Skip to content

Commit 745f7c5

Browse files
mrodrigmrodrig
committed
Merge pull request #31 from mrodrig/allow-different-schemas
Allow different schemas
2 parents 0f87aba + 5f508dd commit 745f7c5

File tree

9 files changed

+168
-23
lines changed

9 files changed

+168
-23
lines changed

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ var converter = require('json-2-csv');
3434
* `FIELD` - String - Field Delimiter. Default: `','`
3535
* `ARRAY` - String - Array Value Delimiter. Default: `';'`
3636
* `WRAP` - String - Wrap values in the delimiter of choice (e.g. wrap values in quotes). Default: `''`
37+
* `CHECK_SCHEMA_DIFFERENCES` - Boolean - Should we require all documents to have the same schema? Default: `true`
3738
* `PREPEND_HEADER` - Boolean - Should the auto-generated header be prepended as the first line in the CSV? Default: `true`
3839
* `EOL` - String - End of Line Delimiter. Default: `'\n'`
3940
* `KEYS` - Array - Specify the keys (as strings) that should be converted. Default: `null`
@@ -164,6 +165,7 @@ Lines : 97.99% ( 146/149 )
164165
- Allows for custom field delimiters, end of line delimiters, etc.
165166
- Promisifiable via bluebird's .promisify(<function>) and .promisifyAll(<object>) (as of 1.1.1)
166167
- Wrapped value support for json2csv and csv2json (as of 1.3.0)
168+
- Support for multiple different schemas (as of 1.4.0)
167169

168170
## F.A.Q.
169171

‎bower.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "json-2-csv",
3-
"version": "1.3.1",
3+
"version": "1.4.0",
44
"homepage": "https://github.com/mrodrig/json-2-csv",
55
"moduleType": [
66
"node"

‎lib/constants.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
"EOL" : "\n",
2929
"PREPEND_HEADER" : true,
3030
"PARSE_CSV_NUMBERS" : false,
31-
"KEYS" : null
31+
"KEYS" : null,
32+
"CHECK_SCHEMA_DIFFERENCES": true
3233
}
3334
}

‎lib/json-2-csv.js‎

Lines changed: 34 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -23,27 +23,41 @@ var generateHeading = function(data) {
2323
}
2424
});
2525

26-
// Check for a consistent schema that does not require the same order:
27-
// if we only have one document - then there is no possibility of multiple schemas
28-
if (keys && keys.length <= 1) {
29-
return Promise.resolve(_.flatten(keys) || []);
26+
// If the user wants to check for the same schema:
27+
if (options.CHECK_SCHEMA_DIFFERENCES) {
28+
// Check for a consistent schema that does not require the same order:
29+
// if we only have one document - then there is no possibility of multiple schemas
30+
if (keys && keys.length <= 1) {
31+
return Promise.resolve(_.flatten(keys) || []);
32+
}
33+
// else - multiple documents - ensure only one schema (regardless of field ordering)
34+
var firstDocSchema = _.flatten(keys[0]),
35+
schemaDifferences = 0;
36+
37+
_.each(keys, function (keyList) {
38+
// If there is a difference between the schemas, increment the counter of schema inconsistencies
39+
var diff = _.difference(firstDocSchema, _.flatten(keyList));
40+
if (!_.isEqual(diff, [])) {
41+
schemaDifferences++;
42+
}
43+
});
44+
45+
// If there are schema inconsistencies, throw a schema not the same error
46+
if (schemaDifferences) {
47+
return Promise.reject(new Error(constants.Errors.json2csv.notSameSchema));
48+
}
49+
50+
return Promise.resolve(_.flatten(keys[0]));
3051
}
31-
// else - multiple documents - ensure only one schema (regardless of field ordering)
32-
var firstDocSchema = _.flatten(keys[0]),
33-
schemaDifferences = 0;
3452

53+
var uniqueKeys = [];
54+
55+
// Otherwise, we do not care if the schemas are different, so we should merge them via union:
3556
_.each(keys, function (keyList) {
36-
// If there is a difference between the schemas, increment the counter of schema inconsistencies
37-
var diff = _.difference(firstDocSchema, _.flatten(keyList));
38-
if (!_.isEqual(diff, [])) {
39-
schemaDifferences++;
40-
}
57+
uniqueKeys = _.union(uniqueKeys, _.flatten(keyList));
4158
});
4259

43-
// If there are schema inconsistencies, throw a schema not the same error
44-
if (schemaDifferences) { return Promise.reject(new Error(constants.Errors.json2csv.notSameSchema)); }
45-
46-
return Promise.resolve(_.flatten(keys[0]));
60+
return Promise.resolve(uniqueKeys);
4761
};
4862

4963
/**
@@ -79,8 +93,11 @@ var generateDocumentHeading = function(heading, data) {
7993
var convertData = function (data, keys) {
8094
// Reduce each key in the data to its CSV value
8195
return _.reduce(keys, function (output, key) {
96+
// Retrieve the appropriate field data
97+
var fieldData = path.evaluatePath(data, key);
98+
if (_.isUndefined(fieldData)) { fieldData = 'null'; }
8299
// Add the CSV representation of the data at the key in the document to the output array
83-
return output.concat(convertField(path.evaluatePath(data, key)));
100+
return output.concat(convertField(fieldData));
84101
}, []);
85102
};
86103

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"author": "mrodrig",
33
"name": "json-2-csv",
44
"description": "A JSON to CSV and CSV to JSON converter that natively supports sub-documents and auto-generates the CSV heading.",
5-
"version": "1.3.1",
5+
"version": "1.4.0",
66
"repository": {
77
"type": "git",
88
"url": "http://github.com/mrodrig/json-2-csv.git"

‎test/CSV/quoted/differentSchemas.csv‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
"carModel","price","color","mileage"
2+
"Audi","10000","blue","7200"
3+
"BMW","15000","red","null"
4+
"Mercedes","20000","yellow","null"
5+
"Porsche","30000","green","null"

‎test/CSV/unQuoted/differentSchemas.csv‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
carModel,price,color,mileage
2+
Audi,10000,blue,7200
3+
BMW,15000,red,null
4+
Mercedes,20000,yellow,null
5+
Porsche,30000,green,null

‎test/testCsvFilesList.json‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@
99
{"key": "nestedQuotes", "file": "test/CSV/unQuoted/nestedQuotes.csv"},
1010
{"key": "noData", "file": "test/CSV/unQuoted/noData.csv"},
1111
{"key": "regularJson", "file": "test/CSV/unQuoted/regularJson.csv"},
12-
{"key": "singleDoc", "file": "test/CSV/unQuoted/singleDoc.csv"}
12+
{"key": "singleDoc", "file": "test/CSV/unQuoted/singleDoc.csv"},
13+
{"key": "differentSchemas", "file": "test/CSV/unQuoted/differentSchemas.csv"}
1314
]
1415
},
1516
{
@@ -24,7 +25,8 @@
2425
{"key": "nestedQuotes", "file": "test/CSV/quoted/nestedQuotes.csv"},
2526
{"key": "noData", "file": "test/CSV/quoted/noData.csv"},
2627
{"key": "regularJson", "file": "test/CSV/quoted/regularJson.csv"},
27-
{"key": "singleDoc", "file": "test/CSV/quoted/singleDoc.csv"}
28+
{"key": "singleDoc", "file": "test/CSV/quoted/singleDoc.csv"},
29+
{"key": "differentSchemas", "file": "test/CSV/quoted/differentSchemas.csv"}
2830
]
2931
}
3032
]

‎test/testJson2Csv.js‎

Lines changed: 114 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,19 @@ var json2csvTests = function () {
9696
});
9797
});
9898

99+
it('should convert two documents with different schemas properly', function (done) {
100+
opts = JSON.parse(JSON.stringify(options));
101+
opts.CHECK_SCHEMA_DIFFERENCES = false;
102+
103+
converter.json2csv(jsonTestData.differentSchemas, function (err, csv) {
104+
if (err) { throw err; }
105+
true.should.equal(_.isEqual(err, null));
106+
csv.should.equal(csvTestData.unQuoted.differentSchemas);
107+
csv.split(options.EOL).length.should.equal(6);
108+
done();
109+
}, opts);
110+
});
111+
99112
it('should throw an error if the documents do not have the same schema', function (done) {
100113
converter.json2csv(jsonTestData.differentSchemas, function (err, csv) {
101114
err.message.should.equal(constants.Errors.json2csv.notSameSchema);
@@ -280,6 +293,19 @@ var json2csvTests = function () {
280293
}, opts);
281294
});
282295

296+
it('should convert two documents with different schemas properly', function (done) {
297+
opts = JSON.parse(JSON.stringify(options));
298+
opts.CHECK_SCHEMA_DIFFERENCES = false;
299+
300+
converter.json2csv(jsonTestData.differentSchemas, function (err, csv) {
301+
if (err) { throw err; }
302+
true.should.equal(_.isEqual(err, null));
303+
csv.should.equal(csvTestData.unQuoted.differentSchemas);
304+
csv.split(options.EOL).length.should.equal(6);
305+
done();
306+
}, opts);
307+
});
308+
283309
it('should throw an error if the documents do not have the same schema', function (done) {
284310
converter.json2csv(jsonTestData.differentSchemas, function (err, csv) {
285311
err.message.should.equal(constants.Errors.json2csv.notSameSchema);
@@ -444,7 +470,7 @@ var json2csvTests = function () {
444470
});
445471

446472
it('should repress the heading', function (done) {
447-
opts = JSON.parse(JSON.stringify(options));
473+
var opts = JSON.parse(JSON.stringify(options));
448474
opts.PREPEND_HEADER = false;
449475

450476
converter.json2csv(jsonTestData.sameSchemaDifferentOrdering, function (err, csv) {
@@ -456,6 +482,20 @@ var json2csvTests = function () {
456482
}, opts);
457483
});
458484

485+
it('should convert two documents with different schemas properly', function (done) {
486+
var opts = JSON.parse(JSON.stringify(options));
487+
opts.PREPEND_HEADER = true;
488+
opts.CHECK_SCHEMA_DIFFERENCES = false;
489+
490+
converter.json2csv(jsonTestData.differentSchemas, function (err, csv) {
491+
if (err) { throw err; }
492+
true.should.equal(_.isEqual(err, null));
493+
csv.should.equal(csvTestData.unQuoted.differentSchemas.replace(/,/g, options.DELIMITER.FIELD).split(options.EOL).join(options.EOL));
494+
csv.split(options.EOL).length.should.equal(6);
495+
done();
496+
}, opts);
497+
});
498+
459499
it('should throw an error if the documents do not have the same schema', function (done) {
460500
converter.json2csv(jsonTestData.differentSchemas, function (err, csv) {
461501
err.message.should.equal(constants.Errors.json2csv.notSameSchema);
@@ -642,6 +682,19 @@ var json2csvTests = function () {
642682
}, opts);
643683
});
644684

685+
it('should convert two documents with different schemas properly', function (done) {
686+
var opts = JSON.parse(JSON.stringify(options));
687+
opts.CHECK_SCHEMA_DIFFERENCES = false;
688+
689+
converter.json2csv(jsonTestData.differentSchemas, function (err, csv) {
690+
if (err) { throw err; }
691+
true.should.equal(_.isEqual(err, null));
692+
csv.should.equal(csvTestData.quoted.differentSchemas);
693+
csv.split(options.EOL).length.should.equal(6);
694+
done();
695+
}, opts);
696+
});
697+
645698
it('should throw an error if the documents do not have the same schema', function (done) {
646699
converter.json2csv(jsonTestData.differentSchemas, function (err, csv) {
647700
err.message.should.equal(constants.Errors.json2csv.notSameSchema);
@@ -846,6 +899,21 @@ var json2csvTests = function () {
846899
});
847900
});
848901

902+
it('should convert two documents with different schemas properly', function (done) {
903+
var opts = JSON.parse(JSON.stringify(options));
904+
opts.CHECK_SCHEMA_DIFFERENCES = false;
905+
906+
converter.json2csvAsync(jsonTestData.differentSchemas, opts)
907+
.then(function (csv) {
908+
csv.should.equal(csvTestData.unQuoted.differentSchemas);
909+
csv.split(options.EOL).length.should.equal(6);
910+
done();
911+
})
912+
.catch(function (err) {
913+
throw err;
914+
});
915+
});
916+
849917
it('should throw an error if the documents do not have the same schema', function (done) {
850918
converter.json2csvAsync(jsonTestData.differentSchemas)
851919
.then(function (csv) {
@@ -1017,6 +1085,21 @@ var json2csvTests = function () {
10171085
});
10181086
});
10191087

1088+
it('should convert two documents with different schemas properly', function (done) {
1089+
var opts = JSON.parse(JSON.stringify(options));
1090+
opts.CHECK_SCHEMA_DIFFERENCES = false;
1091+
1092+
converter.json2csvAsync(jsonTestData.differentSchemas, opts)
1093+
.then(function (csv) {
1094+
csv.should.equal(csvTestData.unQuoted.differentSchemas);
1095+
csv.split(options.EOL).length.should.equal(6);
1096+
done();
1097+
})
1098+
.catch(function (err) {
1099+
throw err;
1100+
});
1101+
});
1102+
10201103
it('should throw an error if the documents do not have the same schema', function (done) {
10211104
converter.json2csvAsync(jsonTestData.differentSchemas, options)
10221105
.then(function (csv) {
@@ -1189,6 +1272,21 @@ var json2csvTests = function () {
11891272
});
11901273
});
11911274

1275+
it('should convert two documents with different schemas properly', function (done) {
1276+
var opts = JSON.parse(JSON.stringify(options));
1277+
opts.CHECK_SCHEMA_DIFFERENCES = false;
1278+
1279+
converter.json2csvAsync(jsonTestData.differentSchemas, opts)
1280+
.then(function (csv) {
1281+
csv.should.equal(csvTestData.unQuoted.differentSchemas.replace(/,/g, options.DELIMITER.FIELD).split(options.EOL).join(options.EOL));
1282+
csv.split(options.EOL).length.should.equal(6);
1283+
done();
1284+
})
1285+
.catch(function (err) {
1286+
throw err;
1287+
});
1288+
});
1289+
11921290
it('should throw an error if the documents do not have the same schema', function (done) {
11931291
converter.json2csvAsync(jsonTestData.differentSchemas, options)
11941292
.then(function (csv) {
@@ -1373,6 +1471,21 @@ var json2csvTests = function () {
13731471
});
13741472
});
13751473

1474+
it('should convert two documents with different schemas properly', function (done) {
1475+
var opts = JSON.parse(JSON.stringify(options));
1476+
opts.CHECK_SCHEMA_DIFFERENCES = false;
1477+
1478+
converter.json2csvAsync(jsonTestData.differentSchemas, opts)
1479+
.then(function (csv) {
1480+
csv.should.equal(csvTestData.quoted.differentSchemas);
1481+
csv.split(options.EOL).length.should.equal(6);
1482+
done();
1483+
})
1484+
.catch(function (err) {
1485+
throw err;
1486+
});
1487+
});
1488+
13761489
it('should throw an error if the documents do not have the same schema', function (done) {
13771490
converter.json2csvAsync(jsonTestData.differentSchemas, options)
13781491
.then(function (csv) {

0 commit comments

Comments
 (0)