Skip to content
This repository was archived by the owner on Mar 13, 2020. It is now read-only.

Commit 5da36e0

Browse files
DavidAmesPupDavidAmesPup
authored
Merge pull request #5 from PageUpPeopleOrg/AddSupportForBoolean
Added bool support
2 parents 8c97e80 + 2fce20a commit 5da36e0

File tree

10 files changed

+56
-34
lines changed

10 files changed

+56
-34
lines changed

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ The destination.type value controls both the data reader type and the destinatio
4646
| numeric | float | sqlalchemy.Numeric | numeric | Stores whole and decimal numbers |
4747
| guid | str | sqlalchemy.dialects.postgresql.UUID | uuid | |
4848
| bigint | int | sqlalchemy.BigInteger | BigInt | Relies on 64big python. Limited to largest number of ~2147483647121212|
49+
| boolean | bool | sqlalchemy.Boolean | Boolean | |
4950

5051

5152
These are implemented in Column_Type_Resolver.py

‎integration_tests/csv_source/assertions/column_test_full_refresh_assertions.sql‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,20 @@ SET client_encoding TO 'UTF8';
33
DROP TABLE IF EXISTS results;
44

55
CREATE TEMPORARY TABLE results AS
6-
WITH expected(id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1, guid_column_1,big_int_column_1) AS (
7-
SELECT 1, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'A Basic String', '57bc8093-fe4c-477a-bbd7-fb5c02055a7e'::UUID,2147483647121212
6+
WITH expected(id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1, guid_column_1,big_int_column_1, bool_column_1) AS (
7+
SELECT 1, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'A Basic String', '57bc8093-fe4c-477a-bbd7-fb5c02055a7e'::UUID,2147483647121212, True
88
UNION ALL
9-
SELECT 2, NULL, NULL, NULL, NULL, NULL, NULL, NULL
9+
SELECT 2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL
1010
UNION ALL
11-
SELECT 3, 333.0, '2001-01-01', 33.333, NULL, 'This Text Has a Quote Before "Dave', NULL, NULL
11+
SELECT 3, 333.0, '2001-01-01', 33.333, NULL, 'This Text Has a Quote Before "Dave', NULL, NULL, True
1212
UNION ALL
13-
SELECT 4, NULL, NULL, NULL, NULL, 'ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ','aabc8093-fe4c-477a-bbd7-fb5c02055a7e', NULL
13+
SELECT 4, NULL, NULL, NULL, NULL, 'ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ','aabc8093-fe4c-477a-bbd7-fb5c02055a7e', NULL, False
1414
UNION ALL
15-
SELECT 5, NULL, NULL, NULL, NULL, 'This row will be updated in the incremental review test', NULL, NULL
15+
SELECT 5, NULL, NULL, NULL, NULL, 'This row will be updated in the incremental review test', NULL, NULL, NULL
1616
),
1717

1818
actual AS (
19-
SELECT id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1,guid_column_1,big_int_column_1
19+
SELECT id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1,guid_column_1,big_int_column_1,bool_column_1
2020
FROM rdl_integration_tests.load_source_data
2121
)
2222

‎integration_tests/csv_source/assertions/column_test_incremental_refresh_assertions.sql‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,24 +2,24 @@ SET client_encoding TO 'UTF8';
22
DROP TABLE IF EXISTS results;
33

44
CREATE TEMPORARY TABLE results AS
5-
WITH expected(id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1, guid_column_1,big_int_column_1) AS (
6-
SELECT 1, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'A Basic String', '57bc8093-fe4c-477a-bbd7-fb5c02055a7e'::UUID,2147483647121212
5+
WITH expected(id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1, guid_column_1,big_int_column_1,bool_column_1) AS (
6+
SELECT 1, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'A Basic String', '57bc8093-fe4c-477a-bbd7-fb5c02055a7e'::UUID,2147483647121212, True
77
UNION ALL
8-
SELECT 2, NULL, NULL, NULL, NULL, NULL, NULL, NULL
8+
SELECT 2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL
99
UNION ALL
10-
SELECT 3, 333.0, '2001-01-01', 33.333, NULL, 'This Text Has a Quote Before "Dave', NULL, NULL
10+
SELECT 3, 333.0, '2001-01-01', 33.333, NULL, 'This Text Has a Quote Before "Dave', NULL, NULL, True
1111
UNION ALL
12-
SELECT 4, NULL, NULL, NULL, NULL, 'ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ', 'aabc8093-fe4c-477a-bbd7-fb5c02055a7e', NULL
12+
SELECT 4, NULL, NULL, NULL, NULL, 'ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ', 'aabc8093-fe4c-477a-bbd7-fb5c02055a7e', NULL, False
1313
UNION ALL
14-
SELECT 5, NULL, NULL, NULL, NULL, 'This row WAS updated in the incremental review test', NULL, NULL
14+
SELECT 5, NULL, NULL, NULL, NULL, 'This row WAS updated in the incremental review test', NULL, NULL, True
1515
UNION ALL
16-
SELECT 6, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'A Basic String', '57bc8093-fe4c-477a-bbd7-fb5c02055a7e', NULL
16+
SELECT 6, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'A Basic String', '57bc8093-fe4c-477a-bbd7-fb5c02055a7e', NULL, False
1717
UNION ALL
18-
SELECT 7, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'Another Basic String', NULL, NULL
18+
SELECT 7, 111.0, '1976-12-01'::DATE, 12.1212, '1976-12-01 01:00:00.000000'::TIMESTAMP, 'Another Basic String', NULL, NULL, True
1919
),
2020

2121
actual AS (
22-
SELECT id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1, guid_column_1,big_int_column_1
22+
SELECT id, int_column_1, date_column_1, decimal_column_1, date_time_column_1, string_column_1, guid_column_1,big_int_column_1,bool_column_1
2323
FROM rdl_integration_tests.load_source_data
2424
)
2525

‎integration_tests/csv_source/config/ColumnTest.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,10 +77,19 @@
7777
"type": "bigint",
7878
"nullable": true
7979
}
80+
},
81+
{
82+
"source_name": "BoolColumn1",
83+
"destination": {
84+
"name": "bool_column_1",
85+
"type": "boolean",
86+
"nullable": true
87+
}
8088
}
8189

8290

8391

8492

93+
8594
]
8695
}

‎integration_tests/csv_source/full_refresh_data/ColumnTest.csv‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
id,StringColumn1,IntColumn1,DecimalColumn1,DateColumn1,DateTimeColumn1,GuidColumn1,BigIntColumn1
2-
1,"A Basic String",111,12.1212,01-Dec-1976,01-dec-1976 1:00 am,57BC8093-FE4C-477A-BBD7-FB5C02055A7E,2147483647121212
3-
2,,,,,,,
4-
3,"This Text Has a Quote Before ""Dave", 333,33.333, 01-01-01,,,
5-
4,"ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ",,,,,AABC8093-FE4C-477A-BBD7-FB5C02055A7E,
1+
id,StringColumn1,IntColumn1,DecimalColumn1,DateColumn1,DateTimeColumn1,GuidColumn1,BigIntColumn1,BoolColumn1
2+
1,"A Basic String",111,12.1212,01-Dec-1976,01-dec-1976 1:00 am,57BC8093-FE4C-477A-BBD7-FB5C02055A7E,2147483647121212,1
3+
2,,,,,,,,
4+
3,"This Text Has a Quote Before ""Dave", 333,33.333, 01-01-01,,,,1
5+
4,"ം ഃ അ ആ ഇ ഈ ഉ ഊ ഋ ഌ എ ഏ",,,,,AABC8093-FE4C-477A-BBD7-FB5C02055A7E,,0
66
5,"This row will be updated in the incremental review test",,

‎integration_tests/csv_source/incremental_refresh_data/ColumnTest.csv‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
id,StringColumn1,IntColumn1,DecimalColumn1,DateColumn1,DateTimeColumn1,GuidColumn1,BigIntColumn1
2-
5,"This row WAS updated in the incremental review test",,,,,,
3-
6,"A Basic String",111,12.1212,01-Dec-1976,01-dec-1976 1:00 am,57BC8093-FE4C-477A-BBD7-FB5C02055A7E,
4-
7,"Another Basic String",111,12.1212,01-Dec-1976,01-dec-1976 1:00 am,,
1+
id,StringColumn1,IntColumn1,DecimalColumn1,DateColumn1,DateTimeColumn1,GuidColumn1,BigIntColumn1,BoolColumn1
2+
5,"This row WAS updated in the incremental review test",,,,,,,1
3+
6,"A Basic String",111,12.1212,01-Dec-1976,01-dec-1976 1:00 am,57BC8093-FE4C-477A-BBD7-FB5C02055A7E,,0
4+
7,"Another Basic String",111,12.1212,01-Dec-1976,01-dec-1976 1:00 am,,,1

‎integration_tests/mssql_source/config/LargeTableTest.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,16 @@
6868
"type": "guid",
6969
"nullable": true
7070
}
71+
},
72+
{
73+
"source_name": "BoolColumn",
74+
"destination": {
75+
"name": "bool_column_1",
76+
"type": "boolean",
77+
"nullable": true
78+
}
7179
}
7280

81+
7382
]
7483
}

‎integration_tests/mssql_source/dockerfile‎

Lines changed: 0 additions & 3 deletions
This file was deleted.

‎integration_tests/mssql_source/source_database_setup/create_large_table.sql‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@ IF object_id('LargeTable') IS NULL
66
DateColumn2 DATE,
77
IntColumn1 INT,
88
StringColumn2 NVARCHAR(100),
9-
GuidColumn UNIQUEIDENTIFIER)
9+
GuidColumn UNIQUEIDENTIFIER,
10+
BoolColumn BIT)
1011
ELSE
1112
TRUNCATE TABLE LargeTable
1213

@@ -29,15 +30,18 @@ INSERT LargeTable
2930
DateColumn2,
3031
IntColumn1,
3132
StringColumn2,
32-
GuidColumn
33+
GuidColumn,
34+
BoolColumn
3335
)
3436
SELECT n,
3537
CASE WHEN n % 3 = 0 THEN NULL ELSE 'Row Number ' + CAST(n as varchar) END,
3638
CASE WHEN n % 5 = 0 THEN NULL ELSE DateAdd(hour, -n, '2000-01-1') END,
3739
CASE WHEN n % 7 = 0 THEN NULL ELSE DateAdd(hour, n, '2000-01-1') END,
3840
CASE WHEN n % 9 = 0 THEN NULL ELSE n * 1000 END,
3941
CASE WHEN n % 11 = 0 THEN NULL ELSE N'काचं शक्नोम्यत्तुम् । नोपहिनस्ति माम् ॥' END,
40-
CASE WHEN n % 13 = 0 THEN NULL ELSE newid() END
42+
CASE WHEN n % 13 = 0 THEN NULL ELSE newid() END,
43+
CASE WHEN n % 3 = 0 THEN NULL ELSE 1 END
44+
4145
FROM Numbers
4246

4347

‎modules/ColumnTypeResolver.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import citext
2-
from sqlalchemy import DateTime, Numeric, Integer, BigInteger
2+
from sqlalchemy import DateTime, Numeric, Integer, BigInteger, Boolean
33
from sqlalchemy.dialects.postgresql import JSONB
44
from sqlalchemy.dialects.postgresql import UUID
55

@@ -10,15 +10,17 @@ class ColumnTypeResolver(object):
1010
'json': str,
1111
'numeric': float,
1212
'guid': str,
13-
'bigint': int}
13+
'bigint': int,
14+
'boolean': bool}
1415

1516
POSTGRES_TYPE_MAP = {'string': citext.CIText,
1617
'datetime': DateTime,
1718
'json': JSONB,
1819
'numeric': Numeric,
1920
'guid': UUID,
2021
'int': Integer,
21-
'bigint': BigInteger}
22+
'bigint': BigInteger,
23+
'boolean': Boolean}
2224

2325
def resolve_postgres_type(self, column):
2426
return self.POSTGRES_TYPE_MAP[column['type']]

0 commit comments

Comments
 (0)