Skip to content

Commit f17e879

Browse files
aosinghaosingh
committed
1.3.0 updates
- Streamlined Incremental materialization as per dbt core - Consolidated time based macros - Added more functional cases
1 parent 43ea08c commit f17e879

File tree

13 files changed

+325
-31
lines changed

13 files changed

+325
-31
lines changed

‎dbt/adapters/oracle/column.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ class OracleColumn(Column):
2929
TYPE_LABELS: ClassVar[Dict[str, str]] = {
3030
"STRING": "VARCHAR2(4000)",
3131
"TIMESTAMP": "TIMESTAMP",
32-
"FLOAT": "FLOAT",
32+
"FLOAT": "NUMBER",
3333
"INTEGER": "INTEGER",
3434
}
3535

‎dbt/adapters/oracle/impl.py‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,9 @@
3939

4040
logger = AdapterLogger("oracle")
4141

42-
42+
# Added 6 random hex letters (56c36b) to table_a and table_b to avoid ORA-32031.
43+
# Some dbt test cases use relation names table_a and table_b
44+
# Oracle error: ORA-32031: illegal reference of a query name in WITH clause
4345
COLUMNS_EQUAL_SQL = '''
4446
with diff_count as (
4547
SELECT
@@ -51,15 +53,15 @@
5153
(SELECT {columns} FROM {relation_b} {except_op}
5254
SELECT {columns} FROM {relation_a})
5355
) a
54-
), table_a as (
56+
), table_a_56c36b as (
5557
SELECT COUNT(*) as num_rows FROM {relation_a}
56-
), table_b as (
58+
), table_b_56c36b as (
5759
SELECT COUNT(*) as num_rows FROM {relation_b}
5860
), row_count_diff as (
5961
select
6062
1 as id,
61-
table_a.num_rows - table_b.num_rows as difference
62-
from table_a, table_b
63+
table_a_56c36b.num_rows - table_b_56c36b.num_rows as difference
64+
from table_a_56c36b, table_b_56c36b
6365
)
6466
select
6567
row_count_diff.difference as row_count_difference,
@@ -314,3 +316,6 @@ def quote_seed_column(
314316
return self.quote(column)
315317
else:
316318
return column
319+
320+
def valid_incremental_strategies(self):
321+
return ["append", "merge"]

‎dbt/include/oracle/macros/adapters.sql‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -346,10 +346,6 @@
346346
{{ return(load_result('list_relations_without_caching').table) }}
347347
{% endmacro %}
348348

349-
{% macro oracle__current_timestamp() -%}
350-
CURRENT_TIMESTAMP
351-
{%- endmacro %}
352-
353349
{% macro oracle__make_temp_relation(base_relation, suffix) %}
354350
{% set dt = modules.datetime.datetime.now() %}
355351
{% set dtstring = dt.strftime("%H%M%S") %}

‎dbt/include/oracle/macros/materializations/incremental/helpers.sql‎

Lines changed: 57 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -73,16 +73,63 @@
7373
{% endmacro %}
7474
7575
76-
{% macro oracle_incremental_upsert(tmp_relation, target_relation, dest_columns, unique_key=none, statement_name="main") %}
76+
{% macro oracle__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}
77+
{%- set default_cols = dest_columns | map(attribute='name') | list -%}
78+
79+
{%- if merge_update_columns and merge_exclude_columns -%}
80+
{{ exceptions.raise_compiler_error(
81+
'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'
82+
)}}
83+
{%- elif merge_update_columns -%}
84+
{%- set update_columns = merge_update_columns -%}
85+
{%- elif merge_exclude_columns -%}
86+
{%- set update_columns = [] -%}
87+
{%- for column in dest_columns -%}
88+
{% if column.column | lower not in merge_exclude_columns | map("lower") | list %}
89+
{%- do update_columns.append(column.name) -%}
90+
{% endif %}
91+
{%- endfor -%}
92+
{%- else -%}
93+
{%- set update_columns = default_cols -%}
94+
{%- endif -%}
95+
96+
{%- set quoted_update_columns = [] -%}
97+
{% for col in update_columns %}
98+
{% do quoted_update_columns.append(adapter.check_and_quote_identifier(col, model.columns)) %}
99+
{% endfor %}
100+
{{ return(quoted_update_columns)}}
101+
{% endmacro %}
102+
103+
104+
{% macro oracle__get_incremental_append_sql(arg_dict) %}
105+
{%- set dest_columns = args_dict["dest_columns"] -%}
106+
{%- set temp_relation = args_dict["temp_relation"] -%}
107+
{%- set target_relation = args_dict["target_relation"] -%}
108+
{%- set dest_column_names = dest_columns | map(attribute='name') | list -%}
109+
{%- set dest_cols_csv = get_quoted_column_csv(model, dest_column_names) -%}
110+
INSERT INTO {{ target_relation }} ({{ dest_cols_csv }})
111+
(
112+
SELECT {{ dest_cols_csv }}
113+
FROM {{ temp_relation }}
114+
)
115+
{% endmacro %}
116+
117+
{% macro oracle__get_incremental_merge_sql(args_dict) %}
118+
{%- set dest_columns = args_dict["dest_columns"] -%}
119+
{%- set temp_relation = args_dict["temp_relation"] -%}
120+
{%- set target_relation = args_dict["target_relation"] -%}
121+
{%- set unique_key = args_dict["unique_key"] -%}
77122
{%- set dest_column_names = dest_columns | map(attribute='name') | list -%}
78123
{%- set dest_cols_csv = get_quoted_column_csv(model, dest_column_names) -%}
79-
{%- set update_columns = oracle_check_and_quote_column_names_for_incremental_merge(dest_column_names) -%}
124+
{%- set merge_update_columns = config.get('merge_update_columns') -%}
125+
{%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}
126+
{%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}
80127
{%- if unique_key -%}
81128
{%- set unique_key_result = oracle_check_and_quote_unique_key_for_incremental_merge(unique_key) -%}
82129
{%- set unique_key_list = unique_key_result['unique_key_list'] -%}
83130
{%- set unique_key_merge_predicates = unique_key_result['unique_key_merge_predicates'] -%}
84131
merge into {{ target_relation }} target
85-
using {{ tmp_relation }} temp
132+
using {{ temp_relation }} temp
86133
on ({{ unique_key_merge_predicates | join(' AND ') }})
87134
when matched then
88135
update set
@@ -100,7 +147,12 @@
100147
insert into {{ target_relation }} ({{ dest_cols_csv }})
101148
(
102149
select {{ dest_cols_csv }}
103-
from {{ tmp_relation }}
150+
from {{ temp_relation }}
104151
)
105152
{%- endif -%}
106-
{%- endmacro %}
153+
{% endmacro %}
154+
155+
{% macro oracle__get_incremental_default_sql(arg_dict) %}
156+
{% do return(get_incremental_merge_sql(arg_dict)) %}
157+
{% endmacro %}
158+

‎dbt/include/oracle/macros/materializations/incremental/incremental.sql‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,13 @@
5656
{% if not dest_columns %}
5757
{% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}
5858
{% endif %}
59-
{% set build_sql = oracle_incremental_upsert(tmp_relation, target_relation, dest_columns, unique_key=unique_key) %}
59+
60+
{#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}
61+
{% set incremental_strategy = config.get('incremental_strategy') or 'default' %}
62+
{% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}
63+
{% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': tmp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns }) %}
64+
{% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}
65+
6066
{% endif %}
6167

6268
{% call statement("main") %}

‎dbt/include/oracle/macros/materializations/snapshot/snapshot.sql‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -311,7 +311,3 @@
311311
{%- endfor -%})
312312
{%- endmacro %}
313313

314-
{% macro oracle__snapshot_string_as_time(timestamp) -%}
315-
{%- set result = "TO_TIMESTAMP('"~ timestamp ~ "','yyyy/mm/dd hh24:mi:ss.FF')" -%}
316-
{{ return(result) }}
317-
{%- endmacro %}

‎dbt/include/oracle/macros/utils/data_types.sql‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
{#
2+
Copyright (c) 2022, Oracle and/or its affiliates.
3+
Copyright (c) 2020, Vitor Avancini
4+
5+
Licensed under the Apache License, Version 2.0 (the "License");
6+
you may not use this file except in compliance with the License.
7+
You may obtain a copy of the License at
8+
9+
https://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
#}
17+
18+
{# bigint = int64 #}
19+
{%- macro oracle__type_bigint() -%}
20+
NUMBER(19)
21+
{%- endmacro -%}
22+
23+
{# int = int32 #}
24+
{%- macro oracle__type_int() -%}
25+
NUMBER(10, 0)
26+
{%- endmacro -%}
27+
28+
29+
{% macro oracle__type_boolean() -%}
30+
NUMBER(1)
31+
{%- endmacro %}

‎dbt/include/oracle/macros/utils/timestamps.sql‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{#
2+
Copyright (c) 2022, Oracle and/or its affiliates.
3+
Copyright (c) 2020, Vitor Avancini
4+
5+
Licensed under the Apache License, Version 2.0 (the "License");
6+
you may not use this file except in compliance with the License.
7+
You may obtain a copy of the License at
8+
9+
https://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
#}
17+
18+
{% macro oracle__current_timestamp() -%}
19+
SYS_EXTRACT_UTC(current_timestamp)
20+
{%- endmacro %}
21+
22+
{% macro oracle__snapshot_string_as_time(timestamp) -%}
23+
{%- set result = "TO_TIMESTAMP('"~ timestamp ~ "','yyyy/mm/dd hh24:mi:ss.FF')" -%}
24+
{{ return(result) }}
25+
{%- endmacro %}

‎tests/functional/adapter/test_basic.py‎

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
from dbt.tests.adapter.basic.test_singular_tests_ephemeral import BaseSingularTestsEphemeral
3131
from dbt.tests.adapter.basic.test_empty import BaseEmpty
3232
from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral
33-
from dbt.tests.adapter.basic.test_incremental import BaseIncremental
33+
from dbt.tests.adapter.basic.test_incremental import BaseIncremental, BaseIncrementalNotSchemaChange
3434
from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests
3535
from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols
3636
from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp
@@ -67,6 +67,18 @@
6767
SELECT 2 as id FROM DUAL
6868
"""
6969

70+
incremental_not_schema_change_sql = """
71+
{{ config(materialized="incremental", unique_key="user_id_current_time",on_schema_change="sync_all_columns") }}
72+
select
73+
1 || '-' || current_timestamp as user_id_current_time,
74+
{% if is_incremental() %}
75+
'thisis18characters' as platform
76+
{% else %}
77+
'okthisis20characters' as platform
78+
{% endif %}
79+
FROM DUAL
80+
"""
81+
7082
models__model_sql = """
7183
7284
{% set upstream = ref('upstream') %}
@@ -91,11 +103,11 @@
91103
"""
92104

93105

94-
class TestSimpleMaterializationsMyAdapter(BaseSimpleMaterializations):
106+
class TestSimpleMaterializationsOracle(BaseSimpleMaterializations):
95107
pass
96108

97109

98-
class TestSingularTestsMyAdapter(BaseSingularTests):
110+
class TestSingularTestsOracle(BaseSingularTests):
99111

100112
@pytest.fixture(scope="class")
101113
def tests(self):
@@ -105,7 +117,7 @@ def tests(self):
105117
}
106118

107119

108-
class TestSingularTestsEphemeralMyAdapter(BaseSingularTestsEphemeral):
120+
class TestSingularTestsEphemeralOracle(BaseSingularTestsEphemeral):
109121

110122
@pytest.fixture(scope="class")
111123
def models(self):
@@ -117,31 +129,31 @@ def models(self):
117129
}
118130

119131

120-
class TestEmptyMyAdapter(BaseEmpty):
132+
class TestEmptyOracle(BaseEmpty):
121133
pass
122134

123135

124-
class TestEphemeralMyAdapter(BaseEphemeral):
136+
class TestEphemeralOracle(BaseEphemeral):
125137
pass
126138

127139

128-
class TestIncrementalMyAdapter(BaseIncremental):
140+
class TestIncrementalOracle(BaseIncremental):
129141
pass
130142

131143

132-
class TestGenericTestsMyAdapter(BaseGenericTests):
144+
class TestGenericTestsOracle(BaseGenericTests):
133145
pass
134146

135147

136-
class TestSnapshotCheckColsMyAdapter(BaseSnapshotCheckCols):
148+
class TestSnapshotCheckColsOracle(BaseSnapshotCheckCols):
137149
pass
138150

139151

140-
class TestSnapshotTimestampMyAdapter(BaseSnapshotTimestamp):
152+
class TestSnapshotTimestampOracle(BaseSnapshotTimestamp):
141153
pass
142154

143155

144-
class TestBaseAdapterMethod(BaseAdapterMethod):
156+
class TestBaseAdapterMethodOracle(BaseAdapterMethod):
145157

146158
@pytest.fixture(scope="class")
147159
def models(self):
@@ -151,3 +163,9 @@ def models(self):
151163
"model.sql": models__model_sql,
152164
}
153165

166+
167+
class TestIncrementalNotSchemaChangeOracle(BaseIncrementalNotSchemaChange):
168+
169+
@pytest.fixture(scope="class")
170+
def models(self):
171+
return {"incremental_not_schema_change.sql": incremental_not_schema_change_sql}

‎tests/functional/adapter/test_concurrency.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
"""
2+
Copyright (c) 2022, Oracle and/or its affiliates.
3+
Copyright (c) 2020, Vitor Avancini
4+
5+
Licensed under the Apache License, Version 2.0 (the "License");
6+
you may not use this file except in compliance with the License.
7+
You may obtain a copy of the License at
8+
9+
https://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
"""
17+
import pytest
18+
19+
from dbt.tests.adapter.concurrency.test_concurrency import TestConcurenncy
20+
21+
22+
class TestConcurrencyOracle(TestConcurenncy):
23+
pass

0 commit comments

Comments
 (0)