Merge pull request #27 from oracle/feature/dbt-core_v1.1

Feature/dbt core v1.1

Author: aosingh

.github/scripts/create_new_user.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -Exeuo pipefail
+
+# Parameters
+DB_USER="${1}"
+DB_PASSWORD="${2}"
+TARGET_PDB="${3:-XEPDB1}"
+
+# Prepare container switch statement to create user in PDB.
+ALTER_SESSION_CMD="ALTER SESSION SET CONTAINER=${TARGET_PDB};"
+
+# 11g XE does not support PDBs, set container switch statement to empty string.
+ORACLE_VERSION=$(sqlplus -version | grep "Release" | awk '{ print $3 }')
+if [[ "${ORACLE_VERSION}" = "11.2"* ]]; then
+   ALTER_SESSION_CMD="";
+fi;
+
+# Create new user in target PDB
+sqlplus -s / as sysdba << EOF
+   -- Exit on any errors
+   WHENEVER SQLERROR EXIT SQL.SQLCODE
+   ${ALTER_SESSION_CMD}
+   CREATE USER ${DB_USER} IDENTIFIED BY "${DB_PASSWORD}" QUOTA UNLIMITED ON USERS;
+   GRANT ALL PRIVILEGES TO ${DB_USER};
+   exit;
+EOF

.github/workflows/oracle-xe-adapter-tests.yml

@@ -1,20 +1,21 @@
-name: dbt-adapter-tests
+name: dbt-tests-adapter
 on: push
 
 jobs:
-  oracle-21_3_0-xe-test-suite:
+  oracle_xe_21_3:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: true
       matrix:
         os: [ ubuntu-latest ]
-        python-version: [ 3.7, 3.8, 3.9 ]
+        python-version: [ '3.7', '3.8', '3.9', '3.10' ]
 
     services:
       oracle_db_xe:
         image: container-registry.oracle.com/database/express:21.3.0-xe
         env:
           ORACLE_PWD: ${{ secrets.DBT_ORACLE_PASSWORD }}
+        options: --name oracle_db_xe
         ports:
           - 1521:1521
 
@@ -32,81 +33,34 @@ jobs:
           chmod +x ${{ github.workspace }}/.github/scripts/install_oracle_instantclient.sh
           ${{ github.workspace }}/.github/scripts/install_oracle_instantclient.sh
 
-      - name: Install dbt-oracle with core dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install pytest pytest-dbt-adapter==0.6.0
-          pip install -r requirements.txt
-          pip install -e .
-
-      - name: Run adapter tests
+      - name: Copy Create User script
         run: |
-          pytest tests/oracle.dbtspec tests/test_config.py
-        env:
-          DBT_ORACLE_USER: ${{ secrets.DBT_ORACLE_USER }}
-          DBT_ORACLE_HOST: localhost
-          DBT_ORACLE_PORT: 1521
-          DBT_ORACLE_SCHEMA: ${{ secrets.DBT_ORACLE_USER }}
-          DBT_ORACLE_PASSWORD: ${{ secrets.DBT_ORACLE_PASSWORD }}
-          DBT_ORACLE_DATABASE: XEPDB1
-          DBT_ORACLE_SERVICE: XEPDB1
-          DBT_ORACLE_PROTOCOL: tcp
-          LD_LIBRARY_PATH: /opt/oracle/instantclient_21_6
-          TNS_ADMIN: /opt/tns_admin
-
-  oracle-21_3_0-xe-py36-test-suite:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: true
-      matrix:
-        os: [ ubuntu-latest ]
-        python-version: [ 3.6 ]
-
-    services:
-      oracle_db_xe:
-        image: container-registry.oracle.com/database/express:21.3.0-xe
-        env:
-          ORACLE_PWD: ${{ secrets.DBT_ORACLE_PASSWORD }}
-        ports:
-          - 1521:1521
-
-    steps:
-      - name: Check out dbt-oracle repository code
-        uses: actions/checkout@v3
+          chmod +x ${{ github.workspace }}/.github/scripts/create_new_user.sh
+          docker cp ${{ github.workspace }}/.github/scripts/create_new_user.sh oracle_db_xe:/home/oracle/create_new_user.sh
 
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v3
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install Oracle Instantclient
+      - name: Create dbt test user
         run: |
-          chmod +x ${{ github.workspace }}/.github/scripts/install_oracle_instantclient.sh
-          ${{ github.workspace }}/.github/scripts/install_oracle_instantclient.sh
+          docker exec oracle_db_xe /home/oracle/create_new_user.sh dbt_test ${{ secrets.DBT_ORACLE_PASSWORD }}
 
       - name: Install dbt-oracle with core dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install pytest pytest-dbt-adapter==0.4.0
+          pip install pytest dbt-tests-adapter==1.1.1
           pip install -r requirements.txt
           pip install -e .
 
       - name: Run adapter tests
         run: |
-          pytest tests/oracle_py36.dbtspec tests/test_config.py
+          pytest
         env:
-          DBT_ORACLE_USER: ${{ secrets.DBT_ORACLE_USER }}
+          DBT_ORACLE_USER: DBT_TEST
           DBT_ORACLE_HOST: localhost
           DBT_ORACLE_PORT: 1521
-          DBT_ORACLE_SCHEMA: ${{ secrets.DBT_ORACLE_USER }}
+          DBT_ORACLE_SCHEMA: DBT_TEST
           DBT_ORACLE_PASSWORD: ${{ secrets.DBT_ORACLE_PASSWORD }}
           DBT_ORACLE_DATABASE: XEPDB1
           DBT_ORACLE_SERVICE: XEPDB1
           DBT_ORACLE_PROTOCOL: tcp
           LD_LIBRARY_PATH: /opt/oracle/instantclient_21_6
           TNS_ADMIN: /opt/tns_admin
 
-
-
-
-

.gitignore

@@ -141,4 +141,5 @@ doc/build.gitbak
 .venvpy37
 .venv1.1.0
 .venv
-.bldenv
+.bldenv
+

Makefile

@@ -1,5 +1,5 @@
 # Configuration variables
-VERSION=1.0.4
+VERSION=1.1.0
 PROJ_DIR?=$(shell pwd)
 VENV_DIR?=${PROJ_DIR}/.bldenv
 BUILD_DIR=${PROJ_DIR}/build
@@ -26,6 +26,7 @@ adbs_local_env_test: wheel clean_venv
 	${VENV_DIR}/bin/pip install ${DIST_DIR}/dbt_oracle-${VERSION}-py3-none-any.whl
 	cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt --version
 	cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt debug --profiles-dir ./
+	cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt deps --profiles-dir ./
 	cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt run-operation drop_schema --args 'relation: ${DBT_ORACLE_SCHEMA}' --profiles-dir ./
 	cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt deps --profiles-dir ./
 	cd dbt_adbs_test_project && ${VENV_DIR}/bin/dbt seed --profiles-dir ./

dbt/adapters/oracle/__version__.py

@@ -14,4 +14,4 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 """
-version = "1.0.8"
+version = "1.1.1"

dbt/adapters/oracle/connections.py

@@ -14,7 +14,7 @@
   See the License for the specific language governing permissions and
   limitations under the License.
 """
-from typing import List, Optional, Tuple, Any, Dict
+from typing import List, Optional, Tuple, Any, Dict, Union
 from contextlib import contextmanager
 from dataclasses import dataclass, field
 import enum
@@ -24,20 +24,13 @@
 from cx_Oracle import Connection
 
 import dbt.exceptions
-from dbt.helper_types import Port
 
 from dbt.adapters.base import Credentials
 from dbt.adapters.sql import SQLConnectionManager
 from dbt.contracts.connection import AdapterResponse
+from dbt.events import AdapterLogger
 
-# dbt.events was introduced in dbt-core==1.0.0 which only supports Python 3.7
-# For Python 3.6, we fallback to the old logger in dbt.logger.
-try:
-    from dbt.events import AdapterLogger
-    logger = AdapterLogger("oracle")
-except ImportError:
-    from dbt.logger import GLOBAL_LOGGER as logger
-
+logger = AdapterLogger("oracle")
 
 
 class OracleConnectionMethod(enum.Enum):
@@ -65,7 +58,7 @@ class OracleAdapterCredentials(Credentials):
     # OracleConnectionMethod.HOST
     protocol: Optional[str] = None
     host: Optional[str] = None
-    port: Optional[Port] = None
+    port: Optional[Union[str, int]] = None
     service: Optional[str] = None
 
     # OracleConnectionMethod.CONNECTION_STRING

dbt/include/oracle/macros/adapters.sql

@@ -234,13 +234,13 @@
   {%- set tmp_column = column_name + "__dbt_alter" -%}
 
   {% call statement('alter_column_type 1', fetch_result=False) %}
-    alter table {{ relation.include(False, True, True).quote(schema=False, identifier=False) }} add column {{ tmp_column }} {{ new_column_type }}
+    alter table {{ relation.include(False, True, True).quote(schema=False, identifier=False) }} add {{ tmp_column }} {{ new_column_type }}
   {% endcall %}
   {% call statement('alter_column_type 2', fetch_result=False) %}
     update {{ relation.include(False, True, True).quote(schema=False, identifier=False)  }} set {{ tmp_column }} = {{ column_name }}
   {% endcall %}
   {% call statement('alter_column_type 3', fetch_result=False) %}
-    alter table {{ relation.include(False, True, True).quote(schema=False, identifier=False) }} drop column {{ column_name }} cascade
+    alter table {{ relation.include(False, True, True).quote(schema=False, identifier=False) }} drop column {{ column_name }} cascade constraints
   {% endcall %}
   {% call statement('alter_column_type 4', fetch_result=False) %}
     alter table {{ relation.include(False, True, True).quote(schema=False, identifier=False) }} rename column {{ tmp_column }} to {{ column_name }}

dbt/include/oracle/macros/columns.sql

@@ -0,0 +1,41 @@
+{#
+ Copyright (c) 2022, Oracle and/or its affiliates.
+ Copyright (c) 2020, Vitor Avancini
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+     https://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+#}
+{% macro oracle__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}
+
+  {% if add_columns is none %}
+    {% set add_columns = [] %}
+  {% endif %}
+  {% if remove_columns is none %}
+    {% set remove_columns = [] %}
+  {% endif %}
+
+  {% set sql -%}
+
+     alter {{ relation.type }} {{ relation }}
+
+            {% for column in add_columns %}
+               add {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}
+            {% endfor %}{{ ',' if add_columns and remove_columns }}
+
+            {% for column in remove_columns %}
+                drop column {{ column.name }}{{ ',' if not loop.last }}
+            {% endfor %}
+
+  {%- endset -%}
+
+  {% do run_query(sql) %}
+{% endmacro %}

dbt/include/oracle/macros/materializations/incremental/helpers.sql

@@ -34,26 +34,40 @@
     )
 {%- endmacro %}
 
-{% macro oracle_incremental_upsert(tmp_relation, target_relation, unique_key=none, statement_name="main") %}
-    {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}
+{% macro oracle_incremental_upsert(tmp_relation, target_relation, dest_columns, unique_key=none, statement_name="main") %}
+    {%- set on_predicates = [] -%}
+    {%- set unique_key_list = [] -%}
     {%- set dest_cols_csv = dest_columns | map(attribute='name') | join(', ') -%}
 
-    {%- if unique_key is not none -%}
-    merge into {{ target_relation }} target
-      using {{ tmp_relation }} temp
-      on (temp.{{ unique_key }} = target.{{ unique_key }})
-    when matched then
-      update set
-      {% for col in dest_columns if col.name.upper() != unique_key.upper() -%}
-        target.{{ col.name }} = temp.{{ col.name }}{% if not loop.last %}, {% endif %}
-      {% endfor -%}
-    when not matched then
-      insert({{ dest_cols_csv }})
-      values(
-        {% for col in dest_columns -%}
-          temp.{{ col.name }}{% if not loop.last %}, {% endif %}
-        {% endfor -%}
-      )
+    {%- if unique_key -%}
+        {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}
+          {% for key in unique_key | unique %}
+                {% do unique_key_list.append(key.upper()) %}
+            {% endfor %}
+        {% else %}
+            {% do unique_key_list.append(unique_key.upper()) %}
+        {% endif %}
+        {% for key in unique_key_list %}
+            {% set this_key_match %}
+                    temp.{{ key }} = target.{{ key }}
+            {% endset %}
+            {% do on_predicates.append(this_key_match) %}
+        {% endfor %}
+        merge into {{ target_relation }} target
+          using {{ tmp_relation }} temp
+          on ({{ on_predicates | join(' AND ') }})
+        when matched then
+          update set
+          {% for col in dest_columns if col.name.upper() not in unique_key_list -%}
+            target.{{ col.name }} = temp.{{ col.name }}{% if not loop.last %}, {% endif %}
+          {% endfor -%}
+        when not matched then
+          insert({{ dest_cols_csv }})
+          values(
+            {% for col in dest_columns -%}
+              temp.{{ col.name }}{% if not loop.last %}, {% endif %}
+            {% endfor -%}
+          )
     {%- else -%}
     insert into {{ target_relation }} ({{ dest_cols_csv }})
     (

dbt/include/oracle/macros/materializations/incremental/incremental.sql

@@ -22,6 +22,8 @@
   {% set target_relation = this.incorporate(type='table') %}
   {% set existing_relation = load_relation(this) %}
   {% set tmp_relation = make_temp_relation(this) %}
+  {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %}
+
 
   {{ run_hooks(pre_hooks, inside_transaction=False) }}
 
@@ -50,7 +52,11 @@
       {% do adapter.expand_target_column_types(
              from_relation=tmp_relation,
              to_relation=target_relation) %}
-      {% set build_sql = oracle_incremental_upsert(tmp_relation, target_relation, unique_key=unique_key) %}
+      {% set dest_columns = process_schema_changes(on_schema_change, tmp_relation, existing_relation) %}
+      {% if not dest_columns %}
+        {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}
+      {% endif %}
+      {% set build_sql = oracle_incremental_upsert(tmp_relation, target_relation, dest_columns, unique_key=unique_key) %}
   {% endif %}
 
   {% call statement("main") %}