Merge pull request #38 from codeforkjeff/dbt-1-3-0-compat

dbt 1.3.0 compatibility

Author: codeforkjeff

Dockerfile

@@ -8,7 +8,7 @@ RUN apt-get update && apt-get -y install git python3 python3-pip python3-venv sq
 WORKDIR /opt/dbt-sqlite
 
 RUN python3 -m pip install --upgrade pip \
-    && python3 -m pip install pytest pytest-dotenv dbt-core~=1.2.0 dbt-tests-adapter~=1.2.0
+    && python3 -m pip install pytest pytest-dotenv dbt-core~=1.3.0 dbt-tests-adapter~=1.3.0
 
 RUN wget -q https://github.com/nalgeon/sqlean/releases/download/0.15.2/crypto.so
 RUN wget -q https://github.com/nalgeon/sqlean/releases/download/0.15.2/math.so

README.md

@@ -137,6 +137,11 @@ not worth the effort.
   - When materialized as a table, a CAST will result in the specified type for
     INT, REAL, TEXT; casts to NUMERIC and BOOLEAN result in a 'NUM' column type.
 
+- To get the best fidelity to your seed data, declare all the column types as TEXT
+  in your [seed configurations](https://docs.getdbt.com/reference/seed-configs)
+  and create a model to do the casts and conversions.
+
+
 ## SQLite Extensions
 
 These modules from SQLean are needed for certain functionality to work:

dbt/include/sqlite/macros/materializations/incremental/incremental.sql

@@ -21,7 +21,13 @@
     ;
 {%- endmacro %}
 
-{# override this just so we can call sqlite_incremental_upsert #}
+{#
+the incremental materialization was overhauled in dbt 1.3.0 to make it easier to override
+adapter-specific pieces, but it's hard to use it as intended in our case, because it renames
+models, which is difficult to handle so we just avoid it
+
+also, we call sqlite_incremental_upsert b/c the syntax for insert differs
+#}
 {% materialization incremental, adapter='sqlite' -%}
 
   {% set unique_key = config.get('unique_key') %}
@@ -63,6 +69,10 @@
 
   {% do persist_docs(target_relation, model) %}
 
+  {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}
+    {% do create_indexes(target_relation) %}
+  {% endif %}
+
   {{ run_hooks(post_hooks, inside_transaction=True) }}
 
   -- `COMMIT` happens here

setup.py

@@ -13,10 +13,6 @@ def _get_plugin_version():
     _version_path = os.path.join(
         this_directory, 'dbt', 'adapters', 'sqlite', '__version__.py'
     )
-    _semver = r"""(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)"""
-    _pre = r"""((?P<prekind>a|b|rc)(?P<pre>\d+))?"""
-    _version_pattern = fr"""version\s*=\s*["']{_semver}{_pre}["']"""
-
     with open(_version_path) as f:
         line = f.read().strip()
         delim = '"' if '"' in line else "'"
@@ -50,7 +46,7 @@ def _get_plugin_version():
         ]
     },
     install_requires=[
-        "dbt-core>=1.2.0"
+        "dbt-core>=1.3.0"
     ],
     classifiers=[
         'Development Status :: 4 - Beta',

tests/functional/adapter/concurrency/test_concurrency.py

@@ -0,0 +1,31 @@
+from dbt.tests.util import (
+    run_dbt,
+    check_relations_equal,
+    rm_file,
+    write_file
+    )
+from dbt.tests.adapter.concurrency.test_concurrency import (
+    BaseConcurrency,
+    seeds__update_csv
+    )
+
+
+class TestConncurenncySqlite(BaseConcurrency):
+
+    def test_conncurrency_sqlite(self, project):
+        run_dbt(["seed", "--select", "seed"])
+        results = run_dbt(["run"], expect_pass=False)
+        assert len(results) == 7
+        check_relations_equal(project.adapter, ["SEED", "VIEW_MODEL"])
+        check_relations_equal(project.adapter, ["SEED", "DEP"])
+        check_relations_equal(project.adapter, ["SEED", "TABLE_A"])
+        check_relations_equal(project.adapter, ["SEED", "TABLE_B"])
+
+        rm_file(project.project_root, "seeds", "seed.csv")
+        write_file(seeds__update_csv, project.project_root + '/seeds', "seed.csv")
+        results = run_dbt(["run"], expect_pass=False)
+        assert len(results) == 7
+        check_relations_equal(project.adapter, ["SEED", "VIEW_MODEL"])
+        check_relations_equal(project.adapter, ["SEED", "DEP"])
+        check_relations_equal(project.adapter, ["SEED", "TABLE_A"])
+        check_relations_equal(project.adapter, ["SEED", "TABLE_B"])

tests/functional/adapter/ephemeral/test_ephemeral.py

@@ -0,0 +1,11 @@
+from dbt.tests.adapter.ephemeral.test_ephemeral import BaseEphemeralMulti
+from dbt.tests.util import run_dbt, check_relations_equal
+
+
+class TestEphemeralMultiSqlite(BaseEphemeralMulti):
+
+   def test_ephemeral_multi_sqlite(self, project):
+        run_dbt(["seed"])
+        results = run_dbt(["run"])
+        assert len(results) == 3
+        check_relations_equal(project.adapter, ["SEED", "DEPENDENT", "DOUBLE_DEPENDENT", "SUPER_DEPENDENT"])

tests/functional/adapter/test_basic.py

@@ -11,6 +11,7 @@
 from dbt.tests.adapter.basic.test_empty import BaseEmpty
 from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral
 from dbt.tests.adapter.basic.test_incremental import BaseIncremental
+from dbt.tests.adapter.basic.test_incremental import BaseIncrementalNotSchemaChange
 from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests
 from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols
 from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp
@@ -42,6 +43,10 @@ class TestIncrementalSqlite(BaseIncremental):
     pass
 
 
+class TestBaseIncrementalNotSchemaChangeSqlite(BaseIncrementalNotSchemaChange):
+    pass
+
+
 class TestGenericTestsSqlite(BaseGenericTests):
     pass
 

tests/functional/adapter/utils/test_data_types.py

@@ -1,5 +1,6 @@
 import pytest
 from dbt.tests.adapter.utils.data_types.test_type_bigint import BaseTypeBigInt
+from dbt.tests.adapter.utils.data_types.test_type_boolean import BaseTypeBoolean
 from dbt.tests.adapter.utils.data_types.test_type_float import BaseTypeFloat
 from dbt.tests.adapter.utils.data_types.test_type_int import BaseTypeInt
 from dbt.tests.adapter.utils.data_types.test_type_numeric import BaseTypeNumeric
@@ -23,7 +24,13 @@
 class TestTypeBigInt(BaseTypeBigInt):
     pass
 
-    
+
+# users should imlement boolean columns as INT with values of 0 or 1
+@pytest.mark.skip("boolean not supported in SQLite")
+class TestTypeBoolean(BaseTypeBoolean):
+    pass
+
+
 class TestTypeFloat(BaseTypeFloat):
 
     models__actual_sql = """

tests/functional/adapter/utils/test_utils.py

@@ -5,9 +5,13 @@
     models__test_datediff_yml,
 )
 from dbt.tests.adapter.utils.test_any_value import BaseAnyValue
+from dbt.tests.adapter.utils.test_array_append import BaseArrayAppend
+from dbt.tests.adapter.utils.test_array_concat import BaseArrayConcat
+from dbt.tests.adapter.utils.test_array_construct import BaseArrayConstruct
 from dbt.tests.adapter.utils.test_bool_or import BaseBoolOr
 from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText
 from dbt.tests.adapter.utils.test_concat import BaseConcat
+from dbt.tests.adapter.utils.test_current_timestamp import BaseCurrentTimestampNaive
 from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd
 #from dbt.tests.adapter.utils.test_datediff import BaseDateDiff
 from dbt.tests.adapter.utils.test_date_trunc import BaseDateTrunc
@@ -31,6 +35,21 @@ class TestAnyValue(BaseAnyValue):
     pass
 
 
+@pytest.mark.skip("arrays not supported in SQLite")
+class TestArrayAppend(BaseArrayAppend):
+    pass
+
+
+@pytest.mark.skip("arrays not supported in SQLite")
+class TestArrayConcat(BaseArrayConcat):
+    pass
+
+
+@pytest.mark.skip("arrays not supported in SQLite")
+class TestArrayConstruct(BaseArrayConstruct):
+    pass
+
+
 class TestBoolOr(BaseBoolOr):
     pass
 
@@ -43,6 +62,11 @@ class TestConcat(BaseConcat):
     pass
 
 
+@pytest.mark.skip("timestamps not supported in SQLite")
+class TestCurrentTimestampNaive(BaseCurrentTimestampNaive):
+    pass
+
+
 class TestDateAdd(BaseDateAdd):
     pass