Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions .github/workflows/dev_dbt.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
---
name: Build and publish dbt-demo

on:
workflow_dispatch:
push:
branches:
- main
# TODO (@NickLarsenNZ): Also build on release branches, but with a stackable0.0.0-dev or stackableXX.X.X tag.
# - release-*
paths:
- demos/airflow-scheduled-job/dbt/Dockerfile
- demos/airflow-scheduled-job/dbt/requirements.txt
- .github/workflows/dev_dbt.yaml

jobs:
build_image:
name: Reusable Workflow
uses: ./.github/workflows/reusable_build_image.yaml
secrets:
harbor-robot-secret: ${{ secrets.HARBOR_ROBOT_DEMOS_GITHUB_ACTION_BUILD_SECRET }}
slack-token: ${{ secrets.SLACK_CONTAINER_IMAGE_TOKEN }}
with:
image-name: dbt-demo
# TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that
# the demo is reproducable for the release and it will be automatically replaced for the release branch.
image-version: 0.0.1
containerfile-path: demos/airflow-scheduled-job/dbt/Dockerfile
41 changes: 41 additions & 0 deletions demos/airflow-scheduled-job/dbt/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
FROM python:3.12-slim-bullseye AS builder

# Install build dependencies
RUN apt-get update && apt-get install -y \
git \
gcc \
&& rm -rf /var/lib/apt/lists/*

# Create virtual environment
RUN python -m venv /opt/venv
ENV PATH="/opt/venv/bin:$PATH"

# Install Python packages
COPY requirements.txt .
RUN pip install --no-cache-dir --upgrade pip && \
pip install --no-cache-dir -r requirements.txt

# Final stage
FROM python:3.12-slim-bullseye

RUN apt-get update && apt-get install -y \
git \
curl \
vim \
&& rm -rf /var/lib/apt/lists/*

# Copy virtual environment from builder
COPY --from=builder /opt/venv /opt/venv
ENV PATH="/opt/venv/bin:$PATH"

WORKDIR /dbt

COPY dbt_test ./dbt_test

# Security: non-root user
RUN useradd -m -u 1000 dbt && chown -R dbt:dbt /dbt
USER dbt

ENV DBT_PROFILES_DIR=/dbt

CMD ["dbt", "run"]
9 changes: 9 additions & 0 deletions demos/airflow-scheduled-job/dbt/dbt_test/dbt_project.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
---
name: 'dbt_demo'
version: '1.0.0'
config-version: 2
profile: 'trino_demo'
model-paths: ['models']
macro-paths: ['macros']
target-path: "target"
clean-targets: ['target']
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{% test id_in_range(model, column_name) %}
select *
from {{ model }}
where {{ column_name }} < 1 or {{ column_name }} > 5
{% endtest %}
2 changes: 2 additions & 0 deletions demos/airflow-scheduled-job/dbt/dbt_test/models/my_table.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
{{ config(materialized='table') }}
select * from (values (1),(2),(3),(4),(5)) as t(id)
13 changes: 13 additions & 0 deletions demos/airflow-scheduled-job/dbt/dbt_test/models/schema.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
---
version: 2

models:
- name: my_table
description: "A simple demo table with integer IDs"
columns:
- name: id
description: "ID value"
tests:
- not_null # built-in dbt test
- unique # optional: ensure no duplicates
- id_in_range
2 changes: 2 additions & 0 deletions demos/airflow-scheduled-job/dbt/dbt_test/packages.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
---
packages: []
17 changes: 17 additions & 0 deletions demos/airflow-scheduled-job/dbt/dbt_test/profiles.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
---
trino_demo:
outputs:
iceberg:
type: trino
method: ldap
user: "{{ env_var('TRINO_USER') }}"
password: "{{ env_var('TRINO_PASSWORD') }}"
catalog: iceberg
host: "{{ env_var('TRINO_HOST') }}"
port: "{{ env_var('TRINO_PORT') | int }}"
schema: dbt_schema
threads: 1
cert: "{{ env_var('CERT_PATH') }}"
verify: true

target: iceberg
2 changes: 2 additions & 0 deletions demos/airflow-scheduled-job/dbt/requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
dbt-core==1.10.13
dbt-trino==1.9.3