Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,42 @@ jobs:
- name: Run integration tests against snowflake
run: ./scripts/ci/test.sh snowflake

# -------- Fusion parse (dbt 2.x) ------------------------------------------
# Parses the package with the dbt Fusion engine and fails on any warning,
# so Fusion deprecations ("will error post preview") are caught before they
# become hard errors. Fusion's parse never connects to the warehouse, so it
# runs with dummy credentials and needs no secrets.
fusion-parse:
name: fusion (parse)
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
- name: Parse with dbt Fusion
run: ./scripts/fusion/check.sh

# -------- Snowflake on Fusion (dbt 2.x) ----------------------------------
# Same two-pass build as `integration-snowflake`, on the dbt Fusion engine.
# Snowflake is the only CI warehouse that Fusion supports and this package
# tests in CI; see scripts/fusion/test.sh.
integration-snowflake-fusion:
name: integration (snowflake, fusion)
runs-on: ubuntu-latest
timeout-minutes: 30
env:
DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_PRIVATE_KEY }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }}
steps:
- name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
- name: Run integration tests against snowflake on Fusion
run: ./scripts/fusion/test.sh snowflake

# -------- BigQuery (cloud, WIF auth) ------------------------------------
integration-bigquery:
name: integration (bigquery)
Expand Down
18 changes: 18 additions & 0 deletions .github/workflows/pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@
# Tier 2/3.
# - Only the latest supported dbt version is tested here. The full
# version matrix runs in Tier 3.
# - The package is also parsed with the dbt Fusion engine (dbt 2.x),
# failing on any warning. Fusion's parse doesn't connect to a warehouse,
# so it runs on dummy credentials. Fusion builds run in Tier 2/3.
#
# Hardening notes:
# - `permissions: contents: read` only. No `id-token`, no escalation.
Expand Down Expand Up @@ -78,3 +81,18 @@ jobs:

- name: Run integration tests against ${{ matrix.warehouse }}
run: ./scripts/ci/test.sh ${{ matrix.warehouse }}

# -------- Fusion parse (dbt 2.x) ------------------------------------------
# Parses the package with the dbt Fusion engine and fails on any warning,
# so Fusion deprecations ("will error post preview") are caught before they
# become hard errors. Fusion's parse never connects to the warehouse, so it
# runs with dummy credentials and needs no secrets.
fusion-parse:
name: fusion (parse)
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
- name: Parse with dbt Fusion
run: ./scripts/fusion/check.sh
2 changes: 1 addition & 1 deletion .github/workflows/publish_docs_on_release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ jobs:
uses: actions/checkout@v3

- name: Install Python packages
run: python -m pip install dbt-snowflake~=1.9.0
run: python -m pip install dbt-snowflake~=1.11.0

- name: Test database connection
run: dbt debug
Expand Down
70 changes: 44 additions & 26 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -122,29 +122,14 @@ jobs:
strategy:
fail-fast: false
# Throttle parallelism to be a good neighbour to the shared cloud
# warehouses and to bound runner billing. With the EOL versions
# dropped (see comment below) the matrix is ~15 slots, so 8 in flight
# completes a release validation quickly.
# warehouses and to bound runner billing. The matrix is 12 slots, so
# 8 in flight completes a release validation quickly.
max-parallel: 8
matrix:
# dbt versions 1.3–1.8 were dropped from CI on 2026-05-26. They
# cannot run on the runner's Python 3.12 (`distutils`, removed in
# 3.12, is imported by dbt-core ≤1.7) and predate the `microbatch`
# incremental strategy (a 1.9 feature) that the test fixture uses.
# Keeping them green would require a legacy Python 3.11 lane plus
# per-version gates — pure tech debt for versions dbt Labs has
# already EOL'd.
#
# This is a CI-COVERAGE change only, not a support drop:
# - `require-dbt-version` in dbt_project.yml still allows ≥1.3.0,
# so consumers on older dbt can still install the package — it's
# simply no longer verified each release.
# - The `tox.ini` envs for 1.3–1.8 are intentionally KEPT so a
# maintainer can still run e.g. `tox -e integration_snowflake_
# 1_5_0` on a local Python 3.11 to debug a consumer report.
# - The formal support-floor bump (require-dbt-version → ≥1.9,
# removing the tox envs) is a separate 3.0.0 decision.
# See specs/ci-rework/README.md §12.3.
# The package's support floor is dbt 1.10 (`require-dbt-version` in
# dbt_project.yml): Fusion requires generic-test arguments under
# `arguments:`, which dbt Core only understands from 1.10. Pinned
# envs below therefore start at 1_10_0. See README "Supported dbt versions".
include:
# "Latest" (unversioned env per warehouse) — picks up the
# highest adapter release at the time the workflow runs.
Expand All @@ -162,22 +147,19 @@ jobs:
# Snowflake — pinned versions, newest first.
- { warehouse: snowflake, dbt_version: "1_11_0" }
- { warehouse: snowflake, dbt_version: "1_10_0" }
- { warehouse: snowflake, dbt_version: "1_9_0" }

# BigQuery
- { warehouse: bigquery, dbt_version: "1_11_0" }
- { warehouse: bigquery, dbt_version: "1_10_0" }
- { warehouse: bigquery, dbt_version: "1_9_0" }

# Postgres — no upstream 1.11 release as of 2026-05-12.
- { warehouse: postgres, dbt_version: "1_10_0" }
- { warehouse: postgres, dbt_version: "1_9_0" }

# Trino — no upstream 1.11 / 1.9 / 1.8 releases; 1.10 is latest.
- { warehouse: trino, dbt_version: "1_10_0" }

# SQL Server — adapter lags; 1.9 is the latest published.
- { warehouse: sqlserver, dbt_version: "1_9_0" }
# SQL Server — floor version; "latest" covers newer minors.
- { warehouse: sqlserver, dbt_version: "1_10_0" }

steps:
- name: Checkout
Expand Down Expand Up @@ -207,6 +189,42 @@ jobs:
- name: Run integration tests
run: ./scripts/ci/test.sh ${{ matrix.warehouse }} ${{ matrix.dbt_version }}

# -------- Fusion parse (dbt 2.x) ------------------------------------------
# Parses the package with the dbt Fusion engine and fails on any warning,
# so Fusion deprecations ("will error post preview") are caught before they
# become hard errors. Fusion's parse never connects to the warehouse, so it
# runs with dummy credentials and needs no secrets.
fusion-parse:
name: fusion (parse)
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
- name: Parse with dbt Fusion
run: ./scripts/fusion/check.sh

# -------- Snowflake on Fusion (dbt 2.x) ----------------------------------
# Same two-pass build as `integration-snowflake`, on the dbt Fusion engine.
# Snowflake is the only CI warehouse that Fusion supports and this package
# tests in CI; see scripts/fusion/test.sh.
integration-snowflake-fusion:
name: integration (snowflake, fusion)
runs-on: ubuntu-latest
timeout-minutes: 30
env:
DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_PRIVATE_KEY }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }}
DBT_ENV_SECRET_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }}
steps:
- name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5
- name: Run integration tests against snowflake on Fusion
run: ./scripts/fusion/test.sh snowflake

# -------- Databricks stub -----------------------------------------------
# Visible in the workflow run so reviewers see Databricks has been
# *considered* and *skipped on purpose*, not forgotten. When Databricks
Expand Down
33 changes: 25 additions & 8 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,14 +47,31 @@ See the generated [dbt docs site](https://brooklyn-data.github.io/dbt_artifacts/

## Supported dbt versions

As of `2.11.0`, the package's CI verifies against **dbt 1.9 and newer**.
Older versions (1.3–1.8) may continue to work but are **no longer verified
in CI** — dbt Labs has marked them end-of-life and they cannot run on the
Python 3.12 toolchain our CI uses. The package still *installs* on dbt
≥1.3.0 (`require-dbt-version` is unchanged), so existing users are not
blocked; we simply can no longer confirm new releases against those
versions. If you rely on dbt < 1.9, pin a known-good `dbt_artifacts`
release in your `packages.yml`.
As of `3.0.0`, the package requires **dbt 1.10 or newer** (dbt Core 1.10+
or the dbt Fusion engine). The package's generic tests pass their arguments
under `arguments:`, which Fusion requires and dbt Core only understands from
1.10. If you're on an older dbt version, pin `dbt_artifacts` to `2.11.1` in
your `packages.yml`:

```yml
packages:
- package: brooklyn-data/dbt_artifacts
version: 2.11.1
```

### dbt Fusion engine (dbt 2.x)

The package is compatible with the [dbt Fusion engine](https://docs.getdbt.com/docs/fusion).
CI parses it with Fusion on every pull request (failing on any deprecation
warning) and runs the full Snowflake integration suite on Fusion before
release. Fusion also supports BigQuery and Databricks; the package parses
cleanly against both, but they are not yet integration-tested on Fusion.
Postgres and Trino are experimental in Fusion, and SQL Server and Spark are
not supported by Fusion, so on those warehouses keep using dbt Core.

Fusion runs `on-run-end` hooks for `dbt compile` and `dbt show`, which dbt
Core does not. `upload_results` skips those commands, so the same invocations
are recorded on both engines.

## Quickstart

Expand Down
2 changes: 1 addition & 1 deletion dbt_project.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name: "dbt_artifacts"
version: "2.11.1"
config-version: 2
require-dbt-version: [">=1.3.0", "<3.0.0"]
require-dbt-version: [">=1.10.0", "<3.0.0"]
profile: "dbt_artifacts"

clean-targets: # folders to be removed by `dbt clean`
Expand Down
2 changes: 1 addition & 1 deletion docs/MAINTAINERS.md
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ env automatically. You no longer need `pyenv virtualenv` or per-version
manual setup — just pass the version:

```bash
./scripts/ci/test.sh snowflake 1_8_0
./scripts/ci/test.sh snowflake 1_10_0
./scripts/ci/test.sh snowflake 1_11_0
```

Expand Down
4 changes: 0 additions & 4 deletions integration_test_project/profiles.yml
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
# HEY! This file is used in the dbt-artifacts integrations tests with GitHub Actions.
# You should __NEVER__ check credentials into version control. Thanks for reading :)

config:
send_anonymous_usage_stats: False
use_colors: True

dbt_artifacts:
target: snowflake
outputs:
Expand Down
6 changes: 4 additions & 2 deletions macros/tests/is_between.sql
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,12 @@
columns:
- name: pct_of_month_smb
tests:
- dbt_artifacts.is_between: {min_value: 0, max_value: 1}
- dbt_artifacts.is_between:
arguments: {min_value: 0, max_value: 1}
- name: runtime_regression_ratio
tests:
- dbt_artifacts.is_between: {min_value: 0} # >= 0, no upper bound
- dbt_artifacts.is_between:
arguments: {min_value: 0} # >= 0, no upper bound
-#}

{% test is_between(model, column_name, min_value=none, max_value=none) %}
Expand Down
10 changes: 9 additions & 1 deletion macros/upload_results/upload_results.sql
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,15 @@

{% macro upload_results(results) -%}

{% if execute %}
{#
dbt Core only fires on-run-end for commands that execute nodes. Fusion
also fires it for `compile` and `show` (including editor previews),
which would upload a phantom `inline` model and an invocation on every
call. Skip those so both engines record the same invocations.
#}
{% set skipped_commands = ['compile', 'show'] %}

{% if execute and invocation_args_dict.get('which') not in skipped_commands %}

{% set datasets_to_load = ['exposures', 'seeds', 'snapshots', 'invocations', 'sources', 'tests', 'models'] %}
{% if results != [] %}
Expand Down
5 changes: 3 additions & 2 deletions models/fct_dbt__consumption_by_model.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,9 @@ models:
description: This model's share of the month's total SMB (0..1), a window over billing_month.
tests:
- dbt_artifacts.is_between:
min_value: 0
max_value: 1
arguments:
min_value: 0
max_value: 1
- name: builds_per_day_avg
description: Average successful builds per active build-day (smb_quantity / distinct_days_built).
- name: distinct_days_built
Expand Down
3 changes: 2 additions & 1 deletion models/fct_dbt__consumption_daily.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ models:
tests:
- not_null
- accepted_values:
values: ['smb', 'active_target_tables']
arguments:
values: ['smb', 'active_target_tables']
- name: quantity
description: Metered quantity for the day x meter (sum of the detail rows).
tests:
Expand Down
3 changes: 2 additions & 1 deletion models/fct_dbt__consumption_daily_detail.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ models:
tests:
- not_null
- accepted_values:
values: ['smb', 'active_target_tables']
arguments:
values: ['smb', 'active_target_tables']
- name: materialization
description: >
Model materialization for 'smb' rows; null for 'active_target_tables'
Expand Down
3 changes: 2 additions & 1 deletion models/fct_dbt__consumption_forecast.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ models:
description: month_to_date_quantity / allowance for the smb meter; null otherwise.
tests:
- dbt_artifacts.is_between:
min_value: 0
arguments:
min_value: 0
- name: days_remaining
description: Calendar days remaining in the month after as_of (0 for fully-elapsed months).
- name: is_on_pace_to_exceed
Expand Down
6 changes: 4 additions & 2 deletions models/fct_dbt__dag_bottlenecks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,13 @@ models:
description: Sum of measured stall seconds attributed to this parent that day.
tests:
- dbt_artifacts.is_between:
min_value: 0
arguments:
min_value: 0
- name: max_stall_seconds
description: Largest single measured stall attributed to this parent that day.
tests:
- dbt_artifacts.is_between:
min_value: 0
arguments:
min_value: 0
- name: invocations_observed
description: Distinct invocations in which this parent gated at least one child that day.
3 changes: 2 additions & 1 deletion models/fct_dbt__dag_bottlenecks_detail.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,5 @@ models:
description: max(child_compile_started_at - parent_query_completed_at, 0) in seconds.
tests:
- dbt_artifacts.is_between:
min_value: 0
arguments:
min_value: 0
5 changes: 3 additions & 2 deletions models/fct_dbt__flaky_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,9 @@ models:
description: flips / executions (0..1).
tests:
- dbt_artifacts.is_between:
min_value: 0
max_value: 1
arguments:
min_value: 0
max_value: 1
- name: last_flip_at
description: run_started_at of the most recent flip's pass that month (null if no flips).
- name: is_flaky
Expand Down
3 changes: 2 additions & 1 deletion models/fct_dbt__model_performance.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@ models:
description: median_runtime / baseline_runtime; null when baseline is null or 0.
tests:
- dbt_artifacts.is_between:
min_value: 0
arguments:
min_value: 0
- name: is_regressed
description: >
True when runtime_regression_ratio > regression_threshold and
Expand Down
5 changes: 3 additions & 2 deletions models/fct_dbt__run_health_daily.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,9 @@ models:
description: successes / (successes + failures + errors); null on days with no such executions.
tests:
- dbt_artifacts.is_between:
min_value: 0
max_value: 1
arguments:
min_value: 0
max_value: 1
- name: total_runtime_seconds
description: Sum of total_node_runtime across all node executions that day.
- name: max_node_runtime_seconds
Expand Down
5 changes: 3 additions & 2 deletions models/fct_dbt__run_health_daily_detail.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,9 @@ models:
description: successes / (successes + failures + errors) for the target.
tests:
- dbt_artifacts.is_between:
min_value: 0
max_value: 1
arguments:
min_value: 0
max_value: 1
- name: total_runtime_seconds
description: Sum of total_node_runtime for the target that day.
- name: max_node_runtime_seconds
Expand Down
Loading
Loading