From be24134fc9dc81fd09e1b261c452294a709740da Mon Sep 17 00:00:00 2001 From: Bruno Antonellini Date: Tue, 31 Dec 2024 10:42:31 -0300 Subject: [PATCH 1/3] DCV-3128 changes to setup template --- copier.yml | 2 +- ...point %}.pre-commit-config.yaml{% endif %}.jinja | 13 ++++++------- ... %} => {% if use_yamllint %}.yamllint{% endif %} | 0 {{_copier_conf.answers_file}}.jinja | 2 -- 4 files changed, 7 insertions(+), 10 deletions(-) rename {% if setup_precommit %}{{ dbt_project_dir }}{% endif %}/{% if use_yamllint %}.yamllint{% endif %} => {% if use_yamllint %}.yamllint{% endif %} (100%) delete mode 100644 {{_copier_conf.answers_file}}.jinja diff --git a/copier.yml b/copier.yml index 653707c..a6e6e47 100644 --- a/copier.yml +++ b/copier.yml @@ -75,5 +75,5 @@ dbt_adapter: - redshift - databricks help: What dbt adapter are you using? - when: "{{not dbt_adapter and (setup_dbt_profile or setup_ci_cd or setup_sqlfluff)}}" + when: "{{not datacoves_dbt_adapter and (setup_dbt_profile or setup_ci_cd or setup_sqlfluff)}}" default: snowflake diff --git a/{% if use_sqlfluff or use_yamllint or use_dbt_checkpoint %}.pre-commit-config.yaml{% endif %}.jinja b/{% if use_sqlfluff or use_yamllint or use_dbt_checkpoint %}.pre-commit-config.yaml{% endif %}.jinja index fff0aa5..863c767 100644 --- a/{% if use_sqlfluff or use_yamllint or use_dbt_checkpoint %}.pre-commit-config.yaml{% endif %}.jinja +++ b/{% if use_sqlfluff or use_yamllint or use_dbt_checkpoint %}.pre-commit-config.yaml{% endif %}.jinja @@ -3,7 +3,7 @@ files: ^{{ dbt_project_dir }}/models/ repos: {% if use_dbt_checkpoint %} - repo: https://github.com/dbt-checkpoint/dbt-checkpoint - rev: v2.0.1 + rev: {{dbt_checkpoint_version}} hooks: - id: check-source-table-has-description @@ -17,7 +17,7 @@ repos: {% if use_sqlfluff %} - repo: https://github.com/sqlfluff/sqlfluff # this is the version of sqlfluff, needs to be updated when using a new sqlfluff version (pip show sqlfluff) - rev: 2.3.2 + rev: {{sqlfluff_version}} hooks: - id: sqlfluff-lint language: python @@ -25,17 +25,16 @@ repos: # sqlfluff-templater-dbt should match the version of sqlfluff above in rev (pip show sqlfluff-templater-dbt) # dbt-snowflake needs to match the version in transform tab of Datacoves (pip show dbt-snowflake) additional_dependencies: - # ["sqlfluff-templater-dbt==2.3.2", "dbt-snowflake==1.6.8", dbt-core==1.6.9] [ - "sqlfluff-templater-dbt==2.3.2", - "dbt-redshift==1.6.7", - dbt-core==1.6.9, + sqlfluff-templater-dbt=={{sqlfluff_version}}, + dbt-redshift=={{dbt_adapter_version}}, + dbt-core=={{dbt_core_version}}, ] args: [--config, "{{ dbt_project_dir }}/.sqlfluff"] {% endif %} {% if use_yamllint %} - repo: https://github.com/adrienverge/yamllint.git - rev: v1.17.0 + rev: {{yamllint_version}} hooks: - id: yamllint args: [-c=.yamllint] diff --git a/{% if setup_precommit %}{{ dbt_project_dir }}{% endif %}/{% if use_yamllint %}.yamllint{% endif %} b/{% if use_yamllint %}.yamllint{% endif %} similarity index 100% rename from {% if setup_precommit %}{{ dbt_project_dir }}{% endif %}/{% if use_yamllint %}.yamllint{% endif %} rename to {% if use_yamllint %}.yamllint{% endif %} diff --git a/{{_copier_conf.answers_file}}.jinja b/{{_copier_conf.answers_file}}.jinja deleted file mode 100644 index 090c992..0000000 --- a/{{_copier_conf.answers_file}}.jinja +++ /dev/null @@ -1,2 +0,0 @@ -# Changes here will be overwritten by Copier -{{ _copier_answers|to_nice_yaml }} From 3083f2977277bf43b40c970e4bbce19f5df2d5e0 Mon Sep 17 00:00:00 2001 From: Bruno Antonellini Date: Tue, 31 Dec 2024 13:24:08 -0300 Subject: [PATCH 2/3] Add env_var usage to CI --- .../workflows/pull_request_build.yml.jinja | 10 +++++----- .../workflows/push-to-main.yml.jinja | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/{%if setup_ci_cd and ci_provider=='github'%}.github{%endif%}/workflows/pull_request_build.yml.jinja b/{%if setup_ci_cd and ci_provider=='github'%}.github{%endif%}/workflows/pull_request_build.yml.jinja index f6b1dd2..50cb598 100644 --- a/{%if setup_ci_cd and ci_provider=='github'%}.github{%endif%}/workflows/pull_request_build.yml.jinja +++ b/{%if setup_ci_cd and ci_provider=='github'%}.github{%endif%}/workflows/pull_request_build.yml.jinja @@ -21,13 +21,13 @@ jobs: runs-on: ubuntu-latest {% if 'snowflake' in dbt_adapter %} - container: datacoves/ci-basic-dbt-snowflake:3.2 + container: datacoves/ci-basic-dbt-snowflake:{{datacoves_env_version}} {% elif 'redshift' in dbt_adapter %} - container: datacoves/ci-basic-dbt-redshift:3.2 + container: datacoves/ci-basic-dbt-redshift:{{datacoves_env_version}} {% elif 'databricks' in dbt_adapter %} - container: datacoves/ci-basic-dbt-databricks:3.2 + container: datacoves/ci-basic-dbt-databricks:{{datacoves_env_version}} {% elif 'bigquery' in dbt_adapter %} - container: datacoves/ci-basic-dbt-bigquery:3.2 + container: datacoves/ci-basic-dbt-bigquery:{{datacoves_env_version}} {% endif %} defaults: @@ -67,7 +67,7 @@ jobs: steps: - name: Checkout branch - uses: actions/checkout@v3.5.0 + uses: actions/checkout@v4 with: fetch-depth: 0 {% raw %}ref: ${{ github.event.pull_request.head.sha }}{% endraw %} diff --git a/{%if setup_ci_cd and ci_provider=='github'%}.github{%endif%}/workflows/push-to-main.yml.jinja b/{%if setup_ci_cd and ci_provider=='github'%}.github{%endif%}/workflows/push-to-main.yml.jinja index 341d2a1..1cd531a 100644 --- a/{%if setup_ci_cd and ci_provider=='github'%}.github{%endif%}/workflows/push-to-main.yml.jinja +++ b/{%if setup_ci_cd and ci_provider=='github'%}.github{%endif%}/workflows/push-to-main.yml.jinja @@ -30,13 +30,13 @@ jobs: # https://github.com///settings/environments # environment: PR_ENV {% if 'snowflake' in dbt_adapter %} - container: datacoves/ci-basic-dbt-snowflake:3.2 + container: datacoves/ci-basic-dbt-snowflake:{{datacoves_env_version}} {% elif 'redshift' in dbt_adapter %} - container: datacoves/ci-basic-dbt-redshift:3.2 + container: datacoves/ci-basic-dbt-redshift:{{datacoves_env_version}} {% elif 'databricks' in dbt_adapter %} - container: datacoves/ci-basic-dbt-databricks:3.2 + container: datacoves/ci-basic-dbt-databricks:{{datacoves_env_version}} {% elif 'bigquery' in dbt_adapter %} - container: datacoves/ci-basic-dbt-bigquery:3.2 + container: datacoves/ci-basic-dbt-bigquery:{{datacoves_env_version}} {% endif %} defaults: run: @@ -75,7 +75,7 @@ jobs: steps: - name: Checkout branch - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: {% raw %}ref: ${{ github.event.push.head.sha }}{% endraw%} fetch-depth: 0 From 47aa74f125c669e1167a4adc2b1407da1a0557a4 Mon Sep 17 00:00:00 2001 From: Bruno Antonellini Date: Tue, 31 Dec 2024 14:12:39 -0300 Subject: [PATCH 3/3] Implement different adapters in precommitconfig --- ...se_dbt_checkpoint %}.pre-commit-config.yaml{% endif %}.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{% if use_sqlfluff or use_yamllint or use_dbt_checkpoint %}.pre-commit-config.yaml{% endif %}.jinja b/{% if use_sqlfluff or use_yamllint or use_dbt_checkpoint %}.pre-commit-config.yaml{% endif %}.jinja index 863c767..ac2afc2 100644 --- a/{% if use_sqlfluff or use_yamllint or use_dbt_checkpoint %}.pre-commit-config.yaml{% endif %}.jinja +++ b/{% if use_sqlfluff or use_yamllint or use_dbt_checkpoint %}.pre-commit-config.yaml{% endif %}.jinja @@ -27,7 +27,7 @@ repos: additional_dependencies: [ sqlfluff-templater-dbt=={{sqlfluff_version}}, - dbt-redshift=={{dbt_adapter_version}}, + {% if 'snowflake' in dbt_adapter %}dbt-snowflake=={{dbt_adapter_version}},{% elif 'redshift' in dbt_adapter %}dbt-redshift=={{dbt_adapter_version}},{% elif 'databricks' in dbt_adapter %}dbt-databricks=={{dbt_adapter_version}},{% elif 'bigquery' in dbt_adapter %}dbt-bigquery=={{dbt_adapter_version}},{% endif %} dbt-core=={{dbt_core_version}}, ] args: [--config, "{{ dbt_project_dir }}/.sqlfluff"]