From 3743dcd25a0b2477f18d461f1635ea79787697ad Mon Sep 17 00:00:00 2001 From: Bruno Antonellini Date: Wed, 19 Mar 2025 10:22:59 -0300 Subject: [PATCH 01/15] DCV-3288 dbt-coves setup improvements --- copier.yml | 47 ++++++--- ....yml.jinja => pull_request_build.yml.jinja | 0 ...o-main.yml.jinja => push-to-main.yml.jinja | 0 workflows/pull_request_build.yml.jinja | 92 +++++++++++++++++ workflows/push-to-main.yml.jinja | 99 +++++++++++++++++++ ...oject' in services %}.gitignore{% endif %} | 0 ...roject' in services %}README.md{% endif %} | 0 .../analyses/.gitkeep | 0 .../dbt_project.yml.jinja | 0 .../macros/.gitkeep | 0 .../models/L1_staging/.gitkeep | 0 .../models/L1_staging/my_model.sql | 0 .../models/L1_staging/my_model.yml | 0 .../models/L1_staging/source.yml | 0 .../models/L2_core/.gitkeep | 0 .../models/L3_marts/.gitkeep | 0 .../packages.yml | 0 .../seeds/.gitkeep | 0 .../snapshots/.gitkeep | 0 .../tests/.gitkeep | 0 ... use_sqlfluff %}.sqlfluffignore{% endif %} | 0 ... use_sqlfluff %}.sqlfluff{% endif %}.jinja | 0 .../sample_dag.py | 0 .../sample_dag.yml | 0 .../profiles.yml.jinja | 0 ...nfig_and_templates %}config.yml{% endif %} | 0 .../model_props.yml | 0 .../source_props.yml | 0 .../staging_model.sql | 0 .../staging_model_props.yml | 0 .../workflows/pull_request_build.yml.jinja | 92 +++++++++++++++++ .../workflows/push-to-main.yml.jinja | 99 +++++++++++++++++++ ...r=='gitlab'%}.gitlab-ci.yml{%endif%}.jinja | 0 {{ _copier_conf.answers_file }}.jinja | 2 + 34 files changed, 420 insertions(+), 11 deletions(-) rename {%if setup_ci_cd and ci_provider=='github'%}.github{%endif%}/workflows/pull_request_build.yml.jinja => pull_request_build.yml.jinja (100%) rename {%if setup_ci_cd and ci_provider=='github'%}.github{%endif%}/workflows/push-to-main.yml.jinja => push-to-main.yml.jinja (100%) create mode 100644 workflows/pull_request_build.yml.jinja create mode 100644 workflows/push-to-main.yml.jinja rename {% if setup_dbt_project %}.gitignore{% endif %} => {% if 'Base dbt project' in services %}.gitignore{% endif %} (100%) rename {% if setup_dbt_project %}README.md{% endif %} => {% if 'Base dbt project' in services %}README.md{% endif %} (100%) rename {{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%} => {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}}/analyses/.gitkeep (100%) rename {{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%} => {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}}/dbt_project.yml.jinja (100%) rename {{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%} => {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}}/macros/.gitkeep (100%) rename {{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%} => {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}}/models/L1_staging/.gitkeep (100%) rename {{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%} => {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}}/models/L1_staging/my_model.sql (100%) rename {{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%} => {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}}/models/L1_staging/my_model.yml (100%) rename {{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%} => {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}}/models/L1_staging/source.yml (100%) rename {{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%} => {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}}/models/L2_core/.gitkeep (100%) rename {{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%} => {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}}/models/L3_marts/.gitkeep (100%) rename {{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%} => {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}}/packages.yml (100%) rename {{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%} => {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}}/seeds/.gitkeep (100%) rename {{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%} => {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}}/snapshots/.gitkeep (100%) rename {{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%} => {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}}/tests/.gitkeep (100%) rename {{% if setup_precommit %}{{ dbt_project_dir }}{% endif %} => {% if 'Initial CI-CD scripts' in services %}{{ dbt_project_dir }}{% endif %}}/{% if use_sqlfluff %}.sqlfluffignore{% endif %} (100%) rename {{% if setup_precommit %}{{ dbt_project_dir }}{% endif %} => {% if 'Initial CI-CD scripts' in services %}{{ dbt_project_dir }}{% endif %}}/{% if use_sqlfluff %}.sqlfluff{% endif %}.jinja (100%) rename {{% if setup_airflow_dag%}{{ dags_path }}{% endif %} => {% if 'Sample Airflow DAGs' in services %}{{ dags_path }}{% endif %}}/sample_dag.py (100%) rename {{% if setup_airflow_dag %}{{ yml_dags_path }}{% endif %} => {% if 'Sample Airflow DAGs' in services %}{{ yml_dags_path }}{% endif %}}/sample_dag.yml (100%) rename {{% if setup_dbt_profile %}{{ airflow_profile_path }}{%endif%} => {% if 'dbt profile for automated runs' in services %}{{ airflow_profile_path }}{%endif%}}/profiles.yml.jinja (100%) rename {{% if setup_dbt_coves or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %} => {% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}}/.dbt_coves/{% if setup_dbtcoves_config or add_dbtcoves_config_and_templates %}config.yml{% endif %} (100%) rename {{% if setup_dbt_coves or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %} => {% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/model_props.yml (100%) rename {{% if setup_dbt_coves or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %} => {% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/source_props.yml (100%) rename {{% if setup_dbt_coves or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %} => {% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/staging_model.sql (100%) rename {{% if setup_dbt_coves or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %} => {% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/staging_model_props.yml (100%) create mode 100644 {%if 'Initial CI-CD scripts' in services and ci_provider=='github'%}.github{%endif%}/workflows/pull_request_build.yml.jinja create mode 100644 {%if 'Initial CI-CD scripts' in services and ci_provider=='github'%}.github{%endif%}/workflows/push-to-main.yml.jinja rename {%if setup_ci_cd and ci_provider=='gitlab'%}.gitlab-ci.yml{%endif%}.jinja => {%if 'Initial CI-CD scripts' in services and ci_provider=='gitlab'%}.gitlab-ci.yml{%endif%}.jinja (100%) create mode 100644 {{ _copier_conf.answers_file }}.jinja diff --git a/copier.yml b/copier.yml index 6564ca2..893e004 100644 --- a/copier.yml +++ b/copier.yml @@ -1,5 +1,7 @@ +_tasks: + - echo 'Hello template' + _exclude: - - copier-answers.yml - ci_requirements.txt - test_load_dagbag.py - copier.yaml @@ -10,61 +12,84 @@ _exclude: - __pycache__ - .DS_Store +services: + type: str + multiselect: true + choices: + - Base dbt project + - dbt profile for automated runs + - Initial CI-CD scripts + - Linting with SQLFluff, dbt-checkpoint and-or YMLLint + - Sample Airflow DAGs + - dbt-coves config and-or templates + help: What services do you want to setup? + +dbt_project_dir: + type: str + when: "{{'Base dbt project' in services}}" + default: "{{ datacoves_env.get('DATACOVES__DBT_HOME', '.') }}" + help: dbt project directory + +dbt_project_name: + type: str + when: "{{'Base dbt project' in services}}" + help: What is the name of the dbt project? + ci_provider: type: str choices: - github - gitlab - when: "{{setup_ci_cd}}" + when: "{{'Initial CI-CD scripts' in services}}" default: github help: CI/CD provider dags_path: type: str - when: "{{airflow_dags_confirm_path}}" + when: "{{'Sample Airflow DAGs' in services and airflow_dags_confirm_path}}" default: "{{tentative_dags_path}}" help: Airflow DAGs path yml_dags_path: type: str - when: "{{yml_dags_confirm_path}}" + when: "{{'Sample Airflow DAGs' in services and yml_dags_confirm_path}}" default: "{{tentative_yml_dags_path}}" help: Airflow YML Dags path use_dbt_checkpoint: type: bool default: false - when: "{{setup_precommit}}" + when: "{{'Linting with SQLFluff, dbt-checkpoint and-or YMLLint' in services}}" help: use dbt-checkpoint? use_sqlfluff: type: bool default: false - when: "{{setup_precommit}}" + when: "{{'Linting with SQLFluff, dbt-checkpoint and-or YMLLint' in services}}" help: use Sqlfluff? use_yamllint: type: bool default: false - when: "{{setup_precommit}}" + when: "{{'Linting with SQLFluff, dbt-checkpoint and-or YMLLint' in services}}" help: use Yamllint? add_dbtcoves_config_and_templates: type: bool default: false - when: "{{not setup_dbt_coves and setup_dbt_project}}" + when: "{{not 'dbt-coves config and-or templates' in services and 'Base dbt project' in services}}" help: Add dbt-coves config and templates? setup_dbtcoves_config: type: bool default: true - when: "{{setup_dbt_coves}}" + when: "{{'dbt-coves config and-or templates' in services}}" help: Setup dbt-coves config? setup_dbtcoves_templates: type: bool default: true - when: "{{setup_dbt_coves}}" + when: "{{'dbt-coves config and-or templates' in services}}" help: Setup dbt-coves templates? dbt_adapter: @@ -75,7 +100,7 @@ dbt_adapter: - redshift - databricks help: What dbt adapter are you using? - when: "{{not datacoves_dbt_adapter and (setup_dbt_profile or setup_ci_cd or setup_sqlfluff)}}" + when: "{{not datacoves_dbt_adapter and ('dbt profile for automated runs' in services or setup_ci_cd or setup_sqlfluff)}}" default: snowflake dbt_checkpoint_version: diff --git a/{%if setup_ci_cd and ci_provider=='github'%}.github{%endif%}/workflows/pull_request_build.yml.jinja b/pull_request_build.yml.jinja similarity index 100% rename from {%if setup_ci_cd and ci_provider=='github'%}.github{%endif%}/workflows/pull_request_build.yml.jinja rename to pull_request_build.yml.jinja diff --git a/{%if setup_ci_cd and ci_provider=='github'%}.github{%endif%}/workflows/push-to-main.yml.jinja b/push-to-main.yml.jinja similarity index 100% rename from {%if setup_ci_cd and ci_provider=='github'%}.github{%endif%}/workflows/push-to-main.yml.jinja rename to push-to-main.yml.jinja diff --git a/workflows/pull_request_build.yml.jinja b/workflows/pull_request_build.yml.jinja new file mode 100644 index 0000000..50cb598 --- /dev/null +++ b/workflows/pull_request_build.yml.jinja @@ -0,0 +1,92 @@ +#### Pre-reqs #### +# Set the correct container based on your data warehouse +# Add all required environment variables/secrets in Github Actions + +name: Test and Check on Pull Request + +on: # yamllint disable-line rule:truthy + pull_request: + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# This cancels a run if another change is pushed to the same branch +concurrency: + {% raw %}group: ${{ github.workflow }}-${{ github.ref }}{%endraw%} + cancel-in-progress: true + +jobs: + dbt: + name: Pull Request dbt Tests + runs-on: ubuntu-latest + + {% if 'snowflake' in dbt_adapter %} + container: datacoves/ci-basic-dbt-snowflake:{{datacoves_env_version}} + {% elif 'redshift' in dbt_adapter %} + container: datacoves/ci-basic-dbt-redshift:{{datacoves_env_version}} + {% elif 'databricks' in dbt_adapter %} + container: datacoves/ci-basic-dbt-databricks:{{datacoves_env_version}} + {% elif 'bigquery' in dbt_adapter %} + container: datacoves/ci-basic-dbt-bigquery:{{datacoves_env_version}} + {% endif %} + + defaults: + run: + {% raw %} working-directory: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{%endraw%}/{{dbt_project_dir}} + + env: + {% raw %}DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{%endraw%}/{{ airflow_profile_path }} + {% raw %}DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{%endraw%}/{{dbt_project_dir}} + + {% if 'snowflake' in dbt_adapter %} + {% raw %}DATACOVES__MAIN__ACCOUNT: ${{ vars.DB_ACCOUNT }} + DATACOVES__PROD__DATABASE: ${{ vars.PROD_DATABASE }} + DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} + DATACOVES__MAIN__ROLE: ${{ vars.DB_ROLE }} + DATACOVES__MAIN__WAREHOUSE: ${{ vars.DB_WAREHOUSE }} + DATACOVES__MAIN__USER: ${{ vars.DB_USER }} + DATACOVES__MAIN__PASSWORD: ${{ secrets.DB_PASSWORD }}{%endraw%} + {% elif 'redshift' in dbt_adapter %} + ##### Redshift ##### + {% raw %}DATACOVES__MAIN__HOST: ${{ vars.DB_HOST }} + DATACOVES__MAIN__USER: ${{ vars.DB_USER }} + DATACOVES__MAIN__PASSWORD: ${{ vars.DB_PASSWORD }} + DATACOVES__MAIN__DATABASE: ${{ vars.PROD_DATABASE }} + DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} + DATACOVES__MAIN__PORT: ${{ vars.DB_PORT }}{%endraw%} + {% elif 'databricks' in dbt_adapter %} + {%raw %}DATACOVES__MAIN__HOST: ${{ vars.DB_HOST }} + DATACOVES__MAIN__HTTP_PATH: ${{ vars.DB_HTTP_PATH }} + DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} + DATACOVES__MAIN__TOKEN: ${{ vars.DB_TOKEN }}{%endraw%} + {% elif 'bigquery' in dbt_adapter %} + {% raw %}DATACOVES__MAIN__KEYFILE_JSON: ${{ vars.DB_KEYFILE_JSON }} + DATACOVES__MAIN__PROJECT: ${{ vars.DB_PROJECT }} + {% endraw %} + {% endif %} + + steps: + - name: Checkout branch + uses: actions/checkout@v4 + with: + fetch-depth: 0 + {% raw %}ref: ${{ github.event.pull_request.head.sha }}{% endraw %} + + - name: Set Secure Directory + {% raw %}run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw %} + + - name: List of files changed + {% raw %}run: "git diff origin/${{ github.event.pull_request.base.ref }} HEAD --name-status"{% endraw %} + + - name: Install dbt packages + run: "dbt deps" + + - name: Run dbt build + run: "dbt build --fail-fast" + + - name: Run governance checks + {% raw %}run: "pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref HEAD"{% endraw %} + + - name: Run governance checks 2 + {% raw %}run: "pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref HEAD"{% endraw %} + diff --git a/workflows/push-to-main.yml.jinja b/workflows/push-to-main.yml.jinja new file mode 100644 index 0000000..1cd531a --- /dev/null +++ b/workflows/push-to-main.yml.jinja @@ -0,0 +1,99 @@ +#### Pre-reqs #### +# Set the correct container based on your data warehouse +# Add all required environment variables/secrets in Github Actions +# In the Datacoves environment settings enable the OBSERVE tab to see production docs +name: Perform deployment to production + +on: # yamllint disable-line rule:truthy + pull_request: + types: + - closed + branches: + - main + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + # Perform the deployment to Prod + build: + # Need to make sure that when the PR was closed, it was actually merged. + if: github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main' + + name: Deployment Script + runs-on: ubuntu-latest + + # Set environment variables in + # https://github.com////settings/variables/actions + # + # Alternatively, You can define multiple ENV for different workflows. + # https://github.com///settings/environments + # environment: PR_ENV + {% if 'snowflake' in dbt_adapter %} + container: datacoves/ci-basic-dbt-snowflake:{{datacoves_env_version}} + {% elif 'redshift' in dbt_adapter %} + container: datacoves/ci-basic-dbt-redshift:{{datacoves_env_version}} + {% elif 'databricks' in dbt_adapter %} + container: datacoves/ci-basic-dbt-databricks:{{datacoves_env_version}} + {% elif 'bigquery' in dbt_adapter %} + container: datacoves/ci-basic-dbt-bigquery:{{datacoves_env_version}} + {% endif %} + defaults: + run: + {% raw %}working-directory: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%}/{{dbt_project_dir}} + + env: + {% raw %}DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%}/{{airflow_profile_path}} + {% raw %}DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%}/{{dbt_project_dir}} + + {% if 'snowflake' in dbt_adapter %} + {% raw %}DATACOVES__MAIN__ACCOUNT: ${{ vars.DB_ACCOUNT }} + DATACOVES__PROD__DATABASE: ${{ vars.PROD_DATABASE }} + DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} + DATACOVES__MAIN__ROLE: ${{ vars.DB_ROLE }} + DATACOVES__MAIN__WAREHOUSE: ${{ vars.DB_WAREHOUSE }} + DATACOVES__MAIN__USER: ${{ vars.DB_USER }} + DATACOVES__MAIN__PASSWORD: ${{ secrets.DB_PASSWORD }}{%endraw%} + {% elif 'redshift' in dbt_adapter %} + ##### Redshift ##### + {% raw %}DATACOVES__MAIN__HOST: ${{ vars.DB_HOST }} + DATACOVES__MAIN__USER: ${{ vars.DB_USER }} + DATACOVES__MAIN__PASSWORD: ${{ vars.DB_PASSWORD }} + DATACOVES__MAIN__DATABASE: ${{ vars.PROD_DATABASE }} + DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} + DATACOVES__MAIN__PORT: ${{ vars.DB_PORT }}{%endraw%} + {% elif 'databricks' in dbt_adapter %} + {%raw %}DATACOVES__MAIN__HOST: ${{ vars.DB_HOST }} + DATACOVES__MAIN__HTTP_PATH: ${{ vars.DB_HTTP_PATH }} + DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} + DATACOVES__MAIN__TOKEN: ${{ vars.DB_TOKEN }}{%endraw%} + {% elif 'bigquery' in dbt_adapter %} + {% raw %}DATACOVES__MAIN__KEYFILE_JSON: ${{ vars.DB_KEYFILE_JSON }} + DATACOVES__MAIN__PROJECT: ${{ vars.DB_PROJECT }} + {% endraw %} + {% endif %} + + steps: + - name: Checkout branch + uses: actions/checkout@v4 + with: + {% raw %}ref: ${{ github.event.push.head.sha }}{% endraw%} + fetch-depth: 0 + + - name: Set Secure Directory + {% raw %}run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%} + + - name: Install dbt packages + run: "dbt deps" + + - name: dbt build + run: "dbt build" + + - name: Generate dbt documentation + run: "dbt docs generate" + + - name: Deploy docs 🚀 + uses: JamesIves/github-pages-deploy-action@4.1.4 + with: + branch: dbt-docs + folder: /target diff --git a/{% if setup_dbt_project %}.gitignore{% endif %} b/{% if 'Base dbt project' in services %}.gitignore{% endif %} similarity index 100% rename from {% if setup_dbt_project %}.gitignore{% endif %} rename to {% if 'Base dbt project' in services %}.gitignore{% endif %} diff --git a/{% if setup_dbt_project %}README.md{% endif %} b/{% if 'Base dbt project' in services %}README.md{% endif %} similarity index 100% rename from {% if setup_dbt_project %}README.md{% endif %} rename to {% if 'Base dbt project' in services %}README.md{% endif %} diff --git a/{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/analyses/.gitkeep b/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/analyses/.gitkeep similarity index 100% rename from {% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/analyses/.gitkeep rename to {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/analyses/.gitkeep diff --git a/{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/dbt_project.yml.jinja b/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/dbt_project.yml.jinja similarity index 100% rename from {% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/dbt_project.yml.jinja rename to {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/dbt_project.yml.jinja diff --git a/{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/macros/.gitkeep b/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/macros/.gitkeep similarity index 100% rename from {% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/macros/.gitkeep rename to {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/macros/.gitkeep diff --git a/{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/.gitkeep b/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/.gitkeep similarity index 100% rename from {% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/.gitkeep rename to {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/.gitkeep diff --git a/{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/my_model.sql b/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/my_model.sql similarity index 100% rename from {% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/my_model.sql rename to {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/my_model.sql diff --git a/{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/my_model.yml b/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/my_model.yml similarity index 100% rename from {% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/my_model.yml rename to {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/my_model.yml diff --git a/{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/source.yml b/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/source.yml similarity index 100% rename from {% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/source.yml rename to {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/source.yml diff --git a/{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/models/L2_core/.gitkeep b/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L2_core/.gitkeep similarity index 100% rename from {% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/models/L2_core/.gitkeep rename to {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L2_core/.gitkeep diff --git a/{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/models/L3_marts/.gitkeep b/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L3_marts/.gitkeep similarity index 100% rename from {% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/models/L3_marts/.gitkeep rename to {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L3_marts/.gitkeep diff --git a/{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/packages.yml b/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/packages.yml similarity index 100% rename from {% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/packages.yml rename to {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/packages.yml diff --git a/{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/seeds/.gitkeep b/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/seeds/.gitkeep similarity index 100% rename from {% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/seeds/.gitkeep rename to {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/seeds/.gitkeep diff --git a/{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/snapshots/.gitkeep b/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/snapshots/.gitkeep similarity index 100% rename from {% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/snapshots/.gitkeep rename to {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/snapshots/.gitkeep diff --git a/{% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/tests/.gitkeep b/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/tests/.gitkeep similarity index 100% rename from {% if setup_dbt_project %}{{ dbt_project_dir }}{%endif%}/tests/.gitkeep rename to {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/tests/.gitkeep diff --git a/{% if setup_precommit %}{{ dbt_project_dir }}{% endif %}/{% if use_sqlfluff %}.sqlfluffignore{% endif %} b/{% if 'Initial CI-CD scripts' in services %}{{ dbt_project_dir }}{% endif %}/{% if use_sqlfluff %}.sqlfluffignore{% endif %} similarity index 100% rename from {% if setup_precommit %}{{ dbt_project_dir }}{% endif %}/{% if use_sqlfluff %}.sqlfluffignore{% endif %} rename to {% if 'Initial CI-CD scripts' in services %}{{ dbt_project_dir }}{% endif %}/{% if use_sqlfluff %}.sqlfluffignore{% endif %} diff --git a/{% if setup_precommit %}{{ dbt_project_dir }}{% endif %}/{% if use_sqlfluff %}.sqlfluff{% endif %}.jinja b/{% if 'Initial CI-CD scripts' in services %}{{ dbt_project_dir }}{% endif %}/{% if use_sqlfluff %}.sqlfluff{% endif %}.jinja similarity index 100% rename from {% if setup_precommit %}{{ dbt_project_dir }}{% endif %}/{% if use_sqlfluff %}.sqlfluff{% endif %}.jinja rename to {% if 'Initial CI-CD scripts' in services %}{{ dbt_project_dir }}{% endif %}/{% if use_sqlfluff %}.sqlfluff{% endif %}.jinja diff --git a/{% if setup_airflow_dag%}{{ dags_path }}{% endif %}/sample_dag.py b/{% if 'Sample Airflow DAGs' in services %}{{ dags_path }}{% endif %}/sample_dag.py similarity index 100% rename from {% if setup_airflow_dag%}{{ dags_path }}{% endif %}/sample_dag.py rename to {% if 'Sample Airflow DAGs' in services %}{{ dags_path }}{% endif %}/sample_dag.py diff --git a/{% if setup_airflow_dag %}{{ yml_dags_path }}{% endif %}/sample_dag.yml b/{% if 'Sample Airflow DAGs' in services %}{{ yml_dags_path }}{% endif %}/sample_dag.yml similarity index 100% rename from {% if setup_airflow_dag %}{{ yml_dags_path }}{% endif %}/sample_dag.yml rename to {% if 'Sample Airflow DAGs' in services %}{{ yml_dags_path }}{% endif %}/sample_dag.yml diff --git a/{% if setup_dbt_profile %}{{ airflow_profile_path }}{%endif%}/profiles.yml.jinja b/{% if 'dbt profile for automated runs' in services %}{{ airflow_profile_path }}{%endif%}/profiles.yml.jinja similarity index 100% rename from {% if setup_dbt_profile %}{{ airflow_profile_path }}{%endif%}/profiles.yml.jinja rename to {% if 'dbt profile for automated runs' in services %}{{ airflow_profile_path }}{%endif%}/profiles.yml.jinja diff --git a/{% if setup_dbt_coves or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_config or add_dbtcoves_config_and_templates %}config.yml{% endif %} b/{% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_config or add_dbtcoves_config_and_templates %}config.yml{% endif %} similarity index 100% rename from {% if setup_dbt_coves or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_config or add_dbtcoves_config_and_templates %}config.yml{% endif %} rename to {% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_config or add_dbtcoves_config_and_templates %}config.yml{% endif %} diff --git a/{% if setup_dbt_coves or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/model_props.yml b/{% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/model_props.yml similarity index 100% rename from {% if setup_dbt_coves or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/model_props.yml rename to {% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/model_props.yml diff --git a/{% if setup_dbt_coves or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/source_props.yml b/{% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/source_props.yml similarity index 100% rename from {% if setup_dbt_coves or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/source_props.yml rename to {% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/source_props.yml diff --git a/{% if setup_dbt_coves or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/staging_model.sql b/{% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/staging_model.sql similarity index 100% rename from {% if setup_dbt_coves or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/staging_model.sql rename to {% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/staging_model.sql diff --git a/{% if setup_dbt_coves or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/staging_model_props.yml b/{% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/staging_model_props.yml similarity index 100% rename from {% if setup_dbt_coves or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/staging_model_props.yml rename to {% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/staging_model_props.yml diff --git a/{%if 'Initial CI-CD scripts' in services and ci_provider=='github'%}.github{%endif%}/workflows/pull_request_build.yml.jinja b/{%if 'Initial CI-CD scripts' in services and ci_provider=='github'%}.github{%endif%}/workflows/pull_request_build.yml.jinja new file mode 100644 index 0000000..50cb598 --- /dev/null +++ b/{%if 'Initial CI-CD scripts' in services and ci_provider=='github'%}.github{%endif%}/workflows/pull_request_build.yml.jinja @@ -0,0 +1,92 @@ +#### Pre-reqs #### +# Set the correct container based on your data warehouse +# Add all required environment variables/secrets in Github Actions + +name: Test and Check on Pull Request + +on: # yamllint disable-line rule:truthy + pull_request: + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# This cancels a run if another change is pushed to the same branch +concurrency: + {% raw %}group: ${{ github.workflow }}-${{ github.ref }}{%endraw%} + cancel-in-progress: true + +jobs: + dbt: + name: Pull Request dbt Tests + runs-on: ubuntu-latest + + {% if 'snowflake' in dbt_adapter %} + container: datacoves/ci-basic-dbt-snowflake:{{datacoves_env_version}} + {% elif 'redshift' in dbt_adapter %} + container: datacoves/ci-basic-dbt-redshift:{{datacoves_env_version}} + {% elif 'databricks' in dbt_adapter %} + container: datacoves/ci-basic-dbt-databricks:{{datacoves_env_version}} + {% elif 'bigquery' in dbt_adapter %} + container: datacoves/ci-basic-dbt-bigquery:{{datacoves_env_version}} + {% endif %} + + defaults: + run: + {% raw %} working-directory: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{%endraw%}/{{dbt_project_dir}} + + env: + {% raw %}DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{%endraw%}/{{ airflow_profile_path }} + {% raw %}DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{%endraw%}/{{dbt_project_dir}} + + {% if 'snowflake' in dbt_adapter %} + {% raw %}DATACOVES__MAIN__ACCOUNT: ${{ vars.DB_ACCOUNT }} + DATACOVES__PROD__DATABASE: ${{ vars.PROD_DATABASE }} + DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} + DATACOVES__MAIN__ROLE: ${{ vars.DB_ROLE }} + DATACOVES__MAIN__WAREHOUSE: ${{ vars.DB_WAREHOUSE }} + DATACOVES__MAIN__USER: ${{ vars.DB_USER }} + DATACOVES__MAIN__PASSWORD: ${{ secrets.DB_PASSWORD }}{%endraw%} + {% elif 'redshift' in dbt_adapter %} + ##### Redshift ##### + {% raw %}DATACOVES__MAIN__HOST: ${{ vars.DB_HOST }} + DATACOVES__MAIN__USER: ${{ vars.DB_USER }} + DATACOVES__MAIN__PASSWORD: ${{ vars.DB_PASSWORD }} + DATACOVES__MAIN__DATABASE: ${{ vars.PROD_DATABASE }} + DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} + DATACOVES__MAIN__PORT: ${{ vars.DB_PORT }}{%endraw%} + {% elif 'databricks' in dbt_adapter %} + {%raw %}DATACOVES__MAIN__HOST: ${{ vars.DB_HOST }} + DATACOVES__MAIN__HTTP_PATH: ${{ vars.DB_HTTP_PATH }} + DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} + DATACOVES__MAIN__TOKEN: ${{ vars.DB_TOKEN }}{%endraw%} + {% elif 'bigquery' in dbt_adapter %} + {% raw %}DATACOVES__MAIN__KEYFILE_JSON: ${{ vars.DB_KEYFILE_JSON }} + DATACOVES__MAIN__PROJECT: ${{ vars.DB_PROJECT }} + {% endraw %} + {% endif %} + + steps: + - name: Checkout branch + uses: actions/checkout@v4 + with: + fetch-depth: 0 + {% raw %}ref: ${{ github.event.pull_request.head.sha }}{% endraw %} + + - name: Set Secure Directory + {% raw %}run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw %} + + - name: List of files changed + {% raw %}run: "git diff origin/${{ github.event.pull_request.base.ref }} HEAD --name-status"{% endraw %} + + - name: Install dbt packages + run: "dbt deps" + + - name: Run dbt build + run: "dbt build --fail-fast" + + - name: Run governance checks + {% raw %}run: "pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref HEAD"{% endraw %} + + - name: Run governance checks 2 + {% raw %}run: "pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref HEAD"{% endraw %} + diff --git a/{%if 'Initial CI-CD scripts' in services and ci_provider=='github'%}.github{%endif%}/workflows/push-to-main.yml.jinja b/{%if 'Initial CI-CD scripts' in services and ci_provider=='github'%}.github{%endif%}/workflows/push-to-main.yml.jinja new file mode 100644 index 0000000..1cd531a --- /dev/null +++ b/{%if 'Initial CI-CD scripts' in services and ci_provider=='github'%}.github{%endif%}/workflows/push-to-main.yml.jinja @@ -0,0 +1,99 @@ +#### Pre-reqs #### +# Set the correct container based on your data warehouse +# Add all required environment variables/secrets in Github Actions +# In the Datacoves environment settings enable the OBSERVE tab to see production docs +name: Perform deployment to production + +on: # yamllint disable-line rule:truthy + pull_request: + types: + - closed + branches: + - main + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + # Perform the deployment to Prod + build: + # Need to make sure that when the PR was closed, it was actually merged. + if: github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main' + + name: Deployment Script + runs-on: ubuntu-latest + + # Set environment variables in + # https://github.com////settings/variables/actions + # + # Alternatively, You can define multiple ENV for different workflows. + # https://github.com///settings/environments + # environment: PR_ENV + {% if 'snowflake' in dbt_adapter %} + container: datacoves/ci-basic-dbt-snowflake:{{datacoves_env_version}} + {% elif 'redshift' in dbt_adapter %} + container: datacoves/ci-basic-dbt-redshift:{{datacoves_env_version}} + {% elif 'databricks' in dbt_adapter %} + container: datacoves/ci-basic-dbt-databricks:{{datacoves_env_version}} + {% elif 'bigquery' in dbt_adapter %} + container: datacoves/ci-basic-dbt-bigquery:{{datacoves_env_version}} + {% endif %} + defaults: + run: + {% raw %}working-directory: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%}/{{dbt_project_dir}} + + env: + {% raw %}DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%}/{{airflow_profile_path}} + {% raw %}DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%}/{{dbt_project_dir}} + + {% if 'snowflake' in dbt_adapter %} + {% raw %}DATACOVES__MAIN__ACCOUNT: ${{ vars.DB_ACCOUNT }} + DATACOVES__PROD__DATABASE: ${{ vars.PROD_DATABASE }} + DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} + DATACOVES__MAIN__ROLE: ${{ vars.DB_ROLE }} + DATACOVES__MAIN__WAREHOUSE: ${{ vars.DB_WAREHOUSE }} + DATACOVES__MAIN__USER: ${{ vars.DB_USER }} + DATACOVES__MAIN__PASSWORD: ${{ secrets.DB_PASSWORD }}{%endraw%} + {% elif 'redshift' in dbt_adapter %} + ##### Redshift ##### + {% raw %}DATACOVES__MAIN__HOST: ${{ vars.DB_HOST }} + DATACOVES__MAIN__USER: ${{ vars.DB_USER }} + DATACOVES__MAIN__PASSWORD: ${{ vars.DB_PASSWORD }} + DATACOVES__MAIN__DATABASE: ${{ vars.PROD_DATABASE }} + DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} + DATACOVES__MAIN__PORT: ${{ vars.DB_PORT }}{%endraw%} + {% elif 'databricks' in dbt_adapter %} + {%raw %}DATACOVES__MAIN__HOST: ${{ vars.DB_HOST }} + DATACOVES__MAIN__HTTP_PATH: ${{ vars.DB_HTTP_PATH }} + DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} + DATACOVES__MAIN__TOKEN: ${{ vars.DB_TOKEN }}{%endraw%} + {% elif 'bigquery' in dbt_adapter %} + {% raw %}DATACOVES__MAIN__KEYFILE_JSON: ${{ vars.DB_KEYFILE_JSON }} + DATACOVES__MAIN__PROJECT: ${{ vars.DB_PROJECT }} + {% endraw %} + {% endif %} + + steps: + - name: Checkout branch + uses: actions/checkout@v4 + with: + {% raw %}ref: ${{ github.event.push.head.sha }}{% endraw%} + fetch-depth: 0 + + - name: Set Secure Directory + {% raw %}run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%} + + - name: Install dbt packages + run: "dbt deps" + + - name: dbt build + run: "dbt build" + + - name: Generate dbt documentation + run: "dbt docs generate" + + - name: Deploy docs 🚀 + uses: JamesIves/github-pages-deploy-action@4.1.4 + with: + branch: dbt-docs + folder: /target diff --git a/{%if setup_ci_cd and ci_provider=='gitlab'%}.gitlab-ci.yml{%endif%}.jinja b/{%if 'Initial CI-CD scripts' in services and ci_provider=='gitlab'%}.gitlab-ci.yml{%endif%}.jinja similarity index 100% rename from {%if setup_ci_cd and ci_provider=='gitlab'%}.gitlab-ci.yml{%endif%}.jinja rename to {%if 'Initial CI-CD scripts' in services and ci_provider=='gitlab'%}.gitlab-ci.yml{%endif%}.jinja diff --git a/{{ _copier_conf.answers_file }}.jinja b/{{ _copier_conf.answers_file }}.jinja new file mode 100644 index 0000000..971b710 --- /dev/null +++ b/{{ _copier_conf.answers_file }}.jinja @@ -0,0 +1,2 @@ +# Changes here will be overwritten by Copier; NEVER EDIT MANUALLY +{{ _copier_answers|to_nice_yaml }} From f7919e2d8afc1f1b10ce9d60d2acadb439c2ad59 Mon Sep 17 00:00:00 2001 From: Bruno Antonellini Date: Wed, 19 Mar 2025 10:27:23 -0300 Subject: [PATCH 02/15] Remove miscommitted workflows --- workflows/pull_request_build.yml.jinja | 92 ------------------------ workflows/push-to-main.yml.jinja | 99 -------------------------- 2 files changed, 191 deletions(-) delete mode 100644 workflows/pull_request_build.yml.jinja delete mode 100644 workflows/push-to-main.yml.jinja diff --git a/workflows/pull_request_build.yml.jinja b/workflows/pull_request_build.yml.jinja deleted file mode 100644 index 50cb598..0000000 --- a/workflows/pull_request_build.yml.jinja +++ /dev/null @@ -1,92 +0,0 @@ -#### Pre-reqs #### -# Set the correct container based on your data warehouse -# Add all required environment variables/secrets in Github Actions - -name: Test and Check on Pull Request - -on: # yamllint disable-line rule:truthy - pull_request: - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# This cancels a run if another change is pushed to the same branch -concurrency: - {% raw %}group: ${{ github.workflow }}-${{ github.ref }}{%endraw%} - cancel-in-progress: true - -jobs: - dbt: - name: Pull Request dbt Tests - runs-on: ubuntu-latest - - {% if 'snowflake' in dbt_adapter %} - container: datacoves/ci-basic-dbt-snowflake:{{datacoves_env_version}} - {% elif 'redshift' in dbt_adapter %} - container: datacoves/ci-basic-dbt-redshift:{{datacoves_env_version}} - {% elif 'databricks' in dbt_adapter %} - container: datacoves/ci-basic-dbt-databricks:{{datacoves_env_version}} - {% elif 'bigquery' in dbt_adapter %} - container: datacoves/ci-basic-dbt-bigquery:{{datacoves_env_version}} - {% endif %} - - defaults: - run: - {% raw %} working-directory: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{%endraw%}/{{dbt_project_dir}} - - env: - {% raw %}DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{%endraw%}/{{ airflow_profile_path }} - {% raw %}DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{%endraw%}/{{dbt_project_dir}} - - {% if 'snowflake' in dbt_adapter %} - {% raw %}DATACOVES__MAIN__ACCOUNT: ${{ vars.DB_ACCOUNT }} - DATACOVES__PROD__DATABASE: ${{ vars.PROD_DATABASE }} - DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} - DATACOVES__MAIN__ROLE: ${{ vars.DB_ROLE }} - DATACOVES__MAIN__WAREHOUSE: ${{ vars.DB_WAREHOUSE }} - DATACOVES__MAIN__USER: ${{ vars.DB_USER }} - DATACOVES__MAIN__PASSWORD: ${{ secrets.DB_PASSWORD }}{%endraw%} - {% elif 'redshift' in dbt_adapter %} - ##### Redshift ##### - {% raw %}DATACOVES__MAIN__HOST: ${{ vars.DB_HOST }} - DATACOVES__MAIN__USER: ${{ vars.DB_USER }} - DATACOVES__MAIN__PASSWORD: ${{ vars.DB_PASSWORD }} - DATACOVES__MAIN__DATABASE: ${{ vars.PROD_DATABASE }} - DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} - DATACOVES__MAIN__PORT: ${{ vars.DB_PORT }}{%endraw%} - {% elif 'databricks' in dbt_adapter %} - {%raw %}DATACOVES__MAIN__HOST: ${{ vars.DB_HOST }} - DATACOVES__MAIN__HTTP_PATH: ${{ vars.DB_HTTP_PATH }} - DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} - DATACOVES__MAIN__TOKEN: ${{ vars.DB_TOKEN }}{%endraw%} - {% elif 'bigquery' in dbt_adapter %} - {% raw %}DATACOVES__MAIN__KEYFILE_JSON: ${{ vars.DB_KEYFILE_JSON }} - DATACOVES__MAIN__PROJECT: ${{ vars.DB_PROJECT }} - {% endraw %} - {% endif %} - - steps: - - name: Checkout branch - uses: actions/checkout@v4 - with: - fetch-depth: 0 - {% raw %}ref: ${{ github.event.pull_request.head.sha }}{% endraw %} - - - name: Set Secure Directory - {% raw %}run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw %} - - - name: List of files changed - {% raw %}run: "git diff origin/${{ github.event.pull_request.base.ref }} HEAD --name-status"{% endraw %} - - - name: Install dbt packages - run: "dbt deps" - - - name: Run dbt build - run: "dbt build --fail-fast" - - - name: Run governance checks - {% raw %}run: "pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref HEAD"{% endraw %} - - - name: Run governance checks 2 - {% raw %}run: "pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref HEAD"{% endraw %} - diff --git a/workflows/push-to-main.yml.jinja b/workflows/push-to-main.yml.jinja deleted file mode 100644 index 1cd531a..0000000 --- a/workflows/push-to-main.yml.jinja +++ /dev/null @@ -1,99 +0,0 @@ -#### Pre-reqs #### -# Set the correct container based on your data warehouse -# Add all required environment variables/secrets in Github Actions -# In the Datacoves environment settings enable the OBSERVE tab to see production docs -name: Perform deployment to production - -on: # yamllint disable-line rule:truthy - pull_request: - types: - - closed - branches: - - main - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -jobs: - # Perform the deployment to Prod - build: - # Need to make sure that when the PR was closed, it was actually merged. - if: github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main' - - name: Deployment Script - runs-on: ubuntu-latest - - # Set environment variables in - # https://github.com////settings/variables/actions - # - # Alternatively, You can define multiple ENV for different workflows. - # https://github.com///settings/environments - # environment: PR_ENV - {% if 'snowflake' in dbt_adapter %} - container: datacoves/ci-basic-dbt-snowflake:{{datacoves_env_version}} - {% elif 'redshift' in dbt_adapter %} - container: datacoves/ci-basic-dbt-redshift:{{datacoves_env_version}} - {% elif 'databricks' in dbt_adapter %} - container: datacoves/ci-basic-dbt-databricks:{{datacoves_env_version}} - {% elif 'bigquery' in dbt_adapter %} - container: datacoves/ci-basic-dbt-bigquery:{{datacoves_env_version}} - {% endif %} - defaults: - run: - {% raw %}working-directory: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%}/{{dbt_project_dir}} - - env: - {% raw %}DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%}/{{airflow_profile_path}} - {% raw %}DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%}/{{dbt_project_dir}} - - {% if 'snowflake' in dbt_adapter %} - {% raw %}DATACOVES__MAIN__ACCOUNT: ${{ vars.DB_ACCOUNT }} - DATACOVES__PROD__DATABASE: ${{ vars.PROD_DATABASE }} - DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} - DATACOVES__MAIN__ROLE: ${{ vars.DB_ROLE }} - DATACOVES__MAIN__WAREHOUSE: ${{ vars.DB_WAREHOUSE }} - DATACOVES__MAIN__USER: ${{ vars.DB_USER }} - DATACOVES__MAIN__PASSWORD: ${{ secrets.DB_PASSWORD }}{%endraw%} - {% elif 'redshift' in dbt_adapter %} - ##### Redshift ##### - {% raw %}DATACOVES__MAIN__HOST: ${{ vars.DB_HOST }} - DATACOVES__MAIN__USER: ${{ vars.DB_USER }} - DATACOVES__MAIN__PASSWORD: ${{ vars.DB_PASSWORD }} - DATACOVES__MAIN__DATABASE: ${{ vars.PROD_DATABASE }} - DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} - DATACOVES__MAIN__PORT: ${{ vars.DB_PORT }}{%endraw%} - {% elif 'databricks' in dbt_adapter %} - {%raw %}DATACOVES__MAIN__HOST: ${{ vars.DB_HOST }} - DATACOVES__MAIN__HTTP_PATH: ${{ vars.DB_HTTP_PATH }} - DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} - DATACOVES__MAIN__TOKEN: ${{ vars.DB_TOKEN }}{%endraw%} - {% elif 'bigquery' in dbt_adapter %} - {% raw %}DATACOVES__MAIN__KEYFILE_JSON: ${{ vars.DB_KEYFILE_JSON }} - DATACOVES__MAIN__PROJECT: ${{ vars.DB_PROJECT }} - {% endraw %} - {% endif %} - - steps: - - name: Checkout branch - uses: actions/checkout@v4 - with: - {% raw %}ref: ${{ github.event.push.head.sha }}{% endraw%} - fetch-depth: 0 - - - name: Set Secure Directory - {% raw %}run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%} - - - name: Install dbt packages - run: "dbt deps" - - - name: dbt build - run: "dbt build" - - - name: Generate dbt documentation - run: "dbt docs generate" - - - name: Deploy docs 🚀 - uses: JamesIves/github-pages-deploy-action@4.1.4 - with: - branch: dbt-docs - folder: /target From b836586f73bcd98ad51e833988a27a621bfe83a1 Mon Sep 17 00:00:00 2001 From: Bruno Antonellini Date: Thu, 20 Mar 2025 09:56:30 -0300 Subject: [PATCH 03/15] Changes after review with Noel --- .github/workflows/main.yml | 2 +- copier-answers.yml | 9 -- copier.yml | 141 +++++++++++------- push-to-main.yml.jinja | 99 ------------ .../sample_dag.py | 0 .../sample_dag.yml | 0 ... use_sqlfluff %}.sqlfluffignore{% endif %} | 0 ... use_sqlfluff %}.sqlfluff{% endif %}.jinja | 0 .../profiles.yml.jinja | 0 ...oject' in services %}.gitignore{% endif %} | 0 ...roject' in services %}README.md{% endif %} | 0 .../analyses/.gitkeep | 0 .../dbt_project.yml.jinja | 0 .../macros/.gitkeep | 0 .../models/L1_staging/.gitkeep | 0 .../models/L1_staging/my_model.sql | 0 .../models/L1_staging/my_model.yml | 0 .../models/L1_staging/source.yml | 0 .../models/L2_core/.gitkeep | 0 .../models/L3_marts/.gitkeep | 0 .../packages.yml | 0 .../seeds/.gitkeep | 0 .../snapshots/.gitkeep | 0 .../tests/.gitkeep | 0 ...nfig_and_templates %}config.yml{% endif %} | 0 .../model_props.yml | 0 .../source_props.yml | 0 .../staging_model.sql | 0 .../staging_model_props.yml | 0 .../workflows/pull_request_build.yml.jinja | 92 ------------ .../workflows/pull_request_build.yml.jinja | 2 +- .../workflows/push-to-main.yml.jinja | 2 +- 32 files changed, 93 insertions(+), 254 deletions(-) delete mode 100644 copier-answers.yml delete mode 100644 push-to-main.yml.jinja rename {{% if 'Sample Airflow DAGs' in services %}{{ dags_path }}{% endif %} => {% if 'setup_airflow_dags' in services %}{{ dags_path }}{% endif %}}/sample_dag.py (100%) rename {{% if 'Sample Airflow DAGs' in services %}{{ yml_dags_path }}{% endif %} => {% if 'setup_airflow_dags' in services %}{{ yml_dags_path }}{% endif %}}/sample_dag.yml (100%) rename {{% if 'Initial CI-CD scripts' in services %}{{ dbt_project_dir }}{% endif %} => {% if 'setup_ci_cd' in services %}{{ dbt_project_dir }}{% endif %}}/{% if use_sqlfluff %}.sqlfluffignore{% endif %} (100%) rename {{% if 'Initial CI-CD scripts' in services %}{{ dbt_project_dir }}{% endif %} => {% if 'setup_ci_cd' in services %}{{ dbt_project_dir }}{% endif %}}/{% if use_sqlfluff %}.sqlfluff{% endif %}.jinja (100%) rename {{% if 'dbt profile for automated runs' in services %}{{ airflow_profile_path }}{%endif%} => {% if 'setup_dbt_profile' in services %}{{ dbt_profiles_path }}{%endif%}}/profiles.yml.jinja (100%) rename {% if 'Base dbt project' in services %}.gitignore{% endif %} => {% if 'setup_dbt_project' in services %}.gitignore{% endif %} (100%) rename {% if 'Base dbt project' in services %}README.md{% endif %} => {% if 'setup_dbt_project' in services %}README.md{% endif %} (100%) rename {{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%} => {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}}/analyses/.gitkeep (100%) rename {{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%} => {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}}/dbt_project.yml.jinja (100%) rename {{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%} => {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}}/macros/.gitkeep (100%) rename {{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%} => {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}}/models/L1_staging/.gitkeep (100%) rename {{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%} => {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}}/models/L1_staging/my_model.sql (100%) rename {{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%} => {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}}/models/L1_staging/my_model.yml (100%) rename {{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%} => {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}}/models/L1_staging/source.yml (100%) rename {{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%} => {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}}/models/L2_core/.gitkeep (100%) rename {{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%} => {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}}/models/L3_marts/.gitkeep (100%) rename {{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%} => {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}}/packages.yml (100%) rename {{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%} => {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}}/seeds/.gitkeep (100%) rename {{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%} => {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}}/snapshots/.gitkeep (100%) rename {{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%} => {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}}/tests/.gitkeep (100%) rename {{% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %} => {% if 'setup_dbtcoves' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}}/.dbt_coves/{% if setup_dbtcoves_config or add_dbtcoves_config_and_templates %}config.yml{% endif %} (100%) rename {{% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %} => {% if 'setup_dbtcoves' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/model_props.yml (100%) rename {{% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %} => {% if 'setup_dbtcoves' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/source_props.yml (100%) rename {{% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %} => {% if 'setup_dbtcoves' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/staging_model.sql (100%) rename {{% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %} => {% if 'setup_dbtcoves' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/staging_model_props.yml (100%) delete mode 100644 {%if 'Initial CI-CD scripts' in services and ci_provider=='github'%}.github{%endif%}/workflows/pull_request_build.yml.jinja rename pull_request_build.yml.jinja => {%if 'setup_ci_cd' in services and ci_provider=='github'%}.github{%endif%}/workflows/pull_request_build.yml.jinja (97%) rename {{%if 'Initial CI-CD scripts' in services and ci_provider=='github'%}.github{%endif%} => {%if 'setup_ci_cd' in services and ci_provider=='github'%}.github{%endif%}}/workflows/push-to-main.yml.jinja (97%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5d80d89..9df97a4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,7 +41,7 @@ jobs: - name: Run Copier run: | - copier --defaults -d setup_dbt_project=True -d setup_dbt_profile=True -d setup_ci_cd=True -d setup_precommit=True -d setup_airflow_dag=True -d dbt_project_dir="." -d dbt_project_name="balboa" -d is_new_project=True -d airflow_profile_path="automate/dbt" -d yml_dags_path="orchestrate/dag_yml_definitions" -d dags_path="orchestrate/dags" -d setup_precommit=True -d use_sqlfluff=True -d use_yamllint=True -d use_dbt_checkpoint=True -a ../copier-answers.yml copy ./ generated/ + copier --defaults -d setup_dbt_project=True -d setup_dbt_profile=True -d setup_ci_cd=True -d setup_precommit=True -d setup_airflow_dag=True -d dbt_project_dir="." -d dbt_project_name="balboa" -d is_new_project=True -d dbt_profiles_path="automate/dbt" -d yml_dags_path="orchestrate/dag_yml_definitions" -d dags_path="orchestrate/dags" -d setup_precommit=True -d use_sqlfluff=True -d use_yamllint=True -d use_dbt_checkpoint=True -a ../copier-answers.yml copy ./ generated/ # run dbt debug and dbt compile - name: Run dbt commands diff --git a/copier-answers.yml b/copier-answers.yml deleted file mode 100644 index c270f93..0000000 --- a/copier-answers.yml +++ /dev/null @@ -1,9 +0,0 @@ -ci_provider: github -dags_path: orchestrate/dags -dbt_adapter: snowflake -use_dbt_checkpoint: true -use_sqlfluff: true -use_yamllint: true -yml_dags_path: orchestrate/dag_yml_definitions -setup_dbtcoves_config: true -setup_dbtcoves_templates: true diff --git a/copier.yml b/copier.yml index 893e004..3d2704b 100644 --- a/copier.yml +++ b/copier.yml @@ -1,5 +1,18 @@ _tasks: - echo 'Hello template' + # - command: ["git init"] + # only run on project setup + # when: "{{ _copier_operation == 'copy' }}" + +_message_before_copy: | + Thanks for generating a project using our template. + +_message_after_copy: | + Your project "{{ dbt_project_name }}" has been created successfully! + +_message_after_update: | + Your project "{{ dbt_project_name }}" has been updated successfully! + In case there _exclude: - ci_requirements.txt @@ -15,83 +28,113 @@ _exclude: services: type: str multiselect: true - choices: - - Base dbt project - - dbt profile for automated runs - - Initial CI-CD scripts - - Linting with SQLFluff, dbt-checkpoint and-or YMLLint - - Sample Airflow DAGs - - dbt-coves config and-or templates help: What services do you want to setup? - + choices: + Base dbt project: setup_dbt_project + dbt profile for automated runs: setup_dbt_profile + setup_ci_cd: setup_ci_cd + Linting with SQLFluff, dbt-checkpoint and-or YMLLint: setup_precommit + Sample Airflow DAGs: setup_airflow_dags + dbt-coves config and-or templates: setup_dbtcoves + +# dbt project related vars dbt_project_dir: type: str - when: "{{'Base dbt project' in services}}" + when: "{{'setup_dbt_project' in services}}" default: "{{ datacoves_env.get('DATACOVES__DBT_HOME', '.') }}" help: dbt project directory dbt_project_name: type: str - when: "{{'Base dbt project' in services}}" + when: "{{'setup_dbt_project' in services}}" help: What is the name of the dbt project? -ci_provider: - type: str - choices: - - github - - gitlab - when: "{{'Initial CI-CD scripts' in services}}" - default: github - help: CI/CD provider - +# Airflow related vars dags_path: type: str - when: "{{'Sample Airflow DAGs' in services and airflow_dags_confirm_path}}" - default: "{{tentative_dags_path}}" + when: "{{'setup_airflow_dags' in services or 'setup_ci_cd' in services}}" + default: "{{ datacoves_env.get('DATACOVES__AIRFLOW_DAGS_PATH', 'orchestrate/dags') }}" help: Airflow DAGs path +airflow_artifacts_path: + type: str + when: "{{'setup_ci_cd' in services}}" + default: "{{ datacoves_env.get('DATACOVES__AIRFLOW_DAGS_PATH', 'orchestrate/dags').split('/')[0] }}" + help: Airflow Artifacts path for unit tests + yml_dags_path: type: str - when: "{{'Sample Airflow DAGs' in services and yml_dags_confirm_path}}" - default: "{{tentative_yml_dags_path}}" + when: "{{'setup_airflow_dags' in services and yml_dags_confirm_path}}" + default: "{{datacoves_env.get('DATACOVES__AIRFLOW_DAGS_YML_PATH', 'orchestrate/dags_yml_definitions')}}" help: Airflow YML Dags path +# dbt-coves related vars +add_dbtcoves_config_and_templates: + type: bool + default: false + when: "{{not 'setup_dbtcoves' in services and 'setup_dbt_project' in services}}" + help: Add dbt-coves config and templates? + +setup_dbtcoves_config: + type: bool + default: true + when: "{{'setup_dbtcoves' in services}}" + help: Setup dbt-coves config? + +setup_dbtcoves_templates: + type: bool + default: true + when: "{{'setup_dbtcoves' in services}}" + help: Setup dbt-coves templates? + +# CI / CD Related vars + +ci_provider: + type: str + choices: + - github + - gitlab + when: "{{'setup_ci_cd' in services}}" + default: github + help: CI/CD provider + use_dbt_checkpoint: type: bool default: false - when: "{{'Linting with SQLFluff, dbt-checkpoint and-or YMLLint' in services}}" + when: "{{'setup_precommit' in services}}" help: use dbt-checkpoint? use_sqlfluff: type: bool default: false - when: "{{'Linting with SQLFluff, dbt-checkpoint and-or YMLLint' in services}}" + when: "{{'setup_precommit' in services}}" help: use Sqlfluff? use_yamllint: type: bool default: false - when: "{{'Linting with SQLFluff, dbt-checkpoint and-or YMLLint' in services}}" + when: "{{'setup_precommit' in services}}" help: use Yamllint? -add_dbtcoves_config_and_templates: - type: bool - default: false - when: "{{not 'dbt-coves config and-or templates' in services and 'Base dbt project' in services}}" - help: Add dbt-coves config and templates? +dbt_checkpoint_version: + type: str + help: What dbt-checkpoint version are you using? + default: "{{ datacoves_env.get('DATACOVES__DBT_CHECKPOINT_VERSION', 'v2.0.6') }}" + when: "{{use_dbt_checkpoint}}" -setup_dbtcoves_config: - type: bool - default: true - when: "{{'dbt-coves config and-or templates' in services}}" - help: Setup dbt-coves config? +yamllint_version: + type: str + help: What yamllint version are you using? + default: "{{ datacoves_env.get('DATACOVES__YAMLLINT_VERSION', '1.36.2') }}" + when: "{{use_yamllint}}" -setup_dbtcoves_templates: - type: bool - default: true - when: "{{'dbt-coves config and-or templates' in services}}" - help: Setup dbt-coves templates? +sqlfluff_version: + type: str + help: What sqlfluff version are you using? + default: "{{ datacoves_env.get('DATACOVES__SQLFLUFF_VERSION', '3.1.1') }}" + when: "{{use_sqlfluff}}" +# dbt related vars dbt_adapter: type: str choices: @@ -100,15 +143,11 @@ dbt_adapter: - redshift - databricks help: What dbt adapter are you using? - when: "{{not datacoves_dbt_adapter and ('dbt profile for automated runs' in services or setup_ci_cd or setup_sqlfluff)}}" - default: snowflake - -dbt_checkpoint_version: - type: str - help: What dbt-checkpoint version are you using? - when: "{{use_dbt_checkpoint and ask_dbt_checkpoint_version}}" + when: "{{not datacoves_dbt_adapter and ('setup_dbt_profile' in services or setup_ci_cd or setup_sqlfluff)}}" + default: "{{ datacoves_env.get('DATACOVES__DBT_ADAPTER', 'snowflake') }}" -yamllint_version: +dbt_profiles_path: type: str - help: What yamllint version are you using? - when: "{{use_yamllint and ask_yamllint_version}}" + when: "{{ 'setup_dbt_profile' in services or 'setup_ci_cd' in services }}" + default: "{{ datacoves_env.get('DATACOVES__AIRFLOW_DBT_PROFILE_PATH', 'automate/dbt') }}" + help: dbt Profile Path for automated runs diff --git a/push-to-main.yml.jinja b/push-to-main.yml.jinja deleted file mode 100644 index 1cd531a..0000000 --- a/push-to-main.yml.jinja +++ /dev/null @@ -1,99 +0,0 @@ -#### Pre-reqs #### -# Set the correct container based on your data warehouse -# Add all required environment variables/secrets in Github Actions -# In the Datacoves environment settings enable the OBSERVE tab to see production docs -name: Perform deployment to production - -on: # yamllint disable-line rule:truthy - pull_request: - types: - - closed - branches: - - main - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -jobs: - # Perform the deployment to Prod - build: - # Need to make sure that when the PR was closed, it was actually merged. - if: github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main' - - name: Deployment Script - runs-on: ubuntu-latest - - # Set environment variables in - # https://github.com////settings/variables/actions - # - # Alternatively, You can define multiple ENV for different workflows. - # https://github.com///settings/environments - # environment: PR_ENV - {% if 'snowflake' in dbt_adapter %} - container: datacoves/ci-basic-dbt-snowflake:{{datacoves_env_version}} - {% elif 'redshift' in dbt_adapter %} - container: datacoves/ci-basic-dbt-redshift:{{datacoves_env_version}} - {% elif 'databricks' in dbt_adapter %} - container: datacoves/ci-basic-dbt-databricks:{{datacoves_env_version}} - {% elif 'bigquery' in dbt_adapter %} - container: datacoves/ci-basic-dbt-bigquery:{{datacoves_env_version}} - {% endif %} - defaults: - run: - {% raw %}working-directory: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%}/{{dbt_project_dir}} - - env: - {% raw %}DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%}/{{airflow_profile_path}} - {% raw %}DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%}/{{dbt_project_dir}} - - {% if 'snowflake' in dbt_adapter %} - {% raw %}DATACOVES__MAIN__ACCOUNT: ${{ vars.DB_ACCOUNT }} - DATACOVES__PROD__DATABASE: ${{ vars.PROD_DATABASE }} - DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} - DATACOVES__MAIN__ROLE: ${{ vars.DB_ROLE }} - DATACOVES__MAIN__WAREHOUSE: ${{ vars.DB_WAREHOUSE }} - DATACOVES__MAIN__USER: ${{ vars.DB_USER }} - DATACOVES__MAIN__PASSWORD: ${{ secrets.DB_PASSWORD }}{%endraw%} - {% elif 'redshift' in dbt_adapter %} - ##### Redshift ##### - {% raw %}DATACOVES__MAIN__HOST: ${{ vars.DB_HOST }} - DATACOVES__MAIN__USER: ${{ vars.DB_USER }} - DATACOVES__MAIN__PASSWORD: ${{ vars.DB_PASSWORD }} - DATACOVES__MAIN__DATABASE: ${{ vars.PROD_DATABASE }} - DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} - DATACOVES__MAIN__PORT: ${{ vars.DB_PORT }}{%endraw%} - {% elif 'databricks' in dbt_adapter %} - {%raw %}DATACOVES__MAIN__HOST: ${{ vars.DB_HOST }} - DATACOVES__MAIN__HTTP_PATH: ${{ vars.DB_HTTP_PATH }} - DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} - DATACOVES__MAIN__TOKEN: ${{ vars.DB_TOKEN }}{%endraw%} - {% elif 'bigquery' in dbt_adapter %} - {% raw %}DATACOVES__MAIN__KEYFILE_JSON: ${{ vars.DB_KEYFILE_JSON }} - DATACOVES__MAIN__PROJECT: ${{ vars.DB_PROJECT }} - {% endraw %} - {% endif %} - - steps: - - name: Checkout branch - uses: actions/checkout@v4 - with: - {% raw %}ref: ${{ github.event.push.head.sha }}{% endraw%} - fetch-depth: 0 - - - name: Set Secure Directory - {% raw %}run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%} - - - name: Install dbt packages - run: "dbt deps" - - - name: dbt build - run: "dbt build" - - - name: Generate dbt documentation - run: "dbt docs generate" - - - name: Deploy docs 🚀 - uses: JamesIves/github-pages-deploy-action@4.1.4 - with: - branch: dbt-docs - folder: /target diff --git a/{% if 'Sample Airflow DAGs' in services %}{{ dags_path }}{% endif %}/sample_dag.py b/{% if 'setup_airflow_dags' in services %}{{ dags_path }}{% endif %}/sample_dag.py similarity index 100% rename from {% if 'Sample Airflow DAGs' in services %}{{ dags_path }}{% endif %}/sample_dag.py rename to {% if 'setup_airflow_dags' in services %}{{ dags_path }}{% endif %}/sample_dag.py diff --git a/{% if 'Sample Airflow DAGs' in services %}{{ yml_dags_path }}{% endif %}/sample_dag.yml b/{% if 'setup_airflow_dags' in services %}{{ yml_dags_path }}{% endif %}/sample_dag.yml similarity index 100% rename from {% if 'Sample Airflow DAGs' in services %}{{ yml_dags_path }}{% endif %}/sample_dag.yml rename to {% if 'setup_airflow_dags' in services %}{{ yml_dags_path }}{% endif %}/sample_dag.yml diff --git a/{% if 'Initial CI-CD scripts' in services %}{{ dbt_project_dir }}{% endif %}/{% if use_sqlfluff %}.sqlfluffignore{% endif %} b/{% if 'setup_ci_cd' in services %}{{ dbt_project_dir }}{% endif %}/{% if use_sqlfluff %}.sqlfluffignore{% endif %} similarity index 100% rename from {% if 'Initial CI-CD scripts' in services %}{{ dbt_project_dir }}{% endif %}/{% if use_sqlfluff %}.sqlfluffignore{% endif %} rename to {% if 'setup_ci_cd' in services %}{{ dbt_project_dir }}{% endif %}/{% if use_sqlfluff %}.sqlfluffignore{% endif %} diff --git a/{% if 'Initial CI-CD scripts' in services %}{{ dbt_project_dir }}{% endif %}/{% if use_sqlfluff %}.sqlfluff{% endif %}.jinja b/{% if 'setup_ci_cd' in services %}{{ dbt_project_dir }}{% endif %}/{% if use_sqlfluff %}.sqlfluff{% endif %}.jinja similarity index 100% rename from {% if 'Initial CI-CD scripts' in services %}{{ dbt_project_dir }}{% endif %}/{% if use_sqlfluff %}.sqlfluff{% endif %}.jinja rename to {% if 'setup_ci_cd' in services %}{{ dbt_project_dir }}{% endif %}/{% if use_sqlfluff %}.sqlfluff{% endif %}.jinja diff --git a/{% if 'dbt profile for automated runs' in services %}{{ airflow_profile_path }}{%endif%}/profiles.yml.jinja b/{% if 'setup_dbt_profile' in services %}{{ dbt_profiles_path }}{%endif%}/profiles.yml.jinja similarity index 100% rename from {% if 'dbt profile for automated runs' in services %}{{ airflow_profile_path }}{%endif%}/profiles.yml.jinja rename to {% if 'setup_dbt_profile' in services %}{{ dbt_profiles_path }}{%endif%}/profiles.yml.jinja diff --git a/{% if 'Base dbt project' in services %}.gitignore{% endif %} b/{% if 'setup_dbt_project' in services %}.gitignore{% endif %} similarity index 100% rename from {% if 'Base dbt project' in services %}.gitignore{% endif %} rename to {% if 'setup_dbt_project' in services %}.gitignore{% endif %} diff --git a/{% if 'Base dbt project' in services %}README.md{% endif %} b/{% if 'setup_dbt_project' in services %}README.md{% endif %} similarity index 100% rename from {% if 'Base dbt project' in services %}README.md{% endif %} rename to {% if 'setup_dbt_project' in services %}README.md{% endif %} diff --git a/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/analyses/.gitkeep b/{% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/analyses/.gitkeep similarity index 100% rename from {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/analyses/.gitkeep rename to {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/analyses/.gitkeep diff --git a/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/dbt_project.yml.jinja b/{% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/dbt_project.yml.jinja similarity index 100% rename from {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/dbt_project.yml.jinja rename to {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/dbt_project.yml.jinja diff --git a/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/macros/.gitkeep b/{% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/macros/.gitkeep similarity index 100% rename from {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/macros/.gitkeep rename to {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/macros/.gitkeep diff --git a/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/.gitkeep b/{% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/.gitkeep similarity index 100% rename from {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/.gitkeep rename to {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/.gitkeep diff --git a/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/my_model.sql b/{% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/my_model.sql similarity index 100% rename from {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/my_model.sql rename to {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/my_model.sql diff --git a/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/my_model.yml b/{% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/my_model.yml similarity index 100% rename from {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/my_model.yml rename to {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/my_model.yml diff --git a/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/source.yml b/{% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/source.yml similarity index 100% rename from {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/source.yml rename to {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/models/L1_staging/source.yml diff --git a/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L2_core/.gitkeep b/{% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/models/L2_core/.gitkeep similarity index 100% rename from {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L2_core/.gitkeep rename to {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/models/L2_core/.gitkeep diff --git a/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L3_marts/.gitkeep b/{% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/models/L3_marts/.gitkeep similarity index 100% rename from {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/models/L3_marts/.gitkeep rename to {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/models/L3_marts/.gitkeep diff --git a/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/packages.yml b/{% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/packages.yml similarity index 100% rename from {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/packages.yml rename to {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/packages.yml diff --git a/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/seeds/.gitkeep b/{% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/seeds/.gitkeep similarity index 100% rename from {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/seeds/.gitkeep rename to {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/seeds/.gitkeep diff --git a/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/snapshots/.gitkeep b/{% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/snapshots/.gitkeep similarity index 100% rename from {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/snapshots/.gitkeep rename to {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/snapshots/.gitkeep diff --git a/{% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/tests/.gitkeep b/{% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/tests/.gitkeep similarity index 100% rename from {% if 'Base dbt project' in services %}{{ dbt_project_dir }}{%endif%}/tests/.gitkeep rename to {% if 'setup_dbt_project' in services %}{{ dbt_project_dir }}{%endif%}/tests/.gitkeep diff --git a/{% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_config or add_dbtcoves_config_and_templates %}config.yml{% endif %} b/{% if 'setup_dbtcoves' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_config or add_dbtcoves_config_and_templates %}config.yml{% endif %} similarity index 100% rename from {% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_config or add_dbtcoves_config_and_templates %}config.yml{% endif %} rename to {% if 'setup_dbtcoves' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_config or add_dbtcoves_config_and_templates %}config.yml{% endif %} diff --git a/{% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/model_props.yml b/{% if 'setup_dbtcoves' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/model_props.yml similarity index 100% rename from {% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/model_props.yml rename to {% if 'setup_dbtcoves' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/model_props.yml diff --git a/{% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/source_props.yml b/{% if 'setup_dbtcoves' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/source_props.yml similarity index 100% rename from {% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/source_props.yml rename to {% if 'setup_dbtcoves' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/source_props.yml diff --git a/{% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/staging_model.sql b/{% if 'setup_dbtcoves' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/staging_model.sql similarity index 100% rename from {% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/staging_model.sql rename to {% if 'setup_dbtcoves' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/staging_model.sql diff --git a/{% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/staging_model_props.yml b/{% if 'setup_dbtcoves' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/staging_model_props.yml similarity index 100% rename from {% if 'dbt-coves config and-or templates' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/staging_model_props.yml rename to {% if 'setup_dbtcoves' in services or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates %}templates{% endif %}/staging_model_props.yml diff --git a/{%if 'Initial CI-CD scripts' in services and ci_provider=='github'%}.github{%endif%}/workflows/pull_request_build.yml.jinja b/{%if 'Initial CI-CD scripts' in services and ci_provider=='github'%}.github{%endif%}/workflows/pull_request_build.yml.jinja deleted file mode 100644 index 50cb598..0000000 --- a/{%if 'Initial CI-CD scripts' in services and ci_provider=='github'%}.github{%endif%}/workflows/pull_request_build.yml.jinja +++ /dev/null @@ -1,92 +0,0 @@ -#### Pre-reqs #### -# Set the correct container based on your data warehouse -# Add all required environment variables/secrets in Github Actions - -name: Test and Check on Pull Request - -on: # yamllint disable-line rule:truthy - pull_request: - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# This cancels a run if another change is pushed to the same branch -concurrency: - {% raw %}group: ${{ github.workflow }}-${{ github.ref }}{%endraw%} - cancel-in-progress: true - -jobs: - dbt: - name: Pull Request dbt Tests - runs-on: ubuntu-latest - - {% if 'snowflake' in dbt_adapter %} - container: datacoves/ci-basic-dbt-snowflake:{{datacoves_env_version}} - {% elif 'redshift' in dbt_adapter %} - container: datacoves/ci-basic-dbt-redshift:{{datacoves_env_version}} - {% elif 'databricks' in dbt_adapter %} - container: datacoves/ci-basic-dbt-databricks:{{datacoves_env_version}} - {% elif 'bigquery' in dbt_adapter %} - container: datacoves/ci-basic-dbt-bigquery:{{datacoves_env_version}} - {% endif %} - - defaults: - run: - {% raw %} working-directory: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{%endraw%}/{{dbt_project_dir}} - - env: - {% raw %}DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{%endraw%}/{{ airflow_profile_path }} - {% raw %}DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{%endraw%}/{{dbt_project_dir}} - - {% if 'snowflake' in dbt_adapter %} - {% raw %}DATACOVES__MAIN__ACCOUNT: ${{ vars.DB_ACCOUNT }} - DATACOVES__PROD__DATABASE: ${{ vars.PROD_DATABASE }} - DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} - DATACOVES__MAIN__ROLE: ${{ vars.DB_ROLE }} - DATACOVES__MAIN__WAREHOUSE: ${{ vars.DB_WAREHOUSE }} - DATACOVES__MAIN__USER: ${{ vars.DB_USER }} - DATACOVES__MAIN__PASSWORD: ${{ secrets.DB_PASSWORD }}{%endraw%} - {% elif 'redshift' in dbt_adapter %} - ##### Redshift ##### - {% raw %}DATACOVES__MAIN__HOST: ${{ vars.DB_HOST }} - DATACOVES__MAIN__USER: ${{ vars.DB_USER }} - DATACOVES__MAIN__PASSWORD: ${{ vars.DB_PASSWORD }} - DATACOVES__MAIN__DATABASE: ${{ vars.PROD_DATABASE }} - DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} - DATACOVES__MAIN__PORT: ${{ vars.DB_PORT }}{%endraw%} - {% elif 'databricks' in dbt_adapter %} - {%raw %}DATACOVES__MAIN__HOST: ${{ vars.DB_HOST }} - DATACOVES__MAIN__HTTP_PATH: ${{ vars.DB_HTTP_PATH }} - DATACOVES__MAIN__SCHEMA: ${{ vars.DB_SCHEMA }} - DATACOVES__MAIN__TOKEN: ${{ vars.DB_TOKEN }}{%endraw%} - {% elif 'bigquery' in dbt_adapter %} - {% raw %}DATACOVES__MAIN__KEYFILE_JSON: ${{ vars.DB_KEYFILE_JSON }} - DATACOVES__MAIN__PROJECT: ${{ vars.DB_PROJECT }} - {% endraw %} - {% endif %} - - steps: - - name: Checkout branch - uses: actions/checkout@v4 - with: - fetch-depth: 0 - {% raw %}ref: ${{ github.event.pull_request.head.sha }}{% endraw %} - - - name: Set Secure Directory - {% raw %}run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw %} - - - name: List of files changed - {% raw %}run: "git diff origin/${{ github.event.pull_request.base.ref }} HEAD --name-status"{% endraw %} - - - name: Install dbt packages - run: "dbt deps" - - - name: Run dbt build - run: "dbt build --fail-fast" - - - name: Run governance checks - {% raw %}run: "pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref HEAD"{% endraw %} - - - name: Run governance checks 2 - {% raw %}run: "pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref HEAD"{% endraw %} - diff --git a/pull_request_build.yml.jinja b/{%if 'setup_ci_cd' in services and ci_provider=='github'%}.github{%endif%}/workflows/pull_request_build.yml.jinja similarity index 97% rename from pull_request_build.yml.jinja rename to {%if 'setup_ci_cd' in services and ci_provider=='github'%}.github{%endif%}/workflows/pull_request_build.yml.jinja index 50cb598..0d7d9ca 100644 --- a/pull_request_build.yml.jinja +++ b/{%if 'setup_ci_cd' in services and ci_provider=='github'%}.github{%endif%}/workflows/pull_request_build.yml.jinja @@ -35,7 +35,7 @@ jobs: {% raw %} working-directory: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{%endraw%}/{{dbt_project_dir}} env: - {% raw %}DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{%endraw%}/{{ airflow_profile_path }} + {% raw %}DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{%endraw%}/{{ dbt_profiles_path }} {% raw %}DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{%endraw%}/{{dbt_project_dir}} {% if 'snowflake' in dbt_adapter %} diff --git a/{%if 'Initial CI-CD scripts' in services and ci_provider=='github'%}.github{%endif%}/workflows/push-to-main.yml.jinja b/{%if 'setup_ci_cd' in services and ci_provider=='github'%}.github{%endif%}/workflows/push-to-main.yml.jinja similarity index 97% rename from {%if 'Initial CI-CD scripts' in services and ci_provider=='github'%}.github{%endif%}/workflows/push-to-main.yml.jinja rename to {%if 'setup_ci_cd' in services and ci_provider=='github'%}.github{%endif%}/workflows/push-to-main.yml.jinja index 1cd531a..53e8378 100644 --- a/{%if 'Initial CI-CD scripts' in services and ci_provider=='github'%}.github{%endif%}/workflows/push-to-main.yml.jinja +++ b/{%if 'setup_ci_cd' in services and ci_provider=='github'%}.github{%endif%}/workflows/push-to-main.yml.jinja @@ -43,7 +43,7 @@ jobs: {% raw %}working-directory: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%}/{{dbt_project_dir}} env: - {% raw %}DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%}/{{airflow_profile_path}} + {% raw %}DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%}/{{dbt_profiles_path}} {% raw %}DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%}/{{dbt_project_dir}} {% if 'snowflake' in dbt_adapter %} From 0fcb8db9937a4e8588bdb28922035040964917b9 Mon Sep 17 00:00:00 2001 From: Bruno Antonellini Date: Thu, 20 Mar 2025 10:11:14 -0300 Subject: [PATCH 04/15] Add setup repo --- copier.yml | 12 ++++++++++++ ...kpoint %}.pre-commit-config.yaml{% endif %}.jinja | 8 ++++++++ 2 files changed, 20 insertions(+) diff --git a/copier.yml b/copier.yml index 3d2704b..e6b061a 100644 --- a/copier.yml +++ b/copier.yml @@ -116,6 +116,12 @@ use_yamllint: when: "{{'setup_precommit' in services}}" help: use Yamllint? +use_newstuff: + type: bool + default: false + when: "{{'setup_precommit' in services}}" + help: use new stuff? + dbt_checkpoint_version: type: str help: What dbt-checkpoint version are you using? @@ -134,6 +140,12 @@ sqlfluff_version: default: "{{ datacoves_env.get('DATACOVES__SQLFLUFF_VERSION', '3.1.1') }}" when: "{{use_sqlfluff}}" +newstuff_version: + type: str + help: What new stuff version are you using? + default: "{{ datacoves_env.get('DATACOVES__NEWSTUFF_VERSION', '0.0.1') }}" + when: "{{use_newstuff}}" + # dbt related vars dbt_adapter: type: str diff --git a/{% if use_sqlfluff or use_yamllint or use_dbt_checkpoint %}.pre-commit-config.yaml{% endif %}.jinja b/{% if use_sqlfluff or use_yamllint or use_dbt_checkpoint %}.pre-commit-config.yaml{% endif %}.jinja index ac2afc2..f0cc0d7 100644 --- a/{% if use_sqlfluff or use_yamllint or use_dbt_checkpoint %}.pre-commit-config.yaml{% endif %}.jinja +++ b/{% if use_sqlfluff or use_yamllint or use_dbt_checkpoint %}.pre-commit-config.yaml{% endif %}.jinja @@ -40,4 +40,12 @@ repos: args: [-c=.yamllint] exclude: ^{{ dbt_project_dir }}/.dbt_coves/templates +{% endif %} +{% if use_newstuff %} + - repo: https://github.com/datacoves/some_new_library.git + rev: {{newstuff_version}} + hooks: + - id: yamllint + args: [-c=.yamllint] + exclude: ^{{ dbt_project_dir }}/.dbt_coves/templates {% endif %} \ No newline at end of file From 035b7ff2b1845866d1cca69d0ec8bc5ac67cc9aa Mon Sep 17 00:00:00 2001 From: Bruno Antonellini Date: Wed, 2 Apr 2025 11:09:23 -0300 Subject: [PATCH 05/15] Attempt Fix CI --- .github/workflows/main.yml | 19 ++++++++++++++++++- copier.yml | 12 ------------ ...%}.pre-commit-config.yaml{% endif %}.jinja | 8 -------- 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9df97a4..b45e77a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,7 +41,24 @@ jobs: - name: Run Copier run: | - copier --defaults -d setup_dbt_project=True -d setup_dbt_profile=True -d setup_ci_cd=True -d setup_precommit=True -d setup_airflow_dag=True -d dbt_project_dir="." -d dbt_project_name="balboa" -d is_new_project=True -d dbt_profiles_path="automate/dbt" -d yml_dags_path="orchestrate/dag_yml_definitions" -d dags_path="orchestrate/dags" -d setup_precommit=True -d use_sqlfluff=True -d use_yamllint=True -d use_dbt_checkpoint=True -a ../copier-answers.yml copy ./ generated/ + copier --defaults \ + -d setup_dbt_project=True \ + -d setup_dbt_profile=True \ + -d setup_ci_cd=True \ + -d setup_precommit=True \ + -d setup_airflow_dag=True \ + -d dbt_project_dir="." \ + -d dbt_project_name="balboa" \ + -d is_new_project=True \ + -d dbt_profiles_path="automate/dbt" \ + -d yml_dags_path="orchestrate/dag_yml_definitions" \ + -d dags_path="orchestrate/dags" \ + -d setup_precommit=True \ + -d use_sqlfluff=True \ + -d use_yamllint=True \ + -d use_dbt_checkpoint=True \ + -d datacoves_env="{ 'DATACOVES__AIRFLOW_DBT_PROFILE_PATH': 'automate/dbt', 'DATACOVES__DBT_HOME': '.', 'DATACOVES__AIRFLOW_DAGS_PATH': 'orchestrate/dags', 'DATACOVES__AIRFLOW_DAGS_YML_PATH': 'orchestrate/dags_yml_definitions' }" \ + -a ../copier-answers.yml copy ./ generated/ # run dbt debug and dbt compile - name: Run dbt commands diff --git a/copier.yml b/copier.yml index e6b061a..3d2704b 100644 --- a/copier.yml +++ b/copier.yml @@ -116,12 +116,6 @@ use_yamllint: when: "{{'setup_precommit' in services}}" help: use Yamllint? -use_newstuff: - type: bool - default: false - when: "{{'setup_precommit' in services}}" - help: use new stuff? - dbt_checkpoint_version: type: str help: What dbt-checkpoint version are you using? @@ -140,12 +134,6 @@ sqlfluff_version: default: "{{ datacoves_env.get('DATACOVES__SQLFLUFF_VERSION', '3.1.1') }}" when: "{{use_sqlfluff}}" -newstuff_version: - type: str - help: What new stuff version are you using? - default: "{{ datacoves_env.get('DATACOVES__NEWSTUFF_VERSION', '0.0.1') }}" - when: "{{use_newstuff}}" - # dbt related vars dbt_adapter: type: str diff --git a/{% if use_sqlfluff or use_yamllint or use_dbt_checkpoint %}.pre-commit-config.yaml{% endif %}.jinja b/{% if use_sqlfluff or use_yamllint or use_dbt_checkpoint %}.pre-commit-config.yaml{% endif %}.jinja index f0cc0d7..ac2afc2 100644 --- a/{% if use_sqlfluff or use_yamllint or use_dbt_checkpoint %}.pre-commit-config.yaml{% endif %}.jinja +++ b/{% if use_sqlfluff or use_yamllint or use_dbt_checkpoint %}.pre-commit-config.yaml{% endif %}.jinja @@ -40,12 +40,4 @@ repos: args: [-c=.yamllint] exclude: ^{{ dbt_project_dir }}/.dbt_coves/templates -{% endif %} -{% if use_newstuff %} - - repo: https://github.com/datacoves/some_new_library.git - rev: {{newstuff_version}} - hooks: - - id: yamllint - args: [-c=.yamllint] - exclude: ^{{ dbt_project_dir }}/.dbt_coves/templates {% endif %} \ No newline at end of file From 06d8b7c1408f710d1edf503e6aa8a767ebdc5c65 Mon Sep 17 00:00:00 2001 From: Bruno Antonellini Date: Wed, 2 Apr 2025 11:29:21 -0300 Subject: [PATCH 06/15] Add services to ci --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b45e77a..09559ba 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -42,6 +42,7 @@ jobs: - name: Run Copier run: | copier --defaults \ + -d services="['setup_dbt_project', 'setup_dbt_profile', 'setup_ci_cd', 'setup_precommit', 'setup_airflow_dags', 'setup_dbtcoves']" \ -d setup_dbt_project=True \ -d setup_dbt_profile=True \ -d setup_ci_cd=True \ From d83993956838294bd97c3c6bc2c4c09a568ea934 Mon Sep 17 00:00:00 2001 From: Bruno Antonellini Date: Fri, 4 Apr 2025 11:16:18 -0300 Subject: [PATCH 07/15] New data file + ci changes --- .github/workflows/main.yml | 21 +-------------------- answers.yml | 4 ++++ ci_requirements.txt | 2 +- copier.yml | 2 ++ data.yml | 26 ++++++++++++++++++++++++++ 5 files changed, 34 insertions(+), 21 deletions(-) create mode 100644 answers.yml create mode 100644 data.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 09559ba..461cbdd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,26 +40,7 @@ jobs: run: pip install -r ci_requirements.txt - name: Run Copier - run: | - copier --defaults \ - -d services="['setup_dbt_project', 'setup_dbt_profile', 'setup_ci_cd', 'setup_precommit', 'setup_airflow_dags', 'setup_dbtcoves']" \ - -d setup_dbt_project=True \ - -d setup_dbt_profile=True \ - -d setup_ci_cd=True \ - -d setup_precommit=True \ - -d setup_airflow_dag=True \ - -d dbt_project_dir="." \ - -d dbt_project_name="balboa" \ - -d is_new_project=True \ - -d dbt_profiles_path="automate/dbt" \ - -d yml_dags_path="orchestrate/dag_yml_definitions" \ - -d dags_path="orchestrate/dags" \ - -d setup_precommit=True \ - -d use_sqlfluff=True \ - -d use_yamllint=True \ - -d use_dbt_checkpoint=True \ - -d datacoves_env="{ 'DATACOVES__AIRFLOW_DBT_PROFILE_PATH': 'automate/dbt', 'DATACOVES__DBT_HOME': '.', 'DATACOVES__AIRFLOW_DAGS_PATH': 'orchestrate/dags', 'DATACOVES__AIRFLOW_DAGS_YML_PATH': 'orchestrate/dags_yml_definitions' }" \ - -a ../copier-answers.yml copy ./ generated/ + run: copier copy --data-file data.yml ./ generated/ # run dbt debug and dbt compile - name: Run dbt commands diff --git a/answers.yml b/answers.yml new file mode 100644 index 0000000..efa94f4 --- /dev/null +++ b/answers.yml @@ -0,0 +1,4 @@ +airflow_artifacts_path: "orchestrate" +setup_dbtcoves_config: true +setup_dbtcoves_templates: true +ci_provider: "github" diff --git a/ci_requirements.txt b/ci_requirements.txt index 9a110a2..9fe2d4b 100644 --- a/ci_requirements.txt +++ b/ci_requirements.txt @@ -1,4 +1,4 @@ -dbt-coves==1.8.15 +git+https://github.com/datacoves/dbt-coves.git@support-pydantic-2.0 dbt-core==1.8.9 dbt-snowflake==1.8.4 pre-commit==3.7.1 \ No newline at end of file diff --git a/copier.yml b/copier.yml index 3d2704b..d529649 100644 --- a/copier.yml +++ b/copier.yml @@ -24,6 +24,8 @@ _exclude: - "*.py[co]" - __pycache__ - .DS_Store + - data.yml + - answers.yml services: type: str diff --git a/data.yml b/data.yml new file mode 100644 index 0000000..abfee30 --- /dev/null +++ b/data.yml @@ -0,0 +1,26 @@ +services: + - setup_dbt_project + - setup_dbt_profile + - setup_ci_cd + - setup_precommit + - setup_airflow_dags + - setup_dbtcoves +setup_dbt_project: true +setup_dbt_profile: true +setup_ci_cd: true +setup_precommit: true +setup_airflow_dag: true +dbt_project_dir: "." +dbt_project_name: "mock_project" +is_new_project: true +dbt_profiles_path: "automate/dbt" +yml_dags_path: "orchestrate/dag_yml_definitions" +dags_path: "orchestrate/dags" +use_sqlfluff: true +use_yamllint: true +use_dbt_checkpoint: true +datacoves_env: + DATACOVES__AIRFLOW_DBT_PROFILE_PATH: "automate/dbt" + DATACOVES__DBT_HOME: "." + DATACOVES__AIRFLOW_DAGS_PATH: "orchestrate/dags" + DATACOVES__AIRFLOW_DAGS_YML_PATH: "orchestrate/dags_yml_definitions" From c640b0dde3227c1efa970d27e3013c72c8f69114 Mon Sep 17 00:00:00 2001 From: Bruno Antonellini Date: Fri, 4 Apr 2025 11:23:46 -0300 Subject: [PATCH 08/15] New data file + ci changes --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 461cbdd..dbfa482 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,7 +40,7 @@ jobs: run: pip install -r ci_requirements.txt - name: Run Copier - run: copier copy --data-file data.yml ./ generated/ + run: copier copy --data-file data.yml -a answers.yml ./ generated/ # run dbt debug and dbt compile - name: Run dbt commands From a3e43dc6a6235aac438d4990cbcdf275d40022a3 Mon Sep 17 00:00:00 2001 From: Bruno Antonellini Date: Fri, 4 Apr 2025 11:37:11 -0300 Subject: [PATCH 09/15] Remove answers, move ci_provider to data file --- .github/workflows/main.yml | 2 +- answers.yml | 4 ---- data.yml | 1 + 3 files changed, 2 insertions(+), 5 deletions(-) delete mode 100644 answers.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dbfa482..461cbdd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,7 +40,7 @@ jobs: run: pip install -r ci_requirements.txt - name: Run Copier - run: copier copy --data-file data.yml -a answers.yml ./ generated/ + run: copier copy --data-file data.yml ./ generated/ # run dbt debug and dbt compile - name: Run dbt commands diff --git a/answers.yml b/answers.yml deleted file mode 100644 index efa94f4..0000000 --- a/answers.yml +++ /dev/null @@ -1,4 +0,0 @@ -airflow_artifacts_path: "orchestrate" -setup_dbtcoves_config: true -setup_dbtcoves_templates: true -ci_provider: "github" diff --git a/data.yml b/data.yml index abfee30..b94d2b2 100644 --- a/data.yml +++ b/data.yml @@ -19,6 +19,7 @@ dags_path: "orchestrate/dags" use_sqlfluff: true use_yamllint: true use_dbt_checkpoint: true +ci_provider: "github" datacoves_env: DATACOVES__AIRFLOW_DBT_PROFILE_PATH: "automate/dbt" DATACOVES__DBT_HOME: "." From e90dfc9fa8fedcba65fcbb87e95251fc29012fec Mon Sep 17 00:00:00 2001 From: Bruno Antonellini Date: Fri, 4 Apr 2025 11:40:08 -0300 Subject: [PATCH 10/15] DCV-3288 complete data.yml file --- data.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/data.yml b/data.yml index b94d2b2..6660f59 100644 --- a/data.yml +++ b/data.yml @@ -25,3 +25,10 @@ datacoves_env: DATACOVES__DBT_HOME: "." DATACOVES__AIRFLOW_DAGS_PATH: "orchestrate/dags" DATACOVES__AIRFLOW_DAGS_YML_PATH: "orchestrate/dags_yml_definitions" +airflow_artifacts_path: "orchestrate" +setup_dbtcoves_config: true +setup_dbtcoves_templates: true +dbt_checkpoint_version: v2.0.6 +yamllint_version: "1.36.2" +sqlfluff_version: "3.1.1" +dbt_adapter: "snowflake" From 9cca8629d59bf438887135fe6d8d82442f761be3 Mon Sep 17 00:00:00 2001 From: Bruno Antonellini Date: Fri, 4 Apr 2025 11:47:28 -0300 Subject: [PATCH 11/15] New data entry --- data.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/data.yml b/data.yml index 6660f59..8974de8 100644 --- a/data.yml +++ b/data.yml @@ -32,3 +32,4 @@ dbt_checkpoint_version: v2.0.6 yamllint_version: "1.36.2" sqlfluff_version: "3.1.1" dbt_adapter: "snowflake" +add_dbtcoves_config_and_templates: true From b12f39e964923de0e23812ffb18781e6d0dc8937 Mon Sep 17 00:00:00 2001 From: Bruno Antonellini Date: Fri, 4 Apr 2025 11:55:54 -0300 Subject: [PATCH 12/15] New data entry --- data.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/data.yml b/data.yml index 8974de8..d6cd721 100644 --- a/data.yml +++ b/data.yml @@ -5,11 +5,6 @@ services: - setup_precommit - setup_airflow_dags - setup_dbtcoves -setup_dbt_project: true -setup_dbt_profile: true -setup_ci_cd: true -setup_precommit: true -setup_airflow_dag: true dbt_project_dir: "." dbt_project_name: "mock_project" is_new_project: true From b786699ef72e1b4229f16642714280f3516b8b33 Mon Sep 17 00:00:00 2001 From: Bruno Antonellini Date: Fri, 4 Apr 2025 12:00:58 -0300 Subject: [PATCH 13/15] Add trust flag --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 461cbdd..26835eb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,7 +40,7 @@ jobs: run: pip install -r ci_requirements.txt - name: Run Copier - run: copier copy --data-file data.yml ./ generated/ + run: copier copy --data-file data.yml ./ generated/ --trust # run dbt debug and dbt compile - name: Run dbt commands From 0fb3cbe16b9220ed4094f082f9d17334c7dd851b Mon Sep 17 00:00:00 2001 From: Bruno Antonellini Date: Fri, 4 Apr 2025 12:03:47 -0300 Subject: [PATCH 14/15] DCV-3288 finish setup changes --- copier.yml | 2 +- data.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/copier.yml b/copier.yml index d529649..afd1633 100644 --- a/copier.yml +++ b/copier.yml @@ -127,7 +127,7 @@ dbt_checkpoint_version: yamllint_version: type: str help: What yamllint version are you using? - default: "{{ datacoves_env.get('DATACOVES__YAMLLINT_VERSION', '1.36.2') }}" + default: "{{ datacoves_env.get('DATACOVES__YAMLLINT_VERSION', 'v1.36.2') }}" when: "{{use_yamllint}}" sqlfluff_version: diff --git a/data.yml b/data.yml index d6cd721..33181a5 100644 --- a/data.yml +++ b/data.yml @@ -24,7 +24,7 @@ airflow_artifacts_path: "orchestrate" setup_dbtcoves_config: true setup_dbtcoves_templates: true dbt_checkpoint_version: v2.0.6 -yamllint_version: "1.36.2" +yamllint_version: "v1.36.2" sqlfluff_version: "3.1.1" dbt_adapter: "snowflake" add_dbtcoves_config_and_templates: true From 13c2a66692d83a18397d8ffff3aeb15d97c0e2d0 Mon Sep 17 00:00:00 2001 From: Bruno Antonellini Date: Fri, 4 Apr 2025 12:15:50 -0300 Subject: [PATCH 15/15] New data entries --- data.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data.yml b/data.yml index 33181a5..81112b6 100644 --- a/data.yml +++ b/data.yml @@ -28,3 +28,5 @@ yamllint_version: "v1.36.2" sqlfluff_version: "3.1.1" dbt_adapter: "snowflake" add_dbtcoves_config_and_templates: true +dbt_core_version: "1.8.9" +dbt_adapter_version: "1.8.4"