Dcv 3288 dbt coves setup improvements

.github/workflows/main.yml

@@ -40,8 +40,7 @@ jobs:
         run: pip install -r ci_requirements.txt
 
       - name: Run Copier
-        run: |
-          copier --defaults -d setup_dbt_project=True -d setup_dbt_profile=True -d setup_ci_cd=True -d setup_precommit=True -d setup_airflow_dag=True -d dbt_project_dir="." -d dbt_project_name="balboa" -d is_new_project=True  -d airflow_profile_path="automate/dbt" -d yml_dags_path="orchestrate/dag_yml_definitions" -d dags_path="orchestrate/dags" -d setup_precommit=True -d use_sqlfluff=True -d use_yamllint=True -d use_dbt_checkpoint=True -a ../copier-answers.yml copy ./ generated/
+        run: copier copy --data-file data.yml ./ generated/ --trust
 
       # run dbt debug and dbt compile
       - name: Run dbt commands

ci_requirements.txt

@@ -1,4 +1,4 @@
-dbt-coves==1.8.15
+git+https://github.com/datacoves/dbt-coves.git@support-pydantic-2.0
 dbt-core==1.8.9
 dbt-snowflake==1.8.4
 pre-commit==3.7.1

copier.yml

@@ -1,5 +1,20 @@
+_tasks:
+  - echo 'Hello template'
+  # - command: ["git init"]
+  # only run on project setup
+  # when: "{{ _copier_operation == 'copy' }}"
+
+_message_before_copy: |
+  Thanks for generating a project using our template.
+
+_message_after_copy: |
+  Your project "{{ dbt_project_name }}" has been created successfully!
+
+_message_after_update: |
+  Your project "{{ dbt_project_name }}" has been updated successfully!
+  In case there
+
 _exclude:
-  - copier-answers.yml
   - ci_requirements.txt
   - test_load_dagbag.py
   - copier.yaml
@@ -9,64 +24,119 @@ _exclude:
   - "*.py[co]"
   - __pycache__
   - .DS_Store
+  - data.yml
+  - answers.yml
 
-ci_provider:
+services:
   type: str
+  multiselect: true
+  help: What services do you want to setup?
   choices:
-    - github
-    - gitlab
-  when: "{{setup_ci_cd}}"
-  default: github
-  help: CI/CD provider
+    Base dbt project: setup_dbt_project
+    dbt profile for automated runs: setup_dbt_profile
+    setup_ci_cd: setup_ci_cd
+    Linting with SQLFluff, dbt-checkpoint and-or YMLLint: setup_precommit
+    Sample Airflow DAGs: setup_airflow_dags
+    dbt-coves config and-or templates: setup_dbtcoves
+
+# dbt project related vars
+dbt_project_dir:
+  type: str
+  when: "{{'setup_dbt_project' in services}}"
+  default: "{{ datacoves_env.get('DATACOVES__DBT_HOME', '.') }}"
+  help: dbt project directory
+
+dbt_project_name:
+  type: str
+  when: "{{'setup_dbt_project' in services}}"
+  help: What is the name of the dbt project?
 
+# Airflow related vars
 dags_path:
   type: str
-  when: "{{airflow_dags_confirm_path}}"
-  default: "{{tentative_dags_path}}"
+  when: "{{'setup_airflow_dags' in services or 'setup_ci_cd' in services}}"
+  default: "{{ datacoves_env.get('DATACOVES__AIRFLOW_DAGS_PATH', 'orchestrate/dags') }}"
   help: Airflow DAGs path
 
+airflow_artifacts_path:
+  type: str
+  when: "{{'setup_ci_cd' in services}}"
+  default: "{{ datacoves_env.get('DATACOVES__AIRFLOW_DAGS_PATH', 'orchestrate/dags').split('/')[0] }}"
+  help: Airflow Artifacts path for unit tests
+
 yml_dags_path:
   type: str
-  when: "{{yml_dags_confirm_path}}"
-  default: "{{tentative_yml_dags_path}}"
+  when: "{{'setup_airflow_dags' in services and yml_dags_confirm_path}}"
+  default: "{{datacoves_env.get('DATACOVES__AIRFLOW_DAGS_YML_PATH', 'orchestrate/dags_yml_definitions')}}"
   help: Airflow YML Dags path
 
+# dbt-coves related vars
+add_dbtcoves_config_and_templates:
+  type: bool
+  default: false
+  when: "{{not 'setup_dbtcoves' in services and 'setup_dbt_project' in services}}"
+  help: Add dbt-coves config and templates?
+
+setup_dbtcoves_config:
+  type: bool
+  default: true
+  when: "{{'setup_dbtcoves' in services}}"
+  help: Setup dbt-coves config?
+
+setup_dbtcoves_templates:
+  type: bool
+  default: true
+  when: "{{'setup_dbtcoves' in services}}"
+  help: Setup dbt-coves templates?
+
+# CI / CD Related vars
+
+ci_provider:
+  type: str
+  choices:
+    - github
+    - gitlab
+  when: "{{'setup_ci_cd' in services}}"
+  default: github
+  help: CI/CD provider
+
 use_dbt_checkpoint:
   type: bool
   default: false
-  when: "{{setup_precommit}}"
+  when: "{{'setup_precommit' in services}}"
   help: use dbt-checkpoint?
 
 use_sqlfluff:
   type: bool
   default: false
-  when: "{{setup_precommit}}"
+  when: "{{'setup_precommit' in services}}"
   help: use Sqlfluff?
 
 use_yamllint:
   type: bool
   default: false
-  when: "{{setup_precommit}}"
+  when: "{{'setup_precommit' in services}}"
   help: use Yamllint?
 
-add_dbtcoves_config_and_templates:
-  type: bool
-  default: false
-  when: "{{not setup_dbt_coves and setup_dbt_project}}"
-  help: Add dbt-coves config and templates?
+dbt_checkpoint_version:
+  type: str
+  help: What dbt-checkpoint version are you using?
+  default: "{{ datacoves_env.get('DATACOVES__DBT_CHECKPOINT_VERSION', 'v2.0.6') }}"
+  when: "{{use_dbt_checkpoint}}"
 
-setup_dbtcoves_config:
-  type: bool
-  default: true
-  when: "{{setup_dbt_coves}}"
-  help: Setup dbt-coves config?
+yamllint_version:
+  type: str
+  help: What yamllint version are you using?
+  default: "{{ datacoves_env.get('DATACOVES__YAMLLINT_VERSION', 'v1.36.2') }}"
+  when: "{{use_yamllint}}"
 
-setup_dbtcoves_templates:
-  type: bool
-  default: true
-  when: "{{setup_dbt_coves}}"
-  help: Setup dbt-coves templates?
+sqlfluff_version:
+  type: str
+  help: What sqlfluff version are you using?
+  default: "{{ datacoves_env.get('DATACOVES__SQLFLUFF_VERSION', '3.1.1') }}"
+  when: "{{use_sqlfluff}}"
 
+# dbt related vars
 dbt_adapter:
   type: str
   choices:
@@ -75,15 +145,11 @@ dbt_adapter:
     - redshift
     - databricks
   help: What dbt adapter are you using?
-  when: "{{not datacoves_dbt_adapter and (setup_dbt_profile or setup_ci_cd or setup_sqlfluff)}}"
-  default: snowflake
-
-dbt_checkpoint_version:
-  type: str
-  help: What dbt-checkpoint version are you using?
-  when: "{{use_dbt_checkpoint and ask_dbt_checkpoint_version}}"
+  when: "{{not datacoves_dbt_adapter and ('setup_dbt_profile' in services or setup_ci_cd or setup_sqlfluff)}}"
+  default: "{{ datacoves_env.get('DATACOVES__DBT_ADAPTER', 'snowflake') }}"
 
-yamllint_version:
+dbt_profiles_path:
   type: str
-  help: What yamllint version are you using?
-  when: "{{use_yamllint and ask_yamllint_version}}"
+  when: "{{ 'setup_dbt_profile' in services or 'setup_ci_cd' in services }}"
+  default: "{{ datacoves_env.get('DATACOVES__AIRFLOW_DBT_PROFILE_PATH', 'automate/dbt') }}"
+  help: dbt Profile Path for automated runs

data.yml

@@ -0,0 +1,32 @@
+services:
+  - setup_dbt_project
+  - setup_dbt_profile
+  - setup_ci_cd
+  - setup_precommit
+  - setup_airflow_dags
+  - setup_dbtcoves
+dbt_project_dir: "."
+dbt_project_name: "mock_project"
+is_new_project: true
+dbt_profiles_path: "automate/dbt"
+yml_dags_path: "orchestrate/dag_yml_definitions"
+dags_path: "orchestrate/dags"
+use_sqlfluff: true
+use_yamllint: true
+use_dbt_checkpoint: true
+ci_provider: "github"
+datacoves_env:
+  DATACOVES__AIRFLOW_DBT_PROFILE_PATH: "automate/dbt"
+  DATACOVES__DBT_HOME: "."
+  DATACOVES__AIRFLOW_DAGS_PATH: "orchestrate/dags"
+  DATACOVES__AIRFLOW_DAGS_YML_PATH: "orchestrate/dags_yml_definitions"
+airflow_artifacts_path: "orchestrate"
+setup_dbtcoves_config: true
+setup_dbtcoves_templates: true
+dbt_checkpoint_version: v2.0.6
+yamllint_version: "v1.36.2"
+sqlfluff_version: "3.1.1"
+dbt_adapter: "snowflake"
+add_dbtcoves_config_and_templates: true
+dbt_core_version: "1.8.9"
+dbt_adapter_version: "1.8.4"

{%if setup_ci_cd and ci_provider=='github'%}.github{%endif%}/workflows/pull_request_build.yml.jinja → {%if 'setup_ci_cd' in services and ci_provider=='github'%}.github{%endif%}/workflows/pull_request_build.yml.jinja

@@ -35,7 +35,7 @@ jobs:
         {% raw %} working-directory: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{%endraw%}/{{dbt_project_dir}}
 
     env:
-      {% raw %}DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{%endraw%}/{{ airflow_profile_path }}
+      {% raw %}DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{%endraw%}/{{ dbt_profiles_path }}
       {% raw %}DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{%endraw%}/{{dbt_project_dir}}
 
       {% if 'snowflake' in dbt_adapter %}

{%if setup_ci_cd and ci_provider=='github'%}.github{%endif%}/workflows/push-to-main.yml.jinja → {%if 'setup_ci_cd' in services and ci_provider=='github'%}.github{%endif%}/workflows/push-to-main.yml.jinja

@@ -43,7 +43,7 @@ jobs:
         {% raw %}working-directory: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%}/{{dbt_project_dir}}
 
     env:
-      {% raw %}DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%}/{{airflow_profile_path}}
+      {% raw %}DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%}/{{dbt_profiles_path}}
       {% raw %}DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}{% endraw%}/{{dbt_project_dir}}
 
       {% if 'snowflake' in dbt_adapter %}

{{ _copier_conf.answers_file }}.jinja

@@ -0,0 +1,2 @@
+# Changes here will be overwritten by Copier; NEVER EDIT MANUALLY
+{{ _copier_answers|to_nice_yaml  }}