feat: schema diff more visibility

Author: Dat Nguyen

integration_tests/dbt_project.yml

@@ -16,23 +16,23 @@ on-run-start:
     {% endif %}
 
 vars:
-  data_diff__database: data_diff
-  data_diff__schema: common
+  # data_diff__database: data_diff
+  data_diff__schema: datadiff
   # For normal (non-async) mode
   data_diff__configured_tables:
     - # src_db: data_diff
-      src_schema: blue_dat
+      src_schema: dbt_blue
       src_table: my_first_dbt_model
       # trg_db: data_diff
-      trg_schema: green_dat
+      trg_schema: dbt_green
       trg_table: my_first_dbt_model
       pk: id # id1,id2
       include_columns: []
       exclude_columns: ["loaded_at"]
       # pipe_name: awesome_thread
-    - src_schema: blue_dat
+    - src_schema: dbt_blue
       src_table: my_second_dbt_model
-      trg_schema: green_dat
+      trg_schema: dbt_green
       trg_table: my_second_dbt_model
       pk: id
       include_columns: []

integration_tests/models/example/my_first_dbt_model.sql

@@ -11,16 +11,16 @@
 
 with source_data as (
 
-    select 1 as id, 'id 1' as id_description
+    select 1 as id, 'id 1' as id_description, {% if target.name.lower() == 'blue' %} '100' {%else %} 100 {% endif %} as amount
     union all
-    select null as id, 'null' as id_description
+    select null as id, 'null' as id_description, {% if target.name.lower() == 'blue' %} '100' {%else %} 100 {% endif %} as amount
     union all
-    select 2 as id, 'id 2' as id_description
+    select 2 as id, 'id 2' as id_description, {% if target.name.lower() == 'blue' %} '100' {%else %} 100 {% endif %} as amount
     union all
-    select 3 as id, {% if target.name.lower() == 'blue' %}'id 3 blue'{% else %}'id 3 green'{% endif %}as id_description
+    select 3 as id, {% if target.name.lower() == 'blue' %}'id 3 blue'{% else %}'id 3 green'{% endif %}as id_description, {% if target.name.lower() == 'blue' %} '100' {%else %} 100 {% endif %} as amount
     {% if target.name.lower() == 'blue' %}
     union all
-    select 4 as id, 'id 4' as id_description
+    select 4 as id, 'id 4' as id_description, {% if target.name.lower() == 'blue' %} '100' {%else %} 100 {% endif %} as amount
     {% endif %}
 
 )

integration_tests/models/example/my_second_dbt_model.sql

@@ -1,7 +1,7 @@
 
 -- Use the `ref` function to select from other models
 
-select *
+select id, id_description
 from {{ ref('my_first_dbt_model') }}
 where true
 {# and id is not null  #}

macros/sis/diff_helper.py

@@ -69,9 +69,17 @@
         )
 
         select      case when r.src_db is null then '🟢' else '🔴' end as result
-                    ,concat(r.number_of_exclusive_source, ' (',upper(r.exclusive_source_list),')') as source_not_found
-                    ,concat(r.number_of_exclusive_source, ' (',upper(r.exclusive_target_list),')') as target_not_found
-                    ,coalesce(1 - r.mutual_columns * 1.0 / r.number_of_columns, 0) as failed_rate
+                    ,case
+                        when r.number_of_exclusive_source > 0 then concat(r.number_of_exclusive_source, ' (',upper(r.exclusive_source_list),')')
+                    end as source_not_found
+                    ,case
+                        when r.number_of_exclusive_source > 0 then concat(r.number_of_exclusive_source, ' (',upper(r.exclusive_target_list),')')
+                    end as target_not_found
+                    ,coalesce(1 - r.mutual_columns * 1.0 / r.number_of_columns, 0) as not_found_rate
+                    ,case
+                        when r.number_of_false_datatype_check > 0 then concat(r.number_of_false_datatype_check, ' (',upper(r.false_datatype_check_list),')')
+                    end as data_type_mismatched
+                    ,coalesce(r.number_of_false_datatype_check  * 1.0 / r.number_of_columns, 0) as mismatched_rate
                     ,concat(
                         c.src_db,'.',c.src_schema,'.',c.src_table,
                         ' ▶️ ',

models/02_schema_diff/schema_check_summary.sql

@@ -16,7 +16,11 @@ select
   , diff_run_id
   , count(*) as number_of_columns
   , sum(case when common_col then 1 else 0 end) as mutual_columns
+
   , sum(case when datatype_check then 0 else 1 end) as number_of_false_datatype_check
+  , listagg(
+    case when not datatype_check then column_name end, ', '
+  ) within group (order by column_name) as false_datatype_check_list
 
   , sum(case when is_exclusive_src then 1 else 0 end) as number_of_exclusive_target
   , listagg(

models/02_schema_diff/schema_check_summary.yml

@@ -23,10 +23,12 @@ models:
         description: Total number of columns
       - name: mutual_columns
         description: Total number of columns existing in both target and source
-      - name: number_of_exclusive_target
-        description: Number of columns exist in target only
       - name: number_of_false_datatype_check
         description: Number of columns which are not matched in the data type
+      - name: false_datatype_check_list
+        description: List of columns have not been matched in data type
+      - name: number_of_exclusive_target
+        description: Number of columns exist in target only
       - name: exclusive_target_list
         description: List of columns exist in target only
       - name: number_of_exclusive_source