Merge pull request #415 from data-integrations/sql_server_datatypes

Mssql batch source and sink documentation changes

Author: Aryan-Verma

mssql-plugin/docs/SqlServer-batchsink.md

@@ -92,9 +92,9 @@ Data Types Mapping
     | BIT              | boolean                |                                                                |
     | CHAR             | string                 |                                                                |
     | DATE             | date                   |                                                                |
-    | DATETIME         | timestamp              |                                                                |
-    | DATETIME2        | timestamp              |                                                                |
-    | DATETIMEOFFSET   | string                 |  DATETIMEOFFSET string literal in the following format:        |
+    | DATETIME         | datetime               |                                                                |
+    | DATETIME2        | datetime               |                                                                |
+    | DATETIMEOFFSET   | timestamp              |  DATETIMEOFFSET string literal in the following format:        |
     |                  |                        |  "2019-06-24 16:19:15.8010000 +03:00"                          |
     | DECIMAL          | decimal                |                                                                |
     | FLOAT            | double                 |                                                                |
@@ -107,7 +107,7 @@ Data Types Mapping
     | NVARCHAR         | string                 |                                                                |
     | NVARCHAR(MAX)    | string                 |                                                                |
     | REAL             | float                  |                                                                |
-    | SMALLDATETIME    | timestamp              |                                                                |
+    | SMALLDATETIME    | datetime               |                                                                |
     | SMALLINT         | int                    |                                                                |
     | SMALLMONEY       | decimal                |                                                                |
     | TEXT             | string                 |                                                                |

mssql-plugin/docs/SqlServer-batchsource.md

@@ -110,7 +110,7 @@ Data Types Mapping
     | DATE             | date                   |                                                                |
     | DATETIME         | datetime               | Users can manually set output schema to map it to timestamp.   |
     | DATETIME2        | datetime               | Users can manually set output schema to map it to timestamp.   |
-    | DATETIMEOFFSET   | datetime               | Users can manually set output schema to map it to string.      |
+    | DATETIMEOFFSET   | timestamp              | Users can manually set output schema to map it to datetime.    |
     | DECIMAL          | decimal                |                                                                |
     | FLOAT            | double                 |                                                                |
     | IMAGE            | bytes                  |                                                                |
@@ -122,7 +122,7 @@ Data Types Mapping
     | NVARCHAR         | string                 |                                                                |
     | NVARCHAR(MAX)    | string                 |                                                                |
     | REAL             | float                  |                                                                |
-    | SMALLDATETIME    | timestamp              |                                                                |
+    | SMALLDATETIME    | datetime               | Users can manually set output schema to map it to timestamp.   |
     | SMALLINT         | int                    |                                                                |
     | SMALLMONEY       | decimal                |                                                                |
     | TEXT             | string                 |                                                                |

mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlFieldsValidator.java

@@ -64,9 +64,8 @@ public boolean isFieldCompatible(Schema.Field field, ResultSetMetaData metadata,
           || super.isFieldCompatible(field, metadata, index);
       case STRING:
         return
-          sqlType == SqlServerSinkSchemaReader.DATETIME_OFFSET_TYPE
           // Value of GEOMETRY and GEOGRAPHY type can be set as Well Known Text string such as "POINT(3 40 5 6)"
-          || sqlType == SqlServerSinkSchemaReader.GEOGRAPHY_TYPE
+          sqlType == SqlServerSinkSchemaReader.GEOGRAPHY_TYPE
           || sqlType == SqlServerSinkSchemaReader.GEOMETRY_TYPE
           || sqlType == SqlServerSinkSchemaReader.SQL_VARIANT
           || sqlType == Types.ROWID

mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java

@@ -197,8 +197,8 @@ public void validate(FailureCollector collector) {
     @Override
     protected void validateField(FailureCollector collector, Schema.Field field, Schema actualFieldSchema,
                                  Schema expectedFieldSchema) {
-      // we allow the case when actual type is Datetime but user manually set it to timestamp (datetime and datetime2)
-      // or string (datetimeoffset). To make it compatible with old behavior that convert datetime to timestamp.
+      // we allow the case when actual type is Datetime but user manually set it to timestamp (datetime and datetime2).
+      // To make it compatible with old behavior that convert datetime to timestamp.
       // below validation is kind of loose, it's possible users try to manually map datetime to string or
       // map datetimeoffset to timestamp which is invalid. In such case runtime will still fail even validation passes.
       // But we don't have the original source type information here and don't want to do big refactoring here
@@ -208,12 +208,6 @@ protected void validateField(FailureCollector collector, Schema.Field field, Sch
         // SmallDateTime does not contain any TimeZone information
         return;
       }
-      if ((actualFieldSchema.getLogicalType() == Schema.LogicalType.DATETIME ||
-        actualFieldSchema.getLogicalType() == Schema.LogicalType.TIMESTAMP_MICROS) &&
-          expectedFieldSchema.getType() == Schema.Type.STRING) {
-        // Case when user manually sets the type to string
-        return;
-      }
       if (actualFieldSchema.getLogicalType() == Schema.LogicalType.TIMESTAMP_MICROS &&
           expectedFieldSchema.getLogicalType() == Schema.LogicalType.DATETIME) {
         // DateTimeOffset case where we map it to CDAP Timestamp as opposed to CDAP DateTime earlier