Skip to content

Commit 4650d1f

Browse files
Nikitapaliwal123Nikitapaliwal123
authored
Added sanity tests for sqlserver (#372)
* Added sanity tests for sqlserver * Added sanity tests for sqlserver
1 parent 64760bb commit 4650d1f

File tree

7 files changed

+349
-10
lines changed

7 files changed

+349
-10
lines changed

‎mssql-plugin/src/e2e-test/features/mssql/Datatype.feature‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
#
2+
# Copyright © 2023 Cask Data, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
# use this file except in compliance with the License. You may obtain a copy of
6+
# the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
# License for the specific language governing permissions and limitations under
14+
# the License.
15+
#
16+
17+
@Mssql
18+
Feature: Mssql - Verify Mssql source data transfer
19+
@MSSQL_SOURCE_DATATYPES_TEST @MSSQL_SINK_TEST @Mssql_Required
20+
Scenario: To verify data is getting transferred from Mssql to Mssql successfully
21+
Given Open Datafusion Project to configure pipeline
22+
When Expand Plugin group in the LHS plugins list: "Source"
23+
When Select plugin: "SQL Server" from the plugins list as: "Source"
24+
When Expand Plugin group in the LHS plugins list: "Sink"
25+
When Select plugin: "SQL Server" from the plugins list as: "Sink"
26+
Then Connect plugins: "SQL Server" and "SQL Server2" to establish connection
27+
Then Navigate to the properties page of plugin: "SQL Server"
28+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
29+
Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
30+
Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
31+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
32+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
33+
Then Enter input plugin property: "referenceName" with value: "sourceRef"
34+
Then Replace input plugin property: "database" with value: "databaseName"
35+
Then Enter textarea plugin property: "importQuery" with value: "selectQuery"
36+
Then Click on the Get Schema button
37+
Then Verify the Output Schema matches the Expected Schema: "outputDatatypesSchema"
38+
Then Validate "SQL Server" plugin properties
39+
Then Close the Plugin Properties page
40+
Then Navigate to the properties page of plugin: "SQL Server2"
41+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
42+
Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
43+
Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
44+
Then Replace input plugin property: "database" with value: "databaseName"
45+
Then Replace input plugin property: "tableName" with value: "targetTable"
46+
Then Replace input plugin property: "dbSchemaName" with value: "schema"
47+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
48+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
49+
Then Enter input plugin property: "referenceName" with value: "targetRef"
50+
Then Validate "SQL Server2" plugin properties
51+
Then Close the Plugin Properties page
52+
Then Save the pipeline
53+
Then Preview and run the pipeline
54+
Then Verify the preview of pipeline is "success"
55+
Then Click on preview data for Mssql sink
56+
Then Verify preview output schema matches the outputSchema captured in properties
57+
Then Close the preview data
58+
Then Deploy the pipeline
59+
Then Run the Pipeline in Runtime
60+
Then Wait till pipeline is in running state
61+
Then Open and capture logs
62+
Then Verify the pipeline status is "Succeeded"
63+
Then Validate records transferred to target table are equal to number of records from the source table
64+
65+

‎mssql-plugin/src/e2e-test/features/mssql/DatatypeImage.feature‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
#
2+
# Copyright © 2023 Cask Data, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
# use this file except in compliance with the License. You may obtain a copy of
6+
# the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
# License for the specific language governing permissions and limitations under
14+
# the License.
15+
#
16+
17+
@Mssql
18+
Feature: Mssql - Verify Mssql source data transfer
19+
@MSSQL_SOURCE_DATATYPES_IMAGE_TEST @MSSQL_SINK_TEST @Mssql_Required
20+
Scenario: To verify data is getting transferred from Mssql to Mssql successfully
21+
Given Open Datafusion Project to configure pipeline
22+
When Expand Plugin group in the LHS plugins list: "Source"
23+
When Select plugin: "SQL Server" from the plugins list as: "Source"
24+
When Expand Plugin group in the LHS plugins list: "Sink"
25+
When Select plugin: "SQL Server" from the plugins list as: "Sink"
26+
Then Connect plugins: "SQL Server" and "SQL Server2" to establish connection
27+
Then Navigate to the properties page of plugin: "SQL Server"
28+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
29+
Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
30+
Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
31+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
32+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
33+
Then Enter input plugin property: "referenceName" with value: "sourceRef"
34+
Then Replace input plugin property: "database" with value: "databaseName"
35+
Then Enter textarea plugin property: "importQuery" with value: "selectQuery"
36+
Then Click on the Get Schema button
37+
Then Verify the Output Schema matches the Expected Schema: "outputDatatypesSchema2"
38+
Then Validate "SQL Server" plugin properties
39+
Then Close the Plugin Properties page
40+
Then Navigate to the properties page of plugin: "SQL Server2"
41+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
42+
Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
43+
Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
44+
Then Replace input plugin property: "database" with value: "databaseName"
45+
Then Replace input plugin property: "tableName" with value: "targetTable"
46+
Then Replace input plugin property: "dbSchemaName" with value: "schema"
47+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
48+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
49+
Then Enter input plugin property: "referenceName" with value: "targetRef"
50+
Then Validate "SQL Server2" plugin properties
51+
Then Close the Plugin Properties page
52+
Then Save the pipeline
53+
Then Preview and run the pipeline
54+
Then Verify the preview of pipeline is "success"
55+
Then Click on preview data for Mssql sink
56+
Then Verify preview output schema matches the outputSchema captured in properties
57+
Then Close the preview data
58+
Then Deploy the pipeline
59+
Then Run the Pipeline in Runtime
60+
Then Wait till pipeline is in running state
61+
Then Open and capture logs
62+
Then Verify the pipeline status is "Succeeded"
63+
Then Validate records transferred to target table are equal to number of records from the source table
64+
65+

‎mssql-plugin/src/e2e-test/features/mssql/DatatypeUniqueIdentifier.feature‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
#
2+
# Copyright © 2023 Cask Data, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
# use this file except in compliance with the License. You may obtain a copy of
6+
# the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
# License for the specific language governing permissions and limitations under
14+
# the License.
15+
#
16+
17+
@Mssql
18+
Feature: Mssql - Verify Mssql source data transfer
19+
@MSSQL_SOURCE_DATATYPES_UIDTYPE_TEST @MSSQL_SINK_TEST @Mssql_Required
20+
Scenario: To verify data is getting transferred from Mssql to Mssql successfully
21+
Given Open Datafusion Project to configure pipeline
22+
When Expand Plugin group in the LHS plugins list: "Source"
23+
When Select plugin: "SQL Server" from the plugins list as: "Source"
24+
When Expand Plugin group in the LHS plugins list: "Sink"
25+
When Select plugin: "SQL Server" from the plugins list as: "Sink"
26+
Then Connect plugins: "SQL Server" and "SQL Server2" to establish connection
27+
Then Navigate to the properties page of plugin: "SQL Server"
28+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
29+
Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
30+
Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
31+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
32+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
33+
Then Enter input plugin property: "referenceName" with value: "sourceRef"
34+
Then Replace input plugin property: "database" with value: "databaseName"
35+
Then Enter textarea plugin property: "importQuery" with value: "selectQuery"
36+
Then Click on the Get Schema button
37+
Then Verify the Output Schema matches the Expected Schema: "outputDatatypesSchema3"
38+
Then Validate "SQL Server" plugin properties
39+
Then Close the Plugin Properties page
40+
Then Navigate to the properties page of plugin: "SQL Server2"
41+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
42+
Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
43+
Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
44+
Then Replace input plugin property: "database" with value: "databaseName"
45+
Then Replace input plugin property: "tableName" with value: "targetTable"
46+
Then Replace input plugin property: "dbSchemaName" with value: "schema"
47+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
48+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
49+
Then Enter input plugin property: "referenceName" with value: "targetRef"
50+
Then Validate "SQL Server2" plugin properties
51+
Then Close the Plugin Properties page
52+
Then Save the pipeline
53+
Then Preview and run the pipeline
54+
Then Verify the preview of pipeline is "success"
55+
Then Click on preview data for Mssql sink
56+
Then Verify preview output schema matches the outputSchema captured in properties
57+
Then Close the preview data
58+
Then Deploy the pipeline
59+
Then Run the Pipeline in Runtime
60+
Then Wait till pipeline is in running state
61+
Then Open and capture logs
62+
Then Verify the pipeline status is "Succeeded"
63+
Then Validate records transferred to target table are equal to number of records from the source table
64+
65+

‎mssql-plugin/src/e2e-test/java/io.cdap.plugin/MssqlClient.java‎

Lines changed: 95 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,8 @@ public static int countRecord(String table, String schema) throws SQLException,
6464
public static void createSourceTable(String sourceTable, String schema) throws SQLException,
6565
ClassNotFoundException {
6666
try (Connection connect = getMssqlConnection(); Statement statement = connect.createStatement()) {
67-
String createSourceTableQuery = "CREATE TABLE " + schema + "." + sourceTable +
68-
"(ID varchar(100), LASTNAME varchar(100))";
67+
String createSourceTableQuery = createTableQuery(sourceTable, schema,
68+
"(ID varchar(100), LASTNAME varchar(100))");
6969
statement.executeUpdate(createSourceTableQuery);
7070

7171
// Insert dummy data.
@@ -79,12 +79,82 @@ public static void createSourceTable(String sourceTable, String schema) throws S
7979
public static void createTargetTable(String targetTable, String schema) throws SQLException,
8080
ClassNotFoundException {
8181
try (Connection connect = getMssqlConnection(); Statement statement = connect.createStatement()) {
82-
String createTargetTableQuery = "CREATE TABLE " + schema + "." + targetTable +
83-
"(ID varchar(100), LASTNAME varchar(100))";
82+
String createTargetTableQuery = createTableQuery(targetTable, schema,
83+
"(ID varchar(100), LASTNAME varchar(100))");
8484
statement.executeUpdate(createTargetTableQuery);
8585
}
8686
}
8787

88+
public static void createSourceDatatypesTable(String sourceTable, String schema) throws SQLException,
89+
ClassNotFoundException {
90+
try (Connection connect = getMssqlConnection(); Statement statement = connect.createStatement()) {
91+
String datatypeColumns = PluginPropertyUtils.pluginProp("datatypeColumns");
92+
String createSourceTableQuery2 = createTableQuery(sourceTable, schema, datatypeColumns);
93+
statement.executeUpdate(createSourceTableQuery2);
94+
95+
// Insert dummy data.
96+
String datatypeValues = PluginPropertyUtils.pluginProp("datatypeValues");
97+
String datatypeColumnsList = PluginPropertyUtils.pluginProp("datatypeColumnsList");
98+
statement.executeUpdate(insertQuery(sourceTable, schema, datatypeColumnsList, datatypeValues));
99+
}
100+
}
101+
102+
public static void createTargetDatatypesTable(String targetTable, String schema) throws SQLException,
103+
ClassNotFoundException {
104+
try (Connection connect = getMssqlConnection(); Statement statement = connect.createStatement()) {
105+
String datatypeColumns = PluginPropertyUtils.pluginProp("datatypeColumns");
106+
String createTargetTableQuery2 = createTableQuery(targetTable, schema, datatypeColumns);
107+
statement.executeUpdate(createTargetTableQuery2);
108+
}
109+
}
110+
111+
public static void createSourceImageTable(String sourceTable, String schema) throws SQLException,
112+
ClassNotFoundException {
113+
try (Connection connect = getMssqlConnection(); Statement statement = connect.createStatement()) {
114+
String imageColumns = PluginPropertyUtils.pluginProp("imageColumns");
115+
String createSourceTableQuery3 = createTableQuery(sourceTable, schema, imageColumns);
116+
statement.executeUpdate(createSourceTableQuery3);
117+
118+
// Insert dummy data.
119+
String imageValues = PluginPropertyUtils.pluginProp("imageValues");
120+
String imageColumnsList = PluginPropertyUtils.pluginProp("imageColumnsList");
121+
statement.executeUpdate(insertQuery(sourceTable, schema, imageColumnsList, imageValues));
122+
}
123+
}
124+
125+
public static void createTargetImageTable(String targetTable, String schema) throws SQLException,
126+
ClassNotFoundException {
127+
try (Connection connect = getMssqlConnection(); Statement statement = connect.createStatement()) {
128+
String imageColumns = PluginPropertyUtils.pluginProp("imageColumns");
129+
String createTargetTableQuery3 = createTableQuery(targetTable, schema, imageColumns);
130+
statement.executeUpdate(createTargetTableQuery3);
131+
}
132+
}
133+
134+
public static void createSourceUniqueIdentifierTable(String sourceTable, String schema) throws SQLException,
135+
ClassNotFoundException {
136+
try (Connection connect = getMssqlConnection(); Statement statement = connect.createStatement()) {
137+
String uniqueIdentifierColumns = PluginPropertyUtils.pluginProp("uniqueIdentifierColumns");
138+
String createSourceTableQuery3 = createTableQuery(sourceTable, schema, uniqueIdentifierColumns);
139+
statement.executeUpdate(createSourceTableQuery3);
140+
141+
// Insert dummy data.
142+
String uniqueIdentifierValues = PluginPropertyUtils.pluginProp("uniqueIdentifierValues");
143+
String uniqueIdentifierColumnsList = PluginPropertyUtils.pluginProp("uniqueIdentifierColumnsList");
144+
statement.executeUpdate(insertQuery(sourceTable, schema, uniqueIdentifierColumnsList,
145+
uniqueIdentifierValues));
146+
}
147+
}
148+
149+
public static void createTargetUniqueIdentifierTable(String targetTable, String schema) throws SQLException,
150+
ClassNotFoundException {
151+
try (Connection connect = getMssqlConnection(); Statement statement = connect.createStatement()) {
152+
String uniqueIdentifierColumns = PluginPropertyUtils.pluginProp("uniqueIdentifierColumns");
153+
String createTargetTableQuery3 = createTableQuery(targetTable, schema, uniqueIdentifierColumns);
154+
statement.executeUpdate(createTargetTableQuery3);
155+
}
156+
}
157+
88158
public static void deleteTables(String schema, String[] tables)
89159
throws SQLException, ClassNotFoundException {
90160
try (Connection connect = getMssqlConnection(); Statement statement = connect.createStatement()) {
@@ -111,6 +181,15 @@ public static boolean validateRecordValues(String schema, String sourceTable, St
111181
}
112182
}
113183

184+
private static String createTableQuery(String table, String schema, String columns) {
185+
return String.format("CREATE TABLE %s.%s %s", schema, table, columns);
186+
}
187+
188+
private static String insertQuery(String table, String schema, String columnList, String columnValues) {
189+
return String.format("INSERT INTO %s.%s %s %s", schema, table,
190+
columnList, columnValues);
191+
}
192+
114193
private static boolean compareResultSetData(ResultSet rsSource, ResultSet rsTarget) throws SQLException {
115194
ResultSetMetaData mdSource = rsSource.getMetaData();
116195
ResultSetMetaData mdTarget = rsTarget.getMetaData();
@@ -124,13 +203,19 @@ private static boolean compareResultSetData(ResultSet rsSource, ResultSet rsTarg
124203
String columnTypeName = mdSource.getColumnTypeName(currentColumnCount);
125204
int columnType = mdSource.getColumnType(currentColumnCount);
126205
String columnName = mdSource.getColumnName(currentColumnCount);
127-
switch (columnType) {
206+
if (columnType == Types.TIMESTAMP) {
207+
GregorianCalendar gc = new GregorianCalendar(TimeZone.getTimeZone("UTC"));
208+
gc.setGregorianChange(new Date(Long.MIN_VALUE));
209+
Timestamp sourceTS = rsSource.getTimestamp(currentColumnCount, gc);
210+
Timestamp targetTS = rsTarget.getTimestamp(currentColumnCount, gc);
211+
Assert.assertEquals(String.format("Different values found for column : %s", columnName),
212+
sourceTS, targetTS);
128213

129-
default:
130-
String sourceString = rsSource.getString(currentColumnCount);
131-
String targetString = rsTarget.getString(currentColumnCount);
132-
Assert.assertTrue(String.format("Different values found for column : %s", columnName),
133-
sourceString.equals(targetString));
214+
} else {
215+
String sourceString = rsSource.getString(currentColumnCount);
216+
String targetString = rsTarget.getString(currentColumnCount);
217+
Assert.assertEquals(String.format("Different values found for column : %s", columnName),
218+
sourceString, targetString);
134219
}
135220
currentColumnCount++;
136221
}

0 commit comments

Comments
 (0)