Skip to content

Commit 335ba95

Browse files
Nikitapaliwal123Nikitapaliwal123
authored
Created basic infra of e2e tests for SQL Server plugin (#363)
* Created basic infra of e2e tests for SQL Server plugin * Created basic infra of e2e tests for SQL Server plugin * Created basic infra of e2e tests for SQL Server plugin * Created basic infra of e2e tests for SQL Server plugin * Created basic infra of e2e tests for SQL Server plugin * Basic e2e infra for mssql * e2e infra
1 parent 6994e09 commit 335ba95

File tree

15 files changed

+500
-7
lines changed

15 files changed

+500
-7
lines changed

‎.github/workflows/e2e.yml‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ jobs:
4040
)
4141
strategy:
4242
matrix:
43-
module: [mysql-plugin, postgresql-plugin, oracle-plugin]
43+
module: [mysql-plugin, postgresql-plugin, oracle-plugin, mssql-plugin]
4444
fail-fast: false
4545

4646
steps:
@@ -83,6 +83,10 @@ jobs:
8383
ORACLE_USERNAME:cdapio-github-builds/ORACLE_USERNAME
8484
ORACLE_PASSWORD:cdapio-github-builds/ORACLE_PASSWORD
8585
ORACLE_PORT:cdapio-github-builds/ORACLE_PORT
86+
MSSQL_HOST:cdapio-github-builds/MSSQL_HOST
87+
MSSQL_USERNAME:cdapio-github-builds/MSSQL_USERNAME
88+
MSSQL_PASSWORD:cdapio-github-builds/MSSQL_PASSWORD
89+
MSSQL_PORT:cdapio-github-builds/MSSQL_PORT
8690
MYSQL_HOST:cdapio-github-builds/MYSQL_HOST
8791
MYSQL_USERNAME:cdapio-github-builds/MYSQL_USERNAME
8892
MYSQL_PASSWORD:cdapio-github-builds/MYSQL_PASSWORD
@@ -96,6 +100,10 @@ jobs:
96100
ORACLE_USERNAME: ${{ steps.secrets.outputs.ORACLE_USERNAME }}
97101
ORACLE_PASSWORD: ${{ steps.secrets.outputs.ORACLE_PASSWORD }}
98102
ORACLE_PORT: ${{ steps.secrets.outputs.ORACLE_PORT }}
103+
MSSQL_HOST: ${{ steps.secrets.outputs.MSSQL_HOST }}
104+
MSSQL_USERNAME: ${{ steps.secrets.outputs.MSSQL_USERNAME }}
105+
MSSQL_PASSWORD: ${{ steps.secrets.outputs.MSSQL_PASSWORD }}
106+
MSSQL_PORT: ${{ steps.secrets.outputs.MSSQL_PORT }}
99107
MYSQL_HOST: ${{ steps.secrets.outputs.MYSQL_HOST }}
100108
MYSQL_USERNAME: ${{ steps.secrets.outputs.MYSQL_USERNAME }}
101109
MYSQL_PASSWORD: ${{ steps.secrets.outputs.MYSQL_PASSWORD }}
@@ -109,6 +117,10 @@ jobs:
109117
ORACLE_USERNAME: ${{ steps.secrets.outputs.ORACLE_USERNAME }}
110118
ORACLE_PASSWORD: ${{ steps.secrets.outputs.ORACLE_PASSWORD }}
111119
ORACLE_PORT: ${{ steps.secrets.outputs.ORACLE_PORT }}
120+
MSSQL_HOST: ${{ steps.secrets.outputs.MSSQL_HOST }}
121+
MSSQL_USERNAME: ${{ steps.secrets.outputs.MSSQL_USERNAME }}
122+
MSSQL_PASSWORD: ${{ steps.secrets.outputs.MSSQL_PASSWORD }}
123+
MSSQL_PORT: ${{ steps.secrets.outputs.MSSQL_PORT }}
112124
MYSQL_HOST: ${{ steps.secrets.outputs.MYSQL_HOST }}
113125
MYSQL_USERNAME: ${{ steps.secrets.outputs.MYSQL_USERNAME }}
114126
MYSQL_PASSWORD: ${{ steps.secrets.outputs.MYSQL_PASSWORD }}

‎mssql-plugin/pom.xml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -81,12 +81,6 @@
8181
<artifactId>cdap-api</artifactId>
8282
<scope>provided</scope>
8383
</dependency>
84-
<dependency>
85-
<groupId>com.microsoft.sqlserver</groupId>
86-
<artifactId>mssql-jdbc</artifactId>
87-
<version>8.2.1.jre8</version>
88-
<scope>test</scope>
89-
</dependency>
9084
<dependency>
9185
<groupId>org.jetbrains</groupId>
9286
<artifactId>annotations</artifactId>

‎mssql-plugin/src/e2e-test/features/mssql/Mssql.feature‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
#
2+
# Copyright © 2023 Cask Data, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
# use this file except in compliance with the License. You may obtain a copy of
6+
# the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
# License for the specific language governing permissions and limitations under
14+
# the License.
15+
#
16+
17+
@Mssql
18+
Feature: Mssql - Verify Mssql source data transfer
19+
@MSSQL_SOURCE_TEST @MSSQL_SINK_TEST @Mssql_Required
20+
Scenario: To verify data is getting transferred from Mssql to Mssql successfully
21+
Given Open Datafusion Project to configure pipeline
22+
When Expand Plugin group in the LHS plugins list: "Source"
23+
When Select plugin: "SQL Server" from the plugins list as: "Source"
24+
When Expand Plugin group in the LHS plugins list: "Sink"
25+
When Select plugin: "SQL Server" from the plugins list as: "Sink"
26+
Then Connect plugins: "SQL Server" and "SQL Server2" to establish connection
27+
Then Navigate to the properties page of plugin: "SQL Server"
28+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
29+
Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
30+
Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
31+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
32+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
33+
Then Enter input plugin property: "referenceName" with value: "sourceRef"
34+
Then Replace input plugin property: "database" with value: "databaseName"
35+
Then Enter textarea plugin property: "importQuery" with value: "selectQuery"
36+
Then Click on the Get Schema button
37+
Then Verify the Output Schema matches the Expected Schema: "outputSchema"
38+
Then Validate "SQL Server" plugin properties
39+
Then Close the Plugin Properties page
40+
Then Navigate to the properties page of plugin: "SQL Server2"
41+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
42+
Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
43+
Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
44+
Then Replace input plugin property: "database" with value: "databaseName"
45+
Then Replace input plugin property: "tableName" with value: "targetTable"
46+
Then Replace input plugin property: "dbSchemaName" with value: "schema"
47+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
48+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
49+
Then Enter input plugin property: "referenceName" with value: "targetRef"
50+
Then Validate "SQL Server2" plugin properties
51+
Then Close the Plugin Properties page
52+
Then Save the pipeline
53+
Then Preview and run the pipeline
54+
Then Verify the preview of pipeline is "success"
55+
Then Click on preview data for Mssql sink
56+
Then Verify preview output schema matches the outputSchema captured in properties
57+
Then Close the preview data
58+
Then Deploy the pipeline
59+
Then Run the Pipeline in Runtime
60+
Then Wait till pipeline is in running state
61+
Then Open and capture logs
62+
Then Verify the pipeline status is "Succeeded"
63+
Then Validate records transferred to target table are equal to number of records from the source table
64+
65+

‎mssql-plugin/src/e2e-test/java/io.cdap.plugin/MssqlClient.java‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
/*
2+
* Copyright © 2023 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
package io.cdap.plugin;
18+
19+
import io.cdap.e2e.utils.PluginPropertyUtils;
20+
import org.junit.Assert;
21+
22+
import java.sql.Blob;
23+
import java.sql.Clob;
24+
import java.sql.Connection;
25+
import java.sql.DriverManager;
26+
import java.sql.ResultSet;
27+
import java.sql.ResultSetMetaData;
28+
import java.sql.SQLException;
29+
import java.sql.Statement;
30+
import java.sql.Timestamp;
31+
import java.sql.Types;
32+
import java.util.Arrays;
33+
import java.util.Date;
34+
import java.util.GregorianCalendar;
35+
import java.util.TimeZone;
36+
37+
/**
38+
* Mssql client.
39+
*/
40+
public class MssqlClient {
41+
42+
private static Connection getMssqlConnection() throws SQLException, ClassNotFoundException {
43+
TimeZone timezone = TimeZone.getTimeZone("UTC");
44+
TimeZone.setDefault(timezone);
45+
Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver");
46+
String databaseName = PluginPropertyUtils.pluginProp("databaseName");
47+
return DriverManager.getConnection("jdbc:sqlserver://" + System.getenv("MSSQL_HOST")
48+
+ ":" + System.getenv("MSSQL_PORT") + ";databaseName=" + databaseName,
49+
System.getenv("MSSQL_USERNAME"), System.getenv("MSSQL_PASSWORD"));
50+
}
51+
52+
public static int countRecord(String table, String schema) throws SQLException, ClassNotFoundException {
53+
String countQuery = "SELECT COUNT(*) as total FROM " + schema + "." + table;
54+
try (Connection connect = getMssqlConnection(); Statement statement = connect.createStatement();
55+
ResultSet rs = statement.executeQuery(countQuery)) {
56+
int num = 0;
57+
while (rs.next()) {
58+
num = (rs.getInt(1));
59+
}
60+
return num;
61+
}
62+
}
63+
64+
public static void createSourceTable(String sourceTable, String schema) throws SQLException,
65+
ClassNotFoundException {
66+
try (Connection connect = getMssqlConnection(); Statement statement = connect.createStatement()) {
67+
String createSourceTableQuery = "CREATE TABLE " + schema + "." + sourceTable +
68+
"(ID varchar(100), LASTNAME varchar(100))";
69+
statement.executeUpdate(createSourceTableQuery);
70+
71+
// Insert dummy data.
72+
statement.executeUpdate("INSERT INTO " + schema + "." + sourceTable + " (ID, LASTNAME)" +
73+
" VALUES ('id1', 'Shelby')");
74+
statement.executeUpdate("INSERT INTO " + schema + "." + sourceTable + " (ID, LASTNAME)" +
75+
" VALUES ('id2', 'Simpson')");
76+
}
77+
}
78+
79+
public static void createTargetTable(String targetTable, String schema) throws SQLException,
80+
ClassNotFoundException {
81+
try (Connection connect = getMssqlConnection(); Statement statement = connect.createStatement()) {
82+
String createTargetTableQuery = "CREATE TABLE " + schema + "." + targetTable +
83+
"(ID varchar(100), LASTNAME varchar(100))";
84+
statement.executeUpdate(createTargetTableQuery);
85+
}
86+
}
87+
88+
public static void deleteTables(String schema, String[] tables)
89+
throws SQLException, ClassNotFoundException {
90+
try (Connection connect = getMssqlConnection(); Statement statement = connect.createStatement()) {
91+
for (String table : tables) {
92+
String dropTableQuery = "DROP TABLE " + schema + "." + table;
93+
statement.execute(dropTableQuery);
94+
}
95+
}
96+
}
97+
98+
public static boolean validateRecordValues(String schema, String sourceTable, String targetTable)
99+
throws SQLException, ClassNotFoundException {
100+
String getSourceQuery = "SELECT * FROM " + schema + "." + sourceTable;
101+
String getTargetQuery = "SELECT * FROM " + schema + "." + targetTable;
102+
try (Connection connect = getMssqlConnection()) {
103+
connect.setHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT);
104+
Statement statement1 = connect.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE,
105+
ResultSet.HOLD_CURSORS_OVER_COMMIT);
106+
Statement statement2 = connect.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE,
107+
ResultSet.HOLD_CURSORS_OVER_COMMIT);
108+
ResultSet rsSource = statement1.executeQuery(getSourceQuery);
109+
ResultSet rsTarget = statement2.executeQuery(getTargetQuery);
110+
return compareResultSetData(rsSource, rsTarget);
111+
}
112+
}
113+
114+
private static boolean compareResultSetData(ResultSet rsSource, ResultSet rsTarget) throws SQLException {
115+
ResultSetMetaData mdSource = rsSource.getMetaData();
116+
ResultSetMetaData mdTarget = rsTarget.getMetaData();
117+
int columnCountSource = mdSource.getColumnCount();
118+
int columnCountTarget = mdTarget.getColumnCount();
119+
Assert.assertEquals("Number of columns in source and target are not equal",
120+
columnCountSource, columnCountTarget);
121+
while (rsSource.next() && rsTarget.next()) {
122+
int currentColumnCount = 1;
123+
while (currentColumnCount <= columnCountSource) {
124+
String columnTypeName = mdSource.getColumnTypeName(currentColumnCount);
125+
int columnType = mdSource.getColumnType(currentColumnCount);
126+
String columnName = mdSource.getColumnName(currentColumnCount);
127+
switch (columnType) {
128+
129+
default:
130+
String sourceString = rsSource.getString(currentColumnCount);
131+
String targetString = rsTarget.getString(currentColumnCount);
132+
Assert.assertTrue(String.format("Different values found for column : %s", columnName),
133+
sourceString.equals(targetString));
134+
}
135+
currentColumnCount++;
136+
}
137+
}
138+
Assert.assertFalse("Number of rows in Source table is greater than the number of rows in Target table",
139+
rsSource.next());
140+
Assert.assertFalse("Number of rows in Target table is greater than the number of rows in Source table",
141+
rsTarget.next());
142+
return true;
143+
}
144+
}

‎mssql-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/TestSetupHooks.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* Copyright © 2023 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
package io.cdap.plugin.common.stepsdesign;
18+
19+
import io.cdap.e2e.utils.PluginPropertyUtils;
20+
import io.cdap.plugin.MssqlClient;
21+
import io.cucumber.java.After;
22+
import io.cucumber.java.Before;
23+
import org.apache.commons.lang3.RandomStringUtils;
24+
25+
import java.sql.SQLException;
26+
27+
/**
28+
* Mssql test hooks.
29+
*/
30+
public class TestSetupHooks {
31+
32+
@Before(order = 1)
33+
public static void setTableName() {
34+
String randomString = RandomStringUtils.randomAlphabetic(10).toUpperCase();
35+
String sourceTableName = String.format("SOURCETABLE_%s", randomString);
36+
String targetTableName = String.format("TARGETTABLE_%s", randomString);
37+
PluginPropertyUtils.addPluginProp("sourceTable", sourceTableName);
38+
PluginPropertyUtils.addPluginProp("targetTable", targetTableName);
39+
String schema = PluginPropertyUtils.pluginProp("schema");
40+
PluginPropertyUtils.addPluginProp("selectQuery", String.format("select * from %s.%s", schema,
41+
sourceTableName));
42+
}
43+
44+
@Before(order = 2, value = "@MSSQL_SOURCE_TEST")
45+
public static void createTables() throws SQLException, ClassNotFoundException {
46+
MssqlClient.createSourceTable(PluginPropertyUtils.pluginProp("sourceTable"),
47+
PluginPropertyUtils.pluginProp("schema"));
48+
MssqlClient.createTargetTable(PluginPropertyUtils.pluginProp("targetTable"),
49+
PluginPropertyUtils.pluginProp("schema"));
50+
}
51+
52+
@After(order = 1, value = "@MSSQL_SINK_TEST")
53+
public static void dropTables() throws SQLException, ClassNotFoundException {
54+
MssqlClient.deleteTables(PluginPropertyUtils.pluginProp("schema"),
55+
new String[]{PluginPropertyUtils.pluginProp("sourceTable"),
56+
PluginPropertyUtils.pluginProp("targetTable")});
57+
}
58+
59+
}

‎mssql-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/package-info.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/*
2+
* Copyright © 2023 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
/**
18+
* Package contains the stepDesign for common features.
19+
*/
20+
package io.cdap.plugin.common.stepsdesign;

‎mssql-plugin/src/e2e-test/java/io.cdap.plugin/mssql/runners/TestRunner.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
* Copyright © 2023 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
package io.cdap.plugin.mssql.runners;
17+
18+
import io.cucumber.junit.Cucumber;
19+
import io.cucumber.junit.CucumberOptions;
20+
import org.junit.runner.RunWith;
21+
22+
/**
23+
* Test Runner to execute Mssql plugin test cases.
24+
*/
25+
@RunWith(Cucumber.class)
26+
@CucumberOptions(
27+
features = {"src/e2e-test/features"},
28+
glue = {"stepsdesign", "io.cdap.plugin.common.stepsdesign", "io.cdap.plugin.mssql.stepsdesign"},
29+
tags = {"@Mssql"},
30+
plugin = {"pretty", "html:target/cucumber-html-report/mssql",
31+
"json:target/cucumber-reports/cucumber-mssql.json",
32+
"junit:target/cucumber-reports/cucumber-mssql.xml"}
33+
)
34+
public class TestRunner {
35+
}

0 commit comments

Comments
 (0)