Skip to content

Commit 64760bb

Browse files
Aryan-VermaAryan-Verma
authored
Merge pull request #370 from data-integrations/postgres_e2e
Added infra setup and sanity check for Postgresql E2E
2 parents 335ba95 + 045dadd commit 64760bb

File tree

12 files changed

+549
-2
lines changed

12 files changed

+549
-2
lines changed

‎.github/workflows/e2e.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,10 @@ jobs:
9191
MYSQL_USERNAME:cdapio-github-builds/MYSQL_USERNAME
9292
MYSQL_PASSWORD:cdapio-github-builds/MYSQL_PASSWORD
9393
MYSQL_PORT:cdapio-github-builds/MYSQL_PORT
94+
POSTGRESQL_HOST:cdapio-github-builds/POSTGRESQL_HOST
95+
POSTGRESQL_USERNAME:cdapio-github-builds/POSTGRESQL_USERNAME
96+
POSTGRESQL_PASSWORD:cdapio-github-builds/POSTGRESQL_PASSWORD
97+
POSTGRESQL_PORT:cdapio-github-builds/POSTGRESQL_PORT
9498
9599
- name: Run required e2e tests
96100
if: github.event_name != 'workflow_dispatch' && github.event_name != 'push' && steps.filter.outputs.e2e-test == 'false'
@@ -108,6 +112,10 @@ jobs:
108112
MYSQL_USERNAME: ${{ steps.secrets.outputs.MYSQL_USERNAME }}
109113
MYSQL_PASSWORD: ${{ steps.secrets.outputs.MYSQL_PASSWORD }}
110114
MYSQL_PORT: ${{ steps.secrets.outputs.MYSQL_PORT }}
115+
POSTGRESQL_HOST: ${{ steps.secrets.outputs.POSTGRESQL_HOST }}
116+
POSTGRESQL_USERNAME: ${{ steps.secrets.outputs.POSTGRESQL_USERNAME }}
117+
POSTGRESQL_PASSWORD: ${{ steps.secrets.outputs.POSTGRESQL_PASSWORD }}
118+
POSTGRESQL_PORT: ${{ steps.secrets.outputs.POSTGRESQL_PORT }}
111119

112120
- name: Run all e2e tests
113121
if: github.event_name == 'workflow_dispatch' || github.event_name == 'push' || steps.filter.outputs.e2e-test == 'true'
@@ -125,6 +133,10 @@ jobs:
125133
MYSQL_USERNAME: ${{ steps.secrets.outputs.MYSQL_USERNAME }}
126134
MYSQL_PASSWORD: ${{ steps.secrets.outputs.MYSQL_PASSWORD }}
127135
MYSQL_PORT: ${{ steps.secrets.outputs.MYSQL_PORT }}
136+
POSTGRESQL_HOST: ${{ steps.secrets.outputs.POSTGRESQL_HOST }}
137+
POSTGRESQL_USERNAME: ${{ steps.secrets.outputs.POSTGRESQL_USERNAME }}
138+
POSTGRESQL_PASSWORD: ${{ steps.secrets.outputs.POSTGRESQL_PASSWORD }}
139+
POSTGRESQL_PORT: ${{ steps.secrets.outputs.POSTGRESQL_PORT }}
128140

129141
- name: Upload report
130142
uses: actions/upload-artifact@v3

‎postgresql-plugin/src/e2e-test/features/postgresql/Postgresql.feature‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#
2+
# Copyright © 2023 Cask Data, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
# use this file except in compliance with the License. You may obtain a copy of
6+
# the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
# License for the specific language governing permissions and limitations under
14+
# the License.
15+
#
16+
17+
@Postgresql
18+
Feature: PostgreSQL - Verify Postgresql source data transfer
19+
@POSTGRESQL_SOURCE_TEST @POSTGRESQL_SINK_TEST @Postgresql_Required
20+
Scenario: To verify data is getting transferred from PostgreSQL to PostgreSQL successfully
21+
Given Open Datafusion Project to configure pipeline
22+
When Expand Plugin group in the LHS plugins list: "Source"
23+
When Select plugin: "PostgreSQL" from the plugins list as: "Source"
24+
When Expand Plugin group in the LHS plugins list: "Sink"
25+
When Select plugin: "PostgreSQL" from the plugins list as: "Sink"
26+
Then Connect plugins: "PostgreSQL" and "PostgreSQL2" to establish connection
27+
Then Navigate to the properties page of plugin: "PostgreSQL"
28+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
29+
Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
30+
Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
31+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
32+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
33+
Then Enter input plugin property: "referenceName" with value: "sourceRef"
34+
Then Replace input plugin property: "database" with value: "databaseName"
35+
Then Enter textarea plugin property: "importQuery" with value: "selectQuery"
36+
Then Click on the Get Schema button
37+
Then Verify the Output Schema matches the Expected Schema: "datatypesSchema"
38+
Then Validate "PostgreSQL" plugin properties
39+
Then Close the Plugin Properties page
40+
Then Navigate to the properties page of plugin: "PostgreSQL2"
41+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
42+
Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
43+
Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
44+
Then Replace input plugin property: "database" with value: "databaseName"
45+
Then Replace input plugin property: "tableName" with value: "targetTable"
46+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
47+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
48+
Then Enter input plugin property: "referenceName" with value: "targetRef"
49+
Then Replace input plugin property: "dbSchemaName" with value: "schema"
50+
Then Validate "PostgreSQL2" plugin properties
51+
Then Close the Plugin Properties page
52+
Then Save the pipeline
53+
Then Preview and run the pipeline
54+
Then Verify the preview of pipeline is "success"
55+
Then Click on preview data for PostgreSQL sink
56+
Then Close the preview data
57+
Then Deploy the pipeline
58+
Then Run the Pipeline in Runtime
59+
Then Wait till pipeline is in running state
60+
Then Open and capture logs
61+
Then Verify the pipeline status is "Succeeded"
62+
Then Validate the values of records transferred to target table is equal to the values from source table

‎postgresql-plugin/src/e2e-test/java/io/cdap/plugin/PostgresqlClient.java‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
/*
2+
* Copyright © 2023 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
package io.cdap.plugin;
18+
19+
import io.cdap.e2e.utils.PluginPropertyUtils;
20+
import org.junit.Assert;
21+
22+
import java.sql.Connection;
23+
import java.sql.DriverManager;
24+
import java.sql.ResultSet;
25+
import java.sql.ResultSetMetaData;
26+
import java.sql.SQLException;
27+
import java.sql.Statement;
28+
import java.sql.Timestamp;
29+
import java.sql.Types;
30+
import java.util.Date;
31+
import java.util.GregorianCalendar;
32+
import java.util.TimeZone;
33+
34+
/**
35+
* PostgreSQL client.
36+
*/
37+
public class PostgresqlClient {
38+
private static final String database = PluginPropertyUtils.pluginProp("databaseName");
39+
40+
private static Connection getPostgresqlConnection() throws SQLException, ClassNotFoundException {
41+
Class.forName("org.postgresql.Driver");
42+
return DriverManager.getConnection("jdbc:postgresql://" + System.getenv("POSTGRESQL_HOST") + ":" +
43+
System.getenv("POSTGRESQL_PORT") + "/" + database,
44+
System.getenv("POSTGRESQL_USERNAME"),
45+
System.getenv("POSTGRESQL_PASSWORD"));
46+
}
47+
48+
public static int countRecord(String table, String schema) throws SQLException, ClassNotFoundException {
49+
String countQuery = "SELECT COUNT(*) as total FROM " + schema + "." + table;
50+
try (Connection connect = getPostgresqlConnection();
51+
Statement statement = connect.createStatement();
52+
ResultSet rs = statement.executeQuery(countQuery)) {
53+
int num = 0;
54+
while (rs.next()) {
55+
num = (rs.getInt(1));
56+
}
57+
return num;
58+
}
59+
}
60+
61+
/**
62+
* Extracts entire data from source and target tables.
63+
* @param sourceTable table at the source side
64+
* @param targetTable table at the sink side
65+
* @return true if the values in source and target side are equal
66+
*/
67+
public static boolean validateRecordValues(String sourceTable, String targetTable, String schema)
68+
throws SQLException, ClassNotFoundException {
69+
String getSourceQuery = "SELECT * FROM " + schema + "." + sourceTable;
70+
String getTargetQuery = "SELECT * FROM " + schema + "." + targetTable;
71+
try (Connection connect = getPostgresqlConnection()) {
72+
connect.setHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT);
73+
Statement statement1 = connect.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE,
74+
ResultSet.HOLD_CURSORS_OVER_COMMIT);
75+
Statement statement2 = connect.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE,
76+
ResultSet.HOLD_CURSORS_OVER_COMMIT);
77+
ResultSet rsSource = statement1.executeQuery(getSourceQuery);
78+
ResultSet rsTarget = statement2.executeQuery(getTargetQuery);
79+
return compareResultSetData(rsSource, rsTarget);
80+
}
81+
}
82+
83+
/**
84+
* Compares the result Set data in source table and sink table..
85+
* @param rsSource result set of the source table data
86+
* @param rsTarget result set of the target table data
87+
* @return true if rsSource matches rsTarget
88+
*/
89+
public static boolean compareResultSetData(ResultSet rsSource, ResultSet rsTarget) throws SQLException {
90+
ResultSetMetaData mdSource = rsSource.getMetaData();
91+
ResultSetMetaData mdTarget = rsTarget.getMetaData();
92+
int columnCountSource = mdSource.getColumnCount();
93+
int columnCountTarget = mdTarget.getColumnCount();
94+
Assert.assertEquals("Number of columns in source and target are not equal",
95+
columnCountSource, columnCountTarget);
96+
while (rsSource.next() && rsTarget.next()) {
97+
int currentColumnCount = 1;
98+
while (currentColumnCount <= columnCountSource) {
99+
String columnTypeName = mdSource.getColumnTypeName(currentColumnCount);
100+
int columnType = mdSource.getColumnType(currentColumnCount);
101+
String columnName = mdSource.getColumnName(currentColumnCount);
102+
if (columnType == Types.TIMESTAMP) {
103+
GregorianCalendar gc = new GregorianCalendar(TimeZone.getTimeZone("UTC"));
104+
gc.setGregorianChange(new Date(Long.MIN_VALUE));
105+
Timestamp sourceTS = rsSource.getTimestamp(currentColumnCount, gc);
106+
Timestamp targetTS = rsTarget.getTimestamp(currentColumnCount, gc);
107+
Assert.assertEquals(String.format("Different values found for column : %s", columnName), sourceTS, targetTS);
108+
} else {
109+
String sourceString = rsSource.getString(currentColumnCount);
110+
String targetString = rsTarget.getString(currentColumnCount);
111+
Assert.assertEquals(String.format("Different values found for column : %s", columnName),
112+
sourceString, targetString);
113+
}
114+
currentColumnCount++;
115+
}
116+
}
117+
Assert.assertFalse("Number of rows in Source table is greater than the number of rows in Target table",
118+
rsSource.next());
119+
Assert.assertFalse("Number of rows in Target table is greater than the number of rows in Source table",
120+
rsTarget.next());
121+
return true;
122+
}
123+
124+
public static void createSourceTable(String sourceTable, String schema) throws SQLException, ClassNotFoundException {
125+
try (Connection connect = getPostgresqlConnection();
126+
Statement statement = connect.createStatement()) {
127+
String datatypesColumns = PluginPropertyUtils.pluginProp("datatypesColumns");
128+
String createSourceTableQuery = "CREATE TABLE " + schema + "." + sourceTable + datatypesColumns;
129+
statement.executeUpdate(createSourceTableQuery);
130+
131+
// Insert dummy data.
132+
String datatypesValues = PluginPropertyUtils.pluginProp("datatypesValues");
133+
String datatypesColumnsList = PluginPropertyUtils.pluginProp("datatypesColumnsList");
134+
statement.executeUpdate("INSERT INTO " + schema + "." + sourceTable + " " + datatypesColumnsList + " " +
135+
datatypesValues);
136+
}
137+
}
138+
139+
public static void createTargetTable(String targetTable, String schema) throws SQLException, ClassNotFoundException {
140+
try (Connection connect = getPostgresqlConnection();
141+
Statement statement = connect.createStatement()) {
142+
String datatypesColumns = PluginPropertyUtils.pluginProp("datatypesColumns");
143+
String createTargetTableQuery = "CREATE TABLE " + schema + "." + targetTable + " " + datatypesColumns;
144+
statement.executeUpdate(createTargetTableQuery);
145+
}
146+
}
147+
148+
public static void dropTables(String[] tables, String schema) throws SQLException, ClassNotFoundException {
149+
try (Connection connect = getPostgresqlConnection();
150+
Statement statement = connect.createStatement()) {
151+
for (String table : tables) {
152+
String dropTableQuery = "Drop Table " + schema + "." + table;
153+
statement.executeUpdate(dropTableQuery);
154+
}
155+
}
156+
}
157+
}

‎postgresql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
/*
2+
* Copyright © 2023 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
package io.cdap.plugin.common.stepsdesign;
18+
19+
import io.cdap.e2e.utils.PluginPropertyUtils;
20+
import io.cdap.plugin.PostgresqlClient;
21+
import io.cucumber.java.After;
22+
import io.cucumber.java.Before;
23+
import org.apache.commons.lang3.RandomStringUtils;
24+
25+
import java.sql.SQLException;
26+
27+
/**
28+
* POSTGRESQL test hooks.
29+
*/
30+
public class TestSetupHooks {
31+
32+
@Before(order = 1)
33+
public static void setTableName() {
34+
String randomString = RandomStringUtils.randomAlphabetic(10).toLowerCase();
35+
String sourceTableName = String.format("sourcetable_%s", randomString);
36+
String targetTableName = String.format("targettable_%s", randomString);
37+
PluginPropertyUtils.addPluginProp("sourceTable", sourceTableName);
38+
PluginPropertyUtils.addPluginProp("targetTable", targetTableName);
39+
String schema = PluginPropertyUtils.pluginProp("schema");
40+
PluginPropertyUtils.addPluginProp("selectQuery",
41+
String.format("select * from %s.%s", schema, sourceTableName));
42+
}
43+
44+
@Before(order = 2, value = "@POSTGRESQL_SOURCE_TEST")
45+
public static void createTables() throws SQLException, ClassNotFoundException {
46+
PostgresqlClient.createSourceTable(PluginPropertyUtils.pluginProp("sourceTable"),
47+
PluginPropertyUtils.pluginProp("schema"));
48+
PostgresqlClient.createTargetTable(PluginPropertyUtils.pluginProp("targetTable"),
49+
PluginPropertyUtils.pluginProp("schema"));
50+
}
51+
52+
@After(order = 2, value = "@POSTGRESQL_SINK_TEST")
53+
public static void dropTables() throws SQLException, ClassNotFoundException {
54+
PostgresqlClient.dropTables(new String[]{PluginPropertyUtils.pluginProp("sourceTable"),
55+
PluginPropertyUtils.pluginProp("targetTable")},
56+
PluginPropertyUtils.pluginProp("schema"));
57+
}
58+
}

‎postgresql-plugin/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/package-info.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/*
2+
* Copyright © 2023 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
/**
18+
* Package contains the stepsDesign for common features.
19+
*/
20+
package io.cdap.plugin.common.stepsdesign;

‎postgresql-plugin/src/e2e-test/java/io/cdap/plugin/package-info.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/*
2+
* Copyright © 2023 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
/**
18+
* Package contains Postgresql client.
19+
*/
20+
package io.cdap.plugin;

‎postgresql-plugin/src/e2e-test/java/io/cdap/plugin/postgresql/runners/TestRunner.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
* Copyright © 2023 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
package io.cdap.plugin.postgresql.runners;
17+
18+
import io.cucumber.junit.Cucumber;
19+
import io.cucumber.junit.CucumberOptions;
20+
import org.junit.runner.RunWith;
21+
22+
/**
23+
* Test Runner to execute Postgresql plugin test cases.
24+
*/
25+
@RunWith(Cucumber.class)
26+
@CucumberOptions(
27+
features = {"src/e2e-test/features"},
28+
glue = {"stepsdesign", "io.cdap.plugin.common.stepsdesign", "io.cdap.plugin.postgresql.stepsdesign"},
29+
tags = {"@Postgresql"},
30+
plugin = {"pretty", "html:target/cucumber-html-report/postgresql",
31+
"json:target/cucumber-reports/cucumber-postgresql.json",
32+
"junit:target/cucumber-reports/cucumber-postgresql.xml"}
33+
)
34+
public class TestRunner {
35+
}

0 commit comments

Comments
 (0)