Skip to content

Commit 9c80909

Browse files
Aryan-VermaAryan-Verma
committed
Added e2e test for oracle plugin
1 parent 34938cf commit 9c80909

File tree

13 files changed

+455
-1
lines changed

13 files changed

+455
-1
lines changed

‎.github/workflows/e2e.yml‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,9 @@ jobs:
4040
)
4141
strategy:
4242
matrix:
43-
module: [mysql-plugin, postgresql-plugin]
43+
module: [mysql-plugin, postgresql-plugin, oracle-plugin]
4444
fail-fast: false
45+
4546
steps:
4647
# Pinned 1.0.0 version
4748
- uses: actions/checkout@v3
@@ -73,13 +74,33 @@ jobs:
7374
restore-keys: |
7475
${{ runner.os }}-maven-${{ github.workflow }}
7576
77+
- name: Get Secrets from GCP Secret Manager
78+
id: secrets
79+
uses: 'google-github-actions/get-secretmanager-secrets@v0'
80+
with:
81+
secrets: |-
82+
ORACLE_HOST:cdapio-github-builds/ORACLE_HOST
83+
ORACLE_USERNAME:cdapio-github-builds/ORACLE_USERNAME
84+
ORACLE_PASSWORD:cdapio-github-builds/ORACLE_PASSWORD
85+
ORACLE_PORT:cdapio-github-builds/ORACLE_PORT
86+
7687
- name: Run required e2e tests
7788
if: github.event_name != 'workflow_dispatch' && github.event_name != 'push' && steps.filter.outputs.e2e-test == 'false'
7889
run: python3 e2e/src/main/scripts/run_e2e_test.py --module ${{ matrix.module }} --testRunner TestRunnerRequired.java
90+
env:
91+
ORACLE_HOST: ${{ steps.secrets.outputs.ORACLE_HOST }}
92+
ORACLE_USERNAME: ${{ steps.secrets.outputs.ORACLE_USERNAME }}
93+
ORACLE_PASSWORD: ${{ steps.secrets.outputs.ORACLE_PASSWORD }}
94+
ORACLE_PORT: ${{ steps.secrets.outputs.ORACLE_PORT }}
7995

8096
- name: Run all e2e tests
8197
if: github.event_name == 'workflow_dispatch' || github.event_name == 'push' || steps.filter.outputs.e2e-test == 'true'
8298
run: python3 e2e/src/main/scripts/run_e2e_test.py --module ${{ matrix.module }}
99+
env:
100+
ORACLE_HOST: ${{ steps.secrets.outputs.ORACLE_HOST }}
101+
ORACLE_USERNAME: ${{ steps.secrets.outputs.ORACLE_USERNAME }}
102+
ORACLE_PASSWORD: ${{ steps.secrets.outputs.ORACLE_PASSWORD }}
103+
ORACLE_PORT: ${{ steps.secrets.outputs.ORACLE_PORT }}
83104

84105
- name: Upload report
85106
uses: actions/upload-artifact@v3

‎oracle-plugin/src/e2e-test/features/oracle/Oracle.feature‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
#
2+
# Copyright © 2023 Cask Data, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
# use this file except in compliance with the License. You may obtain a copy of
6+
# the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
# License for the specific language governing permissions and limitations under
14+
# the License.
15+
#
16+
17+
@Oracle
18+
Feature: Oracle - Verify Oracle source data transfer
19+
@ORACLE_SOURCE_TEST @ORACLE_SINK_TEST @Oracle_Required
20+
Scenario: To verify data is getting transferred from Oracle to Oracle successfully
21+
Given Open Datafusion Project to configure pipeline
22+
When Expand Plugin group in the LHS plugins list: "Source"
23+
When Select plugin: "Oracle" from the plugins list as: "Source"
24+
When Expand Plugin group in the LHS plugins list: "Sink"
25+
When Select plugin: "Oracle" from the plugins list as: "Sink"
26+
Then Connect plugins: "Oracle" and "Oracle2" to establish connection
27+
Then Navigate to the properties page of plugin: "Oracle"
28+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
29+
Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
30+
Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
31+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
32+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
33+
Then Select radio button plugin property: "connectionType" with value: "service"
34+
Then Select radio button plugin property: "role" with value: "sysdba"
35+
Then Enter input plugin property: "referenceName" with value: "sourceRef"
36+
Then Replace input plugin property: "database" with value: "databaseName"
37+
Then Enter textarea plugin property: "importQuery" with value: "selectQuery"
38+
Then Click on the Get Schema button
39+
Then Verify the Output Schema matches the Expected Schema: "outputSchema"
40+
Then Validate "Oracle" plugin properties
41+
Then Close the Plugin Properties page
42+
Then Navigate to the properties page of plugin: "Oracle2"
43+
Then Select dropdown plugin property: "select-jdbcPluginName" with option value: "driverName"
44+
Then Replace input plugin property: "host" with value: "host" for Credentials and Authorization related fields
45+
Then Replace input plugin property: "port" with value: "port" for Credentials and Authorization related fields
46+
Then Replace input plugin property: "database" with value: "databaseName"
47+
Then Replace input plugin property: "tableName" with value: "targetTable"
48+
Then Replace input plugin property: "dbSchemaName" with value: "schema"
49+
Then Replace input plugin property: "user" with value: "username" for Credentials and Authorization related fields
50+
Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields
51+
Then Enter input plugin property: "referenceName" with value: "targetRef"
52+
Then Select radio button plugin property: "connectionType" with value: "service"
53+
Then Select radio button plugin property: "role" with value: "sysdba"
54+
Then Validate "Oracle2" plugin properties
55+
Then Close the Plugin Properties page
56+
Then Save the pipeline
57+
Then Preview and run the pipeline
58+
Then Verify the preview of pipeline is "success"
59+
Then Click on preview data for Oracle sink
60+
Then Verify preview output schema matches the outputSchema captured in properties
61+
Then Close the preview data
62+
Then Deploy the pipeline
63+
Then Run the Pipeline in Runtime
64+
Then Wait till pipeline is in running state
65+
Then Open and capture logs
66+
Then Verify the pipeline status is "Succeeded"
67+
Then Get count of no of records transferred to target Oracle Table
68+
Then Validate records transferred to target table is equal to number of records from source table

‎oracle-plugin/src/e2e-test/java/io.cdap.plugin/OracleClient.java‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
/*
2+
* Copyright © 2023 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
package io.cdap.plugin;
18+
19+
import io.cdap.e2e.utils.PluginPropertyUtils;
20+
21+
import java.sql.Connection;
22+
import java.sql.DriverManager;
23+
import java.sql.ResultSet;
24+
import java.sql.SQLException;
25+
import java.sql.Statement;
26+
import java.util.TimeZone;
27+
28+
/**
29+
* Oracle client.
30+
*/
31+
public class OracleClient {
32+
33+
private static Connection getOracleConnection() throws SQLException, ClassNotFoundException {
34+
TimeZone timezone = TimeZone.getTimeZone("UTC");
35+
TimeZone.setDefault(timezone);
36+
Class.forName("oracle.jdbc.driver.OracleDriver");
37+
String databaseName = PluginPropertyUtils.pluginProp("databaseName");
38+
return DriverManager.getConnection("jdbc:oracle:thin:@//" + System.getenv("ORACLE_HOST")
39+
+ ":" + System.getenv("ORACLE_PORT") + "/" + databaseName,
40+
System.getenv("ORACLE_USERNAME"), System.getenv("ORACLE_PASSWORD"));
41+
}
42+
43+
public static int countRecord(String table, String schema) throws SQLException, ClassNotFoundException {
44+
String countQuery = "SELECT COUNT(*) as total FROM " + schema + "." + table;
45+
try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement();
46+
ResultSet rs = statement.executeQuery(countQuery)) {
47+
int num = 0;
48+
while (rs.next()) {
49+
num = (rs.getInt(1));
50+
}
51+
return num;
52+
}
53+
}
54+
55+
public static void createSourceTable(String sourceTable, String schema) throws SQLException, ClassNotFoundException {
56+
try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) {
57+
String createSourceTableQuery = "CREATE TABLE " + schema + "." + sourceTable +
58+
"(ID number(38), LASTNAME varchar2(100))";
59+
statement.executeUpdate(createSourceTableQuery);
60+
61+
// Insert dummy data.
62+
statement.executeUpdate("INSERT INTO " + schema + "." + sourceTable + " (ID, LASTNAME)" +
63+
" VALUES (1, 'Shelby')");
64+
statement.executeUpdate("INSERT INTO " + schema + "." + sourceTable + " (ID, LASTNAME)" +
65+
" VALUES (2, 'Simpson')");
66+
}
67+
}
68+
69+
public static void createTargetTable(String targetTable, String schema) throws SQLException, ClassNotFoundException {
70+
try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) {
71+
String createTargetTableQuery = "CREATE TABLE " + schema + "." + targetTable +
72+
"(ID number(38), LASTNAME varchar2(100))";
73+
statement.executeUpdate(createTargetTableQuery);
74+
}
75+
}
76+
77+
public static void deleteTables(String schema, String[] tables)
78+
throws SQLException, ClassNotFoundException {
79+
try (Connection connect = getOracleConnection(); Statement statement = connect.createStatement()) {
80+
for (String table : tables) {
81+
String dropTableQuery = "DROP TABLE " + schema + "." + table;
82+
statement.execute(dropTableQuery);
83+
}
84+
}
85+
}
86+
}

‎oracle-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/TestSetupHooks.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/*
2+
* Copyright © 2023 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
package io.cdap.plugin.common.stepsdesign;
18+
19+
import io.cdap.e2e.utils.PluginPropertyUtils;
20+
import io.cdap.plugin.OracleClient;
21+
import io.cucumber.java.After;
22+
import io.cucumber.java.Before;
23+
24+
import java.sql.SQLException;
25+
26+
/**
27+
* Oracle test hooks.
28+
*/
29+
public class TestSetupHooks {
30+
31+
@Before(order = 1, value = "@ORACLE_SOURCE_TEST")
32+
public static void setSelectQuery() {
33+
String sourceTable = PluginPropertyUtils.pluginProp("sourceTable");
34+
String schema = PluginPropertyUtils.pluginProp("schema");
35+
PluginPropertyUtils.addPluginProp("selectQuery",
36+
PluginPropertyUtils.pluginProp("selectQuery").
37+
replace("${table}", sourceTable).replace("${schema}", schema));
38+
}
39+
40+
@Before(order = 2, value = "@ORACLE_SOURCE_TEST")
41+
public static void createTables() throws SQLException, ClassNotFoundException {
42+
OracleClient.createSourceTable(PluginPropertyUtils.pluginProp("sourceTable"),
43+
PluginPropertyUtils.pluginProp("schema"));
44+
OracleClient.createTargetTable(PluginPropertyUtils.pluginProp("targetTable"),
45+
PluginPropertyUtils.pluginProp("schema"));
46+
}
47+
48+
@After(order = 1, value = "@ORACLE_SINK_TEST")
49+
public static void dropTables() throws SQLException, ClassNotFoundException {
50+
OracleClient.deleteTables(PluginPropertyUtils.pluginProp("schema"),
51+
new String[]{PluginPropertyUtils.pluginProp("sourceTable"),
52+
PluginPropertyUtils.pluginProp("targetTable")});
53+
}
54+
55+
}

‎oracle-plugin/src/e2e-test/java/io.cdap.plugin/common.stepsdesign/package-info.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/*
2+
* Copyright © 2023 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
/**
18+
* Package contains the stepDesign for common features.
19+
*/
20+
package io.cdap.plugin.common.stepsdesign;

‎oracle-plugin/src/e2e-test/java/io.cdap.plugin/oracle/runners/TestRunner.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
* Copyright © 2023 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
package io.cdap.plugin.oracle.runners;
17+
18+
import io.cucumber.junit.Cucumber;
19+
import io.cucumber.junit.CucumberOptions;
20+
import org.junit.runner.RunWith;
21+
22+
/**
23+
* Test Runner to execute Oracle plugin test cases.
24+
*/
25+
@RunWith(Cucumber.class)
26+
@CucumberOptions(
27+
features = {"src/e2e-test/features"},
28+
glue = {"stepsdesign", "io.cdap.plugin.common.stepsdesign", "io.cdap.plugin.oracle.stepsdesign"},
29+
tags = {"@Oracle"},
30+
plugin = {"pretty", "html:target/cucumber-html-report/oracle",
31+
"json:target/cucumber-reports/cucumber-oracle.json",
32+
"junit:target/cucumber-reports/cucumber-oracle.xml"}
33+
)
34+
public class TestRunner {
35+
}

‎oracle-plugin/src/e2e-test/java/io.cdap.plugin/oracle/runners/TestRunnerRequired.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
* Copyright © 2023 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
package io.cdap.plugin.oracle.runners;
17+
18+
import io.cucumber.junit.Cucumber;
19+
import io.cucumber.junit.CucumberOptions;
20+
import org.junit.runner.RunWith;
21+
22+
/**
23+
* Test Runner to execute Oracle plugin test cases.
24+
*/
25+
@RunWith(Cucumber.class)
26+
@CucumberOptions(
27+
features = {"src/e2e-test/features"},
28+
glue = {"stepsdesign", "io.cdap.plugin.common.stepsdesign", "io.cdap.plugin.oracle.stepsdesign"},
29+
tags = {"@Oracle_Required"},
30+
plugin = {"pretty", "html:target/cucumber-html-report/oracle-required",
31+
"json:target/cucumber-reports/cucumber-oracle-required.json",
32+
"junit:target/cucumber-reports/cucumber-oracle-required.xml"}
33+
)
34+
public class TestRunnerRequired {
35+
}

‎oracle-plugin/src/e2e-test/java/io.cdap.plugin/oracle/runners/package-info.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/*
2+
* Copyright © 2023 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
/**
18+
* Package contains the runners for Oracle features.
19+
*/
20+
package io.cdap.plugin.oracle.runners;

0 commit comments

Comments
 (0)