Skip to content

Commit f1fcdcb

Browse files
itsankit-googleitsankit-google
committed
Tests for cloudsql-mysql and cloudsql-postgresql connector
1 parent 6271d99 commit f1fcdcb

File tree

7 files changed

+261
-13
lines changed

7 files changed

+261
-13
lines changed

‎cloudsql-mysql-plugin/pom.xml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,54 @@
4343
<groupId>io.cdap.plugin</groupId>
4444
<artifactId>hydrator-common</artifactId>
4545
</dependency>
46+
47+
<!-- test dependencies -->
48+
<dependency>
49+
<groupId>io.cdap.plugin</groupId>
50+
<artifactId>database-commons</artifactId>
51+
<version>${project.version}</version>
52+
<type>test-jar</type>
53+
<scope>test</scope>
54+
</dependency>
4655
<dependency>
4756
<groupId>io.cdap.cdap</groupId>
4857
<artifactId>hydrator-test</artifactId>
58+
</dependency>
59+
<dependency>
60+
<groupId>io.cdap.cdap</groupId>
61+
<artifactId>cdap-data-pipeline2_2.11</artifactId>
62+
</dependency>
63+
<dependency>
64+
<groupId>junit</groupId>
65+
<artifactId>junit</artifactId>
66+
</dependency>
67+
<dependency>
68+
<groupId>io.cdap.cdap</groupId>
69+
<artifactId>cdap-api</artifactId>
70+
<scope>provided</scope>
71+
</dependency>
72+
<dependency>
73+
<groupId>org.mockito</groupId>
74+
<artifactId>mockito-core</artifactId>
75+
</dependency>
76+
<dependency>
77+
<groupId>org.jetbrains</groupId>
78+
<artifactId>annotations</artifactId>
79+
<version>RELEASE</version>
80+
<scope>compile</scope>
81+
</dependency>
82+
<!-- https://mvnrepository.com/artifact/com.google.cloud.sql/mysql-socket-factory-connector-j-8 -->
83+
<dependency>
84+
<groupId>com.google.cloud.sql</groupId>
85+
<artifactId>mysql-socket-factory-connector-j-8</artifactId>
86+
<version>1.0.10</version>
87+
<scope>test</scope>
88+
</dependency>
89+
<!-- https://mvnrepository.com/artifact/mysql/mysql-connector-java -->
90+
<dependency>
91+
<groupId>mysql</groupId>
92+
<artifactId>mysql-connector-java</artifactId>
93+
<version>8.0.11</version>
4994
<scope>test</scope>
5095
</dependency>
5196
</dependencies>

‎cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorConfig.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,17 @@ public class CloudSQLMySQLConnectorConfig extends AbstractDBConnectorConfig {
4848
@Nullable
4949
private String instanceType;
5050

51+
public CloudSQLMySQLConnectorConfig(String user, String password, String jdbcPluginName, String connectionArguments,
52+
String instanceType, String connectionName, String database) {
53+
this.user = user;
54+
this.password = password;
55+
this.jdbcPluginName = jdbcPluginName;
56+
this.connectionArguments = connectionArguments;
57+
this.instanceType = instanceType;
58+
this.connectionName = connectionName;
59+
this.database = database;
60+
}
61+
5162
public String getDatabase() {
5263
return database;
5364
}

‎cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnectorTest.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* Copyright © 2021 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
package io.cdap.plugin.cloudsql.mysql;
18+
19+
import io.cdap.plugin.db.connector.DBSpecificConnectorBaseTest;
20+
import org.apache.parquet.Strings;
21+
import org.junit.Assume;
22+
import org.junit.BeforeClass;
23+
import org.junit.Test;
24+
25+
import java.io.IOException;
26+
27+
/**
28+
* Test for CloudSQL MySQL Connector, it will only be run when below properties are provided:
29+
* -DinstanceType -- Whether the CloudSQL instance to connect to is private or public. If not provided, assumed public
30+
* -DconnectionName -- The CloudSQL instance to connect to. For a public instance, the connection string should be in
31+
* the format <PROJECT_ID>:<REGION>:<INSTANCE_NAME> which can be found in the instance overview page.
32+
* For a private instance, enter the internal IP address of the Compute Engine VM cloudsql proxy is running on.
33+
*/
34+
public class CloudSQLMySQLConnectorTest extends DBSpecificConnectorBaseTest {
35+
36+
private static final String JDBC_DRIVER_CLASS_NAME = "com.google.cloud.sql.mysql.SocketFactory";
37+
private static String instanceType;
38+
private static String connectionName;
39+
40+
@BeforeClass
41+
public static void doSetup() {
42+
// Certain properties need to be configured otherwise the whole tests will be skipped.
43+
String messageTemplate = "%s is not configured, please refer to javadoc of this class for details.";
44+
instanceType = System.getProperty("instanceType");
45+
if (Strings.isNullOrEmpty(instanceType)) {
46+
instanceType = "public";
47+
}
48+
connectionName = System.getProperty("connectionName");
49+
Assume.assumeFalse(String.format(messageTemplate, "connectionName"), connectionName == null);
50+
}
51+
52+
@Test
53+
public void test() throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException {
54+
test(new CloudSQLMySQLConnector(
55+
new CloudSQLMySQLConnectorConfig(username, password, JDBC_PLUGIN_NAME, connectionArguments, instanceType,
56+
connectionName, database)), JDBC_DRIVER_CLASS_NAME,
57+
CloudSQLMySQLConstants.PLUGIN_NAME);
58+
}
59+
}

‎cloudsql-postgresql-plugin/pom.xml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,60 @@
4848
<artifactId>postgresql-plugin</artifactId>
4949
<version>1.7.0-SNAPSHOT</version>
5050
</dependency>
51+
<dependency>
52+
<groupId>com.google.guava</groupId>
53+
<artifactId>guava</artifactId>
54+
<version>23.0</version>
55+
</dependency>
56+
<!-- test dependencies -->
57+
<dependency>
58+
<groupId>io.cdap.plugin</groupId>
59+
<artifactId>database-commons</artifactId>
60+
<version>${project.version}</version>
61+
<type>test-jar</type>
62+
<scope>test</scope>
63+
</dependency>
5164
<dependency>
5265
<groupId>io.cdap.cdap</groupId>
5366
<artifactId>hydrator-test</artifactId>
67+
</dependency>
68+
<dependency>
69+
<groupId>io.cdap.cdap</groupId>
70+
<artifactId>cdap-data-pipeline2_2.11</artifactId>
71+
</dependency>
72+
<dependency>
73+
<groupId>junit</groupId>
74+
<artifactId>junit</artifactId>
75+
</dependency>
76+
<dependency>
77+
<groupId>io.cdap.cdap</groupId>
78+
<artifactId>cdap-api</artifactId>
5479
<scope>provided</scope>
5580
</dependency>
81+
<dependency>
82+
<groupId>org.mockito</groupId>
83+
<artifactId>mockito-core</artifactId>
84+
</dependency>
85+
<dependency>
86+
<groupId>org.jetbrains</groupId>
87+
<artifactId>annotations</artifactId>
88+
<version>RELEASE</version>
89+
<scope>compile</scope>
90+
</dependency>
91+
<!-- https://mvnrepository.com/artifact/com.google.cloud.sql/postgres-socket-factory -->
92+
<dependency>
93+
<groupId>com.google.cloud.sql</groupId>
94+
<artifactId>postgres-socket-factory</artifactId>
95+
<version>1.4.0</version>
96+
<scope>test</scope>
97+
</dependency>
98+
<!-- https://mvnrepository.com/artifact/org.postgresql/postgresql -->
99+
<dependency>
100+
<groupId>org.postgresql</groupId>
101+
<artifactId>postgresql</artifactId>
102+
<version>42.3.1</version>
103+
<scope>test</scope>
104+
</dependency>
56105
</dependencies>
57106

58107
<build>

‎cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnectorConfig.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,18 @@ public class CloudSQLPostgreSQLConnectorConfig extends AbstractDBConnectorConfig
4444
@Nullable
4545
private String instanceType;
4646

47+
public CloudSQLPostgreSQLConnectorConfig(String username, String password, String jdbcPluginName,
48+
String connectionArguments, String instanceType,
49+
String connectionName, String database) {
50+
this.user = username;
51+
this.password = password;
52+
this.jdbcPluginName = jdbcPluginName;
53+
this.connectionArguments = connectionArguments;
54+
this.instanceType = instanceType;
55+
this.connectionName = connectionName;
56+
this.database = database;
57+
}
58+
4759
public String getDatabase() {
4860
return database;
4961
}

‎cloudsql-postgresql-plugin/src/test/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnectorTest.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* Copyright © 2021 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
package io.cdap.plugin.cloudsql.postgres;
18+
19+
import io.cdap.plugin.db.connector.DBSpecificConnectorBaseTest;
20+
import org.apache.parquet.Strings;
21+
import org.junit.Assume;
22+
import org.junit.BeforeClass;
23+
import org.junit.Test;
24+
25+
import java.io.IOException;
26+
27+
/**
28+
* Test for CloudSQL PostgreSQL Connector, it will only be run when below properties are provided:
29+
* -DinstanceType -- Whether the CloudSQL instance to connect to is private or public. If not provided, assumed public
30+
* -DconnectionName -- The CloudSQL instance to connect to. For a public instance, the connection string should be in
31+
* the format <PROJECT_ID>:<REGION>:<INSTANCE_NAME> which can be found in the instance overview page.
32+
* For a private instance, enter the internal IP address of the Compute Engine VM cloudsql proxy is running on.
33+
*/
34+
public class CloudSQLPostgreSQLConnectorTest extends DBSpecificConnectorBaseTest {
35+
36+
private static final String JDBC_DRIVER_CLASS_NAME = "com.google.cloud.sql.postgres.SocketFactory";
37+
private static String instanceType;
38+
private static String connectionName;
39+
40+
@BeforeClass
41+
public static void doSetup() {
42+
// Certain properties need to be configured otherwise the whole tests will be skipped.
43+
String messageTemplate = "%s is not configured, please refer to javadoc of this class for details.";
44+
instanceType = System.getProperty("instanceType");
45+
if (Strings.isNullOrEmpty(instanceType)) {
46+
instanceType = "public";
47+
}
48+
connectionName = System.getProperty("connectionName");
49+
Assume.assumeFalse(String.format(messageTemplate, "connectionName"), connectionName == null);
50+
}
51+
52+
@Test
53+
public void test() throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException {
54+
test(new CloudSQLPostgreSQLConnector(
55+
new CloudSQLPostgreSQLConnectorConfig(username, password, JDBC_PLUGIN_NAME, connectionArguments,
56+
instanceType, connectionName, database)), JDBC_DRIVER_CLASS_NAME,
57+
CloudSQLPostgreSQLConstants.PLUGIN_NAME);
58+
}
59+
}

‎database-commons/src/test/java/io/cdap/plugin/db/connector/DBSpecificConnectorBaseTest.java‎

Lines changed: 26 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import io.cdap.cdap.api.data.schema.Schema;
2222
import io.cdap.cdap.api.plugin.PluginProperties;
2323
import io.cdap.cdap.etl.api.batch.BatchConnector;
24+
import io.cdap.cdap.etl.api.batch.BatchSink;
2425
import io.cdap.cdap.etl.api.batch.BatchSource;
2526
import io.cdap.cdap.etl.api.connector.BrowseDetail;
2627
import io.cdap.cdap.etl.api.connector.BrowseEntity;
@@ -46,13 +47,15 @@
4647
import org.apache.hadoop.mapreduce.TaskID;
4748
import org.apache.hadoop.mapreduce.TaskType;
4849
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
50+
import org.apache.parquet.Strings;
4951
import org.junit.Assert;
5052
import org.junit.Assume;
5153
import org.junit.BeforeClass;
5254
import org.mockito.Mockito;
5355

5456
import java.io.IOException;
5557
import java.util.ArrayList;
58+
import java.util.Iterator;
5659
import java.util.List;
5760
import java.util.Map;
5861
import java.util.Set;
@@ -90,10 +93,11 @@ public static void setupTestClass() throws Exception {
9093
String messageTemplate = "%s is not configured, please refer to javadoc of this class for details.";
9194

9295
host = System.getProperty("host");
93-
Assume.assumeFalse(String.format(messageTemplate, "host"), host == null);
9496

9597
String portStr = System.getProperty("port");
96-
port = Integer.parseInt(portStr);
98+
if (!Strings.isNullOrEmpty(portStr)) {
99+
port = Integer.parseInt(portStr);
100+
}
97101

98102
username = System.getProperty("username");
99103
Assume.assumeFalse(String.format(messageTemplate, "username"), username == null);
@@ -152,7 +156,7 @@ protected void testBrowse(AbstractDBSpecificConnector connector) throws IOExcept
152156
Assert.assertTrue(detail.getEntities().size() > 0);
153157
for (BrowseEntity entity : detail.getEntities()) {
154158
System.out.println(entity.getType() + " : " + entity.getName());
155-
Assert.assertEquals("DATABASE", entity.getType());
159+
Assert.assertEquals("database", entity.getType());
156160
Assert.assertTrue(entity.canBrowse());
157161
Assert.assertFalse(entity.canSample());
158162
}
@@ -165,7 +169,7 @@ protected void testBrowse(AbstractDBSpecificConnector connector) throws IOExcept
165169
Assert.assertTrue(detail.getEntities().size() > 0);
166170
for (BrowseEntity entity : detail.getEntities()) {
167171
System.out.println(entity.getType() + " : " + entity.getName());
168-
Assert.assertEquals("SCHEMA", entity.getType());
172+
Assert.assertEquals("schema", entity.getType());
169173
Assert.assertTrue(entity.canBrowse());
170174
Assert.assertFalse(entity.canSample());
171175
}
@@ -232,7 +236,7 @@ protected void testTest(AbstractDBSpecificConnector connector) {
232236
Assert.assertTrue(validationException.getFailures().isEmpty());
233237
}
234238

235-
protected void testGenerateSpec(AbstractDBSpecificConnector connector, String sourceName) throws IOException {
239+
protected void testGenerateSpec(AbstractDBSpecificConnector connector, String pluginName) throws IOException {
236240
ConnectorSpec connectorSpec = connector.generateSpec(new MockConnectorContext((new MockConnectorConfigurer())),
237241
ConnectorSpecRequest.builder()
238242
.setPath(
@@ -244,16 +248,25 @@ protected void testGenerateSpec(AbstractDBSpecificConnector connector, String so
244248
Assert.assertNotNull(field.getSchema());
245249
}
246250
Set<PluginSpec> relatedPlugins = connectorSpec.getRelatedPlugins();
247-
Assert.assertEquals(1, relatedPlugins.size());
248-
PluginSpec pluginSpec = relatedPlugins.iterator().next();
249-
Assert.assertEquals(sourceName, pluginSpec.getName());
250-
Assert.assertEquals(BatchSource.PLUGIN_TYPE, pluginSpec.getType());
251+
Assert.assertEquals(2, relatedPlugins.size());
252+
Iterator<PluginSpec> relatedPluginsIterator = relatedPlugins.iterator();
253+
PluginSpec pluginSpec = relatedPluginsIterator.next();
254+
Assert.assertEquals(pluginName, pluginSpec.getName());
255+
Assert.assertTrue(pluginSpec.getType().equals(BatchSource.PLUGIN_TYPE) ||
256+
pluginSpec.getType().equals(BatchSink.PLUGIN_TYPE));
257+
PluginSpec pluginSpec1 = relatedPluginsIterator.next();
258+
Assert.assertEquals(pluginName, pluginSpec1.getName());
259+
if (pluginSpec.getType().equals(BatchSink.PLUGIN_TYPE)) {
260+
Assert.assertEquals(BatchSource.PLUGIN_TYPE, pluginSpec1.getType());
261+
} else {
262+
Assert.assertEquals(BatchSink.PLUGIN_TYPE, pluginSpec1.getType());
263+
}
251264

252265
Map<String, String> properties = pluginSpec.getProperties();
253-
Assert.assertEquals("true", properties.get(NAME_USE_CONNECTION));
254-
Assert.assertEquals("${conn(connection-id)}", properties.get(NAME_CONNECTION));
255-
Assert.assertEquals(schema == null ? String.format("SELECT * FROM %s.%s", database, table) :
256-
String.format("SELECT * FROM %s.%s.%s", database, schema, table),
266+
Assert.assertNull(properties.get(NAME_USE_CONNECTION));
267+
Assert.assertNull(properties.get(NAME_CONNECTION));
268+
Assert.assertEquals(connector.getTableQuery(connector.getDBConnectorPath(schema == null ? database + "/" + table :
269+
database + "/" + schema + "/" + table)),
257270
properties.get(IMPORT_QUERY));
258271
properties.put("1", properties.get(NUM_SPLITS));
259272
}

0 commit comments

Comments
 (0)