Skip to content

Commit 81fe1d2

Browse files
author
sishu@dtstack.com
committed
Merge branch 'master' into v1.5.0
2 parents 87d5589 + 5c93bdb commit 81fe1d2

File tree

7 files changed

+208
-10
lines changed

7 files changed

+208
-10
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,4 @@ target/
88
*.eclipse.*
99
*.iml
1010
plugins/
11+
lib/

‎README.md‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,12 @@
1111

1212
# 后续开发计划
1313
* 增加全局缓存功能
14-
* sql增加临时表功能
15-
* 增加redis维表功能
16-
* 增加mongodb维表功能
14+
* 增加临时表功能
15+
* 增加redis维表,结果表功能
16+
* 增加mongodb维表，结果表功能
1717
* 增加oracle维表，结果表功能
1818
* 增加SQlServer维表，结果表功能
19+
* 增加kafka结果表功能
1920

2021
## 1 快速起步
2122
### 1.1 运行模式
@@ -37,14 +38,19 @@
3738

3839
```
3940
mvn clean package -Dmaven.test.skip
41+
42+
打包结束后，项目根目录下会产生plugins目录，plugins目录下存放编译好的数据同步插件包,在lib目下存放job提交的包
4043
```
4144

42-
打包结束后，项目根目录下会产生plugins目录，plugins目录下存放编译好的数据同步插件包
45+
### 1.4 启动
4346

47+
#### 1.4.1 启动命令
4448

45-
### 1.4 启动
49+
```
50+
sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack/150_flinkplugin/sqlplugin -localSqlPluginPath D:\gitspace\flinkStreamSQL\plugins -mode yarn -flinkconf D:\flink_home\kudu150etc -yarnconf D:\hadoop\etc\hadoopkudu -confProp {\"time.characteristic\":\"EventTime\",\"sql.checkpoint.interval\":10000}
51+
```
4652

47-
#### 1.4.1 命令行参数选项
53+
#### 1.4.2 命令行参数选项
4854

4955
* **model**
5056
* 描述：执行模式，也就是flink集群的工作模式
@@ -82,7 +88,7 @@ mvn clean package -Dmaven.test.skip
8288
* **confProp**
8389
* 描述：一些参数设置
8490
* 格式: json
85-
* 必选：否
91+
* 必选：是 （如无参数填写空json即可）
8692
* 默认值：无
8793
* 可选参数:
8894
* sql.env.parallelism: 默认并行度设置
@@ -108,7 +114,7 @@ mvn clean package -Dmaven.test.skip
108114

109115
## 2 结构
110116
### 2.1 源表插件
111-
* [kafka09 源表插件](docs/kafka09Source.md)
117+
* [kafka 源表插件](docs/kafkaSource.md)
112118

113119
### 2.2 结果表插件
114120
* [elasticsearch 结果表插件](docs/elasticsearchSink.md)

‎bin/submit.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/usr/bin/env bash
2+
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one or more
5+
# contributor license agreements. See the NOTICE file distributed with
6+
# this work for additional information regarding copyright ownership.
7+
# The ASF licenses this file to You under the Apache License, Version 2.0
8+
# (the "License"); you may not use this file except in compliance with
9+
# the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
#
19+
20+
set -e
21+
22+
export SQL_HOME="$(cd "`dirname "$0"`"/..; pwd)"
23+
24+
# Find the java binary
25+
if [ -n "${JAVA_HOME}" ]; then
26+
JAVA_RUN="${JAVA_HOME}/bin/java"
27+
else
28+
if [ `command -v java` ]; then
29+
JAVA_RUN="java"
30+
else
31+
echo "JAVA_HOME is not set" >&2
32+
exit 1
33+
fi
34+
fi
35+
36+
JAR_DIR=$SQL_HOME/lib/*
37+
CLASS_NAME=com.dtstack.flink.sql.launcher.LauncherMain
38+
39+
echo "sql submit ..."
40+
nohup $JAVA_RUN -cp $JAR_DIR $CLASS_NAME $@ &

docs/kafka09Source.md renamed to docs/kafkaSource.md

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
## 1.格式：
22
```
3+
数据现在支持json格式{"xx":"bb","cc":"dd"}
4+
35
CREATE TABLE tableName(
46
colName colType,
57
...
@@ -16,7 +18,7 @@ CREATE TABLE tableName(
1618
```
1719

1820
## 2.支持的版本
19-
kafka09
21+
kafka09,kafka10,kafka11
2022

2123
## 3.表结构定义
2224

@@ -55,4 +57,4 @@ CREATE TABLE MyTable(
5557
topic ='nbTest1',
5658
parallelism ='1'
5759
);
58-
```
60+
```

‎launcher/.gitignore‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
target
2+
.idea/
3+
/.idea/*
4+
target/
5+
.class
6+
.project
7+
.classpath
8+
*.eclipse.*
9+
*.iml
10+
plugins/
11+
lib/
12+
dependency-reduced-pom.xml

‎launcher/job/mysqlsideSql.txt‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
CREATE TABLE MyTable(
2+
channel STRING,
3+
pv INT,
4+
xctime bigint,
5+
CHARACTER_LENGTH(channel) as timeLeng,
6+
WATERMARK FOR xctime AS withOffset(xctime,1000)
7+
)WITH(
8+
type='kafka09',
9+
bootstrapServers='172.16.8.198:9092',
10+
offsetReset='latest',
11+
topic='nbTest1'
12+
);
13+
CREATE TABLE MyResult(
14+
channel STRING,
15+
pv INT
16+
)WITH(
17+
type='mysql',
18+
url='jdbc:mysql://172.16.8.104:3306/test?charset=utf8',
19+
userName='dtstack',
20+
password='abc123',
21+
tableName='pv'
22+
);
23+
24+
create table sideTable(
25+
channel String,
26+
xccount int,
27+
PRIMARY KEY(channel),
28+
PERIOD FOR SYSTEM_TIME
29+
)WITH(
30+
type='mysql',
31+
url='jdbc:mysql://172.16.8.104:3306/test?charset=utf8',
32+
userName='dtstack',
33+
password='abc123',
34+
tableName='sidetest',
35+
cache = 'LRU',
36+
cacheTTLMs='10000'
37+
);
38+
39+
insert
40+
into
41+
MyResult
42+
select
43+
a.channel,
44+
b.xccount
45+
from
46+
MyTable a
47+
join
48+
sideTable b
49+
on a.channel=b.channel
50+
where
51+
b.channel = 'xc'
52+
and a.pv=10;

‎launcher/pom.xml‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,89 @@
3131
</dependency>
3232
</dependencies>
3333

34+
<build>
35+
<plugins>
36+
37+
<plugin>
38+
<groupId>org.apache.maven.plugins</groupId>
39+
<artifactId>maven-shade-plugin</artifactId>
40+
<version>3.0.0</version>
41+
<executions>
42+
<execution>
43+
<phase>package</phase>
44+
<goals>
45+
<goal>shade</goal>
46+
</goals>
47+
<configuration>
48+
<transformers>
49+
50+
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer" />
51+
<!-- The service transformer is needed to merge META-INF/services files -->
52+
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
53+
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"/>
54+
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
55+
<resource>reference.conf</resource>
56+
</transformer>
57+
58+
<transformer implementation="org.apache.maven.plugins.shade.resource.XmlAppendingTransformer">
59+
<resource>core-default.xml</resource>
60+
</transformer>
61+
62+
<transformer implementation="org.apache.maven.plugins.shade.resource.XmlAppendingTransformer">
63+
<resource>core-site.xml</resource>
64+
</transformer>
65+
66+
<transformer implementation="org.apache.maven.plugins.shade.resource.XmlAppendingTransformer">
67+
<resource>yarn-default.xml</resource>
68+
</transformer>
69+
70+
<transformer implementation="org.apache.maven.plugins.shade.resource.XmlAppendingTransformer">
71+
<resource>mapred-default.xml</resource>
72+
</transformer>
73+
74+
<transformer implementation="org.apache.maven.plugins.shade.resource.XmlAppendingTransformer">
75+
<resource>mapred-site.xml</resource>
76+
</transformer>
77+
</transformers>
78+
79+
<filters>
80+
<filter>
81+
<artifact>*:*</artifact>
82+
<excludes>
83+
<exclude>META-INF/*.SF</exclude>
84+
<exclude>META-INF/*.DSA</exclude>
85+
<exclude>META-INF/*.RSA</exclude>
86+
</excludes>
87+
</filter>
88+
</filters>
89+
</configuration>
90+
</execution>
91+
</executions>
92+
</plugin>
93+
94+
<plugin>
95+
<artifactId>maven-antrun-plugin</artifactId>
96+
<version>1.2</version>
97+
<executions>
98+
<execution>
99+
<id>copy-resources</id>
100+
<!-- here the phase you need -->
101+
<phase>package</phase>
102+
<goals>
103+
<goal>run</goal>
104+
</goals>
105+
<configuration>
106+
<tasks>
107+
<copy todir="${basedir}/../lib/">
108+
<fileset dir="target/">
109+
<include name="${project.name}-${project.version}.jar" />
110+
</fileset>
111+
</copy>
112+
</tasks>
113+
</configuration>
114+
</execution>
115+
</executions>
116+
</plugin>
117+
</plugins>
118+
</build>
34119
</project>

0 commit comments

Comments
 (0)