Skip to content

Commit 0304f32

Browse files
Zhu4680Zhu4680
committed
Merge branch 'feat_1.8_modifyDoc' of http://gitlab.prod.dtstack.cn/dt-insight-engine/flinkStreamSQL into feat_1.8_modifyDoc
2 parents ccbc0a1 + 2e4721b commit 0304f32

File tree

10 files changed

+733
-196
lines changed

10 files changed

+733
-196
lines changed

‎docs/plugin/clickhouseSide.md‎

Lines changed: 120 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
1-
21
## 1.格式：
3-
```
2+
3+
通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段，
4+
维表JOIN的条件必须与`keyInfo`字段一致。
5+
6+
```sql
47
CREATE TABLE tableName(
58
colName cloType,
69
...
@@ -21,65 +24,132 @@
2124
```
2225

2326
# 2.支持版本
27+
2428
19.14.x、19.15.x、19.16.x
25-
29+
2630
## 3.表结构定义
27-
28-
|参数名称|含义|
29-
|----|---|
30-
| tableName | clickhouse表名称|
31-
| colName | 列名称|
32-
| colType | 列类型 [colType支持的类型](docs/colType.md)|
33-
| PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息|
34-
| PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开|
35-
36-
## 4.参数
37-
38-
|参数名称|含义|是否必填|默认值|
39-
|----|---|---|----|
40-
| type | 表明维表的类型 clickhouse |是||
41-
| url | 连接clickhouse数据库 jdbcUrl |是||
42-
| userName | clickhouse连接用户名 |是||
43-
| password | clickhouse连接密码|是||
44-
| tableName | clickhouse表名称|是||
45-
| tableName | clickhouse 的表名称|是||
46-
| cache | 维表缓存策略(NONE/LRU)|否|NONE|
47-
| partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false|
48-
49-
----------
50-
> 缓存策略
51-
* NONE: 不做内存缓存
52-
* LRU:
53-
* cacheSize: 缓存的条目数量
54-
* cacheTTLMs:缓存的过期时间(ms)
55-
* cacheMode: (unordered|ordered)异步加载是有序还是无序,默认有序。
56-
* asyncCapacity:异步请求容量，默认1000
57-
* asyncTimeout：异步请求超时时间，默认10000毫秒
58-
59-
## 5.样例
31+
32+
[维表参数信息](docs/plugin/sideParams.md)
33+
34+
clinkhose独有的参数信息
35+
36+
| 参数名称 | 含义 | 是否必填 | 默认值 |
37+
| -------- | --------------------- | -------- | ------ |
38+
| type | 维表类型， clinkhouse | 是 | |
39+
| url | 连接数据库 jdbcUrl | 是 | |
40+
| userName | 连接用户名 | 是 | |
41+
| password | 连接密码 | 是 | |
42+
43+
## 4.样例
44+
45+
------
46+
47+
## ALL全量维表定义
48+
49+
```sql
50+
// 定义全量维表
51+
CREATE TABLE sideTable(
52+
id INT,
53+
name VARCHAR,
54+
PRIMARY KEY(id),
55+
PERIOD FOR SYSTEM_TIME
56+
)WITH(
57+
type ='clickhouse',
58+
url ='jdbc:clickhouse://172.16.8.104:3306/zftest',
59+
userName ='dtstack',
60+
password ='abc123',
61+
tableName ='all_test_clinkhousesql',
62+
cache ='ALL',
63+
cacheTTLMs ='60000',
64+
parallelism ='1'
65+
);
6066
```
61-
create table sideTable(
62-
channel varchar,
63-
xccount int,
64-
PRIMARY KEY(channel),
67+
68+
### LRU异步维表定义
69+
70+
```
71+
CREATE TABLE sideTable(
72+
id INT,
73+
name VARCHAR,
74+
PRIMARY KEY(id) ,
6575
PERIOD FOR SYSTEM_TIME
6676
)WITH(
67-
type='clickhouse',
68-
url='jdbc:clickhouse://172.16.8.104:3306/test?charset=utf8',
69-
userName='dtstack',
70-
password='abc123',
71-
tableName='sidetest',
77+
type ='clickhousesql',
78+
url ='jdbc:clickhousesql://172.16.8.104:3306/zftest',
79+
userName ='dtstack',
80+
password ='abc123',
81+
tableName ='lru_test_clickhousesql',
82+
partitionedJoin ='false',
7283
cache ='LRU',
7384
cacheSize ='10000',
7485
cacheTTLMs ='60000',
75-
cacheMode='unordered',
76-
asyncCapacity='1000',
77-
asyncTimeout='10000'
86+
asyncPoolSize ='3',
87+
parallelism ='1'
88+
);
89+
```
90+
91+
### ClickHouseSQL异步维表关联
92+
93+
```sql
94+
CREATE TABLE MyTable(
95+
id int,
96+
name varchar
97+
)WITH(
98+
type ='kafka11',
99+
bootstrapServers ='172.16.8.107:9092',
100+
zookeeperQuorum ='172.16.8.107:2181/kafka',
101+
offsetReset ='latest',
102+
topic ='cannan_zftest01',
103+
timezone='Asia/Shanghai',
104+
enableKeyPartitions ='false',
105+
topicIsPattern ='false',
106+
parallelism ='1'
107+
);
108+
109+
CREATE TABLE MyResult(
110+
id INT,
111+
name VARCHAR
112+
)WITH(
113+
type ='clickhousesql',
114+
url ='jdbc:clickhousesql://172.16.8.104:3306/zftest',
115+
userName ='dtstack',
116+
password ='abc123',
117+
tableName ='test_clickhouse_zf',
118+
updateMode ='append',
78119
parallelism ='1',
79-
partitionedJoin='false'
120+
batchSize ='100',
121+
batchWaitInterval ='1000'
80122
);
81123

124+
CREATE TABLE sideTable(
125+
id INT,
126+
name VARCHAR,
127+
PRIMARY KEY(id) ,
128+
PERIOD FOR SYSTEM_TIME
129+
)WITH(
130+
type ='clickhousesql',
131+
url ='jdbc:clickhousesql://172.16.8.104:3306/zftest',
132+
userName ='dtstack',
133+
password ='abc123',
134+
tableName ='test_clickhouse_10',
135+
partitionedJoin ='false',
136+
cache ='LRU',
137+
cacheSize ='10000',
138+
cacheTTLMs ='60000',
139+
asyncPoolSize ='3',
140+
parallelism ='1'
141+
);
82142

143+
insert
144+
into
145+
MyResult
146+
select
147+
m.id,
148+
s.name
149+
from
150+
MyTable m
151+
join
152+
sideTable s
153+
on m.id=s.id;
83154
```
84155

85-

‎docs/plugin/clickhouseSink.md‎

Lines changed: 81 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ CREATE TABLE tableName(
2424
|----|---|
2525
| tableName| clickhouse表名称|
2626
| colName | 列名称|
27-
| colType | 列类型 [colType支持的类型](docs/colType.md)|
27+
| colType | clickhouse基本数据类型，不包括Array，Tuple,Nested等|
2828

2929
## 4.参数：
3030

@@ -36,18 +36,87 @@ CREATE TABLE tableName(
3636
| password | clickhouse 连接密码|是||
3737
| tableName | clickhouse 表名称|是||
3838
| parallelism | 并行度设置|否|1|
39+
|updateMode| 只支持APPEND模式，过滤掉回撤数据|||
40+
3941

4042
## 5.样例：
43+
44+
4145
```
46+
47+
CREATE TABLE source1 (
48+
id int,
49+
name VARCHAR
50+
)WITH(
51+
type ='kafka11',
52+
bootstrapServers ='172.16.8.107:9092',
53+
zookeeperQuorum ='172.16.8.107:2181/kafka',
54+
offsetReset ='latest',
55+
topic ='mqTest03',
56+
timezone='Asia/Shanghai',
57+
topicIsPattern ='false'
58+
);
59+
60+
61+
62+
CREATE TABLE source2(
63+
id int,
64+
address VARCHAR
65+
)WITH(
66+
type ='kafka11',
67+
bootstrapServers ='172.16.8.107:9092',
68+
zookeeperQuorum ='172.16.8.107:2181/kafka',
69+
offsetReset ='latest',
70+
topic ='mqTest04',
71+
timezone='Asia/Shanghai',
72+
topicIsPattern ='false'
73+
);
74+
75+
4276
CREATE TABLE MyResult(
43-
channel VARCHAR,
44-
pv VARCHAR
45-
)WITH(
46-
type ='clickhouse',
47-
url ='jdbc:clickhouse://172.16.8.104:3306/test?charset=utf8',
48-
userName ='dtstack',
49-
password ='abc123',
50-
tableName ='pv2',
51-
parallelism ='1'
52-
)
53-
```
77+
id int,
78+
name VARCHAR,
79+
address VARCHAR
80+
)WITH(
81+
type='clickhouse',
82+
url='jdbc:clickhouse://172.16.10.168:8123/tudou?charset=utf8',
83+
userName='dtstack',
84+
password='abc123',
85+
tableName='MyResult',
86+
updateMode = 'append'
87+
);
88+
89+
insert into MyResult
90+
select
91+
s1.id,
92+
s1.name,
93+
s2.address
94+
from
95+
source1 s1
96+
left join
97+
source2 s2
98+
on
99+
s1.id = s2.id
100+
101+
102+
```
103+
104+
105+
**Clickhouse建表语句**
106+
107+
108+
```aidl
109+
CREATE TABLE tudou.MyResult (`id` Int64, `name` String, `address` String)
110+
ENGINE = MergeTree PARTITION BY address ORDER BY id SETTINGS index_granularity = 8192
111+
```
112+
113+
114+
115+
116+
数据结果：
117+
118+
向Topic mqTest03 发送数据 {"name":"maqi","id":1001} 插入 (1001,"maqi",null)
119+
120+
向Topic mqTest04 发送数据 {"address":"hz","id":1001} 插入 (1001,"maqi","hz")
121+
122+

0 commit comments

Comments
 (0)