Skip to content

Commit 2065fa6

Browse files
FlechazoWFlechazoW
committed
Merge branch '1.10_release_4.0.x' into hotfix_1.10_4.0.x_30935
2 parents 3935fc2 + 83af3c0 commit 2065fa6

File tree

39 files changed

+1409
-211
lines changed

39 files changed

+1409
-211
lines changed

‎cassandra/cassandra-sink/src/main/java/com/dtstack/flink/sql/sink/cassandra/CassandraOutputFormat.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,9 @@
6161

6262
import java.io.IOException;
6363
import java.net.InetAddress;
64+
import java.sql.Time;
65+
import java.sql.Date;
66+
import java.sql.Timestamp;
6467
import java.sql.DriverManager;
6568
import java.sql.PreparedStatement;
6669
import java.util.ArrayList;
@@ -234,7 +237,10 @@ private String buildSql(Row row) {
234237
if (row.getField(index) == null) {
235238
} else {
236239
fields.append(fieldNames[index] + ",");
237-
if (row.getField(index) instanceof String) {
240+
if (row.getField(index) instanceof String
241+
|| row.getField(index) instanceof Time
242+
|| row.getField(index) instanceof Date
243+
|| row.getField(index) instanceof Timestamp) {
238244
values.append("'" + row.getField(index) + "'" + ",");
239245
} else {
240246
values.append(row.getField(index) + ",");

‎core/src/main/java/com/dtstack/flink/sql/GetPlan.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@
2626
import org.slf4j.Logger;
2727
import org.slf4j.LoggerFactory;
2828

29+
import java.net.URL;
30+
import java.net.URLClassLoader;
31+
2932
/**
3033
* local模式获取sql任务的执行计划
3134
* Date: 2020/2/17
@@ -37,17 +40,23 @@ public class GetPlan {
3740
private static final Logger LOG = LoggerFactory.getLogger(GetPlan.class);
3841

3942
public static String getExecutionPlan(String[] args) {
43+
ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader();
4044
try {
4145
long start = System.currentTimeMillis();
4246
ParamsInfo paramsInfo = ExecuteProcessHelper.parseParams(args);
4347
paramsInfo.setGetPlan(true);
48+
ClassLoader envClassLoader = StreamExecutionEnvironment.class.getClassLoader();
49+
ClassLoader plannerClassLoader = URLClassLoader.newInstance(new URL[0], envClassLoader);
50+
Thread.currentThread().setContextClassLoader(plannerClassLoader);
4451
StreamExecutionEnvironment env = ExecuteProcessHelper.getStreamExecution(paramsInfo);
4552
String executionPlan = env.getExecutionPlan();
4653
long end = System.currentTimeMillis();
4754
return ApiResult.createSuccessResultJsonStr(executionPlan, end - start);
4855
} catch (Exception e) {
4956
LOG.error("Get plan error", e);
5057
return ApiResult.createErrorResultJsonStr(ExceptionUtils.getFullStackTrace(e));
58+
} finally {
59+
Thread.currentThread().setContextClassLoader(currentClassLoader);
5160
}
5261
}
5362
}

‎core/src/main/java/com/dtstack/flink/sql/constrant/ConfigConstrant.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,9 @@ public class ConfigConstrant {
5656
// default 200ms
5757
public static final String AUTO_WATERMARK_INTERVAL_KEY = "autoWatermarkInterval";
5858

59+
// window early trigger
60+
public static final String EARLY_TRIGGER = "early.trigger";
61+
5962
public static final String SQL_TTL_MINTIME = "sql.ttl.min";
6063
public static final String SQL_TTL_MAXTIME = "sql.ttl.max";
6164

‎core/src/main/java/com/dtstack/flink/sql/environment/StreamEnvConfigManager.java‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,6 @@ public static void streamExecutionEnvironmentConfig(StreamExecutionEnvironment s
7373

7474
confProperties = PropertiesUtils.propertiesTrim(confProperties);
7575
streamEnv.getConfig().disableClosureCleaner();
76-
// Disables reusing object
77-
streamEnv.getConfig().enableObjectReuse();
7876

7977
Configuration globalJobParameters = new Configuration();
8078
//Configuration unsupported set properties key-value
@@ -87,8 +85,8 @@ public static void streamExecutionEnvironmentConfig(StreamExecutionEnvironment s
8785
ExecutionConfig exeConfig = streamEnv.getConfig();
8886
if (exeConfig.getGlobalJobParameters() == null) {
8987
exeConfig.setGlobalJobParameters(globalJobParameters);
90-
} else if (exeConfig.getGlobalJobParameters() instanceof Configuration) {
91-
((Configuration) exeConfig.getGlobalJobParameters()).addAll(globalJobParameters);
88+
} else if (exeConfig.getGlobalJobParameters() instanceof ExecutionConfig.GlobalJobParameters) {
89+
exeConfig.setGlobalJobParameters(globalJobParameters);
9290
}
9391

9492
getEnvParallelism(confProperties).ifPresent(streamEnv::setParallelism);
@@ -123,6 +121,21 @@ public static void streamExecutionEnvironmentConfig(StreamExecutionEnvironment s
123121
}
124122
}
125123

124+
/**
125+
* 设置TableEnvironment window提前触发
126+
* @param tableEnv
127+
* @param confProperties
128+
*/
129+
public static void streamTableEnvironmentEarlyTriggerConfig(TableEnvironment tableEnv, Properties confProperties) {
130+
confProperties = PropertiesUtils.propertiesTrim(confProperties);
131+
String triggerTime = confProperties.getProperty(ConfigConstrant.EARLY_TRIGGER);
132+
if (StringUtils.isNumeric(triggerTime)) {
133+
TableConfig qConfig = tableEnv.getConfig();
134+
qConfig.getConfiguration().setString("table.exec.emit.early-fire.enabled", "true");
135+
qConfig.getConfiguration().setString("table.exec.emit.early-fire.delay", triggerTime+"s");
136+
}
137+
}
138+
126139
/**
127140
* 设置TableEnvironment状态超时时间
128141
* @param tableEnv

‎core/src/main/java/com/dtstack/flink/sql/exception/BaseCodeEnum.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package com.dtstack.flink.sql.exception;
20+
21+
/**
22+
* @author: chuixue
23+
* @create: 2020-11-30 16:23
24+
* @description: 公共错误码
25+
**/
26+
public enum BaseCodeEnum implements ErrorCode {
27+
/**
28+
* 未指明的异常
29+
*/
30+
UNSPECIFIED("000", "unknow exception"),
31+
;
32+
33+
/**
34+
* 错误码
35+
*/
36+
private final String code;
37+
38+
/**
39+
* 描述
40+
*/
41+
private final String description;
42+
43+
/**
44+
* @param code 错误码
45+
* @param description 描述
46+
*/
47+
private BaseCodeEnum(final String code, final String description) {
48+
this.code = code;
49+
this.description = description;
50+
}
51+
52+
@Override
53+
public String getCode() {
54+
return code;
55+
}
56+
57+
@Override
58+
public String getDescription() {
59+
return description;
60+
}
61+
}

‎core/src/main/java/com/dtstack/flink/sql/exception/BaseException.java‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package com.dtstack.flink.sql.exception;
20+
21+
22+
/**
23+
* @author: chuixue
24+
* @create: 2020-11-30 14:48
25+
* @description:根异常
26+
**/
27+
public class BaseException extends RuntimeException {
28+
29+
/**
30+
* 错误码
31+
*/
32+
protected final ErrorCode errorCode;
33+
34+
/**
35+
* 无参默认构造UNSPECIFIED
36+
*/
37+
public BaseException() {
38+
super(BaseCodeEnum.UNSPECIFIED.getDescription());
39+
errorCode = BaseCodeEnum.UNSPECIFIED;
40+
}
41+
42+
/**
43+
* 指定错误码构造通用异常
44+
*
45+
* @param errorCode 错误码
46+
*/
47+
public BaseException(ErrorCode errorCode) {
48+
super(errorCode.getDescription());
49+
this.errorCode = errorCode;
50+
}
51+
52+
/**
53+
* 指定详细描述构造通用异常
54+
*
55+
* @param detailedMessage 详细描述
56+
*/
57+
public BaseException(final String detailedMessage) {
58+
super(detailedMessage);
59+
this.errorCode = BaseCodeEnum.UNSPECIFIED;
60+
}
61+
62+
/**
63+
* 指定导火索构造通用异常
64+
*
65+
* @param t 导火索
66+
*/
67+
public BaseException(final Throwable t) {
68+
super(t);
69+
this.errorCode = BaseCodeEnum.UNSPECIFIED;
70+
}
71+
72+
/**
73+
* 构造通用异常
74+
*
75+
* @param errorCode 错误码
76+
* @param detailedMessage 详细描述
77+
*/
78+
public BaseException(final ErrorCode errorCode, final String detailedMessage) {
79+
super(detailedMessage);
80+
this.errorCode = errorCode;
81+
}
82+
83+
/**
84+
* 构造通用异常
85+
*
86+
* @param errorCode 错误码
87+
* @param t 导火索
88+
*/
89+
public BaseException(final ErrorCode errorCode, final Throwable t) {
90+
super(errorCode.getDescription(), t);
91+
this.errorCode = errorCode;
92+
}
93+
94+
/**
95+
* 构造通用异常
96+
*
97+
* @param detailedMessage 详细描述
98+
* @param t 导火索
99+
*/
100+
public BaseException(final String detailedMessage, final Throwable t) {
101+
super(detailedMessage, t);
102+
this.errorCode = BaseCodeEnum.UNSPECIFIED;
103+
}
104+
105+
/**
106+
* 构造通用异常
107+
*
108+
* @param errorCode 错误码
109+
* @param detailedMessage 详细描述
110+
* @param t 导火索
111+
*/
112+
public BaseException(final ErrorCode errorCode, final String detailedMessage,
113+
final Throwable t) {
114+
super(detailedMessage, t);
115+
this.errorCode = errorCode;
116+
}
117+
118+
/**
119+
* 获取错误码
120+
*
121+
* @return
122+
*/
123+
public ErrorCode getErrorCode() {
124+
return errorCode;
125+
}
126+
}

‎core/src/main/java/com/dtstack/flink/sql/exception/ErrorCode.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
package com.dtstack.flink.sql.exception;
2+
3+
/**
4+
* 错误码
5+
*/
6+
public interface ErrorCode {
7+
8+
/**
9+
* 获取错误码
10+
*
11+
* @return
12+
*/
13+
String getCode();
14+
15+
/**
16+
* 获取错误信息
17+
*
18+
* @return
19+
*/
20+
String getDescription();
21+
}

‎core/src/main/java/com/dtstack/flink/sql/exception/sqlparse/SqlParseCodeEnum.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package com.dtstack.flink.sql.exception.sqlparse;
20+
21+
import com.dtstack.flink.sql.exception.ErrorCode;
22+
23+
/**
24+
* @author: chuixue
25+
* @create: 2020-11-30 16:56
26+
* @description:sql解析错误码
27+
**/
28+
public enum SqlParseCodeEnum implements ErrorCode {
29+
/**
30+
* 流join维表时，select、join、group by等字段未使用t.field
31+
*/
32+
WITHOUT_TABLENAME("001", "field invalid , please use like t.field"),
33+
;
34+
35+
/**
36+
* 错误码
37+
*/
38+
private final String code;
39+
40+
/**
41+
* 描述
42+
*/
43+
private final String description;
44+
45+
/**
46+
* @param code 错误码
47+
* @param description 描述
48+
*/
49+
private SqlParseCodeEnum(final String code, final String description) {
50+
this.code = code;
51+
this.description = description;
52+
}
53+
54+
@Override
55+
public String getCode() {
56+
return code;
57+
}
58+
59+
@Override
60+
public String getDescription() {
61+
return description;
62+
}
63+
}

0 commit comments

Comments
 (0)