diff --git a/.gitignore b/.gitignore index 625190ea5..a5f8e1c1c 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,4 @@ streamis-appconn/target **/._* **/.flattened-pom.xml +*.patch \ No newline at end of file diff --git a/assembly/pom.xml b/assembly/pom.xml index fef17c40b..c0d5b19d1 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -20,7 +20,7 @@ streamis com.webank.wedatasphere.streamis - 0.3.1 + ${revision} 4.0.0 @@ -54,7 +54,7 @@ false - wedatasphere-streamis-${streamis.version}-dist + wedatasphere-streamis-${project.version}-dist false false diff --git a/bin/install.sh b/bin/install.sh index 35d03c1b5..ccbf5c3f3 100644 --- a/bin/install.sh +++ b/bin/install.sh @@ -119,12 +119,11 @@ cd $SERVER_HOME/;unzip $SERVERNAME.zip > /dev/null isSuccess "unzip ${SERVERNAME}.zip" echo "$SERVERNAME-step3:subsitution conf" -SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/application.yml -sed -i "s#port:.*#port: $SERVER_PORT#g" $SERVER_CONF_PATH -sed -i "s#defaultZone:.*#defaultZone: $EUREKA_URL#g" $SERVER_CONF_PATH -sed -i "s#hostname:.*#hostname: $SERVER_IP#g" $SERVER_CONF_PATH -isSuccess "subsitution conf of $SERVERNAME" -} +SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/application.properties +sed -i "s#server.port.*#server.port=$SERVER_PORT#g" $SERVER_CONF_PATH +sed -i "s#eureka.client.serviceUrl.defaultZone.*#eureka.client.serviceUrl.defaultZone=$EUREKA_URL#g" $SERVER_CONF_PATH + isSuccess "subsitution conf of $SERVERNAME" + } function setDatasourcePassword(){ PASSWORD=$MYSQL_PASSWORD @@ -152,6 +151,8 @@ setDatasourcePassword sed -i "s#wds.linkis.gateway.ip.*#wds.linkis.gateway.ip=$GATEWAY_INSTALL_IP#g" $SERVER_CONF_PATH sed -i "s#wds.linkis.gateway.port.*#wds.linkis.gateway.port=$GATEWAY_PORT#g" $SERVER_CONF_PATH sed -i "s#wds.linkis.gateway.url.*#wds.linkis.gateway.url=http://${GATEWAY_INSTALL_IP}:${GATEWAY_PORT}#g" $SERVER_CONF_PATH +sed -i "s#\#wds.linkis.token=.*#wds.linkis.token=${STREAMIS_LINKIS_TOKEN}#g" $SERVER_CONF_PATH +sed -i "s#wds.linkis.flink.engine.version=.*#wds.linkis.flink.engine.version=${LINKIS_FLINK_VERSION}#g" $SERVER_CONF_PATH isSuccess "subsitution linkis.properties of $SERVERNAME" echo "<----------------$SERVERNAME:end------------------->" echo "" diff --git a/conf/config.sh b/conf/config.sh index b1a22f287..d5ee955ac 100644 --- a/conf/config.sh +++ b/conf/config.sh @@ -40,6 +40,12 @@ GATEWAY_PORT=9001 STREAMIS_SERVER_INSTALL_IP=127.0.0.1 STREAMIS_SERVER_INSTALL_PORT=9400 -STREAMIS_VERSION=0.3.1 +STREAMIS_VERSION=0.3.2 -STREAMIS_FILE_NAME="STREAMIS-$STREAMIS_VERSION" \ No newline at end of file +STREAMIS_FILE_NAME="STREAMIS-$STREAMIS_VERSION" + +# guarantee that the token is configured within the Linkis management console +STREAMIS_LINKIS_TOKEN=streamis-auth + +# fill in the available Flink versions on Linkis. +LINKIS_FLINK_VERSION=1.16.2 \ No newline at end of file diff --git a/db/streamis_ddl.sql b/db/streamis_ddl.sql index 5f3108bee..115d4eb41 100644 --- a/db/streamis_ddl.sql +++ b/db/streamis_ddl.sql @@ -132,10 +132,11 @@ CREATE TABLE `linkis_stream_job` ( `create_time` datetime DEFAULT NULL, `label` varchar(200) DEFAULT NULL, `description` varchar(200) DEFAULT NULL, - `job_type` varchar(30) DEFAULT NULL COMMENT '目前只支持flink.sql、flink.jar、spark.jar', + `job_type` varchar(30) DEFAULT NULL COMMENT '目前只支持flink.sql、flink.jar', `submit_user` varchar(100) DEFAULT NULL, `workspace_name` varchar(50) DEFAULT NULL, `current_version` varchar(50) DEFAULT NULL, + `enable` tinyint(1) DEFAULT '1', PRIMARY KEY (`id`) USING BTREE, UNIQUE KEY(`project_name`, `name`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT COMMENT='作业表'; @@ -148,14 +149,14 @@ CREATE TABLE `linkis_stream_job_version` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, `job_id` varchar(50) DEFAULT NULL, `version` varchar(20) DEFAULT NULL, - `source` varchar(255) DEFAULT NULL COMMENT '这个版本的来源，比如：用户上传，由某个历史版本回退回来的', + `source` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin COMMENT '这个版本的来源，比如：用户上传，由某个历史版本回退回来的', `job_content` text COMMENT '内容为meta.json', `manage_mode` varchar(30) DEFAULT 'EngineConn' COMMENT 'Manage mode', `comment` varchar(255) DEFAULT NULL, `create_time` datetime DEFAULT NULL, `create_by` varchar(32) DEFAULT NULL, PRIMARY KEY (`id`) USING BTREE, - UNIQUE KEY(`job_id`, `version`) + UNIQUE KEY `job_id`(`job_id`, `version`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT COMMENT='作业表'; /*Table structure for table `linkis_stream_job_version_files` */ @@ -184,6 +185,10 @@ CREATE TABLE `linkis_stream_project` ( `workspace_id` bigint(20) DEFAULT NULL, `name` varchar(100) DEFAULT NULL, `create_by` varchar(50) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `last_update_by` varchar(50) DEFAULT NULL, + `last_update_time` datetime DEFAULT NULL, + `is_deleted` tinyint(3) unsigned DEFAULT 0, PRIMARY KEY (`id`) USING BTREE ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT COMMENT='项目表'; @@ -203,11 +208,47 @@ CREATE TABLE `linkis_stream_project_files` ( `comment` varchar(255) DEFAULT NULL COMMENT '说明', `update_time` datetime DEFAULT NULL, `md5` varchar(100) DEFAULT NULL COMMENT '文件md5', + `source` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL, PRIMARY KEY (`id`) USING BTREE ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT COMMENT='项目表'; + +/*Table structure for table `linkis_stream_project_job_template` */ + +DROP TABLE IF EXISTS `linkis_stream_project_job_template`; + +CREATE TABLE `linkis_stream_project_job_template` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `store_path` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `name` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `meta_json` text COLLATE utf8_bin, + `version` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `date` datetime DEFAULT NULL, + `project_name` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `enable` tinyint(1) DEFAULT '1', + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + +/*Table structure for table `linkis_stream_register_info` */ + + +DROP TABLE IF EXISTS `linkis_stream_register_info`; + +CREATE TABLE `linkis_stream_register_info` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `job_id` bigint(20) DEFAULT NULL, + `application_name` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `password` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `register_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `heartbeat_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=171425 DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + /*Table structure for table `linkis_stream_task` */ + DROP TABLE IF EXISTS `linkis_stream_task`; CREATE TABLE `linkis_stream_task` ( @@ -222,7 +263,10 @@ CREATE TABLE `linkis_stream_task` ( `submit_user` varchar(50) DEFAULT NULL, `linkis_job_id` varchar(200) DEFAULT NULL, `linkis_job_info` mediumtext, - `job_start_config` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL COMMENT 'streamis job startup config', + `server_instance` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL COMMENT 'streamis server instance', + `job_start_config` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin COMMENT 'streamis job startup config', + `solution` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin COMMENT 'error code solution', + `template_id` bigint(20) DEFAULT NULL, PRIMARY KEY (`id`) USING BTREE ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT COMMENT='任务表'; @@ -261,6 +305,7 @@ CREATE TABLE `linkis_stream_error_code` ( `error_desc` varchar(1024) COLLATE utf8_bin NOT NULL, `error_regex` varchar(1024) COLLATE utf8_bin DEFAULT NULL, `error_type` int(3) DEFAULT '0', + `solution` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin COMMENT 'error code solution', PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_bin; @@ -279,13 +324,10 @@ CREATE TABLE `linkis_stream_audit_log` ( `project_name` varchar(100) COLLATE utf8_bin DEFAULT NULL, `client_ip` varchar(100) COLLATE utf8_bin DEFAULT NULL, `cost_time_mills` bigint(20) NULL, + `job_name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_bin; -ALTER TABLE `linkis_stream_project` ADD create_time datetime DEFAULT NULL; -ALTER TABLE `linkis_stream_project` ADD last_update_by varchar(50) DEFAULT NULL; -ALTER TABLE `linkis_stream_project` ADD last_update_time datetime DEFAULT NULL; -ALTER TABLE `linkis_stream_project` ADD is_deleted tinyint unsigned DEFAULT 0; -ALTER TABLE linkis_stream_project_files ADD source varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NULL; + SET FOREIGN_KEY_CHECKS = 1; diff --git a/db/upgrade/0.3.2/upgrade_ddl.sql b/db/upgrade/0.3.2/upgrade_ddl.sql new file mode 100644 index 000000000..43440d636 --- /dev/null +++ b/db/upgrade/0.3.2/upgrade_ddl.sql @@ -0,0 +1,15 @@ + +ALTER TABLE linkis_stream_audit_log + ADD COLUMN `job_name` VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL; + +ALTER TABLE linkis_stream_error_code + ADD COLUMN solution TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin COMMENT 'error code solution'; + +ALTER TABLE linkis_stream_job + ADD COLUMN enable tinyint(1) DEFAULT 1; + +ALTER TABLE linkis_stream_task + ADD COLUMN `solution` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin COMMENT 'error code solution'; + +ALTER TABLE linkis_stream_task + ADD COLUMN `template_id` bigint(20) DEFAULT NULL; diff --git a/new-web/src/pages/auditLogs/index.vue b/new-web/src/pages/auditLogs/index.vue index 633c3e8ea..0985a1431 100644 --- a/new-web/src/pages/auditLogs/index.vue +++ b/new-web/src/pages/auditLogs/index.vue @@ -35,6 +35,14 @@ > +
+ 应用名称 + + +
时间范围 + com.webank.wedatasphere.streamis streamis - 0.3.1 + ${revision} pom Streamis Project Parent POM @@ -45,16 +45,15 @@ - 1.4.0 + 0.3.2 + 1.7.0 4.12 - 1.1.2 - 0.3.1 - 2.11.12 + 1.2.2 + 2.12.17 1.8 3.3.3 2.8.5 1.31 - 2.13.2 3.1.1 4.5.4 4.5.4 @@ -70,10 +69,11 @@ 0.9.10 2.21 1.9.5 - 1.4.19 - 5.1.47 + 8.0.28 2.0.1.Final - 1.4.20 + 1.4.21 + 1.3.0 + 9.4.54.v20240208 @@ -152,11 +152,6 @@ gson ${gson.version} - - com.fasterxml.jackson.core - jackson-databind - ${fasterxml.jackson.version} - org.apache.commons commons-math3 @@ -225,6 +220,31 @@ + + org.codehaus.mojo + flatten-maven-plugin + ${flatten-maven-plugin.version} + + true + resolveCiFriendliesOnly + + + + flatten + + flatten + + process-resources + + + flatten.clean + + clean + + clean + + + org.apache.maven.plugins maven-deploy-plugin @@ -276,7 +296,17 @@ net.alchim31.maven scala-maven-plugin - 3.2.2 + 4.8.1 + + incremental + + -unchecked + -deprecation + -feature + -explaintypes + -target:jvm-8 + + eclipse-add-source @@ -286,31 +316,19 @@ scala-compile-first - process-resources compile + process-resources scala-test-compile-first - process-test-resources testCompile - - - attach-scaladocs - verify - - doc-jar - + process-test-resources - - ${scala.version} - incremental - true - org.apache.maven.plugins @@ -319,5 +337,11 @@ + + + org.codehaus.mojo + flatten-maven-plugin + + \ No newline at end of file diff --git a/streamis-appconn/pom.xml b/streamis-appconn/pom.xml index d71d69abb..b2374c08d 100644 --- a/streamis-appconn/pom.xml +++ b/streamis-appconn/pom.xml @@ -5,7 +5,7 @@ streamis com.webank.wedatasphere.streamis - 0.3.1 + ${revision} 4.0.0 diff --git a/streamis-jobmanager/pom.xml b/streamis-jobmanager/pom.xml index 36aac3adf..f49a9fc4d 100644 --- a/streamis-jobmanager/pom.xml +++ b/streamis-jobmanager/pom.xml @@ -20,7 +20,7 @@ streamis com.webank.wedatasphere.streamis - 0.3.1 + ${revision} 4.0.0 diff --git a/streamis-jobmanager/streamis-job-audit-log/pom.xml b/streamis-jobmanager/streamis-job-audit-log/pom.xml index 575dc209c..822848270 100644 --- a/streamis-jobmanager/streamis-job-audit-log/pom.xml +++ b/streamis-jobmanager/streamis-job-audit-log/pom.xml @@ -20,7 +20,7 @@ streamis com.webank.wedatasphere.streamis - 0.3.1 + ${revision} ../../pom.xml 4.0.0 @@ -37,22 +37,17 @@ org.apache.linkis linkis-common + ${linkis.version} org.apache.linkis linkis-module - - - org.mybatis - mybatis - 3.5.7 - compile + ${linkis.version} com.webank.wedatasphere.streamis streamis-job-manager-service - ${streamis.version} - compile + ${project.version} diff --git a/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/aspect/AuditLogAspect.java b/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/aspect/AuditLogAspect.java index 18e8403e7..191ae3d58 100644 --- a/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/aspect/AuditLogAspect.java +++ b/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/aspect/AuditLogAspect.java @@ -26,11 +26,7 @@ import javax.annotation.PostConstruct; import javax.servlet.http.HttpServletRequest; -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -48,6 +44,9 @@ public class AuditLogAspect { private static final Pattern PROJECT_NAME_PATTERN = Pattern.compile("[?&]projectName=([^&]+)"); + private static final String PROJECT_NAME = "projectName"; + private static final String JOB_ID = "jobId"; + private static final String BULK_SUBJECT = "bulkSubject"; @PostConstruct private void init() { @@ -65,6 +64,9 @@ public Object captureAndLogAuditLog(ProceedingJoinPoint joinPoint) throws Throwa String userName = proxyUserEntity.getUsername(); Map requestParams = getRequestParamsByProceedingJoinPoint(joinPoint); + if (requestParams.containsKey("response")) { + requestParams.remove("response"); + } Object result = null; long startTime = System.currentTimeMillis(); try { @@ -74,14 +76,15 @@ public Object captureAndLogAuditLog(ProceedingJoinPoint joinPoint) throws Throwa throw e; } long costTime = System.currentTimeMillis() - startTime; + String jobName = getJobNameFromRequest(req, requestParams, method); result = Optional.ofNullable(result).orElse("--"); - logAuditInformationAsync(req, requestURI, requestParams, parseObjectToString(result), proxyUser, userName, method, costTime); + logAuditInformationAsync(req, requestURI, requestParams, parseObjectToString(result), proxyUser, userName, method, costTime, jobName); return result; } @Async public void logAuditInformationAsync(HttpServletRequest req, String requestURI, Map requestParams, - String result, String proxyUser, String userName, String method, long costTimeMills) { + String result, String proxyUser, String userName, String method, long costTimeMills,String jobName) { if (!JobAuditConf.AUDIT_LOG_ENABLE.getValue()) { return; } @@ -111,14 +114,14 @@ public void logAuditInformationAsync(HttpServletRequest req, String requestURI, projectName = getProjectNameFromReferer(req.getHeader("Referer")); } try { - logAuditInformation(requestURI, parseObjectToString(requestParams), result, proxyUser, userName, method, projectName, costTimeMills); + logAuditInformation(requestURI, parseObjectToString(requestParams), result, proxyUser, userName, method, projectName, costTimeMills,jobName); } catch (Exception e) { LOG.error("审计日志记录保存失败"); } } - private void logAuditInformation(String requestURI, String requestParams, String result, String proxyUser, String userName, String method, String projectName, long costTimeMills) { + private void logAuditInformation(String requestURI, String requestParams, String result, String proxyUser, String userName, String method, String projectName, long costTimeMills,String jobName) { String apiDesc = InterfaceDescriptionEnum.getDescriptionByUrl(requestURI); String clientIp = getClientIp(); StreamAuditLog auditLog = new StreamAuditLog(); @@ -133,6 +136,7 @@ private void logAuditInformation(String requestURI, String requestParams, String auditLog.setProjectName(projectName); auditLog.setClientIp(clientIp); auditLog.setCostTimeMills(costTimeMills); + auditLog.setJobName(jobName); auditLogService.saveAuditLog(auditLog); } @@ -163,6 +167,13 @@ private Map buildRequestParam(String[] paramNames, Object[] para //如果是批量文件上传 if (value instanceof List) { try { + List tmpList = (List) value; + if (tmpList.size() == 0) { + requestParams.put(paramNames[i], value); + break; + } + Object tmpV1 = tmpList.get(0); + if (MultipartFile.class.isInstance(tmpV1)) { List multipartFiles = castList(value, MultipartFile.class); if (multipartFiles != null) { List fileNames = new ArrayList<>(); @@ -173,8 +184,9 @@ private Map buildRequestParam(String[] paramNames, Object[] para requestParams.put(paramNames[i], fileNames); break; } + } } catch (ClassCastException e) { - + LOG.warn("buildRequestParam: " + value + " failed. " + e.getMessage(), e); } } if (!paramNames[i].equalsIgnoreCase("req") && !paramNames[i].equalsIgnoreCase("request")) { @@ -185,10 +197,14 @@ private Map buildRequestParam(String[] paramNames, Object[] para } private static List castList(Object obj, Class clazz) { - List result = new ArrayList(); + List result = new ArrayList<>(); if (obj instanceof List) { for (Object o : (List) obj) { + if (clazz.isInstance(o)) { result.add(clazz.cast(o)); + } else if (null != o) { + LOG.warn("discarded {}", o.getClass()); + } } return result; } @@ -245,9 +261,9 @@ private String getProjectNameFromGetRequest(HttpServletRequest req) { || req.getRequestURI().equals(InterfaceDescriptionEnum.CONFIG_DEFINITIONS.getUrl())) { return ""; } - String projectName = req.getParameter("projectName"); + String projectName = req.getParameter(PROJECT_NAME); if (projectName == null || projectName.isEmpty()) { - Long jobId = Long.valueOf(req.getParameter("jobId")); + Long jobId = Long.valueOf(req.getParameter(JOB_ID)); projectName = auditLogService.getProjectNameById(jobId); } @@ -257,19 +273,19 @@ private String getProjectNameFromGetRequest(HttpServletRequest req) { private String getProjectNameFromPutRequest(HttpServletRequest req, Map requestParams) { String projectName = ""; if (req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_EXECUTE_INSPECT.getUrl())) { - String[] jobIdArray = req.getParameterValues("jobId"); + String[] jobIdArray = req.getParameterValues(JOB_ID); List jobIdList = Arrays.stream(jobIdArray) .map(Integer::valueOf) .collect(Collectors.toList()); Integer jobId = jobIdList.get(0); projectName = auditLogService.getProjectNameById(Long.valueOf(jobId)); } else if (req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_STATUS.getUrl())) { - Map map = BDPJettyServerHelper.gson().fromJson(requestParams.get("requestMap").toString(), Map.class); + Map map = BDPJettyServerHelper.gson().fromJson(requestParams.get("requestMap").toString(), Map.class); List jobIds = (List) map.get("id_list"); Long jobId = Math.round(jobIds.get(0)); projectName = auditLogService.getProjectNameById(jobId); } else { - String jobId = req.getParameter("jobId"); + String jobId = req.getParameter(JOB_ID); projectName = auditLogService.getProjectNameById(Long.valueOf(jobId)); } return projectName; @@ -279,68 +295,161 @@ private String getProjectNameFromPostRequest(HttpServletRequest req, Map tasks = gson.fromJson(gson.toJson(requestParams.get("bulkUpdateLabelRequest")), Map.class); List> list = gson.fromJson(tasks.get("tasks").toString(), new TypeToken>>() { }.getType()); String jobId = list.get(0).get("id").toString(); projectName = auditLogService.getProjectNameById((long) Double.parseDouble(jobId)); } else if (req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_EXECUTE.getUrl())) { - Map map = gson.fromJson(requestParams.get("json").toString(), Map.class); - Long jobId = (long) Double.parseDouble(map.get("jobId").toString()); + Map map = gson.fromJson(requestParams.get("json").toString(), Map.class); + Long jobId = (long) Double.parseDouble(map.get(JOB_ID).toString()); projectName = auditLogService.getProjectNameById(jobId); } else if (req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_UPDATE_CONTENT.getUrl())) { - Map contentRequest = gson.fromJson(gson.toJson(requestParams.get("contentRequest")), Map.class); - Long jobId = (long) Double.parseDouble(contentRequest.get("jobId").toString()); + Map contentRequest = gson.fromJson(gson.toJson(requestParams.get("contentRequest")), Map.class); + Long jobId = (long) Double.parseDouble(contentRequest.get(JOB_ID).toString()); projectName = auditLogService.getProjectNameById(jobId); } else if (req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_BULK_EXECUTION.getUrl())) { - Map bulkRequest = gson.fromJson(gson.toJson(requestParams.get("execBulkRequest")), Map.class); - List bulkSbjList = (List) bulkRequest.get("bulkSubject"); + Map bulkRequest = gson.fromJson(gson.toJson(requestParams.get("execBulkRequest")), Map.class); + List bulkSbjList = (List) bulkRequest.get(BULK_SUBJECT); Long jobId = Math.round(bulkSbjList.get(0)); projectName = auditLogService.getProjectNameById(jobId); } else if (req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_BULK_PAUSE.getUrl())) { - Map bulkRequest = gson.fromJson(gson.toJson(requestParams.get("pauseRequest")), Map.class); - List bulkSbjList = (List) bulkRequest.get("bulkSubject"); + Map bulkRequest = gson.fromJson(gson.toJson(requestParams.get("pauseRequest")), Map.class); + List bulkSbjList = (List) bulkRequest.get(BULK_SUBJECT); Long jobId = Math.round(bulkSbjList.get(0)); projectName = auditLogService.getProjectNameById(jobId); } else if (req.getRequestURI().equals(InterfaceDescriptionEnum.PROJECT_FILES_UPLOAD.getUrl()) || req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_UPLOAD.getUrl())) { - projectName = req.getParameter("projectName"); + projectName = req.getParameter(PROJECT_NAME); } else if (req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_ENABLE.getUrl()) || req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_BAN.getUrl())) { List jobIdList = (List) requestParams.get("jobIdList"); Long jobId = jobIdList.get(0); projectName = auditLogService.getProjectNameById(jobId); } else if (req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_CREATE_OR_UPDATE.getUrl())) { - Map metaJsonInfo = gson.fromJson(gson.toJson(requestParams.get("metaJsonInfo")), Map.class); - projectName = metaJsonInfo.get("projectName").toString(); + Map metaJsonInfo = gson.fromJson(gson.toJson(requestParams.get("metaJsonInfo")), Map.class); + projectName = metaJsonInfo.get(PROJECT_NAME).toString(); } else if (req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_UPDATE_TASK.getUrl())) { - Map json = gson.fromJson(gson.toJson(requestParams.get("json")), Map.class); - projectName = json.get("projectName").toString(); + Map json = gson.fromJson(gson.toJson(requestParams.get("json")), Map.class); + projectName = json.get(PROJECT_NAME).toString(); } else { - projectName = requestParams.get("projectName").toString(); + projectName = requestParams.get(PROJECT_NAME).toString(); } return projectName; } + private String getJobNameFromRequest(HttpServletRequest req, Map requestParams, String method) { + if ("GET".equalsIgnoreCase(method)) { + return getJobNameFromGetRequest(req); + } else if ("PUT".equalsIgnoreCase(method)) { + return getJobNameFromPutRequest(req, requestParams); + } else if ("POST".equalsIgnoreCase(method)) { + return getJobNameFromPostRequest(req, requestParams); + } + return "--"; + } - private Map parseJsonToMap(String json) { - try { - return BDPJettyServerHelper.jacksonJson().readValue(json, Map.class); - } catch (IOException e) { - e.printStackTrace(); - return Collections.emptyMap(); + private String getJobNameFromGetRequest(HttpServletRequest req) { + String jobIdStr = req.getParameter(JOB_ID); + if (StringUtils.isNotBlank(jobIdStr)) { + Long jobId = Long.valueOf(jobIdStr); + return auditLogService.getJobNameById(jobId); + } else { + return null; + } + } + + private String getJobNameFromPutRequest(HttpServletRequest req, Map requestParams) { + if (req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_EXECUTE_INSPECT.getUrl())) { + String[] jobIdArray = req.getParameterValues(JOB_ID); + List jobIds = Arrays.stream(jobIdArray) + .map(Long::valueOf) + .collect(Collectors.toList()); + List jobNames = auditLogService.getBulkJobNameByIds(jobIds); + return String.join(",", jobNames); + } else if (req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_STATUS.getUrl())) { + Map map = BDPJettyServerHelper.gson().fromJson(requestParams.get("requestMap").toString(), Map.class); + List jobIdDouble = (List) map.get("id_list"); + List jobIds = jobIdDouble.stream().map(Math::round).collect(Collectors.toList()); + List jobNames = auditLogService.getBulkJobNameByIds(jobIds); + return String.join(",", jobNames); + } else if (req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_SNAPSHOT.getUrl())){ + if (requestParams.get(JOB_ID) != null){ + String jobIdStr = requestParams.get(JOB_ID).toString(); + Long jobId = Long.valueOf(jobIdStr); + return auditLogService.getJobNameById(jobId); + } + return null; + } else { + String jobIdStr = req.getParameter(JOB_ID); + if (StringUtils.isNotBlank(jobIdStr)) { + Long jobId = Long.valueOf(jobIdStr); + return auditLogService.getJobNameById(jobId); + }else{ + return null; + } + } + } + + private String getJobNameFromPostRequest(HttpServletRequest req, Map requestParams) { + String jobName = null; + Gson gson = BDPJettyServerHelper.gson(); + if (req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_UPDATE_LABEL.getUrl())) { + Map tasks = gson.fromJson(gson.toJson(requestParams.get("bulkUpdateLabelRequest")), Map.class); + List> list = gson.fromJson(tasks.get("tasks").toString(), new TypeToken>>() { + }.getType()); + String jobId = list.get(0).get("id").toString(); + jobName = auditLogService.getJobNameById((long) Double.parseDouble(jobId)); + } else if (req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_EXECUTE.getUrl())) { + Map map = gson.fromJson(requestParams.get("json").toString(), Map.class); + Long jobId = (long) Double.parseDouble(map.get(JOB_ID).toString()); + jobName = auditLogService.getJobNameById(jobId); + } else if (req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_UPDATE_CONTENT.getUrl())) { + Map contentRequest = gson.fromJson(gson.toJson(requestParams.get("contentRequest")), Map.class); + Long jobId = (long) Double.parseDouble(contentRequest.get(JOB_ID).toString()); + jobName = auditLogService.getJobNameById(jobId); + } else if (req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_BULK_EXECUTION.getUrl())) { + Map bulkRequest = gson.fromJson(gson.toJson(requestParams.get("execBulkRequest")), Map.class); + List bulkSbjList = (List) bulkRequest.get(BULK_SUBJECT); + List jobIds = bulkSbjList.stream().map(Math::round).collect(Collectors.toList()); + List jobNames = auditLogService.getBulkJobNameByIds(jobIds); + jobName = String.join(",", jobNames); + } else if (req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_BULK_PAUSE.getUrl())) { + Map bulkRequest = gson.fromJson(gson.toJson(requestParams.get("pauseRequest")), Map.class); + List bulkSbjList = (List) bulkRequest.get(BULK_SUBJECT); + List jobIds = bulkSbjList.stream().map(Math::round).collect(Collectors.toList()); + List jobNames = auditLogService.getBulkJobNameByIds(jobIds); + jobName = String.join(",", jobNames); + } else if (req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_ENABLE.getUrl()) + || req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_BAN.getUrl())) { + List jobIds = (List) requestParams.get("jobIdList"); + List jobNames = auditLogService.getBulkJobNameByIds(jobIds); + jobName = String.join(",", jobNames); + } else if (req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_CREATE_OR_UPDATE.getUrl())) { + Map metaJsonInfo = gson.fromJson(gson.toJson(requestParams.get("metaJsonInfo")), Map.class); + jobName = metaJsonInfo.get("jobName").toString(); + } else if (req.getRequestURI().equals(InterfaceDescriptionEnum.JOB_UPDATE_TASK.getUrl())) { + Map json = gson.fromJson(gson.toJson(requestParams.get("json")), Map.class); + jobName = json.get("jobName").toString(); + } else { + if (requestParams.get(JOB_ID) != null){ + String jobIdStr = requestParams.get(JOB_ID).toString(); + Long jobId = Long.valueOf(jobIdStr); + jobName = auditLogService.getJobNameById(jobId); + } } + return jobName; } - private String parseObjectToString(Object json) { + private String parseObjectToString(Object obj) { try { - if (ObjectUtils.isEmpty(json)) { + if (ObjectUtils.isEmpty(obj)) { return "--"; } else { - return BDPJettyServerHelper.gson().toJson(json); + return BDPJettyServerHelper.gson().toJson(obj); } } catch (Exception e) { - LOG.error("failed parse map to string "); + LOG.error("failed parse map to string . " + e.getMessage()); } return "--"; } diff --git a/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/aspect/InterfaceDescriptionEnum.java b/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/aspect/InterfaceDescriptionEnum.java index 697716747..8ae3ec5bd 100644 --- a/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/aspect/InterfaceDescriptionEnum.java +++ b/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/aspect/InterfaceDescriptionEnum.java @@ -3,7 +3,9 @@ import org.springframework.util.AntPathMatcher; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; public enum InterfaceDescriptionEnum { @@ -47,16 +49,11 @@ public enum InterfaceDescriptionEnum { JOB_SNAPSHOT("/api/rest_j/v1/streamis/streamJobManager/job/snapshot/{jobId:\\w+}","生成Job快照"), JOB_UPLOAD("/api/rest_j/v1/streamis/streamJobManager/job/upload","上传Job"), JOB_ENABLE("/api/rest_j/v1/streamis/streamJobManager/job/enable","开启Job"), - JOB_BAN("/api/rest_j/v1/streamis/streamJobManager/job/ban","禁用Job"); + JOB_BAN("/api/rest_j/v1/streamis/streamJobManager/job/ban","禁用Job"), + USERNAME_PROXYUSER("/api/rest_j/v1/streamis/streamJobManager/highAvailable/username","得到用户名和代理用户名"); private String url; private String description; - - private static String systemPath = "/api/rest_j/v1"; - - private static String projectManagerRestfulApiPath = "/streamis/streamProjectManager/project"; - - InterfaceDescriptionEnum(String url, String description) { this.url = url; this.description = description; diff --git a/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/conf/JobAuditConf.java b/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/conf/JobAuditConf.java index df5e60c8e..2d068a878 100644 --- a/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/conf/JobAuditConf.java +++ b/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/conf/JobAuditConf.java @@ -9,7 +9,7 @@ public class JobAuditConf { /* divided by comma */ - public static final CommonVars AUDIT_LOG_URI_SKIP = CommonVars.apply("wds.streamis.log.audit.uri.skip", "/api/rest_j/v1/streamis/streamJobManager/log/collect/events,/api/rest_j/v1/streamis/streamJobManager/log/heartbeat"); + public static final CommonVars AUDIT_LOG_URI_SKIP = CommonVars.apply("wds.streamis.log.audit.uri.skip", "/api/rest_j/v1/streamis/streamJobManager/log/collect/events,/api/rest_j/v1/streamis/streamJobManager/log/heartbeat,/api/rest_j/v1/streamis/streamProjectManager/project/files/list,/api/rest_j/v1/streamis/streamProjectManager/project/files/version/list,/api/rest_j/v1/streamis/streamJobManager/job/list,/api/rest_j/v1/streamis/streamJobManager/job/execute/history,/api/rest_j/v1/streamis/streamJobManager/job/logs,/api/rest_j/v1/streamis/streamJobManager/job/alert,/api/rest_j/v1/streamis/streamJobManager/job/status,/api/rest_j/v1/streamis/streamJobManager/highAvailable/username"); } diff --git a/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/dao/StreamAuditLogMapper.java b/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/dao/StreamAuditLogMapper.java index adc15fd3b..b2c4d8cd1 100644 --- a/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/dao/StreamAuditLogMapper.java +++ b/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/dao/StreamAuditLogMapper.java @@ -14,9 +14,16 @@ List searchAuditLogs( @Param("proxyUser") String proxyUser, @Param("startDate") Date startDate, @Param("endDate") Date endDate, - @Param("projectName") String projectName); + @Param("projectName") String projectName, + @Param("jobName") String jobName, + @Param("jobNameIsNull") Boolean jobNameIsNull, + @Param("jobNameLike") String jobNameLike); void saveAuditLog(StreamAuditLog auditLog); String getProjectNameById(@Param("jobId") Long jobId); + + String getJobNameById(@Param("jobId") Long jobId); + + List getBulkJobNameByIds(@Param("jobIds") List jobIds); } diff --git a/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/dao/impl/StreamAuditLogMapper.xml b/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/dao/impl/StreamAuditLogMapper.xml index 683acc2ec..276699c06 100644 --- a/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/dao/impl/StreamAuditLogMapper.xml +++ b/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/dao/impl/StreamAuditLogMapper.xml @@ -38,16 +38,37 @@ AND operate_time <= #{endDate} + + AND job_name IS NULL + + + AND job_name LIKE #{jobNameLike} + order by operate_time desc - INSERT INTO linkis_stream_audit_log (user,api_name,api_type,api_desc,input_parameters, output_parameters, proxy_user, operate_time,project_name,client_ip,cost_time_mills) - VALUES (#{user}, #{apiName},#{apiType},#{apiDesc},#{inputParameters}, #{outputParameters}, #{proxyUser}, #{operateTime},#{projectName},#{clientIp},#{costTimeMills}) + INSERT INTO linkis_stream_audit_log (user,api_name,api_type,api_desc,input_parameters, output_parameters, proxy_user, operate_time,project_name,client_ip,cost_time_mills,job_name) + VALUES (#{user}, #{apiName},#{apiType},#{apiDesc},#{inputParameters}, #{outputParameters}, #{proxyUser}, #{operateTime},#{projectName},#{clientIp},#{costTimeMills},#{jobName}) + + + + \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/entity/StreamAuditLog.java b/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/entity/StreamAuditLog.java index b7783eafd..6f1aaa2b2 100644 --- a/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/entity/StreamAuditLog.java +++ b/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/entity/StreamAuditLog.java @@ -19,6 +19,8 @@ public class StreamAuditLog { private Long costTimeMills; + private String jobName; + public String getClientIp() { return clientIp; } @@ -116,4 +118,12 @@ public Long getCostTimeMills() { public void setCostTimeMills(Long costTimeMills) { this.costTimeMills = costTimeMills; } + + public String getJobName() { + return jobName; + } + + public void setJobName(String jobName) { + this.jobName = jobName; + } } diff --git a/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/restful/api/AuditLogRestfulApi.java b/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/restful/api/AuditLogRestfulApi.java index c4e045c17..0f3e555d1 100644 --- a/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/restful/api/AuditLogRestfulApi.java +++ b/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/restful/api/AuditLogRestfulApi.java @@ -45,9 +45,13 @@ public Message searchAuditLogs(HttpServletRequest req, @RequestParam(value = "proxyUser", required = false) String proxyUser, @RequestParam(value = "startDate", required = false) @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") Date startDate, @RequestParam(value = "endDate", required = false) @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") Date endDate, - @RequestParam(value = "projectName",required = false) String projectName + @RequestParam(value = "projectName",required = false) String projectName, + @RequestParam(value = "jobName",required = false) String jobName ) { String userName = ModuleUserUtils.getOperationUser(req, "Query job audit log"); + if (StringUtils.isBlank(userName)) { + return Message.error("user name cannot be empty(用户名不能为空，请指定)"); + } if(StringUtils.isBlank(projectName)){ return Message.error("Project name cannot be empty(项目名不能为空，请指定)"); } @@ -63,7 +67,7 @@ public Message searchAuditLogs(HttpServletRequest req, PageInfo pageInfo; PageHelper.startPage(pageNow, pageSize); try { - pageInfo = auditLogService.searchAuditLogs(apiName, user, proxyUser, startDate, endDate,projectName); + pageInfo = auditLogService.searchAuditLogs(apiName, user, proxyUser, startDate, endDate,projectName,jobName); } finally { PageHelper.clearPage(); } diff --git a/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/service/AuditLogService.java b/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/service/AuditLogService.java index bdbb5eace..9cadf20ec 100644 --- a/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/service/AuditLogService.java +++ b/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/service/AuditLogService.java @@ -9,9 +9,13 @@ public interface AuditLogService { - PageInfo searchAuditLogs(String apiName, String user, String proxyUser, Date startDate, Date endDate,String projectName); + PageInfo searchAuditLogs(String apiName, String user, String proxyUser, Date startDate, Date endDate,String projectName,String jobName); void saveAuditLog(StreamAuditLog auditLog); String getProjectNameById(Long jobId); + + String getJobNameById(Long jobId); + + List getBulkJobNameByIds(List jobIds); } diff --git a/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/service/impl/AuditLogServiceImpl.java b/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/service/impl/AuditLogServiceImpl.java index 709083180..e02510d54 100644 --- a/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/service/impl/AuditLogServiceImpl.java +++ b/streamis-jobmanager/streamis-job-audit-log/src/main/java/com/webank/wedatasphere/streamis/audit/log/service/impl/AuditLogServiceImpl.java @@ -4,6 +4,7 @@ import com.webank.wedatasphere.streamis.audit.log.dao.StreamAuditLogMapper; import com.webank.wedatasphere.streamis.audit.log.entity.StreamAuditLog; import com.webank.wedatasphere.streamis.audit.log.service.AuditLogService; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -15,8 +16,15 @@ public class AuditLogServiceImpl implements AuditLogService { @Autowired private StreamAuditLogMapper auditLogMapper; - public PageInfo searchAuditLogs(String apiName, String user, String proxyUser, Date startDate, Date endDate,String projectName) { - List streamAuditLogs = auditLogMapper.searchAuditLogs(apiName, user, proxyUser, startDate, endDate,projectName); + public PageInfo searchAuditLogs(String apiName, String user, String proxyUser, Date startDate, Date endDate,String projectName,String jobName) { + Boolean jobNameIsNull = false; + String jobNameLike = null; + if ("*".equals(jobName)) { + jobNameIsNull = true; + } else if (StringUtils.isNotBlank(jobName)) { + jobNameLike = "%" + jobName + "%"; + } + List streamAuditLogs = auditLogMapper.searchAuditLogs(apiName, user, proxyUser, startDate, endDate,projectName,jobName,jobNameIsNull,jobNameLike); return new PageInfo<>(streamAuditLogs); } @@ -30,4 +38,13 @@ public String getProjectNameById(Long jobId) { return auditLogMapper.getProjectNameById(jobId); } + @Override + public String getJobNameById(Long jobId){ + return auditLogMapper.getJobNameById(jobId); + } + + public List getBulkJobNameByIds(List jobIds){ + return auditLogMapper.getBulkJobNameByIds(jobIds); + } + } diff --git a/streamis-jobmanager/streamis-job-entrypoint/pom.xml b/streamis-jobmanager/streamis-job-entrypoint/pom.xml index 6b54298b3..7f7162bae 100644 --- a/streamis-jobmanager/streamis-job-entrypoint/pom.xml +++ b/streamis-jobmanager/streamis-job-entrypoint/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.3.1 + ${revision} 4.0.0 pom diff --git a/streamis-jobmanager/streamis-job-entrypoint/streamis-job-entrypoint-common/pom.xml b/streamis-jobmanager/streamis-job-entrypoint/streamis-job-entrypoint-common/pom.xml index 2ec020dbd..103e8b6e6 100644 --- a/streamis-jobmanager/streamis-job-entrypoint/streamis-job-entrypoint-common/pom.xml +++ b/streamis-jobmanager/streamis-job-entrypoint/streamis-job-entrypoint-common/pom.xml @@ -20,7 +20,7 @@ streamis-job-entrypoint com.webank.wedatasphere.streamis - 0.3.1 + ${revision} 4.0.0 @@ -50,7 +50,7 @@ com.webank.wedatasphere.streamis streamis-jobmanager-common - 0.3.1 + ${project.version} junit diff --git a/streamis-jobmanager/streamis-job-entrypoint/streamis-job-entrypoint-flink/pom.xml b/streamis-jobmanager/streamis-job-entrypoint/streamis-job-entrypoint-flink/pom.xml index 59ccedf7e..0f3d1dc18 100644 --- a/streamis-jobmanager/streamis-job-entrypoint/streamis-job-entrypoint-flink/pom.xml +++ b/streamis-jobmanager/streamis-job-entrypoint/streamis-job-entrypoint-flink/pom.xml @@ -5,7 +5,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.3.1 + ${revision} ../../pom.xml 4.0.0 @@ -21,7 +21,7 @@ com.webank.wedatasphere.streamis streamis-job-entrypoint-common - 0.3.1 + ${project.version} diff --git a/streamis-jobmanager/streamis-job-entrypoint/streamis-job-entrypoint-spark/pom.xml b/streamis-jobmanager/streamis-job-entrypoint/streamis-job-entrypoint-spark/pom.xml index c4c82276b..4e1ffbe67 100644 --- a/streamis-jobmanager/streamis-job-entrypoint/streamis-job-entrypoint-spark/pom.xml +++ b/streamis-jobmanager/streamis-job-entrypoint/streamis-job-entrypoint-spark/pom.xml @@ -5,7 +5,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.3.1 + ${revision} ../../pom.xml 4.0.0 @@ -21,7 +21,7 @@ com.webank.wedatasphere.streamis streamis-job-entrypoint-common - 0.3.1 + ${project.version} diff --git a/streamis-jobmanager/streamis-job-errorcode/pom.xml b/streamis-jobmanager/streamis-job-errorcode/pom.xml index 00f8cae02..ee164d550 100644 --- a/streamis-jobmanager/streamis-job-errorcode/pom.xml +++ b/streamis-jobmanager/streamis-job-errorcode/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.3.1 + ${revision} 4.0.0 @@ -38,16 +38,24 @@ org.apache.linkis linkis-common + provided org.apache.linkis linkis-module + provided org.apache.linkis - linkis-error-code-client + linkis-pes-client ${linkis.version} + + org.junit.jupiter + junit-jupiter + RELEASE + test + diff --git a/streamis-jobmanager/streamis-job-errorcode/src/main/java/com/webank/wedatasphere/streamis/errorcode/handler/StreamisErrorCodeHandler.java b/streamis-jobmanager/streamis-job-errorcode/src/main/java/com/webank/wedatasphere/streamis/errorcode/handler/StreamisErrorCodeHandler.java index ab398e3bb..27af89580 100644 --- a/streamis-jobmanager/streamis-job-errorcode/src/main/java/com/webank/wedatasphere/streamis/errorcode/handler/StreamisErrorCodeHandler.java +++ b/streamis-jobmanager/streamis-job-errorcode/src/main/java/com/webank/wedatasphere/streamis/errorcode/handler/StreamisErrorCodeHandler.java @@ -5,14 +5,9 @@ import com.webank.wedatasphere.streamis.errorcode.utils.StreamisErrorCodeMatcher; import org.apache.linkis.common.utils.Utils; import org.apache.linkis.errorcode.client.ClientConfiguration; -import org.apache.linkis.errorcode.client.handler.LinkisErrorCodeHandler; -import org.apache.linkis.errorcode.client.utils.ErrorCodeMatcher; -import org.apache.linkis.errorcode.common.ErrorCode; -import org.apache.linkis.errorcode.common.LinkisErrorCode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import scala.Option; -import scala.Tuple2; import scala.Tuple3; import java.util.*; diff --git a/streamis-jobmanager/streamis-job-errorcode/src/main/scala/com/webank/wedatasphere/streamis/errorcode/utils/StreamisErrorCodeMatcher.scala b/streamis-jobmanager/streamis-job-errorcode/src/main/scala/com/webank/wedatasphere/streamis/errorcode/utils/StreamisErrorCodeMatcher.scala index d304da560..b0a17ed1c 100644 --- a/streamis-jobmanager/streamis-job-errorcode/src/main/scala/com/webank/wedatasphere/streamis/errorcode/utils/StreamisErrorCodeMatcher.scala +++ b/streamis-jobmanager/streamis-job-errorcode/src/main/scala/com/webank/wedatasphere/streamis/errorcode/utils/StreamisErrorCodeMatcher.scala @@ -29,12 +29,15 @@ object StreamisErrorCodeMatcher extends Logging { Utils.tryCatch { import scala.collection.JavaConverters._ errorCodes.asScala.foreach(e => - if (e.getErrorRegex.r.findFirstIn(log).isDefined) { - val matched = e.getErrorRegex.r.unapplySeq(log) + { + val regexUnanchored = e.getErrorRegex.r.unanchored + if (regexUnanchored.findFirstIn(log).isDefined) { + val matched = regexUnanchored.unapplySeq(log) if (matched.nonEmpty) { return Some(e.getErrorCode, e.getErrorDesc.format(matched.get: _*), e.getSolution) } else return Some(e.getErrorCode, e.getErrorDesc, e.getSolution) } + } ) None } { t: Throwable => diff --git a/streamis-jobmanager/streamis-job-launcher/pom.xml b/streamis-jobmanager/streamis-job-launcher/pom.xml index ba47ee8a5..8a3e70b96 100755 --- a/streamis-jobmanager/streamis-job-launcher/pom.xml +++ b/streamis-jobmanager/streamis-job-launcher/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.3.1 + ${revision} 4.0.0 diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/pom.xml b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/pom.xml index 126096d6a..dff796efc 100755 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/pom.xml +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.3.1 + ${revision} ../../pom.xml 4.0.0 @@ -49,8 +49,9 @@ linkis-mybatis - org.apache.linkis - linkis-module + com.google.code.gson + gson + ${gson.version} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/job/conf/JobConf.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/job/conf/JobConf.scala index 92ace021e..7cb86c431 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/job/conf/JobConf.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/job/conf/JobConf.scala @@ -104,9 +104,9 @@ object JobConf { val DEFAULT_ERROR_MSG: CommonVars[String] = CommonVars("wds.streamis.default.error.msg", "正在分析日志，请稍后") - val ANALYZE_ERROR_MSG: CommonVars[String] = CommonVars("wds.streamis.default.error.msg", "错误原因分析失败") + val ANALYZE_ERROR_MSG: CommonVars[String] = CommonVars("wds.streamis.analyze.error.msg", "错误原因分析失败") - val FINAL_ERROR_MSG: CommonVars[String] = CommonVars("wds.streamis.default.error.msg", "无法匹配到对应错误码，请联系管理员") + val FINAL_ERROR_MSG: CommonVars[String] = CommonVars("wds.streamis.final.error.msg", "无法匹配到对应错误码，请联系管理员") val PROJECT_NAME_STRICT_CHECK_SWITCH: CommonVars[Boolean] = CommonVars("wds.streamis.project.name.strict.check.enable", false) @@ -114,8 +114,6 @@ object JobConf { val HIGHAVAILABLE_SOURCE: CommonVars[String] = CommonVars("wds.streamis.app.highavailable.source", "aomp") -// val HIGHAVAILABLE_POLICY: CommonVars[String] = CommonVars("wds.streamis.app.highavailable.policy.double", "double") - val HIGHAVAILABLE_POLICY_DOUBLE: CommonVars[String] = CommonVars("wds.streamis.app.highavailable.policy.double", "double") val HIGHAVAILABLE_POLICY_DOUBLE_BAK: CommonVars[String] = CommonVars("wds.streamis.app.highavailable.policy.doubleWithBak", "doubleWithBak") @@ -128,6 +126,8 @@ object JobConf { val HIGHAVAILABLE_DEFAULT_POLICY: CommonVars[String] = CommonVars("wds.streamis.app.highavailable.default.policy", "single") + val HIGHAVAILABLE_ENABLE_INTERFACE_UPLOAD: CommonVars[Boolean] = CommonVars("wds.streamis.app.highavailable.enable.interface.upload", true) + val JOB_SCHEMA_SINGLE: CommonVars[String] = CommonVars("wds.streamis.app.job.schema.single", "single") val JOB_SCHEMA_DOUBLE: CommonVars[String] = CommonVars("wds.streamis.app.job.schema.double", "double") @@ -162,7 +162,7 @@ object JobConf { val HIGHAVAILABLE_TOKEN: CommonVars[String] = CommonVars("wds.streamis.app.highavailable.token", "streamis0.3.8") - val STANDARD_AUTHENTICATION_KEY: CommonVars[Boolean] = CommonVars("wds.streamis.standard.authentication.key", true) + val STANDARD_AUTHENTICATION_KEY: CommonVars[Boolean] = CommonVars("wds.streamis.standard.authentication.key", false) val JOB_CONTENT_EDIT_ENABLE: CommonVars[Boolean] = CommonVars("wds.streamis.job.content.edit.enable", true) diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/conf/JobConfKeyConstants.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/job/conf/JobConfKeyConstants.scala similarity index 98% rename from streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/conf/JobConfKeyConstants.scala rename to streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/job/conf/JobConfKeyConstants.scala index 75ad7d215..77a8a4f15 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/conf/JobConfKeyConstants.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-base/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/job/conf/JobConfKeyConstants.scala @@ -13,7 +13,7 @@ * limitations under the License. */ -package com.webank.wedatasphere.streamis.jobmanager.launcher.conf +package com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf import org.apache.linkis.common.conf.CommonVars import org.apache.linkis.governance.common.conf.GovernanceCommonConf diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/pom.xml b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/pom.xml index dd469295d..df94b4c09 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/pom.xml +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/pom.xml @@ -20,7 +20,7 @@ streamis-job-launcher com.webank.wedatasphere.streamis - 0.3.1 + ${revision} 4.0.0 @@ -35,7 +35,7 @@ com.webank.wedatasphere.streamis streamis-job-launcher-base - ${streamis.version} + ${project.version} org.apache.linkis @@ -50,7 +50,7 @@ com.webank.wedatasphere.streamis streamis-job-errorcode - ${streamis.version} + ${project.version} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/utils/HttpClientUtil.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/utils/HttpClientUtil.java index f0a5d1229..ade56f139 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/utils/HttpClientUtil.java +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/utils/HttpClientUtil.java @@ -32,7 +32,7 @@ public class HttpClientUtil { private HttpClientUtil() {} - private static Logger logger = LoggerFactory.getLogger(HttpClientUtil.class); + private static final Logger logger = LoggerFactory.getLogger(HttpClientUtil.class); private static String SPARK_SECRET_PATH = ""; @@ -96,8 +96,8 @@ public static HttpPost getPostRequest(String uri, HttpEntity entity, String... h HttpPost httpPost = new HttpPost(uri); httpPost.setEntity(entity); if(headers.length % 2 == 0){ - for(int i = 0; i < headers.length; i++){ - httpPost.addHeader(headers[i], headers[++i]); + for(int i = 0; i < headers.length; i += 2) { + httpPost.addHeader(headers[i], headers[i + 1]); } } return httpPost; diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/conf/JobLauncherConfiguration.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/conf/JobLauncherConfiguration.scala index 05732f6aa..15b623d6a 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/conf/JobLauncherConfiguration.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/conf/JobLauncherConfiguration.scala @@ -72,6 +72,8 @@ object JobLauncherConfiguration { val FLINK_MANAGER_EXTRA_INIT_CONFIGS: CommonVars[String] = CommonVars("wds.streamis.flink.manager.ec.extra_init.params", "linkis.user.udf.all.load=false") + val FLINK_MANAGER_EC_CLASSLOADER_RESOLVE_ORDER: CommonVars[String] = CommonVars("wds.streamis.flink.manager.ec.classloader.resolve-order", "parent-first") + val LINKIS_EC_EXPIRE_TIME_KEY: CommonVars[String] = CommonVars("linkis.ec.expire.key", "wds.linkis.engineconn.max.free.time") val ENABLE_FLINK_LIST_INSPECT: CommonVars[Boolean] = CommonVars("wds.streamis.job.inspact.list.enable", true) diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerClient.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerClient.scala index ecfa6339c..af02110b3 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerClient.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerClient.scala @@ -70,6 +70,7 @@ class LinkisFlinkManagerClient extends FlinkManagerClient with Logging { initProperties.put(JobLauncherConfiguration.FLINK_MANAGER_EC_KEY.getValue, true.toString) initProperties.put(JobLauncherConfiguration.LINKIS_EC_EXPIRE_TIME_KEY.getValue, JobLauncherConfiguration.FLINKK_MANAGER_EXIT_TIME.getHotValue().toString) initProperties.put(JobLauncherConfiguration.LINKIS_EC_SUPPORT_PARALLEM, true.toString) + initProperties.put("classloader.resolve-order", JobLauncherConfiguration.FLINK_MANAGER_EC_CLASSLOADER_RESOLVE_ORDER.getValue) if (StringUtils.isNotBlank(JobLauncherConfiguration.FLINK_MANAGER_EXTRA_INIT_CONFIGS.getValue)) { JobLauncherConfiguration.FLINK_MANAGER_EXTRA_INIT_CONFIGS.getValue.split(JobConstants.DELIMITER_COMMA).foreach(s => { @@ -310,14 +311,6 @@ class LinkisFlinkManagerClient extends FlinkManagerClient with Logging { null } - private def getManagerInstance(nodeInfo: util.Map[String, Any]): ServiceInstance = - nodeInfo.getOrDefault(ECConstants.MANAGER_SERVICE_INSTANCE_KEY, null) match { - case serviceInstance: ServiceInstance => - serviceInstance - case _ => - null - } - private def getAs[T](map: util.Map[String, Any], key: String): T = map.getOrDefault(key, null).asInstanceOf[T] diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerJobClient.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerJobClient.scala index 572934f68..9df79eb60 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerJobClient.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/client/LinkisFlinkManagerJobClient.scala @@ -55,7 +55,6 @@ class LinkisFlinkManagerJobClient(onceJob: OnceJob, jobInfo: JobInfo, stateManag onceJob match { case simpleOnceJob: SimpleOnceJob => if (StringUtils.isNotBlank(jobInfo.getStatus) && JobConf.isCompleted(JobConf.linkisStatusToStreamisStatus(jobInfo.getStatus))) { -// jobInfo.setStatus(simpleOnceJob.getStatus) logger.info(s"Job : ${simpleOnceJob.getId} is completed : ${jobInfo.getStatus}, no need to get status from linkis.") } else if (refresh && isDetachJob(jobInfo)) { jobInfo match { diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/FlinkJobLaunchManager.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/FlinkJobLaunchManager.scala index 2c00c222b..b7137eb7c 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/FlinkJobLaunchManager.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-linkis/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/linkis/job/manager/FlinkJobLaunchManager.scala @@ -156,11 +156,6 @@ trait FlinkJobLaunchManager extends LinkisJobLaunchManager with Logging { * @return */ def exceptionAnalyze(errorMsg: String, t: Throwable): String = { - // EXCEPTION_PATTERN.findFirstMatchIn(t.getMessage) match { - // case Some(m) => - // errorMsg + s", 原因分析[${m.group(1)}]" - // case _ => errorMsg - // } if (null != t) { val errorCodes = StreamisErrorCodeHandler.getInstance().handle(t.getMessage) if (errorCodes != null && errorCodes.size() > 0) { diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/pom.xml b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/pom.xml index e80502c03..d318f51c5 100755 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/pom.xml +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.3.1 + ${revision} ../../pom.xml 4.0.0 @@ -36,7 +36,7 @@ com.webank.wedatasphere.streamis streamis-job-launcher-linkis - ${streamis.version} + ${project.version} org.apache.linkis @@ -45,12 +45,12 @@ com.webank.wedatasphere.streamis streamis-job-launcher-base - ${streamis.version} + ${project.version} com.webank.wedatasphere.streamis streamis-job-manager-base - ${streamis.version} + ${project.version} org.apache.linkis diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/StreamJobConfMapper.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/StreamJobConfMapper.java deleted file mode 100644 index 8aba65896..000000000 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/StreamJobConfMapper.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2021 WeBank - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.webank.wedatasphere.streamis.jobmanager.launcher.dao; - -import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.JobConfDefinition; -import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.JobConfValue; -import org.apache.ibatis.annotations.Param; - -import java.util.List; - -/** - * Operate the job configuration - */ -public interface StreamJobConfMapper { - - /** - * Select all config definitions - * @return list - */ - List loadAllDefinitions(); - - /** - * Get raw value - * @param jobId job id - * @param key key - * @return - */ - String getRawConfValue(@Param("jobId")Long jobId, @Param("key")String key); - /** - * Get config values by job id - * @param jobId job id - * @return - */ - List getConfValuesByJobId(@Param("jobId")Long jobId); - - /** - * Delete values by job id - * @param jobId job id - */ - int deleteConfValuesByJobId(@Param("jobId")Long jobId); - - /** - * Delete temporary config value - * @param jobId job id - * @return affect rows - */ - int deleteTemporaryConfValue(@Param("jobId")Long jobId); - /** - * Batch insert - * @param values values - */ - void batchInsertValues(@Param("values")List values); -} diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/impl/StreamJobConfMapper.xml b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/impl/StreamJobConfMapper.xml deleted file mode 100644 index 94556b3f9..000000000 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/dao/impl/StreamJobConfMapper.xml +++ /dev/null @@ -1,71 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - DELETE FROM `linkis_stream_job_config` WHERE job_id = #{jobId}; - - - - DELETE c FROM `linkis_stream_job_config` c INNER JOIN `linkis_stream_job_config_def` d ON c.job_id = #{jobId} AND d.id = c.ref_def_id AND d.is_temp = 1; - - - INSERT INTO `linkis_stream_job_config`(`job_id`, `job_name`, `key`, `value`, `ref_def_id`) VALUES - - (#{item.jobId}, #{item.jobName}, #{item.key}, #{item.value}, #{item.referDefId}) - - - \ No newline at end of file diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/utils/JobConfUtil.java b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/utils/JobConfUtil.java index bcaef7f54..98bb41a3d 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/utils/JobConfUtil.java +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/launcher/utils/JobConfUtil.java @@ -1,8 +1,8 @@ package com.webank.wedatasphere.streamis.jobmanager.launcher.utils; -import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants; import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConstants; import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.JobConfValue; +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.JobConfKeyConstants; import org.apache.commons.lang3.StringUtils; import java.util.List; @@ -14,11 +14,10 @@ private JobConfUtil(){} public static List valuesHandler (List jobConfValueList){ String manageMode = JobConfKeyConstants.MANAGE_MODE_KEY().getValue(); - List jobConfValues = jobConfValueList.stream() + return jobConfValueList.stream() .filter(jobConfValue -> jobConfValue.getKey().equals(manageMode) && jobConfValue.getValue().isEmpty()) .peek(jobConfValue -> jobConfValue.setValue(JobConstants.MANAGE_MODE_DETACH())) .collect(Collectors.toList()); - return jobConfValues; } diff --git a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/service/DefaultStreamJobConfService.scala b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/service/DefaultStreamJobConfService.scala index e30ea52ee..a4f2a6ee1 100644 --- a/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/service/DefaultStreamJobConfService.scala +++ b/streamis-jobmanager/streamis-job-launcher/streamis-job-launcher-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/launcher/service/DefaultStreamJobConfService.scala @@ -1,11 +1,11 @@ package com.webank.wedatasphere.streamis.jobmanager.launcher.service -import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants import com.webank.wedatasphere.streamis.jobmanager.launcher.dao.StreamJobConfMapper import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.vo.JobConfValueVo.ValueList import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.vo.{JobConfValueSet, JobConfValueVo} import com.webank.wedatasphere.streamis.jobmanager.launcher.entity.{JobConfDefinition, JobConfValue} import com.webank.wedatasphere.streamis.jobmanager.launcher.exception.ConfigurationException +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.JobConfKeyConstants import com.webank.wedatasphere.streamis.jobmanager.launcher.service.tools.JobConfValueUtils import com.webank.wedatasphere.streamis.jobmanager.launcher.utils.JobConfUtil import com.webank.wedatasphere.streamis.jobmanager.manager.dao.StreamJobMapper diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml index b153de718..404f31f08 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/flink-streamis-log-collector/pom.xml @@ -5,7 +5,7 @@ streamis-job-log com.webank.wedatasphere.streamis - 0.3.1 + ${revision} ../../pom.xml 4.0.0 @@ -24,7 +24,7 @@ com.webank.wedatasphere.streamis streamis-job-log-collector - ${streamis.version} + ${project.version} diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml index dea87c97c..ddd3defdf 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/pom.xml @@ -5,7 +5,7 @@ streamis-job-log com.webank.wedatasphere.streamis - 0.3.1 + ${revision} ../../pom.xml 4.0.0 @@ -23,7 +23,7 @@ com.webank.wedatasphere.streamis streamis-job-log-common - ${streamis.version} + ${project.version} diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcHeartbeatService.java b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcHeartbeatService.java index 1da5b2d2f..60d966224 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcHeartbeatService.java +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector-core/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/collector/sender/RpcHeartbeatService.java @@ -23,12 +23,12 @@ public class RpcHeartbeatService { private CloseableHttpClient httpClient = HttpClients.createDefault(); + private ScheduledThreadPoolExecutor scheduler; + public RpcHeartbeatService(StreamisLogAppenderConfig logAppenderConfig) { this.logAppenderConfig = logAppenderConfig; } - private ScheduledExecutorService scheduler; - private ThreadFactory threadFactory(String threadName, Boolean isDaemon) { return new ThreadFactory() { AtomicInteger num = new AtomicInteger(0); @@ -49,10 +49,9 @@ public void startHeartbeat() { return; } System.out.println("Start to heart register."); - ScheduledThreadPoolExecutor scheduler = new ScheduledThreadPoolExecutor(1, threadFactory("Streamis-Log-Default-Scheduler-Thread-", true)); + this.scheduler = new ScheduledThreadPoolExecutor(1, threadFactory("Streamis-Log-Default-Scheduler-Thread-", true)); scheduler.setMaximumPoolSize(1); scheduler.setKeepAliveTime(30, TimeUnit.MINUTES); - this.scheduler = scheduler; Runtime.getRuntime().addShutdownHook(new Thread(() -> { try { httpClient.close(); diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml index b32d95c66..b4fc967e7 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector/pom.xml @@ -5,7 +5,7 @@ streamis-job-log com.webank.wedatasphere.streamis - 0.3.1 + ${revision} ../../pom.xml 4.0.0 @@ -23,7 +23,7 @@ com.webank.wedatasphere.streamis streamis-job-log-collector-core - ${streamis.version} + ${project.version} diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml index 8ba5bd1c5..f20eb9300 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/streamis-job-log-collector1x/pom.xml @@ -5,7 +5,7 @@ streamis-job-log com.webank.wedatasphere.streamis - 0.3.1 + ${revision} ../../pom.xml 4.0.0 @@ -22,7 +22,7 @@ com.webank.wedatasphere.streamis streamis-job-log-collector-core - ${streamis.version} + ${project.version} diff --git a/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml index 267fe1f84..2c29cb1a5 100644 --- a/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml +++ b/streamis-jobmanager/streamis-job-log/job-log-collector/xspark-streamis-log-collector/pom.xml @@ -5,7 +5,7 @@ streamis-job-log com.webank.wedatasphere.streamis - 0.3.1 + ${revision} ../../pom.xml 4.0.0 @@ -23,7 +23,7 @@ com.webank.wedatasphere.streamis streamis-job-log-collector1x - ${streamis.version} + ${project.version} diff --git a/streamis-jobmanager/streamis-job-log/pom.xml b/streamis-jobmanager/streamis-job-log/pom.xml index 2914e6c52..d6957a371 100644 --- a/streamis-jobmanager/streamis-job-log/pom.xml +++ b/streamis-jobmanager/streamis-job-log/pom.xml @@ -5,7 +5,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.3.1 + ${revision} 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml index 9d2cddb2a..dfe561848 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-common/pom.xml @@ -5,7 +5,7 @@ streamis-job-log com.webank.wedatasphere.streamis - 0.3.1 + ${revision} ../pom.xml 4.0.0 diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml index 7955add92..ce1a4a813 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/pom.xml @@ -5,7 +5,7 @@ streamis-job-log com.webank.wedatasphere.streamis - 0.3.1 + ${revision} ../pom.xml 4.0.0 @@ -21,7 +21,7 @@ com.webank.wedatasphere.streamis streamis-job-log-common - 0.3.1 + ${project.version} org.apache.linkis @@ -36,7 +36,7 @@ com.webank.wedatasphere.streamis streamis-job-manager-base - 0.3.1 + ${project.version} compile diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java index 163283ed9..383eb6b0b 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/JobLogStorage.java @@ -54,7 +54,7 @@ public interface JobLogStorage { StreamJobConfMapper getStreamJobConfMapper(); - void setStreamJobMapper(StreamJobMapper streamJobMapper);; + void setStreamJobMapper(StreamJobMapper streamJobMapper); StreamJobMapper getStreamJobMapper(); diff --git a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java index 9de48798c..e84bb47ac 100644 --- a/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java +++ b/streamis-jobmanager/streamis-job-log/streamis-job-log-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/log/server/storage/StreamisJobLogStorage.java @@ -158,12 +158,7 @@ public synchronized void init() throws StreamJobLogException { SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); // Update the storage context JobLogStorageContext[] contexts = this.storageContexts.toArray(new JobLogStorageContext[0]); - try { - updateContextWeight(contexts); - onContextEvent(new ContextRefreshAllEvent()); - } catch (IOException e) { - logger.warn("Unable to calculate weight array of storage context list", e); - } + updateContextWeightAndEvent(contexts); if (buckets.size() > 0) { StringBuilder builder = new StringBuilder("Buckets(").append(buckets.size()).append(") in LogStorage: [\n"); buckets.forEach((bucketName, bucket) -> { @@ -207,6 +202,14 @@ public synchronized void init() throws StreamJobLogException { } } + private void updateContextWeightAndEvent(JobLogStorageContext[] contexts){ + try { + updateContextWeight(contexts); + onContextEvent(new ContextRefreshAllEvent()); + } catch (IOException e) { + logger.warn("Unable to calculate weight array of storage context list", e); + } + } @Override @PreDestroy @@ -309,7 +312,7 @@ private double[] calculateContextWeight(JobLogStorageContext[] contexts) throws if (freeSpaceInGB <= 0) { freeSpaceInGB = 1; } - weight = context.getScore() * (double) freeSpaceInGB; + weight = context.getScore() * freeSpaceInGB; } weights[i] = weight; if (weight > maxWeight){ diff --git a/streamis-jobmanager/streamis-job-manager/pom.xml b/streamis-jobmanager/streamis-job-manager/pom.xml index 5a18425a7..b5355d586 100755 --- a/streamis-jobmanager/streamis-job-manager/pom.xml +++ b/streamis-jobmanager/streamis-job-manager/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.3.1 + ${revision} 4.0.0 diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/pom.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/pom.xml index 8405fe3c9..e449f61ec 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/pom.xml +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.3.1 + ${revision} ../../pom.xml 4.0.0 @@ -46,6 +46,12 @@ org.apache.linkis linkis-common + + + jackson-databind + com.fasterxml.jackson.core + + commons-io @@ -67,7 +73,7 @@ com.webank.wedatasphere.streamis streamis-job-launcher-linkis - ${streamis.version} + ${project.version} compile diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobManagerConf.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobManagerConf.java index 6d2b08643..dc50908b1 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobManagerConf.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/conf/JobManagerConf.java @@ -6,10 +6,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; public class JobManagerConf { @@ -17,7 +14,7 @@ public class JobManagerConf { private static Map> hookProjectMap = null; - public static final CommonVars ENABLE_JOB_SHUTDOWN_HOOKS = CommonVars.apply("wds.streamis.job.shutdown.hook.enable", true); + public static final CommonVars ENABLE_JOB_SHUTDOWN_HOOKS = CommonVars.apply("wds.streamis.job.shutdown.hook.enable", false); /* @@ -28,7 +25,7 @@ public class JobManagerConf { public static final CommonVars JOB_SHUTDOWN_HOOK_TIMEOUT_MILLS = CommonVars.apply("wds.streamis.job.shutdown.hook.timeout.mills", 5 * 60 * 1000L); private static void initHookMap() { - hookProjectMap = new HashMap>(); + hookProjectMap = new HashMap<>(); String hookConfig = JOB_SHUTDOWN_HOOKS.getValue(); if (StringUtils.isNotBlank(hookConfig)) { for (String hooksAndProjects : hookConfig.split(";")) { @@ -39,9 +36,7 @@ private static void initHookMap() { if (hookProjectArr.length == 2) { String projects = hookProjectArr[1]; String[] projectArr = projects.split(","); - for (String project : projectArr) { - projectList.add(project); - } + projectList.addAll(Arrays.asList(projectArr)); } else { logger.warn("Invalid hook : {} config {}={}", hookName, JOB_SHUTDOWN_HOOKS.key(), JOB_SHUTDOWN_HOOKS.getValue()); } @@ -58,11 +53,9 @@ private static void initHookMap() { public static List getHookNames() { List names = new ArrayList<>(); - if (ENABLE_JOB_SHUTDOWN_HOOKS.getValue()) { - if (null == hookProjectMap) { + if (ENABLE_JOB_SHUTDOWN_HOOKS.getValue() && null == hookProjectMap) { initHookMap(); } - } for (String name : hookProjectMap.keySet()) { names.add(name); } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java index 18bec9db1..c50e327b2 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/StreamJobMapper.java @@ -87,4 +87,5 @@ List getJobLists(@Param("projectName") String projectName, @Para StreamJobVersionFiles getJobFileById(@Param("id") Long jobVersionFileId); + List batchGetJobByJobIdList(@Param("idList") List idList); } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml index 00165d966..5e6ba224c 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamJobMapper.xml @@ -262,6 +262,14 @@ linkis_stream_job_version_files WHERE id = #{id} + UPDATE linkis_stream_job diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamTaskMapper.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamTaskMapper.xml index 71009eed8..3f38a092c 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamTaskMapper.xml +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/StreamTaskMapper.xml @@ -45,7 +45,7 @@ - `id`,`job_version_id`,`job_id`,`status`,`start_time`,`last_update_time`,`err_desc`,`submit_user`, `linkis_job_id`, `linkis_job_info`, `server_instance` + `id`,`job_version_id`,`job_id`,`status`,`start_time`,`last_update_time`,`err_desc`,`submit_user`, `linkis_job_id`, `linkis_job_info`, `server_instance`, `job_start_config` hookMap = new HashMap(); + private static final Map hookMap = new HashMap<>(); public static boolean registerJobShutdownHook(StreamisJobShutdownHook hook) { if (null == hook) { diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/JsonUtils.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/JsonUtils.java index 8312a31ce..639cb944b 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/JsonUtils.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/JsonUtils.java @@ -1,20 +1,27 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.util; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.JsonSyntaxException; +import com.webank.wedatasphere.streamis.jobmanager.manager.exception.JsonFormatWarnException; +import org.apache.commons.lang3.StringUtils; import org.apache.linkis.server.BDPJettyServerHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.text.SimpleDateFormat; import java.util.HashMap; import java.util.List; import java.util.Map; public class JsonUtils { + private JsonUtils(){} private static final Logger LOG = LoggerFactory.getLogger(JsonUtils.class); public static String manageArgs(String jobContent, List args){ try { - Map map = BDPJettyServerHelper.gson().fromJson(jobContent, Map.class); + Map map = BDPJettyServerHelper.gson().fromJson(jobContent, Map.class); map.put("args",args); return BDPJettyServerHelper.gson().toJson(map); } catch (JsonSyntaxException e) { @@ -36,4 +43,29 @@ public static String manageSource(String getSource,Boolean isHighAvailable , Str } return BDPJettyServerHelper.gson().toJson(hashMap); } + + public static ObjectMapper jackson = new ObjectMapper().setDateFormat(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ")) + .setSerializationInclusion(JsonInclude.Include.NON_NULL) + .disable(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES) + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + + public static String toString(Object object) { + if(object == null) { + return null; + } + return LambdaUtils.supply(() -> jackson.writeValueAsString(object), + e -> new JsonFormatWarnException(1000100, "反序列化 " + object.getClass().getSimpleName() + " 失败！")); + } + + public static Map stringToMap(String str) { + return stringToObject(str, Map.class); + } + + public static T stringToObject(String str, Class clazz) { + if(StringUtils.isBlank(str)) { + return null; + } + return LambdaUtils.supply(() -> jackson.readValue(str, clazz), + e -> new JsonFormatWarnException(1000100, str + " 序列化失败！")); + } } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/LambdaUtils.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/LambdaUtils.java new file mode 100644 index 000000000..8e7577c50 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/LambdaUtils.java @@ -0,0 +1,70 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.util; + +import org.apache.linkis.common.exception.WarnException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public interface LambdaUtils { + + static final Logger LOGGER = LoggerFactory.getLogger(LambdaUtils.class); + + static Out supply(Supplier supplier, + java.util.function.Function createException) { + try { + return supplier.get(); + } catch (Exception e) { + throw createException.apply(e); + } + } + + static java.util.function.Consumer consume(Consumer consumer, + java.util.function.Function createException) { + return in -> { + try { + consumer.accept(in); + } catch (Exception e) { + throw createException.apply(e); + } + }; + } + + static java.util.function.Function apply(Function function, + java.util.function.Function createException) { + return in -> { + try { + return function.apply(in); + } catch (Exception e) { + throw createException.apply(e); + } + }; + } + + static java.util.function.Consumer consumeAndWarn(Consumer consumer) { + return in -> { + try { + consumer.accept(in); + } catch (Exception e) { + LOGGER.warn("运行失败，原因：", e); + } + }; + } + + interface Supplier { + + T get() throws Exception; + + } + + interface Consumer { + + void accept(T t) throws Exception; + + } + + interface Function { + + Out apply(In in) throws Exception; + + } + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/ReaderUtils.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/ReaderUtils.java index 96248104c..2b83eda66 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/ReaderUtils.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-base/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/util/ReaderUtils.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.JobConf; +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.JobConfKeyConstants; import com.webank.wedatasphere.streamis.jobmanager.launcher.job.constants.JobConstants; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.MetaJsonInfo; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.PublishRequestVo; @@ -35,19 +36,18 @@ import java.util.regex.Pattern; public class ReaderUtils { - private static final String metaFileName = "meta.txt"; + private static final String metaFileJsonName = "meta.json"; private static final String templateMetaFileJsonName = "-meta.json"; private static final String type = "type"; - private static final String fileName = "filename"; private static final String projectName = "projectname"; private static final String jobName = "jobname"; private static final String tags = "tags"; private static final String description = "description"; private static final String defaultTagList = "prod,streamis"; private static final String version = "v00001"; - private static final String regex = "^[a-z0-9A-Z_-]+$"; + private static final String regex = "^[a-z0-9A-Z._-]+$"; private static final String jarRegex = "^[a-z0-9A-Z._-]+$"; private static final int defaultLength = 64; private static final int descriptionLength = 128; @@ -58,8 +58,6 @@ public class ReaderUtils { private boolean hasProjectName = false; private static final String templateName = "-meta.json"; private static final String JSON_TYPE = ".json"; - private static final String PRODUCE_PARAM = "wds.linkis.flink.produce"; - private static final Logger LOG = LoggerFactory.getLogger(ReaderUtils.class); @@ -334,7 +332,12 @@ public Boolean checkMetaTemplate(String fileName,String inputPath,String project String path = inputPath.replace(JSON_TYPE, ""); MetaJsonInfo metaJsonInfo = parseJson(path,projectName); Map jobConfig = metaJsonInfo.getJobConfig(); - if (jobConfig != null && jobConfig.containsKey(PRODUCE_PARAM)) { + if (jobConfig != null + && jobConfig.containsKey(JobConfKeyConstants.GROUP_PRODUCE().getValue()) + && jobConfig.containsKey(JobConfKeyConstants.GROUP_RESOURCE().getValue())) { + return false; + } + if(metaJsonInfo.getProjectName() == null && metaJsonInfo.getProjectName().isEmpty()) { return false; } if ((metaJsonInfo.getJobName() == null || metaJsonInfo.getJobName().isEmpty()) && diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/pom.xml b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/pom.xml index e7fff5e6d..34a4a1d3f 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/pom.xml +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.3.1 + ${revision} ../../pom.xml 4.0.0 @@ -45,22 +45,22 @@ com.webank.wedatasphere.streamis streamis-job-manager-base - ${streamis.version} + ${project.version} com.webank.wedatasphere.streamis streamis-job-launcher-linkis - ${streamis.version} + ${project.version} com.webank.wedatasphere.streamis streamis-job-launcher-service - ${streamis.version} + ${project.version} com.webank.wedatasphere.streamis streamis-job-errorcode - ${streamis.version} + ${project.version} org.apache.linkis diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/handler/StreamisHeartbeatHandler.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/handler/StreamisHeartbeatHandler.java index 743e00048..5f8221b7c 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/handler/StreamisHeartbeatHandler.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/handler/StreamisHeartbeatHandler.java @@ -43,7 +43,7 @@ public class StreamisHeartbeatHandler { @PostConstruct public void startHeartbeatCheckThread() { - if ((Boolean) JobConf.LOGS_HEARTBEAT_CHECK_ENABLE().getValue()) { + if ((boolean) JobConf.LOGS_HEARTBEAT_CHECK_ENABLE().getValue()) { int interval = Integer.parseInt(JobConf.LOGS_HEARTBEAT_CHECK_INTERVAL().getHotValue().toString()); Utils.defaultScheduler().scheduleAtFixedRate(() -> { try { diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/scheduler/events/StreamisPhaseInSchedulerEvent.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/scheduler/events/StreamisPhaseInSchedulerEvent.java index 03c38fce6..38a4dc8b6 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/scheduler/events/StreamisPhaseInSchedulerEvent.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/scheduler/events/StreamisPhaseInSchedulerEvent.java @@ -110,8 +110,8 @@ public Object getVar(String name){ @SuppressWarnings("unchecked") public T getVar(String name, Class type){ - Object var = variables.get(name); - return Objects.nonNull(var)? (T)var : null; + Object value = variables.get(name); + return Objects.nonNull(value )? (T)value : null; } } } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/utils/SourceUtils.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/utils/SourceUtils.java index 218a20931..a21e1aec9 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/utils/SourceUtils.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/java/com/webank/wedatasphere/streamis/jobmanager/manager/utils/SourceUtils.java @@ -14,18 +14,29 @@ public class SourceUtils { + private static final String HIGH_AVAILABLE_MSG = "highAvailableMessage"; + private static final String DEFAULT_MSG = "高可用信息为空，请联系管理员"; + + private SourceUtils(){} private static final Logger LOG = LoggerFactory.getLogger(SourceUtils.class); public static JobHighAvailableVo manageJobProjectFile(String highAvailablePolicy,String source) { highAvailablePolicy = Optional.ofNullable(highAvailablePolicy).orElse(JobConf.HIGHAVAILABLE_DEFAULT_POLICY().getHotValue()); JobHighAvailableVo highAvailableVo = new JobHighAvailableVo(); if (source.equalsIgnoreCase("update args")) { + if(Boolean.parseBoolean(JobConf.HIGHAVAILABLE_ENABLE_INTERFACE_UPLOAD().getHotValue().toString())){ highAvailableVo.setHighAvailable(true); + highAvailableVo.setMsg("用户在页面手动修改args,跳过一致性检查"); + return highAvailableVo; + } else { + highAvailableVo.setHighAvailable(false); highAvailableVo.setMsg("用户在页面手动修改args,任务不再支持高可用"); return highAvailableVo; } + + } try { - if (!Boolean.parseBoolean(JobConf.HIGHAVAILABLE_ENABLE().getValue().toString())) { + if (!Boolean.parseBoolean(JobConf.HIGHAVAILABLE_ENABLE().getHotValue().toString())) { highAvailableVo.setHighAvailable(true); highAvailableVo.setMsg("管理员未开启高可用配置"); return highAvailableVo; @@ -33,24 +44,36 @@ public static JobHighAvailableVo manageJobProjectFile(String highAvailablePolicy //查job conf wds.streamis.app.highavailable.policy 值 if (highAvailablePolicy.equals(JobConf.HIGHAVAILABLE_POLICY_DOUBLE().getValue()) || highAvailablePolicy.equals(JobConf.HIGHAVAILABLE_POLICY_DOUBLE_BAK().getValue()) || highAvailablePolicy.equals(JobConf.HIGHAVAILABLE_POLICY_MANAGERSLAVE().getValue()) || highAvailablePolicy.equals(JobConf.HIGHAVAILABLE_POLICY_MANAGERSLAVE_BAK().getValue())){ - Map map = BDPJettyServerHelper.gson().fromJson(source, Map.class); + Map map = BDPJettyServerHelper.gson().fromJson(source, Map.class); if (map.containsKey("source")) { String sourceValue = map.get("source").toString(); if (sourceValue.equals(JobConf.HIGHAVAILABLE_SOURCE().getValue())) { if (map.containsKey("isHighAvailable")) { highAvailableVo.setHighAvailable(Boolean.parseBoolean(map.get("isHighAvailable").toString()) ); } - highAvailableVo.setMsg(map.getOrDefault("highAvailableMessage","高可用信息为空，请联系管理员").toString()); + highAvailableVo.setMsg(map.getOrDefault(HIGH_AVAILABLE_MSG,DEFAULT_MSG).toString()); return highAvailableVo; } else { + if(Boolean.parseBoolean(JobConf.HIGHAVAILABLE_ENABLE_INTERFACE_UPLOAD().getHotValue().toString())){ highAvailableVo.setHighAvailable(true); highAvailableVo.setMsg("非标准来源,不检测高可用"); return highAvailableVo; - } } else { + highAvailableVo.setHighAvailable(false); + highAvailableVo.setMsg(map.getOrDefault(HIGH_AVAILABLE_MSG,DEFAULT_MSG).toString()); + return highAvailableVo; + } + } + } else { + if(Boolean.parseBoolean(JobConf.HIGHAVAILABLE_ENABLE_INTERFACE_UPLOAD().getHotValue().toString())){ highAvailableVo.setHighAvailable(true); highAvailableVo.setMsg("非标准来源,不检测高可用"); return highAvailableVo; + } else { + highAvailableVo.setHighAvailable(false); + highAvailableVo.setMsg(map.getOrDefault(HIGH_AVAILABLE_MSG,DEFAULT_MSG).toString()); + return highAvailableVo; + } } } else { highAvailableVo.setHighAvailable(true); diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/constrants/JobConstrants.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/constrants/JobConstrants.scala index 91a46b401..cc3771f6c 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/constrants/JobConstrants.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/constrants/JobConstrants.scala @@ -22,4 +22,6 @@ object JobConstrants { val FIELD_JOB_DESCRIPTION = "description" val PRODUCE_PARAM = "wds.linkis.flink.produce" + + val UNKNOWN_COMMENT = "" } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/hook/DefaultEmptyStreamisJobShutdownHook.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/hook/DefaultEmptyStreamisJobShutdownHook.scala index 662a4ca4a..099fe0f6e 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/hook/DefaultEmptyStreamisJobShutdownHook.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/hook/DefaultEmptyStreamisJobShutdownHook.scala @@ -23,6 +23,6 @@ class DefaultEmptyStreamisJobShutdownHook extends StreamisJobShutdownHook with L } override def getName: String = "emptyJobShutdownHook" - + //empty method override def cancel(): Unit = {} } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultJobTemplateService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultJobTemplateService.scala new file mode 100644 index 000000000..785e31cc6 --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultJobTemplateService.scala @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.service + +import com.google.gson.JsonParser +import com.webank.wedatasphere.streamis.jobmanager.manager.constrants.JobConstrants +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.JobTemplateFiles +import org.springframework.stereotype.Service + +@Service +class DefaultJobTemplateService extends StreamJobTemplateService { + + override def generateJobTemplate(jobTemplate: JobTemplateFiles): String = { + val jsonObj = new JsonParser().parse(jobTemplate.getMetaJson).getAsJsonObject + if (jsonObj.has(JobConstrants.FIELD_METAINFO_NAME)) { + jsonObj.remove(JobConstrants.FIELD_METAINFO_NAME) + } + val parsedConfigJson = jsonObj.toString + parsedConfigJson + } + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobInspectService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobInspectService.scala index 76b56c896..923a1b741 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobInspectService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobInspectService.scala @@ -1,8 +1,7 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.service -import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants import com.webank.wedatasphere.streamis.jobmanager.launcher.dao.StreamJobConfMapper -import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.JobConf +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.{JobConf, JobConfKeyConstants} import com.webank.wedatasphere.streamis.jobmanager.launcher.job.constants.JobConstants import com.webank.wedatasphere.streamis.jobmanager.launcher.job.errorcode.JobLaunchErrorCode import com.webank.wedatasphere.streamis.jobmanager.launcher.job.exception.{JobCreateErrorException, JobErrorException, JobFetchErrorException} @@ -12,7 +11,7 @@ import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamJobMapper, import com.webank.wedatasphere.streamis.jobmanager.manager.entity.{StreamJob, StreamJobVersion, StreamJobVersionFiles} import com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo.{JobHighAvailableVo, JobInspectVo, JobListInspectVo, JobSnapshotInspectVo, JobVersionInspectVo} import com.webank.wedatasphere.streamis.jobmanager.manager.utils.SourceUtils -import org.apache.commons.lang.StringUtils +import org.apache.commons.lang3.StringUtils import org.apache.linkis.common.exception.ErrorException import org.apache.linkis.common.utils.{JsonUtils, Logging, Utils} import org.springframework.beans.factory.annotation.Autowired @@ -96,9 +95,9 @@ class DefaultStreamJobInspectService extends StreamJobInspectService with Loggin */ private def snapshotInspect(streamJob: StreamJob): JobSnapshotInspectVo = { Option(this.streamJobConfMapper.getRawConfValue(streamJob.getId, JobConfKeyConstants.SAVEPOINT.getValue + "path")) match { - case Some(path) => + case path if (path.isDefined && StringUtils.isNotBlank(path.get)) => val inspectVo = new JobSnapshotInspectVo - inspectVo.setPath(new URI(path).toString) + inspectVo.setPath(new URI(path.get).toString) inspectVo case _ => this.streamJobConfMapper.getRawConfValue(streamJob.getId, JobConfKeyConstants.START_AUTO_RESTORE_SWITCH.getValue) match { case "ON" => @@ -183,8 +182,18 @@ class DefaultStreamJobInspectService extends StreamJobInspectService with Loggin inspectVo = SourceUtils.manageJobProjectFile(highAvailablePolicy, source) case None => logger.warn("this job source is null") + if (highAvailablePolicy == JobConf.HIGHAVAILABLE_DEFAULT_POLICY.getValue || highAvailablePolicy == JobConf.HIGHAVAILABLE_POLICY_SINGLE_BAK.getValue){ + inspectVo.setHighAvailable(true) + inspectVo.setMsg("job为单活，跳过高可用检查") + } else { + if (JobConf.HIGHAVAILABLE_ENABLE_INTERFACE_UPLOAD.getHotValue()) { inspectVo.setHighAvailable(true) inspectVo.setMsg("用户直接从页面上传，job的source为空，跳过高可用检查") + } else { + inspectVo.setHighAvailable(false) + inspectVo.setMsg("用户直接从页面上传，高可用检查不通过") + } + } } inspectVo } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala index bdf6f3910..20507d574 100755 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamJobService.scala @@ -18,8 +18,7 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.service import java.util import java.util.{Date, List, Map} import com.github.pagehelper.PageInfo -import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants -import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.JobConf +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.{JobConf, JobConfKeyConstants} import com.webank.wedatasphere.streamis.jobmanager.launcher.job.constants.JobConstants import com.webank.wedatasphere.streamis.jobmanager.launcher.job.exception.{JobCreateErrorException, JobErrorException, JobFetchErrorException} import com.webank.wedatasphere.streamis.jobmanager.launcher.service.StreamJobConfService @@ -249,7 +248,6 @@ class DefaultStreamJobService extends StreamJobService with Logging { }else{ throw new JobCreateErrorException(30030,s"meta_json does not contain jobConfig, please check") } -// this.streamJobConfService.saveJobConfig(version.getJobId, finalJobConfig.asInstanceOf[util.Map[String, AnyRef]]) // 上传所有非meta.json的文件 uploadFiles(metaJsonInfo, version, inputZipPath) version @@ -261,7 +259,6 @@ class DefaultStreamJobService extends StreamJobService with Logging { val validateResult = validateJobDeploy(metaJsonInfo.getProjectName, metaJsonInfo.getJobName, userName) val readerUtils = new ReaderUtils metaJsonInfo.setMetaInfo(readerUtils.readAsJson(metaJsonInfo)) - val projectName = metaJsonInfo.getProjectName val version = deployStreamJob(validateResult.streamJob, metaJsonInfo, userName, validateResult.updateVersion) // Save the job configuration, lock the job again if exists if (null != metaJsonInfo.getJobConfig){ @@ -428,7 +425,7 @@ class DefaultStreamJobService extends StreamJobService with Logging { override def canBeDisabled(jobId: Long): Boolean = { val streamJob = streamJobMapper.getJobById(jobId); - if(streamJob == null) throw new JobFetchErrorException(30030, s"job does not exist.") + if(streamJob == null) throw new JobFetchErrorException(30030, s"job does not exist,please check.") //处理job是首次上传且未被启动的情况 if (streamJob.getStatus == 0){ val streamTask = this.streamTaskMapper.getLatestByJobId(jobId) @@ -458,7 +455,7 @@ class DefaultStreamJobService extends StreamJobService with Logging { override def canbeActivated(jobId: Long): Boolean = { val streamJob = streamJobMapper.getJobById(jobId) - if(streamJob == null) throw new JobFetchErrorException(30030, s"job does not exist.") + if(streamJob == null) throw new JobFetchErrorException(30030, s"job does not exist,check.") if (streamJob.getEnable){ false } else { diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala index c3b8577cc..9ef8fddc5 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/DefaultStreamTaskService.scala @@ -21,8 +21,7 @@ import com.google.gson.JsonParser import java.util import java.util.concurrent.{Executors, Future, ScheduledExecutorService, TimeUnit} import java.util.{Calendar, Map, function} -import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants -import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.{JobConf, StreamJobLauncherConf} +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.{JobConf, JobConfKeyConstants, StreamJobLauncherConf} import com.webank.wedatasphere.streamis.jobmanager.launcher.job.exception.{JobErrorException, JobExecuteErrorException, JobFetchErrorException, JobPauseErrorException, JobTaskErrorException} import com.webank.wedatasphere.streamis.jobmanager.launcher.job.manager.JobLaunchManager import com.webank.wedatasphere.streamis.jobmanager.launcher.job.state.{JobGenericState, JobState} @@ -91,8 +90,6 @@ class DefaultStreamTaskService extends StreamTaskService with Logging{ @Resource private var streamJobConfService: StreamJobConfService = _ - @Resource - private var streamTaskService: StreamTaskService = _ /** * Scheduler */ @@ -446,7 +443,7 @@ class DefaultStreamTaskService extends StreamTaskService with Logging{ } } hookFuture = Utils.defaultScheduler.submit(hookTask) - val rs = hookFuture.get(JobManagerConf.JOB_SHUTDOWN_HOOK_TIMEOUT_MILLS.getHotValue(), TimeUnit.MILLISECONDS) + hookFuture.get(JobManagerConf.JOB_SHUTDOWN_HOOK_TIMEOUT_MILLS.getHotValue(), TimeUnit.MILLISECONDS) logger.info(s"hook : ${hook.getName} outside succeed, costed ${System.currentTimeMillis() - hookStartTimeMills}mills.") } { case e: Exception => @@ -751,7 +748,7 @@ class DefaultStreamTaskService extends StreamTaskService with Logging{ streamTaskMapper.updateTask(streamTask) info(s"StreamJob [${streamJob.getName}] is ${jobInfo.getStatus} with $jobInfo.") if (JobConf.FLINK_JOB_STATUS_FAILED.getValue == streamTask.getStatus){ - val result: Future[_] = streamTaskService.errorCodeMatching(streamJob.getId,streamTask) + val result: Future[_] = errorCodeMatching(streamJob.getId,streamTask) throw new JobExecuteErrorException(-1, s"(提交流式应用状态失败, 请检查日志), errorDesc: ${streamTask.getErrDesc}") } // Drop the temporary configuration @@ -950,7 +947,7 @@ class DefaultStreamTaskService extends StreamTaskService with Logging{ logger.error("errorCodeMatching failed. ", e) val streamTask = new StreamTask() streamTask.setErrDesc(JobConf.ANALYZE_ERROR_MSG.getHotValue()) - streamTaskService.updateTask(streamTask) + updateTask(streamTask) } if (errorMsg.isEmpty){ Utils.tryCatch{ @@ -971,13 +968,13 @@ class DefaultStreamTaskService extends StreamTaskService with Logging{ streamTask.setId(taskId); streamTask.setErrDesc(errorMsg) streamTask.setSolution(solution) - streamTaskService.updateTask(streamTask) + updateTask(streamTask) }{ case e: Exception => logger.error("errorCodeMatching failed. ", e) val streamTask = new StreamTask() streamTask.setErrDesc(JobConf.ANALYZE_ERROR_MSG.getHotValue()) - streamTaskService.updateTask(streamTask) + updateTask(streamTask) } } val streamTask = new StreamTask() @@ -988,7 +985,7 @@ class DefaultStreamTaskService extends StreamTaskService with Logging{ streamTask.setErrDesc(errorMsg) } streamTask.setSolution(solution) - streamTaskService.updateTask(streamTask) + updateTask(streamTask) } }) } @@ -1007,7 +1004,7 @@ class DefaultStreamTaskService extends StreamTaskService with Logging{ payload.setLogType(logType) payload.setFromLine(fromLine+1) payload.setPageSize(100) - val realtimeLog = streamTaskService.getRealtimeLog(jobId, if (null != taskId) taskId else 0L, username, payload) + val realtimeLog = getRealtimeLog(jobId, if (null != taskId) taskId else 0L, username, payload) val logs = realtimeLog.getLogs val logString =logs.asScala.mkString("\n") logString @@ -1070,14 +1067,7 @@ class DefaultStreamTaskService extends StreamTaskService with Logging{ parsedConfigJson } - override def generateJobTemplate(jobTemplate: JobTemplateFiles): String = { - val jsonObj = new JsonParser().parse(jobTemplate.getMetaJson).getAsJsonObject - if (jsonObj.has(JobConstrants.FIELD_METAINFO_NAME)) { - jsonObj.remove(JobConstrants.FIELD_METAINFO_NAME) - } - val parsedConfigJson = jsonObj.toString - parsedConfigJson - } + override def getLatestByJobId(jobId: Long):StreamTask = { val streamTask = streamTaskMapper.getLatestByJobId(jobId) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamJobTemplateService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamJobTemplateService.scala new file mode 100644 index 000000000..d5656096b --- /dev/null +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamJobTemplateService.scala @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.streamis.jobmanager.manager.service + +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.JobTemplateFiles + +trait StreamJobTemplateService { + + + def generateJobTemplate(jobTemplate: JobTemplateFiles): String + + +} diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamTaskService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamTaskService.scala index 312e3cbde..26a3eb1da 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamTaskService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/StreamTaskService.scala @@ -194,7 +194,5 @@ trait StreamTaskService { def generateJobStartConfig(job: StreamJob, jobVersion: StreamJobVersion, creator: String,jobTemplate: JobTemplateFiles): String - def generateJobTemplate(jobTemplate: JobTemplateFiles): String - def getLatestByJobId(jobId: Long ):StreamTask } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskMonitorService.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskMonitorService.scala index 8c2f8e9be..d3a875d96 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskMonitorService.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/service/TaskMonitorService.scala @@ -19,18 +19,18 @@ import java.util import java.util.{Date, Optional, function} import java.util.concurrent.{CompletableFuture, ExecutorService, Future, TimeUnit} import com.webank.wedatasphere.streamis.jobmanager.launcher.JobLauncherAutoConfiguration -import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants import com.webank.wedatasphere.streamis.jobmanager.launcher.dao.StreamJobConfMapper import com.webank.wedatasphere.streamis.jobmanager.launcher.job.JobInfo -import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.JobConf +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.{JobConf, JobConfKeyConstants} import com.webank.wedatasphere.streamis.jobmanager.launcher.job.manager.JobLaunchManager import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.jobInfo.{EngineConnJobInfo, LinkisJobInfo} import com.webank.wedatasphere.streamis.jobmanager.manager.alert.{AlertLevel, Alerter} import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamJobMapper, StreamRegisterMapper, StreamTaskMapper} -import com.webank.wedatasphere.streamis.jobmanager.manager.entity.{StreamJob, StreamTask} +import com.webank.wedatasphere.streamis.jobmanager.manager.entity.{MetaJsonInfo, StreamJob, StreamTask} import com.webank.wedatasphere.streamis.jobmanager.manager.utils.StreamTaskUtils import com.webank.wedatasphere.streamis.errorcode.handler.StreamisErrorCodeHandler import com.webank.wedatasphere.streamis.jobmanager.manager.constrants.JobConstrants +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.utils.JobUtils import javax.annotation.{PostConstruct, PreDestroy, Resource} import org.apache.commons.lang.exception.ExceptionUtils @@ -120,7 +120,7 @@ class TaskMonitorService extends Logging { streamTasks.filter(shouldMonitor).foreach { streamTask => val job = streamJobMapper.getJobById(streamTask.getJobId) if(!JobConf.SUPPORTED_MANAGEMENT_JOB_TYPES.getValue.contains(job.getJobType)) { - val userList = getAlertUsers(job) + val userList = getAlertUsers(job, streamTask) //user val alertMsg = s"Spark Streaming应用[${job.getName}]已经超过 ${Utils.msDurationToString(System.currentTimeMillis - streamTask.getLastUpdateTime.getTime)} 没有更新状态, 请及时确认应用是否正常！" alert(jobService.getAlertLevel(job), alertMsg, userList, streamTask) @@ -169,14 +169,13 @@ class TaskMonitorService extends Logging { alertMsg = s"${alertMsg} 现将自动拉起该应用" restartJob(job,streamTask) case _ => - if (JobConf.AUTO_RESTART_JOB.getHotValue()) { - if (!highAvailablePolicy.equals(JobConf.HIGHAVAILABLE_POLICY_SINGLE_BAK.getValue) && !highAvailablePolicy.equals(JobConf.HIGHAVAILABLE_DEFAULT_POLICY.getValue)) { + if (JobConf.AUTO_RESTART_JOB.getHotValue() && !highAvailablePolicy.equals(JobConf.HIGHAVAILABLE_POLICY_SINGLE_BAK.getValue) + && !highAvailablePolicy.equals(JobConf.HIGHAVAILABLE_DEFAULT_POLICY.getValue)) { alertMsg = s"${alertMsg} 现将自动拉起该应用" restartJob(job,streamTask) } } - } - val userList = getAlertUsers(job) + val userList = getAlertUsers(job, streamTask) alert(jobService.getAlertLevel(job), alertMsg, userList, streamTask) } } @@ -193,7 +192,7 @@ class TaskMonitorService extends Logging { } { case e: Exception => warn(s"Fail to reLaunch the StreamisJob [${job.getName}]", e) - val userList = getAlertUsers(job) + val userList = getAlertUsers(job, null) val alertMsg = s"Fail to reLaunch the StreamisJob [${job.getName}],please be noticed!" alert(AlertLevel.MAJOR, alertMsg, userList, streamTask) } @@ -209,25 +208,58 @@ class TaskMonitorService extends Logging { jobClient.getJobInfo } - protected def getAlertUsers(job: StreamJob): util.List[String] = { + protected def getAlertUsers(job: StreamJob, streamTask: StreamTask): util.List[String] = { + // fist, get alert users from job config in db val allUsers = new util.LinkedHashSet[String]() val alertUsers = jobService.getAlertUsers(job) var isValid = false if (alertUsers!= null) { alertUsers.foreach(user => { - if (StringUtils.isNotBlank(user) && !user.toLowerCase().contains("hduser")) { + if (StringUtils.isNotBlank(user) && !user.toLowerCase().contains("hduser") && !"hadoop".equals(user)) { isValid = true allUsers.add(user) } }) - if (!allUsers.contains(job.getSubmitUser)) { + if (!allUsers.contains(job.getSubmitUser) && !"hadoop".equals(job.getSubmitUser)) { allUsers.add(job.getSubmitUser) } } + if (!isValid && null != streamTask) { + // second, get alert users from job startup config + val metaJsonInfo = Utils.tryCatch {JobUtils.gson.fromJson(streamTask.getJobStartConfig, classOf[MetaJsonInfo])} { + case e: Exception => + logger.error(s"parse jobStartConfig : ${streamTask.getJobStartConfig} error", e) + null + } + if (null != metaJsonInfo) { + val jobStartAlertUsers = metaJsonInfo.getJobConfig.getOrDefault(JobConstrants.PRODUCE_PARAM, null) match { + case map: util.Map[String, Any] => + map.getOrDefault(JobConfKeyConstants.ALERT_USER.getValue, "").toString + case _ => "" + } + if (StringUtils.isNotBlank(jobStartAlertUsers)) { + jobStartAlertUsers.split(",").foreach(user => { + if (StringUtils.isNotBlank(user) && !user.toLowerCase().contains("hduser")) { + isValid = true + allUsers.add(user) + } + }) + } + } + } + if (!isValid){ + if (!"hadoop".equals(job.getSubmitUser)) { allUsers.add(job.getSubmitUser) + } + if (!"hadoop".equals(job.getCreateBy)) { allUsers.add(job.getCreateBy) } + } + if (allUsers.isEmpty) { + logger.error("Got invalid alert users for job : {}, will alert to admin users.", job.getName) + allUsers.addAll(getAdminAlertUsers()); + } new util.ArrayList[String](allUsers) } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala index b3c566fb5..eb3271caf 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/builder/AbstractStreamisTransformJobBuilder.scala @@ -15,12 +15,11 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.transform.builder -import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants -import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.JobConf +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.{JobConf, JobConfKeyConstants} import org.apache.linkis.common.conf.CommonVars import org.apache.linkis.manager.label.entity.engine.RunType.RunType import com.webank.wedatasphere.streamis.jobmanager.launcher.service.StreamJobConfService -import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamJobMapper, StreamJobTemplateMapper} +import com.webank.wedatasphere.streamis.jobmanager.manager.dao.{StreamJobMapper, StreamJobTemplateMapper, StreamTaskMapper} import com.webank.wedatasphere.streamis.jobmanager.manager.entity.{JobTemplateFiles, StreamJob} import com.webank.wedatasphere.streamis.jobmanager.manager.service.{StreamJobService, StreamTaskService} import com.webank.wedatasphere.streamis.jobmanager.manager.transform.StreamisTransformJobBuilder @@ -38,7 +37,7 @@ abstract class AbstractStreamisTransformJobBuilder extends StreamisTransformJobB @Autowired private var streamJobMapper: StreamJobMapper = _ @Autowired private var streamJobConfService: StreamJobConfService = _ @Autowired private var streamJobService: StreamJobService = _ - @Autowired private var streamTaskService: StreamTaskService = _ + @Autowired private var streamTaskMapper: StreamTaskMapper = _ @Autowired private var streamJobTemplateMapper:StreamJobTemplateMapper = _ protected def createStreamisTransformJob(): StreamisTransformJobImpl = new StreamisTransformJobImpl @@ -67,7 +66,7 @@ abstract class AbstractStreamisTransformJobBuilder extends StreamisTransformJobB val streamJobVersions = streamJobMapper.getJobVersions(streamJob.getId) // 无需判断streamJobVersions是否非空，因为TaskService已经判断了 transformJob.setStreamJobVersion(streamJobVersions.get(0)) - val streamTask = streamTaskService.getLatestByJobId(streamJob.getId) + val streamTask = streamTaskMapper.getLatestByJobId(streamJob.getId) val jobTemplate: JobTemplateFiles = if (null != streamTask) { if (streamTask.getStatus.equals(JobConf.FLINK_JOB_STATUS_RUNNING.getValue)) { streamJobTemplateMapper.getJobTemplate(streamTask.getTemplateId,true) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkCheckpointConfigTransform.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkCheckpointConfigTransform.scala index 1e5aab029..5dabbfa82 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkCheckpointConfigTransform.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkCheckpointConfigTransform.scala @@ -16,11 +16,10 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.transform.impl import com.webank.wedatasphere.streamis.jobmanager.launcher.JobLauncherAutoConfiguration -import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants import java.util import com.webank.wedatasphere.streamis.jobmanager.launcher.job.LaunchJob -import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.JobConf +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.{JobConf, JobConfKeyConstants} import com.webank.wedatasphere.streamis.jobmanager.launcher.job.manager.JobLaunchManager import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.job.state.FlinkCheckpoint import com.webank.wedatasphere.streamis.jobmanager.manager.transform.impl.FlinkCheckpointConfigTransform.CHECKPOINT_PATH_CONFIG_NAME diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkExtraConfigTransform.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkExtraConfigTransform.scala index 97c8caeac..5349113e5 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkExtraConfigTransform.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkExtraConfigTransform.scala @@ -15,8 +15,8 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.transform.impl -import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants import com.webank.wedatasphere.streamis.jobmanager.launcher.job.LaunchJob +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.JobConfKeyConstants import java.util import scala.collection.JavaConverters._ diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala index 6ca49d069..ee4f790bc 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkInternalConfigTransform.scala @@ -1,8 +1,7 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.transform.impl -import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants import com.webank.wedatasphere.streamis.jobmanager.launcher.job.LaunchJob -import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.JobConf +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.{JobConf, JobConfKeyConstants} import com.webank.wedatasphere.streamis.jobmanager.manager.transform.impl.FlinkInternalConfigTransform.INTERNAL_CONFIG_MAP import org.apache.linkis.common.conf.CommonVars diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkJarStreamisJobContentTransform.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkJarStreamisJobContentTransform.scala index 00592e91f..44ffbf55d 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkJarStreamisJobContentTransform.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkJarStreamisJobContentTransform.scala @@ -15,9 +15,9 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.transform.impl -import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.{JobConfKeyConstants, JobConstants} +import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConstants import com.webank.wedatasphere.streamis.jobmanager.launcher.job.LaunchJob -import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.JobConf +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.{JobConf, JobConfKeyConstants} import java.util import org.apache.linkis.common.utils.JsonUtils diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkSavepointConfigTransform.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkSavepointConfigTransform.scala index 9465ec7b7..34c2dfa02 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkSavepointConfigTransform.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/FlinkSavepointConfigTransform.scala @@ -1,8 +1,9 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.transform.impl -import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants import com.webank.wedatasphere.streamis.jobmanager.launcher.job.LaunchJob +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.JobConfKeyConstants import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.conf.JobLauncherConfiguration.VAR_FLINK_SAVEPOINT_PATH import com.webank.wedatasphere.streamis.jobmanager.manager.transform.impl.FlinkSavepointConfigTransform.SAVE_POINT_PREFIX +import org.apache.commons.lang3.StringUtils import java.util import scala.collection.JavaConverters._ @@ -21,7 +22,8 @@ class FlinkSavepointConfigTransform extends FlinkConfigTransform { override protected def configGroup(): String = JobConfKeyConstants.GROUP_PRODUCE.getValue override protected def transform(valueSet: util.Map[String, AnyRef], job: LaunchJob): LaunchJob = { - val config: util.Map[String, AnyRef] = valueSet.asScala.filter(_._1.startsWith(JobConfKeyConstants.SAVEPOINT.getValue)) + val config: util.Map[String, AnyRef] = valueSet.asScala.filter(kv => + kv._1.startsWith(JobConfKeyConstants.SAVEPOINT.getValue) && StringUtils.isNotBlank(String.valueOf(kv._2))) .map{ case (key, value) => (FlinkConfigTransform.FLINK_CONFIG_PREFIX + key.replace(JobConfKeyConstants.SAVEPOINT.getValue, SAVE_POINT_PREFIX), value) diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/LaunchConfigTransform.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/LaunchConfigTransform.scala index c988b4b1d..9f84b9558 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/LaunchConfigTransform.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/LaunchConfigTransform.scala @@ -15,9 +15,8 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.transform.impl -import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants import com.webank.wedatasphere.streamis.jobmanager.launcher.job.LaunchJob -import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.JobConf +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.{JobConf, JobConfKeyConstants} import java.util import com.webank.wedatasphere.streamis.jobmanager.manager.transform.Transform diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/ResourceConfigTransform.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/ResourceConfigTransform.scala index 14f972fda..18d582369 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/ResourceConfigTransform.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/impl/ResourceConfigTransform.scala @@ -15,10 +15,9 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.transform.impl -import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants - import java.util import com.webank.wedatasphere.streamis.jobmanager.launcher.job.LaunchJob +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.JobConfKeyConstants import com.webank.wedatasphere.streamis.jobmanager.manager.transform.ConfigTransform import com.webank.wedatasphere.streamis.jobmanager.manager.transform.impl.ResourceConfigTransform.RESOURCE_CONFIG_MAP import com.webank.wedatasphere.streamis.jobmanager.manager.utils.JobUtils diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/AbstractJobContentParser.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/AbstractJobContentParser.scala index 22b454afb..047dc29d9 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/AbstractJobContentParser.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/AbstractJobContentParser.scala @@ -25,7 +25,7 @@ import org.apache.linkis.common.conf.Configuration import org.apache.linkis.common.utils.{JsonUtils, Logging} import com.webank.wedatasphere.streamis.jobmanager.manager.dao.StreamJobMapper import com.webank.wedatasphere.streamis.jobmanager.manager.entity.{JobTemplateFiles, StreamJob, StreamJobVersion, StreamisFile} -import com.webank.wedatasphere.streamis.jobmanager.manager.service.{BMLService, StreamTaskService, StreamiFileService} +import com.webank.wedatasphere.streamis.jobmanager.manager.service.{BMLService, StreamJobTemplateService, StreamTaskService, StreamiFileService} import com.webank.wedatasphere.streamis.jobmanager.manager.transform.JobContentParser import org.apache.commons.io.IOUtils import org.apache.commons.lang.StringUtils @@ -41,7 +41,7 @@ abstract class AbstractJobContentParser extends JobContentParser with Logging { @Autowired private var streamJobMapper: StreamJobMapper = _ @Autowired private var bmlService: BMLService = _ @Autowired private var streamiFileService: StreamiFileService = _ - @Autowired private var streamTaskService: StreamTaskService = _ + @Autowired private var streamJobTemplateService: StreamJobTemplateService = _ private def findFromProject(projectName: String, fileName: String): StreamisFile = fileName match { case AbstractJobContentParser.PROJECT_FILE_REGEX(name, version) => @@ -118,7 +118,7 @@ abstract class AbstractJobContentParser extends JobContentParser with Logging { override def canParse(job: StreamJob, jobVersion: StreamJobVersion): Boolean = jobType == job.getJobType protected def getFinalTemplate(jobTemplate: JobTemplateFiles): String = { - streamTaskService.generateJobTemplate(jobTemplate) + streamJobTemplateService.generateJobTemplate(jobTemplate) } } diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala index c6e00d7ff..37406b2ec 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/transform/parser/SparkTaskMetricsParser.scala @@ -1,5 +1,7 @@ package com.webank.wedatasphere.streamis.jobmanager.manager.transform.parser +import com.webank.wedatasphere.streamis.jobmanager.manager.constrants.JobConstrants.UNKNOWN_COMMENT + import java.util import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJob @@ -50,27 +52,27 @@ class SparkTaskMetricsParser extends AbstractTaskMetricsParser { case _ => val loadConditionDTO = new JobDetailsVo.LoadConditionDTO loadConditionDTO.setType("Driver") - loadConditionDTO.setHost("") - loadConditionDTO.setMemory("") - loadConditionDTO.setTotalMemory("") - loadConditionDTO.setGcLastTime("") - loadConditionDTO.setGcLastConsume("") - loadConditionDTO.setGcTotalTime("") + loadConditionDTO.setHost(UNKNOWN_COMMENT) + loadConditionDTO.setMemory(UNKNOWN_COMMENT) + loadConditionDTO.setTotalMemory(UNKNOWN_COMMENT) + loadConditionDTO.setGcLastTime(UNKNOWN_COMMENT) + loadConditionDTO.setGcLastConsume(UNKNOWN_COMMENT) + loadConditionDTO.setGcTotalTime(UNKNOWN_COMMENT) loadConditionDTOs.add(loadConditionDTO) } val realTimeTrafficDTO = new JobDetailsVo.RealTimeTrafficDTO metricsMap.get("batchMetrics") match { case batchMetrics: util.List[util.Map[String, Object]] if !batchMetrics.isEmpty => val batchMetric = batchMetrics.asScala.maxBy(_.get("batchTime").asInstanceOf[String]) - realTimeTrafficDTO.setSourceKey(metricsMap.getOrDefault("source", "").asInstanceOf[String]) + realTimeTrafficDTO.setSourceKey(metricsMap.getOrDefault("source", UNKNOWN_COMMENT).asInstanceOf[String]) realTimeTrafficDTO.setSourceSpeed(batchMetric.get("inputRecords") + " Records") realTimeTrafficDTO.setTransformKey("processing") - realTimeTrafficDTO.setSinkKey(metricsMap.getOrDefault("sink", "").asInstanceOf[String]) + realTimeTrafficDTO.setSinkKey(metricsMap.getOrDefault("sink", UNKNOWN_COMMENT).asInstanceOf[String]) val sinkSpeed = if (batchMetric.containsKey("totalDelay") && batchMetric.get("totalDelay") != null) Utils.msDurationToString(batchMetric.get("totalDelay").toString.toInt) + " totalDelay" else if (batchMetric.containsKey("taskExecuteTime") && batchMetric.get("taskExecuteTime") != null) Utils.msDurationToString(batchMetric.get("taskExecuteTime").toString.toInt) + " executeTime(Last Batch)" - else "" + else UNKNOWN_COMMENT realTimeTrafficDTO.setSinkSpeed(sinkSpeed) case _ => realTimeTrafficDTO.setSourceKey("") diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/utils/JobContentUtils.scala b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/utils/JobContentUtils.scala index 326936d32..6e6f39d79 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/utils/JobContentUtils.scala +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/utils/JobContentUtils.scala @@ -30,7 +30,6 @@ object JobContentUtils { val jobContentMap = JsonUtils.jackson.readValue(jobContent, classOf[util.Map[String, Object]]) if (StringUtils.isNotBlank(jobTemplate)){ val metaJsonTemplate = JsonUtils.jackson.readValue(jobTemplate, classOf[util.Map[String, Object]]) -// val jobContentTemplate = metaJsonTemplate.get("jobContent").asInstanceOf[JavaMap[String, Object]] val finalJobContent: JavaMap[String, Object] = new java.util.HashMap[String, Object](metaJsonTemplate) MergeUtils.merge(finalJobContent,jobContentMap) finalJobContent @@ -52,10 +51,8 @@ object JobContentUtils { } def getFinalJobConfig(jobConfigMap: JavaMap[String, Object], jobTemplate: String): JavaMap[String, Object] = { -// val jobConfigMap = JsonUtils.jackson.readValue(jobConfig, classOf[util.Map[String, Object]]) if (StringUtils.isNotBlank(jobTemplate)) { val metaJsonTemplate = JsonUtils.jackson.readValue(jobTemplate, classOf[util.Map[String, Object]]) -// val jobConfigTemplate = metaJsonTemplate.get("jobContent").asInstanceOf[JavaMap[String, Object]] val finalJobConfig: JavaMap[String, Object] = new java.util.HashMap[String, Object](metaJsonTemplate) MergeUtils.merge(finalJobConfig,jobConfigMap) finalJobConfig diff --git a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/utils/MergeUtils.java b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/utils/MergeUtils.java index 75b1e1e90..59e8eb77d 100644 --- a/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/utils/MergeUtils.java +++ b/streamis-jobmanager/streamis-job-manager/streamis-job-manager-service/src/main/scala/com/webank/wedatasphere/streamis/jobmanager/manager/utils/MergeUtils.java @@ -3,13 +3,17 @@ import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJob; import org.apache.commons.lang3.StringUtils; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.Map; import java.util.Stack; public class MergeUtils { + private static final String JOB_CONFIG = "jobConfig"; + public static void merge(Map destination, Map source) { - Stack> stackDest = new Stack<>(); - Stack> stackSrc = new Stack<>(); + Deque> stackDest = new ArrayDeque<>(); + Deque> stackSrc = new ArrayDeque<>(); stackDest.push(destination); stackSrc.push(source); @@ -18,7 +22,8 @@ public static void merge(Map destination, Map so Map currentDest = stackDest.pop(); Map currentSrc = stackSrc.pop(); - for (String key : currentSrc.keySet()) { + for (Map.Entry entry : currentSrc.entrySet()) { + String key = entry.getKey(); Object srcValue = currentSrc.get(key); if (currentDest.containsKey(key) && currentDest.get(key) instanceof Map && srcValue instanceof Map) { stackDest.push((Map) currentDest.get(key)); @@ -34,8 +39,8 @@ public static Map getJobTemplateConfMap(String jobTemplate){ Map jobTemplateConfig = null; if(StringUtils.isNotBlank(jobTemplate)){ Map jobTemplateMap = JobContentUtils.getMap(jobTemplate); - if (jobTemplateMap.containsKey("jobConfig") && jobTemplateMap.get("jobConfig") instanceof Map) { - jobTemplateConfig = (Map) jobTemplateMap.get("jobConfig"); + if (jobTemplateMap.containsKey(JOB_CONFIG) && jobTemplateMap.get(JOB_CONFIG) instanceof Map) { + jobTemplateConfig = (Map) jobTemplateMap.get(JOB_CONFIG); } } return jobTemplateConfig; diff --git a/streamis-jobmanager/streamis-jobmanager-common/pom.xml b/streamis-jobmanager/streamis-jobmanager-common/pom.xml index 1a41d16cd..438c7d40b 100644 --- a/streamis-jobmanager/streamis-jobmanager-common/pom.xml +++ b/streamis-jobmanager/streamis-jobmanager-common/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.3.1 + ${revision} 4.0.0 @@ -38,6 +38,12 @@ org.apache.linkis linkis-common + + + jackson-databind + com.fasterxml.jackson.core + + diff --git a/streamis-jobmanager/streamis-jobmanager-server/pom.xml b/streamis-jobmanager/streamis-jobmanager-server/pom.xml index 242a1ff31..258995584 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/pom.xml +++ b/streamis-jobmanager/streamis-jobmanager-server/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.3.1 + ${revision} 4.0.0 @@ -34,12 +34,12 @@ com.webank.wedatasphere.streamis streamis-job-audit-log - ${streamis.version} + ${project.version} com.webank.wedatasphere.streamis streamis-job-launcher-service - ${streamis.version} + ${project.version} org.apache.linkis @@ -50,7 +50,7 @@ com.webank.wedatasphere.streamis streamis-job-manager-service - ${streamis.version} + ${project.version} org.apache.linkis @@ -79,12 +79,12 @@ com.webank.wedatasphere.streamis streamis-jobmanager-common - ${streamis.version} + ${project.version} - + diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/HighAvailableRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/HighAvailableRestfulApi.java index 43941d9e2..b9d36aa10 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/HighAvailableRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/HighAvailableRestfulApi.java @@ -1,15 +1,13 @@ package com.webank.wedatasphere.streamis.jobmanager.restful.api; -import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants; -import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.JobConf; import com.webank.wedatasphere.streamis.jobmanager.launcher.service.StreamJobConfService; -import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJob; import com.webank.wedatasphere.streamis.jobmanager.manager.project.service.ProjectPrivilegeService; import com.webank.wedatasphere.streamis.jobmanager.manager.service.StreamJobService; import com.webank.wedatasphere.streamis.jobmanager.service.HighAvailableService; import com.webank.wedatasphere.streamis.jobmanager.vo.HighAvailableMsg; import org.apache.commons.lang.StringUtils; +import org.apache.linkis.proxy.ProxyUserEntity; import org.apache.linkis.server.Message; import org.apache.linkis.server.utils.ModuleUserUtils; import org.slf4j.Logger; @@ -17,13 +15,10 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; -import java.util.HashMap; -import java.util.Optional; @RequestMapping(path = "/streamis/streamJobManager/highAvailable") @RestController @@ -53,9 +48,12 @@ public Message getClusterMsg(HttpServletRequest request){ @RequestMapping(path = "/username", method = RequestMethod.GET) public Message getUserName(HttpServletRequest request){ Message result = Message.ok("success"); - String userName = ModuleUserUtils.getOperationUser(request, "get user name"); + ProxyUserEntity proxyUserEntity = ModuleUserUtils.getProxyUserEntity(request, "record audit log"); + String proxyUser = proxyUserEntity.getProxyUser(); + String userName = proxyUserEntity.getUsername(); if (StringUtils.isBlank(userName)) return Message.error("current user has no permission"); result.data("userName",userName); + result.data("proxyUser",proxyUser); return result; } } diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobBulkRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobBulkRestfulApi.java index f19575bff..5da45fb4a 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobBulkRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobBulkRestfulApi.java @@ -1,6 +1,6 @@ package com.webank.wedatasphere.streamis.jobmanager.restful.api; -import com.webank.wedatasphere.streamis.jobmanager.launcher.conf.JobConfKeyConstants; +import com.webank.wedatasphere.streamis.jobmanager.launcher.job.conf.JobConfKeyConstants; import com.webank.wedatasphere.streamis.jobmanager.launcher.linkis.conf.JobLauncherConfiguration; import com.webank.wedatasphere.streamis.jobmanager.launcher.service.StreamJobConfService; import com.webank.wedatasphere.streamis.jobmanager.manager.entity.StreamJob; @@ -94,7 +94,7 @@ public Message bulkExecution(@RequestBody JobBulkRequest execBulkRequest, HttpSe return Message.error("The master and backup cluster materials do not match, please check the material"); } if (!streamjobService.getEnableStatus(Long.parseLong(jobId.toString()))){ - return Message.error("current Job " + streamJob.getName() + "has been banned, cannot start,please enable job" ); + return Message.error("The Job " + streamJob.getName() + "has been banned, cannot start,please enable job" ); } if(!highAvailableService.canBeStarted(Long.parseLong(jobId.toString()))){ return Message.error("current Job " + streamJob.getName() + " is in managerSlave mode,please check whether it runs on manager cluster" ); @@ -154,7 +154,7 @@ public Message bulkPause(@RequestBody JobBulkPauseRequest pauseRequest, HttpServ throw new JobExecuteErrorException(-1, "Have no permission to execute StreamJob [" + jobId + "]"); } if (!streamjobService.getEnableStatus(Long.parseLong(jobId.toString()))){ - return Message.error("current Job " + streamJob.getName() + "has been banned, cannot stop,please enable job" ); + return Message.error("Job " + streamJob.getName() + "has been banned, cannot stop,please enable job" ); } } pauseResults = streamTaskService.bulkPause(new ArrayList<>(pauseRequest.getBulkSubject()), diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobConfRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobConfRestfulApi.java index 1c20c976b..4c2a24010 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobConfRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/JobConfRestfulApi.java @@ -55,6 +55,8 @@ public class JobConfRestfulApi { @Resource private ProjectPrivilegeService privilegeService; + private static final String SUCCESS_INFO = "success"; + /** * Definitions * @@ -62,7 +64,7 @@ public class JobConfRestfulApi { */ @RequestMapping(value = "/definitions") public Message definitions() { - Message result = Message.ok("success"); + Message result = Message.ok(SUCCESS_INFO); try { List definitionList = this.streamJobConfService.loadAllDefinitions(); Map definitionRelation = new HashMap<>(); @@ -104,7 +106,7 @@ public Message definitions() { */ @RequestMapping(value = "/json/{jobId:\\w+}", method = RequestMethod.GET) public Message queryConfig(@PathVariable("jobId") Long jobId, HttpServletRequest request) { - Message result = Message.ok("success"); + Message result = Message.ok(SUCCESS_INFO); try { String userName = ModuleUserUtils.getOperationUser(request, "query job config json"); StreamJob streamJob = this.streamJobService.getJobById(jobId); @@ -138,11 +140,11 @@ public Message queryConfig(@PathVariable("jobId") Long jobId, HttpServletRequest @RequestMapping(value = "/json/{jobId:\\w+}", method = RequestMethod.POST) public Message saveConfig(@PathVariable("jobId") Long jobId, @RequestBody Map configContent, HttpServletRequest request) { - Message result = Message.ok("success"); - if (!(Boolean) JobConf.JOB_CONFIG_EDIT_ENABLE().getHotValue()){ + Message result = Message.ok(SUCCESS_INFO); + if (!(boolean) JobConf.JOB_CONFIG_EDIT_ENABLE().getHotValue()){ return Message.error("job config cannot be changed,please contact the admin for advice"); } - if((Boolean) JobConf.PRODUCT_NAME_SWITCH().getHotValue()){ + if((boolean) JobConf.PRODUCT_NAME_SWITCH().getHotValue()){ try { String productValue = Optional.ofNullable(configContent) .map(jovConf -> (Map) jovConf.get("wds.linkis.flink.produce")) @@ -180,7 +182,7 @@ public Message saveConfig(@PathVariable("jobId") Long jobId, @RequestBody Map tasksData = bulkUpdateLabelRequest.getTasks(); @@ -159,10 +176,10 @@ public Message updateLabel(HttpServletRequest req, @RequestBody BulkUpdateLabelR StreamJob streamJob = this.streamJobService.getJobById(jobId); if (!streamJobService.isCreator(jobId, userName) && !this.privilegeService.hasEditPrivilege(req, streamJob.getProjectName())) { - return Message.error("Have no permission to save StreamJob [" + jobId + "] configuration"); + return Message.error("Have no permission to save the configuration of StreamJob [" + jobId + "]"); } if (!streamJobService.getEnableStatus(jobId)){ - return Message.error("current Job " + streamJob.getName() + "has been banned, cannot updateLable,please enable job" ); + return Message.error("The Job " + streamJob.getName() + "has been banned, cannot updateLable,please enable job" ); } String label = bulkUpdateLabel.getLabel(); if (!RegularUtil.matches(label)) @@ -204,7 +221,7 @@ public Message versions(HttpServletRequest req, @PathVariable("jobId")Integer jo PageInfo pageInfo = this.streamJobService.getVersionList(jobId); if (Objects.nonNull(pageInfo)){ result.data("versions", pageInfo.getList()); - result.data("totalPage", pageInfo.getTotal()); + result.data(TOTAL_PAGE, pageInfo.getTotal()); } } catch (Exception e){ result = Message.error("Fail to query job version page (查看任务版本列表失败), message: " + e.getMessage()); @@ -217,7 +234,7 @@ public Message versions(HttpServletRequest req, @PathVariable("jobId")Integer jo public Message version(HttpServletRequest req, @RequestParam(value = "jobId", required = false) Long jobId, @RequestParam(value = "version", required = false) String version) throws JobException { if (jobId == null) { - throw JobExceptionManager.createException(30301, "jobId"); + throw JobExceptionManager.createException(30301, JOB_ID); } if (StringUtils.isEmpty(version)) { throw JobExceptionManager.createException(30301, "version"); @@ -235,7 +252,7 @@ public Message version(HttpServletRequest req, @RequestParam(value = "jobId", re @RequestMapping(path = "/ban", method = RequestMethod.POST) public Message banJob(HttpServletRequest req, @RequestBody List jobIdList) { String userName = ModuleUserUtils.getOperationUser(req, "ban job"); - Message result = Message.ok("success"); + Message result = Message.ok(SUCCESS_MSG); HashMap jobMap = new HashMap<>(); if (jobIdList.size() > 100){ @@ -250,14 +267,14 @@ public Message banJob(HttpServletRequest req, @RequestBody List jobIdList) StreamJob streamJob = this.streamJobService.getJobById(jobId); jobMap.put(jobId,streamJob); if (streamJob == null) { - return Message.error("not exists job " + jobId); + return Message.error(NOT_EXISTS_JOB + jobId); } if (!streamJobService.isCreator(jobId, userName) && !this.privilegeService.hasEditPrivilege(req, streamJob.getProjectName())) { return Message.error("Have no permission to ban StreamJob [" + jobId + "] configuration"); } if (!streamJobService.canBeDisabled(jobId)){ - return Message.error("current job [" + jobId + "] can not be banned, please check"); + return Message.error("Job [" + jobId + "] can not be banned, please check"); } } for (Long jobId : jobIdList) { @@ -274,7 +291,7 @@ public Message banJob(HttpServletRequest req, @RequestBody List jobIdList) @RequestMapping(path = "/enable", method = RequestMethod.POST) public Message enableJob(HttpServletRequest req, @RequestBody List jobIdList) { String userName = ModuleUserUtils.getOperationUser(req, "ban job"); - Message result = Message.ok("success"); + Message result = Message.ok(SUCCESS_MSG); HashMap jobMap = new HashMap<>(); if (jobIdList.size() > 100){ @@ -290,14 +307,14 @@ public Message enableJob(HttpServletRequest req, @RequestBody List jobIdLi StreamJob streamJob = this.streamJobService.getJobById(jobId); jobMap.put(jobId,streamJob); if (streamJob == null) { - return Message.error("not exists job " + jobId); + return Message.error(NOT_EXISTS_JOB + jobId); } if (!streamJobService.isCreator(jobId, userName) && !this.privilegeService.hasEditPrivilege(req, streamJob.getProjectName())) { return Message.error("Have no permission to ban StreamJob [" + jobId + "] configuration"); } if (!streamJobService.canbeActivated(jobId)){ - return Message.error("current job [" + jobId + "] can not be activated, please check"); + return Message.error("Present job [" + jobId + "] can not be activated, please check"); } } for (Long jobId : jobIdList) { @@ -337,7 +354,7 @@ public Message executeInspect(HttpServletRequest req, @RequestParam(value = "job return Message.error("The system does not enable the detach feature ,detach job cannot start [" + jobId + "]"); } if (!streamJobService.getEnableStatus(jobId)){ - return Message.error("current Job " + streamJob.getName() + "has been banned, cannot start,please enable job" ); + return Message.error("The Job " + streamJob.getName() + "has been banned, cannot start,please enable job" ); } try { HashMap jobConfig = new HashMap<>(this.streamJobConfService.getJobConfig(jobId)); @@ -377,13 +394,13 @@ public Message executeInspect(HttpServletRequest req, @RequestParam(value = "job inspections = inspectResult.stream().map(JobInspectVo::getInspectName) .collect(Collectors.toList()); } catch (Exception e){ - LOG.warn(e.getMessage()); + LOG.warn(e.getMessage(), e); return Message.error("Fail to inspect job " + jobId + " of the execution(任务执行前检查失败), message: " + e.getMessage()); } HashMap inspectResultMap = new HashMap<>(); inspectResult.forEach(inspect -> inspectResultMap.put(inspect.getInspectName(), inspect)); - if (!inspectResultMap.containsKey("snapshot")){ + if (!inspectResultMap.containsKey(SNAPSHOT)){ String value = this.streamJobConfService.getJobConfValue(jobId, JobConfKeyConstants.START_AUTO_RESTORE_SWITCH().getValue()); String msg; if (value.equals("ON")){ @@ -391,10 +408,10 @@ public Message executeInspect(HttpServletRequest req, @RequestParam(value = "job }else { msg ="任务未开启快照，无需检查快照地址"; } - inspections.add("snapshot"); + inspections.add(SNAPSHOT); JobSnapshotInspectVo jobSnapshotInspectVo =new JobSnapshotInspectVo(); jobSnapshotInspectVo.setPath(msg); - inspectResultMap.put("snapshot",jobSnapshotInspectVo); + inspectResultMap.put(SNAPSHOT,jobSnapshotInspectVo); } inspectResultMap.put("inspections", inspections); result.setData(inspectResultMap); @@ -404,14 +421,14 @@ public Message executeInspect(HttpServletRequest req, @RequestParam(value = "job @RequestMapping(path = "/execute", method = RequestMethod.POST) public Message executeJob(HttpServletRequest req, @RequestBody Map json) throws JobException { String userName = ModuleUserUtils.getOperationUser(req, "execute job"); - if (!json.containsKey("jobId") || json.get("jobId") == null) { - throw JobExceptionManager.createException(30301, "jobId"); + if (!json.containsKey(JOB_ID) || json.get(JOB_ID) == null) { + throw JobExceptionManager.createException(30301, JOB_ID); } - long jobId = Long.parseLong(json.get("jobId").toString()); + long jobId = Long.parseLong(json.get(JOB_ID).toString()); LOG.info("{} try to execute job {}.", userName, jobId); StreamJob streamJob = this.streamJobService.getJobById(jobId); if(streamJob == null) { - return Message.error("not exists job " + jobId); + return Message.error(NOT_EXISTS_JOB + jobId); } else if(!JobConf.SUPPORTED_MANAGEMENT_JOB_TYPES().getValue().contains(streamJob.getJobType())) { return Message.error("Job " + streamJob.getName() + " is not supported to execute."); } @@ -430,16 +447,14 @@ public Message executeJob(HttpServletRequest req, @RequestBody Map args = contentRequest.getArgs(); if (args == null){ boolean isHighAvailable = contentRequest.isHighAvailable(); String highAvailableMessage = contentRequest.getHighAvailableMessage(); StreamisTransformJobContent jobContent = streamJobService.updateArgs(jobId, version,null,isHighAvailable,highAvailableMessage); - return Message.ok().data("jobContent", jobContent); + return Message.ok().data(JOB_CONTENT, jobContent); } else { - if (!(Boolean) JobConf.JOB_CONTENT_EDIT_ENABLE().getHotValue()){ + if (!(boolean) JobConf.JOB_CONTENT_EDIT_ENABLE().getHotValue()){ return Message.error("job args cannot be changed,please contact the admin for advice"); } int hotValue = Integer.parseInt(JobConf.DEFAULT_ARGS_LENGTH().getHotValue().toString()); @@ -794,7 +809,7 @@ public Message updateContent(HttpServletRequest req, @RequestBody UpdateContentR return Message.error("args length is too long, please less than "+ hotValue); } StreamisTransformJobContent jobContent = streamJobService.updateArgs(jobId, version,args,false,null); - return Message.ok().data("jobContent", jobContent); + return Message.ok().data(JOB_CONTENT, jobContent); } } @@ -822,7 +837,7 @@ public Message getAlert(HttpServletRequest req, } finally { PageHelper.clearPage(); } - return Message.ok().data("list",pageInfo.getList()).data("totalPage", pageInfo.getTotal()); + return Message.ok().data("list",pageInfo.getList()).data(TOTAL_PAGE, pageInfo.getTotal()); } @RequestMapping(path = "/logs", method = RequestMethod.GET) @@ -836,13 +851,13 @@ public Message getLog(HttpServletRequest req, @RequestParam(value = "logType", required = false) String logType, @RequestParam(value = "lastRows", defaultValue = "0") Integer lastRows) throws JobException { if (jobId == null) { - throw JobExceptionManager.createException(30301, "jobId"); + throw JobExceptionManager.createException(30301, JOB_ID); } logType = StringUtils.isBlank(logType) ? "client" : logType; String username = ModuleUserUtils.getOperationUser(req, "view job logs"); StreamJob streamJob = this.streamJobService.getJobById(jobId); if(streamJob == null) { - return Message.error("not exists job " + jobId); + return Message.error(NOT_EXISTS_JOB + jobId); } else if(!JobConf.SUPPORTED_MANAGEMENT_JOB_TYPES().getValue().contains(streamJob.getJobType()) && "client".equals(logType)) { return Message.error("Job " + streamJob.getName() + " is not supported to get client logs."); @@ -872,7 +887,7 @@ public Message status(@RequestBody Map> requestMap){ if (Objects.isNull(jobIds) || jobIds.isEmpty()){ return Message.error("The list of job id which to refresh the status cannot be null or empty"); } - Message result = Message.ok("success"); + Message result = Message.ok(SUCCESS_MSG); try{ result.data("result", this.streamTaskService.getStatusList(new ArrayList<>(jobIds))); }catch (Exception e){ @@ -894,7 +909,7 @@ public Message snapshot(@PathVariable("jobId")Long jobId, HttpServletRequest req String username = ModuleUserUtils.getOperationUser(request, "do snapshot of job"); StreamJob streamJob = this.streamJobService.getJobById(jobId); if(streamJob == null) { - return Message.error("not exists job " + jobId); + return Message.error(NOT_EXISTS_JOB + jobId); } else if(!JobConf.SUPPORTED_MANAGEMENT_JOB_TYPES().getValue().contains(streamJob.getJobType())) { return Message.error("Job " + streamJob.getName() + " is not supported to do snapshot."); } diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/UploadRestfulApi.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/UploadRestfulApi.java index 5f70bf47c..20b4d1524 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/UploadRestfulApi.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/restful/api/UploadRestfulApi.java @@ -95,11 +95,9 @@ public Message uploadJar(HttpServletRequest request, if(!ZipHelper.isZip(fileName)){ throw JobExceptionManager.createException(30302); } - if ((Boolean) JobConf.STANDARD_AUTHENTICATION_KEY().getHotValue()){ - if (!highAvailableService.confirmToken(source)){ + if ((boolean) JobConf.STANDARD_AUTHENTICATION_KEY().getHotValue() && !highAvailableService.confirmToken(source)){ return Message.error("As this job is not from standard release, it is not allowed to upload"); } - } InputStream is = null; OutputStream os = null; File file = null; diff --git a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/service/impl/HighAvailableServiceImpl.java b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/service/impl/HighAvailableServiceImpl.java index b34d1915b..a69f44b8e 100644 --- a/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/service/impl/HighAvailableServiceImpl.java +++ b/streamis-jobmanager/streamis-jobmanager-server/src/main/java/com/webank/wedatasphere/streamis/jobmanager/service/impl/HighAvailableServiceImpl.java @@ -39,13 +39,24 @@ public JobHighAvailableVo getJobHighAvailableVo(long jobId){ String highAvailablePolicy = this.streamJobConfService.getJobConfValue(jobId, JobConf.HIGHAVAILABLE_POLICY_KEY().getValue()); JobHighAvailableVo inspectVo = new JobHighAvailableVo(); Optional sourceOption = Optional.ofNullable(jobVersion.getSource()); - if(sourceOption.isPresent() && JsonUtil.isJson(sourceOption.get())) { + if(sourceOption.isPresent()) { String source = sourceOption.get(); inspectVo = SourceUtils.manageJobProjectFile(highAvailablePolicy, source); } else { LOG.warn("this job source is null"); + if (null == highAvailablePolicy || highAvailablePolicy.equals(JobConf.HIGHAVAILABLE_DEFAULT_POLICY().getValue()) || highAvailablePolicy.equals(JobConf.HIGHAVAILABLE_POLICY_SINGLE_BAK().getValue())){ inspectVo.setHighAvailable(true); - inspectVo.setMsg("User changed params of job not by deploy, will skip to check its highavailable(用户未走发布单独修改了job信息，跳过高可用检查)"); + inspectVo.setMsg("job为单活，跳过高可用检查"); + }else { + if(Boolean.parseBoolean(JobConf.HIGHAVAILABLE_ENABLE_INTERFACE_UPLOAD().getHotValue().toString())){ + inspectVo.setHighAvailable(true); + inspectVo.setMsg("job的source为空，跳过高可用检查"); + }else{ + inspectVo.setHighAvailable(false); + inspectVo.setMsg("job的source为空，高可用检查不通过"); + } + } + } return inspectVo; } @@ -75,17 +86,15 @@ public Boolean confirmToken(String source){ Optional sourceOption = Optional.ofNullable(source); if(sourceOption.isPresent() && JsonUtil.isJson(sourceOption.get())) { String sourceStr = sourceOption.get(); - Map sourceMap = BDPJettyServerHelper.gson().fromJson(sourceStr, Map.class); + Map sourceMap = BDPJettyServerHelper.gson().fromJson(sourceStr, Map.class); if (sourceMap.containsKey("source")) { String sourceValue = sourceMap.get("source").toString(); - if (sourceValue.equals(JobConf.HIGHAVAILABLE_SOURCE().getValue())) { - if (sourceMap.containsKey("token")) { + if (sourceValue.equals(JobConf.HIGHAVAILABLE_SOURCE().getValue()) && sourceMap.containsKey("token")) { String tokenContent = sourceMap.get("token").toString(); return tokenContent.equals(JobConf.HIGHAVAILABLE_TOKEN().getValue()); } } } - } return false; } } diff --git a/streamis-jobmanager/streamis-projectmanager-server/pom.xml b/streamis-jobmanager/streamis-projectmanager-server/pom.xml index 327f04418..dcf54842d 100644 --- a/streamis-jobmanager/streamis-projectmanager-server/pom.xml +++ b/streamis-jobmanager/streamis-projectmanager-server/pom.xml @@ -20,7 +20,7 @@ streamis-jobmanager com.webank.wedatasphere.streamis - 0.3.1 + ${revision} 4.0.0 @@ -34,7 +34,13 @@ com.webank.wedatasphere.streamis streamis-job-audit-log - ${streamis.version} + ${project.version} + + + jackson-core + com.fasterxml.jackson.core + + org.apache.linkis @@ -58,7 +64,7 @@ com.webank.wedatasphere.streamis streamis-job-manager-service - ${streamis.version} + ${project.version} org.apache.linkis diff --git a/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/restful/api/ProjectManagerRestfulApi.java b/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/restful/api/ProjectManagerRestfulApi.java index e5ba552e8..de94d5c05 100644 --- a/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/restful/api/ProjectManagerRestfulApi.java +++ b/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/restful/api/ProjectManagerRestfulApi.java @@ -73,6 +73,8 @@ public class ProjectManagerRestfulApi { private static final String templateName = "-meta.json"; + private static final String NULL_PROJECT_NAME = "projectName is null"; + @RequestMapping(path = "/files/upload", method = RequestMethod.POST) public Message upload(HttpServletRequest req, @RequestParam(name = "version",required = false) String version, @@ -86,7 +88,7 @@ public Message upload(HttpServletRequest req, return Message.error("version is null"); } if (StringUtils.isBlank(projectName)) { - return Message.error("projectName is null"); + return Message.error(NULL_PROJECT_NAME); } if (StringUtils.isBlank(source)) { LOG.info("source的值为空"); @@ -97,17 +99,15 @@ public Message upload(HttpServletRequest req, if (!projectPrivilegeService.hasEditPrivilege(req,projectName)) { return Message.error(NO_OPERATION_PERMISSION_MESSAGE); } - if ((Boolean) JobConf.STANDARD_AUTHENTICATION_KEY().getHotValue()){ - if (!projectManagerService.confirmToken(source)){ + if ((boolean) JobConf.STANDARD_AUTHENTICATION_KEY().getHotValue() && !projectManagerService.confirmToken(source)){ return Message.error("As this file is not from standard release, it is not allowed to upload"); } - } //Only uses 1st file(只取第一个文件) MultipartFile p = files.get(0); String fileName = new String(p.getOriginalFilename().getBytes("ISO8859-1"), StandardCharsets.UTF_8); ReaderUtils readerUtils = new ReaderUtils(); if (!readerUtils.checkName(fileName)) { - return Message.warn("fileName should only contains numeric/English characters and '-_'(仅允许包含数字，英文,中划线,下划线)"); + return Message.warn("fileName " + fileName + " should only contains numeric/English characters and '-_'(仅允许包含数字，英文,中划线,下划线,点号)"); } if (!ReaderUtils.isValidFileFormat(fileName)){ @@ -128,7 +128,7 @@ public Message upload(HttpServletRequest req, os = IoUtils.generateExportOutputStream(inputPath); IOUtils.copy(is, os); if (!p.isEmpty() && p.getOriginalFilename().endsWith(templateName)) { - if(!readerUtils.checkMetaTemplate(fileName,inputPath,projectName)) return Message.error("meta template is not correct,eg:testProject(项目名)-meta.json"); + if(!readerUtils.checkMetaTemplate(fileName,inputPath,projectName)) return Message.error("meta template is not correct,please check"); projectManagerService.upload(username, fileName, version, projectName, inputPath, comment, source); StreamisFile file = projectManagerService.selectFile(fileName,version,projectName); projectManagerService.uploadJobTemplate(username,fileName,inputPath,projectName,version,file.getStorePath()); @@ -157,7 +157,7 @@ public Message list( HttpServletRequest req,@RequestParam(value = "filename",req @RequestParam(value = "pageNow",defaultValue = "1") Integer pageNow, @RequestParam(value = "pageSize",defaultValue = "20") Integer pageSize) { if (StringUtils.isBlank(projectName)) { - return Message.error("projectName is null"); + return Message.error(NULL_PROJECT_NAME); } if (!projectPrivilegeService.hasAccessPrivilege(req,projectName)) return Message.error(NO_OPERATION_PERMISSION_MESSAGE); PageHelper.startPage(pageNow, pageSize); @@ -177,7 +177,7 @@ public Message versionList( HttpServletRequest req, @RequestParam(value = "fileN @RequestParam(value = "pageNow",defaultValue = "1") Integer pageNow, @RequestParam(value = "pageSize",defaultValue = "20") Integer pageSize) { if (StringUtils.isBlank(projectName)) { - return Message.error("projectName is null"); + return Message.error(NULL_PROJECT_NAME); } if (StringUtils.isBlank(fileName)) { return Message.error("fileName is null"); @@ -227,31 +227,39 @@ public Message deleteVersion(HttpServletRequest req, @RequestParam(value = "ids" } @RequestMapping(path = "/files/download", method = RequestMethod.GET) - public Message download( HttpServletRequest req, HttpServletResponse response, + public void download( HttpServletRequest req, HttpServletResponse response, @RequestParam(value = "id",required = false) Long id, @RequestParam(value = "materialType",required = false) String materialType, @RequestParam(value = "projectName",required = false)String projectName) { StreamisFile file = null; String userName = ModuleUserUtils.getOperationUser(req, "download job"); - if (org.apache.commons.lang.StringUtils.isBlank(userName)) return Message.error("current user has no permission"); + if (org.apache.commons.lang.StringUtils.isBlank(userName)) { + LOG.error("current user has no permission"); + return; + } if (StringUtils.isBlank(projectName)) { if (StringUtils.isBlank(materialType)) { - return Message.error("projectName and materialType is null"); + LOG.error("projectName and materialType is null"); + return; } else if (materialType.equals(TYPE_JOB)) { file = streamJobService.getJobFileById(id); } else if (materialType.equals(TYPE_PROJECT)){ file = projectManagerService.getFile(id, projectName); } } else { - if (!projectPrivilegeService.hasEditPrivilege(req, projectName)) - return Message.error(NO_OPERATION_PERMISSION_MESSAGE); + if (!projectPrivilegeService.hasEditPrivilege(req, projectName)) { + LOG.error(NO_OPERATION_PERMISSION_MESSAGE); + return; + } file = projectManagerService.getFile(id, projectName); } if (file == null) { - return Message.error("no such file in this project"); + LOG.error("no such file in this project"); + return; } if (StringUtils.isBlank(file.getStorePath())) { - return Message.error("storePath is null"); + LOG.error("storePath is null"); + return; } response.setContentType("application/x-download"); response.setHeader("content-Disposition", "attachment;filename=" + file.getFileName()); @@ -267,8 +275,6 @@ public Message download( HttpServletRequest req, HttpServletResponse response, os.flush(); } catch (Exception e) { LOG.error("download file: {} failed , message is : {}", file.getFileName(), e); - return Message.error(e.getMessage()); } - return Message.ok(); } } diff --git a/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/service/impl/ProjectManagerServiceImpl.java b/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/service/impl/ProjectManagerServiceImpl.java index a49ed2a8f..3f8b5eeff 100644 --- a/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/service/impl/ProjectManagerServiceImpl.java +++ b/streamis-jobmanager/streamis-projectmanager-server/src/main/java/com/webank/wedatasphere/streamis/projectmanager/service/impl/ProjectManagerServiceImpl.java @@ -77,8 +77,6 @@ public class ProjectManagerServiceImpl implements ProjectManagerService, Streami private static final String templateName = "-meta.json"; - private static final Integer RUNNING = 5; - @Override @Transactional(rollbackFor = Exception.class) public void upload(String username, String fileName, String version, String projectName, String filePath,String comment, String source) throws JsonProcessingException { @@ -256,8 +254,7 @@ public String generateJobTemplate(MetaJsonInfo metaJsonInfo){ if (jsonObj.has(JobConstrants.FIELD_JOB_DESCRIPTION())) { jsonObj.remove(JobConstrants.FIELD_JOB_DESCRIPTION()); } - String metaJson = jsonObj.toString(); - return metaJson; + return jsonObj.toString(); } @Override @@ -265,17 +262,15 @@ public Boolean confirmToken(String source){ Optional sourceOption = Optional.ofNullable(source); if(sourceOption.isPresent() && JsonUtil.isJson(sourceOption.get())) { String sourceStr = sourceOption.get(); - Map sourceMap = BDPJettyServerHelper.gson().fromJson(sourceStr, Map.class); + Map sourceMap = BDPJettyServerHelper.gson().fromJson(sourceStr, Map.class); if (sourceMap.containsKey("source")) { String sourceValue = sourceMap.get("source").toString(); - if (sourceValue.equals(JobConf.HIGHAVAILABLE_SOURCE().getValue())) { - if (sourceMap.containsKey("token")) { + if (sourceValue.equals(JobConf.HIGHAVAILABLE_SOURCE().getValue()) && sourceMap.containsKey("token")) { String tokenContent = sourceMap.get("token").toString(); return tokenContent.equals(JobConf.HIGHAVAILABLE_TOKEN().getValue()); } } } - } return false; } diff --git a/streamis-project/pom.xml b/streamis-project/pom.xml index 2686527fc..b1234338d 100644 --- a/streamis-project/pom.xml +++ b/streamis-project/pom.xml @@ -20,7 +20,7 @@ streamis com.webank.wedatasphere.streamis - 0.3.1 + ${revision} 4.0.0 diff --git a/streamis-project/streamis-project-common/pom.xml b/streamis-project/streamis-project-common/pom.xml index a3e901baa..ddc33edd9 100644 --- a/streamis-project/streamis-project-common/pom.xml +++ b/streamis-project/streamis-project-common/pom.xml @@ -20,7 +20,7 @@ streamis-project com.webank.wedatasphere.streamis - 0.3.1 + ${revision} 4.0.0 diff --git a/streamis-project/streamis-project-server/pom.xml b/streamis-project/streamis-project-server/pom.xml index ebd12c930..98956e96e 100644 --- a/streamis-project/streamis-project-server/pom.xml +++ b/streamis-project/streamis-project-server/pom.xml @@ -20,7 +20,7 @@ streamis-project com.webank.wedatasphere.streamis - 0.3.1 + ${revision} 4.0.0 @@ -41,16 +41,12 @@ ${linkis.version} - jackson-annotations - com.fasterxml.jackson.core + reactor-core + io.projectreactor - jackson-core - com.fasterxml.jackson.core - - - guava - com.google.guava + spring-aop + org.springframework @@ -65,7 +61,7 @@ com.webank.wedatasphere.streamis streamis-project-common - ${streamis.version} + ${project.version} @@ -86,20 +82,30 @@ compile - jackson-annotations - com.fasterxml.jackson.core + org.apache.linkis + linkis-label-common - jackson-databind - com.fasterxml.jackson.core + spring-aop + org.springframework - jackson-core - com.fasterxml.jackson.core + spring-webmvc + org.springframework + + + spring-context-support + org.springframework + + org.apache.linkis + linkis-label-common + ${linkis.version} + + diff --git a/streamis-server/pom.xml b/streamis-server/pom.xml index bacc720c7..2aa4d09e2 100644 --- a/streamis-server/pom.xml +++ b/streamis-server/pom.xml @@ -20,7 +20,7 @@ streamis com.webank.wedatasphere.streamis - 0.3.1 + ${revision} 4.0.0 @@ -40,20 +40,26 @@ - + - + + + org.apache.linkis + linkis-common + ${linkis.version} + + com.webank.wedatasphere.streamis streamis-jobmanager-server - ${streamis.version} + ${project.version} jackson-mapper-asl @@ -67,25 +73,95 @@ snakeyaml org.yaml + + jetty-webapp + org.eclipse.jetty + + + jetty-servlets + org.eclipse.jetty + + + websocket-server + org.eclipse.jetty.websocket + + + javax-websocket-server-impl + org.eclipse.jetty.websocket + + + jackson-core + com.fasterxml.jackson.core + + + jackson-databind + com.fasterxml.jackson.core + + + jackson-annotations + com.fasterxml.jackson.core + + + servlet-api + javax.servlet + + + org.eclipse.jetty + jetty-webapp + ${jetty.version} + + + + org.eclipse.jetty + jetty-servlets + ${jetty.version} + + + + org.eclipse.jetty.websocket + websocket-server + ${jetty.version} + + + + org.eclipse.jetty.websocket + javax-websocket-server-impl + ${jetty.version} + + com.webank.wedatasphere.streamis streamis-projectmanager-server - ${streamis.version} + ${project.version} + + + jackson-databind + com.fasterxml.jackson.core + + com.webank.wedatasphere.streamis streamis-project-server - ${streamis.version} + ${project.version} jackson-mapper-asl org.codehaus.jackson + + jackson-databind + com.fasterxml.jackson.core + + + jackson-annotations + com.fasterxml.jackson.core + @@ -93,7 +169,7 @@ com.webank.wedatasphere.streamis streamis-job-log-server - ${streamis.version} + ${project.version} diff --git a/streamis-server/src/main/assembly/distribution.xml b/streamis-server/src/main/assembly/distribution.xml index b0c8908f7..78f1f9cab 100644 --- a/streamis-server/src/main/assembly/distribution.xml +++ b/streamis-server/src/main/assembly/distribution.xml @@ -19,6 +19,7 @@ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3 http://maven.apache.org/xsd/assembly-1.1.2.xsd"> streamis-server + dir zip true diff --git a/streamis-server/src/main/resources/application.properties b/streamis-server/src/main/resources/application.properties index b6e9bfff9..eefc7b2ba 100644 --- a/streamis-server/src/main/resources/application.properties +++ b/streamis-server/src/main/resources/application.properties @@ -12,3 +12,8 @@ management.endpoints.web.exposure.include=refresh,info logging.config.classpath=log4j2.xml feature.enabled=true + +# after linkis 1.6.0 +#spring.main.allow-circular-references=true +spring.mvc.pathmatch.matching-strategy=ant_path_matcher +spring.cloud.loadbalancer.cache.enabled=false diff --git a/streamis-server/src/main/resources/application.yml b/streamis-server/src/main/resources/application.yml deleted file mode 100644 index 837a97e4e..000000000 --- a/streamis-server/src/main/resources/application.yml +++ /dev/null @@ -1,29 +0,0 @@ -server: - port: 9321 -spring: - application: - name: streamis-server - mvc: - servlet: - path: /api/rest_j/v1 - servlet: - multipart: - enabled: true - max-file-size: 500MB - max-request-size: 500MB - -eureka: - client: - serviceUrl: - defaultZone: http://127.0.0.1:20303/eureka/ - instance: - metadata-map: - test: wedatasphere - -management: - endpoints: - web: - exposure: - include: refresh,info -logging: - config: classpath:log4j2.xml diff --git a/streamis-server/src/main/resources/linkis.properties b/streamis-server/src/main/resources/linkis.properties index da7b03091..f8c0345a2 100644 --- a/streamis-server/src/main/resources/linkis.properties +++ b/streamis-server/src/main/resources/linkis.properties @@ -45,7 +45,8 @@ wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/str classpath*:com/webank/wedatasphere/streamis/jobmanager/launcher/dao/impl/*.xml,\ classpath*:com/webank/wedatasphere/streamis/jobmanager/manager/dao/impl/*.xml,\ classpath*:com/webank/wedatasphere/streamis/projectmanager/dao/impl/*.xml,\ - classpath*:com/webank/wedatasphere/streamis/audit/log/dao/impl/*.xml + classpath*:com/webank/wedatasphere/streamis/audit/log/dao/impl/*.xml,\ + classpath*:com/webank/wedatasphere/streamis/errorcode/dao/impl/*.xml wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.streamis.datasource.manager.domain,\ com.webank.wedatasphere.streamis.jobmanager.launcher.entity,\ @@ -53,7 +54,8 @@ wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.streamis.da com.webank.wedatasphere.streamis.jobmanager.manager.entity.vo,\ com.webank.wedatasphere.streamis.jobmanager.launcher.entity.vo,\ com.webank.wedatasphere.streamis.projectmanager.entity,\ - com.webank.wedatasphere.streamis.audit.log.entity + com.webank.wedatasphere.streamis.audit.log.entity,\ + com.webank.wedatasphere.streamis.errorcode.entity wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.streamis.datasource.manager.dao,\ @@ -62,7 +64,8 @@ wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.streamis.datasourc com.webank.wedatasphere.streamis.jobmanager.launcher.dao,\ com.webank.wedatasphere.streamis.jobmanager.manager.dao,\ com.webank.wedatasphere.streamis.projectmanager.dao,\ - com.webank.wedatasphere.streamis.audit.log.dao + com.webank.wedatasphere.streamis.audit.log.dao,\ + com.webank.wedatasphere.streamis.errorcode.dao # Make sure that can fetch the application info finally wds.streamis.application.info.fetch.max=20 @@ -98,4 +101,15 @@ wds.streamis.job.content.edit.enable=true #edit jobConfig enable wds.streamis.job.config.edit.enable=true #not upgrade linkis version -flink.app.args.separate=' ' \ No newline at end of file +flink.app.args.separate=' ' + +# linkis token +#wds.linkis.token= + +# after linkis 1.6.0 +wds.linkis.rpc.spring.params.enable=true +spring.ribbon.ReadTimeout=100000 +spring.ribbon.ConnectTimeout=100000 + +# linkis flink plugin version +wds.linkis.flink.engine.version=1.16.2 \ No newline at end of file diff --git a/web/package.json b/web/package.json index aab79d38b..fa6583a60 100644 --- a/web/package.json +++ b/web/package.json @@ -23,7 +23,7 @@ ] }, "dependencies": { - "axios": "1.6.4", + "axios": "1.7.4", "babel-polyfill": "6.26.0", "core-js": "2.6.11", "dexie": "3.0.4", @@ -63,7 +63,7 @@ "husky": "1.3.1", "lint-staged": "8.2.1", "monaco-editor-webpack-plugin": "1.8.2", - "node-sass": "4.14.1", + "sass": "1.32.13", "patch-package": "6.2.2", "sass-loader": "7.3.1", "svg-sprite-loader": "5.0.0", diff --git a/web/public/config.js b/web/public/config.js index e6fcce151..3bffa88f0 100644 --- a/web/public/config.js +++ b/web/public/config.js @@ -1,8 +1,8 @@ /** * 知识库URL - * kn.dss.com + * kn.dss.com */ -window.knowledgeLibraryUrl = "http://kn.dss.com/" +window.knowledgeLibraryUrl = "http://127.0.0.1:80" // 是否允许上传、配置、arge参数编辑 true 测试 false 生产 window.enableUpload = true // 水印中region字段的设置 diff --git a/web/src/apps/scriptis/module/workbench/container.vue b/web/src/apps/scriptis/module/workbench/container.vue index f9de3ed6e..e9595e817 100644 --- a/web/src/apps/scriptis/module/workbench/container.vue +++ b/web/src/apps/scriptis/module/workbench/container.vue @@ -1079,7 +1079,7 @@ export default {