diff --git a/iotdb-api/external-service-api/pom.xml b/iotdb-api/external-service-api/pom.xml new file mode 100644 index 0000000000000..adbb6e6ffb001 --- /dev/null +++ b/iotdb-api/external-service-api/pom.xml @@ -0,0 +1,60 @@ + + + + 4.0.0 + + org.apache.iotdb + iotdb-api + 2.0.7-SNAPSHOT + + external-service-api + IoTDB: API: External Service API + + + get-jar-with-dependencies + + + + org.apache.maven.plugins + maven-assembly-plugin + + + jar-with-dependencies + + + + + make-assembly + + + single + + + package + + + + + + + + diff --git a/iotdb-api/external-service-api/src/main/java/org/apache/iotdb/externalservice/api/IExternalService.java b/iotdb-api/external-service-api/src/main/java/org/apache/iotdb/externalservice/api/IExternalService.java new file mode 100644 index 0000000000000..1ae5f30e7a9d3 --- /dev/null +++ b/iotdb-api/external-service-api/src/main/java/org/apache/iotdb/externalservice/api/IExternalService.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.externalservice.api; + +/** An interface to support user-defined service. */ +public interface IExternalService { + + /** Start current service. */ + void start(); + + /** + * Stop current service. If current service uses thread or thread pool, current service should + * guarantee to putBack thread or thread pool. + */ + void stop(); +} diff --git a/iotdb-api/pom.xml b/iotdb-api/pom.xml index 56c3efcd6f086..d724f9d11a973 100644 --- a/iotdb-api/pom.xml +++ b/iotdb-api/pom.xml @@ -31,6 +31,7 @@ IoTDB: API external-api + external-service-api pipe-api trigger-api udf-api diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java index bb533ddbbd578..f16e1526ece33 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java @@ -338,6 +338,11 @@ public enum TSStatusCode { RATIS_READ_UNAVAILABLE(2207), PIPE_CONSENSUS_CLOSE_ERROR(2208), PIPE_CONSENSUS_WAIT_ORDER_TIMEOUT(2209), + + // ExternalService + NO_SUCH_EXTERNAL_SERVICE(2300), + EXTERNAL_SERVICE_ALREADY_EXIST(2301), + GET_BUILTIN_EXTERNAL_SERVICE_ERROR(2302), ; private final int statusCode; diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 index 4a01b352384c2..70f9fced2b364 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 @@ -115,6 +115,7 @@ keyWords | FIRST | FLUSH | FOR + | FORCEDLY | FROM | FULL | FUNCTION @@ -214,6 +215,8 @@ keyWords | SECURITY | SELECT | SERIESSLOTID + | SERVICE + | SERVICES | SESSION | SET | SETTLE diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 index 0d86a02a1cfcb..efe661e05430c 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 @@ -52,6 +52,8 @@ ddlStatement | createFunction | dropFunction | showFunctions // Trigger | createTrigger | dropTrigger | showTriggers | startTrigger | stopTrigger + // ExternalService + | createService | startService | stopService | dropService | showService // Pipe Task | createPipe | alterPipe | dropPipe | startPipe | stopPipe | showPipes // Pipe Plugin @@ -437,6 +439,28 @@ stopTrigger : STOP TRIGGER triggerName=identifier ; +// ExternalService ========================================================================================= +createService + : CREATE SERVICE serviceName=identifier + AS className=STRING_LITERAL + ; + +startService + : START SERVICE serviceName=identifier + ; + +stopService + : STOP SERVICE serviceName=identifier + ; + +dropService + : DROP SERVICE serviceName=identifier FORCEDLY? + + ; + +showService + : SHOW SERVICES (ON targetDataNodeId=INTEGER_LITERAL)? + ; // CQ ============================================================================================== // ---- Create Continuous Query diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 index 85e039b1e5d86..4ff6f0dc12906 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 @@ -382,6 +382,11 @@ FOR : F O R ; +FORCEDLY + : F O R C E D L Y + ; + + FROM : F R O M ; @@ -786,6 +791,14 @@ SERIESSLOTID : S E R I E S S L O T I D ; +SERVICE + : S E R V I C E + ; + +SERVICES + : S E R V I C E S + ; + SESSION : S E S S I O N ; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java index c2f8b1e9d13c7..e5753bf1bd184 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java @@ -60,6 +60,9 @@ public enum CnToDnAsyncRequestType { INACTIVE_TRIGGER_INSTANCE, UPDATE_TRIGGER_LOCATION, + // ExternalService + GET_BUILTIN_SERVICE, + // Pipe Plugin CREATE_PIPE_PLUGIN, DROP_PIPE_PLUGIN, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java index 9227325596d6b..cd69f8b2c846d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java @@ -38,6 +38,7 @@ import org.apache.iotdb.confignode.client.async.handlers.rpc.DataNodeAsyncRequestRPCHandler; import org.apache.iotdb.confignode.client.async.handlers.rpc.DataNodeTSStatusRPCHandler; import org.apache.iotdb.confignode.client.async.handlers.rpc.FetchSchemaBlackListRPCHandler; +import org.apache.iotdb.confignode.client.async.handlers.rpc.GetBuiltInExternalServiceRPCHandler; import org.apache.iotdb.confignode.client.async.handlers.rpc.PipeHeartbeatRPCHandler; import org.apache.iotdb.confignode.client.async.handlers.rpc.PipePushMetaRPCHandler; import org.apache.iotdb.confignode.client.async.handlers.rpc.SchemaUpdateRPCHandler; @@ -486,6 +487,10 @@ protected void initActionMapBuilder() { CnToDnAsyncRequestType.ENABLE_SEPARATION_OF_ADMIN_POWERS, (req, client, handler) -> client.enableSeparationOfAdminPower((DataNodeTSStatusRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.GET_BUILTIN_SERVICE, + (req, client, handler) -> + client.getBuiltInService((GetBuiltInExternalServiceRPCHandler) handler)); } @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java index 4e3fdb09f7ff0..b2e2ec3232781 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java @@ -20,6 +20,7 @@ package org.apache.iotdb.confignode.client.async.handlers.rpc; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TExternalServiceListResp; import org.apache.iotdb.common.rpc.thrift.TPipeHeartbeatResp; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.common.rpc.thrift.TTestConnectionResp; @@ -193,6 +194,14 @@ public static DataNodeAsyncRequestRPCHandler buildHandler( dataNodeLocationMap, (Map) responseMap, countDownLatch); + case GET_BUILTIN_SERVICE: + return new GetBuiltInExternalServiceRPCHandler( + requestType, + requestId, + targetDataNode, + dataNodeLocationMap, + (Map) responseMap, + countDownLatch); case SET_TTL: case CREATE_DATA_REGION: case CREATE_SCHEMA_REGION: diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/GetBuiltInExternalServiceRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/GetBuiltInExternalServiceRPCHandler.java new file mode 100644 index 0000000000000..38b34fe6d4938 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/GetBuiltInExternalServiceRPCHandler.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.client.async.handlers.rpc; + +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TExternalServiceListResp; +import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.concurrent.CountDownLatch; + +public class GetBuiltInExternalServiceRPCHandler + extends DataNodeAsyncRequestRPCHandler { + private static final Logger LOGGER = + LoggerFactory.getLogger(GetBuiltInExternalServiceRPCHandler.class); + + public GetBuiltInExternalServiceRPCHandler( + CnToDnAsyncRequestType requestType, + int requestId, + TDataNodeLocation targetDataNode, + Map dataNodeLocationMap, + Map responseMap, + CountDownLatch countDownLatch) { + super(requestType, requestId, targetDataNode, dataNodeLocationMap, responseMap, countDownLatch); + } + + @Override + public void onComplete(TExternalServiceListResp response) { + // Put response only when success + if (response.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + responseMap.put(requestId, response); + } else { + LOGGER.error( + "Failed to {} on DataNode: {}, response: {}", + requestType, + formattedTargetLocation, + response); + } + + // Always remove to avoid retrying + nodeLocationMap.remove(requestId); + + // Always CountDown + countDownLatch.countDown(); + } + + @Override + public void onError(Exception e) { + String errorMsg = + "Failed to " + + requestType + + " on DataNode: " + + formattedTargetLocation + + ", exception: " + + e.getMessage(); + LOGGER.error(errorMsg, e); + + // Always CountDown + countDownLatch.countDown(); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java index d1bf4c45ced54..7fd7cd029119a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java @@ -46,6 +46,10 @@ import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan; import org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNodePlan; import org.apache.iotdb.confignode.consensus.request.write.datanode.UpdateDataNodePlan; +import org.apache.iotdb.confignode.consensus.request.write.externalservice.CreateExternalServicePlan; +import org.apache.iotdb.confignode.consensus.request.write.externalservice.DropExternalServicePlan; +import org.apache.iotdb.confignode.consensus.request.write.externalservice.StartExternalServicePlan; +import org.apache.iotdb.confignode.consensus.request.write.externalservice.StopExternalServicePlan; import org.apache.iotdb.confignode.consensus.request.write.function.CreateFunctionPlan; import org.apache.iotdb.confignode.consensus.request.write.function.DropTableModelFunctionPlan; import org.apache.iotdb.confignode.consensus.request.write.function.DropTreeModelFunctionPlan; @@ -588,6 +592,18 @@ public static ConfigPhysicalPlan create(final ByteBuffer buffer) throws IOExcept case setThrottleQuota: plan = new SetThrottleQuotaPlan(); break; + case CreateExternalService: + plan = new CreateExternalServicePlan(); + break; + case StartExternalService: + plan = new StartExternalServicePlan(); + break; + case StopExternalService: + plan = new StopExternalServicePlan(); + break; + case DropExternalService: + plan = new DropExternalServicePlan(); + break; default: throw new IOException("unknown PhysicalPlan configPhysicalPlanType: " + planType); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java index 360dd83dd70ce..ecabc2331b2e9 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java @@ -335,6 +335,12 @@ public enum ConfigPhysicalPlanType { EnableSeparationOfAdminPowers((short) 2200), + CreateExternalService((short) 2301), + StartExternalService((short) 2302), + StopExternalService((short) 2303), + DropExternalService((short) 2304), + ShowExternalService((short) 2305), + /** Test Only. */ TestOnly((short) 30000), ; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/exernalservice/ShowExternalServicePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/exernalservice/ShowExternalServicePlan.java new file mode 100644 index 0000000000000..4e013e3ce7bcd --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/exernalservice/ShowExternalServicePlan.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.read.exernalservice; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; + +import java.util.Objects; +import java.util.Set; + +/** Get infos of ExternalService by the DataNode's id. */ +public class ShowExternalServicePlan extends ConfigPhysicalReadPlan { + + private final Set dataNodeIds; + + public ShowExternalServicePlan(Set dataNodeIds) { + super(ConfigPhysicalPlanType.ShowExternalService); + this.dataNodeIds = dataNodeIds; + } + + public Set getDataNodeIds() { + return dataNodeIds; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final ShowExternalServicePlan that = (ShowExternalServicePlan) o; + return dataNodeIds.equals(that.dataNodeIds); + } + + @Override + public int hashCode() { + return Objects.hash(dataNodeIds); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/externalservice/CreateExternalServicePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/externalservice/CreateExternalServicePlan.java new file mode 100644 index 0000000000000..ae65fc05b941d --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/externalservice/CreateExternalServicePlan.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.externalservice; + +import org.apache.iotdb.commons.externalservice.ServiceInfo; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class CreateExternalServicePlan extends ConfigPhysicalPlan { + + private int datanodeId; + private ServiceInfo serviceInfo; + + public CreateExternalServicePlan() { + super(ConfigPhysicalPlanType.CreateExternalService); + } + + public CreateExternalServicePlan(int datanodeId, ServiceInfo serviceInfo) { + super(ConfigPhysicalPlanType.CreateExternalService); + this.datanodeId = datanodeId; + this.serviceInfo = serviceInfo; + } + + public int getDatanodeId() { + return datanodeId; + } + + public ServiceInfo getServiceInfo() { + return serviceInfo; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + + ReadWriteIOUtils.write(datanodeId, stream); + serviceInfo.serialize(stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + datanodeId = ReadWriteIOUtils.readInt(buffer); + serviceInfo = ServiceInfo.deserialize(buffer); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + CreateExternalServicePlan that = (CreateExternalServicePlan) o; + return datanodeId == that.datanodeId && Objects.equals(serviceInfo, that.serviceInfo); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), datanodeId, serviceInfo); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/externalservice/DropExternalServicePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/externalservice/DropExternalServicePlan.java new file mode 100644 index 0000000000000..a25619a9fa08c --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/externalservice/DropExternalServicePlan.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.externalservice; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class DropExternalServicePlan extends ConfigPhysicalPlan { + + private int dataNodeId; + private String serviceName; + + public DropExternalServicePlan() { + super(ConfigPhysicalPlanType.DropExternalService); + } + + public DropExternalServicePlan(int dataNodeId, String serviceName) { + super(ConfigPhysicalPlanType.DropExternalService); + this.dataNodeId = dataNodeId; + this.serviceName = serviceName; + } + + public int getDataNodeId() { + return dataNodeId; + } + + public String getServiceName() { + return serviceName; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + ReadWriteIOUtils.write(dataNodeId, stream); + ReadWriteIOUtils.write(serviceName, stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + dataNodeId = ReadWriteIOUtils.readInt(buffer); + serviceName = ReadWriteIOUtils.readString(buffer); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + DropExternalServicePlan that = (DropExternalServicePlan) o; + return dataNodeId == that.dataNodeId && Objects.equals(serviceName, that.serviceName); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), dataNodeId, serviceName); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/externalservice/StartExternalServicePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/externalservice/StartExternalServicePlan.java new file mode 100644 index 0000000000000..67fd0aa3d450c --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/externalservice/StartExternalServicePlan.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.externalservice; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class StartExternalServicePlan extends ConfigPhysicalPlan { + + private int dataNodeId; + private String serviceName; + + public StartExternalServicePlan() { + super(ConfigPhysicalPlanType.StartExternalService); + } + + public StartExternalServicePlan(int dataNodeId, String serviceName) { + super(ConfigPhysicalPlanType.StartExternalService); + this.dataNodeId = dataNodeId; + this.serviceName = serviceName; + } + + public int getDataNodeId() { + return dataNodeId; + } + + public String getServiceName() { + return serviceName; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + ReadWriteIOUtils.write(dataNodeId, stream); + ReadWriteIOUtils.write(serviceName, stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + dataNodeId = ReadWriteIOUtils.readInt(buffer); + serviceName = ReadWriteIOUtils.readString(buffer); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + StartExternalServicePlan that = (StartExternalServicePlan) o; + return dataNodeId == that.dataNodeId && Objects.equals(serviceName, that.serviceName); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), dataNodeId, serviceName); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/externalservice/StopExternalServicePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/externalservice/StopExternalServicePlan.java new file mode 100644 index 0000000000000..83fcbe5c67d62 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/externalservice/StopExternalServicePlan.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.externalservice; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class StopExternalServicePlan extends ConfigPhysicalPlan { + + private int dataNodeId; + private String serviceName; + + public StopExternalServicePlan() { + super(ConfigPhysicalPlanType.StopExternalService); + } + + public StopExternalServicePlan(int dataNodeId, String serviceName) { + super(ConfigPhysicalPlanType.StopExternalService); + this.dataNodeId = dataNodeId; + this.serviceName = serviceName; + } + + public int getDataNodeId() { + return dataNodeId; + } + + public String getServiceName() { + return serviceName; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + ReadWriteIOUtils.write(dataNodeId, stream); + ReadWriteIOUtils.write(serviceName, stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + dataNodeId = ReadWriteIOUtils.readInt(buffer); + serviceName = ReadWriteIOUtils.readString(buffer); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + StopExternalServicePlan that = (StopExternalServicePlan) o; + return dataNodeId == that.dataNodeId && Objects.equals(serviceName, that.serviceName); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), dataNodeId, serviceName); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/externalservice/ShowExternalServiceResp.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/externalservice/ShowExternalServiceResp.java new file mode 100644 index 0000000000000..217b1e890a2cf --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/externalservice/ShowExternalServiceResp.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.response.externalservice; + +import org.apache.iotdb.common.rpc.thrift.TExternalServiceEntry; +import org.apache.iotdb.common.rpc.thrift.TExternalServiceListResp; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.consensus.common.DataSet; +import org.apache.iotdb.rpc.TSStatusCode; + +import javax.validation.constraints.NotNull; + +import java.util.Comparator; +import java.util.List; + +public class ShowExternalServiceResp implements DataSet { + + private final List serviceInfoEntryList; + + public ShowExternalServiceResp(@NotNull List serviceInfoEntryList) { + this.serviceInfoEntryList = serviceInfoEntryList; + } + + public List getServiceInfoEntryList() { + return serviceInfoEntryList; + } + + public TExternalServiceListResp convertToRpcShowExternalServiceResp() { + serviceInfoEntryList.sort( + Comparator.comparingInt(TExternalServiceEntry::getDataNodId) + .thenComparing(TExternalServiceEntry::getServiceType) + .thenComparing(TExternalServiceEntry::getServiceName)); + return new TExternalServiceListResp( + new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()), serviceInfoEntryList); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 3c3d230419504..0188715094f7d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -25,6 +25,7 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TExternalServiceListResp; import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TPipeHeartbeatResp; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; @@ -109,6 +110,8 @@ import org.apache.iotdb.confignode.consensus.statemachine.ConfigRegionStateMachine; import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; import org.apache.iotdb.confignode.manager.cq.CQManager; +import org.apache.iotdb.confignode.manager.externalservice.ExternalServiceInfo; +import org.apache.iotdb.confignode.manager.externalservice.ExternalServiceManager; import org.apache.iotdb.confignode.manager.load.LoadManager; import org.apache.iotdb.confignode.manager.load.cache.node.NodeHeartbeatSample; import org.apache.iotdb.confignode.manager.node.ClusterNodeStartUtils; @@ -154,6 +157,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListResp; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateConsumerReq; +import org.apache.iotdb.confignode.rpc.thrift.TCreateExternalServiceReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateFunctionReq; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipePluginReq; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; @@ -315,6 +319,9 @@ public class ConfigManager implements IManager { /** Manage procedure. */ private final ProcedureManager procedureManager; + /** ExternalService. */ + private final ExternalServiceManager externalServiceManager; + /** UDF. */ private final UDFManager udfManager; @@ -354,6 +361,7 @@ public ConfigManager() throws IOException { UDFInfo udfInfo = new UDFInfo(); TriggerInfo triggerInfo = new TriggerInfo(); CQInfo cqInfo = new CQInfo(); + ExternalServiceInfo externalServiceInfo = new ExternalServiceInfo(); PipeInfo pipeInfo = new PipeInfo(); QuotaInfo quotaInfo = new QuotaInfo(); TTLInfo ttlInfo = new TTLInfo(); @@ -371,6 +379,7 @@ public ConfigManager() throws IOException { udfInfo, triggerInfo, cqInfo, + externalServiceInfo, pipeInfo, subscriptionInfo, quotaInfo, @@ -389,6 +398,7 @@ public ConfigManager() throws IOException { this.partitionManager = new PartitionManager(this, partitionInfo); this.permissionManager = createPermissionManager(authorInfo); this.procedureManager = createProcedureManager(procedureInfo); + this.externalServiceManager = new ExternalServiceManager(this); this.udfManager = new UDFManager(this, udfInfo); this.triggerManager = new TriggerManager(this, triggerInfo); this.cqManager = new CQManager(this); @@ -1851,6 +1861,15 @@ public TGetDataNodeLocationsResp getReadableDataNodeLocations() { : new TGetDataNodeLocationsResp(status, Collections.emptyList()); } + public Map getReadableDataNodeLocationMap() { + TSStatus status = confirmLeader(); + return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? nodeManager.filterDataNodeThroughStatus(NodeStatus::isReadable).stream() + .map(TDataNodeConfiguration::getLocation) + .collect(Collectors.toMap(TDataNodeLocation::getDataNodeId, location -> location)) + : Collections.emptyMap(); + } + @Override public TRegionRouteMapResp getLatestRegionRouteMap() { final long retryIntervalInMS = 100; @@ -2600,6 +2619,46 @@ public TShowCQResp showCQ() { : new TShowCQResp(status, Collections.emptyList()); } + @Override + public TSStatus createExternalService(TCreateExternalServiceReq req) { + TSStatus status = confirmLeader(); + return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? externalServiceManager.createService(req) + : status; + } + + @Override + public TSStatus startExternalService(int dataNodeId, String serviceName) { + TSStatus status = confirmLeader(); + return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? externalServiceManager.startService(dataNodeId, serviceName) + : status; + } + + @Override + public TSStatus stopExternalService(int dataNodeId, String serviceName) { + TSStatus status = confirmLeader(); + return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? externalServiceManager.stopService(dataNodeId, serviceName) + : status; + } + + @Override + public TSStatus dropExternalService(int dataNodeId, String serviceName) { + TSStatus status = confirmLeader(); + return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? externalServiceManager.dropService(dataNodeId, serviceName) + : status; + } + + @Override + public TExternalServiceListResp showExternalService(int dataNodeId) { + TSStatus status = confirmLeader(); + return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? externalServiceManager.showService(dataNodeId) + : new TExternalServiceListResp(status, Collections.emptyList()); + } + /** * Get all related schemaRegion which may contains the timeseries matched by given patternTree. */ diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java index caa189be81572..f083fd9b966c1 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TExternalServiceListResp; import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TPipeHeartbeatResp; import org.apache.iotdb.common.rpc.thrift.TSStatus; @@ -75,6 +76,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListResp; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateConsumerReq; +import org.apache.iotdb.confignode.rpc.thrift.TCreateExternalServiceReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateFunctionReq; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipePluginReq; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; @@ -862,6 +864,16 @@ TDataPartitionTableResp getOrCreateDataPartition( TShowCQResp showCQ(); + TSStatus createExternalService(TCreateExternalServiceReq req); + + TSStatus startExternalService(int dataNodeId, String serviceName); + + TSStatus stopExternalService(int dataNodeId, String serviceName); + + TSStatus dropExternalService(int dataNodeId, String serviceName); + + TExternalServiceListResp showExternalService(int dataNodeId); + TSStatus checkConfigNodeGlobalConfig(TConfigNodeRegisterReq req); TSStatus transfer(List newUnknownDataList); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/externalservice/ExternalServiceInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/externalservice/ExternalServiceInfo.java new file mode 100644 index 0000000000000..28e321abf0441 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/externalservice/ExternalServiceInfo.java @@ -0,0 +1,314 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager.externalservice; + +import org.apache.iotdb.common.rpc.thrift.TExternalServiceEntry; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.externalservice.ServiceInfo; +import org.apache.iotdb.commons.snapshot.SnapshotProcessor; +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.confignode.consensus.request.write.externalservice.CreateExternalServicePlan; +import org.apache.iotdb.confignode.consensus.request.write.externalservice.DropExternalServicePlan; +import org.apache.iotdb.confignode.consensus.request.write.externalservice.StartExternalServicePlan; +import org.apache.iotdb.confignode.consensus.request.write.externalservice.StopExternalServicePlan; +import org.apache.iotdb.confignode.consensus.response.externalservice.ShowExternalServiceResp; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; +import java.util.zip.CRC32; + +import static com.google.common.base.Preconditions.checkState; + +public class ExternalServiceInfo implements SnapshotProcessor { + + private static final Logger LOGGER = LoggerFactory.getLogger(ExternalServiceInfo.class); + + private final Map> datanodeToServiceInfos; + + private static final String SNAPSHOT_FILENAME = "service_info.bin"; + private static final int SERIALIZATION_VERSION = 1; + private final CRC32 crc32 = new CRC32(); + + public ExternalServiceInfo() { + datanodeToServiceInfos = new ConcurrentHashMap<>(); + } + + /** + * Add a new ExternalService on target DataNode. + * + * @return SUCCESS_STATUS if this service was not existed on target DataNode, otherwise + * EXTERNAL_SERVICE_AlREADY_EXIST + */ + public TSStatus addService(CreateExternalServicePlan plan) { + TSStatus res = new TSStatus(); + Map serviceInfos = + datanodeToServiceInfos.computeIfAbsent( + plan.getDatanodeId(), k -> new ConcurrentHashMap<>()); + String serviceName = plan.getServiceInfo().getServiceName(); + if (serviceInfos.containsKey(serviceName)) { + res.code = TSStatusCode.EXTERNAL_SERVICE_ALREADY_EXIST.getStatusCode(); + res.message = + String.format( + "ExternalService %s has already been created on DataNode %s.", + serviceName, plan.getDatanodeId()); + } else { + serviceInfos.put(serviceName, plan.getServiceInfo()); + res.code = TSStatusCode.SUCCESS_STATUS.getStatusCode(); + } + return res; + } + + /** + * Drop the ExternalService whose name is same as serviceName in plan. + * + * @return SUCCESS_STATUS if this service was existed on target DataNode, otherwise + * NO_SUCH_EXTERNAL_SERVICE + */ + public TSStatus dropService(DropExternalServicePlan plan) { + TSStatus res = new TSStatus(); + Map serviceInfos = + datanodeToServiceInfos.computeIfAbsent( + plan.getDataNodeId(), k -> new ConcurrentHashMap<>()); + String serviceName = plan.getServiceName(); + if (!serviceInfos.containsKey(serviceName)) { + res.code = TSStatusCode.NO_SUCH_EXTERNAL_SERVICE.getStatusCode(); + res.message = + String.format( + "ExternalService %s is not existed on DataNode %s.", + serviceName, plan.getDataNodeId()); + } else { + serviceInfos.remove(serviceName); + res.code = TSStatusCode.SUCCESS_STATUS.getStatusCode(); + } + return res; + } + + /** + * Start the ExternalService whose name is same as serviceName in plan. + * + * @return SUCCESS_STATUS if this service was existed on target DataNode, otherwise + * NO_SUCH_EXTERNAL_SERVICE + */ + public TSStatus startService(StartExternalServicePlan plan) { + TSStatus res = new TSStatus(); + Map serviceInfos = + datanodeToServiceInfos.computeIfAbsent( + plan.getDataNodeId(), k -> new ConcurrentHashMap<>()); + String serviceName = plan.getServiceName(); + if (!serviceInfos.containsKey(serviceName)) { + res.code = TSStatusCode.NO_SUCH_EXTERNAL_SERVICE.getStatusCode(); + res.message = + String.format( + "ExternalService %s is not existed on DataNode %s.", + serviceName, plan.getDataNodeId()); + } else { + ServiceInfo serviceInfo = serviceInfos.get(serviceName); + // The WRITE operations of StateMachine are not concurrent + checkState(serviceInfo != null, "Target serviceInfo should not be null."); + serviceInfo.setState(ServiceInfo.State.RUNNING); + res.code = TSStatusCode.SUCCESS_STATUS.getStatusCode(); + } + return res; + } + + /** + * Stop the ExternalService whose name is same as serviceName in plan. + * + * @return SUCCESS_STATUS if this service was existed on target DataNode, otherwise + * NO_SUCH_EXTERNAL_SERVICE + */ + public TSStatus stopService(StopExternalServicePlan plan) { + TSStatus res = new TSStatus(); + Map serviceInfos = + datanodeToServiceInfos.computeIfAbsent( + plan.getDataNodeId(), k -> new ConcurrentHashMap<>()); + String serviceName = plan.getServiceName(); + if (!serviceInfos.containsKey(serviceName)) { + res.code = TSStatusCode.NO_SUCH_EXTERNAL_SERVICE.getStatusCode(); + res.message = + String.format( + "ExternalService %s is not existed on DataNode %s.", + serviceName, plan.getDataNodeId()); + } else { + ServiceInfo serviceInfo = serviceInfos.get(serviceName); + // The WRITE operations of StateMachine are not concurrent + checkState(serviceInfo != null, "Target serviceInfo should not be null."); + serviceInfo.setState(ServiceInfo.State.STOPPED); + res.code = TSStatusCode.SUCCESS_STATUS.getStatusCode(); + } + return res; + } + + public ShowExternalServiceResp showService(Set dataNodes) { + return new ShowExternalServiceResp( + datanodeToServiceInfos.entrySet().stream() + .filter(entry -> dataNodes.contains(entry.getKey())) + .flatMap( + entry -> + entry.getValue().values().stream() + .map( + serviceInfo -> + new TExternalServiceEntry( + serviceInfo.getServiceName(), + serviceInfo.getClassName(), + serviceInfo.getState().getValue(), + entry.getKey(), + ServiceInfo.ServiceType.USER_DEFINED.getValue()))) + .collect(Collectors.toList())); + } + + private void serializeInfos(OutputStream outputStream) throws IOException { + ReadWriteIOUtils.write(SERIALIZATION_VERSION, outputStream); + ReadWriteIOUtils.write(datanodeToServiceInfos.size(), outputStream); + for (Map.Entry> outerEntry : + datanodeToServiceInfos.entrySet()) { + ReadWriteIOUtils.write(outerEntry.getKey(), outputStream); // DataNode ID + + Map innerMap = outerEntry.getValue(); + // inner Map + ReadWriteIOUtils.write(innerMap.size(), outputStream); + for (ServiceInfo innerEntry : innerMap.values()) { + serializeServiceInfoWithCRC(innerEntry, outputStream); + } + } + } + + private void serializeServiceInfoWithCRC(ServiceInfo serviceInfo, OutputStream outputStream) + throws IOException { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + DataOutputStream tempDos = new DataOutputStream(byteArrayOutputStream); + serviceInfo.serialize(tempDos); + tempDos.flush(); + byte[] bytes = byteArrayOutputStream.toByteArray(); + + crc32.reset(); + crc32.update(bytes, 0, bytes.length); + + ReadWriteIOUtils.write(bytes.length, outputStream); + outputStream.write(bytes); + ReadWriteIOUtils.write(crc32.getValue(), outputStream); + } + + private void deserializeInfos(InputStream inputStream) throws IOException { + if (ReadWriteIOUtils.readInt(inputStream) != SERIALIZATION_VERSION) { + throw new IOException("Incorrect version of " + SNAPSHOT_FILENAME); + } + + int outerSize = ReadWriteIOUtils.readInt(inputStream); + for (int i = 0; i < outerSize; i++) { + int dataNodeId = ReadWriteIOUtils.readInt(inputStream); + int innerSize = ReadWriteIOUtils.readInt(inputStream); + + Map innerMap = + datanodeToServiceInfos.computeIfAbsent( + dataNodeId, k -> new ConcurrentHashMap<>(innerSize)); + for (int j = 0; j < innerSize; j++) { + ServiceInfo value = deserializeServiceInfoConsiderCRC(inputStream); + if (value != null) { + innerMap.put(value.getServiceName(), value); + } + } + datanodeToServiceInfos.put(dataNodeId, innerMap); + } + } + + private ServiceInfo deserializeServiceInfoConsiderCRC(InputStream inputStream) + throws IOException { + int length = ReadWriteIOUtils.readInt(inputStream); + byte[] bytes = new byte[length]; + inputStream.read(bytes); + + crc32.reset(); + crc32.update(bytes, 0, length); + + long expectedCRC = ReadWriteIOUtils.readLong(inputStream); + if (crc32.getValue() != expectedCRC) { + LOGGER.error("Mismatched CRC32 code when deserializing service info."); + return null; + } + + return ServiceInfo.deserialize(ByteBuffer.wrap(bytes)); + } + + @Override + public boolean processTakeSnapshot(File snapshotDir) throws IOException { + File snapshotFile = new File(snapshotDir, SNAPSHOT_FILENAME); + if (snapshotFile.exists() && snapshotFile.isFile()) { + LOGGER.error( + "Failed to take snapshot, because snapshot file [{}] is already exist.", + snapshotFile.getAbsolutePath()); + return false; + } + + try (FileOutputStream fileOutputStream = new FileOutputStream(snapshotFile)) { + + serializeInfos(fileOutputStream); + + // fsync + fileOutputStream.getFD().sync(); + + return true; + } + } + + @Override + public void processLoadSnapshot(File snapshotDir) throws IOException { + File snapshotFile = new File(snapshotDir, SNAPSHOT_FILENAME); + if (!snapshotFile.exists() || !snapshotFile.isFile()) { + LOGGER.error( + "Failed to load snapshot,snapshot file [{}] is not exist.", + snapshotFile.getAbsolutePath()); + return; + } + + try (FileInputStream fileInputStream = new FileInputStream(snapshotFile)) { + + clear(); + + deserializeInfos(fileInputStream); + } + } + + public void clear() { + datanodeToServiceInfos.values().forEach(subMap -> subMap.clear()); + datanodeToServiceInfos.clear(); + } + + @TestOnly + public Map> getRawDatanodeToServiceInfos() { + return datanodeToServiceInfos; + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/externalservice/ExternalServiceManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/externalservice/ExternalServiceManager.java new file mode 100644 index 0000000000000..344479608af20 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/externalservice/ExternalServiceManager.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager.externalservice; + +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TExternalServiceListResp; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.externalservice.ServiceInfo; +import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; +import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager; +import org.apache.iotdb.confignode.client.async.handlers.DataNodeAsyncRequestContext; +import org.apache.iotdb.confignode.consensus.request.read.exernalservice.ShowExternalServicePlan; +import org.apache.iotdb.confignode.consensus.request.write.externalservice.CreateExternalServicePlan; +import org.apache.iotdb.confignode.consensus.request.write.externalservice.DropExternalServicePlan; +import org.apache.iotdb.confignode.consensus.request.write.externalservice.StartExternalServicePlan; +import org.apache.iotdb.confignode.consensus.request.write.externalservice.StopExternalServicePlan; +import org.apache.iotdb.confignode.consensus.response.externalservice.ShowExternalServiceResp; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.rpc.thrift.TCreateExternalServiceReq; +import org.apache.iotdb.consensus.exception.ConsensusException; +import org.apache.iotdb.mpp.rpc.thrift.TCreateFunctionInstanceReq; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.Map; + +public class ExternalServiceManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(ExternalServiceManager.class); + + private final ConfigManager configManager; + + public ExternalServiceManager(ConfigManager configManager) { + this.configManager = configManager; + } + + public TSStatus createService(TCreateExternalServiceReq req) { + try { + return configManager + .getConsensusManager() + .write( + new CreateExternalServicePlan( + req.getDataNodeId(), + new ServiceInfo( + req.getServiceName(), + req.getClassName(), + ServiceInfo.ServiceType.USER_DEFINED))); + } catch (ConsensusException e) { + LOGGER.warn( + "Unexpected error happened while creating Service {} on DataNode {}: ", + req.getServiceName(), + req.getDataNodeId(), + e); + // consensus layer related errors + TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + res.setMessage(e.getMessage()); + return res; + } + } + + public TSStatus startService(int dataNodeId, String serviceName) { + try { + return configManager + .getConsensusManager() + .write(new StartExternalServicePlan(dataNodeId, serviceName)); + } catch (ConsensusException e) { + LOGGER.warn( + "Unexpected error happened while starting Service {} on DataNode {}: ", + serviceName, + dataNodeId, + e); + // consensus layer related errors + TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + res.setMessage(e.getMessage()); + return res; + } + } + + public TSStatus stopService(int dataNodeId, String serviceName) { + try { + return configManager + .getConsensusManager() + .write(new StopExternalServicePlan(dataNodeId, serviceName)); + } catch (ConsensusException e) { + LOGGER.warn( + "Unexpected error happened while stopping Service {} on DataNode {}: ", + serviceName, + dataNodeId, + e); + // consensus layer related errors + TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + res.setMessage(e.getMessage()); + return res; + } + } + + public TSStatus dropService(int dataNodeId, String serviceName) { + try { + return configManager + .getConsensusManager() + .write(new DropExternalServicePlan(dataNodeId, serviceName)); + } catch (ConsensusException e) { + LOGGER.warn( + "Unexpected error happened while dropping Service {} on DataNode {}: ", + serviceName, + dataNodeId, + e); + // consensus layer related errors + TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + res.setMessage(e.getMessage()); + return res; + } + } + + public TExternalServiceListResp showService(int dataNodeId) { + Map targetDataNodes = + configManager.getReadableDataNodeLocationMap(); + + if (targetDataNodes.isEmpty()) { + // no readable DN, return directly + return new TExternalServiceListResp( + new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()), Collections.emptyList()); + } + + if (dataNodeId != -1) { + if (!targetDataNodes.containsKey(dataNodeId)) { + // target DN is not readable, return directly + return new TExternalServiceListResp( + new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()), Collections.emptyList()); + } else { + targetDataNodes = Collections.singletonMap(dataNodeId, targetDataNodes.get(dataNodeId)); + } + } + + // 1. get built-in services info from DN + Map builtInServiceInfos = + getBuiltInServiceInfosFromDataNodes(targetDataNodes); + if (builtInServiceInfos.isEmpty()) { + return new TExternalServiceListResp( + new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()), Collections.emptyList()); + } + + try { + // 2. get user-defined services info from CN consensus + ShowExternalServiceResp response = + (ShowExternalServiceResp) + configManager + .getConsensusManager() + .read(new ShowExternalServicePlan(builtInServiceInfos.keySet())); + + // 3. combined built-in services info and user-defined services info + builtInServiceInfos + .values() + .forEach( + builtInResp -> + response.getServiceInfoEntryList().addAll(builtInResp.getExternalServiceInfos())); + return response.convertToRpcShowExternalServiceResp(); + } catch (ConsensusException e) { + LOGGER.warn("Unexpected error happened while showing Service: ", e); + // consensus layer related errors + TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + res.setMessage(e.getMessage()); + return new TExternalServiceListResp(res, Collections.emptyList()); + } + } + + private Map getBuiltInServiceInfosFromDataNodes( + Map dataNodeLocationMap) { + DataNodeAsyncRequestContext context = + new DataNodeAsyncRequestContext<>( + CnToDnAsyncRequestType.GET_BUILTIN_SERVICE, dataNodeLocationMap); + CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(context); + return context.getResponseMap(); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java index 5cbcfb85881da..12ba1d8840b49 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java @@ -33,6 +33,7 @@ import org.apache.iotdb.confignode.consensus.request.read.database.CountDatabasePlan; import org.apache.iotdb.confignode.consensus.request.read.database.GetDatabasePlan; import org.apache.iotdb.confignode.consensus.request.read.datanode.GetDataNodeConfigurationPlan; +import org.apache.iotdb.confignode.consensus.request.read.exernalservice.ShowExternalServicePlan; import org.apache.iotdb.confignode.consensus.request.read.function.GetFunctionTablePlan; import org.apache.iotdb.confignode.consensus.request.read.function.GetUDFJarPlan; import org.apache.iotdb.confignode.consensus.request.read.partition.CountTimeSlotListPlan; @@ -78,6 +79,10 @@ import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan; import org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNodePlan; import org.apache.iotdb.confignode.consensus.request.write.datanode.UpdateDataNodePlan; +import org.apache.iotdb.confignode.consensus.request.write.externalservice.CreateExternalServicePlan; +import org.apache.iotdb.confignode.consensus.request.write.externalservice.DropExternalServicePlan; +import org.apache.iotdb.confignode.consensus.request.write.externalservice.StartExternalServicePlan; +import org.apache.iotdb.confignode.consensus.request.write.externalservice.StopExternalServicePlan; import org.apache.iotdb.confignode.consensus.request.write.function.CreateFunctionPlan; import org.apache.iotdb.confignode.consensus.request.write.function.DropTableModelFunctionPlan; import org.apache.iotdb.confignode.consensus.request.write.function.DropTreeModelFunctionPlan; @@ -143,6 +148,7 @@ import org.apache.iotdb.confignode.consensus.request.write.trigger.UpdateTriggersOnTransferNodesPlan; import org.apache.iotdb.confignode.consensus.response.partition.SchemaNodeManagementResp; import org.apache.iotdb.confignode.exception.physical.UnknownPhysicalPlanTypeException; +import org.apache.iotdb.confignode.manager.externalservice.ExternalServiceInfo; import org.apache.iotdb.confignode.manager.pipe.agent.PipeConfigNodeAgent; import org.apache.iotdb.confignode.persistence.ClusterInfo; import org.apache.iotdb.confignode.persistence.ProcedureInfo; @@ -204,6 +210,8 @@ public class ConfigPlanExecutor { private final CQInfo cqInfo; + private final ExternalServiceInfo externalServiceInfo; + private final PipeInfo pipeInfo; private final SubscriptionInfo subscriptionInfo; @@ -222,6 +230,7 @@ public ConfigPlanExecutor( UDFInfo udfInfo, TriggerInfo triggerInfo, CQInfo cqInfo, + ExternalServiceInfo externalServiceInfo, PipeInfo pipeInfo, SubscriptionInfo subscriptionInfo, QuotaInfo quotaInfo, @@ -253,6 +262,9 @@ public ConfigPlanExecutor( this.cqInfo = cqInfo; this.snapshotProcessorList.add(cqInfo); + this.externalServiceInfo = externalServiceInfo; + this.snapshotProcessorList.add(externalServiceInfo); + this.pipeInfo = pipeInfo; this.snapshotProcessorList.add(pipeInfo); @@ -344,6 +356,8 @@ public DataSet executeQueryPlan(final ConfigPhysicalReadPlan req) return partitionInfo.getSeriesSlotList((GetSeriesSlotListPlan) req); case SHOW_CQ: return cqInfo.showCQ(); + case ShowExternalService: + return externalServiceInfo.showService(((ShowExternalServicePlan) req).getDataNodeIds()); case GetFunctionTable: return udfInfo.getUDFTable((GetFunctionTablePlan) req); case GetFunctionJar: @@ -635,6 +649,14 @@ public TSStatus executeNonQueryPlan(ConfigPhysicalPlan physicalPlan) return cqInfo.activeCQ((ActiveCQPlan) physicalPlan); case UPDATE_CQ_LAST_EXEC_TIME: return cqInfo.updateCQLastExecutionTime((UpdateCQLastExecTimePlan) physicalPlan); + case CreateExternalService: + return externalServiceInfo.addService((CreateExternalServicePlan) physicalPlan); + case StartExternalService: + return externalServiceInfo.startService((StartExternalServicePlan) physicalPlan); + case StopExternalService: + return externalServiceInfo.stopService((StopExternalServicePlan) physicalPlan); + case DropExternalService: + return externalServiceInfo.dropService((DropExternalServicePlan) physicalPlan); case CreatePipePlugin: return pipeInfo.getPipePluginInfo().createPipePlugin((CreatePipePluginPlan) physicalPlan); case DropPipePlugin: diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index afd399951da8d..26fed0e1c4a3a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TAINodeLocation; import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TExternalServiceListResp; import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TNodeLocations; import org.apache.iotdb.common.rpc.thrift.TPipeHeartbeatResp; @@ -115,6 +116,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListResp; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateConsumerReq; +import org.apache.iotdb.confignode.rpc.thrift.TCreateExternalServiceReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateFunctionReq; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipePluginReq; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; @@ -1363,6 +1365,31 @@ public TShowCQResp showCQ() { return configManager.showCQ(); } + @Override + public TSStatus createExternalService(TCreateExternalServiceReq req) { + return configManager.createExternalService(req); + } + + @Override + public TSStatus startExternalService(int dataNodeId, String serviceName) { + return configManager.startExternalService(dataNodeId, serviceName); + } + + @Override + public TSStatus stopExternalService(int dataNodeId, String serviceName) { + return configManager.stopExternalService(dataNodeId, serviceName); + } + + @Override + public TSStatus dropExternalService(int dataNodeId, String serviceName) { + return configManager.dropExternalService(dataNodeId, serviceName); + } + + @Override + public TExternalServiceListResp showExternalService(int dataNodeId) { + return configManager.showExternalService(dataNodeId); + } + @Override public TSStatus setSpaceQuota(final TSetSpaceQuotaReq req) throws TException { return configManager.setSpaceQuota(req); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/ExternalServiceInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/ExternalServiceInfoTest.java new file mode 100644 index 0000000000000..4dcf6e0047f79 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/ExternalServiceInfoTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.persistence; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.externalservice.ServiceInfo; +import org.apache.iotdb.confignode.consensus.request.write.externalservice.CreateExternalServicePlan; +import org.apache.iotdb.confignode.consensus.request.write.externalservice.StartExternalServicePlan; +import org.apache.iotdb.confignode.manager.externalservice.ExternalServiceInfo; + +import org.apache.thrift.TException; +import org.apache.tsfile.external.commons.io.FileUtils; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +import static org.apache.iotdb.db.utils.constant.TestConstant.BASE_OUTPUT_PATH; + +public class ExternalServiceInfoTest { + + private static ExternalServiceInfo serviceInfo; + private static ExternalServiceInfo serviceInfoBefore; + private static final File snapshotDir = new File(BASE_OUTPUT_PATH, "snapshot"); + + @BeforeClass + public static void setup() throws IOException { + serviceInfo = new ExternalServiceInfo(); + serviceInfoBefore = new ExternalServiceInfo(); + if (!snapshotDir.exists()) { + snapshotDir.mkdirs(); + } + } + + @AfterClass + public static void cleanup() throws IOException { + serviceInfo.clear(); + if (snapshotDir.exists()) { + FileUtils.deleteDirectory(snapshotDir); + } + } + + @Test + public void testSnapshot() throws TException, IOException, IllegalPathException { + CreateExternalServicePlan createExternalServicePlan = + new CreateExternalServicePlan( + 1, new ServiceInfo("TEST1", "testClassName", ServiceInfo.ServiceType.USER_DEFINED)); + serviceInfo.addService(createExternalServicePlan); + serviceInfoBefore.addService(createExternalServicePlan); + + createExternalServicePlan = + new CreateExternalServicePlan( + 2, new ServiceInfo("TEST1", "testClassName", ServiceInfo.ServiceType.USER_DEFINED)); + serviceInfo.addService(createExternalServicePlan); + serviceInfoBefore.addService(createExternalServicePlan); + + StartExternalServicePlan startExternalServicePlan = new StartExternalServicePlan(1, "TEST1"); + serviceInfo.startService(startExternalServicePlan); + serviceInfoBefore.startService(startExternalServicePlan); + + serviceInfo.processTakeSnapshot(snapshotDir); + serviceInfo.clear(); + serviceInfo.processLoadSnapshot(snapshotDir); + + Assert.assertEquals( + serviceInfo.getRawDatanodeToServiceInfos(), + serviceInfoBefore.getRawDatanodeToServiceInfos()); + } +} diff --git a/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-system.properties b/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-system.properties index b396e373f8699..508ab394aa346 100644 --- a/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-system.properties +++ b/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-system.properties @@ -33,8 +33,8 @@ cn_metric_prometheus_reporter_port=9091 timestamp_precision=ms data_region_consensus_protocol_class=org.apache.iotdb.consensus.iot.IoTConsensus schema_region_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus -schema_replication_factor=3 -data_replication_factor=3 +schema_replication_factor=1 +data_replication_factor=1 udf_lib_dir=target/confignode1/ext/udf trigger_lib_dir=target/confignode1/ext/trigger pipe_lib_dir=target/confignode1/ext/pipe diff --git a/iotdb-core/datanode/pom.xml b/iotdb-core/datanode/pom.xml index 85c96595381e9..72c4acba58615 100644 --- a/iotdb-core/datanode/pom.xml +++ b/iotdb-core/datanode/pom.xml @@ -94,6 +94,11 @@ iotdb-thrift-consensus 2.0.7-SNAPSHOT + + org.apache.iotdb + external-service-api + 2.0.7-SNAPSHOT + org.apache.iotdb udf-api diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 35d56ecc3c229..440dd52a9b6cb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -270,6 +270,10 @@ public class IoTDBConfig { private String triggerTemporaryLibDir = triggerDir + File.separator + IoTDBConstant.TMP_FOLDER_NAME; + /** External lib directory for UDF, stores user-uploaded JAR files */ + private String externalServiceDir = + IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.EXTERNAL_SERVICE_FOLDER_NAME; + /** External lib directory for Pipe Plugin, stores user-defined JAR files */ private String pipeDir = IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.PIPE_FOLDER_NAME; @@ -1364,6 +1368,7 @@ private void formulateFolders() { udfTemporaryLibDir = addDataHomeDir(udfTemporaryLibDir); triggerDir = addDataHomeDir(triggerDir); triggerTemporaryLibDir = addDataHomeDir(triggerTemporaryLibDir); + externalServiceDir = addDataHomeDir(externalServiceDir); pipeDir = addDataHomeDir(pipeDir); pipeTemporaryLibDir = addDataHomeDir(pipeTemporaryLibDir); for (int i = 0; i < pipeReceiverFileDirs.length; i++) { @@ -1685,6 +1690,10 @@ public void updateTriggerTemporaryLibDir() { this.triggerTemporaryLibDir = triggerDir + File.separator + IoTDBConstant.TMP_FOLDER_NAME; } + public String getExternalServiceDir() { + return externalServiceDir; + } + public String getPipeLibDir() { return pipeDir; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index 114629c0ed709..e2c04caedfb20 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TExternalServiceListResp; import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TNodeLocations; import org.apache.iotdb.common.rpc.thrift.TPipeHeartbeatResp; @@ -73,6 +74,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListResp; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateConsumerReq; +import org.apache.iotdb.confignode.rpc.thrift.TCreateExternalServiceReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateFunctionReq; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipePluginReq; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; @@ -1342,6 +1344,39 @@ public TShowCQResp showCQ() throws TException { () -> client.showCQ(), resp -> !updateConfigNodeLeader(resp.status)); } + @Override + public TSStatus createExternalService(TCreateExternalServiceReq req) throws TException { + return executeRemoteCallWithRetry( + () -> client.createExternalService(req), resp -> !updateConfigNodeLeader(resp)); + } + + @Override + public TSStatus startExternalService(int dataNodeId, String serviceName) throws TException { + return executeRemoteCallWithRetry( + () -> client.startExternalService(dataNodeId, serviceName), + resp -> !updateConfigNodeLeader(resp)); + } + + @Override + public TSStatus stopExternalService(int dataNodeId, String serviceName) throws TException { + return executeRemoteCallWithRetry( + () -> client.stopExternalService(dataNodeId, serviceName), + resp -> !updateConfigNodeLeader(resp)); + } + + @Override + public TSStatus dropExternalService(int dataNodeId, String serviceName) throws TException { + return executeRemoteCallWithRetry( + () -> client.dropExternalService(dataNodeId, serviceName), + resp -> !updateConfigNodeLeader(resp)); + } + + @Override + public TExternalServiceListResp showExternalService(int dataNodeId) throws TException { + return executeRemoteCallWithRetry( + () -> client.showExternalService(dataNodeId), resp -> !updateConfigNodeLeader(resp.status)); + } + @Override public TSStatus setSpaceQuota(TSetSpaceQuotaReq req) throws TException { return executeRemoteCallWithRetry( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index acdef794fa390..845a525381409 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -23,6 +23,8 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TExternalServiceEntry; +import org.apache.iotdb.common.rpc.thrift.TExternalServiceListResp; import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TLoadSample; import org.apache.iotdb.common.rpc.thrift.TNodeLocations; @@ -186,6 +188,7 @@ import org.apache.iotdb.db.schemaengine.template.TemplateInternalRPCUpdateType; import org.apache.iotdb.db.service.DataNode; import org.apache.iotdb.db.service.RegionMigrateService; +import org.apache.iotdb.db.service.exernalservice.ExternalServiceManagementService; import org.apache.iotdb.db.service.metrics.FileMetrics; import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; @@ -2935,6 +2938,22 @@ public TSStatus dropPipePlugin(TDropPipePluginInstanceReq req) { } } + @Override + public TExternalServiceListResp getBuiltInService() { + try { + + List serviceEntries = + ExternalServiceManagementService.getInstance().getBuiltInServices(); + return new TExternalServiceListResp( + new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()), serviceEntries); + } catch (Exception e) { + return new TExternalServiceListResp( + new TSStatus(TSStatusCode.GET_BUILTIN_EXTERNAL_SERVICE_ERROR.getStatusCode()) + .setMessage(e.getMessage()), + Collections.emptyList()); + } + } + private boolean isSucceed(TSStatus status) { return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java index d7fd3e071c640..4d19daca70d53 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java @@ -113,6 +113,10 @@ public static DatasetHeader getShowTriggersHeader() { return new DatasetHeader(ColumnHeaderConstant.showTriggersColumnHeaders, true); } + public static DatasetHeader getShowExternalServiceHeader() { + return new DatasetHeader(ColumnHeaderConstant.showExternalServiceColumnHeaders, true); + } + public static DatasetHeader getShowPipePluginsHeader() { return new DatasetHeader(ColumnHeaderConstant.showPipePluginsColumnHeaders, true); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java index b8fd3a094db14..8774b670e7a31 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java @@ -24,6 +24,8 @@ import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TExternalServiceEntry; +import org.apache.iotdb.common.rpc.thrift.TExternalServiceListResp; import org.apache.iotdb.commons.audit.UserEntity; import org.apache.iotdb.commons.client.exception.ClientManagerException; import org.apache.iotdb.commons.conf.IoTDBConstant; @@ -126,6 +128,7 @@ import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowFunctionsTask.getFunctionType; import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowPipePluginsTask.PIPE_PLUGIN_TYPE_BUILTIN; import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowPipePluginsTask.PIPE_PLUGIN_TYPE_EXTERNAL; +import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.externalservice.ShowExternalServiceTask.appendServiceEntry; public class InformationSchemaContentSupplierFactory { @@ -178,6 +181,8 @@ public static Iterator getSupplier( return new CurrentQueriesSupplier(dataTypes, predicate, userEntity); case InformationSchema.QUERIES_COSTS_HISTOGRAM: return new QueriesCostsHistogramSupplier(dataTypes, userEntity); + case InformationSchema.SERVICES: + return new ServicesSupplier(dataTypes, userEntity); default: throw new UnsupportedOperationException("Unknown table: " + tableName); } @@ -861,6 +866,38 @@ public boolean hasNext() { } } + private static class ServicesSupplier extends TsBlockSupplier { + + private final Iterator serviceEntryIterator; + + private ServicesSupplier(final List dataTypes, final UserEntity userEntity) + throws Exception { + super(dataTypes); + accessControl.checkUserGlobalSysPrivilege(userEntity); + try (final ConfigNodeClient client = + ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + // -1 means get all services + TExternalServiceListResp resp = client.showExternalService(-1); + if (resp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new IoTDBRuntimeException(resp.getStatus()); + } + + serviceEntryIterator = resp.getExternalServiceInfosIterator(); + } + } + + @Override + protected void constructLine() { + appendServiceEntry(serviceEntryIterator.next(), columnBuilders); + resultBuilder.declarePosition(); + } + + @Override + public boolean hasNext() { + return serviceEntryIterator.hasNext(); + } + } + private static class ConfigurationsSupplier extends TsBlockSupplier { private final Iterator> resultIterator; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java index ffa1f979b0200..0007f5dc79749 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java @@ -70,6 +70,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ClearCache; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateDB; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateExternalService; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateFunction; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateModel; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateTable; @@ -79,6 +80,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DescribeTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropDB; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropExternalService; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropFunction; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropModel; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropTable; @@ -130,7 +132,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowTables; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowVariables; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowVersion; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StartExternalService; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StartRepairData; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StopExternalService; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StopRepairData; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubscriptionStatement; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; @@ -581,6 +585,10 @@ private IQueryExecution createQueryExecutionForTableModel( || statement instanceof CreateFunction || statement instanceof DropFunction || statement instanceof ShowFunctions + || statement instanceof CreateExternalService + || statement instanceof StartExternalService + || statement instanceof StopExternalService + || statement instanceof DropExternalService || statement instanceof RelationalAuthorStatement || statement instanceof MigrateRegion || statement instanceof ReconstructRegion diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 4b609aa87eb78..acbe0dd486519 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -71,6 +71,10 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ai.ShowLoadedModelsTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ai.ShowModelsTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ai.UnloadModelTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.externalservice.CreateExternalServiceTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.externalservice.DropExternalServiceTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.externalservice.StartExternalServiceTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.externalservice.StopExternalServiceTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.ExtendRegionTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.MigrateRegionTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.ReconstructRegionTask; @@ -144,6 +148,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ClearCache; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ColumnDefinition; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateDB; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateExternalService; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateFunction; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateModel; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreatePipe; @@ -159,6 +164,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DescribeTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropDB; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropExternalService; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropFunction; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropModel; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropPipe; @@ -217,8 +223,10 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowTopics; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowVariables; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowVersion; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StartExternalService; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StartPipe; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StartRepairData; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StopExternalService; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StopPipe; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StopRepairData; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.UnloadModel; @@ -1511,6 +1519,37 @@ protected IConfigTask visitDropFunction(DropFunction node, MPPQueryContext conte return new DropFunctionTask(Model.TABLE, node.getUdfName()); } + @Override + protected IConfigTask visitCreateExternalService( + CreateExternalService node, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + return new CreateExternalServiceTask(node); + } + + @Override + protected IConfigTask visitStartExternalService( + StartExternalService node, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + accessControl.checkUserGlobalSysPrivilege(context); + return new StartExternalServiceTask(node.getServiceName()); + } + + @Override + protected IConfigTask visitStopExternalService( + StopExternalService node, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + accessControl.checkUserGlobalSysPrivilege(context); + return new StopExternalServiceTask(node.getServiceName()); + } + + @Override + protected IConfigTask visitDropExternalService( + DropExternalService node, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + accessControl.checkUserGlobalSysPrivilege(context); + return new DropExternalServiceTask(node.getServiceName(), node.isForcedly()); + } + @Override protected IConfigTask visitMigrateRegion(MigrateRegion migrateRegion, MPPQueryContext context) { context.setQueryType(QueryType.WRITE); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java index 3e2d408b5a751..60deee795093b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java @@ -76,6 +76,11 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ai.ShowLoadedModelsTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ai.ShowModelsTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ai.UnloadModelTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.externalservice.CreateExternalServiceTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.externalservice.DropExternalServiceTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.externalservice.ShowExternalServiceTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.externalservice.StartExternalServiceTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.externalservice.StopExternalServiceTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.ExtendRegionTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.MigrateRegionTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.ReconstructRegionTask; @@ -159,6 +164,11 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTriggersStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowVariablesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.UnSetTTLStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice.CreateExternalServiceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice.DropExternalServiceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice.ShowExternalServiceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice.StartExternalServiceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice.StopExternalServiceStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.CreateModelStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.CreateTrainingStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.DropModelStatement; @@ -459,6 +469,37 @@ public IConfigTask visitShowTriggers( return new ShowTriggersTask(); } + @Override + public IConfigTask visitCreateExternalService( + CreateExternalServiceStatement createExternalServiceStatement, MPPQueryContext context) { + return new CreateExternalServiceTask(createExternalServiceStatement); + } + + @Override + public IConfigTask visitStartExternalService( + StartExternalServiceStatement startExternalServiceStatement, MPPQueryContext context) { + return new StartExternalServiceTask(startExternalServiceStatement.getServiceName()); + } + + @Override + public IConfigTask visitStopExternalService( + StopExternalServiceStatement stopExternalServiceStatement, MPPQueryContext context) { + return new StopExternalServiceTask(stopExternalServiceStatement.getServiceName()); + } + + @Override + public IConfigTask visitDropExternalService( + DropExternalServiceStatement dropExternalServiceStatement, MPPQueryContext context) { + return new DropExternalServiceTask( + dropExternalServiceStatement.getServiceName(), dropExternalServiceStatement.isForcedly()); + } + + @Override + public IConfigTask visitShowExternalService( + ShowExternalServiceStatement showExternalServiceStatement, MPPQueryContext context) { + return new ShowExternalServiceTask(showExternalServiceStatement.getDataNodeId()); + } + @Override public IConfigTask visitCreatePipePlugin( CreatePipePluginStatement createPipePluginStatement, MPPQueryContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 4675c9b9e0097..5d59c645fb23e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -35,6 +35,7 @@ import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TExternalServiceEntry; import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; @@ -72,6 +73,8 @@ import org.apache.iotdb.commons.pipe.datastructure.visibility.VisibilityUtils; import org.apache.iotdb.commons.pipe.sink.payload.airgap.AirGapPseudoTPipeTransferRequest; import org.apache.iotdb.commons.schema.cache.CacheClearOptions; +import org.apache.iotdb.commons.schema.column.ColumnHeader; +import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; import org.apache.iotdb.commons.schema.table.AlterOrDropTableOperationType; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; @@ -313,6 +316,8 @@ import org.apache.iotdb.db.schemaengine.template.alter.TemplateAlterOperationUtil; import org.apache.iotdb.db.schemaengine.template.alter.TemplateExtendInfo; import org.apache.iotdb.db.service.DataNodeInternalRPCService; +import org.apache.iotdb.db.service.exernalservice.ExternalServiceManagementException; +import org.apache.iotdb.db.service.exernalservice.ExternalServiceManagementService; import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.db.storageengine.dataregion.compaction.repair.RepairTaskStatus; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionScheduleTaskManager; @@ -342,6 +347,8 @@ import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.external.commons.codec.digest.DigestUtils; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.block.column.TimeColumnBuilder; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -363,6 +370,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -377,6 +385,8 @@ import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_SCOPE; import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_RESULT_NODES; import static org.apache.iotdb.db.protocol.client.ConfigNodeClient.MSG_RECONNECTION_FAIL; +import static org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory.getShowExternalServiceHeader; +import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.externalservice.ShowExternalServiceTask.appendServiceEntry; import static org.apache.iotdb.db.utils.constant.SqlConstant.ROOT; import static org.apache.iotdb.udf.api.type.Type.OBJECT; @@ -4839,4 +4849,82 @@ public void handlePipeConfigClientExit(final String clientId) { LOGGER.warn("Failed to handlePipeConfigClientExit.", e); } } + + @Override + public SettableFuture createExternalService( + String serviceName, String className) { + SettableFuture future = SettableFuture.create(); + try { + ExternalServiceManagementService.getInstance().createService(serviceName, className); + future.set(new ConfigTaskResult(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()))); + } catch (Exception e) { + future.setException(new ExternalServiceManagementException(e.getMessage())); + } + return future; + } + + @Override + public SettableFuture startExternalService(String serviceName) { + SettableFuture future = SettableFuture.create(); + try { + ExternalServiceManagementService.getInstance().startService(serviceName); + future.set(new ConfigTaskResult(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()))); + } catch (Exception e) { + future.setException(new ExternalServiceManagementException(e.getMessage())); + } + return future; + } + + @Override + public SettableFuture stopExternalService(String serviceName) { + SettableFuture future = SettableFuture.create(); + try { + ExternalServiceManagementService.getInstance().stopService(serviceName); + future.set(new ConfigTaskResult(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()))); + } catch (Exception e) { + future.setException(new ExternalServiceManagementException(e.getMessage())); + } + return future; + } + + @Override + public SettableFuture dropExternalService( + String serviceName, boolean forcedly) { + SettableFuture future = SettableFuture.create(); + try { + ExternalServiceManagementService.getInstance().dropService(serviceName, forcedly); + future.set(new ConfigTaskResult(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()))); + } catch (Exception e) { + future.setException(new ExternalServiceManagementException(e.getMessage())); + } + return future; + } + + @Override + public SettableFuture showExternalService(int dataNodeId) { + SettableFuture future = SettableFuture.create(); + try { + Iterator iterator = + ExternalServiceManagementService.getInstance().showService(dataNodeId); + + List outputDataTypes = + ColumnHeaderConstant.showExternalServiceColumnHeaders.stream() + .map(ColumnHeader::getColumnType) + .collect(Collectors.toList()); + TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes); + TimeColumnBuilder timeColumnBuilder = builder.getTimeColumnBuilder(); + while (iterator.hasNext()) { + timeColumnBuilder.writeLong(0L); + appendServiceEntry(iterator.next(), builder.getValueColumnBuilders()); + builder.declarePosition(); + } + + future.set( + new ConfigTaskResult( + TSStatusCode.SUCCESS_STATUS, builder.build(), getShowExternalServiceHeader())); + } catch (Exception e) { + future.setException(e); + } + return future; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java index 4cb28ee6ef694..9e02ba6cff7f4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java @@ -315,6 +315,16 @@ SettableFuture showThrottleQuota( void handlePipeConfigClientExit(String clientId); + SettableFuture createExternalService(String serviceName, String className); + + SettableFuture startExternalService(String serviceName); + + SettableFuture stopExternalService(String serviceName); + + SettableFuture dropExternalService(String serviceName, boolean forcedly); + + SettableFuture showExternalService(int dataNodeId); + // =============================== table syntax ========================================= SettableFuture showDatabases( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/externalservice/CreateExternalServiceTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/externalservice/CreateExternalServiceTask.java new file mode 100644 index 0000000000000..948609333fa80 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/externalservice/CreateExternalServiceTask.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.externalservice; + +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateExternalService; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice.CreateExternalServiceStatement; + +import com.google.common.util.concurrent.ListenableFuture; + +import java.util.Locale; + +public class CreateExternalServiceTask implements IConfigTask { + + private final String serviceName; + private final String className; + + public CreateExternalServiceTask(CreateExternalServiceStatement statement) { + this.serviceName = statement.getServiceName().toUpperCase(Locale.ENGLISH); + this.className = statement.getClassName(); + } + + public CreateExternalServiceTask(CreateExternalService createExternalService) { + this.serviceName = createExternalService.getServiceName().toUpperCase(Locale.ENGLISH); + this.className = createExternalService.getClassName(); + } + + @Override + public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return configTaskExecutor.createExternalService(serviceName, className); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/externalservice/DropExternalServiceTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/externalservice/DropExternalServiceTask.java new file mode 100644 index 0000000000000..7772af7b2a3d8 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/externalservice/DropExternalServiceTask.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.externalservice; + +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; + +import com.google.common.util.concurrent.ListenableFuture; + +import java.util.Locale; + +public class DropExternalServiceTask implements IConfigTask { + + private final String serviceName; + private final boolean forcedly; + + public DropExternalServiceTask(String serviceName, boolean forcedly) { + this.serviceName = serviceName.toUpperCase(Locale.ENGLISH); + this.forcedly = forcedly; + } + + @Override + public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return configTaskExecutor.dropExternalService(serviceName, forcedly); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/externalservice/ShowExternalServiceTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/externalservice/ShowExternalServiceTask.java new file mode 100644 index 0000000000000..66d0e2e6073b8 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/externalservice/ShowExternalServiceTask.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.externalservice; + +import org.apache.iotdb.common.rpc.thrift.TExternalServiceEntry; +import org.apache.iotdb.commons.externalservice.ServiceInfo; +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; + +import com.google.common.util.concurrent.ListenableFuture; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.utils.Binary; + +public class ShowExternalServiceTask implements IConfigTask { + private final int dataNodeId; + + public ShowExternalServiceTask(int dataNodeId) { + this.dataNodeId = dataNodeId; + } + + @Override + public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return configTaskExecutor.showExternalService(dataNodeId); + } + + public static void appendServiceEntry( + TExternalServiceEntry externalServiceEntry, ColumnBuilder[] columnBuilders) { + columnBuilders[0].writeBinary( + new Binary(externalServiceEntry.getServiceName(), TSFileConfig.STRING_CHARSET)); + columnBuilders[1].writeInt(externalServiceEntry.getDataNodId()); + columnBuilders[2].writeBinary( + new Binary( + ServiceInfo.State.deserialize(externalServiceEntry.getState()).toString(), + TSFileConfig.STRING_CHARSET)); + columnBuilders[3].writeBinary( + new Binary(externalServiceEntry.getClassName(), TSFileConfig.STRING_CHARSET)); + columnBuilders[4].writeBinary( + new Binary( + ServiceInfo.ServiceType.deserialize(externalServiceEntry.getServiceType()) + .getDisplayName(), + TSFileConfig.STRING_CHARSET)); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/externalservice/StartExternalServiceTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/externalservice/StartExternalServiceTask.java new file mode 100644 index 0000000000000..5b0d18df8c790 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/externalservice/StartExternalServiceTask.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.externalservice; + +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; + +import com.google.common.util.concurrent.ListenableFuture; + +import java.util.Locale; + +public class StartExternalServiceTask implements IConfigTask { + + private final String serviceName; + + public StartExternalServiceTask(String serviceName) { + this.serviceName = serviceName.toUpperCase(Locale.ENGLISH); + ; + } + + @Override + public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return configTaskExecutor.startExternalService(serviceName); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/externalservice/StopExternalServiceTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/externalservice/StopExternalServiceTask.java new file mode 100644 index 0000000000000..671cad38965d2 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/externalservice/StopExternalServiceTask.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.externalservice; + +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; + +import com.google.common.util.concurrent.ListenableFuture; + +import java.util.Locale; + +public class StopExternalServiceTask implements IConfigTask { + + private final String serviceName; + + public StopExternalServiceTask(String serviceName) { + this.serviceName = serviceName.toUpperCase(Locale.ENGLISH); + } + + @Override + public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return configTaskExecutor.stopExternalService(serviceName); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index 0a95f1afe2097..25d31e7c3382d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -187,6 +187,11 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTriggersStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowVariablesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.UnSetTTLStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice.CreateExternalServiceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice.DropExternalServiceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice.ShowExternalServiceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice.StartExternalServiceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice.StopExternalServiceStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.CreateModelStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.CreateTrainingStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.DropModelStatement; @@ -1093,6 +1098,38 @@ public Statement visitShowTriggers(IoTDBSqlParser.ShowTriggersContext ctx) { return new ShowTriggersStatement(); } + @Override + public Statement visitCreateService(IoTDBSqlParser.CreateServiceContext ctx) { + String serviceName = parseIdentifier(ctx.serviceName.getText()); + String className = parseStringLiteral(ctx.className.getText()); + return new CreateExternalServiceStatement(serviceName, className); + } + + @Override + public Statement visitStartService(IoTDBSqlParser.StartServiceContext ctx) { + return new StartExternalServiceStatement(parseIdentifier(ctx.serviceName.getText())); + } + + @Override + public Statement visitStopService(IoTDBSqlParser.StopServiceContext ctx) { + return new StopExternalServiceStatement(parseIdentifier(ctx.serviceName.getText())); + } + + @Override + public Statement visitDropService(IoTDBSqlParser.DropServiceContext ctx) { + return new DropExternalServiceStatement(parseIdentifier(ctx.serviceName.getText()), false); + } + + @Override + public Statement visitShowService(IoTDBSqlParser.ShowServiceContext ctx) { + // show services on all DNs + int dataNodeId = -1; + if (ctx.ON() != null) { + dataNodeId = Integer.parseInt(ctx.targetDataNodeId.getText()); + } + return new ShowExternalServiceStatement(dataNodeId); + } + // Create PipePlugin ===================================================================== @Override public Statement visitCreatePipePlugin(IoTDBSqlParser.CreatePipePluginContext ctx) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java index d7d755ddc1da6..8e73872f5ba8f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java @@ -104,6 +104,7 @@ public List getDataNodeLocations(final String tableName) { case InformationSchema.NODES: case InformationSchema.CONFIG_NODES: case InformationSchema.DATA_NODES: + case InformationSchema.SERVICES: return Collections.singletonList(DataNodeEndPoints.getLocalDataNodeLocation()); default: throw new UnsupportedOperationException("Unknown table: " + tableName); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java index 7f9cf30e93c2b..3f127a2a0ae37 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/TreeAccessCheckVisitor.java @@ -91,6 +91,11 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTriggersStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowVariablesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.UnSetTTLStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice.CreateExternalServiceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice.DropExternalServiceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice.ShowExternalServiceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice.StartExternalServiceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice.StopExternalServiceStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.CreateModelStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.CreateTrainingStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.DropModelStatement; @@ -936,6 +941,38 @@ private TSStatus checkTriggerManagement(IAuditEntity auditEntity, Supplier ""); + } + + @Override + public TSStatus visitStartExternalService( + StartExternalServiceStatement startExternalServiceStatement, TreeAccessCheckContext context) { + return checkGlobalAuth(context, PrivilegeType.SYSTEM, () -> ""); + } + + @Override + public TSStatus visitStopExternalService( + StopExternalServiceStatement stopExternalServiceStatement, TreeAccessCheckContext context) { + return checkGlobalAuth(context, PrivilegeType.SYSTEM, () -> ""); + } + + @Override + public TSStatus visitDropExternalService( + DropExternalServiceStatement dropExternalServiceStatement, TreeAccessCheckContext context) { + return checkGlobalAuth(context, PrivilegeType.SYSTEM, () -> ""); + } + + @Override + public TSStatus visitShowExternalService( + ShowExternalServiceStatement showExternalServiceStatement, TreeAccessCheckContext context) { + return checkGlobalAuth(context, PrivilegeType.SYSTEM, () -> ""); + } + // ============================== database related =========================== @Override public TSStatus visitSetDatabase( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java index 4b32201aa8e37..e8e5cce22ca93 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java @@ -601,6 +601,26 @@ protected R visitDropFunction(DropFunction node, C context) { return visitStatement(node, context); } + protected R visitCreateExternalService(CreateExternalService node, C context) { + return visitStatement(node, context); + } + + protected R visitStartExternalService(StartExternalService node, C context) { + return visitStatement(node, context); + } + + protected R visitStopExternalService(StopExternalService node, C context) { + return visitStatement(node, context); + } + + protected R visitDropExternalService(DropExternalService node, C context) { + return visitStatement(node, context); + } + + protected R visitShowExternalService(ShowExternalService node, C context) { + return visitStatement(node, context); + } + protected R visitCreateOrUpdateDevice(CreateOrUpdateDevice node, C context) { return visitStatement(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/CreateExternalService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/CreateExternalService.java new file mode 100644 index 0000000000000..36674368e9fd2 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/CreateExternalService.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class CreateExternalService extends Statement { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(CreateExternalService.class); + + private final String serviceName; + private final String className; + + public CreateExternalService(NodeLocation location, String serviceName, String className) { + super(requireNonNull(location, "location is null")); + + this.serviceName = requireNonNull(serviceName, "serviceName is null"); + this.className = requireNonNull(className, "className is null"); + } + + public String getServiceName() { + return serviceName; + } + + public String getClassName() { + return className; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitCreateExternalService(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + CreateExternalService that = (CreateExternalService) o; + return Objects.equals(serviceName, that.serviceName) + && Objects.equals(className, that.className); + } + + @Override + public int hashCode() { + return Objects.hash(serviceName, className); + } + + @Override + public String toString() { + return toStringHelper(this) + .add("serviceName", serviceName) + .add("className", className) + .toString(); + } + + @Override + public long ramBytesUsed() { + long size = INSTANCE_SIZE; + size += AstMemoryEstimationHelper.getEstimatedSizeOfNodeLocation(getLocationInternal()); + size += RamUsageEstimator.sizeOf(serviceName); + size += RamUsageEstimator.sizeOf(className); + return size; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropExternalService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropExternalService.java new file mode 100644 index 0000000000000..602554fa16bd9 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropExternalService.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class DropExternalService extends Statement { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(DropExternalService.class); + + private final String serviceName; + private final boolean forcedly; + + public DropExternalService(NodeLocation location, String serviceName, boolean forcedly) { + super(requireNonNull(location, "location is null")); + this.serviceName = requireNonNull(serviceName, "serviceName is null"); + this.forcedly = forcedly; + } + + public String getServiceName() { + return serviceName; + } + + public boolean isForcedly() { + return forcedly; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitDropExternalService(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DropExternalService that = (DropExternalService) o; + return Objects.equals(serviceName, that.serviceName); + } + + @Override + public int hashCode() { + return Objects.hash(serviceName); + } + + @Override + public String toString() { + return toStringHelper(this).add("serviceName", serviceName).toString(); + } + + @Override + public long ramBytesUsed() { + long size = INSTANCE_SIZE; + size += AstMemoryEstimationHelper.getEstimatedSizeOfNodeLocation(getLocationInternal()); + size += RamUsageEstimator.sizeOf(serviceName); + return size; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowExternalService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowExternalService.java new file mode 100644 index 0000000000000..ca713507e13b0 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowExternalService.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import java.util.Optional; + +public class ShowExternalService extends ShowStatement { + + public ShowExternalService( + NodeLocation location, + String tableName, + Optional where, + Optional orderBy, + Optional offset, + Optional limit) { + super(location, tableName, where, orderBy, offset, limit); + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitShowExternalService(this, context); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StartExternalService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StartExternalService.java new file mode 100644 index 0000000000000..3c72ecce395e3 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StartExternalService.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class StartExternalService extends Statement { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(StartExternalService.class); + + private final String serviceName; + + public StartExternalService(NodeLocation location, String serviceName) { + super(requireNonNull(location, "location is null")); + this.serviceName = requireNonNull(serviceName, "serviceName is null"); + } + + public String getServiceName() { + return serviceName; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitStartExternalService(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + StartExternalService that = (StartExternalService) o; + return Objects.equals(serviceName, that.serviceName); + } + + @Override + public int hashCode() { + return Objects.hash(serviceName); + } + + @Override + public String toString() { + return toStringHelper(this).add("serviceName", serviceName).toString(); + } + + @Override + public long ramBytesUsed() { + long size = INSTANCE_SIZE; + size += AstMemoryEstimationHelper.getEstimatedSizeOfNodeLocation(getLocationInternal()); + size += RamUsageEstimator.sizeOf(serviceName); + return size; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StopExternalService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StopExternalService.java new file mode 100644 index 0000000000000..039c2c9b55905 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StopExternalService.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class StopExternalService extends Statement { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(StopExternalService.class); + + private final String serviceName; + + public StopExternalService(NodeLocation location, String serviceName) { + super(requireNonNull(location, "location is null")); + this.serviceName = requireNonNull(serviceName, "serviceName is null"); + } + + public String getServiceName() { + return serviceName; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitStopExternalService(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + StopExternalService that = (StopExternalService) o; + return Objects.equals(serviceName, that.serviceName); + } + + @Override + public int hashCode() { + return Objects.hash(serviceName); + } + + @Override + public String toString() { + return toStringHelper(this).add("serviceName", serviceName).toString(); + } + + @Override + public long ramBytesUsed() { + long size = INSTANCE_SIZE; + size += AstMemoryEstimationHelper.getEstimatedSizeOfNodeLocation(getLocationInternal()); + size += RamUsageEstimator.sizeOf(serviceName); + return size; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 5c160a1aa5e17..3b8150bd5206c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -58,6 +58,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CountDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CountStatement; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateDB; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateExternalService; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateFunction; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateIndex; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateModel; @@ -80,6 +81,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DoubleLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropDB; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropExternalService; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropFunction; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropIndex; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropModel; @@ -193,6 +195,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDataNodes; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDevice; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowExternalService; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowFunctions; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowIndex; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowLoadedModels; @@ -212,9 +215,11 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SingleColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StartExternalService; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StartPipe; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StartRepairData; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StopExternalService; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StopPipe; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StopRepairData; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral; @@ -1087,6 +1092,52 @@ public Node visitShowFunctionsStatement(RelationalSqlParser.ShowFunctionsStateme return new ShowFunctions(); } + @Override + public Node visitCreateServiceStatement(RelationalSqlParser.CreateServiceStatementContext ctx) { + String serviceName = ((Identifier) visit(ctx.serviceName)).getValue(); + String className = ((StringLiteral) visit(ctx.className)).getValue(); + return new CreateExternalService(getLocation(ctx), serviceName, className); + } + + @Override + public Node visitStartServiceStatement(RelationalSqlParser.StartServiceStatementContext ctx) { + return new StartExternalService( + getLocation(ctx), ((Identifier) visit(ctx.serviceName)).getValue()); + } + + @Override + public Node visitStopServiceStatement(RelationalSqlParser.StopServiceStatementContext ctx) { + return new StopExternalService( + getLocation(ctx), ((Identifier) visit(ctx.serviceName)).getValue()); + } + + @Override + public Node visitDropServiceStatement(RelationalSqlParser.DropServiceStatementContext ctx) { + return new DropExternalService( + getLocation(ctx), ((Identifier) visit(ctx.serviceName)).getValue(), ctx.FORCEDLY() != null); + } + + @Override + public Node visitShowServiceStatement(RelationalSqlParser.ShowServiceStatementContext ctx) { + // show services on all DNs + Optional where = Optional.empty(); + if (ctx.ON() != null) { + where = + Optional.of( + new ComparisonExpression( + ComparisonExpression.Operator.EQUAL, + new Identifier("datanode_id"), + new LongLiteral(ctx.targetDataNodeId.getText()))); + } + return new ShowExternalService( + getLocation(ctx), + InformationSchema.SERVICES, + where, + Optional.empty(), + Optional.empty(), + Optional.empty()); + } + @Override public Node visitLoadTsFileStatement(RelationalSqlParser.LoadTsFileStatementContext ctx) { final Map withAttributes = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/rewrite/ShowRewrite.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/rewrite/ShowRewrite.java index 12fd67d5017a4..d0e386e5926dc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/rewrite/ShowRewrite.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/rewrite/ShowRewrite.java @@ -34,6 +34,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Relation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Select; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowExternalService; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowQueriesStatement; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowStatement; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SingleColumn; @@ -84,6 +85,20 @@ protected Node visitShowStatement(final ShowStatement showStatement, final Void showStatement.getLimit()); } + @Override + protected Node visitShowExternalService(ShowExternalService node, Void context) { + return simpleQuery( + selectList(new AllColumns()), + from(INFORMATION_DATABASE, node.getTableName()), + node.getWhere(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty()); + } + @Override protected Node visitCountStatement(final CountStatement countStatement, final Void context) { return simpleQuery( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java index d75e2e09b7c28..721459ce84d37 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java @@ -192,4 +192,10 @@ public enum StatementType { FAST_LAST_QUERY, SHOW_CONFIGURATION, + + CREATE_EXTERNAL_SERVICE, + START_EXTERNAL_SERVICE, + STOP_EXTERNAL_SERVICE, + DROP_EXTERNAL_SERVICE, + SHOW_EXTERNAL_SERVICE, } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java index 3266ba0726510..29d49453f4445 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java @@ -80,6 +80,11 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTriggersStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowVariablesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.UnSetTTLStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice.CreateExternalServiceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice.DropExternalServiceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice.ShowExternalServiceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice.StartExternalServiceStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice.StopExternalServiceStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.CreateModelStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.CreateTrainingStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.DropModelStatement; @@ -280,6 +285,32 @@ public R visitShowTriggers(ShowTriggersStatement showTriggersStatement, C contex return visitStatement(showTriggersStatement, context); } + // ExternalService + public R visitCreateExternalService( + CreateExternalServiceStatement createExternalServiceStatement, C context) { + return visitStatement(createExternalServiceStatement, context); + } + + public R visitStartExternalService( + StartExternalServiceStatement startExternalServiceStatement, C context) { + return visitStatement(startExternalServiceStatement, context); + } + + public R visitStopExternalService( + StopExternalServiceStatement stopExternalServiceStatement, C context) { + return visitStatement(stopExternalServiceStatement, context); + } + + public R visitDropExternalService( + DropExternalServiceStatement dropExternalServiceStatement, C context) { + return visitStatement(dropExternalServiceStatement, context); + } + + public R visitShowExternalService( + ShowExternalServiceStatement showExternalServiceStatement, C context) { + return visitStatement(showExternalServiceStatement, context); + } + // Pipe Plugin public R visitCreatePipePlugin(CreatePipePluginStatement createPipePluginStatement, C context) { return visitStatement(createPipePluginStatement, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/externalservice/CreateExternalServiceStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/externalservice/CreateExternalServiceStatement.java new file mode 100644 index 0000000000000..83cd86a1128de --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/externalservice/CreateExternalServiceStatement.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; +import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; + +import java.util.Collections; +import java.util.List; + +public class CreateExternalServiceStatement extends Statement implements IConfigStatement { + + private final String serviceName; + private final String className; + + public CreateExternalServiceStatement(String serviceName, String className) { + super(); + statementType = StatementType.CREATE_EXTERNAL_SERVICE; + this.serviceName = serviceName; + this.className = className; + } + + public String getServiceName() { + return serviceName; + } + + public String getClassName() { + return className; + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitCreateExternalService(this, context); + } + + @Override + public QueryType getQueryType() { + return QueryType.WRITE; + } + + @Override + public List getPaths() { + return Collections.emptyList(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/externalservice/DropExternalServiceStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/externalservice/DropExternalServiceStatement.java new file mode 100644 index 0000000000000..b480112d76957 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/externalservice/DropExternalServiceStatement.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; +import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; + +import java.util.Collections; +import java.util.List; + +public class DropExternalServiceStatement extends Statement implements IConfigStatement { + + private final String serviceName; + private final boolean forcedly; + + public DropExternalServiceStatement(String serviceName, boolean forcedly) { + super(); + statementType = StatementType.DROP_EXTERNAL_SERVICE; + this.serviceName = serviceName; + this.forcedly = forcedly; + } + + public String getServiceName() { + return serviceName; + } + + public boolean isForcedly() { + return forcedly; + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitDropExternalService(this, context); + } + + @Override + public QueryType getQueryType() { + return QueryType.WRITE; + } + + @Override + public List getPaths() { + return Collections.emptyList(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/externalservice/ShowExternalServiceStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/externalservice/ShowExternalServiceStatement.java new file mode 100644 index 0000000000000..dfcfbffd30393 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/externalservice/ShowExternalServiceStatement.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; +import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowStatement; + +import java.util.Collections; +import java.util.List; + +public class ShowExternalServiceStatement extends ShowStatement implements IConfigStatement { + + private final int dataNodeId; + + public ShowExternalServiceStatement(int dataNodeId) { + super(); + statementType = StatementType.SHOW_EXTERNAL_SERVICE; + this.dataNodeId = dataNodeId; + } + + public int getDataNodeId() { + return dataNodeId; + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitShowExternalService(this, context); + } + + @Override + public List getPaths() { + return Collections.emptyList(); + } + + @Override + public QueryType getQueryType() { + return QueryType.READ; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/externalservice/StartExternalServiceStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/externalservice/StartExternalServiceStatement.java new file mode 100644 index 0000000000000..f7575a5088537 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/externalservice/StartExternalServiceStatement.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; +import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; + +import java.util.Collections; +import java.util.List; + +public class StartExternalServiceStatement extends Statement implements IConfigStatement { + + private final String serviceName; + + public StartExternalServiceStatement(String serviceName) { + super(); + statementType = StatementType.START_EXTERNAL_SERVICE; + this.serviceName = serviceName; + } + + public String getServiceName() { + return serviceName; + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitStartExternalService(this, context); + } + + @Override + public QueryType getQueryType() { + return QueryType.WRITE; + } + + @Override + public List getPaths() { + return Collections.emptyList(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/externalservice/StopExternalServiceStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/externalservice/StopExternalServiceStatement.java new file mode 100644 index 0000000000000..8de43a8546d46 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/externalservice/StopExternalServiceStatement.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.metadata.externalservice; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; +import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; + +import java.util.Collections; +import java.util.List; + +public class StopExternalServiceStatement extends Statement implements IConfigStatement { + + private final String serviceName; + + public StopExternalServiceStatement(String serviceName) { + super(); + statementType = StatementType.STOP_EXTERNAL_SERVICE; + this.serviceName = serviceName; + } + + public String getServiceName() { + return serviceName; + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitStopExternalService(this, context); + } + + @Override + public QueryType getQueryType() { + return QueryType.WRITE; + } + + @Override + public List getPaths() { + return Collections.emptyList(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java index 1fac05012fe89..01b39023d54ec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java @@ -1287,6 +1287,7 @@ private void initProtocols() throws StartupException { if (IoTDBRestServiceDescriptor.getInstance().getConfig().isEnableRestService()) { registerManager.register(RestService.getInstance()); } + // registerManager.register(ExternalServiceManagementService.getInstance()); if (PipeConfig.getInstance().getPipeAirGapReceiverEnabled()) { registerManager.register(PipeDataNodeAgent.receiver().airGap()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/exernalservice/BuiltinExternalServices.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/exernalservice/BuiltinExternalServices.java new file mode 100644 index 0000000000000..9b9fa403e7fc9 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/exernalservice/BuiltinExternalServices.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.service.exernalservice; + +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.conf.rest.IoTDBRestServiceDescriptor; + +import java.util.function.Supplier; + +public enum BuiltinExternalServices { + MQTT( + "MQTT", + "org.apache.iotdb.externalservice.Mqtt", + IoTDBDescriptor.getInstance().getConfig()::isEnableMQTTService), + REST( + "REST", + "org.apache.iotdb.externalservice.Rest", + IoTDBRestServiceDescriptor.getInstance().getConfig()::isEnableRestService); + + private final String serviceName; + private final String className; + private final Supplier enabledFunction; + + BuiltinExternalServices(String serviceName, String className, Supplier enabledFunction) { + this.serviceName = serviceName; + this.className = className; + this.enabledFunction = enabledFunction; + } + + public String getServiceName() { + return serviceName; + } + + public String getClassName() { + return className; + } + + public boolean isEnabled() { + return enabledFunction.get(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/exernalservice/ExternalServiceClassLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/exernalservice/ExternalServiceClassLoader.java new file mode 100644 index 0000000000000..85166f1a1df6f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/exernalservice/ExternalServiceClassLoader.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.service.exernalservice; + +import org.apache.iotdb.commons.file.SystemFileFactory; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class ExternalServiceClassLoader extends URLClassLoader { + + private static final Logger LOGGER = LoggerFactory.getLogger(ExternalServiceClassLoader.class); + + private final String libRoot; + + public ExternalServiceClassLoader(String libRoot) throws IOException { + super(new URL[0]); + this.libRoot = libRoot; + LOGGER.info("External Service lib root: {}", libRoot); + addURLs(); + } + + private void addURLs() throws IOException { + try (Stream pathStream = + Files.walk(SystemFileFactory.INSTANCE.getFile(libRoot).toPath())) { + for (Path path : + pathStream.filter(path -> !path.toFile().isDirectory()).collect(Collectors.toList())) { + super.addURL(path.toUri().toURL()); + } + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/exernalservice/ExternalServiceManagementException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/exernalservice/ExternalServiceManagementException.java new file mode 100644 index 0000000000000..0ccc917ec6979 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/exernalservice/ExternalServiceManagementException.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.service.exernalservice; + +public class ExternalServiceManagementException extends RuntimeException { + + public ExternalServiceManagementException(String message) { + super(message); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/exernalservice/ExternalServiceManagementService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/exernalservice/ExternalServiceManagementService.java new file mode 100644 index 0000000000000..49d0ce22e8b87 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/exernalservice/ExternalServiceManagementService.java @@ -0,0 +1,342 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.service.exernalservice; + +import org.apache.iotdb.common.rpc.thrift.TExternalServiceEntry; +import org.apache.iotdb.common.rpc.thrift.TExternalServiceListResp; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.exception.ClientManagerException; +import org.apache.iotdb.commons.exception.IoTDBRuntimeException; +import org.apache.iotdb.commons.externalservice.ServiceInfo; +import org.apache.iotdb.confignode.rpc.thrift.TCreateExternalServiceReq; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.protocol.client.ConfigNodeClient; +import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; +import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; +import org.apache.iotdb.db.queryengine.common.QueryId; +import org.apache.iotdb.externalservice.api.IExternalService; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.concurrent.GuardedBy; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; + +import static com.google.common.base.Preconditions.checkState; +import static org.apache.iotdb.commons.externalservice.ServiceInfo.State.RUNNING; +import static org.apache.iotdb.commons.externalservice.ServiceInfo.State.STOPPED; + +public class ExternalServiceManagementService { + @GuardedBy("lock") + private final Map serviceInfos; + + private final String libRoot; + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + + private static final Logger LOGGER = + LoggerFactory.getLogger(ExternalServiceManagementService.class); + + private ExternalServiceManagementService(String libRoot) { + this.serviceInfos = new HashMap<>(); + restoreBuiltInServices(); + this.libRoot = libRoot; + } + + public Iterator showService(int dataNodeId) + throws ClientManagerException, TException { + try { + lock.readLock().lock(); + + ConfigNodeClient client = + ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID); + TExternalServiceListResp resp = client.showExternalService(dataNodeId); + if (resp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new IoTDBRuntimeException(resp.getStatus().message, resp.getStatus().code); + } + return resp.getExternalServiceInfos().iterator(); + } finally { + lock.readLock().unlock(); + } + } + + public void createService(String serviceName, String className) + throws ClientManagerException, TException { + try { + lock.writeLock().lock(); + + // 1. validate + if (serviceInfos.containsKey(serviceName)) { + throw new ExternalServiceManagementException( + String.format("Failed to create External Service %s, it already exists!", serviceName)); + } + + // 2. persist on CN + ConfigNodeClient client = + ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID); + TSStatus status = + client.createExternalService( + new TCreateExternalServiceReq(QueryId.getDataNodeId(), serviceName, className)); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new IoTDBRuntimeException(status.message, status.code); + } + + // 3. modify memory info + serviceInfos.put( + serviceName, + new ServiceInfo(serviceName, className, ServiceInfo.ServiceType.USER_DEFINED)); + } finally { + lock.writeLock().unlock(); + } + } + + public void startService(String serviceName) throws ClientManagerException, TException { + try { + lock.writeLock().lock(); + + // 1. validate + ServiceInfo serviceInfo = serviceInfos.get(serviceName); + if (serviceInfo == null) { + throw new ExternalServiceManagementException( + String.format( + "Failed to start External Service %s, because it is not existed!", serviceName)); + } + + // 2. call start method of ServiceInstance, create if Instance was not created + if (serviceInfo.getState() == RUNNING) { + return; + } else { + // The state is STOPPED + if (serviceInfo.getServiceInstance() != null) { + serviceInfo.getServiceInstance().start(); + } else { + // lazy create Instance + serviceInfo.setServiceInstance(createExternalServiceInstance(serviceName)); + } + } + + // 3. persist on CN, rollback if failed + ConfigNodeClient client = + ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID); + TSStatus status = client.startExternalService(QueryId.getDataNodeId(), serviceName); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + serviceInfo.getServiceInstance().stop(); + throw new IoTDBRuntimeException(status.message, status.code); + } + + // 4. modify memory info + serviceInfo.setState(RUNNING); + } finally { + lock.writeLock().unlock(); + } + } + + private IExternalService createExternalServiceInstance(String serviceName) { + // close ClassLoader automatically to release the file handle + try (ExternalServiceClassLoader classLoader = new ExternalServiceClassLoader(libRoot); ) { + return (IExternalService) + Class.forName(serviceName, true, classLoader).getDeclaredConstructor().newInstance(); + } catch (IOException + | InstantiationException + | InvocationTargetException + | NoSuchMethodException + | IllegalAccessException + | ClassNotFoundException + | ClassCastException e) { + String errorMessage = + String.format( + "Failed to start External Service %s, because its instance can not be constructed successfully. Exception: %s", + serviceName, e); + LOGGER.warn(errorMessage, e); + throw new ExternalServiceManagementException(errorMessage); + } + } + + public void stopService(String serviceName) throws ClientManagerException, TException { + try { + lock.writeLock().lock(); + + // 1. validate + ServiceInfo serviceInfo = serviceInfos.get(serviceName); + if (serviceInfo == null) { + throw new ExternalServiceManagementException( + String.format( + "Failed to start External Service %s, because it is not existed!", serviceName)); + } + + // 2. call stop method of ServiceInstance + if (serviceInfo.getState() == STOPPED) { + return; + } else { + // The state is RUNNING + stopService(serviceInfo); + } + + // 3. persist on CN, rollback if failed + ConfigNodeClient client = + ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID); + TSStatus status = client.stopExternalService(QueryId.getDataNodeId(), serviceName); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + serviceInfo.getServiceInstance().start(); + throw new IoTDBRuntimeException(status.message, status.code); + } + + // 4. modify memory info + serviceInfo.setState(STOPPED); + } finally { + lock.writeLock().unlock(); + } + } + + private void stopService(ServiceInfo serviceInfo) { + checkState( + serviceInfo.getServiceInstance() != null, + "External Service instance is null when state is RUNNING!", + serviceInfo.getServiceName()); + serviceInfo.getServiceInstance().stop(); + } + + public void dropService(String serviceName, boolean forcedly) + throws ClientManagerException, TException { + try { + lock.writeLock().lock(); + + // 1. validate + ServiceInfo serviceInfo = serviceInfos.get(serviceName); + if (serviceInfo == null) { + throw new ExternalServiceManagementException( + String.format( + "Failed to drop External Service %s, because it is not existed!", serviceName)); + } + if (serviceInfo.getServiceType() == ServiceInfo.ServiceType.BUILTIN) { + throw new ExternalServiceManagementException( + String.format( + "Failed to drop External Service %s, because it is BUILT-IN!", serviceName)); + } + + // 2. stop or fail when service are not stopped + if (serviceInfo.getState() == STOPPED) { + // do nothing + } else { + // The state is RUNNING + if (forcedly) { + try { + stopService(serviceInfo); + } catch (Exception e) { + // record errMsg if exception occurs during the stop of service + LOGGER.warn( + "Failed to stop External Service %s because %s. It will be drop forcedly", + serviceName, e.getMessage()); + } + } else { + throw new ExternalServiceManagementException( + String.format( + "Failed to drop External Service %s, because it is RUNNING!", serviceName)); + } + } + + // 3. persist on CN + ConfigNodeClient client = + ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID); + TSStatus status = client.dropExternalService(QueryId.getDataNodeId(), serviceName); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + throw new IoTDBRuntimeException(status.message, status.code); + } + + // 4. modify memory info + serviceInfos.remove(serviceName); + } finally { + lock.writeLock().unlock(); + } + } + + public void restoreRunningServices() { + lock.writeLock().lock(); + try { + // start services with RUNNING state + serviceInfos + .values() + .forEach( + serviceInfo -> { + if (serviceInfo.getState() == RUNNING) { + serviceInfo.setServiceInstance( + createExternalServiceInstance(serviceInfo.getServiceName())); + } + }); + } finally { + lock.writeLock().unlock(); + } + } + + private void restoreBuiltInServices() { + for (BuiltinExternalServices builtinExternalService : BuiltinExternalServices.values()) { + serviceInfos.put( + builtinExternalService.getServiceName(), + new ServiceInfo( + builtinExternalService.getServiceName(), + builtinExternalService.getClassName(), + ServiceInfo.ServiceType.BUILTIN, + builtinExternalService.isEnabled() ? RUNNING : STOPPED)); + } + } + + public List getBuiltInServices() { + try { + lock.readLock().lock(); + return serviceInfos.values().stream() + .filter(serviceInfo -> serviceInfo.getServiceType() == ServiceInfo.ServiceType.BUILTIN) + .map( + serviceInfo -> + new TExternalServiceEntry( + serviceInfo.getServiceName(), + serviceInfo.getClassName(), + serviceInfo.getState().getValue(), + QueryId.getDataNodeId(), + ServiceInfo.ServiceType.BUILTIN.getValue())) + .collect(Collectors.toList()); + } finally { + lock.readLock().unlock(); + } + } + + public static ExternalServiceManagementService getInstance() { + return ExternalServiceManagementServiceHolder.INSTANCE; + } + + private static class ExternalServiceManagementServiceHolder { + + private static final ExternalServiceManagementService INSTANCE = + new ExternalServiceManagementService( + IoTDBDescriptor.getInstance().getConfig().getExternalServiceDir()); + + private ExternalServiceManagementServiceHolder() {} + } +} diff --git a/iotdb-core/node-commons/pom.xml b/iotdb-core/node-commons/pom.xml index 0bf2c01bfb56b..6e54c6b0b4cf8 100644 --- a/iotdb-core/node-commons/pom.xml +++ b/iotdb-core/node-commons/pom.xml @@ -45,6 +45,11 @@ common ${tsfile.version} + + org.apache.iotdb + external-service-api + 2.0.7-SNAPSHOT + org.apache.iotdb udf-api diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java index c7575577ef15e..a94f472b606dd 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java @@ -270,6 +270,7 @@ private IoTDBConstant() {} public static final String EXT_FOLDER_NAME = "ext"; public static final String UDF_FOLDER_NAME = "udf"; public static final String TRIGGER_FOLDER_NAME = "trigger"; + public static final String EXTERNAL_SERVICE_FOLDER_NAME = "external_service"; public static final String PIPE_FOLDER_NAME = "pipe"; public static final String CTE_FOLDER_NAME = "cte"; public static final String TMP_FOLDER_NAME = "tmp"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/externalservice/ServiceInfo.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/externalservice/ServiceInfo.java new file mode 100644 index 0000000000000..f9ef9a73c02a8 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/externalservice/ServiceInfo.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.externalservice; + +import org.apache.iotdb.externalservice.api.IExternalService; + +import com.google.common.base.Objects; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; + +public class ServiceInfo { + private final String serviceName; + private final String className; + // This field needn't to serde, only USER_DEFINED service will be persisted on CN + private final transient ServiceType serviceType; + private State state; + + private transient IExternalService serviceInstance; + + public ServiceInfo(String serviceName, String className, ServiceType serviceType) { + this.serviceName = serviceName; + this.className = className; + this.serviceType = serviceType; + this.state = State.STOPPED; + } + + public ServiceInfo(String serviceName, String className, ServiceType serviceType, State state) { + this.serviceName = serviceName; + this.className = className; + this.serviceType = serviceType; + this.state = state; + } + + public String getClassName() { + return className; + } + + public State getState() { + return state; + } + + public void setState(State state) { + this.state = state; + } + + public ServiceType getServiceType() { + return serviceType; + } + + public String getServiceName() { + return serviceName; + } + + public IExternalService getServiceInstance() { + return serviceInstance; + } + + public void setServiceInstance(IExternalService serviceInstance) { + this.serviceInstance = serviceInstance; + } + + public void serialize(OutputStream stream) throws IOException { + ReadWriteIOUtils.write(serviceName, stream); + ReadWriteIOUtils.write(className, stream); + ReadWriteIOUtils.write(state.getValue(), stream); + } + + public static ServiceInfo deserialize(ByteBuffer buffer) { + String serviceName = ReadWriteIOUtils.readString(buffer); + String className = ReadWriteIOUtils.readString(buffer); + State state = State.deserialize(ReadWriteIOUtils.readByte(buffer)); + return new ServiceInfo(serviceName, className, ServiceType.USER_DEFINED, state); + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + ServiceInfo that = (ServiceInfo) o; + return Objects.equal(serviceName, that.serviceName) + && Objects.equal(className, that.className) + && serviceType == that.serviceType + && state == that.state; + } + + @Override + public int hashCode() { + return Objects.hashCode(serviceName, className, serviceType, state); + } + + public enum ServiceType { + BUILTIN((byte) 0, "built-in"), + USER_DEFINED((byte) 1, "user-defined"); + + private final byte value; + + private final String displayName; + + ServiceType(byte value, String displayName) { + this.value = value; + this.displayName = displayName; + } + + public byte getValue() { + return value; + } + + public String getDisplayName() { + return displayName; + } + + public static ServiceType deserialize(byte t) { + switch (t) { + case 0: + return BUILTIN; + case 1: + return USER_DEFINED; + default: + throw new IllegalArgumentException("Unknown ServiceType: " + t); + } + } + } + + public enum State { + RUNNING((byte) 0), + STOPPED((byte) 1); + + private final byte value; + + State(byte value) { + this.value = value; + } + + public byte getValue() { + return value; + } + + public static State deserialize(byte t) { + switch (t) { + case 0: + return RUNNING; + case 1: + return STOPPED; + default: + throw new IllegalArgumentException("Unknown State: " + t); + } + } + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java index 0459d4d2c86e1..6da2d0e0e6ebb 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java @@ -119,6 +119,13 @@ private ColumnHeaderConstant() { public static final String PATH_PATTERN = "PathPattern"; public static final String CLASS_NAME = "ClassName"; + // column names for show services statement + public static final String SERVICE_NAME = "ServiceName"; + public static final String SERVICE_TYPE = "ServiceType"; + + public static final String SERVICE_NAME_TABLE_MODEL = "service_name"; + public static final String SERVICE_TYPE_TABLE_MODEL = "service_type"; + // column names for show pipe plugins statement public static final String PLUGIN_NAME = "PluginName"; public static final String PLUGIN_TYPE = "PluginType"; @@ -546,6 +553,14 @@ private ColumnHeaderConstant() { new ColumnHeader(CLASS_NAME, TSDataType.TEXT), new ColumnHeader(NODE_ID, TSDataType.TEXT)); + public static final List showExternalServiceColumnHeaders = + ImmutableList.of( + new ColumnHeader(SERVICE_NAME, TSDataType.TEXT), + new ColumnHeader(DATA_NODE_ID, TSDataType.INT32), + new ColumnHeader(STATE, TSDataType.TEXT), + new ColumnHeader(CLASS_NAME, TSDataType.TEXT), + new ColumnHeader(SERVICE_TYPE, TSDataType.TEXT)); + public static final List showPipePluginsColumnHeaders = ImmutableList.of( new ColumnHeader(PLUGIN_NAME, TSDataType.TEXT), diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java index 4f458d34e05a5..c9a994abf6564 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java @@ -52,6 +52,7 @@ public class InformationSchema { public static final String CONNECTIONS = "connections"; public static final String CURRENT_QUERIES = "current_queries"; public static final String QUERIES_COSTS_HISTOGRAM = "queries_costs_histogram"; + public static final String SERVICES = "services"; static { final TsTable queriesTable = new TsTable(QUERIES); @@ -394,6 +395,21 @@ public class InformationSchema { new AttributeColumnSchema(ColumnHeaderConstant.DATANODE_ID, TSDataType.INT32)); queriesCostsHistogramTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(QUERIES_COSTS_HISTOGRAM, queriesCostsHistogramTable); + + final TsTable servicesTable = new TsTable(SERVICES); + servicesTable.addColumnSchema( + new TagColumnSchema(ColumnHeaderConstant.SERVICE_NAME_TABLE_MODEL, TSDataType.STRING)); + servicesTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.DATA_NODE_ID_TABLE_MODEL, TSDataType.INT32)); + servicesTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.STATE_TABLE_MODEL, TSDataType.STRING)); + servicesTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.CLASS_NAME_TABLE_MODEL, TSDataType.STRING)); + servicesTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.SERVICE_TYPE_TABLE_MODEL, TSDataType.STRING)); + servicesTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); + schemaTables.put(SERVICES, servicesTable); } public static Map getSchemaTables() { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/ServiceType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/ServiceType.java index 7267c79a66551..492184d0df0bd 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/ServiceType.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/ServiceType.java @@ -26,6 +26,7 @@ public enum ServiceType { RPC_SERVICE("RPC ServerService", "RPCService"), INFLUX_SERVICE("InfluxDB Protocol Service", "InfluxDB Protocol"), MQTT_SERVICE("MQTTService", "MqttService"), + EXTERNAL_SERVICE("ExternalService Service", "ExternalService"), AIR_GAP_SERVICE("AirGapService", "AirGapService"), MONITOR_SERVICE("Monitor ServerService", "Monitor"), STAT_MONITOR_SERVICE("Statistics ServerService", "StatMonitorService"), diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index 68c29a7a0f413..f2a3bc4947664 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -84,6 +84,13 @@ statement | dropFunctionStatement | createFunctionStatement + // ExternalService Statement + | createServiceStatement + | startServiceStatement + | stopServiceStatement + | dropServiceStatement + | showServiceStatement + // Load Statement | loadTsFileStatement @@ -374,6 +381,27 @@ showFunctionsStatement ; +// -------------------------------------------- ExternalService Statement ---------------------------------------------------------- +createServiceStatement + : CREATE SERVICE serviceName=identifier + AS className=string + ; + +startServiceStatement + : START SERVICE serviceName=identifier + ; + +stopServiceStatement + : STOP SERVICE serviceName=identifier + ; + +dropServiceStatement + : DROP SERVICE serviceName=identifier FORCEDLY? + ; + +showServiceStatement + : SHOW SERVICES (ON targetDataNodeId=INTEGER_VALUE)? + ; // -------------------------------------------- Load Statement --------------------------------------------------------- loadTsFileStatement @@ -1576,6 +1604,7 @@ FIRST: 'FIRST'; FLUSH: 'FLUSH'; FOLLOWING: 'FOLLOWING'; FOR: 'FOR'; +FORCEDLY: 'FORCEDLY'; FORMAT: 'FORMAT'; FROM: 'FROM'; FULL: 'FULL'; @@ -1764,6 +1793,8 @@ SECURITY: 'SECURITY'; SEEK: 'SEEK'; SELECT: 'SELECT'; SERIALIZABLE: 'SERIALIZABLE'; +SERVICE: 'SERVICE'; +SERVICES: 'SERVICES'; SESSION: 'SESSION'; SET: 'SET'; SETS: 'SETS'; diff --git a/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift b/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift index 58c4484a4ae88..df69806250eb8 100644 --- a/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift +++ b/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift @@ -261,6 +261,20 @@ struct TNodeLocations { 2: optional list dataNodeLocations } +struct TExternalServiceListResp { + 1: required TSStatus status + 2: required list externalServiceInfos +} + + +struct TExternalServiceEntry { + 1: required string serviceName + 2: required string className + 3: required byte state + 4: required i32 dataNodId + 5: required byte serviceType +} + enum TAggregationType { COUNT, AVG, diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 9680f11f138c7..2beef6f2570e5 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -1089,6 +1089,14 @@ struct TShowCQResp { 2: required list cqList } +// ==================================================== +// ExternalService +// ==================================================== +struct TCreateExternalServiceReq { + 1: required i32 dataNodeId + 2: required string serviceName + 3: required string className +} struct TDeactivateSchemaTemplateReq { 1: required string queryId @@ -1983,6 +1991,19 @@ service IConfigNodeRPCService { */ TShowCQResp showCQ() + // ==================================================== + // ExternalService + // ==================================================== + common.TSStatus createExternalService(TCreateExternalServiceReq req) + + common.TSStatus startExternalService(i32 dataNodeId, string serviceName) + + common.TSStatus stopExternalService(i32 dataNodeId, string serviceName) + + common.TSStatus dropExternalService(i32 dataNodeId, string serviceName) + + common.TExternalServiceListResp showExternalService(i32 dataNodeId) + // ====================================================== // Quota // ====================================================== diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index 585e5e36be58d..6166e29161aae 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -1027,6 +1027,11 @@ service IDataNodeRPCService { **/ common.TSStatus dropPipePlugin(TDropPipePluginInstanceReq req) + /** + * Config node will get built-in services info from data nodes. + **/ + common.TExternalServiceListResp getBuiltInService() + /* Maintenance Tools */ common.TSStatus merge()