diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java index 37b8907b454a..61bcd5368d83 100644 --- a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java +++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java @@ -18,6 +18,7 @@ import org.apache.cloudstack.acl.ControlledEntity; +import java.util.List; import java.util.Map; import com.cloud.user.Account; @@ -36,5 +37,6 @@ enum KubernetesClusterNodeType { boolean isValidNodeType(String nodeType); Map getServiceOfferingNodeTypeMap(Map> serviceOfferingNodeTypeMap); Map getTemplateNodeTypeMap(Map> templateNodeTypeMap); + Map> getAffinityGroupNodeTypeMap(Map> affinityGroupNodeTypeMap); void cleanupForAccount(Account account); } diff --git a/api/src/main/java/com/cloud/vm/VmDetailConstants.java b/api/src/main/java/com/cloud/vm/VmDetailConstants.java index 596c861218f0..217e9f9224c4 100644 --- a/api/src/main/java/com/cloud/vm/VmDetailConstants.java +++ b/api/src/main/java/com/cloud/vm/VmDetailConstants.java @@ -93,6 +93,7 @@ public interface VmDetailConstants { String CKS_NODE_TYPE = "node"; String OFFERING = "offering"; String TEMPLATE = "template"; + String AFFINITY_GROUP = "affinitygroup"; // VMware to KVM VM migrations specific String VMWARE_TO_KVM_PREFIX = "vmware-to-kvm"; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 3e36d933772b..650f21284b93 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -1236,6 +1236,13 @@ public class ApiConstants { public static final String MAX_SIZE = "maxsize"; public static final String NODE_TYPE_OFFERING_MAP = "nodeofferings"; public static final String NODE_TYPE_TEMPLATE_MAP = "nodetemplates"; + public static final String NODE_TYPE_AFFINITY_GROUP_MAP = "nodeaffinitygroups"; + public static final String CONTROL_AFFINITY_GROUP_IDS = "controlaffinitygroupids"; + public static final String CONTROL_AFFINITY_GROUP_NAMES = "controlaffinitygroupnames"; + public static final String WORKER_AFFINITY_GROUP_IDS = "workeraffinitygroupids"; + public static final String WORKER_AFFINITY_GROUP_NAMES = "workeraffinitygroupnames"; + public static final String ETCD_AFFINITY_GROUP_IDS = "etcdaffinitygroupids"; + public static final String ETCD_AFFINITY_GROUP_NAMES = "etcdaffinitygroupnames"; public static final String BOOT_TYPE = "boottype"; public static final String BOOT_MODE = "bootmode"; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql index 07f394b19c90..a25074c18964 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql @@ -24,6 +24,19 @@ UPDATE `cloud`.`configuration` SET value='random' WHERE name IN ('vm.allocation.algorithm', 'volume.allocation.algorithm') AND value='userconcentratedpod_random'; UPDATE `cloud`.`configuration` SET value='firstfit' WHERE name IN ('vm.allocation.algorithm', 'volume.allocation.algorithm') AND value='userconcentratedpod_firstfit'; +-- Create kubernetes_cluster_affinity_group_map table for CKS per-node-type affinity groups +CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_cluster_affinity_group_map` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `cluster_id` bigint unsigned NOT NULL COMMENT 'kubernetes cluster id', + `node_type` varchar(32) NOT NULL COMMENT 'CONTROL, WORKER, or ETCD', + `affinity_group_id` bigint unsigned NOT NULL COMMENT 'affinity group id', + PRIMARY KEY (`id`), + CONSTRAINT `fk_kubernetes_cluster_ag_map__cluster_id` FOREIGN KEY (`cluster_id`) REFERENCES `kubernetes_cluster`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_kubernetes_cluster_ag_map__ag_id` FOREIGN KEY (`affinity_group_id`) REFERENCES `affinity_group`(`id`) ON DELETE CASCADE, + INDEX `i_kubernetes_cluster_ag_map__cluster_id`(`cluster_id`), + INDEX `i_kubernetes_cluster_ag_map__ag_id`(`affinity_group_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + -- Create webhook_filter table DROP TABLE IF EXISTS `cloud`.`webhook_filter`; CREATE TABLE IF NOT EXISTS `cloud`.`webhook_filter` ( diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterAffinityGroupMapVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterAffinityGroupMapVO.java new file mode 100644 index 000000000000..19babc86690d --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterAffinityGroupMapVO.java @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import org.apache.cloudstack.api.InternalIdentity; + +@Entity +@Table(name = "kubernetes_cluster_affinity_group_map") +public class KubernetesClusterAffinityGroupMapVO implements InternalIdentity { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private Long id; + + @Column(name = "cluster_id") + private long clusterId; + + @Column(name = "node_type") + private String nodeType; + + @Column(name = "affinity_group_id") + private long affinityGroupId; + + public KubernetesClusterAffinityGroupMapVO() { + } + + public KubernetesClusterAffinityGroupMapVO(long clusterId, String nodeType, long affinityGroupId) { + this.clusterId = clusterId; + this.nodeType = nodeType; + this.affinityGroupId = affinityGroupId; + } + + @Override + public long getId() { + return id; + } + + public long getClusterId() { + return clusterId; + } + + public void setClusterId(long clusterId) { + this.clusterId = clusterId; + } + + public String getNodeType() { + return nodeType; + } + + public void setNodeType(String nodeType) { + this.nodeType = nodeType; + } + + public long getAffinityGroupId() { + return affinityGroupId; + } + + public void setAffinityGroupId(long affinityGroupId) { + this.affinityGroupId = affinityGroupId; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index e6ed850fba58..2dad9cc26a71 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -169,6 +169,7 @@ import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterStartWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterStopWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterUpgradeWorker; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterAffinityGroupMapDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; @@ -315,6 +316,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne @Inject public KubernetesClusterDetailsDao kubernetesClusterDetailsDao; @Inject + public KubernetesClusterAffinityGroupMapDao kubernetesClusterAffinityGroupMapDao; + @Inject public KubernetesSupportedVersionDao kubernetesSupportedVersionDao; @Inject protected SSHKeyPairDao sshKeyPairDao; @@ -858,24 +861,38 @@ public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetes List vmResponses = new ArrayList<>(); List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); - ResponseView respView = ResponseView.Restricted; + ResponseView userVmResponseView = ResponseView.Restricted; Account caller = CallContext.current().getCallingAccount(); if (accountService.isRootAdmin(caller.getId())) { - respView = ResponseView.Full; + userVmResponseView = ResponseView.Full; } final String responseName = "virtualmachine"; if (vmList != null && !vmList.isEmpty()) { - for (KubernetesClusterVmMapVO vmMapVO : vmList) { - UserVmJoinVO userVM = userVmJoinDao.findById(vmMapVO.getVmId()); - if (userVM != null) { - UserVmResponse vmResponse = ApiDBUtils.newUserVmResponse(respView, responseName, userVM, - EnumSet.of(VMDetails.nics), caller); + Map vmMapById = vmList.stream() + .collect(Collectors.toMap(KubernetesClusterVmMapVO::getVmId, vm -> vm)); + Long[] vmIds = vmMapById.keySet().toArray(new Long[0]); + List userVmJoinVOs = userVmJoinDao.searchByIds(vmIds); + if (userVmJoinVOs != null && !userVmJoinVOs.isEmpty()) { + Map vmResponseMap = new HashMap<>(); + for (UserVmJoinVO userVM : userVmJoinVOs) { + Long vmId = userVM.getId(); + UserVmResponse vmResponse = vmResponseMap.get(vmId); + if (vmResponse == null) { + vmResponse = ApiDBUtils.newUserVmResponse(userVmResponseView, responseName, userVM, + EnumSet.of(VMDetails.nics, VMDetails.affgrp), caller); + vmResponseMap.put(vmId, vmResponse); + } else { + ApiDBUtils.fillVmDetails(userVmResponseView, vmResponse, userVM); + } + } + for (Map.Entry vmIdResponseEntry : vmResponseMap.entrySet()) { KubernetesUserVmResponse kubernetesUserVmResponse = new KubernetesUserVmResponse(); try { - BeanUtils.copyProperties(kubernetesUserVmResponse, vmResponse); + BeanUtils.copyProperties(kubernetesUserVmResponse, vmIdResponseEntry.getValue()); } catch (IllegalAccessException | InvocationTargetException e) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to generate zone metrics response"); } + KubernetesClusterVmMapVO vmMapVO = vmMapById.get(vmIdResponseEntry.getKey()); kubernetesUserVmResponse.setExternalNode(vmMapVO.isExternalNode()); kubernetesUserVmResponse.setEtcdNode(vmMapVO.isEtcdNode()); kubernetesUserVmResponse.setNodeVersion(vmMapVO.getNodeVersion()); @@ -905,10 +922,45 @@ public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetes response.setClusterType(kubernetesCluster.getClusterType()); response.setCsiEnabled(kubernetesCluster.isCsiEnabled()); response.setCreated(kubernetesCluster.getCreated()); + setNodeTypeAffinityGroupResponse(response, kubernetesCluster.getId()); return response; } + protected void setNodeTypeAffinityGroupResponse(KubernetesClusterResponse response, long clusterId) { + setAffinityGroupResponseForNodeType(response, clusterId, CONTROL.name()); + setAffinityGroupResponseForNodeType(response, clusterId, WORKER.name()); + setAffinityGroupResponseForNodeType(response, clusterId, ETCD.name()); + } + + protected void setAffinityGroupResponseForNodeType(KubernetesClusterResponse response, long clusterId, String nodeType) { + List affinityGroupIds = kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(clusterId, nodeType); + if (affinityGroupIds == null || affinityGroupIds.isEmpty()) { + return; + } + List affinityGroupUuids = new ArrayList<>(); + List affinityGroupNames = new ArrayList<>(); + for (Long affinityGroupId : affinityGroupIds) { + AffinityGroupVO affinityGroup = affinityGroupDao.findById(affinityGroupId); + if (affinityGroup != null) { + affinityGroupUuids.add(affinityGroup.getUuid()); + affinityGroupNames.add(affinityGroup.getName()); + } + } + String affinityGroupUuidsCsv = String.join(",", affinityGroupUuids); + String affinityGroupNamesCsv = String.join(",", affinityGroupNames); + if (CONTROL.name().equals(nodeType)) { + response.setControlAffinityGroupIds(affinityGroupUuidsCsv); + response.setControlAffinityGroupNames(affinityGroupNamesCsv); + } else if (WORKER.name().equals(nodeType)) { + response.setWorkerAffinityGroupIds(affinityGroupUuidsCsv); + response.setWorkerAffinityGroupNames(affinityGroupNamesCsv); + } else if (ETCD.name().equals(nodeType)) { + response.setEtcdAffinityGroupIds(affinityGroupUuidsCsv); + response.setEtcdAffinityGroupNames(affinityGroupNamesCsv); + } + } + private DataCenter validateAndGetZoneForKubernetesCreateParameters(Long zoneId, Long networkId) { DataCenter zone = dataCenterDao.findById(zoneId); if (zone == null) { @@ -1187,6 +1239,20 @@ private Network getKubernetesClusterNetworkIfMissing(final String clusterName, f return network; } + private void persistAffinityGroupMappings(long clusterId, Map> affinityGroupNodeTypeMap) { + if (MapUtils.isEmpty(affinityGroupNodeTypeMap)) { + return; + } + for (Map.Entry> nodeTypeAffinityGroupEntry : affinityGroupNodeTypeMap.entrySet()) { + String nodeType = nodeTypeAffinityGroupEntry.getKey(); + List affinityGroupIds = nodeTypeAffinityGroupEntry.getValue(); + for (Long affinityGroupId : affinityGroupIds) { + kubernetesClusterAffinityGroupMapDao.persist( + new KubernetesClusterAffinityGroupMapVO(clusterId, nodeType, affinityGroupId)); + } + } + } + private void addKubernetesClusterDetails(final KubernetesCluster kubernetesCluster, final Network network, final CreateKubernetesClusterCmd cmd) { final String externalLoadBalancerIpAddress = cmd.getExternalLoadBalancerIpAddress(); final String dockerRegistryUserName = cmd.getDockerRegistryUserName(); @@ -1627,6 +1693,7 @@ public KubernetesCluster createManagedKubernetesCluster(CreateKubernetesClusterC } Map templateNodeTypeMap = cmd.getTemplateNodeTypeMap(); + Map> affinityGroupNodeTypeMap = cmd.getAffinityGroupNodeTypeMap(); final VMTemplateVO finalTemplate = getKubernetesServiceTemplate(zone, hypervisorType, templateNodeTypeMap, DEFAULT, clusterKubernetesVersion); final VMTemplateVO controlNodeTemplate = getKubernetesServiceTemplate(zone, hypervisorType, templateNodeTypeMap, CONTROL, clusterKubernetesVersion); final VMTemplateVO workerNodeTemplate = getKubernetesServiceTemplate(zone, hypervisorType, templateNodeTypeMap, WORKER, clusterKubernetesVersion); @@ -1672,6 +1739,7 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { } newCluster.setCsiEnabled(cmd.getEnableCsi()); kubernetesClusterDao.persist(newCluster); + persistAffinityGroupMappings(newCluster.getId(), affinityGroupNodeTypeMap); addKubernetesClusterDetails(newCluster, defaultNetwork, cmd); return newCluster; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java index 30465c99780d..62712514b2d0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java @@ -18,12 +18,16 @@ import java.lang.reflect.Field; import java.lang.reflect.Modifier; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import javax.inject.Inject; +import org.apache.cloudstack.affinity.AffinityGroup; +import org.apache.cloudstack.affinity.dao.AffinityGroupDao; import com.cloud.exception.InvalidParameterValueException; import com.cloud.offering.ServiceOffering; import com.cloud.service.dao.ServiceOfferingDao; @@ -66,6 +70,8 @@ public class KubernetesServiceHelperImpl extends AdapterBase implements Kubernet @Inject protected VMTemplateDao vmTemplateDao; @Inject + protected AffinityGroupDao affinityGroupDao; + @Inject KubernetesClusterService kubernetesClusterService; protected void setEventTypeEntityDetails(Class eventTypeDefinedClass, Class entityClass) { @@ -244,6 +250,81 @@ public Map getTemplateNodeTypeMap(Map> return mapping; } + protected void checkNodeTypeAffinityGroupEntryCompleteness(String nodeType, String affinityGroupUuids) { + if (StringUtils.isAnyBlank(nodeType, affinityGroupUuids)) { + String error = String.format("Any Node Type to Affinity Group entry should have a valid '%s' and '%s' values", + VmDetailConstants.CKS_NODE_TYPE, VmDetailConstants.AFFINITY_GROUP); + logger.error(error); + throw new InvalidParameterValueException(error); + } + } + + protected void checkNodeTypeAffinityGroupEntryNodeType(String nodeType) { + if (!isValidNodeType(nodeType)) { + String error = String.format("The provided value '%s' for Node Type is invalid", nodeType); + logger.error(error); + throw new InvalidParameterValueException(error); + } + } + + protected Long validateAffinityGroupUuidAndGetId(String affinityGroupUuid) { + if (StringUtils.isBlank(affinityGroupUuid)) { + String error = "Empty affinity group UUID provided"; + logger.error(error); + throw new InvalidParameterValueException(error); + } + AffinityGroup affinityGroup = affinityGroupDao.findByUuid(affinityGroupUuid); + if (affinityGroup == null) { + String error = String.format("Cannot find an affinity group with ID %s", affinityGroupUuid); + logger.error(error); + throw new InvalidParameterValueException(error); + } + return affinityGroup.getId(); + } + + protected List validateAndGetAffinityGroupIds(String affinityGroupUuids) { + String[] uuids = affinityGroupUuids.split(","); + List affinityGroupIds = new ArrayList<>(); + for (String uuid : uuids) { + String trimmedUuid = uuid.trim(); + Long affinityGroupId = validateAffinityGroupUuidAndGetId(trimmedUuid); + affinityGroupIds.add(affinityGroupId); + } + return affinityGroupIds; + } + + protected void addNodeTypeAffinityGroupEntry(String nodeType, List affinityGroupIds, Map> nodeTypeToAffinityGroupIds) { + if (logger.isDebugEnabled()) { + logger.debug(String.format("Node Type: '%s' should use affinity group IDs: '%s'", nodeType, affinityGroupIds)); + } + KubernetesClusterNodeType clusterNodeType = KubernetesClusterNodeType.valueOf(nodeType.toUpperCase()); + nodeTypeToAffinityGroupIds.put(clusterNodeType.name(), affinityGroupIds); + } + + protected void processNodeTypeAffinityGroupEntryAndAddToMappingIfValid(Map nodeTypeAffinityConfig, Map> nodeTypeToAffinityGroupIds) { + if (MapUtils.isEmpty(nodeTypeAffinityConfig)) { + return; + } + String nodeType = nodeTypeAffinityConfig.get(VmDetailConstants.CKS_NODE_TYPE); + String affinityGroupUuids = nodeTypeAffinityConfig.get(VmDetailConstants.AFFINITY_GROUP); + checkNodeTypeAffinityGroupEntryCompleteness(nodeType, affinityGroupUuids); + checkNodeTypeAffinityGroupEntryNodeType(nodeType); + + List affinityGroupIds = validateAndGetAffinityGroupIds(affinityGroupUuids); + addNodeTypeAffinityGroupEntry(nodeType, affinityGroupIds, nodeTypeToAffinityGroupIds); + } + + @Override + public Map> getAffinityGroupNodeTypeMap(Map> affinityGroupNodeTypeMap) { + Map> nodeTypeToAffinityGroupIds = new HashMap<>(); + if (MapUtils.isNotEmpty(affinityGroupNodeTypeMap)) { + for (Map nodeTypeAffinityConfig : affinityGroupNodeTypeMap.values()) { + processNodeTypeAffinityGroupEntryAndAddToMappingIfValid(nodeTypeAffinityConfig, nodeTypeToAffinityGroupIds); + } + } + return nodeTypeToAffinityGroupIds; + } + public void cleanupForAccount(Account account) { kubernetesClusterService.cleanupForAccount(account); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 1d4b6e8d0a84..3e90cbd25d2e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -90,6 +90,7 @@ import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterAffinityGroupMapDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; @@ -217,6 +218,7 @@ public class KubernetesClusterActionWorker { protected KubernetesClusterDao kubernetesClusterDao; protected KubernetesClusterVmMapDao kubernetesClusterVmMapDao; protected KubernetesClusterDetailsDao kubernetesClusterDetailsDao; + protected KubernetesClusterAffinityGroupMapDao kubernetesClusterAffinityGroupMapDao; protected KubernetesSupportedVersionDao kubernetesSupportedVersionDao; protected KubernetesCluster kubernetesCluster; @@ -251,6 +253,7 @@ protected KubernetesClusterActionWorker(final KubernetesCluster kubernetesCluste this.kubernetesClusterDao = clusterManager.kubernetesClusterDao; this.kubernetesClusterDetailsDao = clusterManager.kubernetesClusterDetailsDao; this.kubernetesClusterVmMapDao = clusterManager.kubernetesClusterVmMapDao; + this.kubernetesClusterAffinityGroupMapDao = clusterManager.kubernetesClusterAffinityGroupMapDao; this.kubernetesSupportedVersionDao = clusterManager.kubernetesSupportedVersionDao; this.manager = clusterManager; } @@ -1112,4 +1115,18 @@ public Long getExplicitAffinityGroup(Long domainId, Long accountId) { } return null; } + + protected List getAffinityGroupIdsForNodeType(KubernetesClusterNodeType nodeType) { + return new ArrayList<>(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType( + kubernetesCluster.getId(), nodeType.name())); + } + + protected List getMergedAffinityGroupIds(KubernetesClusterNodeType nodeType, Long domainId, Long accountId) { + List affinityGroupIds = getAffinityGroupIdsForNodeType(nodeType); + Long explicitAffinityGroupId = getExplicitAffinityGroup(domainId, accountId); + if (explicitAffinityGroupId != null && !affinityGroupIds.contains(explicitAffinityGroupId)) { + affinityGroupIds.add(explicitAffinityGroupId); + } + return affinityGroupIds.isEmpty() ? null : affinityGroupIds; + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java index 62bd8b4576a4..dc886117b22e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -348,6 +348,7 @@ public boolean destroy() throws CloudRuntimeException { stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); annotationDao.removeByEntityType(AnnotationService.EntityType.KUBERNETES_CLUSTER.name(), kubernetesCluster.getUuid()); kubernetesClusterDetailsDao.removeDetails(kubernetesCluster.getId()); + kubernetesClusterAffinityGroupMapDao.removeByClusterId(kubernetesCluster.getId()); boolean deleted = kubernetesClusterDao.remove(kubernetesCluster.getId()); if (!deleted) { logMessage(Level.WARN, String.format("Failed to delete Kubernetes cluster: %s", kubernetesCluster), null); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index cf69234d19e0..7442a1eccb46 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -26,7 +26,6 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -426,21 +425,19 @@ protected UserVm createKubernetesNode(String joinIp, Long domainId, Long account if (StringUtils.isNotBlank(kubernetesCluster.getKeyPair())) { keypairs.add(kubernetesCluster.getKeyPair()); } - Long affinityGroupId = getExplicitAffinityGroup(domainId, accountId); + List affinityGroupIds = getMergedAffinityGroupIds(WORKER, domainId, accountId); if (kubernetesCluster.getSecurityGroupId() != null && networkModel.checkSecurityGroupSupportForNetwork(owner, zone, networkIds, List.of(kubernetesCluster.getSecurityGroupId()))) { List securityGroupIds = new ArrayList<>(); securityGroupIds.add(kubernetesCluster.getSecurityGroupId()); nodeVm = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, workerNodeTemplate, networkIds, securityGroupIds, owner, hostName, hostName, null, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, null, null, keypairs, - null, addrs, null, null, Objects.nonNull(affinityGroupId) ? - Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, + null, addrs, null, null, affinityGroupIds, customParameterMap, null, null, null, null, true, null, UserVmManager.CKS_NODE, null, null); } else { nodeVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, workerNodeTemplate, networkIds, owner, hostName, hostName, null, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, null, null, keypairs, - null, addrs, null, null, Objects.nonNull(affinityGroupId) ? - Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null, null, null); + null, addrs, null, null, affinityGroupIds, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null, null, null); } if (logger.isInfoEnabled()) { logger.info("Created node VM : {}, {} in the Kubernetes cluster : {}", hostName, nodeVm, kubernetesCluster.getName()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index aa9317e619b0..4ed5ff0167c2 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -270,7 +270,7 @@ private Pair createKubernetesControlNode(final Network network, S keypairs.add(kubernetesCluster.getKeyPair()); } - Long affinityGroupId = getExplicitAffinityGroup(domainId, accountId); + List affinityGroupIds = getMergedAffinityGroupIds(CONTROL, domainId, accountId); String userDataDetails = kubernetesCluster.getCniConfigDetails(); if (kubernetesCluster.getSecurityGroupId() != null && networkModel.checkSecurityGroupSupportForNetwork(owner, zone, networkIds, @@ -279,15 +279,13 @@ private Pair createKubernetesControlNode(final Network network, S securityGroupIds.add(kubernetesCluster.getSecurityGroupId()); controlVm = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, controlNodeTemplate, networkIds, securityGroupIds, owner, hostName, hostName, null, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, userDataId, userDataDetails, keypairs, - requestedIps, addrs, null, null, Objects.nonNull(affinityGroupId) ? - Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, + requestedIps, addrs, null, null, affinityGroupIds, customParameterMap, null, null, null, null, true, null, UserVmManager.CKS_NODE, null, null); } else { controlVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, controlNodeTemplate, networkIds, owner, hostName, hostName, null, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, userDataId, userDataDetails, keypairs, - requestedIps, addrs, null, null, Objects.nonNull(affinityGroupId) ? - Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null, null, null); + requestedIps, addrs, null, null, affinityGroupIds, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null, null, null); } if (logger.isInfoEnabled()) { logger.info("Created control VM: {}, {} in the Kubernetes cluster: {}", controlVm, hostName, kubernetesCluster); @@ -439,7 +437,7 @@ private UserVm createKubernetesAdditionalControlNode(final String joinIp, final keypairs.add(kubernetesCluster.getKeyPair()); } - Long affinityGroupId = getExplicitAffinityGroup(domainId, accountId); + List affinityGroupIds = getMergedAffinityGroupIds(CONTROL, domainId, accountId); if (kubernetesCluster.getSecurityGroupId() != null && networkModel.checkSecurityGroupSupportForNetwork(owner, zone, networkIds, List.of(kubernetesCluster.getSecurityGroupId()))) { @@ -447,15 +445,13 @@ private UserVm createKubernetesAdditionalControlNode(final String joinIp, final securityGroupIds.add(kubernetesCluster.getSecurityGroupId()); additionalControlVm = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, controlNodeTemplate, networkIds, securityGroupIds, owner, hostName, hostName, null, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, null, null, keypairs, - null, addrs, null, null, Objects.nonNull(affinityGroupId) ? - Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, + null, addrs, null, null, affinityGroupIds, customParameterMap, null, null, null, null, true, null, UserVmManager.CKS_NODE, null, null); } else { additionalControlVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, controlNodeTemplate, networkIds, owner, hostName, hostName, null, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, null, null, keypairs, - null, addrs, null, null, Objects.nonNull(affinityGroupId) ? - Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null, null, null); + null, addrs, null, null, affinityGroupIds, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null, null, null); } if (logger.isInfoEnabled()) { @@ -483,7 +479,7 @@ private UserVm createEtcdNode(List requestedIps, List affinityGroupIds = getMergedAffinityGroupIds(ETCD, domainId, accountId); String hostName = etcdNodeHostnames.get(etcdNodeIndex); Map customParameterMap = new HashMap(); if (zone.isSecurityGroupEnabled()) { @@ -491,15 +487,13 @@ private UserVm createEtcdNode(List requestedIps, List { + + List listByClusterIdAndNodeType(long clusterId, String nodeType); + + List listAffinityGroupIdsByClusterIdAndNodeType(long clusterId, String nodeType); + + int removeByClusterId(long clusterId); +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterAffinityGroupMapDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterAffinityGroupMapDaoImpl.java new file mode 100644 index 000000000000..8b51d1b48c9e --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterAffinityGroupMapDaoImpl.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.dao; + +import java.util.List; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Component; + +import com.cloud.kubernetes.cluster.KubernetesClusterAffinityGroupMapVO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@Component +public class KubernetesClusterAffinityGroupMapDaoImpl extends GenericDaoBase + implements KubernetesClusterAffinityGroupMapDao { + + private final SearchBuilder clusterIdAndNodeTypeSearch; + private final SearchBuilder clusterIdSearch; + + public KubernetesClusterAffinityGroupMapDaoImpl() { + clusterIdAndNodeTypeSearch = createSearchBuilder(); + clusterIdAndNodeTypeSearch.and("clusterId", clusterIdAndNodeTypeSearch.entity().getClusterId(), SearchCriteria.Op.EQ); + clusterIdAndNodeTypeSearch.and("nodeType", clusterIdAndNodeTypeSearch.entity().getNodeType(), SearchCriteria.Op.EQ); + clusterIdAndNodeTypeSearch.done(); + + clusterIdSearch = createSearchBuilder(); + clusterIdSearch.and("clusterId", clusterIdSearch.entity().getClusterId(), SearchCriteria.Op.EQ); + clusterIdSearch.done(); + } + + @Override + public List listByClusterIdAndNodeType(long clusterId, String nodeType) { + SearchCriteria sc = clusterIdAndNodeTypeSearch.create(); + sc.setParameters("clusterId", clusterId); + sc.setParameters("nodeType", nodeType); + return listBy(sc); + } + + @Override + public List listAffinityGroupIdsByClusterIdAndNodeType(long clusterId, String nodeType) { + List maps = listByClusterIdAndNodeType(clusterId, nodeType); + return maps.stream().map(KubernetesClusterAffinityGroupMapVO::getAffinityGroupId).collect(Collectors.toList()); + } + + @Override + public int removeByClusterId(long clusterId) { + SearchCriteria sc = clusterIdSearch.create(); + sc.setParameters("clusterId", clusterId); + return remove(sc); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java index 9ce6dc3ea788..bac3cd964865 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.api.command.user.kubernetes.cluster; import java.security.InvalidParameterException; +import java.util.List; import java.util.Map; import java.util.Objects; @@ -79,7 +80,7 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { @Inject public KubernetesClusterService kubernetesClusterService; @Inject - protected KubernetesServiceHelper kubernetesClusterHelper; + protected KubernetesServiceHelper kubernetesServiceHelper; @Inject private ConfigurationDao configurationDao; @Inject @@ -125,6 +126,12 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { since = "4.21.0") private Map> templateNodeTypeMap; + @ACL(accessType = AccessType.UseEntry) + @Parameter(name = ApiConstants.NODE_TYPE_AFFINITY_GROUP_MAP, type = CommandType.MAP, + description = "(Optional) Node Type to Affinity Group ID mapping. If provided, VMs of each node type will be added to the specified affinity group", + since = "4.23.0") + private Map> affinityGroupNodeTypeMap; + @ACL(accessType = AccessType.UseEntry) @Parameter(name = ApiConstants.ETCD_NODES, type = CommandType.LONG, description = "(Optional) Number of Kubernetes cluster etcd nodes, default is 0." + @@ -314,11 +321,15 @@ public String getClusterType() { } public Map getServiceOfferingNodeTypeMap() { - return kubernetesClusterHelper.getServiceOfferingNodeTypeMap(serviceOfferingNodeTypeMap); + return kubernetesServiceHelper.getServiceOfferingNodeTypeMap(serviceOfferingNodeTypeMap); } public Map getTemplateNodeTypeMap() { - return kubernetesClusterHelper.getTemplateNodeTypeMap(templateNodeTypeMap); + return kubernetesServiceHelper.getTemplateNodeTypeMap(templateNodeTypeMap); + } + + public Map> getAffinityGroupNodeTypeMap() { + return kubernetesServiceHelper.getAffinityGroupNodeTypeMap(affinityGroupNodeTypeMap); } public Hypervisor.HypervisorType getHypervisorType() { diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java index c7ee0b7da92a..1cff2649428d 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java @@ -57,7 +57,7 @@ public class ScaleKubernetesClusterCmd extends BaseAsyncCmd { @Inject public KubernetesClusterService kubernetesClusterService; @Inject - protected KubernetesServiceHelper kubernetesClusterHelper; + protected KubernetesServiceHelper kubernetesServiceHelper; ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// @@ -114,7 +114,7 @@ public Long getServiceOfferingId() { } public Map getServiceOfferingNodeTypeMap() { - return kubernetesClusterHelper.getServiceOfferingNodeTypeMap(this.serviceOfferingNodeTypeMap); + return kubernetesServiceHelper.getServiceOfferingNodeTypeMap(this.serviceOfferingNodeTypeMap); } public Long getClusterSize() { diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java index 0a7e7a97939d..932d722de354 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java @@ -220,6 +220,30 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple @Param(description = "The date when this Kubernetes cluster was created") private Date created; + @SerializedName(ApiConstants.CONTROL_AFFINITY_GROUP_IDS) + @Param(description = "The IDs of affinity groups associated with control nodes", since = "4.23.0") + private String controlAffinityGroupIds; + + @SerializedName(ApiConstants.CONTROL_AFFINITY_GROUP_NAMES) + @Param(description = "The names of affinity groups associated with control nodes", since = "4.23.0") + private String controlAffinityGroupNames; + + @SerializedName(ApiConstants.WORKER_AFFINITY_GROUP_IDS) + @Param(description = "The IDs of affinity groups associated with worker nodes", since = "4.23.0") + private String workerAffinityGroupIds; + + @SerializedName(ApiConstants.WORKER_AFFINITY_GROUP_NAMES) + @Param(description = "The names of affinity groups associated with worker nodes", since = "4.23.0") + private String workerAffinityGroupNames; + + @SerializedName(ApiConstants.ETCD_AFFINITY_GROUP_IDS) + @Param(description = "The IDs of affinity groups associated with etcd nodes", since = "4.23.0") + private String etcdAffinityGroupIds; + + @SerializedName(ApiConstants.ETCD_AFFINITY_GROUP_NAMES) + @Param(description = "The names of affinity groups associated with etcd nodes", since = "4.23.0") + private String etcdAffinityGroupNames; + public KubernetesClusterResponse() { } @@ -535,4 +559,28 @@ public void setCniConfigName(String cniConfigName) { public void setCsiEnabled(Boolean csiEnabled) { isCsiEnabled = csiEnabled; } + + public void setControlAffinityGroupIds(String controlAffinityGroupIds) { + this.controlAffinityGroupIds = controlAffinityGroupIds; + } + + public void setControlAffinityGroupNames(String controlAffinityGroupNames) { + this.controlAffinityGroupNames = controlAffinityGroupNames; + } + + public void setWorkerAffinityGroupIds(String workerAffinityGroupIds) { + this.workerAffinityGroupIds = workerAffinityGroupIds; + } + + public void setWorkerAffinityGroupNames(String workerAffinityGroupNames) { + this.workerAffinityGroupNames = workerAffinityGroupNames; + } + + public void setEtcdAffinityGroupIds(String etcdAffinityGroupIds) { + this.etcdAffinityGroupIds = etcdAffinityGroupIds; + } + + public void setEtcdAffinityGroupNames(String etcdAffinityGroupNames) { + this.etcdAffinityGroupNames = etcdAffinityGroupNames; + } } diff --git a/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml b/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml index 9d236eed26cd..053366786292 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml +++ b/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml @@ -32,6 +32,7 @@ + diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterAffinityGroupMapVOTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterAffinityGroupMapVOTest.java new file mode 100644 index 000000000000..d0aafc7d1e5e --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterAffinityGroupMapVOTest.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster; + +import org.junit.Assert; +import org.junit.Test; + +public class KubernetesClusterAffinityGroupMapVOTest { + + @Test + public void testConstructorAndGetters() { + KubernetesClusterAffinityGroupMapVO vo = + new KubernetesClusterAffinityGroupMapVO(1L, "CONTROL", 100L); + + Assert.assertEquals(1L, vo.getClusterId()); + Assert.assertEquals("CONTROL", vo.getNodeType()); + Assert.assertEquals(100L, vo.getAffinityGroupId()); + } + + @Test + public void testDefaultConstructor() { + KubernetesClusterAffinityGroupMapVO vo = new KubernetesClusterAffinityGroupMapVO(); + Assert.assertNotNull(vo); + } + + @Test + public void testSetClusterId() { + KubernetesClusterAffinityGroupMapVO vo = new KubernetesClusterAffinityGroupMapVO(); + vo.setClusterId(2L); + Assert.assertEquals(2L, vo.getClusterId()); + } + + @Test + public void testSetNodeType() { + KubernetesClusterAffinityGroupMapVO vo = new KubernetesClusterAffinityGroupMapVO(); + vo.setNodeType("WORKER"); + Assert.assertEquals("WORKER", vo.getNodeType()); + } + + @Test + public void testSetAffinityGroupId() { + KubernetesClusterAffinityGroupMapVO vo = new KubernetesClusterAffinityGroupMapVO(); + vo.setAffinityGroupId(200L); + Assert.assertEquals(200L, vo.getAffinityGroupId()); + } + + @Test + public void testAllNodeTypes() { + KubernetesClusterAffinityGroupMapVO controlVo = + new KubernetesClusterAffinityGroupMapVO(1L, "CONTROL", 10L); + KubernetesClusterAffinityGroupMapVO workerVo = + new KubernetesClusterAffinityGroupMapVO(1L, "WORKER", 20L); + KubernetesClusterAffinityGroupMapVO etcdVo = + new KubernetesClusterAffinityGroupMapVO(1L, "ETCD", 30L); + + Assert.assertEquals("CONTROL", controlVo.getNodeType()); + Assert.assertEquals("WORKER", workerVo.getNodeType()); + Assert.assertEquals("ETCD", etcdVo.getNodeType()); + } + + @Test + public void testSettersChain() { + KubernetesClusterAffinityGroupMapVO vo = new KubernetesClusterAffinityGroupMapVO(); + + vo.setClusterId(5L); + vo.setNodeType("ETCD"); + vo.setAffinityGroupId(500L); + + Assert.assertEquals(5L, vo.getClusterId()); + Assert.assertEquals("ETCD", vo.getNodeType()); + Assert.assertEquals(500L, vo.getAffinityGroupId()); + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImplTest.java deleted file mode 100644 index 298f1dfbcd61..000000000000 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImplTest.java +++ /dev/null @@ -1,145 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.kubernetes.cluster; - -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.service.ServiceOfferingVO; -import com.cloud.service.dao.ServiceOfferingDao; -import com.cloud.vm.VmDetailConstants; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; - -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; -import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.ETCD; -import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; - -@RunWith(MockitoJUnitRunner.class) -public class KubernetesClusterHelperImplTest { - - @Mock - private ServiceOfferingDao serviceOfferingDao; - @Mock - private ServiceOfferingVO workerServiceOffering; - @Mock - private ServiceOfferingVO controlServiceOffering; - @Mock - private ServiceOfferingVO etcdServiceOffering; - - private static final String workerNodesOfferingId = UUID.randomUUID().toString(); - private static final String controlNodesOfferingId = UUID.randomUUID().toString(); - private static final String etcdNodesOfferingId = UUID.randomUUID().toString(); - private static final Long workerOfferingId = 1L; - private static final Long controlOfferingId = 2L; - private static final Long etcdOfferingId = 3L; - - private final KubernetesServiceHelperImpl helper = new KubernetesServiceHelperImpl(); - - @Before - public void setUp() { - helper.serviceOfferingDao = serviceOfferingDao; - Mockito.when(serviceOfferingDao.findByUuid(workerNodesOfferingId)).thenReturn(workerServiceOffering); - Mockito.when(serviceOfferingDao.findByUuid(controlNodesOfferingId)).thenReturn(controlServiceOffering); - Mockito.when(serviceOfferingDao.findByUuid(etcdNodesOfferingId)).thenReturn(etcdServiceOffering); - Mockito.when(workerServiceOffering.getId()).thenReturn(workerOfferingId); - Mockito.when(controlServiceOffering.getId()).thenReturn(controlOfferingId); - Mockito.when(etcdServiceOffering.getId()).thenReturn(etcdOfferingId); - } - - @Test - public void testIsValidNodeTypeEmptyNodeType() { - Assert.assertFalse(helper.isValidNodeType(null)); - } - - @Test - public void testIsValidNodeTypeInvalidNodeType() { - String nodeType = "invalidNodeType"; - Assert.assertFalse(helper.isValidNodeType(nodeType)); - } - - @Test - public void testIsValidNodeTypeValidNodeTypeLowercase() { - String nodeType = KubernetesServiceHelper.KubernetesClusterNodeType.WORKER.name().toLowerCase(); - Assert.assertTrue(helper.isValidNodeType(nodeType)); - } - - private Map createMapEntry(KubernetesServiceHelper.KubernetesClusterNodeType nodeType, - String nodeTypeOfferingUuid) { - Map map = new HashMap<>(); - map.put(VmDetailConstants.CKS_NODE_TYPE, nodeType.name().toLowerCase()); - map.put(VmDetailConstants.OFFERING, nodeTypeOfferingUuid); - return map; - } - - @Test - public void testNodeOfferingMap() { - Map> serviceOfferingNodeTypeMap = new HashMap<>(); - Map firstMap = createMapEntry(WORKER, workerNodesOfferingId); - Map secondMap = createMapEntry(CONTROL, controlNodesOfferingId); - serviceOfferingNodeTypeMap.put("map1", firstMap); - serviceOfferingNodeTypeMap.put("map2", secondMap); - Map map = helper.getServiceOfferingNodeTypeMap(serviceOfferingNodeTypeMap); - Assert.assertNotNull(map); - Assert.assertEquals(2, map.size()); - Assert.assertTrue(map.containsKey(WORKER.name()) && map.containsKey(CONTROL.name())); - Assert.assertEquals(workerOfferingId, map.get(WORKER.name())); - Assert.assertEquals(controlOfferingId, map.get(CONTROL.name())); - } - - @Test - public void testNodeOfferingMapNullMap() { - Map map = helper.getServiceOfferingNodeTypeMap(null); - Assert.assertTrue(map.isEmpty()); - } - - @Test - public void testNodeOfferingMapEtcdNodes() { - Map> serviceOfferingNodeTypeMap = new HashMap<>(); - Map firstMap = createMapEntry(ETCD, etcdNodesOfferingId); - serviceOfferingNodeTypeMap.put("map1", firstMap); - Map map = helper.getServiceOfferingNodeTypeMap(serviceOfferingNodeTypeMap); - Assert.assertNotNull(map); - Assert.assertEquals(1, map.size()); - Assert.assertTrue(map.containsKey(ETCD.name())); - Assert.assertEquals(etcdOfferingId, map.get(ETCD.name())); - } - - @Test(expected = InvalidParameterValueException.class) - public void testCheckNodeTypeOfferingEntryCompletenessInvalidParameters() { - helper.checkNodeTypeOfferingEntryCompleteness(WORKER.name(), null); - } - - @Test(expected = InvalidParameterValueException.class) - public void testCheckNodeTypeOfferingEntryValuesInvalidNodeType() { - String invalidNodeType = "invalidNodeTypeName"; - helper.checkNodeTypeOfferingEntryValues(invalidNodeType, workerServiceOffering, workerNodesOfferingId); - } - - @Test(expected = InvalidParameterValueException.class) - public void testCheckNodeTypeOfferingEntryValuesEmptyOffering() { - String nodeType = WORKER.name(); - helper.checkNodeTypeOfferingEntryValues(nodeType, null, workerNodesOfferingId); - } -} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java index 2a381f282de2..9c5ca5fa110a 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java @@ -26,6 +26,7 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterActionWorker; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterAffinityGroupMapDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; import com.cloud.kubernetes.version.KubernetesSupportedVersion; @@ -46,9 +47,12 @@ import com.cloud.utils.net.NetUtils; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.affinity.AffinityGroupVO; +import org.apache.cloudstack.affinity.dao.AffinityGroupDao; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveVirtualMachinesFromKubernetesClusterCmd; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.commons.collections.MapUtils; @@ -103,6 +107,12 @@ public class KubernetesClusterManagerImplTest { @Mock private ServiceOfferingDao serviceOfferingDao; + @Mock + private KubernetesClusterAffinityGroupMapDao kubernetesClusterAffinityGroupMapDao; + + @Mock + private AffinityGroupDao affinityGroupDao; + @Spy @InjectMocks KubernetesClusterManagerImpl kubernetesClusterManager; @@ -441,4 +451,128 @@ public void testGetCksClusterPreferredArchSameArch() { String cksClusterPreferredArch = kubernetesClusterManager.getCksClusterPreferredArch(systemVMArch, cksIso); Assert.assertEquals(CPU.CPUArch.amd64.getType(), cksClusterPreferredArch); } + + @Test + public void testSetAffinityGroupResponseForNodeTypeControl() { + KubernetesClusterResponse response = new KubernetesClusterResponse(); + long clusterId = 1L; + + AffinityGroupVO ag1 = Mockito.mock(AffinityGroupVO.class); + AffinityGroupVO ag2 = Mockito.mock(AffinityGroupVO.class); + Mockito.when(ag1.getUuid()).thenReturn("uuid-1"); + Mockito.when(ag1.getName()).thenReturn("affinity-group-1"); + Mockito.when(ag2.getUuid()).thenReturn("uuid-2"); + Mockito.when(ag2.getName()).thenReturn("affinity-group-2"); + + Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(clusterId, CONTROL.name())) + .thenReturn(Arrays.asList(1L, 2L)); + Mockito.when(affinityGroupDao.findById(1L)).thenReturn(ag1); + Mockito.when(affinityGroupDao.findById(2L)).thenReturn(ag2); + + kubernetesClusterManager.setAffinityGroupResponseForNodeType(response, clusterId, CONTROL.name()); + + Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(clusterId, CONTROL.name()); + Mockito.verify(affinityGroupDao).findById(1L); + Mockito.verify(affinityGroupDao).findById(2L); + } + + @Test + public void testSetAffinityGroupResponseForNodeTypeWorker() { + KubernetesClusterResponse response = new KubernetesClusterResponse(); + long clusterId = 1L; + + AffinityGroupVO ag = Mockito.mock(AffinityGroupVO.class); + Mockito.when(ag.getUuid()).thenReturn("worker-uuid"); + Mockito.when(ag.getName()).thenReturn("worker-affinity"); + + Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(clusterId, WORKER.name())) + .thenReturn(Arrays.asList(10L)); + Mockito.when(affinityGroupDao.findById(10L)).thenReturn(ag); + + kubernetesClusterManager.setAffinityGroupResponseForNodeType(response, clusterId, WORKER.name()); + + Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(clusterId, WORKER.name()); + Mockito.verify(affinityGroupDao).findById(10L); + } + + @Test + public void testSetAffinityGroupResponseForNodeTypeEtcd() { + KubernetesClusterResponse response = new KubernetesClusterResponse(); + long clusterId = 1L; + + AffinityGroupVO ag = Mockito.mock(AffinityGroupVO.class); + Mockito.when(ag.getUuid()).thenReturn("etcd-uuid"); + Mockito.when(ag.getName()).thenReturn("etcd-affinity"); + + Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(clusterId, ETCD.name())) + .thenReturn(Arrays.asList(20L)); + Mockito.when(affinityGroupDao.findById(20L)).thenReturn(ag); + + kubernetesClusterManager.setAffinityGroupResponseForNodeType(response, clusterId, ETCD.name()); + + Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(clusterId, ETCD.name()); + Mockito.verify(affinityGroupDao).findById(20L); + } + + @Test + public void testSetAffinityGroupResponseForNodeTypeEmptyList() { + KubernetesClusterResponse response = new KubernetesClusterResponse(); + long clusterId = 1L; + + Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(clusterId, CONTROL.name())) + .thenReturn(Collections.emptyList()); + + kubernetesClusterManager.setAffinityGroupResponseForNodeType(response, clusterId, CONTROL.name()); + + Mockito.verify(affinityGroupDao, Mockito.never()).findById(Mockito.anyLong()); + } + + @Test + public void testSetAffinityGroupResponseForNodeTypeNullList() { + KubernetesClusterResponse response = new KubernetesClusterResponse(); + long clusterId = 1L; + + Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(clusterId, ETCD.name())) + .thenReturn(null); + + kubernetesClusterManager.setAffinityGroupResponseForNodeType(response, clusterId, ETCD.name()); + + Mockito.verify(affinityGroupDao, Mockito.never()).findById(Mockito.anyLong()); + } + + @Test + public void testSetAffinityGroupResponseForNodeTypeNullAffinityGroup() { + KubernetesClusterResponse response = new KubernetesClusterResponse(); + long clusterId = 1L; + + AffinityGroupVO ag1 = Mockito.mock(AffinityGroupVO.class); + Mockito.when(ag1.getUuid()).thenReturn("uuid-1"); + Mockito.when(ag1.getName()).thenReturn("affinity-group-1"); + + Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(clusterId, CONTROL.name())) + .thenReturn(Arrays.asList(1L, 2L)); + Mockito.when(affinityGroupDao.findById(1L)).thenReturn(ag1); + Mockito.when(affinityGroupDao.findById(2L)).thenReturn(null); + + kubernetesClusterManager.setAffinityGroupResponseForNodeType(response, clusterId, CONTROL.name()); + + Mockito.verify(affinityGroupDao).findById(1L); + Mockito.verify(affinityGroupDao).findById(2L); + } + + @Test + public void testSetNodeTypeAffinityGroupResponse() { + KubernetesClusterResponse response = new KubernetesClusterResponse(); + long clusterId = 1L; + + Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(Mockito.eq(clusterId), Mockito.anyString())) + .thenReturn(Collections.emptyList()); + + kubernetesClusterManager.setNodeTypeAffinityGroupResponse(response, clusterId); + + Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(clusterId, CONTROL.name()); + Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(clusterId, WORKER.name()); + Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(clusterId, ETCD.name()); + } + } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImplTest.java index 3e6688e87577..3994cadc307f 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImplTest.java @@ -17,6 +17,15 @@ package com.cloud.kubernetes.cluster; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.apache.cloudstack.affinity.AffinityGroupVO; +import org.apache.cloudstack.affinity.dao.AffinityGroupDao; +import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; @@ -24,11 +33,16 @@ import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.uservm.UserVm; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.UserVmManager; +import com.cloud.vm.VmDetailConstants; @RunWith(MockitoJUnitRunner.class) public class KubernetesServiceHelperImplTest { @@ -36,6 +50,10 @@ public class KubernetesServiceHelperImplTest { KubernetesClusterVmMapDao kubernetesClusterVmMapDao; @Mock KubernetesClusterDao kubernetesClusterDao; + @Mock + AffinityGroupDao affinityGroupDao; + @Mock + ServiceOfferingDao serviceOfferingDao; @InjectMocks KubernetesServiceHelperImpl kubernetesServiceHelper = new KubernetesServiceHelperImpl(); @@ -84,4 +102,302 @@ public void testCheckVmCanBeDestroyedInExternalManagedCluster() { Mockito.when(kubernetesCluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.ExternalManaged); kubernetesServiceHelper.checkVmCanBeDestroyed(vm); } + + @Test + public void testIsValidNodeTypeEmptyNodeType() { + Assert.assertFalse(kubernetesServiceHelper.isValidNodeType(null)); + } + + @Test + public void testIsValidNodeTypeInvalidNodeType() { + Assert.assertFalse(kubernetesServiceHelper.isValidNodeType("invalidNodeType")); + } + + @Test + public void testIsValidNodeTypeValidNodeTypeLowercase() { + String nodeType = KubernetesClusterNodeType.WORKER.name().toLowerCase(); + Assert.assertTrue(kubernetesServiceHelper.isValidNodeType(nodeType)); + } + + private Map createServiceOfferingMapEntry(KubernetesClusterNodeType nodeType, String offeringUuid) { + Map map = new HashMap<>(); + map.put(VmDetailConstants.CKS_NODE_TYPE, nodeType.name().toLowerCase()); + map.put(VmDetailConstants.OFFERING, offeringUuid); + return map; + } + + @Test + public void testGetServiceOfferingNodeTypeMap() { + String workerOfferingUuid = UUID.randomUUID().toString(); + String controlOfferingUuid = UUID.randomUUID().toString(); + + ServiceOfferingVO workerOffering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(workerOffering.getId()).thenReturn(1L); + Mockito.when(serviceOfferingDao.findByUuid(workerOfferingUuid)).thenReturn(workerOffering); + + ServiceOfferingVO controlOffering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(controlOffering.getId()).thenReturn(2L); + Mockito.when(serviceOfferingDao.findByUuid(controlOfferingUuid)).thenReturn(controlOffering); + + Map> serviceOfferingNodeTypeMap = new HashMap<>(); + serviceOfferingNodeTypeMap.put("map1", createServiceOfferingMapEntry(KubernetesClusterNodeType.WORKER, workerOfferingUuid)); + serviceOfferingNodeTypeMap.put("map2", createServiceOfferingMapEntry(KubernetesClusterNodeType.CONTROL, controlOfferingUuid)); + + Map result = kubernetesServiceHelper.getServiceOfferingNodeTypeMap(serviceOfferingNodeTypeMap); + + Assert.assertNotNull(result); + Assert.assertEquals(2, result.size()); + Assert.assertTrue(result.containsKey(KubernetesClusterNodeType.WORKER.name())); + Assert.assertTrue(result.containsKey(KubernetesClusterNodeType.CONTROL.name())); + Assert.assertEquals(Long.valueOf(1L), result.get(KubernetesClusterNodeType.WORKER.name())); + Assert.assertEquals(Long.valueOf(2L), result.get(KubernetesClusterNodeType.CONTROL.name())); + } + + @Test + public void testGetServiceOfferingNodeTypeMapNullMap() { + Map result = kubernetesServiceHelper.getServiceOfferingNodeTypeMap(null); + Assert.assertTrue(result.isEmpty()); + } + + @Test + public void testGetServiceOfferingNodeTypeMapEtcdNodes() { + String etcdOfferingUuid = UUID.randomUUID().toString(); + + ServiceOfferingVO etcdOffering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(etcdOffering.getId()).thenReturn(3L); + Mockito.when(serviceOfferingDao.findByUuid(etcdOfferingUuid)).thenReturn(etcdOffering); + + Map> serviceOfferingNodeTypeMap = new HashMap<>(); + serviceOfferingNodeTypeMap.put("map1", createServiceOfferingMapEntry(KubernetesClusterNodeType.ETCD, etcdOfferingUuid)); + + Map result = kubernetesServiceHelper.getServiceOfferingNodeTypeMap(serviceOfferingNodeTypeMap); + + Assert.assertNotNull(result); + Assert.assertEquals(1, result.size()); + Assert.assertTrue(result.containsKey(KubernetesClusterNodeType.ETCD.name())); + Assert.assertEquals(Long.valueOf(3L), result.get(KubernetesClusterNodeType.ETCD.name())); + } + + @Test(expected = InvalidParameterValueException.class) + public void testCheckNodeTypeOfferingEntryCompletenessInvalidParameters() { + kubernetesServiceHelper.checkNodeTypeOfferingEntryCompleteness(KubernetesClusterNodeType.WORKER.name(), null); + } + + @Test(expected = InvalidParameterValueException.class) + public void testCheckNodeTypeOfferingEntryValuesInvalidNodeType() { + ServiceOfferingVO offering = Mockito.mock(ServiceOfferingVO.class); + kubernetesServiceHelper.checkNodeTypeOfferingEntryValues("invalidNodeTypeName", offering, "some-uuid"); + } + + @Test(expected = InvalidParameterValueException.class) + public void testCheckNodeTypeOfferingEntryValuesEmptyOffering() { + kubernetesServiceHelper.checkNodeTypeOfferingEntryValues(KubernetesClusterNodeType.WORKER.name(), null, "some-uuid"); + } + + @Test(expected = InvalidParameterValueException.class) + public void testCheckNodeTypeAffinityGroupEntryCompletenessBlankNodeType() { + kubernetesServiceHelper.checkNodeTypeAffinityGroupEntryCompleteness("", "affinity-group-uuid"); + } + + @Test(expected = InvalidParameterValueException.class) + public void testCheckNodeTypeAffinityGroupEntryCompletenessBlankAffinityGroupUuid() { + kubernetesServiceHelper.checkNodeTypeAffinityGroupEntryCompleteness("control", ""); + } + + @Test + public void testCheckNodeTypeAffinityGroupEntryCompletenessValid() { + kubernetesServiceHelper.checkNodeTypeAffinityGroupEntryCompleteness("control", "affinity-group-uuid"); + } + + @Test(expected = InvalidParameterValueException.class) + public void testCheckNodeTypeAffinityGroupEntryNodeTypeInvalid() { + kubernetesServiceHelper.checkNodeTypeAffinityGroupEntryNodeType("invalid-node-type"); + } + + @Test + public void testCheckNodeTypeAffinityGroupEntryNodeTypeValid() { + kubernetesServiceHelper.checkNodeTypeAffinityGroupEntryNodeType("control"); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateAffinityGroupUuidAndGetIdBlank() { + kubernetesServiceHelper.validateAffinityGroupUuidAndGetId(""); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateAffinityGroupUuidAndGetIdNotFound() { + Mockito.when(affinityGroupDao.findByUuid("non-existent-uuid")).thenReturn(null); + kubernetesServiceHelper.validateAffinityGroupUuidAndGetId("non-existent-uuid"); + } + + @Test + public void testValidateAffinityGroupUuidAndGetIdValid() { + AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class); + Mockito.when(affinityGroup.getId()).thenReturn(100L); + Mockito.when(affinityGroupDao.findByUuid("valid-uuid")).thenReturn(affinityGroup); + Long result = kubernetesServiceHelper.validateAffinityGroupUuidAndGetId("valid-uuid"); + Assert.assertEquals(Long.valueOf(100L), result); + } + + @Test + public void testValidateAndGetAffinityGroupIdsSingleUuid() { + AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class); + Mockito.when(affinityGroup.getId()).thenReturn(1L); + Mockito.when(affinityGroupDao.findByUuid("uuid1")).thenReturn(affinityGroup); + + List result = kubernetesServiceHelper.validateAndGetAffinityGroupIds("uuid1"); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(Long.valueOf(1L), result.get(0)); + } + + @Test + public void testValidateAndGetAffinityGroupIdsMultipleUuids() { + AffinityGroupVO affinityGroup1 = Mockito.mock(AffinityGroupVO.class); + AffinityGroupVO affinityGroup2 = Mockito.mock(AffinityGroupVO.class); + AffinityGroupVO affinityGroup3 = Mockito.mock(AffinityGroupVO.class); + Mockito.when(affinityGroup1.getId()).thenReturn(1L); + Mockito.when(affinityGroup2.getId()).thenReturn(2L); + Mockito.when(affinityGroup3.getId()).thenReturn(3L); + Mockito.when(affinityGroupDao.findByUuid("uuid1")).thenReturn(affinityGroup1); + Mockito.when(affinityGroupDao.findByUuid("uuid2")).thenReturn(affinityGroup2); + Mockito.when(affinityGroupDao.findByUuid("uuid3")).thenReturn(affinityGroup3); + + List result = kubernetesServiceHelper.validateAndGetAffinityGroupIds("uuid1,uuid2,uuid3"); + Assert.assertEquals(3, result.size()); + Assert.assertEquals(Arrays.asList(1L, 2L, 3L), result); + } + + @Test + public void testValidateAndGetAffinityGroupIdsWithSpaces() { + AffinityGroupVO affinityGroup1 = Mockito.mock(AffinityGroupVO.class); + AffinityGroupVO affinityGroup2 = Mockito.mock(AffinityGroupVO.class); + Mockito.when(affinityGroup1.getId()).thenReturn(1L); + Mockito.when(affinityGroup2.getId()).thenReturn(2L); + Mockito.when(affinityGroupDao.findByUuid("uuid1")).thenReturn(affinityGroup1); + Mockito.when(affinityGroupDao.findByUuid("uuid2")).thenReturn(affinityGroup2); + + List result = kubernetesServiceHelper.validateAndGetAffinityGroupIds(" uuid1 , uuid2 "); + Assert.assertEquals(2, result.size()); + Assert.assertEquals(Arrays.asList(1L, 2L), result); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateAndGetAffinityGroupIdsOneInvalid() { + AffinityGroupVO affinityGroup1 = Mockito.mock(AffinityGroupVO.class); + Mockito.when(affinityGroupDao.findByUuid("uuid1")).thenReturn(affinityGroup1); + Mockito.when(affinityGroupDao.findByUuid("invalid-uuid")).thenReturn(null); + + kubernetesServiceHelper.validateAndGetAffinityGroupIds("uuid1,invalid-uuid"); + } + + @Test + public void testAddNodeTypeAffinityGroupEntry() { + Map> mapping = new HashMap<>(); + kubernetesServiceHelper.addNodeTypeAffinityGroupEntry("control", Arrays.asList(1L, 2L), mapping); + Assert.assertEquals(1, mapping.size()); + Assert.assertEquals(Arrays.asList(1L, 2L), mapping.get("CONTROL")); + } + + @Test + public void testProcessNodeTypeAffinityGroupEntryAndAddToMappingIfValidEmptyEntry() { + Map> mapping = new HashMap<>(); + kubernetesServiceHelper.processNodeTypeAffinityGroupEntryAndAddToMappingIfValid(new HashMap<>(), mapping); + Assert.assertTrue(mapping.isEmpty()); + } + + @Test + public void testProcessNodeTypeAffinityGroupEntryAndAddToMappingIfValidValidEntry() { + AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class); + Mockito.when(affinityGroup.getId()).thenReturn(100L); + Mockito.when(affinityGroupDao.findByUuid("affinity-group-uuid")).thenReturn(affinityGroup); + + Map entry = new HashMap<>(); + entry.put(VmDetailConstants.CKS_NODE_TYPE, "control"); + entry.put(VmDetailConstants.AFFINITY_GROUP, "affinity-group-uuid"); + + Map> mapping = new HashMap<>(); + kubernetesServiceHelper.processNodeTypeAffinityGroupEntryAndAddToMappingIfValid(entry, mapping); + Assert.assertEquals(1, mapping.size()); + Assert.assertEquals(Arrays.asList(100L), mapping.get("CONTROL")); + } + + @Test + public void testProcessNodeTypeAffinityGroupEntryAndAddToMappingIfValidMultipleUuids() { + AffinityGroupVO affinityGroup1 = Mockito.mock(AffinityGroupVO.class); + AffinityGroupVO affinityGroup2 = Mockito.mock(AffinityGroupVO.class); + Mockito.when(affinityGroup1.getId()).thenReturn(1L); + Mockito.when(affinityGroup2.getId()).thenReturn(2L); + Mockito.when(affinityGroupDao.findByUuid("uuid1")).thenReturn(affinityGroup1); + Mockito.when(affinityGroupDao.findByUuid("uuid2")).thenReturn(affinityGroup2); + + Map entry = new HashMap<>(); + entry.put(VmDetailConstants.CKS_NODE_TYPE, "worker"); + entry.put(VmDetailConstants.AFFINITY_GROUP, "uuid1,uuid2"); + + Map> mapping = new HashMap<>(); + kubernetesServiceHelper.processNodeTypeAffinityGroupEntryAndAddToMappingIfValid(entry, mapping); + Assert.assertEquals(1, mapping.size()); + Assert.assertEquals(Arrays.asList(1L, 2L), mapping.get("WORKER")); + } + + @Test + public void testGetAffinityGroupNodeTypeMapEmptyMap() { + Map> result = kubernetesServiceHelper.getAffinityGroupNodeTypeMap(null); + Assert.assertTrue(result.isEmpty()); + + result = kubernetesServiceHelper.getAffinityGroupNodeTypeMap(new HashMap<>()); + Assert.assertTrue(result.isEmpty()); + } + + @Test + public void testGetAffinityGroupNodeTypeMapValidEntries() { + AffinityGroupVO controlAffinityGroup = Mockito.mock(AffinityGroupVO.class); + Mockito.when(controlAffinityGroup.getId()).thenReturn(100L); + Mockito.when(affinityGroupDao.findByUuid("control-affinity-uuid")).thenReturn(controlAffinityGroup); + + AffinityGroupVO workerAffinityGroup = Mockito.mock(AffinityGroupVO.class); + Mockito.when(workerAffinityGroup.getId()).thenReturn(200L); + Mockito.when(affinityGroupDao.findByUuid("worker-affinity-uuid")).thenReturn(workerAffinityGroup); + + Map> affinityGroupNodeTypeMap = new HashMap<>(); + + Map controlEntry = new HashMap<>(); + controlEntry.put(VmDetailConstants.CKS_NODE_TYPE, "control"); + controlEntry.put(VmDetailConstants.AFFINITY_GROUP, "control-affinity-uuid"); + affinityGroupNodeTypeMap.put("0", controlEntry); + + Map workerEntry = new HashMap<>(); + workerEntry.put(VmDetailConstants.CKS_NODE_TYPE, "worker"); + workerEntry.put(VmDetailConstants.AFFINITY_GROUP, "worker-affinity-uuid"); + affinityGroupNodeTypeMap.put("1", workerEntry); + + Map> result = kubernetesServiceHelper.getAffinityGroupNodeTypeMap(affinityGroupNodeTypeMap); + Assert.assertEquals(2, result.size()); + Assert.assertEquals(Arrays.asList(100L), result.get("CONTROL")); + Assert.assertEquals(Arrays.asList(200L), result.get("WORKER")); + } + + @Test + public void testGetAffinityGroupNodeTypeMapMultipleIdsPerNodeType() { + AffinityGroupVO ag1 = Mockito.mock(AffinityGroupVO.class); + AffinityGroupVO ag2 = Mockito.mock(AffinityGroupVO.class); + AffinityGroupVO ag3 = Mockito.mock(AffinityGroupVO.class); + Mockito.when(ag1.getId()).thenReturn(1L); + Mockito.when(ag2.getId()).thenReturn(2L); + Mockito.when(ag3.getId()).thenReturn(3L); + Mockito.when(affinityGroupDao.findByUuid("ag1")).thenReturn(ag1); + Mockito.when(affinityGroupDao.findByUuid("ag2")).thenReturn(ag2); + Mockito.when(affinityGroupDao.findByUuid("ag3")).thenReturn(ag3); + + Map> affinityGroupNodeTypeMap = new HashMap<>(); + + Map controlEntry = new HashMap<>(); + controlEntry.put(VmDetailConstants.CKS_NODE_TYPE, "control"); + controlEntry.put(VmDetailConstants.AFFINITY_GROUP, "ag1,ag2,ag3"); + affinityGroupNodeTypeMap.put("0", controlEntry); + + Map> result = kubernetesServiceHelper.getAffinityGroupNodeTypeMap(affinityGroupNodeTypeMap); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(Arrays.asList(1L, 2L, 3L), result.get("CONTROL")); + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java index 1eb55808e09d..a25ec55cc04a 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorkerTest.java @@ -16,8 +16,14 @@ // under the License. package com.cloud.kubernetes.cluster.actionworkers; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.UUID; +import org.apache.cloudstack.affinity.AffinityGroupVO; +import org.apache.cloudstack.affinity.dao.AffinityGroupDao; import org.apache.cloudstack.api.ApiConstants; import org.junit.Assert; import org.junit.Before; @@ -30,6 +36,8 @@ import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterAffinityGroupMapDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; @@ -60,6 +68,12 @@ public class KubernetesClusterActionWorkerTest { @Mock IPAddressDao ipAddressDao; + @Mock + KubernetesClusterAffinityGroupMapDao kubernetesClusterAffinityGroupMapDao; + + @Mock + AffinityGroupDao affinityGroupDao; + KubernetesClusterActionWorker actionWorker = null; final static Long DEFAULT_ID = 1L; @@ -70,10 +84,12 @@ public void setUp() throws Exception { kubernetesClusterManager.kubernetesSupportedVersionDao = kubernetesSupportedVersionDao; kubernetesClusterManager.kubernetesClusterDetailsDao = kubernetesClusterDetailsDao; kubernetesClusterManager.kubernetesClusterVmMapDao = kubernetesClusterVmMapDao; + kubernetesClusterManager.kubernetesClusterAffinityGroupMapDao = kubernetesClusterAffinityGroupMapDao; KubernetesCluster kubernetesCluster = Mockito.mock(KubernetesCluster.class); Mockito.when(kubernetesCluster.getId()).thenReturn(DEFAULT_ID); actionWorker = new KubernetesClusterActionWorker(kubernetesCluster, kubernetesClusterManager); actionWorker.ipAddressDao = ipAddressDao; + actionWorker.affinityGroupDao = affinityGroupDao; } @Test @@ -130,4 +146,87 @@ public void testGetVpcTierKubernetesPublicIpValid() { IpAddress result = actionWorker.getVpcTierKubernetesPublicIp(mockNetworkForGetVpcTierKubernetesPublicIpTest()); Assert.assertNotNull(result); } + + @Test + public void testGetAffinityGroupIdsForNodeTypeReturnsIds() { + Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(DEFAULT_ID, "CONTROL")) + .thenReturn(Arrays.asList(1L, 2L)); + + List result = actionWorker.getAffinityGroupIdsForNodeType(KubernetesClusterNodeType.CONTROL); + + Assert.assertEquals(2, result.size()); + Assert.assertTrue(result.containsAll(Arrays.asList(1L, 2L))); + } + + @Test + public void testGetAffinityGroupIdsForNodeTypeReturnsEmptyList() { + Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(DEFAULT_ID, "WORKER")) + .thenReturn(Collections.emptyList()); + + List result = actionWorker.getAffinityGroupIdsForNodeType(KubernetesClusterNodeType.WORKER); + + Assert.assertTrue(result.isEmpty()); + } + + @Test + public void testGetMergedAffinityGroupIdsWithExplicitDedication() { + Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(DEFAULT_ID, "CONTROL")) + .thenReturn(new ArrayList<>(Arrays.asList(1L))); + + AffinityGroupVO explicitGroup = Mockito.mock(AffinityGroupVO.class); + Mockito.when(explicitGroup.getId()).thenReturn(99L); + Mockito.when(affinityGroupDao.findByAccountAndType(Mockito.anyLong(), Mockito.eq("ExplicitDedication"))) + .thenReturn(explicitGroup); + + List result = actionWorker.getMergedAffinityGroupIds(KubernetesClusterNodeType.CONTROL, 1L, 1L); + + Assert.assertEquals(2, result.size()); + Assert.assertTrue(result.contains(1L)); + Assert.assertTrue(result.contains(99L)); + } + + @Test + public void testGetMergedAffinityGroupIdsNoExplicitDedication() { + Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(DEFAULT_ID, "WORKER")) + .thenReturn(new ArrayList<>(Arrays.asList(1L, 2L))); + Mockito.when(affinityGroupDao.findByAccountAndType(Mockito.anyLong(), Mockito.eq("ExplicitDedication"))) + .thenReturn(null); + Mockito.when(affinityGroupDao.findDomainLevelGroupByType(Mockito.anyLong(), Mockito.eq("ExplicitDedication"))) + .thenReturn(null); + + List result = actionWorker.getMergedAffinityGroupIds(KubernetesClusterNodeType.WORKER, 1L, 1L); + + Assert.assertEquals(2, result.size()); + } + + @Test + public void testGetMergedAffinityGroupIdsReturnsNullWhenEmpty() { + Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(DEFAULT_ID, "ETCD")) + .thenReturn(new ArrayList<>()); + Mockito.when(affinityGroupDao.findByAccountAndType(Mockito.anyLong(), Mockito.anyString())) + .thenReturn(null); + Mockito.when(affinityGroupDao.findDomainLevelGroupByType(Mockito.anyLong(), Mockito.anyString())) + .thenReturn(null); + + List result = actionWorker.getMergedAffinityGroupIds(KubernetesClusterNodeType.ETCD, 1L, 1L); + + Assert.assertNull(result); + } + + @Test + public void testGetMergedAffinityGroupIdsExplicitDedicationAlreadyInList() { + Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(DEFAULT_ID, "CONTROL")) + .thenReturn(new ArrayList<>(Arrays.asList(99L, 2L))); + + AffinityGroupVO explicitGroup = Mockito.mock(AffinityGroupVO.class); + Mockito.when(explicitGroup.getId()).thenReturn(99L); + Mockito.when(affinityGroupDao.findByAccountAndType(Mockito.anyLong(), Mockito.eq("ExplicitDedication"))) + .thenReturn(explicitGroup); + + List result = actionWorker.getMergedAffinityGroupIds(KubernetesClusterNodeType.CONTROL, 1L, 1L); + + Assert.assertEquals(2, result.size()); + Assert.assertTrue(result.contains(99L)); + Assert.assertTrue(result.contains(2L)); + } }