From 7835a48275e421a1b33d517b778827fd67273896 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 20 Mar 2024 09:45:51 +0100 Subject: [PATCH 01/44] New feature: Import volume from storage pool --- .../main/java/com/cloud/event/EventTypes.java | 2 + .../apache/cloudstack/api/ApiConstants.java | 2 + .../command/admin/volume/ImportVolumeCmd.java | 178 ++++++++ .../admin/volume/ListVolumesForImportCmd.java | 95 +++++ .../admin/volume/UnmanageVolumeCmd.java | 127 ++++++ .../api/response/VolumeForImportResponse.java | 140 +++++++ .../volume/VolumeImportUnmanageService.java | 35 ++ .../storage/volume/VolumeOnStorageTO.java | 137 ++++++ .../storage/volume/VolumeOnStorageTOTest.java | 48 +++ .../agent/api/GetVolumesOnStorageAnswer.java | 42 ++ .../agent/api/GetVolumesOnStorageCommand.java | 49 +++ .../api/GetVolumesOnStorageAnswerTest.java | 66 +++ .../api/GetVolumesOnStorageCommandTest.java | 38 ++ .../service/VolumeOrchestrationService.java | 3 +- .../orchestration/VolumeOrchestrator.java | 10 +- ...virtGetVolumesOnStorageCommandWrapper.java | 149 +++++++ .../apache/cloudstack/utils/qemu/QemuImg.java | 6 +- .../vmware/resource/VmwareResource.java | 54 +++ .../VolumeImportUnmanagedManagerImpl.java | 392 ++++++++++++++++++ .../vm/UnmanagedVMsManagerImpl.java | 2 +- .../spring-server-core-managers-context.xml | 2 + .../smoke/test_import_unmanage_volumes.py | 168 ++++++++ 22 files changed, 1737 insertions(+), 8 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ListVolumesForImportCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/response/VolumeForImportResponse.java create mode 100644 api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java create mode 100644 api/src/main/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTO.java create mode 100644 api/src/test/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTOTest.java create mode 100644 core/src/main/java/com/cloud/agent/api/GetVolumesOnStorageAnswer.java create mode 100644 core/src/main/java/com/cloud/agent/api/GetVolumesOnStorageCommand.java create mode 100644 core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageAnswerTest.java create mode 100644 core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageCommandTest.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java create mode 100644 server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanagedManagerImpl.java create mode 100644 test/integration/smoke/test_import_unmanage_volumes.py diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index d385fa9ed07f..0122319cd3db 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -316,6 +316,8 @@ public class EventTypes { public static final String EVENT_VOLUME_UPDATE = "VOLUME.UPDATE"; public static final String EVENT_VOLUME_DESTROY = "VOLUME.DESTROY"; public static final String EVENT_VOLUME_RECOVER = "VOLUME.RECOVER"; + public static final String EVENT_VOLUME_IMPORT = "VOLUME.IMPORT"; + public static final String EVENT_VOLUME_UNMANAGE = "VOLUME.UNMANAGE"; public static final String EVENT_VOLUME_CHANGE_DISK_OFFERING = "VOLUME.CHANGE.DISK.OFFERING"; // Domains diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 18d25a0cfc3f..fa8f9067fa1c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -117,6 +117,7 @@ public class ApiConstants { public static final String CURRENT_START_IP = "currentstartip"; public static final String CURRENT_END_IP = "currentendip"; public static final String ENCRYPT = "encrypt"; + public static final String ENCRYPT_FORMAT = "encryptformat"; public static final String ENCRYPT_ROOT = "encryptroot"; public static final String ENCRYPTION_SUPPORTED = "encryptionsupported"; public static final String MIN_IOPS = "miniops"; @@ -191,6 +192,7 @@ public class ApiConstants { public static final String FORMAT = "format"; public static final String FOR_VIRTUAL_NETWORK = "forvirtualnetwork"; public static final String FOR_SYSTEM_VMS = "forsystemvms"; + public static final String FULL_PATH = "fullpath"; public static final String GATEWAY = "gateway"; public static final String IP6_GATEWAY = "ip6gateway"; public static final String GROUP = "group"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java new file mode 100644 index 000000000000..cfd49222f66e --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.volume; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.user.Account; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.DiskOfferingResponse; +import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ProjectResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; +import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.storage.volume.VolumeImportUnmanageService; + +import javax.inject.Inject; + +@APICommand(name = "importVolume", + description = "Import an unmanaged volume from a storage pool on a host into CloudStack", + responseObject = VolumeResponse.class, + responseView = ResponseObject.ResponseView.Full, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = true, + authorized = {RoleType.Admin}, + since = "4.19.1") +public class ImportVolumeCmd extends BaseAsyncCmd { + + @Inject + public VolumeImportUnmanageService volumeImportService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + + @Parameter(name = ApiConstants.PATH, + type = BaseCmd.CommandType.STRING, + required = true, + description = "the path of the volume") + private String path; + + @Parameter(name = ApiConstants.STORAGE_ID, + type = BaseCmd.CommandType.UUID, + required = true, + entityType = StoragePoolResponse.class, + description = "the ID of the storage pool") + private Long storageId; + + @Parameter(name = ApiConstants.DISK_OFFERING_ID, + type = BaseCmd.CommandType.UUID, + entityType = DiskOfferingResponse.class, + description = "the ID of the disk offering linked to the volume") + private Long diskOfferingId; + + @Parameter(name = ApiConstants.ACCOUNT, + type = BaseCmd.CommandType.STRING, + description = "an optional account for the virtual machine. Must be used with domainId.") + private String accountName; + + @Parameter(name = ApiConstants.DOMAIN_ID, + type = BaseCmd.CommandType.UUID, + entityType = DomainResponse.class, + description = "import instance to the domain specified") + private Long domainId; + + @Parameter(name = ApiConstants.PROJECT_ID, + type = BaseCmd.CommandType.UUID, + entityType = ProjectResponse.class, + description = "import instance for the project") + private Long projectId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public Long getStorageId() { + return storageId; + } + + public void setStorageId(Long storageId) { + this.storageId = storageId; + } + + public Long getDiskOfferingId() { + return diskOfferingId; + } + + public void setDiskOfferingId(Long diskOfferingId) { + this.diskOfferingId = diskOfferingId; + } + + public String getAccountName() { + return accountName; + } + + public void setAccountName(String accountName) { + this.accountName = accountName; + } + + public Long getDomainId() { + return domainId; + } + + public void setDomainId(Long domainId) { + this.domainId = domainId; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + @Override + public String getEventType() { + return EventTypes.EVENT_VOLUME_IMPORT; + } + + @Override + public String getEventDescription() { + return String.format("Importing unmanaged Volume with path: %s", path); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + VolumeResponse response = volumeImportService.importVolume(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + Account caller = CallContext.current().getCallingAccount(); + return caller.getAccountId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ListVolumesForImportCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ListVolumesForImportCmd.java new file mode 100644 index 000000000000..6dbadf3c1cb9 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ListVolumesForImportCmd.java @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.volume; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; +import org.apache.cloudstack.api.response.VolumeForImportResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.storage.volume.VolumeImportUnmanageService; +import org.apache.cloudstack.storage.volume.VolumeOnStorageTO; + +import javax.inject.Inject; + +@APICommand(name = "listVolumesForImport", + description = "Lists unmanaged volumes on a storage pool", + responseObject = VolumeForImportResponse.class, + responseView = ResponseObject.ResponseView.Full, + entityType = {VolumeOnStorageTO.class}, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = true, + authorized = {RoleType.Admin}, + since = "4.19.1") +public class ListVolumesForImportCmd extends BaseListCmd { + + @Inject + public VolumeImportUnmanageService volumeImportService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.STORAGE_ID, + type = BaseCmd.CommandType.UUID, + required = true, + entityType = StoragePoolResponse.class, + description = "the ID of the storage pool") + private Long storageId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getStorageId() { + return storageId; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + ListResponse response = volumeImportService.listVolumesForImport(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + Account account = CallContext.current().getCallingAccount(); + if (account != null) { + return account.getId(); + } + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmd.java new file mode 100644 index 000000000000..773f587f43b0 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmd.java @@ -0,0 +1,127 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.api.command.admin.volume; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.storage.Volume; +import com.cloud.user.Account; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.storage.volume.VolumeImportUnmanageService; + +import javax.inject.Inject; + +@APICommand(name = "unmanageVolume", + description = "Unmanage a volume on storage pool.", + entityType = {Volume.class}, + responseObject = SuccessResponse.class, + requestHasSensitiveInfo = false, + authorized = {RoleType.Admin}, + since = "4.19.1") +public class UnmanageVolumeCmd extends BaseAsyncCmd { + + @Inject + public VolumeImportUnmanageService volumeImportService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, + type = CommandType.UUID, + entityType = VolumeResponse.class, + required = true, + description = "The ID of the volume to unmanage") + private Long volumeId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + + public Long getVolumeId() { + return volumeId; + } + + @Override + public String getEventType() { + return EventTypes.EVENT_VOLUME_UNMANAGE; + } + + @Override + public String getEventDescription() { + return String.format("unmanaging Volume with ID %s", volumeId); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, + ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + try { + boolean result = volumeImportService.unmanageVolume(volumeId); + if (result) { + SuccessResponse response = new SuccessResponse(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to unmanage the volume"); + } + + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getLocalizedMessage()); + } + } + + @Override + public long getEntityOwnerId() { + Volume volume = _responseGenerator.findVolumeById(volumeId); + if (volume != null) { + return volume.getAccountId(); + } + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.Volume; + } + + @Override + public Long getApiResourceId() { + return volumeId; + } + +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/VolumeForImportResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VolumeForImportResponse.java new file mode 100644 index 000000000000..045a7b85e0d3 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/VolumeForImportResponse.java @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.response; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; +import org.apache.cloudstack.storage.volume.VolumeOnStorageTO; + +import java.util.Map; + +@EntityReference(value = VolumeOnStorageTO.class) +public class VolumeForImportResponse extends BaseResponse { + + @SerializedName(ApiConstants.NAME) + @Param(description = "the name of the volume") + private String name; + + @SerializedName(ApiConstants.PATH) + @Param(description = "the path of the volume") + private String path; + + @SerializedName(ApiConstants.FULL_PATH) + @Param(description = "the full path of the volume") + private String fullPath; + + @SerializedName(ApiConstants.FORMAT) + @Param(description = "the format of the volume") + private String format; + + @SerializedName(ApiConstants.SIZE) + @Param(description = "the size of the volume") + private long size; + + @SerializedName(ApiConstants.VIRTUAL_SIZE) + @Param(description = "the virtual size of the volume") + private long virtualSize; + + @SerializedName(ApiConstants.ENCRYPT_FORMAT) + @Param(description = "the encrypt format of the volume") + private String qemuEncryptFormat; + + @SerializedName(ApiConstants.DETAILS) + @Param(description = "volume details in key/value pairs.") + private Map details; + + @SerializedName(ApiConstants.CHAIN_INFO) + @Param(description = "the chain info of the volume") + String chainInfo; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getFullPath() { + return fullPath; + } + + public void setFullPath(String fullPath) { + this.fullPath = fullPath; + } + + public String getFormat() { + return format; + } + + public void setFormat(String format) { + this.format = format; + } + + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } + + public long getVirtualSize() { + return virtualSize; + } + + public void setVirtualSize(long virtualSize) { + this.virtualSize = virtualSize; + } + + public String getQemuEncryptFormat() { + return qemuEncryptFormat; + } + + public void setQemuEncryptFormat(String qemuEncryptFormat) { + this.qemuEncryptFormat = qemuEncryptFormat; + } + + public Map getDetails() { + return details; + } + + public void setDetails(Map details) { + this.details = details; + } + + public String getChainInfo() { + return chainInfo; + } + + public void setChainInfo(String chainInfo) { + this.chainInfo = chainInfo; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java b/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java new file mode 100644 index 000000000000..f908fd9c9a4d --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.volume; + +import com.cloud.utils.component.PluggableService; +import org.apache.cloudstack.api.command.admin.volume.ListVolumesForImportCmd; +import org.apache.cloudstack.api.command.admin.volume.ImportVolumeCmd; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.VolumeForImportResponse; +import org.apache.cloudstack.api.response.VolumeResponse; + +public interface VolumeImportUnmanageService extends PluggableService { + + ListResponse listVolumesForImport(ListVolumesForImportCmd cmd); + + VolumeResponse importVolume(ImportVolumeCmd cmd); + + boolean unmanageVolume(long volumeId); + +} diff --git a/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTO.java b/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTO.java new file mode 100644 index 000000000000..80e93c9f4225 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTO.java @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.volume; + +import com.cloud.hypervisor.Hypervisor; + +import java.util.HashMap; +import java.util.Map; + +public class VolumeOnStorageTO { + Hypervisor.HypervisorType hypervisorType; + private String path; + private String fullPath; + + private String name; + + private String format; + private long size; + private long virtualSize; + private String qemuEncryptFormat; + private Map details = new HashMap<>(); + + public enum Detail { + BACKING_FILE, BACKING_FILE_FORMAT, CLUSTER_SIZE, FILE_FORMAT, IS_LOCKED + } + + public VolumeOnStorageTO() { + } + + public VolumeOnStorageTO(Hypervisor.HypervisorType hypervisorType, String path, String name, String fullPath, String format, long size, long virtualSize) { + this.hypervisorType = hypervisorType; + this.path = path; + this.name = name; + this.fullPath = fullPath; + this.format = format; + this.size = size; + this.virtualSize = virtualSize; + } + + public VolumeOnStorageTO(Hypervisor.HypervisorType hypervisorType, String path, String name, long size) { + this.hypervisorType = hypervisorType; + this.path = path; + this.name = name; + this.size = size; + } + + public Hypervisor.HypervisorType getHypervisorType() { + return hypervisorType; + } + + public void setHypervisorType(Hypervisor.HypervisorType hypervisorType) { + this.hypervisorType = hypervisorType; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getFullPath() { + return fullPath; + } + + public void setFullPath(String fullPath) { + this.fullPath = fullPath; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getFormat() { + return format; + } + + public void setFormat(String format) { + this.format = format; + } + + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } + + public long getVirtualSize() { + return virtualSize; + } + + public void setVirtualSize(long virtualSize) { + this.virtualSize = virtualSize; + } + + public String getQemuEncryptFormat() { + return qemuEncryptFormat; + } + + public void setQemuEncryptFormat(String qemuEncryptFormat) { + this.qemuEncryptFormat = qemuEncryptFormat; + } + + public Map getDetails() { + return details; + } + + public void setDetails(Map details) { + this.details = details; + } + + public void addDetail(Detail detail, String value) { + details.put(detail, value); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTOTest.java b/api/src/test/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTOTest.java new file mode 100644 index 000000000000..b36c6d9d8731 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTOTest.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.volume; + +import com.cloud.hypervisor.Hypervisor; +import org.junit.Assert; +import org.junit.Test; + +public class VolumeOnStorageTOTest { + + private static String path = "path"; + private static String name = "name"; + private static String fullPath = "fullPath"; + private static String format = "qcow2"; + private static long size = 10; + private static long virtualSize = 20; + private static String encryptFormat = "LUKS"; + + @Test + public void testVolumeOnStorageTO() { + VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(Hypervisor.HypervisorType.KVM, path, name, fullPath, + format, size, virtualSize); + volumeOnStorageTO.setQemuEncryptFormat(encryptFormat); + + Assert.assertEquals(path, volumeOnStorageTO.getPath()); + Assert.assertEquals(name, volumeOnStorageTO.getName()); + Assert.assertEquals(fullPath, volumeOnStorageTO.getFullPath()); + Assert.assertEquals(format, volumeOnStorageTO.getFormat()); + Assert.assertEquals(size, volumeOnStorageTO.getSize()); + Assert.assertEquals(virtualSize, volumeOnStorageTO.getVirtualSize()); + Assert.assertEquals(encryptFormat, volumeOnStorageTO.getQemuEncryptFormat()); + Assert.assertEquals(path, volumeOnStorageTO.getPath()); + } +} diff --git a/core/src/main/java/com/cloud/agent/api/GetVolumesOnStorageAnswer.java b/core/src/main/java/com/cloud/agent/api/GetVolumesOnStorageAnswer.java new file mode 100644 index 000000000000..89f256867c29 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/GetVolumesOnStorageAnswer.java @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.agent.api; + +import org.apache.cloudstack.storage.volume.VolumeOnStorageTO; + +import java.util.List; + +public class GetVolumesOnStorageAnswer extends Answer { + private List volumes; + + GetVolumesOnStorageAnswer() { + } + + public GetVolumesOnStorageAnswer(GetVolumesOnStorageCommand cmd, List volumes) { + super(cmd, true, null); + this.volumes = volumes; + } + + public GetVolumesOnStorageAnswer(final GetVolumesOnStorageCommand cmd, final boolean success, final String details) { + super(cmd, success, details); + } + + public List getVolumes() { + return volumes; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/GetVolumesOnStorageCommand.java b/core/src/main/java/com/cloud/agent/api/GetVolumesOnStorageCommand.java new file mode 100644 index 000000000000..8fbbac914b36 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/GetVolumesOnStorageCommand.java @@ -0,0 +1,49 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.agent.api; + +import com.cloud.agent.api.to.StorageFilerTO; + +public class GetVolumesOnStorageCommand extends Command { + + StorageFilerTO pool; + private String volumePath; //filter by file path + + public GetVolumesOnStorageCommand() { + } + + public GetVolumesOnStorageCommand(StorageFilerTO pool, String filePath) { + this.pool = pool; + this.volumePath = filePath; + } + + public StorageFilerTO getPool() { + return pool; + } + + public String getVolumePath() { + return volumePath; + } + + @Override + public boolean executeInSequence() { + return false; + } +} diff --git a/core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageAnswerTest.java b/core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageAnswerTest.java new file mode 100644 index 000000000000..92205559826d --- /dev/null +++ b/core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageAnswerTest.java @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api; + +import com.cloud.hypervisor.Hypervisor; +import org.apache.cloudstack.storage.volume.VolumeOnStorageTO; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.List; + +public class GetVolumesOnStorageAnswerTest { + + private static String path = "path"; + private static String name = "name"; + private static String fullPath = "fullPath"; + private static String format = "qcow2"; + private static long size = 10; + private static long virtualSize = 20; + private static String encryptFormat = "LUKS"; + + private static GetVolumesOnStorageCommand command = Mockito.mock(GetVolumesOnStorageCommand.class); + + @Test + public void testGetVolumesOnStorageAnswer() { + VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(Hypervisor.HypervisorType.KVM, path, name, fullPath, + format, size, virtualSize); + volumeOnStorageTO.setQemuEncryptFormat(encryptFormat); + + List volumesOnStorageTO = new ArrayList<>(); + volumesOnStorageTO.add(volumeOnStorageTO); + + GetVolumesOnStorageAnswer answer = new GetVolumesOnStorageAnswer(command, volumesOnStorageTO); + List volumes = answer.getVolumes(); + + Assert.assertEquals(1, volumes.size()); + VolumeOnStorageTO volume = volumes.get(0); + + Assert.assertEquals(Hypervisor.HypervisorType.KVM, volume.getHypervisorType()); + Assert.assertEquals(path, volume.getPath()); + Assert.assertEquals(name, volume.getName()); + Assert.assertEquals(fullPath, volume.getFullPath()); + Assert.assertEquals(format, volume.getFormat()); + Assert.assertEquals(size, volume.getSize()); + Assert.assertEquals(virtualSize, volume.getVirtualSize()); + Assert.assertEquals(encryptFormat, volume.getQemuEncryptFormat()); + Assert.assertEquals(path, volume.getPath()); + } + +} diff --git a/core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageCommandTest.java b/core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageCommandTest.java new file mode 100644 index 000000000000..fba772b9d74e --- /dev/null +++ b/core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageCommandTest.java @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api; + +import com.cloud.agent.api.to.StorageFilerTO; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +public class GetVolumesOnStorageCommandTest { + + final StorageFilerTO pool = Mockito.mock(StorageFilerTO.class); + + final String localPath = "localPath"; + final String volumePath = "volumePath"; + + @Test + public void testGetVolumesOnStorageCommand() { + GetVolumesOnStorageCommand command = new GetVolumesOnStorageCommand(pool, volumePath); + + Assert.assertEquals(pool, command.getPool()); + Assert.assertEquals(volumePath, command.getVolumePath()); + } +} diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java index c4fbc2505aa4..74ededaf1f4b 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java @@ -165,7 +165,8 @@ List allocateTemplatedVolumes(Type type, String name, DiskOffering * @param chainInfo chain info for the volume. Hypervisor specific. * @return DiskProfile of imported volume */ - DiskProfile importVolume(Type type, String name, DiskOffering offering, Long sizeInBytes, Long minIops, Long maxIops, VirtualMachine vm, VirtualMachineTemplate template, + DiskProfile importVolume(Type type, String name, DiskOffering offering, Long sizeInBytes, Long minIops, Long maxIops, + Long zoneId, HypervisorType hypervisorType, VirtualMachine vm, VirtualMachineTemplate template, Account owner, Long deviceId, Long poolId, String path, String chainInfo); DiskProfile updateImportedVolume(Type type, DiskOffering offering, VirtualMachine vm, VirtualMachineTemplate template, diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 409b5388d72f..e2ac16f57ed9 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -2187,7 +2187,7 @@ public void updateVolumeDiskChain(long volumeId, String path, String chainInfo, @Override public DiskProfile importVolume(Type type, String name, DiskOffering offering, Long sizeInBytes, Long minIops, Long maxIops, - VirtualMachine vm, VirtualMachineTemplate template, Account owner, + Long zoneId, HypervisorType hypervisorType, VirtualMachine vm, VirtualMachineTemplate template, Account owner, Long deviceId, Long poolId, String path, String chainInfo) { if (sizeInBytes == null) { sizeInBytes = offering.getDiskSize(); @@ -2196,9 +2196,10 @@ public DiskProfile importVolume(Type type, String name, DiskOffering offering, L minIops = minIops != null ? minIops : offering.getMinIops(); maxIops = maxIops != null ? maxIops : offering.getMaxIops(); - VolumeVO vol = new VolumeVO(type, name, vm.getDataCenterId(), owner.getDomainId(), owner.getId(), offering.getId(), offering.getProvisioningType(), sizeInBytes, minIops, maxIops, null); + VolumeVO vol = new VolumeVO(type, name, zoneId, owner.getDomainId(), owner.getId(), offering.getId(), offering.getProvisioningType(), sizeInBytes, minIops, maxIops, null); if (vm != null) { vol.setInstanceId(vm.getId()); + vol.setAttached(new Date()); } if (deviceId != null) { @@ -2221,17 +2222,16 @@ public DiskProfile importVolume(Type type, String name, DiskOffering offering, L } // display flag matters only for the User vms - if (VirtualMachine.Type.User.equals(vm.getType())) { + if (vm != null && VirtualMachine.Type.User.equals(vm.getType())) { UserVmVO userVm = _userVmDao.findById(vm.getId()); vol.setDisplayVolume(userVm.isDisplayVm()); } - vol.setFormat(getSupportedImageFormatForCluster(vm.getHypervisorType())); + vol.setFormat(getSupportedImageFormatForCluster(hypervisorType)); vol.setPoolId(poolId); vol.setPath(path); vol.setChainInfo(chainInfo); vol.setState(Volume.State.Ready); - vol.setAttached(new Date()); vol = _volsDao.persist(vol); return toDiskProfile(vol, offering); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java new file mode 100644 index 000000000000..f4b9cb1de798 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java @@ -0,0 +1,149 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.GetVolumesOnStorageAnswer; +import com.cloud.agent.api.GetVolumesOnStorageCommand; +import com.cloud.agent.api.to.StorageFilerTO; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.storage.Storage.StoragePoolType; +import org.apache.cloudstack.storage.volume.VolumeOnStorageTO; +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.apache.commons.lang3.StringUtils; +import org.libvirt.LibvirtException; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +@ResourceWrapper(handles = GetVolumesOnStorageCommand.class) +public final class LibvirtGetVolumesOnStorageCommandWrapper extends CommandWrapper { + + static final List SUPPORTED_STORAGE_POOL_TYPES = Arrays.asList(StoragePoolType.NetworkFilesystem, + StoragePoolType.Filesystem, StoragePoolType.RBD); + + @Override + public Answer execute(final GetVolumesOnStorageCommand command, final LibvirtComputingResource libvirtComputingResource) { + + final StorageFilerTO pool = command.getPool(); + if (!SUPPORTED_STORAGE_POOL_TYPES.contains(pool.getType())) { + return new GetVolumesOnStorageAnswer(command, false, String.format("pool type %s is unsupported", pool.getType())); + } + final String volumePath = command.getVolumePath(); + + List volumes = new ArrayList<>(); + + final KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr(); + KVMStoragePool storagePool = storagePoolMgr.getStoragePool(pool.getType(), pool.getUuid(), true); + + if (volumePath != null) { + KVMPhysicalDisk disk = storagePool.getPhysicalDisk(volumePath); + if (disk != null) { + if (!isDiskFormatSupported(disk)) { + return new GetVolumesOnStorageAnswer(command, false, String.format("disk format %s is unsupported", disk.getFormat())); + } + Map info = getDiskFileInfo(storagePool, disk, true); + if (info == null) { + return new GetVolumesOnStorageAnswer(command, false, "failed to get information of disk file. The disk might be locked or unsupported"); + } + VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(Hypervisor.HypervisorType.KVM, disk.getName(), disk.getName(), disk.getPath(), + disk.getFormat().toString(), disk.getSize(), disk.getVirtualSize()); + if (disk.getQemuEncryptFormat() != null) { + volumeOnStorageTO.setQemuEncryptFormat(disk.getQemuEncryptFormat().toString()); + } + String backingFilePath = info.get(QemuImg.BACKING_FILE); + if (StringUtils.isNotBlank(backingFilePath)) { + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.BACKING_FILE, backingFilePath); + } + String backingFileFormat = info.get(QemuImg.BACKING_FILE_FORMAT); + if (StringUtils.isNotBlank(backingFileFormat)) { + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.BACKING_FILE_FORMAT, backingFileFormat); + } + String clusterSize = info.get(QemuImg.CLUSTER_SIZE); + if (StringUtils.isNotBlank(clusterSize)) { + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.CLUSTER_SIZE, clusterSize); + } + String fileFormat = info.get(QemuImg.FILE_FORMAT); + if (StringUtils.isNotBlank(clusterSize)) { + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.FILE_FORMAT, fileFormat); + } + Boolean isLocked = isDiskFileLocked(storagePool, disk); + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.IS_LOCKED, String.valueOf(isLocked)); + volumes.add(volumeOnStorageTO); + } + } else { + for (KVMPhysicalDisk disk: storagePool.listPhysicalDisks()) { + if (!isDiskFormatSupported(disk)) { + continue; + } + VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(Hypervisor.HypervisorType.KVM, disk.getName(), disk.getName(), disk.getPath(), + disk.getFormat().toString(), disk.getSize(), disk.getVirtualSize()); + if (disk.getQemuEncryptFormat() != null) { + volumeOnStorageTO.setQemuEncryptFormat(disk.getQemuEncryptFormat().toString()); + } + volumes.add(volumeOnStorageTO); + } + } + + return new GetVolumesOnStorageAnswer(command, volumes); + } + + private boolean isDiskFormatSupported(KVMPhysicalDisk disk) { + return PhysicalDiskFormat.QCOW2.equals(disk.getFormat()) || PhysicalDiskFormat.RAW.equals(disk.getFormat()); + } + + private boolean isDiskFileLocked(KVMStoragePool pool, KVMPhysicalDisk disk) { + if (PhysicalDiskFormat.QCOW2.equals(disk.getFormat())) { + Map info = getDiskFileInfo(pool, disk, false); + return info == null; + } + return false; // unknown + } + + private Map getDiskFileInfo(KVMStoragePool pool, KVMPhysicalDisk disk, boolean secure) { + try { + QemuImg qemu = new QemuImg(0); + QemuImgFile qemuFile = new QemuImgFile(disk.getPath(), disk.getFormat()); + if (StoragePoolType.RBD.equals(pool.getType())) { + String rbdDestFile = KVMPhysicalDisk.RBDStringBuilder(pool.getSourceHost(), + pool.getSourcePort(), + pool.getAuthUserName(), + pool.getAuthSecret(), + disk.getPath()); + qemuFile = new QemuImgFile(rbdDestFile, disk.getFormat()); + } + return qemu.info(qemuFile, secure); + } catch (QemuImgException | LibvirtException ex) { + logger.error("Failed to get info of disk file: " + ex.getMessage()); + } + return null; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java index 360c762deb0b..04f7b36bbae0 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java @@ -554,9 +554,13 @@ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, Strin * @return A HashMap with string key-value information as returned by 'qemu-img info'. */ public Map info(final QemuImgFile file) throws QemuImgException { + return info(file, true); + } + + public Map info(final QemuImgFile file, boolean secure) throws QemuImgException { final Script s = new Script(_qemuImgPath); s.add("info"); - if (this.version >= QEMU_2_10) { + if (this.version >= QEMU_2_10 && secure) { s.add("-U"); } s.add(file.getFileName()); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 1c244645033e..f1ab96682bbf 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -48,6 +48,8 @@ import javax.naming.ConfigurationException; import javax.xml.datatype.XMLGregorianCalendar; +import com.cloud.agent.api.GetVolumesOnStorageAnswer; +import com.cloud.agent.api.GetVolumesOnStorageCommand; import com.cloud.hypervisor.vmware.mo.HostDatastoreBrowserMO; import com.vmware.vim25.FileInfo; import com.vmware.vim25.FileQueryFlags; @@ -66,6 +68,7 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.storage.volume.VolumeOnStorageTO; import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo; import org.apache.cloudstack.vm.UnmanagedInstanceTO; import org.apache.commons.collections.CollectionUtils; @@ -527,6 +530,8 @@ public Answer executeRequest(Command cmd) { answer = execute((DeleteStoragePoolCommand) cmd); } else if (clz == CopyVolumeCommand.class) { answer = execute((CopyVolumeCommand) cmd); + } else if (clz == GetVolumesOnStorageCommand.class) { + answer = execute((GetVolumesOnStorageCommand) cmd); } else if (clz == AttachIsoCommand.class) { answer = execute((AttachIsoCommand) cmd); } else if (clz == ValidateSnapshotCommand.class) { @@ -5935,6 +5940,55 @@ public CopyVolumeAnswer execute(CopyVolumeCommand cmd) { } } + private GetVolumesOnStorageAnswer execute(GetVolumesOnStorageCommand cmd) { + List volumes = new ArrayList<>(); + try { + VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext()); + StorageFilerTO pool = cmd.getPool(); + + if (!Arrays.asList(StoragePoolType.NetworkFilesystem, StoragePoolType.VMFS, StoragePoolType.PreSetup, + StoragePoolType.DatastoreCluster).contains(pool.getType())) { + throw new Exception("Unsupported storage pool type " + pool.getType()); + } + + ManagedObjectReference morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, pool.getUuid()); + + if (morDatastore == null) { + boolean vmfsDatastore = Arrays.asList(StoragePoolType.VMFS, StoragePoolType.PreSetup, StoragePoolType.DatastoreCluster).contains(pool.getType()); + morDatastore = hyperHost.mountDatastore(vmfsDatastore, pool.getHost(), pool.getPort(), pool.getPath(), pool.getUuid().replace("-", ""), true); + } + + assert (morDatastore != null); + + DatastoreMO dsMo = new DatastoreMO(getServiceContext(), morDatastore); + + HostDatastoreBrowserMO browserMo = dsMo.getHostDatastoreBrowserMO(); + FileQueryFlags fqf = new FileQueryFlags(); + fqf.setFileSize(true); + fqf.setFileType(true); + fqf.setModification(true); + fqf.setFileOwner(false); + + HostDatastoreBrowserSearchSpec spec = new HostDatastoreBrowserSearchSpec(); + spec.setSearchCaseInsensitive(true); + spec.setDetails(fqf); + + String dsPath = String.format("[%s]", dsMo.getName()); + + HostDatastoreBrowserSearchResults results = browserMo.searchDatastore(dsPath, spec); + List fileInfoList = results.getFile(); + for (FileInfo file : fileInfoList) { + VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(HypervisorType.VMware, file.getPath(), + file.getFriendlyName(), file.getFileSize()); + volumes.add(volumeOnStorageTO); + } + } catch (Exception e) { + return new GetVolumesOnStorageAnswer(cmd, false, e.getMessage()); + + } + return new GetVolumesOnStorageAnswer(cmd, volumes); + } + @Override public void disconnected() { } diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanagedManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanagedManagerImpl.java new file mode 100644 index 000000000000..dc525e0d1812 --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanagedManagerImpl.java @@ -0,0 +1,392 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.volume; + +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.GetVolumesOnStorageAnswer; +import com.cloud.agent.api.GetVolumesOnStorageCommand; +import com.cloud.agent.api.to.StorageFilerTO; +import com.cloud.configuration.ConfigurationManager; +import com.cloud.configuration.Resource; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.event.EventTypes; +import com.cloud.event.UsageEventUtils; +import com.cloud.exception.PermissionDeniedException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.offering.DiskOffering; +import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; +import com.cloud.storage.StoragePoolHostVO; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.storage.dao.StoragePoolHostDao; +import com.cloud.storage.dao.VMTemplatePoolDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.ResourceLimitService; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.DiskProfile; + +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.ResponseGenerator; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.admin.volume.ImportVolumeCmd; +import org.apache.cloudstack.api.command.admin.volume.ListVolumesForImportCmd; +import org.apache.cloudstack.api.command.admin.volume.UnmanageVolumeCmd; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.VolumeForImportResponse; +import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class VolumeImportUnmanagedManagerImpl implements VolumeImportUnmanageService { + protected Logger logger = LogManager.getLogger(VolumeImportUnmanagedManagerImpl.class); + + private static final List volumeImportUnmanageSupportedHypervisors = + Arrays.asList(Hypervisor.HypervisorType.KVM); + + @Inject + private AccountManager accountMgr; + @Inject + private AgentManager agentManager; + @Inject + private HostDao hostDao; + @Inject + private DiskOfferingDao diskOfferingDao; + @Inject + private ResourceLimitService resourceLimitService; + @Inject + private ResponseGenerator responseGenerator; + @Inject + private VolumeDao volumeDao; + @Inject + private PrimaryDataStoreDao primaryDataStoreDao; + @Inject + private StoragePoolHostDao storagePoolHostDao; + @Inject + private ConfigurationManager configMgr; + @Inject + private DataCenterDao dcDao; + @Inject + private VolumeOrchestrationService volumeManager; + @Inject + private VMTemplatePoolDao templatePoolDao; + + private static final String DEFAULT_DISK_OFFERING_NAME = "Default Custom Offering for Volume Import"; + private static final String DEFAULT_DISK_OFFERING_UNIQUE_NAME = "Custom-Offering-Volume-Import"; + + private void logFailureAndThrowException(String msg) { + logger.error(msg); + throw new CloudRuntimeException(msg); + } + + @Override + public List> getCommands() { + final List> cmdList = new ArrayList<>(); + cmdList.add(ListVolumesForImportCmd.class); + cmdList.add(ImportVolumeCmd.class); + cmdList.add(UnmanageVolumeCmd.class); + return cmdList; + } + + @Override + public ListResponse listVolumesForImport(ListVolumesForImportCmd cmd) { + Long poolId = cmd.getStorageId(); + + List volumes = listVolumesForImportInternal(poolId, null); + StoragePoolVO pool = checkIfPoolAvailable(poolId); + + List responses = new ArrayList<>(); + for (VolumeOnStorageTO volume : volumes) { + if (checkIfVolumeManaged(pool, volume.getPath()) || checkIfVolumeForTemplate(pool, volume.getPath())) { + continue; + } + responses.add(createVolumeForImportResponse(volume, pool)); + } + ListResponse listResponses = new ListResponse<>(); + listResponses.setResponses(responses, responses.size()); + return listResponses; + } + + @Override + public VolumeResponse importVolume(ImportVolumeCmd cmd) { + // 1. verify owner + final Account caller = CallContext.current().getCallingAccount(); + if (caller.getType() != Account.Type.ADMIN) { + throw new PermissionDeniedException(String.format("Cannot import VM as the caller account [%s] is not ROOT Admin.", caller.getUuid())); + } + Account owner = accountMgr.finalizeOwner(caller, cmd.getAccountName(), cmd.getDomainId(), cmd.getProjectId()); + if (owner == null) { + logFailureAndThrowException("Cannot import volume due to unknown owner"); + } + + // 2. check if pool exists and not in maintenance + Long poolId = cmd.getStorageId(); + StoragePoolVO pool = checkIfPoolAvailable(poolId); + + // 3. check if the volume already exists in cloudstack by path + String volumePath = cmd.getPath(); + if (checkIfVolumeManaged(pool, volumePath)){ + logFailureAndThrowException("Volume is already managed by CloudStack: " + volumePath); + } + if (checkIfVolumeForTemplate(pool, volumePath)) { + logFailureAndThrowException("Volume is a base image of a template: " + volumePath); + } + + // 4. send a command to hypervisor to check + List volumes = listVolumesForImportInternal(poolId, volumePath); + if (CollectionUtils.isEmpty(volumes)) { + logFailureAndThrowException("Cannot find volume on storage pool: " + volumePath); + } + + VolumeOnStorageTO volume = volumes.get(0); + + // 5. check resource limitation + checkResourceLimitForImportVolume(owner, volume); + + // 6. get disk offering + DiskOfferingVO diskOffering = getOrCreateDiskOffering(owner, cmd.getDiskOfferingId(), pool.getDataCenterId()); + + // 7. create records + VolumeVO volumeVO = createRecordsForVolumeImport(volume, diskOffering, owner, pool); + + // 8. Update resource count + updateResourceLimitForVolumeImport(volumeVO); + + // 9. Publish event + publicUsageEventForVolumeImportAndUnmanage(volumeVO, true); + + return responseGenerator.createVolumeResponse(ResponseObject.ResponseView.Full, volumeVO); + } + + private List listVolumesForImportInternal(Long poolId, String volumePath) { + StoragePoolVO pool = checkIfPoolAvailable(poolId); + + Pair hostAndLocalPath = findHostAndLocalPathForVolumeImport(pool); + HostVO host = hostAndLocalPath.first(); + if (!volumeImportUnmanageSupportedHypervisors.contains(host.getHypervisorType())) { + logFailureAndThrowException("Import VM is not supported for hypervisor: " + host.getHypervisorType()); + } + + StorageFilerTO storageTO = new StorageFilerTO(pool); + GetVolumesOnStorageCommand command = new GetVolumesOnStorageCommand(storageTO, volumePath); + Answer answer = agentManager.easySend(host.getId(), command); + if (answer == null || !(answer instanceof GetVolumesOnStorageAnswer)) { + logFailureAndThrowException("Cannot get volumes on storage pool via host " + host.getName()); + } + if (!answer.getResult()) { + logFailureAndThrowException("Volume cannot be imported due to " + answer.getDetails()); + } + List volumes = ((GetVolumesOnStorageAnswer) answer).getVolumes(); + return volumes; + } + + @Override + public boolean unmanageVolume(long volumeId) { + // 1. check if volume can be unmanaged + VolumeVO volume = checkIfVolumeCanBeUnmanaged(volumeId); + + // 2. check if pool available + StoragePoolVO pool = checkIfPoolAvailable(volume.getPoolId()); + + // 3. Update resource count + updateResourceLimitForVolumeUnmanage(volume); + + // 4. publish events + publicUsageEventForVolumeImportAndUnmanage(volume, false); + + // 5. update the state/removed of record + unmanageVolumeFromDatabase(volume); + + return true; + } + + private StoragePoolVO checkIfPoolAvailable(Long poolId) { + StoragePoolVO pool = primaryDataStoreDao.findById(poolId); + if (pool == null) { + logFailureAndThrowException("Storage pool does not exist: ID = " + poolId); + } + if (pool.isInMaintenance()) { + logFailureAndThrowException("Storage pool is in maintenance: " + pool.getName()); + } + return pool; + } + + private Pair findHostAndLocalPathForVolumeImport(StoragePoolVO pool) { + List hosts = new ArrayList<>(); + if (ScopeType.HOST.equals(pool.getScope())) { + List storagePoolHostVOs = storagePoolHostDao.listByPoolId(pool.getId()); + if (CollectionUtils.isNotEmpty(storagePoolHostVOs)) { + for (StoragePoolHostVO storagePoolHostVO : storagePoolHostVOs) { + HostVO host = hostDao.findById(storagePoolHostVO.getHostId()); + if (host != null) { + return new Pair<>(host, storagePoolHostVO.getLocalPath()); + } + } + } + } else if (ScopeType.CLUSTER.equals(pool.getScope())) { + hosts = hostDao.findHypervisorHostInCluster((pool.getClusterId())); + } else if (ScopeType.ZONE.equals(pool.getScope())) { + hosts = hostDao.listAllHostsUpByZoneAndHypervisor(pool.getDataCenterId(), pool.getHypervisor()); + } + for (HostVO host : hosts) { + StoragePoolHostVO storagePoolHostVO = storagePoolHostDao.findByPoolHost(pool.getId(), host.getId()); + if (storagePoolHostVO != null) { + return new Pair<>(host, storagePoolHostVO.getLocalPath()); + } + } + logFailureAndThrowException("No host found to perform volume import"); + return null; + } + + private VolumeForImportResponse createVolumeForImportResponse(VolumeOnStorageTO volume, StoragePoolVO pool) { + VolumeForImportResponse response = new VolumeForImportResponse(); + response.setPath(volume.getPath()); + response.setName(volume.getName()); + response.setFullPath(volume.getFullPath()); + response.setFormat(volume.getFormat()); // TODO: always qcow2 for kvm, which is incorrect + response.setSize(volume.getSize()); + response.setVirtualSize(volume.getVirtualSize()); + response.setQemuEncryptFormat(volume.getQemuEncryptFormat()); + // TODO: add more information of storage pool + + response.setObjectName("volumeforimport"); + return response; + } + + private boolean checkIfVolumeManaged(StoragePoolVO pool, String volumePath) { + return volumeDao.findByPoolIdAndPath(pool.getId(), volumePath) != null; + } + + private boolean checkIfVolumeForTemplate(StoragePoolVO pool, String volumePath) { + return templatePoolDao.findByPoolPath(pool.getId(), volumePath) != null; + } + + private DiskOfferingVO getOrCreateDiskOffering(Account owner, Long diskOfferingId, Long zoneId) { + if (diskOfferingId != null) { + // check if disk offering exists and active + DiskOfferingVO diskOfferingVO = diskOfferingDao.findById(diskOfferingId); + if (diskOfferingVO == null) { + logFailureAndThrowException(String.format("Disk offering does not exist", diskOfferingId)); + } + if (!DiskOffering.State.Active.equals(diskOfferingVO.getState())) { + logFailureAndThrowException(String.format("Disk offering with ID %s is not active", diskOfferingId)); + } + // check if disk offering is accessible by the account/owner + try { + configMgr.checkDiskOfferingAccess(owner, diskOfferingVO, dcDao.findById(zoneId)); + return diskOfferingVO; + } catch (PermissionDeniedException ignored) { + logFailureAndThrowException(String.format("Disk offering with ID %s is not accessible by account %s", diskOfferingId, owner.getAccountName())); + } + } + return getOrCreateDefaultDiskOfferingIdForVolumeImport(); + } + + + private DiskOfferingVO getOrCreateDefaultDiskOfferingIdForVolumeImport() { + DiskOfferingVO diskOffering = diskOfferingDao.findByUniqueName(DEFAULT_DISK_OFFERING_UNIQUE_NAME); + if (diskOffering != null) { + return diskOffering; + } + DiskOfferingVO newDiskOffering = new DiskOfferingVO(DEFAULT_DISK_OFFERING_NAME, DEFAULT_DISK_OFFERING_NAME, + Storage.ProvisioningType.THIN, 0, null, true, null, null, null); + newDiskOffering.setUniqueName(DEFAULT_DISK_OFFERING_UNIQUE_NAME); + newDiskOffering = diskOfferingDao.persistDefaultDiskOffering(newDiskOffering); + return newDiskOffering; + } + + private VolumeVO createRecordsForVolumeImport(VolumeOnStorageTO volume, DiskOfferingVO diskOffering, + Account owner, StoragePoolVO pool) { + DiskProfile diskProfile = volumeManager.importVolume(Volume.Type.DATADISK, volume.getName(), diskOffering, + volume.getVirtualSize(), null, null, pool.getDataCenterId(), pool.getHypervisor(), null, null, + owner, null, pool.getId(), volume.getPath(), null); + return volumeDao.findById(diskProfile.getVolumeId()); + } + + private void checkResourceLimitForImportVolume(Account owner, VolumeOnStorageTO volume) { + Long volumeSize = volume.getSize(); + try { + resourceLimitService.checkResourceLimit(owner, Resource.ResourceType.volume, 1); + resourceLimitService.checkResourceLimit(owner, Resource.ResourceType.primary_storage, volumeSize); + } catch (ResourceAllocationException e) { + logger.error(String.format("VM resource allocation error for account: %s", owner.getUuid()), e); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("VM resource allocation error for account: %s. %s", owner.getUuid(), StringUtils.defaultString(e.getMessage()))); + } + } + + private void updateResourceLimitForVolumeImport(VolumeVO volumeVO) { + resourceLimitService.incrementResourceCount(volumeVO.getAccountId(), Resource.ResourceType.volume); + resourceLimitService.incrementResourceCount(volumeVO.getAccountId(), Resource.ResourceType.primary_storage, volumeVO.getSize()); + } + + private void publicUsageEventForVolumeImportAndUnmanage(VolumeVO volumeVO, boolean isImport) { + try { + String eventType = isImport ? EventTypes.EVENT_VOLUME_IMPORT: EventTypes.EVENT_VOLUME_UNMANAGE; + UsageEventUtils.publishUsageEvent(eventType, volumeVO.getAccountId(), volumeVO.getDataCenterId(), + volumeVO.getId(), volumeVO.getName(), volumeVO.getDiskOfferingId(), null, volumeVO.getSize(), + Volume.class.getName(), volumeVO.getUuid(), volumeVO.isDisplayVolume()); + } catch (Exception e) { + logger.error(String.format("Failed to publish volume ID: %s usage records during volume import/unmanage", volumeVO.getUuid()), e); + } + } + + private void updateResourceLimitForVolumeUnmanage(VolumeVO volumeVO) { + resourceLimitService.decrementResourceCount(volumeVO.getAccountId(), Resource.ResourceType.volume); + resourceLimitService.decrementResourceCount(volumeVO.getAccountId(), Resource.ResourceType.primary_storage, volumeVO.getSize()); + } + + private VolumeVO checkIfVolumeCanBeUnmanaged(long volumeId) { + VolumeVO volumeVO = volumeDao.findById(volumeId); + if (volumeVO == null) { + logFailureAndThrowException(String.format("Volume (ID: %s) does not exist", volumeId)); + } + if (!Volume.State.Ready.equals(volumeVO.getState())) { + logFailureAndThrowException(String.format("Volume (ID: %s) is not ready", volumeId)); + } + if (volumeVO.getAttached() != null || volumeVO.getInstanceId() != null) { + logFailureAndThrowException(String.format("Volume (ID: %s) is attached to VM (ID: %s)", volumeId, volumeVO.getInstanceId())); + } + return volumeVO; + } + + private boolean unmanageVolumeFromDatabase(VolumeVO volume) { + volumeDao.remove(volume.getId()); + return true; + } +} diff --git a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java index 1ed5a8f1648a..674252b6a5f3 100644 --- a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java @@ -850,7 +850,7 @@ private Pair importDisk(UnmanagedInstanceTO.Disk disk, } StoragePool storagePool = getStoragePool(disk, zone, cluster); DiskProfile profile = volumeManager.importVolume(type, name, diskOffering, diskSize, - minIops, maxIops, vm, template, owner, deviceId, storagePool.getId(), path, chainInfo); + minIops, maxIops, vm.getDataCenterId(), vm.getHypervisorType(), vm, template, owner, deviceId, storagePool.getId(), path, chainInfo); return new Pair(profile, storagePool); } diff --git a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml index 7227264e2297..6d8a8a853bb7 100644 --- a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml @@ -360,4 +360,6 @@ + + diff --git a/test/integration/smoke/test_import_unmanage_volumes.py b/test/integration/smoke/test_import_unmanage_volumes.py new file mode 100644 index 000000000000..c2ea416536e8 --- /dev/null +++ b/test/integration/smoke/test_import_unmanage_volumes.py @@ -0,0 +1,168 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" Tests for importVolume and unmanageVolume APIs +""" +# Import Local Modules +from marvin.cloudstackAPI import unmanageVolume, listVolumesForImport, importVolume +from marvin.cloudstackTestCase import cloudstackTestCase, unittest +from marvin.codes import FAILED +from marvin.lib.base import (Account, + Domain, + Volume, + ServiceOffering, + DiskOffering, + VirtualMachine) +from marvin.lib.common import (get_domain, get_zone, get_suitable_test_template) + +# Import System modules +from nose.plugins.attrib import attr + +_multiprocess_shared_ = True + + +class TestImportAndUnmanageVolumes(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + testClient = super(TestImportAndUnmanageVolumes, cls).getClsTestClient() + cls.apiclient = testClient.getApiClient() + cls.testdata = cls.testClient.getParsedTestDataConfig() + + cls.services = testClient.getParsedTestDataConfig() + cls.hypervisor = testClient.getHypervisorInfo() + cls.domain = get_domain(cls.apiclient) + cls.zone = get_zone(cls.apiclient) + cls._cleanup = [] + + cls.service_offering = ServiceOffering.create( + cls.apiclient, + cls.services["service_offerings"]["tiny"] + ) + cls._cleanup.append(cls.service_offering) + + template = get_suitable_test_template( + cls.apiclient, + cls.zone.id, + cls.services["ostype"], + cls.hypervisor + ) + if template == FAILED: + assert False, "get_test_template() failed to return template" + + cls.services["virtual_machine"]["zoneid"] = cls.zone.id + cls.services["mode"] = cls.zone.networktype + + cls.disk_offering = DiskOffering.create(cls.apiclient, + cls.services["disk_offering"]) + cls._cleanup.append(cls.disk_offering) + + cls.test_domain = Domain.create( + cls.apiclient, + cls.services["domain"]) + cls._cleanup.append(cls.test_domain) + + cls.test_account = Account.create( + cls.apiclient, + cls.services["account"], + admin=True, + domainid=cls.test_domain.id) + cls._cleanup.append(cls.test_account) + + # Create VM + cls.virtual_machine = VirtualMachine.create( + cls.apiclient, + cls.services["virtual_machine"], + templateid=template.id, + accountid=cls.test_account.name, + domainid=cls.test_account.domainid, + serviceofferingid=cls.service_offering.id, + mode=cls.services["mode"] + ) + cls._cleanup.append(cls.virtual_machine) + + cls.virtual_machine.stop(cls.apiclient, forced=True) + + @classmethod + def tearDownClass(cls): + super(TestImportAndUnmanageVolumes, cls).tearDownClass() + + def setUp(self): + if self.testClient.getHypervisorInfo().lower() != "kvm": + raise unittest.SkipTest("This is only available for KVM") + + @attr(tags=['advanced', 'basic', 'sg'], required_hardware=False) + def test_01_detach_unmanage_import_volume(self): + """Test listing Volumes with account & domain filter + """ + # Create DATA volume + volume = Volume.create( + self.apiclient, + self.testdata["volume"], + zoneid=self.zone.id, + account=self.test_account.name, + domainid=self.test_account.domainid, + diskofferingid=self.disk_offering.id + ) + + # Attach and Detach volume + try: + self.virtual_machine.attach_volume(self.apiclient, volume) + except Exception as e: + self.fail("Attach volume failed with Exception: %s" % e) + + self.virtual_machine.detach_volume(self.apiclient, volume) + + # List volume by id + volumes = Volume.list(self.apiclient, + id = volume.id) + self.assertTrue(isinstance(volumes, list), + "listVolumes response should return a valid list" + ) + self.assertTrue(len(volumes) > 0, + "listVolumes response should return a non-empty list" + ) + volume = volumes[0] + + # Unmanage volume + cmd = unmanageVolume.unmanageVolumeCmd() + cmd.id = volume.id + self.apiclient.unmanageVolume(cmd) + + # List VMs for import + cmd = listVolumesForImport.listVolumesForImportCmd() + cmd.storageid = volume.storageid + volumesForImport = self.apiclient.listVolumesForImport(cmd) + self.assertTrue(isinstance(volumesForImport, list), + "Check listVolumesForImport response returns a valid list" + ) + + # Import volume + cmd = importVolume.importVolumeCmd() + cmd.storageid = volume.storageid + cmd.path = volume.path + self.apiclient.importVolume(cmd) + + # List volume by name + volumes = Volume.list(self.apiclient, + storageid = volume.storageid, + name=volume.path) + self.assertTrue(isinstance(volumes, list), + "listVolumes response should return a valid list" + ) + self.assertTrue(len(volumes) > 0, + "listVolumes response should return a non-empty list" + ) \ No newline at end of file From 3863353fafb4f658ca6daf0d6000969ad1bd90f0 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 20 Mar 2024 10:06:16 +0100 Subject: [PATCH 02/44] Update 8808: part1 --- .../apache/cloudstack/api/ApiConstants.java | 1 + .../api/response/VolumeForImportResponse.java | 24 +++++++++++++++++++ .../agent/api/GetVolumesOnStorageAnswer.java | 6 ++--- .../VolumeImportUnmanagedManagerImpl.java | 18 +++++++------- .../smoke/test_import_unmanage_volumes.py | 2 +- 5 files changed, 38 insertions(+), 13 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index fa8f9067fa1c..b5fab14ccb66 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -552,6 +552,7 @@ public class ApiConstants { public static final String ALLOCATION_STATE = "allocationstate"; public static final String MANAGED_STATE = "managedstate"; public static final String MANAGEMENT_SERVER_ID = "managementserverid"; + public static final String STORAGE = "storage"; public static final String STORAGE_ID = "storageid"; public static final String PING_STORAGE_SERVER_IP = "pingstorageserverip"; public static final String PING_DIR = "pingdir"; diff --git a/api/src/main/java/org/apache/cloudstack/api/response/VolumeForImportResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VolumeForImportResponse.java index 045a7b85e0d3..a3fc944451a3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/VolumeForImportResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/VolumeForImportResponse.java @@ -58,6 +58,14 @@ public class VolumeForImportResponse extends BaseResponse { @Param(description = "the encrypt format of the volume") private String qemuEncryptFormat; + @SerializedName(ApiConstants.STORAGE_ID) + @Param(description = "id of the primary storage hosting the volume") + private String storagePoolId; + + @SerializedName(ApiConstants.STORAGE) + @Param(description = "name of the primary storage hosting the volume") + private String storagePoolName; + @SerializedName(ApiConstants.DETAILS) @Param(description = "volume details in key/value pairs.") private Map details; @@ -122,6 +130,22 @@ public void setQemuEncryptFormat(String qemuEncryptFormat) { this.qemuEncryptFormat = qemuEncryptFormat; } + public String getStoragePoolId() { + return storagePoolId; + } + + public void setStoragePoolId(String storagePoolId) { + this.storagePoolId = storagePoolId; + } + + public String getStoragePoolName() { + return storagePoolName; + } + + public void setStoragePoolName(String storagePoolName) { + this.storagePoolName = storagePoolName; + } + public Map getDetails() { return details; } diff --git a/core/src/main/java/com/cloud/agent/api/GetVolumesOnStorageAnswer.java b/core/src/main/java/com/cloud/agent/api/GetVolumesOnStorageAnswer.java index 89f256867c29..3c46994499d9 100644 --- a/core/src/main/java/com/cloud/agent/api/GetVolumesOnStorageAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/GetVolumesOnStorageAnswer.java @@ -22,12 +22,12 @@ import java.util.List; public class GetVolumesOnStorageAnswer extends Answer { - private List volumes; + private List volumes; GetVolumesOnStorageAnswer() { } - public GetVolumesOnStorageAnswer(GetVolumesOnStorageCommand cmd, List volumes) { + public GetVolumesOnStorageAnswer(GetVolumesOnStorageCommand cmd, List volumes) { super(cmd, true, null); this.volumes = volumes; } @@ -36,7 +36,7 @@ public GetVolumesOnStorageAnswer(final GetVolumesOnStorageCommand cmd, final boo super(cmd, success, details); } - public List getVolumes() { + public List getVolumes() { return volumes; } } diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanagedManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanagedManagerImpl.java index dc525e0d1812..cb45fb863b39 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanagedManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanagedManagerImpl.java @@ -128,8 +128,8 @@ public List> getCommands() { public ListResponse listVolumesForImport(ListVolumesForImportCmd cmd) { Long poolId = cmd.getStorageId(); - List volumes = listVolumesForImportInternal(poolId, null); StoragePoolVO pool = checkIfPoolAvailable(poolId); + List volumes = listVolumesForImportInternal(poolId, null); List responses = new ArrayList<>(); for (VolumeOnStorageTO volume : volumes) { @@ -169,7 +169,7 @@ public VolumeResponse importVolume(ImportVolumeCmd cmd) { } // 4. send a command to hypervisor to check - List volumes = listVolumesForImportInternal(poolId, volumePath); + List volumes = listVolumesForImportInternal(poolId, volumePath); if (CollectionUtils.isEmpty(volumes)) { logFailureAndThrowException("Cannot find volume on storage pool: " + volumePath); } @@ -194,7 +194,7 @@ public VolumeResponse importVolume(ImportVolumeCmd cmd) { return responseGenerator.createVolumeResponse(ResponseObject.ResponseView.Full, volumeVO); } - private List listVolumesForImportInternal(Long poolId, String volumePath) { + private List listVolumesForImportInternal(Long poolId, String volumePath) { StoragePoolVO pool = checkIfPoolAvailable(poolId); Pair hostAndLocalPath = findHostAndLocalPathForVolumeImport(pool); @@ -212,8 +212,7 @@ private List listVolumesForImportInternal(Long pool if (!answer.getResult()) { logFailureAndThrowException("Volume cannot be imported due to " + answer.getDetails()); } - List volumes = ((GetVolumesOnStorageAnswer) answer).getVolumes(); - return volumes; + return ((GetVolumesOnStorageAnswer) answer).getVolumes(); } @Override @@ -279,12 +278,13 @@ private VolumeForImportResponse createVolumeForImportResponse(VolumeOnStorageTO response.setPath(volume.getPath()); response.setName(volume.getName()); response.setFullPath(volume.getFullPath()); - response.setFormat(volume.getFormat()); // TODO: always qcow2 for kvm, which is incorrect + response.setFormat(volume.getFormat()); response.setSize(volume.getSize()); response.setVirtualSize(volume.getVirtualSize()); response.setQemuEncryptFormat(volume.getQemuEncryptFormat()); - // TODO: add more information of storage pool - + response.setStoragePoolId(pool.getUuid()); + response.setStoragePoolName(pool.getName()); + response.setDetails(volume.getDetails()); response.setObjectName("volumeforimport"); return response; } @@ -302,7 +302,7 @@ private DiskOfferingVO getOrCreateDiskOffering(Account owner, Long diskOfferingI // check if disk offering exists and active DiskOfferingVO diskOfferingVO = diskOfferingDao.findById(diskOfferingId); if (diskOfferingVO == null) { - logFailureAndThrowException(String.format("Disk offering does not exist", diskOfferingId)); + logFailureAndThrowException(String.format("Disk offering %s does not exist", diskOfferingId)); } if (!DiskOffering.State.Active.equals(diskOfferingVO.getState())) { logFailureAndThrowException(String.format("Disk offering with ID %s is not active", diskOfferingId)); diff --git a/test/integration/smoke/test_import_unmanage_volumes.py b/test/integration/smoke/test_import_unmanage_volumes.py index c2ea416536e8..d32633252b07 100644 --- a/test/integration/smoke/test_import_unmanage_volumes.py +++ b/test/integration/smoke/test_import_unmanage_volumes.py @@ -165,4 +165,4 @@ def test_01_detach_unmanage_import_volume(self): ) self.assertTrue(len(volumes) > 0, "listVolumes response should return a non-empty list" - ) \ No newline at end of file + ) From 5a28f97c39c6ed1f6f2d024f3289e52bbd878524 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 20 Mar 2024 13:45:22 +0100 Subject: [PATCH 03/44] Update 8808: part2 for local storage --- .../api/response/VolumeForImportResponse.java | 12 +++++ ...oudStackPrimaryDataStoreLifeCycleImpl.java | 2 +- .../VolumeImportUnmanagedManagerImpl.java | 50 +++++++++++++------ 3 files changed, 47 insertions(+), 17 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/response/VolumeForImportResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VolumeForImportResponse.java index a3fc944451a3..803f154816a8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/VolumeForImportResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/VolumeForImportResponse.java @@ -66,6 +66,10 @@ public class VolumeForImportResponse extends BaseResponse { @Param(description = "name of the primary storage hosting the volume") private String storagePoolName; + @SerializedName(ApiConstants.STORAGE_TYPE) + @Param(description = "type of the primary storage hosting the volume") + private String storagePoolType; + @SerializedName(ApiConstants.DETAILS) @Param(description = "volume details in key/value pairs.") private Map details; @@ -146,6 +150,14 @@ public void setStoragePoolName(String storagePoolName) { this.storagePoolName = storagePoolName; } + public String getStoragePoolType() { + return storagePoolType; + } + + public void setStoragePoolType(String storagePoolType) { + this.storagePoolType = storagePoolType; + } + public Map getDetails() { return details; } diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java index 685565d73b03..419d4c0ce7c6 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java @@ -147,7 +147,7 @@ public DataStore initialize(Map dsInfos) { String uri = String.format("%s://%s%s", scheme, storageHost, hostPath); Object localStorage = dsInfos.get("localStorage"); - if (localStorage != null) { + if (localStorage != null) { hostPath = hostPath.contains("//") ? hostPath.replaceFirst("/", "") : hostPath; hostPath = hostPath.replace("+", " "); } diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanagedManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanagedManagerImpl.java index cb45fb863b39..d0e3a550a6a4 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanagedManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanagedManagerImpl.java @@ -108,7 +108,9 @@ public class VolumeImportUnmanagedManagerImpl implements VolumeImportUnmanageSer private VMTemplatePoolDao templatePoolDao; private static final String DEFAULT_DISK_OFFERING_NAME = "Default Custom Offering for Volume Import"; - private static final String DEFAULT_DISK_OFFERING_UNIQUE_NAME = "Custom-Offering-Volume-Import"; + private static final String DEFAULT_DISK_OFFERING_UNIQUE_NAME = "Custom-Volume-Import"; + private static final String DEFAULT_DISK_OFFERING_NAME_LOCAL = DEFAULT_DISK_OFFERING_NAME + " - Local Storage"; + private static final String DEFAULT_DISK_OFFERING_UNIQUE_NAME_LOCAL = DEFAULT_DISK_OFFERING_UNIQUE_NAME + "-Local"; private void logFailureAndThrowException(String msg) { logger.error(msg); @@ -180,7 +182,7 @@ public VolumeResponse importVolume(ImportVolumeCmd cmd) { checkResourceLimitForImportVolume(owner, volume); // 6. get disk offering - DiskOfferingVO diskOffering = getOrCreateDiskOffering(owner, cmd.getDiskOfferingId(), pool.getDataCenterId()); + DiskOfferingVO diskOffering = getOrCreateDiskOffering(owner, cmd.getDiskOfferingId(), pool.getDataCenterId(), pool.isLocal()); // 7. create records VolumeVO volumeVO = createRecordsForVolumeImport(volume, diskOffering, owner, pool); @@ -284,6 +286,7 @@ private VolumeForImportResponse createVolumeForImportResponse(VolumeOnStorageTO response.setQemuEncryptFormat(volume.getQemuEncryptFormat()); response.setStoragePoolId(pool.getUuid()); response.setStoragePoolName(pool.getName()); + response.setStoragePoolType(String.valueOf(pool.getPoolType())); response.setDetails(volume.getDetails()); response.setObjectName("volumeforimport"); return response; @@ -297,7 +300,7 @@ private boolean checkIfVolumeForTemplate(StoragePoolVO pool, String volumePath) return templatePoolDao.findByPoolPath(pool.getId(), volumePath) != null; } - private DiskOfferingVO getOrCreateDiskOffering(Account owner, Long diskOfferingId, Long zoneId) { + private DiskOfferingVO getOrCreateDiskOffering(Account owner, Long diskOfferingId, Long zoneId, boolean isLocal) { if (diskOfferingId != null) { // check if disk offering exists and active DiskOfferingVO diskOfferingVO = diskOfferingDao.findById(diskOfferingId); @@ -307,28 +310,43 @@ private DiskOfferingVO getOrCreateDiskOffering(Account owner, Long diskOfferingI if (!DiskOffering.State.Active.equals(diskOfferingVO.getState())) { logFailureAndThrowException(String.format("Disk offering with ID %s is not active", diskOfferingId)); } + if (diskOfferingVO.isUseLocalStorage() != isLocal) { + logFailureAndThrowException(String.format("Disk offering with ID %s should use %s storage", diskOfferingId, isLocal ? "local": "shared")); + } // check if disk offering is accessible by the account/owner try { configMgr.checkDiskOfferingAccess(owner, diskOfferingVO, dcDao.findById(zoneId)); return diskOfferingVO; - } catch (PermissionDeniedException ignored) { - logFailureAndThrowException(String.format("Disk offering with ID %s is not accessible by account %s", diskOfferingId, owner.getAccountName())); + } catch (PermissionDeniedException ex) { + logFailureAndThrowException(String.format("Disk offering with ID %s is not accessible by owner %s", diskOfferingId, owner)); } } - return getOrCreateDefaultDiskOfferingIdForVolumeImport(); + return getOrCreateDefaultDiskOfferingIdForVolumeImport(isLocal); } - - private DiskOfferingVO getOrCreateDefaultDiskOfferingIdForVolumeImport() { - DiskOfferingVO diskOffering = diskOfferingDao.findByUniqueName(DEFAULT_DISK_OFFERING_UNIQUE_NAME); - if (diskOffering != null) { - return diskOffering; + private DiskOfferingVO getOrCreateDefaultDiskOfferingIdForVolumeImport(boolean isLocalStorage) { + if (isLocalStorage) { + DiskOfferingVO diskOffering = diskOfferingDao.findByUniqueName(DEFAULT_DISK_OFFERING_UNIQUE_NAME_LOCAL); + if (diskOffering != null) { + return diskOffering; + } + DiskOfferingVO newDiskOffering = new DiskOfferingVO(DEFAULT_DISK_OFFERING_NAME_LOCAL, DEFAULT_DISK_OFFERING_NAME_LOCAL, + Storage.ProvisioningType.THIN, 0, null, true, null, null, null); + newDiskOffering.setUseLocalStorage(true); + newDiskOffering.setUniqueName(DEFAULT_DISK_OFFERING_UNIQUE_NAME_LOCAL); + newDiskOffering = diskOfferingDao.persistDefaultDiskOffering(newDiskOffering); + return newDiskOffering; + } else { + DiskOfferingVO diskOffering = diskOfferingDao.findByUniqueName(DEFAULT_DISK_OFFERING_UNIQUE_NAME); + if (diskOffering != null) { + return diskOffering; + } + DiskOfferingVO newDiskOffering = new DiskOfferingVO(DEFAULT_DISK_OFFERING_NAME, DEFAULT_DISK_OFFERING_NAME, + Storage.ProvisioningType.THIN, 0, null, true, null, null, null); + newDiskOffering.setUniqueName(DEFAULT_DISK_OFFERING_UNIQUE_NAME); + newDiskOffering = diskOfferingDao.persistDefaultDiskOffering(newDiskOffering); + return newDiskOffering; } - DiskOfferingVO newDiskOffering = new DiskOfferingVO(DEFAULT_DISK_OFFERING_NAME, DEFAULT_DISK_OFFERING_NAME, - Storage.ProvisioningType.THIN, 0, null, true, null, null, null); - newDiskOffering.setUniqueName(DEFAULT_DISK_OFFERING_UNIQUE_NAME); - newDiskOffering = diskOfferingDao.persistDefaultDiskOffering(newDiskOffering); - return newDiskOffering; } private VolumeVO createRecordsForVolumeImport(VolumeOnStorageTO volume, DiskOfferingVO diskOffering, From bb35235e3eae72b53cb242293fb508fbe8607028 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 20 Mar 2024 14:00:30 +0100 Subject: [PATCH 04/44] Update 8808: rename VolumeImportUnmanagedManagerImpl to VolumeImportUnmanageManagerImpl --- ...dManagerImpl.java => VolumeImportUnmanageManagerImpl.java} | 4 ++-- .../cloudstack/core/spring-server-core-managers-context.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename server/src/main/java/org/apache/cloudstack/storage/volume/{VolumeImportUnmanagedManagerImpl.java => VolumeImportUnmanageManagerImpl.java} (99%) diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanagedManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java similarity index 99% rename from server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanagedManagerImpl.java rename to server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index d0e3a550a6a4..f837422d25b0 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanagedManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -74,8 +74,8 @@ import java.util.Arrays; import java.util.List; -public class VolumeImportUnmanagedManagerImpl implements VolumeImportUnmanageService { - protected Logger logger = LogManager.getLogger(VolumeImportUnmanagedManagerImpl.class); +public class VolumeImportUnmanageManagerImpl implements VolumeImportUnmanageService { + protected Logger logger = LogManager.getLogger(VolumeImportUnmanageManagerImpl.class); private static final List volumeImportUnmanageSupportedHypervisors = Arrays.asList(Hypervisor.HypervisorType.KVM); diff --git a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml index 6d8a8a853bb7..735a9075f9e9 100644 --- a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml @@ -361,5 +361,5 @@ - + From 388e7e2e3aaf37dd541ef0332f89c68de5acda5a Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 21 Mar 2024 09:55:42 +0100 Subject: [PATCH 05/44] Update 8808: list volumesforimport by path --- .../admin/volume/ListVolumesForImportCmd.java | 9 +++++++++ ...bvirtGetVolumesOnStorageCommandWrapper.java | 18 ++++++++---------- .../VolumeImportUnmanageManagerImpl.java | 3 ++- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ListVolumesForImportCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ListVolumesForImportCmd.java index 6dbadf3c1cb9..ea1111f3efe2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ListVolumesForImportCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ListVolumesForImportCmd.java @@ -65,6 +65,11 @@ public class ListVolumesForImportCmd extends BaseListCmd { description = "the ID of the storage pool") private Long storageId; + @Parameter(name = ApiConstants.PATH, + type = BaseCmd.CommandType.STRING, + description = "the path of the volume on the storage pool") + private String path; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -73,6 +78,10 @@ public Long getStorageId() { return storageId; } + public String getPath() { + return path; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java index f4b9cb1de798..aff5b0b8951e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java @@ -41,6 +41,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -54,9 +55,6 @@ public final class LibvirtGetVolumesOnStorageCommandWrapper extends CommandWrapp public Answer execute(final GetVolumesOnStorageCommand command, final LibvirtComputingResource libvirtComputingResource) { final StorageFilerTO pool = command.getPool(); - if (!SUPPORTED_STORAGE_POOL_TYPES.contains(pool.getType())) { - return new GetVolumesOnStorageAnswer(command, false, String.format("pool type %s is unsupported", pool.getType())); - } final String volumePath = command.getVolumePath(); List volumes = new ArrayList<>(); @@ -64,7 +62,7 @@ public Answer execute(final GetVolumesOnStorageCommand command, final LibvirtCom final KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr(); KVMStoragePool storagePool = storagePoolMgr.getStoragePool(pool.getType(), pool.getUuid(), true); - if (volumePath != null) { + if (StringUtils.isNotBlank(volumePath)) { KVMPhysicalDisk disk = storagePool.getPhysicalDisk(volumePath); if (disk != null) { if (!isDiskFormatSupported(disk)) { @@ -121,14 +119,14 @@ private boolean isDiskFormatSupported(KVMPhysicalDisk disk) { } private boolean isDiskFileLocked(KVMStoragePool pool, KVMPhysicalDisk disk) { - if (PhysicalDiskFormat.QCOW2.equals(disk.getFormat())) { - Map info = getDiskFileInfo(pool, disk, false); - return info == null; - } - return false; // unknown + Map info = getDiskFileInfo(pool, disk, false); + return info == null; } private Map getDiskFileInfo(KVMStoragePool pool, KVMPhysicalDisk disk, boolean secure) { + if (!SUPPORTED_STORAGE_POOL_TYPES.contains(pool.getType())) { + return new HashMap<>(); // unknown + } try { QemuImg qemu = new QemuImg(0); QemuImgFile qemuFile = new QemuImgFile(disk.getPath(), disk.getFormat()); @@ -143,7 +141,7 @@ private Map getDiskFileInfo(KVMStoragePool pool, KVMPhysicalDisk return qemu.info(qemuFile, secure); } catch (QemuImgException | LibvirtException ex) { logger.error("Failed to get info of disk file: " + ex.getMessage()); + return null; } - return null; } } diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index f837422d25b0..0db9b2dc6194 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -129,9 +129,10 @@ public List> getCommands() { @Override public ListResponse listVolumesForImport(ListVolumesForImportCmd cmd) { Long poolId = cmd.getStorageId(); + String path = cmd.getPath(); StoragePoolVO pool = checkIfPoolAvailable(poolId); - List volumes = listVolumesForImportInternal(poolId, null); + List volumes = listVolumesForImportInternal(poolId, path); List responses = new ArrayList<>(); for (VolumeOnStorageTO volume : volumes) { From b77c6eb02c73aba4320decb3d3e5b96af964ab90 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 21 Mar 2024 10:51:29 +0100 Subject: [PATCH 06/44] Update 8808: check if volume is locked --- .../volume/VolumeImportUnmanageManagerImpl.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index 0db9b2dc6194..2d5fbf575767 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -73,6 +73,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; public class VolumeImportUnmanageManagerImpl implements VolumeImportUnmanageService { protected Logger logger = LogManager.getLogger(VolumeImportUnmanageManagerImpl.class); @@ -179,6 +180,9 @@ public VolumeResponse importVolume(ImportVolumeCmd cmd) { VolumeOnStorageTO volume = volumes.get(0); + // check if volume is locked + checkIfVolumeIsLocked(volume); + // 5. check resource limitation checkResourceLimitForImportVolume(owner, volume); @@ -301,6 +305,16 @@ private boolean checkIfVolumeForTemplate(StoragePoolVO pool, String volumePath) return templatePoolDao.findByPoolPath(pool.getId(), volumePath) != null; } + private void checkIfVolumeIsLocked(VolumeOnStorageTO volume) { + Map volumeDetails = volume.getDetails(); + if (volumeDetails != null && volumeDetails.containsKey(VolumeOnStorageTO.Detail.IS_LOCKED)) { + String isLocked = volumeDetails.get(VolumeOnStorageTO.Detail.IS_LOCKED); + if (Boolean.parseBoolean(isLocked)) { + logFailureAndThrowException("Volume is locked"); + } + } + } + private DiskOfferingVO getOrCreateDiskOffering(Account owner, Long diskOfferingId, Long zoneId, boolean isLocal) { if (diskOfferingId != null) { // check if disk offering exists and active From 01059e977ed1ae8d6a9778ffbd43844c6c4aee82 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 21 Mar 2024 11:58:33 +0100 Subject: [PATCH 07/44] Update 8808: Do not import Encrypted disk (could be supported later) --- .../storage/volume/VolumeOnStorageTO.java | 2 +- .../LibvirtGetVolumesOnStorageCommandWrapper.java | 5 +++++ .../org/apache/cloudstack/utils/qemu/QemuImg.java | 1 + .../volume/VolumeImportUnmanageManagerImpl.java | 13 ++++++++++++- .../smoke/test_import_unmanage_volumes.py | 2 +- 5 files changed, 20 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTO.java b/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTO.java index 80e93c9f4225..b44767bd8d82 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTO.java +++ b/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTO.java @@ -36,7 +36,7 @@ public class VolumeOnStorageTO { private Map details = new HashMap<>(); public enum Detail { - BACKING_FILE, BACKING_FILE_FORMAT, CLUSTER_SIZE, FILE_FORMAT, IS_LOCKED + BACKING_FILE, BACKING_FILE_FORMAT, CLUSTER_SIZE, FILE_FORMAT, IS_LOCKED, IS_ENCRYPTED } public VolumeOnStorageTO() { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java index aff5b0b8951e..4971eb333bbb 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java @@ -93,8 +93,13 @@ public Answer execute(final GetVolumesOnStorageCommand command, final LibvirtCom if (StringUtils.isNotBlank(clusterSize)) { volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.FILE_FORMAT, fileFormat); } + String encrypted = info.get(QemuImg.ENCRYPTED); + if (StringUtils.isNotBlank(encrypted) && encrypted.toLowerCase().equals("yes")) { + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.IS_ENCRYPTED, String.valueOf(Boolean.TRUE)); + } Boolean isLocked = isDiskFileLocked(storagePool, disk); volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.IS_LOCKED, String.valueOf(isLocked)); + volumes.add(volumeOnStorageTO); } } else { diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java index 04f7b36bbae0..ca9ab4c8308d 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java @@ -45,6 +45,7 @@ public class QemuImg { public static final String FILE_FORMAT = "file_format"; public static final String IMAGE = "image"; public static final String VIRTUAL_SIZE = "virtual_size"; + public static final String ENCRYPTED = "encrypted"; public static final String ENCRYPT_FORMAT = "encrypt.format"; public static final String ENCRYPT_KEY_SECRET = "encrypt.key-secret"; public static final String TARGET_ZERO_FLAG = "--target-is-zero"; diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index 2d5fbf575767..7bbbbfe53818 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -182,6 +182,7 @@ public VolumeResponse importVolume(ImportVolumeCmd cmd) { // check if volume is locked checkIfVolumeIsLocked(volume); + checkIfVolumeIsEncrypted(volume); // 5. check resource limitation checkResourceLimitForImportVolume(owner, volume); @@ -310,7 +311,17 @@ private void checkIfVolumeIsLocked(VolumeOnStorageTO volume) { if (volumeDetails != null && volumeDetails.containsKey(VolumeOnStorageTO.Detail.IS_LOCKED)) { String isLocked = volumeDetails.get(VolumeOnStorageTO.Detail.IS_LOCKED); if (Boolean.parseBoolean(isLocked)) { - logFailureAndThrowException("Volume is locked"); + logFailureAndThrowException("Locked volume cannot be imported."); + } + } + } + + private void checkIfVolumeIsEncrypted(VolumeOnStorageTO volume) { + Map volumeDetails = volume.getDetails(); + if (volumeDetails != null && volumeDetails.containsKey(VolumeOnStorageTO.Detail.IS_ENCRYPTED)) { + String isEncrypted = volumeDetails.get(VolumeOnStorageTO.Detail.IS_ENCRYPTED); + if (Boolean.parseBoolean(isEncrypted)) { + logFailureAndThrowException("Encrypted volume cannot be imported for now."); } } } diff --git a/test/integration/smoke/test_import_unmanage_volumes.py b/test/integration/smoke/test_import_unmanage_volumes.py index d32633252b07..abd49215834e 100644 --- a/test/integration/smoke/test_import_unmanage_volumes.py +++ b/test/integration/smoke/test_import_unmanage_volumes.py @@ -106,7 +106,7 @@ def setUp(self): @attr(tags=['advanced', 'basic', 'sg'], required_hardware=False) def test_01_detach_unmanage_import_volume(self): - """Test listing Volumes with account & domain filter + """Test attach/detach/unmanage/import volume """ # Create DATA volume volume = Volume.create( From c61d13f823e5b91541c270d5898b0ba9322dff06 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 21 Mar 2024 14:59:17 +0100 Subject: [PATCH 08/44] Update 8088: revert VMware changes --- .../vmware/resource/VmwareResource.java | 54 ------------------- 1 file changed, 54 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index f1ab96682bbf..1c244645033e 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -48,8 +48,6 @@ import javax.naming.ConfigurationException; import javax.xml.datatype.XMLGregorianCalendar; -import com.cloud.agent.api.GetVolumesOnStorageAnswer; -import com.cloud.agent.api.GetVolumesOnStorageCommand; import com.cloud.hypervisor.vmware.mo.HostDatastoreBrowserMO; import com.vmware.vim25.FileInfo; import com.vmware.vim25.FileQueryFlags; @@ -68,7 +66,6 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; -import org.apache.cloudstack.storage.volume.VolumeOnStorageTO; import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo; import org.apache.cloudstack.vm.UnmanagedInstanceTO; import org.apache.commons.collections.CollectionUtils; @@ -530,8 +527,6 @@ public Answer executeRequest(Command cmd) { answer = execute((DeleteStoragePoolCommand) cmd); } else if (clz == CopyVolumeCommand.class) { answer = execute((CopyVolumeCommand) cmd); - } else if (clz == GetVolumesOnStorageCommand.class) { - answer = execute((GetVolumesOnStorageCommand) cmd); } else if (clz == AttachIsoCommand.class) { answer = execute((AttachIsoCommand) cmd); } else if (clz == ValidateSnapshotCommand.class) { @@ -5940,55 +5935,6 @@ public CopyVolumeAnswer execute(CopyVolumeCommand cmd) { } } - private GetVolumesOnStorageAnswer execute(GetVolumesOnStorageCommand cmd) { - List volumes = new ArrayList<>(); - try { - VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext()); - StorageFilerTO pool = cmd.getPool(); - - if (!Arrays.asList(StoragePoolType.NetworkFilesystem, StoragePoolType.VMFS, StoragePoolType.PreSetup, - StoragePoolType.DatastoreCluster).contains(pool.getType())) { - throw new Exception("Unsupported storage pool type " + pool.getType()); - } - - ManagedObjectReference morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, pool.getUuid()); - - if (morDatastore == null) { - boolean vmfsDatastore = Arrays.asList(StoragePoolType.VMFS, StoragePoolType.PreSetup, StoragePoolType.DatastoreCluster).contains(pool.getType()); - morDatastore = hyperHost.mountDatastore(vmfsDatastore, pool.getHost(), pool.getPort(), pool.getPath(), pool.getUuid().replace("-", ""), true); - } - - assert (morDatastore != null); - - DatastoreMO dsMo = new DatastoreMO(getServiceContext(), morDatastore); - - HostDatastoreBrowserMO browserMo = dsMo.getHostDatastoreBrowserMO(); - FileQueryFlags fqf = new FileQueryFlags(); - fqf.setFileSize(true); - fqf.setFileType(true); - fqf.setModification(true); - fqf.setFileOwner(false); - - HostDatastoreBrowserSearchSpec spec = new HostDatastoreBrowserSearchSpec(); - spec.setSearchCaseInsensitive(true); - spec.setDetails(fqf); - - String dsPath = String.format("[%s]", dsMo.getName()); - - HostDatastoreBrowserSearchResults results = browserMo.searchDatastore(dsPath, spec); - List fileInfoList = results.getFile(); - for (FileInfo file : fileInfoList) { - VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(HypervisorType.VMware, file.getPath(), - file.getFriendlyName(), file.getFileSize()); - volumes.add(volumeOnStorageTO); - } - } catch (Exception e) { - return new GetVolumesOnStorageAnswer(cmd, false, e.getMessage()); - - } - return new GetVolumesOnStorageAnswer(cmd, volumes); - } - @Override public void disconnected() { } From deb5e12e0e4609599ab3947c33bfc7545193069d Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 21 Mar 2024 12:54:46 +0100 Subject: [PATCH 09/44] Update 8088: do not unmanage encrypted volumes --- .../storage/volume/VolumeImportUnmanageManagerImpl.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index 7bbbbfe53818..affb14ef7e2d 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -423,14 +423,16 @@ private VolumeVO checkIfVolumeCanBeUnmanaged(long volumeId) { if (!Volume.State.Ready.equals(volumeVO.getState())) { logFailureAndThrowException(String.format("Volume (ID: %s) is not ready", volumeId)); } + if (volumeVO.getEncryptFormat() != null) { + logFailureAndThrowException(String.format("Volume (ID: %s) is encrypted", volumeId)); + } if (volumeVO.getAttached() != null || volumeVO.getInstanceId() != null) { logFailureAndThrowException(String.format("Volume (ID: %s) is attached to VM (ID: %s)", volumeId, volumeVO.getInstanceId())); } return volumeVO; } - private boolean unmanageVolumeFromDatabase(VolumeVO volume) { + private void unmanageVolumeFromDatabase(VolumeVO volume) { volumeDao.remove(volume.getId()); - return true; } } From 167f5902be3acb236210b5aa72aae292827705dc Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 21 Mar 2024 14:59:43 +0100 Subject: [PATCH 10/44] Update 8808: add unit tests --- .../command/admin/volume/ImportVolumeCmd.java | 24 ------ .../admin/volume/ImportVolumeCmdTest.java | 69 +++++++++++++++++ .../volume/ListVolumesForImportCmdTest.java | 58 +++++++++++++++ .../admin/volume/UnmanageVolumeCmdTest.java | 54 ++++++++++++++ .../response/VolumeForImportResponseTest.java | 74 +++++++++++++++++++ .../storage/volume/VolumeOnStorageTOTest.java | 49 +++++++++++- .../api/GetVolumesOnStorageAnswerTest.java | 7 ++ .../api/GetVolumesOnStorageCommandTest.java | 1 + 8 files changed, 310 insertions(+), 26 deletions(-) create mode 100644 api/src/test/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmdTest.java create mode 100644 api/src/test/java/org/apache/cloudstack/api/command/admin/volume/ListVolumesForImportCmdTest.java create mode 100644 api/src/test/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmdTest.java create mode 100644 api/src/test/java/org/apache/cloudstack/api/response/VolumeForImportResponseTest.java diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java index cfd49222f66e..944e1cc6b239 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java @@ -105,50 +105,26 @@ public String getPath() { return path; } - public void setPath(String path) { - this.path = path; - } - public Long getStorageId() { return storageId; } - public void setStorageId(Long storageId) { - this.storageId = storageId; - } - public Long getDiskOfferingId() { return diskOfferingId; } - public void setDiskOfferingId(Long diskOfferingId) { - this.diskOfferingId = diskOfferingId; - } - public String getAccountName() { return accountName; } - public void setAccountName(String accountName) { - this.accountName = accountName; - } - public Long getDomainId() { return domainId; } - public void setDomainId(Long domainId) { - this.domainId = domainId; - } - public Long getProjectId() { return projectId; } - public void setProjectId(Long projectId) { - this.projectId = projectId; - } - @Override public String getEventType() { return EventTypes.EVENT_VOLUME_IMPORT; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmdTest.java new file mode 100644 index 000000000000..36f2f82d0e13 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmdTest.java @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.volume; + +import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.storage.volume.VolumeImportUnmanageService; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +@RunWith(MockitoJUnitRunner.class) +public class ImportVolumeCmdTest { + + VolumeImportUnmanageService volumeImportService = Mockito.spy(VolumeImportUnmanageService.class); + + @Test + public void testImportVolumeCmd() { + String path = "file path"; + Long storageId = 2L; + Long diskOfferingId = 3L; + String accountName = "account"; + Long domainId = 4L; + Long projectId = 5L; + + ImportVolumeCmd cmd = new ImportVolumeCmd(); + ReflectionTestUtils.setField(cmd, "path", path); + ReflectionTestUtils.setField(cmd, "storageId", storageId); + ReflectionTestUtils.setField(cmd, "diskOfferingId", diskOfferingId); + ReflectionTestUtils.setField(cmd, "accountName", accountName); + ReflectionTestUtils.setField(cmd, "domainId", domainId); + ReflectionTestUtils.setField(cmd, "projectId", projectId); + ReflectionTestUtils.setField(cmd,"volumeImportService", volumeImportService); + + Assert.assertEquals(path, cmd.getPath()); + Assert.assertEquals(storageId, cmd.getStorageId()); + Assert.assertEquals(diskOfferingId, cmd.getDiskOfferingId()); + Assert.assertEquals(accountName, cmd.getAccountName()); + Assert.assertEquals(domainId, cmd.getDomainId()); + Assert.assertEquals(projectId, cmd.getProjectId()); + + VolumeResponse response = Mockito.mock(VolumeResponse.class); + Mockito.when(volumeImportService.importVolume(cmd)).thenReturn(response); + try { + cmd.execute(); + } catch (Exception ignored) { + } + + Assert.assertEquals(response, cmd.getResponseObject()); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/ListVolumesForImportCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/ListVolumesForImportCmdTest.java new file mode 100644 index 000000000000..959940d2a91d --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/ListVolumesForImportCmdTest.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.volume; + +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.VolumeForImportResponse; +import org.apache.cloudstack.storage.volume.VolumeImportUnmanageService; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +@RunWith(MockitoJUnitRunner.class) +public class ListVolumesForImportCmdTest { + + VolumeImportUnmanageService volumeImportService = Mockito.spy(VolumeImportUnmanageService.class); + + @Test + public void testListVolumesForImportCmd() { + Long storageId = 2L; + String filePath = "file path"; + + ListVolumesForImportCmd cmd = new ListVolumesForImportCmd(); + ReflectionTestUtils.setField(cmd, "storageId", storageId); + ReflectionTestUtils.setField(cmd, "path", filePath); + ReflectionTestUtils.setField(cmd,"volumeImportService", volumeImportService); + + Assert.assertEquals(storageId, cmd.getStorageId()); + Assert.assertEquals(filePath, cmd.getPath()); + + ListResponse response = Mockito.mock(ListResponse.class); + Mockito.when(volumeImportService.listVolumesForImport(cmd)).thenReturn(response); + try { + cmd.execute(); + } catch (Exception ignored) { + } + + Assert.assertEquals(response, cmd.getResponseObject()); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmdTest.java new file mode 100644 index 000000000000..8ab8c219339b --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmdTest.java @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.volume; + +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.storage.volume.VolumeImportUnmanageService; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +@RunWith(MockitoJUnitRunner.class) +public class UnmanageVolumeCmdTest { + + VolumeImportUnmanageService volumeImportService = Mockito.spy(VolumeImportUnmanageService.class); + + @Test + public void testUnmanageVolumeCmd() { + Long volumeId = 3L; + + UnmanageVolumeCmd cmd = new UnmanageVolumeCmd(); + ReflectionTestUtils.setField(cmd, "volumeId", volumeId); + ReflectionTestUtils.setField(cmd,"volumeImportService", volumeImportService); + + Assert.assertEquals(volumeId, cmd.getVolumeId()); + + Mockito.when(volumeImportService.unmanageVolume(volumeId)).thenReturn(true); + try { + cmd.execute(); + } catch (Exception ignored) { + } + + Object response = cmd.getResponseObject(); + Assert.assertTrue(response instanceof SuccessResponse); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/response/VolumeForImportResponseTest.java b/api/src/test/java/org/apache/cloudstack/api/response/VolumeForImportResponseTest.java new file mode 100644 index 000000000000..72bf0a8efde7 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/response/VolumeForImportResponseTest.java @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.Storage; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.HashMap; +import java.util.Map; + +@RunWith(MockitoJUnitRunner.class) +public final class VolumeForImportResponseTest { + + private static String path = "path"; + private static String name = "name"; + private static String fullPath = "fullPath"; + private static String format = "qcow2"; + private static long size = 10; + private static long virtualSize = 20; + private static String encryptFormat = "LUKS"; + private static Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.KVM; + private static String storagePoolId = "storage pool uuid"; + private static String storagePoolName = "storage pool 1"; + private static String storagePoolType = Storage.StoragePoolType.NetworkFilesystem.name(); + + @Test + public void testVolumeForImportResponse() { + final VolumeForImportResponse response = new VolumeForImportResponse(); + + response.setPath(path); + response.setName(name); + response.setFullPath(fullPath); + response.setFormat(format); + response.setSize(size); + response.setVirtualSize(virtualSize); + response.setQemuEncryptFormat(encryptFormat); + response.setStoragePoolType(storagePoolType); + response.setStoragePoolName(storagePoolName); + response.setStoragePoolId(storagePoolId); + Map details = new HashMap<>(); + details.put("key", "value"); + response.setDetails(details); + + Assert.assertEquals(path, response.getPath()); + Assert.assertEquals(name, response.getName()); + Assert.assertEquals(fullPath, response.getFullPath()); + Assert.assertEquals(format, response.getFormat()); + Assert.assertEquals(size, response.getSize()); + Assert.assertEquals(virtualSize, response.getVirtualSize()); + Assert.assertEquals(encryptFormat, response.getQemuEncryptFormat()); + Assert.assertEquals(storagePoolType, response.getStoragePoolType()); + Assert.assertEquals(storagePoolName, response.getStoragePoolName()); + Assert.assertEquals(storagePoolId, response.getStoragePoolId()); + Assert.assertEquals(details, response.getDetails()); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTOTest.java b/api/src/test/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTOTest.java index b36c6d9d8731..1660d0b472e6 100644 --- a/api/src/test/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTOTest.java +++ b/api/src/test/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTOTest.java @@ -20,6 +20,9 @@ import org.junit.Assert; import org.junit.Test; +import java.util.HashMap; +import java.util.Map; + public class VolumeOnStorageTOTest { private static String path = "path"; @@ -29,13 +32,51 @@ public class VolumeOnStorageTOTest { private static long size = 10; private static long virtualSize = 20; private static String encryptFormat = "LUKS"; + private static Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.KVM; + private static String BACKING_FILE = "backing file"; + private static String BACKING_FILE_FORMAT = "qcow2"; @Test public void testVolumeOnStorageTO() { - VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(Hypervisor.HypervisorType.KVM, path, name, fullPath, + VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(hypervisorType, path, name, fullPath, format, size, virtualSize); + + Assert.assertEquals(hypervisorType, volumeOnStorageTO.getHypervisorType()); + Assert.assertEquals(path, volumeOnStorageTO.getPath()); + Assert.assertEquals(name, volumeOnStorageTO.getName()); + Assert.assertEquals(fullPath, volumeOnStorageTO.getFullPath()); + Assert.assertEquals(format, volumeOnStorageTO.getFormat()); + Assert.assertEquals(size, volumeOnStorageTO.getSize()); + Assert.assertEquals(virtualSize, volumeOnStorageTO.getVirtualSize()); + } + + @Test + public void testVolumeOnStorageTO2() { + VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(hypervisorType, path, name, size); + Assert.assertEquals(hypervisorType, volumeOnStorageTO.getHypervisorType()); + Assert.assertEquals(path, volumeOnStorageTO.getPath()); + Assert.assertEquals(name, volumeOnStorageTO.getName()); + Assert.assertEquals(size, volumeOnStorageTO.getSize()); + } + + @Test + public void testVolumeOnStorageTO3() { + VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(); + volumeOnStorageTO.setHypervisorType(hypervisorType); + volumeOnStorageTO.setPath(path); + volumeOnStorageTO.setFullPath(fullPath); + volumeOnStorageTO.setName(name); + volumeOnStorageTO.setFormat(format); + volumeOnStorageTO.setSize(size); + volumeOnStorageTO.setVirtualSize(virtualSize); volumeOnStorageTO.setQemuEncryptFormat(encryptFormat); + Map details = new HashMap<>(); + details.put(VolumeOnStorageTO.Detail.BACKING_FILE, BACKING_FILE); + volumeOnStorageTO.setDetails(details); + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.BACKING_FILE_FORMAT, BACKING_FILE_FORMAT); + + Assert.assertEquals(hypervisorType, volumeOnStorageTO.getHypervisorType()); Assert.assertEquals(path, volumeOnStorageTO.getPath()); Assert.assertEquals(name, volumeOnStorageTO.getName()); Assert.assertEquals(fullPath, volumeOnStorageTO.getFullPath()); @@ -43,6 +84,10 @@ public void testVolumeOnStorageTO() { Assert.assertEquals(size, volumeOnStorageTO.getSize()); Assert.assertEquals(virtualSize, volumeOnStorageTO.getVirtualSize()); Assert.assertEquals(encryptFormat, volumeOnStorageTO.getQemuEncryptFormat()); - Assert.assertEquals(path, volumeOnStorageTO.getPath()); + + details = volumeOnStorageTO.getDetails(); + Assert.assertEquals(2, details.size()); + Assert.assertEquals(BACKING_FILE, details.get(VolumeOnStorageTO.Detail.BACKING_FILE)); + Assert.assertEquals(BACKING_FILE_FORMAT, details.get(VolumeOnStorageTO.Detail.BACKING_FILE_FORMAT)); } } diff --git a/core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageAnswerTest.java b/core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageAnswerTest.java index 92205559826d..de402fdf81c4 100644 --- a/core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageAnswerTest.java +++ b/core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageAnswerTest.java @@ -63,4 +63,11 @@ public void testGetVolumesOnStorageAnswer() { Assert.assertEquals(path, volume.getPath()); } + @Test + public void testGetVolumesOnStorageAnswer2() { + String details = "details"; + GetVolumesOnStorageAnswer answer = new GetVolumesOnStorageAnswer(command, false, details); + Assert.assertFalse(answer.getResult()); + Assert.assertEquals(details, answer.getDetails()); + } } diff --git a/core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageCommandTest.java b/core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageCommandTest.java index fba772b9d74e..e7883a2e02e9 100644 --- a/core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageCommandTest.java +++ b/core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageCommandTest.java @@ -34,5 +34,6 @@ public void testGetVolumesOnStorageCommand() { Assert.assertEquals(pool, command.getPool()); Assert.assertEquals(volumePath, command.getVolumePath()); + Assert.assertFalse(command.executeInSequence()); } } From 083d7155a10bc461087580bf959dc185a838d4d2 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 21 Mar 2024 15:01:12 +0100 Subject: [PATCH 11/44] Update 8088: code optimization --- .../VolumeImportUnmanageManagerImpl.java | 42 +++++++++---------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index affb14ef7e2d..b264e583f320 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -109,9 +109,9 @@ public class VolumeImportUnmanageManagerImpl implements VolumeImportUnmanageServ private VMTemplatePoolDao templatePoolDao; private static final String DEFAULT_DISK_OFFERING_NAME = "Default Custom Offering for Volume Import"; - private static final String DEFAULT_DISK_OFFERING_UNIQUE_NAME = "Custom-Volume-Import"; - private static final String DEFAULT_DISK_OFFERING_NAME_LOCAL = DEFAULT_DISK_OFFERING_NAME + " - Local Storage"; - private static final String DEFAULT_DISK_OFFERING_UNIQUE_NAME_LOCAL = DEFAULT_DISK_OFFERING_UNIQUE_NAME + "-Local"; + private static final String DEFAULT_DISK_OFFERING_UNIQUE_NAME = "Volume-Import"; + private static final String DISK_OFFERING_NAME_SUFFIX_LOCAL = " - Local Storage"; + private static final String DISK_OFFERING_UNIQUE_NAME_SUFFIX_LOCAL = "-Local"; private void logFailureAndThrowException(String msg) { logger.error(msg); @@ -351,28 +351,24 @@ private DiskOfferingVO getOrCreateDiskOffering(Account owner, Long diskOfferingI } private DiskOfferingVO getOrCreateDefaultDiskOfferingIdForVolumeImport(boolean isLocalStorage) { + final StringBuilder diskOfferingNameBuilder = new StringBuilder(DEFAULT_DISK_OFFERING_NAME); + final StringBuilder uniqueNameBuilder = new StringBuilder(DEFAULT_DISK_OFFERING_UNIQUE_NAME); if (isLocalStorage) { - DiskOfferingVO diskOffering = diskOfferingDao.findByUniqueName(DEFAULT_DISK_OFFERING_UNIQUE_NAME_LOCAL); - if (diskOffering != null) { - return diskOffering; - } - DiskOfferingVO newDiskOffering = new DiskOfferingVO(DEFAULT_DISK_OFFERING_NAME_LOCAL, DEFAULT_DISK_OFFERING_NAME_LOCAL, - Storage.ProvisioningType.THIN, 0, null, true, null, null, null); - newDiskOffering.setUseLocalStorage(true); - newDiskOffering.setUniqueName(DEFAULT_DISK_OFFERING_UNIQUE_NAME_LOCAL); - newDiskOffering = diskOfferingDao.persistDefaultDiskOffering(newDiskOffering); - return newDiskOffering; - } else { - DiskOfferingVO diskOffering = diskOfferingDao.findByUniqueName(DEFAULT_DISK_OFFERING_UNIQUE_NAME); - if (diskOffering != null) { - return diskOffering; - } - DiskOfferingVO newDiskOffering = new DiskOfferingVO(DEFAULT_DISK_OFFERING_NAME, DEFAULT_DISK_OFFERING_NAME, - Storage.ProvisioningType.THIN, 0, null, true, null, null, null); - newDiskOffering.setUniqueName(DEFAULT_DISK_OFFERING_UNIQUE_NAME); - newDiskOffering = diskOfferingDao.persistDefaultDiskOffering(newDiskOffering); - return newDiskOffering; + diskOfferingNameBuilder.append(DISK_OFFERING_NAME_SUFFIX_LOCAL); + uniqueNameBuilder.append(DISK_OFFERING_UNIQUE_NAME_SUFFIX_LOCAL); + } + final String diskOfferingName = diskOfferingNameBuilder.toString(); + final String uniqueName = uniqueNameBuilder.toString(); + DiskOfferingVO diskOffering = diskOfferingDao.findByUniqueName(uniqueName); + if (diskOffering != null) { + return diskOffering; } + DiskOfferingVO newDiskOffering = new DiskOfferingVO(diskOfferingName, diskOfferingName, + Storage.ProvisioningType.THIN, 0, null, true, null, null, null); + newDiskOffering.setUseLocalStorage(isLocalStorage); + newDiskOffering.setUniqueName(uniqueName); + newDiskOffering = diskOfferingDao.persistDefaultDiskOffering(newDiskOffering); + return newDiskOffering; } private VolumeVO createRecordsForVolumeImport(VolumeOnStorageTO volume, DiskOfferingVO diskOffering, From a7d1796fa71810029fe431483a5c4c14064fa03c Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 22 Mar 2024 08:49:49 +0100 Subject: [PATCH 12/44] Update 8808: add unit tests part2 --- .../command/admin/volume/ImportVolumeCmd.java | 8 ++- .../admin/volume/ListVolumesForImportCmd.java | 11 ---- .../admin/volume/UnmanageVolumeCmd.java | 2 +- .../admin/volume/ImportVolumeCmdTest.java | 11 ++++ .../admin/volume/UnmanageVolumeCmdTest.java | 16 ++++++ .../response/VolumeForImportResponseTest.java | 3 + .../orchestration/VolumeOrchestratorTest.java | 55 ++++++++++++++++++- 7 files changed, 90 insertions(+), 16 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java index 944e1cc6b239..e6874da5638f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java @@ -23,7 +23,6 @@ import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; -import com.cloud.user.Account; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; @@ -148,7 +147,10 @@ public void execute() throws ResourceUnavailableException, InsufficientCapacityE @Override public long getEntityOwnerId() { - Account caller = CallContext.current().getCallingAccount(); - return caller.getAccountId(); + Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + if (accountId == null) { + return CallContext.current().getCallingAccount().getId(); + } + return accountId; } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ListVolumesForImportCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ListVolumesForImportCmd.java index ea1111f3efe2..dbe3d37e4069 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ListVolumesForImportCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ListVolumesForImportCmd.java @@ -22,7 +22,6 @@ import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; -import com.cloud.user.Account; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; @@ -34,7 +33,6 @@ import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.VolumeForImportResponse; -import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.storage.volume.VolumeImportUnmanageService; import org.apache.cloudstack.storage.volume.VolumeOnStorageTO; @@ -92,13 +90,4 @@ public void execute() throws ResourceUnavailableException, InsufficientCapacityE response.setResponseName(getCommandName()); setResponseObject(response); } - - @Override - public long getEntityOwnerId() { - Account account = CallContext.current().getCallingAccount(); - if (account != null) { - return account.getId(); - } - return Account.ACCOUNT_ID_SYSTEM; - } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmd.java index 773f587f43b0..dcc8b2af8d74 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmd.java @@ -81,7 +81,7 @@ public String getEventType() { @Override public String getEventDescription() { - return String.format("unmanaging Volume with ID %s", volumeId); + return String.format("Unmanaging Volume with ID %s", volumeId); } ///////////////////////////////////////////////////// diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmdTest.java index 36f2f82d0e13..9de7ee88f783 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmdTest.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.api.command.admin.volume; +import com.cloud.event.EventTypes; +import com.cloud.user.AccountService; import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.cloudstack.storage.volume.VolumeImportUnmanageService; import org.junit.Assert; @@ -31,6 +33,7 @@ public class ImportVolumeCmdTest { VolumeImportUnmanageService volumeImportService = Mockito.spy(VolumeImportUnmanageService.class); + AccountService accountService = Mockito.spy(AccountService.class); @Test public void testImportVolumeCmd() { @@ -40,6 +43,9 @@ public void testImportVolumeCmd() { String accountName = "account"; Long domainId = 4L; Long projectId = 5L; + long accountId = 6L; + + Mockito.when(accountService.finalyzeAccountId(accountName, domainId, projectId, true)).thenReturn(accountId); ImportVolumeCmd cmd = new ImportVolumeCmd(); ReflectionTestUtils.setField(cmd, "path", path); @@ -49,6 +55,7 @@ public void testImportVolumeCmd() { ReflectionTestUtils.setField(cmd, "domainId", domainId); ReflectionTestUtils.setField(cmd, "projectId", projectId); ReflectionTestUtils.setField(cmd,"volumeImportService", volumeImportService); + ReflectionTestUtils.setField(cmd, "_accountService", accountService); Assert.assertEquals(path, cmd.getPath()); Assert.assertEquals(storageId, cmd.getStorageId()); @@ -57,6 +64,10 @@ public void testImportVolumeCmd() { Assert.assertEquals(domainId, cmd.getDomainId()); Assert.assertEquals(projectId, cmd.getProjectId()); + Assert.assertEquals(EventTypes.EVENT_VOLUME_IMPORT, cmd.getEventType()); + Assert.assertEquals("Importing unmanaged Volume with path: " + path, cmd.getEventDescription()); + Assert.assertEquals(accountId, cmd.getEntityOwnerId()); + VolumeResponse response = Mockito.mock(VolumeResponse.class); Mockito.when(volumeImportService.importVolume(cmd)).thenReturn(response); try { diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmdTest.java index 8ab8c219339b..ba7e351a8a8e 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmdTest.java @@ -17,6 +17,10 @@ package org.apache.cloudstack.api.command.admin.volume; +import com.cloud.event.EventTypes; +import com.cloud.storage.Volume; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ResponseGenerator; import org.apache.cloudstack.api.response.SuccessResponse; import org.apache.cloudstack.storage.volume.VolumeImportUnmanageService; import org.junit.Assert; @@ -31,16 +35,28 @@ public class UnmanageVolumeCmdTest { VolumeImportUnmanageService volumeImportService = Mockito.spy(VolumeImportUnmanageService.class); + ResponseGenerator responseGenerator = Mockito.spy(ResponseGenerator.class); @Test public void testUnmanageVolumeCmd() { + long accountId = 2L; Long volumeId = 3L; + Volume volume = Mockito.mock(Volume.class); + + Mockito.when(responseGenerator.findVolumeById(volumeId)).thenReturn(volume); + Mockito.when(volume.getAccountId()).thenReturn(accountId); UnmanageVolumeCmd cmd = new UnmanageVolumeCmd(); ReflectionTestUtils.setField(cmd, "volumeId", volumeId); ReflectionTestUtils.setField(cmd,"volumeImportService", volumeImportService); + ReflectionTestUtils.setField(cmd,"_responseGenerator", responseGenerator); Assert.assertEquals(volumeId, cmd.getVolumeId()); + Assert.assertEquals(accountId, cmd.getEntityOwnerId()); + Assert.assertEquals(volumeId, cmd.getApiResourceId()); + Assert.assertEquals(ApiCommandResourceType.Volume, cmd.getApiResourceType()); + Assert.assertEquals(EventTypes.EVENT_VOLUME_UNMANAGE, cmd.getEventType()); + Assert.assertEquals("Unmanaging Volume with ID " + volumeId, cmd.getEventDescription()); Mockito.when(volumeImportService.unmanageVolume(volumeId)).thenReturn(true); try { diff --git a/api/src/test/java/org/apache/cloudstack/api/response/VolumeForImportResponseTest.java b/api/src/test/java/org/apache/cloudstack/api/response/VolumeForImportResponseTest.java index 72bf0a8efde7..7d0538836bff 100644 --- a/api/src/test/java/org/apache/cloudstack/api/response/VolumeForImportResponseTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/response/VolumeForImportResponseTest.java @@ -40,6 +40,7 @@ public final class VolumeForImportResponseTest { private static String storagePoolId = "storage pool uuid"; private static String storagePoolName = "storage pool 1"; private static String storagePoolType = Storage.StoragePoolType.NetworkFilesystem.name(); + private static String chainInfo = "chain info"; @Test public void testVolumeForImportResponse() { @@ -55,6 +56,7 @@ public void testVolumeForImportResponse() { response.setStoragePoolType(storagePoolType); response.setStoragePoolName(storagePoolName); response.setStoragePoolId(storagePoolId); + response.setChainInfo(chainInfo); Map details = new HashMap<>(); details.put("key", "value"); response.setDetails(details); @@ -69,6 +71,7 @@ public void testVolumeForImportResponse() { Assert.assertEquals(storagePoolType, response.getStoragePoolType()); Assert.assertEquals(storagePoolName, response.getStoragePoolName()); Assert.assertEquals(storagePoolId, response.getStoragePoolId()); + Assert.assertEquals(chainInfo, response.getChainInfo()); Assert.assertEquals(details, response.getDetails()); } } diff --git a/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java b/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java index e817f61a0980..a37845c86c2f 100644 --- a/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java +++ b/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java @@ -17,7 +17,14 @@ package org.apache.cloudstack.engine.orchestration; import java.util.ArrayList; - +import java.util.Date; + +import com.cloud.hypervisor.Hypervisor; +import com.cloud.offering.DiskOffering; +import com.cloud.storage.Storage; +import com.cloud.storage.Volume; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.Account; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; @@ -32,6 +39,7 @@ import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockedConstruction; import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; @@ -42,6 +50,7 @@ import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.storage.VolumeVO; +import com.cloud.storage.Volume.Type; import com.cloud.user.ResourceLimitService; import com.cloud.utils.exception.CloudRuntimeException; @@ -54,6 +63,8 @@ public class VolumeOrchestratorTest { protected VolumeService volumeService; @Mock protected VolumeDataFactory volumeDataFactory; + @Mock + protected VolumeDao volumeDao; @Spy @InjectMocks @@ -155,4 +166,46 @@ public void testGrantVolumeAccessToHostIfNeededDriverNeedsButException() { volumeOrchestrator.grantVolumeAccessToHostIfNeeded(store, 1L, Mockito.mock(HostVO.class), ""); } + + @Test + public void testImportVolume() { + Type volumeType = Type.DATADISK; + String name = "new-volume"; + Long sizeInBytes = 1000000L; + Long zoneId = 1L; + Long domainId = 2L; + Long accountId = 3L; + Long diskOfferingId = 4L; + DiskOffering diskOffering = Mockito.mock(DiskOffering.class); + Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.KVM; + Account owner = Mockito.mock(Account.class); + Mockito.when(owner.getDomainId()).thenReturn(domainId); + Mockito.when(owner.getId()).thenReturn(accountId); + Mockito.when(diskOffering.getId()).thenReturn(diskOfferingId); + Long deviceId = 2L; + Long poolId = 3L; + String path = "volume path"; + String chainInfo = "chain info"; + + MockedConstruction volumeVOMockedConstructionConstruction = Mockito.mockConstruction(VolumeVO.class, (mock, context) -> { + }); + + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + Mockito.when(volumeDao.persist(Mockito.any(VolumeVO.class))).thenReturn(volumeVO); + + volumeOrchestrator.importVolume(volumeType, name, diskOffering, sizeInBytes, null, null, + zoneId, hypervisorType, null, null, owner, + deviceId, poolId, path, chainInfo); + + VolumeVO volume = volumeVOMockedConstructionConstruction.constructed().get(0); + Mockito.verify(volume, Mockito.never()).setInstanceId(Mockito.anyLong()); + Mockito.verify(volume, Mockito.never()).setAttached(Mockito.any(Date.class)); + Mockito.verify(volume, Mockito.times(1)).setDeviceId(deviceId); + Mockito.verify(volume, Mockito.never()).setDisplayVolume(Mockito.any(Boolean.class)); + Mockito.verify(volume, Mockito.times(1)).setFormat(Storage.ImageFormat.QCOW2); + Mockito.verify(volume, Mockito.times(1)).setPoolId(poolId); + Mockito.verify(volume, Mockito.times(1)).setPath(path); + Mockito.verify(volume, Mockito.times(1)).setChainInfo(chainInfo); + Mockito.verify(volume, Mockito.times(1)).setState(Volume.State.Ready); + } } From 55196a4d851a6702466cdbc81527ce6679a0181d Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 22 Mar 2024 14:56:27 +0100 Subject: [PATCH 13/44] Update 8808: add unit tests part3 --- ...GetVolumesOnStorageCommandWrapperTest.java | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapperTest.java diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapperTest.java new file mode 100644 index 000000000000..0fbbec5a76fe --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapperTest.java @@ -0,0 +1,155 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.GetVolumesOnStorageAnswer; +import com.cloud.agent.api.GetVolumesOnStorageCommand; +import com.cloud.agent.api.to.StorageFilerTO; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.storage.Storage; +import org.apache.cloudstack.storage.volume.VolumeOnStorageTO; +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.apache.cloudstack.utils.qemu.QemuObject; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.MockedConstruction; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.mockito.Mockito.times; + +@RunWith(MockitoJUnitRunner.class) +public class LibvirtGetVolumesOnStorageCommandWrapperTest { + + @Mock + LibvirtComputingResource libvirtComputingResource; + + @Mock + KVMStoragePoolManager storagePoolMgr; + @Mock + KVMStoragePool storagePool; + + @Mock + StorageFilerTO pool; + @Mock + Map qemuImgInfo; + + + private final Storage.StoragePoolType poolType = Storage.StoragePoolType.NetworkFilesystem; + private final String poolUuid = "pool-uuid"; + private final String volumePath = "volume-path"; + + private final String backingFilePath = "backing file path"; + private final String backingFileFormat = "QCOW2"; + private final String clusterSize = "4096"; + private final String fileFormat = "QCOW2"; + private final String encrypted = "yes"; + + @Spy + LibvirtGetVolumesOnStorageCommandWrapper libvirtGetVolumesOnStorageCommandWrapper = new LibvirtGetVolumesOnStorageCommandWrapper(); + + MockedConstruction qemuImg; + MockedConstruction volumeOnStorageTOMock; + + @Before + public void setUp() { + Mockito.when(pool.getUuid()).thenReturn(poolUuid); + Mockito.when(pool.getType()).thenReturn(poolType); + Mockito.when(libvirtComputingResource.getStoragePoolMgr()).thenReturn(storagePoolMgr); + Mockito.when(storagePoolMgr.getStoragePool(poolType, poolUuid, true)).thenReturn(storagePool); + + qemuImg = Mockito.mockConstruction(QemuImg.class, (mock, context) -> { + Mockito.when(mock.info(Mockito.any(QemuImgFile.class), Mockito.eq(true))).thenReturn(qemuImgInfo); + }); + volumeOnStorageTOMock = Mockito.mockConstruction(VolumeOnStorageTO.class); + } + + @After + public void tearDown() { + qemuImg.close(); + volumeOnStorageTOMock.close(); + } + + @Test + public void testLibvirtGetVolumesOnStorageCommandWrapperForAllVolumes() { + GetVolumesOnStorageCommand command = new GetVolumesOnStorageCommand(pool, null); + List physicalDisks = new ArrayList<>(); + int numberDisks = 3; + for (int i = 0; i < numberDisks; i++) { + KVMPhysicalDisk disk = Mockito.mock(KVMPhysicalDisk.class); + Mockito.when(disk.getFormat()).thenReturn(QemuImg.PhysicalDiskFormat.QCOW2); + Mockito.when(disk.getQemuEncryptFormat()).thenReturn(QemuObject.EncryptFormat.LUKS); + physicalDisks.add(disk); + } + Mockito.when(storagePool.listPhysicalDisks()).thenReturn(physicalDisks); + + Answer answer = libvirtGetVolumesOnStorageCommandWrapper.execute(command, libvirtComputingResource); + Assert.assertTrue(answer instanceof GetVolumesOnStorageAnswer); + Assert.assertTrue(answer.getResult()); + List volumes = ((GetVolumesOnStorageAnswer) answer).getVolumes(); + Assert.assertEquals(numberDisks, volumes.size()); + volumeOnStorageTOMock.constructed().forEach(s -> Mockito.verify(s, times(1)).setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS.toString())); + } + + @Test + public void testLibvirtGetVolumesOnStorageCommandWrapperForVolume() { + KVMPhysicalDisk disk = Mockito.mock(KVMPhysicalDisk.class); + Mockito.when(disk.getFormat()).thenReturn(QemuImg.PhysicalDiskFormat.QCOW2); + Mockito.when(disk.getQemuEncryptFormat()).thenReturn(QemuObject.EncryptFormat.LUKS); + Mockito.when(storagePool.getPhysicalDisk(volumePath)).thenReturn(disk); + Mockito.when(storagePool.getType()).thenReturn(poolType); + + Mockito.when(qemuImgInfo.get(QemuImg.BACKING_FILE)).thenReturn(backingFilePath); + Mockito.when(qemuImgInfo.get(QemuImg.BACKING_FILE_FORMAT)).thenReturn(backingFileFormat); + Mockito.when(qemuImgInfo.get(QemuImg.CLUSTER_SIZE)).thenReturn(clusterSize); + Mockito.when(qemuImgInfo.get(QemuImg.FILE_FORMAT)).thenReturn(fileFormat); + Mockito.when(qemuImgInfo.get(QemuImg.ENCRYPTED)).thenReturn(encrypted); + + GetVolumesOnStorageCommand command = new GetVolumesOnStorageCommand(pool, volumePath); + Answer answer = libvirtGetVolumesOnStorageCommandWrapper.execute(command, libvirtComputingResource); + Assert.assertTrue(answer instanceof GetVolumesOnStorageAnswer); + Assert.assertTrue(answer.getResult()); + List volumes = ((GetVolumesOnStorageAnswer) answer).getVolumes(); + Assert.assertEquals(1, volumes.size()); + + VolumeOnStorageTO volumeOnStorageTO = volumeOnStorageTOMock.constructed().get(0); + Mockito.verify(volumeOnStorageTO).setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS.toString()); + Mockito.verify(volumeOnStorageTO).addDetail(VolumeOnStorageTO.Detail.BACKING_FILE, backingFilePath); + Mockito.verify(volumeOnStorageTO).addDetail(VolumeOnStorageTO.Detail.BACKING_FILE_FORMAT, backingFileFormat); + Mockito.verify(volumeOnStorageTO).addDetail(VolumeOnStorageTO.Detail.CLUSTER_SIZE, clusterSize); + Mockito.verify(volumeOnStorageTO).addDetail(VolumeOnStorageTO.Detail.FILE_FORMAT, fileFormat); + Mockito.verify(volumeOnStorageTO).addDetail(VolumeOnStorageTO.Detail.IS_ENCRYPTED, "true"); + Mockito.verify(volumeOnStorageTO).addDetail(VolumeOnStorageTO.Detail.IS_LOCKED, "false"); + } +} From b14fdf2153b9cf280336c9025919d02ec1cccb8f Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 25 Mar 2024 09:18:02 +0100 Subject: [PATCH 14/44] Update 8088: remove unused constructor VolumeOnStorageTO --- .../cloudstack/storage/volume/VolumeOnStorageTO.java | 7 ------- .../cloudstack/storage/volume/VolumeOnStorageTOTest.java | 9 --------- 2 files changed, 16 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTO.java b/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTO.java index b44767bd8d82..1a8fd6ee273a 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTO.java +++ b/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTO.java @@ -52,13 +52,6 @@ public VolumeOnStorageTO(Hypervisor.HypervisorType hypervisorType, String path, this.virtualSize = virtualSize; } - public VolumeOnStorageTO(Hypervisor.HypervisorType hypervisorType, String path, String name, long size) { - this.hypervisorType = hypervisorType; - this.path = path; - this.name = name; - this.size = size; - } - public Hypervisor.HypervisorType getHypervisorType() { return hypervisorType; } diff --git a/api/src/test/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTOTest.java b/api/src/test/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTOTest.java index 1660d0b472e6..59de3b8ac4ee 100644 --- a/api/src/test/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTOTest.java +++ b/api/src/test/java/org/apache/cloudstack/storage/volume/VolumeOnStorageTOTest.java @@ -50,15 +50,6 @@ public void testVolumeOnStorageTO() { Assert.assertEquals(virtualSize, volumeOnStorageTO.getVirtualSize()); } - @Test - public void testVolumeOnStorageTO2() { - VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(hypervisorType, path, name, size); - Assert.assertEquals(hypervisorType, volumeOnStorageTO.getHypervisorType()); - Assert.assertEquals(path, volumeOnStorageTO.getPath()); - Assert.assertEquals(name, volumeOnStorageTO.getName()); - Assert.assertEquals(size, volumeOnStorageTO.getSize()); - } - @Test public void testVolumeOnStorageTO3() { VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(); From 6d921e3d538c192939f42dee6479333c525508ad Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 25 Mar 2024 09:18:10 +0100 Subject: [PATCH 15/44] Update 8088: import volume with name --- .../api/command/admin/volume/ImportVolumeCmd.java | 9 +++++++++ .../command/admin/volume/ImportVolumeCmdTest.java | 3 +++ .../volume/VolumeImportUnmanageManagerImpl.java | 14 +++++++++----- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java index e6874da5638f..662de4cfefe4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java @@ -66,6 +66,11 @@ public class ImportVolumeCmd extends BaseAsyncCmd { description = "the path of the volume") private String path; + @Parameter(name = ApiConstants.NAME, + type = BaseCmd.CommandType.STRING, + description = "the name of the volume. If not set, it will be set to the path of the volume.") + private String name; + @Parameter(name = ApiConstants.STORAGE_ID, type = BaseCmd.CommandType.UUID, required = true, @@ -104,6 +109,10 @@ public String getPath() { return path; } + public String getName() { + return name; + } + public Long getStorageId() { return storageId; } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmdTest.java index 9de7ee88f783..a7c41b9271b1 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmdTest.java @@ -37,6 +37,7 @@ public class ImportVolumeCmdTest { @Test public void testImportVolumeCmd() { + String name = "volume name"; String path = "file path"; Long storageId = 2L; Long diskOfferingId = 3L; @@ -49,6 +50,7 @@ public void testImportVolumeCmd() { ImportVolumeCmd cmd = new ImportVolumeCmd(); ReflectionTestUtils.setField(cmd, "path", path); + ReflectionTestUtils.setField(cmd, "name", name); ReflectionTestUtils.setField(cmd, "storageId", storageId); ReflectionTestUtils.setField(cmd, "diskOfferingId", diskOfferingId); ReflectionTestUtils.setField(cmd, "accountName", accountName); @@ -58,6 +60,7 @@ public void testImportVolumeCmd() { ReflectionTestUtils.setField(cmd, "_accountService", accountService); Assert.assertEquals(path, cmd.getPath()); + Assert.assertEquals(name, cmd.getName()); Assert.assertEquals(storageId, cmd.getStorageId()); Assert.assertEquals(diskOfferingId, cmd.getDiskOfferingId()); Assert.assertEquals(accountName, cmd.getAccountName()); diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index b264e583f320..0aa028a759ab 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -165,6 +165,9 @@ public VolumeResponse importVolume(ImportVolumeCmd cmd) { // 3. check if the volume already exists in cloudstack by path String volumePath = cmd.getPath(); + if (StringUtils.isBlank(volumePath)) { + logFailureAndThrowException("Volume path is null or blank: " + volumePath); + } if (checkIfVolumeManaged(pool, volumePath)){ logFailureAndThrowException("Volume is already managed by CloudStack: " + volumePath); } @@ -191,7 +194,8 @@ public VolumeResponse importVolume(ImportVolumeCmd cmd) { DiskOfferingVO diskOffering = getOrCreateDiskOffering(owner, cmd.getDiskOfferingId(), pool.getDataCenterId(), pool.isLocal()); // 7. create records - VolumeVO volumeVO = createRecordsForVolumeImport(volume, diskOffering, owner, pool); + String volumeName = StringUtils.isNotBlank(cmd.getName()) ? cmd.getName().trim() : volumePath; + VolumeVO volumeVO = importVolumeInternal(volume, diskOffering, owner, pool, volumeName); // 8. Update resource count updateResourceLimitForVolumeImport(volumeVO); @@ -371,10 +375,10 @@ private DiskOfferingVO getOrCreateDefaultDiskOfferingIdForVolumeImport(boolean i return newDiskOffering; } - private VolumeVO createRecordsForVolumeImport(VolumeOnStorageTO volume, DiskOfferingVO diskOffering, - Account owner, StoragePoolVO pool) { - DiskProfile diskProfile = volumeManager.importVolume(Volume.Type.DATADISK, volume.getName(), diskOffering, - volume.getVirtualSize(), null, null, pool.getDataCenterId(), pool.getHypervisor(), null, null, + private VolumeVO importVolumeInternal(VolumeOnStorageTO volume, DiskOfferingVO diskOffering, + Account owner, StoragePoolVO pool, String volumeName) { + DiskProfile diskProfile = volumeManager.importVolume(Volume.Type.DATADISK, volumeName, diskOffering, + volume.getVirtualSize(), null, null, pool.getDataCenterId(), volume.getHypervisorType(), null, null, owner, null, pool.getId(), volume.getPath(), null); return volumeDao.findById(diskProfile.getVolumeId()); } From 6207a97eb22486ba6c1c397c8cb629fc7a15c7e2 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 25 Mar 2024 09:18:18 +0100 Subject: [PATCH 16/44] Update 8088: sort the volumes by name --- .../wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java | 5 ++++- .../LibvirtGetVolumesOnStorageCommandWrapperTest.java | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java index 4971eb333bbb..314a94b2f371 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java @@ -41,6 +41,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -103,7 +104,9 @@ public Answer execute(final GetVolumesOnStorageCommand command, final LibvirtCom volumes.add(volumeOnStorageTO); } } else { - for (KVMPhysicalDisk disk: storagePool.listPhysicalDisks()) { + List disks = storagePool.listPhysicalDisks(); + disks.sort(Comparator.comparing(KVMPhysicalDisk::getName)); + for (KVMPhysicalDisk disk: disks) { if (!isDiskFormatSupported(disk)) { continue; } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapperTest.java index 0fbbec5a76fe..e4bce1988e48 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapperTest.java @@ -108,6 +108,7 @@ public void testLibvirtGetVolumesOnStorageCommandWrapperForAllVolumes() { int numberDisks = 3; for (int i = 0; i < numberDisks; i++) { KVMPhysicalDisk disk = Mockito.mock(KVMPhysicalDisk.class); + Mockito.when(disk.getName()).thenReturn("name-" + (numberDisks - i)); Mockito.when(disk.getFormat()).thenReturn(QemuImg.PhysicalDiskFormat.QCOW2); Mockito.when(disk.getQemuEncryptFormat()).thenReturn(QemuObject.EncryptFormat.LUKS); physicalDisks.add(disk); From 70a83d13ee949fbd948c8c1cfdcff0c843e59052 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 25 Mar 2024 09:18:30 +0100 Subject: [PATCH 17/44] Update 8088: add unit tests VolumeImportUnmanageManagerImplTest --- .../api/GetVolumesOnStorageAnswerTest.java | 2 +- .../VolumeImportUnmanageManagerImpl.java | 28 +- .../VolumeImportUnmanageManagerImplTest.java | 486 ++++++++++++++++++ 3 files changed, 501 insertions(+), 15 deletions(-) create mode 100644 server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java diff --git a/core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageAnswerTest.java b/core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageAnswerTest.java index de402fdf81c4..26948f2b8232 100644 --- a/core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageAnswerTest.java +++ b/core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageAnswerTest.java @@ -47,7 +47,7 @@ public void testGetVolumesOnStorageAnswer() { volumesOnStorageTO.add(volumeOnStorageTO); GetVolumesOnStorageAnswer answer = new GetVolumesOnStorageAnswer(command, volumesOnStorageTO); - List volumes = answer.getVolumes(); + List volumes = answer.getVolumes(); Assert.assertEquals(1, volumes.size()); VolumeOnStorageTO volume = volumes.get(0); diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index 0aa028a759ab..37d73e0596a5 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -108,10 +108,10 @@ public class VolumeImportUnmanageManagerImpl implements VolumeImportUnmanageServ @Inject private VMTemplatePoolDao templatePoolDao; - private static final String DEFAULT_DISK_OFFERING_NAME = "Default Custom Offering for Volume Import"; - private static final String DEFAULT_DISK_OFFERING_UNIQUE_NAME = "Volume-Import"; - private static final String DISK_OFFERING_NAME_SUFFIX_LOCAL = " - Local Storage"; - private static final String DISK_OFFERING_UNIQUE_NAME_SUFFIX_LOCAL = "-Local"; + static final String DEFAULT_DISK_OFFERING_NAME = "Default Custom Offering for Volume Import"; + static final String DEFAULT_DISK_OFFERING_UNIQUE_NAME = "Volume-Import"; + static final String DISK_OFFERING_NAME_SUFFIX_LOCAL = " - Local Storage"; + static final String DISK_OFFERING_UNIQUE_NAME_SUFFIX_LOCAL = "-Local"; private void logFailureAndThrowException(String msg) { logger.error(msg); @@ -206,7 +206,7 @@ public VolumeResponse importVolume(ImportVolumeCmd cmd) { return responseGenerator.createVolumeResponse(ResponseObject.ResponseView.Full, volumeVO); } - private List listVolumesForImportInternal(Long poolId, String volumePath) { + protected List listVolumesForImportInternal(Long poolId, String volumePath) { StoragePoolVO pool = checkIfPoolAvailable(poolId); Pair hostAndLocalPath = findHostAndLocalPathForVolumeImport(pool); @@ -247,7 +247,7 @@ public boolean unmanageVolume(long volumeId) { return true; } - private StoragePoolVO checkIfPoolAvailable(Long poolId) { + protected StoragePoolVO checkIfPoolAvailable(Long poolId) { StoragePoolVO pool = primaryDataStoreDao.findById(poolId); if (pool == null) { logFailureAndThrowException("Storage pool does not exist: ID = " + poolId); @@ -258,7 +258,7 @@ private StoragePoolVO checkIfPoolAvailable(Long poolId) { return pool; } - private Pair findHostAndLocalPathForVolumeImport(StoragePoolVO pool) { + protected Pair findHostAndLocalPathForVolumeImport(StoragePoolVO pool) { List hosts = new ArrayList<>(); if (ScopeType.HOST.equals(pool.getScope())) { List storagePoolHostVOs = storagePoolHostDao.listByPoolId(pool.getId()); @@ -285,7 +285,7 @@ private Pair findHostAndLocalPathForVolumeImport(StoragePoolVO p return null; } - private VolumeForImportResponse createVolumeForImportResponse(VolumeOnStorageTO volume, StoragePoolVO pool) { + protected VolumeForImportResponse createVolumeForImportResponse(VolumeOnStorageTO volume, StoragePoolVO pool) { VolumeForImportResponse response = new VolumeForImportResponse(); response.setPath(volume.getPath()); response.setName(volume.getName()); @@ -310,7 +310,7 @@ private boolean checkIfVolumeForTemplate(StoragePoolVO pool, String volumePath) return templatePoolDao.findByPoolPath(pool.getId(), volumePath) != null; } - private void checkIfVolumeIsLocked(VolumeOnStorageTO volume) { + protected void checkIfVolumeIsLocked(VolumeOnStorageTO volume) { Map volumeDetails = volume.getDetails(); if (volumeDetails != null && volumeDetails.containsKey(VolumeOnStorageTO.Detail.IS_LOCKED)) { String isLocked = volumeDetails.get(VolumeOnStorageTO.Detail.IS_LOCKED); @@ -320,7 +320,7 @@ private void checkIfVolumeIsLocked(VolumeOnStorageTO volume) { } } - private void checkIfVolumeIsEncrypted(VolumeOnStorageTO volume) { + protected void checkIfVolumeIsEncrypted(VolumeOnStorageTO volume) { Map volumeDetails = volume.getDetails(); if (volumeDetails != null && volumeDetails.containsKey(VolumeOnStorageTO.Detail.IS_ENCRYPTED)) { String isEncrypted = volumeDetails.get(VolumeOnStorageTO.Detail.IS_ENCRYPTED); @@ -330,7 +330,7 @@ private void checkIfVolumeIsEncrypted(VolumeOnStorageTO volume) { } } - private DiskOfferingVO getOrCreateDiskOffering(Account owner, Long diskOfferingId, Long zoneId, boolean isLocal) { + protected DiskOfferingVO getOrCreateDiskOffering(Account owner, Long diskOfferingId, Long zoneId, boolean isLocal) { if (diskOfferingId != null) { // check if disk offering exists and active DiskOfferingVO diskOfferingVO = diskOfferingDao.findById(diskOfferingId); @@ -383,10 +383,10 @@ private VolumeVO importVolumeInternal(VolumeOnStorageTO volume, DiskOfferingVO d return volumeDao.findById(diskProfile.getVolumeId()); } - private void checkResourceLimitForImportVolume(Account owner, VolumeOnStorageTO volume) { - Long volumeSize = volume.getSize(); + protected void checkResourceLimitForImportVolume(Account owner, VolumeOnStorageTO volume) { + Long volumeSize = volume.getVirtualSize(); try { - resourceLimitService.checkResourceLimit(owner, Resource.ResourceType.volume, 1); + resourceLimitService.checkResourceLimit(owner, Resource.ResourceType.volume); resourceLimitService.checkResourceLimit(owner, Resource.ResourceType.primary_storage, volumeSize); } catch (ResourceAllocationException e) { logger.error(String.format("VM resource allocation error for account: %s", owner.getUuid()), e); diff --git a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java new file mode 100644 index 000000000000..8d6975c3b38c --- /dev/null +++ b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java @@ -0,0 +1,486 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.volume; + +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.GetVolumesOnStorageAnswer; +import com.cloud.agent.api.GetVolumesOnStorageCommand; +import com.cloud.configuration.ConfigurationManager; +import com.cloud.configuration.Resource; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.event.UsageEventUtils; +import com.cloud.exception.PermissionDeniedException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.offering.DiskOffering; +import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; +import com.cloud.storage.StoragePoolHostVO; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.storage.dao.StoragePoolHostDao; +import com.cloud.storage.dao.VMTemplatePoolDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; +import com.cloud.user.ResourceLimitService; +import com.cloud.user.User; +import com.cloud.user.UserVO; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.DiskProfile; +import org.apache.cloudstack.api.ResponseGenerator; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.command.admin.volume.ImportVolumeCmd; +import org.apache.cloudstack.api.command.admin.volume.ListVolumesForImportCmd; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.VolumeForImportResponse; +import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.UUID; + +import static org.apache.cloudstack.storage.volume.VolumeImportUnmanageManagerImpl.DEFAULT_DISK_OFFERING_UNIQUE_NAME; +import static org.apache.cloudstack.storage.volume.VolumeImportUnmanageManagerImpl.DISK_OFFERING_UNIQUE_NAME_SUFFIX_LOCAL; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class VolumeImportUnmanageManagerImplTest { + + @Spy + @InjectMocks + VolumeImportUnmanageManagerImpl volumeImportUnmanageManager; + + @Mock + private AccountManager accountMgr; + @Mock + private AgentManager agentManager; + @Mock + private HostDao hostDao; + @Mock + private DiskOfferingDao diskOfferingDao; + @Mock + private ResourceLimitService resourceLimitService; + @Mock + private ResponseGenerator responseGenerator; + @Mock + private VolumeDao volumeDao; + @Mock + private PrimaryDataStoreDao primaryDataStoreDao; + @Mock + private StoragePoolHostDao storagePoolHostDao; + @Mock + private ConfigurationManager configMgr; + @Mock + private DataCenterDao dcDao; + @Mock + private VolumeOrchestrationService volumeManager; + @Mock + private VMTemplatePoolDao templatePoolDao; + + @Mock + StoragePoolVO storagePoolVO; + @Mock + VolumeVO volumeVO; + @Mock + DiskProfile diskProfile; + @Mock + HostVO hostVO; + @Mock + StoragePoolHostVO storagePoolHostVO; + @Mock + DiskOfferingVO diskOfferingVO; + @Mock + DataCenterVO dataCenterVO; + + final static long accountId = 10L; + final static long zoneId = 11L; + final static long clusterId = 11L; + final static long hostId = 13L; + final static long poolId = 100L; + final static boolean isLocal = true; + final static long volumeId = 101L; + final static String volumeName = "import volume"; + final static long diskOfferingId = 120L; + final static String localPath = "/mnt/localPath"; + + private static String path = "path"; + private static String name = "name"; + private static String fullPath = "fullPath"; + private static String format = "qcow2"; + private static long size = 100000L; + private static long virtualSize = 20000000L; + private static String encryptFormat = "LUKS"; + private static Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.KVM; + private static String BACKING_FILE = "backing file"; + private static String BACKING_FILE_FORMAT = "qcow2"; + private static String storagePoolUuid = "pool-uuid"; + private static String storagePoolName = "pool-name"; + private static Storage.StoragePoolType storagePoolType = Storage.StoragePoolType.NetworkFilesystem; + + AccountVO account; + + @Before + public void setUp() { + CallContext.unregister(); + account = new AccountVO("admin", 1L, "", Account.Type.ADMIN, "uuid"); + account.setId(accountId); + UserVO user = new UserVO(1, "admin", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); + CallContext.register(user, account); + when(accountMgr.finalizeOwner(any(Account.class), nullable(String.class), nullable(Long.class), nullable(Long.class))).thenReturn(account); + + when(primaryDataStoreDao.findById(poolId)).thenReturn(storagePoolVO); + when(storagePoolVO.getId()).thenReturn(poolId); + when(storagePoolVO.getDataCenterId()).thenReturn(zoneId); + when(storagePoolVO.isLocal()).thenReturn(isLocal); + when(storagePoolVO.getHypervisor()).thenReturn(hypervisorType); + when(storagePoolVO.getUuid()).thenReturn(storagePoolUuid); + when(storagePoolVO.getName()).thenReturn(storagePoolName); + when(storagePoolVO.getPoolType()).thenReturn(storagePoolType); + + when(volumeDao.findById(volumeId)).thenReturn(volumeVO); + when(volumeVO.getId()).thenReturn(volumeId); + when(volumeVO.getAccountId()).thenReturn(accountId); + when(volumeVO.getSize()).thenReturn(virtualSize); + when(volumeVO.getDataCenterId()).thenReturn(zoneId); + when(volumeVO.getName()).thenReturn(volumeName); + + when(hostVO.getHypervisorType()).thenReturn(hypervisorType); + when(hostVO.getId()).thenReturn(hostId); + when(hostDao.findById(hostId)).thenReturn(hostVO); + + when(storagePoolHostVO.getLocalPath()).thenReturn(localPath); + when(storagePoolHostDao.findByPoolHost(poolId, hostId)).thenReturn(storagePoolHostVO); + when(storagePoolHostVO.getHostId()).thenReturn(hostId); + + when(dcDao.findById(zoneId)).thenReturn(dataCenterVO); + } + + @Test + public void testListVolumesForImport() { + ListVolumesForImportCmd cmd = mock(ListVolumesForImportCmd.class); + when(cmd.getPath()).thenReturn(path); + when(cmd.getStorageId()).thenReturn(poolId); + + when(volumeDao.findByPoolIdAndPath(poolId, path)).thenReturn(null); + when(templatePoolDao.findByPoolPath(poolId, path)).thenReturn(null); + + VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(hypervisorType, path, name, fullPath, + format, size, virtualSize); + volumeOnStorageTO.setQemuEncryptFormat(encryptFormat); + List volumesOnStorageTO = new ArrayList<>(); + volumesOnStorageTO.add(volumeOnStorageTO); + doReturn(volumesOnStorageTO).when(volumeImportUnmanageManager).listVolumesForImportInternal(poolId, path); + + ListResponse listResponses = volumeImportUnmanageManager.listVolumesForImport(cmd); + Assert.assertEquals(1, listResponses.getResponses().size()); + VolumeForImportResponse response = listResponses.getResponses().get(0); + + Assert.assertEquals(path, response.getPath()); + Assert.assertEquals(name, response.getName()); + Assert.assertEquals(fullPath, response.getFullPath()); + Assert.assertEquals(format, response.getFormat()); + Assert.assertEquals(size, response.getSize()); + Assert.assertEquals(virtualSize, response.getVirtualSize()); + Assert.assertEquals(encryptFormat, response.getQemuEncryptFormat()); + Assert.assertEquals(storagePoolType.name(), response.getStoragePoolType()); + Assert.assertEquals(storagePoolName, response.getStoragePoolName()); + Assert.assertEquals(storagePoolUuid, response.getStoragePoolId()); + } + + @Test + public void testImportVolumeAllGood() throws ResourceAllocationException { + ImportVolumeCmd cmd = mock(ImportVolumeCmd.class); + when(cmd.getPath()).thenReturn(path); + when(cmd.getStorageId()).thenReturn(poolId); + when(cmd.getDiskOfferingId()).thenReturn(diskOfferingId); + when(volumeDao.findByPoolIdAndPath(poolId, path)).thenReturn(null); + when(templatePoolDao.findByPoolPath(poolId, path)).thenReturn(null); + + VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(hypervisorType, path, name, fullPath, + format, size, virtualSize); + volumeOnStorageTO.setQemuEncryptFormat(encryptFormat); + List volumesOnStorageTO = new ArrayList<>(); + volumesOnStorageTO.add(volumeOnStorageTO); + + doReturn(volumesOnStorageTO).when(volumeImportUnmanageManager).listVolumesForImportInternal(poolId, path); + + doNothing().when(volumeImportUnmanageManager).checkIfVolumeIsLocked(volumeOnStorageTO); + doNothing().when(volumeImportUnmanageManager).checkIfVolumeIsEncrypted(volumeOnStorageTO); + + doNothing().when(resourceLimitService).checkResourceLimit(account, Resource.ResourceType.volume); + doNothing().when(resourceLimitService).checkResourceLimit(account, Resource.ResourceType.primary_storage, virtualSize); + + DiskOfferingVO diskOffering = mock(DiskOfferingVO.class); + doReturn(diskOffering).when(volumeImportUnmanageManager).getOrCreateDiskOffering(account, diskOfferingId, zoneId, isLocal); + doReturn(diskProfile).when(volumeManager).importVolume(any(), anyString(), any(), eq(virtualSize), isNull(), isNull(), anyLong(), + any(), isNull(), isNull(), any(), isNull(), anyLong(), anyString(), isNull()); + when(diskProfile.getVolumeId()).thenReturn(volumeId); + when(volumeDao.findById(volumeId)).thenReturn(volumeVO); + + doNothing().when(resourceLimitService).incrementResourceCount(accountId, Resource.ResourceType.volume); + doNothing().when(resourceLimitService).incrementResourceCount(accountId, Resource.ResourceType.primary_storage, virtualSize); + + VolumeResponse response = mock(VolumeResponse.class); + doReturn(response).when(responseGenerator).createVolumeResponse(ResponseObject.ResponseView.Full, volumeVO); + try (MockedStatic ignored = Mockito.mockStatic(UsageEventUtils.class)) { + VolumeResponse result = volumeImportUnmanageManager.importVolume(cmd); + Assert.assertEquals(response, result); + } + } + + @Test + public void testListVolumesForImportInternal() { + Pair hostAndLocalPath = mock(Pair.class); + doReturn(hostAndLocalPath).when(volumeImportUnmanageManager).findHostAndLocalPathForVolumeImport(storagePoolVO); + when(hostAndLocalPath.first()).thenReturn(hostVO); + + VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(hypervisorType, path, name, fullPath, + format, size, virtualSize); + volumeOnStorageTO.setQemuEncryptFormat(encryptFormat); + List volumesOnStorageTO = new ArrayList<>(); + volumesOnStorageTO.add(volumeOnStorageTO); + GetVolumesOnStorageAnswer answer = mock(GetVolumesOnStorageAnswer.class); + when(answer.getResult()).thenReturn(true); + when(answer.getVolumes()).thenReturn(volumesOnStorageTO); + doReturn(answer).when(agentManager).easySend(eq(hostId), any(GetVolumesOnStorageCommand.class)); + + List result = volumeImportUnmanageManager.listVolumesForImportInternal(poolId, path); + Assert.assertEquals(volumesOnStorageTO, result); + } + + @Test(expected = CloudRuntimeException.class) + public void testCheckIfVolumeIsLocked() { + VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(hypervisorType, path, name, fullPath, + format, size, virtualSize); + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.IS_LOCKED, "true"); + volumeImportUnmanageManager.checkIfVolumeIsLocked(volumeOnStorageTO); + } + + @Test(expected = CloudRuntimeException.class) + public void testCheckIfVolumeIsEncrypted() { + VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(hypervisorType, path, name, fullPath, + format, size, virtualSize); + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.IS_ENCRYPTED, "true"); + volumeImportUnmanageManager.checkIfVolumeIsEncrypted(volumeOnStorageTO); + } + + @Test + public void testUnmanageVolume() { + when(volumeVO.getState()).thenReturn(Volume.State.Ready); + when(volumeVO.getPoolId()).thenReturn(poolId); + when(volumeVO.getInstanceId()).thenReturn(null); + doNothing().when(resourceLimitService).decrementResourceCount(accountId, Resource.ResourceType.volume); + doNothing().when(resourceLimitService).decrementResourceCount(accountId, Resource.ResourceType.primary_storage, virtualSize); + + try (MockedStatic ignored = Mockito.mockStatic(UsageEventUtils.class)) { + volumeImportUnmanageManager.unmanageVolume(volumeId); + } + + verify(resourceLimitService).decrementResourceCount(volumeVO.getAccountId(), Resource.ResourceType.volume); + verify(resourceLimitService).decrementResourceCount(volumeVO.getAccountId(), Resource.ResourceType.primary_storage, virtualSize); + verify(volumeDao).remove(volumeId); + } + + @Test(expected = CloudRuntimeException.class) + public void testUnmanageVolumeNotExist() { + when(volumeDao.findById(volumeId)).thenReturn(null); + volumeImportUnmanageManager.unmanageVolume(volumeId); + } + + @Test(expected = CloudRuntimeException.class) + public void testUnmanageVolumeNotReady() { + when(volumeVO.getState()).thenReturn(Volume.State.Allocated); + volumeImportUnmanageManager.unmanageVolume(volumeId); + } + + + @Test(expected = CloudRuntimeException.class) + public void testUnmanageVolumeEncrypted() { + when(volumeVO.getState()).thenReturn(Volume.State.Ready); + when(volumeVO.getEncryptFormat()).thenReturn(encryptFormat); + volumeImportUnmanageManager.unmanageVolume(volumeId); + } + + @Test(expected = CloudRuntimeException.class) + public void testUnmanageVolumeAttached() { + when(volumeVO.getState()).thenReturn(Volume.State.Ready); + when(volumeVO.getAttached()).thenReturn(new Date()); + volumeImportUnmanageManager.unmanageVolume(volumeId); + } + + @Test(expected = CloudRuntimeException.class) + public void testCheckIfPoolAvailableNotExist() { + when(primaryDataStoreDao.findById(poolId)).thenReturn(null); + volumeImportUnmanageManager.checkIfPoolAvailable(poolId); + } + + @Test(expected = CloudRuntimeException.class) + public void testCheckIfPoolAvailableInMaintenance() { + when(primaryDataStoreDao.findById(poolId)).thenReturn(storagePoolVO); + when(storagePoolVO.isInMaintenance()).thenReturn(true); + volumeImportUnmanageManager.checkIfPoolAvailable(poolId); + } + + @Test + public void testFindHostAndLocalPathForVolumeImportZoneScope() { + when(storagePoolVO.getScope()).thenReturn(ScopeType.ZONE); + List hosts = new ArrayList<>(); + hosts.add(hostVO); + when(hostDao.listAllHostsUpByZoneAndHypervisor(zoneId, hypervisorType)).thenReturn(hosts); + + Pair result = volumeImportUnmanageManager.findHostAndLocalPathForVolumeImport(storagePoolVO); + Assert.assertNotNull(result); + Assert.assertEquals(hostVO, result.first()); + Assert.assertEquals(localPath, result.second()); + } + + @Test + public void testFindHostAndLocalPathForVolumeImportClusterScope() { + when(storagePoolVO.getScope()).thenReturn(ScopeType.CLUSTER); + when(storagePoolVO.getClusterId()).thenReturn(clusterId); + + List hosts = new ArrayList<>(); + hosts.add(hostVO); + when(hostDao.findHypervisorHostInCluster(clusterId)).thenReturn(hosts); + + Pair result = volumeImportUnmanageManager.findHostAndLocalPathForVolumeImport(storagePoolVO); + Assert.assertNotNull(result); + Assert.assertEquals(hostVO, result.first()); + Assert.assertEquals(localPath, result.second()); + } + + @Test + public void testFindHostAndLocalPathForVolumeImportLocalHost() { + when(storagePoolVO.getScope()).thenReturn(ScopeType.HOST); + + List storagePoolHostVOs = new ArrayList<>(); + storagePoolHostVOs.add(storagePoolHostVO); + when(storagePoolHostDao.listByPoolId(poolId)).thenReturn(storagePoolHostVOs); + + Pair result = volumeImportUnmanageManager.findHostAndLocalPathForVolumeImport(storagePoolVO); + Assert.assertNotNull(result); + Assert.assertEquals(hostVO, result.first()); + Assert.assertEquals(localPath, result.second()); + } + + @Test + public void testGetOrCreateDiskOfferingAllGood() { + when(diskOfferingDao.findById(diskOfferingId)).thenReturn(diskOfferingVO); + when(diskOfferingVO.getState()).thenReturn(DiskOffering.State.Active); + when(diskOfferingVO.isUseLocalStorage()).thenReturn(isLocal); + doNothing().when(configMgr).checkDiskOfferingAccess(account, diskOfferingVO, dataCenterVO); + + DiskOfferingVO result = volumeImportUnmanageManager.getOrCreateDiskOffering(account, diskOfferingId, zoneId, isLocal); + Assert.assertEquals(diskOfferingVO, result); + } + + @Test(expected = CloudRuntimeException.class) + public void testGetOrCreateDiskOfferingNotExist() { + when(diskOfferingDao.findById(diskOfferingId)).thenReturn(null); + + volumeImportUnmanageManager.getOrCreateDiskOffering(account, diskOfferingId, zoneId, isLocal); + } + + @Test(expected = CloudRuntimeException.class) + public void testGetOrCreateDiskOfferingNotActive() { + when(diskOfferingDao.findById(diskOfferingId)).thenReturn(diskOfferingVO); + when(diskOfferingVO.getState()).thenReturn(DiskOffering.State.Inactive); + + volumeImportUnmanageManager.getOrCreateDiskOffering(account, diskOfferingId, zoneId, isLocal); + } + + @Test(expected = CloudRuntimeException.class) + public void testGetOrCreateDiskOfferingNotLocal() { + when(diskOfferingDao.findById(diskOfferingId)).thenReturn(diskOfferingVO); + when(diskOfferingVO.getState()).thenReturn(DiskOffering.State.Active); + when(diskOfferingVO.isUseLocalStorage()).thenReturn(!isLocal); + + volumeImportUnmanageManager.getOrCreateDiskOffering(account, diskOfferingId, zoneId, isLocal); + } + + @Test(expected = CloudRuntimeException.class) + public void testGetOrCreateDiskOfferingNoPermission() { + when(diskOfferingDao.findById(diskOfferingId)).thenReturn(diskOfferingVO); + when(diskOfferingVO.getState()).thenReturn(DiskOffering.State.Active); + when(diskOfferingVO.isUseLocalStorage()).thenReturn(isLocal); + doThrow(PermissionDeniedException.class).when(configMgr).checkDiskOfferingAccess(account, diskOfferingVO, dataCenterVO); + + volumeImportUnmanageManager.getOrCreateDiskOffering(account, diskOfferingId, zoneId, isLocal); + } + + @Test + public void testGetOrCreateDefaultDiskOfferingIdForVolumeImportExist() { + String uniqueName = DEFAULT_DISK_OFFERING_UNIQUE_NAME + (isLocal ? DISK_OFFERING_UNIQUE_NAME_SUFFIX_LOCAL : ""); + when(diskOfferingDao.findByUniqueName(uniqueName)).thenReturn(diskOfferingVO); + + DiskOfferingVO result = volumeImportUnmanageManager.getOrCreateDiskOffering(account, null, zoneId, isLocal); + Assert.assertEquals(diskOfferingVO, result); + } + + @Test + public void testGetOrCreateDefaultDiskOfferingIdForVolumeImportNotExist() { + String uniqueName = DEFAULT_DISK_OFFERING_UNIQUE_NAME + (isLocal ? DISK_OFFERING_UNIQUE_NAME_SUFFIX_LOCAL : ""); + when(diskOfferingDao.findByUniqueName(uniqueName)).thenReturn(null); + when(diskOfferingDao.persistDefaultDiskOffering(any())).thenReturn(diskOfferingVO); + + try ( + MockedConstruction diskOfferingVOMockedConstruction = Mockito.mockConstruction(DiskOfferingVO.class); + ) { + DiskOfferingVO result = volumeImportUnmanageManager.getOrCreateDiskOffering(account, null, zoneId, isLocal); + Assert.assertEquals(diskOfferingVO, result); + + DiskOfferingVO diskOfferingVOMock = diskOfferingVOMockedConstruction.constructed().get(0); + verify(diskOfferingVOMock).setUseLocalStorage(isLocal); + verify(diskOfferingVOMock).setUniqueName(uniqueName); + } + } +} From 4b9ef71135466642c0bf1217b68b201071afc76a Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 25 Mar 2024 09:18:39 +0100 Subject: [PATCH 18/44] Update 8088: search by keyword --- .../cloud/agent/api/GetVolumesOnStorageCommand.java | 10 ++++++++-- .../agent/api/GetVolumesOnStorageCommandTest.java | 4 +++- .../LibvirtGetVolumesOnStorageCommandWrapper.java | 5 +++++ ...LibvirtGetVolumesOnStorageCommandWrapperTest.java | 7 ++++--- .../volume/VolumeImportUnmanageManagerImpl.java | 12 ++++++++---- .../volume/VolumeImportUnmanageManagerImplTest.java | 6 +++--- 6 files changed, 31 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/com/cloud/agent/api/GetVolumesOnStorageCommand.java b/core/src/main/java/com/cloud/agent/api/GetVolumesOnStorageCommand.java index 8fbbac914b36..6bc3356eb671 100644 --- a/core/src/main/java/com/cloud/agent/api/GetVolumesOnStorageCommand.java +++ b/core/src/main/java/com/cloud/agent/api/GetVolumesOnStorageCommand.java @@ -24,14 +24,16 @@ public class GetVolumesOnStorageCommand extends Command { StorageFilerTO pool; - private String volumePath; //filter by file path + private String volumePath; //search by file path + private String keyword; //filter by keyword public GetVolumesOnStorageCommand() { } - public GetVolumesOnStorageCommand(StorageFilerTO pool, String filePath) { + public GetVolumesOnStorageCommand(StorageFilerTO pool, String filePath, String keyword) { this.pool = pool; this.volumePath = filePath; + this.keyword = keyword; } public StorageFilerTO getPool() { @@ -42,6 +44,10 @@ public String getVolumePath() { return volumePath; } + public String getKeyword() { + return keyword; + } + @Override public boolean executeInSequence() { return false; diff --git a/core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageCommandTest.java b/core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageCommandTest.java index e7883a2e02e9..7b8a91145716 100644 --- a/core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageCommandTest.java +++ b/core/src/test/java/com/cloud/agent/api/GetVolumesOnStorageCommandTest.java @@ -27,13 +27,15 @@ public class GetVolumesOnStorageCommandTest { final String localPath = "localPath"; final String volumePath = "volumePath"; + final String keyword = "keyword"; @Test public void testGetVolumesOnStorageCommand() { - GetVolumesOnStorageCommand command = new GetVolumesOnStorageCommand(pool, volumePath); + GetVolumesOnStorageCommand command = new GetVolumesOnStorageCommand(pool, volumePath, keyword); Assert.assertEquals(pool, command.getPool()); Assert.assertEquals(volumePath, command.getVolumePath()); + Assert.assertEquals(keyword, command.getKeyword()); Assert.assertFalse(command.executeInSequence()); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java index 314a94b2f371..884dc6523c79 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java @@ -45,6 +45,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @ResourceWrapper(handles = GetVolumesOnStorageCommand.class) public final class LibvirtGetVolumesOnStorageCommandWrapper extends CommandWrapper { @@ -57,6 +58,7 @@ public Answer execute(final GetVolumesOnStorageCommand command, final LibvirtCom final StorageFilerTO pool = command.getPool(); final String volumePath = command.getVolumePath(); + final String keyword = command.getKeyword(); List volumes = new ArrayList<>(); @@ -105,6 +107,9 @@ public Answer execute(final GetVolumesOnStorageCommand command, final LibvirtCom } } else { List disks = storagePool.listPhysicalDisks(); + if (StringUtils.isNotBlank(keyword)) { + disks = disks.stream().filter(disk -> disk.getName().contains(keyword)).collect(Collectors.toList()); + } disks.sort(Comparator.comparing(KVMPhysicalDisk::getName)); for (KVMPhysicalDisk disk: disks) { if (!isDiskFormatSupported(disk)) { diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapperTest.java index e4bce1988e48..73887caab792 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapperTest.java @@ -75,6 +75,7 @@ public class LibvirtGetVolumesOnStorageCommandWrapperTest { private final String clusterSize = "4096"; private final String fileFormat = "QCOW2"; private final String encrypted = "yes"; + private final String diskNamePrefix = "disk-"; @Spy LibvirtGetVolumesOnStorageCommandWrapper libvirtGetVolumesOnStorageCommandWrapper = new LibvirtGetVolumesOnStorageCommandWrapper(); @@ -103,12 +104,12 @@ public void tearDown() { @Test public void testLibvirtGetVolumesOnStorageCommandWrapperForAllVolumes() { - GetVolumesOnStorageCommand command = new GetVolumesOnStorageCommand(pool, null); + GetVolumesOnStorageCommand command = new GetVolumesOnStorageCommand(pool, null, diskNamePrefix); List physicalDisks = new ArrayList<>(); int numberDisks = 3; for (int i = 0; i < numberDisks; i++) { KVMPhysicalDisk disk = Mockito.mock(KVMPhysicalDisk.class); - Mockito.when(disk.getName()).thenReturn("name-" + (numberDisks - i)); + Mockito.when(disk.getName()).thenReturn(diskNamePrefix + (numberDisks - i)); Mockito.when(disk.getFormat()).thenReturn(QemuImg.PhysicalDiskFormat.QCOW2); Mockito.when(disk.getQemuEncryptFormat()).thenReturn(QemuObject.EncryptFormat.LUKS); physicalDisks.add(disk); @@ -137,7 +138,7 @@ public void testLibvirtGetVolumesOnStorageCommandWrapperForVolume() { Mockito.when(qemuImgInfo.get(QemuImg.FILE_FORMAT)).thenReturn(fileFormat); Mockito.when(qemuImgInfo.get(QemuImg.ENCRYPTED)).thenReturn(encrypted); - GetVolumesOnStorageCommand command = new GetVolumesOnStorageCommand(pool, volumePath); + GetVolumesOnStorageCommand command = new GetVolumesOnStorageCommand(pool, volumePath, null); Answer answer = libvirtGetVolumesOnStorageCommandWrapper.execute(command, libvirtComputingResource); Assert.assertTrue(answer instanceof GetVolumesOnStorageAnswer); Assert.assertTrue(answer.getResult()); diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index 37d73e0596a5..8dcb00c606c2 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -131,9 +131,13 @@ public List> getCommands() { public ListResponse listVolumesForImport(ListVolumesForImportCmd cmd) { Long poolId = cmd.getStorageId(); String path = cmd.getPath(); + String keyword = cmd.getKeyword(); + if (StringUtils.isNotBlank(keyword)) { + keyword = keyword.trim(); + } StoragePoolVO pool = checkIfPoolAvailable(poolId); - List volumes = listVolumesForImportInternal(poolId, path); + List volumes = listVolumesForImportInternal(poolId, path, keyword); List responses = new ArrayList<>(); for (VolumeOnStorageTO volume : volumes) { @@ -176,7 +180,7 @@ public VolumeResponse importVolume(ImportVolumeCmd cmd) { } // 4. send a command to hypervisor to check - List volumes = listVolumesForImportInternal(poolId, volumePath); + List volumes = listVolumesForImportInternal(poolId, volumePath, null); if (CollectionUtils.isEmpty(volumes)) { logFailureAndThrowException("Cannot find volume on storage pool: " + volumePath); } @@ -206,7 +210,7 @@ public VolumeResponse importVolume(ImportVolumeCmd cmd) { return responseGenerator.createVolumeResponse(ResponseObject.ResponseView.Full, volumeVO); } - protected List listVolumesForImportInternal(Long poolId, String volumePath) { + protected List listVolumesForImportInternal(Long poolId, String volumePath, String keyword) { StoragePoolVO pool = checkIfPoolAvailable(poolId); Pair hostAndLocalPath = findHostAndLocalPathForVolumeImport(pool); @@ -216,7 +220,7 @@ protected List listVolumesForImportInternal(Long poolId, Stri } StorageFilerTO storageTO = new StorageFilerTO(pool); - GetVolumesOnStorageCommand command = new GetVolumesOnStorageCommand(storageTO, volumePath); + GetVolumesOnStorageCommand command = new GetVolumesOnStorageCommand(storageTO, volumePath, keyword); Answer answer = agentManager.easySend(host.getId(), command); if (answer == null || !(answer instanceof GetVolumesOnStorageAnswer)) { logFailureAndThrowException("Cannot get volumes on storage pool via host " + host.getName()); diff --git a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java index 8d6975c3b38c..e891290a5947 100644 --- a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java @@ -218,7 +218,7 @@ public void testListVolumesForImport() { volumeOnStorageTO.setQemuEncryptFormat(encryptFormat); List volumesOnStorageTO = new ArrayList<>(); volumesOnStorageTO.add(volumeOnStorageTO); - doReturn(volumesOnStorageTO).when(volumeImportUnmanageManager).listVolumesForImportInternal(poolId, path); + doReturn(volumesOnStorageTO).when(volumeImportUnmanageManager).listVolumesForImportInternal(poolId, path, null); ListResponse listResponses = volumeImportUnmanageManager.listVolumesForImport(cmd); Assert.assertEquals(1, listResponses.getResponses().size()); @@ -251,7 +251,7 @@ public void testImportVolumeAllGood() throws ResourceAllocationException { List volumesOnStorageTO = new ArrayList<>(); volumesOnStorageTO.add(volumeOnStorageTO); - doReturn(volumesOnStorageTO).when(volumeImportUnmanageManager).listVolumesForImportInternal(poolId, path); + doReturn(volumesOnStorageTO).when(volumeImportUnmanageManager).listVolumesForImportInternal(poolId, path, null); doNothing().when(volumeImportUnmanageManager).checkIfVolumeIsLocked(volumeOnStorageTO); doNothing().when(volumeImportUnmanageManager).checkIfVolumeIsEncrypted(volumeOnStorageTO); @@ -293,7 +293,7 @@ public void testListVolumesForImportInternal() { when(answer.getVolumes()).thenReturn(volumesOnStorageTO); doReturn(answer).when(agentManager).easySend(eq(hostId), any(GetVolumesOnStorageCommand.class)); - List result = volumeImportUnmanageManager.listVolumesForImportInternal(poolId, path); + List result = volumeImportUnmanageManager.listVolumesForImportInternal(poolId, path, null); Assert.assertEquals(volumesOnStorageTO, result); } From b9b71166d21b203b0ff7d4c3fde038531f9cd032 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 25 Mar 2024 12:36:18 +0100 Subject: [PATCH 19/44] Update 8088: fix typo in ImportVolumeCmd --- .../api/command/admin/volume/ImportVolumeCmd.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java index 662de4cfefe4..57c3ee586d35 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java @@ -86,19 +86,19 @@ public class ImportVolumeCmd extends BaseAsyncCmd { @Parameter(name = ApiConstants.ACCOUNT, type = BaseCmd.CommandType.STRING, - description = "an optional account for the virtual machine. Must be used with domainId.") + description = "an optional account for the volume. Must be used with domainId.") private String accountName; @Parameter(name = ApiConstants.DOMAIN_ID, type = BaseCmd.CommandType.UUID, entityType = DomainResponse.class, - description = "import instance to the domain specified") + description = "import volume to the domain specified") private Long domainId; @Parameter(name = ApiConstants.PROJECT_ID, type = BaseCmd.CommandType.UUID, entityType = ProjectResponse.class, - description = "import instance for the project") + description = "import volume for the project") private Long projectId; ///////////////////////////////////////////////////// From b13607b77f4704857d4d4e93b9b86da9732e99bb Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 25 Mar 2024 13:34:57 +0100 Subject: [PATCH 20/44] Update 8088: mark unmanaged volume as Destroy --- .../storage/volume/VolumeImportUnmanageManagerImpl.java | 5 ++++- .../storage/volume/VolumeImportUnmanageManagerImplTest.java | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index 8dcb00c606c2..333294c16b8a 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -72,6 +72,7 @@ import javax.inject.Inject; import java.util.ArrayList; import java.util.Arrays; +import java.util.Date; import java.util.List; import java.util.Map; @@ -437,6 +438,8 @@ private VolumeVO checkIfVolumeCanBeUnmanaged(long volumeId) { } private void unmanageVolumeFromDatabase(VolumeVO volume) { - volumeDao.remove(volume.getId()); + volume.setState(Volume.State.Destroy); + volume.setRemoved(new Date()); + volumeDao.update(volume.getId(), volume); } } diff --git a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java index e891290a5947..e1616de3af6d 100644 --- a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java @@ -327,7 +327,7 @@ public void testUnmanageVolume() { verify(resourceLimitService).decrementResourceCount(volumeVO.getAccountId(), Resource.ResourceType.volume); verify(resourceLimitService).decrementResourceCount(volumeVO.getAccountId(), Resource.ResourceType.primary_storage, virtualSize); - verify(volumeDao).remove(volumeId); + verify(volumeDao).update(eq(volumeId), any()); } @Test(expected = CloudRuntimeException.class) From dc906baf9522380db520925a0299cfdd7fc0f008 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 25 Mar 2024 14:00:03 +0100 Subject: [PATCH 21/44] Update 8088: skip tests in setUpClass --- test/integration/smoke/test_import_unmanage_volumes.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/integration/smoke/test_import_unmanage_volumes.py b/test/integration/smoke/test_import_unmanage_volumes.py index abd49215834e..9001e97a79ed 100644 --- a/test/integration/smoke/test_import_unmanage_volumes.py +++ b/test/integration/smoke/test_import_unmanage_volumes.py @@ -44,6 +44,9 @@ def setUpClass(cls): cls.services = testClient.getParsedTestDataConfig() cls.hypervisor = testClient.getHypervisorInfo() + if cls.testClient.getHypervisorInfo().lower() != "kvm": + raise unittest.SkipTest("This is only available for KVM") + cls.domain = get_domain(cls.apiclient) cls.zone = get_zone(cls.apiclient) cls._cleanup = [] @@ -100,10 +103,6 @@ def setUpClass(cls): def tearDownClass(cls): super(TestImportAndUnmanageVolumes, cls).tearDownClass() - def setUp(self): - if self.testClient.getHypervisorInfo().lower() != "kvm": - raise unittest.SkipTest("This is only available for KVM") - @attr(tags=['advanced', 'basic', 'sg'], required_hardware=False) def test_01_detach_unmanage_import_volume(self): """Test attach/detach/unmanage/import volume From b085e3208841055cb306cef30fbc807fded9bc3e Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 25 Mar 2024 14:23:19 +0100 Subject: [PATCH 22/44] Update 8088: refactor findHostAndLocalPathForVolumeImportForHost --- .../VolumeImportUnmanageManagerImpl.java | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index 333294c16b8a..9b510c93dfeb 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -34,7 +34,6 @@ import com.cloud.hypervisor.Hypervisor; import com.cloud.offering.DiskOffering; import com.cloud.storage.DiskOfferingVO; -import com.cloud.storage.ScopeType; import com.cloud.storage.Storage; import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.Volume; @@ -265,20 +264,15 @@ protected StoragePoolVO checkIfPoolAvailable(Long poolId) { protected Pair findHostAndLocalPathForVolumeImport(StoragePoolVO pool) { List hosts = new ArrayList<>(); - if (ScopeType.HOST.equals(pool.getScope())) { - List storagePoolHostVOs = storagePoolHostDao.listByPoolId(pool.getId()); - if (CollectionUtils.isNotEmpty(storagePoolHostVOs)) { - for (StoragePoolHostVO storagePoolHostVO : storagePoolHostVOs) { - HostVO host = hostDao.findById(storagePoolHostVO.getHostId()); - if (host != null) { - return new Pair<>(host, storagePoolHostVO.getLocalPath()); - } - } - } - } else if (ScopeType.CLUSTER.equals(pool.getScope())) { - hosts = hostDao.findHypervisorHostInCluster((pool.getClusterId())); - } else if (ScopeType.ZONE.equals(pool.getScope())) { - hosts = hostDao.listAllHostsUpByZoneAndHypervisor(pool.getDataCenterId(), pool.getHypervisor()); + switch (pool.getScope()) { + case HOST: + return findHostAndLocalPathForVolumeImportForHostScope(pool.getId()); + case CLUSTER: + hosts = hostDao.findHypervisorHostInCluster((pool.getClusterId())); + break; + case ZONE: + hosts = hostDao.listAllHostsUpByZoneAndHypervisor(pool.getDataCenterId(), pool.getHypervisor()); + break; } for (HostVO host : hosts) { StoragePoolHostVO storagePoolHostVO = storagePoolHostDao.findByPoolHost(pool.getId(), host.getId()); @@ -290,6 +284,20 @@ protected Pair findHostAndLocalPathForVolumeImport(StoragePoolVO return null; } + private Pair findHostAndLocalPathForVolumeImportForHostScope(Long poolId) { + List storagePoolHostVOs = storagePoolHostDao.listByPoolId(poolId); + if (CollectionUtils.isNotEmpty(storagePoolHostVOs)) { + for (StoragePoolHostVO storagePoolHostVO : storagePoolHostVOs) { + HostVO host = hostDao.findById(storagePoolHostVO.getHostId()); + if (host != null) { + return new Pair<>(host, storagePoolHostVO.getLocalPath()); + } + } + } + logFailureAndThrowException("No host found to perform volume import on pool: " + poolId); + return null; + } + protected VolumeForImportResponse createVolumeForImportResponse(VolumeOnStorageTO volume, StoragePoolVO pool) { VolumeForImportResponse response = new VolumeForImportResponse(); response.setPath(volume.getPath()); From 7a14fbd5dfd543a33affeb913dadaf9cfb0c91dd Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 25 Mar 2024 14:30:29 +0100 Subject: [PATCH 23/44] Update 8088: refactor LibvirtGetVolumesOnStorageCommandWrapper --- ...virtGetVolumesOnStorageCommandWrapper.java | 122 ++++++++++-------- 1 file changed, 66 insertions(+), 56 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java index 884dc6523c79..67da52069f1b 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java @@ -60,70 +60,80 @@ public Answer execute(final GetVolumesOnStorageCommand command, final LibvirtCom final String volumePath = command.getVolumePath(); final String keyword = command.getKeyword(); - List volumes = new ArrayList<>(); - final KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr(); - KVMStoragePool storagePool = storagePoolMgr.getStoragePool(pool.getType(), pool.getUuid(), true); + final KVMStoragePool storagePool = storagePoolMgr.getStoragePool(pool.getType(), pool.getUuid(), true); if (StringUtils.isNotBlank(volumePath)) { - KVMPhysicalDisk disk = storagePool.getPhysicalDisk(volumePath); - if (disk != null) { - if (!isDiskFormatSupported(disk)) { - return new GetVolumesOnStorageAnswer(command, false, String.format("disk format %s is unsupported", disk.getFormat())); - } - Map info = getDiskFileInfo(storagePool, disk, true); - if (info == null) { - return new GetVolumesOnStorageAnswer(command, false, "failed to get information of disk file. The disk might be locked or unsupported"); - } - VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(Hypervisor.HypervisorType.KVM, disk.getName(), disk.getName(), disk.getPath(), - disk.getFormat().toString(), disk.getSize(), disk.getVirtualSize()); - if (disk.getQemuEncryptFormat() != null) { - volumeOnStorageTO.setQemuEncryptFormat(disk.getQemuEncryptFormat().toString()); - } - String backingFilePath = info.get(QemuImg.BACKING_FILE); - if (StringUtils.isNotBlank(backingFilePath)) { - volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.BACKING_FILE, backingFilePath); - } - String backingFileFormat = info.get(QemuImg.BACKING_FILE_FORMAT); - if (StringUtils.isNotBlank(backingFileFormat)) { - volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.BACKING_FILE_FORMAT, backingFileFormat); - } - String clusterSize = info.get(QemuImg.CLUSTER_SIZE); - if (StringUtils.isNotBlank(clusterSize)) { - volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.CLUSTER_SIZE, clusterSize); - } - String fileFormat = info.get(QemuImg.FILE_FORMAT); - if (StringUtils.isNotBlank(clusterSize)) { - volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.FILE_FORMAT, fileFormat); - } - String encrypted = info.get(QemuImg.ENCRYPTED); - if (StringUtils.isNotBlank(encrypted) && encrypted.toLowerCase().equals("yes")) { - volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.IS_ENCRYPTED, String.valueOf(Boolean.TRUE)); - } - Boolean isLocked = isDiskFileLocked(storagePool, disk); - volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.IS_LOCKED, String.valueOf(isLocked)); - - volumes.add(volumeOnStorageTO); - } + return addVolumeByVolumePath(command, storagePool, volumePath); } else { - List disks = storagePool.listPhysicalDisks(); - if (StringUtils.isNotBlank(keyword)) { - disks = disks.stream().filter(disk -> disk.getName().contains(keyword)).collect(Collectors.toList()); + return addAllVolumes(command, storagePool, keyword); + } + } + + private GetVolumesOnStorageAnswer addVolumeByVolumePath(final GetVolumesOnStorageCommand command, final KVMStoragePool storagePool, String volumePath) { + List volumes = new ArrayList<>(); + + KVMPhysicalDisk disk = storagePool.getPhysicalDisk(volumePath); + if (disk != null) { + if (!isDiskFormatSupported(disk)) { + return new GetVolumesOnStorageAnswer(command, false, String.format("disk format %s is unsupported", disk.getFormat())); + } + Map info = getDiskFileInfo(storagePool, disk, true); + if (info == null) { + return new GetVolumesOnStorageAnswer(command, false, "failed to get information of disk file. The disk might be locked or unsupported"); + } + VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(Hypervisor.HypervisorType.KVM, disk.getName(), disk.getName(), disk.getPath(), + disk.getFormat().toString(), disk.getSize(), disk.getVirtualSize()); + if (disk.getQemuEncryptFormat() != null) { + volumeOnStorageTO.setQemuEncryptFormat(disk.getQemuEncryptFormat().toString()); + } + String backingFilePath = info.get(QemuImg.BACKING_FILE); + if (StringUtils.isNotBlank(backingFilePath)) { + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.BACKING_FILE, backingFilePath); + } + String backingFileFormat = info.get(QemuImg.BACKING_FILE_FORMAT); + if (StringUtils.isNotBlank(backingFileFormat)) { + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.BACKING_FILE_FORMAT, backingFileFormat); } - disks.sort(Comparator.comparing(KVMPhysicalDisk::getName)); - for (KVMPhysicalDisk disk: disks) { - if (!isDiskFormatSupported(disk)) { - continue; - } - VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(Hypervisor.HypervisorType.KVM, disk.getName(), disk.getName(), disk.getPath(), - disk.getFormat().toString(), disk.getSize(), disk.getVirtualSize()); - if (disk.getQemuEncryptFormat() != null) { - volumeOnStorageTO.setQemuEncryptFormat(disk.getQemuEncryptFormat().toString()); - } - volumes.add(volumeOnStorageTO); + String clusterSize = info.get(QemuImg.CLUSTER_SIZE); + if (StringUtils.isNotBlank(clusterSize)) { + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.CLUSTER_SIZE, clusterSize); } + String fileFormat = info.get(QemuImg.FILE_FORMAT); + if (StringUtils.isNotBlank(clusterSize)) { + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.FILE_FORMAT, fileFormat); + } + String encrypted = info.get(QemuImg.ENCRYPTED); + if (StringUtils.isNotBlank(encrypted) && encrypted.toLowerCase().equals("yes")) { + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.IS_ENCRYPTED, String.valueOf(Boolean.TRUE)); + } + Boolean isLocked = isDiskFileLocked(storagePool, disk); + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.IS_LOCKED, String.valueOf(isLocked)); + + volumes.add(volumeOnStorageTO); } + return new GetVolumesOnStorageAnswer(command, volumes); + } + private GetVolumesOnStorageAnswer addAllVolumes(final GetVolumesOnStorageCommand command, final KVMStoragePool storagePool, String keyword) { + List volumes = new ArrayList<>(); + + List disks = storagePool.listPhysicalDisks(); + if (StringUtils.isNotBlank(keyword)) { + disks = disks.stream().filter(disk -> disk.getName().contains(keyword)).collect(Collectors.toList()); + } + disks.sort(Comparator.comparing(KVMPhysicalDisk::getName)); + for (KVMPhysicalDisk disk: disks) { + if (!isDiskFormatSupported(disk)) { + continue; + } + VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(Hypervisor.HypervisorType.KVM, disk.getName(), disk.getName(), disk.getPath(), + disk.getFormat().toString(), disk.getSize(), disk.getVirtualSize()); + if (disk.getQemuEncryptFormat() != null) { + volumeOnStorageTO.setQemuEncryptFormat(disk.getQemuEncryptFormat().toString()); + } + volumes.add(volumeOnStorageTO); + } return new GetVolumesOnStorageAnswer(command, volumes); } From a2db5cf78e349dd7bad514919cda1ff3531a8c2a Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 25 Mar 2024 20:26:56 +0100 Subject: [PATCH 24/44] Update 8808: add test testLogFailureAndThrowException --- .../volume/VolumeImportUnmanageManagerImpl.java | 2 +- .../volume/VolumeImportUnmanageManagerImplTest.java | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index 9b510c93dfeb..8df0372c99fc 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -113,7 +113,7 @@ public class VolumeImportUnmanageManagerImpl implements VolumeImportUnmanageServ static final String DISK_OFFERING_NAME_SUFFIX_LOCAL = " - Local Storage"; static final String DISK_OFFERING_UNIQUE_NAME_SUFFIX_LOCAL = "-Local"; - private void logFailureAndThrowException(String msg) { + protected void logFailureAndThrowException(String msg) { logger.error(msg); throw new CloudRuntimeException(msg); } diff --git a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java index e1616de3af6d..f843275ef230 100644 --- a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java @@ -483,4 +483,14 @@ public void testGetOrCreateDefaultDiskOfferingIdForVolumeImportNotExist() { verify(diskOfferingVOMock).setUniqueName(uniqueName); } } + + @Test + public void testLogFailureAndThrowException() { + String message = "error message"; + try { + volumeImportUnmanageManager.logFailureAndThrowException(message); + } catch (CloudRuntimeException ex) { + Assert.assertEquals(message, ex.getMessage()); + } + } } From bc832091d1fff9b8a165e23c6026092ef188bc44 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 25 Mar 2024 20:48:45 +0100 Subject: [PATCH 25/44] Update 8088: UI for volume import/unmanage --- ui/public/locales/en.json | 16 + ui/src/config/section/tools.js | 9 + ui/src/views/tools/ManageVolumes.vue | 1099 ++++++++++++++++++++++++++ 3 files changed, 1124 insertions(+) create mode 100644 ui/src/views/tools/ManageVolumes.vue diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 17a70b423b39..4e9adb6d54d1 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -138,6 +138,7 @@ "label.action.image.store.read.only": "Make image store read-only", "label.action.image.store.read.write": "Make image store read-write", "label.action.import.export.instances": "Import-Export Instances", +"label.action.import.unmanage.volumes": "Import-Unmanage Volumes", "label.action.ingest.instances": "Ingest instances", "label.action.iso.permission": "Update ISO permissions", "label.action.iso.share": "Update ISO sharing", @@ -190,6 +191,8 @@ "label.action.unmanage.instance": "Unmanage Instance", "label.action.unmanage.instances": "Unmanage Instances", "label.action.unmanage.virtualmachine": "Unmanage Instance", +"label.action.unmanage.volume": "Unmanage Volume", +"label.action.unmanage.volumes": "Unmanage Volumes", "label.action.update.offering.access": "Update offering access", "label.action.update.resource.count": "Update resource count", "label.action.value": "Action/Value", @@ -679,6 +682,7 @@ "label.desc.import.ext.kvm.wizard": "Import Instance from remote KVM host", "label.desc.import.local.kvm.wizard": "Import QCOW2 image from Local Storage", "label.desc.import.shared.kvm.wizard": "Import QCOW2 image from Shared Storage", +"label.desc.import.unmanage.volume": "Import and unmanage volume on Storage Pools", "label.desc.ingesttinstancewizard": "Ingest instances from an external KVM host", "label.desc.importmigratefromvmwarewizard": "Import instances from VMware into a KVM cluster", "label.desc.usage.stats": "Usage Server Statistics", @@ -888,6 +892,7 @@ "label.featured": "Featured", "label.fetch.instances": "Fetch Instances", "label.fetch.latest": "Fetch latest", +"label.filename": "File Name", "label.files": "Alternate files to retrieve", "label.filter": "Filter", "label.filter.annotations.all": "All comments", @@ -1020,6 +1025,7 @@ "label.import.instance": "Import Instance", "label.import.offering": "Import offering", "label.import.role": "Import role", +"label.import.volume": "Import Volume", "label.in.progress": "in progress", "label.in.progress.for": "in progress for", "label.info": "Info", @@ -1255,6 +1261,7 @@ "label.manage": "Manage", "label.manage.vpn.user": "Manage VPN Users", "label.managed.instances": "Managed Instances", +"label.managed.volumes": "Managed Volumes", "label.managedstate": "Managed state", "label.management": "Management", "label.management.ips": "Management IP addresses", @@ -2155,8 +2162,10 @@ "label.unlimited": "Unlimited", "label.unmanaged": "Unmanaged", "label.unmanage.instance": "Unmanage Instance", +"label.unmanage.volume": "Unmanage Volume", "label.unmanaged.instance": "Unmanaged Instance", "label.unmanaged.instances": "Unmanaged Instances", +"label.unmanaged.volumes": "Unmanaged Volumes", "label.untagged": "Untagged", "label.up": "Up", "label.updateinsequence": "Update in sequence", @@ -2475,6 +2484,8 @@ "message.action.unmanage.instance": "Please confirm that you want to unmanage the Instance.", "message.action.unmanage.instances": "Please confirm that you want to unmanage the Instances.", "message.action.unmanage.virtualmachine": "Please confirm that you want to unmanage the Instance.", +"message.action.unmanage.volume": "Please confirm that you want to unmanage the Volume.", +"message.action.unmanage.volumes": "Please confirm that you want to unmanage the Volumes.", "message.action.vmsnapshot.delete": "Please confirm that you want to delete this Instance Snapshot.
Please notice that the Instance will be paused before the Snapshot deletion, and resumed after deletion, if it runs on KVM.", "message.activate.project": "Are you sure you want to activate this project?", "message.add.egress.rule.failed": "Adding new egress rule failed.", @@ -2704,6 +2715,7 @@ "message.desc.import.ext.kvm.wizard": "Import libvirt domain from External KVM Host not managed by CloudStack", "message.desc.import.local.kvm.wizard": "Import QCOW2 image from Local Storage of selected KVM Host", "message.desc.import.shared.kvm.wizard": "Import QCOW2 image from selected Primary Storage Pool", +"message.desc.import.unmanage.volume": "Please choose a storage pool that you want to import or unmanage volumes. This feature only supports KVM.", "message.desc.importexportinstancewizard": "By choosing to manage an Instance, CloudStack takes over the orchestration of that Instance. Unmanaging an Instance removes CloudStack ability to manage it. In both cases, the Instance is left running and no changes are done to the VM on the hypervisor.

For KVM, managing a VM is an experimental feature.", "message.desc.importmigratefromvmwarewizard": "By selecting an existing or external VMware Datacenter and an instance to import, CloudStack migrates the selected instance from VMware to KVM on a conversion host using virt-v2v and imports it into a KVM cluster", "message.desc.primary.storage": "Each cluster must contain one or more primary storage servers. We will add the first one now. Primary storage contains the disk volumes for all the Instances running on hosts in the cluster. Use any standards-compliant protocol that is supported by the underlying hypervisor.", @@ -3179,6 +3191,7 @@ "message.success.edit.rule": "Successfully edited rule", "message.success.enable.saml.auth": "Successfully enabled SAML Authorization", "message.success.import.instance": "Successfully imported Instance", +"message.success.import.volume": "Successfully imported Volume", "message.success.migrate.volume": "Successfully migrated volume", "message.success.migrating": "Migration completed successfully for", "message.success.migration": "Migration completed successfully", @@ -3210,6 +3223,7 @@ "message.success.resize.volume": "Successfully resized volume", "message.success.scale.kubernetes": "Successfully scaled Kubernetes cluster", "message.success.unmanage.instance": "Successfully unmanaged Instance", +"message.success.unmanage.volume": "Successfully unmanaged Volume", "message.success.update.bucket": "Successfully updated bucket", "message.success.update.condition": "Successfully updated condition", "message.success.update.ipaddress": "Successfully updated IP address", @@ -3322,6 +3336,8 @@ "message.volume.state.uploadinprogress": "Volume upload is in progress.", "message.volume.state.uploadop": "The volume upload operation is in progress and will be on secondary storage shortly.", "message.volume.state.primary.storage.suitability": "The suitability of a primary storage for a volume depends on the disk offering of the volume and on the virtual machine allocation (if the volume is attached to a virtual machine).", +"message.volumes.managed": "Volumes controlled by CloudStack.", +"message.volumes.unmanaged": "Volumes not controlled by CloudStack.", "message.vr.alert.upon.network.offering.creation.l2": "As virtual routers are not created for L2 Networks, the compute offering will not be used.", "message.vr.alert.upon.network.offering.creation.others": "As none of the obligatory services for creating a virtual router (VPN, DHCP, DNS, Firewall, LB, UserData, SourceNat, StaticNat, PortForwarding) are enabled, the virtual router will not be created and the compute offering will not be used.", "message.warn.filetype": "jpg, jpeg, png, bmp and svg are the only supported image formats.", diff --git a/ui/src/config/section/tools.js b/ui/src/config/section/tools.js index a04563c477a4..78439f648ce1 100644 --- a/ui/src/config/section/tools.js +++ b/ui/src/config/section/tools.js @@ -68,6 +68,15 @@ export default { resourceType: 'UserVm', permission: ['listInfrastructure', 'listUnmanagedInstances'], component: () => import('@/views/tools/ManageInstances.vue') + }, + { + name: 'managevolumes', + title: 'label.action.import.unmanage.volumes', + icon: 'interaction-outlined', + docHelp: 'adminguide/virtual_machines.html#importing-and-unmanaging-volume', + resourceType: 'UserVm', + permission: ['listInfrastructure', 'listVolumesForImport'], + component: () => import('@/views/tools/ManageVolumes.vue') } ] } diff --git a/ui/src/views/tools/ManageVolumes.vue b/ui/src/views/tools/ManageVolumes.vue new file mode 100644 index 000000000000..9651adeff408 --- /dev/null +++ b/ui/src/views/tools/ManageVolumes.vue @@ -0,0 +1,1099 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + From fd4b237f7f07eb4a19ec6b927ea766c790bb8549 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 25 Mar 2024 20:49:07 +0100 Subject: [PATCH 26/44] Fix PR 8808 against 4.19 --- .../storage/volume/VolumeImportUnmanageManagerImpl.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index 8df0372c99fc..7691a1ca3302 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -65,8 +65,7 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import org.apache.log4j.Logger; import javax.inject.Inject; import java.util.ArrayList; @@ -76,7 +75,7 @@ import java.util.Map; public class VolumeImportUnmanageManagerImpl implements VolumeImportUnmanageService { - protected Logger logger = LogManager.getLogger(VolumeImportUnmanageManagerImpl.class); + protected Logger logger = Logger.getLogger(VolumeImportUnmanageManagerImpl.class); private static final List volumeImportUnmanageSupportedHypervisors = Arrays.asList(Hypervisor.HypervisorType.KVM); From cd0e313f309d504a8d90344121f5b7b54c4cf0e2 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 26 Mar 2024 11:41:48 +0100 Subject: [PATCH 27/44] Update 8088: clean search keyword on UI when change zone/pod/cluster/host/pod --- ui/src/views/tools/ManageVolumes.vue | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ui/src/views/tools/ManageVolumes.vue b/ui/src/views/tools/ManageVolumes.vue index 9651adeff408..91d06fbf939b 100644 --- a/ui/src/views/tools/ManageVolumes.vue +++ b/ui/src/views/tools/ManageVolumes.vue @@ -766,6 +766,8 @@ export default { if (['zoneid', 'podid', 'clusterid', 'hostid', 'poolid'].includes(field)) { query.managedpage = 1 query.unmanagedpage = 1 + this.searchParams.managed.keyword = null + this.searchParams.unmanaged.keyword = null } this.$router.push({ query }) }, From 1750a32609520b35c8d2d17bb03b8fec3d1aef4f Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 27 Mar 2024 09:32:48 +0100 Subject: [PATCH 28/44] Update 8088: import volume for specified account on UI --- ui/public/locales/en.json | 1 + ui/src/views/tools/ManageVolumes.vue | 220 ++++++++++++++++++++++++++- 2 files changed, 214 insertions(+), 7 deletions(-) diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 4e9adb6d54d1..46175c5a521e 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -2922,6 +2922,7 @@ "message.host.dedicated": "Host Dedicated", "message.host.dedication.released": "Host dedication released.", "message.import.running.instance.warning": "The selected VM is powered-on on the VMware Datacenter. The recommended state to convert a VMware VM into KVM is powered-off after a graceful shutdown of the guest OS.", +"message.import.volume": "Please specify the domain, account or project name.
If not set, the volume will be imported for the caller.", "message.info.cloudian.console": "Cloudian Management Console should open in another window.", "message.installwizard.cloudstack.helptext.website": " * Project website:\t ", "message.infra.setup.tungsten.description": "This zone must contain a Tungsten-Fabric provider because the isolation method is TF", diff --git a/ui/src/views/tools/ManageVolumes.vue b/ui/src/views/tools/ManageVolumes.vue index 91d06fbf939b..cb7a1bc61c4f 100644 --- a/ui/src/views/tools/ManageVolumes.vue +++ b/ui/src/views/tools/ManageVolumes.vue @@ -326,9 +326,14 @@ :maskClosable="false" :footer="null" :cancelText="$t('label.cancel')" - @cancel="showImportForm = false" + @cancel="onCloseImportVolumeForm" centered width="auto"> + + + + + + + + {{ $t('label.account') }} + {{ $t('label.project') }} + + + + + + + + + + {{ domain.path || domain.name || domain.description }} + + + + + + + + + + + + {{ account.name }} + + + + {{ $t('label.required') }} + + + + + + + + + {{ project.name }} + + + + {{ $t('label.required') }} +
- {{ $t('label.cancel') }} + {{ $t('label.cancel') }} {{ $t('label.ok') }}
@@ -419,6 +513,9 @@ export default { } ] return { + domains: [], + accounts: [], + projects: [], options: { zones: [], pods: [], @@ -485,6 +582,7 @@ export default { this.page.managed = parseInt(this.$route.query.managedpage || 1) this.initForm() this.fetchData() + this.fetchDomains() }, computed: { isPageAllowed () { @@ -842,6 +940,65 @@ export default { this.updateQuery('scope', value) this.fetchOptions(this.params.zones, 'zones', value) }, + fetchDomains () { + api('listDomains', { + response: 'json', + listAll: true, + showicon: true, + details: 'min' + }).then(response => { + this.domains = response.listdomainsresponse.domain || [] + }).catch(error => { + this.$notifyError(error) + }).finally(() => { + this.loading = false + }) + }, + fetchAccounts () { + this.loading = true + api('listAccounts', { + response: 'json', + domainId: this.importForm.selectedDomain, + showicon: true, + state: 'Enabled', + isrecursive: false + }).then(response => { + this.accounts = response.listaccountsresponse.account || [] + }).catch(error => { + this.$notifyError(error) + }).finally(() => { + this.loading = false + }) + }, + fetchProjects () { + this.loading = true + api('listProjects', { + response: 'json', + domainId: this.importForm.selectedDomain, + state: 'Active', + showicon: true, + details: 'min', + isrecursive: false + }).then(response => { + this.projects = response.listprojectsresponse.project || [] + }).catch(error => { + this.$notifyError(error) + }).finally(() => { + this.loading = false + }) + }, + changeDomain () { + this.importForm.selectedAccount = null + this.importForm.selectedProject = null + this.fetchAccounts() + this.fetchProjects() + }, + changeAccount () { + this.importForm.selectedProject = null + }, + changeProject () { + this.importForm.selectedAccount = null + }, fetchVolumes () { this.fetchUnmanagedVolumes() this.fetchManagedVolumes() @@ -961,7 +1118,28 @@ export default { } this.values = toRaw(this.importForm) const volumeName = this.selectedUnmanagedVolume.name + + let variableKey = '' + let variableValue = '' + if (this.values.selectedAccountType === 'Account') { + if (!this.values.selectedAccount) { + this.importForm.accountError = true + return + } + variableKey = 'account' + variableValue = this.values.selectedAccount + } else if (this.values.selectedAccountType === 'Project') { + if (!this.values.selectedProject) { + this.importForm.projectError = true + return + } + variableKey = 'projectid' + variableValue = this.values.selectedProject + } + var params = { + domainid: this.importForm.selectedDomain, + [variableKey]: variableValue, storageid: this.poolId, path: this.selectedUnmanagedVolume.path, name: this.values.name @@ -985,7 +1163,14 @@ export default { this.loading = false }) this.selectedUnmanagedVolume = null + this.onCloseImportVolumeForm() + }, + onCloseImportVolumeForm () { this.showImportForm = false + this.importForm.selectedAccountType = null + this.importForm.selectedDomain = null + this.importForm.selectedAccount = null + this.importForm.selectedProject = null }, onUnmanageVolumeAction () { const self = this @@ -1044,11 +1229,32 @@ export default { } .import-form { - width: 80vw; + width: 85vw; + + @media (min-width: 760px) { + width: 500px; + } + + display: flex; + flex-direction: column; - @media (min-width: 500px) { - min-width: 400px; + &__item { + display: flex; + flex-direction: column; width: 100%; + margin-bottom: 10px; + } + + &__label { + display: flex; + font-weight: bold; + margin-bottom: 5px; + } + + .required { + margin-right: 2px; + color: red; + font-weight: bold; } } .volumes-card { From 0a96796385888cb6253c6c5a6cb4da84d2ad251b Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 27 Mar 2024 11:48:51 +0100 Subject: [PATCH 29/44] Update 8088: add validateCustomDiskOfferingSizeRange --- .../storage/volume/VolumeImportUnmanageManagerImpl.java | 9 ++++++++- .../volume/VolumeImportUnmanageManagerImplTest.java | 5 +++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index 7691a1ca3302..d53a9a35319b 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -37,6 +37,7 @@ import com.cloud.storage.Storage; import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.Volume; +import com.cloud.storage.VolumeApiService; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.StoragePoolHostDao; @@ -63,6 +64,7 @@ import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; @@ -106,6 +108,8 @@ public class VolumeImportUnmanageManagerImpl implements VolumeImportUnmanageServ private VolumeOrchestrationService volumeManager; @Inject private VMTemplatePoolDao templatePoolDao; + @Inject + private VolumeApiService volumeApiService; static final String DEFAULT_DISK_OFFERING_NAME = "Default Custom Offering for Volume Import"; static final String DEFAULT_DISK_OFFERING_UNIQUE_NAME = "Volume-Import"; @@ -186,7 +190,7 @@ public VolumeResponse importVolume(ImportVolumeCmd cmd) { VolumeOnStorageTO volume = volumes.get(0); - // check if volume is locked + // check if volume is locked, encrypted or volume size is allowed checkIfVolumeIsLocked(volume); checkIfVolumeIsEncrypted(volume); @@ -195,6 +199,9 @@ public VolumeResponse importVolume(ImportVolumeCmd cmd) { // 6. get disk offering DiskOfferingVO diskOffering = getOrCreateDiskOffering(owner, cmd.getDiskOfferingId(), pool.getDataCenterId(), pool.isLocal()); + if (diskOffering.isCustomized()) { + volumeApiService.validateCustomDiskOfferingSizeRange(volume.getVirtualSize() / ByteScaleUtils.GiB); + } // 7. create records String volumeName = StringUtils.isNotBlank(cmd.getName()) ? cmd.getName().trim() : volumePath; diff --git a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java index f843275ef230..7f08c6e0fb81 100644 --- a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java @@ -35,6 +35,7 @@ import com.cloud.storage.Storage; import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.Volume; +import com.cloud.storage.VolumeApiService; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.StoragePoolHostDao; @@ -125,6 +126,8 @@ public class VolumeImportUnmanageManagerImplTest { private VolumeOrchestrationService volumeManager; @Mock private VMTemplatePoolDao templatePoolDao; + @Mock + private VolumeApiService volumeApiService; @Mock StoragePoolVO storagePoolVO; @@ -260,7 +263,9 @@ public void testImportVolumeAllGood() throws ResourceAllocationException { doNothing().when(resourceLimitService).checkResourceLimit(account, Resource.ResourceType.primary_storage, virtualSize); DiskOfferingVO diskOffering = mock(DiskOfferingVO.class); + when(diskOffering.isCustomized()).thenReturn(true); doReturn(diskOffering).when(volumeImportUnmanageManager).getOrCreateDiskOffering(account, diskOfferingId, zoneId, isLocal); + doNothing().when(volumeApiService).validateCustomDiskOfferingSizeRange(anyLong()); doReturn(diskProfile).when(volumeManager).importVolume(any(), anyString(), any(), eq(virtualSize), isNull(), isNull(), anyLong(), any(), isNull(), isNull(), any(), isNull(), anyLong(), anyString(), isNull()); when(diskProfile.getVolumeId()).thenReturn(volumeId); From d16a7ed156686c2a5de023d58020335025de32e3 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 27 Mar 2024 12:11:16 +0100 Subject: [PATCH 30/44] Update 8088: do not import volume with backing file --- .../VolumeImportUnmanageManagerImpl.java | 13 +++++- .../VolumeImportUnmanageManagerImplTest.java | 45 ++++++++++++++----- 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index d53a9a35319b..2eff36b8068b 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -193,6 +193,7 @@ public VolumeResponse importVolume(ImportVolumeCmd cmd) { // check if volume is locked, encrypted or volume size is allowed checkIfVolumeIsLocked(volume); checkIfVolumeIsEncrypted(volume); + checkIfVolumeHasBackingFile(volume); // 5. check resource limitation checkResourceLimitForImportVolume(owner, volume); @@ -344,7 +345,17 @@ protected void checkIfVolumeIsEncrypted(VolumeOnStorageTO volume) { if (volumeDetails != null && volumeDetails.containsKey(VolumeOnStorageTO.Detail.IS_ENCRYPTED)) { String isEncrypted = volumeDetails.get(VolumeOnStorageTO.Detail.IS_ENCRYPTED); if (Boolean.parseBoolean(isEncrypted)) { - logFailureAndThrowException("Encrypted volume cannot be imported for now."); + logFailureAndThrowException("Encrypted volume cannot be imported."); + } + } + } + + protected void checkIfVolumeHasBackingFile(VolumeOnStorageTO volume) { + Map volumeDetails = volume.getDetails(); + if (volumeDetails != null && volumeDetails.containsKey(VolumeOnStorageTO.Detail.BACKING_FILE)) { + String backingFile = volumeDetails.get(VolumeOnStorageTO.Detail.BACKING_FILE); + if (StringUtils.isNotBlank(backingFile)) { + logFailureAndThrowException("Volume with backing file cannot be imported."); } } } diff --git a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java index 7f08c6e0fb81..28708c23722f 100644 --- a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java @@ -258,6 +258,7 @@ public void testImportVolumeAllGood() throws ResourceAllocationException { doNothing().when(volumeImportUnmanageManager).checkIfVolumeIsLocked(volumeOnStorageTO); doNothing().when(volumeImportUnmanageManager).checkIfVolumeIsEncrypted(volumeOnStorageTO); + doNothing().when(volumeImportUnmanageManager).checkIfVolumeHasBackingFile(volumeOnStorageTO); doNothing().when(resourceLimitService).checkResourceLimit(account, Resource.ResourceType.volume); doNothing().when(resourceLimitService).checkResourceLimit(account, Resource.ResourceType.primary_storage, virtualSize); @@ -302,20 +303,44 @@ public void testListVolumesForImportInternal() { Assert.assertEquals(volumesOnStorageTO, result); } - @Test(expected = CloudRuntimeException.class) + @Test public void testCheckIfVolumeIsLocked() { - VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(hypervisorType, path, name, fullPath, - format, size, virtualSize); - volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.IS_LOCKED, "true"); - volumeImportUnmanageManager.checkIfVolumeIsLocked(volumeOnStorageTO); + try { + VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(hypervisorType, path, name, fullPath, + format, size, virtualSize); + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.IS_LOCKED, "true"); + volumeImportUnmanageManager.checkIfVolumeIsLocked(volumeOnStorageTO); + Assert.fail("It should fail as the volume is locked"); + } catch (CloudRuntimeException ex) { + Assert.assertEquals("Locked volume cannot be imported.", ex.getMessage()); + } } - @Test(expected = CloudRuntimeException.class) + @Test public void testCheckIfVolumeIsEncrypted() { - VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(hypervisorType, path, name, fullPath, - format, size, virtualSize); - volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.IS_ENCRYPTED, "true"); - volumeImportUnmanageManager.checkIfVolumeIsEncrypted(volumeOnStorageTO); + try { + VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(hypervisorType, path, name, fullPath, + format, size, virtualSize); + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.IS_ENCRYPTED, "true"); + volumeImportUnmanageManager.checkIfVolumeIsEncrypted(volumeOnStorageTO); + Assert.fail("It should fail as the volume is encrypted"); + } catch (CloudRuntimeException ex) { + Assert.assertEquals("Encrypted volume cannot be imported.", ex.getMessage()); + } + } + + @Test + public void testCheckIfVolumeHasBackingFile() { + try { + VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(hypervisorType, path, name, fullPath, + format, size, virtualSize); + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.BACKING_FILE, BACKING_FILE); + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.BACKING_FILE_FORMAT, BACKING_FILE_FORMAT); + volumeImportUnmanageManager.checkIfVolumeHasBackingFile(volumeOnStorageTO); + Assert.fail("It should fail as the volume has backing file"); + } catch (CloudRuntimeException ex) { + Assert.assertEquals("Volume with backing file cannot be imported.", ex.getMessage()); + } } @Test From 42cd52900049db91ee5fa80a4eaf1d9ec4955d4d Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 27 Mar 2024 13:15:50 +0100 Subject: [PATCH 31/44] Update 8088: do not import volumes with different file format as supported --- .../wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java index 67da52069f1b..128d625df333 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java @@ -100,7 +100,10 @@ private GetVolumesOnStorageAnswer addVolumeByVolumePath(final GetVolumesOnStorag volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.CLUSTER_SIZE, clusterSize); } String fileFormat = info.get(QemuImg.FILE_FORMAT); - if (StringUtils.isNotBlank(clusterSize)) { + if (StringUtils.isNotBlank(fileFormat)) { + if (!fileFormat.equalsIgnoreCase(disk.getFormat().toString())) { + return new GetVolumesOnStorageAnswer(command, false, String.format("The file format is %s, but expected to be %s", fileFormat, disk.getFormat())); + } volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.FILE_FORMAT, fileFormat); } String encrypted = info.get(QemuImg.ENCRYPTED); From 8b06f57ac2a46d1c0538759119fd03b92c8931fe Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 27 Mar 2024 13:16:04 +0100 Subject: [PATCH 32/44] Update 8088: check if volume is a reference of snapshot --- .../volume/VolumeImportUnmanageManagerImpl.java | 16 +++++++++++++++- .../VolumeImportUnmanageManagerImplTest.java | 5 +++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index 2eff36b8068b..0b0851283fc9 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -33,6 +33,7 @@ import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.offering.DiskOffering; +import com.cloud.storage.DataStoreRole; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.Storage; import com.cloud.storage.StoragePoolHostVO; @@ -63,6 +64,7 @@ import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.commons.collections.CollectionUtils; @@ -110,6 +112,8 @@ public class VolumeImportUnmanageManagerImpl implements VolumeImportUnmanageServ private VMTemplatePoolDao templatePoolDao; @Inject private VolumeApiService volumeApiService; + @Inject + private SnapshotDataStoreDao snapshotDataStoreDao; static final String DEFAULT_DISK_OFFERING_NAME = "Default Custom Offering for Volume Import"; static final String DEFAULT_DISK_OFFERING_UNIQUE_NAME = "Volume-Import"; @@ -144,7 +148,9 @@ public ListResponse listVolumesForImport(ListVolumesFor List responses = new ArrayList<>(); for (VolumeOnStorageTO volume : volumes) { - if (checkIfVolumeManaged(pool, volume.getPath()) || checkIfVolumeForTemplate(pool, volume.getPath())) { + if (checkIfVolumeManaged(pool, volume.getPath()) + || checkIfVolumeForTemplate(pool, volume.getPath()) + || checkIfVolumeForSnapshot(pool, volume.getFullPath())) { continue; } responses.add(createVolumeForImportResponse(volume, pool)); @@ -194,6 +200,9 @@ public VolumeResponse importVolume(ImportVolumeCmd cmd) { checkIfVolumeIsLocked(volume); checkIfVolumeIsEncrypted(volume); checkIfVolumeHasBackingFile(volume); + if (checkIfVolumeForSnapshot(pool, volume.getFullPath())) { + logFailureAndThrowException("Volume is a reference of snapshot on primary: " + volume.getFullPath()); + } // 5. check resource limitation checkResourceLimitForImportVolume(owner, volume); @@ -330,6 +339,11 @@ private boolean checkIfVolumeForTemplate(StoragePoolVO pool, String volumePath) return templatePoolDao.findByPoolPath(pool.getId(), volumePath) != null; } + private boolean checkIfVolumeForSnapshot(StoragePoolVO pool, String fullVolumePath) { + List absPathList = Arrays.asList(fullVolumePath); + return CollectionUtils.isNotEmpty(snapshotDataStoreDao.listByStoreAndInstallPaths(pool.getId(), DataStoreRole.Primary, absPathList)); + } + protected void checkIfVolumeIsLocked(VolumeOnStorageTO volume) { Map volumeDetails = volume.getDetails(); if (volumeDetails != null && volumeDetails.containsKey(VolumeOnStorageTO.Detail.IS_LOCKED)) { diff --git a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java index 28708c23722f..11cee8b16a57 100644 --- a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java @@ -30,6 +30,7 @@ import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.offering.DiskOffering; +import com.cloud.storage.DataStoreRole; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.ScopeType; import com.cloud.storage.Storage; @@ -60,6 +61,7 @@ import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.junit.Assert; import org.junit.Before; @@ -128,6 +130,8 @@ public class VolumeImportUnmanageManagerImplTest { private VMTemplatePoolDao templatePoolDao; @Mock private VolumeApiService volumeApiService; + @Mock + private SnapshotDataStoreDao snapshotDataStoreDao; @Mock StoragePoolVO storagePoolVO; @@ -215,6 +219,7 @@ public void testListVolumesForImport() { when(volumeDao.findByPoolIdAndPath(poolId, path)).thenReturn(null); when(templatePoolDao.findByPoolPath(poolId, path)).thenReturn(null); + when(snapshotDataStoreDao.listByStoreAndInstallPaths(eq(poolId), eq(DataStoreRole.Primary), any())).thenReturn(null); VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(hypervisorType, path, name, fullPath, format, size, virtualSize); From a619041cf1a49b7b160e84d47292e3bfc0e3a030 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 28 Mar 2024 11:42:15 +0100 Subject: [PATCH 33/44] Update 8088: publish event when import/unmanage is completed --- .../storage/volume/VolumeImportUnmanageManagerImpl.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index 0b0851283fc9..157e3ea478d4 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -25,7 +25,9 @@ import com.cloud.configuration.ConfigurationManager; import com.cloud.configuration.Resource; import com.cloud.dc.dao.DataCenterDao; +import com.cloud.event.ActionEventUtils; import com.cloud.event.EventTypes; +import com.cloud.event.EventVO; import com.cloud.event.UsageEventUtils; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceAllocationException; @@ -47,10 +49,12 @@ import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.ResourceLimitService; +import com.cloud.user.User; import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.DiskProfile; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.ResponseGenerator; import org.apache.cloudstack.api.ResponseObject; @@ -446,11 +450,14 @@ private void updateResourceLimitForVolumeImport(VolumeVO volumeVO) { private void publicUsageEventForVolumeImportAndUnmanage(VolumeVO volumeVO, boolean isImport) { try { String eventType = isImport ? EventTypes.EVENT_VOLUME_IMPORT: EventTypes.EVENT_VOLUME_UNMANAGE; + String eventDescription = isImport ? "Successfully imported volume " + volumeVO.getUuid(): "Successfully unmanaged volume " + volumeVO.getUuid(); + ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, volumeVO.getAccountId(), EventVO.LEVEL_INFO, + eventType, eventDescription, volumeVO.getId(), ApiCommandResourceType.Volume.toString(),0); UsageEventUtils.publishUsageEvent(eventType, volumeVO.getAccountId(), volumeVO.getDataCenterId(), volumeVO.getId(), volumeVO.getName(), volumeVO.getDiskOfferingId(), null, volumeVO.getSize(), Volume.class.getName(), volumeVO.getUuid(), volumeVO.isDisplayVolume()); } catch (Exception e) { - logger.error(String.format("Failed to publish volume ID: %s usage records during volume import/unmanage", volumeVO.getUuid()), e); + logger.error(String.format("Failed to publish volume ID: %s event or usage records during volume import/unmanage", volumeVO.getUuid()), e); } } From 37e3436f2359a36c7df997902a98a4afabc5f000 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 28 Mar 2024 13:28:34 +0100 Subject: [PATCH 34/44] Update 8088: change label --- ui/public/locales/en.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 46175c5a521e..31133ac6a9ec 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -138,7 +138,7 @@ "label.action.image.store.read.only": "Make image store read-only", "label.action.image.store.read.write": "Make image store read-write", "label.action.import.export.instances": "Import-Export Instances", -"label.action.import.unmanage.volumes": "Import-Unmanage Volumes", +"label.action.import.unmanage.volumes": "Import Data Volumes", "label.action.ingest.instances": "Ingest instances", "label.action.iso.permission": "Update ISO permissions", "label.action.iso.share": "Update ISO sharing", From fb672cc4f1fad63f42de4f01bcf7a53c86e0f1ed Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 28 Mar 2024 16:54:15 +0100 Subject: [PATCH 35/44] Update 8088: add MockedStatic --- .../volume/VolumeImportUnmanageManagerImplTest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java index 11cee8b16a57..3700ccd05421 100644 --- a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java @@ -23,6 +23,7 @@ import com.cloud.configuration.Resource; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; +import com.cloud.event.ActionEventUtils; import com.cloud.event.UsageEventUtils; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceAllocationException; @@ -282,7 +283,8 @@ public void testImportVolumeAllGood() throws ResourceAllocationException { VolumeResponse response = mock(VolumeResponse.class); doReturn(response).when(responseGenerator).createVolumeResponse(ResponseObject.ResponseView.Full, volumeVO); - try (MockedStatic ignored = Mockito.mockStatic(UsageEventUtils.class)) { + try (MockedStatic ignored = Mockito.mockStatic(UsageEventUtils.class); + MockedStatic ignoredtoo = Mockito.mockStatic(ActionEventUtils.class)) { VolumeResponse result = volumeImportUnmanageManager.importVolume(cmd); Assert.assertEquals(response, result); } @@ -356,7 +358,8 @@ public void testUnmanageVolume() { doNothing().when(resourceLimitService).decrementResourceCount(accountId, Resource.ResourceType.volume); doNothing().when(resourceLimitService).decrementResourceCount(accountId, Resource.ResourceType.primary_storage, virtualSize); - try (MockedStatic ignored = Mockito.mockStatic(UsageEventUtils.class)) { + try (MockedStatic ignored = Mockito.mockStatic(UsageEventUtils.class); + MockedStatic ignoredtoo = Mockito.mockStatic(ActionEventUtils.class)) { volumeImportUnmanageManager.unmanageVolume(volumeId); } From 1092bbcd0989961e08dc7b7fadc8b08e7e319915 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 28 Mar 2024 20:31:58 +0100 Subject: [PATCH 36/44] Update 8088: refactor VolumeImportUnmanageManagerImplTest --- .../VolumeImportUnmanageManagerImpl.java | 4 +- .../VolumeImportUnmanageManagerImplTest.java | 132 ++++++++++++------ 2 files changed, 94 insertions(+), 42 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index 157e3ea478d4..7bdc82992230 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -274,10 +274,10 @@ public boolean unmanageVolume(long volumeId) { protected StoragePoolVO checkIfPoolAvailable(Long poolId) { StoragePoolVO pool = primaryDataStoreDao.findById(poolId); if (pool == null) { - logFailureAndThrowException("Storage pool does not exist: ID = " + poolId); + logFailureAndThrowException(String.format("Storage pool (ID: %s) does not exist", poolId)); } if (pool.isInMaintenance()) { - logFailureAndThrowException("Storage pool is in maintenance: " + pool.getName()); + logFailureAndThrowException(String.format("Storage pool (name: %s) is in maintenance: ", pool.getName())); } return pool; } diff --git a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java index 3700ccd05421..8c6196c1d6a5 100644 --- a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java @@ -320,6 +320,7 @@ public void testCheckIfVolumeIsLocked() { Assert.fail("It should fail as the volume is locked"); } catch (CloudRuntimeException ex) { Assert.assertEquals("Locked volume cannot be imported.", ex.getMessage()); + verify(volumeImportUnmanageManager).logFailureAndThrowException("Locked volume cannot be imported."); } } @@ -333,6 +334,7 @@ public void testCheckIfVolumeIsEncrypted() { Assert.fail("It should fail as the volume is encrypted"); } catch (CloudRuntimeException ex) { Assert.assertEquals("Encrypted volume cannot be imported.", ex.getMessage()); + verify(volumeImportUnmanageManager).logFailureAndThrowException("Encrypted volume cannot be imported."); } } @@ -347,6 +349,7 @@ public void testCheckIfVolumeHasBackingFile() { Assert.fail("It should fail as the volume has backing file"); } catch (CloudRuntimeException ex) { Assert.assertEquals("Volume with backing file cannot be imported.", ex.getMessage()); + verify(volumeImportUnmanageManager).logFailureAndThrowException("Volume with backing file cannot be imported."); } } @@ -368,44 +371,74 @@ public void testUnmanageVolume() { verify(volumeDao).update(eq(volumeId), any()); } - @Test(expected = CloudRuntimeException.class) + @Test public void testUnmanageVolumeNotExist() { - when(volumeDao.findById(volumeId)).thenReturn(null); - volumeImportUnmanageManager.unmanageVolume(volumeId); + try { + when(volumeDao.findById(volumeId)).thenReturn(null); + volumeImportUnmanageManager.unmanageVolume(volumeId); + Assert.fail("it should fail"); + } catch (CloudRuntimeException ex) { + verify(volumeImportUnmanageManager).logFailureAndThrowException(String.format("Volume (ID: %s) does not exist", volumeId)); + } } - @Test(expected = CloudRuntimeException.class) + @Test public void testUnmanageVolumeNotReady() { - when(volumeVO.getState()).thenReturn(Volume.State.Allocated); - volumeImportUnmanageManager.unmanageVolume(volumeId); + try { + when(volumeVO.getState()).thenReturn(Volume.State.Allocated); + volumeImportUnmanageManager.unmanageVolume(volumeId); + Assert.fail("it should fail"); + } catch (CloudRuntimeException ex) { + verify(volumeImportUnmanageManager).logFailureAndThrowException(String.format("Volume (ID: %s) is not ready", volumeId)); + } } - @Test(expected = CloudRuntimeException.class) + @Test public void testUnmanageVolumeEncrypted() { - when(volumeVO.getState()).thenReturn(Volume.State.Ready); - when(volumeVO.getEncryptFormat()).thenReturn(encryptFormat); - volumeImportUnmanageManager.unmanageVolume(volumeId); + try { + when(volumeVO.getState()).thenReturn(Volume.State.Ready); + when(volumeVO.getEncryptFormat()).thenReturn(encryptFormat); + volumeImportUnmanageManager.unmanageVolume(volumeId); + Assert.fail("it should fail"); + } catch (CloudRuntimeException ex) { + verify(volumeImportUnmanageManager).logFailureAndThrowException(String.format("Volume (ID: %s) is encrypted", volumeId)); + } } - @Test(expected = CloudRuntimeException.class) + @Test public void testUnmanageVolumeAttached() { - when(volumeVO.getState()).thenReturn(Volume.State.Ready); - when(volumeVO.getAttached()).thenReturn(new Date()); - volumeImportUnmanageManager.unmanageVolume(volumeId); + try { + when(volumeVO.getState()).thenReturn(Volume.State.Ready); + when(volumeVO.getAttached()).thenReturn(new Date()); + volumeImportUnmanageManager.unmanageVolume(volumeId); + Assert.fail("it should fail"); + } catch (CloudRuntimeException ex) { + verify(volumeImportUnmanageManager).logFailureAndThrowException(String.format("Volume (ID: %s) is attached to VM (ID: %s)", volumeId, volumeVO.getInstanceId())); + } } - @Test(expected = CloudRuntimeException.class) + @Test public void testCheckIfPoolAvailableNotExist() { - when(primaryDataStoreDao.findById(poolId)).thenReturn(null); - volumeImportUnmanageManager.checkIfPoolAvailable(poolId); + try { + when(primaryDataStoreDao.findById(poolId)).thenReturn(null); + volumeImportUnmanageManager.checkIfPoolAvailable(poolId); + Assert.fail("it should fail"); + } catch (CloudRuntimeException ex) { + verify(volumeImportUnmanageManager).logFailureAndThrowException(String.format("Storage pool (ID: %s) does not exist", poolId)); + } } - @Test(expected = CloudRuntimeException.class) + @Test public void testCheckIfPoolAvailableInMaintenance() { - when(primaryDataStoreDao.findById(poolId)).thenReturn(storagePoolVO); - when(storagePoolVO.isInMaintenance()).thenReturn(true); - volumeImportUnmanageManager.checkIfPoolAvailable(poolId); + try { + when(primaryDataStoreDao.findById(poolId)).thenReturn(storagePoolVO); + when(storagePoolVO.isInMaintenance()).thenReturn(true); + volumeImportUnmanageManager.checkIfPoolAvailable(poolId); + Assert.fail("it should fail"); + } catch (CloudRuntimeException ex) { + verify(volumeImportUnmanageManager).logFailureAndThrowException(String.format("Storage pool (name: %s) is in maintenance: ", storagePoolName)); + } } @Test @@ -461,38 +494,57 @@ public void testGetOrCreateDiskOfferingAllGood() { Assert.assertEquals(diskOfferingVO, result); } - @Test(expected = CloudRuntimeException.class) + @Test public void testGetOrCreateDiskOfferingNotExist() { - when(diskOfferingDao.findById(diskOfferingId)).thenReturn(null); - - volumeImportUnmanageManager.getOrCreateDiskOffering(account, diskOfferingId, zoneId, isLocal); + try { + when(diskOfferingDao.findById(diskOfferingId)).thenReturn(null); + volumeImportUnmanageManager.getOrCreateDiskOffering(account, diskOfferingId, zoneId, isLocal); + Assert.fail("it should fail"); + } catch (CloudRuntimeException ex) { + verify(volumeImportUnmanageManager).logFailureAndThrowException(String.format("Disk offering %s does not exist", diskOfferingId)); + } } - @Test(expected = CloudRuntimeException.class) + @Test public void testGetOrCreateDiskOfferingNotActive() { - when(diskOfferingDao.findById(diskOfferingId)).thenReturn(diskOfferingVO); - when(diskOfferingVO.getState()).thenReturn(DiskOffering.State.Inactive); + try { + when(diskOfferingDao.findById(diskOfferingId)).thenReturn(diskOfferingVO); + when(diskOfferingVO.getState()).thenReturn(DiskOffering.State.Inactive); - volumeImportUnmanageManager.getOrCreateDiskOffering(account, diskOfferingId, zoneId, isLocal); + volumeImportUnmanageManager.getOrCreateDiskOffering(account, diskOfferingId, zoneId, isLocal); + Assert.fail("it should fail"); + } catch (CloudRuntimeException ex) { + verify(volumeImportUnmanageManager).logFailureAndThrowException(String.format("Disk offering with ID %s is not active", diskOfferingId)); + } } - @Test(expected = CloudRuntimeException.class) + @Test public void testGetOrCreateDiskOfferingNotLocal() { - when(diskOfferingDao.findById(diskOfferingId)).thenReturn(diskOfferingVO); - when(diskOfferingVO.getState()).thenReturn(DiskOffering.State.Active); - when(diskOfferingVO.isUseLocalStorage()).thenReturn(!isLocal); + try { + when(diskOfferingDao.findById(diskOfferingId)).thenReturn(diskOfferingVO); + when(diskOfferingVO.getState()).thenReturn(DiskOffering.State.Active); + when(diskOfferingVO.isUseLocalStorage()).thenReturn(!isLocal); - volumeImportUnmanageManager.getOrCreateDiskOffering(account, diskOfferingId, zoneId, isLocal); + volumeImportUnmanageManager.getOrCreateDiskOffering(account, diskOfferingId, zoneId, isLocal); + Assert.fail("it should fail"); + } catch (CloudRuntimeException ex) { + verify(volumeImportUnmanageManager).logFailureAndThrowException(String.format("Disk offering with ID %s should use %s storage", diskOfferingId, isLocal ? "local" : "shared")); + } } - @Test(expected = CloudRuntimeException.class) + @Test public void testGetOrCreateDiskOfferingNoPermission() { - when(diskOfferingDao.findById(diskOfferingId)).thenReturn(diskOfferingVO); - when(diskOfferingVO.getState()).thenReturn(DiskOffering.State.Active); - when(diskOfferingVO.isUseLocalStorage()).thenReturn(isLocal); - doThrow(PermissionDeniedException.class).when(configMgr).checkDiskOfferingAccess(account, diskOfferingVO, dataCenterVO); + try { + when(diskOfferingDao.findById(diskOfferingId)).thenReturn(diskOfferingVO); + when(diskOfferingVO.getState()).thenReturn(DiskOffering.State.Active); + when(diskOfferingVO.isUseLocalStorage()).thenReturn(isLocal); + doThrow(PermissionDeniedException.class).when(configMgr).checkDiskOfferingAccess(account, diskOfferingVO, dataCenterVO); - volumeImportUnmanageManager.getOrCreateDiskOffering(account, diskOfferingId, zoneId, isLocal); + volumeImportUnmanageManager.getOrCreateDiskOffering(account, diskOfferingId, zoneId, isLocal); + Assert.fail("it should fail"); + } catch (CloudRuntimeException ex) { + verify(volumeImportUnmanageManager).logFailureAndThrowException(String.format("Disk offering with ID %s is not accessible by owner %s", diskOfferingId, account)); + } } @Test From dff5fd859880080f12429d20ee9ae9ea8091d22c Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 2 Apr 2024 11:28:06 +0200 Subject: [PATCH 37/44] Update 8088: select disk offering on UI --- ui/src/views/tools/ManageVolumes.vue | 67 ++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/ui/src/views/tools/ManageVolumes.vue b/ui/src/views/tools/ManageVolumes.vue index cb7a1bc61c4f..2ec713c3b433 100644 --- a/ui/src/views/tools/ManageVolumes.vue +++ b/ui/src/views/tools/ManageVolumes.vue @@ -353,6 +353,7 @@ ref="accounttype" :label="$t('label.accounttype')"> {{ $t('label.required') }} + + + + + {{ offering.displaytext || offering.name }} + + + +
{{ $t('label.cancel') }} {{ $t('label.ok') }} @@ -516,6 +540,7 @@ export default { domains: [], accounts: [], projects: [], + diskOfferings: [], options: { zones: [], pods: [], @@ -987,17 +1012,57 @@ export default { this.loading = false }) }, + changeAccountType () { + this.importForm.selectedDomain = null + this.importForm.selectedAccount = null + this.importForm.selectedProject = null + this.importForm.selectedDiskoffering = null + this.diskOfferings = {} + }, changeDomain () { this.importForm.selectedAccount = null this.importForm.selectedProject = null + this.importForm.selectedDiskoffering = null + this.diskOfferings = {} this.fetchAccounts() this.fetchProjects() }, changeAccount () { this.importForm.selectedProject = null + this.importForm.selectedDiskoffering = null + this.diskOfferings = {} + this.fetchDiskOfferings() }, changeProject () { this.importForm.selectedAccount = null + this.importForm.selectedDiskoffering = null + this.diskOfferings = {} + this.fetchDiskOfferings() + }, + fetchDiskOfferings () { + this.loading = true + const selectedPool = this.options.pools.filter(pool => pool.id === this.poolId) + const storagetype = selectedPool[0].scope === 'HOST' ? 'local' : 'shared' + var params = { + zoneid: this.zoneId, + storageid: this.poolId, + storagetype: storagetype, + encrypt: false, + listall: true + } + if (this.importForm.selectedAccountType === 'Account') { + params.domainid = this.importForm.selectedDomain + params.account = this.importForm.selectedAccount + } else if (this.importForm.selectedAccountType === 'Project') { + params.domainid = this.importForm.selectedDomain + params.projectid = this.importForm.selectedProject + } + + api('listDiskOfferings', params).then(json => { + this.diskOfferings = json.listdiskofferingsresponse.diskoffering || [] + }).finally(() => { + this.loading = false + }) }, fetchVolumes () { this.fetchUnmanagedVolumes() @@ -1110,6 +1175,7 @@ export default { this.selectedUnmanagedVolume = this.unmanagedVolumes[this.unmanagedVolumesSelectedRowKeys[0]] this.importForm.name = this.selectedUnmanagedVolume.name } + this.fetchDiskOfferings() this.showImportForm = true }, handleSubmitImportVolumeForm () { @@ -1138,6 +1204,7 @@ export default { } var params = { + diskofferingid: this.importForm.selectedDiskoffering, domainid: this.importForm.selectedDomain, [variableKey]: variableValue, storageid: this.poolId, From e0b30f01cfa456e33f4ed77cb023e136b077da3b Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 2 Apr 2024 09:56:48 +0200 Subject: [PATCH 38/44] Update 8808: Disk offering should not support volume encryption --- .../volume/VolumeImportUnmanageManagerImpl.java | 3 +++ .../VolumeImportUnmanageManagerImplTest.java | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index 7bdc82992230..1b87538ef7df 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -391,6 +391,9 @@ protected DiskOfferingVO getOrCreateDiskOffering(Account owner, Long diskOfferin if (diskOfferingVO.isUseLocalStorage() != isLocal) { logFailureAndThrowException(String.format("Disk offering with ID %s should use %s storage", diskOfferingId, isLocal ? "local": "shared")); } + if (diskOfferingVO.getEncrypt()) { + logFailureAndThrowException(String.format("Disk offering with ID %s should not support volume encryption", diskOfferingId)); + } // check if disk offering is accessible by the account/owner try { configMgr.checkDiskOfferingAccess(owner, diskOfferingVO, dcDao.findById(zoneId)); diff --git a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java index 8c6196c1d6a5..a80a02cb986f 100644 --- a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java @@ -488,6 +488,7 @@ public void testGetOrCreateDiskOfferingAllGood() { when(diskOfferingDao.findById(diskOfferingId)).thenReturn(diskOfferingVO); when(diskOfferingVO.getState()).thenReturn(DiskOffering.State.Active); when(diskOfferingVO.isUseLocalStorage()).thenReturn(isLocal); + when(diskOfferingVO.getEncrypt()).thenReturn(false); doNothing().when(configMgr).checkDiskOfferingAccess(account, diskOfferingVO, dataCenterVO); DiskOfferingVO result = volumeImportUnmanageManager.getOrCreateDiskOffering(account, diskOfferingId, zoneId, isLocal); @@ -532,6 +533,21 @@ public void testGetOrCreateDiskOfferingNotLocal() { } } + @Test + public void testGetOrCreateDiskOfferingForVolumeEncryption() { + try { + when(diskOfferingDao.findById(diskOfferingId)).thenReturn(diskOfferingVO); + when(diskOfferingVO.getState()).thenReturn(DiskOffering.State.Active); + when(diskOfferingVO.isUseLocalStorage()).thenReturn(isLocal); + when(diskOfferingVO.getEncrypt()).thenReturn(true); + + volumeImportUnmanageManager.getOrCreateDiskOffering(account, diskOfferingId, zoneId, isLocal); + Assert.fail("it should fail"); + } catch (CloudRuntimeException ex) { + verify(volumeImportUnmanageManager).logFailureAndThrowException(String.format("Disk offering with ID %s should not support volume encryption", diskOfferingId)); + } + } + @Test public void testGetOrCreateDiskOfferingNoPermission() { try { From ff261cad4fb874b0724f415f8f0c50a59abb0a15 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 2 Apr 2024 10:35:19 +0200 Subject: [PATCH 39/44] Update 8808: storage pool should be Up --- .../volume/VolumeImportUnmanageManagerImpl.java | 6 +++++- .../VolumeImportUnmanageManagerImplTest.java | 17 ++++++++++++++++- ui/public/locales/en.json | 2 +- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index 1b87538ef7df..7fb2bd7e2e06 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -39,6 +39,7 @@ import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.Storage; import com.cloud.storage.StoragePoolHostVO; +import com.cloud.storage.StoragePoolStatus; import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiService; import com.cloud.storage.VolumeVO; @@ -277,7 +278,10 @@ protected StoragePoolVO checkIfPoolAvailable(Long poolId) { logFailureAndThrowException(String.format("Storage pool (ID: %s) does not exist", poolId)); } if (pool.isInMaintenance()) { - logFailureAndThrowException(String.format("Storage pool (name: %s) is in maintenance: ", pool.getName())); + logFailureAndThrowException(String.format("Storage pool (name: %s) is in maintenance", pool.getName())); + } + if (!StoragePoolStatus.Up.equals(pool.getStatus())) { + logFailureAndThrowException(String.format("Storage pool (ID: %s) is not Up: %s", pool.getName(), pool.getStatus())); } return pool; } diff --git a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java index a80a02cb986f..696d54ee4786 100644 --- a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java @@ -36,6 +36,7 @@ import com.cloud.storage.ScopeType; import com.cloud.storage.Storage; import com.cloud.storage.StoragePoolHostVO; +import com.cloud.storage.StoragePoolStatus; import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiService; import com.cloud.storage.VolumeVO; @@ -193,6 +194,7 @@ public void setUp() { when(storagePoolVO.getUuid()).thenReturn(storagePoolUuid); when(storagePoolVO.getName()).thenReturn(storagePoolName); when(storagePoolVO.getPoolType()).thenReturn(storagePoolType); + when(storagePoolVO.getStatus()).thenReturn(StoragePoolStatus.Up); when(volumeDao.findById(volumeId)).thenReturn(volumeVO); when(volumeVO.getId()).thenReturn(volumeId); @@ -437,7 +439,20 @@ public void testCheckIfPoolAvailableInMaintenance() { volumeImportUnmanageManager.checkIfPoolAvailable(poolId); Assert.fail("it should fail"); } catch (CloudRuntimeException ex) { - verify(volumeImportUnmanageManager).logFailureAndThrowException(String.format("Storage pool (name: %s) is in maintenance: ", storagePoolName)); + verify(volumeImportUnmanageManager).logFailureAndThrowException(String.format("Storage pool (name: %s) is in maintenance", storagePoolName)); + } + } + + @Test + public void testCheckIfPoolAvailableDisabled() { + try { + when(primaryDataStoreDao.findById(poolId)).thenReturn(storagePoolVO); + when(storagePoolVO.isInMaintenance()).thenReturn(false); + when(storagePoolVO.getStatus()).thenReturn(StoragePoolStatus.Disabled); + volumeImportUnmanageManager.checkIfPoolAvailable(poolId); + Assert.fail("it should fail"); + } catch (CloudRuntimeException ex) { + verify(volumeImportUnmanageManager).logFailureAndThrowException(String.format("Storage pool (ID: %s) is not Up: %s", storagePoolName, StoragePoolStatus.Disabled)); } } diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 31133ac6a9ec..03e26d3df1b3 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -2715,7 +2715,7 @@ "message.desc.import.ext.kvm.wizard": "Import libvirt domain from External KVM Host not managed by CloudStack", "message.desc.import.local.kvm.wizard": "Import QCOW2 image from Local Storage of selected KVM Host", "message.desc.import.shared.kvm.wizard": "Import QCOW2 image from selected Primary Storage Pool", -"message.desc.import.unmanage.volume": "Please choose a storage pool that you want to import or unmanage volumes. This feature only supports KVM.", +"message.desc.import.unmanage.volume": "Please choose a storage pool that you want to import or unmanage volumes. The storage pool should be in Up status.
This feature only supports KVM.", "message.desc.importexportinstancewizard": "By choosing to manage an Instance, CloudStack takes over the orchestration of that Instance. Unmanaging an Instance removes CloudStack ability to manage it. In both cases, the Instance is left running and no changes are done to the VM on the hypervisor.

For KVM, managing a VM is an experimental feature.", "message.desc.importmigratefromvmwarewizard": "By selecting an existing or external VMware Datacenter and an instance to import, CloudStack migrates the selected instance from VMware to KVM on a conversion host using virt-v2v and imports it into a KVM cluster", "message.desc.primary.storage": "Each cluster must contain one or more primary storage servers. We will add the first one now. Primary storage contains the disk volumes for all the Instances running on hosts in the cluster. Use any standards-compliant protocol that is supported by the underlying hypervisor.", From ef6d167d3b1237f7d8c5f7fafa2fbc2e4b0ecfa5 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 2 Apr 2024 10:59:31 +0200 Subject: [PATCH 40/44] Update 8808: check volume path on storage pool on kvm hosts --- .../wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java | 4 ++++ .../wrapper/LibvirtGetVolumesOnStorageCommandWrapperTest.java | 1 + 2 files changed, 5 insertions(+) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java index 128d625df333..e4f8bdf1498d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java @@ -75,6 +75,10 @@ private GetVolumesOnStorageAnswer addVolumeByVolumePath(final GetVolumesOnStorag KVMPhysicalDisk disk = storagePool.getPhysicalDisk(volumePath); if (disk != null) { + if (!volumePath.equals(disk.getPath()) && !volumePath.equals(disk.getName())) { + String error = String.format("Volume path mismatch. Expected volume path (%s) is not the same as the actual name (%s) and path (%s)", volumePath, disk.getName(), disk.getPath()); + return new GetVolumesOnStorageAnswer(command, false, error); + } if (!isDiskFormatSupported(disk)) { return new GetVolumesOnStorageAnswer(command, false, String.format("disk format %s is unsupported", disk.getFormat())); } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapperTest.java index 73887caab792..4e039f318928 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapperTest.java @@ -127,6 +127,7 @@ public void testLibvirtGetVolumesOnStorageCommandWrapperForAllVolumes() { @Test public void testLibvirtGetVolumesOnStorageCommandWrapperForVolume() { KVMPhysicalDisk disk = Mockito.mock(KVMPhysicalDisk.class); + Mockito.when(disk.getPath()).thenReturn(volumePath); Mockito.when(disk.getFormat()).thenReturn(QemuImg.PhysicalDiskFormat.QCOW2); Mockito.when(disk.getQemuEncryptFormat()).thenReturn(QemuObject.EncryptFormat.LUKS); Mockito.when(storagePool.getPhysicalDisk(volumePath)).thenReturn(disk); From ffe9c96c618c5e16359e8f5bb2b8849ec50ae305 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 2 Apr 2024 11:09:26 +0200 Subject: [PATCH 41/44] Update 8808: check disk offering tags --- .../storage/volume/VolumeImportUnmanageManagerImpl.java | 3 +++ .../storage/volume/VolumeImportUnmanageManagerImplTest.java | 1 + 2 files changed, 4 insertions(+) diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index 7fb2bd7e2e06..8caea58ec6b4 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -217,6 +217,9 @@ public VolumeResponse importVolume(ImportVolumeCmd cmd) { if (diskOffering.isCustomized()) { volumeApiService.validateCustomDiskOfferingSizeRange(volume.getVirtualSize() / ByteScaleUtils.GiB); } + if (!volumeApiService.doesTargetStorageSupportDiskOffering(pool, diskOffering.getTags())) { + logFailureAndThrowException(String.format("Disk offering: %s storage tags are not compatible with selected storage pool: %s", diskOffering.getUuid(), pool.getUuid())); + } // 7. create records String volumeName = StringUtils.isNotBlank(cmd.getName()) ? cmd.getName().trim() : volumePath; diff --git a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java index 696d54ee4786..15cafc5576b6 100644 --- a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java @@ -275,6 +275,7 @@ public void testImportVolumeAllGood() throws ResourceAllocationException { when(diskOffering.isCustomized()).thenReturn(true); doReturn(diskOffering).when(volumeImportUnmanageManager).getOrCreateDiskOffering(account, diskOfferingId, zoneId, isLocal); doNothing().when(volumeApiService).validateCustomDiskOfferingSizeRange(anyLong()); + doReturn(true).when(volumeApiService).doesTargetStorageSupportDiskOffering(any(), isNull()); doReturn(diskProfile).when(volumeManager).importVolume(any(), anyString(), any(), eq(virtualSize), isNull(), isNull(), anyLong(), any(), isNull(), isNull(), any(), isNull(), anyLong(), anyString(), isNull()); when(diskProfile.getVolumeId()).thenReturn(volumeId); From 3fa226ba8310ec66bca7ed928f2941c31c8df4dd Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 8 Apr 2024 13:43:33 +0200 Subject: [PATCH 42/44] Update 8808: define list of supported hypervisors and storage types --- .../volume/VolumeImportUnmanageService.java | 11 ++++++ ...virtGetVolumesOnStorageCommandWrapper.java | 6 ++-- .../VolumeImportUnmanageManagerImpl.java | 34 +++++++++++++------ .../VolumeImportUnmanageManagerImplTest.java | 17 ++++++++-- 4 files changed, 51 insertions(+), 17 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java b/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java index f908fd9c9a4d..5f69f3e46e73 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java +++ b/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.storage.volume; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.Storage; import com.cloud.utils.component.PluggableService; import org.apache.cloudstack.api.command.admin.volume.ListVolumesForImportCmd; import org.apache.cloudstack.api.command.admin.volume.ImportVolumeCmd; @@ -24,8 +26,17 @@ import org.apache.cloudstack.api.response.VolumeForImportResponse; import org.apache.cloudstack.api.response.VolumeResponse; +import java.util.Arrays; +import java.util.List; + public interface VolumeImportUnmanageService extends PluggableService { + List SUPPORTED_HYPERVISORS = + Arrays.asList(Hypervisor.HypervisorType.KVM, Hypervisor.HypervisorType.VMware); + + List SUPPORTED_STORAGE_POOL_TYPES_FOR_KVM = Arrays.asList(Storage.StoragePoolType.NetworkFilesystem, + Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.RBD); + ListResponse listVolumesForImport(ListVolumesForImportCmd cmd); VolumeResponse importVolume(ImportVolumeCmd cmd); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java index e4f8bdf1498d..821a80f5ccac 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java @@ -50,7 +50,7 @@ @ResourceWrapper(handles = GetVolumesOnStorageCommand.class) public final class LibvirtGetVolumesOnStorageCommandWrapper extends CommandWrapper { - static final List SUPPORTED_STORAGE_POOL_TYPES = Arrays.asList(StoragePoolType.NetworkFilesystem, + static final List STORAGE_POOL_TYPES_SUPPORTED_BY_QEMU_IMG = Arrays.asList(StoragePoolType.NetworkFilesystem, StoragePoolType.Filesystem, StoragePoolType.RBD); @Override @@ -111,7 +111,7 @@ private GetVolumesOnStorageAnswer addVolumeByVolumePath(final GetVolumesOnStorag volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.FILE_FORMAT, fileFormat); } String encrypted = info.get(QemuImg.ENCRYPTED); - if (StringUtils.isNotBlank(encrypted) && encrypted.toLowerCase().equals("yes")) { + if (StringUtils.isNotBlank(encrypted) && encrypted.equalsIgnoreCase("yes")) { volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.IS_ENCRYPTED, String.valueOf(Boolean.TRUE)); } Boolean isLocked = isDiskFileLocked(storagePool, disk); @@ -154,7 +154,7 @@ private boolean isDiskFileLocked(KVMStoragePool pool, KVMPhysicalDisk disk) { } private Map getDiskFileInfo(KVMStoragePool pool, KVMPhysicalDisk disk, boolean secure) { - if (!SUPPORTED_STORAGE_POOL_TYPES.contains(pool.getType())) { + if (!STORAGE_POOL_TYPES_SUPPORTED_BY_QEMU_IMG.contains(pool.getType())) { return new HashMap<>(); // unknown } try { diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index 8caea58ec6b4..400f5028a95c 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -86,9 +86,6 @@ public class VolumeImportUnmanageManagerImpl implements VolumeImportUnmanageService { protected Logger logger = Logger.getLogger(VolumeImportUnmanageManagerImpl.class); - private static final List volumeImportUnmanageSupportedHypervisors = - Arrays.asList(Hypervisor.HypervisorType.KVM); - @Inject private AccountManager accountMgr; @Inject @@ -149,7 +146,7 @@ public ListResponse listVolumesForImport(ListVolumesFor } StoragePoolVO pool = checkIfPoolAvailable(poolId); - List volumes = listVolumesForImportInternal(poolId, path, keyword); + List volumes = listVolumesForImportInternal(pool, path, keyword); List responses = new ArrayList<>(); for (VolumeOnStorageTO volume : volumes) { @@ -194,7 +191,7 @@ public VolumeResponse importVolume(ImportVolumeCmd cmd) { } // 4. send a command to hypervisor to check - List volumes = listVolumesForImportInternal(poolId, volumePath, null); + List volumes = listVolumesForImportInternal(pool, volumePath, null); if (CollectionUtils.isEmpty(volumes)) { logFailureAndThrowException("Cannot find volume on storage pool: " + volumePath); } @@ -234,14 +231,10 @@ public VolumeResponse importVolume(ImportVolumeCmd cmd) { return responseGenerator.createVolumeResponse(ResponseObject.ResponseView.Full, volumeVO); } - protected List listVolumesForImportInternal(Long poolId, String volumePath, String keyword) { - StoragePoolVO pool = checkIfPoolAvailable(poolId); - + protected List listVolumesForImportInternal(StoragePoolVO pool, String volumePath, String keyword) { Pair hostAndLocalPath = findHostAndLocalPathForVolumeImport(pool); HostVO host = hostAndLocalPath.first(); - if (!volumeImportUnmanageSupportedHypervisors.contains(host.getHypervisorType())) { - logFailureAndThrowException("Import VM is not supported for hypervisor: " + host.getHypervisorType()); - } + checkIfHostAndPoolSupported(host, pool); StorageFilerTO storageTO = new StorageFilerTO(pool); GetVolumesOnStorageCommand command = new GetVolumesOnStorageCommand(storageTO, volumePath, keyword); @@ -263,6 +256,9 @@ public boolean unmanageVolume(long volumeId) { // 2. check if pool available StoragePoolVO pool = checkIfPoolAvailable(volume.getPoolId()); + // 3. unmanage volume internally + unmanageVolumeInternal(pool, volume.getPath()); + // 3. Update resource count updateResourceLimitForVolumeUnmanage(volume); @@ -325,6 +321,22 @@ private Pair findHostAndLocalPathForVolumeImportForHostScope(Lon return null; } + private void checkIfHostAndPoolSupported(HostVO host, StoragePoolVO pool) { + if (!SUPPORTED_HYPERVISORS.contains(host.getHypervisorType())) { + logFailureAndThrowException("Importing and unmanaging volume are not supported for hypervisor: " + host.getHypervisorType()); + } + + if (Hypervisor.HypervisorType.KVM.equals(host.getHypervisorType()) && !SUPPORTED_STORAGE_POOL_TYPES_FOR_KVM.contains(pool.getPoolType())) { + logFailureAndThrowException(String.format("Importing and unmanaging volume are not supported for pool type %s on hypervisor %s", pool.getPoolType(), host.getHypervisorType())); + } + } + + protected void unmanageVolumeInternal(StoragePoolVO pool, String volumePath) { + Pair hostAndLocalPath = findHostAndLocalPathForVolumeImport(pool); + HostVO host = hostAndLocalPath.first(); + checkIfHostAndPoolSupported(host, pool); + } + protected VolumeForImportResponse createVolumeForImportResponse(VolumeOnStorageTO volume, StoragePoolVO pool) { VolumeForImportResponse response = new VolumeForImportResponse(); response.setPath(volume.getPath()); diff --git a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java index 15cafc5576b6..ba17de2c3adf 100644 --- a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java @@ -229,7 +229,7 @@ public void testListVolumesForImport() { volumeOnStorageTO.setQemuEncryptFormat(encryptFormat); List volumesOnStorageTO = new ArrayList<>(); volumesOnStorageTO.add(volumeOnStorageTO); - doReturn(volumesOnStorageTO).when(volumeImportUnmanageManager).listVolumesForImportInternal(poolId, path, null); + doReturn(volumesOnStorageTO).when(volumeImportUnmanageManager).listVolumesForImportInternal(storagePoolVO, path, null); ListResponse listResponses = volumeImportUnmanageManager.listVolumesForImport(cmd); Assert.assertEquals(1, listResponses.getResponses().size()); @@ -262,7 +262,7 @@ public void testImportVolumeAllGood() throws ResourceAllocationException { List volumesOnStorageTO = new ArrayList<>(); volumesOnStorageTO.add(volumeOnStorageTO); - doReturn(volumesOnStorageTO).when(volumeImportUnmanageManager).listVolumesForImportInternal(poolId, path, null); + doReturn(volumesOnStorageTO).when(volumeImportUnmanageManager).listVolumesForImportInternal(storagePoolVO, path, null); doNothing().when(volumeImportUnmanageManager).checkIfVolumeIsLocked(volumeOnStorageTO); doNothing().when(volumeImportUnmanageManager).checkIfVolumeIsEncrypted(volumeOnStorageTO); @@ -309,7 +309,7 @@ public void testListVolumesForImportInternal() { when(answer.getVolumes()).thenReturn(volumesOnStorageTO); doReturn(answer).when(agentManager).easySend(eq(hostId), any(GetVolumesOnStorageCommand.class)); - List result = volumeImportUnmanageManager.listVolumesForImportInternal(poolId, path, null); + List result = volumeImportUnmanageManager.listVolumesForImportInternal(storagePoolVO, path, null); Assert.assertEquals(volumesOnStorageTO, result); } @@ -361,6 +361,8 @@ public void testUnmanageVolume() { when(volumeVO.getState()).thenReturn(Volume.State.Ready); when(volumeVO.getPoolId()).thenReturn(poolId); when(volumeVO.getInstanceId()).thenReturn(null); + when(volumeVO.getPath()).thenReturn(path); + doNothing().when(volumeImportUnmanageManager).unmanageVolumeInternal(storagePoolVO, path); doNothing().when(resourceLimitService).decrementResourceCount(accountId, Resource.ResourceType.volume); doNothing().when(resourceLimitService).decrementResourceCount(accountId, Resource.ResourceType.primary_storage, virtualSize); @@ -421,6 +423,15 @@ public void testUnmanageVolumeAttached() { } } + @Test + public void testUnmanageVolumeInternal() { + Pair hostAndLocalPath = mock(Pair.class); + doReturn(hostAndLocalPath).when(volumeImportUnmanageManager).findHostAndLocalPathForVolumeImport(storagePoolVO); + when(hostAndLocalPath.first()).thenReturn(hostVO); + + volumeImportUnmanageManager.unmanageVolumeInternal(storagePoolVO, path); + } + @Test public void testCheckIfPoolAvailableNotExist() { try { From e5f6072f047d02e96e58f05fedafdf8cd4738f69 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 22 Apr 2024 12:46:52 +0200 Subject: [PATCH 43/44] Update 8808: get volume via host and check for both import/unmanage --- .../VolumeImportUnmanageManagerImpl.java | 44 ++++++++++--------- .../VolumeImportUnmanageManagerImplTest.java | 25 ++++------- 2 files changed, 32 insertions(+), 37 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index 400f5028a95c..b3b164395f6a 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -190,18 +190,9 @@ public VolumeResponse importVolume(ImportVolumeCmd cmd) { logFailureAndThrowException("Volume is a base image of a template: " + volumePath); } - // 4. send a command to hypervisor to check - List volumes = listVolumesForImportInternal(pool, volumePath, null); - if (CollectionUtils.isEmpty(volumes)) { - logFailureAndThrowException("Cannot find volume on storage pool: " + volumePath); - } + // 4. get volume info on storage through host and check + VolumeOnStorageTO volume = getVolumeOnStorageAndCheck(pool, volumePath); - VolumeOnStorageTO volume = volumes.get(0); - - // check if volume is locked, encrypted or volume size is allowed - checkIfVolumeIsLocked(volume); - checkIfVolumeIsEncrypted(volume); - checkIfVolumeHasBackingFile(volume); if (checkIfVolumeForSnapshot(pool, volume.getFullPath())) { logFailureAndThrowException("Volume is a reference of snapshot on primary: " + volume.getFullPath()); } @@ -231,6 +222,23 @@ public VolumeResponse importVolume(ImportVolumeCmd cmd) { return responseGenerator.createVolumeResponse(ResponseObject.ResponseView.Full, volumeVO); } + protected VolumeOnStorageTO getVolumeOnStorageAndCheck(StoragePoolVO pool, String volumePath) { + // send a command to hypervisor to check + List volumes = listVolumesForImportInternal(pool, volumePath, null); + if (CollectionUtils.isEmpty(volumes)) { + logFailureAndThrowException("Cannot find volume on storage pool: " + volumePath); + } + + VolumeOnStorageTO volume = volumes.get(0); + + // check if volume is locked, encrypted or has backing file + checkIfVolumeIsLocked(volume); + checkIfVolumeIsEncrypted(volume); + checkIfVolumeHasBackingFile(volume); + + return volume; + } + protected List listVolumesForImportInternal(StoragePoolVO pool, String volumePath, String keyword) { Pair hostAndLocalPath = findHostAndLocalPathForVolumeImport(pool); HostVO host = hostAndLocalPath.first(); @@ -257,7 +265,7 @@ public boolean unmanageVolume(long volumeId) { StoragePoolVO pool = checkIfPoolAvailable(volume.getPoolId()); // 3. unmanage volume internally - unmanageVolumeInternal(pool, volume.getPath()); + getVolumeOnStorageAndCheck(pool, volume.getPath()); // 3. Update resource count updateResourceLimitForVolumeUnmanage(volume); @@ -331,12 +339,6 @@ private void checkIfHostAndPoolSupported(HostVO host, StoragePoolVO pool) { } } - protected void unmanageVolumeInternal(StoragePoolVO pool, String volumePath) { - Pair hostAndLocalPath = findHostAndLocalPathForVolumeImport(pool); - HostVO host = hostAndLocalPath.first(); - checkIfHostAndPoolSupported(host, pool); - } - protected VolumeForImportResponse createVolumeForImportResponse(VolumeOnStorageTO volume, StoragePoolVO pool) { VolumeForImportResponse response = new VolumeForImportResponse(); response.setPath(volume.getPath()); @@ -372,7 +374,7 @@ protected void checkIfVolumeIsLocked(VolumeOnStorageTO volume) { if (volumeDetails != null && volumeDetails.containsKey(VolumeOnStorageTO.Detail.IS_LOCKED)) { String isLocked = volumeDetails.get(VolumeOnStorageTO.Detail.IS_LOCKED); if (Boolean.parseBoolean(isLocked)) { - logFailureAndThrowException("Locked volume cannot be imported."); + logFailureAndThrowException("Locked volume cannot be imported or unmanaged."); } } } @@ -382,7 +384,7 @@ protected void checkIfVolumeIsEncrypted(VolumeOnStorageTO volume) { if (volumeDetails != null && volumeDetails.containsKey(VolumeOnStorageTO.Detail.IS_ENCRYPTED)) { String isEncrypted = volumeDetails.get(VolumeOnStorageTO.Detail.IS_ENCRYPTED); if (Boolean.parseBoolean(isEncrypted)) { - logFailureAndThrowException("Encrypted volume cannot be imported."); + logFailureAndThrowException("Encrypted volume cannot be imported or unmanaged."); } } } @@ -392,7 +394,7 @@ protected void checkIfVolumeHasBackingFile(VolumeOnStorageTO volume) { if (volumeDetails != null && volumeDetails.containsKey(VolumeOnStorageTO.Detail.BACKING_FILE)) { String backingFile = volumeDetails.get(VolumeOnStorageTO.Detail.BACKING_FILE); if (StringUtils.isNotBlank(backingFile)) { - logFailureAndThrowException("Volume with backing file cannot be imported."); + logFailureAndThrowException("Volume with backing file cannot be imported or unmanaged."); } } } diff --git a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java index ba17de2c3adf..dab465954381 100644 --- a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java @@ -322,8 +322,8 @@ public void testCheckIfVolumeIsLocked() { volumeImportUnmanageManager.checkIfVolumeIsLocked(volumeOnStorageTO); Assert.fail("It should fail as the volume is locked"); } catch (CloudRuntimeException ex) { - Assert.assertEquals("Locked volume cannot be imported.", ex.getMessage()); - verify(volumeImportUnmanageManager).logFailureAndThrowException("Locked volume cannot be imported."); + Assert.assertEquals("Locked volume cannot be imported or unmanaged.", ex.getMessage()); + verify(volumeImportUnmanageManager).logFailureAndThrowException("Locked volume cannot be imported or unmanaged."); } } @@ -336,8 +336,8 @@ public void testCheckIfVolumeIsEncrypted() { volumeImportUnmanageManager.checkIfVolumeIsEncrypted(volumeOnStorageTO); Assert.fail("It should fail as the volume is encrypted"); } catch (CloudRuntimeException ex) { - Assert.assertEquals("Encrypted volume cannot be imported.", ex.getMessage()); - verify(volumeImportUnmanageManager).logFailureAndThrowException("Encrypted volume cannot be imported."); + Assert.assertEquals("Encrypted volume cannot be imported or unmanaged.", ex.getMessage()); + verify(volumeImportUnmanageManager).logFailureAndThrowException("Encrypted volume cannot be imported or unmanaged."); } } @@ -351,8 +351,8 @@ public void testCheckIfVolumeHasBackingFile() { volumeImportUnmanageManager.checkIfVolumeHasBackingFile(volumeOnStorageTO); Assert.fail("It should fail as the volume has backing file"); } catch (CloudRuntimeException ex) { - Assert.assertEquals("Volume with backing file cannot be imported.", ex.getMessage()); - verify(volumeImportUnmanageManager).logFailureAndThrowException("Volume with backing file cannot be imported."); + Assert.assertEquals("Volume with backing file cannot be imported or unmanaged.", ex.getMessage()); + verify(volumeImportUnmanageManager).logFailureAndThrowException("Volume with backing file cannot be imported or unmanaged."); } } @@ -362,7 +362,9 @@ public void testUnmanageVolume() { when(volumeVO.getPoolId()).thenReturn(poolId); when(volumeVO.getInstanceId()).thenReturn(null); when(volumeVO.getPath()).thenReturn(path); - doNothing().when(volumeImportUnmanageManager).unmanageVolumeInternal(storagePoolVO, path); + VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(hypervisorType, path, name, fullPath, + format, size, virtualSize); + doReturn(volumeOnStorageTO).when(volumeImportUnmanageManager).getVolumeOnStorageAndCheck(storagePoolVO, path); doNothing().when(resourceLimitService).decrementResourceCount(accountId, Resource.ResourceType.volume); doNothing().when(resourceLimitService).decrementResourceCount(accountId, Resource.ResourceType.primary_storage, virtualSize); @@ -423,15 +425,6 @@ public void testUnmanageVolumeAttached() { } } - @Test - public void testUnmanageVolumeInternal() { - Pair hostAndLocalPath = mock(Pair.class); - doReturn(hostAndLocalPath).when(volumeImportUnmanageManager).findHostAndLocalPathForVolumeImport(storagePoolVO); - when(hostAndLocalPath.first()).thenReturn(hostVO); - - volumeImportUnmanageManager.unmanageVolumeInternal(storagePoolVO, path); - } - @Test public void testCheckIfPoolAvailableNotExist() { try { From 8e6d67332fa98f7441f0485f25b2d36b4f88d08e Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 22 Apr 2024 12:47:07 +0200 Subject: [PATCH 44/44] Update 8808: allow only unmanage detached/Ready/KVM volume on UI --- ui/src/views/tools/ManageVolumes.vue | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ui/src/views/tools/ManageVolumes.vue b/ui/src/views/tools/ManageVolumes.vue index 2ec713c3b433..94c06b4ce9c4 100644 --- a/ui/src/views/tools/ManageVolumes.vue +++ b/ui/src/views/tools/ManageVolumes.vue @@ -747,7 +747,12 @@ export default { return { type: 'checkbox', selectedRowKeys: this.managedVolumesSelectedRowKeys || [], - onChange: this.onManagedVolumeSelectRow + onChange: this.onManagedVolumeSelectRow, + getCheckboxProps: (record) => { + return { + disabled: record.virtualmachineid !== undefined || record.state !== 'Ready' || record.hypervisor !== 'KVM' + } + } } }, selectedCluster () {