vmware: fix inter-cluster stopped vm and volume migration (#4895)

Fixes #4838

For inter-cluster migration without shared storage, VMware needs a host to be specified. Fix is to specify an appropriate host in the target cluster during a stopped VM migration. Also, find target datastore using the host in the target cluster.

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

Author: shwstppr

core/src/main/java/com/cloud/agent/api/MigrateVmToPoolCommand.java

@@ -18,10 +18,10 @@
 //
 package com.cloud.agent.api;
 
-import com.cloud.agent.api.to.VolumeTO;
-
 import java.util.Collection;
 
+import com.cloud.agent.api.to.VolumeTO;
+
 /**
  * used to tell the agent to migrate a vm to a different primary storage pool.
  * It is for now only implemented on Vmware and is supposed to work irrespective of whether the VM is started or not.
@@ -32,6 +32,7 @@ public class MigrateVmToPoolCommand extends Command {
     private String vmName;
     private String destinationPool;
     private boolean executeInSequence = false;
+    private String hostGuidInTargetCluster;
 
     protected MigrateVmToPoolCommand() {
     }
@@ -41,15 +42,22 @@ protected MigrateVmToPoolCommand() {
      * @param vmName the name of the VM to migrate
      * @param volumes used to supply feedback on vmware generated names
      * @param destinationPool the primary storage pool to migrate the VM to
+     * @param hostGuidInTargetCluster GUID of host in target cluster when migrating across clusters
      * @param executeInSequence
      */
-    public MigrateVmToPoolCommand(String vmName, Collection<VolumeTO> volumes, String destinationPool, boolean executeInSequence) {
+    public MigrateVmToPoolCommand(String vmName, Collection<VolumeTO> volumes, String destinationPool,
+                                  String hostGuidInTargetCluster, boolean executeInSequence) {
         this.vmName = vmName;
         this.volumes = volumes;
         this.destinationPool = destinationPool;
+        this.hostGuidInTargetCluster = hostGuidInTargetCluster;
         this.executeInSequence = executeInSequence;
     }
 
+    public MigrateVmToPoolCommand(String vmName, Collection<VolumeTO> volumes, String destinationPool, boolean executeInSequence) {
+        this(vmName, volumes, destinationPool, null, executeInSequence);
+    }
+
     public Collection<VolumeTO> getVolumes() {
         return volumes;
     }
@@ -62,6 +70,10 @@ public String getVmName() {
         return vmName;
     }
 
+    public String getHostGuidInTargetCluster() {
+        return hostGuidInTargetCluster;
+    }
+
     @Override
     public boolean executeInSequence() {
         return executeInSequence;

core/src/main/java/com/cloud/agent/api/storage/MigrateVolumeCommand.java

@@ -34,6 +34,7 @@ public class MigrateVolumeCommand extends Command {
     StorageFilerTO sourcePool;
     String attachedVmName;
     Volume.Type volumeType;
+    private String hostGuidInTargetCluster;
 
     private DataTO srcData;
     private DataTO destData;
@@ -54,9 +55,10 @@ public MigrateVolumeCommand(long volumeId, String volumePath, StoragePool pool,
         this.setWait(timeout);
     }
 
-    public MigrateVolumeCommand(long volumeId, String volumePath, StoragePool sourcePool, StoragePool targetPool) {
+    public MigrateVolumeCommand(long volumeId, String volumePath, StoragePool sourcePool, StoragePool targetPool, String hostGuidInTargetCluster) {
         this(volumeId,volumePath,targetPool, null, Volume.Type.UNKNOWN, -1);
         this.sourcePool = new StorageFilerTO(sourcePool);
+        this.hostGuidInTargetCluster = hostGuidInTargetCluster;
     }
 
     public MigrateVolumeCommand(DataTO srcData, DataTO destData, Map<String, String> srcDetails, Map<String, String> destDetails, int timeout) {
@@ -101,6 +103,10 @@ public Volume.Type getVolumeType() {
         return volumeType;
     }
 
+    public String getHostGuidInTargetCluster() {
+        return hostGuidInTargetCluster;
+    }
+
     public DataTO getSrcData() {
         return srcData;
     }

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -40,7 +40,6 @@
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
-import com.cloud.deployasis.dao.UserVmDeployAsIsDetailsDao;
 import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.command.admin.vm.MigrateVMCmd;
@@ -142,6 +141,7 @@
 import com.cloud.deploy.DeploymentPlanner;
 import com.cloud.deploy.DeploymentPlanner.ExcludeList;
 import com.cloud.deploy.DeploymentPlanningManager;
+import com.cloud.deployasis.dao.UserVmDeployAsIsDetailsDao;
 import com.cloud.event.EventTypes;
 import com.cloud.event.UsageEventUtils;
 import com.cloud.event.UsageEventVO;
@@ -2210,23 +2210,17 @@ private void orchestrateStorageMigration(final String vmUuid, final StoragePool
         }
     }
 
-    private Answer[] attemptHypervisorMigration(StoragePool destPool, VMInstanceVO vm) {
+    private Answer[] attemptHypervisorMigration(StoragePool destPool, VMInstanceVO vm, Long hostId) {
+        if (hostId == null) {
+            return null;
+        }
         final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vm.getHypervisorType());
         // OfflineVmwareMigration: in case of vmware call vcenter to do it for us.
         // OfflineVmwareMigration: should we check the proximity of source and destination
         // OfflineVmwareMigration: if we are in the same cluster/datacentre/pool or whatever?
         // OfflineVmwareMigration: we are checking on success to optionally delete an old vm if we are not
         List<Command> commandsToSend = hvGuru.finalizeMigrate(vm, destPool);
 
-        Long hostId = vm.getHostId();
-        // OfflineVmwareMigration: probably this is null when vm is stopped
-        if(hostId == null) {
-            hostId = vm.getLastHostId();
-            if (s_logger.isDebugEnabled()) {
-                s_logger.debug(String.format("host id is null, using last host id %d", hostId) );
-            }
-        }
-
         if(CollectionUtils.isNotEmpty(commandsToSend)) {
             Commands commandsContainer = new Commands(Command.OnError.Stop);
             commandsContainer.addCommands(commandsToSend);
@@ -2241,7 +2235,7 @@ private Answer[] attemptHypervisorMigration(StoragePool destPool, VMInstanceVO v
         return null;
     }
 
-    private void afterHypervisorMigrationCleanup(StoragePool destPool, VMInstanceVO vm, HostVO srcHost, Long srcClusterId, Answer[] hypervisorMigrationResults) throws InsufficientCapacityException {
+    private void afterHypervisorMigrationCleanup(StoragePool destPool, VMInstanceVO vm, Long srcClusterId, Answer[] hypervisorMigrationResults) throws InsufficientCapacityException {
         boolean isDebugEnabled = s_logger.isDebugEnabled();
         if(isDebugEnabled) {
             String msg = String.format("cleaning up after hypervisor pool migration volumes for VM %s(%s) to pool %s(%s)", vm.getInstanceName(), vm.getUuid(), destPool.getName(), destPool.getUuid());
@@ -2250,18 +2244,23 @@ private void afterHypervisorMigrationCleanup(StoragePool destPool, VMInstanceVO
         setDestinationPoolAndReallocateNetwork(destPool, vm);
         // OfflineVmwareMigration: don't set this to null or have another way to address the command; twice migrating will lead to an NPE
         Long destPodId = destPool.getPodId();
-        Long vmPodId = vm.getPodIdToDeployIn();
-        if (destPodId == null || ! destPodId.equals(vmPodId)) {
+
+        if (destPodId == null || !destPodId.equals(vm.getPodIdToDeployIn())) {
             if(isDebugEnabled) {
                 String msg = String.format("resetting lasHost for VM %s(%s) as pod (%s) is no good.", vm.getInstanceName(), vm.getUuid(), destPodId);
                 s_logger.debug(msg);
             }
-
             vm.setLastHostId(null);
             vm.setPodIdToDeployIn(destPodId);
             // OfflineVmwareMigration: a consecutive migration will fail probably (no host not pod)
-        }// else keep last host set for this vm
-        markVolumesInPool(vm,destPool, hypervisorMigrationResults);
+        } else if (srcClusterId != null && destPool.getClusterId() != null && !srcClusterId.equals(destPool.getClusterId())) {
+            if(isDebugEnabled) {
+                String msg = String.format("resetting lasHost for VM %s(%s) as cluster changed", vm.getInstanceName(), vm.getUuid());
+                s_logger.debug(msg);
+            }
+            vm.setLastHostId(null);
+        } // else keep last host set for this vm
+        markVolumesInPool(vm, destPool, hypervisorMigrationResults);
         // OfflineVmwareMigration: deal with answers, if (hypervisorMigrationResults.length > 0)
         // OfflineVmwareMigration: iterate over the volumes for data updates
     }
@@ -2295,23 +2294,60 @@ private void markVolumesInPool(VMInstanceVO vm, StoragePool destPool, Answer[] h
         }
     }
 
+    private Pair<Long, Long> findClusterAndHostIdForVm(VMInstanceVO vm) {
+        Long hostId = vm.getHostId();
+        Long clusterId = null;
+        // OfflineVmwareMigration: probably this is null when vm is stopped
+        if(hostId == null) {
+            hostId = vm.getLastHostId();
+            if (s_logger.isDebugEnabled()) {
+                s_logger.debug(String.format("host id is null, using last host id %d", hostId) );
+            }
+        }
+        if (hostId == null) {
+            List<VolumeVO> volumes = _volsDao.findByInstanceAndType(vm.getId(), Type.ROOT);
+            if (CollectionUtils.isNotEmpty(volumes)) {
+                for (VolumeVO rootVolume : volumes) {
+                    if (rootVolume.getPoolId() != null) {
+                        StoragePoolVO pool = _storagePoolDao.findById(rootVolume.getPoolId());
+                        if (pool != null && pool.getClusterId() != null) {
+                            clusterId = pool.getClusterId();
+                            List<HostVO> hosts = _hostDao.findHypervisorHostInCluster(pool.getClusterId());
+                            if (CollectionUtils.isNotEmpty(hosts)) {
+                                hostId = hosts.get(0).getId();
+                                break;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        if (clusterId == null && hostId != null) {
+            HostVO host = _hostDao.findById(hostId);
+            if (host != null) {
+                clusterId = host.getClusterId();
+            }
+        }
+        return new Pair<>(clusterId, hostId);
+    }
+
     private void migrateThroughHypervisorOrStorage(StoragePool destPool, VMInstanceVO vm) throws StorageUnavailableException, InsufficientCapacityException {
         final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm);
-        final Long srchostId = vm.getHostId() != null ? vm.getHostId() : vm.getLastHostId();
-        final HostVO srcHost = _hostDao.findById(srchostId);
-        final Long srcClusterId = srcHost.getClusterId();
-        Answer[] hypervisorMigrationResults = attemptHypervisorMigration(destPool, vm);
+        Pair<Long, Long> vmClusterAndHost = findClusterAndHostIdForVm(vm);
+        final Long sourceClusterId = vmClusterAndHost.first();
+        final Long sourceHostId = vmClusterAndHost.second();
+        Answer[] hypervisorMigrationResults = attemptHypervisorMigration(destPool, vm, sourceHostId);
         boolean migrationResult = false;
         if (hypervisorMigrationResults == null) {
             // OfflineVmwareMigration: if the HypervisorGuru can't do it, let the volume manager take care of it.
             migrationResult = volumeMgr.storageMigration(profile, destPool);
             if (migrationResult) {
-                afterStorageMigrationCleanup(destPool, vm, srcHost, srcClusterId);
+                afterStorageMigrationCleanup(destPool, vm, sourceHostId, sourceClusterId);
             } else {
                 s_logger.debug("Storage migration failed");
             }
         } else {
-            afterHypervisorMigrationCleanup(destPool, vm, srcHost, srcClusterId, hypervisorMigrationResults);
+            afterHypervisorMigrationCleanup(destPool, vm, sourceClusterId, hypervisorMigrationResults);
         }
     }
 
@@ -2366,7 +2402,7 @@ static boolean matches(List<String> volumeTags, List<String> storagePoolTags) {
     }
 
 
-    private void afterStorageMigrationCleanup(StoragePool destPool, VMInstanceVO vm, HostVO srcHost, Long srcClusterId) throws InsufficientCapacityException {
+    private void afterStorageMigrationCleanup(StoragePool destPool, VMInstanceVO vm, Long srcHostId, Long srcClusterId) throws InsufficientCapacityException {
         setDestinationPoolAndReallocateNetwork(destPool, vm);
 
         //when start the vm next time, don;'t look at last_host_id, only choose the host based on volume/storage pool
@@ -2376,7 +2412,7 @@ private void afterStorageMigrationCleanup(StoragePool destPool, VMInstanceVO vm,
         // If VM was cold migrated between clusters belonging to two different VMware DCs,
         // unregister the VM from the source host and cleanup the associated VM files.
         if (vm.getHypervisorType().equals(HypervisorType.VMware)) {
-            afterStorageMigrationVmwareVMcleanup(destPool, vm, srcHost, srcClusterId);
+            afterStorageMigrationVmwareVMCleanup(destPool, vm, srcHostId, srcClusterId);
         }
     }
 
@@ -2394,14 +2430,14 @@ private void setDestinationPoolAndReallocateNetwork(StoragePool destPool, VMInst
         }
     }
 
-    private void afterStorageMigrationVmwareVMcleanup(StoragePool destPool, VMInstanceVO vm, HostVO srcHost, Long srcClusterId) {
+    private void afterStorageMigrationVmwareVMCleanup(StoragePool destPool, VMInstanceVO vm, Long srcHostId, Long srcClusterId) {
         // OfflineVmwareMigration: this should only happen on storage migration, else the guru would already have issued the command
         final Long destClusterId = destPool.getClusterId();
-        if (srcClusterId != null && destClusterId != null && ! srcClusterId.equals(destClusterId)) {
+        if (srcHostId != null && srcClusterId != null && destClusterId != null && ! srcClusterId.equals(destClusterId)) {
             final String srcDcName = _clusterDetailsDao.getVmwareDcName(srcClusterId);
             final String destDcName = _clusterDetailsDao.getVmwareDcName(destClusterId);
             if (srcDcName != null && destDcName != null && !srcDcName.equals(destDcName)) {
-                removeStaleVmFromSource(vm, srcHost);
+                removeStaleVmFromSource(vm, _hostDao.findById(srcHostId));
             }
         }
     }