From f2baa688025f4b2e966be8ea0bd19f5def5faba2 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Fri, 26 Jul 2024 12:04:40 +0530 Subject: [PATCH 01/68] backup: Simple NAS backup plugin for KVM This is an experimental simple NAS backup plugin for KVM which may be later expanded for other hypervisors. This backup plugin aims to use shared NAS storage on KVM hosts such as NFS or CephFS, which is used to backup fully cloned VMs for backup & restore operations. This may not be as efficient and performant as some of the other B&R providers, but maybe useful for some KVM environments. Signed-off-by: Rohit Yadav --- client/pom.xml | 5 + plugins/backup/nas/pom.xml | 54 ++ .../cloudstack/backup/NASBackupProvider.java | 601 ++++++++++++++++++ .../META-INF/cloudstack/nas/module.properties | 18 + .../nas/spring-backup-nas-context.xml | 26 + plugins/pom.xml | 1 + 6 files changed, 705 insertions(+) create mode 100644 plugins/backup/nas/pom.xml create mode 100644 plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java create mode 100644 plugins/backup/nas/src/main/resources/META-INF/cloudstack/nas/module.properties create mode 100644 plugins/backup/nas/src/main/resources/META-INF/cloudstack/nas/spring-backup-nas-context.xml diff --git a/client/pom.xml b/client/pom.xml index 23e0f1886bdb..70c38caede93 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -628,6 +628,11 @@ cloud-plugin-backup-networker ${project.version} + + org.apache.cloudstack + cloud-plugin-backup-nas + ${project.version} + org.apache.cloudstack cloud-plugin-integrations-kubernetes-service diff --git a/plugins/backup/nas/pom.xml b/plugins/backup/nas/pom.xml new file mode 100644 index 000000000000..096bf45c67ed --- /dev/null +++ b/plugins/backup/nas/pom.xml @@ -0,0 +1,54 @@ + + + 4.0.0 + cloud-plugin-backup-nas + Apache CloudStack Plugin - KVM NAS Backup and Recovery Plugin + + cloudstack-plugins + org.apache.cloudstack + 4.20.0.0-SNAPSHOT + ../../pom.xml + + + + org.apache.cloudstack + cloud-plugin-hypervisor-kvm + ${project.version} + + + org.apache.commons + commons-lang3 + ${cs.commons-lang3.version} + + + com.fasterxml.jackson.core + jackson-databind + ${cs.jackson.version} + + + com.github.tomakehurst + wiremock-standalone + ${cs.wiremock.version} + test + + + diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java new file mode 100644 index 000000000000..b33d04c877dc --- /dev/null +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -0,0 +1,601 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup; + +import com.cloud.dc.dao.ClusterDao; +import com.cloud.host.HostVO; +import com.cloud.host.Status; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.StoragePoolHostVO; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.StoragePoolHostDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.utils.Pair; +import com.cloud.utils.Ternary; +import com.cloud.utils.component.AdapterBase; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionStatus; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.ssh.SshHelper; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.cloudstack.backup.dao.BackupOfferingDaoImpl; +import org.apache.cloudstack.backup.networker.NASClient; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.commons.collections.CollectionUtils; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.xml.utils.URI; +import org.apache.cloudstack.backup.networker.api.NASBackup; +import javax.inject.Inject; +import java.net.URISyntaxException; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.HashMap; +import java.util.Date; +import java.util.Objects; +import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import com.cloud.utils.script.Script; + +public class NASBackupProvider extends AdapterBase implements BackupProvider, Configurable { + private static final Logger LOG = LogManager.getLogger(NASBackupProvider.class); + + private final ConfigKey NASDetails = new ConfigKey<>("Advanced", String.class, + "backup.plugin.nas.details", "Default", + "The NFS/NAS storage details", true, ConfigKey.Scope.Zone); + + @Inject + private BackupDao backupDao; + + @Inject + private HostDao hostDao; + + @Inject + private ClusterDao clusterDao; + + @Inject + private VolumeDao volumeDao; + + @Inject + private StoragePoolHostDao storagePoolHostDao; + + @Inject + private VMInstanceDao vmInstanceDao; + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[]{ + NASDetails + }; + } + + @Override + public String getName() { + return "nas"; + } + + @Override + public String getDescription() { + return "NAS Backup Plugin"; + } + + @Override + public String getConfigComponentName() { + return BackupService.class.getSimpleName(); + } + + protected HostVO getLastVMHypervisorHost(VirtualMachine vm) { + HostVO host; + Long hostId = vm.getLastHostId(); + + if (hostId == null) { + LOG.debug("Cannot find last host for vm. This should never happen, please check your database."); + return null; + } + host = hostDao.findById(hostId); + + if ( host.getStatus() == Status.Up ) { + return host; + } else { + // Try to find a host in the same cluster + List altClusterHosts = hostDao.findHypervisorHostInCluster(host.getClusterId()); + for (final HostVO candidateClusterHost : altClusterHosts) { + if ( candidateClusterHost.getStatus() == Status.Up ) { + LOG.debug("Found Host " + candidateClusterHost.getName()); + return candidateClusterHost; + } + } + } + // Try to find a Host in the zone + List altZoneHosts = hostDao.findByDataCenterId(host.getDataCenterId()); + for (final HostVO candidateZoneHost : altZoneHosts) { + if ( candidateZoneHost.getStatus() == Status.Up && candidateZoneHost.getHypervisorType() == Hypervisor.HypervisorType.KVM ) { + LOG.debug("Found Host " + candidateZoneHost.getName()); + return candidateZoneHost; + } + } + return null; + } + + protected HostVO getRunningVMHypervisorHost(VirtualMachine vm) { + + HostVO host; + Long hostId = vm.getHostId(); + + if (hostId == null) { + throw new CloudRuntimeException("Unable to find the HYPERVISOR for " + vm.getName() + ". Make sure the virtual machine is running"); + } + + host = hostDao.findById(hostId); + + return host; + } + + protected String getVMHypervisorCluster(HostVO host) { + return clusterDao.findById(host.getClusterId()).getName(); + } + + protected Ternary getKVMHyperisorCredentials(HostVO host) { + String username = null; + String password = null; + + if (host != null && host.getHypervisorType() == Hypervisor.HypervisorType.KVM) { + hostDao.loadDetails(host); + password = host.getDetail("password"); + username = host.getDetail("username"); + } + if ( password == null || username == null) { + throw new CloudRuntimeException("Cannot find login credentials for HYPERVISOR " + Objects.requireNonNull(host).getUuid()); + } + + return new Ternary<>(username, password, null); + } + + private String executeBackupCommand(HostVO host, String username, String password, String command) { + String nstRegex = "\\bcompleted savetime=([0-9]{10})"; + Pattern saveTimePattern = Pattern.compile(nstRegex); + + try { + Pair response = SshHelper.sshExecute(host.getPrivateIpAddress(), 22, + username, null, password, command, 120000, 120000, 3600000); + if (!response.first()) { + LOG.error(String.format("Backup Script failed on HYPERVISOR %s due to: %s", host, response.second())); + } else { + LOG.debug(String.format("NAS Backup Results: %s", response.second())); + } + Matcher saveTimeMatcher = saveTimePattern.matcher(response.second()); + if (saveTimeMatcher.find()) { + LOG.debug(String.format("Got saveTimeMatcher: %s", saveTimeMatcher.group(1))); + return saveTimeMatcher.group(1); + } + } catch (final Exception e) { + throw new CloudRuntimeException(String.format("Failed to take backup on host %s due to: %s", host.getName(), e.getMessage())); + } + + return null; + } + private boolean executeRestoreCommand(HostVO host, String username, String password, String command) { + + try { + Pair response = SshHelper.sshExecute(host.getPrivateIpAddress(), 22, + username, null, password, command, 120000, 120000, 3600000); + + if (!response.first()) { + LOG.error(String.format("Restore Script failed on HYPERVISOR %s due to: %s", host, response.second())); + } else { + LOG.debug(String.format("NAS Restore Results: %s",response.second())); + return true; + } + } catch (final Exception e) { + throw new CloudRuntimeException(String.format("Failed to restore backup on host %s due to: %s", host.getName(), e.getMessage())); + } + return false; + } + + private Object getClient(final Long zoneId) { + try { + return null; // TODO: as a native object we don't need an API client + } catch (Exception e) { + LOG.error("Failed to build NAS API client due to: ", e); + } + throw new CloudRuntimeException("Failed to build NAS API client"); + } + + @Override + public List listBackupOfferings(Long zoneId) { + List policies = new ArrayList<>(); + for (final BackupOffering policy : getClient(zoneId).listPolicies()) { + if (!policy.getName().contains(BACKUP_IDENTIFIER)) { + policies.add(policy); + } + } + + return policies; + } + + @Override + public boolean isValidProviderOffering(Long zoneId, String uuid) { + List policies = listBackupOfferings(zoneId); + if (CollectionUtils.isEmpty(policies)) { + return false; + } + for (final BackupOffering policy : policies) { + if (Objects.equals(policy.getExternalId(), uuid)) { + return true; + } + } + return false; + } + + @Override + public boolean assignVMToBackupOffering(VirtualMachine vm, BackupOffering backupOffering) { return true; } + + @Override + public boolean removeVMFromBackupOffering(VirtualMachine vm) { + LOG.debug("Removing VirtualMachine from Backup offering and Deleting any existing backups"); + + List backupsTaken = getClient(vm.getDataCenterId()).getBackupsForVm(vm); + + for (String backupId : backupsTaken) { + LOG.debug("Trying to remove backup with id" + backupId); + getClient(vm.getDataCenterId()).deleteBackupForVM(backupId); + } + + return true; + } + + @Override + public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { + String networkerServer; + HostVO hostVO; + + final Long zoneId = backup.getZoneId(); + final String externalBackupId = backup.getExternalId(); + final NASBackup networkerBackup=getClient(zoneId).getNASBackupInfo(externalBackupId); + final String SSID = networkerBackup.getShortId(); + + LOG.debug("Restoring vm " + vm.getUuid() + "from backup " + backup.getUuid() + " on the NAS Backup Provider"); + + if ( SSID.isEmpty() ) { + LOG.debug("There was an error retrieving the SSID for backup with id " + externalBackupId + " from NAS"); + return false; + } + + // Find where the VM was last running + hostVO = getLastVMHypervisorHost(vm); + // Get credentials for that host + Ternary credentials = getKVMHyperisorCredentials(hostVO); + LOG.debug("The SSID was reported successfully " + externalBackupId); + try { + networkerServer = getUrlDomain(NASUrl.value()); + } catch (URISyntaxException e) { + throw new CloudRuntimeException(String.format("Failed to convert API to HOST : %s", e)); + } + String networkerRestoreScr = "/usr/share/cloudstack-common/scripts/vm/hypervisor/kvm/nsrkvmrestore.sh"; + final Script script = new Script(networkerRestoreScr); + script.add("-s"); + script.add(networkerServer); + script.add("-S"); + script.add(SSID); + + if ( Boolean.TRUE.equals(NASClientVerboseLogs.value()) ) + script.add("-v"); + + Date restoreJobStart = new Date(); + LOG.debug("Starting Restore for VM ID " + vm.getUuid() + " and SSID" + SSID + " at " + restoreJobStart); + + if ( executeRestoreCommand(hostVO, credentials.first(), credentials.second(), script.toString()) ) { + Date restoreJobEnd = new Date(); + LOG.debug("Restore Job for SSID " + SSID + " completed successfully at " + restoreJobEnd); + return true; + } else { + LOG.debug("Restore Job for SSID " + SSID + " failed!"); + return false; + } + } + + @Override + public Pair restoreBackedUpVolume(Backup backup, String volumeUuid, String hostIp, String dataStoreUuid) { + String networkerServer; + VolumeVO volume = volumeDao.findByUuid(volumeUuid); + VMInstanceVO backupSourceVm = vmInstanceDao.findById(backup.getVmId()); + StoragePoolHostVO dataStore = storagePoolHostDao.findByUuid(dataStoreUuid); + HostVO hostVO = hostDao.findByIp(hostIp); + + final Long zoneId = backup.getZoneId(); + final String externalBackupId = backup.getExternalId(); + final NASBackup networkerBackup=getClient(zoneId).getNASBackupInfo(externalBackupId); + final String SSID = networkerBackup.getShortId(); + final String clusterName = networkerBackup.getClientHostname(); + final String destinationNASClient = hostVO.getName().split("\\.")[0]; + Long restoredVolumeDiskSize = 0L; + + LOG.debug("Restoring volume " + volumeUuid + "from backup " + backup.getUuid() + " on the NAS Backup Provider"); + + if ( SSID.isEmpty() ) { + LOG.debug("There was an error retrieving the SSID for backup with id " + externalBackupId + " from NAS"); + return null; + } + + Ternary credentials = getKVMHyperisorCredentials(hostVO); + LOG.debug("The SSID was reported successfully " + externalBackupId); + try { + networkerServer = getUrlDomain(NASUrl.value()); + } catch (URISyntaxException e) { + throw new CloudRuntimeException(String.format("Failed to convert API to HOST : %s", e)); + } + + // Find volume size from backup vols + for ( Backup.VolumeInfo VMVolToRestore : backupSourceVm.getBackupVolumeList()) { + if (VMVolToRestore.getUuid().equals(volumeUuid)) + restoredVolumeDiskSize = (VMVolToRestore.getSize()); + } + + VolumeVO restoredVolume = new VolumeVO(Volume.Type.DATADISK, null, backup.getZoneId(), + backup.getDomainId(), backup.getAccountId(), 0, null, + backup.getSize(), null, null, null); + + restoredVolume.setName("RV-"+volume.getName()); + restoredVolume.setProvisioningType(volume.getProvisioningType()); + restoredVolume.setUpdated(new Date()); + restoredVolume.setUuid(UUID.randomUUID().toString()); + restoredVolume.setRemoved(null); + restoredVolume.setDisplayVolume(true); + restoredVolume.setPoolId(volume.getPoolId()); + restoredVolume.setPath(restoredVolume.getUuid()); + restoredVolume.setState(Volume.State.Copying); + restoredVolume.setSize(restoredVolumeDiskSize); + restoredVolume.setDiskOfferingId(volume.getDiskOfferingId()); + + try { + volumeDao.persist(restoredVolume); + } catch (Exception e) { + throw new CloudRuntimeException("Unable to craft restored volume due to: "+e); + } + String networkerRestoreScr = "/usr/share/cloudstack-common/scripts/vm/hypervisor/kvm/nsrkvmrestore.sh"; + final Script script = new Script(networkerRestoreScr); + script.add("-s"); + script.add(networkerServer); + script.add("-c"); + script.add(clusterName); + script.add("-d"); + script.add(destinationNASClient); + script.add("-n"); + script.add(restoredVolume.getUuid()); + script.add("-p"); + script.add(dataStore.getLocalPath()); + script.add("-a"); + script.add(volume.getUuid()); + + if ( Boolean.TRUE.equals(NASClientVerboseLogs.value()) ) + script.add("-v"); + + Date restoreJobStart = new Date(); + LOG.debug("Starting Restore for Volume UUID " + volume.getUuid() + " and SSID" + SSID + " at " + restoreJobStart); + + if ( executeRestoreCommand(hostVO, credentials.first(), credentials.second(), script.toString()) ) { + Date restoreJobEnd = new Date(); + LOG.debug("Restore Job for SSID " + SSID + " completed successfully at " + restoreJobEnd); + return new Pair<>(true,restoredVolume.getUuid()); + } else { + volumeDao.expunge(restoredVolume.getId()); + LOG.debug("Restore Job for SSID " + SSID + " failed!"); + return null; + } + } + + @Override + public boolean takeBackup(VirtualMachine vm) { + String networkerServer; + String clusterName; + + try { + networkerServer = getUrlDomain(NASUrl.value()); + } catch (URISyntaxException e) { + throw new CloudRuntimeException(String.format("Failed to convert API to HOST : %s", e)); + } + + // Find where the VM is currently running + HostVO hostVO = getRunningVMHypervisorHost(vm); + // Get credentials for that host + Ternary credentials = getKVMHyperisorCredentials(hostVO); + // Get retention Period for our Backup + BackupOfferingVO vmBackupOffering = new BackupOfferingDaoImpl().findById(vm.getBackupOfferingId()); + final String backupProviderPolicyId = vmBackupOffering.getExternalId(); + String backupRentionPeriod = getClient(vm.getDataCenterId()).getBackupPolicyRetentionInterval(backupProviderPolicyId); + + if ( backupRentionPeriod == null ) { + LOG.warn("There is no retention setting for NAS Policy, setting default for 1 day"); + backupRentionPeriod = "1 Day"; + } + + // Get Cluster + clusterName = getVMHypervisorCluster(hostVO); + String networkerBackupScr = "/usr/share/cloudstack-common/scripts/vm/hypervisor/kvm/nsrkvmbackup.sh"; + final Script script = new Script(networkerBackupScr); + script.add("-s"); + script.add(networkerServer); + script.add("-R"); + script.add("'"+backupRentionPeriod+"'"); + script.add("-P"); + script.add(NASMediaPool.valueIn(vm.getDataCenterId())); + script.add("-c"); + script.add(clusterName); + script.add("-u"); + script.add(vm.getUuid()); + script.add("-t"); + script.add(vm.getName()); + if ( Boolean.TRUE.equals(NASClientVerboseLogs.value()) ) + script.add("-v"); + + LOG.debug("Starting backup for VM ID " + vm.getUuid() + " on NAS provider"); + Date backupJobStart = new Date(); + + String saveTime = executeBackupCommand(hostVO, credentials.first(), credentials.second(), script.toString()); + LOG.info ("NAS finished backup job for vm " + vm.getName() + " with saveset Time: " + saveTime); + BackupVO backup = getClient(vm.getDataCenterId()).registerBackupForVm(vm, backupJobStart, saveTime); + if (backup != null) { + backupDao.persist(backup); + return true; + } else { + LOG.error("Could not register backup for vm " + vm.getName() + " with saveset Time: " + saveTime); + // We need to handle this rare situation where backup is successful but can't be registered properly. + return false; + } + } + + @Override + public boolean deleteBackup(Backup backup, boolean forced) { + + final Long zoneId = backup.getZoneId(); + final String externalBackupId = backup.getExternalId(); + + if (getClient(zoneId).deleteBackupForVM(externalBackupId)) { + LOG.debug("NAS successfully deleted backup with id " + externalBackupId); + return true; + } else { + LOG.debug("There was an error removing the backup with id " + externalBackupId + " from NAS"); + } + return false; + } + + @Override + public Map getBackupMetrics(Long zoneId, List vms) { + final Map metrics = new HashMap<>(); + Long vmBackupSize=0L; + Long vmBackupProtectedSize=0L; + + if (CollectionUtils.isEmpty(vms)) { + LOG.warn("Unable to get VM Backup Metrics because the list of VMs is empty."); + return metrics; + } + + for (final VirtualMachine vm : vms) { + for ( Backup.VolumeInfo thisVMVol : vm.getBackupVolumeList()) { + vmBackupSize += (thisVMVol.getSize() / 1024L / 1024L); + } + final ArrayList vmBackups = getClient(zoneId).getBackupsForVm(vm); + for ( String vmBackup : vmBackups ) { + NASBackup vmNwBackup = getClient(zoneId).getNASBackupInfo(vmBackup); + vmBackupProtectedSize+= vmNwBackup.getSize().getValue() / 1024L; + } + Backup.Metric vmBackupMetric = new Backup.Metric(vmBackupSize,vmBackupProtectedSize); + LOG.debug(String.format("Metrics for VM [uuid: %s, name: %s] is [backup size: %s, data size: %s].", vm.getUuid(), + vm.getInstanceName(), vmBackupMetric.getBackupSize(), vmBackupMetric.getDataSize())); + metrics.put(vm, vmBackupMetric); + } + return metrics; + } + + @Override + public void syncBackups(VirtualMachine vm, Backup.Metric metric) { + final Long zoneId = vm.getDataCenterId(); + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + final List backupsInDb = backupDao.listByVmId(null, vm.getId()); + final ArrayList backupsInNAS = getClient(zoneId).getBackupsForVm(vm); + final List removeList = backupsInDb.stream().map(InternalIdentity::getId).collect(Collectors.toList()); + for (final String networkerBackupId : backupsInNAS ) { + Long vmBackupSize=0L; + boolean backupExists = false; + for (final Backup backupInDb : backupsInDb) { + LOG.debug("Checking if Backup with external ID " + backupInDb.getName() + " for VM " + backupInDb.getVmId() + "is valid"); + if ( networkerBackupId.equals(backupInDb.getExternalId()) ) { + LOG.debug("Found Backup with id " + backupInDb.getId() + " in both Database and NAS"); + backupExists = true; + removeList.remove(backupInDb.getId()); + if (metric != null) { + LOG.debug(String.format("Update backup with [uuid: %s, external id: %s] from [size: %s, protected size: %s] to [size: %s, protected size: %s].", + backupInDb.getUuid(), backupInDb.getExternalId(), backupInDb.getSize(), backupInDb.getProtectedSize(), + metric.getBackupSize(), metric.getDataSize())); + ((BackupVO) backupInDb).setSize(metric.getBackupSize()); + ((BackupVO) backupInDb).setProtectedSize(metric.getDataSize()); + backupDao.update(backupInDb.getId(), ((BackupVO) backupInDb)); + } + break; + } + } + if (backupExists) { + continue; + } + // Technically an administrator can manually create a backup for a VM by utilizing the KVM scripts + // with the proper parameters. So we will register any backups taken on the NAS side from + // outside Cloudstack. If ever NAS will support KVM out of the box this functionality also will + // ensure that SLA like backups will be found and registered. + NASBackup strayNASBackup = getClient(vm.getDataCenterId()).getNASBackupInfo(networkerBackupId); + // Since running backups are already present in NAS Server but not completed + // make sure the backup is not in progress at this time. + if ( strayNASBackup.getCompletionTime() != null) { + BackupVO strayBackup = new BackupVO(); + strayBackup.setVmId(vm.getId()); + strayBackup.setExternalId(strayNASBackup.getId()); + strayBackup.setType(strayNASBackup.getType()); + SimpleDateFormat formatterDateTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); + try { + strayBackup.setDate(formatterDateTime.parse(strayNASBackup.getSaveTime())); + } catch (ParseException e) { + String msg = String.format("Unable to parse date [%s].", strayNASBackup.getSaveTime()); + LOG.error(msg, e); + throw new CloudRuntimeException(msg, e); + } + strayBackup.setStatus(Backup.Status.BackedUp); + for ( Backup.VolumeInfo thisVMVol : vm.getBackupVolumeList()) { + vmBackupSize += (thisVMVol.getSize() / 1024L /1024L); + } + strayBackup.setSize(vmBackupSize); + strayBackup.setProtectedSize(strayNASBackup.getSize().getValue() / 1024L ); + strayBackup.setBackupOfferingId(vm.getBackupOfferingId()); + strayBackup.setAccountId(vm.getAccountId()); + strayBackup.setDomainId(vm.getDomainId()); + strayBackup.setZoneId(vm.getDataCenterId()); + LOG.debug(String.format("Creating a new entry in backups: [uuid: %s, vm_id: %s, external_id: %s, type: %s, date: %s, backup_offering_id: %s, account_id: %s, " + + "domain_id: %s, zone_id: %s].", strayBackup.getUuid(), strayBackup.getVmId(), strayBackup.getExternalId(), + strayBackup.getType(), strayBackup.getDate(), strayBackup.getBackupOfferingId(), strayBackup.getAccountId(), + strayBackup.getDomainId(), strayBackup.getZoneId())); + backupDao.persist(strayBackup); + LOG.warn("Added backup found in provider with ID: [" + strayBackup.getId() + "]"); + } else { + LOG.debug ("Backup is in progress, skipping addition for this run"); + } + } + for (final Long backupIdToRemove : removeList) { + LOG.warn(String.format("Removing backup with ID: [%s].", backupIdToRemove)); + backupDao.remove(backupIdToRemove); + } + } + }); + } + + @Override + public boolean willDeleteBackupsOnOfferingRemoval() { return false; } +} diff --git a/plugins/backup/nas/src/main/resources/META-INF/cloudstack/nas/module.properties b/plugins/backup/nas/src/main/resources/META-INF/cloudstack/nas/module.properties new file mode 100644 index 000000000000..2e101ef02314 --- /dev/null +++ b/plugins/backup/nas/src/main/resources/META-INF/cloudstack/nas/module.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name=nas +parent=backup diff --git a/plugins/backup/nas/src/main/resources/META-INF/cloudstack/nas/spring-backup-nas-context.xml b/plugins/backup/nas/src/main/resources/META-INF/cloudstack/nas/spring-backup-nas-context.xml new file mode 100644 index 000000000000..635ca66fbdee --- /dev/null +++ b/plugins/backup/nas/src/main/resources/META-INF/cloudstack/nas/spring-backup-nas-context.xml @@ -0,0 +1,26 @@ + + + + + + + diff --git a/plugins/pom.xml b/plugins/pom.xml index 279067e2c97f..048f064a5481 100755 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -62,6 +62,7 @@ backup/dummy backup/networker + backup/nas ca/root-ca From 2604f812648725ce87537f21f3ac94ec80ff7376 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Fri, 26 Jul 2024 13:09:27 +0530 Subject: [PATCH 02/68] cleanup and leave TODOs Signed-off-by: Rohit Yadav --- .../cloudstack/backup/NASBackupProvider.java | 478 +++--------------- 1 file changed, 61 insertions(+), 417 deletions(-) diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index b33d04c877dc..424024b82600 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.backup; import com.cloud.dc.dao.ClusterDao; +import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; @@ -27,44 +28,22 @@ import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.utils.Pair; -import com.cloud.utils.Ternary; import com.cloud.utils.component.AdapterBase; -import com.cloud.utils.db.Transaction; -import com.cloud.utils.db.TransactionCallbackNoReturn; -import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.ssh.SshHelper; -import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; -import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.BackupOfferingDaoImpl; -import org.apache.cloudstack.backup.networker.NASClient; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.commons.collections.CollectionUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; -import org.apache.xml.utils.URI; -import org.apache.cloudstack.backup.networker.api.NASBackup; import javax.inject.Inject; -import java.net.URISyntaxException; -import java.security.KeyManagementException; -import java.security.NoSuchAlgorithmException; -import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.HashMap; -import java.util.Date; -import java.util.Objects; -import java.util.UUID; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import com.cloud.utils.script.Script; public class NASBackupProvider extends AdapterBase implements BackupProvider, Configurable { private static final Logger LOG = LogManager.getLogger(NASBackupProvider.class); @@ -105,7 +84,7 @@ public String getName() { @Override public String getDescription() { - return "NAS Backup Plugin"; + return "NAS KVM Backup Plugin"; } @Override @@ -113,364 +92,95 @@ public String getConfigComponentName() { return BackupService.class.getSimpleName(); } - protected HostVO getLastVMHypervisorHost(VirtualMachine vm) { - HostVO host; + protected Host getLastVMHypervisorHost(VirtualMachine vm) { Long hostId = vm.getLastHostId(); - if (hostId == null) { LOG.debug("Cannot find last host for vm. This should never happen, please check your database."); return null; } - host = hostDao.findById(hostId); + Host host = hostDao.findById(hostId); - if ( host.getStatus() == Status.Up ) { + if (host.getStatus() == Status.Up) { return host; } else { - // Try to find a host in the same cluster - List altClusterHosts = hostDao.findHypervisorHostInCluster(host.getClusterId()); - for (final HostVO candidateClusterHost : altClusterHosts) { - if ( candidateClusterHost.getStatus() == Status.Up ) { - LOG.debug("Found Host " + candidateClusterHost.getName()); - return candidateClusterHost; + // Try to find any Up host in the same cluster + for (final Host hostInCluster : hostDao.findHypervisorHostInCluster(host.getClusterId())) { + if (hostInCluster.getStatus() == Status.Up) { + LOG.debug("Found Host " + hostInCluster.getName()); + return hostInCluster; } } } - // Try to find a Host in the zone - List altZoneHosts = hostDao.findByDataCenterId(host.getDataCenterId()); - for (final HostVO candidateZoneHost : altZoneHosts) { - if ( candidateZoneHost.getStatus() == Status.Up && candidateZoneHost.getHypervisorType() == Hypervisor.HypervisorType.KVM ) { - LOG.debug("Found Host " + candidateZoneHost.getName()); - return candidateZoneHost; + // Try to find any Host in the zone + for (final HostVO hostInZone : hostDao.listByDataCenterIdAndHypervisorType(host.getDataCenterId(), Hypervisor.HypervisorType.KVM)) { + if (hostInZone.getStatus() == Status.Up) { + LOG.debug("Found Host " + hostInZone.getName()); + return hostInZone; } } return null; } - protected HostVO getRunningVMHypervisorHost(VirtualMachine vm) { - - HostVO host; + protected Host getRunningVMHypervisorHost(VirtualMachine vm) { Long hostId = vm.getHostId(); - if (hostId == null) { throw new CloudRuntimeException("Unable to find the HYPERVISOR for " + vm.getName() + ". Make sure the virtual machine is running"); } - - host = hostDao.findById(hostId); - - return host; - } - - protected String getVMHypervisorCluster(HostVO host) { - return clusterDao.findById(host.getClusterId()).getName(); - } - - protected Ternary getKVMHyperisorCredentials(HostVO host) { - String username = null; - String password = null; - - if (host != null && host.getHypervisorType() == Hypervisor.HypervisorType.KVM) { - hostDao.loadDetails(host); - password = host.getDetail("password"); - username = host.getDetail("username"); - } - if ( password == null || username == null) { - throw new CloudRuntimeException("Cannot find login credentials for HYPERVISOR " + Objects.requireNonNull(host).getUuid()); - } - - return new Ternary<>(username, password, null); - } - - private String executeBackupCommand(HostVO host, String username, String password, String command) { - String nstRegex = "\\bcompleted savetime=([0-9]{10})"; - Pattern saveTimePattern = Pattern.compile(nstRegex); - - try { - Pair response = SshHelper.sshExecute(host.getPrivateIpAddress(), 22, - username, null, password, command, 120000, 120000, 3600000); - if (!response.first()) { - LOG.error(String.format("Backup Script failed on HYPERVISOR %s due to: %s", host, response.second())); - } else { - LOG.debug(String.format("NAS Backup Results: %s", response.second())); - } - Matcher saveTimeMatcher = saveTimePattern.matcher(response.second()); - if (saveTimeMatcher.find()) { - LOG.debug(String.format("Got saveTimeMatcher: %s", saveTimeMatcher.group(1))); - return saveTimeMatcher.group(1); - } - } catch (final Exception e) { - throw new CloudRuntimeException(String.format("Failed to take backup on host %s due to: %s", host.getName(), e.getMessage())); - } - - return null; - } - private boolean executeRestoreCommand(HostVO host, String username, String password, String command) { - - try { - Pair response = SshHelper.sshExecute(host.getPrivateIpAddress(), 22, - username, null, password, command, 120000, 120000, 3600000); - - if (!response.first()) { - LOG.error(String.format("Restore Script failed on HYPERVISOR %s due to: %s", host, response.second())); - } else { - LOG.debug(String.format("NAS Restore Results: %s",response.second())); - return true; - } - } catch (final Exception e) { - throw new CloudRuntimeException(String.format("Failed to restore backup on host %s due to: %s", host.getName(), e.getMessage())); - } - return false; - } - - private Object getClient(final Long zoneId) { - try { - return null; // TODO: as a native object we don't need an API client - } catch (Exception e) { - LOG.error("Failed to build NAS API client due to: ", e); - } - throw new CloudRuntimeException("Failed to build NAS API client"); - } - - @Override - public List listBackupOfferings(Long zoneId) { - List policies = new ArrayList<>(); - for (final BackupOffering policy : getClient(zoneId).listPolicies()) { - if (!policy.getName().contains(BACKUP_IDENTIFIER)) { - policies.add(policy); - } - } - - return policies; - } - - @Override - public boolean isValidProviderOffering(Long zoneId, String uuid) { - List policies = listBackupOfferings(zoneId); - if (CollectionUtils.isEmpty(policies)) { - return false; - } - for (final BackupOffering policy : policies) { - if (Objects.equals(policy.getExternalId(), uuid)) { - return true; - } - } - return false; - } - - @Override - public boolean assignVMToBackupOffering(VirtualMachine vm, BackupOffering backupOffering) { return true; } - - @Override - public boolean removeVMFromBackupOffering(VirtualMachine vm) { - LOG.debug("Removing VirtualMachine from Backup offering and Deleting any existing backups"); - - List backupsTaken = getClient(vm.getDataCenterId()).getBackupsForVm(vm); - - for (String backupId : backupsTaken) { - LOG.debug("Trying to remove backup with id" + backupId); - getClient(vm.getDataCenterId()).deleteBackupForVM(backupId); - } - - return true; + return hostDao.findById(hostId); } @Override public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { - String networkerServer; - HostVO hostVO; - final Long zoneId = backup.getZoneId(); - final String externalBackupId = backup.getExternalId(); - final NASBackup networkerBackup=getClient(zoneId).getNASBackupInfo(externalBackupId); - final String SSID = networkerBackup.getShortId(); LOG.debug("Restoring vm " + vm.getUuid() + "from backup " + backup.getUuid() + " on the NAS Backup Provider"); - if ( SSID.isEmpty() ) { - LOG.debug("There was an error retrieving the SSID for backup with id " + externalBackupId + " from NAS"); - return false; - } - // Find where the VM was last running - hostVO = getLastVMHypervisorHost(vm); - // Get credentials for that host - Ternary credentials = getKVMHyperisorCredentials(hostVO); - LOG.debug("The SSID was reported successfully " + externalBackupId); - try { - networkerServer = getUrlDomain(NASUrl.value()); - } catch (URISyntaxException e) { - throw new CloudRuntimeException(String.format("Failed to convert API to HOST : %s", e)); - } - String networkerRestoreScr = "/usr/share/cloudstack-common/scripts/vm/hypervisor/kvm/nsrkvmrestore.sh"; - final Script script = new Script(networkerRestoreScr); - script.add("-s"); - script.add(networkerServer); - script.add("-S"); - script.add(SSID); - - if ( Boolean.TRUE.equals(NASClientVerboseLogs.value()) ) - script.add("-v"); - - Date restoreJobStart = new Date(); - LOG.debug("Starting Restore for VM ID " + vm.getUuid() + " and SSID" + SSID + " at " + restoreJobStart); - - if ( executeRestoreCommand(hostVO, credentials.first(), credentials.second(), script.toString()) ) { - Date restoreJobEnd = new Date(); - LOG.debug("Restore Job for SSID " + SSID + " completed successfully at " + restoreJobEnd); - return true; - } else { - LOG.debug("Restore Job for SSID " + SSID + " failed!"); - return false; - } + final Host hostVO = getLastVMHypervisorHost(vm); + + // TODO: get KVM agent to restore VM backup + + return true; } @Override public Pair restoreBackedUpVolume(Backup backup, String volumeUuid, String hostIp, String dataStoreUuid) { - String networkerServer; - VolumeVO volume = volumeDao.findByUuid(volumeUuid); - VMInstanceVO backupSourceVm = vmInstanceDao.findById(backup.getVmId()); - StoragePoolHostVO dataStore = storagePoolHostDao.findByUuid(dataStoreUuid); - HostVO hostVO = hostDao.findByIp(hostIp); - + final Volume volume = volumeDao.findByUuid(volumeUuid); + final VirtualMachine backupSourceVm = vmInstanceDao.findById(backup.getVmId()); + final StoragePoolHostVO dataStore = storagePoolHostDao.findByUuid(dataStoreUuid); + final HostVO hostVO = hostDao.findByIp(hostIp); final Long zoneId = backup.getZoneId(); - final String externalBackupId = backup.getExternalId(); - final NASBackup networkerBackup=getClient(zoneId).getNASBackupInfo(externalBackupId); - final String SSID = networkerBackup.getShortId(); - final String clusterName = networkerBackup.getClientHostname(); - final String destinationNASClient = hostVO.getName().split("\\.")[0]; - Long restoredVolumeDiskSize = 0L; - - LOG.debug("Restoring volume " + volumeUuid + "from backup " + backup.getUuid() + " on the NAS Backup Provider"); - - if ( SSID.isEmpty() ) { - LOG.debug("There was an error retrieving the SSID for backup with id " + externalBackupId + " from NAS"); - return null; - } - Ternary credentials = getKVMHyperisorCredentials(hostVO); - LOG.debug("The SSID was reported successfully " + externalBackupId); - try { - networkerServer = getUrlDomain(NASUrl.value()); - } catch (URISyntaxException e) { - throw new CloudRuntimeException(String.format("Failed to convert API to HOST : %s", e)); - } - - // Find volume size from backup vols - for ( Backup.VolumeInfo VMVolToRestore : backupSourceVm.getBackupVolumeList()) { - if (VMVolToRestore.getUuid().equals(volumeUuid)) - restoredVolumeDiskSize = (VMVolToRestore.getSize()); - } + // TODO: Find volume from backup volumes VolumeVO restoredVolume = new VolumeVO(Volume.Type.DATADISK, null, backup.getZoneId(), backup.getDomainId(), backup.getAccountId(), 0, null, backup.getSize(), null, null, null); - restoredVolume.setName("RV-"+volume.getName()); - restoredVolume.setProvisioningType(volume.getProvisioningType()); - restoredVolume.setUpdated(new Date()); - restoredVolume.setUuid(UUID.randomUUID().toString()); - restoredVolume.setRemoved(null); - restoredVolume.setDisplayVolume(true); - restoredVolume.setPoolId(volume.getPoolId()); - restoredVolume.setPath(restoredVolume.getUuid()); - restoredVolume.setState(Volume.State.Copying); - restoredVolume.setSize(restoredVolumeDiskSize); - restoredVolume.setDiskOfferingId(volume.getDiskOfferingId()); - + // TODO: fill restored volume VO try { volumeDao.persist(restoredVolume); } catch (Exception e) { throw new CloudRuntimeException("Unable to craft restored volume due to: "+e); } - String networkerRestoreScr = "/usr/share/cloudstack-common/scripts/vm/hypervisor/kvm/nsrkvmrestore.sh"; - final Script script = new Script(networkerRestoreScr); - script.add("-s"); - script.add(networkerServer); - script.add("-c"); - script.add(clusterName); - script.add("-d"); - script.add(destinationNASClient); - script.add("-n"); - script.add(restoredVolume.getUuid()); - script.add("-p"); - script.add(dataStore.getLocalPath()); - script.add("-a"); - script.add(volume.getUuid()); - - if ( Boolean.TRUE.equals(NASClientVerboseLogs.value()) ) - script.add("-v"); - - Date restoreJobStart = new Date(); - LOG.debug("Starting Restore for Volume UUID " + volume.getUuid() + " and SSID" + SSID + " at " + restoreJobStart); - - if ( executeRestoreCommand(hostVO, credentials.first(), credentials.second(), script.toString()) ) { - Date restoreJobEnd = new Date(); - LOG.debug("Restore Job for SSID " + SSID + " completed successfully at " + restoreJobEnd); - return new Pair<>(true,restoredVolume.getUuid()); - } else { - volumeDao.expunge(restoredVolume.getId()); - LOG.debug("Restore Job for SSID " + SSID + " failed!"); - return null; - } + + // TODO: get KVM agent to copy/restore the specific volume to datastore + + return null; } @Override public boolean takeBackup(VirtualMachine vm) { - String networkerServer; - String clusterName; - - try { - networkerServer = getUrlDomain(NASUrl.value()); - } catch (URISyntaxException e) { - throw new CloudRuntimeException(String.format("Failed to convert API to HOST : %s", e)); - } - // Find where the VM is currently running - HostVO hostVO = getRunningVMHypervisorHost(vm); - // Get credentials for that host - Ternary credentials = getKVMHyperisorCredentials(hostVO); + Host hostVO = getRunningVMHypervisorHost(vm); // Get retention Period for our Backup BackupOfferingVO vmBackupOffering = new BackupOfferingDaoImpl().findById(vm.getBackupOfferingId()); - final String backupProviderPolicyId = vmBackupOffering.getExternalId(); - String backupRentionPeriod = getClient(vm.getDataCenterId()).getBackupPolicyRetentionInterval(backupProviderPolicyId); - - if ( backupRentionPeriod == null ) { - LOG.warn("There is no retention setting for NAS Policy, setting default for 1 day"); - backupRentionPeriod = "1 Day"; - } - - // Get Cluster - clusterName = getVMHypervisorCluster(hostVO); - String networkerBackupScr = "/usr/share/cloudstack-common/scripts/vm/hypervisor/kvm/nsrkvmbackup.sh"; - final Script script = new Script(networkerBackupScr); - script.add("-s"); - script.add(networkerServer); - script.add("-R"); - script.add("'"+backupRentionPeriod+"'"); - script.add("-P"); - script.add(NASMediaPool.valueIn(vm.getDataCenterId())); - script.add("-c"); - script.add(clusterName); - script.add("-u"); - script.add(vm.getUuid()); - script.add("-t"); - script.add(vm.getName()); - if ( Boolean.TRUE.equals(NASClientVerboseLogs.value()) ) - script.add("-v"); LOG.debug("Starting backup for VM ID " + vm.getUuid() + " on NAS provider"); - Date backupJobStart = new Date(); - - String saveTime = executeBackupCommand(hostVO, credentials.first(), credentials.second(), script.toString()); - LOG.info ("NAS finished backup job for vm " + vm.getName() + " with saveset Time: " + saveTime); - BackupVO backup = getClient(vm.getDataCenterId()).registerBackupForVm(vm, backupJobStart, saveTime); - if (backup != null) { - backupDao.persist(backup); - return true; - } else { - LOG.error("Could not register backup for vm " + vm.getName() + " with saveset Time: " + saveTime); - // We need to handle this rare situation where backup is successful but can't be registered properly. - return false; - } + // TODO: initiate backup to NAS by KVM agent + // TODO: perisist object based on Answer: backupDao.persist(backup); + + return true; } @Override @@ -479,12 +189,8 @@ public boolean deleteBackup(Backup backup, boolean forced) { final Long zoneId = backup.getZoneId(); final String externalBackupId = backup.getExternalId(); - if (getClient(zoneId).deleteBackupForVM(externalBackupId)) { - LOG.debug("NAS successfully deleted backup with id " + externalBackupId); - return true; - } else { - LOG.debug("There was an error removing the backup with id " + externalBackupId + " from NAS"); - } + // TODO: delete backup from NAS + return false; } @@ -500,14 +206,7 @@ public Map getBackupMetrics(Long zoneId, List vmBackups = getClient(zoneId).getBackupsForVm(vm); - for ( String vmBackup : vmBackups ) { - NASBackup vmNwBackup = getClient(zoneId).getNASBackupInfo(vmBackup); - vmBackupProtectedSize+= vmNwBackup.getSize().getValue() / 1024L; - } + // TODO: get per VM backup usage Backup.Metric vmBackupMetric = new Backup.Metric(vmBackupSize,vmBackupProtectedSize); LOG.debug(String.format("Metrics for VM [uuid: %s, name: %s] is [backup size: %s, data size: %s].", vm.getUuid(), vm.getInstanceName(), vmBackupMetric.getBackupSize(), vmBackupMetric.getDataSize())); @@ -516,84 +215,29 @@ public Map getBackupMetrics(Long zoneId, List listBackupOfferings(Long zoneId) { + return new ArrayList<>(); + } + + @Override + public boolean isValidProviderOffering(Long zoneId, String uuid) { + // TODO + return true; + } + + @Override + public boolean assignVMToBackupOffering(VirtualMachine vm, BackupOffering backupOffering) { + return Hypervisor.HypervisorType.KVM.equals(vm.getHypervisorType()); + } + + @Override + public boolean removeVMFromBackupOffering(VirtualMachine vm) { + return true; + } + @Override public void syncBackups(VirtualMachine vm, Backup.Metric metric) { - final Long zoneId = vm.getDataCenterId(); - Transaction.execute(new TransactionCallbackNoReturn() { - @Override - public void doInTransactionWithoutResult(TransactionStatus status) { - final List backupsInDb = backupDao.listByVmId(null, vm.getId()); - final ArrayList backupsInNAS = getClient(zoneId).getBackupsForVm(vm); - final List removeList = backupsInDb.stream().map(InternalIdentity::getId).collect(Collectors.toList()); - for (final String networkerBackupId : backupsInNAS ) { - Long vmBackupSize=0L; - boolean backupExists = false; - for (final Backup backupInDb : backupsInDb) { - LOG.debug("Checking if Backup with external ID " + backupInDb.getName() + " for VM " + backupInDb.getVmId() + "is valid"); - if ( networkerBackupId.equals(backupInDb.getExternalId()) ) { - LOG.debug("Found Backup with id " + backupInDb.getId() + " in both Database and NAS"); - backupExists = true; - removeList.remove(backupInDb.getId()); - if (metric != null) { - LOG.debug(String.format("Update backup with [uuid: %s, external id: %s] from [size: %s, protected size: %s] to [size: %s, protected size: %s].", - backupInDb.getUuid(), backupInDb.getExternalId(), backupInDb.getSize(), backupInDb.getProtectedSize(), - metric.getBackupSize(), metric.getDataSize())); - ((BackupVO) backupInDb).setSize(metric.getBackupSize()); - ((BackupVO) backupInDb).setProtectedSize(metric.getDataSize()); - backupDao.update(backupInDb.getId(), ((BackupVO) backupInDb)); - } - break; - } - } - if (backupExists) { - continue; - } - // Technically an administrator can manually create a backup for a VM by utilizing the KVM scripts - // with the proper parameters. So we will register any backups taken on the NAS side from - // outside Cloudstack. If ever NAS will support KVM out of the box this functionality also will - // ensure that SLA like backups will be found and registered. - NASBackup strayNASBackup = getClient(vm.getDataCenterId()).getNASBackupInfo(networkerBackupId); - // Since running backups are already present in NAS Server but not completed - // make sure the backup is not in progress at this time. - if ( strayNASBackup.getCompletionTime() != null) { - BackupVO strayBackup = new BackupVO(); - strayBackup.setVmId(vm.getId()); - strayBackup.setExternalId(strayNASBackup.getId()); - strayBackup.setType(strayNASBackup.getType()); - SimpleDateFormat formatterDateTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); - try { - strayBackup.setDate(formatterDateTime.parse(strayNASBackup.getSaveTime())); - } catch (ParseException e) { - String msg = String.format("Unable to parse date [%s].", strayNASBackup.getSaveTime()); - LOG.error(msg, e); - throw new CloudRuntimeException(msg, e); - } - strayBackup.setStatus(Backup.Status.BackedUp); - for ( Backup.VolumeInfo thisVMVol : vm.getBackupVolumeList()) { - vmBackupSize += (thisVMVol.getSize() / 1024L /1024L); - } - strayBackup.setSize(vmBackupSize); - strayBackup.setProtectedSize(strayNASBackup.getSize().getValue() / 1024L ); - strayBackup.setBackupOfferingId(vm.getBackupOfferingId()); - strayBackup.setAccountId(vm.getAccountId()); - strayBackup.setDomainId(vm.getDomainId()); - strayBackup.setZoneId(vm.getDataCenterId()); - LOG.debug(String.format("Creating a new entry in backups: [uuid: %s, vm_id: %s, external_id: %s, type: %s, date: %s, backup_offering_id: %s, account_id: %s, " - + "domain_id: %s, zone_id: %s].", strayBackup.getUuid(), strayBackup.getVmId(), strayBackup.getExternalId(), - strayBackup.getType(), strayBackup.getDate(), strayBackup.getBackupOfferingId(), strayBackup.getAccountId(), - strayBackup.getDomainId(), strayBackup.getZoneId())); - backupDao.persist(strayBackup); - LOG.warn("Added backup found in provider with ID: [" + strayBackup.getId() + "]"); - } else { - LOG.debug ("Backup is in progress, skipping addition for this run"); - } - } - for (final Long backupIdToRemove : removeList) { - LOG.warn(String.format("Removing backup with ID: [%s].", backupIdToRemove)); - backupDao.remove(backupIdToRemove); - } - } - }); } @Override From b961a4d70bb115cb74103a626a83d63b1bdb835e Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Fri, 26 Jul 2024 13:25:58 +0530 Subject: [PATCH 03/68] cleanup, refactor; add plugin config keys Signed-off-by: Rohit Yadav --- .../cloudstack/backup/NASBackupProvider.java | 106 ++++++++++-------- 1 file changed, 61 insertions(+), 45 deletions(-) diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 424024b82600..48a00be0c301 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -48,9 +48,20 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Configurable { private static final Logger LOG = LogManager.getLogger(NASBackupProvider.class); - private final ConfigKey NASDetails = new ConfigKey<>("Advanced", String.class, - "backup.plugin.nas.details", "Default", - "The NFS/NAS storage details", true, ConfigKey.Scope.Zone); + private final ConfigKey NasType = new ConfigKey<>("Advanced", String.class, + "backup.plugin.nas.target.type", "nfs", + "The NAS storage target type. Only supported: nfs and cephfs", true, ConfigKey.Scope.Zone); + private final ConfigKey NfsPool = new ConfigKey<>("Advanced", String.class, + "backup.plugin.nas.nfs.pool", "", + "The NFS NAS storage pool URL (format :", true, ConfigKey.Scope.Zone); + + private final ConfigKey CephFSPool = new ConfigKey<>("Advanced", String.class, + "backup.plugin.nas.cephfs.pool", "", + "The CephFS storage pool URL (format: :)", true, ConfigKey.Scope.Zone); + + private final ConfigKey CephFSPoolCredentials = new ConfigKey<>("Advanced", String.class, + "backup.plugin.nas.cephfs.credentials", "", + "The CephFS storage pool URL (format: )", true, ConfigKey.Scope.Zone); @Inject private BackupDao backupDao; @@ -70,28 +81,6 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Inject private VMInstanceDao vmInstanceDao; - @Override - public ConfigKey[] getConfigKeys() { - return new ConfigKey[]{ - NASDetails - }; - } - - @Override - public String getName() { - return "nas"; - } - - @Override - public String getDescription() { - return "NAS KVM Backup Plugin"; - } - - @Override - public String getConfigComponentName() { - return BackupService.class.getSimpleName(); - } - protected Host getLastVMHypervisorHost(VirtualMachine vm) { Long hostId = vm.getLastHostId(); if (hostId == null) { @@ -129,6 +118,20 @@ protected Host getRunningVMHypervisorHost(VirtualMachine vm) { return hostDao.findById(hostId); } + @Override + public boolean takeBackup(VirtualMachine vm) { + // Find where the VM is currently running + Host hostVO = getRunningVMHypervisorHost(vm); + // Get retention Period for our Backup + BackupOfferingVO vmBackupOffering = new BackupOfferingDaoImpl().findById(vm.getBackupOfferingId()); + + LOG.debug("Starting backup for VM ID " + vm.getUuid() + " on NAS provider"); + // TODO: initiate backup to NAS by KVM agent + // TODO: perisist object based on Answer: backupDao.persist(backup); + + return true; + } + @Override public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { final Long zoneId = backup.getZoneId(); @@ -169,20 +172,6 @@ public Pair restoreBackedUpVolume(Backup backup, String volumeU return null; } - @Override - public boolean takeBackup(VirtualMachine vm) { - // Find where the VM is currently running - Host hostVO = getRunningVMHypervisorHost(vm); - // Get retention Period for our Backup - BackupOfferingVO vmBackupOffering = new BackupOfferingDaoImpl().findById(vm.getBackupOfferingId()); - - LOG.debug("Starting backup for VM ID " + vm.getUuid() + " on NAS provider"); - // TODO: initiate backup to NAS by KVM agent - // TODO: perisist object based on Answer: backupDao.persist(backup); - - return true; - } - @Override public boolean deleteBackup(Backup backup, boolean forced) { @@ -215,6 +204,25 @@ public Map getBackupMetrics(Long zoneId, List listBackupOfferings(Long zoneId) { return new ArrayList<>(); @@ -227,19 +235,27 @@ public boolean isValidProviderOffering(Long zoneId, String uuid) { } @Override - public boolean assignVMToBackupOffering(VirtualMachine vm, BackupOffering backupOffering) { - return Hypervisor.HypervisorType.KVM.equals(vm.getHypervisorType()); + public ConfigKey[] getConfigKeys() { + return new ConfigKey[]{ + NasType, + NfsPool, + CephFSPool, + CephFSPoolCredentials + }; } @Override - public boolean removeVMFromBackupOffering(VirtualMachine vm) { - return true; + public String getName() { + return "nas"; } @Override - public void syncBackups(VirtualMachine vm, Backup.Metric metric) { + public String getDescription() { + return "NAS KVM Backup Plugin"; } @Override - public boolean willDeleteBackupsOnOfferingRemoval() { return false; } + public String getConfigComponentName() { + return BackupService.class.getSimpleName(); + } } From 9a8bbbbc08f99657475a469b2c6069af8b330fef Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Fri, 26 Jul 2024 13:37:37 +0530 Subject: [PATCH 04/68] WIP B&R tool Signed-off-by: Rohit Yadav --- scripts/vm/hypervisor/kvm/nasbackup.sh | 45 ++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 scripts/vm/hypervisor/kvm/nasbackup.sh diff --git a/scripts/vm/hypervisor/kvm/nasbackup.sh b/scripts/vm/hypervisor/kvm/nasbackup.sh new file mode 100755 index 000000000000..053c97243cc4 --- /dev/null +++ b/scripts/vm/hypervisor/kvm/nasbackup.sh @@ -0,0 +1,45 @@ +#!/bin/bash +## Licensed to the Apache Software Foundation (ASF) under one +## or more contributor license agreements. See the NOTICE file +## distributed with this work for additional information +## regarding copyright ownership. The ASF licenses this file +## to you under the Apache License, Version 2.0 (the +## "License"); you may not use this file except in compliance +## with the License. You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, +## software distributed under the License is distributed on an +## "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +## KIND, either express or implied. See the License for the +## specific language governing permissions and limitations +## under the License. + +# CloudStack B&R NAS (KVM) Backup and Recovery Tool + +# TODO: do libvirt version & other dependency checks + +# TODO: logging & args/opts handling + +backup_domain() { + domain=$1 # TODO: get as opt? + # TODO: santiy checks on domain + # get domblklist + # gather external snapshot diskspec/target + # ensure the target nas is mounted / path is known or available + # virsh snapshot-create-as -> snapshot domain with --no-metadata--atomic --quiesce (?) --disk-only + # merge/commit target -> domblklist; blockcommit + # backup this domain & its disks, xml completely + # cleanup snapshot(s) +} + +restore_all_volumes() { + # check and mount nas target & copy files + # check and restore all volumes +} + +restore_volume() { + # check and mount nas target & copy files + # check and restore specific volume (qcow2) +} From d5503e7735ef7216b0aef11844e996e5d469c8da Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Thu, 1 Aug 2024 13:16:52 +0530 Subject: [PATCH 05/68] add cmd-answer classes Signed-off-by: Rohit Yadav --- .../cloudstack/backup/BackupAnswer.java | 59 ++++++++++++++++ .../cloudstack/backup/TakeBackupCommand.java | 68 +++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java create mode 100644 core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java diff --git a/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java new file mode 100644 index 000000000000..c73ad3ab0826 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java @@ -0,0 +1,59 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Answer; + +public class BackupAnswer extends Answer { + Long size; + Long virtualSize; + String path; + + public BackupAnswer(TakeBackupCommand command, Long size, Long virtualSize, String path) { + super(command); + this.size = size; + this.virtualSize = virtualSize; + this.path = path; + } + + public Long getSize() { + return size; + } + + public void setSize(Long size) { + this.size = size; + } + + public Long getVirtualSize() { + return virtualSize; + } + + public void setVirtualSize(Long virtualSize) { + this.virtualSize = virtualSize; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java new file mode 100644 index 000000000000..a72c6744e0b6 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java @@ -0,0 +1,68 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; +import com.cloud.agent.api.LogLevel; + +import java.util.Map; + +public class TakeBackupCommand extends Command { + private String vmName; + private String backupStoragePath; + @LogLevel(LogLevel.Log4jLevel.Off) + private Map details; + + public TakeBackupCommand(String vmName, String backupStoragePath, Map details) { + super(); + this.vmName = vmName; + this.backupStoragePath = backupStoragePath; + this.details = details; + } + + public String getVmName() { + return vmName; + } + + public void setVmName(String vmName) { + this.vmName = vmName; + } + + public String getBackupStoragePath() { + return backupStoragePath; + } + + public void setBackupStoragePath(String backupStoragePath) { + this.backupStoragePath = backupStoragePath; + } + + public Map getDetails() { + return details; + } + + public void setDetails(Map details) { + this.details = details; + } + + @Override + public boolean executeInSequence() { + return true; + } +} From f3bc03955a45cf703d42d1e5c44fcf458e0d83d4 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Thu, 1 Aug 2024 13:17:11 +0530 Subject: [PATCH 06/68] barebone take backup implementation Signed-off-by: Rohit Yadav --- .../cloudstack/backup/NASBackupProvider.java | 82 +++++++++++++++---- 1 file changed, 67 insertions(+), 15 deletions(-) diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 48a00be0c301..d18ab7aa7a39 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -16,7 +16,10 @@ // under the License. package org.apache.cloudstack.backup; +import com.cloud.agent.AgentManager; import com.cloud.dc.dao.ClusterDao; +import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; @@ -41,13 +44,14 @@ import org.apache.logging.log4j.LogManager; import javax.inject.Inject; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; import java.util.List; import java.util.Map; import java.util.HashMap; public class NASBackupProvider extends AdapterBase implements BackupProvider, Configurable { private static final Logger LOG = LogManager.getLogger(NASBackupProvider.class); - private final ConfigKey NasType = new ConfigKey<>("Advanced", String.class, "backup.plugin.nas.target.type", "nfs", "The NAS storage target type. Only supported: nfs and cephfs", true, ConfigKey.Scope.Zone); @@ -81,6 +85,21 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Inject private VMInstanceDao vmInstanceDao; + @Inject + private AgentManager agentManager; + + protected String getNasType(final Long zoneId) { + return NasType.valueIn(zoneId); + } + + protected String getBackupStoragePath(final Long zoneId) { + final String type = getNasType(zoneId); + if ("nfs".equalsIgnoreCase(type)) { + return NfsPool.valueIn(zoneId); + } + throw new CloudRuntimeException("NAS backup plugin not configured"); + } + protected Host getLastVMHypervisorHost(VirtualMachine vm) { Long hostId = vm.getLastHostId(); if (hostId == null) { @@ -120,16 +139,46 @@ protected Host getRunningVMHypervisorHost(VirtualMachine vm) { @Override public boolean takeBackup(VirtualMachine vm) { - // Find where the VM is currently running - Host hostVO = getRunningVMHypervisorHost(vm); - // Get retention Period for our Backup - BackupOfferingVO vmBackupOffering = new BackupOfferingDaoImpl().findById(vm.getBackupOfferingId()); + Host host = getRunningVMHypervisorHost(vm); + if (host == null || !Status.Up.equals(host.getStatus()) || !Hypervisor.HypervisorType.KVM.equals(host.getHypervisorType())) { + throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); + } - LOG.debug("Starting backup for VM ID " + vm.getUuid() + " on NAS provider"); - // TODO: initiate backup to NAS by KVM agent - // TODO: perisist object based on Answer: backupDao.persist(backup); + BackupOfferingVO backupOffering = new BackupOfferingDaoImpl().findById(vm.getBackupOfferingId()); + final String backupStoragePath = getBackupStoragePath(vm.getDataCenterId()); + final String nasType = getNasType(vm.getDataCenterId()); + final Map backupDetails = Map.of( + "type", nasType + ); - return true; + TakeBackupCommand command = new TakeBackupCommand(vm.getInstanceName(), backupStoragePath, backupDetails); + + BackupAnswer answer = null; + try { + answer = (BackupAnswer) agentManager.send(host.getId(), command); + } catch (AgentUnavailableException e) { + throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); + } catch (OperationTimedoutException e) { + throw new CloudRuntimeException("Operation to initiate backup timed out, please try again"); + } + + if (answer != null) { + BackupVO backup = new BackupVO(); + backup.setVmId(vm.getId()); + backup.setExternalId(String.format("%s|%s|%s", nasType, backupStoragePath, answer.getPath())); + backup.setType("FULL"); + backup.setDate(new Date()); + backup.setSize(answer.getSize()); + backup.setProtectedSize(answer.getVirtualSize()); + backup.setStatus(Backup.Status.BackedUp); + backup.setBackupOfferingId(vm.getBackupOfferingId()); + backup.setAccountId(vm.getAccountId()); + backup.setDomainId(vm.getDomainId()); + backup.setZoneId(vm.getDataCenterId()); + return backupDao.persist(backup) != null; + } + + return false; } @Override @@ -186,16 +235,18 @@ public boolean deleteBackup(Backup backup, boolean forced) { @Override public Map getBackupMetrics(Long zoneId, List vms) { final Map metrics = new HashMap<>(); - Long vmBackupSize=0L; - Long vmBackupProtectedSize=0L; - if (CollectionUtils.isEmpty(vms)) { LOG.warn("Unable to get VM Backup Metrics because the list of VMs is empty."); return metrics; } for (final VirtualMachine vm : vms) { - // TODO: get per VM backup usage + Long vmBackupSize = 0L; + Long vmBackupProtectedSize = 0L; + for (final Backup backup: backupDao.listByVmId(null, vm.getId())) { + vmBackupSize += backup.getSize(); + vmBackupProtectedSize += backup.getProtectedSize(); + } Backup.Metric vmBackupMetric = new Backup.Metric(vmBackupSize,vmBackupProtectedSize); LOG.debug(String.format("Metrics for VM [uuid: %s, name: %s] is [backup size: %s, data size: %s].", vm.getUuid(), vm.getInstanceName(), vmBackupMetric.getBackupSize(), vmBackupMetric.getDataSize())); @@ -221,16 +272,17 @@ public boolean willDeleteBackupsOnOfferingRemoval() { @Override public void syncBackups(VirtualMachine vm, Backup.Metric metric) { + // TODO: check and sum/return backups metrics on per VM basis } @Override public List listBackupOfferings(Long zoneId) { - return new ArrayList<>(); + BackupOffering policy = new BackupOfferingVO(zoneId, "default", getName(), "Default", "Default Backup Offering", true); + return List.of(policy); } @Override public boolean isValidProviderOffering(Long zoneId, String uuid) { - // TODO return true; } From f13769d33d275d303e2aaac28ea1770fff11e46e Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Thu, 1 Aug 2024 14:31:13 +0530 Subject: [PATCH 07/68] prototype backup e2e Signed-off-by: Rohit Yadav --- .../cloudstack/backup/BackupAnswer.java | 19 ++-- .../cloudstack/backup/TakeBackupCommand.java | 12 +- .../cloudstack/backup/NASBackupProvider.java | 11 +- .../resource/LibvirtComputingResource.java | 10 ++ .../LibvirtTakeBackupCommandWrapper.java | 52 +++++++++ scripts/vm/hypervisor/kvm/nasbackup.sh | 105 ++++++++++++++---- 6 files changed, 171 insertions(+), 38 deletions(-) create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java diff --git a/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java index c73ad3ab0826..c5e52850a33c 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java +++ b/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java @@ -21,16 +21,15 @@ import com.cloud.agent.api.Answer; +import java.util.Map; + public class BackupAnswer extends Answer { Long size; Long virtualSize; - String path; + Map volumes; - public BackupAnswer(TakeBackupCommand command, Long size, Long virtualSize, String path) { - super(command); - this.size = size; - this.virtualSize = virtualSize; - this.path = path; + public BackupAnswer(final TakeBackupCommand command, final boolean success, final String details) { + super(command, success, details); } public Long getSize() { @@ -49,11 +48,11 @@ public void setVirtualSize(Long virtualSize) { this.virtualSize = virtualSize; } - public String getPath() { - return path; + public Map getVolumes() { + return volumes; } - public void setPath(String path) { - this.path = path; + public void setVolumes(Map volumes) { + this.volumes = volumes; } } diff --git a/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java index a72c6744e0b6..91a49f1ce30d 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java @@ -26,13 +26,15 @@ public class TakeBackupCommand extends Command { private String vmName; + private String backupPath; private String backupStoragePath; @LogLevel(LogLevel.Log4jLevel.Off) private Map details; - public TakeBackupCommand(String vmName, String backupStoragePath, Map details) { + public TakeBackupCommand(String vmName, String backupPath, String backupStoragePath, Map details) { super(); this.vmName = vmName; + this.backupPath = backupPath; this.backupStoragePath = backupStoragePath; this.details = details; } @@ -45,6 +47,14 @@ public void setVmName(String vmName) { this.vmName = vmName; } + public String getBackupPath() { + return backupPath; + } + + public void setBackupPath(String backupPath) { + this.backupPath = backupPath; + } + public String getBackupStoragePath() { return backupStoragePath; } diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index d18ab7aa7a39..5dfaf4f3008d 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -43,6 +43,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import javax.inject.Inject; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; @@ -139,19 +140,21 @@ protected Host getRunningVMHypervisorHost(VirtualMachine vm) { @Override public boolean takeBackup(VirtualMachine vm) { - Host host = getRunningVMHypervisorHost(vm); + final Host host = getRunningVMHypervisorHost(vm); if (host == null || !Status.Up.equals(host.getStatus()) || !Hypervisor.HypervisorType.KVM.equals(host.getHypervisorType())) { throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); } - BackupOfferingVO backupOffering = new BackupOfferingDaoImpl().findById(vm.getBackupOfferingId()); + final BackupOfferingVO backupOffering = new BackupOfferingDaoImpl().findById(vm.getBackupOfferingId()); + final String backupStoragePath = getBackupStoragePath(vm.getDataCenterId()); final String nasType = getNasType(vm.getDataCenterId()); final Map backupDetails = Map.of( "type", nasType ); + final String backupPath = new SimpleDateFormat("yyyy.MM.dd.HH.mm.ss").format(new java.util.Date()); - TakeBackupCommand command = new TakeBackupCommand(vm.getInstanceName(), backupStoragePath, backupDetails); + TakeBackupCommand command = new TakeBackupCommand(vm.getInstanceName(), backupPath, backupStoragePath, backupDetails); BackupAnswer answer = null; try { @@ -165,7 +168,7 @@ public boolean takeBackup(VirtualMachine vm) { if (answer != null) { BackupVO backup = new BackupVO(); backup.setVmId(vm.getId()); - backup.setExternalId(String.format("%s|%s|%s", nasType, backupStoragePath, answer.getPath())); + backup.setExternalId(String.format("%s|%s|%s", nasType, backupStoragePath, backupPath)); backup.setType("FULL"); backup.setDate(new Date()); backup.setSize(answer.getSize()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index dec7f70e62fd..7ed6e7203deb 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -326,6 +326,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private String createTmplPath; private String heartBeatPath; private String vmActivityCheckPath; + private String nasBackupPath; private String securityGroupPath; private String ovsPvlanDhcpHostPath; private String ovsPvlanVmPath; @@ -714,6 +715,10 @@ public String getVmActivityCheckPath() { return vmActivityCheckPath; } + public String getNasBackupPath() { + return nasBackupPath; + } + public String getOvsPvlanDhcpHostPath() { return ovsPvlanDhcpHostPath; } @@ -984,6 +989,11 @@ public boolean configure(final String name, final Map params) th throw new ConfigurationException("Unable to find kvmvmactivity.sh"); } + nasBackupPath = Script.findScript(kvmScriptsDir, "nasbackup.sh"); + if (nasBackupPath == null) { + throw new ConfigurationException("Unable to find nasbackup.sh"); + } + createTmplPath = Script.findScript(storageScriptsDir, "createtmplt.sh"); if (createTmplPath == null) { throw new ConfigurationException("Unable to find the createtmplt.sh"); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java new file mode 100644 index 000000000000..740749376b43 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java @@ -0,0 +1,52 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.backup.BackupAnswer; +import org.apache.cloudstack.backup.TakeBackupCommand; + +import java.io.File; + +@ResourceWrapper(handles = TakeBackupCommand.class) +public class LibvirtTakeBackupCommandWrapper extends CommandWrapper { + @Override + public Answer execute(TakeBackupCommand command, LibvirtComputingResource libvirtComputingResource) { + final String vmName = command.getVmName(); + final String backupStoragePath = command.getBackupStoragePath(); + final String backupFolder = command.getBackupPath(); + Script cmd = new Script(libvirtComputingResource.getNasBackupPath(), libvirtComputingResource.getCmdsTimeout(), logger); + cmd.add("-b", vmName); + cmd.add("-s", backupStoragePath); + cmd.add("-p", String.format("%s%s%s", vmName, File.separator, backupFolder)); + String result = cmd.execute(); + if (result == null) { + logger.debug("Failed to take VM backup: " + result); + return new BackupAnswer(command, false, result); + } + BackupAnswer answer = new BackupAnswer(command, true, null); + answer.setSize(Long.valueOf(result)); + return answer; + } +} diff --git a/scripts/vm/hypervisor/kvm/nasbackup.sh b/scripts/vm/hypervisor/kvm/nasbackup.sh index 053c97243cc4..d354e991b748 100755 --- a/scripts/vm/hypervisor/kvm/nasbackup.sh +++ b/scripts/vm/hypervisor/kvm/nasbackup.sh @@ -16,30 +16,89 @@ ## specific language governing permissions and limitations ## under the License. -# CloudStack B&R NAS (KVM) Backup and Recovery Tool - -# TODO: do libvirt version & other dependency checks - -# TODO: logging & args/opts handling - -backup_domain() { - domain=$1 # TODO: get as opt? - # TODO: santiy checks on domain - # get domblklist - # gather external snapshot diskspec/target - # ensure the target nas is mounted / path is known or available - # virsh snapshot-create-as -> snapshot domain with --no-metadata--atomic --quiesce (?) --disk-only - # merge/commit target -> domblklist; blockcommit - # backup this domain & its disks, xml completely - # cleanup snapshot(s) -} +# CloudStack B&R NAS Backup and Recovery Tool for KVM + +# TODO: do libvirt/logging etc checks + +backup_vm() { + vm=$1 + path=$2 + storage=$3 + + mount_point=$(mktemp -d -t csbackup.XXXXX) + dest="$mount_point/$path" + + mount -t nfs $storage $mount_point + mkdir -p $dest + + echo "" > $dest/backup.xml + for disk in $(virsh -c qemu:///system domblklist $vm --details | awk '/disk/{print$3}'); do + echo "" >> $dest/backup.xml + done + echo "" > $dest/backup.xml -restore_all_volumes() { - # check and mount nas target & copy files - # check and restore all volumes + virsh -c qemu:///system backup-begin --domain $vm --backupxml $dest/backup.xml + virsh -c qemu:///system dumpxml $vm > $dest/domain-$vm.xml + sync + + # Print directory size + du -sb $dest | cut -f1 + umount $mount_point + rmdir $mount_point } -restore_volume() { - # check and mount nas target & copy files - # check and restore specific volume (qcow2) +OP="" +VM="" +PATH="" +NAS="" +TYPE="" + +function usage { + echo "" + echo "Usage: $0 -b -s -p " + echo "" + exit 1 } + +while [[ $# -gt 0 ]]; do + case $1 in + -b|--backup) + OP="backup" + VM="$2" + shift + shift + ;; + -s|--storage) + NAS="$2" + TYPE="nfs" + shift + shift + ;; + -p|--path) + PATH="$2" + shift + shift + ;; + -r|--recover) + OP="recover" + shift + ;; + -rv|--recover) + OP="recover-volume" + shift + ;; + -h|--help) + usage + shift + ;; + *) + echo "Invalid option: $1" + usage + ;; + esac +done + +if [ "$OP" = "backup" ]; then + backup_vm $VM $PATH $NAS +fi + From 1933f9e3eef78a6fec5dab66b049397e6f4212b4 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Thu, 1 Aug 2024 16:03:21 +0530 Subject: [PATCH 08/68] fix backup script Signed-off-by: Rohit Yadav --- .../cloudstack/backup/NASBackupProvider.java | 2 -- scripts/vm/hypervisor/kvm/nasbackup.sh | 20 +++++++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 5dfaf4f3008d..390458b214cc 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -44,8 +44,6 @@ import org.apache.logging.log4j.LogManager; import javax.inject.Inject; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Date; import java.util.List; import java.util.Map; diff --git a/scripts/vm/hypervisor/kvm/nasbackup.sh b/scripts/vm/hypervisor/kvm/nasbackup.sh index d354e991b748..c0b1e00500ad 100755 --- a/scripts/vm/hypervisor/kvm/nasbackup.sh +++ b/scripts/vm/hypervisor/kvm/nasbackup.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/bash ## Licensed to the Apache Software Foundation (ASF) under one ## or more contributor license agreements. See the NOTICE file ## distributed with this work for additional information @@ -16,11 +16,15 @@ ## specific language governing permissions and limitations ## under the License. +set -e + # CloudStack B&R NAS Backup and Recovery Tool for KVM # TODO: do libvirt/logging etc checks + backup_vm() { + export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" vm=$1 path=$2 storage=$3 @@ -32,16 +36,20 @@ backup_vm() { mkdir -p $dest echo "" > $dest/backup.xml - for disk in $(virsh -c qemu:///system domblklist $vm --details | awk '/disk/{print$3}'); do + for disk in $(virsh -c qemu:///system domblklist $vm --details 2>/dev/null | awk '/disk/{print$3}'); do echo "" >> $dest/backup.xml done - echo "" > $dest/backup.xml + echo "" >> $dest/backup.xml - virsh -c qemu:///system backup-begin --domain $vm --backupxml $dest/backup.xml - virsh -c qemu:///system dumpxml $vm > $dest/domain-$vm.xml - sync + virsh -c qemu:///system backup-begin --domain $vm --backupxml $dest/backup.xml > /dev/null 2>/dev/null + virsh -c qemu:///system dumpxml $vm > $dest/domain-$vm.xml 2>/dev/null + + until virsh -c qemu:///system domjobinfo $vm --completed 2>/dev/null | grep "Completed" > /dev/null; do + sleep 5 + done # Print directory size + sync du -sb $dest | cut -f1 umount $mount_point rmdir $mount_point From e066c5a91987b9b3e3b9a69ae1d89b4f353e7c09 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Thu, 1 Aug 2024 16:40:35 +0530 Subject: [PATCH 09/68] fix command execution handling Signed-off-by: Rohit Yadav --- .../cloudstack/backup/BackupAnswer.java | 6 ++--- .../cloudstack/backup/NASBackupProvider.java | 2 +- .../LibvirtTakeBackupCommandWrapper.java | 26 ++++++++++++------- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java index c5e52850a33c..aae774fadbf2 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java +++ b/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java @@ -24,9 +24,9 @@ import java.util.Map; public class BackupAnswer extends Answer { - Long size; - Long virtualSize; - Map volumes; + private Long size; + private Long virtualSize; + private Map volumes; public BackupAnswer(final TakeBackupCommand command, final boolean success, final String details) { super(command, success, details); diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 390458b214cc..39df00678e44 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -166,7 +166,7 @@ public boolean takeBackup(VirtualMachine vm) { if (answer != null) { BackupVO backup = new BackupVO(); backup.setVmId(vm.getId()); - backup.setExternalId(String.format("%s|%s|%s", nasType, backupStoragePath, backupPath)); + backup.setExternalId(String.format("%s:%s/%s", nasType, vm.getInstanceName(), backupPath)); backup.setType("FULL"); backup.setDate(new Date()); backup.setSize(answer.getSize()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java index 740749376b43..489610f982c6 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java @@ -23,11 +23,14 @@ import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.Pair; import com.cloud.utils.script.Script; import org.apache.cloudstack.backup.BackupAnswer; import org.apache.cloudstack.backup.TakeBackupCommand; import java.io.File; +import java.util.ArrayList; +import java.util.List; @ResourceWrapper(handles = TakeBackupCommand.class) public class LibvirtTakeBackupCommandWrapper extends CommandWrapper { @@ -36,17 +39,22 @@ public Answer execute(TakeBackupCommand command, LibvirtComputingResource libvir final String vmName = command.getVmName(); final String backupStoragePath = command.getBackupStoragePath(); final String backupFolder = command.getBackupPath(); - Script cmd = new Script(libvirtComputingResource.getNasBackupPath(), libvirtComputingResource.getCmdsTimeout(), logger); - cmd.add("-b", vmName); - cmd.add("-s", backupStoragePath); - cmd.add("-p", String.format("%s%s%s", vmName, File.separator, backupFolder)); - String result = cmd.execute(); - if (result == null) { - logger.debug("Failed to take VM backup: " + result); - return new BackupAnswer(command, false, result); + + List commands = new ArrayList<>(); + commands.add(new String[]{libvirtComputingResource.getNasBackupPath(), + "-b", vmName, + "-s", backupStoragePath, + "-p", String.format("%s%s%s", vmName, File.separator, backupFolder) }); + + Pair result = Script.executePipedCommands(commands, libvirtComputingResource.getCmdsTimeout()); + + if (result.first() != 0) { + logger.debug("Failed to take VM backup: " + result.second()); + return new BackupAnswer(command, false, result.second()); } + BackupAnswer answer = new BackupAnswer(command, true, null); - answer.setSize(Long.valueOf(result)); + answer.setSize(Long.valueOf(result.second())); return answer; } } From 50e407c1fa685be20c74bad3d013754724e3ace8 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Thu, 1 Aug 2024 17:54:43 +0530 Subject: [PATCH 10/68] implement backup delete Signed-off-by: Rohit Yadav --- .../cloudstack/backup/BackupAnswer.java | 3 +- .../backup/DeleteBackupCommand.java | 68 +++++++++++++++++++ .../cloudstack/backup/NASBackupProvider.java | 41 ++++++++--- .../LibvirtDeleteBackupCommandWrapper.java | 58 ++++++++++++++++ .../LibvirtTakeBackupCommandWrapper.java | 10 ++- 5 files changed, 165 insertions(+), 15 deletions(-) create mode 100644 core/src/main/java/org/apache/cloudstack/backup/DeleteBackupCommand.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteBackupCommandWrapper.java diff --git a/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java index aae774fadbf2..09f9c5621502 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java +++ b/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java @@ -20,6 +20,7 @@ package org.apache.cloudstack.backup; import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; import java.util.Map; @@ -28,7 +29,7 @@ public class BackupAnswer extends Answer { private Long virtualSize; private Map volumes; - public BackupAnswer(final TakeBackupCommand command, final boolean success, final String details) { + public BackupAnswer(final Command command, final boolean success, final String details) { super(command, success, details); } diff --git a/core/src/main/java/org/apache/cloudstack/backup/DeleteBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/DeleteBackupCommand.java new file mode 100644 index 000000000000..2dd4e3d4f7f1 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/backup/DeleteBackupCommand.java @@ -0,0 +1,68 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.backup; + +import com.cloud.agent.api.Command; +import com.cloud.agent.api.LogLevel; + +import java.util.Map; + +public class DeleteBackupCommand extends Command { + private String backupPath; + private String backupStoragePath; + @LogLevel(LogLevel.Log4jLevel.Off) + private Map details; + + public DeleteBackupCommand(String backupPath, String backupStoragePath, Map details) { + super(); + this.backupPath = backupPath; + this.backupStoragePath = backupStoragePath; + this.details = details; + } + + public String getBackupPath() { + return backupPath; + } + + public void setBackupPath(String backupPath) { + this.backupPath = backupPath; + } + + public String getBackupStoragePath() { + return backupStoragePath; + } + + public void setBackupStoragePath(String backupStoragePath) { + this.backupStoragePath = backupStoragePath; + } + + public Map getDetails() { + return details; + } + + public void setDetails(Map details) { + this.details = details; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 39df00678e44..c93ef15f323c 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -36,7 +36,6 @@ import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.backup.dao.BackupDao; -import org.apache.cloudstack.backup.dao.BackupOfferingDaoImpl; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.commons.collections.CollectionUtils; @@ -133,17 +132,16 @@ protected Host getRunningVMHypervisorHost(VirtualMachine vm) { if (hostId == null) { throw new CloudRuntimeException("Unable to find the HYPERVISOR for " + vm.getName() + ". Make sure the virtual machine is running"); } - return hostDao.findById(hostId); - } - - @Override - public boolean takeBackup(VirtualMachine vm) { - final Host host = getRunningVMHypervisorHost(vm); + final Host host = hostDao.findById(hostId); if (host == null || !Status.Up.equals(host.getStatus()) || !Hypervisor.HypervisorType.KVM.equals(host.getHypervisorType())) { throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); } + return host; + } - final BackupOfferingVO backupOffering = new BackupOfferingDaoImpl().findById(vm.getBackupOfferingId()); + @Override + public boolean takeBackup(final VirtualMachine vm) { + final Host host = getRunningVMHypervisorHost(vm); final String backupStoragePath = getBackupStoragePath(vm.getDataCenterId()); final String nasType = getNasType(vm.getDataCenterId()); @@ -224,11 +222,32 @@ public Pair restoreBackedUpVolume(Backup backup, String volumeU @Override public boolean deleteBackup(Backup backup, boolean forced) { - final Long zoneId = backup.getZoneId(); - final String externalBackupId = backup.getExternalId(); + final String backupStoragePath = getBackupStoragePath(zoneId); + final String nasType = getNasType(zoneId); + final Map backupDetails = Map.of( + "type", nasType + ); + final String backupPath = backup.getExternalId().split(":")[1]; + + // TODO: this can be any host in the cluster or last host + final VirtualMachine vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); + final Host host = getRunningVMHypervisorHost(vm); + + DeleteBackupCommand command = new DeleteBackupCommand(backupPath, backupStoragePath, backupDetails); + + BackupAnswer answer = null; + try { + answer = (BackupAnswer) agentManager.send(host.getId(), command); + } catch (AgentUnavailableException e) { + throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); + } catch (OperationTimedoutException e) { + throw new CloudRuntimeException("Operation to initiate backup timed out, please try again"); + } - // TODO: delete backup from NAS + if (answer != null && answer.getResult()) { + return backupDao.remove(backup.getId()); + } return false; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteBackupCommandWrapper.java new file mode 100644 index 000000000000..cc53fe779c62 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteBackupCommandWrapper.java @@ -0,0 +1,58 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.Pair; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.backup.BackupAnswer; +import org.apache.cloudstack.backup.DeleteBackupCommand; + +import java.util.ArrayList; +import java.util.List; + +@ResourceWrapper(handles = DeleteBackupCommand.class) +public class LibvirtDeleteBackupCommandWrapper extends CommandWrapper { + @Override + public Answer execute(DeleteBackupCommand command, LibvirtComputingResource libvirtComputingResource) { + final String backupStoragePath = command.getBackupStoragePath(); + final String backupFolder = command.getBackupPath(); + + List commands = new ArrayList<>(); + commands.add(new String[]{ + libvirtComputingResource.getNasBackupPath(), + "-d", backupFolder, + "-s", backupStoragePath + }); + + Pair result = Script.executePipedCommands(commands, libvirtComputingResource.getCmdsTimeout()); + + logger.debug("Backup delete result: " + result.second() + ", exit code: " + result.first()); + + if (result.first() != 0) { + logger.debug("Failed to delete VM backup: " + result.second()); + return new BackupAnswer(command, false, result.second()); + } + return new BackupAnswer(command, true, null); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java index 489610f982c6..7938628653d9 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java @@ -41,20 +41,24 @@ public Answer execute(TakeBackupCommand command, LibvirtComputingResource libvir final String backupFolder = command.getBackupPath(); List commands = new ArrayList<>(); - commands.add(new String[]{libvirtComputingResource.getNasBackupPath(), + commands.add(new String[]{ + libvirtComputingResource.getNasBackupPath(), "-b", vmName, "-s", backupStoragePath, - "-p", String.format("%s%s%s", vmName, File.separator, backupFolder) }); + "-p", String.format("%s%s%s", vmName, File.separator, backupFolder) + }); Pair result = Script.executePipedCommands(commands, libvirtComputingResource.getCmdsTimeout()); + logger.debug("VM Backup Result: " + result.second() + ", exit code: " + result.first()); + if (result.first() != 0) { logger.debug("Failed to take VM backup: " + result.second()); return new BackupAnswer(command, false, result.second()); } BackupAnswer answer = new BackupAnswer(command, true, null); - answer.setSize(Long.valueOf(result.second())); + answer.setSize(Long.parseLong(result.second().trim())); return answer; } } From 9b0be25a5c9c468038233b8bbcb8470b5ad9fb23 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Thu, 1 Aug 2024 17:55:49 +0530 Subject: [PATCH 11/68] fix nasbackup script Signed-off-by: Rohit Yadav --- scripts/vm/hypervisor/kvm/nasbackup.sh | 29 ++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/scripts/vm/hypervisor/kvm/nasbackup.sh b/scripts/vm/hypervisor/kvm/nasbackup.sh index c0b1e00500ad..d29e50f0e10e 100755 --- a/scripts/vm/hypervisor/kvm/nasbackup.sh +++ b/scripts/vm/hypervisor/kvm/nasbackup.sh @@ -24,7 +24,6 @@ set -e backup_vm() { - export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" vm=$1 path=$2 storage=$3 @@ -55,9 +54,23 @@ backup_vm() { rmdir $mount_point } +delete_backup() { + path=$1 + storage=$2 + + mount_point=$(mktemp -d -t csbackup.XXXXX) + dest="$mount_point/$path" + + mount -t nfs $storage $mount_point + rm -frv $dest + sync + umount $mount_point + rmdir $mount_point +} + OP="" VM="" -PATH="" +DEST="" NAS="" TYPE="" @@ -76,6 +89,12 @@ while [[ $# -gt 0 ]]; do shift shift ;; + -d|--delete) + OP="delete" + DEST="$2" + shift + shift + ;; -s|--storage) NAS="$2" TYPE="nfs" @@ -83,7 +102,7 @@ while [[ $# -gt 0 ]]; do shift ;; -p|--path) - PATH="$2" + DEST="$2" shift shift ;; @@ -107,6 +126,8 @@ while [[ $# -gt 0 ]]; do done if [ "$OP" = "backup" ]; then - backup_vm $VM $PATH $NAS + backup_vm $VM $DEST $NAS +elif [ "$OP" = "backup" ]; then + delete_backup $DEST $NAS fi From 11cbea2b155783d96156df364300dca5239c03dc Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Thu, 1 Aug 2024 18:05:21 +0530 Subject: [PATCH 12/68] fix op typo Signed-off-by: Rohit Yadav --- scripts/vm/hypervisor/kvm/nasbackup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/vm/hypervisor/kvm/nasbackup.sh b/scripts/vm/hypervisor/kvm/nasbackup.sh index d29e50f0e10e..2cfae0d4e560 100755 --- a/scripts/vm/hypervisor/kvm/nasbackup.sh +++ b/scripts/vm/hypervisor/kvm/nasbackup.sh @@ -127,7 +127,7 @@ done if [ "$OP" = "backup" ]; then backup_vm $VM $DEST $NAS -elif [ "$OP" = "backup" ]; then +elif [ "$OP" = "delete" ]; then delete_backup $DEST $NAS fi From 18e05894c718764aaa2339966ee409fe6fba2a0b Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Thu, 1 Aug 2024 18:15:11 +0530 Subject: [PATCH 13/68] add todos Signed-off-by: Rohit Yadav --- .../java/org/apache/cloudstack/backup/NASBackupProvider.java | 4 +++- scripts/vm/hypervisor/kvm/nasbackup.sh | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index c93ef15f323c..24c0c7cc6701 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -141,6 +141,8 @@ protected Host getRunningVMHypervisorHost(VirtualMachine vm) { @Override public boolean takeBackup(final VirtualMachine vm) { + // TODO: currently works for only running VMs + // TODO: add support for backup of stopped VMs final Host host = getRunningVMHypervisorHost(vm); final String backupStoragePath = getBackupStoragePath(vm.getDataCenterId()); @@ -187,7 +189,7 @@ public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { LOG.debug("Restoring vm " + vm.getUuid() + "from backup " + backup.getUuid() + " on the NAS Backup Provider"); // Find where the VM was last running - final Host hostVO = getLastVMHypervisorHost(vm); + final Host host = getLastVMHypervisorHost(vm); // TODO: get KVM agent to restore VM backup diff --git a/scripts/vm/hypervisor/kvm/nasbackup.sh b/scripts/vm/hypervisor/kvm/nasbackup.sh index 2cfae0d4e560..1595e7fb036c 100755 --- a/scripts/vm/hypervisor/kvm/nasbackup.sh +++ b/scripts/vm/hypervisor/kvm/nasbackup.sh @@ -42,6 +42,7 @@ backup_vm() { virsh -c qemu:///system backup-begin --domain $vm --backupxml $dest/backup.xml > /dev/null 2>/dev/null virsh -c qemu:///system dumpxml $vm > $dest/domain-$vm.xml 2>/dev/null + rm -f $dest/backup.xml until virsh -c qemu:///system domjobinfo $vm --completed 2>/dev/null | grep "Completed" > /dev/null; do sleep 5 From 97ce34ce2b37a4feca3e5fd69e85127ad2ad129d Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Fri, 2 Aug 2024 12:58:40 +0530 Subject: [PATCH 14/68] introduce backup repository concept for NAS & other backups Each backup repository is for a specific zone & provider. Eack NAS bkcp offering is tied to a back repo & each bkup offer assinged to VM -> defined its backup repository Signed-off-by: Rohit Yadav --- .../cloudstack/backup/BackupRepository.java | 34 ++++ .../backup/DeleteBackupCommand.java | 38 +++-- .../cloudstack/backup/TakeBackupCommand.java | 35 ++-- .../cloudstack/backup/BackupRepositoryVO.java | 152 ++++++++++++++++++ .../backup/dao/BackupRepositoryDao.java | 31 ++++ .../backup/dao/BackupRepositoryDaoImpl.java | 67 ++++++++ ...spring-engine-schema-core-daos-context.xml | 1 + .../META-INF/db/schema-41910to42000.sql | 20 +++ .../cloudstack/backup/NASBackupProvider.java | 81 ++++------ .../cloudstack/backup/NasBackupOffering.java | 75 +++++++++ .../LibvirtDeleteBackupCommandWrapper.java | 13 +- .../LibvirtTakeBackupCommandWrapper.java | 17 +- scripts/vm/hypervisor/kvm/nasbackup.sh | 67 ++++---- 13 files changed, 509 insertions(+), 122 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDao.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDaoImpl.java create mode 100644 plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NasBackupOffering.java diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java b/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java new file mode 100644 index 000000000000..8e5c9740e690 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java @@ -0,0 +1,34 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.backup; + +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +import java.util.Date; + +public interface BackupRepository extends InternalIdentity, Identity { + String getProvider(); + long getZoneId(); + String getName(); + String getType(); + String getAddress(); + String getMountOptions(); + Long getCapacityBytes(); + Long getUsedBytes(); + Date getCreated(); +} diff --git a/core/src/main/java/org/apache/cloudstack/backup/DeleteBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/DeleteBackupCommand.java index 2dd4e3d4f7f1..ceb6426a316f 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/DeleteBackupCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/DeleteBackupCommand.java @@ -22,19 +22,19 @@ import com.cloud.agent.api.Command; import com.cloud.agent.api.LogLevel; -import java.util.Map; - public class DeleteBackupCommand extends Command { private String backupPath; - private String backupStoragePath; + private String backupRepoType; + private String backupRepoAddress; @LogLevel(LogLevel.Log4jLevel.Off) - private Map details; + private String mountOptions; - public DeleteBackupCommand(String backupPath, String backupStoragePath, Map details) { + public DeleteBackupCommand(String backupPath, String backupRepoType, String backupRepoAddress, String mountOptions) { super(); this.backupPath = backupPath; - this.backupStoragePath = backupStoragePath; - this.details = details; + this.backupRepoType = backupRepoType; + this.backupRepoAddress = backupRepoAddress; + this.mountOptions = mountOptions; } public String getBackupPath() { @@ -45,20 +45,28 @@ public void setBackupPath(String backupPath) { this.backupPath = backupPath; } - public String getBackupStoragePath() { - return backupStoragePath; + public String getBackupRepoType() { + return backupRepoType; + } + + public void setBackupRepoType(String backupRepoType) { + this.backupRepoType = backupRepoType; + } + + public String getBackupRepoAddress() { + return backupRepoAddress; } - public void setBackupStoragePath(String backupStoragePath) { - this.backupStoragePath = backupStoragePath; + public void setBackupRepoAddress(String backupRepoAddress) { + this.backupRepoAddress = backupRepoAddress; } - public Map getDetails() { - return details; + public String getMountOptions() { + return mountOptions; } - public void setDetails(Map details) { - this.details = details; + public void setMountOptions(String mountOptions) { + this.mountOptions = mountOptions; } @Override diff --git a/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java index 91a49f1ce30d..3deb3eb47b0c 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java @@ -22,21 +22,18 @@ import com.cloud.agent.api.Command; import com.cloud.agent.api.LogLevel; -import java.util.Map; - public class TakeBackupCommand extends Command { private String vmName; private String backupPath; - private String backupStoragePath; + private String backupRepoType; + private String backupRepoAddress; @LogLevel(LogLevel.Log4jLevel.Off) - private Map details; + private String mountOptions; - public TakeBackupCommand(String vmName, String backupPath, String backupStoragePath, Map details) { + public TakeBackupCommand(String vmName, String backupPath) { super(); this.vmName = vmName; this.backupPath = backupPath; - this.backupStoragePath = backupStoragePath; - this.details = details; } public String getVmName() { @@ -55,20 +52,28 @@ public void setBackupPath(String backupPath) { this.backupPath = backupPath; } - public String getBackupStoragePath() { - return backupStoragePath; + public String getBackupRepoType() { + return backupRepoType; + } + + public void setBackupRepoType(String backupRepoType) { + this.backupRepoType = backupRepoType; + } + + public String getBackupRepoAddress() { + return backupRepoAddress; } - public void setBackupStoragePath(String backupStoragePath) { - this.backupStoragePath = backupStoragePath; + public void setBackupRepoAddress(String backupRepoAddress) { + this.backupRepoAddress = backupRepoAddress; } - public Map getDetails() { - return details; + public String getMountOptions() { + return mountOptions; } - public void setDetails(Map details) { - this.details = details; + public void setMountOptions(String mountOptions) { + this.mountOptions = mountOptions; } @Override diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java new file mode 100644 index 000000000000..8ae1cf91449b --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup; + +import java.util.Date; +import java.util.UUID; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +@Entity +@Table(name = "backup_repository") +public class BackupRepositoryVO implements BackupRepository { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "uuid") + private String uuid; + + @Column(name = "name") + private String name; + + @Column(name = "zone_id", nullable = false) + private long zoneId; + + @Column(name = "provider", nullable = false) + private String provider; + + @Column(name = "type", nullable = false) + private String type; + + @Column(name = "address", nullable = false) + private String address; + + @Column(name = "mount_opts") + private String mountOptions; + + @Column(name = "used_bytes",nullable = true) + private Long usedBytes; + + @Column(name = "capacity_bytes", nullable = true) + private Long capacityBytes; + + @Column(name = "created") + @Temporal(value = TemporalType.TIMESTAMP) + private Date created; + + @Column(name = "removed") + @Temporal(value = TemporalType.TIMESTAMP) + private Date removed; + + public BackupRepositoryVO() { + this.uuid = UUID.randomUUID().toString(); + } + + public BackupRepositoryVO(final long zoneId, final String provider, final String name, final String type, final String address, final String mountOptions, final Long capacityBytes) { + this(); + this.zoneId = zoneId; + this.provider = provider; + this.name = name; + this.type = type; + this.address = address; + this.mountOptions = mountOptions; + this.capacityBytes = capacityBytes; + this.created = new Date(); + } + + public String getUuid() { + return uuid; + } + + public long getId() { + return id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @Override + public long getZoneId() { + return zoneId; + } + + @Override + public String getProvider() { + return provider; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public void setAddress(String address) { + this.address = address; + } + + @Override + public String getAddress() { + return address; + } + + @Override + public String getMountOptions() { + return mountOptions; + } + + @Override + public Long getUsedBytes() { + return usedBytes; + } + + @Override + public Long getCapacityBytes() { + return capacityBytes; + } + + public Date getCreated() { + return created; + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDao.java new file mode 100644 index 000000000000..0034bfb30ab6 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDao.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup.dao; + +import java.util.List; + +import org.apache.cloudstack.backup.BackupRepository; +import org.apache.cloudstack.backup.BackupRepositoryVO; + +import com.cloud.utils.db.GenericDao; + +public interface BackupRepositoryDao extends GenericDao { + List listByZoneAndProvider(Long zoneId, String provider); + + BackupRepository findByBackupOfferingId(Long backupOfferingId); +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDaoImpl.java new file mode 100644 index 000000000000..460b6d8aba45 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupRepositoryDaoImpl.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup.dao; + +import java.util.ArrayList; +import java.util.List; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; + +import org.apache.cloudstack.backup.BackupOfferingVO; +import org.apache.cloudstack.backup.BackupRepository; +import org.apache.cloudstack.backup.BackupRepositoryVO; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +public class BackupRepositoryDaoImpl extends GenericDaoBase implements BackupRepositoryDao { + @Inject + BackupOfferingDao backupOfferingDao; + + private SearchBuilder backupRepoSearch; + + public BackupRepositoryDaoImpl() { + } + + @PostConstruct + protected void init() { + backupRepoSearch = createSearchBuilder(); + backupRepoSearch.and("zone_id", backupRepoSearch.entity().getZoneId(), SearchCriteria.Op.EQ); + backupRepoSearch.and("provider", backupRepoSearch.entity().getProvider(), SearchCriteria.Op.EQ); + backupRepoSearch.done(); + } + + @Override + public List listByZoneAndProvider(Long zoneId, String provider) { + SearchCriteria sc = backupRepoSearch.create(); + sc.setParameters("zone_id", zoneId); + sc.setParameters("provider", provider); + return new ArrayList<>(listBy(sc)); + } + + @Override + public BackupRepository findByBackupOfferingId(Long backupOfferingId) { + BackupOfferingVO offering = backupOfferingDao.findByIdIncludingRemoved(backupOfferingId); + if (offering == null) { + return null; + } + return findByUuid(offering.getExternalId()); + } +} diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 8ab60a766246..305184bcf7f6 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -269,6 +269,7 @@ + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql b/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql index f59eda5c06c9..370abc3f9f24 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql @@ -134,6 +134,26 @@ CREATE TABLE `cloud`.`webhook_delivery` ( CONSTRAINT `fk_webhook__webhook_id` FOREIGN KEY (`webhook_id`) REFERENCES `webhook`(`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8; +-- Backup Repository NAS feature +DROP TABLE IF EXISTS `cloud`.`backup_repository`; +CREATE TABLE `cloud`.`backup_repository` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id of the backup repository', + `uuid` varchar(255) NOT NULL COMMENT 'uuid of the backup repository', + `name` varchar(255) NOT NULL COMMENT 'name of the backup repository', + `zone_id` bigint unsigned NOT NULL COMMENT 'id of zone', + `provider` varchar(255) NOT NULL COMMENT 'backup provider name', + `type` varchar(255) NOT NULL COMMENT 'backup repo type', + `address` varchar(1024) NOT NULL COMMENT 'url of the backup repository', + `mount_opts` varchar(1024) COMMENT 'mount options for the backup repository', + `used_bytes` bigint unsigned, + `capacity_bytes` bigint unsigned, + `created` datetime, + `removed` datetime, + PRIMARY KEY(`id`), + INDEX `i_backup_repository__uuid`(`uuid`), + INDEX `i_backup_repository__zone_id_provider`(`zone_id`, `provider`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + -- Normalize quota.usage.smtp.useStartTLS, quota.usage.smtp.useAuth, alert.smtp.useAuth and project.smtp.useAuth values UPDATE `cloud`.`configuration` diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 24c0c7cc6701..0e6b3122a3a9 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -36,6 +36,8 @@ import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.cloudstack.backup.dao.BackupOfferingDao; +import org.apache.cloudstack.backup.dao.BackupRepositoryDao; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.commons.collections.CollectionUtils; @@ -43,6 +45,7 @@ import org.apache.logging.log4j.LogManager; import javax.inject.Inject; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; @@ -50,23 +53,15 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Configurable { private static final Logger LOG = LogManager.getLogger(NASBackupProvider.class); - private final ConfigKey NasType = new ConfigKey<>("Advanced", String.class, - "backup.plugin.nas.target.type", "nfs", - "The NAS storage target type. Only supported: nfs and cephfs", true, ConfigKey.Scope.Zone); - private final ConfigKey NfsPool = new ConfigKey<>("Advanced", String.class, - "backup.plugin.nas.nfs.pool", "", - "The NFS NAS storage pool URL (format :", true, ConfigKey.Scope.Zone); - private final ConfigKey CephFSPool = new ConfigKey<>("Advanced", String.class, - "backup.plugin.nas.cephfs.pool", "", - "The CephFS storage pool URL (format: :)", true, ConfigKey.Scope.Zone); + @Inject + private BackupDao backupDao; - private final ConfigKey CephFSPoolCredentials = new ConfigKey<>("Advanced", String.class, - "backup.plugin.nas.cephfs.credentials", "", - "The CephFS storage pool URL (format: )", true, ConfigKey.Scope.Zone); + @Inject + private BackupRepositoryDao backupRepositoryDao; @Inject - private BackupDao backupDao; + private BackupOfferingDao backupOfferingDao; @Inject private HostDao hostDao; @@ -86,18 +81,6 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Inject private AgentManager agentManager; - protected String getNasType(final Long zoneId) { - return NasType.valueIn(zoneId); - } - - protected String getBackupStoragePath(final Long zoneId) { - final String type = getNasType(zoneId); - if ("nfs".equalsIgnoreCase(type)) { - return NfsPool.valueIn(zoneId); - } - throw new CloudRuntimeException("NAS backup plugin not configured"); - } - protected Host getLastVMHypervisorHost(VirtualMachine vm) { Long hostId = vm.getLastHostId(); if (hostId == null) { @@ -145,14 +128,18 @@ public boolean takeBackup(final VirtualMachine vm) { // TODO: add support for backup of stopped VMs final Host host = getRunningVMHypervisorHost(vm); - final String backupStoragePath = getBackupStoragePath(vm.getDataCenterId()); - final String nasType = getNasType(vm.getDataCenterId()); - final Map backupDetails = Map.of( - "type", nasType - ); - final String backupPath = new SimpleDateFormat("yyyy.MM.dd.HH.mm.ss").format(new java.util.Date()); + final BackupRepository backupRepository = backupRepositoryDao.findByBackupOfferingId(vm.getBackupOfferingId()); + if (backupRepository == null) { + throw new CloudRuntimeException("No valid backup repository found for the VM, please check the attached backup offering"); + } + + final String backupPath = String.format("%s/%s", vm.getInstanceName(), + new SimpleDateFormat("yyyy.MM.dd.HH.mm.ss").format(new java.util.Date())); - TakeBackupCommand command = new TakeBackupCommand(vm.getInstanceName(), backupPath, backupStoragePath, backupDetails); + TakeBackupCommand command = new TakeBackupCommand(vm.getInstanceName(), backupPath); + command.setBackupRepoType(backupRepository.getType()); + command.setBackupRepoAddress(backupRepository.getAddress()); + command.setMountOptions(backupRepository.getMountOptions()); BackupAnswer answer = null; try { @@ -166,7 +153,7 @@ public boolean takeBackup(final VirtualMachine vm) { if (answer != null) { BackupVO backup = new BackupVO(); backup.setVmId(vm.getId()); - backup.setExternalId(String.format("%s:%s/%s", nasType, vm.getInstanceName(), backupPath)); + backup.setExternalId(backupPath); backup.setType("FULL"); backup.setDate(new Date()); backup.setSize(answer.getSize()); @@ -224,19 +211,17 @@ public Pair restoreBackedUpVolume(Backup backup, String volumeU @Override public boolean deleteBackup(Backup backup, boolean forced) { - final Long zoneId = backup.getZoneId(); - final String backupStoragePath = getBackupStoragePath(zoneId); - final String nasType = getNasType(zoneId); - final Map backupDetails = Map.of( - "type", nasType - ); - final String backupPath = backup.getExternalId().split(":")[1]; + final BackupRepository backupRepository = backupRepositoryDao.findByBackupOfferingId(backup.getBackupOfferingId()); + if (backupRepository == null) { + throw new CloudRuntimeException("No valid backup repository found for the VM, please check the attached backup offering"); + } // TODO: this can be any host in the cluster or last host final VirtualMachine vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); final Host host = getRunningVMHypervisorHost(vm); - DeleteBackupCommand command = new DeleteBackupCommand(backupPath, backupStoragePath, backupDetails); + DeleteBackupCommand command = new DeleteBackupCommand(backup.getExternalId(), backupRepository.getType(), + backupRepository.getAddress(), backupRepository.getMountOptions()); BackupAnswer answer = null; try { @@ -299,8 +284,12 @@ public void syncBackups(VirtualMachine vm, Backup.Metric metric) { @Override public List listBackupOfferings(Long zoneId) { - BackupOffering policy = new BackupOfferingVO(zoneId, "default", getName(), "Default", "Default Backup Offering", true); - return List.of(policy); + final List repositories = backupRepositoryDao.listByZoneAndProvider(zoneId, getName()); + final List offerings = new ArrayList<>(); + for (final BackupRepository repository : repositories) { + offerings.add(new NasBackupOffering(repository.getName(), repository.getUuid())); + } + return offerings; } @Override @@ -311,10 +300,6 @@ public boolean isValidProviderOffering(Long zoneId, String uuid) { @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[]{ - NasType, - NfsPool, - CephFSPool, - CephFSPoolCredentials }; } @@ -325,7 +310,7 @@ public String getName() { @Override public String getDescription() { - return "NAS KVM Backup Plugin"; + return "NAS Backup Plugin"; } @Override diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NasBackupOffering.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NasBackupOffering.java new file mode 100644 index 000000000000..91df74166e58 --- /dev/null +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NasBackupOffering.java @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup; + +import java.util.Date; + +public class NasBackupOffering implements BackupOffering { + + private String name; + private String uid; + + public NasBackupOffering(String name, String uid) { + this.name = name; + this.uid = uid; + } + + @Override + public String getExternalId() { + return uid; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getDescription() { + return "NAS Backup Offering (Repository)"; + } + + @Override + public long getZoneId() { + return -1; + } + + @Override + public boolean isUserDrivenBackupAllowed() { + return true; + } + + @Override + public String getProvider() { + return "nas"; + } + + @Override + public Date getCreated() { + return null; + } + + @Override + public String getUuid() { + return uid; + } + + @Override + public long getId() { + return -1; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteBackupCommandWrapper.java index cc53fe779c62..cb1e34858242 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteBackupCommandWrapper.java @@ -35,14 +35,19 @@ public class LibvirtDeleteBackupCommandWrapper extends CommandWrapper { @Override public Answer execute(DeleteBackupCommand command, LibvirtComputingResource libvirtComputingResource) { - final String backupStoragePath = command.getBackupStoragePath(); - final String backupFolder = command.getBackupPath(); + final String backupPath = command.getBackupPath(); + final String backupRepoType = command.getBackupRepoType(); + final String backupRepoAddress = command.getBackupRepoAddress(); + final String mountOptions = command.getMountOptions(); List commands = new ArrayList<>(); commands.add(new String[]{ libvirtComputingResource.getNasBackupPath(), - "-d", backupFolder, - "-s", backupStoragePath + "-o", "delete", + "-t", backupRepoType, + "-s", backupRepoAddress, + "-m", mountOptions, + "-p", backupPath }); Pair result = Script.executePipedCommands(commands, libvirtComputingResource.getCmdsTimeout()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java index 7938628653d9..30f530ffd8a2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java @@ -29,6 +29,8 @@ import org.apache.cloudstack.backup.TakeBackupCommand; import java.io.File; +import java.net.MalformedURLException; +import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -37,15 +39,20 @@ public class LibvirtTakeBackupCommandWrapper extends CommandWrapper commands = new ArrayList<>(); commands.add(new String[]{ libvirtComputingResource.getNasBackupPath(), - "-b", vmName, - "-s", backupStoragePath, - "-p", String.format("%s%s%s", vmName, File.separator, backupFolder) + "-b", "backup", + "-v", vmName, + "-t", backupRepoType, + "-s", backupRepoAddress, + "-m", mountOptions, + "-p", backupPath }); Pair result = Script.executePipedCommands(commands, libvirtComputingResource.getCmdsTimeout()); diff --git a/scripts/vm/hypervisor/kvm/nasbackup.sh b/scripts/vm/hypervisor/kvm/nasbackup.sh index 1595e7fb036c..5c6cbc5476aa 100755 --- a/scripts/vm/hypervisor/kvm/nasbackup.sh +++ b/scripts/vm/hypervisor/kvm/nasbackup.sh @@ -22,16 +22,22 @@ set -e # TODO: do libvirt/logging etc checks +### Declare variables ### -backup_vm() { - vm=$1 - path=$2 - storage=$3 +OP="" +VM="" +NAS_TYPE="" +NAS_ADDRESS="" +MOUNT_OPTS="" +BACKUP_DIR="" +### Operation methods ### + +backup_vm() { mount_point=$(mktemp -d -t csbackup.XXXXX) - dest="$mount_point/$path" + dest="$mount_point/${BACKUP_DIR}" - mount -t nfs $storage $mount_point + mount -t ${NAS_TYPE} ${NAS_ADDRESS} ${mount_point} $([[ ! -z "${MOUNT_OPTS}" ]] && echo -o ${MOUNT_OPTS}) mkdir -p $dest echo "" > $dest/backup.xml @@ -42,11 +48,11 @@ backup_vm() { virsh -c qemu:///system backup-begin --domain $vm --backupxml $dest/backup.xml > /dev/null 2>/dev/null virsh -c qemu:///system dumpxml $vm > $dest/domain-$vm.xml 2>/dev/null - rm -f $dest/backup.xml until virsh -c qemu:///system domjobinfo $vm --completed 2>/dev/null | grep "Completed" > /dev/null; do sleep 5 done + rm -f $dest/backup.xml # Print directory size sync @@ -56,25 +62,18 @@ backup_vm() { } delete_backup() { - path=$1 - storage=$2 - mount_point=$(mktemp -d -t csbackup.XXXXX) - dest="$mount_point/$path" + dest="$mount_point/${BACKUP_DIR}" + + mount -t ${NAS_TYPE} ${NAS_ADDRESS} ${mount_point} $([[ ! -z "${MOUNT_OPTS}" ]] && echo -o ${MOUNT_OPTS}) - mount -t nfs $storage $mount_point rm -frv $dest sync + umount $mount_point rmdir $mount_point } -OP="" -VM="" -DEST="" -NAS="" -TYPE="" - function usage { echo "" echo "Usage: $0 -b -s -p " @@ -84,35 +83,34 @@ function usage { while [[ $# -gt 0 ]]; do case $1 in - -b|--backup) - OP="backup" + -o|--operation) + OP="$2" + shift + shift + ;; + -v|--vm) VM="$2" shift shift ;; - -d|--delete) - OP="delete" - DEST="$2" + -t|--type) + NAS_TYPE="$2" shift shift ;; -s|--storage) - NAS="$2" - TYPE="nfs" + NAS_ADDRESS="$2" shift shift ;; - -p|--path) - DEST="$2" + -m|--mount) + MOUNT_OPTS="$2" shift shift ;; - -r|--recover) - OP="recover" + -p|--path) + BACKUP_DIR="$2" shift - ;; - -rv|--recover) - OP="recover-volume" shift ;; -h|--help) @@ -127,8 +125,7 @@ while [[ $# -gt 0 ]]; do done if [ "$OP" = "backup" ]; then - backup_vm $VM $DEST $NAS + backup_vm elif [ "$OP" = "delete" ]; then - delete_backup $DEST $NAS + delete_backup fi - From 0b9db75c4f53dafeff6d01d3e86c5a624f192c27 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Fri, 2 Aug 2024 13:38:54 +0530 Subject: [PATCH 15/68] complete poc Signed-off-by: Rohit Yadav --- .../cloudstack/backup/NASBackupProvider.java | 14 +++++++++++++- .../wrapper/LibvirtTakeBackupCommandWrapper.java | 14 +++++++++----- scripts/vm/hypervisor/kvm/nasbackup.sh | 14 ++++++++++---- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 0e6b3122a3a9..83c55f625129 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -46,6 +46,7 @@ import javax.inject.Inject; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Comparator; import java.util.Date; import java.util.List; import java.util.Map; @@ -157,7 +158,13 @@ public boolean takeBackup(final VirtualMachine vm) { backup.setType("FULL"); backup.setDate(new Date()); backup.setSize(answer.getSize()); - backup.setProtectedSize(answer.getVirtualSize()); + Long virtualSize = 0L; + for (final Volume volume: volumeDao.findByInstance(vm.getId())) { + if (Volume.State.Ready.equals(volume.getState())) { + virtualSize += volume.getSize(); + } + } + backup.setProtectedSize(virtualSize); backup.setStatus(Backup.Status.BackedUp); backup.setBackupOfferingId(vm.getBackupOfferingId()); backup.setAccountId(vm.getAccountId()); @@ -173,6 +180,10 @@ public boolean takeBackup(final VirtualMachine vm) { public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { final Long zoneId = backup.getZoneId(); + List volumes = volumeDao.findByInstance(vm.getId()); + volumes.removeIf(x -> !Volume.State.Ready.equals(x.getState())); + volumes.sort(Comparator.comparing(VolumeVO::getDeviceId)); + LOG.debug("Restoring vm " + vm.getUuid() + "from backup " + backup.getUuid() + " on the NAS Backup Provider"); // Find where the VM was last running @@ -192,6 +203,7 @@ public Pair restoreBackedUpVolume(Backup backup, String volumeU final Long zoneId = backup.getZoneId(); // TODO: Find volume from backup volumes + volume.getDeviceId(); // restore by device ID VolumeVO restoredVolume = new VolumeVO(Volume.Type.DATADISK, null, backup.getZoneId(), backup.getDomainId(), backup.getAccountId(), 0, null, diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java index 30f530ffd8a2..d228c48296a4 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java @@ -28,10 +28,8 @@ import org.apache.cloudstack.backup.BackupAnswer; import org.apache.cloudstack.backup.TakeBackupCommand; -import java.io.File; -import java.net.MalformedURLException; -import java.net.URL; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; @ResourceWrapper(handles = TakeBackupCommand.class) @@ -64,8 +62,14 @@ public Answer execute(TakeBackupCommand command, LibvirtComputingResource libvir return new BackupAnswer(command, false, result.second()); } - BackupAnswer answer = new BackupAnswer(command, true, null); - answer.setSize(Long.parseLong(result.second().trim())); + List outputLines = Arrays.asList(result.second().trim().split("\n")); + Long backupSize = 0L; + if (outputLines.size() > 0) { + backupSize = Long.parseLong(outputLines.get(outputLines.size()-1).trim()); + } + + BackupAnswer answer = new BackupAnswer(command, true, result.second()); + answer.setSize(backupSize); return answer; } } diff --git a/scripts/vm/hypervisor/kvm/nasbackup.sh b/scripts/vm/hypervisor/kvm/nasbackup.sh index 5c6cbc5476aa..ae21255dc781 100755 --- a/scripts/vm/hypervisor/kvm/nasbackup.sh +++ b/scripts/vm/hypervisor/kvm/nasbackup.sh @@ -40,23 +40,29 @@ backup_vm() { mount -t ${NAS_TYPE} ${NAS_ADDRESS} ${mount_point} $([[ ! -z "${MOUNT_OPTS}" ]] && echo -o ${MOUNT_OPTS}) mkdir -p $dest + deviceId=0 + name="root" echo "" > $dest/backup.xml for disk in $(virsh -c qemu:///system domblklist $vm --details 2>/dev/null | awk '/disk/{print$3}'); do - echo "" >> $dest/backup.xml + echo "" >> $dest/backup.xml + deviceId=$((devideId+1)) + name="datadisk" done echo "" >> $dest/backup.xml virsh -c qemu:///system backup-begin --domain $vm --backupxml $dest/backup.xml > /dev/null 2>/dev/null virsh -c qemu:///system dumpxml $vm > $dest/domain-$vm.xml 2>/dev/null - until virsh -c qemu:///system domjobinfo $vm --completed 2>/dev/null | grep "Completed" > /dev/null; do + until virsh -c qemu:///system domjobinfo $vm --completed --keep-completed 2>/dev/null | grep "Completed" > /dev/null; do sleep 5 done rm -f $dest/backup.xml - - # Print directory size sync + + # Print statistics + virsh -c qemu:///system domjobinfo $vm --completed du -sb $dest | cut -f1 + umount $mount_point rmdir $mount_point } From 00b22d462b1586c3e4815146e79115352d1e132b Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Fri, 2 Aug 2024 14:27:40 +0530 Subject: [PATCH 16/68] fix backup script Signed-off-by: Rohit Yadav --- .../cloudstack/backup/NASBackupProvider.java | 7 ++++--- .../LibvirtTakeBackupCommandWrapper.java | 8 +++----- scripts/vm/hypervisor/kvm/nasbackup.sh | 19 +++++++++++++------ 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 83c55f625129..33a648ac8fb2 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -134,8 +134,9 @@ public boolean takeBackup(final VirtualMachine vm) { throw new CloudRuntimeException("No valid backup repository found for the VM, please check the attached backup offering"); } + final Date creationDate = new Date(); final String backupPath = String.format("%s/%s", vm.getInstanceName(), - new SimpleDateFormat("yyyy.MM.dd.HH.mm.ss").format(new java.util.Date())); + new SimpleDateFormat("yyyy.MM.dd.HH.mm.ss").format(creationDate)); TakeBackupCommand command = new TakeBackupCommand(vm.getInstanceName(), backupPath); command.setBackupRepoType(backupRepository.getType()); @@ -151,12 +152,12 @@ public boolean takeBackup(final VirtualMachine vm) { throw new CloudRuntimeException("Operation to initiate backup timed out, please try again"); } - if (answer != null) { + if (answer != null && answer.getResult()) { BackupVO backup = new BackupVO(); backup.setVmId(vm.getId()); backup.setExternalId(backupPath); backup.setType("FULL"); - backup.setDate(new Date()); + backup.setDate(creationDate); backup.setSize(answer.getSize()); Long virtualSize = 0L; for (final Volume volume: volumeDao.findByInstance(vm.getId())) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java index d228c48296a4..af02f2aa06a3 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java @@ -45,7 +45,7 @@ public Answer execute(TakeBackupCommand command, LibvirtComputingResource libvir List commands = new ArrayList<>(); commands.add(new String[]{ libvirtComputingResource.getNasBackupPath(), - "-b", "backup", + "-o", "backup", "-v", vmName, "-t", backupRepoType, "-s", backupRepoAddress, @@ -55,11 +55,9 @@ public Answer execute(TakeBackupCommand command, LibvirtComputingResource libvir Pair result = Script.executePipedCommands(commands, libvirtComputingResource.getCmdsTimeout()); - logger.debug("VM Backup Result: " + result.second() + ", exit code: " + result.first()); - if (result.first() != 0) { logger.debug("Failed to take VM backup: " + result.second()); - return new BackupAnswer(command, false, result.second()); + return new BackupAnswer(command, false, result.second().trim()); } List outputLines = Arrays.asList(result.second().trim().split("\n")); @@ -68,7 +66,7 @@ public Answer execute(TakeBackupCommand command, LibvirtComputingResource libvir backupSize = Long.parseLong(outputLines.get(outputLines.size()-1).trim()); } - BackupAnswer answer = new BackupAnswer(command, true, result.second()); + BackupAnswer answer = new BackupAnswer(command, true, result.second().trim()); answer.setSize(backupSize); return answer; } diff --git a/scripts/vm/hypervisor/kvm/nasbackup.sh b/scripts/vm/hypervisor/kvm/nasbackup.sh index ae21255dc781..4caae9c4da7c 100755 --- a/scripts/vm/hypervisor/kvm/nasbackup.sh +++ b/scripts/vm/hypervisor/kvm/nasbackup.sh @@ -43,24 +43,31 @@ backup_vm() { deviceId=0 name="root" echo "" > $dest/backup.xml - for disk in $(virsh -c qemu:///system domblklist $vm --details 2>/dev/null | awk '/disk/{print$3}'); do - echo "" >> $dest/backup.xml + for disk in $(virsh -c qemu:///system domblklist $VM --details 2>/dev/null | awk '/disk/{print$3}'); do + volpath=$(virsh -c qemu:///system domblklist $VM --details | awk "/$disk/{print $4}" | sed 's/.*\///') + echo "" >> $dest/backup.xml deviceId=$((devideId+1)) name="datadisk" done echo "" >> $dest/backup.xml - virsh -c qemu:///system backup-begin --domain $vm --backupxml $dest/backup.xml > /dev/null 2>/dev/null - virsh -c qemu:///system dumpxml $vm > $dest/domain-$vm.xml 2>/dev/null + # Start push backup + virsh -c qemu:///system backup-begin --domain $VM --backupxml $dest/backup.xml > /dev/null 2>/dev/null - until virsh -c qemu:///system domjobinfo $vm --completed --keep-completed 2>/dev/null | grep "Completed" > /dev/null; do + # Backup domain information + virsh -c qemu:///system dumpxml $VM > $dest/domain-config.xml 2>/dev/null + virsh -c qemu:///system dominfo $VM > $dest/dominfo.xml 2>/dev/null + virsh -c qemu:///system domiflist $VM > $dest/domiflist.xml 2>/dev/null + virsh -c qemu:///system domblklist $VM > $dest/domblklist.xml 2>/dev/null + + until virsh -c qemu:///system domjobinfo $VM --completed --keep-completed 2>/dev/null | grep "Completed" > /dev/null; do sleep 5 done rm -f $dest/backup.xml sync # Print statistics - virsh -c qemu:///system domjobinfo $vm --completed + virsh -c qemu:///system domjobinfo $VM --completed du -sb $dest | cut -f1 umount $mount_point From 7db5178e660e9c273092355a01cf3ea13a9e88cd Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Fri, 2 Aug 2024 15:31:06 +0530 Subject: [PATCH 17/68] ui tweaks --- ui/public/locales/en.json | 1 + ui/src/components/view/ListResourceTable.vue | 4 ++++ ui/src/components/view/ListView.vue | 2 +- ui/src/config/section/config.js | 21 ++++++++++++++++++++ ui/src/config/section/storage.js | 2 +- ui/src/views/compute/InstanceTab.vue | 4 ++-- 6 files changed, 30 insertions(+), 4 deletions(-) diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 8f2c9fa6d151..393a9067c714 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -398,6 +398,7 @@ "label.backup.offering.assign": "Assign Instance to backup offering", "label.backup.offering.remove": "Remove Instance from backup offering", "label.backup.offerings": "Backup offerings", +"label.backup.repository": "Backup Repository", "label.backup.restore": "Restore Instance backup", "label.backupofferingid": "Backup offering", "label.backupofferingname": "Backup offering", diff --git a/ui/src/components/view/ListResourceTable.vue b/ui/src/components/view/ListResourceTable.vue index a7e805b54439..001c81aa043a 100644 --- a/ui/src/components/view/ListResourceTable.vue +++ b/ui/src/components/view/ListResourceTable.vue @@ -50,6 +50,10 @@ {{ $toLocaleDate(text) }} + + diff --git a/ui/src/components/view/ListView.vue b/ui/src/components/view/ListView.vue index 2a379b5bf520..11035b95d680 100644 --- a/ui/src/components/view/ListView.vue +++ b/ui/src/components/view/ListView.vue @@ -174,7 +174,7 @@ {{ text }} {{ text }} -