Merge remote-tracking branch 'origin/4.15'

Fix db upgrade path conflict, add 4.15.1.0->4.16.0.0 for master, bump
systemvmtemplate version to 4.16.

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>

Author: yadvr

engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java

@@ -69,6 +69,8 @@
 import com.cloud.upgrade.dao.Upgrade41300to41310;
 import com.cloud.upgrade.dao.Upgrade41310to41400;
 import com.cloud.upgrade.dao.Upgrade41400to41500;
+import com.cloud.upgrade.dao.Upgrade41500to41510;
+import com.cloud.upgrade.dao.Upgrade41510to41600;
 import com.cloud.upgrade.dao.Upgrade420to421;
 import com.cloud.upgrade.dao.Upgrade421to430;
 import com.cloud.upgrade.dao.Upgrade430to440;
@@ -193,6 +195,9 @@ public DatabaseUpgradeChecker() {
                 .next("4.13.0.0", new Upgrade41300to41310())
                 .next("4.13.1.0", new Upgrade41310to41400())
                 .next("4.14.0.0", new Upgrade41400to41500())
+                .next("4.14.1.0", new Upgrade41400to41500())
+                .next("4.15.0.0", new Upgrade41500to41510())
+                .next("4.15.1.0", new Upgrade41510to41600())
                 .build();
     }
 

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41400to41500.java

@@ -23,16 +23,12 @@
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import org.apache.log4j.Logger;
 
-import com.cloud.hypervisor.Hypervisor;
 import com.cloud.utils.exception.CloudRuntimeException;
 
 public class Upgrade41400to41500 implements DbUpgrade {
@@ -67,178 +63,9 @@ public InputStream[] getPrepareScripts() {
 
     @Override
     public void performDataMigration(Connection conn) {
-        updateSystemVmTemplates(conn);
         addRolePermissionsForNewReadOnlyAndSupportRoles(conn);
     }
 
-    @SuppressWarnings("serial")
-    private void updateSystemVmTemplates(final Connection conn) {
-        LOG.debug("Updating System Vm template IDs");
-        final Set<Hypervisor.HypervisorType> hypervisorsListInUse = new HashSet<Hypervisor.HypervisorType>();
-        try (PreparedStatement pstmt = conn.prepareStatement("select distinct(hypervisor_type) from `cloud`.`cluster` where removed is null"); ResultSet rs = pstmt.executeQuery()) {
-            while (rs.next()) {
-                switch (Hypervisor.HypervisorType.getType(rs.getString(1))) {
-                    case XenServer:
-                        hypervisorsListInUse.add(Hypervisor.HypervisorType.XenServer);
-                        break;
-                    case KVM:
-                        hypervisorsListInUse.add(Hypervisor.HypervisorType.KVM);
-                        break;
-                    case VMware:
-                        hypervisorsListInUse.add(Hypervisor.HypervisorType.VMware);
-                        break;
-                    case Hyperv:
-                        hypervisorsListInUse.add(Hypervisor.HypervisorType.Hyperv);
-                        break;
-                    case LXC:
-                        hypervisorsListInUse.add(Hypervisor.HypervisorType.LXC);
-                        break;
-                    case Ovm3:
-                        hypervisorsListInUse.add(Hypervisor.HypervisorType.Ovm3);
-                        break;
-                    default:
-                        break;
-                }
-            }
-        } catch (final SQLException e) {
-            LOG.error("updateSystemVmTemplates: Exception caught while getting hypervisor types from clusters: " + e.getMessage());
-            throw new CloudRuntimeException("updateSystemVmTemplates:Exception while getting hypervisor types from clusters", e);
-        }
-
-        final Map<Hypervisor.HypervisorType, String> NewTemplateNameList = new HashMap<Hypervisor.HypervisorType, String>() {
-            {
-                put(Hypervisor.HypervisorType.KVM, "systemvm-kvm-4.15.0");
-                put(Hypervisor.HypervisorType.VMware, "systemvm-vmware-4.15.0");
-                put(Hypervisor.HypervisorType.XenServer, "systemvm-xenserver-4.15.0");
-                put(Hypervisor.HypervisorType.Hyperv, "systemvm-hyperv-4.15.0");
-                put(Hypervisor.HypervisorType.LXC, "systemvm-lxc-4.15.0");
-                put(Hypervisor.HypervisorType.Ovm3, "systemvm-ovm3-4.15.0");
-            }
-        };
-
-        final Map<Hypervisor.HypervisorType, String> routerTemplateConfigurationNames = new HashMap<Hypervisor.HypervisorType, String>() {
-            {
-                put(Hypervisor.HypervisorType.KVM, "router.template.kvm");
-                put(Hypervisor.HypervisorType.VMware, "router.template.vmware");
-                put(Hypervisor.HypervisorType.XenServer, "router.template.xenserver");
-                put(Hypervisor.HypervisorType.Hyperv, "router.template.hyperv");
-                put(Hypervisor.HypervisorType.LXC, "router.template.lxc");
-                put(Hypervisor.HypervisorType.Ovm3, "router.template.ovm3");
-            }
-        };
-
-        final Map<Hypervisor.HypervisorType, String> newTemplateUrl = new HashMap<Hypervisor.HypervisorType, String>() {
-            {
-                put(Hypervisor.HypervisorType.KVM, "https://download.cloudstack.org/systemvm/4.15/systemvmtemplate-4.15.0-kvm.qcow2.bz2");
-                put(Hypervisor.HypervisorType.VMware, "https://download.cloudstack.org/systemvm/4.15/systemvmtemplate-4.15.0-vmware.ova");
-                put(Hypervisor.HypervisorType.XenServer, "https://download.cloudstack.org/systemvm/4.15/systemvmtemplate-4.15.0-xen.vhd.bz2");
-                put(Hypervisor.HypervisorType.Hyperv, "https://download.cloudstack.org/systemvm/4.15/systemvmtemplate-4.15.0-hyperv.vhd.zip");
-                put(Hypervisor.HypervisorType.LXC, "https://download.cloudstack.org/systemvm/4.15/systemvmtemplate-4.15.0-kvm.qcow2.bz2");
-                put(Hypervisor.HypervisorType.Ovm3, "https://download.cloudstack.org/systemvm/4.15/systemvmtemplate-4.15.0-ovm.raw.bz2");
-            }
-        };
-
-        final Map<Hypervisor.HypervisorType, String> newTemplateChecksum = new HashMap<Hypervisor.HypervisorType, String>() {
-            {
-                put(Hypervisor.HypervisorType.KVM, "81b3e48bb934784a13555a43c5ef5ffb");
-                put(Hypervisor.HypervisorType.XenServer, "1b178a5dbdbe090555515340144c6017");
-                put(Hypervisor.HypervisorType.VMware, "e6a88e518c57d6f36c096c4204c3417f");
-                put(Hypervisor.HypervisorType.Hyperv, "5c94da45337cf3e1910dcbe084d4b9ad");
-                put(Hypervisor.HypervisorType.LXC, "81b3e48bb934784a13555a43c5ef5ffb");
-                put(Hypervisor.HypervisorType.Ovm3, "875c5c65455fc06c4a012394410db375");
-            }
-        };
-
-        for (final Map.Entry<Hypervisor.HypervisorType, String> hypervisorAndTemplateName : NewTemplateNameList.entrySet()) {
-            LOG.debug("Updating " + hypervisorAndTemplateName.getKey() + " System Vms");
-            try (PreparedStatement pstmt = conn.prepareStatement("select id from `cloud`.`vm_template` where name = ? and removed is null order by id desc limit 1")) {
-                // Get systemvm template id for corresponding hypervisor
-                long templateId = -1;
-                pstmt.setString(1, hypervisorAndTemplateName.getValue());
-                try (ResultSet rs = pstmt.executeQuery()) {
-                    if (rs.next()) {
-                        templateId = rs.getLong(1);
-                    }
-                } catch (final SQLException e) {
-                    LOG.error("updateSystemVmTemplates: Exception caught while getting ids of templates: " + e.getMessage());
-                    throw new CloudRuntimeException("updateSystemVmTemplates: Exception caught while getting ids of templates", e);
-                }
-
-                // change template type to SYSTEM
-                if (templateId != -1) {
-                    try (PreparedStatement templ_type_pstmt = conn.prepareStatement("update `cloud`.`vm_template` set type='SYSTEM' where id = ?");) {
-                        templ_type_pstmt.setLong(1, templateId);
-                        templ_type_pstmt.executeUpdate();
-                    } catch (final SQLException e) {
-                        LOG.error("updateSystemVmTemplates:Exception while updating template with id " + templateId + " to be marked as 'system': " + e.getMessage());
-                        throw new CloudRuntimeException("updateSystemVmTemplates:Exception while updating template with id " + templateId + " to be marked as 'system'", e);
-                    }
-                    // update template ID of system Vms
-                    try (PreparedStatement update_templ_id_pstmt = conn
-                            .prepareStatement("update `cloud`.`vm_instance` set vm_template_id = ? where type <> 'User' and hypervisor_type = ? and removed is NULL");) {
-                        update_templ_id_pstmt.setLong(1, templateId);
-                        update_templ_id_pstmt.setString(2, hypervisorAndTemplateName.getKey().toString());
-                        update_templ_id_pstmt.executeUpdate();
-                    } catch (final Exception e) {
-                        LOG.error("updateSystemVmTemplates:Exception while setting template for " + hypervisorAndTemplateName.getKey().toString() + " to " + templateId
-                                + ": " + e.getMessage());
-                        throw new CloudRuntimeException("updateSystemVmTemplates:Exception while setting template for " + hypervisorAndTemplateName.getKey().toString() + " to "
-                                + templateId, e);
-                    }
-
-                    // Change value of global configuration parameter
-                    // router.template.* for the corresponding hypervisor
-                    try (PreparedStatement update_pstmt = conn.prepareStatement("UPDATE `cloud`.`configuration` SET value = ? WHERE name = ?");) {
-                        update_pstmt.setString(1, hypervisorAndTemplateName.getValue());
-                        update_pstmt.setString(2, routerTemplateConfigurationNames.get(hypervisorAndTemplateName.getKey()));
-                        update_pstmt.executeUpdate();
-                    } catch (final SQLException e) {
-                        LOG.error("updateSystemVmTemplates:Exception while setting " + routerTemplateConfigurationNames.get(hypervisorAndTemplateName.getKey()) + " to "
-                                + hypervisorAndTemplateName.getValue() + ": " + e.getMessage());
-                        throw new CloudRuntimeException("updateSystemVmTemplates:Exception while setting "
-                                + routerTemplateConfigurationNames.get(hypervisorAndTemplateName.getKey()) + " to " + hypervisorAndTemplateName.getValue(), e);
-                    }
-
-                    // Change value of global configuration parameter
-                    // minreq.sysvmtemplate.version for the ACS version
-                    try (PreparedStatement update_pstmt = conn.prepareStatement("UPDATE `cloud`.`configuration` SET value = ? WHERE name = ?");) {
-                        update_pstmt.setString(1, "4.15.0");
-                        update_pstmt.setString(2, "minreq.sysvmtemplate.version");
-                        update_pstmt.executeUpdate();
-                    } catch (final SQLException e) {
-                        LOG.error("updateSystemVmTemplates:Exception while setting 'minreq.sysvmtemplate.version' to 4.15.0: " + e.getMessage());
-                        throw new CloudRuntimeException("updateSystemVmTemplates:Exception while setting 'minreq.sysvmtemplate.version' to 4.15.0", e);
-                    }
-                } else {
-                    if (hypervisorsListInUse.contains(hypervisorAndTemplateName.getKey())) {
-                        throw new CloudRuntimeException(getUpgradedVersion() + hypervisorAndTemplateName.getKey() + " SystemVm template not found. Cannot upgrade system Vms");
-                    } else {
-                        LOG.warn(getUpgradedVersion() + hypervisorAndTemplateName.getKey() + " SystemVm template not found. " + hypervisorAndTemplateName.getKey()
-                                + " hypervisor is not used, so not failing upgrade");
-                        // Update the latest template URLs for corresponding
-                        // hypervisor
-                        try (PreparedStatement update_templ_url_pstmt = conn
-                                .prepareStatement("UPDATE `cloud`.`vm_template` SET url = ? , checksum = ? WHERE hypervisor_type = ? AND type = 'SYSTEM' AND removed is null order by id desc limit 1");) {
-                            update_templ_url_pstmt.setString(1, newTemplateUrl.get(hypervisorAndTemplateName.getKey()));
-                            update_templ_url_pstmt.setString(2, newTemplateChecksum.get(hypervisorAndTemplateName.getKey()));
-                            update_templ_url_pstmt.setString(3, hypervisorAndTemplateName.getKey().toString());
-                            update_templ_url_pstmt.executeUpdate();
-                        } catch (final SQLException e) {
-                            LOG.error("updateSystemVmTemplates:Exception while updating 'url' and 'checksum' for hypervisor type "
-                                    + hypervisorAndTemplateName.getKey().toString() + ": " + e.getMessage());
-                            throw new CloudRuntimeException("updateSystemVmTemplates:Exception while updating 'url' and 'checksum' for hypervisor type "
-                                    + hypervisorAndTemplateName.getKey().toString(), e);
-                        }
-                    }
-                }
-            } catch (final SQLException e) {
-                LOG.error("updateSystemVmTemplates:Exception while getting ids of templates: " + e.getMessage());
-                throw new CloudRuntimeException("updateSystemVmTemplates:Exception while getting ids of templates", e);
-            }
-        }
-        LOG.debug("Updating System Vm Template IDs Complete");
-    }
-
     private void addRolePermissionsForNewReadOnlyAndSupportRoles(final Connection conn) {
         addRolePermissionsForReadOnlyAdmin(conn);
         addRolePermissionsForReadOnlyUser(conn);

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41500to41510.java

@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.upgrade.dao;
+
+import java.io.InputStream;
+import java.sql.Connection;
+
+import org.apache.log4j.Logger;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+
+public class Upgrade41500to41510 implements DbUpgrade {
+
+    final static Logger LOG = Logger.getLogger(Upgrade41500to41510.class);
+
+    @Override
+    public String[] getUpgradableVersionRange() {
+        return new String[] {"4.15.0.0", "4.15.1.0"};
+    }
+
+    @Override
+    public String getUpgradedVersion() {
+        return "4.15.1.0";
+    }
+
+    @Override
+    public boolean supportsRollingUpgrade() {
+        return false;
+    }
+
+    @Override
+    public InputStream[] getPrepareScripts() {
+        final String scriptFile = "META-INF/db/schema-41500to41510.sql";
+        final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile);
+        if (script == null) {
+            throw new CloudRuntimeException("Unable to find " + scriptFile);
+        }
+
+        return new InputStream[] {script};
+    }
+
+    @Override
+    public void performDataMigration(Connection conn) {
+    }
+
+    @Override
+    public InputStream[] getCleanupScripts() {
+        final String scriptFile = "META-INF/db/schema-41500to41510-cleanup.sql";
+        final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile);
+        if (script == null) {
+            throw new CloudRuntimeException("Unable to find " + scriptFile);
+        }
+
+        return new InputStream[] {script};
+    }
+}