Merge branch '4.20' into polish_server

Author: harikrishna-patnala

agent/src/main/java/com/cloud/agent/Agent.java

@@ -1228,7 +1228,14 @@ public void doTask(final Task task) throws TaskExecutionException {
                     logger.error("Error parsing task", e);
                 }
             } else if (task.getType() == Task.Type.DISCONNECT) {
-                logger.debug("Executing disconnect task - {}", () -> getLinkLog(task.getLink()));
+                try {
+                    // an issue has been found if reconnect immediately after disconnecting.
+                    // wait 5 seconds before reconnecting
+                    logger.debug("Wait for 5 secs before reconnecting, disconnect task - {}", () -> getLinkLog(task.getLink()));
+                    Thread.sleep(5000);
+                } catch (InterruptedException e) {
+                }
+                logger.debug("Executing disconnect task - {} and reconnecting", () -> getLinkLog(task.getLink()));
                 reconnect(task.getLink());
             } else if (task.getType() == Task.Type.OTHER) {
                 processOtherTask(task);

api/src/main/java/com/cloud/server/ManagementService.java

@@ -71,6 +71,8 @@
 import com.cloud.capacity.Capacity;
 import com.cloud.dc.Pod;
 import com.cloud.dc.Vlan;
+import com.cloud.deploy.DeploymentPlan;
+import com.cloud.deploy.DeploymentPlanner.ExcludeList;
 import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.ManagementServerException;
 import com.cloud.exception.ResourceUnavailableException;
@@ -91,6 +93,7 @@
 import com.cloud.vm.InstanceGroup;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachine.Type;
+import com.cloud.vm.VirtualMachineProfile;
 
 /**
  * Hopefully this is temporary.
@@ -452,6 +455,19 @@ public interface ManagementService {
 
     Ternary<Pair<List<? extends Host>, Integer>, List<? extends Host>, Map<Host, Boolean>> listHostsForMigrationOfVM(VirtualMachine vm, Long startIndex, Long pageSize, String keyword, List<VirtualMachine> vmList);
 
+    /**
+     * Apply affinity group constraints and other exclusion rules for VM migration.
+     * This is a helper method that can be used independently for per-iteration affinity checks in DRS.
+     *
+     * @param vm The virtual machine to migrate
+     * @param vmProfile The VM profile
+     * @param plan The deployment plan
+     * @param vmList List of VMs with current/simulated placements for affinity processing
+     * @return ExcludeList containing hosts to avoid
+     */
+    ExcludeList applyAffinityConstraints(VirtualMachine vm, VirtualMachineProfile vmProfile,
+            DeploymentPlan plan, List<VirtualMachine> vmList);
+
     /**
      * List storage pools for live migrating of a volume. The API returns list of all pools in the cluster to which the
      * volume can be migrated. Current pool is not included in the list. In case of vSphere datastore cluster storage pools,

api/src/main/java/org/apache/cloudstack/api/command/admin/network/CreateManagementNetworkIpRangeCmd.java

@@ -34,6 +34,7 @@
 import com.cloud.exception.ResourceAllocationException;
 import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.user.Account;
+import com.cloud.utils.StringUtils;
 
 @APICommand(name = "createManagementNetworkIpRange",
         description = "Creates a Management network IP range.",
@@ -118,7 +119,7 @@ public Boolean isForSystemVms() {
     }
 
     public String getVlan() {
-        if (vlan == null || vlan.isEmpty()) {
+        if (StringUtils.isBlank(vlan)) {
             vlan = "untagged";
         }
         return vlan;

api/src/main/java/org/apache/cloudstack/api/command/admin/vlan/CreateVlanIpRangeCmd.java

@@ -17,6 +17,7 @@
 package org.apache.cloudstack.api.command.admin.vlan;
 
 import com.cloud.utils.net.NetUtils;
+import com.cloud.utils.StringUtils;
 
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiConstants;
@@ -162,7 +163,7 @@ public boolean isForNsx() {
     }
 
     public String getVlan() {
-        if ((vlan == null || vlan.isEmpty()) && !isForNsx()) {
+        if (StringUtils.isBlank(vlan) && !isForNsx()) {
             vlan = "untagged";
         }
         return vlan;

api/src/main/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmd.java

@@ -211,7 +211,7 @@ public String getUrl() {
     }
 
     public String getTemplateTag() {
-        return templateTag;
+        return StringUtils.isBlank(templateTag) ? null : templateTag;
     }
 
     public Map getDetails() {

api/src/main/java/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java

@@ -160,7 +160,7 @@ public Boolean isSshKeyEnabled() {
     }
 
     public String getTemplateTag() {
-        return templateTag;
+        return StringUtils.isBlank(templateTag) ? null : templateTag;
     }
 
     public boolean isDeployAsIs() {

api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java

@@ -265,7 +265,7 @@ public String getChecksum() {
     }
 
     public String getTemplateTag() {
-        return templateTag;
+        return StringUtils.isBlank(templateTag) ? null : templateTag;
     }
 
     public Map getDetails() {

api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsAlgorithm.java

@@ -22,10 +22,10 @@
 import com.cloud.host.Host;
 import com.cloud.offering.ServiceOffering;
 import com.cloud.org.Cluster;
-import com.cloud.utils.Pair;
 import com.cloud.utils.Ternary;
 import com.cloud.utils.component.Adapter;
 import com.cloud.vm.VirtualMachine;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.math3.stat.descriptive.moment.Mean;
 import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
 
@@ -40,6 +40,9 @@
 
 public interface ClusterDrsAlgorithm extends Adapter {
 
+    Mean MEAN_CALCULATOR = new Mean();
+    StandardDeviation STDDEV_CALCULATOR = new StandardDeviation(false);
+
     /**
      * Determines whether a DRS operation is needed for a given cluster and host-VM
      * mapping.
@@ -59,79 +62,121 @@ public interface ClusterDrsAlgorithm extends Adapter {
     boolean needsDrs(Cluster cluster, List<Ternary<Long, Long, Long>> cpuList,
                      List<Ternary<Long, Long, Long>> memoryList) throws ConfigurationException;
 
-
     /**
-     * Determines the metrics for a given virtual machine and destination host in a DRS cluster.
-     *
-     * @param clusterId
-     *         the ID of the cluster to check
-     * @param vm
-     *         the virtual machine to check
-     * @param serviceOffering
-     *         the service offering for the virtual machine
-     * @param destHost
-     *         the destination host for the virtual machine
-     * @param hostCpuMap
-     *         a map of host IDs to the Ternary of used, reserved and total CPU on each host
-     * @param hostMemoryMap
-     *         a map of host IDs to the Ternary of used, reserved and total memory on each host
-     * @param requiresStorageMotion
-     *         whether storage motion is required for the virtual machine
+     * Calculates the metrics (improvement, cost, benefit) for migrating a VM to a destination host. Improvement is
+     * calculated based on the change in cluster imbalance before and after the migration.
      *
+     * @param cluster the cluster to check
+     * @param vm the virtual machine to check
+     * @param serviceOffering the service offering for the virtual machine
+     * @param destHost the destination host for the virtual machine
+     * @param hostCpuMap a map of host IDs to the Ternary of used, reserved and total CPU on each host
+     * @param hostMemoryMap a map of host IDs to the Ternary of used, reserved and total memory on each host
+     * @param requiresStorageMotion whether storage motion is required for the virtual machine
+     * @param preImbalance the pre-calculated cluster imbalance before migration (null to calculate it)
+     * @param baseMetricsArray pre-calculated array of all host metrics before migration
+     * @param hostIdToIndexMap mapping from host ID to index in the metrics array
      * @return a ternary containing improvement, cost, benefit
      */
     Ternary<Double, Double, Double> getMetrics(Cluster cluster, VirtualMachine vm, ServiceOffering serviceOffering,
             Host destHost, Map<Long, Ternary<Long, Long, Long>> hostCpuMap,
             Map<Long, Ternary<Long, Long, Long>> hostMemoryMap,
-            Boolean requiresStorageMotion) throws ConfigurationException;
+            Boolean requiresStorageMotion, Double preImbalance,
+            double[] baseMetricsArray, Map<Long, Integer> hostIdToIndexMap) throws ConfigurationException;
 
     /**
-     * Calculates the imbalance of the cluster after a virtual machine migration.
+     * Calculates the cluster imbalance after migrating a VM to a destination host.
      *
-     * @param serviceOffering
-     *         the service offering for the virtual machine
-     * @param vm
-     *         the virtual machine being migrated
-     * @param destHost
-     *         the destination host for the virtual machine
-     * @param hostCpuMap
-     *         a map of host IDs to the Ternary of used, reserved and total CPU on each host
-     * @param hostMemoryMap
-     *         a map of host IDs to the Ternary of used, reserved and total memory on each host
+     * @param vm the virtual machine being migrated
+     * @param destHost the destination host for the virtual machine
+     * @param clusterId the cluster ID
+     * @param vmMetric the VM's resource consumption metric
+     * @param baseMetricsArray pre-calculated array of all host metrics before migration
+     * @param hostIdToIndexMap mapping from host ID to index in the metrics array
+     * @return the cluster imbalance after migration
+     */
+    default Double getImbalancePostMigration(VirtualMachine vm,
+            Host destHost, Long clusterId, long vmMetric, double[] baseMetricsArray,
+            Map<Long, Integer> hostIdToIndexMap, Map<Long, Ternary<Long, Long, Long>> hostCpuMap,
+            Map<Long, Ternary<Long, Long, Long>> hostMemoryMap) {
+        // Create a copy of the base array and adjust only the two affected hosts
+        double[] adjustedMetrics = new double[baseMetricsArray.length];
+        System.arraycopy(baseMetricsArray, 0, adjustedMetrics, 0, baseMetricsArray.length);
+
+        long destHostId = destHost.getId();
+        long vmHostId = vm.getHostId();
+
+        // Adjust source host (remove VM resources)
+        Integer sourceIndex = hostIdToIndexMap.get(vmHostId);
+        if (sourceIndex != null && sourceIndex < adjustedMetrics.length) {
+            Map<Long, Ternary<Long, Long, Long>> sourceMetricsMap = getClusterDrsMetric(clusterId).equals("cpu") ? hostCpuMap : hostMemoryMap;
+            Ternary<Long, Long, Long> sourceMetrics = sourceMetricsMap.get(vmHostId);
+            if (sourceMetrics != null) {
+                adjustedMetrics[sourceIndex] = getMetricValuePostMigration(clusterId, sourceMetrics, vmMetric, vmHostId, destHostId, vmHostId);
+            }
+        }
+
+        // Adjust destination host (add VM resources)
+        Integer destIndex = hostIdToIndexMap.get(destHostId);
+        if (destIndex != null && destIndex < adjustedMetrics.length) {
+            Map<Long, Ternary<Long, Long, Long>> destMetricsMap = getClusterDrsMetric(clusterId).equals("cpu") ? hostCpuMap : hostMemoryMap;
+            Ternary<Long, Long, Long> destMetrics = destMetricsMap.get(destHostId);
+            if (destMetrics != null) {
+                adjustedMetrics[destIndex] = getMetricValuePostMigration(clusterId, destMetrics, vmMetric, destHostId, destHostId, vmHostId);
+            }
+        }
+
+        return calculateImbalance(adjustedMetrics);
+    }
+
+    /**
+     * Calculate imbalance from an array of metric values.
+     * Imbalance is defined as standard deviation divided by mean.
      *
-     * @return a pair containing the CPU and memory imbalance of the cluster after the migration
+     * Uses reusable stateless calculator objects to avoid object creation overhead.
+     * @param values array of metric values
+     * @return calculated imbalance
      */
-    default Double getImbalancePostMigration(ServiceOffering serviceOffering, VirtualMachine vm,
-            Host destHost, Map<Long, Ternary<Long, Long, Long>> hostCpuMap,
-            Map<Long, Ternary<Long, Long, Long>> hostMemoryMap) throws ConfigurationException {
-        Pair<Long, Map<Long, Ternary<Long, Long, Long>>> pair = getHostMetricsMapAndType(destHost.getClusterId(), serviceOffering, hostCpuMap, hostMemoryMap);
-        long vmMetric = pair.first();
-        Map<Long, Ternary<Long, Long, Long>> hostMetricsMap = pair.second();
+    private static double calculateImbalance(double[] values) {
+        if (values == null || values.length == 0) {
+            return 0.0;
+        }
 
-        List<Double> list = new ArrayList<>();
-        for (Long hostId : hostMetricsMap.keySet()) {
-            list.add(getMetricValuePostMigration(destHost.getClusterId(), hostMetricsMap.get(hostId), vmMetric, hostId, destHost.getId(), vm.getHostId()));
+        double mean = MEAN_CALCULATOR.evaluate(values);
+        if (mean == 0.0) {
+            return 0.0; // Avoid division by zero
         }
-        return getImbalance(list);
+        double stdDev = STDDEV_CALCULATOR.evaluate(values, mean);
+        return stdDev / mean;
     }
 
-    private Pair<Long, Map<Long, Ternary<Long, Long, Long>>> getHostMetricsMapAndType(Long clusterId,
-            ServiceOffering serviceOffering, Map<Long, Ternary<Long, Long, Long>> hostCpuMap,
-            Map<Long, Ternary<Long, Long, Long>> hostMemoryMap) throws ConfigurationException {
+    /**
+     * Helper method to get VM metric based on cluster configuration.
+     */
+    static long getVmMetric(ServiceOffering serviceOffering, Long clusterId) throws ConfigurationException {
         String metric = getClusterDrsMetric(clusterId);
-        Pair<Long, Map<Long, Ternary<Long, Long, Long>>> pair;
         switch (metric) {
             case "cpu":
-                pair = new Pair<>((long) serviceOffering.getCpu() * serviceOffering.getSpeed(), hostCpuMap);
-                break;
+                return (long) serviceOffering.getCpu() * serviceOffering.getSpeed();
             case "memory":
-                pair = new Pair<>(serviceOffering.getRamSize() * 1024L * 1024L, hostMemoryMap);
-                break;
+                return serviceOffering.getRamSize() * 1024L * 1024L;
             default:
                 throw new ConfigurationException(
                         String.format("Invalid metric: %s for cluster: %d", metric, clusterId));
         }
-        return pair;
+    }
+
+    /**
+     * Helper method to calculate metrics from pre and post imbalance values.
+     */
+    default Ternary<Double, Double, Double> calculateMetricsFromImbalances(Double preImbalance, Double postImbalance) {
+        // This needs more research to determine the cost and benefit of a migration
+        // TODO: Cost should be a factor of the VM size and the host capacity
+        // TODO: Benefit should be a factor of the VM size and the host capacity and the number of VMs on the host
+        final double improvement = preImbalance - postImbalance;
+        final double cost = 0.0;
+        final double benefit = 1.0;
+        return new Ternary<>(improvement, cost, benefit);
     }
 
     private Double getMetricValuePostMigration(Long clusterId, Ternary<Long, Long, Long> metrics, long vmMetric,
@@ -151,9 +196,26 @@ private Double getMetricValuePostMigration(Long clusterId, Ternary<Long, Long, L
     }
 
     private static Double getImbalance(List<Double> metricList) {
-        Double clusterMeanMetric = getClusterMeanMetric(metricList);
-        Double clusterStandardDeviation = getClusterStandardDeviation(metricList, clusterMeanMetric);
-        return clusterStandardDeviation / clusterMeanMetric;
+        if (CollectionUtils.isEmpty(metricList)) {
+            return 0.0;
+        }
+        // Convert List<Double> to double[] once, avoiding repeated conversions
+        double[] values = new double[metricList.size()];
+        int index = 0;
+        for (Double value : metricList) {
+            if (value != null) {
+                values[index++] = value;
+            }
+        }
+
+        // Trim array if some values were null
+        if (index < values.length) {
+            double[] trimmed = new double[index];
+            System.arraycopy(values, 0, trimmed, 0, index);
+            values = trimmed;
+        }
+
+        return calculateImbalance(values);
     }
 
     static String getClusterDrsMetric(long clusterId) {
@@ -181,36 +243,6 @@ static Double getMetricValue(long clusterId, long used, long free, long total, F
         return null;
     }
 
-    /**
-     * Mean is the average of a collection or set of metrics. In context of a DRS
-     * cluster, the cluster metrics defined as the average metrics value for some
-     * metric (such as CPU, memory etc.) for every resource such as host.
-     * Cluster Mean Metric, mavg = (∑mi) / N, where mi is a measurable metric for a
-     * resource ‘i’ in a cluster with total N number of resources.
-     */
-    static Double getClusterMeanMetric(List<Double> metricList) {
-        return new Mean().evaluate(metricList.stream().mapToDouble(i -> i).toArray());
-    }
-
-    /**
-     * Standard deviation is defined as the square root of the absolute squared sum
-     * of difference of a metric from its mean for every resource divided by the
-     * total number of resources. In context of the DRS, the cluster standard
-     * deviation is the standard deviation based on a metric of resources in a
-     * cluster such as for the allocation or utilisation CPU/memory metric of hosts
-     * in a cluster.
-     * Cluster Standard Deviation, σc = sqrt((∑∣mi−mavg∣^2) / N), where mavg is the
-     * mean metric value and mi is a measurable metric for some resource ‘i’ in the
-     * cluster with total N number of resources.
-     */
-    static Double getClusterStandardDeviation(List<Double> metricList, Double mean) {
-        if (mean != null) {
-            return new StandardDeviation(false).evaluate(metricList.stream().mapToDouble(i -> i).toArray(), mean);
-        } else {
-            return new StandardDeviation(false).evaluate(metricList.stream().mapToDouble(i -> i).toArray());
-        }
-    }
-
     static boolean getDrsMetricUseRatio(long clusterId) {
         return ClusterDrsMetricUseRatio.valueIn(clusterId);
     }

debian/rules

@@ -4,7 +4,6 @@ VERSION := $(shell grep '<version>' pom.xml | head -2 | tail -1 | cut -d'>' -f2
 PACKAGE = $(shell dh_listpackages|head -n 1|cut -d '-' -f 1)
 SYSCONFDIR = "/etc"
 DESTDIR = "debian/tmp"
-CMK_REL := $(shell wget -O - "https://api.github.com/repos/apache/cloudstack-cloudmonkey/releases" 2>/dev/null | jq -r '.[0].tag_name')
 
 %:
 	dh $@ --with systemd
@@ -86,7 +85,7 @@ override_dh_auto_install:
 	rm -rf $(DESTDIR)/usr/share/$(PACKAGE)-management/templates/systemvm/md5sum.txt
 
 	# Bundle cmk in cloudstack-management
-	wget https://github.com/apache/cloudstack-cloudmonkey/releases/download/$(CMK_REL)/cmk.linux.x86-64 -O $(DESTDIR)/usr/bin/cmk
+	wget https://github.com/apache/cloudstack-cloudmonkey/releases/latest/download/cmk.linux.x86-64 -O $(DESTDIR)/usr/bin/cmk
 	chmod +x $(DESTDIR)/usr/bin/cmk
 
 	# nast hack for a couple of configuration files