merge forward 4.22 to main

Author: Daan Hoogland

api/src/main/java/com/cloud/server/ManagementService.java

@@ -77,6 +77,8 @@
 import com.cloud.capacity.Capacity;
 import com.cloud.dc.Pod;
 import com.cloud.dc.Vlan;
+import com.cloud.deploy.DeploymentPlan;
+import com.cloud.deploy.DeploymentPlanner.ExcludeList;
 import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.ManagementServerException;
 import com.cloud.exception.ResourceUnavailableException;
@@ -97,6 +99,7 @@
 import com.cloud.vm.InstanceGroup;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachine.Type;
+import com.cloud.vm.VirtualMachineProfile;
 
 /**
  * Hopefully this is temporary.
@@ -478,6 +481,19 @@ public interface ManagementService {
 
     Ternary<Pair<List<? extends Host>, Integer>, List<? extends Host>, Map<Host, Boolean>> listHostsForMigrationOfVM(VirtualMachine vm, Long startIndex, Long pageSize, String keyword, List<VirtualMachine> vmList);
 
+    /**
+     * Apply affinity group constraints and other exclusion rules for VM migration.
+     * This is a helper method that can be used independently for per-iteration affinity checks in DRS.
+     *
+     * @param vm The virtual machine to migrate
+     * @param vmProfile The VM profile
+     * @param plan The deployment plan
+     * @param vmList List of VMs with current/simulated placements for affinity processing
+     * @return ExcludeList containing hosts to avoid
+     */
+    ExcludeList applyAffinityConstraints(VirtualMachine vm, VirtualMachineProfile vmProfile,
+            DeploymentPlan plan, List<VirtualMachine> vmList);
+
     /**
      * List storage pools for live migrating of a volume. The API returns list of all pools in the cluster to which the
      * volume can be migrated. Current pool is not included in the list. In case of vSphere datastore cluster storage pools,

api/src/main/java/org/apache/cloudstack/api/command/admin/network/CreateManagementNetworkIpRangeCmd.java

@@ -34,6 +34,7 @@
 import com.cloud.exception.ResourceAllocationException;
 import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.user.Account;
+import com.cloud.utils.StringUtils;
 
 @APICommand(name = "createManagementNetworkIpRange",
         description = "Creates a Management network IP range.",
@@ -118,7 +119,7 @@ public Boolean isForSystemVms() {
     }
 
     public String getVlan() {
-        if (vlan == null || vlan.isEmpty()) {
+        if (StringUtils.isBlank(vlan)) {
             vlan = "untagged";
         }
         return vlan;

api/src/main/java/org/apache/cloudstack/api/command/admin/vlan/CreateVlanIpRangeCmd.java

@@ -19,6 +19,7 @@
 import com.cloud.configuration.ConfigurationService;
 import com.cloud.network.Network;
 import com.cloud.utils.net.NetUtils;
+import com.cloud.utils.StringUtils;
 
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiConstants;
@@ -163,7 +164,7 @@ public Network.Provider getProvider() {
     }
 
     public String getVlan() {
-        if ((vlan == null || vlan.isEmpty()) && !ConfigurationService.IsIpRangeForProvider(getProvider())) {
+        if (StringUtils.isBlank(vlan) && !ConfigurationService.IsIpRangeForProvider(getProvider())) {
             vlan = "untagged";
         }
         return vlan;

api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsAlgorithm.java

@@ -22,10 +22,10 @@
 import com.cloud.host.Host;
 import com.cloud.offering.ServiceOffering;
 import com.cloud.org.Cluster;
-import com.cloud.utils.Pair;
 import com.cloud.utils.Ternary;
 import com.cloud.utils.component.Adapter;
 import com.cloud.vm.VirtualMachine;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.math3.stat.descriptive.moment.Mean;
 import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
 
@@ -40,6 +40,9 @@
 
 public interface ClusterDrsAlgorithm extends Adapter {
 
+    Mean MEAN_CALCULATOR = new Mean();
+    StandardDeviation STDDEV_CALCULATOR = new StandardDeviation(false);
+
     /**
      * Determines whether a DRS operation is needed for a given cluster and host-VM
      * mapping.
@@ -59,79 +62,121 @@ public interface ClusterDrsAlgorithm extends Adapter {
     boolean needsDrs(Cluster cluster, List<Ternary<Long, Long, Long>> cpuList,
                      List<Ternary<Long, Long, Long>> memoryList) throws ConfigurationException;
 
-
     /**
-     * Determines the metrics for a given virtual machine and destination host in a DRS cluster.
-     *
-     * @param clusterId
-     *         the ID of the cluster to check
-     * @param vm
-     *         the virtual machine to check
-     * @param serviceOffering
-     *         the service offering for the virtual machine
-     * @param destHost
-     *         the destination host for the virtual machine
-     * @param hostCpuMap
-     *         a map of host IDs to the Ternary of used, reserved and total CPU on each host
-     * @param hostMemoryMap
-     *         a map of host IDs to the Ternary of used, reserved and total memory on each host
-     * @param requiresStorageMotion
-     *         whether storage motion is required for the virtual machine
+     * Calculates the metrics (improvement, cost, benefit) for migrating a VM to a destination host. Improvement is
+     * calculated based on the change in cluster imbalance before and after the migration.
      *
+     * @param cluster the cluster to check
+     * @param vm the virtual machine to check
+     * @param serviceOffering the service offering for the virtual machine
+     * @param destHost the destination host for the virtual machine
+     * @param hostCpuMap a map of host IDs to the Ternary of used, reserved and total CPU on each host
+     * @param hostMemoryMap a map of host IDs to the Ternary of used, reserved and total memory on each host
+     * @param requiresStorageMotion whether storage motion is required for the virtual machine
+     * @param preImbalance the pre-calculated cluster imbalance before migration (null to calculate it)
+     * @param baseMetricsArray pre-calculated array of all host metrics before migration
+     * @param hostIdToIndexMap mapping from host ID to index in the metrics array
      * @return a ternary containing improvement, cost, benefit
      */
     Ternary<Double, Double, Double> getMetrics(Cluster cluster, VirtualMachine vm, ServiceOffering serviceOffering,
             Host destHost, Map<Long, Ternary<Long, Long, Long>> hostCpuMap,
             Map<Long, Ternary<Long, Long, Long>> hostMemoryMap,
-            Boolean requiresStorageMotion) throws ConfigurationException;
+            Boolean requiresStorageMotion, Double preImbalance,
+            double[] baseMetricsArray, Map<Long, Integer> hostIdToIndexMap) throws ConfigurationException;
 
     /**
-     * Calculates the imbalance of the cluster after a virtual machine migration.
+     * Calculates the cluster imbalance after migrating a VM to a destination host.
      *
-     * @param serviceOffering
-     *         the service offering for the virtual machine
-     * @param vm
-     *         the virtual machine being migrated
-     * @param destHost
-     *         the destination host for the virtual machine
-     * @param hostCpuMap
-     *         a map of host IDs to the Ternary of used, reserved and total CPU on each host
-     * @param hostMemoryMap
-     *         a map of host IDs to the Ternary of used, reserved and total memory on each host
+     * @param vm the virtual machine being migrated
+     * @param destHost the destination host for the virtual machine
+     * @param clusterId the cluster ID
+     * @param vmMetric the VM's resource consumption metric
+     * @param baseMetricsArray pre-calculated array of all host metrics before migration
+     * @param hostIdToIndexMap mapping from host ID to index in the metrics array
+     * @return the cluster imbalance after migration
+     */
+    default Double getImbalancePostMigration(VirtualMachine vm,
+            Host destHost, Long clusterId, long vmMetric, double[] baseMetricsArray,
+            Map<Long, Integer> hostIdToIndexMap, Map<Long, Ternary<Long, Long, Long>> hostCpuMap,
+            Map<Long, Ternary<Long, Long, Long>> hostMemoryMap) {
+        // Create a copy of the base array and adjust only the two affected hosts
+        double[] adjustedMetrics = new double[baseMetricsArray.length];
+        System.arraycopy(baseMetricsArray, 0, adjustedMetrics, 0, baseMetricsArray.length);
+
+        long destHostId = destHost.getId();
+        long vmHostId = vm.getHostId();
+
+        // Adjust source host (remove VM resources)
+        Integer sourceIndex = hostIdToIndexMap.get(vmHostId);
+        if (sourceIndex != null && sourceIndex < adjustedMetrics.length) {
+            Map<Long, Ternary<Long, Long, Long>> sourceMetricsMap = getClusterDrsMetric(clusterId).equals("cpu") ? hostCpuMap : hostMemoryMap;
+            Ternary<Long, Long, Long> sourceMetrics = sourceMetricsMap.get(vmHostId);
+            if (sourceMetrics != null) {
+                adjustedMetrics[sourceIndex] = getMetricValuePostMigration(clusterId, sourceMetrics, vmMetric, vmHostId, destHostId, vmHostId);
+            }
+        }
+
+        // Adjust destination host (add VM resources)
+        Integer destIndex = hostIdToIndexMap.get(destHostId);
+        if (destIndex != null && destIndex < adjustedMetrics.length) {
+            Map<Long, Ternary<Long, Long, Long>> destMetricsMap = getClusterDrsMetric(clusterId).equals("cpu") ? hostCpuMap : hostMemoryMap;
+            Ternary<Long, Long, Long> destMetrics = destMetricsMap.get(destHostId);
+            if (destMetrics != null) {
+                adjustedMetrics[destIndex] = getMetricValuePostMigration(clusterId, destMetrics, vmMetric, destHostId, destHostId, vmHostId);
+            }
+        }
+
+        return calculateImbalance(adjustedMetrics);
+    }
+
+    /**
+     * Calculate imbalance from an array of metric values.
+     * Imbalance is defined as standard deviation divided by mean.
      *
-     * @return a pair containing the CPU and memory imbalance of the cluster after the migration
+     * Uses reusable stateless calculator objects to avoid object creation overhead.
+     * @param values array of metric values
+     * @return calculated imbalance
      */
-    default Double getImbalancePostMigration(ServiceOffering serviceOffering, VirtualMachine vm,
-            Host destHost, Map<Long, Ternary<Long, Long, Long>> hostCpuMap,
-            Map<Long, Ternary<Long, Long, Long>> hostMemoryMap) throws ConfigurationException {
-        Pair<Long, Map<Long, Ternary<Long, Long, Long>>> pair = getHostMetricsMapAndType(destHost.getClusterId(), serviceOffering, hostCpuMap, hostMemoryMap);
-        long vmMetric = pair.first();
-        Map<Long, Ternary<Long, Long, Long>> hostMetricsMap = pair.second();
+    private static double calculateImbalance(double[] values) {
+        if (values == null || values.length == 0) {
+            return 0.0;
+        }
 
-        List<Double> list = new ArrayList<>();
-        for (Long hostId : hostMetricsMap.keySet()) {
-            list.add(getMetricValuePostMigration(destHost.getClusterId(), hostMetricsMap.get(hostId), vmMetric, hostId, destHost.getId(), vm.getHostId()));
+        double mean = MEAN_CALCULATOR.evaluate(values);
+        if (mean == 0.0) {
+            return 0.0; // Avoid division by zero
         }
-        return getImbalance(list);
+        double stdDev = STDDEV_CALCULATOR.evaluate(values, mean);
+        return stdDev / mean;
     }
 
-    private Pair<Long, Map<Long, Ternary<Long, Long, Long>>> getHostMetricsMapAndType(Long clusterId,
-            ServiceOffering serviceOffering, Map<Long, Ternary<Long, Long, Long>> hostCpuMap,
-            Map<Long, Ternary<Long, Long, Long>> hostMemoryMap) throws ConfigurationException {
+    /**
+     * Helper method to get VM metric based on cluster configuration.
+     */
+    static long getVmMetric(ServiceOffering serviceOffering, Long clusterId) throws ConfigurationException {
         String metric = getClusterDrsMetric(clusterId);
-        Pair<Long, Map<Long, Ternary<Long, Long, Long>>> pair;
         switch (metric) {
             case "cpu":
-                pair = new Pair<>((long) serviceOffering.getCpu() * serviceOffering.getSpeed(), hostCpuMap);
-                break;
+                return (long) serviceOffering.getCpu() * serviceOffering.getSpeed();
             case "memory":
-                pair = new Pair<>(serviceOffering.getRamSize() * 1024L * 1024L, hostMemoryMap);
-                break;
+                return serviceOffering.getRamSize() * 1024L * 1024L;
             default:
                 throw new ConfigurationException(
                         String.format("Invalid metric: %s for cluster: %d", metric, clusterId));
         }
-        return pair;
+    }
+
+    /**
+     * Helper method to calculate metrics from pre and post imbalance values.
+     */
+    default Ternary<Double, Double, Double> calculateMetricsFromImbalances(Double preImbalance, Double postImbalance) {
+        // This needs more research to determine the cost and benefit of a migration
+        // TODO: Cost should be a factor of the VM size and the host capacity
+        // TODO: Benefit should be a factor of the VM size and the host capacity and the number of VMs on the host
+        final double improvement = preImbalance - postImbalance;
+        final double cost = 0.0;
+        final double benefit = 1.0;
+        return new Ternary<>(improvement, cost, benefit);
     }
 
     private Double getMetricValuePostMigration(Long clusterId, Ternary<Long, Long, Long> metrics, long vmMetric,
@@ -151,9 +196,26 @@ private Double getMetricValuePostMigration(Long clusterId, Ternary<Long, Long, L
     }
 
     private static Double getImbalance(List<Double> metricList) {
-        Double clusterMeanMetric = getClusterMeanMetric(metricList);
-        Double clusterStandardDeviation = getClusterStandardDeviation(metricList, clusterMeanMetric);
-        return clusterStandardDeviation / clusterMeanMetric;
+        if (CollectionUtils.isEmpty(metricList)) {
+            return 0.0;
+        }
+        // Convert List<Double> to double[] once, avoiding repeated conversions
+        double[] values = new double[metricList.size()];
+        int index = 0;
+        for (Double value : metricList) {
+            if (value != null) {
+                values[index++] = value;
+            }
+        }
+
+        // Trim array if some values were null
+        if (index < values.length) {
+            double[] trimmed = new double[index];
+            System.arraycopy(values, 0, trimmed, 0, index);
+            values = trimmed;
+        }
+
+        return calculateImbalance(values);
     }
 
     static String getClusterDrsMetric(long clusterId) {
@@ -181,36 +243,6 @@ static Double getMetricValue(long clusterId, long used, long free, long total, F
         return null;
     }
 
-    /**
-     * Mean is the average of a collection or set of metrics. In context of a DRS
-     * cluster, the cluster metrics defined as the average metrics value for some
-     * metric (such as CPU, memory etc.) for every resource such as host.
-     * Cluster Mean Metric, mavg = (∑mi) / N, where mi is a measurable metric for a
-     * resource ‘i’ in a cluster with total N number of resources.
-     */
-    static Double getClusterMeanMetric(List<Double> metricList) {
-        return new Mean().evaluate(metricList.stream().mapToDouble(i -> i).toArray());
-    }
-
-    /**
-     * Standard deviation is defined as the square root of the absolute squared sum
-     * of difference of a metric from its mean for every resource divided by the
-     * total number of resources. In context of the DRS, the cluster standard
-     * deviation is the standard deviation based on a metric of resources in a
-     * cluster such as for the allocation or utilisation CPU/memory metric of hosts
-     * in a cluster.
-     * Cluster Standard Deviation, σc = sqrt((∑∣mi−mavg∣^2) / N), where mavg is the
-     * mean metric value and mi is a measurable metric for some resource ‘i’ in the
-     * cluster with total N number of resources.
-     */
-    static Double getClusterStandardDeviation(List<Double> metricList, Double mean) {
-        if (mean != null) {
-            return new StandardDeviation(false).evaluate(metricList.stream().mapToDouble(i -> i).toArray(), mean);
-        } else {
-            return new StandardDeviation(false).evaluate(metricList.stream().mapToDouble(i -> i).toArray());
-        }
-    }
-
     static boolean getDrsMetricUseRatio(long clusterId) {
         return ClusterDrsMetricUseRatio.valueIn(clusterId);
     }

api/src/main/java/org/apache/cloudstack/config/ApiServiceConfiguration.java

@@ -16,10 +16,15 @@
 // under the License.
 package org.apache.cloudstack.config;
 
+import com.cloud.exception.InvalidParameterValueException;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.Configurable;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 public class ApiServiceConfiguration implements Configurable {
+    protected static Logger LOGGER = LogManager.getLogger(ApiServiceConfiguration.class);
     public static final ConfigKey<String> ManagementServerAddresses = new ConfigKey<>(String.class, "host", "Advanced", "localhost", "The ip address of management server. This can also accept comma separated addresses.", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.CSV, null);
     public static final ConfigKey<String> ApiServletPath = new ConfigKey<String>("Advanced", String.class, "endpoint.url", "http://localhost:8080/client/api",
             "API end point. Can be used by CS components/services deployed remotely, for sending CS API requests", true);
@@ -29,6 +34,20 @@ public class ApiServiceConfiguration implements Configurable {
             "true", "Are the source checks on API calls enabled (true) or not (false)? See api.allowed.source.cidr.list", true, ConfigKey.Scope.Global);
     public static final ConfigKey<String> ApiAllowedSourceCidrList = new ConfigKey<>(String.class, "api.allowed.source.cidr.list", "Advanced",
             "0.0.0.0/0,::/0", "Comma separated list of IPv4/IPv6 CIDRs from which API calls can be performed. Can be set on Global and Account levels.", true, ConfigKey.Scope.Account, null, null, null, null, null, ConfigKey.Kind.CSV, null);
+
+
+    public static void validateEndpointUrl() {
+        String csUrl = getApiServletPathValue();
+        if (StringUtils.isBlank(csUrl) || StringUtils.containsAny(csUrl, "localhost", "127.0.0.1", "[::1]")) {
+            LOGGER.error("Global setting [{}] cannot contain localhost or be blank. Current value: {}", ApiServletPath.key(), csUrl);
+            throw new InvalidParameterValueException("Unable to complete this operation. Contact your cloud admin.");
+        }
+    }
+
+    public static String getApiServletPathValue() {
+        return ApiServletPath.value();
+    }
+
     @Override
     public String getConfigComponentName() {
         return ApiServiceConfiguration.class.getSimpleName();