framework/cluster: improve cluster service and integration API service

- mTLS implementation for cluster service communication
- Listen only on the specified cluster node IP address instead of all interfaces
- Validate incoming cluster service requests are from peer management servers based on the server's certificate dns name which can be through global config - ca.framework.cert.management.custom.san
- Hardening of KVM command wrapper script execution
- Improve API server integration port check
- cloudstack-management.default: don't have JMX configuration if not needed. JMX is used for instrumentation; users who need to use it should enable it explicitly

Co-authored-by: Abhishek Kumar <abhishek.mrt22@gmail.com>
Co-authored-by: Wei Zhou <weizhou@apache.org>
Co-authored-by: Rohit Yadav <rohit.yadav@shapeblue.com>

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

Author: shwstppr

api/src/main/java/org/apache/cloudstack/ca/CAManager.java

@@ -77,6 +77,14 @@ public interface CAManager extends CAService, Configurable, PluggableService {
                                                     "15",
                                                     "The number of days before expiry of a client certificate, the validations are checked. Admins are alerted when auto-renewal is not allowed, otherwise auto-renewal is attempted.", true, ConfigKey.Scope.Cluster);
 
+
+    ConfigKey<String> CertManagementCustomSubjectAlternativeName = new ConfigKey<>("Advanced", String.class,
+            "ca.framework.cert.management.custom.san",
+            "cloudstack.internal",
+            "The custom Subject Alternative Name that will be added to the management server certificate. " +
+                    "The actual implementation will depend on the configured CA provider.",
+            false);
+
     /**
      * Returns a list of available CA provider plugins
      * @return returns list of CAProvider

core/src/main/java/com/cloud/resource/CommandWrapper.java

@@ -19,9 +19,12 @@
 
 package com.cloud.resource;
 
+import org.apache.log4j.Logger;
+
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.Command;
-import org.apache.log4j.Logger;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.script.Script;
 
 public abstract class CommandWrapper<T extends Command, A extends Answer, R extends ServerResource> {
     protected Logger logger = Logger.getLogger(getClass());
@@ -32,4 +35,26 @@ public abstract class CommandWrapper<T extends Command, A extends Answer, R exte
      * @return A and the Answer from the command.
      */
     public abstract A execute(T command, R serverResource);
+
+    protected String sanitizeBashCommandArgument(String input) {
+        StringBuilder sanitized = new StringBuilder();
+        for (char c : input.toCharArray()) {
+            if ("\\\"'`$|&;()<>*?![]{}~".indexOf(c) != -1) {
+                sanitized.append('\\');
+            }
+            sanitized.append(c);
+        }
+        return sanitized.toString();
+    }
+
+    public void removeDpdkPort(String portToRemove) {
+        logger.debug("Removing DPDK port: " + portToRemove);
+        int port;
+        try {
+            port = Integer.valueOf(portToRemove);
+        } catch (NumberFormatException nfe) {
+            throw new CloudRuntimeException(String.format("Invalid DPDK port specified: '%s'", portToRemove));
+        }
+        Script.executeCommand("ovs-vsctl", "del-port", String.valueOf(port));
+    }
 }

framework/ca/src/main/java/org/apache/cloudstack/framework/ca/CAProvider.java

@@ -22,6 +22,7 @@
 import java.security.GeneralSecurityException;
 import java.security.KeyStore;
 import java.security.KeyStoreException;
+import java.security.cert.CertificateParsingException;
 import java.security.cert.X509Certificate;
 import java.util.List;
 import java.util.Map;
@@ -45,6 +46,7 @@ public interface CAProvider {
 
     /**
      * Issues certificate with provided options
+     *
      * @param domainNames
      * @param ipAddresses
      * @param validityDays
@@ -104,4 +106,6 @@ public interface CAProvider {
      * @return returns description
      */
     String getDescription();
+
+    boolean isManagementCertificate(java.security.cert.Certificate certificate) throws CertificateParsingException;
 }

framework/ca/src/main/java/org/apache/cloudstack/framework/ca/CAService.java

@@ -21,6 +21,7 @@
 import java.security.GeneralSecurityException;
 import java.security.KeyStore;
 import java.security.KeyStoreException;
+import java.security.cert.CertificateParsingException;
 
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLEngine;
@@ -47,4 +48,6 @@ public interface CAService {
      * @return returns char[] passphrase
      */
     char[] getKeyStorePassphrase();
+
+    boolean isManagementCertificate(java.security.cert.Certificate certificate) throws CertificateParsingException;
 }

framework/cluster/src/main/java/com/cloud/cluster/ClusterManager.java

@@ -16,8 +16,8 @@
 // under the License.
 package com.cloud.cluster;
 
-import org.apache.cloudstack.management.ManagementServerHost;
 import org.apache.cloudstack.framework.config.ConfigKey;
+import org.apache.cloudstack.management.ManagementServerHost;
 
 import com.cloud.utils.component.Manager;
 
@@ -77,6 +77,8 @@ public interface ClusterManager extends Manager {
      */
     String getSelfPeerName();
 
+    String getSelfNodeIP();
+
     long getManagementNodeId();
 
     /**

framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java

@@ -40,17 +40,17 @@
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
-import com.cloud.cluster.dao.ManagementServerStatusDao;
-import org.apache.cloudstack.management.ManagementServerHost;
 import org.apache.cloudstack.framework.config.ConfigDepot;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.Configurable;
 import org.apache.cloudstack.managed.context.ManagedContextRunnable;
+import org.apache.cloudstack.management.ManagementServerHost;
 import org.apache.cloudstack.utils.identity.ManagementServerNode;
 import org.apache.log4j.Logger;
 
 import com.cloud.cluster.dao.ManagementServerHostDao;
 import com.cloud.cluster.dao.ManagementServerHostPeerDao;
+import com.cloud.cluster.dao.ManagementServerStatusDao;
 import com.cloud.utils.DateUtil;
 import com.cloud.utils.Profiler;
 import com.cloud.utils.component.ComponentLifecycle;
@@ -130,7 +130,7 @@ public ClusterManagerImpl() {
         // recursive remote calls between nodes
         //
         _executor = Executors.newCachedThreadPool(new NamedThreadFactory("Cluster-Worker"));
-        setRunLevel(ComponentLifecycle.RUN_LEVEL_FRAMEWORK);
+        setRunLevel(ComponentLifecycle.RUN_LEVEL_COMPONENT);
     }
 
     private void registerRequestPdu(final ClusterServiceRequestPdu pdu) {
@@ -475,6 +475,7 @@ public String getSelfPeerName() {
         return Long.toString(_msId);
     }
 
+    @Override
     public String getSelfNodeIP() {
         return _clusterNodeIP;
     }

framework/cluster/src/main/java/com/cloud/cluster/ClusterServiceAdapter.java

@@ -28,7 +28,5 @@ public interface ClusterServiceAdapter extends Adapter {
 
     public ClusterService getPeerService(String strPeer) throws RemoteException;
 
-    public String getServiceEndpointName(String strPeer);
-
     public int getServicePort();
 }

framework/cluster/src/main/java/com/cloud/cluster/ClusterServiceServletAdapter.java

@@ -23,8 +23,9 @@
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
-import org.apache.log4j.Logger;
+import org.apache.cloudstack.ca.CAManager;
 import org.apache.cloudstack.framework.config.ConfigDepot;
+import org.apache.log4j.Logger;
 
 import com.cloud.cluster.dao.ManagementServerHostDao;
 import com.cloud.utils.NumbersUtil;
@@ -44,14 +45,16 @@ public class ClusterServiceServletAdapter extends AdapterBase implements Cluster
     @Inject
     private ManagementServerHostDao _mshostDao;
     @Inject
+    private CAManager caService;
+    @Inject
     protected ConfigDepot _configDepot;
 
     private ClusterServiceServletContainer _servletContainer;
 
     private int _clusterServicePort = DEFAULT_SERVICE_PORT;
 
     public ClusterServiceServletAdapter() {
-        setRunLevel(ComponentLifecycle.RUN_LEVEL_FRAMEWORK);
+        setRunLevel(ComponentLifecycle.RUN_LEVEL_COMPONENT);
     }
 
     @Override
@@ -66,12 +69,10 @@ public ClusterService getPeerService(String strPeer) throws RemoteException {
         String serviceUrl = getServiceEndpointName(strPeer);
         if (serviceUrl == null)
             return null;
-
-        return new ClusterServiceServletImpl(serviceUrl);
+        return new ClusterServiceServletImpl(serviceUrl, caService);
     }
 
-    @Override
-    public String getServiceEndpointName(String strPeer) {
+    protected String getServiceEndpointName(String strPeer) {
         try {
             init();
         } catch (ConfigurationException e) {
@@ -95,7 +96,7 @@ public int getServicePort() {
 
     private String composeEndpointName(String nodeIP, int port) {
         StringBuffer sb = new StringBuffer();
-        sb.append("http://").append(nodeIP).append(":").append(port).append("/clusterservice");
+        sb.append("https://").append(nodeIP).append(":").append(port).append("/clusterservice");
         return sb.toString();
     }
 
@@ -108,7 +109,8 @@ public boolean configure(String name, Map<String, Object> params) throws Configu
     @Override
     public boolean start() {
         _servletContainer = new ClusterServiceServletContainer();
-        _servletContainer.start(new ClusterServiceServletHttpHandler(_manager), _clusterServicePort);
+        _servletContainer.start(new ClusterServiceServletHttpHandler(_manager), _manager.getSelfNodeIP(),
+                _clusterServicePort, caService);
         return true;
     }