[WIP] Centralized logging capability. Follow up on the #4108

server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManager.java

@@ -44,6 +44,12 @@ public interface ConsoleProxyManager extends Manager, ConsoleProxyService {
     public static final ConfigKey<Boolean> NoVncConsoleSourceIpCheckEnabled = new ConfigKey<Boolean>("Advanced", Boolean.class, "novnc.console.sourceip.check.enabled", "false",
         "If true, The source IP to access novnc console must be same as the IP in request to management server for console URL. Needs to reconnect CPVM to management server when this changes (via restart CPVM, or management server, or cloud service in CPVM)", false);
 
+    static final ConfigKey<Boolean> ConsoleProxyRemoteLoggingEnabled = new ConfigKey<>("Advanced", Boolean.class, "consoleproxy.remote.logging.enabled", "false",
+            "If true, console proxy will forward console proxy logs to <consoleproxy.remote.logging.address>", true, ConfigKey.Scope.Global);
+
+    static final ConfigKey<String> ConsoleProxyRemoteLoggingAddress = new ConfigKey<>("Advanced", String.class, "consoleproxy.remote.logging.address", "",
+            "Console proxy forwards its logs to this address", true, ConfigKey.Scope.Global);
+
     public void setManagementState(ConsoleProxyManagementState state);
 
     public ConsoleProxyManagementState getManagementState();

server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java

@@ -1219,6 +1219,9 @@ public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, Depl
         }
         buf.append(" zone=").append(dest.getDataCenter().getId());
         buf.append(" pod=").append(dest.getPod().getId());
+        buf.append(" remoteloggingenabled=").append(ConsoleProxyRemoteLoggingEnabled.value());
+        buf.append(" remoteloggingaddress=").append(ConsoleProxyRemoteLoggingAddress.value());
+
         buf.append(" guid=Proxy.").append(profile.getId());
         buf.append(" proxy_vm=").append(profile.getId());
         if (disableRpFilter) {
@@ -1580,7 +1583,8 @@ public String getConfigComponentName() {
 
     @Override
     public ConfigKey<?>[] getConfigKeys() {
-        return new ConfigKey<?>[] { NoVncConsoleDefault, NoVncConsoleSourceIpCheckEnabled };
+        return new ConfigKey<?>[] { NoVncConsoleDefault, NoVncConsoleSourceIpCheckEnabled,
+                ConsoleProxyRemoteLoggingEnabled, ConsoleProxyRemoteLoggingAddress };
     }
 
     protected ConsoleProxyStatus parseJsonToConsoleProxyStatus(String json) throws JsonParseException {

server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManager.java

@@ -114,6 +114,12 @@ public interface VirtualNetworkApplianceManager extends Manager, VirtualNetworkA
     static final ConfigKey<Double> RouterHealthChecksMaxMemoryUsageThreshold = new ConfigKey<Double>(Double.class, "router.health.checks.max.memory.usage.threshold",
             "Advanced", "100", "Max Memory Usage threshold as % above which check is considered a failure.",
             true, ConfigKey.Scope.Zone, null);
+    static final ConfigKey<Boolean> RouterRemoteLoggingEnabled = new ConfigKey<>(Boolean.class, "router.remote.logging.enabled",
+            "Advanced", "false", "If true, Router VM will forward console proxy logs to <router.remote.logging.address>",
+            true, ConfigKey.Scope.Zone, null);
+    static final ConfigKey<String> RouterRemoteLoggingAddress = new ConfigKey<>(String.class, "router.remote.logging.address",
+            "Advanced",  "", "Router VMs forward their logs to this address",
+            true, ConfigKey.Scope.Zone, null);
 
     public static final int DEFAULT_ROUTER_VM_RAMSIZE = 256;            // 256M
     public static final int DEFAULT_ROUTER_CPU_MHZ = 500;                // 500 MHz

server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java

@@ -2050,6 +2050,9 @@ public boolean finalizeVirtualMachineProfile(final VirtualMachineProfile profile
 
         buf.append(" type=" + type + rpFilter);
 
+        buf.append(" remoteloggingenabled=").append(RouterRemoteLoggingEnabled.value());
+        buf.append(" remoteloggingaddress=").append(RouterRemoteLoggingAddress.value());
+
         final String domain_suffix = dc.getDetail(ZoneConfig.DnsSearchOrder.getName());
         if (domain_suffix != null) {
             buf.append(" dnssearchorder=").append(domain_suffix);
@@ -3239,7 +3242,9 @@ public ConfigKey<?>[] getConfigKeys() {
                 RouterHealthChecksFreeDiskSpaceThreshold,
                 RouterHealthChecksMaxCpuUsageThreshold,
                 RouterHealthChecksMaxMemoryUsageThreshold,
-                ExposeDnsAndBootpServer
+                ExposeDnsAndBootpServer,
+                RouterRemoteLoggingEnabled,
+                RouterRemoteLoggingAddress
         };
     }
 

services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java

@@ -266,6 +266,12 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar
     static final ConfigKey<Integer> MaxNumberOfSsvmsForMigration = new ConfigKey<Integer>("Advanced", Integer.class, "max.ssvm.count", "5",
             "Number of additional SSVMs to handle migration of data objects concurrently", true, ConfigKey.Scope.Global);
 
+    static final ConfigKey<Boolean> SecondaryStorageRemoteLoggingEnabled = new ConfigKey<>( "Advanced", Boolean.class,"secondary.storage.remote.logging.enabled", "false",
+            "If true, secondary storage will forward secondary storage logs to <secondary.storage.remote.logging.address>", false, ConfigKey.Scope.Global);
+
+    static final ConfigKey<String> SecondaryStorageRemoteLoggingAddress = new ConfigKey<>( "secondary.storage.remote.logging.address", String.class, "Advanced", "",
+            "Secondary storage forwards its logs to this address", true, ConfigKey.Scope.Global);
+
     public SecondaryStorageManagerImpl() {
     }
 
@@ -1080,6 +1086,8 @@ public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, Depl
 
         buf.append(" zone=").append(dest.getDataCenter().getId());
         buf.append(" pod=").append(dest.getPod().getId());
+        buf.append(" remoteloggingenabled=").append(SecondaryStorageRemoteLoggingEnabled.value());
+        buf.append(" remoteloggingaddress=").append(SecondaryStorageRemoteLoggingAddress.value());
 
         buf.append(" guid=").append(profile.getVirtualMachine().getHostName());
 
@@ -1420,7 +1428,7 @@ public String getConfigComponentName() {
 
     @Override
     public ConfigKey<?>[] getConfigKeys() {
-        return new ConfigKey<?>[] {NTPServerConfig, MaxNumberOfSsvmsForMigration};
+        return new ConfigKey<?>[] {NTPServerConfig, MaxNumberOfSsvmsForMigration, SecondaryStorageRemoteLoggingEnabled, SecondaryStorageRemoteLoggingAddress};
     }
 
 }

systemvm/debian/etc/rsyslog.d/49-cloud.conf

@@ -0,0 +1,8 @@
+$InputFileName /var/log/cloud.log
+$InputFileTag cloud
+$InputFileStateFile stat-file1-cloud
+$InputFileSeverity info
+$InputFileFacility local7
+$InputFilePollInterval 1
+$InputFilePersistStateInterval 1
+$InputRunFileMonitor

systemvm/debian/etc/rsyslog.d/49-haproxy.conf

@@ -0,0 +1,12 @@
+# Create an additional socket in haproxy's chroot in order to allow logging via
+# /dev/log to chroot'ed HAProxy processes
+$AddUnixListenSocket /var/lib/haproxy/dev/log
+
+$InputFileName /var/log/haproxy.log
+$InputFileTag haproxy
+$InputFileStateFile stat-file1-haproxy
+$InputFileSeverity info
+$InputFileFacility local7
+$InputFilePollInterval 1
+$InputFilePersistStateInterval 1
+$InputRunFileMonitor

systemvm/debian/opt/cloud/bin/setup/common.sh

@@ -759,6 +759,12 @@ parse_cmd_line() {
         ntpserverlist)
             export NTP_SERVER_LIST=$VALUE
             ;;
+        remoteloggingenabled)
+            export REMOTE_LOGGGING_ENABLED=$VALUE
+            ;;
+        remoteloggingaddress)
+            export REMOTE_LOGGGING_ADDRESS=$VALUE
+            ;;
       esac
   done
   echo -e "\n\t}\n}" >> ${CHEF_TMP_FILE}

systemvm/debian/opt/cloud/bin/setup/postinit.sh

@@ -20,10 +20,28 @@
 
 # Eject cdrom if any
 eject || true
+. /opt/cloud/bin/setup/common.sh
+
+setup_remote_logging(){
+    echo "${REMOTE_LOGGGING_ENABLED,,}"
+    # Remove curent configuration
+    sed -i '/imudp/d' /etc/rsyslog.conf
+    sed -i '/3914/d' /etc/rsyslog.conf
+    if [[ "${REMOTE_LOGGGING_ENABLED,,}" == "true" ]]; then
+        sed -i '/^# provides UDP syslog*/a $ModLoad imudp\n$UDPServerRun 3914' /etc/rsyslog.conf
+    fi
+
+    echo "*.* @${REMOTE_LOGGGING_ADDRESS}" > /etc/rsyslog.d/99-fwd.conf
+
+    systemctl restart rsyslog
+}
 
 # Restart journald for setting changes to apply
 systemctl restart systemd-journald
 
+# Setup remote logging
+setup_remote_logging
+
 TYPE=$(grep -Po 'type=\K[a-zA-Z]*' /var/cache/cloud/cmdline)
 if [ "$TYPE" == "router" ] || [ "$TYPE" == "vpcrouter" ] || [ "$TYPE" == "dhcpsrvr" ]
 then