[monitoring] Expose prometheus via TCP services

deploy/services/helm-charts/dss/templates/_networking-google.tpl

@@ -1,3 +1,6 @@
+{{- define "google-lb-default-annotations" -}}
+{{- end -}}
+
 {{- define "google-lb-crdb-annotations" -}}
 {{- end -}}
 

deploy/services/helm-charts/dss/templates/prometheus-loadbalancers.yaml

@@ -3,113 +3,36 @@
 {{- if $.Values.monitoring.enabled }}
 {{- if $.Values.monitoring.externalService.enabled }}
 
-{{- if eq $cloudProvider "google" }}
-
----
-apiVersion: cloud.google.com/v1
-kind: BackendConfig
-metadata:
-  name: prometheus-external
-spec:
-  securityPolicy:
-    name: "{{ $.Values.monitoring.externalService.allowedIPsPolicy }}"
-
 ---
 apiVersion: v1
 kind: Service
-metadata:
-  labels:
-    app: {{$.Release.Name}}-prometheus
-    name: {{$.Release.Name}}-prometheus-external
-  annotations:
-    cloud.google.com/backend-config: '{"default": "prometheus-external"}'
-  name: {{$.Release.Name}}-prometheus-external
-spec:
-  ports:
-    - name: prometheus
-      port: 9090
-      targetPort: 9090
-  publishNotReadyAddresses: true
-  selector:
-     app.kubernetes.io/instance: "{{$.Release.Name}}"
-     app.kubernetes.io/name: "prometheus"
-  type: ClusterIP
-
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
 metadata:
   annotations:
-    {{- include (printf "%s-ingress-prometheus-annotations" $cloudProvider)
+    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
+    {{- include (printf "%s-lb-default-annotations" $cloudProvider)
       (dict
-        "certName" (printf "%s-prometheus-https-certificate" $.Release.Name)
+        "name" "prometheus-external"
         "ip" $.Values.monitoring.externalService.ip
-        "frontendConfig" (empty .sslPolicy | ternary "" "ssl-frontend-config")
+        "subnet" $.Values.monitoring.externalService.subnet
+        "cloudProvider" $cloudProvider
       ) | nindent 4
     }}
   labels:
-    name: {{$.Release.Name}}-prometheus-https-ingress
-  name: {{$.Release.Name}}-prometheus-https-ingress
-spec:
-  {{- include (printf "%s-ingress-spec" $cloudProvider) . | nindent 2 }}
-  rules:
-    - http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: {{$.Release.Name}}-prometheus-external
-                port:
-                  number: 9090
-
----
-apiVersion: networking.gke.io/v1
-kind: ManagedCertificate
-metadata:
-  labels:
-    name: {{$.Release.Name}}-prometheus-https-certificate
-  name: {{$.Release.Name}}-prometheus-https-certificate
-spec:
-  domains:
-    - {{ $.Values.monitoring.externalService.hostname }}
-
-{{- else }}
-
----
-apiVersion: v1
-kind: Service
-metadata:
-  annotations:
-    {{- include (printf "%s-ingress-prometheus-annotations" $cloudProvider)
-      (merge $.Values.monitoring.externalService
-        (dict
-          "name" "prometheus-external"
-          "cloudProvider" $cloudProvider
-        )
-      ) | nindent 4
-    }}
-  labels:
-    app: {{$.Release.Name}}-prometheus
-    name: {{$.Release.Name}}-prometheus-external
-  name: {{$.Release.Name}}-prometheus-external
+    app: prometheus
+    name: prometheus-external
+  name: prometheus-external
+  namespace: default
 spec:
   {{- include (printf "%s-lb-spec" $cloudProvider) (dict "ip" $.Values.monitoring.externalService.ip) | nindent 2}}
-  loadBalancerSourceRanges:
-{{- range $i, $ip := $.Values.monitoring.externalService.allowedIPs }}
-    - {{$ip}}
-{{- end }}
   ports:
-    - name: prometheus
-      port: 443
+    - name: prometheus-external
+      port: 9090
       targetPort: 9090
   publishNotReadyAddresses: true
   selector:
-     app.kubernetes.io/instance: "{{$.Release.Name}}"
-     app.kubernetes.io/name: "prometheus"
-  type: LoadBalancer
+    app.kubernetes.io/name: prometheus
 
-{{- end }}
+  type: LoadBalancer
 
 {{- end }}
 {{- end }}

deploy/services/tanka/metadata_base.libsonnet

@@ -75,7 +75,6 @@
     image: 'prom/prometheus:v3.8.1',
     expose_external: false,
     IP: '',  // This is the static external ip address for promethus ingress, leaving blank means your cloud provider will assign an ephemeral IP
-    whitelist_ip_ranges: error 'must specify whitelisted CIDR IP Blocks, or empty list for fully public access',
     retention: '15d',
     storage_size: '100Gi',
     storageClass: 'standard',

deploy/services/tanka/prometheus.libsonnet

@@ -29,16 +29,30 @@ local PrometheusWebConfig(metadata) = {
    }
 };
 
-local PrometheusExternalService(metadata) = base.Service(metadata, 'prometheus-external') {
-  app:: 'prometheus',
+local googleExternalLB(metadata, name, ip) = base.Service(metadata, name) {
   port:: 9090,
+  app:: 'prometheus',
   spec+: {
     type: 'LoadBalancer',
-    loadBalancerIP: metadata.prometheus.IP,
-    loadBalancerSourceRanges: metadata.prometheus.whitelist_ip_ranges
-  }
+    loadBalancerIP: ip,
+  },
+};
+
+local awsExternalLB(metadata, name, ip) = base.AWSLoadBalancer(metadata, name, [ip], metadata.subnet) {
+  port:: 9090,
+  app:: 'prometheus',
+};
+
+local minikubeExternalLB(metadata, name, ip) = base.Service(metadata, name) {
+  port:: 9090,
+  app:: 'prometheus',
 };
 
+local externalLB(metadata, name, ip) =
+    if metadata.cloud_provider == "google" then googleExternalLB(metadata, name, ip)
+    else if metadata.cloud_provider == "aws" then awsExternalLB(metadata, name, ip)
+    else if metadata.cloud_provider == "minikube" then minikubeExternalLB(metadata, name, ip);
+
 {
   all(metadata) : {
     clusterRole: base.ClusterRole(metadata, 'prometheus') {
@@ -231,7 +245,7 @@ local PrometheusExternalService(metadata) = base.Service(metadata, 'prometheus-e
         ],
       },
     },
-    externalService: if metadata.prometheus.expose_external == true then PrometheusExternalService(metadata),
+    externalLB: if metadata.prometheus.expose_external == true then externalLB(metadata, "prometheus", metadata.prometheus.IP),
     internalService: base.Service(metadata, 'prometheus-service') {
       app:: 'prometheus',
       port:: 9090,