From 6fb58a222953e2a8b80950c7e4f61b864d84449e Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 20 Jan 2026 09:38:31 +0100 Subject: [PATCH] [monitoring] Expose prometheus as TCP services --- .../dss/templates/_networking-google.tpl | 3 + .../templates/prometheus-loadbalancers.yaml | 103 +++--------------- deploy/services/tanka/metadata_base.libsonnet | 1 - deploy/services/tanka/prometheus.libsonnet | 26 ++++- 4 files changed, 36 insertions(+), 97 deletions(-) diff --git a/deploy/services/helm-charts/dss/templates/_networking-google.tpl b/deploy/services/helm-charts/dss/templates/_networking-google.tpl index a9cdc412f..09764ba9f 100644 --- a/deploy/services/helm-charts/dss/templates/_networking-google.tpl +++ b/deploy/services/helm-charts/dss/templates/_networking-google.tpl @@ -1,3 +1,6 @@ +{{- define "google-lb-default-annotations" -}} +{{- end -}} + {{- define "google-lb-crdb-annotations" -}} {{- end -}} diff --git a/deploy/services/helm-charts/dss/templates/prometheus-loadbalancers.yaml b/deploy/services/helm-charts/dss/templates/prometheus-loadbalancers.yaml index c06cb7d46..90dd46033 100644 --- a/deploy/services/helm-charts/dss/templates/prometheus-loadbalancers.yaml +++ b/deploy/services/helm-charts/dss/templates/prometheus-loadbalancers.yaml @@ -3,113 +3,36 @@ {{- if $.Values.monitoring.enabled }} {{- if $.Values.monitoring.externalService.enabled }} -{{- if eq $cloudProvider "google" }} - ---- -apiVersion: cloud.google.com/v1 -kind: BackendConfig -metadata: - name: prometheus-external -spec: - securityPolicy: - name: "{{ $.Values.monitoring.externalService.allowedIPsPolicy }}" - --- apiVersion: v1 kind: Service -metadata: - labels: - app: {{$.Release.Name}}-prometheus - name: {{$.Release.Name}}-prometheus-external - annotations: - cloud.google.com/backend-config: '{"default": "prometheus-external"}' - name: {{$.Release.Name}}-prometheus-external -spec: - ports: - - name: prometheus - port: 9090 - targetPort: 9090 - publishNotReadyAddresses: true - selector: - app.kubernetes.io/instance: "{{$.Release.Name}}" - app.kubernetes.io/name: "prometheus" - type: ClusterIP - ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress metadata: annotations: - {{- include (printf "%s-ingress-prometheus-annotations" $cloudProvider) + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + {{- include (printf "%s-lb-default-annotations" $cloudProvider) (dict - "certName" (printf "%s-prometheus-https-certificate" $.Release.Name) + "name" "prometheus-external" "ip" $.Values.monitoring.externalService.ip - "frontendConfig" (empty .sslPolicy | ternary "" "ssl-frontend-config") + "subnet" $.Values.monitoring.externalService.subnet + "cloudProvider" $cloudProvider ) | nindent 4 }} labels: - name: {{$.Release.Name}}-prometheus-https-ingress - name: {{$.Release.Name}}-prometheus-https-ingress -spec: - {{- include (printf "%s-ingress-spec" $cloudProvider) . | nindent 2 }} - rules: - - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: {{$.Release.Name}}-prometheus-external - port: - number: 9090 - ---- -apiVersion: networking.gke.io/v1 -kind: ManagedCertificate -metadata: - labels: - name: {{$.Release.Name}}-prometheus-https-certificate - name: {{$.Release.Name}}-prometheus-https-certificate -spec: - domains: - - {{ $.Values.monitoring.externalService.hostname }} - -{{- else }} - ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - {{- include (printf "%s-ingress-prometheus-annotations" $cloudProvider) - (merge $.Values.monitoring.externalService - (dict - "name" "prometheus-external" - "cloudProvider" $cloudProvider - ) - ) | nindent 4 - }} - labels: - app: {{$.Release.Name}}-prometheus - name: {{$.Release.Name}}-prometheus-external - name: {{$.Release.Name}}-prometheus-external + app: prometheus + name: prometheus-external + name: prometheus-external + namespace: default spec: {{- include (printf "%s-lb-spec" $cloudProvider) (dict "ip" $.Values.monitoring.externalService.ip) | nindent 2}} - loadBalancerSourceRanges: -{{- range $i, $ip := $.Values.monitoring.externalService.allowedIPs }} - - {{$ip}} -{{- end }} ports: - - name: prometheus - port: 443 + - name: prometheus-external + port: 9090 targetPort: 9090 publishNotReadyAddresses: true selector: - app.kubernetes.io/instance: "{{$.Release.Name}}" - app.kubernetes.io/name: "prometheus" - type: LoadBalancer + app.kubernetes.io/name: prometheus -{{- end }} + type: LoadBalancer {{- end }} {{- end }} diff --git a/deploy/services/tanka/metadata_base.libsonnet b/deploy/services/tanka/metadata_base.libsonnet index 646eda16c..1cf23151b 100644 --- a/deploy/services/tanka/metadata_base.libsonnet +++ b/deploy/services/tanka/metadata_base.libsonnet @@ -75,7 +75,6 @@ image: 'prom/prometheus:v3.8.1', expose_external: false, IP: '', // This is the static external ip address for promethus ingress, leaving blank means your cloud provider will assign an ephemeral IP - whitelist_ip_ranges: error 'must specify whitelisted CIDR IP Blocks, or empty list for fully public access', retention: '15d', storage_size: '100Gi', storageClass: 'standard', diff --git a/deploy/services/tanka/prometheus.libsonnet b/deploy/services/tanka/prometheus.libsonnet index 5cce0831b..5e1dd79d4 100644 --- a/deploy/services/tanka/prometheus.libsonnet +++ b/deploy/services/tanka/prometheus.libsonnet @@ -29,16 +29,30 @@ local PrometheusWebConfig(metadata) = { } }; -local PrometheusExternalService(metadata) = base.Service(metadata, 'prometheus-external') { - app:: 'prometheus', +local googleExternalLB(metadata, name, ip) = base.Service(metadata, name) { port:: 9090, + app:: 'prometheus', spec+: { type: 'LoadBalancer', - loadBalancerIP: metadata.prometheus.IP, - loadBalancerSourceRanges: metadata.prometheus.whitelist_ip_ranges - } + loadBalancerIP: ip, + }, +}; + +local awsExternalLB(metadata, name, ip) = base.AWSLoadBalancer(metadata, name, [ip], metadata.subnet) { + port:: 9090, + app:: 'prometheus', +}; + +local minikubeExternalLB(metadata, name, ip) = base.Service(metadata, name) { + port:: 9090, + app:: 'prometheus', }; +local externalLB(metadata, name, ip) = + if metadata.cloud_provider == "google" then googleExternalLB(metadata, name, ip) + else if metadata.cloud_provider == "aws" then awsExternalLB(metadata, name, ip) + else if metadata.cloud_provider == "minikube" then minikubeExternalLB(metadata, name, ip); + { all(metadata) : { clusterRole: base.ClusterRole(metadata, 'prometheus') { @@ -231,7 +245,7 @@ local PrometheusExternalService(metadata) = base.Service(metadata, 'prometheus-e ], }, }, - externalService: if metadata.prometheus.expose_external == true then PrometheusExternalService(metadata), + externalLB: if metadata.prometheus.expose_external == true then externalLB(metadata, "prometheus", metadata.prometheus.IP), internalService: base.Service(metadata, 'prometheus-service') { app:: 'prometheus', port:: 9090,