frr-k8s: use OperatorPKI for metrics TLS to fix bootstrap deadlock

The FRR DaemonSet requires a TLS secret for kube-rbac-proxy to start.
Previously this used service-ca, but service-ca is not available during
bootstrap (it depends on CNI being ready first), causing a deadlock.

This commit adds a second OperatorPKI for metrics certificates, so both
webhook and metrics use OperatorPKI with no service-ca dependency during
bootstrap.

Changes:
- Add frr-k8s-metrics OperatorPKI to 003-pki.yaml
- Update frr-k8s.yaml to use frr-k8s-metrics-cert secret
- Remove service-ca annotation from monitor.yaml Service
- Use insecureSkipVerify in ServiceMonitor (Prometheus doesn't have
  OperatorPKI CA in its trust bundle, but TLS encryption is still active)

Signed-off-by: Riccardo Ravaioli <rravaiol@redhat.com>

Author: ricky-rav

bindata/network/frr-k8s/003-pki.yaml

@@ -1,5 +1,5 @@
 # Request that the cluster network operator PKI controller
-# creates certificates for the FRR-k8s webhook.
+# creates certificates for the FRR-k8s webhook and metrics.
 # This avoids dependency on service-ca operator during bootstrap,
 # which is critical because the webhook must be ready before OVN-Kubernetes
 # starts when RouteAdvertisements are enabled.
@@ -15,3 +15,12 @@ metadata:
 spec:
   targetCert:
     commonName: frr-k8s-webhook-service.openshift-frr-k8s.svc
+---
+apiVersion: network.operator.openshift.io/v1
+kind: OperatorPKI
+metadata:
+  name: frr-k8s-metrics
+  namespace: openshift-frr-k8s
+spec:
+  targetCert:
+    commonName: frr-k8s-monitor-service.openshift-frr-k8s.svc

bindata/network/frr-k8s/frr-k8s.yaml

@@ -51,7 +51,7 @@ spec:
         emptyDir: {}
       - name: metrics-certs
         secret:
-          secretName: frr-k8s-metrics-certs
+          secretName: frr-k8s-metrics-cert
       initContainers:
       # Copies the initial config files with the right permissions to the shared volume.
       - name: cp-frr-files

bindata/network/frr-k8s/monitor.yaml

@@ -8,7 +8,6 @@ metadata:
     name: frr-k8s-monitor-service
   annotations:
     prometheus.io/scrape: "true"
-    service.beta.openshift.io/serving-cert-secret-name: frr-k8s-metrics-certs
 spec:
   selector:
     app: frr-k8s
@@ -41,15 +40,13 @@ spec:
     port: metricshttps
     scheme: https
     tlsConfig:
-      caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
-      serverName: frr-k8s-monitor-service.openshift-frr-k8s.svc
+      insecureSkipVerify: true
   - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
     honorLabels: true
     port: frrmetricshttps
     scheme: https
     tlsConfig:
-      caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
-      serverName: frr-k8s-monitor-service.openshift-frr-k8s.svc
+      insecureSkipVerify: true
   jobLabel: app
   namespaceSelector:
     matchNames:

pkg/network/render_test.go

@@ -638,7 +638,7 @@ func Test_renderAdditionalRoutingCapabilities(t *testing.T) {
 					},
 				},
 			},
-			want:        21, // 19 original + 1 OperatorPKI + 1 document separator
+			want:        22, // 19 original + 2 OperatorPKI (webhook + metrics) + 1 document separator
 			expectedErr: nil,
 		},
 	}