CLOUDP-111968: Watch for changes in CA certificate (#910)

* Watch for changes in CA certificate

* Watch for CA secret changes

* Add missing autogenerated Prometheus methods

* Force re-adding service binding annotations

* Ensure CA cert secret is extended by modifying dnsnames

* Remove usage of caResourceType

Author: mircea-cosbuc

api/v1/mongodbcommunity_types.go

@@ -726,6 +726,10 @@ func (m MongoDBCommunity) PrometheusTLSSecretNamespacedName() types.NamespacedNa
 	return types.NamespacedName{Name: m.Spec.Prometheus.TLSSecretRef.Name, Namespace: m.Namespace}
 }
 
+func (m MongoDBCommunity) TLSOperatorCASecretNamespacedName() types.NamespacedName {
+	return types.NamespacedName{Name: m.Name + "-ca-certificate", Namespace: m.Namespace}
+}
+
 // TLSOperatorSecretNamespacedName will get the namespaced name of the Secret created by the operator
 // containing the combined certificate and key.
 func (m MongoDBCommunity) TLSOperatorSecretNamespacedName() types.NamespacedName {

api/v1/zz_generated.deepcopy.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/mongodb/mongodb-kubernetes-operator/controllers/construct"
 	"github.com/mongodb/mongodb-kubernetes-operator/pkg/automationconfig"
+	"github.com/pkg/errors"
 
 	"github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret"
 
@@ -15,7 +16,6 @@ import (
 	"github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/podtemplatespec"
 	"github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/statefulset"
 
-	v1 "k8s.io/api/core/v1"
 	apiErrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
 
@@ -40,35 +40,17 @@ func (r *ReplicaSetReconciler) validateTLSConfig(mdb mdbv1.MongoDBCommunity) (bo
 	r.log.Info("Ensuring TLS is correctly configured")
 
 	// Ensure CA cert is configured
-	var caResourceName types.NamespacedName
-	var caResourceType string
-	var caData map[string]string
-	var err error
-	if mdb.Spec.Security.TLS.CaCertificateSecret != nil {
-		caResourceName = mdb.TLSCaCertificateSecretNamespacedName()
-		caResourceType = "Secret"
-		caData, err = secret.ReadStringData(r.client, caResourceName)
-	} else {
-		caResourceName = mdb.TLSConfigMapNamespacedName()
-		caResourceType = "ConfigMap"
-		caData, err = configmap.ReadData(r.client, caResourceName)
-	}
+	_, err := getCaCrt(r.client, r.client, mdb)
 
 	if err != nil {
 		if apiErrors.IsNotFound(err) {
-			r.log.Warnf(`CA %s "%s" not found`, caResourceType, caResourceName)
+			r.log.Warnf("CA resource not found: %s", err)
 			return false, nil
 		}
 
 		return false, err
 	}
 
-	// Ensure Secret or ConfigMap has a "ca.crt" field
-	if cert, ok := caData[tlsCACertName]; !ok || cert == "" {
-		r.log.Warnf(`%s "%s" should have a CA certificate in field "%s"`, caResourceType, caResourceName, tlsCACertName)
-		return false, nil
-	}
-
 	// Ensure Secret exists
 	_, err = secret.ReadStringData(r.client, mdb.TLSSecretNamespacedName())
 	if err != nil {
@@ -91,23 +73,35 @@ func (r *ReplicaSetReconciler) validateTLSConfig(mdb mdbv1.MongoDBCommunity) (bo
 	// Watch certificate-key secret to handle rotations
 	r.secretWatcher.Watch(mdb.TLSSecretNamespacedName(), mdb.NamespacedName())
 
+	// Watch CA certificate changes
+	if mdb.Spec.Security.TLS.CaCertificateSecret != nil {
+		r.secretWatcher.Watch(mdb.TLSCaCertificateSecretNamespacedName(), mdb.NamespacedName())
+	} else {
+		r.configMapWatcher.Watch(mdb.TLSConfigMapNamespacedName(), mdb.NamespacedName())
+	}
+
 	r.log.Infof("Successfully validated TLS config")
 	return true, nil
 }
 
 // getTLSConfigModification creates a modification function which enables TLS in the automation config.
 // It will also ensure that the combined cert-key secret is created.
-func getTLSConfigModification(getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDBCommunity) (automationconfig.Modification, error) {
+func getTLSConfigModification(cmGetter configmap.Getter, secretGetter secret.Getter, mdb mdbv1.MongoDBCommunity) (automationconfig.Modification, error) {
 	if !mdb.Spec.Security.TLS.Enabled {
 		return automationconfig.NOOP(), nil
 	}
 
-	certKey, err := getPemOrConcatenatedCrtAndKey(getUpdateCreator, mdb, mdb.TLSSecretNamespacedName())
+	caCert, err := getCaCrt(cmGetter, secretGetter, mdb)
 	if err != nil {
 		return automationconfig.NOOP(), err
 	}
 
-	return tlsConfigModification(mdb, certKey), nil
+	certKey, err := getPemOrConcatenatedCrtAndKey(secretGetter, mdb, mdb.TLSSecretNamespacedName())
+	if err != nil {
+		return automationconfig.NOOP(), err
+	}
+
+	return tlsConfigModification(mdb, certKey, caCert), nil
 }
 
 // getCertAndKey will fetch the certificate and key from the user-provided Secret.
@@ -162,6 +156,48 @@ func getPemOrConcatenatedCrtAndKey(getter secret.Getter, mdb mdbv1.MongoDBCommun
 	return certKey, nil
 }
 
+func getCaCrt(cmGetter configmap.Getter, secretGetter secret.Getter, mdb mdbv1.MongoDBCommunity) (string, error) {
+	var caResourceName types.NamespacedName
+	var caData map[string]string
+	var err error
+	if mdb.Spec.Security.TLS.CaCertificateSecret != nil {
+		caResourceName = mdb.TLSCaCertificateSecretNamespacedName()
+		caData, err = secret.ReadStringData(secretGetter, caResourceName)
+	} else {
+		caResourceName = mdb.TLSConfigMapNamespacedName()
+		caData, err = configmap.ReadData(cmGetter, caResourceName)
+	}
+	if err != nil {
+		return "", err
+	}
+
+	if cert, ok := caData[tlsCACertName]; !ok || cert == "" {
+		return "", errors.Errorf(`CA certificate resource "%s" should have a CA certificate in field "%s"`, caResourceName, tlsCACertName)
+	} else {
+		return cert, nil
+	}
+}
+
+// ensureCASecret will create or update the operator managed Secret containing
+// the CA certficate from the user provided Secret or ConfigMap.
+func ensureCASecret(cmGetter configmap.Getter, secretGetter secret.Getter, getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDBCommunity) error {
+	cert, err := getCaCrt(cmGetter, secretGetter, mdb)
+	if err != nil {
+		return err
+	}
+
+	caFileName := tlsOperatorSecretFileName(cert)
+
+	operatorSecret := secret.Builder().
+		SetName(mdb.TLSOperatorCASecretNamespacedName().Name).
+		SetNamespace(mdb.TLSOperatorCASecretNamespacedName().Namespace).
+		SetField(caFileName, cert).
+		SetOwnerReferences(mdb.GetOwnerReferences()).
+		Build()
+
+	return secret.CreateOrUpdate(getUpdateCreator, operatorSecret)
+}
+
 // ensureTLSSecret will create or update the operator-managed Secret containing
 // the concatenated certificate and key from the user-provided Secret.
 func ensureTLSSecret(getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDBCommunity) error {
@@ -214,8 +250,8 @@ func tlsOperatorSecretFileName(certKey string) string {
 }
 
 // tlsConfigModification will enable TLS in the automation config.
-func tlsConfigModification(mdb mdbv1.MongoDBCommunity, certKey string) automationconfig.Modification {
-	caCertificatePath := tlsCAMountPath + tlsCACertName
+func tlsConfigModification(mdb mdbv1.MongoDBCommunity, certKey, caCert string) automationconfig.Modification {
+	caCertificatePath := tlsCAMountPath + tlsOperatorSecretFileName(caCert)
 	certificateKeyPath := tlsOperatorSecretMountPath + tlsOperatorSecretFileName(certKey)
 
 	mode := automationconfig.TLSModeRequired
@@ -247,12 +283,7 @@ func buildTLSPodSpecModification(mdb mdbv1.MongoDBCommunity) podtemplatespec.Mod
 
 	// Configure a volume which mounts the CA certificate from either a Secret or a ConfigMap
 	// The certificate is used by both mongod and the agent
-	var caVolume v1.Volume
-	if mdb.Spec.Security.TLS.CaCertificateSecret != nil {
-		caVolume = statefulset.CreateVolumeFromSecret("tls-ca", mdb.Spec.Security.TLS.CaCertificateSecret.Name)
-	} else {
-		caVolume = statefulset.CreateVolumeFromConfigMap("tls-ca", mdb.Spec.Security.TLS.CaConfigMap.Name)
-	}
+	caVolume := statefulset.CreateVolumeFromSecret("tls-ca", mdb.TLSOperatorCASecretNamespacedName().Name)
 	caVolumeMount := statefulset.CreateVolumeMount(caVolume.Name, tlsCAMountPath, statefulset.WithReadOnly(true))
 
 	// Configure a volume which mounts the secret holding the server key and certificate

controllers/mongodb_tls.go

@@ -39,17 +39,16 @@ func TestStatefulSet_IsCorrectlyConfiguredWithTLS(t *testing.T) {
 
 	// Assert that all TLS volumes have been added.
 	assert.Len(t, sts.Spec.Template.Spec.Volumes, 7)
+	permission := int32(416)
 	assert.Contains(t, sts.Spec.Template.Spec.Volumes, corev1.Volume{
 		Name: "tls-ca",
 		VolumeSource: corev1.VolumeSource{
-			ConfigMap: &corev1.ConfigMapVolumeSource{
-				LocalObjectReference: corev1.LocalObjectReference{
-					Name: mdb.Spec.Security.TLS.CaConfigMap.Name,
-				},
+			Secret: &corev1.SecretVolumeSource{
+				SecretName:  mdb.TLSOperatorCASecretNamespacedName().Name,
+				DefaultMode: &permission,
 			},
 		},
 	})
-	permission := int32(416)
 	assert.Contains(t, sts.Spec.Template.Spec.Volumes, corev1.Volume{
 		Name: "tls-secret",
 		VolumeSource: corev1.VolumeSource{
@@ -90,7 +89,7 @@ func TestAutomationConfig_IsCorrectlyConfiguredWithTLS(t *testing.T) {
 		err = createTLSConfigMap(client, mdb)
 		assert.NoError(t, err)
 
-		tlsModification, err := getTLSConfigModification(client, mdb)
+		tlsModification, err := getTLSConfigModification(client, client, mdb)
 		assert.NoError(t, err)
 		ac, err := buildAutomationConfig(mdb, automationconfig.Auth{}, automationconfig.AutomationConfig{}, tlsModification)
 		assert.NoError(t, err)
@@ -117,7 +116,7 @@ func TestAutomationConfig_IsCorrectlyConfiguredWithTLS(t *testing.T) {
 		ac := createAC(mdb)
 
 		assert.Equal(t, &automationconfig.TLS{
-			CAFilePath:            tlsCAMountPath + tlsCACertName,
+			CAFilePath:            tlsCAMountPath + tlsOperatorSecretFileName("CERT"),
 			ClientCertificateMode: automationconfig.ClientCertificateModeOptional,
 		}, ac.TLSConfig)
 
@@ -126,7 +125,7 @@ func TestAutomationConfig_IsCorrectlyConfiguredWithTLS(t *testing.T) {
 
 			assert.Equal(t, automationconfig.TLSModeRequired, process.Args26.Get("net.tls.mode").Data())
 			assert.Equal(t, tlsOperatorSecretMountPath+operatorSecretFileName, process.Args26.Get("net.tls.certificateKeyFile").Data())
-			assert.Equal(t, tlsCAMountPath+tlsCACertName, process.Args26.Get("net.tls.CAFile").Data())
+			assert.Equal(t, tlsCAMountPath+tlsOperatorSecretFileName("CERT"), process.Args26.Get("net.tls.CAFile").Data())
 			assert.True(t, process.Args26.Get("net.tls.allowConnectionsWithoutCertificates").MustBool())
 		}
 	})
@@ -137,7 +136,7 @@ func TestAutomationConfig_IsCorrectlyConfiguredWithTLS(t *testing.T) {
 		ac := createAC(mdb)
 
 		assert.Equal(t, &automationconfig.TLS{
-			CAFilePath:            tlsCAMountPath + tlsCACertName,
+			CAFilePath:            tlsCAMountPath + tlsOperatorSecretFileName("CERT"),
 			ClientCertificateMode: automationconfig.ClientCertificateModeOptional,
 		}, ac.TLSConfig)
 
@@ -146,7 +145,7 @@ func TestAutomationConfig_IsCorrectlyConfiguredWithTLS(t *testing.T) {
 
 			assert.Equal(t, automationconfig.TLSModePreferred, process.Args26.Get("net.tls.mode").Data())
 			assert.Equal(t, tlsOperatorSecretMountPath+operatorSecretFileName, process.Args26.Get("net.tls.certificateKeyFile").Data())
-			assert.Equal(t, tlsCAMountPath+tlsCACertName, process.Args26.Get("net.tls.CAFile").Data())
+			assert.Equal(t, tlsCAMountPath+tlsOperatorSecretFileName("CERT"), process.Args26.Get("net.tls.CAFile").Data())
 			assert.True(t, process.Args26.Get("net.tls.allowConnectionsWithoutCertificates").MustBool())
 		}
 	})

controllers/mongodb_tls_test.go

@@ -72,12 +72,14 @@ func init() {
 func NewReconciler(mgr manager.Manager) *ReplicaSetReconciler {
 	mgrClient := mgr.GetClient()
 	secretWatcher := watch.New()
+	configMapWatcher := watch.New()
 
 	return &ReplicaSetReconciler{
-		client:        kubernetesClient.NewClient(mgrClient),
-		scheme:        mgr.GetScheme(),
-		log:           zap.S(),
-		secretWatcher: &secretWatcher,
+		client:           kubernetesClient.NewClient(mgrClient),
+		scheme:           mgr.GetScheme(),
+		log:              zap.S(),
+		secretWatcher:    &secretWatcher,
+		configMapWatcher: &configMapWatcher,
 	}
 }
 
@@ -87,6 +89,7 @@ func (r *ReplicaSetReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		WithOptions(controller.Options{MaxConcurrentReconciles: 3}).
 		For(&mdbv1.MongoDBCommunity{}, builder.WithPredicates(predicates.OnlyOnSpecChange())).
 		Watches(&source.Kind{Type: &corev1.Secret{}}, r.secretWatcher).
+		Watches(&source.Kind{Type: &corev1.ConfigMap{}}, r.configMapWatcher).
 		Owns(&appsv1.StatefulSet{}).
 		Complete(r)
 }
@@ -95,10 +98,11 @@ func (r *ReplicaSetReconciler) SetupWithManager(mgr ctrl.Manager) error {
 type ReplicaSetReconciler struct {
 	// This client, initialized using mgr.Client() above, is a split client
 	// that reads objects from the cache and writes to the apiserver
-	client        kubernetesClient.Client
-	scheme        *runtime.Scheme
-	log           *zap.SugaredLogger
-	secretWatcher *watch.ResourceWatcher
+	client           kubernetesClient.Client
+	scheme           *runtime.Scheme
+	log              *zap.SugaredLogger
+	secretWatcher    *watch.ResourceWatcher
+	configMapWatcher *watch.ResourceWatcher
 }
 
 // +kubebuilder:rbac:groups=mongodbcommunity.mongodb.com,resources=mongodbcommunity,verbs=get;list;watch;create;update;patch;delete
@@ -297,6 +301,10 @@ func (r *ReplicaSetReconciler) ensureTLSResources(mdb mdbv1.MongoDBCommunity) er
 	// the TLS secret needs to be created beforehand, as both the StatefulSet and AutomationConfig
 	// require the contents.
 	if mdb.Spec.Security.TLS.Enabled {
+		r.log.Infof("TLS is enabled, creating/updating CA secret")
+		if err := ensureCASecret(r.client, r.client, r.client, mdb); err != nil {
+			return errors.Errorf("could not ensure CA secret: %s", err)
+		}
 		r.log.Infof("TLS is enabled, creating/updating TLS secret")
 		if err := ensureTLSSecret(r.client, mdb); err != nil {
 			return errors.Errorf("could not ensure TLS secret: %s", err)
@@ -597,7 +605,7 @@ func getCustomRolesModification(mdb mdbv1.MongoDBCommunity) (automationconfig.Mo
 }
 
 func (r ReplicaSetReconciler) buildAutomationConfig(mdb mdbv1.MongoDBCommunity) (automationconfig.AutomationConfig, error) {
-	tlsModification, err := getTLSConfigModification(r.client, mdb)
+	tlsModification, err := getTLSConfigModification(r.client, r.client, mdb)
 	if err != nil {
 		return automationconfig.AutomationConfig{}, errors.Errorf("could not configure TLS modification: %s", err)
 	}

controllers/replica_set_controller.go

@@ -173,6 +173,7 @@ rules:
   - watch
   - create
   - update
+  - patch
   - delete
   - patch
   - deletecollection

deploy/e2e/role.yaml

@@ -22,6 +22,7 @@ import (
 	mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/api/v1"
 
 	// Needed for running tests on GCP
+	"k8s.io/client-go/dynamic"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 )
 
@@ -88,8 +89,9 @@ type E2ETestClient struct {
 	Client client.Client
 	// We need the core API client for some operations that the controller-runtime client doesn't support
 	// (e.g. exec into the container)
-	CoreV1Client corev1client.CoreV1Client
-	restConfig   *rest.Config
+	CoreV1Client  corev1client.CoreV1Client
+	DynamicClient dynamic.Interface
+	restConfig    *rest.Config
 }
 
 // NewE2ETestClient creates a new E2ETestClient.
@@ -102,7 +104,11 @@ func newE2ETestClient(config *rest.Config, scheme *runtime.Scheme) (*E2ETestClie
 	if err != nil {
 		return nil, err
 	}
-	return &E2ETestClient{Client: cli, CoreV1Client: *coreClient, restConfig: config}, err
+	dynamicClient, err := dynamic.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+	return &E2ETestClient{Client: cli, CoreV1Client: *coreClient, DynamicClient: dynamicClient, restConfig: config}, err
 }
 
 // Create wraps client.Create to provide post-test cleanup functionality.