From d244fa70b7d22f427292d00fbf53b45a10f5470a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B4=BE=E6=A5=9A?= Date: Fri, 17 Jan 2020 10:43:34 +0000 Subject: [PATCH 01/61] repair update mysqlclusters version field error --- .../pkg/controllers/cluster/controller.go | 622 ++++++++++++++++++ 1 file changed, 622 insertions(+) create mode 100644 src/github.com/oracle/mysql-operator/pkg/controllers/cluster/controller.go diff --git a/src/github.com/oracle/mysql-operator/pkg/controllers/cluster/controller.go b/src/github.com/oracle/mysql-operator/pkg/controllers/cluster/controller.go new file mode 100644 index 000000000..b7159e9aa --- /dev/null +++ b/src/github.com/oracle/mysql-operator/pkg/controllers/cluster/controller.go @@ -0,0 +1,622 @@ +// Copyright 2018 Oracle and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cluster + +import ( + "context" + "fmt" + "strings" + "time" + + apps "k8s.io/api/apps/v1beta1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + wait "k8s.io/apimachinery/pkg/util/wait" + appsinformers "k8s.io/client-go/informers/apps/v1beta1" + coreinformers "k8s.io/client-go/informers/core/v1" + kubernetes "k8s.io/client-go/kubernetes" + scheme "k8s.io/client-go/kubernetes/scheme" + typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + appslisters "k8s.io/client-go/listers/apps/v1beta1" + corelisters "k8s.io/client-go/listers/core/v1" + cache "k8s.io/client-go/tools/cache" + record "k8s.io/client-go/tools/record" + workqueue "k8s.io/client-go/util/workqueue" + + "github.com/coreos/go-semver/semver" + "github.com/golang/glog" + "github.com/pkg/errors" + + clusterutil "github.com/oracle/mysql-operator/pkg/api/cluster" + v1alpha1 "github.com/oracle/mysql-operator/pkg/apis/mysql/v1alpha1" + constants "github.com/oracle/mysql-operator/pkg/constants" + controllerutils "github.com/oracle/mysql-operator/pkg/controllers/util" + clientset "github.com/oracle/mysql-operator/pkg/generated/clientset/versioned" + opscheme "github.com/oracle/mysql-operator/pkg/generated/clientset/versioned/scheme" + informersv1alpha1 "github.com/oracle/mysql-operator/pkg/generated/informers/externalversions/mysql/v1alpha1" + listersv1alpha1 "github.com/oracle/mysql-operator/pkg/generated/listers/mysql/v1alpha1" + + operatoropts "github.com/oracle/mysql-operator/pkg/options/operator" + secrets "github.com/oracle/mysql-operator/pkg/resources/secrets" + services "github.com/oracle/mysql-operator/pkg/resources/services" + statefulsets "github.com/oracle/mysql-operator/pkg/resources/statefulsets" + metrics "github.com/oracle/mysql-operator/pkg/util/metrics" + buildversion "github.com/oracle/mysql-operator/pkg/version" +) + +const controllerAgentName = "mysql-operator" + +const ( + // SuccessSynced is used as part of the Event 'reason' when a MySQSL is + // synced. + SuccessSynced = "Synced" + // ErrResourceExists is used as part of the Event 'reason' when a + // Cluster fails to sync due to a resource of the same name already + // existing. + ErrResourceExists = "ErrResourceExists" + + // MessageResourceExists is the message used for Events when a resource + // fails to sync due to a resource already existing. + MessageResourceExists = "%s %s/%s already exists and is not managed by Cluster" + // MessageResourceSynced is the message used for an Event fired when a + // Cluster is synced successfully + MessageResourceSynced = "Cluster synced successfully" +) + +// The MySQLController watches the Kubernetes API for changes to MySQL resources +type MySQLController struct { + // Global MySQLOperator configuration options. + opConfig operatoropts.MySQLOperatorOpts + + kubeClient kubernetes.Interface + opClient clientset.Interface + + shutdown bool + queue workqueue.RateLimitingInterface + + // clusterLister is able to list/get Clusters from a shared informer's + // store. + clusterLister listersv1alpha1.ClusterLister + // clusterListerSynced returns true if the Cluster shared informer has + // synced at least once. + clusterListerSynced cache.InformerSynced + // clusterUpdater implements control logic for updating Cluster + // statuses. Implemented as an interface to enable testing. + clusterUpdater clusterUpdaterInterface + + // statefulSetLister is able to list/get StatefulSets from a shared + // informer's store. + statefulSetLister appslisters.StatefulSetLister + // statefulSetListerSynced returns true if the StatefulSet shared informer + // has synced at least once. + statefulSetListerSynced cache.InformerSynced + // statefulSetControl enables control of StatefulSets associated with + // Clusters. + statefulSetControl StatefulSetControlInterface + + // podLister is able to list/get Pods from a shared + // informer's store. + podLister corelisters.PodLister + // podListerSynced returns true if the Pod shared informer + // has synced at least once. + podListerSynced cache.InformerSynced + // podControl enables control of Pods associated with + // Clusters. + podControl PodControlInterface + + // serviceLister is able to list/get Services from a shared informer's + // store. + serviceLister corelisters.ServiceLister + // serviceListerSynced returns true if the Service shared informer + // has synced at least once. + serviceListerSynced cache.InformerSynced + + // serviceControl enables control of Services associated with Clusters. + serviceControl ServiceControlInterface + + // secretControl enables control of Services associated with Clusters. + secretControl SecretControlInterface + + // recorder is an event recorder for recording Event resources to the + // Kubernetes API. + recorder record.EventRecorder +} + +// NewController creates a new MySQLController. +func NewController( + opConfig operatoropts.MySQLOperatorOpts, + opClient clientset.Interface, + kubeClient kubernetes.Interface, + clusterInformer informersv1alpha1.ClusterInformer, + statefulSetInformer appsinformers.StatefulSetInformer, + podInformer coreinformers.PodInformer, + serviceInformer coreinformers.ServiceInformer, + resyncPeriod time.Duration, + namespace string, +) *MySQLController { + opscheme.AddToScheme(scheme.Scheme) // TODO: This shouldn't be done here I don't think. + + // Create event broadcaster. + glog.V(4).Info("Creating event broadcaster") + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartLogging(glog.Infof) + eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")}) + recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerAgentName}) + + m := MySQLController{ + opConfig: opConfig, + + opClient: opClient, + kubeClient: kubeClient, + + clusterLister: clusterInformer.Lister(), + clusterListerSynced: clusterInformer.Informer().HasSynced, + clusterUpdater: newClusterUpdater(opClient, clusterInformer.Lister()), + + serviceLister: serviceInformer.Lister(), + serviceListerSynced: serviceInformer.Informer().HasSynced, + serviceControl: NewRealServiceControl(kubeClient, serviceInformer.Lister()), + + statefulSetLister: statefulSetInformer.Lister(), + statefulSetListerSynced: statefulSetInformer.Informer().HasSynced, + statefulSetControl: NewRealStatefulSetControl(kubeClient, statefulSetInformer.Lister()), + + podLister: podInformer.Lister(), + podListerSynced: podInformer.Informer().HasSynced, + podControl: NewRealPodControl(kubeClient, podInformer.Lister()), + + secretControl: NewRealSecretControl(kubeClient), + + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "mysqlcluster"), + recorder: recorder, + } + + clusterInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: m.enqueueCluster, + UpdateFunc: func(old, new interface{}) { + m.enqueueCluster(new) + }, + DeleteFunc: func(obj interface{}) { + cluster, ok := obj.(*v1alpha1.Cluster) + if ok { + m.onClusterDeleted(cluster.Name) + } + }, + }) + + statefulSetInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: m.handleObject, + UpdateFunc: func(old, new interface{}) { + newStatefulSet := new.(*apps.StatefulSet) + oldStatefulSet := old.(*apps.StatefulSet) + if newStatefulSet.ResourceVersion == oldStatefulSet.ResourceVersion { + return + } + + // If cluster is ready ... + if newStatefulSet.Status.ReadyReplicas == newStatefulSet.Status.Replicas { + clusterName, ok := newStatefulSet.Labels[constants.ClusterLabel] + if ok { + m.onClusterReady(clusterName) + } + } + m.handleObject(new) + }, + DeleteFunc: m.handleObject, + }) + + return &m +} + +// Run will set up the event handlers for types we are interested in, as well +// as syncing informer caches and starting workers. It will block until stopCh +// is closed, at which point it will shutdown the workqueue and wait for +// workers to finish processing their current work items. +func (m *MySQLController) Run(ctx context.Context, threadiness int) { + defer utilruntime.HandleCrash() + defer m.queue.ShutDown() + + glog.Info("Starting Cluster controller") + + // Wait for the caches to be synced before starting workers + glog.Info("Waiting for Cluster controller informer caches to sync") + if !controllerutils.WaitForCacheSync("mysql cluster", ctx.Done(), + m.clusterListerSynced, + m.statefulSetListerSynced, + m.podListerSynced, + m.serviceListerSynced) { + return + } + + glog.Info("Starting Cluster controller workers") + // Launch two workers to process Foo resources + for i := 0; i < threadiness; i++ { + go wait.Until(m.runWorker, time.Second, ctx.Done()) + } + + glog.Info("Started Cluster controller workers") + defer glog.Info("Shutting down Cluster controller workers") + <-ctx.Done() +} + +// worker runs a worker goroutine that invokes processNextWorkItem until the +// controller's queue is closed. +func (m *MySQLController) runWorker() { + for m.processNextWorkItem() { + } +} + +// processNextWorkItem will read a single work item off the workqueue and +// attempt to process it, by calling the syncHandler. +func (m *MySQLController) processNextWorkItem() bool { + obj, shutdown := m.queue.Get() + + if shutdown { + return false + } + + err := func(obj interface{}) error { + defer m.queue.Done(obj) + key, ok := obj.(string) + if !ok { + m.queue.Forget(obj) + utilruntime.HandleError(fmt.Errorf("expected string in queue but got %#v", obj)) + return nil + } + if err := m.syncHandler(key); err != nil { + return fmt.Errorf("error syncing '%s': %s", key, err.Error()) + } + m.queue.Forget(obj) + glog.Infof("Successfully synced '%s'", key) + return nil + }(obj) + + if err != nil { + utilruntime.HandleError(err) + return true + } + + return true +} + +// syncHandler compares the actual state with the desired, and attempts to +// converge the two. It then updates the Status block of the Cluster +// resource with the current status of the resource. +func (m *MySQLController) syncHandler(key string) error { + // Convert the namespace/name string into a distinct namespace and name. + namespace, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + nsName := types.NamespacedName{Namespace: namespace, Name: name} + + // Get the Cluster resource with this namespace/name. + cluster, err := m.clusterLister.Clusters(namespace).Get(name) + if err != nil { + // The Cluster resource may no longer exist, in which case we stop processing. + if apierrors.IsNotFound(err) { + utilruntime.HandleError(fmt.Errorf("mysqlcluster '%s' in work queue no longer exists", key)) + return nil + } + return err + } + + cluster.EnsureDefaults() + if err = cluster.Validate(); err != nil { + return errors.Wrap(err, "validating Cluster") + } + + if cluster.Spec.Repository == "" { + cluster.Spec.Repository = m.opConfig.Images.DefaultMySQLServerImage + } + + operatorVersion := buildversion.GetBuildVersion() + // Ensure that the required labels are set on the cluster. + sel := combineSelectors(SelectorForCluster(cluster), SelectorForClusterOperatorVersion(operatorVersion)) + if !sel.Matches(labels.Set(cluster.Labels)) { + glog.V(2).Infof("Setting labels on cluster %s", SelectorForCluster(cluster).String()) + if cluster.Labels == nil { + cluster.Labels = make(map[string]string) + } + cluster.Labels[constants.ClusterLabel] = cluster.Name + cluster.Labels[constants.MySQLOperatorVersionLabel] = buildversion.GetBuildVersion() + return m.clusterUpdater.UpdateClusterLabels(cluster.DeepCopy(), labels.Set(cluster.Labels)) + } + + // Create a MySQL root password secret for the cluster if required. + if cluster.RequiresSecret() { + err = m.secretControl.CreateSecret(secrets.NewMysqlRootPassword(cluster)) + if err != nil && !apierrors.IsAlreadyExists(err) { + return errors.Wrap(err, "creating root password Secret") + } + } + + svc, err := m.serviceLister.Services(cluster.Namespace).Get(cluster.Name) + // If the resource doesn't exist, we'll create it + if apierrors.IsNotFound(err) { + glog.V(2).Infof("Creating a new Service for cluster %q", nsName) + svc = services.NewForCluster(cluster) + err = m.serviceControl.CreateService(svc) + } + + // If an error occurs during Get/Create, we'll requeue the item so we can + // attempt processing again later. This could have been caused by a + // temporary network failure, or any other transient reason. + if err != nil { + return err + } + + // If the Service is not controlled by this Cluster resource, we should + // log a warning to the event recorder and return. + if !metav1.IsControlledBy(svc, cluster) { + msg := fmt.Sprintf(MessageResourceExists, "Service", svc.Namespace, svc.Name) + m.recorder.Event(cluster, corev1.EventTypeWarning, ErrResourceExists, msg) + return errors.New(msg) + } + + ss, err := m.statefulSetLister.StatefulSets(cluster.Namespace).Get(cluster.Name) + // If the resource doesn't exist, we'll create it + if apierrors.IsNotFound(err) { + glog.V(2).Infof("Creating a new StatefulSet for cluster %q", nsName) + ss = statefulsets.NewForCluster(cluster, m.opConfig.Images, svc.Name) + err = m.statefulSetControl.CreateStatefulSet(ss) + } + + // If an error occurs during Get/Create, we'll requeue the item so we can + // attempt processing again later. This could have been caused by a + // temporary network failure, or any other transient reason. + if err != nil { + return err + } + + // If the StatefulSet is not controlled by this Cluster resource, we + // should log a warning to the event recorder and return. + if !metav1.IsControlledBy(ss, cluster) { + msg := fmt.Sprintf(MessageResourceExists, "StatefulSet", ss.Namespace, ss.Name) + m.recorder.Event(cluster, corev1.EventTypeWarning, ErrResourceExists, msg) + return fmt.Errorf(msg) + } + + // Upgrade the required component resources the current MySQLOperator version. + if err := m.ensureMySQLOperatorVersion(cluster, ss, buildversion.GetBuildVersion()); err != nil { + return errors.Wrap(err, "ensuring MySQL Operator version") + } + + // Upgrade the MySQL server version if required. + if err := m.ensureMySQLVersion(cluster, ss); err != nil { + return errors.Wrap(err, "ensuring MySQL version") + } + + // If this number of the members on the Cluster does not equal the + // current desired replicas on the StatefulSet, we should update the + // StatefulSet resource. + if cluster.Spec.Members != *ss.Spec.Replicas { + glog.V(4).Infof("Updating %q: clusterMembers=%d statefulSetReplicas=%d", + nsName, cluster.Spec.Members, ss.Spec.Replicas) + old := ss.DeepCopy() + ss = statefulsets.NewForCluster(cluster, m.opConfig.Images, svc.Name) + if err := m.statefulSetControl.Patch(old, ss); err != nil { + // Requeue the item so we can attempt processing again later. + // This could have been caused by a temporary network failure etc. + return err + } + } + + // Finally, we update the status block of the Cluster resource to + // reflect the current state of the world. + err = m.updateClusterStatus(cluster, ss) + if err != nil { + return err + } + + m.recorder.Event(cluster, corev1.EventTypeNormal, SuccessSynced, MessageResourceSynced) + return nil +} + +func getMySQLContainerIndex(containers []corev1.Container) (int, error) { + for i, c := range containers { + if c.Name == statefulsets.MySQLServerName { + return i, nil + } + } + + return 0, errors.Errorf("no %q container found", statefulsets.MySQLServerName) +} + +// splitImage splits an image into its name and version. +func splitImage(image string) (string, string, error) { + parts := strings.Split(image, ":") + if len(parts) < 2 { + return "", "", errors.Errorf("invalid image %q", image) + } + return strings.Join(parts[:len(parts)-1], ":"), parts[len(parts)-1], nil +} + +func (m *MySQLController) ensureMySQLVersion(c *v1alpha1.Cluster, ss *apps.StatefulSet) error { + index, err := getMySQLContainerIndex(ss.Spec.Template.Spec.Containers) + if err != nil { + return errors.Wrapf(err, "getting MySQL container for StatefulSet %q", ss.Name) + } + imageName, actualVersion, err := splitImage(ss.Spec.Template.Spec.Containers[index].Image) + if err != nil { + return errors.Wrapf(err, "getting MySQL version for StatefulSet %q", ss.Name) + } + + actual, err := semver.NewVersion(actualVersion) + if err != nil { + return errors.Wrap(err, "parsing StatuefulSet MySQL version") + } + expected, err := semver.NewVersion(c.Spec.Version) + if err != nil { + return errors.Wrap(err, "parsing Cluster MySQL version") + } + + switch expected.Compare(*actual) { + case -1: + return errors.Errorf("attempted unsupported downgrade from %q to %q", actual, expected) + case 0: + return nil + } + + updated := ss.DeepCopy() + updated.Spec.Template.Spec.Containers[index].Image = fmt.Sprintf("%s:%s", imageName, c.Spec.Version) + // NOTE: We do this as previously we defaulted to the OnDelete strategy + // so clusters created with previous versions would not support upgrades. + updated.Spec.UpdateStrategy = apps.StatefulSetUpdateStrategy{ + Type: apps.RollingUpdateStatefulSetStrategyType, + } + + err = m.statefulSetControl.Patch(ss, updated) + if err != nil { + return errors.Wrap(err, "patching StatefulSet") + } + + return nil +} + +// ensureMySQLOperatorVersion updates the MySQLOperator resource types that +//require it to make it consistent with the specified operator version. +func (m *MySQLController) ensureMySQLOperatorVersion(c *v1alpha1.Cluster, ss *apps.StatefulSet, operatorVersion string) error { + // Ensure the Pods belonging to the Cluster are updated to the correct 'mysql-agent' image for the current MySQLOperator version. + container := statefulsets.MySQLAgentName + pods, err := m.podLister.List(SelectorForCluster(c)) + if err != nil { + return errors.Wrapf(err, "listing pods matching %q", SelectorForCluster(c).String()) + } + for _, pod := range pods { + if requiresMySQLAgentPodUpgrade(pod, container, operatorVersion) && canUpgradeMySQLAgent(pod) { + glog.Infof("Upgrading cluster pod '%s/%s' to latest operator version: %s", pod.Namespace, pod.Name, operatorVersion) + updated := updatePodToOperatorVersion(pod.DeepCopy(), m.opConfig.Images.MySQLAgentImage, operatorVersion) + err = m.podControl.PatchPod(pod, updated) + if err != nil { + return errors.Wrap(err, "upgrade operator version: PatchPod failed") + } + } + } + + // Ensure the StatefulSet is updated with the correct template 'mysql-agent' image for the current MySQLOperator version. + if requiresMySQLAgentStatefulSetUpgrade(ss, container, operatorVersion) { + glog.Infof("Upgrading cluster statefulset '%s/%s' to latest operator version: %s", ss.Namespace, ss.Name, operatorVersion) + updated := updateStatefulSetToOperatorVersion(ss.DeepCopy(), m.opConfig.Images.MySQLAgentImage, operatorVersion) + err = m.statefulSetControl.Patch(ss, updated) + if err != nil { + return errors.Wrap(err, "upgrade operator version: PatchStatefulSet failed") + } + } + + // Ensure the Cluster is updated with the correct MySQLOperator version. + if !SelectorForClusterOperatorVersion(operatorVersion).Matches(labels.Set(c.Labels)) { + glog.Infof("Upgrading cluster statefulset '%s/%s' to latest operator version: %s", c.Namespace, c.Name, operatorVersion) + copy := c.DeepCopy() + copy.Labels[constants.MySQLOperatorVersionLabel] = operatorVersion + err := m.clusterUpdater.UpdateClusterLabels(copy, labels.Set(copy.Labels)) + if err != nil { + return errors.Wrap(err, "upgrade operator version: ClusterUpdate failed") + } + } + return nil +} + +// updateClusterStatusForSS updates Cluster statuses based on changes to their associated StatefulSets. +func (m *MySQLController) updateClusterStatus(cluster *v1alpha1.Cluster, ss *apps.StatefulSet) error { + glog.V(4).Infof("%s/%s: ss.Spec.Replicas=%d, ss.Status.ReadyReplicas=%d, ss.Status.Replicas=%d", + cluster.Namespace, cluster.Name, *ss.Spec.Replicas, ss.Status.ReadyReplicas, ss.Status.Replicas) + + status := cluster.Status.DeepCopy() + _, condition := clusterutil.GetClusterCondition(&cluster.Status, v1alpha1.ClusterReady) + if condition == nil { + condition = &v1alpha1.ClusterCondition{Type: v1alpha1.ClusterReady} + } + if ss.Status.ReadyReplicas == ss.Status.Replicas && ss.Status.ReadyReplicas == cluster.Spec.Members { + condition.Status = corev1.ConditionTrue + } else { + condition.Status = corev1.ConditionFalse + } + + if updated := clusterutil.UpdateClusterCondition(status, condition); updated { + return m.clusterUpdater.UpdateClusterStatus(cluster.DeepCopy(), status) + } + return nil +} + +// enqueueCluster takes a Cluster resource and converts it into a +// namespace/name string which is then put onto the work queue. This method +// should *not* be passed resources of any type other than Cluster. +func (m *MySQLController) enqueueCluster(obj interface{}) { + key, err := cache.MetaNamespaceKeyFunc(obj) + if err != nil { + utilruntime.HandleError(err) + return + } + m.queue.AddRateLimited(key) +} + +// handleObject will take any resource implementing metav1.Object and attempt +// to find the MySQLResource that 'owns' it. It does this by looking at the +// objects metadata.ownerReferences field for an appropriate OwnerReference. +// It then enqueues that Foo resource to be processed. If the object does not +// have an appropriate OwnerReference, it will simply be skipped. +func (m *MySQLController) handleObject(obj interface{}) { + object, ok := obj.(metav1.Object) + if !ok { + tombstone, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + utilruntime.HandleError(fmt.Errorf("error decoding object, invalid type")) + return + } + object, ok = tombstone.Obj.(metav1.Object) + if !ok { + utilruntime.HandleError(fmt.Errorf("error decoding object tombstone, invalid type")) + return + } + glog.V(4).Infof("Recovered deleted object '%s' from tombstone", object.GetName()) + } + + glog.V(4).Infof("Processing object: %s", object.GetName()) + if ownerRef := metav1.GetControllerOf(object); ownerRef != nil { + // If this object is not owned by a Cluster, we should not do + // anything more with it. + if ownerRef.Kind != v1alpha1.ClusterCRDResourceKind { + return + } + + cluster, err := m.clusterLister.Clusters(object.GetNamespace()).Get(ownerRef.Name) + if err != nil { + glog.V(4).Infof("ignoring orphaned object '%s' of Cluster '%s'", object.GetSelfLink(), ownerRef.Name) + return + } + + m.enqueueCluster(cluster) + return + } +} + +func (m *MySQLController) onClusterReady(clusterName string) { + glog.V(2).Infof("Cluster %s ready", clusterName) + metrics.IncEventCounter(clustersCreatedCount) + metrics.IncEventGauge(clustersTotalCount) +} + +func (m *MySQLController) onClusterDeleted(clusterName string) { + glog.V(2).Infof("Cluster %s deleted", clusterName) + metrics.IncEventCounter(clustersDeletedCount) + metrics.DecEventGauge(clustersTotalCount) +} From d5ea26af57019e22b9317a57ed8e8d3c86ee08c5 Mon Sep 17 00:00:00 2001 From: jiachu Date: Sun, 19 Jan 2020 14:33:02 +0800 Subject: [PATCH 02/61] repair json decode problem --- pkg/util/mysqlsh/mysqlsh.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pkg/util/mysqlsh/mysqlsh.go b/pkg/util/mysqlsh/mysqlsh.go index 283ac2d3a..b6f29caaf 100644 --- a/pkg/util/mysqlsh/mysqlsh.go +++ b/pkg/util/mysqlsh/mysqlsh.go @@ -118,9 +118,22 @@ func (r *runner) GetClusterStatus(ctx context.Context) (*innodb.ClusterStatus, e if err != nil { return nil, err } - + + //repair json decode problem + rawJson := string(output) + firstBraceIndex := strings.Index(rawJson, "{") + + var jsonData string + + if firstGraceIndex == -1 { + return nil, errors.Errorf("no json found in output: %q", rawJson) + } + + jsonData = rawJson[firstBraceIndex:] + glog.V(2).Infof("mysqlsh clusterstatus: %q", jsonData) + status := &innodb.ClusterStatus{} - err = json.Unmarshal(sanitizeJSON(output), status) + err = json.Unmarshal(sanitizeJSON([]byte(jsonData)), status) if err != nil { return nil, errors.Wrapf(err, "decoding cluster status output: %q", output) } From 68aa589b08c184b32b84ad914bf4654ffe7117fa Mon Sep 17 00:00:00 2001 From: jiachu Date: Sun, 19 Jan 2020 14:57:52 +0800 Subject: [PATCH 03/61] debug --- pkg/util/mysqlsh/mysqlsh.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/mysqlsh/mysqlsh.go b/pkg/util/mysqlsh/mysqlsh.go index b6f29caaf..ebc7f38f2 100644 --- a/pkg/util/mysqlsh/mysqlsh.go +++ b/pkg/util/mysqlsh/mysqlsh.go @@ -125,7 +125,7 @@ func (r *runner) GetClusterStatus(ctx context.Context) (*innodb.ClusterStatus, e var jsonData string - if firstGraceIndex == -1 { + if firstBraceIndex == -1 { return nil, errors.Errorf("no json found in output: %q", rawJson) } From eb8fbbd871bf6576a915b68cf4437263cfb5c83e Mon Sep 17 00:00:00 2001 From: jiachu Date: Sun, 19 Jan 2020 18:10:33 +0800 Subject: [PATCH 04/61] increase timeout --- pkg/controllers/cluster/manager/innodb_cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controllers/cluster/manager/innodb_cluster.go b/pkg/controllers/cluster/manager/innodb_cluster.go index 2039cc1b3..9f3b1b823 100644 --- a/pkg/controllers/cluster/manager/innodb_cluster.go +++ b/pkg/controllers/cluster/manager/innodb_cluster.go @@ -35,7 +35,7 @@ import ( var errNoClusterFound = errors.New("no cluster found on any of the seed nodes") -const defaultTimeout = 10 * time.Second +const defaultTimeout = 60 * time.Second // isDatabaseRunning returns true if a connection can be made to the MySQL // database running in the pod instance in which this function is called. From 04c63a01bbe5fa1b503dcb4fda0bc0ebe1a0553d Mon Sep 17 00:00:00 2001 From: jiachu Date: Fri, 14 Feb 2020 14:33:40 +0800 Subject: [PATCH 05/61] update clusterspec --- pkg/apis/mysql/v1alpha1/types.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/apis/mysql/v1alpha1/types.go b/pkg/apis/mysql/v1alpha1/types.go index dcc8735cf..ebe6bfa42 100644 --- a/pkg/apis/mysql/v1alpha1/types.go +++ b/pkg/apis/mysql/v1alpha1/types.go @@ -43,6 +43,8 @@ type ClusterSpec struct { // all instances will be R/W. If false (the default), only a single instance // will be R/W and the rest will be R/O. MultiMaster bool `json:"multiMaster,omitempty"` + //hostnetwork + HostNetwork bool `json:"hostNetwork,omitempty"` // NodeSelector is a selector which must be true for the pod to fit on a node. // Selector which must match a node's labels for the pod to be scheduled on that node. // More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ From dc1ed4fdc2f2f1d3fcabdcf301f590fce4f8fabb Mon Sep 17 00:00:00 2001 From: jiachu Date: Fri, 14 Feb 2020 14:36:19 +0800 Subject: [PATCH 06/61] update statefulsets resource --- pkg/resources/statefulsets/statefulset.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index b3ae46f3c..3dbfd654a 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -416,6 +416,7 @@ func NewForCluster(cluster *v1alpha1.Cluster, images operatoropts.Images, servic ServiceAccountName: "mysql-agent", NodeSelector: cluster.Spec.NodeSelector, Affinity: cluster.Spec.Affinity, + HostNetwork: cluster.Spec.HostNetwork, Containers: containers, Volumes: podVolumes, }, From b2d68f6d4ba62a497993504c581666ca4109c0de Mon Sep 17 00:00:00 2001 From: jiachu Date: Fri, 14 Feb 2020 18:44:06 +0800 Subject: [PATCH 07/61] add debug info --- pkg/controllers/cluster/manager/cluster_manager.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/controllers/cluster/manager/cluster_manager.go b/pkg/controllers/cluster/manager/cluster_manager.go index 3f3648a0b..74cef52f2 100644 --- a/pkg/controllers/cluster/manager/cluster_manager.go +++ b/pkg/controllers/cluster/manager/cluster_manager.go @@ -122,6 +122,7 @@ func (m *ClusterManager) Sync(ctx context.Context) bool { clusterStatus, err := m.getClusterStatus(ctx) if err != nil { myshErr, ok := errors.Cause(err).(*mysqlsh.Error) + glog.V(2).Infof("get cluster*** error: %+v", err) if !ok { glog.Errorf("Failed to get the cluster status: %+v", err) return false From 4cb581fb2b9628e4444a5e74ce20384da337787e Mon Sep 17 00:00:00 2001 From: jiachu Date: Fri, 14 Feb 2020 19:09:13 +0800 Subject: [PATCH 08/61] add debug info --- pkg/controllers/cluster/manager/cluster_manager.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/controllers/cluster/manager/cluster_manager.go b/pkg/controllers/cluster/manager/cluster_manager.go index 74cef52f2..a1b4ce10c 100644 --- a/pkg/controllers/cluster/manager/cluster_manager.go +++ b/pkg/controllers/cluster/manager/cluster_manager.go @@ -100,6 +100,7 @@ func (m *ClusterManager) getClusterStatus(ctx context.Context) (*innodb.ClusterS if localMSHErr != nil { var err error clusterStatus, err = getClusterStatusFromGroupSeeds(ctx, m.kubeClient, m.Instance) + glog.V(2).Infof("get cluster seeds*** error: %+v", err) if err != nil { // NOTE: We return the localMSHErr rather than the error here so that we // can dispatch on it. From 9f9821c413529d749c037d04401eda15101651e8 Mon Sep 17 00:00:00 2001 From: jiachu Date: Mon, 17 Feb 2020 10:37:25 +0800 Subject: [PATCH 09/61] update getclusterstatus --- pkg/cluster/instance.go | 2 +- pkg/resources/statefulsets/statefulset.go | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/cluster/instance.go b/pkg/cluster/instance.go index 16e448db6..600e5ce68 100644 --- a/pkg/cluster/instance.go +++ b/pkg/cluster/instance.go @@ -62,7 +62,7 @@ func NewInstance(namespace, clusterName, parentName string, ordinal, port int, m // NewLocalInstance creates a new instance of this structure, with it's name and index // populated from os.Hostname(). func NewLocalInstance() (*Instance, error) { - hostname, err := os.Hostname() + hostname, err := os.Getenv("MY_POD_NAME") if err != nil { return nil, err } diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index 3dbfd654a..9a8de032b 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -249,6 +249,14 @@ func mysqlServerContainer(cluster *v1alpha1.Cluster, mysqlServerImage string, ro Name: "MYSQL_LOG_CONSOLE", Value: "true", }, + { + Name: "MY_POD_NAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + } }, Resources: resourceLimits, } From 46d39f27b9ce3a721cfb09bcb0fd9036dfd0af59 Mon Sep 17 00:00:00 2001 From: jiachu Date: Mon, 17 Feb 2020 10:44:43 +0800 Subject: [PATCH 10/61] update instance.go --- pkg/cluster/instance.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/cluster/instance.go b/pkg/cluster/instance.go index 600e5ce68..8eb295b77 100644 --- a/pkg/cluster/instance.go +++ b/pkg/cluster/instance.go @@ -62,11 +62,11 @@ func NewInstance(namespace, clusterName, parentName string, ordinal, port int, m // NewLocalInstance creates a new instance of this structure, with it's name and index // populated from os.Hostname(). func NewLocalInstance() (*Instance, error) { - hostname, err := os.Getenv("MY_POD_NAME") - if err != nil { - return nil, err + pod_name := os.Getenv("MY_POD_NAME") + if pod_name == nil { + return nil, errors.Errorf("env MY_POD_NAME is empty!!!") } - name, ordinal := GetParentNameAndOrdinal(hostname) + name, ordinal := GetParentNameAndOrdinal(pod_name) multiMaster, _ := strconv.ParseBool(os.Getenv("MYSQL_CLUSTER_MULTI_MASTER")) return &Instance{ Namespace: os.Getenv("POD_NAMESPACE"), From 59c524e712c2a749c1983dd06562f75cbd94119d Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Mon, 17 Feb 2020 10:50:28 +0800 Subject: [PATCH 11/61] debug statefulsets --- pkg/resources/statefulsets/statefulset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index 9a8de032b..f8fa7bf89 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -256,7 +256,7 @@ func mysqlServerContainer(cluster *v1alpha1.Cluster, mysqlServerImage string, ro FieldPath: "metadata.name", }, }, - } + }, }, Resources: resourceLimits, } From 1cfd5d294d18e8229d767528374590d0ee92d568 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Mon, 17 Feb 2020 11:04:08 +0800 Subject: [PATCH 12/61] debug os env --- pkg/cluster/instance.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/instance.go b/pkg/cluster/instance.go index 8eb295b77..53d523ac4 100644 --- a/pkg/cluster/instance.go +++ b/pkg/cluster/instance.go @@ -63,7 +63,7 @@ func NewInstance(namespace, clusterName, parentName string, ordinal, port int, m // populated from os.Hostname(). func NewLocalInstance() (*Instance, error) { pod_name := os.Getenv("MY_POD_NAME") - if pod_name == nil { + if pod_name == "" { return nil, errors.Errorf("env MY_POD_NAME is empty!!!") } name, ordinal := GetParentNameAndOrdinal(pod_name) From a618fabe614962ad919a7d8fa5fa48efc081ec1b Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Mon, 17 Feb 2020 11:28:36 +0800 Subject: [PATCH 13/61] debug mysql-agent container --- pkg/resources/statefulsets/statefulset.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index f8fa7bf89..aeb78d705 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -220,7 +220,7 @@ func mysqlServerContainer(cluster *v1alpha1.Cluster, mysqlServerImage string, ro # Finds the replica index from the hostname, and uses this to define # a unique server id for this instance. - index=$(cat /etc/hostname | grep -o '[^-]*$') + index=$( $MY_POD_NAME | grep -o '[^-]*$') /entrypoint.sh %s`, baseServerID, entryPointArgs) var resourceLimits corev1.ResourceRequirements @@ -294,6 +294,14 @@ func mysqlAgentContainer(cluster *v1alpha1.Cluster, mysqlAgentImage string, root }, }, }, + { + Name: "MY_POD_NAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, }, LivenessProbe: &v1.Probe{ Handler: v1.Handler{ From 3d732fad60b229a6102bd29ce72abb4097d54e94 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Mon, 17 Feb 2020 11:40:38 +0800 Subject: [PATCH 14/61] update statefulset.go --- pkg/resources/statefulsets/statefulset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index aeb78d705..b58453261 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -220,7 +220,7 @@ func mysqlServerContainer(cluster *v1alpha1.Cluster, mysqlServerImage string, ro # Finds the replica index from the hostname, and uses this to define # a unique server id for this instance. - index=$( $MY_POD_NAME | grep -o '[^-]*$') + index=$(echo $MY_POD_NAME | grep -o '[^-]*$') /entrypoint.sh %s`, baseServerID, entryPointArgs) var resourceLimits corev1.ResourceRequirements From 04e22422fe3a7c36c890316983e292369c511a62 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Mon, 17 Feb 2020 11:48:22 +0800 Subject: [PATCH 15/61] update mysql-agent debug level --- pkg/resources/statefulsets/statefulset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index b58453261..d4ccffa99 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -278,7 +278,7 @@ func mysqlAgentContainer(cluster *v1alpha1.Cluster, mysqlAgentImage string, root return v1.Container{ Name: MySQLAgentName, Image: fmt.Sprintf("%s:%s", mysqlAgentImage, agentVersion), - Args: []string{"--v=4"}, + Args: []string{"--v=6"}, VolumeMounts: volumeMounts(cluster), Env: []v1.EnvVar{ clusterNameEnvVar(cluster), From a3627bfea1676f6e43e62bcc8cd27afdcff605ae Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Mon, 17 Feb 2020 14:32:18 +0800 Subject: [PATCH 16/61] update image pull policy --- pkg/resources/statefulsets/statefulset.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index d4ccffa99..1eadfee4c 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -232,6 +232,7 @@ func mysqlServerContainer(cluster *v1alpha1.Cluster, mysqlServerImage string, ro Name: MySQLServerName, // TODO(apryde): Add BaseImage to cluster CRD. Image: fmt.Sprintf("%s:%s", mysqlServerImage, cluster.Spec.Version), + ImagePullPolicy: v1.PullAlways, Ports: []v1.ContainerPort{ { ContainerPort: 3306, @@ -278,6 +279,7 @@ func mysqlAgentContainer(cluster *v1alpha1.Cluster, mysqlAgentImage string, root return v1.Container{ Name: MySQLAgentName, Image: fmt.Sprintf("%s:%s", mysqlAgentImage, agentVersion), + ImagePullPolicy: v1.PullAlways, Args: []string{"--v=6"}, VolumeMounts: volumeMounts(cluster), Env: []v1.EnvVar{ From d6130cefee5e6f8b7cf360d4f039b74ffc19ae9e Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Mon, 17 Feb 2020 18:13:10 +0800 Subject: [PATCH 17/61] update port --- pkg/apis/mysql/v1alpha1/helpers.go | 5 +++++ pkg/apis/mysql/v1alpha1/types.go | 2 ++ pkg/controllers/cluster/manager/cluster_manager.go | 8 ++++++++ pkg/resources/statefulsets/statefulset.go | 10 +++++++--- pkg/util/mysqlsh/mysqlsh.go | 1 + 5 files changed, 23 insertions(+), 3 deletions(-) diff --git a/pkg/apis/mysql/v1alpha1/helpers.go b/pkg/apis/mysql/v1alpha1/helpers.go index df25b1465..8adbab56f 100644 --- a/pkg/apis/mysql/v1alpha1/helpers.go +++ b/pkg/apis/mysql/v1alpha1/helpers.go @@ -22,6 +22,7 @@ import ( const ( // DefaultVersion is the MySQL version to use if not specified explicitly by user DefaultVersion = "8.0.12" + DefaultReplicationGroupPort = 33061 defaultMembers = 3 defaultBaseServerID = 1000 // maxBaseServerID is the maximum safe value for BaseServerID calculated @@ -69,6 +70,10 @@ func (c *Cluster) EnsureDefaults() *Cluster { c.Spec.Version = DefaultVersion } + if c.Spec.GroupPort == 0 { + c.Spec.GroupPort = DefaultReplicationGroupPort + } + return c } diff --git a/pkg/apis/mysql/v1alpha1/types.go b/pkg/apis/mysql/v1alpha1/types.go index ebe6bfa42..ca390a04e 100644 --- a/pkg/apis/mysql/v1alpha1/types.go +++ b/pkg/apis/mysql/v1alpha1/types.go @@ -45,6 +45,8 @@ type ClusterSpec struct { MultiMaster bool `json:"multiMaster,omitempty"` //hostnetwork HostNetwork bool `json:"hostNetwork,omitempty"` + //replicationGroupPort + GroupPort uint32 `json:"groupPort,omitempty"` // NodeSelector is a selector which must be true for the pod to fit on a node. // Selector which must match a node's labels for the pod to be scheduled on that node. // More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ diff --git a/pkg/controllers/cluster/manager/cluster_manager.go b/pkg/controllers/cluster/manager/cluster_manager.go index a1b4ce10c..c7f49a035 100644 --- a/pkg/controllers/cluster/manager/cluster_manager.go +++ b/pkg/controllers/cluster/manager/cluster_manager.go @@ -324,9 +324,17 @@ func (m *ClusterManager) createCluster(ctx context.Context) (*innodb.ClusterStat if err != nil { return nil, errors.Wrap(err, "getting CIDR to whitelist for GR") } + + localAddress := fmt.Sprintf("%s:%s", m.instance.Name(), os.Getenv("GROUP_PORT")) + groupSeeds := os.Getenv("REPLICATION_GROUP_SEEDS") + + glog.Infof("localAddress: %s, groupSeeds: %s", localAddress, groupSeeds) + opts := mysqlsh.Options{ "memberSslMode": "REQUIRED", "ipWhitelist": whitelistCIDR, + "localAddress": localAddress, + "groupSeeds": groupSeeds, } if m.Instance.MultiMaster { opts["force"] = "True" diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index 1eadfee4c..e7ae9c9bb 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -151,10 +151,10 @@ func mysqlRootPassword(cluster *v1alpha1.Cluster) v1.EnvVar { } } -func getReplicationGroupSeeds(name string, members int) string { +func getReplicationGroupSeeds(name string, members int, groupPort uint32) string { seeds := []string{} for i := 0; i < members; i++ { - seeds = append(seeds, fmt.Sprintf("%[1]s-%[2]d.%[1]s:%[3]d", name, i, replicationGroupPort)) + seeds = append(seeds, fmt.Sprintf("%[1]s-%[2]d.%[1]s:%[3]d", name, i, groupPort)) } return strings.Join(seeds, ",") } @@ -269,7 +269,7 @@ func mysqlAgentContainer(cluster *v1alpha1.Cluster, mysqlAgentImage string, root agentVersion = version } - replicationGroupSeeds := getReplicationGroupSeeds(cluster.Name, members) + replicationGroupSeeds := getReplicationGroupSeeds(cluster.Name, members, cluster.Spec.GroupPort) var resourceLimits corev1.ResourceRequirements if cluster.Spec.Resources != nil && cluster.Spec.Resources.Agent != nil { @@ -288,6 +288,10 @@ func mysqlAgentContainer(cluster *v1alpha1.Cluster, mysqlAgentImage string, root replicationGroupSeedsEnvVar(replicationGroupSeeds), multiMasterEnvVar(cluster.Spec.MultiMaster), rootPassword, + { + Name: "GROUP_PORT", + Value: cluster.Spec.GroupPort, + }, { Name: "MY_POD_IP", ValueFrom: &v1.EnvVarSource{ diff --git a/pkg/util/mysqlsh/mysqlsh.go b/pkg/util/mysqlsh/mysqlsh.go index ebc7f38f2..7ec2e9743 100644 --- a/pkg/util/mysqlsh/mysqlsh.go +++ b/pkg/util/mysqlsh/mysqlsh.go @@ -87,6 +87,7 @@ func (r *runner) IsClustered(ctx context.Context) bool { func (r *runner) CreateCluster(ctx context.Context, opts Options) (*innodb.ClusterStatus, error) { python := fmt.Sprintf("print dba.create_cluster('%s', %s).status()", innodb.DefaultClusterName, opts) + glog.V(2).Infof("mysqlsh CreateCluster: %q", python) output, err := r.run(ctx, python) if err != nil { return nil, err From fe4a1ad3f3d5105e205366d811a07ce1143ed69f Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Mon, 17 Feb 2020 18:22:34 +0800 Subject: [PATCH 18/61] update statefulsets --- pkg/resources/statefulsets/statefulset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index e7ae9c9bb..80d67b518 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -290,7 +290,7 @@ func mysqlAgentContainer(cluster *v1alpha1.Cluster, mysqlAgentImage string, root rootPassword, { Name: "GROUP_PORT", - Value: cluster.Spec.GroupPort, + Value: strconv.FromInt(cluster.Spec.GroupPort), }, { Name: "MY_POD_IP", From 89c4c77f22fdcf1a82956ee7f8109147009f4268 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Mon, 17 Feb 2020 18:27:17 +0800 Subject: [PATCH 19/61] update statefulsets --- pkg/resources/statefulsets/statefulset.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index 80d67b518..c364a0ea6 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -127,6 +127,12 @@ func multiMasterEnvVar(enabled bool) v1.EnvVar { } } +func replicationGroupPortEnvVar(groupPort uint32) v1.EnvVar { + return v1.EnvVar{ + Name: "GROUP_PORT", + Value: strconv.FromInt(groupPort), + } +} // Returns the MySQL_ROOT_PASSWORD environment variable // If a user specifies a secret in the spec we use that // else we create a secret with a random password @@ -287,11 +293,8 @@ func mysqlAgentContainer(cluster *v1alpha1.Cluster, mysqlAgentImage string, root namespaceEnvVar(), replicationGroupSeedsEnvVar(replicationGroupSeeds), multiMasterEnvVar(cluster.Spec.MultiMaster), + replicationGroupPortEnvVar(cluster.Spec.GroupPort), rootPassword, - { - Name: "GROUP_PORT", - Value: strconv.FromInt(cluster.Spec.GroupPort), - }, { Name: "MY_POD_IP", ValueFrom: &v1.EnvVarSource{ From 462874805ed9fb9a68014c4c8c40a0020aeda8f9 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Mon, 17 Feb 2020 18:33:16 +0800 Subject: [PATCH 20/61] update statefulsets --- pkg/resources/statefulsets/statefulset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index c364a0ea6..3fa377cae 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -130,7 +130,7 @@ func multiMasterEnvVar(enabled bool) v1.EnvVar { func replicationGroupPortEnvVar(groupPort uint32) v1.EnvVar { return v1.EnvVar{ Name: "GROUP_PORT", - Value: strconv.FromInt(groupPort), + Value: strconv.FormatUint(groupPort), } } // Returns the MySQL_ROOT_PASSWORD environment variable From c7f55d400d397ee1ab2eea700da9e978b1c01ee0 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Mon, 17 Feb 2020 18:38:14 +0800 Subject: [PATCH 21/61] update statefulsets --- pkg/resources/statefulsets/statefulset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index 3fa377cae..736ac655f 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -130,7 +130,7 @@ func multiMasterEnvVar(enabled bool) v1.EnvVar { func replicationGroupPortEnvVar(groupPort uint32) v1.EnvVar { return v1.EnvVar{ Name: "GROUP_PORT", - Value: strconv.FormatUint(groupPort), + Value: strconv.FormatUint(groupPort, 10), } } // Returns the MySQL_ROOT_PASSWORD environment variable From 3243d65ad8f4c851821986beeca68edc4ba4aa55 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Mon, 17 Feb 2020 18:56:04 +0800 Subject: [PATCH 22/61] update statefulsets --- pkg/resources/statefulsets/statefulset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index 736ac655f..cf6a9916a 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -130,7 +130,7 @@ func multiMasterEnvVar(enabled bool) v1.EnvVar { func replicationGroupPortEnvVar(groupPort uint32) v1.EnvVar { return v1.EnvVar{ Name: "GROUP_PORT", - Value: strconv.FormatUint(groupPort, 10), + Value: strconv.FormatUint(uint64(groupPort), 10), } } // Returns the MySQL_ROOT_PASSWORD environment variable From d62e4fde6f7ae41817e328ce616929caaae1dcf7 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Mon, 17 Feb 2020 18:57:53 +0800 Subject: [PATCH 23/61] update statefulsets --- pkg/controllers/cluster/manager/cluster_manager.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controllers/cluster/manager/cluster_manager.go b/pkg/controllers/cluster/manager/cluster_manager.go index c7f49a035..e1510e2ef 100644 --- a/pkg/controllers/cluster/manager/cluster_manager.go +++ b/pkg/controllers/cluster/manager/cluster_manager.go @@ -325,11 +325,11 @@ func (m *ClusterManager) createCluster(ctx context.Context) (*innodb.ClusterStat return nil, errors.Wrap(err, "getting CIDR to whitelist for GR") } - localAddress := fmt.Sprintf("%s:%s", m.instance.Name(), os.Getenv("GROUP_PORT")) + localAddress := fmt.Sprintf("%s:%s", m.Instance.Name(), os.Getenv("GROUP_PORT")) groupSeeds := os.Getenv("REPLICATION_GROUP_SEEDS") glog.Infof("localAddress: %s, groupSeeds: %s", localAddress, groupSeeds) - + opts := mysqlsh.Options{ "memberSslMode": "REQUIRED", "ipWhitelist": whitelistCIDR, From fc23319e5ace4c516d2e4ba1e7940d78d6037fa8 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Mon, 17 Feb 2020 18:59:26 +0800 Subject: [PATCH 24/61] update statefulsets --- pkg/controllers/cluster/manager/cluster_manager.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/controllers/cluster/manager/cluster_manager.go b/pkg/controllers/cluster/manager/cluster_manager.go index e1510e2ef..79e6221b7 100644 --- a/pkg/controllers/cluster/manager/cluster_manager.go +++ b/pkg/controllers/cluster/manager/cluster_manager.go @@ -17,6 +17,7 @@ package manager import ( "context" "fmt" + "os" "strings" "time" From c4040d560a1b6e154d0c3c50a3d524d87078ea08 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Tue, 18 Feb 2020 10:05:49 +0800 Subject: [PATCH 25/61] update log-bin basename --- pkg/resources/statefulsets/statefulset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index cf6a9916a..d982dc861 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -194,7 +194,7 @@ func mysqlServerContainer(cluster *v1alpha1.Cluster, mysqlServerImage string, ro "--datadir=/var/lib/mysql", "--user=mysql", "--gtid_mode=ON", - "--log-bin", + "--log-bin=$MY_POD_NAME", "--binlog_checksum=NONE", "--enforce_gtid_consistency=ON", "--log-slave-updates=ON", From 82825cecf74cd2c5306b40a4ff68d7b37b340683 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Tue, 18 Feb 2020 11:46:08 +0800 Subject: [PATCH 26/61] update agent listen port --- cmd/mysql-agent/app/mysql_agent.go | 8 +++++--- pkg/apis/mysql/v1alpha1/helpers.go | 9 +++++++++ pkg/apis/mysql/v1alpha1/types.go | 4 ++++ .../cluster/manager/cluster_manager.go | 7 +++++++ pkg/options/agent/options.go | 4 +++- pkg/resources/statefulsets/statefulset.go | 19 ++++++++++++++++++- 6 files changed, 46 insertions(+), 5 deletions(-) diff --git a/cmd/mysql-agent/app/mysql_agent.go b/cmd/mysql-agent/app/mysql_agent.go index 85c1eba95..511ddec31 100644 --- a/cmd/mysql-agent/app/mysql_agent.go +++ b/cmd/mysql-agent/app/mysql_agent.go @@ -22,6 +22,7 @@ import ( "strconv" "sync" "time" + "os" "github.com/golang/glog" "github.com/heptiolabs/healthcheck" @@ -100,15 +101,16 @@ func Run(opts *agentopts.MySQLAgentOpts) error { if err != nil { return errors.Wrap(err, "failed to create new local MySQL InnoDB cluster manager") } - - // Initialise the agent metrics. + // agent prometheus port + agentPromePort := os.Getenv("AGENT_PROME_PORT") + glog.Info("agent prometheus port: %d", agentPromePort) metrics.RegisterPodName(opts.Hostname) metrics.RegisterClusterName(manager.Instance.ClusterName) clustermgr.RegisterMetrics() backupcontroller.RegisterMetrics() restorecontroller.RegisterMetrics() http.Handle("/metrics", prometheus.Handler()) - go http.ListenAndServe(metricsEndpoint, nil) + go http.ListenAndServe(agentPromePort, nil) // Block until local instance successfully initialised. for !manager.Sync(ctx) { diff --git a/pkg/apis/mysql/v1alpha1/helpers.go b/pkg/apis/mysql/v1alpha1/helpers.go index 8adbab56f..c71ce3fc9 100644 --- a/pkg/apis/mysql/v1alpha1/helpers.go +++ b/pkg/apis/mysql/v1alpha1/helpers.go @@ -23,6 +23,8 @@ const ( // DefaultVersion is the MySQL version to use if not specified explicitly by user DefaultVersion = "8.0.12" DefaultReplicationGroupPort = 33061 + DefaultAgentHealthCheckPort = 10512 + DefaultAgentPromePort = 8080 defaultMembers = 3 defaultBaseServerID = 1000 // maxBaseServerID is the maximum safe value for BaseServerID calculated @@ -74,6 +76,13 @@ func (c *Cluster) EnsureDefaults() *Cluster { c.Spec.GroupPort = DefaultReplicationGroupPort } + if c.Spec.AgentCheckPort == 0 { + c.Spec.AgentCheckPort = DefaultAgentHealthCheckPort + } + + if c.Spec.AgentPromePort == 0 { + c.Spe.AgentPromePort = DefaultAgentPromePort + } return c } diff --git a/pkg/apis/mysql/v1alpha1/types.go b/pkg/apis/mysql/v1alpha1/types.go index ca390a04e..15774bf68 100644 --- a/pkg/apis/mysql/v1alpha1/types.go +++ b/pkg/apis/mysql/v1alpha1/types.go @@ -47,6 +47,10 @@ type ClusterSpec struct { HostNetwork bool `json:"hostNetwork,omitempty"` //replicationGroupPort GroupPort uint32 `json:"groupPort,omitempty"` + //agent healthcheck port + AgentCheckPort uint32 `json:"groupPort,omitempty"` + //agent prometheus port + AgentPromePort uint32 `json:"groupPort,omitempty"` // NodeSelector is a selector which must be true for the pod to fit on a node. // Selector which must match a node's labels for the pod to be scheduled on that node. // More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ diff --git a/pkg/controllers/cluster/manager/cluster_manager.go b/pkg/controllers/cluster/manager/cluster_manager.go index 79e6221b7..cbf722d5c 100644 --- a/pkg/controllers/cluster/manager/cluster_manager.go +++ b/pkg/controllers/cluster/manager/cluster_manager.go @@ -293,9 +293,16 @@ func (m *ClusterManager) handleInstanceNotFound(ctx context.Context, primaryAddr return false } + localAddress := fmt.Sprintf("%s:%s", m.Instance.Name(), os.Getenv("GROUP_PORT")) + groupSeeds := os.Getenv("REPLICATION_GROUP_SEEDS") + + glog.Infof("localAddress: %s, groupSeeds: %s", localAddress, groupSeeds) + if err := psh.AddInstanceToCluster(ctx, m.Instance.GetShellURI(), mysqlsh.Options{ "memberSslMode": "REQUIRED", "ipWhitelist": whitelistCIDR, + "localAddress": localAddress, + "groupSeeds": groupSeeds, }); err != nil { glog.Errorf("Failed to add to cluster: %v", err) return false diff --git a/pkg/options/agent/options.go b/pkg/options/agent/options.go index 77ccd7501..484513b02 100644 --- a/pkg/options/agent/options.go +++ b/pkg/options/agent/options.go @@ -65,8 +65,10 @@ func NewMySQLAgentOpts() *MySQLAgentOpts { } namespace := os.Getenv("POD_NAMESPACE") clusterName := os.Getenv("MYSQL_CLUSTER_NAME") + healthcheckPort := os.Getenv("AGENT_HEALTHCHECK_PORT") + glog.V(2).Infof("mysql-agent healthcheckPort: %d", healthcheckPort) return &MySQLAgentOpts{ - HealthcheckPort: DefaultMySQLAgentHeathcheckPort, + HealthcheckPort: healthcheckPort, Address: "0.0.0.0", Namespace: namespace, ClusterName: clusterName, diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index d982dc861..bd99d548c 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -133,6 +133,21 @@ func replicationGroupPortEnvVar(groupPort uint32) v1.EnvVar { Value: strconv.FormatUint(uint64(groupPort), 10), } } + +func agentHealthCheckPortEnvVar(agentCheckPort uint32) v1.EnvVar { + return v1.EnvVar{ + Name: "AGENT_HEALTHCHECK_PORT", + Value: strconv.FormatUint(uint64(agentCheckPort), 10), + } +} + +func agentPromePortEnvVar(agentPromePort uint32) v1.EnvVar { + return v1.EnvVar{ + Name: "AGENT_PROME_PORT", + Value: strconv.FormatUint(uint64(agentPromePort), 10), + } +} + // Returns the MySQL_ROOT_PASSWORD environment variable // If a user specifies a secret in the spec we use that // else we create a secret with a random password @@ -194,7 +209,7 @@ func mysqlServerContainer(cluster *v1alpha1.Cluster, mysqlServerImage string, ro "--datadir=/var/lib/mysql", "--user=mysql", "--gtid_mode=ON", - "--log-bin=$MY_POD_NAME", + "--log-bin=${MY_POD_NAME}-bin", "--binlog_checksum=NONE", "--enforce_gtid_consistency=ON", "--log-slave-updates=ON", @@ -294,6 +309,8 @@ func mysqlAgentContainer(cluster *v1alpha1.Cluster, mysqlAgentImage string, root replicationGroupSeedsEnvVar(replicationGroupSeeds), multiMasterEnvVar(cluster.Spec.MultiMaster), replicationGroupPortEnvVar(cluster.Spec.GroupPort), + agentHealthCheckPortEnvVar(cluster.Spec.AgentCheckPort), + agentPromePortEnvVar(cluster.Spec.AgentPromePort), rootPassword, { Name: "MY_POD_IP", From a2a05dd64e7857694b81bb2164c697a63e647cd4 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Tue, 18 Feb 2020 11:52:33 +0800 Subject: [PATCH 27/61] debug change port --- cmd/mysql-agent/app/mysql_agent.go | 6 ++++-- pkg/apis/mysql/v1alpha1/helpers.go | 2 +- pkg/options/agent/options.go | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/mysql-agent/app/mysql_agent.go b/cmd/mysql-agent/app/mysql_agent.go index 511ddec31..4f9a74c12 100644 --- a/cmd/mysql-agent/app/mysql_agent.go +++ b/cmd/mysql-agent/app/mysql_agent.go @@ -23,6 +23,7 @@ import ( "sync" "time" "os" + "fmt" "github.com/golang/glog" "github.com/heptiolabs/healthcheck" @@ -103,14 +104,15 @@ func Run(opts *agentopts.MySQLAgentOpts) error { } // agent prometheus port agentPromePort := os.Getenv("AGENT_PROME_PORT") - glog.Info("agent prometheus port: %d", agentPromePort) + promeMetricsEndpoint := fmt.Sprintf("0.0.0.0:%s", agentPromePort) + glog.Info("agent prometheus endpoint: %s", promeMetricsEndpoint) metrics.RegisterPodName(opts.Hostname) metrics.RegisterClusterName(manager.Instance.ClusterName) clustermgr.RegisterMetrics() backupcontroller.RegisterMetrics() restorecontroller.RegisterMetrics() http.Handle("/metrics", prometheus.Handler()) - go http.ListenAndServe(agentPromePort, nil) + go http.ListenAndServe(promeMetricsEndpoint, nil) // Block until local instance successfully initialised. for !manager.Sync(ctx) { diff --git a/pkg/apis/mysql/v1alpha1/helpers.go b/pkg/apis/mysql/v1alpha1/helpers.go index c71ce3fc9..5235e5ca9 100644 --- a/pkg/apis/mysql/v1alpha1/helpers.go +++ b/pkg/apis/mysql/v1alpha1/helpers.go @@ -81,7 +81,7 @@ func (c *Cluster) EnsureDefaults() *Cluster { } if c.Spec.AgentPromePort == 0 { - c.Spe.AgentPromePort = DefaultAgentPromePort + c.Spec.AgentPromePort = DefaultAgentPromePort } return c } diff --git a/pkg/options/agent/options.go b/pkg/options/agent/options.go index 484513b02..b669f62d0 100644 --- a/pkg/options/agent/options.go +++ b/pkg/options/agent/options.go @@ -65,7 +65,7 @@ func NewMySQLAgentOpts() *MySQLAgentOpts { } namespace := os.Getenv("POD_NAMESPACE") clusterName := os.Getenv("MYSQL_CLUSTER_NAME") - healthcheckPort := os.Getenv("AGENT_HEALTHCHECK_PORT") + healthcheckPort := int(os.Getenv("AGENT_HEALTHCHECK_PORT")) glog.V(2).Infof("mysql-agent healthcheckPort: %d", healthcheckPort) return &MySQLAgentOpts{ HealthcheckPort: healthcheckPort, From 56b10d3f8d56a79828db46d5d9c2ecd74cba61d1 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Tue, 18 Feb 2020 12:07:04 +0800 Subject: [PATCH 28/61] debug string to int --- pkg/options/agent/options.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/options/agent/options.go b/pkg/options/agent/options.go index b669f62d0..efacd4b01 100644 --- a/pkg/options/agent/options.go +++ b/pkg/options/agent/options.go @@ -18,6 +18,7 @@ import ( "fmt" "os" "time" + "strconv" "github.com/golang/glog" "github.com/spf13/pflag" @@ -65,7 +66,7 @@ func NewMySQLAgentOpts() *MySQLAgentOpts { } namespace := os.Getenv("POD_NAMESPACE") clusterName := os.Getenv("MYSQL_CLUSTER_NAME") - healthcheckPort := int(os.Getenv("AGENT_HEALTHCHECK_PORT")) + healthcheckPort := strconv.ParseInt(os.Getenv("AGENT_HEALTHCHECK_PORT"), 10, 32) glog.V(2).Infof("mysql-agent healthcheckPort: %d", healthcheckPort) return &MySQLAgentOpts{ HealthcheckPort: healthcheckPort, From 0e2157183392c40fb2d9999499aa93ca2cfbc015 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Tue, 18 Feb 2020 14:33:29 +0800 Subject: [PATCH 29/61] debug string to int --- pkg/options/agent/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/options/agent/options.go b/pkg/options/agent/options.go index efacd4b01..db9f9b13e 100644 --- a/pkg/options/agent/options.go +++ b/pkg/options/agent/options.go @@ -66,7 +66,7 @@ func NewMySQLAgentOpts() *MySQLAgentOpts { } namespace := os.Getenv("POD_NAMESPACE") clusterName := os.Getenv("MYSQL_CLUSTER_NAME") - healthcheckPort := strconv.ParseInt(os.Getenv("AGENT_HEALTHCHECK_PORT"), 10, 32) + healthcheckPort, _ := strconv.ParseInt(os.Getenv("AGENT_HEALTHCHECK_PORT"), 10, 32) glog.V(2).Infof("mysql-agent healthcheckPort: %d", healthcheckPort) return &MySQLAgentOpts{ HealthcheckPort: healthcheckPort, From 59e07fedbcfa803af4d6343e1d021a8b852fa365 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Tue, 18 Feb 2020 15:07:23 +0800 Subject: [PATCH 30/61] debug string to int --- pkg/options/agent/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/options/agent/options.go b/pkg/options/agent/options.go index db9f9b13e..7a8d85d42 100644 --- a/pkg/options/agent/options.go +++ b/pkg/options/agent/options.go @@ -66,7 +66,7 @@ func NewMySQLAgentOpts() *MySQLAgentOpts { } namespace := os.Getenv("POD_NAMESPACE") clusterName := os.Getenv("MYSQL_CLUSTER_NAME") - healthcheckPort, _ := strconv.ParseInt(os.Getenv("AGENT_HEALTHCHECK_PORT"), 10, 32) + healthcheckPort, _ := int32(strconv.ParseInt(os.Getenv("AGENT_HEALTHCHECK_PORT"), 10, 32)) glog.V(2).Infof("mysql-agent healthcheckPort: %d", healthcheckPort) return &MySQLAgentOpts{ HealthcheckPort: healthcheckPort, From bcd98959f3d1396ae36caf70bacd09d25b2aabc1 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Tue, 18 Feb 2020 15:08:41 +0800 Subject: [PATCH 31/61] debug string to int --- pkg/options/agent/options.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/options/agent/options.go b/pkg/options/agent/options.go index 7a8d85d42..8fd8ba580 100644 --- a/pkg/options/agent/options.go +++ b/pkg/options/agent/options.go @@ -66,10 +66,10 @@ func NewMySQLAgentOpts() *MySQLAgentOpts { } namespace := os.Getenv("POD_NAMESPACE") clusterName := os.Getenv("MYSQL_CLUSTER_NAME") - healthcheckPort, _ := int32(strconv.ParseInt(os.Getenv("AGENT_HEALTHCHECK_PORT"), 10, 32)) + healthcheckPort, _ := strconv.ParseInt(os.Getenv("AGENT_HEALTHCHECK_PORT"), 10, 32) glog.V(2).Infof("mysql-agent healthcheckPort: %d", healthcheckPort) return &MySQLAgentOpts{ - HealthcheckPort: healthcheckPort, + HealthcheckPort: int32(healthcheckPort), Address: "0.0.0.0", Namespace: namespace, ClusterName: clusterName, From 7a1e91825bc842ff8013edd566a5fabfe8b44f68 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Tue, 18 Feb 2020 15:18:30 +0800 Subject: [PATCH 32/61] debug cluster struct --- pkg/apis/mysql/v1alpha1/types.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/apis/mysql/v1alpha1/types.go b/pkg/apis/mysql/v1alpha1/types.go index 15774bf68..b0c3d1698 100644 --- a/pkg/apis/mysql/v1alpha1/types.go +++ b/pkg/apis/mysql/v1alpha1/types.go @@ -48,9 +48,9 @@ type ClusterSpec struct { //replicationGroupPort GroupPort uint32 `json:"groupPort,omitempty"` //agent healthcheck port - AgentCheckPort uint32 `json:"groupPort,omitempty"` + AgentCheckPort uint32 `json:"agentCheckPort,omitempty"` //agent prometheus port - AgentPromePort uint32 `json:"groupPort,omitempty"` + AgentPromePort uint32 `json:"agentPromePort,omitempty"` // NodeSelector is a selector which must be true for the pod to fit on a node. // Selector which must match a node's labels for the pod to be scheduled on that node. // More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ From ffb227db237a99190b23733a3eaa17cc2e089607 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Tue, 18 Feb 2020 15:54:15 +0800 Subject: [PATCH 33/61] debug liveprobe readiness --- pkg/resources/statefulsets/statefulset.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index bd99d548c..f15d2cc80 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -297,6 +297,8 @@ func mysqlAgentContainer(cluster *v1alpha1.Cluster, mysqlAgentImage string, root resourceLimits = *cluster.Spec.Resources.Agent } + agentPromePort := strconv.FormatUint(uint64(cluster.Spec.AgentPromePort), 10) + return v1.Container{ Name: MySQLAgentName, Image: fmt.Sprintf("%s:%s", mysqlAgentImage, agentVersion), @@ -333,7 +335,7 @@ func mysqlAgentContainer(cluster *v1alpha1.Cluster, mysqlAgentImage string, root Handler: v1.Handler{ HTTPGet: &v1.HTTPGetAction{ Path: "/live", - Port: intstr.FromInt(int(agentopts.DefaultMySQLAgentHeathcheckPort)), + Port: agentPromePort, }, }, }, @@ -341,7 +343,7 @@ func mysqlAgentContainer(cluster *v1alpha1.Cluster, mysqlAgentImage string, root Handler: v1.Handler{ HTTPGet: &v1.HTTPGetAction{ Path: "/ready", - Port: intstr.FromInt(int(agentopts.DefaultMySQLAgentHeathcheckPort)), + Port: agentPromePort, }, }, }, From b46886679ddfb333e360e7bf357c1c6e7fb22a9a Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Tue, 18 Feb 2020 15:55:57 +0800 Subject: [PATCH 34/61] debug liveprobe readiness --- pkg/resources/statefulsets/statefulset.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index f15d2cc80..b7eaeee9d 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -297,8 +297,6 @@ func mysqlAgentContainer(cluster *v1alpha1.Cluster, mysqlAgentImage string, root resourceLimits = *cluster.Spec.Resources.Agent } - agentPromePort := strconv.FormatUint(uint64(cluster.Spec.AgentPromePort), 10) - return v1.Container{ Name: MySQLAgentName, Image: fmt.Sprintf("%s:%s", mysqlAgentImage, agentVersion), @@ -335,7 +333,7 @@ func mysqlAgentContainer(cluster *v1alpha1.Cluster, mysqlAgentImage string, root Handler: v1.Handler{ HTTPGet: &v1.HTTPGetAction{ Path: "/live", - Port: agentPromePort, + Port: intstr.FromInt(int(cluster.Spec.agentPromePort)), }, }, }, @@ -343,7 +341,7 @@ func mysqlAgentContainer(cluster *v1alpha1.Cluster, mysqlAgentImage string, root Handler: v1.Handler{ HTTPGet: &v1.HTTPGetAction{ Path: "/ready", - Port: agentPromePort, + Port: intstr.FromInt(int(cluster.Spec.agentPromePort)), }, }, }, From f6e85d21127ff1efd9111ee37bfdd44004756a29 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Tue, 18 Feb 2020 15:57:30 +0800 Subject: [PATCH 35/61] debug liveprobe readiness --- pkg/resources/statefulsets/statefulset.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index b7eaeee9d..21b296b54 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -29,7 +29,7 @@ import ( "github.com/oracle/mysql-operator/pkg/apis/mysql/v1alpha1" "github.com/oracle/mysql-operator/pkg/constants" - agentopts "github.com/oracle/mysql-operator/pkg/options/agent" + //agentopts "github.com/oracle/mysql-operator/pkg/options/agent" operatoropts "github.com/oracle/mysql-operator/pkg/options/operator" "github.com/oracle/mysql-operator/pkg/resources/secrets" "github.com/oracle/mysql-operator/pkg/version" @@ -333,7 +333,7 @@ func mysqlAgentContainer(cluster *v1alpha1.Cluster, mysqlAgentImage string, root Handler: v1.Handler{ HTTPGet: &v1.HTTPGetAction{ Path: "/live", - Port: intstr.FromInt(int(cluster.Spec.agentPromePort)), + Port: intstr.FromInt(int(cluster.Spec.AgentPromePort)), }, }, }, @@ -341,7 +341,7 @@ func mysqlAgentContainer(cluster *v1alpha1.Cluster, mysqlAgentImage string, root Handler: v1.Handler{ HTTPGet: &v1.HTTPGetAction{ Path: "/ready", - Port: intstr.FromInt(int(cluster.Spec.agentPromePort)), + Port: intstr.FromInt(int(cluster.Spec.AgentPromePort)), }, }, }, From cc7c2394ef66e74c868551098dc4c08faa1e91b9 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Tue, 18 Feb 2020 16:21:27 +0800 Subject: [PATCH 36/61] debug liveprobe readiness --- pkg/resources/statefulsets/statefulset.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index 21b296b54..573cff883 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -333,7 +333,7 @@ func mysqlAgentContainer(cluster *v1alpha1.Cluster, mysqlAgentImage string, root Handler: v1.Handler{ HTTPGet: &v1.HTTPGetAction{ Path: "/live", - Port: intstr.FromInt(int(cluster.Spec.AgentPromePort)), + Port: intstr.FromInt(int(cluster.Spec.AgentCheckPort)), }, }, }, @@ -341,7 +341,7 @@ func mysqlAgentContainer(cluster *v1alpha1.Cluster, mysqlAgentImage string, root Handler: v1.Handler{ HTTPGet: &v1.HTTPGetAction{ Path: "/ready", - Port: intstr.FromInt(int(cluster.Spec.AgentPromePort)), + Port: intstr.FromInt(int(cluster.Spec.AgentCheckPort)), }, }, }, From 62eaa20edcffa56a747a1e9593dfd9303189c9a9 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Tue, 18 Feb 2020 16:29:31 +0800 Subject: [PATCH 37/61] debug liveprobe readiness --- cmd/mysql-agent/app/mysql_agent.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/mysql-agent/app/mysql_agent.go b/cmd/mysql-agent/app/mysql_agent.go index 4f9a74c12..7cb76f4d4 100644 --- a/cmd/mysql-agent/app/mysql_agent.go +++ b/cmd/mysql-agent/app/mysql_agent.go @@ -104,8 +104,8 @@ func Run(opts *agentopts.MySQLAgentOpts) error { } // agent prometheus port agentPromePort := os.Getenv("AGENT_PROME_PORT") - promeMetricsEndpoint := fmt.Sprintf("0.0.0.0:%s", agentPromePort) - glog.Info("agent prometheus endpoint: %s", promeMetricsEndpoint) + promeMetricsEndpoint := fmt.Sprintf("0.0.0.0: %s", agentPromePort) + glog.Info("agent prometheus endpoint: ", promeMetricsEndpoint) metrics.RegisterPodName(opts.Hostname) metrics.RegisterClusterName(manager.Instance.ClusterName) clustermgr.RegisterMetrics() From d8df60e39429cd279f5a086fdbc7c06471012882 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Tue, 18 Feb 2020 17:09:06 +0800 Subject: [PATCH 38/61] debug database not yet --- pkg/apis/mysql/v1alpha1/helpers.go | 5 +++++ pkg/apis/mysql/v1alpha1/types.go | 2 ++ pkg/controllers/cluster/manager/innodb_cluster.go | 1 + pkg/resources/statefulsets/statefulset.go | 8 ++++++++ 4 files changed, 16 insertions(+) diff --git a/pkg/apis/mysql/v1alpha1/helpers.go b/pkg/apis/mysql/v1alpha1/helpers.go index 5235e5ca9..ab557d7de 100644 --- a/pkg/apis/mysql/v1alpha1/helpers.go +++ b/pkg/apis/mysql/v1alpha1/helpers.go @@ -25,6 +25,7 @@ const ( DefaultReplicationGroupPort = 33061 DefaultAgentHealthCheckPort = 10512 DefaultAgentPromePort = 8080 + DefaultMysqlPort = 3306 defaultMembers = 3 defaultBaseServerID = 1000 // maxBaseServerID is the maximum safe value for BaseServerID calculated @@ -83,6 +84,10 @@ func (c *Cluster) EnsureDefaults() *Cluster { if c.Spec.AgentPromePort == 0 { c.Spec.AgentPromePort = DefaultAgentPromePort } + + if c.Spec.MysqlPort == 0 { + c.Spec.MysqlPort = DefaultMysqlPort + } return c } diff --git a/pkg/apis/mysql/v1alpha1/types.go b/pkg/apis/mysql/v1alpha1/types.go index b0c3d1698..3c8e8401b 100644 --- a/pkg/apis/mysql/v1alpha1/types.go +++ b/pkg/apis/mysql/v1alpha1/types.go @@ -51,6 +51,8 @@ type ClusterSpec struct { AgentCheckPort uint32 `json:"agentCheckPort,omitempty"` //agent prometheus port AgentPromePort uint32 `json:"agentPromePort,omitempty"` + //mysql_port + MysqlPort uint32 `json:"mysqlPort,omitempty"` // NodeSelector is a selector which must be true for the pod to fit on a node. // Selector which must match a node's labels for the pod to be scheduled on that node. // More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ diff --git a/pkg/controllers/cluster/manager/innodb_cluster.go b/pkg/controllers/cluster/manager/innodb_cluster.go index 9f3b1b823..289382a06 100644 --- a/pkg/controllers/cluster/manager/innodb_cluster.go +++ b/pkg/controllers/cluster/manager/innodb_cluster.go @@ -46,6 +46,7 @@ func isDatabaseRunning(ctx context.Context) bool { "mysqladmin", "--protocol", "tcp", "-u", "root", + os.ExpandEnv("-P$MYSQL_PORT"), os.ExpandEnv("-p$MYSQL_ROOT_PASSWORD"), "status", ).Run() diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index 573cff883..adc24a73b 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -148,6 +148,13 @@ func agentPromePortEnvVar(agentPromePort uint32) v1.EnvVar { } } +func mysqlPortEnvVar(mysqlPort uint32) v1.EnvVar { + return v1.EnvVar{ + Name: "MYSQL_PORT", + Value: strconv.FormatUint(uint64(mysqlPort), 10), + } +} + // Returns the MySQL_ROOT_PASSWORD environment variable // If a user specifies a secret in the spec we use that // else we create a secret with a random password @@ -311,6 +318,7 @@ func mysqlAgentContainer(cluster *v1alpha1.Cluster, mysqlAgentImage string, root replicationGroupPortEnvVar(cluster.Spec.GroupPort), agentHealthCheckPortEnvVar(cluster.Spec.AgentCheckPort), agentPromePortEnvVar(cluster.Spec.AgentPromePort), + mysqlPortEnvVar(cluster.Spec.MysqlPort), rootPassword, { Name: "MY_POD_IP", From 07549d90f56f8b49dc6101d53472d67d5226f50c Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Tue, 18 Feb 2020 17:36:02 +0800 Subject: [PATCH 39/61] debug database not yet --- pkg/cluster/instance.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/cluster/instance.go b/pkg/cluster/instance.go index 53d523ac4..424aceac5 100644 --- a/pkg/cluster/instance.go +++ b/pkg/cluster/instance.go @@ -24,7 +24,7 @@ import ( "github.com/pkg/errors" - "github.com/oracle/mysql-operator/pkg/cluster/innodb" + //"github.com/oracle/mysql-operator/pkg/cluster/innodb" ) // Instance represents the local MySQL instance. @@ -68,12 +68,13 @@ func NewLocalInstance() (*Instance, error) { } name, ordinal := GetParentNameAndOrdinal(pod_name) multiMaster, _ := strconv.ParseBool(os.Getenv("MYSQL_CLUSTER_MULTI_MASTER")) + mysqlPort, _ := strconv.ParseInt(os.Getenv("MYSQL_PORT"), 10, 32) return &Instance{ Namespace: os.Getenv("POD_NAMESPACE"), ClusterName: os.Getenv("MYSQL_CLUSTER_NAME"), ParentName: name, Ordinal: ordinal, - Port: innodb.MySQLDBPort, + Port: mysqlPort, MultiMaster: multiMaster, IP: net.ParseIP(os.Getenv("MY_POD_IP")), }, nil @@ -90,12 +91,13 @@ func NewInstanceFromGroupSeed(seed string) (*Instance, error) { // MySQLDB port not its group replication port. parentName, ordinal := GetParentNameAndOrdinal(podName) multiMaster, _ := strconv.ParseBool(os.Getenv("MYSQL_CLUSTER_MULTI_MASTER")) + mysqlPort, _ := strconv.ParseInt(os.Getenv("MYSQL_PORT"), 10, 32) return &Instance{ ClusterName: os.Getenv("MYSQL_CLUSTER_NAME"), Namespace: os.Getenv("POD_NAMESPACE"), ParentName: parentName, Ordinal: ordinal, - Port: innodb.MySQLDBPort, + Port: mysqlPort, MultiMaster: multiMaster, }, nil } From 0bd80a85c41f23037b4f3cbeeda498c2e9005de7 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Tue, 18 Feb 2020 17:38:01 +0800 Subject: [PATCH 40/61] debug database not yet --- pkg/cluster/instance.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cluster/instance.go b/pkg/cluster/instance.go index 424aceac5..3ced3bcbb 100644 --- a/pkg/cluster/instance.go +++ b/pkg/cluster/instance.go @@ -74,7 +74,7 @@ func NewLocalInstance() (*Instance, error) { ClusterName: os.Getenv("MYSQL_CLUSTER_NAME"), ParentName: name, Ordinal: ordinal, - Port: mysqlPort, + Port: int(mysqlPort), MultiMaster: multiMaster, IP: net.ParseIP(os.Getenv("MY_POD_IP")), }, nil @@ -97,7 +97,7 @@ func NewInstanceFromGroupSeed(seed string) (*Instance, error) { Namespace: os.Getenv("POD_NAMESPACE"), ParentName: parentName, Ordinal: ordinal, - Port: mysqlPort, + Port: int(mysqlPort), MultiMaster: multiMaster, }, nil } From 184b5b19a7ae626482c7013332b74c598e60f8b7 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Tue, 18 Feb 2020 17:55:42 +0800 Subject: [PATCH 41/61] debug statefulset port --- pkg/resources/statefulsets/statefulset.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index adc24a73b..f4463f19f 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -263,12 +263,13 @@ func mysqlServerContainer(cluster *v1alpha1.Cluster, mysqlServerImage string, ro ImagePullPolicy: v1.PullAlways, Ports: []v1.ContainerPort{ { - ContainerPort: 3306, + ContainerPort: cluster.Spec.MysqlPort, }, }, VolumeMounts: volumeMounts(cluster), Command: []string{"/bin/bash", "-ecx", cmd}, Env: []v1.EnvVar{ + //mysqlPortEnvVar(cluster.Spec.MysqlPort), rootPassword, { Name: "MYSQL_ROOT_HOST", From 7ec556e7c9f5466f4703f0f2d435a56fcb047883 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Tue, 18 Feb 2020 18:01:30 +0800 Subject: [PATCH 42/61] debug statefulset port --- pkg/resources/statefulsets/statefulset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index f4463f19f..b18e3ebff 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -457,7 +457,7 @@ func NewForCluster(cluster *v1alpha1.Cluster, images operatoropts.Images, servic Labels: podLabels, Annotations: map[string]string{ "prometheus.io/scrape": "true", - "prometheus.io/port": "8080", + "prometheus.io/port": cluster.Spec.AgentPromePort, }, }, Spec: v1.PodSpec{ From 77a6e90074e39c4e2d3c486b2e67671a0eaef850 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Tue, 18 Feb 2020 18:04:17 +0800 Subject: [PATCH 43/61] debug statefulset port --- pkg/resources/statefulsets/statefulset.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index b18e3ebff..52ec09fff 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -263,7 +263,7 @@ func mysqlServerContainer(cluster *v1alpha1.Cluster, mysqlServerImage string, ro ImagePullPolicy: v1.PullAlways, Ports: []v1.ContainerPort{ { - ContainerPort: cluster.Spec.MysqlPort, + ContainerPort: int32(cluster.Spec.MysqlPort), }, }, VolumeMounts: volumeMounts(cluster), @@ -457,7 +457,7 @@ func NewForCluster(cluster *v1alpha1.Cluster, images operatoropts.Images, servic Labels: podLabels, Annotations: map[string]string{ "prometheus.io/scrape": "true", - "prometheus.io/port": cluster.Spec.AgentPromePort, + "prometheus.io/port": int32(cluster.Spec.AgentPromePort), }, }, Spec: v1.PodSpec{ From 6dc2503017ee1bacb8c6836c003a1ab084a36b80 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Tue, 18 Feb 2020 18:07:39 +0800 Subject: [PATCH 44/61] debug statefulset port --- pkg/resources/statefulsets/statefulset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index 52ec09fff..bb104f67c 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -457,7 +457,7 @@ func NewForCluster(cluster *v1alpha1.Cluster, images operatoropts.Images, servic Labels: podLabels, Annotations: map[string]string{ "prometheus.io/scrape": "true", - "prometheus.io/port": int32(cluster.Spec.AgentPromePort), + "prometheus.io/port": strconv.FormatUint(uint64(cluster.Spec.AgentPromePort), 10), }, }, Spec: v1.PodSpec{ From 3fb933739e7e79b26873a1610e53a8baf5eac50c Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Tue, 18 Feb 2020 18:36:04 +0800 Subject: [PATCH 45/61] debug statefulset port --- pkg/cluster/innodb/innodb.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/cluster/innodb/innodb.go b/pkg/cluster/innodb/innodb.go index eb2103646..316658849 100644 --- a/pkg/cluster/innodb/innodb.go +++ b/pkg/cluster/innodb/innodb.go @@ -17,6 +17,7 @@ package innodb import ( "fmt" "net" + "os" ) // DefaultClusterName is the default name assigned to InnoDB clusters created by @@ -123,7 +124,7 @@ func (s *ClusterStatus) GetInstanceStatus(name string) InstanceStatus { if s.DefaultReplicaSet.Topology == nil { return InstanceStatusNotFound } - if is, ok := s.DefaultReplicaSet.Topology[fmt.Sprintf("%s:%d", name, MySQLDBPort)]; ok { + if is, ok := s.DefaultReplicaSet.Topology[fmt.Sprintf("%s:%s", name, os.Getenv("MYSQL_PORT"))]; ok { return is.Status } return InstanceStatusNotFound From d97125425bff0a8415369074d7f185436f8da5b0 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Thu, 20 Feb 2020 10:59:34 +0800 Subject: [PATCH 46/61] update mysqlsh group port default --- .../cluster/manager/cluster_manager.go | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/pkg/controllers/cluster/manager/cluster_manager.go b/pkg/controllers/cluster/manager/cluster_manager.go index cbf722d5c..3432818a1 100644 --- a/pkg/controllers/cluster/manager/cluster_manager.go +++ b/pkg/controllers/cluster/manager/cluster_manager.go @@ -292,17 +292,17 @@ func (m *ClusterManager) handleInstanceNotFound(ctx context.Context, primaryAddr glog.Errorf("Getting CIDR to whitelist for GR: %v", err) return false } + //use deault mysqlPort + 1 + //localAddress := fmt.Sprintf("%s:%s", m.Instance.Name(), os.Getenv("GROUP_PORT")) + //groupSeeds := os.Getenv("REPLICATION_GROUP_SEEDS") - localAddress := fmt.Sprintf("%s:%s", m.Instance.Name(), os.Getenv("GROUP_PORT")) - groupSeeds := os.Getenv("REPLICATION_GROUP_SEEDS") - - glog.Infof("localAddress: %s, groupSeeds: %s", localAddress, groupSeeds) + //glog.Infof("localAddress: %s, groupSeeds: %s", localAddress, groupSeeds) if err := psh.AddInstanceToCluster(ctx, m.Instance.GetShellURI(), mysqlsh.Options{ "memberSslMode": "REQUIRED", "ipWhitelist": whitelistCIDR, - "localAddress": localAddress, - "groupSeeds": groupSeeds, + // "localAddress": localAddress, + // "groupSeeds": groupSeeds, }); err != nil { glog.Errorf("Failed to add to cluster: %v", err) return false @@ -333,16 +333,17 @@ func (m *ClusterManager) createCluster(ctx context.Context) (*innodb.ClusterStat return nil, errors.Wrap(err, "getting CIDR to whitelist for GR") } - localAddress := fmt.Sprintf("%s:%s", m.Instance.Name(), os.Getenv("GROUP_PORT")) - groupSeeds := os.Getenv("REPLICATION_GROUP_SEEDS") + // use deault mysql_port + 1 + //localAddress := fmt.Sprintf("%s:%s", m.Instance.Name(), os.Getenv("GROUP_PORT")) + //groupSeeds := os.Getenv("REPLICATION_GROUP_SEEDS") - glog.Infof("localAddress: %s, groupSeeds: %s", localAddress, groupSeeds) + //glog.Infof("localAddress: %s, groupSeeds: %s", localAddress, groupSeeds) opts := mysqlsh.Options{ "memberSslMode": "REQUIRED", "ipWhitelist": whitelistCIDR, - "localAddress": localAddress, - "groupSeeds": groupSeeds, + // "localAddress": localAddress, + // "groupSeeds": groupSeeds, } if m.Instance.MultiMaster { opts["force"] = "True" From 8ecea2676f9484e45369693f7d4ee97f481c5905 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Thu, 20 Feb 2020 11:23:51 +0800 Subject: [PATCH 47/61] debug cluster_manager --- pkg/controllers/cluster/manager/cluster_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controllers/cluster/manager/cluster_manager.go b/pkg/controllers/cluster/manager/cluster_manager.go index 3432818a1..7733a4cc9 100644 --- a/pkg/controllers/cluster/manager/cluster_manager.go +++ b/pkg/controllers/cluster/manager/cluster_manager.go @@ -17,7 +17,7 @@ package manager import ( "context" "fmt" - "os" + //"os" "strings" "time" From 7736b2d6369314cf173b8c1c8798e57bd35e86b9 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Fri, 21 Feb 2020 10:09:05 +0800 Subject: [PATCH 48/61] debug headless service port --- pkg/resources/services/service.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/resources/services/service.go b/pkg/resources/services/service.go index 9aa177a3b..11d6983f2 100644 --- a/pkg/resources/services/service.go +++ b/pkg/resources/services/service.go @@ -16,6 +16,8 @@ package services import ( corev1 "k8s.io/api/core/v1" + "os" + "strconv" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -25,7 +27,8 @@ import ( // NewForCluster will return a new headless Kubernetes service for a MySQL cluster func NewForCluster(cluster *v1alpha1.Cluster) *corev1.Service { - mysqlPort := corev1.ServicePort{Port: 3306} + port := strconv.ParseInt(os.Getenv("MYSQL_PORT"), 10, 32) + mysqlPort := corev1.ServicePort{Port: int32(port)} svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{constants.ClusterLabel: cluster.Name}, From 4c282a425db4bd08619c40afd385fc7c19102f23 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Fri, 21 Feb 2020 10:12:32 +0800 Subject: [PATCH 49/61] debug headless service port --- pkg/resources/services/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/resources/services/service.go b/pkg/resources/services/service.go index 11d6983f2..cf2707394 100644 --- a/pkg/resources/services/service.go +++ b/pkg/resources/services/service.go @@ -27,7 +27,7 @@ import ( // NewForCluster will return a new headless Kubernetes service for a MySQL cluster func NewForCluster(cluster *v1alpha1.Cluster) *corev1.Service { - port := strconv.ParseInt(os.Getenv("MYSQL_PORT"), 10, 32) + port, _ := strconv.ParseInt(os.Getenv("MYSQL_PORT"), 10, 32) mysqlPort := corev1.ServicePort{Port: int32(port)} svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ From c48e20f22d0fb2f8162050f5facff1a42f621394 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Fri, 21 Feb 2020 11:25:11 +0800 Subject: [PATCH 50/61] debug headless service port --- pkg/resources/services/service.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/resources/services/service.go b/pkg/resources/services/service.go index cf2707394..e31a84d07 100644 --- a/pkg/resources/services/service.go +++ b/pkg/resources/services/service.go @@ -27,8 +27,7 @@ import ( // NewForCluster will return a new headless Kubernetes service for a MySQL cluster func NewForCluster(cluster *v1alpha1.Cluster) *corev1.Service { - port, _ := strconv.ParseInt(os.Getenv("MYSQL_PORT"), 10, 32) - mysqlPort := corev1.ServicePort{Port: int32(port)} + mysqlPort := corev1.ServicePort{Port: int32(cluster.Spec.MysqlPort)} svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{constants.ClusterLabel: cluster.Name}, From 4102c645a1c168b491701e5dd19054525f69d5e2 Mon Sep 17 00:00:00 2001 From: jiachu <1156083606@163.com> Date: Fri, 21 Feb 2020 11:26:26 +0800 Subject: [PATCH 51/61] debug headless service port --- pkg/resources/services/service.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/resources/services/service.go b/pkg/resources/services/service.go index e31a84d07..36289d00d 100644 --- a/pkg/resources/services/service.go +++ b/pkg/resources/services/service.go @@ -16,8 +16,6 @@ package services import ( corev1 "k8s.io/api/core/v1" - "os" - "strconv" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" From c9ce7523236265787a848d3e1ddadd42adbcee72 Mon Sep 17 00:00:00 2001 From: jiachu Date: Thu, 27 Feb 2020 09:12:20 +0800 Subject: [PATCH 52/61] debug --- pkg/controllers/cluster/manager/cluster_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controllers/cluster/manager/cluster_manager.go b/pkg/controllers/cluster/manager/cluster_manager.go index 7733a4cc9..e1e061342 100644 --- a/pkg/controllers/cluster/manager/cluster_manager.go +++ b/pkg/controllers/cluster/manager/cluster_manager.go @@ -37,7 +37,7 @@ import ( "github.com/oracle/mysql-operator/pkg/util/mysqlsh" ) -const pollingIntervalSeconds = 15 +const pollingIntervalSeconds = 60 // ClusterManager manages the local MySQL instance's membership of an InnoDB cluster. type ClusterManager struct { From c0407e3a74127b34b3858e931220067d41ed6796 Mon Sep 17 00:00:00 2001 From: jiachu Date: Thu, 27 Feb 2020 10:10:56 +0800 Subject: [PATCH 53/61] change agent run interval --- pkg/apis/mysql/v1alpha1/helpers.go | 4 ++++ pkg/apis/mysql/v1alpha1/types.go | 2 ++ pkg/controllers/cluster/manager/cluster_manager.go | 8 ++++++-- pkg/resources/statefulsets/statefulset.go | 8 ++++++++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/pkg/apis/mysql/v1alpha1/helpers.go b/pkg/apis/mysql/v1alpha1/helpers.go index ab557d7de..c5a0bc18d 100644 --- a/pkg/apis/mysql/v1alpha1/helpers.go +++ b/pkg/apis/mysql/v1alpha1/helpers.go @@ -28,6 +28,7 @@ const ( DefaultMysqlPort = 3306 defaultMembers = 3 defaultBaseServerID = 1000 + defaultAgentIntervalTime = 15 // maxBaseServerID is the maximum safe value for BaseServerID calculated // as max MySQL server_id value - max Replication Group size. maxBaseServerID uint32 = 4294967295 - 9 @@ -88,6 +89,9 @@ func (c *Cluster) EnsureDefaults() *Cluster { if c.Spec.MysqlPort == 0 { c.Spec.MysqlPort = DefaultMysqlPort } + if c.Spec.AgentIntervalTime == 0 { + c.Spec.AgentIntervalTime = defaultAgentIntervalTime + } return c } diff --git a/pkg/apis/mysql/v1alpha1/types.go b/pkg/apis/mysql/v1alpha1/types.go index 3c8e8401b..2fb56cf95 100644 --- a/pkg/apis/mysql/v1alpha1/types.go +++ b/pkg/apis/mysql/v1alpha1/types.go @@ -53,6 +53,8 @@ type ClusterSpec struct { AgentPromePort uint32 `json:"agentPromePort,omitempty"` //mysql_port MysqlPort uint32 `json:"mysqlPort,omitempty"` + //agent execute interval + AgentIntervalTime `json:"agentIntervalTime,omitempty"` // NodeSelector is a selector which must be true for the pod to fit on a node. // Selector which must match a node's labels for the pod to be scheduled on that node. // More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ diff --git a/pkg/controllers/cluster/manager/cluster_manager.go b/pkg/controllers/cluster/manager/cluster_manager.go index e1e061342..7e6ff0d0f 100644 --- a/pkg/controllers/cluster/manager/cluster_manager.go +++ b/pkg/controllers/cluster/manager/cluster_manager.go @@ -17,8 +17,9 @@ package manager import ( "context" "fmt" - //"os" + "os" "strings" + "strconv" "time" "github.com/golang/glog" @@ -374,7 +375,10 @@ func (m *ClusterManager) rebootFromOutage(ctx context.Context) (*innodb.ClusterS // Run runs the ClusterManager controller. // NOTE: ctx is not currently used for cancellation by caller (the stopCh is). func (m *ClusterManager) Run(ctx context.Context) { - wait.Until(func() { m.Sync(ctx) }, time.Second*pollingIntervalSeconds, ctx.Done()) + interval_time, _ := stronv.ParseUint(os.Getenv("AGENT_INTERVAL"), 10 32) + glog.Info("***agent run interval: %d", interval_time) + //wait.Until(func() { m.Sync(ctx) }, time.Second*pollingIntervalSeconds, ctx.Done()) + wait.Until(func() { m.Sync(ctx) }, time.Second*interval_time, ctx.Done()) <-ctx.Done() diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index bb104f67c..a089e43f5 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -155,6 +155,13 @@ func mysqlPortEnvVar(mysqlPort uint32) v1.EnvVar { } } +func numberEnvVar(field string, num uint32) v1.EnvVar { + return v1.EnvVar{ + Name: field, + Value: strconv.FormatUint(uint64(num), 10), + } +} + // Returns the MySQL_ROOT_PASSWORD environment variable // If a user specifies a secret in the spec we use that // else we create a secret with a random password @@ -320,6 +327,7 @@ func mysqlAgentContainer(cluster *v1alpha1.Cluster, mysqlAgentImage string, root agentHealthCheckPortEnvVar(cluster.Spec.AgentCheckPort), agentPromePortEnvVar(cluster.Spec.AgentPromePort), mysqlPortEnvVar(cluster.Spec.MysqlPort), + numberEnvVar("AGENT_INTERVAL", cluster.Spec.AgentIntervalTime), rootPassword, { Name: "MY_POD_IP", From 1abb5e9cc7585327fab9c81879fd2837748e627c Mon Sep 17 00:00:00 2001 From: jiachu Date: Thu, 27 Feb 2020 10:14:04 +0800 Subject: [PATCH 54/61] change agent run interval --- pkg/apis/mysql/v1alpha1/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apis/mysql/v1alpha1/types.go b/pkg/apis/mysql/v1alpha1/types.go index 2fb56cf95..62f358cac 100644 --- a/pkg/apis/mysql/v1alpha1/types.go +++ b/pkg/apis/mysql/v1alpha1/types.go @@ -54,7 +54,7 @@ type ClusterSpec struct { //mysql_port MysqlPort uint32 `json:"mysqlPort,omitempty"` //agent execute interval - AgentIntervalTime `json:"agentIntervalTime,omitempty"` + AgentIntervalTime uint32 `json:"agentIntervalTime,omitempty"` // NodeSelector is a selector which must be true for the pod to fit on a node. // Selector which must match a node's labels for the pod to be scheduled on that node. // More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ From cbd24a589756f4b0baa66ce57b300d67da05bebf Mon Sep 17 00:00:00 2001 From: jiachu Date: Thu, 27 Feb 2020 10:15:53 +0800 Subject: [PATCH 55/61] change agent run interval --- pkg/controllers/cluster/manager/cluster_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controllers/cluster/manager/cluster_manager.go b/pkg/controllers/cluster/manager/cluster_manager.go index 7e6ff0d0f..80a265dbb 100644 --- a/pkg/controllers/cluster/manager/cluster_manager.go +++ b/pkg/controllers/cluster/manager/cluster_manager.go @@ -375,7 +375,7 @@ func (m *ClusterManager) rebootFromOutage(ctx context.Context) (*innodb.ClusterS // Run runs the ClusterManager controller. // NOTE: ctx is not currently used for cancellation by caller (the stopCh is). func (m *ClusterManager) Run(ctx context.Context) { - interval_time, _ := stronv.ParseUint(os.Getenv("AGENT_INTERVAL"), 10 32) + interval_time, _ := stronv.ParseUint(os.Getenv("AGENT_INTERVAL"), 10, 32) glog.Info("***agent run interval: %d", interval_time) //wait.Until(func() { m.Sync(ctx) }, time.Second*pollingIntervalSeconds, ctx.Done()) wait.Until(func() { m.Sync(ctx) }, time.Second*interval_time, ctx.Done()) From df736a9331a2e43724d0f183e347797fe34e4b92 Mon Sep 17 00:00:00 2001 From: jiachu Date: Thu, 27 Feb 2020 10:19:38 +0800 Subject: [PATCH 56/61] change agent run interval --- pkg/controllers/cluster/manager/cluster_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controllers/cluster/manager/cluster_manager.go b/pkg/controllers/cluster/manager/cluster_manager.go index 80a265dbb..0fc8cec5c 100644 --- a/pkg/controllers/cluster/manager/cluster_manager.go +++ b/pkg/controllers/cluster/manager/cluster_manager.go @@ -375,7 +375,7 @@ func (m *ClusterManager) rebootFromOutage(ctx context.Context) (*innodb.ClusterS // Run runs the ClusterManager controller. // NOTE: ctx is not currently used for cancellation by caller (the stopCh is). func (m *ClusterManager) Run(ctx context.Context) { - interval_time, _ := stronv.ParseUint(os.Getenv("AGENT_INTERVAL"), 10, 32) + interval_time, _ := strconv.ParseUint(os.Getenv("AGENT_INTERVAL"), 10, 32) glog.Info("***agent run interval: %d", interval_time) //wait.Until(func() { m.Sync(ctx) }, time.Second*pollingIntervalSeconds, ctx.Done()) wait.Until(func() { m.Sync(ctx) }, time.Second*interval_time, ctx.Done()) From a03ee9311c583e6a1b08c87994879ab9f18bb0de Mon Sep 17 00:00:00 2001 From: jiachu Date: Thu, 27 Feb 2020 10:22:04 +0800 Subject: [PATCH 57/61] change agent run interval --- pkg/controllers/cluster/manager/cluster_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controllers/cluster/manager/cluster_manager.go b/pkg/controllers/cluster/manager/cluster_manager.go index 0fc8cec5c..63d96728e 100644 --- a/pkg/controllers/cluster/manager/cluster_manager.go +++ b/pkg/controllers/cluster/manager/cluster_manager.go @@ -375,7 +375,7 @@ func (m *ClusterManager) rebootFromOutage(ctx context.Context) (*innodb.ClusterS // Run runs the ClusterManager controller. // NOTE: ctx is not currently used for cancellation by caller (the stopCh is). func (m *ClusterManager) Run(ctx context.Context) { - interval_time, _ := strconv.ParseUint(os.Getenv("AGENT_INTERVAL"), 10, 32) + interval_time, _ := strconv.ParseUint(os.Getenv("AGENT_INTERVAL"), 10, 64) glog.Info("***agent run interval: %d", interval_time) //wait.Until(func() { m.Sync(ctx) }, time.Second*pollingIntervalSeconds, ctx.Done()) wait.Until(func() { m.Sync(ctx) }, time.Second*interval_time, ctx.Done()) From 3c0695718053cdc0237ccd70d6c8123455489403 Mon Sep 17 00:00:00 2001 From: jiachu Date: Thu, 27 Feb 2020 10:32:25 +0800 Subject: [PATCH 58/61] change agent run interval --- pkg/controllers/cluster/manager/cluster_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controllers/cluster/manager/cluster_manager.go b/pkg/controllers/cluster/manager/cluster_manager.go index 63d96728e..9a1ea5f2c 100644 --- a/pkg/controllers/cluster/manager/cluster_manager.go +++ b/pkg/controllers/cluster/manager/cluster_manager.go @@ -375,7 +375,7 @@ func (m *ClusterManager) rebootFromOutage(ctx context.Context) (*innodb.ClusterS // Run runs the ClusterManager controller. // NOTE: ctx is not currently used for cancellation by caller (the stopCh is). func (m *ClusterManager) Run(ctx context.Context) { - interval_time, _ := strconv.ParseUint(os.Getenv("AGENT_INTERVAL"), 10, 64) + interval_time, _ := strconv.ParseInt(os.Getenv("AGENT_INTERVAL"), 10, 32) glog.Info("***agent run interval: %d", interval_time) //wait.Until(func() { m.Sync(ctx) }, time.Second*pollingIntervalSeconds, ctx.Done()) wait.Until(func() { m.Sync(ctx) }, time.Second*interval_time, ctx.Done()) From 1cb709a3116a21ac088590354c90962f8234463d Mon Sep 17 00:00:00 2001 From: jiachu Date: Thu, 27 Feb 2020 10:36:58 +0800 Subject: [PATCH 59/61] change agent run interval --- pkg/controllers/cluster/manager/cluster_manager.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controllers/cluster/manager/cluster_manager.go b/pkg/controllers/cluster/manager/cluster_manager.go index 9a1ea5f2c..b004a979c 100644 --- a/pkg/controllers/cluster/manager/cluster_manager.go +++ b/pkg/controllers/cluster/manager/cluster_manager.go @@ -375,10 +375,10 @@ func (m *ClusterManager) rebootFromOutage(ctx context.Context) (*innodb.ClusterS // Run runs the ClusterManager controller. // NOTE: ctx is not currently used for cancellation by caller (the stopCh is). func (m *ClusterManager) Run(ctx context.Context) { - interval_time, _ := strconv.ParseInt(os.Getenv("AGENT_INTERVAL"), 10, 32) + interval_time, _ := strconv.ParseUint(os.Getenv("AGENT_INTERVAL"), 10, 32) glog.Info("***agent run interval: %d", interval_time) //wait.Until(func() { m.Sync(ctx) }, time.Second*pollingIntervalSeconds, ctx.Done()) - wait.Until(func() { m.Sync(ctx) }, time.Second*interval_time, ctx.Done()) + wait.Until(func() { m.Sync(ctx) }, time.Second*time.Duration(interval_time), ctx.Done()) <-ctx.Done() From 66a19c6927004f23befea2b4fc6aeaad412b3d71 Mon Sep 17 00:00:00 2001 From: jiachu Date: Thu, 27 Feb 2020 10:51:02 +0800 Subject: [PATCH 60/61] change agent run interval --- pkg/controllers/cluster/manager/cluster_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controllers/cluster/manager/cluster_manager.go b/pkg/controllers/cluster/manager/cluster_manager.go index b004a979c..f953b45a0 100644 --- a/pkg/controllers/cluster/manager/cluster_manager.go +++ b/pkg/controllers/cluster/manager/cluster_manager.go @@ -376,7 +376,7 @@ func (m *ClusterManager) rebootFromOutage(ctx context.Context) (*innodb.ClusterS // NOTE: ctx is not currently used for cancellation by caller (the stopCh is). func (m *ClusterManager) Run(ctx context.Context) { interval_time, _ := strconv.ParseUint(os.Getenv("AGENT_INTERVAL"), 10, 32) - glog.Info("***agent run interval: %d", interval_time) + glog.Info("***agent run interval: ", interval_time) //wait.Until(func() { m.Sync(ctx) }, time.Second*pollingIntervalSeconds, ctx.Done()) wait.Until(func() { m.Sync(ctx) }, time.Second*time.Duration(interval_time), ctx.Done()) From 99fd90ce82f03fe9cae674c2de2f64d246ece9d6 Mon Sep 17 00:00:00 2001 From: jiachu Date: Thu, 5 Mar 2020 19:10:36 +0800 Subject: [PATCH 61/61] drop exit-state-action --- pkg/resources/statefulsets/statefulset.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/resources/statefulsets/statefulset.go b/pkg/resources/statefulsets/statefulset.go index a089e43f5..0e70ae6d6 100644 --- a/pkg/resources/statefulsets/statefulset.go +++ b/pkg/resources/statefulsets/statefulset.go @@ -243,9 +243,9 @@ func mysqlServerContainer(cluster *v1alpha1.Cluster, mysqlServerImage string, ro "--ssl-key=/etc/ssl/mysql/tls.key") } - if checkSupportGroupExitStateArgs(cluster.Spec.Version) { - args = append(args, "--loose-group-replication-exit-state-action=READ_ONLY") - } + //if checkSupportGroupExitStateArgs(cluster.Spec.Version) { + // args = append(args, "--loose-group-replication-exit-state-action=READ_ONLY") + //} entryPointArgs := strings.Join(args, " ")