Merge pull request #607 from CS-SI/bug/circuitbreaker

Bugfixes: cluster

Author: oscarpicas

cli/safescale/commands/cluster.go

@@ -514,10 +514,6 @@ var clusterCreateCommand = cli.Command{
 			}
 		}
 		if c.IsSet("master-sizing") {
-			if strings.Contains(c.String("master-sizing"), "count") {
-				return clitools.FailureResponse(clitools.ExitOnInvalidArgument("the number of masters cannot be changed yet: count cannot be included in 'master-sizing' flag"))
-			}
-
 			mastersDef, err = constructHostDefinitionStringFromCLI(c, "master-sizing")
 			if err != nil {
 				return err

go.mod

@@ -37,6 +37,7 @@ require (
 	github.com/schollz/progressbar/v3 v3.9.0
 	github.com/sethvargo/go-password v0.2.0
 	github.com/sirupsen/logrus v1.9.0
+	github.com/sony/gobreaker v0.5.0
 	github.com/spf13/viper v1.11.0
 	github.com/stretchr/testify v1.8.1
 	github.com/urfave/cli v1.22.10
@@ -50,7 +51,7 @@ require (
 	golang.org/x/text v0.6.0
 	gomodules.xyz/stow v0.2.4
 	google.golang.org/api v0.107.0
-	google.golang.org/grpc v1.51.0
+	google.golang.org/grpc v1.52.3
 	google.golang.org/protobuf v1.28.1
 	gopkg.in/fsnotify.v1 v1.4.7
 )

go.sum

@@ -484,6 +484,8 @@ github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0
 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/smartystreets/assertions v1.13.0 h1:Dx1kYM01xsSqKPno3aqLnrwac2LetPvN23diwyr69Qs=
 github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=
+github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg=
+github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo=
 github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo=
@@ -902,8 +904,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
 google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U=
-google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww=
+google.golang.org/grpc v1.52.3 h1:pf7sOysg4LdgBqduXveGKrcEwbStiK2rtfghdzlUYDQ=
+google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

lib/backend/resources/abstract/cluster.go

@@ -20,6 +20,7 @@ import (
 	stdjson "encoding/json"
 	"fmt"
 	"github.com/CS-SI/SafeScale/v22/lib"
+	uuidpkg "github.com/gofrs/uuid"
 	"time"
 
 	"github.com/CS-SI/SafeScale/v22/lib/backend/resources/enums/clustercomplexity"
@@ -44,11 +45,12 @@ type ClusterRequest struct {
 	MastersDef              HostSizingRequirements // sizing of Masters
 	NodesDef                HostSizingRequirements // sizing of nodes
 	InitialNodeCount        uint                   // contains the initial count of nodes to create (cannot be less than flavor requirement)
-	OS                      string                 // contains the name of the linux distribution wanted
-	DisabledDefaultFeatures map[string]struct{}    // contains the list of features that should be installed by default, but we don't want actually
-	Force                   bool                   // set to True in order to ignore sizing recommendations
-	FeatureParameters       []string               // contains parameter values of automatically installed Features
-	DefaultSshPort          uint                   // default ssh port for gateways // nolint
+	InitialMasterCount      uint
+	OS                      string              // contains the name of the linux distribution wanted
+	DisabledDefaultFeatures map[string]struct{} // contains the list of features that should be installed by default, but we don't want actually
+	Force                   bool                // set to True in order to ignore sizing recommendations
+	FeatureParameters       []string            // contains parameter values of automatically installed Features
+	DefaultSshPort          uint                // default ssh port for gateways // nolint
 }
 
 // ClusterIdentity contains the bare minimum information about a cluster
@@ -59,11 +61,18 @@ type ClusterIdentity struct {
 	Keypair       *KeyPair               `json:"keypair"`        // Keypair contains the key-pair used inside the Cluster
 	AdminPassword string                 `json:"admin_password"` // contains the password of the cladm account
 	Tags          map[string]string      `json:"tags,omitempty"`
+	ID            string                 `json:"ID"`
 }
 
 // NewClusterIdentity ...
 func NewClusterIdentity() *ClusterIdentity {
+	aid := ""
+	if uuid, err := uuidpkg.NewV4(); err == nil {
+		aid = uuid.String()
+	}
+
 	ci := &ClusterIdentity{
+		ID:   aid,
 		Tags: make(map[string]string),
 	}
 	ci.Tags["CreationDate"] = time.Now().Format(time.RFC3339)
@@ -109,10 +118,10 @@ func (instance ClusterIdentity) GetName() string {
 	return instance.Name
 }
 
-// GetID returns the ID of the cluster (== GetName)
+// GetID returns the ID of the cluster
 // Satisfies interface data.Identifiable
 func (instance ClusterIdentity) GetID() (string, error) {
-	return instance.GetName(), nil
+	return instance.ID, nil
 }
 
 // OK ...

lib/backend/resources/abstract/cluster_test.go

@@ -236,7 +236,4 @@ func TestClusterIdentity_Replace(t *testing.T) {
 	require.EqualValues(t, result, cluster2)
 	require.EqualValues(t, result.(*ClusterIdentity).Keypair, kp2)
 	require.EqualValues(t, result.(*ClusterIdentity).GetName(), cluster2.Name)
-	clid, _ := result.(*ClusterIdentity).GetID()
-	require.EqualValues(t, clid, cluster2.Name)
-
 }

lib/backend/resources/abstract/host.go

@@ -50,7 +50,7 @@ func (kp *KeyPair) IsNull() bool {
 func NewKeyPair(name string) (*KeyPair, fail.Error) {
 	id, err := uuid.NewV4()
 	if err != nil {
-		return nil, fail.Wrap(err, "failed to create host UUID")
+		return nil, fail.Wrap(err, "failed to create keypair UUID")
 	}
 
 	if name == "" {

lib/backend/resources/abstract/subnet.go

@@ -43,6 +43,7 @@ type SubnetRequest struct {
 	ImageRef       string         // contains the reference (ID or name) of the image requested for gateway(s)
 	DefaultSSHPort uint32         // contains the port to use for SSH on all hosts of the subnet by default
 	KeepOnFailure  bool           // tells if resources have to be kept in case of failure (default behavior is to delete them)
+	ClusterID      string
 }
 
 // Subnet represents a subnet

lib/backend/resources/cluster.go

@@ -49,7 +49,6 @@ type Cluster interface {
 	DeleteSpecificNode(ctx context.Context, hostID string, selectedMasterID string) fail.Error                                                                // deletes a node identified by its ID
 	Delete(ctx context.Context, force bool) fail.Error                                                                                                        // deletes the cluster (Delete is not used to not collision with metadata)
 	FindAvailableMaster(ctx context.Context) (Host, fail.Error)                                                                                               // returns ID of the first master available to execute order
-	FindAvailableNode(ctx context.Context) (Host, fail.Error)                                                                                                 // returns node instance of the first node available to execute order
 	GetIdentity(ctx context.Context) (abstract.ClusterIdentity, fail.Error)                                                                                   // returns Cluster Identity
 	GetFlavor(ctx context.Context) (clusterflavor.Enum, fail.Error)                                                                                           // returns the flavor of the cluster
 	GetComplexity(ctx context.Context) (clustercomplexity.Enum, fail.Error)                                                                                   // returns the complexity of the cluster
@@ -62,7 +61,6 @@ type Cluster interface {
 	ListInstalledFeatures(ctx context.Context) ([]Feature, fail.Error)                                                                                        // returns the list of installed features on the Cluster
 	ListMasters(ctx context.Context) (IndexedListOfClusterNodes, fail.Error)                                                                                  // lists the node instances corresponding to masters (if there is such masters in the flavor...)
 	ListNodes(ctx context.Context) (IndexedListOfClusterNodes, fail.Error)                                                                                    // lists node instances corresponding to the nodes in the cluster
-	LookupNode(ctx context.Context, ref string) (bool, fail.Error)                                                                                            // tells if the ID of the host passed as parameter is a node
 	RemoveFeature(ctx context.Context, name string, vars data.Map, settings FeatureSettings) (Results, fail.Error)                                            // removes feature from cluster
 	Shrink(ctx context.Context, name string, count uint) ([]*propertiesv3.ClusterNode, fail.Error)                                                            // reduce the size of the cluster of 'count' nodes (the last created)
 	Start(ctx context.Context) fail.Error                                                                                                                     // starts the cluster

lib/backend/resources/operations/cluster.go

@@ -59,6 +59,8 @@ import (
 	"github.com/CS-SI/SafeScale/v22/lib/utils/retry"
 	"github.com/CS-SI/SafeScale/v22/lib/utils/strprocess"
 	"github.com/CS-SI/SafeScale/v22/lib/utils/template"
+
+	_ "github.com/sony/gobreaker"
 )
 
 const (
@@ -1239,9 +1241,12 @@ func (instance *Cluster) AddNodes(ctx context.Context, cluName string, count uin
 		}
 	}
 
+	ctx, ca := context.WithTimeout(ctx, timeout)
+	defer ca()
+
 	// for OVH, we have to ignore the errors and keep trying until we have 'count'
-	nodesChan := make(chan StdResult, count)
-	err = runWindow(ctx, count, uint(math.Min(float64(count), float64(winSize))), timeout, nodesChan, instance.taskCreateNode, taskCreateNodeParameters{
+	nodesChan := make(chan StdResult, 4*count)
+	err = runWindow(ctx, count, uint(math.Min(float64(count), float64(winSize))), nodesChan, instance.taskCreateNode, taskCreateNodeParameters{
 		nodeDef:       nodeDef,
 		timeout:       timings.HostCreationTimeout(),
 		keepOnFailure: keepOnFailure,
@@ -1579,67 +1584,6 @@ func (instance *Cluster) ListNodeNames(ctx context.Context) (list data.IndexedLi
 	return list, nil
 }
 
-// FindAvailableNode returns node instance of the first node available to execute order
-func (instance *Cluster) FindAvailableNode(ctx context.Context) (node resources.Host, ferr fail.Error) {
-	defer fail.OnPanic(&ferr)
-
-	if valid.IsNil(instance) {
-		return nil, fail.InvalidInstanceError()
-	}
-	if ctx == nil {
-		return nil, fail.InvalidParameterCannotBeNilError("ctx")
-	}
-
-	return instance.unsafeFindAvailableNode(ctx)
-}
-
-// LookupNode tells if the ID of the master passed as parameter is a node
-func (instance *Cluster) LookupNode(ctx context.Context, ref string) (found bool, ferr fail.Error) {
-	defer fail.OnPanic(&ferr)
-
-	if valid.IsNil(instance) {
-		return false, fail.InvalidInstanceError()
-	}
-	if ctx == nil {
-		return false, fail.InvalidParameterCannotBeNilError("ctx")
-	}
-	if ref == "" {
-		return false, fail.InvalidParameterError("ref", "cannot be empty string")
-	}
-
-	xerr := instance.beingRemoved(ctx)
-	xerr = debug.InjectPlannedFail(xerr)
-	if xerr != nil {
-		return false, xerr
-	}
-
-	var hostInstance resources.Host
-	hostInstance, xerr = LoadHost(ctx, instance.Service(), ref)
-	xerr = debug.InjectPlannedFail(xerr)
-	if xerr != nil {
-		return false, xerr
-	}
-
-	found = false
-	xerr = instance.Inspect(ctx, func(_ data.Clonable, props *serialize.JSONProperties) fail.Error {
-		return props.Inspect(clusterproperty.NodesV3, func(clonable data.Clonable) fail.Error {
-			nodesV3, ok := clonable.(*propertiesv3.ClusterNodes)
-			if !ok {
-				return fail.InconsistentError("'*propertiesv3.ClusterNodes' expected, '%s' provided", reflect.TypeOf(clonable).String())
-			}
-
-			hid, err := hostInstance.GetID()
-			if err != nil {
-				return fail.ConvertError(err)
-			}
-
-			_, found = nodesV3.PrivateNodeByID[hid]
-			return nil
-		})
-	})
-	return found, xerr
-}
-
 // deleteMaster deletes the master specified by its ID
 func (instance *Cluster) deleteMaster(ctx context.Context, host string) (ferr fail.Error) {
 	if valid.IsNil(instance) {

lib/backend/resources/operations/cluster_test.go

@@ -986,74 +986,6 @@ func TestCluster_ListNodes(t *testing.T) {
 
 }
 
-func TestCluster_FindAvailableNode(t *testing.T) {
-
-	ctx := context.Background()
-
-	err := NewServiceTest(t, func(svc *ServiceTest) {
-
-		svc._setLogLevel(0)
-
-		_, xerr := svc._CreateCluster(ctx, createClusterRequest(), true)
-		require.Nil(t, xerr)
-
-		cluster, xerr := LoadCluster(ctx, svc, "ClusterName")
-		require.Nil(t, xerr)
-
-		ocluster, ok := cluster.(*Cluster)
-		if !ok {
-			t.Error("ressources.Cluster not castable to operation.Cluster")
-			t.FailNow()
-		}
-
-		node, xerr := ocluster.FindAvailableNode(ctx)
-		require.Nil(t, xerr)
-		require.EqualValues(t, node.GetName(), "ClusterName-node-1")
-
-	})
-	require.Nil(t, err)
-
-}
-
-func TestCluster_LookupNode(t *testing.T) {
-
-	ctx := context.Background()
-
-	err := NewServiceTest(t, func(svc *ServiceTest) {
-
-		svc._setLogLevel(0)
-
-		_, xerr := svc._CreateCluster(ctx, createClusterRequest(), true)
-		require.Nil(t, xerr)
-
-		cluster, xerr := LoadCluster(ctx, svc, "ClusterName")
-		require.Nil(t, xerr)
-
-		ocluster, ok := cluster.(*Cluster)
-		if !ok {
-			t.Error("ressources.Cluster not castable to operation.Cluster")
-			t.FailNow()
-		}
-
-		found, xerr := ocluster.LookupNode(nil, "ClusterName-node-1")
-		require.Contains(t, xerr.Error(), "invalid parameter: ctx")
-
-		found, xerr = ocluster.LookupNode(ctx, "")
-		require.Contains(t, xerr.Error(), "invalid parameter: ref")
-
-		found, xerr = ocluster.LookupNode(ctx, "ClusterName-node-1")
-		require.Nil(t, xerr)
-		require.True(t, found)
-
-		found, xerr = ocluster.LookupNode(ctx, "ClusterName-node-2")
-		require.EqualValues(t, reflect.TypeOf(xerr).String(), "*fail.ErrNotFound")
-		require.False(t, found)
-
-	})
-	require.Nil(t, err)
-
-}
-
 func TestCluster_Delete(t *testing.T) {}
 
 func TestCluster_ToProtocol(t *testing.T) {