wave 1 error improvements

pkg/capabilities/consensus/ocr3/aggregators/identical.go

@@ -83,7 +83,7 @@ func (a *identicalAggregator) collectHighestCounts(counters []map[[32]byte]*coun
 			}
 		}
 		if highestCount < 2*f+1 {
-			return nil, fmt.Errorf("can't reach consensus on observations with index %d", idx)
+			return nil, fmt.Errorf("consensus failed: cannot reach agreement on observation at index %d. Fewer than %d nodes (2f+1, f=%d) agreed on the same value. This may indicate data source inconsistency across nodes or a non-deterministic computation", idx, 2*f+1, f)
 		}
 		if useOverrides {
 			outcome[a.config.KeyOverrides[idx]] = highestObservation

pkg/capabilities/consensus/ocr3/aggregators/identical_test.go

@@ -78,7 +78,7 @@ func TestDataFeedsAggregator_Aggregate_NoConsensus(t *testing.T) {
 	}
 	outcome, err := agg.Aggregate(logger.Nop(), nil, observations, 1)
 	require.Nil(t, outcome)
-	require.ErrorContains(t, err, "can't reach consensus on observations with index 0")
+	require.ErrorContains(t, err, "consensus failed: cannot reach agreement on observation at index 0")
 }
 
 func getConfigIdenticalAggregator(t *testing.T, overrideKeys []string) *values.Map {

pkg/capabilities/consensus/ocr3/aggregators/reduce_aggregator.go

@@ -98,7 +98,7 @@ var _ types.Aggregator = (*reduceAggregator)(nil)
 // Condenses multiple observations into a single encodable outcome
 func (a *reduceAggregator) Aggregate(lggr logger.Logger, previousOutcome *types.AggregationOutcome, observations map[ocrcommon.OracleID][]values.Value, f int) (*types.AggregationOutcome, error) {
 	if len(observations) < 2*f+1 {
-		return nil, fmt.Errorf("not enough observations, have %d want %d", len(observations), 2*f+1)
+		return nil, fmt.Errorf("consensus failed: insufficient observations, received %d but need at least %d (2f+1, f=%d). Not enough DON nodes responded in time", len(observations), 2*f+1, f)
 	}
 
 	currentState, err := a.initializeCurrentState(lggr, previousOutcome)
@@ -114,7 +114,7 @@ func (a *reduceAggregator) Aggregate(lggr logger.Logger, previousOutcome *types.
 
 		// only proceed if every field has reached the minimum number of observations
 		if len(vals) < 2*f+1 {
-			return nil, fmt.Errorf("not enough observations provided %s, have %d want %d", field.InputKey, len(vals), 2*f+1)
+			return nil, fmt.Errorf("consensus failed: insufficient observations for field %q, received %d but need at least %d (2f+1, f=%d). Not enough DON nodes provided data for this field", field.InputKey, len(vals), 2*f+1, f)
 		}
 
 		singleValue, err := reduce(field.Method, vals, f, field.ModeQuorum)
@@ -486,12 +486,12 @@ func modeHasQuorum(quorumType string, count int, f int) error {
 		return nil
 	case MODE_QUORUM_OCR:
 		if count < f+1 {
-			return fmt.Errorf("mode quorum not reached. have: %d, want: %d", count, f+1)
+			return fmt.Errorf("consensus failed: mode quorum not reached, %d nodes agreed but need at least %d (f+1, f=%d). DON nodes disagree too much on the value", count, f+1, f)
 		}
 		return nil
 	case MODE_QUORUM_ALL:
 		if count < 2*f+1 {
-			return fmt.Errorf("mode quorum not reached. have: %d, want: %d", count, 2*f+1)
+			return fmt.Errorf("consensus failed: mode quorum not reached, %d nodes agreed but need at least %d (2f+1, f=%d). DON nodes disagree too much on the value", count, 2*f+1, f)
 		}
 		return nil
 	default:

pkg/capabilities/consensus/ocr3/aggregators/reduce_test.go

@@ -687,7 +687,7 @@ func TestReduceAggregator_Aggregate(t *testing.T) {
 				observationsFactory: func() map[commontypes.OracleID][]values.Value {
 					return map[commontypes.OracleID][]values.Value{}
 				},
-				errString: "not enough observations, have 0 want 3",
+				errString: "consensus failed: insufficient observations, received 0 but need at least 3 (2f+1, f=1). Not enough DON nodes responded in time",
 			},
 			{
 				name: "invalid previous outcome not pb",
@@ -725,7 +725,7 @@ func TestReduceAggregator_Aggregate(t *testing.T) {
 					mockValueEmpty := values.EmptyMap()
 					return map[commontypes.OracleID][]values.Value{1: {mockValue}, 2: {mockValue}, 3: {mockValueEmpty}}
 				},
-				errString: "not enough observations provided Price, have 2 want 3",
+				errString: "consensus failed: insufficient observations for field \"Price\", received 2 but need at least 3 (2f+1, f=1). Not enough DON nodes provided data for this field",
 			},
 			{
 				name:            "reduce error median",
@@ -764,7 +764,7 @@ func TestReduceAggregator_Aggregate(t *testing.T) {
 					require.NoError(t, err)
 					return map[commontypes.OracleID][]values.Value{1: {mockValue}, 2: {mockValue2}, 3: {mockValue3}}
 				},
-				errString: "unable to reduce on method mode, err: mode quorum not reached. have: 1, want: 2",
+				errString: "unable to reduce on method mode, err: consensus failed: mode quorum not reached, 1 nodes agreed but need at least 2 (f+1, f=1). DON nodes disagree too much on the value",
 			},
 			{
 				name:            "reduce error mode with mode quorum of: all",
@@ -784,7 +784,7 @@ func TestReduceAggregator_Aggregate(t *testing.T) {
 					require.NoError(t, err)
 					return map[commontypes.OracleID][]values.Value{1: {mockValue}, 2: {mockValue2}, 3: {mockValue2}}
 				},
-				errString: "unable to reduce on method mode, err: mode quorum not reached. have: 2, want: 3",
+				errString: "unable to reduce on method mode, err: consensus failed: mode quorum not reached, 2 nodes agreed but need at least 3 (2f+1, f=1). DON nodes disagree too much on the value",
 			},
 		}
 		for _, tt := range cases {

pkg/config/errors.go

@@ -47,7 +47,7 @@ type KeyNotFoundError struct {
 }
 
 func (e KeyNotFoundError) Error() string {
-	return fmt.Sprintf("unable to find %s key with id %s", e.KeyType, e.ID)
+	return fmt.Sprintf("key not found: no %s key exists with ID %q. Verify the key ID is correct and that the key has been created", e.KeyType, e.ID)
 }
 
 // UniqueStrings is a helper for tracking unique values in string form.

pkg/loop/internal/goplugin/service.go

@@ -13,7 +13,7 @@ import (
 	"github.com/smartcontractkit/chainlink-common/pkg/services"
 )
 
-var ErrPluginUnavailable = errors.New("plugin unavailable")
+var ErrPluginUnavailable = errors.New("plugin unavailable: the capability plugin is not running or has become unresponsive. This is typically a node operator issue - the plugin may need to be restarted")
 
 var (
 	_ services.Service = (*ServiceClient)(nil)

pkg/settings/limits/bound_test.go

@@ -28,7 +28,7 @@ func ExampleBoundLimiter_Check() {
 	fn(4)
 	fn(10)
 	// Output:
-	// limited: cannot use 11, limit is 10
+	// limited: cannot use 11, maximum allowed is 10. Reduce usage or request a limit increase
 	// used 4
 	// used 10
 }

pkg/settings/limits/errors.go

@@ -34,7 +34,7 @@ func (e ErrorRateLimited) Is(target error) bool {
 
 func (e ErrorRateLimited) Error() string {
 	which, who := errArgs(e.Key, e.Scope, e.Tenant)
-	msg := fmt.Sprintf("%srate limited%s", which, who)
+	msg := fmt.Sprintf("%srate limited%s: request rate has exceeded the allowed limit. Please reduce request frequency or wait before retrying", which, who)
 	if e.Err == nil {
 		return msg
 	}
@@ -61,7 +61,7 @@ func (e ErrorResourceLimited[N]) Is(target error) bool {
 
 func (e ErrorResourceLimited[N]) Error() string {
 	which, who := errArgs(e.Key, e.Scope, e.Tenant)
-	return fmt.Sprintf("%sresource limited%s: cannot use %v, already using %v/%v", which, who, e.Amount, e.Used, e.Limit)
+	return fmt.Sprintf("%sresource limited%s: cannot allocate %v, already using %v of %v maximum. Free existing resources or request a limit increase", which, who, e.Amount, e.Used, e.Limit)
 }
 
 type ErrorTimeLimited struct {
@@ -84,7 +84,7 @@ func (e ErrorTimeLimited) Is(target error) bool {
 
 func (e ErrorTimeLimited) Error() string {
 	which, who := errArgs(e.Key, e.Scope, e.Tenant)
-	return fmt.Sprintf("%stime limited%s to %s", which, who, e.Timeout)
+	return fmt.Sprintf("%stime limited%s: operation exceeded the maximum allowed duration of %s. Consider simplifying the operation or requesting a timeout increase", which, who, e.Timeout)
 }
 
 func errArgs(key string, scope settings.Scope, tenant string) (which, who string) {
@@ -117,7 +117,7 @@ func (e ErrorBoundLimited[N]) Is(target error) bool {
 
 func (e ErrorBoundLimited[N]) Error() string {
 	which, who := errArgs(e.Key, e.Scope, e.Tenant)
-	return fmt.Sprintf("%slimited%s: cannot use %v, limit is %v", which, who, e.Amount, e.Limit)
+	return fmt.Sprintf("%slimited%s: cannot use %v, maximum allowed is %v. Reduce usage or request a limit increase", which, who, e.Amount, e.Limit)
 }
 
 type ErrorQueueFull struct {
@@ -140,7 +140,7 @@ func (e ErrorQueueFull) Is(target error) bool {
 
 func (e ErrorQueueFull) Error() string {
 	which, who := errArgs(e.Key, e.Scope, e.Tenant)
-	return fmt.Sprintf("%slimited%s: queue of %d is full", which, who, e.Limit)
+	return fmt.Sprintf("%slimited%s: queue is full (capacity: %d). New items are being rejected (failed to enqueue). Consider reducing submission rate or requesting a capacity increase", which, who, e.Limit)
 }
 
 var ErrQueueEmpty = fmt.Errorf("queue is empty")
@@ -163,5 +163,5 @@ func (e ErrorNotAllowed) Is(target error) bool {
 
 func (e ErrorNotAllowed) Error() string {
 	which, who := errArgs(e.Key, e.Scope, e.Tenant)
-	return fmt.Sprintf("%slimited%s: not allowed", which, who)
+	return fmt.Sprintf("%slimited%s: operation not allowed. This action is restricted by current configuration and gate settings", which, who)
 }

pkg/settings/limits/errors_test.go

@@ -31,7 +31,7 @@ func TestErrorRateLimited(t *testing.T) {
 				N:      42,
 				Err:    wrapped,
 			},
-			exp: "foo rate limited for workflow[wf]: wrapper",
+			exp: "foo rate limited for workflow[wf]: request rate has exceeded the allowed limit. Please reduce request frequency or wait before retrying: wrapper",
 		},
 		{
 			name: "no-err",
@@ -41,22 +41,22 @@ func TestErrorRateLimited(t *testing.T) {
 				Tenant: "wf",
 				N:      42,
 			},
-			exp: "foo rate limited for workflow[wf]",
+			exp: "foo rate limited for workflow[wf]: request rate has exceeded the allowed limit. Please reduce request frequency or wait before retrying",
 		},
 		{
 			name: "no-err-tenant",
 			err: ErrorRateLimited{
 				Key: "foo",
 				N:   42,
 			},
-			exp: "foo rate limited",
+			exp: "foo rate limited: request rate has exceeded the allowed limit. Please reduce request frequency or wait before retrying",
 		},
 		{
 			name: "no-err-tenant-key",
 			err: ErrorRateLimited{
 				N: 42,
 			},
-			exp: "rate limited",
+			exp: "rate limited: request rate has exceeded the allowed limit. Please reduce request frequency or wait before retrying",
 		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
@@ -88,7 +88,7 @@ func TestErrorResourceLimited(t *testing.T) {
 				Used:   42,
 				Amount: 13,
 			},
-			exp: "foo resource limited for workflow[wf]: cannot use 13, already using 42/100",
+			exp: "foo resource limited for workflow[wf]: cannot allocate 13, already using 42 of 100 maximum. Free existing resources or request a limit increase",
 		},
 		{
 			name: "no-tenant",
@@ -99,7 +99,7 @@ func TestErrorResourceLimited(t *testing.T) {
 				Used:   42,
 				Amount: 13,
 			},
-			exp: "foo resource limited: cannot use 13, already using 42/100",
+			exp: "foo resource limited: cannot allocate 13, already using 42 of 100 maximum. Free existing resources or request a limit increase",
 		},
 		{
 			name: "no-tenant-key",
@@ -108,7 +108,7 @@ func TestErrorResourceLimited(t *testing.T) {
 				Used:   42,
 				Amount: 13,
 			},
-			exp: "resource limited: cannot use 13, already using 42/100",
+			exp: "resource limited: cannot allocate 13, already using 42 of 100 maximum. Free existing resources or request a limit increase",
 		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
@@ -138,7 +138,7 @@ func TestErrorTimeLimited(t *testing.T) {
 				Tenant:  "wf",
 				Timeout: time.Minute,
 			},
-			exp: "foo time limited for workflow[wf] to 1m0s",
+			exp: "foo time limited for workflow[wf]: operation exceeded the maximum allowed duration of 1m0s. Consider simplifying the operation or requesting a timeout increase",
 		},
 		{
 			name: "no-tenant",
@@ -147,14 +147,14 @@ func TestErrorTimeLimited(t *testing.T) {
 				Scope:   settings.ScopeWorkflow,
 				Timeout: time.Minute,
 			},
-			exp: "foo time limited to 1m0s",
+			exp: "foo time limited: operation exceeded the maximum allowed duration of 1m0s. Consider simplifying the operation or requesting a timeout increase",
 		},
 		{
 			name: "no-tenant-key",
 			err: ErrorTimeLimited{
 				Timeout: time.Minute,
 			},
-			exp: "time limited to 1m0s",
+			exp: "time limited: operation exceeded the maximum allowed duration of 1m0s. Consider simplifying the operation or requesting a timeout increase",
 		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
@@ -185,7 +185,7 @@ func TestErrorBoundLimited(t *testing.T) {
 				Limit:  13,
 				Amount: 100,
 			},
-			exp: "foo limited for workflow[wf]: cannot use 100, limit is 13",
+			exp: "foo limited for workflow[wf]: cannot use 100, maximum allowed is 13. Reduce usage or request a limit increase",
 		},
 		{
 			name: "no-tenant",
@@ -195,15 +195,15 @@ func TestErrorBoundLimited(t *testing.T) {
 				Limit:  13,
 				Amount: 100,
 			},
-			exp: "foo limited: cannot use 100, limit is 13",
+			exp: "foo limited: cannot use 100, maximum allowed is 13. Reduce usage or request a limit increase",
 		},
 		{
 			name: "no-tenant-key",
 			err: ErrorBoundLimited[int]{
 				Limit:  13,
 				Amount: 100,
 			},
-			exp: "limited: cannot use 100, limit is 13",
+			exp: "limited: cannot use 100, maximum allowed is 13. Reduce usage or request a limit increase",
 		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {

pkg/settings/limits/gate_test.go

@@ -44,7 +44,7 @@ func ExampleGateLimiter_AllowErr() {
 	// open: true
 	// allow: <nil>
 	// open: false
-	// allow: limited: not allowed
+	// allow: limited: operation not allowed. This action is restricted by current configuration and gate settings
 }
 
 func TestMakeGateLimiter(t *testing.T) {

pkg/settings/limits/rate_test.go

@@ -96,7 +96,7 @@ func ExampleRateLimiter_AllowErr() {
 	}
 
 	// Output:
-	// 5: rate limited
+	// 5: rate limited: request rate has exceeded the allowed limit. Please reduce request frequency or wait before retrying
 	// 4: success
 }
 
@@ -168,7 +168,7 @@ func ExampleMultiRateLimiter() {
 
 	// Output:
 	// A: success
-	// A&B: rate limited
+	// A&B: rate limited: request rate has exceeded the allowed limit. Please reduce request frequency or wait before retrying
 }
 
 func TestFactory_NewRateLimiter(t *testing.T) {

pkg/settings/limits/resource_test.go

@@ -67,7 +67,7 @@ func ExampleResourceLimiter_Use() {
 	defer limiter.Free(ctx, 1)
 
 	// Output:
-	// Try failed: resource limited: cannot use 1, already using 5/5
+	// Try failed: resource limited: cannot allocate 1, already using 5 of 5 maximum. Free existing resources or request a limit increase
 }
 
 func ExampleMultiResourcePoolLimiter() {
@@ -118,10 +118,10 @@ func ExampleMultiResourcePoolLimiter() {
 	}
 	free()
 	// Output:
-	// resource limited: cannot use 10, already using 95/100
-	// resource limited for org[org-id]: cannot use 10, already using 45/50
-	// resource limited for owner[owner-id]: cannot use 10, already using 15/20
-	// resource limited for workflow[workflow-id]: cannot use 10, already using 5/10
+	// resource limited: cannot allocate 10, already using 95 of 100 maximum. Free existing resources or request a limit increase
+	// resource limited for org[org-id]: cannot allocate 10, already using 45 of 50 maximum. Free existing resources or request a limit increase
+	// resource limited for owner[owner-id]: cannot allocate 10, already using 15 of 20 maximum. Free existing resources or request a limit increase
+	// resource limited for workflow[workflow-id]: cannot allocate 10, already using 5 of 10 maximum. Free existing resources or request a limit increase
 	// <nil>
 }