diff --git a/pkg/capabilities/consensus/ocr3/aggregators/identical.go b/pkg/capabilities/consensus/ocr3/aggregators/identical.go index 41002caa3c..88bb724dfc 100644 --- a/pkg/capabilities/consensus/ocr3/aggregators/identical.go +++ b/pkg/capabilities/consensus/ocr3/aggregators/identical.go @@ -83,7 +83,7 @@ func (a *identicalAggregator) collectHighestCounts(counters []map[[32]byte]*coun } } if highestCount < 2*f+1 { - return nil, fmt.Errorf("can't reach consensus on observations with index %d", idx) + return nil, fmt.Errorf("consensus failed: cannot reach agreement on observation at index %d. Fewer than %d nodes (2f+1, f=%d) agreed on the same value. This may indicate data source inconsistency across nodes or a non-deterministic computation", idx, 2*f+1, f) } if useOverrides { outcome[a.config.KeyOverrides[idx]] = highestObservation diff --git a/pkg/capabilities/consensus/ocr3/aggregators/identical_test.go b/pkg/capabilities/consensus/ocr3/aggregators/identical_test.go index 8051131b6f..b94b49677d 100644 --- a/pkg/capabilities/consensus/ocr3/aggregators/identical_test.go +++ b/pkg/capabilities/consensus/ocr3/aggregators/identical_test.go @@ -78,7 +78,7 @@ func TestDataFeedsAggregator_Aggregate_NoConsensus(t *testing.T) { } outcome, err := agg.Aggregate(logger.Nop(), nil, observations, 1) require.Nil(t, outcome) - require.ErrorContains(t, err, "can't reach consensus on observations with index 0") + require.ErrorContains(t, err, "consensus failed: cannot reach agreement on observation at index 0") } func getConfigIdenticalAggregator(t *testing.T, overrideKeys []string) *values.Map { diff --git a/pkg/capabilities/consensus/ocr3/aggregators/reduce_aggregator.go b/pkg/capabilities/consensus/ocr3/aggregators/reduce_aggregator.go index f63df075a4..8cedc47abd 100644 --- a/pkg/capabilities/consensus/ocr3/aggregators/reduce_aggregator.go +++ b/pkg/capabilities/consensus/ocr3/aggregators/reduce_aggregator.go @@ -98,7 +98,7 @@ var _ types.Aggregator = (*reduceAggregator)(nil) // Condenses multiple observations into a single encodable outcome func (a *reduceAggregator) Aggregate(lggr logger.Logger, previousOutcome *types.AggregationOutcome, observations map[ocrcommon.OracleID][]values.Value, f int) (*types.AggregationOutcome, error) { if len(observations) < 2*f+1 { - return nil, fmt.Errorf("not enough observations, have %d want %d", len(observations), 2*f+1) + return nil, fmt.Errorf("consensus failed: insufficient observations, received %d but need at least %d (2f+1, f=%d). Not enough DON nodes responded in time", len(observations), 2*f+1, f) } currentState, err := a.initializeCurrentState(lggr, previousOutcome) @@ -114,7 +114,7 @@ func (a *reduceAggregator) Aggregate(lggr logger.Logger, previousOutcome *types. // only proceed if every field has reached the minimum number of observations if len(vals) < 2*f+1 { - return nil, fmt.Errorf("not enough observations provided %s, have %d want %d", field.InputKey, len(vals), 2*f+1) + return nil, fmt.Errorf("consensus failed: insufficient observations for field %q, received %d but need at least %d (2f+1, f=%d). Not enough DON nodes provided data for this field", field.InputKey, len(vals), 2*f+1, f) } singleValue, err := reduce(field.Method, vals, f, field.ModeQuorum) @@ -486,12 +486,12 @@ func modeHasQuorum(quorumType string, count int, f int) error { return nil case MODE_QUORUM_OCR: if count < f+1 { - return fmt.Errorf("mode quorum not reached. have: %d, want: %d", count, f+1) + return fmt.Errorf("consensus failed: mode quorum not reached, %d nodes agreed but need at least %d (f+1, f=%d). DON nodes disagree too much on the value", count, f+1, f) } return nil case MODE_QUORUM_ALL: if count < 2*f+1 { - return fmt.Errorf("mode quorum not reached. have: %d, want: %d", count, 2*f+1) + return fmt.Errorf("consensus failed: mode quorum not reached, %d nodes agreed but need at least %d (2f+1, f=%d). DON nodes disagree too much on the value", count, 2*f+1, f) } return nil default: diff --git a/pkg/capabilities/consensus/ocr3/aggregators/reduce_test.go b/pkg/capabilities/consensus/ocr3/aggregators/reduce_test.go index b48e60c36e..6d086d5540 100644 --- a/pkg/capabilities/consensus/ocr3/aggregators/reduce_test.go +++ b/pkg/capabilities/consensus/ocr3/aggregators/reduce_test.go @@ -687,7 +687,7 @@ func TestReduceAggregator_Aggregate(t *testing.T) { observationsFactory: func() map[commontypes.OracleID][]values.Value { return map[commontypes.OracleID][]values.Value{} }, - errString: "not enough observations, have 0 want 3", + errString: "consensus failed: insufficient observations, received 0 but need at least 3 (2f+1, f=1). Not enough DON nodes responded in time", }, { name: "invalid previous outcome not pb", @@ -725,7 +725,7 @@ func TestReduceAggregator_Aggregate(t *testing.T) { mockValueEmpty := values.EmptyMap() return map[commontypes.OracleID][]values.Value{1: {mockValue}, 2: {mockValue}, 3: {mockValueEmpty}} }, - errString: "not enough observations provided Price, have 2 want 3", + errString: "consensus failed: insufficient observations for field \"Price\", received 2 but need at least 3 (2f+1, f=1). Not enough DON nodes provided data for this field", }, { name: "reduce error median", @@ -764,7 +764,7 @@ func TestReduceAggregator_Aggregate(t *testing.T) { require.NoError(t, err) return map[commontypes.OracleID][]values.Value{1: {mockValue}, 2: {mockValue2}, 3: {mockValue3}} }, - errString: "unable to reduce on method mode, err: mode quorum not reached. have: 1, want: 2", + errString: "unable to reduce on method mode, err: consensus failed: mode quorum not reached, 1 nodes agreed but need at least 2 (f+1, f=1). DON nodes disagree too much on the value", }, { name: "reduce error mode with mode quorum of: all", @@ -784,7 +784,7 @@ func TestReduceAggregator_Aggregate(t *testing.T) { require.NoError(t, err) return map[commontypes.OracleID][]values.Value{1: {mockValue}, 2: {mockValue2}, 3: {mockValue2}} }, - errString: "unable to reduce on method mode, err: mode quorum not reached. have: 2, want: 3", + errString: "unable to reduce on method mode, err: consensus failed: mode quorum not reached, 2 nodes agreed but need at least 3 (2f+1, f=1). DON nodes disagree too much on the value", }, } for _, tt := range cases { diff --git a/pkg/config/errors.go b/pkg/config/errors.go index ba3bef9577..1f533178b4 100644 --- a/pkg/config/errors.go +++ b/pkg/config/errors.go @@ -47,7 +47,7 @@ type KeyNotFoundError struct { } func (e KeyNotFoundError) Error() string { - return fmt.Sprintf("unable to find %s key with id %s", e.KeyType, e.ID) + return fmt.Sprintf("key not found: no %s key exists with ID %q. Verify the key ID is correct and that the key has been created", e.KeyType, e.ID) } // UniqueStrings is a helper for tracking unique values in string form. diff --git a/pkg/loop/internal/goplugin/service.go b/pkg/loop/internal/goplugin/service.go index f8f1557513..2e4eb07d13 100644 --- a/pkg/loop/internal/goplugin/service.go +++ b/pkg/loop/internal/goplugin/service.go @@ -13,7 +13,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/services" ) -var ErrPluginUnavailable = errors.New("plugin unavailable") +var ErrPluginUnavailable = errors.New("plugin unavailable: the capability plugin is not running or has become unresponsive. This is typically a node operator issue - the plugin may need to be restarted") var ( _ services.Service = (*ServiceClient)(nil) diff --git a/pkg/settings/limits/bound_test.go b/pkg/settings/limits/bound_test.go index ed4ad954a0..94bc5611a6 100644 --- a/pkg/settings/limits/bound_test.go +++ b/pkg/settings/limits/bound_test.go @@ -28,7 +28,7 @@ func ExampleBoundLimiter_Check() { fn(4) fn(10) // Output: - // limited: cannot use 11, limit is 10 + // limited: cannot use 11, maximum allowed is 10. Reduce usage or request a limit increase // used 4 // used 10 } diff --git a/pkg/settings/limits/errors.go b/pkg/settings/limits/errors.go index 0aede8454e..72117bcbd5 100644 --- a/pkg/settings/limits/errors.go +++ b/pkg/settings/limits/errors.go @@ -34,7 +34,7 @@ func (e ErrorRateLimited) Is(target error) bool { func (e ErrorRateLimited) Error() string { which, who := errArgs(e.Key, e.Scope, e.Tenant) - msg := fmt.Sprintf("%srate limited%s", which, who) + msg := fmt.Sprintf("%srate limited%s: request rate has exceeded the allowed limit. Please reduce request frequency or wait before retrying", which, who) if e.Err == nil { return msg } @@ -61,7 +61,7 @@ func (e ErrorResourceLimited[N]) Is(target error) bool { func (e ErrorResourceLimited[N]) Error() string { which, who := errArgs(e.Key, e.Scope, e.Tenant) - return fmt.Sprintf("%sresource limited%s: cannot use %v, already using %v/%v", which, who, e.Amount, e.Used, e.Limit) + return fmt.Sprintf("%sresource limited%s: cannot allocate %v, already using %v of %v maximum. Free existing resources or request a limit increase", which, who, e.Amount, e.Used, e.Limit) } type ErrorTimeLimited struct { @@ -84,7 +84,7 @@ func (e ErrorTimeLimited) Is(target error) bool { func (e ErrorTimeLimited) Error() string { which, who := errArgs(e.Key, e.Scope, e.Tenant) - return fmt.Sprintf("%stime limited%s to %s", which, who, e.Timeout) + return fmt.Sprintf("%stime limited%s: operation exceeded the maximum allowed duration of %s. Consider simplifying the operation or requesting a timeout increase", which, who, e.Timeout) } func errArgs(key string, scope settings.Scope, tenant string) (which, who string) { @@ -117,7 +117,7 @@ func (e ErrorBoundLimited[N]) Is(target error) bool { func (e ErrorBoundLimited[N]) Error() string { which, who := errArgs(e.Key, e.Scope, e.Tenant) - return fmt.Sprintf("%slimited%s: cannot use %v, limit is %v", which, who, e.Amount, e.Limit) + return fmt.Sprintf("%slimited%s: cannot use %v, maximum allowed is %v. Reduce usage or request a limit increase", which, who, e.Amount, e.Limit) } type ErrorQueueFull struct { @@ -140,7 +140,7 @@ func (e ErrorQueueFull) Is(target error) bool { func (e ErrorQueueFull) Error() string { which, who := errArgs(e.Key, e.Scope, e.Tenant) - return fmt.Sprintf("%slimited%s: queue of %d is full", which, who, e.Limit) + return fmt.Sprintf("%slimited%s: queue is full (capacity: %d). New items are being rejected (failed to enqueue). Consider reducing submission rate or requesting a capacity increase", which, who, e.Limit) } var ErrQueueEmpty = fmt.Errorf("queue is empty") @@ -163,5 +163,5 @@ func (e ErrorNotAllowed) Is(target error) bool { func (e ErrorNotAllowed) Error() string { which, who := errArgs(e.Key, e.Scope, e.Tenant) - return fmt.Sprintf("%slimited%s: not allowed", which, who) + return fmt.Sprintf("%slimited%s: operation not allowed. This action is restricted by current configuration and gate settings", which, who) } diff --git a/pkg/settings/limits/errors_test.go b/pkg/settings/limits/errors_test.go index e139e340c4..3e5f5a8dbd 100644 --- a/pkg/settings/limits/errors_test.go +++ b/pkg/settings/limits/errors_test.go @@ -31,7 +31,7 @@ func TestErrorRateLimited(t *testing.T) { N: 42, Err: wrapped, }, - exp: "foo rate limited for workflow[wf]: wrapper", + exp: "foo rate limited for workflow[wf]: request rate has exceeded the allowed limit. Please reduce request frequency or wait before retrying: wrapper", }, { name: "no-err", @@ -41,7 +41,7 @@ func TestErrorRateLimited(t *testing.T) { Tenant: "wf", N: 42, }, - exp: "foo rate limited for workflow[wf]", + exp: "foo rate limited for workflow[wf]: request rate has exceeded the allowed limit. Please reduce request frequency or wait before retrying", }, { name: "no-err-tenant", @@ -49,14 +49,14 @@ func TestErrorRateLimited(t *testing.T) { Key: "foo", N: 42, }, - exp: "foo rate limited", + exp: "foo rate limited: request rate has exceeded the allowed limit. Please reduce request frequency or wait before retrying", }, { name: "no-err-tenant-key", err: ErrorRateLimited{ N: 42, }, - exp: "rate limited", + exp: "rate limited: request rate has exceeded the allowed limit. Please reduce request frequency or wait before retrying", }, } { t.Run(tt.name, func(t *testing.T) { @@ -88,7 +88,7 @@ func TestErrorResourceLimited(t *testing.T) { Used: 42, Amount: 13, }, - exp: "foo resource limited for workflow[wf]: cannot use 13, already using 42/100", + exp: "foo resource limited for workflow[wf]: cannot allocate 13, already using 42 of 100 maximum. Free existing resources or request a limit increase", }, { name: "no-tenant", @@ -99,7 +99,7 @@ func TestErrorResourceLimited(t *testing.T) { Used: 42, Amount: 13, }, - exp: "foo resource limited: cannot use 13, already using 42/100", + exp: "foo resource limited: cannot allocate 13, already using 42 of 100 maximum. Free existing resources or request a limit increase", }, { name: "no-tenant-key", @@ -108,7 +108,7 @@ func TestErrorResourceLimited(t *testing.T) { Used: 42, Amount: 13, }, - exp: "resource limited: cannot use 13, already using 42/100", + exp: "resource limited: cannot allocate 13, already using 42 of 100 maximum. Free existing resources or request a limit increase", }, } { t.Run(tt.name, func(t *testing.T) { @@ -138,7 +138,7 @@ func TestErrorTimeLimited(t *testing.T) { Tenant: "wf", Timeout: time.Minute, }, - exp: "foo time limited for workflow[wf] to 1m0s", + exp: "foo time limited for workflow[wf]: operation exceeded the maximum allowed duration of 1m0s. Consider simplifying the operation or requesting a timeout increase", }, { name: "no-tenant", @@ -147,14 +147,14 @@ func TestErrorTimeLimited(t *testing.T) { Scope: settings.ScopeWorkflow, Timeout: time.Minute, }, - exp: "foo time limited to 1m0s", + exp: "foo time limited: operation exceeded the maximum allowed duration of 1m0s. Consider simplifying the operation or requesting a timeout increase", }, { name: "no-tenant-key", err: ErrorTimeLimited{ Timeout: time.Minute, }, - exp: "time limited to 1m0s", + exp: "time limited: operation exceeded the maximum allowed duration of 1m0s. Consider simplifying the operation or requesting a timeout increase", }, } { t.Run(tt.name, func(t *testing.T) { @@ -185,7 +185,7 @@ func TestErrorBoundLimited(t *testing.T) { Limit: 13, Amount: 100, }, - exp: "foo limited for workflow[wf]: cannot use 100, limit is 13", + exp: "foo limited for workflow[wf]: cannot use 100, maximum allowed is 13. Reduce usage or request a limit increase", }, { name: "no-tenant", @@ -195,7 +195,7 @@ func TestErrorBoundLimited(t *testing.T) { Limit: 13, Amount: 100, }, - exp: "foo limited: cannot use 100, limit is 13", + exp: "foo limited: cannot use 100, maximum allowed is 13. Reduce usage or request a limit increase", }, { name: "no-tenant-key", @@ -203,7 +203,7 @@ func TestErrorBoundLimited(t *testing.T) { Limit: 13, Amount: 100, }, - exp: "limited: cannot use 100, limit is 13", + exp: "limited: cannot use 100, maximum allowed is 13. Reduce usage or request a limit increase", }, } { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/settings/limits/gate_test.go b/pkg/settings/limits/gate_test.go index 6e98e8e8bd..a0d8a6d09e 100644 --- a/pkg/settings/limits/gate_test.go +++ b/pkg/settings/limits/gate_test.go @@ -44,7 +44,7 @@ func ExampleGateLimiter_AllowErr() { // open: true // allow: // open: false - // allow: limited: not allowed + // allow: limited: operation not allowed. This action is restricted by current configuration and gate settings } func TestMakeGateLimiter(t *testing.T) { diff --git a/pkg/settings/limits/rate_test.go b/pkg/settings/limits/rate_test.go index dba2dcfa06..ed1ff9b244 100644 --- a/pkg/settings/limits/rate_test.go +++ b/pkg/settings/limits/rate_test.go @@ -96,7 +96,7 @@ func ExampleRateLimiter_AllowErr() { } // Output: - // 5: rate limited + // 5: rate limited: request rate has exceeded the allowed limit. Please reduce request frequency or wait before retrying // 4: success } @@ -168,7 +168,7 @@ func ExampleMultiRateLimiter() { // Output: // A: success - // A&B: rate limited + // A&B: rate limited: request rate has exceeded the allowed limit. Please reduce request frequency or wait before retrying } func TestFactory_NewRateLimiter(t *testing.T) { diff --git a/pkg/settings/limits/resource_test.go b/pkg/settings/limits/resource_test.go index 8e7de3ffb2..66a3cb4a01 100644 --- a/pkg/settings/limits/resource_test.go +++ b/pkg/settings/limits/resource_test.go @@ -67,7 +67,7 @@ func ExampleResourceLimiter_Use() { defer limiter.Free(ctx, 1) // Output: - // Try failed: resource limited: cannot use 1, already using 5/5 + // Try failed: resource limited: cannot allocate 1, already using 5 of 5 maximum. Free existing resources or request a limit increase } func ExampleMultiResourcePoolLimiter() { @@ -118,10 +118,10 @@ func ExampleMultiResourcePoolLimiter() { } free() // Output: - // resource limited: cannot use 10, already using 95/100 - // resource limited for org[org-id]: cannot use 10, already using 45/50 - // resource limited for owner[owner-id]: cannot use 10, already using 15/20 - // resource limited for workflow[workflow-id]: cannot use 10, already using 5/10 + // resource limited: cannot allocate 10, already using 95 of 100 maximum. Free existing resources or request a limit increase + // resource limited for org[org-id]: cannot allocate 10, already using 45 of 50 maximum. Free existing resources or request a limit increase + // resource limited for owner[owner-id]: cannot allocate 10, already using 15 of 20 maximum. Free existing resources or request a limit increase + // resource limited for workflow[workflow-id]: cannot allocate 10, already using 5 of 10 maximum. Free existing resources or request a limit increase // }