From c3c405ae23ede4abc0b3e8dc80072cfa51de7248 Mon Sep 17 00:00:00 2001 From: Dustin Byrne Date: Wed, 17 Dec 2025 16:33:02 -0500 Subject: [PATCH 1/3] feat: Set `$feature_flag_error` on `$feature_flag_called` If an error occurs while fetching flags, the property will be set to indicate what happened. --- lib/posthog.rb | 1 + lib/posthog/client.rb | 3 +- lib/posthog/feature_flag_error.rb | 36 ++++ lib/posthog/feature_flags.rb | 46 +++- spec/posthog/feature_flag_error_spec.rb | 274 ++++++++++++++++++++++++ 5 files changed, 356 insertions(+), 4 deletions(-) create mode 100644 lib/posthog/feature_flag_error.rb create mode 100644 spec/posthog/feature_flag_error_spec.rb diff --git a/lib/posthog.rb b/lib/posthog.rb index 275086f..618aa9f 100644 --- a/lib/posthog.rb +++ b/lib/posthog.rb @@ -10,3 +10,4 @@ require 'posthog/response' require 'posthog/logging' require 'posthog/exception_capture' +require 'posthog/feature_flag_error' diff --git a/lib/posthog/client.rb b/lib/posthog/client.rb index 3bdd957..e9952f5 100644 --- a/lib/posthog/client.rb +++ b/lib/posthog/client.rb @@ -285,7 +285,7 @@ def get_feature_flag( person_properties, group_properties ) - feature_flag_response, flag_was_locally_evaluated, request_id, evaluated_at = + feature_flag_response, flag_was_locally_evaluated, request_id, evaluated_at, feature_flag_error = @feature_flags_poller.get_feature_flag( key, distinct_id, @@ -304,6 +304,7 @@ def get_feature_flag( } properties['$feature_flag_request_id'] = request_id if request_id properties['$feature_flag_evaluated_at'] = evaluated_at if evaluated_at + properties['$feature_flag_error'] = feature_flag_error if feature_flag_error capture( { diff --git a/lib/posthog/feature_flag_error.rb b/lib/posthog/feature_flag_error.rb new file mode 100644 index 0000000..3430ad3 --- /dev/null +++ b/lib/posthog/feature_flag_error.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module PostHog + # Error type constants for the $feature_flag_error property. + # + # These values are sent in analytics events to track flag evaluation failures. + # They should not be changed without considering impact on existing dashboards + # and queries that filter on these values. + # + # Error values: + # ERRORS_WHILE_COMPUTING: Server returned errorsWhileComputingFlags=true + # FLAG_MISSING: Requested flag not in API response + # QUOTA_LIMITED: Rate/quota limit exceeded + # TIMEOUT: Request timed out + # CONNECTION_ERROR: Network connectivity issue + # UNKNOWN_ERROR: Unexpected exceptions + # + # For API errors with status codes, use the api_error() method which returns + # a string like "api_error_500". + class FeatureFlagError + ERRORS_WHILE_COMPUTING = 'errors_while_computing_flags' + FLAG_MISSING = 'flag_missing' + QUOTA_LIMITED = 'quota_limited' + TIMEOUT = 'timeout' + CONNECTION_ERROR = 'connection_error' + UNKNOWN_ERROR = 'unknown_error' + + # Generate API error string with status code. + # + # @param status [Integer, String] The HTTP status code + # @return [String] Error string in format "api_error_STATUS" + def self.api_error(status) + "api_error_#{status}" + end + end +end diff --git a/lib/posthog/feature_flags.rb b/lib/posthog/feature_flags.rb index 8e15898..88310be 100644 --- a/lib/posthog/feature_flags.rb +++ b/lib/posthog/feature_flags.rb @@ -193,9 +193,11 @@ def get_feature_flag( request_id = nil evaluated_at = nil + feature_flag_error = nil if !flag_was_locally_evaluated && !only_evaluate_locally begin + errors = [] flags_data = get_all_flags_and_payloads(distinct_id, groups, person_properties, group_properties, only_evaluate_locally, true) if flags_data.key?(:featureFlags) @@ -207,15 +209,41 @@ def get_feature_flag( flags = {} end + status = flags_data[:status] + if status && status >= 400 + errors << FeatureFlagError.api_error(status) + end + + if flags_data[:errorsWhileComputingFlags] + errors << FeatureFlagError::ERRORS_WHILE_COMPUTING + end + + if flags_data[:quotaLimited]&.include?('feature_flags') + errors << FeatureFlagError::QUOTA_LIMITED + end + + unless flags.key?(key.to_s) + errors << FeatureFlagError::FLAG_MISSING + end + response = flags[key] response = false if response.nil? + feature_flag_error = errors.join(',') unless errors.empty? + logger.debug "Successfully computed flag remotely: #{key} -> #{response}" + rescue Timeout::Error, Net::ReadTimeout, Net::WriteTimeout + @on_error.call(-1, 'Timeout while fetching flags remotely') + feature_flag_error = FeatureFlagError::TIMEOUT + rescue Errno::ECONNRESET, Errno::ECONNREFUSED, EOFError, SocketError => e + @on_error.call(-1, "Connection error while fetching flags remotely: #{e}") + feature_flag_error = FeatureFlagError::CONNECTION_ERROR rescue StandardError => e @on_error.call(-1, "Error computing flag remotely: #{e}. #{e.backtrace.join("\n")}") + feature_flag_error = FeatureFlagError::UNKNOWN_ERROR end end - [response, flag_was_locally_evaluated, request_id, evaluated_at] + [response, flag_was_locally_evaluated, request_id, evaluated_at, feature_flag_error] end def get_all_flags( @@ -270,19 +298,28 @@ def get_all_flags_and_payloads( fallback_to_server = true end + errors_while_computing = false + quota_limited = nil + status_code = nil + if fallback_to_server && !only_evaluate_locally begin flags_and_payloads = get_flags(distinct_id, groups, person_properties, group_properties) + errors_while_computing = flags_and_payloads[:errorsWhileComputingFlags] || false + quota_limited = flags_and_payloads[:quotaLimited] + status_code = flags_and_payloads[:status] unless flags_and_payloads.key?(:featureFlags) raise StandardError, "Error flags response: #{flags_and_payloads}" end # Check if feature_flags are quota limited - if flags_and_payloads[:quotaLimited]&.include?('feature_flags') + if quota_limited&.include?('feature_flags') logger.warn '[FEATURE FLAGS] Quota limited for feature flags' flags = {} payloads = {} + request_id = flags_and_payloads[:requestId] + evaluated_at = flags_and_payloads[:evaluatedAt] else flags = stringify_keys(flags_and_payloads[:featureFlags] || {}) payloads = stringify_keys(flags_and_payloads[:featureFlagPayloads] || {}) @@ -299,7 +336,10 @@ def get_all_flags_and_payloads( featureFlags: flags, featureFlagPayloads: payloads, requestId: request_id, - evaluatedAt: evaluated_at + evaluatedAt: evaluated_at, + errorsWhileComputingFlags: errors_while_computing, + quotaLimited: quota_limited, + status: status_code } end diff --git a/spec/posthog/feature_flag_error_spec.rb b/spec/posthog/feature_flag_error_spec.rb new file mode 100644 index 0000000..0e60dab --- /dev/null +++ b/spec/posthog/feature_flag_error_spec.rb @@ -0,0 +1,274 @@ +# frozen_string_literal: true + +require 'spec_helper' + +module PostHog + describe 'Feature Flag Error Tracking' do + let(:flags_endpoint) { 'https://app.posthog.com/flags/?v=2' } + let(:feature_flag_endpoint) { 'https://app.posthog.com/api/feature_flag/local_evaluation?token=testsecret&send_cohorts=true' } + let(:client) { Client.new(api_key: API_KEY, personal_api_key: API_KEY, test_mode: true) } + + before do + # Stub the initial feature flag definitions request + stub_request(:get, feature_flag_endpoint) + .to_return(status: 200, body: { flags: [] }.to_json) + end + + describe '$feature_flag_error property' do + context 'when flag is missing from response' do + it 'adds flag_missing error to $feature_flag_called event' do + # Mock response without the requested flag + flags_response = { + 'featureFlags' => { 'other-flag' => true }, + 'featureFlagPayloads' => {} + } + + stub_request(:post, flags_endpoint) + .to_return(status: 200, body: flags_response.to_json) + + result = client.get_feature_flag('missing-flag', 'test-user') + + expect(result).to eq(false) + + captured_message = client.dequeue_last_message + expect(captured_message[:event]).to eq('$feature_flag_called') + expect(captured_message[:properties]['$feature_flag']).to eq('missing-flag') + expect(captured_message[:properties]['$feature_flag_error']).to eq(FeatureFlagError::FLAG_MISSING) + end + end + + context 'when server returns errorsWhileComputingFlags' do + it 'adds errors_while_computing_flags error to $feature_flag_called event' do + flags_response = { + 'featureFlags' => { 'test-flag' => true }, + 'featureFlagPayloads' => {}, + 'errorsWhileComputingFlags' => true + } + + stub_request(:post, flags_endpoint) + .to_return(status: 200, body: flags_response.to_json) + + result = client.get_feature_flag('test-flag', 'test-user') + + expect(result).to eq(true) + + captured_message = client.dequeue_last_message + expect(captured_message[:event]).to eq('$feature_flag_called') + expect(captured_message[:properties]['$feature_flag_error']).to eq(FeatureFlagError::ERRORS_WHILE_COMPUTING) + end + end + + context 'when quota limited' do + it 'adds quota_limited and flag_missing errors to $feature_flag_called event' do + # When quota limited, the response includes quotaLimited field and empty flags + flags_response = { + 'featureFlags' => {}, + 'featureFlagPayloads' => {}, + 'quotaLimited' => ['feature_flags'] + } + + stub_request(:post, flags_endpoint) + .to_return(status: 200, body: flags_response.to_json) + + result = client.get_feature_flag('test-flag', 'test-user') + + # Flag is nil because quota limiting returns empty flags + expect(result).to eq(false) + + captured_message = client.dequeue_last_message + expect(captured_message[:event]).to eq('$feature_flag_called') + # Both quota_limited and flag_missing are reported since the flag is not in the empty response + expected_error = "#{FeatureFlagError::QUOTA_LIMITED},#{FeatureFlagError::FLAG_MISSING}" + expect(captured_message[:properties]['$feature_flag_error']).to eq(expected_error) + end + end + + context 'when both errorsWhileComputingFlags and flag_missing occur' do + it 'joins multiple errors with commas' do + flags_response = { + 'featureFlags' => { 'other-flag' => true }, + 'featureFlagPayloads' => {}, + 'errorsWhileComputingFlags' => true + } + + stub_request(:post, flags_endpoint) + .to_return(status: 200, body: flags_response.to_json) + + result = client.get_feature_flag('missing-flag', 'test-user') + + expect(result).to eq(false) + + captured_message = client.dequeue_last_message + expect(captured_message[:event]).to eq('$feature_flag_called') + expected_error = "#{FeatureFlagError::ERRORS_WHILE_COMPUTING},#{FeatureFlagError::FLAG_MISSING}" + expect(captured_message[:properties]['$feature_flag_error']).to eq(expected_error) + end + end + + context 'when API returns error status code' do + it 'adds api_error_500 for server error' do + stub_request(:post, flags_endpoint) + .to_return(status: 500, body: { 'featureFlags' => {} }.to_json) + + result = client.get_feature_flag('test-flag', 'test-user') + + expect(result).to eq(false) + + captured_message = client.dequeue_last_message + expect(captured_message[:event]).to eq('$feature_flag_called') + expect(captured_message[:properties]['$feature_flag_error']).to eq('api_error_500,flag_missing') + end + + it 'adds api_error_503 for service unavailable' do + stub_request(:post, flags_endpoint) + .to_return(status: 503, body: { 'featureFlags' => { 'test-flag' => true } }.to_json) + + result = client.get_feature_flag('test-flag', 'test-user') + + expect(result).to eq(true) + + captured_message = client.dequeue_last_message + expect(captured_message[:event]).to eq('$feature_flag_called') + expect(captured_message[:properties]['$feature_flag_error']).to eq('api_error_503') + end + + it 'adds api_error_400 for bad request' do + stub_request(:post, flags_endpoint) + .to_return(status: 400, body: { 'featureFlags' => {} }.to_json) + + result = client.get_feature_flag('test-flag', 'test-user') + + captured_message = client.dequeue_last_message + expect(captured_message[:properties]['$feature_flag_error']).to eq('api_error_400,flag_missing') + end + end + + context 'when request fails completely' do + it 'adds timeout error to $feature_flag_called event on timeout' do + stub_request(:post, flags_endpoint) + .to_timeout + + result = client.get_feature_flag('test-flag', 'test-user') + + expect(result).to eq(nil) + + captured_message = client.dequeue_last_message + expect(captured_message[:event]).to eq('$feature_flag_called') + expect(captured_message[:properties]['$feature_flag_error']).to eq(FeatureFlagError::TIMEOUT) + end + + it 'adds connection_error to $feature_flag_called event on connection error' do + stub_request(:post, flags_endpoint) + .to_raise(Errno::ECONNREFUSED) + + result = client.get_feature_flag('test-flag', 'test-user') + + expect(result).to eq(nil) + + captured_message = client.dequeue_last_message + expect(captured_message[:event]).to eq('$feature_flag_called') + expect(captured_message[:properties]['$feature_flag_error']).to eq(FeatureFlagError::CONNECTION_ERROR) + end + + it 'adds unknown_error to $feature_flag_called event on unexpected error' do + stub_request(:post, flags_endpoint) + .to_raise(StandardError.new('Unexpected error')) + + result = client.get_feature_flag('test-flag', 'test-user') + + expect(result).to eq(nil) + + captured_message = client.dequeue_last_message + expect(captured_message[:event]).to eq('$feature_flag_called') + expect(captured_message[:properties]['$feature_flag_error']).to eq(FeatureFlagError::UNKNOWN_ERROR) + end + end + + context 'when request succeeds with no errors' do + it 'does not add $feature_flag_error property' do + flags_response = { + 'featureFlags' => { 'test-flag' => true }, + 'featureFlagPayloads' => {} + } + + stub_request(:post, flags_endpoint) + .to_return(status: 200, body: flags_response.to_json) + + result = client.get_feature_flag('test-flag', 'test-user') + + expect(result).to eq(true) + + captured_message = client.dequeue_last_message + expect(captured_message[:event]).to eq('$feature_flag_called') + expect(captured_message[:properties]).not_to have_key('$feature_flag_error') + end + end + + context 'when local evaluation succeeds' do + it 'does not add $feature_flag_error property' do + api_feature_flag_res = { + 'flags' => [ + { + 'id' => 1, + 'name' => 'Beta Feature', + 'key' => 'beta-feature', + 'active' => true, + 'is_simple_flag' => true, + 'rollout_percentage' => 100, + 'filters' => { + 'groups' => [ + { + 'properties' => [], + 'rollout_percentage' => 100 + } + ] + } + } + ] + } + + stub_request(:get, feature_flag_endpoint) + .to_return(status: 200, body: api_feature_flag_res.to_json) + + new_client = Client.new(api_key: API_KEY, personal_api_key: API_KEY, test_mode: true) + + new_client.get_feature_flag('beta-feature', 'test-user') + + captured_message = new_client.dequeue_last_message + expect(captured_message[:event]).to eq('$feature_flag_called') + expect(captured_message[:properties]['locally_evaluated']).to eq(true) + expect(captured_message[:properties]).not_to have_key('$feature_flag_error') + end + end + + context 'when send_feature_flag_events is false' do + it 'does not capture events even on error' do + stub_request(:post, flags_endpoint) + .to_raise(StandardError.new('Network error')) + + result = client.get_feature_flag('test-flag', 'test-user', send_feature_flag_events: false) + + expect(result).to eq(nil) + expect(client.queued_messages).to eq(0) + end + end + end + + describe 'FeatureFlagError constants' do + it 'has all required error constants' do + expect(FeatureFlagError::ERRORS_WHILE_COMPUTING).to eq('errors_while_computing_flags') + expect(FeatureFlagError::FLAG_MISSING).to eq('flag_missing') + expect(FeatureFlagError::QUOTA_LIMITED).to eq('quota_limited') + expect(FeatureFlagError::TIMEOUT).to eq('timeout') + expect(FeatureFlagError::CONNECTION_ERROR).to eq('connection_error') + expect(FeatureFlagError::UNKNOWN_ERROR).to eq('unknown_error') + end + + it 'generates api_error strings with status codes' do + expect(FeatureFlagError.api_error(500)).to eq('api_error_500') + expect(FeatureFlagError.api_error(404)).to eq('api_error_404') + expect(FeatureFlagError.api_error(503)).to eq('api_error_503') + end + end + end +end From a0ec1e3204f23c545914cb4f333662056d646f93 Mon Sep 17 00:00:00 2001 From: Dustin Byrne Date: Fri, 19 Dec 2025 14:42:59 -0500 Subject: [PATCH 2/3] chore: Fix up rubocop suggestions --- lib/posthog/feature_flags.rb | 30 ++++++++----------------- spec/posthog/feature_flag_error_spec.rb | 2 +- 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/lib/posthog/feature_flags.rb b/lib/posthog/feature_flags.rb index 88310be..119b928 100644 --- a/lib/posthog/feature_flags.rb +++ b/lib/posthog/feature_flags.rb @@ -210,29 +210,18 @@ def get_feature_flag( end status = flags_data[:status] - if status && status >= 400 - errors << FeatureFlagError.api_error(status) - end - - if flags_data[:errorsWhileComputingFlags] - errors << FeatureFlagError::ERRORS_WHILE_COMPUTING - end - - if flags_data[:quotaLimited]&.include?('feature_flags') - errors << FeatureFlagError::QUOTA_LIMITED - end - - unless flags.key?(key.to_s) - errors << FeatureFlagError::FLAG_MISSING - end + errors << FeatureFlagError.api_error(status) if status && status >= 400 + errors << FeatureFlagError::ERRORS_WHILE_COMPUTING if flags_data[:errorsWhileComputingFlags] + errors << FeatureFlagError::QUOTA_LIMITED if flags_data[:quotaLimited]&.include?('feature_flags') + errors << FeatureFlagError::FLAG_MISSING unless flags.key?(key.to_s) response = flags[key] response = false if response.nil? feature_flag_error = errors.join(',') unless errors.empty? logger.debug "Successfully computed flag remotely: #{key} -> #{response}" - rescue Timeout::Error, Net::ReadTimeout, Net::WriteTimeout - @on_error.call(-1, 'Timeout while fetching flags remotely') + rescue Timeout::Error => e + @on_error.call(-1, "Timeout while fetching flags remotely: #{e}") feature_flag_error = FeatureFlagError::TIMEOUT rescue Errno::ECONNRESET, Errno::ECONNREFUSED, EOFError, SocketError => e @on_error.call(-1, "Connection error while fetching flags remotely: #{e}") @@ -313,18 +302,17 @@ def get_all_flags_and_payloads( raise StandardError, "Error flags response: #{flags_and_payloads}" end + request_id = flags_and_payloads[:requestId] + evaluated_at = flags_and_payloads[:evaluatedAt] + # Check if feature_flags are quota limited if quota_limited&.include?('feature_flags') logger.warn '[FEATURE FLAGS] Quota limited for feature flags' flags = {} payloads = {} - request_id = flags_and_payloads[:requestId] - evaluated_at = flags_and_payloads[:evaluatedAt] else flags = stringify_keys(flags_and_payloads[:featureFlags] || {}) payloads = stringify_keys(flags_and_payloads[:featureFlagPayloads] || {}) - request_id = flags_and_payloads[:requestId] - evaluated_at = flags_and_payloads[:evaluatedAt] end rescue StandardError => e @on_error.call(-1, "Error computing flag remotely: #{e}") diff --git a/spec/posthog/feature_flag_error_spec.rb b/spec/posthog/feature_flag_error_spec.rb index 0e60dab..eb53d95 100644 --- a/spec/posthog/feature_flag_error_spec.rb +++ b/spec/posthog/feature_flag_error_spec.rb @@ -136,7 +136,7 @@ module PostHog stub_request(:post, flags_endpoint) .to_return(status: 400, body: { 'featureFlags' => {} }.to_json) - result = client.get_feature_flag('test-flag', 'test-user') + client.get_feature_flag('test-flag', 'test-user') captured_message = client.dequeue_last_message expect(captured_message[:properties]['$feature_flag_error']).to eq('api_error_400,flag_missing') From d41dbcee2d92517fa69f06077a5f4fc992057e57 Mon Sep 17 00:00:00 2001 From: Dustin Byrne Date: Fri, 19 Dec 2025 14:44:11 -0500 Subject: [PATCH 3/3] fix: Always downcase the api error status Considering it can be a string --- lib/posthog/feature_flag_error.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/posthog/feature_flag_error.rb b/lib/posthog/feature_flag_error.rb index 3430ad3..7a1319e 100644 --- a/lib/posthog/feature_flag_error.rb +++ b/lib/posthog/feature_flag_error.rb @@ -30,7 +30,7 @@ class FeatureFlagError # @param status [Integer, String] The HTTP status code # @return [String] Error string in format "api_error_STATUS" def self.api_error(status) - "api_error_#{status}" + "api_error_#{status.to_s.downcase}" end end end