Honor Sentry rate limiting globally (#953)

Author: dnsbty

config/config.exs

@@ -9,6 +9,7 @@ if config_env() == :test do
     hackney_opts: [recv_timeout: 50, pool: :sentry_pool],
     send_result: :sync,
     send_max_attempts: 1,
+    start_rate_limiter: false,
     dedup_events: false,
     test_mode: true,
     traces_sample_rate: 1.0

lib/sentry/application.ex

@@ -33,6 +33,14 @@ defmodule Sentry.Application do
         []
       end
 
+    # Don't start RateLimiter in test environment - tests start their own instances
+    maybe_rate_limiter =
+      if Application.get_env(:sentry, :start_rate_limiter, true) == false do
+        []
+      else
+        [Sentry.Transport.RateLimiter]
+      end
+
     children =
       [
         {Registry, keys: :unique, name: Sentry.Transport.SenderRegistry},
@@ -47,6 +55,7 @@ defmodule Sentry.Application do
       ] ++
         maybe_http_client_spec ++
         maybe_span_storage ++
+        maybe_rate_limiter ++
         [Sentry.Transport.SenderPool]
 
     cache_loaded_applications()

lib/sentry/client_error.ex

@@ -33,6 +33,7 @@ defmodule Sentry.ClientError do
   """
   @type reason() ::
           :too_many_retries
+          | :rate_limited
           | :server_error
           | {:invalid_json, Exception.t()}
           | {:request_failure, reason :: :inet.posix() | term()}
@@ -73,6 +74,10 @@ defmodule Sentry.ClientError do
     "Sentry responded with status 429 - Too Many Requests and the SDK exhausted the configured retries"
   end
 
+  defp format(:rate_limited) do
+    "the event was dropped because the category is currently rate-limited by Sentry"
+  end
+
   defp format({:invalid_json, reason}) do
     formatted =
       if is_exception(reason) do

lib/sentry/client_report/sender.ex

@@ -14,7 +14,7 @@ defmodule Sentry.ClientReport.Sender do
 
   @spec start_link([]) :: GenServer.on_start()
   def start_link(opts \\ []) do
-    GenServer.start_link(__MODULE__, nil, name: Keyword.get(opts, :name, __MODULE__))
+    GenServer.start_link(__MODULE__, opts, name: Keyword.get(opts, :name, __MODULE__))
   end
 
   def record_discarded_events(reason, info, genserver \\ __MODULE__)
@@ -51,8 +51,13 @@ defmodule Sentry.ClientReport.Sender do
   ## Callbacks
 
   @impl true
-  def init(nil) do
+  def init(opts) do
     schedule_report()
+
+    if rate_limiter_table_name = Keyword.get(opts, :rate_limiter_table_name) do
+      Process.put(:rate_limiter_table_name, rate_limiter_table_name)
+    end
+
     {:ok, _state = %{}}
   end
 

lib/sentry/transport.ex

@@ -4,6 +4,7 @@ defmodule Sentry.Transport do
   # This module is exclusively responsible for encoding and POSTing envelopes to Sentry.
 
   alias Sentry.{ClientError, ClientReport, Config, Envelope, LoggerUtils}
+  alias Sentry.Transport.RateLimiter
 
   @default_retries [1000, 2000, 4000, 8000]
   @sentry_version 5
@@ -47,27 +48,13 @@ defmodule Sentry.Transport do
          retries_left,
          envelope_items
        ) do
-    case request(client, endpoint, headers, payload) do
+    case request(client, endpoint, headers, payload, envelope_items) do
       {:ok, id} ->
         {:ok, id}
 
-      # If Sentry gives us a Retry-After header, we listen to that instead of our
-      # own retry.
-      {:retry_after, delay_ms} when retries_left != [] ->
-        Process.sleep(delay_ms)
-
-        post_envelope_with_retries(
-          client,
-          endpoint,
-          headers,
-          payload,
-          tl(retries_left),
-          envelope_items
-        )
-
-      {:retry_after, _delay_ms} ->
+      {:error, :rate_limited} ->
         ClientReport.Sender.record_discarded_events(:ratelimit_backoff, envelope_items)
-        {:error, ClientError.new(:too_many_retries)}
+        {:error, ClientError.new(:rate_limited)}
 
       {:error, _reason} when retries_left != [] ->
         [sleep_interval | retries_left] = retries_left
@@ -92,29 +79,32 @@ defmodule Sentry.Transport do
     end
   end
 
-  defp request(client, endpoint, headers, body) do
-    with {:ok, 200, _headers, body} <-
+  defp check_rate_limited(envelope_items) do
+    rate_limited? =
+      Enum.any?(envelope_items, fn item ->
+        category = Envelope.get_data_category(item)
+        RateLimiter.rate_limited?(category)
+      end)
+
+    if rate_limited?, do: {:error, :rate_limited}, else: :ok
+  end
+
+  defp request(client, endpoint, headers, body, envelope_items) do
+    with :ok <- check_rate_limited(envelope_items),
+         {:ok, 200, _headers, body} <-
            client_post_and_validate_return_value(client, endpoint, headers, body),
          {:ok, json} <- Sentry.JSON.decode(body, Config.json_library()) do
       {:ok, Map.get(json, "id")}
     else
-      {:ok, 429, headers, _body} ->
-        delay_ms =
-          with timeout when is_binary(timeout) <-
-                 :proplists.get_value("Retry-After", headers, nil),
-               {delay_s, ""} <- Integer.parse(timeout) do
-            delay_s * 1000
-          else
-            _ ->
-              # https://develop.sentry.dev/sdk/rate-limiting/#stage-1-parse-response-headers
-              60_000
-          end
-
-        {:retry_after, delay_ms}
+      {:ok, 429, _headers, _body} ->
+        {:error, :rate_limited}
 
       {:ok, status, headers, body} ->
         {:error, {:http, {status, headers, body}}}
 
+      {:error, :rate_limited} ->
+        {:error, :rate_limited}
+
       {:error, reason} ->
         {:error, {:request_failure, reason}}
     end
@@ -127,6 +117,7 @@ defmodule Sentry.Transport do
       {:ok, status, resp_headers, resp_body}
       when is_integer(status) and status in 200..599 and is_list(resp_headers) and
              is_binary(resp_body) ->
+        update_rate_limits(resp_headers, status)
         {:ok, status, resp_headers, resp_body}
 
       {:ok, status, resp_headers, resp_body} ->
@@ -137,6 +128,35 @@ defmodule Sentry.Transport do
     end
   end
 
+  defp update_rate_limits(headers, status) do
+    rate_limits_header = :proplists.get_value("X-Sentry-Rate-Limits", headers, nil)
+
+    cond do
+      is_binary(rate_limits_header) ->
+        # Use categorized rate limits if present
+        RateLimiter.update_rate_limits(rate_limits_header)
+
+      status == 429 ->
+        # Use global rate limit from Retry-After if no categorized limits are present
+        delay_seconds = get_global_delay(headers)
+        RateLimiter.update_global_rate_limit(delay_seconds)
+
+      true ->
+        :ok
+    end
+  end
+
+  defp get_global_delay(headers) do
+    with timeout when is_binary(timeout) <- :proplists.get_value("Retry-After", headers, nil),
+         {delay, ""} <- Integer.parse(timeout) do
+      delay
+    else
+      # Per the spec, if Retry-After is missing or malformed, default to 60 seconds
+      # https://develop.sentry.dev/sdk/rate-limiting/#stage-1-parse-response-headers
+      _ -> 60
+    end
+  end
+
   defp get_endpoint_and_headers do
     %Sentry.DSN{} = dsn = Config.dsn()