From 44a618d254f9806ca7689c9883707495e95d5bc0 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 6 Nov 2025 13:55:15 +0000 Subject: [PATCH 1/6] Accept gzip encoded payloads --- lib/lightning_web/plug_configs.ex | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/lightning_web/plug_configs.ex b/lib/lightning_web/plug_configs.ex index 0a175a806b..f8d469dd26 100644 --- a/lib/lightning_web/plug_configs.ex +++ b/lib/lightning_web/plug_configs.ex @@ -15,7 +15,22 @@ defmodule LightningWeb.PlugConfigs do } ], pass: ["*/*"], + body_reader: {__MODULE__, :decompress_body, []}, json_decoder: Phoenix.json_library() ] end + + @doc """ + Custom body reader that decompresses gzipped request bodies. + """ + def decompress_body(conn, opts) do + case Plug.Conn.get_req_header(conn, "content-encoding") do + ["gzip" | _] -> + {:ok, body, conn} = Plug.Conn.read_body(conn, opts) + {:ok, :zlib.gunzip(body), conn} + + _ -> + Plug.Conn.read_body(conn, opts) + end + end end From 336753801b5cb06f6057b953a757f3b7fc4963be Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 6 Nov 2025 14:02:39 +0000 Subject: [PATCH 2/6] unit test --- test/lightning_web/plug_configs_test.exs | 74 ++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 test/lightning_web/plug_configs_test.exs diff --git a/test/lightning_web/plug_configs_test.exs b/test/lightning_web/plug_configs_test.exs new file mode 100644 index 0000000000..2397b9c243 --- /dev/null +++ b/test/lightning_web/plug_configs_test.exs @@ -0,0 +1,74 @@ +defmodule LightningWeb.PlugConfigsTest do + use LightningWeb.ConnCase, async: true + + alias LightningWeb.PlugConfigs + + describe "decompress_body/2" do + test "decompresses gzipped request body", %{conn: conn} do + json_data = %{"traitors" => ["alan"], "faithfuls" => ["david"]} + json_string = Jason.encode!(json_data) + + # Compress the JSON string + gzipped_body = :zlib.gzip(json_string) + + # Simulate a connection with gzipped body + conn = + conn + |> put_req_header("content-encoding", "gzip") + |> Map.put(:body_params, %{}) + + # Mock the body reading by setting adapter data + conn = %{ + conn + | adapter: {Plug.Adapters.Test.Conn, %{chunks: [gzipped_body]}} + } + + # Call the decompress function + {:ok, decompressed_body, _conn} = PlugConfigs.decompress_body(conn, []) + + assert decompressed_body == json_string + end + + test "passes through non-gzipped request body", %{conn: conn} do + json_data = %{"traitors" => ["alan"], "faithfuls" => ["david"]} + json_string = Jason.encode!(json_data) + + # Simulate a connection without compression + conn = + conn + |> Map.put(:body_params, %{}) + + # Mock the body reading + conn = %{ + conn + | adapter: {Plug.Adapters.Test.Conn, %{chunks: [json_string]}} + } + + # Call the decompress function (should just read normally) + {:ok, body, _conn} = PlugConfigs.decompress_body(conn, []) + + assert body == json_string + end + + test "handles multiple content-encoding headers", %{conn: conn} do + json_string = Jason.encode!(%{"test" => "data"}) + gzipped_body = :zlib.gzip(json_string) + + # Multiple headers with gzip first + conn = + conn + |> put_req_header("content-encoding", "gzip") + |> put_req_header("content-encoding", "identity") + |> Map.put(:body_params, %{}) + + conn = %{ + conn + | adapter: {Plug.Adapters.Test.Conn, %{chunks: [gzipped_body]}} + } + + {:ok, decompressed_body, _conn} = PlugConfigs.decompress_body(conn, []) + + assert decompressed_body == json_string + end + end +end From 37ba9874e25daa4d50da53e9c4a951e121bc6c71 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 6 Nov 2025 14:15:20 +0000 Subject: [PATCH 3/6] refactor --- lib/lightning_web/plug_configs.ex | 16 +---- test/lightning_web/plug_configs_test.exs | 74 ------------------------ 2 files changed, 1 insertion(+), 89 deletions(-) delete mode 100644 test/lightning_web/plug_configs_test.exs diff --git a/lib/lightning_web/plug_configs.ex b/lib/lightning_web/plug_configs.ex index f8d469dd26..aba376975d 100644 --- a/lib/lightning_web/plug_configs.ex +++ b/lib/lightning_web/plug_configs.ex @@ -15,22 +15,8 @@ defmodule LightningWeb.PlugConfigs do } ], pass: ["*/*"], - body_reader: {__MODULE__, :decompress_body, []}, + body_reader: {LightningWeb.Plugs.DecompressBody, :read_body, []}, json_decoder: Phoenix.json_library() ] end - - @doc """ - Custom body reader that decompresses gzipped request bodies. - """ - def decompress_body(conn, opts) do - case Plug.Conn.get_req_header(conn, "content-encoding") do - ["gzip" | _] -> - {:ok, body, conn} = Plug.Conn.read_body(conn, opts) - {:ok, :zlib.gunzip(body), conn} - - _ -> - Plug.Conn.read_body(conn, opts) - end - end end diff --git a/test/lightning_web/plug_configs_test.exs b/test/lightning_web/plug_configs_test.exs deleted file mode 100644 index 2397b9c243..0000000000 --- a/test/lightning_web/plug_configs_test.exs +++ /dev/null @@ -1,74 +0,0 @@ -defmodule LightningWeb.PlugConfigsTest do - use LightningWeb.ConnCase, async: true - - alias LightningWeb.PlugConfigs - - describe "decompress_body/2" do - test "decompresses gzipped request body", %{conn: conn} do - json_data = %{"traitors" => ["alan"], "faithfuls" => ["david"]} - json_string = Jason.encode!(json_data) - - # Compress the JSON string - gzipped_body = :zlib.gzip(json_string) - - # Simulate a connection with gzipped body - conn = - conn - |> put_req_header("content-encoding", "gzip") - |> Map.put(:body_params, %{}) - - # Mock the body reading by setting adapter data - conn = %{ - conn - | adapter: {Plug.Adapters.Test.Conn, %{chunks: [gzipped_body]}} - } - - # Call the decompress function - {:ok, decompressed_body, _conn} = PlugConfigs.decompress_body(conn, []) - - assert decompressed_body == json_string - end - - test "passes through non-gzipped request body", %{conn: conn} do - json_data = %{"traitors" => ["alan"], "faithfuls" => ["david"]} - json_string = Jason.encode!(json_data) - - # Simulate a connection without compression - conn = - conn - |> Map.put(:body_params, %{}) - - # Mock the body reading - conn = %{ - conn - | adapter: {Plug.Adapters.Test.Conn, %{chunks: [json_string]}} - } - - # Call the decompress function (should just read normally) - {:ok, body, _conn} = PlugConfigs.decompress_body(conn, []) - - assert body == json_string - end - - test "handles multiple content-encoding headers", %{conn: conn} do - json_string = Jason.encode!(%{"test" => "data"}) - gzipped_body = :zlib.gzip(json_string) - - # Multiple headers with gzip first - conn = - conn - |> put_req_header("content-encoding", "gzip") - |> put_req_header("content-encoding", "identity") - |> Map.put(:body_params, %{}) - - conn = %{ - conn - | adapter: {Plug.Adapters.Test.Conn, %{chunks: [gzipped_body]}} - } - - {:ok, decompressed_body, _conn} = PlugConfigs.decompress_body(conn, []) - - assert decompressed_body == json_string - end - end -end From e62d74340e0b69dbf26b0683fc4484c5aada0baf Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 6 Nov 2025 14:16:28 +0000 Subject: [PATCH 4/6] changelog --- CHANGELOG.md | 2 + gzip.js | 22 ++++++ lib/lightning_web/plugs/decompress_body.ex | 51 +++++++++++++ .../plugs/decompress_body_test.exs | 74 +++++++++++++++++++ 4 files changed, 149 insertions(+) create mode 100644 gzip.js create mode 100644 lib/lightning_web/plugs/decompress_body.ex create mode 100644 test/lightning_web/plugs/decompress_body_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 0503226c95..30f47591a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ and this project adheres to - Shared doc lookup in clustered environments now works across nodes instead of only searching locally [#3910](https://github.com/OpenFn/lightning/issues/3910) +- Accept gzip-compressed payloads on + webhooks[#3937](https://github.com/OpenFn/lightning/issues/3937) ## [2.14.14] - 2025-11-05 diff --git a/gzip.js b/gzip.js new file mode 100644 index 0000000000..695244770d --- /dev/null +++ b/gzip.js @@ -0,0 +1,22 @@ +import { gzipSync } from "node:zlib"; +const url = "http://localhost:4000/i/ab1801c2-3af0-40cf-8f0c-b743a0afce9b"; + +const json = { + traitors: ["alan"], + faithfuls: ["david"], +}; + +const gzippedData = gzipSync(Buffer.from(JSON.stringify(json))); + +// Post with fetch +const response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Encoding": "gzip", + }, + body: gzippedData, +}); + +console.log("Status:", response.status); +console.log("Response:", await response.text()); diff --git a/lib/lightning_web/plugs/decompress_body.ex b/lib/lightning_web/plugs/decompress_body.ex new file mode 100644 index 0000000000..eec52414b3 --- /dev/null +++ b/lib/lightning_web/plugs/decompress_body.ex @@ -0,0 +1,51 @@ +defmodule LightningWeb.Plugs.DecompressBody do + @moduledoc """ + A Plug body reader that decompresses gzipped request bodies. + + This plug checks for the `Content-Encoding: gzip` header and automatically + decompresses the request body before it's passed to the JSON parser. + + ## Usage + + Add this as a body_reader option to Plug.Parsers: + + plug Plug.Parsers, + parsers: [:json], + body_reader: {LightningWeb.Plugs.DecompressBody, :read_body, []} + + """ + + @doc """ + Reads and optionally decompresses the request body based on Content-Encoding header. + + If the `content-encoding` header is set to `gzip`, the body will be decompressed + using `:zlib.gunzip/1`. Otherwise, the body is read normally without modification. + + ## Parameters + + * `conn` - The `Plug.Conn` struct + * `opts` - Options passed to `Plug.Conn.read_body/2` + + ## Returns + + * `{:ok, body, conn}` - The decompressed (or original) body and updated connection + * `{:more, partial_body, conn}` - For chunked requests (passed through from read_body) + * `{:error, term}` - If reading fails + + """ + def read_body(conn, opts) do + case Plug.Conn.get_req_header(conn, "content-encoding") do + ["gzip" | _] -> + case Plug.Conn.read_body(conn, opts) do + {:ok, body, conn} -> + {:ok, :zlib.gunzip(body), conn} + + other -> + other + end + + _ -> + Plug.Conn.read_body(conn, opts) + end + end +end diff --git a/test/lightning_web/plugs/decompress_body_test.exs b/test/lightning_web/plugs/decompress_body_test.exs new file mode 100644 index 0000000000..bf8f0d6e1f --- /dev/null +++ b/test/lightning_web/plugs/decompress_body_test.exs @@ -0,0 +1,74 @@ +defmodule LightningWeb.Plugs.DecompressBodyTest do + use LightningWeb.ConnCase, async: true + + alias LightningWeb.Plugs.DecompressBody + + describe "read_body/2" do + test "decompresses gzipped request body", %{conn: conn} do + json_data = %{"traitors" => ["alan"], "faithfuls" => ["david"]} + json_string = Jason.encode!(json_data) + + # Compress the JSON string + gzipped_body = :zlib.gzip(json_string) + + # Simulate a connection with gzipped body + conn = + conn + |> put_req_header("content-encoding", "gzip") + |> Map.put(:body_params, %{}) + + # Mock the body reading by setting adapter data + conn = %{ + conn + | adapter: {Plug.Adapters.Test.Conn, %{chunks: [gzipped_body]}} + } + + # Call the decompress function + {:ok, decompressed_body, _conn} = DecompressBody.read_body(conn, []) + + assert decompressed_body == json_string + end + + test "passes through non-gzipped request body", %{conn: conn} do + json_data = %{"traitors" => ["alan"], "faithfuls" => ["david"]} + json_string = Jason.encode!(json_data) + + # Simulate a connection without compression + conn = + conn + |> Map.put(:body_params, %{}) + + # Mock the body reading + conn = %{ + conn + | adapter: {Plug.Adapters.Test.Conn, %{chunks: [json_string]}} + } + + # Call the read_body function (should just read normally) + {:ok, body, _conn} = DecompressBody.read_body(conn, []) + + assert body == json_string + end + + test "handles multiple content-encoding headers", %{conn: conn} do + json_string = Jason.encode!(%{"test" => "data"}) + gzipped_body = :zlib.gzip(json_string) + + # Multiple headers with gzip first + conn = + conn + |> put_req_header("content-encoding", "gzip") + |> put_req_header("content-encoding", "identity") + |> Map.put(:body_params, %{}) + + conn = %{ + conn + | adapter: {Plug.Adapters.Test.Conn, %{chunks: [gzipped_body]}} + } + + {:ok, decompressed_body, _conn} = DecompressBody.read_body(conn, []) + + assert decompressed_body == json_string + end + end +end From 8177130408a4effbce018decc46a53e05c7146bb Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 6 Nov 2025 14:16:47 +0000 Subject: [PATCH 5/6] tidy --- gzip.js | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 gzip.js diff --git a/gzip.js b/gzip.js deleted file mode 100644 index 695244770d..0000000000 --- a/gzip.js +++ /dev/null @@ -1,22 +0,0 @@ -import { gzipSync } from "node:zlib"; -const url = "http://localhost:4000/i/ab1801c2-3af0-40cf-8f0c-b743a0afce9b"; - -const json = { - traitors: ["alan"], - faithfuls: ["david"], -}; - -const gzippedData = gzipSync(Buffer.from(JSON.stringify(json))); - -// Post with fetch -const response = await fetch(url, { - method: "POST", - headers: { - "Content-Type": "application/json", - "Content-Encoding": "gzip", - }, - body: gzippedData, -}); - -console.log("Status:", response.status); -console.log("Response:", await response.text()); From 3468386a73ab9e902ab98bd4902bd03012989098 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 6 Nov 2025 14:17:53 +0000 Subject: [PATCH 6/6] remove docs --- lib/lightning_web/plugs/decompress_body.ex | 32 +--------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/lib/lightning_web/plugs/decompress_body.ex b/lib/lightning_web/plugs/decompress_body.ex index eec52414b3..0be3ab4659 100644 --- a/lib/lightning_web/plugs/decompress_body.ex +++ b/lib/lightning_web/plugs/decompress_body.ex @@ -1,37 +1,7 @@ defmodule LightningWeb.Plugs.DecompressBody do @moduledoc """ - A Plug body reader that decompresses gzipped request bodies. - - This plug checks for the `Content-Encoding: gzip` header and automatically + A Plug body reader that decompresses gzipped request bodies. Checks for the `Content-Encoding: gzip` header and automatically decompresses the request body before it's passed to the JSON parser. - - ## Usage - - Add this as a body_reader option to Plug.Parsers: - - plug Plug.Parsers, - parsers: [:json], - body_reader: {LightningWeb.Plugs.DecompressBody, :read_body, []} - - """ - - @doc """ - Reads and optionally decompresses the request body based on Content-Encoding header. - - If the `content-encoding` header is set to `gzip`, the body will be decompressed - using `:zlib.gunzip/1`. Otherwise, the body is read normally without modification. - - ## Parameters - - * `conn` - The `Plug.Conn` struct - * `opts` - Options passed to `Plug.Conn.read_body/2` - - ## Returns - - * `{:ok, body, conn}` - The decompressed (or original) body and updated connection - * `{:more, partial_body, conn}` - For chunked requests (passed through from read_body) - * `{:error, term}` - If reading fails - """ def read_body(conn, opts) do case Plug.Conn.get_req_header(conn, "content-encoding") do