From c92cc346db4695321b4f90261f20e8956eba30ad Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 3 Dec 2025 03:59:46 +0100 Subject: [PATCH] feat(gateway): add AllowCodecConversion config option Add AllowCodecConversion to gateway.Config to control codec conversion behavior per IPIP-0524. When false (default), the gateway returns 406 Not Acceptable if the requested format doesn't match the block's codec. When true, conversions between codecs are performed for backward compatibility. Codec conversion tests moved here from gateway-conformance since conversions are now an optional implementation feature, not a spec requirement. Gateway-conformance now tests for 406 responses. Ref: https://github.com/ipfs/specs/pull/524 Ref: https://github.com/ipfs/gateway-conformance/pull/254 --- .github/workflows/gateway-conformance.yml | 12 ++-- CHANGELOG.md | 2 + examples/gateway/car-file/main_test.go | 56 +++++++---------- examples/go.mod | 2 +- gateway/gateway.go | 13 ++++ gateway/gateway_test.go | 59 +++++++++++++++++- gateway/handler_codec.go | 15 ++++- .../path_gateway_dag/dag-cbor-traversal.car | Bin 0 -> 318 bytes gateway/utilities_test.go | 1 + 9 files changed, 115 insertions(+), 45 deletions(-) create mode 100644 gateway/testdata/path_gateway_dag/dag-cbor-traversal.car diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 47bb861e7..6a8d97de1 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -22,7 +22,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.8 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: output: fixtures merged: true @@ -47,7 +47,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@v0.8 + uses: ipfs/gateway-conformance/.github/actions/test@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: gateway-url: http://127.0.0.1:8040 subdomain-url: http://example.net:8040 @@ -84,7 +84,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.8 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: output: fixtures merged: true @@ -114,7 +114,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@v0.8 + uses: ipfs/gateway-conformance/.github/actions/test@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote block gateway subdomain-url: http://example.net:8040 @@ -152,7 +152,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.8 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: output: fixtures merged: true @@ -182,7 +182,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@v0.8 + uses: ipfs/gateway-conformance/.github/actions/test@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote car gateway subdomain-url: http://example.net:8040 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b6576431..56b9a6712 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ The following emojis are used to highlight certain changes: ### Changed +- `gateway`: 🛠 Codec conversions (e.g., dag-pb to dag-json, dag-json to dag-cbor) are no longer performed by default per [IPIP-0524](https://github.com/ipfs/specs/pull/524). Requesting a format that differs from the block's codec now returns HTTP 406 Not Acceptable. Clients should fetch raw blocks (`?format=raw`) and convert in userland. Set `Config.AllowCodecConversion` to `true` to restore the old behavior. + ### Removed ### Fixed diff --git a/examples/gateway/car-file/main_test.go b/examples/gateway/car-file/main_test.go index 6c8fc22d7..6e611d819 100644 --- a/examples/gateway/car-file/main_test.go +++ b/examples/gateway/car-file/main_test.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "io" "net/http" "net/http/httptest" @@ -8,8 +9,6 @@ import ( "github.com/ipfs/boxo/examples/gateway/common" "github.com/ipfs/boxo/gateway" - "github.com/ipld/go-ipld-prime/codec/dagjson" - "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/stretchr/testify/assert" ) @@ -62,48 +61,35 @@ func TestFile(t *testing.T) { assert.EqualValues(t, string(body), "hello world\n") } -func TestDirectoryAsDAG(t *testing.T) { +func TestDirectoryAsRawBlock(t *testing.T) { ts, f, err := newTestServer() assert.NoError(t, err) defer f.Close() - res, err := http.Get(ts.URL + "/ipfs/" + BaseCID + "?format=dag-json") + res, err := http.Get(ts.URL + "/ipfs/" + BaseCID + "?format=raw") assert.NoError(t, err) defer res.Body.Close() - contentType := res.Header.Get("Content-Type") - assert.EqualValues(t, contentType, "application/vnd.ipld.dag-json") - - // Parses the DAG-JSON response. - dag := basicnode.Prototype.Any.NewBuilder() - err = dagjson.Decode(dag, res.Body) - assert.NoError(t, err) - - // Checks for the links inside the logical model. - links, err := dag.Build().LookupByString("Links") - assert.NoError(t, err) - - // Checks if there are 2 links. - assert.EqualValues(t, links.Length(), 2) - - // Check if the first item is correct. - n, err := links.LookupByIndex(0) - assert.NoError(t, err) - assert.NotNil(t, n) + assert.Equal(t, http.StatusOK, res.StatusCode) - nameNode, err := n.LookupByString("Name") - assert.NoError(t, err) - assert.NotNil(t, nameNode) - - name, err := nameNode.AsString() - assert.NoError(t, err) - assert.EqualValues(t, name, "eye.png") + contentType := res.Header.Get("Content-Type") + assert.Equal(t, "application/vnd.ipld.raw", contentType) - hashNode, err := n.LookupByString("Hash") + body, err := io.ReadAll(res.Body) assert.NoError(t, err) - assert.NotNil(t, hashNode) - hash, err := hashNode.AsLink() - assert.NoError(t, err) - assert.EqualValues(t, hash.String(), "bafybeigmlfksb374fdkxih4urny2yiyazyra2375y2e4a72b3jcrnthnau") + // Raw bytes of the dag-pb directory block + expected := []byte{ + 0x12, 0x33, 0x0a, 0x24, 0x01, 0x70, 0x12, 0x20, 0xcc, 0x59, 0x55, 0x20, + 0xef, 0xfc, 0x28, 0xd5, 0x74, 0x1f, 0x94, 0x8b, 0x71, 0xac, 0x23, 0x00, + 0xce, 0x22, 0x0d, 0x6f, 0xfd, 0xc6, 0x89, 0xc0, 0x7f, 0x41, 0xda, 0x45, + 0x16, 0xcc, 0xed, 0x05, 0x12, 0x07, 0x65, 0x79, 0x65, 0x2e, 0x70, 0x6e, + 0x67, 0x18, 0xd0, 0xc8, 0x10, 0x12, 0x33, 0x0a, 0x24, 0x01, 0x55, 0x12, + 0x20, 0xa9, 0x48, 0x90, 0x4f, 0x2f, 0x0f, 0x47, 0x9b, 0x8f, 0x81, 0x97, + 0x69, 0x4b, 0x30, 0x18, 0x4b, 0x0d, 0x2e, 0xd1, 0xc1, 0xcd, 0x2a, 0x1e, + 0xc0, 0xfb, 0x85, 0xd2, 0x99, 0xa1, 0x92, 0xa4, 0x47, 0x12, 0x09, 0x68, + 0x65, 0x6c, 0x6c, 0x6f, 0x2e, 0x74, 0x78, 0x74, 0x18, 0x0c, 0x0a, 0x02, + 0x08, 0x01, + } + assert.True(t, bytes.Equal(body, expected), "raw block bytes should match") } diff --git a/examples/go.mod b/examples/go.mod index ebdaacd90..92112616e 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -8,7 +8,6 @@ require ( github.com/ipfs/go-cid v0.6.0 github.com/ipfs/go-datastore v0.9.0 github.com/ipld/go-car/v2 v2.16.0 - github.com/ipld/go-ipld-prime v0.21.0 github.com/libp2p/go-libp2p v0.45.0 github.com/multiformats/go-multiaddr v0.16.1 github.com/multiformats/go-multicodec v0.10.0 @@ -63,6 +62,7 @@ require ( github.com/ipfs/go-peertaskqueue v0.8.2 // indirect github.com/ipfs/go-unixfsnode v1.10.2 // indirect github.com/ipld/go-codec-dagpb v1.7.0 // indirect + github.com/ipld/go-ipld-prime v0.21.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/klauspost/compress v1.18.0 // indirect diff --git a/gateway/gateway.go b/gateway/gateway.go index 8cec10f21..cd78f8280 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -52,6 +52,19 @@ type Config struct { // [Trustless Gateway]: https://specs.ipfs.tech/http-gateways/trustless-gateway/ DeserializedResponses bool + // AllowCodecConversion enables automatic conversion between codecs when + // the requested format differs from the block's native codec. For example, + // converting dag-pb (UnixFS) to dag-json. + // + // When false (default), the gateway returns 406 Not Acceptable if the + // requested format doesn't match the block's codec. This follows the + // behavior specified in IPIP-0524. + // + // When true, the gateway attempts to convert between legacy IPLD formats. + // This is provided for backwards compatibility but is not required by + // the gateway specification. + AllowCodecConversion bool + // NoDNSLink configures the gateway to _not_ perform DNS TXT record lookups in // response to requests with values in `Host` HTTP header. This flag can be // overridden per FQDN in PublicGateways. To be used with WithHostname. diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index b0fc7fa7d..5f37fdb05 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -521,6 +521,7 @@ func TestHeaders(t *testing.T) { }, }, DeserializedResponses: true, + AllowCodecConversion: true, // Test tests various format conversions }) runTest := func(name, path, accept, host, expectedContentLocationHdr string) { @@ -1073,7 +1074,8 @@ func TestDeserializedResponses(t *testing.T) { backend, root := newMockBackend(t, "fixtures.car") ts := newTestServerWithConfig(t, backend, Config{ - NoDNSLink: false, + NoDNSLink: false, + AllowCodecConversion: true, // Test expects codec conversions to work PublicGateways: map[string]*PublicGateway{ "trustless.com": { Paths: []string{"/ipfs", "/ipns"}, @@ -1152,7 +1154,8 @@ func TestDeserializedResponses(t *testing.T) { backend.namesys["/ipns/trusted.com"] = newMockNamesysItem(path.FromCid(root), 0) ts := newTestServerWithConfig(t, backend, Config{ - NoDNSLink: false, + NoDNSLink: false, + AllowCodecConversion: true, // Test expects codec conversions to work PublicGateways: map[string]*PublicGateway{ "trustless.com": { Paths: []string{"/ipfs", "/ipns"}, @@ -1186,6 +1189,58 @@ func TestDeserializedResponses(t *testing.T) { }) } +func TestAllowCodecConversion(t *testing.T) { + t.Parallel() + + // Use dag-cbor fixture + backend, dagCborRoot := newMockBackend(t, "path_gateway_dag/dag-cbor-traversal.car") + + t.Run("AllowCodecConversion=false returns 406 for codec mismatch", func(t *testing.T) { + t.Parallel() + + ts := newTestServerWithConfig(t, backend, Config{ + DeserializedResponses: true, + AllowCodecConversion: false, // IPIP-0524 behavior + }) + + // Request dag-json for a dag-cbor block - should return 406 + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-json", nil) + res := mustDoWithoutRedirect(t, req) + defer res.Body.Close() + assert.Equal(t, http.StatusNotAcceptable, res.StatusCode) + }) + + t.Run("AllowCodecConversion=false allows matching codec", func(t *testing.T) { + t.Parallel() + + ts := newTestServerWithConfig(t, backend, Config{ + DeserializedResponses: true, + AllowCodecConversion: false, // IPIP-0524 behavior + }) + + // Request dag-cbor for a dag-cbor block - should return 200 + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-cbor", nil) + res := mustDoWithoutRedirect(t, req) + defer res.Body.Close() + assert.Equal(t, http.StatusOK, res.StatusCode) + }) + + t.Run("AllowCodecConversion=true allows codec conversion", func(t *testing.T) { + t.Parallel() + + ts := newTestServerWithConfig(t, backend, Config{ + DeserializedResponses: true, + AllowCodecConversion: true, // Legacy behavior + }) + + // Request dag-json for a dag-cbor block - should return 200 with conversion + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-json", nil) + res := mustDoWithoutRedirect(t, req) + defer res.Body.Close() + assert.Equal(t, http.StatusOK, res.StatusCode) + }) +} + type errorMockBackend struct { err error } diff --git a/gateway/handler_codec.go b/gateway/handler_codec.go index 49b2e9952..122756423 100644 --- a/gateway/handler_codec.go +++ b/gateway/handler_codec.go @@ -148,7 +148,20 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt return false } - // This handles DAG-* conversions and validations. + // IPIP-0524: Check if codec conversion is allowed + if !i.config.AllowCodecConversion && toCodec != cidCodec { + // Conversion not allowed and codecs don't match - return 406 + err := fmt.Errorf("format %q requested but block has codec %q: codec conversion is not supported", rq.responseFormat, cidCodec.String()) + i.webError(w, r, err, http.StatusNotAcceptable) + return false + } + + // If codecs match, serve raw (no conversion needed) + if toCodec == cidCodec { + return i.serveCodecRaw(ctx, w, r, blockSize, blockData, rq.contentPath, modtime, rq.begin) + } + + // AllowCodecConversion is true - perform DAG-* conversion return i.serveCodecConverted(ctx, w, r, blockCid, blockData, rq.contentPath, toCodec, modtime, rq.begin) } diff --git a/gateway/testdata/path_gateway_dag/dag-cbor-traversal.car b/gateway/testdata/path_gateway_dag/dag-cbor-traversal.car new file mode 100644 index 0000000000000000000000000000000000000000..92c3d4f3e21718627007431c6b6d0cc1693be206 GIT binary patch literal 318 zcmcColvlXV(2oIMVptN{kPzW{vS&r0ub_{_K>mhu-w1 zv^b}ir4|)u=I1dM5Tklwa$0`=qLiG>yll7)cbQITWL7ctu-q3j&R(lE-F}^dl&{i> zKYDy?e%dYj{535