diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 47bb861e7..6a8d97de1 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -22,7 +22,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.8 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: output: fixtures merged: true @@ -47,7 +47,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@v0.8 + uses: ipfs/gateway-conformance/.github/actions/test@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: gateway-url: http://127.0.0.1:8040 subdomain-url: http://example.net:8040 @@ -84,7 +84,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.8 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: output: fixtures merged: true @@ -114,7 +114,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@v0.8 + uses: ipfs/gateway-conformance/.github/actions/test@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote block gateway subdomain-url: http://example.net:8040 @@ -152,7 +152,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.8 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: output: fixtures merged: true @@ -182,7 +182,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests without IPNS and DNSLink - uses: ipfs/gateway-conformance/.github/actions/test@v0.8 + uses: ipfs/gateway-conformance/.github/actions/test@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships with: gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote car gateway subdomain-url: http://example.net:8040 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b6576431..56b9a6712 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ The following emojis are used to highlight certain changes: ### Changed +- `gateway`: 🛠 Codec conversions (e.g., dag-pb to dag-json, dag-json to dag-cbor) are no longer performed by default per [IPIP-0524](https://github.com/ipfs/specs/pull/524). Requesting a format that differs from the block's codec now returns HTTP 406 Not Acceptable. Clients should fetch raw blocks (`?format=raw`) and convert in userland. Set `Config.AllowCodecConversion` to `true` to restore the old behavior. + ### Removed ### Fixed diff --git a/examples/gateway/car-file/main_test.go b/examples/gateway/car-file/main_test.go index 6c8fc22d7..6e611d819 100644 --- a/examples/gateway/car-file/main_test.go +++ b/examples/gateway/car-file/main_test.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "io" "net/http" "net/http/httptest" @@ -8,8 +9,6 @@ import ( "github.com/ipfs/boxo/examples/gateway/common" "github.com/ipfs/boxo/gateway" - "github.com/ipld/go-ipld-prime/codec/dagjson" - "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/stretchr/testify/assert" ) @@ -62,48 +61,35 @@ func TestFile(t *testing.T) { assert.EqualValues(t, string(body), "hello world\n") } -func TestDirectoryAsDAG(t *testing.T) { +func TestDirectoryAsRawBlock(t *testing.T) { ts, f, err := newTestServer() assert.NoError(t, err) defer f.Close() - res, err := http.Get(ts.URL + "/ipfs/" + BaseCID + "?format=dag-json") + res, err := http.Get(ts.URL + "/ipfs/" + BaseCID + "?format=raw") assert.NoError(t, err) defer res.Body.Close() - contentType := res.Header.Get("Content-Type") - assert.EqualValues(t, contentType, "application/vnd.ipld.dag-json") - - // Parses the DAG-JSON response. - dag := basicnode.Prototype.Any.NewBuilder() - err = dagjson.Decode(dag, res.Body) - assert.NoError(t, err) - - // Checks for the links inside the logical model. - links, err := dag.Build().LookupByString("Links") - assert.NoError(t, err) - - // Checks if there are 2 links. - assert.EqualValues(t, links.Length(), 2) - - // Check if the first item is correct. - n, err := links.LookupByIndex(0) - assert.NoError(t, err) - assert.NotNil(t, n) + assert.Equal(t, http.StatusOK, res.StatusCode) - nameNode, err := n.LookupByString("Name") - assert.NoError(t, err) - assert.NotNil(t, nameNode) - - name, err := nameNode.AsString() - assert.NoError(t, err) - assert.EqualValues(t, name, "eye.png") + contentType := res.Header.Get("Content-Type") + assert.Equal(t, "application/vnd.ipld.raw", contentType) - hashNode, err := n.LookupByString("Hash") + body, err := io.ReadAll(res.Body) assert.NoError(t, err) - assert.NotNil(t, hashNode) - hash, err := hashNode.AsLink() - assert.NoError(t, err) - assert.EqualValues(t, hash.String(), "bafybeigmlfksb374fdkxih4urny2yiyazyra2375y2e4a72b3jcrnthnau") + // Raw bytes of the dag-pb directory block + expected := []byte{ + 0x12, 0x33, 0x0a, 0x24, 0x01, 0x70, 0x12, 0x20, 0xcc, 0x59, 0x55, 0x20, + 0xef, 0xfc, 0x28, 0xd5, 0x74, 0x1f, 0x94, 0x8b, 0x71, 0xac, 0x23, 0x00, + 0xce, 0x22, 0x0d, 0x6f, 0xfd, 0xc6, 0x89, 0xc0, 0x7f, 0x41, 0xda, 0x45, + 0x16, 0xcc, 0xed, 0x05, 0x12, 0x07, 0x65, 0x79, 0x65, 0x2e, 0x70, 0x6e, + 0x67, 0x18, 0xd0, 0xc8, 0x10, 0x12, 0x33, 0x0a, 0x24, 0x01, 0x55, 0x12, + 0x20, 0xa9, 0x48, 0x90, 0x4f, 0x2f, 0x0f, 0x47, 0x9b, 0x8f, 0x81, 0x97, + 0x69, 0x4b, 0x30, 0x18, 0x4b, 0x0d, 0x2e, 0xd1, 0xc1, 0xcd, 0x2a, 0x1e, + 0xc0, 0xfb, 0x85, 0xd2, 0x99, 0xa1, 0x92, 0xa4, 0x47, 0x12, 0x09, 0x68, + 0x65, 0x6c, 0x6c, 0x6f, 0x2e, 0x74, 0x78, 0x74, 0x18, 0x0c, 0x0a, 0x02, + 0x08, 0x01, + } + assert.True(t, bytes.Equal(body, expected), "raw block bytes should match") } diff --git a/examples/go.mod b/examples/go.mod index ebdaacd90..92112616e 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -8,7 +8,6 @@ require ( github.com/ipfs/go-cid v0.6.0 github.com/ipfs/go-datastore v0.9.0 github.com/ipld/go-car/v2 v2.16.0 - github.com/ipld/go-ipld-prime v0.21.0 github.com/libp2p/go-libp2p v0.45.0 github.com/multiformats/go-multiaddr v0.16.1 github.com/multiformats/go-multicodec v0.10.0 @@ -63,6 +62,7 @@ require ( github.com/ipfs/go-peertaskqueue v0.8.2 // indirect github.com/ipfs/go-unixfsnode v1.10.2 // indirect github.com/ipld/go-codec-dagpb v1.7.0 // indirect + github.com/ipld/go-ipld-prime v0.21.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/klauspost/compress v1.18.0 // indirect diff --git a/gateway/gateway.go b/gateway/gateway.go index 8cec10f21..cd78f8280 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -52,6 +52,19 @@ type Config struct { // [Trustless Gateway]: https://specs.ipfs.tech/http-gateways/trustless-gateway/ DeserializedResponses bool + // AllowCodecConversion enables automatic conversion between codecs when + // the requested format differs from the block's native codec. For example, + // converting dag-pb (UnixFS) to dag-json. + // + // When false (default), the gateway returns 406 Not Acceptable if the + // requested format doesn't match the block's codec. This follows the + // behavior specified in IPIP-0524. + // + // When true, the gateway attempts to convert between legacy IPLD formats. + // This is provided for backwards compatibility but is not required by + // the gateway specification. + AllowCodecConversion bool + // NoDNSLink configures the gateway to _not_ perform DNS TXT record lookups in // response to requests with values in `Host` HTTP header. This flag can be // overridden per FQDN in PublicGateways. To be used with WithHostname. diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index b0fc7fa7d..5f37fdb05 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -521,6 +521,7 @@ func TestHeaders(t *testing.T) { }, }, DeserializedResponses: true, + AllowCodecConversion: true, // Test tests various format conversions }) runTest := func(name, path, accept, host, expectedContentLocationHdr string) { @@ -1073,7 +1074,8 @@ func TestDeserializedResponses(t *testing.T) { backend, root := newMockBackend(t, "fixtures.car") ts := newTestServerWithConfig(t, backend, Config{ - NoDNSLink: false, + NoDNSLink: false, + AllowCodecConversion: true, // Test expects codec conversions to work PublicGateways: map[string]*PublicGateway{ "trustless.com": { Paths: []string{"/ipfs", "/ipns"}, @@ -1152,7 +1154,8 @@ func TestDeserializedResponses(t *testing.T) { backend.namesys["/ipns/trusted.com"] = newMockNamesysItem(path.FromCid(root), 0) ts := newTestServerWithConfig(t, backend, Config{ - NoDNSLink: false, + NoDNSLink: false, + AllowCodecConversion: true, // Test expects codec conversions to work PublicGateways: map[string]*PublicGateway{ "trustless.com": { Paths: []string{"/ipfs", "/ipns"}, @@ -1186,6 +1189,58 @@ func TestDeserializedResponses(t *testing.T) { }) } +func TestAllowCodecConversion(t *testing.T) { + t.Parallel() + + // Use dag-cbor fixture + backend, dagCborRoot := newMockBackend(t, "path_gateway_dag/dag-cbor-traversal.car") + + t.Run("AllowCodecConversion=false returns 406 for codec mismatch", func(t *testing.T) { + t.Parallel() + + ts := newTestServerWithConfig(t, backend, Config{ + DeserializedResponses: true, + AllowCodecConversion: false, // IPIP-0524 behavior + }) + + // Request dag-json for a dag-cbor block - should return 406 + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-json", nil) + res := mustDoWithoutRedirect(t, req) + defer res.Body.Close() + assert.Equal(t, http.StatusNotAcceptable, res.StatusCode) + }) + + t.Run("AllowCodecConversion=false allows matching codec", func(t *testing.T) { + t.Parallel() + + ts := newTestServerWithConfig(t, backend, Config{ + DeserializedResponses: true, + AllowCodecConversion: false, // IPIP-0524 behavior + }) + + // Request dag-cbor for a dag-cbor block - should return 200 + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-cbor", nil) + res := mustDoWithoutRedirect(t, req) + defer res.Body.Close() + assert.Equal(t, http.StatusOK, res.StatusCode) + }) + + t.Run("AllowCodecConversion=true allows codec conversion", func(t *testing.T) { + t.Parallel() + + ts := newTestServerWithConfig(t, backend, Config{ + DeserializedResponses: true, + AllowCodecConversion: true, // Legacy behavior + }) + + // Request dag-json for a dag-cbor block - should return 200 with conversion + req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-json", nil) + res := mustDoWithoutRedirect(t, req) + defer res.Body.Close() + assert.Equal(t, http.StatusOK, res.StatusCode) + }) +} + type errorMockBackend struct { err error } diff --git a/gateway/handler_codec.go b/gateway/handler_codec.go index 49b2e9952..122756423 100644 --- a/gateway/handler_codec.go +++ b/gateway/handler_codec.go @@ -148,7 +148,20 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt return false } - // This handles DAG-* conversions and validations. + // IPIP-0524: Check if codec conversion is allowed + if !i.config.AllowCodecConversion && toCodec != cidCodec { + // Conversion not allowed and codecs don't match - return 406 + err := fmt.Errorf("format %q requested but block has codec %q: codec conversion is not supported", rq.responseFormat, cidCodec.String()) + i.webError(w, r, err, http.StatusNotAcceptable) + return false + } + + // If codecs match, serve raw (no conversion needed) + if toCodec == cidCodec { + return i.serveCodecRaw(ctx, w, r, blockSize, blockData, rq.contentPath, modtime, rq.begin) + } + + // AllowCodecConversion is true - perform DAG-* conversion return i.serveCodecConverted(ctx, w, r, blockCid, blockData, rq.contentPath, toCodec, modtime, rq.begin) } diff --git a/gateway/testdata/path_gateway_dag/dag-cbor-traversal.car b/gateway/testdata/path_gateway_dag/dag-cbor-traversal.car new file mode 100644 index 000000000..92c3d4f3e Binary files /dev/null and b/gateway/testdata/path_gateway_dag/dag-cbor-traversal.car differ diff --git a/gateway/utilities_test.go b/gateway/utilities_test.go index c186455ea..b7743ef7a 100644 --- a/gateway/utilities_test.go +++ b/gateway/utilities_test.go @@ -234,6 +234,7 @@ func newTestServerAndNode(t *testing.T, fixturesFile string) (*httptest.Server, func newTestServer(t *testing.T, backend IPFSBackend) *httptest.Server { return newTestServerWithConfig(t, backend, Config{ DeserializedResponses: true, + AllowCodecConversion: true, // Enable for backwards compatibility in tests MetricsRegistry: prometheus.NewRegistry(), }) }