1
0
mirror of https://github.com/ipfs/kubo.git synced 2025-08-06 03:19:47 +08:00

feat: Gateway.DeserializedResponses config flag (#9789)

Co-authored-by: Marcin Rataj <lidel@lidel.org>
This commit is contained in:
Henrique Dias
2023-05-30 00:59:34 +02:00
committed by GitHub
parent 38556b8d21
commit c10b804449
10 changed files with 235 additions and 42 deletions

View File

@ -1,6 +1,9 @@
package config
const DefaultInlineDNSLink = false
const (
DefaultInlineDNSLink = false
DefaultDeserializedResponses = true
)
type GatewaySpec struct {
// Paths is explicit list of path prefixes that should be handled by
@ -25,6 +28,11 @@ type GatewaySpec struct {
// (FQDN) into a single DNS label in order to interop with wildcard TLS certs
// and Origin per CID isolation provided by rules like https://publicsuffix.org
InlineDNSLink Flag
// DeserializedResponses configures this gateway to respond to deserialized
// responses. Disabling this option enables a Trustless Gateway, as per:
// https://specs.ipfs.tech/http-gateways/trustless-gateway/.
DeserializedResponses Flag
}
// Gateway contains options for the HTTP gateway server.
@ -56,6 +64,12 @@ type Gateway struct {
// This flag can be overridden per FQDN in PublicGateways.
NoDNSLink bool
// DeserializedResponses configures this gateway to respond to deserialized
// requests. Disabling this option enables a Trustless only gateway, as per:
// https://specs.ipfs.tech/http-gateways/trustless-gateway/. This can
// be overridden per FQDN in PublicGateways.
DeserializedResponses Flag
// PublicGateways configures behavior of known public gateways.
// Each key is a fully qualified domain name (FQDN).
PublicGateways map[string]*GatewaySpec

View File

@ -28,22 +28,11 @@ import (
func GatewayOption(paths ...string) ServeOption {
return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
cfg, err := n.Repo.Config()
gwConfig, err := getGatewayConfig(n)
if err != nil {
return nil, err
}
headers := make(map[string][]string, len(cfg.Gateway.HTTPHeaders))
for h, v := range cfg.Gateway.HTTPHeaders {
headers[http.CanonicalHeaderKey(h)] = v
}
gateway.AddAccessControlHeaders(headers)
gwConfig := gateway.Config{
Headers: headers,
}
gwAPI, err := newGatewayBackend(n)
if err != nil {
return nil, err
@ -65,7 +54,7 @@ func GatewayOption(paths ...string) ServeOption {
func HostnameOption() ServeOption {
return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
cfg, err := n.Repo.Config()
gwConfig, err := getGatewayConfig(n)
if err != nil {
return nil, err
}
@ -75,9 +64,8 @@ func HostnameOption() ServeOption {
return nil, err
}
publicGateways := convertPublicGateways(cfg.Gateway.PublicGateways)
childMux := http.NewServeMux()
mux.HandleFunc("/", gateway.WithHostname(childMux, gwAPI, publicGateways, cfg.Gateway.NoDNSLink).ServeHTTP)
mux.HandleFunc("/", gateway.WithHostname(gwConfig, gwAPI, childMux).ServeHTTP)
return childMux, nil
}
}
@ -212,30 +200,49 @@ var defaultKnownGateways = map[string]*gateway.Specification{
"localhost": subdomainGatewaySpec,
}
func convertPublicGateways(publicGateways map[string]*config.GatewaySpec) map[string]*gateway.Specification {
gws := map[string]*gateway.Specification{}
// First, implicit defaults such as subdomain gateway on localhost
for hostname, gw := range defaultKnownGateways {
gws[hostname] = gw
func getGatewayConfig(n *core.IpfsNode) (gateway.Config, error) {
cfg, err := n.Repo.Config()
if err != nil {
return gateway.Config{}, err
}
// Then apply values from Gateway.PublicGateways, if present in the config
for hostname, gw := range publicGateways {
// Parse configuration headers and add the default Access Control Headers.
headers := make(map[string][]string, len(cfg.Gateway.HTTPHeaders))
for h, v := range cfg.Gateway.HTTPHeaders {
headers[http.CanonicalHeaderKey(h)] = v
}
gateway.AddAccessControlHeaders(headers)
// Initialize gateway configuration, with empty PublicGateways, handled after.
gwCfg := gateway.Config{
Headers: headers,
DeserializedResponses: cfg.Gateway.DeserializedResponses.WithDefault(config.DefaultDeserializedResponses),
NoDNSLink: cfg.Gateway.NoDNSLink,
PublicGateways: map[string]*gateway.Specification{},
}
// Add default implicit known gateways, such as subdomain gateway on localhost.
for hostname, gw := range defaultKnownGateways {
gwCfg.PublicGateways[hostname] = gw
}
// Apply values from cfg.Gateway.PublicGateways if they exist.
for hostname, gw := range cfg.Gateway.PublicGateways {
if gw == nil {
// Remove any implicit defaults, if present. This is useful when one
// wants to disable subdomain gateway on localhost etc.
delete(gws, hostname)
// wants to disable subdomain gateway on localhost, etc.
delete(gwCfg.PublicGateways, hostname)
continue
}
gws[hostname] = &gateway.Specification{
gwCfg.PublicGateways[hostname] = &gateway.Specification{
Paths: gw.Paths,
NoDNSLink: gw.NoDNSLink,
UseSubdomains: gw.UseSubdomains,
InlineDNSLink: gw.InlineDNSLink.WithDefault(config.DefaultInlineDNSLink),
DeserializedResponses: gw.DeserializedResponses.WithDefault(gwCfg.DeserializedResponses),
}
}
return gws
return gwCfg, nil
}

View File

@ -14,6 +14,7 @@ import (
core "github.com/ipfs/kubo/core"
"github.com/ipfs/kubo/core/coreapi"
repo "github.com/ipfs/kubo/repo"
"github.com/stretchr/testify/assert"
iface "github.com/ipfs/boxo/coreiface"
nsopts "github.com/ipfs/boxo/coreiface/options/namesys"
@ -173,3 +174,42 @@ func TestVersion(t *testing.T) {
t.Fatalf("response doesn't contain protocol version:\n%s", s)
}
}
func TestDeserializedResponsesInheritance(t *testing.T) {
for _, testCase := range []struct {
globalSetting config.Flag
gatewaySetting config.Flag
expectedGatewaySetting bool
}{
{config.True, config.Default, true},
{config.False, config.Default, false},
{config.False, config.True, true},
{config.True, config.False, false},
} {
c := config.Config{
Identity: config.Identity{
PeerID: "QmTFauExutTsy4XP6JbMFcw2Wa9645HJt2bTqL6qYDCKfe", // required by offline node
},
Gateway: config.Gateway{
DeserializedResponses: testCase.globalSetting,
PublicGateways: map[string]*config.GatewaySpec{
"example.com": {
DeserializedResponses: testCase.gatewaySetting,
},
},
},
}
r := &repo.Mock{
C: c,
D: syncds.MutexWrap(datastore.NewMapDatastore()),
}
n, err := core.NewNode(context.Background(), &core.BuildCfg{Repo: r})
assert.NoError(t, err)
gwCfg, err := getGatewayConfig(n)
assert.NoError(t, err)
assert.Contains(t, gwCfg.PublicGateways, "example.com")
assert.Equal(t, testCase.expectedGatewaySetting, gwCfg.PublicGateways["example.com"].DeserializedResponses)
}
}

View File

@ -7,6 +7,7 @@
- [Overview](#overview)
- [🔦 Highlights](#-highlights)
- [Saving previously seen nodes for later bootstrapping](#saving-previously-seen-nodes-for-later-bootstrapping)
- [`Gateway.DeserializedResponses` config flag](#gatewaydeserializedresponses-config-flag)
- [📝 Changelog](#-changelog)
- [👨‍👩‍👧‍👦 Contributors](#-contributors)
@ -29,6 +30,32 @@ enabled.
With this update, the same level of robustness is applied to peers that lack
mDNS peers and solely rely on the public DHT.
#### `Gateway.DeserializedResponses` config flag
This release introduces the
[`Gateway.DeserializedResponses`](https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewaydeserializedresponses)
configuration flag.
With this flag, one can explicitly configure whether the gateway responds to
deserialized requests or not. By default, this flag is enabled.
Disabling deserialized responses allows the
gateway to operate
as a [Trustless Gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway/)
limited to three [verifiable](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval)
response types:
[application/vnd.ipld.raw](https://www.iana.org/assignments/media-types/application/vnd.ipld.raw),
[application/vnd.ipld.car](https://www.iana.org/assignments/media-types/application/vnd.ipld.car),
and [application/vnd.ipfs.ipns-record](https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record).
With deserialized responses disabled, the Kubo gateway can serve as a block
backend for other software (like
[bifrost-gateway](https://github.com/ipfs/bifrost-gateway#readme),
[IPFS in Chromium](https://github.com/little-bear-labs/ipfs-chromium/blob/main/README.md)
etc) without the usual risks associated with hosting deserialized data behind
third-party CIDs.
### 📝 Changelog
### 👨‍👩‍👧‍👦 Contributors

View File

@ -50,6 +50,7 @@ config file at runtime.
- [`Gateway`](#gateway)
- [`Gateway.NoFetch`](#gatewaynofetch)
- [`Gateway.NoDNSLink`](#gatewaynodnslink)
- [`Gateway.DeserializedResponses`](#gatewaydeserializedresponses)
- [`Gateway.HTTPHeaders`](#gatewayhttpheaders)
- [`Gateway.RootRedirect`](#gatewayrootredirect)
- [`Gateway.FastDirIndexThreshold`](#gatewayfastdirindexthreshold)
@ -60,6 +61,7 @@ config file at runtime.
- [`Gateway.PublicGateways: UseSubdomains`](#gatewaypublicgateways-usesubdomains)
- [`Gateway.PublicGateways: NoDNSLink`](#gatewaypublicgateways-nodnslink)
- [`Gateway.PublicGateways: InlineDNSLink`](#gatewaypublicgateways-inlinednslink)
- [`Gateway.PublicGateways: DeserializedResponses`](#gatewaypublicgateways-deserializedresponses)
- [Implicit defaults of `Gateway.PublicGateways`](#implicit-defaults-of-gatewaypublicgateways)
- [`Gateway` recipes](#gateway-recipes)
- [`Identity`](#identity)
@ -646,6 +648,16 @@ Default: `false`
Type: `bool`
#### `Gateway.DeserializedResponses`
An optional flag to explicitly configure whether this gateway responds to deserialized
requests, or not. By default, it is enabled. When disabling this option, the gateway
operates as a Trustless Gateway only: https://specs.ipfs.tech/http-gateways/trustless-gateway/.
Default: `true`
Type: `flag`
### `Gateway.HTTPHeaders`
Headers to set on gateway responses.
@ -790,6 +802,16 @@ Default: `false`
Type: `flag`
#### `Gateway.PublicGateways: DeserializedResponses`
An optional flag to explicitly configure whether this gateway responds to deserialized
requests, or not. By default, it is enabled. When disabling this option, the gateway
operates as a Trustless Gateway only: https://specs.ipfs.tech/http-gateways/trustless-gateway/.
Default: same as global `Gateway.DeserializedResponses`
Type: `flag`
#### Implicit defaults of `Gateway.PublicGateways`
Default entries for `localhost` hostname and loopback IPs are always present.

View File

@ -7,7 +7,7 @@ go 1.18
replace github.com/ipfs/kubo => ./../../..
require (
github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49
github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad
github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
github.com/libp2p/go-libp2p v0.27.3
github.com/multiformats/go-multiaddr v0.9.0

View File

@ -321,8 +321,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49 h1:hi2x0dCINl9fHIV6YM+IH+Bah45pRAFekjM5MMKWJO4=
github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y=
github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad h1:2vkMvvVa5f9fWzts7OcJL6ZS0QaKCcEeOV6I+doPMo0=
github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y=
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=

2
go.mod
View File

@ -16,7 +16,7 @@ require (
github.com/gogo/protobuf v1.3.2
github.com/google/uuid v1.3.0
github.com/hashicorp/go-multierror v1.1.1
github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49
github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad
github.com/ipfs/go-block-format v0.1.2
github.com/ipfs/go-cid v0.4.1
github.com/ipfs/go-cidutil v0.1.0

4
go.sum
View File

@ -356,8 +356,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49 h1:hi2x0dCINl9fHIV6YM+IH+Bah45pRAFekjM5MMKWJO4=
github.com/ipfs/boxo v0.8.2-0.20230525115135-a8533c998f49/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y=
github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad h1:2vkMvvVa5f9fWzts7OcJL6ZS0QaKCcEeOV6I+doPMo0=
github.com/ipfs/boxo v0.8.2-0.20230529214945-86cdb2485dad/go.mod h1:Ej2r08Z4VIaFKqY08UXMNhwcLf6VekHhK8c+KqA1B9Y=
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=

View File

@ -513,4 +513,87 @@ func TestGateway(t *testing.T) {
})
})
})
t.Run("DeserializedResponses", func(t *testing.T) {
type testCase struct {
globalValue config.Flag
gatewayValue config.Flag
deserializedGlobalStatusCode int
deserializedGatewayStaticCode int
message string
}
setHost := func(r *http.Request) {
r.Host = "example.com"
}
withAccept := func(accept string) func(r *http.Request) {
return func(r *http.Request) {
r.Header.Set("Accept", accept)
}
}
withHostAndAccept := func(accept string) func(r *http.Request) {
return func(r *http.Request) {
setHost(r)
withAccept(accept)(r)
}
}
makeTest := func(test *testCase) func(t *testing.T) {
return func(t *testing.T) {
t.Parallel()
node := harness.NewT(t).NewNode().Init()
node.UpdateConfig(func(cfg *config.Config) {
cfg.Gateway.DeserializedResponses = test.globalValue
cfg.Gateway.PublicGateways = map[string]*config.GatewaySpec{
"example.com": {
Paths: []string{"/ipfs", "/ipns"},
DeserializedResponses: test.gatewayValue,
},
}
})
node.StartDaemon()
cidFoo := node.IPFSAddStr("foo")
client := node.GatewayClient()
deserializedPath := "/ipfs/" + cidFoo
blockPath := deserializedPath + "?format=raw"
carPath := deserializedPath + "?format=car"
// Global Check (Gateway.DeserializedResponses)
assert.Equal(t, http.StatusOK, client.Get(blockPath).StatusCode)
assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withAccept("application/vnd.ipld.raw")).StatusCode)
assert.Equal(t, http.StatusOK, client.Get(carPath).StatusCode)
assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withAccept("application/vnd.ipld.car")).StatusCode)
assert.Equal(t, test.deserializedGlobalStatusCode, client.Get(deserializedPath).StatusCode)
assert.Equal(t, test.deserializedGlobalStatusCode, client.Get(deserializedPath, withAccept("application/json")).StatusCode)
// Public Gateway (example.com) Check (Gateway.PublicGateways[example.com].DeserializedResponses)
assert.Equal(t, http.StatusOK, client.Get(blockPath, setHost).StatusCode)
assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withHostAndAccept("application/vnd.ipld.raw")).StatusCode)
assert.Equal(t, http.StatusOK, client.Get(carPath, setHost).StatusCode)
assert.Equal(t, http.StatusOK, client.Get(deserializedPath, withHostAndAccept("application/vnd.ipld.car")).StatusCode)
assert.Equal(t, test.deserializedGatewayStaticCode, client.Get(deserializedPath, setHost).StatusCode)
assert.Equal(t, test.deserializedGatewayStaticCode, client.Get(deserializedPath, withHostAndAccept("application/json")).StatusCode)
}
}
for _, test := range []*testCase{
{config.True, config.Default, http.StatusOK, http.StatusOK, "when Gateway.DeserializedResponses is globally enabled, leaving implicit default for Gateway.PublicGateways[example.com] should inherit the global setting (enabled)"},
{config.False, config.Default, http.StatusNotAcceptable, http.StatusNotAcceptable, "when Gateway.DeserializedResponses is globally disabled, leaving implicit default on Gateway.PublicGateways[example.com] should inherit the global setting (disabled)"},
{config.False, config.True, http.StatusNotAcceptable, http.StatusOK, "when Gateway.DeserializedResponses is globally disabled, explicitly enabling on Gateway.PublicGateways[example.com] should override global (enabled)"},
{config.True, config.False, http.StatusOK, http.StatusNotAcceptable, "when Gateway.DeserializedResponses is globally enabled, explicitly disabling on Gateway.PublicGateways[example.com] should override global (disabled)"},
} {
t.Run(test.message, makeTest(test))
}
})
}