mirror of
https://github.com/ipfs/kubo.git
synced 2025-08-06 11:31:54 +08:00
refactor: switch gateway code to new API from go-libipfs (#9681)
Co-authored-by: Marcin Rataj <lidel@lidel.org> Co-authored-by: Henrique Dias <hacdias@gmail.com>
This commit is contained in:
Adin Schmahmann
@ -2,24 +2,26 @@ package corehttp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
|
||||
"github.com/ipfs/boxo/blockservice"
|
||||
iface "github.com/ipfs/boxo/coreiface"
|
||||
options "github.com/ipfs/boxo/coreiface/options"
|
||||
nsopts "github.com/ipfs/boxo/coreiface/options/namesys"
|
||||
"github.com/ipfs/boxo/coreiface/path"
|
||||
"github.com/ipfs/boxo/exchange/offline"
|
||||
"github.com/ipfs/boxo/files"
|
||||
"github.com/ipfs/boxo/gateway"
|
||||
"github.com/ipfs/boxo/namesys"
|
||||
"github.com/ipfs/go-block-format"
|
||||
offlineroute "github.com/ipfs/boxo/routing/offline"
|
||||
cid "github.com/ipfs/go-cid"
|
||||
version "github.com/ipfs/kubo"
|
||||
config "github.com/ipfs/kubo/config"
|
||||
core "github.com/ipfs/kubo/core"
|
||||
coreapi "github.com/ipfs/kubo/core/coreapi"
|
||||
"github.com/ipfs/kubo/core/node"
|
||||
"github.com/libp2p/go-libp2p/core/routing"
|
||||
id "github.com/libp2p/go-libp2p/p2p/protocol/identify"
|
||||
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
|
||||
)
|
||||
@ -42,7 +44,7 @@ func GatewayOption(paths ...string) ServeOption {
|
||||
Headers: headers,
|
||||
}
|
||||
|
||||
gwAPI, err := newGatewayAPI(n)
|
||||
gwAPI, err := newGatewayBackend(n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -70,7 +72,7 @@ func HostnameOption() ServeOption {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
gwAPI, err := newGatewayAPI(n)
|
||||
gwAPI, err := newGatewayBackend(n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -93,83 +95,120 @@ func VersionOption() ServeOption {
|
||||
}
|
||||
}
|
||||
|
||||
type gatewayAPI struct {
|
||||
ns namesys.NameSystem
|
||||
api iface.CoreAPI
|
||||
offlineAPI iface.CoreAPI
|
||||
}
|
||||
|
||||
func newGatewayAPI(n *core.IpfsNode) (*gatewayAPI, error) {
|
||||
func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) {
|
||||
cfg, err := n.Repo.Config()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
api, err := coreapi.NewCoreAPI(n, options.Api.FetchBlocks(!cfg.Gateway.NoFetch))
|
||||
bserv := n.Blocks
|
||||
var vsRouting routing.ValueStore = n.Routing
|
||||
nsys := n.Namesys
|
||||
if cfg.Gateway.NoFetch {
|
||||
bserv = blockservice.New(bserv.Blockstore(), offline.Exchange(bserv.Blockstore()))
|
||||
|
||||
cs := cfg.Ipns.ResolveCacheSize
|
||||
if cs == 0 {
|
||||
cs = node.DefaultIpnsCacheSize
|
||||
}
|
||||
if cs < 0 {
|
||||
return nil, fmt.Errorf("cannot specify negative resolve cache size")
|
||||
}
|
||||
|
||||
vsRouting = offlineroute.NewOfflineRouter(n.Repo.Datastore(), n.RecordValidator)
|
||||
nsys, err = namesys.NewNameSystem(vsRouting,
|
||||
namesys.WithDatastore(n.Repo.Datastore()),
|
||||
namesys.WithDNSResolver(n.DNSResolver),
|
||||
namesys.WithCache(cs))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error constructing namesys: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
gw, err := gateway.NewBlocksGateway(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
offlineAPI, err := api.WithOptions(options.Api.Offline(true))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return &offlineGatewayErrWrapper{gwimpl: gw}, nil
|
||||
}
|
||||
|
||||
type offlineGatewayErrWrapper struct {
|
||||
gwimpl gateway.IPFSBackend
|
||||
}
|
||||
|
||||
func offlineErrWrap(err error) error {
|
||||
if errors.Is(err, iface.ErrOffline) {
|
||||
return fmt.Errorf("%s : %w", err.Error(), gateway.ErrServiceUnavailable)
|
||||
}
|
||||
|
||||
return &gatewayAPI{
|
||||
ns: n.Namesys,
|
||||
api: api,
|
||||
offlineAPI: offlineAPI,
|
||||
}, nil
|
||||
return err
|
||||
}
|
||||
|
||||
func (gw *gatewayAPI) GetUnixFsNode(ctx context.Context, pth path.Resolved) (files.Node, error) {
|
||||
return gw.api.Unixfs().Get(ctx, pth)
|
||||
func (o *offlineGatewayErrWrapper) Get(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, *gateway.GetResponse, error) {
|
||||
md, n, err := o.gwimpl.Get(ctx, path)
|
||||
err = offlineErrWrap(err)
|
||||
return md, n, err
|
||||
}
|
||||
|
||||
func (gw *gatewayAPI) LsUnixFsDir(ctx context.Context, pth path.Resolved) (<-chan iface.DirEntry, error) {
|
||||
// Optimization: use Unixfs.Ls without resolving children, but using the
|
||||
// cumulative DAG size as the file size. This allows for a fast listing
|
||||
// while keeping a good enough Size field.
|
||||
return gw.api.Unixfs().Ls(ctx, pth,
|
||||
options.Unixfs.ResolveChildren(false),
|
||||
options.Unixfs.UseCumulativeSize(true),
|
||||
)
|
||||
func (o *offlineGatewayErrWrapper) GetRange(ctx context.Context, path gateway.ImmutablePath, ranges ...gateway.GetRange) (gateway.ContentPathMetadata, files.File, error) {
|
||||
md, n, err := o.gwimpl.GetRange(ctx, path, ranges...)
|
||||
err = offlineErrWrap(err)
|
||||
return md, n, err
|
||||
}
|
||||
|
||||
func (gw *gatewayAPI) GetBlock(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
|
||||
r, err := gw.api.Block().Get(ctx, path.IpfsPath(cid))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return blocks.NewBlockWithCid(data, cid)
|
||||
func (o *offlineGatewayErrWrapper) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) {
|
||||
md, n, err := o.gwimpl.GetAll(ctx, path)
|
||||
err = offlineErrWrap(err)
|
||||
return md, n, err
|
||||
}
|
||||
|
||||
func (gw *gatewayAPI) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) {
|
||||
return gw.api.Routing().Get(ctx, "/ipns/"+c.String())
|
||||
func (o *offlineGatewayErrWrapper) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) {
|
||||
md, n, err := o.gwimpl.GetBlock(ctx, path)
|
||||
err = offlineErrWrap(err)
|
||||
return md, n, err
|
||||
}
|
||||
|
||||
func (gw *gatewayAPI) GetDNSLinkRecord(ctx context.Context, hostname string) (path.Path, error) {
|
||||
p, err := gw.ns.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1))
|
||||
if err == namesys.ErrResolveRecursion {
|
||||
err = nil
|
||||
}
|
||||
return path.New(p.String()), err
|
||||
func (o *offlineGatewayErrWrapper) Head(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) {
|
||||
md, n, err := o.gwimpl.Head(ctx, path)
|
||||
err = offlineErrWrap(err)
|
||||
return md, n, err
|
||||
}
|
||||
|
||||
func (gw *gatewayAPI) IsCached(ctx context.Context, pth path.Path) bool {
|
||||
_, err := gw.offlineAPI.Block().Stat(ctx, pth)
|
||||
return err == nil
|
||||
func (o *offlineGatewayErrWrapper) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) {
|
||||
md, err := o.gwimpl.ResolvePath(ctx, path)
|
||||
err = offlineErrWrap(err)
|
||||
return md, err
|
||||
}
|
||||
|
||||
func (gw *gatewayAPI) ResolvePath(ctx context.Context, pth path.Path) (path.Resolved, error) {
|
||||
return gw.api.ResolvePath(ctx, pth)
|
||||
func (o *offlineGatewayErrWrapper) GetCAR(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, io.ReadCloser, <-chan error, error) {
|
||||
md, data, errCh, err := o.gwimpl.GetCAR(ctx, path)
|
||||
err = offlineErrWrap(err)
|
||||
return md, data, errCh, err
|
||||
}
|
||||
|
||||
func (o *offlineGatewayErrWrapper) IsCached(ctx context.Context, path path.Path) bool {
|
||||
return o.gwimpl.IsCached(ctx, path)
|
||||
}
|
||||
|
||||
func (o *offlineGatewayErrWrapper) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) {
|
||||
rec, err := o.gwimpl.GetIPNSRecord(ctx, c)
|
||||
err = offlineErrWrap(err)
|
||||
return rec, err
|
||||
}
|
||||
|
||||
func (o *offlineGatewayErrWrapper) ResolveMutable(ctx context.Context, path path.Path) (gateway.ImmutablePath, error) {
|
||||
imPath, err := o.gwimpl.ResolveMutable(ctx, path)
|
||||
err = offlineErrWrap(err)
|
||||
return imPath, err
|
||||
}
|
||||
|
||||
func (o *offlineGatewayErrWrapper) GetDNSLinkRecord(ctx context.Context, s string) (path.Path, error) {
|
||||
p, err := o.gwimpl.GetDNSLinkRecord(ctx, s)
|
||||
err = offlineErrWrap(err)
|
||||
return p, err
|
||||
}
|
||||
|
||||
var _ gateway.IPFSBackend = (*offlineGatewayErrWrapper)(nil)
|
||||
|
||||
var defaultPaths = []string{"/ipfs/", "/ipns/", "/api/", "/p2p/"}
|
||||
|
||||
var subdomainGatewaySpec = &gateway.Specification{
|
||||
|
||||
@ -1,88 +1,20 @@
|
||||
package node
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ipfs/boxo/gateway"
|
||||
config "github.com/ipfs/kubo/config"
|
||||
doh "github.com/libp2p/go-doh-resolver"
|
||||
madns "github.com/multiformats/go-multiaddr-dns"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
)
|
||||
|
||||
var defaultResolvers = map[string]string{
|
||||
"eth.": "https://resolver.cloudflare-eth.com/dns-query",
|
||||
"crypto.": "https://resolver.cloudflare-eth.com/dns-query",
|
||||
}
|
||||
|
||||
func newResolver(url string, opts ...doh.Option) (madns.BasicResolver, error) {
|
||||
if !strings.HasPrefix(url, "https://") {
|
||||
return nil, fmt.Errorf("invalid resolver url: %s", url)
|
||||
}
|
||||
|
||||
return doh.NewResolver(url, opts...)
|
||||
}
|
||||
|
||||
func DNSResolver(cfg *config.Config) (*madns.Resolver, error) {
|
||||
var opts []madns.Option
|
||||
var err error
|
||||
|
||||
var dohOpts []doh.Option
|
||||
if !cfg.DNS.MaxCacheTTL.IsDefault() {
|
||||
dohOpts = append(dohOpts, doh.WithMaxCacheTTL(cfg.DNS.MaxCacheTTL.WithDefault(time.Duration(math.MaxUint32)*time.Second)))
|
||||
}
|
||||
|
||||
domains := make(map[string]struct{}) // to track overridden default resolvers
|
||||
rslvrs := make(map[string]madns.BasicResolver) // to reuse resolvers for the same URL
|
||||
|
||||
for domain, url := range cfg.DNS.Resolvers {
|
||||
if domain != "." && !dns.IsFqdn(domain) {
|
||||
return nil, fmt.Errorf("invalid domain %s; must be FQDN", domain)
|
||||
}
|
||||
|
||||
domains[domain] = struct{}{}
|
||||
if url == "" {
|
||||
// allow overriding of implicit defaults with the default resolver
|
||||
continue
|
||||
}
|
||||
|
||||
rslv, ok := rslvrs[url]
|
||||
if !ok {
|
||||
rslv, err = newResolver(url, dohOpts...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bad resolver for %s: %w", domain, err)
|
||||
}
|
||||
rslvrs[url] = rslv
|
||||
}
|
||||
|
||||
if domain != "." {
|
||||
opts = append(opts, madns.WithDomainResolver(domain, rslv))
|
||||
} else {
|
||||
opts = append(opts, madns.WithDefaultResolver(rslv))
|
||||
}
|
||||
}
|
||||
|
||||
// fill in defaults if not overridden by the user
|
||||
for domain, url := range defaultResolvers {
|
||||
_, ok := domains[domain]
|
||||
if ok {
|
||||
continue
|
||||
}
|
||||
|
||||
rslv, ok := rslvrs[url]
|
||||
if !ok {
|
||||
rslv, err = newResolver(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bad resolver for %s: %w", domain, err)
|
||||
}
|
||||
rslvrs[url] = rslv
|
||||
}
|
||||
|
||||
opts = append(opts, madns.WithDomainResolver(domain, rslv))
|
||||
}
|
||||
|
||||
return madns.NewResolver(opts...)
|
||||
return gateway.NewDNSResolver(cfg.DNS.Resolvers, dohOpts...)
|
||||
}
|
||||
|
||||
@ -7,7 +7,7 @@ go 1.18
|
||||
replace github.com/ipfs/kubo => ./../../..
|
||||
|
||||
require (
|
||||
github.com/ipfs/boxo v0.8.0-rc2
|
||||
github.com/ipfs/boxo v0.8.0-rc3
|
||||
github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
|
||||
github.com/libp2p/go-libp2p v0.26.4
|
||||
github.com/multiformats/go-multiaddr v0.8.0
|
||||
@ -41,6 +41,7 @@ require (
|
||||
github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect
|
||||
github.com/flynn/noise v1.0.0 // indirect
|
||||
github.com/francoispqt/gojay v1.2.13 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.1 // indirect
|
||||
github.com/go-logr/logr v1.2.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
|
||||
@ -77,6 +78,7 @@ require (
|
||||
github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
|
||||
github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
|
||||
github.com/ipfs/go-ipfs-pq v0.0.3 // indirect
|
||||
github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect
|
||||
github.com/ipfs/go-ipfs-util v0.0.2 // indirect
|
||||
github.com/ipfs/go-ipld-cbor v0.0.6 // indirect
|
||||
github.com/ipfs/go-ipld-format v0.4.0 // indirect
|
||||
@ -153,6 +155,7 @@ require (
|
||||
github.com/samber/lo v1.36.0 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
|
||||
github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
|
||||
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
|
||||
github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect
|
||||
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
|
||||
|
||||
@ -170,6 +170,8 @@ github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0X
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
|
||||
github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q=
|
||||
github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
|
||||
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
|
||||
@ -335,8 +337,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
|
||||
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
|
||||
github.com/ipfs/boxo v0.8.0-rc2 h1:JnSlLKlIURiVsTfPs1BLv3NNCygjV1b07wGva3w4sLY=
|
||||
github.com/ipfs/boxo v0.8.0-rc2/go.mod h1:EgDiNox/+W/+ySwEotRrHlvdmrhbSAB4p22ELg+ZsCc=
|
||||
github.com/ipfs/boxo v0.8.0-rc3 h1:rttpGdhLE0zeTec8f2/e5YDgCYzEQf7dI4eRglu2ktc=
|
||||
github.com/ipfs/boxo v0.8.0-rc3/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA=
|
||||
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
|
||||
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
|
||||
github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
|
||||
@ -393,6 +395,8 @@ github.com/ipfs/go-ipfs-files v0.3.0 h1:fallckyc5PYjuMEitPNrjRfpwl7YFt69heCOUhsb
|
||||
github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs=
|
||||
github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE=
|
||||
github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3kERkRb4=
|
||||
github.com/ipfs/go-ipfs-redirects-file v0.1.1 h1:Io++k0Vf/wK+tfnhEh63Yte1oQK5VGT2hIEYpD0Rzx8=
|
||||
github.com/ipfs/go-ipfs-redirects-file v0.1.1/go.mod h1:tAwRjCV0RjLTjH8DR/AU7VYvfQECg+lpUy2Mdzv7gyk=
|
||||
github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc=
|
||||
github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8=
|
||||
github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ=
|
||||
@ -800,8 +804,11 @@ github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
|
||||
github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
|
||||
github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M=
|
||||
github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
|
||||
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
|
||||
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
|
||||
github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb h1:Ywfo8sUltxogBpFuMOFRrrSifO788kAFxmvVw31PtQQ=
|
||||
github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb/go.mod h1:ikPs9bRWicNw3S7XpJ8sK/smGwU9WcSVU3dy9qahYBM=
|
||||
github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
|
||||
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
|
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
@ -997,6 +1004,7 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx
|
||||
golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
|
||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
@ -1088,6 +1096,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
|
||||
|
||||
4
go.mod
4
go.mod
@ -16,7 +16,7 @@ require (
|
||||
github.com/gogo/protobuf v1.3.2
|
||||
github.com/google/uuid v1.3.0
|
||||
github.com/hashicorp/go-multierror v1.1.1
|
||||
github.com/ipfs/boxo v0.8.0-rc2
|
||||
github.com/ipfs/boxo v0.8.0-rc3
|
||||
github.com/ipfs/go-block-format v0.1.2
|
||||
github.com/ipfs/go-cid v0.4.0
|
||||
github.com/ipfs/go-cidutil v0.1.0
|
||||
@ -55,7 +55,6 @@ require (
|
||||
github.com/libp2p/go-libp2p-routing-helpers v0.6.1
|
||||
github.com/libp2p/go-libp2p-testing v0.12.0
|
||||
github.com/libp2p/go-socket-activation v0.1.0
|
||||
github.com/miekg/dns v1.1.50
|
||||
github.com/mitchellh/go-homedir v1.1.0
|
||||
github.com/multiformats/go-multiaddr v0.8.0
|
||||
github.com/multiformats/go-multiaddr-dns v0.3.1
|
||||
@ -173,6 +172,7 @@ require (
|
||||
github.com/mattn/go-runewidth v0.0.4 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
|
||||
github.com/miekg/dns v1.1.50 // indirect
|
||||
github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
|
||||
github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
|
||||
github.com/minio/sha256-simd v1.0.0 // indirect
|
||||
|
||||
4
go.sum
4
go.sum
@ -355,8 +355,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
|
||||
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
|
||||
github.com/ipfs/boxo v0.8.0-rc2 h1:JnSlLKlIURiVsTfPs1BLv3NNCygjV1b07wGva3w4sLY=
|
||||
github.com/ipfs/boxo v0.8.0-rc2/go.mod h1:EgDiNox/+W/+ySwEotRrHlvdmrhbSAB4p22ELg+ZsCc=
|
||||
github.com/ipfs/boxo v0.8.0-rc3 h1:rttpGdhLE0zeTec8f2/e5YDgCYzEQf7dI4eRglu2ktc=
|
||||
github.com/ipfs/boxo v0.8.0-rc3/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA=
|
||||
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
|
||||
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
|
||||
github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
|
||||
|
||||
72
test/cli/fixtures/README.md
Normal file
72
test/cli/fixtures/README.md
Normal file
@ -0,0 +1,72 @@
|
||||
# Dataset Description / Sources
|
||||
|
||||
TestGatewayHAMTDirectory.car generated with:
|
||||
|
||||
```bash
|
||||
ipfs version
|
||||
# ipfs version 0.19.0
|
||||
|
||||
export HAMT_DIR=bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm
|
||||
export IPFS_PATH=$(mktemp -d)
|
||||
|
||||
# Init and start daemon, ensure we have an empty repository.
|
||||
ipfs init --empty-repo
|
||||
ipfs daemon &> /dev/null &
|
||||
export IPFS_PID=$!
|
||||
|
||||
# Retrieve the directory listing, forcing the daemon to download all required DAGs. Kill daemon.
|
||||
curl -o dir.html http://127.0.0.1:8080/ipfs/$HAMT_DIR/
|
||||
kill $IPFS_PID
|
||||
|
||||
# Get the list with all the downloaded refs and sanity check.
|
||||
ipfs refs local > required_refs
|
||||
cat required_refs | wc -l
|
||||
# 962
|
||||
|
||||
# Get the list of all the files CIDs inside the directory and sanity check.
|
||||
cat dir.html| pup '#content tbody .ipfs-hash attr{href}' | sed 's/\/ipfs\///g;s/\?filename=.*//g' > files_refs
|
||||
cat files_refs | wc -l
|
||||
# 10100
|
||||
|
||||
# Make and export our fixture.
|
||||
ipfs files mkdir --cid-version 1 /fixtures
|
||||
cat required_refs | xargs -I {} ipfs files cp /ipfs/{} /fixtures/{}
|
||||
cat files_refs | ipfs files write --create /fixtures/files_refs
|
||||
export FIXTURE_CID=$(ipfs files stat --hash /fixtures/)
|
||||
echo $FIXTURE_CID
|
||||
# bafybeig3yoibxe56aolixqa4zk55gp5sug3qgaztkakpndzk2b2ynobd4i
|
||||
ipfs dag export $FIXTURE_CID > TestGatewayHAMTDirectory.car
|
||||
```
|
||||
|
||||
TestGatewayMultiRange.car generated with:
|
||||
|
||||
|
||||
```sh
|
||||
ipfs version
|
||||
# ipfs version 0.19.0
|
||||
|
||||
export FILE_CID=bafybeiae5abzv6j3ucqbzlpnx3pcqbr2otbnpot7d2k5pckmpymin4guau
|
||||
export IPFS_PATH=$(mktemp -d)
|
||||
|
||||
# Init and start daemon, ensure we have an empty repository.
|
||||
ipfs init --empty-repo
|
||||
ipfs daemon &> /dev/null &
|
||||
export IPFS_PID=$!
|
||||
|
||||
# Get a specific byte range from the file.
|
||||
curl http://127.0.0.1:8080/ipfs/$FILE_CID -i -H "Range: bytes=1276-1279, 29839070-29839080"
|
||||
kill $IPFS_PID
|
||||
|
||||
# Get the list with all the downloaded refs and sanity check.
|
||||
ipfs refs local > required_refs
|
||||
cat required_refs | wc -l
|
||||
# 19
|
||||
|
||||
# Make and export our fixture.
|
||||
ipfs files mkdir --cid-version 1 /fixtures
|
||||
cat required_refs | xargs -I {} ipfs files cp /ipfs/{} /fixtures/{}
|
||||
export FIXTURE_CID=$(ipfs files stat --hash /fixtures/)
|
||||
echo $FIXTURE_CID
|
||||
# bafybeicgsg3lwyn3yl75lw7sn4zhyj5dxtb7wfxwscpq6yzippetmr2w3y
|
||||
ipfs dag export $FIXTURE_CID > TestGatewayMultiRange.car
|
||||
```
|
||||
BIN
test/cli/fixtures/TestGatewayHAMTDirectory.car
Normal file
BIN
test/cli/fixtures/TestGatewayHAMTDirectory.car
Normal file
Binary file not shown.
BIN
test/cli/fixtures/TestGatewayMultiRange.car
Normal file
BIN
test/cli/fixtures/TestGatewayMultiRange.car
Normal file
Binary file not shown.
75
test/cli/gateway_range_test.go
Normal file
75
test/cli/gateway_range_test.go
Normal file
@ -0,0 +1,75 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/ipfs/kubo/test/cli/harness"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestGatewayHAMTDirectory(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
// The CID of the HAMT-sharded directory that has 10k items
|
||||
hamtCid = "bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm"
|
||||
|
||||
// fixtureCid is the CID of root of the DAG that is a subset of hamtCid DAG
|
||||
// representing the minimal set of blocks necessary for directory listing.
|
||||
// It also includes a "files_refs" file with the list of the references
|
||||
// we do NOT needs to fetch (files inside the directory)
|
||||
fixtureCid = "bafybeig3yoibxe56aolixqa4zk55gp5sug3qgaztkakpndzk2b2ynobd4i"
|
||||
)
|
||||
|
||||
// Start node
|
||||
h := harness.NewT(t)
|
||||
node := h.NewNode().Init("--empty-repo", "--profile=test").StartDaemon("--offline")
|
||||
client := node.GatewayClient()
|
||||
|
||||
// Import fixtures
|
||||
r, err := os.Open("./fixtures/TestGatewayHAMTDirectory.car")
|
||||
assert.Nil(t, err)
|
||||
defer r.Close()
|
||||
err = node.IPFSDagImport(r, fixtureCid)
|
||||
assert.Nil(t, err)
|
||||
|
||||
// Fetch HAMT directory succeeds with minimal refs
|
||||
resp := client.Get(fmt.Sprintf("/ipfs/%s/", hamtCid))
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
}
|
||||
|
||||
func TestGatewayMultiRange(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
// fileCid is the CID of the large HAMT-sharded file.
|
||||
fileCid = "bafybeiae5abzv6j3ucqbzlpnx3pcqbr2otbnpot7d2k5pckmpymin4guau"
|
||||
|
||||
// fixtureCid is the CID of root of the DAG that is a subset of fileCid DAG
|
||||
// representing the minimal set of blocks necessary for a simple byte range request.
|
||||
fixtureCid = "bafybeicgsg3lwyn3yl75lw7sn4zhyj5dxtb7wfxwscpq6yzippetmr2w3y"
|
||||
)
|
||||
|
||||
// Start node
|
||||
h := harness.NewT(t)
|
||||
node := h.NewNode().Init("--empty-repo", "--profile=test").StartDaemon("--offline")
|
||||
client := node.GatewayClient()
|
||||
|
||||
// Import fixtures
|
||||
r, err := os.Open("./fixtures/TestGatewayMultiRange.car")
|
||||
assert.Nil(t, err)
|
||||
defer r.Close()
|
||||
err = node.IPFSDagImport(r, fixtureCid)
|
||||
assert.Nil(t, err)
|
||||
|
||||
// Succeeds fetching a range of blocks we have
|
||||
resp := client.Get(fmt.Sprintf("/ipfs/%s", fileCid), func(r *http.Request) {
|
||||
r.Header.Set("Range", "bytes=1276-1279, 29839070-29839080")
|
||||
})
|
||||
assert.Equal(t, http.StatusPartialContent, resp.StatusCode)
|
||||
assert.Contains(t, resp.Body, "Content-Range: bytes 1276-1279/109266405\r\nContent-Type: text/plain; charset=utf-8\r\n\r\niana\r\n")
|
||||
assert.Contains(t, resp.Body, "Content-Range: bytes 29839070-29839080/109266405\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nEXAMPLE.COM\r\n")
|
||||
}
|
||||
@ -78,3 +78,22 @@ func (n *Node) IPFSAdd(content io.Reader, args ...string) string {
|
||||
log.Debugf("add result: %q", out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (n *Node) IPFSDagImport(content io.Reader, cid string, args ...string) error {
|
||||
log.Debugf("node %d dag import with args: %v", n.ID, args)
|
||||
fullArgs := []string{"dag", "import", "--pin-roots=false"}
|
||||
fullArgs = append(fullArgs, args...)
|
||||
res := n.Runner.MustRun(RunRequest{
|
||||
Path: n.IPFSBin,
|
||||
Args: fullArgs,
|
||||
CmdOpts: []CmdOpt{RunWithStdin(content)},
|
||||
})
|
||||
if res.Err != nil {
|
||||
return res.Err
|
||||
}
|
||||
res = n.Runner.MustRun(RunRequest{
|
||||
Path: n.IPFSBin,
|
||||
Args: []string{"block", "stat", "--offline", cid},
|
||||
})
|
||||
return res.Err
|
||||
}
|
||||
|
||||
@ -29,6 +29,21 @@ FILE_CID=bafkreihhpc5y2pqvl5rbe5uuyhqjouybfs3rvlmisccgzue2kkt5zq6upq # ./dir/asc
|
||||
test_cmp expected_block curl_ipfs_dir_block_accept_output
|
||||
'
|
||||
|
||||
test_expect_success "GET for application/vnd.ipld.raw with single range request includes correct bytes" '
|
||||
echo -n "application" > expected_file_block_single_range &&
|
||||
curl -sX GET -H "Accept: application/vnd.ipld.raw" -H "Range: bytes=6-16" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" -o curl_ipfs_file_block_single_range &&
|
||||
test_cmp expected_file_block_single_range curl_ipfs_file_block_single_range
|
||||
'
|
||||
|
||||
test_expect_success "GET for application/vnd.ipld.raw with multiple range request includes correct bytes" '
|
||||
curl -sX GET -H "Accept: application/vnd.ipld.raw" -H "Range: bytes=6-16,0-4" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" -o curl_ipfs_file_block_multiple_range &&
|
||||
test_should_contain "Content-Range: bytes 6-16/31" curl_ipfs_file_block_multiple_range &&
|
||||
test_should_contain "Content-Type: application/vnd.ipld.raw" curl_ipfs_file_block_multiple_range &&
|
||||
test_should_contain "application" curl_ipfs_file_block_multiple_range &&
|
||||
test_should_contain "Content-Range: bytes 0-4/31" curl_ipfs_file_block_multiple_range &&
|
||||
test_should_contain "hello" curl_ipfs_file_block_multiple_range
|
||||
'
|
||||
|
||||
# Make sure expected HTTP headers are returned with the block bytes
|
||||
|
||||
test_expect_success "GET response for application/vnd.ipld.raw has expected Content-Type" '
|
||||
|
||||
Reference in New Issue
Block a user