opensource/kubo
1
0
Fork 0
mirror of https://github.com/ipfs/kubo.git synced 2025-05-17 06:57:40 +08:00
Code Issues Projects Releases Wiki Activity

Update to boxo with refactored providerQueryManager. (#10595)

Browse Source
This commit is contained in:
Hector Sanjuan
2024-11-26 12:34:06 +01:00
committed by GitHub
parent ef58568964
commit 37c5060742
19 changed files with 93 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
core/commands/bitswap.go
View File

@ -5,7 +5,6 @@ import (
"io"
cmdenv "github.com/ipfs/kubo/core/commands/cmdenv"
e "github.com/ipfs/kubo/core/commands/e"
humanize "github.com/dustin/go-humanize"
bitswap "github.com/ipfs/boxo/bitswap"
@ -53,10 +52,7 @@ Print out all blocks currently on the bitswap wantlist for the local peer.`,
return ErrNotOnline
}
bs, ok := nd.Exchange.(*bitswap.Bitswap)
if !ok {
return e.TypeErr(bs, nd.Exchange)
}
bs := nd.Bitswap
pstr, found := req.Options[peerOptionName].(string)
if found {
@ -112,12 +108,7 @@ var bitswapStatCmd = &cmds.Command{
return cmds.Errorf(cmds.ErrClient, "unable to run offline: %s", ErrNotOnline)
}
bs, ok := nd.Exchange.(*bitswap.Bitswap)
if !ok {
return e.TypeErr(bs, nd.Exchange)
}
st, err := bs.Stat()
st, err := nd.Bitswap.Stat()
if err != nil {
return err
}
@ -134,7 +125,6 @@ var bitswapStatCmd = &cmds.Command{
human, _ := req.Options[bitswapHumanOptionName].(bool)
fmt.Fprintln(w, "bitswap status")
fmt.Fprintf(w, "\tprovides buffer: %d / %d\n", s.ProvideBufLen, bitswap.HasBlockBufferSize)
fmt.Fprintf(w, "\tblocks received: %d\n", s.BlocksReceived)
fmt.Fprintf(w, "\tblocks sent: %d\n", s.BlocksSent)
if human {
@ -190,17 +180,12 @@ prints the ledger associated with a given peer.
return ErrNotOnline
}
bs, ok := nd.Exchange.(*bitswap.Bitswap)
if !ok {
return e.TypeErr(bs, nd.Exchange)
}
partner, err := peer.Decode(req.Arguments[0])
if err != nil {
return err
}
return cmds.EmitOnce(res, bs.LedgerForPeer(partner))
return cmds.EmitOnce(res, nd.Bitswap.LedgerForPeer(partner))
},
Encoders: cmds.EncoderMap{
cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *server.Receipt) error {

4
core/core.go
View File

@ -19,6 +19,7 @@ import (
pin "github.com/ipfs/boxo/pinning/pinner"
"github.com/ipfs/go-datastore"
bitswap "github.com/ipfs/boxo/bitswap"
bserv "github.com/ipfs/boxo/blockservice"
bstore "github.com/ipfs/boxo/blockstore"
exchange "github.com/ipfs/boxo/exchange"
@ -102,7 +103,8 @@ type IpfsNode struct {
UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` // The UnixFS path resolver
OfflineIPLDPathResolver pathresolver.Resolver `name:"offlineIpldPathResolver"` // The IPLD path resolver that uses only locally available blocks
OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"` // The UnixFS path resolver that uses only locally available blocks
Exchange exchange.Interface // the block exchange + strategy (bitswap)
Exchange exchange.Interface // the block exchange + strategy
Bitswap *bitswap.Bitswap `optional:"true"` // The Bitswap instance
Namesys namesys.NameSystem // the name system, resolves paths to hashes
Provider provider.System // the value provider system
IpnsRepub *ipnsrp.Republisher `optional:"true"`

2
core/coreapi/pin.go
View File

@ -44,7 +44,7 @@ func (api *PinAPI) Add(ctx context.Context, p path.Path, opts ...caopts.PinAddOp
return fmt.Errorf("pin: %s", err)
}
if err := api.provider.Provide(dagNode.Cid()); err != nil {
if err := api.provider.Provide(ctx, dagNode.Cid(), true); err != nil {
return err
}

2
core/coreapi/unixfs.go
View File

@ -173,7 +173,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options
}
if !settings.OnlyHash {
if err := api.provider.Provide(nd.Cid()); err != nil {
if err := api.provider.Provide(ctx, nd.Cid(), true); err != nil {
return path.ImmutablePath{}, err
}
}

63
core/node/bitswap.go
View File

@ -5,9 +5,12 @@ import (
"time"
"github.com/ipfs/boxo/bitswap"
"github.com/ipfs/boxo/bitswap/client"
"github.com/ipfs/boxo/bitswap/network"
blockstore "github.com/ipfs/boxo/blockstore"
exchange "github.com/ipfs/boxo/exchange"
"github.com/ipfs/boxo/exchange/providing"
provider "github.com/ipfs/boxo/provider"
"github.com/ipfs/kubo/config"
irouting "github.com/ipfs/kubo/routing"
"github.com/libp2p/go-libp2p/core/host"
@ -34,7 +37,7 @@ type bitswapOptionsOut struct {
// BitswapOptions creates configuration options for Bitswap from the config file
// and whether to provide data.
func BitswapOptions(cfg *config.Config, provide bool) interface{} {
func BitswapOptions(cfg *config.Config) interface{} {
return func() bitswapOptionsOut {
var internalBsCfg config.InternalBitswap
if cfg.Internal.Bitswap != nil {
@ -42,7 +45,6 @@ func BitswapOptions(cfg *config.Config, provide bool) interface{} {
}
opts := []bitswap.Option{
bitswap.ProvideEnabled(provide),
bitswap.ProviderSearchDelay(internalBsCfg.ProviderSearchDelay.WithDefault(DefaultProviderSearchDelay)), // See https://github.com/ipfs/go-ipfs/issues/8807 for rationale
bitswap.EngineBlockstoreWorkerCount(int(internalBsCfg.EngineBlockstoreWorkerCount.WithDefault(DefaultEngineBlockstoreWorkerCount))),
bitswap.TaskWorkerCount(int(internalBsCfg.TaskWorkerCount.WithDefault(DefaultTaskWorkerCount))),
@ -55,7 +57,7 @@ func BitswapOptions(cfg *config.Config, provide bool) interface{} {
}
}
type onlineExchangeIn struct {
type bitswapIn struct {
fx.In
Mctx helpers.MetricsCtx
@ -65,19 +67,62 @@ type onlineExchangeIn struct {
BitswapOpts []bitswap.Option `group:"bitswap-options"`
}
// OnlineExchange creates new LibP2P backed block exchange (BitSwap).
// Bitswap creates the BitSwap server/client instance.
// Additional options to bitswap.New can be provided via the "bitswap-options"
// group.
func OnlineExchange() interface{} {
return func(in onlineExchangeIn, lc fx.Lifecycle) exchange.Interface {
bitswapNetwork := network.NewFromIpfsHost(in.Host, in.Rt)
func Bitswap(provide bool) interface{} {
return func(in bitswapIn, lc fx.Lifecycle) *bitswap.Bitswap {
bitswapNetwork := network.NewFromIpfsHost(in.Host)
var provider client.ProviderFinder
if provide {
provider = in.Rt
}
bs := bitswap.New(helpers.LifecycleCtx(in.Mctx, lc), bitswapNetwork, provider, in.Bs, in.BitswapOpts...)
exch := bitswap.New(helpers.LifecycleCtx(in.Mctx, lc), bitswapNetwork, in.Bs, in.BitswapOpts...)
lc.Append(fx.Hook{
OnStop: func(ctx context.Context) error {
return exch.Close()
return bs.Close()
},
})
return bs
}
}
// OnlineExchange creates new LibP2P backed block exchange.
func OnlineExchange() interface{} {
return func(in *bitswap.Bitswap, lc fx.Lifecycle) exchange.Interface {
lc.Append(fx.Hook{
OnStop: func(ctx context.Context) error {
return in.Close()
},
})
return in
}
}
type providingExchangeIn struct {
fx.In
BaseExch exchange.Interface
Provider provider.System
}
// ProvidingExchange creates a providing.Exchange with the existing exchange
// and the provider.System.
// We cannot do this in OnlineExchange because it causes cycles so this is for
// a decorator.
func ProvidingExchange(provide bool) interface{} {
return func(in providingExchangeIn, lc fx.Lifecycle) exchange.Interface {
exch := in.BaseExch
if provide {
exch = providing.New(in.BaseExch, in.Provider)
lc.Append(fx.Hook{
OnStop: func(ctx context.Context) error {
return exch.Close()
},
})
}
return exch
}
}

5
core/node/groups.go
View File

@ -293,8 +293,11 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part
shouldBitswapProvide := !cfg.Experimental.StrategicProviding
return fx.Options(
fx.Provide(BitswapOptions(cfg, shouldBitswapProvide)),
fx.Provide(BitswapOptions(cfg)),
fx.Provide(Bitswap(shouldBitswapProvide)),
fx.Provide(OnlineExchange()),
// Replace our Exchange with a Providing exchange!
fx.Decorate(ProvidingExchange(shouldBitswapProvide)),
fx.Provide(DNSResolver),
fx.Provide(Namesys(ipnsCacheSize, cfg.Ipns.MaxCacheTTL.WithDefault(config.DefaultIpnsMaxCacheTTL))),
fx.Provide(Peering),

2
docs/changelogs/v0.32.md
View File

@ -1,6 +1,6 @@
# Kubo changelog v0.32
- [v0.32.0](#v0310)
- [v0.32.0](#v0320)
## v0.32.0

2
docs/changelogs/v0.33.md
View File

@ -16,6 +16,8 @@
#### 📦️ Dependency updates
- update `boxo` to [v0.24.TODO](https://github.com/ipfs/boxo/releases/tag/v0.24.TODO)
### 📝 Changelog
### 👨‍👩‍👧‍👦 Contributors

2
docs/examples/kubo-as-a-library/go.mod
View File

@ -7,7 +7,7 @@ go 1.23
replace github.com/ipfs/kubo => ./../../..
require (
github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f
github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1
github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
github.com/libp2p/go-libp2p v0.37.0
github.com/multiformats/go-multiaddr v0.13.0

4
docs/examples/kubo-as-a-library/go.sum
View File

@ -298,8 +298,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7Uy
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f h1:3qgBUQ6BYfEAPaoSYoH90PKwVT1/iFLX7fDGGkvXZ8Y=
github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4=
github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1 h1:Ox1qTlON8qG46rUL7dDEwnIt7W9MhaidtvR/97RywWw=
github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4=
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=

2
go.mod
View File

@ -22,7 +22,7 @@ require (
github.com/hashicorp/go-version v1.7.0
github.com/ipfs-shipyard/nopfs v0.0.12
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c
github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f
github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1
github.com/ipfs/go-block-format v0.2.0
github.com/ipfs/go-cid v0.4.1
github.com/ipfs/go-cidutil v0.1.0

4
go.sum
View File

@ -362,8 +362,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7Uy
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f h1:3qgBUQ6BYfEAPaoSYoH90PKwVT1/iFLX7fDGGkvXZ8Y=
github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4=
github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1 h1:Ox1qTlON8qG46rUL7dDEwnIt7W9MhaidtvR/97RywWw=
github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4=
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=

6
test/cli/delegated_routing_v1_http_proxy_test.go
View File

@ -60,8 +60,10 @@ func TestRoutingV1Proxy(t *testing.T) {
nodes := setupNodes(t)
cidStr := nodes[0].IPFSAddStr(testutils.RandomStr(1000))
res := nodes[1].IPFS("routing", "findprovs", cidStr)
// Reprovide as initialProviderDelay still ongoing
res := nodes[0].IPFS("bitswap", "reprovide")
require.NoError(t, res.Err)
res = nodes[1].IPFS("routing", "findprovs", cidStr)
assert.Equal(t, nodes[0].PeerID().String(), res.Stdout.Trimmed())
})

4
test/cli/delegated_routing_v1_http_server_test.go
View File

@ -14,6 +14,7 @@ import (
"github.com/ipfs/kubo/test/cli/harness"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRoutingV1Server(t *testing.T) {
@ -38,6 +39,9 @@ func TestRoutingV1Server(t *testing.T) {
text := "hello world " + uuid.New().String()
cidStr := nodes[2].IPFSAddStr(text)
_ = nodes[3].IPFSAddStr(text)
// Reprovide as initialProviderDelay still ongoing
res := nodes[3].IPFS("bitswap", "reprovide")
require.NoError(t, res.Err)
cid, err := cid.Decode(cidStr)
assert.NoError(t, err)

3
test/cli/provider_test.go
View File

@ -42,6 +42,9 @@ func TestProvider(t *testing.T) {
defer nodes.StopDaemons()
cid := nodes[0].IPFSAddStr(time.Now().String())
// Reprovide as initialProviderDelay still ongoing
res := nodes[0].IPFS("bitswap", "reprovide")
require.NoError(t, res.Err)
expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...)
})

5
test/cli/routing_dht_test.go
View File

@ -84,7 +84,10 @@ func testRoutingDHT(t *testing.T, enablePubsub bool) {
t.Run("ipfs routing findprovs", func(t *testing.T) {
t.Parallel()
hash := nodes[3].IPFSAddStr("some stuff")
res := nodes[4].IPFS("routing", "findprovs", hash)
// Reprovide as initialProviderDelay still ongoing
res := nodes[3].IPFS("bitswap", "reprovide")
require.NoError(t, res.Err)
res = nodes[4].IPFS("routing", "findprovs", hash)
assert.Equal(t, nodes[3].PeerID().String(), res.Stdout.Trimmed())
})

2
test/dependencies/go.mod
View File

@ -119,7 +119,7 @@ require (
github.com/huin/goupnp v1.3.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/ipfs/bbloom v0.0.4 // indirect
github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f // indirect
github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1 // indirect
github.com/ipfs/go-block-format v0.2.0 // indirect
github.com/ipfs/go-cid v0.4.1 // indirect
github.com/ipfs/go-datastore v0.6.0 // indirect

4
test/dependencies/go.sum
View File

@ -318,8 +318,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f h1:3qgBUQ6BYfEAPaoSYoH90PKwVT1/iFLX7fDGGkvXZ8Y=
github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4=
github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1 h1:Ox1qTlON8qG46rUL7dDEwnIt7W9MhaidtvR/97RywWw=
github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4=
github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs=
github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM=
github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s=

3
test/sharness/t0220-bitswap.sh
View File

@ -18,7 +18,6 @@ test_expect_success "'ipfs bitswap stat' succeeds" '
test_expect_success "'ipfs bitswap stat' output looks good" '
cat <<EOF | unexpand -t2 >expected &&
bitswap status
provides buffer: 0 / 256
blocks received: 0
blocks sent: 0
data received: 0
@ -56,7 +55,6 @@ test_expect_success "'ipfs bitswap stat' succeeds" '
test_expect_success "'ipfs bitswap stat' output looks good" '
cat <<EOF | unexpand -t2 >expected &&
bitswap status
provides buffer: 0 / 256
blocks received: 0
blocks sent: 0
data received: 0
@ -85,7 +83,6 @@ test_expect_success "'ipfs bitswap stat --human' succeeds" '
test_expect_success "'ipfs bitswap stat --human' output looks good" '
cat <<EOF | unexpand -t2 >expected &&
bitswap status
provides buffer: 0 / 256
blocks received: 0
blocks sent: 0
data received: 0 B