mirror of
https://github.com/ipfs/kubo.git
synced 2025-05-17 23:16:11 +08:00
fix(config): wire up Provider.Enabled
flag (#10804)
* fix(config): explicit Provider.Enabled flag Adds missing config option described in https://github.com/ipfs/kubo/issues/10803 * refactor: remove Experimental.StrategicProviding removing experiment, replaced with Provider.Enabled * test(cli): routing [re]provide updated and added tests for manually triggering provide and reprovide and making them respect global configuration flag to avoid inconsistent behaviors * docs: improve DelegatedRouters * refactor: default DefaultProviderWorkerCount=16 - simplified default for both - 16 is safer for non-accelerated DHT client - acceletated DHT performs better without limit anyway - updated docs
This commit is contained in:
@ -491,6 +491,11 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
|
|||||||
if cfg.Provider.Strategy.WithDefault("") != "" && cfg.Reprovider.Strategy.IsDefault() {
|
if cfg.Provider.Strategy.WithDefault("") != "" && cfg.Reprovider.Strategy.IsDefault() {
|
||||||
log.Fatal("Invalid config. Remove unused Provider.Strategy and set Reprovider.Strategy instead. Documentation: https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy")
|
log.Fatal("Invalid config. Remove unused Provider.Strategy and set Reprovider.Strategy instead. Documentation: https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy")
|
||||||
}
|
}
|
||||||
|
if cfg.Experimental.StrategicProviding {
|
||||||
|
log.Error("Experimental.StrategicProviding was removed. Remove it from your config and set Provider.Enabled=false to remove this message. Documentation: https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#strategic-providing")
|
||||||
|
cfg.Experimental.StrategicProviding = false
|
||||||
|
cfg.Provider.Enabled = config.False
|
||||||
|
}
|
||||||
|
|
||||||
printLibp2pPorts(node)
|
printLibp2pPorts(node)
|
||||||
|
|
||||||
@ -625,17 +630,19 @@ take effect.
|
|||||||
}()
|
}()
|
||||||
|
|
||||||
if !offline {
|
if !offline {
|
||||||
// Warn users who were victims of 'lowprofile' footgun (https://github.com/ipfs/kubo/pull/10524)
|
// Warn users when provide systems are disabled
|
||||||
if cfg.Experimental.StrategicProviding {
|
if !cfg.Provider.Enabled.WithDefault(config.DefaultProviderEnabled) {
|
||||||
fmt.Print(`
|
fmt.Print(`
|
||||||
⚠️ Reprovide system is disabled due to 'Experimental.StrategicProviding=true'
|
|
||||||
|
⚠️ Provide and Reprovide systems are disabled due to 'Provide.Enabled=false'
|
||||||
⚠️ Local CIDs will not be announced to Amino DHT, making them impossible to retrieve without manual peering
|
⚠️ Local CIDs will not be announced to Amino DHT, making them impossible to retrieve without manual peering
|
||||||
⚠️ If this is not intentional, call 'ipfs config profile apply announce-on'
|
⚠️ If this is not intentional, call 'ipfs config profile apply announce-on' or set Provide.Enabled=true'
|
||||||
|
|
||||||
`)
|
`)
|
||||||
} else if cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval) == 0 {
|
} else if cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval) == 0 {
|
||||||
fmt.Print(`
|
fmt.Print(`
|
||||||
⚠️ Reprovider system is disabled due to 'Reprovider.Interval=0'
|
|
||||||
|
⚠️ Provide and Reprovide systems are disabled due to 'Reprovider.Interval=0'
|
||||||
⚠️ Local CIDs will not be announced to Amino DHT, making them impossible to retrieve without manual peering
|
⚠️ Local CIDs will not be announced to Amino DHT, making them impossible to retrieve without manual peering
|
||||||
⚠️ If this is not intentional, call 'ipfs config profile apply announce-on', or set 'Reprovider.Interval=22h'
|
⚠️ If this is not intentional, call 'ipfs config profile apply announce-on', or set 'Reprovider.Interval=22h'
|
||||||
|
|
||||||
|
@ -6,7 +6,7 @@ type Experiments struct {
|
|||||||
ShardingEnabled bool `json:",omitempty"` // deprecated by autosharding: https://github.com/ipfs/kubo/pull/8527
|
ShardingEnabled bool `json:",omitempty"` // deprecated by autosharding: https://github.com/ipfs/kubo/pull/8527
|
||||||
Libp2pStreamMounting bool
|
Libp2pStreamMounting bool
|
||||||
P2pHttpProxy bool //nolint
|
P2pHttpProxy bool //nolint
|
||||||
StrategicProviding bool
|
StrategicProviding bool `json:",omitempty"` // removed, use Provider.Enabled instead
|
||||||
OptimisticProvide bool
|
OptimisticProvide bool
|
||||||
OptimisticProvideJobsPoolSize int
|
OptimisticProvideJobsPoolSize int
|
||||||
GatewayOverLibp2p bool `json:",omitempty"`
|
GatewayOverLibp2p bool `json:",omitempty"`
|
||||||
|
@ -270,7 +270,7 @@ fetching may be degraded.
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
"announce-off": {
|
"announce-off": {
|
||||||
Description: `Disables Reprovide system (and announcing to Amino DHT).
|
Description: `Disables Provide and Reprovide systems (announcing to Amino DHT).
|
||||||
|
|
||||||
USE WITH CAUTION:
|
USE WITH CAUTION:
|
||||||
The main use case for this is setups with manual Peering.Peers config.
|
The main use case for this is setups with manual Peering.Peers config.
|
||||||
@ -279,16 +279,16 @@ fetching may be degraded.
|
|||||||
one hosting it, and other peers are not already connected to it.
|
one hosting it, and other peers are not already connected to it.
|
||||||
`,
|
`,
|
||||||
Transform: func(c *Config) error {
|
Transform: func(c *Config) error {
|
||||||
|
c.Provider.Enabled = False
|
||||||
c.Reprovider.Interval = NewOptionalDuration(0) // 0 disables periodic reprovide
|
c.Reprovider.Interval = NewOptionalDuration(0) // 0 disables periodic reprovide
|
||||||
c.Experimental.StrategicProviding = true // this is not a typo (the name is counter-intuitive)
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"announce-on": {
|
"announce-on": {
|
||||||
Description: `Re-enables Reprovide system (reverts announce-off profile).`,
|
Description: `Re-enables Provide and Reprovide systems (reverts announce-off profile).`,
|
||||||
Transform: func(c *Config) error {
|
Transform: func(c *Config) error {
|
||||||
|
c.Provider.Enabled = True
|
||||||
c.Reprovider.Interval = NewOptionalDuration(DefaultReproviderInterval) // have to apply explicit default because nil would be ignored
|
c.Reprovider.Interval = NewOptionalDuration(DefaultReproviderInterval) // have to apply explicit default because nil would be ignored
|
||||||
c.Experimental.StrategicProviding = false // this is not a typo (the name is counter-intuitive)
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
@ -1,12 +1,14 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
const (
|
const (
|
||||||
DefaultProviderWorkerCount = 64
|
DefaultProviderEnabled = true
|
||||||
|
DefaultProviderWorkerCount = 16
|
||||||
)
|
)
|
||||||
|
|
||||||
// Provider configuration describes how NEW CIDs are announced the moment they are created.
|
// Provider configuration describes how NEW CIDs are announced the moment they are created.
|
||||||
// For periodical reprovide configuration, see Reprovider.*
|
// For periodical reprovide configuration, see Reprovider.*
|
||||||
type Provider struct {
|
type Provider struct {
|
||||||
|
Enabled Flag `json:",omitempty"`
|
||||||
Strategy *OptionalString `json:",omitempty"` // Unused, you are likely looking for Reprovider.Strategy instead
|
Strategy *OptionalString `json:",omitempty"` // Unused, you are likely looking for Reprovider.Strategy instead
|
||||||
WorkerCount *OptionalInteger `json:",omitempty"` // Number of concurrent provides allowed, 0 means unlimited
|
WorkerCount *OptionalInteger `json:",omitempty"` // Number of concurrent provides allowed, 0 means unlimited
|
||||||
}
|
}
|
||||||
|
@ -48,10 +48,11 @@ type Routing struct {
|
|||||||
|
|
||||||
IgnoreProviders []string `json:",omitempty"`
|
IgnoreProviders []string `json:",omitempty"`
|
||||||
|
|
||||||
|
// Simplified configuration used by default when Routing.Type=auto|autoclient
|
||||||
DelegatedRouters []string `json:",omitempty"`
|
DelegatedRouters []string `json:",omitempty"`
|
||||||
|
|
||||||
|
// Advanced configuration used when Routing.Type=custom
|
||||||
Routers Routers `json:",omitempty"`
|
Routers Routers `json:",omitempty"`
|
||||||
|
|
||||||
Methods Methods `json:",omitempty"`
|
Methods Methods `json:",omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -9,6 +9,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/ipfs/kubo/config"
|
||||||
cmdenv "github.com/ipfs/kubo/core/commands/cmdenv"
|
cmdenv "github.com/ipfs/kubo/core/commands/cmdenv"
|
||||||
|
|
||||||
dag "github.com/ipfs/boxo/ipld/merkledag"
|
dag "github.com/ipfs/boxo/ipld/merkledag"
|
||||||
@ -158,6 +159,14 @@ var provideRefRoutingCmd = &cmds.Command{
|
|||||||
if !nd.IsOnline {
|
if !nd.IsOnline {
|
||||||
return ErrNotOnline
|
return ErrNotOnline
|
||||||
}
|
}
|
||||||
|
// respect global config
|
||||||
|
cfg, err := nd.Repo.Config()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !cfg.Provider.Enabled.WithDefault(config.DefaultProviderEnabled) {
|
||||||
|
return errors.New("invalid configuration: Provider.Enabled is set to 'false'")
|
||||||
|
}
|
||||||
|
|
||||||
if len(nd.PeerHost.Network().Conns()) == 0 {
|
if len(nd.PeerHost.Network().Conns()) == 0 {
|
||||||
return errors.New("cannot provide, no connected peers")
|
return errors.New("cannot provide, no connected peers")
|
||||||
@ -254,6 +263,18 @@ Trigger reprovider to announce our data to network.
|
|||||||
return ErrNotOnline
|
return ErrNotOnline
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// respect global config
|
||||||
|
cfg, err := nd.Repo.Config()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !cfg.Provider.Enabled.WithDefault(config.DefaultProviderEnabled) {
|
||||||
|
return errors.New("invalid configuration: Provider.Enabled is set to 'false'")
|
||||||
|
}
|
||||||
|
if cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval) == 0 {
|
||||||
|
return errors.New("invalid configuration: Reprovider.Interval is set to '0'")
|
||||||
|
}
|
||||||
|
|
||||||
err = nd.Provider.Reprovide(req.Context)
|
err = nd.Provider.Reprovide(req.Context)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -83,7 +83,7 @@ type bitswapIn struct {
|
|||||||
// Bitswap creates the BitSwap server/client instance.
|
// Bitswap creates the BitSwap server/client instance.
|
||||||
// If Bitswap.ServerEnabled is false, the node will act only as a client
|
// If Bitswap.ServerEnabled is false, the node will act only as a client
|
||||||
// using an empty blockstore to prevent serving blocks to other peers.
|
// using an empty blockstore to prevent serving blocks to other peers.
|
||||||
func Bitswap(serverEnabled bool) interface{} {
|
func Bitswap(serverEnabled, libp2pEnabled, httpEnabled bool) interface{} {
|
||||||
return func(in bitswapIn, lc fx.Lifecycle) (*bitswap.Bitswap, error) {
|
return func(in bitswapIn, lc fx.Lifecycle) (*bitswap.Bitswap, error) {
|
||||||
var bitswapNetworks, bitswapLibp2p network.BitSwapNetwork
|
var bitswapNetworks, bitswapLibp2p network.BitSwapNetwork
|
||||||
var bitswapBlockstore blockstore.Blockstore = in.Bs
|
var bitswapBlockstore blockstore.Blockstore = in.Bs
|
||||||
@ -93,7 +93,8 @@ func Bitswap(serverEnabled bool) interface{} {
|
|||||||
bitswapLibp2p = bsnet.NewFromIpfsHost(in.Host)
|
bitswapLibp2p = bsnet.NewFromIpfsHost(in.Host)
|
||||||
}
|
}
|
||||||
|
|
||||||
if httpCfg := in.Cfg.HTTPRetrieval; httpCfg.Enabled.WithDefault(config.DefaultHTTPRetrievalEnabled) {
|
if httpEnabled {
|
||||||
|
httpCfg := in.Cfg.HTTPRetrieval
|
||||||
maxBlockSize, err := humanize.ParseBytes(httpCfg.MaxBlockSize.WithDefault(config.DefaultHTTPRetrievalMaxBlockSize))
|
maxBlockSize, err := humanize.ParseBytes(httpCfg.MaxBlockSize.WithDefault(config.DefaultHTTPRetrievalMaxBlockSize))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@ -136,7 +137,7 @@ func Bitswap(serverEnabled bool) interface{} {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Explicitly enable/disable server to ensure desired provide mode
|
// Explicitly enable/disable server
|
||||||
in.BitswapOpts = append(in.BitswapOpts, bitswap.WithServerEnabled(serverEnabled))
|
in.BitswapOpts = append(in.BitswapOpts, bitswap.WithServerEnabled(serverEnabled))
|
||||||
|
|
||||||
bs := bitswap.New(helpers.LifecycleCtx(in.Mctx, lc), bitswapNetworks, providerQueryMgr, bitswapBlockstore, in.BitswapOpts...)
|
bs := bitswap.New(helpers.LifecycleCtx(in.Mctx, lc), bitswapNetworks, providerQueryMgr, bitswapBlockstore, in.BitswapOpts...)
|
||||||
|
@ -337,16 +337,18 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part
|
|||||||
|
|
||||||
isBitswapLibp2pEnabled := cfg.Bitswap.Libp2pEnabled.WithDefault(config.DefaultBitswapLibp2pEnabled)
|
isBitswapLibp2pEnabled := cfg.Bitswap.Libp2pEnabled.WithDefault(config.DefaultBitswapLibp2pEnabled)
|
||||||
isBitswapServerEnabled := cfg.Bitswap.ServerEnabled.WithDefault(config.DefaultBitswapServerEnabled)
|
isBitswapServerEnabled := cfg.Bitswap.ServerEnabled.WithDefault(config.DefaultBitswapServerEnabled)
|
||||||
|
isHTTPRetrievalEnabled := cfg.HTTPRetrieval.Enabled.WithDefault(config.DefaultHTTPRetrievalEnabled)
|
||||||
|
|
||||||
// Don't provide from bitswap when the legacy noop experiment "strategic provider service" is active
|
// Right now Provider and Reprovider systems are tied together - disabling Reprovider by setting interval to 0 disables Provider
|
||||||
isBitswapServerEnabled = isBitswapServerEnabled && !cfg.Experimental.StrategicProviding
|
// and vice versa: Provider.Enabled=false will disable both Provider of new CIDs and the Reprovider of old ones.
|
||||||
|
isProviderEnabled := cfg.Provider.Enabled.WithDefault(config.DefaultProviderEnabled) && cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval) != 0
|
||||||
|
|
||||||
return fx.Options(
|
return fx.Options(
|
||||||
fx.Provide(BitswapOptions(cfg)),
|
fx.Provide(BitswapOptions(cfg)),
|
||||||
fx.Provide(Bitswap(isBitswapServerEnabled)),
|
fx.Provide(Bitswap(isBitswapServerEnabled, isBitswapLibp2pEnabled, isHTTPRetrievalEnabled)),
|
||||||
fx.Provide(OnlineExchange(isBitswapLibp2pEnabled)),
|
fx.Provide(OnlineExchange(isBitswapLibp2pEnabled)),
|
||||||
// Replace our Exchange with a Providing exchange!
|
// Replace our Exchange with a Providing exchange!
|
||||||
fx.Decorate(ProvidingExchange(isBitswapServerEnabled)),
|
fx.Decorate(ProvidingExchange(isProviderEnabled && isBitswapServerEnabled)),
|
||||||
fx.Provide(DNSResolver),
|
fx.Provide(DNSResolver),
|
||||||
fx.Provide(Namesys(ipnsCacheSize, cfg.Ipns.MaxCacheTTL.WithDefault(config.DefaultIpnsMaxCacheTTL))),
|
fx.Provide(Namesys(ipnsCacheSize, cfg.Ipns.MaxCacheTTL.WithDefault(config.DefaultIpnsMaxCacheTTL))),
|
||||||
fx.Provide(Peering),
|
fx.Provide(Peering),
|
||||||
@ -358,7 +360,7 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part
|
|||||||
|
|
||||||
LibP2P(bcfg, cfg, userResourceOverrides),
|
LibP2P(bcfg, cfg, userResourceOverrides),
|
||||||
OnlineProviders(
|
OnlineProviders(
|
||||||
cfg.Experimental.StrategicProviding,
|
isProviderEnabled,
|
||||||
cfg.Reprovider.Strategy.WithDefault(config.DefaultReproviderStrategy),
|
cfg.Reprovider.Strategy.WithDefault(config.DefaultReproviderStrategy),
|
||||||
cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval),
|
cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval),
|
||||||
cfg.Routing.AcceleratedDHTClient.WithDefault(config.DefaultAcceleratedDHTClient),
|
cfg.Routing.AcceleratedDHTClient.WithDefault(config.DefaultAcceleratedDHTClient),
|
||||||
|
@ -132,8 +132,8 @@ https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtcli
|
|||||||
// ONLINE/OFFLINE
|
// ONLINE/OFFLINE
|
||||||
|
|
||||||
// OnlineProviders groups units managing provider routing records online
|
// OnlineProviders groups units managing provider routing records online
|
||||||
func OnlineProviders(useStrategicProviding bool, reprovideStrategy string, reprovideInterval time.Duration, acceleratedDHTClient bool, provideWorkerCount int) fx.Option {
|
func OnlineProviders(provide bool, reprovideStrategy string, reprovideInterval time.Duration, acceleratedDHTClient bool, provideWorkerCount int) fx.Option {
|
||||||
if useStrategicProviding {
|
if !provide {
|
||||||
return OfflineProviders()
|
return OfflineProviders()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -181,12 +181,16 @@ to delays in initial advertisements (provides).
|
|||||||
Provides and Reprovides now have separate queues, allowing for immediate
|
Provides and Reprovides now have separate queues, allowing for immediate
|
||||||
provide of new CIDs and optimised batching of reprovides.
|
provide of new CIDs and optimised batching of reprovides.
|
||||||
|
|
||||||
This change introduces a new configuration option for limiting the number of
|
###### New `Provider` configuration options
|
||||||
concurrent provide operations:
|
|
||||||
[`Provider.WorkerCount`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providerworkercount).
|
This change introduces a new configuration options:
|
||||||
|
|
||||||
|
- [`Provider.Enabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providerenabled) is a global flag for disabling both [Provider](https://github.com/ipfs/kubo/blob/master/docs/config.md#provider) and [Reprovider](https://github.com/ipfs/kubo/blob/master/docs/config.md#reprovider) systems (announcing new/old CIDs to amino DHT).
|
||||||
|
- [`Provider.WorkerCount`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providerworkercount) for limiting the number of concurrent provide operations, allows for fine-tuning the trade-off between announcement speed and system load when announcing new CIDs.
|
||||||
|
- Removed `Experimental.StrategicProviding`. Superseded by `Provider.Enabled`, `Reprovider.Interval` and [`Reprovider.Strategy`](https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy).
|
||||||
|
|
||||||
> [!TIP]
|
> [!TIP]
|
||||||
> Users who need to provide large volumes of content immediately should consider removing the cap on concurrent provide operations and also set `Routing.AcceleratedDHTClient` to `true`.
|
> Users who need to provide large volumes of content immediately should consider setting `Routing.AcceleratedDHTClient` to `true`. If that is not enough, consider adjusting `Provider.WorkerCount` to a higher value.
|
||||||
|
|
||||||
###### Deprecated `ipfs stats provider`
|
###### Deprecated `ipfs stats provider`
|
||||||
|
|
||||||
|
@ -110,6 +110,7 @@ config file at runtime.
|
|||||||
- [`Pinning.RemoteServices: Policies.MFS.PinName`](#pinningremoteservices-policiesmfspinname)
|
- [`Pinning.RemoteServices: Policies.MFS.PinName`](#pinningremoteservices-policiesmfspinname)
|
||||||
- [`Pinning.RemoteServices: Policies.MFS.RepinInterval`](#pinningremoteservices-policiesmfsrepininterval)
|
- [`Pinning.RemoteServices: Policies.MFS.RepinInterval`](#pinningremoteservices-policiesmfsrepininterval)
|
||||||
- [`Provider`](#provider)
|
- [`Provider`](#provider)
|
||||||
|
- [`Provider.Enabled`](#providerenabled)
|
||||||
- [`Provider.Strategy`](#providerstrategy)
|
- [`Provider.Strategy`](#providerstrategy)
|
||||||
- [`Provider.WorkerCount`](#providerworkercount)
|
- [`Provider.WorkerCount`](#providerworkercount)
|
||||||
- [`Pubsub`](#pubsub)
|
- [`Pubsub`](#pubsub)
|
||||||
@ -1110,7 +1111,7 @@ $ ipfs config --json Gateway.PublicGateways '{"localhost": null }'
|
|||||||
|
|
||||||
### `Gateway` recipes
|
### `Gateway` recipes
|
||||||
|
|
||||||
Below is a list of the most common public gateway setups.
|
Below is a list of the most common gateway setups.
|
||||||
|
|
||||||
* Public [subdomain gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway) at `http://{cid}.ipfs.dweb.link` (each content root gets its own Origin)
|
* Public [subdomain gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway) at `http://{cid}.ipfs.dweb.link` (each content root gets its own Origin)
|
||||||
```console
|
```console
|
||||||
@ -1121,6 +1122,7 @@ Below is a list of the most common public gateway setups.
|
|||||||
}
|
}
|
||||||
}'
|
}'
|
||||||
```
|
```
|
||||||
|
- **Performance:** consider running with `Routing.AcceleratedDHTClient=true` and either `Provider.Enabled=false` (avoid providing newly retrieved blocks) or `Provider.WorkerCount=0` (provide as fast as possible, at the cost of increased load)
|
||||||
- **Backward-compatible:** this feature enables automatic redirects from content paths to subdomains:
|
- **Backward-compatible:** this feature enables automatic redirects from content paths to subdomains:
|
||||||
|
|
||||||
`http://dweb.link/ipfs/{cid}` → `http://{cid}.ipfs.dweb.link`
|
`http://dweb.link/ipfs/{cid}` → `http://{cid}.ipfs.dweb.link`
|
||||||
@ -1145,6 +1147,7 @@ Below is a list of the most common public gateway setups.
|
|||||||
}
|
}
|
||||||
}'
|
}'
|
||||||
```
|
```
|
||||||
|
- **Performance:** when running an open, recursive gateway consider running with `Routing.AcceleratedDHTClient=true` and either `Provider.Enabled=false` (avoid providing newly retrieved blocks) or `Provider.WorkerCount=0` (provide as fast as possible, at the cost of increased load)
|
||||||
|
|
||||||
* Public [DNSLink](https://dnslink.io/) gateway resolving every hostname passed in `Host` header.
|
* Public [DNSLink](https://dnslink.io/) gateway resolving every hostname passed in `Host` header.
|
||||||
```console
|
```console
|
||||||
@ -1503,15 +1506,28 @@ commands.
|
|||||||
|
|
||||||
For periodical DHT reprovide settings, see [`Reprovide.*`](#reprovider).
|
For periodical DHT reprovide settings, see [`Reprovide.*`](#reprovider).
|
||||||
|
|
||||||
|
### `Provider.Enabled`
|
||||||
|
|
||||||
|
Controls whether Kubo provider and reprovide systems are enabled.
|
||||||
|
|
||||||
|
> [!CAUTION]
|
||||||
|
> Disabling this, will disable BOTH `Provider` system for new CIDs
|
||||||
|
> and the periodical reprovide ([`Reprovider.Interval`](#reprovider)) of old CIDs.
|
||||||
|
|
||||||
|
Default: `true`
|
||||||
|
|
||||||
|
Type: `flag`
|
||||||
|
|
||||||
### `Provider.Strategy`
|
### `Provider.Strategy`
|
||||||
|
|
||||||
Legacy, not used at the moment, see [`Reprovider.Strategy`](#reproviderstrategy) instead.
|
Legacy, not used at the moment, see [`Reprovider.Strategy`](#reproviderstrategy) instead.
|
||||||
|
|
||||||
### `Provider.WorkerCount`
|
### `Provider.WorkerCount`
|
||||||
|
|
||||||
Sets the maximum number of _concurrent_ DHT provide operations. DHT reprovides
|
Sets the maximum number of _concurrent_ DHT provide operations (announcement of new CIDs).
|
||||||
operations do **not** count against that limit. A value of `0` allows an
|
|
||||||
unlimited number of provide workers.
|
[`Reprovider`](#reprovider) operations do **not** count against this limit.
|
||||||
|
A value of `0` allows an unlimited number of provide workers.
|
||||||
|
|
||||||
If the [accelerated DHT client](#routingaccelerateddhtclient) is enabled, each
|
If the [accelerated DHT client](#routingaccelerateddhtclient) is enabled, each
|
||||||
provide operation opens ~20 connections in parallel. With the standard DHT
|
provide operation opens ~20 connections in parallel. With the standard DHT
|
||||||
@ -1520,13 +1536,17 @@ connections, with at most 10 active at once. Provides complete more quickly
|
|||||||
when using the accelerated client. Be mindful of how many simultaneous
|
when using the accelerated client. Be mindful of how many simultaneous
|
||||||
connections this setting can generate.
|
connections this setting can generate.
|
||||||
|
|
||||||
For nodes without strict connection limits that need to provide large volumes
|
> [!CAUTION]
|
||||||
of content immediately, we recommend enabling the `Routing.AcceleratedDHTClient` and
|
> For nodes without strict connection limits that need to provide large volumes
|
||||||
setting `Provider.WorkerCount` to `0` (unlimited).
|
> of content immediately, we recommend enabling the `Routing.AcceleratedDHTClient` and
|
||||||
|
> setting `Provider.WorkerCount` to `0` (unlimited).
|
||||||
|
>
|
||||||
|
> At the same time, mind that raising this value too high may lead to increased load.
|
||||||
|
> Proceed with caution, ensure proper hardware and networking are in place.
|
||||||
|
|
||||||
Default: `64`
|
Default: `16`
|
||||||
|
|
||||||
Type: `integer` (non-negative; `0` means unlimited number of workers)
|
Type: `optionalInteger` (non-negative; `0` means unlimited number of workers)
|
||||||
|
|
||||||
## `Pubsub`
|
## `Pubsub`
|
||||||
|
|
||||||
@ -1704,7 +1724,11 @@ system.
|
|||||||
Note: disabling content reproviding will result in other nodes on the network
|
Note: disabling content reproviding will result in other nodes on the network
|
||||||
not being able to discover that you have the objects that you have. If you want
|
not being able to discover that you have the objects that you have. If you want
|
||||||
to have this disabled and keep the network aware of what you have, you must
|
to have this disabled and keep the network aware of what you have, you must
|
||||||
manually announce your content periodically.
|
manually announce your content periodically or run your own routing system
|
||||||
|
and convince users to add it to [`Routing.DelegatedRouters`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingdelegatedrouters).
|
||||||
|
|
||||||
|
> [!CAUTION]
|
||||||
|
> To maintain backward-compatibility, setting `Reprovider.Interval=0` will also disable Provider system (equivalent of `Provider.Enabled=false`)
|
||||||
|
|
||||||
Default: `22h` (`DefaultReproviderInterval`)
|
Default: `22h` (`DefaultReproviderInterval`)
|
||||||
|
|
||||||
@ -1868,12 +1892,13 @@ Type: `array[string]`
|
|||||||
|
|
||||||
### `Routing.DelegatedRouters`
|
### `Routing.DelegatedRouters`
|
||||||
|
|
||||||
This is an array of URL hostnames that support the [Delegated Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/) which are used alongside the DHT when [`Routing.Type`](#routingtype) is set to `auto` or `autoclient`.
|
An array of URL hostnames for delegated routers to be queried in addition to the Amino DHT when `Routing.Type` is set to `auto` (default) or `autoclient`.
|
||||||
|
These endpoints must support the [Delegated Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/).
|
||||||
|
|
||||||
> [!TIP]
|
> [!TIP]
|
||||||
> Delegated routing allows IPFS implementations to offload tasks like content routing, peer routing, and naming to a separate process or server while also benefiting from HTTP caching.
|
> Delegated routing allows IPFS implementations to offload tasks like content routing, peer routing, and naming to a separate process or server while also benefiting from HTTP caching.
|
||||||
>
|
>
|
||||||
> One can run their own delegated router either by implementing the [Delegated Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/) themselves, or by using [Someguy](https://github.com/ipfs/someguy), a turn-key implementation that proxies requests to the Amino DHT and other delegated routing servers, such as the Network Indexer at `cid.contact`. Public utility instance of Someguy is hosted at [`https://delegated-ipfs.dev`](https://docs.ipfs.tech/concepts/public-utilities/#delegated-routing).
|
> One can run their own delegated router either by implementing the [Delegated Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/) themselves, or by using [Someguy](https://github.com/ipfs/someguy), a turn-key implementation that proxies requests to other routing systems. A public utility instance of Someguy is hosted at [`https://delegated-ipfs.dev`](https://docs.ipfs.tech/concepts/public-utilities/#delegated-routing).
|
||||||
|
|
||||||
Default: `["https://cid.contact"]` (empty or `nil` will also use this default; to disable delegated routing, set `Routing.Type` to `dht` or `dhtclient`)
|
Default: `["https://cid.contact"]` (empty or `nil` will also use this default; to disable delegated routing, set `Routing.Type` to `dht` or `dhtclient`)
|
||||||
|
|
||||||
@ -1881,11 +1906,14 @@ Type: `array[string]`
|
|||||||
|
|
||||||
### `Routing.Routers`
|
### `Routing.Routers`
|
||||||
|
|
||||||
**EXPERIMENTAL: `Routing.Routers` configuration may change in future release**
|
Alternative configuration used when `Routing.Type=custom`.
|
||||||
|
|
||||||
Map of additional Routers.
|
> [!WARNING]
|
||||||
|
> **EXPERIMENTAL: `Routing.Routers` configuration may change in future release**
|
||||||
|
>
|
||||||
|
> Consider this advanced low-level config: Most users can simply use `Routing.Type=auto` or `autoclient` and set up basic config in user-friendly [`Routing.DelegatedRouters`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingdelegatedrouters).
|
||||||
|
|
||||||
Allows for extending the default routing (Amino DHT) with alternative Router
|
Allows for replacing the default routing (Amino DHT) with alternative Router
|
||||||
implementations.
|
implementations.
|
||||||
|
|
||||||
The map key is a name of a Router, and the value is its configuration.
|
The map key is a name of a Router, and the value is its configuration.
|
||||||
@ -1945,7 +1973,14 @@ Type: `object[string->string]`
|
|||||||
|
|
||||||
### `Routing: Methods`
|
### `Routing: Methods`
|
||||||
|
|
||||||
`Methods:map` will define which routers will be executed per method. The key will be the name of the method: `"provide"`, `"find-providers"`, `"find-peers"`, `"put-ipns"`, `"get-ipns"`. All methods must be added to the list.
|
`Methods:map` will define which routers will be executed per method used when `Routing.Type=custom`.
|
||||||
|
|
||||||
|
> [!WARNING]
|
||||||
|
> **EXPERIMENTAL: `Routing.Routers` configuration may change in future release**
|
||||||
|
>
|
||||||
|
> Consider this advanced low-level config: Most users can simply use `Routing.Type=auto` or `autoclient` and set up basic config in user-friendly [`Routing.DelegatedRouters`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingdelegatedrouters).
|
||||||
|
|
||||||
|
The key will be the name of the method: `"provide"`, `"find-providers"`, `"find-peers"`, `"put-ipns"`, `"get-ipns"`. All methods must be added to the list.
|
||||||
|
|
||||||
The value will contain:
|
The value will contain:
|
||||||
- `RouterName:string`: Name of the router. It should be one of the previously added to `Routing.Routers` list.
|
- `RouterName:string`: Name of the router. It should be one of the previously added to `Routing.Routers` list.
|
||||||
|
@ -537,27 +537,9 @@ ipfs config --json Swarm.RelayClient.Enabled true
|
|||||||
|
|
||||||
### State
|
### State
|
||||||
|
|
||||||
Experimental, disabled by default.
|
`Experimental.StrategicProviding` was removed in Kubo v0.35.
|
||||||
|
|
||||||
Replaces the existing provide mechanism with a robust, strategic provider system. Currently enabling this option will provide nothing.
|
Replaced by [`Provide.Enabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providerenabled) and [`Reprovider.Strategy`](https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy).
|
||||||
|
|
||||||
### How to enable
|
|
||||||
|
|
||||||
Modify your ipfs config:
|
|
||||||
|
|
||||||
```
|
|
||||||
ipfs config --json Experimental.StrategicProviding true
|
|
||||||
```
|
|
||||||
|
|
||||||
### Road to being a real feature
|
|
||||||
|
|
||||||
- [ ] needs real-world testing
|
|
||||||
- [ ] needs adoption
|
|
||||||
- [ ] needs to support all provider subsystem features
|
|
||||||
- [X] provide nothing
|
|
||||||
- [ ] provide roots
|
|
||||||
- [ ] provide all
|
|
||||||
- [ ] provide strategic
|
|
||||||
|
|
||||||
## GraphSync
|
## GraphSync
|
||||||
|
|
||||||
|
@ -7,6 +7,7 @@ import (
|
|||||||
|
|
||||||
"github.com/ipfs/kubo/test/cli/harness"
|
"github.com/ipfs/kubo/test/cli/harness"
|
||||||
"github.com/ipfs/kubo/test/cli/testutils"
|
"github.com/ipfs/kubo/test/cli/testutils"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -33,11 +34,11 @@ func TestProvider(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Run("Basic Providing", func(t *testing.T) {
|
t.Run("Provider.Enabled=true announces new CIDs created by ipfs add", func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
nodes := initNodes(t, 2, func(n *harness.Node) {
|
nodes := initNodes(t, 2, func(n *harness.Node) {
|
||||||
n.SetIPFSConfig("Experimental.StrategicProviding", false)
|
n.SetIPFSConfig("Provider.Enabled", true)
|
||||||
})
|
})
|
||||||
defer nodes.StopDaemons()
|
defer nodes.StopDaemons()
|
||||||
|
|
||||||
@ -48,11 +49,11 @@ func TestProvider(t *testing.T) {
|
|||||||
expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...)
|
expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...)
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("Basic Strategic Providing", func(t *testing.T) {
|
t.Run("Provider.Enabled=false disables announcement of new CID from ipfs add", func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
nodes := initNodes(t, 2, func(n *harness.Node) {
|
nodes := initNodes(t, 2, func(n *harness.Node) {
|
||||||
n.SetIPFSConfig("Experimental.StrategicProviding", true)
|
n.SetIPFSConfig("Provider.Enabled", false)
|
||||||
})
|
})
|
||||||
defer nodes.StopDaemons()
|
defer nodes.StopDaemons()
|
||||||
|
|
||||||
@ -60,6 +61,75 @@ func TestProvider(t *testing.T) {
|
|||||||
expectNoProviders(t, cid, nodes[1:]...)
|
expectNoProviders(t, cid, nodes[1:]...)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("Provider.Enabled=false disables manual announcement via RPC command", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
nodes := initNodes(t, 2, func(n *harness.Node) {
|
||||||
|
n.SetIPFSConfig("Provider.Enabled", false)
|
||||||
|
})
|
||||||
|
defer nodes.StopDaemons()
|
||||||
|
|
||||||
|
cid := nodes[0].IPFSAddStr(time.Now().String())
|
||||||
|
res := nodes[0].RunIPFS("routing", "provide", cid)
|
||||||
|
assert.Contains(t, res.Stderr.Trimmed(), "invalid configuration: Provider.Enabled is set to 'false'")
|
||||||
|
assert.Equal(t, 1, res.ExitCode())
|
||||||
|
|
||||||
|
expectNoProviders(t, cid, nodes[1:]...)
|
||||||
|
})
|
||||||
|
|
||||||
|
// Right now Provide and Reprovide are tied together
|
||||||
|
t.Run("Reprovide.Interval=0 disables announcement of new CID too", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
nodes := initNodes(t, 2, func(n *harness.Node) {
|
||||||
|
n.SetIPFSConfig("Reprovider.Interval", "0")
|
||||||
|
})
|
||||||
|
defer nodes.StopDaemons()
|
||||||
|
|
||||||
|
cid := nodes[0].IPFSAddStr(time.Now().String())
|
||||||
|
expectNoProviders(t, cid, nodes[1:]...)
|
||||||
|
})
|
||||||
|
|
||||||
|
// It is a lesser evil - forces users to fix their config and have some sort of interval
|
||||||
|
t.Run("Manual Reprovider trigger does not work when periodic Reprovider is disabled", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
nodes := initNodes(t, 2, func(n *harness.Node) {
|
||||||
|
n.SetIPFSConfig("Reprovider.Interval", "0")
|
||||||
|
})
|
||||||
|
defer nodes.StopDaemons()
|
||||||
|
|
||||||
|
cid := nodes[0].IPFSAddStr(time.Now().String(), "--offline")
|
||||||
|
|
||||||
|
expectNoProviders(t, cid, nodes[1:]...)
|
||||||
|
|
||||||
|
res := nodes[0].RunIPFS("routing", "reprovide")
|
||||||
|
assert.Contains(t, res.Stderr.Trimmed(), "invalid configuration: Reprovider.Interval is set to '0'")
|
||||||
|
assert.Equal(t, 1, res.ExitCode())
|
||||||
|
|
||||||
|
expectNoProviders(t, cid, nodes[1:]...)
|
||||||
|
})
|
||||||
|
|
||||||
|
// It is a lesser evil - forces users to fix their config and have some sort of interval
|
||||||
|
t.Run("Manual Reprovider trigger does not work when Provider system is disabled", func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
nodes := initNodes(t, 2, func(n *harness.Node) {
|
||||||
|
n.SetIPFSConfig("Provider.Enabled", false)
|
||||||
|
})
|
||||||
|
defer nodes.StopDaemons()
|
||||||
|
|
||||||
|
cid := nodes[0].IPFSAddStr(time.Now().String(), "--offline")
|
||||||
|
|
||||||
|
expectNoProviders(t, cid, nodes[1:]...)
|
||||||
|
|
||||||
|
res := nodes[0].RunIPFS("routing", "reprovide")
|
||||||
|
assert.Contains(t, res.Stderr.Trimmed(), "invalid configuration: Provider.Enabled is set to 'false'")
|
||||||
|
assert.Equal(t, 1, res.ExitCode())
|
||||||
|
|
||||||
|
expectNoProviders(t, cid, nodes[1:]...)
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("Reprovides with 'all' strategy", func(t *testing.T) {
|
t.Run("Reprovides with 'all' strategy", func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@ -149,20 +219,4 @@ func TestProvider(t *testing.T) {
|
|||||||
expectProviders(t, cidBarDir, nodes[0].PeerID().String(), nodes[1:]...)
|
expectProviders(t, cidBarDir, nodes[0].PeerID().String(), nodes[1:]...)
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("Providing works without ticking", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
nodes := initNodes(t, 2, func(n *harness.Node) {
|
|
||||||
n.SetIPFSConfig("Reprovider.Interval", "0")
|
|
||||||
})
|
|
||||||
defer nodes.StopDaemons()
|
|
||||||
|
|
||||||
cid := nodes[0].IPFSAddStr(time.Now().String(), "--offline")
|
|
||||||
|
|
||||||
expectNoProviders(t, cid, nodes[1:]...)
|
|
||||||
|
|
||||||
nodes[0].IPFS("routing", "reprovide")
|
|
||||||
|
|
||||||
expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
Reference in New Issue
Block a user