feat: periodic version check and json config (#10438)

Co-authored-by: Lucas Molas <schomatis@gmail.com>
Co-authored-by: Marcin Rataj <lidel@lidel.org>

cmd/ipfs/kubo/daemon.go

@@ -1,9 +1,11 @@
 package kubo
 
 import (
+	"context"
 	"errors"
 	_ "expvar"
 	"fmt"
+	"math"
 	"net"
 	"net/http"
 	_ "net/http/pprof"
@@ -438,9 +440,11 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
 		return fmt.Errorf("unrecognized routing option: %s", routingOption)
 	}
 
-	agentVersionSuffixString, _ := req.Options[agentVersionSuffix].(string)
-	if agentVersionSuffixString != "" {
-		version.SetUserAgentSuffix(agentVersionSuffixString)
+	// Set optional agent version suffix
+	versionSuffixFromCli, _ := req.Options[agentVersionSuffix].(string)
+	versionSuffix := cfg.Version.AgentSuffix.WithDefault(versionSuffixFromCli)
+	if versionSuffix != "" {
+		version.SetUserAgentSuffix(versionSuffix)
 	}
 
 	node, err := core.NewNode(req.Context, ncfg)
@@ -610,6 +614,15 @@ take effect.
 			}
 			if len(peers) == 0 {
 				log.Error("failed to bootstrap (no peers found): consider updating Bootstrap or Peering section of your config")
+			} else {
+				// After 1 minute we should have enough peers
+				// to run informed version check
+				startVersionChecker(
+					cctx.Context(),
+					node,
+					cfg.Version.SwarmCheckEnabled.WithDefault(true),
+					cfg.Version.SwarmCheckPercentThreshold.WithDefault(config.DefaultSwarmCheckPercentThreshold),
+				)
 			}
 		})
 	}
@@ -1056,3 +1069,41 @@ func printVersion() {
 	fmt.Printf("System version: %s\n", runtime.GOARCH+"/"+runtime.GOOS)
 	fmt.Printf("Golang version: %s\n", runtime.Version())
 }
+
+func startVersionChecker(ctx context.Context, nd *core.IpfsNode, enabled bool, percentThreshold int64) {
+	if !enabled {
+		return
+	}
+	ticker := time.NewTicker(time.Hour)
+	defer ticker.Stop()
+	go func() {
+		for {
+			o, err := commands.DetectNewKuboVersion(nd, percentThreshold)
+			if err != nil {
+				// The version check is best-effort, and may fail in custom
+				// configurations that do not run standard WAN DHT. If it
+				// errors here, no point in spamming logs: og once and exit.
+				log.Errorw("initial version check failed, will not be run again", "error", err)
+				return
+			}
+			if o.UpdateAvailable {
+				newerPercent := fmt.Sprintf("%.0f%%", math.Round(float64(o.WithGreaterVersion)/float64(o.PeersSampled)*100))
+				log.Errorf(`
+⚠️ A NEW VERSION OF KUBO DETECTED
+
+This Kubo node is running an outdated version (%s).
+%s of the sampled Kubo peers are running a higher version.
+Visit https://github.com/ipfs/kubo/releases or https://dist.ipfs.tech/#kubo and update to version %s or later.`,
+					o.RunningVersion, newerPercent, o.GreatestVersion)
+			}
+			select {
+			case <-ctx.Done():
+				return
+			case <-nd.Process.Closing():
+				return
+			case <-ticker.C:
+				continue
+			}
+		}
+	}()
+}

config/config.go

@@ -37,6 +37,7 @@ type Config struct {
 	Plugins      Plugins
 	Pinning      Pinning
 	Import       Import
+	Version      Version
 
 	Internal Internal // experimental/unstable options
 }

config/version.go

@@ -0,0 +1,14 @@
+package config
+
+const DefaultSwarmCheckPercentThreshold = 5
+
+// Version allows controling things like custom user agent and update checks.
+type Version struct {
+	// Optional suffix to the AgentVersion presented by `ipfs id` and exposed
+	// via libp2p identify protocol.
+	AgentSuffix *OptionalString `json:",omitempty"`
+
+	// Detect when to warn about new version when observed via libp2p identify
+	SwarmCheckEnabled          Flag             `json:",omitempty"`
+	SwarmCheckPercentThreshold *OptionalInteger `json:",omitempty"`
+}

core/commands/commands_test.go

@@ -199,6 +199,7 @@ func TestCommands(t *testing.T) {
 		"/swarm/resources",
 		"/update",
 		"/version",
+		"/version/check",
 		"/version/deps",
 	}
 

core/commands/version.go

@@ -5,10 +5,17 @@ import (
 	"fmt"
 	"io"
 	"runtime/debug"
+	"strings"
 
-	version "github.com/ipfs/kubo"
-
+	versioncmp "github.com/hashicorp/go-version"
 	cmds "github.com/ipfs/go-ipfs-cmds"
+	version "github.com/ipfs/kubo"
+	"github.com/ipfs/kubo/config"
+	"github.com/ipfs/kubo/core"
+	"github.com/ipfs/kubo/core/commands/cmdenv"
+	"github.com/libp2p/go-libp2p-kad-dht/fullrt"
+	peer "github.com/libp2p/go-libp2p/core/peer"
+	pstore "github.com/libp2p/go-libp2p/core/peerstore"
 )
 
 const (
@@ -16,6 +23,7 @@ const (
 	versionCommitOptionName         = "commit"
 	versionRepoOptionName           = "repo"
 	versionAllOptionName            = "all"
+	versionCheckThresholdOptionName = "min-percent"
 )
 
 var VersionCmd = &cmds.Command{
@@ -25,6 +33,7 @@ var VersionCmd = &cmds.Command{
 	},
 	Subcommands: map[string]*cmds.Command{
 		"deps":  depsVersionCommand,
+		"check": checkVersionCommand,
 	},
 
 	Options: []cmds.Option{
@@ -130,3 +139,161 @@ Print out all dependencies and their versions.`,
 		}),
 	},
 }
+
+const DefaultMinimalVersionFraction = 0.05 // 5%
+
+type VersionCheckOutput struct {
+	UpdateAvailable    bool
+	RunningVersion     string
+	GreatestVersion    string
+	PeersSampled       int
+	WithGreaterVersion int
+}
+
+var checkVersionCommand = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Checks Kubo version against connected peers.",
+		ShortDescription: `
+This command uses the libp2p identify protocol to check the 'AgentVersion'
+of connected peers and see if the Kubo version we're running is outdated.
+
+Peers with an AgentVersion that doesn't start with 'kubo/' are ignored.
+'UpdateAvailable' is set to true only if the 'min-fraction' criteria are met.
+
+The 'ipfs daemon' does the same check regularly and logs when a new version
+is available. You can stop these regular checks by setting
+Version.SwarmCheckEnabled:false in the config.
+`,
+	},
+	Options: []cmds.Option{
+		cmds.IntOption(versionCheckThresholdOptionName, "t", "Percentage (1-100) of sampled peers with the new Kubo version needed to trigger an update warning.").WithDefault(config.DefaultSwarmCheckPercentThreshold),
+	},
+	Type: VersionCheckOutput{},
+
+	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
+		nd, err := cmdenv.GetNode(env)
+		if err != nil {
+			return err
+		}
+
+		if !nd.IsOnline {
+			return ErrNotOnline
+		}
+
+		minPercent, _ := req.Options[versionCheckThresholdOptionName].(int64)
+		output, err := DetectNewKuboVersion(nd, minPercent)
+		if err != nil {
+			return err
+		}
+
+		if err := cmds.EmitOnce(res, output); err != nil {
+			return err
+		}
+		return nil
+	},
+}
+
+// DetectNewKuboVersion observers kubo version reported by other peers via
+// libp2p identify protocol and notifies when threshold fraction of seen swarm
+// is running updated Kubo. It is used by RPC and CLI at 'ipfs version check'
+// and also periodically when 'ipfs daemon' is running.
+func DetectNewKuboVersion(nd *core.IpfsNode, minPercent int64) (VersionCheckOutput, error) {
+	ourVersion, err := versioncmp.NewVersion(version.CurrentVersionNumber)
+	if err != nil {
+		return VersionCheckOutput{}, fmt.Errorf("could not parse our own version %q: %w",
+			version.CurrentVersionNumber, err)
+	}
+	// MAJOR.MINOR.PATCH without any suffix
+	ourVersion = ourVersion.Core()
+
+	greatestVersionSeen := ourVersion
+	totalPeersSampled := 1 // Us (and to avoid division-by-zero edge case)
+	withGreaterVersion := 0
+
+	recordPeerVersion := func(agentVersion string) {
+		// We process the version as is it assembled in GetUserAgentVersion
+		segments := strings.Split(agentVersion, "/")
+		if len(segments) < 2 {
+			return
+		}
+		if segments[0] != "kubo" {
+			return
+		}
+		versionNumber := segments[1] // As in our CurrentVersionNumber
+
+		peerVersion, err := versioncmp.NewVersion(versionNumber)
+		if err != nil {
+			// Do not error on invalid remote versions, just ignore
+			return
+		}
+
+		// Ignore prerelases and development releases (-dev, -rcX)
+		if peerVersion.Metadata() != "" || peerVersion.Prerelease() != "" {
+			return
+		}
+
+		// MAJOR.MINOR.PATCH without any suffix
+		peerVersion = peerVersion.Core()
+
+		// Valid peer version number
+		totalPeersSampled += 1
+		if ourVersion.LessThan(peerVersion) {
+			withGreaterVersion += 1
+		}
+		if peerVersion.GreaterThan(greatestVersionSeen) {
+			greatestVersionSeen = peerVersion
+		}
+	}
+
+	processPeerstoreEntry := func(id peer.ID) {
+		if v, err := nd.Peerstore.Get(id, "AgentVersion"); err == nil {
+			recordPeerVersion(v.(string))
+		} else if errors.Is(err, pstore.ErrNotFound) { // ignore noop
+		} else { // a bug, usually.
+			log.Errorw("failed to get agent version from peerstore", "error", err)
+		}
+	}
+
+	// Amino DHT client keeps information about previously seen peers
+	if nd.DHTClient != nd.DHT && nd.DHTClient != nil {
+		client, ok := nd.DHTClient.(*fullrt.FullRT)
+		if !ok {
+			return VersionCheckOutput{}, errors.New("could not perform version check due to missing or incompatible DHT configuration")
+		}
+		for _, p := range client.Stat() {
+			processPeerstoreEntry(p)
+		}
+	} else if nd.DHT != nil && nd.DHT.WAN != nil {
+		for _, pi := range nd.DHT.WAN.RoutingTable().GetPeerInfos() {
+			processPeerstoreEntry(pi.Id)
+		}
+	} else if nd.DHT != nil && nd.DHT.LAN != nil {
+		for _, pi := range nd.DHT.LAN.RoutingTable().GetPeerInfos() {
+			processPeerstoreEntry(pi.Id)
+		}
+	} else {
+		return VersionCheckOutput{}, errors.New("could not perform version check due to missing or incompatible DHT configuration")
+	}
+
+	if minPercent < 1 || minPercent > 100 {
+		if minPercent == 0 {
+			minPercent = config.DefaultSwarmCheckPercentThreshold
+		} else {
+			return VersionCheckOutput{}, errors.New("Version.SwarmCheckPercentThreshold must be between 1 and 100")
+		}
+	}
+
+	minFraction := float64(minPercent) / 100.0
+
+	// UpdateAvailable flag is set only if minFraction was reached
+	greaterFraction := float64(withGreaterVersion) / float64(totalPeersSampled)
+
+	// Gathered metric are returned every time
+	return VersionCheckOutput{
+		UpdateAvailable:    (greaterFraction >= minFraction),
+		RunningVersion:     ourVersion.String(),
+		GreatestVersion:    greatestVersionSeen.String(),
+		PeersSampled:       totalPeersSampled,
+		WithGreaterVersion: withGreaterVersion,
+	}, nil
+}

docs/changelogs/v0.30.md

@@ -6,6 +6,8 @@
 
 - [Overview](#overview)
 - [🔦 Highlights](#-highlights)
+  - [Automated `ipfs version check`](#automated-ipfs-version-check)
+  - [Version Suffix Configuration](#version-suffix-configuration)
 - [📝 Changelog](#-changelog)
 - [👨‍👩‍👧‍👦 Contributors](#-contributors)
 
@@ -13,6 +15,21 @@
 
 ### 🔦 Highlights
 
+#### Automated `ipfs version check`
+
+Kubo now performs privacy-preserving version checks using the [libp2p identify protocol](https://github.com/libp2p/specs/blob/master/identify/README.md) on peers detected by the Amino DHT client.
+If more than 5% of Kubo peers seen by your node are running a newer version, you will receive a log message notification.
+
+- For manual checks, refer to `ipfs version check --help` for details.
+- To disable automated checks, set [`Version.SwarmCheckEnabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#versionswarmcheckenabled) to `false`.
+
+#### Version Suffix Configuration
+
+Defining the optional agent version suffix is now simpler. The [`Version.AgentSuffix`](https://github.com/ipfs/kubo/blob/master/docs/config.md#agentsuffix) value from the Kubo config takes precedence over any value provided via `ipfs daemon --agent-version-suffix` (which is still supported).
+
+> [!NOTE]
+> Setting a custom version suffix helps with ecosystem analysis, such as Amino DHT reports published at https://stats.ipfs.network
+
 ### 📝 Changelog
 
 ### 👨‍👩‍👧‍👦 Contributors

docs/config.md

@@ -180,6 +180,10 @@ config file at runtime.
     - [`Import.UnixFSRawLeaves`](#importunixfsrawleaves)
     - [`Import.UnixFSChunker`](#importunixfschunker)
     - [`Import.HashFunction`](#importhashfunction)
+  - [`Version`](#version)
+    - [`Version.AgentSuffix`](#versionagentsuffix)
+    - [`Version.SwarmCheckEnabled`](#versionswarmcheckenabled)
+    - [`Version.SwarmCheckPercentThreshold`](#versionswarmcheckpercentthreshold)
 
 ## Profiles
 
@@ -2435,3 +2439,39 @@ The default hash function. Commands affected: `ipfs add`, `ipfs block put`, `ipf
 Default: `sha2-256`
 
 Type: `optionalString`
+
+## `Version`
+
+Options to configure agent version announced to the swarm, and leveraging
+other peers version for detecting when there is time to update.
+
+### `Version.AgentSuffix`
+
+Optional suffix to the AgentVersion presented by `ipfs id` and exposed via [libp2p identify protocol](https://github.com/libp2p/specs/blob/master/identify/README.md#agentversion).
+
+The value from config takes precedence over value passed via `ipfs daemon --agent-version-suffix`.
+
+> [!NOTE]
+> Setting a custom version suffix helps with ecosystem analysis, such as Amino DHT reports published at https://stats.ipfs.network
+
+Default: `""` (no suffix, or value from `ipfs daemon --agent-version-suffix=`)
+
+Type: `optionalString`
+
+### `Version.SwarmCheckEnabled`
+
+Observe the AgentVersion of swarm peers and log warning when
+`SwarmCheckPercentThreshold` of peers runs version higher than this node.
+
+Default: `true`
+
+Type: `flag`
+
+### `Version.SwarmCheckPercentThreshold`
+
+Control the percentage of `kubo/` peers running new version required to
+trigger update warning.
+
+Default: `5`
+
+Type: `optionalInteger` (1-100)

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/fsnotify/fsnotify v1.6.0
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/go-multierror v1.1.1
+	github.com/hashicorp/go-version v1.6.0
 	github.com/ipfs-shipyard/nopfs v0.0.12
 	github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c
 	github.com/ipfs/boxo v0.21.0

go.sum

@@ -310,6 +310,8 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
+github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=

test/sharness/t0026-id.sh

@@ -65,5 +65,16 @@ iptb stop
 
 test_kill_ipfs_daemon
 
+# Version.AgentSuffix overrides --agent-version-suffix (local, offline)
+test_expect_success "setting Version.AgentSuffix in config" '
+  ipfs config Version.AgentSuffix json-config-suffix
+'
+test_launch_ipfs_daemon --agent-version-suffix=ignored-cli-suffix
+test_expect_success "checking AgentVersion with suffix set via JSON config" '
+  test_id_compute_agent json-config-suffix > expected-agent-version &&
+  ipfs id -f "<aver>\n" > actual-agent-version &&
+  test_cmp expected-agent-version actual-agent-version
+'
+test_kill_ipfs_daemon
 
 test_done