mirror of
https://github.com/ipfs/kubo.git
synced 2025-05-17 06:57:40 +08:00
feat: pebbleds profile and plugin (#10530)
* include pebble as built-in plugin Pebble provides a high-performance alternative to leveldb as the datastore, and will serve as a replacement for badger1. There are a number of tuning parameters available for tuning pebble's performance to your specific needs. Default values are used for any that are not configured or are set to the parameter's zero-value. Requires https://github.com/ipfs/go-ds-pebble/pull/39 Closes #10347 * docs: remove mention of ipfs-ds-convert. Rationale: https://github.com/ipfs/ipfs-ds-convert/issues/50 * docs: pebbleds profile * test: meaningful t0025-datastores.sh * Update config/init.go * Update docs/config.md * Do not hard-code zero values into pebble config
This commit is contained in:
Andrew Gillis
@ -138,6 +138,17 @@ func DefaultDatastoreConfig() Datastore {
|
||||
}
|
||||
}
|
||||
|
||||
func pebbleSpec() map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"type": "measure",
|
||||
"prefix": "pebble.datastore",
|
||||
"child": map[string]interface{}{
|
||||
"type": "pebbleds",
|
||||
"path": "pebbleds",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func badgerSpec() map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"type": "measure",
|
||||
|
||||
@ -135,7 +135,11 @@ You should use this datastore if:
|
||||
* You want to minimize memory usage.
|
||||
* You are ok with the default speed of data import, or prefer to use --nocopy.
|
||||
|
||||
This profile may only be applied when first initializing the node.
|
||||
See configuration documentation at:
|
||||
https://github.com/ipfs/kubo/blob/master/docs/datastores.md#flatfs
|
||||
|
||||
NOTE: This profile may only be applied when first initializing node at IPFS_PATH
|
||||
via 'ipfs init --profile flatfs'
|
||||
`,
|
||||
|
||||
InitOnly: true,
|
||||
@ -144,6 +148,32 @@ This profile may only be applied when first initializing the node.
|
||||
return nil
|
||||
},
|
||||
},
|
||||
"pebbleds": {
|
||||
Description: `Configures the node to use the pebble high-performance datastore.
|
||||
|
||||
Pebble is a LevelDB/RocksDB inspired key-value store focused on performance
|
||||
and internal usage by CockroachDB.
|
||||
You should use this datastore if:
|
||||
|
||||
- You need a datastore that is focused on performance.
|
||||
- You need reliability by default, but may choose to disable WAL for maximum performance when reliability is not critical.
|
||||
- This datastore is good for multi-terabyte data sets.
|
||||
- May benefit from tuning depending on read/write patterns and throughput.
|
||||
- Performance is helped significantly by running on a system with plenty of memory.
|
||||
|
||||
See configuration documentation at:
|
||||
https://github.com/ipfs/kubo/blob/master/docs/datastores.md#pebbleds
|
||||
|
||||
NOTE: This profile may only be applied when first initializing node at IPFS_PATH
|
||||
via 'ipfs init --profile pebbleds'
|
||||
`,
|
||||
|
||||
InitOnly: true,
|
||||
Transform: func(c *Config) error {
|
||||
c.Datastore.Spec = pebbleSpec()
|
||||
return nil
|
||||
},
|
||||
},
|
||||
"badgerds": {
|
||||
Description: `Configures the node to use the legacy badgerv1 datastore.
|
||||
|
||||
@ -160,7 +190,12 @@ Other caveats:
|
||||
* Good for medium-size datastores, but may run into performance issues
|
||||
if your dataset is bigger than a terabyte.
|
||||
|
||||
This profile may only be applied when first initializing the node.`,
|
||||
See configuration documentation at:
|
||||
https://github.com/ipfs/kubo/blob/master/docs/datastores.md#badgerds
|
||||
|
||||
NOTE: This profile may only be applied when first initializing node at IPFS_PATH
|
||||
via 'ipfs init --profile badgerds'
|
||||
`,
|
||||
|
||||
InitOnly: true,
|
||||
Transform: func(c *Config) error {
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
|
||||
- [Overview](#overview)
|
||||
- [🔦 Highlights](#-highlights)
|
||||
- [Experimental Pebble Datastore](#experimental-pebble-datastore)
|
||||
- [New metrics](#new-metrics)
|
||||
- [`lowpower` profile no longer breaks DHT announcements](#lowpower-profile-no-longer-breaks-dht-announcements)
|
||||
- [📝 Changelog](#-changelog)
|
||||
@ -15,6 +16,15 @@
|
||||
|
||||
### 🔦 Highlights
|
||||
|
||||
#### Experimental Pebble Datastore
|
||||
|
||||
[Pebble](https://github.com/ipfs/kubo/blob/master/docs/config.md#pebbleds-profile) provides a high-performance alternative to leveldb as the datastore, and provides a modern replacement for [legacy badgerv1](https://github.com/ipfs/kubo/blob/master/docs/config.md#badgerds-profile).
|
||||
|
||||
A fresh Kubo node can be initialized with [`pebbleds` profile](https://github.com/ipfs/kubo/blob/master/docs/config.md#pebbleds-profile) via `ipfs init --profile pebbleds`.
|
||||
|
||||
There are a number of parameters available for tuning pebble's performance to your specific needs. Default values are used for any parameters that are not configured or are set to their zero-value.
|
||||
For a description of the available tuning parameters, see [kubo/docs/datastores.md#pebbleds](https://github.com/ipfs/kubo/blob/master/docs/datastores.md#pebbleds).
|
||||
|
||||
#### New metrics
|
||||
|
||||
- Added 3 new go metrics: `go_gc_gogc_percent`, `go_gc_gomemlimit_bytes` and `go_sched_gomaxprocs_threads` as those are [recommended by the Go team](https://github.com/prometheus/client_golang/pull/1559)
|
||||
|
||||
@ -181,6 +181,7 @@ config file at runtime.
|
||||
- [`local-discovery` profile](#local-discovery-profile)
|
||||
- [`default-networking` profile](#default-networking-profile)
|
||||
- [`flatfs` profile](#flatfs-profile)
|
||||
- [`pebbleds` profile](#pebbleds-profile)
|
||||
- [`badgerds` profile](#badgerds-profile)
|
||||
- [`lowpower` profile](#lowpower-profile)
|
||||
- [`announce-off` profile](#announce-off-profile)
|
||||
@ -524,13 +525,8 @@ Spec defines the structure of the ipfs datastore. It is a composable structure,
|
||||
where each datastore is represented by a json object. Datastores can wrap other
|
||||
datastores to provide extra functionality (eg metrics, logging, or caching).
|
||||
|
||||
This can be changed manually, however, if you make any changes that require a
|
||||
different on-disk structure, you will need to run the [ipfs-ds-convert
|
||||
tool](https://github.com/ipfs/ipfs-ds-convert) to migrate data into the new
|
||||
structures.
|
||||
|
||||
For more information on possible values for this configuration option, see
|
||||
[docs/datastores.md](datastores.md)
|
||||
> [!NOTE]
|
||||
> For more information on possible values for this configuration option, see [`kubo/docs/datastores.md`](datastores.md)
|
||||
|
||||
Default:
|
||||
```
|
||||
@ -2403,9 +2399,9 @@ Inverse profile of the test profile.
|
||||
|
||||
### `flatfs` profile
|
||||
|
||||
Configures the node to use the flatfs datastore. Flatfs is the default datastore.
|
||||
Configures the node to use the flatfs datastore.
|
||||
Flatfs is the default, most battle-tested and reliable datastore.
|
||||
|
||||
This is the most battle-tested and reliable datastore.
|
||||
You should use this datastore if:
|
||||
|
||||
- You need a very simple and very reliable datastore, and you trust your
|
||||
@ -2416,7 +2412,30 @@ You should use this datastore if:
|
||||
- You want to minimize memory usage.
|
||||
- You are ok with the default speed of data import, or prefer to use `--nocopy`.
|
||||
|
||||
This profile may only be applied when first initializing the node.
|
||||
> [!WARNING]
|
||||
> This profile may only be applied when first initializing the node via `ipfs init --profile flatfs`
|
||||
|
||||
> [!NOTE]
|
||||
> See caveats and configuration options at [`datastores.md#flatfs`](datastores.md#flatfs)
|
||||
|
||||
### `pebbleds` profile
|
||||
|
||||
Configures the node to use the pebble high-performance datastore.
|
||||
|
||||
Pebble is a LevelDB/RocksDB inspired key-value store focused on performance and internal usage by CockroachDB.
|
||||
You should use this datastore if:
|
||||
|
||||
- You need a datastore that is focused on performance.
|
||||
- You need reliability by default, but may choose to disable WAL for maximum performance when reliability is not critical.
|
||||
- This datastore is good for multi-terrabyte data sets.
|
||||
- May benefit from tuning depending on read/write patterns and throughput.
|
||||
- Performance is helped significantly by running on a system with plenty of memory.
|
||||
|
||||
> [!WARNING]
|
||||
> This profile may only be applied when first initializing the node via `ipfs init --profile pebbleds`
|
||||
|
||||
> [!NOTE]
|
||||
> See other caveats and configuration options at [`datastores.md#pebbleds`](datastores.md#pebbleds)
|
||||
|
||||
### `badgerds` profile
|
||||
|
||||
@ -2437,7 +2456,11 @@ Also, be aware that:
|
||||
- Good for medium-size datastores, but may run into performance issues if your dataset is bigger than a terabyte.
|
||||
- The current implementation is based on old badger 1.x which is no longer supported by the upstream team.
|
||||
|
||||
This profile may only be applied when first initializing the node.
|
||||
> [!WARNING]
|
||||
> This profile may only be applied when first initializing the node via `ipfs init --profile badgerds`
|
||||
|
||||
> [!NOTE]
|
||||
> See other caveats and configuration options at [`datastores.md#pebbleds`](datastores.md#pebbleds)
|
||||
|
||||
### `lowpower` profile
|
||||
|
||||
|
||||
@ -3,6 +3,13 @@
|
||||
This document describes the different possible values for the `Datastore.Spec`
|
||||
field in the ipfs configuration file.
|
||||
|
||||
- [flatfs](#flatfs)
|
||||
- [levelds](#levelds)
|
||||
- [pebbleds](#pebbleds)
|
||||
- [badgerds](#badgerds)
|
||||
- [mount](#mount)
|
||||
- [measure](#measure)
|
||||
|
||||
## flatfs
|
||||
|
||||
Stores each key value pair as a file on the filesystem.
|
||||
@ -35,6 +42,35 @@ Uses a leveldb database to store key value pairs.
|
||||
}
|
||||
```
|
||||
|
||||
## pebbleds
|
||||
|
||||
Uses [pebble](https://github.com/cockroachdb/pebble) as a key value store.
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "pebbleds",
|
||||
"path": "<location of pebble inside repo>",
|
||||
}
|
||||
```
|
||||
|
||||
The following options are availble for tuning pebble.
|
||||
If they are not configured (or assigned their zero-valued), then default values are used.
|
||||
|
||||
* `bytesPerSync`: int, Sync sstables periodically in order to smooth out writes to disk. (default: 512KB)
|
||||
* `bisableWAL`: true|false, Disable the write-ahead log (WAL) at expense of prohibiting crash recovery. (default: false)
|
||||
* `cacheSize`: Size of pebble's shared block cache. (default: 8MB)
|
||||
* `l0CompactionThreshold`: int, Count of L0 files necessary to trigger an L0 compaction.
|
||||
* `l0StopWritesThreshold`: int, Limit on L0 read-amplification, computed as the number of L0 sublevels.
|
||||
* `lBaseMaxBytes`: int, Maximum number of bytes for LBase. The base level is the level which L0 is compacted into.
|
||||
* `maxConcurrentCompactions`: int, Maximum number of concurrent compactions. (default: 1)
|
||||
* `memTableSize`: int, Size of a MemTable in steady state. The actual MemTable size starts at min(256KB, MemTableSize) and doubles for each subsequent MemTable up to MemTableSize (default: 4MB)
|
||||
* `memTableStopWritesThreshold`: int, Limit on the number of queued of MemTables. (default: 2)
|
||||
* `walBytesPerSync`: int: Sets the number of bytes to write to a WAL before calling Sync on it in the background. (default: 0, no background syncing)
|
||||
* `walMinSyncSeconds`: int: Sets the minimum duration between syncs of the WAL. (default: 0)
|
||||
|
||||
> [!TIP]
|
||||
> Start using pebble with only default values and configure tuning items are needed for your needs. For a more complete description of these values, see: `https://pkg.go.dev/github.com/cockroachdb/pebble@vA.B.C#Options` (where `A.B.C` is pebble version from Kubo's `go.mod`).
|
||||
|
||||
## badgerds
|
||||
|
||||
Uses [badger](https://github.com/dgraph-io/badger) as a key value store.
|
||||
|
||||
@ -16,6 +16,7 @@ require (
|
||||
require (
|
||||
bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc // indirect
|
||||
github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
|
||||
github.com/DataDog/zstd v1.4.5 // indirect
|
||||
github.com/Jorropo/jsync v1.0.1 // indirect
|
||||
github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 // indirect
|
||||
github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect
|
||||
@ -25,6 +26,12 @@ require (
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
|
||||
github.com/ceramicnetwork/go-dag-jose v0.1.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/cockroachdb/errors v1.11.3 // indirect
|
||||
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
|
||||
github.com/cockroachdb/pebble v1.1.2 // indirect
|
||||
github.com/cockroachdb/redact v1.1.5 // indirect
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
|
||||
github.com/containerd/cgroups v1.1.0 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
|
||||
github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 // indirect
|
||||
@ -43,6 +50,7 @@ require (
|
||||
github.com/francoispqt/gojay v1.2.13 // indirect
|
||||
github.com/fsnotify/fsnotify v1.7.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.4 // indirect
|
||||
github.com/getsentry/sentry-go v0.27.0 // indirect
|
||||
github.com/go-logr/logr v1.4.2 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
|
||||
@ -74,6 +82,7 @@ require (
|
||||
github.com/ipfs/go-ds-flatfs v0.5.1 // indirect
|
||||
github.com/ipfs/go-ds-leveldb v0.5.0 // indirect
|
||||
github.com/ipfs/go-ds-measure v0.2.0 // indirect
|
||||
github.com/ipfs/go-ds-pebble v0.4.0 // indirect
|
||||
github.com/ipfs/go-fs-lock v0.0.7 // indirect
|
||||
github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect
|
||||
github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
|
||||
@ -103,6 +112,8 @@ require (
|
||||
github.com/klauspost/compress v1.17.9 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
|
||||
github.com/koron/go-ssdp v0.0.4 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
|
||||
github.com/libp2p/go-cidranger v1.1.0 // indirect
|
||||
github.com/libp2p/go-doh-resolver v0.4.0 // indirect
|
||||
@ -172,6 +183,7 @@ require (
|
||||
github.com/quic-go/quic-go v0.45.2 // indirect
|
||||
github.com/quic-go/webtransport-go v0.8.0 // indirect
|
||||
github.com/raulk/go-watchdog v1.3.0 // indirect
|
||||
github.com/rogpeppe/go-internal v1.12.0 // indirect
|
||||
github.com/samber/lo v1.46.0 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/stretchr/testify v1.9.0 // indirect
|
||||
|
||||
@ -29,6 +29,8 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIo
|
||||
github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
|
||||
github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
|
||||
github.com/Jorropo/jsync v1.0.1 h1:6HgRolFZnsdfzRUj+ImB9og1JYOxQoReSywkHOGSaUU=
|
||||
github.com/Jorropo/jsync v1.0.1/go.mod h1:jCOZj3vrBCri3bSU3ErUYvevKlnbssrXeCivybS5ABQ=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
@ -74,6 +76,20 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
|
||||
github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
|
||||
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
|
||||
github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
|
||||
github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8=
|
||||
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4=
|
||||
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M=
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
|
||||
github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA=
|
||||
github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU=
|
||||
github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
|
||||
github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
|
||||
github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE=
|
||||
github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM=
|
||||
github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw=
|
||||
@ -143,10 +159,14 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
|
||||
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
|
||||
github.com/gabriel-vasile/mimetype v1.4.4 h1:QjV6pZ7/XZ7ryI2KuyeEDE8wnh7fHP9YnQy+R0LnH8I=
|
||||
github.com/gabriel-vasile/mimetype v1.4.4/go.mod h1:JwLei5XPtWdGiMFB5Pjle1oEeoSeEuJfJE+TtfvdB/s=
|
||||
github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
|
||||
github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
|
||||
github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
|
||||
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
|
||||
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
|
||||
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
@ -301,6 +321,8 @@ github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUN
|
||||
github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q=
|
||||
github.com/ipfs/go-ds-measure v0.2.0 h1:sG4goQe0KDTccHMyT45CY1XyUbxe5VwTKpg2LjApYyQ=
|
||||
github.com/ipfs/go-ds-measure v0.2.0/go.mod h1:SEUD/rE2PwRa4IQEC5FuNAmjJCyYObZr9UvVh8V3JxE=
|
||||
github.com/ipfs/go-ds-pebble v0.4.0 h1:88lgFAs2ck8jCQ8lMYRBtksEg18r9BlvTxIMnNJkZaQ=
|
||||
github.com/ipfs/go-ds-pebble v0.4.0/go.mod h1:ZyYU+weIni+4NG/Yjva+cPkU3ghlsU1HA2R/VLHJ9sM=
|
||||
github.com/ipfs/go-fs-lock v0.0.7 h1:6BR3dajORFrFTkb5EpCUFIAypsoxpGpDSVUdFwzgL9U=
|
||||
github.com/ipfs/go-fs-lock v0.0.7/go.mod h1:Js8ka+FNYmgQRLrRXzU3CB/+Csr1BwrRilEcvYrHhhc=
|
||||
github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ=
|
||||
@ -585,6 +607,8 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhM
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk=
|
||||
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=
|
||||
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
|
||||
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
|
||||
github.com/pion/datachannel v1.5.8 h1:ph1P1NsGkazkjrvyMfhRBUAWMxugJjq2HfQifaOoSNo=
|
||||
github.com/pion/datachannel v1.5.8/go.mod h1:PgmdpoaNBLX9HNzNClmdki4DYW5JtI7Yibu8QzbL3tI=
|
||||
github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s=
|
||||
@ -627,6 +651,7 @@ github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc=
|
||||
github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
|
||||
github.com/pion/webrtc/v3 v3.3.0 h1:Rf4u6n6U5t5sUxhYPQk/samzU/oDv7jk6BA5hyO2F9I=
|
||||
github.com/pion/webrtc/v3 v3.3.0/go.mod h1:hVmrDJvwhEertRWObeb1xzulzHGeVUoPlWvxdGzcfU0=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
@ -658,6 +683,7 @@ github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtB
|
||||
github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
|
||||
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
|
||||
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
|
||||
|
||||
14
go.mod
14
go.mod
@ -10,6 +10,7 @@ require (
|
||||
github.com/cenkalti/backoff/v4 v4.3.0
|
||||
github.com/ceramicnetwork/go-dag-jose v0.1.0
|
||||
github.com/cheggaaa/pb v1.0.29
|
||||
github.com/cockroachdb/pebble v1.1.2
|
||||
github.com/coreos/go-systemd/v22 v22.5.0
|
||||
github.com/dustin/go-humanize v1.0.1
|
||||
github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302
|
||||
@ -30,6 +31,7 @@ require (
|
||||
github.com/ipfs/go-ds-flatfs v0.5.1
|
||||
github.com/ipfs/go-ds-leveldb v0.5.0
|
||||
github.com/ipfs/go-ds-measure v0.2.0
|
||||
github.com/ipfs/go-ds-pebble v0.4.0
|
||||
github.com/ipfs/go-fs-lock v0.0.7
|
||||
github.com/ipfs/go-ipfs-cmds v0.13.0
|
||||
github.com/ipfs/go-ipld-cbor v0.2.0
|
||||
@ -95,12 +97,18 @@ require (
|
||||
|
||||
require (
|
||||
github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
|
||||
github.com/DataDog/zstd v1.4.5 // indirect
|
||||
github.com/Jorropo/jsync v1.0.1 // indirect
|
||||
github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 // indirect
|
||||
github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash v1.1.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/cockroachdb/errors v1.11.3 // indirect
|
||||
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
|
||||
github.com/cockroachdb/redact v1.1.5 // indirect
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
|
||||
github.com/containerd/cgroups v1.1.0 // indirect
|
||||
github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 // indirect
|
||||
github.com/cskr/pubsub v1.0.2 // indirect
|
||||
@ -115,6 +123,7 @@ require (
|
||||
github.com/flynn/noise v1.1.0 // indirect
|
||||
github.com/francoispqt/gojay v1.2.13 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.4 // indirect
|
||||
github.com/getsentry/sentry-go v0.27.0 // indirect
|
||||
github.com/go-kit/log v0.2.1 // indirect
|
||||
github.com/go-logfmt/logfmt v0.5.1 // indirect
|
||||
github.com/go-logr/logr v1.4.2 // indirect
|
||||
@ -151,6 +160,8 @@ require (
|
||||
github.com/klauspost/compress v1.17.9 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
|
||||
github.com/koron/go-ssdp v0.0.4 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
|
||||
github.com/libp2p/go-cidranger v1.1.0 // indirect
|
||||
github.com/libp2p/go-flow-metrics v0.1.0 // indirect
|
||||
@ -164,7 +175,7 @@ require (
|
||||
github.com/libp2p/go-yamux/v4 v4.0.1 // indirect
|
||||
github.com/libp2p/zeroconf/v2 v2.2.0 // indirect
|
||||
github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
|
||||
github.com/mattn/go-colorable v0.1.6 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.4 // indirect
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
|
||||
@ -210,6 +221,7 @@ require (
|
||||
github.com/quic-go/quic-go v0.45.2 // indirect
|
||||
github.com/quic-go/webtransport-go v0.8.0 // indirect
|
||||
github.com/raulk/go-watchdog v1.3.0 // indirect
|
||||
github.com/rogpeppe/go-internal v1.12.0 // indirect
|
||||
github.com/rs/cors v1.10.1 // indirect
|
||||
github.com/samber/lo v1.46.0 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
|
||||
33
go.sum
33
go.sum
@ -47,6 +47,8 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIo
|
||||
github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
|
||||
github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
|
||||
github.com/Jorropo/jsync v1.0.1 h1:6HgRolFZnsdfzRUj+ImB9og1JYOxQoReSywkHOGSaUU=
|
||||
github.com/Jorropo/jsync v1.0.1/go.mod h1:jCOZj3vrBCri3bSU3ErUYvevKlnbssrXeCivybS5ABQ=
|
||||
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
|
||||
@ -104,6 +106,20 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
|
||||
github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
|
||||
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
|
||||
github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
|
||||
github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8=
|
||||
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4=
|
||||
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M=
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
|
||||
github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA=
|
||||
github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU=
|
||||
github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
|
||||
github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
|
||||
github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE=
|
||||
github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM=
|
||||
github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw=
|
||||
@ -177,10 +193,14 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4
|
||||
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
|
||||
github.com/gabriel-vasile/mimetype v1.4.4 h1:QjV6pZ7/XZ7ryI2KuyeEDE8wnh7fHP9YnQy+R0LnH8I=
|
||||
github.com/gabriel-vasile/mimetype v1.4.4/go.mod h1:JwLei5XPtWdGiMFB5Pjle1oEeoSeEuJfJE+TtfvdB/s=
|
||||
github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
|
||||
github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
|
||||
github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
|
||||
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
|
||||
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
|
||||
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
@ -365,6 +385,8 @@ github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUN
|
||||
github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q=
|
||||
github.com/ipfs/go-ds-measure v0.2.0 h1:sG4goQe0KDTccHMyT45CY1XyUbxe5VwTKpg2LjApYyQ=
|
||||
github.com/ipfs/go-ds-measure v0.2.0/go.mod h1:SEUD/rE2PwRa4IQEC5FuNAmjJCyYObZr9UvVh8V3JxE=
|
||||
github.com/ipfs/go-ds-pebble v0.4.0 h1:88lgFAs2ck8jCQ8lMYRBtksEg18r9BlvTxIMnNJkZaQ=
|
||||
github.com/ipfs/go-ds-pebble v0.4.0/go.mod h1:ZyYU+weIni+4NG/Yjva+cPkU3ghlsU1HA2R/VLHJ9sM=
|
||||
github.com/ipfs/go-fs-lock v0.0.7 h1:6BR3dajORFrFTkb5EpCUFIAypsoxpGpDSVUdFwzgL9U=
|
||||
github.com/ipfs/go-fs-lock v0.0.7/go.mod h1:Js8ka+FNYmgQRLrRXzU3CB/+Csr1BwrRilEcvYrHhhc=
|
||||
github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ=
|
||||
@ -560,15 +582,15 @@ github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
|
||||
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||
github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE=
|
||||
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
|
||||
@ -685,6 +707,8 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhM
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk=
|
||||
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=
|
||||
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
|
||||
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
|
||||
github.com/pion/datachannel v1.5.8 h1:ph1P1NsGkazkjrvyMfhRBUAWMxugJjq2HfQifaOoSNo=
|
||||
github.com/pion/datachannel v1.5.8/go.mod h1:PgmdpoaNBLX9HNzNClmdki4DYW5JtI7Yibu8QzbL3tI=
|
||||
github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s=
|
||||
@ -727,6 +751,7 @@ github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc=
|
||||
github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
|
||||
github.com/pion/webrtc/v3 v3.3.0 h1:Rf4u6n6U5t5sUxhYPQk/samzU/oDv7jk6BA5hyO2F9I=
|
||||
github.com/pion/webrtc/v3 v3.3.0/go.mod h1:hVmrDJvwhEertRWObeb1xzulzHGeVUoPlWvxdGzcfU0=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
@ -785,6 +810,7 @@ github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtB
|
||||
github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
|
||||
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
|
||||
github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo=
|
||||
@ -1200,6 +1226,7 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
|
||||
@ -8,6 +8,7 @@ import (
|
||||
pluginipldgit "github.com/ipfs/kubo/plugin/plugins/git"
|
||||
pluginlevelds "github.com/ipfs/kubo/plugin/plugins/levelds"
|
||||
pluginnopfs "github.com/ipfs/kubo/plugin/plugins/nopfs"
|
||||
pluginpebbleds "github.com/ipfs/kubo/plugin/plugins/pebbleds"
|
||||
pluginpeerlog "github.com/ipfs/kubo/plugin/plugins/peerlog"
|
||||
)
|
||||
|
||||
@ -21,6 +22,7 @@ func init() {
|
||||
Preload(pluginbadgerds.Plugins...)
|
||||
Preload(pluginflatfs.Plugins...)
|
||||
Preload(pluginlevelds.Plugins...)
|
||||
Preload(pluginpebbleds.Plugins...)
|
||||
Preload(pluginpeerlog.Plugins...)
|
||||
Preload(pluginfxtest.Plugins...)
|
||||
Preload(pluginnopfs.Plugins...)
|
||||
|
||||
@ -9,6 +9,7 @@ iplddagjose github.com/ipfs/kubo/plugin/plugins/dagjose *
|
||||
badgerds github.com/ipfs/kubo/plugin/plugins/badgerds *
|
||||
flatfs github.com/ipfs/kubo/plugin/plugins/flatfs *
|
||||
levelds github.com/ipfs/kubo/plugin/plugins/levelds *
|
||||
pebbleds github.com/ipfs/kubo/plugin/plugins/pebbleds *
|
||||
peerlog github.com/ipfs/kubo/plugin/plugins/peerlog *
|
||||
fxtest github.com/ipfs/kubo/plugin/plugins/fxtest *
|
||||
nopfs github.com/ipfs/kubo/plugin/plugins/nopfs *
|
||||
nopfs github.com/ipfs/kubo/plugin/plugins/nopfs *
|
||||
|
||||
@ -42,7 +42,7 @@ type datastoreConfig struct {
|
||||
syncField bool
|
||||
}
|
||||
|
||||
// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore
|
||||
// BadgerdsDatastoreConfig returns a configuration stub for a flatfs datastore
|
||||
// from the given parameters.
|
||||
func (*flatfsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap {
|
||||
return func(params map[string]interface{}) (fsrepo.DatastoreConfig, error) {
|
||||
|
||||
@ -42,7 +42,7 @@ type datastoreConfig struct {
|
||||
compression ldbopts.Compression
|
||||
}
|
||||
|
||||
// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore
|
||||
// DatastoreConfigParser returns a configuration stub for a badger datastore
|
||||
// from the given parameters.
|
||||
func (*leveldsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap {
|
||||
return func(params map[string]interface{}) (fsrepo.DatastoreConfig, error) {
|
||||
|
||||
246
plugin/plugins/pebbleds/pebbleds.go
Normal file
246
plugin/plugins/pebbleds/pebbleds.go
Normal file
@ -0,0 +1,246 @@
|
||||
package pebbleds
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/cockroachdb/pebble"
|
||||
pebbleds "github.com/ipfs/go-ds-pebble"
|
||||
"github.com/ipfs/kubo/plugin"
|
||||
"github.com/ipfs/kubo/repo"
|
||||
"github.com/ipfs/kubo/repo/fsrepo"
|
||||
)
|
||||
|
||||
// Plugins is exported list of plugins that will be loaded.
|
||||
var Plugins = []plugin.Plugin{
|
||||
&pebbledsPlugin{},
|
||||
}
|
||||
|
||||
type pebbledsPlugin struct{}
|
||||
|
||||
var _ plugin.PluginDatastore = (*pebbledsPlugin)(nil)
|
||||
|
||||
func (*pebbledsPlugin) Name() string {
|
||||
return "ds-pebble"
|
||||
}
|
||||
|
||||
func (*pebbledsPlugin) Version() string {
|
||||
return "0.1.0"
|
||||
}
|
||||
|
||||
func (*pebbledsPlugin) Init(_ *plugin.Environment) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (*pebbledsPlugin) DatastoreTypeName() string {
|
||||
return "pebbleds"
|
||||
}
|
||||
|
||||
type datastoreConfig struct {
|
||||
path string
|
||||
cacheSize int64
|
||||
|
||||
// Documentation of these values: https://pkg.go.dev/github.com/cockroachdb/pebble@v1.1.2#Options
|
||||
pebbleOpts *pebble.Options
|
||||
}
|
||||
|
||||
// PebbleDatastoreConfig returns a configuration stub for a pebble datastore
|
||||
// from the given parameters.
|
||||
func (*pebbledsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap {
|
||||
return func(params map[string]any) (fsrepo.DatastoreConfig, error) {
|
||||
var c datastoreConfig
|
||||
var ok bool
|
||||
|
||||
c.path, ok = params["path"].(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("'path' field is missing or not string")
|
||||
}
|
||||
|
||||
cacheSize, err := getConfigInt("cacheSize", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.cacheSize = int64(cacheSize)
|
||||
|
||||
bytesPerSync, err := getConfigInt("bytesPerSync", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
disableWAL, err := getConfigBool("disableWAL", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l0CompactionThreshold, err := getConfigInt("l0CompactionThreshold", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
l0StopWritesThreshold, err := getConfigInt("l0StopWritesThreshold", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lBaseMaxBytes, err := getConfigInt("lBaseMaxBytes", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
maxConcurrentCompactions, err := getConfigInt("maxConcurrentCompactions", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
memTableSize, err := getConfigInt("memTableSize", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
memTableStopWritesThreshold, err := getConfigInt("memTableStopWritesThreshold", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
walBytesPerSync, err := getConfigInt("walBytesPerSync", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
walMinSyncSec, err := getConfigInt("walMinSyncIntervalSeconds", params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if bytesPerSync != 0 || disableWAL || l0CompactionThreshold != 0 || l0StopWritesThreshold != 0 || lBaseMaxBytes != 0 || maxConcurrentCompactions != 0 || memTableSize != 0 || memTableStopWritesThreshold != 0 || walBytesPerSync != 0 || walMinSyncSec != 0 {
|
||||
c.pebbleOpts = &pebble.Options{
|
||||
BytesPerSync: bytesPerSync,
|
||||
DisableWAL: disableWAL,
|
||||
L0CompactionThreshold: l0CompactionThreshold,
|
||||
L0StopWritesThreshold: l0StopWritesThreshold,
|
||||
LBaseMaxBytes: int64(lBaseMaxBytes),
|
||||
MemTableSize: uint64(memTableSize),
|
||||
MemTableStopWritesThreshold: memTableStopWritesThreshold,
|
||||
WALBytesPerSync: walBytesPerSync,
|
||||
}
|
||||
if maxConcurrentCompactions != 0 {
|
||||
c.pebbleOpts.MaxConcurrentCompactions = func() int { return maxConcurrentCompactions }
|
||||
}
|
||||
if walMinSyncSec != 0 {
|
||||
c.pebbleOpts.WALMinSyncInterval = func() time.Duration { return time.Duration(walMinSyncSec) * time.Second }
|
||||
}
|
||||
}
|
||||
|
||||
return &c, nil
|
||||
}
|
||||
}
|
||||
|
||||
func getConfigBool(name string, params map[string]any) (bool, error) {
|
||||
val, ok := params[name]
|
||||
if ok {
|
||||
bval, ok := val.(bool)
|
||||
if !ok {
|
||||
return false, fmt.Errorf("%q field was not a bool", name)
|
||||
}
|
||||
return bval, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func getConfigInt(name string, params map[string]any) (int, error) {
|
||||
val, ok := params[name]
|
||||
if ok {
|
||||
// TODO: see why val may be an int or a float64.
|
||||
ival, ok := val.(int)
|
||||
if !ok {
|
||||
fval, ok := val.(float64)
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("%q field was not an integer or a float64", name)
|
||||
}
|
||||
return int(fval), nil
|
||||
}
|
||||
return ival, nil
|
||||
}
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (c *datastoreConfig) DiskSpec() fsrepo.DiskSpec {
|
||||
return map[string]interface{}{
|
||||
"type": "pebbleds",
|
||||
"path": c.path,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *datastoreConfig) Create(path string) (repo.Datastore, error) {
|
||||
p := c.path
|
||||
if !filepath.IsAbs(p) {
|
||||
p = filepath.Join(path, p)
|
||||
}
|
||||
|
||||
if err := dirWritable(p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return pebbleds.NewDatastore(p, pebbleds.WithCacheSize(c.cacheSize), pebbleds.WithPebbleOpts(c.pebbleOpts))
|
||||
}
|
||||
|
||||
// dirWritable checks if a directory is writable. If the directory does
|
||||
// not exist it is created with writable permission.
|
||||
func dirWritable(dir string) error {
|
||||
if dir == "" {
|
||||
return errors.New("directory not specified")
|
||||
}
|
||||
var err error
|
||||
dir, err = expandHome(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fi, err := os.Stat(dir)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
// Directory does not exist, so create it.
|
||||
err = os.Mkdir(dir, 0775)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if errors.Is(err, fs.ErrPermission) {
|
||||
err = fs.ErrPermission
|
||||
}
|
||||
return fmt.Errorf("directory not writable: %s: %w", dir, err)
|
||||
}
|
||||
if !fi.IsDir() {
|
||||
return fmt.Errorf("not a directory: %s", dir)
|
||||
}
|
||||
|
||||
// Directory exists, check that a file can be written.
|
||||
file, err := os.CreateTemp(dir, "writetest")
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrPermission) {
|
||||
err = fs.ErrPermission
|
||||
}
|
||||
return fmt.Errorf("directory not writable: %s: %w", dir, err)
|
||||
}
|
||||
file.Close()
|
||||
return os.Remove(file.Name())
|
||||
}
|
||||
|
||||
// expandHome expands the path to include the home directory if the path is
|
||||
// prefixed with `~`. If it isn't prefixed with `~`, the path is returned
|
||||
// as-is.
|
||||
func expandHome(path string) (string, error) {
|
||||
if path == "" {
|
||||
return path, nil
|
||||
}
|
||||
|
||||
if path[0] != '~' {
|
||||
return path, nil
|
||||
}
|
||||
|
||||
if len(path) > 1 && path[1] != '/' && path[1] != '\\' {
|
||||
return "", errors.New("cannot expand user-specific home dir")
|
||||
}
|
||||
|
||||
dir, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return filepath.Join(dir, path[1:]), nil
|
||||
}
|
||||
@ -281,7 +281,7 @@ test_config_cmd() {
|
||||
|
||||
# won't work as it changes datastore definition, which makes ipfs not launch
|
||||
# without converting first
|
||||
# test_profile_apply_revert badgerds
|
||||
# test_profile_apply_revert pebbleds
|
||||
|
||||
test_expect_success "cleanup config backups" '
|
||||
find "$IPFS_PATH" -name "config-*" -exec rm {} \;
|
||||
|
||||
@ -4,13 +4,20 @@ test_description="Test non-standard datastores"
|
||||
|
||||
. lib/test-lib.sh
|
||||
|
||||
test_expect_success "'ipfs init --empty-repo=false --profile=badgerds' succeeds" '
|
||||
BITS="2048" &&
|
||||
ipfs init --empty-repo=false --profile=badgerds
|
||||
'
|
||||
profiles=("flatfs" "pebbleds" "badgerds")
|
||||
proot="$(mktemp -d "${TMPDIR:-/tmp}/t0025.XXXXXX")"
|
||||
|
||||
test_expect_success "'ipfs pin ls' works" '
|
||||
ipfs pin ls | wc -l | grep 9
|
||||
'
|
||||
for profile in "${profiles[@]}"; do
|
||||
test_expect_success "'ipfs init --empty-repo=false --profile=$profile' succeeds" '
|
||||
BITS="2048" &&
|
||||
IPFS_PATH="$proot/$profile" &&
|
||||
ipfs init --empty-repo=false --profile=$profile
|
||||
'
|
||||
test_expect_success "'ipfs pin add' and 'pin ls' works with $profile" '
|
||||
export IPFS_PATH="$proot/$profile" &&
|
||||
echo -n "hello_$profile" | ipfs block put --pin=true > hello_cid &&
|
||||
ipfs pin ls -t recursive "$(cat hello_cid)"
|
||||
'
|
||||
done
|
||||
|
||||
test_done
|
||||
|
||||
Reference in New Issue
Block a user