From 007ffd40bd439451193f9084687286e741eff8bf Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 16:45:54 -0800 Subject: [PATCH 01/23] refactor: move LatencyConfig --- test/epictest/addcat_test.go | 15 ++++---- test/epictest/bench_test.go | 18 ++++++---- test/epictest/core.go | 3 +- test/epictest/test_config.go | 47 ------------------------- test/epictest/three_legged_cat_test.go | 5 +-- util/testutil/latency_config.go | 48 ++++++++++++++++++++++++++ 6 files changed, 72 insertions(+), 64 deletions(-) create mode 100644 util/testutil/latency_config.go diff --git a/test/epictest/addcat_test.go b/test/epictest/addcat_test.go index ff6e899db..d214f0bab 100644 --- a/test/epictest/addcat_test.go +++ b/test/epictest/addcat_test.go @@ -13,12 +13,13 @@ import ( random "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-random" mocknet "github.com/jbenet/go-ipfs/p2p/net/mock" errors "github.com/jbenet/go-ipfs/util/debugerror" + testutil "github.com/jbenet/go-ipfs/util/testutil" ) const kSeed = 1 func Test1KBInstantaneous(t *testing.T) { - conf := Config{ + conf := testutil.LatencyConfig{ NetworkLatency: 0, RoutingLatency: 0, BlockstoreLatency: 0, @@ -31,7 +32,7 @@ func Test1KBInstantaneous(t *testing.T) { func TestDegenerateSlowBlockstore(t *testing.T) { SkipUnlessEpic(t) - conf := Config{BlockstoreLatency: 50 * time.Millisecond} + conf := testutil.LatencyConfig{BlockstoreLatency: 50 * time.Millisecond} if err := AddCatPowers(conf, 128); err != nil { t.Fatal(err) } @@ -39,7 +40,7 @@ func TestDegenerateSlowBlockstore(t *testing.T) { func TestDegenerateSlowNetwork(t *testing.T) { SkipUnlessEpic(t) - conf := Config{NetworkLatency: 400 * time.Millisecond} + conf := testutil.LatencyConfig{NetworkLatency: 400 * time.Millisecond} if err := AddCatPowers(conf, 128); err != nil { t.Fatal(err) } @@ -47,7 +48,7 @@ func TestDegenerateSlowNetwork(t *testing.T) { func TestDegenerateSlowRouting(t *testing.T) { SkipUnlessEpic(t) - conf := Config{RoutingLatency: 400 * time.Millisecond} + conf := testutil.LatencyConfig{RoutingLatency: 400 * time.Millisecond} if err := AddCatPowers(conf, 128); err != nil { t.Fatal(err) } @@ -55,13 +56,13 @@ func TestDegenerateSlowRouting(t *testing.T) { func Test100MBMacbookCoastToCoast(t *testing.T) { SkipUnlessEpic(t) - conf := Config{}.Network_NYtoSF().Blockstore_SlowSSD2014().Routing_Slow() + conf := testutil.LatencyConfig{}.Network_NYtoSF().Blockstore_SlowSSD2014().Routing_Slow() if err := DirectAddCat(RandomBytes(100*1024*1024), conf); err != nil { t.Fatal(err) } } -func AddCatPowers(conf Config, megabytesMax int64) error { +func AddCatPowers(conf testutil.LatencyConfig, megabytesMax int64) error { var i int64 for i = 1; i < megabytesMax; i = i * 2 { fmt.Printf("%d MB\n", i) @@ -78,7 +79,7 @@ func RandomBytes(n int64) []byte { return data.Bytes() } -func DirectAddCat(data []byte, conf Config) error { +func DirectAddCat(data []byte, conf testutil.LatencyConfig) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() const numPeers = 2 diff --git a/test/epictest/bench_test.go b/test/epictest/bench_test.go index 323ad6265..72c82871b 100644 --- a/test/epictest/bench_test.go +++ b/test/epictest/bench_test.go @@ -1,8 +1,12 @@ package epictest -import "testing" +import ( + "testing" -func benchmarkAddCat(numBytes int64, conf Config, b *testing.B) { + testutil "github.com/jbenet/go-ipfs/util/testutil" +) + +func benchmarkAddCat(numBytes int64, conf testutil.LatencyConfig, b *testing.B) { b.StopTimer() b.SetBytes(numBytes) @@ -16,7 +20,7 @@ func benchmarkAddCat(numBytes int64, conf Config, b *testing.B) { } } -var instant = Config{}.All_Instantaneous() +var instant = testutil.LatencyConfig{}.All_Instantaneous() func BenchmarkInstantaneousAddCat1KB(b *testing.B) { benchmarkAddCat(1*KB, instant, b) } func BenchmarkInstantaneousAddCat1MB(b *testing.B) { benchmarkAddCat(1*MB, instant, b) } @@ -29,7 +33,7 @@ func BenchmarkInstantaneousAddCat64MB(b *testing.B) { benchmarkAddCat(64*MB, in func BenchmarkInstantaneousAddCat128MB(b *testing.B) { benchmarkAddCat(128*MB, instant, b) } func BenchmarkInstantaneousAddCat256MB(b *testing.B) { benchmarkAddCat(256*MB, instant, b) } -var routing = Config{}.Routing_Slow() +var routing = testutil.LatencyConfig{}.Routing_Slow() func BenchmarkRoutingSlowAddCat1MB(b *testing.B) { benchmarkAddCat(1*MB, routing, b) } func BenchmarkRoutingSlowAddCat2MB(b *testing.B) { benchmarkAddCat(2*MB, routing, b) } @@ -42,7 +46,7 @@ func BenchmarkRoutingSlowAddCat128MB(b *testing.B) { benchmarkAddCat(128*MB, rou func BenchmarkRoutingSlowAddCat256MB(b *testing.B) { benchmarkAddCat(256*MB, routing, b) } func BenchmarkRoutingSlowAddCat512MB(b *testing.B) { benchmarkAddCat(512*MB, routing, b) } -var network = Config{}.Network_NYtoSF() +var network = testutil.LatencyConfig{}.Network_NYtoSF() func BenchmarkNetworkSlowAddCat1MB(b *testing.B) { benchmarkAddCat(1*MB, network, b) } func BenchmarkNetworkSlowAddCat2MB(b *testing.B) { benchmarkAddCat(2*MB, network, b) } @@ -54,7 +58,7 @@ func BenchmarkNetworkSlowAddCat64MB(b *testing.B) { benchmarkAddCat(64*MB, netw func BenchmarkNetworkSlowAddCat128MB(b *testing.B) { benchmarkAddCat(128*MB, network, b) } func BenchmarkNetworkSlowAddCat256MB(b *testing.B) { benchmarkAddCat(256*MB, network, b) } -var hdd = Config{}.Blockstore_7200RPM() +var hdd = testutil.LatencyConfig{}.Blockstore_7200RPM() func BenchmarkBlockstoreSlowAddCat1MB(b *testing.B) { benchmarkAddCat(1*MB, hdd, b) } func BenchmarkBlockstoreSlowAddCat2MB(b *testing.B) { benchmarkAddCat(2*MB, hdd, b) } @@ -66,7 +70,7 @@ func BenchmarkBlockstoreSlowAddCat64MB(b *testing.B) { benchmarkAddCat(64*MB, h func BenchmarkBlockstoreSlowAddCat128MB(b *testing.B) { benchmarkAddCat(128*MB, hdd, b) } func BenchmarkBlockstoreSlowAddCat256MB(b *testing.B) { benchmarkAddCat(256*MB, hdd, b) } -var mixed = Config{}.Network_NYtoSF().Blockstore_SlowSSD2014().Routing_Slow() +var mixed = testutil.LatencyConfig{}.Network_NYtoSF().Blockstore_SlowSSD2014().Routing_Slow() func BenchmarkMixedAddCat1MBXX(b *testing.B) { benchmarkAddCat(1*MB, mixed, b) } func BenchmarkMixedAddCat2MBXX(b *testing.B) { benchmarkAddCat(2*MB, mixed, b) } diff --git a/test/epictest/core.go b/test/epictest/core.go index e89db1522..4493e3350 100644 --- a/test/epictest/core.go +++ b/test/epictest/core.go @@ -9,6 +9,7 @@ import ( blockstore "github.com/jbenet/go-ipfs/blocks/blockstore" blockservice "github.com/jbenet/go-ipfs/blockservice" + testutil "github.com/jbenet/go-ipfs/util/testutil" exchange "github.com/jbenet/go-ipfs/exchange" bitswap "github.com/jbenet/go-ipfs/exchange/bitswap" bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network" @@ -130,7 +131,7 @@ func (r *repo) Exchange() exchange.Interface { return r.exchange } -func MocknetTestRepo(p peer.ID, h host.Host, conf Config) RepoFactory { +func MocknetTestRepo(p peer.ID, h host.Host, conf testutil.LatencyConfig) RepoFactory { return func(ctx context.Context) (Repo, error) { const kWriteCacheElems = 100 const alwaysSendToPeer = true diff --git a/test/epictest/test_config.go b/test/epictest/test_config.go index e3d34110e..619ed8ae4 100644 --- a/test/epictest/test_config.go +++ b/test/epictest/test_config.go @@ -1,48 +1 @@ package epictest - -import "time" - -type Config struct { - BlockstoreLatency time.Duration - NetworkLatency time.Duration - RoutingLatency time.Duration -} - -func (c Config) All_Instantaneous() Config { - // Could use a zero value but whatever. Consistency of interface - c.NetworkLatency = 0 - c.RoutingLatency = 0 - c.BlockstoreLatency = 0 - return c -} - -func (c Config) Network_NYtoSF() Config { - c.NetworkLatency = 20 * time.Millisecond - return c -} - -func (c Config) Network_IntraDatacenter2014() Config { - c.NetworkLatency = 250 * time.Microsecond - return c -} - -func (c Config) Blockstore_FastSSD2014() Config { - const iops = 100000 - c.BlockstoreLatency = (1 / iops) * time.Second - return c -} - -func (c Config) Blockstore_SlowSSD2014() Config { - c.BlockstoreLatency = 150 * time.Microsecond - return c -} - -func (c Config) Blockstore_7200RPM() Config { - c.BlockstoreLatency = 8 * time.Millisecond - return c -} - -func (c Config) Routing_Slow() Config { - c.RoutingLatency = 200 * time.Millisecond - return c -} diff --git a/test/epictest/three_legged_cat_test.go b/test/epictest/three_legged_cat_test.go index 38147e426..7ce396a34 100644 --- a/test/epictest/three_legged_cat_test.go +++ b/test/epictest/three_legged_cat_test.go @@ -9,10 +9,11 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" mocknet "github.com/jbenet/go-ipfs/p2p/net/mock" errors "github.com/jbenet/go-ipfs/util/debugerror" + testutil "github.com/jbenet/go-ipfs/util/testutil" ) func TestThreeLeggedCat(t *testing.T) { - conf := Config{ + conf := testutil.LatencyConfig{ NetworkLatency: 0, RoutingLatency: 0, BlockstoreLatency: 0, @@ -22,7 +23,7 @@ func TestThreeLeggedCat(t *testing.T) { } } -func RunThreeLeggedCat(data []byte, conf Config) error { +func RunThreeLeggedCat(data []byte, conf testutil.LatencyConfig) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() const numPeers = 3 diff --git a/util/testutil/latency_config.go b/util/testutil/latency_config.go new file mode 100644 index 000000000..767b9a638 --- /dev/null +++ b/util/testutil/latency_config.go @@ -0,0 +1,48 @@ +package testutil + +import "time" + +type LatencyConfig struct { + BlockstoreLatency time.Duration + NetworkLatency time.Duration + RoutingLatency time.Duration +} + +func (c LatencyConfig) All_Instantaneous() LatencyConfig { + // Could use a zero value but whatever. Consistency of interface + c.NetworkLatency = 0 + c.RoutingLatency = 0 + c.BlockstoreLatency = 0 + return c +} + +func (c LatencyConfig) Network_NYtoSF() LatencyConfig { + c.NetworkLatency = 20 * time.Millisecond + return c +} + +func (c LatencyConfig) Network_IntraDatacenter2014() LatencyConfig { + c.NetworkLatency = 250 * time.Microsecond + return c +} + +func (c LatencyConfig) Blockstore_FastSSD2014() LatencyConfig { + const iops = 100000 + c.BlockstoreLatency = (1 / iops) * time.Second + return c +} + +func (c LatencyConfig) Blockstore_SlowSSD2014() LatencyConfig { + c.BlockstoreLatency = 150 * time.Microsecond + return c +} + +func (c LatencyConfig) Blockstore_7200RPM() LatencyConfig { + c.BlockstoreLatency = 8 * time.Millisecond + return c +} + +func (c LatencyConfig) Routing_Slow() LatencyConfig { + c.RoutingLatency = 200 * time.Millisecond + return c +} From 57b3ffa533026d0597426f4f335ad85c93217e35 Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 17:23:11 -0800 Subject: [PATCH 02/23] feat: new core constructor + config options (Standard, Online, Offline) --- cmd/ipfs/init.go | 2 +- cmd/ipfs/main.go | 2 +- core/core.go | 31 +++++++++++++++++++++++++++++++ core/core_test.go | 4 ++-- 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/cmd/ipfs/init.go b/cmd/ipfs/init.go index fc4563f4e..697f9516d 100644 --- a/cmd/ipfs/init.go +++ b/cmd/ipfs/init.go @@ -121,7 +121,7 @@ func doInit(configRoot string, dspathOverride string, force bool, nBitsForKeypai func addTheWelcomeFile(conf *config.Config) error { // TODO extract this file creation operation into a function ctx, cancel := context.WithCancel(context.Background()) - nd, err := core.NewIpfsNode(ctx, conf, false) + nd, err := core.NewIPFSNode(ctx, core.Offline(conf)) if err != nil { return err } diff --git a/cmd/ipfs/main.go b/cmd/ipfs/main.go index 4d7f4350c..b58b8bf07 100644 --- a/cmd/ipfs/main.go +++ b/cmd/ipfs/main.go @@ -188,7 +188,7 @@ func (i *cmdInvocation) constructNodeFunc(ctx context.Context) func() (*core.Ipf // ok everything is good. set it on the invocation (for ownership) // and return it. - i.node, err = core.NewIpfsNode(ctx, cfg, cmdctx.Online) + i.node, err = core.NewIPFSNode(ctx, core.Standard(cfg, cmdctx.Online)) return i.node, err } } diff --git a/core/core.go b/core/core.go index d847411e4..c375872cf 100644 --- a/core/core.go +++ b/core/core.go @@ -1,6 +1,7 @@ package core import ( + "errors" "fmt" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" @@ -75,7 +76,37 @@ type Mounts struct { Ipns mount.Mount } +var errTODO = errors.New("TODO") + +type Configuration *IpfsNode // define a different type + +type ConfigOption func(ctx context.Context) (Configuration, error) + +func NewIPFSNode(ctx context.Context, option ConfigOption) (*IpfsNode, error) { + config, err := option(ctx) + if err != nil { + return nil, err + } + return config, nil +} + +func Offline(cfg *config.Config) ConfigOption { + return Standard(cfg, false) +} + +func Online(cfg *config.Config) ConfigOption { + return Standard(cfg, true) +} + +// DEPRECATED: use Online, Offline functions +func Standard(cfg *config.Config, online bool) ConfigOption { + return func(ctx context.Context) (Configuration, error) { + return NewIpfsNode(ctx, cfg, online) + } +} + // NewIpfsNode constructs a new IpfsNode based on the given config. +// DEPRECATED: use `NewIPFSNode` func NewIpfsNode(ctx context.Context, cfg *config.Config, online bool) (n *IpfsNode, err error) { success := false // flip to true after all sub-system inits succeed defer func() { diff --git a/core/core_test.go b/core/core_test.go index 619adbb50..12c8e554f 100644 --- a/core/core_test.go +++ b/core/core_test.go @@ -45,14 +45,14 @@ func TestInitialization(t *testing.T) { } for i, c := range good { - n, err := NewIpfsNode(ctx, c, false) + n, err := NewIPFSNode(ctx, Standard(c, false)) if n == nil || err != nil { t.Error("Should have constructed.", i, err) } } for i, c := range bad { - n, err := NewIpfsNode(ctx, c, false) + n, err := NewIPFSNode(ctx, Standard(c, false)) if n != nil || err == nil { t.Error("Should have failed to construct.", i) } From 4c2eda21234482da8a2265ee792b60d6e49ea406 Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 17:24:56 -0800 Subject: [PATCH 03/23] refactor: remove deprecated function --- core/core.go | 121 ++++++++++++++++++++++++--------------------------- 1 file changed, 57 insertions(+), 64 deletions(-) diff --git a/core/core.go b/core/core.go index c375872cf..956e49740 100644 --- a/core/core.go +++ b/core/core.go @@ -78,9 +78,7 @@ type Mounts struct { var errTODO = errors.New("TODO") -type Configuration *IpfsNode // define a different type - -type ConfigOption func(ctx context.Context) (Configuration, error) +type ConfigOption func(ctx context.Context) (*IpfsNode, error) func NewIPFSNode(ctx context.Context, option ConfigOption) (*IpfsNode, error) { config, err := option(ctx) @@ -100,73 +98,68 @@ func Online(cfg *config.Config) ConfigOption { // DEPRECATED: use Online, Offline functions func Standard(cfg *config.Config, online bool) ConfigOption { - return func(ctx context.Context) (Configuration, error) { - return NewIpfsNode(ctx, cfg, online) - } -} + return func(ctx context.Context) (n *IpfsNode, err error) { -// NewIpfsNode constructs a new IpfsNode based on the given config. -// DEPRECATED: use `NewIPFSNode` -func NewIpfsNode(ctx context.Context, cfg *config.Config, online bool) (n *IpfsNode, err error) { - success := false // flip to true after all sub-system inits succeed - defer func() { - if !success && n != nil { - n.Close() + success := false // flip to true after all sub-system inits succeed + defer func() { + if !success && n != nil { + n.Close() + } + }() + + if cfg == nil { + return nil, debugerror.Errorf("configuration required") } - }() - - if cfg == nil { - return nil, debugerror.Errorf("configuration required") - } - - n = &IpfsNode{ - onlineMode: online, - Config: cfg, - } - n.ContextGroup = ctxgroup.WithContextAndTeardown(ctx, n.teardown) - ctx = n.ContextGroup.Context() - - // setup Peerstore - n.Peerstore = peer.NewPeerstore() - - // setup datastore. - if n.Datastore, err = makeDatastore(cfg.Datastore); err != nil { - return nil, debugerror.Wrap(err) - } - - // setup local peer ID (private key is loaded in online setup) - if err := n.loadID(); err != nil { - return nil, err - } - - n.Blockstore, err = bstore.WriteCached(bstore.NewBlockstore(n.Datastore), kSizeBlockstoreWriteCache) - if err != nil { - return nil, debugerror.Wrap(err) - } - - // setup online services - if online { - if err := n.StartOnlineServices(); err != nil { - return nil, err // debugerror.Wraps. + n = &IpfsNode{ + onlineMode: online, + Config: cfg, } - } else { - n.Exchange = offline.Exchange(n.Blockstore) - } - n.Blocks, err = bserv.New(n.Blockstore, n.Exchange) - if err != nil { - return nil, debugerror.Wrap(err) - } + n.ContextGroup = ctxgroup.WithContextAndTeardown(ctx, n.teardown) + ctx = n.ContextGroup.Context() - n.DAG = merkledag.NewDAGService(n.Blocks) - n.Pinning, err = pin.LoadPinner(n.Datastore, n.DAG) - if err != nil { - n.Pinning = pin.NewPinner(n.Datastore, n.DAG) - } - n.Resolver = &path.Resolver{DAG: n.DAG} + // setup Peerstore + n.Peerstore = peer.NewPeerstore() - success = true - return n, nil + // setup datastore. + if n.Datastore, err = makeDatastore(cfg.Datastore); err != nil { + return nil, debugerror.Wrap(err) + } + + // setup local peer ID (private key is loaded in online setup) + if err := n.loadID(); err != nil { + return nil, err + } + + n.Blockstore, err = bstore.WriteCached(bstore.NewBlockstore(n.Datastore), kSizeBlockstoreWriteCache) + if err != nil { + return nil, debugerror.Wrap(err) + } + + // setup online services + if online { + if err := n.StartOnlineServices(); err != nil { + return nil, err // debugerror.Wraps. + } + } else { + n.Exchange = offline.Exchange(n.Blockstore) + } + + n.Blocks, err = bserv.New(n.Blockstore, n.Exchange) + if err != nil { + return nil, debugerror.Wrap(err) + } + + n.DAG = merkledag.NewDAGService(n.Blocks) + n.Pinning, err = pin.LoadPinner(n.Datastore, n.DAG) + if err != nil { + n.Pinning = pin.NewPinner(n.Datastore, n.DAG) + } + n.Resolver = &path.Resolver{DAG: n.DAG} + + success = true + return n, nil + } } func (n *IpfsNode) StartOnlineServices() error { From 2c3fb43350f825812bbf5320ca20824806e58733 Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 19:25:18 -0800 Subject: [PATCH 04/23] feat: expose IpfsNode.Bootstrap() method --- core/bootstrap.go | 15 ++------------- core/core.go | 31 ++++++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/core/bootstrap.go b/core/bootstrap.go index 8f11596ca..b2f8a59a2 100644 --- a/core/bootstrap.go +++ b/core/bootstrap.go @@ -29,7 +29,7 @@ func superviseConnections(parent context.Context, h host.Host, route *dht.IpfsDHT, // TODO depend on abstract interface for testing purposes store peer.Peerstore, - peers []config.BootstrapPeer) error { + peers []peer.PeerInfo) error { for { ctx, _ := context.WithTimeout(parent, connectiontimeout) @@ -51,7 +51,7 @@ func bootstrap(ctx context.Context, h host.Host, r *dht.IpfsDHT, ps peer.Peerstore, - boots []config.BootstrapPeer) error { + bootstrapPeers []peer.PeerInfo) error { connectedPeers := h.Network().Peers() if len(connectedPeers) >= recoveryThreshold { @@ -66,17 +66,6 @@ func bootstrap(ctx context.Context, log.Event(ctx, "bootstrapStart", h.ID()) log.Debugf("%s bootstrapping to %d more nodes", h.ID(), numCxnsToCreate) - var bootstrapPeers []peer.PeerInfo - for _, bootstrap := range boots { - p, err := toPeer(bootstrap) - if err != nil { - log.Event(ctx, "bootstrapError", h.ID(), lgbl.Error(err)) - log.Errorf("%s bootstrap error: %s", h.ID(), err) - return err - } - bootstrapPeers = append(bootstrapPeers, p) - } - var notConnected []peer.PeerInfo for _, p := range bootstrapPeers { if h.Network().Connectedness(p.ID) != inet.Connected { diff --git a/core/core.go b/core/core.go index 956e49740..02cbc9331 100644 --- a/core/core.go +++ b/core/core.go @@ -32,6 +32,7 @@ import ( ds2 "github.com/jbenet/go-ipfs/util/datastore2" debugerror "github.com/jbenet/go-ipfs/util/debugerror" eventlog "github.com/jbenet/go-ipfs/util/eventlog" + lgbl "github.com/jbenet/go-ipfs/util/eventlog/loggables" ) const IpnsValidatorTag = "ipns" @@ -66,6 +67,11 @@ type IpfsNode struct { Diagnostics *diag.Diagnostics // the diagnostics service ctxgroup.ContextGroup + + // dht allows node to Bootstrap when dht is present + // TODO privatize before merging. This is here temporarily during the + // migration of the TestNet constructor + DHT *dht.IpfsDHT } // Mounts defines what the node's mount state is. This should @@ -185,6 +191,7 @@ func (n *IpfsNode) StartOnlineServices() error { dhtRouting := dht.NewDHT(ctx, n.PeerHost, n.Datastore) dhtRouting.Validators[IpnsValidatorTag] = namesys.ValidateIpnsRecord n.Routing = dhtRouting + n.DHT = dhtRouting n.AddChildGroup(dhtRouting) // setup exchange service @@ -202,7 +209,17 @@ func (n *IpfsNode) StartOnlineServices() error { // an Exchange, Network, or Routing component and have the constructor // manage the wiring. In that scenario, this dangling function is a bit // awkward. - go superviseConnections(ctx, n.PeerHost, dhtRouting, n.Peerstore, n.Config.Bootstrap) + var bootstrapPeers []peer.PeerInfo + for _, bootstrap := range n.Config.Bootstrap { + p, err := toPeer(bootstrap) + if err != nil { + log.Event(ctx, "bootstrapError", n.Identity, lgbl.Error(err)) + log.Errorf("%s bootstrap error: %s", n.Identity, err) + return err + } + bootstrapPeers = append(bootstrapPeers, p) + } + go superviseConnections(ctx, n.PeerHost, dhtRouting, n.Peerstore, bootstrapPeers) return nil } @@ -245,6 +262,18 @@ func (n *IpfsNode) OnlineMode() bool { return n.onlineMode } +func (n *IpfsNode) Bootstrap(ctx context.Context, peers []peer.PeerInfo) error { + if n.DHT != nil { + for _, p := range peers { + // TODO bootstrap(ctx, n.PeerHost, n.DHT, n.Peerstore, peers) + if err := n.DHT.Connect(ctx, p.ID); err != nil { + return err + } + } + } + return nil +} + func (n *IpfsNode) loadID() error { if n.Identity != "" { return debugerror.New("identity already loaded") From 85401d53fd832e644de93dccf625e98f9be1a1c4 Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 19:48:24 -0800 Subject: [PATCH 05/23] refactor: use the Core in the integration test --- test/epictest/addcat_test.go | 4 +- test/epictest/core.go | 65 ++++++++++++-------------- test/epictest/three_legged_cat_test.go | 6 +-- 3 files changed, 35 insertions(+), 40 deletions(-) diff --git a/test/epictest/addcat_test.go b/test/epictest/addcat_test.go index d214f0bab..94fb67076 100644 --- a/test/epictest/addcat_test.go +++ b/test/epictest/addcat_test.go @@ -109,8 +109,8 @@ func DirectAddCat(data []byte, conf testutil.LatencyConfig) error { return err } - adder.Bootstrap(ctx, catter.ID()) - catter.Bootstrap(ctx, adder.ID()) + adder.Bootstrap(ctx, catter.Peerstore.PeerInfo(catter.PeerHost.ID())) + catter.Bootstrap(ctx, adder.Peerstore.PeerInfo(adder.PeerHost.ID())) keyAdded, err := adder.Add(bytes.NewReader(data)) if err != nil { diff --git a/test/epictest/core.go b/test/epictest/core.go index 4493e3350..dcdddc99c 100644 --- a/test/epictest/core.go +++ b/test/epictest/core.go @@ -9,7 +9,7 @@ import ( blockstore "github.com/jbenet/go-ipfs/blocks/blockstore" blockservice "github.com/jbenet/go-ipfs/blockservice" - testutil "github.com/jbenet/go-ipfs/util/testutil" + core "github.com/jbenet/go-ipfs/core" exchange "github.com/jbenet/go-ipfs/exchange" bitswap "github.com/jbenet/go-ipfs/exchange/bitswap" bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network" @@ -25,30 +25,26 @@ import ( "github.com/jbenet/go-ipfs/util/datastore2" delay "github.com/jbenet/go-ipfs/util/delay" eventlog "github.com/jbenet/go-ipfs/util/eventlog" + testutil "github.com/jbenet/go-ipfs/util/testutil" ) var log = eventlog.Logger("epictest") // TODO merge with core.IpfsNode -type core struct { - repo Repo - - blockService *blockservice.BlockService - blockstore blockstore.Blockstore - dag merkledag.DAGService - id peer.ID +type Core struct { + *core.IpfsNode } -func (c *core) ID() peer.ID { - return c.repo.ID() +func (c *Core) ID() peer.ID { + return c.IpfsNode.Identity } -func (c *core) Bootstrap(ctx context.Context, p peer.ID) error { - return c.repo.Bootstrap(ctx, p) +func (c *Core) Bootstrap(ctx context.Context, p peer.PeerInfo) error { + return c.IpfsNode.Bootstrap(ctx, []peer.PeerInfo{p}) } -func (c *core) Cat(k util.Key) (io.Reader, error) { - catterdag := c.dag +func (c *Core) Cat(k util.Key) (io.Reader, error) { + catterdag := c.IpfsNode.DAG nodeCatted, err := (&path.Resolver{catterdag}).ResolvePath(k.String()) if err != nil { return nil, err @@ -56,10 +52,10 @@ func (c *core) Cat(k util.Key) (io.Reader, error) { return uio.NewDagReader(nodeCatted, catterdag) } -func (c *core) Add(r io.Reader) (util.Key, error) { +func (c *Core) Add(r io.Reader) (util.Key, error) { nodeAdded, err := importer.BuildDagFromReader( r, - c.dag, + c.IpfsNode.DAG, nil, chunk.DefaultSplitter, ) @@ -69,28 +65,26 @@ func (c *core) Add(r io.Reader) (util.Key, error) { return nodeAdded.Key() } -func makeCore(ctx context.Context, rf RepoFactory) (*core, error) { - repo, err := rf(ctx) +func makeCore(ctx context.Context, rf RepoFactory) (*Core, error) { + node, err := rf(ctx) if err != nil { return nil, err } - bss, err := blockservice.New(repo.Blockstore(), repo.Exchange()) + node.Blocks, err = blockservice.New(node.Blockstore, node.Exchange) if err != nil { return nil, err } - dag := merkledag.NewDAGService(bss) + node.DAG = merkledag.NewDAGService(node.Blocks) // to make sure nothing is omitted, init each individual field and assign // all at once at the bottom. - return &core{ - repo: repo, - blockService: bss, - dag: dag, + return &Core{ + IpfsNode: node, }, nil } -type RepoFactory func(ctx context.Context) (Repo, error) +type RepoFactory func(ctx context.Context) (*core.IpfsNode, error) type Repo interface { ID() peer.ID @@ -132,11 +126,11 @@ func (r *repo) Exchange() exchange.Interface { } func MocknetTestRepo(p peer.ID, h host.Host, conf testutil.LatencyConfig) RepoFactory { - return func(ctx context.Context) (Repo, error) { + return func(ctx context.Context) (*core.IpfsNode, error) { const kWriteCacheElems = 100 const alwaysSendToPeer = true dsDelay := delay.Fixed(conf.BlockstoreLatency) - ds := sync.MutexWrap(datastore2.WithDelay(datastore.NewMapDatastore(), dsDelay)) + ds := datastore2.CloserWrap(sync.MutexWrap(datastore2.WithDelay(datastore.NewMapDatastore(), dsDelay))) log.Debugf("MocknetTestRepo: %s %s %s", p, h.ID(), h) dhtt := dht.NewDHT(ctx, h, ds) @@ -146,14 +140,15 @@ func MocknetTestRepo(p peer.ID, h host.Host, conf testutil.LatencyConfig) RepoFa return nil, err } exch := bitswap.New(ctx, p, bsn, bstore, alwaysSendToPeer) - return &repo{ - bitSwapNetwork: bsn, - blockstore: bstore, - exchange: exch, - datastore: ds, - host: h, - dht: dhtt, - id: p, + return &core.IpfsNode{ + Peerstore: h.Peerstore(), + Blockstore: bstore, + Exchange: exch, + Datastore: ds, + PeerHost: h, + Routing: dhtt, + Identity: p, + DHT: dhtt, }, nil } } diff --git a/test/epictest/three_legged_cat_test.go b/test/epictest/three_legged_cat_test.go index 7ce396a34..997f4b3a0 100644 --- a/test/epictest/three_legged_cat_test.go +++ b/test/epictest/three_legged_cat_test.go @@ -55,9 +55,9 @@ func RunThreeLeggedCat(data []byte, conf testutil.LatencyConfig) error { if err != nil { return err } - - adder.Bootstrap(ctx, bootstrap.ID()) - catter.Bootstrap(ctx, bootstrap.ID()) + boostrapInfo := bootstrap.Peerstore.PeerInfo(bootstrap.PeerHost.ID()) + adder.Bootstrap(ctx, boostrapInfo) + catter.Bootstrap(ctx, boostrapInfo) keyAdded, err := adder.Add(bytes.NewReader(data)) if err != nil { From efb75ee5ba5b87fb4755b884c807be9652e48097 Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 19:52:50 -0800 Subject: [PATCH 06/23] refactor: move add and cat to the core --- core/core.go | 33 +++++++++++++++++++++++++++++++++ test/epictest/core.go | 29 ----------------------------- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/core/core.go b/core/core.go index 02cbc9331..8ea5979cb 100644 --- a/core/core.go +++ b/core/core.go @@ -3,6 +3,7 @@ package core import ( "errors" "fmt" + "io" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58" @@ -18,6 +19,8 @@ import ( bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network" "github.com/jbenet/go-ipfs/exchange/offline" mount "github.com/jbenet/go-ipfs/fuse/mount" + importer "github.com/jbenet/go-ipfs/importer" + chunk "github.com/jbenet/go-ipfs/importer/chunk" merkledag "github.com/jbenet/go-ipfs/merkledag" namesys "github.com/jbenet/go-ipfs/namesys" ic "github.com/jbenet/go-ipfs/p2p/crypto" @@ -29,6 +32,8 @@ import ( pin "github.com/jbenet/go-ipfs/pin" routing "github.com/jbenet/go-ipfs/routing" dht "github.com/jbenet/go-ipfs/routing/dht" + uio "github.com/jbenet/go-ipfs/unixfs/io" + u "github.com/jbenet/go-ipfs/util" ds2 "github.com/jbenet/go-ipfs/util/datastore2" debugerror "github.com/jbenet/go-ipfs/util/debugerror" eventlog "github.com/jbenet/go-ipfs/util/eventlog" @@ -274,6 +279,34 @@ func (n *IpfsNode) Bootstrap(ctx context.Context, peers []peer.PeerInfo) error { return nil } +// TODO we may not want to add these methods to the core. Maybe they should be +// defined as free functions in another package that use public fields on the +// node. +// +// e.g. reader, err := unix.Cat(node) + +func (n *IpfsNode) Cat(k u.Key) (io.Reader, error) { + catterdag := n.DAG + nodeCatted, err := (&path.Resolver{catterdag}).ResolvePath(k.String()) + if err != nil { + return nil, err + } + return uio.NewDagReader(nodeCatted, catterdag) +} + +func (n *IpfsNode) Add(r io.Reader) (u.Key, error) { + nodeAdded, err := importer.BuildDagFromReader( + r, + n.DAG, + nil, + chunk.DefaultSplitter, + ) + if err != nil { + return "", err + } + return nodeAdded.Key() +} + func (n *IpfsNode) loadID() error { if n.Identity != "" { return debugerror.New("identity already loaded") diff --git a/test/epictest/core.go b/test/epictest/core.go index dcdddc99c..72e56459f 100644 --- a/test/epictest/core.go +++ b/test/epictest/core.go @@ -1,8 +1,6 @@ package epictest import ( - "io" - context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" datastore "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" sync "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync" @@ -13,15 +11,10 @@ import ( exchange "github.com/jbenet/go-ipfs/exchange" bitswap "github.com/jbenet/go-ipfs/exchange/bitswap" bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network" - importer "github.com/jbenet/go-ipfs/importer" - chunk "github.com/jbenet/go-ipfs/importer/chunk" merkledag "github.com/jbenet/go-ipfs/merkledag" host "github.com/jbenet/go-ipfs/p2p/host" peer "github.com/jbenet/go-ipfs/p2p/peer" - path "github.com/jbenet/go-ipfs/path" dht "github.com/jbenet/go-ipfs/routing/dht" - uio "github.com/jbenet/go-ipfs/unixfs/io" - util "github.com/jbenet/go-ipfs/util" "github.com/jbenet/go-ipfs/util/datastore2" delay "github.com/jbenet/go-ipfs/util/delay" eventlog "github.com/jbenet/go-ipfs/util/eventlog" @@ -43,28 +36,6 @@ func (c *Core) Bootstrap(ctx context.Context, p peer.PeerInfo) error { return c.IpfsNode.Bootstrap(ctx, []peer.PeerInfo{p}) } -func (c *Core) Cat(k util.Key) (io.Reader, error) { - catterdag := c.IpfsNode.DAG - nodeCatted, err := (&path.Resolver{catterdag}).ResolvePath(k.String()) - if err != nil { - return nil, err - } - return uio.NewDagReader(nodeCatted, catterdag) -} - -func (c *Core) Add(r io.Reader) (util.Key, error) { - nodeAdded, err := importer.BuildDagFromReader( - r, - c.IpfsNode.DAG, - nil, - chunk.DefaultSplitter, - ) - if err != nil { - return "", err - } - return nodeAdded.Key() -} - func makeCore(ctx context.Context, rf RepoFactory) (*Core, error) { node, err := rf(ctx) if err != nil { From 401b8f413d1dd04124eaf6bdf92c491e8efa4d79 Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 20:00:39 -0800 Subject: [PATCH 07/23] misc: move initialization sqaush --- test/epictest/core.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/epictest/core.go b/test/epictest/core.go index 72e56459f..a75562705 100644 --- a/test/epictest/core.go +++ b/test/epictest/core.go @@ -42,12 +42,6 @@ func makeCore(ctx context.Context, rf RepoFactory) (*Core, error) { return nil, err } - node.Blocks, err = blockservice.New(node.Blockstore, node.Exchange) - if err != nil { - return nil, err - } - - node.DAG = merkledag.NewDAGService(node.Blocks) // to make sure nothing is omitted, init each individual field and assign // all at once at the bottom. return &Core{ @@ -111,12 +105,19 @@ func MocknetTestRepo(p peer.ID, h host.Host, conf testutil.LatencyConfig) RepoFa return nil, err } exch := bitswap.New(ctx, p, bsn, bstore, alwaysSendToPeer) + blockservice, err := blockservice.New(bstore, exch) + if err != nil { + return nil, err + } + return &core.IpfsNode{ Peerstore: h.Peerstore(), Blockstore: bstore, Exchange: exch, Datastore: ds, PeerHost: h, + DAG: merkledag.NewDAGService(blockservice), + Blocks: blockservice, Routing: dhtt, Identity: p, DHT: dhtt, From 5c46597c532b70ec1aee11d45199cc331272d29f Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 20:05:53 -0800 Subject: [PATCH 08/23] use the core.NewIPFSNode constructor NB: all commits (from master) pass tests up to this point. fix: squashme into use core constructor --- test/epictest/addcat_test.go | 10 ++++++---- test/epictest/three_legged_cat_test.go | 12 +++++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/test/epictest/addcat_test.go b/test/epictest/addcat_test.go index 94fb67076..fa92bb943 100644 --- a/test/epictest/addcat_test.go +++ b/test/epictest/addcat_test.go @@ -11,7 +11,9 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" random "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-random" + "github.com/jbenet/go-ipfs/core" mocknet "github.com/jbenet/go-ipfs/p2p/net/mock" + "github.com/jbenet/go-ipfs/p2p/peer" errors "github.com/jbenet/go-ipfs/util/debugerror" testutil "github.com/jbenet/go-ipfs/util/testutil" ) @@ -100,17 +102,17 @@ func DirectAddCat(data []byte, conf testutil.LatencyConfig) error { return errors.New("test initialization error") } - adder, err := makeCore(ctx, MocknetTestRepo(peers[0], mn.Host(peers[0]), conf)) + adder, err := core.NewIPFSNode(ctx, core.ConfigOption(MocknetTestRepo(peers[0], mn.Host(peers[0]), conf))) if err != nil { return err } - catter, err := makeCore(ctx, MocknetTestRepo(peers[1], mn.Host(peers[1]), conf)) + catter, err := core.NewIPFSNode(ctx, core.ConfigOption(MocknetTestRepo(peers[1], mn.Host(peers[1]), conf))) if err != nil { return err } - adder.Bootstrap(ctx, catter.Peerstore.PeerInfo(catter.PeerHost.ID())) - catter.Bootstrap(ctx, adder.Peerstore.PeerInfo(adder.PeerHost.ID())) + catter.Bootstrap(ctx, []peer.PeerInfo{adder.Peerstore.PeerInfo(adder.Identity)}) + adder.Bootstrap(ctx, []peer.PeerInfo{catter.Peerstore.PeerInfo(catter.Identity)}) keyAdded, err := adder.Add(bytes.NewReader(data)) if err != nil { diff --git a/test/epictest/three_legged_cat_test.go b/test/epictest/three_legged_cat_test.go index 997f4b3a0..3ec6cbde3 100644 --- a/test/epictest/three_legged_cat_test.go +++ b/test/epictest/three_legged_cat_test.go @@ -7,7 +7,9 @@ import ( "testing" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" + "github.com/jbenet/go-ipfs/core" mocknet "github.com/jbenet/go-ipfs/p2p/net/mock" + "github.com/jbenet/go-ipfs/p2p/peer" errors "github.com/jbenet/go-ipfs/util/debugerror" testutil "github.com/jbenet/go-ipfs/util/testutil" ) @@ -43,21 +45,21 @@ func RunThreeLeggedCat(data []byte, conf testutil.LatencyConfig) error { if len(peers) < numPeers { return errors.New("test initialization error") } - adder, err := makeCore(ctx, MocknetTestRepo(peers[0], mn.Host(peers[0]), conf)) + adder, err := core.NewIPFSNode(ctx, core.ConfigOption(MocknetTestRepo(peers[0], mn.Host(peers[0]), conf))) if err != nil { return err } - catter, err := makeCore(ctx, MocknetTestRepo(peers[1], mn.Host(peers[1]), conf)) + catter, err := core.NewIPFSNode(ctx, core.ConfigOption(MocknetTestRepo(peers[1], mn.Host(peers[1]), conf))) if err != nil { return err } - bootstrap, err := makeCore(ctx, MocknetTestRepo(peers[2], mn.Host(peers[2]), conf)) + bootstrap, err := core.NewIPFSNode(ctx, core.ConfigOption(MocknetTestRepo(peers[2], mn.Host(peers[2]), conf))) if err != nil { return err } boostrapInfo := bootstrap.Peerstore.PeerInfo(bootstrap.PeerHost.ID()) - adder.Bootstrap(ctx, boostrapInfo) - catter.Bootstrap(ctx, boostrapInfo) + adder.Bootstrap(ctx, []peer.PeerInfo{boostrapInfo}) + catter.Bootstrap(ctx, []peer.PeerInfo{boostrapInfo}) keyAdded, err := adder.Add(bytes.NewReader(data)) if err != nil { From 7cebb33e814ef32b8d8d9f6830e57be9b03be6a9 Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 20:19:37 -0800 Subject: [PATCH 09/23] fix: remove dead code --- test/epictest/core.go | 70 ++----------------------------------------- 1 file changed, 2 insertions(+), 68 deletions(-) diff --git a/test/epictest/core.go b/test/epictest/core.go index a75562705..92202e2c5 100644 --- a/test/epictest/core.go +++ b/test/epictest/core.go @@ -8,7 +8,6 @@ import ( blockstore "github.com/jbenet/go-ipfs/blocks/blockstore" blockservice "github.com/jbenet/go-ipfs/blockservice" core "github.com/jbenet/go-ipfs/core" - exchange "github.com/jbenet/go-ipfs/exchange" bitswap "github.com/jbenet/go-ipfs/exchange/bitswap" bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network" merkledag "github.com/jbenet/go-ipfs/merkledag" @@ -23,74 +22,9 @@ import ( var log = eventlog.Logger("epictest") -// TODO merge with core.IpfsNode -type Core struct { - *core.IpfsNode -} +type ConfigOption func(ctx context.Context) (*core.IpfsNode, error) -func (c *Core) ID() peer.ID { - return c.IpfsNode.Identity -} - -func (c *Core) Bootstrap(ctx context.Context, p peer.PeerInfo) error { - return c.IpfsNode.Bootstrap(ctx, []peer.PeerInfo{p}) -} - -func makeCore(ctx context.Context, rf RepoFactory) (*Core, error) { - node, err := rf(ctx) - if err != nil { - return nil, err - } - - // to make sure nothing is omitted, init each individual field and assign - // all at once at the bottom. - return &Core{ - IpfsNode: node, - }, nil -} - -type RepoFactory func(ctx context.Context) (*core.IpfsNode, error) - -type Repo interface { - ID() peer.ID - Blockstore() blockstore.Blockstore - Exchange() exchange.Interface - - Bootstrap(ctx context.Context, peer peer.ID) error -} - -type repo struct { - // DHT, Exchange, Network,Datastore - bitSwapNetwork bsnet.BitSwapNetwork - blockstore blockstore.Blockstore - exchange exchange.Interface - datastore datastore.ThreadSafeDatastore - host host.Host - dht *dht.IpfsDHT - id peer.ID -} - -func (r *repo) ID() peer.ID { - return r.id -} - -func (c *repo) Bootstrap(ctx context.Context, p peer.ID) error { - return c.dht.Connect(ctx, p) -} - -func (r *repo) Datastore() datastore.ThreadSafeDatastore { - return r.datastore -} - -func (r *repo) Blockstore() blockstore.Blockstore { - return r.blockstore -} - -func (r *repo) Exchange() exchange.Interface { - return r.exchange -} - -func MocknetTestRepo(p peer.ID, h host.Host, conf testutil.LatencyConfig) RepoFactory { +func MocknetTestRepo(p peer.ID, h host.Host, conf testutil.LatencyConfig) ConfigOption { return func(ctx context.Context) (*core.IpfsNode, error) { const kWriteCacheElems = 100 const alwaysSendToPeer = true From 91808e18f73e8a4cb0cb8b69dc9aee566875a3f1 Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 20:20:39 -0800 Subject: [PATCH 10/23] refactor: use core.ConfigOption return type --- test/epictest/core.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/epictest/core.go b/test/epictest/core.go index 92202e2c5..118cd89c9 100644 --- a/test/epictest/core.go +++ b/test/epictest/core.go @@ -22,9 +22,7 @@ import ( var log = eventlog.Logger("epictest") -type ConfigOption func(ctx context.Context) (*core.IpfsNode, error) - -func MocknetTestRepo(p peer.ID, h host.Host, conf testutil.LatencyConfig) ConfigOption { +func MocknetTestRepo(p peer.ID, h host.Host, conf testutil.LatencyConfig) core.ConfigOption { return func(ctx context.Context) (*core.IpfsNode, error) { const kWriteCacheElems = 100 const alwaysSendToPeer = true @@ -43,7 +41,6 @@ func MocknetTestRepo(p peer.ID, h host.Host, conf testutil.LatencyConfig) Config if err != nil { return nil, err } - return &core.IpfsNode{ Peerstore: h.Peerstore(), Blockstore: bstore, From 7fa5d810935461c04c5089c4f4375ae86cd991e8 Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 20:21:21 -0800 Subject: [PATCH 11/23] refactor(core): rename --- core/core.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/core.go b/core/core.go index 8ea5979cb..ea13edc09 100644 --- a/core/core.go +++ b/core/core.go @@ -92,11 +92,11 @@ var errTODO = errors.New("TODO") type ConfigOption func(ctx context.Context) (*IpfsNode, error) func NewIPFSNode(ctx context.Context, option ConfigOption) (*IpfsNode, error) { - config, err := option(ctx) + node, err := option(ctx) if err != nil { return nil, err } - return config, nil + return node, nil } func Offline(cfg *config.Config) ConfigOption { From 34e8f3ee26a0d13606763bafe082265c5d8efca7 Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 20:24:34 -0800 Subject: [PATCH 12/23] refactor(core): init DAG and its dependents in shared constructor fix: remove merkledag import may need to squash this commit into the merkledag move commit --- core/core.go | 18 +++++++++++------- test/epictest/core.go | 2 -- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/core/core.go b/core/core.go index ea13edc09..f9ae30970 100644 --- a/core/core.go +++ b/core/core.go @@ -96,6 +96,17 @@ func NewIPFSNode(ctx context.Context, option ConfigOption) (*IpfsNode, error) { if err != nil { return nil, err } + + // Need to make sure it's perfectly clear 1) which variables are expected + // to be initialized at this point, and 2) which variables will be + // initialized after this point. + + node.DAG = merkledag.NewDAGService(node.Blocks) + node.Pinning, err = pin.LoadPinner(node.Datastore, node.DAG) + if err != nil { + node.Pinning = pin.NewPinner(node.Datastore, node.DAG) + } + node.Resolver = &path.Resolver{DAG: node.DAG} return node, nil } @@ -161,13 +172,6 @@ func Standard(cfg *config.Config, online bool) ConfigOption { return nil, debugerror.Wrap(err) } - n.DAG = merkledag.NewDAGService(n.Blocks) - n.Pinning, err = pin.LoadPinner(n.Datastore, n.DAG) - if err != nil { - n.Pinning = pin.NewPinner(n.Datastore, n.DAG) - } - n.Resolver = &path.Resolver{DAG: n.DAG} - success = true return n, nil } diff --git a/test/epictest/core.go b/test/epictest/core.go index 118cd89c9..f7e245e3c 100644 --- a/test/epictest/core.go +++ b/test/epictest/core.go @@ -10,7 +10,6 @@ import ( core "github.com/jbenet/go-ipfs/core" bitswap "github.com/jbenet/go-ipfs/exchange/bitswap" bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network" - merkledag "github.com/jbenet/go-ipfs/merkledag" host "github.com/jbenet/go-ipfs/p2p/host" peer "github.com/jbenet/go-ipfs/p2p/peer" dht "github.com/jbenet/go-ipfs/routing/dht" @@ -47,7 +46,6 @@ func MocknetTestRepo(p peer.ID, h host.Host, conf testutil.LatencyConfig) core.C Exchange: exch, Datastore: ds, PeerHost: h, - DAG: merkledag.NewDAGService(blockservice), Blocks: blockservice, Routing: dhtt, Identity: p, From 707874c3a595c87d5d46cecab3e6a1bb8ac0b997 Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 20:33:50 -0800 Subject: [PATCH 13/23] refactor(core): init node.Blocks in shared NewIPFSNode constructor --- core/core.go | 9 ++++----- test/epictest/core.go | 6 ------ 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/core/core.go b/core/core.go index f9ae30970..a265ad11b 100644 --- a/core/core.go +++ b/core/core.go @@ -101,6 +101,10 @@ func NewIPFSNode(ctx context.Context, option ConfigOption) (*IpfsNode, error) { // to be initialized at this point, and 2) which variables will be // initialized after this point. + node.Blocks, err = bserv.New(node.Blockstore, node.Exchange) + if err != nil { + return nil, debugerror.Wrap(err) + } node.DAG = merkledag.NewDAGService(node.Blocks) node.Pinning, err = pin.LoadPinner(node.Datastore, node.DAG) if err != nil { @@ -167,11 +171,6 @@ func Standard(cfg *config.Config, online bool) ConfigOption { n.Exchange = offline.Exchange(n.Blockstore) } - n.Blocks, err = bserv.New(n.Blockstore, n.Exchange) - if err != nil { - return nil, debugerror.Wrap(err) - } - success = true return n, nil } diff --git a/test/epictest/core.go b/test/epictest/core.go index f7e245e3c..d5aef22f4 100644 --- a/test/epictest/core.go +++ b/test/epictest/core.go @@ -6,7 +6,6 @@ import ( sync "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync" blockstore "github.com/jbenet/go-ipfs/blocks/blockstore" - blockservice "github.com/jbenet/go-ipfs/blockservice" core "github.com/jbenet/go-ipfs/core" bitswap "github.com/jbenet/go-ipfs/exchange/bitswap" bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network" @@ -36,17 +35,12 @@ func MocknetTestRepo(p peer.ID, h host.Host, conf testutil.LatencyConfig) core.C return nil, err } exch := bitswap.New(ctx, p, bsn, bstore, alwaysSendToPeer) - blockservice, err := blockservice.New(bstore, exch) - if err != nil { - return nil, err - } return &core.IpfsNode{ Peerstore: h.Peerstore(), Blockstore: bstore, Exchange: exch, Datastore: ds, PeerHost: h, - Blocks: blockservice, Routing: dhtt, Identity: p, DHT: dhtt, From 8e42e86d874946a34affac4ea2a198d656b2cb06 Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 23:26:29 -0800 Subject: [PATCH 14/23] hack(core): instantiate peerstore at the end if it hasn't already been instantiated this is a dirty hack --- core/core.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/core.go b/core/core.go index a265ad11b..eacc24d4e 100644 --- a/core/core.go +++ b/core/core.go @@ -105,6 +105,9 @@ func NewIPFSNode(ctx context.Context, option ConfigOption) (*IpfsNode, error) { if err != nil { return nil, debugerror.Wrap(err) } + if node.Peerstore == nil { + node.Peerstore = peer.NewPeerstore() + } node.DAG = merkledag.NewDAGService(node.Blocks) node.Pinning, err = pin.LoadPinner(node.Datastore, node.DAG) if err != nil { From eb6c40d5dd0db765464f5f06f05a43cb1ffc6a90 Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 20:57:32 -0800 Subject: [PATCH 15/23] refactor(core): isolate the complex initialization of PeerHost construction --- core/core.go | 60 ++++++++++++++++++++++++++-------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/core/core.go b/core/core.go index eacc24d4e..3f2f5122f 100644 --- a/core/core.go +++ b/core/core.go @@ -191,9 +191,11 @@ func (n *IpfsNode) StartOnlineServices() error { return err } - if err := n.startNetwork(); err != nil { - return err + peerhost, err := constructPeerHost(ctx, n.ContextGroup, n.Config, n.Identity, n.Peerstore) + if err != nil { + return debugerror.Wrap(err) } + n.PeerHost = peerhost // setup diagnostics service n.Diagnostics = diag.NewDiagnostics(n.Identity, n.PeerHost) @@ -234,34 +236,6 @@ func (n *IpfsNode) StartOnlineServices() error { return nil } -func (n *IpfsNode) startNetwork() error { - ctx := n.Context() - - // setup the network - listenAddrs, err := listenAddresses(n.Config) - if err != nil { - return debugerror.Wrap(err) - } - // make sure we dont error out if our config includes some addresses we cant use. - listenAddrs = swarm.FilterAddrs(listenAddrs) - network, err := swarm.NewNetwork(ctx, listenAddrs, n.Identity, n.Peerstore) - if err != nil { - return debugerror.Wrap(err) - } - n.AddChildGroup(network.CtxGroup()) - n.PeerHost = p2pbhost.New(network) - - // explicitly set these as our listen addrs. - // (why not do it inside inet.NewNetwork? because this way we can - // listen on addresses without necessarily advertising those publicly.) - addrs, err := n.PeerHost.Network().InterfaceListenAddresses() - if err != nil { - return debugerror.Wrap(err) - } - n.Peerstore.AddAddresses(n.Identity, addrs) - return nil -} - func (n *IpfsNode) teardown() error { if err := n.Datastore.Close(); err != nil { return err @@ -381,3 +355,29 @@ func listenAddresses(cfg *config.Config) ([]ma.Multiaddr, error) { return listen, nil } + +// isolates the complex initialization steps +func constructPeerHost(ctx context.Context, ctxg ctxgroup.ContextGroup, cfg *config.Config, id peer.ID, ps peer.Peerstore) (p2phost.Host, error) { + listenAddrs, err := listenAddresses(cfg) + // make sure we dont error out if our config includes some addresses we cant use. + filteredAddrs := swarm.FilterAddrs(listenAddrs) + if err != nil { + return nil, debugerror.Wrap(err) + } + network, err := swarm.NewNetwork(ctx, filteredAddrs, id, ps) + if err != nil { + return nil, debugerror.Wrap(err) + } + ctxg.AddChildGroup(network.CtxGroup()) + + peerhost := p2pbhost.New(network) + // explicitly set these as our listen addrs. + // (why not do it inside inet.NewNetwork? because this way we can + // listen on addresses without necessarily advertising those publicly.) + addrs, err := peerhost.Network().InterfaceListenAddresses() + if err != nil { + return nil, debugerror.Wrap(err) + } + ps.AddAddresses(id, addrs) + return peerhost, nil +} From ca8190a889322c287694b8fc628c50116f0f258d Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 21:09:25 -0800 Subject: [PATCH 16/23] refactor(core): isolate complex DHT initialization --- core/core.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/core/core.go b/core/core.go index 3f2f5122f..418f60004 100644 --- a/core/core.go +++ b/core/core.go @@ -8,6 +8,7 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58" ctxgroup "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup" + datastore "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" bstore "github.com/jbenet/go-ipfs/blocks/blockstore" @@ -201,11 +202,12 @@ func (n *IpfsNode) StartOnlineServices() error { n.Diagnostics = diag.NewDiagnostics(n.Identity, n.PeerHost) // setup routing service - dhtRouting := dht.NewDHT(ctx, n.PeerHost, n.Datastore) - dhtRouting.Validators[IpnsValidatorTag] = namesys.ValidateIpnsRecord - n.Routing = dhtRouting + dhtRouting, err := constructDHTRouting(ctx, n.ContextGroup, n.PeerHost, n.Datastore) + if err != nil { + return debugerror.Wrap(err) + } n.DHT = dhtRouting - n.AddChildGroup(dhtRouting) + n.Routing = dhtRouting // setup exchange service const alwaysSendToPeer = true // use YesManStrategy @@ -232,7 +234,7 @@ func (n *IpfsNode) StartOnlineServices() error { } bootstrapPeers = append(bootstrapPeers, p) } - go superviseConnections(ctx, n.PeerHost, dhtRouting, n.Peerstore, bootstrapPeers) + go superviseConnections(ctx, n.PeerHost, n.DHT, n.Peerstore, bootstrapPeers) return nil } @@ -381,3 +383,10 @@ func constructPeerHost(ctx context.Context, ctxg ctxgroup.ContextGroup, cfg *con ps.AddAddresses(id, addrs) return peerhost, nil } + +func constructDHTRouting(ctx context.Context, ctxg ctxgroup.ContextGroup, host p2phost.Host, ds datastore.ThreadSafeDatastore) (*dht.IpfsDHT, error) { + dhtRouting := dht.NewDHT(ctx, host, ds) + dhtRouting.Validators[IpnsValidatorTag] = namesys.ValidateIpnsRecord + ctxg.AddChildGroup(dhtRouting) + return dhtRouting, nil +} From 9d438843bdd5e973a28cec525fa4e945174b75ed Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 21:33:17 -0800 Subject: [PATCH 17/23] refactor(core): distinguish Online services --- core/core.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/core/core.go b/core/core.go index 418f60004..a7f19779c 100644 --- a/core/core.go +++ b/core/core.go @@ -52,7 +52,6 @@ type IpfsNode struct { // Self Config *config.Config // the node's configuration Identity peer.ID // the local node's identity - PrivateKey ic.PrivKey // the local node's private Key onlineMode bool // alternatively, offline // Local node @@ -61,16 +60,19 @@ type IpfsNode struct { Mounts Mounts // current mount state, if any. // Services - Peerstore peer.Peerstore // storage for other Peer instances - PeerHost p2phost.Host // the network host (server+client) - Routing routing.IpfsRouting // the routing system. recommend ipfs-dht - Exchange exchange.Interface // the block exchange + strategy (bitswap) - Blockstore bstore.Blockstore // the block store (lower level) - Blocks *bserv.BlockService // the block service, get/add blocks. - DAG merkledag.DAGService // the merkle dag service, get/add objects. - Resolver *path.Resolver // the path resolution system - Namesys namesys.NameSystem // the name system, resolves paths to hashes - Diagnostics *diag.Diagnostics // the diagnostics service + Peerstore peer.Peerstore // storage for other Peer instances + Blockstore bstore.Blockstore // the block store (lower level) + Blocks *bserv.BlockService // the block service, get/add blocks. + DAG merkledag.DAGService // the merkle dag service, get/add objects. + Resolver *path.Resolver // the path resolution system + + // Online + PrivateKey ic.PrivKey // the local node's private Key + PeerHost p2phost.Host // the network host (server+client) + Routing routing.IpfsRouting // the routing system. recommend ipfs-dht + Exchange exchange.Interface // the block exchange + strategy (bitswap) + Namesys namesys.NameSystem // the name system, resolves paths to hashes + Diagnostics *diag.Diagnostics // the diagnostics service ctxgroup.ContextGroup From b0f7143250a3387f23e2dc462b4aa4bd63fb846e Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 21:35:30 -0800 Subject: [PATCH 18/23] refactor(core): distinguish repo.Repo components --- core/core.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/core/core.go b/core/core.go index a7f19779c..a6906edaa 100644 --- a/core/core.go +++ b/core/core.go @@ -50,14 +50,16 @@ var log = eventlog.Logger("core") type IpfsNode struct { // Self - Config *config.Config // the node's configuration - Identity peer.ID // the local node's identity - onlineMode bool // alternatively, offline + Identity peer.ID // the local node's identity + onlineMode bool // alternatively, offline + + // TODO abstract as repo.Repo + Config *config.Config // the node's configuration + Datastore ds2.ThreadSafeDatastoreCloser // the local datastore // Local node - Datastore ds2.ThreadSafeDatastoreCloser // the local datastore - Pinning pin.Pinner // the pinning manager - Mounts Mounts // current mount state, if any. + Pinning pin.Pinner // the pinning manager + Mounts Mounts // current mount state, if any. // Services Peerstore peer.Peerstore // storage for other Peer instances From 130532c5323a445149b278efdec4563e8acfe68e Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 21:36:22 -0800 Subject: [PATCH 19/23] rm errTODO --- core/core.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/core.go b/core/core.go index a6906edaa..528bc5c22 100644 --- a/core/core.go +++ b/core/core.go @@ -1,7 +1,6 @@ package core import ( - "errors" "fmt" "io" @@ -92,8 +91,6 @@ type Mounts struct { Ipns mount.Mount } -var errTODO = errors.New("TODO") - type ConfigOption func(ctx context.Context) (*IpfsNode, error) func NewIPFSNode(ctx context.Context, option ConfigOption) (*IpfsNode, error) { From ddf14bee78928d29a3cdf29e0d60bda7943777dd Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 22:52:49 -0800 Subject: [PATCH 20/23] refactor(core): replace online bool with mode type --- core/core.go | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/core/core.go b/core/core.go index 528bc5c22..fd18e5527 100644 --- a/core/core.go +++ b/core/core.go @@ -17,7 +17,7 @@ import ( exchange "github.com/jbenet/go-ipfs/exchange" bitswap "github.com/jbenet/go-ipfs/exchange/bitswap" bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network" - "github.com/jbenet/go-ipfs/exchange/offline" + offline "github.com/jbenet/go-ipfs/exchange/offline" mount "github.com/jbenet/go-ipfs/fuse/mount" importer "github.com/jbenet/go-ipfs/importer" chunk "github.com/jbenet/go-ipfs/importer/chunk" @@ -45,12 +45,20 @@ const kSizeBlockstoreWriteCache = 100 var log = eventlog.Logger("core") +type mode int + +const ( + // zero value is not a valid mode, must be explicitly set + invalidMode mode = iota + offlineMode + onlineMode +) + // IpfsNode is IPFS Core module. It represents an IPFS instance. type IpfsNode struct { // Self - Identity peer.ID // the local node's identity - onlineMode bool // alternatively, offline + Identity peer.ID // the local node's identity // TODO abstract as repo.Repo Config *config.Config // the node's configuration @@ -80,7 +88,8 @@ type IpfsNode struct { // dht allows node to Bootstrap when dht is present // TODO privatize before merging. This is here temporarily during the // migration of the TestNet constructor - DHT *dht.IpfsDHT + DHT *dht.IpfsDHT + mode mode } // Mounts defines what the node's mount state is. This should @@ -142,8 +151,13 @@ func Standard(cfg *config.Config, online bool) ConfigOption { return nil, debugerror.Errorf("configuration required") } n = &IpfsNode{ - onlineMode: online, - Config: cfg, + mode: func() mode { + if online { + return onlineMode + } + return offlineMode + }(), + Config: cfg, } n.ContextGroup = ctxgroup.WithContextAndTeardown(ctx, n.teardown) @@ -247,7 +261,12 @@ func (n *IpfsNode) teardown() error { } func (n *IpfsNode) OnlineMode() bool { - return n.onlineMode + switch n.mode { + case onlineMode: + return true + default: + return false + } } func (n *IpfsNode) Bootstrap(ctx context.Context, peers []peer.PeerInfo) error { From 223ee4df1a778dbe5654787e12221f6616b2354c Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 23:12:26 -0800 Subject: [PATCH 21/23] refactor(core): move Add, Cat to core/io --- core/core.go | 33 ------------------------ core/io/add.go | 35 ++++++++++++++++++++++++++ core/io/cat.go | 30 ++++++++++++++++++++++ test/epictest/addcat_test.go | 5 ++-- test/epictest/three_legged_cat_test.go | 7 +++--- 5 files changed, 72 insertions(+), 38 deletions(-) create mode 100644 core/io/add.go create mode 100644 core/io/cat.go diff --git a/core/core.go b/core/core.go index fd18e5527..c6ab76ab7 100644 --- a/core/core.go +++ b/core/core.go @@ -2,7 +2,6 @@ package core import ( "fmt" - "io" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58" @@ -19,8 +18,6 @@ import ( bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network" offline "github.com/jbenet/go-ipfs/exchange/offline" mount "github.com/jbenet/go-ipfs/fuse/mount" - importer "github.com/jbenet/go-ipfs/importer" - chunk "github.com/jbenet/go-ipfs/importer/chunk" merkledag "github.com/jbenet/go-ipfs/merkledag" namesys "github.com/jbenet/go-ipfs/namesys" ic "github.com/jbenet/go-ipfs/p2p/crypto" @@ -32,8 +29,6 @@ import ( pin "github.com/jbenet/go-ipfs/pin" routing "github.com/jbenet/go-ipfs/routing" dht "github.com/jbenet/go-ipfs/routing/dht" - uio "github.com/jbenet/go-ipfs/unixfs/io" - u "github.com/jbenet/go-ipfs/util" ds2 "github.com/jbenet/go-ipfs/util/datastore2" debugerror "github.com/jbenet/go-ipfs/util/debugerror" eventlog "github.com/jbenet/go-ipfs/util/eventlog" @@ -281,34 +276,6 @@ func (n *IpfsNode) Bootstrap(ctx context.Context, peers []peer.PeerInfo) error { return nil } -// TODO we may not want to add these methods to the core. Maybe they should be -// defined as free functions in another package that use public fields on the -// node. -// -// e.g. reader, err := unix.Cat(node) - -func (n *IpfsNode) Cat(k u.Key) (io.Reader, error) { - catterdag := n.DAG - nodeCatted, err := (&path.Resolver{catterdag}).ResolvePath(k.String()) - if err != nil { - return nil, err - } - return uio.NewDagReader(nodeCatted, catterdag) -} - -func (n *IpfsNode) Add(r io.Reader) (u.Key, error) { - nodeAdded, err := importer.BuildDagFromReader( - r, - n.DAG, - nil, - chunk.DefaultSplitter, - ) - if err != nil { - return "", err - } - return nodeAdded.Key() -} - func (n *IpfsNode) loadID() error { if n.Identity != "" { return debugerror.New("identity already loaded") diff --git a/core/io/add.go b/core/io/add.go new file mode 100644 index 000000000..384cbba90 --- /dev/null +++ b/core/io/add.go @@ -0,0 +1,35 @@ +package core_io + +// TODO rename package to something that doesn't conflict with io/ioutil. +// Pretty names are hard to find. +// +// Candidates: +// +// go-ipfs/core/unix +// go-ipfs/core/io +// go-ipfs/core/ioutil +// go-ipfs/core/coreio +// go-ipfs/core/coreunix + +import ( + "io" + + core "github.com/jbenet/go-ipfs/core" + importer "github.com/jbenet/go-ipfs/importer" + chunk "github.com/jbenet/go-ipfs/importer/chunk" + u "github.com/jbenet/go-ipfs/util" +) + +func Add(n *core.IpfsNode, r io.Reader) (u.Key, error) { + // TODO more attractive function signature importer.BuildDagFromReader + dagNode, err := importer.BuildDagFromReader( + r, + n.DAG, + nil, + chunk.DefaultSplitter, + ) + if err != nil { + return "", err + } + return dagNode.Key() +} diff --git a/core/io/cat.go b/core/io/cat.go new file mode 100644 index 000000000..29dacf9d6 --- /dev/null +++ b/core/io/cat.go @@ -0,0 +1,30 @@ +package core_io + +// TODO rename package to something that doesn't conflict with io/ioutil. +// Pretty names are hard to find. +// +// Candidates: +// +// go-ipfs/core/unix +// go-ipfs/core/io +// go-ipfs/core/ioutil +// go-ipfs/core/coreio +// go-ipfs/core/coreunix + +import ( + "io" + + core "github.com/jbenet/go-ipfs/core" + path "github.com/jbenet/go-ipfs/path" + uio "github.com/jbenet/go-ipfs/unixfs/io" + u "github.com/jbenet/go-ipfs/util" +) + +func Cat(n *core.IpfsNode, k u.Key) (io.Reader, error) { + dag := n.DAG + dagNode, err := (&path.Resolver{dag}).ResolvePath(k.String()) + if err != nil { + return nil, err + } + return uio.NewDagReader(dagNode, dag) +} diff --git a/test/epictest/addcat_test.go b/test/epictest/addcat_test.go index fa92bb943..d3a99e932 100644 --- a/test/epictest/addcat_test.go +++ b/test/epictest/addcat_test.go @@ -12,6 +12,7 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" random "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-random" "github.com/jbenet/go-ipfs/core" + core_io "github.com/jbenet/go-ipfs/core/io" mocknet "github.com/jbenet/go-ipfs/p2p/net/mock" "github.com/jbenet/go-ipfs/p2p/peer" errors "github.com/jbenet/go-ipfs/util/debugerror" @@ -114,12 +115,12 @@ func DirectAddCat(data []byte, conf testutil.LatencyConfig) error { catter.Bootstrap(ctx, []peer.PeerInfo{adder.Peerstore.PeerInfo(adder.Identity)}) adder.Bootstrap(ctx, []peer.PeerInfo{catter.Peerstore.PeerInfo(catter.Identity)}) - keyAdded, err := adder.Add(bytes.NewReader(data)) + keyAdded, err := core_io.Add(adder, bytes.NewReader(data)) if err != nil { return err } - readerCatted, err := catter.Cat(keyAdded) + readerCatted, err := core_io.Cat(catter, keyAdded) if err != nil { return err } diff --git a/test/epictest/three_legged_cat_test.go b/test/epictest/three_legged_cat_test.go index 3ec6cbde3..b33b07077 100644 --- a/test/epictest/three_legged_cat_test.go +++ b/test/epictest/three_legged_cat_test.go @@ -7,7 +7,8 @@ import ( "testing" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" - "github.com/jbenet/go-ipfs/core" + core "github.com/jbenet/go-ipfs/core" + core_io "github.com/jbenet/go-ipfs/core/io" mocknet "github.com/jbenet/go-ipfs/p2p/net/mock" "github.com/jbenet/go-ipfs/p2p/peer" errors "github.com/jbenet/go-ipfs/util/debugerror" @@ -61,12 +62,12 @@ func RunThreeLeggedCat(data []byte, conf testutil.LatencyConfig) error { adder.Bootstrap(ctx, []peer.PeerInfo{boostrapInfo}) catter.Bootstrap(ctx, []peer.PeerInfo{boostrapInfo}) - keyAdded, err := adder.Add(bytes.NewReader(data)) + keyAdded, err := core_io.Add(adder, bytes.NewReader(data)) if err != nil { return err } - readerCatted, err := catter.Cat(keyAdded) + readerCatted, err := core_io.Cat(catter, keyAdded) if err != nil { return err } From 033e33dadf6ab0aca22e7bbd370598a5ac4b167d Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 23:17:19 -0800 Subject: [PATCH 22/23] feat(core): expose IpfsNode.Resolve --- core/core.go | 5 +++++ core/io/cat.go | 6 ++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/core/core.go b/core/core.go index c6ab76ab7..b5b95cf0f 100644 --- a/core/core.go +++ b/core/core.go @@ -29,6 +29,7 @@ import ( pin "github.com/jbenet/go-ipfs/pin" routing "github.com/jbenet/go-ipfs/routing" dht "github.com/jbenet/go-ipfs/routing/dht" + util "github.com/jbenet/go-ipfs/util" ds2 "github.com/jbenet/go-ipfs/util/datastore2" debugerror "github.com/jbenet/go-ipfs/util/debugerror" eventlog "github.com/jbenet/go-ipfs/util/eventlog" @@ -264,6 +265,10 @@ func (n *IpfsNode) OnlineMode() bool { } } +func (n *IpfsNode) Resolve(k util.Key) (*merkledag.Node, error) { + return (&path.Resolver{n.DAG}).ResolvePath(k.String()) +} + func (n *IpfsNode) Bootstrap(ctx context.Context, peers []peer.PeerInfo) error { if n.DHT != nil { for _, p := range peers { diff --git a/core/io/cat.go b/core/io/cat.go index 29dacf9d6..0e0f2712a 100644 --- a/core/io/cat.go +++ b/core/io/cat.go @@ -15,16 +15,14 @@ import ( "io" core "github.com/jbenet/go-ipfs/core" - path "github.com/jbenet/go-ipfs/path" uio "github.com/jbenet/go-ipfs/unixfs/io" u "github.com/jbenet/go-ipfs/util" ) func Cat(n *core.IpfsNode, k u.Key) (io.Reader, error) { - dag := n.DAG - dagNode, err := (&path.Resolver{dag}).ResolvePath(k.String()) + dagNode, err := n.Resolve(k) if err != nil { return nil, err } - return uio.NewDagReader(dagNode, dag) + return uio.NewDagReader(dagNode, n.DAG) } From 63c0d41614a56cdc88a0b4a1ebf94c3b5970df67 Mon Sep 17 00:00:00 2001 From: Brian Tiger Chow Date: Sat, 10 Jan 2015 23:23:39 -0800 Subject: [PATCH 23/23] fix(core): perform sophisticated boostrap operation --- core/core.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/core/core.go b/core/core.go index b5b95cf0f..9408d76ff 100644 --- a/core/core.go +++ b/core/core.go @@ -269,14 +269,13 @@ func (n *IpfsNode) Resolve(k util.Key) (*merkledag.Node, error) { return (&path.Resolver{n.DAG}).ResolvePath(k.String()) } +// Bootstrap is undefined when node is not in OnlineMode func (n *IpfsNode) Bootstrap(ctx context.Context, peers []peer.PeerInfo) error { + + // TODO what should return value be when in offlineMode? + if n.DHT != nil { - for _, p := range peers { - // TODO bootstrap(ctx, n.PeerHost, n.DHT, n.Peerstore, peers) - if err := n.DHT.Connect(ctx, p.ID); err != nil { - return err - } - } + return bootstrap(ctx, n.PeerHost, n.DHT, n.Peerstore, peers) } return nil }