diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index ffd4f1ceb..61c05dd9a 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -21,13 +21,13 @@ }, { "ImportPath": "code.google.com/p/go.crypto/blowfish", - "Comment": "null-219", - "Rev": "00a7d3b31bbab5795b4a51933c04fc2768242970" + "Comment": "null-236", + "Rev": "69e2a90ed92d03812364aeb947b7068dc42e561e" }, { "ImportPath": "code.google.com/p/go.crypto/sha3", - "Comment": "null-219", - "Rev": "00a7d3b31bbab5795b4a51933c04fc2768242970" + "Comment": "null-236", + "Rev": "69e2a90ed92d03812364aeb947b7068dc42e561e" }, { "ImportPath": "code.google.com/p/go.net/context", @@ -118,7 +118,7 @@ }, { "ImportPath": "github.com/jbenet/go-msgio", - "Rev": "5e7289d3a0cd046a5bee30b187cc844c31f54dce" + "Rev": "dbae89193876910c736b2ce1291fa8bbcf299d77" }, { "ImportPath": "github.com/jbenet/go-multiaddr", @@ -131,8 +131,8 @@ }, { "ImportPath": "github.com/jbenet/go-multihash", - "Comment": "0.1.0-5-g1976046", - "Rev": "1976046c2b0db0b668791b3e541d76a38b7c1af7" + "Comment": "0.1.0-19-g8ce5cb1", + "Rev": "8ce5cb1b82e1b4c1bea1fdf3cd467ef49301734e" }, { "ImportPath": "github.com/jbenet/go-peerstream", diff --git a/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/doc.go b/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/doc.go new file mode 100644 index 000000000..0ba123b1f --- /dev/null +++ b/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/doc.go @@ -0,0 +1,68 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package sha3 implements the SHA-3 fixed-output-length hash functions and +// the SHAKE variable-output-length hash functions defined by FIPS-202. +// +// Both types of hash function use the "sponge" construction and the Keccak +// permutation. For a detailed specification see http://keccak.noekeon.org/ +// +// +// Guidance +// +// If you aren't sure what function you need, use SHAKE256 with at least 64 +// bytes of output. +// +// If you need a secret-key MAC (message authentication code), prepend the +// secret key to the input, hash with SHAKE256 and read at least 32 bytes of +// output. +// +// +// Security strengths +// +// The SHA3-x functions have a security strength against preimage attacks of x +// bits. Since they only produce x bits of output, their collision-resistance +// is only x/2 bits. +// +// The SHAKE-x functions have a generic security strength of x bits against +// all attacks, provided that at least 2x bits of their output is used. +// Requesting more than 2x bits of output does not increase the collision- +// resistance of the SHAKE functions. +// +// +// The sponge construction +// +// A sponge builds a pseudo-random function from a pseudo-random permutation, +// by applying the permutation to a state of "rate + capacity" bytes, but +// hiding "capacity" of the bytes. +// +// A sponge starts out with a zero state. To hash an input using a sponge, up +// to "rate" bytes of the input are XORed into the sponge's state. The sponge +// has thus been "filled up" and the permutation is applied. This process is +// repeated until all the input has been "absorbed". The input is then padded. +// The digest is "squeezed" from the sponge by the same method, except that +// output is copied out. +// +// A sponge is parameterized by its generic security strength, which is equal +// to half its capacity; capacity + rate is equal to the permutation's width. +// +// Since the KeccakF-1600 permutation is 1600 bits (200 bytes) wide, this means +// that security_strength == (1600 - bitrate) / 2. +// +// +// Recommendations, detailed +// +// The SHAKE functions are recommended for most new uses. They can produce +// output of arbitrary length. SHAKE256, with an output length of at least +// 64 bytes, provides 256-bit security against all attacks. +// +// The Keccak team recommends SHAKE256 for most applications upgrading from +// SHA2-512. (NIST chose a much stronger, but much slower, sponge instance +// for SHA3-512.) +// +// The SHA-3 functions are "drop-in" replacements for the SHA-2 functions. +// They produce output of the same length, with the same security strengths +// against all attacks. This means, in particular, that SHA3-256 only has +// 128-bit collision resistance, because its output length is 32 bytes. +package sha3 diff --git a/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/hashes.go b/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/hashes.go new file mode 100644 index 000000000..2b51cf4e9 --- /dev/null +++ b/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/hashes.go @@ -0,0 +1,65 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sha3 + +// This file provides functions for creating instances of the SHA-3 +// and SHAKE hash functions, as well as utility functions for hashing +// bytes. + +import ( + "hash" +) + +// New224 creates a new SHA3-224 hash. +// Its generic security strength is 224 bits against preimage attacks, +// and 112 bits against collision attacks. +func New224() hash.Hash { return &state{rate: 144, outputLen: 28, dsbyte: 0x06} } + +// New256 creates a new SHA3-256 hash. +// Its generic security strength is 256 bits against preimage attacks, +// and 128 bits against collision attacks. +func New256() hash.Hash { return &state{rate: 136, outputLen: 32, dsbyte: 0x06} } + +// New384 creates a new SHA3-384 hash. +// Its generic security strength is 384 bits against preimage attacks, +// and 192 bits against collision attacks. +func New384() hash.Hash { return &state{rate: 104, outputLen: 48, dsbyte: 0x06} } + +// New512 creates a new SHA3-512 hash. +// Its generic security strength is 512 bits against preimage attacks, +// and 256 bits against collision attacks. +func New512() hash.Hash { return &state{rate: 72, outputLen: 64, dsbyte: 0x06} } + +// Sum224 returns the SHA3-224 digest of the data. +func Sum224(data []byte) (digest [28]byte) { + h := New224() + h.Write(data) + h.Sum(digest[:0]) + return +} + +// Sum256 returns the SHA3-256 digest of the data. +func Sum256(data []byte) (digest [32]byte) { + h := New256() + h.Write(data) + h.Sum(digest[:0]) + return +} + +// Sum384 returns the SHA3-384 digest of the data. +func Sum384(data []byte) (digest [48]byte) { + h := New384() + h.Write(data) + h.Sum(digest[:0]) + return +} + +// Sum512 returns the SHA3-512 digest of the data. +func Sum512(data []byte) (digest [64]byte) { + h := New512() + h.Write(data) + h.Sum(digest[:0]) + return +} diff --git a/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/keccakKats.json.deflate b/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/keccakKats.json.deflate new file mode 100644 index 000000000..62e85ae24 Binary files /dev/null and b/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/keccakKats.json.deflate differ diff --git a/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/keccakf.go b/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/keccakf.go index 76c0312a0..13e7058fa 100644 --- a/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/keccakf.go +++ b/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/keccakf.go @@ -1,16 +1,11 @@ -// Copyright 2013 The Go Authors. All rights reserved. +// Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package sha3 -// This file implements the core Keccak permutation function necessary for computing SHA3. -// This is implemented in a separate file to allow for replacement by an optimized implementation. -// Nothing in this package is exported. -// For the detailed specification, refer to the Keccak web site (http://keccak.noekeon.org/). - // rc stores the round constants for use in the ι step. -var rc = [...]uint64{ +var rc = [24]uint64{ 0x0000000000000001, 0x0000000000008082, 0x800000000000808A, @@ -37,129 +32,379 @@ var rc = [...]uint64{ 0x8000000080008008, } -// keccakF computes the complete Keccak-f function consisting of 24 rounds with a different -// constant (rc) in each round. This implementation fully unrolls the round function to avoid -// inner loops, as well as pre-calculating shift offsets. -func keccakF(a *[numLanes]uint64) { - var t, bc0, bc1, bc2, bc3, bc4 uint64 - for _, roundConstant := range rc { - // θ step +// keccakF1600 applies the Keccak permutation to a 1600b-wide +// state represented as a slice of 25 uint64s. +func keccakF1600(a *[25]uint64) { + // Implementation translated from Keccak-inplace.c + // in the keccak reference code. + var t, bc0, bc1, bc2, bc3, bc4, d0, d1, d2, d3, d4 uint64 + + for i := 0; i < 24; i += 4 { + // Combines the 5 steps in each round into 2 steps. + // Unrolls 4 rounds per loop and spreads some steps across rounds. + + // Round 1 bc0 = a[0] ^ a[5] ^ a[10] ^ a[15] ^ a[20] bc1 = a[1] ^ a[6] ^ a[11] ^ a[16] ^ a[21] bc2 = a[2] ^ a[7] ^ a[12] ^ a[17] ^ a[22] bc3 = a[3] ^ a[8] ^ a[13] ^ a[18] ^ a[23] bc4 = a[4] ^ a[9] ^ a[14] ^ a[19] ^ a[24] - t = bc4 ^ (bc1<<1 ^ bc1>>63) - a[0] ^= t - a[5] ^= t - a[10] ^= t - a[15] ^= t - a[20] ^= t - t = bc0 ^ (bc2<<1 ^ bc2>>63) - a[1] ^= t - a[6] ^= t - a[11] ^= t - a[16] ^= t - a[21] ^= t - t = bc1 ^ (bc3<<1 ^ bc3>>63) - a[2] ^= t - a[7] ^= t - a[12] ^= t - a[17] ^= t - a[22] ^= t - t = bc2 ^ (bc4<<1 ^ bc4>>63) - a[3] ^= t - a[8] ^= t - a[13] ^= t - a[18] ^= t - a[23] ^= t - t = bc3 ^ (bc0<<1 ^ bc0>>63) - a[4] ^= t - a[9] ^= t - a[14] ^= t - a[19] ^= t - a[24] ^= t + d0 = bc4 ^ (bc1<<1 | bc1>>63) + d1 = bc0 ^ (bc2<<1 | bc2>>63) + d2 = bc1 ^ (bc3<<1 | bc3>>63) + d3 = bc2 ^ (bc4<<1 | bc4>>63) + d4 = bc3 ^ (bc0<<1 | bc0>>63) - // ρ and π steps - t = a[1] - t, a[10] = a[10], t<<1^t>>(64-1) - t, a[7] = a[7], t<<3^t>>(64-3) - t, a[11] = a[11], t<<6^t>>(64-6) - t, a[17] = a[17], t<<10^t>>(64-10) - t, a[18] = a[18], t<<15^t>>(64-15) - t, a[3] = a[3], t<<21^t>>(64-21) - t, a[5] = a[5], t<<28^t>>(64-28) - t, a[16] = a[16], t<<36^t>>(64-36) - t, a[8] = a[8], t<<45^t>>(64-45) - t, a[21] = a[21], t<<55^t>>(64-55) - t, a[24] = a[24], t<<2^t>>(64-2) - t, a[4] = a[4], t<<14^t>>(64-14) - t, a[15] = a[15], t<<27^t>>(64-27) - t, a[23] = a[23], t<<41^t>>(64-41) - t, a[19] = a[19], t<<56^t>>(64-56) - t, a[13] = a[13], t<<8^t>>(64-8) - t, a[12] = a[12], t<<25^t>>(64-25) - t, a[2] = a[2], t<<43^t>>(64-43) - t, a[20] = a[20], t<<62^t>>(64-62) - t, a[14] = a[14], t<<18^t>>(64-18) - t, a[22] = a[22], t<<39^t>>(64-39) - t, a[9] = a[9], t<<61^t>>(64-61) - t, a[6] = a[6], t<<20^t>>(64-20) - a[1] = t<<44 ^ t>>(64-44) + bc0 = a[0] ^ d0 + t = a[6] ^ d1 + bc1 = t<<44 | t>>(64-44) + t = a[12] ^ d2 + bc2 = t<<43 | t>>(64-43) + t = a[18] ^ d3 + bc3 = t<<21 | t>>(64-21) + t = a[24] ^ d4 + bc4 = t<<14 | t>>(64-14) + a[0] = bc0 ^ (bc2 &^ bc1) ^ rc[i] + a[6] = bc1 ^ (bc3 &^ bc2) + a[12] = bc2 ^ (bc4 &^ bc3) + a[18] = bc3 ^ (bc0 &^ bc4) + a[24] = bc4 ^ (bc1 &^ bc0) - // χ step - bc0 = a[0] - bc1 = a[1] - bc2 = a[2] - bc3 = a[3] - bc4 = a[4] - a[0] ^= bc2 &^ bc1 - a[1] ^= bc3 &^ bc2 - a[2] ^= bc4 &^ bc3 - a[3] ^= bc0 &^ bc4 - a[4] ^= bc1 &^ bc0 - bc0 = a[5] - bc1 = a[6] - bc2 = a[7] - bc3 = a[8] - bc4 = a[9] - a[5] ^= bc2 &^ bc1 - a[6] ^= bc3 &^ bc2 - a[7] ^= bc4 &^ bc3 - a[8] ^= bc0 &^ bc4 - a[9] ^= bc1 &^ bc0 - bc0 = a[10] - bc1 = a[11] - bc2 = a[12] - bc3 = a[13] - bc4 = a[14] - a[10] ^= bc2 &^ bc1 - a[11] ^= bc3 &^ bc2 - a[12] ^= bc4 &^ bc3 - a[13] ^= bc0 &^ bc4 - a[14] ^= bc1 &^ bc0 - bc0 = a[15] - bc1 = a[16] - bc2 = a[17] - bc3 = a[18] - bc4 = a[19] - a[15] ^= bc2 &^ bc1 - a[16] ^= bc3 &^ bc2 - a[17] ^= bc4 &^ bc3 - a[18] ^= bc0 &^ bc4 - a[19] ^= bc1 &^ bc0 - bc0 = a[20] - bc1 = a[21] - bc2 = a[22] - bc3 = a[23] - bc4 = a[24] - a[20] ^= bc2 &^ bc1 - a[21] ^= bc3 &^ bc2 - a[22] ^= bc4 &^ bc3 - a[23] ^= bc0 &^ bc4 - a[24] ^= bc1 &^ bc0 + t = a[10] ^ d0 + bc2 = t<<3 | t>>(64-3) + t = a[16] ^ d1 + bc3 = t<<45 | t>>(64-45) + t = a[22] ^ d2 + bc4 = t<<61 | t>>(64-61) + t = a[3] ^ d3 + bc0 = t<<28 | t>>(64-28) + t = a[9] ^ d4 + bc1 = t<<20 | t>>(64-20) + a[10] = bc0 ^ (bc2 &^ bc1) + a[16] = bc1 ^ (bc3 &^ bc2) + a[22] = bc2 ^ (bc4 &^ bc3) + a[3] = bc3 ^ (bc0 &^ bc4) + a[9] = bc4 ^ (bc1 &^ bc0) - // ι step - a[0] ^= roundConstant + t = a[20] ^ d0 + bc4 = t<<18 | t>>(64-18) + t = a[1] ^ d1 + bc0 = t<<1 | t>>(64-1) + t = a[7] ^ d2 + bc1 = t<<6 | t>>(64-6) + t = a[13] ^ d3 + bc2 = t<<25 | t>>(64-25) + t = a[19] ^ d4 + bc3 = t<<8 | t>>(64-8) + a[20] = bc0 ^ (bc2 &^ bc1) + a[1] = bc1 ^ (bc3 &^ bc2) + a[7] = bc2 ^ (bc4 &^ bc3) + a[13] = bc3 ^ (bc0 &^ bc4) + a[19] = bc4 ^ (bc1 &^ bc0) + + t = a[5] ^ d0 + bc1 = t<<36 | t>>(64-36) + t = a[11] ^ d1 + bc2 = t<<10 | t>>(64-10) + t = a[17] ^ d2 + bc3 = t<<15 | t>>(64-15) + t = a[23] ^ d3 + bc4 = t<<56 | t>>(64-56) + t = a[4] ^ d4 + bc0 = t<<27 | t>>(64-27) + a[5] = bc0 ^ (bc2 &^ bc1) + a[11] = bc1 ^ (bc3 &^ bc2) + a[17] = bc2 ^ (bc4 &^ bc3) + a[23] = bc3 ^ (bc0 &^ bc4) + a[4] = bc4 ^ (bc1 &^ bc0) + + t = a[15] ^ d0 + bc3 = t<<41 | t>>(64-41) + t = a[21] ^ d1 + bc4 = t<<2 | t>>(64-2) + t = a[2] ^ d2 + bc0 = t<<62 | t>>(64-62) + t = a[8] ^ d3 + bc1 = t<<55 | t>>(64-55) + t = a[14] ^ d4 + bc2 = t<<39 | t>>(64-39) + a[15] = bc0 ^ (bc2 &^ bc1) + a[21] = bc1 ^ (bc3 &^ bc2) + a[2] = bc2 ^ (bc4 &^ bc3) + a[8] = bc3 ^ (bc0 &^ bc4) + a[14] = bc4 ^ (bc1 &^ bc0) + + // Round 2 + bc0 = a[0] ^ a[5] ^ a[10] ^ a[15] ^ a[20] + bc1 = a[1] ^ a[6] ^ a[11] ^ a[16] ^ a[21] + bc2 = a[2] ^ a[7] ^ a[12] ^ a[17] ^ a[22] + bc3 = a[3] ^ a[8] ^ a[13] ^ a[18] ^ a[23] + bc4 = a[4] ^ a[9] ^ a[14] ^ a[19] ^ a[24] + d0 = bc4 ^ (bc1<<1 | bc1>>63) + d1 = bc0 ^ (bc2<<1 | bc2>>63) + d2 = bc1 ^ (bc3<<1 | bc3>>63) + d3 = bc2 ^ (bc4<<1 | bc4>>63) + d4 = bc3 ^ (bc0<<1 | bc0>>63) + + bc0 = a[0] ^ d0 + t = a[16] ^ d1 + bc1 = t<<44 | t>>(64-44) + t = a[7] ^ d2 + bc2 = t<<43 | t>>(64-43) + t = a[23] ^ d3 + bc3 = t<<21 | t>>(64-21) + t = a[14] ^ d4 + bc4 = t<<14 | t>>(64-14) + a[0] = bc0 ^ (bc2 &^ bc1) ^ rc[i+1] + a[16] = bc1 ^ (bc3 &^ bc2) + a[7] = bc2 ^ (bc4 &^ bc3) + a[23] = bc3 ^ (bc0 &^ bc4) + a[14] = bc4 ^ (bc1 &^ bc0) + + t = a[20] ^ d0 + bc2 = t<<3 | t>>(64-3) + t = a[11] ^ d1 + bc3 = t<<45 | t>>(64-45) + t = a[2] ^ d2 + bc4 = t<<61 | t>>(64-61) + t = a[18] ^ d3 + bc0 = t<<28 | t>>(64-28) + t = a[9] ^ d4 + bc1 = t<<20 | t>>(64-20) + a[20] = bc0 ^ (bc2 &^ bc1) + a[11] = bc1 ^ (bc3 &^ bc2) + a[2] = bc2 ^ (bc4 &^ bc3) + a[18] = bc3 ^ (bc0 &^ bc4) + a[9] = bc4 ^ (bc1 &^ bc0) + + t = a[15] ^ d0 + bc4 = t<<18 | t>>(64-18) + t = a[6] ^ d1 + bc0 = t<<1 | t>>(64-1) + t = a[22] ^ d2 + bc1 = t<<6 | t>>(64-6) + t = a[13] ^ d3 + bc2 = t<<25 | t>>(64-25) + t = a[4] ^ d4 + bc3 = t<<8 | t>>(64-8) + a[15] = bc0 ^ (bc2 &^ bc1) + a[6] = bc1 ^ (bc3 &^ bc2) + a[22] = bc2 ^ (bc4 &^ bc3) + a[13] = bc3 ^ (bc0 &^ bc4) + a[4] = bc4 ^ (bc1 &^ bc0) + + t = a[10] ^ d0 + bc1 = t<<36 | t>>(64-36) + t = a[1] ^ d1 + bc2 = t<<10 | t>>(64-10) + t = a[17] ^ d2 + bc3 = t<<15 | t>>(64-15) + t = a[8] ^ d3 + bc4 = t<<56 | t>>(64-56) + t = a[24] ^ d4 + bc0 = t<<27 | t>>(64-27) + a[10] = bc0 ^ (bc2 &^ bc1) + a[1] = bc1 ^ (bc3 &^ bc2) + a[17] = bc2 ^ (bc4 &^ bc3) + a[8] = bc3 ^ (bc0 &^ bc4) + a[24] = bc4 ^ (bc1 &^ bc0) + + t = a[5] ^ d0 + bc3 = t<<41 | t>>(64-41) + t = a[21] ^ d1 + bc4 = t<<2 | t>>(64-2) + t = a[12] ^ d2 + bc0 = t<<62 | t>>(64-62) + t = a[3] ^ d3 + bc1 = t<<55 | t>>(64-55) + t = a[19] ^ d4 + bc2 = t<<39 | t>>(64-39) + a[5] = bc0 ^ (bc2 &^ bc1) + a[21] = bc1 ^ (bc3 &^ bc2) + a[12] = bc2 ^ (bc4 &^ bc3) + a[3] = bc3 ^ (bc0 &^ bc4) + a[19] = bc4 ^ (bc1 &^ bc0) + + // Round 3 + bc0 = a[0] ^ a[5] ^ a[10] ^ a[15] ^ a[20] + bc1 = a[1] ^ a[6] ^ a[11] ^ a[16] ^ a[21] + bc2 = a[2] ^ a[7] ^ a[12] ^ a[17] ^ a[22] + bc3 = a[3] ^ a[8] ^ a[13] ^ a[18] ^ a[23] + bc4 = a[4] ^ a[9] ^ a[14] ^ a[19] ^ a[24] + d0 = bc4 ^ (bc1<<1 | bc1>>63) + d1 = bc0 ^ (bc2<<1 | bc2>>63) + d2 = bc1 ^ (bc3<<1 | bc3>>63) + d3 = bc2 ^ (bc4<<1 | bc4>>63) + d4 = bc3 ^ (bc0<<1 | bc0>>63) + + bc0 = a[0] ^ d0 + t = a[11] ^ d1 + bc1 = t<<44 | t>>(64-44) + t = a[22] ^ d2 + bc2 = t<<43 | t>>(64-43) + t = a[8] ^ d3 + bc3 = t<<21 | t>>(64-21) + t = a[19] ^ d4 + bc4 = t<<14 | t>>(64-14) + a[0] = bc0 ^ (bc2 &^ bc1) ^ rc[i+2] + a[11] = bc1 ^ (bc3 &^ bc2) + a[22] = bc2 ^ (bc4 &^ bc3) + a[8] = bc3 ^ (bc0 &^ bc4) + a[19] = bc4 ^ (bc1 &^ bc0) + + t = a[15] ^ d0 + bc2 = t<<3 | t>>(64-3) + t = a[1] ^ d1 + bc3 = t<<45 | t>>(64-45) + t = a[12] ^ d2 + bc4 = t<<61 | t>>(64-61) + t = a[23] ^ d3 + bc0 = t<<28 | t>>(64-28) + t = a[9] ^ d4 + bc1 = t<<20 | t>>(64-20) + a[15] = bc0 ^ (bc2 &^ bc1) + a[1] = bc1 ^ (bc3 &^ bc2) + a[12] = bc2 ^ (bc4 &^ bc3) + a[23] = bc3 ^ (bc0 &^ bc4) + a[9] = bc4 ^ (bc1 &^ bc0) + + t = a[5] ^ d0 + bc4 = t<<18 | t>>(64-18) + t = a[16] ^ d1 + bc0 = t<<1 | t>>(64-1) + t = a[2] ^ d2 + bc1 = t<<6 | t>>(64-6) + t = a[13] ^ d3 + bc2 = t<<25 | t>>(64-25) + t = a[24] ^ d4 + bc3 = t<<8 | t>>(64-8) + a[5] = bc0 ^ (bc2 &^ bc1) + a[16] = bc1 ^ (bc3 &^ bc2) + a[2] = bc2 ^ (bc4 &^ bc3) + a[13] = bc3 ^ (bc0 &^ bc4) + a[24] = bc4 ^ (bc1 &^ bc0) + + t = a[20] ^ d0 + bc1 = t<<36 | t>>(64-36) + t = a[6] ^ d1 + bc2 = t<<10 | t>>(64-10) + t = a[17] ^ d2 + bc3 = t<<15 | t>>(64-15) + t = a[3] ^ d3 + bc4 = t<<56 | t>>(64-56) + t = a[14] ^ d4 + bc0 = t<<27 | t>>(64-27) + a[20] = bc0 ^ (bc2 &^ bc1) + a[6] = bc1 ^ (bc3 &^ bc2) + a[17] = bc2 ^ (bc4 &^ bc3) + a[3] = bc3 ^ (bc0 &^ bc4) + a[14] = bc4 ^ (bc1 &^ bc0) + + t = a[10] ^ d0 + bc3 = t<<41 | t>>(64-41) + t = a[21] ^ d1 + bc4 = t<<2 | t>>(64-2) + t = a[7] ^ d2 + bc0 = t<<62 | t>>(64-62) + t = a[18] ^ d3 + bc1 = t<<55 | t>>(64-55) + t = a[4] ^ d4 + bc2 = t<<39 | t>>(64-39) + a[10] = bc0 ^ (bc2 &^ bc1) + a[21] = bc1 ^ (bc3 &^ bc2) + a[7] = bc2 ^ (bc4 &^ bc3) + a[18] = bc3 ^ (bc0 &^ bc4) + a[4] = bc4 ^ (bc1 &^ bc0) + + // Round 4 + bc0 = a[0] ^ a[5] ^ a[10] ^ a[15] ^ a[20] + bc1 = a[1] ^ a[6] ^ a[11] ^ a[16] ^ a[21] + bc2 = a[2] ^ a[7] ^ a[12] ^ a[17] ^ a[22] + bc3 = a[3] ^ a[8] ^ a[13] ^ a[18] ^ a[23] + bc4 = a[4] ^ a[9] ^ a[14] ^ a[19] ^ a[24] + d0 = bc4 ^ (bc1<<1 | bc1>>63) + d1 = bc0 ^ (bc2<<1 | bc2>>63) + d2 = bc1 ^ (bc3<<1 | bc3>>63) + d3 = bc2 ^ (bc4<<1 | bc4>>63) + d4 = bc3 ^ (bc0<<1 | bc0>>63) + + bc0 = a[0] ^ d0 + t = a[1] ^ d1 + bc1 = t<<44 | t>>(64-44) + t = a[2] ^ d2 + bc2 = t<<43 | t>>(64-43) + t = a[3] ^ d3 + bc3 = t<<21 | t>>(64-21) + t = a[4] ^ d4 + bc4 = t<<14 | t>>(64-14) + a[0] = bc0 ^ (bc2 &^ bc1) ^ rc[i+3] + a[1] = bc1 ^ (bc3 &^ bc2) + a[2] = bc2 ^ (bc4 &^ bc3) + a[3] = bc3 ^ (bc0 &^ bc4) + a[4] = bc4 ^ (bc1 &^ bc0) + + t = a[5] ^ d0 + bc2 = t<<3 | t>>(64-3) + t = a[6] ^ d1 + bc3 = t<<45 | t>>(64-45) + t = a[7] ^ d2 + bc4 = t<<61 | t>>(64-61) + t = a[8] ^ d3 + bc0 = t<<28 | t>>(64-28) + t = a[9] ^ d4 + bc1 = t<<20 | t>>(64-20) + a[5] = bc0 ^ (bc2 &^ bc1) + a[6] = bc1 ^ (bc3 &^ bc2) + a[7] = bc2 ^ (bc4 &^ bc3) + a[8] = bc3 ^ (bc0 &^ bc4) + a[9] = bc4 ^ (bc1 &^ bc0) + + t = a[10] ^ d0 + bc4 = t<<18 | t>>(64-18) + t = a[11] ^ d1 + bc0 = t<<1 | t>>(64-1) + t = a[12] ^ d2 + bc1 = t<<6 | t>>(64-6) + t = a[13] ^ d3 + bc2 = t<<25 | t>>(64-25) + t = a[14] ^ d4 + bc3 = t<<8 | t>>(64-8) + a[10] = bc0 ^ (bc2 &^ bc1) + a[11] = bc1 ^ (bc3 &^ bc2) + a[12] = bc2 ^ (bc4 &^ bc3) + a[13] = bc3 ^ (bc0 &^ bc4) + a[14] = bc4 ^ (bc1 &^ bc0) + + t = a[15] ^ d0 + bc1 = t<<36 | t>>(64-36) + t = a[16] ^ d1 + bc2 = t<<10 | t>>(64-10) + t = a[17] ^ d2 + bc3 = t<<15 | t>>(64-15) + t = a[18] ^ d3 + bc4 = t<<56 | t>>(64-56) + t = a[19] ^ d4 + bc0 = t<<27 | t>>(64-27) + a[15] = bc0 ^ (bc2 &^ bc1) + a[16] = bc1 ^ (bc3 &^ bc2) + a[17] = bc2 ^ (bc4 &^ bc3) + a[18] = bc3 ^ (bc0 &^ bc4) + a[19] = bc4 ^ (bc1 &^ bc0) + + t = a[20] ^ d0 + bc3 = t<<41 | t>>(64-41) + t = a[21] ^ d1 + bc4 = t<<2 | t>>(64-2) + t = a[22] ^ d2 + bc0 = t<<62 | t>>(64-62) + t = a[23] ^ d3 + bc1 = t<<55 | t>>(64-55) + t = a[24] ^ d4 + bc2 = t<<39 | t>>(64-39) + a[20] = bc0 ^ (bc2 &^ bc1) + a[21] = bc1 ^ (bc3 &^ bc2) + a[22] = bc2 ^ (bc4 &^ bc3) + a[23] = bc3 ^ (bc0 &^ bc4) + a[24] = bc4 ^ (bc1 &^ bc0) } } diff --git a/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/register.go b/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/register.go new file mode 100644 index 000000000..3cf6a22e0 --- /dev/null +++ b/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/register.go @@ -0,0 +1,18 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.4 + +package sha3 + +import ( + "crypto" +) + +func init() { + crypto.RegisterHash(crypto.SHA3_224, New224) + crypto.RegisterHash(crypto.SHA3_256, New256) + crypto.RegisterHash(crypto.SHA3_384, New384) + crypto.RegisterHash(crypto.SHA3_512, New512) +} diff --git a/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/sha3.go b/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/sha3.go index e1f9aa85c..8d775684c 100644 --- a/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/sha3.go +++ b/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/sha3.go @@ -1,213 +1,226 @@ -// Copyright 2013 The Go Authors. All rights reserved. +// Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package sha3 implements the SHA3 hash algorithm (formerly called Keccak) chosen by NIST in 2012. -// This file provides a SHA3 implementation which implements the standard hash.Hash interface. -// Writing input data, including padding, and reading output data are computed in this file. -// Note that the current implementation can compute the hash of an integral number of bytes only. -// This is a consequence of the hash interface in which a buffer of bytes is passed in. -// The internals of the Keccak-f function are computed in keccakf.go. -// For the detailed specification, refer to the Keccak web site (http://keccak.noekeon.org/). package sha3 import ( "encoding/binary" - "hash" ) -// laneSize is the size in bytes of each "lane" of the internal state of SHA3 (5 * 5 * 8). -// Note that changing this size would requires using a type other than uint64 to store each lane. -const laneSize = 8 +// spongeDirection indicates the direction bytes are flowing through the sponge. +type spongeDirection int -// sliceSize represents the dimensions of the internal state, a square matrix of -// sliceSize ** 2 lanes. This is the size of both the "rows" and "columns" dimensions in the -// terminology of the SHA3 specification. -const sliceSize = 5 +const ( + // spongeAbsorbing indicates that the sponge is absorbing input. + spongeAbsorbing spongeDirection = iota + // spongeSqueezing indicates that the sponge is being squeezed. + spongeSqueezing +) -// numLanes represents the total number of lanes in the state. -const numLanes = sliceSize * sliceSize +const ( + // maxRate is the maximum size of the internal buffer. SHAKE-256 + // currently needs the largest buffer. + maxRate = 168 +) -// stateSize is the size in bytes of the internal state of SHA3 (5 * 5 * WSize). -const stateSize = laneSize * numLanes +type state struct { + // Generic sponge components. + a [25]uint64 // main state of the hash + buf []byte // points into storage + rate int // the number of bytes of state to use -// digest represents the partial evaluation of a checksum. -// Note that capacity, and not outputSize, is the critical security parameter, as SHA3 can output -// an arbitrary number of bytes for any given capacity. The Keccak proposal recommends that -// capacity = 2*outputSize to ensure that finding a collision of size outputSize requires -// O(2^{outputSize/2}) computations (the birthday lower bound). Future standards may modify the -// capacity/outputSize ratio to allow for more output with lower cryptographic security. -type digest struct { - a [numLanes]uint64 // main state of the hash - outputSize int // desired output size in bytes - capacity int // number of bytes to leave untouched during squeeze/absorb - absorbed int // number of bytes absorbed thus far + // dsbyte contains the "domain separation" value and the first bit of + // the padding. In sections 6.1 and 6.2 of [1], the SHA-3 and SHAKE + // functions are defined with bits appended to the message: SHA-3 + // functions have 01 and SHAKE functions have 1111. Because of the way + // that bits are numbered from the LSB upwards, that ends up as + // 00000010b and 00001111b, respectively. Then the padding rule from + // section 5.1 is applied to pad to a multiple of the rate, which + // involves adding a 1 bit, zero or more zero bits and then a final one + // bit. The first one bit from the padding is merged into the dsbyte + // value giving 00000110b (0x06) and 00011111b (0x1f), respectively. + // + // [1] http://csrc.nist.gov/publications/drafts/fips-202/fips_202_draft.pdf, + dsbyte byte + storage [maxRate]byte + + // Specific to SHA-3 and SHAKE. + fixedOutput bool // whether this is a fixed-ouput-length instance + outputLen int // the default output size in bytes + state spongeDirection // current direction of the sponge } -// minInt returns the lesser of two integer arguments, to simplify the absorption routine. -func minInt(v1, v2 int) int { - if v1 <= v2 { - return v1 - } - return v2 -} +// BlockSize returns the rate of sponge underlying this hash function. +func (d *state) BlockSize() int { return d.rate } -// rate returns the number of bytes of the internal state which can be absorbed or squeezed -// in between calls to the permutation function. -func (d *digest) rate() int { - return stateSize - d.capacity -} +// Size returns the output size of the hash function in bytes. +func (d *state) Size() int { return d.outputLen } -// Reset clears the internal state by zeroing bytes in the state buffer. -// This can be skipped for a newly-created hash state; the default zero-allocated state is correct. -func (d *digest) Reset() { - d.absorbed = 0 +// Reset clears the internal state by zeroing the sponge state and +// the byte buffer, and setting Sponge.state to absorbing. +func (d *state) Reset() { + // Zero the permutation's state. for i := range d.a { d.a[i] = 0 } + d.state = spongeAbsorbing + d.buf = d.storage[:0] } -// BlockSize, required by the hash.Hash interface, does not have a standard intepretation -// for a sponge-based construction like SHA3. We return the data rate: the number of bytes which -// can be absorbed per invocation of the permutation function. For Merkle-Damgård based hashes -// (ie SHA1, SHA2, MD5) the output size of the internal compression function is returned. -// We consider this to be roughly equivalent because it represents the number of bytes of output -// produced per cryptographic operation. -func (d *digest) BlockSize() int { return d.rate() } - -// Size returns the output size of the hash function in bytes. -func (d *digest) Size() int { - return d.outputSize -} - -// unalignedAbsorb is a helper function for Write, which absorbs data that isn't aligned with an -// 8-byte lane. This requires shifting the individual bytes into position in a uint64. -func (d *digest) unalignedAbsorb(p []byte) { - var t uint64 - for i := len(p) - 1; i >= 0; i-- { - t <<= 8 - t |= uint64(p[i]) +func (d *state) clone() *state { + ret := *d + if ret.state == spongeAbsorbing { + ret.buf = ret.storage[:len(ret.buf)] + } else { + ret.buf = ret.storage[d.rate-cap(d.buf) : d.rate] } - offset := (d.absorbed) % d.rate() - t <<= 8 * uint(offset%laneSize) - d.a[offset/laneSize] ^= t - d.absorbed += len(p) + + return &ret } -// Write "absorbs" bytes into the state of the SHA3 hash, updating as needed when the sponge -// "fills up" with rate() bytes. Since lanes are stored internally as type uint64, this requires -// converting the incoming bytes into uint64s using a little endian interpretation. This -// implementation is optimized for large, aligned writes of multiples of 8 bytes (laneSize). -// Non-aligned or uneven numbers of bytes require shifting and are slower. -func (d *digest) Write(p []byte) (int, error) { - // An initial offset is needed if the we aren't absorbing to the first lane initially. - offset := d.absorbed % d.rate() - toWrite := len(p) +// xorIn xors a buffer into the state, byte-swapping to +// little-endian as necessary; it returns the number of bytes +// copied, including any zeros appended to the bytestring. +func (d *state) xorIn(buf []byte) { + n := len(buf) / 8 - // The first lane may need to absorb unaligned and/or incomplete data. - if (offset%laneSize != 0 || len(p) < 8) && len(p) > 0 { - toAbsorb := minInt(laneSize-(offset%laneSize), len(p)) - d.unalignedAbsorb(p[:toAbsorb]) - p = p[toAbsorb:] - offset = (d.absorbed) % d.rate() + for i := 0; i < n; i++ { + a := binary.LittleEndian.Uint64(buf) + d.a[i] ^= a + buf = buf[8:] + } + if len(buf) != 0 { + // XOR in the last partial ulint64. + a := uint64(0) + for i, v := range buf { + a |= uint64(v) << uint64(8*i) + } + d.a[n] ^= a + } +} - // For every rate() bytes absorbed, the state must be permuted via the F Function. - if (d.absorbed)%d.rate() == 0 { - keccakF(&d.a) +// copyOut copies ulint64s to a byte buffer. +func (d *state) copyOut(b []byte) { + for i := 0; len(b) >= 8; i++ { + binary.LittleEndian.PutUint64(b, d.a[i]) + b = b[8:] + } +} + +// permute applies the KeccakF-1600 permutation. It handles +// any input-output buffering. +func (d *state) permute() { + switch d.state { + case spongeAbsorbing: + // If we're absorbing, we need to xor the input into the state + // before applying the permutation. + d.xorIn(d.buf) + d.buf = d.storage[:0] + keccakF1600(&d.a) + case spongeSqueezing: + // If we're squeezing, we need to apply the permutatin before + // copying more output. + keccakF1600(&d.a) + d.buf = d.storage[:d.rate] + d.copyOut(d.buf) + } +} + +// pads appends the domain separation bits in dsbyte, applies +// the multi-bitrate 10..1 padding rule, and permutes the state. +func (d *state) padAndPermute(dsbyte byte) { + if d.buf == nil { + d.buf = d.storage[:0] + } + // Pad with this instance's domain-separator bits. We know that there's + // at least one byte of space in d.buf because, if it were full, + // permute would have been called to empty it. dsbyte also contains the + // first one bit for the padding. See the comment in the state struct. + d.buf = append(d.buf, dsbyte) + zerosStart := len(d.buf) + d.buf = d.storage[:d.rate] + for i := zerosStart; i < d.rate; i++ { + d.buf[i] = 0 + } + // This adds the final one bit for the padding. Because of the way that + // bits are numbered from the LSB upwards, the final bit is the MSB of + // the last byte. + d.buf[d.rate-1] ^= 0x80 + // Apply the permutation + d.permute() + d.state = spongeSqueezing + d.buf = d.storage[:d.rate] + d.copyOut(d.buf) +} + +// Write absorbs more data into the hash's state. It produces an error +// if more data is written to the ShakeHash after writing +func (d *state) Write(p []byte) (written int, err error) { + if d.state != spongeAbsorbing { + panic("sha3: write to sponge after read") + } + if d.buf == nil { + d.buf = d.storage[:0] + } + written = len(p) + + for len(p) > 0 { + if len(d.buf) == 0 && len(p) >= d.rate { + // The fast path; absorb a full "rate" bytes of input and apply the permutation. + d.xorIn(p[:d.rate]) + p = p[d.rate:] + keccakF1600(&d.a) + } else { + // The slow path; buffer the input until we can fill the sponge, and then xor it in. + todo := d.rate - len(d.buf) + if todo > len(p) { + todo = len(p) + } + d.buf = append(d.buf, p[:todo]...) + p = p[todo:] + + // If the sponge is full, apply the permutation. + if len(d.buf) == d.rate { + d.permute() + } } } - // This loop should absorb the bulk of the data into full, aligned lanes. - // It will call the update function as necessary. - for len(p) > 7 { - firstLane := offset / laneSize - lastLane := minInt(d.rate()/laneSize, firstLane+len(p)/laneSize) - - // This inner loop absorbs input bytes into the state in groups of 8, converted to uint64s. - for lane := firstLane; lane < lastLane; lane++ { - d.a[lane] ^= binary.LittleEndian.Uint64(p[:laneSize]) - p = p[laneSize:] - } - d.absorbed += (lastLane - firstLane) * laneSize - // For every rate() bytes absorbed, the state must be permuted via the F Function. - if (d.absorbed)%d.rate() == 0 { - keccakF(&d.a) - } - - offset = 0 - } - - // If there are insufficient bytes to fill the final lane, an unaligned absorption. - // This should always start at a correct lane boundary though, or else it would be caught - // by the uneven opening lane case above. - if len(p) > 0 { - d.unalignedAbsorb(p) - } - - return toWrite, nil + return } -// pad computes the SHA3 padding scheme based on the number of bytes absorbed. -// The padding is a 1 bit, followed by an arbitrary number of 0s and then a final 1 bit, such that -// the input bits plus padding bits are a multiple of rate(). Adding the padding simply requires -// xoring an opening and closing bit into the appropriate lanes. -func (d *digest) pad() { - offset := d.absorbed % d.rate() - // The opening pad bit must be shifted into position based on the number of bytes absorbed - padOpenLane := offset / laneSize - d.a[padOpenLane] ^= 0x0000000000000001 << uint(8*(offset%laneSize)) - // The closing padding bit is always in the last position - padCloseLane := (d.rate() / laneSize) - 1 - d.a[padCloseLane] ^= 0x8000000000000000 -} - -// finalize prepares the hash to output data by padding and one final permutation of the state. -func (d *digest) finalize() { - d.pad() - keccakF(&d.a) -} - -// squeeze outputs an arbitrary number of bytes from the hash state. -// Squeezing can require multiple calls to the F function (one per rate() bytes squeezed), -// although this is not the case for standard SHA3 parameters. This implementation only supports -// squeezing a single time, subsequent squeezes may lose alignment. Future implementations -// may wish to support multiple squeeze calls, for example to support use as a PRNG. -func (d *digest) squeeze(in []byte, toSqueeze int) []byte { - // Because we read in blocks of laneSize, we need enough room to read - // an integral number of lanes - needed := toSqueeze + (laneSize-toSqueeze%laneSize)%laneSize - if cap(in)-len(in) < needed { - newIn := make([]byte, len(in), len(in)+needed) - copy(newIn, in) - in = newIn +// Read squeezes an arbitrary number of bytes from the sponge. +func (d *state) Read(out []byte) (n int, err error) { + // If we're still absorbing, pad and apply the permutation. + if d.state == spongeAbsorbing { + d.padAndPermute(d.dsbyte) } - out := in[len(in) : len(in)+needed] + n = len(out) + + // Now, do the squeezing. for len(out) > 0 { - for i := 0; i < d.rate() && len(out) > 0; i += laneSize { - binary.LittleEndian.PutUint64(out[:], d.a[i/laneSize]) - out = out[laneSize:] - } - if len(out) > 0 { - keccakF(&d.a) + n := copy(out, d.buf) + d.buf = d.buf[n:] + out = out[n:] + + // Apply the permutation if we've squeezed the sponge dry. + if len(d.buf) == 0 { + d.permute() } } - return in[:len(in)+toSqueeze] // Re-slice in case we wrote extra data. + + return } -// Sum applies padding to the hash state and then squeezes out the desired nubmer of output bytes. -func (d *digest) Sum(in []byte) []byte { - // Make a copy of the original hash so that caller can keep writing and summing. - dup := *d - dup.finalize() - return dup.squeeze(in, dup.outputSize) +// Sum applies padding to the hash state and then squeezes out the desired +// number of output bytes. +func (d *state) Sum(in []byte) []byte { + // Make a copy of the original hash so that caller can keep writing + // and summing. + dup := d.clone() + hash := make([]byte, dup.outputLen) + dup.Read(hash) + return append(in, hash...) } - -// The NewKeccakX constructors enable initializing a hash in any of the four recommend sizes -// from the Keccak specification, all of which set capacity=2*outputSize. Note that the final -// NIST standard for SHA3 may specify different input/output lengths. -// The output size is indicated in bits but converted into bytes internally. -func NewKeccak224() hash.Hash { return &digest{outputSize: 224 / 8, capacity: 2 * 224 / 8} } -func NewKeccak256() hash.Hash { return &digest{outputSize: 256 / 8, capacity: 2 * 256 / 8} } -func NewKeccak384() hash.Hash { return &digest{outputSize: 384 / 8, capacity: 2 * 384 / 8} } -func NewKeccak512() hash.Hash { return &digest{outputSize: 512 / 8, capacity: 2 * 512 / 8} } diff --git a/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/sha3_test.go b/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/sha3_test.go index 05e7c9589..6f84863ac 100644 --- a/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/sha3_test.go +++ b/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/sha3_test.go @@ -1,34 +1,56 @@ -// Copyright 2013 The Go Authors. All rights reserved. +// Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package sha3 -// These tests are a subset of those provided by the Keccak web site(http://keccak.noekeon.org/). +// Tests include all the ShortMsgKATs provided by the Keccak team at +// https://github.com/gvanas/KeccakCodePackage +// +// They only include the zero-bit case of the utterly useless bitwise +// testvectors published by NIST in the draft of FIPS-202. import ( "bytes" + "compress/flate" "encoding/hex" - "fmt" + "encoding/json" "hash" + "os" "strings" "testing" ) -// testDigests maintains a digest state of each standard type. -var testDigests = map[string]*digest{ - "Keccak224": {outputSize: 224 / 8, capacity: 2 * 224 / 8}, - "Keccak256": {outputSize: 256 / 8, capacity: 2 * 256 / 8}, - "Keccak384": {outputSize: 384 / 8, capacity: 2 * 384 / 8}, - "Keccak512": {outputSize: 512 / 8, capacity: 2 * 512 / 8}, +const ( + testString = "brekeccakkeccak koax koax" + katFilename = "keccakKats.json.deflate" +) + +// Internal-use instances of SHAKE used to test against KATs. +func newHashShake128() hash.Hash { + return &state{rate: 168, dsbyte: 0x1f, outputLen: 512} +} +func newHashShake256() hash.Hash { + return &state{rate: 136, dsbyte: 0x1f, outputLen: 512} } -// testVector represents a test input and expected outputs from multiple algorithm variants. -type testVector struct { - desc string - input []byte - repeat int // input will be concatenated the input this many times. - want map[string]string +// testDigests contains functions returning hash.Hash instances +// with output-length equal to the KAT length for both SHA-3 and +// SHAKE instances. +var testDigests = map[string]func() hash.Hash{ + "SHA3-224": New224, + "SHA3-256": New256, + "SHA3-384": New384, + "SHA3-512": New512, + "SHAKE128": newHashShake128, + "SHAKE256": newHashShake256, +} + +// testShakes contains functions returning ShakeHash instances for +// testing the ShakeHash-specific interface. +var testShakes = map[string]func() ShakeHash{ + "SHAKE128": NewShake128, + "SHAKE256": NewShake256, } // decodeHex converts an hex-encoded string into a raw byte string. @@ -40,102 +62,61 @@ func decodeHex(s string) []byte { return b } -// shortTestVectors stores a series of short testVectors. -// Inputs of 8, 248, and 264 bits from http://keccak.noekeon.org/ are included below. -// The standard defines additional test inputs of all sizes between 0 and 2047 bits. -// Because the current implementation can only handle an integral number of bytes, -// most of the standard test inputs can't be used. -var shortKeccakTestVectors = []testVector{ - { - desc: "short-8b", - input: decodeHex("CC"), - repeat: 1, - want: map[string]string{ - "Keccak224": "A9CAB59EB40A10B246290F2D6086E32E3689FAF1D26B470C899F2802", - "Keccak256": "EEAD6DBFC7340A56CAEDC044696A168870549A6A7F6F56961E84A54BD9970B8A", - "Keccak384": "1B84E62A46E5A201861754AF5DC95C4A1A69CAF4A796AE405680161E29572641F5FA1E8641D7958336EE7B11C58F73E9", - "Keccak512": "8630C13CBD066EA74BBE7FE468FEC1DEE10EDC1254FB4C1B7C5FD69B646E44160B8CE01D05A0908CA790DFB080F4B513BC3B6225ECE7A810371441A5AC666EB9", - }, - }, - { - desc: "short-248b", - input: decodeHex("84FB51B517DF6C5ACCB5D022F8F28DA09B10232D42320FFC32DBECC3835B29"), - repeat: 1, - want: map[string]string{ - "Keccak224": "81AF3A7A5BD4C1F948D6AF4B96F93C3B0CF9C0E7A6DA6FCD71EEC7F6", - "Keccak256": "D477FB02CAAA95B3280EC8EE882C29D9E8A654B21EF178E0F97571BF9D4D3C1C", - "Keccak384": "503DCAA4ADDA5A9420B2E436DD62D9AB2E0254295C2982EF67FCE40F117A2400AB492F7BD5D133C6EC2232268BC27B42", - "Keccak512": "9D8098D8D6EDBBAA2BCFC6FB2F89C3EAC67FEC25CDFE75AA7BD570A648E8C8945FF2EC280F6DCF73386109155C5BBC444C707BB42EAB873F5F7476657B1BC1A8", - }, - }, - { - desc: "short-264b", - input: decodeHex("DE8F1B3FAA4B7040ED4563C3B8E598253178E87E4D0DF75E4FF2F2DEDD5A0BE046"), - repeat: 1, - want: map[string]string{ - "Keccak224": "F217812E362EC64D4DC5EACFABC165184BFA456E5C32C2C7900253D0", - "Keccak256": "E78C421E6213AFF8DE1F025759A4F2C943DB62BBDE359C8737E19B3776ED2DD2", - "Keccak384": "CF38764973F1EC1C34B5433AE75A3AAD1AAEF6AB197850C56C8617BCD6A882F6666883AC17B2DCCDBAA647075D0972B5", - "Keccak512": "9A7688E31AAF40C15575FC58C6B39267AAD3722E696E518A9945CF7F7C0FEA84CB3CB2E9F0384A6B5DC671ADE7FB4D2B27011173F3EEEAF17CB451CF26542031", - }, - }, -} - -// longTestVectors stores longer testVectors (currently only one). -// The computed test vector is 64 MiB long and is a truncated version of the -// ExtremelyLongMsgKAT taken from http://keccak.noekeon.org/. -var longKeccakTestVectors = []testVector{ - { - desc: "long-64MiB", - input: []byte("abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmno"), - repeat: 1024 * 1024, - want: map[string]string{ - "Keccak224": "50E35E40980FEEFF1EA490957B0E970257F75EA0D410EE0F0B8A7A58", - "Keccak256": "5015A4935F0B51E091C6550A94DCD262C08998232CCAA22E7F0756DEAC0DC0D0", - "Keccak384": "7907A8D0FAA7BC6A90FE14C6C958C956A0877E751455D8F13ACDB96F144B5896E716C06EC0CB56557A94EF5C3355F6F3", - "Keccak512": "3EC327D6759F769DEB74E80CA70C831BC29CAB048A4BF4190E4A1DD5C6507CF2B4B58937FDE81D36014E7DFE1B1DD8B0F27CB7614F9A645FEC114F1DAAEFC056", - }, - }, -} - -// TestKeccakVectors checks that correct output is produced for a set of known testVectors. -func TestKeccakVectors(t *testing.T) { - testCases := append([]testVector{}, shortKeccakTestVectors...) - if !testing.Short() { - testCases = append(testCases, longKeccakTestVectors...) +// structs used to marshal JSON test-cases. +type KeccakKats struct { + Kats map[string][]struct { + Digest string `json:"digest"` + Length int64 `json:"length"` + Message string `json:"message"` } - for _, tc := range testCases { - for alg, want := range tc.want { - d := testDigests[alg] +} + +// TestKeccakKats tests the SHA-3 and Shake implementations against all the +// ShortMsgKATs from https://github.com/gvanas/KeccakCodePackage +// (The testvectors are stored in keccakKats.json.deflate due to their length.) +func TestKeccakKats(t *testing.T) { + // Read the KATs. + deflated, err := os.Open(katFilename) + if err != nil { + t.Errorf("Error opening %s: %s", katFilename, err) + } + file := flate.NewReader(deflated) + dec := json.NewDecoder(file) + var katSet KeccakKats + err = dec.Decode(&katSet) + if err != nil { + t.Errorf("%s", err) + } + + // Do the KATs. + for functionName, kats := range katSet.Kats { + d := testDigests[functionName]() + t.Logf("%s", functionName) + for _, kat := range kats { d.Reset() - for i := 0; i < tc.repeat; i++ { - d.Write(tc.input) + in, err := hex.DecodeString(kat.Message) + if err != nil { + t.Errorf("%s", err) } + d.Write(in[:kat.Length/8]) got := strings.ToUpper(hex.EncodeToString(d.Sum(nil))) + want := kat.Digest if got != want { - t.Errorf("%s, alg=%s\ngot %q, want %q", tc.desc, alg, got, want) + t.Errorf("function=%s, length=%d\nmessage:\n %s\ngot:\n %s\nwanted:\n %s", + functionName, kat.Length, kat.Message, got, want) + t.Logf("wanted %+v", kat) + t.FailNow() } } } } -// dumpState is a debugging function to pretty-print the internal state of the hash. -func (d *digest) dumpState() { - fmt.Printf("SHA3 hash, %d B output, %d B capacity (%d B rate)\n", d.outputSize, d.capacity, d.rate()) - fmt.Printf("Internal state after absorbing %d B:\n", d.absorbed) - - for x := 0; x < sliceSize; x++ { - for y := 0; y < sliceSize; y++ { - fmt.Printf("%v, ", d.a[x*sliceSize+y]) - } - fmt.Println("") - } -} - -// TestUnalignedWrite tests that writing data in an arbitrary pattern with small input buffers. +// TestUnalignedWrite tests that writing data in an arbitrary pattern with +// small input buffers. func TestUnalignedWrite(t *testing.T) { buf := sequentialBytes(0x10000) - for alg, d := range testDigests { + for alg, df := range testDigests { + d := df() d.Reset() d.Write(buf) want := d.Sum(nil) @@ -145,7 +126,9 @@ func TestUnalignedWrite(t *testing.T) { // Because 137 is prime this sequence should exercise all corner cases. offsets := [17]int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1} for _, j := range offsets { - j = minInt(j, len(buf)-i) + if v := len(buf) - i; v < j { + j = v + } d.Write(buf[i : i+j]) i += j } @@ -157,8 +140,9 @@ func TestUnalignedWrite(t *testing.T) { } } +// Test that appending works when reallocation is necessary. func TestAppend(t *testing.T) { - d := NewKeccak224() + d := New224() for capacity := 2; capacity < 64; capacity += 64 { // The first time around the loop, Sum will have to reallocate. @@ -167,24 +151,57 @@ func TestAppend(t *testing.T) { d.Reset() d.Write([]byte{0xcc}) buf = d.Sum(buf) - expected := "0000A9CAB59EB40A10B246290F2D6086E32E3689FAF1D26B470C899F2802" + expected := "0000DF70ADC49B2E76EEE3A6931B93FA41841C3AF2CDF5B32A18B5478C39" if got := strings.ToUpper(hex.EncodeToString(buf)); got != expected { t.Errorf("got %s, want %s", got, expected) } } } +// Test that appending works when no reallocation is necessary. func TestAppendNoRealloc(t *testing.T) { buf := make([]byte, 1, 200) - d := NewKeccak224() + d := New224() d.Write([]byte{0xcc}) buf = d.Sum(buf) - expected := "00A9CAB59EB40A10B246290F2D6086E32E3689FAF1D26B470C899F2802" + expected := "00DF70ADC49B2E76EEE3A6931B93FA41841C3AF2CDF5B32A18B5478C39" if got := strings.ToUpper(hex.EncodeToString(buf)); got != expected { t.Errorf("got %s, want %s", got, expected) } } +// TestSqueezing checks that squeezing the full output a single time produces +// the same output as repeatedly squeezing the instance. +func TestSqueezing(t *testing.T) { + for functionName, newShakeHash := range testShakes { + t.Logf("%s", functionName) + d0 := newShakeHash() + d0.Write([]byte(testString)) + ref := make([]byte, 32) + d0.Read(ref) + + d1 := newShakeHash() + d1.Write([]byte(testString)) + var multiple []byte + for _ = range ref { + one := make([]byte, 1) + d1.Read(one) + multiple = append(multiple, one...) + } + if !bytes.Equal(ref, multiple) { + t.Errorf("squeezing %d bytes one at a time failed", len(ref)) + } + } +} + +func TestReadSimulation(t *testing.T) { + d := NewShake256() + d.Write(nil) + dwr := make([]byte, 32) + d.Read(dwr) + +} + // sequentialBytes produces a buffer of size consecutive bytes 0x00, 0x01, ..., used for testing. func sequentialBytes(size int) []byte { result := make([]byte, size) @@ -194,77 +211,39 @@ func sequentialBytes(size int) []byte { return result } -// benchmarkBlockWrite tests the speed of writing data and never calling the permutation function. -func benchmarkBlockWrite(b *testing.B, d *digest) { - b.StopTimer() - d.Reset() - // Write all but the last byte of a block, to ensure that the permutation is not called. - data := sequentialBytes(d.rate() - 1) - b.SetBytes(int64(len(data))) - b.StartTimer() - for i := 0; i < b.N; i++ { - d.absorbed = 0 // Reset absorbed to avoid ever calling the permutation function - d.Write(data) - } - b.StopTimer() - d.Reset() -} - -// BenchmarkPermutationFunction measures the speed of the permutation function with no input data. +// BenchmarkPermutationFunction measures the speed of the permutation function +// with no input data. func BenchmarkPermutationFunction(b *testing.B) { - b.SetBytes(int64(stateSize)) - var lanes [numLanes]uint64 + b.SetBytes(int64(200)) + var lanes [25]uint64 for i := 0; i < b.N; i++ { - keccakF(&lanes) + keccakF1600(&lanes) } } -// BenchmarkSingleByteWrite tests the latency from writing a single byte -func BenchmarkSingleByteWrite(b *testing.B) { - b.StopTimer() - d := testDigests["Keccak512"] - d.Reset() - data := sequentialBytes(1) //1 byte buffer - b.SetBytes(int64(d.rate()) - 1) - b.StartTimer() - for i := 0; i < b.N; i++ { - d.absorbed = 0 // Reset absorbed to avoid ever calling the permutation function - - // Write all but the last byte of a block, one byte at a time. - for j := 0; j < d.rate()-1; j++ { - d.Write(data) - } - } - b.StopTimer() - d.Reset() -} - -// BenchmarkSingleByteX measures the block write speed for each size of the digest. -func BenchmarkBlockWrite512(b *testing.B) { benchmarkBlockWrite(b, testDigests["Keccak512"]) } -func BenchmarkBlockWrite384(b *testing.B) { benchmarkBlockWrite(b, testDigests["Keccak384"]) } -func BenchmarkBlockWrite256(b *testing.B) { benchmarkBlockWrite(b, testDigests["Keccak256"]) } -func BenchmarkBlockWrite224(b *testing.B) { benchmarkBlockWrite(b, testDigests["Keccak224"]) } - -// benchmarkBulkHash tests the speed to hash a 16 KiB buffer. -func benchmarkBulkHash(b *testing.B, h hash.Hash) { +// benchmarkBulkHash tests the speed to hash a buffer of buflen. +func benchmarkBulkHash(b *testing.B, h hash.Hash, size int) { b.StopTimer() h.Reset() - size := 1 << 14 data := sequentialBytes(size) b.SetBytes(int64(size)) b.StartTimer() - var digest []byte + var state []byte for i := 0; i < b.N; i++ { h.Write(data) - digest = h.Sum(digest[:0]) + state = h.Sum(state[:0]) } b.StopTimer() h.Reset() } -// benchmarkBulkKeccakX test the speed to hash a 16 KiB buffer by calling benchmarkBulkHash. -func BenchmarkBulkKeccak512(b *testing.B) { benchmarkBulkHash(b, NewKeccak512()) } -func BenchmarkBulkKeccak384(b *testing.B) { benchmarkBulkHash(b, NewKeccak384()) } -func BenchmarkBulkKeccak256(b *testing.B) { benchmarkBulkHash(b, NewKeccak256()) } -func BenchmarkBulkKeccak224(b *testing.B) { benchmarkBulkHash(b, NewKeccak224()) } +func BenchmarkSha3_512_MTU(b *testing.B) { benchmarkBulkHash(b, New512(), 1350) } +func BenchmarkSha3_384_MTU(b *testing.B) { benchmarkBulkHash(b, New384(), 1350) } +func BenchmarkSha3_256_MTU(b *testing.B) { benchmarkBulkHash(b, New256(), 1350) } +func BenchmarkSha3_224_MTU(b *testing.B) { benchmarkBulkHash(b, New224(), 1350) } +func BenchmarkShake256_MTU(b *testing.B) { benchmarkBulkHash(b, newHashShake256(), 1350) } +func BenchmarkShake128_MTU(b *testing.B) { benchmarkBulkHash(b, newHashShake128(), 1350) } + +func BenchmarkSha3_512_1MiB(b *testing.B) { benchmarkBulkHash(b, New512(), 1<<20) } +func BenchmarkShake256_1MiB(b *testing.B) { benchmarkBulkHash(b, newHashShake256(), 1<<20) } diff --git a/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/shake.go b/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/shake.go new file mode 100644 index 000000000..841f9860f --- /dev/null +++ b/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3/shake.go @@ -0,0 +1,60 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sha3 + +// This file defines the ShakeHash interface, and provides +// functions for creating SHAKE instances, as well as utility +// functions for hashing bytes to arbitrary-length output. + +import ( + "io" +) + +// ShakeHash defines the interface to hash functions that +// support arbitrary-length output. +type ShakeHash interface { + // Write absorbs more data into the hash's state. It panics if input is + // written to it after output has been read from it. + io.Writer + + // Read reads more output from the hash; reading affects the hash's + // state. (ShakeHash.Read is thus very different from Hash.Sum) + // It never returns an error. + io.Reader + + // Clone returns a copy of the ShakeHash in its current state. + Clone() ShakeHash + + // Reset resets the ShakeHash to its initial state. + Reset() +} + +func (d *state) Clone() ShakeHash { + return d.clone() +} + +// NewShake128 creates a new SHAKE128 variable-output-length ShakeHash. +// Its generic security strength is 128 bits against all attacks if at +// least 32 bytes of its output are used. +func NewShake128() ShakeHash { return &state{rate: 168, dsbyte: 0x1f} } + +// NewShake256 creates a new SHAKE128 variable-output-length ShakeHash. +// Its generic security strength is 256 bits against all attacks if +// at least 64 bytes of its output are used. +func NewShake256() ShakeHash { return &state{rate: 136, dsbyte: 0x1f} } + +// ShakeSum128 writes an arbitrary-length digest of data into hash. +func ShakeSum128(hash, data []byte) { + h := NewShake128() + h.Write(data) + h.Read(hash) +} + +// ShakeSum256 writes an arbitrary-length digest of data into hash. +func ShakeSum256(hash, data []byte) { + h := NewShake256() + h.Write(data) + h.Read(hash) +} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-msgio/msgio_test.go b/Godeps/_workspace/src/github.com/jbenet/go-msgio/msgio_test.go index cef3e9cd0..05eca7b24 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-msgio/msgio_test.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-msgio/msgio_test.go @@ -15,6 +15,24 @@ func TestReadWrite(t *testing.T) { buf := bytes.NewBuffer(nil) writer := NewWriter(buf) reader := NewReader(buf) + SubtestReadWrite(t, writer, reader) +} + +func TestReadWriteMsg(t *testing.T) { + buf := bytes.NewBuffer(nil) + writer := NewWriter(buf) + reader := NewReader(buf) + SubtestReadWriteMsg(t, writer, reader) +} + +func TestReadWriteMsgSync(t *testing.T) { + buf := bytes.NewBuffer(nil) + writer := NewWriter(buf) + reader := NewReader(buf) + SubtestReadWriteMsgSync(t, writer, reader) +} + +func SubtestReadWrite(t *testing.T, writer WriteCloser, reader ReadCloser) { msgs := [1000][]byte{} r := rand.New(rand.NewSource(time.Now().UnixNano())) @@ -58,10 +76,7 @@ func TestReadWrite(t *testing.T) { } } -func TestReadWriteMsg(t *testing.T) { - buf := bytes.NewBuffer(nil) - writer := NewWriter(buf) - reader := NewReader(buf) +func SubtestReadWriteMsg(t *testing.T, writer WriteCloser, reader ReadCloser) { msgs := [1000][]byte{} r := rand.New(rand.NewSource(time.Now().UnixNano())) @@ -100,10 +115,7 @@ func TestReadWriteMsg(t *testing.T) { } } -func TestReadWriteMsgSync(t *testing.T) { - buf := bytes.NewBuffer(nil) - writer := NewWriter(buf) - reader := NewReader(buf) +func SubtestReadWriteMsgSync(t *testing.T, writer WriteCloser, reader ReadCloser) { msgs := [1000][]byte{} r := rand.New(rand.NewSource(time.Now().UnixNano())) diff --git a/Godeps/_workspace/src/github.com/jbenet/go-msgio/varint.go b/Godeps/_workspace/src/github.com/jbenet/go-msgio/varint.go new file mode 100644 index 000000000..613470503 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-msgio/varint.go @@ -0,0 +1,182 @@ +package msgio + +import ( + "encoding/binary" + "io" + "sync" + + mpool "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-msgio/mpool" +) + +// varintWriter is the underlying type that implements the Writer interface. +type varintWriter struct { + W io.Writer + + lbuf []byte // for encoding varints + lock sync.Locker // for threadsafe writes +} + +// NewVarintWriter wraps an io.Writer with a varint msgio framed writer. +// The msgio.Writer will write the length prefix of every message written +// as a varint, using https://golang.org/pkg/encoding/binary/#PutUvarint +func NewVarintWriter(w io.Writer) WriteCloser { + return &varintWriter{ + W: w, + lbuf: make([]byte, binary.MaxVarintLen64), + lock: new(sync.Mutex), + } +} + +func (s *varintWriter) Write(msg []byte) (int, error) { + err := s.WriteMsg(msg) + if err != nil { + return 0, err + } + return len(msg), nil +} + +func (s *varintWriter) WriteMsg(msg []byte) error { + s.lock.Lock() + defer s.lock.Unlock() + + length := uint64(len(msg)) + n := binary.PutUvarint(s.lbuf, length) + if _, err := s.W.Write(s.lbuf[:n]); err != nil { + return err + } + _, err := s.W.Write(msg) + return err +} + +func (s *varintWriter) Close() error { + s.lock.Lock() + defer s.lock.Unlock() + + if c, ok := s.W.(io.Closer); ok { + return c.Close() + } + return nil +} + +// varintReader is the underlying type that implements the Reader interface. +type varintReader struct { + R io.Reader + br io.ByteReader // for reading varints. + + lbuf []byte + next int + pool *mpool.Pool + lock sync.Locker +} + +// NewVarintReader wraps an io.Reader with a varint msgio framed reader. +// The msgio.Reader will read whole messages at a time (using the length). +// Varints read according to https://golang.org/pkg/encoding/binary/#ReadUvarint +// Assumes an equivalent writer on the other side. +func NewVarintReader(r io.Reader) ReadCloser { + return NewVarintReaderWithPool(r, &mpool.ByteSlicePool) +} + +// NewVarintReaderWithPool wraps an io.Reader with a varint msgio framed reader. +// The msgio.Reader will read whole messages at a time (using the length). +// Varints read according to https://golang.org/pkg/encoding/binary/#ReadUvarint +// Assumes an equivalent writer on the other side. It uses a given mpool.Pool +func NewVarintReaderWithPool(r io.Reader, p *mpool.Pool) ReadCloser { + if p == nil { + panic("nil pool") + } + return &varintReader{ + R: r, + br: &simpleByteReader{R: r}, + lbuf: make([]byte, binary.MaxVarintLen64), + next: -1, + pool: p, + lock: new(sync.Mutex), + } +} + +// NextMsgLen reads the length of the next msg into s.lbuf, and returns it. +// WARNING: like Read, NextMsgLen is destructive. It reads from the internal +// reader. +func (s *varintReader) NextMsgLen() (int, error) { + s.lock.Lock() + defer s.lock.Unlock() + return s.nextMsgLen() +} + +func (s *varintReader) nextMsgLen() (int, error) { + if s.next == -1 { + length, err := binary.ReadUvarint(s.br) + if err != nil { + return 0, err + } + s.next = int(length) + } + return s.next, nil +} + +func (s *varintReader) Read(msg []byte) (int, error) { + s.lock.Lock() + defer s.lock.Unlock() + + length, err := s.nextMsgLen() + if err != nil { + return 0, err + } + + if length > len(msg) { + return 0, io.ErrShortBuffer + } + _, err = io.ReadFull(s.R, msg[:length]) + s.next = -1 // signal we've consumed this msg + return length, err +} + +func (s *varintReader) ReadMsg() ([]byte, error) { + s.lock.Lock() + defer s.lock.Unlock() + + length, err := s.nextMsgLen() + if err != nil { + return nil, err + } + + msgb := s.pool.Get(uint32(length)) + if msgb == nil { + return nil, io.ErrShortBuffer + } + msg := msgb.([]byte)[:length] + _, err = io.ReadFull(s.R, msg) + s.next = -1 // signal we've consumed this msg + return msg, err +} + +func (s *varintReader) ReleaseMsg(msg []byte) { + s.pool.Put(uint32(cap(msg)), msg) +} + +func (s *varintReader) Close() error { + s.lock.Lock() + defer s.lock.Unlock() + + if c, ok := s.R.(io.Closer); ok { + return c.Close() + } + return nil +} + +type simpleByteReader struct { + R io.Reader + buf []byte +} + +func (r *simpleByteReader) ReadByte() (c byte, err error) { + if r.buf == nil { + r.buf = make([]byte, 1) + } + + if _, err := io.ReadFull(r.R, r.buf); err != nil { + return 0, err + } + return r.buf[0], nil +} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-msgio/varint_test.go b/Godeps/_workspace/src/github.com/jbenet/go-msgio/varint_test.go new file mode 100644 index 000000000..c63415089 --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-msgio/varint_test.go @@ -0,0 +1,66 @@ +package msgio + +import ( + "bytes" + "encoding/binary" + "testing" +) + +func TestVarintReadWrite(t *testing.T) { + buf := bytes.NewBuffer(nil) + writer := NewVarintWriter(buf) + reader := NewVarintReader(buf) + SubtestReadWrite(t, writer, reader) +} + +func TestVarintReadWriteMsg(t *testing.T) { + buf := bytes.NewBuffer(nil) + writer := NewVarintWriter(buf) + reader := NewVarintReader(buf) + SubtestReadWriteMsg(t, writer, reader) +} + +func TestVarintReadWriteMsgSync(t *testing.T) { + buf := bytes.NewBuffer(nil) + writer := NewVarintWriter(buf) + reader := NewVarintReader(buf) + SubtestReadWriteMsgSync(t, writer, reader) +} + +func TestVarintWrite(t *testing.T) { + SubtestVarintWrite(t, []byte("hello world")) + SubtestVarintWrite(t, []byte("hello world hello world hello world")) + SubtestVarintWrite(t, make([]byte, 1<<20)) + SubtestVarintWrite(t, []byte("")) +} + +func SubtestVarintWrite(t *testing.T, msg []byte) { + buf := bytes.NewBuffer(nil) + writer := NewVarintWriter(buf) + + if err := writer.WriteMsg(msg); err != nil { + t.Fatal(err) + } + + bb := buf.Bytes() + + sbr := simpleByteReader{R: buf} + length, err := binary.ReadUvarint(&sbr) + if err != nil { + t.Fatal(err) + } + + t.Logf("checking varint is %d", len(msg)) + if int(length) != len(msg) { + t.Fatalf("incorrect varint: %d != %d", length, len(msg)) + } + + lbuf := make([]byte, binary.MaxVarintLen64) + n := binary.PutUvarint(lbuf, length) + + bblen := int(length) + n + t.Logf("checking wrote (%d + %d) bytes", length, n) + if len(bb) != bblen { + t.Fatalf("wrote incorrect number of bytes: %d != %d", len(bb), bblen) + } +} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-multihash/io.go b/Godeps/_workspace/src/github.com/jbenet/go-multihash/io.go new file mode 100644 index 000000000..cdd234c4b --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-multihash/io.go @@ -0,0 +1,79 @@ +package multihash + +import ( + "fmt" + "io" +) + +// Reader is an io.Reader wrapper that exposes a function +// to read a whole multihash, parse it, and return it. +type Reader interface { + io.Reader + + ReadMultihash() (Multihash, error) +} + +// Writer is an io.Writer wrapper that exposes a function +// to write a whole multihash. +type Writer interface { + io.Writer + + WriteMultihash(Multihash) error +} + +// NewReader wraps an io.Reader with a multihash.Reader +func NewReader(r io.Reader) Reader { + return &mhReader{r} +} + +// NewWriter wraps an io.Writer with a multihash.Writer +func NewWriter(w io.Writer) Writer { + return &mhWriter{w} +} + +type mhReader struct { + r io.Reader +} + +func (r *mhReader) Read(buf []byte) (n int, err error) { + return r.r.Read(buf) +} + +func (r *mhReader) ReadMultihash() (Multihash, error) { + mhhdr := make([]byte, 2) + if _, err := io.ReadFull(r.r, mhhdr); err != nil { + return nil, err + } + + // first byte is the algo, the second is the length. + + // (varints someday...) + length := uint(mhhdr[1]) + + if length > 127 { + return nil, fmt.Errorf("varints not yet supported (length is %d)", length) + } + + buf := make([]byte, length+2) + buf[0] = mhhdr[0] + buf[1] = mhhdr[1] + + if _, err := io.ReadFull(r.r, buf[2:]); err != nil { + return nil, err + } + + return Cast(buf) +} + +type mhWriter struct { + w io.Writer +} + +func (w *mhWriter) Write(buf []byte) (n int, err error) { + return w.w.Write(buf) +} + +func (w *mhWriter) WriteMultihash(m Multihash) error { + _, err := w.w.Write([]byte(m)) + return err +} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-multihash/io_test.go b/Godeps/_workspace/src/github.com/jbenet/go-multihash/io_test.go new file mode 100644 index 000000000..44da78e1f --- /dev/null +++ b/Godeps/_workspace/src/github.com/jbenet/go-multihash/io_test.go @@ -0,0 +1,69 @@ +package multihash + +import ( + "bytes" + "io" + "testing" +) + +func TestReader(t *testing.T) { + + var buf bytes.Buffer + + for _, tc := range testCases { + m, err := tc.Multihash() + if err != nil { + t.Fatal(err) + } + + buf.Write([]byte(m)) + } + + r := NewReader(&buf) + + for _, tc := range testCases { + h, err := tc.Multihash() + if err != nil { + t.Fatal(err) + } + + h2, err := r.ReadMultihash() + if err != nil { + t.Error(err) + continue + } + + if !bytes.Equal(h, h2) { + t.Error("h and h2 should be equal") + } + } +} + +func TestWriter(t *testing.T) { + + var buf bytes.Buffer + w := NewWriter(&buf) + + for _, tc := range testCases { + m, err := tc.Multihash() + if err != nil { + t.Error(err) + continue + } + + if err := w.WriteMultihash(m); err != nil { + t.Error(err) + continue + } + + buf2 := make([]byte, len(m)) + if _, err := io.ReadFull(&buf, buf2); err != nil { + t.Error(err) + continue + } + + if !bytes.Equal(m, buf2) { + t.Error("m and buf2 should be equal") + } + } +} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-multihash/multihash.go b/Godeps/_workspace/src/github.com/jbenet/go-multihash/multihash.go index a56bc162b..9dbe7fd7f 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-multihash/multihash.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-multihash/multihash.go @@ -2,43 +2,67 @@ package multihash import ( "encoding/hex" + "errors" "fmt" + b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58" ) +// errors +var ( + ErrUnknownCode = errors.New("unknown multihash code") + ErrTooShort = errors.New("multihash too short. must be > 3 bytes") + ErrTooLong = errors.New("multihash too long. must be < 129 bytes") + ErrLenNotSupported = errors.New("multihash does not yet support digests longer than 127 bytes") +) + +// ErrInconsistentLen is returned when a decoded multihash has an inconsistent length +type ErrInconsistentLen struct { + dm *DecodedMultihash +} + +func (e ErrInconsistentLen) Error() string { + return fmt.Sprintf("multihash length inconsistent: %v", e.dm) +} + // constants -const SHA1 = 0x11 -const SHA2_256 = 0x12 -const SHA2_512 = 0x13 -const SHA3 = 0x14 -const BLAKE2B = 0x40 -const BLAKE2S = 0x41 +const ( + SHA1 = 0x11 + SHA2_256 = 0x12 + SHA2_512 = 0x13 + SHA3 = 0x14 + BLAKE2B = 0x40 + BLAKE2S = 0x41 +) +// Names maps the name of a hash to the code var Names = map[string]int{ - "sha1": 0x11, - "sha2-256": 0x12, - "sha2-512": 0x13, - "sha3": 0x14, - "blake2b": 0x40, - "blake2s": 0x41, + "sha1": SHA1, + "sha2-256": SHA2_256, + "sha2-512": SHA2_512, + "sha3": SHA3, + "blake2b": BLAKE2B, + "blake2s": BLAKE2S, } +// Codes maps a hash code to it's name var Codes = map[int]string{ - 0x11: "sha1", - 0x12: "sha2-256", - 0x13: "sha2-512", - 0x14: "sha3", - 0x40: "blake2b", - 0x41: "blake2s", + SHA1: "sha1", + SHA2_256: "sha2-256", + SHA2_512: "sha2-512", + SHA3: "sha3", + BLAKE2B: "blake2b", + BLAKE2S: "blake2s", } +// DefaultLengths maps a hash code to it's default length var DefaultLengths = map[int]int{ - 0x11: 20, - 0x12: 32, - 0x13: 64, - 0x14: 64, - 0x40: 64, - 0x41: 32, + SHA1: 20, + SHA2_256: 32, + SHA2_512: 64, + SHA3: 64, + BLAKE2B: 64, + BLAKE2S: 32, } type DecodedMultihash struct { @@ -50,8 +74,12 @@ type DecodedMultihash struct { type Multihash []byte -func (m Multihash) HexString() string { - return hex.EncodeToString([]byte(m)) +func (m *Multihash) HexString() string { + return hex.EncodeToString([]byte(*m)) +} + +func (m *Multihash) String() string { + return m.HexString() } func FromHexString(s string) (Multihash, error) { @@ -88,21 +116,21 @@ func Cast(buf []byte) (Multihash, error) { } if !ValidCode(dm.Code) { - return Multihash{}, fmt.Errorf("unknown multihash code") + return Multihash{}, ErrUnknownCode } return Multihash(buf), nil } -// Decodes a hash from the given Multihash. +// Decode a hash from the given Multihash. func Decode(buf []byte) (*DecodedMultihash, error) { if len(buf) < 3 { - return nil, fmt.Errorf("multihash too short. must be > 3 bytes.") + return nil, ErrTooShort } if len(buf) > 129 { - return nil, fmt.Errorf("multihash too long. must be < 129 bytes.") + return nil, ErrTooLong } dm := &DecodedMultihash{ @@ -113,23 +141,22 @@ func Decode(buf []byte) (*DecodedMultihash, error) { } if len(dm.Digest) != dm.Length { - return nil, fmt.Errorf("multihash length inconsistent: %v", dm) + return nil, ErrInconsistentLen{dm} } return dm, nil } -// Encodes a hash digest along with the specified function code. +// Encode a hash digest along with the specified function code. // Note: the length is derived from the length of the digest itself. func Encode(buf []byte, code int) ([]byte, error) { if !ValidCode(code) { - return nil, fmt.Errorf("unknown multihash code") + return nil, ErrUnknownCode } if len(buf) > 127 { - m := "multihash does not yet support digests longer than 127 bytes." - return nil, fmt.Errorf(m) + return nil, ErrLenNotSupported } pre := make([]byte, 2) @@ -142,7 +169,7 @@ func EncodeName(buf []byte, name string) ([]byte, error) { return Encode(buf, Names[name]) } -// Checks whether a multihash code is valid. +// ValidCode checks whether a multihash code is valid. func ValidCode(code int) bool { if AppCode(code) { return true @@ -155,7 +182,7 @@ func ValidCode(code int) bool { return false } -// Checks whether a multihash code is part of the App range. +// AppCode checks whether a multihash code is part of the App range. func AppCode(code int) bool { return code >= 0 && code < 0x10 } diff --git a/Godeps/_workspace/src/github.com/jbenet/go-multihash/multihash_test.go b/Godeps/_workspace/src/github.com/jbenet/go-multihash/multihash_test.go index fac9f721b..dfa858e4f 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-multihash/multihash_test.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-multihash/multihash_test.go @@ -3,6 +3,7 @@ package multihash import ( "bytes" "encoding/hex" + "fmt" "testing" ) @@ -31,6 +32,19 @@ var testCases = []TestCase{ TestCase{"0beec7b5ea3f0fdbc9", 0x40, "blake2b"}, } +func (tc TestCase) Multihash() (Multihash, error) { + ob, err := hex.DecodeString(tc.hex) + if err != nil { + return nil, err + } + + b := make([]byte, 2+len(ob)) + b[0] = byte(uint8(tc.code)) + b[1] = byte(uint8(len(ob))) + copy(b[2:], ob) + return Cast(b) +} + func TestEncode(t *testing.T) { for _, tc := range testCases { ob, err := hex.DecodeString(tc.hex) @@ -63,9 +77,28 @@ func TestEncode(t *testing.T) { if !bytes.Equal(encN, nb) { t.Error("encoded byte mismatch: ", encN, nb) } + + h, err := tc.Multihash() + if err != nil { + t.Error(err) + } + if !bytes.Equal(h, nb) { + t.Error("Multihash func mismatch.") + } } } +func ExampleEncodeName() { + // ignores errors for simplicity - don't do that at home. + buf, _ := hex.DecodeString("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") + mhbuf, _ := EncodeName(buf, "sha1") + mhhex := hex.EncodeToString(mhbuf) + fmt.Printf("hex: %v\n", mhhex) + + // Output: + // hex: 11140beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33 +} + func TestDecode(t *testing.T) { for _, tc := range testCases { ob, err := hex.DecodeString(tc.hex) @@ -114,6 +147,18 @@ func TestTable(t *testing.T) { } } +func ExampleDecode() { + // ignores errors for simplicity - don't do that at home. + buf, _ := hex.DecodeString("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") + mhbuf, _ := EncodeName(buf, "sha1") + o, _ := Decode(mhbuf) + mhhex := hex.EncodeToString(o.Digest) + fmt.Printf("obj: %v 0x%x %d %s\n", o.Name, o.Code, o.Length, mhhex) + + // Output: + // obj: sha1 0x11 20 0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33 +} + func TestValidCode(t *testing.T) { for i := 0; i < 0xff; i++ { _, ok := tCodes[i] @@ -127,7 +172,7 @@ func TestValidCode(t *testing.T) { func TestAppCode(t *testing.T) { for i := 0; i < 0xff; i++ { - b := i > 0 && i < 0x10 + b := i >= 0 && i < 0x10 if AppCode(i) != b { t.Error("AppCode incorrect for: ", i) } @@ -190,3 +235,36 @@ func TestHex(t *testing.T) { } } } + +func BenchmarkEncode(b *testing.B) { + tc := testCases[0] + ob, err := hex.DecodeString(tc.hex) + if err != nil { + b.Error(err) + return + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + Encode(ob, tc.code) + } +} + +func BenchmarkDecode(b *testing.B) { + tc := testCases[0] + ob, err := hex.DecodeString(tc.hex) + if err != nil { + b.Error(err) + return + } + + pre := make([]byte, 2) + pre[0] = byte(uint8(tc.code)) + pre[1] = byte(uint8(len(ob))) + nb := append(pre, ob...) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + Decode(nb) + } +} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-multihash/sum.go b/Godeps/_workspace/src/github.com/jbenet/go-multihash/sum.go index 767be814e..4dc29ee3c 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-multihash/sum.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-multihash/sum.go @@ -4,10 +4,14 @@ import ( "crypto/sha1" "crypto/sha256" "crypto/sha512" + "errors" "fmt" + sha3 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.crypto/sha3" ) +var ErrSumNotSupported = errors.New("Function not implemented. Complain to lib maintainer.") + func Sum(data []byte, code int, length int) (Multihash, error) { m := Multihash{} err := error(nil) @@ -26,7 +30,7 @@ func Sum(data []byte, code int, length int) (Multihash, error) { case SHA3: d, err = sumSHA3(data) default: - return m, fmt.Errorf("Function not implemented. Complain to lib maintainer.") + return m, ErrSumNotSupported } if err != nil { @@ -60,7 +64,7 @@ func sumSHA512(data []byte) []byte { } func sumSHA3(data []byte) ([]byte, error) { - h := sha3.NewKeccak512() + h := sha3.New512() if _, err := h.Write(data); err != nil { return nil, err } diff --git a/Godeps/_workspace/src/github.com/jbenet/go-multihash/sum_test.go b/Godeps/_workspace/src/github.com/jbenet/go-multihash/sum_test.go index 3431d6f67..1e5891bbb 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-multihash/sum_test.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-multihash/sum_test.go @@ -57,3 +57,10 @@ func TestSum(t *testing.T) { } } } + +func BenchmarkSum(b *testing.B) { + tc := sumTestCases[0] + for i := 0; i < b.N; i++ { + Sum([]byte(tc.input), tc.code, tc.length) + } +} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-multihash/test/foo.go b/Godeps/_workspace/src/github.com/jbenet/go-multihash/test/foo.go deleted file mode 100644 index 21d33d658..000000000 --- a/Godeps/_workspace/src/github.com/jbenet/go-multihash/test/foo.go +++ /dev/null @@ -1,21 +0,0 @@ -package main - -import ( - "encoding/hex" - "fmt" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" -) - -func main() { - // ignores errors for simplicity. - // don't do that at home. - - buf, _ := hex.DecodeString("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") - mhbuf, _ := multihash.EncodeName(buf, "sha1") - mhhex := hex.EncodeToString(mhbuf) - fmt.Printf("hex: %v\n", mhhex) - - o, _ := multihash.Decode(mhbuf) - mhhex = hex.EncodeToString(o.Digest) - fmt.Printf("obj: %v 0x%x %d %s\n", o.Name, o.Code, o.Length, mhhex) -} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-multihash/test/test b/Godeps/_workspace/src/github.com/jbenet/go-multihash/test/test new file mode 100644 index 000000000..9e5199531 Binary files /dev/null and b/Godeps/_workspace/src/github.com/jbenet/go-multihash/test/test differ diff --git a/cmd/ipfs/init.go b/cmd/ipfs/init.go index bc609659f..5d2c3e773 100644 --- a/cmd/ipfs/init.go +++ b/cmd/ipfs/init.go @@ -12,10 +12,10 @@ import ( cmds "github.com/jbenet/go-ipfs/commands" config "github.com/jbenet/go-ipfs/config" core "github.com/jbenet/go-ipfs/core" - ci "github.com/jbenet/go-ipfs/crypto" imp "github.com/jbenet/go-ipfs/importer" chunk "github.com/jbenet/go-ipfs/importer/chunk" - peer "github.com/jbenet/go-ipfs/peer" + ci "github.com/jbenet/go-ipfs/p2p/crypto" + peer "github.com/jbenet/go-ipfs/p2p/peer" repo "github.com/jbenet/go-ipfs/repo" u "github.com/jbenet/go-ipfs/util" debugerror "github.com/jbenet/go-ipfs/util/debugerror" diff --git a/cmd/seccat/seccat.go b/cmd/seccat/seccat.go index 625f64ec0..62fd1d024 100644 --- a/cmd/seccat/seccat.go +++ b/cmd/seccat/seccat.go @@ -18,9 +18,9 @@ import ( "os/signal" "syscall" - ci "github.com/jbenet/go-ipfs/crypto" - secio "github.com/jbenet/go-ipfs/crypto/secio" - peer "github.com/jbenet/go-ipfs/peer" + ci "github.com/jbenet/go-ipfs/p2p/crypto" + secio "github.com/jbenet/go-ipfs/p2p/crypto/secio" + peer "github.com/jbenet/go-ipfs/p2p/peer" u "github.com/jbenet/go-ipfs/util" ) diff --git a/config/config.go b/config/config.go index d165f9886..ed8e66b21 100644 --- a/config/config.go +++ b/config/config.go @@ -6,7 +6,7 @@ import ( "os" "path/filepath" - ic "github.com/jbenet/go-ipfs/crypto" + ic "github.com/jbenet/go-ipfs/p2p/crypto" u "github.com/jbenet/go-ipfs/util" "github.com/jbenet/go-ipfs/util/debugerror" ) diff --git a/core/bootstrap.go b/core/bootstrap.go index 1539a761f..8045b8f26 100644 --- a/core/bootstrap.go +++ b/core/bootstrap.go @@ -8,8 +8,9 @@ import ( "time" config "github.com/jbenet/go-ipfs/config" - inet "github.com/jbenet/go-ipfs/net" - peer "github.com/jbenet/go-ipfs/peer" + host "github.com/jbenet/go-ipfs/p2p/host" + inet "github.com/jbenet/go-ipfs/p2p/net" + peer "github.com/jbenet/go-ipfs/p2p/peer" dht "github.com/jbenet/go-ipfs/routing/dht" lgbl "github.com/jbenet/go-ipfs/util/eventlog/loggables" math2 "github.com/jbenet/go-ipfs/util/math2" @@ -25,7 +26,7 @@ const ( ) func superviseConnections(parent context.Context, - n inet.Network, + h host.Host, route *dht.IpfsDHT, // TODO depend on abstract interface for testing purposes store peer.Peerstore, peers []*config.BootstrapPeer) error { @@ -34,7 +35,7 @@ func superviseConnections(parent context.Context, ctx, _ := context.WithTimeout(parent, connectiontimeout) // TODO get config from disk so |peers| always reflects the latest // information - if err := bootstrap(ctx, n, route, store, peers); err != nil { + if err := bootstrap(ctx, h, route, store, peers); err != nil { log.Error(err) } select { @@ -47,30 +48,30 @@ func superviseConnections(parent context.Context, } func bootstrap(ctx context.Context, - n inet.Network, + h host.Host, r *dht.IpfsDHT, ps peer.Peerstore, boots []*config.BootstrapPeer) error { - connectedPeers := n.Peers() + connectedPeers := h.Network().Peers() if len(connectedPeers) >= recoveryThreshold { - log.Event(ctx, "bootstrapSkip", n.LocalPeer()) + log.Event(ctx, "bootstrapSkip", h.ID()) log.Debugf("%s bootstrap skipped -- connected to %d (> %d) nodes", - n.LocalPeer(), len(connectedPeers), recoveryThreshold) + h.ID(), len(connectedPeers), recoveryThreshold) return nil } numCxnsToCreate := recoveryThreshold - len(connectedPeers) - log.Event(ctx, "bootstrapStart", n.LocalPeer()) - log.Debugf("%s bootstrapping to %d more nodes", n.LocalPeer(), numCxnsToCreate) + log.Event(ctx, "bootstrapStart", h.ID()) + log.Debugf("%s bootstrapping to %d more nodes", h.ID(), numCxnsToCreate) var bootstrapPeers []peer.PeerInfo for _, bootstrap := range boots { p, err := toPeer(bootstrap) if err != nil { - log.Event(ctx, "bootstrapError", n.LocalPeer(), lgbl.Error(err)) - log.Errorf("%s bootstrap error: %s", n.LocalPeer(), err) + log.Event(ctx, "bootstrapError", h.ID(), lgbl.Error(err)) + log.Errorf("%s bootstrap error: %s", h.ID(), err) return err } bootstrapPeers = append(bootstrapPeers, p) @@ -78,7 +79,7 @@ func bootstrap(ctx context.Context, var notConnected []peer.PeerInfo for _, p := range bootstrapPeers { - if n.Connectedness(p.ID) != inet.Connected { + if h.Network().Connectedness(p.ID) != inet.Connected { notConnected = append(notConnected, p) } } @@ -86,17 +87,17 @@ func bootstrap(ctx context.Context, if len(notConnected) < 1 { s := "must bootstrap to %d more nodes, but already connected to all candidates" err := fmt.Errorf(s, numCxnsToCreate) - log.Event(ctx, "bootstrapError", n.LocalPeer(), lgbl.Error(err)) - log.Errorf("%s bootstrap error: %s", n.LocalPeer(), err) + log.Event(ctx, "bootstrapError", h.ID(), lgbl.Error(err)) + log.Errorf("%s bootstrap error: %s", h.ID(), err) return err } var randomSubset = randomSubsetOfPeers(notConnected, numCxnsToCreate) - log.Debugf("%s bootstrapping to %d nodes: %s", n.LocalPeer(), numCxnsToCreate, randomSubset) + log.Debugf("%s bootstrapping to %d nodes: %s", h.ID(), numCxnsToCreate, randomSubset) if err := connect(ctx, ps, r, randomSubset); err != nil { - log.Event(ctx, "bootstrapError", n.LocalPeer(), lgbl.Error(err)) - log.Errorf("%s bootstrap error: %s", n.LocalPeer(), err) + log.Event(ctx, "bootstrapError", h.ID(), lgbl.Error(err)) + log.Errorf("%s bootstrap error: %s", h.ID(), err) return err } return nil diff --git a/core/bootstrap_test.go b/core/bootstrap_test.go index 636a8f808..385c77fdf 100644 --- a/core/bootstrap_test.go +++ b/core/bootstrap_test.go @@ -3,7 +3,7 @@ package core import ( "testing" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" testutil "github.com/jbenet/go-ipfs/util/testutil" ) diff --git a/core/commands/id.go b/core/commands/id.go index 96e462a09..c71b0cddb 100644 --- a/core/commands/id.go +++ b/core/commands/id.go @@ -11,8 +11,8 @@ import ( b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58" cmds "github.com/jbenet/go-ipfs/commands" - ic "github.com/jbenet/go-ipfs/crypto" - "github.com/jbenet/go-ipfs/peer" + ic "github.com/jbenet/go-ipfs/p2p/crypto" + "github.com/jbenet/go-ipfs/p2p/peer" kb "github.com/jbenet/go-ipfs/routing/kbucket" u "github.com/jbenet/go-ipfs/util" ) diff --git a/core/commands/publish.go b/core/commands/publish.go index 97455b9c7..9b6448a7a 100644 --- a/core/commands/publish.go +++ b/core/commands/publish.go @@ -6,8 +6,8 @@ import ( cmds "github.com/jbenet/go-ipfs/commands" core "github.com/jbenet/go-ipfs/core" - crypto "github.com/jbenet/go-ipfs/crypto" nsys "github.com/jbenet/go-ipfs/namesys" + crypto "github.com/jbenet/go-ipfs/p2p/crypto" u "github.com/jbenet/go-ipfs/util" ) @@ -53,7 +53,7 @@ Publish a to another public key: args := req.Arguments() - if n.Network == nil { + if n.PeerHost == nil { return nil, errNotOnline } diff --git a/core/commands/resolve.go b/core/commands/resolve.go index dad72d755..02de15177 100644 --- a/core/commands/resolve.go +++ b/core/commands/resolve.go @@ -47,7 +47,7 @@ Resolve te value of another name: var name string - if n.Network == nil { + if n.PeerHost == nil { return nil, errNotOnline } diff --git a/core/commands/swarm.go b/core/commands/swarm.go index 9b583f3ae..a27c672ea 100644 --- a/core/commands/swarm.go +++ b/core/commands/swarm.go @@ -6,7 +6,7 @@ import ( "path" cmds "github.com/jbenet/go-ipfs/commands" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" errors "github.com/jbenet/go-ipfs/util/debugerror" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" @@ -51,11 +51,11 @@ ipfs swarm peers lists the set of peers this node is connected to. return nil, err } - if n.Network == nil { + if n.PeerHost == nil { return nil, errNotOnline } - conns := n.Network.Conns() + conns := n.PeerHost.Network().Conns() addrs := make([]string, len(conns)) for i, c := range conns { pid := c.RemotePeer() @@ -95,7 +95,7 @@ ipfs swarm connect /ip4/104.131.131.82/tcp/4001/QmaCpDMGvV2BGHeYERUEnRQAwe3N8Szb addrs := req.Arguments() - if n.Network == nil { + if n.PeerHost == nil { return nil, errNotOnline } @@ -108,7 +108,7 @@ ipfs swarm connect /ip4/104.131.131.82/tcp/4001/QmaCpDMGvV2BGHeYERUEnRQAwe3N8Szb for i, p := range peers { output[i] = "connect " + p.Pretty() - err := n.Network.DialPeer(ctx, p) + err := n.PeerHost.Connect(ctx, peer.PeerInfo{ID: p}) if err != nil { output[i] += " failure: " + err.Error() } else { diff --git a/core/core.go b/core/core.go index 7cabef281..b48e80f9c 100644 --- a/core/core.go +++ b/core/core.go @@ -11,7 +11,6 @@ import ( bstore "github.com/jbenet/go-ipfs/blocks/blockstore" bserv "github.com/jbenet/go-ipfs/blockservice" config "github.com/jbenet/go-ipfs/config" - ic "github.com/jbenet/go-ipfs/crypto" diag "github.com/jbenet/go-ipfs/diagnostics" exchange "github.com/jbenet/go-ipfs/exchange" bitswap "github.com/jbenet/go-ipfs/exchange/bitswap" @@ -20,9 +19,12 @@ import ( mount "github.com/jbenet/go-ipfs/fuse/mount" merkledag "github.com/jbenet/go-ipfs/merkledag" namesys "github.com/jbenet/go-ipfs/namesys" - inet "github.com/jbenet/go-ipfs/net" + ic "github.com/jbenet/go-ipfs/p2p/crypto" + p2phost "github.com/jbenet/go-ipfs/p2p/host" + p2pbhost "github.com/jbenet/go-ipfs/p2p/host/basic" + swarm "github.com/jbenet/go-ipfs/p2p/net/swarm" + peer "github.com/jbenet/go-ipfs/p2p/peer" path "github.com/jbenet/go-ipfs/path" - peer "github.com/jbenet/go-ipfs/peer" pin "github.com/jbenet/go-ipfs/pin" routing "github.com/jbenet/go-ipfs/routing" dht "github.com/jbenet/go-ipfs/routing/dht" @@ -52,7 +54,7 @@ type IpfsNode struct { // Services Peerstore peer.Peerstore // storage for other Peer instances - Network inet.Network // the network message stream + PeerHost p2phost.Host // the network host (server+client) Routing routing.IpfsRouting // the routing system. recommend ipfs-dht Exchange exchange.Interface // the block exchange + strategy (bitswap) Blocks *bserv.BlockService // the block service, get/add blocks. @@ -121,16 +123,17 @@ func NewIpfsNode(ctx context.Context, cfg *config.Config, online bool) (n *IpfsN return nil, debugerror.Wrap(err) } - n.Network, err = inet.NewNetwork(ctx, listenAddrs, n.Identity, n.Peerstore) + network, err := swarm.NewNetwork(ctx, listenAddrs, n.Identity, n.Peerstore) if err != nil { return nil, debugerror.Wrap(err) } - n.AddChildGroup(n.Network.CtxGroup()) + n.AddChildGroup(network.CtxGroup()) + n.PeerHost = p2pbhost.New(network) // explicitly set these as our listen addrs. // (why not do it inside inet.NewNetwork? because this way we can // listen on addresses without necessarily advertising those publicly.) - addrs, err := n.Network.InterfaceListenAddresses() + addrs, err := n.PeerHost.Network().InterfaceListenAddresses() if err != nil { return nil, debugerror.Wrap(err) } @@ -138,10 +141,10 @@ func NewIpfsNode(ctx context.Context, cfg *config.Config, online bool) (n *IpfsN n.Peerstore.AddAddresses(n.Identity, addrs) // setup diagnostics service - n.Diagnostics = diag.NewDiagnostics(n.Identity, n.Network) + n.Diagnostics = diag.NewDiagnostics(n.Identity, n.PeerHost) // setup routing service - dhtRouting := dht.NewDHT(ctx, n.Identity, n.Network, n.Datastore) + dhtRouting := dht.NewDHT(ctx, n.PeerHost, n.Datastore) dhtRouting.Validators[IpnsValidatorTag] = namesys.ValidateIpnsRecord // TODO(brian): perform this inside NewDHT factory method @@ -150,7 +153,7 @@ func NewIpfsNode(ctx context.Context, cfg *config.Config, online bool) (n *IpfsN // setup exchange service const alwaysSendToPeer = true // use YesManStrategy - bitswapNetwork := bsnet.NewFromIpfsNetwork(n.Network, n.Routing) + bitswapNetwork := bsnet.NewFromIpfsHost(n.PeerHost, n.Routing) n.Exchange = bitswap.New(ctx, n.Identity, bitswapNetwork, blockstore, alwaysSendToPeer) @@ -160,7 +163,7 @@ func NewIpfsNode(ctx context.Context, cfg *config.Config, online bool) (n *IpfsN // an Exchange, Network, or Routing component and have the constructor // manage the wiring. In that scenario, this dangling function is a bit // awkward. - go superviseConnections(ctx, n.Network, dhtRouting, n.Peerstore, n.Config.Bootstrap) + go superviseConnections(ctx, n.PeerHost, dhtRouting, n.Peerstore, n.Config.Bootstrap) } // TODO(brian): when offline instantiate the BlockService with a bitswap diff --git a/core/mock.go b/core/mock.go index 758b30588..b448b2517 100644 --- a/core/mock.go +++ b/core/mock.go @@ -7,16 +7,16 @@ import ( syncds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync" "github.com/jbenet/go-ipfs/blocks/blockstore" blockservice "github.com/jbenet/go-ipfs/blockservice" - ci "github.com/jbenet/go-ipfs/crypto" "github.com/jbenet/go-ipfs/exchange/offline" mdag "github.com/jbenet/go-ipfs/merkledag" nsys "github.com/jbenet/go-ipfs/namesys" - "github.com/jbenet/go-ipfs/net/mock" + ci "github.com/jbenet/go-ipfs/p2p/crypto" + mocknet "github.com/jbenet/go-ipfs/p2p/net/mock" + peer "github.com/jbenet/go-ipfs/p2p/peer" path "github.com/jbenet/go-ipfs/path" - peer "github.com/jbenet/go-ipfs/peer" dht "github.com/jbenet/go-ipfs/routing/dht" ds2 "github.com/jbenet/go-ipfs/util/datastore2" - "github.com/jbenet/go-ipfs/util/testutil" + testutil "github.com/jbenet/go-ipfs/util/testutil" ) // TODO this is super sketch. Deprecate and initialize one that shares code @@ -44,16 +44,18 @@ func NewMockNode() (*IpfsNode, error) { nd.Peerstore = peer.NewPeerstore() nd.Peerstore.AddPrivKey(p, sk) nd.Peerstore.AddPubKey(p, pk) - nd.Network, err = mocknet.New(ctx).AddPeer(sk, testutil.RandLocalTCPAddress()) // effectively offline + + nd.PeerHost, err = mocknet.New(ctx).AddPeer(sk, testutil.RandLocalTCPAddress()) // effectively offline if err != nil { return nil, err } + // Temp Datastore dstore := ds.NewMapDatastore() nd.Datastore = ds2.CloserWrap(syncds.MutexWrap(dstore)) // Routing - dht := dht.NewDHT(ctx, nd.Identity, nd.Network, nd.Datastore) + dht := dht.NewDHT(ctx, nd.PeerHost, nd.Datastore) nd.Routing = dht // Bitswap diff --git a/diagnostics/diag.go b/diagnostics/diag.go index 80a598f51..85625d3e0 100644 --- a/diagnostics/diag.go +++ b/diagnostics/diag.go @@ -17,21 +17,27 @@ import ( ggio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/io" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" + host "github.com/jbenet/go-ipfs/p2p/host" + inet "github.com/jbenet/go-ipfs/p2p/net" + peer "github.com/jbenet/go-ipfs/p2p/peer" + protocol "github.com/jbenet/go-ipfs/p2p/protocol" + pb "github.com/jbenet/go-ipfs/diagnostics/internal/pb" - net "github.com/jbenet/go-ipfs/net" - peer "github.com/jbenet/go-ipfs/peer" util "github.com/jbenet/go-ipfs/util" ) var log = util.Logger("diagnostics") +// ProtocolDiag is the diagnostics protocol.ID +var ProtocolDiag protocol.ID = "/ipfs/diagnostics" + const ResponseTimeout = time.Second * 10 // Diagnostics is a net service that manages requesting and responding to diagnostic // requests type Diagnostics struct { - network net.Network - self peer.ID + host host.Host + self peer.ID diagLock sync.Mutex diagMap map[string]time.Time @@ -39,15 +45,15 @@ type Diagnostics struct { } // NewDiagnostics instantiates a new diagnostics service running on the given network -func NewDiagnostics(self peer.ID, inet net.Network) *Diagnostics { +func NewDiagnostics(self peer.ID, h host.Host) *Diagnostics { d := &Diagnostics{ - network: inet, + host: h, self: self, birth: time.Now(), diagMap: make(map[string]time.Time), } - inet.SetHandler(net.ProtocolDiag, d.handleNewStream) + h.SetStreamHandler(ProtocolDiag, d.handleNewStream) return d } @@ -92,7 +98,7 @@ func (di *DiagInfo) Marshal() []byte { } func (d *Diagnostics) getPeers() []peer.ID { - return d.network.Peers() + return d.host.Network().Peers() } func (d *Diagnostics) getDiagInfo() *DiagInfo { @@ -101,10 +107,11 @@ func (d *Diagnostics) getDiagInfo() *DiagInfo { di.ID = d.self.Pretty() di.LifeSpan = time.Since(d.birth) di.Keys = nil // Currently no way to query datastore - di.BwIn, di.BwOut = d.network.BandwidthTotals() + + // di.BwIn, di.BwOut = d.host.BandwidthTotals() //TODO fix this. for _, p := range d.getPeers() { - d := connDiagInfo{d.network.Peerstore().LatencyEWMA(p), p.Pretty()} + d := connDiagInfo{d.host.Peerstore().LatencyEWMA(p), p.Pretty()} di.Connections = append(di.Connections, d) } return di @@ -197,13 +204,13 @@ func newMessage(diagID string) *pb.Message { func (d *Diagnostics) sendRequest(ctx context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) { - s, err := d.network.NewStream(net.ProtocolDiag, p) + s, err := d.host.NewStream(ProtocolDiag, p) if err != nil { return nil, err } defer s.Close() - r := ggio.NewDelimitedReader(s, net.MessageSizeMax) + r := ggio.NewDelimitedReader(s, inet.MessageSizeMax) w := ggio.NewDelimitedWriter(s) start := time.Now() @@ -274,7 +281,7 @@ func (d *Diagnostics) handleDiagnostic(p peer.ID, pmes *pb.Message) (*pb.Message return resp, nil } -func (d *Diagnostics) HandleMessage(ctx context.Context, s net.Stream) error { +func (d *Diagnostics) HandleMessage(ctx context.Context, s inet.Stream) error { r := ggio.NewDelimitedReader(s, 32768) // maxsize w := ggio.NewDelimitedWriter(s) @@ -312,10 +319,7 @@ func (d *Diagnostics) HandleMessage(ctx context.Context, s net.Stream) error { return nil } -func (d *Diagnostics) handleNewStream(s net.Stream) { - - go func() { - d.HandleMessage(context.Background(), s) - }() - +func (d *Diagnostics) handleNewStream(s inet.Stream) { + d.HandleMessage(context.Background(), s) + s.Close() } diff --git a/epictest/addcat_test.go b/epictest/addcat_test.go index c9ccb931a..ff6e899db 100644 --- a/epictest/addcat_test.go +++ b/epictest/addcat_test.go @@ -11,7 +11,7 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" random "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-random" - mocknet "github.com/jbenet/go-ipfs/net/mock" + mocknet "github.com/jbenet/go-ipfs/p2p/net/mock" errors "github.com/jbenet/go-ipfs/util/debugerror" ) @@ -99,11 +99,11 @@ func DirectAddCat(data []byte, conf Config) error { return errors.New("test initialization error") } - adder, err := makeCore(ctx, MocknetTestRepo(peers[0], mn.Net(peers[0]), conf)) + adder, err := makeCore(ctx, MocknetTestRepo(peers[0], mn.Host(peers[0]), conf)) if err != nil { return err } - catter, err := makeCore(ctx, MocknetTestRepo(peers[1], mn.Net(peers[1]), conf)) + catter, err := makeCore(ctx, MocknetTestRepo(peers[1], mn.Host(peers[1]), conf)) if err != nil { return err } diff --git a/epictest/core.go b/epictest/core.go index bb2f00756..13c129ec5 100644 --- a/epictest/core.go +++ b/epictest/core.go @@ -15,9 +15,9 @@ import ( importer "github.com/jbenet/go-ipfs/importer" chunk "github.com/jbenet/go-ipfs/importer/chunk" merkledag "github.com/jbenet/go-ipfs/merkledag" - net "github.com/jbenet/go-ipfs/net" + host "github.com/jbenet/go-ipfs/p2p/host" + peer "github.com/jbenet/go-ipfs/p2p/peer" path "github.com/jbenet/go-ipfs/path" - peer "github.com/jbenet/go-ipfs/peer" dht "github.com/jbenet/go-ipfs/routing/dht" uio "github.com/jbenet/go-ipfs/unixfs/io" util "github.com/jbenet/go-ipfs/util" @@ -101,7 +101,7 @@ type repo struct { blockstore blockstore.Blockstore exchange exchange.Interface datastore datastore.ThreadSafeDatastore - network net.Network + host host.Host dht *dht.IpfsDHT id peer.ID } @@ -126,16 +126,16 @@ func (r *repo) Exchange() exchange.Interface { return r.exchange } -func MocknetTestRepo(p peer.ID, n net.Network, conf Config) RepoFactory { +func MocknetTestRepo(p peer.ID, h host.Host, conf Config) RepoFactory { return func(ctx context.Context) (Repo, error) { const kWriteCacheElems = 100 const alwaysSendToPeer = true dsDelay := delay.Fixed(conf.BlockstoreLatency) ds := sync.MutexWrap(datastore2.WithDelay(datastore.NewMapDatastore(), dsDelay)) - log.Debugf("MocknetTestRepo: %s %s %s", p, n.LocalPeer(), n) - dhtt := dht.NewDHT(ctx, p, n, ds) - bsn := bsnet.NewFromIpfsNetwork(n, dhtt) + log.Debugf("MocknetTestRepo: %s %s %s", p, h.ID(), h) + dhtt := dht.NewDHT(ctx, h, ds) + bsn := bsnet.NewFromIpfsHost(h, dhtt) bstore, err := blockstore.WriteCached(blockstore.NewBlockstore(ds), kWriteCacheElems) if err != nil { return nil, err @@ -146,7 +146,7 @@ func MocknetTestRepo(p peer.ID, n net.Network, conf Config) RepoFactory { blockstore: bstore, exchange: exch, datastore: ds, - network: n, + host: h, dht: dhtt, id: p, }, nil diff --git a/epictest/three_legged_cat_test.go b/epictest/three_legged_cat_test.go index d66578305..38147e426 100644 --- a/epictest/three_legged_cat_test.go +++ b/epictest/three_legged_cat_test.go @@ -7,7 +7,7 @@ import ( "testing" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" - mocknet "github.com/jbenet/go-ipfs/net/mock" + mocknet "github.com/jbenet/go-ipfs/p2p/net/mock" errors "github.com/jbenet/go-ipfs/util/debugerror" ) @@ -42,15 +42,15 @@ func RunThreeLeggedCat(data []byte, conf Config) error { if len(peers) < numPeers { return errors.New("test initialization error") } - adder, err := makeCore(ctx, MocknetTestRepo(peers[0], mn.Net(peers[0]), conf)) + adder, err := makeCore(ctx, MocknetTestRepo(peers[0], mn.Host(peers[0]), conf)) if err != nil { return err } - catter, err := makeCore(ctx, MocknetTestRepo(peers[1], mn.Net(peers[1]), conf)) + catter, err := makeCore(ctx, MocknetTestRepo(peers[1], mn.Host(peers[1]), conf)) if err != nil { return err } - bootstrap, err := makeCore(ctx, MocknetTestRepo(peers[2], mn.Net(peers[2]), conf)) + bootstrap, err := makeCore(ctx, MocknetTestRepo(peers[2], mn.Host(peers[2]), conf)) if err != nil { return err } diff --git a/exchange/bitswap/bitswap.go b/exchange/bitswap/bitswap.go index 4ff23aee2..fe20a406a 100644 --- a/exchange/bitswap/bitswap.go +++ b/exchange/bitswap/bitswap.go @@ -17,7 +17,7 @@ import ( bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network" notifications "github.com/jbenet/go-ipfs/exchange/bitswap/notifications" wantlist "github.com/jbenet/go-ipfs/exchange/bitswap/wantlist" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" u "github.com/jbenet/go-ipfs/util" errors "github.com/jbenet/go-ipfs/util/debugerror" "github.com/jbenet/go-ipfs/util/delay" diff --git a/exchange/bitswap/decision/engine.go b/exchange/bitswap/decision/engine.go index da5ccfe6d..582d96e08 100644 --- a/exchange/bitswap/decision/engine.go +++ b/exchange/bitswap/decision/engine.go @@ -7,7 +7,7 @@ import ( bstore "github.com/jbenet/go-ipfs/blocks/blockstore" bsmsg "github.com/jbenet/go-ipfs/exchange/bitswap/message" wl "github.com/jbenet/go-ipfs/exchange/bitswap/wantlist" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" u "github.com/jbenet/go-ipfs/util" ) diff --git a/exchange/bitswap/decision/engine_test.go b/exchange/bitswap/decision/engine_test.go index 0196863b3..08e729dc8 100644 --- a/exchange/bitswap/decision/engine_test.go +++ b/exchange/bitswap/decision/engine_test.go @@ -11,7 +11,7 @@ import ( blocks "github.com/jbenet/go-ipfs/blocks" blockstore "github.com/jbenet/go-ipfs/blocks/blockstore" message "github.com/jbenet/go-ipfs/exchange/bitswap/message" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" ) type peerAndEngine struct { diff --git a/exchange/bitswap/decision/ledger.go b/exchange/bitswap/decision/ledger.go index f2b824603..273c3e706 100644 --- a/exchange/bitswap/decision/ledger.go +++ b/exchange/bitswap/decision/ledger.go @@ -4,7 +4,7 @@ import ( "time" wl "github.com/jbenet/go-ipfs/exchange/bitswap/wantlist" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" u "github.com/jbenet/go-ipfs/util" ) diff --git a/exchange/bitswap/decision/taskqueue.go b/exchange/bitswap/decision/taskqueue.go index c86a73371..11af3db35 100644 --- a/exchange/bitswap/decision/taskqueue.go +++ b/exchange/bitswap/decision/taskqueue.go @@ -4,7 +4,7 @@ import ( "sync" wantlist "github.com/jbenet/go-ipfs/exchange/bitswap/wantlist" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" u "github.com/jbenet/go-ipfs/util" ) diff --git a/exchange/bitswap/message/message.go b/exchange/bitswap/message/message.go index 7f7f1d08e..117758d9e 100644 --- a/exchange/bitswap/message/message.go +++ b/exchange/bitswap/message/message.go @@ -6,7 +6,7 @@ import ( blocks "github.com/jbenet/go-ipfs/blocks" pb "github.com/jbenet/go-ipfs/exchange/bitswap/message/internal/pb" wantlist "github.com/jbenet/go-ipfs/exchange/bitswap/wantlist" - inet "github.com/jbenet/go-ipfs/net" + inet "github.com/jbenet/go-ipfs/p2p/net" u "github.com/jbenet/go-ipfs/util" ggio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/io" diff --git a/exchange/bitswap/network/interface.go b/exchange/bitswap/network/interface.go index 1bc14ca88..7c34a352b 100644 --- a/exchange/bitswap/network/interface.go +++ b/exchange/bitswap/network/interface.go @@ -4,10 +4,13 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" bsmsg "github.com/jbenet/go-ipfs/exchange/bitswap/message" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" + protocol "github.com/jbenet/go-ipfs/p2p/protocol" u "github.com/jbenet/go-ipfs/util" ) +var ProtocolBitswap protocol.ID = "/ipfs/bitswap" + // BitSwapNetwork provides network connectivity for BitSwap sessions type BitSwapNetwork interface { diff --git a/exchange/bitswap/network/ipfs_impl.go b/exchange/bitswap/network/ipfs_impl.go index 5388c8e6d..4e349dbed 100644 --- a/exchange/bitswap/network/ipfs_impl.go +++ b/exchange/bitswap/network/ipfs_impl.go @@ -3,29 +3,29 @@ package network import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" bsmsg "github.com/jbenet/go-ipfs/exchange/bitswap/message" - inet "github.com/jbenet/go-ipfs/net" - peer "github.com/jbenet/go-ipfs/peer" + host "github.com/jbenet/go-ipfs/p2p/host" + inet "github.com/jbenet/go-ipfs/p2p/net" + peer "github.com/jbenet/go-ipfs/p2p/peer" routing "github.com/jbenet/go-ipfs/routing" util "github.com/jbenet/go-ipfs/util" ) var log = util.Logger("bitswap_network") -// NewFromIpfsNetwork returns a BitSwapNetwork supported by underlying IPFS -// Dialer & Service -func NewFromIpfsNetwork(n inet.Network, r routing.IpfsRouting) BitSwapNetwork { +// NewFromIpfsHost returns a BitSwapNetwork supported by underlying IPFS host +func NewFromIpfsHost(host host.Host, r routing.IpfsRouting) BitSwapNetwork { bitswapNetwork := impl{ - network: n, + host: host, routing: r, } - n.SetHandler(inet.ProtocolBitswap, bitswapNetwork.handleNewStream) + host.SetStreamHandler(ProtocolBitswap, bitswapNetwork.handleNewStream) return &bitswapNetwork } // impl transforms the ipfs network interface, which sends and receives // NetMessage objects, into the bitswap network interface. type impl struct { - network inet.Network + host host.Host routing routing.IpfsRouting // inbound messages from the network are forwarded to the receiver @@ -33,7 +33,7 @@ type impl struct { } func (bsnet *impl) DialPeer(ctx context.Context, p peer.ID) error { - return bsnet.network.DialPeer(ctx, p) + return bsnet.host.Connect(ctx, peer.PeerInfo{ID: p}) } func (bsnet *impl) SendMessage( @@ -41,7 +41,7 @@ func (bsnet *impl) SendMessage( p peer.ID, outgoing bsmsg.BitSwapMessage) error { - s, err := bsnet.network.NewStream(inet.ProtocolBitswap, p) + s, err := bsnet.host.NewStream(ProtocolBitswap, p) if err != nil { return err } @@ -55,7 +55,7 @@ func (bsnet *impl) SendRequest( p peer.ID, outgoing bsmsg.BitSwapMessage) (bsmsg.BitSwapMessage, error) { - s, err := bsnet.network.NewStream(inet.ProtocolBitswap, p) + s, err := bsnet.host.NewStream(ProtocolBitswap, p) if err != nil { return nil, err } @@ -79,7 +79,7 @@ func (bsnet *impl) FindProvidersAsync(ctx context.Context, k util.Key, max int) defer close(out) providers := bsnet.routing.FindProvidersAsync(ctx, k, max) for info := range providers { - bsnet.network.Peerstore().AddAddresses(info.ID, info.Addrs) + bsnet.host.Peerstore().AddAddresses(info.ID, info.Addrs) select { case <-ctx.Done(): case out <- info.ID: diff --git a/exchange/bitswap/testnet/interface.go b/exchange/bitswap/testnet/interface.go index 029ea704e..4b6f46aaf 100644 --- a/exchange/bitswap/testnet/interface.go +++ b/exchange/bitswap/testnet/interface.go @@ -2,7 +2,7 @@ package bitswap import ( bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" "github.com/jbenet/go-ipfs/util/testutil" ) diff --git a/exchange/bitswap/testnet/network_test.go b/exchange/bitswap/testnet/network_test.go index 6f6275896..bbf84995c 100644 --- a/exchange/bitswap/testnet/network_test.go +++ b/exchange/bitswap/testnet/network_test.go @@ -8,7 +8,7 @@ import ( blocks "github.com/jbenet/go-ipfs/blocks" bsmsg "github.com/jbenet/go-ipfs/exchange/bitswap/message" bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" mockrouting "github.com/jbenet/go-ipfs/routing/mock" delay "github.com/jbenet/go-ipfs/util/delay" testutil "github.com/jbenet/go-ipfs/util/testutil" diff --git a/exchange/bitswap/testnet/peernet.go b/exchange/bitswap/testnet/peernet.go index 905d78a6a..1d1d22408 100644 --- a/exchange/bitswap/testnet/peernet.go +++ b/exchange/bitswap/testnet/peernet.go @@ -4,8 +4,8 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network" - mockpeernet "github.com/jbenet/go-ipfs/net/mock" - peer "github.com/jbenet/go-ipfs/peer" + mockpeernet "github.com/jbenet/go-ipfs/p2p/net/mock" + peer "github.com/jbenet/go-ipfs/p2p/peer" mockrouting "github.com/jbenet/go-ipfs/routing/mock" testutil "github.com/jbenet/go-ipfs/util/testutil" ) @@ -25,7 +25,7 @@ func (pn *peernet) Adapter(p testutil.Identity) bsnet.BitSwapNetwork { panic(err.Error()) } routing := pn.routingserver.ClientWithDatastore(context.TODO(), p, ds.NewMapDatastore()) - return bsnet.NewFromIpfsNetwork(client, routing) + return bsnet.NewFromIpfsHost(client, routing) } func (pn *peernet) HasPeer(p peer.ID) bool { diff --git a/exchange/bitswap/testnet/virtual.go b/exchange/bitswap/testnet/virtual.go index 887d29bee..9426176a2 100644 --- a/exchange/bitswap/testnet/virtual.go +++ b/exchange/bitswap/testnet/virtual.go @@ -7,7 +7,7 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" bsmsg "github.com/jbenet/go-ipfs/exchange/bitswap/message" bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" routing "github.com/jbenet/go-ipfs/routing" mockrouting "github.com/jbenet/go-ipfs/routing/mock" util "github.com/jbenet/go-ipfs/util" diff --git a/exchange/bitswap/testutils.go b/exchange/bitswap/testutils.go index 0d1aa4fec..dd96e5f46 100644 --- a/exchange/bitswap/testutils.go +++ b/exchange/bitswap/testutils.go @@ -9,7 +9,7 @@ import ( blockstore "github.com/jbenet/go-ipfs/blocks/blockstore" exchange "github.com/jbenet/go-ipfs/exchange" tn "github.com/jbenet/go-ipfs/exchange/bitswap/testnet" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" datastore2 "github.com/jbenet/go-ipfs/util/datastore2" delay "github.com/jbenet/go-ipfs/util/delay" testutil "github.com/jbenet/go-ipfs/util/testutil" diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index 7eabe74c3..3172dd57e 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -14,9 +14,9 @@ import ( proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" core "github.com/jbenet/go-ipfs/core" - ci "github.com/jbenet/go-ipfs/crypto" chunk "github.com/jbenet/go-ipfs/importer/chunk" mdag "github.com/jbenet/go-ipfs/merkledag" + ci "github.com/jbenet/go-ipfs/p2p/crypto" ft "github.com/jbenet/go-ipfs/unixfs" uio "github.com/jbenet/go-ipfs/unixfs/io" ftpb "github.com/jbenet/go-ipfs/unixfs/pb" diff --git a/namesys/interface.go b/namesys/interface.go index eef1fc32b..c2e39afec 100644 --- a/namesys/interface.go +++ b/namesys/interface.go @@ -3,7 +3,7 @@ package namesys import ( "errors" - ci "github.com/jbenet/go-ipfs/crypto" + ci "github.com/jbenet/go-ipfs/p2p/crypto" ) // ErrResolveFailed signals an error when attempting to resolve. diff --git a/namesys/namesys.go b/namesys/namesys.go index 2ea9a30bd..cc11d9ddc 100644 --- a/namesys/namesys.go +++ b/namesys/namesys.go @@ -1,7 +1,7 @@ package namesys import ( - ci "github.com/jbenet/go-ipfs/crypto" + ci "github.com/jbenet/go-ipfs/p2p/crypto" routing "github.com/jbenet/go-ipfs/routing" ) diff --git a/namesys/publisher.go b/namesys/publisher.go index be838b2f0..75cccf9e4 100644 --- a/namesys/publisher.go +++ b/namesys/publisher.go @@ -10,8 +10,8 @@ import ( proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" - ci "github.com/jbenet/go-ipfs/crypto" pb "github.com/jbenet/go-ipfs/namesys/internal/pb" + ci "github.com/jbenet/go-ipfs/p2p/crypto" routing "github.com/jbenet/go-ipfs/routing" u "github.com/jbenet/go-ipfs/util" ) diff --git a/namesys/routing.go b/namesys/routing.go index c990b492b..709f9424c 100644 --- a/namesys/routing.go +++ b/namesys/routing.go @@ -7,8 +7,8 @@ import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" - ci "github.com/jbenet/go-ipfs/crypto" pb "github.com/jbenet/go-ipfs/namesys/internal/pb" + ci "github.com/jbenet/go-ipfs/p2p/crypto" routing "github.com/jbenet/go-ipfs/routing" u "github.com/jbenet/go-ipfs/util" ) diff --git a/net/conn/handshake.go b/net/conn/handshake.go deleted file mode 100644 index 3a995bc9a..000000000 --- a/net/conn/handshake.go +++ /dev/null @@ -1,52 +0,0 @@ -package conn - -import ( - "fmt" - - handshake "github.com/jbenet/go-ipfs/net/handshake" - hspb "github.com/jbenet/go-ipfs/net/handshake/pb" - - context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" - ggprotoio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/io" -) - -// Handshake1 exchanges local and remote versions and compares them -// closes remote and returns an error in case of major difference -func Handshake1(ctx context.Context, c Conn) error { - rpeer := c.RemotePeer() - lpeer := c.LocalPeer() - - // setup up protobuf io - maxSize := 4096 - r := ggprotoio.NewDelimitedReader(c, maxSize) - w := ggprotoio.NewDelimitedWriter(c) - localH := handshake.Handshake1Msg() - remoteH := new(hspb.Handshake1) - - // send the outgoing handshake message - if err := w.WriteMsg(localH); err != nil { - return err - } - log.Debugf("%p sent my version (%s) to %s", c, localH, rpeer) - log.Event(ctx, "handshake1Sent", lpeer) - - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - if err := r.ReadMsg(remoteH); err != nil { - return fmt.Errorf("could not receive remote version: %q", err) - } - log.Debugf("%p received remote version (%s) from %s", c, remoteH, rpeer) - log.Event(ctx, "handshake1Received", lpeer) - - if err := handshake.Handshake1Compatible(localH, remoteH); err != nil { - log.Infof("%s (%s) incompatible version with %s (%s)", lpeer, localH, rpeer, remoteH) - return err - } - - log.Debugf("%s version handshake compatible %s", lpeer, rpeer) - return nil -} diff --git a/net/handshake/README.md b/net/handshake/README.md deleted file mode 100644 index 329112c45..000000000 --- a/net/handshake/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# IFPS Handshake - -The IPFS Protocol Handshake is divided into three sequential steps - -1. Version Handshake (`Hanshake1`) -2. Secure Channel (`NewSecureConn`) -3. Services (`Handshake3`) - -Currently these parts currently happen sequentially (costing an awful 5 RTT), -but can be optimized to 2 RTT. - -### Version Handshake - -The Version Handshake ensures that nodes speaking to each other can interoperate. -They send each other protocol versions and ensure there is a match on the major -version (semver). - -### Secure Channel - -The second part exchanges keys and establishes a secure comm channel. This -follows ECDHE TLS, but *isn't* TLS. (why will be written up elsewhere). - -### Services - -The Services portion sends any additional information on nodes needed -by the nodes, e.g. Listen Address (the received address could be a Dial addr), -and later on can include Service listing (dht, exchange, ipns, etc). diff --git a/net/handshake/doc.go b/net/handshake/doc.go deleted file mode 100644 index 3c4bff006..000000000 --- a/net/handshake/doc.go +++ /dev/null @@ -1,34 +0,0 @@ -/* -package handshake implements the ipfs handshake protocol - -IPFS Handshake - -The IPFS Protocol Handshake is divided into three sequential steps - -1. Version Handshake (`Hanshake1`) - -2. Secure Channel (`NewSecureConn`) - -3. Services (`Handshake3`) - -Currently these parts currently happen sequentially (costing an awful 5 RTT), -but can be optimized to 2 RTT. - -Version Handshake - -The Version Handshake ensures that nodes speaking to each other can interoperate. -They send each other protocol versions and ensure there is a match on the major -version (semver). - -Secure Channel - -The second part exchanges keys and establishes a secure comm channel. This -follows ECDHE TLS, but *isn't* TLS. (why will be written up elsewhere). - -Services - -The Services portion sends any additional information on nodes needed -by the nodes, e.g. Listen Address (the received address could be a Dial addr), -and later on can include Service listing (dht, exchange, ipns, etc). -*/ -package handshake diff --git a/net/handshake/handshake1.go b/net/handshake/handshake1.go deleted file mode 100644 index 17ca44f09..000000000 --- a/net/handshake/handshake1.go +++ /dev/null @@ -1,68 +0,0 @@ -package handshake - -import ( - "errors" - "fmt" - - config "github.com/jbenet/go-ipfs/config" - pb "github.com/jbenet/go-ipfs/net/handshake/pb" - u "github.com/jbenet/go-ipfs/util" - - semver "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/coreos/go-semver/semver" -) - -var log = u.Logger("handshake") - -// IpfsVersion holds the current protocol version for a client running this code -var IpfsVersion *semver.Version -var ClientVersion = "go-ipfs/" + config.CurrentVersionNumber - -func init() { - var err error - IpfsVersion, err = semver.NewVersion("0.0.1") - if err != nil { - panic(fmt.Errorf("invalid protocol version: %v", err)) - } -} - -// Handshake1Msg returns the current protocol version as a protobuf message -func Handshake1Msg() *pb.Handshake1 { - return NewHandshake1(IpfsVersion.String(), ClientVersion) -} - -// ErrVersionMismatch is returned when two clients don't share a protocol version -var ErrVersionMismatch = errors.New("protocol missmatch") - -// Handshake1Compatible checks whether two versions are compatible -// returns nil if they are fine -func Handshake1Compatible(handshakeA, handshakeB *pb.Handshake1) error { - a, err := semver.NewVersion(*handshakeA.ProtocolVersion) - if err != nil { - return err - } - b, err := semver.NewVersion(*handshakeB.ProtocolVersion) - if err != nil { - return err - } - - if a.Major != b.Major { - return ErrVersionMismatch - } - - return nil -} - -// NewHandshake1 creates a new Handshake1 from the two strings -func NewHandshake1(protoVer, agentVer string) *pb.Handshake1 { - if protoVer == "" { - protoVer = IpfsVersion.String() - } - if agentVer == "" { - agentVer = ClientVersion - } - - return &pb.Handshake1{ - ProtocolVersion: &protoVer, - AgentVersion: &agentVer, - } -} diff --git a/net/handshake/handshake1_test.go b/net/handshake/handshake1_test.go deleted file mode 100644 index 78b3e6f0f..000000000 --- a/net/handshake/handshake1_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package handshake - -import "testing" - -func TestH1Compatible(t *testing.T) { - tcases := []struct { - a, b string - expected error - }{ - {"0.0.0", "0.0.0", nil}, - {"1.0.0", "1.1.0", nil}, - {"1.0.0", "1.0.1", nil}, - {"0.0.0", "1.0.0", ErrVersionMismatch}, - {"1.0.0", "0.0.0", ErrVersionMismatch}, - } - - for i, tcase := range tcases { - - if Handshake1Compatible(NewHandshake1(tcase.a, ""), NewHandshake1(tcase.b, "")) != tcase.expected { - t.Fatalf("case[%d] failed", i) - } - } -} diff --git a/net/id_test.go b/net/id_test.go deleted file mode 100644 index 524616a2d..000000000 --- a/net/id_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package net_test - -import ( - "testing" - "time" - - inet "github.com/jbenet/go-ipfs/net" - handshake "github.com/jbenet/go-ipfs/net/handshake" - peer "github.com/jbenet/go-ipfs/peer" - testutil "github.com/jbenet/go-ipfs/util/testutil" - - context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" - ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" -) - -func GenNetwork(t *testing.T, ctx context.Context) inet.Network { - p := testutil.RandPeerNetParamsOrFatal(t) - ps := peer.NewPeerstore() - ps.AddAddress(p.ID, p.Addr) - ps.AddPubKey(p.ID, p.PubKey) - ps.AddPrivKey(p.ID, p.PrivKey) - n, err := inet.NewNetwork(ctx, ps.Addresses(p.ID), p.ID, ps) - if err != nil { - t.Fatal(err) - } - return n -} - -func DivulgeAddresses(a, b inet.Network) { - id := a.LocalPeer() - addrs := a.Peerstore().Addresses(id) - b.Peerstore().AddAddresses(id, addrs) -} - -func subtestIDService(t *testing.T, postDialWait time.Duration) { - - // the generated networks should have the id service wired in. - ctx := context.Background() - n1 := GenNetwork(t, ctx) - n2 := GenNetwork(t, ctx) - - n1p := n1.LocalPeer() - n2p := n2.LocalPeer() - - testKnowsAddrs(t, n1, n2p, []ma.Multiaddr{}) // nothing - testKnowsAddrs(t, n2, n1p, []ma.Multiaddr{}) // nothing - - // have n2 tell n1, so we can dial... - DivulgeAddresses(n2, n1) - - testKnowsAddrs(t, n1, n2p, n2.Peerstore().Addresses(n2p)) // has them - testKnowsAddrs(t, n2, n1p, []ma.Multiaddr{}) // nothing - - if err := n1.DialPeer(ctx, n2p); err != nil { - t.Fatalf("Failed to dial:", err) - } - - // we need to wait here if Dial returns before ID service is finished. - if postDialWait > 0 { - <-time.After(postDialWait) - } - - // the IDService should be opened automatically, by the network. - // what we should see now is that both peers know about each others listen addresses. - testKnowsAddrs(t, n1, n2p, n2.Peerstore().Addresses(n2p)) // has them - testHasProtocolVersions(t, n1, n2p) - - // now, this wait we do have to do. it's the wait for the Listening side - // to be done identifying the connection. - c := n2.ConnsToPeer(n1.LocalPeer()) - if len(c) < 1 { - t.Fatal("should have connection by now at least.") - } - <-n2.IdentifyProtocol().IdentifyWait(c[0]) - - // and the protocol versions. - testKnowsAddrs(t, n2, n1p, n1.Peerstore().Addresses(n1p)) // has them - testHasProtocolVersions(t, n2, n1p) -} - -func testKnowsAddrs(t *testing.T, n inet.Network, p peer.ID, expected []ma.Multiaddr) { - actual := n.Peerstore().Addresses(p) - - if len(actual) != len(expected) { - t.Error("dont have the same addresses") - } - - have := map[string]struct{}{} - for _, addr := range actual { - have[addr.String()] = struct{}{} - } - for _, addr := range expected { - if _, found := have[addr.String()]; !found { - t.Errorf("%s did not have addr for %s: %s", n.LocalPeer(), p, addr) - // panic("ahhhhhhh") - } - } -} - -func testHasProtocolVersions(t *testing.T, n inet.Network, p peer.ID) { - v, err := n.Peerstore().Get(p, "ProtocolVersion") - if v == nil { - t.Error("no protocol version") - return - } - if v.(string) != handshake.IpfsVersion.String() { - t.Error("protocol mismatch", err) - } - v, err = n.Peerstore().Get(p, "AgentVersion") - if v.(string) != handshake.ClientVersion { - t.Error("agent version mismatch", err) - } -} - -// TestIDServiceWait gives the ID service 100ms to finish after dialing -// this is becasue it used to be concurrent. Now, Dial wait till the -// id service is done. -func TestIDServiceWait(t *testing.T) { - N := 3 - for i := 0; i < N; i++ { - subtestIDService(t, 100*time.Millisecond) - } -} - -func TestIDServiceNoWait(t *testing.T) { - N := 3 - for i := 0; i < N; i++ { - subtestIDService(t, 0) - } -} diff --git a/net/net.go b/net/net.go deleted file mode 100644 index 39afc6b10..000000000 --- a/net/net.go +++ /dev/null @@ -1,302 +0,0 @@ -// Package net provides an interface for ipfs to interact with the network through -package net - -import ( - "fmt" - - ic "github.com/jbenet/go-ipfs/crypto" - swarm "github.com/jbenet/go-ipfs/net/swarm" - peer "github.com/jbenet/go-ipfs/peer" - - context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" - ctxgroup "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup" - ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" -) - -type stream swarm.Stream - -func (s *stream) SwarmStream() *swarm.Stream { - return (*swarm.Stream)(s) -} - -// Conn returns the connection this stream is part of. -func (s *stream) Conn() Conn { - c := s.SwarmStream().Conn() - return (*conn_)(c) -} - -// Conn returns the connection this stream is part of. -func (s *stream) Close() error { - return s.SwarmStream().Close() -} - -// Read reads bytes from a stream. -func (s *stream) Read(p []byte) (n int, err error) { - return s.SwarmStream().Read(p) -} - -// Write writes bytes to a stream, flushing for each call. -func (s *stream) Write(p []byte) (n int, err error) { - return s.SwarmStream().Write(p) -} - -type conn_ swarm.Conn - -func (s *conn_) String() string { - return s.SwarmConn().String() -} - -func (c *conn_) SwarmConn() *swarm.Conn { - return (*swarm.Conn)(c) -} - -func (c *conn_) NewStreamWithProtocol(pr ProtocolID) (Stream, error) { - s, err := (*swarm.Conn)(c).NewStream() - if err != nil { - return nil, err - } - - ss := (*stream)(s) - - if err := WriteProtocolHeader(pr, ss); err != nil { - ss.Close() - return nil, err - } - - return ss, nil -} - -func (c *conn_) LocalMultiaddr() ma.Multiaddr { - return c.SwarmConn().LocalMultiaddr() -} - -func (c *conn_) RemoteMultiaddr() ma.Multiaddr { - return c.SwarmConn().RemoteMultiaddr() -} - -func (c *conn_) LocalPeer() peer.ID { - return c.SwarmConn().LocalPeer() -} - -func (c *conn_) RemotePeer() peer.ID { - return c.SwarmConn().RemotePeer() -} - -func (c *conn_) LocalPrivateKey() ic.PrivKey { - return c.SwarmConn().LocalPrivateKey() -} - -func (c *conn_) RemotePublicKey() ic.PubKey { - return c.SwarmConn().RemotePublicKey() -} - -// network implements the Network interface, -type network struct { - local peer.ID // local peer - mux Mux // protocol multiplexing - swarm *swarm.Swarm // peer connection multiplexing - ps peer.Peerstore - ids *IDService - - cg ctxgroup.ContextGroup // for Context closing -} - -// NewNetwork constructs a new network and starts listening on given addresses. -func NewNetwork(ctx context.Context, listen []ma.Multiaddr, local peer.ID, - peers peer.Peerstore) (Network, error) { - - s, err := swarm.NewSwarm(ctx, listen, local, peers) - if err != nil { - return nil, err - } - - n := &network{ - local: local, - swarm: s, - mux: Mux{Handlers: StreamHandlerMap{}}, - cg: ctxgroup.WithContext(ctx), - ps: peers, - } - - n.cg.SetTeardown(n.close) - n.cg.AddChildGroup(s.CtxGroup()) - - s.SetStreamHandler(func(s *swarm.Stream) { - n.mux.Handle((*stream)(s)) - }) - - // setup a conn handler that immediately "asks the other side about them" - // this is ProtocolIdentify. - n.ids = NewIDService(n) - s.SetConnHandler(n.newConnHandler) - - return n, nil -} - -func (n *network) newConnHandler(c *swarm.Conn) { - cc := (*conn_)(c) - n.ids.IdentifyConn(cc) -} - -// DialPeer attempts to establish a connection to a given peer. -// Respects the context. -func (n *network) DialPeer(ctx context.Context, p peer.ID) error { - log.Debugf("[%s] network dialing peer [%s]", n.local, p) - sc, err := n.swarm.Dial(ctx, p) - if err != nil { - return err - } - - // identify the connection before returning. - done := make(chan struct{}) - go func() { - n.ids.IdentifyConn((*conn_)(sc)) - close(done) - }() - - // respect don contexteone - select { - case <-done: - case <-ctx.Done(): - return ctx.Err() - } - - log.Debugf("network for %s finished dialing %s", n.local, p) - return nil -} - -func (n *network) Protocols() []ProtocolID { - return n.mux.Protocols() -} - -// CtxGroup returns the network's ContextGroup -func (n *network) CtxGroup() ctxgroup.ContextGroup { - return n.cg -} - -// Swarm returns the network's peerstream.Swarm -func (n *network) Swarm() *swarm.Swarm { - return n.Swarm() -} - -// LocalPeer the network's LocalPeer -func (n *network) LocalPeer() peer.ID { - return n.swarm.LocalPeer() -} - -// Peers returns the connected peers -func (n *network) Peers() []peer.ID { - return n.swarm.Peers() -} - -// Peers returns the connected peers -func (n *network) Peerstore() peer.Peerstore { - return n.ps -} - -// Conns returns the connected peers -func (n *network) Conns() []Conn { - conns1 := n.swarm.Connections() - out := make([]Conn, len(conns1)) - for i, c := range conns1 { - out[i] = (*conn_)(c) - } - return out -} - -// ConnsToPeer returns the connections in this Netowrk for given peer. -func (n *network) ConnsToPeer(p peer.ID) []Conn { - conns1 := n.swarm.ConnectionsToPeer(p) - out := make([]Conn, len(conns1)) - for i, c := range conns1 { - out[i] = (*conn_)(c) - } - return out -} - -// ClosePeer connection to peer -func (n *network) ClosePeer(p peer.ID) error { - return n.swarm.CloseConnection(p) -} - -// close is the real teardown function -func (n *network) close() error { - return n.swarm.Close() -} - -// Close calls the ContextCloser func -func (n *network) Close() error { - return n.cg.Close() -} - -// BandwidthTotals returns the total amount of bandwidth transferred -func (n *network) BandwidthTotals() (in uint64, out uint64) { - // need to implement this. probably best to do it in swarm this time. - // need a "metrics" object - return 0, 0 -} - -// ListenAddresses returns a list of addresses at which this network listens. -func (n *network) ListenAddresses() []ma.Multiaddr { - return n.swarm.ListenAddresses() -} - -// InterfaceListenAddresses returns a list of addresses at which this network -// listens. It expands "any interface" addresses (/ip4/0.0.0.0, /ip6/::) to -// use the known local interfaces. -func (n *network) InterfaceListenAddresses() ([]ma.Multiaddr, error) { - return swarm.InterfaceListenAddresses(n.swarm) -} - -// Connectedness returns a state signaling connection capabilities -// For now only returns Connected || NotConnected. Expand into more later. -func (n *network) Connectedness(p peer.ID) Connectedness { - c := n.swarm.ConnectionsToPeer(p) - if c != nil && len(c) > 0 { - return Connected - } - return NotConnected -} - -// NewStream returns a new stream to given peer p. -// If there is no connection to p, attempts to create one. -// If ProtocolID is "", writes no header. -func (n *network) NewStream(pr ProtocolID, p peer.ID) (Stream, error) { - log.Debugf("[%s] network opening stream to peer [%s]: %s", n.local, p, pr) - s, err := n.swarm.NewStreamWithPeer(p) - if err != nil { - return nil, err - } - - ss := (*stream)(s) - - if err := WriteProtocolHeader(pr, ss); err != nil { - ss.Close() - return nil, err - } - - return ss, nil -} - -// SetHandler sets the protocol handler on the Network's Muxer. -// This operation is threadsafe. -func (n *network) SetHandler(p ProtocolID, h StreamHandler) { - n.mux.SetHandler(p, h) -} - -func (n *network) String() string { - return fmt.Sprintf("", n.LocalPeer()) -} - -func (n *network) IdentifyProtocol() *IDService { - return n.ids -} - -func WriteProtocolHeader(pr ProtocolID, s Stream) error { - if pr != "" { // only write proper protocol headers - if err := WriteLengthPrefix(s, string(pr)); err != nil { - return err - } - } - return nil -} diff --git a/crypto/internal/pb/Makefile b/p2p/crypto/internal/pb/Makefile similarity index 100% rename from crypto/internal/pb/Makefile rename to p2p/crypto/internal/pb/Makefile diff --git a/crypto/internal/pb/crypto.pb.go b/p2p/crypto/internal/pb/crypto.pb.go similarity index 100% rename from crypto/internal/pb/crypto.pb.go rename to p2p/crypto/internal/pb/crypto.pb.go diff --git a/crypto/internal/pb/crypto.proto b/p2p/crypto/internal/pb/crypto.proto similarity index 100% rename from crypto/internal/pb/crypto.proto rename to p2p/crypto/internal/pb/crypto.proto diff --git a/crypto/key.go b/p2p/crypto/key.go similarity index 99% rename from crypto/key.go rename to p2p/crypto/key.go index df9a8b512..78c50dce1 100644 --- a/crypto/key.go +++ b/p2p/crypto/key.go @@ -21,7 +21,7 @@ import ( proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" - pb "github.com/jbenet/go-ipfs/crypto/internal/pb" + pb "github.com/jbenet/go-ipfs/p2p/crypto/internal/pb" u "github.com/jbenet/go-ipfs/util" ) diff --git a/crypto/key_test.go b/p2p/crypto/key_test.go similarity index 97% rename from crypto/key_test.go rename to p2p/crypto/key_test.go index 16b13a2f8..fa2ad7799 100644 --- a/crypto/key_test.go +++ b/p2p/crypto/key_test.go @@ -1,7 +1,7 @@ package crypto_test import ( - . "github.com/jbenet/go-ipfs/crypto" + . "github.com/jbenet/go-ipfs/p2p/crypto" "bytes" tu "github.com/jbenet/go-ipfs/util/testutil" diff --git a/crypto/rsa.go b/p2p/crypto/rsa.go similarity index 97% rename from crypto/rsa.go rename to p2p/crypto/rsa.go index 9f939807d..b308f6856 100644 --- a/crypto/rsa.go +++ b/p2p/crypto/rsa.go @@ -10,7 +10,7 @@ import ( proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" - pb "github.com/jbenet/go-ipfs/crypto/internal/pb" + pb "github.com/jbenet/go-ipfs/p2p/crypto/internal/pb" ) type RsaPrivateKey struct { diff --git a/crypto/secio/al.go b/p2p/crypto/secio/al.go similarity index 98% rename from crypto/secio/al.go rename to p2p/crypto/secio/al.go index e9db3ad60..2ca7d2619 100644 --- a/crypto/secio/al.go +++ b/p2p/crypto/secio/al.go @@ -15,7 +15,7 @@ import ( bfish "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.crypto/blowfish" - ci "github.com/jbenet/go-ipfs/crypto" + ci "github.com/jbenet/go-ipfs/p2p/crypto" ) // List of supported ECDH curves diff --git a/crypto/secio/interface.go b/p2p/crypto/secio/interface.go similarity index 96% rename from crypto/secio/interface.go rename to p2p/crypto/secio/interface.go index 2ae52d740..934a5fad8 100644 --- a/crypto/secio/interface.go +++ b/p2p/crypto/secio/interface.go @@ -4,12 +4,12 @@ package secio import ( "io" - ci "github.com/jbenet/go-ipfs/crypto" + ci "github.com/jbenet/go-ipfs/p2p/crypto" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" msgio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-msgio" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" ) // SessionGenerator constructs secure communication sessions for a peer. diff --git a/crypto/secio/internal/pb/Makefile b/p2p/crypto/secio/internal/pb/Makefile similarity index 100% rename from crypto/secio/internal/pb/Makefile rename to p2p/crypto/secio/internal/pb/Makefile diff --git a/crypto/secio/internal/pb/spipe.pb.go b/p2p/crypto/secio/internal/pb/spipe.pb.go similarity index 100% rename from crypto/secio/internal/pb/spipe.pb.go rename to p2p/crypto/secio/internal/pb/spipe.pb.go diff --git a/crypto/secio/internal/pb/spipe.proto b/p2p/crypto/secio/internal/pb/spipe.proto similarity index 100% rename from crypto/secio/internal/pb/spipe.proto rename to p2p/crypto/secio/internal/pb/spipe.proto diff --git a/crypto/secio/io_test.go b/p2p/crypto/secio/io_test.go similarity index 100% rename from crypto/secio/io_test.go rename to p2p/crypto/secio/io_test.go diff --git a/crypto/secio/pb/Makefile b/p2p/crypto/secio/pb/Makefile similarity index 100% rename from crypto/secio/pb/Makefile rename to p2p/crypto/secio/pb/Makefile diff --git a/crypto/secio/pb/spipe.pb.go b/p2p/crypto/secio/pb/spipe.pb.go similarity index 100% rename from crypto/secio/pb/spipe.pb.go rename to p2p/crypto/secio/pb/spipe.pb.go diff --git a/crypto/secio/pb/spipe.proto b/p2p/crypto/secio/pb/spipe.proto similarity index 100% rename from crypto/secio/pb/spipe.proto rename to p2p/crypto/secio/pb/spipe.proto diff --git a/crypto/secio/protocol.go b/p2p/crypto/secio/protocol.go similarity index 98% rename from crypto/secio/protocol.go rename to p2p/crypto/secio/protocol.go index 9dc492ff2..503c09149 100644 --- a/crypto/secio/protocol.go +++ b/p2p/crypto/secio/protocol.go @@ -10,9 +10,9 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" msgio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-msgio" - ci "github.com/jbenet/go-ipfs/crypto" - pb "github.com/jbenet/go-ipfs/crypto/secio/internal/pb" - peer "github.com/jbenet/go-ipfs/peer" + ci "github.com/jbenet/go-ipfs/p2p/crypto" + pb "github.com/jbenet/go-ipfs/p2p/crypto/secio/internal/pb" + peer "github.com/jbenet/go-ipfs/p2p/peer" u "github.com/jbenet/go-ipfs/util" eventlog "github.com/jbenet/go-ipfs/util/eventlog" ) diff --git a/crypto/secio/rw.go b/p2p/crypto/secio/rw.go similarity index 100% rename from crypto/secio/rw.go rename to p2p/crypto/secio/rw.go diff --git a/p2p/host/basic/basic_host.go b/p2p/host/basic/basic_host.go new file mode 100644 index 000000000..5d11174e5 --- /dev/null +++ b/p2p/host/basic/basic_host.go @@ -0,0 +1,149 @@ +package basichost + +import ( + context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" + + eventlog "github.com/jbenet/go-ipfs/util/eventlog" + + inet "github.com/jbenet/go-ipfs/p2p/net" + peer "github.com/jbenet/go-ipfs/p2p/peer" + protocol "github.com/jbenet/go-ipfs/p2p/protocol" + identify "github.com/jbenet/go-ipfs/p2p/protocol/identify" + relay "github.com/jbenet/go-ipfs/p2p/protocol/relay" +) + +var log = eventlog.Logger("p2p/host/basic") + +type BasicHost struct { + network inet.Network + mux protocol.Mux + ids *identify.IDService + relay *relay.RelayService +} + +// New constructs and sets up a new *BasicHost with given Network +func New(net inet.Network) *BasicHost { + h := &BasicHost{ + network: net, + mux: protocol.Mux{Handlers: protocol.StreamHandlerMap{}}, + } + + // setup host services + h.ids = identify.NewIDService(h) + h.relay = relay.NewRelayService(h, h.Mux().HandleSync) + + net.SetConnHandler(h.newConnHandler) + net.SetStreamHandler(h.newStreamHandler) + + return h +} + +// newConnHandler is the remote-opened conn handler for inet.Network +func (h *BasicHost) newConnHandler(c inet.Conn) { + h.ids.IdentifyConn(c) +} + +// newStreamHandler is the remote-opened stream handler for inet.Network +func (h *BasicHost) newStreamHandler(s inet.Stream) { + h.Mux().Handle(s) +} + +// ID returns the (local) peer.ID associated with this Host +func (h *BasicHost) ID() peer.ID { + return h.Network().LocalPeer() +} + +// Peerstore returns the Host's repository of Peer Addresses and Keys. +func (h *BasicHost) Peerstore() peer.Peerstore { + return h.Network().Peerstore() +} + +// Networks returns the Network interface of the Host +func (h *BasicHost) Network() inet.Network { + return h.network +} + +// Mux returns the Mux multiplexing incoming streams to protocol handlers +func (h *BasicHost) Mux() *protocol.Mux { + return &h.mux +} + +func (h *BasicHost) IDService() *identify.IDService { + return h.ids +} + +// SetStreamHandler sets the protocol handler on the Host's Mux. +// This is equivalent to: +// host.Mux().SetHandler(proto, handler) +// (Threadsafe) +func (h *BasicHost) SetStreamHandler(pid protocol.ID, handler inet.StreamHandler) { + h.Mux().SetHandler(pid, handler) +} + +// NewStream opens a new stream to given peer p, and writes a p2p/protocol +// header with given protocol.ID. If there is no connection to p, attempts +// to create one. If ProtocolID is "", writes no header. +// (Threadsafe) +func (h *BasicHost) NewStream(pid protocol.ID, p peer.ID) (inet.Stream, error) { + s, err := h.Network().NewStream(p) + if err != nil { + return nil, err + } + + if err := protocol.WriteHeader(s, pid); err != nil { + s.Close() + return nil, err + } + + return s, nil +} + +// Connect ensures there is a connection between this host and the peer with +// given peer.ID. Connect will absorb the addresses in pi into its internal +// peerstore. If there is not an active connection, Connect will issue a +// h.Network.Dial, and block until a connection is open, or an error is +// returned. // TODO: Relay + NAT. +func (h *BasicHost) Connect(ctx context.Context, pi peer.PeerInfo) error { + + // absorb addresses into peerstore + h.Peerstore().AddPeerInfo(pi) + + cs := h.Network().ConnsToPeer(pi.ID) + if len(cs) > 0 { + return nil + } + + return h.dialPeer(ctx, pi.ID) +} + +// dialPeer opens a connection to peer, and makes sure to identify +// the connection once it has been opened. +func (h *BasicHost) dialPeer(ctx context.Context, p peer.ID) error { + log.Debugf("host %s dialing %s", h.ID, p) + c, err := h.Network().DialPeer(ctx, p) + if err != nil { + return err + } + + // identify the connection before returning. + done := make(chan struct{}) + go func() { + h.ids.IdentifyConn(c) + close(done) + }() + + // respect don contexteone + select { + case <-done: + case <-ctx.Done(): + return ctx.Err() + } + + log.Debugf("host %s finished dialing %s", h.ID, p) + return nil +} + +// Close shuts down the Host's services (network, etc). +func (h *BasicHost) Close() error { + return h.Network().Close() +} diff --git a/p2p/host/basic/basic_host_test.go b/p2p/host/basic/basic_host_test.go new file mode 100644 index 000000000..0b79d3308 --- /dev/null +++ b/p2p/host/basic/basic_host_test.go @@ -0,0 +1,63 @@ +package basichost_test + +import ( + "bytes" + "io" + "testing" + + inet "github.com/jbenet/go-ipfs/p2p/net" + protocol "github.com/jbenet/go-ipfs/p2p/protocol" + testutil "github.com/jbenet/go-ipfs/p2p/test/util" + + context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" +) + +func TestHostSimple(t *testing.T) { + + ctx := context.Background() + h1 := testutil.GenHostSwarm(t, ctx) + h2 := testutil.GenHostSwarm(t, ctx) + defer h1.Close() + defer h2.Close() + + h2pi := h2.Peerstore().PeerInfo(h2.ID()) + if err := h1.Connect(ctx, h2pi); err != nil { + t.Fatal(err) + } + + piper, pipew := io.Pipe() + h2.SetStreamHandler(protocol.TestingID, func(s inet.Stream) { + defer s.Close() + w := io.MultiWriter(s, pipew) + io.Copy(w, s) // mirror everything + }) + + s, err := h1.NewStream(protocol.TestingID, h2pi.ID) + if err != nil { + t.Fatal(err) + } + + // write to the stream + buf1 := []byte("abcdefghijkl") + if _, err := s.Write(buf1); err != nil { + t.Fatal(err) + } + + // get it from the stream (echoed) + buf2 := make([]byte, len(buf1)) + if _, err := io.ReadFull(s, buf2); err != nil { + t.Fatal(err) + } + if !bytes.Equal(buf1, buf2) { + t.Fatal("buf1 != buf2 -- %x != %x", buf1, buf2) + } + + // get it from the pipe (tee) + buf3 := make([]byte, len(buf1)) + if _, err := io.ReadFull(piper, buf3); err != nil { + t.Fatal(err) + } + if !bytes.Equal(buf1, buf3) { + t.Fatal("buf1 != buf3 -- %x != %x", buf1, buf3) + } +} diff --git a/p2p/host/host.go b/p2p/host/host.go new file mode 100644 index 000000000..020322116 --- /dev/null +++ b/p2p/host/host.go @@ -0,0 +1,54 @@ +package host + +import ( + context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" + + eventlog "github.com/jbenet/go-ipfs/util/eventlog" + + inet "github.com/jbenet/go-ipfs/p2p/net" + peer "github.com/jbenet/go-ipfs/p2p/peer" + protocol "github.com/jbenet/go-ipfs/p2p/protocol" +) + +var log = eventlog.Logger("p2p/host") + +// Host is an object participating in a p2p network, which +// implements protocols or provides services. It handles +// requests like a Server, and issues requests like a Client. +// It is called Host because it is both Server and Client (and Peer +// may be confusing). +type Host interface { + // ID returns the (local) peer.ID associated with this Host + ID() peer.ID + + // Peerstore returns the Host's repository of Peer Addresses and Keys. + Peerstore() peer.Peerstore + + // Networks returns the Network interface of the Host + Network() inet.Network + + // Mux returns the Mux multiplexing incoming streams to protocol handlers + Mux() *protocol.Mux + + // Connect ensures there is a connection between this host and the peer with + // given peer.ID. Connect will absorb the addresses in pi into its internal + // peerstore. If there is not an active connection, Connect will issue a + // h.Network.Dial, and block until a connection is open, or an error is + // returned. // TODO: Relay + NAT. + Connect(ctx context.Context, pi peer.PeerInfo) error + + // SetStreamHandler sets the protocol handler on the Host's Mux. + // This is equivalent to: + // host.Mux().SetHandler(proto, handler) + // (Threadsafe) + SetStreamHandler(pid protocol.ID, handler inet.StreamHandler) + + // NewStream opens a new stream to given peer p, and writes a p2p/protocol + // header with given protocol.ID. If there is no connection to p, attempts + // to create one. If ProtocolID is "", writes no header. + // (Threadsafe) + NewStream(pid protocol.ID, p peer.ID) (inet.Stream, error) + + // Close shuts down the host, its Network, and services. + Close() error +} diff --git a/net/README.md b/p2p/net/README.md similarity index 100% rename from net/README.md rename to p2p/net/README.md diff --git a/net/conn/conn.go b/p2p/net/conn/conn.go similarity index 91% rename from net/conn/conn.go rename to p2p/net/conn/conn.go index 7ee303dee..65b12f23e 100644 --- a/net/conn/conn.go +++ b/p2p/net/conn/conn.go @@ -11,19 +11,14 @@ import ( ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" manet "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr-net" - ic "github.com/jbenet/go-ipfs/crypto" - peer "github.com/jbenet/go-ipfs/peer" + ic "github.com/jbenet/go-ipfs/p2p/crypto" + peer "github.com/jbenet/go-ipfs/p2p/peer" u "github.com/jbenet/go-ipfs/util" eventlog "github.com/jbenet/go-ipfs/util/eventlog" ) var log = eventlog.Logger("conn") -const ( - // MaxMessageSize is the size of the largest single message. (4MB) - MaxMessageSize = 1 << 22 -) - // ReleaseBuffer puts the given byte array back into the buffer pool, // first verifying that it is the correct size func ReleaseBuffer(b []byte) { @@ -48,15 +43,8 @@ func newSingleConn(ctx context.Context, local, remote peer.ID, maconn manet.Conn maconn: maconn, msgrw: msgio.NewReadWriter(maconn), } + log.Debugf("newSingleConn %p: %v to %v", conn, local, remote) - - // version handshake - if err := Handshake1(ctx, conn); err != nil { - conn.Close() - return nil, fmt.Errorf("Handshake1 failed: %s", err) - } - - log.Debugf("newSingleConn %p: %v to %v finished", conn, local, remote) return conn, nil } diff --git a/net/conn/conn_test.go b/p2p/net/conn/conn_test.go similarity index 100% rename from net/conn/conn_test.go rename to p2p/net/conn/conn_test.go diff --git a/net/conn/dial.go b/p2p/net/conn/dial.go similarity index 98% rename from net/conn/dial.go rename to p2p/net/conn/dial.go index 8ffb441d3..1294f2241 100644 --- a/net/conn/dial.go +++ b/p2p/net/conn/dial.go @@ -8,7 +8,7 @@ import ( ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" manet "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr-net" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" debugerror "github.com/jbenet/go-ipfs/util/debugerror" ) diff --git a/net/conn/dial_test.go b/p2p/net/conn/dial_test.go similarity index 94% rename from net/conn/dial_test.go rename to p2p/net/conn/dial_test.go index bf24ab09a..b757a1b69 100644 --- a/net/conn/dial_test.go +++ b/p2p/net/conn/dial_test.go @@ -51,7 +51,6 @@ func setupConn(t *testing.T, ctx context.Context, secure bool) (a, b Conn, p1, p p1 = tu.RandPeerNetParamsOrFatal(t) p2 = tu.RandPeerNetParamsOrFatal(t) - laddr := p1.Addr key1 := p1.PrivKey key2 := p2.PrivKey @@ -59,10 +58,11 @@ func setupConn(t *testing.T, ctx context.Context, secure bool) (a, b Conn, p1, p key1 = nil key2 = nil } - l1, err := Listen(ctx, laddr, p1.ID, key1) + l1, err := Listen(ctx, p1.Addr, p1.ID, key1) if err != nil { t.Fatal(err) } + p1.Addr = l1.Multiaddr() // Addr has been determined by kernel. d2 := &Dialer{ LocalPeer: p2.ID, @@ -110,6 +110,7 @@ func testDialer(t *testing.T, secure bool) { if err != nil { t.Fatal(err) } + p1.Addr = l1.Multiaddr() // Addr has been determined by kernel. d2 := &Dialer{ LocalPeer: p2.ID, diff --git a/net/conn/interface.go b/p2p/net/conn/interface.go similarity index 96% rename from net/conn/interface.go rename to p2p/net/conn/interface.go index d42a9a97a..1601da1aa 100644 --- a/net/conn/interface.go +++ b/p2p/net/conn/interface.go @@ -5,8 +5,8 @@ import ( "net" "time" - ic "github.com/jbenet/go-ipfs/crypto" - peer "github.com/jbenet/go-ipfs/peer" + ic "github.com/jbenet/go-ipfs/p2p/crypto" + peer "github.com/jbenet/go-ipfs/p2p/peer" u "github.com/jbenet/go-ipfs/util" msgio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-msgio" diff --git a/net/conn/listen.go b/p2p/net/conn/listen.go similarity index 80% rename from net/conn/listen.go rename to p2p/net/conn/listen.go index e8eddb997..0cc1a75c8 100644 --- a/net/conn/listen.go +++ b/p2p/net/conn/listen.go @@ -9,33 +9,32 @@ import ( ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" manet "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr-net" - ic "github.com/jbenet/go-ipfs/crypto" - peer "github.com/jbenet/go-ipfs/peer" + ic "github.com/jbenet/go-ipfs/p2p/crypto" + peer "github.com/jbenet/go-ipfs/p2p/peer" ) // listener is an object that can accept connections. It implements Listener type listener struct { manet.Listener - maddr ma.Multiaddr // Local multiaddr to listen on - local peer.ID // LocalPeer is the identity of the local Peer - privk ic.PrivKey // private key to use to initialize secure conns + local peer.ID // LocalPeer is the identity of the local Peer + privk ic.PrivKey // private key to use to initialize secure conns cg ctxgroup.ContextGroup } func (l *listener) teardown() error { - defer log.Debugf("listener closed: %s %s", l.local, l.maddr) + defer log.Debugf("listener closed: %s %s", l.local, l.Multiaddr()) return l.Listener.Close() } func (l *listener) Close() error { - log.Debugf("listener closing: %s %s", l.local, l.maddr) + log.Debugf("listener closing: %s %s", l.local, l.Multiaddr()) return l.cg.Close() } func (l *listener) String() string { - return fmt.Sprintf("", l.local, l.maddr) + return fmt.Sprintf("", l.local, l.Multiaddr()) } // Accept waits for and returns the next connection to the listener. @@ -73,8 +72,14 @@ func (l *listener) Addr() net.Addr { } // Multiaddr is the identity of the local Peer. +// If there is an error converting from net.Addr to ma.Multiaddr, +// the return value will be nil. func (l *listener) Multiaddr() ma.Multiaddr { - return l.maddr + maddr, err := manet.FromNetAddr(l.Addr()) + if err != nil { + return nil // error + } + return maddr } // LocalPeer is the identity of the local Peer. @@ -102,7 +107,6 @@ func Listen(ctx context.Context, addr ma.Multiaddr, local peer.ID, sk ic.PrivKey l := &listener{ Listener: ml, - maddr: addr, local: local, privk: sk, cg: ctxgroup.WithContext(ctx), diff --git a/net/conn/secure_conn.go b/p2p/net/conn/secure_conn.go similarity index 96% rename from net/conn/secure_conn.go rename to p2p/net/conn/secure_conn.go index dda1f5b55..6d8cca6d5 100644 --- a/net/conn/secure_conn.go +++ b/p2p/net/conn/secure_conn.go @@ -8,9 +8,9 @@ import ( msgio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-msgio" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - ic "github.com/jbenet/go-ipfs/crypto" - secio "github.com/jbenet/go-ipfs/crypto/secio" - peer "github.com/jbenet/go-ipfs/peer" + ic "github.com/jbenet/go-ipfs/p2p/crypto" + secio "github.com/jbenet/go-ipfs/p2p/crypto/secio" + peer "github.com/jbenet/go-ipfs/p2p/peer" errors "github.com/jbenet/go-ipfs/util/debugerror" ) diff --git a/net/conn/secure_conn_test.go b/p2p/net/conn/secure_conn_test.go similarity index 98% rename from net/conn/secure_conn_test.go rename to p2p/net/conn/secure_conn_test.go index 7b400da06..7e364d12b 100644 --- a/net/conn/secure_conn_test.go +++ b/p2p/net/conn/secure_conn_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - ic "github.com/jbenet/go-ipfs/crypto" + ic "github.com/jbenet/go-ipfs/p2p/crypto" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ) diff --git a/net/interface.go b/p2p/net/interface.go similarity index 67% rename from net/interface.go rename to p2p/net/interface.go index 7f9f3e617..74add2bb6 100644 --- a/net/interface.go +++ b/p2p/net/interface.go @@ -3,28 +3,14 @@ package net import ( "io" - conn "github.com/jbenet/go-ipfs/net/conn" - // swarm "github.com/jbenet/go-ipfs/net/swarm2" - peer "github.com/jbenet/go-ipfs/peer" + conn "github.com/jbenet/go-ipfs/p2p/net/conn" + peer "github.com/jbenet/go-ipfs/p2p/peer" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ctxgroup "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" ) -// ProtocolID is an identifier used to write protocol headers in streams. -type ProtocolID string - -// These are the ProtocolIDs of the protocols running. It is useful -// to keep them in one place. -const ( - ProtocolTesting ProtocolID = "/ipfs/testing" - ProtocolBitswap ProtocolID = "/ipfs/bitswap" - ProtocolDHT ProtocolID = "/ipfs/dht" - ProtocolIdentify ProtocolID = "/ipfs/id" - ProtocolDiag ProtocolID = "/ipfs/diagnostics" -) - // MessageSizeMax is a soft (recommended) maximum for network messages. // One can write more, as the interface is a stream. But it is useful // to bunch it up into multiple read/writes when the whole message is @@ -44,12 +30,10 @@ type Stream interface { Conn() Conn } -// StreamHandler is the function protocols who wish to listen to -// incoming streams must implement. +// StreamHandler is the type of function used to listen for +// streams opened by the remote side. type StreamHandler func(Stream) -type StreamHandlerMap map[ProtocolID]StreamHandler - // Conn is a connection to a remote peer. It multiplexes streams. // Usually there is no need to use a Conn directly, but it may // be useful to get information about the peer on the other side: @@ -57,11 +41,15 @@ type StreamHandlerMap map[ProtocolID]StreamHandler type Conn interface { conn.PeerConn - // NewStreamWithProtocol constructs a new Stream over this conn. - NewStreamWithProtocol(pr ProtocolID) (Stream, error) + // NewStream constructs a new Stream over this conn. + NewStream() (Stream, error) } -// Network is the interface IPFS uses for connecting to the world. +// ConnHandler is the type of function used to listen for +// connections opened by the remote side. +type ConnHandler func(Conn) + +// Network is the interface used to connect to the outside world. // It dials and listens for connections. it uses a Swarm to pool // connnections (see swarm pkg, and peerstream.Swarm). Connections // are encrypted with a TLS-like protocol. @@ -69,22 +57,17 @@ type Network interface { Dialer io.Closer - // SetHandler sets the protocol handler on the Network's Muxer. - // This operation is threadsafe. - SetHandler(ProtocolID, StreamHandler) + // SetStreamHandler sets the handler for new streams opened by the + // remote side. This operation is threadsafe. + SetStreamHandler(StreamHandler) - // Protocols returns the list of protocols this network currently - // has registered handlers for. - Protocols() []ProtocolID + // SetConnHandler sets the handler for new connections opened by the + // remote side. This operation is threadsafe. + SetConnHandler(ConnHandler) // NewStream returns a new stream to given peer p. // If there is no connection to p, attempts to create one. - // If ProtocolID is "", writes no header. - NewStream(ProtocolID, peer.ID) (Stream, error) - - // BandwidthTotals returns the total number of bytes passed through - // the network since it was instantiated - BandwidthTotals() (uint64, uint64) + NewStream(peer.ID) (Stream, error) // ListenAddresses returns a list of addresses at which this network listens. ListenAddresses() []ma.Multiaddr @@ -96,11 +79,6 @@ type Network interface { // CtxGroup returns the network's contextGroup CtxGroup() ctxgroup.ContextGroup - - // IdentifyProtocol returns the instance of the object running the Identify - // Protocol. This is what runs the ifps handshake-- this should be removed - // if this abstracted out to its own package. - IdentifyProtocol() *IDService } // Dialer represents a service that can dial out to peers @@ -116,8 +94,8 @@ type Dialer interface { // LocalPeer returns the local peer associated with this network LocalPeer() peer.ID - // DialPeer attempts to establish a connection to a given peer - DialPeer(context.Context, peer.ID) error + // DialPeer establishes a connection to a given peer + DialPeer(context.Context, peer.ID) (Conn, error) // ClosePeer closes the connection to a given peer ClosePeer(peer.ID) error diff --git a/net/mock/interface.go b/p2p/net/mock/interface.go similarity index 86% rename from net/mock/interface.go rename to p2p/net/mock/interface.go index be6bd4bb2..fac6dd747 100644 --- a/net/mock/interface.go +++ b/p2p/net/mock/interface.go @@ -10,9 +10,10 @@ import ( "io" "time" - ic "github.com/jbenet/go-ipfs/crypto" - inet "github.com/jbenet/go-ipfs/net" - peer "github.com/jbenet/go-ipfs/peer" + ic "github.com/jbenet/go-ipfs/p2p/crypto" + host "github.com/jbenet/go-ipfs/p2p/host" + inet "github.com/jbenet/go-ipfs/p2p/net" + peer "github.com/jbenet/go-ipfs/p2p/peer" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" ) @@ -20,16 +21,18 @@ import ( type Mocknet interface { // GenPeer generates a peer and its inet.Network in the Mocknet - GenPeer() (inet.Network, error) + GenPeer() (host.Host, error) // AddPeer adds an existing peer. we need both a privkey and addr. // ID is derived from PrivKey - AddPeer(ic.PrivKey, ma.Multiaddr) (inet.Network, error) + AddPeer(ic.PrivKey, ma.Multiaddr) (host.Host, error) // retrieve things (with randomized iteration order) Peers() []peer.ID Net(peer.ID) inet.Network Nets() []inet.Network + Host(peer.ID) host.Host + Hosts() []host.Host Links() LinkMap LinksBetweenPeers(a, b peer.ID) []Link LinksBetweenNets(a, b inet.Network) []Link @@ -52,8 +55,8 @@ type Mocknet interface { // Connections are the usual. Connecting means Dialing. // **to succeed, peers must be linked beforehand** - ConnectPeers(peer.ID, peer.ID) error - ConnectNets(inet.Network, inet.Network) error + ConnectPeers(peer.ID, peer.ID) (inet.Conn, error) + ConnectNets(inet.Network, inet.Network) (inet.Conn, error) DisconnectPeers(peer.ID, peer.ID) error DisconnectNets(inet.Network, inet.Network) error } diff --git a/net/mock/mock.go b/p2p/net/mock/mock.go similarity index 96% rename from net/mock/mock.go rename to p2p/net/mock/mock.go index b892a74ed..5403358cb 100644 --- a/net/mock/mock.go +++ b/p2p/net/mock/mock.go @@ -53,7 +53,7 @@ func FullMeshConnected(ctx context.Context, n int) (Mocknet, error) { nets := m.Nets() for _, n1 := range nets { for _, n2 := range nets { - if err := m.ConnectNets(n1, n2); err != nil { + if _, err := m.ConnectNets(n1, n2); err != nil { return nil, err } } diff --git a/net/mock/mock_conn.go b/p2p/net/mock/mock_conn.go similarity index 88% rename from net/mock/mock_conn.go rename to p2p/net/mock/mock_conn.go index e92ff6fa4..7e58eaae5 100644 --- a/net/mock/mock_conn.go +++ b/p2p/net/mock/mock_conn.go @@ -4,9 +4,9 @@ import ( "container/list" "sync" - ic "github.com/jbenet/go-ipfs/crypto" - inet "github.com/jbenet/go-ipfs/net" - peer "github.com/jbenet/go-ipfs/peer" + ic "github.com/jbenet/go-ipfs/p2p/crypto" + inet "github.com/jbenet/go-ipfs/p2p/net" + peer "github.com/jbenet/go-ipfs/p2p/peer" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" ) @@ -82,14 +82,10 @@ func (c *conn) openStream() *stream { return sl } -func (c *conn) NewStreamWithProtocol(pr inet.ProtocolID) (inet.Stream, error) { +func (c *conn) NewStream() (inet.Stream, error) { log.Debugf("Conn.NewStreamWithProtocol: %s --> %s", c.local, c.remote) s := c.openStream() - if err := inet.WriteProtocolHeader(pr, s); err != nil { - s.Close() - return nil, err - } return s, nil } diff --git a/net/mock/mock_link.go b/p2p/net/mock/mock_link.go similarity index 94% rename from net/mock/mock_link.go rename to p2p/net/mock/mock_link.go index 38d99886c..cfd4be142 100644 --- a/net/mock/mock_link.go +++ b/p2p/net/mock/mock_link.go @@ -4,8 +4,8 @@ import ( "io" "sync" - inet "github.com/jbenet/go-ipfs/net" - peer "github.com/jbenet/go-ipfs/peer" + inet "github.com/jbenet/go-ipfs/p2p/net" + peer "github.com/jbenet/go-ipfs/p2p/peer" ) // link implements mocknet.Link diff --git a/net/mock/mock_net.go b/p2p/net/mock/mock_net.go similarity index 84% rename from net/mock/mock_net.go rename to p2p/net/mock/mock_net.go index 2be03f836..d380e5173 100644 --- a/net/mock/mock_net.go +++ b/p2p/net/mock/mock_net.go @@ -3,10 +3,13 @@ package mocknet import ( "fmt" "sync" + "time" - ic "github.com/jbenet/go-ipfs/crypto" - inet "github.com/jbenet/go-ipfs/net" - peer "github.com/jbenet/go-ipfs/peer" + ic "github.com/jbenet/go-ipfs/p2p/crypto" + host "github.com/jbenet/go-ipfs/p2p/host" + bhost "github.com/jbenet/go-ipfs/p2p/host/basic" + inet "github.com/jbenet/go-ipfs/p2p/net" + peer "github.com/jbenet/go-ipfs/p2p/peer" testutil "github.com/jbenet/go-ipfs/util/testutil" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" @@ -16,9 +19,8 @@ import ( // mocknet implements mocknet.Mocknet type mocknet struct { - // must map on peer.ID (instead of peer.ID) because - // each inet.Network has different peerstore - nets map[peer.ID]*peernet + nets map[peer.ID]*peernet + hosts map[peer.ID]*bhost.BasicHost // links make it possible to connect two peers. // think of links as the physical medium. @@ -35,33 +37,36 @@ type mocknet struct { func New(ctx context.Context) Mocknet { return &mocknet{ nets: map[peer.ID]*peernet{}, + hosts: map[peer.ID]*bhost.BasicHost{}, links: map[peer.ID]map[peer.ID]map[*link]struct{}{}, cg: ctxgroup.WithContext(ctx), } } -func (mn *mocknet) GenPeer() (inet.Network, error) { - sk, _, err := testutil.SeededKeyPair(int64(len(mn.nets))) +func (mn *mocknet) GenPeer() (host.Host, error) { + sk, _, err := testutil.SeededKeyPair(time.Now().UnixNano()) if err != nil { return nil, err } a := testutil.RandLocalTCPAddress() - n, err := mn.AddPeer(sk, a) + h, err := mn.AddPeer(sk, a) if err != nil { return nil, err } - return n, nil + return h, nil } -func (mn *mocknet) AddPeer(k ic.PrivKey, a ma.Multiaddr) (inet.Network, error) { +func (mn *mocknet) AddPeer(k ic.PrivKey, a ma.Multiaddr) (host.Host, error) { n, err := newPeernet(mn.cg.Context(), mn, k, a) if err != nil { return nil, err } + h := bhost.New(n) + // make sure to add listening address! // this makes debugging things simpler as remembering to register // an address may cause unexpected failure. @@ -72,8 +77,9 @@ func (mn *mocknet) AddPeer(k ic.PrivKey, a ma.Multiaddr) (inet.Network, error) { mn.Lock() mn.nets[n.peer] = n + mn.hosts[n.peer] = h mn.Unlock() - return n, nil + return h, nil } func (mn *mocknet) Peers() []peer.ID { @@ -87,16 +93,29 @@ func (mn *mocknet) Peers() []peer.ID { return cp } +func (mn *mocknet) Host(pid peer.ID) host.Host { + mn.RLock() + host := mn.hosts[pid] + mn.RUnlock() + return host +} + func (mn *mocknet) Net(pid peer.ID) inet.Network { + mn.RLock() + n := mn.nets[pid] + mn.RUnlock() + return n +} + +func (mn *mocknet) Hosts() []host.Host { mn.RLock() defer mn.RUnlock() - for _, n := range mn.nets { - if n.peer == pid { - return n - } + cp := make([]host.Host, 0, len(mn.hosts)) + for _, h := range mn.hosts { + cp = append(cp, h) } - return nil + return cp } func (mn *mocknet) Nets() []inet.Network { @@ -269,7 +288,7 @@ func (mn *mocknet) ConnectAll() error { continue } - if err := mn.ConnectNets(n1, n2); err != nil { + if _, err := mn.ConnectNets(n1, n2); err != nil { return err } } @@ -277,11 +296,11 @@ func (mn *mocknet) ConnectAll() error { return nil } -func (mn *mocknet) ConnectPeers(a, b peer.ID) error { +func (mn *mocknet) ConnectPeers(a, b peer.ID) (inet.Conn, error) { return mn.Net(a).DialPeer(mn.cg.Context(), b) } -func (mn *mocknet) ConnectNets(a, b inet.Network) error { +func (mn *mocknet) ConnectNets(a, b inet.Network) (inet.Conn, error) { return a.DialPeer(mn.cg.Context(), b.LocalPeer()) } diff --git a/net/mock/mock_peernet.go b/p2p/net/mock/mock_peernet.go similarity index 83% rename from net/mock/mock_peernet.go rename to p2p/net/mock/mock_peernet.go index 51ffb1e72..18f1ff451 100644 --- a/net/mock/mock_peernet.go +++ b/p2p/net/mock/mock_peernet.go @@ -5,9 +5,9 @@ import ( "math/rand" "sync" - ic "github.com/jbenet/go-ipfs/crypto" - inet "github.com/jbenet/go-ipfs/net" - peer "github.com/jbenet/go-ipfs/peer" + ic "github.com/jbenet/go-ipfs/p2p/crypto" + inet "github.com/jbenet/go-ipfs/p2p/net" + peer "github.com/jbenet/go-ipfs/p2p/peer" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ctxgroup "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup" @@ -27,9 +27,9 @@ type peernet struct { connsByPeer map[peer.ID]map[*conn]struct{} connsByLink map[*link]map[*conn]struct{} - // needed to implement inet.Network - mux inet.Mux - ids *inet.IDService + // implement inet.Network + streamHandler inet.StreamHandler + connHandler inet.ConnHandler cg ctxgroup.ContextGroup sync.RWMutex @@ -54,7 +54,6 @@ func newPeernet(ctx context.Context, m *mocknet, k ic.PrivKey, mocknet: m, peer: p, ps: ps, - mux: inet.Mux{Handlers: inet.StreamHandlerMap{}}, cg: ctxgroup.WithContext(ctx), connsByPeer: map[peer.ID]map[*conn]struct{}{}, @@ -62,11 +61,6 @@ func newPeernet(ctx context.Context, m *mocknet, k ic.PrivKey, } n.cg.SetTeardown(n.teardown) - - // setup a conn handler that immediately "asks the other side about them" - // this is ProtocolIdentify. - n.ids = inet.NewIDService(n) - return n, nil } @@ -97,10 +91,6 @@ func (pn *peernet) Close() error { return pn.cg.Close() } -func (pn *peernet) Protocols() []inet.ProtocolID { - return pn.mux.Protocols() -} - func (pn *peernet) Peerstore() peer.Peerstore { return pn.ps } @@ -109,24 +99,41 @@ func (pn *peernet) String() string { return fmt.Sprintf("", pn.peer, len(pn.allConns())) } -// handleNewStream is an internal function to trigger the muxer handler +// handleNewStream is an internal function to trigger the client's handler func (pn *peernet) handleNewStream(s inet.Stream) { - go pn.mux.Handle(s) + pn.RLock() + handler := pn.streamHandler + pn.RUnlock() + if handler != nil { + go handler(s) + } +} + +// handleNewConn is an internal function to trigger the client's handler +func (pn *peernet) handleNewConn(c inet.Conn) { + pn.RLock() + handler := pn.connHandler + pn.RUnlock() + if handler != nil { + go handler(c) + } } // DialPeer attempts to establish a connection to a given peer. // Respects the context. -func (pn *peernet) DialPeer(ctx context.Context, p peer.ID) error { +func (pn *peernet) DialPeer(ctx context.Context, p peer.ID) (inet.Conn, error) { return pn.connect(p) } -func (pn *peernet) connect(p peer.ID) error { +func (pn *peernet) connect(p peer.ID) (*conn, error) { // first, check if we already have live connections pn.RLock() cs, found := pn.connsByPeer[p] pn.RUnlock() if found && len(cs) > 0 { - return nil + for c := range cs { + return c, nil + } } log.Debugf("%s (newly) dialing %s", pn.peer, p) @@ -134,7 +141,7 @@ func (pn *peernet) connect(p peer.ID) error { // ok, must create a new connection. we need a link links := pn.mocknet.LinksBetweenPeers(pn.peer, p) if len(links) < 1 { - return fmt.Errorf("%s cannot connect to %s", pn.peer, p) + return nil, fmt.Errorf("%s cannot connect to %s", pn.peer, p) } // if many links found, how do we select? for now, randomly... @@ -144,8 +151,8 @@ func (pn *peernet) connect(p peer.ID) error { log.Debugf("%s dialing %s openingConn", pn.peer, p) // create a new connection with link - pn.openConn(p, l.(*link)) - return nil + c := pn.openConn(p, l.(*link)) + return c, nil } func (pn *peernet) openConn(r peer.ID, l *link) *conn { @@ -159,16 +166,15 @@ func (pn *peernet) openConn(r peer.ID, l *link) *conn { func (pn *peernet) remoteOpenedConn(c *conn) { log.Debugf("%s accepting connection from %s", pn.LocalPeer(), c.RemotePeer()) pn.addConn(c) + pn.handleNewConn(c) } // addConn constructs and adds a connection // to given remote peer over given link func (pn *peernet) addConn(c *conn) { - - // run the Identify protocol/handshake. - pn.ids.IdentifyConn(c) - pn.Lock() + defer pn.Unlock() + cs, found := pn.connsByPeer[c.RemotePeer()] if !found { cs = map[*conn]struct{}{} @@ -182,7 +188,6 @@ func (pn *peernet) addConn(c *conn) { pn.connsByLink[c.link] = cs } pn.connsByLink[c.link][c] = struct{}{} - pn.Unlock() } // removeConn removes a given conn @@ -307,15 +312,14 @@ func (pn *peernet) Connectedness(p peer.ID) inet.Connectedness { // NewStream returns a new stream to given peer p. // If there is no connection to p, attempts to create one. -// If ProtocolID is "", writes no header. -func (pn *peernet) NewStream(pr inet.ProtocolID, p peer.ID) (inet.Stream, error) { +func (pn *peernet) NewStream(p peer.ID) (inet.Stream, error) { pn.Lock() - defer pn.Unlock() - cs, found := pn.connsByPeer[p] if !found || len(cs) < 1 { + pn.Unlock() return nil, fmt.Errorf("no connection to peer") } + pn.Unlock() // if many conns are found, how do we select? for now, randomly... // this would be an interesting place to test logic that can measure @@ -329,15 +333,21 @@ func (pn *peernet) NewStream(pr inet.ProtocolID, p peer.ID) (inet.Stream, error) n-- } - return c.NewStreamWithProtocol(pr) + return c.NewStream() } -// SetHandler sets the protocol handler on the Network's Muxer. +// SetStreamHandler sets the new stream handler on the Network. // This operation is threadsafe. -func (pn *peernet) SetHandler(p inet.ProtocolID, h inet.StreamHandler) { - pn.mux.SetHandler(p, h) +func (pn *peernet) SetStreamHandler(h inet.StreamHandler) { + pn.Lock() + pn.streamHandler = h + pn.Unlock() } -func (pn *peernet) IdentifyProtocol() *inet.IDService { - return pn.ids +// SetConnHandler sets the new conn handler on the Network. +// This operation is threadsafe. +func (pn *peernet) SetConnHandler(h inet.ConnHandler) { + pn.Lock() + pn.connHandler = h + pn.Unlock() } diff --git a/net/mock/mock_printer.go b/p2p/net/mock/mock_printer.go similarity index 89% rename from net/mock/mock_printer.go rename to p2p/net/mock/mock_printer.go index 4adb7b544..47b6e8178 100644 --- a/net/mock/mock_printer.go +++ b/p2p/net/mock/mock_printer.go @@ -4,8 +4,8 @@ import ( "fmt" "io" - inet "github.com/jbenet/go-ipfs/net" - peer "github.com/jbenet/go-ipfs/peer" + inet "github.com/jbenet/go-ipfs/p2p/net" + peer "github.com/jbenet/go-ipfs/p2p/peer" ) // separate object so our interfaces are separate :) diff --git a/net/mock/mock_stream.go b/p2p/net/mock/mock_stream.go similarity index 89% rename from net/mock/mock_stream.go rename to p2p/net/mock/mock_stream.go index 9bf3f7d46..71a0ba66d 100644 --- a/net/mock/mock_stream.go +++ b/p2p/net/mock/mock_stream.go @@ -3,7 +3,7 @@ package mocknet import ( "io" - inet "github.com/jbenet/go-ipfs/net" + inet "github.com/jbenet/go-ipfs/p2p/net" ) // stream implements inet.Stream diff --git a/net/mock/mock_test.go b/p2p/net/mock/mock_test.go similarity index 75% rename from net/mock/mock_test.go rename to p2p/net/mock/mock_test.go index be10cabea..268b35f4d 100644 --- a/net/mock/mock_test.go +++ b/p2p/net/mock/mock_test.go @@ -7,8 +7,9 @@ import ( "sync" "testing" - inet "github.com/jbenet/go-ipfs/net" - peer "github.com/jbenet/go-ipfs/peer" + inet "github.com/jbenet/go-ipfs/p2p/net" + peer "github.com/jbenet/go-ipfs/p2p/peer" + protocol "github.com/jbenet/go-ipfs/p2p/protocol" testutil "github.com/jbenet/go-ipfs/util/testutil" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" @@ -46,31 +47,44 @@ func TestNetworkSetup(t *testing.T) { a2 := testutil.RandLocalTCPAddress() a3 := testutil.RandLocalTCPAddress() - n1, err := mn.AddPeer(sk1, a1) + h1, err := mn.AddPeer(sk1, a1) if err != nil { t.Fatal(err) } - p1 := n1.LocalPeer() + p1 := h1.ID() - n2, err := mn.AddPeer(sk2, a2) + h2, err := mn.AddPeer(sk2, a2) if err != nil { t.Fatal(err) } - p2 := n2.LocalPeer() + p2 := h2.ID() - n3, err := mn.AddPeer(sk3, a3) + h3, err := mn.AddPeer(sk3, a3) if err != nil { t.Fatal(err) } - p3 := n3.LocalPeer() + p3 := h3.ID() // check peers and net + if mn.Host(p1) != h1 { + t.Error("host for p1.ID != h1") + } + if mn.Host(p2) != h2 { + t.Error("host for p2.ID != h2") + } + if mn.Host(p3) != h3 { + t.Error("host for p3.ID != h3") + } + + n1 := h1.Network() if mn.Net(p1) != n1 { t.Error("net for p1.ID != n1") } + n2 := h2.Network() if mn.Net(p2) != n2 { t.Error("net for p2.ID != n1") } + n3 := h3.Network() if mn.Net(p3) != n3 { t.Error("net for p3.ID != n1") } @@ -177,7 +191,7 @@ func TestNetworkSetup(t *testing.T) { } // connect p2->p3 - if err := n2.DialPeer(ctx, p3); err != nil { + if _, err := n2.DialPeer(ctx, p3); err != nil { t.Error(err) } @@ -191,41 +205,41 @@ func TestNetworkSetup(t *testing.T) { // p.NetworkConns(n3) // can create a stream 2->3, 3->2, - if _, err := n2.NewStream(inet.ProtocolDiag, p3); err != nil { + if _, err := n2.NewStream(p3); err != nil { t.Error(err) } - if _, err := n3.NewStream(inet.ProtocolDiag, p2); err != nil { + if _, err := n3.NewStream(p2); err != nil { t.Error(err) } // but not 1->2 nor 2->2 (not linked), nor 1->1 (not connected) - if _, err := n1.NewStream(inet.ProtocolDiag, p2); err == nil { + if _, err := n1.NewStream(p2); err == nil { t.Error("should not be able to connect") } - if _, err := n2.NewStream(inet.ProtocolDiag, p2); err == nil { + if _, err := n2.NewStream(p2); err == nil { t.Error("should not be able to connect") } - if _, err := n1.NewStream(inet.ProtocolDiag, p1); err == nil { + if _, err := n1.NewStream(p1); err == nil { t.Error("should not be able to connect") } // connect p1->p1 (should work) - if err := n1.DialPeer(ctx, p1); err != nil { + if _, err := n1.DialPeer(ctx, p1); err != nil { t.Error("p1 should be able to dial self.", err) } // and a stream too - if _, err := n1.NewStream(inet.ProtocolDiag, p1); err != nil { + if _, err := n1.NewStream(p1); err != nil { t.Error(err) } // connect p1->p2 - if err := n1.DialPeer(ctx, p2); err == nil { + if _, err := n1.DialPeer(ctx, p2); err == nil { t.Error("p1 should not be able to dial p2, not connected...") } // connect p3->p1 - if err := n3.DialPeer(ctx, p1); err == nil { + if _, err := n3.DialPeer(ctx, p1); err == nil { t.Error("p3 should not be able to dial p1, not connected...") } @@ -243,12 +257,12 @@ func TestNetworkSetup(t *testing.T) { // should now be able to connect // connect p1->p2 - if err := n1.DialPeer(ctx, p2); err != nil { + if _, err := n1.DialPeer(ctx, p2); err != nil { t.Error(err) } // and a stream should work now too :) - if _, err := n2.NewStream(inet.ProtocolDiag, p3); err != nil { + if _, err := n2.NewStream(p3); err != nil { t.Error(err) } @@ -262,27 +276,25 @@ func TestStreams(t *testing.T) { } handler := func(s inet.Stream) { - go func() { - b := make([]byte, 4) - if _, err := io.ReadFull(s, b); err != nil { - panic(err) - } - if !bytes.Equal(b, []byte("beep")) { - panic("bytes mismatch") - } - if _, err := s.Write([]byte("boop")); err != nil { - panic(err) - } - s.Close() - }() + b := make([]byte, 4) + if _, err := io.ReadFull(s, b); err != nil { + panic(err) + } + if !bytes.Equal(b, []byte("beep")) { + panic("bytes mismatch") + } + if _, err := s.Write([]byte("boop")); err != nil { + panic(err) + } + s.Close() } - nets := mn.Nets() - for _, n := range nets { - n.SetHandler(inet.ProtocolDHT, handler) + hosts := mn.Hosts() + for _, h := range mn.Hosts() { + h.SetStreamHandler(protocol.TestingID, handler) } - s, err := nets[0].NewStream(inet.ProtocolDHT, nets[1].LocalPeer()) + s, err := hosts[0].NewStream(protocol.TestingID, hosts[1].ID()) if err != nil { t.Fatal(err) } @@ -352,17 +364,10 @@ func TestStreamsStress(t *testing.T) { t.Fatal(err) } - protos := []inet.ProtocolID{ - inet.ProtocolDHT, - inet.ProtocolBitswap, - inet.ProtocolDiag, - } - - nets := mn.Nets() - for _, n := range nets { - for _, p := range protos { - n.SetHandler(p, makePonger(string(p))) - } + hosts := mn.Hosts() + for _, h := range hosts { + ponger := makePonger(string(protocol.TestingID)) + h.SetStreamHandler(protocol.TestingID, ponger) } var wg sync.WaitGroup @@ -370,18 +375,16 @@ func TestStreamsStress(t *testing.T) { wg.Add(1) go func(i int) { defer wg.Done() - from := rand.Intn(len(nets)) - to := rand.Intn(len(nets)) - p := rand.Intn(3) - proto := protos[p] - s, err := nets[from].NewStream(protos[p], nets[to].LocalPeer()) + from := rand.Intn(len(hosts)) + to := rand.Intn(len(hosts)) + s, err := hosts[from].NewStream(protocol.TestingID, hosts[to].ID()) if err != nil { - log.Debugf("%d (%s) %d (%s) %d (%s)", from, nets[from], to, nets[to], p, protos[p]) + log.Debugf("%d (%s) %d (%s)", from, hosts[from], to, hosts[to]) panic(err) } log.Infof("%d start pinging", i) - makePinger(string(proto), rand.Intn(100))(s) + makePinger("pingpong", rand.Intn(100))(s) log.Infof("%d done pinging", i) }(i) } @@ -401,12 +404,12 @@ func TestAdding(t *testing.T) { } a := testutil.RandLocalTCPAddress() - n, err := mn.AddPeer(sk, a) + h, err := mn.AddPeer(sk, a) if err != nil { t.Fatal(err) } - peers = append(peers, n.LocalPeer()) + peers = append(peers, h.ID()) } p1 := peers[0] @@ -422,40 +425,38 @@ func TestAdding(t *testing.T) { } // set the new stream handler on p2 - n2 := mn.Net(p2) - if n2 == nil { - t.Fatalf("no network for %s", p2) + h2 := mn.Host(p2) + if h2 == nil { + t.Fatalf("no host for %s", p2) } - n2.SetHandler(inet.ProtocolBitswap, func(s inet.Stream) { - go func() { - defer s.Close() + h2.SetStreamHandler(protocol.TestingID, func(s inet.Stream) { + defer s.Close() - b := make([]byte, 4) - if _, err := io.ReadFull(s, b); err != nil { - panic(err) - } - if string(b) != "beep" { - panic("did not beep!") - } + b := make([]byte, 4) + if _, err := io.ReadFull(s, b); err != nil { + panic(err) + } + if string(b) != "beep" { + panic("did not beep!") + } - if _, err := s.Write([]byte("boop")); err != nil { - panic(err) - } - }() + if _, err := s.Write([]byte("boop")); err != nil { + panic(err) + } }) // connect p1 to p2 - if err := mn.ConnectPeers(p1, p2); err != nil { + if _, err := mn.ConnectPeers(p1, p2); err != nil { t.Fatal(err) } // talk to p2 - n1 := mn.Net(p1) - if n1 == nil { + h1 := mn.Host(p1) + if h1 == nil { t.Fatalf("no network for %s", p1) } - s, err := n1.NewStream(inet.ProtocolBitswap, p2) + s, err := h1.NewStream(protocol.TestingID, p2) if err != nil { t.Fatal(err) } diff --git a/net/swarm/addr.go b/p2p/net/swarm/addr.go similarity index 98% rename from net/swarm/addr.go rename to p2p/net/swarm/addr.go index 5f074df94..01cb39717 100644 --- a/net/swarm/addr.go +++ b/p2p/net/swarm/addr.go @@ -1,7 +1,7 @@ package swarm import ( - conn "github.com/jbenet/go-ipfs/net/conn" + conn "github.com/jbenet/go-ipfs/p2p/net/conn" eventlog "github.com/jbenet/go-ipfs/util/eventlog" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" diff --git a/net/swarm/simul_test.go b/p2p/net/swarm/simul_test.go similarity index 79% rename from net/swarm/simul_test.go rename to p2p/net/swarm/simul_test.go index 4e2c3feaa..c87df91c3 100644 --- a/net/swarm/simul_test.go +++ b/p2p/net/swarm/simul_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" @@ -15,13 +15,14 @@ func TestSimultOpen(t *testing.T) { // t.Skip("skipping for another test") ctx := context.Background() - swarms, peers := makeSwarms(ctx, t, 2) + swarms := makeSwarms(ctx, t, 2) // connect everyone { var wg sync.WaitGroup connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { // copy for other peer + log.Debugf("TestSimultOpen: connecting: %s --> %s (%s)", s.local, dst, addr) s.peers.AddAddress(dst, addr) if _, err := s.Dial(ctx, dst); err != nil { t.Fatal("error swarm dialing to peer", err) @@ -31,8 +32,8 @@ func TestSimultOpen(t *testing.T) { log.Info("Connecting swarms simultaneously.") wg.Add(2) - go connect(swarms[0], swarms[1].local, peers[1].Addr) - go connect(swarms[1], swarms[0].local, peers[0].Addr) + go connect(swarms[0], swarms[1].local, swarms[1].ListenAddresses()[0]) + go connect(swarms[1], swarms[0].local, swarms[0].ListenAddresses()[0]) wg.Wait() } diff --git a/net/swarm/swarm.go b/p2p/net/swarm/swarm.go similarity index 96% rename from net/swarm/swarm.go rename to p2p/net/swarm/swarm.go index 25c27adbf..96c07d015 100644 --- a/net/swarm/swarm.go +++ b/p2p/net/swarm/swarm.go @@ -3,7 +3,8 @@ package swarm import ( - peer "github.com/jbenet/go-ipfs/peer" + inet "github.com/jbenet/go-ipfs/p2p/net" + peer "github.com/jbenet/go-ipfs/p2p/peer" eventlog "github.com/jbenet/go-ipfs/util/eventlog" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" @@ -92,7 +93,7 @@ func (s *Swarm) SetConnHandler(handler ConnHandler) { // SetStreamHandler assigns the handler for new streams. // See peerstream. -func (s *Swarm) SetStreamHandler(handler StreamHandler) { +func (s *Swarm) SetStreamHandler(handler inet.StreamHandler) { s.swarm.SetStreamHandler(func(s *ps.Stream) { handler(wrapStream(s)) }) diff --git a/net/swarm/swarm_conn.go b/p2p/net/swarm/swarm_conn.go similarity index 91% rename from net/swarm/swarm_conn.go rename to p2p/net/swarm/swarm_conn.go index 36be0bc50..580f88fae 100644 --- a/net/swarm/swarm_conn.go +++ b/p2p/net/swarm/swarm_conn.go @@ -3,9 +3,10 @@ package swarm import ( "fmt" - ic "github.com/jbenet/go-ipfs/crypto" - conn "github.com/jbenet/go-ipfs/net/conn" - peer "github.com/jbenet/go-ipfs/peer" + ic "github.com/jbenet/go-ipfs/p2p/crypto" + inet "github.com/jbenet/go-ipfs/p2p/net" + conn "github.com/jbenet/go-ipfs/p2p/net/conn" + peer "github.com/jbenet/go-ipfs/p2p/peer" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" @@ -74,12 +75,18 @@ func (c *Conn) RemotePublicKey() ic.PubKey { return c.RawConn().RemotePublicKey() } -// NewStream returns a new Stream from this connection -func (c *Conn) NewStream() (*Stream, error) { +// NewSwarmStream returns a new Stream from this connection +func (c *Conn) NewSwarmStream() (*Stream, error) { s, err := c.StreamConn().NewStream() return wrapStream(s), err } +// NewStream returns a new Stream from this connection +func (c *Conn) NewStream() (inet.Stream, error) { + s, err := c.NewSwarmStream() + return inet.Stream(s), err +} + func (c *Conn) Close() error { return c.StreamConn().Close() } diff --git a/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go similarity index 96% rename from net/swarm/swarm_dial.go rename to p2p/net/swarm/swarm_dial.go index 03e596e9e..dde967fbc 100644 --- a/net/swarm/swarm_dial.go +++ b/p2p/net/swarm/swarm_dial.go @@ -4,8 +4,8 @@ import ( "errors" "fmt" - conn "github.com/jbenet/go-ipfs/net/conn" - peer "github.com/jbenet/go-ipfs/peer" + conn "github.com/jbenet/go-ipfs/p2p/net/conn" + peer "github.com/jbenet/go-ipfs/p2p/peer" lgbl "github.com/jbenet/go-ipfs/util/eventlog/loggables" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" diff --git a/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go similarity index 98% rename from net/swarm/swarm_listen.go rename to p2p/net/swarm/swarm_listen.go index bcc55cad6..ba0bee2e7 100644 --- a/net/swarm/swarm_listen.go +++ b/p2p/net/swarm/swarm_listen.go @@ -1,7 +1,7 @@ package swarm import ( - conn "github.com/jbenet/go-ipfs/net/conn" + conn "github.com/jbenet/go-ipfs/p2p/net/conn" lgbl "github.com/jbenet/go-ipfs/util/eventlog/loggables" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" diff --git a/p2p/net/swarm/swarm_net.go b/p2p/net/swarm/swarm_net.go new file mode 100644 index 000000000..42230105f --- /dev/null +++ b/p2p/net/swarm/swarm_net.go @@ -0,0 +1,156 @@ +package swarm + +import ( + "fmt" + + peer "github.com/jbenet/go-ipfs/p2p/peer" + + inet "github.com/jbenet/go-ipfs/p2p/net" + + context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" + ctxgroup "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup" + ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" +) + +// Network implements the inet.Network interface. +// It is simply a swarm, with a few different functions +// to implement inet.Network. +type Network Swarm + +// NewNetwork constructs a new network and starts listening on given addresses. +func NewNetwork(ctx context.Context, listen []ma.Multiaddr, local peer.ID, + peers peer.Peerstore) (*Network, error) { + + s, err := NewSwarm(ctx, listen, local, peers) + if err != nil { + return nil, err + } + + return (*Network)(s), nil +} + +// DialPeer attempts to establish a connection to a given peer. +// Respects the context. +func (n *Network) DialPeer(ctx context.Context, p peer.ID) (inet.Conn, error) { + log.Debugf("[%s] network dialing peer [%s]", n.local, p) + sc, err := n.Swarm().Dial(ctx, p) + if err != nil { + return nil, err + } + + log.Debugf("network for %s finished dialing %s", n.local, p) + return inet.Conn(sc), nil +} + +// CtxGroup returns the network's ContextGroup +func (n *Network) CtxGroup() ctxgroup.ContextGroup { + return n.cg +} + +// Swarm returns the network's peerstream.Swarm +func (n *Network) Swarm() *Swarm { + return (*Swarm)(n) +} + +// LocalPeer the network's LocalPeer +func (n *Network) LocalPeer() peer.ID { + return n.Swarm().LocalPeer() +} + +// Peers returns the connected peers +func (n *Network) Peers() []peer.ID { + return n.Swarm().Peers() +} + +// Peers returns the connected peers +func (n *Network) Peerstore() peer.Peerstore { + return n.Swarm().peers +} + +// Conns returns the connected peers +func (n *Network) Conns() []inet.Conn { + conns1 := n.Swarm().Connections() + out := make([]inet.Conn, len(conns1)) + for i, c := range conns1 { + out[i] = inet.Conn(c) + } + return out +} + +// ConnsToPeer returns the connections in this Netowrk for given peer. +func (n *Network) ConnsToPeer(p peer.ID) []inet.Conn { + conns1 := n.Swarm().ConnectionsToPeer(p) + out := make([]inet.Conn, len(conns1)) + for i, c := range conns1 { + out[i] = inet.Conn(c) + } + return out +} + +// ClosePeer connection to peer +func (n *Network) ClosePeer(p peer.ID) error { + return n.Swarm().CloseConnection(p) +} + +// close is the real teardown function +func (n *Network) close() error { + return n.Swarm().Close() +} + +// Close calls the ContextCloser func +func (n *Network) Close() error { + return n.Swarm().cg.Close() +} + +// ListenAddresses returns a list of addresses at which this network listens. +func (n *Network) ListenAddresses() []ma.Multiaddr { + return n.Swarm().ListenAddresses() +} + +// InterfaceListenAddresses returns a list of addresses at which this network +// listens. It expands "any interface" addresses (/ip4/0.0.0.0, /ip6/::) to +// use the known local interfaces. +func (n *Network) InterfaceListenAddresses() ([]ma.Multiaddr, error) { + return InterfaceListenAddresses(n.Swarm()) +} + +// Connectedness returns a state signaling connection capabilities +// For now only returns Connected || NotConnected. Expand into more later. +func (n *Network) Connectedness(p peer.ID) inet.Connectedness { + c := n.Swarm().ConnectionsToPeer(p) + if c != nil && len(c) > 0 { + return inet.Connected + } + return inet.NotConnected +} + +// NewStream returns a new stream to given peer p. +// If there is no connection to p, attempts to create one. +func (n *Network) NewStream(p peer.ID) (inet.Stream, error) { + log.Debugf("[%s] network opening stream to peer [%s]", n.local, p) + s, err := n.Swarm().NewStreamWithPeer(p) + if err != nil { + return nil, err + } + + return inet.Stream(s), nil +} + +// SetHandler sets the protocol handler on the Network's Muxer. +// This operation is threadsafe. +func (n *Network) SetStreamHandler(h inet.StreamHandler) { + n.Swarm().SetStreamHandler(h) +} + +// SetConnHandler sets the conn handler on the Network. +// This operation is threadsafe. +func (n *Network) SetConnHandler(h inet.ConnHandler) { + n.Swarm().SetConnHandler(func(c *Conn) { + h(inet.Conn(c)) + }) +} + +// String returns a string representation of Network. +func (n *Network) String() string { + return fmt.Sprintf("", n.LocalPeer()) +} diff --git a/net/net_test.go b/p2p/net/swarm/swarm_net_test.go similarity index 87% rename from net/net_test.go rename to p2p/net/swarm/swarm_net_test.go index 0704ec25c..3849838ba 100644 --- a/net/net_test.go +++ b/p2p/net/swarm/swarm_net_test.go @@ -1,4 +1,4 @@ -package net_test +package swarm_test import ( "fmt" @@ -6,7 +6,9 @@ import ( "time" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" - inet "github.com/jbenet/go-ipfs/net" + + inet "github.com/jbenet/go-ipfs/p2p/net" + testutil "github.com/jbenet/go-ipfs/p2p/test/util" ) // TestConnectednessCorrect starts a few networks, connects a few @@ -17,14 +19,14 @@ func TestConnectednessCorrect(t *testing.T) { nets := make([]inet.Network, 4) for i := 0; i < 4; i++ { - nets[i] = GenNetwork(t, ctx) + nets[i] = testutil.GenSwarmNetwork(t, ctx) } // connect 0-1, 0-2, 0-3, 1-2, 2-3 dial := func(a, b inet.Network) { - DivulgeAddresses(b, a) - if err := a.DialPeer(ctx, b.LocalPeer()); err != nil { + testutil.DivulgeAddresses(b, a) + if _, err := a.DialPeer(ctx, b.LocalPeer()); err != nil { t.Fatalf("Failed to dial: %s", err) } } diff --git a/net/swarm/swarm_stream.go b/p2p/net/swarm/swarm_stream.go similarity index 78% rename from net/swarm/swarm_stream.go rename to p2p/net/swarm/swarm_stream.go index 7fe23f92b..abd641ff0 100644 --- a/net/swarm/swarm_stream.go +++ b/p2p/net/swarm/swarm_stream.go @@ -1,6 +1,8 @@ package swarm import ( + inet "github.com/jbenet/go-ipfs/p2p/net" + ps "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-peerstream" ) @@ -8,17 +10,18 @@ import ( // our Conn and Swarm (instead of just the ps.Conn and ps.Swarm) type Stream ps.Stream -// StreamHandler is called when new streams are opened from remote peers. -// See peerstream.StreamHandler -type StreamHandler func(*Stream) - // Stream returns the underlying peerstream.Stream func (s *Stream) Stream() *ps.Stream { return (*ps.Stream)(s) } -// Conn returns the Conn associated with this Stream -func (s *Stream) Conn() *Conn { +// Conn returns the Conn associated with this Stream, as an inet.Conn +func (s *Stream) Conn() inet.Conn { + return s.SwarmConn() +} + +// SwarmConn returns the Conn associated with this Stream, as a *Conn +func (s *Stream) SwarmConn() *Conn { return (*Conn)(s.Stream().Conn()) } diff --git a/net/swarm/swarm_test.go b/p2p/net/swarm/swarm_test.go similarity index 83% rename from net/swarm/swarm_test.go rename to p2p/net/swarm/swarm_test.go index b532566b8..4588ec55f 100644 --- a/net/swarm/swarm_test.go +++ b/p2p/net/swarm/swarm_test.go @@ -7,7 +7,8 @@ import ( "testing" "time" - peer "github.com/jbenet/go-ipfs/peer" + inet "github.com/jbenet/go-ipfs/p2p/net" + peer "github.com/jbenet/go-ipfs/p2p/peer" errors "github.com/jbenet/go-ipfs/util/debugerror" testutil "github.com/jbenet/go-ipfs/util/testutil" @@ -15,12 +16,12 @@ import ( ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" ) -func EchoStreamHandler(stream *Stream) { +func EchoStreamHandler(stream inet.Stream) { go func() { defer stream.Close() // pull out the ipfs conn - c := stream.Conn().RawConn() + c := stream.Conn() log.Debugf("%s ponging to %s", c.LocalPeer(), c.RemotePeer()) buf := make([]byte, 4) @@ -46,20 +47,17 @@ func EchoStreamHandler(stream *Stream) { }() } -func makeSwarms(ctx context.Context, t *testing.T, num int) ([]*Swarm, []testutil.PeerNetParams) { +func makeSwarms(ctx context.Context, t *testing.T, num int) []*Swarm { swarms := make([]*Swarm, 0, num) - peersnp := make([]testutil.PeerNetParams, 0, num) for i := 0; i < num; i++ { localnp := testutil.RandPeerNetParamsOrFatal(t) - peersnp = append(peersnp, localnp) peerstore := peer.NewPeerstore() - peerstore.AddAddress(localnp.ID, localnp.Addr) peerstore.AddPubKey(localnp.ID, localnp.PubKey) peerstore.AddPrivKey(localnp.ID, localnp.PrivKey) - addrs := peerstore.Addresses(localnp.ID) + addrs := []ma.Multiaddr{localnp.Addr} swarm, err := NewSwarm(ctx, addrs, localnp.ID, peerstore) if err != nil { t.Fatal(err) @@ -69,10 +67,10 @@ func makeSwarms(ctx context.Context, t *testing.T, num int) ([]*Swarm, []testuti swarms = append(swarms, swarm) } - return swarms, peersnp + return swarms } -func connectSwarms(t *testing.T, ctx context.Context, swarms []*Swarm, peersnp []testutil.PeerNetParams) { +func connectSwarms(t *testing.T, ctx context.Context, swarms []*Swarm) { var wg sync.WaitGroup connect := func(s *Swarm, dst peer.ID, addr ma.Multiaddr) { @@ -85,11 +83,11 @@ func connectSwarms(t *testing.T, ctx context.Context, swarms []*Swarm, peersnp [ } log.Info("Connecting swarms simultaneously.") - for _, s := range swarms { - for _, p := range peersnp { - if p.ID != s.local { // don't connect to self. + for _, s1 := range swarms { + for _, s2 := range swarms { + if s2.local != s1.local { // don't connect to self. wg.Add(1) - connect(s, p.ID, p.Addr) + connect(s1, s2.LocalPeer(), s2.ListenAddresses()[0]) // try the first. } } } @@ -104,10 +102,10 @@ func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { // t.Skip("skipping for another test") ctx := context.Background() - swarms, peersnp := makeSwarms(ctx, t, SwarmNum) + swarms := makeSwarms(ctx, t, SwarmNum) // connect everyone - connectSwarms(t, ctx, swarms, peersnp) + connectSwarms(t, ctx, swarms) // ping/pong for _, s1 := range swarms { @@ -117,7 +115,7 @@ func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { _, cancel := context.WithCancel(ctx) got := map[peer.ID]int{} - errChan := make(chan error, MsgNum*len(peersnp)) + errChan := make(chan error, MsgNum*len(swarms)) streamChan := make(chan *Stream, MsgNum) // send out "ping" x MsgNum to every peer @@ -149,13 +147,13 @@ func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { streamChan <- stream } - for _, p := range peersnp { - if p.ID == s1.local { + for _, s2 := range swarms { + if s2.local == s1.local { continue // dont send to self... } wg.Add(1) - go send(p.ID) + go send(s2.local) } wg.Wait() }() @@ -164,7 +162,7 @@ func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { go func() { defer close(errChan) count := 0 - countShouldBe := MsgNum * (len(peersnp) - 1) + countShouldBe := MsgNum * (len(swarms) - 1) for stream := range streamChan { // one per peer defer stream.Close() @@ -208,8 +206,8 @@ func SubtestSwarm(t *testing.T, SwarmNum int, MsgNum int) { } log.Debugf("%s got pongs", s1.local) - if (len(peersnp) - 1) != len(got) { - t.Errorf("got (%d) less messages than sent (%d).", len(got), len(peersnp)) + if (len(swarms) - 1) != len(got) { + t.Errorf("got (%d) less messages than sent (%d).", len(got), len(swarms)) } for p, n := range got { @@ -240,14 +238,14 @@ func TestConnHandler(t *testing.T) { // t.Skip("skipping for another test") ctx := context.Background() - swarms, peersnp := makeSwarms(ctx, t, 5) + swarms := makeSwarms(ctx, t, 5) gotconn := make(chan struct{}, 10) swarms[0].SetConnHandler(func(conn *Conn) { gotconn <- struct{}{} }) - connectSwarms(t, ctx, swarms, peersnp) + connectSwarms(t, ctx, swarms) <-time.After(time.Millisecond) // should've gotten 5 by now. diff --git a/peer/metrics.go b/p2p/peer/metrics.go similarity index 100% rename from peer/metrics.go rename to p2p/peer/metrics.go diff --git a/peer/metrics_test.go b/p2p/peer/metrics_test.go similarity index 93% rename from peer/metrics_test.go rename to p2p/peer/metrics_test.go index 0301792c3..7b07cb139 100644 --- a/peer/metrics_test.go +++ b/p2p/peer/metrics_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" testutil "github.com/jbenet/go-ipfs/util/testutil" ) diff --git a/peer/peer.go b/p2p/peer/peer.go similarity index 98% rename from peer/peer.go rename to p2p/peer/peer.go index 9d57f9811..fa4e448d6 100644 --- a/peer/peer.go +++ b/p2p/peer/peer.go @@ -9,7 +9,7 @@ import ( ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" - ic "github.com/jbenet/go-ipfs/crypto" + ic "github.com/jbenet/go-ipfs/p2p/crypto" u "github.com/jbenet/go-ipfs/util" ) diff --git a/peer/peer_test.go b/p2p/peer/peer_test.go similarity index 97% rename from peer/peer_test.go rename to p2p/peer/peer_test.go index e8d1d29f3..b19d0faef 100644 --- a/peer/peer_test.go +++ b/p2p/peer/peer_test.go @@ -6,8 +6,8 @@ import ( "strings" "testing" - ic "github.com/jbenet/go-ipfs/crypto" - . "github.com/jbenet/go-ipfs/peer" + ic "github.com/jbenet/go-ipfs/p2p/crypto" + . "github.com/jbenet/go-ipfs/p2p/peer" u "github.com/jbenet/go-ipfs/util" tu "github.com/jbenet/go-ipfs/util/testutil" diff --git a/peer/peerstore.go b/p2p/peer/peerstore.go similarity index 95% rename from peer/peerstore.go rename to p2p/peer/peerstore.go index 46de50f0d..32d6207ea 100644 --- a/peer/peerstore.go +++ b/p2p/peer/peerstore.go @@ -4,7 +4,7 @@ import ( "errors" "sync" - ic "github.com/jbenet/go-ipfs/crypto" + ic "github.com/jbenet/go-ipfs/p2p/crypto" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" dssync "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync" @@ -26,6 +26,9 @@ type Peerstore interface { // that peer, useful to other services. PeerInfo(ID) PeerInfo + // AddPeerInfo absorbs the information listed in given PeerInfo. + AddPeerInfo(PeerInfo) + // Get/Put is a simple registry for other peer-related key/value pairs. // if we find something we use often, it should become its own set of // methods. this is a last resort. @@ -235,6 +238,10 @@ func (ps *peerstore) PeerInfo(p ID) PeerInfo { } } +func (ps *peerstore) AddPeerInfo(pi PeerInfo) { + ps.AddAddresses(pi.ID, pi.Addrs) +} + func PeerInfos(ps Peerstore, peers []ID) []PeerInfo { pi := make([]PeerInfo, len(peers)) for i, p := range peers { diff --git a/peer/peerstore_test.go b/p2p/peer/peerstore_test.go similarity index 100% rename from peer/peerstore_test.go rename to p2p/peer/peerstore_test.go diff --git a/peer/queue/distance.go b/p2p/peer/queue/distance.go similarity index 97% rename from peer/queue/distance.go rename to p2p/peer/queue/distance.go index ebe876bb1..a0e376fab 100644 --- a/peer/queue/distance.go +++ b/p2p/peer/queue/distance.go @@ -5,7 +5,7 @@ import ( "math/big" "sync" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" ks "github.com/jbenet/go-ipfs/routing/keyspace" u "github.com/jbenet/go-ipfs/util" ) diff --git a/peer/queue/interface.go b/p2p/peer/queue/interface.go similarity index 90% rename from peer/queue/interface.go rename to p2p/peer/queue/interface.go index c2106164e..b611422c4 100644 --- a/peer/queue/interface.go +++ b/p2p/peer/queue/interface.go @@ -1,6 +1,6 @@ package queue -import peer "github.com/jbenet/go-ipfs/peer" +import peer "github.com/jbenet/go-ipfs/p2p/peer" // PeerQueue maintains a set of peers ordered according to a metric. // Implementations of PeerQueue could order peers based on distances along diff --git a/peer/queue/queue_test.go b/p2p/peer/queue/queue_test.go similarity index 98% rename from peer/queue/queue_test.go rename to p2p/peer/queue/queue_test.go index d3e661400..957564fca 100644 --- a/peer/queue/queue_test.go +++ b/p2p/peer/queue/queue_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" u "github.com/jbenet/go-ipfs/util" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" diff --git a/peer/queue/sync.go b/p2p/peer/queue/sync.go similarity index 96% rename from peer/queue/sync.go rename to p2p/peer/queue/sync.go index 7a00eeb43..3f75cd0cf 100644 --- a/peer/queue/sync.go +++ b/p2p/peer/queue/sync.go @@ -3,7 +3,7 @@ package queue import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" ) // ChanQueue makes any PeerQueue synchronizable through channels. diff --git a/net/id.go b/p2p/protocol/identify/id.go similarity index 56% rename from net/id.go rename to p2p/protocol/identify/id.go index 802d54794..2479a28f9 100644 --- a/net/id.go +++ b/p2p/protocol/identify/id.go @@ -1,15 +1,41 @@ -package net +package identify import ( + "fmt" "sync" - handshake "github.com/jbenet/go-ipfs/net/handshake" - pb "github.com/jbenet/go-ipfs/net/handshake/pb" - + context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ggio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/io" + semver "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/coreos/go-semver/semver" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" + + config "github.com/jbenet/go-ipfs/config" + eventlog "github.com/jbenet/go-ipfs/util/eventlog" + + host "github.com/jbenet/go-ipfs/p2p/host" + inet "github.com/jbenet/go-ipfs/p2p/net" + protocol "github.com/jbenet/go-ipfs/p2p/protocol" + + pb "github.com/jbenet/go-ipfs/p2p/protocol/identify/pb" ) +var log = eventlog.Logger("net/identify") + +// ID is the protocol.ID of the Identify Service. +const ID protocol.ID = "/ipfs/identify" + +// IpfsVersion holds the current protocol version for a client running this code +var IpfsVersion *semver.Version +var ClientVersion = "go-ipfs/" + config.CurrentVersionNumber + +func init() { + var err error + IpfsVersion, err = semver.NewVersion("0.0.1") + if err != nil { + panic(fmt.Errorf("invalid protocol version: %v", err)) + } +} + // IDService is a structure that implements ProtocolIdentify. // It is a trivial service that gives the other peer some // useful information about the local peer. A sort of hello. @@ -19,24 +45,24 @@ import ( // * Our IPFS Agent Version // * Our public Listen Addresses type IDService struct { - Network Network + Host host.Host // connections undergoing identification // for wait purposes - currid map[Conn]chan struct{} + currid map[inet.Conn]chan struct{} currmu sync.RWMutex } -func NewIDService(n Network) *IDService { +func NewIDService(h host.Host) *IDService { s := &IDService{ - Network: n, - currid: make(map[Conn]chan struct{}), + Host: h, + currid: make(map[inet.Conn]chan struct{}), } - n.SetHandler(ProtocolIdentify, s.RequestHandler) + h.SetStreamHandler(ID, s.RequestHandler) return s } -func (ids *IDService) IdentifyConn(c Conn) { +func (ids *IDService) IdentifyConn(c inet.Conn) { ids.currmu.Lock() if wait, found := ids.currid[c]; found { ids.currmu.Unlock() @@ -47,13 +73,17 @@ func (ids *IDService) IdentifyConn(c Conn) { ids.currid[c] = make(chan struct{}) ids.currmu.Unlock() - s, err := c.NewStreamWithProtocol(ProtocolIdentify) + s, err := c.NewStream() if err != nil { - log.Error("network: unable to open initial stream for %s", ProtocolIdentify) - log.Event(ids.Network.CtxGroup().Context(), "IdentifyOpenFailed", c.RemotePeer()) + log.Error("error opening initial stream for %s", ID) + log.Event(context.TODO(), "IdentifyOpenFailed", c.RemotePeer()) } else { // ok give the response to our handler. + if err := protocol.WriteHeader(s, ID); err != nil { + log.Error("error writing stream header for %s", ID) + log.Event(context.TODO(), "IdentifyOpenFailed", c.RemotePeer()) + } ids.ResponseHandler(s) } @@ -70,40 +100,40 @@ func (ids *IDService) IdentifyConn(c Conn) { close(ch) // release everyone waiting. } -func (ids *IDService) RequestHandler(s Stream) { +func (ids *IDService) RequestHandler(s inet.Stream) { defer s.Close() c := s.Conn() w := ggio.NewDelimitedWriter(s) - mes := pb.Handshake3{} + mes := pb.Identify{} ids.populateMessage(&mes, s.Conn()) w.WriteMsg(&mes) - log.Debugf("%s sent message to %s %s", ProtocolIdentify, + log.Debugf("%s sent message to %s %s", ID, c.RemotePeer(), c.RemoteMultiaddr()) } -func (ids *IDService) ResponseHandler(s Stream) { +func (ids *IDService) ResponseHandler(s inet.Stream) { defer s.Close() c := s.Conn() r := ggio.NewDelimitedReader(s, 2048) - mes := pb.Handshake3{} + mes := pb.Identify{} if err := r.ReadMsg(&mes); err != nil { - log.Errorf("%s error receiving message from %s %s", ProtocolIdentify, + log.Errorf("%s error receiving message from %s %s", ID, c.RemotePeer(), c.RemoteMultiaddr()) return } ids.consumeMessage(&mes, c) - log.Debugf("%s received message from %s %s", ProtocolIdentify, + log.Debugf("%s received message from %s %s", ID, c.RemotePeer(), c.RemoteMultiaddr()) } -func (ids *IDService) populateMessage(mes *pb.Handshake3, c Conn) { +func (ids *IDService) populateMessage(mes *pb.Identify, c inet.Conn) { // set protocols this node is currently handling - protos := ids.Network.Protocols() + protos := ids.Host.Mux().Protocols() mes.Protocols = make([]string, len(protos)) for i, p := range protos { mes.Protocols[i] = string(p) @@ -114,7 +144,7 @@ func (ids *IDService) populateMessage(mes *pb.Handshake3, c Conn) { mes.ObservedAddr = c.RemoteMultiaddr().Bytes() // set listen addrs - laddrs, err := ids.Network.InterfaceListenAddresses() + laddrs, err := ids.Host.Network().InterfaceListenAddresses() if err != nil { log.Error(err) } else { @@ -126,10 +156,12 @@ func (ids *IDService) populateMessage(mes *pb.Handshake3, c Conn) { } // set protocol versions - mes.H1 = handshake.NewHandshake1("", "") + s := IpfsVersion.String() + mes.ProtocolVersion = &s + mes.AgentVersion = &ClientVersion } -func (ids *IDService) consumeMessage(mes *pb.Handshake3, c Conn) { +func (ids *IDService) consumeMessage(mes *pb.Identify, c inet.Conn) { p := c.RemotePeer() // mes.Protocols @@ -141,22 +173,22 @@ func (ids *IDService) consumeMessage(mes *pb.Handshake3, c Conn) { for _, addr := range laddrs { maddr, err := ma.NewMultiaddrBytes(addr) if err != nil { - log.Errorf("%s failed to parse multiaddr from %s %s", ProtocolIdentify, p, - c.RemoteMultiaddr()) + log.Errorf("%s failed to parse multiaddr from %s %s", ID, + p, c.RemoteMultiaddr()) continue } lmaddrs = append(lmaddrs, maddr) } // update our peerstore with the addresses. - ids.Network.Peerstore().AddAddresses(p, lmaddrs) + ids.Host.Peerstore().AddAddresses(p, lmaddrs) log.Debugf("%s received listen addrs for %s: %s", c.LocalPeer(), c.RemotePeer(), lmaddrs) // get protocol versions - pv := *mes.H1.ProtocolVersion - av := *mes.H1.AgentVersion - ids.Network.Peerstore().Put(p, "ProtocolVersion", pv) - ids.Network.Peerstore().Put(p, "AgentVersion", av) + pv := *mes.ProtocolVersion + av := *mes.AgentVersion + ids.Host.Peerstore().Put(p, "ProtocolVersion", pv) + ids.Host.Peerstore().Put(p, "AgentVersion", av) } // IdentifyWait returns a channel which will be closed once @@ -164,7 +196,7 @@ func (ids *IDService) consumeMessage(mes *pb.Handshake3, c Conn) { // This happens async so the connection can start to be used // even if handshake3 knowledge is not necesary. // Users **MUST** call IdentifyWait _after_ IdentifyConn -func (ids *IDService) IdentifyWait(c Conn) <-chan struct{} { +func (ids *IDService) IdentifyWait(c inet.Conn) <-chan struct{} { ids.currmu.Lock() ch, found := ids.currid[c] ids.currmu.Unlock() diff --git a/p2p/protocol/identify/id_test.go b/p2p/protocol/identify/id_test.go new file mode 100644 index 000000000..c6707b0ac --- /dev/null +++ b/p2p/protocol/identify/id_test.go @@ -0,0 +1,106 @@ +package identify_test + +import ( + "testing" + "time" + + host "github.com/jbenet/go-ipfs/p2p/host" + peer "github.com/jbenet/go-ipfs/p2p/peer" + identify "github.com/jbenet/go-ipfs/p2p/protocol/identify" + testutil "github.com/jbenet/go-ipfs/p2p/test/util" + + context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" + ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" +) + +func subtestIDService(t *testing.T, postDialWait time.Duration) { + + // the generated networks should have the id service wired in. + ctx := context.Background() + h1 := testutil.GenHostSwarm(t, ctx) + h2 := testutil.GenHostSwarm(t, ctx) + + h1p := h1.ID() + h2p := h2.ID() + + testKnowsAddrs(t, h1, h2p, []ma.Multiaddr{}) // nothing + testKnowsAddrs(t, h2, h1p, []ma.Multiaddr{}) // nothing + + h2pi := h2.Peerstore().PeerInfo(h2p) + if err := h1.Connect(ctx, h2pi); err != nil { + t.Fatal(err) + } + + // we need to wait here if Dial returns before ID service is finished. + if postDialWait > 0 { + <-time.After(postDialWait) + } + + // the IDService should be opened automatically, by the network. + // what we should see now is that both peers know about each others listen addresses. + testKnowsAddrs(t, h1, h2p, h2.Peerstore().Addresses(h2p)) // has them + testHasProtocolVersions(t, h1, h2p) + + // now, this wait we do have to do. it's the wait for the Listening side + // to be done identifying the connection. + c := h2.Network().ConnsToPeer(h1.ID()) + if len(c) < 1 { + t.Fatal("should have connection by now at least.") + } + <-h2.IDService().IdentifyWait(c[0]) + + // and the protocol versions. + testKnowsAddrs(t, h2, h1p, h1.Peerstore().Addresses(h1p)) // has them + testHasProtocolVersions(t, h2, h1p) +} + +func testKnowsAddrs(t *testing.T, h host.Host, p peer.ID, expected []ma.Multiaddr) { + actual := h.Peerstore().Addresses(p) + + if len(actual) != len(expected) { + t.Error("dont have the same addresses") + } + + have := map[string]struct{}{} + for _, addr := range actual { + have[addr.String()] = struct{}{} + } + for _, addr := range expected { + if _, found := have[addr.String()]; !found { + t.Errorf("%s did not have addr for %s: %s", h.ID(), p, addr) + // panic("ahhhhhhh") + } + } +} + +func testHasProtocolVersions(t *testing.T, h host.Host, p peer.ID) { + v, err := h.Peerstore().Get(p, "ProtocolVersion") + if v == nil { + t.Error("no protocol version") + return + } + if v.(string) != identify.IpfsVersion.String() { + t.Error("protocol mismatch", err) + } + v, err = h.Peerstore().Get(p, "AgentVersion") + if v.(string) != identify.ClientVersion { + t.Error("agent version mismatch", err) + } +} + +// TestIDServiceWait gives the ID service 100ms to finish after dialing +// this is becasue it used to be concurrent. Now, Dial wait till the +// id service is done. +func TestIDServiceWait(t *testing.T) { + N := 3 + for i := 0; i < N; i++ { + subtestIDService(t, 100*time.Millisecond) + } +} + +func TestIDServiceNoWait(t *testing.T) { + N := 3 + for i := 0; i < N; i++ { + subtestIDService(t, 0) + } +} diff --git a/net/handshake/pb/Makefile b/p2p/protocol/identify/pb/Makefile similarity index 100% rename from net/handshake/pb/Makefile rename to p2p/protocol/identify/pb/Makefile diff --git a/net/handshake/pb/handshake.pb.go b/p2p/protocol/identify/pb/identify.pb.go similarity index 51% rename from net/handshake/pb/handshake.pb.go rename to p2p/protocol/identify/pb/identify.pb.go index e4164788d..e656514c4 100644 --- a/net/handshake/pb/handshake.pb.go +++ b/p2p/protocol/identify/pb/identify.pb.go @@ -1,18 +1,17 @@ // Code generated by protoc-gen-gogo. -// source: handshake.proto +// source: identify.proto // DO NOT EDIT! /* -Package handshake_pb is a generated protocol buffer package. +Package identify_pb is a generated protocol buffer package. It is generated from these files: - handshake.proto + identify.proto It has these top-level messages: - Handshake1 - Handshake3 + Identify */ -package handshake_pb +package identify_pb import proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/proto" import json "encoding/json" @@ -23,91 +22,72 @@ var _ = proto.Marshal var _ = &json.SyntaxError{} var _ = math.Inf -// Handshake1 is delivered _before_ the secure channel is initialized -type Handshake1 struct { +type Identify struct { // protocolVersion determines compatibility between peers - ProtocolVersion *string `protobuf:"bytes,1,opt,name=protocolVersion" json:"protocolVersion,omitempty"` + ProtocolVersion *string `protobuf:"bytes,5,opt,name=protocolVersion" json:"protocolVersion,omitempty"` // agentVersion is like a UserAgent string in browsers, or client version in bittorrent - // includes the client name and client. e.g. "go-ipfs/0.1.0" - AgentVersion *string `protobuf:"bytes,2,opt,name=agentVersion" json:"agentVersion,omitempty"` - XXX_unrecognized []byte `json:"-"` -} - -func (m *Handshake1) Reset() { *m = Handshake1{} } -func (m *Handshake1) String() string { return proto.CompactTextString(m) } -func (*Handshake1) ProtoMessage() {} - -func (m *Handshake1) GetProtocolVersion() string { - if m != nil && m.ProtocolVersion != nil { - return *m.ProtocolVersion - } - return "" -} - -func (m *Handshake1) GetAgentVersion() string { - if m != nil && m.AgentVersion != nil { - return *m.AgentVersion - } - return "" -} - -// Handshake3 is delivered _after_ the secure channel is initialized -type Handshake3 struct { - // can include all the values in handshake1, for protocol version, etc. - H1 *Handshake1 `protobuf:"bytes,5,opt,name=h1" json:"h1,omitempty"` + // includes the client name and client. + AgentVersion *string `protobuf:"bytes,6,opt,name=agentVersion" json:"agentVersion,omitempty"` // publicKey is this node's public key (which also gives its node.ID) // - may not need to be sent, as secure channel implies it has been sent. // - then again, if we change / disable secure channel, may still want it. PublicKey []byte `protobuf:"bytes,1,opt,name=publicKey" json:"publicKey,omitempty"` // listenAddrs are the multiaddrs the sender node listens for open connections on ListenAddrs [][]byte `protobuf:"bytes,2,rep,name=listenAddrs" json:"listenAddrs,omitempty"` - // protocols are the services this node is running - Protocols []string `protobuf:"bytes,3,rep,name=protocols" json:"protocols,omitempty"` // oservedAddr is the multiaddr of the remote endpoint that the sender node perceives // this is useful information to convey to the other side, as it helps the remote endpoint // determine whether its connection to the local peer goes through NAT. - ObservedAddr []byte `protobuf:"bytes,4,opt,name=observedAddr" json:"observedAddr,omitempty"` - XXX_unrecognized []byte `json:"-"` + ObservedAddr []byte `protobuf:"bytes,4,opt,name=observedAddr" json:"observedAddr,omitempty"` + // protocols are the services this node is running + Protocols []string `protobuf:"bytes,3,rep,name=protocols" json:"protocols,omitempty"` + XXX_unrecognized []byte `json:"-"` } -func (m *Handshake3) Reset() { *m = Handshake3{} } -func (m *Handshake3) String() string { return proto.CompactTextString(m) } -func (*Handshake3) ProtoMessage() {} +func (m *Identify) Reset() { *m = Identify{} } +func (m *Identify) String() string { return proto.CompactTextString(m) } +func (*Identify) ProtoMessage() {} -func (m *Handshake3) GetH1() *Handshake1 { - if m != nil { - return m.H1 +func (m *Identify) GetProtocolVersion() string { + if m != nil && m.ProtocolVersion != nil { + return *m.ProtocolVersion } - return nil + return "" } -func (m *Handshake3) GetPublicKey() []byte { +func (m *Identify) GetAgentVersion() string { + if m != nil && m.AgentVersion != nil { + return *m.AgentVersion + } + return "" +} + +func (m *Identify) GetPublicKey() []byte { if m != nil { return m.PublicKey } return nil } -func (m *Handshake3) GetListenAddrs() [][]byte { +func (m *Identify) GetListenAddrs() [][]byte { if m != nil { return m.ListenAddrs } return nil } -func (m *Handshake3) GetProtocols() []string { - if m != nil { - return m.Protocols - } - return nil -} - -func (m *Handshake3) GetObservedAddr() []byte { +func (m *Identify) GetObservedAddr() []byte { if m != nil { return m.ObservedAddr } return nil } +func (m *Identify) GetProtocols() []string { + if m != nil { + return m.Protocols + } + return nil +} + func init() { } diff --git a/net/handshake/pb/handshake.proto b/p2p/protocol/identify/pb/identify.proto similarity index 61% rename from net/handshake/pb/handshake.proto rename to p2p/protocol/identify/pb/identify.proto index 8eb699559..7d31e0474 100644 --- a/net/handshake/pb/handshake.proto +++ b/p2p/protocol/identify/pb/identify.proto @@ -1,24 +1,13 @@ -package handshake.pb; +package identify.pb; -//import "github.com/jbenet/go-ipfs/net/mux/mux.proto"; +message Identify { -// Handshake1 is delivered _before_ the secure channel is initialized -message Handshake1 { // protocolVersion determines compatibility between peers - optional string protocolVersion = 1; // semver + optional string protocolVersion = 5; // e.g. ipfs/1.0.0 // agentVersion is like a UserAgent string in browsers, or client version in bittorrent - // includes the client name and client. e.g. "go-ipfs/0.1.0" - optional string agentVersion = 2; // semver - - // we'll have more fields here later. -} - -// Handshake3 is delivered _after_ the secure channel is initialized -message Handshake3 { - - // can include all the values in handshake1, for protocol version, etc. - optional Handshake1 h1 = 5; + // includes the client name and client. + optional string agentVersion = 6; // e.g. go-ipfs/0.1.0 // publicKey is this node's public key (which also gives its node.ID) // - may not need to be sent, as secure channel implies it has been sent. @@ -28,11 +17,11 @@ message Handshake3 { // listenAddrs are the multiaddrs the sender node listens for open connections on repeated bytes listenAddrs = 2; - // protocols are the services this node is running - repeated string protocols = 3; - // oservedAddr is the multiaddr of the remote endpoint that the sender node perceives // this is useful information to convey to the other side, as it helps the remote endpoint // determine whether its connection to the local peer goes through NAT. optional bytes observedAddr = 4; + + // protocols are the services this node is running + repeated string protocols = 3; } diff --git a/net/mux.go b/p2p/protocol/mux.go similarity index 61% rename from net/mux.go rename to p2p/protocol/mux.go index d2bb7dc22..711b99991 100644 --- a/net/mux.go +++ b/p2p/protocol/mux.go @@ -1,4 +1,4 @@ -package net +package protocol import ( "fmt" @@ -6,11 +6,15 @@ import ( "sync" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" + + inet "github.com/jbenet/go-ipfs/p2p/net" eventlog "github.com/jbenet/go-ipfs/util/eventlog" lgbl "github.com/jbenet/go-ipfs/util/eventlog/loggables" ) -var log = eventlog.Logger("network") +var log = eventlog.Logger("net/mux") + +type StreamHandlerMap map[ID]inet.StreamHandler // Mux provides simple stream multixplexing. // It helps you precisely when: @@ -20,26 +24,19 @@ var log = eventlog.Logger("network") // It contains the handlers for each protocol accepted. // It dispatches handlers for streams opened by remote peers. // -// We use a totally ad-hoc encoding: -// <1 byte length in bytes> -// So "bitswap" is 0x0762697473776170 -// -// NOTE: only the dialer specifies this muxing line. -// This is because we're using Streams :) -// // WARNING: this datastructure IS NOT threadsafe. // do not modify it once the network is using it. type Mux struct { - Default StreamHandler // handles unknown protocols. + Default inet.StreamHandler // handles unknown protocols. Handlers StreamHandlerMap sync.RWMutex } // Protocols returns the list of protocols this muxer has handlers for -func (m *Mux) Protocols() []ProtocolID { +func (m *Mux) Protocols() []ID { m.RLock() - l := make([]ProtocolID, 0, len(m.Handlers)) + l := make([]ID, 0, len(m.Handlers)) for p := range m.Handlers { l = append(l, p) } @@ -47,27 +44,27 @@ func (m *Mux) Protocols() []ProtocolID { return l } -// ReadProtocolHeader reads the stream and returns the next Handler function +// readHeader reads the stream and returns the next Handler function // according to the muxer encoding. -func (m *Mux) ReadProtocolHeader(s io.Reader) (string, StreamHandler, error) { +func (m *Mux) readHeader(s io.Reader) (ID, inet.StreamHandler, error) { // log.Error("ReadProtocolHeader") - name, err := ReadLengthPrefix(s) + p, err := ReadHeader(s) if err != nil { return "", nil, err } - // log.Debug("ReadProtocolHeader got:", name) + // log.Debug("readHeader got:", p) m.RLock() - h, found := m.Handlers[ProtocolID(name)] + h, found := m.Handlers[p] m.RUnlock() switch { case !found && m.Default != nil: - return name, m.Default, nil + return p, m.Default, nil case !found && m.Default == nil: - return name, nil, fmt.Errorf("%s no handler with name: %s (%d)", m, name, len(name)) + return p, nil, fmt.Errorf("%s no handler with name: %s (%d)", m, p, len(p)) default: - return name, h, nil + return p, h, nil } } @@ -80,26 +77,35 @@ func (m *Mux) String() string { // SetHandler sets the protocol handler on the Network's Muxer. // This operation is threadsafe. -func (m *Mux) SetHandler(p ProtocolID, h StreamHandler) { +func (m *Mux) SetHandler(p ID, h inet.StreamHandler) { log.Debugf("%s setting handler for protocol: %s (%d)", m, p, len(p)) m.Lock() m.Handlers[p] = h m.Unlock() } -// Handle reads the next name off the Stream, and calls a function -func (m *Mux) Handle(s Stream) { +// Handle reads the next name off the Stream, and calls a handler function +// This is done in its own goroutine, to avoid blocking the caller. +func (m *Mux) Handle(s inet.Stream) { + go m.HandleSync(s) +} + +// HandleSync reads the next name off the Stream, and calls a handler function +// This is done synchronously. The handler function will return before +// HandleSync returns. +func (m *Mux) HandleSync(s inet.Stream) { ctx := context.Background() - name, handler, err := m.ReadProtocolHeader(s) + name, handler, err := m.readHeader(s) if err != nil { err = fmt.Errorf("protocol mux error: %s", err) log.Error(err) log.Event(ctx, "muxError", lgbl.Error(err)) + s.Close() return } - log.Info("muxer handle protocol: %s", name) + log.Infof("muxer handle protocol: %s", name) log.Event(ctx, "muxHandle", eventlog.Metadata{"protocol": name}) handler(s) } @@ -122,14 +128,3 @@ func ReadLengthPrefix(r io.Reader) (string, error) { return string(name), nil } - -// WriteLengthPrefix writes the name into Writer with a length-byte-prefix. -func WriteLengthPrefix(w io.Writer, name string) error { - // log.Error("WriteLengthPrefix", name) - s := make([]byte, len(name)+1) - s[0] = byte(len(name)) - copy(s[1:], []byte(name)) - - _, err := w.Write(s) - return err -} diff --git a/net/mux_test.go b/p2p/protocol/mux_test.go similarity index 51% rename from net/mux_test.go rename to p2p/protocol/mux_test.go index 2dff54c33..dfac205da 100644 --- a/net/mux_test.go +++ b/p2p/protocol/mux_test.go @@ -1,21 +1,25 @@ -package net +package protocol import ( "bytes" "testing" + + inet "github.com/jbenet/go-ipfs/p2p/net" ) var testCases = map[string]string{ - "bitswap": "\u0007bitswap", - "dht": "\u0003dht", - "ipfs": "\u0004ipfs", - "ipfsdksnafkasnfkdajfkdajfdsjadosiaaodjasofdias": ".ipfsdksnafkasnfkdajfkdajfdsjadosiaaodjasofdias", + "/bitswap": "\u0009/bitswap\n", + "/dht": "\u0005/dht\n", + "/ipfs": "\u0006/ipfs\n", + "/ipfs/dksnafkasnfkdajfkdajfdsjadosiaaodj": ")/ipfs/dksnafkasnfkdajfkdajfdsjadosiaaodj\n", } func TestWrite(t *testing.T) { for k, v := range testCases { var buf bytes.Buffer - WriteLengthPrefix(&buf, k) + if err := WriteHeader(&buf, ID(k)); err != nil { + t.Fatal(err) + } v2 := buf.Bytes() if !bytes.Equal(v2, []byte(v)) { @@ -28,18 +32,18 @@ func TestHandler(t *testing.T) { outs := make(chan string, 10) - h := func(n string) func(s Stream) { - return func(s Stream) { + h := func(n string) func(s inet.Stream) { + return func(s inet.Stream) { outs <- n } } m := Mux{Handlers: StreamHandlerMap{}} m.Default = h("default") - m.Handlers["dht"] = h("bitswap") - // m.Handlers["ipfs"] = h("bitswap") // default! - m.Handlers["bitswap"] = h("bitswap") - m.Handlers["ipfsdksnafkasnfkdajfkdajfdsjadosiaaodjasofdias"] = h("bitswap") + m.Handlers["/dht"] = h("bitswap") + // m.Handlers["/ipfs"] = h("bitswap") // default! + m.Handlers["/bitswap"] = h("bitswap") + m.Handlers["/ipfs/dksnafkasnfkdajfkdajfdsjadosiaaodj"] = h("bitswap") for k, v := range testCases { var buf bytes.Buffer @@ -48,13 +52,13 @@ func TestHandler(t *testing.T) { continue } - name, _, err := m.ReadProtocolHeader(&buf) + name, err := ReadHeader(&buf) if err != nil { t.Error(err) continue } - if name != k { + if name != ID(k) { t.Errorf("name mismatch: %s != %s", k, name) continue } diff --git a/p2p/protocol/protocol.go b/p2p/protocol/protocol.go new file mode 100644 index 000000000..e192fb6a3 --- /dev/null +++ b/p2p/protocol/protocol.go @@ -0,0 +1,40 @@ +package protocol + +import ( + "io" + + msgio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-msgio" +) + +// ID is an identifier used to write protocol headers in streams. +type ID string + +// These are reserved protocol.IDs. +const ( + TestingID ID = "/p2p/_testing" +) + +// WriteHeader writes a protocol.ID header to an io.Writer. This is so +// multiple protocols can be multiplexed on top of the same transport. +// +// We use go-msgio varint encoding: +// \n +// (the varint includes the \n) +func WriteHeader(w io.Writer, id ID) error { + vw := msgio.NewVarintWriter(w) + s := string(id) + "\n" // add \n + return vw.WriteMsg([]byte(s)) +} + +// ReadHeader reads a protocol.ID header from an io.Reader. This is so +// multiple protocols can be multiplexed on top of the same transport. +// See WriteHeader. +func ReadHeader(r io.Reader) (ID, error) { + vr := msgio.NewVarintReader(r) + msg, err := vr.ReadMsg() + if err != nil { + return ID(""), err + } + msg = msg[:len(msg)-1] // remove \n + return ID(msg), nil +} diff --git a/p2p/protocol/relay/relay.go b/p2p/protocol/relay/relay.go new file mode 100644 index 000000000..4e4f0ba22 --- /dev/null +++ b/p2p/protocol/relay/relay.go @@ -0,0 +1,156 @@ +package relay + +import ( + "fmt" + "io" + + mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" + + host "github.com/jbenet/go-ipfs/p2p/host" + inet "github.com/jbenet/go-ipfs/p2p/net" + peer "github.com/jbenet/go-ipfs/p2p/peer" + protocol "github.com/jbenet/go-ipfs/p2p/protocol" + eventlog "github.com/jbenet/go-ipfs/util/eventlog" +) + +var log = eventlog.Logger("p2p/protocol/relay") + +// ID is the protocol.ID of the Relay Service. +const ID protocol.ID = "/ipfs/relay" + +// Relay is a structure that implements ProtocolRelay. +// It is a simple relay service which forwards traffic +// between two directly connected peers. +// +// the protocol is very simple: +// +// /ipfs/relay\n +// +// +// +// +type RelayService struct { + host host.Host + handler inet.StreamHandler // for streams sent to us locally. +} + +func NewRelayService(h host.Host, sh inet.StreamHandler) *RelayService { + s := &RelayService{ + host: h, + handler: sh, + } + h.SetStreamHandler(ID, s.requestHandler) + return s +} + +// requestHandler is the function called by clients +func (rs *RelayService) requestHandler(s inet.Stream) { + if err := rs.handleStream(s); err != nil { + log.Error("RelayService error:", err) + } +} + +// handleStream is our own handler, which returns an error for simplicity. +func (rs *RelayService) handleStream(s inet.Stream) error { + defer s.Close() + + // read the header (src and dst peer.IDs) + src, dst, err := ReadHeader(s) + if err != nil { + return fmt.Errorf("stream with bad header: %s", err) + } + + local := rs.host.ID() + + switch { + case src == local: + return fmt.Errorf("relaying from self") + case dst == local: // it's for us! yaaay. + log.Debugf("%s consuming stream from %s", local, src) + return rs.consumeStream(s) + default: // src and dst are not local. relay it. + log.Debugf("%s relaying stream %s <--> %s", local, src, dst) + return rs.pipeStream(src, dst, s) + } +} + +// consumeStream connects streams directed to the local peer +// to our handler, with the header now stripped (read). +func (rs *RelayService) consumeStream(s inet.Stream) error { + rs.handler(s) // boom. + return nil +} + +// pipeStream relays over a stream to a remote peer. It's like `cat` +func (rs *RelayService) pipeStream(src, dst peer.ID, s inet.Stream) error { + s2, err := rs.openStreamToPeer(dst) + if err != nil { + return fmt.Errorf("failed to open stream to peer: %s -- %s", dst, err) + } + + if err := WriteHeader(s2, src, dst); err != nil { + return err + } + + // connect the series of tubes. + done := make(chan retio, 2) + go func() { + n, err := io.Copy(s2, s) + done <- retio{n, err} + }() + go func() { + n, err := io.Copy(s, s2) + done <- retio{n, err} + }() + + r1 := <-done + r2 := <-done + log.Infof("%s relayed %d/%d bytes between %s and %s", rs.host.ID(), r1.n, r2.n, src, dst) + + if r1.err != nil { + return r1.err + } + return r2.err +} + +// openStreamToPeer opens a pipe to a remote endpoint +// for now, can only open streams to directly connected peers. +// maybe we can do some routing later on. +func (rs *RelayService) openStreamToPeer(p peer.ID) (inet.Stream, error) { + return rs.host.NewStream(ID, p) +} + +func ReadHeader(r io.Reader) (src, dst peer.ID, err error) { + + mhr := mh.NewReader(r) + + s, err := mhr.ReadMultihash() + if err != nil { + return "", "", err + } + + d, err := mhr.ReadMultihash() + if err != nil { + return "", "", err + } + + return peer.ID(s), peer.ID(d), nil +} + +func WriteHeader(w io.Writer, src, dst peer.ID) error { + // write header to w. + mhw := mh.NewWriter(w) + if err := mhw.WriteMultihash(mh.Multihash(src)); err != nil { + return fmt.Errorf("failed to write relay header: %s -- %s", dst, err) + } + if err := mhw.WriteMultihash(mh.Multihash(dst)); err != nil { + return fmt.Errorf("failed to write relay header: %s -- %s", dst, err) + } + + return nil +} + +type retio struct { + n int64 + err error +} diff --git a/p2p/protocol/relay/relay_test.go b/p2p/protocol/relay/relay_test.go new file mode 100644 index 000000000..14145044e --- /dev/null +++ b/p2p/protocol/relay/relay_test.go @@ -0,0 +1,303 @@ +package relay_test + +import ( + "io" + "testing" + + inet "github.com/jbenet/go-ipfs/p2p/net" + protocol "github.com/jbenet/go-ipfs/p2p/protocol" + relay "github.com/jbenet/go-ipfs/p2p/protocol/relay" + testutil "github.com/jbenet/go-ipfs/p2p/test/util" + eventlog "github.com/jbenet/go-ipfs/util/eventlog" + + context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" +) + +var log = eventlog.Logger("relay_test") + +func TestRelaySimple(t *testing.T) { + + ctx := context.Background() + + // these networks have the relay service wired in already. + n1 := testutil.GenHostSwarm(t, ctx) + n2 := testutil.GenHostSwarm(t, ctx) + n3 := testutil.GenHostSwarm(t, ctx) + + n1p := n1.ID() + n2p := n2.ID() + n3p := n3.ID() + + n2pi := n2.Peerstore().PeerInfo(n2p) + if err := n1.Connect(ctx, n2pi); err != nil { + t.Fatal("Failed to connect:", err) + } + if err := n3.Connect(ctx, n2pi); err != nil { + t.Fatal("Failed to connect:", err) + } + + // setup handler on n3 to copy everything over to the pipe. + piper, pipew := io.Pipe() + n3.SetStreamHandler(protocol.TestingID, func(s inet.Stream) { + log.Debug("relay stream opened to n3!") + log.Debug("piping and echoing everything") + w := io.MultiWriter(s, pipew) + io.Copy(w, s) + log.Debug("closing stream") + s.Close() + }) + + // ok, now we can try to relay n1--->n2--->n3. + log.Debug("open relay stream") + s, err := n1.NewStream(relay.ID, n2p) + if err != nil { + t.Fatal(err) + } + + // ok first thing we write the relay header n1->n3 + log.Debug("write relay header") + if err := relay.WriteHeader(s, n1p, n3p); err != nil { + t.Fatal(err) + } + + // ok now the header's there, we can write the next protocol header. + log.Debug("write testing header") + if err := protocol.WriteHeader(s, protocol.TestingID); err != nil { + t.Fatal(err) + } + + // okay, now we should be able to write text, and read it out. + buf1 := []byte("abcdefghij") + buf2 := make([]byte, 10) + buf3 := make([]byte, 10) + log.Debug("write in some text.") + if _, err := s.Write(buf1); err != nil { + t.Fatal(err) + } + + // read it out from the pipe. + log.Debug("read it out from the pipe.") + if _, err := io.ReadFull(piper, buf2); err != nil { + t.Fatal(err) + } + if string(buf1) != string(buf2) { + t.Fatal("should've gotten that text out of the pipe") + } + + // read it out from the stream (echoed) + log.Debug("read it out from the stream (echoed).") + if _, err := io.ReadFull(s, buf3); err != nil { + t.Fatal(err) + } + if string(buf1) != string(buf3) { + t.Fatal("should've gotten that text out of the stream") + } + + // sweet. relay works. + log.Debug("sweet, relay works.") + s.Close() +} + +func TestRelayAcrossFour(t *testing.T) { + + ctx := context.Background() + + // these networks have the relay service wired in already. + n1 := testutil.GenHostSwarm(t, ctx) + n2 := testutil.GenHostSwarm(t, ctx) + n3 := testutil.GenHostSwarm(t, ctx) + n4 := testutil.GenHostSwarm(t, ctx) + n5 := testutil.GenHostSwarm(t, ctx) + + n1p := n1.ID() + n2p := n2.ID() + n3p := n3.ID() + n4p := n4.ID() + n5p := n5.ID() + + n2pi := n2.Peerstore().PeerInfo(n2p) + n4pi := n4.Peerstore().PeerInfo(n4p) + + if err := n1.Connect(ctx, n2pi); err != nil { + t.Fatalf("Failed to dial:", err) + } + if err := n3.Connect(ctx, n2pi); err != nil { + t.Fatalf("Failed to dial:", err) + } + if err := n3.Connect(ctx, n4pi); err != nil { + t.Fatalf("Failed to dial:", err) + } + if err := n5.Connect(ctx, n4pi); err != nil { + t.Fatalf("Failed to dial:", err) + } + + // setup handler on n5 to copy everything over to the pipe. + piper, pipew := io.Pipe() + n5.SetStreamHandler(protocol.TestingID, func(s inet.Stream) { + log.Debug("relay stream opened to n5!") + log.Debug("piping and echoing everything") + w := io.MultiWriter(s, pipew) + io.Copy(w, s) + log.Debug("closing stream") + s.Close() + }) + + // ok, now we can try to relay n1--->n2--->n3--->n4--->n5 + log.Debug("open relay stream") + s, err := n1.NewStream(relay.ID, n2p) + if err != nil { + t.Fatal(err) + } + + log.Debugf("write relay header n1->n3 (%s -> %s)", n1p, n3p) + if err := relay.WriteHeader(s, n1p, n3p); err != nil { + t.Fatal(err) + } + + log.Debugf("write relay header n1->n4 (%s -> %s)", n1p, n4p) + if err := protocol.WriteHeader(s, relay.ID); err != nil { + t.Fatal(err) + } + if err := relay.WriteHeader(s, n1p, n4p); err != nil { + t.Fatal(err) + } + + log.Debugf("write relay header n1->n5 (%s -> %s)", n1p, n5p) + if err := protocol.WriteHeader(s, relay.ID); err != nil { + t.Fatal(err) + } + if err := relay.WriteHeader(s, n1p, n5p); err != nil { + t.Fatal(err) + } + + // ok now the header's there, we can write the next protocol header. + log.Debug("write testing header") + if err := protocol.WriteHeader(s, protocol.TestingID); err != nil { + t.Fatal(err) + } + + // okay, now we should be able to write text, and read it out. + buf1 := []byte("abcdefghij") + buf2 := make([]byte, 10) + buf3 := make([]byte, 10) + log.Debug("write in some text.") + if _, err := s.Write(buf1); err != nil { + t.Fatal(err) + } + + // read it out from the pipe. + log.Debug("read it out from the pipe.") + if _, err := io.ReadFull(piper, buf2); err != nil { + t.Fatal(err) + } + if string(buf1) != string(buf2) { + t.Fatal("should've gotten that text out of the pipe") + } + + // read it out from the stream (echoed) + log.Debug("read it out from the stream (echoed).") + if _, err := io.ReadFull(s, buf3); err != nil { + t.Fatal(err) + } + if string(buf1) != string(buf3) { + t.Fatal("should've gotten that text out of the stream") + } + + // sweet. relay works. + log.Debug("sweet, relaying across 4 works.") + s.Close() +} + +func TestRelayStress(t *testing.T) { + buflen := 1 << 18 + iterations := 10 + + ctx := context.Background() + + // these networks have the relay service wired in already. + n1 := testutil.GenHostSwarm(t, ctx) + n2 := testutil.GenHostSwarm(t, ctx) + n3 := testutil.GenHostSwarm(t, ctx) + + n1p := n1.ID() + n2p := n2.ID() + n3p := n3.ID() + + n2pi := n2.Peerstore().PeerInfo(n2p) + if err := n1.Connect(ctx, n2pi); err != nil { + t.Fatalf("Failed to dial:", err) + } + if err := n3.Connect(ctx, n2pi); err != nil { + t.Fatalf("Failed to dial:", err) + } + + // setup handler on n3 to copy everything over to the pipe. + piper, pipew := io.Pipe() + n3.SetStreamHandler(protocol.TestingID, func(s inet.Stream) { + log.Debug("relay stream opened to n3!") + log.Debug("piping and echoing everything") + w := io.MultiWriter(s, pipew) + io.Copy(w, s) + log.Debug("closing stream") + s.Close() + }) + + // ok, now we can try to relay n1--->n2--->n3. + log.Debug("open relay stream") + s, err := n1.NewStream(relay.ID, n2p) + if err != nil { + t.Fatal(err) + } + + // ok first thing we write the relay header n1->n3 + log.Debug("write relay header") + if err := relay.WriteHeader(s, n1p, n3p); err != nil { + t.Fatal(err) + } + + // ok now the header's there, we can write the next protocol header. + log.Debug("write testing header") + if err := protocol.WriteHeader(s, protocol.TestingID); err != nil { + t.Fatal(err) + } + + // okay, now write lots of text and read it back out from both + // the pipe and the stream. + buf1 := make([]byte, buflen) + buf2 := make([]byte, len(buf1)) + buf3 := make([]byte, len(buf1)) + + fillbuf := func(buf []byte, b byte) { + for i := range buf { + buf[i] = b + } + } + + for i := 0; i < iterations; i++ { + fillbuf(buf1, byte(int('a')+i)) + log.Debugf("writing %d bytes (%d/%d)", len(buf1), i, iterations) + if _, err := s.Write(buf1); err != nil { + t.Fatal(err) + } + + log.Debug("read it out from the pipe.") + if _, err := io.ReadFull(piper, buf2); err != nil { + t.Fatal(err) + } + if string(buf1) != string(buf2) { + t.Fatal("should've gotten that text out of the pipe") + } + + // read it out from the stream (echoed) + log.Debug("read it out from the stream (echoed).") + if _, err := io.ReadFull(s, buf3); err != nil { + t.Fatal(err) + } + if string(buf1) != string(buf3) { + t.Fatal("should've gotten that text out of the stream") + } + } + + log.Debug("sweet, relay works under stress.") + s.Close() +} diff --git a/net/backpressure/backpressure.go b/p2p/test/backpressure/backpressure.go similarity index 100% rename from net/backpressure/backpressure.go rename to p2p/test/backpressure/backpressure.go diff --git a/net/backpressure/backpressure_test.go b/p2p/test/backpressure/backpressure_test.go similarity index 85% rename from net/backpressure/backpressure_test.go rename to p2p/test/backpressure/backpressure_test.go index 7a8593ee4..946b87b3a 100644 --- a/net/backpressure/backpressure_test.go +++ b/p2p/test/backpressure/backpressure_test.go @@ -7,31 +7,18 @@ import ( "testing" "time" - inet "github.com/jbenet/go-ipfs/net" - peer "github.com/jbenet/go-ipfs/peer" + host "github.com/jbenet/go-ipfs/p2p/host" + inet "github.com/jbenet/go-ipfs/p2p/net" + peer "github.com/jbenet/go-ipfs/p2p/peer" + protocol "github.com/jbenet/go-ipfs/p2p/protocol" + testutil "github.com/jbenet/go-ipfs/p2p/test/util" eventlog "github.com/jbenet/go-ipfs/util/eventlog" - testutil "github.com/jbenet/go-ipfs/util/testutil" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ) var log = eventlog.Logger("backpressure") -func GenNetwork(t *testing.T, ctx context.Context) (inet.Network, error) { - p := testutil.RandPeerNetParamsOrFatal(t) - ps := peer.NewPeerstore() - ps.AddAddress(p.ID, p.Addr) - ps.AddPubKey(p.ID, p.PubKey) - ps.AddPrivKey(p.ID, p.PrivKey) - return inet.NewNetwork(ctx, ps.Addresses(p.ID), p.ID, ps) -} - -func divulgeAddresses(a, b inet.Network) { - id := a.LocalPeer() - addrs := a.Peerstore().Addresses(id) - b.Peerstore().AddAddresses(id, addrs) -} - // TestBackpressureStreamHandler tests whether mux handler // ratelimiting works. Meaning, since the handler is sequential // it should block senders. @@ -87,7 +74,7 @@ a problem. } // the sender opens streams as fast as possible - sender := func(net inet.Network, remote peer.ID) { + sender := func(host host.Host, remote peer.ID) { var s inet.Stream var err error defer func() { @@ -97,7 +84,7 @@ a problem. }() for { - s, err = net.NewStream(inet.ProtocolTesting, remote) + s, err = host.NewStream(protocol.TestingID, remote) if err != nil { return } @@ -149,25 +136,20 @@ a problem. // ok that's enough setup. let's do it! ctx := context.Background() - n1, err := GenNetwork(t, ctx) - if err != nil { - t.Fatal(err) - } - n2, err := GenNetwork(t, ctx) - if err != nil { - t.Fatal(err) - } + h1 := testutil.GenHostSwarm(t, ctx) + h2 := testutil.GenHostSwarm(t, ctx) // setup receiver handler - n1.SetHandler(inet.ProtocolTesting, receiver) + h1.SetStreamHandler(protocol.TestingID, receiver) - log.Debugf("dialing %s", n2.ListenAddresses()) - if err := n1.DialPeer(ctx, n2.LocalPeer()); err != nil { - t.Fatalf("Failed to dial:", err) + h2pi := h2.Peerstore().PeerInfo(h2.ID()) + log.Debugf("dialing %s", h2pi.Addrs) + if err := h1.Connect(ctx, h2pi); err != nil { + t.Fatalf("Failed to connect:", err) } // launch sender! - go sender(n2, n1.LocalPeer()) + go sender(h2, h1.ID()) // ok, what do we expect to happen? the receiver should // receive 10 requests and stop receiving, blocking the sender. @@ -201,7 +183,7 @@ a problem. // now for the sugar on top: let's tear down the receiver. it should // exit the sender. - n1.Close() + h1.Close() // shouldn't have opened anything more testStreamsOpened(0) @@ -291,28 +273,23 @@ func TestStBackpressureStreamWrite(t *testing.T) { // setup the networks ctx := context.Background() - n1, err := GenNetwork(t, ctx) - if err != nil { - t.Fatal(err) - } - n2, err := GenNetwork(t, ctx) - if err != nil { - t.Fatal(err) - } - - divulgeAddresses(n1, n2) - divulgeAddresses(n2, n1) + h1 := testutil.GenHostSwarm(t, ctx) + h2 := testutil.GenHostSwarm(t, ctx) // setup sender handler on 1 - n1.SetHandler(inet.ProtocolTesting, sender) + h1.SetStreamHandler(protocol.TestingID, sender) - log.Debugf("dialing %s", n2.ListenAddresses()) - if err := n1.DialPeer(ctx, n2.LocalPeer()); err != nil { - t.Fatalf("Failed to dial:", err) + h2pi := h2.Peerstore().PeerInfo(h2.ID()) + log.Debugf("dialing %s", h2pi.Addrs) + if err := h1.Connect(ctx, h2pi); err != nil { + t.Fatalf("Failed to connect:", err) } // open a stream, from 2->1, this is our reader - s, err := n2.NewStream(inet.ProtocolTesting, n1.LocalPeer()) + s, err := h2.NewStream(protocol.TestingID, h1.ID()) + if err != nil { + t.Fatal(err) + } // let's make sure r/w works. testSenderWrote := func(bytesE int) { diff --git a/p2p/test/util/util.go b/p2p/test/util/util.go new file mode 100644 index 000000000..30c9d1f99 --- /dev/null +++ b/p2p/test/util/util.go @@ -0,0 +1,38 @@ +package testutil + +import ( + "testing" + + bhost "github.com/jbenet/go-ipfs/p2p/host/basic" + inet "github.com/jbenet/go-ipfs/p2p/net" + swarm "github.com/jbenet/go-ipfs/p2p/net/swarm" + peer "github.com/jbenet/go-ipfs/p2p/peer" + tu "github.com/jbenet/go-ipfs/util/testutil" + + context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" + ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" +) + +func GenSwarmNetwork(t *testing.T, ctx context.Context) *swarm.Network { + p := tu.RandPeerNetParamsOrFatal(t) + ps := peer.NewPeerstore() + ps.AddPubKey(p.ID, p.PubKey) + ps.AddPrivKey(p.ID, p.PrivKey) + n, err := swarm.NewNetwork(ctx, []ma.Multiaddr{p.Addr}, p.ID, ps) + if err != nil { + t.Fatal(err) + } + ps.AddAddresses(p.ID, n.ListenAddresses()) + return n +} + +func DivulgeAddresses(a, b inet.Network) { + id := a.LocalPeer() + addrs := a.Peerstore().Addresses(id) + b.Peerstore().AddAddresses(id, addrs) +} + +func GenHostSwarm(t *testing.T, ctx context.Context) *bhost.BasicHost { + n := GenSwarmNetwork(t, ctx) + return bhost.New(n) +} diff --git a/routing/dht/dht.go b/routing/dht/dht.go index fb7c5a49b..2a576629a 100644 --- a/routing/dht/dht.go +++ b/routing/dht/dht.go @@ -10,8 +10,9 @@ import ( "sync" "time" - inet "github.com/jbenet/go-ipfs/net" - peer "github.com/jbenet/go-ipfs/peer" + host "github.com/jbenet/go-ipfs/p2p/host" + peer "github.com/jbenet/go-ipfs/p2p/peer" + protocol "github.com/jbenet/go-ipfs/p2p/protocol" routing "github.com/jbenet/go-ipfs/routing" pb "github.com/jbenet/go-ipfs/routing/dht/pb" kb "github.com/jbenet/go-ipfs/routing/kbucket" @@ -26,6 +27,8 @@ import ( var log = eventlog.Logger("dht") +var ProtocolDHT protocol.ID = "/ipfs/dht" + const doPinging = false // NumBootstrapQueries defines the number of random dht queries to do to @@ -37,7 +40,7 @@ const NumBootstrapQueries = 5 // IpfsDHT is an implementation of Kademlia with Coral and S/Kademlia modifications. // It is used to implement the base IpfsRouting module. type IpfsDHT struct { - network inet.Network // the network services we need + host host.Host // the network services we need self peer.ID // Local peer (yourself) peerstore peer.Peerstore // Peer Registry @@ -56,19 +59,19 @@ type IpfsDHT struct { } // NewDHT creates a new DHT object with the given peer as the 'local' host -func NewDHT(ctx context.Context, p peer.ID, n inet.Network, dstore ds.ThreadSafeDatastore) *IpfsDHT { +func NewDHT(ctx context.Context, h host.Host, dstore ds.ThreadSafeDatastore) *IpfsDHT { dht := new(IpfsDHT) dht.datastore = dstore - dht.self = p - dht.peerstore = n.Peerstore() + dht.self = h.ID() + dht.peerstore = h.Peerstore() dht.ContextGroup = ctxgroup.WithContext(ctx) - dht.network = n - n.SetHandler(inet.ProtocolDHT, dht.handleNewStream) + dht.host = h + h.SetStreamHandler(ProtocolDHT, dht.handleNewStream) - dht.providers = NewProviderManager(dht.Context(), p) + dht.providers = NewProviderManager(dht.Context(), dht.self) dht.AddChildGroup(dht.providers) - dht.routingTable = kb.NewRoutingTable(20, kb.ConvertPeerID(p), time.Minute, dht.peerstore) + dht.routingTable = kb.NewRoutingTable(20, kb.ConvertPeerID(dht.self), time.Minute, dht.peerstore) dht.birth = time.Now() dht.Validators = make(map[string]ValidatorFunc) @@ -88,7 +91,8 @@ func (dht *IpfsDHT) LocalPeer() peer.ID { // Connect to a new peer at the given address, ping and add to the routing table func (dht *IpfsDHT) Connect(ctx context.Context, npeer peer.ID) error { - if err := dht.network.DialPeer(ctx, npeer); err != nil { + // TODO: change interface to accept a PeerInfo as well. + if err := dht.host.Connect(ctx, peer.PeerInfo{ID: npeer}); err != nil { return err } @@ -127,7 +131,7 @@ func (dht *IpfsDHT) putProvider(ctx context.Context, p peer.ID, key string) erro // add self as the provider pi := dht.peerstore.PeerInfo(dht.self) - pmes.ProviderPeers = pb.PeerInfosToPBPeers(dht.network, []peer.PeerInfo{pi}) + pmes.ProviderPeers = pb.PeerInfosToPBPeers(dht.host.Network(), []peer.PeerInfo{pi}) err := dht.sendMessage(ctx, p, pmes) if err != nil { @@ -304,7 +308,7 @@ func (dht *IpfsDHT) ensureConnectedToPeer(ctx context.Context, p peer.ID) error } // dial connection - return dht.network.DialPeer(ctx, p) + return dht.host.Connect(ctx, peer.PeerInfo{ID: p}) } // PingRoutine periodically pings nearest neighbors. diff --git a/routing/dht/dht_net.go b/routing/dht/dht_net.go index d247cf3af..fd088e02c 100644 --- a/routing/dht/dht_net.go +++ b/routing/dht/dht_net.go @@ -4,8 +4,8 @@ import ( "errors" "time" - inet "github.com/jbenet/go-ipfs/net" - peer "github.com/jbenet/go-ipfs/peer" + inet "github.com/jbenet/go-ipfs/p2p/net" + peer "github.com/jbenet/go-ipfs/p2p/peer" pb "github.com/jbenet/go-ipfs/routing/dht/pb" ctxutil "github.com/jbenet/go-ipfs/util/ctx" @@ -74,7 +74,7 @@ func (dht *IpfsDHT) handleNewMessage(s inet.Stream) { func (dht *IpfsDHT) sendRequest(ctx context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) { log.Debugf("%s dht starting stream", dht.self) - s, err := dht.network.NewStream(inet.ProtocolDHT, p) + s, err := dht.host.NewStream(ProtocolDHT, p) if err != nil { return nil, err } @@ -116,7 +116,7 @@ func (dht *IpfsDHT) sendRequest(ctx context.Context, p peer.ID, pmes *pb.Message func (dht *IpfsDHT) sendMessage(ctx context.Context, p peer.ID, pmes *pb.Message) error { log.Debugf("%s dht starting stream", dht.self) - s, err := dht.network.NewStream(inet.ProtocolDHT, p) + s, err := dht.host.NewStream(ProtocolDHT, p) if err != nil { return err } diff --git a/routing/dht/dht_test.go b/routing/dht/dht_test.go index 5eeb3a2bc..133f7a27c 100644 --- a/routing/dht/dht_test.go +++ b/routing/dht/dht_test.go @@ -14,11 +14,10 @@ import ( dssync "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - inet "github.com/jbenet/go-ipfs/net" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" + netutil "github.com/jbenet/go-ipfs/p2p/test/util" routing "github.com/jbenet/go-ipfs/routing" u "github.com/jbenet/go-ipfs/util" - testutil "github.com/jbenet/go-ipfs/util/testutil" ) var testCaseValues = map[u.Key][]byte{} @@ -32,30 +31,11 @@ func init() { } } -func setupDHT(ctx context.Context, t *testing.T, addr ma.Multiaddr) *IpfsDHT { - - sk, pk, err := testutil.SeededKeyPair(time.Now().UnixNano()) - if err != nil { - t.Fatal(err) - } - - p, err := peer.IDFromPublicKey(pk) - if err != nil { - t.Fatal(err) - } - - peerstore := peer.NewPeerstore() - peerstore.AddPrivKey(p, sk) - peerstore.AddPubKey(p, pk) - peerstore.AddAddress(p, addr) - - n, err := inet.NewNetwork(ctx, []ma.Multiaddr{addr}, p, peerstore) - if err != nil { - t.Fatal(err) - } +func setupDHT(ctx context.Context, t *testing.T) *IpfsDHT { + h := netutil.GenHostSwarm(t, ctx) dss := dssync.MutexWrap(ds.NewMapDatastore()) - d := NewDHT(ctx, p, n, dss) + d := NewDHT(ctx, h, dss) d.Validators["v"] = func(u.Key, []byte) error { return nil @@ -69,9 +49,9 @@ func setupDHTS(ctx context.Context, n int, t *testing.T) ([]ma.Multiaddr, []peer peers := make([]peer.ID, n) for i := 0; i < n; i++ { - addrs[i] = testutil.RandLocalTCPAddress() - dhts[i] = setupDHT(ctx, t, addrs[i]) + dhts[i] = setupDHT(ctx, t) peers[i] = dhts[i].self + addrs[i] = dhts[i].peerstore.Addresses(dhts[i].self)[0] } return addrs, peers, dhts @@ -116,19 +96,16 @@ func TestPing(t *testing.T) { // t.Skip("skipping test to debug another") ctx := context.Background() - addrA := testutil.RandLocalTCPAddress() - addrB := testutil.RandLocalTCPAddress() - - dhtA := setupDHT(ctx, t, addrA) - dhtB := setupDHT(ctx, t, addrB) + dhtA := setupDHT(ctx, t) + dhtB := setupDHT(ctx, t) peerA := dhtA.self peerB := dhtB.self defer dhtA.Close() defer dhtB.Close() - defer dhtA.network.Close() - defer dhtB.network.Close() + defer dhtA.host.Close() + defer dhtB.host.Close() connect(t, ctx, dhtA, dhtB) @@ -149,16 +126,13 @@ func TestValueGetSet(t *testing.T) { ctx := context.Background() - addrA := testutil.RandLocalTCPAddress() - addrB := testutil.RandLocalTCPAddress() - - dhtA := setupDHT(ctx, t, addrA) - dhtB := setupDHT(ctx, t, addrB) + dhtA := setupDHT(ctx, t) + dhtB := setupDHT(ctx, t) defer dhtA.Close() defer dhtB.Close() - defer dhtA.network.Close() - defer dhtB.network.Close() + defer dhtA.host.Close() + defer dhtB.host.Close() vf := func(u.Key, []byte) error { return nil @@ -200,7 +174,7 @@ func TestProvides(t *testing.T) { defer func() { for i := 0; i < 4; i++ { dhts[i].Close() - defer dhts[i].network.Close() + defer dhts[i].host.Close() } }() @@ -268,7 +242,7 @@ func TestBootstrap(t *testing.T) { defer func() { for i := 0; i < nDHTs; i++ { dhts[i].Close() - defer dhts[i].network.Close() + defer dhts[i].host.Close() } }() @@ -312,7 +286,7 @@ func TestProvidesMany(t *testing.T) { defer func() { for i := 0; i < nDHTs; i++ { dhts[i].Close() - defer dhts[i].network.Close() + defer dhts[i].host.Close() } }() @@ -424,7 +398,7 @@ func TestProvidesAsync(t *testing.T) { defer func() { for i := 0; i < 4; i++ { dhts[i].Close() - defer dhts[i].network.Close() + defer dhts[i].host.Close() } }() @@ -478,7 +452,7 @@ func TestLayeredGet(t *testing.T) { defer func() { for i := 0; i < 4; i++ { dhts[i].Close() - defer dhts[i].network.Close() + defer dhts[i].host.Close() } }() @@ -518,7 +492,7 @@ func TestFindPeer(t *testing.T) { defer func() { for i := 0; i < 4; i++ { dhts[i].Close() - dhts[i].network.Close() + dhts[i].host.Close() } }() @@ -554,7 +528,7 @@ func TestFindPeersConnectedToPeer(t *testing.T) { defer func() { for i := 0; i < 4; i++ { dhts[i].Close() - dhts[i].network.Close() + dhts[i].host.Close() } }() @@ -633,11 +607,11 @@ func TestConnectCollision(t *testing.T) { ctx := context.Background() - addrA := testutil.RandLocalTCPAddress() - addrB := testutil.RandLocalTCPAddress() + dhtA := setupDHT(ctx, t) + dhtB := setupDHT(ctx, t) - dhtA := setupDHT(ctx, t, addrA) - dhtB := setupDHT(ctx, t, addrB) + addrA := dhtA.peerstore.Addresses(dhtA.self)[0] + addrB := dhtB.peerstore.Addresses(dhtB.self)[0] peerA := dhtA.self peerB := dhtB.self @@ -674,7 +648,7 @@ func TestConnectCollision(t *testing.T) { dhtA.Close() dhtB.Close() - dhtA.network.Close() - dhtB.network.Close() + dhtA.host.Close() + dhtB.host.Close() } } diff --git a/routing/dht/diag.go b/routing/dht/diag.go index 96d2b1a01..79b4709e9 100644 --- a/routing/dht/diag.go +++ b/routing/dht/diag.go @@ -4,7 +4,7 @@ import ( "encoding/json" "time" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" ) type connDiagInfo struct { diff --git a/routing/dht/ext_test.go b/routing/dht/ext_test.go index 8441c1f72..2be8127c7 100644 --- a/routing/dht/ext_test.go +++ b/routing/dht/ext_test.go @@ -1,12 +1,14 @@ package dht import ( + "io" + "io/ioutil" "math/rand" "testing" - inet "github.com/jbenet/go-ipfs/net" - mocknet "github.com/jbenet/go-ipfs/net/mock" - peer "github.com/jbenet/go-ipfs/peer" + inet "github.com/jbenet/go-ipfs/p2p/net" + mocknet "github.com/jbenet/go-ipfs/p2p/net/mock" + peer "github.com/jbenet/go-ipfs/p2p/peer" routing "github.com/jbenet/go-ipfs/routing" pb "github.com/jbenet/go-ipfs/routing/dht/pb" u "github.com/jbenet/go-ipfs/util" @@ -29,13 +31,20 @@ func TestGetFailures(t *testing.T) { if err != nil { t.Fatal(err) } - nets := mn.Nets() + hosts := mn.Hosts() peers := mn.Peers() tsds := dssync.MutexWrap(ds.NewMapDatastore()) - d := NewDHT(ctx, peers[0], nets[0], tsds) + d := NewDHT(ctx, hosts[0], tsds) d.Update(ctx, peers[1]) + // u.POut("NotFound Test\n") + // Reply with failures to every message + hosts[1].SetStreamHandler(ProtocolDHT, func(s inet.Stream) { + defer s.Close() + io.Copy(ioutil.Discard, s) + }) + // This one should time out // u.POut("Timout Test\n") ctx1, _ := context.WithTimeout(context.Background(), time.Second) @@ -50,7 +59,7 @@ func TestGetFailures(t *testing.T) { t.Log("Timeout test passed.") // Reply with failures to every message - nets[1].SetHandler(inet.ProtocolDHT, func(s inet.Stream) { + hosts[1].SetStreamHandler(ProtocolDHT, func(s inet.Stream) { defer s.Close() pbr := ggio.NewDelimitedReader(s, inet.MessageSizeMax) @@ -97,7 +106,7 @@ func TestGetFailures(t *testing.T) { } // u.POut("handleGetValue Test\n") - s, err := nets[1].NewStream(inet.ProtocolDHT, peers[0]) + s, err := hosts[1].NewStream(ProtocolDHT, hosts[0].ID()) if err != nil { t.Fatal(err) } @@ -133,19 +142,19 @@ func TestNotFound(t *testing.T) { if err != nil { t.Fatal(err) } - nets := mn.Nets() + hosts := mn.Hosts() peers := mn.Peers() tsds := dssync.MutexWrap(ds.NewMapDatastore()) - d := NewDHT(ctx, peers[0], nets[0], tsds) + d := NewDHT(ctx, hosts[0], tsds) for _, p := range peers { d.Update(ctx, p) } // Reply with random peers to every message - for _, neti := range nets { - neti := neti // shadow loop var - neti.SetHandler(inet.ProtocolDHT, func(s inet.Stream) { + for _, host := range hosts { + host := host // shadow loop var + host.SetStreamHandler(ProtocolDHT, func(s inet.Stream) { defer s.Close() pbr := ggio.NewDelimitedReader(s, inet.MessageSizeMax) @@ -163,11 +172,11 @@ func TestNotFound(t *testing.T) { ps := []peer.PeerInfo{} for i := 0; i < 7; i++ { p := peers[rand.Intn(len(peers))] - pi := neti.Peerstore().PeerInfo(p) + pi := host.Peerstore().PeerInfo(p) ps = append(ps, pi) } - resp.CloserPeers = pb.PeerInfosToPBPeers(d.network, ps) + resp.CloserPeers = pb.PeerInfosToPBPeers(d.host.Network(), ps) if err := pbw.WriteMsg(resp); err != nil { panic(err) } @@ -205,20 +214,20 @@ func TestLessThanKResponses(t *testing.T) { if err != nil { t.Fatal(err) } - nets := mn.Nets() + hosts := mn.Hosts() peers := mn.Peers() tsds := dssync.MutexWrap(ds.NewMapDatastore()) - d := NewDHT(ctx, peers[0], nets[0], tsds) + d := NewDHT(ctx, hosts[0], tsds) for i := 1; i < 5; i++ { d.Update(ctx, peers[i]) } // Reply with random peers to every message - for _, neti := range nets { - neti := neti // shadow loop var - neti.SetHandler(inet.ProtocolDHT, func(s inet.Stream) { + for _, host := range hosts { + host := host // shadow loop var + host.SetStreamHandler(ProtocolDHT, func(s inet.Stream) { defer s.Close() pbr := ggio.NewDelimitedReader(s, inet.MessageSizeMax) @@ -231,10 +240,10 @@ func TestLessThanKResponses(t *testing.T) { switch pmes.GetType() { case pb.Message_GET_VALUE: - pi := neti.Peerstore().PeerInfo(peers[1]) + pi := host.Peerstore().PeerInfo(peers[1]) resp := &pb.Message{ Type: pmes.Type, - CloserPeers: pb.PeerInfosToPBPeers(d.network, []peer.PeerInfo{pi}), + CloserPeers: pb.PeerInfosToPBPeers(d.host.Network(), []peer.PeerInfo{pi}), } if err := pbw.WriteMsg(resp); err != nil { diff --git a/routing/dht/handlers.go b/routing/dht/handlers.go index acb052248..3670c570d 100644 --- a/routing/dht/handlers.go +++ b/routing/dht/handlers.go @@ -8,7 +8,7 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" pb "github.com/jbenet/go-ipfs/routing/dht/pb" u "github.com/jbenet/go-ipfs/util" ) @@ -89,7 +89,7 @@ func (dht *IpfsDHT) handleGetValue(ctx context.Context, p peer.ID, pmes *pb.Mess provinfos := peer.PeerInfos(dht.peerstore, provs) if len(provs) > 0 { log.Debugf("handleGetValue returning %d provider[s]", len(provs)) - resp.ProviderPeers = pb.PeerInfosToPBPeers(dht.network, provinfos) + resp.ProviderPeers = pb.PeerInfosToPBPeers(dht.host.Network(), provinfos) } // Find closest peer on given cluster to desired key and reply with that info @@ -106,7 +106,7 @@ func (dht *IpfsDHT) handleGetValue(ctx context.Context, p peer.ID, pmes *pb.Mess } } - resp.CloserPeers = pb.PeerInfosToPBPeers(dht.network, closerinfos) + resp.CloserPeers = pb.PeerInfosToPBPeers(dht.host.Network(), closerinfos) } return resp, nil @@ -161,7 +161,7 @@ func (dht *IpfsDHT) handleFindPeer(ctx context.Context, p peer.ID, pmes *pb.Mess } } - resp.CloserPeers = pb.PeerInfosToPBPeers(dht.network, withAddresses) + resp.CloserPeers = pb.PeerInfosToPBPeers(dht.host.Network(), withAddresses) return resp, nil } @@ -185,14 +185,14 @@ func (dht *IpfsDHT) handleGetProviders(ctx context.Context, p peer.ID, pmes *pb. if providers != nil && len(providers) > 0 { infos := peer.PeerInfos(dht.peerstore, providers) - resp.ProviderPeers = pb.PeerInfosToPBPeers(dht.network, infos) + resp.ProviderPeers = pb.PeerInfosToPBPeers(dht.host.Network(), infos) } // Also send closer peers. closer := dht.betterPeersToQuery(pmes, p, CloserPeerCount) if closer != nil { infos := peer.PeerInfos(dht.peerstore, providers) - resp.CloserPeers = pb.PeerInfosToPBPeers(dht.network, infos) + resp.CloserPeers = pb.PeerInfosToPBPeers(dht.host.Network(), infos) } return resp, nil diff --git a/routing/dht/pb/message.go b/routing/dht/pb/message.go index 570c7cf18..61bf41ebb 100644 --- a/routing/dht/pb/message.go +++ b/routing/dht/pb/message.go @@ -3,8 +3,8 @@ package dht_pb import ( ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - inet "github.com/jbenet/go-ipfs/net" - peer "github.com/jbenet/go-ipfs/peer" + inet "github.com/jbenet/go-ipfs/p2p/net" + peer "github.com/jbenet/go-ipfs/p2p/peer" eventlog "github.com/jbenet/go-ipfs/util/eventlog" ) diff --git a/routing/dht/providers.go b/routing/dht/providers.go index 861c25f0c..9e96eff36 100644 --- a/routing/dht/providers.go +++ b/routing/dht/providers.go @@ -4,7 +4,7 @@ import ( "time" ctxgroup "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" u "github.com/jbenet/go-ipfs/util" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" diff --git a/routing/dht/providers_test.go b/routing/dht/providers_test.go index 35ff92dfe..2781a3c59 100644 --- a/routing/dht/providers_test.go +++ b/routing/dht/providers_test.go @@ -3,7 +3,7 @@ package dht import ( "testing" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" u "github.com/jbenet/go-ipfs/util" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" diff --git a/routing/dht/query.go b/routing/dht/query.go index 4790e814c..0056bee1d 100644 --- a/routing/dht/query.go +++ b/routing/dht/query.go @@ -3,9 +3,8 @@ package dht import ( "sync" - inet "github.com/jbenet/go-ipfs/net" - peer "github.com/jbenet/go-ipfs/peer" - queue "github.com/jbenet/go-ipfs/peer/queue" + peer "github.com/jbenet/go-ipfs/p2p/peer" + queue "github.com/jbenet/go-ipfs/p2p/peer/queue" "github.com/jbenet/go-ipfs/routing" u "github.com/jbenet/go-ipfs/util" pset "github.com/jbenet/go-ipfs/util/peerset" @@ -18,17 +17,10 @@ import ( var maxQueryConcurrency = AlphaValue type dhtQuery struct { - // the key we're querying for - key u.Key - - // dialer used to ensure we're connected to peers - dialer inet.Dialer - - // the function to execute per peer - qfunc queryFunc - - // the concurrency parameter - concurrency int + dht *IpfsDHT + key u.Key // the key we're querying for + qfunc queryFunc // the function to execute per peer + concurrency int // the concurrency parameter } type dhtQueryResult struct { @@ -40,10 +32,10 @@ type dhtQueryResult struct { } // constructs query -func newQuery(k u.Key, d inet.Dialer, f queryFunc) *dhtQuery { +func (dht *IpfsDHT) newQuery(k u.Key, f queryFunc) *dhtQuery { return &dhtQuery{ key: k, - dialer: d, + dht: dht, qfunc: f, concurrency: maxQueryConcurrency, } @@ -155,7 +147,7 @@ func (r *dhtQueryRunner) Run(peers []peer.ID) (*dhtQueryResult, error) { func (r *dhtQueryRunner) addPeerToQuery(ctx context.Context, next peer.ID) { // if new peer is ourselves... - if next == r.query.dialer.LocalPeer() { + if next == r.query.dht.self { return } @@ -222,10 +214,11 @@ func (r *dhtQueryRunner) queryPeer(cg ctxgroup.ContextGroup, p peer.ID) { }() // make sure we're connected to the peer. - if conns := r.query.dialer.ConnsToPeer(p); len(conns) == 0 { + if conns := r.query.dht.host.Network().ConnsToPeer(p); len(conns) == 0 { log.Infof("worker for: %v -- not connected. dial start", p) - if err := r.query.dialer.DialPeer(cg.Context(), p); err != nil { + pi := peer.PeerInfo{ID: p} + if err := r.query.dht.host.Connect(cg.Context(), pi); err != nil { log.Debugf("ERROR worker for: %v -- err connecting: %v", p, err) r.Lock() r.errs = append(r.errs, err) @@ -257,12 +250,7 @@ func (r *dhtQueryRunner) queryPeer(cg ctxgroup.ContextGroup, p peer.ID) { log.Debugf("PEERS CLOSER -- worker for: %v (%d closer peers)", p, len(res.closerPeers)) for _, next := range res.closerPeers { // add their addresses to the dialer's peerstore - conns := r.query.dialer.ConnsToPeer(next.ID) - if len(conns) == 0 { - log.Infof("PEERS CLOSER -- worker for %v FOUND NEW PEER: %s %s", p, next.ID, next.Addrs) - } - - r.query.dialer.Peerstore().AddAddresses(next.ID, next.Addrs) + r.query.dht.peerstore.AddPeerInfo(next) r.addPeerToQuery(cg.Context(), next.ID) log.Debugf("PEERS CLOSER -- worker for: %v added %v (%v)", p, next.ID, next.Addrs) } diff --git a/routing/dht/records.go b/routing/dht/records.go index cf383916b..0791f80a3 100644 --- a/routing/dht/records.go +++ b/routing/dht/records.go @@ -9,8 +9,8 @@ import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" - ci "github.com/jbenet/go-ipfs/crypto" - "github.com/jbenet/go-ipfs/peer" + ci "github.com/jbenet/go-ipfs/p2p/crypto" + "github.com/jbenet/go-ipfs/p2p/peer" pb "github.com/jbenet/go-ipfs/routing/dht/pb" u "github.com/jbenet/go-ipfs/util" ctxutil "github.com/jbenet/go-ipfs/util/ctx" diff --git a/routing/dht/routing.go b/routing/dht/routing.go index 2a948f4be..2f00929b6 100644 --- a/routing/dht/routing.go +++ b/routing/dht/routing.go @@ -6,8 +6,8 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" - inet "github.com/jbenet/go-ipfs/net" - peer "github.com/jbenet/go-ipfs/peer" + inet "github.com/jbenet/go-ipfs/p2p/net" + peer "github.com/jbenet/go-ipfs/p2p/peer" "github.com/jbenet/go-ipfs/routing" pb "github.com/jbenet/go-ipfs/routing/dht/pb" kb "github.com/jbenet/go-ipfs/routing/kbucket" @@ -82,7 +82,7 @@ func (dht *IpfsDHT) GetValue(ctx context.Context, key u.Key) ([]byte, error) { } // setup the Query - query := newQuery(key, dht.network, func(ctx context.Context, p peer.ID) (*dhtQueryResult, error) { + query := dht.newQuery(key, func(ctx context.Context, p peer.ID) (*dhtQueryResult, error) { val, peers, err := dht.getValueOrPeers(ctx, p, key) if err != nil { @@ -170,7 +170,7 @@ func (dht *IpfsDHT) getClosestPeers(ctx context.Context, key u.Key) (<-chan peer peerset.Add(p) } - query := newQuery(key, dht.network, func(ctx context.Context, p peer.ID) (*dhtQueryResult, error) { + query := dht.newQuery(key, func(ctx context.Context, p peer.ID) (*dhtQueryResult, error) { closer, err := dht.closerPeersSingle(ctx, key, p) if err != nil { log.Errorf("error getting closer peers: %s", err) @@ -253,7 +253,7 @@ func (dht *IpfsDHT) findProvidersAsyncRoutine(ctx context.Context, key u.Key, co } // setup the Query - query := newQuery(key, dht.network, func(ctx context.Context, p peer.ID) (*dhtQueryResult, error) { + query := dht.newQuery(key, func(ctx context.Context, p peer.ID) (*dhtQueryResult, error) { pmes, err := dht.findProvidersSingle(ctx, p, key) if err != nil { @@ -312,7 +312,7 @@ func (dht *IpfsDHT) FindPeer(ctx context.Context, id peer.ID) (peer.PeerInfo, er } // setup the Query - query := newQuery(u.Key(id), dht.network, func(ctx context.Context, p peer.ID) (*dhtQueryResult, error) { + query := dht.newQuery(u.Key(id), func(ctx context.Context, p peer.ID) (*dhtQueryResult, error) { pmes, err := dht.findPeerSingle(ctx, p, id) if err != nil { @@ -361,7 +361,7 @@ func (dht *IpfsDHT) FindPeersConnectedToPeer(ctx context.Context, id peer.ID) (< } // setup the Query - query := newQuery(u.Key(id), dht.network, func(ctx context.Context, p peer.ID) (*dhtQueryResult, error) { + query := dht.newQuery(u.Key(id), func(ctx context.Context, p peer.ID) (*dhtQueryResult, error) { pmes, err := dht.findPeerSingle(ctx, p, id) if err != nil { diff --git a/routing/kbucket/bucket.go b/routing/kbucket/bucket.go index 2fa5586db..e158f70f9 100644 --- a/routing/kbucket/bucket.go +++ b/routing/kbucket/bucket.go @@ -4,7 +4,7 @@ import ( "container/list" "sync" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" ) // Bucket holds a list of peers. diff --git a/routing/kbucket/sorting.go b/routing/kbucket/sorting.go index a3a68767b..7995b39ed 100644 --- a/routing/kbucket/sorting.go +++ b/routing/kbucket/sorting.go @@ -2,7 +2,7 @@ package kbucket import ( "container/list" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" "sort" ) diff --git a/routing/kbucket/table.go b/routing/kbucket/table.go index 90ba65530..62bfa0646 100644 --- a/routing/kbucket/table.go +++ b/routing/kbucket/table.go @@ -7,7 +7,7 @@ import ( "sync" "time" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" u "github.com/jbenet/go-ipfs/util" ) diff --git a/routing/kbucket/table_test.go b/routing/kbucket/table_test.go index db93ddf86..3e44cf66a 100644 --- a/routing/kbucket/table_test.go +++ b/routing/kbucket/table_test.go @@ -7,7 +7,7 @@ import ( tu "github.com/jbenet/go-ipfs/util/testutil" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" ) // Test basic features of the bucket struct diff --git a/routing/kbucket/util.go b/routing/kbucket/util.go index 2d06b5f08..80c08de9e 100644 --- a/routing/kbucket/util.go +++ b/routing/kbucket/util.go @@ -5,7 +5,7 @@ import ( "crypto/sha256" "errors" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" ks "github.com/jbenet/go-ipfs/routing/keyspace" u "github.com/jbenet/go-ipfs/util" ) diff --git a/routing/mock/centralized_client.go b/routing/mock/centralized_client.go index 6b5a455a7..2aeafe026 100644 --- a/routing/mock/centralized_client.go +++ b/routing/mock/centralized_client.go @@ -6,7 +6,7 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" routing "github.com/jbenet/go-ipfs/routing" u "github.com/jbenet/go-ipfs/util" "github.com/jbenet/go-ipfs/util/testutil" diff --git a/routing/mock/centralized_server.go b/routing/mock/centralized_server.go index 030227b1b..dc462797a 100644 --- a/routing/mock/centralized_server.go +++ b/routing/mock/centralized_server.go @@ -7,7 +7,7 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" u "github.com/jbenet/go-ipfs/util" "github.com/jbenet/go-ipfs/util/testutil" ) diff --git a/routing/mock/centralized_test.go b/routing/mock/centralized_test.go index dcaf165b1..526d63c68 100644 --- a/routing/mock/centralized_test.go +++ b/routing/mock/centralized_test.go @@ -5,7 +5,7 @@ import ( "time" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" u "github.com/jbenet/go-ipfs/util" delay "github.com/jbenet/go-ipfs/util/delay" "github.com/jbenet/go-ipfs/util/testutil" diff --git a/routing/mock/dht.go b/routing/mock/dht.go index 1dfa415e0..2235970f5 100644 --- a/routing/mock/dht.go +++ b/routing/mock/dht.go @@ -4,7 +4,7 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" sync "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync" - mocknet "github.com/jbenet/go-ipfs/net/mock" + mocknet "github.com/jbenet/go-ipfs/p2p/net/mock" dht "github.com/jbenet/go-ipfs/routing/dht" "github.com/jbenet/go-ipfs/util/testutil" ) @@ -27,12 +27,12 @@ func (rs *mocknetserver) ClientWithDatastore(ctx context.Context, p testutil.Ide // FIXME AddPeer doesn't appear to be idempotent - net, err := rs.mn.AddPeer(p.PrivateKey(), p.Address()) + host, err := rs.mn.AddPeer(p.PrivateKey(), p.Address()) if err != nil { panic("FIXME") // return nil, debugerror.Wrap(err) } - return dht.NewDHT(ctx, p.ID(), net, sync.MutexWrap(ds)) + return dht.NewDHT(ctx, host, sync.MutexWrap(ds)) } var _ Server = &mocknetserver{} diff --git a/routing/mock/interface.go b/routing/mock/interface.go index 0bb54f365..d7dca8348 100644 --- a/routing/mock/interface.go +++ b/routing/mock/interface.go @@ -7,7 +7,7 @@ package mockrouting import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" routing "github.com/jbenet/go-ipfs/routing" u "github.com/jbenet/go-ipfs/util" delay "github.com/jbenet/go-ipfs/util/delay" diff --git a/routing/routing.go b/routing/routing.go index ae9acad44..1fbd79d25 100644 --- a/routing/routing.go +++ b/routing/routing.go @@ -6,7 +6,7 @@ import ( context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" u "github.com/jbenet/go-ipfs/util" ) diff --git a/util/peerset/peerset.go b/util/peerset/peerset.go index 35a80a12d..b45758ec3 100644 --- a/util/peerset/peerset.go +++ b/util/peerset/peerset.go @@ -1,7 +1,7 @@ package peerset import ( - peer "github.com/jbenet/go-ipfs/peer" + peer "github.com/jbenet/go-ipfs/p2p/peer" "sync" ) diff --git a/util/testutil/gen.go b/util/testutil/gen.go index bcf1f283e..93bca120d 100644 --- a/util/testutil/gen.go +++ b/util/testutil/gen.go @@ -9,13 +9,26 @@ import ( "sync" "testing" - ci "github.com/jbenet/go-ipfs/crypto" - peer "github.com/jbenet/go-ipfs/peer" + ci "github.com/jbenet/go-ipfs/p2p/crypto" + peer "github.com/jbenet/go-ipfs/p2p/peer" u "github.com/jbenet/go-ipfs/util" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" ) +// ZeroLocalTCPAddress is the "zero" tcp local multiaddr. This means: +// /ip4/127.0.0.1/tcp/0 +var ZeroLocalTCPAddress ma.Multiaddr + +func init() { + // initialize ZeroLocalTCPAddress + maddr, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/0") + if err != nil { + panic(err) + } + ZeroLocalTCPAddress = maddr +} + func RandKeyPair(bits int) (ci.PrivKey, ci.PubKey, error) { return ci.GenerateKeyPairWithReader(ci.RSA, bits, u.NewTimeSeededRand()) } @@ -48,6 +61,11 @@ func RandPeerIDFatal(t testing.TB) peer.ID { // RandLocalTCPAddress returns a random multiaddr. it suppresses errors // for nice composability-- do check the address isn't nil. +// +// Note: for real network tests, use ZeroLocalTCPAddress so the kernel +// assigns an unused TCP port. otherwise you may get clashes. This +// function remains here so that p2p/net/mock (which does not touch the +// real network) can assign different addresses to peers. func RandLocalTCPAddress() ma.Multiaddr { // chances are it will work out, but it **might** fail if the port is in use @@ -123,7 +141,7 @@ func RandPeerNetParamsOrFatal(t *testing.T) PeerNetParams { func RandPeerNetParams() (*PeerNetParams, error) { var p PeerNetParams var err error - p.Addr = RandLocalTCPAddress() + p.Addr = ZeroLocalTCPAddress p.PrivKey, p.PubKey, err = RandKeyPair(512) if err != nil { return nil, err diff --git a/util/testutil/identity.go b/util/testutil/identity.go index 34a536859..f1ab232cf 100644 --- a/util/testutil/identity.go +++ b/util/testutil/identity.go @@ -4,8 +4,8 @@ import ( "testing" ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" - ci "github.com/jbenet/go-ipfs/crypto" - peer "github.com/jbenet/go-ipfs/peer" + ci "github.com/jbenet/go-ipfs/p2p/crypto" + peer "github.com/jbenet/go-ipfs/p2p/peer" ) type Identity interface {