diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index cb5bf158f..47945f05d 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -60,6 +60,10 @@ "ImportPath": "github.com/camlistore/lock", "Rev": "ae27720f340952636b826119b58130b9c1a847a0" }, + { + "ImportPath": "github.com/cenkalti/backoff", + "Rev": "9831e1e25c874e0a0601b6dc43641071414eec7a" + }, { "ImportPath": "github.com/coreos/go-semver/semver", "Rev": "6fe83ccda8fb9b7549c9ab4ba47f47858bc950aa" @@ -118,7 +122,7 @@ }, { "ImportPath": "github.com/jbenet/go-fuse-version", - "Rev": "ff72c39433f95ada15f116fa493a51eeec2bd52e" + "Rev": "c723f93ceeb1d1e21eb7fe6fd39aa21a9fe7db99" }, { "ImportPath": "github.com/jbenet/go-is-domain", diff --git a/Godeps/_workspace/src/github.com/cenkalti/backoff/.gitignore b/Godeps/_workspace/src/github.com/cenkalti/backoff/.gitignore new file mode 100644 index 000000000..00268614f --- /dev/null +++ b/Godeps/_workspace/src/github.com/cenkalti/backoff/.gitignore @@ -0,0 +1,22 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe diff --git a/Godeps/_workspace/src/github.com/cenkalti/backoff/.travis.yml b/Godeps/_workspace/src/github.com/cenkalti/backoff/.travis.yml new file mode 100644 index 000000000..ce9cb6233 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cenkalti/backoff/.travis.yml @@ -0,0 +1,2 @@ +language: go +go: 1.3.3 diff --git a/Godeps/_workspace/src/github.com/cenkalti/backoff/LICENSE b/Godeps/_workspace/src/github.com/cenkalti/backoff/LICENSE new file mode 100644 index 000000000..89b817996 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cenkalti/backoff/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2014 Cenk Altı + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Godeps/_workspace/src/github.com/cenkalti/backoff/README.md b/Godeps/_workspace/src/github.com/cenkalti/backoff/README.md new file mode 100644 index 000000000..8e2612e63 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cenkalti/backoff/README.md @@ -0,0 +1,69 @@ +# backoff + +[![GoDoc](https://godoc.org/github.com/cenkalti/backoff?status.png)](https://godoc.org/github.com/cenkalti/backoff) +[![Build Status](https://travis-ci.org/cenkalti/backoff.png)](https://travis-ci.org/cenkalti/backoff) + +This is a Go port of the exponential backoff algorithm from +[google-http-java-client](https://code.google.com/p/google-http-java-client/wiki/ExponentialBackoff). + +[Exponential backoff](http://en.wikipedia.org/wiki/Exponential_backoff) +is an algorithm that uses feedback to multiplicatively decrease the rate of some process, +in order to gradually find an acceptable rate. +The retries exponentially increase and stop increasing when a certain threshold is met. + + + + +## Install + +```bash +go get github.com/cenkalti/backoff +``` + +## Example + +Simple retry helper that uses exponential back-off algorithm: + +```go +operation := func() error { + // An operation that might fail +} + +err := backoff.Retry(operation, backoff.NewExponentialBackOff()) +if err != nil { + // handle error +} + +// operation is successfull +``` + +Ticker example: + +```go +operation := func() error { + // An operation that may fail +} + +b := backoff.NewExponentialBackOff() +ticker := backoff.NewTicker(b) + +var err error + +// Ticks will continue to arrive when the previous operation is still running, +// so operations that take a while to fail could run in quick succession. +for t = range ticker.C { + if err = operation(); err != nil { + log.Println(err, "will retry...") + continue + } + + ticker.Stop() + break +} + +if err != nil { + // Operation has failed. +} + +// Operation is successfull. +``` diff --git a/Godeps/_workspace/src/github.com/cenkalti/backoff/backoff.go b/Godeps/_workspace/src/github.com/cenkalti/backoff/backoff.go new file mode 100644 index 000000000..25870d2fc --- /dev/null +++ b/Godeps/_workspace/src/github.com/cenkalti/backoff/backoff.go @@ -0,0 +1,56 @@ +// Package backoff implements backoff algorithms for retrying operations. +// +// Also has a Retry() helper for retrying operations that may fail. +package backoff + +import "time" + +// Back-off policy when retrying an operation. +type BackOff interface { + // Gets the duration to wait before retrying the operation or + // backoff.Stop to indicate that no retries should be made. + // + // Example usage: + // + // duration := backoff.NextBackOff(); + // if (duration == backoff.Stop) { + // // do not retry operation + // } else { + // // sleep for duration and retry operation + // } + // + NextBackOff() time.Duration + + // Reset to initial state. + Reset() +} + +// Indicates that no more retries should be made for use in NextBackOff(). +const Stop time.Duration = -1 + +// ZeroBackOff is a fixed back-off policy whose back-off time is always zero, +// meaning that the operation is retried immediately without waiting. +type ZeroBackOff struct{} + +func (b *ZeroBackOff) Reset() {} + +func (b *ZeroBackOff) NextBackOff() time.Duration { return 0 } + +// StopBackOff is a fixed back-off policy that always returns backoff.Stop for +// NextBackOff(), meaning that the operation should not be retried. +type StopBackOff struct{} + +func (b *StopBackOff) Reset() {} + +func (b *StopBackOff) NextBackOff() time.Duration { return Stop } + +type ConstantBackOff struct { + Interval time.Duration +} + +func (b *ConstantBackOff) Reset() {} +func (b *ConstantBackOff) NextBackOff() time.Duration { return b.Interval } + +func NewConstantBackOff(d time.Duration) *ConstantBackOff { + return &ConstantBackOff{Interval: d} +} diff --git a/Godeps/_workspace/src/github.com/cenkalti/backoff/backoff_test.go b/Godeps/_workspace/src/github.com/cenkalti/backoff/backoff_test.go new file mode 100644 index 000000000..24c49947b --- /dev/null +++ b/Godeps/_workspace/src/github.com/cenkalti/backoff/backoff_test.go @@ -0,0 +1,28 @@ +package backoff + +import ( + "time" + + "testing" +) + +func TestNextBackOffMillis(t *testing.T) { + subtestNextBackOff(t, 0, new(ZeroBackOff)) + subtestNextBackOff(t, Stop, new(StopBackOff)) +} + +func subtestNextBackOff(t *testing.T, expectedValue time.Duration, backOffPolicy BackOff) { + for i := 0; i < 10; i++ { + next := backOffPolicy.NextBackOff() + if next != expectedValue { + t.Errorf("got: %d expected: %d", next, expectedValue) + } + } +} + +func TestConstantBackOff(t *testing.T) { + backoff := NewConstantBackOff(time.Second) + if backoff.NextBackOff() != time.Second { + t.Error("invalid interval") + } +} diff --git a/Godeps/_workspace/src/github.com/cenkalti/backoff/exponential.go b/Godeps/_workspace/src/github.com/cenkalti/backoff/exponential.go new file mode 100644 index 000000000..e81e9c67a --- /dev/null +++ b/Godeps/_workspace/src/github.com/cenkalti/backoff/exponential.go @@ -0,0 +1,141 @@ +package backoff + +import ( + "math/rand" + "time" +) + +/* +ExponentialBackOff is an implementation of BackOff that increases the back off +period for each retry attempt using a randomization function that grows exponentially. + +NextBackOff() is calculated using the following formula: + + randomized_interval = + retry_interval * (random value in range [1 - randomization_factor, 1 + randomization_factor]) + +In other words NextBackOff() will range between the randomization factor +percentage below and above the retry interval. For example, using 2 seconds as the base retry +interval and 0.5 as the randomization factor, the actual back off period used in the next retry +attempt will be between 1 and 3 seconds. + +Note: max_interval caps the retry_interval and not the randomized_interval. + +If the time elapsed since an ExponentialBackOff instance is created goes past the +max_elapsed_time then the method NextBackOff() starts returning backoff.Stop. +The elapsed time can be reset by calling Reset(). + +Example: The default retry_interval is .5 seconds, default randomization_factor is 0.5, default +multiplier is 1.5 and the default max_interval is 1 minute. For 10 tries the sequence will be +(values in seconds) and assuming we go over the max_elapsed_time on the 10th try: + + request# retry_interval randomized_interval + + 1 0.5 [0.25, 0.75] + 2 0.75 [0.375, 1.125] + 3 1.125 [0.562, 1.687] + 4 1.687 [0.8435, 2.53] + 5 2.53 [1.265, 3.795] + 6 3.795 [1.897, 5.692] + 7 5.692 [2.846, 8.538] + 8 8.538 [4.269, 12.807] + 9 12.807 [6.403, 19.210] + 10 19.210 backoff.Stop + +Implementation is not thread-safe. +*/ +type ExponentialBackOff struct { + InitialInterval time.Duration + RandomizationFactor float64 + Multiplier float64 + MaxInterval time.Duration + // After MaxElapsedTime the ExponentialBackOff stops. + // It never stops if MaxElapsedTime == 0. + MaxElapsedTime time.Duration + Clock Clock + + currentInterval time.Duration + startTime time.Time +} + +// Clock is an interface that returns current time for BackOff. +type Clock interface { + Now() time.Time +} + +// Default values for ExponentialBackOff. +const ( + DefaultInitialInterval = 500 * time.Millisecond + DefaultRandomizationFactor = 0.5 + DefaultMultiplier = 1.5 + DefaultMaxInterval = 60 * time.Second + DefaultMaxElapsedTime = 15 * time.Minute +) + +// NewExponentialBackOff creates an instance of ExponentialBackOff using default values. +func NewExponentialBackOff() *ExponentialBackOff { + return &ExponentialBackOff{ + InitialInterval: DefaultInitialInterval, + RandomizationFactor: DefaultRandomizationFactor, + Multiplier: DefaultMultiplier, + MaxInterval: DefaultMaxInterval, + MaxElapsedTime: DefaultMaxElapsedTime, + Clock: SystemClock, + } +} + +type systemClock struct{} + +func (t systemClock) Now() time.Time { + return time.Now() +} + +// SystemClock implements Clock interface that uses time.Now(). +var SystemClock = systemClock{} + +// Reset the interval back to the initial retry interval and restarts the timer. +func (b *ExponentialBackOff) Reset() { + b.currentInterval = b.InitialInterval + b.startTime = b.Clock.Now() +} + +// NextBackOff calculates the next back off interval using the formula: +// randomized_interval = retry_interval +/- (randomization_factor * retry_interval) +func (b *ExponentialBackOff) NextBackOff() time.Duration { + // Make sure we have not gone over the maximum elapsed time. + if b.MaxElapsedTime != 0 && b.GetElapsedTime() > b.MaxElapsedTime { + return Stop + } + defer b.incrementCurrentInterval() + return getRandomValueFromInterval(b.RandomizationFactor, rand.Float64(), b.currentInterval) +} + +// GetElapsedTime returns the elapsed time since an ExponentialBackOff instance +// is created and is reset when Reset() is called. +// +// The elapsed time is computed using time.Now().UnixNano(). +func (b *ExponentialBackOff) GetElapsedTime() time.Duration { + return b.Clock.Now().Sub(b.startTime) +} + +// Increments the current interval by multiplying it with the multiplier. +func (b *ExponentialBackOff) incrementCurrentInterval() { + // Check for overflow, if overflow is detected set the current interval to the max interval. + if float64(b.currentInterval) >= float64(b.MaxInterval)/b.Multiplier { + b.currentInterval = b.MaxInterval + } else { + b.currentInterval = time.Duration(float64(b.currentInterval) * b.Multiplier) + } +} + +// Returns a random value from the interval: +// [randomizationFactor * currentInterval, randomizationFactor * currentInterval]. +func getRandomValueFromInterval(randomizationFactor, random float64, currentInterval time.Duration) time.Duration { + var delta = randomizationFactor * float64(currentInterval) + var minInterval = float64(currentInterval) - delta + var maxInterval = float64(currentInterval) + delta + // Get a random value from the range [minInterval, maxInterval]. + // The formula used below has a +1 because if the minInterval is 1 and the maxInterval is 3 then + // we want a 33% chance for selecting either 1, 2 or 3. + return time.Duration(minInterval + (random * (maxInterval - minInterval + 1))) +} diff --git a/Godeps/_workspace/src/github.com/cenkalti/backoff/exponential_test.go b/Godeps/_workspace/src/github.com/cenkalti/backoff/exponential_test.go new file mode 100644 index 000000000..2af22b8bd --- /dev/null +++ b/Godeps/_workspace/src/github.com/cenkalti/backoff/exponential_test.go @@ -0,0 +1,111 @@ +package backoff + +import ( + "math" + "testing" + "time" +) + +func TestBackOff(t *testing.T) { + var ( + testInitialInterval = 500 * time.Millisecond + testRandomizationFactor = 0.1 + testMultiplier = 2.0 + testMaxInterval = 5 * time.Second + testMaxElapsedTime = 15 * time.Minute + ) + + exp := NewExponentialBackOff() + exp.InitialInterval = testInitialInterval + exp.RandomizationFactor = testRandomizationFactor + exp.Multiplier = testMultiplier + exp.MaxInterval = testMaxInterval + exp.MaxElapsedTime = testMaxElapsedTime + exp.Reset() + + var expectedResults = []time.Duration{500, 1000, 2000, 4000, 5000, 5000, 5000, 5000, 5000, 5000} + for i, d := range expectedResults { + expectedResults[i] = d * time.Millisecond + } + + for _, expected := range expectedResults { + assertEquals(t, expected, exp.currentInterval) + // Assert that the next back off falls in the expected range. + var minInterval = expected - time.Duration(testRandomizationFactor*float64(expected)) + var maxInterval = expected + time.Duration(testRandomizationFactor*float64(expected)) + var actualInterval = exp.NextBackOff() + if !(minInterval <= actualInterval && actualInterval <= maxInterval) { + t.Error("error") + } + } +} + +func TestGetRandomizedInterval(t *testing.T) { + // 33% chance of being 1. + assertEquals(t, 1, getRandomValueFromInterval(0.5, 0, 2)) + assertEquals(t, 1, getRandomValueFromInterval(0.5, 0.33, 2)) + // 33% chance of being 2. + assertEquals(t, 2, getRandomValueFromInterval(0.5, 0.34, 2)) + assertEquals(t, 2, getRandomValueFromInterval(0.5, 0.66, 2)) + // 33% chance of being 3. + assertEquals(t, 3, getRandomValueFromInterval(0.5, 0.67, 2)) + assertEquals(t, 3, getRandomValueFromInterval(0.5, 0.99, 2)) +} + +type TestClock struct { + i time.Duration + start time.Time +} + +func (c *TestClock) Now() time.Time { + t := c.start.Add(c.i) + c.i += time.Second + return t +} + +func TestGetElapsedTime(t *testing.T) { + var exp = NewExponentialBackOff() + exp.Clock = &TestClock{} + exp.Reset() + + var elapsedTime = exp.GetElapsedTime() + if elapsedTime != time.Second { + t.Errorf("elapsedTime=%d", elapsedTime) + } +} + +func TestMaxElapsedTime(t *testing.T) { + var exp = NewExponentialBackOff() + exp.Clock = &TestClock{start: time.Time{}.Add(10000 * time.Second)} + if exp.NextBackOff() != Stop { + t.Error("error2") + } + // Change the currentElapsedTime to be 0 ensuring that the elapsed time will be greater + // than the max elapsed time. + exp.startTime = time.Time{} + assertEquals(t, Stop, exp.NextBackOff()) +} + +func TestBackOffOverflow(t *testing.T) { + var ( + testInitialInterval time.Duration = math.MaxInt64 / 2 + testMaxInterval time.Duration = math.MaxInt64 + testMultiplier float64 = 2.1 + ) + + exp := NewExponentialBackOff() + exp.InitialInterval = testInitialInterval + exp.Multiplier = testMultiplier + exp.MaxInterval = testMaxInterval + exp.Reset() + + exp.NextBackOff() + // Assert that when an overflow is possible the current varerval time.Duration is set to the max varerval time.Duration . + assertEquals(t, testMaxInterval, exp.currentInterval) +} + +func assertEquals(t *testing.T, expected, value time.Duration) { + if expected != value { + t.Errorf("got: %d, expected: %d", value, expected) + } +} diff --git a/Godeps/_workspace/src/github.com/cenkalti/backoff/retry.go b/Godeps/_workspace/src/github.com/cenkalti/backoff/retry.go new file mode 100644 index 000000000..80c547767 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cenkalti/backoff/retry.go @@ -0,0 +1,47 @@ +package backoff + +import "time" + +// Retry the function f until it does not return error or BackOff stops. +// f is guaranteed to be run at least once. +// It is the caller's responsibility to reset b after Retry returns. +// +// Retry sleeps the goroutine for the duration returned by BackOff after a +// failed operation returns. +// +// Usage: +// operation := func() error { +// // An operation that may fail +// } +// +// err := backoff.Retry(operation, backoff.NewExponentialBackOff()) +// if err != nil { +// // Operation has failed. +// } +// +// // Operation is successfull. +// +func Retry(f func() error, b BackOff) error { return RetryNotify(f, b, nil) } + +// RetryNotify calls notify function with the error and wait duration for each failed attempt before sleep. +func RetryNotify(f func() error, b BackOff, notify func(err error, wait time.Duration)) error { + var err error + var next time.Duration + + b.Reset() + for { + if err = f(); err == nil { + return nil + } + + if next = b.NextBackOff(); next == Stop { + return err + } + + if notify != nil { + notify(err, next) + } + + time.Sleep(next) + } +} diff --git a/Godeps/_workspace/src/github.com/cenkalti/backoff/retry_test.go b/Godeps/_workspace/src/github.com/cenkalti/backoff/retry_test.go new file mode 100644 index 000000000..c0d25ab76 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cenkalti/backoff/retry_test.go @@ -0,0 +1,34 @@ +package backoff + +import ( + "errors" + "log" + "testing" +) + +func TestRetry(t *testing.T) { + const successOn = 3 + var i = 0 + + // This function is successfull on "successOn" calls. + f := func() error { + i++ + log.Printf("function is called %d. time\n", i) + + if i == successOn { + log.Println("OK") + return nil + } + + log.Println("error") + return errors.New("error") + } + + err := Retry(f, NewExponentialBackOff()) + if err != nil { + t.Errorf("unexpected error: %s", err.Error()) + } + if i != successOn { + t.Errorf("invalid number of retries: %d", i) + } +} diff --git a/Godeps/_workspace/src/github.com/cenkalti/backoff/ticker.go b/Godeps/_workspace/src/github.com/cenkalti/backoff/ticker.go new file mode 100644 index 000000000..17ace5660 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cenkalti/backoff/ticker.go @@ -0,0 +1,105 @@ +package backoff + +import ( + "runtime" + "sync" + "time" +) + +// Ticker holds a channel that delivers `ticks' of a clock at times reported by a BackOff. +// +// Ticks will continue to arrive when the previous operation is still running, +// so operations that take a while to fail could run in quick succession. +// +// Usage: +// operation := func() error { +// // An operation that may fail +// } +// +// b := backoff.NewExponentialBackOff() +// ticker := backoff.NewTicker(b) +// +// var err error +// for _ = range ticker.C { +// if err = operation(); err != nil { +// log.Println(err, "will retry...") +// continue +// } +// +// ticker.Stop() +// break +// } +// +// if err != nil { +// // Operation has failed. +// } +// +// // Operation is successfull. +// +type Ticker struct { + C <-chan time.Time + c chan time.Time + b BackOff + stop chan struct{} + stopOnce sync.Once +} + +// NewTicker returns a new Ticker containing a channel that will send the time at times +// specified by the BackOff argument. Ticker is guaranteed to tick at least once. +// The channel is closed when Stop method is called or BackOff stops. +func NewTicker(b BackOff) *Ticker { + c := make(chan time.Time) + t := &Ticker{ + C: c, + c: c, + b: b, + stop: make(chan struct{}), + } + go t.run() + runtime.SetFinalizer(t, (*Ticker).Stop) + return t +} + +// Stop turns off a ticker. After Stop, no more ticks will be sent. +func (t *Ticker) Stop() { + t.stopOnce.Do(func() { close(t.stop) }) +} + +func (t *Ticker) run() { + c := t.c + defer close(c) + t.b.Reset() + + // Ticker is guaranteed to tick at least once. + afterC := t.send(time.Now()) + + for { + if afterC == nil { + return + } + + select { + case tick := <-afterC: + afterC = t.send(tick) + case <-t.stop: + t.c = nil // Prevent future ticks from being sent to the channel. + return + } + } +} + +func (t *Ticker) send(tick time.Time) <-chan time.Time { + select { + case t.c <- tick: + case <-t.stop: + return nil + } + + next := t.b.NextBackOff() + if next == Stop { + t.Stop() + return nil + } + + return time.After(next) +} diff --git a/Godeps/_workspace/src/github.com/cenkalti/backoff/ticker_test.go b/Godeps/_workspace/src/github.com/cenkalti/backoff/ticker_test.go new file mode 100644 index 000000000..7c392df46 --- /dev/null +++ b/Godeps/_workspace/src/github.com/cenkalti/backoff/ticker_test.go @@ -0,0 +1,45 @@ +package backoff + +import ( + "errors" + "log" + "testing" +) + +func TestTicker(t *testing.T) { + const successOn = 3 + var i = 0 + + // This function is successfull on "successOn" calls. + f := func() error { + i++ + log.Printf("function is called %d. time\n", i) + + if i == successOn { + log.Println("OK") + return nil + } + + log.Println("error") + return errors.New("error") + } + + b := NewExponentialBackOff() + ticker := NewTicker(b) + + var err error + for _ = range ticker.C { + if err = f(); err != nil { + t.Log(err) + continue + } + + break + } + if err != nil { + t.Errorf("unexpected error: %s", err.Error()) + } + if i != successOn { + t.Errorf("invalid number of retries: %d", i) + } +} diff --git a/Godeps/_workspace/src/github.com/jbenet/go-fuse-version/version_bsd.go b/Godeps/_workspace/src/github.com/jbenet/go-fuse-version/version_bsd.go index a1d2460c8..850fb1e4b 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-fuse-version/version_bsd.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-fuse-version/version_bsd.go @@ -3,9 +3,10 @@ package fuseversion import ( + "fmt" "runtime" ) func getLocalFuseSystems() (*Systems, error) { - return nil, fmt.Sprintf(notImplYet, runtime.GOARCH()) + return nil, fmt.Errorf(notImplYet, runtime.GOARCH) } diff --git a/Godeps/_workspace/src/github.com/jbenet/go-fuse-version/version_linux.go b/Godeps/_workspace/src/github.com/jbenet/go-fuse-version/version_linux.go index e4944d1c0..6ad4db598 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-fuse-version/version_linux.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-fuse-version/version_linux.go @@ -1,9 +1,10 @@ package fuseversion import ( + "fmt" "runtime" ) func getLocalFuseSystems() (*Systems, error) { - return nil, fmt.Sprintf(notImplYet, runtime.GOARCH()) + return nil, fmt.Errorf(notImplYet, runtime.GOARCH) } diff --git a/Godeps/_workspace/src/github.com/jbenet/go-fuse-version/version_windows.go b/Godeps/_workspace/src/github.com/jbenet/go-fuse-version/version_windows.go index e4944d1c0..6ad4db598 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-fuse-version/version_windows.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-fuse-version/version_windows.go @@ -1,9 +1,10 @@ package fuseversion import ( + "fmt" "runtime" ) func getLocalFuseSystems() (*Systems, error) { - return nil, fmt.Sprintf(notImplYet, runtime.GOARCH()) + return nil, fmt.Errorf(notImplYet, runtime.GOARCH) } diff --git a/core/core.go b/core/core.go index 2477354f6..60dfaeae0 100644 --- a/core/core.go +++ b/core/core.go @@ -2,6 +2,7 @@ package core import ( "fmt" + "time" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" b58 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-base58" @@ -16,6 +17,7 @@ import ( bitswap "github.com/jbenet/go-ipfs/exchange/bitswap" bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network" offline "github.com/jbenet/go-ipfs/exchange/offline" + rp "github.com/jbenet/go-ipfs/exchange/reprovide" mount "github.com/jbenet/go-ipfs/fuse/mount" merkledag "github.com/jbenet/go-ipfs/merkledag" namesys "github.com/jbenet/go-ipfs/namesys" @@ -79,6 +81,7 @@ type IpfsNode struct { Exchange exchange.Interface // the block exchange + strategy (bitswap) Namesys namesys.NameSystem // the name system, resolves paths to hashes Diagnostics *diag.Diagnostics // the diagnostics service + Reprovider *rp.Reprovider // the value reprovider system ctxgroup.ContextGroup @@ -247,6 +250,10 @@ func (n *IpfsNode) StartOnlineServices() error { bootstrapPeers = append(bootstrapPeers, p) } go superviseConnections(ctx, n.PeerHost, n.DHT, n.Peerstore, bootstrapPeers) + + // Start up reprovider system + n.Reprovider = rp.NewReprovider(n.Routing, n.Blockstore) + go n.Reprovider.ProvideEvery(ctx, time.Hour*12) return nil } diff --git a/exchange/reprovide/reprovide.go b/exchange/reprovide/reprovide.go new file mode 100644 index 000000000..d0ac69470 --- /dev/null +++ b/exchange/reprovide/reprovide.go @@ -0,0 +1,71 @@ +package reprovide + +import ( + "time" + + context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" + backoff "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/cenkalti/backoff" + + blocks "github.com/jbenet/go-ipfs/blocks/blockstore" + routing "github.com/jbenet/go-ipfs/routing" + debugerror "github.com/jbenet/go-ipfs/util/debugerror" + eventlog "github.com/jbenet/go-ipfs/util/eventlog" +) + +var log = eventlog.Logger("reprovider") + +type Reprovider struct { + // The routing system to provide values through + rsys routing.IpfsRouting + + // The backing store for blocks to be provided + bstore blocks.Blockstore +} + +func NewReprovider(rsys routing.IpfsRouting, bstore blocks.Blockstore) *Reprovider { + return &Reprovider{ + rsys: rsys, + bstore: bstore, + } +} + +func (rp *Reprovider) ProvideEvery(ctx context.Context, tick time.Duration) { + after := time.After(0) + for { + select { + case <-ctx.Done(): + return + case <-after: + err := rp.Reprovide(ctx) + if err != nil { + log.Error(err) + } + after = time.After(tick) + } + } +} + +func (rp *Reprovider) Reprovide(ctx context.Context) error { + keychan, err := rp.bstore.AllKeysChan(ctx, 0, 1<<16) + if err != nil { + return debugerror.Errorf("Failed to get key chan from blockstore: %s", err) + } + for k := range keychan { + op := func() error { + err := rp.rsys.Provide(ctx, k) + if err != nil { + log.Warningf("Failed to provide key: %s", err) + } + return err + } + + // TODO: this backoff library does not respect our context, we should + // eventually work contexts into it. low priority. + err := backoff.Retry(op, backoff.NewExponentialBackOff()) + if err != nil { + log.Errorf("Providing failed after number of retries: %s", err) + return err + } + } + return nil +} diff --git a/exchange/reprovide/reprovide_test.go b/exchange/reprovide/reprovide_test.go new file mode 100644 index 000000000..18fc4d0b1 --- /dev/null +++ b/exchange/reprovide/reprovide_test.go @@ -0,0 +1,53 @@ +package reprovide_test + +import ( + "testing" + + context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" + dssync "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync" + + blocks "github.com/jbenet/go-ipfs/blocks" + blockstore "github.com/jbenet/go-ipfs/blocks/blockstore" + mock "github.com/jbenet/go-ipfs/routing/mock" + testutil "github.com/jbenet/go-ipfs/util/testutil" + + . "github.com/jbenet/go-ipfs/exchange/reprovide" +) + +func TestReprovide(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + mrserv := mock.NewServer() + + idA := testutil.RandIdentityOrFatal(t) + idB := testutil.RandIdentityOrFatal(t) + + clA := mrserv.Client(idA) + clB := mrserv.Client(idB) + + bstore := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) + + blk := blocks.NewBlock([]byte("this is a test")) + bstore.Put(blk) + + reprov := NewReprovider(clA, bstore) + err := reprov.Reprovide(ctx) + if err != nil { + t.Fatal(err) + } + + provs, err := clB.FindProviders(ctx, blk.Key()) + if err != nil { + t.Fatal(err) + } + + if len(provs) == 0 { + t.Fatal("Should have gotten a provider") + } + + if provs[0].ID != idA.ID() { + t.Fatal("Somehow got the wrong peer back as a provider.") + } +}