From 56a32a306c4048221f98c9baef1b0ac3e3f53d9f Mon Sep 17 00:00:00 2001 From: Henry Date: Wed, 4 Mar 2015 14:50:17 +0100 Subject: [PATCH 1/5] using pollEndpoint to block in tests for 'daemon ready' (updates #844) --- cmd/ipfs/daemon.go | 1 + cmd/pollEndpoint/main.go | 106 ++++++++++++++++++++++++++++++++++ test/Makefile | 6 +- test/sharness/lib/test-lib.sh | 5 +- 4 files changed, 114 insertions(+), 4 deletions(-) create mode 100644 cmd/pollEndpoint/main.go diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index abd6818d3..6c4e996be 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -244,6 +244,7 @@ func daemonFunc(req cmds.Request, res cmds.Response) { corehttp.CommandsOption(*req.Context()), corehttp.WebUIOption, gateway.ServeOption(), + corehttp.VersionOption(), } if rootRedirect != nil { opts = append(opts, rootRedirect) diff --git a/cmd/pollEndpoint/main.go b/cmd/pollEndpoint/main.go new file mode 100644 index 000000000..0261d3f8b --- /dev/null +++ b/cmd/pollEndpoint/main.go @@ -0,0 +1,106 @@ +// pollEndpoint is a helper utility that waits for a http endpoint to be reachable and return with http.StatusOK +package main + +import ( + "flag" + "net" + "net/http" + "net/url" + "os" + "syscall" + "time" + + log "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/Sirupsen/logrus" + ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr" + manet "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr-net" +) + +var ( + host = flag.String("host", "/ip4/127.0.0.1/tcp/5001", "the multiaddr host to dial on") + endpoint = flag.String("ep", "/version", "which http endpoint path to hit") + tries = flag.Int("tries", 10, "how many tries to make before failing") + timeout = flag.Duration("tout", time.Second, "how long to wait between attempts") + verbose = flag.Bool("v", false, "verbose logging") +) + +func main() { + flag.Parse() + + // extract address from host flag + addr, err := ma.NewMultiaddr(*host) + if err != nil { + log.WithField("err", err).Fatal("NewMultiaddr() failed") + } + p := addr.Protocols() + if len(p) < 2 { + log.WithField("addr", addr).Fatal("need to protocolls in host flag.") + } + _, host, err := manet.DialArgs(addr) + if err != nil { + log.WithField("err", err).Fatal("manet.DialArgs() failed") + } + + if *verbose { // lower log level + log.SetLevel(log.DebugLevel) + } + + // construct url to dial + var u url.URL + u.Scheme = "http" + u.Host = host + u.Path = *endpoint + + // show what we got + start := time.Now() + log.WithFields(log.Fields{ + "when": start, + "tries": *tries, + "timeout": *timeout, + "url": u.String(), + }).Debug("starting") + + for *tries > 0 { + + f := log.Fields{"tries": *tries} + + resp, err := http.Get(u.String()) + + if err == nil { + resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + f["took"] = time.Since(start) + log.WithFields(f).Println("status ok - endpoint reachable") + os.Exit(0) + } + + f["status"] = resp.Status + log.WithFields(f).Warn("response not okay") + + } else if urlErr, ok := err.(*url.Error); ok { // expected error from http.Get() + f["urlErr"] = urlErr + + if urlErr.Op != "Get" || urlErr.URL != *endpoint { + f["op"] = urlErr.Op + f["url"] = urlErr.URL + log.WithFields(f).Error("way to funky buisness..!") + } + + if opErr, ok := urlErr.Err.(*net.OpError); ok { + f["opErr"] = opErr + f["connRefused"] = opErr.Err == syscall.ECONNREFUSED + f["temporary"] = opErr.Temporary() + log.WithFields(f).Println("net.OpError") + } + } else { // unexpected error from http.Get() + f["err"] = err + log.WithFields(f).Error("unknown error") + } + + time.Sleep(*timeout) + *tries-- + } + + log.Println("failed.") + os.Exit(1) +} diff --git a/test/Makefile b/test/Makefile index 52f18d210..3b16ff82e 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,9 +1,10 @@ -BINS = bin/random bin/multihash bin/ipfs +BINS = bin/random bin/multihash bin/ipfs bin/pollEndpoint IPFS_ROOT = ../ IPFS_CMD = ../cmd/ipfs RANDOM_SRC = ../Godeps/_workspace/src/github.com/jbenet/go-random MULTIHASH_SRC = ../Godeps/_workspace/src/github.com/jbenet/go-multihash +POLLENDPOINT_SRC= ../cmd/pollEndpoint all: deps @@ -23,6 +24,9 @@ bin/multihash: $(MULTIHASH_SRC)/**/*.go bin/ipfs: $(IPFS_ROOT)/**/*.go go build -o bin/ipfs $(IPFS_CMD) +bin/pollEndpoint: $(POLLENDPOINT_SRC)/*.go + go build -o bin/pollEndpoint $(POLLENDPOINT_SRC) + test: test_expensive test_expensive: diff --git a/test/sharness/lib/test-lib.sh b/test/sharness/lib/test-lib.sh index 25724e25b..65fba7a53 100644 --- a/test/sharness/lib/test-lib.sh +++ b/test/sharness/lib/test-lib.sh @@ -175,14 +175,13 @@ test_launch_ipfs_daemon() { ADDR_API="/ip4/127.0.0.1/tcp/5001" test_expect_success "'ipfs daemon' is ready" ' IPFS_PID=$! && - test_wait_output_n_lines_60_sec actual_daemon 2 && - test_run_repeat_60_sec "grep \"API server listening on $ADDR_API\" actual_daemon" || + pollEndpoint -ep=/version -host=$ADDR_API -v -tout=1s -tries=60 2>poll_err > poll_apiout || test_fsh cat actual_daemon || test_fsh cat daemon_err ' if test "$ADDR_GWAY" != ""; then test_expect_success "'ipfs daemon' output includes Gateway address" ' - test_run_repeat_60_sec "grep \"Gateway server listening on $ADDR_GWAY\" actual_daemon" || + pollEndpoint -ep=/version -host=$ADDR_GWAY -v -tout=1s -tries=60 2>poll_err > poll_gwout || test_fsh cat daemon_err ' fi From 0204a0c17492e5a1d22492b772ddecb089a36ba5 Mon Sep 17 00:00:00 2001 From: Henry Date: Wed, 4 Mar 2015 14:53:30 +0100 Subject: [PATCH 2/5] cat log output on fail --- test/sharness/lib/test-lib.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/sharness/lib/test-lib.sh b/test/sharness/lib/test-lib.sh index 65fba7a53..d2896c60c 100644 --- a/test/sharness/lib/test-lib.sh +++ b/test/sharness/lib/test-lib.sh @@ -175,14 +175,14 @@ test_launch_ipfs_daemon() { ADDR_API="/ip4/127.0.0.1/tcp/5001" test_expect_success "'ipfs daemon' is ready" ' IPFS_PID=$! && - pollEndpoint -ep=/version -host=$ADDR_API -v -tout=1s -tries=60 2>poll_err > poll_apiout || - test_fsh cat actual_daemon || test_fsh cat daemon_err + pollEndpoint -ep=/version -host=$ADDR_API -v -tout=1s -tries=60 2>poll_apierr > poll_apiout || + test_fsh cat actual_daemon || test_fsh cat daemon_err || test_fsh cat poll_apierr || test_fsh cat poll_apiout ' if test "$ADDR_GWAY" != ""; then test_expect_success "'ipfs daemon' output includes Gateway address" ' - pollEndpoint -ep=/version -host=$ADDR_GWAY -v -tout=1s -tries=60 2>poll_err > poll_gwout || - test_fsh cat daemon_err + pollEndpoint -ep=/version -host=$ADDR_GWAY -v -tout=1s -tries=60 2>poll_gwerr > poll_gwout || + test_fsh cat daemon_err || test_fsh cat poll_gwerr || test_fsh cat poll_gwout ' fi } From aa745fef07056bc25cf4759bfff9ab1a6ef11a5d Mon Sep 17 00:00:00 2001 From: Henry Date: Wed, 4 Mar 2015 15:06:26 +0100 Subject: [PATCH 3/5] specify new dep in sharenss Makefile --- test/Makefile | 2 +- test/sharness/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Makefile b/test/Makefile index 3b16ff82e..91304ae87 100644 --- a/test/Makefile +++ b/test/Makefile @@ -24,7 +24,7 @@ bin/multihash: $(MULTIHASH_SRC)/**/*.go bin/ipfs: $(IPFS_ROOT)/**/*.go go build -o bin/ipfs $(IPFS_CMD) -bin/pollEndpoint: $(POLLENDPOINT_SRC)/*.go +bin/pollEndpoint: $(POLLENDPOINT_SRC)/*.go go build -o bin/pollEndpoint $(POLLENDPOINT_SRC) test: test_expensive diff --git a/test/sharness/Makefile b/test/sharness/Makefile index 392826632..144262f98 100644 --- a/test/sharness/Makefile +++ b/test/sharness/Makefile @@ -7,7 +7,7 @@ # NOTE: run with TEST_VERBOSE=1 for verbose sharness tests. T = $(sort $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)) -BINS = bin/random bin/multihash bin/ipfs +BINS = bin/random bin/multihash bin/ipfs bin/pollEndpoint SHARNESS = lib/sharness/sharness.sh IPFS_ROOT = ../.. From 3f932af14dcebb0b902816e07a7fc65904286cfe Mon Sep 17 00:00:00 2001 From: Henry Date: Wed, 4 Mar 2015 16:18:41 +0100 Subject: [PATCH 4/5] removed crazy logging --- cmd/pollEndpoint/main.go | 59 ++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 36 deletions(-) diff --git a/cmd/pollEndpoint/main.go b/cmd/pollEndpoint/main.go index 0261d3f8b..de9e56523 100644 --- a/cmd/pollEndpoint/main.go +++ b/cmd/pollEndpoint/main.go @@ -3,11 +3,10 @@ package main import ( "flag" - "net" + "fmt" "net/http" "net/url" "os" - "syscall" "time" log "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/Sirupsen/logrus" @@ -33,7 +32,7 @@ func main() { } p := addr.Protocols() if len(p) < 2 { - log.WithField("addr", addr).Fatal("need to protocolls in host flag.") + log.WithField("addr", addr).Fatal("need two protocols in host flag (/ip/tcp)") } _, host, err := manet.DialArgs(addr) if err != nil { @@ -60,43 +59,16 @@ func main() { }).Debug("starting") for *tries > 0 { - f := log.Fields{"tries": *tries} - resp, err := http.Get(u.String()) - + err := checkOK(http.Get(u.String())) if err == nil { - resp.Body.Close() - - if resp.StatusCode == http.StatusOK { - f["took"] = time.Since(start) - log.WithFields(f).Println("status ok - endpoint reachable") - os.Exit(0) - } - - f["status"] = resp.Status - log.WithFields(f).Warn("response not okay") - - } else if urlErr, ok := err.(*url.Error); ok { // expected error from http.Get() - f["urlErr"] = urlErr - - if urlErr.Op != "Get" || urlErr.URL != *endpoint { - f["op"] = urlErr.Op - f["url"] = urlErr.URL - log.WithFields(f).Error("way to funky buisness..!") - } - - if opErr, ok := urlErr.Err.(*net.OpError); ok { - f["opErr"] = opErr - f["connRefused"] = opErr.Err == syscall.ECONNREFUSED - f["temporary"] = opErr.Temporary() - log.WithFields(f).Println("net.OpError") - } - } else { // unexpected error from http.Get() - f["err"] = err - log.WithFields(f).Error("unknown error") + f["took"] = time.Since(start) + log.WithFields(f).Println("status ok - endpoint reachable") + os.Exit(0) } - + f["error"] = err + log.WithFields(f).Debug("get failed") time.Sleep(*timeout) *tries-- } @@ -104,3 +76,18 @@ func main() { log.Println("failed.") os.Exit(1) } + +func checkOK(resp *http.Response, err error) error { + if err == nil { // request worked + resp.Body.Close() + if resp.StatusCode == http.StatusOK { + return nil + } + return fmt.Errorf("Response not OK. %d %s", resp.StatusCode, resp.Status) + } else if urlErr, ok := err.(*url.Error); ok { // expected error from http.Get() + if urlErr.Op != "Get" || urlErr.URL != *endpoint { + return fmt.Errorf("wrong url or endpoint error from http.Get() %#v", urlErr) + } + } + return err +} From 14090d4a11ef4f194dd8179ebc97c764a1724165 Mon Sep 17 00:00:00 2001 From: Henry Date: Wed, 4 Mar 2015 16:25:41 +0100 Subject: [PATCH 5/5] moved pollEndpoint to thirdparty/ --- test/Makefile | 2 +- {cmd => thirdparty}/pollEndpoint/main.go | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename {cmd => thirdparty}/pollEndpoint/main.go (100%) diff --git a/test/Makefile b/test/Makefile index 91304ae87..868eac731 100644 --- a/test/Makefile +++ b/test/Makefile @@ -4,7 +4,7 @@ IPFS_ROOT = ../ IPFS_CMD = ../cmd/ipfs RANDOM_SRC = ../Godeps/_workspace/src/github.com/jbenet/go-random MULTIHASH_SRC = ../Godeps/_workspace/src/github.com/jbenet/go-multihash -POLLENDPOINT_SRC= ../cmd/pollEndpoint +POLLENDPOINT_SRC= ../thirdparty/pollEndpoint all: deps diff --git a/cmd/pollEndpoint/main.go b/thirdparty/pollEndpoint/main.go similarity index 100% rename from cmd/pollEndpoint/main.go rename to thirdparty/pollEndpoint/main.go