From 284bb1866fe494c0edb038793e4f7b34ffd18171 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sat, 16 Apr 2016 12:12:21 -0700 Subject: [PATCH 01/10] add a dist_get script for getting bins from dist.ipfs.io License: MIT Signed-off-by: Jeromy --- Makefile | 28 ++++++---- bin/dist_get | 143 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 10 deletions(-) create mode 100755 bin/dist_get diff --git a/Makefile b/Makefile index 014eb74dd..26d0d0a84 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,12 @@ else go_test=go test endif + +gx_bin=bin/gx-v0.6.0 +gx-go_bin=bin/gx-go-v1.1.0 + +# use things in our bin before any other system binaries +export PATH := bin:$(PATH) export IPFS_API ?= v04x.ipfs.io all: help @@ -21,21 +27,23 @@ toolkit_upgrade: gx_upgrade gxgo_upgrade go_check: @bin/check_go_version $(IPFS_MIN_GO_VERSION) -gx_upgrade: - go get -u github.com/whyrusleeping/gx +bin/gx-%: + @echo "installing gx $(@:bin/gx-%=%)" + @bin/dist_get gx $@ $(@:bin/gx-%=%) -gxgo_upgrade: - go get -u github.com/whyrusleeping/gx-go +bin/gx-go-%: + @echo "installing gx-go $(@:bin/gx-go-%=%)" + @bin/dist_get gx-go $@ $(@:bin/gx-go-%=%) + rm -f bin/gx-go + ln -s $(@:bin/%=%) bin/gx-go + +gx_check: ${gx_bin} ${gx-go_bin} path_check: @bin/check_go_path $(realpath $(shell pwd)) $(realpath $(GOPATH)/src/github.com/ipfs/go-ipfs) -gx_check: - @bin/check_gx_program "gx" $(IPFS_MIN_GX_VERSION) 'Upgrade or install gx using your package manager or run `make gx_upgrade`' - @bin/check_gx_program "gx-go" $(IPFS_MIN_GX_GO_VERSION) 'Upgrade or install gx-go using your package manager or run `make gxgo_upgrade`' - deps: go_check gx_check path_check - gx --verbose install --global + ${gx_bin} --verbose install --global # saves/vendors third-party dependencies to Godeps/_workspace # -r flag rewrites import paths to use the vendored path @@ -58,7 +66,7 @@ clean: uninstall: make -C cmd/ipfs uninstall -PHONY += all help godep toolkit_upgrade gx_upgrade gxgo_upgrade gx_check +PHONY += all help godep toolkit_upgrade gx_check PHONY += go_check deps vendor install build nofuse clean uninstall ############################################################## diff --git a/bin/dist_get b/bin/dist_get new file mode 100755 index 000000000..06f1be5a8 --- /dev/null +++ b/bin/dist_get @@ -0,0 +1,143 @@ +#!/bin/bash + +die() { + echo "$@" >&2 + exit 1 +} + +have_binary() { + type "$1" > /dev/null +} + +check_writeable() { + printf "" > "$1" && rm "$1" +} + +download() { + local url="$1" + local output="$2" + + if [ -z "$url" ] || [ -z "$output" ]; then + die "download takes exactly two arguments. was given '$@'" + fi + + if ! check_writeable "$output"; then + die "download error: cannot write to $output" + fi + + printf 'Downloading "%s" to "%s"\n' "$url" "$output" + if have_binary wget; then + wget "$url" -O "$output" + elif have_binary curl; then + curl --silent "$url" > "$output" + elif have_binary fetch; then + fetch "$url" -o "$output" + else + die "no binary found to download $url. exiting." + fi +} + +unarchive() { + local archivetype="$1" + local infile="$2" + local outfile="$3" + local distname="$4" + + if ! check_writeable "$outfile"; then + die "unarchive error: cannot write to $outfile" + fi + + case $archivetype in + tar.gz) + if have_binary tar; then + echo "==> using 'tar' to extract binary from archive" + tar -x "$distname/$distname" -f "$infile" -O > "$outfile" + else + die "no binary on system for extracting tar files" + fi + ;; + zip) + if have_binary unzip; then + echo "==> using 'unzip' to extract binary from archive" + unzip -p "$infile" "$distname/$distname" > "$outfile" + else + die "no installed method for extracting .zip archives" + fi + ;; + *) + die "unrecognized archive type '$archivetype'" + esac + + chmod +x "$outfile" +} + +get_go_vars() { + if [ ! -z "$GOOS" ] && [ ! -z "$GOARCH" ]; then + printf "%s-%s" "$GOOS" "$GOARCH" + fi + + if have_binary go; then + printf "%s-%s" "$(go env GOOS)" "$(go env GOARCH)" + else + die "no way of determining system GOOS and GOARCH\nPlease manually set GOOS and GOARCH then retry." + fi +} + +mkurl() { + local name="$1" + local vers="$2" + local archive="$3" + + local govars=$(get_go_vars) + + echo "http://dist.ipfs.io/$name/$vers/${name}_${vers}_$govars.$archive" +} + +distname="$1" +outpath="$2" +version="$3" + +if [ -z "$distname" ] || [ -z "$outpath" ] || [ -z "$version" ]; then + die "usage: dist_get " +fi + +if [ ${version:0:1} != "v" ]; then + die "versions must begin with 'v', for example: v0.4.0" +fi + +# TODO: don't depend on the go tool being installed to detect this +goenv=$(get_go_vars) + +case $goenv in + linux-*) + archive="tar.gz" + ;; + darwin-*) + archive="tar.gz" + ;; + windows-*) + archive="zip" + ;; + freebsd-*) + archive="tar.gz" + ;; + *) + echo "unrecognized system environment: $goenv" >&2 + die "currently only linux, darwin, windows and freebsd are supported by this script" +esac + + +mkdir -p bin/tmp + +url=$(mkurl "$distname" "$version" "$archive") +tmpfi="bin/tmp/$distname.$archive" + +download "$url" "$tmpfi" +if [ $? -ne 0 ]; then + die "failed to download $url to $tmpfi" +fi + +unarchive "$archive" "$tmpfi" "$outpath" "$distname" +if [ $? -ne 0 ]; then + die "failed to exract archive $tmpfi" +fi From cd11a487cc1e5b2369c72ee41f6edae586a12956 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 19 Apr 2016 08:35:46 -0700 Subject: [PATCH 02/10] debugging travisCI License: MIT Signed-off-by: Jeromy --- bin/dist_get | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/dist_get b/bin/dist_get index 06f1be5a8..c9656603c 100755 --- a/bin/dist_get +++ b/bin/dist_get @@ -102,6 +102,7 @@ if [ -z "$distname" ] || [ -z "$outpath" ] || [ -z "$version" ]; then fi if [ ${version:0:1} != "v" ]; then + echo "invalid version '$version'" >&2 die "versions must begin with 'v', for example: v0.4.0" fi From 4b6828ebfe5e4f19ca07be3aa819aa08b8131eb1 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Tue, 19 Apr 2016 08:53:19 -0700 Subject: [PATCH 03/10] stricter makefile targets License: MIT Signed-off-by: Jeromy --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 26d0d0a84..8be346aa5 100644 --- a/Makefile +++ b/Makefile @@ -27,11 +27,11 @@ toolkit_upgrade: gx_upgrade gxgo_upgrade go_check: @bin/check_go_version $(IPFS_MIN_GO_VERSION) -bin/gx-%: +bin/gx-v%: @echo "installing gx $(@:bin/gx-%=%)" @bin/dist_get gx $@ $(@:bin/gx-%=%) -bin/gx-go-%: +bin/gx-go-v%: @echo "installing gx-go $(@:bin/gx-go-%=%)" @bin/dist_get gx-go $@ $(@:bin/gx-go-%=%) rm -f bin/gx-go From 4a0882701a4143f1e58dad0fff275ccb7236e578 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 27 Apr 2016 12:22:41 -0700 Subject: [PATCH 04/10] update dist_get script to be more cross platform License: MIT Signed-off-by: Jeromy --- bin/dist_get | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bin/dist_get b/bin/dist_get index c9656603c..d6bdd1249 100755 --- a/bin/dist_get +++ b/bin/dist_get @@ -6,7 +6,7 @@ die() { } have_binary() { - type "$1" > /dev/null + type "$1" > /dev/null 2> /dev/null } check_writeable() { @@ -25,12 +25,14 @@ download() { die "download error: cannot write to $output" fi - printf 'Downloading "%s" to "%s"\n' "$url" "$output" if have_binary wget; then + printf 'Using wget to download "%s" to "%s"\n' "$url" "$output" wget "$url" -O "$output" elif have_binary curl; then + printf 'Using curl to download "%s" to "%s"\n' "$url" "$output" curl --silent "$url" > "$output" elif have_binary fetch; then + printf 'Using fetch to download "%s" to "%s"\n' "$url" "$output" fetch "$url" -o "$output" else die "no binary found to download $url. exiting." @@ -51,7 +53,7 @@ unarchive() { tar.gz) if have_binary tar; then echo "==> using 'tar' to extract binary from archive" - tar -x "$distname/$distname" -f "$infile" -O > "$outfile" + cat "$infile" | tar -O -z -x "$distname/$distname" > "$outfile" else die "no binary on system for extracting tar files" fi From 06a8a5bdb6a9e5a8aec9089c1eaf625b20cc0831 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 27 Apr 2016 12:28:20 -0700 Subject: [PATCH 05/10] use gx-go 1.2.0 License: MIT Signed-off-by: Jeromy --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8be346aa5..b94bd3f41 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ endif gx_bin=bin/gx-v0.6.0 -gx-go_bin=bin/gx-go-v1.1.0 +gx-go_bin=bin/gx-go-v1.2.0 # use things in our bin before any other system binaries export PATH := bin:$(PATH) From 3c7fad8af19a3a1d76418429b13db5c707ebf194 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 27 Apr 2016 15:32:39 -0700 Subject: [PATCH 06/10] address some CR feedback License: MIT Signed-off-by: Jeromy --- bin/dist_get | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/bin/dist_get b/bin/dist_get index d6bdd1249..895862ef9 100755 --- a/bin/dist_get +++ b/bin/dist_get @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh die() { echo "$@" >&2 @@ -14,46 +14,48 @@ check_writeable() { } download() { - local url="$1" - local output="$2" + dl_url="$1" + dl_output="$2" - if [ -z "$url" ] || [ -z "$output" ]; then - die "download takes exactly two arguments. was given '$@'" - fi + test "$#" -eq "2" || die "download requires exactly two arguments, was given $@" if ! check_writeable "$output"; then die "download error: cannot write to $output" fi if have_binary wget; then - printf 'Using wget to download "%s" to "%s"\n' "$url" "$output" + printf '==> Using wget to download "%s" to "%s"\n' "$url" "$output" wget "$url" -O "$output" elif have_binary curl; then - printf 'Using curl to download "%s" to "%s"\n' "$url" "$output" + printf '==> Using curl to download "%s" to "%s"\n' "$url" "$output" curl --silent "$url" > "$output" elif have_binary fetch; then - printf 'Using fetch to download "%s" to "%s"\n' "$url" "$output" + printf '==> Using fetch to download "%s" to "%s"\n' "$url" "$output" fetch "$url" -o "$output" else die "no binary found to download $url. exiting." fi + if [ "$?" -ne 0 ]; then + return $? + fi + echo "==> download complete!" } unarchive() { - local archivetype="$1" - local infile="$2" - local outfile="$3" - local distname="$4" + ua_archivetype="$1" + ua_infile="$2" + ua_outfile="$3" + ua_distname="$4" - if ! check_writeable "$outfile"; then - die "unarchive error: cannot write to $outfile" + if ! check_writeable "$ua_outfile"; then + die "unarchive error: cannot write to $ua_outfile" fi - case $archivetype in + case "$ua_archivetype" in tar.gz) if have_binary tar; then echo "==> using 'tar' to extract binary from archive" - cat "$infile" | tar -O -z -x "$distname/$distname" > "$outfile" + cat "$ua_infile" | tar -O -z -x "$ua_distname/$ua_distname" > "$ua_outfile" else die "no binary on system for extracting tar files" fi @@ -61,16 +63,16 @@ unarchive() { zip) if have_binary unzip; then echo "==> using 'unzip' to extract binary from archive" - unzip -p "$infile" "$distname/$distname" > "$outfile" + unzip -p "$ua_infile" "$ua_distname/$ua_distname" > "$ua_outfile" else die "no installed method for extracting .zip archives" fi ;; *) - die "unrecognized archive type '$archivetype'" + die "unrecognized archive type '$ua_archivetype'" esac - chmod +x "$outfile" + chmod +x "$ua_outfile" } get_go_vars() { @@ -142,5 +144,5 @@ fi unarchive "$archive" "$tmpfi" "$outpath" "$distname" if [ $? -ne 0 ]; then - die "failed to exract archive $tmpfi" + die "failed to extract archive $tmpfi" fi From 5f624efe734c4d77231bcce63f8c9baff19d39f2 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 28 Apr 2016 10:54:56 -0700 Subject: [PATCH 07/10] fix up dist_get script License: MIT Signed-off-by: Jeromy --- bin/dist_get | 47 ++++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/bin/dist_get b/bin/dist_get index 895862ef9..6cdd7e078 100755 --- a/bin/dist_get +++ b/bin/dist_get @@ -19,24 +19,21 @@ download() { test "$#" -eq "2" || die "download requires exactly two arguments, was given $@" - if ! check_writeable "$output"; then - die "download error: cannot write to $output" + if ! check_writeable "$dl_output"; then + die "download error: cannot write to $dl_output" fi if have_binary wget; then - printf '==> Using wget to download "%s" to "%s"\n' "$url" "$output" - wget "$url" -O "$output" + printf '==> Using wget to download "%s" to "%s"\n' "$dl_url" "$dl_output" + wget "$dl_url" -O "$dl_output" || return elif have_binary curl; then - printf '==> Using curl to download "%s" to "%s"\n' "$url" "$output" - curl --silent "$url" > "$output" + printf '==> Using curl to download "%s" to "%s"\n' "$dl_url" "$dl_output" + curl --silent "$dl_url" > "$dl_output" || return elif have_binary fetch; then - printf '==> Using fetch to download "%s" to "%s"\n' "$url" "$output" - fetch "$url" -o "$output" + printf '==> Using fetch to download "%s" to "%s"\n' "$dl_url" "$dl_output" + fetch "$dl_url" -o "$dl_output" || return else - die "no binary found to download $url. exiting." - fi - if [ "$?" -ne 0 ]; then - return $? + die "no binary found to download $dl_url. exiting." fi echo "==> download complete!" } @@ -88,13 +85,12 @@ get_go_vars() { } mkurl() { - local name="$1" - local vers="$2" - local archive="$3" + m_name="$1" + m_vers="$2" + m_archive="$3" + m_govars=$(get_go_vars) || die "could not get go env vars" - local govars=$(get_go_vars) - - echo "http://dist.ipfs.io/$name/$vers/${name}_${vers}_$govars.$archive" + echo "http://dist.ipfs.io/$m_name/$m_vers/${m_name}_${m_vers}_$m_govars.$m_archive" } distname="$1" @@ -105,13 +101,18 @@ if [ -z "$distname" ] || [ -z "$outpath" ] || [ -z "$version" ]; then die "usage: dist_get " fi -if [ ${version:0:1} != "v" ]; then - echo "invalid version '$version'" >&2 - die "versions must begin with 'v', for example: v0.4.0" -fi +case $version in + v*) + # correct input + ;; + *) + echo "invalid version '$version'" >&2 + die "versions must begin with 'v', for example: v0.4.0" + ;; +esac # TODO: don't depend on the go tool being installed to detect this -goenv=$(get_go_vars) +goenv=$(get_go_vars) || die "could not get go env vars" case $goenv in linux-*) From f2aa10914f9a9134cd08b1c21518ffa8858bd275 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 9 May 2016 11:26:51 -0700 Subject: [PATCH 08/10] fix travis.yml License: MIT Signed-off-by: Jeromy --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9b511be90..3de3336bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,6 @@ env: - TEST_NO_FUSE=1 TEST_VERBOSE=1 TEST_SUITE=test_sharness_expensive install: - - make toolkit_upgrade - make install script: From 1e31839cc2ad09871ca447e737d4ca685b3be15a Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 9 May 2016 12:48:06 -0700 Subject: [PATCH 09/10] use repo level bins in test Makefile License: MIT Signed-off-by: Jeromy --- Makefile | 2 ++ test/Makefile | 2 ++ 2 files changed, 4 insertions(+) diff --git a/Makefile b/Makefile index b94bd3f41..7a7131955 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,8 @@ go_check: bin/gx-v%: @echo "installing gx $(@:bin/gx-%=%)" @bin/dist_get gx $@ $(@:bin/gx-%=%) + rm -f bin/gx + ln -s $(@:bin/%=%) bin/gx bin/gx-go-v%: @echo "installing gx-go $(@:bin/gx-go-%=%)" diff --git a/test/Makefile b/test/Makefile index 689e22afa..26b8b5b1b 100644 --- a/test/Makefile +++ b/test/Makefile @@ -9,6 +9,8 @@ GOSLEEP_SRC = ./dependencies/go-sleep GX_RELATIVE_PATH = ../../../../gx/ipfs +export PATH := ../bin:${PATH} + # User might want to override those on the command line GOFLAGS = From db5d730fb9d964e2dac3c108b35156c7ab9ef168 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 9 May 2016 13:19:14 -0700 Subject: [PATCH 10/10] bump gx version to 0.7.0 License: MIT Signed-off-by: Jeromy --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7a7131955..496002174 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ else endif -gx_bin=bin/gx-v0.6.0 +gx_bin=bin/gx-v0.7.0 gx-go_bin=bin/gx-go-v1.2.0 # use things in our bin before any other system binaries