From 81c8cffee9c01d2c8dc1efac76441ece9c3bdca2 Mon Sep 17 00:00:00 2001 From: Lars Gierth Date: Thu, 4 Feb 2016 04:19:14 +0100 Subject: [PATCH] Rework the Dockerfile - Have two Dockerfiles doing essentially the same, but optimized for build time (for tests) and image size (for Docker Hub) - Fetch gx dependencies - Expose port 4002 for utp - Specify go version, currently 1.5.3-r0 - Create ephemeral fs-repo if none is mounted - Have t0300-docker-image actually test IPFS, not just an echo - Make everything a bit less hardcoded - Remove dead shacheck License: MIT Signed-off-by: Lars Gierth --- .dockerignore | 4 +- Dockerfile | 89 +++++++++++++++++++++-------- bin/container_daemon | 17 +++--- bin/container_shacheck | 10 ---- test/3nodetest/Makefile | 2 +- test/Dockerfile | 54 +++++++++++++++++ test/ipfs-test-lib.sh | 8 +-- test/sharness/t0300-docker-image.sh | 18 ++++-- 8 files changed, 148 insertions(+), 54 deletions(-) delete mode 100644 bin/container_shacheck create mode 100644 test/Dockerfile diff --git a/.dockerignore b/.dockerignore index 6b8710a71..4b2bacc27 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,3 @@ -.git +cmd/ipfs/ipfs +vendor/gx/ +test/ diff --git a/Dockerfile b/Dockerfile index a4773cd20..b307ac0cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,34 +1,73 @@ FROM alpine:3.3 -MAINTAINER Brian Tiger Chow +MAINTAINER Lars Gierth +# There is a copy of this Dockerfile in test/sharness, +# which is optimized for build time, instead of image size. +# +# Please keep these two Dockerfiles in sync. + + +# Ports for Swarm TCP, Swarm uTP, API, Gateway +EXPOSE 4001 +EXPOSE 4002/udp +EXPOSE 5001 +EXPOSE 8080 + +# Volume for mounting an IPFS fs-repo +# This is moved to the bottom for technical reasons. +#VOLUME $IPFS_PATH + +# IPFS API to use for fetching gx packages. +# This can be a gateway too, since its read-only API provides all gx needs. +# - e.g. /ip4/172.17.0.1/tcp/8080 if the Docker host +# has the IPFS gateway listening on the bridge interface +# provided by Docker's default networking. +# - if empty, the public gateway at ipfs.io is used. +ENV GX_IPFS "" +# The IPFS fs-repo within the container ENV IPFS_PATH /data/ipfs -ENV GOPATH /go:/go/src/github.com/ipfs/go-ipfs/Godeps/_workspace +# Golang stuff +ENV GO_VERSION 1.5.3-r0 +ENV GOPATH /go +ENV PATH /go/bin:$PATH +ENV SRC_PATH /go/src/github.com/ipfs/go-ipfs -EXPOSE 4001 5001 8080 -# 4001 = Swarm, 5001 = API, 8080 = HTTP transport +# Get the go-ipfs sourcecode +COPY . $SRC_PATH -ADD bin/container_daemon /usr/local/bin/start_ipfs -ADD bin/container_shacheck /usr/local/bin/shacheck - -ADD . /go/src/github.com/ipfs/go-ipfs -WORKDIR /go/src/github.com/ipfs/go-ipfs/cmd/ipfs - -RUN adduser -D -h /data -u 1000 ipfs \ - && mkdir -p /data/ipfs && chown ipfs:ipfs /data/ipfs \ - && apk add --update bash ca-certificates git go \ - && go install -ldflags "-X github.com/ipfs/go-ipfs/repo/config.CurrentCommit=$(git rev-parse --short HEAD 2> /dev/null || echo unknown)" \ - && mv /go/bin/ipfs /usr/local/bin/ipfs \ - && chmod 755 /usr/local/bin/start_ipfs \ - && apk del --purge go git - -WORKDIR / -RUN rm -rf /go/src/github.com/ipfs/go-ipfs +RUN apk add --update musl go=$GO_VERSION git bash wget ca-certificates \ + # Setup user and fs-repo directory + && mkdir -p $IPFS_PATH \ + && adduser -D -h $IPFS_PATH -u 1000 ipfs \ + && chown ipfs:ipfs $IPFS_PATH && chmod 755 $IPFS_PATH \ + # Install gx + && go get -u github.com/whyrusleeping/gx \ + && go get -u github.com/whyrusleeping/gx-go \ + # Point gx to a specific IPFS API + && ([ -z "$GX_IPFS" ] || echo $GX_IPFS > $IPFS_PATH/api) \ + # Invoke gx + && cd $SRC_PATH \ + && gx --verbose install --global \ + # Build and install IPFS and entrypoint script + && cd $SRC_PATH/cmd/ipfs \ + && go build -ldflags "-X github.com/ipfs/go-ipfs/repo/config.CurrentCommit=$(git rev-parse --short HEAD 2> /dev/null)" \ + && cp ipfs /usr/local/bin/ipfs \ + && cp $SRC_PATH/bin/container_daemon /usr/local/bin/start_ipfs \ + && chmod 755 /usr/local/bin/start_ipfs \ + # Remove all build-time dependencies + && apk del --purge musl go git && rm -rf $GOPATH && rm -vf $IPFS_PATH/api +# Call uid 1000 "ipfs" USER ipfs -VOLUME /data/ipfs +# Expose the fs-repo as a volume. +# We're doing this down here (and not at the top), +# so that the overlay directory is owned by the ipfs user. +# start_ipfs initializes an ephemeral fs-repo if none is mounted, +# which is why uid=1000 needs write permissions there. +VOLUME $IPFS_PATH + +# This just makes sure that: +# 1. There's an fs-repo, and initializes one if there isn't. +# 2. The API and Gateway are accessible from outside the container. ENTRYPOINT ["/usr/local/bin/start_ipfs"] - -# build: docker build -t go-ipfs . -# run: docker run -p 4001:4001 -p 5001:5001 go-ipfs:latest -# run: docker run -p 8080:8080 -p 4001:4001 -p 5001:5001 go-ipfs:latest diff --git a/bin/container_daemon b/bin/container_daemon index 4b85fc9ec..82fa26400 100644 --- a/bin/container_daemon +++ b/bin/container_daemon @@ -1,17 +1,18 @@ -#!/bin/bash +#!/bin/sh + +user=$(whoami) +repo="$IPFS_PATH" # Test whether the mounted directory is writable for us -if ( touch /data/ipfs/write_test 2>/dev/null ); then - rm /data/ipfs/write_test -else - echo "ERR: /data/ipfs is not writable for user 'ipfs' (UID 1000)" +if [ ! -w "$repo" 2>/dev/null ]; then + echo "error: $repo is not writable for user $user (uid=$(id -u $user))" exit 1 fi -echo "Running $(ipfs version)..." +ipfs version -if [ -e /data/ipfs/config ]; then - echo "Found ipfs repository. Not initializing." +if [ -e "$repo/config" ]; then + echo "Found IPFS fs-repo at $repo" else ipfs init ipfs config Addresses.API /ip4/0.0.0.0/tcp/5001 diff --git a/bin/container_shacheck b/bin/container_shacheck deleted file mode 100644 index fc53ec6d1..000000000 --- a/bin/container_shacheck +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -xe - -VERSION=$1 -FILENAME=$2 - -ONLINE_SHA=$( curl "https://gobuilder.me/api/v1/github.com/ipfs/go-ipfs/cmd/ipfs/signed-hashes/${VERSION}" 2>/dev/null | grep -A 4 ${FILENAME} | grep sha1 | awk '{ print $3 }' ) - -echo "Checking SHA1: ${ONLINE_SHA} == $(sha1sum ${FILENAME} | awk '{print $1}')" - -echo "${ONLINE_SHA} ${FILENAME}" | sha1sum -cw diff --git a/test/3nodetest/Makefile b/test/3nodetest/Makefile index 1c20a6837..b769a2ee5 100644 --- a/test/3nodetest/Makefile +++ b/test/3nodetest/Makefile @@ -25,7 +25,7 @@ bin/random: $(RANDOMSRC)/**/*.go # just build it every time... this part isn't # even the lengthy part, and it decreases pain. docker_ipfs_image: - cd $(IPFS_ROOT) && docker build -t $(IMAGE_NAME) . + docker build -t $(IMAGE_NAME) -f test/Dockerfile . docker images | grep $(IMAGE_NAME) clean: diff --git a/test/Dockerfile b/test/Dockerfile new file mode 100644 index 000000000..b4adec1ce --- /dev/null +++ b/test/Dockerfile @@ -0,0 +1,54 @@ +FROM alpine:3.3 +MAINTAINER Lars Gierth + +# This is a copy of the root Dockerfile, +# except that we optimize for build time, instead of image size. +# +# Please keep these two Dockerfiles in sync. +# +# Only sections different from the root Dockerfile are commented. + + +EXPOSE 4001 +EXPOSE 4002/udp +EXPOSE 5001 +EXPOSE 8080 + +ENV GX_IPFS "" +ENV IPFS_PATH /data/ipfs +ENV GO_VERSION 1.5.3-r0 +ENV GOPATH /go +ENV PATH /go/bin:$PATH +ENV SRC_PATH /go/src/github.com/ipfs/go-ipfs + +# This is an optimization which avoids rebuilding +# of the gx dependencies every time anything changes. +# gx will only be invoked if the dependencies have changed. +# +# Put differently: if package.json has changed, +# the image-id after this COPY command will change, +# and trigger a re-run of all following commands. +COPY ./package.json $SRC_PATH/package.json + +RUN apk add --update musl go=$GO_VERSION git bash wget ca-certificates \ + && mkdir -p $IPFS_PATH \ + && adduser -D -h $IPFS_PATH -u 1000 ipfs \ + && chown ipfs:ipfs $IPFS_PATH && chmod 755 $IPFS_PATH \ + && go get -u github.com/whyrusleeping/gx \ + && go get -u github.com/whyrusleeping/gx-go \ + && ([ -z "$GX_IPFS" ] || echo $GX_IPFS > $IPFS_PATH/api) \ + && cd $SRC_PATH \ + && gx --verbose install --global + +COPY . $SRC_PATH + +RUN cd $SRC_PATH/cmd/ipfs \ + && go build -ldflags "-X github.com/ipfs/go-ipfs/repo/config.CurrentCommit=$(git rev-parse --short HEAD 2> /dev/null)" \ + && cp ipfs /usr/local/bin/ipfs \ + && cp $SRC_PATH/bin/container_daemon /usr/local/bin/start_ipfs \ + && chmod 755 /usr/local/bin/start_ipfs \ + && apk del --purge musl go git && rm -rf $GOPATH && rm -vf $IPFS_PATH/api + +USER ipfs +VOLUME $IPFS_PATH +ENTRYPOINT ["/usr/local/bin/start_ipfs"] diff --git a/test/ipfs-test-lib.sh b/test/ipfs-test-lib.sh index d1897f386..50770080f 100644 --- a/test/ipfs-test-lib.sh +++ b/test/ipfs-test-lib.sh @@ -38,14 +38,14 @@ shellquote() { # Docker -# This takes a directory, that should contain a Dockerfile, as argument +# This takes a Dockerfile, and a build context directory docker_build() { - docker build --rm "$1" + docker build --rm -f "$1" "$2" } # This takes an image as argument and writes a docker ID on stdout docker_run() { - docker run -it -d -p 8080:8080 -p 4001:4001 -p 5001:5001 "$1" + docker run -d "$1" } # This takes a docker ID and a command as arguments @@ -54,7 +54,7 @@ docker_exec() { then sudo lxc-attach -n "$(docker inspect --format '{{.Id}}' $1)" -- /bin/bash -c "$2" else - docker exec -i "$1" /bin/bash -c "$2" + docker exec -t "$1" /bin/bash -c "$2" fi } diff --git a/test/sharness/t0300-docker-image.sh b/test/sharness/t0300-docker-image.sh index 1d0759e77..478b24344 100755 --- a/test/sharness/t0300-docker-image.sh +++ b/test/sharness/t0300-docker-image.sh @@ -33,7 +33,7 @@ TEST_TESTS_DIR=$(dirname "$TEST_SCRIPTS_DIR") APP_ROOT_DIR=$(dirname "$TEST_TESTS_DIR") test_expect_success "docker image build succeeds" ' - docker_build "$APP_ROOT_DIR" >actual + docker_build "$TEST_TESTS_DIR/Dockerfile" "$APP_ROOT_DIR" >actual ' test_expect_success "docker image build output looks good" ' @@ -46,12 +46,20 @@ test_expect_success "docker image runs" ' DOC_ID=$(docker_run "$IMAGE_ID") ' -test_expect_success "simple command can be run in docker container" ' - docker_exec "$DOC_ID" "echo Hello Worlds" >actual +test_expect_success "docker image gateway is up" ' + docker_exec "$DOC_ID" "wget --retry-connrefused --waitretry=1 --timeout=30 -t 30 \ + -q -O - http://localhost:8080/version >/dev/null" ' -test_expect_success "simple command output looks good" ' - echo "Hello Worlds" >expected && +test_expect_success "docker image API is up" ' + docker_exec "$DOC_ID" "wget --retry-connrefused --waitretry=1 --timeout=30 -t 30 \ + -q -O - http://localhost:5001/api/v0/version >/dev/null" +' + +test_expect_success "simple ipfs add/cat can be run in docker container" ' + expected="Hello Worlds" && + HASH=$(docker_exec "$DOC_ID" "echo $(cat expected) | ipfs add | cut -d' ' -f2") && + docker_exec "$DOC_ID" "ipfs cat $HASH" >actual && test_cmp expected actual '