Rework the Dockerfile

- Have two Dockerfiles doing essentially the same,
  but optimized for build time (for tests)
  and image size (for Docker Hub)
- Fetch gx dependencies
- Expose port 4002 for utp
- Specify go version, currently 1.5.3-r0
- Create ephemeral fs-repo if none is mounted
- Have t0300-docker-image actually test IPFS, not just an echo
- Make everything a bit less hardcoded
- Remove dead shacheck

License: MIT
Signed-off-by: Lars Gierth <larsg@systemli.org>

.dockerignore

-.git
+cmd/ipfs/ipfs
+vendor/gx/
+test/

Dockerfile

 FROM alpine:3.3
-MAINTAINER Brian Tiger Chow <btc@perfmode.com>
+MAINTAINER Lars Gierth <lgierth@ipfs.io>
+
+# There is a copy of this Dockerfile in test/sharness,
+# which is optimized for build time, instead of image size.
+#
+# Please keep these two Dockerfiles in sync.
 
-ENV IPFS_PATH /data/ipfs
-ENV GOPATH /go:/go/src/github.com/ipfs/go-ipfs/Godeps/_workspace
 
-EXPOSE 4001 5001 8080
-# 4001 = Swarm, 5001 = API, 8080 = HTTP transport
+# Ports for Swarm TCP, Swarm uTP, API, Gateway
+EXPOSE 4001
+EXPOSE 4002/udp
+EXPOSE 5001
+EXPOSE 8080
 
-ADD bin/container_daemon /usr/local/bin/start_ipfs
-ADD bin/container_shacheck /usr/local/bin/shacheck
+# Volume for mounting an IPFS fs-repo
+# This is moved to the bottom for technical reasons.
+#VOLUME $IPFS_PATH
 
-ADD . /go/src/github.com/ipfs/go-ipfs
-WORKDIR /go/src/github.com/ipfs/go-ipfs/cmd/ipfs
+# IPFS API to use for fetching gx packages.
+# This can be a gateway too, since its read-only API provides all gx needs.
+# - e.g. /ip4/172.17.0.1/tcp/8080 if the Docker host
+#   has the IPFS gateway listening on the bridge interface
+#   provided by Docker's default networking.
+# - if empty, the public gateway at ipfs.io is used.
+ENV GX_IPFS   ""
+# The IPFS fs-repo within the container
+ENV IPFS_PATH /data/ipfs
+# Golang stuff
+ENV GO_VERSION 1.5.3-r0
+ENV GOPATH     /go
+ENV PATH       /go/bin:$PATH
+ENV SRC_PATH   /go/src/github.com/ipfs/go-ipfs
 
-RUN adduser -D -h /data -u 1000 ipfs \
- && mkdir -p /data/ipfs && chown ipfs:ipfs /data/ipfs \
- && apk add --update bash ca-certificates git go \
- && go install -ldflags "-X github.com/ipfs/go-ipfs/repo/config.CurrentCommit=$(git rev-parse --short HEAD 2> /dev/null || echo unknown)" \
- && mv /go/bin/ipfs /usr/local/bin/ipfs \
- && chmod 755 /usr/local/bin/start_ipfs \
- && apk del --purge go git
+# Get the go-ipfs sourcecode
+COPY . $SRC_PATH
 
-WORKDIR /
-RUN rm -rf /go/src/github.com/ipfs/go-ipfs
+RUN apk add --update musl go=$GO_VERSION git bash wget ca-certificates \
+	# Setup user and fs-repo directory
+	&& mkdir -p $IPFS_PATH \
+	&& adduser -D -h $IPFS_PATH -u 1000 ipfs \
+	&& chown ipfs:ipfs $IPFS_PATH && chmod 755 $IPFS_PATH \
+	# Install gx
+	&& go get -u github.com/whyrusleeping/gx \
+	&& go get -u github.com/whyrusleeping/gx-go \
+	# Point gx to a specific IPFS API
+	&& ([ -z "$GX_IPFS" ] || echo $GX_IPFS > $IPFS_PATH/api) \
+	# Invoke gx
+	&& cd $SRC_PATH \
+	&& gx --verbose install --global \
+	# Build and install IPFS and entrypoint script
+	&& cd $SRC_PATH/cmd/ipfs \
+	&& go build -ldflags "-X github.com/ipfs/go-ipfs/repo/config.CurrentCommit=$(git rev-parse --short HEAD 2> /dev/null)" \
+	&& cp ipfs /usr/local/bin/ipfs \
+	&& cp $SRC_PATH/bin/container_daemon /usr/local/bin/start_ipfs \
+	&& chmod 755 /usr/local/bin/start_ipfs \
+	# Remove all build-time dependencies
+	&& apk del --purge musl go git && rm -rf $GOPATH && rm -vf $IPFS_PATH/api
 
+# Call uid 1000 "ipfs"
 USER ipfs
-VOLUME /data/ipfs
 
-ENTRYPOINT ["/usr/local/bin/start_ipfs"]
+# Expose the fs-repo as a volume.
+# We're doing this down here (and not at the top),
+# so that the overlay directory is owned by the ipfs user.
+# start_ipfs initializes an ephemeral fs-repo if none is mounted,
+# which is why uid=1000 needs write permissions there.
+VOLUME $IPFS_PATH
 
-# build:    docker build -t go-ipfs .
-# run:      docker run -p 4001:4001 -p 5001:5001 go-ipfs:latest
-# run:      docker run -p 8080:8080 -p 4001:4001 -p 5001:5001 go-ipfs:latest
+# This just makes sure that:
+# 1. There's an fs-repo, and initializes one if there isn't.
+# 2. The API and Gateway are accessible from outside the container.
+ENTRYPOINT ["/usr/local/bin/start_ipfs"]

bin/container_daemon

-#!/bin/bash
+#!/bin/sh
+
+user=$(whoami)
+repo="$IPFS_PATH"
 
 # Test whether the mounted directory is writable for us
-if ( touch /data/ipfs/write_test 2>/dev/null ); then
-  rm /data/ipfs/write_test
-else
-  echo "ERR: /data/ipfs is not writable for user 'ipfs' (UID 1000)"
+if [ ! -w "$repo" 2>/dev/null ]; then
+  echo "error: $repo is not writable for user $user (uid=$(id -u $user))"
   exit 1
 fi
 
-echo "Running $(ipfs version)..."
+ipfs version
 
-if [ -e /data/ipfs/config ]; then
-  echo "Found ipfs repository. Not initializing."
+if [ -e "$repo/config" ]; then
+  echo "Found IPFS fs-repo at $repo"
 else
   ipfs init
   ipfs config Addresses.API /ip4/0.0.0.0/tcp/5001

bin/container_shacheck

-#!/bin/bash -xe
-
-VERSION=$1
-FILENAME=$2
-
-ONLINE_SHA=$( curl "https://gobuilder.me/api/v1/github.com/ipfs/go-ipfs/cmd/ipfs/signed-hashes/${VERSION}" 2>/dev/null | grep -A 4 ${FILENAME} | grep sha1 | awk '{ print $3 }' )
-
-echo "Checking SHA1: ${ONLINE_SHA} == $(sha1sum ${FILENAME} | awk '{print $1}')"
-
-echo "${ONLINE_SHA}  ${FILENAME}" | sha1sum -cw

test/3nodetest/Makefile

@@ -25,7 +25,7 @@ bin/random: $(RANDOMSRC)/**/*.go
 # just build it every time... this part isn't
 # even the lengthy part, and it decreases pain.
 docker_ipfs_image:
-	cd $(IPFS_ROOT) && docker build -t $(IMAGE_NAME) .
+	docker build -t $(IMAGE_NAME) -f test/Dockerfile .
 	docker images | grep $(IMAGE_NAME)
 
 clean:

test/Dockerfile

+FROM alpine:3.3
+MAINTAINER Lars Gierth <lgierth@ipfs.io>
+
+# This is a copy of the root Dockerfile,
+# except that we optimize for build time, instead of image size.
+#
+# Please keep these two Dockerfiles in sync.
+#
+# Only sections different from the root Dockerfile are commented.
+
+
+EXPOSE 4001
+EXPOSE 4002/udp
+EXPOSE 5001
+EXPOSE 8080
+
+ENV GX_IPFS   ""
+ENV IPFS_PATH /data/ipfs
+ENV GO_VERSION 1.5.3-r0
+ENV GOPATH     /go
+ENV PATH       /go/bin:$PATH
+ENV SRC_PATH   /go/src/github.com/ipfs/go-ipfs
+
+# This is an optimization which avoids rebuilding
+# of the gx dependencies every time anything changes.
+# gx will only be invoked if the dependencies have changed.
+#
+# Put differently: if package.json has changed,
+# the image-id after this COPY command will change,
+# and trigger a re-run of all following commands.
+COPY ./package.json $SRC_PATH/package.json
+
+RUN apk add --update musl go=$GO_VERSION git bash wget ca-certificates \
+	&& mkdir -p $IPFS_PATH \
+	&& adduser -D -h $IPFS_PATH -u 1000 ipfs \
+	&& chown ipfs:ipfs $IPFS_PATH && chmod 755 $IPFS_PATH \
+	&& go get -u github.com/whyrusleeping/gx \
+	&& go get -u github.com/whyrusleeping/gx-go \
+	&& ([ -z "$GX_IPFS" ] || echo $GX_IPFS > $IPFS_PATH/api) \
+	&& cd $SRC_PATH \
+	&& gx --verbose install --global
+
+COPY . $SRC_PATH
+
+RUN cd $SRC_PATH/cmd/ipfs \
+	&& go build -ldflags "-X github.com/ipfs/go-ipfs/repo/config.CurrentCommit=$(git rev-parse --short HEAD 2> /dev/null)" \
+	&& cp ipfs /usr/local/bin/ipfs \
+	&& cp $SRC_PATH/bin/container_daemon /usr/local/bin/start_ipfs \
+	&& chmod 755 /usr/local/bin/start_ipfs \
+	&& apk del --purge musl go git && rm -rf $GOPATH && rm -vf $IPFS_PATH/api
+
+USER ipfs
+VOLUME $IPFS_PATH
+ENTRYPOINT ["/usr/local/bin/start_ipfs"]

test/ipfs-test-lib.sh

@@ -38,14 +38,14 @@ shellquote() {
 
 # Docker
 
-# This takes a directory, that should contain a Dockerfile, as argument
+# This takes a Dockerfile, and a build context directory
 docker_build() {
-    docker build --rm "$1"
+    docker build --rm -f "$1" "$2"
 }
 
 # This takes an image as argument and writes a docker ID on stdout
 docker_run() {
-    docker run -it -d -p 8080:8080 -p 4001:4001 -p 5001:5001 "$1"
+    docker run -d "$1"
 }
 
 # This takes a docker ID and a command as arguments
@@ -54,7 +54,7 @@ docker_exec() {
     then
         sudo lxc-attach -n "$(docker inspect --format '{{.Id}}' $1)" -- /bin/bash -c "$2"
     else
-	docker exec -i "$1" /bin/bash -c "$2"
+	docker exec -t "$1" /bin/bash -c "$2"
     fi
 }
 

test/sharness/t0300-docker-image.sh

@@ -33,7 +33,7 @@ TEST_TESTS_DIR=$(dirname "$TEST_SCRIPTS_DIR")
 APP_ROOT_DIR=$(dirname "$TEST_TESTS_DIR")
 
 test_expect_success "docker image build succeeds" '
-	docker_build "$APP_ROOT_DIR" >actual
+	docker_build "$TEST_TESTS_DIR/Dockerfile" "$APP_ROOT_DIR" >actual
 '
 
 test_expect_success "docker image build output looks good" '
@@ -46,12 +46,20 @@ test_expect_success "docker image runs" '
 	DOC_ID=$(docker_run "$IMAGE_ID")
 '
 
-test_expect_success "simple command can be run in docker container" '
-	docker_exec "$DOC_ID" "echo Hello Worlds" >actual
+test_expect_success "docker image gateway is up" '
+	docker_exec "$DOC_ID" "wget --retry-connrefused --waitretry=1 --timeout=30 -t 30 \
+		-q -O - http://localhost:8080/version >/dev/null"
 '
 
-test_expect_success "simple command output looks good" '
-	echo "Hello Worlds" >expected &&
+test_expect_success "docker image API is up" '
+	docker_exec "$DOC_ID" "wget --retry-connrefused --waitretry=1 --timeout=30 -t 30 \
+		-q -O - http://localhost:5001/api/v0/version >/dev/null"
+'
+
+test_expect_success "simple ipfs add/cat can be run in docker container" '
+	expected="Hello Worlds" &&
+	HASH=$(docker_exec "$DOC_ID" "echo $(cat expected) | ipfs add | cut -d' ' -f2") &&
+	docker_exec "$DOC_ID" "ipfs cat $HASH" >actual &&
 	test_cmp expected actual
 '