fix and improve the writable gateway

1. Fix handling of PUT. The simple implementation was the correct
   implementation, I have no idea what was going on here.
2. Use MFS everywhere to reduce code duplication and add support for sharded
   directories.
3. _Correctly_ block IPNS.
4. Remove the dependency on `core.IpfsNode`.
5. Remove support for putting empty directories with a well-known CID. It was
   useless as directories are automatically created.

core/corehttp/gateway.go

@@ -87,7 +87,7 @@ func GatewayOption(writable bool, paths ...string) ServeOption {
 				"X-Stream-Output",
 			}, headers[ACEHeadersName]...))
 
-		gateway := newGatewayHandler(n, GatewayConfig{
+		gateway := newGatewayHandler(GatewayConfig{
 			Headers:      headers,
 			Writable:     writable,
 			PathPrefixes: cfg.Gateway.PathPrefixes,

core/corehttp/gateway_handler.go

@@ -2,30 +2,23 @@ package corehttp
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
+	"os"
 	gopath "path"
 	"runtime/debug"
 	"strings"
 	"time"
 
-	"github.com/ipfs/go-ipfs/core"
-	"github.com/ipfs/go-ipfs/dagutils"
-	"github.com/ipfs/go-ipfs/namesys/resolve"
-
 	"github.com/dustin/go-humanize"
 	"github.com/ipfs/go-cid"
-	chunker "github.com/ipfs/go-ipfs-chunker"
 	files "github.com/ipfs/go-ipfs-files"
-	ipld "github.com/ipfs/go-ipld-format"
 	dag "github.com/ipfs/go-merkledag"
+	"github.com/ipfs/go-mfs"
 	"github.com/ipfs/go-path"
 	"github.com/ipfs/go-path/resolver"
-	ft "github.com/ipfs/go-unixfs"
-	"github.com/ipfs/go-unixfs/importer"
 	coreiface "github.com/ipfs/interface-go-ipfs-core"
 	ipath "github.com/ipfs/interface-go-ipfs-core/path"
 	routing "github.com/libp2p/go-libp2p-core/routing"
@@ -40,27 +33,36 @@ const (
 // gatewayHandler is a HTTP handler that serves IPFS objects (accessible by default at /ipfs/<path>)
 // (it serves requests like GET /ipfs/QmVRzPKPzNtSrEzBFm2UZfxmPAgnaLke4DMcerbsGGSaFe/link)
 type gatewayHandler struct {
-	node   *core.IpfsNode
 	config GatewayConfig
 	api    coreiface.CoreAPI
 }
 
-func newGatewayHandler(n *core.IpfsNode, c GatewayConfig, api coreiface.CoreAPI) *gatewayHandler {
+func newGatewayHandler(c GatewayConfig, api coreiface.CoreAPI) *gatewayHandler {
 	i := &gatewayHandler{
-		node:   n,
 		config: c,
 		api:    api,
 	}
 	return i
 }
 
-// TODO(cryptix):  find these helpers somewhere else
-func (i *gatewayHandler) newDagFromReader(r io.Reader) (ipld.Node, error) {
-	// TODO(cryptix): change and remove this helper once PR1136 is merged
-	// return ufs.AddFromReader(i.node, r.Body)
-	return importer.BuildDagFromReader(
-		i.node.DAG,
-		chunker.DefaultSplitter(r))
+func parseIpfsPath(p string) (cid.Cid, string, error) {
+	rootPath, err := path.ParsePath(p)
+	if err != nil {
+		return cid.Cid{}, "", err
+	}
+
+	// Check the path.
+	rsegs := rootPath.Segments()
+	if rsegs[0] != "ipfs" {
+		return cid.Cid{}, "", fmt.Errorf("WritableGateway: only ipfs paths supported")
+	}
+
+	rootCid, err := cid.Decode(rsegs[1])
+	if err != nil {
+		return cid.Cid{}, "", err
+	}
+
+	return rootCid, path.Join(rsegs[2:]), nil
 }
 
 func (i *gatewayHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -160,10 +162,12 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 
 	// Resolve path to the final DAG node for the ETag
 	resolvedPath, err := i.api.ResolvePath(r.Context(), parsedPath)
-	if err == coreiface.ErrOffline && !i.node.IsOnline {
+	switch err {
+	case nil:
+	case coreiface.ErrOffline:
 		webError(w, "ipfs resolve -r "+escapedURLPath, err, http.StatusServiceUnavailable)
 		return
-	} else if err != nil {
+	default:
 		webError(w, "ipfs resolve -r "+escapedURLPath, err, http.StatusNotFound)
 		return
 	}
@@ -395,102 +399,91 @@ func (i *gatewayHandler) postHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
-	rootPath, err := path.ParsePath(r.URL.Path)
+	ctx := r.Context()
+	ds := i.api.Dag()
+
+	// Parse the path
+	rootCid, newPath, err := parseIpfsPath(r.URL.Path)
 	if err != nil {
-		webError(w, "putHandler: IPFS path not valid", err, http.StatusBadRequest)
+		webError(w, "WritableGateway: failed to parse the path", err, http.StatusBadRequest)
 		return
 	}
-
-	rsegs := rootPath.Segments()
-	if rsegs[0] == ipnsPathPrefix {
-		webError(w, "putHandler: updating named entries not supported", errors.New("WritableGateway: ipns put not supported"), http.StatusBadRequest)
+	if newPath == "" || newPath == "/" {
+		http.Error(w, "WritableGateway: empty path", http.StatusBadRequest)
 		return
 	}
+	newDirectory, newFileName := gopath.Split(newPath)
 
-	var newnode ipld.Node
-	if rsegs[len(rsegs)-1] == "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn" {
-		newnode = ft.EmptyDirNode()
-	} else {
-		putNode, err := i.newDagFromReader(r.Body)
-		if err != nil {
-			webError(w, "putHandler: Could not create DAG from request", err, http.StatusInternalServerError)
-			return
-		}
-		newnode = putNode
-	}
+	// Resolve the old root.
 
-	var newPath string
-	if len(rsegs) > 1 {
-		newPath = path.Join(rsegs[2:])
+	rnode, err := ds.Get(ctx, rootCid)
+	if err != nil {
+		webError(w, "WritableGateway: Could not create DAG from request", err, http.StatusInternalServerError)
+		return
 	}
 
-	var newcid cid.Cid
-	rnode, err := resolve.Resolve(r.Context(), i.node.Namesys, i.node.Resolver, rootPath)
-	switch ev := err.(type) {
-	case resolver.ErrNoLink:
-		// ev.Node < node where resolve failed
-		// ev.Name < new link
-		// but we need to patch from the root
-		c, err := cid.Decode(rsegs[1])
-		if err != nil {
-			webError(w, "putHandler: bad input path", err, http.StatusBadRequest)
-			return
-		}
-
-		rnode, err := i.node.DAG.Get(r.Context(), c)
-		if err != nil {
-			webError(w, "putHandler: Could not create DAG from request", err, http.StatusInternalServerError)
-			return
-		}
-
-		pbnd, ok := rnode.(*dag.ProtoNode)
-		if !ok {
-			webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest)
-			return
-		}
-
-		e := dagutils.NewDagEditor(pbnd, i.node.DAG)
-		err = e.InsertNodeAtPath(r.Context(), newPath, newnode, ft.EmptyDirNode)
-		if err != nil {
-			webError(w, "putHandler: InsertNodeAtPath failed", err, http.StatusInternalServerError)
-			return
-		}
-
-		nnode, err := e.Finalize(r.Context(), i.node.DAG)
-		if err != nil {
-			webError(w, "putHandler: could not get node", err, http.StatusInternalServerError)
-			return
-		}
+	pbnd, ok := rnode.(*dag.ProtoNode)
+	if !ok {
+		webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest)
+		return
+	}
 
-		newcid = nnode.Cid()
+	// Create the new file.
+	newFilePath, err := i.api.Unixfs().Add(ctx, files.NewReaderFile(r.Body))
+	if err != nil {
+		webError(w, "WritableGateway: could not create DAG from request", err, http.StatusInternalServerError)
+		return
+	}
 
-	case nil:
-		pbnd, ok := rnode.(*dag.ProtoNode)
-		if !ok {
-			webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest)
-			return
-		}
+	newFile, err := ds.Get(ctx, newFilePath.Cid())
+	if err != nil {
+		webError(w, "WritableGateway: failed to resolve new file", err, http.StatusInternalServerError)
+		return
+	}
 
-		pbnewnode, ok := newnode.(*dag.ProtoNode)
-		if !ok {
-			webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest)
-			return
-		}
+	// Patch the new file into the old root.
 
-		// object set-data case
-		pbnd.SetData(pbnewnode.Data())
+	root, err := mfs.NewRoot(ctx, ds, pbnd, nil)
+	if err != nil {
+		webError(w, "WritableGateway: failed to create MFS root", err, http.StatusBadRequest)
+		return
+	}
 
-		newcid = pbnd.Cid()
-		err = i.node.DAG.Add(r.Context(), pbnd)
+	if newDirectory != "" {
+		err := mfs.Mkdir(root, newDirectory, mfs.MkdirOpts{Mkparents: true, Flush: false})
 		if err != nil {
-			nnk := newnode.Cid()
-			webError(w, fmt.Sprintf("putHandler: Could not add newnode(%q) to root(%q)", nnk.String(), newcid.String()), err, http.StatusInternalServerError)
+			webError(w, "WritableGateway: failed to create MFS directory", err, http.StatusInternalServerError)
 			return
 		}
+	}
+	dirNode, err := mfs.Lookup(root, newDirectory)
+	if err != nil {
+		webError(w, "WritableGateway: failed to lookup directory", err, http.StatusInternalServerError)
+		return
+	}
+	dir, ok := dirNode.(*mfs.Directory)
+	if !ok {
+		http.Error(w, "WritableGateway: target directory is not a directory", http.StatusBadRequest)
+		return
+	}
+	err = dir.Unlink(newFileName)
+	switch err {
+	case os.ErrNotExist, nil:
 	default:
-		webError(w, "could not resolve root DAG", ev, http.StatusInternalServerError)
+		webError(w, "WritableGateway: failed to replace existing file", err, http.StatusBadRequest)
 		return
 	}
+	err = dir.AddChild(newFileName, newFile)
+	if err != nil {
+		webError(w, "WritableGateway: failed to link file into directory", err, http.StatusInternalServerError)
+		return
+	}
+	nnode, err := root.GetDirectory().GetNode()
+	if err != nil {
+		webError(w, "WritableGateway: failed to finalize", err, http.StatusInternalServerError)
+		return
+	}
+	newcid := nnode.Cid()
 
 	i.addUserHeaders(w) // ok, _now_ write user's headers.
 	w.Header().Set("IPFS-Hash", newcid.String())
@@ -498,91 +491,75 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func (i *gatewayHandler) deleteHandler(w http.ResponseWriter, r *http.Request) {
-	urlPath := r.URL.Path
+	ctx := r.Context()
+
+	// parse the path
 
-	p, err := path.ParsePath(urlPath)
+	rootCid, newPath, err := parseIpfsPath(r.URL.Path)
 	if err != nil {
-		webError(w, "failed to parse path", err, http.StatusBadRequest)
+		webError(w, "WritableGateway: failed to parse the path", err, http.StatusBadRequest)
 		return
 	}
-
-	c, components, err := path.SplitAbsPath(p)
-	if err != nil {
-		webError(w, "Could not split path", err, http.StatusInternalServerError)
+	if newPath == "" || newPath == "/" {
+		http.Error(w, "WritableGateway: empty path", http.StatusBadRequest)
 		return
 	}
+	directory, filename := gopath.Split(newPath)
+
+	// lookup the root
 
-	pathNodes, err := i.resolvePathComponents(r.Context(), c, components)
+	rootNodeIPLD, err := i.api.Dag().Get(ctx, rootCid)
 	if err != nil {
-		webError(w, "Could not resolve path components", err, http.StatusBadRequest)
+		webError(w, "WritableGateway: failed to resolve root CID", err, http.StatusInternalServerError)
 		return
 	}
-
-	pbnd, ok := pathNodes[len(pathNodes)-1].(*dag.ProtoNode)
+	rootNode, ok := rootNodeIPLD.(*dag.ProtoNode)
 	if !ok {
-		webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest)
+		http.Error(w, "WritableGateway: empty path", http.StatusInternalServerError)
 		return
 	}
 
-	// TODO(cyrptix): assumes len(pathNodes) > 1 - not found is an error above?
-	err = pbnd.RemoveNodeLink(components[len(components)-1])
+	// construct the mfs root
+
+	root, err := mfs.NewRoot(ctx, i.api.Dag(), rootNode, nil)
 	if err != nil {
-		webError(w, "Could not delete link", err, http.StatusBadRequest)
+		webError(w, "WritableGateway: failed to construct the MFS root", err, http.StatusBadRequest)
 		return
 	}
 
-	var newnode *dag.ProtoNode = pbnd
-	for j := len(pathNodes) - 2; j >= 0; j-- {
-		if err := i.node.DAG.Add(r.Context(), newnode); err != nil {
-			webError(w, "Could not add node", err, http.StatusInternalServerError)
-			return
-		}
+	// lookup the parent directory
 
-		pathpb, ok := pathNodes[j].(*dag.ProtoNode)
-		if !ok {
-			webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest)
-			return
-		}
-
-		newnode, err = pathpb.UpdateNodeLink(components[j], newnode)
-		if err != nil {
-			webError(w, "Could not update node links", err, http.StatusInternalServerError)
-			return
-		}
+	parentNode, err := mfs.Lookup(root, directory)
+	if err != nil {
+		webError(w, "WritableGateway: failed to look up parent", err, http.StatusInternalServerError)
+		return
 	}
 
-	if err := i.node.DAG.Add(r.Context(), newnode); err != nil {
-		webError(w, "Could not add root node", err, http.StatusInternalServerError)
+	parent, ok := parentNode.(*mfs.Directory)
+	if !ok {
+		http.Error(w, "WritableGateway: parent is not a directory", http.StatusInternalServerError)
 		return
 	}
 
-	// Redirect to new path
-	ncid := newnode.Cid()
+	// delete the file
 
-	i.addUserHeaders(w) // ok, _now_ write user's headers.
-	w.Header().Set("IPFS-Hash", ncid.String())
-	http.Redirect(w, r, gopath.Join(ipfsPathPrefix+ncid.String(), path.Join(components[:len(components)-1])), http.StatusCreated)
-}
-
-func (i *gatewayHandler) resolvePathComponents(
-	ctx context.Context,
-	c cid.Cid,
-	components []string,
-) ([]ipld.Node, error) {
-	tctx, cancel := context.WithTimeout(ctx, time.Minute)
-	defer cancel()
-
-	rootnd, err := i.node.Resolver.DAG.Get(tctx, c)
-	if err != nil {
-		return nil, fmt.Errorf("Could not resolve root object: %s", err)
+	switch parent.Unlink(filename) {
+	case nil, os.ErrNotExist:
+	default:
+		webError(w, "WritableGateway: failed to remove file", err, http.StatusInternalServerError)
+		return
 	}
 
-	pathNodes, err := i.node.Resolver.ResolveLinks(tctx, rootnd, components[:len(components)-1])
+	nnode, err := root.GetDirectory().GetNode()
 	if err != nil {
-		return nil, fmt.Errorf("Could not resolve parent object: %s", err)
+		webError(w, "WritableGateway: failed to finalize", err, http.StatusInternalServerError)
 	}
+	ncid := nnode.Cid()
 
-	return pathNodes, nil
+	i.addUserHeaders(w) // ok, _now_ write user's headers.
+	w.Header().Set("IPFS-Hash", ncid.String())
+	// note: StatusCreated is technically correct here as we created a new resource.
+	http.Redirect(w, r, gopath.Join(ipfsPathPrefix+ncid.String(), directory), http.StatusCreated)
 }
 
 func (i *gatewayHandler) addUserHeaders(w http.ResponseWriter) {

go.mod

@@ -45,12 +45,12 @@ require (
 	github.com/ipfs/go-ipld-git v0.0.2
 	github.com/ipfs/go-ipns v0.0.1
 	github.com/ipfs/go-log v0.0.1
-	github.com/ipfs/go-merkledag v0.2.0
+	github.com/ipfs/go-merkledag v0.2.3
 	github.com/ipfs/go-metrics-interface v0.0.1
 	github.com/ipfs/go-metrics-prometheus v0.0.2
 	github.com/ipfs/go-mfs v0.1.0
 	github.com/ipfs/go-path v0.0.7
-	github.com/ipfs/go-unixfs v0.2.0
+	github.com/ipfs/go-unixfs v0.2.1
 	github.com/ipfs/go-verifcid v0.0.1
 	github.com/ipfs/hang-fds v0.0.1
 	github.com/ipfs/interface-go-ipfs-core v0.1.0

go.sum

@@ -301,6 +301,8 @@ github.com/ipfs/go-merkledag v0.1.0 h1:CAEXjRFEDPvealQj3TgEjV1IJckwjvmxAqtq5QSXJ
 github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
 github.com/ipfs/go-merkledag v0.2.0 h1:EAjIQCgZ6/DnOAlKY3+59j72FD9BsYtNaCRSmN0xIbU=
 github.com/ipfs/go-merkledag v0.2.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
+github.com/ipfs/go-merkledag v0.2.3 h1:aMdkK9G1hEeNvn3VXfiEMLY0iJnbiQQUHnM0HFJREsE=
+github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
 github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
 github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
 github.com/ipfs/go-metrics-prometheus v0.0.2 h1:9i2iljLg12S78OhC6UAiXi176xvQGiZaGVF1CUVdE+s=
@@ -320,8 +322,8 @@ github.com/ipfs/go-todocounter v0.0.1/go.mod h1:l5aErvQc8qKE2r7NDMjmq5UNAvuZy0rC
 github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vUpWyfz8=
 github.com/ipfs/go-unixfs v0.0.8 h1:AHahQ+gdNZd9BhKVLf8XP1EWeKa78eTzYgCygp7N/Pg=
 github.com/ipfs/go-unixfs v0.0.8/go.mod h1:cK2vDJ7L4YnWB6oLefpVNesgx0x/zPTRVDw6B4Y+03U=
-github.com/ipfs/go-unixfs v0.2.0 h1:mfdI8rgsEifWfhLECrH2WphHvslNoPbdvlmsJ05Fu0M=
-github.com/ipfs/go-unixfs v0.2.0/go.mod h1:sy/j20FKUxdUYOl6GZb3wsQ593C2xk5r1i8U7W+5iio=
+github.com/ipfs/go-unixfs v0.2.1 h1:g51t9ODICFZ3F51FPivm8dE7NzYcdAQNUL9wGP5AYa0=
+github.com/ipfs/go-unixfs v0.2.1/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k=
 github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E=
 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
 github.com/ipfs/hang-fds v0.0.1 h1:KGAxiGtJPT3THVRNT6yxgpdFPeX4ZemUjENOt6NlOn4=

test/sharness/t0111-gateway-writeable.sh

@@ -54,17 +54,8 @@ test_expect_success "We can HTTP GET file just created" '
   test_cmp infile outfile
 '
 
-test_expect_success "HTTP PUT empty directory" '
-  URL="http://localhost:$port/ipfs/$HASH_EMPTY_DIR/" &&
-  echo "PUT $URL" &&
-  curl -svX PUT "$URL" 2>curl_putEmpty.out &&
-  cat curl_putEmpty.out &&
-  grep "Ipfs-Hash: $HASH_EMPTY_DIR" curl_putEmpty.out &&
-  grep "Location: /ipfs/$HASH_EMPTY_DIR" curl_putEmpty.out &&
-  grep "HTTP/1.1 201 Created" curl_putEmpty.out
-'
-
 test_expect_success "HTTP GET empty directory" '
+  URL="http://localhost:$port/ipfs/$HASH_EMPTY_DIR/" &&
   echo "GET $URL" &&
   curl -so outfile "$URL" 2>curl_getEmpty.out &&
   grep "Index of /ipfs/$HASH_EMPTY_DIR/" outfile
@@ -105,6 +96,24 @@ test_expect_success "We can HTTP GET file just updated" '
   test_cmp infile2 outfile2
 '
 
+test_expect_success "HTTP PUT to replace a directory" '
+  echo "$RANDOM" >infile3 &&
+  URL="http://localhost:$port/ipfs/$HASH/test" &&
+  echo "PUT $URL" &&
+  curl -svX PUT --data-binary @infile3 "$URL" 2>curl_putOverDirectory.out &&
+  grep "HTTP/1.1 201 Created" curl_putOverDirectory.out &&
+  LOCATION=$(grep Location curl_putOverDirectory.out) &&
+  HASH=$(expr "$LOCATION" : "< Location: /ipfs/\(.*\)/test")
+'
+
+test_expect_success "We can HTTP GET file just put over a directory" '
+  URL="http://localhost:$port/ipfs/$HASH/test" &&
+  echo "GET $URL" &&
+  curl -svo outfile3 "$URL" 2>curl_getOverDirectory.out &&
+  test_cmp infile3 outfile3
+'
+
+
 test_kill_ipfs_daemon
 
 test_done