Merge pull request #6539 from ipfs/fix/writable-gateway

fix and improve the writable gateway

core/commands/dht.go

@@ -326,7 +326,7 @@ func provideKeysRec(ctx context.Context, r routing.Routing, dserv ipld.DAGServic
 	for _, c := range cids {
 		kset := cid.NewSet()
 
-		err := dag.WalkParallel(ctx, dag.GetLinksDirect(dserv), c, kset.Visit)
+		err := dag.Walk(ctx, dag.GetLinksDirect(dserv), c, kset.Visit)
 		if err != nil {
 			return err
 		}

core/commands/pin.go

@@ -503,11 +503,7 @@ func pinLsAll(req *cmds.Request, typeStr string, n *core.IpfsNode, emit func(val
 	if typeStr == "indirect" || typeStr == "all" {
 		for _, k := range n.Pinning.RecursiveKeys() {
 			var visitErr error
-			err := dag.WalkParallelDepth(req.Context, dag.GetLinksWithDAG(n.DAG), k, 0, func(c cid.Cid, depth int) bool {
-				if depth == 0 {
-					// skip it without visiting it.
-					return true
-				}
+			err := dag.Walk(req.Context, dag.GetLinksWithDAG(n.DAG), k, func(c cid.Cid) bool {
 				r := keys.Visit(c)
 				if r {
 					err := emit(&PinLsOutputWrapper{
@@ -521,7 +517,7 @@ func pinLsAll(req *cmds.Request, typeStr string, n *core.IpfsNode, emit func(val
 					}
 				}
 				return r
-			})
+			}, dag.SkipRoot(), dag.Concurrent())
 
 			if visitErr != nil {
 				return visitErr

core/coreapi/dht.go

@@ -114,7 +114,7 @@ func provideKeysRec(ctx context.Context, r routing.Routing, bs blockstore.Blocks
 	go func() {
 		dserv := dag.NewDAGService(blockservice.New(bs, offline.Exchange(bs)))
 		for _, c := range cids {
-			err := dag.WalkParallel(ctx, dag.GetLinksDirect(dserv), c, provided.Visitor(ctx))
+			err := dag.Walk(ctx, dag.GetLinksDirect(dserv), c, provided.Visitor(ctx))
 			if err != nil {
 				errCh <- err
 			}

core/coreapi/pin.go

@@ -209,12 +209,10 @@ func (api *PinAPI) pinLsAll(typeStr string, ctx context.Context) ([]coreiface.Pi
 	if typeStr == "indirect" || typeStr == "all" {
 		set := cid.NewSet()
 		for _, k := range api.pinning.RecursiveKeys() {
-			err := merkledag.WalkParallelDepth(
-				ctx, merkledag.GetLinksWithDAG(api.dag), k, 0,
-				func(c cid.Cid, depth int) bool {
-					// don't visit the root node, that doesn't count.
-					return depth == 0 || set.Visit(c)
-				},
+			err := merkledag.Walk(
+				ctx, merkledag.GetLinksWithDAG(api.dag), k,
+				set.Visit,
+				merkledag.SkipRoot(), merkledag.Concurrent(),
 			)
 			if err != nil {
 				return nil, err

core/corehttp/gateway.go

@@ -87,7 +87,7 @@ func GatewayOption(writable bool, paths ...string) ServeOption {
 				"X-Stream-Output",
 			}, headers[ACEHeadersName]...))
 
-		gateway := newGatewayHandler(n, GatewayConfig{
+		gateway := newGatewayHandler(GatewayConfig{
 			Headers:      headers,
 			Writable:     writable,
 			PathPrefixes: cfg.Gateway.PathPrefixes,

dagutils/diffenum.go

@@ -40,7 +40,7 @@ func DiffEnumerate(ctx context.Context, dserv ipld.NodeGetter, from, to cid.Cid)
 			if sset.Has(c.aft) {
 				continue
 			}
-			err := mdag.WalkParallel(ctx, mdag.GetLinksDirect(dserv), c.aft, sset.Visit)
+			err := mdag.Walk(ctx, mdag.GetLinksDirect(dserv), c.aft, sset.Visit, mdag.Concurrent())
 			if err != nil {
 				return err
 			}

go.mod

@@ -45,12 +45,12 @@ require (
 	github.com/ipfs/go-ipld-git v0.0.2
 	github.com/ipfs/go-ipns v0.0.1
 	github.com/ipfs/go-log v0.0.1
-	github.com/ipfs/go-merkledag v0.2.0
+	github.com/ipfs/go-merkledag v0.2.3
 	github.com/ipfs/go-metrics-interface v0.0.1
 	github.com/ipfs/go-metrics-prometheus v0.0.2
 	github.com/ipfs/go-mfs v0.1.0
 	github.com/ipfs/go-path v0.0.7
-	github.com/ipfs/go-unixfs v0.2.0
+	github.com/ipfs/go-unixfs v0.2.1
 	github.com/ipfs/go-verifcid v0.0.1
 	github.com/ipfs/hang-fds v0.0.1
 	github.com/ipfs/interface-go-ipfs-core v0.1.0

go.sum

@@ -301,6 +301,8 @@ github.com/ipfs/go-merkledag v0.1.0 h1:CAEXjRFEDPvealQj3TgEjV1IJckwjvmxAqtq5QSXJ
 github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
 github.com/ipfs/go-merkledag v0.2.0 h1:EAjIQCgZ6/DnOAlKY3+59j72FD9BsYtNaCRSmN0xIbU=
 github.com/ipfs/go-merkledag v0.2.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
+github.com/ipfs/go-merkledag v0.2.3 h1:aMdkK9G1hEeNvn3VXfiEMLY0iJnbiQQUHnM0HFJREsE=
+github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
 github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
 github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
 github.com/ipfs/go-metrics-prometheus v0.0.2 h1:9i2iljLg12S78OhC6UAiXi176xvQGiZaGVF1CUVdE+s=
@@ -320,8 +322,8 @@ github.com/ipfs/go-todocounter v0.0.1/go.mod h1:l5aErvQc8qKE2r7NDMjmq5UNAvuZy0rC
 github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vUpWyfz8=
 github.com/ipfs/go-unixfs v0.0.8 h1:AHahQ+gdNZd9BhKVLf8XP1EWeKa78eTzYgCygp7N/Pg=
 github.com/ipfs/go-unixfs v0.0.8/go.mod h1:cK2vDJ7L4YnWB6oLefpVNesgx0x/zPTRVDw6B4Y+03U=
-github.com/ipfs/go-unixfs v0.2.0 h1:mfdI8rgsEifWfhLECrH2WphHvslNoPbdvlmsJ05Fu0M=
-github.com/ipfs/go-unixfs v0.2.0/go.mod h1:sy/j20FKUxdUYOl6GZb3wsQ593C2xk5r1i8U7W+5iio=
+github.com/ipfs/go-unixfs v0.2.1 h1:g51t9ODICFZ3F51FPivm8dE7NzYcdAQNUL9wGP5AYa0=
+github.com/ipfs/go-unixfs v0.2.1/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k=
 github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E=
 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
 github.com/ipfs/hang-fds v0.0.1 h1:KGAxiGtJPT3THVRNT6yxgpdFPeX4ZemUjENOt6NlOn4=

pin/gc/gc.go

@@ -171,7 +171,7 @@ func Descendants(ctx context.Context, getLinks dag.GetLinks, set *cid.Set, roots
 
 	for _, c := range roots {
 		// Walk recursively walks the dag and adds the keys to the given set
-		err := dag.WalkParallel(ctx, verifyGetLinks, c, set.Visit)
+		err := dag.Walk(ctx, verifyGetLinks, c, set.Visit, dag.Concurrent())
 
 		if err != nil {
 			err = verboseCidError(err)

test/sharness/t0111-gateway-writeable.sh

@@ -54,17 +54,13 @@ test_expect_success "We can HTTP GET file just created" '
   test_cmp infile outfile
 '
 
-test_expect_success "HTTP PUT empty directory" '
-  URL="http://localhost:$port/ipfs/$HASH_EMPTY_DIR/" &&
-  echo "PUT $URL" &&
-  curl -svX PUT "$URL" 2>curl_putEmpty.out &&
-  cat curl_putEmpty.out &&
-  grep "Ipfs-Hash: $HASH_EMPTY_DIR" curl_putEmpty.out &&
-  grep "Location: /ipfs/$HASH_EMPTY_DIR" curl_putEmpty.out &&
-  grep "HTTP/1.1 201 Created" curl_putEmpty.out
+test_expect_success "We got the correct hash" '
+  ADD_HASH="/ipfs/$(ipfs add -q infile)" &&
+  test "x$ADD_HASH" = "x$HASH" || test_fsh echo "$ADD_HASH != $HASH"
 '
 
 test_expect_success "HTTP GET empty directory" '
+  URL="http://localhost:$port/ipfs/$HASH_EMPTY_DIR/" &&
   echo "GET $URL" &&
   curl -so outfile "$URL" 2>curl_getEmpty.out &&
   grep "Index of /ipfs/$HASH_EMPTY_DIR/" outfile
@@ -105,6 +101,32 @@ test_expect_success "We can HTTP GET file just updated" '
   test_cmp infile2 outfile2
 '
 
+test_expect_success "HTTP PUT to replace a directory" '
+  echo "$RANDOM" >infile3 &&
+  URL="http://localhost:$port/ipfs/$HASH/test" &&
+  echo "PUT $URL" &&
+  curl -svX PUT --data-binary @infile3 "$URL" 2>curl_putOverDirectory.out &&
+  grep "HTTP/1.1 201 Created" curl_putOverDirectory.out &&
+  LOCATION=$(grep Location curl_putOverDirectory.out) &&
+  HASH=$(expr "$LOCATION" : "< Location: /ipfs/\(.*\)/test")
+'
+
+test_expect_success "We can HTTP GET file just put over a directory" '
+  URL="http://localhost:$port/ipfs/$HASH/test" &&
+  echo "GET $URL" &&
+  curl -svo outfile3 "$URL" 2>curl_getOverDirectory.out &&
+  test_cmp infile3 outfile3
+'
+
+test_expect_success "HTTP PUT to /ipns fails" '
+  PEERID=`ipfs id --format="<id>"` &&
+  URL="http://localhost:$port/ipns/$PEERID/test.txt" &&
+  echo "PUT $URL" &&
+  curl -svX PUT --data-binary @infile1 "$URL" 2>curl_putIpns.out &&
+  grep "HTTP/1.1 400 Bad Request" curl_putIpns.out
+'
+
+
 test_kill_ipfs_daemon
 
 test_done