iterator technique for unixfs dir listing

License: MIT
Signed-off-by: Jeromy <why@ipfs.io>

mfs/dir.go

@@ -221,71 +221,59 @@ func (d *Directory) ListNames() ([]string, error) {
 	d.lock.Lock()
 	defer d.lock.Unlock()
 
-	names := make(map[string]struct{})
-	for n, _ := range d.childDirs {
-		names[n] = struct{}{}
-	}
-	for n, _ := range d.files {
-		names[n] = struct{}{}
-	}
-
-	links, err := d.dirbuilder.Links()
+	var out []string
+	err := d.dirbuilder.ForEachLink(func(l *node.Link) error {
+		out = append(out, l.Name)
+		return nil
+	})
 	if err != nil {
 		return nil, err
 	}
 
-	for _, l := range links {
-		names[l.Name] = struct{}{}
-	}
-
-	var out []string
-	for n, _ := range names {
-		out = append(out, n)
-	}
 	sort.Strings(out)
 
 	return out, nil
 }
 
 func (d *Directory) List() ([]NodeListing, error) {
-	d.lock.Lock()
-	defer d.lock.Unlock()
-
 	var out []NodeListing
+	err := d.ForEachEntry(context.TODO(), func(nl NodeListing) error {
+		out = append(out, nl)
+		return nil
+	})
+	return out, err
+}
 
-	links, err := d.dirbuilder.Links()
-	if err != nil {
-		return nil, err
-	}
-
-	for _, l := range links {
-		child := NodeListing{}
-		child.Name = l.Name
-
+func (d *Directory) ForEachEntry(ctx context.Context, f func(NodeListing) error) error {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+	return d.dirbuilder.ForEachLink(func(l *node.Link) error {
 		c, err := d.childUnsync(l.Name)
 		if err != nil {
-			return nil, err
+			return err
+		}
+
+		nd, err := c.GetNode()
+		if err != nil {
+			return err
+		}
+
+		child := NodeListing{
+			Name: l.Name,
+			Type: int(c.Type()),
+			Hash: nd.Cid().String(),
 		}
 
-		child.Type = int(c.Type())
 		if c, ok := c.(*File); ok {
 			size, err := c.Size()
 			if err != nil {
-				return nil, err
+				return err
 			}
 			child.Size = size
 		}
-		nd, err := c.GetNode()
-		if err != nil {
-			return nil, err
-		}
 
-		child.Hash = nd.Cid().String()
-
-		out = append(out, child)
-	}
-
-	return out, nil
+		return f(child)
+	})
 }
 
 func (d *Directory) Mkdir(name string) (*Directory, error) {
@@ -433,5 +421,5 @@ func (d *Directory) GetNode() (node.Node, error) {
 		return nil, err
 	}
 
-	return nd, err
+	return nd.Copy(), err
 }

unixfs/hamt/hamt.go

@@ -321,7 +321,15 @@ func (ds *HamtShard) getValue(ctx context.Context, hv *hashBits, key string, cb
 
 func (ds *HamtShard) EnumLinks() ([]*node.Link, error) {
 	var links []*node.Link
-	err := ds.walkTrie(func(sv *shardValue) error {
+	err := ds.ForEachLink(func(l *node.Link) error {
+		links = append(links, l)
+		return nil
+	})
+	return links, err
+}
+
+func (ds *HamtShard) ForEachLink(f func(*node.Link) error) error {
+	return ds.walkTrie(func(sv *shardValue) error {
 		lnk, err := node.MakeLink(sv.val)
 		if err != nil {
 			return err
@@ -329,14 +337,8 @@ func (ds *HamtShard) EnumLinks() ([]*node.Link, error) {
 
 		lnk.Name = sv.key
 
-		links = append(links, lnk)
-		return nil
+		return f(lnk)
 	})
-	if err != nil {
-		return nil, err
-	}
-
-	return links, nil
 }
 
 func (ds *HamtShard) walkTrie(cb func(*shardValue) error) error {

unixfs/io/dirbuilder.go

@@ -102,6 +102,19 @@ func (d *Directory) switchToSharding(ctx context.Context) error {
 	return nil
 }
 
+func (d *Directory) ForEachLink(f func(*node.Link) error) error {
+	if d.shard == nil {
+		for _, l := range d.dirnode.Links() {
+			if err := f(l); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+
+	return d.shard.ForEachLink(f)
+}
+
 func (d *Directory) Links() ([]*node.Link, error) {
 	if d.shard == nil {
 		return d.dirnode.Links(), nil