Merge pull request #553 from jbenet/serial-file

Serial File Opening

commands/cli/parse.go

@@ -5,12 +5,11 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	fp "path"
 	"runtime"
-	"sort"
 	"strings"
 
 	cmds "github.com/jbenet/go-ipfs/commands"
+	files "github.com/jbenet/go-ipfs/commands/files"
 	u "github.com/jbenet/go-ipfs/util"
 )
 
@@ -65,7 +64,7 @@ func Parse(input []string, stdin *os.File, root *cmds.Command) (cmds.Request, *c
 	}
 	req.SetArguments(stringArgs)
 
-	file := &cmds.SliceFile{"", fileArgs}
+	file := &files.SliceFile{"", fileArgs}
 	req.SetFiles(file)
 
 	err = cmd.CheckArguments(req)
@@ -139,7 +138,7 @@ func parseOptions(input []string) (map[string]interface{}, []string, error) {
 	return opts, args, nil
 }
 
-func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursive bool) ([]string, []cmds.File, error) {
+func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursive bool) ([]string, []files.File, error) {
 	// ignore stdin on Windows
 	if runtime.GOOS == "windows" {
 		stdin = nil
@@ -176,7 +175,7 @@ func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursi
 	}
 
 	stringArgs := make([]string, 0, numInputs)
-	fileArgs := make([]cmds.File, 0, numInputs)
+	fileArgs := make([]files.File, 0, numInputs)
 
 	argDefIndex := 0 // the index of the current argument definition
 	for i := 0; i < numInputs; i++ {
@@ -264,7 +263,7 @@ func appendStdinAsString(args []string, stdin *os.File) ([]string, *os.File, err
 	return append(args, strings.Split(input, "\n")...), nil, nil
 }
 
-func appendFile(args []cmds.File, inputs []string, argDef *cmds.Argument, recursive bool) ([]cmds.File, []string, error) {
+func appendFile(args []files.File, inputs []string, argDef *cmds.Argument, recursive bool) ([]files.File, []string, error) {
 	path := inputs[0]
 
 	file, err := os.Open(path)
@@ -290,7 +289,7 @@ func appendFile(args []cmds.File, inputs []string, argDef *cmds.Argument, recurs
 		}
 	}
 
-	arg, err := openPath(file, path)
+	arg, err := files.NewSerialFile(path, file)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -298,51 +297,11 @@ func appendFile(args []cmds.File, inputs []string, argDef *cmds.Argument, recurs
 	return append(args, arg), inputs[1:], nil
 }
 
-func appendStdinAsFile(args []cmds.File, stdin *os.File) ([]cmds.File, *os.File) {
-	arg := &cmds.ReaderFile{"", stdin}
+func appendStdinAsFile(args []files.File, stdin *os.File) ([]files.File, *os.File) {
+	arg := &files.ReaderFile{"", stdin}
 	return append(args, arg), nil
 }
 
-// recursively get file or directory contents as a cmds.File
-func openPath(file *os.File, path string) (cmds.File, error) {
-	stat, err := file.Stat()
-	if err != nil {
-		return nil, err
-	}
-
-	// for non-directories, return a ReaderFile
-	if !stat.IsDir() {
-		return &cmds.ReaderFile{path, file}, nil
-	}
-
-	// for directories, recursively iterate though children then return as a SliceFile
-	contents, err := file.Readdir(0)
-	if err != nil {
-		return nil, err
-	}
-
-	// make sure contents are sorted so -- repeatably -- we get the same inputs.
-	sort.Sort(sortFIByName(contents))
-
-	files := make([]cmds.File, 0, len(contents))
-	for _, child := range contents {
-		childPath := fp.Join(path, child.Name())
-		childFile, err := os.Open(childPath)
-		if err != nil {
-			return nil, err
-		}
-
-		f, err := openPath(childFile, childPath)
-		if err != nil {
-			return nil, err
-		}
-
-		files = append(files, f)
-	}
-
-	return &cmds.SliceFile{path, files}, nil
-}
-
 // isTerminal returns true if stdin is a Stdin pipe (e.g. `cat file | ipfs`),
 // and false otherwise (e.g. nothing is being piped in, so stdin is
 // coming from the terminal)
@@ -355,9 +314,3 @@ func isTerminal(stdin *os.File) (bool, error) {
 	// if stdin is a CharDevice, return true
 	return ((stat.Mode() & os.ModeCharDevice) != 0), nil
 }
-
-type sortFIByName []os.FileInfo
-
-func (es sortFIByName) Len() int           { return len(es) }
-func (es sortFIByName) Swap(i, j int)      { es[i], es[j] = es[j], es[i] }
-func (es sortFIByName) Less(i, j int) bool { return es[i].Name() < es[j].Name() }

commands/files/file.go

+package files
+
+import (
+	"errors"
+	"io"
+)
+
+var (
+	ErrNotDirectory = errors.New("Couln't call NextFile(), this isn't a directory")
+	ErrNotReader    = errors.New("This file is a directory, can't use Reader functions")
+)
+
+// File is an interface that provides functionality for handling files/directories
+// as values that can be supplied to commands. For directories, child files are
+// accessed serially by calling `NextFile()`.
+type File interface {
+	// Files implement ReadCloser, but can only be read from or closed if they are not directories
+	io.ReadCloser
+
+	// FileName returns a full filename path associated with this file
+	FileName() string
+
+	// IsDirectory returns true if the File is a directory (and therefore supports calling `NextFile`)
+	// and false if the File is a normal file (and therefor supports calling `Read` and `Close`)
+	IsDirectory() bool
+
+	// NextFile returns the next child file available (if the File is a directory).
+	// It will return (nil, io.EOF) if no more files are available.
+	// If the file is a regular file (not a directory), NextFile will return a non-nil error.
+	NextFile() (File, error)
+}

commands/file_test.go → commands/files/file_test.go

-package commands
+package files
 
 import (
 	"io"
+	"io/ioutil"
 	"mime/multipart"
 	"strings"
 	"testing"
@@ -10,9 +11,9 @@ import (
 func TestSliceFiles(t *testing.T) {
 	name := "testname"
 	files := []File{
-		&ReaderFile{"file.txt", strings.NewReader("Some text!\n")},
-		&ReaderFile{"beep.txt", strings.NewReader("beep")},
-		&ReaderFile{"boop.txt", strings.NewReader("boop")},
+		&ReaderFile{"file.txt", ioutil.NopCloser(strings.NewReader("Some text!\n"))},
+		&ReaderFile{"beep.txt", ioutil.NopCloser(strings.NewReader("beep"))},
+		&ReaderFile{"boop.txt", ioutil.NopCloser(strings.NewReader("boop"))},
 	}
 	buf := make([]byte, 20)
 
@@ -54,7 +55,7 @@ func TestSliceFiles(t *testing.T) {
 
 func TestReaderFiles(t *testing.T) {
 	message := "beep boop"
-	rf := &ReaderFile{"file.txt", strings.NewReader(message)}
+	rf := &ReaderFile{"file.txt", ioutil.NopCloser(strings.NewReader(message))}
 	buf := make([]byte, len(message))
 
 	if rf.IsDirectory() {

commands/file.go → commands/files/multipartfile.go

-package commands
+package files
 
 import (
-	"errors"
-	"io"
 	"mime"
 	"mime/multipart"
 	"net/http"
@@ -15,31 +13,6 @@ const (
 	contentTypeHeader = "Content-Type"
 )
 
-var (
-	ErrNotDirectory = errors.New("Couln't call NextFile(), this isn't a directory")
-	ErrNotReader    = errors.New("This file is a directory, can't use Reader functions")
-)
-
-// File is an interface that provides functionality for handling files/directories
-// as values that can be supplied to commands. For directories, child files are
-// accessed serially by calling `NextFile()`.
-type File interface {
-	// Files implement ReadCloser, but can only be read from or closed if they are not directories
-	io.ReadCloser
-
-	// FileName returns a full filename path associated with this file
-	FileName() string
-
-	// IsDirectory returns true if the File is a directory (and therefore supports calling `NextFile`)
-	// and false if the File is a normal file (and therefor supports calling `Read` and `Close`)
-	IsDirectory() bool
-
-	// NextFile returns the next child file available (if the File is a directory).
-	// It will return (nil, io.EOF) if no more files are available.
-	// If the file is a regular file (not a directory), NextFile will return a non-nil error.
-	NextFile() (File, error)
-}
-
 // MultipartFile implements File, and is created from a `multipart.Part`.
 // It can be either a directory or file (checked by calling `IsDirectory()`).
 type MultipartFile struct {
@@ -110,67 +83,3 @@ func (f *MultipartFile) Close() error {
 	}
 	return f.Part.Close()
 }
-
-// SliceFile implements File, and provides simple directory handling.
-// It contains children files, and is created from a `[]File`.
-// SliceFiles are always directories, and can't be read from or closed.
-type SliceFile struct {
-	Filename string
-	Files    []File
-}
-
-func (f *SliceFile) IsDirectory() bool {
-	return true
-}
-
-func (f *SliceFile) NextFile() (File, error) {
-	if len(f.Files) == 0 {
-		return nil, io.EOF
-	}
-	file := f.Files[0]
-	f.Files = f.Files[1:]
-	return file, nil
-}
-
-func (f *SliceFile) FileName() string {
-	return f.Filename
-}
-
-func (f *SliceFile) Read(p []byte) (int, error) {
-	return 0, ErrNotReader
-}
-
-func (f *SliceFile) Close() error {
-	return ErrNotReader
-}
-
-// ReaderFile is a implementation of File created from an `io.Reader`.
-// ReaderFiles are never directories, and can be read from and closed.
-type ReaderFile struct {
-	Filename string
-	Reader   io.Reader
-}
-
-func (f *ReaderFile) IsDirectory() bool {
-	return false
-}
-
-func (f *ReaderFile) NextFile() (File, error) {
-	return nil, ErrNotDirectory
-}
-
-func (f *ReaderFile) FileName() string {
-	return f.Filename
-}
-
-func (f *ReaderFile) Read(p []byte) (int, error) {
-	return f.Reader.Read(p)
-}
-
-func (f *ReaderFile) Close() error {
-	closer, ok := f.Reader.(io.Closer)
-	if !ok {
-		return nil
-	}
-	return closer.Close()
-}

commands/files/readerfile.go

+package files
+
+import "io"
+
+// ReaderFile is a implementation of File created from an `io.Reader`.
+// ReaderFiles are never directories, and can be read from and closed.
+type ReaderFile struct {
+	Filename string
+	Reader   io.ReadCloser
+}
+
+func (f *ReaderFile) IsDirectory() bool {
+	return false
+}
+
+func (f *ReaderFile) NextFile() (File, error) {
+	return nil, ErrNotDirectory
+}
+
+func (f *ReaderFile) FileName() string {
+	return f.Filename
+}
+
+func (f *ReaderFile) Read(p []byte) (int, error) {
+	return f.Reader.Read(p)
+}
+
+func (f *ReaderFile) Close() error {
+	return f.Reader.Close()
+}

commands/files/serialfile.go

+package files
+
+import (
+	"io"
+	"os"
+	fp "path"
+	"sort"
+	"syscall"
+)
+
+type sortFIByName []os.FileInfo
+
+func (es sortFIByName) Len() int           { return len(es) }
+func (es sortFIByName) Swap(i, j int)      { es[i], es[j] = es[j], es[i] }
+func (es sortFIByName) Less(i, j int) bool { return es[i].Name() < es[j].Name() }
+
+// serialFile implements File, and reads from a path on the OS filesystem.
+// No more than one file will be opened at a time (directories will advance
+// to the next file when NextFile() is called).
+type serialFile struct {
+	path    string
+	files   []os.FileInfo
+	current *os.File
+}
+
+func NewSerialFile(path string, file *os.File) (File, error) {
+	stat, err := file.Stat()
+	if err != nil {
+		return nil, err
+	}
+
+	return newSerialFile(path, file, stat)
+}
+
+func newSerialFile(path string, file *os.File, stat os.FileInfo) (File, error) {
+	// for non-directories, return a ReaderFile
+	if !stat.IsDir() {
+		return &ReaderFile{path, file}, nil
+	}
+
+	// for directories, stat all of the contents first, so we know what files to
+	// open when NextFile() is called
+	contents, err := file.Readdir(0)
+	if err != nil {
+		return nil, err
+	}
+
+	// we no longer need our root directory file (we already statted the contents),
+	// so close it
+	err = file.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	// make sure contents are sorted so -- repeatably -- we get the same inputs.
+	sort.Sort(sortFIByName(contents))
+
+	return &serialFile{path, contents, nil}, nil
+}
+
+func (f *serialFile) IsDirectory() bool {
+	// non-directories get created as a ReaderFile, so serialFiles should only
+	// represent directories
+	return true
+}
+
+func (f *serialFile) NextFile() (File, error) {
+	// if a file was opened previously, close it
+	err := f.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	// if there aren't any files left in the root directory, we're done
+	if len(f.files) == 0 {
+		return nil, io.EOF
+	}
+
+	stat := f.files[0]
+	f.files = f.files[1:]
+
+	// open the next file
+	filePath := fp.Join(f.path, stat.Name())
+	file, err := os.Open(filePath)
+	if err != nil {
+		return nil, err
+	}
+	f.current = file
+
+	// recursively call the constructor on the next file
+	// if it's a regular file, we will open it as a ReaderFile
+	// if it's a directory, files in it will be opened serially
+	return newSerialFile(filePath, file, stat)
+}
+
+func (f *serialFile) FileName() string {
+	return f.path
+}
+
+func (f *serialFile) Read(p []byte) (int, error) {
+	return 0, ErrNotReader
+}
+
+func (f *serialFile) Close() error {
+	// close the current file if there is one
+	if f.current != nil {
+		err := f.current.Close()
+		// ignore EINVAL error, the file might have already been closed
+		if err != nil && err != syscall.EINVAL {
+			return err
+		}
+	}
+
+	return nil
+}

commands/files/slicefile.go

+package files
+
+import "io"
+
+// SliceFile implements File, and provides simple directory handling.
+// It contains children files, and is created from a `[]File`.
+// SliceFiles are always directories, and can't be read from or closed.
+type SliceFile struct {
+	Filename string
+	Files    []File
+}
+
+func (f *SliceFile) IsDirectory() bool {
+	return true
+}
+
+func (f *SliceFile) NextFile() (File, error) {
+	if len(f.Files) == 0 {
+		return nil, io.EOF
+	}
+	file := f.Files[0]
+	f.Files = f.Files[1:]
+	return file, nil
+}
+
+func (f *SliceFile) FileName() string {
+	return f.Filename
+}
+
+func (f *SliceFile) Read(p []byte) (int, error) {
+	return 0, ErrNotReader
+}
+
+func (f *SliceFile) Close() error {
+	return ErrNotReader
+}

commands/http/multifilereader.go

@@ -8,7 +8,7 @@ import (
 	"net/textproto"
 	"sync"
 
-	cmds "github.com/jbenet/go-ipfs/commands"
+	files "github.com/jbenet/go-ipfs/commands/files"
 )
 
 // MultiFileReader reads from a `commands.File` (which can be a directory of files
@@ -16,7 +16,7 @@ import (
 type MultiFileReader struct {
 	io.Reader
 
-	files       cmds.File
+	files       files.File
 	currentFile io.Reader
 	buf         bytes.Buffer
 	mpWriter    *multipart.Writer
@@ -31,7 +31,7 @@ type MultiFileReader struct {
 // NewMultiFileReader constructs a MultiFileReader. `file` can be any `commands.File`.
 // If `form` is set to true, the multipart data will have a Content-Type of 'multipart/form-data',
 // if `form` is false, the Content-Type will be 'multipart/mixed'.
-func NewMultiFileReader(file cmds.File, form bool) *MultiFileReader {
+func NewMultiFileReader(file files.File, form bool) *MultiFileReader {
 	mfr := &MultiFileReader{
 		files: file,
 		form:  form,

commands/http/multifilereader_test.go

@@ -2,24 +2,25 @@ package http
 
 import (
 	"io"
+	"io/ioutil"
 	"mime/multipart"
 	"strings"
 	"testing"
 
-	cmds "github.com/jbenet/go-ipfs/commands"
+	files "github.com/jbenet/go-ipfs/commands/files"
 )
 
 func TestOutput(t *testing.T) {
 	text := "Some text! :)"
-	files := []cmds.File{
-		&cmds.ReaderFile{"file.txt", strings.NewReader(text)},
-		&cmds.SliceFile{"boop", []cmds.File{
-			&cmds.ReaderFile{"boop/a.txt", strings.NewReader("bleep")},
-			&cmds.ReaderFile{"boop/b.txt", strings.NewReader("bloop")},
+	fileset := []files.File{
+		&files.ReaderFile{"file.txt", ioutil.NopCloser(strings.NewReader(text))},
+		&files.SliceFile{"boop", []files.File{
+			&files.ReaderFile{"boop/a.txt", ioutil.NopCloser(strings.NewReader("bleep"))},
+			&files.ReaderFile{"boop/b.txt", ioutil.NopCloser(strings.NewReader("bloop"))},
 		}},
-		&cmds.ReaderFile{"beep.txt", strings.NewReader("beep")},
+		&files.ReaderFile{"beep.txt", ioutil.NopCloser(strings.NewReader("beep"))},
 	}
-	sf := &cmds.SliceFile{"", files}
+	sf := &files.SliceFile{"", fileset}
 	buf := make([]byte, 20)
 
 	// testing output by reading it with the go stdlib "mime/multipart" Reader
@@ -30,7 +31,7 @@ func TestOutput(t *testing.T) {
 	if part == nil || err != nil {
 		t.Error("Expected non-nil part, nil error")
 	}
-	mpf, err := cmds.NewFileFromPart(part)
+	mpf, err := files.NewFileFromPart(part)
 	if mpf == nil || err != nil {
 		t.Error("Expected non-nil MultipartFile, nil error")
 	}
@@ -51,7 +52,7 @@ func TestOutput(t *testing.T) {
 	if part == nil || err != nil {
 		t.Error("Expected non-nil part, nil error")
 	}
-	mpf, err = cmds.NewFileFromPart(part)
+	mpf, err = files.NewFileFromPart(part)
 	if mpf == nil || err != nil {
 		t.Error("Expected non-nil MultipartFile, nil error")
 	}
@@ -93,7 +94,7 @@ func TestOutput(t *testing.T) {
 	if part == nil || err != nil {
 		t.Error("Expected non-nil part, nil error")
 	}
-	mpf, err = cmds.NewFileFromPart(part)
+	mpf, err = files.NewFileFromPart(part)
 	if mpf == nil || err != nil {
 		t.Error("Expected non-nil MultipartFile, nil error")
 	}

commands/http/parse.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	cmds "github.com/jbenet/go-ipfs/commands"
+	files "github.com/jbenet/go-ipfs/commands/files"
 )
 
 // Parse parses the data in a http.Request and returns a command Request object
@@ -94,9 +95,9 @@ func Parse(r *http.Request, root *cmds.Command) (cmds.Request, error) {
 	contentType := r.Header.Get(contentTypeHeader)
 	mediatype, _, _ := mime.ParseMediaType(contentType)
 
-	var f *cmds.MultipartFile
+	var f *files.MultipartFile
 	if mediatype == "multipart/form-data" {
-		f = &cmds.MultipartFile{Mediatype: mediatype}
+		f = &files.MultipartFile{Mediatype: mediatype}
 		f.Reader, err = r.MultipartReader()
 		if err != nil {
 			return nil, err

commands/request.go

@@ -8,6 +8,7 @@ import (
 
 	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
 
+	"github.com/jbenet/go-ipfs/commands/files"
 	"github.com/jbenet/go-ipfs/core"
 	"github.com/jbenet/go-ipfs/repo/config"
 	u "github.com/jbenet/go-ipfs/util"
@@ -71,8 +72,8 @@ type Request interface {
 	SetOptions(opts map[string]interface{}) error
 	Arguments() []string
 	SetArguments([]string)
-	Files() File
-	SetFiles(File)
+	Files() files.File
+	SetFiles(files.File)
 	Context() *Context
 	SetContext(Context)
 	Command() *Command
@@ -84,7 +85,7 @@ type request struct {
 	path       []string
 	options    optMap
 	arguments  []string
-	files      File
+	files      files.File
 	cmd        *Command
 	ctx        Context
 	optionDefs map[string]Option
@@ -159,11 +160,11 @@ func (r *request) SetArguments(args []string) {
 	r.arguments = args
 }
 
-func (r *request) Files() File {
+func (r *request) Files() files.File {
 	return r.files
 }
 
-func (r *request) SetFiles(f File) {
+func (r *request) SetFiles(f files.File) {
 	r.files = f
 }
 
@@ -259,7 +260,7 @@ func NewEmptyRequest() (Request, error) {
 
 // NewRequest returns a request initialized with given arguments
 // An non-nil error will be returned if the provided option values are invalid
-func NewRequest(path []string, opts optMap, args []string, file File, cmd *Command, optDefs map[string]Option) (Request, error) {
+func NewRequest(path []string, opts optMap, args []string, file files.File, cmd *Command, optDefs map[string]Option) (Request, error) {
 	if path == nil {
 		path = make([]string, 0)
 	}

core/commands/add.go

@@ -8,6 +8,7 @@ import (
 	"path"
 
 	cmds "github.com/jbenet/go-ipfs/commands"
+	files "github.com/jbenet/go-ipfs/commands/files"
 	core "github.com/jbenet/go-ipfs/core"
 	importer "github.com/jbenet/go-ipfs/importer"
 	"github.com/jbenet/go-ipfs/importer/chunk"
@@ -138,7 +139,7 @@ func addNode(n *core.IpfsNode, node *dag.Node) error {
 	return nil
 }
 
-func addFile(n *core.IpfsNode, file cmds.File, out chan interface{}) (*dag.Node, error) {
+func addFile(n *core.IpfsNode, file files.File, out chan interface{}) (*dag.Node, error) {
 	if file.IsDirectory() {
 		return addDir(n, file, out)
 	}
@@ -155,7 +156,7 @@ func addFile(n *core.IpfsNode, file cmds.File, out chan interface{}) (*dag.Node,
 	return dns[len(dns)-1], nil // last dag node is the file.
 }
 
-func addDir(n *core.IpfsNode, dir cmds.File, out chan interface{}) (*dag.Node, error) {
+func addDir(n *core.IpfsNode, dir files.File, out chan interface{}) (*dag.Node, error) {
 	log.Infof("adding directory: %s", dir.FileName())
 
 	tree := &dag.Node{Data: ft.FolderPBData()}