added random-files tool for testing

License: MIT
Signed-off-by: Juan Batiz-Benet <juan@benet.ai>

Godeps/Godeps.json

@@ -186,6 +186,10 @@
 			"ImportPath": "github.com/jbenet/go-random",
 			"Rev": "cd535bd25356746b9b1e824871dda7da932460e2"
 		},
+		{
+			"ImportPath": "github.com/jbenet/go-random-files",
+			"Rev": "737479700b40b4b50e914e963ce8d9d44603e3c8"
+		},
 		{
 			"ImportPath": "github.com/jbenet/go-reuseport",
 			"Rev": "48959f1fad204b6cf2c0e8d086ef69f03f2de961"

Godeps/_workspace/src/github.com/jbenet/go-random-files/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Juan Batiz-Benet
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

Godeps/_workspace/src/github.com/jbenet/go-random-files/Makefile

@@ -0,0 +1,5 @@
+build:
+	cd random-files && go build
+
+install:
+	cd random-files && go install

Godeps/_workspace/src/github.com/jbenet/go-random-files/README.md

@@ -0,0 +1,108 @@
+# go-random-files - create random fs hierarchies
+
+Useful for testing filesystems.
+
+
+```
+# library
+go get -u github.com/jbenet/go-random-files
+
+# binary
+go get -u github.com/jbenet/go-random-files/random-files
+```
+
+### Library godoc: https://godoc.org/github.com/jbenet/go-random-files
+
+# random-files - create random fs hierarchies
+
+See more about the binary at: github.com/jbenet/go-random-files/tree/master/random-files
+
+Useful for testing filesystems.
+
+## Install
+
+```
+go get -u github.com/jbenet/go-random-files/random-files
+```
+
+## Usage
+
+```sh
+> random-files --help
+usage: random-files [options] <path>...
+Write a random filesystem hierarchy to each <path>
+
+Options:
+  -alphabet="easy": alphabet for filenames {easy, hard}
+  -depth=2: fanout depth - how deep the hierarchy goes
+  -dirs=5: fanout dirs - number of dirs per dir (or max)
+  -files=10: fanout files - number of files per dir (or max
+  -filesize=4096: filesize - how big to make each file (or max)
+  -q=false: quiet output
+  -random-crypto=false: use cryptographic randomness for files
+  -random-fanout=false: randomize fanout numbers
+  -random-size=true: randomize filesize
+  -seed=0: random seed - 0 for current time
+```
+
+### Examples
+
+```sh
+> random-files --depth=2 --files=3 foo
+foo/h20uo3jrpihb
+foo/x6tef1
+foo/jh0c2vdci
+foo/fden012m368
+foo/fden012m368/p6n0chy4kg
+foo/fden012m368/h92_
+foo/fden012m368/kvjiya98p3
+foo/e_i6hwav1tb
+foo/e_i6hwav1tb/oj0-a
+foo/e_i6hwav1tb/1-pfgvim
+foo/e_i6hwav1tb/s_unf
+foo/bgvy8x-_hsm
+foo/bgvy8x-_hsm/98zcoz-9ng
+foo/bgvy8x-_hsm/j0see3qv
+foo/bgvy8x-_hsm/qntuf0r
+foo/6zjkw3ejm2awwt
+foo/6zjkw3ejm2awwt/iba52dh1lhnewh
+foo/6zjkw3ejm2awwt/n1bwcv5zpe
+foo/6zjkw3ejm2awwt/o8k89cc
+foo/efp_6
+foo/efp_6/qfap2
+foo/efp_6/v_kl_wlefsaa
+foo/efp_6/r7sdbph
+```
+
+It made:
+
+```
+> tree foo
+foo
+├── 6zjkw3ejm2awwt
+│   ├── iba52dh1lhnewh
+│   ├── n1bwcv5zpe
+│   └── o8k89cc
+├── bgvy8x-_hsm
+│   ├── 98zcoz-9ng
+│   ├── j0see3qv
+│   └── qntuf0r
+├── e_i6hwav1tb
+│   ├── 1-pfgvim
+│   ├── oj0-a
+│   └── s_unf
+├── efp_6
+│   ├── qfap2
+│   ├── r7sdbph
+│   └── v_kl_wlefsaa
+├── fden012m368
+│   ├── h92_
+│   ├── kvjiya98p3
+│   └── p6n0chy4kg
+├── h20uo3jrpihb
+├── jh0c2vdci
+└── x6tef1
+
+5 directories, 18 files
+```
+

Godeps/_workspace/src/github.com/jbenet/go-random-files/lib.go

@@ -0,0 +1,110 @@
+package randomfiles
+
+import (
+	"fmt"
+	"io"
+	"math/rand"
+	"os"
+	"path"
+)
+
+type Options struct {
+	Out    io.Writer // output progress
+	Source io.Reader // randomness source
+
+	FileSize int    // the size per file.
+	Alphabet []rune // for filenames
+
+	FanoutDepth int // how deep the hierarchy goes
+	FanoutFiles int // how many files per dir
+	FanoutDirs  int // how many dirs per dir
+
+	RandomSeed   int64 // use a random seed. if 0, use a random seed
+	RandomSize   bool  // randomize file sizes
+	RandomFanout bool  // randomize fanout numbers
+}
+
+func WriteRandomFiles(root string, depth int, opts *Options) error {
+
+	numfiles := opts.FanoutFiles
+	if opts.RandomFanout {
+		numfiles = rand.Intn(numfiles) + 1
+	}
+
+	for i := 0; i < numfiles; i++ {
+		if err := WriteRandomFile(root, opts); err != nil {
+			return err
+		}
+	}
+
+	if depth+1 <= opts.FanoutDepth {
+		numdirs := opts.FanoutDirs
+		if opts.RandomFanout {
+			numdirs = rand.Intn(numdirs) + 1
+		}
+
+		for i := 0; i < numdirs; i++ {
+			if err := WriteRandomDir(root, depth+1, opts); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+var FilenameSize = 16
+var RunesEasy = []rune("abcdefghijklmnopqrstuvwxyz01234567890-_")
+var RunesHard = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890!@#$%^&*()-_+= ;.,<>'\"[]{}() ")
+
+func RandomFilename(length int, alphabet []rune) string {
+	b := make([]rune, length)
+	for i := range b {
+		b[i] = alphabet[rand.Intn(len(alphabet))]
+	}
+	return string(b)
+}
+
+func WriteRandomFile(root string, opts *Options) error {
+	filesize := int64(opts.FileSize)
+	if opts.RandomSize {
+		filesize = rand.Int63n(filesize) + 1
+	}
+
+	n := rand.Intn(FilenameSize-4) + 4
+	name := RandomFilename(n, opts.Alphabet)
+	filepath := path.Join(root, name)
+	f, err := os.Create(filepath)
+	if err != nil {
+		return err
+	}
+
+	if _, err := io.CopyN(f, opts.Source, filesize); err != nil {
+		return err
+	}
+
+	if opts.Out != nil {
+		fmt.Fprintln(opts.Out, filepath)
+	}
+
+	return f.Close()
+}
+
+func WriteRandomDir(root string, depth int, opts *Options) error {
+	if depth > opts.FanoutDepth {
+		return nil
+	}
+
+	n := rand.Intn(FilenameSize-4) + 4
+	name := RandomFilename(n, opts.Alphabet)
+	root = path.Join(root, name)
+	if err := os.MkdirAll(root, 0755); err != nil {
+		return err
+	}
+
+	if opts.Out != nil {
+		fmt.Fprintln(opts.Out, root)
+	}
+
+	return WriteRandomFiles(root, depth, opts)
+}

Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files/.gitignore

@@ -0,0 +1 @@
+random-files

Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files/README.md

@@ -0,0 +1,90 @@
+# random-files - create random fs hierarchies
+
+random-files creates random fs hierarchies. Useful for testing filesystems.
+
+## Install
+
+```
+go get -u github.com/jbenet/go-random-files/random-files
+```
+
+## Usage
+
+```sh
+> random-files --help
+usage: random-files [options] <path>...
+Write a random filesystem hierarchy to each <path>
+
+Options:
+  -alphabet="easy": alphabet for filenames {easy, hard}
+  -depth=2: fanout depth - how deep the hierarchy goes
+  -dirs=5: fanout dirs - number of dirs per dir (or max)
+  -files=10: fanout files - number of files per dir (or max
+  -filesize=4096: filesize - how big to make each file (or max)
+  -q=false: quiet output
+  -random-crypto=false: use cryptographic randomness for files
+  -random-fanout=false: randomize fanout numbers
+  -random-size=true: randomize filesize
+  -seed=0: random seed - 0 for current time
+```
+
+## Examples
+
+```sh
+> random-files --depth=2 --files=3 foo
+foo/h20uo3jrpihb
+foo/x6tef1
+foo/jh0c2vdci
+foo/fden012m368
+foo/fden012m368/p6n0chy4kg
+foo/fden012m368/h92_
+foo/fden012m368/kvjiya98p3
+foo/e_i6hwav1tb
+foo/e_i6hwav1tb/oj0-a
+foo/e_i6hwav1tb/1-pfgvim
+foo/e_i6hwav1tb/s_unf
+foo/bgvy8x-_hsm
+foo/bgvy8x-_hsm/98zcoz-9ng
+foo/bgvy8x-_hsm/j0see3qv
+foo/bgvy8x-_hsm/qntuf0r
+foo/6zjkw3ejm2awwt
+foo/6zjkw3ejm2awwt/iba52dh1lhnewh
+foo/6zjkw3ejm2awwt/n1bwcv5zpe
+foo/6zjkw3ejm2awwt/o8k89cc
+foo/efp_6
+foo/efp_6/qfap2
+foo/efp_6/v_kl_wlefsaa
+foo/efp_6/r7sdbph
+```
+
+It made:
+
+```
+> tree foo
+foo
+├── 6zjkw3ejm2awwt
+│   ├── iba52dh1lhnewh
+│   ├── n1bwcv5zpe
+│   └── o8k89cc
+├── bgvy8x-_hsm
+│   ├── 98zcoz-9ng
+│   ├── j0see3qv
+│   └── qntuf0r
+├── e_i6hwav1tb
+│   ├── 1-pfgvim
+│   ├── oj0-a
+│   └── s_unf
+├── efp_6
+│   ├── qfap2
+│   ├── r7sdbph
+│   └── v_kl_wlefsaa
+├── fden012m368
+│   ├── h92_
+│   ├── kvjiya98p3
+│   └── p6n0chy4kg
+├── h20uo3jrpihb
+├── jh0c2vdci
+└── x6tef1
+
+5 directories, 18 files
+```

Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files/main.go

@@ -0,0 +1,119 @@
+package main
+
+import (
+	crand "crypto/rand"
+	"errors"
+	"flag"
+	"fmt"
+	"math/rand"
+	"os"
+	"time"
+
+	randomfiles "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-random-files"
+	ringreader "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-random-files/ringreader"
+)
+
+var usage = `usage: %s [options] <path>...
+Write a random filesystem hierarchy to each <path>
+
+Options:
+`
+
+// flags
+var opts randomfiles.Options
+var quiet bool
+var alphabet string
+var paths []string
+var cryptorand bool
+
+func init() {
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, usage, os.Args[0])
+		flag.PrintDefaults()
+	}
+
+	flag.BoolVar(&quiet, "q", false, "quiet output")
+	flag.BoolVar(&cryptorand, "random-crypto", false, "use cryptographic randomness for files")
+	flag.StringVar(&alphabet, "alphabet", "easy", "alphabet for filenames {easy, hard}")
+	flag.IntVar(&opts.FileSize, "filesize", 4096, "filesize - how big to make each file (or max)")
+
+	flag.IntVar(&opts.FanoutDepth, "depth", 2, "fanout depth - how deep the hierarchy goes")
+	flag.IntVar(&opts.FanoutDirs, "dirs", 5, "fanout dirs - number of dirs per dir (or max)")
+	flag.IntVar(&opts.FanoutFiles, "files", 10, "fanout files - number of files per dir (or max")
+
+	flag.Int64Var(&opts.RandomSeed, "seed", 0, "random seed - 0 for current time")
+	flag.BoolVar(&opts.RandomFanout, "random-fanout", false, "randomize fanout numbers")
+	flag.BoolVar(&opts.RandomSize, "random-size", true, "randomize filesize")
+}
+
+func parseArgs() error {
+	flag.Parse()
+
+	switch alphabet {
+	case "easy":
+		opts.Alphabet = randomfiles.RunesEasy
+	case "hard":
+		opts.Alphabet = randomfiles.RunesHard
+	default:
+		return errors.New("alphabet must be one of: easy, hard")
+	}
+
+	paths = flag.Args()
+	if len(paths) < 1 {
+		flag.Usage()
+		os.Exit(0)
+	}
+
+	if !quiet {
+		opts.Out = os.Stdout
+	}
+
+	switch opts.RandomSeed {
+	case 0:
+		rand.Seed(time.Now().UnixNano())
+	default:
+		rand.Seed(opts.RandomSeed)
+	}
+
+	// prepare randomn source.
+	if cryptorand {
+		opts.Source = crand.Reader
+	} else {
+		// if not crypto, we don't need a lot of random
+		// data. we just need to sample from a sequence.
+		s := 16777216 // 16MB
+		r, err := ringreader.NewReader(s)
+		if err != nil {
+			return err
+		}
+		opts.Source = r
+	}
+
+	return nil
+}
+
+func run() error {
+	if err := parseArgs(); err != nil {
+		return err
+	}
+
+	for _, root := range paths {
+		if err := os.MkdirAll(root, 0755); err != nil {
+			return err
+		}
+
+		err := randomfiles.WriteRandomFiles(root, 1, &opts)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Fprintln(os.Stderr, "error:", err)
+		os.Exit(1)
+	}
+}

Godeps/_workspace/src/github.com/jbenet/go-random-files/ringreader/ringreader.go

@@ -0,0 +1,40 @@
+package ringreader
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand"
+
+	random "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-random"
+)
+
+type Reader struct {
+	Buf []byte
+}
+
+func NewReader(bufsize int) (*Reader, error) {
+	buf := bytes.NewBuffer(nil)
+	err := random.WritePseudoRandomBytes(int64(bufsize), buf, rand.Int63())
+	return &Reader{Buf: buf.Bytes()}, err
+}
+
+func (r *Reader) Read(buf []byte) (n int, err error) {
+	ibufl := len(r.Buf)
+	left := len(buf)
+	copied := 0
+
+	for copied < left {
+		pos1 := rand.Intn(len(r.Buf))
+		pos2 := pos1 + left
+		if pos2 > ibufl {
+			pos2 = ibufl
+		}
+		copied += copy(buf[copied:], r.Buf[pos1:pos2])
+	}
+
+	if copied != left {
+		err := fmt.Errorf("copied a different ammount: %d != %d", copied, left)
+		panic(err.Error())
+	}
+	return copied, nil
+}

Godeps/_workspace/src/github.com/jbenet/go-random-files/ringreader/ringreader_test.go

@@ -0,0 +1,22 @@
+package ringreader
+
+import (
+	"testing"
+)
+
+func TestRingReader(t *testing.T) {
+	r, _ := NewReader(256)
+	t.Log("buffer:", r.Buf)
+
+	for i := 1; i < 1048576; i = i * 2 {
+		buf := make([]byte, i)
+		n, err := r.Read(buf)
+		if err != nil {
+			t.Error(err)
+		}
+		if n != len(buf) {
+			t.Error("did not read %d bytes", n)
+		}
+		t.Log("read:", buf)
+	}
+}

test/Makefile

@@ -3,6 +3,7 @@ BINS = bin/random bin/multihash bin/ipfs bin/pollEndpoint bin/iptb bin/go-sleep
 IPFS_ROOT = ../
 IPFS_CMD = ../cmd/ipfs
 RANDOM_SRC = ../Godeps/_workspace/src/github.com/jbenet/go-random
+RANDOM_FILES_SRC = ../Godeps/_workspace/src/github.com/jbenet/go-random-files
 MULTIHASH_SRC = ../Godeps/_workspace/src/github.com/jbenet/go-multihash
 IPTB_SRC = ../Godeps/_workspace/src/github.com/whyrusleeping/iptb
 POLLENDPOINT_SRC= ../thirdparty/pollEndpoint
@@ -26,6 +27,10 @@ bin/random: $(call find_go_files, $(RANDOM_SRC)) IPFS-BUILD-OPTIONS
 	@echo "*** installing $@ ***"
 	go build $(GOFLAGS) -o bin/random $(RANDOM_SRC)/random
 
+bin/random-files:
+	@echo "*** installing $@ ***"
+	go build $(GOFLAGS) -o bin/random-files $(RANDOM_FILES_SRC)/random-files
+
 bin/multihash: $(call find_go_files, $(MULTIHASH_SRC)) IPFS-BUILD-OPTIONS
 	@echo "*** installing $@ ***"
 	go build $(GOFLAGS) -o bin/multihash $(MULTIHASH_SRC)/multihash

test/sharness/Makefile

@@ -8,7 +8,7 @@
 
 T = $(sort $(wildcard t[0-9][0-9][0-9][0-9]-*.sh))
 BINS = bin/random bin/multihash bin/ipfs bin/pollEndpoint \
-	   bin/iptb bin/go-sleep
+	   bin/iptb bin/go-sleep bin/random-files
 SHARNESS = lib/sharness/sharness.sh
 IPFS_ROOT = ../..
 

test/sharness/lib/random-dep.go

@@ -5,4 +5,5 @@ package randomdep
 
 import (
 	_ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-random"
+	_ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-random-files"
 )