From 136ef70f00dcc54dc5842f6aa4c09a6d52ebd9ff Mon Sep 17 00:00:00 2001
From: Juan Batiz-Benet <juan@benet.ai>
Date: Thu, 8 Jan 2015 16:18:45 -0800
Subject: [PATCH] calc_test for picking link block size

---
 importer/calc_test.go              | 50 ++++++++++++++++++++++++++++++
 importer/importer.go               | 19 +++++++++++-
 test/sharness/t0040-add-and-cat.sh |  3 +-
 3 files changed, 69 insertions(+), 3 deletions(-)
 create mode 100644 importer/calc_test.go

diff --git a/importer/calc_test.go b/importer/calc_test.go
new file mode 100644
index 000000000..ef16fc85e
--- /dev/null
+++ b/importer/calc_test.go
@@ -0,0 +1,50 @@
+package importer
+
+import (
+	"math"
+	"testing"
+
+	humanize "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/dustin/go-humanize"
+)
+
+func TestCalculateSizes(t *testing.T) {
+
+	//  d := ((lbs/271) ^ layer) * dbs
+
+	increments := func(a, b int) []int {
+		ints := []int{}
+		for ; a <= b; a *= 2 {
+			ints = append(ints, a)
+		}
+		return ints
+	}
+
+	layers := 7
+	roughLinkSize := roughLinkSize // from importer pkg
+	dataBlockSizes := increments(1<<12, 1<<18)
+	linkBlockSizes := increments(1<<12, 1<<14)
+
+	t.Logf("rough link size:  %d", roughLinkSize)
+	t.Logf("data block sizes: %v", dataBlockSizes)
+	t.Logf("link block sizes: %v", linkBlockSizes)
+	for _, dbs := range dataBlockSizes {
+		t.Logf("")
+		t.Logf("with data block size: %d", dbs)
+		for _, lbs := range linkBlockSizes {
+			t.Logf("")
+			t.Logf("\twith data block size: %d", dbs)
+			t.Logf("\twith link block size: %d", lbs)
+
+			lpb := lbs / roughLinkSize
+			t.Logf("\tlinks per block: %d", lpb)
+
+			for l := 1; l < layers; l++ {
+				total := int(math.Pow(float64(lpb), float64(l))) * dbs
+				htotal := humanize.Bytes(uint64(total))
+				t.Logf("\t\t\tlayer %d: %s\t%d", l, htotal, total)
+			}
+
+		}
+	}
+
+}
diff --git a/importer/importer.go b/importer/importer.go
index eac532ff2..c84dfe3ab 100644
--- a/importer/importer.go
+++ b/importer/importer.go
@@ -20,7 +20,24 @@ var log = util.Logger("importer")
 // BlockSizeLimit specifies the maximum size an imported block can have.
 var BlockSizeLimit = 1048576 // 1 MB
 
-var DefaultLinksPerBlock = 8192
+// rough estimates on expected sizes
+var roughDataBlockSize = chunk.DefaultBlockSize
+var roughLinkBlockSize = 1 << 13 // 8KB
+var roughLinkSize = 258 + 8 + 5  // sha256 multihash + size + no name + protobuf framing
+
+// DefaultLinksPerBlock governs how the importer decides how many links there
+// will be per block. This calculation is based on expected distributions of:
+//  * the expected distribution of block sizes
+//  * the expected distribution of link sizes
+//  * desired access speed
+// For now, we use:
+//
+//   var roughLinkBlockSize = 1 << 13 // 8KB
+//   var roughLinkSize = 288          // sha256 + framing + name
+//   var DefaultLinksPerBlock = (roughLinkBlockSize / roughLinkSize)
+//
+// See calc_test.go
+var DefaultLinksPerBlock = (roughLinkBlockSize / roughLinkSize)
 
 // ErrSizeLimitExceeded signals that a block is larger than BlockSizeLimit.
 var ErrSizeLimitExceeded = fmt.Errorf("object size limit exceeded")
diff --git a/test/sharness/t0040-add-and-cat.sh b/test/sharness/t0040-add-and-cat.sh
index 15a1de1c8..86ca3dc9b 100755
--- a/test/sharness/t0040-add-and-cat.sh
+++ b/test/sharness/t0040-add-and-cat.sh
@@ -107,7 +107,6 @@ test_expect_success "'ipfs add bigfile' output looks good" '
 	echo "added $HASH mountdir/bigfile" >expected &&
 	test_cmp expected actual
 '
-
 test_expect_success "'ipfs cat' succeeds" '
 	ipfs cat $HASH >actual
 '
@@ -139,7 +138,7 @@ test_expect_success EXPENSIVE "ipfs add bigfile succeeds" '
 '
 
 test_expect_success EXPENSIVE "ipfs add bigfile output looks good" '
-	HASH="QmbprabK1ucRoPLPns2zKtjAqZrTANDhZMgmcx6sDKPK92" &&
+	HASH="QmSVxWkYfbJ3cowQUUgF4iF4CQd92vubxw7bs2aZAVRUD9" &&
 	echo "added $HASH mountdir/bigfile" >expected &&
 	test_cmp expected actual
 '