calc_test for picking link block size

importer/calc_test.go

@@ -0,0 +1,50 @@
+package importer
+
+import (
+	"math"
+	"testing"
+
+	humanize "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/dustin/go-humanize"
+)
+
+func TestCalculateSizes(t *testing.T) {
+
+	//  d := ((lbs/271) ^ layer) * dbs
+
+	increments := func(a, b int) []int {
+		ints := []int{}
+		for ; a <= b; a *= 2 {
+			ints = append(ints, a)
+		}
+		return ints
+	}
+
+	layers := 7
+	roughLinkSize := roughLinkSize // from importer pkg
+	dataBlockSizes := increments(1<<12, 1<<18)
+	linkBlockSizes := increments(1<<12, 1<<14)
+
+	t.Logf("rough link size:  %d", roughLinkSize)
+	t.Logf("data block sizes: %v", dataBlockSizes)
+	t.Logf("link block sizes: %v", linkBlockSizes)
+	for _, dbs := range dataBlockSizes {
+		t.Logf("")
+		t.Logf("with data block size: %d", dbs)
+		for _, lbs := range linkBlockSizes {
+			t.Logf("")
+			t.Logf("\twith data block size: %d", dbs)
+			t.Logf("\twith link block size: %d", lbs)
+
+			lpb := lbs / roughLinkSize
+			t.Logf("\tlinks per block: %d", lpb)
+
+			for l := 1; l < layers; l++ {
+				total := int(math.Pow(float64(lpb), float64(l))) * dbs
+				htotal := humanize.Bytes(uint64(total))
+				t.Logf("\t\t\tlayer %d: %s\t%d", l, htotal, total)
+			}
+
+		}
+	}
+
+}

importer/importer.go

@@ -20,7 +20,24 @@ var log = util.Logger("importer")
 // BlockSizeLimit specifies the maximum size an imported block can have.
 var BlockSizeLimit = 1048576 // 1 MB
 
-var DefaultLinksPerBlock = 8192
+// rough estimates on expected sizes
+var roughDataBlockSize = chunk.DefaultBlockSize
+var roughLinkBlockSize = 1 << 13 // 8KB
+var roughLinkSize = 258 + 8 + 5  // sha256 multihash + size + no name + protobuf framing
+
+// DefaultLinksPerBlock governs how the importer decides how many links there
+// will be per block. This calculation is based on expected distributions of:
+//  * the expected distribution of block sizes
+//  * the expected distribution of link sizes
+//  * desired access speed
+// For now, we use:
+//
+//   var roughLinkBlockSize = 1 << 13 // 8KB
+//   var roughLinkSize = 288          // sha256 + framing + name
+//   var DefaultLinksPerBlock = (roughLinkBlockSize / roughLinkSize)
+//
+// See calc_test.go
+var DefaultLinksPerBlock = (roughLinkBlockSize / roughLinkSize)
 
 // ErrSizeLimitExceeded signals that a block is larger than BlockSizeLimit.
 var ErrSizeLimitExceeded = fmt.Errorf("object size limit exceeded")

test/sharness/t0040-add-and-cat.sh

@@ -107,7 +107,6 @@ test_expect_success "'ipfs add bigfile' output looks good" '
 	echo "added $HASH mountdir/bigfile" >expected &&
 	test_cmp expected actual
 '
-
 test_expect_success "'ipfs cat' succeeds" '
 	ipfs cat $HASH >actual
 '
@@ -139,7 +138,7 @@ test_expect_success EXPENSIVE "ipfs add bigfile succeeds" '
 '
 
 test_expect_success EXPENSIVE "ipfs add bigfile output looks good" '
-	HASH="QmbprabK1ucRoPLPns2zKtjAqZrTANDhZMgmcx6sDKPK92" &&
+	HASH="QmSVxWkYfbJ3cowQUUgF4iF4CQd92vubxw7bs2aZAVRUD9" &&
 	echo "added $HASH mountdir/bigfile" >expected &&
 	test_cmp expected actual
 '