Reland: limit write concurrency of leveldb to max open file handles limit

chunks/leveldb_store.go

@@ -22,9 +22,10 @@ func toChunkKey(r ref.Ref) []byte {
 }
 
 type LevelDBStore struct {
-	db       *leveldb.DB
+	db                   *leveldb.DB
-	mu       *sync.Mutex
+	mu                   *sync.Mutex
-	putCount int // for testing
+	putCount             int // for testing
+	concurrentWriteLimit chan struct{}
 }
 
 func NewLevelDBStore(dir string, maxFileHandles int) *LevelDBStore {
@@ -34,10 +35,15 @@ func NewLevelDBStore(dir string, maxFileHandles int) *LevelDBStore {
 		Compression:            opt.NoCompression,
 		Filter:                 filter.NewBloomFilter(10), // 10 bits/key
 		OpenFilesCacheCapacity: maxFileHandles,
-		WriteBuffer:            1 << 24, // 16MiB
+		WriteBuffer:            1 << 24, // 16MiB,
 	})
 	d.Chk.NoError(err)
-	return &LevelDBStore{db, &sync.Mutex{}, 0}
+	return &LevelDBStore{
+		db,
+		&sync.Mutex{},
+		0,
+		make(chan struct{}, maxFileHandles),
+	}
 }
 
 func (l *LevelDBStore) Root() ref.Ref {
@@ -86,9 +92,11 @@ func (l *LevelDBStore) Put(c Chunk) {
 		return
 	}
 
+	l.concurrentWriteLimit <- struct{}{}
 	err := l.db.Put(toChunkKey(c.Ref()), c.Data(), nil)
 	d.Chk.NoError(err)
 	l.putCount += 1
+	<-l.concurrentWriteLimit
 }
 
 func (l *LevelDBStore) Close() error {

chunks/leveldb_store_test.go

@@ -21,7 +21,7 @@ func (suite *LevelDBStoreTestSuite) SetupTest() {
 	var err error
 	suite.dir, err = ioutil.TempDir(os.TempDir(), "")
 	suite.NoError(err)
-	store := NewLevelDBStore(suite.dir, 0)
+	store := NewLevelDBStore(suite.dir, 24)
 	suite.putCountFn = func() int {
 		return store.putCount
 	}

http/http_server.go

@@ -23,16 +23,15 @@ const (
 )
 
 type httpServer struct {
-	cs         chunks.ChunkStore
+	cs    chunks.ChunkStore
-	port       int
+	port  int
-	l          *net.Listener
+	l     *net.Listener
-	conns      map[net.Conn]http.ConnState
+	conns map[net.Conn]http.ConnState
-	writeLimit chan struct{}
 }
 
 func NewHttpServer(cs chunks.ChunkStore, port int) *httpServer {
 	return &httpServer{
-		cs, port, nil, map[net.Conn]http.ConnState{}, make(chan struct{}, maxConcurrentPuts),
+		cs, port, nil, map[net.Conn]http.ConnState{},
 	}
 }