Use latest (1.20+) LevelDB (#326)

config/config.go

@@ -35,6 +35,7 @@ type LevelDBConfig struct {
 	WriteBufferSize int  `toml:"write_buffer_size"`
 	CacheSize       int  `toml:"cache_size"`
 	MaxOpenFiles    int  `toml:"max_open_files"`
+	MaxFileSize     int  `toml:"max_file_size"`
 }
 
 type RocksDBConfig struct {
@@ -244,6 +245,7 @@ func (cfg *LevelDBConfig) adjust() {
 	cfg.BlockSize = getDefault(4*KB, cfg.BlockSize)
 	cfg.WriteBufferSize = getDefault(4*MB, cfg.WriteBufferSize)
 	cfg.MaxOpenFiles = getDefault(1024, cfg.MaxOpenFiles)
+	cfg.MaxFileSize = getDefault(32*MB, cfg.MaxFileSize)
 }
 
 func (cfg *RocksDBConfig) adjust() {

config/config.toml

@@ -70,6 +70,7 @@ block_size = 32768
 write_buffer_size = 67108864
 cache_size = 524288000
 max_open_files = 1024
+max_file_size = 33554432
 
 [rocksdb]
 # rocksdb has many many configurations,

store/leveldb/db.go

@@ -128,6 +128,8 @@ func (db *DB) initOptions(cfg *config.LevelDBConfig) {
 
 	opts.SetMaxOpenFiles(cfg.MaxOpenFiles)
 
+	opts.SetMaxFileSize(cfg.MaxFileSize)
+
 	db.opts = opts
 
 	db.readOpts = NewReadOptions()

store/leveldb/options.go

@@ -69,6 +69,10 @@ func (o *Options) SetMaxOpenFiles(n int) {
 	C.leveldb_options_set_max_open_files(o.Opt, C.int(n))
 }
 
+func (o *Options) SetMaxFileSize(n int) {
+	C.leveldb_options_set_max_file_size(o.Opt, C.size_t(n))
+}
+
 func (o *Options) SetBlockSize(s int) {
 	C.leveldb_options_set_block_size(o.Opt, C.size_t(s))
 }

tools/build_leveldb.sh

@@ -5,6 +5,8 @@ LEVELDB_DIR=/usr/local/leveldb
 
 ROOT_DIR=$(pwd)
 
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
 BUILD_DIR=/tmp/build_leveldb
 
 mkdir -p $BUILD_DIR
@@ -26,12 +28,14 @@ fi
 cd $BUILD_DIR
 
 if [ ! -f $LEVELDB_DIR/lib/libleveldb.a ]; then
-    (git clone https://github.com/siddontang/leveldb.git ; \
+    (git clone git@github.com:google/leveldb.git ; \
         cd ./leveldb && \
+        git checkout 47cb9e2a211e1d7157078ba7bab536beb29e56dc && \
+        patch -p0 < $SCRIPT_DIR/leveldb.patch
         echo "echo \"PLATFORM_CFLAGS+=-I$SNAPPY_DIR/include\" >> build_config.mk" >> build_detect_platform &&
         echo "echo \"PLATFORM_CXXFLAGS+=-I$SNAPPY_DIR/include\" >> build_config.mk" >> build_detect_platform &&
         echo "echo \"PLATFORM_LDFLAGS+=-L $SNAPPY_DIR/lib -lsnappy\" >> build_config.mk" >> build_detect_platform &&
-        make SNAPPY=1 && \
+        make HAVE_SNAPPY=1 && \
         make && \
         mkdir -p $LEVELDB_DIR/include/leveldb && \
         install include/leveldb/*.h $LEVELDB_DIR/include/leveldb && \

tools/leveldb.patch

@@ -0,0 +1,28 @@
+diff -Naur ./db/dbformat.h ../leveldb/db/dbformat.h
+--- ./db/dbformat.h	2017-11-03 18:04:26.000000000 -0400
++++ ../leveldb/db/dbformat.h	2017-11-23 13:11:30.167396736 -0500
+@@ -25,10 +25,10 @@
+ static const int kL0_CompactionTrigger = 4;
+
+ // Soft limit on number of level-0 files.  We slow down writes at this point.
+-static const int kL0_SlowdownWritesTrigger = 8;
++static const int kL0_SlowdownWritesTrigger = 16;
+
+ // Maximum number of level-0 files.  We stop writes at this point.
+-static const int kL0_StopWritesTrigger = 12;
++static const int kL0_StopWritesTrigger = 64;
+
+ // Maximum level to which a new compacted memtable is pushed if it
+ // does not create overlap.  We try to push to level 2 to avoid the
+diff -Naur ./db/version_set.cc ../leveldb/db/version_set.cc
+--- ./db/version_set.cc	2017-11-03 18:04:26.000000000 -0400
++++ ../leveldb/db/version_set.cc	2017-11-23 13:12:12.700870777 -0500
+@@ -27,7 +27,7 @@
+ // Maximum bytes of overlaps in grandparent (i.e., level+2) before we
+ // stop building a single file in a level->level+1 compaction.
+ static int64_t MaxGrandParentOverlapBytes(const Options* options) {
+-  return 10 * TargetFileSize(options);
++  return 20 * TargetFileSize(options);
+ }
+
+ // Maximum number of bytes in all compacted files.  We avoid expanding