Use latest (1.20+) LevelDB (#326)

config/config.go

@@ -35,6 +35,7 @@ type LevelDBConfig struct {
 	WriteBufferSize int  `toml:"write_buffer_size"`
 	CacheSize       int  `toml:"cache_size"`
 	MaxOpenFiles    int  `toml:"max_open_files"`
+	MaxFileSize     int  `toml:"max_file_size"`
 }
 
 type RocksDBConfig struct {
@@ -244,6 +245,7 @@ func (cfg *LevelDBConfig) adjust() {
 	cfg.BlockSize = getDefault(4*KB, cfg.BlockSize)
 	cfg.WriteBufferSize = getDefault(4*MB, cfg.WriteBufferSize)
 	cfg.MaxOpenFiles = getDefault(1024, cfg.MaxOpenFiles)
+	cfg.MaxFileSize = getDefault(32*MB, cfg.MaxFileSize)
 }
 
 func (cfg *RocksDBConfig) adjust() {

config/config.toml

@@ -25,7 +25,7 @@ access_log = ""
 # Any write operations except flushall and replication will be disabled in slave mode.
 slaveof = ""
 
-# Readonly mode, slave server is always readonly even readonly = false  
+# Readonly mode, slave server is always readonly even readonly = false
 # for readonly mode, only replication and flushall can write
 readonly = false
 
@@ -35,16 +35,16 @@ readonly = false
 #   rocksdb
 #   goleveldb
 #   memory
-#   
+#
 db_name = "leveldb"
 
 # If not set, use data_dir/"db_name"_data
 db_path = ""
 
 # Sync commit to disk if possible
-#  0: no sync 
+#  0: no sync
-#  1: sync every second 
+#  1: sync every second
-#  2: sync every commit 
+#  2: sync every commit
 db_sync_commit = 0
 
 # enable replication or not
@@ -55,8 +55,8 @@ use_replication = false
 conn_read_buffer_size = 10240
 conn_write_buffer_size = 10240
 
-# if connection receives no data after n seconds, it may be dead, close 
+# if connection receives no data after n seconds, it may be dead, close
-# 0 to disable and not check 
+# 0 to disable and not check
 conn_keepalive_interval = 0
 
 # checking TTL (time to live) data every n seconds
@@ -70,13 +70,14 @@ block_size = 32768
 write_buffer_size = 67108864
 cache_size = 524288000
 max_open_files = 1024
+max_file_size = 33554432
 
 [rocksdb]
-# rocksdb has many many configurations, 
+# rocksdb has many many configurations,
 # we only list little now, but may add more later.
 # good luck!
 
-# 0:no, 1:snappy, 2:zlib, 3:bz2, 4:lz4, 5:lz4hc 
+# 0:no, 1:snappy, 2:zlib, 3:bz2, 4:lz4, 5:lz4hc
 compression = 0
 block_size = 65536
 write_buffer_size = 134217728
@@ -92,9 +93,9 @@ target_file_size_base = 67108864
 target_file_size_multiplier = 1
 max_bytes_for_level_base = 536870912
 max_bytes_for_level_multiplier = 8
-disable_auto_compactions = false 
+disable_auto_compactions = false
-disable_data_sync = false 
+disable_data_sync = false
-use_fsync = false 
+use_fsync = false
 background_theads = 16
 high_priority_background_threads = 1
 max_background_compactions = 15
@@ -114,11 +115,11 @@ nosync = true
 
 [replication]
 # Path to store replication information(write ahead log, commit log, etc.)
-# if not set, use data_dir/rpl 
+# if not set, use data_dir/rpl
 path = ""
 
-# If sync is true, the new log must be sent to some slaves, and then commit. 
+# If sync is true, the new log must be sent to some slaves, and then commit.
-# It will reduce performance but have better high availability. 
+# It will reduce performance but have better high availability.
 sync = false
 
 # If sync is true, wait at last wait_sync_time milliseconds for slave syncing this log
@@ -146,9 +147,9 @@ max_log_file_num = 0
 use_mmap = true
 
 # Sync log to disk if possible
-#  0: no sync 
+#  0: no sync
-#  1: sync every second 
+#  1: sync every second
-#  2: sync every commit 
+#  2: sync every commit
 sync_log = 0
 
 # Compress the log or not

store/leveldb/db.go

@@ -128,6 +128,8 @@ func (db *DB) initOptions(cfg *config.LevelDBConfig) {
 
 	opts.SetMaxOpenFiles(cfg.MaxOpenFiles)
 
+	opts.SetMaxFileSize(cfg.MaxFileSize)
+
 	db.opts = opts
 
 	db.readOpts = NewReadOptions()

store/leveldb/options.go

@@ -69,6 +69,10 @@ func (o *Options) SetMaxOpenFiles(n int) {
 	C.leveldb_options_set_max_open_files(o.Opt, C.int(n))
 }
 
+func (o *Options) SetMaxFileSize(n int) {
+	C.leveldb_options_set_max_file_size(o.Opt, C.size_t(n))
+}
+
 func (o *Options) SetBlockSize(s int) {
 	C.leveldb_options_set_block_size(o.Opt, C.size_t(s))
 }

tools/build_leveldb.sh

@@ -5,6 +5,8 @@ LEVELDB_DIR=/usr/local/leveldb
 
 ROOT_DIR=$(pwd)
 
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
 BUILD_DIR=/tmp/build_leveldb
 
 mkdir -p $BUILD_DIR
@@ -26,12 +28,14 @@ fi
 cd $BUILD_DIR
 
 if [ ! -f $LEVELDB_DIR/lib/libleveldb.a ]; then
-    (git clone https://github.com/siddontang/leveldb.git ; \
+    (git clone git@github.com:google/leveldb.git ; \
         cd ./leveldb && \
+        git checkout 47cb9e2a211e1d7157078ba7bab536beb29e56dc && \
+        patch -p0 < $SCRIPT_DIR/leveldb.patch
         echo "echo \"PLATFORM_CFLAGS+=-I$SNAPPY_DIR/include\" >> build_config.mk" >> build_detect_platform &&
         echo "echo \"PLATFORM_CXXFLAGS+=-I$SNAPPY_DIR/include\" >> build_config.mk" >> build_detect_platform &&
         echo "echo \"PLATFORM_LDFLAGS+=-L $SNAPPY_DIR/lib -lsnappy\" >> build_config.mk" >> build_detect_platform &&
-        make SNAPPY=1 && \
+        make HAVE_SNAPPY=1 && \
         make && \
         mkdir -p $LEVELDB_DIR/include/leveldb && \
         install include/leveldb/*.h $LEVELDB_DIR/include/leveldb && \

tools/leveldb.patch

@@ -0,0 +1,28 @@
+diff -Naur ./db/dbformat.h ../leveldb/db/dbformat.h
+--- ./db/dbformat.h	2017-11-03 18:04:26.000000000 -0400
++++ ../leveldb/db/dbformat.h	2017-11-23 13:11:30.167396736 -0500
+@@ -25,10 +25,10 @@
+ static const int kL0_CompactionTrigger = 4;
+
+ // Soft limit on number of level-0 files.  We slow down writes at this point.
+-static const int kL0_SlowdownWritesTrigger = 8;
++static const int kL0_SlowdownWritesTrigger = 16;
+
+ // Maximum number of level-0 files.  We stop writes at this point.
+-static const int kL0_StopWritesTrigger = 12;
++static const int kL0_StopWritesTrigger = 64;
+
+ // Maximum level to which a new compacted memtable is pushed if it
+ // does not create overlap.  We try to push to level 2 to avoid the
+diff -Naur ./db/version_set.cc ../leveldb/db/version_set.cc
+--- ./db/version_set.cc	2017-11-03 18:04:26.000000000 -0400
++++ ../leveldb/db/version_set.cc	2017-11-23 13:12:12.700870777 -0500
+@@ -27,7 +27,7 @@
+ // Maximum bytes of overlaps in grandparent (i.e., level+2) before we
+ // stop building a single file in a level->level+1 compaction.
+ static int64_t MaxGrandParentOverlapBytes(const Options* options) {
+-  return 10 * TargetFileSize(options);
++  return 20 * TargetFileSize(options);
+ }
+
+ // Maximum number of bytes in all compacted files.  We avoid expanding