From 81f50c08aa7c500429575e585a98e89063c8f22f Mon Sep 17 00:00:00 2001 From: Maciej Lisiewski Date: Sun, 26 Nov 2017 19:51:18 -0500 Subject: [PATCH] Use latest (1.20+) LevelDB (#326) --- config/config.go | 2 ++ config/config.toml | 37 +++++++++++++++++++------------------ store/leveldb/db.go | 2 ++ store/leveldb/options.go | 4 ++++ tools/build_leveldb.sh | 8 ++++++-- tools/leveldb.patch | 28 ++++++++++++++++++++++++++++ 6 files changed, 61 insertions(+), 20 deletions(-) create mode 100644 tools/leveldb.patch diff --git a/config/config.go b/config/config.go index 620c2ae..f8aa639 100644 --- a/config/config.go +++ b/config/config.go @@ -35,6 +35,7 @@ type LevelDBConfig struct { WriteBufferSize int `toml:"write_buffer_size"` CacheSize int `toml:"cache_size"` MaxOpenFiles int `toml:"max_open_files"` + MaxFileSize int `toml:"max_file_size"` } type RocksDBConfig struct { @@ -244,6 +245,7 @@ func (cfg *LevelDBConfig) adjust() { cfg.BlockSize = getDefault(4*KB, cfg.BlockSize) cfg.WriteBufferSize = getDefault(4*MB, cfg.WriteBufferSize) cfg.MaxOpenFiles = getDefault(1024, cfg.MaxOpenFiles) + cfg.MaxFileSize = getDefault(32*MB, cfg.MaxFileSize) } func (cfg *RocksDBConfig) adjust() { diff --git a/config/config.toml b/config/config.toml index 86d51e1..a75409c 100644 --- a/config/config.toml +++ b/config/config.toml @@ -25,7 +25,7 @@ access_log = "" # Any write operations except flushall and replication will be disabled in slave mode. slaveof = "" -# Readonly mode, slave server is always readonly even readonly = false +# Readonly mode, slave server is always readonly even readonly = false # for readonly mode, only replication and flushall can write readonly = false @@ -35,16 +35,16 @@ readonly = false # rocksdb # goleveldb # memory -# +# db_name = "leveldb" # If not set, use data_dir/"db_name"_data db_path = "" # Sync commit to disk if possible -# 0: no sync -# 1: sync every second -# 2: sync every commit +# 0: no sync +# 1: sync every second +# 2: sync every commit db_sync_commit = 0 # enable replication or not @@ -55,8 +55,8 @@ use_replication = false conn_read_buffer_size = 10240 conn_write_buffer_size = 10240 -# if connection receives no data after n seconds, it may be dead, close -# 0 to disable and not check +# if connection receives no data after n seconds, it may be dead, close +# 0 to disable and not check conn_keepalive_interval = 0 # checking TTL (time to live) data every n seconds @@ -70,13 +70,14 @@ block_size = 32768 write_buffer_size = 67108864 cache_size = 524288000 max_open_files = 1024 +max_file_size = 33554432 [rocksdb] -# rocksdb has many many configurations, +# rocksdb has many many configurations, # we only list little now, but may add more later. # good luck! -# 0:no, 1:snappy, 2:zlib, 3:bz2, 4:lz4, 5:lz4hc +# 0:no, 1:snappy, 2:zlib, 3:bz2, 4:lz4, 5:lz4hc compression = 0 block_size = 65536 write_buffer_size = 134217728 @@ -92,9 +93,9 @@ target_file_size_base = 67108864 target_file_size_multiplier = 1 max_bytes_for_level_base = 536870912 max_bytes_for_level_multiplier = 8 -disable_auto_compactions = false -disable_data_sync = false -use_fsync = false +disable_auto_compactions = false +disable_data_sync = false +use_fsync = false background_theads = 16 high_priority_background_threads = 1 max_background_compactions = 15 @@ -114,11 +115,11 @@ nosync = true [replication] # Path to store replication information(write ahead log, commit log, etc.) -# if not set, use data_dir/rpl +# if not set, use data_dir/rpl path = "" -# If sync is true, the new log must be sent to some slaves, and then commit. -# It will reduce performance but have better high availability. +# If sync is true, the new log must be sent to some slaves, and then commit. +# It will reduce performance but have better high availability. sync = false # If sync is true, wait at last wait_sync_time milliseconds for slave syncing this log @@ -146,9 +147,9 @@ max_log_file_num = 0 use_mmap = true # Sync log to disk if possible -# 0: no sync -# 1: sync every second -# 2: sync every commit +# 0: no sync +# 1: sync every second +# 2: sync every commit sync_log = 0 # Compress the log or not diff --git a/store/leveldb/db.go b/store/leveldb/db.go index c9cf83f..7f1ee67 100644 --- a/store/leveldb/db.go +++ b/store/leveldb/db.go @@ -128,6 +128,8 @@ func (db *DB) initOptions(cfg *config.LevelDBConfig) { opts.SetMaxOpenFiles(cfg.MaxOpenFiles) + opts.SetMaxFileSize(cfg.MaxFileSize) + db.opts = opts db.readOpts = NewReadOptions() diff --git a/store/leveldb/options.go b/store/leveldb/options.go index bd3ad16..68733bb 100644 --- a/store/leveldb/options.go +++ b/store/leveldb/options.go @@ -69,6 +69,10 @@ func (o *Options) SetMaxOpenFiles(n int) { C.leveldb_options_set_max_open_files(o.Opt, C.int(n)) } +func (o *Options) SetMaxFileSize(n int) { + C.leveldb_options_set_max_file_size(o.Opt, C.size_t(n)) +} + func (o *Options) SetBlockSize(s int) { C.leveldb_options_set_block_size(o.Opt, C.size_t(s)) } diff --git a/tools/build_leveldb.sh b/tools/build_leveldb.sh index dbf9979..c458c4e 100644 --- a/tools/build_leveldb.sh +++ b/tools/build_leveldb.sh @@ -5,6 +5,8 @@ LEVELDB_DIR=/usr/local/leveldb ROOT_DIR=$(pwd) +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + BUILD_DIR=/tmp/build_leveldb mkdir -p $BUILD_DIR @@ -26,12 +28,14 @@ fi cd $BUILD_DIR if [ ! -f $LEVELDB_DIR/lib/libleveldb.a ]; then - (git clone https://github.com/siddontang/leveldb.git ; \ + (git clone git@github.com:google/leveldb.git ; \ cd ./leveldb && \ + git checkout 47cb9e2a211e1d7157078ba7bab536beb29e56dc && \ + patch -p0 < $SCRIPT_DIR/leveldb.patch echo "echo \"PLATFORM_CFLAGS+=-I$SNAPPY_DIR/include\" >> build_config.mk" >> build_detect_platform && echo "echo \"PLATFORM_CXXFLAGS+=-I$SNAPPY_DIR/include\" >> build_config.mk" >> build_detect_platform && echo "echo \"PLATFORM_LDFLAGS+=-L $SNAPPY_DIR/lib -lsnappy\" >> build_config.mk" >> build_detect_platform && - make SNAPPY=1 && \ + make HAVE_SNAPPY=1 && \ make && \ mkdir -p $LEVELDB_DIR/include/leveldb && \ install include/leveldb/*.h $LEVELDB_DIR/include/leveldb && \ diff --git a/tools/leveldb.patch b/tools/leveldb.patch new file mode 100644 index 0000000..f810e0c --- /dev/null +++ b/tools/leveldb.patch @@ -0,0 +1,28 @@ +diff -Naur ./db/dbformat.h ../leveldb/db/dbformat.h +--- ./db/dbformat.h 2017-11-03 18:04:26.000000000 -0400 ++++ ../leveldb/db/dbformat.h 2017-11-23 13:11:30.167396736 -0500 +@@ -25,10 +25,10 @@ + static const int kL0_CompactionTrigger = 4; + + // Soft limit on number of level-0 files. We slow down writes at this point. +-static const int kL0_SlowdownWritesTrigger = 8; ++static const int kL0_SlowdownWritesTrigger = 16; + + // Maximum number of level-0 files. We stop writes at this point. +-static const int kL0_StopWritesTrigger = 12; ++static const int kL0_StopWritesTrigger = 64; + + // Maximum level to which a new compacted memtable is pushed if it + // does not create overlap. We try to push to level 2 to avoid the +diff -Naur ./db/version_set.cc ../leveldb/db/version_set.cc +--- ./db/version_set.cc 2017-11-03 18:04:26.000000000 -0400 ++++ ../leveldb/db/version_set.cc 2017-11-23 13:12:12.700870777 -0500 +@@ -27,7 +27,7 @@ + // Maximum bytes of overlaps in grandparent (i.e., level+2) before we + // stop building a single file in a level->level+1 compaction. + static int64_t MaxGrandParentOverlapBytes(const Options* options) { +- return 10 * TargetFileSize(options); ++ return 20 * TargetFileSize(options); + } + + // Maximum number of bytes in all compacted files. We avoid expanding \ No newline at end of file