Use latest (1.20+) LevelDB (#326)

This commit is contained in:
Maciej Lisiewski 2017-11-26 19:51:18 -05:00 committed by siddontang
parent 2ff56553d9
commit 81f50c08aa
6 changed files with 61 additions and 20 deletions

View File

@ -35,6 +35,7 @@ type LevelDBConfig struct {
WriteBufferSize int `toml:"write_buffer_size"`
CacheSize int `toml:"cache_size"`
MaxOpenFiles int `toml:"max_open_files"`
MaxFileSize int `toml:"max_file_size"`
}
type RocksDBConfig struct {
@ -244,6 +245,7 @@ func (cfg *LevelDBConfig) adjust() {
cfg.BlockSize = getDefault(4*KB, cfg.BlockSize)
cfg.WriteBufferSize = getDefault(4*MB, cfg.WriteBufferSize)
cfg.MaxOpenFiles = getDefault(1024, cfg.MaxOpenFiles)
cfg.MaxFileSize = getDefault(32*MB, cfg.MaxFileSize)
}
func (cfg *RocksDBConfig) adjust() {

View File

@ -25,7 +25,7 @@ access_log = ""
# Any write operations except flushall and replication will be disabled in slave mode.
slaveof = ""
# Readonly mode, slave server is always readonly even readonly = false
# Readonly mode, slave server is always readonly even readonly = false
# for readonly mode, only replication and flushall can write
readonly = false
@ -35,16 +35,16 @@ readonly = false
# rocksdb
# goleveldb
# memory
#
#
db_name = "leveldb"
# If not set, use data_dir/"db_name"_data
db_path = ""
# Sync commit to disk if possible
# 0: no sync
# 1: sync every second
# 2: sync every commit
# 0: no sync
# 1: sync every second
# 2: sync every commit
db_sync_commit = 0
# enable replication or not
@ -55,8 +55,8 @@ use_replication = false
conn_read_buffer_size = 10240
conn_write_buffer_size = 10240
# if connection receives no data after n seconds, it may be dead, close
# 0 to disable and not check
# if connection receives no data after n seconds, it may be dead, close
# 0 to disable and not check
conn_keepalive_interval = 0
# checking TTL (time to live) data every n seconds
@ -70,13 +70,14 @@ block_size = 32768
write_buffer_size = 67108864
cache_size = 524288000
max_open_files = 1024
max_file_size = 33554432
[rocksdb]
# rocksdb has many many configurations,
# rocksdb has many many configurations,
# we only list little now, but may add more later.
# good luck!
# 0:no, 1:snappy, 2:zlib, 3:bz2, 4:lz4, 5:lz4hc
# 0:no, 1:snappy, 2:zlib, 3:bz2, 4:lz4, 5:lz4hc
compression = 0
block_size = 65536
write_buffer_size = 134217728
@ -92,9 +93,9 @@ target_file_size_base = 67108864
target_file_size_multiplier = 1
max_bytes_for_level_base = 536870912
max_bytes_for_level_multiplier = 8
disable_auto_compactions = false
disable_data_sync = false
use_fsync = false
disable_auto_compactions = false
disable_data_sync = false
use_fsync = false
background_theads = 16
high_priority_background_threads = 1
max_background_compactions = 15
@ -114,11 +115,11 @@ nosync = true
[replication]
# Path to store replication information(write ahead log, commit log, etc.)
# if not set, use data_dir/rpl
# if not set, use data_dir/rpl
path = ""
# If sync is true, the new log must be sent to some slaves, and then commit.
# It will reduce performance but have better high availability.
# If sync is true, the new log must be sent to some slaves, and then commit.
# It will reduce performance but have better high availability.
sync = false
# If sync is true, wait at last wait_sync_time milliseconds for slave syncing this log
@ -146,9 +147,9 @@ max_log_file_num = 0
use_mmap = true
# Sync log to disk if possible
# 0: no sync
# 1: sync every second
# 2: sync every commit
# 0: no sync
# 1: sync every second
# 2: sync every commit
sync_log = 0
# Compress the log or not

View File

@ -128,6 +128,8 @@ func (db *DB) initOptions(cfg *config.LevelDBConfig) {
opts.SetMaxOpenFiles(cfg.MaxOpenFiles)
opts.SetMaxFileSize(cfg.MaxFileSize)
db.opts = opts
db.readOpts = NewReadOptions()

View File

@ -69,6 +69,10 @@ func (o *Options) SetMaxOpenFiles(n int) {
C.leveldb_options_set_max_open_files(o.Opt, C.int(n))
}
func (o *Options) SetMaxFileSize(n int) {
C.leveldb_options_set_max_file_size(o.Opt, C.size_t(n))
}
func (o *Options) SetBlockSize(s int) {
C.leveldb_options_set_block_size(o.Opt, C.size_t(s))
}

View File

@ -5,6 +5,8 @@ LEVELDB_DIR=/usr/local/leveldb
ROOT_DIR=$(pwd)
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
BUILD_DIR=/tmp/build_leveldb
mkdir -p $BUILD_DIR
@ -26,12 +28,14 @@ fi
cd $BUILD_DIR
if [ ! -f $LEVELDB_DIR/lib/libleveldb.a ]; then
(git clone https://github.com/siddontang/leveldb.git ; \
(git clone git@github.com:google/leveldb.git ; \
cd ./leveldb && \
git checkout 47cb9e2a211e1d7157078ba7bab536beb29e56dc && \
patch -p0 < $SCRIPT_DIR/leveldb.patch
echo "echo \"PLATFORM_CFLAGS+=-I$SNAPPY_DIR/include\" >> build_config.mk" >> build_detect_platform &&
echo "echo \"PLATFORM_CXXFLAGS+=-I$SNAPPY_DIR/include\" >> build_config.mk" >> build_detect_platform &&
echo "echo \"PLATFORM_LDFLAGS+=-L $SNAPPY_DIR/lib -lsnappy\" >> build_config.mk" >> build_detect_platform &&
make SNAPPY=1 && \
make HAVE_SNAPPY=1 && \
make && \
mkdir -p $LEVELDB_DIR/include/leveldb && \
install include/leveldb/*.h $LEVELDB_DIR/include/leveldb && \

28
tools/leveldb.patch Normal file
View File

@ -0,0 +1,28 @@
diff -Naur ./db/dbformat.h ../leveldb/db/dbformat.h
--- ./db/dbformat.h 2017-11-03 18:04:26.000000000 -0400
+++ ../leveldb/db/dbformat.h 2017-11-23 13:11:30.167396736 -0500
@@ -25,10 +25,10 @@
static const int kL0_CompactionTrigger = 4;
// Soft limit on number of level-0 files. We slow down writes at this point.
-static const int kL0_SlowdownWritesTrigger = 8;
+static const int kL0_SlowdownWritesTrigger = 16;
// Maximum number of level-0 files. We stop writes at this point.
-static const int kL0_StopWritesTrigger = 12;
+static const int kL0_StopWritesTrigger = 64;
// Maximum level to which a new compacted memtable is pushed if it
// does not create overlap. We try to push to level 2 to avoid the
diff -Naur ./db/version_set.cc ../leveldb/db/version_set.cc
--- ./db/version_set.cc 2017-11-03 18:04:26.000000000 -0400
+++ ../leveldb/db/version_set.cc 2017-11-23 13:12:12.700870777 -0500
@@ -27,7 +27,7 @@
// Maximum bytes of overlaps in grandparent (i.e., level+2) before we
// stop building a single file in a level->level+1 compaction.
static int64_t MaxGrandParentOverlapBytes(const Options* options) {
- return 10 * TargetFileSize(options);
+ return 20 * TargetFileSize(options);
}
// Maximum number of bytes in all compacted files. We avoid expanding