From 3cbbbb8f85de247bf13c002f8698ee88c019a80a Mon Sep 17 00:00:00 2001 From: siddontang Date: Sun, 12 Mar 2017 21:21:49 +0800 Subject: [PATCH] update rocksdb 5.1+ (#282) --- README.md | 4 ++-- config/config.go | 4 ---- store/rocksdb/batch.go | 5 ++++- store/rocksdb/db.go | 2 -- store/rocksdb/options.go | 10 +--------- store/store_test.go | 16 ++++++++-------- store/writebatch.go | 2 -- 7 files changed, 15 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index eedfd1d..8e1b7c2 100644 --- a/README.md +++ b/README.md @@ -57,9 +57,9 @@ Create a workspace and checkout ledisdb source + Set `LEVELDB_DIR` and `SNAPPY_DIR` to the actual install path in dev.sh. + `make clean && make` -## RocksDB support +## RocksDB support -+ [Install rocksdb](https://github.com/facebook/rocksdb/blob/master/INSTALL.md)(`make shared_lib`) and snappy first. ++ [Install rocksdb(5.1+)](https://github.com/facebook/rocksdb/blob/master/INSTALL.md)(`make shared_lib`) and snappy first. LedisDB has not yet supplied a simple script to install. diff --git a/config/config.go b/config/config.go index b6c2b82..f0ac896 100644 --- a/config/config.go +++ b/config/config.go @@ -52,11 +52,9 @@ type RocksDBConfig struct { MaxBytesForLevelBase int `toml:"max_bytes_for_level_base"` MaxBytesForLevelMultiplier int `toml:"max_bytes_for_level_multiplier"` DisableAutoCompactions bool `toml:"disable_auto_compactions"` - DisableDataSync bool `toml:"disable_data_sync"` UseFsync bool `toml:"use_fsync"` MaxBackgroundCompactions int `toml:"max_background_compactions"` MaxBackgroundFlushes int `toml:"max_background_flushes"` - AllowOsBuffer bool `toml:"allow_os_buffer"` EnableStatistics bool `toml:"enable_statistics"` StatsDumpPeriodSec int `toml:"stats_dump_period_sec"` BackgroundThreads int `toml:"background_theads"` @@ -192,11 +190,9 @@ func NewConfigDefault() *Config { cfg.Replication.UseMmap = true cfg.Snapshot.MaxNum = 1 - cfg.RocksDB.AllowOsBuffer = true cfg.RocksDB.EnableStatistics = false cfg.RocksDB.UseFsync = false cfg.RocksDB.DisableAutoCompactions = false - cfg.RocksDB.AllowOsBuffer = true cfg.RocksDB.DisableWAL = false cfg.adjust() diff --git a/store/rocksdb/batch.go b/store/rocksdb/batch.go index bb727e7..1d7ce0f 100644 --- a/store/rocksdb/batch.go +++ b/store/rocksdb/batch.go @@ -75,9 +75,12 @@ func (w *WriteBatch) commit(wb *WriteOptions) error { return nil } +const batchDataHeadLen = 12 + func (w *WriteBatch) Data() []byte { var vallen C.size_t value := C.rocksdb_writebatch_data(w.wbatch, &vallen) - return slice(unsafe.Pointer(value), int(vallen)) + buf := slice(unsafe.Pointer(value), int(vallen)) + return buf[batchDataHeadLen:] } diff --git a/store/rocksdb/db.go b/store/rocksdb/db.go index 8b97e18..a7ee720 100644 --- a/store/rocksdb/db.go +++ b/store/rocksdb/db.go @@ -140,12 +140,10 @@ func (db *DB) initOptions(cfg *config.RocksDBConfig) { opts.SetTargetFileSizeMultiplier(cfg.TargetFileSizeMultiplier) opts.SetMaxBytesForLevelBase(cfg.MaxBytesForLevelBase) opts.SetMaxBytesForLevelMultiplier(cfg.MaxBytesForLevelMultiplier) - opts.DisableDataSync(cfg.DisableDataSync) opts.SetMinWriteBufferNumberToMerge(cfg.MinWriteBufferNumberToMerge) opts.DisableAutoCompactions(cfg.DisableAutoCompactions) opts.EnableStatistics(cfg.EnableStatistics) opts.UseFsync(cfg.UseFsync) - opts.AllowOsBuffer(cfg.AllowOsBuffer) opts.SetStatsDumpPeriodSec(cfg.StatsDumpPeriodSec) db.opts = opts diff --git a/store/rocksdb/options.go b/store/rocksdb/options.go index 2783679..aeb4a1f 100644 --- a/store/rocksdb/options.go +++ b/store/rocksdb/options.go @@ -139,7 +139,7 @@ func (o *Options) SetMaxBytesForLevelBase(n int) { } func (o *Options) SetMaxBytesForLevelMultiplier(n int) { - C.rocksdb_options_set_max_bytes_for_level_multiplier(o.Opt, C.int(n)) + C.rocksdb_options_set_max_bytes_for_level_multiplier(o.Opt, C.double(n)) } func (o *Options) SetBlockBasedTableFactory(opt *BlockBasedTableOptions) { @@ -150,10 +150,6 @@ func (o *Options) SetMinWriteBufferNumberToMerge(n int) { C.rocksdb_options_set_min_write_buffer_number_to_merge(o.Opt, C.int(n)) } -func (o *Options) DisableDataSync(b bool) { - C.rocksdb_options_set_disable_data_sync(o.Opt, boolToInt(b)) -} - func (o *Options) DisableAutoCompactions(b bool) { C.rocksdb_options_set_disable_auto_compactions(o.Opt, boolToInt(b)) } @@ -162,10 +158,6 @@ func (o *Options) UseFsync(b bool) { C.rocksdb_options_set_use_fsync(o.Opt, boolToInt(b)) } -func (o *Options) AllowOsBuffer(b bool) { - C.rocksdb_options_set_allow_os_buffer(o.Opt, boolToUchar(b)) -} - func (o *Options) EnableStatistics(b bool) { if b { C.rocksdb_options_enable_statistics(o.Opt) diff --git a/store/store_test.go b/store/store_test.go index d886b97..ffe954f 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -356,15 +356,15 @@ func testBatchData(db *DB, t *testing.T) { d := w.BatchData() + expected := []BatchItem{ + {[]byte("a"), []byte("1")}, + {[]byte("b"), []byte(nil)}, + {[]byte("c"), nil}, + } + if kvs, err := d.Items(); err != nil { t.Fatal(err) - } else if len(kvs) != 3 { - t.Fatal(len(kvs)) - } else if !reflect.DeepEqual(kvs[0], BatchItem{[]byte("a"), []byte("1")}) { - t.Fatal("must equal") - } else if !reflect.DeepEqual(kvs[1], BatchItem{[]byte("b"), []byte(nil)}) { - t.Fatalf("must equal") - } else if !reflect.DeepEqual(kvs[2], BatchItem{[]byte("c"), nil}) { - t.Fatal("must equal") + } else if !reflect.DeepEqual(kvs, expected) { + t.Fatalf("%v != %v", kvs, expected) } } diff --git a/store/writebatch.go b/store/writebatch.go index 7df7c7e..73760d7 100644 --- a/store/writebatch.go +++ b/store/writebatch.go @@ -75,8 +75,6 @@ func (wb *WriteBatch) Data() []byte { return b.Data() } -const BatchDataHeadLen = 12 - /* see leveldb batch data format for more information */