update rocksdb 5.1+ (#282)

README.md

@@ -57,9 +57,9 @@ Create a workspace and checkout ledisdb source
 + Set `LEVELDB_DIR` and `SNAPPY_DIR` to the actual install path in dev.sh.
 + `make clean && make` 
 
-## RocksDB support
+## RocksDB support 
 
-+ [Install rocksdb](https://github.com/facebook/rocksdb/blob/master/INSTALL.md)(`make shared_lib`) and snappy first.
++ [Install rocksdb(5.1+)](https://github.com/facebook/rocksdb/blob/master/INSTALL.md)(`make shared_lib`) and snappy first.
 
     LedisDB has not yet supplied a simple script to install.
 

config/config.go

@@ -52,11 +52,9 @@ type RocksDBConfig struct {
 	MaxBytesForLevelBase           int  `toml:"max_bytes_for_level_base"`
 	MaxBytesForLevelMultiplier     int  `toml:"max_bytes_for_level_multiplier"`
 	DisableAutoCompactions         bool `toml:"disable_auto_compactions"`
-	DisableDataSync                bool `toml:"disable_data_sync"`
 	UseFsync                       bool `toml:"use_fsync"`
 	MaxBackgroundCompactions       int  `toml:"max_background_compactions"`
 	MaxBackgroundFlushes           int  `toml:"max_background_flushes"`
-	AllowOsBuffer                  bool `toml:"allow_os_buffer"`
 	EnableStatistics               bool `toml:"enable_statistics"`
 	StatsDumpPeriodSec             int  `toml:"stats_dump_period_sec"`
 	BackgroundThreads              int  `toml:"background_theads"`
@@ -192,11 +190,9 @@ func NewConfigDefault() *Config {
 	cfg.Replication.UseMmap = true
 	cfg.Snapshot.MaxNum = 1
 
-	cfg.RocksDB.AllowOsBuffer = true
 	cfg.RocksDB.EnableStatistics = false
 	cfg.RocksDB.UseFsync = false
 	cfg.RocksDB.DisableAutoCompactions = false
-	cfg.RocksDB.AllowOsBuffer = true
 	cfg.RocksDB.DisableWAL = false
 
 	cfg.adjust()

store/rocksdb/batch.go

@@ -75,9 +75,12 @@ func (w *WriteBatch) commit(wb *WriteOptions) error {
 	return nil
 }
 
+const batchDataHeadLen = 12
+
 func (w *WriteBatch) Data() []byte {
 	var vallen C.size_t
 	value := C.rocksdb_writebatch_data(w.wbatch, &vallen)
 
-	return slice(unsafe.Pointer(value), int(vallen))
+	buf := slice(unsafe.Pointer(value), int(vallen))
+	return buf[batchDataHeadLen:]
 }

store/rocksdb/db.go

@@ -140,12 +140,10 @@ func (db *DB) initOptions(cfg *config.RocksDBConfig) {
 	opts.SetTargetFileSizeMultiplier(cfg.TargetFileSizeMultiplier)
 	opts.SetMaxBytesForLevelBase(cfg.MaxBytesForLevelBase)
 	opts.SetMaxBytesForLevelMultiplier(cfg.MaxBytesForLevelMultiplier)
-	opts.DisableDataSync(cfg.DisableDataSync)
 	opts.SetMinWriteBufferNumberToMerge(cfg.MinWriteBufferNumberToMerge)
 	opts.DisableAutoCompactions(cfg.DisableAutoCompactions)
 	opts.EnableStatistics(cfg.EnableStatistics)
 	opts.UseFsync(cfg.UseFsync)
-	opts.AllowOsBuffer(cfg.AllowOsBuffer)
 	opts.SetStatsDumpPeriodSec(cfg.StatsDumpPeriodSec)
 
 	db.opts = opts

store/rocksdb/options.go

@@ -139,7 +139,7 @@ func (o *Options) SetMaxBytesForLevelBase(n int) {
 }
 
 func (o *Options) SetMaxBytesForLevelMultiplier(n int) {
-	C.rocksdb_options_set_max_bytes_for_level_multiplier(o.Opt, C.int(n))
+	C.rocksdb_options_set_max_bytes_for_level_multiplier(o.Opt, C.double(n))
 }
 
 func (o *Options) SetBlockBasedTableFactory(opt *BlockBasedTableOptions) {
@@ -150,10 +150,6 @@ func (o *Options) SetMinWriteBufferNumberToMerge(n int) {
 	C.rocksdb_options_set_min_write_buffer_number_to_merge(o.Opt, C.int(n))
 }
 
-func (o *Options) DisableDataSync(b bool) {
-	C.rocksdb_options_set_disable_data_sync(o.Opt, boolToInt(b))
-}
-
 func (o *Options) DisableAutoCompactions(b bool) {
 	C.rocksdb_options_set_disable_auto_compactions(o.Opt, boolToInt(b))
 }
@@ -162,10 +158,6 @@ func (o *Options) UseFsync(b bool) {
 	C.rocksdb_options_set_use_fsync(o.Opt, boolToInt(b))
 }
 
-func (o *Options) AllowOsBuffer(b bool) {
-	C.rocksdb_options_set_allow_os_buffer(o.Opt, boolToUchar(b))
-}
-
 func (o *Options) EnableStatistics(b bool) {
 	if b {
 		C.rocksdb_options_enable_statistics(o.Opt)

store/store_test.go

@@ -356,15 +356,15 @@ func testBatchData(db *DB, t *testing.T) {
 
 	d := w.BatchData()
 
+	expected := []BatchItem{
+		{[]byte("a"), []byte("1")},
+		{[]byte("b"), []byte(nil)},
+		{[]byte("c"), nil},
+	}
+
 	if kvs, err := d.Items(); err != nil {
 		t.Fatal(err)
-	} else if len(kvs) != 3 {
+	} else if !reflect.DeepEqual(kvs, expected) {
-		t.Fatal(len(kvs))
+		t.Fatalf("%v != %v", kvs, expected)
-	} else if !reflect.DeepEqual(kvs[0], BatchItem{[]byte("a"), []byte("1")}) {
-		t.Fatal("must equal")
-	} else if !reflect.DeepEqual(kvs[1], BatchItem{[]byte("b"), []byte(nil)}) {
-		t.Fatalf("must equal")
-	} else if !reflect.DeepEqual(kvs[2], BatchItem{[]byte("c"), nil}) {
-		t.Fatal("must equal")
 	}
 }

store/writebatch.go

@@ -75,8 +75,6 @@ func (wb *WriteBatch) Data() []byte {
 	return b.Data()
 }
 
-const BatchDataHeadLen = 12
-
 /*
 	see leveldb batch data format for more information
 */