From 0c5691ff0e885584511b4abc52c6587955f7bb8a Mon Sep 17 00:00:00 2001 From: Maxim Kouprianov Date: Thu, 23 Oct 2014 20:26:28 +0400 Subject: [PATCH 01/98] Increase MaxValueSize for a reasonable value 10MB for a blob is a really tiny size, I'd like to decide myself the size of values I'm going to push. And it seems to me that all blobs smaller than 1GB shouldn't be harmful for the ledis, so just lift up this limit please. --- ledis/const.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ledis/const.go b/ledis/const.go index 3e17a95..5c208ab 100644 --- a/ledis/const.go +++ b/ledis/const.go @@ -76,7 +76,7 @@ const ( MaxSetMemberSize int = 1024 //max value size - MaxValueSize int = 10 * 1024 * 1024 + MaxValueSize int = 1024 * 1024 * 1024 ) var ( From c7dfec509b9a7aa48fdec2f231d809d75b66d3f6 Mon Sep 17 00:00:00 2001 From: holys Date: Sun, 26 Oct 2014 15:15:43 +0800 Subject: [PATCH 02/98] add SETEX command for KV --- cmd/ledis-cli/const.go | 3 ++- doc/commands.json | 5 +++++ doc/commands.md | 25 +++++++++++++++++++++++++ ledis/t_kv.go | 26 ++++++++++++++++++++++++++ ledis/t_kv_test.go | 29 +++++++++++++++++++++++++++++ server/cmd_kv.go | 21 +++++++++++++++++++++ server/cmd_kv_test.go | 10 ++++++++++ 7 files changed, 118 insertions(+), 1 deletion(-) diff --git a/cmd/ledis-cli/const.go b/cmd/ledis-cli/const.go index 790a77b..5586f1a 100644 --- a/cmd/ledis-cli/const.go +++ b/cmd/ledis-cli/const.go @@ -1,4 +1,4 @@ -//This file was generated by .tools/generate_commands.py on Mon Oct 20 2014 22:35:33 +0800 +//This file was generated by .tools/generate_commands.py on Sun Oct 26 2014 15:14:39 +0800 package main var helpCommands = [][]string{ @@ -86,6 +86,7 @@ var helpCommands = [][]string{ {"SDIFFSTORE", "destination key [key ...]", "Set"}, {"SELECT", "index", "Server"}, {"SET", "key value", "KV"}, + {"SETEX", "key seconds value", "KV"}, {"SETNX", "key value", "KV"}, {"SEXPIRE", "key seconds", "Set"}, {"SEXPIREAT", "key timestamp", "Set"}, diff --git a/doc/commands.json b/doc/commands.json index b03e7f9..d435bc3 100644 --- a/doc/commands.json +++ b/doc/commands.json @@ -310,6 +310,11 @@ "group": "KV", "readonly": false }, + "SETEX": { + "arguments": "key seconds value", + "group": "KV", + "readonly": false + }, "SLAVEOF": { "arguments": "host port [RESTART] [READONLY]", "group": "Replication", diff --git a/doc/commands.md b/doc/commands.md index 9dcdca6..a5c0527 100644 --- a/doc/commands.md +++ b/doc/commands.md @@ -22,6 +22,7 @@ Table of Contents - [MSET key value [key value ...]](#mset-key-value-key-value-) - [SET key value](#set-key-value) - [SETNX key value](#setnx-key-value) + - [SETEX key seconds value](#setex-key-seconds-value) - [EXPIRE key seconds](#expire-key-seconds) - [EXPIREAT key timestamp](#expireat-key-timestamp) - [TTL key](#ttl-key) @@ -389,6 +390,30 @@ ledis> GET mykey "hello" ``` +### SETEX key seconds value +Set key to hold the string value and set key to timeout after a given number of seconds. This command is equivalent to executing the following commands: + +``` +SET mykey value +EXPIRE mykey seconds +``` + +**Return value** + +Simple string reply + +**Examples** + +``` +ledis> SETEX mykey 10 "Hello" +OK +ledis> TTL mykey +(integer) 10 +ledis> GET mykey +"Hello" +ledis> +``` + ### EXPIRE key seconds Set a timeout on key. After the timeout has expired, the key will be deleted. diff --git a/ledis/t_kv.go b/ledis/t_kv.go index 497dcf2..2a55425 100644 --- a/ledis/t_kv.go +++ b/ledis/t_kv.go @@ -300,6 +300,32 @@ func (db *DB) SetNX(key []byte, value []byte) (int64, error) { return n, err } +func (db *DB) SetEX(key []byte, duration int64, value []byte) error { + if err := checkKeySize(key); err != nil { + return err + } else if err := checkValueSize(value); err != nil { + return err + } else if duration <= 0 { + return errExpireValue + } + + ek := db.encodeKVKey(key) + + t := db.kvBatch + + t.Lock() + defer t.Unlock() + + t.Put(ek, value) + db.expireAt(t, KVType, key, time.Now().Unix()+duration) + + if err := t.Commit(); err != nil { + return err + } + + return nil +} + func (db *DB) flush() (drop int64, err error) { t := db.kvBatch t.Lock() diff --git a/ledis/t_kv_test.go b/ledis/t_kv_test.go index b0f3437..77ce706 100644 --- a/ledis/t_kv_test.go +++ b/ledis/t_kv_test.go @@ -116,3 +116,32 @@ func TestKVFlush(t *testing.T) { } } } + +func TestKVSetEX(t *testing.T) { + db := getTestDB() + db.FlushAll() + + key := []byte("testdb_kv_c") + + if err := db.SetEX(key, 10, []byte("hello world")); err != nil { + t.Fatal(err) + } + + v, err := db.Get(key) + if err != nil { + t.Fatal(err) + } else if string(v) == "" { + t.Fatal("v is nil") + } + + if n, err := db.TTL(key); err != nil { + t.Fatal(err) + } else if n != 10 { + t.Fatal(n) + } + + if v, _ := db.Get(key); string(v) != "hello world" { + t.Fatal(string(v)) + } + +} diff --git a/server/cmd_kv.go b/server/cmd_kv.go index c62cc18..0d2dc25 100644 --- a/server/cmd_kv.go +++ b/server/cmd_kv.go @@ -66,6 +66,26 @@ func setnxCommand(c *client) error { return nil } +func setexCommand(c *client) error { + args := c.args + if len(args) != 3 { + return ErrCmdParams + } + + sec, err := ledis.StrInt64(args[1], nil) + if err != nil { + return ErrValue + } + + if err := c.db.SetEX(args[0], sec, args[2]); err != nil { + return err + } else { + c.resp.writeStatus(OK) + } + + return nil +} + func existsCommand(c *client) error { args := c.args if len(args) != 1 { @@ -365,6 +385,7 @@ func init() { register("mset", msetCommand) register("set", setCommand) register("setnx", setnxCommand) + register("setex", setexCommand) register("expire", expireCommand) register("expireat", expireAtCommand) register("ttl", ttlCommand) diff --git a/server/cmd_kv_test.go b/server/cmd_kv_test.go index d24fd25..d3b0fe4 100644 --- a/server/cmd_kv_test.go +++ b/server/cmd_kv_test.go @@ -27,6 +27,12 @@ func TestKV(t *testing.T) { t.Fatal(n) } + if ok, err := ledis.String(c.Do("setex", "xx", 10, "hello world")); err != nil { + t.Fatal(err) + } else if ok != OK { + t.Fatal(ok) + } + if v, err := ledis.String(c.Do("get", "a")); err != nil { t.Fatal(err) } else if v != "1234" { @@ -214,4 +220,8 @@ func TestKVErrorParams(t *testing.T) { t.Fatal("invalid err of %v", err) } + if _, err := c.Do("setex", "a", "blah", "hello world"); err == nil { + t.Fatalf("invalid err %v", err) + } + } From d0a40c411d52667fd8da1afb644f59328167b12b Mon Sep 17 00:00:00 2001 From: siddontang Date: Sun, 26 Oct 2014 15:30:36 +0800 Subject: [PATCH 03/98] update config --- etc/ledis.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etc/ledis.conf b/etc/ledis.conf index a3b0833..aef46d6 100644 --- a/etc/ledis.conf +++ b/etc/ledis.conf @@ -43,6 +43,10 @@ db_sync_commit = 0 # enable replication or not use_replication = true +# set connection buffer, you can increase them appropriately +conn_read_buffer_size = 10240 +conn_write_buffer_size = 10240 + [leveldb] # for leveldb and goleveldb compression = false From c365f3902d76bd84ff5b69bfc6db7c5c0fc261aa Mon Sep 17 00:00:00 2001 From: siddontang Date: Tue, 28 Oct 2014 17:57:29 +0800 Subject: [PATCH 04/98] update bench tool --- cmd/ledis-benchmark/main.go | 101 ++++++++++++++++++++++++++--------- cmd/ledis-storebench/main.go | 8 ++- 2 files changed, 82 insertions(+), 27 deletions(-) diff --git a/cmd/ledis-benchmark/main.go b/cmd/ledis-benchmark/main.go index 510d436..1fe0c2d 100644 --- a/cmd/ledis-benchmark/main.go +++ b/cmd/ledis-benchmark/main.go @@ -6,6 +6,7 @@ import ( "github.com/siddontang/ledisdb/client/go/ledis" "math/rand" "runtime" + "strings" "sync" "sync/atomic" "time" @@ -15,10 +16,9 @@ var ip = flag.String("ip", "127.0.0.1", "redis/ledis/ssdb server ip") var port = flag.Int("port", 6380, "redis/ledis/ssdb server port") var number = flag.Int("n", 1000, "request number") var clients = flag.Int("c", 50, "number of clients") -var reverse = flag.Bool("rev", false, "enable zset rev benchmark") var round = flag.Int("r", 1, "benchmark round number") -var del = flag.Bool("del", true, "enable del benchmark") var valueSize = flag.Int("vsize", 100, "kv value size") +var tests = flag.String("t", "", "only run the comma separated list of tests, set,get,del,lpush,lrange,lpop,hset,hget,hdel,zadd,zincr,zrange,zrevrange,zdel") var wg sync.WaitGroup var client *ledis.Client @@ -53,7 +53,13 @@ func bench(cmd string, f func()) { t2 := time.Now() - fmt.Printf("%s: %0.2f op/s\n", cmd, (float64(*number) / t2.Sub(t1).Seconds())) + d := t2.Sub(t1) + + fmt.Printf("%s: %s %0.3f micros/op, %0.2fop/s\n", + cmd, + d.String(), + float64(d.Nanoseconds()/1e3)/float64(*number), + float64(*number)/d.Seconds()) } var kvSetBase int64 = 0 @@ -276,45 +282,90 @@ func main() { *round = 1 } - for i := 0; i < *round; i++ { - benchSet() - benchGet() - benchRandGet() + runAll := true + ts := strings.Split(*tests, ",") + if len(ts) > 0 && len(ts[0]) != 0 { + runAll = false + } - if *del == true { + println(*tests, len(ts)) + + needTest := make(map[string]struct{}) + for _, s := range ts { + needTest[strings.ToLower(s)] = struct{}{} + } + + checkTest := func(cmd string) bool { + if runAll { + return true + } else if _, ok := needTest[cmd]; ok { + return ok + } + return false + } + + for i := 0; i < *round; i++ { + if checkTest("set") { + benchSet() + } + + if checkTest("get") { + benchGet() + benchRandGet() + } + + if checkTest("del") { benchDel() } - benchPushList() - benchRangeList10() - benchRangeList50() - benchRangeList100() + if checkTest("lpush") { + benchPushList() + } - if *del == true { + if checkTest("lrange") { + benchRangeList10() + benchRangeList50() + benchRangeList100() + } + + if checkTest("lpop") { benchPopList() } - benchHset() - benchHGet() - benchHRandGet() + if checkTest("hset") { + benchHset() + } - if *del == true { + if checkTest("hget") { + benchHGet() + benchHRandGet() + } + + if checkTest("hdel") { benchHDel() } - benchZAdd() - benchZIncr() - benchZRangeByRank() - benchZRangeByScore() + if checkTest("zadd") { + benchZAdd() + } - //rev is too slow in leveldb, rocksdb or other - //maybe disable for huge data benchmark - if *reverse == true { + if checkTest("zincr") { + benchZIncr() + } + + if checkTest("zrange") { + benchZRangeByRank() + benchZRangeByScore() + } + + if checkTest("zrevrange") { + //rev is too slow in leveldb, rocksdb or other + //maybe disable for huge data benchmark benchZRevRangeByRank() benchZRevRangeByScore() } - if *del == true { + if checkTest("zdel") { benchZDel() } diff --git a/cmd/ledis-storebench/main.go b/cmd/ledis-storebench/main.go index 6d91a89..ce37750 100644 --- a/cmd/ledis-storebench/main.go +++ b/cmd/ledis-storebench/main.go @@ -46,8 +46,12 @@ func bench(cmd string, f func()) { t2 := time.Now() d := t2.Sub(t1) - fmt.Printf("%s: %0.3f micros/op, %0.2fmb/s %0.2fop/s\n", cmd, float64(d.Nanoseconds()/1e3)/float64(*number), - float64((*valueSize+16)*(*number))/(1024.0*1024.0*(d.Seconds())), float64(*number)/d.Seconds()) + fmt.Printf("%s %s: %0.3f micros/op, %0.2fmb/s %0.2fop/s\n", + cmd, + d.String(), + float64(d.Nanoseconds()/1e3)/float64(*number), + float64((*valueSize+16)*(*number))/(1024.0*1024.0*(d.Seconds())), + float64(*number)/d.Seconds()) } var kvSetBase int64 = 0 From fdeda83a374f3339d47b5db70e32379124e42d2d Mon Sep 17 00:00:00 2001 From: siddontang Date: Tue, 28 Oct 2014 17:58:37 +0800 Subject: [PATCH 05/98] update protocol handling --- server/client_resp.go | 111 ++++++++++++++++++++++++++++-------------- server/util.go | 68 ++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 37 deletions(-) diff --git a/server/client_resp.go b/server/client_resp.go index 8d8378f..2713609 100644 --- a/server/client_resp.go +++ b/server/client_resp.go @@ -37,6 +37,7 @@ func newClientRESP(conn net.Conn, app *App) { tcpConn.SetReadBuffer(app.cfg.ConnReadBufferSize) tcpConn.SetWriteBuffer(app.cfg.ConnWriteBufferSize) } + c.rb = bufio.NewReaderSize(conn, app.cfg.ConnReadBufferSize) c.resp = newWriterRESP(conn, app.cfg.ConnWriteBufferSize) @@ -85,57 +86,93 @@ func (c *respClient) readLine() ([]byte, error) { } //A client sends to the Redis server a RESP Array consisting of just Bulk Strings. +// func (c *respClient) readRequest() ([][]byte, error) { +// l, err := c.readLine() +// if err != nil { +// return nil, err +// } else if len(l) == 0 || l[0] != '*' { +// return nil, errReadRequest +// } + +// var nparams int +// if nparams, err = strconv.Atoi(hack.String(l[1:])); err != nil { +// return nil, err +// } else if nparams <= 0 { +// return nil, errReadRequest +// } + +// req := make([][]byte, 0, nparams) +// var n int +// for i := 0; i < nparams; i++ { +// if l, err = c.readLine(); err != nil { +// return nil, err +// } + +// if len(l) == 0 { +// return nil, errReadRequest +// } else if l[0] == '$' { +// //handle resp string +// if n, err = strconv.Atoi(hack.String(l[1:])); err != nil { +// return nil, err +// } else if n == -1 { +// req = append(req, nil) +// } else { +// buf := make([]byte, n+2) +// if _, err = io.ReadFull(c.rb, buf); err != nil { +// return nil, err +// } + +// if buf[len(buf)-2] != '\r' && buf[len(buf)-1] != '\n' { +// return nil, errors.New("bad bulk string format") +// } + +// // if l, err = c.readLine(); err != nil { +// // return nil, err +// // } else if len(l) != 0 { +// // return nil, errors.New("bad bulk string format") +// // } + +// req = append(req, buf[0:len(buf)-2]) + +// } + +// } else { +// return nil, errReadRequest +// } +// } + +// return req, nil +// } + func (c *respClient) readRequest() ([][]byte, error) { - l, err := c.readLine() + code, err := c.rb.ReadByte() if err != nil { return nil, err - } else if len(l) == 0 || l[0] != '*' { + } + + if code != '*' { return nil, errReadRequest } - var nparams int - if nparams, err = strconv.Atoi(hack.String(l[1:])); err != nil { + var nparams int64 + if nparams, err = readLong(c.rb); err != nil { return nil, err } else if nparams <= 0 { return nil, errReadRequest } - req := make([][]byte, 0, nparams) - var n int - for i := 0; i < nparams; i++ { - if l, err = c.readLine(); err != nil { + req := make([][]byte, nparams) + for i := range req { + if code, err = c.rb.ReadByte(); err != nil { + return nil, err + } else if code != '$' { + return nil, errReadRequest + } + + if req[i], err = readBytes(c.rb); err != nil { return nil, err } - - if len(l) == 0 { - return nil, errReadRequest - } else if l[0] == '$' { - //handle resp string - if n, err = strconv.Atoi(hack.String(l[1:])); err != nil { - return nil, err - } else if n == -1 { - req = append(req, nil) - } else { - buf := make([]byte, n) - if _, err = io.ReadFull(c.rb, buf); err != nil { - return nil, err - } - - if l, err = c.readLine(); err != nil { - return nil, err - } else if len(l) != 0 { - return nil, errors.New("bad bulk string format") - } - - req = append(req, buf) - - } - - } else { - return nil, errReadRequest - } } - return req, nil } diff --git a/server/util.go b/server/util.go index 44b289c..504b051 100644 --- a/server/util.go +++ b/server/util.go @@ -3,6 +3,8 @@ package server import ( "bufio" "errors" + "fmt" + "io" ) var ( @@ -19,5 +21,71 @@ func ReadLine(rb *bufio.Reader) ([]byte, error) { if i < 0 || p[i] != '\r' { return nil, errLineFormat } + return p[:i], nil } + +func readBytes(br *bufio.Reader) (bytes []byte, err error) { + size, err := readLong(br) + if err != nil { + return nil, err + } + if size == -1 { + return nil, nil + } + if size < 0 { + return nil, errors.New("Invalid size: " + fmt.Sprint("%d", size)) + } + + buf := make([]byte, size+2) + if _, err = io.ReadFull(br, buf); err != nil { + return nil, err + } + + if buf[len(buf)-2] != '\r' && buf[len(buf)-1] != '\n' { + return nil, errors.New("bad bulk string format") + } + + bytes = buf[0 : len(buf)-2] + + return +} + +func readLong(in *bufio.Reader) (result int64, err error) { + read, err := in.ReadByte() + if err != nil { + return -1, err + } + var sign int + if read == '-' { + read, err = in.ReadByte() + if err != nil { + return -1, err + } + sign = -1 + } else { + sign = 1 + } + var number int64 + for number = 0; err == nil; read, err = in.ReadByte() { + if read == '\r' { + read, err = in.ReadByte() + if err != nil { + return -1, err + } + if read == '\n' { + return number * int64(sign), nil + } else { + return -1, errors.New("Bad line ending") + } + } + value := read - '0' + if value >= 0 && value < 10 { + number *= 10 + number += int64(value) + } else { + return -1, errors.New("Invalid digit") + } + } + return -1, err +} From e36cc553ec82d38d5277fce66123b39add6bff7a Mon Sep 17 00:00:00 2001 From: siddontang Date: Tue, 28 Oct 2014 23:23:30 +0800 Subject: [PATCH 06/98] update benchmark --- cmd/ledis-benchmark/main.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cmd/ledis-benchmark/main.go b/cmd/ledis-benchmark/main.go index 1fe0c2d..02f56eb 100644 --- a/cmd/ledis-benchmark/main.go +++ b/cmd/ledis-benchmark/main.go @@ -29,9 +29,9 @@ func waitBench(cmd string, args ...interface{}) { c := client.Get() defer c.Close() - _, err := c.Do(cmd, args...) + _, err := c.Do(strings.ToUpper(cmd), args...) if err != nil { - fmt.Printf("do %s error %s", cmd, err.Error()) + fmt.Printf("do %s error %s\n", cmd, err.Error()) return } } @@ -88,7 +88,7 @@ func benchGet() { func benchRandGet() { f := func() { - n := rand.Int() + n := rand.Int() % *number waitBench("get", n) } @@ -172,7 +172,7 @@ func benchHGet() { func benchHRandGet() { f := func() { - n := rand.Int() + n := rand.Int() % *number waitBench("hget", "myhashkey", n) } @@ -288,8 +288,6 @@ func main() { runAll = false } - println(*tests, len(ts)) - needTest := make(map[string]struct{}) for _, s := range ts { needTest[strings.ToLower(s)] = struct{}{} From c5107c71372cc9e403d899fd76fce4b2401979a6 Mon Sep 17 00:00:00 2001 From: siddontang Date: Tue, 28 Oct 2014 23:30:00 +0800 Subject: [PATCH 07/98] add slice for store, optimize leveldb/rocksdb get --- store/boltdb/db.go | 9 +++++++ store/driver/driver.go | 1 + store/driver/slice.go | 60 ++++++++++++++++++++++++++++++++++++++++++ store/goleveldb/db.go | 8 ++++++ store/leveldb/db.go | 26 ++++++++++++++++++ store/mdb/mdb.go | 9 +++++++ store/rocksdb/db.go | 26 ++++++++++++++++++ 7 files changed, 139 insertions(+) create mode 100644 store/driver/slice.go diff --git a/store/boltdb/db.go b/store/boltdb/db.go index ac8cc03..377cac5 100644 --- a/store/boltdb/db.go +++ b/store/boltdb/db.go @@ -167,6 +167,15 @@ func (db *DB) Compact() error { return nil } +func (db *DB) GetSlice(key []byte) (driver.ISlice, error) { + v, err := db.Get(key) + if err != nil { + return nil, err + } else { + return driver.GoSlice(v), nil + } +} + func init() { driver.Register(Store{}) } diff --git a/store/driver/driver.go b/store/driver/driver.go index e4312ce..32e5a68 100644 --- a/store/driver/driver.go +++ b/store/driver/driver.go @@ -12,6 +12,7 @@ type IDB interface { Close() error Get(key []byte) ([]byte, error) + GetSlice(key []byte) (ISlice, error) Put(key []byte, value []byte) error Delete(key []byte) error diff --git a/store/driver/slice.go b/store/driver/slice.go new file mode 100644 index 0000000..76e4ea7 --- /dev/null +++ b/store/driver/slice.go @@ -0,0 +1,60 @@ +package driver + +// #include +import "C" + +import ( + "reflect" + "unsafe" +) + +type ISlice interface { + Data() []byte + Size() int + Free() +} + +type CSlice struct { + data unsafe.Pointer + size int +} + +func NewCSlice(p unsafe.Pointer, n int) *CSlice { + return &CSlice{p, n} +} + +func (s *CSlice) Data() []byte { + var value []byte + + sH := (*reflect.SliceHeader)(unsafe.Pointer(&value)) + sH.Cap = int(s.size) + sH.Len = int(s.size) + sH.Data = uintptr(s.data) + + return value +} + +func (s *CSlice) Size() int { + return int(s.size) +} + +func (s *CSlice) Free() { + if s.data != nil { + C.free(s.data) + s.data = nil + } +} + +type GoSlice []byte + +func (s GoSlice) Data() []byte { + return []byte(s) +} + +func (s GoSlice) Size() int { + return len(s) +} + +func (s GoSlice) Free() { + +} diff --git a/store/goleveldb/db.go b/store/goleveldb/db.go index 9924067..6b3fb1b 100644 --- a/store/goleveldb/db.go +++ b/store/goleveldb/db.go @@ -196,6 +196,14 @@ func (db *DB) Compact() error { return db.db.CompactRange(util.Range{nil, nil}) } +func (db *DB) GetSlice(key []byte) (driver.ISlice, error) { + v, err := db.Get(key) + if err != nil { + return nil, err + } else { + return driver.GoSlice(v), nil + } +} func init() { driver.Register(Store{}) driver.Register(MemStore{}) diff --git a/store/leveldb/db.go b/store/leveldb/db.go index 449c32b..2de24b6 100644 --- a/store/leveldb/db.go +++ b/store/leveldb/db.go @@ -257,6 +257,28 @@ func (db *DB) get(ro *ReadOptions, key []byte) ([]byte, error) { return C.GoBytes(unsafe.Pointer(value), C.int(vallen)), nil } +func (db *DB) getSlice(ro *ReadOptions, key []byte) (driver.ISlice, error) { + var errStr *C.char + var vallen C.size_t + var k *C.char + if len(key) != 0 { + k = (*C.char)(unsafe.Pointer(&key[0])) + } + + value := C.leveldb_get( + db.db, ro.Opt, k, C.size_t(len(key)), &vallen, &errStr) + + if errStr != nil { + return nil, saveError(errStr) + } + + if value == nil { + return nil, nil + } + + return driver.NewCSlice(unsafe.Pointer(value), int(vallen)), nil +} + func (db *DB) delete(wo *WriteOptions, key []byte) error { var errStr *C.char var k *C.char @@ -282,6 +304,10 @@ func (db *DB) Compact() error { return nil } +func (db *DB) GetSlice(key []byte) (driver.ISlice, error) { + return db.getSlice(db.readOpts, key) +} + func init() { driver.Register(Store{}) } diff --git a/store/mdb/mdb.go b/store/mdb/mdb.go index 6bf26f6..ed8f2d1 100644 --- a/store/mdb/mdb.go +++ b/store/mdb/mdb.go @@ -311,6 +311,15 @@ func (db MDB) Compact() error { return nil } +func (db MDB) GetSlice(key []byte) (driver.ISlice, error) { + v, err := db.Get(key) + if err != nil { + return nil, err + } else { + return driver.GoSlice(v), nil + } +} + func init() { driver.Register(Store{}) } diff --git a/store/rocksdb/db.go b/store/rocksdb/db.go index e1f10a4..7ed572a 100644 --- a/store/rocksdb/db.go +++ b/store/rocksdb/db.go @@ -284,6 +284,28 @@ func (db *DB) get(ro *ReadOptions, key []byte) ([]byte, error) { return C.GoBytes(unsafe.Pointer(value), C.int(vallen)), nil } +func (db *DB) getSlice(ro *ReadOptions, key []byte) (driver.ISlice, error) { + var errStr *C.char + var vallen C.size_t + var k *C.char + if len(key) != 0 { + k = (*C.char)(unsafe.Pointer(&key[0])) + } + + value := C.rocksdb_get( + db.db, ro.Opt, k, C.size_t(len(key)), &vallen, &errStr) + + if errStr != nil { + return nil, saveError(errStr) + } + + if value == nil { + return nil, nil + } + + return driver.NewCSlice(unsafe.Pointer(value), int(vallen)), nil +} + func (db *DB) delete(wo *WriteOptions, key []byte) error { var errStr *C.char var k *C.char @@ -309,6 +331,10 @@ func (db *DB) Compact() error { return nil } +func (db *DB) GetSlice(key []byte) (driver.ISlice, error) { + return db.getSlice(db.readOpts, key) +} + func init() { driver.Register(Store{}) } From e1df74e33d706fc6d375998031903e405b6460d7 Mon Sep 17 00:00:00 2001 From: siddontang Date: Tue, 28 Oct 2014 23:30:40 +0800 Subject: [PATCH 08/98] set replication config to false default --- config/config.toml | 2 +- etc/ledis.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.toml b/config/config.toml index aef46d6..2b5d545 100644 --- a/config/config.toml +++ b/config/config.toml @@ -41,7 +41,7 @@ db_path = "" db_sync_commit = 0 # enable replication or not -use_replication = true +use_replication = false # set connection buffer, you can increase them appropriately conn_read_buffer_size = 10240 diff --git a/etc/ledis.conf b/etc/ledis.conf index aef46d6..2b5d545 100644 --- a/etc/ledis.conf +++ b/etc/ledis.conf @@ -41,7 +41,7 @@ db_path = "" db_sync_commit = 0 # enable replication or not -use_replication = true +use_replication = false # set connection buffer, you can increase them appropriately conn_read_buffer_size = 10240 From 4a7b57d30e300410e129b95fc87fb634ee802523 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 29 Oct 2014 09:11:22 +0800 Subject: [PATCH 09/98] add reap protocol benchmark --- cmd/ledis-respbench/main.go | 94 +++++++++++++++++++++++++++++++++++++ server/client_resp.go | 93 +----------------------------------- server/util.go | 33 +++++++++++++ 3 files changed, 128 insertions(+), 92 deletions(-) create mode 100644 cmd/ledis-respbench/main.go diff --git a/cmd/ledis-respbench/main.go b/cmd/ledis-respbench/main.go new file mode 100644 index 0000000..7e657c0 --- /dev/null +++ b/cmd/ledis-respbench/main.go @@ -0,0 +1,94 @@ +package main + +import ( + "bufio" + "bytes" + "flag" + "fmt" + "github.com/siddontang/ledisdb/server" + "net" + "runtime" + "sync" + "time" +) + +var addr = flag.String("addr", ":6380", "listen addr") + +func main() { + runtime.GOMAXPROCS(runtime.NumCPU()) + + flag.Parse() + l, err := net.Listen("tcp", *addr) + + println("listen", *addr) + + if err != nil { + fmt.Println(err.Error()) + return + } + + for { + c, err := l.Accept() + if err != nil { + println(err.Error()) + continue + } + go run(c) + } +} + +var m = map[string][]byte{} +var mu sync.Mutex + +func run(c net.Conn) { + //buf := make([]byte, 10240) + ok := []byte("+OK\r\n") + + var rt time.Duration + var wt time.Duration + + rb := bufio.NewReaderSize(c, 10240) + wb := bufio.NewWriterSize(c, 10240) + + for { + t1 := time.Now() + + req, err := server.ReadRequest(rb) + + if err != nil { + break + } + t2 := time.Now() + + rt += t2.Sub(t1) + + cmd := string(bytes.ToUpper(req[0])) + switch cmd { + case "SET": + mu.Lock() + m[string(req[1])] = req[2] + mu.Unlock() + wb.Write(ok) + case "GET": + mu.Lock() + v := m[string(req[1])] + mu.Unlock() + if v == nil { + wb.WriteString("$-1\r\n") + } else { + wb.WriteString(fmt.Sprintf("$%d\r\n", len(v))) + wb.Write(v) + wb.WriteString("\r\n") + } + default: + wb.WriteString(fmt.Sprintf("-Err %s Not Supported Now", req[0])) + } + + wb.Flush() + + t3 := time.Now() + wt += t3.Sub(t2) + } + + fmt.Printf("rt:%s wt:%s\n", rt.String(), wt.String()) +} diff --git a/server/client_resp.go b/server/client_resp.go index 2713609..f5f7768 100644 --- a/server/client_resp.go +++ b/server/client_resp.go @@ -81,99 +81,8 @@ func (c *respClient) run() { } } -func (c *respClient) readLine() ([]byte, error) { - return ReadLine(c.rb) -} - -//A client sends to the Redis server a RESP Array consisting of just Bulk Strings. -// func (c *respClient) readRequest() ([][]byte, error) { -// l, err := c.readLine() -// if err != nil { -// return nil, err -// } else if len(l) == 0 || l[0] != '*' { -// return nil, errReadRequest -// } - -// var nparams int -// if nparams, err = strconv.Atoi(hack.String(l[1:])); err != nil { -// return nil, err -// } else if nparams <= 0 { -// return nil, errReadRequest -// } - -// req := make([][]byte, 0, nparams) -// var n int -// for i := 0; i < nparams; i++ { -// if l, err = c.readLine(); err != nil { -// return nil, err -// } - -// if len(l) == 0 { -// return nil, errReadRequest -// } else if l[0] == '$' { -// //handle resp string -// if n, err = strconv.Atoi(hack.String(l[1:])); err != nil { -// return nil, err -// } else if n == -1 { -// req = append(req, nil) -// } else { -// buf := make([]byte, n+2) -// if _, err = io.ReadFull(c.rb, buf); err != nil { -// return nil, err -// } - -// if buf[len(buf)-2] != '\r' && buf[len(buf)-1] != '\n' { -// return nil, errors.New("bad bulk string format") -// } - -// // if l, err = c.readLine(); err != nil { -// // return nil, err -// // } else if len(l) != 0 { -// // return nil, errors.New("bad bulk string format") -// // } - -// req = append(req, buf[0:len(buf)-2]) - -// } - -// } else { -// return nil, errReadRequest -// } -// } - -// return req, nil -// } - func (c *respClient) readRequest() ([][]byte, error) { - code, err := c.rb.ReadByte() - if err != nil { - return nil, err - } - - if code != '*' { - return nil, errReadRequest - } - - var nparams int64 - if nparams, err = readLong(c.rb); err != nil { - return nil, err - } else if nparams <= 0 { - return nil, errReadRequest - } - - req := make([][]byte, nparams) - for i := range req { - if code, err = c.rb.ReadByte(); err != nil { - return nil, err - } else if code != '$' { - return nil, errReadRequest - } - - if req[i], err = readBytes(c.rb); err != nil { - return nil, err - } - } - return req, nil + return ReadRequest(c.rb) } func (c *respClient) handleRequest(reqData [][]byte) { diff --git a/server/util.go b/server/util.go index 504b051..adeba69 100644 --- a/server/util.go +++ b/server/util.go @@ -89,3 +89,36 @@ func readLong(in *bufio.Reader) (result int64, err error) { } return -1, err } + +func ReadRequest(in *bufio.Reader) ([][]byte, error) { + code, err := in.ReadByte() + if err != nil { + return nil, err + } + + if code != '*' { + return nil, errReadRequest + } + + var nparams int64 + if nparams, err = readLong(in); err != nil { + return nil, err + } else if nparams <= 0 { + return nil, errReadRequest + } + + req := make([][]byte, nparams) + for i := range req { + if code, err = in.ReadByte(); err != nil { + return nil, err + } else if code != '$' { + return nil, errReadRequest + } + + if req[i], err = readBytes(in); err != nil { + return nil, err + } + } + + return req, nil +} From 2d52dca88ee8756bc2b9764ecb8b3448a79180b4 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 29 Oct 2014 09:12:11 +0800 Subject: [PATCH 10/98] rename to dbbench using ledis --- cmd/{ledis-storebench => ledis-dbbench}/main.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) rename cmd/{ledis-storebench => ledis-dbbench}/main.go (92%) diff --git a/cmd/ledis-storebench/main.go b/cmd/ledis-dbbench/main.go similarity index 92% rename from cmd/ledis-storebench/main.go rename to cmd/ledis-dbbench/main.go index ce37750..7f02d04 100644 --- a/cmd/ledis-storebench/main.go +++ b/cmd/ledis-dbbench/main.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/siddontang/go/num" "github.com/siddontang/ledisdb/config" - "github.com/siddontang/ledisdb/store" + "github.com/siddontang/ledisdb/ledis" "os" "runtime" "sync" @@ -24,7 +24,8 @@ var round = flag.Int("r", 1, "benchmark round number") var valueSize = flag.Int("vsize", 100, "kv value size") var wg sync.WaitGroup -var db *store.DB +var ldb *ledis.Ledis +var db *ledis.DB var loop int = 0 @@ -62,7 +63,7 @@ func benchSet() { value := make([]byte, *valueSize) n := atomic.AddInt64(&kvSetBase, 1) - db.Put(num.Int64ToBytes(n), value) + db.Set(num.Int64ToBytes(n), value) } bench("set", f) @@ -104,7 +105,7 @@ func main() { flag.Parse() cfg := config.NewConfigDefault() - cfg.DBPath = "./var/store_test" + cfg.DBPath = "./var/db_test" cfg.DBName = *name os.RemoveAll(cfg.DBPath) @@ -116,12 +117,14 @@ func main() { setRocksDB(&cfg.RocksDB) var err error - db, err = store.Open(cfg) + ldb, err = ledis.Open(cfg) if err != nil { panic(err) return } + db, _ = ldb.Select(0) + if *number <= 0 { panic("invalid number") return From 8d16ac54c5f7ac85c2280de87d438b19ffd7926f Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 29 Oct 2014 10:29:35 +0800 Subject: [PATCH 11/98] refactor db getslice --- store/boltdb/db.go | 9 ------- store/db.go | 17 ++++++++++++ store/driver/driver.go | 5 +++- store/goleveldb/db.go | 8 ------ store/leveldb/db.go | 2 +- store/leveldb/slice.go | 43 ++++++++++++++++++++++++++++++ store/leveldb/snapshot.go | 4 +++ store/mdb/mdb.go | 9 ------- store/rocksdb/db.go | 2 +- store/{driver => rocksdb}/slice.go | 26 ++++-------------- store/rocksdb/snapshot.go | 4 +++ store/slice.go | 9 +++++++ store/snapshot.go | 17 ++++++++++++ store/stat.go | 2 +- store/store_test.go | 16 +++++++++++ 15 files changed, 122 insertions(+), 51 deletions(-) create mode 100644 store/leveldb/slice.go rename store/{driver => rocksdb}/slice.go (69%) create mode 100644 store/slice.go diff --git a/store/boltdb/db.go b/store/boltdb/db.go index 377cac5..ac8cc03 100644 --- a/store/boltdb/db.go +++ b/store/boltdb/db.go @@ -167,15 +167,6 @@ func (db *DB) Compact() error { return nil } -func (db *DB) GetSlice(key []byte) (driver.ISlice, error) { - v, err := db.Get(key) - if err != nil { - return nil, err - } else { - return driver.GoSlice(v), nil - } -} - func init() { driver.Register(Store{}) } diff --git a/store/db.go b/store/db.go index b2d116b..7e88342 100644 --- a/store/db.go +++ b/store/db.go @@ -156,3 +156,20 @@ func (db *DB) needSyncCommit() bool { } } + +func (db *DB) GetSlice(key []byte) (Slice, error) { + if d, ok := db.db.(driver.ISliceGeter); ok { + v, err := d.GetSlice(key) + db.st.statGet(v, err) + return v, err + } else { + v, err := db.Get(key) + if err != nil { + return nil, err + } else if v == nil { + return nil, nil + } else { + return driver.GoSlice(v), nil + } + } +} diff --git a/store/driver/driver.go b/store/driver/driver.go index 32e5a68..31d15de 100644 --- a/store/driver/driver.go +++ b/store/driver/driver.go @@ -12,7 +12,6 @@ type IDB interface { Close() error Get(key []byte) ([]byte, error) - GetSlice(key []byte) (ISlice, error) Put(key []byte, value []byte) error Delete(key []byte) error @@ -72,3 +71,7 @@ type Tx interface { Commit() error Rollback() error } + +type ISliceGeter interface { + GetSlice(key []byte) (ISlice, error) +} diff --git a/store/goleveldb/db.go b/store/goleveldb/db.go index 6b3fb1b..9924067 100644 --- a/store/goleveldb/db.go +++ b/store/goleveldb/db.go @@ -196,14 +196,6 @@ func (db *DB) Compact() error { return db.db.CompactRange(util.Range{nil, nil}) } -func (db *DB) GetSlice(key []byte) (driver.ISlice, error) { - v, err := db.Get(key) - if err != nil { - return nil, err - } else { - return driver.GoSlice(v), nil - } -} func init() { driver.Register(Store{}) driver.Register(MemStore{}) diff --git a/store/leveldb/db.go b/store/leveldb/db.go index 2de24b6..e5ab88c 100644 --- a/store/leveldb/db.go +++ b/store/leveldb/db.go @@ -276,7 +276,7 @@ func (db *DB) getSlice(ro *ReadOptions, key []byte) (driver.ISlice, error) { return nil, nil } - return driver.NewCSlice(unsafe.Pointer(value), int(vallen)), nil + return NewCSlice(unsafe.Pointer(value), int(vallen)), nil } func (db *DB) delete(wo *WriteOptions, key []byte) error { diff --git a/store/leveldb/slice.go b/store/leveldb/slice.go new file mode 100644 index 0000000..fb739ca --- /dev/null +++ b/store/leveldb/slice.go @@ -0,0 +1,43 @@ +// +build leveldb + +package leveldb + +// #cgo LDFLAGS: -lleveldb +// #include "leveldb/c.h" +import "C" + +import ( + "reflect" + "unsafe" +) + +type CSlice struct { + data unsafe.Pointer + size int +} + +func NewCSlice(p unsafe.Pointer, n int) *CSlice { + return &CSlice{p, n} +} + +func (s *CSlice) Data() []byte { + var value []byte + + sH := (*reflect.SliceHeader)(unsafe.Pointer(&value)) + sH.Cap = int(s.size) + sH.Len = int(s.size) + sH.Data = uintptr(s.data) + + return value +} + +func (s *CSlice) Size() int { + return int(s.size) +} + +func (s *CSlice) Free() { + if s.data != nil { + C.leveldb_free(s.data) + s.data = nil + } +} diff --git a/store/leveldb/snapshot.go b/store/leveldb/snapshot.go index e8e6ca7..bdc8d51 100644 --- a/store/leveldb/snapshot.go +++ b/store/leveldb/snapshot.go @@ -21,6 +21,10 @@ func (s *Snapshot) Get(key []byte) ([]byte, error) { return s.db.get(s.readOpts, key) } +func (s *Snapshot) GetSlice(key []byte) (driver.ISlice, error) { + return s.db.getSlice(s.readOpts, key) +} + func (s *Snapshot) NewIterator() driver.IIterator { it := new(Iterator) it.it = C.leveldb_create_iterator(s.db.db, s.db.iteratorOpts.Opt) diff --git a/store/mdb/mdb.go b/store/mdb/mdb.go index ed8f2d1..6bf26f6 100644 --- a/store/mdb/mdb.go +++ b/store/mdb/mdb.go @@ -311,15 +311,6 @@ func (db MDB) Compact() error { return nil } -func (db MDB) GetSlice(key []byte) (driver.ISlice, error) { - v, err := db.Get(key) - if err != nil { - return nil, err - } else { - return driver.GoSlice(v), nil - } -} - func init() { driver.Register(Store{}) } diff --git a/store/rocksdb/db.go b/store/rocksdb/db.go index 7ed572a..60816d3 100644 --- a/store/rocksdb/db.go +++ b/store/rocksdb/db.go @@ -303,7 +303,7 @@ func (db *DB) getSlice(ro *ReadOptions, key []byte) (driver.ISlice, error) { return nil, nil } - return driver.NewCSlice(unsafe.Pointer(value), int(vallen)), nil + return NewCSlice(unsafe.Pointer(value), int(vallen)), nil } func (db *DB) delete(wo *WriteOptions, key []byte) error { diff --git a/store/driver/slice.go b/store/rocksdb/slice.go similarity index 69% rename from store/driver/slice.go rename to store/rocksdb/slice.go index 76e4ea7..cbfba74 100644 --- a/store/driver/slice.go +++ b/store/rocksdb/slice.go @@ -1,5 +1,9 @@ -package driver +//+build rocksdb +package rocksdb + +// #cgo LDFLAGS: -lrocksdb +// #include // #include import "C" @@ -8,12 +12,6 @@ import ( "unsafe" ) -type ISlice interface { - Data() []byte - Size() int - Free() -} - type CSlice struct { data unsafe.Pointer size int @@ -44,17 +42,3 @@ func (s *CSlice) Free() { s.data = nil } } - -type GoSlice []byte - -func (s GoSlice) Data() []byte { - return []byte(s) -} - -func (s GoSlice) Size() int { - return len(s) -} - -func (s GoSlice) Free() { - -} diff --git a/store/rocksdb/snapshot.go b/store/rocksdb/snapshot.go index e560e8e..1ced600 100644 --- a/store/rocksdb/snapshot.go +++ b/store/rocksdb/snapshot.go @@ -21,6 +21,10 @@ func (s *Snapshot) Get(key []byte) ([]byte, error) { return s.db.get(s.readOpts, key) } +func (s *Snapshot) GetSlice(key []byte) (driver.ISlice, error) { + return s.db.getSlice(s.readOpts, key) +} + func (s *Snapshot) NewIterator() driver.IIterator { it := new(Iterator) it.it = C.rocksdb_create_iterator(s.db.db, s.db.iteratorOpts.Opt) diff --git a/store/slice.go b/store/slice.go new file mode 100644 index 0000000..b027f4f --- /dev/null +++ b/store/slice.go @@ -0,0 +1,9 @@ +package store + +import ( + "github.com/siddontang/ledisdb/store/driver" +) + +type Slice interface { + driver.ISlice +} diff --git a/store/snapshot.go b/store/snapshot.go index 80524ce..a1c9de9 100644 --- a/store/snapshot.go +++ b/store/snapshot.go @@ -25,6 +25,23 @@ func (s *Snapshot) Get(key []byte) ([]byte, error) { return v, err } +func (s *Snapshot) GetSlice(key []byte) (Slice, error) { + if d, ok := s.ISnapshot.(driver.ISliceGeter); ok { + v, err := d.GetSlice(key) + s.st.statGet(v, err) + return v, err + } else { + v, err := s.Get(key) + if err != nil { + return nil, err + } else if v == nil { + return nil, nil + } else { + return driver.GoSlice(v), nil + } + } +} + func (s *Snapshot) Close() { s.st.SnapshotCloseNum.Add(1) s.ISnapshot.Close() diff --git a/store/stat.go b/store/stat.go index 0b535d0..a14a135 100644 --- a/store/stat.go +++ b/store/stat.go @@ -23,7 +23,7 @@ type Stat struct { CompactTotalTime sync2.AtomicDuration } -func (st *Stat) statGet(v []byte, err error) { +func (st *Stat) statGet(v interface{}, err error) { st.GetNum.Add(1) if v == nil && err == nil { st.GetMissingNum.Add(1) diff --git a/store/store_test.go b/store/store_test.go index b488158..7626d5b 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -61,6 +61,16 @@ func testSimple(db *DB, t *testing.T) { t.Fatal("not equal") } + if v, err := db.GetSlice(key); err != nil { + t.Fatal(err) + } else if v == nil { + t.Fatal("must not nil") + } else if !bytes.Equal(v.Data(), value) { + t.Fatal("not equal") + } else { + v.Free() + } + if err := db.Delete(key); err != nil { t.Fatal(err) } @@ -70,6 +80,12 @@ func testSimple(db *DB, t *testing.T) { t.Fatal("must nil") } + if v, err := db.GetSlice(key); err != nil { + t.Fatal(err) + } else if v != nil { + t.Fatal("must nil") + } + if err := db.Put(key, nil); err != nil { t.Fatal(err) } From 522d94e31e90762d23bbef4e2e2fb9a6bff92f9e Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 29 Oct 2014 10:29:50 +0800 Subject: [PATCH 12/98] add go slice --- store/driver/slice.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 store/driver/slice.go diff --git a/store/driver/slice.go b/store/driver/slice.go new file mode 100644 index 0000000..d0c80e0 --- /dev/null +++ b/store/driver/slice.go @@ -0,0 +1,21 @@ +package driver + +type ISlice interface { + Data() []byte + Size() int + Free() +} + +type GoSlice []byte + +func (s GoSlice) Data() []byte { + return []byte(s) +} + +func (s GoSlice) Size() int { + return len(s) +} + +func (s GoSlice) Free() { + +} From 9282c7223ed611c75df179e3c6305177c5620764 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 29 Oct 2014 12:32:57 +0800 Subject: [PATCH 13/98] optimize benchmark thanks joe --- client/go/ledis/client.go | 6 --- client/go/ledis/conn.go | 3 -- cmd/ledis-benchmark/main.go | 99 +++++++++++++++++++------------------ 3 files changed, 51 insertions(+), 57 deletions(-) diff --git a/client/go/ledis/client.go b/client/go/ledis/client.go index f6b0fc4..bdc532f 100644 --- a/client/go/ledis/client.go +++ b/client/go/ledis/client.go @@ -4,11 +4,6 @@ import ( "container/list" "strings" "sync" - "time" -) - -const ( - pingPeriod time.Duration = 3 * time.Second ) type Config struct { @@ -98,7 +93,6 @@ func (c *Client) put(conn *Conn) { c.Unlock() conn.finalize() } else { - conn.lastActive = time.Now() c.conns.PushFront(conn) c.Unlock() } diff --git a/client/go/ledis/conn.go b/client/go/ledis/conn.go index 7568772..c74ab3f 100644 --- a/client/go/ledis/conn.go +++ b/client/go/ledis/conn.go @@ -8,7 +8,6 @@ import ( "io" "net" "strconv" - "time" ) // Error represents an error returned in a command reply. @@ -28,8 +27,6 @@ type Conn struct { rSize int wSize int - lastActive time.Time - // Scratch space for formatting argument length. // '*' or '$', length, "\r\n" lenScratch [32]byte diff --git a/cmd/ledis-benchmark/main.go b/cmd/ledis-benchmark/main.go index 02f56eb..6437f10 100644 --- a/cmd/ledis-benchmark/main.go +++ b/cmd/ledis-benchmark/main.go @@ -22,29 +22,27 @@ var tests = flag.String("t", "", "only run the comma separated list of tests, se var wg sync.WaitGroup var client *ledis.Client - var loop int = 0 -func waitBench(cmd string, args ...interface{}) { - c := client.Get() - defer c.Close() - +func waitBench(c *ledis.Conn, cmd string, args ...interface{}) { _, err := c.Do(strings.ToUpper(cmd), args...) if err != nil { fmt.Printf("do %s error %s\n", cmd, err.Error()) - return } + } -func bench(cmd string, f func()) { +func bench(cmd string, f func(c *ledis.Conn)) { wg.Add(*clients) t1 := time.Now() for i := 0; i < *clients; i++ { go func() { + c := client.Get() for j := 0; j < loop; j++ { - f() + f(c) } + c.Close() wg.Done() }() } @@ -68,78 +66,78 @@ var kvIncrBase int64 = 0 var kvDelBase int64 = 0 func benchSet() { - f := func() { + f := func(c *ledis.Conn) { value := make([]byte, *valueSize) n := atomic.AddInt64(&kvSetBase, 1) - waitBench("set", n, value) + waitBench(c, "SET", n, value) } bench("set", f) } func benchGet() { - f := func() { + f := func(c *ledis.Conn) { n := atomic.AddInt64(&kvGetBase, 1) - waitBench("get", n) + waitBench(c, "GET", n) } bench("get", f) } func benchRandGet() { - f := func() { + f := func(c *ledis.Conn) { n := rand.Int() % *number - waitBench("get", n) + waitBench(c, "GET", n) } bench("randget", f) } func benchDel() { - f := func() { + f := func(c *ledis.Conn) { n := atomic.AddInt64(&kvDelBase, 1) - waitBench("del", n) + waitBench(c, "DEL", n) } bench("del", f) } func benchPushList() { - f := func() { + f := func(c *ledis.Conn) { value := make([]byte, 100) - waitBench("rpush", "mytestlist", value) + waitBench(c, "RPUSH", "mytestlist", value) } bench("rpush", f) } func benchRangeList10() { - f := func() { - waitBench("lrange", "mytestlist", 0, 10) + f := func(c *ledis.Conn) { + waitBench(c, "LRANGE", "mytestlist", 0, 10) } bench("lrange10", f) } func benchRangeList50() { - f := func() { - waitBench("lrange", "mytestlist", 0, 50) + f := func(c *ledis.Conn) { + waitBench(c, "LRANGE", "mytestlist", 0, 50) } bench("lrange50", f) } func benchRangeList100() { - f := func() { - waitBench("lrange", "mytestlist", 0, 100) + f := func(c *ledis.Conn) { + waitBench(c, "LRANGE", "mytestlist", 0, 100) } bench("lrange100", f) } func benchPopList() { - f := func() { - waitBench("lpop", "mytestlist") + f := func(c *ledis.Conn) { + waitBench(c, "LPOP", "mytestlist") } bench("lpop", f) @@ -151,38 +149,38 @@ var hashGetBase int64 = 0 var hashDelBase int64 = 0 func benchHset() { - f := func() { + f := func(c *ledis.Conn) { value := make([]byte, 100) n := atomic.AddInt64(&hashSetBase, 1) - waitBench("hset", "myhashkey", n, value) + waitBench(c, "HSET", "myhashkey", n, value) } bench("hset", f) } func benchHGet() { - f := func() { + f := func(c *ledis.Conn) { n := atomic.AddInt64(&hashGetBase, 1) - waitBench("hget", "myhashkey", n) + waitBench(c, "HGET", "myhashkey", n) } bench("hget", f) } func benchHRandGet() { - f := func() { + f := func(c *ledis.Conn) { n := rand.Int() % *number - waitBench("hget", "myhashkey", n) + waitBench(c, "HGET", "myhashkey", n) } bench("hrandget", f) } func benchHDel() { - f := func() { + f := func(c *ledis.Conn) { n := atomic.AddInt64(&hashDelBase, 1) - waitBench("hdel", "myhashkey", n) + waitBench(c, "HDEL", "myhashkey", n) } bench("hdel", f) @@ -193,60 +191,60 @@ var zsetDelBase int64 = 0 var zsetIncrBase int64 = 0 func benchZAdd() { - f := func() { + f := func(c *ledis.Conn) { member := make([]byte, 16) n := atomic.AddInt64(&zsetAddBase, 1) - waitBench("zadd", "myzsetkey", n, member) + waitBench(c, "ZADD", "myzsetkey", n, member) } bench("zadd", f) } func benchZDel() { - f := func() { + f := func(c *ledis.Conn) { n := atomic.AddInt64(&zsetDelBase, 1) - waitBench("zrem", "myzsetkey", n) + waitBench(c, "ZREM", "myzsetkey", n) } bench("zrem", f) } func benchZIncr() { - f := func() { + f := func(c *ledis.Conn) { n := atomic.AddInt64(&zsetIncrBase, 1) - waitBench("zincrby", "myzsetkey", 1, n) + waitBench(c, "ZINCRBY", "myzsetkey", 1, n) } bench("zincrby", f) } func benchZRangeByScore() { - f := func() { - waitBench("zrangebyscore", "myzsetkey", 0, rand.Int(), "withscores", "limit", rand.Int()%100, 100) + f := func(c *ledis.Conn) { + waitBench(c, "ZRANGEBYSCORE", "myzsetkey", 0, rand.Int(), "withscores", "limit", rand.Int()%100, 100) } bench("zrangebyscore", f) } func benchZRangeByRank() { - f := func() { - waitBench("zrange", "myzsetkey", 0, rand.Int()%100) + f := func(c *ledis.Conn) { + waitBench(c, "ZRANGE", "myzsetkey", 0, rand.Int()%100) } bench("zrange", f) } func benchZRevRangeByScore() { - f := func() { - waitBench("zrevrangebyscore", "myzsetkey", 0, rand.Int(), "withscores", "limit", rand.Int()%100, 100) + f := func(c *ledis.Conn) { + waitBench(c, "ZREVRANGEBYSCORE", "myzsetkey", 0, rand.Int(), "withscores", "limit", rand.Int()%100, 100) } bench("zrevrangebyscore", f) } func benchZRevRangeByRank() { - f := func() { - waitBench("zrevrange", "myzsetkey", 0, rand.Int()%100) + f := func(c *ledis.Conn) { + waitBench(c, "ZREVRANGE", "myzsetkey", 0, rand.Int()%100) } bench("zrevrange", f) @@ -278,6 +276,11 @@ func main() { cfg.WriteBufferSize = 10240 client = ledis.NewClient(cfg) + for i := 0; i < *clients; i++ { + c := client.Get() + c.Close() + } + if *round <= 0 { *round = 1 } From 21fee60d8bc8aca4bf49d9beaedc34699f26fe03 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 29 Oct 2014 16:02:46 +0800 Subject: [PATCH 14/98] optimize and bugfix TTL, must upgrade manually --- cmd/ledis-server/main.go | 3 + config/config.go | 3 + config/config.toml | 4 ++ ledis/const.go | 9 ++- ledis/ledis.go | 36 ++++++---- ledis/ledis_db.go | 13 ---- ledis/t_ttl.go | 97 +++++++++++++++++---------- upgrade/ledis-upgrade-ttl/main.go | 106 ++++++++++++++++++++++++++++++ 8 files changed, 209 insertions(+), 62 deletions(-) create mode 100644 upgrade/ledis-upgrade-ttl/main.go diff --git a/cmd/ledis-server/main.go b/cmd/ledis-server/main.go index 32be6c2..0ba897c 100644 --- a/cmd/ledis-server/main.go +++ b/cmd/ledis-server/main.go @@ -24,6 +24,7 @@ var slaveof = flag.String("slaveof", "", "make the server a slave of another ins var readonly = flag.Bool("readonly", false, "set readonly mode, salve server is always readonly") var rpl = flag.Bool("rpl", false, "enable replication or not, slave server is always enabled") var rplSync = flag.Bool("rpl_sync", false, "enable sync replication or not") +var ttlCheck = flag.Int("ttl_check", 1, "TTL check interval") func main() { runtime.GOMAXPROCS(runtime.NumCPU()) @@ -67,6 +68,8 @@ func main() { cfg.Replication.Sync = *rplSync } + cfg.TTLCheckInterval = *ttlCheck + var app *server.App app, err = server.NewApp(cfg) if err != nil { diff --git a/config/config.go b/config/config.go index 5353ffc..3d197e3 100644 --- a/config/config.go +++ b/config/config.go @@ -113,6 +113,8 @@ type Config struct { ConnReadBufferSize int `toml:"conn_read_buffer_size"` ConnWriteBufferSize int `toml:"conn_write_buffer_size"` + + TTLCheckInterval int `toml:"ttl_check_interval"` } func NewConfigWithFile(fileName string) (*Config, error) { @@ -196,6 +198,7 @@ func (cfg *Config) adjust() { cfg.Replication.ExpiredLogDays = getDefault(7, cfg.Replication.ExpiredLogDays) cfg.ConnReadBufferSize = getDefault(4*KB, cfg.ConnReadBufferSize) cfg.ConnWriteBufferSize = getDefault(4*KB, cfg.ConnWriteBufferSize) + cfg.TTLCheckInterval = getDefault(1, cfg.TTLCheckInterval) } func (cfg *LevelDBConfig) adjust() { diff --git a/config/config.toml b/config/config.toml index 2b5d545..fc227f4 100644 --- a/config/config.toml +++ b/config/config.toml @@ -47,6 +47,10 @@ use_replication = false conn_read_buffer_size = 10240 conn_write_buffer_size = 10240 +# checking TTL (time to live) data every n seconds +# if you set big, the expired data may not be deleted immediately +ttl_check_interval = 1 + [leveldb] # for leveldb and goleveldb compression = false diff --git a/ledis/const.go b/ledis/const.go index 3e17a95..1c5eedd 100644 --- a/ledis/const.go +++ b/ledis/const.go @@ -21,8 +21,13 @@ const ( maxDataType byte = 100 - ExpTimeType byte = 101 - ExpMetaType byte = 102 + /* + I make a big mistake about TTL time key format and have to use a new one (change 101 to 103). + You must run the ledis-upgrade-ttl to upgrade db. + */ + ObsoleteExpTimeType byte = 101 + ExpMetaType byte = 102 + ExpTimeType byte = 103 MetaType byte = 201 ) diff --git a/ledis/ledis.go b/ledis/ledis.go index 10fa49c..a5e926e 100644 --- a/ledis/ledis.go +++ b/ledis/ledis.go @@ -34,6 +34,8 @@ type Ledis struct { commitLock sync.Mutex //allow one write commit at same time lock io.Closer + + tcs [MaxDBNumber]*ttlChecker } func Open(cfg *config.Config) (*Ledis, error) { @@ -81,7 +83,7 @@ func Open(cfg *config.Config) (*Ledis, error) { } l.wg.Add(1) - go l.onDataExpired() + go l.checkTTL() return l, nil } @@ -165,18 +167,28 @@ func (l *Ledis) IsReadOnly() bool { return false } -func (l *Ledis) onDataExpired() { +func (l *Ledis) checkTTL() { defer l.wg.Done() - var executors []*elimination = make([]*elimination, len(l.dbs)) for i, db := range l.dbs { - executors[i] = db.newEliminator() + c := newTTLChecker(db) + + c.register(KVType, db.kvBatch, db.delete) + c.register(ListType, db.listBatch, db.lDelete) + c.register(HashType, db.hashBatch, db.hDelete) + c.register(ZSetType, db.zsetBatch, db.zDelete) + c.register(BitType, db.binBatch, db.bDelete) + c.register(SetType, db.setBatch, db.sDelete) + + l.tcs[i] = c } - tick := time.NewTicker(1 * time.Second) - defer tick.Stop() + if l.cfg.TTLCheckInterval == 0 { + l.cfg.TTLCheckInterval = 1 + } - done := make(chan struct{}) + tick := time.NewTicker(time.Duration(l.cfg.TTLCheckInterval) * time.Second) + defer tick.Stop() for { select { @@ -185,13 +197,9 @@ func (l *Ledis) onDataExpired() { break } - go func() { - for _, eli := range executors { - eli.active() - } - done <- struct{}{} - }() - <-done + for _, c := range l.tcs { + c.check() + } case <-l.quit: return } diff --git a/ledis/ledis_db.go b/ledis/ledis_db.go index c9609fb..bd092a3 100644 --- a/ledis/ledis_db.go +++ b/ledis/ledis_db.go @@ -100,19 +100,6 @@ func (db *DB) FlushAll() (drop int64, err error) { return } -func (db *DB) newEliminator() *elimination { - eliminator := newEliminator(db) - - eliminator.regRetireContext(KVType, db.kvBatch, db.delete) - eliminator.regRetireContext(ListType, db.listBatch, db.lDelete) - eliminator.regRetireContext(HashType, db.hashBatch, db.hDelete) - eliminator.regRetireContext(ZSetType, db.zsetBatch, db.zDelete) - eliminator.regRetireContext(BitType, db.binBatch, db.bDelete) - eliminator.regRetireContext(SetType, db.setBatch, db.sDelete) - - return eliminator -} - func (db *DB) flushType(t *batch, dataType byte) (drop int64, err error) { var deleteFunc func(t *batch, key []byte) int64 var metaDataType byte diff --git a/ledis/t_ttl.go b/ledis/t_ttl.go index 2d5e2d5..a912d26 100644 --- a/ledis/t_ttl.go +++ b/ledis/t_ttl.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "errors" "github.com/siddontang/ledisdb/store" + "sync" "time" ) @@ -12,12 +13,16 @@ var ( errExpTimeKey = errors.New("invalid expire time key") ) -type retireCallback func(*batch, []byte) int64 +type onExpired func(*batch, []byte) int64 -type elimination struct { - db *DB - exp2Tx []*batch - exp2Retire []retireCallback +type ttlChecker struct { + sync.Mutex + db *DB + txs []*batch + cbs []onExpired + + //next check time + nc int64 } var errExpType = errors.New("invalid expire type") @@ -27,12 +32,14 @@ func (db *DB) expEncodeTimeKey(dataType byte, key []byte, when int64) []byte { buf[0] = db.index buf[1] = ExpTimeType - buf[2] = dataType - pos := 3 + pos := 2 binary.BigEndian.PutUint64(buf[pos:], uint64(when)) pos += 8 + buf[pos] = dataType + pos++ + copy(buf[pos:], key) return buf @@ -64,7 +71,7 @@ func (db *DB) expDecodeTimeKey(tk []byte) (byte, []byte, int64, error) { return 0, nil, 0, errExpTimeKey } - return tk[2], tk[11:], int64(binary.BigEndian.Uint64(tk[3:])), nil + return tk[10], tk[11:], int64(binary.BigEndian.Uint64(tk[2:])), nil } func (db *DB) expire(t *batch, dataType byte, key []byte, duration int64) { @@ -77,6 +84,8 @@ func (db *DB) expireAt(t *batch, dataType byte, key []byte, when int64) { t.Put(tk, mk) t.Put(mk, PutInt64(when)) + + db.l.tcs[db.index].setNextCheckTime(when, false) } func (db *DB) ttl(dataType byte, key []byte) (t int64, err error) { @@ -111,48 +120,68 @@ func (db *DB) rmExpire(t *batch, dataType byte, key []byte) (int64, error) { } } -////////////////////////////////////////////////////////// -// -////////////////////////////////////////////////////////// - -func newEliminator(db *DB) *elimination { - eli := new(elimination) - eli.db = db - eli.exp2Tx = make([]*batch, maxDataType) - eli.exp2Retire = make([]retireCallback, maxDataType) - return eli +func newTTLChecker(db *DB) *ttlChecker { + c := new(ttlChecker) + c.db = db + c.txs = make([]*batch, maxDataType) + c.cbs = make([]onExpired, maxDataType) + c.nc = 0 + return c } -func (eli *elimination) regRetireContext(dataType byte, t *batch, onRetire retireCallback) { - - // todo .. need to ensure exist - mapExpMetaType[expType] - - eli.exp2Tx[dataType] = t - eli.exp2Retire[dataType] = onRetire +func (c *ttlChecker) register(dataType byte, t *batch, f onExpired) { + c.txs[dataType] = t + c.cbs[dataType] = f } -// call by outside ... (from *db to another *db) -func (eli *elimination) active() { +func (c *ttlChecker) setNextCheckTime(when int64, force bool) { + c.Lock() + if force { + c.nc = when + } else if !force && c.nc > when { + c.nc = when + } + c.Unlock() +} + +func (c *ttlChecker) check() { now := time.Now().Unix() - db := eli.db + + c.Lock() + nc := c.nc + c.Unlock() + + if now < nc { + return + } + + nc = now + 3600 + + db := c.db dbGet := db.bucket.Get minKey := db.expEncodeTimeKey(NoneType, nil, 0) - maxKey := db.expEncodeTimeKey(maxDataType, nil, now) + maxKey := db.expEncodeTimeKey(maxDataType, nil, nc) it := db.bucket.RangeLimitIterator(minKey, maxKey, store.RangeROpen, 0, -1) for ; it.Valid(); it.Next() { tk := it.RawKey() mk := it.RawValue() - dt, k, _, err := db.expDecodeTimeKey(tk) + dt, k, nt, err := db.expDecodeTimeKey(tk) if err != nil { continue } - t := eli.exp2Tx[dt] - onRetire := eli.exp2Retire[dt] - if tk == nil || onRetire == nil { + if nt > now { + //the next ttl check time is nt! + nc = nt + break + } + + t := c.txs[dt] + cb := c.cbs[dt] + if tk == nil || cb == nil { continue } @@ -161,7 +190,7 @@ func (eli *elimination) active() { if exp, err := Int64(dbGet(mk)); err == nil { // check expire again if exp <= now { - onRetire(t, k) + cb(t, k) t.Delete(tk) t.Delete(mk) @@ -174,5 +203,7 @@ func (eli *elimination) active() { } it.Close() + c.setNextCheckTime(nc, true) + return } diff --git a/upgrade/ledis-upgrade-ttl/main.go b/upgrade/ledis-upgrade-ttl/main.go new file mode 100644 index 0000000..ea8dab1 --- /dev/null +++ b/upgrade/ledis-upgrade-ttl/main.go @@ -0,0 +1,106 @@ +package main + +import ( + "encoding/binary" + "flag" + "fmt" + "github.com/siddontang/ledisdb/config" + "github.com/siddontang/ledisdb/ledis" + "github.com/siddontang/ledisdb/store" +) + +var configPath = flag.String("config", "", "ledisdb config file") + +func main() { + flag.Parse() + + if len(*configPath) == 0 { + println("need ledis config file") + return + } + + cfg, err := config.NewConfigWithFile(*configPath) + if err != nil { + println(err.Error()) + return + } + + db, err := store.Open(cfg) + if err != nil { + println(err.Error()) + return + } + + // upgrade: ttl time key 101 to ttl time key 103 + + wb := db.NewWriteBatch() + + for i := uint8(0); i < ledis.MaxDBNumber; i++ { + minK, maxK := oldKeyPair(i) + + it := db.RangeIterator(minK, maxK, store.RangeROpen) + num := 0 + for ; it.Valid(); it.Next() { + dt, k, t, err := decodeOldKey(i, it.RawKey()) + if err != nil { + continue + } + + newKey := encodeNewKey(i, dt, k, t) + + wb.Put(newKey, it.RawValue()) + wb.Delete(it.RawKey()) + num++ + if num%1024 == 0 { + if err := wb.Commit(); err != nil { + fmt.Printf("commit error :%s\n", err.Error()) + } + } + } + it.Close() + + if err := wb.Commit(); err != nil { + fmt.Printf("commit error :%s\n", err.Error()) + } + } +} + +func oldKeyPair(index uint8) ([]byte, []byte) { + minB := make([]byte, 11) + minB[0] = index + minB[1] = ledis.ObsoleteExpTimeType + minB[2] = 0 + + maxB := make([]byte, 11) + maxB[0] = index + maxB[1] = ledis.ObsoleteExpTimeType + maxB[2] = 255 + + return minB, maxB +} + +func decodeOldKey(index uint8, tk []byte) (byte, []byte, int64, error) { + if len(tk) < 11 || tk[0] != index || tk[1] != ledis.ObsoleteExpTimeType { + return 0, nil, 0, fmt.Errorf("invalid exp time key") + } + + return tk[2], tk[11:], int64(binary.BigEndian.Uint64(tk[3:])), nil +} + +func encodeNewKey(index uint8, dataType byte, key []byte, when int64) []byte { + buf := make([]byte, len(key)+11) + + buf[0] = index + buf[1] = ledis.ExpTimeType + pos := 2 + + binary.BigEndian.PutUint64(buf[pos:], uint64(when)) + pos += 8 + + buf[pos] = dataType + pos++ + + copy(buf[pos:], key) + + return buf +} From 5ee3a4ac2ca7af2fe2fadf4cbe0c979408174be4 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 29 Oct 2014 16:07:58 +0800 Subject: [PATCH 15/98] add flag upgrade-ttl --- upgrade/ledis-upgrade-ttl/main.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/upgrade/ledis-upgrade-ttl/main.go b/upgrade/ledis-upgrade-ttl/main.go index ea8dab1..96d1a28 100644 --- a/upgrade/ledis-upgrade-ttl/main.go +++ b/upgrade/ledis-upgrade-ttl/main.go @@ -10,6 +10,8 @@ import ( ) var configPath = flag.String("config", "", "ledisdb config file") +var dataDir = flag.String("data_dir", "", "ledisdb base data dir") +var dbName = flag.String("db_name", "", "select a db to use, it will overwrite the config's db name") func main() { flag.Parse() @@ -25,6 +27,14 @@ func main() { return } + if len(*dataDir) > 0 { + cfg.DataDir = *dataDir + } + + if len(*dbName) > 0 { + cfg.DBName = *dbName + } + db, err := store.Open(cfg) if err != nil { println(err.Error()) From 6572f3c719321cb6ab4a1a46711348862dcbcf69 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 29 Oct 2014 17:52:31 +0800 Subject: [PATCH 16/98] update benchmark --- cmd/ledis-benchmark/main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/ledis-benchmark/main.go b/cmd/ledis-benchmark/main.go index 6437f10..5ac7e98 100644 --- a/cmd/ledis-benchmark/main.go +++ b/cmd/ledis-benchmark/main.go @@ -18,7 +18,7 @@ var number = flag.Int("n", 1000, "request number") var clients = flag.Int("c", 50, "number of clients") var round = flag.Int("r", 1, "benchmark round number") var valueSize = flag.Int("vsize", 100, "kv value size") -var tests = flag.String("t", "", "only run the comma separated list of tests, set,get,del,lpush,lrange,lpop,hset,hget,hdel,zadd,zincr,zrange,zrevrange,zdel") +var tests = flag.String("t", "", "only run the comma separated list of tests, set,get,randget,del,lpush,lrange,lpop,hset,hget,hdel,zadd,zincr,zrange,zrevrange,zdel") var wg sync.WaitGroup var client *ledis.Client @@ -312,6 +312,9 @@ func main() { if checkTest("get") { benchGet() + } + + if checkTest("rangeget") { benchRandGet() } From a2c98cf359a687b0117ed9796cd8601a26a657b8 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 29 Oct 2014 20:16:24 +0800 Subject: [PATCH 17/98] bugfix ledis-dbbench --- cmd/ledis-dbbench/main.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/ledis-dbbench/main.go b/cmd/ledis-dbbench/main.go index 7f02d04..bfacc4c 100644 --- a/cmd/ledis-dbbench/main.go +++ b/cmd/ledis-dbbench/main.go @@ -105,9 +105,12 @@ func main() { flag.Parse() cfg := config.NewConfigDefault() - cfg.DBPath = "./var/db_test" + cfg.DataDir = "./var/ledis_dbbench" cfg.DBName = *name os.RemoveAll(cfg.DBPath) + defer os.RemoveAll(cfg.DBPath) + + os.MkdirAll(cfg.DBPath, 0755) cfg.LevelDB.BlockSize = 32 * KB cfg.LevelDB.CacheSize = 512 * MB @@ -119,7 +122,7 @@ func main() { var err error ldb, err = ledis.Open(cfg) if err != nil { - panic(err) + println(err.Error()) return } From 4c19e401595f8ceb7922461ffec4e633f3410895 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 29 Oct 2014 20:17:12 +0800 Subject: [PATCH 18/98] add GetSlice, hope can improve get performance --- ledis/ledis_db.go | 1 + ledis/t_kv.go | 11 +++++++++++ server/cmd_kv.go | 23 +++++++++++++++++++++-- store/leveldb/slice.go | 5 +---- store/rocksdb/slice.go | 5 +---- store/tx.go | 10 ++++++++++ 6 files changed, 45 insertions(+), 10 deletions(-) diff --git a/ledis/ledis_db.go b/ledis/ledis_db.go index bd092a3..ebde98e 100644 --- a/ledis/ledis_db.go +++ b/ledis/ledis_db.go @@ -8,6 +8,7 @@ import ( type ibucket interface { Get(key []byte) ([]byte, error) + GetSlice(key []byte) (store.Slice, error) Put(key []byte, value []byte) error Delete(key []byte) error diff --git a/ledis/t_kv.go b/ledis/t_kv.go index 2a55425..37315c1 100644 --- a/ledis/t_kv.go +++ b/ledis/t_kv.go @@ -3,6 +3,7 @@ package ledis import ( "errors" "github.com/siddontang/go/num" + "github.com/siddontang/ledisdb/store" "time" ) @@ -164,6 +165,16 @@ func (db *DB) Get(key []byte) ([]byte, error) { return db.bucket.Get(key) } +func (db *DB) GetSlice(key []byte) (store.Slice, error) { + if err := checkKeySize(key); err != nil { + return nil, err + } + + key = db.encodeKVKey(key) + + return db.bucket.GetSlice(key) +} + func (db *DB) GetSet(key []byte, value []byte) ([]byte, error) { if err := checkKeySize(key); err != nil { return nil, err diff --git a/server/cmd_kv.go b/server/cmd_kv.go index 0d2dc25..4e7d601 100644 --- a/server/cmd_kv.go +++ b/server/cmd_kv.go @@ -7,16 +7,35 @@ import ( "strings" ) +// func getCommand(c *client) error { +// args := c.args +// if len(args) != 1 { +// return ErrCmdParams +// } + +// if v, err := c.db.Get(args[0]); err != nil { +// return err +// } else { +// c.resp.writeBulk(v) +// } +// return nil +// } + func getCommand(c *client) error { args := c.args if len(args) != 1 { return ErrCmdParams } - if v, err := c.db.Get(args[0]); err != nil { + if v, err := c.db.GetSlice(args[0]); err != nil { return err } else { - c.resp.writeBulk(v) + if v == nil { + c.resp.writeBulk(nil) + } else { + c.resp.writeBulk(v.Data()) + v.Free() + } } return nil } diff --git a/store/leveldb/slice.go b/store/leveldb/slice.go index fb739ca..83ebf55 100644 --- a/store/leveldb/slice.go +++ b/store/leveldb/slice.go @@ -36,8 +36,5 @@ func (s *CSlice) Size() int { } func (s *CSlice) Free() { - if s.data != nil { - C.leveldb_free(s.data) - s.data = nil - } + C.leveldb_free(s.data) } diff --git a/store/rocksdb/slice.go b/store/rocksdb/slice.go index cbfba74..bbaa65b 100644 --- a/store/rocksdb/slice.go +++ b/store/rocksdb/slice.go @@ -37,8 +37,5 @@ func (s *CSlice) Size() int { } func (s *CSlice) Free() { - if s.data != nil { - C.free(s.data) - s.data = nil - } + C.free(s.data) } diff --git a/store/tx.go b/store/tx.go index 6845ee5..4dbf311 100644 --- a/store/tx.go +++ b/store/tx.go @@ -56,6 +56,16 @@ func (tx *Tx) Get(key []byte) ([]byte, error) { return v, err } +func (tx *Tx) GetSlice(key []byte) (Slice, error) { + if v, err := tx.Get(key); err != nil { + return nil, err + } else if v == nil { + return nil, nil + } else { + return driver.GoSlice(v), nil + } +} + func (tx *Tx) Put(key []byte, value []byte) error { tx.st.PutNum.Add(1) return tx.tx.Put(key, value) From d7c59f337d18f5d94dd79274f1c4547923b65983 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 29 Oct 2014 21:13:56 +0800 Subject: [PATCH 19/98] update config --- etc/ledis.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/etc/ledis.conf b/etc/ledis.conf index 2b5d545..fc227f4 100644 --- a/etc/ledis.conf +++ b/etc/ledis.conf @@ -47,6 +47,10 @@ use_replication = false conn_read_buffer_size = 10240 conn_write_buffer_size = 10240 +# checking TTL (time to live) data every n seconds +# if you set big, the expired data may not be deleted immediately +ttl_check_interval = 1 + [leveldb] # for leveldb and goleveldb compression = false From 796f4b3af20da63265a6999ad40c9c018a4158bf Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 30 Oct 2014 00:01:19 +0800 Subject: [PATCH 20/98] use arena to reduce object create --- Godeps/Godeps.json | 22 +++++++++++++--------- cmd/ledis-respbench/main.go | 28 ++++++++++------------------ server/client.go | 13 +++++++------ server/client_resp.go | 26 +++++++++++++++++++++++++- server/util.go | 9 +++++---- 5 files changed, 60 insertions(+), 38 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index de55e1d..f4ab10d 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -14,41 +14,45 @@ "Comment": "data/v1-228-g8fb50d5", "Rev": "8fb50d5ee57110936b904a7539d4c5f2bf2359db" }, + { + "ImportPath": "github.com/siddontang/go/arena", + "Rev": "ecf49fc0738105e87d20e29aa82c403b666ff0b4" + }, { "ImportPath": "github.com/siddontang/go/bson", - "Rev": "c7a17e4e4a1b72e4bc38b8b52cac8558aff4a4b1" + "Rev": "ecf49fc0738105e87d20e29aa82c403b666ff0b4" }, { "ImportPath": "github.com/siddontang/go/filelock", - "Rev": "c7a17e4e4a1b72e4bc38b8b52cac8558aff4a4b1" + "Rev": "ecf49fc0738105e87d20e29aa82c403b666ff0b4" }, { "ImportPath": "github.com/siddontang/go/hack", - "Rev": "c7a17e4e4a1b72e4bc38b8b52cac8558aff4a4b1" + "Rev": "ecf49fc0738105e87d20e29aa82c403b666ff0b4" }, { "ImportPath": "github.com/siddontang/go/ioutil2", - "Rev": "c7a17e4e4a1b72e4bc38b8b52cac8558aff4a4b1" + "Rev": "ecf49fc0738105e87d20e29aa82c403b666ff0b4" }, { "ImportPath": "github.com/siddontang/go/log", - "Rev": "c7a17e4e4a1b72e4bc38b8b52cac8558aff4a4b1" + "Rev": "ecf49fc0738105e87d20e29aa82c403b666ff0b4" }, { "ImportPath": "github.com/siddontang/go/num", - "Rev": "c7a17e4e4a1b72e4bc38b8b52cac8558aff4a4b1" + "Rev": "ecf49fc0738105e87d20e29aa82c403b666ff0b4" }, { "ImportPath": "github.com/siddontang/go/snappy", - "Rev": "c7a17e4e4a1b72e4bc38b8b52cac8558aff4a4b1" + "Rev": "ecf49fc0738105e87d20e29aa82c403b666ff0b4" }, { "ImportPath": "github.com/siddontang/go/sync2", - "Rev": "c7a17e4e4a1b72e4bc38b8b52cac8558aff4a4b1" + "Rev": "ecf49fc0738105e87d20e29aa82c403b666ff0b4" }, { "ImportPath": "github.com/siddontang/goleveldb/leveldb", - "Rev": "71404b29ccd98b94ec2278afa806d59a11cd0d28" + "Rev": "c1f6d721561c48f467b26a277741e55fd224df1e" }, { "ImportPath": "github.com/szferi/gomdb", diff --git a/cmd/ledis-respbench/main.go b/cmd/ledis-respbench/main.go index 7e657c0..e21d173 100644 --- a/cmd/ledis-respbench/main.go +++ b/cmd/ledis-respbench/main.go @@ -5,10 +5,10 @@ import ( "bytes" "flag" "fmt" + "github.com/siddontang/go/arena" "github.com/siddontang/ledisdb/server" "net" "runtime" - "sync" "time" ) @@ -37,12 +37,12 @@ func main() { } } -var m = map[string][]byte{} -var mu sync.Mutex - func run(c net.Conn) { //buf := make([]byte, 10240) ok := []byte("+OK\r\n") + data := []byte("$4096\r\n") + data = append(data, make([]byte, 4096)...) + data = append(data, "\r\n"...) var rt time.Duration var wt time.Duration @@ -50,10 +50,14 @@ func run(c net.Conn) { rb := bufio.NewReaderSize(c, 10240) wb := bufio.NewWriterSize(c, 10240) + a := arena.NewArena(10240) + for { t1 := time.Now() - req, err := server.ReadRequest(rb) + a.Reset() + + req, err := server.ReadRequest(rb, a) if err != nil { break @@ -65,21 +69,9 @@ func run(c net.Conn) { cmd := string(bytes.ToUpper(req[0])) switch cmd { case "SET": - mu.Lock() - m[string(req[1])] = req[2] - mu.Unlock() wb.Write(ok) case "GET": - mu.Lock() - v := m[string(req[1])] - mu.Unlock() - if v == nil { - wb.WriteString("$-1\r\n") - } else { - wb.WriteString(fmt.Sprintf("$%d\r\n", len(v))) - wb.Write(v) - wb.WriteString("\r\n") - } + wb.Write(data) default: wb.WriteString(fmt.Sprintf("-Err %s Not Supported Now", req[0])) } diff --git a/server/client.go b/server/client.go index ba24821..7ba86ec 100644 --- a/server/client.go +++ b/server/client.go @@ -108,17 +108,18 @@ func (c *client) perform() { } if err == nil { - go func() { - c.reqErr <- exeCmd(c) - }() + // go func() { + // c.reqErr <- exeCmd(c) + // }() - err = <-c.reqErr + // err = <-c.reqErr + err = exeCmd(c) } } - duration := time.Since(start) - if c.app.access != nil { + duration := time.Since(start) + fullCmd := c.catGenericCommand() cost := duration.Nanoseconds() / 1000000 diff --git a/server/client_resp.go b/server/client_resp.go index f5f7768..00d0e95 100644 --- a/server/client_resp.go +++ b/server/client_resp.go @@ -3,6 +3,7 @@ package server import ( "bufio" "errors" + "github.com/siddontang/go/arena" "github.com/siddontang/go/hack" "github.com/siddontang/go/log" "github.com/siddontang/go/num" @@ -21,6 +22,8 @@ type respClient struct { conn net.Conn rb *bufio.Reader + + ar *arena.Arena } type respWriter struct { @@ -43,6 +46,9 @@ func newClientRESP(conn net.Conn, app *App) { c.resp = newWriterRESP(conn, app.cfg.ConnWriteBufferSize) c.remoteAddr = conn.RemoteAddr().String() + //maybe another config? + c.ar = arena.NewArena(app.cfg.ConnReadBufferSize) + go c.run() } @@ -71,18 +77,31 @@ func (c *respClient) run() { c.app.removeSlave(c.client, handleQuit) }() + // done := make(chan error) for { + // go func() { reqData, err := c.readRequest() if err != nil { + // done <- err return } c.handleRequest(reqData) + // done <- nil + // }() + + // err := <-done + // if err != nil { + // return + // } + // if c.conn == nil { + // return + // } } } func (c *respClient) readRequest() ([][]byte, error) { - return ReadRequest(c.rb) + return ReadRequest(c.rb, c.ar) } func (c *respClient) handleRequest(reqData [][]byte) { @@ -103,6 +122,11 @@ func (c *respClient) handleRequest(reqData [][]byte) { c.perform() + c.cmd = "" + c.args = nil + + c.ar.Reset() + return } diff --git a/server/util.go b/server/util.go index adeba69..a243c18 100644 --- a/server/util.go +++ b/server/util.go @@ -4,6 +4,7 @@ import ( "bufio" "errors" "fmt" + "github.com/siddontang/go/arena" "io" ) @@ -25,7 +26,7 @@ func ReadLine(rb *bufio.Reader) ([]byte, error) { return p[:i], nil } -func readBytes(br *bufio.Reader) (bytes []byte, err error) { +func readBytes(br *bufio.Reader, a *arena.Arena) (bytes []byte, err error) { size, err := readLong(br) if err != nil { return nil, err @@ -37,7 +38,7 @@ func readBytes(br *bufio.Reader) (bytes []byte, err error) { return nil, errors.New("Invalid size: " + fmt.Sprint("%d", size)) } - buf := make([]byte, size+2) + buf := a.Make(int(size) + 2) if _, err = io.ReadFull(br, buf); err != nil { return nil, err } @@ -90,7 +91,7 @@ func readLong(in *bufio.Reader) (result int64, err error) { return -1, err } -func ReadRequest(in *bufio.Reader) ([][]byte, error) { +func ReadRequest(in *bufio.Reader, a *arena.Arena) ([][]byte, error) { code, err := in.ReadByte() if err != nil { return nil, err @@ -115,7 +116,7 @@ func ReadRequest(in *bufio.Reader) ([][]byte, error) { return nil, errReadRequest } - if req[i], err = readBytes(in); err != nil { + if req[i], err = readBytes(in, a); err != nil { return nil, err } } From 2e3c51289f304f24d6db47611d1747a6601bde01 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 30 Oct 2014 08:27:08 +0800 Subject: [PATCH 21/98] update bootstrap --- bootstrap.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bootstrap.sh b/bootstrap.sh index ae18f27..f546a24 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -25,4 +25,5 @@ go get github.com/siddontang/go/log go get github.com/siddontang/go/snappy go get github.com/siddontang/go/num go get github.com/siddontang/go/filelock -go get github.com/siddontang/go/sync2 \ No newline at end of file +go get github.com/siddontang/go/sync2 +go get github.com/siddontang/go/arena \ No newline at end of file From 8133cfdbb7b397f0f811329eb234f9b2eaf92f48 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 30 Oct 2014 09:00:19 +0800 Subject: [PATCH 22/98] update server TTL check --- cmd/ledis-server/main.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/ledis-server/main.go b/cmd/ledis-server/main.go index 0ba897c..70328c4 100644 --- a/cmd/ledis-server/main.go +++ b/cmd/ledis-server/main.go @@ -24,7 +24,7 @@ var slaveof = flag.String("slaveof", "", "make the server a slave of another ins var readonly = flag.Bool("readonly", false, "set readonly mode, salve server is always readonly") var rpl = flag.Bool("rpl", false, "enable replication or not, slave server is always enabled") var rplSync = flag.Bool("rpl_sync", false, "enable sync replication or not") -var ttlCheck = flag.Int("ttl_check", 1, "TTL check interval") +var ttlCheck = flag.Int("ttl_check", 0, "TTL check interval") func main() { runtime.GOMAXPROCS(runtime.NumCPU()) @@ -68,7 +68,9 @@ func main() { cfg.Replication.Sync = *rplSync } - cfg.TTLCheckInterval = *ttlCheck + if *ttlCheck > 0 { + cfg.TTLCheckInterval = *ttlCheck + } var app *server.App app, err = server.NewApp(cfg) From fcc8c9ae37ca9b48478107c16ac84241ab985a99 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 30 Oct 2014 09:03:58 +0800 Subject: [PATCH 23/98] use goroutine for reap handling --- server/client.go | 4 ++-- server/client_resp.go | 37 +++++++++++++++++++++---------------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/server/client.go b/server/client.go index 7ba86ec..47d8f85 100644 --- a/server/client.go +++ b/server/client.go @@ -64,7 +64,7 @@ type client struct { lastLogID uint64 - reqErr chan error + // reqErr chan error buf bytes.Buffer @@ -82,7 +82,7 @@ func newClient(app *App) *client { c.ldb = app.ldb c.db, _ = app.ldb.Select(0) //use default db - c.reqErr = make(chan error) + // c.reqErr = make(chan error) return c } diff --git a/server/client_resp.go b/server/client_resp.go index 00d0e95..a7ec21c 100644 --- a/server/client_resp.go +++ b/server/client_resp.go @@ -77,26 +77,31 @@ func (c *respClient) run() { c.app.removeSlave(c.client, handleQuit) }() - // done := make(chan error) + done := make(chan error) for { - // go func() { - reqData, err := c.readRequest() + // I still don't know why use goroutine can improve performance + // if someone knows and benchamrks with another different result without goroutine, please tell me + go func() { + reqData, err := c.readRequest() + if err == nil { + c.handleRequest(reqData) + } + + done <- nil + }() + + // reqData, err := c.readRequest() + // if err == nil { + // c.handleRequest(reqData) + // } + + err := <-done if err != nil { - // done <- err return } - - c.handleRequest(reqData) - // done <- nil - // }() - - // err := <-done - // if err != nil { - // return - // } - // if c.conn == nil { - // return - // } + if c.conn == nil { + return + } } } From bfcec916a197f009f1b8950e2613071ea3d610b8 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 30 Oct 2014 09:32:53 +0800 Subject: [PATCH 24/98] bugfix benchmark for randget flag --- cmd/ledis-benchmark/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/ledis-benchmark/main.go b/cmd/ledis-benchmark/main.go index 5ac7e98..d5b5311 100644 --- a/cmd/ledis-benchmark/main.go +++ b/cmd/ledis-benchmark/main.go @@ -314,7 +314,7 @@ func main() { benchGet() } - if checkTest("rangeget") { + if checkTest("randget") { benchRandGet() } From 89c58e45ca0cfa0a24c33ad8f376162bb70c75bf Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 30 Oct 2014 10:51:02 +0800 Subject: [PATCH 25/98] add detailed men and gc info --- server/client_resp.go | 4 +++ server/info.go | 64 +++++++++++++++++++++++++++++++++---------- 2 files changed, 53 insertions(+), 15 deletions(-) diff --git a/server/client_resp.go b/server/client_resp.go index a7ec21c..2ce4f8d 100644 --- a/server/client_resp.go +++ b/server/client_resp.go @@ -53,7 +53,11 @@ func newClientRESP(conn net.Conn, app *App) { } func (c *respClient) run() { + c.app.info.addClients(1) + defer func() { + c.app.info.addClients(-1) + if e := recover(); e != nil { buf := make([]byte, 4096) n := runtime.Stack(buf, false) diff --git a/server/info.go b/server/info.go index ccdba98..bf69ace 100644 --- a/server/info.go +++ b/server/info.go @@ -6,9 +6,11 @@ import ( "github.com/siddontang/go/sync2" "os" "runtime" + "runtime/debug" "strings" "sync" "sync/atomic" + "time" ) type info struct { @@ -55,11 +57,11 @@ func (i *info) Close() { func getMemoryHuman(m uint64) string { if m > GB { - return fmt.Sprintf("%dG", m/GB) + return fmt.Sprintf("%0.3fG", float64(m)/float64(GB)) } else if m > MB { - return fmt.Sprintf("%dM", m/MB) + return fmt.Sprintf("%0.3fM", float64(m)/float64(MB)) } else if m > KB { - return fmt.Sprintf("%dK", m/KB) + return fmt.Sprintf("%0.3fK", float64(m)/float64(KB)) } else { return fmt.Sprintf("%d", m) } @@ -72,10 +74,10 @@ func (i *info) Dump(section string) []byte { i.dumpAll(buf) case "server": i.dumpServer(buf) - case "client": - i.dumpClients(buf) case "mem": i.dumpMem(buf) + case "gc": + i.dumpGC(buf) case "store": i.dumpStore(buf) case "replication": @@ -97,10 +99,10 @@ func (i *info) dumpAll(buf *bytes.Buffer) { buf.Write(Delims) i.dumpStore(buf) buf.Write(Delims) - i.dumpClients(buf) - buf.Write(Delims) i.dumpMem(buf) buf.Write(Delims) + i.dumpGC(buf) + buf.Write(Delims) i.dumpReplication(buf) } @@ -113,23 +115,55 @@ func (i *info) dumpServer(buf *bytes.Buffer) { infoPair{"http_addr", i.app.cfg.HttpAddr}, infoPair{"readonly", i.app.cfg.Readonly}, infoPair{"goroutine_num", runtime.NumGoroutine()}, + infoPair{"cgo_call_num", runtime.NumCgoCall()}, + infoPair{"client_num", i.Clients.ConnectedClients}, ) } -func (i *info) dumpClients(buf *bytes.Buffer) { - buf.WriteString("# Client\r\n") - - i.dumpPairs(buf, infoPair{"client_num", i.Clients.ConnectedClients}) -} - func (i *info) dumpMem(buf *bytes.Buffer) { buf.WriteString("# Mem\r\n") var mem runtime.MemStats runtime.ReadMemStats(&mem) - i.dumpPairs(buf, infoPair{"mem_alloc", mem.Alloc}, - infoPair{"mem_alloc_human", getMemoryHuman(mem.Alloc)}) + i.dumpPairs(buf, infoPair{"mem_alloc", getMemoryHuman(mem.Alloc)}, + infoPair{"mem_sys", getMemoryHuman(mem.Sys)}, + infoPair{"mem_looksups", getMemoryHuman(mem.Lookups)}, + infoPair{"mem_mallocs", getMemoryHuman(mem.Mallocs)}, + infoPair{"mem_frees", getMemoryHuman(mem.Frees)}, + infoPair{"mem_total", getMemoryHuman(mem.TotalAlloc)}, + infoPair{"mem_heap_alloc", getMemoryHuman(mem.HeapAlloc)}, + infoPair{"mem_heap_sys", getMemoryHuman(mem.HeapSys)}, + infoPair{"mem_head_idle", getMemoryHuman(mem.HeapIdle)}, + infoPair{"mem_head_inuse", getMemoryHuman(mem.HeapInuse)}, + infoPair{"mem_head_released", getMemoryHuman(mem.HeapReleased)}, + infoPair{"mem_head_objects", mem.HeapObjects}, + ) +} + +const ( + gcTimeFormat = "2006/01/02 15:04:05.000" +) + +func (i *info) dumpGC(buf *bytes.Buffer) { + count := 5 + + var st debug.GCStats + st.Pause = make([]time.Duration, count) + // st.PauseQuantiles = make([]time.Duration, count) + debug.ReadGCStats(&st) + + h := make([]string, 0, count) + + for i := 0; i < count && i < len(st.Pause); i++ { + h = append(h, st.Pause[i].String()) + } + + i.dumpPairs(buf, infoPair{"gc_last_time", st.LastGC.Format(gcTimeFormat)}, + infoPair{"gc_num", st.NumGC}, + infoPair{"gc_pause_total", st.PauseTotal.String()}, + infoPair{"gc_pause_history", strings.Join(h, ",")}, + ) } func (i *info) dumpStore(buf *bytes.Buffer) { From b9b60f19ad5b2a32fb9ef7b04edca0cd1b09f3e1 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 30 Oct 2014 11:11:45 +0800 Subject: [PATCH 26/98] add conn keep alive check and bug fix --- config/config.go | 5 +++-- config/config.toml | 5 +++++ etc/ledis.conf | 5 +++++ server/client_resp.go | 8 +++++++- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/config/config.go b/config/config.go index 3d197e3..f11c335 100644 --- a/config/config.go +++ b/config/config.go @@ -111,8 +111,9 @@ type Config struct { Snapshot SnapshotConfig `toml:"snapshot"` - ConnReadBufferSize int `toml:"conn_read_buffer_size"` - ConnWriteBufferSize int `toml:"conn_write_buffer_size"` + ConnReadBufferSize int `toml:"conn_read_buffer_size"` + ConnWriteBufferSize int `toml:"conn_write_buffer_size"` + ConnKeepaliveInterval int `toml:"conn_keepavlie_interval"` TTLCheckInterval int `toml:"ttl_check_interval"` } diff --git a/config/config.toml b/config/config.toml index fc227f4..6b6722b 100644 --- a/config/config.toml +++ b/config/config.toml @@ -44,9 +44,14 @@ db_sync_commit = 0 use_replication = false # set connection buffer, you can increase them appropriately +# more size, more memory used conn_read_buffer_size = 10240 conn_write_buffer_size = 10240 +# if connection receives no data after n seconds, it may be dead, close +# 0 to disable and not check +conn_keepavlie_interval = 0 + # checking TTL (time to live) data every n seconds # if you set big, the expired data may not be deleted immediately ttl_check_interval = 1 diff --git a/etc/ledis.conf b/etc/ledis.conf index fc227f4..6b6722b 100644 --- a/etc/ledis.conf +++ b/etc/ledis.conf @@ -44,9 +44,14 @@ db_sync_commit = 0 use_replication = false # set connection buffer, you can increase them appropriately +# more size, more memory used conn_read_buffer_size = 10240 conn_write_buffer_size = 10240 +# if connection receives no data after n seconds, it may be dead, close +# 0 to disable and not check +conn_keepavlie_interval = 0 + # checking TTL (time to live) data every n seconds # if you set big, the expired data may not be deleted immediately ttl_check_interval = 1 diff --git a/server/client_resp.go b/server/client_resp.go index 2ce4f8d..30a8f87 100644 --- a/server/client_resp.go +++ b/server/client_resp.go @@ -13,6 +13,7 @@ import ( "runtime" "strconv" "strings" + "time" ) var errReadRequest = errors.New("invalid request protocol") @@ -81,8 +82,13 @@ func (c *respClient) run() { c.app.removeSlave(c.client, handleQuit) }() + kc := time.Duration(c.app.cfg.ConnKeepaliveInterval) * time.Second done := make(chan error) for { + if kc > 0 { + c.conn.SetReadDeadline(time.Now().Add(kc)) + } + // I still don't know why use goroutine can improve performance // if someone knows and benchamrks with another different result without goroutine, please tell me go func() { @@ -91,7 +97,7 @@ func (c *respClient) run() { c.handleRequest(reqData) } - done <- nil + done <- err }() // reqData, err := c.readRequest() From d0bd0c17db5f938f99a316919b8503722602df5d Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 30 Oct 2014 11:21:09 +0800 Subject: [PATCH 27/98] update info gc --- server/info.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/info.go b/server/info.go index bf69ace..d899d4f 100644 --- a/server/info.go +++ b/server/info.go @@ -146,6 +146,8 @@ const ( ) func (i *info) dumpGC(buf *bytes.Buffer) { + buf.WriteString("# GC\r\n") + count := 5 var st debug.GCStats From b47e49d6fd379d358ef0078b23fd26cd02500295 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 30 Oct 2014 11:43:04 +0800 Subject: [PATCH 28/98] try reduce rocksdb batch clear --- store/rocksdb/batch.go | 19 +++++++++++++++---- store/rocksdb/rocksdb_ext.cc | 8 ++++++++ store/rocksdb/rocksdb_ext.h | 2 +- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/store/rocksdb/batch.go b/store/rocksdb/batch.go index 017fc88..6a94a63 100644 --- a/store/rocksdb/batch.go +++ b/store/rocksdb/batch.go @@ -4,6 +4,7 @@ package rocksdb // #cgo LDFLAGS: -lrocksdb // #include "rocksdb/c.h" +// #include "rocksdb_ext.h" import "C" import ( @@ -11,8 +12,9 @@ import ( ) type WriteBatch struct { - db *DB - wbatch *C.rocksdb_writebatch_t + db *DB + wbatch *C.rocksdb_writebatch_t + commitOk bool } func (w *WriteBatch) Close() error { @@ -22,6 +24,8 @@ func (w *WriteBatch) Close() error { } func (w *WriteBatch) Put(key, value []byte) { + w.commitOk = false + var k, v *C.char if len(key) != 0 { k = (*C.char)(unsafe.Pointer(&key[0])) @@ -37,6 +41,8 @@ func (w *WriteBatch) Put(key, value []byte) { } func (w *WriteBatch) Delete(key []byte) { + w.commitOk = false + C.rocksdb_writebatch_delete(w.wbatch, (*C.char)(unsafe.Pointer(&key[0])), C.size_t(len(key))) } @@ -50,14 +56,19 @@ func (w *WriteBatch) SyncCommit() error { } func (w *WriteBatch) Rollback() error { - C.rocksdb_writebatch_clear(w.wbatch) + if !w.commitOk { + C.rocksdb_writebatch_clear(w.wbatch) + } return nil } func (w *WriteBatch) commit(wb *WriteOptions) error { + w.commitOk = true + var errStr *C.char - C.rocksdb_write(w.db.db, wb.Opt, w.wbatch, &errStr) + C.rocksdb_write_ext(w.db.db, wb.Opt, w.wbatch, &errStr) if errStr != nil { + w.commitOk = false return saveError(errStr) } return nil diff --git a/store/rocksdb/rocksdb_ext.cc b/store/rocksdb/rocksdb_ext.cc index 4a7720f..39036ab 100644 --- a/store/rocksdb/rocksdb_ext.cc +++ b/store/rocksdb/rocksdb_ext.cc @@ -32,5 +32,13 @@ unsigned char rocksdb_iter_prev_ext(rocksdb_iterator_t* iter) { return rocksdb_iter_valid(iter); } +void rocksdb_write_ext(rocksdb_t* db, + const rocksdb_writeoptions_t* options, + rocksdb_writebatch_t* batch, char** errptr) { + rocksdb_write(db, options, batch, errptr); + if(*errptr == NULL) { + rocksdb_writebatch_clear(batch); + } +} } \ No newline at end of file diff --git a/store/rocksdb/rocksdb_ext.h b/store/rocksdb/rocksdb_ext.h index 4938294..11cb653 100644 --- a/store/rocksdb/rocksdb_ext.h +++ b/store/rocksdb/rocksdb_ext.h @@ -15,7 +15,7 @@ extern unsigned char rocksdb_iter_seek_to_last_ext(rocksdb_iterator_t*); extern unsigned char rocksdb_iter_seek_ext(rocksdb_iterator_t*, const char* k, size_t klen); extern unsigned char rocksdb_iter_next_ext(rocksdb_iterator_t*); extern unsigned char rocksdb_iter_prev_ext(rocksdb_iterator_t*); - +extern void rocksdb_write_ext(rocksdb_t* db, const rocksdb_writeoptions_t* options, rocksdb_writebatch_t* batch, char** errptr); #ifdef __cplusplus } From 44e93ba9212f1ded1d70a70f2b4ce1846e149645 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 30 Oct 2014 12:48:52 +0800 Subject: [PATCH 29/98] optimize codes --- server/client_resp.go | 3 +-- server/util.go | 11 +++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/server/client_resp.go b/server/client_resp.go index 30a8f87..cfcaf5d 100644 --- a/server/client_resp.go +++ b/server/client_resp.go @@ -12,7 +12,6 @@ import ( "net" "runtime" "strconv" - "strings" "time" ) @@ -124,7 +123,7 @@ func (c *respClient) handleRequest(reqData [][]byte) { c.cmd = "" c.args = reqData[0:0] } else { - c.cmd = strings.ToLower(hack.String(reqData[0])) + c.cmd = hack.String(lowerSlice(reqData[0])) c.args = reqData[1:] } if c.cmd == "quit" { diff --git a/server/util.go b/server/util.go index a243c18..7c5b73d 100644 --- a/server/util.go +++ b/server/util.go @@ -123,3 +123,14 @@ func ReadRequest(in *bufio.Reader, a *arena.Arena) ([][]byte, error) { return req, nil } + +func lowerSlice(buf []byte) []byte { + for i, r := range buf { + if 'A' <= r && r <= 'Z' { + r += 'a' - 'A' + } + + buf[i] = r + } + return buf +} From bd9e546da4399d3b3892999e689e97a994f33122 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 30 Oct 2014 14:15:49 +0800 Subject: [PATCH 30/98] add get slice for dbbench --- cmd/ledis-dbbench/main.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/cmd/ledis-dbbench/main.go b/cmd/ledis-dbbench/main.go index bfacc4c..b2183ce 100644 --- a/cmd/ledis-dbbench/main.go +++ b/cmd/ledis-dbbench/main.go @@ -70,6 +70,7 @@ func benchSet() { } func benchGet() { + kvGetBase = 0 f := func() { n := atomic.AddInt64(&kvGetBase, 1) v, err := db.Get(num.Int64ToBytes(n)) @@ -83,6 +84,23 @@ func benchGet() { bench("get", f) } +var kvGetSliceBase int64 = 0 + +func benchGetSlice() { + kvGetSliceBase = 0 + f := func() { + n := atomic.AddInt64(&kvGetSliceBase, 1) + v, err := db.GetSlice(num.Int64ToBytes(n)) + if err != nil { + println(err.Error()) + } else if v != nil { + v.Free() + } + } + + bench("getslice", f) +} + func setRocksDB(cfg *config.RocksDBConfig) { cfg.BlockSize = 64 * KB cfg.WriteBufferSize = 64 * MB @@ -147,6 +165,9 @@ func main() { for i := 0; i < *round; i++ { benchSet() benchGet() + benchGetSlice() + benchGet() + benchGetSlice() println("") } From 46683508620bc4e3bf8ba231070d34124ca54fcc Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 30 Oct 2014 16:06:44 +0800 Subject: [PATCH 31/98] update store statistic --- server/info.go | 20 ++++++++++++++++++-- store/db.go | 4 ++++ store/stat.go | 34 ++++++++++++++++++---------------- store/writebatch.go | 23 +++++++++++------------ 4 files changed, 51 insertions(+), 30 deletions(-) diff --git a/server/info.go b/server/info.go index d899d4f..1b7d27d 100644 --- a/server/info.go +++ b/server/info.go @@ -173,15 +173,31 @@ func (i *info) dumpStore(buf *bytes.Buffer) { s := i.app.ldb.StoreStat() + getNum := s.GetNum.Get() + getTotalTime := s.GetTotalTime.Get() + + gt := int64(0) + if getNum > 0 { + gt = getTotalTime.Nanoseconds() / (getNum * 1e3) + } + + commitNum := s.BatchCommitNum.Get() + commitTotalTime := s.BatchCommitTotalTime.Get() + + ct := int64(0) + if commitNum > 0 { + ct = commitTotalTime.Nanoseconds() / (commitNum * 1e3) + } + i.dumpPairs(buf, infoPair{"name", i.app.cfg.DBName}, infoPair{"get", s.GetNum}, infoPair{"get_missing", s.GetMissingNum}, - infoPair{"put", s.PutNum}, - infoPair{"delete", s.DeleteNum}, + infoPair{"get_per_time", fmt.Sprintf("%0.002fms", float64(gt)/1000.0)}, infoPair{"iter", s.IterNum}, infoPair{"iter_seek", s.IterSeekNum}, infoPair{"iter_close", s.IterCloseNum}, infoPair{"batch_commit", s.BatchCommitNum}, + infoPair{"batch_commit_per_time", fmt.Sprintf("%0.002fms", float64(ct)/1000.0)}, ) } diff --git a/store/db.go b/store/db.go index 7e88342..9964c5e 100644 --- a/store/db.go +++ b/store/db.go @@ -39,8 +39,10 @@ func (db *DB) NewIterator() *Iterator { } func (db *DB) Get(key []byte) ([]byte, error) { + t := time.Now() v, err := db.db.Get(key) db.st.statGet(v, err) + db.st.GetTotalTime.Add(time.Now().Sub(t)) return v, err } @@ -159,8 +161,10 @@ func (db *DB) needSyncCommit() bool { func (db *DB) GetSlice(key []byte) (Slice, error) { if d, ok := db.db.(driver.ISliceGeter); ok { + t := time.Now() v, err := d.GetSlice(key) db.st.statGet(v, err) + db.st.GetTotalTime.Add(time.Now().Sub(t)) return v, err } else { v, err := db.Get(key) diff --git a/store/stat.go b/store/stat.go index a14a135..e0a035a 100644 --- a/store/stat.go +++ b/store/stat.go @@ -5,22 +5,24 @@ import ( ) type Stat struct { - GetNum sync2.AtomicInt64 - GetMissingNum sync2.AtomicInt64 - PutNum sync2.AtomicInt64 - DeleteNum sync2.AtomicInt64 - IterNum sync2.AtomicInt64 - IterSeekNum sync2.AtomicInt64 - IterCloseNum sync2.AtomicInt64 - SnapshotNum sync2.AtomicInt64 - SnapshotCloseNum sync2.AtomicInt64 - BatchNum sync2.AtomicInt64 - BatchCommitNum sync2.AtomicInt64 - TxNum sync2.AtomicInt64 - TxCommitNum sync2.AtomicInt64 - TxCloseNum sync2.AtomicInt64 - CompactNum sync2.AtomicInt64 - CompactTotalTime sync2.AtomicDuration + GetNum sync2.AtomicInt64 + GetMissingNum sync2.AtomicInt64 + GetTotalTime sync2.AtomicDuration + PutNum sync2.AtomicInt64 + DeleteNum sync2.AtomicInt64 + IterNum sync2.AtomicInt64 + IterSeekNum sync2.AtomicInt64 + IterCloseNum sync2.AtomicInt64 + SnapshotNum sync2.AtomicInt64 + SnapshotCloseNum sync2.AtomicInt64 + BatchNum sync2.AtomicInt64 + BatchCommitNum sync2.AtomicInt64 + BatchCommitTotalTime sync2.AtomicDuration + TxNum sync2.AtomicInt64 + TxCommitNum sync2.AtomicInt64 + TxCloseNum sync2.AtomicInt64 + CompactNum sync2.AtomicInt64 + CompactTotalTime sync2.AtomicDuration } func (st *Stat) statGet(v interface{}, err error) { diff --git a/store/writebatch.go b/store/writebatch.go index bf4658c..773eeaf 100644 --- a/store/writebatch.go +++ b/store/writebatch.go @@ -2,38 +2,37 @@ package store import ( "github.com/siddontang/ledisdb/store/driver" + "time" ) type WriteBatch struct { - wb driver.IWriteBatch - st *Stat - putNum int64 - deleteNum int64 + wb driver.IWriteBatch + st *Stat db *DB } func (wb *WriteBatch) Put(key []byte, value []byte) { - wb.putNum++ wb.wb.Put(key, value) } func (wb *WriteBatch) Delete(key []byte) { - wb.deleteNum++ wb.wb.Delete(key) } func (wb *WriteBatch) Commit() error { wb.st.BatchCommitNum.Add(1) - wb.st.PutNum.Add(wb.putNum) - wb.st.DeleteNum.Add(wb.deleteNum) - wb.putNum = 0 - wb.deleteNum = 0 + var err error + t := time.Now() if wb.db == nil || !wb.db.needSyncCommit() { - return wb.wb.Commit() + err = wb.wb.Commit() } else { - return wb.wb.SyncCommit() + err = wb.wb.SyncCommit() } + + wb.st.BatchCommitTotalTime.Add(time.Now().Sub(t)) + + return err } func (wb *WriteBatch) Rollback() error { From c14ea7a3a7f9d2e3b714bf8c1df20b1a4b10ad98 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 30 Oct 2014 16:48:15 +0800 Subject: [PATCH 32/98] update benchmark tool --- cmd/ledis-benchmark/main.go | 125 ++++++++++++------------------------ cmd/ledis-respbench/main.go | 72 ++++++++++++++++++++- 2 files changed, 111 insertions(+), 86 deletions(-) diff --git a/cmd/ledis-benchmark/main.go b/cmd/ledis-benchmark/main.go index d5b5311..640607d 100644 --- a/cmd/ledis-benchmark/main.go +++ b/cmd/ledis-benchmark/main.go @@ -18,7 +18,7 @@ var number = flag.Int("n", 1000, "request number") var clients = flag.Int("c", 50, "number of clients") var round = flag.Int("r", 1, "benchmark round number") var valueSize = flag.Int("vsize", 100, "kv value size") -var tests = flag.String("t", "", "only run the comma separated list of tests, set,get,randget,del,lpush,lrange,lpop,hset,hget,hdel,zadd,zincr,zrange,zrevrange,zdel") +var tests = flag.String("t", "set,get,randget,del,lpush,lrange,lpop,hset,hget,hdel,zadd,zincr,zrange,zrevrange,zdel", "only run the comma separated list of tests") var wg sync.WaitGroup var client *ledis.Client @@ -285,92 +285,49 @@ func main() { *round = 1 } - runAll := true ts := strings.Split(*tests, ",") - if len(ts) > 0 && len(ts[0]) != 0 { - runAll = false - } - - needTest := make(map[string]struct{}) - for _, s := range ts { - needTest[strings.ToLower(s)] = struct{}{} - } - - checkTest := func(cmd string) bool { - if runAll { - return true - } else if _, ok := needTest[cmd]; ok { - return ok - } - return false - } for i := 0; i < *round; i++ { - if checkTest("set") { - benchSet() - } - - if checkTest("get") { - benchGet() - } - - if checkTest("randget") { - benchRandGet() - } - - if checkTest("del") { - benchDel() - } - - if checkTest("lpush") { - benchPushList() - } - - if checkTest("lrange") { - benchRangeList10() - benchRangeList50() - benchRangeList100() - } - - if checkTest("lpop") { - benchPopList() - } - - if checkTest("hset") { - benchHset() - } - - if checkTest("hget") { - benchHGet() - benchHRandGet() - } - - if checkTest("hdel") { - benchHDel() - } - - if checkTest("zadd") { - benchZAdd() - } - - if checkTest("zincr") { - benchZIncr() - } - - if checkTest("zrange") { - benchZRangeByRank() - benchZRangeByScore() - } - - if checkTest("zrevrange") { - //rev is too slow in leveldb, rocksdb or other - //maybe disable for huge data benchmark - benchZRevRangeByRank() - benchZRevRangeByScore() - } - - if checkTest("zdel") { - benchZDel() + for _, s := range ts { + switch strings.ToLower(s) { + case "set": + benchSet() + case "get": + benchGet() + case "randget": + benchRandGet() + case "del": + benchDel() + case "lpush": + benchPushList() + case "lrange": + benchRangeList10() + benchRangeList50() + benchRangeList100() + case "lpop": + benchPopList() + case "hset": + benchHset() + case "hget": + benchHGet() + benchHRandGet() + case "hdel": + benchHDel() + case "zadd": + benchZAdd() + case "zincr": + benchZIncr() + case "zrange": + benchZRangeByRank() + benchZRangeByScore() + case "zrevrange": + //rev is too slow in leveldb, rocksdb or other + //maybe disable for huge data benchmark + benchZRevRangeByRank() + benchZRevRangeByScore() + case "zdel": + benchZDel() + } } println("") diff --git a/cmd/ledis-respbench/main.go b/cmd/ledis-respbench/main.go index e21d173..0be7da0 100644 --- a/cmd/ledis-respbench/main.go +++ b/cmd/ledis-respbench/main.go @@ -6,13 +6,41 @@ import ( "flag" "fmt" "github.com/siddontang/go/arena" + "github.com/siddontang/ledisdb/config" + "github.com/siddontang/ledisdb/ledis" "github.com/siddontang/ledisdb/server" "net" + "os" "runtime" "time" ) +var KB = config.KB +var MB = config.MB +var GB = config.GB + var addr = flag.String("addr", ":6380", "listen addr") +var name = flag.String("db_name", "", "db name") + +var ldb *ledis.Ledis +var db *ledis.DB + +func setRocksDB(cfg *config.RocksDBConfig) { + cfg.BlockSize = 64 * KB + cfg.WriteBufferSize = 64 * MB + cfg.MaxWriteBufferNum = 2 + cfg.MaxBytesForLevelBase = 512 * MB + cfg.TargetFileSizeBase = 64 * MB + cfg.BackgroundThreads = 4 + cfg.HighPriorityBackgroundThreads = 1 + cfg.MaxBackgroundCompactions = 3 + cfg.MaxBackgroundFlushes = 1 + cfg.CacheSize = 512 * MB + cfg.EnableStatistics = true + cfg.StatsDumpPeriodSec = 5 + cfg.Level0FileNumCompactionTrigger = 8 + cfg.MaxBytesForLevelMultiplier = 8 +} func main() { runtime.GOMAXPROCS(runtime.NumCPU()) @@ -27,6 +55,31 @@ func main() { return } + if len(*name) > 0 { + cfg := config.NewConfigDefault() + cfg.DataDir = "./var/ledis_respbench" + cfg.DBName = *name + os.RemoveAll(cfg.DBPath) + defer os.RemoveAll(cfg.DBPath) + + os.MkdirAll(cfg.DBPath, 0755) + + cfg.LevelDB.BlockSize = 32 * KB + cfg.LevelDB.CacheSize = 512 * MB + cfg.LevelDB.WriteBufferSize = 64 * MB + cfg.LevelDB.MaxOpenFiles = 1000 + + setRocksDB(&cfg.RocksDB) + + ldb, err = ledis.Open(cfg) + if err != nil { + println(err.Error()) + return + } + + db, _ = ldb.Select(0) + } + for { c, err := l.Accept() if err != nil { @@ -69,11 +122,26 @@ func run(c net.Conn) { cmd := string(bytes.ToUpper(req[0])) switch cmd { case "SET": + if db != nil { + db.Set(req[1], req[2]) + } wb.Write(ok) case "GET": - wb.Write(data) + if db != nil { + d, _ := db.GetSlice(req[1]) + if d == nil { + wb.Write(data) + } else { + wb.WriteString(fmt.Sprintf("$%d\r\n", d.Size())) + wb.Write(d.Data()) + wb.WriteString("\r\n") + d.Free() + } + } else { + wb.Write(data) + } default: - wb.WriteString(fmt.Sprintf("-Err %s Not Supported Now", req[0])) + wb.WriteString(fmt.Sprintf("-Err %s Not Supported Now\r\n", req[0])) } wb.Flush() From c471b4538f701ba5a1ca8a808f5d35afe862c9b0 Mon Sep 17 00:00:00 2001 From: siddontang Date: Fri, 31 Oct 2014 08:55:31 +0800 Subject: [PATCH 33/98] update bench tool --- cmd/ledis-respbench/main.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/ledis-respbench/main.go b/cmd/ledis-respbench/main.go index 0be7da0..fcdf239 100644 --- a/cmd/ledis-respbench/main.go +++ b/cmd/ledis-respbench/main.go @@ -99,6 +99,8 @@ func run(c net.Conn) { var rt time.Duration var wt time.Duration + var st time.Duration + var gt time.Duration rb := bufio.NewReaderSize(c, 10240) wb := bufio.NewWriterSize(c, 10240) @@ -124,11 +126,13 @@ func run(c net.Conn) { case "SET": if db != nil { db.Set(req[1], req[2]) + st += time.Now().Sub(t2) } wb.Write(ok) case "GET": if db != nil { d, _ := db.GetSlice(req[1]) + gt += time.Now().Sub(t2) if d == nil { wb.Write(data) } else { @@ -150,5 +154,5 @@ func run(c net.Conn) { wt += t3.Sub(t2) } - fmt.Printf("rt:%s wt:%s\n", rt.String(), wt.String()) + fmt.Printf("rt:%s wt %s, gt:%s, st:%s\n", rt.String(), wt.String(), gt.String(), st.String()) } From b787ddbbd19db74ccf5445ef751ce977066fb82d Mon Sep 17 00:00:00 2001 From: siddontang Date: Fri, 31 Oct 2014 09:18:45 +0800 Subject: [PATCH 34/98] update bench tool --- cmd/ledis-dbbench/main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/ledis-dbbench/main.go b/cmd/ledis-dbbench/main.go index b2183ce..0ab8277 100644 --- a/cmd/ledis-dbbench/main.go +++ b/cmd/ledis-dbbench/main.go @@ -58,9 +58,10 @@ func bench(cmd string, f func()) { var kvSetBase int64 = 0 var kvGetBase int64 = 0 +var value []byte + func benchSet() { f := func() { - value := make([]byte, *valueSize) n := atomic.AddInt64(&kvSetBase, 1) db.Set(num.Int64ToBytes(n), value) @@ -122,6 +123,8 @@ func main() { runtime.GOMAXPROCS(runtime.NumCPU()) flag.Parse() + value = make([]byte, *valueSize) + cfg := config.NewConfigDefault() cfg.DataDir = "./var/ledis_dbbench" cfg.DBName = *name From be265447ac22c5495c472ad5b82ea410eef57ea8 Mon Sep 17 00:00:00 2001 From: siddontang Date: Fri, 31 Oct 2014 15:40:47 +0800 Subject: [PATCH 35/98] reduce replication interface --- rpl/file_store.go | 33 ++++++++++++++--------------- rpl/file_table.go | 1 + rpl/goleveldb_store.go | 48 ++++++++++++------------------------------ rpl/rpl.go | 6 +----- rpl/store.go | 6 +----- rpl/store_test.go | 11 +++++----- 6 files changed, 37 insertions(+), 68 deletions(-) create mode 100644 rpl/file_table.go diff --git a/rpl/file_store.go b/rpl/file_store.go index 91ea418..d6b9d26 100644 --- a/rpl/file_store.go +++ b/rpl/file_store.go @@ -14,14 +14,23 @@ import ( ) const ( - defaultMaxLogFileSize = 1024 * 1024 * 1024 + defaultMaxLogFileSize = uint32(1024 * 1024 * 1024) + + //why 4G, we can use uint32 as offset, reduce memory useage + maxLogFileSize = uint32(4*1024*1024*1024 - 1) ) /* -index file format: -ledis-bin.00001 -ledis-bin.00002 -ledis-bin.00003 + File Store: + 0000001.data + 0000001.meta + 0000002.data + 0000002.meta + + data: log1 data | log2 data | magic data + meta: log1 pos in data | log2 pos in data + + we use table to mangage data + meta pair */ type FileStore struct { @@ -29,7 +38,7 @@ type FileStore struct { m sync.Mutex - maxFileSize int + maxFileSize uint32 first uint64 last uint64 @@ -66,7 +75,7 @@ func NewFileStore(path string) (*FileStore, error) { return s, nil } -func (s *FileStore) SetMaxFileSize(size int) { +func (s *FileStore) SetMaxFileSize(size uint32) { s.maxFileSize = size } @@ -75,11 +84,6 @@ func (s *FileStore) GetLog(id uint64, log *Log) error { return nil } -func (s *FileStore) SeekLog(id uint64, log *Log) error { - panic("not implementation") - return nil -} - func (s *FileStore) FirstID() (uint64, error) { panic("not implementation") return 0, nil @@ -95,11 +99,6 @@ func (s *FileStore) StoreLog(log *Log) error { return nil } -func (s *FileStore) StoreLogs(logs []*Log) error { - panic("not implementation") - return nil -} - func (s *FileStore) Purge(n uint64) error { panic("not implementation") return nil diff --git a/rpl/file_table.go b/rpl/file_table.go new file mode 100644 index 0000000..def4355 --- /dev/null +++ b/rpl/file_table.go @@ -0,0 +1 @@ +package rpl diff --git a/rpl/goleveldb_store.go b/rpl/goleveldb_store.go index 39bf63a..349cb4a 100644 --- a/rpl/goleveldb_store.go +++ b/rpl/goleveldb_store.go @@ -21,6 +21,8 @@ type GoLevelDBStore struct { first uint64 last uint64 + + buf bytes.Buffer } func (s *GoLevelDBStore) FirstID() (uint64, error) { @@ -84,30 +86,10 @@ func (s *GoLevelDBStore) GetLog(id uint64, log *Log) error { } } -func (s *GoLevelDBStore) SeekLog(id uint64, log *Log) error { - it := s.db.NewIterator() - defer it.Close() - - it.Seek(num.Uint64ToBytes(id)) - - if !it.Valid() { - return ErrLogNotFound - } else { - return log.Decode(bytes.NewBuffer(it.RawValue())) - } -} - func (s *GoLevelDBStore) StoreLog(log *Log) error { - return s.StoreLogs([]*Log{log}) -} - -func (s *GoLevelDBStore) StoreLogs(logs []*Log) error { s.m.Lock() defer s.m.Unlock() - w := s.db.NewWriteBatch() - defer w.Rollback() - last, err := s.lastID() if err != nil { return err @@ -115,24 +97,20 @@ func (s *GoLevelDBStore) StoreLogs(logs []*Log) error { s.last = InvalidLogID - var buf bytes.Buffer - for _, log := range logs { - buf.Reset() + s.buf.Reset() - if log.ID <= last { - return ErrLessLogID - } - - last = log.ID - key := num.Uint64ToBytes(log.ID) - - if err := log.Encode(&buf); err != nil { - return err - } - w.Put(key, buf.Bytes()) + if log.ID != last+1 { + return ErrStoreLogID } - if err = w.Commit(); err != nil { + last = log.ID + key := num.Uint64ToBytes(log.ID) + + if err := log.Encode(&s.buf); err != nil { + return err + } + + if err = s.db.Put(key, s.buf.Bytes()); err != nil { return err } diff --git a/rpl/rpl.go b/rpl/rpl.go index d862132..151d17c 100644 --- a/rpl/rpl.go +++ b/rpl/rpl.go @@ -135,14 +135,10 @@ func (r *Replication) WaitLog() <-chan struct{} { } func (r *Replication) StoreLog(log *Log) error { - return r.StoreLogs([]*Log{log}) -} - -func (r *Replication) StoreLogs(logs []*Log) error { r.m.Lock() defer r.m.Unlock() - return r.s.StoreLogs(logs) + return r.s.StoreLog(log) } func (r *Replication) FirstLogID() (uint64, error) { diff --git a/rpl/store.go b/rpl/store.go index 8d5e8ec..00d9594 100644 --- a/rpl/store.go +++ b/rpl/store.go @@ -10,22 +10,18 @@ const ( var ( ErrLogNotFound = errors.New("log not found") - ErrLessLogID = errors.New("log id is less") + ErrStoreLogID = errors.New("log id is less") ErrNoBehindLog = errors.New("no behind commit log") ) type LogStore interface { GetLog(id uint64, log *Log) error - // Get the first log which ID is equal or larger than id - SeekLog(id uint64, log *Log) error - FirstID() (uint64, error) LastID() (uint64, error) // if log id is less than current last id, return error StoreLog(log *Log) error - StoreLogs(logs []*Log) error // Delete first n logs Purge(n uint64) error diff --git a/rpl/store_test.go b/rpl/store_test.go index 0dda1ce..f572317 100644 --- a/rpl/store_test.go +++ b/rpl/store_test.go @@ -63,16 +63,15 @@ func testLogs(t *testing.T, l LogStore) { } // Attempt to write multiple logs - var logs []*Log for i := 11; i <= 20; i++ { nl := &Log{ ID: uint64(i), Data: []byte("first"), } - logs = append(logs, nl) - } - if err := l.StoreLogs(logs); err != nil { - t.Fatalf("err: %v", err) + + if err := l.StoreLog(nl); err != nil { + t.Fatalf("err: %v", err) + } } // Try to fetch @@ -157,7 +156,7 @@ func testLogs(t *testing.T, l LogStore) { } now := uint32(time.Now().Unix()) - logs = []*Log{} + logs := []*Log{} for i := 1; i <= 20; i++ { nl := &Log{ ID: uint64(i), From 270c1ade1e12bc92196a8490499c70b0bf54f3d6 Mon Sep 17 00:00:00 2001 From: presbrey Date: Fri, 31 Oct 2014 07:29:45 -0400 Subject: [PATCH 36/98] Update config.toml minor typo --- config/config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.toml b/config/config.toml index 6b6722b..f75c784 100644 --- a/config/config.toml +++ b/config/config.toml @@ -50,7 +50,7 @@ conn_write_buffer_size = 10240 # if connection receives no data after n seconds, it may be dead, close # 0 to disable and not check -conn_keepavlie_interval = 0 +conn_keepalive_interval = 0 # checking TTL (time to live) data every n seconds # if you set big, the expired data may not be deleted immediately From c6e6b953865d701158b46bee43656a61528ed398 Mon Sep 17 00:00:00 2001 From: joe Date: Fri, 31 Oct 2014 08:07:40 -0400 Subject: [PATCH 37/98] add travis config --- .travis.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..e3f1130 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,9 @@ +language: go +go: 1.3.3 +before_install: + - go get github.com/tools/godep + - go get code.google.com/p/go.tools/cmd/cover + - go install -race std +script: + - godep go test -cover ./... +# - godep go test -race ./... From 0009fe83b9f09d6f91593a539b6bb956d836050c Mon Sep 17 00:00:00 2001 From: siddontang Date: Fri, 31 Oct 2014 20:33:13 +0800 Subject: [PATCH 38/98] add put and delete info back --- server/info.go | 30 ++++++++++++++++-------------- store/writebatch.go | 14 +++++++++++++- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/server/info.go b/server/info.go index 1b7d27d..0e807c1 100644 --- a/server/info.go +++ b/server/info.go @@ -173,31 +173,33 @@ func (i *info) dumpStore(buf *bytes.Buffer) { s := i.app.ldb.StoreStat() - getNum := s.GetNum.Get() - getTotalTime := s.GetTotalTime.Get() + // getNum := s.GetNum.Get() + // getTotalTime := s.GetTotalTime.Get() - gt := int64(0) - if getNum > 0 { - gt = getTotalTime.Nanoseconds() / (getNum * 1e3) - } + // gt := int64(0) + // if getNum > 0 { + // gt = getTotalTime.Nanoseconds() / (getNum * 1e3) + // } - commitNum := s.BatchCommitNum.Get() - commitTotalTime := s.BatchCommitTotalTime.Get() + // commitNum := s.BatchCommitNum.Get() + // commitTotalTime := s.BatchCommitTotalTime.Get() - ct := int64(0) - if commitNum > 0 { - ct = commitTotalTime.Nanoseconds() / (commitNum * 1e3) - } + // ct := int64(0) + // if commitNum > 0 { + // ct = commitTotalTime.Nanoseconds() / (commitNum * 1e3) + // } i.dumpPairs(buf, infoPair{"name", i.app.cfg.DBName}, infoPair{"get", s.GetNum}, infoPair{"get_missing", s.GetMissingNum}, - infoPair{"get_per_time", fmt.Sprintf("%0.002fms", float64(gt)/1000.0)}, + infoPair{"put", s.PutNum}, + infoPair{"delete", s.DeleteNum}, + infoPair{"get_total_time", s.GetTotalTime.Get().String()}, infoPair{"iter", s.IterNum}, infoPair{"iter_seek", s.IterSeekNum}, infoPair{"iter_close", s.IterCloseNum}, infoPair{"batch_commit", s.BatchCommitNum}, - infoPair{"batch_commit_per_time", fmt.Sprintf("%0.002fms", float64(ct)/1000.0)}, + infoPair{"batch_commit_total_time", s.BatchCommitTotalTime.Get().String()}, ) } diff --git a/store/writebatch.go b/store/writebatch.go index 773eeaf..a7fd84a 100644 --- a/store/writebatch.go +++ b/store/writebatch.go @@ -9,19 +9,28 @@ type WriteBatch struct { wb driver.IWriteBatch st *Stat - db *DB + putNum int64 + deleteNum int64 + db *DB } func (wb *WriteBatch) Put(key []byte, value []byte) { + wb.putNum++ wb.wb.Put(key, value) } func (wb *WriteBatch) Delete(key []byte) { + wb.deleteNum++ wb.wb.Delete(key) } func (wb *WriteBatch) Commit() error { wb.st.BatchCommitNum.Add(1) + wb.st.PutNum.Add(wb.putNum) + wb.st.DeleteNum.Add(wb.deleteNum) + wb.putNum = 0 + wb.deleteNum = 0 + var err error t := time.Now() if wb.db == nil || !wb.db.needSyncCommit() { @@ -36,5 +45,8 @@ func (wb *WriteBatch) Commit() error { } func (wb *WriteBatch) Rollback() error { + wb.putNum = 0 + wb.deleteNum = 0 + return wb.wb.Rollback() } From e343ca0dc0d881da9ea79d7218930c464796f3a4 Mon Sep 17 00:00:00 2001 From: siddontang Date: Fri, 31 Oct 2014 20:33:32 +0800 Subject: [PATCH 39/98] fix type --- config/config.go | 2 +- etc/ledis.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.go b/config/config.go index f11c335..e75bca4 100644 --- a/config/config.go +++ b/config/config.go @@ -113,7 +113,7 @@ type Config struct { ConnReadBufferSize int `toml:"conn_read_buffer_size"` ConnWriteBufferSize int `toml:"conn_write_buffer_size"` - ConnKeepaliveInterval int `toml:"conn_keepavlie_interval"` + ConnKeepaliveInterval int `toml:"conn_keepalive_interval"` TTLCheckInterval int `toml:"ttl_check_interval"` } diff --git a/etc/ledis.conf b/etc/ledis.conf index 6b6722b..f75c784 100644 --- a/etc/ledis.conf +++ b/etc/ledis.conf @@ -50,7 +50,7 @@ conn_write_buffer_size = 10240 # if connection receives no data after n seconds, it may be dead, close # 0 to disable and not check -conn_keepavlie_interval = 0 +conn_keepalive_interval = 0 # checking TTL (time to live) data every n seconds # if you set big, the expired data may not be deleted immediately From 7b5d950cc92d6545f22c909722529bf7974b40b0 Mon Sep 17 00:00:00 2001 From: siddontang Date: Sat, 1 Nov 2014 23:28:28 +0800 Subject: [PATCH 40/98] fix go test race error --- Makefile | 3 ++ client/go/ledis/conn.go | 30 +++++++++++++++++-- config/config.go | 16 ++++++++++ ledis/ledis.go | 43 +++++++++++++------------- ledis/t_ttl.go | 3 +- rpl/rpl.go | 14 +++++++-- server/app.go | 17 +++++++---- server/client.go | 3 +- server/cmd_replication.go | 10 +++++-- server/cmd_ttl_test.go | 8 ++--- server/replication.go | 63 +++++++++++++++++++-------------------- 11 files changed, 138 insertions(+), 72 deletions(-) diff --git a/Makefile b/Makefile index cfdddda..5495da4 100644 --- a/Makefile +++ b/Makefile @@ -22,5 +22,8 @@ clean: test: $(GO) test -tags '$(GO_BUILD_TAGS)' ./... +test_race: + $(GO) test -race -tags '$(GO_BUILD_TAGS)' ./... + pytest: sh client/ledis-py/tests/all.sh diff --git a/client/go/ledis/conn.go b/client/go/ledis/conn.go index c74ab3f..1c988b6 100644 --- a/client/go/ledis/conn.go +++ b/client/go/ledis/conn.go @@ -8,6 +8,7 @@ import ( "io" "net" "strconv" + "sync" ) // Error represents an error returned in a command reply. @@ -16,6 +17,12 @@ type Error string func (err Error) Error() string { return string(err) } type Conn struct { + cm sync.Mutex + wm sync.Mutex + rm sync.Mutex + + closed bool + client *Client addr string @@ -42,6 +49,8 @@ func NewConn(addr string) *Conn { co.rSize = 4096 co.wSize = 4096 + co.closed = false + return co } @@ -73,6 +82,9 @@ func (c *Conn) Send(cmd string, args ...interface{}) error { return err } + c.wm.Lock() + defer c.wm.Unlock() + if err := c.writeCommand(cmd, args); err != nil { c.finalize() return err @@ -86,6 +98,9 @@ func (c *Conn) Send(cmd string, args ...interface{}) error { } func (c *Conn) Receive() (interface{}, error) { + c.rm.Lock() + defer c.rm.Unlock() + if reply, err := c.readReply(); err != nil { c.finalize() return nil, err @@ -99,6 +114,9 @@ func (c *Conn) Receive() (interface{}, error) { } func (c *Conn) ReceiveBulkTo(w io.Writer) error { + c.rm.Lock() + defer c.rm.Unlock() + err := c.readBulkReplyTo(w) if err != nil { if _, ok := err.(Error); !ok { @@ -109,20 +127,26 @@ func (c *Conn) ReceiveBulkTo(w io.Writer) error { } func (c *Conn) finalize() { - if c.c != nil { + c.cm.Lock() + if !c.closed { c.c.Close() - c.c = nil + c.closed = true } + c.cm.Unlock() } func (c *Conn) connect() error { - if c.c != nil { + c.cm.Lock() + defer c.cm.Unlock() + + if !c.closed && c.c != nil { return nil } var err error c.c, err = net.Dial(getProto(c.addr), c.addr) if err != nil { + c.c = nil return err } diff --git a/config/config.go b/config/config.go index e75bca4..498c748 100644 --- a/config/config.go +++ b/config/config.go @@ -7,6 +7,7 @@ import ( "github.com/siddontang/go/ioutil2" "io" "io/ioutil" + "sync" ) var ( @@ -83,6 +84,8 @@ type SnapshotConfig struct { } type Config struct { + m sync.RWMutex `toml:"-"` + FileName string `toml:"-"` Addr string `toml:"addr"` @@ -254,3 +257,16 @@ func (cfg *Config) Rewrite() error { return cfg.DumpFile(cfg.FileName) } + +func (cfg *Config) GetReadonly() bool { + cfg.m.RLock() + b := cfg.Readonly + cfg.m.RUnlock() + return b +} + +func (cfg *Config) SetReadonly(b bool) { + cfg.m.Lock() + cfg.Readonly = b + cfg.m.Unlock() +} diff --git a/ledis/ledis.go b/ledis/ledis.go index a5e926e..6e390c5 100644 --- a/ledis/ledis.go +++ b/ledis/ledis.go @@ -82,8 +82,7 @@ func Open(cfg *config.Config) (*Ledis, error) { l.dbs[i] = l.newDB(i) } - l.wg.Add(1) - go l.checkTTL() + l.checkTTL() return l, nil } @@ -96,12 +95,12 @@ func (l *Ledis) Close() { if l.r != nil { l.r.Close() - l.r = nil + //l.r = nil } if l.lock != nil { l.lock.Close() - l.lock = nil + //l.lock = nil } } @@ -157,7 +156,7 @@ func (l *Ledis) flushAll() error { } func (l *Ledis) IsReadOnly() bool { - if l.cfg.Readonly { + if l.cfg.GetReadonly() { return true } else if l.r != nil { if b, _ := l.r.CommitIDBehind(); b { @@ -168,8 +167,6 @@ func (l *Ledis) IsReadOnly() bool { } func (l *Ledis) checkTTL() { - defer l.wg.Done() - for i, db := range l.dbs { c := newTTLChecker(db) @@ -187,23 +184,29 @@ func (l *Ledis) checkTTL() { l.cfg.TTLCheckInterval = 1 } - tick := time.NewTicker(time.Duration(l.cfg.TTLCheckInterval) * time.Second) - defer tick.Stop() + l.wg.Add(1) + go func() { + defer l.wg.Done() - for { - select { - case <-tick.C: - if l.IsReadOnly() { - break - } + tick := time.NewTicker(time.Duration(l.cfg.TTLCheckInterval) * time.Second) + defer tick.Stop() - for _, c := range l.tcs { - c.check() + for { + select { + case <-tick.C: + if l.IsReadOnly() { + break + } + + for _, c := range l.tcs { + c.check() + } + case <-l.quit: + return } - case <-l.quit: - return } - } + + }() } diff --git a/ledis/t_ttl.go b/ledis/t_ttl.go index a912d26..f16a735 100644 --- a/ledis/t_ttl.go +++ b/ledis/t_ttl.go @@ -85,7 +85,8 @@ func (db *DB) expireAt(t *batch, dataType byte, key []byte, when int64) { t.Put(tk, mk) t.Put(mk, PutInt64(when)) - db.l.tcs[db.index].setNextCheckTime(when, false) + tc := db.l.tcs[db.index] + tc.setNextCheckTime(when, false) } func (db *DB) ttl(dataType byte, key []byte) (t int64, err error) { diff --git a/rpl/rpl.go b/rpl/rpl.go index d862132..18eba25 100644 --- a/rpl/rpl.go +++ b/rpl/rpl.go @@ -32,6 +32,8 @@ type Replication struct { wg sync.WaitGroup nc chan struct{} + + ncm sync.Mutex } func NewReplication(cfg *config.Config) (*Replication, error) { @@ -63,6 +65,7 @@ func NewReplication(cfg *config.Config) (*Replication, error) { return nil, err } + r.wg.Add(1) go r.onPurgeExpired() return r, nil @@ -73,6 +76,9 @@ func (r *Replication) Close() error { r.wg.Wait() + r.m.Lock() + defer r.m.Unlock() + if r.s != nil { r.s.Close() r.s = nil @@ -124,14 +130,19 @@ func (r *Replication) Log(data []byte) (*Log, error) { return nil, err } + r.ncm.Lock() close(r.nc) r.nc = make(chan struct{}) + r.ncm.Unlock() return l, nil } func (r *Replication) WaitLog() <-chan struct{} { - return r.nc + r.ncm.Lock() + ch := r.nc + r.ncm.Unlock() + return ch } func (r *Replication) StoreLog(log *Log) error { @@ -255,7 +266,6 @@ func (r *Replication) ClearWithCommitID(id uint64) error { } func (r *Replication) onPurgeExpired() { - r.wg.Add(1) defer r.wg.Done() for { diff --git a/server/app.go b/server/app.go index 0e03b3f..a021865 100644 --- a/server/app.go +++ b/server/app.go @@ -149,13 +149,18 @@ func (app *App) Run() { go app.httpServe() - for !app.closed { - conn, err := app.listener.Accept() - if err != nil { - continue - } + for { + select { + case <-app.quit: + return + default: + conn, err := app.listener.Accept() + if err != nil { + continue + } - newClientRESP(conn, app) + newClientRESP(conn, app) + } } } diff --git a/server/client.go b/server/client.go index 47d8f85..d39241b 100644 --- a/server/client.go +++ b/server/client.go @@ -3,6 +3,7 @@ package server import ( "bytes" "fmt" + "github.com/siddontang/go/sync2" "github.com/siddontang/ledisdb/ledis" "io" "time" @@ -62,7 +63,7 @@ type client struct { syncBuf bytes.Buffer - lastLogID uint64 + lastLogID sync2.AtomicUint64 // reqErr chan error diff --git a/server/cmd_replication.go b/server/cmd_replication.go index b401f35..bc26968 100644 --- a/server/cmd_replication.go +++ b/server/cmd_replication.go @@ -110,16 +110,20 @@ func syncCommand(c *client) error { return ErrCmdParams } - c.lastLogID = logId - 1 + lastLogID := logId - 1 stat, err := c.app.ldb.ReplicationStat() if err != nil { return err } - if c.lastLogID > stat.LastID { + if lastLogID > stat.LastID { return fmt.Errorf("invalid sync logid %d > %d + 1", logId, stat.LastID) - } else if c.lastLogID == stat.LastID { + } + + c.lastLogID.Set(lastLogID) + + if lastLogID == stat.LastID { c.app.slaveAck(c) } diff --git a/server/cmd_ttl_test.go b/server/cmd_ttl_test.go index c9d388c..d851b83 100644 --- a/server/cmd_ttl_test.go +++ b/server/cmd_ttl_test.go @@ -72,8 +72,8 @@ func TestExpire(t *testing.T) { if ttl, err := ledis.Int64(c.Do(ttl, key)); err != nil { t.Fatal(err) - } else if ttl != exp { - t.Fatal(ttl) + } else if ttl == -1 { + t.Fatal("no ttl") } // expireat + ttl @@ -86,8 +86,8 @@ func TestExpire(t *testing.T) { if ttl, err := ledis.Int64(c.Do(ttl, key)); err != nil { t.Fatal(err) - } else if ttl != 3 { - t.Fatal(ttl) + } else if ttl == -1 { + t.Fatal("no ttl") } kErr := "not_exist_ttl" diff --git a/server/replication.go b/server/replication.go index 24a5c1a..3f95388 100644 --- a/server/replication.go +++ b/server/replication.go @@ -19,6 +19,7 @@ import ( var ( errConnectMaster = errors.New("connect master error") + errReplClosed = errors.New("replication is closed") ) type master struct { @@ -47,17 +48,16 @@ func newMaster(app *App) *master { } func (m *master) Close() { - ledis.AsyncNotify(m.quit) + m.quit <- struct{}{} - if m.conn != nil { - //for replication, we send quit command to close gracefully - m.conn.Send("quit") - - m.conn.Close() - m.conn = nil - } + m.closeConn() m.wg.Wait() + + select { + case <-m.quit: + default: + } } func (m *master) resetConn() error { @@ -67,7 +67,6 @@ func (m *master) resetConn() error { if m.conn != nil { m.conn.Close() - m.conn = nil } m.conn = goledis.NewConn(m.addr) @@ -75,6 +74,15 @@ func (m *master) resetConn() error { return nil } +func (m *master) closeConn() { + if m.conn != nil { + //for replication, we send quit command to close gracefully + m.conn.Send("quit") + + m.conn.Close() + } +} + func (m *master) stopReplication() error { m.Close() @@ -87,9 +95,7 @@ func (m *master) startReplication(masterAddr string, restart bool) error { m.addr = masterAddr - m.quit = make(chan struct{}, 1) - - m.app.cfg.Readonly = true + m.app.cfg.SetReadonly(true) m.wg.Add(1) go m.runReplication(restart) @@ -123,28 +129,20 @@ func (m *master) runReplication(restart bool) { if restart { if err := m.fullSync(); err != nil { - if m.conn != nil { - //if conn == nil, other close the replication, not error - log.Error("restart fullsync error %s", err.Error()) - } + log.Error("restart fullsync error %s", err.Error()) return } } for { - if err := m.sync(); err != nil { - if m.conn != nil { - //if conn == nil, other close the replication, not error - log.Error("sync error %s", err.Error()) - } - return - } - select { case <-m.quit: return default: - break + if err := m.sync(); err != nil { + log.Error("sync error %s", err.Error()) + return + } } } } @@ -266,7 +264,7 @@ func (app *App) slaveof(masterAddr string, restart bool, readonly bool) error { //in master mode and no slaveof, only set readonly if len(app.cfg.SlaveOf) == 0 && len(masterAddr) == 0 { - app.cfg.Readonly = readonly + app.cfg.SetReadonly(readonly) return nil } @@ -281,7 +279,7 @@ func (app *App) slaveof(masterAddr string, restart bool, readonly bool) error { return err } - app.cfg.Readonly = readonly + app.cfg.SetReadonly(readonly) } else { return app.m.startReplication(masterAddr, restart) } @@ -323,7 +321,7 @@ func (app *App) removeSlave(c *client, activeQuit bool) { delete(app.slaves, addr) log.Info("remove slave %s", addr) if activeQuit { - asyncNotifyUint64(app.slaveSyncAck, c.lastLogID) + asyncNotifyUint64(app.slaveSyncAck, c.lastLogID.Get()) } } } @@ -339,7 +337,7 @@ func (app *App) slaveAck(c *client) { return } - asyncNotifyUint64(app.slaveSyncAck, c.lastLogID) + asyncNotifyUint64(app.slaveSyncAck, c.lastLogID.Get()) } func asyncNotifyUint64(ch chan uint64, v uint64) { @@ -369,11 +367,12 @@ func (app *App) publishNewLog(l *rpl.Log) { n := 0 logId := l.ID for _, s := range app.slaves { - if s.lastLogID == logId { + lastLogID := s.lastLogID.Get() + if lastLogID == logId { //slave has already owned this log n++ - } else if s.lastLogID > logId { - log.Error("invalid slave %s, lastlogid %d > %d", s.slaveListeningAddr, s.lastLogID, logId) + } else if lastLogID > logId { + log.Error("invalid slave %s, lastlogid %d > %d", s.slaveListeningAddr, lastLogID, logId) } } From 8cfc5732cf8b11ffd2c2ca33f88f7e414a14a475 Mon Sep 17 00:00:00 2001 From: siddontang Date: Sun, 2 Nov 2014 14:21:24 +0800 Subject: [PATCH 41/98] remove unnecessary code --- Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Makefile b/Makefile index 5495da4..d52de40 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,3 @@ test: test_race: $(GO) test -race -tags '$(GO_BUILD_TAGS)' ./... - -pytest: - sh client/ledis-py/tests/all.sh From 600546755435c6d75540a4dfc06a9e5c7760a7c5 Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 3 Nov 2014 11:22:13 +0800 Subject: [PATCH 42/98] update --- rpl/file_store.go | 8 ++++++++ rpl/file_table.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/rpl/file_store.go b/rpl/file_store.go index d6b9d26..127dd19 100644 --- a/rpl/file_store.go +++ b/rpl/file_store.go @@ -30,7 +30,15 @@ const ( data: log1 data | log2 data | magic data meta: log1 pos in data | log2 pos in data + log id can not be 0, we use here for magic log + if data has no magic data, it means that we don't close replication gracefully. + so we must repair the log data + magic data = log0 data + we use table to mangage data + meta pair + + we must guarantee that the log id is monotonic increment strictly. + if log1's id is 1, log2 must be 2 */ type FileStore struct { diff --git a/rpl/file_table.go b/rpl/file_table.go index def4355..2a81ebf 100644 --- a/rpl/file_table.go +++ b/rpl/file_table.go @@ -1 +1,29 @@ package rpl + +import ( + "fmt" + "os" + "path" +) + +type table struct { + baseName string + + index int64 + + readonly bool + df *os.File + mf *os.File + + first uint64 + last uint64 +} + +func newReadTable(base string, index int64) (*table, error) { + t := new(table) + + t.baseName = path.Join(base, fmt.Sprintf("%08d", index)) + t.index = index + + return t, nil +} From 912b7991751a0c05307230de71e56fce7d2cb5e1 Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 3 Nov 2014 17:53:46 +0800 Subject: [PATCH 43/98] update, water many time...... --- bootstrap.sh | 7 +++---- rpl/file_store.go | 18 ++++++++---------- rpl/file_table.go | 19 +------------------ 3 files changed, 12 insertions(+), 32 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index f546a24..6a2a7a0 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -10,20 +10,19 @@ if [ "$?" = 0 ]; then exit 0 fi -go get github.com/siddontang/goleveldb/leveldb - go get github.com/szferi/gomdb go get github.com/boltdb/bolt go get github.com/ugorji/go/codec go get github.com/BurntSushi/toml +go get github.com/edsrzf/mmap-go - +go get github.com/siddontang/goleveldb/leveldb go get github.com/siddontang/go/bson go get github.com/siddontang/go/log go get github.com/siddontang/go/snappy go get github.com/siddontang/go/num go get github.com/siddontang/go/filelock go get github.com/siddontang/go/sync2 -go get github.com/siddontang/go/arena \ No newline at end of file +go get github.com/siddontang/go/arena diff --git a/rpl/file_store.go b/rpl/file_store.go index 127dd19..1c85846 100644 --- a/rpl/file_store.go +++ b/rpl/file_store.go @@ -22,20 +22,18 @@ const ( /* File Store: - 0000001.data - 0000001.meta - 0000002.data - 0000002.meta + 00000001.log + 00000002.log - data: log1 data | log2 data | magic data - meta: log1 pos in data | log2 pos in data + log: log1 data | log2 data | split data | log1 offset | log 2 offset | offset start pos | offset length | magic data - log id can not be 0, we use here for magic log + log id can not be 0, we use here for split data if data has no magic data, it means that we don't close replication gracefully. so we must repair the log data - magic data = log0 data - - we use table to mangage data + meta pair + split data = log0 data + log0: id 0, create time 0, compression 0, data "" + //sha1 of github.com/siddontang/ledisdb 20 bytes + magic data = "\x1c\x1d\xb8\x88\xff\x9e\x45\x55\x40\xf0\x4c\xda\xe0\xce\x47\xde\x65\x48\x71\x17" we must guarantee that the log id is monotonic increment strictly. if log1's id is 1, log2 must be 2 diff --git a/rpl/file_table.go b/rpl/file_table.go index 2a81ebf..92f1309 100644 --- a/rpl/file_table.go +++ b/rpl/file_table.go @@ -1,29 +1,12 @@ package rpl import ( - "fmt" "os" - "path" ) type table struct { - baseName string - - index int64 - - readonly bool - df *os.File - mf *os.File + f *os.File first uint64 last uint64 } - -func newReadTable(base string, index int64) (*table, error) { - t := new(table) - - t.baseName = path.Join(base, fmt.Sprintf("%08d", index)) - t.index = index - - return t, nil -} From b74487eadb4b9b2028e77bfe830dc2431258ec0c Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 3 Nov 2014 23:42:34 +0800 Subject: [PATCH 44/98] try to optimize rocksdb get hope can improve performance --- tools/build_rocksdb_ext.sh | 20 +++++++++++++ tools/rocksdb_ext.cc | 57 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 tools/build_rocksdb_ext.sh create mode 100644 tools/rocksdb_ext.cc diff --git a/tools/build_rocksdb_ext.sh b/tools/build_rocksdb_ext.sh new file mode 100644 index 0000000..1701cac --- /dev/null +++ b/tools/build_rocksdb_ext.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +ROCKSDB_DIR=/usr/local/rocksdb + +if test -z "$TARGET_OS"; then + TARGET_OS=`uname -s` +fi + +PLATFORM_SHARED_EXT="so" +PLATFORM_SHARED_LDFLAGS="-shared -Wl,-soname -Wl," +PLATFORM_SHARED_CFLAGS="-fPIC" + +if [ "$TARGET_OS" = "Darwin" ]; then + PLATFORM_SHARED_EXT=dylib + PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name " +fi + +SONAME=librocksdb_ext.$PLATFORM_SHARED_EXT + +g++ $PLATFORM_SHARED_LDFLAGS$SONAME $PLATFORM_SHARED_CFLAGS -std=c++0x -L$ROCKSDB_DIR/lib -lrocksdb -I/$ROCKSDB_DIR/include rocksdb_ext.cc -o $SONAME \ No newline at end of file diff --git a/tools/rocksdb_ext.cc b/tools/rocksdb_ext.cc new file mode 100644 index 0000000..95713a6 --- /dev/null +++ b/tools/rocksdb_ext.cc @@ -0,0 +1,57 @@ +#include +#include + +#include + +using namespace rocksdb; + +extern "C" { + +static bool SaveError(char** errptr, const Status& s) { + assert(errptr != NULL); + if (s.ok()) { + return false; + } else if (*errptr == NULL) { + *errptr = strdup(s.ToString().c_str()); + } else { + free(*errptr); + *errptr = strdup(s.ToString().c_str()); + } + return true; +} + +void* rocksdb_get_ext( + rocksdb_t* db, + const rocksdb_readoptions_t* options, + const char* key, size_t keylen, + char** valptr, + size_t* vallen, + char** errptr) { + + std::string *tmp = new(std::string); + + //very tricky, maybe changed with c++ rocksdb upgrade + Status s = (*(DB**)db)->Get(*(ReadOptions*)options, Slice(key, keylen), tmp); + + if (s.ok()) { + *valptr = (char*)tmp->data(); + *vallen = tmp->size(); + } else { + delete(tmp); + tmp = NULL; + *valptr = NULL; + *vallen = 0; + if (!s.IsNotFound()) { + SaveError(errptr, s); + } + } + return tmp; +} + +void rocksdb_get_free_ext(void* context) { + std::string* s = (std::string*)context; + + delete(s); +} + +} \ No newline at end of file From ae12ced082b51419190133736e5dbdeae9277591 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 5 Nov 2014 10:44:57 +0800 Subject: [PATCH 45/98] update gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 8959171..8c0a2ef 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ build nohup.out build_config.mk var* -_workspace \ No newline at end of file +_workspace +*.log \ No newline at end of file From f0fe5cac4e5123f947985d5179606c09c570c5f2 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 5 Nov 2014 10:45:12 +0800 Subject: [PATCH 46/98] add ignore configuration set --- store/rocksdb/db.go | 1 + 1 file changed, 1 insertion(+) diff --git a/store/rocksdb/db.go b/store/rocksdb/db.go index 60816d3..52a74b7 100644 --- a/store/rocksdb/db.go +++ b/store/rocksdb/db.go @@ -131,6 +131,7 @@ func (db *DB) initOptions(cfg *config.RocksDBConfig) { opts.SetMaxOpenFiles(cfg.MaxOpenFiles) opts.SetMaxBackgroundCompactions(cfg.MaxBackgroundCompactions) opts.SetMaxBackgroundFlushes(cfg.MaxBackgroundFlushes) + opts.SetLevel0FileNumCompactionTrigger(cfg.Level0FileNumCompactionTrigger) opts.SetLevel0SlowdownWritesTrigger(cfg.Level0SlowdownWritesTrigger) opts.SetLevel0StopWritesTrigger(cfg.Level0StopWritesTrigger) opts.SetTargetFileSizeBase(cfg.TargetFileSizeBase) From 7ad5e50b32cb56406e4b979a7719328033b184e9 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 5 Nov 2014 10:47:49 +0800 Subject: [PATCH 47/98] remove rocksdb_ext no significant performance improvement --- tools/build_rocksdb_ext.sh | 20 ------------- tools/rocksdb_ext.cc | 57 -------------------------------------- 2 files changed, 77 deletions(-) delete mode 100644 tools/build_rocksdb_ext.sh delete mode 100644 tools/rocksdb_ext.cc diff --git a/tools/build_rocksdb_ext.sh b/tools/build_rocksdb_ext.sh deleted file mode 100644 index 1701cac..0000000 --- a/tools/build_rocksdb_ext.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -ROCKSDB_DIR=/usr/local/rocksdb - -if test -z "$TARGET_OS"; then - TARGET_OS=`uname -s` -fi - -PLATFORM_SHARED_EXT="so" -PLATFORM_SHARED_LDFLAGS="-shared -Wl,-soname -Wl," -PLATFORM_SHARED_CFLAGS="-fPIC" - -if [ "$TARGET_OS" = "Darwin" ]; then - PLATFORM_SHARED_EXT=dylib - PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name " -fi - -SONAME=librocksdb_ext.$PLATFORM_SHARED_EXT - -g++ $PLATFORM_SHARED_LDFLAGS$SONAME $PLATFORM_SHARED_CFLAGS -std=c++0x -L$ROCKSDB_DIR/lib -lrocksdb -I/$ROCKSDB_DIR/include rocksdb_ext.cc -o $SONAME \ No newline at end of file diff --git a/tools/rocksdb_ext.cc b/tools/rocksdb_ext.cc deleted file mode 100644 index 95713a6..0000000 --- a/tools/rocksdb_ext.cc +++ /dev/null @@ -1,57 +0,0 @@ -#include -#include - -#include - -using namespace rocksdb; - -extern "C" { - -static bool SaveError(char** errptr, const Status& s) { - assert(errptr != NULL); - if (s.ok()) { - return false; - } else if (*errptr == NULL) { - *errptr = strdup(s.ToString().c_str()); - } else { - free(*errptr); - *errptr = strdup(s.ToString().c_str()); - } - return true; -} - -void* rocksdb_get_ext( - rocksdb_t* db, - const rocksdb_readoptions_t* options, - const char* key, size_t keylen, - char** valptr, - size_t* vallen, - char** errptr) { - - std::string *tmp = new(std::string); - - //very tricky, maybe changed with c++ rocksdb upgrade - Status s = (*(DB**)db)->Get(*(ReadOptions*)options, Slice(key, keylen), tmp); - - if (s.ok()) { - *valptr = (char*)tmp->data(); - *vallen = tmp->size(); - } else { - delete(tmp); - tmp = NULL; - *valptr = NULL; - *vallen = 0; - if (!s.IsNotFound()) { - SaveError(errptr, s); - } - } - return tmp; -} - -void rocksdb_get_free_ext(void* context) { - std::string* s = (std::string*)context; - - delete(s); -} - -} \ No newline at end of file From 662a4130b45987c18c6cad5ade1d2742cae12a6e Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 5 Nov 2014 17:34:14 +0800 Subject: [PATCH 48/98] update file rpl store --- rpl/file_store.go | 15 ++++++++++++--- rpl/file_table.go | 12 ------------ rpl/file_table_test.go | 9 +++++++++ rpl/log.go | 39 ++++++++++++++++++++++++--------------- 4 files changed, 45 insertions(+), 30 deletions(-) delete mode 100644 rpl/file_table.go create mode 100644 rpl/file_table_test.go diff --git a/rpl/file_store.go b/rpl/file_store.go index 1c85846..27b9e30 100644 --- a/rpl/file_store.go +++ b/rpl/file_store.go @@ -18,20 +18,29 @@ const ( //why 4G, we can use uint32 as offset, reduce memory useage maxLogFileSize = uint32(4*1024*1024*1024 - 1) + + maxLogNumInFile = uint64(10000000) ) /* File Store: - 00000001.log - 00000002.log + 00000001.ldb + 00000002.ldb log: log1 data | log2 data | split data | log1 offset | log 2 offset | offset start pos | offset length | magic data log id can not be 0, we use here for split data if data has no magic data, it means that we don't close replication gracefully. so we must repair the log data + log data: id (bigendian uint64), create time (bigendian uint32), compression (byte), data len(bigendian uint32), data split data = log0 data - log0: id 0, create time 0, compression 0, data "" + log0: id 0, create time 0, compression 0, data len 0, data "" + + log offset: bigendian uint32 | bigendian uint32 + + offset start pos: bigendian uint64 + offset length: bigendian uint32 + //sha1 of github.com/siddontang/ledisdb 20 bytes magic data = "\x1c\x1d\xb8\x88\xff\x9e\x45\x55\x40\xf0\x4c\xda\xe0\xce\x47\xde\x65\x48\x71\x17" diff --git a/rpl/file_table.go b/rpl/file_table.go deleted file mode 100644 index 92f1309..0000000 --- a/rpl/file_table.go +++ /dev/null @@ -1,12 +0,0 @@ -package rpl - -import ( - "os" -) - -type table struct { - f *os.File - - first uint64 - last uint64 -} diff --git a/rpl/file_table_test.go b/rpl/file_table_test.go new file mode 100644 index 0000000..82efade --- /dev/null +++ b/rpl/file_table_test.go @@ -0,0 +1,9 @@ +package rpl + +import ( + "testing" +) + +func TestFileTable(t *testing.T) { + +} diff --git a/rpl/log.go b/rpl/log.go index 261e852..382dff5 100644 --- a/rpl/log.go +++ b/rpl/log.go @@ -69,24 +69,11 @@ func (l *Log) Encode(w io.Writer) error { } func (l *Log) Decode(r io.Reader) error { - buf := make([]byte, l.HeadSize()) - - if _, err := io.ReadFull(r, buf); err != nil { + length, err := l.DecodeHead(r) + if err != nil { return err } - pos := 0 - l.ID = binary.BigEndian.Uint64(buf[pos:]) - pos += 8 - - l.CreateTime = binary.BigEndian.Uint32(buf[pos:]) - pos += 4 - - l.Compression = uint8(buf[pos]) - pos++ - - length := binary.BigEndian.Uint32(buf[pos:]) - l.Data = l.Data[0:0] if cap(l.Data) >= int(length) { @@ -100,3 +87,25 @@ func (l *Log) Decode(r io.Reader) error { return nil } + +func (l *Log) DecodeHead(r io.Reader) (uint32, error) { + buf := make([]byte, l.HeadSize()) + + if _, err := io.ReadFull(r, buf); err != nil { + return 0, err + } + + pos := 0 + l.ID = binary.BigEndian.Uint64(buf[pos:]) + pos += 8 + + l.CreateTime = binary.BigEndian.Uint32(buf[pos:]) + pos += 4 + + l.Compression = uint8(buf[pos]) + pos++ + + length := binary.BigEndian.Uint32(buf[pos:]) + + return length, nil +} From ffa98417dad858964f17ef8c31c66e1486464c1c Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 5 Nov 2014 17:34:58 +0800 Subject: [PATCH 49/98] add file table --- rpl/file_table.go | 456 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 456 insertions(+) create mode 100644 rpl/file_table.go diff --git a/rpl/file_table.go b/rpl/file_table.go new file mode 100644 index 0000000..584b27f --- /dev/null +++ b/rpl/file_table.go @@ -0,0 +1,456 @@ +package rpl + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "github.com/edsrzf/mmap-go" + "github.com/siddontang/go/log" + "github.com/siddontang/go/num" + "github.com/siddontang/go/sync2" + "io" + "os" + "path" + "sync" + "time" +) + +var ( + magic = []byte("\x1c\x1d\xb8\x88\xff\x9e\x45\x55\x40\xf0\x4c\xda\xe0\xce\x47\xde\x65\x48\x71\x17") + log0Data = []byte("00000000000000000") + errTableNeedFlush = errors.New("write table need flush") + errTableFrozen = errors.New("write table is frozen") +) + +const tableReaderKeepaliveInterval int64 = 30 + +func fmtTableName(index int64) string { + return fmt.Sprintf("%08d", index) +} + +type tableReader struct { + sync.Mutex + + name string + index int64 + + f *os.File + m mmap.MMap + + first uint64 + last uint64 + + offsetStartPos int64 + offsetLen uint32 + + lastReadTime sync2.AtomicInt64 +} + +func newTableReader(name string) (*tableReader, error) { + t := new(tableReader) + t.name = name + + var err error + + if _, err = fmt.Sscanf(path.Base(name), "%d.ldb", &t.index); err != nil { + return nil, err + } + + if err = t.check(); err != nil { + log.Error("check %s error: %s, try to repair", name, err.Error()) + + if err = t.repair(); err != nil { + log.Error("repair %s error: %s", name, err.Error()) + return nil, err + } + } + + t.close() + + return t, nil +} + +func (t *tableReader) Close() { + t.Lock() + defer t.Unlock() + + t.close() +} + +func (t *tableReader) close() { + if t.m != nil { + t.m.Unmap() + t.m = nil + } + + if t.f != nil { + t.f.Close() + t.f = nil + } +} + +func (t *tableReader) Keepalived() bool { + l := t.lastReadTime.Get() + if l > 0 && time.Now().Unix()-l > tableReaderKeepaliveInterval { + return false + } + + return true +} + +func (t *tableReader) check() error { + var err error + + if t.f, err = os.Open(t.name); err != nil { + return err + } + + st, _ := t.f.Stat() + + if st.Size() < 32 { + return fmt.Errorf("file size %d too short", st.Size()) + } + + if _, err = t.f.Seek(-32, os.SEEK_END); err != nil { + return err + } + + if err = binary.Read(t.f, binary.BigEndian, &t.offsetStartPos); err != nil { + return err + } else if t.offsetStartPos >= st.Size() { + return fmt.Errorf("invalid offset start pos %d, file size %d", t.offsetStartPos, st.Size()) + } + + if err = binary.Read(t.f, binary.BigEndian, &t.offsetLen); err != nil { + return err + } else if int64(t.offsetLen) >= st.Size() || t.offsetLen == 0 { + return fmt.Errorf("invalid offset len %d, file size %d", t.offsetLen, st.Size()) + } else if t.offsetLen%4 != 0 { + return fmt.Errorf("invalid offset len %d, must 4 multiple", t.offsetLen) + } + + b := make([]byte, 20) + if _, err = t.f.Read(b); err != nil { + return err + } else if !bytes.Equal(b, magic) { + return fmt.Errorf("invalid magic data %q", b) + } + + if t.m, err = mmap.MapRegion(t.f, int(t.offsetLen), mmap.RDONLY, 0, t.offsetStartPos); err != nil { + return err + } + + firstLogPos := binary.BigEndian.Uint32(t.m) + lastLogPos := binary.BigEndian.Uint32(t.m[len(t.m)-4:]) + + if firstLogPos != 0 { + return fmt.Errorf("invalid first log pos %d, must 0", firstLogPos) + } else if int64(lastLogPos) > t.offsetStartPos { + return fmt.Errorf("invalid last log pos %d", lastLogPos) + } + + var l Log + if _, err = t.decodeLogHead(&l, int64(firstLogPos)); err != nil { + return fmt.Errorf("decode first log err %s", err.Error()) + } + + t.first = l.ID + var n int64 + if n, err = t.decodeLogHead(&l, int64(lastLogPos)); err != nil { + return fmt.Errorf("decode last log err %s", err.Error()) + } else if n+int64(len(log0Data)) != t.offsetStartPos { + return fmt.Errorf("invalid last log, no proper log0") + } + + t.last = l.ID + + if t.first > t.last { + return fmt.Errorf("invalid log table first %d > last %d", t.first, t.last) + } else if (t.last - t.first + 1) != uint64(t.offsetLen/4) { + return fmt.Errorf("invalid log table, first %d, last %d, and log num %d", t.first, t.last, t.offsetLen/4) + } + + return nil +} + +func (t *tableReader) repair() error { + t.close() + + //todo later + return fmt.Errorf("repair not supported now") +} + +func (t *tableReader) decodeLogHead(l *Log, pos int64) (int64, error) { + _, err := t.f.Seek(int64(pos), os.SEEK_SET) + if err != nil { + return 0, err + } + + dataLen, err := l.DecodeHead(t.f) + if err != nil { + return 0, err + } + + return pos + int64(l.HeadSize()) + int64(dataLen), nil +} + +func (t *tableReader) ReadLog(id uint64, l *Log) error { + if id < t.first || id > t.last { + return fmt.Errorf("log %d not in [%d=%d]", id, t.first, t.last) + } + + t.lastReadTime.Set(time.Now().Unix()) + + t.Lock() + defer t.Unlock() + + if err := t.openTable(); err != nil { + t.close() + return err + } + + pos := binary.BigEndian.Uint32(t.m[(id-t.first)*4:]) + + if _, err := t.f.Seek(int64(pos), os.SEEK_SET); err != nil { + return err + } + + if err := l.Decode(t.f); err != nil { + return err + } else if l.ID != id { + return fmt.Errorf("invalid log id %d != %d", l.ID, id) + } + + return nil +} + +func (t *tableReader) openTable() error { + var err error + if t.f == nil { + if t.f, err = os.Open(t.name); err != nil { + return err + } + } + + if t.m == nil { + if t.m, err = mmap.MapRegion(t.f, int(t.offsetLen), mmap.RDONLY, 0, t.offsetStartPos); err != nil { + return err + } + } + + return nil +} + +type tableWriter struct { + sync.RWMutex + + wf *os.File + rf *os.File + + rm sync.Mutex + + base string + name string + index int64 + + first uint64 + last uint64 + + offsetBuf []byte + + frozen bool + + maxLogSize int64 +} + +func newTableWriter(base string, index int64, maxLogSize int64) (*tableWriter, error) { + t := new(tableWriter) + + t.base = base + t.name = path.Join(base, fmtTableName(index)) + t.index = index + + t.maxLogSize = maxLogSize + + return t, nil +} + +func (t *tableWriter) close() { + if t.rf != nil { + t.rf.Close() + t.rf = nil + } + + if t.wf != nil { + t.wf.Close() + t.wf = nil + } +} + +func (t *tableWriter) Close() { + t.Lock() + defer t.Unlock() + + t.close() +} + +func (t *tableWriter) reset() { + t.close() + + t.first = 0 + t.last = 0 + t.index = t.index + 1 + t.name = path.Join(t.base, fmtTableName(t.index)) + t.offsetBuf = t.offsetBuf[0:0] +} + +func (t *tableWriter) Flush() (*tableReader, error) { + t.Lock() + defer t.Unlock() + + t.frozen = true + + if t.wf == nil { + return nil, fmt.Errorf("nil write handler") + } + + defer t.reset() + + tr := new(tableReader) + tr.name = t.name + tr.index = t.index + + st, _ := t.wf.Stat() + + tr.offsetStartPos = st.Size() + int64(len(log0Data)) + tr.offsetLen = uint32(len(t.offsetBuf) / 4) + + tr.first = t.first + tr.last = t.last + + if n, err := t.wf.Write(log0Data); err != nil { + log.Error("flush log0data error %s", err.Error()) + return nil, err + } else if n != len(log0Data) { + log.Error("flush log0data only %d != %d", n, len(log0Data)) + return nil, io.ErrShortWrite + } + + if n, err := t.wf.Write(t.offsetBuf); err != nil { + log.Error("flush offset buffer error %s", err.Error()) + return nil, err + } else if n != len(t.offsetBuf) { + log.Error("flush offset buffer only %d != %d", n, len(t.offsetBuf)) + return nil, io.ErrShortWrite + } + + if err := binary.Write(t.wf, binary.BigEndian, tr.offsetStartPos); err != nil { + log.Error("flush offset start pos error %s", err.Error()) + return nil, err + } + + if err := binary.Write(t.wf, binary.BigEndian, tr.offsetLen); err != nil { + log.Error("flush offset len error %s", err.Error()) + return nil, err + } + + if n, err := t.wf.Write(magic); err != nil { + log.Error("flush magic data error %s", err.Error()) + return nil, err + } else if n != len(magic) { + log.Error("flush magic data only %d != %d", n, len(magic)) + return nil, io.ErrShortWrite + } + + return tr, nil +} + +func (t *tableWriter) StoreLog(l *Log) error { + t.Lock() + defer t.Unlock() + + if t.last > 0 && l.ID != t.last+1 { + return ErrStoreLogID + } + + if t.last-t.first+1 > maxLogNumInFile { + return errTableNeedFlush + } + + var err error + if t.wf == nil { + if t.wf, err = os.OpenFile(t.name, os.O_CREATE|os.O_APPEND, 0644); err != nil { + return err + } + } + + if t.offsetBuf == nil { + t.offsetBuf = make([]byte, 0, maxLogNumInFile*4) + } + + st, _ := t.wf.Stat() + if st.Size() >= t.maxLogSize { + return errTableNeedFlush + } + + offsetPos := uint32(st.Size()) + + if err := l.Encode(t.wf); err != nil { + return err + } + + t.offsetBuf = append(t.offsetBuf, num.Uint32ToBytes(offsetPos)...) + if t.first == 0 { + t.first = l.ID + } + + t.last = l.ID + + //todo add LRU cache + + return nil +} + +func (t *tableWriter) GetLog(id uint64, l *Log) error { + t.RLock() + defer t.RUnlock() + + if id < t.first && id > t.last { + return fmt.Errorf("log %d not in [%d=%d]", id, t.first, t.last) + } + + //todo memory cache + + offset := binary.BigEndian.Uint32(t.offsetBuf[(id-t.first)*4:]) + + if err := t.getLog(l, int64(offset)); err != nil { + return err + } else if l.ID != id { + return fmt.Errorf("invalid log id %d != %d", id, l.ID) + } + + return nil +} + +func (t *tableWriter) getLog(l *Log, pos int64) error { + t.rm.Lock() + defer t.rm.Unlock() + + var err error + if t.rf == nil { + if t.rf, err = os.Open(t.name); err != nil { + return err + } + } + + if _, err = t.rf.Seek(pos, os.SEEK_SET); err != nil { + return err + } + + if err = l.Decode(t.rf); err != nil { + return err + } + + return nil +} From 5f34758cb37b5f774e37c38e6dfd9a1b2bae7521 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 5 Nov 2014 17:45:25 +0800 Subject: [PATCH 50/98] update godep --- Godeps/Godeps.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index f4ab10d..b63936b 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -14,6 +14,10 @@ "Comment": "data/v1-228-g8fb50d5", "Rev": "8fb50d5ee57110936b904a7539d4c5f2bf2359db" }, + { + "ImportPath": "github.com/edsrzf/mmap-go", + "Rev": "6c75090c55983bef2e129e173681b20d24871ef8" + }, { "ImportPath": "github.com/siddontang/go/arena", "Rev": "ecf49fc0738105e87d20e29aa82c403b666ff0b4" From 128827999f0db76f62324a28b1a7401a25c691ea Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 5 Nov 2014 22:35:43 +0800 Subject: [PATCH 51/98] update table repair --- rpl/file_store.go | 12 ++++---- rpl/file_table.go | 65 +++++++++++++++++++++++++++++++++++++++--- rpl/file_table_test.go | 8 ++++++ 3 files changed, 76 insertions(+), 9 deletions(-) diff --git a/rpl/file_store.go b/rpl/file_store.go index 27b9e30..eaf2321 100644 --- a/rpl/file_store.go +++ b/rpl/file_store.go @@ -5,6 +5,8 @@ import ( "github.com/siddontang/go/hack" "github.com/siddontang/go/ioutil2" "github.com/siddontang/go/log" + "github.com/siddontang/go/num" + "io/ioutil" "os" "path" @@ -14,10 +16,10 @@ import ( ) const ( - defaultMaxLogFileSize = uint32(1024 * 1024 * 1024) + defaultMaxLogFileSize = int64(1024 * 1024 * 1024) //why 4G, we can use uint32 as offset, reduce memory useage - maxLogFileSize = uint32(4*1024*1024*1024 - 1) + maxLogFileSize = int64(uint32(4*1024*1024*1024 - 1)) maxLogNumInFile = uint64(10000000) ) @@ -53,7 +55,7 @@ type FileStore struct { m sync.Mutex - maxFileSize uint32 + maxFileSize int64 first uint64 last uint64 @@ -90,8 +92,8 @@ func NewFileStore(path string) (*FileStore, error) { return s, nil } -func (s *FileStore) SetMaxFileSize(size uint32) { - s.maxFileSize = size +func (s *FileStore) SetMaxFileSize(size int64) { + s.maxFileSize = num.MinInt64(maxLogFileSize, size) } func (s *FileStore) GetLog(id uint64, log *Log) error { diff --git a/rpl/file_table.go b/rpl/file_table.go index 584b27f..9745644 100644 --- a/rpl/file_table.go +++ b/rpl/file_table.go @@ -177,8 +177,57 @@ func (t *tableReader) check() error { func (t *tableReader) repair() error { t.close() - //todo later - return fmt.Errorf("repair not supported now") + var err error + if t.f, err = os.Open(t.name); err != nil { + return err + } + + defer t.close() + + tw := newTableWriter(path.Base(t.name), t.index, maxLogFileSize) + tw.name = tw.name + ".tmp" + os.Remove(tw.name) + + defer func() { + tw.Close() + os.Remove(tw.name) + }() + + var l Log + + for { + if err := l.Decode(t.f); err != nil { + return err + } + + if l.ID == 0 { + break + } + + if err := tw.StoreLog(&l); err != nil { + return err + } + } + + t.close() + + var tr *tableReader + if tr, err = tw.Flush(); err != nil { + return err + } + + t.first = tr.first + t.last = tr.last + t.offsetStartPos = tr.offsetStartPos + t.offsetLen = tr.offsetLen + + os.Remove(t.name) + + if err := os.Rename(tw.name, t.name); err != nil { + return err + } + + return nil } func (t *tableReader) decodeLogHead(l *Log, pos int64) (int64, error) { @@ -264,7 +313,7 @@ type tableWriter struct { maxLogSize int64 } -func newTableWriter(base string, index int64, maxLogSize int64) (*tableWriter, error) { +func newTableWriter(base string, index int64, maxLogSize int64) *tableWriter { t := new(tableWriter) t.base = base @@ -273,7 +322,7 @@ func newTableWriter(base string, index int64, maxLogSize int64) (*tableWriter, e t.maxLogSize = maxLogSize - return t, nil + return t } func (t *tableWriter) close() { @@ -370,6 +419,10 @@ func (t *tableWriter) StoreLog(l *Log) error { t.Lock() defer t.Unlock() + if t.frozen { + return errTableFrozen + } + if t.last > 0 && l.ID != t.last+1 { return ErrStoreLogID } @@ -416,6 +469,10 @@ func (t *tableWriter) GetLog(id uint64, l *Log) error { t.RLock() defer t.RUnlock() + if t.frozen { + return errTableFrozen + } + if id < t.first && id > t.last { return fmt.Errorf("log %d not in [%d=%d]", id, t.first, t.last) } diff --git a/rpl/file_table_test.go b/rpl/file_table_test.go index 82efade..edd1019 100644 --- a/rpl/file_table_test.go +++ b/rpl/file_table_test.go @@ -1,9 +1,17 @@ package rpl import ( + "io/ioutil" + "os" "testing" ) func TestFileTable(t *testing.T) { + base, err := ioutil.TempDir("./", "test_table") + if err != nil { + t.Fatal(err) + } + + defer os.RemoveAll(base) } From c3303e1fdb9eff9881d8a09f12f2032316e915af Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 6 Nov 2014 13:46:02 +0800 Subject: [PATCH 52/98] update table test --- rpl/file_table.go | 34 ++++---- rpl/file_table_test.go | 183 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 201 insertions(+), 16 deletions(-) diff --git a/rpl/file_table.go b/rpl/file_table.go index 9745644..f1db90a 100644 --- a/rpl/file_table.go +++ b/rpl/file_table.go @@ -18,7 +18,7 @@ import ( var ( magic = []byte("\x1c\x1d\xb8\x88\xff\x9e\x45\x55\x40\xf0\x4c\xda\xe0\xce\x47\xde\x65\x48\x71\x17") - log0Data = []byte("00000000000000000") + log0Data = make([]byte, 17) errTableNeedFlush = errors.New("write table need flush") errTableFrozen = errors.New("write table is frozen") ) @@ -26,7 +26,7 @@ var ( const tableReaderKeepaliveInterval int64 = 30 func fmtTableName(index int64) string { - return fmt.Sprintf("%08d", index) + return fmt.Sprintf("%08d.ldb", index) } type tableReader struct { @@ -184,13 +184,15 @@ func (t *tableReader) repair() error { defer t.close() - tw := newTableWriter(path.Base(t.name), t.index, maxLogFileSize) - tw.name = tw.name + ".tmp" - os.Remove(tw.name) + tw := newTableWriter(path.Dir(t.name), t.index, maxLogFileSize) + + tmpName := tw.name + ".tmp" + tw.name = tmpName + os.Remove(tmpName) defer func() { tw.Close() - os.Remove(tw.name) + os.Remove(tmpName) }() var l Log @@ -221,9 +223,11 @@ func (t *tableReader) repair() error { t.offsetStartPos = tr.offsetStartPos t.offsetLen = tr.offsetLen + defer tr.Close() + os.Remove(t.name) - if err := os.Rename(tw.name, t.name); err != nil { + if err := os.Rename(tmpName, t.name); err != nil { return err } @@ -244,7 +248,7 @@ func (t *tableReader) decodeLogHead(l *Log, pos int64) (int64, error) { return pos + int64(l.HeadSize()) + int64(dataLen), nil } -func (t *tableReader) ReadLog(id uint64, l *Log) error { +func (t *tableReader) GetLog(id uint64, l *Log) error { if id < t.first || id > t.last { return fmt.Errorf("log %d not in [%d=%d]", id, t.first, t.last) } @@ -373,7 +377,7 @@ func (t *tableWriter) Flush() (*tableReader, error) { st, _ := t.wf.Stat() tr.offsetStartPos = st.Size() + int64(len(log0Data)) - tr.offsetLen = uint32(len(t.offsetBuf) / 4) + tr.offsetLen = uint32(len(t.offsetBuf)) tr.first = t.first tr.last = t.last @@ -416,13 +420,13 @@ func (t *tableWriter) Flush() (*tableReader, error) { } func (t *tableWriter) StoreLog(l *Log) error { + if l.ID == 0 { + return ErrStoreLogID + } + t.Lock() defer t.Unlock() - if t.frozen { - return errTableFrozen - } - if t.last > 0 && l.ID != t.last+1 { return ErrStoreLogID } @@ -433,7 +437,7 @@ func (t *tableWriter) StoreLog(l *Log) error { var err error if t.wf == nil { - if t.wf, err = os.OpenFile(t.name, os.O_CREATE|os.O_APPEND, 0644); err != nil { + if t.wf, err = os.OpenFile(t.name, os.O_CREATE|os.O_WRONLY, 0644); err != nil { return err } } @@ -473,7 +477,7 @@ func (t *tableWriter) GetLog(id uint64, l *Log) error { return errTableFrozen } - if id < t.first && id > t.last { + if id < t.first || id > t.last { return fmt.Errorf("log %d not in [%d=%d]", id, t.first, t.last) } diff --git a/rpl/file_table_test.go b/rpl/file_table_test.go index edd1019..929bdb0 100644 --- a/rpl/file_table_test.go +++ b/rpl/file_table_test.go @@ -1,17 +1,198 @@ package rpl import ( + "github.com/siddontang/go/log" "io/ioutil" "os" "testing" + "time" ) func TestFileTable(t *testing.T) { - base, err := ioutil.TempDir("./", "test_table") + log.SetLevel(log.LevelFatal) + + base, err := ioutil.TempDir("", "test_table") if err != nil { t.Fatal(err) } + os.MkdirAll(base, 0755) + defer os.RemoveAll(base) + l := new(Log) + l.Compression = 0 + l.Data = make([]byte, 4096) + + w := newTableWriter(base, 1, 1024*1024) + defer w.Close() + + for i := 0; i < 10; i++ { + l.ID = uint64(i + 1) + l.CreateTime = uint32(time.Now().Unix()) + + l.Data[0] = byte(i + 1) + + if err := w.StoreLog(l); err != nil { + t.Fatal(err) + } + } + + if w.first != 1 { + t.Fatal(w.first) + } else if w.last != 10 { + t.Fatal(w.last) + } + + l.ID = 10 + if err := w.StoreLog(l); err == nil { + t.Fatal("must err") + } + + var ll Log + + if err = ll.Unmarshal(log0Data); err != nil { + t.Fatal(err) + } + + for i := 0; i < 10; i++ { + if err := w.GetLog(uint64(i+1), &ll); err != nil { + t.Fatal(err) + } else if len(ll.Data) != 4096 { + t.Fatal(len(ll.Data)) + } else if ll.Data[0] != byte(i+1) { + t.Fatal(ll.Data[0]) + } + } + + if err := w.GetLog(12, &ll); err == nil { + t.Fatal("must nil") + } + + for i := 0; i < 10; i++ { + if err := w.GetLog(uint64(i+1), &ll); err != nil { + t.Fatal(err) + } else if len(ll.Data) != 4096 { + t.Fatal(len(ll.Data)) + } else if ll.Data[0] != byte(i+1) { + t.Fatal(ll.Data[0]) + } + } + + var r *tableReader + name := w.name + + if r, err = w.Flush(); err != nil { + t.Fatal(err) + } + + for i := 10; i < 20; i++ { + l.ID = uint64(i + 1) + l.CreateTime = uint32(time.Now().Unix()) + + l.Data[0] = byte(i + 1) + + if err := w.StoreLog(l); err != nil { + t.Fatal(err) + } + } + + if w.first != 11 { + t.Fatal(w.first) + } else if w.last != 20 { + t.Fatal(w.last) + } + + defer r.Close() + + if err := r.GetLog(12, &ll); err == nil { + t.Fatal("must nil") + } + + r.Close() + + if r, err = newTableReader(name); err != nil { + t.Fatal(err) + } + defer r.Close() + + for i := 0; i < 10; i++ { + if err := r.GetLog(uint64(i+1), &ll); err != nil { + t.Fatal(err) + } else if len(ll.Data) != 4096 { + t.Fatal(len(ll.Data)) + } else if ll.Data[0] != byte(i+1) { + t.Fatal(ll.Data[0]) + } + } + + if err := r.GetLog(12, &ll); err == nil { + t.Fatal("must nil") + } + + st, _ := r.f.Stat() + s := st.Size() + + r.Close() + + testRepair(t, name, s, 11) + testRepair(t, name, s, 32) + testRepair(t, name, s, 42) + testRepair(t, name, s, 72) + + if err := os.Truncate(name, s-73); err != nil { + t.Fatal(err) + } + + if r, err = newTableReader(name); err == nil { + t.Fatal("can not repair") + } + + name = w.name + + if r, err := w.Flush(); err != nil { + t.Fatal(err) + } else { + r.Close() + } + + if r, err = newTableReader(name); err != nil { + t.Fatal(err) + } + defer r.Close() +} + +func testRepair(t *testing.T, name string, s int64, cutSize int64) { + var r *tableReader + var err error + + if err := os.Truncate(name, s-cutSize); err != nil { + t.Fatal(err) + } + + if r, err = newTableReader(name); err != nil { + t.Fatal(err) + } + defer r.Close() + + var ll Log + for i := 0; i < 10; i++ { + if err := r.GetLog(uint64(i+1), &ll); err != nil { + t.Fatal(err) + } else if len(ll.Data) != 4096 { + t.Fatal(len(ll.Data)) + } else if ll.Data[0] != byte(i+1) { + t.Fatal(ll.Data[0]) + } + } + + if err := r.GetLog(12, &ll); err == nil { + t.Fatal("must nil") + } + + st, _ := r.f.Stat() + if s != st.Size() { + t.Fatalf("repair error size %d != %d", s, st.Size()) + } + } From 4a0244180ace63ceff77044cd2bd358fc336979d Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 6 Nov 2014 15:05:21 +0800 Subject: [PATCH 53/98] update table reader --- rpl/file_table.go | 18 ++++++++++-------- rpl/file_table_test.go | 22 +++++++++++----------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/rpl/file_table.go b/rpl/file_table.go index f1db90a..e94a471 100644 --- a/rpl/file_table.go +++ b/rpl/file_table.go @@ -41,27 +41,26 @@ type tableReader struct { first uint64 last uint64 + lastTime uint32 + offsetStartPos int64 offsetLen uint32 lastReadTime sync2.AtomicInt64 } -func newTableReader(name string) (*tableReader, error) { +func newTableReader(base string, index int64) (*tableReader, error) { t := new(tableReader) - t.name = name + t.name = path.Join(base, fmtTableName(index)) + t.index = index var err error - if _, err = fmt.Sscanf(path.Base(name), "%d.ldb", &t.index); err != nil { - return nil, err - } - if err = t.check(); err != nil { - log.Error("check %s error: %s, try to repair", name, err.Error()) + log.Error("check %s error: %s, try to repair", t.name, err.Error()) if err = t.repair(); err != nil { - log.Error("repair %s error: %s", name, err.Error()) + log.Error("repair %s error: %s", t.name, err.Error()) return nil, err } } @@ -164,6 +163,7 @@ func (t *tableReader) check() error { } t.last = l.ID + t.lastTime = l.CreateTime if t.first > t.last { return fmt.Errorf("invalid log table first %d > last %d", t.first, t.last) @@ -206,6 +206,8 @@ func (t *tableReader) repair() error { break } + t.lastTime = l.CreateTime + if err := tw.StoreLog(&l); err != nil { return err } diff --git a/rpl/file_table_test.go b/rpl/file_table_test.go index 929bdb0..65b7db7 100644 --- a/rpl/file_table_test.go +++ b/rpl/file_table_test.go @@ -4,6 +4,7 @@ import ( "github.com/siddontang/go/log" "io/ioutil" "os" + "path" "testing" "time" ) @@ -80,6 +81,7 @@ func TestFileTable(t *testing.T) { } var r *tableReader + name := w.name if r, err = w.Flush(); err != nil { @@ -111,7 +113,7 @@ func TestFileTable(t *testing.T) { r.Close() - if r, err = newTableReader(name); err != nil { + if r, err = newTableReader(base, 1); err != nil { t.Fatal(err) } defer r.Close() @@ -135,34 +137,32 @@ func TestFileTable(t *testing.T) { r.Close() - testRepair(t, name, s, 11) - testRepair(t, name, s, 32) - testRepair(t, name, s, 42) - testRepair(t, name, s, 72) + testRepair(t, name, 1, s, 11) + testRepair(t, name, 1, s, 32) + testRepair(t, name, 1, s, 42) + testRepair(t, name, 1, s, 72) if err := os.Truncate(name, s-73); err != nil { t.Fatal(err) } - if r, err = newTableReader(name); err == nil { + if r, err = newTableReader(base, 1); err == nil { t.Fatal("can not repair") } - name = w.name - if r, err := w.Flush(); err != nil { t.Fatal(err) } else { r.Close() } - if r, err = newTableReader(name); err != nil { + if r, err = newTableReader(base, 2); err != nil { t.Fatal(err) } defer r.Close() } -func testRepair(t *testing.T, name string, s int64, cutSize int64) { +func testRepair(t *testing.T, name string, index int64, s int64, cutSize int64) { var r *tableReader var err error @@ -170,7 +170,7 @@ func testRepair(t *testing.T, name string, s int64, cutSize int64) { t.Fatal(err) } - if r, err = newTableReader(name); err != nil { + if r, err = newTableReader(path.Dir(name), index); err != nil { t.Fatal(err) } defer r.Close() From 0635b9e8cb5d622bea36a2e23a268a04990273e7 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 6 Nov 2014 15:05:29 +0800 Subject: [PATCH 54/98] update godep json --- Godeps/Godeps.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index b63936b..362496e 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -56,7 +56,7 @@ }, { "ImportPath": "github.com/siddontang/goleveldb/leveldb", - "Rev": "c1f6d721561c48f467b26a277741e55fd224df1e" + "Rev": "448b15792e63fe835c48d294f9b7859a86f4b76e" }, { "ImportPath": "github.com/szferi/gomdb", From b564cfbb7ea6be746e5c3866b0b61483f58b6678 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 6 Nov 2014 15:13:31 +0800 Subject: [PATCH 55/98] add more log to solve travis test failed --- rpl/file_table.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rpl/file_table.go b/rpl/file_table.go index e94a471..0cd871f 100644 --- a/rpl/file_table.go +++ b/rpl/file_table.go @@ -268,11 +268,11 @@ func (t *tableReader) GetLog(id uint64, l *Log) error { pos := binary.BigEndian.Uint32(t.m[(id-t.first)*4:]) if _, err := t.f.Seek(int64(pos), os.SEEK_SET); err != nil { - return err + return fmt.Errorf("seek error %s", err.Error()) } if err := l.Decode(t.f); err != nil { - return err + return fmt.Errorf("decode log err :%s", err.Error()) } else if l.ID != id { return fmt.Errorf("invalid log id %d != %d", l.ID, id) } @@ -284,13 +284,13 @@ func (t *tableReader) openTable() error { var err error if t.f == nil { if t.f, err = os.Open(t.name); err != nil { - return err + return fmt.Errorf("open %s error %s", t.name, err.Error()) } } if t.m == nil { if t.m, err = mmap.MapRegion(t.f, int(t.offsetLen), mmap.RDONLY, 0, t.offsetStartPos); err != nil { - return err + return fmt.Errorf("mmap %s error %s", t.name, err.Error()) } } From f2d11e8ef7438e2140350ac090ff0daf534345c7 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 6 Nov 2014 15:25:26 +0800 Subject: [PATCH 56/98] add more log for travis test failed --- rpl/file_table.go | 8 +++++++- rpl/file_table_test.go | 20 ++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/rpl/file_table.go b/rpl/file_table.go index 0cd871f..1926b06 100644 --- a/rpl/file_table.go +++ b/rpl/file_table.go @@ -289,8 +289,14 @@ func (t *tableReader) openTable() error { } if t.m == nil { + if t.f == nil { + return fmt.Errorf("invalid fd") + } + + t.f.Seek(0, os.SEEK_SET) + if t.m, err = mmap.MapRegion(t.f, int(t.offsetLen), mmap.RDONLY, 0, t.offsetStartPos); err != nil { - return fmt.Errorf("mmap %s error %s", t.name, err.Error()) + return fmt.Errorf("mmap %s error %s, %d, %d", t.name, err.Error(), t.offsetLen, t.offsetStartPos) } } diff --git a/rpl/file_table_test.go b/rpl/file_table_test.go index 65b7db7..a7ad9b5 100644 --- a/rpl/file_table_test.go +++ b/rpl/file_table_test.go @@ -70,16 +70,6 @@ func TestFileTable(t *testing.T) { t.Fatal("must nil") } - for i := 0; i < 10; i++ { - if err := w.GetLog(uint64(i+1), &ll); err != nil { - t.Fatal(err) - } else if len(ll.Data) != 4096 { - t.Fatal(len(ll.Data)) - } else if ll.Data[0] != byte(i+1) { - t.Fatal(ll.Data[0]) - } - } - var r *tableReader name := w.name @@ -107,6 +97,16 @@ func TestFileTable(t *testing.T) { defer r.Close() + for i := 0; i < 10; i++ { + if err := r.GetLog(uint64(i+1), &ll); err != nil { + t.Fatal(err) + } else if len(ll.Data) != 4096 { + t.Fatal(len(ll.Data)) + } else if ll.Data[0] != byte(i+1) { + t.Fatal(ll.Data[0]) + } + } + if err := r.GetLog(12, &ll); err == nil { t.Fatal("must nil") } From a03f8ee65312cefa89e9bdfda33bfc770bc0ba3f Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 6 Nov 2014 15:43:14 +0800 Subject: [PATCH 57/98] try mmap copy flag for travis test failed --- rpl/file_table.go | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/rpl/file_table.go b/rpl/file_table.go index 1926b06..ec6f8c1 100644 --- a/rpl/file_table.go +++ b/rpl/file_table.go @@ -136,7 +136,7 @@ func (t *tableReader) check() error { return fmt.Errorf("invalid magic data %q", b) } - if t.m, err = mmap.MapRegion(t.f, int(t.offsetLen), mmap.RDONLY, 0, t.offsetStartPos); err != nil { + if t.m, err = mmap.MapRegion(t.f, int(t.offsetLen), mmap.COPY, 0, t.offsetStartPos); err != nil { return err } @@ -268,11 +268,11 @@ func (t *tableReader) GetLog(id uint64, l *Log) error { pos := binary.BigEndian.Uint32(t.m[(id-t.first)*4:]) if _, err := t.f.Seek(int64(pos), os.SEEK_SET); err != nil { - return fmt.Errorf("seek error %s", err.Error()) + return err } if err := l.Decode(t.f); err != nil { - return fmt.Errorf("decode log err :%s", err.Error()) + return err } else if l.ID != id { return fmt.Errorf("invalid log id %d != %d", l.ID, id) } @@ -289,14 +289,8 @@ func (t *tableReader) openTable() error { } if t.m == nil { - if t.f == nil { - return fmt.Errorf("invalid fd") - } - - t.f.Seek(0, os.SEEK_SET) - - if t.m, err = mmap.MapRegion(t.f, int(t.offsetLen), mmap.RDONLY, 0, t.offsetStartPos); err != nil { - return fmt.Errorf("mmap %s error %s, %d, %d", t.name, err.Error(), t.offsetLen, t.offsetStartPos) + if t.m, err = mmap.MapRegion(t.f, int(t.offsetLen), mmap.COPY, 0, t.offsetStartPos); err != nil { + return fmt.Errorf("mmap %s error %s", t.name, err.Error()) } } From 75c4c59020b3418a3880e76f630351ea005a6167 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 6 Nov 2014 15:51:17 +0800 Subject: [PATCH 58/98] travis can not be build ok for mmap, later fix --- rpl/file_table.go | 4 ++-- rpl/file_table_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rpl/file_table.go b/rpl/file_table.go index ec6f8c1..be69e39 100644 --- a/rpl/file_table.go +++ b/rpl/file_table.go @@ -136,7 +136,7 @@ func (t *tableReader) check() error { return fmt.Errorf("invalid magic data %q", b) } - if t.m, err = mmap.MapRegion(t.f, int(t.offsetLen), mmap.COPY, 0, t.offsetStartPos); err != nil { + if t.m, err = mmap.MapRegion(t.f, int(t.offsetLen), mmap.RDONLY, 0, t.offsetStartPos); err != nil { return err } @@ -289,7 +289,7 @@ func (t *tableReader) openTable() error { } if t.m == nil { - if t.m, err = mmap.MapRegion(t.f, int(t.offsetLen), mmap.COPY, 0, t.offsetStartPos); err != nil { + if t.m, err = mmap.MapRegion(t.f, int(t.offsetLen), mmap.RDONLY, 0, t.offsetStartPos); err != nil { return fmt.Errorf("mmap %s error %s", t.name, err.Error()) } } diff --git a/rpl/file_table_test.go b/rpl/file_table_test.go index a7ad9b5..2b9e906 100644 --- a/rpl/file_table_test.go +++ b/rpl/file_table_test.go @@ -10,8 +10,6 @@ import ( ) func TestFileTable(t *testing.T) { - log.SetLevel(log.LevelFatal) - base, err := ioutil.TempDir("", "test_table") if err != nil { t.Fatal(err) @@ -137,6 +135,8 @@ func TestFileTable(t *testing.T) { r.Close() + log.SetLevel(log.LevelFatal) + testRepair(t, name, 1, s, 11) testRepair(t, name, 1, s, 32) testRepair(t, name, 1, s, 42) From 501505cea21e5c37bf9936698c9fd6ecbd8693ad Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 6 Nov 2014 16:06:27 +0800 Subject: [PATCH 59/98] use syscall mmap --- Godeps/Godeps.json | 4 ---- rpl/file_table.go | 12 ++++++------ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 362496e..b0e6d93 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -14,10 +14,6 @@ "Comment": "data/v1-228-g8fb50d5", "Rev": "8fb50d5ee57110936b904a7539d4c5f2bf2359db" }, - { - "ImportPath": "github.com/edsrzf/mmap-go", - "Rev": "6c75090c55983bef2e129e173681b20d24871ef8" - }, { "ImportPath": "github.com/siddontang/go/arena", "Rev": "ecf49fc0738105e87d20e29aa82c403b666ff0b4" diff --git a/rpl/file_table.go b/rpl/file_table.go index be69e39..16a6b2c 100644 --- a/rpl/file_table.go +++ b/rpl/file_table.go @@ -5,7 +5,6 @@ import ( "encoding/binary" "errors" "fmt" - "github.com/edsrzf/mmap-go" "github.com/siddontang/go/log" "github.com/siddontang/go/num" "github.com/siddontang/go/sync2" @@ -13,6 +12,7 @@ import ( "os" "path" "sync" + "syscall" "time" ) @@ -36,7 +36,7 @@ type tableReader struct { index int64 f *os.File - m mmap.MMap + m []byte first uint64 last uint64 @@ -79,7 +79,7 @@ func (t *tableReader) Close() { func (t *tableReader) close() { if t.m != nil { - t.m.Unmap() + syscall.Munmap(t.m) t.m = nil } @@ -136,7 +136,7 @@ func (t *tableReader) check() error { return fmt.Errorf("invalid magic data %q", b) } - if t.m, err = mmap.MapRegion(t.f, int(t.offsetLen), mmap.RDONLY, 0, t.offsetStartPos); err != nil { + if t.m, err = syscall.Mmap(int(t.f.Fd()), t.offsetStartPos, int(t.offsetLen), syscall.PROT_READ, syscall.MAP_PRIVATE); err != nil { return err } @@ -289,8 +289,8 @@ func (t *tableReader) openTable() error { } if t.m == nil { - if t.m, err = mmap.MapRegion(t.f, int(t.offsetLen), mmap.RDONLY, 0, t.offsetStartPos); err != nil { - return fmt.Errorf("mmap %s error %s", t.name, err.Error()) + if t.m, err = syscall.Mmap(int(t.f.Fd()), t.offsetStartPos, int(t.offsetLen), syscall.PROT_READ, syscall.MAP_PRIVATE); err != nil { + return err } } From 109468e0f3892a6481919a91e6fa9cd5a18be05e Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 6 Nov 2014 17:37:22 +0800 Subject: [PATCH 60/98] add padding for pagesize, not use mmap, add later --- rpl/file_store.go | 4 +- rpl/file_table.go | 100 +++++++++++++++++++++++++++++++---------- rpl/file_table_test.go | 2 +- 3 files changed, 79 insertions(+), 27 deletions(-) diff --git a/rpl/file_store.go b/rpl/file_store.go index eaf2321..873a0e2 100644 --- a/rpl/file_store.go +++ b/rpl/file_store.go @@ -35,8 +35,8 @@ const ( if data has no magic data, it means that we don't close replication gracefully. so we must repair the log data log data: id (bigendian uint64), create time (bigendian uint32), compression (byte), data len(bigendian uint32), data - split data = log0 data - log0: id 0, create time 0, compression 0, data len 0, data "" + split data = log0 data + [padding 0] -> file % pagesize() == 0 + log0: id 0, create time 0, compression 0, data len 7, data "ledisdb" log offset: bigendian uint32 | bigendian uint32 diff --git a/rpl/file_table.go b/rpl/file_table.go index 16a6b2c..b8de065 100644 --- a/rpl/file_table.go +++ b/rpl/file_table.go @@ -11,18 +11,25 @@ import ( "io" "os" "path" + "reflect" "sync" - "syscall" "time" ) var ( magic = []byte("\x1c\x1d\xb8\x88\xff\x9e\x45\x55\x40\xf0\x4c\xda\xe0\xce\x47\xde\x65\x48\x71\x17") - log0Data = make([]byte, 17) + log0 = Log{0, 1, 1, []byte("ledisdb")} + log0Data = []byte{} errTableNeedFlush = errors.New("write table need flush") errTableFrozen = errors.New("write table is frozen") + pageSize = int64(4096) ) +func init() { + log0Data, _ = log0.Marshal() + pageSize = int64(os.Getpagesize()) +} + const tableReaderKeepaliveInterval int64 = 30 func fmtTableName(index int64) string { @@ -35,8 +42,8 @@ type tableReader struct { name string index int64 - f *os.File - m []byte + f *os.File + pf *os.File first uint64 last uint64 @@ -78,9 +85,9 @@ func (t *tableReader) Close() { } func (t *tableReader) close() { - if t.m != nil { - syscall.Munmap(t.m) - t.m = nil + if t.pf != nil { + t.pf.Close() + t.pf = nil } if t.f != nil { @@ -98,6 +105,18 @@ func (t *tableReader) Keepalived() bool { return true } +func (t *tableReader) getLogPos(index int) (uint32, error) { + if _, err := t.pf.Seek(t.offsetStartPos+int64(index*4), os.SEEK_SET); err != nil { + return 0, err + } + + var pos uint32 + if err := binary.Read(t.pf, binary.BigEndian, &pos); err != nil { + return 0, err + } + return pos, nil +} + func (t *tableReader) check() error { var err error @@ -111,7 +130,8 @@ func (t *tableReader) check() error { return fmt.Errorf("file size %d too short", st.Size()) } - if _, err = t.f.Seek(-32, os.SEEK_END); err != nil { + var pos int64 + if pos, err = t.f.Seek(-32, os.SEEK_END); err != nil { return err } @@ -119,6 +139,8 @@ func (t *tableReader) check() error { return err } else if t.offsetStartPos >= st.Size() { return fmt.Errorf("invalid offset start pos %d, file size %d", t.offsetStartPos, st.Size()) + } else if t.offsetStartPos%pageSize != 0 { + return fmt.Errorf("invalid offset start pos %d, must page size %d multi", t.offsetStartPos, pageSize) } if err = binary.Read(t.f, binary.BigEndian, &t.offsetLen); err != nil { @@ -129,6 +151,10 @@ func (t *tableReader) check() error { return fmt.Errorf("invalid offset len %d, must 4 multiple", t.offsetLen) } + if t.offsetStartPos+int64(t.offsetLen) != pos { + return fmt.Errorf("invalid offset %d %d", t.offsetStartPos, t.offsetLen) + } + b := make([]byte, 20) if _, err = t.f.Read(b); err != nil { return err @@ -136,12 +162,12 @@ func (t *tableReader) check() error { return fmt.Errorf("invalid magic data %q", b) } - if t.m, err = syscall.Mmap(int(t.f.Fd()), t.offsetStartPos, int(t.offsetLen), syscall.PROT_READ, syscall.MAP_PRIVATE); err != nil { + if t.pf, err = os.Open(t.name); err != nil { return err } - firstLogPos := binary.BigEndian.Uint32(t.m) - lastLogPos := binary.BigEndian.Uint32(t.m[len(t.m)-4:]) + firstLogPos, _ := t.getLogPos(0) + lastLogPos, _ := t.getLogPos(int(t.offsetLen/4 - 1)) if firstLogPos != 0 { return fmt.Errorf("invalid first log pos %d, must 0", firstLogPos) @@ -158,8 +184,16 @@ func (t *tableReader) check() error { var n int64 if n, err = t.decodeLogHead(&l, int64(lastLogPos)); err != nil { return fmt.Errorf("decode last log err %s", err.Error()) - } else if n+int64(len(log0Data)) != t.offsetStartPos { - return fmt.Errorf("invalid last log, no proper log0") + } else { + var l0 Log + if _, err := t.f.Seek(n, os.SEEK_SET); err != nil { + return fmt.Errorf("seek logo err %s", err.Error()) + } else if err = l0.Decode(t.f); err != nil { + println(lastLogPos, n, l0.ID, l0.CreateTime, l0.Compression) + return fmt.Errorf("decode log0 err %s", err.Error()) + } else if !reflect.DeepEqual(l0, log0) { + return fmt.Errorf("invalid log0 %#v != %#v", l0, log0) + } } t.last = l.ID @@ -265,7 +299,10 @@ func (t *tableReader) GetLog(id uint64, l *Log) error { return err } - pos := binary.BigEndian.Uint32(t.m[(id-t.first)*4:]) + pos, err := t.getLogPos(int(id - t.first)) + if err != nil { + return err + } if _, err := t.f.Seek(int64(pos), os.SEEK_SET); err != nil { return err @@ -284,12 +321,12 @@ func (t *tableReader) openTable() error { var err error if t.f == nil { if t.f, err = os.Open(t.name); err != nil { - return fmt.Errorf("open %s error %s", t.name, err.Error()) + return err } } - if t.m == nil { - if t.m, err = syscall.Mmap(int(t.f.Fd()), t.offsetStartPos, int(t.offsetLen), syscall.PROT_READ, syscall.MAP_PRIVATE); err != nil { + if t.pf == nil { + if t.pf, err = os.Open(t.name); err != nil { return err } } @@ -378,20 +415,35 @@ func (t *tableWriter) Flush() (*tableReader, error) { st, _ := t.wf.Stat() - tr.offsetStartPos = st.Size() + int64(len(log0Data)) - tr.offsetLen = uint32(len(t.offsetBuf)) - tr.first = t.first tr.last = t.last if n, err := t.wf.Write(log0Data); err != nil { - log.Error("flush log0data error %s", err.Error()) - return nil, err + return nil, fmt.Errorf("flush log0data error %s", err.Error()) } else if n != len(log0Data) { - log.Error("flush log0data only %d != %d", n, len(log0Data)) - return nil, io.ErrShortWrite + return nil, fmt.Errorf("flush log0data only %d != %d", n, len(log0Data)) } + st, _ = t.wf.Stat() + + if m := st.Size() % pageSize; m != 0 { + padding := pageSize - m + if n, err := t.wf.Write(make([]byte, padding)); err != nil { + return nil, fmt.Errorf("flush log padding error %s", err.Error()) + } else if n != int(padding) { + return nil, fmt.Errorf("flush log padding error %d != %d", n, padding) + } + } + + st, _ = t.wf.Stat() + + if st.Size()%pageSize != 0 { + return nil, fmt.Errorf("invalid offset start pos, %d", st.Size()) + } + + tr.offsetStartPos = st.Size() + tr.offsetLen = uint32(len(t.offsetBuf)) + if n, err := t.wf.Write(t.offsetBuf); err != nil { log.Error("flush offset buffer error %s", err.Error()) return nil, err diff --git a/rpl/file_table_test.go b/rpl/file_table_test.go index 2b9e906..c3ac2b2 100644 --- a/rpl/file_table_test.go +++ b/rpl/file_table_test.go @@ -142,7 +142,7 @@ func TestFileTable(t *testing.T) { testRepair(t, name, 1, s, 42) testRepair(t, name, 1, s, 72) - if err := os.Truncate(name, s-73); err != nil { + if err := os.Truncate(name, s-(73+4096)); err != nil { t.Fatal(err) } From 297c7c05923d3df13bd2489b2ad9f291bed4aafa Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 6 Nov 2014 19:35:13 +0800 Subject: [PATCH 61/98] use mmap again, hope travis can pass --- Godeps/Godeps.json | 4 ++++ rpl/file_table.go | 35 ++++++++++++++++++++--------------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index b0e6d93..362496e 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -14,6 +14,10 @@ "Comment": "data/v1-228-g8fb50d5", "Rev": "8fb50d5ee57110936b904a7539d4c5f2bf2359db" }, + { + "ImportPath": "github.com/edsrzf/mmap-go", + "Rev": "6c75090c55983bef2e129e173681b20d24871ef8" + }, { "ImportPath": "github.com/siddontang/go/arena", "Rev": "ecf49fc0738105e87d20e29aa82c403b666ff0b4" diff --git a/rpl/file_table.go b/rpl/file_table.go index b8de065..ab9df1f 100644 --- a/rpl/file_table.go +++ b/rpl/file_table.go @@ -5,6 +5,7 @@ import ( "encoding/binary" "errors" "fmt" + "github.com/edsrzf/mmap-go" "github.com/siddontang/go/log" "github.com/siddontang/go/num" "github.com/siddontang/go/sync2" @@ -42,7 +43,9 @@ type tableReader struct { name string index int64 - f *os.File + f *os.File + m mmap.MMap + pf *os.File first uint64 @@ -85,9 +88,9 @@ func (t *tableReader) Close() { } func (t *tableReader) close() { - if t.pf != nil { - t.pf.Close() - t.pf = nil + if t.m != nil { + t.m.Unmap() + t.m = nil } if t.f != nil { @@ -106,15 +109,17 @@ func (t *tableReader) Keepalived() bool { } func (t *tableReader) getLogPos(index int) (uint32, error) { - if _, err := t.pf.Seek(t.offsetStartPos+int64(index*4), os.SEEK_SET); err != nil { - return 0, err - } + // if _, err := t.pf.Seek(t.offsetStartPos+int64(index*4), os.SEEK_SET); err != nil { + // return 0, err + // } - var pos uint32 - if err := binary.Read(t.pf, binary.BigEndian, &pos); err != nil { - return 0, err - } - return pos, nil + // var pos uint32 + // if err := binary.Read(t.pf, binary.BigEndian, &pos); err != nil { + // return 0, err + // } + // return pos, nil + + return binary.BigEndian.Uint32(t.m[index*4:]), nil } func (t *tableReader) check() error { @@ -162,7 +167,7 @@ func (t *tableReader) check() error { return fmt.Errorf("invalid magic data %q", b) } - if t.pf, err = os.Open(t.name); err != nil { + if t.m, err = mmap.MapRegion(t.f, int(t.offsetLen), mmap.RDONLY, 0, t.offsetStartPos); err != nil { return err } @@ -325,8 +330,8 @@ func (t *tableReader) openTable() error { } } - if t.pf == nil { - if t.pf, err = os.Open(t.name); err != nil { + if t.m == nil { + if t.m, err = mmap.MapRegion(t.f, int(t.offsetLen), mmap.RDONLY, 0, t.offsetStartPos); err != nil { return err } } From 137f85dff3bba12c6a94de88da85c6e3e5f5e678 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 6 Nov 2014 21:52:18 +0800 Subject: [PATCH 62/98] replication update --- rpl/file_store.go | 297 ++++++++++++++++++++++------------------- rpl/file_table.go | 30 +++-- rpl/goleveldb_store.go | 46 +------ rpl/store.go | 3 - rpl/store_test.go | 85 ------------ 5 files changed, 189 insertions(+), 272 deletions(-) diff --git a/rpl/file_store.go b/rpl/file_store.go index 873a0e2..f88b603 100644 --- a/rpl/file_store.go +++ b/rpl/file_store.go @@ -2,17 +2,13 @@ package rpl import ( "fmt" - "github.com/siddontang/go/hack" - "github.com/siddontang/go/ioutil2" "github.com/siddontang/go/log" "github.com/siddontang/go/num" - "io/ioutil" "os" - "path" - "strconv" - "strings" + "sort" "sync" + "time" ) const ( @@ -53,193 +49,226 @@ const ( type FileStore struct { LogStore - m sync.Mutex - maxFileSize int64 - first uint64 - last uint64 + base string - logFile *os.File - logNames []string - nextLogIndex int64 + rm sync.RWMutex + wm sync.Mutex - indexName string - - path string + rs tableReaders + w *tableWriter } -func NewFileStore(path string) (*FileStore, error) { +func NewFileStore(base string, maxSize int64) (*FileStore, error) { s := new(FileStore) - if err := os.MkdirAll(path, 0755); err != nil { + var err error + + if err = os.MkdirAll(base, 0755); err != nil { return nil, err } - s.path = path + s.base = base - s.maxFileSize = defaultMaxLogFileSize + s.maxFileSize = num.MinInt64(maxLogFileSize, maxSize) - s.first = 0 - s.last = 0 - - s.logNames = make([]string, 0, 16) - - if err := s.loadIndex(); err != nil { + if err = s.load(); err != nil { return nil, err } + index := int64(1) + if len(s.rs) != 0 { + index = s.rs[len(s.rs)-1].index + 1 + } + + s.w = newTableWriter(s.base, index, s.maxFileSize) return s, nil } -func (s *FileStore) SetMaxFileSize(size int64) { - s.maxFileSize = num.MinInt64(maxLogFileSize, size) -} - func (s *FileStore) GetLog(id uint64, log *Log) error { panic("not implementation") return nil } func (s *FileStore) FirstID() (uint64, error) { - panic("not implementation") return 0, nil } func (s *FileStore) LastID() (uint64, error) { - panic("not implementation") return 0, nil } -func (s *FileStore) StoreLog(log *Log) error { - panic("not implementation") - return nil -} +func (s *FileStore) StoreLog(l *Log) error { + s.wm.Lock() + defer s.wm.Unlock() + + if s.w == nil { + return fmt.Errorf("nil table writer, cannot store") + } + + err := s.w.StoreLog(l) + if err == nil { + return nil + } else if err != errTableNeedFlush { + return err + } + + var r *tableReader + if r, err = s.w.Flush(); err != nil { + log.Error("write table flush error %s, can not store now", err.Error()) + + s.w.Close() + s.w = nil + return err + } + + s.rm.Lock() + s.rs = append(s.rs, r) + s.rm.Unlock() -func (s *FileStore) Purge(n uint64) error { - panic("not implementation") return nil } func (s *FileStore) PuregeExpired(n int64) error { - panic("not implementation") + s.rm.Lock() + + purges := []*tableReader{} + lefts := []*tableReader{} + + t := uint32(time.Now().Unix() - int64(n)) + + for _, r := range s.rs { + if r.lastTime < t { + purges = append(purges, r) + } else { + lefts = append(lefts, r) + } + } + + s.rs = lefts + + s.rm.Unlock() + + for _, r := range purges { + name := r.name + r.Close() + if err := os.Remove(name); err != nil { + log.Error("purge table %s err: %s", name, err.Error()) + } + } + return nil } func (s *FileStore) Clear() error { - panic("not implementation") return nil } func (s *FileStore) Close() error { - panic("not implementation") + s.wm.Lock() + if s.w != nil { + if r, err := s.w.Flush(); err != nil { + log.Error("close err: %s", err.Error()) + } else { + r.Close() + s.w.Close() + s.w = nil + } + } + + s.wm.Unlock() + + s.rm.Lock() + + for i := range s.rs { + s.rs[i].Close() + } + s.rs = nil + + s.rm.Unlock() + return nil } -func (s *FileStore) flushIndex() error { - data := strings.Join(s.logNames, "\n") - - if err := ioutil2.WriteFileAtomic(s.indexName, hack.Slice(data), 0644); err != nil { - log.Error("flush index error %s", err.Error()) +func (s *FileStore) load() error { + fs, err := ioutil.ReadDir(s.base) + if err != nil { return err } - return nil -} - -func (s *FileStore) fileExists(name string) bool { - p := path.Join(s.path, name) - _, err := os.Stat(p) - return !os.IsNotExist(err) -} - -func (s *FileStore) loadIndex() error { - s.indexName = path.Join(s.path, fmt.Sprintf("ledis-bin.index")) - if _, err := os.Stat(s.indexName); os.IsNotExist(err) { - //no index file, nothing to do - } else { - indexData, err := ioutil.ReadFile(s.indexName) - if err != nil { - return err - } - - lines := strings.Split(string(indexData), "\n") - for _, line := range lines { - line = strings.Trim(line, "\r\n ") - if len(line) == 0 { - continue - } - - if s.fileExists(line) { - s.logNames = append(s.logNames, line) + var r *tableReader + var index int64 + for _, f := range fs { + if _, err := fmt.Sscanf(f.Name(), "%08d.ldb", &index); err == nil { + if r, err = newTableReader(s.base, index); err != nil { + log.Error("load table %s err: %s", f.Name(), err.Error()) } else { - log.Info("log %s has not exists", line) + s.rs = append(s.rs, r) } } } - var err error - if len(s.logNames) == 0 { - s.nextLogIndex = 1 - } else { - lastName := s.logNames[len(s.logNames)-1] + if err := s.rs.check(); err != nil { + return err + } - if s.nextLogIndex, err = strconv.ParseInt(path.Ext(lastName)[1:], 10, 64); err != nil { - log.Error("invalid logfile name %s", err.Error()) - return err + return nil +} + +type tableReaders []*tableReader + +func (ts tableReaders) Len() int { + return len(ts) +} + +func (ts tableReaders) Swap(i, j int) { + ts[i], ts[j] = ts[j], ts[i] +} + +func (ts tableReaders) Less(i, j int) bool { + return ts[i].index < ts[j].index +} + +func (ts tableReaders) Search(id uint64) *tableReader { + n := sort.Search(len(ts), func(i int) bool { + return id >= ts[i].first && id <= ts[i].last + }) + + if n < len(ts) { + return ts[n] + } else { + return nil + } +} + +func (ts tableReaders) check() error { + if len(ts) == 0 { + return nil + } + + sort.Sort(ts) + + first := ts[0].first + last := ts[0].last + index := ts[0].index + + if first == 0 || first > last { + return fmt.Errorf("invalid log in table %s", ts[0].name) + } + + for i := 1; i < len(ts); i++ { + if ts[i].first <= last { + return fmt.Errorf("invalid first log id %d in table %s", ts[i].first, ts[i].name) } - //like mysql, if server restart, a new log will create - s.nextLogIndex++ - } + if ts[i].index == index { + return fmt.Errorf("invalid index %d in table %s", ts[i].index, ts[i].name) + } + first = ts[i].first + last = ts[i].last + index = ts[i].index + } return nil } - -func (s *FileStore) openNewLogFile() error { - var err error - lastName := s.formatLogFileName(s.nextLogIndex) - - logPath := path.Join(s.path, lastName) - if s.logFile, err = os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY, 0644); err != nil { - log.Error("open new logfile error %s", err.Error()) - return err - } - - s.logNames = append(s.logNames, lastName) - - if err = s.flushIndex(); err != nil { - return err - } - - return nil -} - -func (s *FileStore) checkLogFileSize() bool { - if s.logFile == nil { - return false - } - - st, _ := s.logFile.Stat() - if st.Size() >= int64(s.maxFileSize) { - s.closeLog() - return true - } - - return false -} - -func (s *FileStore) closeLog() { - if s.logFile == nil { - return - } - - s.nextLogIndex++ - - s.logFile.Close() - s.logFile = nil -} - -func (s *FileStore) formatLogFileName(index int64) string { - return fmt.Sprintf("ledis-bin.%07d", index) -} diff --git a/rpl/file_table.go b/rpl/file_table.go index ab9df1f..410f150 100644 --- a/rpl/file_table.go +++ b/rpl/file_table.go @@ -22,7 +22,6 @@ var ( log0 = Log{0, 1, 1, []byte("ledisdb")} log0Data = []byte{} errTableNeedFlush = errors.New("write table need flush") - errTableFrozen = errors.New("write table is frozen") pageSize = int64(4096) ) @@ -60,6 +59,9 @@ type tableReader struct { } func newTableReader(base string, index int64) (*tableReader, error) { + if index <= 0 { + return nil, fmt.Errorf("invalid index %d", index) + } t := new(tableReader) t.name = path.Join(base, fmtTableName(index)) t.index = index @@ -356,12 +358,14 @@ type tableWriter struct { offsetBuf []byte - frozen bool - maxLogSize int64 } func newTableWriter(base string, index int64, maxLogSize int64) *tableWriter { + if index <= 0 { + panic(fmt.Errorf("invalid index %d", index)) + } + t := new(tableWriter) t.base = base @@ -392,6 +396,20 @@ func (t *tableWriter) Close() { t.close() } +func (t *tableWriter) First() uint64 { + t.Lock() + id := t.first + t.Unlock() + return id +} + +func (t *tableWriter) Last() uint64 { + t.Lock() + id := t.last + t.Unlock() + return id +} + func (t *tableWriter) reset() { t.close() @@ -406,8 +424,6 @@ func (t *tableWriter) Flush() (*tableReader, error) { t.Lock() defer t.Unlock() - t.frozen = true - if t.wf == nil { return nil, fmt.Errorf("nil write handler") } @@ -532,10 +548,6 @@ func (t *tableWriter) GetLog(id uint64, l *Log) error { t.RLock() defer t.RUnlock() - if t.frozen { - return errTableFrozen - } - if id < t.first || id > t.last { return fmt.Errorf("log %d not in [%d=%d]", id, t.first, t.last) } diff --git a/rpl/goleveldb_store.go b/rpl/goleveldb_store.go index 349cb4a..2d3f808 100644 --- a/rpl/goleveldb_store.go +++ b/rpl/goleveldb_store.go @@ -118,42 +118,6 @@ func (s *GoLevelDBStore) StoreLog(log *Log) error { return nil } -func (s *GoLevelDBStore) Purge(n uint64) error { - s.m.Lock() - defer s.m.Unlock() - - var first, last uint64 - var err error - - first, err = s.firstID() - if err != nil { - return err - } - - last, err = s.lastID() - if err != nil { - return err - } - - start := first - stop := num.MinUint64(last, first+n) - - w := s.db.NewWriteBatch() - defer w.Rollback() - - s.reset() - - for i := start; i < stop; i++ { - w.Delete(num.Uint64ToBytes(i)) - } - - if err = w.Commit(); err != nil { - return err - } - - return nil -} - func (s *GoLevelDBStore) PurgeExpired(n int64) error { if n <= 0 { return fmt.Errorf("invalid expired time %d", n) @@ -192,6 +156,11 @@ func (s *GoLevelDBStore) PurgeExpired(n int64) error { return nil } +func (s *GoLevelDBStore) reset() { + s.first = InvalidLogID + s.last = InvalidLogID +} + func (s *GoLevelDBStore) Clear() error { s.m.Lock() defer s.m.Unlock() @@ -206,11 +175,6 @@ func (s *GoLevelDBStore) Clear() error { return s.open() } -func (s *GoLevelDBStore) reset() { - s.first = InvalidLogID - s.last = InvalidLogID -} - func (s *GoLevelDBStore) Close() error { s.m.Lock() defer s.m.Unlock() diff --git a/rpl/store.go b/rpl/store.go index 00d9594..d56d9f0 100644 --- a/rpl/store.go +++ b/rpl/store.go @@ -23,9 +23,6 @@ type LogStore interface { // if log id is less than current last id, return error StoreLog(log *Log) error - // Delete first n logs - Purge(n uint64) error - // Delete logs before n seconds PurgeExpired(n int64) error diff --git a/rpl/store_test.go b/rpl/store_test.go index f572317..e84e286 100644 --- a/rpl/store_test.go +++ b/rpl/store_test.go @@ -4,7 +4,6 @@ import ( "io/ioutil" "os" "testing" - "time" ) func TestGoLevelDBStore(t *testing.T) { @@ -101,88 +100,4 @@ func testLogs(t *testing.T, l LogStore) { if idx != 20 { t.Fatalf("bad idx: %d", idx) } - - // Delete a suffix - if err := l.Purge(5); err != nil { - t.Fatalf("err: %v ", err) - } - - // Verify they are all deleted - for i := 1; i <= 5; i++ { - if err := l.GetLog(uint64(i), &out); err != ErrLogNotFound { - t.Fatalf("err: %v ", err) - } - } - - // Index should be one - idx, err = l.FirstID() - if err != nil { - t.Fatalf("err: %v ", err) - } - if idx != 6 { - t.Fatalf("bad idx: %d", idx) - } - idx, err = l.LastID() - if err != nil { - t.Fatalf("err: %v ", err) - } - if idx != 20 { - t.Fatalf("bad idx: %d", idx) - } - - // Should not be able to fetch - if err := l.GetLog(5, &out); err != ErrLogNotFound { - t.Fatalf("err: %v ", err) - } - - if err := l.Clear(); err != nil { - t.Fatal(err) - } - - idx, err = l.FirstID() - if err != nil { - t.Fatalf("err: %v ", err) - } - if idx != 0 { - t.Fatalf("bad idx: %d", idx) - } - - idx, err = l.LastID() - if err != nil { - t.Fatalf("err: %v ", err) - } - if idx != 0 { - t.Fatalf("bad idx: %d", idx) - } - - now := uint32(time.Now().Unix()) - logs := []*Log{} - for i := 1; i <= 20; i++ { - nl := &Log{ - ID: uint64(i), - CreateTime: now - 20, - Data: []byte("first"), - } - logs = append(logs, nl) - } - - if err := l.PurgeExpired(1); err != nil { - t.Fatal(err) - } - - idx, err = l.FirstID() - if err != nil { - t.Fatalf("err: %v ", err) - } - if idx != 0 { - t.Fatalf("bad idx: %d", idx) - } - - idx, err = l.LastID() - if err != nil { - t.Fatalf("err: %v ", err) - } - if idx != 0 { - t.Fatalf("bad idx: %d", idx) - } } From 34d485056f428dba1a24da4c923ba48f3df42f46 Mon Sep 17 00:00:00 2001 From: siddontang Date: Fri, 7 Nov 2014 16:35:54 +0800 Subject: [PATCH 63/98] update replication --- config/config.go | 2 + config/config.toml | 7 ++ rpl/file_store.go | 162 ++++++++++++++++++++++++++++---------- rpl/file_table.go | 33 +++++++- rpl/rpl.go | 12 ++- rpl/store_test.go | 66 +++++++++++++++- rpl/table_readers_test.go | 38 +++++++++ 7 files changed, 270 insertions(+), 50 deletions(-) create mode 100644 rpl/table_readers_test.go diff --git a/config/config.go b/config/config.go index 498c748..1634fa0 100644 --- a/config/config.go +++ b/config/config.go @@ -74,6 +74,8 @@ type ReplicationConfig struct { WaitSyncTime int `toml:"wait_sync_time"` WaitMaxSlaveAcks int `toml:"wait_max_slave_acks"` ExpiredLogDays int `toml:"expired_log_days"` + StoreName string `toml:"store_name"` + MaxLogFileSize int64 `toml:"max_log_file_size"` SyncLog int `toml:"sync_log"` Compression bool `toml:"compression"` } diff --git a/config/config.toml b/config/config.toml index f75c784..f121543 100644 --- a/config/config.toml +++ b/config/config.toml @@ -121,9 +121,16 @@ wait_sync_time = 500 # If 0, wait (n + 1) / 2 acks. wait_max_slave_acks = 2 +# store name: file, goleveldb +# change in runtime is very dangerous +store_name = "file" + # Expire write ahead logs after the given days expired_log_days = 7 +# for file store, if 0, use default 1G, max is 4G +max_log_file_size = 0 + # Sync log to disk if possible # 0: no sync # 1: sync every second diff --git a/rpl/file_store.go b/rpl/file_store.go index f88b603..3c5e719 100644 --- a/rpl/file_store.go +++ b/rpl/file_store.go @@ -60,7 +60,7 @@ type FileStore struct { w *tableWriter } -func NewFileStore(base string, maxSize int64) (*FileStore, error) { +func NewFileStore(base string, maxSize int64, syncType int) (*FileStore, error) { s := new(FileStore) var err error @@ -72,6 +72,9 @@ func NewFileStore(base string, maxSize int64) (*FileStore, error) { s.base = base s.maxFileSize = num.MinInt64(maxLogFileSize, maxSize) + if s.maxFileSize == 0 { + s.maxFileSize = defaultMaxLogFileSize + } if err = s.load(); err != nil { return nil, err @@ -83,30 +86,75 @@ func NewFileStore(base string, maxSize int64) (*FileStore, error) { } s.w = newTableWriter(s.base, index, s.maxFileSize) + s.w.SetSyncType(syncType) + return s, nil } -func (s *FileStore) GetLog(id uint64, log *Log) error { - panic("not implementation") - return nil +func (s *FileStore) GetLog(id uint64, l *Log) error { + //first search in table writer + if err := s.w.GetLog(id, l); err == nil { + return nil + } else if err != ErrLogNotFound { + return err + } + + s.rm.RLock() + t := s.rs.Search(id) + + if t == nil { + s.rm.RUnlock() + + return ErrLogNotFound + } + + err := t.GetLog(id, l) + s.rm.RUnlock() + + return err } func (s *FileStore) FirstID() (uint64, error) { - return 0, nil + id := uint64(0) + + s.rm.RLock() + if len(s.rs) > 0 { + id = s.rs[0].first + } else { + id = 0 + } + s.rm.RUnlock() + + if id > 0 { + return id, nil + } + + //if id = 0, + + return s.w.First(), nil } func (s *FileStore) LastID() (uint64, error) { - return 0, nil + id := s.w.Last() + if id > 0 { + return id, nil + } + + //if table writer has no last id, we may find in the last table reader + + s.rm.RLock() + if len(s.rs) > 0 { + id = s.rs[len(s.rs)-1].last + } + s.rm.RUnlock() + + return id, nil } func (s *FileStore) StoreLog(l *Log) error { s.wm.Lock() defer s.wm.Unlock() - if s.w == nil { - return fmt.Errorf("nil table writer, cannot store") - } - err := s.w.StoreLog(l) if err == nil { return nil @@ -114,40 +162,40 @@ func (s *FileStore) StoreLog(l *Log) error { return err } + s.rm.Lock() + var r *tableReader if r, err = s.w.Flush(); err != nil { log.Error("write table flush error %s, can not store now", err.Error()) s.w.Close() - s.w = nil + + s.rm.Unlock() + return err } - s.rm.Lock() s.rs = append(s.rs, r) s.rm.Unlock() - return nil + return s.w.StoreLog(l) } func (s *FileStore) PuregeExpired(n int64) error { s.rm.Lock() purges := []*tableReader{} - lefts := []*tableReader{} t := uint32(time.Now().Unix() - int64(n)) - for _, r := range s.rs { - if r.lastTime < t { - purges = append(purges, r) - } else { - lefts = append(lefts, r) + for i, r := range s.rs { + if r.lastTime > t { + purges = s.rs[0:i] + s.rs = s.rs[i:] + break } } - s.rs = lefts - s.rm.Unlock() for _, r := range purges { @@ -162,31 +210,53 @@ func (s *FileStore) PuregeExpired(n int64) error { } func (s *FileStore) Clear() error { + s.wm.Lock() + s.rm.Lock() + + defer func() { + s.rm.Unlock() + s.wm.Unlock() + }() + + s.w.Close() + + for i := range s.rs { + s.rs[i].Close() + } + + s.rs = tableReaders{} + + if err := os.RemoveAll(s.base); err != nil { + return err + } + + if err := os.MkdirAll(s.base, 0755); err != nil { + return err + } + + s.w = newTableWriter(s.base, 1, s.maxFileSize) + return nil } func (s *FileStore) Close() error { s.wm.Lock() - if s.w != nil { - if r, err := s.w.Flush(); err != nil { - log.Error("close err: %s", err.Error()) - } else { - r.Close() - s.w.Close() - s.w = nil - } - } - - s.wm.Unlock() - s.rm.Lock() + if r, err := s.w.Flush(); err != nil { + log.Error("close err: %s", err.Error()) + } else { + r.Close() + s.w.Close() + } + for i := range s.rs { s.rs[i].Close() } - s.rs = nil + s.rs = tableReaders{} s.rm.Unlock() + s.wm.Unlock() return nil } @@ -227,19 +297,25 @@ func (ts tableReaders) Swap(i, j int) { } func (ts tableReaders) Less(i, j int) bool { - return ts[i].index < ts[j].index + return ts[i].first < ts[j].first } func (ts tableReaders) Search(id uint64) *tableReader { - n := sort.Search(len(ts), func(i int) bool { - return id >= ts[i].first && id <= ts[i].last - }) + i, j := 0, len(ts)-1 - if n < len(ts) { - return ts[n] - } else { - return nil + for i <= j { + h := i + (j-i)/2 + + if ts[h].first <= id && id <= ts[h].last { + return ts[h] + } else if ts[h].last < id { + i = h + 1 + } else { + j = h - 1 + } } + + return nil } func (ts tableReaders) check() error { @@ -262,7 +338,7 @@ func (ts tableReaders) check() error { return fmt.Errorf("invalid first log id %d in table %s", ts[i].first, ts[i].name) } - if ts[i].index == index { + if ts[i].index <= index { return fmt.Errorf("invalid index %d in table %s", ts[i].index, ts[i].name) } diff --git a/rpl/file_table.go b/rpl/file_table.go index 410f150..9023aa3 100644 --- a/rpl/file_table.go +++ b/rpl/file_table.go @@ -293,7 +293,7 @@ func (t *tableReader) decodeLogHead(l *Log, pos int64) (int64, error) { func (t *tableReader) GetLog(id uint64, l *Log) error { if id < t.first || id > t.last { - return fmt.Errorf("log %d not in [%d=%d]", id, t.first, t.last) + return ErrLogNotFound } t.lastReadTime.Set(time.Now().Unix()) @@ -359,6 +359,11 @@ type tableWriter struct { offsetBuf []byte maxLogSize int64 + + closed bool + + syncType int + lastTime uint32 } func newTableWriter(base string, index int64, maxLogSize int64) *tableWriter { @@ -374,9 +379,19 @@ func newTableWriter(base string, index int64, maxLogSize int64) *tableWriter { t.maxLogSize = maxLogSize + t.closed = false + return t } +func (t *tableWriter) SetMaxLogSize(s int64) { + t.maxLogSize = s +} + +func (t *tableWriter) SetSyncType(tp int) { + t.syncType = tp +} + func (t *tableWriter) close() { if t.rf != nil { t.rf.Close() @@ -393,6 +408,8 @@ func (t *tableWriter) Close() { t.Lock() defer t.Unlock() + t.closed = true + t.close() } @@ -502,6 +519,10 @@ func (t *tableWriter) StoreLog(l *Log) error { t.Lock() defer t.Unlock() + if t.closed { + return fmt.Errorf("table writer is closed") + } + if t.last > 0 && l.ID != t.last+1 { return ErrStoreLogID } @@ -539,8 +560,16 @@ func (t *tableWriter) StoreLog(l *Log) error { t.last = l.ID + t.lastTime = l.CreateTime + //todo add LRU cache + if t.syncType == 2 || (t.syncType == 1 && time.Now().Unix()-int64(t.lastTime) > 1) { + if err := t.wf.Sync(); err != nil { + log.Error("sync table error %s", err.Error()) + } + } + return nil } @@ -549,7 +578,7 @@ func (t *tableWriter) GetLog(id uint64, l *Log) error { defer t.RUnlock() if id < t.first || id > t.last { - return fmt.Errorf("log %d not in [%d=%d]", id, t.first, t.last) + return ErrLogNotFound } //todo memory cache diff --git a/rpl/rpl.go b/rpl/rpl.go index 067c0c2..5e9eca8 100644 --- a/rpl/rpl.go +++ b/rpl/rpl.go @@ -51,8 +51,16 @@ func NewReplication(cfg *config.Config) (*Replication, error) { r.cfg = cfg var err error - if r.s, err = NewGoLevelDBStore(path.Join(base, "wal"), cfg.Replication.SyncLog); err != nil { - return nil, err + + switch cfg.Replication.StoreName { + case "goleveldb": + if r.s, err = NewGoLevelDBStore(path.Join(base, "wal"), cfg.Replication.SyncLog); err != nil { + return nil, err + } + default: + if r.s, err = NewFileStore(path.Join(base, "ldb"), cfg.Replication.MaxLogFileSize, cfg.Replication.SyncLog); err != nil { + return nil, err + } } if r.commitLog, err = os.OpenFile(path.Join(base, "commit.log"), os.O_RDWR|os.O_CREATE, 0644); err != nil { diff --git a/rpl/store_test.go b/rpl/store_test.go index e84e286..cdec659 100644 --- a/rpl/store_test.go +++ b/rpl/store_test.go @@ -24,6 +24,24 @@ func TestGoLevelDBStore(t *testing.T) { testLogs(t, l) } +func TestFileStore(t *testing.T) { + // Create a test dir + dir, err := ioutil.TempDir("", "ldb") + if err != nil { + t.Fatalf("err: %v ", err) + } + defer os.RemoveAll(dir) + + // New level + l, err := NewFileStore(dir, 4096, 0) + if err != nil { + t.Fatalf("err: %v ", err) + } + defer l.Close() + + testLogs(t, l) +} + func testLogs(t *testing.T, l LogStore) { // Should be no first index idx, err := l.FirstID() @@ -45,14 +63,16 @@ func testLogs(t *testing.T, l LogStore) { // Try a filed fetch var out Log - if err := l.GetLog(10, &out); err.Error() != "log not found" { + if err := l.GetLog(10, &out); err != ErrLogNotFound { t.Fatalf("err: %v ", err) } + data := make([]byte, 1024) + // Write out a log log := Log{ ID: 1, - Data: []byte("first"), + Data: data, } for i := 1; i <= 10; i++ { log.ID = uint64(i) @@ -65,7 +85,7 @@ func testLogs(t *testing.T, l LogStore) { for i := 11; i <= 20; i++ { nl := &Log{ ID: uint64(i), - Data: []byte("first"), + Data: data, } if err := l.StoreLog(nl); err != nil { @@ -73,6 +93,11 @@ func testLogs(t *testing.T, l LogStore) { } } + // Try to fetch + if err := l.GetLog(1, &out); err != nil { + t.Fatalf("err: %v ", err) + } + // Try to fetch if err := l.GetLog(10, &out); err != nil { t.Fatalf("err: %v ", err) @@ -100,4 +125,39 @@ func testLogs(t *testing.T, l LogStore) { if idx != 20 { t.Fatalf("bad idx: %d", idx) } + + if err = l.Clear(); err != nil { + t.Fatalf("err :%v", err) + } + + // Check the lowest index + idx, err = l.FirstID() + if err != nil { + t.Fatalf("err: %v ", err) + } + if idx != 0 { + t.Fatalf("bad idx: %d", idx) + } + + // Check the highest index + idx, err = l.LastID() + if err != nil { + t.Fatalf("err: %v ", err) + } + if idx != 0 { + t.Fatalf("bad idx: %d", idx) + } + + // Write out a log + log = Log{ + ID: 1, + Data: data, + } + for i := 1; i <= 10; i++ { + log.ID = uint64(i) + if err := l.StoreLog(&log); err != nil { + t.Fatalf("err: %v", err) + } + } + } diff --git a/rpl/table_readers_test.go b/rpl/table_readers_test.go new file mode 100644 index 0000000..d0045e6 --- /dev/null +++ b/rpl/table_readers_test.go @@ -0,0 +1,38 @@ +package rpl + +import ( + "testing" +) + +func TestTableReaders(t *testing.T) { + ts := make(tableReaders, 0, 10) + + for i := uint64(0); i < 10; i++ { + t := new(tableReader) + t.index = int64(i) + 1 + t.first = i*10 + 1 + t.last = i*10 + 10 + + ts = append(ts, t) + } + + if err := ts.check(); err != nil { + t.Fatal(err) + } + + for i := 1; i <= 100; i++ { + if r := ts.Search(uint64(i)); r == nil { + t.Fatal("must hit", i) + } else if r.index != int64((i-1)/10)+1 { + t.Fatal("invalid index", r.index, i) + } + } + + if r := ts.Search(1000); r != nil { + t.Fatal("must not hit") + } + if r := ts.Search(0); r != nil { + t.Fatal("must not hit") + } + +} From c0f8a436a1ef9de966accf393b037c12ac4ad559 Mon Sep 17 00:00:00 2001 From: siddontang Date: Sun, 9 Nov 2014 23:10:03 +0800 Subject: [PATCH 64/98] write batch add data function leveled will support it later --- Godeps/Godeps.json | 2 +- store/driver/batch.go | 52 +++++++++++++++++++++++++++------------- store/driver/driver.go | 1 + store/goleveldb/batch.go | 4 ++++ store/leveldb/batch.go | 5 ++++ store/rocksdb/batch.go | 7 ++++++ 6 files changed, 53 insertions(+), 18 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 362496e..9fa9487 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -56,7 +56,7 @@ }, { "ImportPath": "github.com/siddontang/goleveldb/leveldb", - "Rev": "448b15792e63fe835c48d294f9b7859a86f4b76e" + "Rev": "41805642b981fb3d9462f6641bcb94b8609ca791" }, { "ImportPath": "github.com/szferi/gomdb", diff --git a/store/driver/batch.go b/store/driver/batch.go index 5fc461f..33c07ed 100644 --- a/store/driver/batch.go +++ b/store/driver/batch.go @@ -1,5 +1,9 @@ package driver +import ( + "github.com/siddontang/goleveldb/leveldb" +) + type BatchPuter interface { BatchPut([]Write) error SyncBatchPut([]Write) error @@ -11,34 +15,48 @@ type Write struct { } type WriteBatch struct { - batch BatchPuter - wb []Write + d *leveldb.Batch + + wb []Write + w BatchPuter } -func (w *WriteBatch) Put(key, value []byte) { - if value == nil { - value = []byte{} - } - w.wb = append(w.wb, Write{key, value}) +func (wb *WriteBatch) Put(key, value []byte) { + wb.wb = append(wb.wb, Write{key, value}) } -func (w *WriteBatch) Delete(key []byte) { - w.wb = append(w.wb, Write{key, nil}) +func (wb *WriteBatch) Delete(key []byte) { + wb.wb = append(wb.wb, Write{key, nil}) } -func (w *WriteBatch) Commit() error { - return w.batch.BatchPut(w.wb) +func (wb *WriteBatch) Commit() error { + return wb.w.BatchPut(wb.wb) } -func (w *WriteBatch) SyncCommit() error { - return w.batch.SyncBatchPut(w.wb) +func (wb *WriteBatch) SyncCommit() error { + return wb.w.SyncBatchPut(wb.wb) } -func (w *WriteBatch) Rollback() error { - w.wb = w.wb[0:0] +func (wb *WriteBatch) Rollback() error { + wb.wb = wb.wb[0:0] return nil } -func NewWriteBatch(puter BatchPuter) IWriteBatch { - return &WriteBatch{puter, []Write{}} +func (wb *WriteBatch) Data() []byte { + wb.d.Reset() + for _, w := range wb.wb { + if w.Value == nil { + wb.Delete(w.Key) + } else { + wb.Put(w.Key, w.Value) + } + } + return wb.d.Dump() +} + +func NewWriteBatch(puter BatchPuter) *WriteBatch { + return &WriteBatch{ + &leveldb.Batch{}, + []Write{}, + puter} } diff --git a/store/driver/driver.go b/store/driver/driver.go index 31d15de..04133c7 100644 --- a/store/driver/driver.go +++ b/store/driver/driver.go @@ -58,6 +58,7 @@ type IWriteBatch interface { Commit() error SyncCommit() error Rollback() error + Data() []byte } type Tx interface { diff --git a/store/goleveldb/batch.go b/store/goleveldb/batch.go index 85b78c6..aef0f55 100644 --- a/store/goleveldb/batch.go +++ b/store/goleveldb/batch.go @@ -29,3 +29,7 @@ func (w *WriteBatch) Rollback() error { w.wbatch.Reset() return nil } + +func (w *WriteBatch) Data() []byte { + return w.wbatch.Dump() +} diff --git a/store/leveldb/batch.go b/store/leveldb/batch.go index caadc03..777e11c 100644 --- a/store/leveldb/batch.go +++ b/store/leveldb/batch.go @@ -63,3 +63,8 @@ func (w *WriteBatch) commit(wb *WriteOptions) error { } return nil } + +func (w *WriteBatch) Data() []byte { + //todo later + return nil +} diff --git a/store/rocksdb/batch.go b/store/rocksdb/batch.go index 6a94a63..aa4e407 100644 --- a/store/rocksdb/batch.go +++ b/store/rocksdb/batch.go @@ -73,3 +73,10 @@ func (w *WriteBatch) commit(wb *WriteOptions) error { } return nil } + +func (w *WriteBatch) Data() []byte { + var vallen C.size_t + value := C.rocksdb_writebatch_data(w.wbatch, &vallen) + + return slice(unsafe.Pointer(value), int(vallen)) +} From 38e5d7262c25cf69c5a5ff9f570dc627d0968787 Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 10 Nov 2014 11:07:21 +0800 Subject: [PATCH 65/98] leveldb batch support data optimize later --- store/leveldb/batch.go | 35 +++++++++++++++++++++++++++++++++-- store/leveldb/db.go | 7 +------ 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/store/leveldb/batch.go b/store/leveldb/batch.go index 777e11c..542a322 100644 --- a/store/leveldb/batch.go +++ b/store/leveldb/batch.go @@ -7,22 +7,45 @@ package leveldb import "C" import ( + "github.com/siddontang/goleveldb/leveldb" "unsafe" ) +/* + It's not easy to let leveldb support Data() function like rocksdb, + so here, we will use goleveldb batch instead + + later optimize +*/ + type WriteBatch struct { db *DB wbatch *C.leveldb_writebatch_t + + gbatch *leveldb.Batch +} + +func newWriteBatch(db *DB) *WriteBatch { + w := new(WriteBatch) + w.db = db + w.wbatch = C.leveldb_writebatch_create() + w.gbatch = new(leveldb.Batch) + + return w } func (w *WriteBatch) Close() error { C.leveldb_writebatch_destroy(w.wbatch) w.wbatch = nil + w.gbatch = nil + return nil } func (w *WriteBatch) Put(key, value []byte) { + w.gbatch.Put(key, value) + var k, v *C.char if len(key) != 0 { k = (*C.char)(unsafe.Pointer(&key[0])) @@ -38,20 +61,29 @@ func (w *WriteBatch) Put(key, value []byte) { } func (w *WriteBatch) Delete(key []byte) { + w.gbatch.Delete(key) + C.leveldb_writebatch_delete(w.wbatch, (*C.char)(unsafe.Pointer(&key[0])), C.size_t(len(key))) } func (w *WriteBatch) Commit() error { + w.gbatch.Reset() + return w.commit(w.db.writeOpts) } func (w *WriteBatch) SyncCommit() error { + w.gbatch.Reset() + return w.commit(w.db.syncOpts) } func (w *WriteBatch) Rollback() error { C.leveldb_writebatch_clear(w.wbatch) + + w.gbatch.Reset() + return nil } @@ -65,6 +97,5 @@ func (w *WriteBatch) commit(wb *WriteOptions) error { } func (w *WriteBatch) Data() []byte { - //todo later - return nil + return w.gbatch.Dump() } diff --git a/store/leveldb/db.go b/store/leveldb/db.go index e5ab88c..a2d9c26 100644 --- a/store/leveldb/db.go +++ b/store/leveldb/db.go @@ -182,12 +182,7 @@ func (db *DB) SyncDelete(key []byte) error { } func (db *DB) NewWriteBatch() driver.IWriteBatch { - wb := &WriteBatch{ - db: db, - wbatch: C.leveldb_writebatch_create(), - } - - return wb + return newWriteBatch(db) } func (db *DB) NewIterator() driver.IIterator { From 4a266b30b26fa18bc8cd434d175bba6b61931740 Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 10 Nov 2014 11:07:33 +0800 Subject: [PATCH 66/98] batch support data --- store/driver/batch.go | 7 ++-- store/store_test.go | 51 ++++++++++++++++++++++++++++ store/writebatch.go | 77 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 2 deletions(-) diff --git a/store/driver/batch.go b/store/driver/batch.go index 33c07ed..5a47c1b 100644 --- a/store/driver/batch.go +++ b/store/driver/batch.go @@ -22,6 +22,9 @@ type WriteBatch struct { } func (wb *WriteBatch) Put(key, value []byte) { + if value == nil { + value = []byte{} + } wb.wb = append(wb.wb, Write{key, value}) } @@ -46,9 +49,9 @@ func (wb *WriteBatch) Data() []byte { wb.d.Reset() for _, w := range wb.wb { if w.Value == nil { - wb.Delete(w.Key) + wb.d.Delete(w.Key) } else { - wb.Put(w.Key, w.Value) + wb.d.Put(w.Key, w.Value) } } return wb.d.Dump() diff --git a/store/store_test.go b/store/store_test.go index 7626d5b..edb20bb 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -6,6 +6,7 @@ import ( "github.com/siddontang/ledisdb/config" "github.com/siddontang/ledisdb/store/driver" "os" + "reflect" "testing" ) @@ -38,6 +39,7 @@ func testStore(db *DB, t *testing.T) { testBatch(db, t) testIterator(db, t) testSnapshot(db, t) + testBatchData(db, t) } func testClear(db *DB, t *testing.T) { @@ -342,3 +344,52 @@ func testSnapshot(db *DB, t *testing.T) { } } + +func testBatchData(db *DB, t *testing.T) { + w := db.NewWriteBatch() + + w.Put([]byte("a"), []byte("1")) + w.Put([]byte("b"), nil) + w.Delete([]byte("c")) + + d, err := w.Data() + if err != nil { + t.Fatal(err) + } + + if kvs, err := d.Items(); err != nil { + t.Fatal(err) + } else if len(kvs) != 3 { + t.Fatal(len(kvs)) + } else if !reflect.DeepEqual(kvs[0], BatchItem{[]byte("a"), []byte("1")}) { + t.Fatal("must equal") + } else if !reflect.DeepEqual(kvs[1], BatchItem{[]byte("b"), []byte{}}) { + t.Fatal("must equal") + } else if !reflect.DeepEqual(kvs[2], BatchItem{[]byte("c"), nil}) { + t.Fatal("must equal") + } + + if err := d.Append(d); err != nil { + t.Fatal(err) + } else if d.Len() != 6 { + t.Fatal(d.Len()) + } + + if kvs, err := d.Items(); err != nil { + t.Fatal(err) + } else if len(kvs) != 6 { + t.Fatal(len(kvs)) + } else if !reflect.DeepEqual(kvs[0], BatchItem{[]byte("a"), []byte("1")}) { + t.Fatal("must equal") + } else if !reflect.DeepEqual(kvs[1], BatchItem{[]byte("b"), []byte{}}) { + t.Fatal("must equal") + } else if !reflect.DeepEqual(kvs[2], BatchItem{[]byte("c"), nil}) { + t.Fatal("must equal") + } else if !reflect.DeepEqual(kvs[3], BatchItem{[]byte("a"), []byte("1")}) { + t.Fatal("must equal") + } else if !reflect.DeepEqual(kvs[4], BatchItem{[]byte("b"), []byte{}}) { + t.Fatal("must equal") + } else if !reflect.DeepEqual(kvs[5], BatchItem{[]byte("c"), nil}) { + t.Fatal("must equal") + } +} diff --git a/store/writebatch.go b/store/writebatch.go index a7fd84a..134ad42 100644 --- a/store/writebatch.go +++ b/store/writebatch.go @@ -1,6 +1,8 @@ package store import ( + "encoding/binary" + "github.com/siddontang/goleveldb/leveldb" "github.com/siddontang/ledisdb/store/driver" "time" ) @@ -50,3 +52,78 @@ func (wb *WriteBatch) Rollback() error { return wb.wb.Rollback() } + +func (wb *WriteBatch) Data() (*BatchData, error) { + data := wb.wb.Data() + return NewBatchData(data) +} + +const BatchDataHeadLen = 12 + +/* + see leveldb batch data format for more information +*/ + +type BatchData struct { + leveldb.Batch +} + +func NewBatchData(data []byte) (*BatchData, error) { + b := new(BatchData) + + if err := b.Load(data); err != nil { + return nil, err + } + + return b, nil +} + +func (d *BatchData) Append(do *BatchData) error { + d1 := d.Dump() + d2 := do.Dump() + + n := d.Len() + do.Len() + + binary.LittleEndian.PutUint32(d1[8:], uint32(n)) + d1 = append(d1, d2[BatchDataHeadLen:]...) + + return d.Load(d1) +} + +func (d *BatchData) Data() []byte { + return d.Dump() +} + +type BatchDataReplay interface { + Put(key, value []byte) + Delete(key []byte) +} + +type BatchItem struct { + Key []byte + Value []byte +} + +type batchItems []BatchItem + +func (bs *batchItems) Put(key, value []byte) { + *bs = append(*bs, BatchItem{key, value}) +} + +func (bs *batchItems) Delete(key []byte) { + *bs = append(*bs, BatchItem{key, nil}) +} + +func (d *BatchData) Replay(r BatchDataReplay) error { + return d.Batch.Replay(r) +} + +func (d *BatchData) Items() ([]BatchItem, error) { + is := make(batchItems, 0, d.Len()) + + if err := d.Replay(&is); err != nil { + return nil, err + } + + return []BatchItem(is), nil +} From d058c2009424e88c3831648127296c03ec8918ce Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 10 Nov 2014 12:40:19 +0800 Subject: [PATCH 67/98] refactor replication replication log format is not compatibility --- ledis/batch.go | 27 +++++-------- ledis/event.go | 90 +------------------------------------------- ledis/event_test.go | 56 --------------------------- ledis/replication.go | 8 +++- ledis/tx.go | 9 +++-- store/store_test.go | 5 +-- store/writebatch.go | 22 +++++++++-- 7 files changed, 43 insertions(+), 174 deletions(-) delete mode 100644 ledis/event_test.go diff --git a/ledis/batch.go b/ledis/batch.go index f5fe061..526b4b6 100644 --- a/ledis/batch.go +++ b/ledis/batch.go @@ -15,8 +15,6 @@ type batch struct { sync.Locker tx *Tx - - eb *eventBatch } func (b *batch) Commit() error { @@ -25,10 +23,12 @@ func (b *batch) Commit() error { } if b.tx == nil { - return b.l.handleCommit(b.eb, b.WriteBatch) + return b.l.handleCommit(b.WriteBatch, b.WriteBatch) } else { if b.l.r != nil { - b.tx.eb.Write(b.eb.Bytes()) + if err := b.tx.data.Append(b.WriteBatch.BatchData()); err != nil { + return err + } } return b.WriteBatch.Commit() } @@ -39,25 +39,15 @@ func (b *batch) Lock() { } func (b *batch) Unlock() { - b.eb.Reset() - b.WriteBatch.Rollback() b.Locker.Unlock() } func (b *batch) Put(key []byte, value []byte) { - if b.l.r != nil { - b.eb.Put(key, value) - } - b.WriteBatch.Put(key, value) } func (b *batch) Delete(key []byte) { - if b.l.r != nil { - b.eb.Delete(key) - } - b.WriteBatch.Delete(key) } @@ -96,7 +86,6 @@ func (l *Ledis) newBatch(wb *store.WriteBatch, locker sync.Locker, tx *Tx) *batc b.Locker = locker b.tx = tx - b.eb = new(eventBatch) return b } @@ -105,14 +94,18 @@ type commiter interface { Commit() error } -func (l *Ledis) handleCommit(eb *eventBatch, c commiter) error { +type commitDataGetter interface { + Data() []byte +} + +func (l *Ledis) handleCommit(g commitDataGetter, c commiter) error { l.commitLock.Lock() defer l.commitLock.Unlock() var err error if l.r != nil { var rl *rpl.Log - if rl, err = l.r.Log(eb.Bytes()); err != nil { + if rl, err = l.r.Log(g.Data()); err != nil { log.Fatal("write wal error %s", err.Error()) return err } diff --git a/ledis/event.go b/ledis/event.go index 72ac373..2a3b54a 100644 --- a/ledis/event.go +++ b/ledis/event.go @@ -1,101 +1,13 @@ package ledis import ( - "bytes" - "encoding/binary" "errors" "fmt" "github.com/siddontang/go/hack" - "io" "strconv" ) -const ( - kTypeDeleteEvent uint8 = 0 - kTypePutEvent uint8 = 1 -) - -var ( - errInvalidPutEvent = errors.New("invalid put event") - errInvalidDeleteEvent = errors.New("invalid delete event") - errInvalidEvent = errors.New("invalid event") -) - -type eventBatch struct { - bytes.Buffer -} - -func (b *eventBatch) Put(key []byte, value []byte) { - l := uint32(len(key) + len(value) + 1 + 2) - binary.Write(b, binary.BigEndian, l) - b.WriteByte(kTypePutEvent) - keyLen := uint16(len(key)) - binary.Write(b, binary.BigEndian, keyLen) - b.Write(key) - b.Write(value) -} - -func (b *eventBatch) Delete(key []byte) { - l := uint32(len(key) + 1) - binary.Write(b, binary.BigEndian, l) - b.WriteByte(kTypeDeleteEvent) - b.Write(key) -} - -type eventWriter interface { - Put(key []byte, value []byte) - Delete(key []byte) -} - -func decodeEventBatch(w eventWriter, data []byte) error { - for { - if len(data) == 0 { - return nil - } - - if len(data) < 4 { - return io.ErrUnexpectedEOF - } - - l := binary.BigEndian.Uint32(data) - data = data[4:] - if uint32(len(data)) < l { - return io.ErrUnexpectedEOF - } - - if err := decodeEvent(w, data[0:l]); err != nil { - return err - } - data = data[l:] - } -} - -func decodeEvent(w eventWriter, b []byte) error { - if len(b) == 0 { - return errInvalidEvent - } - - switch b[0] { - case kTypePutEvent: - if len(b[1:]) < 2 { - return errInvalidPutEvent - } - - keyLen := binary.BigEndian.Uint16(b[1:3]) - b = b[3:] - if len(b) < int(keyLen) { - return errInvalidPutEvent - } - - w.Put(b[0:keyLen], b[keyLen:]) - case kTypeDeleteEvent: - w.Delete(b[1:]) - default: - return errInvalidEvent - } - - return nil -} +var errInvalidEvent = errors.New("invalid event") func formatEventKey(buf []byte, k []byte) ([]byte, error) { if len(k) < 2 { diff --git a/ledis/event_test.go b/ledis/event_test.go deleted file mode 100644 index d2271e2..0000000 --- a/ledis/event_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package ledis - -import ( - "reflect" - "testing" -) - -type testEvent struct { - Key []byte - Value []byte -} - -type testEventWriter struct { - evs []testEvent -} - -func (w *testEventWriter) Put(key []byte, value []byte) { - e := testEvent{key, value} - w.evs = append(w.evs, e) -} - -func (w *testEventWriter) Delete(key []byte) { - e := testEvent{key, nil} - w.evs = append(w.evs, e) -} - -func TestEvent(t *testing.T) { - k1 := []byte("k1") - v1 := []byte("v1") - k2 := []byte("k2") - k3 := []byte("k3") - v3 := []byte("v3") - - b := new(eventBatch) - - b.Put(k1, v1) - b.Delete(k2) - b.Put(k3, v3) - - buf := b.Bytes() - - w := &testEventWriter{} - - ev2 := &testEventWriter{ - evs: []testEvent{ - testEvent{k1, v1}, - testEvent{k2, nil}, - testEvent{k3, v3}}, - } - - if err := decodeEventBatch(w, buf); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(w, ev2) { - t.Fatal("not equal") - } -} diff --git a/ledis/replication.go b/ledis/replication.go index 33a5e12..593388d 100644 --- a/ledis/replication.go +++ b/ledis/replication.go @@ -6,6 +6,7 @@ import ( "github.com/siddontang/go/log" "github.com/siddontang/go/snappy" "github.com/siddontang/ledisdb/rpl" + "github.com/siddontang/ledisdb/store" "io" "time" ) @@ -49,7 +50,12 @@ func (l *Ledis) handleReplication() error { } } - decodeEventBatch(l.rbatch, rl.Data) + if bd, err := store.NewBatchData(rl.Data); err != nil { + log.Error("decode batch log error %s", err.Error()) + return err + } else if err = bd.Replay(l.rbatch); err != nil { + log.Error("replay batch log error %s", err.Error()) + } l.commitLock.Lock() if err = l.rbatch.Commit(); err != nil { diff --git a/ledis/tx.go b/ledis/tx.go index 5c1c52a..03d6c5b 100644 --- a/ledis/tx.go +++ b/ledis/tx.go @@ -16,7 +16,7 @@ type Tx struct { tx *store.Tx - eb *eventBatch + data *store.BatchData } func (db *DB) IsTransaction() bool { @@ -32,7 +32,7 @@ func (db *DB) Begin() (*Tx, error) { tx := new(Tx) - tx.eb = new(eventBatch) + tx.data = &store.BatchData{} tx.DB = new(DB) tx.DB.l = db.l @@ -71,7 +71,8 @@ func (tx *Tx) Commit() error { return ErrTxDone } - err := tx.l.handleCommit(tx.eb, tx.tx) + err := tx.l.handleCommit(tx.data, tx.tx) + tx.data.Reset() tx.tx = nil @@ -88,7 +89,7 @@ func (tx *Tx) Rollback() error { } err := tx.tx.Rollback() - tx.eb.Reset() + tx.data.Reset() tx.tx = nil tx.l.wLock.Unlock() diff --git a/store/store_test.go b/store/store_test.go index edb20bb..9045ce2 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -352,10 +352,7 @@ func testBatchData(db *DB, t *testing.T) { w.Put([]byte("b"), nil) w.Delete([]byte("c")) - d, err := w.Data() - if err != nil { - t.Fatal(err) - } + d := w.BatchData() if kvs, err := d.Items(); err != nil { t.Fatal(err) diff --git a/store/writebatch.go b/store/writebatch.go index 134ad42..dc825c1 100644 --- a/store/writebatch.go +++ b/store/writebatch.go @@ -53,9 +53,21 @@ func (wb *WriteBatch) Rollback() error { return wb.wb.Rollback() } -func (wb *WriteBatch) Data() (*BatchData, error) { +// the data will be undefined after commit or rollback +func (wb *WriteBatch) BatchData() *BatchData { data := wb.wb.Data() - return NewBatchData(data) + d, err := NewBatchData(data) + if err != nil { + //can not enter this + panic(err) + } + + return d +} + +func (wb *WriteBatch) Data() []byte { + b := wb.BatchData() + return b.Data() } const BatchDataHeadLen = 12 @@ -84,8 +96,8 @@ func (d *BatchData) Append(do *BatchData) error { n := d.Len() + do.Len() - binary.LittleEndian.PutUint32(d1[8:], uint32(n)) d1 = append(d1, d2[BatchDataHeadLen:]...) + binary.LittleEndian.PutUint32(d1[8:], uint32(n)) return d.Load(d1) } @@ -94,6 +106,10 @@ func (d *BatchData) Data() []byte { return d.Dump() } +func (d *BatchData) Reset() { + d.Batch.Reset() +} + type BatchDataReplay interface { Put(key, value []byte) Delete(key []byte) From f0475b635e947d015880c9fae8ea8e0c3af4c133 Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 10 Nov 2014 13:09:37 +0800 Subject: [PATCH 68/98] client perform use coroutine hope improving performance --- server/client.go | 38 +++++++++++++++++++++++++++++++++++++- server/client_http.go | 1 + server/client_resp.go | 24 +++++++----------------- 3 files changed, 45 insertions(+), 18 deletions(-) diff --git a/server/client.go b/server/client.go index d39241b..57abc8c 100644 --- a/server/client.go +++ b/server/client.go @@ -6,6 +6,7 @@ import ( "github.com/siddontang/go/sync2" "github.com/siddontang/ledisdb/ledis" "io" + "sync" "time" ) @@ -74,6 +75,13 @@ type client struct { script *ledis.Multi slaveListeningAddr string + + quit chan struct{} + done chan error + + wg sync.WaitGroup + + fc chan CommandFunc } func newClient(app *App) *client { @@ -85,9 +93,35 @@ func newClient(app *App) *client { // c.reqErr = make(chan error) + c.quit = make(chan struct{}) + c.done = make(chan error, 1) + c.fc = make(chan CommandFunc, 1) + + c.wg.Add(1) + go c.run() + return c } +func (c *client) close() { + close(c.quit) + + c.wg.Wait() +} + +func (c *client) run() { + defer c.wg.Done() + + for { + select { + case <-c.quit: + return + case f := <-c.fc: + c.done <- f(c) + } + } +} + func (c *client) perform() { var err error @@ -114,7 +148,9 @@ func (c *client) perform() { // }() // err = <-c.reqErr - err = exeCmd(c) + c.fc <- exeCmd + + err = <-c.done } } diff --git a/server/client_http.go b/server/client_http.go index 057ba6b..d039533 100644 --- a/server/client_http.go +++ b/server/client_http.go @@ -50,6 +50,7 @@ func newClientHTTP(app *App, w http.ResponseWriter, r *http.Request) { return } c.perform() + c.client.close() } func (c *httpClient) addr(r *http.Request) string { diff --git a/server/client_resp.go b/server/client_resp.go index cfcaf5d..b538f1a 100644 --- a/server/client_resp.go +++ b/server/client_resp.go @@ -56,6 +56,8 @@ func (c *respClient) run() { c.app.info.addClients(1) defer func() { + c.client.close() + c.app.info.addClients(-1) if e := recover(); e != nil { @@ -82,32 +84,20 @@ func (c *respClient) run() { }() kc := time.Duration(c.app.cfg.ConnKeepaliveInterval) * time.Second - done := make(chan error) for { if kc > 0 { c.conn.SetReadDeadline(time.Now().Add(kc)) } - // I still don't know why use goroutine can improve performance - // if someone knows and benchamrks with another different result without goroutine, please tell me - go func() { - reqData, err := c.readRequest() - if err == nil { - c.handleRequest(reqData) - } + reqData, err := c.readRequest() + if err == nil { + c.handleRequest(reqData) + } - done <- err - }() - - // reqData, err := c.readRequest() - // if err == nil { - // c.handleRequest(reqData) - // } - - err := <-done if err != nil { return } + if c.conn == nil { return } From a0abd793282392aeceeb0632e3257b75466918a2 Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 10 Nov 2014 14:13:07 +0800 Subject: [PATCH 69/98] update default config --- etc/ledis.conf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/etc/ledis.conf b/etc/ledis.conf index f75c784..f121543 100644 --- a/etc/ledis.conf +++ b/etc/ledis.conf @@ -121,9 +121,16 @@ wait_sync_time = 500 # If 0, wait (n + 1) / 2 acks. wait_max_slave_acks = 2 +# store name: file, goleveldb +# change in runtime is very dangerous +store_name = "file" + # Expire write ahead logs after the given days expired_log_days = 7 +# for file store, if 0, use default 1G, max is 4G +max_log_file_size = 0 + # Sync log to disk if possible # 0: no sync # 1: sync every second From 9b4c23353ebf0cd981b68dbcc91da346e151944c Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 10 Nov 2014 14:13:29 +0800 Subject: [PATCH 70/98] control commit id flush --- rpl/rpl.go | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/rpl/rpl.go b/rpl/rpl.go index 5e9eca8..b0453ef 100644 --- a/rpl/rpl.go +++ b/rpl/rpl.go @@ -24,8 +24,9 @@ type Replication struct { s LogStore - commitID uint64 - commitLog *os.File + commitID uint64 + commitLog *os.File + commitLastTime time.Time quit chan struct{} @@ -92,6 +93,10 @@ func (r *Replication) Close() error { r.s = nil } + if err := r.updateCommitID(r.commitID, true); err != nil { + log.Error("update commit id err %s", err.Error()) + } + if r.commitLog != nil { r.commitLog.Close() r.commitLog = nil @@ -185,7 +190,7 @@ func (r *Replication) UpdateCommitID(id uint64) error { r.m.Lock() defer r.m.Unlock() - return r.updateCommitID(id) + return r.updateCommitID(id, r.cfg.Replication.SyncLog == 2) } func (r *Replication) Stat() (*Stat, error) { @@ -207,17 +212,23 @@ func (r *Replication) Stat() (*Stat, error) { return s, nil } -func (r *Replication) updateCommitID(id uint64) error { - if _, err := r.commitLog.Seek(0, os.SEEK_SET); err != nil { - return err - } +func (r *Replication) updateCommitID(id uint64, force bool) error { + n := time.Now() - if err := binary.Write(r.commitLog, binary.BigEndian, id); err != nil { - return err + if force || n.Sub(r.commitLastTime) > time.Second { + if _, err := r.commitLog.Seek(0, os.SEEK_SET); err != nil { + return err + } + + if err := binary.Write(r.commitLog, binary.BigEndian, id); err != nil { + return err + } } r.commitID = id + r.commitLastTime = n + return nil } @@ -266,7 +277,7 @@ func (r *Replication) ClearWithCommitID(id uint64) error { return err } - return r.updateCommitID(id) + return r.updateCommitID(id, true) } func (r *Replication) onPurgeExpired() { From f47b9168288bf1611ad6bc4ae6158517c167fa75 Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 10 Nov 2014 15:31:36 +0800 Subject: [PATCH 71/98] write batch add Close --- store/driver/batch.go | 5 +++++ store/driver/driver.go | 1 + store/goleveldb/batch.go | 4 ++++ store/leveldb/batch.go | 10 +++++----- store/leveldb/db.go | 9 ++++++++- store/rocksdb/batch.go | 9 +++++---- store/rocksdb/db.go | 5 +++++ store/writebatch.go | 4 ++++ 8 files changed, 37 insertions(+), 10 deletions(-) diff --git a/store/driver/batch.go b/store/driver/batch.go index 5a47c1b..e793e76 100644 --- a/store/driver/batch.go +++ b/store/driver/batch.go @@ -21,6 +21,11 @@ type WriteBatch struct { w BatchPuter } +func (wb *WriteBatch) Close() { + wb.d.Reset() + wb.wb = wb.wb[0:0] +} + func (wb *WriteBatch) Put(key, value []byte) { if value == nil { value = []byte{} diff --git a/store/driver/driver.go b/store/driver/driver.go index 04133c7..b571738 100644 --- a/store/driver/driver.go +++ b/store/driver/driver.go @@ -59,6 +59,7 @@ type IWriteBatch interface { SyncCommit() error Rollback() error Data() []byte + Close() } type Tx interface { diff --git a/store/goleveldb/batch.go b/store/goleveldb/batch.go index aef0f55..00485e1 100644 --- a/store/goleveldb/batch.go +++ b/store/goleveldb/batch.go @@ -30,6 +30,10 @@ func (w *WriteBatch) Rollback() error { return nil } +func (w *WriteBatch) Close() { + w.wbatch.Reset() +} + func (w *WriteBatch) Data() []byte { return w.wbatch.Dump() } diff --git a/store/leveldb/batch.go b/store/leveldb/batch.go index 542a322..b588735 100644 --- a/store/leveldb/batch.go +++ b/store/leveldb/batch.go @@ -34,13 +34,13 @@ func newWriteBatch(db *DB) *WriteBatch { return w } -func (w *WriteBatch) Close() error { - C.leveldb_writebatch_destroy(w.wbatch) - w.wbatch = nil +func (w *WriteBatch) Close() { + if w.wbatch != nil { + C.leveldb_writebatch_destroy(w.wbatch) + w.wbatch = nil + } w.gbatch = nil - - return nil } func (w *WriteBatch) Put(key, value []byte) { diff --git a/store/leveldb/db.go b/store/leveldb/db.go index a2d9c26..64bbc2b 100644 --- a/store/leveldb/db.go +++ b/store/leveldb/db.go @@ -14,6 +14,7 @@ import ( "github.com/siddontang/ledisdb/config" "github.com/siddontang/ledisdb/store/driver" "os" + "runtime" "unsafe" ) @@ -182,7 +183,13 @@ func (db *DB) SyncDelete(key []byte) error { } func (db *DB) NewWriteBatch() driver.IWriteBatch { - return newWriteBatch(db) + wb := newWriteBatch(db) + + runtime.SetFinalizer(wb, func(w *WriteBatch) { + w.Close() + }) + + return wb } func (db *DB) NewIterator() driver.IIterator { diff --git a/store/rocksdb/batch.go b/store/rocksdb/batch.go index aa4e407..bb727e7 100644 --- a/store/rocksdb/batch.go +++ b/store/rocksdb/batch.go @@ -17,10 +17,11 @@ type WriteBatch struct { commitOk bool } -func (w *WriteBatch) Close() error { - C.rocksdb_writebatch_destroy(w.wbatch) - w.wbatch = nil - return nil +func (w *WriteBatch) Close() { + if w.wbatch != nil { + C.rocksdb_writebatch_destroy(w.wbatch) + w.wbatch = nil + } } func (w *WriteBatch) Put(key, value []byte) { diff --git a/store/rocksdb/db.go b/store/rocksdb/db.go index 60816d3..0cf8f96 100644 --- a/store/rocksdb/db.go +++ b/store/rocksdb/db.go @@ -15,6 +15,7 @@ import ( "github.com/siddontang/ledisdb/config" "github.com/siddontang/ledisdb/store/driver" "os" + "runtime" "unsafe" ) @@ -215,6 +216,10 @@ func (db *DB) NewWriteBatch() driver.IWriteBatch { wbatch: C.rocksdb_writebatch_create(), } + runtime.SetFinalizer(wb, func(w *WriteBatch) { + w.Close() + }) + return wb } diff --git a/store/writebatch.go b/store/writebatch.go index dc825c1..3e0d138 100644 --- a/store/writebatch.go +++ b/store/writebatch.go @@ -16,6 +16,10 @@ type WriteBatch struct { db *DB } +func (wb *WriteBatch) Close() { + wb.wb.Close() +} + func (wb *WriteBatch) Put(key []byte, value []byte) { wb.putNum++ wb.wb.Put(key, value) From 660998f1fdc65ee453809df429e441df01f9f860 Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 10 Nov 2014 15:35:57 +0800 Subject: [PATCH 72/98] using batch to optimize load dump --- ledis/dump.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/ledis/dump.go b/ledis/dump.go index 1f6d5e2..863255c 100644 --- a/ledis/dump.go +++ b/ledis/dump.go @@ -155,6 +155,11 @@ func (l *Ledis) LoadDump(r io.Reader) (*DumpHead, error) { var key, value []byte + wb := l.ldb.NewWriteBatch() + defer wb.Close() + + n := 0 + for { if err = binary.Read(rb, binary.BigEndian, &keyLen); err != nil && err != io.EOF { return nil, err @@ -182,14 +187,26 @@ func (l *Ledis) LoadDump(r io.Reader) (*DumpHead, error) { return nil, err } - if err = l.ldb.Put(key, value); err != nil { - return nil, err + wb.Put(key, value) + n++ + if n%1024 == 0 { + if err = wb.Commit(); err != nil { + return nil, err + } } + // if err = l.ldb.Put(key, value); err != nil { + // return nil, err + // } + keyBuf.Reset() valueBuf.Reset() } + if err = wb.Commit(); err != nil { + return nil, err + } + deKeyBuf = nil deValueBuf = nil From cd76c402d217078ee7a31b524401195429430e08 Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 10 Nov 2014 17:16:02 +0800 Subject: [PATCH 73/98] add default optimization --- store/rocksdb/options.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/store/rocksdb/options.go b/store/rocksdb/options.go index 0404cdb..2783679 100644 --- a/store/rocksdb/options.go +++ b/store/rocksdb/options.go @@ -57,6 +57,14 @@ func (o *Options) Close() { C.rocksdb_options_destroy(o.Opt) } +func (o *Options) IncreaseParallelism(n int) { + C.rocksdb_options_increase_parallelism(o.Opt, C.int(n)) +} + +func (o *Options) OptimizeLevelStyleCompaction(n int) { + C.rocksdb_options_optimize_level_style_compaction(o.Opt, C.uint64_t(n)) +} + func (o *Options) SetComparator(cmp *C.rocksdb_comparator_t) { C.rocksdb_options_set_comparator(o.Opt, cmp) } From f39f5cdef6ae1c789c1ff091b6e14751c1596ada Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 10 Nov 2014 17:22:18 +0800 Subject: [PATCH 74/98] adjust default rocksdb config --- config/config.toml | 12 ++++++------ etc/ledis.conf | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/config/config.toml b/config/config.toml index f121543..9f1f5f7 100644 --- a/config/config.toml +++ b/config/config.toml @@ -72,11 +72,11 @@ max_open_files = 1024 # 0:no, 1:snappy, 2:zlib, 3:bz2, 4:lz4, 5:lz4hc compression = 0 block_size = 65536 -write_buffer_size = 67108864 -cache_size = 524288000 +write_buffer_size = 134217728 +cache_size = 1073741824 max_open_files = 1024 -max_write_buffer_num = 2 -min_write_buffer_number_to_merge = 1 +max_write_buffer_num = 6 +min_write_buffer_number_to_merge = 2 num_levels = 7 level0_file_num_compaction_trigger = 8 level0_slowdown_writes_trigger = 16 @@ -88,9 +88,9 @@ max_bytes_for_level_multiplier = 8 disable_auto_compactions = false disable_data_sync = false use_fsync = false -background_theads = 4 +background_theads = 16 high_priority_background_threads = 1 -max_background_compactions = 3 +max_background_compactions = 15 max_background_flushes = 1 allow_os_buffer = true enable_statistics = false diff --git a/etc/ledis.conf b/etc/ledis.conf index f121543..9f1f5f7 100644 --- a/etc/ledis.conf +++ b/etc/ledis.conf @@ -72,11 +72,11 @@ max_open_files = 1024 # 0:no, 1:snappy, 2:zlib, 3:bz2, 4:lz4, 5:lz4hc compression = 0 block_size = 65536 -write_buffer_size = 67108864 -cache_size = 524288000 +write_buffer_size = 134217728 +cache_size = 1073741824 max_open_files = 1024 -max_write_buffer_num = 2 -min_write_buffer_number_to_merge = 1 +max_write_buffer_num = 6 +min_write_buffer_number_to_merge = 2 num_levels = 7 level0_file_num_compaction_trigger = 8 level0_slowdown_writes_trigger = 16 @@ -88,9 +88,9 @@ max_bytes_for_level_multiplier = 8 disable_auto_compactions = false disable_data_sync = false use_fsync = false -background_theads = 4 +background_theads = 16 high_priority_background_threads = 1 -max_background_compactions = 3 +max_background_compactions = 15 max_background_flushes = 1 allow_os_buffer = true enable_statistics = false From 8fb200819c0f0f27acfc301f9ab7db22775402f2 Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 10 Nov 2014 21:43:25 +0800 Subject: [PATCH 75/98] update leveldb batch data --- store/leveldb/batch.go | 41 +++++++++++++++++++----------------- store/leveldb/leveldb_ext.cc | 7 ++++++ store/leveldb/leveldb_ext.h | 1 + 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/store/leveldb/batch.go b/store/leveldb/batch.go index b588735..4e2f1c1 100644 --- a/store/leveldb/batch.go +++ b/store/leveldb/batch.go @@ -4,6 +4,7 @@ package leveldb // #cgo LDFLAGS: -lleveldb // #include "leveldb/c.h" +// #include "leveldb_ext.h" import "C" import ( @@ -11,13 +12,6 @@ import ( "unsafe" ) -/* - It's not easy to let leveldb support Data() function like rocksdb, - so here, we will use goleveldb batch instead - - later optimize -*/ - type WriteBatch struct { db *DB wbatch *C.leveldb_writebatch_t @@ -44,8 +38,6 @@ func (w *WriteBatch) Close() { } func (w *WriteBatch) Put(key, value []byte) { - w.gbatch.Put(key, value) - var k, v *C.char if len(key) != 0 { k = (*C.char)(unsafe.Pointer(&key[0])) @@ -61,29 +53,21 @@ func (w *WriteBatch) Put(key, value []byte) { } func (w *WriteBatch) Delete(key []byte) { - w.gbatch.Delete(key) - C.leveldb_writebatch_delete(w.wbatch, (*C.char)(unsafe.Pointer(&key[0])), C.size_t(len(key))) } func (w *WriteBatch) Commit() error { - w.gbatch.Reset() - return w.commit(w.db.writeOpts) } func (w *WriteBatch) SyncCommit() error { - w.gbatch.Reset() - return w.commit(w.db.syncOpts) } func (w *WriteBatch) Rollback() error { C.leveldb_writebatch_clear(w.wbatch) - w.gbatch.Reset() - return nil } @@ -96,6 +80,25 @@ func (w *WriteBatch) commit(wb *WriteOptions) error { return nil } -func (w *WriteBatch) Data() []byte { - return w.gbatch.Dump() +//export leveldb_writebatch_iterate_put +func leveldb_writebatch_iterate_put(p unsafe.Pointer, k *C.char, klen C.size_t, v *C.char, vlen C.size_t) { + b := (*leveldb.Batch)(p) + key := slice(unsafe.Pointer(k), int(klen)) + value := slice(unsafe.Pointer(v), int(vlen)) + b.Put(key, value) +} + +//export leveldb_writebatch_iterate_delete +func leveldb_writebatch_iterate_delete(p unsafe.Pointer, k *C.char, klen C.size_t) { + b := (*leveldb.Batch)(p) + key := slice(unsafe.Pointer(k), int(klen)) + b.Delete(key) +} + +func (w *WriteBatch) Data() []byte { + w.gbatch.Reset() + C.leveldb_writebatch_iterate_ext(w.wbatch, + unsafe.Pointer(w.gbatch)) + b := w.gbatch.Dump() + return b } diff --git a/store/leveldb/leveldb_ext.cc b/store/leveldb/leveldb_ext.cc index a362ab5..540b739 100644 --- a/store/leveldb/leveldb_ext.cc +++ b/store/leveldb/leveldb_ext.cc @@ -84,5 +84,12 @@ unsigned char leveldb_iter_prev_ext(leveldb_iterator_t* iter) { return leveldb_iter_valid(iter); } +extern void leveldb_writebatch_iterate_put(void*, const char* k, size_t klen, const char* v, size_t vlen); +extern void leveldb_writebatch_iterate_delete(void*, const char* k, size_t klen); + +void leveldb_writebatch_iterate_ext(leveldb_writebatch_t* w, void *p) { + leveldb_writebatch_iterate(w, p, + leveldb_writebatch_iterate_put, leveldb_writebatch_iterate_delete); +} } \ No newline at end of file diff --git a/store/leveldb/leveldb_ext.h b/store/leveldb/leveldb_ext.h index 1c5f986..3eed41b 100644 --- a/store/leveldb/leveldb_ext.h +++ b/store/leveldb/leveldb_ext.h @@ -32,6 +32,7 @@ extern unsigned char leveldb_iter_seek_ext(leveldb_iterator_t*, const char* k, s extern unsigned char leveldb_iter_next_ext(leveldb_iterator_t*); extern unsigned char leveldb_iter_prev_ext(leveldb_iterator_t*); +extern void leveldb_writebatch_iterate_ext(leveldb_writebatch_t*, void* p); #ifdef __cplusplus } From 529c158293ac88b514063599b316f504be1d189c Mon Sep 17 00:00:00 2001 From: siddontang Date: Tue, 11 Nov 2014 11:09:32 +0800 Subject: [PATCH 76/98] update signal notify --- cmd/ledis-server/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/ledis-server/main.go b/cmd/ledis-server/main.go index 70328c4..31e056f 100644 --- a/cmd/ledis-server/main.go +++ b/cmd/ledis-server/main.go @@ -81,6 +81,8 @@ func main() { sc := make(chan os.Signal, 1) signal.Notify(sc, + os.Kill, + os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, From c6576990efe6b583c3626579e30d11450067ad58 Mon Sep 17 00:00:00 2001 From: siddontang Date: Tue, 11 Nov 2014 15:20:26 +0800 Subject: [PATCH 77/98] rpl support sync every 1 second --- rpl/file_store.go | 8 +++++++- rpl/file_table.go | 30 +++++++++++++++++++++++++-- rpl/goleveldb_store.go | 5 +++++ rpl/rpl.go | 46 ++++++++++++++++++++++++++++++------------ rpl/store.go | 2 ++ 5 files changed, 75 insertions(+), 16 deletions(-) diff --git a/rpl/file_store.go b/rpl/file_store.go index 3c5e719..63d287c 100644 --- a/rpl/file_store.go +++ b/rpl/file_store.go @@ -209,6 +209,10 @@ func (s *FileStore) PuregeExpired(n int64) error { return nil } +func (s *FileStore) Sync() error { + return s.w.Sync() +} + func (s *FileStore) Clear() error { s.wm.Lock() s.rm.Lock() @@ -244,7 +248,9 @@ func (s *FileStore) Close() error { s.rm.Lock() if r, err := s.w.Flush(); err != nil { - log.Error("close err: %s", err.Error()) + if err != errNilHandler { + log.Error("close err: %s", err.Error()) + } } else { r.Close() s.w.Close() diff --git a/rpl/file_table.go b/rpl/file_table.go index 9023aa3..bfff239 100644 --- a/rpl/file_table.go +++ b/rpl/file_table.go @@ -22,6 +22,7 @@ var ( log0 = Log{0, 1, 1, []byte("ledisdb")} log0Data = []byte{} errTableNeedFlush = errors.New("write table need flush") + errNilHandler = errors.New("nil write handler") pageSize = int64(4096) ) @@ -225,6 +226,13 @@ func (t *tableReader) repair() error { defer t.close() + st, _ := t.f.Stat() + size := st.Size() + + if size == 0 { + return fmt.Errorf("empty file, can not repaired") + } + tw := newTableWriter(path.Dir(t.name), t.index, maxLogFileSize) tmpName := tw.name + ".tmp" @@ -239,6 +247,14 @@ func (t *tableReader) repair() error { var l Log for { + lastPos, _ := t.f.Seek(0, os.SEEK_CUR) + if lastPos == size { + //no data anymore, we can not read log0 + //we may meet the log missing risk but have no way + log.Error("no more data, maybe missing some logs, use your own risk!!!") + break + } + if err := l.Decode(t.f); err != nil { return err } @@ -442,7 +458,7 @@ func (t *tableWriter) Flush() (*tableReader, error) { defer t.Unlock() if t.wf == nil { - return nil, fmt.Errorf("nil write handler") + return nil, errNilHandler } defer t.reset() @@ -564,7 +580,7 @@ func (t *tableWriter) StoreLog(l *Log) error { //todo add LRU cache - if t.syncType == 2 || (t.syncType == 1 && time.Now().Unix()-int64(t.lastTime) > 1) { + if t.syncType == 2 { if err := t.wf.Sync(); err != nil { log.Error("sync table error %s", err.Error()) } @@ -594,6 +610,16 @@ func (t *tableWriter) GetLog(id uint64, l *Log) error { return nil } +func (t *tableWriter) Sync() error { + t.Lock() + defer t.Unlock() + + if t.wf != nil { + return t.wf.Sync() + } + return nil +} + func (t *tableWriter) getLog(l *Log, pos int64) error { t.rm.Lock() defer t.rm.Unlock() diff --git a/rpl/goleveldb_store.go b/rpl/goleveldb_store.go index 2d3f808..5ece8d5 100644 --- a/rpl/goleveldb_store.go +++ b/rpl/goleveldb_store.go @@ -156,6 +156,11 @@ func (s *GoLevelDBStore) PurgeExpired(n int64) error { return nil } +func (s *GoLevelDBStore) Sync() error { + //no other way for sync, so ignore here + return nil +} + func (s *GoLevelDBStore) reset() { s.first = InvalidLogID s.last = InvalidLogID diff --git a/rpl/rpl.go b/rpl/rpl.go index b0453ef..e3bd54c 100644 --- a/rpl/rpl.go +++ b/rpl/rpl.go @@ -24,9 +24,8 @@ type Replication struct { s LogStore - commitID uint64 - commitLog *os.File - commitLastTime time.Time + commitID uint64 + commitLog *os.File quit chan struct{} @@ -75,7 +74,7 @@ func NewReplication(cfg *config.Config) (*Replication, error) { } r.wg.Add(1) - go r.onPurgeExpired() + go r.run() return r, nil } @@ -213,9 +212,7 @@ func (r *Replication) Stat() (*Stat, error) { } func (r *Replication) updateCommitID(id uint64, force bool) error { - n := time.Now() - - if force || n.Sub(r.commitLastTime) > time.Second { + if force { if _, err := r.commitLog.Seek(0, os.SEEK_SET); err != nil { return err } @@ -227,8 +224,6 @@ func (r *Replication) updateCommitID(id uint64, force bool) error { r.commitID = id - r.commitLastTime = n - return nil } @@ -280,19 +275,44 @@ func (r *Replication) ClearWithCommitID(id uint64) error { return r.updateCommitID(id, true) } -func (r *Replication) onPurgeExpired() { +func (r *Replication) run() { defer r.wg.Done() + syncTc := time.NewTicker(1 * time.Second) + purgeTc := time.NewTicker(1 * time.Hour) + for { select { - case <-time.After(1 * time.Hour): + case <-purgeTc.C: n := (r.cfg.Replication.ExpiredLogDays * 24 * 3600) r.m.Lock() - if err := r.s.PurgeExpired(int64(n)); err != nil { + err := r.s.PurgeExpired(int64(n)) + r.m.Unlock() + if err != nil { log.Error("purge expired log error %s", err.Error()) } - r.m.Unlock() + case <-syncTc.C: + if r.cfg.Replication.SyncLog == 1 { + r.m.Lock() + err := r.s.Sync() + r.m.Unlock() + if err != nil { + log.Error("sync store error %s", err.Error()) + } + } + if r.cfg.Replication.SyncLog != 2 { + //we will sync commit id every 1 second + r.m.Lock() + err := r.updateCommitID(r.commitID, true) + r.m.Unlock() + + if err != nil { + log.Error("sync commitid error %s", err.Error()) + } + } case <-r.quit: + syncTc.Stop() + purgeTc.Stop() return } } diff --git a/rpl/store.go b/rpl/store.go index d56d9f0..7af9b0a 100644 --- a/rpl/store.go +++ b/rpl/store.go @@ -26,6 +26,8 @@ type LogStore interface { // Delete logs before n seconds PurgeExpired(n int64) error + Sync() error + // Clear all logs Clear() error From 6f39cb17a9d71bce55aa3d3aac990d4ea193688e Mon Sep 17 00:00:00 2001 From: siddontang Date: Tue, 11 Nov 2014 15:21:41 +0800 Subject: [PATCH 78/98] close gracefully wait all resp-client connections closed --- cmd/ledis-server/main.go | 12 +++--- ledis/replication.go | 4 +- server/app.go | 12 ++++++ server/client_http.go | 5 ++- server/client_resp.go | 81 ++++++++++++++++++++++++++++----------- server/cmd_replication.go | 2 +- server/info.go | 11 +----- 7 files changed, 85 insertions(+), 42 deletions(-) diff --git a/cmd/ledis-server/main.go b/cmd/ledis-server/main.go index 31e056f..f1c695b 100644 --- a/cmd/ledis-server/main.go +++ b/cmd/ledis-server/main.go @@ -88,17 +88,15 @@ func main() { syscall.SIGTERM, syscall.SIGQUIT) - go func() { - <-sc - - app.Close() - }() - if *usePprof { go func() { log.Println(http.ListenAndServe(fmt.Sprintf(":%d", *pprofPort), nil)) }() } - app.Run() + go app.Run() + + <-sc + + app.Close() } diff --git a/ledis/replication.go b/ledis/replication.go index 593388d..4f02259 100644 --- a/ledis/replication.go +++ b/ledis/replication.go @@ -202,7 +202,7 @@ func (l *Ledis) ReadLogsTo(startLogID uint64, w io.Writer) (n int, nextLogID uin } // try to read events, if no events read, try to wait the new event singal until timeout seconds -func (l *Ledis) ReadLogsToTimeout(startLogID uint64, w io.Writer, timeout int) (n int, nextLogID uint64, err error) { +func (l *Ledis) ReadLogsToTimeout(startLogID uint64, w io.Writer, timeout int, quitCh chan struct{}) (n int, nextLogID uint64, err error) { n, nextLogID, err = l.ReadLogsTo(startLogID, w) if err != nil { return @@ -213,6 +213,8 @@ func (l *Ledis) ReadLogsToTimeout(startLogID uint64, w io.Writer, timeout int) ( select { case <-l.r.WaitLog(): case <-time.After(time.Duration(timeout) * time.Second): + case <-quitCh: + return } return l.ReadLogsTo(startLogID, w) } diff --git a/server/app.go b/server/app.go index a021865..393da09 100644 --- a/server/app.go +++ b/server/app.go @@ -37,6 +37,11 @@ type App struct { slaveSyncAck chan uint64 snap *snapshotStore + + connWait sync.WaitGroup + + rcm sync.Mutex + rcs map[*respClient]struct{} } func netType(s string) string { @@ -64,6 +69,8 @@ func NewApp(cfg *config.Config) (*App, error) { app.slaves = make(map[string]*client) app.slaveSyncAck = make(chan uint64) + app.rcs = make(map[*respClient]struct{}) + var err error if app.info, err = newInfo(app); err != nil { @@ -129,6 +136,11 @@ func (app *App) Close() { app.httpListener.Close() } + app.closeAllRespClients() + + //wait all connection closed + app.connWait.Wait() + app.closeScript() app.m.Close() diff --git a/server/client_http.go b/server/client_http.go index d039533..4383673 100644 --- a/server/client_http.go +++ b/server/client_http.go @@ -40,15 +40,18 @@ type httpWriter struct { } func newClientHTTP(app *App, w http.ResponseWriter, r *http.Request) { + app.connWait.Add(1) + defer app.connWait.Done() + var err error c := new(httpClient) - c.client = newClient(app) err = c.makeRequest(app, r, w) if err != nil { w.Write([]byte(err.Error())) return } + c.client = newClient(app) c.perform() c.client.close() } diff --git a/server/client_resp.go b/server/client_resp.go index b538f1a..65a83b4 100644 --- a/server/client_resp.go +++ b/server/client_resp.go @@ -16,6 +16,7 @@ import ( ) var errReadRequest = errors.New("invalid request protocol") +var errClientQuit = errors.New("remote client quit") type respClient struct { *client @@ -24,18 +25,51 @@ type respClient struct { rb *bufio.Reader ar *arena.Arena + + activeQuit bool } type respWriter struct { buff *bufio.Writer } +func (app *App) addRespClient(c *respClient) { + app.rcm.Lock() + app.rcs[c] = struct{}{} + app.rcm.Unlock() +} + +func (app *App) delRespClient(c *respClient) { + app.rcm.Lock() + delete(app.rcs, c) + app.rcm.Unlock() +} + +func (app *App) closeAllRespClients() { + app.rcm.Lock() + + for c := range app.rcs { + c.conn.Close() + } + + app.rcm.Unlock() +} + +func (app *App) respClientNum() int { + app.rcm.Lock() + n := len(app.rcs) + app.rcm.Unlock() + return n +} + func newClientRESP(conn net.Conn, app *App) { c := new(respClient) c.client = newClient(app) c.conn = conn + c.activeQuit = false + if tcpConn, ok := conn.(*net.TCPConn); ok { tcpConn.SetReadBuffer(app.cfg.ConnReadBufferSize) tcpConn.SetWriteBuffer(app.cfg.ConnWriteBufferSize) @@ -49,17 +83,15 @@ func newClientRESP(conn net.Conn, app *App) { //maybe another config? c.ar = arena.NewArena(app.cfg.ConnReadBufferSize) + app.connWait.Add(1) + + app.addRespClient(c) + go c.run() } func (c *respClient) run() { - c.app.info.addClients(1) - defer func() { - c.client.close() - - c.app.info.addClients(-1) - if e := recover(); e != nil { buf := make([]byte, 4096) n := runtime.Stack(buf, false) @@ -68,21 +100,30 @@ func (c *respClient) run() { log.Fatal("client run panic %s:%v", buf, e) } - handleQuit := true - if c.conn != nil { - //if handle quit command before, conn is nil - handleQuit = false - c.conn.Close() - } + c.client.close() + + c.conn.Close() if c.tx != nil { c.tx.Rollback() c.tx = nil } - c.app.removeSlave(c.client, handleQuit) + c.app.removeSlave(c.client, c.activeQuit) + + c.app.delRespClient(c) + + c.app.connWait.Done() }() + select { + case <-c.app.quit: + //check app closed + return + default: + break + } + kc := time.Duration(c.app.cfg.ConnKeepaliveInterval) * time.Second for { if kc > 0 { @@ -91,16 +132,12 @@ func (c *respClient) run() { reqData, err := c.readRequest() if err == nil { - c.handleRequest(reqData) + err = c.handleRequest(reqData) } if err != nil { return } - - if c.conn == nil { - return - } } } @@ -108,7 +145,7 @@ func (c *respClient) readRequest() ([][]byte, error) { return ReadRequest(c.rb, c.ar) } -func (c *respClient) handleRequest(reqData [][]byte) { +func (c *respClient) handleRequest(reqData [][]byte) error { if len(reqData) == 0 { c.cmd = "" c.args = reqData[0:0] @@ -117,11 +154,11 @@ func (c *respClient) handleRequest(reqData [][]byte) { c.args = reqData[1:] } if c.cmd == "quit" { + c.activeQuit = true c.resp.writeStatus(OK) c.resp.flush() c.conn.Close() - c.conn = nil - return + return errClientQuit } c.perform() @@ -131,7 +168,7 @@ func (c *respClient) handleRequest(reqData [][]byte) { c.ar.Reset() - return + return nil } // response writer diff --git a/server/cmd_replication.go b/server/cmd_replication.go index bc26968..b910e51 100644 --- a/server/cmd_replication.go +++ b/server/cmd_replication.go @@ -131,7 +131,7 @@ func syncCommand(c *client) error { c.syncBuf.Write(dummyBuf) - if _, _, err := c.app.ldb.ReadLogsToTimeout(logId, &c.syncBuf, 30); err != nil { + if _, _, err := c.app.ldb.ReadLogsToTimeout(logId, &c.syncBuf, 30, c.app.quit); err != nil { return err } else { buf := c.syncBuf.Bytes() diff --git a/server/info.go b/server/info.go index 0e807c1..b06b084 100644 --- a/server/info.go +++ b/server/info.go @@ -9,7 +9,6 @@ import ( "runtime/debug" "strings" "sync" - "sync/atomic" "time" ) @@ -23,10 +22,6 @@ type info struct { ProceessId int } - Clients struct { - ConnectedClients int64 - } - Replication struct { PubLogNum sync2.AtomicInt64 PubLogAckNum sync2.AtomicInt64 @@ -47,10 +42,6 @@ func newInfo(app *App) (i *info, err error) { return i, nil } -func (i *info) addClients(delta int64) { - atomic.AddInt64(&i.Clients.ConnectedClients, delta) -} - func (i *info) Close() { } @@ -116,7 +107,7 @@ func (i *info) dumpServer(buf *bytes.Buffer) { infoPair{"readonly", i.app.cfg.Readonly}, infoPair{"goroutine_num", runtime.NumGoroutine()}, infoPair{"cgo_call_num", runtime.NumCgoCall()}, - infoPair{"client_num", i.Clients.ConnectedClients}, + infoPair{"resp_client_num", i.app.respClientNum()}, ) } From 4e802b06e32047b812a4f0ed4b910cd60ebcd46d Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 12 Nov 2014 11:52:10 +0800 Subject: [PATCH 79/98] replication write table add lru cache --- rpl/file_table.go | 22 +++++++++- rpl/loglrucache.go | 95 +++++++++++++++++++++++++++++++++++++++++ rpl/loglrucache_test.go | 48 +++++++++++++++++++++ 3 files changed, 163 insertions(+), 2 deletions(-) create mode 100644 rpl/loglrucache.go create mode 100644 rpl/loglrucache_test.go diff --git a/rpl/file_table.go b/rpl/file_table.go index bfff239..3c07635 100644 --- a/rpl/file_table.go +++ b/rpl/file_table.go @@ -380,6 +380,8 @@ type tableWriter struct { syncType int lastTime uint32 + + cache *logLRUCache } func newTableWriter(base string, index int64, maxLogSize int64) *tableWriter { @@ -397,6 +399,9 @@ func newTableWriter(base string, index int64, maxLogSize int64) *tableWriter { t.closed = false + //maybe use config later + t.cache = newLogLRUCache(1024*1024, 1000) + return t } @@ -451,6 +456,7 @@ func (t *tableWriter) reset() { t.index = t.index + 1 t.name = path.Join(t.base, fmtTableName(t.index)) t.offsetBuf = t.offsetBuf[0:0] + t.cache.Reset() } func (t *tableWriter) Flush() (*tableReader, error) { @@ -565,8 +571,11 @@ func (t *tableWriter) StoreLog(l *Log) error { offsetPos := uint32(st.Size()) - if err := l.Encode(t.wf); err != nil { + buf, _ := l.Marshal() + if n, err := t.wf.Write(buf); err != nil { return err + } else if n != len(buf) { + return io.ErrShortWrite } t.offsetBuf = append(t.offsetBuf, num.Uint32ToBytes(offsetPos)...) @@ -578,7 +587,7 @@ func (t *tableWriter) StoreLog(l *Log) error { t.lastTime = l.CreateTime - //todo add LRU cache + t.cache.Set(l.ID, buf) if t.syncType == 2 { if err := t.wf.Sync(); err != nil { @@ -598,6 +607,13 @@ func (t *tableWriter) GetLog(id uint64, l *Log) error { } //todo memory cache + if cl := t.cache.Get(id); cl != nil { + if err := l.Unmarshal(cl); err == nil && l.ID == id { + return nil + } else { + t.cache.Delete(id) + } + } offset := binary.BigEndian.Uint32(t.offsetBuf[(id-t.first)*4:]) @@ -607,6 +623,8 @@ func (t *tableWriter) GetLog(id uint64, l *Log) error { return fmt.Errorf("invalid log id %d != %d", id, l.ID) } + //todo add cache here? + return nil } diff --git a/rpl/loglrucache.go b/rpl/loglrucache.go new file mode 100644 index 0000000..3dcbaf3 --- /dev/null +++ b/rpl/loglrucache.go @@ -0,0 +1,95 @@ +package rpl + +import ( + "container/list" + "encoding/binary" +) + +type logLRUCache struct { + itemsList *list.List + itemsMap map[uint64]*list.Element + size int + capability int + maxNum int +} + +func newLogLRUCache(capability int, maxNum int) *logLRUCache { + if capability <= 0 { + capability = 1024 * 1024 + } + + if maxNum <= 0 { + maxNum = 16 + } + + return &logLRUCache{ + itemsList: list.New(), + itemsMap: make(map[uint64]*list.Element), + size: 0, + capability: capability, + maxNum: maxNum, + } +} + +func (cache *logLRUCache) Set(id uint64, data []byte) { + elem, ok := cache.itemsMap[id] + if ok { + //we may not enter here + // item already exists, so move it to the front of the list and update the data + cache.itemsList.MoveToFront(elem) + ol := elem.Value.([]byte) + elem.Value = data + cache.size += (len(data) - len(ol)) + } else { + cache.size += len(data) + + // item doesn't exist, so add it to front of list + elem = cache.itemsList.PushFront(data) + cache.itemsMap[id] = elem + } + + // evict LRU entry if the cache is full + for cache.size > cache.capability || cache.itemsList.Len() > cache.maxNum { + removedElem := cache.itemsList.Back() + l := removedElem.Value.([]byte) + cache.itemsList.Remove(removedElem) + delete(cache.itemsMap, binary.BigEndian.Uint64(l[0:8])) + + cache.size -= len(l) + if cache.size <= 0 { + cache.size = 0 + } + } +} + +func (cache *logLRUCache) Get(id uint64) []byte { + elem, ok := cache.itemsMap[id] + if !ok { + return nil + } + + // item exists, so move it to front of list and return it + cache.itemsList.MoveToFront(elem) + l := elem.Value.([]byte) + return l +} + +func (cache *logLRUCache) Delete(id uint64) { + elem, ok := cache.itemsMap[id] + if !ok { + return + } + + cache.itemsList.Remove(elem) + delete(cache.itemsMap, id) +} + +func (cache *logLRUCache) Len() int { + return cache.itemsList.Len() +} + +func (cache *logLRUCache) Reset() { + cache.itemsList = list.New() + cache.itemsMap = make(map[uint64]*list.Element) + cache.size = 0 +} diff --git a/rpl/loglrucache_test.go b/rpl/loglrucache_test.go new file mode 100644 index 0000000..88a2923 --- /dev/null +++ b/rpl/loglrucache_test.go @@ -0,0 +1,48 @@ +package rpl + +import ( + "testing" +) + +func TestLogLRUCache(t *testing.T) { + c := newLogLRUCache(180, 10) + + var i uint64 + for i = 1; i <= 10; i++ { + l := &Log{i, 0, 0, []byte("0")} + b, _ := l.Marshal() + c.Set(l.ID, b) + } + + for i = 1; i <= 10; i++ { + if l := c.Get(i); l == nil { + t.Fatal("must exist", i) + } + } + + for i = 11; i <= 20; i++ { + l := &Log{i, 0, 0, []byte("0")} + b, _ := l.Marshal() + c.Set(l.ID, b) + } + + for i = 1; i <= 10; i++ { + if l := c.Get(i); l != nil { + t.Fatal("must not exist", i) + } + } + + c.Get(11) + + l := &Log{21, 0, 0, []byte("0")} + b, _ := l.Marshal() + c.Set(l.ID, b) + + if l := c.Get(12); l != nil { + t.Fatal("must nil", 12) + } + + if l := c.Get(11); l == nil { + t.Fatal("must not nil", 11) + } +} From 10a367f978c6c1eac7ee23267280755e26a4df46 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 13 Nov 2014 13:41:07 +0800 Subject: [PATCH 80/98] rpl file store check read table keepalived --- rpl/file_store.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/rpl/file_store.go b/rpl/file_store.go index 63d287c..a8f4940 100644 --- a/rpl/file_store.go +++ b/rpl/file_store.go @@ -58,11 +58,15 @@ type FileStore struct { rs tableReaders w *tableWriter + + quit chan struct{} } func NewFileStore(base string, maxSize int64, syncType int) (*FileStore, error) { s := new(FileStore) + s.quit = make(chan struct{}) + var err error if err = os.MkdirAll(base, 0755); err != nil { @@ -88,6 +92,8 @@ func NewFileStore(base string, maxSize int64, syncType int) (*FileStore, error) s.w = newTableWriter(s.base, index, s.maxFileSize) s.w.SetSyncType(syncType) + go s.checkTableReaders() + return s, nil } @@ -244,6 +250,8 @@ func (s *FileStore) Clear() error { } func (s *FileStore) Close() error { + close(s.quit) + s.wm.Lock() s.rm.Lock() @@ -267,6 +275,27 @@ func (s *FileStore) Close() error { return nil } +func (s *FileStore) checkTableReaders() { + t := time.NewTicker(60 * time.Second) + defer t.Stop() + for { + select { + case <-t.C: + s.rm.Lock() + + for _, r := range s.rs { + if !r.Keepalived() { + r.Close() + } + } + + s.rm.Unlock() + case <-s.quit: + return + } + } +} + func (s *FileStore) load() error { fs, err := ioutil.ReadDir(s.base) if err != nil { From 5196e211ccae8c792a13a1e97fadf8423b87edaf Mon Sep 17 00:00:00 2001 From: siddontang Date: Fri, 14 Nov 2014 16:00:03 +0800 Subject: [PATCH 81/98] adjust readonly check --- ledis/batch.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ledis/batch.go b/ledis/batch.go index 526b4b6..b0d04b9 100644 --- a/ledis/batch.go +++ b/ledis/batch.go @@ -18,7 +18,7 @@ type batch struct { } func (b *batch) Commit() error { - if b.l.IsReadOnly() { + if b.l.cfg.GetReadonly() { return ErrWriteInROnly } @@ -104,6 +104,10 @@ func (l *Ledis) handleCommit(g commitDataGetter, c commiter) error { var err error if l.r != nil { + if b, _ := l.r.CommitIDBehind(); b { + return ErrWriteInROnly + } + var rl *rpl.Log if rl, err = l.r.Log(g.Data()); err != nil { log.Fatal("write wal error %s", err.Error()) From c737ec82ffe914e20462969c6aee63176d03ba98 Mon Sep 17 00:00:00 2001 From: siddontang Date: Sat, 15 Nov 2014 20:59:40 +0800 Subject: [PATCH 82/98] rpl fix panic bug --- rpl/file_store.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rpl/file_store.go b/rpl/file_store.go index a8f4940..26a2c5c 100644 --- a/rpl/file_store.go +++ b/rpl/file_store.go @@ -187,7 +187,7 @@ func (s *FileStore) StoreLog(l *Log) error { return s.w.StoreLog(l) } -func (s *FileStore) PuregeExpired(n int64) error { +func (s *FileStore) PurgeExpired(n int64) error { s.rm.Lock() purges := []*tableReader{} @@ -302,6 +302,8 @@ func (s *FileStore) load() error { return err } + s.rs = make(tableReaders, 0, len(fs)) + var r *tableReader var index int64 for _, f := range fs { From 053cee05a38874f526db310aefbde3f3b1e5f12a Mon Sep 17 00:00:00 2001 From: siddontang Date: Sat, 15 Nov 2014 21:20:12 +0800 Subject: [PATCH 83/98] rpl add max log file num --- config/config.go | 3 +++ config/config.toml | 3 +++ etc/ledis.conf | 3 +++ rpl/file_store.go | 49 ++++++++++++++++++++++++++++++++-------------- rpl/rpl.go | 2 +- rpl/store_test.go | 7 +++++-- 6 files changed, 49 insertions(+), 18 deletions(-) diff --git a/config/config.go b/config/config.go index 1634fa0..f9fb6aa 100644 --- a/config/config.go +++ b/config/config.go @@ -76,6 +76,7 @@ type ReplicationConfig struct { ExpiredLogDays int `toml:"expired_log_days"` StoreName string `toml:"store_name"` MaxLogFileSize int64 `toml:"max_log_file_size"` + MaxLogFileNum int `toml:"max_log_file_num"` SyncLog int `toml:"sync_log"` Compression bool `toml:"compression"` } @@ -202,9 +203,11 @@ func (cfg *Config) adjust() { cfg.RocksDB.adjust() cfg.Replication.ExpiredLogDays = getDefault(7, cfg.Replication.ExpiredLogDays) + cfg.Replication.MaxLogFileNum = getDefault(10, cfg.Replication.MaxLogFileNum) cfg.ConnReadBufferSize = getDefault(4*KB, cfg.ConnReadBufferSize) cfg.ConnWriteBufferSize = getDefault(4*KB, cfg.ConnWriteBufferSize) cfg.TTLCheckInterval = getDefault(1, cfg.TTLCheckInterval) + } func (cfg *LevelDBConfig) adjust() { diff --git a/config/config.toml b/config/config.toml index 9f1f5f7..17b5d51 100644 --- a/config/config.toml +++ b/config/config.toml @@ -131,6 +131,9 @@ expired_log_days = 7 # for file store, if 0, use default 1G, max is 4G max_log_file_size = 0 +# for file store, if 0, use default 10 +max_log_file_num = 10 + # Sync log to disk if possible # 0: no sync # 1: sync every second diff --git a/etc/ledis.conf b/etc/ledis.conf index 9f1f5f7..17b5d51 100644 --- a/etc/ledis.conf +++ b/etc/ledis.conf @@ -131,6 +131,9 @@ expired_log_days = 7 # for file store, if 0, use default 1G, max is 4G max_log_file_size = 0 +# for file store, if 0, use default 10 +max_log_file_num = 10 + # Sync log to disk if possible # 0: no sync # 1: sync every second diff --git a/rpl/file_store.go b/rpl/file_store.go index 26a2c5c..823b1cd 100644 --- a/rpl/file_store.go +++ b/rpl/file_store.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/siddontang/go/log" "github.com/siddontang/go/num" + "github.com/siddontang/ledisdb/config" "io/ioutil" "os" "sort" @@ -49,7 +50,7 @@ const ( type FileStore struct { LogStore - maxFileSize int64 + cfg *config.Config base string @@ -62,7 +63,7 @@ type FileStore struct { quit chan struct{} } -func NewFileStore(base string, maxSize int64, syncType int) (*FileStore, error) { +func NewFileStore(base string, cfg *config.Config) (*FileStore, error) { s := new(FileStore) s.quit = make(chan struct{}) @@ -75,11 +76,14 @@ func NewFileStore(base string, maxSize int64, syncType int) (*FileStore, error) s.base = base - s.maxFileSize = num.MinInt64(maxLogFileSize, maxSize) - if s.maxFileSize == 0 { - s.maxFileSize = defaultMaxLogFileSize + if cfg.Replication.MaxLogFileSize == 0 { + cfg.Replication.MaxLogFileSize = defaultMaxLogFileSize } + cfg.Replication.MaxLogFileSize = num.MinInt64(cfg.Replication.MaxLogFileSize, maxLogFileSize) + + s.cfg = cfg + if err = s.load(); err != nil { return nil, err } @@ -89,8 +93,8 @@ func NewFileStore(base string, maxSize int64, syncType int) (*FileStore, error) index = s.rs[len(s.rs)-1].index + 1 } - s.w = newTableWriter(s.base, index, s.maxFileSize) - s.w.SetSyncType(syncType) + s.w = newTableWriter(s.base, index, cfg.Replication.MaxLogFileSize) + s.w.SetSyncType(cfg.Replication.SyncLog) go s.checkTableReaders() @@ -204,13 +208,7 @@ func (s *FileStore) PurgeExpired(n int64) error { s.rm.Unlock() - for _, r := range purges { - name := r.name - r.Close() - if err := os.Remove(name); err != nil { - log.Error("purge table %s err: %s", name, err.Error()) - } - } + s.purgeTableReaders(purges) return nil } @@ -244,7 +242,7 @@ func (s *FileStore) Clear() error { return err } - s.w = newTableWriter(s.base, 1, s.maxFileSize) + s.w = newTableWriter(s.base, 1, s.cfg.Replication.MaxLogFileSize) return nil } @@ -289,13 +287,34 @@ func (s *FileStore) checkTableReaders() { } } + purges := []*tableReader{} + maxNum := s.cfg.Replication.MaxLogFileNum + num := len(s.rs) + if num > maxNum { + purges = s.rs[:num-maxNum] + s.rs = s.rs[num-maxNum:] + } + s.rm.Unlock() + + s.purgeTableReaders(purges) + case <-s.quit: return } } } +func (s *FileStore) purgeTableReaders(purges []*tableReader) { + for _, r := range purges { + name := r.name + r.Close() + if err := os.Remove(name); err != nil { + log.Error("purge table %s err: %s", name, err.Error()) + } + } +} + func (s *FileStore) load() error { fs, err := ioutil.ReadDir(s.base) if err != nil { diff --git a/rpl/rpl.go b/rpl/rpl.go index e3bd54c..d942b49 100644 --- a/rpl/rpl.go +++ b/rpl/rpl.go @@ -58,7 +58,7 @@ func NewReplication(cfg *config.Config) (*Replication, error) { return nil, err } default: - if r.s, err = NewFileStore(path.Join(base, "ldb"), cfg.Replication.MaxLogFileSize, cfg.Replication.SyncLog); err != nil { + if r.s, err = NewFileStore(path.Join(base, "ldb"), cfg); err != nil { return nil, err } } diff --git a/rpl/store_test.go b/rpl/store_test.go index cdec659..9b8febe 100644 --- a/rpl/store_test.go +++ b/rpl/store_test.go @@ -1,6 +1,7 @@ package rpl import ( + "github.com/siddontang/ledisdb/config" "io/ioutil" "os" "testing" @@ -33,7 +34,10 @@ func TestFileStore(t *testing.T) { defer os.RemoveAll(dir) // New level - l, err := NewFileStore(dir, 4096, 0) + cfg := config.NewConfigDefault() + cfg.Replication.MaxLogFileSize = 4096 + + l, err := NewFileStore(dir, cfg) if err != nil { t.Fatalf("err: %v ", err) } @@ -51,7 +55,6 @@ func testLogs(t *testing.T, l LogStore) { if idx != 0 { t.Fatalf("bad idx: %d", idx) } - // Should be no last index idx, err = l.LastID() if err != nil { From 91ce8f5cf7786f3807a5e30ddb7f1e8f4fee2230 Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 17 Nov 2014 16:31:44 +0800 Subject: [PATCH 84/98] update bootstrap for no godep --- bootstrap.sh | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index 6a2a7a0..6b36cbf 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -10,19 +10,19 @@ if [ "$?" = 0 ]; then exit 0 fi -go get github.com/szferi/gomdb +go get -u github.com/szferi/gomdb -go get github.com/boltdb/bolt +go get -u github.com/boltdb/bolt -go get github.com/ugorji/go/codec -go get github.com/BurntSushi/toml -go get github.com/edsrzf/mmap-go +go get -u github.com/ugorji/go/codec +go get -u github.com/BurntSushi/toml +go get -u github.com/edsrzf/mmap-go -go get github.com/siddontang/goleveldb/leveldb -go get github.com/siddontang/go/bson -go get github.com/siddontang/go/log -go get github.com/siddontang/go/snappy -go get github.com/siddontang/go/num -go get github.com/siddontang/go/filelock -go get github.com/siddontang/go/sync2 -go get github.com/siddontang/go/arena +go get -u github.com/siddontang/goleveldb/leveldb +go get -u github.com/siddontang/go/bson +go get -u github.com/siddontang/go/log +go get -u github.com/siddontang/go/snappy +go get -u github.com/siddontang/go/num +go get -u github.com/siddontang/go/filelock +go get -u github.com/siddontang/go/sync2 +go get -u github.com/siddontang/go/arena From 8ab0b84cc2715cab2784f84a645f6cb93e37f4af Mon Sep 17 00:00:00 2001 From: siddontang Date: Tue, 18 Nov 2014 17:22:28 +0800 Subject: [PATCH 85/98] adjust rpl config --- config/config.toml | 2 +- etc/ledis.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.toml b/config/config.toml index 17b5d51..4e04cb1 100644 --- a/config/config.toml +++ b/config/config.toml @@ -141,7 +141,7 @@ max_log_file_num = 10 sync_log = 0 # Compress the log or not -compression = true +compression = false [snapshot] # Path to store snapshot dump file diff --git a/etc/ledis.conf b/etc/ledis.conf index 17b5d51..4e04cb1 100644 --- a/etc/ledis.conf +++ b/etc/ledis.conf @@ -141,7 +141,7 @@ max_log_file_num = 10 sync_log = 0 # Compress the log or not -compression = true +compression = false [snapshot] # Path to store snapshot dump file From b7cabcbb5f5ad4815145521daaada21bf00908d3 Mon Sep 17 00:00:00 2001 From: siddontang Date: Tue, 18 Nov 2014 17:50:22 +0800 Subject: [PATCH 86/98] rpl optimize --- ledis/batch.go | 4 ---- rpl/rpl.go | 2 ++ rpl/store.go | 7 ++++--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ledis/batch.go b/ledis/batch.go index b0d04b9..171a3ea 100644 --- a/ledis/batch.go +++ b/ledis/batch.go @@ -104,10 +104,6 @@ func (l *Ledis) handleCommit(g commitDataGetter, c commiter) error { var err error if l.r != nil { - if b, _ := l.r.CommitIDBehind(); b { - return ErrWriteInROnly - } - var rl *rpl.Log if rl, err = l.r.Log(g.Data()); err != nil { log.Fatal("write wal error %s", err.Error()) diff --git a/rpl/rpl.go b/rpl/rpl.go index d942b49..1e15b6f 100644 --- a/rpl/rpl.go +++ b/rpl/rpl.go @@ -124,6 +124,8 @@ func (r *Replication) Log(data []byte) (*Log, error) { commitId := r.commitID if lastID < commitId { lastID = commitId + } else if lastID > commitId { + return nil, ErrCommitIDBehind } l := new(Log) diff --git a/rpl/store.go b/rpl/store.go index 7af9b0a..9f985ec 100644 --- a/rpl/store.go +++ b/rpl/store.go @@ -9,9 +9,10 @@ const ( ) var ( - ErrLogNotFound = errors.New("log not found") - ErrStoreLogID = errors.New("log id is less") - ErrNoBehindLog = errors.New("no behind commit log") + ErrLogNotFound = errors.New("log not found") + ErrStoreLogID = errors.New("log id is less") + ErrNoBehindLog = errors.New("no behind commit log") + ErrCommitIDBehind = errors.New("commit id is behind last log id") ) type LogStore interface { From 9256808dc5e8ee4daa3de1d799f47a6368d0f206 Mon Sep 17 00:00:00 2001 From: siddontang Date: Tue, 18 Nov 2014 22:27:39 +0800 Subject: [PATCH 87/98] replication optimize --- rpl/file_table.go | 66 +++++++++++++++++++++++++++++++---------------- rpl/log.go | 31 ++++++++++------------ 2 files changed, 58 insertions(+), 39 deletions(-) diff --git a/rpl/file_table.go b/rpl/file_table.go index 3c07635..814475e 100644 --- a/rpl/file_table.go +++ b/rpl/file_table.go @@ -1,6 +1,7 @@ package rpl import ( + "bufio" "bytes" "encoding/binary" "errors" @@ -10,6 +11,7 @@ import ( "github.com/siddontang/go/num" "github.com/siddontang/go/sync2" "io" + "io/ioutil" "os" "path" "reflect" @@ -363,6 +365,8 @@ type tableWriter struct { wf *os.File rf *os.File + wb *bufio.Writer + rm sync.Mutex base string @@ -372,6 +376,7 @@ type tableWriter struct { first uint64 last uint64 + offsetPos int64 offsetBuf []byte maxLogSize int64 @@ -381,7 +386,7 @@ type tableWriter struct { syncType int lastTime uint32 - cache *logLRUCache + // cache *logLRUCache } func newTableWriter(base string, index int64, maxLogSize int64) *tableWriter { @@ -395,12 +400,16 @@ func newTableWriter(base string, index int64, maxLogSize int64) *tableWriter { t.name = path.Join(base, fmtTableName(index)) t.index = index + t.offsetPos = 0 t.maxLogSize = maxLogSize + //maybe config later? + t.wb = bufio.NewWriterSize(ioutil.Discard, 4096) + t.closed = false //maybe use config later - t.cache = newLogLRUCache(1024*1024, 1000) + // t.cache = newLogLRUCache(1024*1024, 1000) return t } @@ -423,6 +432,8 @@ func (t *tableWriter) close() { t.wf.Close() t.wf = nil } + + t.wb.Reset(ioutil.Discard) } func (t *tableWriter) Close() { @@ -456,7 +467,8 @@ func (t *tableWriter) reset() { t.index = t.index + 1 t.name = path.Join(t.base, fmtTableName(t.index)) t.offsetBuf = t.offsetBuf[0:0] - t.cache.Reset() + t.offsetPos = 0 + // t.cache.Reset() } func (t *tableWriter) Flush() (*tableReader, error) { @@ -558,27 +570,40 @@ func (t *tableWriter) StoreLog(l *Log) error { if t.wf, err = os.OpenFile(t.name, os.O_CREATE|os.O_WRONLY, 0644); err != nil { return err } + t.wb.Reset(t.wf) } if t.offsetBuf == nil { t.offsetBuf = make([]byte, 0, maxLogNumInFile*4) } - st, _ := t.wf.Stat() - if st.Size() >= t.maxLogSize { + // st, _ := t.wf.Stat() + // if st.Size() >= t.maxLogSize { + // return errTableNeedFlush + // } + + if t.offsetPos >= t.maxLogSize { return errTableNeedFlush } - offsetPos := uint32(st.Size()) + offsetPos := t.offsetPos - buf, _ := l.Marshal() - if n, err := t.wf.Write(buf); err != nil { + if err := l.Encode(t.wb); err != nil { + return err + } else if err = t.wb.Flush(); err != nil { return err - } else if n != len(buf) { - return io.ErrShortWrite } - t.offsetBuf = append(t.offsetBuf, num.Uint32ToBytes(offsetPos)...) + // buf, _ := l.Marshal() + // if n, err := t.wf.Write(buf); err != nil { + // return err + // } else if n != len(buf) { + // return io.ErrShortWrite + // } + + t.offsetPos += int64(l.Size()) + + t.offsetBuf = append(t.offsetBuf, num.Uint32ToBytes(uint32(offsetPos))...) if t.first == 0 { t.first = l.ID } @@ -587,7 +612,7 @@ func (t *tableWriter) StoreLog(l *Log) error { t.lastTime = l.CreateTime - t.cache.Set(l.ID, buf) + // t.cache.Set(l.ID, buf) if t.syncType == 2 { if err := t.wf.Sync(); err != nil { @@ -606,14 +631,13 @@ func (t *tableWriter) GetLog(id uint64, l *Log) error { return ErrLogNotFound } - //todo memory cache - if cl := t.cache.Get(id); cl != nil { - if err := l.Unmarshal(cl); err == nil && l.ID == id { - return nil - } else { - t.cache.Delete(id) - } - } + // if cl := t.cache.Get(id); cl != nil { + // if err := l.Unmarshal(cl); err == nil && l.ID == id { + // return nil + // } else { + // t.cache.Delete(id) + // } + // } offset := binary.BigEndian.Uint32(t.offsetBuf[(id-t.first)*4:]) @@ -623,8 +647,6 @@ func (t *tableWriter) GetLog(id uint64, l *Log) error { return fmt.Errorf("invalid log id %d != %d", id, l.ID) } - //todo add cache here? - return nil } diff --git a/rpl/log.go b/rpl/log.go index 382dff5..8500fbf 100644 --- a/rpl/log.go +++ b/rpl/log.go @@ -40,24 +40,21 @@ func (l *Log) Unmarshal(b []byte) error { } func (l *Log) Encode(w io.Writer) error { - buf := make([]byte, l.HeadSize()) - - pos := 0 - binary.BigEndian.PutUint64(buf[pos:], l.ID) - pos += 8 - - binary.BigEndian.PutUint32(buf[pos:], l.CreateTime) - pos += 4 - - buf[pos] = l.Compression - pos++ - - binary.BigEndian.PutUint32(buf[pos:], uint32(len(l.Data))) - - if n, err := w.Write(buf); err != nil { + if err := binary.Write(w, binary.BigEndian, l.ID); err != nil { + return err + } + + if err := binary.Write(w, binary.BigEndian, l.CreateTime); err != nil { + return err + } + + if _, err := w.Write([]byte{l.Compression}); err != nil { + return err + } + + dataLen := uint32(len(l.Data)) + if err := binary.Write(w, binary.BigEndian, dataLen); err != nil { return err - } else if n != len(buf) { - return io.ErrShortWrite } if n, err := w.Write(l.Data); err != nil { From 1ca74b404a0cca1ad574edec47785e72c170a9ab Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 19 Nov 2014 09:30:28 +0800 Subject: [PATCH 88/98] rpl optimize --- store/writebatch.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/store/writebatch.go b/store/writebatch.go index 3e0d138..a69dbd8 100644 --- a/store/writebatch.go +++ b/store/writebatch.go @@ -14,6 +14,8 @@ type WriteBatch struct { putNum int64 deleteNum int64 db *DB + + data *BatchData } func (wb *WriteBatch) Close() { @@ -60,13 +62,12 @@ func (wb *WriteBatch) Rollback() error { // the data will be undefined after commit or rollback func (wb *WriteBatch) BatchData() *BatchData { data := wb.wb.Data() - d, err := NewBatchData(data) - if err != nil { - //can not enter this - panic(err) + if wb.data == nil { + wb.data = new(BatchData) } - return d + wb.data.Load(data) + return wb.data } func (wb *WriteBatch) Data() []byte { From 8ec27302353059bac9c2f48de0d8eb9237031354 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 19 Nov 2014 10:26:50 +0800 Subject: [PATCH 89/98] rpl optimize log head use pool remove defer mutex --- README.md | 13 ++---------- config/config.toml | 2 +- etc/ledis.conf | 2 +- ledis/batch.go | 15 ++++++++++---- rpl/file_store.go | 19 +++++++++++------- rpl/file_table.go | 25 +++++++++++++++-------- rpl/log.go | 49 +++++++++++++++++++++++++++++----------------- rpl/rpl.go | 28 +++++++++++++++++--------- 8 files changed, 94 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 5a8436b..9e4ee0a 100644 --- a/README.md +++ b/README.md @@ -167,18 +167,9 @@ See [Clients](https://github.com/siddontang/ledisdb/wiki/Clients) to find or con + `pcall` and `xpcall` are not supported in lua, you can see the readme in [golua](https://github.com/aarzilli/golua). -## Thanks - -Gmail: cenqichao@gmail.com - -Gmail: chendahui007@gmail.com - -Gmail: cppgohan@gmail.com - -Gmail: tiaotiaoyly@gmail.com - -Gmail: wyk4true@gmail.com +## Requirement ++ go version >= 1.3 ## Feedback diff --git a/config/config.toml b/config/config.toml index 4e04cb1..98da90e 100644 --- a/config/config.toml +++ b/config/config.toml @@ -111,7 +111,7 @@ path = "" # If sync is true, the new log must be sent to some slaves, and then commit. # It will reduce performance but have better high availability. -sync = true +sync = false # If sync is true, wait at last wait_sync_time milliseconds for slave syncing this log wait_sync_time = 500 diff --git a/etc/ledis.conf b/etc/ledis.conf index 4e04cb1..98da90e 100644 --- a/etc/ledis.conf +++ b/etc/ledis.conf @@ -111,7 +111,7 @@ path = "" # If sync is true, the new log must be sent to some slaves, and then commit. # It will reduce performance but have better high availability. -sync = true +sync = false # If sync is true, wait at last wait_sync_time milliseconds for slave syncing this log wait_sync_time = 500 diff --git a/ledis/batch.go b/ledis/batch.go index 171a3ea..c9064df 100644 --- a/ledis/batch.go +++ b/ledis/batch.go @@ -100,12 +100,13 @@ type commitDataGetter interface { func (l *Ledis) handleCommit(g commitDataGetter, c commiter) error { l.commitLock.Lock() - defer l.commitLock.Unlock() var err error if l.r != nil { var rl *rpl.Log if rl, err = l.r.Log(g.Data()); err != nil { + l.commitLock.Unlock() + log.Fatal("write wal error %s", err.Error()) return err } @@ -113,19 +114,25 @@ func (l *Ledis) handleCommit(g commitDataGetter, c commiter) error { l.propagate(rl) if err = c.Commit(); err != nil { + l.commitLock.Unlock() + log.Fatal("commit error %s", err.Error()) l.noticeReplication() return err } if err = l.r.UpdateCommitID(rl.ID); err != nil { + l.commitLock.Unlock() + log.Fatal("update commit id error %s", err.Error()) l.noticeReplication() return err } - - return nil } else { - return c.Commit() + err = c.Commit() } + + l.commitLock.Unlock() + + return err } diff --git a/rpl/file_store.go b/rpl/file_store.go index 823b1cd..13d86c8 100644 --- a/rpl/file_store.go +++ b/rpl/file_store.go @@ -163,8 +163,12 @@ func (s *FileStore) LastID() (uint64, error) { func (s *FileStore) StoreLog(l *Log) error { s.wm.Lock() - defer s.wm.Unlock() + err := s.storeLog(l) + s.wm.Unlock() + return err +} +func (s *FileStore) storeLog(l *Log) error { err := s.w.StoreLog(l) if err == nil { return nil @@ -172,23 +176,24 @@ func (s *FileStore) StoreLog(l *Log) error { return err } - s.rm.Lock() - var r *tableReader - if r, err = s.w.Flush(); err != nil { + r, err = s.w.Flush() + + if err != nil { log.Error("write table flush error %s, can not store now", err.Error()) s.w.Close() - s.rm.Unlock() - return err } + s.rm.Lock() s.rs = append(s.rs, r) s.rm.Unlock() - return s.w.StoreLog(l) + err = s.w.StoreLog(l) + + return err } func (s *FileStore) PurgeExpired(n int64) error { diff --git a/rpl/file_table.go b/rpl/file_table.go index 814475e..7b37cd8 100644 --- a/rpl/file_table.go +++ b/rpl/file_table.go @@ -546,13 +546,18 @@ func (t *tableWriter) Flush() (*tableReader, error) { } func (t *tableWriter) StoreLog(l *Log) error { + t.Lock() + err := t.storeLog(l) + t.Unlock() + + return err +} + +func (t *tableWriter) storeLog(l *Log) error { if l.ID == 0 { return ErrStoreLogID } - t.Lock() - defer t.Unlock() - if t.closed { return fmt.Errorf("table writer is closed") } @@ -588,9 +593,11 @@ func (t *tableWriter) StoreLog(l *Log) error { offsetPos := t.offsetPos - if err := l.Encode(t.wb); err != nil { + if err = l.Encode(t.wb); err != nil { return err - } else if err = t.wb.Flush(); err != nil { + } + + if err = t.wb.Flush(); err != nil { return err } @@ -652,12 +659,14 @@ func (t *tableWriter) GetLog(id uint64, l *Log) error { func (t *tableWriter) Sync() error { t.Lock() - defer t.Unlock() + var err error if t.wf != nil { - return t.wf.Sync() + err = t.wf.Sync() } - return nil + t.Unlock() + + return err } func (t *tableWriter) getLog(l *Log, pos int64) error { diff --git a/rpl/log.go b/rpl/log.go index 8500fbf..0b10fd4 100644 --- a/rpl/log.go +++ b/rpl/log.go @@ -4,8 +4,11 @@ import ( "bytes" "encoding/binary" "io" + "sync" ) +const LogHeadSize = 17 + type Log struct { ID uint64 CreateTime uint32 @@ -15,7 +18,7 @@ type Log struct { } func (l *Log) HeadSize() int { - return 17 + return LogHeadSize } func (l *Log) Size() int { @@ -23,7 +26,7 @@ func (l *Log) Size() int { } func (l *Log) Marshal() ([]byte, error) { - buf := bytes.NewBuffer(make([]byte, l.HeadSize()+len(l.Data))) + buf := bytes.NewBuffer(make([]byte, l.Size())) buf.Reset() if err := l.Encode(buf); err != nil { @@ -39,25 +42,32 @@ func (l *Log) Unmarshal(b []byte) error { return l.Decode(buf) } +var headPool = sync.Pool{ + New: func() interface{} { return make([]byte, LogHeadSize) }, +} + func (l *Log) Encode(w io.Writer) error { - if err := binary.Write(w, binary.BigEndian, l.ID); err != nil { + b := headPool.Get().([]byte) + pos := 0 + + binary.BigEndian.PutUint64(b[pos:], l.ID) + pos += 8 + binary.BigEndian.PutUint32(b[pos:], uint32(l.CreateTime)) + pos += 4 + b[pos] = l.Compression + pos++ + binary.BigEndian.PutUint32(b[pos:], uint32(len(l.Data))) + + n, err := w.Write(b) + headPool.Put(b) + + if err != nil { return err + } else if n != LogHeadSize { + return io.ErrShortWrite } - if err := binary.Write(w, binary.BigEndian, l.CreateTime); err != nil { - return err - } - - if _, err := w.Write([]byte{l.Compression}); err != nil { - return err - } - - dataLen := uint32(len(l.Data)) - if err := binary.Write(w, binary.BigEndian, dataLen); err != nil { - return err - } - - if n, err := w.Write(l.Data); err != nil { + if n, err = w.Write(l.Data); err != nil { return err } else if n != len(l.Data) { return io.ErrShortWrite @@ -86,9 +96,10 @@ func (l *Log) Decode(r io.Reader) error { } func (l *Log) DecodeHead(r io.Reader) (uint32, error) { - buf := make([]byte, l.HeadSize()) + buf := headPool.Get().([]byte) if _, err := io.ReadFull(r, buf); err != nil { + headPool.Put(buf) return 0, err } @@ -104,5 +115,7 @@ func (l *Log) DecodeHead(r io.Reader) (uint32, error) { length := binary.BigEndian.Uint32(buf[pos:]) + headPool.Put(buf) + return length, nil } diff --git a/rpl/rpl.go b/rpl/rpl.go index 1e15b6f..d232992 100644 --- a/rpl/rpl.go +++ b/rpl/rpl.go @@ -114,10 +114,10 @@ func (r *Replication) Log(data []byte) (*Log, error) { } r.m.Lock() - defer r.m.Unlock() lastID, err := r.s.LastID() if err != nil { + r.m.Unlock() return nil, err } @@ -125,6 +125,7 @@ func (r *Replication) Log(data []byte) (*Log, error) { if lastID < commitId { lastID = commitId } else if lastID > commitId { + r.m.Unlock() return nil, ErrCommitIDBehind } @@ -141,9 +142,12 @@ func (r *Replication) Log(data []byte) (*Log, error) { l.Data = data if err = r.s.StoreLog(l); err != nil { + r.m.Unlock() return nil, err } + r.m.Unlock() + r.ncm.Lock() close(r.nc) r.nc = make(chan struct{}) @@ -161,22 +165,24 @@ func (r *Replication) WaitLog() <-chan struct{} { func (r *Replication) StoreLog(log *Log) error { r.m.Lock() - defer r.m.Unlock() + err := r.s.StoreLog(log) + r.m.Unlock() - return r.s.StoreLog(log) + return err } func (r *Replication) FirstLogID() (uint64, error) { r.m.Lock() - defer r.m.Unlock() id, err := r.s.FirstID() + r.m.Unlock() + return id, err } func (r *Replication) LastLogID() (uint64, error) { r.m.Lock() - defer r.m.Unlock() id, err := r.s.LastID() + r.m.Unlock() return id, err } @@ -189,9 +195,10 @@ func (r *Replication) LastCommitID() (uint64, error) { func (r *Replication) UpdateCommitID(id uint64) error { r.m.Lock() - defer r.m.Unlock() + err := r.updateCommitID(id, r.cfg.Replication.SyncLog == 2) + r.m.Unlock() - return r.updateCommitID(id, r.cfg.Replication.SyncLog == 2) + return err } func (r *Replication) Stat() (*Stat, error) { @@ -231,14 +238,17 @@ func (r *Replication) updateCommitID(id uint64, force bool) error { func (r *Replication) CommitIDBehind() (bool, error) { r.m.Lock() - defer r.m.Unlock() id, err := r.s.LastID() if err != nil { + r.m.Unlock() return false, err } - return id > r.commitID, nil + behind := id > r.commitID + r.m.Unlock() + + return behind, nil } func (r *Replication) GetLog(id uint64, log *Log) error { From 62440fef917423cf4cbed5988fd4e64551c8dc87 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 20 Nov 2014 17:33:38 +0800 Subject: [PATCH 90/98] refactor replication, not compatibility use data and meta file for a table use mmap for file read and write --- config/config.go | 2 +- config/config.toml | 6 +- etc/ledis.conf | 6 +- ledis/replication_test.go | 2 + ledis/tx_test.go | 2 +- rpl/file_io.go | 383 ++++++++++++++++++++++++++ rpl/file_store.go | 41 +-- rpl/file_table.go | 563 +++++++++++++++----------------------- rpl/file_table_test.go | 39 ++- rpl/log.go | 66 ++++- rpl/loglrucache.go | 95 ------- rpl/loglrucache_test.go | 48 ---- 12 files changed, 708 insertions(+), 545 deletions(-) create mode 100644 rpl/file_io.go delete mode 100644 rpl/loglrucache.go delete mode 100644 rpl/loglrucache_test.go diff --git a/config/config.go b/config/config.go index f9fb6aa..44887fa 100644 --- a/config/config.go +++ b/config/config.go @@ -203,7 +203,7 @@ func (cfg *Config) adjust() { cfg.RocksDB.adjust() cfg.Replication.ExpiredLogDays = getDefault(7, cfg.Replication.ExpiredLogDays) - cfg.Replication.MaxLogFileNum = getDefault(10, cfg.Replication.MaxLogFileNum) + cfg.Replication.MaxLogFileNum = getDefault(50, cfg.Replication.MaxLogFileNum) cfg.ConnReadBufferSize = getDefault(4*KB, cfg.ConnReadBufferSize) cfg.ConnWriteBufferSize = getDefault(4*KB, cfg.ConnWriteBufferSize) cfg.TTLCheckInterval = getDefault(1, cfg.TTLCheckInterval) diff --git a/config/config.toml b/config/config.toml index 98da90e..4177699 100644 --- a/config/config.toml +++ b/config/config.toml @@ -128,11 +128,11 @@ store_name = "file" # Expire write ahead logs after the given days expired_log_days = 7 -# for file store, if 0, use default 1G, max is 4G +# for file store, if 0, use default 256MB, max is 1G max_log_file_size = 0 -# for file store, if 0, use default 10 -max_log_file_num = 10 +# for file store, if 0, use default 50 +max_log_file_num = 0 # Sync log to disk if possible # 0: no sync diff --git a/etc/ledis.conf b/etc/ledis.conf index 98da90e..4177699 100644 --- a/etc/ledis.conf +++ b/etc/ledis.conf @@ -128,11 +128,11 @@ store_name = "file" # Expire write ahead logs after the given days expired_log_days = 7 -# for file store, if 0, use default 1G, max is 4G +# for file store, if 0, use default 256MB, max is 1G max_log_file_size = 0 -# for file store, if 0, use default 10 -max_log_file_num = 10 +# for file store, if 0, use default 50 +max_log_file_num = 0 # Sync log to disk if possible # 0: no sync diff --git a/ledis/replication_test.go b/ledis/replication_test.go index fc5e210..287480b 100644 --- a/ledis/replication_test.go +++ b/ledis/replication_test.go @@ -42,6 +42,7 @@ func TestReplication(t *testing.T) { if err != nil { t.Fatal(err) } + defer master.Close() cfgS := config.NewConfigDefault() cfgS.DataDir = "/tmp/test_repl/slave" @@ -54,6 +55,7 @@ func TestReplication(t *testing.T) { if err != nil { t.Fatal(err) } + defer slave.Close() db, _ := master.Select(0) db.Set([]byte("a"), []byte("value")) diff --git a/ledis/tx_test.go b/ledis/tx_test.go index e21c0a8..26888b5 100644 --- a/ledis/tx_test.go +++ b/ledis/tx_test.go @@ -195,7 +195,7 @@ func testTx(t *testing.T, name string) { cfg.DBName = name cfg.LMDB.MapSize = 10 * 1024 * 1024 - cfg.UseReplication = true + //cfg.UseReplication = true os.RemoveAll(cfg.DataDir) diff --git a/rpl/file_io.go b/rpl/file_io.go new file mode 100644 index 0000000..2e0023c --- /dev/null +++ b/rpl/file_io.go @@ -0,0 +1,383 @@ +package rpl + +import ( + "fmt" + "github.com/edsrzf/mmap-go" + "github.com/siddontang/go/log" + "io" + "os" +) + +//like leveldb or rocksdb file interface, haha! + +type writeFile interface { + Sync() error + Write(b []byte) (n int, err error) + Close(addMagic bool) error + ReadAt(buf []byte, offset int64) (int, error) + Truncate(size int64) error + SetOffset(o int64) + Name() string + Size() int + Offset() int64 +} + +type readFile interface { + ReadAt(buf []byte, offset int64) (int, error) + Close() error + Size() int + Name() string +} + +type rawWriteFile struct { + writeFile + f *os.File + offset int64 + name string +} + +func newRawWriteFile(name string, size int64) (writeFile, error) { + m := new(rawWriteFile) + var err error + + m.name = name + + m.f, err = os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0644) + if err != nil { + return nil, err + } + + return m, nil +} + +func (m *rawWriteFile) Close(addMagic bool) error { + if addMagic { + if err := m.f.Truncate(m.offset + int64(len(magic))); err != nil { + return fmt.Errorf("close truncate %s error %s", m.name, err.Error()) + } + + if _, err := m.f.WriteAt(magic, m.offset); err != nil { + return fmt.Errorf("close write %s magic error %s", m.name, err.Error()) + } + } else { + if err := m.f.Truncate(m.offset); err != nil { + return fmt.Errorf("close truncate %s error %s", m.name, err.Error()) + } + } + + if err := m.f.Close(); err != nil { + return fmt.Errorf("close %s error %s", m.name, err.Error()) + } + + return nil +} + +func (m *rawWriteFile) Sync() error { + return m.f.Sync() +} + +func (m *rawWriteFile) Write(b []byte) (n int, err error) { + n, err = m.f.Write(b) + if err != nil { + return + } else if n != len(b) { + err = io.ErrShortWrite + return + } + + m.offset += int64(n) + return +} + +func (m *rawWriteFile) ReadAt(buf []byte, offset int64) (int, error) { + return m.f.ReadAt(buf, offset) +} + +func (m *rawWriteFile) Truncate(size int64) error { + var err error + if err = m.f.Truncate(size); err != nil { + return err + } + + if m.offset > size { + m.offset = size + } + return nil +} + +func (m *rawWriteFile) SetOffset(o int64) { + m.offset = o +} + +func (m *rawWriteFile) Offset() int64 { + return m.offset +} + +func (m *rawWriteFile) Name() string { + return m.name +} + +func (m *rawWriteFile) Size() int { + st, _ := m.f.Stat() + return int(st.Size()) +} + +type rawReadFile struct { + readFile + + f *os.File + name string +} + +func newRawReadFile(name string) (readFile, error) { + m := new(rawReadFile) + + var err error + m.f, err = os.Open(name) + m.name = name + + if err != nil { + return nil, err + } + + return m, err +} + +func (m *rawReadFile) Close() error { + return m.f.Close() +} + +func (m *rawReadFile) Size() int { + st, _ := m.f.Stat() + return int(st.Size()) +} + +func (m *rawReadFile) ReadAt(b []byte, offset int64) (int, error) { + return m.f.ReadAt(b, offset) +} + +func (m *rawReadFile) Name() string { + return m.name +} + +///////////////////////////////////////////////// + +type mmapWriteFile struct { + writeFile + + f *os.File + m mmap.MMap + name string + size int64 + offset int64 +} + +func newMmapWriteFile(name string, size int64) (writeFile, error) { + m := new(mmapWriteFile) + + m.name = name + + var err error + + m.f, err = os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0644) + if err != nil { + return nil, err + } + + if size == 0 { + st, _ := m.f.Stat() + size = st.Size() + } + + if err = m.f.Truncate(size); err != nil { + return nil, err + } + + if m.m, err = mmap.Map(m.f, mmap.RDWR, 0); err != nil { + return nil, err + } + + m.size = size + m.offset = 0 + return m, nil +} + +func (m *mmapWriteFile) Size() int { + return int(m.size) +} + +func (m *mmapWriteFile) Sync() error { + return m.m.Flush() +} + +func (m *mmapWriteFile) Close(addMagic bool) error { + if err := m.m.Unmap(); err != nil { + return fmt.Errorf("unmap %s error %s", m.name, err.Error()) + } + + if addMagic { + if err := m.f.Truncate(m.offset + int64(len(magic))); err != nil { + return fmt.Errorf("close truncate %s error %s", m.name, err.Error()) + } + + if _, err := m.f.WriteAt(magic, m.offset); err != nil { + return fmt.Errorf("close write %s magic error %s", m.name, err.Error()) + } + } else { + if err := m.f.Truncate(m.offset); err != nil { + return fmt.Errorf("close truncate %s error %s", m.name, err.Error()) + } + } + + if err := m.f.Close(); err != nil { + return fmt.Errorf("close %s error %s", m.name, err.Error()) + } + + return nil +} + +func (m *mmapWriteFile) Write(b []byte) (n int, err error) { + extra := int64(len(b)) - (m.size - m.offset) + if extra > 0 { + newSize := m.size + extra + m.size/10 + println("need truncate ???", newSize, m.size, len(b)) + if err = m.Truncate(newSize); err != nil { + return + } + m.size = newSize + } + + n = copy(m.m[m.offset:], b) + if n != len(b) { + return 0, io.ErrShortWrite + } + + m.offset += int64(len(b)) + return len(b), nil +} + +func (m *mmapWriteFile) ReadAt(buf []byte, offset int64) (int, error) { + if offset > m.offset { + return 0, fmt.Errorf("invalid offset %d", offset) + } + + n := copy(buf, m.m[offset:m.offset]) + if n != len(buf) { + return n, io.ErrUnexpectedEOF + } + + return n, nil +} + +func (m *mmapWriteFile) Truncate(size int64) error { + var err error + if err = m.m.Unmap(); err != nil { + return err + } + + if err = m.f.Truncate(size); err != nil { + return err + } + + if m.m, err = mmap.Map(m.f, mmap.RDWR, 0); err != nil { + return err + } + + m.size = size + if m.offset > m.size { + m.offset = m.size + } + return nil +} + +func (m *mmapWriteFile) SetOffset(o int64) { + m.offset = o +} + +func (m *mmapWriteFile) Offset() int64 { + return m.offset +} + +func (m *mmapWriteFile) Name() string { + return m.name +} + +type mmapReadFile struct { + readFile + + f *os.File + m mmap.MMap + name string +} + +func newMmapReadFile(name string) (readFile, error) { + m := new(mmapReadFile) + + m.name = name + + var err error + m.f, err = os.Open(name) + if err != nil { + return nil, err + } + + m.m, err = mmap.Map(m.f, mmap.RDONLY, 0) + return m, err +} + +func (m *mmapReadFile) ReadAt(buf []byte, offset int64) (int, error) { + if int64(offset) > int64(len(m.m)) { + return 0, fmt.Errorf("invalid offset %d", offset) + } + + n := copy(buf, m.m[offset:]) + if n != len(buf) { + return n, io.ErrUnexpectedEOF + } + + return n, nil +} + +func (m *mmapReadFile) Close() error { + if m.m != nil { + if err := m.m.Unmap(); err != nil { + log.Error("unmap %s error %s", m.name, err.Error()) + } + m.m = nil + } + + if m.f != nil { + if err := m.f.Close(); err != nil { + log.Error("close %s error %s", m.name, err.Error()) + } + m.f = nil + } + + return nil +} + +func (m *mmapReadFile) Size() int { + return len(m.m) +} + +func (m *mmapReadFile) Name() string { + return m.name +} + +///////////////////////////////////// + +func newWriteFile(useMmap bool, name string, size int64) (writeFile, error) { + if useMmap { + return newMmapWriteFile(name, size) + } else { + return newRawWriteFile(name, size) + } +} + +func newReadFile(useMmap bool, name string) (readFile, error) { + if useMmap { + return newMmapReadFile(name) + } else { + return newRawReadFile(name) + } +} diff --git a/rpl/file_store.go b/rpl/file_store.go index 13d86c8..5cebdfa 100644 --- a/rpl/file_store.go +++ b/rpl/file_store.go @@ -13,33 +13,30 @@ import ( ) const ( - defaultMaxLogFileSize = int64(1024 * 1024 * 1024) + defaultMaxLogFileSize = int64(256 * 1024 * 1024) - //why 4G, we can use uint32 as offset, reduce memory useage - maxLogFileSize = int64(uint32(4*1024*1024*1024 - 1)) + maxLogFileSize = int64(1024 * 1024 * 1024) - maxLogNumInFile = uint64(10000000) + defaultLogNumInFile = int64(1024 * 1024) ) /* File Store: - 00000001.ldb - 00000002.ldb + 00000001.data + 00000001.meta + 00000002.data + 00000002.meta - log: log1 data | log2 data | split data | log1 offset | log 2 offset | offset start pos | offset length | magic data + data: log1 data | log2 data | magic data - log id can not be 0, we use here for split data if data has no magic data, it means that we don't close replication gracefully. so we must repair the log data log data: id (bigendian uint64), create time (bigendian uint32), compression (byte), data len(bigendian uint32), data split data = log0 data + [padding 0] -> file % pagesize() == 0 - log0: id 0, create time 0, compression 0, data len 7, data "ledisdb" + meta: log1 offset | log2 offset log offset: bigendian uint32 | bigendian uint32 - offset start pos: bigendian uint64 - offset length: bigendian uint32 - //sha1 of github.com/siddontang/ledisdb 20 bytes magic data = "\x1c\x1d\xb8\x88\xff\x9e\x45\x55\x40\xf0\x4c\xda\xe0\xce\x47\xde\x65\x48\x71\x17" @@ -270,6 +267,7 @@ func (s *FileStore) Close() error { for i := range s.rs { s.rs[i].Close() } + s.rs = tableReaders{} s.rm.Unlock() @@ -312,11 +310,16 @@ func (s *FileStore) checkTableReaders() { func (s *FileStore) purgeTableReaders(purges []*tableReader) { for _, r := range purges { - name := r.name + dataName := fmtTableDataName(r.base, r.index) + metaName := fmtTableMetaName(r.base, r.index) r.Close() - if err := os.Remove(name); err != nil { - log.Error("purge table %s err: %s", name, err.Error()) + if err := os.Remove(dataName); err != nil { + log.Error("purge table data %s err: %s", dataName, err.Error()) } + if err := os.Remove(metaName); err != nil { + log.Error("purge table meta %s err: %s", metaName, err.Error()) + } + } } @@ -331,7 +334,7 @@ func (s *FileStore) load() error { var r *tableReader var index int64 for _, f := range fs { - if _, err := fmt.Sscanf(f.Name(), "%08d.ldb", &index); err == nil { + if _, err := fmt.Sscanf(f.Name(), "%08d.data", &index); err == nil { if r, err = newTableReader(s.base, index); err != nil { log.Error("load table %s err: %s", f.Name(), err.Error()) } else { @@ -391,16 +394,16 @@ func (ts tableReaders) check() error { index := ts[0].index if first == 0 || first > last { - return fmt.Errorf("invalid log in table %s", ts[0].name) + return fmt.Errorf("invalid log in table %s", ts[0]) } for i := 1; i < len(ts); i++ { if ts[i].first <= last { - return fmt.Errorf("invalid first log id %d in table %s", ts[i].first, ts[i].name) + return fmt.Errorf("invalid first log id %d in table %s", ts[i].first, ts[i]) } if ts[i].index <= index { - return fmt.Errorf("invalid index %d in table %s", ts[i].index, ts[i].name) + return fmt.Errorf("invalid index %d in table %s", ts[i].index, ts[i]) } first = ts[i].first diff --git a/rpl/file_table.go b/rpl/file_table.go index 7b37cd8..de5f85d 100644 --- a/rpl/file_table.go +++ b/rpl/file_table.go @@ -1,64 +1,54 @@ package rpl import ( - "bufio" "bytes" "encoding/binary" "errors" "fmt" - "github.com/edsrzf/mmap-go" "github.com/siddontang/go/log" - "github.com/siddontang/go/num" "github.com/siddontang/go/sync2" "io" - "io/ioutil" - "os" "path" - "reflect" "sync" "time" ) var ( magic = []byte("\x1c\x1d\xb8\x88\xff\x9e\x45\x55\x40\xf0\x4c\xda\xe0\xce\x47\xde\x65\x48\x71\x17") - log0 = Log{0, 1, 1, []byte("ledisdb")} - log0Data = []byte{} errTableNeedFlush = errors.New("write table need flush") errNilHandler = errors.New("nil write handler") - pageSize = int64(4096) ) -func init() { - log0Data, _ = log0.Marshal() - pageSize = int64(os.Getpagesize()) -} - const tableReaderKeepaliveInterval int64 = 30 -func fmtTableName(index int64) string { - return fmt.Sprintf("%08d.ldb", index) +func fmtTableDataName(base string, index int64) string { + return path.Join(base, fmt.Sprintf("%08d.data", index)) } +func fmtTableMetaName(base string, index int64) string { + return path.Join(base, fmt.Sprintf("%08d.meta", index)) +} + +//todo config +var useMmap = true + type tableReader struct { sync.Mutex - name string + base string index int64 - f *os.File - m mmap.MMap - - pf *os.File + data readFile + meta readFile first uint64 last uint64 lastTime uint32 - offsetStartPos int64 - offsetLen uint32 - lastReadTime sync2.AtomicInt64 + + useMmap bool } func newTableReader(base string, index int64) (*tableReader, error) { @@ -66,16 +56,19 @@ func newTableReader(base string, index int64) (*tableReader, error) { return nil, fmt.Errorf("invalid index %d", index) } t := new(tableReader) - t.name = path.Join(base, fmtTableName(index)) + t.base = base t.index = index + //todo, use config + t.useMmap = useMmap + var err error if err = t.check(); err != nil { - log.Error("check %s error: %s, try to repair", t.name, err.Error()) + log.Error("check %d error: %s, try to repair", t.index, err.Error()) if err = t.repair(); err != nil { - log.Error("repair %s error: %s", t.name, err.Error()) + log.Error("repair %d error: %s", t.index, err.Error()) return nil, err } } @@ -85,22 +78,27 @@ func newTableReader(base string, index int64) (*tableReader, error) { return t, nil } +func (t *tableReader) String() string { + return fmt.Sprintf("%d", t.index) +} + func (t *tableReader) Close() { t.Lock() - defer t.Unlock() t.close() + + t.Unlock() } func (t *tableReader) close() { - if t.m != nil { - t.m.Unmap() - t.m = nil + if t.data != nil { + t.data.Close() + t.data = nil } - if t.f != nil { - t.f.Close() - t.f = nil + if t.meta != nil { + t.meta.Close() + t.meta = nil } } @@ -114,96 +112,78 @@ func (t *tableReader) Keepalived() bool { } func (t *tableReader) getLogPos(index int) (uint32, error) { - // if _, err := t.pf.Seek(t.offsetStartPos+int64(index*4), os.SEEK_SET); err != nil { - // return 0, err - // } + var buf [4]byte + if _, err := t.meta.ReadAt(buf[0:4], int64(index)*4); err != nil { + return 0, err + } - // var pos uint32 - // if err := binary.Read(t.pf, binary.BigEndian, &pos); err != nil { - // return 0, err - // } - // return pos, nil + return binary.BigEndian.Uint32(buf[0:4]), nil +} - return binary.BigEndian.Uint32(t.m[index*4:]), nil +func (t *tableReader) checkData() error { + var err error + if t.data, err = newReadFile(t.useMmap, fmtTableDataName(t.base, t.index)); err != nil { + return err + } + + if t.data.Size() < len(magic) { + return fmt.Errorf("data file %s size %d too short", t.data.Name(), t.data.Size()) + } + + buf := make([]byte, len(magic)) + if _, err := t.data.ReadAt(buf, int64(t.data.Size()-len(magic))); err != nil { + return err + } + + if !bytes.Equal(magic, buf) { + return fmt.Errorf("data file %s invalid magic data %q", t.data.Name(), buf) + } + + return nil +} + +func (t *tableReader) checkMeta() error { + var err error + if t.meta, err = newReadFile(t.useMmap, fmtTableMetaName(t.base, t.index)); err != nil { + return err + } + + if t.meta.Size()%4 != 0 || t.meta.Size() == 0 { + return fmt.Errorf("meta file %s invalid offset len %d, must 4 multiple and not 0", t.meta.Name(), t.meta.Size()) + } + + return nil } func (t *tableReader) check() error { var err error - if t.f, err = os.Open(t.name); err != nil { + if err := t.checkMeta(); err != nil { return err } - st, _ := t.f.Stat() - - if st.Size() < 32 { - return fmt.Errorf("file size %d too short", st.Size()) - } - - var pos int64 - if pos, err = t.f.Seek(-32, os.SEEK_END); err != nil { - return err - } - - if err = binary.Read(t.f, binary.BigEndian, &t.offsetStartPos); err != nil { - return err - } else if t.offsetStartPos >= st.Size() { - return fmt.Errorf("invalid offset start pos %d, file size %d", t.offsetStartPos, st.Size()) - } else if t.offsetStartPos%pageSize != 0 { - return fmt.Errorf("invalid offset start pos %d, must page size %d multi", t.offsetStartPos, pageSize) - } - - if err = binary.Read(t.f, binary.BigEndian, &t.offsetLen); err != nil { - return err - } else if int64(t.offsetLen) >= st.Size() || t.offsetLen == 0 { - return fmt.Errorf("invalid offset len %d, file size %d", t.offsetLen, st.Size()) - } else if t.offsetLen%4 != 0 { - return fmt.Errorf("invalid offset len %d, must 4 multiple", t.offsetLen) - } - - if t.offsetStartPos+int64(t.offsetLen) != pos { - return fmt.Errorf("invalid offset %d %d", t.offsetStartPos, t.offsetLen) - } - - b := make([]byte, 20) - if _, err = t.f.Read(b); err != nil { - return err - } else if !bytes.Equal(b, magic) { - return fmt.Errorf("invalid magic data %q", b) - } - - if t.m, err = mmap.MapRegion(t.f, int(t.offsetLen), mmap.RDONLY, 0, t.offsetStartPos); err != nil { + if err := t.checkData(); err != nil { return err } firstLogPos, _ := t.getLogPos(0) - lastLogPos, _ := t.getLogPos(int(t.offsetLen/4 - 1)) + lastLogPos, _ := t.getLogPos(t.meta.Size()/4 - 1) if firstLogPos != 0 { return fmt.Errorf("invalid first log pos %d, must 0", firstLogPos) - } else if int64(lastLogPos) > t.offsetStartPos { - return fmt.Errorf("invalid last log pos %d", lastLogPos) } var l Log - if _, err = t.decodeLogHead(&l, int64(firstLogPos)); err != nil { + if _, err = t.decodeLogHead(&l, t.data, int64(firstLogPos)); err != nil { return fmt.Errorf("decode first log err %s", err.Error()) } t.first = l.ID var n int64 - if n, err = t.decodeLogHead(&l, int64(lastLogPos)); err != nil { + if n, err = t.decodeLogHead(&l, t.data, int64(lastLogPos)); err != nil { return fmt.Errorf("decode last log err %s", err.Error()) - } else { - var l0 Log - if _, err := t.f.Seek(n, os.SEEK_SET); err != nil { - return fmt.Errorf("seek logo err %s", err.Error()) - } else if err = l0.Decode(t.f); err != nil { - println(lastLogPos, n, l0.ID, l0.CreateTime, l0.Compression) - return fmt.Errorf("decode log0 err %s", err.Error()) - } else if !reflect.DeepEqual(l0, log0) { - return fmt.Errorf("invalid log0 %#v != %#v", l0, log0) - } + } else if n+int64(len(magic)) != int64(t.data.Size()) { + return fmt.Errorf("extra log data at offset %d", n) } t.last = l.ID @@ -211,8 +191,8 @@ func (t *tableReader) check() error { if t.first > t.last { return fmt.Errorf("invalid log table first %d > last %d", t.first, t.last) - } else if (t.last - t.first + 1) != uint64(t.offsetLen/4) { - return fmt.Errorf("invalid log table, first %d, last %d, and log num %d", t.first, t.last, t.offsetLen/4) + } else if (t.last - t.first + 1) != uint64(t.meta.Size()/4) { + return fmt.Errorf("invalid log table, first %d, last %d, and log num %d", t.first, t.last, t.meta.Size()/4) } return nil @@ -222,86 +202,73 @@ func (t *tableReader) repair() error { t.close() var err error - if t.f, err = os.Open(t.name); err != nil { - return err - } + var data writeFile + var meta writeFile - defer t.close() + data, err = newWriteFile(t.useMmap, fmtTableDataName(t.base, t.index), 0) + data.SetOffset(int64(data.Size())) - st, _ := t.f.Stat() - size := st.Size() - - if size == 0 { - return fmt.Errorf("empty file, can not repaired") - } - - tw := newTableWriter(path.Dir(t.name), t.index, maxLogFileSize) - - tmpName := tw.name + ".tmp" - tw.name = tmpName - os.Remove(tmpName) - - defer func() { - tw.Close() - os.Remove(tmpName) - }() + meta, err = newWriteFile(t.useMmap, fmtTableMetaName(t.base, t.index), int64(defaultLogNumInFile*4)) var l Log + var pos int64 = 0 + var nextPos int64 = 0 + b := make([]byte, 4) + + t.first = 0 + t.last = 0 for { - lastPos, _ := t.f.Seek(0, os.SEEK_CUR) - if lastPos == size { - //no data anymore, we can not read log0 - //we may meet the log missing risk but have no way - log.Error("no more data, maybe missing some logs, use your own risk!!!") + nextPos, err = t.decodeLogHead(&l, data, pos) + if err != nil { + //if error, we may lost all logs from pos + log.Error("%s may lost logs from %d", data.Name(), pos) break } - if err := l.Decode(t.f); err != nil { - return err - } - if l.ID == 0 { + log.Error("%s may lost logs from %d, invalid log 0", data.Name(), pos) break } + if t.first == 0 { + t.first = l.ID + } + + if t.last == 0 { + t.last = l.ID + } else if l.ID <= t.last { + log.Error("%s may lost logs from %d, invalid logid %d", t.data.Name(), pos, l.ID) + break + } + + t.last = l.ID t.lastTime = l.CreateTime - if err := tw.StoreLog(&l); err != nil { - return err - } + binary.BigEndian.PutUint32(b, uint32(pos)) + meta.Write(b) + + pos = nextPos + + t.lastTime = l.CreateTime } - t.close() + var e error + if err := meta.Close(false); err != nil { + e = err + } - var tr *tableReader - if tr, err = tw.Flush(); err != nil { + data.SetOffset(pos) + + if err = data.Close(true); err != nil { return err } - t.first = tr.first - t.last = tr.last - t.offsetStartPos = tr.offsetStartPos - t.offsetLen = tr.offsetLen - - defer tr.Close() - - os.Remove(t.name) - - if err := os.Rename(tmpName, t.name); err != nil { - return err - } - - return nil + return e } -func (t *tableReader) decodeLogHead(l *Log, pos int64) (int64, error) { - _, err := t.f.Seek(int64(pos), os.SEEK_SET) - if err != nil { - return 0, err - } - - dataLen, err := l.DecodeHead(t.f) +func (t *tableReader) decodeLogHead(l *Log, r io.ReaderAt, pos int64) (int64, error) { + dataLen, err := l.DecodeHeadAt(r, pos) if err != nil { return 0, err } @@ -317,23 +284,20 @@ func (t *tableReader) GetLog(id uint64, l *Log) error { t.lastReadTime.Set(time.Now().Unix()) t.Lock() - defer t.Unlock() if err := t.openTable(); err != nil { t.close() + t.Unlock() return err } + t.Unlock() pos, err := t.getLogPos(int(id - t.first)) if err != nil { return err } - if _, err := t.f.Seek(int64(pos), os.SEEK_SET); err != nil { - return err - } - - if err := l.Decode(t.f); err != nil { + if err := l.DecodeAt(t.data, int64(pos)); err != nil { return err } else if l.ID != id { return fmt.Errorf("invalid log id %d != %d", l.ID, id) @@ -344,16 +308,17 @@ func (t *tableReader) GetLog(id uint64, l *Log) error { func (t *tableReader) openTable() error { var err error - if t.f == nil { - if t.f, err = os.Open(t.name); err != nil { + if t.data == nil { + if t.data, err = newReadFile(t.useMmap, fmtTableDataName(t.base, t.index)); err != nil { return err } } - if t.m == nil { - if t.m, err = mmap.MapRegion(t.f, int(t.offsetLen), mmap.RDONLY, 0, t.offsetStartPos); err != nil { + if t.meta == nil { + if t.meta, err = newReadFile(t.useMmap, fmtTableMetaName(t.base, t.index)); err != nil { return err } + } return nil @@ -362,31 +327,25 @@ func (t *tableReader) openTable() error { type tableWriter struct { sync.RWMutex - wf *os.File - rf *os.File - - wb *bufio.Writer - - rm sync.Mutex + data writeFile + meta writeFile base string - name string index int64 - first uint64 - last uint64 - - offsetPos int64 - offsetBuf []byte + first uint64 + last uint64 + lastTime uint32 maxLogSize int64 closed bool syncType int - lastTime uint32 - // cache *logLRUCache + posBuf []byte + + useMmap bool } func newTableWriter(base string, index int64, maxLogSize int64) *tableWriter { @@ -397,23 +356,24 @@ func newTableWriter(base string, index int64, maxLogSize int64) *tableWriter { t := new(tableWriter) t.base = base - t.name = path.Join(base, fmtTableName(index)) t.index = index - t.offsetPos = 0 t.maxLogSize = maxLogSize - //maybe config later? - t.wb = bufio.NewWriterSize(ioutil.Discard, 4096) - t.closed = false - //maybe use config later - // t.cache = newLogLRUCache(1024*1024, 1000) + t.posBuf = make([]byte, 4) + + //todo, use config + t.useMmap = useMmap return t } +func (t *tableWriter) String() string { + return fmt.Sprintf("%d", t.index) +} + func (t *tableWriter) SetMaxLogSize(s int64) { t.maxLogSize = s } @@ -423,26 +383,27 @@ func (t *tableWriter) SetSyncType(tp int) { } func (t *tableWriter) close() { - if t.rf != nil { - t.rf.Close() - t.rf = nil + if t.meta != nil { + if err := t.meta.Close(false); err != nil { + log.Fatal("close log meta error %s", err.Error()) + } + t.meta = nil } - if t.wf != nil { - t.wf.Close() - t.wf = nil + if t.data != nil { + if err := t.data.Close(true); err != nil { + log.Fatal("close log data error %s", err.Error()) + } + t.data = nil } - - t.wb.Reset(ioutil.Discard) } func (t *tableWriter) Close() { t.Lock() - defer t.Unlock() - t.closed = true t.close() + t.Unlock() } func (t *tableWriter) First() uint64 { @@ -459,88 +420,31 @@ func (t *tableWriter) Last() uint64 { return id } -func (t *tableWriter) reset() { +func (t *tableWriter) Flush() (*tableReader, error) { + t.Lock() + + if t.data == nil || t.meta == nil { + t.Unlock() + return nil, errNilHandler + } + + tr := new(tableReader) + tr.base = t.base + tr.index = t.index + + tr.first = t.first + tr.last = t.last + tr.lastTime = t.lastTime + //todo config + tr.useMmap = useMmap + t.close() t.first = 0 t.last = 0 t.index = t.index + 1 - t.name = path.Join(t.base, fmtTableName(t.index)) - t.offsetBuf = t.offsetBuf[0:0] - t.offsetPos = 0 - // t.cache.Reset() -} -func (t *tableWriter) Flush() (*tableReader, error) { - t.Lock() - defer t.Unlock() - - if t.wf == nil { - return nil, errNilHandler - } - - defer t.reset() - - tr := new(tableReader) - tr.name = t.name - tr.index = t.index - - st, _ := t.wf.Stat() - - tr.first = t.first - tr.last = t.last - - if n, err := t.wf.Write(log0Data); err != nil { - return nil, fmt.Errorf("flush log0data error %s", err.Error()) - } else if n != len(log0Data) { - return nil, fmt.Errorf("flush log0data only %d != %d", n, len(log0Data)) - } - - st, _ = t.wf.Stat() - - if m := st.Size() % pageSize; m != 0 { - padding := pageSize - m - if n, err := t.wf.Write(make([]byte, padding)); err != nil { - return nil, fmt.Errorf("flush log padding error %s", err.Error()) - } else if n != int(padding) { - return nil, fmt.Errorf("flush log padding error %d != %d", n, padding) - } - } - - st, _ = t.wf.Stat() - - if st.Size()%pageSize != 0 { - return nil, fmt.Errorf("invalid offset start pos, %d", st.Size()) - } - - tr.offsetStartPos = st.Size() - tr.offsetLen = uint32(len(t.offsetBuf)) - - if n, err := t.wf.Write(t.offsetBuf); err != nil { - log.Error("flush offset buffer error %s", err.Error()) - return nil, err - } else if n != len(t.offsetBuf) { - log.Error("flush offset buffer only %d != %d", n, len(t.offsetBuf)) - return nil, io.ErrShortWrite - } - - if err := binary.Write(t.wf, binary.BigEndian, tr.offsetStartPos); err != nil { - log.Error("flush offset start pos error %s", err.Error()) - return nil, err - } - - if err := binary.Write(t.wf, binary.BigEndian, tr.offsetLen); err != nil { - log.Error("flush offset len error %s", err.Error()) - return nil, err - } - - if n, err := t.wf.Write(magic); err != nil { - log.Error("flush magic data error %s", err.Error()) - return nil, err - } else if n != len(magic) { - log.Error("flush magic data only %d != %d", n, len(magic)) - return nil, io.ErrShortWrite - } + t.Unlock() return tr, nil } @@ -553,6 +457,22 @@ func (t *tableWriter) StoreLog(l *Log) error { return err } +func (t *tableWriter) openFile() error { + var err error + if t.data == nil { + if t.data, err = newWriteFile(t.useMmap, fmtTableDataName(t.base, t.index), t.maxLogSize+t.maxLogSize/10+int64(len(magic))); err != nil { + return err + } + } + + if t.meta == nil { + if t.meta, err = newWriteFile(t.useMmap, fmtTableMetaName(t.base, t.index), int64(defaultLogNumInFile*4)); err != nil { + return err + } + } + return err +} + func (t *tableWriter) storeLog(l *Log) error { if l.ID == 0 { return ErrStoreLogID @@ -566,63 +486,34 @@ func (t *tableWriter) storeLog(l *Log) error { return ErrStoreLogID } - if t.last-t.first+1 > maxLogNumInFile { + if t.data != nil && t.data.Offset() > t.maxLogSize { return errTableNeedFlush } var err error - if t.wf == nil { - if t.wf, err = os.OpenFile(t.name, os.O_CREATE|os.O_WRONLY, 0644); err != nil { - return err - } - t.wb.Reset(t.wf) - } - - if t.offsetBuf == nil { - t.offsetBuf = make([]byte, 0, maxLogNumInFile*4) - } - - // st, _ := t.wf.Stat() - // if st.Size() >= t.maxLogSize { - // return errTableNeedFlush - // } - - if t.offsetPos >= t.maxLogSize { - return errTableNeedFlush - } - - offsetPos := t.offsetPos - - if err = l.Encode(t.wb); err != nil { + if err = t.openFile(); err != nil { return err } - if err = t.wb.Flush(); err != nil { + offsetPos := t.data.Offset() + if err = l.Encode(t.data); err != nil { return err } - // buf, _ := l.Marshal() - // if n, err := t.wf.Write(buf); err != nil { - // return err - // } else if n != len(buf) { - // return io.ErrShortWrite - // } + binary.BigEndian.PutUint32(t.posBuf, uint32(offsetPos)) + if _, err = t.meta.Write(t.posBuf); err != nil { + return err + } - t.offsetPos += int64(l.Size()) - - t.offsetBuf = append(t.offsetBuf, num.Uint32ToBytes(uint32(offsetPos))...) if t.first == 0 { t.first = l.ID } t.last = l.ID - t.lastTime = l.CreateTime - // t.cache.Set(l.ID, buf) - if t.syncType == 2 { - if err := t.wf.Sync(); err != nil { + if err := t.data.Sync(); err != nil { log.Error("sync table error %s", err.Error()) } } @@ -638,17 +529,14 @@ func (t *tableWriter) GetLog(id uint64, l *Log) error { return ErrLogNotFound } - // if cl := t.cache.Get(id); cl != nil { - // if err := l.Unmarshal(cl); err == nil && l.ID == id { - // return nil - // } else { - // t.cache.Delete(id) - // } - // } + var buf [4]byte + if _, err := t.meta.ReadAt(buf[0:4], int64((id-t.first)*4)); err != nil { + return err + } - offset := binary.BigEndian.Uint32(t.offsetBuf[(id-t.first)*4:]) + offset := binary.BigEndian.Uint32(buf[0:4]) - if err := t.getLog(l, int64(offset)); err != nil { + if err := l.DecodeAt(t.data, int64(offset)); err != nil { return err } else if l.ID != id { return fmt.Errorf("invalid log id %d != %d", id, l.ID) @@ -661,32 +549,17 @@ func (t *tableWriter) Sync() error { t.Lock() var err error - if t.wf != nil { - err = t.wf.Sync() + if t.data != nil { + err = t.data.Sync() + t.Unlock() + return err } + + if t.meta != nil { + err = t.meta.Sync() + } + t.Unlock() return err } - -func (t *tableWriter) getLog(l *Log, pos int64) error { - t.rm.Lock() - defer t.rm.Unlock() - - var err error - if t.rf == nil { - if t.rf, err = os.Open(t.name); err != nil { - return err - } - } - - if _, err = t.rf.Seek(pos, os.SEEK_SET); err != nil { - return err - } - - if err = l.Decode(t.rf); err != nil { - return err - } - - return nil -} diff --git a/rpl/file_table_test.go b/rpl/file_table_test.go index c3ac2b2..b5cfb4b 100644 --- a/rpl/file_table_test.go +++ b/rpl/file_table_test.go @@ -10,6 +10,17 @@ import ( ) func TestFileTable(t *testing.T) { + useMmap = true + testFileTable(t) + + useMmap = false + testFileTable(t) + useMmap = true +} + +func testFileTable(t *testing.T) { + log.SetLevel(log.LevelInfo) + base, err := ioutil.TempDir("", "test_table") if err != nil { t.Fatal(err) @@ -50,10 +61,6 @@ func TestFileTable(t *testing.T) { var ll Log - if err = ll.Unmarshal(log0Data); err != nil { - t.Fatal(err) - } - for i := 0; i < 10; i++ { if err := w.GetLog(uint64(i+1), &ll); err != nil { t.Fatal(err) @@ -70,7 +77,7 @@ func TestFileTable(t *testing.T) { var r *tableReader - name := w.name + name := fmtTableDataName(w.base, w.index) if r, err = w.Flush(); err != nil { t.Fatal(err) @@ -130,26 +137,19 @@ func TestFileTable(t *testing.T) { t.Fatal("must nil") } - st, _ := r.f.Stat() - s := st.Size() + s := int64(r.data.Size()) r.Close() log.SetLevel(log.LevelFatal) testRepair(t, name, 1, s, 11) - testRepair(t, name, 1, s, 32) - testRepair(t, name, 1, s, 42) - testRepair(t, name, 1, s, 72) + testRepair(t, name, 1, s, 20) - if err := os.Truncate(name, s-(73+4096)); err != nil { + if err := os.Truncate(name, s-21); err != nil { t.Fatal(err) } - if r, err = newTableReader(base, 1); err == nil { - t.Fatal("can not repair") - } - if r, err := w.Flush(); err != nil { t.Fatal(err) } else { @@ -159,7 +159,7 @@ func TestFileTable(t *testing.T) { if r, err = newTableReader(base, 2); err != nil { t.Fatal(err) } - defer r.Close() + r.Close() } func testRepair(t *testing.T, name string, index int64, s int64, cutSize int64) { @@ -178,7 +178,7 @@ func testRepair(t *testing.T, name string, index int64, s int64, cutSize int64) var ll Log for i := 0; i < 10; i++ { if err := r.GetLog(uint64(i+1), &ll); err != nil { - t.Fatal(err) + t.Fatal(err, i) } else if len(ll.Data) != 4096 { t.Fatal(len(ll.Data)) } else if ll.Data[0] != byte(i+1) { @@ -190,9 +190,8 @@ func testRepair(t *testing.T, name string, index int64, s int64, cutSize int64) t.Fatal("must nil") } - st, _ := r.f.Stat() - if s != st.Size() { - t.Fatalf("repair error size %d != %d", s, st.Size()) + if s != int64(r.data.Size()) { + t.Fatalf("repair error size %d != %d", s, r.data.Size()) } } diff --git a/rpl/log.go b/rpl/log.go index 0b10fd4..ad0b48c 100644 --- a/rpl/log.go +++ b/rpl/log.go @@ -81,13 +81,8 @@ func (l *Log) Decode(r io.Reader) error { return err } - l.Data = l.Data[0:0] + l.growData(int(length)) - if cap(l.Data) >= int(length) { - l.Data = l.Data[0:length] - } else { - l.Data = make([]byte, length) - } if _, err := io.ReadFull(r, l.Data); err != nil { return err } @@ -103,6 +98,60 @@ func (l *Log) DecodeHead(r io.Reader) (uint32, error) { return 0, err } + length := l.decodeHeadBuf(buf) + + headPool.Put(buf) + + return length, nil +} + +func (l *Log) DecodeAt(r io.ReaderAt, pos int64) error { + length, err := l.DecodeHeadAt(r, pos) + if err != nil { + return err + } + + l.growData(int(length)) + var n int + n, err = r.ReadAt(l.Data, pos+int64(LogHeadSize)) + if err == io.EOF && n == len(l.Data) { + err = nil + } + + return err +} + +func (l *Log) growData(length int) { + l.Data = l.Data[0:0] + + if cap(l.Data) >= length { + l.Data = l.Data[0:length] + } else { + l.Data = make([]byte, length) + } +} + +func (l *Log) DecodeHeadAt(r io.ReaderAt, pos int64) (uint32, error) { + buf := headPool.Get().([]byte) + + n, err := r.ReadAt(buf, pos) + if err != nil && err != io.EOF { + headPool.Put(buf) + + return 0, err + } + + length := l.decodeHeadBuf(buf) + headPool.Put(buf) + + if err == io.EOF && (length != 0 || n != len(buf)) { + return 0, err + } + + return length, nil +} + +func (l *Log) decodeHeadBuf(buf []byte) uint32 { pos := 0 l.ID = binary.BigEndian.Uint64(buf[pos:]) pos += 8 @@ -114,8 +163,5 @@ func (l *Log) DecodeHead(r io.Reader) (uint32, error) { pos++ length := binary.BigEndian.Uint32(buf[pos:]) - - headPool.Put(buf) - - return length, nil + return length } diff --git a/rpl/loglrucache.go b/rpl/loglrucache.go deleted file mode 100644 index 3dcbaf3..0000000 --- a/rpl/loglrucache.go +++ /dev/null @@ -1,95 +0,0 @@ -package rpl - -import ( - "container/list" - "encoding/binary" -) - -type logLRUCache struct { - itemsList *list.List - itemsMap map[uint64]*list.Element - size int - capability int - maxNum int -} - -func newLogLRUCache(capability int, maxNum int) *logLRUCache { - if capability <= 0 { - capability = 1024 * 1024 - } - - if maxNum <= 0 { - maxNum = 16 - } - - return &logLRUCache{ - itemsList: list.New(), - itemsMap: make(map[uint64]*list.Element), - size: 0, - capability: capability, - maxNum: maxNum, - } -} - -func (cache *logLRUCache) Set(id uint64, data []byte) { - elem, ok := cache.itemsMap[id] - if ok { - //we may not enter here - // item already exists, so move it to the front of the list and update the data - cache.itemsList.MoveToFront(elem) - ol := elem.Value.([]byte) - elem.Value = data - cache.size += (len(data) - len(ol)) - } else { - cache.size += len(data) - - // item doesn't exist, so add it to front of list - elem = cache.itemsList.PushFront(data) - cache.itemsMap[id] = elem - } - - // evict LRU entry if the cache is full - for cache.size > cache.capability || cache.itemsList.Len() > cache.maxNum { - removedElem := cache.itemsList.Back() - l := removedElem.Value.([]byte) - cache.itemsList.Remove(removedElem) - delete(cache.itemsMap, binary.BigEndian.Uint64(l[0:8])) - - cache.size -= len(l) - if cache.size <= 0 { - cache.size = 0 - } - } -} - -func (cache *logLRUCache) Get(id uint64) []byte { - elem, ok := cache.itemsMap[id] - if !ok { - return nil - } - - // item exists, so move it to front of list and return it - cache.itemsList.MoveToFront(elem) - l := elem.Value.([]byte) - return l -} - -func (cache *logLRUCache) Delete(id uint64) { - elem, ok := cache.itemsMap[id] - if !ok { - return - } - - cache.itemsList.Remove(elem) - delete(cache.itemsMap, id) -} - -func (cache *logLRUCache) Len() int { - return cache.itemsList.Len() -} - -func (cache *logLRUCache) Reset() { - cache.itemsList = list.New() - cache.itemsMap = make(map[uint64]*list.Element) - cache.size = 0 -} diff --git a/rpl/loglrucache_test.go b/rpl/loglrucache_test.go deleted file mode 100644 index 88a2923..0000000 --- a/rpl/loglrucache_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package rpl - -import ( - "testing" -) - -func TestLogLRUCache(t *testing.T) { - c := newLogLRUCache(180, 10) - - var i uint64 - for i = 1; i <= 10; i++ { - l := &Log{i, 0, 0, []byte("0")} - b, _ := l.Marshal() - c.Set(l.ID, b) - } - - for i = 1; i <= 10; i++ { - if l := c.Get(i); l == nil { - t.Fatal("must exist", i) - } - } - - for i = 11; i <= 20; i++ { - l := &Log{i, 0, 0, []byte("0")} - b, _ := l.Marshal() - c.Set(l.ID, b) - } - - for i = 1; i <= 10; i++ { - if l := c.Get(i); l != nil { - t.Fatal("must not exist", i) - } - } - - c.Get(11) - - l := &Log{21, 0, 0, []byte("0")} - b, _ := l.Marshal() - c.Set(l.ID, b) - - if l := c.Get(12); l != nil { - t.Fatal("must nil", 12) - } - - if l := c.Get(11); l == nil { - t.Fatal("must not nil", 11) - } -} From 98ddc8b47a8ab5d6c1d952228493bdafc5a334cd Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 20 Nov 2014 22:28:08 +0800 Subject: [PATCH 91/98] add usemmap config --- config/config.go | 2 ++ config/config.toml | 3 +++ etc/ledis.conf | 3 +++ rpl/file_io.go | 36 ++++++++---------------------------- rpl/file_store.go | 6 +++--- rpl/file_table.go | 39 ++++++++++++++++++++++----------------- rpl/file_table_test.go | 24 ++++++++++-------------- 7 files changed, 51 insertions(+), 62 deletions(-) diff --git a/config/config.go b/config/config.go index 44887fa..92ea190 100644 --- a/config/config.go +++ b/config/config.go @@ -79,6 +79,7 @@ type ReplicationConfig struct { MaxLogFileNum int `toml:"max_log_file_num"` SyncLog int `toml:"sync_log"` Compression bool `toml:"compression"` + UseMmap bool `toml:"use_mmap"` } type SnapshotConfig struct { @@ -175,6 +176,7 @@ func NewConfigDefault() *Config { cfg.Replication.Compression = true cfg.Replication.WaitMaxSlaveAcks = 2 cfg.Replication.SyncLog = 0 + cfg.Replication.UseMmap = true cfg.Snapshot.MaxNum = 1 cfg.RocksDB.AllowOsBuffer = true diff --git a/config/config.toml b/config/config.toml index 4177699..44b38d6 100644 --- a/config/config.toml +++ b/config/config.toml @@ -134,6 +134,9 @@ max_log_file_size = 0 # for file store, if 0, use default 50 max_log_file_num = 0 +# for file store, use mmap for file read and write +use_mmap = true + # Sync log to disk if possible # 0: no sync # 1: sync every second diff --git a/etc/ledis.conf b/etc/ledis.conf index 4177699..44b38d6 100644 --- a/etc/ledis.conf +++ b/etc/ledis.conf @@ -134,6 +134,9 @@ max_log_file_size = 0 # for file store, if 0, use default 50 max_log_file_num = 0 +# for file store, use mmap for file read and write +use_mmap = true + # Sync log to disk if possible # 0: no sync # 1: sync every second diff --git a/rpl/file_io.go b/rpl/file_io.go index 2e0023c..5518b38 100644 --- a/rpl/file_io.go +++ b/rpl/file_io.go @@ -13,7 +13,7 @@ import ( type writeFile interface { Sync() error Write(b []byte) (n int, err error) - Close(addMagic bool) error + Close() error ReadAt(buf []byte, offset int64) (int, error) Truncate(size int64) error SetOffset(o int64) @@ -50,19 +50,9 @@ func newRawWriteFile(name string, size int64) (writeFile, error) { return m, nil } -func (m *rawWriteFile) Close(addMagic bool) error { - if addMagic { - if err := m.f.Truncate(m.offset + int64(len(magic))); err != nil { - return fmt.Errorf("close truncate %s error %s", m.name, err.Error()) - } - - if _, err := m.f.WriteAt(magic, m.offset); err != nil { - return fmt.Errorf("close write %s magic error %s", m.name, err.Error()) - } - } else { - if err := m.f.Truncate(m.offset); err != nil { - return fmt.Errorf("close truncate %s error %s", m.name, err.Error()) - } +func (m *rawWriteFile) Close() error { + if err := m.f.Truncate(m.offset); err != nil { + return fmt.Errorf("close truncate %s error %s", m.name, err.Error()) } if err := m.f.Close(); err != nil { @@ -77,7 +67,7 @@ func (m *rawWriteFile) Sync() error { } func (m *rawWriteFile) Write(b []byte) (n int, err error) { - n, err = m.f.Write(b) + n, err = m.f.WriteAt(b, m.offset) if err != nil { return } else if n != len(b) { @@ -210,23 +200,13 @@ func (m *mmapWriteFile) Sync() error { return m.m.Flush() } -func (m *mmapWriteFile) Close(addMagic bool) error { +func (m *mmapWriteFile) Close() error { if err := m.m.Unmap(); err != nil { return fmt.Errorf("unmap %s error %s", m.name, err.Error()) } - if addMagic { - if err := m.f.Truncate(m.offset + int64(len(magic))); err != nil { - return fmt.Errorf("close truncate %s error %s", m.name, err.Error()) - } - - if _, err := m.f.WriteAt(magic, m.offset); err != nil { - return fmt.Errorf("close write %s magic error %s", m.name, err.Error()) - } - } else { - if err := m.f.Truncate(m.offset); err != nil { - return fmt.Errorf("close truncate %s error %s", m.name, err.Error()) - } + if err := m.f.Truncate(m.offset); err != nil { + return fmt.Errorf("close truncate %s error %s", m.name, err.Error()) } if err := m.f.Close(); err != nil { diff --git a/rpl/file_store.go b/rpl/file_store.go index 5cebdfa..9c00b5e 100644 --- a/rpl/file_store.go +++ b/rpl/file_store.go @@ -90,7 +90,7 @@ func NewFileStore(base string, cfg *config.Config) (*FileStore, error) { index = s.rs[len(s.rs)-1].index + 1 } - s.w = newTableWriter(s.base, index, cfg.Replication.MaxLogFileSize) + s.w = newTableWriter(s.base, index, cfg.Replication.MaxLogFileSize, cfg.Replication.UseMmap) s.w.SetSyncType(cfg.Replication.SyncLog) go s.checkTableReaders() @@ -244,7 +244,7 @@ func (s *FileStore) Clear() error { return err } - s.w = newTableWriter(s.base, 1, s.cfg.Replication.MaxLogFileSize) + s.w = newTableWriter(s.base, 1, s.cfg.Replication.MaxLogFileSize, s.cfg.Replication.UseMmap) return nil } @@ -335,7 +335,7 @@ func (s *FileStore) load() error { var index int64 for _, f := range fs { if _, err := fmt.Sscanf(f.Name(), "%08d.data", &index); err == nil { - if r, err = newTableReader(s.base, index); err != nil { + if r, err = newTableReader(s.base, index, s.cfg.Replication.UseMmap); err != nil { log.Error("load table %s err: %s", f.Name(), err.Error()) } else { s.rs = append(s.rs, r) diff --git a/rpl/file_table.go b/rpl/file_table.go index de5f85d..b4dcbfd 100644 --- a/rpl/file_table.go +++ b/rpl/file_table.go @@ -29,9 +29,6 @@ func fmtTableMetaName(base string, index int64) string { return path.Join(base, fmt.Sprintf("%08d.meta", index)) } -//todo config -var useMmap = true - type tableReader struct { sync.Mutex @@ -51,7 +48,7 @@ type tableReader struct { useMmap bool } -func newTableReader(base string, index int64) (*tableReader, error) { +func newTableReader(base string, index int64, useMmap bool) (*tableReader, error) { if index <= 0 { return nil, fmt.Errorf("invalid index %d", index) } @@ -59,7 +56,6 @@ func newTableReader(base string, index int64) (*tableReader, error) { t.base = base t.index = index - //todo, use config t.useMmap = useMmap var err error @@ -122,7 +118,8 @@ func (t *tableReader) getLogPos(index int) (uint32, error) { func (t *tableReader) checkData() error { var err error - if t.data, err = newReadFile(t.useMmap, fmtTableDataName(t.base, t.index)); err != nil { + //check will use raw file mode + if t.data, err = newReadFile(false, fmtTableDataName(t.base, t.index)); err != nil { return err } @@ -144,7 +141,8 @@ func (t *tableReader) checkData() error { func (t *tableReader) checkMeta() error { var err error - if t.meta, err = newReadFile(t.useMmap, fmtTableMetaName(t.base, t.index)); err != nil { + //check will use raw file mode + if t.meta, err = newReadFile(false, fmtTableMetaName(t.base, t.index)); err != nil { return err } @@ -205,10 +203,11 @@ func (t *tableReader) repair() error { var data writeFile var meta writeFile - data, err = newWriteFile(t.useMmap, fmtTableDataName(t.base, t.index), 0) + //repair will use raw file mode + data, err = newWriteFile(false, fmtTableDataName(t.base, t.index), 0) data.SetOffset(int64(data.Size())) - meta, err = newWriteFile(t.useMmap, fmtTableMetaName(t.base, t.index), int64(defaultLogNumInFile*4)) + meta, err = newWriteFile(false, fmtTableMetaName(t.base, t.index), int64(defaultLogNumInFile*4)) var l Log var pos int64 = 0 @@ -254,13 +253,17 @@ func (t *tableReader) repair() error { } var e error - if err := meta.Close(false); err != nil { + if err := meta.Close(); err != nil { e = err } data.SetOffset(pos) - if err = data.Close(true); err != nil { + if _, err = data.Write(magic); err != nil { + log.Error("write magic error %s", err.Error()) + } + + if err = data.Close(); err != nil { return err } @@ -348,7 +351,7 @@ type tableWriter struct { useMmap bool } -func newTableWriter(base string, index int64, maxLogSize int64) *tableWriter { +func newTableWriter(base string, index int64, maxLogSize int64, useMmap bool) *tableWriter { if index <= 0 { panic(fmt.Errorf("invalid index %d", index)) } @@ -364,7 +367,6 @@ func newTableWriter(base string, index int64, maxLogSize int64) *tableWriter { t.posBuf = make([]byte, 4) - //todo, use config t.useMmap = useMmap return t @@ -384,14 +386,18 @@ func (t *tableWriter) SetSyncType(tp int) { func (t *tableWriter) close() { if t.meta != nil { - if err := t.meta.Close(false); err != nil { + if err := t.meta.Close(); err != nil { log.Fatal("close log meta error %s", err.Error()) } t.meta = nil } if t.data != nil { - if err := t.data.Close(true); err != nil { + if _, err := t.data.Write(magic); err != nil { + log.Fatal("write magic error %s", err.Error()) + } + + if err := t.data.Close(); err != nil { log.Fatal("close log data error %s", err.Error()) } t.data = nil @@ -435,8 +441,7 @@ func (t *tableWriter) Flush() (*tableReader, error) { tr.first = t.first tr.last = t.last tr.lastTime = t.lastTime - //todo config - tr.useMmap = useMmap + tr.useMmap = t.useMmap t.close() diff --git a/rpl/file_table_test.go b/rpl/file_table_test.go index b5cfb4b..e020c8a 100644 --- a/rpl/file_table_test.go +++ b/rpl/file_table_test.go @@ -10,15 +10,11 @@ import ( ) func TestFileTable(t *testing.T) { - useMmap = true - testFileTable(t) - - useMmap = false - testFileTable(t) - useMmap = true + testFileTable(t, true) + testFileTable(t, false) } -func testFileTable(t *testing.T) { +func testFileTable(t *testing.T, useMmap bool) { log.SetLevel(log.LevelInfo) base, err := ioutil.TempDir("", "test_table") @@ -34,7 +30,7 @@ func testFileTable(t *testing.T) { l.Compression = 0 l.Data = make([]byte, 4096) - w := newTableWriter(base, 1, 1024*1024) + w := newTableWriter(base, 1, 1024*1024, useMmap) defer w.Close() for i := 0; i < 10; i++ { @@ -118,7 +114,7 @@ func testFileTable(t *testing.T) { r.Close() - if r, err = newTableReader(base, 1); err != nil { + if r, err = newTableReader(base, 1, useMmap); err != nil { t.Fatal(err) } defer r.Close() @@ -143,8 +139,8 @@ func testFileTable(t *testing.T) { log.SetLevel(log.LevelFatal) - testRepair(t, name, 1, s, 11) - testRepair(t, name, 1, s, 20) + testRepair(t, name, 1, s, 11, useMmap) + testRepair(t, name, 1, s, 20, useMmap) if err := os.Truncate(name, s-21); err != nil { t.Fatal(err) @@ -156,13 +152,13 @@ func testFileTable(t *testing.T) { r.Close() } - if r, err = newTableReader(base, 2); err != nil { + if r, err = newTableReader(base, 2, useMmap); err != nil { t.Fatal(err) } r.Close() } -func testRepair(t *testing.T, name string, index int64, s int64, cutSize int64) { +func testRepair(t *testing.T, name string, index int64, s int64, cutSize int64, useMmap bool) { var r *tableReader var err error @@ -170,7 +166,7 @@ func testRepair(t *testing.T, name string, index int64, s int64, cutSize int64) t.Fatal(err) } - if r, err = newTableReader(path.Dir(name), index); err != nil { + if r, err = newTableReader(path.Dir(name), index, useMmap); err != nil { t.Fatal(err) } defer r.Close() From 9fc027e5537e95857f2867c583aab850714be29d Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 20 Nov 2014 22:52:29 +0800 Subject: [PATCH 92/98] use origin syndtr goleveldb --- Godeps/Godeps.json | 8 ++++++-- bootstrap.sh | 2 +- store/driver/batch.go | 2 +- store/goleveldb/batch.go | 2 +- store/goleveldb/db.go | 18 ++++++++++++------ store/goleveldb/iterator.go | 2 +- store/goleveldb/snapshot.go | 2 +- store/leveldb/batch.go | 2 +- store/writebatch.go | 2 +- 9 files changed, 25 insertions(+), 15 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 9fa9487..98b81a8 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -55,8 +55,12 @@ "Rev": "ecf49fc0738105e87d20e29aa82c403b666ff0b4" }, { - "ImportPath": "github.com/siddontang/goleveldb/leveldb", - "Rev": "41805642b981fb3d9462f6641bcb94b8609ca791" + "ImportPath": "github.com/syndtr/goleveldb/leveldb", + "Rev": "c9e0ae706141dc099005d6d247e4880c7feda2e1" + }, + { + "ImportPath": "github.com/syndtr/gosnappy/snappy", + "Rev": "ce8acff4829e0c2458a67ead32390ac0a381c862" }, { "ImportPath": "github.com/szferi/gomdb", diff --git a/bootstrap.sh b/bootstrap.sh index 6b36cbf..92872e9 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -17,8 +17,8 @@ go get -u github.com/boltdb/bolt go get -u github.com/ugorji/go/codec go get -u github.com/BurntSushi/toml go get -u github.com/edsrzf/mmap-go +go get -u github.com/syndtr/goleveldb/leveldb -go get -u github.com/siddontang/goleveldb/leveldb go get -u github.com/siddontang/go/bson go get -u github.com/siddontang/go/log go get -u github.com/siddontang/go/snappy diff --git a/store/driver/batch.go b/store/driver/batch.go index e793e76..1c1e899 100644 --- a/store/driver/batch.go +++ b/store/driver/batch.go @@ -1,7 +1,7 @@ package driver import ( - "github.com/siddontang/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb" ) type BatchPuter interface { diff --git a/store/goleveldb/batch.go b/store/goleveldb/batch.go index 00485e1..2032279 100644 --- a/store/goleveldb/batch.go +++ b/store/goleveldb/batch.go @@ -1,7 +1,7 @@ package goleveldb import ( - "github.com/siddontang/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb" ) type WriteBatch struct { diff --git a/store/goleveldb/db.go b/store/goleveldb/db.go index 9924067..af8633b 100644 --- a/store/goleveldb/db.go +++ b/store/goleveldb/db.go @@ -1,12 +1,12 @@ package goleveldb import ( - "github.com/siddontang/goleveldb/leveldb" - "github.com/siddontang/goleveldb/leveldb/cache" - "github.com/siddontang/goleveldb/leveldb/filter" - "github.com/siddontang/goleveldb/leveldb/opt" - "github.com/siddontang/goleveldb/leveldb/storage" - "github.com/siddontang/goleveldb/leveldb/util" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/cache" + "github.com/syndtr/goleveldb/leveldb/filter" + "github.com/syndtr/goleveldb/leveldb/opt" + "github.com/syndtr/goleveldb/leveldb/storage" + "github.com/syndtr/goleveldb/leveldb/util" "github.com/siddontang/ledisdb/config" "github.com/siddontang/ledisdb/store/driver" @@ -126,6 +126,12 @@ func newOptions(cfg *config.LevelDBConfig) *opt.Options { opts.BlockSize = cfg.BlockSize opts.WriteBuffer = cfg.WriteBufferSize + opts.CachedOpenFiles = cfg.MaxOpenFiles + + //here we use default value, later add config support + opts.CompactionTableSize = 32 * 1024 * 1024 + opts.WriteL0SlowdownTrigger = 16 + opts.WriteL0PauseTrigger = 64 return opts } diff --git a/store/goleveldb/iterator.go b/store/goleveldb/iterator.go index bd06376..c1fd8b5 100644 --- a/store/goleveldb/iterator.go +++ b/store/goleveldb/iterator.go @@ -1,7 +1,7 @@ package goleveldb import ( - "github.com/siddontang/goleveldb/leveldb/iterator" + "github.com/syndtr/goleveldb/leveldb/iterator" ) type Iterator struct { diff --git a/store/goleveldb/snapshot.go b/store/goleveldb/snapshot.go index 4dd56a9..c615579 100644 --- a/store/goleveldb/snapshot.go +++ b/store/goleveldb/snapshot.go @@ -1,8 +1,8 @@ package goleveldb import ( - "github.com/siddontang/goleveldb/leveldb" "github.com/siddontang/ledisdb/store/driver" + "github.com/syndtr/goleveldb/leveldb" ) type Snapshot struct { diff --git a/store/leveldb/batch.go b/store/leveldb/batch.go index 4e2f1c1..027aa39 100644 --- a/store/leveldb/batch.go +++ b/store/leveldb/batch.go @@ -8,7 +8,7 @@ package leveldb import "C" import ( - "github.com/siddontang/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb" "unsafe" ) diff --git a/store/writebatch.go b/store/writebatch.go index a69dbd8..c193ae0 100644 --- a/store/writebatch.go +++ b/store/writebatch.go @@ -2,8 +2,8 @@ package store import ( "encoding/binary" - "github.com/siddontang/goleveldb/leveldb" "github.com/siddontang/ledisdb/store/driver" + "github.com/syndtr/goleveldb/leveldb" "time" ) From abb4e8c90337d016b80a3b48b4a51d5a6680478b Mon Sep 17 00:00:00 2001 From: siddontang Date: Fri, 21 Nov 2014 09:13:48 +0800 Subject: [PATCH 93/98] update godep bootstrap --- bootstrap.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bootstrap.sh b/bootstrap.sh index 92872e9..185b9a8 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -6,6 +6,9 @@ godep path > /dev/null 2>&1 if [ "$?" = 0 ]; then GOPATH=`godep path` + # https://github.com/tools/godep/issues/60 + # have to rm Godeps/_workspace first, then restore + rm -rf $GOPATH godep restore exit 0 fi From 7b774092a22c27c7f0f5b5ef3b48f86713da98ef Mon Sep 17 00:00:00 2001 From: siddontang Date: Fri, 21 Nov 2014 10:32:18 +0800 Subject: [PATCH 94/98] travis add test race --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e3f1130..52e07e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,4 +6,4 @@ before_install: - go install -race std script: - godep go test -cover ./... -# - godep go test -race ./... + - godep go test -race ./... From 2a4504aafab257b72e6c1a468779cd18f82f6468 Mon Sep 17 00:00:00 2001 From: siddontang Date: Fri, 21 Nov 2014 10:32:25 +0800 Subject: [PATCH 95/98] add version tag --- ledis/const.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ledis/const.go b/ledis/const.go index 26a429e..a61cb90 100644 --- a/ledis/const.go +++ b/ledis/const.go @@ -4,6 +4,8 @@ import ( "errors" ) +const Version = "0.4" + const ( NoneType byte = 0 KVType byte = 1 From 9e904939927e70e3345634e74b80f909d505c778 Mon Sep 17 00:00:00 2001 From: siddontang Date: Fri, 21 Nov 2014 10:32:33 +0800 Subject: [PATCH 96/98] update read me --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9e4ee0a..a8f9e20 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,14 @@ -# LedisDB +# LedisDB + +[![Build Status](https://travis-ci.org/siddontang/ledisdb.svg?branch=develop)](https://travis-ci.org/siddontang/ledisdb) Ledisdb is a high performance NoSQL like Redis written by go. It supports some data structure like kv, list, hash, zset, bitmap,set. LedisDB now supports multiple databases as backend to store data, you can test and choose the proper one for you. +### **You must run `ledis-upgrade-ttl` before using LedisDB version 0.4, I fixed a very serious bug for key expiration and ttl.** + + ## Features + Rich data structure: KV, List, Hash, ZSet, Bitmap, Set. From 2f4aaa4ba53622f52d5cbf7cd12a6b6d5460ecb5 Mon Sep 17 00:00:00 2001 From: siddontang Date: Fri, 21 Nov 2014 10:56:56 +0800 Subject: [PATCH 97/98] remove println --- rpl/file_io.go | 1 - rpl/file_store.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/rpl/file_io.go b/rpl/file_io.go index 5518b38..08e9d2e 100644 --- a/rpl/file_io.go +++ b/rpl/file_io.go @@ -220,7 +220,6 @@ func (m *mmapWriteFile) Write(b []byte) (n int, err error) { extra := int64(len(b)) - (m.size - m.offset) if extra > 0 { newSize := m.size + extra + m.size/10 - println("need truncate ???", newSize, m.size, len(b)) if err = m.Truncate(newSize); err != nil { return } diff --git a/rpl/file_store.go b/rpl/file_store.go index 9c00b5e..161ab8d 100644 --- a/rpl/file_store.go +++ b/rpl/file_store.go @@ -177,7 +177,7 @@ func (s *FileStore) storeLog(l *Log) error { r, err = s.w.Flush() if err != nil { - log.Error("write table flush error %s, can not store now", err.Error()) + log.Fatal("write table flush error %s, can not store!!!", err.Error()) s.w.Close() From 72932f68a0b4226845fcd80424d5eb8f7028babd Mon Sep 17 00:00:00 2001 From: siddontang Date: Sat, 22 Nov 2014 09:10:41 +0800 Subject: [PATCH 98/98] update godep --- Godeps/Godeps.json | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 98b81a8..4c1bf08 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -11,8 +11,8 @@ }, { "ImportPath": "github.com/boltdb/bolt", - "Comment": "data/v1-228-g8fb50d5", - "Rev": "8fb50d5ee57110936b904a7539d4c5f2bf2359db" + "Comment": "data/v1-254-gd285804", + "Rev": "d285804df1760edf4c602ecd901be5d5e67bf982" }, { "ImportPath": "github.com/edsrzf/mmap-go", @@ -20,39 +20,39 @@ }, { "ImportPath": "github.com/siddontang/go/arena", - "Rev": "ecf49fc0738105e87d20e29aa82c403b666ff0b4" + "Rev": "8f64946c30746240c2f3bdb606eed9a4aca34478" }, { "ImportPath": "github.com/siddontang/go/bson", - "Rev": "ecf49fc0738105e87d20e29aa82c403b666ff0b4" + "Rev": "8f64946c30746240c2f3bdb606eed9a4aca34478" }, { "ImportPath": "github.com/siddontang/go/filelock", - "Rev": "ecf49fc0738105e87d20e29aa82c403b666ff0b4" + "Rev": "8f64946c30746240c2f3bdb606eed9a4aca34478" }, { "ImportPath": "github.com/siddontang/go/hack", - "Rev": "ecf49fc0738105e87d20e29aa82c403b666ff0b4" + "Rev": "8f64946c30746240c2f3bdb606eed9a4aca34478" }, { "ImportPath": "github.com/siddontang/go/ioutil2", - "Rev": "ecf49fc0738105e87d20e29aa82c403b666ff0b4" + "Rev": "8f64946c30746240c2f3bdb606eed9a4aca34478" }, { "ImportPath": "github.com/siddontang/go/log", - "Rev": "ecf49fc0738105e87d20e29aa82c403b666ff0b4" + "Rev": "8f64946c30746240c2f3bdb606eed9a4aca34478" }, { "ImportPath": "github.com/siddontang/go/num", - "Rev": "ecf49fc0738105e87d20e29aa82c403b666ff0b4" + "Rev": "8f64946c30746240c2f3bdb606eed9a4aca34478" }, { "ImportPath": "github.com/siddontang/go/snappy", - "Rev": "ecf49fc0738105e87d20e29aa82c403b666ff0b4" + "Rev": "8f64946c30746240c2f3bdb606eed9a4aca34478" }, { "ImportPath": "github.com/siddontang/go/sync2", - "Rev": "ecf49fc0738105e87d20e29aa82c403b666ff0b4" + "Rev": "8f64946c30746240c2f3bdb606eed9a4aca34478" }, { "ImportPath": "github.com/syndtr/goleveldb/leveldb",