From 38355c59f7567b411e68809485030e17b495fc14 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 28 Aug 2014 16:07:48 +0800 Subject: [PATCH 01/32] update doc --- doc/commands.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/commands.md b/doc/commands.md index 535602d..fe5437f 100644 --- a/doc/commands.md +++ b/doc/commands.md @@ -869,7 +869,7 @@ ledis> HPERSIST not_exists_key Iterate Hash keys incrementally. -See `SCAN` for more information. +See [Scan](#scan-key-match-match-count-count) for more information. ## List @@ -1166,7 +1166,7 @@ ledis> LPERSIST b Iterate list keys incrementally. -See `SCAN` for more information. +See [Scan](#scan-key-match-match-count-count) for more information. ## Set @@ -1594,7 +1594,7 @@ ledis> STTL key Iterate Set keys incrementally. -See `SCAN` for more information. +See [Scan](#scan-key-match-match-count-count) for more information. ## ZSet @@ -2220,7 +2220,7 @@ ledis> ZRANGE out 0 -1 WITHSCORES Iterate ZSet keys incrementally. -See `SCAN` for more information. +See [Scan](#scan-key-match-match-count-count) for more information. ## Bitmap @@ -2386,7 +2386,7 @@ ledis> BCOUNT flag 5 6 Iterate Bitmap keys incrementally. -See `SCAN` for more information. +See [Scan](#scan-key-match-match-count-count) for more information. ## Replication From 15636bc0cb17f08548cbd0f13235ba21a55e5e71 Mon Sep 17 00:00:00 2001 From: holys Date: Thu, 28 Aug 2014 17:00:37 +0800 Subject: [PATCH 02/32] refactor test, reduce code --- server/cmd_ttl_test.go | 502 ++++++++--------------------------------- 1 file changed, 100 insertions(+), 402 deletions(-) diff --git a/server/cmd_ttl_test.go b/server/cmd_ttl_test.go index 702348d..c9d388c 100644 --- a/server/cmd_ttl_test.go +++ b/server/cmd_ttl_test.go @@ -1,6 +1,7 @@ package server import ( + "fmt" "github.com/siddontang/ledisdb/client/go/ledis" "testing" "time" @@ -10,422 +11,119 @@ func now() int64 { return time.Now().Unix() } -func TestKVExpire(t *testing.T) { +func TestExpire(t *testing.T) { + // test for kv, list, hash, set, zset, bitmap in all + ttlType := []string{"k", "l", "h", "s", "z", "b"} + + var ( + expire string + expireat string + ttl string + persist string + key string + ) + c := getTestConn() defer c.Close() - k := "a_ttl" - c.Do("set", k, "123") + idx := 1 + for _, tt := range ttlType { + if tt == "k" { + expire = "expire" + expireat = "expireat" + ttl = "ttl" + persist = "persist" - // expire + ttl - exp := int64(10) - if n, err := ledis.Int(c.Do("expire", k, exp)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } + } else { + expire = fmt.Sprintf("%sexpire", tt) + expireat = fmt.Sprintf("%sexpireat", tt) + ttl = fmt.Sprintf("%sttl", tt) + persist = fmt.Sprintf("%spersist", tt) + } - if ttl, err := ledis.Int64(c.Do("ttl", k)); err != nil { - t.Fatal(err) - } else if ttl != exp { - t.Fatal(ttl) - } + switch tt { + case "k": + key = "kv_ttl" + c.Do("set", key, "123") + case "l": + key = "list_ttl" + c.Do("rpush", key, "123") + case "h": + key = "hash_ttl" + c.Do("hset", key, "a", "123") + case "s": + key = "set_ttl" + c.Do("sadd", key, "123") + case "z": + key = "zset_ttl" + c.Do("zadd", key, 123, "a") + case "b": + key = "bitmap_ttl" + c.Do("bsetbit", key, 0, 1) + } - // expireat + ttl - tm := now() + 3 - if n, err := ledis.Int(c.Do("expireat", k, tm)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } + // expire + ttl + exp := int64(10) + if n, err := ledis.Int(c.Do(expire, key, exp)); err != nil { + t.Fatal(err) + } else if n != 1 { + t.Fatal(n) + } - if ttl, err := ledis.Int64(c.Do("ttl", k)); err != nil { - t.Fatal(err) - } else if ttl != 3 { - t.Fatal(ttl) - } + if ttl, err := ledis.Int64(c.Do(ttl, key)); err != nil { + t.Fatal(err) + } else if ttl != exp { + t.Fatal(ttl) + } - kErr := "not_exist_ttl" + // expireat + ttl + tm := now() + 3 + if n, err := ledis.Int(c.Do(expireat, key, tm)); err != nil { + t.Fatal(err) + } else if n != 1 { + t.Fatal(n) + } - // err - expire, expireat - if n, err := ledis.Int(c.Do("expire", kErr, tm)); err != nil || n != 0 { - t.Fatal(false) - } + if ttl, err := ledis.Int64(c.Do(ttl, key)); err != nil { + t.Fatal(err) + } else if ttl != 3 { + t.Fatal(ttl) + } - if n, err := ledis.Int(c.Do("expireat", kErr, tm)); err != nil || n != 0 { - t.Fatal(false) - } + kErr := "not_exist_ttl" - if n, err := ledis.Int(c.Do("ttl", kErr)); err != nil || n != -1 { - t.Fatal(false) - } + // err - expire, expireat + if n, err := ledis.Int(c.Do(expire, kErr, tm)); err != nil || n != 0 { + t.Fatal(false) + } - if n, err := ledis.Int(c.Do("persist", k)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } + if n, err := ledis.Int(c.Do(expireat, kErr, tm)); err != nil || n != 0 { + t.Fatal(false) + } - if n, err := ledis.Int(c.Do("expire", k, 10)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } + if n, err := ledis.Int(c.Do(ttl, kErr)); err != nil || n != -1 { + t.Fatal(false) + } - if n, err := ledis.Int(c.Do("persist", k)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - -} - -func TestSetExpire(t *testing.T) { - c := getTestConn() - defer c.Close() - - k := "set_ttl" - c.Do("sadd", k, "123") - - // expire + ttl - exp := int64(10) - if n, err := ledis.Int(c.Do("sexpire", k, exp)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - - if ttl, err := ledis.Int64(c.Do("sttl", k)); err != nil { - t.Fatal(err) - } else if ttl != exp { - t.Fatal(ttl) - } - - // expireat + ttl - tm := now() + 3 - if n, err := ledis.Int(c.Do("sexpireat", k, tm)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - - if ttl, err := ledis.Int64(c.Do("sttl", k)); err != nil { - t.Fatal(err) - } else if ttl != 3 { - t.Fatal(ttl) - } - - kErr := "not_exist_ttl" - - // err - expire, expireat - if n, err := ledis.Int(c.Do("sexpire", kErr, tm)); err != nil || n != 0 { - t.Fatal(false) - } - - if n, err := ledis.Int(c.Do("sexpireat", kErr, tm)); err != nil || n != 0 { - t.Fatal(false) - } - - if n, err := ledis.Int(c.Do("sttl", kErr)); err != nil || n != -1 { - t.Fatal(false) - } - - if n, err := ledis.Int(c.Do("spersist", k)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - - if n, err := ledis.Int(c.Do("sexpire", k, 10)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - - if n, err := ledis.Int(c.Do("spersist", k)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - -} - -func TestListExpire(t *testing.T) { - c := getTestConn() - defer c.Close() - - k := "list_ttl" - c.Do("rpush", k, "123") - - // expire + ttl - exp := int64(10) - if n, err := ledis.Int(c.Do("lexpire", k, exp)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - - if ttl, err := ledis.Int64(c.Do("lttl", k)); err != nil { - t.Fatal(err) - } else if ttl != exp { - t.Fatal(ttl) - } - - // expireat + ttl - tm := now() + 3 - if n, err := ledis.Int(c.Do("lexpireat", k, tm)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - - if ttl, err := ledis.Int64(c.Do("lttl", k)); err != nil { - t.Fatal(err) - } else if ttl != 3 { - t.Fatal(ttl) - } - - kErr := "not_exist_ttl" - - // err - expire, expireat - if n, err := ledis.Int(c.Do("lexpire", kErr, tm)); err != nil || n != 0 { - t.Fatal(false) - } - - if n, err := ledis.Int(c.Do("lexpireat", kErr, tm)); err != nil || n != 0 { - t.Fatal(false) - } - - if n, err := ledis.Int(c.Do("lttl", kErr)); err != nil || n != -1 { - t.Fatal(false) - } - - if n, err := ledis.Int(c.Do("lpersist", k)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - - if n, err := ledis.Int(c.Do("lexpire", k, 10)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - - if n, err := ledis.Int(c.Do("lpersist", k)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - -} - -func TestHashExpire(t *testing.T) { - c := getTestConn() - defer c.Close() - - k := "hash_ttl" - c.Do("hset", k, "f", 123) - - // expire + ttl - exp := int64(10) - if n, err := ledis.Int(c.Do("hexpire", k, exp)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - - if ttl, err := ledis.Int64(c.Do("httl", k)); err != nil { - t.Fatal(err) - } else if ttl != exp { - t.Fatal(ttl) - } - - // expireat + ttl - tm := now() + 3 - if n, err := ledis.Int(c.Do("hexpireat", k, tm)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - - if ttl, err := ledis.Int64(c.Do("httl", k)); err != nil { - t.Fatal(err) - } else if ttl != 3 { - t.Fatal(ttl) - } - - kErr := "not_exist_ttl" - - // err - expire, expireat - if n, err := ledis.Int(c.Do("hexpire", kErr, tm)); err != nil || n != 0 { - t.Fatal(false) - } - - if n, err := ledis.Int(c.Do("hexpireat", kErr, tm)); err != nil || n != 0 { - t.Fatal(false) - } - - if n, err := ledis.Int(c.Do("httl", kErr)); err != nil || n != -1 { - t.Fatal(false) - } - - if n, err := ledis.Int(c.Do("hpersist", k)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - - if n, err := ledis.Int(c.Do("hexpire", k, 10)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - - if n, err := ledis.Int(c.Do("hpersist", k)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - -} - -func TestZsetExpire(t *testing.T) { - c := getTestConn() - defer c.Close() - - k := "zset_ttl" - c.Do("zadd", k, 123, "m") - - // expire + ttl - exp := int64(10) - if n, err := ledis.Int(c.Do("zexpire", k, exp)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - - if ttl, err := ledis.Int64(c.Do("zttl", k)); err != nil { - t.Fatal(err) - } else if ttl != exp { - t.Fatal(ttl) - } - - // expireat + ttl - tm := now() + 3 - if n, err := ledis.Int(c.Do("zexpireat", k, tm)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - - if ttl, err := ledis.Int64(c.Do("zttl", k)); err != nil { - t.Fatal(err) - } else if ttl != 3 { - t.Fatal(ttl) - } - - kErr := "not_exist_ttl" - - // err - expire, expireat - if n, err := ledis.Int(c.Do("zexpire", kErr, tm)); err != nil || n != 0 { - t.Fatal(false) - } - - if n, err := ledis.Int(c.Do("zexpireat", kErr, tm)); err != nil || n != 0 { - t.Fatal(false) - } - - if n, err := ledis.Int(c.Do("zttl", kErr)); err != nil || n != -1 { - t.Fatal(false) - } - - if n, err := ledis.Int(c.Do("zpersist", k)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - - if n, err := ledis.Int(c.Do("zexpire", k, 10)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - - if n, err := ledis.Int(c.Do("zpersist", k)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - -} - -func TestBitmapExpire(t *testing.T) { - c := getTestConn() - defer c.Close() - - k := "bit_ttl" - c.Do("bsetbit", k, 0, 1) - - // expire + ttl - exp := int64(10) - if n, err := ledis.Int(c.Do("bexpire", k, exp)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - - if ttl, err := ledis.Int64(c.Do("bttl", k)); err != nil { - t.Fatal(err) - } else if ttl != exp { - t.Fatal(ttl) - } - - // expireat + ttl - tm := now() + 3 - if n, err := ledis.Int(c.Do("bexpireat", k, tm)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - - if ttl, err := ledis.Int64(c.Do("bttl", k)); err != nil { - t.Fatal(err) - } else if ttl != 3 { - t.Fatal(ttl) - } - - kErr := "not_exist_ttl" - - // err - expire, expireat - if n, err := ledis.Int(c.Do("bexpire", kErr, tm)); err != nil || n != 0 { - t.Fatal(false) - } - - if n, err := ledis.Int(c.Do("bexpireat", kErr, tm)); err != nil || n != 0 { - t.Fatal(false) - } - - if n, err := ledis.Int(c.Do("bttl", kErr)); err != nil || n != -1 { - t.Fatal(false) - } - - if n, err := ledis.Int(c.Do("bpersist", k)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - - if n, err := ledis.Int(c.Do("bexpire", k, 10)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) - } - - if n, err := ledis.Int(c.Do("bpersist", k)); err != nil { - t.Fatal(err) - } else if n != 1 { - t.Fatal(n) + if n, err := ledis.Int(c.Do(persist, key)); err != nil { + t.Fatal(err) + } else if n != 1 { + t.Fatal(n) + } + + if n, err := ledis.Int(c.Do(expire, key, 10)); err != nil { + t.Fatal(err) + } else if n != 1 { + t.Fatal(n) + } + + if n, err := ledis.Int(c.Do(persist, key)); err != nil { + t.Fatal(err) + } else if n != 1 { + t.Fatal(n) + } + + idx++ } } From ed982db4cc7941aa6447d1f76ed049a6f66c649d Mon Sep 17 00:00:00 2001 From: siddontang Date: Sat, 30 Aug 2014 10:17:04 +0800 Subject: [PATCH 03/32] use import `_` to load backend storage --- store/store.go | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/store/store.go b/store/store.go index 50d2744..e2a6b85 100644 --- a/store/store.go +++ b/store/store.go @@ -7,12 +7,12 @@ import ( "os" "path" - "github.com/siddontang/ledisdb/store/boltdb" - "github.com/siddontang/ledisdb/store/goleveldb" - "github.com/siddontang/ledisdb/store/hyperleveldb" - "github.com/siddontang/ledisdb/store/leveldb" - "github.com/siddontang/ledisdb/store/mdb" - "github.com/siddontang/ledisdb/store/rocksdb" + _ "github.com/siddontang/ledisdb/store/boltdb" + _ "github.com/siddontang/ledisdb/store/goleveldb" + _ "github.com/siddontang/ledisdb/store/hyperleveldb" + _ "github.com/siddontang/ledisdb/store/leveldb" + _ "github.com/siddontang/ledisdb/store/mdb" + _ "github.com/siddontang/ledisdb/store/rocksdb" ) func getStorePath(cfg *config.Config) string { @@ -53,10 +53,4 @@ func Repair(cfg *config.Config) error { } func init() { - _ = boltdb.DBName - _ = goleveldb.DBName - _ = hyperleveldb.DBName - _ = leveldb.DBName - _ = mdb.DBName - _ = rocksdb.DBName } From 577d54548637fda5450c038d1c5d9c43ed501fed Mon Sep 17 00:00:00 2001 From: siddontang Date: Sat, 30 Aug 2014 17:19:10 +0800 Subject: [PATCH 04/32] store remove tricky code --- store/db.go | 10 +++ store/hyperleveldb/db.go | 8 +-- store/hyperleveldb/hyperleveldb_ext.cc | 86 +++++++++++++------------- store/hyperleveldb/hyperleveldb_ext.h | 24 +++---- store/leveldb/db.go | 8 +-- store/leveldb/leveldb_ext.cc | 86 +++++++++++++------------- store/leveldb/leveldb_ext.h | 24 +++---- store/snapshot.go | 16 +++++ 8 files changed, 142 insertions(+), 120 deletions(-) create mode 100644 store/snapshot.go diff --git a/store/db.go b/store/db.go index 76c63e3..ca57326 100644 --- a/store/db.go +++ b/store/db.go @@ -19,6 +19,16 @@ func (db *DB) NewWriteBatch() WriteBatch { return db.IDB.NewWriteBatch() } +func (db *DB) NewSnapshot() (*Snapshot, error) { + var err error + s := &Snapshot{} + if s.ISnapshot, err = db.IDB.NewSnapshot(); err != nil { + return nil, err + } + + return s, nil +} + func (db *DB) RangeIterator(min []byte, max []byte, rangeType uint8) *RangeLimitIterator { return NewRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, &Limit{0, -1}) } diff --git a/store/hyperleveldb/db.go b/store/hyperleveldb/db.go index e8ec944..6d0e176 100644 --- a/store/hyperleveldb/db.go +++ b/store/hyperleveldb/db.go @@ -234,10 +234,8 @@ func (db *DB) get(ro *ReadOptions, key []byte) ([]byte, error) { k = (*C.char)(unsafe.Pointer(&key[0])) } - var value *C.char - - c := C.hyperleveldb_get_ext( - db.db, ro.Opt, k, C.size_t(len(key)), &value, &vallen, &errStr) + value := C.leveldb_get( + db.db, ro.Opt, k, C.size_t(len(key)), &vallen, &errStr) if errStr != nil { return nil, saveError(errStr) @@ -247,7 +245,7 @@ func (db *DB) get(ro *ReadOptions, key []byte) ([]byte, error) { return nil, nil } - defer C.hyperleveldb_get_free_ext(unsafe.Pointer(c)) + defer C.leveldb_free(unsafe.Pointer(value)) return C.GoBytes(unsafe.Pointer(value), C.int(vallen)), nil } diff --git a/store/hyperleveldb/hyperleveldb_ext.cc b/store/hyperleveldb/hyperleveldb_ext.cc index dab687c..f775ee9 100644 --- a/store/hyperleveldb/hyperleveldb_ext.cc +++ b/store/hyperleveldb/hyperleveldb_ext.cc @@ -3,60 +3,60 @@ #include "hyperleveldb_ext.h" #include -#include +//#include -#include "hyperleveldb/db.h" +//#include "hyperleveldb/db.h" -using namespace leveldb; +//using namespace leveldb; extern "C" { -static bool SaveError(char** errptr, const Status& s) { - assert(errptr != NULL); - if (s.ok()) { - return false; - } else if (*errptr == NULL) { - *errptr = strdup(s.ToString().c_str()); - } else { - free(*errptr); - *errptr = strdup(s.ToString().c_str()); - } - return true; -} +// static bool SaveError(char** errptr, const Status& s) { +// assert(errptr != NULL); +// if (s.ok()) { +// return false; +// } else if (*errptr == NULL) { +// *errptr = strdup(s.ToString().c_str()); +// } else { +// free(*errptr); +// *errptr = strdup(s.ToString().c_str()); +// } +// return true; +// } -void* hyperleveldb_get_ext( - leveldb_t* db, - const leveldb_readoptions_t* options, - const char* key, size_t keylen, - char** valptr, - size_t* vallen, - char** errptr) { +// void* hyperleveldb_get_ext( +// leveldb_t* db, +// const leveldb_readoptions_t* options, +// const char* key, size_t keylen, +// char** valptr, +// size_t* vallen, +// char** errptr) { - std::string *tmp = new(std::string); +// std::string *tmp = new(std::string); - //very tricky, maybe changed with c++ leveldb upgrade - Status s = (*(DB**)db)->Get(*(ReadOptions*)options, Slice(key, keylen), tmp); +// //very tricky, maybe changed with c++ leveldb upgrade +// Status s = (*(DB**)db)->Get(*(ReadOptions*)options, Slice(key, keylen), tmp); - if (s.ok()) { - *valptr = (char*)tmp->data(); - *vallen = tmp->size(); - } else { - delete(tmp); - tmp = NULL; - *valptr = NULL; - *vallen = 0; - if (!s.IsNotFound()) { - SaveError(errptr, s); - } - } - return tmp; -} +// if (s.ok()) { +// *valptr = (char*)tmp->data(); +// *vallen = tmp->size(); +// } else { +// delete(tmp); +// tmp = NULL; +// *valptr = NULL; +// *vallen = 0; +// if (!s.IsNotFound()) { +// SaveError(errptr, s); +// } +// } +// return tmp; +// } -void hyperleveldb_get_free_ext(void* context) { - std::string* s = (std::string*)context; +// void hyperleveldb_get_free_ext(void* context) { +// std::string* s = (std::string*)context; - delete(s); -} +// delete(s); +// } unsigned char hyperleveldb_iter_seek_to_first_ext(leveldb_iterator_t* iter) { diff --git a/store/hyperleveldb/hyperleveldb_ext.h b/store/hyperleveldb/hyperleveldb_ext.h index 940a090..9182768 100644 --- a/store/hyperleveldb/hyperleveldb_ext.h +++ b/store/hyperleveldb/hyperleveldb_ext.h @@ -10,19 +10,19 @@ extern "C" { #include "hyperleveldb/c.h" -/* Returns NULL if not found. Otherwise stores the value in **valptr. - Stores the length of the value in *vallen. - Returns a context must be later to free*/ -extern void* hyperleveldb_get_ext( - leveldb_t* db, - const leveldb_readoptions_t* options, - const char* key, size_t keylen, - char** valptr, - size_t* vallen, - char** errptr); +// /* Returns NULL if not found. Otherwise stores the value in **valptr. +// Stores the length of the value in *vallen. +// Returns a context must be later to free*/ +// extern void* hyperleveldb_get_ext( +// leveldb_t* db, +// const leveldb_readoptions_t* options, +// const char* key, size_t keylen, +// char** valptr, +// size_t* vallen, +// char** errptr); -// Free context returns by hyperleveldb_get_ext -extern void hyperleveldb_get_free_ext(void* context); +// // Free context returns by hyperleveldb_get_ext +// extern void hyperleveldb_get_free_ext(void* context); // Below iterator functions like leveldb iterator but returns valid status for iterator diff --git a/store/leveldb/db.go b/store/leveldb/db.go index 0a40953..43ee0c2 100644 --- a/store/leveldb/db.go +++ b/store/leveldb/db.go @@ -234,10 +234,8 @@ func (db *DB) get(ro *ReadOptions, key []byte) ([]byte, error) { k = (*C.char)(unsafe.Pointer(&key[0])) } - var value *C.char - - c := C.leveldb_get_ext( - db.db, ro.Opt, k, C.size_t(len(key)), &value, &vallen, &errStr) + value := C.leveldb_get( + db.db, ro.Opt, k, C.size_t(len(key)), &vallen, &errStr) if errStr != nil { return nil, saveError(errStr) @@ -247,7 +245,7 @@ func (db *DB) get(ro *ReadOptions, key []byte) ([]byte, error) { return nil, nil } - defer C.leveldb_get_free_ext(unsafe.Pointer(c)) + defer C.leveldb_free(unsafe.Pointer(value)) return C.GoBytes(unsafe.Pointer(value), C.int(vallen)), nil } diff --git a/store/leveldb/leveldb_ext.cc b/store/leveldb/leveldb_ext.cc index 96d6541..a362ab5 100644 --- a/store/leveldb/leveldb_ext.cc +++ b/store/leveldb/leveldb_ext.cc @@ -3,60 +3,60 @@ #include "leveldb_ext.h" #include -#include +//#include -#include "leveldb/db.h" +//#include "leveldb/db.h" -using namespace leveldb; +//using namespace leveldb; extern "C" { -static bool SaveError(char** errptr, const Status& s) { - assert(errptr != NULL); - if (s.ok()) { - return false; - } else if (*errptr == NULL) { - *errptr = strdup(s.ToString().c_str()); - } else { - free(*errptr); - *errptr = strdup(s.ToString().c_str()); - } - return true; -} +// static bool SaveError(char** errptr, const Status& s) { +// assert(errptr != NULL); +// if (s.ok()) { +// return false; +// } else if (*errptr == NULL) { +// *errptr = strdup(s.ToString().c_str()); +// } else { +// free(*errptr); +// *errptr = strdup(s.ToString().c_str()); +// } +// return true; +// } -void* leveldb_get_ext( - leveldb_t* db, - const leveldb_readoptions_t* options, - const char* key, size_t keylen, - char** valptr, - size_t* vallen, - char** errptr) { +// void* leveldb_get_ext( +// leveldb_t* db, +// const leveldb_readoptions_t* options, +// const char* key, size_t keylen, +// char** valptr, +// size_t* vallen, +// char** errptr) { - std::string *tmp = new(std::string); +// std::string *tmp = new(std::string); - //very tricky, maybe changed with c++ leveldb upgrade - Status s = (*(DB**)db)->Get(*(ReadOptions*)options, Slice(key, keylen), tmp); +// //very tricky, maybe changed with c++ leveldb upgrade +// Status s = (*(DB**)db)->Get(*(ReadOptions*)options, Slice(key, keylen), tmp); - if (s.ok()) { - *valptr = (char*)tmp->data(); - *vallen = tmp->size(); - } else { - delete(tmp); - tmp = NULL; - *valptr = NULL; - *vallen = 0; - if (!s.IsNotFound()) { - SaveError(errptr, s); - } - } - return tmp; -} +// if (s.ok()) { +// *valptr = (char*)tmp->data(); +// *vallen = tmp->size(); +// } else { +// delete(tmp); +// tmp = NULL; +// *valptr = NULL; +// *vallen = 0; +// if (!s.IsNotFound()) { +// SaveError(errptr, s); +// } +// } +// return tmp; +// } -void leveldb_get_free_ext(void* context) { - std::string* s = (std::string*)context; +// void leveldb_get_free_ext(void* context) { +// std::string* s = (std::string*)context; - delete(s); -} +// delete(s); +// } unsigned char leveldb_iter_seek_to_first_ext(leveldb_iterator_t* iter) { diff --git a/store/leveldb/leveldb_ext.h b/store/leveldb/leveldb_ext.h index 8222ae3..1c5f986 100644 --- a/store/leveldb/leveldb_ext.h +++ b/store/leveldb/leveldb_ext.h @@ -10,19 +10,19 @@ extern "C" { #include "leveldb/c.h" -/* Returns NULL if not found. Otherwise stores the value in **valptr. - Stores the length of the value in *vallen. - Returns a context must be later to free*/ -extern void* leveldb_get_ext( - leveldb_t* db, - const leveldb_readoptions_t* options, - const char* key, size_t keylen, - char** valptr, - size_t* vallen, - char** errptr); +// /* Returns NULL if not found. Otherwise stores the value in **valptr. +// Stores the length of the value in *vallen. +// Returns a context must be later to free*/ +// extern void* leveldb_get_ext( +// leveldb_t* db, +// const leveldb_readoptions_t* options, +// const char* key, size_t keylen, +// char** valptr, +// size_t* vallen, +// char** errptr); -// Free context returns by leveldb_get_ext -extern void leveldb_get_free_ext(void* context); +// // Free context returns by leveldb_get_ext +// extern void leveldb_get_free_ext(void* context); // Below iterator functions like leveldb iterator but returns valid status for iterator diff --git a/store/snapshot.go b/store/snapshot.go new file mode 100644 index 0000000..3f7538a --- /dev/null +++ b/store/snapshot.go @@ -0,0 +1,16 @@ +package store + +import ( + "github.com/siddontang/ledisdb/store/driver" +) + +type Snapshot struct { + driver.ISnapshot +} + +func (s *Snapshot) NewIterator() *Iterator { + it := new(Iterator) + it.it = s.ISnapshot.NewIterator() + + return it +} From 882e20a3e31a357e94ad02fbfac3c3ef858818aa Mon Sep 17 00:00:00 2001 From: siddontang Date: Sat, 30 Aug 2014 17:39:44 +0800 Subject: [PATCH 05/32] refactor ledis lock and binlog --- ledis/binlog.go | 29 ++++++++++-- ledis/binlog_util.go | 9 ---- ledis/dump.go | 22 +++++---- ledis/ledis.go | 7 ++- ledis/ledis_db.go | 4 -- ledis/ledis_test.go | 4 +- ledis/replication.go | 110 ++++++++++++++++++++++++++++--------------- ledis/tx.go | 95 ++++++++++++++++++------------------- 8 files changed, 161 insertions(+), 119 deletions(-) diff --git a/ledis/binlog.go b/ledis/binlog.go index 087c13f..3bdf50a 100644 --- a/ledis/binlog.go +++ b/ledis/binlog.go @@ -11,6 +11,7 @@ import ( "path" "strconv" "strings" + "sync" "time" ) @@ -27,6 +28,8 @@ timestamp(bigendian uint32, seconds)|PayloadLen(bigendian uint32)|PayloadData */ type BinLog struct { + sync.Mutex + path string cfg *config.BinLogConfig @@ -177,16 +180,20 @@ func (l *BinLog) checkLogFileSize() bool { st, _ := l.logFile.Stat() if st.Size() >= int64(l.cfg.MaxFileSize) { - l.lastLogIndex++ - - l.logFile.Close() - l.logFile = nil + l.closeLog() return true } return false } +func (l *BinLog) closeLog() { + l.lastLogIndex++ + + l.logFile.Close() + l.logFile = nil +} + func (l *BinLog) purge(n int) { for i := 0; i < n; i++ { logPath := path.Join(l.path, l.logNames[i]) @@ -238,6 +245,9 @@ func (l *BinLog) LogPath() string { } func (l *BinLog) Purge(n int) error { + l.Lock() + defer l.Unlock() + if len(l.logNames) == 0 { return nil } @@ -255,7 +265,18 @@ func (l *BinLog) Purge(n int) error { return l.flushIndex() } +func (l *BinLog) PurgeAll() error { + l.Lock() + defer l.Unlock() + + l.closeLog() + return l.openNewLogFile() +} + func (l *BinLog) Log(args ...[]byte) error { + l.Lock() + defer l.Unlock() + var err error if l.logFile == nil { diff --git a/ledis/binlog_util.go b/ledis/binlog_util.go index 5167b40..da058bd 100644 --- a/ledis/binlog_util.go +++ b/ledis/binlog_util.go @@ -54,15 +54,6 @@ func decodeBinLogPut(sz []byte) ([]byte, []byte, error) { return sz[3 : 3+keyLen], sz[3+keyLen:], nil } -func encodeBinLogCommand(commandType uint8, args ...[]byte) []byte { - //to do - return nil -} - -func decodeBinLogCommand(sz []byte) (uint8, [][]byte, error) { - return 0, nil, errBinLogCommandType -} - func FormatBinLogEvent(event []byte) (string, error) { logType := uint8(event[0]) diff --git a/ledis/dump.go b/ledis/dump.go index 14d7ff7..67908a8 100644 --- a/ledis/dump.go +++ b/ledis/dump.go @@ -57,16 +57,17 @@ func (l *Ledis) DumpFile(path string) error { func (l *Ledis) Dump(w io.Writer) error { var m *MasterInfo = new(MasterInfo) - l.Lock() - defer l.Unlock() + + var err error + + l.wLock.Lock() + defer l.wLock.Unlock() if l.binlog != nil { m.LogFileIndex = l.binlog.LogFileIndex() m.LogPos = l.binlog.LogFilePos() } - var err error - wb := bufio.NewWriterSize(w, 4096) if err = m.WriteTo(wb); err != nil { return err @@ -128,8 +129,8 @@ func (l *Ledis) LoadDumpFile(path string) (*MasterInfo, error) { } func (l *Ledis) LoadDump(r io.Reader) (*MasterInfo, error) { - l.Lock() - defer l.Unlock() + l.wLock.Lock() + defer l.wLock.Unlock() info := new(MasterInfo) @@ -182,10 +183,6 @@ func (l *Ledis) LoadDump(r io.Reader) (*MasterInfo, error) { return nil, err } - if l.binlog != nil { - err = l.binlog.Log(encodeBinLogPut(key, value)) - } - keyBuf.Reset() valueBuf.Reset() } @@ -193,5 +190,10 @@ func (l *Ledis) LoadDump(r io.Reader) (*MasterInfo, error) { deKeyBuf = nil deValueBuf = nil + //if binlog enable, we will delete all binlogs and open a new one for handling simply + if l.binlog != nil { + l.binlog.PurgeAll() + } + return info, nil } diff --git a/ledis/ledis.go b/ledis/ledis.go index 70d22d1..f3c1c8c 100644 --- a/ledis/ledis.go +++ b/ledis/ledis.go @@ -10,8 +10,6 @@ import ( ) type Ledis struct { - sync.Mutex - cfg *config.Config ldb *store.DB @@ -21,11 +19,13 @@ type Ledis struct { jobs *sync.WaitGroup binlog *BinLog + + wLock sync.RWMutex //allow one write at same time + commitLock sync.Mutex //allow one write commit at same time } func Open(cfg *config.Config) (*Ledis, error) { if len(cfg.DataDir) == 0 { - fmt.Printf("no datadir set, use default %s\n", config.DefaultDataDir) cfg.DataDir = config.DefaultDataDir } @@ -42,7 +42,6 @@ func Open(cfg *config.Config) (*Ledis, error) { l.ldb = ldb if cfg.BinLog.MaxFileNum > 0 && cfg.BinLog.MaxFileSize > 0 { - println("binlog will be refactored later, use your own risk!!!") l.binlog, err = NewBinLog(cfg) if err != nil { return nil, err diff --git a/ledis/ledis_db.go b/ledis/ledis_db.go index c774d03..9241b1d 100644 --- a/ledis/ledis_db.go +++ b/ledis/ledis_db.go @@ -3,7 +3,6 @@ package ledis import ( "fmt" "github.com/siddontang/ledisdb/store" - "sync" ) type ibucket interface { @@ -29,8 +28,6 @@ type DB struct { bucket ibucket - dbLock *sync.RWMutex - index uint8 kvBatch *batch @@ -54,7 +51,6 @@ func (l *Ledis) newDB(index uint8) *DB { d.isTx = false d.index = index - d.dbLock = &sync.RWMutex{} d.kvBatch = d.newBatch() d.listBatch = d.newBatch() diff --git a/ledis/ledis_test.go b/ledis/ledis_test.go index aff4ebe..d5a5476 100644 --- a/ledis/ledis_test.go +++ b/ledis/ledis_test.go @@ -14,8 +14,8 @@ func getTestDB() *DB { f := func() { cfg := new(config.Config) cfg.DataDir = "/tmp/test_ledis" - cfg.BinLog.MaxFileSize = 1073741824 - cfg.BinLog.MaxFileNum = 3 + // cfg.BinLog.MaxFileSize = 1073741824 + // cfg.BinLog.MaxFileNum = 3 os.RemoveAll(cfg.DataDir) diff --git a/ledis/replication.go b/ledis/replication.go index bd6c192..2b19cfe 100644 --- a/ledis/replication.go +++ b/ledis/replication.go @@ -6,6 +6,7 @@ import ( "encoding/binary" "errors" "github.com/siddontang/go-log/log" + "github.com/siddontang/ledisdb/store/driver" "io" "os" ) @@ -19,7 +20,41 @@ var ( errInvalidBinLogFile = errors.New("invalid binlog file") ) -func (l *Ledis) ReplicateEvent(event []byte) error { +type replBatch struct { + wb driver.IWriteBatch + events [][]byte + createTime uint32 + l *Ledis +} + +func (b *replBatch) Commit() error { + b.l.commitLock.Lock() + defer b.l.commitLock.Unlock() + + err := b.wb.Commit() + if err != nil { + b.Rollback() + return err + } + + if b.l.binlog != nil { + if err = b.l.binlog.Log(b.events...); err != nil { + b.Rollback() + return err + } + } + + return nil +} + +func (b *replBatch) Rollback() error { + b.wb.Rollback() + b.events = [][]byte{} + b.createTime = 0 + return nil +} + +func (l *Ledis) replicateEvent(b *replBatch, event []byte) error { if len(event) == 0 { return errInvalidBinLogEvent } @@ -27,52 +62,42 @@ func (l *Ledis) ReplicateEvent(event []byte) error { logType := uint8(event[0]) switch logType { case BinLogTypePut: - return l.replicatePutEvent(event) + return l.replicatePutEvent(b, event) case BinLogTypeDeletion: - return l.replicateDeleteEvent(event) - case BinLogTypeCommand: - return l.replicateCommandEvent(event) + return l.replicateDeleteEvent(b, event) default: return errInvalidBinLogEvent } } -func (l *Ledis) replicatePutEvent(event []byte) error { +func (l *Ledis) replicatePutEvent(b *replBatch, event []byte) error { key, value, err := decodeBinLogPut(event) if err != nil { return err } - if err = l.ldb.Put(key, value); err != nil { - return err + b.wb.Put(key, value) + + if b.l.binlog != nil { + b.events = append(b.events, event) } - if l.binlog != nil { - err = l.binlog.Log(event) - } - - return err + return nil } -func (l *Ledis) replicateDeleteEvent(event []byte) error { +func (l *Ledis) replicateDeleteEvent(b *replBatch, event []byte) error { key, err := decodeBinLogDelete(event) if err != nil { return err } - if err = l.ldb.Delete(key); err != nil { - return err + b.wb.Delete(key) + + if b.l.binlog != nil { + b.events = append(b.events, event) } - if l.binlog != nil { - err = l.binlog.Log(event) - } - - return err -} - -func (l *Ledis) replicateCommandEvent(event []byte) error { - return errors.New("command event not supported now") + return nil } func ReadEventFromReader(rb io.Reader, f func(createTime uint32, event []byte) error) error { @@ -110,8 +135,23 @@ func ReadEventFromReader(rb io.Reader, f func(createTime uint32, event []byte) e } func (l *Ledis) ReplicateFromReader(rb io.Reader) error { + b := new(replBatch) + + b.wb = l.ldb.NewWriteBatch() + b.l = l + f := func(createTime uint32, event []byte) error { - err := l.ReplicateEvent(event) + if b.createTime == 0 { + b.createTime = createTime + } else if b.createTime != createTime { + if err := b.Commit(); err != nil { + log.Fatal("replication error %s, skip to next", err.Error()) + return ErrSkipEvent + } + b.createTime = createTime + } + + err := l.replicateEvent(b, event) if err != nil { log.Fatal("replication error %s, skip to next", err.Error()) return ErrSkipEvent @@ -119,15 +159,18 @@ func (l *Ledis) ReplicateFromReader(rb io.Reader) error { return nil } - return ReadEventFromReader(rb, f) + err := ReadEventFromReader(rb, f) + if err != nil { + b.Rollback() + return err + } + return b.Commit() } func (l *Ledis) ReplicateFromData(data []byte) error { rb := bytes.NewReader(data) - l.Lock() err := l.ReplicateFromReader(rb) - l.Unlock() return err } @@ -140,17 +183,13 @@ func (l *Ledis) ReplicateFromBinLog(filePath string) error { rb := bufio.NewReaderSize(f, 4096) - l.Lock() err = l.ReplicateFromReader(rb) - l.Unlock() f.Close() return err } -const maxSyncEvents = 64 - func (l *Ledis) ReadEventsTo(info *MasterInfo, w io.Writer) (n int, err error) { n = 0 if l.binlog == nil { @@ -205,8 +244,6 @@ func (l *Ledis) ReadEventsTo(info *MasterInfo, w io.Writer) (n int, err error) { var createTime uint32 var dataLen uint32 - var eventsNum int = 0 - for { if err = binary.Read(f, binary.BigEndian, &createTime); err != nil { if err == io.EOF { @@ -222,13 +259,10 @@ func (l *Ledis) ReadEventsTo(info *MasterInfo, w io.Writer) (n int, err error) { } } - eventsNum++ if lastCreateTime == 0 { lastCreateTime = createTime } else if lastCreateTime != createTime { return - } else if eventsNum > maxSyncEvents { - return } if err = binary.Read(f, binary.BigEndian, &dataLen); err != nil { diff --git a/ledis/tx.go b/ledis/tx.go index 38eb626..7488233 100644 --- a/ledis/tx.go +++ b/ledis/tx.go @@ -24,42 +24,49 @@ type batch struct { } type dbBatchLocker struct { - sync.Mutex - dbLock *sync.RWMutex + l *sync.Mutex + wrLock *sync.RWMutex +} + +func (l *dbBatchLocker) Lock() { + l.wrLock.RLock() + l.l.Lock() +} + +func (l *dbBatchLocker) Unlock() { + l.l.Unlock() + l.wrLock.RUnlock() } type txBatchLocker struct { } -func (l *txBatchLocker) Lock() { -} +func (l *txBatchLocker) Lock() {} +func (l *txBatchLocker) Unlock() {} -func (l *txBatchLocker) Unlock() { -} - -func (l *dbBatchLocker) Lock() { - l.dbLock.RLock() - l.Mutex.Lock() -} - -func (l *dbBatchLocker) Unlock() { - l.Mutex.Unlock() - l.dbLock.RUnlock() -} - -func (db *DB) newBatch() *batch { +func (l *Ledis) newBatch(wb store.WriteBatch, tx *Tx) *batch { b := new(batch) + b.l = l + b.WriteBatch = wb - b.WriteBatch = db.bucket.NewWriteBatch() - b.Locker = &dbBatchLocker{dbLock: db.dbLock} - b.l = db.l + b.tx = tx + if tx == nil { + b.Locker = &dbBatchLocker{l: &sync.Mutex{}, wrLock: &l.wLock} + } else { + b.Locker = &txBatchLocker{} + } + b.logs = [][]byte{} return b } +func (db *DB) newBatch() *batch { + return db.l.newBatch(db.bucket.NewWriteBatch(), nil) +} + func (b *batch) Commit() error { - b.l.Lock() - defer b.l.Unlock() + b.l.commitLock.Lock() + defer b.l.commitLock.Unlock() err := b.WriteBatch.Commit() @@ -85,7 +92,7 @@ func (b *batch) Unlock() { if b.l.binlog != nil { b.logs = [][]byte{} } - b.Rollback() + b.WriteBatch.Rollback() b.Locker.Unlock() } @@ -129,11 +136,10 @@ func (db *DB) Begin() (*Tx, error) { tx := new(Tx) tx.DB = new(DB) - tx.DB.dbLock = db.dbLock - - tx.DB.dbLock.Lock() - tx.DB.l = db.l + + tx.l.wLock.Lock() + tx.index = db.index tx.DB.sdb = db.sdb @@ -141,7 +147,7 @@ func (db *DB) Begin() (*Tx, error) { var err error tx.tx, err = db.sdb.Begin() if err != nil { - tx.DB.dbLock.Unlock() + tx.l.wLock.Unlock() return nil, err } @@ -151,12 +157,12 @@ func (db *DB) Begin() (*Tx, error) { tx.DB.index = db.index - tx.DB.kvBatch = tx.newBatch() - tx.DB.listBatch = tx.newBatch() - tx.DB.hashBatch = tx.newBatch() - tx.DB.zsetBatch = tx.newBatch() - tx.DB.binBatch = tx.newBatch() - tx.DB.setBatch = tx.newBatch() + tx.DB.kvBatch = tx.newTxBatch() + tx.DB.listBatch = tx.newTxBatch() + tx.DB.hashBatch = tx.newTxBatch() + tx.DB.zsetBatch = tx.newTxBatch() + tx.DB.binBatch = tx.newTxBatch() + tx.DB.setBatch = tx.newTxBatch() return tx, nil } @@ -166,7 +172,7 @@ func (tx *Tx) Commit() error { return ErrTxDone } - tx.l.Lock() + tx.l.commitLock.Lock() err := tx.tx.Commit() tx.tx = nil @@ -174,9 +180,9 @@ func (tx *Tx) Commit() error { tx.l.binlog.Log(tx.logs...) } - tx.l.Unlock() + tx.l.commitLock.Unlock() - tx.DB.dbLock.Unlock() + tx.l.wLock.Unlock() tx.DB = nil return err } @@ -189,20 +195,13 @@ func (tx *Tx) Rollback() error { err := tx.tx.Rollback() tx.tx = nil - tx.DB.dbLock.Unlock() + tx.l.wLock.Unlock() tx.DB = nil return err } -func (tx *Tx) newBatch() *batch { - b := new(batch) - - b.l = tx.l - b.WriteBatch = tx.tx.NewWriteBatch() - b.Locker = &txBatchLocker{} - b.tx = tx - - return b +func (tx *Tx) newTxBatch() *batch { + return tx.l.newBatch(tx.tx.NewWriteBatch(), tx) } func (tx *Tx) Index() int { From d94f4629c2b35ea4b97e86a554cb324497ac3cba Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 1 Sep 2014 08:33:35 +0800 Subject: [PATCH 06/32] bitmap unlock bugfix --- ledis/t_bit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ledis/t_bit.go b/ledis/t_bit.go index fadab4d..496c37a 100644 --- a/ledis/t_bit.go +++ b/ledis/t_bit.go @@ -506,6 +506,7 @@ func (db *DB) BSetBit(key []byte, offset int32, val uint8) (ori uint8, err error if setBit(segment, off, val) { t := db.binBatch t.Lock() + defer t.Unlock() t.Put(bk, segment) if _, _, e := db.bUpdateMeta(t, key, seq, off); e != nil { @@ -514,7 +515,6 @@ func (db *DB) BSetBit(key []byte, offset int32, val uint8) (ori uint8, err error } err = t.Commit() - t.Unlock() } } From 356c2eea1787d0c6635e29903c5b165d9bbb862b Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 1 Sep 2014 08:34:03 +0800 Subject: [PATCH 07/32] dump use raw key value --- ledis/dump.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ledis/dump.go b/ledis/dump.go index 67908a8..63c1d58 100644 --- a/ledis/dump.go +++ b/ledis/dump.go @@ -81,8 +81,8 @@ func (l *Ledis) Dump(w io.Writer) error { var key []byte var value []byte for ; it.Valid(); it.Next() { - key = it.Key() - value = it.Value() + key = it.RawKey() + value = it.RawValue() if key, err = snappy.Encode(compressBuf, key); err != nil { return err From cfa6d86acefc445d37e557e05ba0368c3e93f241 Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 1 Sep 2014 10:16:20 +0800 Subject: [PATCH 08/32] refactor binlog can not compatibility --- cmd/ledis-binlog/main.go | 6 +-- ledis/binlog.go | 84 ++++++++++++++++++++++++++++++++++----- ledis/replication.go | 86 ++++++++++++++++++++-------------------- 3 files changed, 120 insertions(+), 56 deletions(-) diff --git a/cmd/ledis-binlog/main.go b/cmd/ledis-binlog/main.go index 7212b0e..3725920 100644 --- a/cmd/ledis-binlog/main.go +++ b/cmd/ledis-binlog/main.go @@ -63,12 +63,12 @@ func main() { } } -func printEvent(createTime uint32, event []byte) error { - if createTime < startTime || createTime > stopTime { +func printEvent(head *ledis.BinLogHead, event []byte) error { + if head.CreateTime < startTime || head.CreateTime > stopTime { return nil } - t := time.Unix(int64(createTime), 0) + t := time.Unix(int64(head.CreateTime), 0) fmt.Printf("%s ", t.Format(TimeFormat)) diff --git a/ledis/binlog.go b/ledis/binlog.go index 3bdf50a..6eb0c30 100644 --- a/ledis/binlog.go +++ b/ledis/binlog.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/siddontang/go-log/log" "github.com/siddontang/ledisdb/config" + "io" "io/ioutil" "os" "path" @@ -15,6 +16,65 @@ import ( "time" ) +type BinLogHead struct { + CreateTime uint32 + BatchId uint32 + PayloadLen uint32 +} + +func (h *BinLogHead) Len() int { + return 12 +} + +func (h *BinLogHead) Write(w io.Writer) error { + if err := binary.Write(w, binary.BigEndian, h.CreateTime); err != nil { + return err + } + + if err := binary.Write(w, binary.BigEndian, h.BatchId); err != nil { + return err + } + + if err := binary.Write(w, binary.BigEndian, h.PayloadLen); err != nil { + return err + } + + return nil +} + +func (h *BinLogHead) handleReadError(err error) error { + if err == io.EOF { + return io.ErrUnexpectedEOF + } else { + return err + } +} + +func (h *BinLogHead) Read(r io.Reader) error { + var err error + if err = binary.Read(r, binary.BigEndian, &h.CreateTime); err != nil { + return err + } + + if err = binary.Read(r, binary.BigEndian, &h.BatchId); err != nil { + return h.handleReadError(err) + } + + if err = binary.Read(r, binary.BigEndian, &h.PayloadLen); err != nil { + return h.handleReadError(err) + } + + return nil +} + +func (h *BinLogHead) InSameBatch(ho *BinLogHead) bool { + if h.CreateTime == ho.CreateTime && h.BatchId == ho.BatchId { + return true + } else { + return false + } +} + /* index file format: ledis-bin.00001 @@ -23,7 +83,9 @@ ledis-bin.00003 log file format -timestamp(bigendian uint32, seconds)|PayloadLen(bigendian uint32)|PayloadData +Log: Head|PayloadData + +Head: createTime|batchId|payloadData */ @@ -41,6 +103,8 @@ type BinLog struct { indexName string logNames []string lastLogIndex int64 + + batchId uint32 } func NewBinLog(cfg *config.Config) (*BinLog, error) { @@ -49,7 +113,7 @@ func NewBinLog(cfg *config.Config) (*BinLog, error) { l.cfg = &cfg.BinLog l.cfg.Adjust() - l.path = path.Join(cfg.DataDir, "bin_log") + l.path = path.Join(cfg.DataDir, "binlog") if err := os.MkdirAll(l.path, os.ModePerm); err != nil { return nil, err @@ -285,17 +349,17 @@ func (l *BinLog) Log(args ...[]byte) error { } } - //we treat log many args as a batch, so use same createTime - createTime := uint32(time.Now().Unix()) + head := &BinLogHead{} + + head.CreateTime = uint32(time.Now().Unix()) + head.BatchId = l.batchId + + l.batchId++ for _, data := range args { - payLoadLen := uint32(len(data)) + head.PayloadLen = uint32(len(data)) - if err := binary.Write(l.logWb, binary.BigEndian, createTime); err != nil { - return err - } - - if err := binary.Write(l.logWb, binary.BigEndian, payLoadLen); err != nil { + if err := head.Write(l.logWb); err != nil { return err } diff --git a/ledis/replication.go b/ledis/replication.go index 2b19cfe..2d8db48 100644 --- a/ledis/replication.go +++ b/ledis/replication.go @@ -3,7 +3,6 @@ package ledis import ( "bufio" "bytes" - "encoding/binary" "errors" "github.com/siddontang/go-log/log" "github.com/siddontang/ledisdb/store/driver" @@ -11,6 +10,11 @@ import ( "os" ) +const ( + maxReplBatchNum = 100 + maxReplLogSize = 1 * 1024 * 1024 +) + var ( ErrSkipEvent = errors.New("skip to next event") ) @@ -21,10 +25,11 @@ var ( ) type replBatch struct { - wb driver.IWriteBatch - events [][]byte - createTime uint32 - l *Ledis + wb driver.IWriteBatch + events [][]byte + l *Ledis + + lastHead *BinLogHead } func (b *replBatch) Commit() error { @@ -50,7 +55,7 @@ func (b *replBatch) Commit() error { func (b *replBatch) Rollback() error { b.wb.Rollback() b.events = [][]byte{} - b.createTime = 0 + b.lastHead = nil return nil } @@ -100,14 +105,13 @@ func (l *Ledis) replicateDeleteEvent(b *replBatch, event []byte) error { return nil } -func ReadEventFromReader(rb io.Reader, f func(createTime uint32, event []byte) error) error { - var createTime uint32 - var dataLen uint32 +func ReadEventFromReader(rb io.Reader, f func(head *BinLogHead, event []byte) error) error { + head := &BinLogHead{} var dataBuf bytes.Buffer var err error for { - if err = binary.Read(rb, binary.BigEndian, &createTime); err != nil { + if err = head.Read(rb); err != nil { if err == io.EOF { break } else { @@ -115,15 +119,11 @@ func ReadEventFromReader(rb io.Reader, f func(createTime uint32, event []byte) e } } - if err = binary.Read(rb, binary.BigEndian, &dataLen); err != nil { + if _, err = io.CopyN(&dataBuf, rb, int64(head.PayloadLen)); err != nil { return err } - if _, err = io.CopyN(&dataBuf, rb, int64(dataLen)); err != nil { - return err - } - - err = f(createTime, dataBuf.Bytes()) + err = f(head, dataBuf.Bytes()) if err != nil && err != ErrSkipEvent { return err } @@ -140,15 +140,15 @@ func (l *Ledis) ReplicateFromReader(rb io.Reader) error { b.wb = l.ldb.NewWriteBatch() b.l = l - f := func(createTime uint32, event []byte) error { - if b.createTime == 0 { - b.createTime = createTime - } else if b.createTime != createTime { + f := func(head *BinLogHead, event []byte) error { + if b.lastHead == nil { + b.lastHead = head + } else if !b.lastHead.InSameBatch(head) { if err := b.Commit(); err != nil { log.Fatal("replication error %s, skip to next", err.Error()) return ErrSkipEvent } - b.createTime = createTime + b.lastHead = head } err := l.replicateEvent(b, event) @@ -240,12 +240,14 @@ func (l *Ledis) ReadEventsTo(info *MasterInfo, w io.Writer) (n int, err error) { return } - var lastCreateTime uint32 = 0 - var createTime uint32 - var dataLen uint32 + var lastHead *BinLogHead = nil + + head := &BinLogHead{} + + batchNum := 0 for { - if err = binary.Read(f, binary.BigEndian, &createTime); err != nil { + if err = head.Read(f); err != nil { if err == io.EOF { //we will try to use next binlog if index < l.binlog.LogFileIndex() { @@ -257,32 +259,30 @@ func (l *Ledis) ReadEventsTo(info *MasterInfo, w io.Writer) (n int, err error) { } else { return } + } - if lastCreateTime == 0 { - lastCreateTime = createTime - } else if lastCreateTime != createTime { + if lastHead == nil { + lastHead = head + batchNum++ + } else if !lastHead.InSameBatch(head) { + lastHead = head + batchNum++ + if batchNum > maxReplBatchNum || n > maxReplLogSize { + return + } + } + + if err = head.Write(w); err != nil { return } - if err = binary.Read(f, binary.BigEndian, &dataLen); err != nil { + if _, err = io.CopyN(w, f, int64(head.PayloadLen)); err != nil { return } - if err = binary.Write(w, binary.BigEndian, createTime); err != nil { - return - } - - if err = binary.Write(w, binary.BigEndian, dataLen); err != nil { - return - } - - if _, err = io.CopyN(w, f, int64(dataLen)); err != nil { - return - } - - n += (8 + int(dataLen)) - info.LogPos = info.LogPos + 8 + int64(dataLen) + n += (head.Len() + int(head.PayloadLen)) + info.LogPos = info.LogPos + int64(head.Len()) + int64(head.PayloadLen) } return From 03c8e4c24c04994903e746f4752f57255831c902 Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 1 Sep 2014 17:45:47 +0800 Subject: [PATCH 09/32] store add memory support based on goleveldb --- store/goleveldb/const.go | 1 + store/goleveldb/db.go | 44 +++++++++++++++++++++++++++++++++------- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/store/goleveldb/const.go b/store/goleveldb/const.go index 6486e3f..2fffa7c 100644 --- a/store/goleveldb/const.go +++ b/store/goleveldb/const.go @@ -1,3 +1,4 @@ package goleveldb const DBName = "goleveldb" +const MemDBName = "memory" diff --git a/store/goleveldb/db.go b/store/goleveldb/db.go index ca2de60..e873feb 100644 --- a/store/goleveldb/db.go +++ b/store/goleveldb/db.go @@ -5,6 +5,8 @@ import ( "github.com/siddontang/goleveldb/leveldb/cache" "github.com/siddontang/goleveldb/leveldb/filter" "github.com/siddontang/goleveldb/leveldb/opt" + "github.com/siddontang/goleveldb/leveldb/storage" + "github.com/siddontang/ledisdb/config" "github.com/siddontang/ledisdb/store/driver" @@ -20,6 +22,13 @@ func (s Store) String() string { return DBName } +type MemStore struct { +} + +func (s MemStore) String() string { + return MemDBName +} + type DB struct { path string @@ -45,7 +54,12 @@ func (s Store) Open(path string, cfg *config.Config) (driver.IDB, error) { db.path = path db.cfg = &cfg.LevelDB - if err := db.open(); err != nil { + db.initOpts() + + var err error + db.db, err = leveldb.OpenFile(db.path, db.opts) + + if err != nil { return nil, err } @@ -62,16 +76,31 @@ func (s Store) Repair(path string, cfg *config.Config) error { return nil } -func (db *DB) open() error { +func (s MemStore) Open(path string, cfg *config.Config) (driver.IDB, error) { + db := new(DB) + db.path = path + db.cfg = &cfg.LevelDB + + db.initOpts() + + var err error + db.db, err = leveldb.Open(storage.NewMemStorage(), db.opts) + if err != nil { + return nil, err + } + + return db, nil +} + +func (s MemStore) Repair(path string, cfg *config.Config) error { + return nil +} + +func (db *DB) initOpts() { db.opts = newOptions(db.cfg) db.iteratorOpts = &opt.ReadOptions{} db.iteratorOpts.DontFillCache = true - - var err error - db.db, err = leveldb.OpenFile(db.path, db.opts) - - return err } func newOptions(cfg *config.LevelDBConfig) *opt.Options { @@ -153,4 +182,5 @@ func (db *DB) NewSnapshot() (driver.ISnapshot, error) { func init() { driver.Register(Store{}) + driver.Register(MemStore{}) } From 93f3cc534346fe64ecb8e30a77cb3fc6d0b91ca9 Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 1 Sep 2014 23:26:35 +0800 Subject: [PATCH 10/32] lua resp writer --- bootstrap.sh | 2 + server/cmd_script.go | 158 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 server/cmd_script.go diff --git a/bootstrap.sh b/bootstrap.sh index ee260b7..a0eae73 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -15,3 +15,5 @@ go get github.com/ugorji/go/codec go get github.com/BurntSushi/toml go get github.com/siddontang/go-bson/bson + +go get github.com/aarzilli/golua/lua \ No newline at end of file diff --git a/server/cmd_script.go b/server/cmd_script.go new file mode 100644 index 0000000..20f5b63 --- /dev/null +++ b/server/cmd_script.go @@ -0,0 +1,158 @@ +package server + +import ( + "fmt" + "github.com/aarzilli/golua/lua" + "github.com/siddontang/ledisdb/ledis" + "io" +) + +//ledis <-> lua type conversion, same as http://redis.io/commands/eval + +type luaClient struct { + l *lua.State +} + +type luaWriter struct { + l *lua.State +} + +func (w *luaWriter) writeError(err error) { + w.l.NewTable() + top := w.l.GetTop() + + w.l.PushString("err") + w.l.PushString(err.Error()) + w.l.SetTable(top) +} + +func (w *luaWriter) writeStatus(status string) { + w.l.NewTable() + top := w.l.GetTop() + + w.l.PushString("ok") + w.l.PushString(status) + w.l.SetTable(top) +} + +func (w *luaWriter) writeInteger(n int64) { + w.l.PushInteger(n) +} + +func (w *luaWriter) writeBulk(b []byte) { + if b == nil { + w.l.PushBoolean(false) + } else { + w.l.PushString(ledis.String(b)) + } +} + +func (w *luaWriter) writeArray(lst []interface{}) { + if lst == nil { + w.l.PushBoolean(false) + return + } + + base := w.l.GetTop() + defer func() { + if e := recover(); e != nil { + w.l.SetTop(base) + w.writeError(fmt.Errorf("%v", e)) + } + }() + + w.l.CreateTable(len(lst), 0) + top := w.l.GetTop() + + for i, _ := range lst { + w.l.PushInteger(int64(i) + 1) + + switch v := lst[i].(type) { + case []interface{}: + w.writeArray(v) + case [][]byte: + w.writeSliceArray(v) + case []byte: + w.writeBulk(v) + case nil: + w.writeBulk(nil) + case int64: + w.writeInteger(v) + default: + panic("invalid array type") + } + + w.l.SetTable(top) + } +} + +func (w *luaWriter) writeSliceArray(lst [][]byte) { + if lst == nil { + w.l.PushBoolean(false) + return + } + + w.l.CreateTable(len(lst), 0) + top := w.l.GetTop() + for i, v := range lst { + w.l.PushInteger(int64(i) + 1) + w.l.PushString(ledis.String(v)) + w.l.SetTable(top) + } +} + +func (w *luaWriter) writeFVPairArray(lst []ledis.FVPair) { + if lst == nil { + w.l.PushBoolean(false) + return + } + + w.l.CreateTable(len(lst)*2, 0) + top := w.l.GetTop() + for i, v := range lst { + w.l.PushInteger(int64(2*i) + 1) + w.l.PushString(ledis.String(v.Field)) + w.l.SetTable(top) + + w.l.PushInteger(int64(2*i) + 2) + w.l.PushString(ledis.String(v.Value)) + w.l.SetTable(top) + } +} + +func (w *luaWriter) writeScorePairArray(lst []ledis.ScorePair, withScores bool) { + if lst == nil { + w.l.PushBoolean(false) + return + } + + if withScores { + w.l.CreateTable(len(lst)*2, 0) + top := w.l.GetTop() + for i, v := range lst { + w.l.PushInteger(int64(2*i) + 1) + w.l.PushString(ledis.String(v.Member)) + w.l.SetTable(top) + + w.l.PushInteger(int64(2*i) + 2) + w.l.PushString(ledis.String(ledis.StrPutInt64(v.Score))) + w.l.SetTable(top) + } + } else { + w.l.CreateTable(len(lst), 0) + top := w.l.GetTop() + for i, v := range lst { + w.l.PushInteger(int64(i) + 1) + w.l.PushString(ledis.String(v.Member)) + w.l.SetTable(top) + } + } +} + +func (w *luaWriter) writeBulkFrom(n int64, rb io.Reader) { + w.writeError(fmt.Errorf("unsupport")) +} + +func (w *luaWriter) flush() { + +} From ab1ae62bf709effe14a91d13097dd9b11f6cd894 Mon Sep 17 00:00:00 2001 From: siddontang Date: Tue, 2 Sep 2014 17:55:12 +0800 Subject: [PATCH 11/32] add lua support --- ledis/batch.go | 105 +++++++++++ ledis/const.go | 6 + ledis/ledis_db.go | 13 +- ledis/multi.go | 73 ++++++++ ledis/multi_test.go | 51 +++++ ledis/replication_test.go | 6 + ledis/tx.go | 209 --------------------- ledis/tx_test.go | 46 +++++ server/app.go | 6 + server/client.go | 49 ++++- server/cmd_script.go | 317 +++++++++++++++++-------------- server/cmd_script_test.go | 29 +++ server/command.go | 22 ++- server/script.go | 380 ++++++++++++++++++++++++++++++++++++++ server/script_test.go | 177 ++++++++++++++++++ 15 files changed, 1132 insertions(+), 357 deletions(-) create mode 100644 ledis/batch.go create mode 100644 ledis/multi.go create mode 100644 ledis/multi_test.go delete mode 100644 ledis/tx.go create mode 100644 server/cmd_script_test.go create mode 100644 server/script.go create mode 100644 server/script_test.go diff --git a/ledis/batch.go b/ledis/batch.go new file mode 100644 index 0000000..b23cc47 --- /dev/null +++ b/ledis/batch.go @@ -0,0 +1,105 @@ +package ledis + +import ( + "github.com/siddontang/ledisdb/store" + "sync" +) + +type batch struct { + l *Ledis + + store.WriteBatch + + sync.Locker + + logs [][]byte + + tx *Tx +} + +func (b *batch) Commit() error { + b.l.commitLock.Lock() + defer b.l.commitLock.Unlock() + + err := b.WriteBatch.Commit() + + if b.l.binlog != nil { + if err == nil { + if b.tx == nil { + b.l.binlog.Log(b.logs...) + } else { + b.tx.logs = append(b.tx.logs, b.logs...) + } + } + b.logs = [][]byte{} + } + + return err +} + +func (b *batch) Lock() { + b.Locker.Lock() +} + +func (b *batch) Unlock() { + if b.l.binlog != nil { + b.logs = [][]byte{} + } + b.WriteBatch.Rollback() + b.Locker.Unlock() +} + +func (b *batch) Put(key []byte, value []byte) { + if b.l.binlog != nil { + buf := encodeBinLogPut(key, value) + b.logs = append(b.logs, buf) + } + b.WriteBatch.Put(key, value) +} + +func (b *batch) Delete(key []byte) { + if b.l.binlog != nil { + buf := encodeBinLogDelete(key) + b.logs = append(b.logs, buf) + } + b.WriteBatch.Delete(key) +} + +type dbBatchLocker struct { + l *sync.Mutex + wrLock *sync.RWMutex +} + +func (l *dbBatchLocker) Lock() { + l.wrLock.RLock() + l.l.Lock() +} + +func (l *dbBatchLocker) Unlock() { + l.l.Unlock() + l.wrLock.RUnlock() +} + +type txBatchLocker struct { +} + +func (l *txBatchLocker) Lock() {} +func (l *txBatchLocker) Unlock() {} + +type multiBatchLocker struct { +} + +func (l *multiBatchLocker) Lock() {} +func (l *multiBatchLocker) Unlock() {} + +func (l *Ledis) newBatch(wb store.WriteBatch, locker sync.Locker, tx *Tx) *batch { + b := new(batch) + b.l = l + b.WriteBatch = wb + + b.tx = tx + b.Locker = locker + + b.logs = [][]byte{} + return b +} diff --git a/ledis/const.go b/ledis/const.go index ef416de..e889f4e 100644 --- a/ledis/const.go +++ b/ledis/const.go @@ -86,3 +86,9 @@ const ( BinLogTypePut uint8 = 0x1 BinLogTypeCommand uint8 = 0x2 ) + +const ( + DBAutoCommit uint8 = 0x0 + DBInTransaction uint8 = 0x1 + DBInMulti uint8 = 0x2 +) diff --git a/ledis/ledis_db.go b/ledis/ledis_db.go index 9241b1d..dd8ff74 100644 --- a/ledis/ledis_db.go +++ b/ledis/ledis_db.go @@ -3,6 +3,7 @@ package ledis import ( "fmt" "github.com/siddontang/ledisdb/store" + "sync" ) type ibucket interface { @@ -37,7 +38,7 @@ type DB struct { binBatch *batch setBatch *batch - isTx bool + status uint8 } func (l *Ledis) newDB(index uint8) *DB { @@ -49,7 +50,7 @@ func (l *Ledis) newDB(index uint8) *DB { d.bucket = d.sdb - d.isTx = false + d.status = DBAutoCommit d.index = index d.kvBatch = d.newBatch() @@ -62,10 +63,18 @@ func (l *Ledis) newDB(index uint8) *DB { return d } +func (db *DB) newBatch() *batch { + return db.l.newBatch(db.bucket.NewWriteBatch(), &dbBatchLocker{l: &sync.Mutex{}, wrLock: &db.l.wLock}, nil) +} + func (db *DB) Index() int { return int(db.index) } +func (db *DB) IsAutoCommit() bool { + return db.status == DBAutoCommit +} + func (db *DB) FlushAll() (drop int64, err error) { all := [...](func() (int64, error)){ db.flush, diff --git a/ledis/multi.go b/ledis/multi.go new file mode 100644 index 0000000..a549c2c --- /dev/null +++ b/ledis/multi.go @@ -0,0 +1,73 @@ +package ledis + +import ( + "errors" + "fmt" +) + +var ( + ErrNestMulti = errors.New("nest multi not supported") + ErrMultiDone = errors.New("multi has been closed") +) + +type Multi struct { + *DB +} + +func (db *DB) IsInMulti() bool { + return db.status == DBInMulti +} + +// begin a mutli to execute commands, +// it will block any other write operations before you close the multi, unlike transaction, mutli can not rollback +func (db *DB) Multi() (*Multi, error) { + if db.IsInMulti() { + return nil, ErrNestMulti + } + + m := new(Multi) + + m.DB = new(DB) + m.DB.status = DBInMulti + + m.DB.l = db.l + + m.l.wLock.Lock() + + m.DB.sdb = db.sdb + + m.DB.bucket = db.sdb + + m.DB.index = db.index + + m.DB.kvBatch = m.newBatch() + m.DB.listBatch = m.newBatch() + m.DB.hashBatch = m.newBatch() + m.DB.zsetBatch = m.newBatch() + m.DB.binBatch = m.newBatch() + m.DB.setBatch = m.newBatch() + + return m, nil +} + +func (m *Multi) newBatch() *batch { + return m.l.newBatch(m.bucket.NewWriteBatch(), &multiBatchLocker{}, nil) +} + +func (m *Multi) Close() error { + if m.bucket == nil { + return ErrMultiDone + } + m.l.wLock.Unlock() + m.bucket = nil + return nil +} + +func (m *Multi) Select(index int) error { + if index < 0 || index >= int(MaxDBNumber) { + return fmt.Errorf("invalid db index %d", index) + } + + m.DB.index = uint8(index) + return nil +} diff --git a/ledis/multi_test.go b/ledis/multi_test.go new file mode 100644 index 0000000..936c141 --- /dev/null +++ b/ledis/multi_test.go @@ -0,0 +1,51 @@ +package ledis + +import ( + "sync" + "testing" +) + +func TestMulti(t *testing.T) { + db := getTestDB() + + key := []byte("test_multi_1") + v1 := []byte("v1") + v2 := []byte("v2") + + m, err := db.Multi() + if err != nil { + t.Fatal(err) + } + + wg := sync.WaitGroup{} + + wg.Add(1) + + go func() { + if err := db.Set(key, v2); err != nil { + t.Fatal(err) + } + wg.Done() + }() + + if err := m.Set(key, v1); err != nil { + t.Fatal(err) + } + + if v, err := m.Get(key); err != nil { + t.Fatal(err) + } else if string(v) != string(v1) { + t.Fatal(string(v)) + } + + m.Close() + + wg.Wait() + + if v, err := db.Get(key); err != nil { + t.Fatal(err) + } else if string(v) != string(v2) { + t.Fatal(string(v)) + } + +} diff --git a/ledis/replication_test.go b/ledis/replication_test.go index 96bb10a..2a64a11 100644 --- a/ledis/replication_test.go +++ b/ledis/replication_test.go @@ -70,6 +70,12 @@ func TestReplication(t *testing.T) { db.HSet([]byte("c"), []byte("3"), []byte("value")) } + m, _ := db.Multi() + m.Set([]byte("a1"), []byte("value")) + m.Set([]byte("b1"), []byte("value")) + m.Set([]byte("c1"), []byte("value")) + m.Close() + for _, name := range master.binlog.LogNames() { p := path.Join(master.binlog.LogPath(), name) diff --git a/ledis/tx.go b/ledis/tx.go deleted file mode 100644 index 7488233..0000000 --- a/ledis/tx.go +++ /dev/null @@ -1,209 +0,0 @@ -package ledis - -import ( - "errors" - "github.com/siddontang/ledisdb/store" - "sync" -) - -var ( - ErrNestTx = errors.New("nest transaction not supported") - ErrTxDone = errors.New("Transaction has already been committed or rolled back") -) - -type batch struct { - l *Ledis - - store.WriteBatch - - sync.Locker - - logs [][]byte - - tx *Tx -} - -type dbBatchLocker struct { - l *sync.Mutex - wrLock *sync.RWMutex -} - -func (l *dbBatchLocker) Lock() { - l.wrLock.RLock() - l.l.Lock() -} - -func (l *dbBatchLocker) Unlock() { - l.l.Unlock() - l.wrLock.RUnlock() -} - -type txBatchLocker struct { -} - -func (l *txBatchLocker) Lock() {} -func (l *txBatchLocker) Unlock() {} - -func (l *Ledis) newBatch(wb store.WriteBatch, tx *Tx) *batch { - b := new(batch) - b.l = l - b.WriteBatch = wb - - b.tx = tx - if tx == nil { - b.Locker = &dbBatchLocker{l: &sync.Mutex{}, wrLock: &l.wLock} - } else { - b.Locker = &txBatchLocker{} - } - - b.logs = [][]byte{} - return b -} - -func (db *DB) newBatch() *batch { - return db.l.newBatch(db.bucket.NewWriteBatch(), nil) -} - -func (b *batch) Commit() error { - b.l.commitLock.Lock() - defer b.l.commitLock.Unlock() - - err := b.WriteBatch.Commit() - - if b.l.binlog != nil { - if err == nil { - if b.tx == nil { - b.l.binlog.Log(b.logs...) - } else { - b.tx.logs = append(b.tx.logs, b.logs...) - } - } - b.logs = [][]byte{} - } - - return err -} - -func (b *batch) Lock() { - b.Locker.Lock() -} - -func (b *batch) Unlock() { - if b.l.binlog != nil { - b.logs = [][]byte{} - } - b.WriteBatch.Rollback() - b.Locker.Unlock() -} - -func (b *batch) Put(key []byte, value []byte) { - if b.l.binlog != nil { - buf := encodeBinLogPut(key, value) - b.logs = append(b.logs, buf) - } - b.WriteBatch.Put(key, value) -} - -func (b *batch) Delete(key []byte) { - if b.l.binlog != nil { - buf := encodeBinLogDelete(key) - b.logs = append(b.logs, buf) - } - b.WriteBatch.Delete(key) -} - -type Tx struct { - *DB - - tx *store.Tx - - logs [][]byte - - index uint8 -} - -func (db *DB) IsTransaction() bool { - return db.isTx -} - -// Begin a transaction, it will block all other write operations before calling Commit or Rollback. -// You must be very careful to prevent long-time transaction. -func (db *DB) Begin() (*Tx, error) { - if db.isTx { - return nil, ErrNestTx - } - - tx := new(Tx) - - tx.DB = new(DB) - tx.DB.l = db.l - - tx.l.wLock.Lock() - - tx.index = db.index - - tx.DB.sdb = db.sdb - - var err error - tx.tx, err = db.sdb.Begin() - if err != nil { - tx.l.wLock.Unlock() - return nil, err - } - - tx.DB.bucket = tx.tx - - tx.DB.isTx = true - - tx.DB.index = db.index - - tx.DB.kvBatch = tx.newTxBatch() - tx.DB.listBatch = tx.newTxBatch() - tx.DB.hashBatch = tx.newTxBatch() - tx.DB.zsetBatch = tx.newTxBatch() - tx.DB.binBatch = tx.newTxBatch() - tx.DB.setBatch = tx.newTxBatch() - - return tx, nil -} - -func (tx *Tx) Commit() error { - if tx.tx == nil { - return ErrTxDone - } - - tx.l.commitLock.Lock() - err := tx.tx.Commit() - tx.tx = nil - - if len(tx.logs) > 0 { - tx.l.binlog.Log(tx.logs...) - } - - tx.l.commitLock.Unlock() - - tx.l.wLock.Unlock() - tx.DB = nil - return err -} - -func (tx *Tx) Rollback() error { - if tx.tx == nil { - return ErrTxDone - } - - err := tx.tx.Rollback() - tx.tx = nil - - tx.l.wLock.Unlock() - tx.DB = nil - return err -} - -func (tx *Tx) newTxBatch() *batch { - return tx.l.newBatch(tx.tx.NewWriteBatch(), tx) -} - -func (tx *Tx) Index() int { - return int(tx.index) -} diff --git a/ledis/tx_test.go b/ledis/tx_test.go index bf06012..026b70d 100644 --- a/ledis/tx_test.go +++ b/ledis/tx_test.go @@ -144,6 +144,51 @@ func testTxCommit(t *testing.T, db *DB) { } } +func testTxSelect(t *testing.T, db *DB) { + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + + defer tx.Rollback() + + tx.Set([]byte("tx_select_1"), []byte("a")) + + tx.Select(1) + + tx.Set([]byte("tx_select_2"), []byte("b")) + + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + + if v, err := db.Get([]byte("tx_select_1")); err != nil { + t.Fatal(err) + } else if string(v) != "a" { + t.Fatal(string(v)) + } + + if v, err := db.Get([]byte("tx_select_2")); err != nil { + t.Fatal(err) + } else if v != nil { + t.Fatal("must nil") + } + + db, _ = db.l.Select(1) + + if v, err := db.Get([]byte("tx_select_2")); err != nil { + t.Fatal(err) + } else if string(v) != "b" { + t.Fatal(string(v)) + } + + if v, err := db.Get([]byte("tx_select_1")); err != nil { + t.Fatal(err) + } else if v != nil { + t.Fatal("must nil") + } +} + func testTx(t *testing.T, name string) { cfg := new(config.Config) cfg.DataDir = "/tmp/ledis_test_tx" @@ -164,6 +209,7 @@ func testTx(t *testing.T, name string) { testTxRollback(t, db) testTxCommit(t, db) + testTxSelect(t, db) } //only lmdb, boltdb support Transaction diff --git a/server/app.go b/server/app.go index d5c77c9..edd65c8 100644 --- a/server/app.go +++ b/server/app.go @@ -27,6 +27,8 @@ type App struct { m *master info *info + + s *script } func netType(s string) string { @@ -85,6 +87,8 @@ func NewApp(cfg *config.Config) (*App, error) { app.m = newMaster(app) + app.openScript() + return app, nil } @@ -103,6 +107,8 @@ func (app *App) Close() { app.httpListener.Close() } + app.closeScript() + app.m.Close() if app.access != nil { diff --git a/server/client.go b/server/client.go index f28a930..27e08b1 100644 --- a/server/client.go +++ b/server/client.go @@ -16,6 +16,18 @@ var txUnsupportedCmds = map[string]struct{}{ "begin": struct{}{}, "flushall": struct{}{}, "flushdb": struct{}{}, + "eval": struct{}{}, +} + +var scriptUnsupportedCmds = map[string]struct{}{ + "slaveof": struct{}{}, + "fullsync": struct{}{}, + "sync": struct{}{}, + "begin": struct{}{}, + "commit": struct{}{}, + "rollback": struct{}{}, + "flushall": struct{}{}, + "flushdb": struct{}{}, } type responseWriter interface { @@ -34,7 +46,8 @@ type responseWriter interface { type client struct { app *App ldb *ledis.Ledis - db *ledis.DB + + db *ledis.DB remoteAddr string cmd string @@ -49,7 +62,8 @@ type client struct { buf bytes.Buffer - tx *ledis.Tx + tx *ledis.Tx + script *ledis.Multi } func newClient(app *App) *client { @@ -59,16 +73,12 @@ func newClient(app *App) *client { c.ldb = app.ldb c.db, _ = app.ldb.Select(0) //use default db - c.compressBuf = make([]byte, 256) + c.compressBuf = []byte{} c.reqErr = make(chan error) return c } -func (c *client) isInTransaction() bool { - return c.tx != nil -} - func (c *client) perform() { var err error @@ -79,10 +89,14 @@ func (c *client) perform() { } else if exeCmd, ok := regCmds[c.cmd]; !ok { err = ErrNotFound } else { - if c.isInTransaction() { + if c.db.IsTransaction() { if _, ok := txUnsupportedCmds[c.cmd]; ok { err = fmt.Errorf("%s not supported in transaction", c.cmd) } + } else if c.db.IsInMulti() { + if _, ok := scriptUnsupportedCmds[c.cmd]; ok { + err = fmt.Errorf("%s not supported in multi", c.cmd) + } } if err == nil { @@ -128,3 +142,22 @@ func (c *client) catGenericCommand() []byte { return buffer.Bytes() } + +func writeValue(w responseWriter, value interface{}) { + switch v := value.(type) { + case []interface{}: + w.writeArray(v) + case [][]byte: + w.writeSliceArray(v) + case []byte: + w.writeBulk(v) + case string: + w.writeStatus(v) + case nil: + w.writeBulk(nil) + case int64: + w.writeInteger(v) + default: + panic("invalid value type") + } +} diff --git a/server/cmd_script.go b/server/cmd_script.go index 20f5b63..f0dabd1 100644 --- a/server/cmd_script.go +++ b/server/cmd_script.go @@ -1,158 +1,207 @@ package server import ( + "encoding/hex" "fmt" "github.com/aarzilli/golua/lua" "github.com/siddontang/ledisdb/ledis" - "io" + "strconv" + "strings" ) -//ledis <-> lua type conversion, same as http://redis.io/commands/eval - -type luaClient struct { - l *lua.State -} - -type luaWriter struct { - l *lua.State -} - -func (w *luaWriter) writeError(err error) { - w.l.NewTable() - top := w.l.GetTop() - - w.l.PushString("err") - w.l.PushString(err.Error()) - w.l.SetTable(top) -} - -func (w *luaWriter) writeStatus(status string) { - w.l.NewTable() - top := w.l.GetTop() - - w.l.PushString("ok") - w.l.PushString(status) - w.l.SetTable(top) -} - -func (w *luaWriter) writeInteger(n int64) { - w.l.PushInteger(n) -} - -func (w *luaWriter) writeBulk(b []byte) { - if b == nil { - w.l.PushBoolean(false) - } else { - w.l.PushString(ledis.String(b)) - } -} - -func (w *luaWriter) writeArray(lst []interface{}) { - if lst == nil { - w.l.PushBoolean(false) - return +func parseEvalArgs(l *lua.State, c *client) error { + args := c.args + if len(args) < 2 { + return ErrCmdParams } - base := w.l.GetTop() + args = args[1:] + + n, err := strconv.Atoi(ledis.String(args[0])) + if err != nil { + return err + } + + if n > len(args)-1 { + return ErrCmdParams + } + + luaSetGlobalArray(l, "KEYS", args[1:n+1]) + luaSetGlobalArray(l, "ARGV", args[n+1:]) + + return nil +} + +func evalGenericCommand(c *client, evalSha1 bool) error { + m, err := c.db.Multi() + if err != nil { + return err + } + + s := c.app.s + luaClient := s.c + l := s.l + + s.Lock() + + base := l.GetTop() + defer func() { - if e := recover(); e != nil { - w.l.SetTop(base) - w.writeError(fmt.Errorf("%v", e)) - } + l.SetTop(base) + luaClient.db = nil + luaClient.script = nil + + s.Unlock() + + m.Close() }() - w.l.CreateTable(len(lst), 0) - top := w.l.GetTop() + luaClient.db = m.DB + luaClient.script = m + luaClient.remoteAddr = c.remoteAddr - for i, _ := range lst { - w.l.PushInteger(int64(i) + 1) - - switch v := lst[i].(type) { - case []interface{}: - w.writeArray(v) - case [][]byte: - w.writeSliceArray(v) - case []byte: - w.writeBulk(v) - case nil: - w.writeBulk(nil) - case int64: - w.writeInteger(v) - default: - panic("invalid array type") - } - - w.l.SetTable(top) - } -} - -func (w *luaWriter) writeSliceArray(lst [][]byte) { - if lst == nil { - w.l.PushBoolean(false) - return + if err := parseEvalArgs(l, c); err != nil { + return err } - w.l.CreateTable(len(lst), 0) - top := w.l.GetTop() - for i, v := range lst { - w.l.PushInteger(int64(i) + 1) - w.l.PushString(ledis.String(v)) - w.l.SetTable(top) - } -} - -func (w *luaWriter) writeFVPairArray(lst []ledis.FVPair) { - if lst == nil { - w.l.PushBoolean(false) - return - } - - w.l.CreateTable(len(lst)*2, 0) - top := w.l.GetTop() - for i, v := range lst { - w.l.PushInteger(int64(2*i) + 1) - w.l.PushString(ledis.String(v.Field)) - w.l.SetTable(top) - - w.l.PushInteger(int64(2*i) + 2) - w.l.PushString(ledis.String(v.Value)) - w.l.SetTable(top) - } -} - -func (w *luaWriter) writeScorePairArray(lst []ledis.ScorePair, withScores bool) { - if lst == nil { - w.l.PushBoolean(false) - return - } - - if withScores { - w.l.CreateTable(len(lst)*2, 0) - top := w.l.GetTop() - for i, v := range lst { - w.l.PushInteger(int64(2*i) + 1) - w.l.PushString(ledis.String(v.Member)) - w.l.SetTable(top) - - w.l.PushInteger(int64(2*i) + 2) - w.l.PushString(ledis.String(ledis.StrPutInt64(v.Score))) - w.l.SetTable(top) - } + var sha1 string + if !evalSha1 { + sha1 = hex.EncodeToString(c.args[0]) } else { - w.l.CreateTable(len(lst), 0) - top := w.l.GetTop() - for i, v := range lst { - w.l.PushInteger(int64(i) + 1) - w.l.PushString(ledis.String(v.Member)) - w.l.SetTable(top) + sha1 = ledis.String(c.args[0]) + } + + l.GetGlobal(sha1) + + if l.IsNil(-1) { + l.Pop(1) + + if evalSha1 { + return fmt.Errorf("missing %s script", sha1) + } + + if r := l.LoadString(ledis.String(c.args[0])); r != 0 { + err := fmt.Errorf("%s", l.ToString(-1)) + l.Pop(1) + return err + } else { + l.PushValue(-1) + l.SetGlobal(sha1) + + s.chunks[sha1] = struct{}{} } } + + if err := l.Call(0, lua.LUA_MULTRET); err != nil { + return err + } else { + r := luaReplyToLedisReply(l) + m.Close() + writeValue(c.resp, r) + } + + return nil } -func (w *luaWriter) writeBulkFrom(n int64, rb io.Reader) { - w.writeError(fmt.Errorf("unsupport")) +func evalCommand(c *client) error { + return evalGenericCommand(c, false) } -func (w *luaWriter) flush() { - +func evalshaCommand(c *client) error { + return evalGenericCommand(c, true) +} + +func scriptCommand(c *client) error { + s := c.app.s + l := s.l + + s.Lock() + + base := l.GetTop() + + defer func() { + l.SetTop(base) + s.Unlock() + }() + + args := c.args + if len(args) < 1 { + return ErrCmdParams + } + + switch strings.ToLower(c.cmd) { + case "script load": + return scriptLoadCommand(c) + case "script exists": + return scriptExistsCommand(c) + case "script flush": + return scriptFlushCommand(c) + default: + return fmt.Errorf("invalid scirpt cmd %s", args[0]) + } + + return nil +} + +func scriptLoadCommand(c *client) error { + s := c.app.s + l := s.l + + if len(c.args) != 1 { + return ErrCmdParams + } + + sha1 := hex.EncodeToString(c.args[1]) + + if r := l.LoadString(ledis.String(c.args[1])); r != 0 { + err := fmt.Errorf("%s", l.ToString(-1)) + l.Pop(1) + return err + } else { + l.PushValue(-1) + l.SetGlobal(sha1) + + s.chunks[sha1] = struct{}{} + } + + c.resp.writeBulk(ledis.Slice(sha1)) + return nil +} + +func scriptExistsCommand(c *client) error { + s := c.app.s + + ay := make([]interface{}, len(c.args[1:])) + for i, n := range c.args[1:] { + if _, ok := s.chunks[ledis.String(n)]; ok { + ay[i] = int64(1) + } else { + ay[i] = int64(0) + } + } + + c.resp.writeArray(ay) + return nil +} + +func scriptFlushCommand(c *client) error { + s := c.app.s + l := s.l + + for n, _ := range s.chunks { + l.PushNil() + l.SetGlobal(n) + } + + c.resp.writeStatus(OK) + + return nil +} + +func init() { + register("eval", evalCommand) + register("evalsha", evalshaCommand) + register("script load", scriptCommand) + register("script flush", scriptCommand) + register("script exists", scriptCommand) } diff --git a/server/cmd_script_test.go b/server/cmd_script_test.go new file mode 100644 index 0000000..5c47866 --- /dev/null +++ b/server/cmd_script_test.go @@ -0,0 +1,29 @@ +package server + +import ( + "fmt" + "github.com/siddontang/ledisdb/client/go/ledis" + "reflect" + "testing" +) + +func TestCmdEval(t *testing.T) { + c := getTestConn() + defer c.Close() + + if v, err := ledis.Strings(c.Do("eval", "return {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}", 2, "key1", "key2", "first", "second")); err != nil { + t.Fatal(err) + } else if len(v) != 4 { + t.Fatal(err) + } else if !reflect.DeepEqual(v, []string{"key1", "key2", "first", "second"}) { + t.Fatal(fmt.Sprintf("%v", v)) + } + + if v, err := ledis.Strings(c.Do("eval", "return {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}", 2, "key1", "key2", "first", "second")); err != nil { + t.Fatal(err) + } else if len(v) != 4 { + t.Fatal(err) + } else if !reflect.DeepEqual(v, []string{"key1", "key2", "first", "second"}) { + t.Fatal(fmt.Sprintf("%v", v)) + } +} diff --git a/server/command.go b/server/command.go index af95244..458343b 100644 --- a/server/command.go +++ b/server/command.go @@ -41,12 +41,26 @@ func selectCommand(c *client) error { if index, err := strconv.Atoi(ledis.String(c.args[0])); err != nil { return err } else { - if db, err := c.ldb.Select(index); err != nil { - return err + if c.db.IsTransaction() { + if err := c.tx.Select(index); err != nil { + return err + } else { + c.db = c.tx.DB + } + } else if c.db.IsInMulti() { + if err := c.script.Select(index); err != nil { + return err + } else { + c.db = c.script.DB + } } else { - c.db = db - c.resp.writeStatus(OK) + if db, err := c.ldb.Select(index); err != nil { + return err + } else { + c.db = db + } } + c.resp.writeStatus(OK) } return nil diff --git a/server/script.go b/server/script.go new file mode 100644 index 0000000..c2c8ff2 --- /dev/null +++ b/server/script.go @@ -0,0 +1,380 @@ +package server + +import ( + "encoding/hex" + "fmt" + "github.com/aarzilli/golua/lua" + "github.com/siddontang/ledisdb/ledis" + "io" + "sync" +) + +//ledis <-> lua type conversion, same as http://redis.io/commands/eval + +type luaWriter struct { + l *lua.State +} + +func (w *luaWriter) writeError(err error) { + panic(err) +} + +func (w *luaWriter) writeStatus(status string) { + w.l.NewTable() + top := w.l.GetTop() + + w.l.PushString("ok") + w.l.PushString(status) + w.l.SetTable(top) +} + +func (w *luaWriter) writeInteger(n int64) { + w.l.PushInteger(n) +} + +func (w *luaWriter) writeBulk(b []byte) { + if b == nil { + w.l.PushBoolean(false) + } else { + w.l.PushString(ledis.String(b)) + } +} + +func (w *luaWriter) writeArray(lst []interface{}) { + if lst == nil { + w.l.PushBoolean(false) + return + } + + w.l.CreateTable(len(lst), 0) + top := w.l.GetTop() + + for i, _ := range lst { + w.l.PushInteger(int64(i) + 1) + + switch v := lst[i].(type) { + case []interface{}: + w.writeArray(v) + case [][]byte: + w.writeSliceArray(v) + case []byte: + w.writeBulk(v) + case nil: + w.writeBulk(nil) + case int64: + w.writeInteger(v) + default: + panic("invalid array type") + } + + w.l.SetTable(top) + } +} + +func (w *luaWriter) writeSliceArray(lst [][]byte) { + if lst == nil { + w.l.PushBoolean(false) + return + } + + w.l.CreateTable(len(lst), 0) + for i, v := range lst { + w.l.PushString(ledis.String(v)) + w.l.RawSeti(-2, i+1) + } +} + +func (w *luaWriter) writeFVPairArray(lst []ledis.FVPair) { + if lst == nil { + w.l.PushBoolean(false) + return + } + + w.l.CreateTable(len(lst)*2, 0) + for i, v := range lst { + w.l.PushString(ledis.String(v.Field)) + w.l.RawSeti(-2, 2*i+1) + + w.l.PushString(ledis.String(v.Value)) + w.l.RawSeti(-2, 2*i+2) + } +} + +func (w *luaWriter) writeScorePairArray(lst []ledis.ScorePair, withScores bool) { + if lst == nil { + w.l.PushBoolean(false) + return + } + + if withScores { + w.l.CreateTable(len(lst)*2, 0) + for i, v := range lst { + w.l.PushString(ledis.String(v.Member)) + w.l.RawSeti(-2, 2*i+1) + + w.l.PushString(ledis.String(ledis.StrPutInt64(v.Score))) + w.l.RawSeti(-2, 2*i+2) + } + } else { + w.l.CreateTable(len(lst), 0) + for i, v := range lst { + w.l.PushString(ledis.String(v.Member)) + w.l.RawSeti(-2, i+1) + } + } +} + +func (w *luaWriter) writeBulkFrom(n int64, rb io.Reader) { + w.writeError(fmt.Errorf("unsupport")) +} + +func (w *luaWriter) flush() { + +} + +type script struct { + sync.Mutex + + app *App + l *lua.State + c *client + + chunks map[string]struct{} +} + +func (app *App) openScript() { + s := new(script) + s.app = app + + s.chunks = make(map[string]struct{}) + + app.s = s + + l := lua.NewState() + + l.OpenBase() + l.OpenLibs() + l.OpenMath() + l.OpenString() + l.OpenTable() + l.OpenPackage() + + s.l = l + s.c = newClient(app) + s.c.db = nil + + w := new(luaWriter) + w.l = l + s.c.resp = w + + l.NewTable() + l.PushString("call") + l.PushGoFunction(luaCall) + l.SetTable(-3) + + l.PushString("pcall") + l.PushGoFunction(luaPCall) + l.SetTable(-3) + + l.PushString("sha1hex") + l.PushGoFunction(luaSha1Hex) + l.SetTable(-3) + + l.PushString("error_reply") + l.PushGoFunction(luaErrorReply) + l.SetTable(-3) + + l.PushString("status_reply") + l.PushGoFunction(luaStatusReply) + l.SetTable(-3) + + l.SetGlobal("ledis") + + setMapState(l, s) +} + +func (app *App) closeScript() { + app.s.l.Close() + delMapState(app.s.l) + app.s = nil +} + +var mapState = map[*lua.State]*script{} +var stateLock sync.Mutex + +func setMapState(l *lua.State, s *script) { + stateLock.Lock() + defer stateLock.Unlock() + + mapState[l] = s +} + +func getMapState(l *lua.State) *script { + stateLock.Lock() + defer stateLock.Unlock() + + return mapState[l] +} + +func delMapState(l *lua.State) { + stateLock.Lock() + defer stateLock.Unlock() + + delete(mapState, l) +} + +func luaCall(l *lua.State) int { + return luaCallGenericCommand(l) +} + +func luaPCall(l *lua.State) (n int) { + defer func() { + if e := recover(); e != nil { + luaPushError(l, fmt.Sprintf("%v", e)) + n = 1 + } + return + }() + return luaCallGenericCommand(l) +} + +func luaErrorReply(l *lua.State) int { + return luaReturnSingleFieldTable(l, "err") +} + +func luaStatusReply(l *lua.State) int { + return luaReturnSingleFieldTable(l, "ok") +} + +func luaReturnSingleFieldTable(l *lua.State, filed string) int { + if l.GetTop() != 1 || l.Type(-1) != lua.LUA_TSTRING { + luaPushError(l, "wrong number or type of arguments") + return 1 + } + + l.NewTable() + l.PushString(filed) + l.PushValue(-3) + l.SetTable(-3) + return 1 +} + +func luaSha1Hex(l *lua.State) int { + argc := l.GetTop() + if argc != 1 { + luaPushError(l, "wrong number of arguments") + return 1 + } + + s := l.ToString(1) + s = hex.EncodeToString(ledis.Slice(s)) + + l.PushString(s) + return 1 +} + +func luaPushError(l *lua.State, msg string) { + l.NewTable() + l.PushString("err") + err := l.NewError(msg) + l.PushString(err.Error()) + l.SetTable(-3) +} + +func luaCallGenericCommand(l *lua.State) int { + s := getMapState(l) + if s == nil { + panic("Invalid lua call") + } else if s.c.db == nil { + panic("Invalid lua call, not prepared") + } + + c := s.c + + argc := l.GetTop() + if argc < 1 { + panic("Please specify at least one argument for ledis.call()") + } + + c.cmd = l.ToString(1) + + c.args = make([][]byte, argc-1) + + for i := 2; i <= argc; i++ { + switch l.Type(i) { + case lua.LUA_TNUMBER: + c.args[i-2] = []byte(fmt.Sprintf("%.17g", l.ToNumber(i))) + case lua.LUA_TSTRING: + c.args[i-2] = []byte(l.ToString(i)) + default: + panic("Lua ledis() command arguments must be strings or integers") + } + } + + c.perform() + + return 1 +} + +func luaSetGlobalArray(l *lua.State, name string, ay [][]byte) { + l.NewTable() + + for i := 0; i < len(ay); i++ { + l.PushString(ledis.String(ay[i])) + l.RawSeti(-2, i+1) + } + + l.SetGlobal(name) +} + +func luaReplyToLedisReply(l *lua.State) interface{} { + base := l.GetTop() + defer func() { + l.SetTop(base - 1) + }() + + switch l.Type(-1) { + case lua.LUA_TSTRING: + return ledis.Slice(l.ToString(-1)) + case lua.LUA_TBOOLEAN: + if l.ToBoolean(-1) { + return int64(1) + } else { + return nil + } + case lua.LUA_TNUMBER: + return int64(l.ToInteger(-1)) + case lua.LUA_TTABLE: + l.PushString("err") + l.GetTable(-2) + if l.Type(-1) == lua.LUA_TSTRING { + return fmt.Errorf("%s", l.ToString(-1)) + } + + l.Pop(1) + l.PushString("ok") + l.GetTable(-2) + if l.Type(-1) == lua.LUA_TSTRING { + return l.ToString(-1) + } else { + l.Pop(1) + + ay := make([]interface{}, 0) + + for i := 1; ; i++ { + l.PushInteger(int64(i)) + l.GetTable(-2) + if l.Type(-1) == lua.LUA_TNIL { + l.Pop(1) + break + } + + ay = append(ay, luaReplyToLedisReply(l)) + } + return ay + + } + default: + return nil + } +} diff --git a/server/script_test.go b/server/script_test.go new file mode 100644 index 0000000..0d91231 --- /dev/null +++ b/server/script_test.go @@ -0,0 +1,177 @@ +package server + +import ( + "fmt" + "github.com/aarzilli/golua/lua" + "github.com/siddontang/ledisdb/config" + + "testing" +) + +var testLuaWriter = &luaWriter{} + +func testLuaWriteError(l *lua.State) int { + testLuaWriter.writeError(fmt.Errorf("test error")) + return 1 +} + +func testLuaWriteArray(l *lua.State) int { + ay := make([]interface{}, 2) + ay[0] = []byte("1") + b := make([]interface{}, 2) + b[0] = int64(10) + b[1] = []byte("11") + + ay[1] = b + + testLuaWriter.writeArray(ay) + + return 1 +} + +func TestLuaWriter(t *testing.T) { + l := lua.NewState() + + l.OpenBase() + + testLuaWriter.l = l + + l.Register("WriteError", testLuaWriteError) + + str := ` + WriteError() + ` + + err := l.DoString(str) + + if err == nil { + t.Fatal("must error") + } + + l.Register("WriteArray", testLuaWriteArray) + + str = ` + local a = WriteArray() + + if #a ~= 2 then + error("len a must 2") + elseif a[1] ~= "1" then + error("a[1] must 1") + elseif #a[2] ~= 2 then + error("len a[2] must 2") + elseif a[2][1] ~= 10 then + error("a[2][1] must 10") + elseif a[2][2] ~= "11" then + error("a[2][2] must 11") + end + ` + + err = l.DoString(str) + if err != nil { + t.Fatal(err) + } + + l.Close() +} + +var testScript1 = ` + return {1,2,3} +` + +var testScript2 = ` + return ledis.call("ping") +` + +var testScript3 = ` + ledis.call("set", 1, "a") + + local a = ledis.call("get", 1) + if type(a) ~= "string" then + error("must string") + elseif a ~= "a" then + error("must a") + end +` + +var testScript4 = ` + ledis.call("select", 2) + ledis.call("set", 2, "b") +` + +func TestLuaCall(t *testing.T) { + cfg := new(config.Config) + cfg.Addr = ":11188" + cfg.DataDir = "/tmp/testscript" + cfg.DBName = "memory" + + app, e := NewApp(cfg) + if e != nil { + t.Fatal(e) + } + go app.Run() + + defer app.Close() + + db, _ := app.ldb.Select(0) + m, _ := db.Multi() + defer m.Close() + + luaClient := app.s.c + luaClient.db = m.DB + luaClient.script = m + + l := app.s.l + + err := app.s.l.DoString(testScript1) + if err != nil { + t.Fatal(err) + } + + v := luaReplyToLedisReply(l) + if vv, ok := v.([]interface{}); ok { + if len(vv) != 3 { + t.Fatal(len(vv)) + } + } else { + t.Fatal(fmt.Sprintf("%v %T", v, v)) + } + + err = app.s.l.DoString(testScript2) + if err != nil { + t.Fatal(err) + } + + v = luaReplyToLedisReply(l) + if vv := v.(string); vv != "PONG" { + t.Fatal(fmt.Sprintf("%v %T", v, v)) + } + + err = app.s.l.DoString(testScript3) + if err != nil { + t.Fatal(err) + } + + if v, err := db.Get([]byte("1")); err != nil { + t.Fatal(err) + } else if string(v) != "a" { + t.Fatal(string(v)) + } + + err = app.s.l.DoString(testScript4) + if err != nil { + t.Fatal(err) + } + + if luaClient.db.Index() != 2 { + t.Fatal(luaClient.db.Index()) + } + + db2, _ := app.ldb.Select(2) + if v, err := db2.Get([]byte("2")); err != nil { + t.Fatal(err) + } else if string(v) != "b" { + t.Fatal(string(v)) + } + + luaClient.db = nil +} From 0af4758e798d9b5ca45d686d1eb5eae46143022e Mon Sep 17 00:00:00 2001 From: siddontang Date: Tue, 2 Sep 2014 22:04:18 +0800 Subject: [PATCH 12/32] lua update --- ledis/tx.go | 112 ++++++++++++++++++++++++++++++++++++++ server/cmd_script.go | 42 ++++++++------ server/cmd_script_test.go | 34 +++++++++++- 3 files changed, 167 insertions(+), 21 deletions(-) create mode 100644 ledis/tx.go diff --git a/ledis/tx.go b/ledis/tx.go new file mode 100644 index 0000000..6339bae --- /dev/null +++ b/ledis/tx.go @@ -0,0 +1,112 @@ +package ledis + +import ( + "errors" + "fmt" + "github.com/siddontang/ledisdb/store" +) + +var ( + ErrNestTx = errors.New("nest transaction not supported") + ErrTxDone = errors.New("Transaction has already been committed or rolled back") +) + +type Tx struct { + *DB + + tx *store.Tx + + logs [][]byte +} + +func (db *DB) IsTransaction() bool { + return db.status == DBInTransaction +} + +// Begin a transaction, it will block all other write operations before calling Commit or Rollback. +// You must be very careful to prevent long-time transaction. +func (db *DB) Begin() (*Tx, error) { + if db.IsTransaction() { + return nil, ErrNestTx + } + + tx := new(Tx) + + tx.DB = new(DB) + tx.DB.l = db.l + + tx.l.wLock.Lock() + + tx.DB.sdb = db.sdb + + var err error + tx.tx, err = db.sdb.Begin() + if err != nil { + tx.l.wLock.Unlock() + return nil, err + } + + tx.DB.bucket = tx.tx + + tx.DB.status = DBInTransaction + + tx.DB.index = db.index + + tx.DB.kvBatch = tx.newBatch() + tx.DB.listBatch = tx.newBatch() + tx.DB.hashBatch = tx.newBatch() + tx.DB.zsetBatch = tx.newBatch() + tx.DB.binBatch = tx.newBatch() + tx.DB.setBatch = tx.newBatch() + + return tx, nil +} + +func (tx *Tx) Commit() error { + if tx.tx == nil { + return ErrTxDone + } + + tx.l.commitLock.Lock() + err := tx.tx.Commit() + tx.tx = nil + + if len(tx.logs) > 0 { + tx.l.binlog.Log(tx.logs...) + } + + tx.l.commitLock.Unlock() + + tx.l.wLock.Unlock() + + tx.DB.bucket = nil + + return err +} + +func (tx *Tx) Rollback() error { + if tx.tx == nil { + return ErrTxDone + } + + err := tx.tx.Rollback() + tx.tx = nil + + tx.l.wLock.Unlock() + tx.DB.bucket = nil + + return err +} + +func (tx *Tx) newBatch() *batch { + return tx.l.newBatch(tx.tx.NewWriteBatch(), &txBatchLocker{}, tx) +} + +func (tx *Tx) Select(index int) error { + if index < 0 || index >= int(MaxDBNumber) { + return fmt.Errorf("invalid db index %d", index) + } + + tx.DB.index = uint8(index) + return nil +} diff --git a/server/cmd_script.go b/server/cmd_script.go index f0dabd1..1325477 100644 --- a/server/cmd_script.go +++ b/server/cmd_script.go @@ -1,6 +1,7 @@ package server import ( + "crypto/sha1" "encoding/hex" "fmt" "github.com/aarzilli/golua/lua" @@ -64,20 +65,21 @@ func evalGenericCommand(c *client, evalSha1 bool) error { return err } - var sha1 string + var key string if !evalSha1 { - sha1 = hex.EncodeToString(c.args[0]) + h := sha1.Sum(c.args[0]) + key = hex.EncodeToString(h[0:20]) } else { - sha1 = ledis.String(c.args[0]) + key = strings.ToLower(ledis.String(c.args[0])) } - l.GetGlobal(sha1) + l.GetGlobal(key) if l.IsNil(-1) { l.Pop(1) if evalSha1 { - return fmt.Errorf("missing %s script", sha1) + return fmt.Errorf("missing %s script", key) } if r := l.LoadString(ledis.String(c.args[0])); r != 0 { @@ -86,9 +88,9 @@ func evalGenericCommand(c *client, evalSha1 bool) error { return err } else { l.PushValue(-1) - l.SetGlobal(sha1) + l.SetGlobal(key) - s.chunks[sha1] = struct{}{} + s.chunks[key] = struct{}{} } } @@ -125,9 +127,6 @@ func scriptCommand(c *client) error { }() args := c.args - if len(args) < 1 { - return ErrCmdParams - } switch strings.ToLower(c.cmd) { case "script load": @@ -137,7 +136,7 @@ func scriptCommand(c *client) error { case "script flush": return scriptFlushCommand(c) default: - return fmt.Errorf("invalid scirpt cmd %s", args[0]) + return fmt.Errorf("invalid script cmd %s", args[0]) } return nil @@ -151,28 +150,33 @@ func scriptLoadCommand(c *client) error { return ErrCmdParams } - sha1 := hex.EncodeToString(c.args[1]) + h := sha1.Sum(c.args[0]) + key := hex.EncodeToString(h[0:20]) - if r := l.LoadString(ledis.String(c.args[1])); r != 0 { + if r := l.LoadString(ledis.String(c.args[0])); r != 0 { err := fmt.Errorf("%s", l.ToString(-1)) l.Pop(1) return err } else { l.PushValue(-1) - l.SetGlobal(sha1) + l.SetGlobal(key) - s.chunks[sha1] = struct{}{} + s.chunks[key] = struct{}{} } - c.resp.writeBulk(ledis.Slice(sha1)) + c.resp.writeBulk(ledis.Slice(key)) return nil } func scriptExistsCommand(c *client) error { s := c.app.s - ay := make([]interface{}, len(c.args[1:])) - for i, n := range c.args[1:] { + if len(c.args) < 1 { + return ErrCmdParams + } + + ay := make([]interface{}, len(c.args)) + for i, n := range c.args { if _, ok := s.chunks[ledis.String(n)]; ok { ay[i] = int64(1) } else { @@ -193,6 +197,8 @@ func scriptFlushCommand(c *client) error { l.SetGlobal(n) } + s.chunks = map[string]struct{}{} + c.resp.writeStatus(OK) return nil diff --git a/server/cmd_script_test.go b/server/cmd_script_test.go index 5c47866..d652e08 100644 --- a/server/cmd_script_test.go +++ b/server/cmd_script_test.go @@ -13,17 +13,45 @@ func TestCmdEval(t *testing.T) { if v, err := ledis.Strings(c.Do("eval", "return {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}", 2, "key1", "key2", "first", "second")); err != nil { t.Fatal(err) - } else if len(v) != 4 { - t.Fatal(err) } else if !reflect.DeepEqual(v, []string{"key1", "key2", "first", "second"}) { t.Fatal(fmt.Sprintf("%v", v)) } if v, err := ledis.Strings(c.Do("eval", "return {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}", 2, "key1", "key2", "first", "second")); err != nil { t.Fatal(err) - } else if len(v) != 4 { + } else if !reflect.DeepEqual(v, []string{"key1", "key2", "first", "second"}) { + t.Fatal(fmt.Sprintf("%v", v)) + } + + var sha1 string + var err error + if sha1, err = ledis.String(c.Do("script load", "return {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}")); err != nil { + t.Fatal(err) + } else if len(sha1) != 40 { + t.Fatal(sha1) + } + + if v, err := ledis.Strings(c.Do("evalsha", sha1, 2, "key1", "key2", "first", "second")); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(v, []string{"key1", "key2", "first", "second"}) { t.Fatal(fmt.Sprintf("%v", v)) } + + if ay, err := ledis.Values(c.Do("script exists", sha1, "01234567890123456789")); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ay, []interface{}{int64(1), int64(0)}) { + t.Fatal(fmt.Sprintf("%v", ay)) + } + + if ok, err := ledis.String(c.Do("script flush")); err != nil { + t.Fatal(err) + } else if ok != "OK" { + t.Fatal(ok) + } + + if ay, err := ledis.Values(c.Do("script exists", sha1)); err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(ay, []interface{}{int64(0)}) { + t.Fatal(fmt.Sprintf("%v", ay)) + } } From d917eeacd0a844d72fe57211897fb10001d00153 Mon Sep 17 00:00:00 2001 From: siddontang Date: Tue, 2 Sep 2014 22:36:50 +0800 Subject: [PATCH 13/32] update lua --- README.md | 1 + client/ledis-py/ledis/client.py | 15 ++++++++++++ client/ledis-py/tests/test_cmd_script.py | 24 ++++++++++++++++++ client/nodejs/ledis/lib/commands.js | 5 ++++ client/openresty/ledis.lua | 9 ++++++- cmd/ledis-cli/const.go | 7 +++++- doc/commands.json | 31 ++++++++++++++++++++++++ doc/commands.md | 26 ++++++++++++++++++-- 8 files changed, 114 insertions(+), 4 deletions(-) create mode 100644 client/ledis-py/tests/test_cmd_script.py diff --git a/README.md b/README.md index 4524884..726de3a 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ LedisDB now supports multiple databases as backend to store data, you can test a + Stores lots of data, over the memory limit. + Various backend database to use: LevelDB, goleveldb, LMDB, RocksDB, BoltDB, HyperLevelDB. + Supports transaction using LMDB or BotlDB. ++ Supports lua scripting. + Supports expiration and ttl. + Redis clients, like redis-cli, are supported directly. + Multiple client API supports, including Go, Python, Lua(Openresty), C/C++, Node.js. diff --git a/client/ledis-py/ledis/client.py b/client/ledis-py/ledis/client.py index a083ca6..6c8a9fb 100644 --- a/client/ledis-py/ledis/client.py +++ b/client/ledis-py/ledis/client.py @@ -905,6 +905,21 @@ class Ledis(object): def bscan(self, key, match = "", count = 10): return self.execute_command("BSCAN", key, match, count) + def eval(self, script, keys, *args): + return self.execute_command('EVAL', script, len(keys), *keys, *args) + + def evalsha(self, sha1, keys, *args): + return self.execute_command('EVALSHA', sha1, len(keys), *keys, *args) + + def scriptload(self, script): + return self.execute_command('SCRIPT LOAD', script) + + def scriptexists(self, *args): + return self.execute_command('SCRIPT EXISTS', *args) + + def scriptflush(self): + return self.execute_command('SCRIPT FLUSH') + class Transaction(Ledis): def __init__(self, connection_pool, response_callbacks): self.connection_pool = connection_pool diff --git a/client/ledis-py/tests/test_cmd_script.py b/client/ledis-py/tests/test_cmd_script.py new file mode 100644 index 0000000..b2e2f18 --- /dev/null +++ b/client/ledis-py/tests/test_cmd_script.py @@ -0,0 +1,24 @@ +# coding: utf-8 +# Test Cases for bit commands + +import unittest +import sys +sys.path.append('..') + +import ledis +from ledis._compat import b +from util import expire_at, expire_at_seconds + +l = ledis.Ledis(port=6380) + + +class TestCmdScript(unittest.TestCase): + def setUp(self): + pass + + def tearDown(self): + pass + + + + \ No newline at end of file diff --git a/client/nodejs/ledis/lib/commands.js b/client/nodejs/ledis/lib/commands.js index 696bc7e..26408e2 100644 --- a/client/nodejs/ledis/lib/commands.js +++ b/client/nodejs/ledis/lib/commands.js @@ -129,4 +129,9 @@ module.exports = [ "rollback", "commit", + "eval", + "evalsha", + "script load", + "script exists", + "script flush", ]; diff --git a/client/openresty/ledis.lua b/client/openresty/ledis.lua index a25319e..2778a56 100644 --- a/client/openresty/ledis.lua +++ b/client/openresty/ledis.lua @@ -151,7 +151,14 @@ local commands = { -- [[transaction]] "begin", "commit", - "rollback" + "rollback", + + -- [[script]] + "eval", + "evalsha", + "script load", + "script exists", + "script flush" } diff --git a/cmd/ledis-cli/const.go b/cmd/ledis-cli/const.go index 48b78aa..9560c44 100644 --- a/cmd/ledis-cli/const.go +++ b/cmd/ledis-cli/const.go @@ -1,4 +1,4 @@ -//This file was generated by .tools/generate_commands.py on Wed Aug 27 2014 11:14:50 +0800 +//This file was generated by .tools/generate_commands.py on Tue Sep 02 2014 22:27:45 +0800 package main var helpCommands = [][]string{ @@ -20,6 +20,8 @@ var helpCommands = [][]string{ {"DECRBY", "key decrement", "KV"}, {"DEL", "key [key ...]", "KV"}, {"ECHO", "message", "Server"}, + {"EVAL", "script numkeys key [key ...] arg [arg ...]", "Script"}, + {"EVALSHA", "sha1 numkeys key [key ...] arg [arg ...]", "Script"}, {"EXISTS", "key", "KV"}, {"EXPIRE", "key seconds", "KV"}, {"EXPIREAT", "key timestamp", "KV"}, @@ -72,6 +74,9 @@ var helpCommands = [][]string{ {"SCAN", "key [MATCH match] [COUNT count]", "KV"}, {"SCARD", "key", "Set"}, {"SCLEAR", "key", "Set"}, + {"SCRIPT EXISTS", "script [script ...]", "Script"}, + {"SCRIPT FLUSH", "-", "Script"}, + {"SCRIPT LOAD", "script", "Script"}, {"SDIFF", "key [key ...]", "Set"}, {"SDIFFSTORE", "destination key [key ...]", "Set"}, {"SELECT", "index", "Server"}, diff --git a/doc/commands.json b/doc/commands.json index e6ad2e2..370d588 100644 --- a/doc/commands.json +++ b/doc/commands.json @@ -580,5 +580,36 @@ "arguments": "[section]", "group": "Server", "readonly": true + }, + + "EVAL": { + "arguments": "script numkeys key [key ...] arg [arg ...]", + "group": "Script", + "readonly": false + }, + + "EVALSHA": { + "arguments": "sha1 numkeys key [key ...] arg [arg ...]", + "group": "Script", + "readonly": false + }, + + "SCRIPT LOAD": { + "arguments": "script", + "group": "Script", + "readonly": false + }, + + "SCRIPT EXISTS": { + "arguments": "script [script ...]", + "group": "Script", + "readonly": false + }, + + "SCRIPT FLUSH": { + "arguments" : "-", + "group": "Script", + "readonly": false } + } diff --git a/doc/commands.md b/doc/commands.md index fe5437f..d3c3b39 100644 --- a/doc/commands.md +++ b/doc/commands.md @@ -70,7 +70,7 @@ Table of Contents - [SINTERSTORE destination key [key ...]](#sinterstore-destination-key-key-) - [SISMEMBER key member](#sismember-key-member) - [SMEMBERS key](#smembers-key) - - [SREM key member [member]](#srem-key-member-member-) + - [SREM key member [member ...]](#srem-key-member-member-) - [SUNION key [key ...]](#sunion-key-key-) - [SUNIONSTORE destination key [key ...]](#sunionstore-destination-key-key-) - [SCLEAR key](#sclear-key) @@ -133,7 +133,12 @@ Table of Contents - [BEGIN](#begin) - [ROLLBACK](#rollback) - [COMMIT](#commit) - +- [Script](#script) + - [EVAL script numkeys key [key ...] arg [arg ...]](#eval-script-numkeys-key-key--arg-arg-) + - [EVALSHA sha1 numkeys key [key ...] arg [arg ...]](#evalsha-sha1-numkeys-key-key--arg-arg-) + - [SCRIPT LOAD script](#script-load-script) + - [SCRIPT EXISTS script [script ...]](#script-exists-script-script-) + - [SCRIPT FLUSH](#script-flush) ## KV @@ -2562,4 +2567,21 @@ ledis> GET HELLO "WORLD" ``` +## Script + +LedisDB's script is refer to Redis, you can see more [http://redis.io/commands/eval](http://redis.io/commands/eval) + +You must notice that executing lua will block any other write operations. + +### EVAL script numkeys key [key ...] arg [arg ...] + +### EVALSHA sha1 numkeys key [key ...] arg [arg ...] + +### SCRIPT LOAD script + +### SCRIPT EXISTS script [script ...] + +### SCRIPT FLUSH + + Thanks [doctoc](http://doctoc.herokuapp.com/) From d9a8da770d1967eb2cf883bc81e231ba9edf3f55 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 3 Sep 2014 07:49:46 +0800 Subject: [PATCH 14/32] return lua error in eval --- server/cmd_script.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/cmd_script.go b/server/cmd_script.go index 1325477..e5c3264 100644 --- a/server/cmd_script.go +++ b/server/cmd_script.go @@ -99,6 +99,11 @@ func evalGenericCommand(c *client, evalSha1 bool) error { } else { r := luaReplyToLedisReply(l) m.Close() + + if v, ok := r.(error); ok { + return v + } + writeValue(c.resp, r) } From 9628817ca43cacc91ef76a4105e8b8dd2d6007c4 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 3 Sep 2014 09:03:49 +0800 Subject: [PATCH 15/32] replace lua error error may cause c stack overflow https://github.com/aarzilli/golua/issues/24 --- README.md | 1 + server/script.go | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/README.md b/README.md index 726de3a..44d5309 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ Choosing a store database to use is very simple, you have two ways: + You must known that changing store database runtime is very dangerous, LedisDB will not guarantee the data validation if you do it. + Begin a transaction will block any other write operators before you call `commit` or `rollback`. Don't use long-time transaction. ++ `pcall` and `xpcall` are not supported in lua, you can see the readme in [golua](https://github.com/aarzilli/golua). ## Configuration diff --git a/server/script.go b/server/script.go index c2c8ff2..7ecdfc9 100644 --- a/server/script.go +++ b/server/script.go @@ -159,6 +159,8 @@ func (app *App) openScript() { l.OpenTable() l.OpenPackage() + l.Register("error", luaErrorHandler) + s.l = l s.c = newClient(app) s.c.db = nil @@ -223,6 +225,11 @@ func delMapState(l *lua.State) { delete(mapState, l) } +func luaErrorHandler(l *lua.State) int { + msg := l.ToString(1) + panic(fmt.Errorf(msg)) +} + func luaCall(l *lua.State) int { return luaCallGenericCommand(l) } From b4c66c7eed43c96c213b3c5f194282e1feaf72b5 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 3 Sep 2014 17:00:03 +0800 Subject: [PATCH 16/32] bugfix for script command --- client/ledis-py/ledis/client.py | 6 ++--- client/nodejs/ledis/lib/commands.js | 4 +--- client/openresty/ledis.lua | 5 +---- server/cmd_script.go | 34 +++++++++++++++++------------ server/cmd_script_test.go | 8 +++---- 5 files changed, 29 insertions(+), 28 deletions(-) diff --git a/client/ledis-py/ledis/client.py b/client/ledis-py/ledis/client.py index 6c8a9fb..cffac29 100644 --- a/client/ledis-py/ledis/client.py +++ b/client/ledis-py/ledis/client.py @@ -912,13 +912,13 @@ class Ledis(object): return self.execute_command('EVALSHA', sha1, len(keys), *keys, *args) def scriptload(self, script): - return self.execute_command('SCRIPT LOAD', script) + return self.execute_command('SCRIPT', 'LOAD', script) def scriptexists(self, *args): - return self.execute_command('SCRIPT EXISTS', *args) + return self.execute_command('SCRIPT', 'EXISTS', *args) def scriptflush(self): - return self.execute_command('SCRIPT FLUSH') + return self.execute_command('SCRIPT', 'FLUSH') class Transaction(Ledis): def __init__(self, connection_pool, response_callbacks): diff --git a/client/nodejs/ledis/lib/commands.js b/client/nodejs/ledis/lib/commands.js index 26408e2..7271f3d 100644 --- a/client/nodejs/ledis/lib/commands.js +++ b/client/nodejs/ledis/lib/commands.js @@ -131,7 +131,5 @@ module.exports = [ "eval", "evalsha", - "script load", - "script exists", - "script flush", + "script", ]; diff --git a/client/openresty/ledis.lua b/client/openresty/ledis.lua index 2778a56..973d489 100644 --- a/client/openresty/ledis.lua +++ b/client/openresty/ledis.lua @@ -156,9 +156,7 @@ local commands = { -- [[script]] "eval", "evalsha", - "script load", - "script exists", - "script flush" + "script" } @@ -398,7 +396,6 @@ function _M.hmset(self, hashname, ...) return _do_cmd(self, "hmset", hashname, ...) end - function _M.array_to_hash(self, t) local n = #t -- print("n = ", n) diff --git a/server/cmd_script.go b/server/cmd_script.go index e5c3264..d2a3a1c 100644 --- a/server/cmd_script.go +++ b/server/cmd_script.go @@ -133,15 +133,19 @@ func scriptCommand(c *client) error { args := c.args - switch strings.ToLower(c.cmd) { - case "script load": + if len(args) < 1 { + return ErrCmdParams + } + + switch strings.ToLower(ledis.String(args[0])) { + case "load": return scriptLoadCommand(c) - case "script exists": + case "exists": return scriptExistsCommand(c) - case "script flush": + case "flush": return scriptFlushCommand(c) default: - return fmt.Errorf("invalid script cmd %s", args[0]) + return fmt.Errorf("invalid script %s", args[0]) } return nil @@ -151,14 +155,14 @@ func scriptLoadCommand(c *client) error { s := c.app.s l := s.l - if len(c.args) != 1 { + if len(c.args) != 2 { return ErrCmdParams } - h := sha1.Sum(c.args[0]) + h := sha1.Sum(c.args[1]) key := hex.EncodeToString(h[0:20]) - if r := l.LoadString(ledis.String(c.args[0])); r != 0 { + if r := l.LoadString(ledis.String(c.args[1])); r != 0 { err := fmt.Errorf("%s", l.ToString(-1)) l.Pop(1) return err @@ -176,12 +180,12 @@ func scriptLoadCommand(c *client) error { func scriptExistsCommand(c *client) error { s := c.app.s - if len(c.args) < 1 { + if len(c.args) < 2 { return ErrCmdParams } - ay := make([]interface{}, len(c.args)) - for i, n := range c.args { + ay := make([]interface{}, len(c.args[1:])) + for i, n := range c.args[1:] { if _, ok := s.chunks[ledis.String(n)]; ok { ay[i] = int64(1) } else { @@ -197,6 +201,10 @@ func scriptFlushCommand(c *client) error { s := c.app.s l := s.l + if len(c.args) != 1 { + return ErrCmdParams + } + for n, _ := range s.chunks { l.PushNil() l.SetGlobal(n) @@ -212,7 +220,5 @@ func scriptFlushCommand(c *client) error { func init() { register("eval", evalCommand) register("evalsha", evalshaCommand) - register("script load", scriptCommand) - register("script flush", scriptCommand) - register("script exists", scriptCommand) + register("script", scriptCommand) } diff --git a/server/cmd_script_test.go b/server/cmd_script_test.go index d652e08..ae3e713 100644 --- a/server/cmd_script_test.go +++ b/server/cmd_script_test.go @@ -25,7 +25,7 @@ func TestCmdEval(t *testing.T) { var sha1 string var err error - if sha1, err = ledis.String(c.Do("script load", "return {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}")); err != nil { + if sha1, err = ledis.String(c.Do("script", "load", "return {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}")); err != nil { t.Fatal(err) } else if len(sha1) != 40 { t.Fatal(sha1) @@ -37,19 +37,19 @@ func TestCmdEval(t *testing.T) { t.Fatal(fmt.Sprintf("%v", v)) } - if ay, err := ledis.Values(c.Do("script exists", sha1, "01234567890123456789")); err != nil { + if ay, err := ledis.Values(c.Do("script", "exists", sha1, "01234567890123456789")); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(ay, []interface{}{int64(1), int64(0)}) { t.Fatal(fmt.Sprintf("%v", ay)) } - if ok, err := ledis.String(c.Do("script flush")); err != nil { + if ok, err := ledis.String(c.Do("script", "flush")); err != nil { t.Fatal(err) } else if ok != "OK" { t.Fatal(ok) } - if ay, err := ledis.Values(c.Do("script exists", sha1)); err != nil { + if ay, err := ledis.Values(c.Do("script", "exists", sha1)); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(ay, []interface{}{int64(0)}) { t.Fatal(fmt.Sprintf("%v", ay)) From 855682ff2be7d4fc21eb4c34a6397ac3ab131463 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 3 Sep 2014 17:01:06 +0800 Subject: [PATCH 17/32] make cli output pretty, needs more work --- cmd/ledis-cli/main.go | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/cmd/ledis-cli/main.go b/cmd/ledis-cli/main.go index 70f0b93..be509f3 100644 --- a/cmd/ledis-cli/main.go +++ b/cmd/ledis-cli/main.go @@ -65,22 +65,19 @@ func main() { if cmd == "help" || cmd == "?" { printHelp(cmds) } else { - if len(cmds) == 2 && strings.ToLower(cmds[0]) == "select" { - if db, _ := strconv.Atoi(cmds[1]); db < 16 && db >= 0 { - *dbn = db - } - - } - r, err := c.Do(cmds[0], args...) + if err == nil && strings.ToLower(cmds[0]) == "select" { + *dbn, _ = strconv.Atoi(cmds[1]) + } + if err != nil { fmt.Printf("%s", err.Error()) } else { if cmd == "info" { printInfo(r.([]byte)) } else { - printReply(cmd, r) + printReply(0, r) } } @@ -95,7 +92,7 @@ func printInfo(s []byte) { fmt.Printf("%s", s) } -func printReply(cmd string, reply interface{}) { +func printReply(level int, reply interface{}) { switch reply := reply.(type) { case int64: fmt.Printf("(integer) %d", reply) @@ -108,13 +105,14 @@ func printReply(cmd string, reply interface{}) { case ledis.Error: fmt.Printf("%s", string(reply)) case []interface{}: + if level > 0 { + //todo for better pretty + fmt.Printf("%q", reply) + return + } for i, v := range reply { fmt.Printf("%d) ", i+1) - if v == nil { - fmt.Printf("(nil)") - } else { - fmt.Printf("%q", v) - } + printReply(level+1, v) if i != len(reply)-1 { fmt.Printf("\n") } From 0f4c6281460c5816f6d1d557c6c98a39574419d9 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 3 Sep 2014 17:18:58 +0800 Subject: [PATCH 18/32] update py client --- client/ledis-py/ledis/client.py | 10 +++++++--- client/ledis-py/tests/test_cmd_script.py | 3 ++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/client/ledis-py/ledis/client.py b/client/ledis-py/ledis/client.py index cffac29..034f2e3 100644 --- a/client/ledis-py/ledis/client.py +++ b/client/ledis-py/ledis/client.py @@ -906,11 +906,15 @@ class Ledis(object): return self.execute_command("BSCAN", key, match, count) def eval(self, script, keys, *args): - return self.execute_command('EVAL', script, len(keys), *keys, *args) + n = len(keys) + args = list_or_args(keys, args) + return self.execute_command('EVAL', script, n, *args) def evalsha(self, sha1, keys, *args): - return self.execute_command('EVALSHA', sha1, len(keys), *keys, *args) - + n = len(keys) + args = list_or_args(keys, args) + return self.execute_command('EVALSHA', sha1, n, *args) + def scriptload(self, script): return self.execute_command('SCRIPT', 'LOAD', script) diff --git a/client/ledis-py/tests/test_cmd_script.py b/client/ledis-py/tests/test_cmd_script.py index b2e2f18..02d7630 100644 --- a/client/ledis-py/tests/test_cmd_script.py +++ b/client/ledis-py/tests/test_cmd_script.py @@ -19,6 +19,7 @@ class TestCmdScript(unittest.TestCase): def tearDown(self): pass - + def testEval(self): + assert l.eval("return {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}", ["key1", "key2"], "first", "second") == ["key1", "key2", "first", "second"] \ No newline at end of file From c516efa800e9b5912a23a1291b03726c03f79679 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 3 Sep 2014 17:52:38 +0800 Subject: [PATCH 19/32] cli output pretty --- cmd/ledis-cli/main.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cmd/ledis-cli/main.go b/cmd/ledis-cli/main.go index be509f3..ba97b30 100644 --- a/cmd/ledis-cli/main.go +++ b/cmd/ledis-cli/main.go @@ -105,13 +105,14 @@ func printReply(level int, reply interface{}) { case ledis.Error: fmt.Printf("%s", string(reply)) case []interface{}: - if level > 0 { - //todo for better pretty - fmt.Printf("%q", reply) - return - } for i, v := range reply { - fmt.Printf("%d) ", i+1) + if i != 0 { + fmt.Printf("%s", strings.Repeat(" ", level*4)) + } + + s := fmt.Sprintf("%d) ", i+1) + fmt.Printf("%-4s", s) + printReply(level+1, v) if i != len(reply)-1 { fmt.Printf("\n") From 210d51258b403aa3d8a6a78d9b8a22ff821f20aa Mon Sep 17 00:00:00 2001 From: holys Date: Wed, 3 Sep 2014 22:00:31 +0800 Subject: [PATCH 20/32] info bug fix --- client/ledis-py/ledis/client.py | 39 ++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/client/ledis-py/ledis/client.py b/client/ledis-py/ledis/client.py index 034f2e3..5547d6a 100644 --- a/client/ledis-py/ledis/client.py +++ b/client/ledis-py/ledis/client.py @@ -64,6 +64,30 @@ def int_or_none(response): return int(response) +def parse_info(response): + + info = {} + response = nativestr(response) + + def get_value(value): + if ',' not in value or '=' not in value: + try: + if '.' in value: + return float(value) + else: + return int(value) + except ValueError: + return value + + for line in response.splitlines(): + if line and not line.startswith('#'): + if line.find(':') != -1: + key, value = line.split(':', 1) + info[key] = get_value(value) + + return info + + class Ledis(object): """ Implementation of the Redis protocol. @@ -111,7 +135,8 @@ class Ledis(object): 'HGETALL': lambda r: r and pairs_to_dict(r) or {}, 'PING': lambda r: nativestr(r) == 'PONG', 'SET': lambda r: r and nativestr(r) == 'OK', - }, + 'INFO': parse_info, + } ) @@ -219,8 +244,15 @@ class Ledis(object): db = 0 return self.execute_command('SELECT', db) - def info(self, section): - return self.execute_command('PING', section) + def info(self, section=None): + """ + Return + """ + + if section is None: + return self.execute_command("INFO") + else: + return self.execute_command('INFO', section) def flushall(self): return self.execute_command('FLUSHALL') @@ -924,6 +956,7 @@ class Ledis(object): def scriptflush(self): return self.execute_command('SCRIPT', 'FLUSH') + class Transaction(Ledis): def __init__(self, connection_pool, response_callbacks): self.connection_pool = connection_pool From 6f3f04970703a567adc08b14771540cd9bee1dc6 Mon Sep 17 00:00:00 2001 From: holys Date: Wed, 3 Sep 2014 23:01:10 +0800 Subject: [PATCH 21/32] add tests for py client --- client/ledis-py/tests/all.sh | 7 +++++ client/ledis-py/tests/test_cmd_script.py | 36 ++++++++++++++++++++++-- 2 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 client/ledis-py/tests/all.sh diff --git a/client/ledis-py/tests/all.sh b/client/ledis-py/tests/all.sh new file mode 100644 index 0000000..163ee04 --- /dev/null +++ b/client/ledis-py/tests/all.sh @@ -0,0 +1,7 @@ +dbs=(leveldb rocksdb hyperleveldb goleveldb boltdb lmdb) +for db in "${dbs[@]}" +do + ledis-server -db_name=$db & + py.test + killall ledis-server +done diff --git a/client/ledis-py/tests/test_cmd_script.py b/client/ledis-py/tests/test_cmd_script.py index 02d7630..4a08cb5 100644 --- a/client/ledis-py/tests/test_cmd_script.py +++ b/client/ledis-py/tests/test_cmd_script.py @@ -12,14 +12,44 @@ from util import expire_at, expire_at_seconds l = ledis.Ledis(port=6380) +simple_script = "return {KEYS[1], KEYS[2], ARGV[1], ARGV[2]}" + + class TestCmdScript(unittest.TestCase): def setUp(self): pass def tearDown(self): - pass + l.flushdb() + + def test_eval(self): + assert l.eval(simple_script, ["key1", "key2"], "first", "second") == ["key1", "key2", "first", "second"] + + def test_evalsha(self): + sha1 = l.scriptload(simple_script) + assert len(sha1) == 40 + + assert l.evalsha(sha1, ["key1", "key2"], "first", "second") == ["key1", "key2", "first", "second"] + + def test_scriptload(self): + sha1 = l.scriptload(simple_script) + assert len(sha1) == 40 + + def test_scriptexists(self): + sha1 = l.scriptload(simple_script) + assert l.scriptexists(sha1) == [1L] + + def test_scriptflush(self): + sha1 = l.scriptload(simple_script) + assert l.scriptexists(sha1) == [1L] + assert l.scriptflush() == 'OK' + + assert l.scriptexists(sha1) == [0L] + + + + + - def testEval(self): - assert l.eval("return {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}", ["key1", "key2"], "first", "second") == ["key1", "key2", "first", "second"] \ No newline at end of file From ba9f611488337352447b785a3f150b7fb87a9073 Mon Sep 17 00:00:00 2001 From: holys Date: Thu, 4 Sep 2014 12:53:19 +0800 Subject: [PATCH 22/32] *scan commands bug fix; add more tests --- Makefile | 3 + client/ledis-py/Makefile | 5 ++ client/ledis-py/ledis/client.py | 49 ++++++++++---- client/ledis-py/ledis/connection.py | 20 ++++++ client/ledis-py/tests/all.sh | 2 +- client/ledis-py/tests/test_cmd_bit.py | 3 +- client/ledis-py/tests/test_cmd_hash.py | 2 +- client/ledis-py/tests/test_cmd_kv.py | 2 +- client/ledis-py/tests/test_cmd_list.py | 2 +- client/ledis-py/tests/test_cmd_set.py | 2 +- client/ledis-py/tests/test_cmd_zset.py | 2 +- client/ledis-py/tests/test_others.py | 94 +++++++++++++++++++++++++- client/ledis-py/tests/test_tx.py | 12 +++- 13 files changed, 173 insertions(+), 25 deletions(-) create mode 100644 client/ledis-py/Makefile diff --git a/Makefile b/Makefile index e2725be..d98fc4d 100644 --- a/Makefile +++ b/Makefile @@ -23,3 +23,6 @@ clean: test: go test -tags '$(GO_BUILD_TAGS)' ./... + +pytest: + sh client/ledis-py/tests/all.sh diff --git a/client/ledis-py/Makefile b/client/ledis-py/Makefile new file mode 100644 index 0000000..9f53bed --- /dev/null +++ b/client/ledis-py/Makefile @@ -0,0 +1,5 @@ +.PHONY: test + + +test: + sh tests/all.sh diff --git a/client/ledis-py/ledis/client.py b/client/ledis-py/ledis/client.py index 5547d6a..6615c91 100644 --- a/client/ledis-py/ledis/client.py +++ b/client/ledis-py/ledis/client.py @@ -4,7 +4,7 @@ import time as mod_time from itertools import chain, starmap from ledis._compat import (b, izip, imap, iteritems, basestring, long, nativestr, bytes) -from ledis.connection import ConnectionPool, UnixDomainSocketConnection +from ledis.connection import ConnectionPool, UnixDomainSocketConnection, Token from ledis.exceptions import ( ConnectionError, DataError, @@ -87,6 +87,7 @@ def parse_info(response): return info +# def parse_lscan(response, ) class Ledis(object): """ @@ -138,6 +139,7 @@ class Ledis(object): 'INFO': parse_info, } + ) @classmethod @@ -382,8 +384,21 @@ class Ledis(object): "Removes an expiration on name" return self.execute_command('PERSIST', name) - def scan(self, key, match = "", count = 10): - return self.execute_command("SCAN", key, match, count) + def scan(self, key="" , match=None, count=10): + pieces = [key] + if match is not None: + pieces.extend(["MATCH", match]) + + pieces.extend(["COUNT", count]) + + return self.execute_command("SCAN", *pieces) + + def scan_iter(self, match=None, count=10): + key = "" + while key != "": + key, data = self.scan(key=key, match=match, count=count) + for item in data: + yield item #### LIST COMMANDS #### def lindex(self, name, index): @@ -460,8 +475,8 @@ class Ledis(object): "Removes an expiration on ``name``" return self.execute_command('LPERSIST', name) - def lscan(self, key, match = "", count = 10): - return self.execute_command("LSCAN", key, match, count) + def lscan(self, key="", match=None, count=10): + return self.scan_generic("LSCAN", key=key, match=match, count=count) #### SET COMMANDS #### @@ -560,8 +575,8 @@ class Ledis(object): "Removes an expiration on name" return self.execute_command('SPERSIST', name) - def sscan(self, key, match = "", count = 10): - return self.execute_command("SSCAN", key, match, count) + def sscan(self, key="", match=None, count = 10): + return self.scan_generic("SSCAN", key=key, match=match, count=count) #### SORTED SET COMMANDS #### @@ -759,9 +774,17 @@ class Ledis(object): "Removes an expiration on name" return self.execute_command('ZPERSIST', name) - def zscan(self, key, match = "", count = 10): - return self.execute_command("ZSCAN", key, match, count) + def scan_generic(self, scan_type, key="", match=None, count=10): + pieces = [key] + if match is not None: + pieces.extend([Token("MATCH"), match]) + pieces.extend([Token("count"), count]) + scan_type = scan_type.upper() + return self.execute_command(scan_type, *pieces) + + def zscan(self, key="", match=None, count=10): + return self.scan_generic("ZSCAN", key=key, match=match, count=count) #### HASH COMMANDS #### def hdel(self, name, *keys): @@ -855,8 +878,8 @@ class Ledis(object): "Removes an expiration on name" return self.execute_command('HPERSIST', name) - def hscan(self, key, match = "", count = 10): - return self.execute_command("HSCAN", key, match, count) + def hscan(self, key="", match=None, count=10): + return self.scan_generic("HSCAN", key=key, match=match, count=count) ### BIT COMMANDS @@ -934,8 +957,8 @@ class Ledis(object): "Removes an expiration on name" return self.execute_command('BPERSIST', name) - def bscan(self, key, match = "", count = 10): - return self.execute_command("BSCAN", key, match, count) + def bscan(self, key="", match=None, count=10): + return self.scan_generic("BSCAN", key=key, match=match, count=count) def eval(self, script, keys, *args): n = len(keys) diff --git a/client/ledis-py/ledis/connection.py b/client/ledis-py/ledis/connection.py index 4a39317..5372838 100644 --- a/client/ledis-py/ledis/connection.py +++ b/client/ledis-py/ledis/connection.py @@ -588,3 +588,23 @@ class BlockingConnectionPool(object): timeout=self.timeout, connection_class=self.connection_class, queue_class=self.queue_class, **self.connection_kwargs) + + +class Token(object): + """ + Literal strings in Redis commands, such as the command names and any + hard-coded arguments are wrapped in this class so we know not to apply + and encoding rules on them. + """ + def __init__(self, value): + if isinstance(value, Token): + value = value.value + self.value = value + + def __repr__(self): + return self.value + + def __str__(self): + return self.value + + diff --git a/client/ledis-py/tests/all.sh b/client/ledis-py/tests/all.sh index 163ee04..8b7ae0f 100644 --- a/client/ledis-py/tests/all.sh +++ b/client/ledis-py/tests/all.sh @@ -1,7 +1,7 @@ dbs=(leveldb rocksdb hyperleveldb goleveldb boltdb lmdb) for db in "${dbs[@]}" do + killall ledis-server ledis-server -db_name=$db & py.test - killall ledis-server done diff --git a/client/ledis-py/tests/test_cmd_bit.py b/client/ledis-py/tests/test_cmd_bit.py index c1976a7..3d8d0c5 100644 --- a/client/ledis-py/tests/test_cmd_bit.py +++ b/client/ledis-py/tests/test_cmd_bit.py @@ -17,8 +17,7 @@ class TestCmdBit(unittest.TestCase): pass def tearDown(self): - l.bdelete('a') - l.bdelete('non_exists_key') + l.flushdb() def test_bget(self): "bget is the same as get in K/V commands" diff --git a/client/ledis-py/tests/test_cmd_hash.py b/client/ledis-py/tests/test_cmd_hash.py index 8a89af2..5efc86f 100644 --- a/client/ledis-py/tests/test_cmd_hash.py +++ b/client/ledis-py/tests/test_cmd_hash.py @@ -19,7 +19,7 @@ class TestCmdHash(unittest.TestCase): pass def tearDown(self): - l.hmclear('myhash', 'a') + l.flushdb() def test_hdel(self): diff --git a/client/ledis-py/tests/test_cmd_kv.py b/client/ledis-py/tests/test_cmd_kv.py index 774b800..b556c7a 100644 --- a/client/ledis-py/tests/test_cmd_kv.py +++ b/client/ledis-py/tests/test_cmd_kv.py @@ -18,7 +18,7 @@ class TestCmdKv(unittest.TestCase): pass def tearDown(self): - l.delete('a', 'b', 'c', 'non_exist_key') + l.flushdb() def test_decr(self): assert l.delete('a') == 1 diff --git a/client/ledis-py/tests/test_cmd_list.py b/client/ledis-py/tests/test_cmd_list.py index f87b8a1..065cee5 100644 --- a/client/ledis-py/tests/test_cmd_list.py +++ b/client/ledis-py/tests/test_cmd_list.py @@ -18,7 +18,7 @@ class TestCmdList(unittest.TestCase): pass def tearDown(self): - l.lmclear('mylist', 'mylist1', 'mylist2') + l.flushdb() def test_lindex(self): l.rpush('mylist', '1', '2', '3') diff --git a/client/ledis-py/tests/test_cmd_set.py b/client/ledis-py/tests/test_cmd_set.py index 0d2eec9..e98a762 100644 --- a/client/ledis-py/tests/test_cmd_set.py +++ b/client/ledis-py/tests/test_cmd_set.py @@ -20,7 +20,7 @@ class TestCmdSet(unittest.TestCase): pass def tearDown(self): - l.smclear('a', 'b', 'c') + l.flushdb() def test_sadd(self): members = set([b('1'), b('2'), b('3')]) diff --git a/client/ledis-py/tests/test_cmd_zset.py b/client/ledis-py/tests/test_cmd_zset.py index 08233fc..9277fce 100644 --- a/client/ledis-py/tests/test_cmd_zset.py +++ b/client/ledis-py/tests/test_cmd_zset.py @@ -17,7 +17,7 @@ class TestCmdZset(unittest.TestCase): pass def tearDown(self): - l.zclear('a') + l.flushdb() def test_zadd(self): l.zadd('a', a1=1, a2=2, a3=3) diff --git a/client/ledis-py/tests/test_others.py b/client/ledis-py/tests/test_others.py index d57d332..2cd7110 100644 --- a/client/ledis-py/tests/test_others.py +++ b/client/ledis-py/tests/test_others.py @@ -10,13 +10,14 @@ from ledis._compat import b from ledis import ResponseError l = ledis.Ledis(port=6380) +dbs = ["leveldb", "rocksdb", "goleveldb", "hyperleveldb", "lmdb", "boltdb"] class TestOtherCommands(unittest.TestCase): def setUp(self): pass def tearDown(self): - pass + l.flushdb() # server information def test_echo(self): @@ -28,4 +29,93 @@ class TestOtherCommands(unittest.TestCase): def test_select(self): assert l.select('1') assert l.select('15') - self.assertRaises(ResponseError, lambda: l.select('16')) \ No newline at end of file + self.assertRaises(ResponseError, lambda: l.select('16')) + + + def test_info(self): + info1 = l.info() + assert info1.get("db_name") in dbs + info2 = l.info(section="server") + assert info2.get("os") in ["linux", "darwin"] + + def test_flushdb(self): + l.set("a", 1) + assert l.flushdb() == "OK" + assert l.get("a") is None + + def test_flushall(self): + l.select(1) + l.set("a", 1) + assert l.get("a") == b("1") + + l.select(10) + l.set("a", 1) + assert l.get("a") == b("1") + + assert l.flushall() == "OK" + + assert l.get("a") is None + l.select(1) + assert l.get("a") is None + + + # test *scan commands + + def check_keys(self, scan_type): + d = { + "scan": l.scan, + "sscan": l.sscan, + "lscan": l.lscan, + "hscan": l.hscan, + "zscan": l.zscan, + "bscan": l.bscan + } + + key, keys = d[scan_type]() + assert key == "" + assert set(keys) == set([b("a"), b("b"), b("c")]) + + _, keys = d[scan_type](match="a") + assert set(keys) == set([b("a")]) + + _, keys = d[scan_type](key="a") + assert set(keys) == set([b("b"), b("c")]) + + + def test_scan(self): + d = {"a":1, "b":2, "c": 3} + l.mset(d) + self.check_keys("scan") + + + def test_lscan(self): + l.rpush("a", 1) + l.rpush("b", 1) + l.rpush("c", 1) + self.check_keys("lscan") + + + def test_hscan(self): + l.hset("a", "hello", "world") + l.hset("b", "hello", "world") + l.hset("c", "hello", "world") + self.check_keys("hscan") + + def test_sscan(self): + l.sadd("a", 1) + l.sadd("b", 2) + l.sadd("c", 3) + self.check_keys("sscan") + + def test_zscan(self): + l.zadd("a", 1, "a") + l.zadd("b", 1, "a") + l.zadd("c", 1, "a") + self.check_keys("zscan") + + def test_bscan(self): + l.bsetbit("a", 1, 1) + l.bsetbit("b", 1, 1) + l.bsetbit("c", 1, 1) + self.check_keys("bscan") + diff --git a/client/ledis-py/tests/test_tx.py b/client/ledis-py/tests/test_tx.py index b589dc7..cfbab20 100644 --- a/client/ledis-py/tests/test_tx.py +++ b/client/ledis-py/tests/test_tx.py @@ -4,14 +4,21 @@ sys.path.append("..") import ledis +global_l = ledis.Ledis() + +#db that do not support transaction +dbs = ["leveldb", "rocksdb", "hyperleveldb", "goleveldb"] +check = global_l.info().get("db_name") in dbs + class TestTx(unittest.TestCase): def setUp(self): self.l = ledis.Ledis(port=6380) def tearDown(self): - self.l.delete("a") - + self.l.flushdb() + + @unittest.skipIf(check, reason="db not support transaction") def test_commit(self): tx = self.l.tx() self.l.set("a", "no-tx") @@ -24,6 +31,7 @@ class TestTx(unittest.TestCase): tx.commit() assert self.l.get("a") == "tx" + @unittest.skipIf(check, reason="db not support transaction") def test_rollback(self): tx = self.l.tx() self.l.set("a", "no-tx") From 9eaab3fba2ede4f3952aeefd4adca6518d06c351 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 4 Sep 2014 14:00:32 +0800 Subject: [PATCH 23/32] adjust benchmark --- cmd/ledis-benchmark/main.go | 72 +++++++++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 10 deletions(-) diff --git a/cmd/ledis-benchmark/main.go b/cmd/ledis-benchmark/main.go index 0084df1..55f9613 100644 --- a/cmd/ledis-benchmark/main.go +++ b/cmd/ledis-benchmark/main.go @@ -6,6 +6,7 @@ import ( "github.com/siddontang/ledisdb/client/go/ledis" "math/rand" "sync" + "sync/atomic" "time" ) @@ -13,6 +14,7 @@ var ip = flag.String("ip", "127.0.0.1", "redis/ledis/ssdb server ip") var port = flag.Int("port", 6380, "redis/ledis/ssdb server port") var number = flag.Int("n", 1000, "request number") var clients = flag.Int("c", 50, "number of clients") +var reverse = flag.Bool("rev", false, "enable zset rev benchmark") var wg sync.WaitGroup @@ -52,9 +54,14 @@ func bench(cmd string, f func()) { fmt.Printf("%s: %0.2f requests per second\n", cmd, (float64(*number) / delta)) } +var kvSetBase int64 = 0 +var kvGetBase int64 = 0 +var kvIncrBase int64 = 0 +var kvDelBase int64 = 0 + func benchSet() { f := func() { - n := rand.Int() + n := atomic.AddInt64(&kvSetBase, 1) waitBench("set", n, n) } @@ -62,6 +69,15 @@ func benchSet() { } func benchGet() { + f := func() { + n := atomic.AddInt64(&kvGetBase, 1) + waitBench("get", n) + } + + bench("get", f) +} + +func benchRandGet() { f := func() { n := rand.Int() waitBench("get", n) @@ -72,13 +88,22 @@ func benchGet() { func benchIncr() { f := func() { - n := rand.Int() + n := atomic.AddInt64(&kvIncrBase, 1) waitBench("incr", n) } bench("incr", f) } +func benchDel() { + f := func() { + n := atomic.AddInt64(&kvDelBase, 1) + waitBench("del", n) + } + + bench("del", f) +} + func benchPushList() { f := func() { n := rand.Int() @@ -120,9 +145,14 @@ func benchPopList() { bench("lpop", f) } +var hashSetBase int64 = 0 +var hashIncrBase int64 = 0 +var hashGetBase int64 = 0 +var hashDelBase int64 = 0 + func benchHset() { f := func() { - n := rand.Int() + n := atomic.AddInt64(&hashSetBase, 1) waitBench("hset", "myhashkey", n, n) } @@ -131,7 +161,7 @@ func benchHset() { func benchHIncr() { f := func() { - n := rand.Int() + n := atomic.AddInt64(&hashIncrBase, 1) waitBench("hincrby", "myhashkey", n, 1) } @@ -139,6 +169,15 @@ func benchHIncr() { } func benchHGet() { + f := func() { + n := atomic.AddInt64(&hashGetBase, 1) + waitBench("hget", "myhashkey", n) + } + + bench("hget", f) +} + +func benchHRandGet() { f := func() { n := rand.Int() waitBench("hget", "myhashkey", n) @@ -149,16 +188,20 @@ func benchHGet() { func benchHDel() { f := func() { - n := rand.Int() + n := atomic.AddInt64(&hashDelBase, 1) waitBench("hdel", "myhashkey", n) } bench("hdel", f) } +var zsetAddBase int64 = 0 +var zsetDelBase int64 = 0 +var zsetIncrBase int64 = 0 + func benchZAdd() { f := func() { - n := rand.Int() + n := atomic.AddInt64(&zsetAddBase, 1) waitBench("zadd", "myzsetkey", n, n) } @@ -167,7 +210,7 @@ func benchZAdd() { func benchZDel() { f := func() { - n := rand.Int() + n := atomic.AddInt64(&zsetDelBase, 1) waitBench("zrem", "myzsetkey", n) } @@ -176,7 +219,7 @@ func benchZDel() { func benchZIncr() { f := func() { - n := rand.Int() + n := atomic.AddInt64(&zsetIncrBase, 1) waitBench("zincrby", "myzsetkey", 1, n) } @@ -239,6 +282,8 @@ func main() { benchSet() benchIncr() benchGet() + benchRandGet() + benchDel() benchPushList() benchRangeList10() @@ -249,13 +294,20 @@ func main() { benchHset() benchHGet() benchHIncr() + benchHRandGet() benchHDel() benchZAdd() benchZIncr() benchZRangeByRank() benchZRangeByScore() - benchZRevRangeByRank() - benchZRevRangeByScore() + + //rev is too slow in leveldb, rocksdb or other + //maybe disable for huge data benchmark + if *reverse == true { + benchZRevRangeByRank() + benchZRevRangeByScore() + } + benchZDel() } From 2efd35de5a59524caa4b3fda292a5c6adee48cb2 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 4 Sep 2014 14:02:47 +0800 Subject: [PATCH 24/32] update config --- config/config.go | 1 + config/config.toml | 2 ++ etc/ledis.conf | 2 ++ 3 files changed, 5 insertions(+) diff --git a/config/config.go b/config/config.go index 48a45f7..ca93d29 100644 --- a/config/config.go +++ b/config/config.go @@ -102,6 +102,7 @@ func NewConfigDefault() *Config { // disable access log cfg.AccessLog = "" + cfg.LMDB.MapSize = 20 * 1024 * 1024 cfg.LMDB.NoSync = true return cfg diff --git a/config/config.toml b/config/config.toml index 573db9a..2a3a246 100644 --- a/config/config.toml +++ b/config/config.toml @@ -22,6 +22,8 @@ slaveof = "" # goleveldb # lmdb # boltdb +# hyperleveldb +# memory # db_name = "leveldb" diff --git a/etc/ledis.conf b/etc/ledis.conf index 2097f65..d3adbd8 100644 --- a/etc/ledis.conf +++ b/etc/ledis.conf @@ -24,6 +24,8 @@ slaveof = "" # goleveldb # lmdb # boltdb +# hyperleveldb +# memory # db_name = "leveldb" From c77f938b147d10d6c20129bde5048f4747267fdf Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 4 Sep 2014 14:02:55 +0800 Subject: [PATCH 25/32] update read me --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 44d5309..07437da 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ LedisDB now supports multiple databases as backend to store data, you can test a + Rich data structure: KV, List, Hash, ZSet, Bitmap, Set. + Stores lots of data, over the memory limit. -+ Various backend database to use: LevelDB, goleveldb, LMDB, RocksDB, BoltDB, HyperLevelDB. ++ Various backend database to use: LevelDB, goleveldb, LMDB, RocksDB, BoltDB, HyperLevelDB, Memory. + Supports transaction using LMDB or BotlDB. + Supports lua scripting. + Supports expiration and ttl. From 42f6a3679f70d987222a163ef1514f6b90287430 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 4 Sep 2014 14:55:26 +0800 Subject: [PATCH 26/32] bugfix for lmdb --- store/mdb/mdb.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/store/mdb/mdb.go b/store/mdb/mdb.go index 171c088..d5c3987 100644 --- a/store/mdb/mdb.go +++ b/store/mdb/mdb.go @@ -92,14 +92,16 @@ func (s Store) Repair(path string, c *config.Config) error { func (db MDB) Put(key, value []byte) error { itr := db.iterator(false) + defer itr.Close() itr.err = itr.c.Put(key, value, 0) itr.setState() - return itr.Close() + return itr.err } func (db MDB) BatchPut(writes []driver.Write) error { itr := db.iterator(false) + defer itr.Close() for _, w := range writes { if w.Value == nil { @@ -117,7 +119,7 @@ func (db MDB) BatchPut(writes []driver.Write) error { } itr.setState() - return itr.Close() + return itr.err } func (db MDB) Get(key []byte) ([]byte, error) { @@ -208,6 +210,8 @@ func (itr *MDBIterator) setState() { itr.err = nil } itr.valid = false + } else { + itr.valid = true } } From 6fac39c96b1718ab7c7cfa334e1651617a4d4f18 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 4 Sep 2014 14:57:26 +0800 Subject: [PATCH 27/32] bugfix for replication --- ledis/replication.go | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/ledis/replication.go b/ledis/replication.go index 2d8db48..421a5ab 100644 --- a/ledis/replication.go +++ b/ledis/replication.go @@ -49,6 +49,9 @@ func (b *replBatch) Commit() error { } } + b.events = [][]byte{} + b.lastHead = nil + return nil } @@ -64,6 +67,8 @@ func (l *Ledis) replicateEvent(b *replBatch, event []byte) error { return errInvalidBinLogEvent } + b.events = append(b.events, event) + logType := uint8(event[0]) switch logType { case BinLogTypePut: @@ -83,10 +88,6 @@ func (l *Ledis) replicatePutEvent(b *replBatch, event []byte) error { b.wb.Put(key, value) - if b.l.binlog != nil { - b.events = append(b.events, event) - } - return nil } @@ -98,16 +99,11 @@ func (l *Ledis) replicateDeleteEvent(b *replBatch, event []byte) error { b.wb.Delete(key) - if b.l.binlog != nil { - b.events = append(b.events, event) - } - return nil } func ReadEventFromReader(rb io.Reader, f func(head *BinLogHead, event []byte) error) error { head := &BinLogHead{} - var dataBuf bytes.Buffer var err error for { @@ -119,6 +115,8 @@ func ReadEventFromReader(rb io.Reader, f func(head *BinLogHead, event []byte) er } } + var dataBuf bytes.Buffer + if _, err = io.CopyN(&dataBuf, rb, int64(head.PayloadLen)); err != nil { return err } @@ -127,8 +125,6 @@ func ReadEventFromReader(rb io.Reader, f func(head *BinLogHead, event []byte) er if err != nil && err != ErrSkipEvent { return err } - - dataBuf.Reset() } return nil From 8ad6eabd6a1d1f288878d43a8d85fffe867380d8 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 4 Sep 2014 15:49:51 +0800 Subject: [PATCH 28/32] update benchmark --- cmd/ledis-benchmark/main.go | 102 ++++++++++++++++++------------------ 1 file changed, 50 insertions(+), 52 deletions(-) diff --git a/cmd/ledis-benchmark/main.go b/cmd/ledis-benchmark/main.go index 55f9613..d849e23 100644 --- a/cmd/ledis-benchmark/main.go +++ b/cmd/ledis-benchmark/main.go @@ -1,6 +1,7 @@ package main import ( + crand "crypto/rand" "flag" "fmt" "github.com/siddontang/ledisdb/client/go/ledis" @@ -15,6 +16,7 @@ var port = flag.Int("port", 6380, "redis/ledis/ssdb server port") var number = flag.Int("n", 1000, "request number") var clients = flag.Int("c", 50, "number of clients") var reverse = flag.Bool("rev", false, "enable zset rev benchmark") +var round = flag.Int("r", 1, "benchmark round number") var wg sync.WaitGroup @@ -61,8 +63,10 @@ var kvDelBase int64 = 0 func benchSet() { f := func() { + value := make([]byte, 100) + crand.Read(value) n := atomic.AddInt64(&kvSetBase, 1) - waitBench("set", n, n) + waitBench("set", n, value) } bench("set", f) @@ -86,15 +90,6 @@ func benchRandGet() { bench("get", f) } -func benchIncr() { - f := func() { - n := atomic.AddInt64(&kvIncrBase, 1) - waitBench("incr", n) - } - - bench("incr", f) -} - func benchDel() { f := func() { n := atomic.AddInt64(&kvDelBase, 1) @@ -106,8 +101,9 @@ func benchDel() { func benchPushList() { f := func() { - n := rand.Int() - waitBench("rpush", "mytestlist", n) + value := make([]byte, 10) + crand.Read(value) + waitBench("rpush", "mytestlist", value) } bench("rpush", f) @@ -152,22 +148,16 @@ var hashDelBase int64 = 0 func benchHset() { f := func() { + value := make([]byte, 100) + crand.Read(value) + n := atomic.AddInt64(&hashSetBase, 1) - waitBench("hset", "myhashkey", n, n) + waitBench("hset", "myhashkey", n, value) } bench("hset", f) } -func benchHIncr() { - f := func() { - n := atomic.AddInt64(&hashIncrBase, 1) - waitBench("hincrby", "myhashkey", n, 1) - } - - bench("hincrby", f) -} - func benchHGet() { f := func() { n := atomic.AddInt64(&hashGetBase, 1) @@ -201,8 +191,10 @@ var zsetIncrBase int64 = 0 func benchZAdd() { f := func() { + member := make([]byte, 16) + crand.Read(member) n := atomic.AddInt64(&zsetAddBase, 1) - waitBench("zadd", "myzsetkey", n, n) + waitBench("zadd", "myzsetkey", n, member) } bench("zadd", f) @@ -279,35 +271,41 @@ func main() { cfg.Addr = addr client = ledis.NewClient(cfg) - benchSet() - benchIncr() - benchGet() - benchRandGet() - benchDel() - - benchPushList() - benchRangeList10() - benchRangeList50() - benchRangeList100() - benchPopList() - - benchHset() - benchHGet() - benchHIncr() - benchHRandGet() - benchHDel() - - benchZAdd() - benchZIncr() - benchZRangeByRank() - benchZRangeByScore() - - //rev is too slow in leveldb, rocksdb or other - //maybe disable for huge data benchmark - if *reverse == true { - benchZRevRangeByRank() - benchZRevRangeByScore() + if *round <= 0 { + *round = 1 } - benchZDel() + for i := 0; i < *round; i++ { + benchSet() + benchGet() + benchRandGet() + benchDel() + + benchPushList() + benchRangeList10() + benchRangeList50() + benchRangeList100() + benchPopList() + + benchHset() + benchHGet() + benchHRandGet() + benchHDel() + + benchZAdd() + benchZIncr() + benchZRangeByRank() + benchZRangeByScore() + + //rev is too slow in leveldb, rocksdb or other + //maybe disable for huge data benchmark + if *reverse == true { + benchZRevRangeByRank() + benchZRevRangeByScore() + } + + benchZDel() + + println("") + } } From b9ea592026de06d0db7f79a8cf11b4544259fbf0 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 4 Sep 2014 16:39:23 +0800 Subject: [PATCH 29/32] add lua build tag maybe install golua error, and lua cannot be used --- bootstrap.sh | 2 +- dev.sh | 7 +++++++ server/cmd_script.go | 4 +++- server/cmd_script_test.go | 2 ++ server/script.go | 8 +++++++- server/script_dummy.go | 10 ++++++++++ server/script_test.go | 4 +++- tools/check_lua.go | 8 ++++++++ 8 files changed, 41 insertions(+), 4 deletions(-) create mode 100644 server/script_dummy.go create mode 100644 tools/check_lua.go diff --git a/bootstrap.sh b/bootstrap.sh index a0eae73..ffb4c46 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -16,4 +16,4 @@ go get github.com/BurntSushi/toml go get github.com/siddontang/go-bson/bson -go get github.com/aarzilli/golua/lua \ No newline at end of file +go get github.com/siddontang/golua/lua \ No newline at end of file diff --git a/dev.sh b/dev.sh index 798ffab..7b42c8e 100644 --- a/dev.sh +++ b/dev.sh @@ -74,6 +74,13 @@ if [ -f $HYPERLEVELDB_DIR/include/hyperleveldb/c.h ]; then GO_BUILD_TAGS="$GO_BUILD_TAGS hyperleveldb" fi +#check lua +CHECK_LUA_FILE="$LEDISTOP/tools/check_lua.go" +go run $CHECK_LUA_FILE 2>/dev/null +if [ "$?" = 0 ]; then + GO_BUILD_TAGS="$GO_BUILD_TAGS lua" +fi + export CGO_CFLAGS export CGO_CXXFLAGS export CGO_LDFLAGS diff --git a/server/cmd_script.go b/server/cmd_script.go index d2a3a1c..de1844f 100644 --- a/server/cmd_script.go +++ b/server/cmd_script.go @@ -1,10 +1,12 @@ +// +build lua + package server import ( "crypto/sha1" "encoding/hex" "fmt" - "github.com/aarzilli/golua/lua" + "github.com/siddontang/golua/lua" "github.com/siddontang/ledisdb/ledis" "strconv" "strings" diff --git a/server/cmd_script_test.go b/server/cmd_script_test.go index ae3e713..017e527 100644 --- a/server/cmd_script_test.go +++ b/server/cmd_script_test.go @@ -1,3 +1,5 @@ +// +build lua + package server import ( diff --git a/server/script.go b/server/script.go index 7ecdfc9..4f230fa 100644 --- a/server/script.go +++ b/server/script.go @@ -1,9 +1,11 @@ +// +build lua + package server import ( "encoding/hex" "fmt" - "github.com/aarzilli/golua/lua" + "github.com/siddontang/golua/lua" "github.com/siddontang/ledisdb/ledis" "io" "sync" @@ -159,6 +161,10 @@ func (app *App) openScript() { l.OpenTable() l.OpenPackage() + l.OpenCJson() + l.OpenCMsgpack() + l.OpenStruct() + l.Register("error", luaErrorHandler) s.l = l diff --git a/server/script_dummy.go b/server/script_dummy.go new file mode 100644 index 0000000..f19f3b8 --- /dev/null +++ b/server/script_dummy.go @@ -0,0 +1,10 @@ +// +build !lua + +package server + +type script struct { +} + +func (app *App) openScript() {} + +func (app *App) closeScript() {} diff --git a/server/script_test.go b/server/script_test.go index 0d91231..74160d3 100644 --- a/server/script_test.go +++ b/server/script_test.go @@ -1,8 +1,10 @@ +// +build lua + package server import ( "fmt" - "github.com/aarzilli/golua/lua" + "github.com/siddontang/golua/lua" "github.com/siddontang/ledisdb/config" "testing" diff --git a/tools/check_lua.go b/tools/check_lua.go new file mode 100644 index 0000000..f0fccf8 --- /dev/null +++ b/tools/check_lua.go @@ -0,0 +1,8 @@ +package main + +import "github.com/siddontang/golua/lua" + +func main() { + L := lua.NewState() + L.Close() +} From 0cf898bf2779739f31befc8054f325c7574b0660 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 4 Sep 2014 16:55:18 +0800 Subject: [PATCH 30/32] update lua build --- Makefile | 2 +- dev.sh | 2 +- tools/check_lua.go | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d98fc4d..f5b6dcd 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ INSTALL_PATH ?= $(CURDIR) -$(shell ./bootstrap.sh) +$(shell ./bootstrap.sh >> /dev/null 2>&1) $(shell ./tools/build_config.sh build_config.mk $INSTALL_PATH) diff --git a/dev.sh b/dev.sh index 7b42c8e..a9be046 100644 --- a/dev.sh +++ b/dev.sh @@ -76,7 +76,7 @@ fi #check lua CHECK_LUA_FILE="$LEDISTOP/tools/check_lua.go" -go run $CHECK_LUA_FILE 2>/dev/null +go run $CHECK_LUA_FILE >> /dev/null 2>&1 if [ "$?" = 0 ]; then GO_BUILD_TAGS="$GO_BUILD_TAGS lua" fi diff --git a/tools/check_lua.go b/tools/check_lua.go index f0fccf8..bc82c04 100644 --- a/tools/check_lua.go +++ b/tools/check_lua.go @@ -1,3 +1,5 @@ +// +build ignore + package main import "github.com/siddontang/golua/lua" From 9569c4d08afa1c9497a4910a98e8c5bbd9952642 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 4 Sep 2014 17:17:47 +0800 Subject: [PATCH 31/32] update benchmark --- cmd/ledis-benchmark/main.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/ledis-benchmark/main.go b/cmd/ledis-benchmark/main.go index d849e23..d5f81d5 100644 --- a/cmd/ledis-benchmark/main.go +++ b/cmd/ledis-benchmark/main.go @@ -87,7 +87,7 @@ func benchRandGet() { waitBench("get", n) } - bench("get", f) + bench("randget", f) } func benchDel() { @@ -114,7 +114,7 @@ func benchRangeList10() { waitBench("lrange", "mytestlist", 0, 10) } - bench("lrange", f) + bench("lrange10", f) } func benchRangeList50() { @@ -122,7 +122,7 @@ func benchRangeList50() { waitBench("lrange", "mytestlist", 0, 50) } - bench("lrange", f) + bench("lrange50", f) } func benchRangeList100() { @@ -130,7 +130,7 @@ func benchRangeList100() { waitBench("lrange", "mytestlist", 0, 100) } - bench("lrange", f) + bench("lrange100", f) } func benchPopList() { @@ -173,7 +173,7 @@ func benchHRandGet() { waitBench("hget", "myhashkey", n) } - bench("hget", f) + bench("hrandget", f) } func benchHDel() { From 138be8fb06d6e88f978afaa8b5ecd63b0c5e8536 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 4 Sep 2014 17:51:43 +0800 Subject: [PATCH 32/32] update benchmark --- cmd/ledis-benchmark/main.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/cmd/ledis-benchmark/main.go b/cmd/ledis-benchmark/main.go index d5f81d5..f91c98c 100644 --- a/cmd/ledis-benchmark/main.go +++ b/cmd/ledis-benchmark/main.go @@ -25,17 +25,13 @@ var client *ledis.Client var loop int = 0 func waitBench(cmd string, args ...interface{}) { - defer wg.Done() - c := client.Get() defer c.Close() - for i := 0; i < loop; i++ { - _, err := c.Do(cmd, args...) - if err != nil { - fmt.Printf("do %s error %s", cmd, err.Error()) - return - } + _, err := c.Do(cmd, args...) + if err != nil { + fmt.Printf("do %s error %s", cmd, err.Error()) + return } } @@ -44,7 +40,12 @@ func bench(cmd string, f func()) { t1 := time.Now().UnixNano() for i := 0; i < *clients; i++ { - go f() + go func() { + for i := 0; i < loop; i++ { + f() + } + wg.Done() + }() } wg.Wait() @@ -269,6 +270,7 @@ func main() { cfg := new(ledis.Config) cfg.Addr = addr + cfg.MaxIdleConns = *clients client = ledis.NewClient(cfg) if *round <= 0 {