From 9b1c6c4223507bb313b77eafc9ab4986dc09fc77 Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 25 Aug 2014 14:18:23 +0800 Subject: [PATCH] refactor --- .gitignore | 2 +- README.md | 4 +- client/nodejs/ledis/lib/commands.js | 6 +- cmd/ledis-cli/const.go | 7 +- cmd/ledis-cli/main.go | 14 +- config/config.go | 2 + doc/commands.json | 20 +- doc/commands.md | 70 ++- ledis/binlog_util.go | 14 + ledis/info.go | 26 ++ ledis/ledis.go | 40 +- ledis/ledis_db.go | 86 +++- ledis/replication_test.go | 26 +- ledis/scan.go | 2 +- ledis/t_bit.go | 32 +- ledis/t_bit_test.go | 9 +- ledis/t_hash.go | 50 ++- ledis/t_kv.go | 34 +- ledis/t_list.go | 28 +- ledis/t_set.go | 40 +- ledis/t_ttl.go | 24 +- ledis/t_zset.go | 54 +-- ledis/tx.go | 229 +++++++--- ledis/tx_test.go | 173 ++++++++ server/app.go | 6 + server/client.go | 128 ++++++ server/client_http.go | 45 +- server/client_resp.go | 46 +- server/cmd_bit.go | 88 ++-- server/cmd_hash.go | 136 +++--- server/cmd_kv.go | 134 +++--- server/cmd_list.go | 104 ++--- server/cmd_replication.go | 36 +- server/cmd_set.go | 136 +++--- server/cmd_tx.go | 57 +++ server/cmd_zset.go | 272 ++++++------ server/command.go | 43 +- server/command_cnf.go | 510 ---------------------- server/const.go | 15 + server/info.go | 166 +++++++ server/request.go | 116 ----- store/boltdb/db.go | 11 +- store/boltdb/snapshot.go | 37 ++ store/db.go | 45 +- store/driver/driver.go | 8 + store/driver/store.go | 2 +- store/goleveldb/db.go | 14 + store/goleveldb/snapshot.go | 26 ++ store/hyperleveldb/db.go | 14 + store/hyperleveldb/options.go | 8 + store/hyperleveldb/snapshot.go | 35 ++ store/leveldb/db.go | 14 + store/leveldb/options.go | 8 + store/leveldb/snapshot.go | 35 ++ store/mdb/mdb.go | 4 + store/mdb/snapshot.go | 43 ++ store/mdb/tx.go | 6 +- store/rocksdb/db.go | 14 + store/rocksdb/options.go | 8 + store/rocksdb/snapshot.go | 35 ++ store/store_test.go | 64 +++ store/tx.go | 35 +- store/tx_test.go | 4 +- generate.py => tools/generate_commands.py | 34 +- 64 files changed, 2053 insertions(+), 1481 deletions(-) create mode 100644 ledis/info.go create mode 100644 ledis/tx_test.go create mode 100644 server/client.go create mode 100644 server/cmd_tx.go delete mode 100644 server/command_cnf.go create mode 100644 server/info.go delete mode 100644 server/request.go create mode 100644 store/boltdb/snapshot.go create mode 100644 store/goleveldb/snapshot.go create mode 100644 store/hyperleveldb/snapshot.go create mode 100644 store/leveldb/snapshot.go create mode 100644 store/mdb/snapshot.go create mode 100644 store/rocksdb/snapshot.go rename generate.py => tools/generate_commands.py (66%) diff --git a/.gitignore b/.gitignore index c6ac2ca..1955ca3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ build .DS_Store nohup.out build_config.mk -var/ +var diff --git a/README.md b/README.md index c6ee195..af9d6ea 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ LedisDB now supports multiple databases as backend to store data, you can test a + Rich data structure: KV, List, Hash, ZSet, Bitmap, Set. + Stores lots of data, over the memory limit. + Various backend database to use: LevelDB, goleveldb, LMDB, RocksDB, BoltDB, HyperLevelDB. ++ Supports transaction using LMDB or BotlDB. + Supports expiration and ttl. + Redis clients, like redis-cli, are supported directly. + Multiple client API supports, including Go, Python, Lua(Openresty), C/C++, Node.js. @@ -89,7 +90,8 @@ Choosing a store database to use is very simple, you have two ways: **Caveat** -You must known that changing store database runtime is very dangerous, LedisDB will not guarantee the data validation if you do it. ++ You must known that changing store database runtime is very dangerous, LedisDB will not guarantee the data validation if you do it. ++ Begin a transaction will block any other write operators before you call `commit` or `rollback`. Don't use long-time transaction. ## Configuration diff --git a/client/nodejs/ledis/lib/commands.js b/client/nodejs/ledis/lib/commands.js index 38ce362..8a53ca2 100644 --- a/client/nodejs/ledis/lib/commands.js +++ b/client/nodejs/ledis/lib/commands.js @@ -117,6 +117,10 @@ module.exports = [ "sexpire", "sexpireat", "sttl", - "spersist" + "spersist", + + "begin", + "rollback", + "commit", ]; diff --git a/cmd/ledis-cli/const.go b/cmd/ledis-cli/const.go index ab76d34..2adcdad 100644 --- a/cmd/ledis-cli/const.go +++ b/cmd/ledis-cli/const.go @@ -1,9 +1,10 @@ -//This file was generated by ./generate.py on Fri Aug 15 2014 16:40:03 +0800 +//This file was generated by ./generate.py on Fri Aug 22 2014 14:32:10 +0800 package main var helpCommands = [][]string{ {"BCOUNT", "key [start end]", "Bitmap"}, {"BDELETE", "key", "ZSet"}, + {"BEGIN", "-", "Transaction"}, {"BEXPIRE", "key seconds", "Bitmap"}, {"BEXPIREAT", "key timestamp", "Bitmap"}, {"BGET", "key", "Bitmap"}, @@ -13,6 +14,7 @@ var helpCommands = [][]string{ {"BPERSIST", "key", "Bitmap"}, {"BSETBIT", "key offset value", "Bitmap"}, {"BTTL", "key", "Bitmap"}, + {"COMMIT", "-", "Transaction"}, {"DECR", "key", "KV"}, {"DECRBY", "key decrement", "KV"}, {"DEL", "key [key ...]", "KV"}, @@ -57,6 +59,7 @@ var helpCommands = [][]string{ {"MSET", "key value [key value ...]", "KV"}, {"PERSIST", "key", "KV"}, {"PING", "-", "Server"}, + {"ROLLBACK", "-", "Transaction"}, {"RPOP", "key", "List"}, {"RPUSH", "key value [value ...]", "List"}, {"SADD", "key member [member ...]", "Set"}, @@ -89,6 +92,7 @@ var helpCommands = [][]string{ {"ZEXPIRE", "key seconds", "ZSet"}, {"ZEXPIREAT", "key timestamp", "ZSet"}, {"ZINCRBY", "key increment member", "ZSet"}, + {"ZINTERSTORE", "destkey numkeys key [key ...] [WEIGHTS weight [weight ...]] [AGGREGATE SUM|MIN|MAX]", "ZSet"}, {"ZMCLEAR", "key [key ...]", "ZSet"}, {"ZPERSIST", "key", "ZSet"}, {"ZRANGE", "key start stop [WITHSCORES]", "ZSet"}, @@ -102,4 +106,5 @@ var helpCommands = [][]string{ {"ZREVRANK", "key member", "ZSet"}, {"ZSCORE", "key member", "ZSet"}, {"ZTTL", "key", "ZSet"}, + {"ZUNIONSTORE", "destkey numkeys key [key ...] [WEIGHTS weight [weight ...]] [AGGREGATE SUM|MIN|MAX]", "ZSet"}, } diff --git a/cmd/ledis-cli/main.go b/cmd/ledis-cli/main.go index a8f4397..70f0b93 100644 --- a/cmd/ledis-cli/main.go +++ b/cmd/ledis-cli/main.go @@ -61,8 +61,8 @@ func main() { args[i] = strings.Trim(string(cmds[1+i]), "\"'") } - cmd := cmds[0] - if strings.ToLower(cmd) == "help" || cmd == "?" { + cmd := strings.ToLower(cmds[0]) + if cmd == "help" || cmd == "?" { printHelp(cmds) } else { if len(cmds) == 2 && strings.ToLower(cmds[0]) == "select" { @@ -77,7 +77,11 @@ func main() { if err != nil { fmt.Printf("%s", err.Error()) } else { - printReply(cmd, r) + if cmd == "info" { + printInfo(r.([]byte)) + } else { + printReply(cmd, r) + } } fmt.Printf("\n") @@ -87,6 +91,10 @@ func main() { } } +func printInfo(s []byte) { + fmt.Printf("%s", s) +} + func printReply(cmd string, reply interface{}) { switch reply := reply.(type) { case int64: diff --git a/config/config.go b/config/config.go index 0939afb..e956a02 100644 --- a/config/config.go +++ b/config/config.go @@ -106,6 +106,8 @@ func NewConfigDefault() *Config { // disable access log cfg.AccessLog = "" + cfg.LMDB.NoSync = true + return cfg } diff --git a/doc/commands.json b/doc/commands.json index 5ff7070..ede7cab 100644 --- a/doc/commands.json +++ b/doc/commands.json @@ -288,7 +288,7 @@ "SELECT": { "arguments": "index", "group": "Server", - "readonly": false + "readonly": true }, "SET": { "arguments": "key value", @@ -448,7 +448,7 @@ "ZRANGE": { "arguments": "key start stop [WITHSCORES]", "group": "ZSet", - "readonly": false + "readonly": true }, "ZRANGEBYSCORE": { "arguments": "key min max [WITHSCORES] [LIMIT offset count]", @@ -510,5 +510,21 @@ "arguments": "destkey numkeys key [key ...] [WEIGHTS weight [weight ...]] [AGGREGATE SUM|MIN|MAX]", "group": "ZSet", "readonly": false + }, + + "BEGIN": { + "arguments": "-", + "group": "Transaction", + "readonly": false + }, + "COMMIT": { + "arguments": "-", + "group": "Transaction", + "readonly": false + }, + "ROLLBACK": { + "arguments": "-", + "group": "Transaction", + "readonly": false } } diff --git a/doc/commands.md b/doc/commands.md index 9880826..66c227b 100644 --- a/doc/commands.md +++ b/doc/commands.md @@ -2,7 +2,7 @@ ledisdb use redis protocol called RESP(REdis Serialization Protocol), [here](http://redis.io/topics/protocol). -ledisdb all commands return RESP fomrat and it will use `int64` instead of `RESP integer`, `string` instead of `RESP simple string`, `bulk string` instead of `RESP bulk string`, and `array` instead of `RESP arrays` below. +ledisdb all commands return RESP format and it will use `int64` instead of `RESP integer`, `string` instead of `RESP simple string`, `bulk string` instead of `RESP bulk string`, and `array` instead of `RESP arrays` below. Table of Contents ================= @@ -124,6 +124,10 @@ Table of Contents - [PING](#ping) - [ECHO message](#echo-message) - [SELECT index](#select-index) +- [Transaction](#transaction) + - [BEGIN](#begin) + - [ROLLBACK](#rollback) + - [COMMIT](#commit) ## KV @@ -2394,4 +2398,68 @@ ledis> SELECT 16 ERR invalid db index 16 ``` +## Transaction + +### BEGIN + +Marks the start of a transaction block. Subsequent commands will be in a transaction context util using COMMIT or ROLLBACK. + +You must known that `BEGIN` will block any other write operators before you `COMMIT` or `ROLLBACK`. Don't use long-time transaction. + +**Return value** + +Returns `OK` if the backend store engine in use supports transaction, otherwise, returns `Err`. + +**Examples** +``` +ledis> BEGIN +OK +ledis> SET HELLO WORLD +OK +ledis> COMMIT +OK +``` + +### ROLLBACK + +Discards all the changes of previously commands in a transaction and restores the connection state to normal. + +**Return value** +Returns `OK` if in a transaction context, otherwise, `Err` + +**Examples** +``` +ledis> BEGIN +OK +ledis> SET HELLO WORLD +OK +ledis> GET HELLO +"WORLD" +ledis> ROLLBACK +OK +ledis> GET HELLO +(nil) +``` + +### COMMIT + +Persists the changes of all the commands in a transaction and restores the connection state to normal. + +**Return value** +Returns `OK` if in a transaction context, otherwise, `Err` + +**Examples** +``` +ledis> BEGIN +OK +ledis> SET HELLO WORLD +OK +ledis> GET HELLO +"WORLD" +ledis> COMMIT +OK +ledis> GET HELLO +"WORLD" +``` + Thanks [doctoc](http://doctoc.herokuapp.com/) diff --git a/ledis/binlog_util.go b/ledis/binlog_util.go index 766d3e6..5167b40 100644 --- a/ledis/binlog_util.go +++ b/ledis/binlog_util.go @@ -184,6 +184,20 @@ func formatDataKey(buf []byte, k []byte) ([]byte, error) { } else { buf = strconv.AppendQuote(buf, String(key)) } + case SetType: + if key, member, err := db.sDecodeSetKey(k); err != nil { + return nil, err + } else { + buf = strconv.AppendQuote(buf, String(key)) + buf = append(buf, ' ') + buf = strconv.AppendQuote(buf, String(member)) + } + case SSizeType: + if key, err := db.sDecodeSizeKey(k); err != nil { + return nil, err + } else { + buf = strconv.AppendQuote(buf, String(key)) + } case ExpTimeType: if tp, key, t, err := db.expDecodeTimeKey(k); err != nil { return nil, err diff --git a/ledis/info.go b/ledis/info.go new file mode 100644 index 0000000..df3ee72 --- /dev/null +++ b/ledis/info.go @@ -0,0 +1,26 @@ +package ledis + +import () + +// todo, add info + +// type Keyspace struct { +// Kvs int `json:"kvs"` +// KvExpires int `json:"kv_expires"` + +// Lists int `json:"lists"` +// ListExpires int `json:"list_expires"` + +// Bitmaps int `json:"bitmaps"` +// BitmapExpires int `json:"bitmap_expires"` + +// ZSets int `json:"zsets"` +// ZSetExpires int `json:"zset_expires"` + +// Hashes int `json:"hashes"` +// HashExpires int `json:"hahsh_expires"` +// } + +// type Info struct { +// KeySpaces [MaxDBNumber]Keyspace +// } diff --git a/ledis/ledis.go b/ledis/ledis.go index 65ab1ec..70d22d1 100644 --- a/ledis/ledis.go +++ b/ledis/ledis.go @@ -9,21 +9,6 @@ import ( "time" ) -type DB struct { - l *Ledis - - db *store.DB - - index uint8 - - kvTx *tx - listTx *tx - hashTx *tx - zsetTx *tx - binTx *tx - setTx *tx -} - type Ledis struct { sync.Mutex @@ -32,10 +17,10 @@ type Ledis struct { ldb *store.DB dbs [MaxDBNumber]*DB - binlog *BinLog - quit chan struct{} jobs *sync.WaitGroup + + binlog *BinLog } func Open(cfg *config.Config) (*Ledis, error) { @@ -67,7 +52,7 @@ func Open(cfg *config.Config) (*Ledis, error) { } for i := uint8(0); i < MaxDBNumber; i++ { - l.dbs[i] = newDB(l, i) + l.dbs[i] = l.newDB(i) } l.activeExpireCycle() @@ -75,25 +60,6 @@ func Open(cfg *config.Config) (*Ledis, error) { return l, nil } -func newDB(l *Ledis, index uint8) *DB { - d := new(DB) - - d.l = l - - d.db = l.ldb - - d.index = index - - d.kvTx = newTx(l) - d.listTx = newTx(l) - d.hashTx = newTx(l) - d.zsetTx = newTx(l) - d.binTx = newTx(l) - d.setTx = newTx(l) - - return d -} - func (l *Ledis) Close() { close(l.quit) l.jobs.Wait() diff --git a/ledis/ledis_db.go b/ledis/ledis_db.go index 0283d2f..104ac6b 100644 --- a/ledis/ledis_db.go +++ b/ledis/ledis_db.go @@ -3,8 +3,73 @@ package ledis import ( "fmt" "github.com/siddontang/ledisdb/store" + "sync" ) +type ibucket interface { + Get(key []byte) ([]byte, error) + + Put(key []byte, value []byte) error + Delete(key []byte) error + + NewIterator() *store.Iterator + + NewWriteBatch() store.WriteBatch + + RangeIterator(min []byte, max []byte, rangeType uint8) *store.RangeLimitIterator + RevRangeIterator(min []byte, max []byte, rangeType uint8) *store.RangeLimitIterator + RangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *store.RangeLimitIterator + RevRangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *store.RangeLimitIterator +} + +type DB struct { + l *Ledis + + sdb *store.DB + + bucket ibucket + + dbLock *sync.RWMutex + + index uint8 + + kvBatch *batch + listBatch *batch + hashBatch *batch + zsetBatch *batch + binBatch *batch + setBatch *batch + + isTx bool +} + +func (l *Ledis) newDB(index uint8) *DB { + d := new(DB) + + d.l = l + + d.sdb = l.ldb + + d.bucket = d.sdb + + d.isTx = false + d.index = index + d.dbLock = &sync.RWMutex{} + + d.kvBatch = d.newBatch() + d.listBatch = d.newBatch() + d.hashBatch = d.newBatch() + d.zsetBatch = d.newBatch() + d.binBatch = d.newBatch() + d.setBatch = d.newBatch() + + return d +} + +func (db *DB) Index() int { + return int(db.index) +} + func (db *DB) FlushAll() (drop int64, err error) { all := [...](func() (int64, error)){ db.flush, @@ -28,18 +93,19 @@ func (db *DB) FlushAll() (drop int64, err error) { func (db *DB) newEliminator() *elimination { eliminator := newEliminator(db) - eliminator.regRetireContext(KVType, db.kvTx, db.delete) - eliminator.regRetireContext(ListType, db.listTx, db.lDelete) - eliminator.regRetireContext(HashType, db.hashTx, db.hDelete) - eliminator.regRetireContext(ZSetType, db.zsetTx, db.zDelete) - eliminator.regRetireContext(BitType, db.binTx, db.bDelete) - eliminator.regRetireContext(SetType, db.setTx, db.sDelete) + + eliminator.regRetireContext(KVType, db.kvBatch, db.delete) + eliminator.regRetireContext(ListType, db.listBatch, db.lDelete) + eliminator.regRetireContext(HashType, db.hashBatch, db.hDelete) + eliminator.regRetireContext(ZSetType, db.zsetBatch, db.zDelete) + eliminator.regRetireContext(BitType, db.binBatch, db.bDelete) + eliminator.regRetireContext(SetType, db.setBatch, db.sDelete) return eliminator } -func (db *DB) flushRegion(t *tx, minKey []byte, maxKey []byte) (drop int64, err error) { - it := db.db.RangeIterator(minKey, maxKey, store.RangeROpen) +func (db *DB) flushRegion(t *batch, minKey []byte, maxKey []byte) (drop int64, err error) { + it := db.bucket.RangeIterator(minKey, maxKey, store.RangeROpen) for ; it.Valid(); it.Next() { t.Delete(it.RawKey()) drop++ @@ -53,8 +119,8 @@ func (db *DB) flushRegion(t *tx, minKey []byte, maxKey []byte) (drop int64, err return } -func (db *DB) flushType(t *tx, dataType byte) (drop int64, err error) { - var deleteFunc func(t *tx, key []byte) int64 +func (db *DB) flushType(t *batch, dataType byte) (drop int64, err error) { + var deleteFunc func(t *batch, key []byte) int64 var metaDataType byte switch dataType { case KVType: diff --git a/ledis/replication_test.go b/ledis/replication_test.go index 0e6f024..96bb10a 100644 --- a/ledis/replication_test.go +++ b/ledis/replication_test.go @@ -59,9 +59,16 @@ func TestReplication(t *testing.T) { db.Set([]byte("b"), []byte("value")) db.Set([]byte("c"), []byte("value")) - db.HSet([]byte("a"), []byte("1"), []byte("value")) - db.HSet([]byte("b"), []byte("2"), []byte("value")) - db.HSet([]byte("c"), []byte("3"), []byte("value")) + if tx, err := db.Begin(); err == nil { + tx.HSet([]byte("a"), []byte("1"), []byte("value")) + tx.HSet([]byte("b"), []byte("2"), []byte("value")) + tx.HSet([]byte("c"), []byte("3"), []byte("value")) + tx.Commit() + } else { + db.HSet([]byte("a"), []byte("1"), []byte("value")) + db.HSet([]byte("b"), []byte("2"), []byte("value")) + db.HSet([]byte("c"), []byte("3"), []byte("value")) + } for _, name := range master.binlog.LogNames() { p := path.Join(master.binlog.LogPath(), name) @@ -78,14 +85,21 @@ func TestReplication(t *testing.T) { slave.FlushAll() - db.Set([]byte("a1"), []byte("1")) - db.Set([]byte("b1"), []byte("2")) - db.Set([]byte("c1"), []byte("3")) + db.Set([]byte("a1"), []byte("value")) + db.Set([]byte("b1"), []byte("value")) + db.Set([]byte("c1"), []byte("value")) db.HSet([]byte("a1"), []byte("1"), []byte("value")) db.HSet([]byte("b1"), []byte("2"), []byte("value")) db.HSet([]byte("c1"), []byte("3"), []byte("value")) + if tx, err := db.Begin(); err == nil { + tx.HSet([]byte("a1"), []byte("1"), []byte("value1")) + tx.HSet([]byte("b1"), []byte("2"), []byte("value1")) + tx.HSet([]byte("c1"), []byte("3"), []byte("value1")) + tx.Rollback() + } + info := new(MasterInfo) info.LogFileIndex = 1 info.LogPos = 0 diff --git a/ledis/scan.go b/ledis/scan.go index d64eae5..205bc07 100644 --- a/ledis/scan.go +++ b/ledis/scan.go @@ -40,7 +40,7 @@ func (db *DB) scan(dataType byte, key []byte, count int, inclusive bool) ([][]by rangeType = store.RangeOpen } - it := db.db.RangeLimitIterator(minKey, maxKey, rangeType, 0, count) + it := db.bucket.RangeLimitIterator(minKey, maxKey, rangeType, 0, count) for ; it.Valid(); it.Next() { if k, err := db.decodeMetaKey(dataType, it.Key()); err != nil { diff --git a/ledis/t_bit.go b/ledis/t_bit.go index b32e77b..3af59dc 100644 --- a/ledis/t_bit.go +++ b/ledis/t_bit.go @@ -204,7 +204,7 @@ func (db *DB) bGetMeta(key []byte) (tailSeq int32, tailOff int32, err error) { var v []byte mk := db.bEncodeMetaKey(key) - v, err = db.db.Get(mk) + v, err = db.bucket.Get(mk) if err != nil { return } @@ -219,7 +219,7 @@ func (db *DB) bGetMeta(key []byte) (tailSeq int32, tailOff int32, err error) { return } -func (db *DB) bSetMeta(t *tx, key []byte, tailSeq uint32, tailOff uint32) { +func (db *DB) bSetMeta(t *batch, key []byte, tailSeq uint32, tailOff uint32) { ek := db.bEncodeMetaKey(key) buf := make([]byte, 8) @@ -230,7 +230,7 @@ func (db *DB) bSetMeta(t *tx, key []byte, tailSeq uint32, tailOff uint32) { return } -func (db *DB) bUpdateMeta(t *tx, key []byte, seq uint32, off uint32) (tailSeq uint32, tailOff uint32, err error) { +func (db *DB) bUpdateMeta(t *batch, key []byte, seq uint32, off uint32) (tailSeq uint32, tailOff uint32, err error) { var tseq, toff int32 var update bool = false @@ -252,13 +252,13 @@ func (db *DB) bUpdateMeta(t *tx, key []byte, seq uint32, off uint32) (tailSeq ui return } -func (db *DB) bDelete(t *tx, key []byte) (drop int64) { +func (db *DB) bDelete(t *batch, key []byte) (drop int64) { mk := db.bEncodeMetaKey(key) t.Delete(mk) minKey := db.bEncodeBinKey(key, minSeq) maxKey := db.bEncodeBinKey(key, maxSeq) - it := db.db.RangeIterator(minKey, maxKey, store.RangeClose) + it := db.bucket.RangeIterator(minKey, maxKey, store.RangeClose) for ; it.Valid(); it.Next() { t.Delete(it.RawKey()) drop++ @@ -270,7 +270,7 @@ func (db *DB) bDelete(t *tx, key []byte) (drop int64) { func (db *DB) bGetSegment(key []byte, seq uint32) ([]byte, []byte, error) { bk := db.bEncodeBinKey(key, seq) - segment, err := db.db.Get(bk) + segment, err := db.bucket.Get(bk) if err != nil { return bk, nil, err } @@ -288,7 +288,7 @@ func (db *DB) bAllocateSegment(key []byte, seq uint32) ([]byte, []byte, error) { func (db *DB) bIterator(key []byte) *store.RangeLimitIterator { sk := db.bEncodeBinKey(key, minSeq) ek := db.bEncodeBinKey(key, maxSeq) - return db.db.RangeIterator(sk, ek, store.RangeClose) + return db.bucket.RangeIterator(sk, ek, store.RangeClose) } func (db *DB) bSegAnd(a []byte, b []byte, res *[]byte) { @@ -361,7 +361,7 @@ func (db *DB) bSegXor(a []byte, b []byte, res *[]byte) { } func (db *DB) bExpireAt(key []byte, when int64) (int64, error) { - t := db.binTx + t := db.binBatch t.Lock() defer t.Unlock() @@ -451,7 +451,7 @@ func (db *DB) BGet(key []byte) (data []byte, err error) { minKey := db.bEncodeBinKey(key, minSeq) maxKey := db.bEncodeBinKey(key, tailSeq) - it := db.db.RangeIterator(minKey, maxKey, store.RangeClose) + it := db.bucket.RangeIterator(minKey, maxKey, store.RangeClose) var seq, s, e uint32 for ; it.Valid(); it.Next() { @@ -474,7 +474,7 @@ func (db *DB) BDelete(key []byte) (drop int64, err error) { return } - t := db.binTx + t := db.binBatch t.Lock() defer t.Unlock() @@ -504,7 +504,7 @@ func (db *DB) BSetBit(key []byte, offset int32, val uint8) (ori uint8, err error if segment != nil { ori = getBit(segment, off) if setBit(segment, off, val) { - t := db.binTx + t := db.binBatch t.Lock() t.Put(bk, segment) @@ -554,7 +554,7 @@ func (db *DB) BMSetBit(key []byte, args ...BitPair) (place int64, err error) { } // #2 : execute bit set in order - t := db.binTx + t := db.binBatch t.Lock() defer t.Unlock() @@ -667,7 +667,7 @@ func (db *DB) BCount(key []byte, start int32, end int32) (cnt int32, err error) skey := db.bEncodeBinKey(key, sseq) ekey := db.bEncodeBinKey(key, eseq) - it := db.db.RangeIterator(skey, ekey, store.RangeOpen) + it := db.bucket.RangeIterator(skey, ekey, store.RangeOpen) for ; it.Valid(); it.Next() { segment = it.RawValue() for _, bt := range segment { @@ -715,7 +715,7 @@ func (db *DB) BOperation(op uint8, dstkey []byte, srckeys ...[]byte) (blen int32 return } - t := db.binTx + t := db.binBatch t.Lock() defer t.Unlock() @@ -895,7 +895,7 @@ func (db *DB) BPersist(key []byte) (int64, error) { return 0, err } - t := db.binTx + t := db.binBatch t.Lock() defer t.Unlock() @@ -913,7 +913,7 @@ func (db *DB) BScan(key []byte, count int, inclusive bool) ([][]byte, error) { } func (db *DB) bFlush() (drop int64, err error) { - t := db.binTx + t := db.binBatch t.Lock() defer t.Unlock() diff --git a/ledis/t_bit_test.go b/ledis/t_bit_test.go index 59355da..e04fe65 100644 --- a/ledis/t_bit_test.go +++ b/ledis/t_bit_test.go @@ -43,6 +43,7 @@ func TestBinary(t *testing.T) { testOpNot(t) testMSetBit(t) testBitExpire(t) + testBFlush(t) } func testSimple(t *testing.T) { @@ -543,7 +544,7 @@ func testBFlush(t *testing.T) { db.FlushAll() for i := 0; i < 2000; i++ { - key := []byte{} + key := make([]byte, 4) binary.LittleEndian.PutUint32(key, uint32(i)) if _, err := db.BSetBit(key, 1, 1); err != nil { t.Fatal(err.Error()) @@ -557,7 +558,7 @@ func testBFlush(t *testing.T) { } for i := 0; i < 2000; i++ { - key := []byte{} + key := make([]byte, 4) binary.LittleEndian.PutUint32(key, uint32(i)) if v, err := db.BGetBit(key, 1); err != nil { t.Fatal(err.Error()) @@ -574,12 +575,12 @@ func testBFlush(t *testing.T) { if v, err := db.BScan(nil, 3000, true); err != nil { t.Fatal(err.Error()) - } else if v != nil { + } else if len(v) != 0 { t.Fatal("invalid value length ", len(v)) } for i := 0; i < 2000; i++ { - key := []byte{} + key := make([]byte, 4) binary.LittleEndian.PutUint32(key, uint32(i)) if v, err := db.BGet(key); err != nil { t.Fatal(err.Error()) diff --git a/ledis/t_hash.go b/ledis/t_hash.go index 0782e63..bdcf16d 100644 --- a/ledis/t_hash.go +++ b/ledis/t_hash.go @@ -107,12 +107,12 @@ func (db *DB) hEncodeStopKey(key []byte) []byte { } func (db *DB) hSetItem(key []byte, field []byte, value []byte) (int64, error) { - t := db.hashTx + t := db.hashBatch ek := db.hEncodeHashKey(key, field) var n int64 = 1 - if v, _ := db.db.Get(ek); v != nil { + if v, _ := db.bucket.Get(ek); v != nil { n = 0 } else { if _, err := db.hIncrSize(key, 1); err != nil { @@ -126,13 +126,13 @@ func (db *DB) hSetItem(key []byte, field []byte, value []byte) (int64, error) { // ps : here just focus on deleting the hash data, // any other likes expire is ignore. -func (db *DB) hDelete(t *tx, key []byte) int64 { +func (db *DB) hDelete(t *batch, key []byte) int64 { sk := db.hEncodeSizeKey(key) start := db.hEncodeStartKey(key) stop := db.hEncodeStopKey(key) var num int64 = 0 - it := db.db.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) + it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) for ; it.Valid(); it.Next() { t.Delete(it.Key()) num++ @@ -144,7 +144,7 @@ func (db *DB) hDelete(t *tx, key []byte) int64 { } func (db *DB) hExpireAt(key []byte, when int64) (int64, error) { - t := db.hashTx + t := db.hashBatch t.Lock() defer t.Unlock() @@ -164,7 +164,7 @@ func (db *DB) HLen(key []byte) (int64, error) { return 0, err } - return Int64(db.db.Get(db.hEncodeSizeKey(key))) + return Int64(db.bucket.Get(db.hEncodeSizeKey(key))) } func (db *DB) HSet(key []byte, field []byte, value []byte) (int64, error) { @@ -174,7 +174,7 @@ func (db *DB) HSet(key []byte, field []byte, value []byte) (int64, error) { return 0, err } - t := db.hashTx + t := db.hashBatch t.Lock() defer t.Unlock() @@ -194,11 +194,11 @@ func (db *DB) HGet(key []byte, field []byte) ([]byte, error) { return nil, err } - return db.db.Get(db.hEncodeHashKey(key, field)) + return db.bucket.Get(db.hEncodeHashKey(key, field)) } func (db *DB) HMset(key []byte, args ...FVPair) error { - t := db.hashTx + t := db.hashBatch t.Lock() defer t.Unlock() @@ -214,7 +214,7 @@ func (db *DB) HMset(key []byte, args ...FVPair) error { ek = db.hEncodeHashKey(key, args[i].Field) - if v, err := db.db.Get(ek); err != nil { + if v, err := db.bucket.Get(ek); err != nil { return err } else if v == nil { num++ @@ -235,7 +235,7 @@ func (db *DB) HMset(key []byte, args ...FVPair) error { func (db *DB) HMget(key []byte, args ...[]byte) ([][]byte, error) { var ek []byte - it := db.db.NewIterator() + it := db.bucket.NewIterator() defer it.Close() r := make([][]byte, len(args)) @@ -253,7 +253,7 @@ func (db *DB) HMget(key []byte, args ...[]byte) ([][]byte, error) { } func (db *DB) HDel(key []byte, args ...[]byte) (int64, error) { - t := db.hashTx + t := db.hashBatch var ek []byte var v []byte @@ -262,7 +262,7 @@ func (db *DB) HDel(key []byte, args ...[]byte) (int64, error) { t.Lock() defer t.Unlock() - it := db.db.NewIterator() + it := db.bucket.NewIterator() defer it.Close() var num int64 = 0 @@ -292,12 +292,12 @@ func (db *DB) HDel(key []byte, args ...[]byte) (int64, error) { } func (db *DB) hIncrSize(key []byte, delta int64) (int64, error) { - t := db.hashTx + t := db.hashBatch sk := db.hEncodeSizeKey(key) var err error var size int64 = 0 - if size, err = Int64(db.db.Get(sk)); err != nil { + if size, err = Int64(db.bucket.Get(sk)); err != nil { return 0, err } else { size += delta @@ -318,7 +318,7 @@ func (db *DB) HIncrBy(key []byte, field []byte, delta int64) (int64, error) { return 0, err } - t := db.hashTx + t := db.hashBatch var ek []byte var err error @@ -328,7 +328,7 @@ func (db *DB) HIncrBy(key []byte, field []byte, delta int64) (int64, error) { ek = db.hEncodeHashKey(key, field) var n int64 = 0 - if n, err = StrInt64(db.db.Get(ek)); err != nil { + if n, err = StrInt64(db.bucket.Get(ek)); err != nil { return 0, err } @@ -354,7 +354,7 @@ func (db *DB) HGetAll(key []byte) ([]FVPair, error) { v := make([]FVPair, 0, 16) - it := db.db.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) + it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) for ; it.Valid(); it.Next() { _, f, err := db.hDecodeHashKey(it.Key()) if err != nil { @@ -379,7 +379,7 @@ func (db *DB) HKeys(key []byte) ([][]byte, error) { v := make([][]byte, 0, 16) - it := db.db.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) + it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) for ; it.Valid(); it.Next() { _, f, err := db.hDecodeHashKey(it.Key()) if err != nil { @@ -403,7 +403,7 @@ func (db *DB) HValues(key []byte) ([][]byte, error) { v := make([][]byte, 0, 16) - it := db.db.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) + it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) for ; it.Valid(); it.Next() { _, _, err := db.hDecodeHashKey(it.Key()) if err != nil { @@ -423,7 +423,7 @@ func (db *DB) HClear(key []byte) (int64, error) { return 0, err } - t := db.hashTx + t := db.hashBatch t.Lock() defer t.Unlock() @@ -435,7 +435,7 @@ func (db *DB) HClear(key []byte) (int64, error) { } func (db *DB) HMclear(keys ...[]byte) (int64, error) { - t := db.hashTx + t := db.hashBatch t.Lock() defer t.Unlock() @@ -453,9 +453,11 @@ func (db *DB) HMclear(keys ...[]byte) (int64, error) { } func (db *DB) hFlush() (drop int64, err error) { - t := db.hashTx + t := db.hashBatch + t.Lock() defer t.Unlock() + return db.flushType(t, HashType) } @@ -492,7 +494,7 @@ func (db *DB) HPersist(key []byte) (int64, error) { return 0, err } - t := db.hashTx + t := db.hashBatch t.Lock() defer t.Unlock() diff --git a/ledis/t_kv.go b/ledis/t_kv.go index b7bf4d6..3d5bb0f 100644 --- a/ledis/t_kv.go +++ b/ledis/t_kv.go @@ -62,13 +62,13 @@ func (db *DB) incr(key []byte, delta int64) (int64, error) { var err error key = db.encodeKVKey(key) - t := db.kvTx + t := db.kvBatch t.Lock() defer t.Unlock() var n int64 - n, err = StrInt64(db.db.Get(key)) + n, err = StrInt64(db.bucket.Get(key)) if err != nil { return 0, err } @@ -85,14 +85,14 @@ func (db *DB) incr(key []byte, delta int64) (int64, error) { // ps : here just focus on deleting the key-value data, // any other likes expire is ignore. -func (db *DB) delete(t *tx, key []byte) int64 { +func (db *DB) delete(t *batch, key []byte) int64 { key = db.encodeKVKey(key) t.Delete(key) return 1 } func (db *DB) setExpireAt(key []byte, when int64) (int64, error) { - t := db.kvTx + t := db.kvBatch t.Lock() defer t.Unlock() @@ -125,7 +125,7 @@ func (db *DB) Del(keys ...[]byte) (int64, error) { codedKeys[i] = db.encodeKVKey(k) } - t := db.kvTx + t := db.kvBatch t.Lock() defer t.Unlock() @@ -147,7 +147,7 @@ func (db *DB) Exists(key []byte) (int64, error) { key = db.encodeKVKey(key) var v []byte - v, err = db.db.Get(key) + v, err = db.bucket.Get(key) if v != nil && err == nil { return 1, nil } @@ -162,7 +162,7 @@ func (db *DB) Get(key []byte) ([]byte, error) { key = db.encodeKVKey(key) - return db.db.Get(key) + return db.bucket.Get(key) } func (db *DB) GetSet(key []byte, value []byte) ([]byte, error) { @@ -174,12 +174,12 @@ func (db *DB) GetSet(key []byte, value []byte) ([]byte, error) { key = db.encodeKVKey(key) - t := db.kvTx + t := db.kvBatch t.Lock() defer t.Unlock() - oldValue, err := db.db.Get(key) + oldValue, err := db.bucket.Get(key) if err != nil { return nil, err } @@ -203,7 +203,7 @@ func (db *DB) IncrBy(key []byte, increment int64) (int64, error) { func (db *DB) MGet(keys ...[]byte) ([][]byte, error) { values := make([][]byte, len(keys)) - it := db.db.NewIterator() + it := db.bucket.NewIterator() defer it.Close() for i := range keys { @@ -222,7 +222,7 @@ func (db *DB) MSet(args ...KVPair) error { return nil } - t := db.kvTx + t := db.kvBatch var err error var key []byte @@ -261,15 +261,13 @@ func (db *DB) Set(key []byte, value []byte) error { var err error key = db.encodeKVKey(key) - t := db.kvTx + t := db.kvBatch t.Lock() defer t.Unlock() t.Put(key, value) - //todo, binlog - err = t.Commit() return err @@ -287,12 +285,12 @@ func (db *DB) SetNX(key []byte, value []byte) (int64, error) { var n int64 = 1 - t := db.kvTx + t := db.kvBatch t.Lock() defer t.Unlock() - if v, err := db.db.Get(key); err != nil { + if v, err := db.bucket.Get(key); err != nil { return 0, err } else if v != nil { n = 0 @@ -308,7 +306,7 @@ func (db *DB) SetNX(key []byte, value []byte) (int64, error) { } func (db *DB) flush() (drop int64, err error) { - t := db.kvTx + t := db.kvBatch t.Lock() defer t.Unlock() return db.flushType(t, KVType) @@ -348,7 +346,7 @@ func (db *DB) Persist(key []byte) (int64, error) { return 0, err } - t := db.kvTx + t := db.kvBatch t.Lock() defer t.Unlock() n, err := db.rmExpire(t, KVType, key) diff --git a/ledis/t_list.go b/ledis/t_list.go index a97ce97..599dc9f 100644 --- a/ledis/t_list.go +++ b/ledis/t_list.go @@ -84,7 +84,7 @@ func (db *DB) lpush(key []byte, whereSeq int32, args ...[]byte) (int64, error) { var size int32 var err error - t := db.listTx + t := db.listBatch t.Lock() defer t.Unlock() @@ -139,7 +139,7 @@ func (db *DB) lpop(key []byte, whereSeq int32) ([]byte, error) { return nil, err } - t := db.listTx + t := db.listBatch t.Lock() defer t.Unlock() @@ -161,7 +161,7 @@ func (db *DB) lpop(key []byte, whereSeq int32) ([]byte, error) { } itemKey := db.lEncodeListKey(key, seq) - value, err = db.db.Get(itemKey) + value, err = db.bucket.Get(itemKey) if err != nil { return nil, err } @@ -184,14 +184,14 @@ func (db *DB) lpop(key []byte, whereSeq int32) ([]byte, error) { // ps : here just focus on deleting the list data, // any other likes expire is ignore. -func (db *DB) lDelete(t *tx, key []byte) int64 { +func (db *DB) lDelete(t *batch, key []byte) int64 { mk := db.lEncodeMetaKey(key) var headSeq int32 var tailSeq int32 var err error - it := db.db.NewIterator() + it := db.bucket.NewIterator() defer it.Close() headSeq, tailSeq, _, err = db.lGetMeta(it, mk) @@ -219,7 +219,7 @@ func (db *DB) lGetMeta(it *store.Iterator, ek []byte) (headSeq int32, tailSeq in if it != nil { v = it.Find(ek) } else { - v, err = db.db.Get(ek) + v, err = db.bucket.Get(ek) } if err != nil { return @@ -237,7 +237,7 @@ func (db *DB) lGetMeta(it *store.Iterator, ek []byte) (headSeq int32, tailSeq in } func (db *DB) lSetMeta(ek []byte, headSeq int32, tailSeq int32) int32 { - t := db.listTx + t := db.listBatch var size int32 = tailSeq - headSeq + 1 if size < 0 { @@ -257,7 +257,7 @@ func (db *DB) lSetMeta(ek []byte, headSeq int32, tailSeq int32) int32 { } func (db *DB) lExpireAt(key []byte, when int64) (int64, error) { - t := db.listTx + t := db.listBatch t.Lock() defer t.Unlock() @@ -284,7 +284,7 @@ func (db *DB) LIndex(key []byte, index int32) ([]byte, error) { metaKey := db.lEncodeMetaKey(key) - it := db.db.NewIterator() + it := db.bucket.NewIterator() defer it.Close() headSeq, tailSeq, _, err = db.lGetMeta(it, metaKey) @@ -333,7 +333,7 @@ func (db *DB) LRange(key []byte, start int32, stop int32) ([][]byte, error) { metaKey := db.lEncodeMetaKey(key) - it := db.db.NewIterator() + it := db.bucket.NewIterator() defer it.Close() if headSeq, _, llen, err = db.lGetMeta(it, metaKey); err != nil { @@ -393,7 +393,7 @@ func (db *DB) LClear(key []byte) (int64, error) { return 0, err } - t := db.listTx + t := db.listBatch t.Lock() defer t.Unlock() @@ -405,7 +405,7 @@ func (db *DB) LClear(key []byte) (int64, error) { } func (db *DB) LMclear(keys ...[]byte) (int64, error) { - t := db.listTx + t := db.listBatch t.Lock() defer t.Unlock() @@ -424,7 +424,7 @@ func (db *DB) LMclear(keys ...[]byte) (int64, error) { } func (db *DB) lFlush() (drop int64, err error) { - t := db.listTx + t := db.listBatch t.Lock() defer t.Unlock() return db.flushType(t, ListType) @@ -459,7 +459,7 @@ func (db *DB) LPersist(key []byte) (int64, error) { return 0, err } - t := db.listTx + t := db.listBatch t.Lock() defer t.Unlock() diff --git a/ledis/t_set.go b/ledis/t_set.go index 40e4452..5a656c1 100644 --- a/ledis/t_set.go +++ b/ledis/t_set.go @@ -106,20 +106,20 @@ func (db *DB) sEncodeStopKey(key []byte) []byte { func (db *DB) sFlush() (drop int64, err error) { - t := db.setTx + t := db.setBatch t.Lock() defer t.Unlock() return db.flushType(t, SetType) } -func (db *DB) sDelete(t *tx, key []byte) int64 { +func (db *DB) sDelete(t *batch, key []byte) int64 { sk := db.sEncodeSizeKey(key) start := db.sEncodeStartKey(key) stop := db.sEncodeStopKey(key) var num int64 = 0 - it := db.db.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) + it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) for ; it.Valid(); it.Next() { t.Delete(it.RawKey()) num++ @@ -132,12 +132,12 @@ func (db *DB) sDelete(t *tx, key []byte) int64 { } func (db *DB) sIncrSize(key []byte, delta int64) (int64, error) { - t := db.setTx + t := db.setBatch sk := db.sEncodeSizeKey(key) var err error var size int64 = 0 - if size, err = Int64(db.db.Get(sk)); err != nil { + if size, err = Int64(db.bucket.Get(sk)); err != nil { return 0, err } else { size += delta @@ -154,7 +154,7 @@ func (db *DB) sIncrSize(key []byte, delta int64) (int64, error) { } func (db *DB) sExpireAt(key []byte, when int64) (int64, error) { - t := db.setTx + t := db.setBatch t.Lock() defer t.Unlock() @@ -172,11 +172,11 @@ func (db *DB) sExpireAt(key []byte, when int64) (int64, error) { } func (db *DB) sSetItem(key []byte, member []byte) (int64, error) { - t := db.setTx + t := db.setBatch ek := db.sEncodeSetKey(key, member) var n int64 = 1 - if v, _ := db.db.Get(ek); v != nil { + if v, _ := db.bucket.Get(ek); v != nil { n = 0 } else { if _, err := db.sIncrSize(key, 1); err != nil { @@ -189,7 +189,7 @@ func (db *DB) sSetItem(key []byte, member []byte) (int64, error) { } func (db *DB) SAdd(key []byte, args ...[]byte) (int64, error) { - t := db.setTx + t := db.setBatch t.Lock() defer t.Unlock() @@ -203,7 +203,7 @@ func (db *DB) SAdd(key []byte, args ...[]byte) (int64, error) { ek = db.sEncodeSetKey(key, args[i]) - if v, err := db.db.Get(ek); err != nil { + if v, err := db.bucket.Get(ek); err != nil { return 0, err } else if v == nil { num++ @@ -228,7 +228,7 @@ func (db *DB) SCard(key []byte) (int64, error) { sk := db.sEncodeSizeKey(key) - return Int64(db.db.Get(sk)) + return Int64(db.bucket.Get(sk)) } func (db *DB) sDiffGeneric(keys ...[]byte) ([][]byte, error) { @@ -354,7 +354,7 @@ func (db *DB) SIsMember(key []byte, member []byte) (int64, error) { ek := db.sEncodeSetKey(key, member) var n int64 = 1 - if v, err := db.db.Get(ek); err != nil { + if v, err := db.bucket.Get(ek); err != nil { return 0, err } else if v == nil { n = 0 @@ -372,7 +372,7 @@ func (db *DB) SMembers(key []byte) ([][]byte, error) { v := make([][]byte, 0, 16) - it := db.db.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) + it := db.bucket.RangeLimitIterator(start, stop, store.RangeROpen, 0, -1) for ; it.Valid(); it.Next() { _, m, err := db.sDecodeSetKey(it.Key()) if err != nil { @@ -388,7 +388,7 @@ func (db *DB) SMembers(key []byte) ([][]byte, error) { } func (db *DB) SRem(key []byte, args ...[]byte) (int64, error) { - t := db.setTx + t := db.setBatch t.Lock() defer t.Unlock() @@ -396,7 +396,7 @@ func (db *DB) SRem(key []byte, args ...[]byte) (int64, error) { var v []byte var err error - it := db.db.NewIterator() + it := db.bucket.NewIterator() defer it.Close() var num int64 = 0 @@ -471,7 +471,7 @@ func (db *DB) sStoreGeneric(dstKey []byte, optType byte, keys ...[]byte) (int64, return 0, err } - t := db.setTx + t := db.setBatch t.Lock() defer t.Unlock() @@ -501,7 +501,7 @@ func (db *DB) sStoreGeneric(dstKey []byte, optType byte, keys ...[]byte) (int64, ek = db.sEncodeSetKey(dstKey, m) - if _, err := db.db.Get(ek); err != nil { + if _, err := db.bucket.Get(ek); err != nil { return 0, err } @@ -523,7 +523,7 @@ func (db *DB) SClear(key []byte) (int64, error) { return 0, err } - t := db.setTx + t := db.setBatch t.Lock() defer t.Unlock() @@ -535,7 +535,7 @@ func (db *DB) SClear(key []byte) (int64, error) { } func (db *DB) SMclear(keys ...[]byte) (int64, error) { - t := db.setTx + t := db.setBatch t.Lock() defer t.Unlock() @@ -583,7 +583,7 @@ func (db *DB) SPersist(key []byte) (int64, error) { return 0, err } - t := db.setTx + t := db.setBatch t.Lock() defer t.Unlock() diff --git a/ledis/t_ttl.go b/ledis/t_ttl.go index e41ed10..3d12606 100644 --- a/ledis/t_ttl.go +++ b/ledis/t_ttl.go @@ -12,11 +12,11 @@ var ( errExpTimeKey = errors.New("invalid expire time key") ) -type retireCallback func(*tx, []byte) int64 +type retireCallback func(*batch, []byte) int64 type elimination struct { db *DB - exp2Tx []*tx + exp2Tx []*batch exp2Retire []retireCallback } @@ -67,11 +67,11 @@ func (db *DB) expDecodeTimeKey(tk []byte) (byte, []byte, int64, error) { return tk[2], tk[11:], int64(binary.BigEndian.Uint64(tk[3:])), nil } -func (db *DB) expire(t *tx, dataType byte, key []byte, duration int64) { +func (db *DB) expire(t *batch, dataType byte, key []byte, duration int64) { db.expireAt(t, dataType, key, time.Now().Unix()+duration) } -func (db *DB) expireAt(t *tx, dataType byte, key []byte, when int64) { +func (db *DB) expireAt(t *batch, dataType byte, key []byte, when int64) { mk := db.expEncodeMetaKey(dataType, key) tk := db.expEncodeTimeKey(dataType, key, when) @@ -82,7 +82,7 @@ func (db *DB) expireAt(t *tx, dataType byte, key []byte, when int64) { func (db *DB) ttl(dataType byte, key []byte) (t int64, err error) { mk := db.expEncodeMetaKey(dataType, key) - if t, err = Int64(db.db.Get(mk)); err != nil || t == 0 { + if t, err = Int64(db.bucket.Get(mk)); err != nil || t == 0 { t = -1 } else { t -= time.Now().Unix() @@ -95,9 +95,9 @@ func (db *DB) ttl(dataType byte, key []byte) (t int64, err error) { return t, err } -func (db *DB) rmExpire(t *tx, dataType byte, key []byte) (int64, error) { +func (db *DB) rmExpire(t *batch, dataType byte, key []byte) (int64, error) { mk := db.expEncodeMetaKey(dataType, key) - if v, err := db.db.Get(mk); err != nil { + if v, err := db.bucket.Get(mk); err != nil { return 0, err } else if v == nil { return 0, nil @@ -111,7 +111,7 @@ func (db *DB) rmExpire(t *tx, dataType byte, key []byte) (int64, error) { } } -func (db *DB) expFlush(t *tx, dataType byte) (err error) { +func (db *DB) expFlush(t *batch, dataType byte) (err error) { minKey := make([]byte, 3) minKey[0] = db.index minKey[1] = ExpTimeType @@ -134,12 +134,12 @@ func (db *DB) expFlush(t *tx, dataType byte) (err error) { func newEliminator(db *DB) *elimination { eli := new(elimination) eli.db = db - eli.exp2Tx = make([]*tx, maxDataType) + eli.exp2Tx = make([]*batch, maxDataType) eli.exp2Retire = make([]retireCallback, maxDataType) return eli } -func (eli *elimination) regRetireContext(dataType byte, t *tx, onRetire retireCallback) { +func (eli *elimination) regRetireContext(dataType byte, t *batch, onRetire retireCallback) { // todo .. need to ensure exist - mapExpMetaType[expType] @@ -151,12 +151,12 @@ func (eli *elimination) regRetireContext(dataType byte, t *tx, onRetire retireCa func (eli *elimination) active() { now := time.Now().Unix() db := eli.db - dbGet := db.db.Get + dbGet := db.bucket.Get minKey := db.expEncodeTimeKey(NoneType, nil, 0) maxKey := db.expEncodeTimeKey(maxDataType, nil, now) - it := db.db.RangeLimitIterator(minKey, maxKey, store.RangeROpen, 0, -1) + it := db.bucket.RangeLimitIterator(minKey, maxKey, store.RangeROpen, 0, -1) for ; it.Valid(); it.Next() { tk := it.RawKey() mk := it.RawValue() diff --git a/ledis/t_zset.go b/ledis/t_zset.go index 202543a..c7edb6c 100644 --- a/ledis/t_zset.go +++ b/ledis/t_zset.go @@ -199,7 +199,7 @@ func (db *DB) zDecodeScoreKey(ek []byte) (key []byte, member []byte, score int64 return } -func (db *DB) zSetItem(t *tx, key []byte, score int64, member []byte) (int64, error) { +func (db *DB) zSetItem(t *batch, key []byte, score int64, member []byte) (int64, error) { if score <= MinScore || score >= MaxScore { return 0, errScoreOverflow } @@ -207,7 +207,7 @@ func (db *DB) zSetItem(t *tx, key []byte, score int64, member []byte) (int64, er var exists int64 = 0 ek := db.zEncodeSetKey(key, member) - if v, err := db.db.Get(ek); err != nil { + if v, err := db.bucket.Get(ek); err != nil { return 0, err } else if v != nil { exists = 1 @@ -228,9 +228,9 @@ func (db *DB) zSetItem(t *tx, key []byte, score int64, member []byte) (int64, er return exists, nil } -func (db *DB) zDelItem(t *tx, key []byte, member []byte, skipDelScore bool) (int64, error) { +func (db *DB) zDelItem(t *batch, key []byte, member []byte, skipDelScore bool) (int64, error) { ek := db.zEncodeSetKey(key, member) - if v, err := db.db.Get(ek); err != nil { + if v, err := db.bucket.Get(ek); err != nil { return 0, err } else if v == nil { //not exists @@ -253,14 +253,14 @@ func (db *DB) zDelItem(t *tx, key []byte, member []byte, skipDelScore bool) (int return 1, nil } -func (db *DB) zDelete(t *tx, key []byte) int64 { +func (db *DB) zDelete(t *batch, key []byte) int64 { delMembCnt, _ := db.zRemRange(t, key, MinScore, MaxScore, 0, -1) // todo : log err return delMembCnt } func (db *DB) zExpireAt(key []byte, when int64) (int64, error) { - t := db.zsetTx + t := db.zsetBatch t.Lock() defer t.Unlock() @@ -280,7 +280,7 @@ func (db *DB) ZAdd(key []byte, args ...ScorePair) (int64, error) { return 0, nil } - t := db.zsetTx + t := db.zsetBatch t.Lock() defer t.Unlock() @@ -310,10 +310,10 @@ func (db *DB) ZAdd(key []byte, args ...ScorePair) (int64, error) { return num, err } -func (db *DB) zIncrSize(t *tx, key []byte, delta int64) (int64, error) { +func (db *DB) zIncrSize(t *batch, key []byte, delta int64) (int64, error) { sk := db.zEncodeSizeKey(key) - size, err := Int64(db.db.Get(sk)) + size, err := Int64(db.bucket.Get(sk)) if err != nil { return 0, err } else { @@ -336,7 +336,7 @@ func (db *DB) ZCard(key []byte) (int64, error) { } sk := db.zEncodeSizeKey(key) - return Int64(db.db.Get(sk)) + return Int64(db.bucket.Get(sk)) } func (db *DB) ZScore(key []byte, member []byte) (int64, error) { @@ -347,7 +347,7 @@ func (db *DB) ZScore(key []byte, member []byte) (int64, error) { var score int64 = InvalidScore k := db.zEncodeSetKey(key, member) - if v, err := db.db.Get(k); err != nil { + if v, err := db.bucket.Get(k); err != nil { return InvalidScore, err } else if v == nil { return InvalidScore, ErrScoreMiss @@ -365,7 +365,7 @@ func (db *DB) ZRem(key []byte, members ...[]byte) (int64, error) { return 0, nil } - t := db.zsetTx + t := db.zsetBatch t.Lock() defer t.Unlock() @@ -395,14 +395,14 @@ func (db *DB) ZIncrBy(key []byte, delta int64, member []byte) (int64, error) { return InvalidScore, err } - t := db.zsetTx + t := db.zsetBatch t.Lock() defer t.Unlock() ek := db.zEncodeSetKey(key, member) var oldScore int64 = 0 - v, err := db.db.Get(ek) + v, err := db.bucket.Get(ek) if err != nil { return InvalidScore, err } else if v == nil { @@ -441,7 +441,7 @@ func (db *DB) ZCount(key []byte, min int64, max int64) (int64, error) { rangeType := store.RangeROpen - it := db.db.RangeLimitIterator(minKey, maxKey, rangeType, 0, -1) + it := db.bucket.RangeLimitIterator(minKey, maxKey, rangeType, 0, -1) var n int64 = 0 for ; it.Valid(); it.Next() { n++ @@ -458,7 +458,7 @@ func (db *DB) zrank(key []byte, member []byte, reverse bool) (int64, error) { k := db.zEncodeSetKey(key, member) - it := db.db.NewIterator() + it := db.bucket.NewIterator() defer it.Close() if v := it.Find(k); v == nil { @@ -504,13 +504,13 @@ func (db *DB) zIterator(key []byte, min int64, max int64, offset int, count int, maxKey := db.zEncodeStopScoreKey(key, max) if !reverse { - return db.db.RangeLimitIterator(minKey, maxKey, store.RangeClose, offset, count) + return db.bucket.RangeLimitIterator(minKey, maxKey, store.RangeClose, offset, count) } else { - return db.db.RevRangeLimitIterator(minKey, maxKey, store.RangeClose, offset, count) + return db.bucket.RevRangeLimitIterator(minKey, maxKey, store.RangeClose, offset, count) } } -func (db *DB) zRemRange(t *tx, key []byte, min int64, max int64, offset int, count int) (int64, error) { +func (db *DB) zRemRange(t *batch, key []byte, min int64, max int64, offset int, count int) (int64, error) { if len(key) > MaxKeySize { return 0, errKeySize } @@ -626,7 +626,7 @@ func (db *DB) zParseLimit(key []byte, start int, stop int) (offset int, count in } func (db *DB) ZClear(key []byte) (int64, error) { - t := db.zsetTx + t := db.zsetBatch t.Lock() defer t.Unlock() @@ -639,7 +639,7 @@ func (db *DB) ZClear(key []byte) (int64, error) { } func (db *DB) ZMclear(keys ...[]byte) (int64, error) { - t := db.zsetTx + t := db.zsetBatch t.Lock() defer t.Unlock() @@ -677,7 +677,7 @@ func (db *DB) ZRemRangeByRank(key []byte, start int, stop int) (int64, error) { var rmCnt int64 - t := db.zsetTx + t := db.zsetBatch t.Lock() defer t.Unlock() @@ -691,7 +691,7 @@ func (db *DB) ZRemRangeByRank(key []byte, start int, stop int) (int64, error) { //min and max must be inclusive func (db *DB) ZRemRangeByScore(key []byte, min int64, max int64) (int64, error) { - t := db.zsetTx + t := db.zsetBatch t.Lock() defer t.Unlock() @@ -735,7 +735,7 @@ func (db *DB) ZRangeByScoreGeneric(key []byte, min int64, max int64, } func (db *DB) zFlush() (drop int64, err error) { - t := db.zsetTx + t := db.zsetBatch t.Lock() defer t.Unlock() return db.flushType(t, ZSetType) @@ -770,7 +770,7 @@ func (db *DB) ZPersist(key []byte) (int64, error) { return 0, err } - t := db.zsetTx + t := db.zsetBatch t.Lock() defer t.Unlock() @@ -842,7 +842,7 @@ func (db *DB) ZUnionStore(destKey []byte, srcKeys [][]byte, weights []int64, agg } } - t := db.zsetTx + t := db.zsetBatch t.Lock() defer t.Unlock() @@ -912,7 +912,7 @@ func (db *DB) ZInterStore(destKey []byte, srcKeys [][]byte, weights []int64, agg destMap = tmpMap } - t := db.zsetTx + t := db.zsetBatch t.Lock() defer t.Unlock() diff --git a/ledis/tx.go b/ledis/tx.go index 09e4cea..009bfac 100644 --- a/ledis/tx.go +++ b/ledis/tx.go @@ -1,84 +1,203 @@ package ledis import ( + "errors" "github.com/siddontang/ledisdb/store" "sync" ) -type tx struct { - m sync.Mutex +var ( + ErrNestTx = errors.New("nest transaction not supported") + ErrTxDone = errors.New("Transaction has already been committed or rolled back") +) - l *Ledis - wb store.WriteBatch +type batch struct { + l *Ledis - binlog *BinLog - batch [][]byte + store.WriteBatch + + sync.Locker + + logs [][]byte + + tx *Tx } -func newTx(l *Ledis) *tx { - t := new(tx) - - t.l = l - t.wb = l.ldb.NewWriteBatch() - - t.batch = make([][]byte, 0, 4) - t.binlog = l.binlog - return t +type dbBatchLocker struct { + sync.Mutex + dbLock *sync.RWMutex } -func (t *tx) Close() { - t.wb = nil +type txBatchLocker struct { } -func (t *tx) Put(key []byte, value []byte) { - t.wb.Put(key, value) - - if t.binlog != nil { - buf := encodeBinLogPut(key, value) - t.batch = append(t.batch, buf) - } +func (l *txBatchLocker) Lock() { } -func (t *tx) Delete(key []byte) { - t.wb.Delete(key) - - if t.binlog != nil { - buf := encodeBinLogDelete(key) - t.batch = append(t.batch, buf) - } +func (l *txBatchLocker) Unlock() { } -func (t *tx) Lock() { - t.m.Lock() +func (l *dbBatchLocker) Lock() { + l.dbLock.RLock() + l.Mutex.Lock() } -func (t *tx) Unlock() { - t.batch = t.batch[0:0] - t.wb.Rollback() - t.m.Unlock() +func (l *dbBatchLocker) Unlock() { + l.Mutex.Unlock() + l.dbLock.RUnlock() } -func (t *tx) Commit() error { - var err error - if t.binlog != nil { - t.l.Lock() - err = t.wb.Commit() - if err != nil { - t.l.Unlock() - return err +func (db *DB) newBatch() *batch { + b := new(batch) + + b.WriteBatch = db.bucket.NewWriteBatch() + b.Locker = &dbBatchLocker{dbLock: db.dbLock} + b.l = db.l + + return b +} + +func (b *batch) Commit() error { + b.l.Lock() + defer b.l.Unlock() + + err := b.WriteBatch.Commit() + + if b.l.binlog != nil { + if err == nil { + if b.tx == nil { + b.l.binlog.Log(b.logs...) + } else { + b.tx.logs = append(b.tx.logs, b.logs...) + } } - - err = t.binlog.Log(t.batch...) - - t.l.Unlock() - } else { - t.l.Lock() - err = t.wb.Commit() - t.l.Unlock() + b.logs = [][]byte{} } + return err } -func (t *tx) Rollback() { - t.wb.Rollback() +func (b *batch) Lock() { + b.Locker.Lock() +} + +func (b *batch) Unlock() { + if b.l.binlog != nil { + b.logs = [][]byte{} + } + b.Rollback() + b.Locker.Unlock() +} + +func (b *batch) Put(key []byte, value []byte) { + if b.l.binlog != nil { + buf := encodeBinLogPut(key, value) + b.logs = append(b.logs, buf) + } + b.WriteBatch.Put(key, value) +} + +func (b *batch) Delete(key []byte) { + if b.l.binlog != nil { + buf := encodeBinLogDelete(key) + b.logs = append(b.logs, buf) + } + b.WriteBatch.Delete(key) +} + +type Tx struct { + *DB + + tx *store.Tx + + logs [][]byte +} + +func (db *DB) IsTransaction() bool { + return db.isTx +} + +// Begin a transaction, it will block all other write operations before calling Commit or Rollback. +// You must be very careful to prevent long-time transaction. +func (db *DB) Begin() (*Tx, error) { + if db.isTx { + return nil, ErrNestTx + } + + tx := new(Tx) + + tx.DB = new(DB) + tx.DB.dbLock = db.dbLock + + tx.DB.dbLock.Lock() + + tx.DB.l = db.l + + tx.DB.sdb = db.sdb + + var err error + tx.tx, err = db.sdb.Begin() + if err != nil { + tx.DB.dbLock.Unlock() + return nil, err + } + + tx.DB.bucket = tx.tx + + tx.DB.isTx = true + + tx.DB.index = db.index + + tx.DB.kvBatch = tx.newBatch() + tx.DB.listBatch = tx.newBatch() + tx.DB.hashBatch = tx.newBatch() + tx.DB.zsetBatch = tx.newBatch() + tx.DB.binBatch = tx.newBatch() + tx.DB.setBatch = tx.newBatch() + + return tx, nil +} + +func (tx *Tx) Commit() error { + if tx.tx == nil { + return ErrTxDone + } + + tx.l.Lock() + err := tx.tx.Commit() + tx.tx = nil + + if len(tx.logs) > 0 { + tx.l.binlog.Log(tx.logs...) + } + + tx.l.Unlock() + + tx.DB.dbLock.Unlock() + tx.DB = nil + return err +} + +func (tx *Tx) Rollback() error { + if tx.tx == nil { + return ErrTxDone + } + + err := tx.tx.Rollback() + tx.tx = nil + + tx.DB.dbLock.Unlock() + tx.DB = nil + return err +} + +func (tx *Tx) newBatch() *batch { + b := new(batch) + + b.l = tx.l + b.WriteBatch = tx.tx.NewWriteBatch() + b.Locker = &txBatchLocker{} + b.tx = tx + + return b } diff --git a/ledis/tx_test.go b/ledis/tx_test.go new file mode 100644 index 0000000..bf06012 --- /dev/null +++ b/ledis/tx_test.go @@ -0,0 +1,173 @@ +package ledis + +import ( + "github.com/siddontang/ledisdb/config" + "os" + "testing" +) + +func testTxRollback(t *testing.T, db *DB) { + var err error + key1 := []byte("tx_key1") + key2 := []byte("tx_key2") + field2 := []byte("tx_field2") + + err = db.Set(key1, []byte("value")) + if err != nil { + t.Fatal(err) + } + + _, err = db.HSet(key2, field2, []byte("value")) + if err != nil { + t.Fatal(err) + } + + var tx *Tx + tx, err = db.Begin() + if err != nil { + t.Fatal(err) + } + + defer tx.Rollback() + + err = tx.Set(key1, []byte("1")) + + if err != nil { + t.Fatal(err) + } + + _, err = tx.HSet(key2, field2, []byte("2")) + + if err != nil { + t.Fatal(err) + } + + _, err = tx.HSet([]byte("no_key"), field2, []byte("2")) + + if err != nil { + t.Fatal(err) + } + + if v, err := tx.Get(key1); err != nil { + t.Fatal(err) + } else if string(v) != "1" { + t.Fatal(string(v)) + } + + if v, err := tx.HGet(key2, field2); err != nil { + t.Fatal(err) + } else if string(v) != "2" { + t.Fatal(string(v)) + } + + err = tx.Rollback() + if err != nil { + t.Fatal(err) + } + + if v, err := db.Get(key1); err != nil { + t.Fatal(err) + } else if string(v) != "value" { + t.Fatal(string(v)) + } + + if v, err := db.HGet(key2, field2); err != nil { + t.Fatal(err) + } else if string(v) != "value" { + t.Fatal(string(v)) + } +} + +func testTxCommit(t *testing.T, db *DB) { + var err error + key1 := []byte("tx_key1") + key2 := []byte("tx_key2") + field2 := []byte("tx_field2") + + err = db.Set(key1, []byte("value")) + if err != nil { + t.Fatal(err) + } + + _, err = db.HSet(key2, field2, []byte("value")) + if err != nil { + t.Fatal(err) + } + + var tx *Tx + tx, err = db.Begin() + if err != nil { + t.Fatal(err) + } + + defer tx.Rollback() + + err = tx.Set(key1, []byte("1")) + + if err != nil { + t.Fatal(err) + } + + _, err = tx.HSet(key2, field2, []byte("2")) + + if err != nil { + t.Fatal(err) + } + + if v, err := tx.Get(key1); err != nil { + t.Fatal(err) + } else if string(v) != "1" { + t.Fatal(string(v)) + } + + if v, err := tx.HGet(key2, field2); err != nil { + t.Fatal(err) + } else if string(v) != "2" { + t.Fatal(string(v)) + } + + err = tx.Commit() + if err != nil { + t.Fatal(err) + } + + if v, err := db.Get(key1); err != nil { + t.Fatal(err) + } else if string(v) != "1" { + t.Fatal(string(v)) + } + + if v, err := db.HGet(key2, field2); err != nil { + t.Fatal(err) + } else if string(v) != "2" { + t.Fatal(string(v)) + } +} + +func testTx(t *testing.T, name string) { + cfg := new(config.Config) + cfg.DataDir = "/tmp/ledis_test_tx" + + cfg.DBName = name + cfg.LMDB.MapSize = 10 * 1024 * 1024 + + os.RemoveAll(cfg.DataDir) + + l, err := Open(cfg) + if err != nil { + t.Fatal(err) + } + + defer l.Close() + + db, _ := l.Select(0) + + testTxRollback(t, db) + testTxCommit(t, db) +} + +//only lmdb, boltdb support Transaction +func TestTx(t *testing.T) { + testTx(t, "lmdb") + testTx(t, "boltdb") +} diff --git a/server/app.go b/server/app.go index 5e1c3b3..d5c77c9 100644 --- a/server/app.go +++ b/server/app.go @@ -25,6 +25,8 @@ type App struct { //for slave replication m *master + + info *info } func netType(s string) string { @@ -51,6 +53,10 @@ func NewApp(cfg *config.Config) (*App, error) { var err error + if app.info, err = newInfo(app); err != nil { + return nil, err + } + if app.listener, err = net.Listen(netType(cfg.Addr), cfg.Addr); err != nil { return nil, err } diff --git a/server/client.go b/server/client.go new file mode 100644 index 0000000..7613562 --- /dev/null +++ b/server/client.go @@ -0,0 +1,128 @@ +package server + +import ( + "bytes" + "fmt" + "github.com/siddontang/ledisdb/ledis" + "io" + "time" +) + +var txUnsupportedCmds = map[string]struct{}{ + "select": struct{}{}, + "slaveof": struct{}{}, + "fullsync": struct{}{}, + "sync": struct{}{}, + "begin": struct{}{}, +} + +type responseWriter interface { + writeError(error) + writeStatus(string) + writeInteger(int64) + writeBulk([]byte) + writeArray([]interface{}) + writeSliceArray([][]byte) + writeFVPairArray([]ledis.FVPair) + writeScorePairArray([]ledis.ScorePair, bool) + writeBulkFrom(int64, io.Reader) + flush() +} + +type client struct { + app *App + ldb *ledis.Ledis + db *ledis.DB + + remoteAddr string + cmd string + args [][]byte + + resp responseWriter + + syncBuf bytes.Buffer + compressBuf []byte + + reqErr chan error + + buf bytes.Buffer + + tx *ledis.Tx +} + +func newClient(app *App) *client { + c := new(client) + + c.app = app + c.ldb = app.ldb + c.db, _ = app.ldb.Select(0) //use default db + + c.compressBuf = make([]byte, 256) + c.reqErr = make(chan error) + + return c +} + +func (c *client) isInTransaction() bool { + return c.tx != nil +} + +func (c *client) perform() { + var err error + + start := time.Now() + + if len(c.cmd) == 0 { + err = ErrEmptyCommand + } else if exeCmd, ok := regCmds[c.cmd]; !ok { + err = ErrNotFound + } else { + if c.isInTransaction() { + if _, ok := txUnsupportedCmds[c.cmd]; ok { + err = fmt.Errorf("%s not supported in transaction", c.cmd) + } + } + + if err == nil { + go func() { + c.reqErr <- exeCmd(c) + }() + + err = <-c.reqErr + } + } + + duration := time.Since(start) + + if c.app.access != nil { + fullCmd := c.catGenericCommand() + cost := duration.Nanoseconds() / 1000000 + + truncateLen := len(fullCmd) + if truncateLen > 256 { + truncateLen = 256 + } + + c.app.access.Log(c.remoteAddr, cost, fullCmd[:truncateLen], err) + } + + if err != nil { + c.resp.writeError(err) + } + c.resp.flush() + return +} + +func (c *client) catGenericCommand() []byte { + buffer := c.buf + buffer.Reset() + + buffer.Write([]byte(c.cmd)) + + for _, arg := range c.args { + buffer.WriteByte(' ') + buffer.Write(arg) + } + + return buffer.Bytes() +} diff --git a/server/client_http.go b/server/client_http.go index 01272d5..0db0843 100644 --- a/server/client_http.go +++ b/server/client_http.go @@ -18,20 +18,18 @@ var allowedContentTypes = map[string]struct{}{ "bson": struct{}{}, "msgpack": struct{}{}, } -var unsupportedCommands = map[string]struct{}{ +var httpUnsupportedCommands = map[string]struct{}{ "slaveof": struct{}{}, "fullsync": struct{}{}, "sync": struct{}{}, "quit": struct{}{}, + "begin": struct{}{}, + "commit": struct{}{}, + "rollback": struct{}{}, } type httpClient struct { - app *App - db *ledis.DB - ldb *ledis.Ledis - - resp responseWriter - req *requestContext + *client } type httpWriter struct { @@ -43,58 +41,51 @@ type httpWriter struct { func newClientHTTP(app *App, w http.ResponseWriter, r *http.Request) { var err error c := new(httpClient) - c.app = app - c.ldb = app.ldb - c.db, err = c.ldb.Select(0) - if err != nil { - w.Write([]byte(err.Error())) - return - } + c.client = newClient(app) - c.req, err = c.makeRequest(app, r, w) + err = c.makeRequest(app, r, w) if err != nil { w.Write([]byte(err.Error())) return } - c.req.perform() + c.perform() } func (c *httpClient) addr(r *http.Request) string { return r.RemoteAddr } -func (c *httpClient) makeRequest(app *App, r *http.Request, w http.ResponseWriter) (*requestContext, error) { +func (c *httpClient) makeRequest(app *App, r *http.Request, w http.ResponseWriter) error { var err error db, cmd, argsStr, contentType := c.parseReqPath(r) c.db, err = app.ldb.Select(db) if err != nil { - return nil, err + return err } contentType = strings.ToLower(contentType) if _, ok := allowedContentTypes[contentType]; !ok { - return nil, fmt.Errorf("unsupported content type: '%s', only json, bson, msgpack are supported", contentType) + return fmt.Errorf("unsupported content type: '%s', only json, bson, msgpack are supported", contentType) } - req := newRequestContext(app) args := make([][]byte, len(argsStr)) for i, arg := range argsStr { args[i] = []byte(arg) } - req.cmd = strings.ToLower(cmd) - if _, ok := unsupportedCommands[req.cmd]; ok { - return nil, fmt.Errorf("unsupported command: '%s'", cmd) + c.cmd = strings.ToLower(cmd) + if _, ok := httpUnsupportedCommands[c.cmd]; ok { + return fmt.Errorf("unsupported command: '%s'", cmd) } - req.args = args + c.args = args - req.remoteAddr = c.addr(r) - req.resp = &httpWriter{contentType, cmd, w} - return req, nil + c.remoteAddr = c.addr(r) + c.resp = &httpWriter{contentType, cmd, w} + return nil } func (c *httpClient) parseReqPath(r *http.Request) (db int, cmd string, args []string, contentType string) { diff --git a/server/client_resp.go b/server/client_resp.go index e89ad96..22e6e69 100644 --- a/server/client_resp.go +++ b/server/client_resp.go @@ -15,14 +15,10 @@ import ( var errReadRequest = errors.New("invalid request protocol") type respClient struct { - app *App - ldb *ledis.Ledis - db *ledis.DB + *client conn net.Conn rb *bufio.Reader - - req *requestContext } type respWriter struct { @@ -32,16 +28,13 @@ type respWriter struct { func newClientRESP(conn net.Conn, app *App) { c := new(respClient) - c.app = app + c.client = newClient(app) c.conn = conn - c.ldb = app.ldb - c.db, _ = app.ldb.Select(0) c.rb = bufio.NewReaderSize(conn, 256) - c.req = newRequestContext(app) - c.req.resp = newWriterRESP(conn) - c.req.remoteAddr = conn.RemoteAddr().String() + c.resp = newWriterRESP(conn) + c.remoteAddr = conn.RemoteAddr().String() go c.run() } @@ -56,7 +49,14 @@ func (c *respClient) run() { log.Fatal("client run panic %s:%v", buf, e) } - c.conn.Close() + if c.conn != nil { + c.conn.Close() + } + + if c.tx != nil { + c.tx.Rollback() + c.tx = nil + } }() for { @@ -129,27 +129,21 @@ func (c *respClient) readRequest() ([][]byte, error) { } func (c *respClient) handleRequest(reqData [][]byte) { - req := c.req - if len(reqData) == 0 { - c.req.cmd = "" - c.req.args = reqData[0:0] + c.cmd = "" + c.args = reqData[0:0] } else { - c.req.cmd = strings.ToLower(ledis.String(reqData[0])) - c.req.args = reqData[1:] + c.cmd = strings.ToLower(ledis.String(reqData[0])) + c.args = reqData[1:] } - if c.req.cmd == "quit" { - c.req.resp.writeStatus(OK) - c.req.resp.flush() + if c.cmd == "quit" { + c.resp.writeStatus(OK) + c.resp.flush() c.conn.Close() return } - req.db = c.db - - c.req.perform() - - c.db = req.db // "SELECT" + c.perform() return } diff --git a/server/cmd_bit.go b/server/cmd_bit.go index a08a966..5845f28 100644 --- a/server/cmd_bit.go +++ b/server/cmd_bit.go @@ -5,36 +5,36 @@ import ( "strings" ) -func bgetCommand(req *requestContext) error { - args := req.args +func bgetCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if v, err := req.db.BGet(args[0]); err != nil { + if v, err := c.db.BGet(args[0]); err != nil { return err } else { - req.resp.writeBulk(v) + c.resp.writeBulk(v) } return nil } -func bdeleteCommand(req *requestContext) error { - args := req.args +func bdeleteCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if n, err := req.db.BDelete(args[0]); err != nil { + if n, err := c.db.BDelete(args[0]); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func bsetbitCommand(req *requestContext) error { - args := req.args +func bsetbitCommand(c *client) error { + args := c.args if len(args) != 3 { return ErrCmdParams } @@ -58,16 +58,16 @@ func bsetbitCommand(req *requestContext) error { return ErrBool } - if ori, err := req.db.BSetBit(args[0], offset, uint8(val)); err != nil { + if ori, err := c.db.BSetBit(args[0], offset, uint8(val)); err != nil { return err } else { - req.resp.writeInteger(int64(ori)) + c.resp.writeInteger(int64(ori)) } return nil } -func bgetbitCommand(req *requestContext) error { - args := req.args +func bgetbitCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } @@ -78,16 +78,16 @@ func bgetbitCommand(req *requestContext) error { return ErrOffset } - if v, err := req.db.BGetBit(args[0], offset); err != nil { + if v, err := c.db.BGetBit(args[0], offset); err != nil { return err } else { - req.resp.writeInteger(int64(v)) + c.resp.writeInteger(int64(v)) } return nil } -func bmsetbitCommand(req *requestContext) error { - args := req.args +func bmsetbitCommand(c *client) error { + args := c.args if len(args) < 3 { return ErrCmdParams } @@ -124,16 +124,16 @@ func bmsetbitCommand(req *requestContext) error { pairs[i].Val = uint8(val) } - if place, err := req.db.BMSetBit(key, pairs...); err != nil { + if place, err := c.db.BMSetBit(key, pairs...); err != nil { return err } else { - req.resp.writeInteger(place) + c.resp.writeInteger(place) } return nil } -func bcountCommand(req *requestContext) error { - args := req.args +func bcountCommand(c *client) error { + args := c.args argCnt := len(args) if !(argCnt > 0 && argCnt <= 3) { @@ -159,16 +159,16 @@ func bcountCommand(req *requestContext) error { } } - if cnt, err := req.db.BCount(args[0], start, end); err != nil { + if cnt, err := c.db.BCount(args[0], start, end); err != nil { return err } else { - req.resp.writeInteger(int64(cnt)) + c.resp.writeInteger(int64(cnt)) } return nil } -func boptCommand(req *requestContext) error { - args := req.args +func boptCommand(c *client) error { + args := c.args if len(args) < 2 { return ErrCmdParams } @@ -194,16 +194,16 @@ func boptCommand(req *requestContext) error { if len(srcKeys) == 0 { return ErrCmdParams } - if blen, err := req.db.BOperation(op, dstKey, srcKeys...); err != nil { + if blen, err := c.db.BOperation(op, dstKey, srcKeys...); err != nil { return err } else { - req.resp.writeInteger(int64(blen)) + c.resp.writeInteger(int64(blen)) } return nil } -func bexpireCommand(req *requestContext) error { - args := req.args +func bexpireCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } @@ -213,17 +213,17 @@ func bexpireCommand(req *requestContext) error { return ErrValue } - if v, err := req.db.BExpire(args[0], duration); err != nil { + if v, err := c.db.BExpire(args[0], duration); err != nil { return err } else { - req.resp.writeInteger(v) + c.resp.writeInteger(v) } return nil } -func bexpireAtCommand(req *requestContext) error { - args := req.args +func bexpireAtCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } @@ -233,40 +233,40 @@ func bexpireAtCommand(req *requestContext) error { return ErrValue } - if v, err := req.db.BExpireAt(args[0], when); err != nil { + if v, err := c.db.BExpireAt(args[0], when); err != nil { return err } else { - req.resp.writeInteger(v) + c.resp.writeInteger(v) } return nil } -func bttlCommand(req *requestContext) error { - args := req.args +func bttlCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if v, err := req.db.BTTL(args[0]); err != nil { + if v, err := c.db.BTTL(args[0]); err != nil { return err } else { - req.resp.writeInteger(v) + c.resp.writeInteger(v) } return nil } -func bpersistCommand(req *requestContext) error { - args := req.args +func bpersistCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if n, err := req.db.BPersist(args[0]); err != nil { + if n, err := c.db.BPersist(args[0]); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil diff --git a/server/cmd_hash.go b/server/cmd_hash.go index 0d195f6..5c336a3 100644 --- a/server/cmd_hash.go +++ b/server/cmd_hash.go @@ -4,87 +4,87 @@ import ( "github.com/siddontang/ledisdb/ledis" ) -func hsetCommand(req *requestContext) error { - args := req.args +func hsetCommand(c *client) error { + args := c.args if len(args) != 3 { return ErrCmdParams } - if n, err := req.db.HSet(args[0], args[1], args[2]); err != nil { + if n, err := c.db.HSet(args[0], args[1], args[2]); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func hgetCommand(req *requestContext) error { - args := req.args +func hgetCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } - if v, err := req.db.HGet(args[0], args[1]); err != nil { + if v, err := c.db.HGet(args[0], args[1]); err != nil { return err } else { - req.resp.writeBulk(v) + c.resp.writeBulk(v) } return nil } -func hexistsCommand(req *requestContext) error { - args := req.args +func hexistsCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } var n int64 = 1 - if v, err := req.db.HGet(args[0], args[1]); err != nil { + if v, err := c.db.HGet(args[0], args[1]); err != nil { return err } else { if v == nil { n = 0 } - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func hdelCommand(req *requestContext) error { - args := req.args +func hdelCommand(c *client) error { + args := c.args if len(args) < 2 { return ErrCmdParams } - if n, err := req.db.HDel(args[0], args[1:]...); err != nil { + if n, err := c.db.HDel(args[0], args[1:]...); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func hlenCommand(req *requestContext) error { - args := req.args +func hlenCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if n, err := req.db.HLen(args[0]); err != nil { + if n, err := c.db.HLen(args[0]); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func hincrbyCommand(req *requestContext) error { - args := req.args +func hincrbyCommand(c *client) error { + args := c.args if len(args) != 3 { return ErrCmdParams } @@ -95,16 +95,16 @@ func hincrbyCommand(req *requestContext) error { } var n int64 - if n, err = req.db.HIncrBy(args[0], args[1], delta); err != nil { + if n, err = c.db.HIncrBy(args[0], args[1], delta); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func hmsetCommand(req *requestContext) error { - args := req.args +func hmsetCommand(c *client) error { + args := c.args if len(args) < 3 { return ErrCmdParams } @@ -123,107 +123,107 @@ func hmsetCommand(req *requestContext) error { kvs[i].Value = args[2*i+1] } - if err := req.db.HMset(key, kvs...); err != nil { + if err := c.db.HMset(key, kvs...); err != nil { return err } else { - req.resp.writeStatus(OK) + c.resp.writeStatus(OK) } return nil } -func hmgetCommand(req *requestContext) error { - args := req.args +func hmgetCommand(c *client) error { + args := c.args if len(args) < 2 { return ErrCmdParams } - if v, err := req.db.HMget(args[0], args[1:]...); err != nil { + if v, err := c.db.HMget(args[0], args[1:]...); err != nil { return err } else { - req.resp.writeSliceArray(v) + c.resp.writeSliceArray(v) } return nil } -func hgetallCommand(req *requestContext) error { - args := req.args +func hgetallCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if v, err := req.db.HGetAll(args[0]); err != nil { + if v, err := c.db.HGetAll(args[0]); err != nil { return err } else { - req.resp.writeFVPairArray(v) + c.resp.writeFVPairArray(v) } return nil } -func hkeysCommand(req *requestContext) error { - args := req.args +func hkeysCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if v, err := req.db.HKeys(args[0]); err != nil { + if v, err := c.db.HKeys(args[0]); err != nil { return err } else { - req.resp.writeSliceArray(v) + c.resp.writeSliceArray(v) } return nil } -func hvalsCommand(req *requestContext) error { - args := req.args +func hvalsCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if v, err := req.db.HValues(args[0]); err != nil { + if v, err := c.db.HValues(args[0]); err != nil { return err } else { - req.resp.writeSliceArray(v) + c.resp.writeSliceArray(v) } return nil } -func hclearCommand(req *requestContext) error { - args := req.args +func hclearCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if n, err := req.db.HClear(args[0]); err != nil { + if n, err := c.db.HClear(args[0]); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func hmclearCommand(req *requestContext) error { - args := req.args +func hmclearCommand(c *client) error { + args := c.args if len(args) < 1 { return ErrCmdParams } - if n, err := req.db.HMclear(args...); err != nil { + if n, err := c.db.HMclear(args...); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func hexpireCommand(req *requestContext) error { - args := req.args +func hexpireCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } @@ -233,17 +233,17 @@ func hexpireCommand(req *requestContext) error { return ErrValue } - if v, err := req.db.HExpire(args[0], duration); err != nil { + if v, err := c.db.HExpire(args[0], duration); err != nil { return err } else { - req.resp.writeInteger(v) + c.resp.writeInteger(v) } return nil } -func hexpireAtCommand(req *requestContext) error { - args := req.args +func hexpireAtCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } @@ -253,40 +253,40 @@ func hexpireAtCommand(req *requestContext) error { return ErrValue } - if v, err := req.db.HExpireAt(args[0], when); err != nil { + if v, err := c.db.HExpireAt(args[0], when); err != nil { return err } else { - req.resp.writeInteger(v) + c.resp.writeInteger(v) } return nil } -func httlCommand(req *requestContext) error { - args := req.args +func httlCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if v, err := req.db.HTTL(args[0]); err != nil { + if v, err := c.db.HTTL(args[0]); err != nil { return err } else { - req.resp.writeInteger(v) + c.resp.writeInteger(v) } return nil } -func hpersistCommand(req *requestContext) error { - args := req.args +func hpersistCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if n, err := req.db.HPersist(args[0]); err != nil { + if n, err := c.db.HPersist(args[0]); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil diff --git a/server/cmd_kv.go b/server/cmd_kv.go index 83b6e67..cd0cac0 100644 --- a/server/cmd_kv.go +++ b/server/cmd_kv.go @@ -4,112 +4,112 @@ import ( "github.com/siddontang/ledisdb/ledis" ) -func getCommand(req *requestContext) error { - args := req.args +func getCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if v, err := req.db.Get(args[0]); err != nil { + if v, err := c.db.Get(args[0]); err != nil { return err } else { - req.resp.writeBulk(v) + c.resp.writeBulk(v) } return nil } -func setCommand(req *requestContext) error { - args := req.args +func setCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } - if err := req.db.Set(args[0], args[1]); err != nil { + if err := c.db.Set(args[0], args[1]); err != nil { return err } else { - req.resp.writeStatus(OK) + c.resp.writeStatus(OK) } return nil } -func getsetCommand(req *requestContext) error { - args := req.args +func getsetCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } - if v, err := req.db.GetSet(args[0], args[1]); err != nil { + if v, err := c.db.GetSet(args[0], args[1]); err != nil { return err } else { - req.resp.writeBulk(v) + c.resp.writeBulk(v) } return nil } -func setnxCommand(req *requestContext) error { - args := req.args +func setnxCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } - if n, err := req.db.SetNX(args[0], args[1]); err != nil { + if n, err := c.db.SetNX(args[0], args[1]); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func existsCommand(req *requestContext) error { - args := req.args +func existsCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if n, err := req.db.Exists(args[0]); err != nil { + if n, err := c.db.Exists(args[0]); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func incrCommand(req *requestContext) error { - args := req.args +func incrCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if n, err := req.db.Incr(req.args[0]); err != nil { + if n, err := c.db.Incr(c.args[0]); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func decrCommand(req *requestContext) error { - args := req.args +func decrCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if n, err := req.db.Decr(req.args[0]); err != nil { + if n, err := c.db.Decr(c.args[0]); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func incrbyCommand(req *requestContext) error { - args := req.args +func incrbyCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } @@ -119,17 +119,17 @@ func incrbyCommand(req *requestContext) error { return ErrValue } - if n, err := req.db.IncrBy(req.args[0], delta); err != nil { + if n, err := c.db.IncrBy(c.args[0], delta); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func decrbyCommand(req *requestContext) error { - args := req.args +func decrbyCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } @@ -139,32 +139,32 @@ func decrbyCommand(req *requestContext) error { return ErrValue } - if n, err := req.db.DecrBy(req.args[0], delta); err != nil { + if n, err := c.db.DecrBy(c.args[0], delta); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func delCommand(req *requestContext) error { - args := req.args +func delCommand(c *client) error { + args := c.args if len(args) == 0 { return ErrCmdParams } - if n, err := req.db.Del(args...); err != nil { + if n, err := c.db.Del(args...); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func msetCommand(req *requestContext) error { - args := req.args +func msetCommand(c *client) error { + args := c.args if len(args) == 0 || len(args)%2 != 0 { return ErrCmdParams } @@ -175,36 +175,36 @@ func msetCommand(req *requestContext) error { kvs[i].Value = args[2*i+1] } - if err := req.db.MSet(kvs...); err != nil { + if err := c.db.MSet(kvs...); err != nil { return err } else { - req.resp.writeStatus(OK) + c.resp.writeStatus(OK) } return nil } -// func setexCommand(req *requestContext) error { +// func setexCommand(c *client) error { // return nil // } -func mgetCommand(req *requestContext) error { - args := req.args +func mgetCommand(c *client) error { + args := c.args if len(args) == 0 { return ErrCmdParams } - if v, err := req.db.MGet(args...); err != nil { + if v, err := c.db.MGet(args...); err != nil { return err } else { - req.resp.writeSliceArray(v) + c.resp.writeSliceArray(v) } return nil } -func expireCommand(req *requestContext) error { - args := req.args +func expireCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } @@ -214,17 +214,17 @@ func expireCommand(req *requestContext) error { return ErrValue } - if v, err := req.db.Expire(args[0], duration); err != nil { + if v, err := c.db.Expire(args[0], duration); err != nil { return err } else { - req.resp.writeInteger(v) + c.resp.writeInteger(v) } return nil } -func expireAtCommand(req *requestContext) error { - args := req.args +func expireAtCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } @@ -234,49 +234,45 @@ func expireAtCommand(req *requestContext) error { return ErrValue } - if v, err := req.db.ExpireAt(args[0], when); err != nil { + if v, err := c.db.ExpireAt(args[0], when); err != nil { return err } else { - req.resp.writeInteger(v) + c.resp.writeInteger(v) } return nil } -func ttlCommand(req *requestContext) error { - args := req.args +func ttlCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if v, err := req.db.TTL(args[0]); err != nil { + if v, err := c.db.TTL(args[0]); err != nil { return err } else { - req.resp.writeInteger(v) + c.resp.writeInteger(v) } return nil } -func persistCommand(req *requestContext) error { - args := req.args +func persistCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if n, err := req.db.Persist(args[0]); err != nil { + if n, err := c.db.Persist(args[0]); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -// func (db *DB) Expire(key []byte, duration int6 -// func (db *DB) ExpireAt(key []byte, when int64) -// func (db *DB) TTL(key []byte) (int64, error) - func init() { register("decr", decrCommand) register("decrby", decrbyCommand) diff --git a/server/cmd_list.go b/server/cmd_list.go index 7d26893..f893643 100644 --- a/server/cmd_list.go +++ b/server/cmd_list.go @@ -4,83 +4,83 @@ import ( "github.com/siddontang/ledisdb/ledis" ) -func lpushCommand(req *requestContext) error { - args := req.args +func lpushCommand(c *client) error { + args := c.args if len(args) < 2 { return ErrCmdParams } - if n, err := req.db.LPush(args[0], args[1:]...); err != nil { + if n, err := c.db.LPush(args[0], args[1:]...); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func rpushCommand(req *requestContext) error { - args := req.args +func rpushCommand(c *client) error { + args := c.args if len(args) < 2 { return ErrCmdParams } - if n, err := req.db.RPush(args[0], args[1:]...); err != nil { + if n, err := c.db.RPush(args[0], args[1:]...); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func lpopCommand(req *requestContext) error { - args := req.args +func lpopCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if v, err := req.db.LPop(args[0]); err != nil { + if v, err := c.db.LPop(args[0]); err != nil { return err } else { - req.resp.writeBulk(v) + c.resp.writeBulk(v) } return nil } -func rpopCommand(req *requestContext) error { - args := req.args +func rpopCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if v, err := req.db.RPop(args[0]); err != nil { + if v, err := c.db.RPop(args[0]); err != nil { return err } else { - req.resp.writeBulk(v) + c.resp.writeBulk(v) } return nil } -func llenCommand(req *requestContext) error { - args := req.args +func llenCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if n, err := req.db.LLen(args[0]); err != nil { + if n, err := c.db.LLen(args[0]); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func lindexCommand(req *requestContext) error { - args := req.args +func lindexCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } @@ -90,17 +90,17 @@ func lindexCommand(req *requestContext) error { return ErrValue } - if v, err := req.db.LIndex(args[0], int32(index)); err != nil { + if v, err := c.db.LIndex(args[0], int32(index)); err != nil { return err } else { - req.resp.writeBulk(v) + c.resp.writeBulk(v) } return nil } -func lrangeCommand(req *requestContext) error { - args := req.args +func lrangeCommand(c *client) error { + args := c.args if len(args) != 3 { return ErrCmdParams } @@ -119,47 +119,47 @@ func lrangeCommand(req *requestContext) error { return ErrValue } - if v, err := req.db.LRange(args[0], int32(start), int32(stop)); err != nil { + if v, err := c.db.LRange(args[0], int32(start), int32(stop)); err != nil { return err } else { - req.resp.writeSliceArray(v) + c.resp.writeSliceArray(v) } return nil } -func lclearCommand(req *requestContext) error { - args := req.args +func lclearCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if n, err := req.db.LClear(args[0]); err != nil { + if n, err := c.db.LClear(args[0]); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func lmclearCommand(req *requestContext) error { - args := req.args +func lmclearCommand(c *client) error { + args := c.args if len(args) < 1 { return ErrCmdParams } - if n, err := req.db.LMclear(args...); err != nil { + if n, err := c.db.LMclear(args...); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func lexpireCommand(req *requestContext) error { - args := req.args +func lexpireCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } @@ -169,17 +169,17 @@ func lexpireCommand(req *requestContext) error { return ErrValue } - if v, err := req.db.LExpire(args[0], duration); err != nil { + if v, err := c.db.LExpire(args[0], duration); err != nil { return err } else { - req.resp.writeInteger(v) + c.resp.writeInteger(v) } return nil } -func lexpireAtCommand(req *requestContext) error { - args := req.args +func lexpireAtCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } @@ -189,40 +189,40 @@ func lexpireAtCommand(req *requestContext) error { return ErrValue } - if v, err := req.db.LExpireAt(args[0], when); err != nil { + if v, err := c.db.LExpireAt(args[0], when); err != nil { return err } else { - req.resp.writeInteger(v) + c.resp.writeInteger(v) } return nil } -func lttlCommand(req *requestContext) error { - args := req.args +func lttlCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if v, err := req.db.LTTL(args[0]); err != nil { + if v, err := c.db.LTTL(args[0]); err != nil { return err } else { - req.resp.writeInteger(v) + c.resp.writeInteger(v) } return nil } -func lpersistCommand(req *requestContext) error { - args := req.args +func lpersistCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if n, err := req.db.LPersist(args[0]); err != nil { + if n, err := c.db.LPersist(args[0]); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil diff --git a/server/cmd_replication.go b/server/cmd_replication.go index 85c0861..fe84191 100644 --- a/server/cmd_replication.go +++ b/server/cmd_replication.go @@ -11,8 +11,8 @@ import ( "strings" ) -func slaveofCommand(req *requestContext) error { - args := req.args +func slaveofCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams @@ -31,23 +31,23 @@ func slaveofCommand(req *requestContext) error { masterAddr = fmt.Sprintf("%s:%s", args[0], args[1]) } - if err := req.app.slaveof(masterAddr); err != nil { + if err := c.app.slaveof(masterAddr); err != nil { return err } - req.resp.writeStatus(OK) + c.resp.writeStatus(OK) return nil } -func fullsyncCommand(req *requestContext) error { +func fullsyncCommand(c *client) error { //todo, multi fullsync may use same dump file - dumpFile, err := ioutil.TempFile(req.app.cfg.DataDir, "dump_") + dumpFile, err := ioutil.TempFile(c.app.cfg.DataDir, "dump_") if err != nil { return err } - if err = req.app.ldb.Dump(dumpFile); err != nil { + if err = c.app.ldb.Dump(dumpFile); err != nil { return err } @@ -56,7 +56,7 @@ func fullsyncCommand(req *requestContext) error { dumpFile.Seek(0, os.SEEK_SET) - req.resp.writeBulkFrom(n, dumpFile) + c.resp.writeBulkFrom(n, dumpFile) name := dumpFile.Name() dumpFile.Close() @@ -68,8 +68,8 @@ func fullsyncCommand(req *requestContext) error { var reserveInfoSpace = make([]byte, 16) -func syncCommand(req *requestContext) error { - args := req.args +func syncCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } @@ -87,32 +87,32 @@ func syncCommand(req *requestContext) error { return ErrCmdParams } - req.syncBuf.Reset() + c.syncBuf.Reset() //reserve space to write master info - if _, err := req.syncBuf.Write(reserveInfoSpace); err != nil { + if _, err := c.syncBuf.Write(reserveInfoSpace); err != nil { return err } m := &ledis.MasterInfo{logIndex, logPos} - if _, err := req.app.ldb.ReadEventsTo(m, &req.syncBuf); err != nil { + if _, err := c.app.ldb.ReadEventsTo(m, &c.syncBuf); err != nil { return err } else { - buf := req.syncBuf.Bytes() + buf := c.syncBuf.Bytes() binary.BigEndian.PutUint64(buf[0:], uint64(m.LogFileIndex)) binary.BigEndian.PutUint64(buf[8:], uint64(m.LogPos)) - if len(req.compressBuf) < snappy.MaxEncodedLen(len(buf)) { - req.compressBuf = make([]byte, snappy.MaxEncodedLen(len(buf))) + if len(c.compressBuf) < snappy.MaxEncodedLen(len(buf)) { + c.compressBuf = make([]byte, snappy.MaxEncodedLen(len(buf))) } - if buf, err = snappy.Encode(req.compressBuf, buf); err != nil { + if buf, err = snappy.Encode(c.compressBuf, buf); err != nil { return err } - req.resp.writeBulk(buf) + c.resp.writeBulk(buf) } return nil diff --git a/server/cmd_set.go b/server/cmd_set.go index 815c3a8..335d8cc 100644 --- a/server/cmd_set.go +++ b/server/cmd_set.go @@ -4,23 +4,23 @@ import ( "github.com/siddontang/ledisdb/ledis" ) -func saddCommand(req *requestContext) error { - args := req.args +func saddCommand(c *client) error { + args := c.args if len(args) < 2 { return ErrCmdParams } - if n, err := req.db.SAdd(args[0], args[1:]...); err != nil { + if n, err := c.db.SAdd(args[0], args[1:]...); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func soptGeneric(req *requestContext, optType byte) error { - args := req.args +func soptGeneric(c *client, optType byte) error { + args := c.args if len(args) < 1 { return ErrCmdParams } @@ -30,25 +30,25 @@ func soptGeneric(req *requestContext, optType byte) error { switch optType { case ledis.UnionType: - v, err = req.db.SUnion(args...) + v, err = c.db.SUnion(args...) case ledis.DiffType: - v, err = req.db.SDiff(args...) + v, err = c.db.SDiff(args...) case ledis.InterType: - v, err = req.db.SInter(args...) + v, err = c.db.SInter(args...) } if err != nil { return err } else { - req.resp.writeSliceArray(v) + c.resp.writeSliceArray(v) } return nil } -func soptStoreGeneric(req *requestContext, optType byte) error { - args := req.args +func soptStoreGeneric(c *client, optType byte) error { + args := c.args if len(args) < 2 { return ErrCmdParams } @@ -58,141 +58,141 @@ func soptStoreGeneric(req *requestContext, optType byte) error { switch optType { case ledis.UnionType: - n, err = req.db.SUnionStore(args[0], args[1:]...) + n, err = c.db.SUnionStore(args[0], args[1:]...) case ledis.DiffType: - n, err = req.db.SDiffStore(args[0], args[1:]...) + n, err = c.db.SDiffStore(args[0], args[1:]...) case ledis.InterType: - n, err = req.db.SInterStore(args[0], args[1:]...) + n, err = c.db.SInterStore(args[0], args[1:]...) } if err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func scardCommand(req *requestContext) error { - args := req.args +func scardCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if n, err := req.db.SCard(args[0]); err != nil { + if n, err := c.db.SCard(args[0]); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func sdiffCommand(req *requestContext) error { - return soptGeneric(req, ledis.DiffType) +func sdiffCommand(c *client) error { + return soptGeneric(c, ledis.DiffType) } -func sdiffstoreCommand(req *requestContext) error { - return soptStoreGeneric(req, ledis.DiffType) +func sdiffstoreCommand(c *client) error { + return soptStoreGeneric(c, ledis.DiffType) } -func sinterCommand(req *requestContext) error { - return soptGeneric(req, ledis.InterType) +func sinterCommand(c *client) error { + return soptGeneric(c, ledis.InterType) } -func sinterstoreCommand(req *requestContext) error { - return soptStoreGeneric(req, ledis.InterType) +func sinterstoreCommand(c *client) error { + return soptStoreGeneric(c, ledis.InterType) } -func sismemberCommand(req *requestContext) error { - args := req.args +func sismemberCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } - if n, err := req.db.SIsMember(args[0], args[1]); err != nil { + if n, err := c.db.SIsMember(args[0], args[1]); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func smembersCommand(req *requestContext) error { - args := req.args +func smembersCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if v, err := req.db.SMembers(args[0]); err != nil { + if v, err := c.db.SMembers(args[0]); err != nil { return err } else { - req.resp.writeSliceArray(v) + c.resp.writeSliceArray(v) } return nil } -func sremCommand(req *requestContext) error { - args := req.args +func sremCommand(c *client) error { + args := c.args if len(args) < 2 { return ErrCmdParams } - if n, err := req.db.SRem(args[0], args[1:]...); err != nil { + if n, err := c.db.SRem(args[0], args[1:]...); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func sunionCommand(req *requestContext) error { - return soptGeneric(req, ledis.UnionType) +func sunionCommand(c *client) error { + return soptGeneric(c, ledis.UnionType) } -func sunionstoreCommand(req *requestContext) error { - return soptStoreGeneric(req, ledis.UnionType) +func sunionstoreCommand(c *client) error { + return soptStoreGeneric(c, ledis.UnionType) } -func sclearCommand(req *requestContext) error { - args := req.args +func sclearCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if n, err := req.db.SClear(args[0]); err != nil { + if n, err := c.db.SClear(args[0]); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func smclearCommand(req *requestContext) error { - args := req.args +func smclearCommand(c *client) error { + args := c.args if len(args) < 1 { return ErrCmdParams } - if n, err := req.db.SMclear(args...); err != nil { + if n, err := c.db.SMclear(args...); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func sexpireCommand(req *requestContext) error { - args := req.args +func sexpireCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } @@ -202,17 +202,17 @@ func sexpireCommand(req *requestContext) error { return ErrValue } - if v, err := req.db.SExpire(args[0], duration); err != nil { + if v, err := c.db.SExpire(args[0], duration); err != nil { return err } else { - req.resp.writeInteger(v) + c.resp.writeInteger(v) } return nil } -func sexpireAtCommand(req *requestContext) error { - args := req.args +func sexpireAtCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } @@ -222,41 +222,41 @@ func sexpireAtCommand(req *requestContext) error { return ErrValue } - if v, err := req.db.SExpireAt(args[0], when); err != nil { + if v, err := c.db.SExpireAt(args[0], when); err != nil { return err } else { - req.resp.writeInteger(v) + c.resp.writeInteger(v) } return nil } -func sttlCommand(req *requestContext) error { - args := req.args +func sttlCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if v, err := req.db.STTL(args[0]); err != nil { + if v, err := c.db.STTL(args[0]); err != nil { return err } else { - req.resp.writeInteger(v) + c.resp.writeInteger(v) } return nil } -func spersistCommand(req *requestContext) error { - args := req.args +func spersistCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if n, err := req.db.SPersist(args[0]); err != nil { + if n, err := c.db.SPersist(args[0]); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil diff --git a/server/cmd_tx.go b/server/cmd_tx.go new file mode 100644 index 0000000..19eb5c1 --- /dev/null +++ b/server/cmd_tx.go @@ -0,0 +1,57 @@ +package server + +import ( + "errors" +) + +var errTxMiss = errors.New("transaction miss") + +func beginCommand(c *client) error { + tx, err := c.db.Begin() + if err == nil { + c.tx = tx + c.db = tx.DB + c.resp.writeStatus(OK) + } + + return err +} + +func commitCommand(c *client) error { + if c.tx == nil { + return errTxMiss + } + + err := c.tx.Commit() + c.db, _ = c.ldb.Select(c.tx.Index()) + c.tx = nil + + if err == nil { + c.resp.writeStatus(OK) + } + + return err +} + +func rollbackCommand(c *client) error { + if c.tx == nil { + return errTxMiss + } + + err := c.tx.Rollback() + + c.db, _ = c.ldb.Select(c.tx.Index()) + c.tx = nil + + if err == nil { + c.resp.writeStatus(OK) + } + + return err +} + +func init() { + register("begin", beginCommand) + register("commit", commitCommand) + register("rollback", rollbackCommand) +} diff --git a/server/cmd_zset.go b/server/cmd_zset.go index f8117fc..2964af5 100644 --- a/server/cmd_zset.go +++ b/server/cmd_zset.go @@ -12,20 +12,20 @@ import ( var errScoreOverflow = errors.New("zset score overflow") -func zaddCommand(req *requestContext) error { - args := req.args +func zaddCommand(c *client) error { + args := c.args if len(args) < 3 { return ErrCmdParams } key := args[0] - if len(args[1:])%2 != 0 { + if len(args[1:])&1 != 0 { return ErrCmdParams } args = args[1:] - params := make([]ledis.ScorePair, len(args)/2) + params := make([]ledis.ScorePair, len(args)>>1) for i := 0; i < len(params); i++ { score, err := ledis.StrInt64(args[2*i], nil) if err != nil { @@ -36,66 +36,66 @@ func zaddCommand(req *requestContext) error { params[i].Member = args[2*i+1] } - if n, err := req.db.ZAdd(key, params...); err != nil { - return err - } else { - req.resp.writeInteger(n) + n, err := c.db.ZAdd(key, params...) + + if err == nil { + c.resp.writeInteger(n) } - return nil + return err } -func zcardCommand(req *requestContext) error { - args := req.args +func zcardCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if n, err := req.db.ZCard(args[0]); err != nil { + if n, err := c.db.ZCard(args[0]); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func zscoreCommand(req *requestContext) error { - args := req.args +func zscoreCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } - if s, err := req.db.ZScore(args[0], args[1]); err != nil { + if s, err := c.db.ZScore(args[0], args[1]); err != nil { if err == ledis.ErrScoreMiss { - req.resp.writeBulk(nil) + c.resp.writeBulk(nil) } else { return err } } else { - req.resp.writeBulk(ledis.StrPutInt64(s)) + c.resp.writeBulk(ledis.StrPutInt64(s)) } return nil } -func zremCommand(req *requestContext) error { - args := req.args +func zremCommand(c *client) error { + args := c.args if len(args) < 2 { return ErrCmdParams } - if n, err := req.db.ZRem(args[0], args[1:]...); err != nil { - return err - } else { - req.resp.writeInteger(n) + n, err := c.db.ZRem(args[0], args[1:]...) + + if err == nil { + c.resp.writeInteger(n) } - return nil + return err } -func zincrbyCommand(req *requestContext) error { - args := req.args +func zincrbyCommand(c *client) error { + args := c.args if len(args) != 3 { return ErrCmdParams } @@ -107,13 +107,13 @@ func zincrbyCommand(req *requestContext) error { return ErrValue } - if v, err := req.db.ZIncrBy(key, delta, args[2]); err != nil { - return err - } else { - req.resp.writeBulk(ledis.StrPutInt64(v)) + v, err := c.db.ZIncrBy(key, delta, args[2]) + + if err == nil { + c.resp.writeBulk(ledis.StrPutInt64(v)) } - return nil + return err } func zparseScoreRange(minBuf []byte, maxBuf []byte) (min int64, max int64, err error) { @@ -186,8 +186,8 @@ func zparseScoreRange(minBuf []byte, maxBuf []byte) (min int64, max int64, err e return } -func zcountCommand(req *requestContext) error { - args := req.args +func zcountCommand(c *client) error { + args := c.args if len(args) != 3 { return ErrCmdParams } @@ -198,77 +198,77 @@ func zcountCommand(req *requestContext) error { } if min > max { - req.resp.writeInteger(0) + c.resp.writeInteger(0) return nil } - if n, err := req.db.ZCount(args[0], min, max); err != nil { + if n, err := c.db.ZCount(args[0], min, max); err != nil { return err } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func zrankCommand(req *requestContext) error { - args := req.args +func zrankCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } - if n, err := req.db.ZRank(args[0], args[1]); err != nil { + if n, err := c.db.ZRank(args[0], args[1]); err != nil { return err } else if n == -1 { - req.resp.writeBulk(nil) + c.resp.writeBulk(nil) } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func zrevrankCommand(req *requestContext) error { - args := req.args +func zrevrankCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } - if n, err := req.db.ZRevRank(args[0], args[1]); err != nil { + if n, err := c.db.ZRevRank(args[0], args[1]); err != nil { return err } else if n == -1 { - req.resp.writeBulk(nil) + c.resp.writeBulk(nil) } else { - req.resp.writeInteger(n) + c.resp.writeInteger(n) } return nil } -func zremrangebyrankCommand(req *requestContext) error { - args := req.args +func zremrangebyrankCommand(c *client) error { + args := c.args if len(args) != 3 { return ErrCmdParams } key := args[0] - start, stop, err := zparseRange(req, args[1], args[2]) + start, stop, err := zparseRange(c, args[1], args[2]) if err != nil { return ErrValue } - if n, err := req.db.ZRemRangeByRank(key, start, stop); err != nil { - return err - } else { - req.resp.writeInteger(n) + n, err := c.db.ZRemRangeByRank(key, start, stop) + + if err == nil { + c.resp.writeInteger(n) } - return nil + return err } -func zremrangebyscoreCommand(req *requestContext) error { - args := req.args +func zremrangebyscoreCommand(c *client) error { + args := c.args if len(args) != 3 { return ErrCmdParams } @@ -279,16 +279,16 @@ func zremrangebyscoreCommand(req *requestContext) error { return err } - if n, err := req.db.ZRemRangeByScore(key, min, max); err != nil { - return err - } else { - req.resp.writeInteger(n) + n, err := c.db.ZRemRangeByScore(key, min, max) + + if err == nil { + c.resp.writeInteger(n) } - return nil + return err } -func zparseRange(req *requestContext, a1 []byte, a2 []byte) (start int, stop int, err error) { +func zparseRange(c *client, a1 []byte, a2 []byte) (start int, stop int, err error) { if start, err = strconv.Atoi(ledis.String(a1)); err != nil { return } @@ -300,15 +300,15 @@ func zparseRange(req *requestContext, a1 []byte, a2 []byte) (start int, stop int return } -func zrangeGeneric(req *requestContext, reverse bool) error { - args := req.args +func zrangeGeneric(c *client, reverse bool) error { + args := c.args if len(args) < 3 { return ErrCmdParams } key := args[0] - start, stop, err := zparseRange(req, args[1], args[2]) + start, stop, err := zparseRange(c, args[1], args[2]) if err != nil { return ErrValue } @@ -327,24 +327,24 @@ func zrangeGeneric(req *requestContext, reverse bool) error { } } - if datas, err := req.db.ZRangeGeneric(key, start, stop, reverse); err != nil { + if datas, err := c.db.ZRangeGeneric(key, start, stop, reverse); err != nil { return err } else { - req.resp.writeScorePairArray(datas, withScores) + c.resp.writeScorePairArray(datas, withScores) } return nil } -func zrangeCommand(req *requestContext) error { - return zrangeGeneric(req, false) +func zrangeCommand(c *client) error { + return zrangeGeneric(c, false) } -func zrevrangeCommand(req *requestContext) error { - return zrangeGeneric(req, true) +func zrevrangeCommand(c *client) error { + return zrangeGeneric(c, true) } -func zrangebyscoreGeneric(req *requestContext, reverse bool) error { - args := req.args +func zrangebyscoreGeneric(c *client, reverse bool) error { + args := c.args if len(args) < 3 { return ErrCmdParams } @@ -400,59 +400,59 @@ func zrangebyscoreGeneric(req *requestContext, reverse bool) error { if offset < 0 { //for ledis, if offset < 0, a empty will return //so here we directly return a empty array - req.resp.writeArray([]interface{}{}) + c.resp.writeArray([]interface{}{}) return nil } - if datas, err := req.db.ZRangeByScoreGeneric(key, min, max, offset, count, reverse); err != nil { + if datas, err := c.db.ZRangeByScoreGeneric(key, min, max, offset, count, reverse); err != nil { return err } else { - req.resp.writeScorePairArray(datas, withScores) + c.resp.writeScorePairArray(datas, withScores) } return nil } -func zrangebyscoreCommand(req *requestContext) error { - return zrangebyscoreGeneric(req, false) +func zrangebyscoreCommand(c *client) error { + return zrangebyscoreGeneric(c, false) } -func zrevrangebyscoreCommand(req *requestContext) error { - return zrangebyscoreGeneric(req, true) +func zrevrangebyscoreCommand(c *client) error { + return zrangebyscoreGeneric(c, true) } -func zclearCommand(req *requestContext) error { - args := req.args +func zclearCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if n, err := req.db.ZClear(args[0]); err != nil { - return err - } else { - req.resp.writeInteger(n) + n, err := c.db.ZClear(args[0]) + + if err == nil { + c.resp.writeInteger(n) } - return nil + return err } -func zmclearCommand(req *requestContext) error { - args := req.args +func zmclearCommand(c *client) error { + args := c.args if len(args) < 1 { return ErrCmdParams } - if n, err := req.db.ZMclear(args...); err != nil { - return err - } else { - req.resp.writeInteger(n) + n, err := c.db.ZMclear(args...) + + if err == nil { + c.resp.writeInteger(n) } - return nil + return err } -func zexpireCommand(req *requestContext) error { - args := req.args +func zexpireCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } @@ -462,17 +462,17 @@ func zexpireCommand(req *requestContext) error { return ErrValue } - if v, err := req.db.ZExpire(args[0], duration); err != nil { - return err - } else { - req.resp.writeInteger(v) + v, err := c.db.ZExpire(args[0], duration) + + if err == nil { + c.resp.writeInteger(v) } - return nil + return err } -func zexpireAtCommand(req *requestContext) error { - args := req.args +func zexpireAtCommand(c *client) error { + args := c.args if len(args) != 2 { return ErrCmdParams } @@ -482,42 +482,43 @@ func zexpireAtCommand(req *requestContext) error { return ErrValue } - if v, err := req.db.ZExpireAt(args[0], when); err != nil { - return err - } else { - req.resp.writeInteger(v) + v, err := c.db.ZExpireAt(args[0], when) + + if err == nil { + c.resp.writeInteger(v) } - return nil + + return err } -func zttlCommand(req *requestContext) error { - args := req.args +func zttlCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if v, err := req.db.ZTTL(args[0]); err != nil { + if v, err := c.db.ZTTL(args[0]); err != nil { return err } else { - req.resp.writeInteger(v) + c.resp.writeInteger(v) } return nil } -func zpersistCommand(req *requestContext) error { - args := req.args +func zpersistCommand(c *client) error { + args := c.args if len(args) != 1 { return ErrCmdParams } - if n, err := req.db.ZPersist(args[0]); err != nil { - return err - } else { - req.resp.writeInteger(n) + n, err := c.db.ZPersist(args[0]) + + if err == nil { + c.resp.writeInteger(n) } - return nil + return err } func zparseZsetoptStore(args [][]byte) (destKey []byte, srcKeys [][]byte, weights []int64, aggregate byte, err error) { @@ -597,8 +598,8 @@ func zparseZsetoptStore(args [][]byte) (destKey []byte, srcKeys [][]byte, weight return } -func zunionstoreCommand(req *requestContext) error { - args := req.args +func zunionstoreCommand(c *client) error { + args := c.args if len(args) < 2 { return ErrCmdParams } @@ -607,17 +608,18 @@ func zunionstoreCommand(req *requestContext) error { if err != nil { return err } - if n, err := req.db.ZUnionStore(destKey, srcKeys, weights, aggregate); err != nil { - return err - } else { - req.resp.writeInteger(n) + + n, err := c.db.ZUnionStore(destKey, srcKeys, weights, aggregate) + + if err == nil { + c.resp.writeInteger(n) } - return nil + return err } -func zinterstoreCommand(req *requestContext) error { - args := req.args +func zinterstoreCommand(c *client) error { + args := c.args if len(args) < 2 { return ErrCmdParams } @@ -626,12 +628,14 @@ func zinterstoreCommand(req *requestContext) error { if err != nil { return err } - if n, err := req.db.ZInterStore(destKey, srcKeys, weights, aggregate); err != nil { - return err - } else { - req.resp.writeInteger(n) + + n, err := c.db.ZInterStore(destKey, srcKeys, weights, aggregate) + + if err == nil { + c.resp.writeInteger(n) } - return nil + + return err } func init() { diff --git a/server/command.go b/server/command.go index 440a177..2bf86a5 100644 --- a/server/command.go +++ b/server/command.go @@ -4,11 +4,10 @@ import ( "fmt" "github.com/siddontang/ledisdb/ledis" "strconv" - "strings" ) -type CommandFunc func(req *requestContext) error +type CommandFunc func(c *client) error var regCmds = map[string]CommandFunc{} @@ -20,40 +19,58 @@ func register(name string, f CommandFunc) { regCmds[name] = f } -func pingCommand(req *requestContext) error { - req.resp.writeStatus(PONG) +func pingCommand(c *client) error { + c.resp.writeStatus(PONG) return nil } -func echoCommand(req *requestContext) error { - if len(req.args) != 1 { +func echoCommand(c *client) error { + if len(c.args) != 1 { return ErrCmdParams } - req.resp.writeBulk(req.args[0]) + c.resp.writeBulk(c.args[0]) return nil } -func selectCommand(req *requestContext) error { - if len(req.args) != 1 { +func selectCommand(c *client) error { + if len(c.args) != 1 { return ErrCmdParams } - if index, err := strconv.Atoi(ledis.String(req.args[0])); err != nil { + if index, err := strconv.Atoi(ledis.String(c.args[0])); err != nil { return err } else { - if db, err := req.ldb.Select(index); err != nil { + if db, err := c.ldb.Select(index); err != nil { return err } else { - req.db = db - req.resp.writeStatus(OK) + c.db = db + c.resp.writeStatus(OK) } } + + return nil +} + +func infoCommand(c *client) error { + if len(c.args) > 1 { + return ErrSyntax + } + var section string + if len(c.args) == 1 { + section = strings.ToLower(ledis.String(c.args[0])) + } + + buf := c.app.info.Dump(section) + c.resp.writeBulk(buf) + return nil } func init() { + register("ping", pingCommand) register("echo", echoCommand) register("select", selectCommand) + register("info", infoCommand) } diff --git a/server/command_cnf.go b/server/command_cnf.go deleted file mode 100644 index e830592..0000000 --- a/server/command_cnf.go +++ /dev/null @@ -1,510 +0,0 @@ -//This file was generated by ./generate.py on Mon Aug 11 2014 12:35:56 +0800 -package server - -type cmdConf struct { - name string - argDesc string - group string - readonly bool -} - -var cnfCmds = []cmdConf{ - { - "ZRANGEBYSCORE", - "key min max [WITHSCORES] [LIMIT offset count]", - "ZSet", - true, - }, - { - "ZPERSIST", - "key", - "ZSet", - false, - }, - { - "LTTL", - "key", - "List", - true, - }, - { - "LINDEX", - "key index", - "List", - true, - }, - { - "FULLSYNC", - "-", - "Replication", - false, - }, - { - "ZREVRANK", - "key member", - "ZSet", - true, - }, - { - "ZEXPIRE", - "key seconds", - "ZSet", - false, - }, - { - "SYNC", - "index offset", - "Replication", - false, - }, - { - "BMSETBIT", - "key offset value [offset value ...]", - "Bitmap", - false, - }, - { - "LPOP", - "key", - "List", - false, - }, - { - "HPERSIST", - "key", - "Hash", - false, - }, - { - "EXPIRE", - "key seconds", - "KV", - false, - }, - { - "DEL", - "key [key ...]", - "KV", - false, - }, - { - "LPUSH", - "key value [value ...]", - "List", - false, - }, - { - "PERSIST", - "key", - "KV", - false, - }, - { - "HTTL", - "key", - "Hash", - true, - }, - { - "LEXPIREAT", - "key timestamp", - "List", - false, - }, - { - "ZEXPIREAT", - "key timestamp", - "ZSet", - false, - }, - { - "DECR", - "key", - "KV", - false, - }, - { - "SLAVEOF", - "host port", - "Replication", - false, - }, - { - "INCR", - "key", - "KV", - false, - }, - { - "MSET", - "key value [key value ...]", - "KV", - false, - }, - { - "LEXPIRE", - "key seconds", - "List", - false, - }, - { - "HINCRBY", - "key field increment", - "Hash", - false, - }, - { - "GET", - "key", - "KV", - true, - }, - { - "ZREVRANGE", - "key start stop [WITHSCORES]", - "ZSet", - true, - }, - { - "ZINCRBY", - "key increment member", - "ZSet", - false, - }, - { - "LPERSIST", - "key", - "List", - false, - }, - { - "HEXISTS", - "key field", - "Hash", - true, - }, - { - "ZREM", - "key member [member ...]", - "ZSet", - false, - }, - { - "BOPT", - "operation destkey key [key ...]", - "Bitmap", - false, - }, - { - "ZCLEAR", - "key", - "ZSet", - false, - }, - { - "LCLEAR", - "key", - "List", - false, - }, - { - "ZRANK", - "key member", - "ZSet", - true, - }, - { - "TTL", - "key", - "KV", - true, - }, - { - "ZADD", - "key score member [score member ...]", - "ZSet", - false, - }, - { - "HEXPIRE", - "key seconds", - "Hash", - false, - }, - { - "HDEL", - "key field [field ...]", - "Hash", - false, - }, - { - "HSET", - "key field value", - "Hash", - false, - }, - { - "LLEN", - "key", - "List", - true, - }, - { - "HVALS", - "key", - "Hash", - true, - }, - { - "BCOUNT", - "key [start end]", - "Bitmap", - true, - }, - { - "BGET", - "key", - "Bitmap", - true, - }, - { - "MGET", - "key [key ...]", - "KV", - true, - }, - { - "EXISTS", - "key", - "KV", - true, - }, - { - "HMCLEAR", - "key [key ...]", - "Hash", - false, - }, - { - "ZCOUNT", - "key min max", - "ZSet", - true, - }, - { - "SELECT", - "index", - "Server", - false, - }, - { - "ECHO", - "message", - "Server", - true, - }, - { - "ZTTL", - "key", - "ZSet", - true, - }, - { - "HKEYS", - "key", - "Hash", - true, - }, - { - "HGETALL", - "key", - "Hash", - true, - }, - { - "RPOP", - "key", - "List", - false, - }, - { - "HMGET", - "key field [field ...]", - "Hash", - true, - }, - { - "SETNX", - "key value", - "KV", - false, - }, - { - "HGET", - "key field", - "Hash", - true, - }, - { - "BPERSIST", - "key", - "Bitmap", - false, - }, - { - "INCRBY", - "key increment", - "KV", - false, - }, - { - "BDELETE", - "key", - "ZSet", - false, - }, - { - "ZMCLEAR", - "key [key ...]", - "ZSet", - false, - }, - { - "RPUSH", - "key value [value ...]", - "List", - false, - }, - { - "LRANGE", - "key start stop", - "List", - true, - }, - { - "HLEN", - "key", - "Hash", - true, - }, - { - "ZSCORE", - "key member", - "ZSet", - true, - }, - { - "LMCLEAR", - "key [key ...]", - "List", - false, - }, - { - "EXPIREAT", - "key timestamp", - "KV", - false, - }, - { - "ZREMRANGEBYSCORE", - "key min max", - "ZSet", - false, - }, - { - "ZCARD", - "key", - "ZSet", - true, - }, - { - "ZREMRANGEBYRANK", - "key start stop", - "ZSet", - false, - }, - { - "PING", - "-", - "Server", - true, - }, - { - "HMSET", - "key field value [field value ...]", - "Hash", - false, - }, - { - "BTTL", - "key", - "Bitmap", - true, - }, - { - "HCLEAR", - "key", - "Hash", - false, - }, - { - "ZRANGE", - "key start stop [WITHSCORES]", - "ZSet", - false, - }, - { - "ZREVRANGEBYSCORE", - "key max min [WITHSCORES][LIMIT offset count]", - "ZSet", - true, - }, - { - "BSETBIT", - "key offset value", - "Bitmap", - false, - }, - { - "BEXPIREAT", - "key timestamp", - "Bitmap", - false, - }, - { - "SET", - "key value", - "KV", - false, - }, - { - "BGETBIT", - "key offset", - "Bitmap", - true, - }, - { - "BEXPIRE", - "key seconds", - "Bitmap", - false, - }, - { - "GETSET", - " key value", - "KV", - false, - }, - { - "DECRBY", - "key decrement", - "KV", - false, - }, - { - "HEXPIREAT", - "key timestamp", - "Hash", - false, - }, -} diff --git a/server/const.go b/server/const.go index cfb7595..f1123e3 100644 --- a/server/const.go +++ b/server/const.go @@ -23,3 +23,18 @@ var ( PONG = "PONG" OK = "OK" ) + +const ( + KV = iota + LIST + HASH + SET + ZSET + BIT +) + +const ( + GB uint64 = 1024 * 1024 * 1024 + MB uint64 = 1024 * 1024 + KB uint64 = 1024 +) diff --git a/server/info.go b/server/info.go new file mode 100644 index 0000000..fb6c099 --- /dev/null +++ b/server/info.go @@ -0,0 +1,166 @@ +package server + +import ( + "bytes" + "fmt" + "github.com/siddontang/ledisdb/config" + "os" + "runtime" + "strings" + "sync" + "sync/atomic" + "syscall" +) + +type info struct { + sync.Mutex + + app *App + + Server struct { + OS string + ProceessId int + } + + Clients struct { + ConnectedClients int64 + } + + Persistence struct { + DBName string + } +} + +func newInfo(app *App) (i *info, err error) { + i = new(info) + + i.app = app + + i.Server.OS = runtime.GOOS + i.Server.ProceessId = os.Getpid() + + if app.cfg.DBName != "" { + i.Persistence.DBName = app.cfg.DBName + } else { + i.Persistence.DBName = config.DefaultDBName + } + + return i, nil +} + +func (i *info) addClients(delta int64) { + atomic.AddInt64(&i.Clients.ConnectedClients, delta) +} +func (i *info) Close() { + +} + +func getMemoryHuman(m uint64) string { + if m > GB { + return fmt.Sprintf("%dG", m/GB) + } else if m > MB { + return fmt.Sprintf("%dM", m/MB) + } else if m > KB { + return fmt.Sprintf("%dK", m/KB) + } else { + return fmt.Sprintf("%d", m) + } +} + +func (i *info) Dump(section string) []byte { + buf := &bytes.Buffer{} + switch strings.ToLower(section) { + case "": + i.dumpAll(buf) + case "server": + i.dumpServer(buf) + case "client": + i.dumpClients(buf) + case "cpu": + i.dumpCPU(buf) + case "mem": + i.dumpMem(buf) + case "persistence": + i.dumpPersistence(buf) + case "goroutine": + i.dumpGoroutine(buf) + default: + buf.WriteString(fmt.Sprintf("# %s\r\n", section)) + } + + return buf.Bytes() +} + +type infoPair struct { + Key string + Value interface{} +} + +func (i *info) dumpAll(buf *bytes.Buffer) { + i.dumpServer(buf) + buf.Write(Delims) + i.dumpPersistence(buf) + buf.Write(Delims) + i.dumpClients(buf) + buf.Write(Delims) + i.dumpCPU(buf) + buf.Write(Delims) + i.dumpMem(buf) + buf.Write(Delims) + i.dumpGoroutine(buf) +} + +func (i *info) dumpServer(buf *bytes.Buffer) { + buf.WriteString("# Server\r\n") + + i.dumpPairs(buf, infoPair{"os", i.Server.OS}, + infoPair{"process_id", i.Server.ProceessId}, + infoPair{"addr", i.app.cfg.Addr}, + infoPair{"http_addr", i.app.cfg.HttpAddr}) +} + +func (i *info) dumpClients(buf *bytes.Buffer) { + buf.WriteString("# Client\r\n") + + i.dumpPairs(buf, infoPair{"client_num", i.Clients.ConnectedClients}) +} + +func (i *info) dumpCPU(buf *bytes.Buffer) { + buf.WriteString("# CPU\r\n") + + var rusage syscall.Rusage + if err := syscall.Getrusage(syscall.RUSAGE_SELF, &rusage); err != nil { + return + } + + i.dumpPairs(buf, infoPair{"cpu_sys", rusage.Stime.Usec}, + infoPair{"cpu_user", rusage.Utime.Usec}) +} + +func (i *info) dumpMem(buf *bytes.Buffer) { + buf.WriteString("# Mem\r\n") + + var mem runtime.MemStats + runtime.ReadMemStats(&mem) + + i.dumpPairs(buf, infoPair{"mem_alloc", mem.Alloc}, + infoPair{"mem_alloc_human", getMemoryHuman(mem.Alloc)}) +} + +func (i *info) dumpGoroutine(buf *bytes.Buffer) { + buf.WriteString("# Goroutine\r\n") + + i.dumpPairs(buf, infoPair{"goroutine_num", runtime.NumGoroutine()}) +} + +func (i *info) dumpPersistence(buf *bytes.Buffer) { + buf.WriteString("# Persistence\r\n") + + i.dumpPairs(buf, infoPair{"db_name", i.Persistence.DBName}) +} + +func (i *info) dumpPairs(buf *bytes.Buffer, pairs ...infoPair) { + for _, v := range pairs { + buf.WriteString(fmt.Sprintf("%s:%v\r\n", v.Key, v.Value)) + } +} diff --git a/server/request.go b/server/request.go deleted file mode 100644 index 0531a37..0000000 --- a/server/request.go +++ /dev/null @@ -1,116 +0,0 @@ -package server - -import ( - "bytes" - "github.com/siddontang/ledisdb/ledis" - "io" - "time" -) - -type responseWriter interface { - writeError(error) - writeStatus(string) - writeInteger(int64) - writeBulk([]byte) - writeArray([]interface{}) - writeSliceArray([][]byte) - writeFVPairArray([]ledis.FVPair) - writeScorePairArray([]ledis.ScorePair, bool) - writeBulkFrom(int64, io.Reader) - flush() -} - -type requestContext struct { - app *App - ldb *ledis.Ledis - db *ledis.DB - - remoteAddr string - cmd string - args [][]byte - - resp responseWriter - - syncBuf bytes.Buffer - compressBuf []byte - - reqErr chan error - - buf bytes.Buffer -} - -func newRequestContext(app *App) *requestContext { - req := new(requestContext) - - req.app = app - req.ldb = app.ldb - req.db, _ = app.ldb.Select(0) //use default db - - req.compressBuf = make([]byte, 256) - req.reqErr = make(chan error) - - return req -} - -func (req *requestContext) perform() { - var err error - - start := time.Now() - - if len(req.cmd) == 0 { - err = ErrEmptyCommand - } else if exeCmd, ok := regCmds[req.cmd]; !ok { - err = ErrNotFound - } else { - go func() { - req.reqErr <- exeCmd(req) - }() - - err = <-req.reqErr - } - - duration := time.Since(start) - - if req.app.access != nil { - fullCmd := req.catGenericCommand() - cost := duration.Nanoseconds() / 1000000 - - truncateLen := len(fullCmd) - if truncateLen > 256 { - truncateLen = 256 - } - - req.app.access.Log(req.remoteAddr, cost, fullCmd[:truncateLen], err) - } - - if err != nil { - req.resp.writeError(err) - } - req.resp.flush() - return -} - -// func (h *requestHandler) catFullCommand(req *requestContext) []byte { -// -// // if strings.HasSuffix(cmd, "expire") { -// // catExpireCommand(c, buffer) -// // } else { -// // catGenericCommand(c, buffer) -// // } -// -// return h.catGenericCommand(req) -// } - -func (req *requestContext) catGenericCommand() []byte { - buffer := req.buf - buffer.Reset() - - buffer.Write([]byte(req.cmd)) - - for _, arg := range req.args { - buffer.WriteByte(' ') - buffer.Write(arg) - } - - return buffer.Bytes() -} diff --git a/store/boltdb/db.go b/store/boltdb/db.go index 6e126b6..8212d08 100644 --- a/store/boltdb/db.go +++ b/store/boltdb/db.go @@ -71,14 +71,11 @@ func (db *DB) Get(key []byte) ([]byte, error) { if err != nil { return nil, err } + defer t.Rollback() + b := t.Bucket(bucketName) value = b.Get(key) - err = t.Rollback() - - if err != nil { - return nil, err - } if value == nil { return nil, nil @@ -132,6 +129,10 @@ func (db *DB) Begin() (driver.Tx, error) { }, nil } +func (db *DB) NewSnapshot() (driver.ISnapshot, error) { + return newSnapshot(db) +} + func (db *DB) BatchPut(writes []driver.Write) error { err := db.db.Update(func(tx *bolt.Tx) error { b := tx.Bucket(bucketName) diff --git a/store/boltdb/snapshot.go b/store/boltdb/snapshot.go new file mode 100644 index 0000000..ad4df90 --- /dev/null +++ b/store/boltdb/snapshot.go @@ -0,0 +1,37 @@ +package boltdb + +import ( + "github.com/boltdb/bolt" + "github.com/siddontang/ledisdb/store/driver" +) + +type Snapshot struct { + tx *bolt.Tx + b *bolt.Bucket +} + +func newSnapshot(db *DB) (*Snapshot, error) { + tx, err := db.db.Begin(false) + if err != nil { + return nil, err + } + + return &Snapshot{ + tx: tx, + b: tx.Bucket(bucketName)}, nil +} + +func (s *Snapshot) Get(key []byte) ([]byte, error) { + return s.b.Get(key), nil +} + +func (s *Snapshot) NewIterator() driver.IIterator { + return &Iterator{ + tx: nil, + it: s.b.Cursor(), + } +} + +func (s *Snapshot) Close() { + s.tx.Rollback() +} diff --git a/store/db.go b/store/db.go index f32b1c3..76c63e3 100644 --- a/store/db.go +++ b/store/db.go @@ -5,51 +5,18 @@ import ( ) type DB struct { - db driver.IDB -} - -// Close database -// -// Caveat -// Any other DB operations like Get, Put, etc... may cause a panic after Close -// -func (db *DB) Close() error { - if db.db == nil { - return nil - } - - err := db.db.Close() - db.db = nil - - return err -} - -// Get Value with Key -func (db *DB) Get(key []byte) ([]byte, error) { - return db.db.Get(key) -} - -// Put value with key -func (db *DB) Put(key []byte, value []byte) error { - err := db.db.Put(key, value) - return err -} - -// Delete by key -func (db *DB) Delete(key []byte) error { - err := db.db.Delete(key) - return err + driver.IDB } func (db *DB) NewIterator() *Iterator { it := new(Iterator) - it.it = db.db.NewIterator() + it.it = db.IDB.NewIterator() return it } func (db *DB) NewWriteBatch() WriteBatch { - return db.db.NewWriteBatch() + return db.IDB.NewWriteBatch() } func (db *DB) RangeIterator(min []byte, max []byte, rangeType uint8) *RangeLimitIterator { @@ -74,11 +41,11 @@ func (db *DB) RevRangeLimitIterator(min []byte, max []byte, rangeType uint8, off return NewRevRangeLimitIterator(db.NewIterator(), &Range{min, max, rangeType}, &Limit{offset, count}) } -func (db *DB) Begin() (Tx, error) { - tx, err := db.db.Begin() +func (db *DB) Begin() (*Tx, error) { + tx, err := db.IDB.Begin() if err != nil { return nil, err } - return tx, nil + return &Tx{tx}, nil } diff --git a/store/driver/driver.go b/store/driver/driver.go index a557d3e..6da67df 100644 --- a/store/driver/driver.go +++ b/store/driver/driver.go @@ -20,9 +20,17 @@ type IDB interface { NewWriteBatch() IWriteBatch + NewSnapshot() (ISnapshot, error) + Begin() (Tx, error) } +type ISnapshot interface { + Get(key []byte) ([]byte, error) + NewIterator() IIterator + Close() +} + type IIterator interface { Close() error diff --git a/store/driver/store.go b/store/driver/store.go index 91f4c66..dffa670 100644 --- a/store/driver/store.go +++ b/store/driver/store.go @@ -8,7 +8,7 @@ import ( type Store interface { String() string Open(path string, cfg *config.Config) (IDB, error) - Repair(paht string, cfg *config.Config) error + Repair(path string, cfg *config.Config) error } var dbs = map[string]Store{} diff --git a/store/goleveldb/db.go b/store/goleveldb/db.go index 33e408f..ca2de60 100644 --- a/store/goleveldb/db.go +++ b/store/goleveldb/db.go @@ -137,6 +137,20 @@ func (db *DB) Begin() (driver.Tx, error) { return nil, driver.ErrTxSupport } +func (db *DB) NewSnapshot() (driver.ISnapshot, error) { + snapshot, err := db.db.GetSnapshot() + if err != nil { + return nil, err + } + + s := &Snapshot{ + db: db, + snp: snapshot, + } + + return s, nil +} + func init() { driver.Register(Store{}) } diff --git a/store/goleveldb/snapshot.go b/store/goleveldb/snapshot.go new file mode 100644 index 0000000..4dd56a9 --- /dev/null +++ b/store/goleveldb/snapshot.go @@ -0,0 +1,26 @@ +package goleveldb + +import ( + "github.com/siddontang/goleveldb/leveldb" + "github.com/siddontang/ledisdb/store/driver" +) + +type Snapshot struct { + db *DB + snp *leveldb.Snapshot +} + +func (s *Snapshot) Get(key []byte) ([]byte, error) { + return s.snp.Get(key, s.db.iteratorOpts) +} + +func (s *Snapshot) NewIterator() driver.IIterator { + it := &Iterator{ + s.snp.NewIterator(nil, s.db.iteratorOpts), + } + return it +} + +func (s *Snapshot) Close() { + s.snp.Release() +} diff --git a/store/hyperleveldb/db.go b/store/hyperleveldb/db.go index 34d6d3f..e8ec944 100644 --- a/store/hyperleveldb/db.go +++ b/store/hyperleveldb/db.go @@ -191,6 +191,20 @@ func (db *DB) NewIterator() driver.IIterator { return it } +func (db *DB) NewSnapshot() (driver.ISnapshot, error) { + snap := &Snapshot{ + db: db, + snap: C.leveldb_create_snapshot(db.db), + readOpts: NewReadOptions(), + iteratorOpts: NewReadOptions(), + } + snap.readOpts.SetSnapshot(snap) + snap.iteratorOpts.SetSnapshot(snap) + snap.iteratorOpts.SetFillCache(false) + + return snap, nil +} + func (db *DB) put(wo *WriteOptions, key, value []byte) error { var errStr *C.char var k, v *C.char diff --git a/store/hyperleveldb/options.go b/store/hyperleveldb/options.go index 09c9a02..43764f6 100644 --- a/store/hyperleveldb/options.go +++ b/store/hyperleveldb/options.go @@ -105,6 +105,14 @@ func (ro *ReadOptions) SetFillCache(b bool) { C.leveldb_readoptions_set_fill_cache(ro.Opt, boolToUchar(b)) } +func (ro *ReadOptions) SetSnapshot(snap *Snapshot) { + var s *C.leveldb_snapshot_t + if snap != nil { + s = snap.snap + } + C.leveldb_readoptions_set_snapshot(ro.Opt, s) +} + func (wo *WriteOptions) Close() { C.leveldb_writeoptions_destroy(wo.Opt) } diff --git a/store/hyperleveldb/snapshot.go b/store/hyperleveldb/snapshot.go new file mode 100644 index 0000000..5054152 --- /dev/null +++ b/store/hyperleveldb/snapshot.go @@ -0,0 +1,35 @@ +// +build hyperleveldb + +package hyperleveldb + +// #cgo LDFLAGS: -lhyperleveldb +// #include "hyperleveldb/c.h" +import "C" + +import ( + "github.com/siddontang/ledisdb/store/driver" +) + +type Snapshot struct { + db *DB + snap *C.leveldb_snapshot_t + readOpts *ReadOptions + iteratorOpts *ReadOptions +} + +func (s *Snapshot) Get(key []byte) ([]byte, error) { + return s.db.get(s.readOpts, key) +} + +func (s *Snapshot) NewIterator() driver.IIterator { + it := new(Iterator) + it.it = C.leveldb_create_iterator(s.db.db, s.db.iteratorOpts.Opt) + return it + +} + +func (s *Snapshot) Close() { + C.leveldb_release_snapshot(s.db.db, s.snap) + s.iteratorOpts.Close() + s.readOpts.Close() +} diff --git a/store/leveldb/db.go b/store/leveldb/db.go index f42726f..0a40953 100644 --- a/store/leveldb/db.go +++ b/store/leveldb/db.go @@ -191,6 +191,20 @@ func (db *DB) NewIterator() driver.IIterator { return it } +func (db *DB) NewSnapshot() (driver.ISnapshot, error) { + snap := &Snapshot{ + db: db, + snap: C.leveldb_create_snapshot(db.db), + readOpts: NewReadOptions(), + iteratorOpts: NewReadOptions(), + } + snap.readOpts.SetSnapshot(snap) + snap.iteratorOpts.SetSnapshot(snap) + snap.iteratorOpts.SetFillCache(false) + + return snap, nil +} + func (db *DB) put(wo *WriteOptions, key, value []byte) error { var errStr *C.char var k, v *C.char diff --git a/store/leveldb/options.go b/store/leveldb/options.go index 83009be..bd3ad16 100644 --- a/store/leveldb/options.go +++ b/store/leveldb/options.go @@ -105,6 +105,14 @@ func (ro *ReadOptions) SetFillCache(b bool) { C.leveldb_readoptions_set_fill_cache(ro.Opt, boolToUchar(b)) } +func (ro *ReadOptions) SetSnapshot(snap *Snapshot) { + var s *C.leveldb_snapshot_t + if snap != nil { + s = snap.snap + } + C.leveldb_readoptions_set_snapshot(ro.Opt, s) +} + func (wo *WriteOptions) Close() { C.leveldb_writeoptions_destroy(wo.Opt) } diff --git a/store/leveldb/snapshot.go b/store/leveldb/snapshot.go new file mode 100644 index 0000000..e8e6ca7 --- /dev/null +++ b/store/leveldb/snapshot.go @@ -0,0 +1,35 @@ +// +build leveldb + +package leveldb + +// #cgo LDFLAGS: -lleveldb +// #include "leveldb/c.h" +import "C" + +import ( + "github.com/siddontang/ledisdb/store/driver" +) + +type Snapshot struct { + db *DB + snap *C.leveldb_snapshot_t + readOpts *ReadOptions + iteratorOpts *ReadOptions +} + +func (s *Snapshot) Get(key []byte) ([]byte, error) { + return s.db.get(s.readOpts, key) +} + +func (s *Snapshot) NewIterator() driver.IIterator { + it := new(Iterator) + it.it = C.leveldb_create_iterator(s.db.db, s.db.iteratorOpts.Opt) + return it + +} + +func (s *Snapshot) Close() { + C.leveldb_release_snapshot(s.db.db, s.snap) + s.iteratorOpts.Close() + s.readOpts.Close() +} diff --git a/store/mdb/mdb.go b/store/mdb/mdb.go index 177a3d1..171c088 100644 --- a/store/mdb/mdb.go +++ b/store/mdb/mdb.go @@ -278,6 +278,10 @@ func (db MDB) Begin() (driver.Tx, error) { return newTx(db) } +func (db MDB) NewSnapshot() (driver.ISnapshot, error) { + return newSnapshot(db) +} + func init() { driver.Register(Store{}) } diff --git a/store/mdb/snapshot.go b/store/mdb/snapshot.go new file mode 100644 index 0000000..270907d --- /dev/null +++ b/store/mdb/snapshot.go @@ -0,0 +1,43 @@ +// +build !windows + +package mdb + +import ( + "github.com/siddontang/ledisdb/store/driver" + mdb "github.com/szferi/gomdb" +) + +type Snapshot struct { + db mdb.DBI + tx *mdb.Txn +} + +func newSnapshot(db MDB) (*Snapshot, error) { + tx, err := db.env.BeginTxn(nil, mdb.RDONLY) + if err != nil { + return nil, err + } + + return &Snapshot{db.db, tx}, nil +} + +func (s *Snapshot) Get(key []byte) ([]byte, error) { + v, err := s.tx.Get(s.db, key) + if err == mdb.NotFound { + return nil, nil + } + return v, err +} + +func (s *Snapshot) NewIterator() driver.IIterator { + c, err := s.tx.CursorOpen(s.db) + if err != nil { + return &MDBIterator{nil, nil, nil, nil, false, err, false} + } + + return &MDBIterator{nil, nil, c, s.tx, true, nil, false} +} + +func (s *Snapshot) Close() { + s.tx.Commit() +} diff --git a/store/mdb/tx.go b/store/mdb/tx.go index a9ed5ef..e98a5f6 100644 --- a/store/mdb/tx.go +++ b/store/mdb/tx.go @@ -22,7 +22,11 @@ func newTx(db MDB) (*Tx, error) { } func (t *Tx) Get(key []byte) ([]byte, error) { - return t.tx.Get(t.db, key) + v, err := t.tx.Get(t.db, key) + if err == mdb.NotFound { + return nil, nil + } + return v, err } func (t *Tx) Put(key []byte, value []byte) error { diff --git a/store/rocksdb/db.go b/store/rocksdb/db.go index d8f26c4..90183f6 100644 --- a/store/rocksdb/db.go +++ b/store/rocksdb/db.go @@ -211,6 +211,20 @@ func (db *DB) NewIterator() driver.IIterator { return it } +func (db *DB) NewSnapshot() (driver.ISnapshot, error) { + snap := &Snapshot{ + db: db, + snap: C.rocksdb_create_snapshot(db.db), + readOpts: NewReadOptions(), + iteratorOpts: NewReadOptions(), + } + snap.readOpts.SetSnapshot(snap) + snap.iteratorOpts.SetSnapshot(snap) + snap.iteratorOpts.SetFillCache(false) + + return snap, nil +} + func (db *DB) put(wo *WriteOptions, key, value []byte) error { var errStr *C.char var k, v *C.char diff --git a/store/rocksdb/options.go b/store/rocksdb/options.go index 3a9f043..aaa9a06 100644 --- a/store/rocksdb/options.go +++ b/store/rocksdb/options.go @@ -153,6 +153,14 @@ func (ro *ReadOptions) SetFillCache(b bool) { C.rocksdb_readoptions_set_fill_cache(ro.Opt, boolToUchar(b)) } +func (ro *ReadOptions) SetSnapshot(snap *Snapshot) { + var s *C.rocksdb_snapshot_t + if snap != nil { + s = snap.snap + } + C.rocksdb_readoptions_set_snapshot(ro.Opt, s) +} + func (wo *WriteOptions) Close() { C.rocksdb_writeoptions_destroy(wo.Opt) } diff --git a/store/rocksdb/snapshot.go b/store/rocksdb/snapshot.go new file mode 100644 index 0000000..e560e8e --- /dev/null +++ b/store/rocksdb/snapshot.go @@ -0,0 +1,35 @@ +// +build rocksdb + +package rocksdb + +// #cgo LDFLAGS: -lrocksdb +// #include "rocksdb/c.h" +import "C" + +import ( + "github.com/siddontang/ledisdb/store/driver" +) + +type Snapshot struct { + db *DB + snap *C.rocksdb_snapshot_t + readOpts *ReadOptions + iteratorOpts *ReadOptions +} + +func (s *Snapshot) Get(key []byte) ([]byte, error) { + return s.db.get(s.readOpts, key) +} + +func (s *Snapshot) NewIterator() driver.IIterator { + it := new(Iterator) + it.it = C.rocksdb_create_iterator(s.db.db, s.db.iteratorOpts.Opt) + return it + +} + +func (s *Snapshot) Close() { + C.rocksdb_release_snapshot(s.db.db, s.snap) + s.iteratorOpts.Close() + s.readOpts.Close() +} diff --git a/store/store_test.go b/store/store_test.go index 2c3517c..5fdd95d 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -37,6 +37,7 @@ func testStore(db *DB, t *testing.T) { testSimple(db, t) testBatch(db, t) testIterator(db, t) + testSnapshot(db, t) } func testClear(db *DB, t *testing.T) { @@ -44,6 +45,7 @@ func testClear(db *DB, t *testing.T) { for ; it.Valid(); it.Next() { db.Delete(it.RawKey()) } + it.Close() } func testSimple(db *DB, t *testing.T) { @@ -259,3 +261,65 @@ func testIterator(db *DB, t *testing.T) { } it.Close() } + +func testSnapshot(db *DB, t *testing.T) { + foo := []byte("foo") + bar := []byte("bar") + v1 := []byte("v1") + v2 := []byte("v2") + + db.Put(foo, v1) + db.Put(bar, v1) + + snap, err := db.NewSnapshot() + if err != nil { + t.Fatal(err) + } + + i := snap.NewIterator() + + i.Seek([]byte("foo")) + + if !i.Valid() { + t.Fatal("must valid") + } else if string(i.Value()) != "v1" { + t.Fatal(string(i.Value())) + } + i.Close() + + db.Put(foo, v2) + db.Put(bar, v2) + + if v, err := snap.Get(foo); err != nil { + t.Fatal(err) + } else if string(v) != "v1" { + t.Fatal(string(v)) + } + + if v, err := snap.Get(bar); err != nil { + t.Fatal(err) + } else if string(v) != "v1" { + t.Fatal(string(v)) + } + + if v, err := db.Get(foo); err != nil { + t.Fatal(err) + } else if string(v) != "v2" { + t.Fatal(string(v)) + } + + if v, err := db.Get(bar); err != nil { + t.Fatal(err) + } else if string(v) != "v2" { + t.Fatal(string(v)) + } + + snap.Close() + + if v, err := db.Get(foo); err != nil { + t.Fatal(err) + } else if string(v) != "v2" { + t.Fatal(string(v)) + } + +} diff --git a/store/tx.go b/store/tx.go index 1a074c7..8ee72a8 100644 --- a/store/tx.go +++ b/store/tx.go @@ -4,6 +4,39 @@ import ( "github.com/siddontang/ledisdb/store/driver" ) -type Tx interface { +type Tx struct { driver.Tx } + +func (tx *Tx) NewIterator() *Iterator { + it := new(Iterator) + it.it = tx.Tx.NewIterator() + + return it +} + +func (tx *Tx) NewWriteBatch() WriteBatch { + return tx.Tx.NewWriteBatch() +} + +func (tx *Tx) RangeIterator(min []byte, max []byte, rangeType uint8) *RangeLimitIterator { + return NewRangeLimitIterator(tx.NewIterator(), &Range{min, max, rangeType}, &Limit{0, -1}) +} + +func (tx *Tx) RevRangeIterator(min []byte, max []byte, rangeType uint8) *RangeLimitIterator { + return NewRevRangeLimitIterator(tx.NewIterator(), &Range{min, max, rangeType}, &Limit{0, -1}) +} + +//count < 0, unlimit. +// +//offset must >= 0, if < 0, will get nothing. +func (tx *Tx) RangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *RangeLimitIterator { + return NewRangeLimitIterator(tx.NewIterator(), &Range{min, max, rangeType}, &Limit{offset, count}) +} + +//count < 0, unlimit. +// +//offset must >= 0, if < 0, will get nothing. +func (tx *Tx) RevRangeLimitIterator(min []byte, max []byte, rangeType uint8, offset int, count int) *RangeLimitIterator { + return NewRevRangeLimitIterator(tx.NewIterator(), &Range{min, max, rangeType}, &Limit{offset, count}) +} diff --git a/store/tx_test.go b/store/tx_test.go index 2b86c64..1f89436 100644 --- a/store/tx_test.go +++ b/store/tx_test.go @@ -59,7 +59,7 @@ func testTx(db *DB, t *testing.T) { t.Fatal(string(it.Value())) } - it.First() + it.SeekToFirst() if !it.Valid() { t.Fatal("must valid") @@ -83,7 +83,7 @@ func testTx(db *DB, t *testing.T) { t.Fatal(string(it.Value())) } - it.Last() + it.SeekToLast() if !it.Valid() { t.Fatal("must valid") diff --git a/generate.py b/tools/generate_commands.py similarity index 66% rename from generate.py rename to tools/generate_commands.py index 1295e22..7d2d9bc 100644 --- a/generate.py +++ b/tools/generate_commands.py @@ -6,15 +6,6 @@ import sys import os from collections import OrderedDict as dict -content = u"""\n -type cmdConf struct { - name string - argDesc string - group string - readonly bool -} -""" - def json_to_js(json_path, js_path): """Convert `commands.json` to `commands.js`""" @@ -44,24 +35,8 @@ def json_to_go_array(json_path, go_path): g_fp.close() -def json_to_command_cnf(json_path, go_path): - g_fp = open(go_path, "w") - - with open(json_path) as fp: - _json = json.load(fp) - generate_time(g_fp) - g_fp.write("package server") - print >> g_fp, content - g_fp.write("var cnfCmds = []cmdConf{\n") - for k, v in _json.iteritems(): - g_fp.write('\t{\n\t\t"%s",\n\t\t"%s",\n\t\t"%s", \n\t\t%s,\n\t},\n' % - (k, v["arguments"], v["group"], "true" if v["readonly"] else "false" )) - g_fp.write("}\n") - g_fp.close() - - def generate_time(fp): - fp.write("//This file was generated by ./generate.py on %s \n" % + fp.write("//This file was generated by .tools/generate_commands.py on %s \n" % time.strftime('%a %b %d %Y %H:%M:%S %z')) @@ -77,10 +52,6 @@ if __name__ == "__main__": python generate.py /path/to/commands.json /path/to/const.go - 3. for server/command_cnf.go - - python generate.py /path/to/commands.json /path/to/command_cnf.go - """ if len(sys.argv) != 3: @@ -95,8 +66,5 @@ if __name__ == "__main__": elif dst_path_base.startswith("const.go"): json_to_go_array(src_path, dst_path) - elif dst_path_base.startswith("command"): - json_to_command_cnf(src_path, dst_path) - else: print "Not support arguments"