diff --git a/.gitignore b/.gitignore index 8458dd7..d1c3459 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ build *.pyc .DS_Store +nohup.out \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1dfe2f3 --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +all: build + +build: + go install ./... + +clean: + go clean -i ./... + +test: + go test ./... + go test -race ./... \ No newline at end of file diff --git a/README.md b/README.md index 6d7f61f..846ac17 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Ledisdb is a high performance NoSQL like Redis based on LevelDB written by go. I + Supports expiration and ttl. + Redis clients, like redis-cli, are supported directly. + Multi client API supports, including Golang, Python, Lua(Openresty). -+ Easily to embed in Golang application. ++ Easy to embed in Golang application. + Replication to guarantee data safe. + Supplies tools to load, dump, repair database. @@ -25,13 +25,15 @@ Ledisdb is a high performance NoSQL like Redis based on LevelDB written by go. I + Install leveldb and snappy, if you have installed, skip. - I supply a simple shell to install leveldb and snappy, you can use: + LedisDB supplies a simple shell to install leveldb and snappy: sh build_leveldb.sh - It will default install leveldb at /usr/local/leveldb and snappy at /usr/local/snappy + It will default install leveldb at /usr/local/leveldb and snappy at /usr/local/snappy. -+ Change LEVELDB_DIR and SNAPPY_DIR to real install path in dev.sh. + LedisDB use the modified LevelDB for better performance, see [here](https://github.com/siddontang/ledisdb/wiki/leveldb-source-modification). + ++ Set LEVELDB_DIR and SNAPPY_DIR to the actual install path in dev.sh. + Then: @@ -40,7 +42,7 @@ Ledisdb is a high performance NoSQL like Redis based on LevelDB written by go. I go install ./... -## Run +## Server Example ./ledis-server -config=/etc/ledis.json @@ -52,7 +54,7 @@ Ledisdb is a high performance NoSQL like Redis based on LevelDB written by go. I ledis 127.0.0.1:6380> get a "1" -## Lib +## Package Example import "github.com/siddontang/ledisdb/ledis" l, _ := ledis.Open(cfg) @@ -63,9 +65,9 @@ Ledisdb is a high performance NoSQL like Redis based on LevelDB written by go. I db.Get(key) -## Replication +## Replication Example -set slaveof in config or dynamiclly +Set slaveof in config or dynamiclly ledis-cli -p 6381 @@ -78,15 +80,16 @@ See benchmark.md for more. ## Todo -+ Admin +See [Issues todo](https://github.com/siddontang/ledisdb/issues?labels=todo&page=1&state=open) -## GoDoc -[![GoDoc](https://godoc.org/github.com/siddontang/ledisdb?status.png)](https://godoc.org/github.com/siddontang/ledisdb) +## Links -## Commands ++ [Official Website](http://ledisdb.com) ++ [Author's Chinese Blog](http://blog.csdn.net/siddontang/article/category/2264003) ++ [GoDoc](https://godoc.org/github.com/siddontang/ledisdb) ++ [Server Commands](https://github.com/siddontang/ledisdb/wiki/Commands) -Some server commands explaintions are [here](https://github.com/siddontang/ledisdb/wiki/Commands), others will add continuate. ## Thanks @@ -94,6 +97,13 @@ Gmail: cenqichao@gmail.com Gmail: chendahui007@gmail.com +Gmail: cppgohan@gmail.com + +Gmail: tiaotiaoyly@gmail.com + +Gmail: wyk4true@gmail.com + + ## Feedback -Gmail: siddontang@gmail.com \ No newline at end of file +Gmail: siddontang@gmail.com diff --git a/client/ledis-py/ledis/client.py b/client/ledis-py/ledis/client.py index a855e84..7d64457 100644 --- a/client/ledis-py/ledis/client.py +++ b/client/ledis-py/ledis/client.py @@ -1,7 +1,7 @@ from __future__ import with_statement import datetime import time as mod_time -from ledis._compat import (b, izip, imap, iteritems, iterkeys, itervalues, +from ledis._compat import (b, izip, imap, iteritems, basestring, long, nativestr, urlparse, bytes) from ledis.connection import ConnectionPool, UnixDomainSocketConnection from ledis.exceptions import ( @@ -53,9 +53,8 @@ def zset_score_pairs(response, **options): """ if not response or not options['withscores']: return response - score_cast_func = options.get('score_cast_func', int) it = iter(response) - return list(izip(it, imap(score_cast_func, it))) + return list(izip(it, imap(int, it))) def int_or_none(response): @@ -77,13 +76,14 @@ class Ledis(object): RESPONSE_CALLBACKS = dict_merge( string_keys_to_dict( 'EXISTS EXPIRE EXPIREAT HEXISTS HMSET SETNX ' - 'PERSIST HPERSIST LPERSIST ZPERSIST', + 'PERSIST HPERSIST LPERSIST ZPERSIST BEXPIRE ' + 'BEXPIREAT BPERSIST BDELETE', bool ), string_keys_to_dict( 'DECRBY DEL HDEL HLEN INCRBY LLEN ' 'ZADD ZCARD ZREM ZREMRANGEBYRANK ZREMRANGEBYSCORE' - 'LMCLEAR HMCLEAR ZMCLEAR', + 'LMCLEAR HMCLEAR ZMCLEAR BCOUNT BGETBIT BSETBIT BOPT BMSETBIT', int ), string_keys_to_dict( @@ -102,7 +102,9 @@ class Ledis(object): { 'HGETALL': lambda r: r and pairs_to_dict(r) or {}, 'PING': lambda r: nativestr(r) == 'PONG', - 'SET': lambda r: r and nativestr(r) == 'OK', } + 'SET': lambda r: r and nativestr(r) == 'OK', + }, + ) @classmethod @@ -187,7 +189,6 @@ class Ledis(object): return self.response_callbacks[command_name](response, **options) return response - #### SERVER INFORMATION #### def echo(self, value): "Echo the string back from the server" @@ -205,7 +206,6 @@ class Ledis(object): db = 0 return self.execute_command('SELECT', db) - #### BASIC KEY COMMANDS #### def decr(self, name, amount=1): """ @@ -327,7 +327,6 @@ class Ledis(object): "Removes an expiration on name" return self.execute_command('PERSIST', name) - #### LIST COMMANDS #### def lindex(self, name, index): """ @@ -433,8 +432,8 @@ class Ledis(object): def zcount(self, name, min, max): """ - Return the number of elements in the sorted set at key ``name`` with a score - between ``min`` and ``max``. + Return the number of elements in the sorted set at key ``name`` with a score + between ``min`` and ``max``. The min and max arguments have the same semantic as described for ZRANGEBYSCORE. """ return self.execute_command('ZCOUNT', name, min, max) @@ -454,17 +453,15 @@ class Ledis(object): ``withscores`` indicates to return the scores along with the values. The return type is a list of (value, score) pairs - - ``score_cast_func`` a callable used to cast the score return value """ if desc: - return self.zrevrange(name, start, end, withscores, - score_cast_func) + return self.zrevrange(name, start, end, withscores) + pieces = ['ZRANGE', name, start, end] if withscores: pieces.append('withscores') options = { - 'withscores': withscores, 'score_cast_func': int} + 'withscores': withscores} return self.execute_command(*pieces, **options) def zrangebyscore(self, name, min, max, start=None, num=None, @@ -478,8 +475,6 @@ class Ledis(object): ``withscores`` indicates to return the scores along with the values. The return type is a list of (value, score) pairs - - `score_cast_func`` a callable used to cast the score return value """ if (start is not None and num is None) or \ (num is not None and start is None): @@ -490,7 +485,7 @@ class Ledis(object): if withscores: pieces.append('withscores') options = { - 'withscores': withscores, 'score_cast_func': int} + 'withscores': withscores} return self.execute_command(*pieces, **options) def zrank(self, name, value): @@ -529,14 +524,11 @@ class Ledis(object): ``withscores`` indicates to return the scores along with the values The return type is a list of (value, score) pairs - - ``score_cast_func`` a callable used to cast the score return value """ pieces = ['ZREVRANGE', name, start, num] if withscores: pieces.append('withscores') - options = { - 'withscores': withscores, 'score_cast_func': int} + options = {'withscores': withscores} return self.execute_command(*pieces, **options) def zrevrangebyscore(self, name, min, max, start=None, num=None, @@ -550,8 +542,6 @@ class Ledis(object): ``withscores`` indicates to return the scores along with the values. The return type is a list of (value, score) pairs - - ``score_cast_func`` a callable used to cast the score return value """ if (start is not None and num is None) or \ (num is not None and start is None): @@ -561,8 +551,7 @@ class Ledis(object): pieces.extend(['LIMIT', start, num]) if withscores: pieces.append('withscores') - options = { - 'withscores': withscores, 'score_cast_func': int} + options = {'withscores': withscores} return self.execute_command(*pieces, **options) def zrevrank(self, name, value): @@ -703,3 +692,79 @@ class Ledis(object): "Removes an expiration on name" return self.execute_command('HPERSIST', name) + + + ### BIT COMMANDS + def bget(self, name): + "" + return self.execute_command("BGET", name) + + def bdelete(self, name): + "" + return self.execute_command("BDELETE", name) + + def bsetbit(self, name, offset, value): + "" + value = value and 1 or 0 + return self.execute_command("BSETBIT", name, offset, value) + + def bgetbit(self, name, offset): + "" + return self.execute_command("BGETBIT", name, offset) + + def bmsetbit(self, name, *args): + """ + Set any number of offset, value pairs to the key ``name``. Pairs can be + specified in the following way: + + offset1, value1, offset2, value2, ... + """ + pieces = [] + if args: + if len(args) % 2 != 0: + raise LedisError("BMSETBIT requires an equal number of " + "offset and value") + pieces.extend(args) + return self.execute_command("BMSETBIT", name, *pieces) + + def bcount(self, key, start=None, end=None): + "" + params = [key] + if start is not None and end is not None: + params.append(start) + params.append(end) + elif (start is not None and end is None) or \ + (start is None and end is not None): + raise LedisError("Both start and end must be specified") + return self.execute_command("BCOUNT", *params) + + def bopt(self, operation, dest, *keys): + """ + Perform a bitwise operation using ``operation`` between ``keys`` and + store the result in ``dest``. + ``operation`` is one of `and`, `or`, `xor`, `not`. + """ + return self.execute_command('BOPT', operation, dest, *keys) + + def bexpire(self, name, time): + "Set timeout on key ``name`` with ``time``" + if isinstance(time, datetime.timedelta): + time = time.seconds + time.days * 24 * 3600 + return self.execute_command('BEXPIRE', name, time) + + def bexpireat(self, name, when): + """ + Set an expire flag on key name for time seconds. time can be represented by + an integer or a Python timedelta object. + """ + if isinstance(when, datetime.datetime): + when = int(mod_time.mktime(when.timetuple())) + return self.execute_command('BEXPIREAT', name, when) + + def bttl(self, name): + "Returns the number of seconds until the key name will expire" + return self.execute_command('BTTL', name) + + def bpersist(self, name): + "Removes an expiration on name" + return self.execute_command('BPERSIST', name) \ No newline at end of file diff --git a/client/ledis-py/tests/test_cmd_bit.py b/client/ledis-py/tests/test_cmd_bit.py new file mode 100644 index 0000000..7863d21 --- /dev/null +++ b/client/ledis-py/tests/test_cmd_bit.py @@ -0,0 +1,118 @@ +# coding: utf-8 +# Test Cases for bit commands + +import unittest +import sys +import datetime, time +sys.path.append('..') + +import ledis +from ledis._compat import b +from ledis import ResponseError + + +l = ledis.Ledis(port=6380) + + +def current_time(): + return datetime.datetime.now() + + +class TestCmdBit(unittest.TestCase): + def setUp(self): + pass + + def tearDown(self): + l.bdelete('a') + l.bdelete('non_exists_key') + + def test_bget(self): + "bget is the same as get in K/V commands" + l.bmsetbit('a', 0, 1, 1, 1, 2, 1, 3, 1, 4, 1, 5, 1, 6, 1, 7, 0) + assert l.bget('a') == b('\x7f') + + def test_bdelete(self): + l.bsetbit('a', 0, 1) + assert l.bdelete('a') + assert not l.bdelete('non_exists_key') + + def test_get_set_bit(self): + assert not l.bgetbit('a', 5) + assert not l.bsetbit('a', 5, True) + assert l.bgetbit('a', 5) + + assert not l.bsetbit('a', 4, False) + assert not l.bgetbit('a', 4) + + assert not l.bsetbit('a', 4, True) + assert l.bgetbit('a', 4) + + assert l.bsetbit('a', 5, True) + assert l.bgetbit('a', 5) + + def test_bmsetbit(self): + assert l.bmsetbit('a', 0, 1, 2, 1, 3, 1) == 3 + + def test_bcount(self): + l.bsetbit('a', 5, 1) + assert l.bcount('a') == 1 + l.bsetbit('a', 6, 1) + assert l.bcount('a') == 2 + l.bsetbit('a', 5, 0) + assert l.bcount('a') == 1 + l.bmsetbit('a', 10, 1, 20, 1, 30, 1, 40, 1) + assert l.bcount('a') == 5 + assert l.bcount('a', 0, 10) == 2 + assert l.bcount('a', 20, 30) == 2 + assert l.bcount('a', 10, 10) == 1 + + def test_bopt_not_empty_string(self): + l.bopt('not', 'r', 'a') + assert l.bget('r') is None + + def test_bopt(self): + l.bmsetbit('a1', 10, 1, 30, 1, 50, 1, 70, 1, 90, 1) + l.bmsetbit('a2', 20, 1, 40, 1, 60, 1, 80, 1, 100, 1) + assert l.bopt('and', 'res1', 'a1', 'a2') == 101 + assert l.bcount('res1') == 0 + + assert l.bopt('or', 'res2', 'a1', 'a2') == 101 + assert l.bcount('res2') == 10 + + assert l.bopt('xor', 'res3', 'a1', 'a2') == 101 + assert l.bcount('res3') == 10 + + assert l.bopt('not', 'res4', 'a1') == 91 + assert l.bcount('res4') == 86 + + def test_bexpire(self): + assert not l.bexpire('a', 100) + l.bsetbit('a', 1, True) + assert l.bexpire('a', 100) + assert 0 < l.bttl('a') <= 100 + assert l.bpersist('a') + assert l.bttl('a') == -1 + + def test_bexpireat_datetime(self): + expire_at = current_time() + datetime.timedelta(minutes=1) + l.bsetbit('a', 1, True) + assert l.bexpireat('a', expire_at) + assert 0 < l.bttl('a') <= 61 + + def test_bexpireat_unixtime(self): + expire_at = current_time() + datetime.timedelta(minutes=1) + l.bsetbit('a', 1, True) + expire_at_seconds = int(time.mktime(expire_at.timetuple())) + assert l.bexpireat('a', expire_at_seconds) + assert 0 < l.bttl('a') <= 61 + + def test_bexpireat_no_key(self): + expire_at = current_time() + datetime.timedelta(minutes=1) + assert not l.bexpireat('a', expire_at) + + def test_bttl_and_bpersist(self): + l.bsetbit('a', 1, True) + l.bexpire('a', 100) + assert 0 < l.bttl('a') <= 100 + assert l.bpersist('a') + assert l.bttl('a') == -1 diff --git a/client/ledis-py/tests/test_cmd_hash.py b/client/ledis-py/tests/test_cmd_hash.py index 599eb77..d0fe5ab 100644 --- a/client/ledis-py/tests/test_cmd_hash.py +++ b/client/ledis-py/tests/test_cmd_hash.py @@ -122,7 +122,7 @@ class TestCmdHash(unittest.TestCase): assert l.hexpireat('a', expire_at_seconds) assert 0 < l.httl('a') <= 61 - def test_zexpireat_no_key(self): + def test_hexpireat_no_key(self): expire_at = current_time() + datetime.timedelta(minutes=1) assert not l.hexpireat('a', expire_at) diff --git a/client/openresty/ledis.lua b/client/openresty/ledis.lua index fa149f6..06d9161 100644 --- a/client/openresty/ledis.lua +++ b/client/openresty/ledis.lua @@ -100,6 +100,19 @@ local commands = { "zttl", "zpersist", + --[[bit]] + "bget", + "bdelete", + "bsetbit", + "bgetbit", + "bmsetbit", + "bcount", + "bopt", + "bexpire", + "bexpireat", + "bttl", + "bpersist", + --[[server]] "ping", "echo", diff --git a/cmd/ledis-cli/const.go b/cmd/ledis-cli/const.go new file mode 100644 index 0000000..1cdff8f --- /dev/null +++ b/cmd/ledis-cli/const.go @@ -0,0 +1,84 @@ +package main + +var helpCommands = [][]string{ + {"DECR", "key", "KV"}, + {"DECRBY", "key decrement", "KV"}, + {"DEL", "key [key ...]", "KV"}, + {"EXISTS", "key", "KV"}, + {"GET", "key", "KV"}, + {"GETSET", " key value", "KV"}, + {"INCR", "key", "KV"}, + {"INCRBY", "key increment", "KV"}, + {"MGET", "key [key ...]", "KV"}, + {"MSET", "key value [key value ...]", "KV"}, + {"SET", "key value", "KV"}, + {"SETNX", "key value", "KV"}, + {"EXPIRE", "key seconds", "KV"}, + {"EXPIREAT", "key timestamp", "KV"}, + {"TTL", "PERSIST", "KV"}, + {"HDEL", "key field [field ...]", "Hash"}, + {"HEXISTS", "key field", "Hash"}, + {"HGET", "key field", "Hash"}, + {"HGETALL", "key", "Hash"}, + {"HINCRBY", "key field increment", "Hash"}, + {"HKEYS", "key", "Hash"}, + {"HLEN", "key", "Hash"}, + {"HMGET", "key field [field ...]", "Hash"}, + {"HMSET", "key field value [field value ...]", "Hash"}, + {"HSET", "key field value", "Hash"}, + {"HVALS", "key", "Hash"}, + {"HCLEAR", "key", "Hash"}, + {"HMCLEAR", "key [key ...]", "Hash"}, + {"HEXPIRE", "key seconds", "Hash"}, + {"HEXPIREAT", "key timestamp", "Hash"}, + {"HTTL", "key", "Hash"}, + {"HPERSIST", "key", "Hash"}, + {"LINDEX", "key index", "List"}, + {"LLEN", "key", "List"}, + {"LPOP", "key", "List"}, + {"LPUSH", "key value [value ...]", "List"}, + {"LRANGE", "key start stop", "List"}, + {"RPOP", "key", "List"}, + {"RPUSH", "key value [value ...]", "List"}, + {"LCLEAR", "key", "List"}, + {"LMCLEAR", "key [key ...]", "List"}, + {"LEXPIRE", "key seconds", "List"}, + {"LEXPIREAT", "key timestamp", "List"}, + {"LTTL", "key", "List"}, + {"LPERSIST", "key", "List"}, + {"ZADD", "key score member [score member ...]", "ZSet"}, + {"ZCARD", "key", "ZSet"}, + {"ZCOUNT", "key min max", "ZSet"}, + {"ZINCRBY", "key increment member", "ZSet"}, + {"ZRANGE", "key start stop [WITHSCORES]", "ZSet"}, + {"ZRANGEBYSCORE", "key min max [WITHSCORES] [LIMIT offset count]", "ZSet"}, + {"ZRANK", "key member", "ZSet"}, + {"ZREM", "key member [member ...]", "ZSet"}, + {"ZREMRANGEBYRANK", "key start stop", "ZSet"}, + {"ZREMRANGEBYSCORE", "key min max", "ZSet"}, + {"ZREVRANGEBYSCORE", "key max min [WITHSCORES][LIMIT offset count]", "ZSet"}, + {"ZREVRANK", "key member", "ZSet"}, + {"ZSCORE", "key member", "ZSet"}, + {"ZCLEAR", "key", "ZSet"}, + {"ZMCLEAR", "key [key ...]", "ZSet"}, + {"ZEXPIRE", "key seconds", "ZSet"}, + {"ZEXPIREAT", "key timestamp", "ZSet"}, + {"ZTTL", "key", "ZSet"}, + {"ZPERSIST", "key", "ZSet"}, + {"BGET", "key", "Bitmap"}, + {"BGETBIT", "key offset", "Bitmap"}, + {"BSETBIT", "key offset value", "Bitmap"}, + {"BMSETBIT", "key offset value [offset value ...]", "Bitmap"}, + {"BOPT", "operation destkey key [key ...]", "Bitmap"}, + {"BCOUNT", "key [start end]", "Bitmap"}, + {"BEXPIRE", "key seconds", "Bitmap"}, + {"BEXPIREAT", "key timestamp", "Bitmap"}, + {"BTTL", "key", "Bitmap"}, + {"BPERSIST", "key", "Bitmap"}, + {"SLAVEOF", "host port", "Replication"}, + {"FULLSYNC", "-", "Replication"}, + {"SYNC", "index offset", "Replication"}, + {"PING", "-", "Server"}, + {"ECHO", "message", "Server"}, + {"SELECT", "index", "Server"}, +} diff --git a/cmd/ledis-cli/main.go b/cmd/ledis-cli/main.go index 4149055..bcd42ac 100644 --- a/cmd/ledis-cli/main.go +++ b/cmd/ledis-cli/main.go @@ -5,12 +5,14 @@ import ( "fmt" "github.com/siddontang/ledisdb/client/go/ledis" "regexp" + "strconv" "strings" ) var ip = flag.String("h", "127.0.0.1", "ledisdb server ip (default 127.0.0.1)") var port = flag.Int("p", 6380, "ledisdb server port (default 6380)") var socket = flag.String("s", "", "ledisdb server socket, overwrite ip and port") +var dbn = flag.Int("n", 0, "ledisdb database number(default 0)") func main() { flag.Parse() @@ -30,8 +32,20 @@ func main() { reg, _ := regexp.Compile(`'.*?'|".*?"|\S+`) + prompt := "" + for { - cmd, err := line(fmt.Sprintf("%s> ", cfg.Addr)) + if *dbn >= 16 { + fmt.Printf("ERR invalid db index %d. Auto switch back db 0.\n", *dbn) + *dbn = 0 + continue + } else if *dbn > 0 && *dbn < 16 { + prompt = fmt.Sprintf("%s[%d]>", cfg.Addr, *dbn) + } else { + prompt = fmt.Sprintf("%s>", cfg.Addr) + } + + cmd, err := line(prompt) if err != nil { fmt.Printf("%s\n", err.Error()) return @@ -44,17 +58,29 @@ func main() { addHistory(cmd) args := make([]interface{}, len(cmds[1:])) + for i := range args { args[i] = strings.Trim(string(cmds[1+i]), "\"'") } - r, err := c.Do(cmds[0], args...) - if err != nil { - fmt.Printf("%s", err.Error()) + + cmd := cmds[0] + if strings.ToLower(cmd) == "help" || cmd == "?" { + printHelp(cmds) } else { - printReply(cmd, r) + r, err := c.Do(cmds[0], args...) + + if err != nil { + fmt.Printf("%s", err.Error()) + } else if nb, _ := strconv.Atoi(cmds[1]); strings.ToLower(cmds[0]) == "select" && nb < 16 { + *dbn = nb + printReply(cmd, r) + } else { + printReply(cmd, r) + } + + fmt.Printf("\n") } - fmt.Printf("\n") } } } @@ -66,7 +92,7 @@ func printReply(cmd string, reply interface{}) { case string: fmt.Printf("%s", reply) case []byte: - fmt.Printf("%s", string(reply)) + fmt.Printf("%q", reply) case nil: fmt.Printf("(nil)") case ledis.Error: @@ -87,3 +113,34 @@ func printReply(cmd string, reply interface{}) { fmt.Printf("invalid ledis reply") } } + +func printGenericHelp() { + msg := + `ledis-cli +Type: "help " for help on + ` + fmt.Println(msg) +} + +func printCommandHelp(arr []string) { + fmt.Println() + fmt.Printf("\t%s %s \n", arr[0], arr[1]) + fmt.Printf("\tGroup: %s \n", arr[2]) + fmt.Println() +} + +func printHelp(cmds []string) { + args := cmds[1:] + if len(args) == 0 { + printGenericHelp() + } else if len(args) > 1 { + fmt.Println() + } else { + cmd := strings.ToUpper(args[0]) + for i := 0; i < len(helpCommands); i++ { + if helpCommands[i][0] == cmd { + printCommandHelp(helpCommands[i]) + } + } + } +} diff --git a/cmd/ledis-load/main.go b/cmd/ledis-load/main.go index da1fad3..0995b29 100644 --- a/cmd/ledis-load/main.go +++ b/cmd/ledis-load/main.go @@ -3,16 +3,13 @@ package main import ( "encoding/json" "flag" + "fmt" "github.com/siddontang/ledisdb/ledis" - "github.com/siddontang/ledisdb/server" "io/ioutil" - "path" ) var configPath = flag.String("config", "/etc/ledis.json", "ledisdb config file") var dumpPath = flag.String("dump_file", "", "ledisdb dump file") -var masterAddr = flag.String("master_addr", "", - "master addr to set where dump file comes from, if not set correctly, next slaveof may cause fullsync") func main() { flag.Parse() @@ -74,13 +71,11 @@ func loadDump(cfg *ledis.Config, ldb *ledis.Ledis) error { return err } - info := new(server.MasterInfo) + //master enable binlog, here output this like mysql + if head.LogFileIndex != 0 && head.LogPos != 0 { + format := "MASTER_LOG_FILE='binlog.%07d', MASTER_LOG_POS=%d;\n" + fmt.Printf(format, head.LogFileIndex, head.LogPos) + } - info.Addr = *masterAddr - info.LogFileIndex = head.LogFileIndex - info.LogPos = head.LogPos - - infoFile := path.Join(cfg.DataDir, "master.info") - - return info.Save(infoFile) + return nil } diff --git a/doc/commands.md b/doc/commands.md index 90d7199..2157cbd 100644 --- a/doc/commands.md +++ b/doc/commands.md @@ -70,7 +70,7 @@ Table of Contents - [ZREMRANGEBYRANK key start stop](#zremrangebyrank-key-start-stop) - [ZREMRANGEBYSCORE key min max](#zremrangebyscore-key-min-max) - [ZREVRANGE key start stop [WITHSCORES]](#zrevrange-key-start-stop-withscores) - - [ZREVRANGEBYSCORE key max min [WITHSCORES] [LIMIT offset count]](#zrevrangebyscore--key-max-min-withscores-limit-offset-count) + - [ZREVRANGEBYSCORE key max min [WITHSCORES] [LIMIT offset count]](#zrevrangebyscore-key-max-min-withscores-limit-offset-count) - [ZREVRANK key member](#zrevrank-key-member) - [ZSCORE key member](#zscore-key-member) - [ZCLEAR key](#zclear-key) @@ -79,6 +79,19 @@ Table of Contents - [ZEXPIREAT key timestamp](#zexpireat-key-timestamp) - [ZTTL key](#zttl-key) - [ZPERSIST key](#zpersist-key) +- [Bitmap](#bitmap) + + - [BGET key](#bget-key) + - [BGETBIT key offset](#bgetbit-key-offset) + - [BSETBIT key offset value](#bsetbit-key-offset-value) + - [BMSETBIT key offset value[offset value ...]](#bmsetbit-key-offset-value-offset-value-) + - [BOPT operation destkey key [key ...]](#bopt-operation-destkey-key-key-) + - [BCOUNT key [start, end]](#bcount-key-start-end) + - [BEXPIRE key seconds](#bexpire-key-seconds) + - [BEXPIREAT key timestamp](#bexpireat-key-timestamp) + - [BTTL key](#bttl-key) + - [BPERSIST key](#bpersist-key) + - [Replication](#replication) - [SLAVEOF host port](#slaveof-host-port) - [FULLSYNC](#fullsync) @@ -1138,7 +1151,7 @@ ledis> ZRANGE myset 0 -1 WITHSCORES 6) "2" 7) "three" 8) "3" -ledis> zcard myset +ledis> ZCARD myset (integer) 4 ``` @@ -1168,9 +1181,9 @@ ledis> ZRANGE myset 0 -1 WITHSCORES 6) "2" 7) "three" 8) "3" -ledis> zcount myset -inf +inf +ledis> ZCOUNT myset -inf +inf (integer) 4 -ledis> zcount myset (1 3 +ledis> ZCOUNT myset (1 3 (integer) 2 ``` @@ -1444,13 +1457,13 @@ Use ZRANK to get the rank of an element with the scores ordered from low to high **Examples** ``` -127.0.0.1:6380> zadd myset 1 one +ledis> ZADD myset 1 one (integer) 1 -127.0.0.1:6380> zadd myset 2 two +ledis> ZADD myset 2 two (integer) 1 -127.0.0.1:6380> zrevrank myset one +ledis> ZREVRANK myset one (integer) 1 -127.0.0.1:6380> zrevrank myset three +ledis> ZREVRANK myset three (nil) ``` @@ -1581,13 +1594,13 @@ int64: TTL in seconds **Examples** ``` -ledis> zadd myset 1 'one' +ledis> ZADD myset 1 'one' (integer) 1 -ledis> zexpire myset 100 +ledis> ZEXPIRE myset 100 (integer) 1 -ledis> zttl myset +ledis> ZTTL myset (integer) 97 -ledis> zttl myset2 +ledis> ZTTL myset2 (integer) -1 ``` @@ -1617,6 +1630,166 @@ ledis> ZTTL mset ``` + +## Bitmap + + +### BGET key + +Returns the whole binary data stored at `key`. + +**Return value** + +bulk: the raw value of key, or nil when key does not exist. + +**Examples** + +``` +ledis> BMSETBIT flag 0 1 5 1 6 1 +(integer) 3 +ledis> BGET flag +a +``` + + +### BGETBIT key offset + +Returns the bit value at `offset` in the string value stored at `key`. +When *offset* beyond the data length, ot the target data is not exist, the bit value will be 0 always. + +**Return value** + +int64 : the bit value stored at offset. + +**Examples** + +``` +ledis> BSETBIT flag 1024 1 +(integer) 0 +ledis> BGETBIT flag 0 +(integer) 0 +ledis> BGETBIT flag 1024 +(integer) 1 +ledis> BGETBIT flag 65535 +(integer) 0 +``` + + +### BSETBIT key offset value + +Sets or clear the bit at `offset` in the binary data sotred at `key`. +The bit is either set or cleared depending on `value`, which can be either `0` or `1`. +The *offset* argument is required to be qual to 0, and smaller than +2^23 (this means bitmap limits to 8MB). + +**Return value** + +int64 : the original bit value stored at offset. + +**Examples** + +``` +ledis> BSETBIT flag 0 1 +(integer) 0 +ledis> BSETBIT flag 0 0 +(integer) 1 +ledis> BGETBIT flag 0 99 +ERR invalid command param +``` + +### BMSETBIT key offset value [offset value ...] +Sets the given *offset* to their respective values. + +**Return value** + +int64 : The number of input *offset* + +**Examples** + +``` +ledis> BMSETBIT flag 0 1 1 1 2 0 3 1 +(integer) 4 +ledis> BCOUNT flag +(integer) 3 +``` + + +### BOPT operation destkey key [key ...] +Perform a bitwise operation between multiple keys (containing string values) and store the result in the destination key. + +**Return value** + +Int64: +The size of the string stored in the destination key, that is equal to the size of the longest input string. +**Examples** + +``` +ledis> BMSETBIT a 0 1 2 1 +(integer) 2 +ledis> BMSETBIT b 1 1 +(integer) 1 +ledis> BOPT AND res a b +(integer) 3 +ledis> BCOUNT res +(integer) 0 +ledis> BOPT OR res2 a b +(integer) 3 +ledis> BCOUNT res2 +(integer) 3 +ledis> BOPT XOR res3 a b +(integer) 3 +ledis> BCOUNT res3 +(integer) 3 +``` + +### BCOUNT key [start end] + +Count the number of set bits in a bitmap. + +**Return value** + +int64 : The number of bits set to 1. + +**Examples** + +``` +ledis> BMSETBIT flag 0 1 5 1 6 1 +(integer) 3 +ledis> BGET flag +a +ledis> BCOUNT flag +(integer) 3 +ledis> BCOUNT flag 0 0s +(integer) 1 +ledis> BCOUNT flag 0 4 +(integer) 1 +ledis> BCOUNT flag 0 5 +(integer) 2 +ledis> BCOUNT flag 5 6 +(integer) 2 +``` + + +### BEXPIRE key seconds + +(refer to [EXPIRE](#expire-key-seconds) api for other types) + + +### BEXPIREAT key timestamp + +(refer to [EXPIREAT](#expireat-key-timestamp) api for other types) + + +### BTTL key + +(refer to [TTL](#ttl-key) api for other types) + + +### BPERSIST key + +(refer to [PERSIST](#persist-key) api for other types) + + ## Replication ### SLAVEOF host port diff --git a/ledis/t_bit.go b/ledis/t_bit.go index 4111df8..ab14a7d 100644 --- a/ledis/t_bit.go +++ b/ledis/t_bit.go @@ -371,6 +371,65 @@ func (db *DB) bExpireAt(key []byte, when int64) (int64, error) { return 1, nil } +func (db *DB) bCountByte(val byte, soff uint32, eoff uint32) int32 { + if soff > eoff { + soff, eoff = eoff, soff + } + + mask := uint8(0) + if soff > 0 { + mask |= fillBits[soff-1] + } + if eoff < 7 { + mask |= (fillBits[7] ^ fillBits[eoff]) + } + mask = fillBits[7] ^ mask + + return bitsInByte[val&mask] +} + +func (db *DB) bCountSeg(key []byte, seq uint32, soff uint32, eoff uint32) (cnt int32, err error) { + if soff >= segBitSize || soff < 0 || + eoff >= segBitSize || eoff < 0 { + return + } + + var segment []byte + if _, segment, err = db.bGetSegment(key, seq); err != nil { + return + } + + if segment == nil { + return + } + + if soff > eoff { + soff, eoff = eoff, soff + } + + headIdx := int(soff >> 3) + endIdx := int(eoff >> 3) + sByteOff := soff - ((soff >> 3) << 3) + eByteOff := eoff - ((eoff >> 3) << 3) + + if headIdx == endIdx { + cnt = db.bCountByte(segment[headIdx], sByteOff, eByteOff) + } else { + cnt = db.bCountByte(segment[headIdx], sByteOff, 7) + + db.bCountByte(segment[endIdx], 0, eByteOff) + } + + // sum up following bytes + for idx, end := headIdx+1, endIdx-1; idx <= end; idx += 1 { + cnt += bitsInByte[segment[idx]] + if idx == end { + break + } + } + + return +} + func (db *DB) BGet(key []byte) (data []byte, err error) { if err = checkKeySize(key); err != nil { return @@ -561,25 +620,53 @@ func (db *DB) BGetBit(key []byte, offset int32) (uint8, error) { // } func (db *DB) BCount(key []byte, start int32, end int32) (cnt int32, err error) { - var sseq uint32 - if sseq, _, err = db.bParseOffset(key, start); err != nil { + var sseq, soff uint32 + if sseq, soff, err = db.bParseOffset(key, start); err != nil { return } - var eseq uint32 - if eseq, _, err = db.bParseOffset(key, end); err != nil { + var eseq, eoff uint32 + if eseq, eoff, err = db.bParseOffset(key, end); err != nil { return } + if sseq > eseq || (sseq == eseq && soff > eoff) { + sseq, eseq = eseq, sseq + soff, eoff = eoff, soff + } + + var segCnt int32 + if eseq == sseq { + if segCnt, err = db.bCountSeg(key, sseq, soff, eoff); err != nil { + return 0, err + } + + cnt = segCnt + + } else { + if segCnt, err = db.bCountSeg(key, sseq, soff, segBitSize-1); err != nil { + return 0, err + } else { + cnt += segCnt + } + + if segCnt, err = db.bCountSeg(key, eseq, 0, eoff); err != nil { + return 0, err + } else { + cnt += segCnt + } + } + + // middle segs var segment []byte skey := db.bEncodeBinKey(key, sseq) ekey := db.bEncodeBinKey(key, eseq) - it := db.db.RangeIterator(skey, ekey, leveldb.RangeClose) + it := db.db.RangeIterator(skey, ekey, leveldb.RangeOpen) for ; it.Valid(); it.Next() { segment = it.Value() - for _, bit := range segment { - cnt += bitsInByte[bit] + for _, bt := range segment { + cnt += bitsInByte[bt] } } it.Close() @@ -604,7 +691,7 @@ func (db *DB) BTail(key []byte) (int32, error) { func (db *DB) BOperation(op uint8, dstkey []byte, srckeys ...[]byte) (blen int32, err error) { // blen - - // the size of the string stored in the destination key, + // the total bit size of data stored in destination key, // that is equal to the size of the longest input string. var exeOp func([]byte, []byte, *[]byte) @@ -759,7 +846,8 @@ func (db *DB) BOperation(op uint8, dstkey []byte, srckeys ...[]byte) (blen int32 err = t.Commit() if err == nil { - blen = int32(maxDstSeq<