From 8be1cf8b540e899c66ea88567b9c24cf10afb127 Mon Sep 17 00:00:00 2001 From: holys Date: Sun, 13 Jul 2014 00:31:30 +0800 Subject: [PATCH 01/27] rename Bin to Bit --- server/cmd_bit_test.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/server/cmd_bit_test.go b/server/cmd_bit_test.go index e49b84c..902ec10 100644 --- a/server/cmd_bit_test.go +++ b/server/cmd_bit_test.go @@ -5,14 +5,14 @@ import ( "testing" ) -func TestBin(t *testing.T) { - testBinGetSet(t) - testBinMset(t) - testBinCount(t) - testBinOpt(t) +func TestBit(t *testing.T) { + testBitGetSet(t) + testBitMset(t) + testBitCount(t) + testBitOpt(t) } -func testBinGetSet(t *testing.T) { +func testBitGetSet(t *testing.T) { c := getTestConn() defer c.Close() @@ -66,7 +66,7 @@ func testBinGetSet(t *testing.T) { } } -func testBinMset(t *testing.T) { +func testBitMset(t *testing.T) { c := getTestConn() defer c.Close() @@ -103,7 +103,7 @@ func testBinMset(t *testing.T) { } } -func testBinCount(t *testing.T) { +func testBitCount(t *testing.T) { c := getTestConn() defer c.Close() @@ -121,7 +121,7 @@ func testBinCount(t *testing.T) { } } -func testBinOpt(t *testing.T) { +func testBitOpt(t *testing.T) { c := getTestConn() defer c.Close() From 9d8e153c98aca3998817b0cb697492b74050c910 Mon Sep 17 00:00:00 2001 From: siddontang Date: Sun, 13 Jul 2014 19:56:46 +0800 Subject: [PATCH 02/27] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6d7f61f..98163a9 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Ledisdb is a high performance NoSQL like Redis based on LevelDB written by go. I + Supports expiration and ttl. + Redis clients, like redis-cli, are supported directly. + Multi client API supports, including Golang, Python, Lua(Openresty). -+ Easily to embed in Golang application. ++ Easy to embed in Golang application. + Replication to guarantee data safe. + Supplies tools to load, dump, repair database. From e2e7db795a67bb416fa50cd3765a587054774b30 Mon Sep 17 00:00:00 2001 From: holys Date: Mon, 14 Jul 2014 22:48:33 +0800 Subject: [PATCH 03/27] add bit commands support; remove score_cast_func, we only support int64 --- client/ledis-py/ledis/client.py | 119 +++++++++++++++++++------ client/ledis-py/tests/test_cmd_bit.py | 105 ++++++++++++++++++++++ client/ledis-py/tests/test_cmd_hash.py | 2 +- 3 files changed, 198 insertions(+), 28 deletions(-) create mode 100644 client/ledis-py/tests/test_cmd_bit.py diff --git a/client/ledis-py/ledis/client.py b/client/ledis-py/ledis/client.py index a855e84..7d64457 100644 --- a/client/ledis-py/ledis/client.py +++ b/client/ledis-py/ledis/client.py @@ -1,7 +1,7 @@ from __future__ import with_statement import datetime import time as mod_time -from ledis._compat import (b, izip, imap, iteritems, iterkeys, itervalues, +from ledis._compat import (b, izip, imap, iteritems, basestring, long, nativestr, urlparse, bytes) from ledis.connection import ConnectionPool, UnixDomainSocketConnection from ledis.exceptions import ( @@ -53,9 +53,8 @@ def zset_score_pairs(response, **options): """ if not response or not options['withscores']: return response - score_cast_func = options.get('score_cast_func', int) it = iter(response) - return list(izip(it, imap(score_cast_func, it))) + return list(izip(it, imap(int, it))) def int_or_none(response): @@ -77,13 +76,14 @@ class Ledis(object): RESPONSE_CALLBACKS = dict_merge( string_keys_to_dict( 'EXISTS EXPIRE EXPIREAT HEXISTS HMSET SETNX ' - 'PERSIST HPERSIST LPERSIST ZPERSIST', + 'PERSIST HPERSIST LPERSIST ZPERSIST BEXPIRE ' + 'BEXPIREAT BPERSIST BDELETE', bool ), string_keys_to_dict( 'DECRBY DEL HDEL HLEN INCRBY LLEN ' 'ZADD ZCARD ZREM ZREMRANGEBYRANK ZREMRANGEBYSCORE' - 'LMCLEAR HMCLEAR ZMCLEAR', + 'LMCLEAR HMCLEAR ZMCLEAR BCOUNT BGETBIT BSETBIT BOPT BMSETBIT', int ), string_keys_to_dict( @@ -102,7 +102,9 @@ class Ledis(object): { 'HGETALL': lambda r: r and pairs_to_dict(r) or {}, 'PING': lambda r: nativestr(r) == 'PONG', - 'SET': lambda r: r and nativestr(r) == 'OK', } + 'SET': lambda r: r and nativestr(r) == 'OK', + }, + ) @classmethod @@ -187,7 +189,6 @@ class Ledis(object): return self.response_callbacks[command_name](response, **options) return response - #### SERVER INFORMATION #### def echo(self, value): "Echo the string back from the server" @@ -205,7 +206,6 @@ class Ledis(object): db = 0 return self.execute_command('SELECT', db) - #### BASIC KEY COMMANDS #### def decr(self, name, amount=1): """ @@ -327,7 +327,6 @@ class Ledis(object): "Removes an expiration on name" return self.execute_command('PERSIST', name) - #### LIST COMMANDS #### def lindex(self, name, index): """ @@ -433,8 +432,8 @@ class Ledis(object): def zcount(self, name, min, max): """ - Return the number of elements in the sorted set at key ``name`` with a score - between ``min`` and ``max``. + Return the number of elements in the sorted set at key ``name`` with a score + between ``min`` and ``max``. The min and max arguments have the same semantic as described for ZRANGEBYSCORE. """ return self.execute_command('ZCOUNT', name, min, max) @@ -454,17 +453,15 @@ class Ledis(object): ``withscores`` indicates to return the scores along with the values. The return type is a list of (value, score) pairs - - ``score_cast_func`` a callable used to cast the score return value """ if desc: - return self.zrevrange(name, start, end, withscores, - score_cast_func) + return self.zrevrange(name, start, end, withscores) + pieces = ['ZRANGE', name, start, end] if withscores: pieces.append('withscores') options = { - 'withscores': withscores, 'score_cast_func': int} + 'withscores': withscores} return self.execute_command(*pieces, **options) def zrangebyscore(self, name, min, max, start=None, num=None, @@ -478,8 +475,6 @@ class Ledis(object): ``withscores`` indicates to return the scores along with the values. The return type is a list of (value, score) pairs - - `score_cast_func`` a callable used to cast the score return value """ if (start is not None and num is None) or \ (num is not None and start is None): @@ -490,7 +485,7 @@ class Ledis(object): if withscores: pieces.append('withscores') options = { - 'withscores': withscores, 'score_cast_func': int} + 'withscores': withscores} return self.execute_command(*pieces, **options) def zrank(self, name, value): @@ -529,14 +524,11 @@ class Ledis(object): ``withscores`` indicates to return the scores along with the values The return type is a list of (value, score) pairs - - ``score_cast_func`` a callable used to cast the score return value """ pieces = ['ZREVRANGE', name, start, num] if withscores: pieces.append('withscores') - options = { - 'withscores': withscores, 'score_cast_func': int} + options = {'withscores': withscores} return self.execute_command(*pieces, **options) def zrevrangebyscore(self, name, min, max, start=None, num=None, @@ -550,8 +542,6 @@ class Ledis(object): ``withscores`` indicates to return the scores along with the values. The return type is a list of (value, score) pairs - - ``score_cast_func`` a callable used to cast the score return value """ if (start is not None and num is None) or \ (num is not None and start is None): @@ -561,8 +551,7 @@ class Ledis(object): pieces.extend(['LIMIT', start, num]) if withscores: pieces.append('withscores') - options = { - 'withscores': withscores, 'score_cast_func': int} + options = {'withscores': withscores} return self.execute_command(*pieces, **options) def zrevrank(self, name, value): @@ -703,3 +692,79 @@ class Ledis(object): "Removes an expiration on name" return self.execute_command('HPERSIST', name) + + + ### BIT COMMANDS + def bget(self, name): + "" + return self.execute_command("BGET", name) + + def bdelete(self, name): + "" + return self.execute_command("BDELETE", name) + + def bsetbit(self, name, offset, value): + "" + value = value and 1 or 0 + return self.execute_command("BSETBIT", name, offset, value) + + def bgetbit(self, name, offset): + "" + return self.execute_command("BGETBIT", name, offset) + + def bmsetbit(self, name, *args): + """ + Set any number of offset, value pairs to the key ``name``. Pairs can be + specified in the following way: + + offset1, value1, offset2, value2, ... + """ + pieces = [] + if args: + if len(args) % 2 != 0: + raise LedisError("BMSETBIT requires an equal number of " + "offset and value") + pieces.extend(args) + return self.execute_command("BMSETBIT", name, *pieces) + + def bcount(self, key, start=None, end=None): + "" + params = [key] + if start is not None and end is not None: + params.append(start) + params.append(end) + elif (start is not None and end is None) or \ + (start is None and end is not None): + raise LedisError("Both start and end must be specified") + return self.execute_command("BCOUNT", *params) + + def bopt(self, operation, dest, *keys): + """ + Perform a bitwise operation using ``operation`` between ``keys`` and + store the result in ``dest``. + ``operation`` is one of `and`, `or`, `xor`, `not`. + """ + return self.execute_command('BOPT', operation, dest, *keys) + + def bexpire(self, name, time): + "Set timeout on key ``name`` with ``time``" + if isinstance(time, datetime.timedelta): + time = time.seconds + time.days * 24 * 3600 + return self.execute_command('BEXPIRE', name, time) + + def bexpireat(self, name, when): + """ + Set an expire flag on key name for time seconds. time can be represented by + an integer or a Python timedelta object. + """ + if isinstance(when, datetime.datetime): + when = int(mod_time.mktime(when.timetuple())) + return self.execute_command('BEXPIREAT', name, when) + + def bttl(self, name): + "Returns the number of seconds until the key name will expire" + return self.execute_command('BTTL', name) + + def bpersist(self, name): + "Removes an expiration on name" + return self.execute_command('BPERSIST', name) \ No newline at end of file diff --git a/client/ledis-py/tests/test_cmd_bit.py b/client/ledis-py/tests/test_cmd_bit.py new file mode 100644 index 0000000..67b5bf9 --- /dev/null +++ b/client/ledis-py/tests/test_cmd_bit.py @@ -0,0 +1,105 @@ +# coding: utf-8 +# Test Cases for bit commands + +import unittest +import sys +import datetime, time +sys.path.append('..') + +import ledis +from ledis._compat import b +from ledis import ResponseError + + +l = ledis.Ledis(port=6380) + + +def current_time(): + return datetime.datetime.now() + + +class TestCmdBit(unittest.TestCase): + def setUp(self): + pass + + def tearDown(self): + l.bdelete('a') + l.bdelete('non_exists_key') + + def test_bget(self): + "bget is the same as get in K/V commands" + l.bmsetbit('a', 0, 1, 1, 1, 2, 1, 3, 1, 4, 1, 5, 1, 6, 1, 7, 0) + assert l.bget('a') == b('\x7f') + + def test_bdelete(self): + l.bsetbit('a', 0, 1) + assert l.bdelete('a') + assert not l.bdelete('non_exists_key') + + def test_get_set_bit(self): + assert not l.bgetbit('a', 5) + assert not l.bsetbit('a', 5, True) + assert l.bgetbit('a', 5) + + assert not l.bsetbit('a', 4, False) + assert not l.bgetbit('a', 4) + + assert not l.bsetbit('a', 4, True) + assert l.bgetbit('a', 4) + + assert l.bsetbit('a', 5, True) + assert l.bgetbit('a', 5) + + def test_bmsetbit(self): + assert l.bmsetbit('a', 0, 1, 2, 1, 3, 1) == 3 + + def test_bcount(self): + l.bsetbit('a', 5, 1) + assert l.bcount('a') == 1 + l.bsetbit('a', 6, 1) + assert l.bcount('a') == 2 + l.bsetbit('a', 5, 0) + assert l.bcount('a') == 1 + l.bmsetbit('a', 10, 1, 20, 1, 30, 1, 40, 1) + assert l.bcount('a') == 5 + #TODO test bcount with start/end option + + def test_bopt_not_empty_string(self): + l.bopt('not', 'r', 'a') + assert l.bget('r') is None + + def test_bopt_not(self): + #TODO + pass + + def test_bexpire(self): + assert not l.bexpire('a', 100) + l.bsetbit('a', 1, True) + assert l.bexpire('a', 100) + assert 0 < l.bttl('a') <= 100 + assert l.bpersist('a') + assert l.bttl('a') == -1 + + def test_bexpireat_datetime(self): + expire_at = current_time() + datetime.timedelta(minutes=1) + l.bsetbit('a', 1, True) + assert l.bexpireat('a', expire_at) + assert 0 < l.bttl('a') <= 61 + + def test_bexpireat_unixtime(self): + expire_at = current_time() + datetime.timedelta(minutes=1) + l.bsetbit('a', 1, True) + expire_at_seconds = int(time.mktime(expire_at.timetuple())) + assert l.bexpireat('a', expire_at_seconds) + assert 0 < l.bttl('a') <= 61 + + def test_bexpireat_no_key(self): + expire_at = current_time() + datetime.timedelta(minutes=1) + assert not l.bexpireat('a', expire_at) + + def test_bttl_and_bpersist(self): + l.bsetbit('a', 1, True) + l.bexpire('a', 100) + assert 0 < l.bttl('a') <= 100 + assert l.bpersist('a') + assert l.bttl('a') == -1 diff --git a/client/ledis-py/tests/test_cmd_hash.py b/client/ledis-py/tests/test_cmd_hash.py index 599eb77..d0fe5ab 100644 --- a/client/ledis-py/tests/test_cmd_hash.py +++ b/client/ledis-py/tests/test_cmd_hash.py @@ -122,7 +122,7 @@ class TestCmdHash(unittest.TestCase): assert l.hexpireat('a', expire_at_seconds) assert 0 < l.httl('a') <= 61 - def test_zexpireat_no_key(self): + def test_hexpireat_no_key(self): expire_at = current_time() + datetime.timedelta(minutes=1) assert not l.hexpireat('a', expire_at) From ce64e7364c74d84d0b6844a4bbf53c0fde7d9b2e Mon Sep 17 00:00:00 2001 From: silentsai Date: Tue, 15 Jul 2014 18:15:03 +0800 Subject: [PATCH 04/27] add some api describtion for the bit type --- doc/commands.md | 110 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 109 insertions(+), 1 deletion(-) diff --git a/doc/commands.md b/doc/commands.md index 90d7199..007eea0 100644 --- a/doc/commands.md +++ b/doc/commands.md @@ -70,7 +70,7 @@ Table of Contents - [ZREMRANGEBYRANK key start stop](#zremrangebyrank-key-start-stop) - [ZREMRANGEBYSCORE key min max](#zremrangebyscore-key-min-max) - [ZREVRANGE key start stop [WITHSCORES]](#zrevrange-key-start-stop-withscores) - - [ZREVRANGEBYSCORE key max min [WITHSCORES] [LIMIT offset count]](#zrevrangebyscore--key-max-min-withscores-limit-offset-count) + - [ZREVRANGEBYSCORE key max min [WITHSCORES] [LIMIT offset count]](#zrevrangebyscore-key-max-min-withscores-limit-offset-count) - [ZREVRANK key member](#zrevrank-key-member) - [ZSCORE key member](#zscore-key-member) - [ZCLEAR key](#zclear-key) @@ -79,6 +79,16 @@ Table of Contents - [ZEXPIREAT key timestamp](#zexpireat-key-timestamp) - [ZTTL key](#zttl-key) - [ZPERSIST key](#zpersist-key) +- [Bitmap](#bitmap) + - [BSETBIT key offset value](#bsetbit-key-offset-value) + - [BGETBIT key offset](#bsetbit-key-offset) + - [BGET key](#bget-key) + - [BCOUNT key [start, end]](#bcount-key-start-end) + - [BEXPIRE key seconds](#bexpire-key-seconds) + - [BEXPIREAT key timestamp](#bexpireat-key-timestamp) + - [BTTL key](#bttl-key) + - [BPERSIST key](#bpersist-key) + - [Replication](#replication) - [SLAVEOF host port](#slaveof-host-port) - [FULLSYNC](#fullsync) @@ -1617,6 +1627,104 @@ ledis> ZTTL mset ``` +## bitmap + +### BSETBIT key offset value + +Sets or clear the bit at `offset` in the binary data sotred at `key`. +The bit is either set or cleared depending on `value`, which can be either `0` or `1`. +The *offset* argument is required to be qual to 0, and smaller than +2^23 (this means bitmap limits to 8MB). + +**Return value** + +int64 : the original bit value stored at offset. + +**Examples** + +``` +ledis> BSETBIT flag 0 1 +(integer) 0 +ledis> BSETBIT flag 0 0 +(integer) 1 +ledis> BGETBIT flag 0 99 +ERR invalid command param +``` + + +### BGETBIT key offset + +Returns the bit value at `offset` in the string value stored at `key`. +When *offset* beyond the data length, ot the target data is not exist, the bit value will be 0 always. + +**Return value** + +int64 : the bit value stored at offset. + +**Examples** + +``` +ledis> BSETBIT flag 1024 1 +(integer) 0 +ledis> BGETBIT flag 0 +(integer) 0 +ledis> BGETBIT flag 1024 +(integer) 1 +ledis> BGETBIT flag 65535 +(integer) 0 +``` + + +### BCOUNT key [start end] + +Count the number of set bits in a bitmap. + +**Return value** + +int64 : The number of bits set to 1. + +**Examples** + +``` + +``` + + +### BGET key + +Returns the whole binary data stored at `key`. + +**Return value** + +bulk: the raw value of key, or nil when key does not exist. + +**Examples** + +``` + +``` + + +### BEXPIRE key seconds + +(refer to `expire` api for other types) + + +### BEXPIREAT key timestamp + +(refer to `expireat` api for other types) + + +### BTTL key + +(refer to `ttl` api for other types) + + +### PERSIST key + +(refer to `persist` api for other types) + + ## Replication ### SLAVEOF host port From 1131e637b03b4da7cb26b1674aa78b4cdbf6866a Mon Sep 17 00:00:00 2001 From: holys Date: Tue, 15 Jul 2014 23:44:56 +0800 Subject: [PATCH 05/27] add bitmap commands support for openresty(lua) --- client/openresty/ledis.lua | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/client/openresty/ledis.lua b/client/openresty/ledis.lua index fa149f6..06d9161 100644 --- a/client/openresty/ledis.lua +++ b/client/openresty/ledis.lua @@ -100,6 +100,19 @@ local commands = { "zttl", "zpersist", + --[[bit]] + "bget", + "bdelete", + "bsetbit", + "bgetbit", + "bmsetbit", + "bcount", + "bopt", + "bexpire", + "bexpireat", + "bttl", + "bpersist", + --[[server]] "ping", "echo", From dfee4df618e555c7e31ee5de720b231dfe2ce64d Mon Sep 17 00:00:00 2001 From: holys Date: Wed, 16 Jul 2014 00:10:50 +0800 Subject: [PATCH 06/27] UPPER case of Commands --- doc/commands.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/doc/commands.md b/doc/commands.md index 007eea0..72b24fc 100644 --- a/doc/commands.md +++ b/doc/commands.md @@ -1148,7 +1148,7 @@ ledis> ZRANGE myset 0 -1 WITHSCORES 6) "2" 7) "three" 8) "3" -ledis> zcard myset +ledis> ZCARD myset (integer) 4 ``` @@ -1178,9 +1178,9 @@ ledis> ZRANGE myset 0 -1 WITHSCORES 6) "2" 7) "three" 8) "3" -ledis> zcount myset -inf +inf +ledis> ZCOUNT myset -inf +inf (integer) 4 -ledis> zcount myset (1 3 +ledis> ZCOUNT myset (1 3 (integer) 2 ``` @@ -1454,13 +1454,13 @@ Use ZRANK to get the rank of an element with the scores ordered from low to high **Examples** ``` -127.0.0.1:6380> zadd myset 1 one +127.0.0.1:6380> ZADD myset 1 one (integer) 1 -127.0.0.1:6380> zadd myset 2 two +127.0.0.1:6380> ZADD myset 2 two (integer) 1 -127.0.0.1:6380> zrevrank myset one +127.0.0.1:6380> ZREVRANK myset one (integer) 1 -127.0.0.1:6380> zrevrank myset three +127.0.0.1:6380> ZREVRANK myset three (nil) ``` @@ -1591,13 +1591,13 @@ int64: TTL in seconds **Examples** ``` -ledis> zadd myset 1 'one' +ledis> ZADD myset 1 'one' (integer) 1 -ledis> zexpire myset 100 +ledis> ZEXPIRE myset 100 (integer) 1 -ledis> zttl myset +ledis> ZTTL myset (integer) 97 -ledis> zttl myset2 +ledis> ZTTL myset2 (integer) -1 ``` @@ -1627,7 +1627,7 @@ ledis> ZTTL mset ``` -## bitmap +## Bitmap ### BSETBIT key offset value @@ -1707,22 +1707,22 @@ bulk: the raw value of key, or nil when key does not exist. ### BEXPIRE key seconds -(refer to `expire` api for other types) +(refer to [EXPIRE](#expire-key-seconds) api for other types) ### BEXPIREAT key timestamp -(refer to `expireat` api for other types) +(refer to [EXPIREAT](#expireat-key-timestamp) api for other types) ### BTTL key -(refer to `ttl` api for other types) +(refer to [TTL](#ttl-key) api for other types) ### PERSIST key -(refer to `persist` api for other types) +(refer to [PERSIST](#persist-key) api for other types) ## Replication From 4f203e9b5d9db4fcd38a88a1784b41e5fcc18393 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 16 Jul 2014 11:29:44 +0800 Subject: [PATCH 07/27] parseint bitsize is 8 --- ledis/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ledis/util.go b/ledis/util.go index d5f24d1..770bca1 100644 --- a/ledis/util.go +++ b/ledis/util.go @@ -79,7 +79,7 @@ func StrInt8(v []byte, err error) (int8, error) { } else if v == nil { return 0, nil } else { - res, err := strconv.ParseInt(String(v), 10, 32) + res, err := strconv.ParseInt(String(v), 10, 8) return int8(res), err } } From 8f0b86b5725c3b5db9fd451c22b9961242df5de6 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 16 Jul 2014 11:29:57 +0800 Subject: [PATCH 08/27] update doc --- doc/commands.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/commands.md b/doc/commands.md index 72b24fc..c9045e1 100644 --- a/doc/commands.md +++ b/doc/commands.md @@ -1720,7 +1720,7 @@ bulk: the raw value of key, or nil when key does not exist. (refer to [TTL](#ttl-key) api for other types) -### PERSIST key +### BPERSIST key (refer to [PERSIST](#persist-key) api for other types) From 3da485602b5f07e2f0942bd7dcbdcfb652c7289a Mon Sep 17 00:00:00 2001 From: silentsai Date: Wed, 16 Jul 2014 11:48:02 +0800 Subject: [PATCH 09/27] problem of incorrect bit count within specified pos range is solved; corret itoperation return value --- ledis/t_bit.go | 106 ++++++++++++++++++++++++++++++++++++++++---- ledis/t_bit_test.go | 63 ++++++++++++++++++++++++++ 2 files changed, 160 insertions(+), 9 deletions(-) diff --git a/ledis/t_bit.go b/ledis/t_bit.go index 4111df8..ab14a7d 100644 --- a/ledis/t_bit.go +++ b/ledis/t_bit.go @@ -371,6 +371,65 @@ func (db *DB) bExpireAt(key []byte, when int64) (int64, error) { return 1, nil } +func (db *DB) bCountByte(val byte, soff uint32, eoff uint32) int32 { + if soff > eoff { + soff, eoff = eoff, soff + } + + mask := uint8(0) + if soff > 0 { + mask |= fillBits[soff-1] + } + if eoff < 7 { + mask |= (fillBits[7] ^ fillBits[eoff]) + } + mask = fillBits[7] ^ mask + + return bitsInByte[val&mask] +} + +func (db *DB) bCountSeg(key []byte, seq uint32, soff uint32, eoff uint32) (cnt int32, err error) { + if soff >= segBitSize || soff < 0 || + eoff >= segBitSize || eoff < 0 { + return + } + + var segment []byte + if _, segment, err = db.bGetSegment(key, seq); err != nil { + return + } + + if segment == nil { + return + } + + if soff > eoff { + soff, eoff = eoff, soff + } + + headIdx := int(soff >> 3) + endIdx := int(eoff >> 3) + sByteOff := soff - ((soff >> 3) << 3) + eByteOff := eoff - ((eoff >> 3) << 3) + + if headIdx == endIdx { + cnt = db.bCountByte(segment[headIdx], sByteOff, eByteOff) + } else { + cnt = db.bCountByte(segment[headIdx], sByteOff, 7) + + db.bCountByte(segment[endIdx], 0, eByteOff) + } + + // sum up following bytes + for idx, end := headIdx+1, endIdx-1; idx <= end; idx += 1 { + cnt += bitsInByte[segment[idx]] + if idx == end { + break + } + } + + return +} + func (db *DB) BGet(key []byte) (data []byte, err error) { if err = checkKeySize(key); err != nil { return @@ -561,25 +620,53 @@ func (db *DB) BGetBit(key []byte, offset int32) (uint8, error) { // } func (db *DB) BCount(key []byte, start int32, end int32) (cnt int32, err error) { - var sseq uint32 - if sseq, _, err = db.bParseOffset(key, start); err != nil { + var sseq, soff uint32 + if sseq, soff, err = db.bParseOffset(key, start); err != nil { return } - var eseq uint32 - if eseq, _, err = db.bParseOffset(key, end); err != nil { + var eseq, eoff uint32 + if eseq, eoff, err = db.bParseOffset(key, end); err != nil { return } + if sseq > eseq || (sseq == eseq && soff > eoff) { + sseq, eseq = eseq, sseq + soff, eoff = eoff, soff + } + + var segCnt int32 + if eseq == sseq { + if segCnt, err = db.bCountSeg(key, sseq, soff, eoff); err != nil { + return 0, err + } + + cnt = segCnt + + } else { + if segCnt, err = db.bCountSeg(key, sseq, soff, segBitSize-1); err != nil { + return 0, err + } else { + cnt += segCnt + } + + if segCnt, err = db.bCountSeg(key, eseq, 0, eoff); err != nil { + return 0, err + } else { + cnt += segCnt + } + } + + // middle segs var segment []byte skey := db.bEncodeBinKey(key, sseq) ekey := db.bEncodeBinKey(key, eseq) - it := db.db.RangeIterator(skey, ekey, leveldb.RangeClose) + it := db.db.RangeIterator(skey, ekey, leveldb.RangeOpen) for ; it.Valid(); it.Next() { segment = it.Value() - for _, bit := range segment { - cnt += bitsInByte[bit] + for _, bt := range segment { + cnt += bitsInByte[bt] } } it.Close() @@ -604,7 +691,7 @@ func (db *DB) BTail(key []byte) (int32, error) { func (db *DB) BOperation(op uint8, dstkey []byte, srckeys ...[]byte) (blen int32, err error) { // blen - - // the size of the string stored in the destination key, + // the total bit size of data stored in destination key, // that is equal to the size of the longest input string. var exeOp func([]byte, []byte, *[]byte) @@ -759,7 +846,8 @@ func (db *DB) BOperation(op uint8, dstkey []byte, srckeys ...[]byte) (blen int32 err = t.Commit() if err == nil { - blen = int32(maxDstSeq< Date: Wed, 16 Jul 2014 14:34:08 +0800 Subject: [PATCH 10/27] update read me --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 98163a9..f853e9b 100644 --- a/README.md +++ b/README.md @@ -25,13 +25,13 @@ Ledisdb is a high performance NoSQL like Redis based on LevelDB written by go. I + Install leveldb and snappy, if you have installed, skip. - I supply a simple shell to install leveldb and snappy, you can use: + LedisDB supplies a simple shell to install leveldb and snappy: sh build_leveldb.sh It will default install leveldb at /usr/local/leveldb and snappy at /usr/local/snappy -+ Change LEVELDB_DIR and SNAPPY_DIR to real install path in dev.sh. ++ Set LEVELDB_DIR and SNAPPY_DIR to the actual install path in dev.sh. + Then: @@ -65,7 +65,7 @@ Ledisdb is a high performance NoSQL like Redis based on LevelDB written by go. I ## Replication -set slaveof in config or dynamiclly +Set slaveof in config or dynamiclly ledis-cli -p 6381 @@ -86,7 +86,7 @@ See benchmark.md for more. ## Commands -Some server commands explaintions are [here](https://github.com/siddontang/ledisdb/wiki/Commands), others will add continuate. +[Server Commands](https://github.com/siddontang/ledisdb/wiki/Commands). ## Thanks @@ -94,6 +94,8 @@ Gmail: cenqichao@gmail.com Gmail: chendahui007@gmail.com +Gamil: tiaotiaoyly@gmail.com + ## Feedback Gmail: siddontang@gmail.com \ No newline at end of file From 5eed29a4db9c48448f10caf623039abcc7c75293 Mon Sep 17 00:00:00 2001 From: silentsai Date: Wed, 16 Jul 2014 15:26:54 +0800 Subject: [PATCH 11/27] add some test for the bit type --- ledis/t_bit_test.go | 76 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/ledis/t_bit_test.go b/ledis/t_bit_test.go index 1b7ac27..a356539 100644 --- a/ledis/t_bit_test.go +++ b/ledis/t_bit_test.go @@ -135,7 +135,7 @@ func testCount(t *testing.T) { db := getTestDB() db.FlushAll() - dstKey := []byte("test_bin_count") + key := []byte("test_bin_count") if ori, _ := db.BSetBit(key, 0, 1); ori != 0 { t.Error(ori) @@ -191,6 +191,40 @@ func testCount(t *testing.T) { t.Error(sum) } + key = []byte("test_bin_count_2") + + db.BSetBit(key, 1, 1) + db.BSetBit(key, 2, 1) + db.BSetBit(key, 4, 1) + db.BSetBit(key, 6, 1) + + if sum, _ := db.BCount(key, 0, -1); sum != 4 { + t.Error(sum) + } + + if sum, _ := db.BCount(key, 1, 1); sum != 1 { + t.Error(sum) + } + + if sum, _ := db.BCount(key, 0, 7); sum != 4 { + t.Error(sum) + } + + if ori, _ := db.BSetBit(key, 8, 1); ori != 0 { + t.Error(ori) + } + + if ori, _ := db.BSetBit(key, 11, 1); ori != 0 { + t.Error(ori) + } + + if sum, _ := db.BCount(key, 0, -1); sum != 6 { + t.Error(sum) + } + + if sum, _ := db.BCount(key, 0, 16); sum != 6 { + t.Error(sum) + } } func testOpAndOr(t *testing.T) { @@ -319,6 +353,30 @@ func testOpAndOr(t *testing.T) { } +func testOpAnd(t *testing.T) { + db := getTestDB() + db.FlushAll() + + dstKey := []byte("test_bin_or") + + k0 := []byte("op_or_0") + k1 := []byte("op_or_01") + srcKeys := [][]byte{k0, k1} + + db.BSetBit(k0, 0, 1) + db.BSetBit(k0, 2, 1) + + db.BSetBit(k1, 1, 1) + + if blen, _ := db.BOperation(OPand, dstKey, srcKeys...); blen != 3 { + t.Fatal(blen) + } + + if cnt, _ := db.BCount(dstKey, 0, -1); cnt != 1 { + t.Fatal(1) + } +} + func testOpXor(t *testing.T) { db := getTestDB() db.FlushAll() @@ -393,6 +451,22 @@ func testOpNot(t *testing.T) { if cmpBytes(data, stdData) { t.Fatal(false) } + + k1 := []byte("op_not_2") + srcKeys = [][]byte{k1} + + db.BSetBit(k1, 0, 1) + db.BSetBit(k1, 2, 1) + db.BSetBit(k1, 4, 1) + db.BSetBit(k1, 6, 1) + + if blen, _ := db.BOperation(OPnot, dstKey, srcKeys...); blen != 7 { + t.Fatal(blen) + } + + if cnt, _ := db.BCount(dstKey, 0, -1); cnt != 3 { + t.Fatal(cnt) + } } func testMSetBit(t *testing.T) { From 56e68d91c60cbdec09a47588a664d325a23ef657 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 16 Jul 2014 16:58:03 +0800 Subject: [PATCH 12/27] update read me --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f853e9b..f050288 100644 --- a/README.md +++ b/README.md @@ -80,13 +80,14 @@ See benchmark.md for more. + Admin -## GoDoc -[![GoDoc](https://godoc.org/github.com/siddontang/ledisdb?status.png)](https://godoc.org/github.com/siddontang/ledisdb) +## Links -## Commands ++ [Official Website](http://ledisdb.com) ++ [Author's Blog](http://blog.csdn.net/siddontang/article/category/2264003) ++ [![GoDoc](https://godoc.org/github.com/siddontang/ledisdb?status.png)](https://godoc.org/github.com/siddontang/ledisdb) ++ [Server Commands](https://github.com/siddontang/ledisdb/wiki/Commands). -[Server Commands](https://github.com/siddontang/ledisdb/wiki/Commands). ## Thanks From ea5fabb6925425924085325b6e2c7aece82eaf6c Mon Sep 17 00:00:00 2001 From: holys Date: Wed, 16 Jul 2014 16:57:38 +0800 Subject: [PATCH 13/27] Update command documents, Add missing bitmap commands --- .gitignore | 1 + doc/commands.md | 133 +++++++++++++++++++++++++++++++++++------------- reinstall.sh | 16 ++++++ 3 files changed, 116 insertions(+), 34 deletions(-) create mode 100644 reinstall.sh diff --git a/.gitignore b/.gitignore index 8458dd7..d1c3459 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ build *.pyc .DS_Store +nohup.out \ No newline at end of file diff --git a/doc/commands.md b/doc/commands.md index c9045e1..2157cbd 100644 --- a/doc/commands.md +++ b/doc/commands.md @@ -80,9 +80,12 @@ Table of Contents - [ZTTL key](#zttl-key) - [ZPERSIST key](#zpersist-key) - [Bitmap](#bitmap) - - [BSETBIT key offset value](#bsetbit-key-offset-value) - - [BGETBIT key offset](#bsetbit-key-offset) + - [BGET key](#bget-key) + - [BGETBIT key offset](#bgetbit-key-offset) + - [BSETBIT key offset value](#bsetbit-key-offset-value) + - [BMSETBIT key offset value[offset value ...]](#bmsetbit-key-offset-value-offset-value-) + - [BOPT operation destkey key [key ...]](#bopt-operation-destkey-key-key-) - [BCOUNT key [start, end]](#bcount-key-start-end) - [BEXPIRE key seconds](#bexpire-key-seconds) - [BEXPIREAT key timestamp](#bexpireat-key-timestamp) @@ -1454,13 +1457,13 @@ Use ZRANK to get the rank of an element with the scores ordered from low to high **Examples** ``` -127.0.0.1:6380> ZADD myset 1 one +ledis> ZADD myset 1 one (integer) 1 -127.0.0.1:6380> ZADD myset 2 two +ledis> ZADD myset 2 two (integer) 1 -127.0.0.1:6380> ZREVRANK myset one +ledis> ZREVRANK myset one (integer) 1 -127.0.0.1:6380> ZREVRANK myset three +ledis> ZREVRANK myset three (nil) ``` @@ -1627,28 +1630,25 @@ ledis> ZTTL mset ``` + ## Bitmap -### BSETBIT key offset value -Sets or clear the bit at `offset` in the binary data sotred at `key`. -The bit is either set or cleared depending on `value`, which can be either `0` or `1`. -The *offset* argument is required to be qual to 0, and smaller than -2^23 (this means bitmap limits to 8MB). +### BGET key + +Returns the whole binary data stored at `key`. **Return value** -int64 : the original bit value stored at offset. +bulk: the raw value of key, or nil when key does not exist. **Examples** ``` -ledis> BSETBIT flag 0 1 -(integer) 0 -ledis> BSETBIT flag 0 0 -(integer) 1 -ledis> BGETBIT flag 0 99 -ERR invalid command param +ledis> BMSETBIT flag 0 1 5 1 6 1 +(integer) 3 +ledis> BGET flag +a ``` @@ -1675,6 +1675,73 @@ ledis> BGETBIT flag 65535 ``` +### BSETBIT key offset value + +Sets or clear the bit at `offset` in the binary data sotred at `key`. +The bit is either set or cleared depending on `value`, which can be either `0` or `1`. +The *offset* argument is required to be qual to 0, and smaller than +2^23 (this means bitmap limits to 8MB). + +**Return value** + +int64 : the original bit value stored at offset. + +**Examples** + +``` +ledis> BSETBIT flag 0 1 +(integer) 0 +ledis> BSETBIT flag 0 0 +(integer) 1 +ledis> BGETBIT flag 0 99 +ERR invalid command param +``` + +### BMSETBIT key offset value [offset value ...] +Sets the given *offset* to their respective values. + +**Return value** + +int64 : The number of input *offset* + +**Examples** + +``` +ledis> BMSETBIT flag 0 1 1 1 2 0 3 1 +(integer) 4 +ledis> BCOUNT flag +(integer) 3 +``` + + +### BOPT operation destkey key [key ...] +Perform a bitwise operation between multiple keys (containing string values) and store the result in the destination key. + +**Return value** + +Int64: +The size of the string stored in the destination key, that is equal to the size of the longest input string. +**Examples** + +``` +ledis> BMSETBIT a 0 1 2 1 +(integer) 2 +ledis> BMSETBIT b 1 1 +(integer) 1 +ledis> BOPT AND res a b +(integer) 3 +ledis> BCOUNT res +(integer) 0 +ledis> BOPT OR res2 a b +(integer) 3 +ledis> BCOUNT res2 +(integer) 3 +ledis> BOPT XOR res3 a b +(integer) 3 +ledis> BCOUNT res3 +(integer) 3 +``` + ### BCOUNT key [start end] Count the number of set bits in a bitmap. @@ -1686,22 +1753,20 @@ int64 : The number of bits set to 1. **Examples** ``` - -``` - - -### BGET key - -Returns the whole binary data stored at `key`. - -**Return value** - -bulk: the raw value of key, or nil when key does not exist. - -**Examples** - -``` - +ledis> BMSETBIT flag 0 1 5 1 6 1 +(integer) 3 +ledis> BGET flag +a +ledis> BCOUNT flag +(integer) 3 +ledis> BCOUNT flag 0 0s +(integer) 1 +ledis> BCOUNT flag 0 4 +(integer) 1 +ledis> BCOUNT flag 0 5 +(integer) 2 +ledis> BCOUNT flag 5 6 +(integer) 2 ``` diff --git a/reinstall.sh b/reinstall.sh new file mode 100644 index 0000000..3593328 --- /dev/null +++ b/reinstall.sh @@ -0,0 +1,16 @@ +#/bin/bash +len=$(git status |grep modified |wc | awk '{print $1}') +if [ "$len" -gt 0 ]; then + printf "\nYou have local modified files\n" + exit 1 +fi + +git pull --rebase +ps -ef |grep -v grep |grep ledis| awk '{print $2}'|xargs kill -9 + +go install ./... + +source ./dev.sh +nohup ledis-server & +day=$(ps aux|grep -v grep |grep ledis-server | awk '{print $9}') +printf "ledis-server 启动于 $day" From 955cd1647722d37a2987efafabbad25b8df84199 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 16 Jul 2014 16:59:42 +0800 Subject: [PATCH 14/27] update read me --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f050288..50e165d 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ See benchmark.md for more. + [Official Website](http://ledisdb.com) + [Author's Blog](http://blog.csdn.net/siddontang/article/category/2264003) -+ [![GoDoc](https://godoc.org/github.com/siddontang/ledisdb?status.png)](https://godoc.org/github.com/siddontang/ledisdb) ++ [GoDoc](https://godoc.org/github.com/siddontang/ledisdb) + [Server Commands](https://github.com/siddontang/ledisdb/wiki/Commands). From 17f1fa4bc1aeff0113daa6b6b60c5f4a14740abe Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 16 Jul 2014 16:59:59 +0800 Subject: [PATCH 15/27] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 50e165d..4473536 100644 --- a/README.md +++ b/README.md @@ -84,9 +84,9 @@ See benchmark.md for more. ## Links + [Official Website](http://ledisdb.com) -+ [Author's Blog](http://blog.csdn.net/siddontang/article/category/2264003) ++ [Author's Chinese Blog](http://blog.csdn.net/siddontang/article/category/2264003) + [GoDoc](https://godoc.org/github.com/siddontang/ledisdb) -+ [Server Commands](https://github.com/siddontang/ledisdb/wiki/Commands). ++ [Server Commands](https://github.com/siddontang/ledisdb/wiki/Commands) ## Thanks @@ -99,4 +99,4 @@ Gamil: tiaotiaoyly@gmail.com ## Feedback -Gmail: siddontang@gmail.com \ No newline at end of file +Gmail: siddontang@gmail.com From 3826557b6ab3de013b6cec43d2f5f6eaf8ad604f Mon Sep 17 00:00:00 2001 From: holys Date: Wed, 16 Jul 2014 17:08:32 +0800 Subject: [PATCH 16/27] delete temporary file --- reinstall.sh | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 reinstall.sh diff --git a/reinstall.sh b/reinstall.sh deleted file mode 100644 index 3593328..0000000 --- a/reinstall.sh +++ /dev/null @@ -1,16 +0,0 @@ -#/bin/bash -len=$(git status |grep modified |wc | awk '{print $1}') -if [ "$len" -gt 0 ]; then - printf "\nYou have local modified files\n" - exit 1 -fi - -git pull --rebase -ps -ef |grep -v grep |grep ledis| awk '{print $2}'|xargs kill -9 - -go install ./... - -source ./dev.sh -nohup ledis-server & -day=$(ps aux|grep -v grep |grep ledis-server | awk '{print $9}') -printf "ledis-server 启动于 $day" From 633ced379c27226b9f0dbe3731103c163d11132c Mon Sep 17 00:00:00 2001 From: holys Date: Thu, 17 Jul 2014 00:44:40 +0800 Subject: [PATCH 17/27] fix bitmap test --- server/cmd_bit_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/cmd_bit_test.go b/server/cmd_bit_test.go index 902ec10..f5438f9 100644 --- a/server/cmd_bit_test.go +++ b/server/cmd_bit_test.go @@ -163,7 +163,7 @@ func testBitOpt(t *testing.T) { if blen, err := ledis.Int( c.Do("bopt", "and", dstk, k0, k1)); err != nil { t.Fatal(err) - } else if blen != 100 { + } else if blen != 101 { t.Fatal(blen) } @@ -183,7 +183,7 @@ func testBitOpt(t *testing.T) { if blen, err := ledis.Int( c.Do("bopt", "or", dstk, k0, k1)); err != nil { t.Fatal(err) - } else if blen != 100 { + } else if blen != 101 { t.Fatal(blen) } @@ -203,7 +203,7 @@ func testBitOpt(t *testing.T) { if blen, err := ledis.Int( c.Do("bopt", "xor", dstk, k0, k1)); err != nil { t.Fatal(err) - } else if blen != 100 { + } else if blen != 101 { t.Fatal(blen) } From 78ac5c98424c249711fce090874e940cdf41c948 Mon Sep 17 00:00:00 2001 From: holys Date: Thu, 17 Jul 2014 02:18:53 +0800 Subject: [PATCH 18/27] fix unprintable character --- cmd/ledis-cli/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/ledis-cli/main.go b/cmd/ledis-cli/main.go index 4149055..e4e1bc2 100644 --- a/cmd/ledis-cli/main.go +++ b/cmd/ledis-cli/main.go @@ -66,7 +66,7 @@ func printReply(cmd string, reply interface{}) { case string: fmt.Printf("%s", reply) case []byte: - fmt.Printf("%s", string(reply)) + fmt.Printf("%q", reply) case nil: fmt.Printf("(nil)") case ledis.Error: From c85844629c635451e75c4ba1d0f107fc72ab26b8 Mon Sep 17 00:00:00 2001 From: holys Date: Thu, 17 Jul 2014 11:08:14 +0800 Subject: [PATCH 19/27] add select display support; plus the -n flag in command line parameter --- cmd/ledis-cli/main.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/cmd/ledis-cli/main.go b/cmd/ledis-cli/main.go index e4e1bc2..1b06f00 100644 --- a/cmd/ledis-cli/main.go +++ b/cmd/ledis-cli/main.go @@ -5,12 +5,14 @@ import ( "fmt" "github.com/siddontang/ledisdb/client/go/ledis" "regexp" + "strconv" "strings" ) var ip = flag.String("h", "127.0.0.1", "ledisdb server ip (default 127.0.0.1)") var port = flag.Int("p", 6380, "ledisdb server port (default 6380)") var socket = flag.String("s", "", "ledisdb server socket, overwrite ip and port") +var dbnum = flag.Int("n", 0, "ledisdb database number(default 0)") func main() { flag.Parse() @@ -30,8 +32,16 @@ func main() { reg, _ := regexp.Compile(`'.*?'|".*?"|\S+`) + prompt := "" + for { - cmd, err := line(fmt.Sprintf("%s> ", cfg.Addr)) + if *dbnum != 0 { + prompt = fmt.Sprintf("%s[%d]>", cfg.Addr, *dbnum) + } else { + prompt = fmt.Sprintf("%s>", cfg.Addr) + } + + cmd, err := line(prompt) if err != nil { fmt.Printf("%s\n", err.Error()) return @@ -48,6 +58,9 @@ func main() { args[i] = strings.Trim(string(cmds[1+i]), "\"'") } r, err := c.Do(cmds[0], args...) + if strings.ToLower(cmds[0]) == "select" { + *dbnum, _ = strconv.Atoi(cmds[1]) + } if err != nil { fmt.Printf("%s", err.Error()) } else { From 82711d6fa8c48b40f36e6c1a4d3654a1b252b853 Mon Sep 17 00:00:00 2001 From: holys Date: Thu, 17 Jul 2014 12:06:07 +0800 Subject: [PATCH 20/27] fix display of db number out of range --- cmd/ledis-cli/main.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/cmd/ledis-cli/main.go b/cmd/ledis-cli/main.go index 1b06f00..e26af0b 100644 --- a/cmd/ledis-cli/main.go +++ b/cmd/ledis-cli/main.go @@ -12,7 +12,7 @@ import ( var ip = flag.String("h", "127.0.0.1", "ledisdb server ip (default 127.0.0.1)") var port = flag.Int("p", 6380, "ledisdb server port (default 6380)") var socket = flag.String("s", "", "ledisdb server socket, overwrite ip and port") -var dbnum = flag.Int("n", 0, "ledisdb database number(default 0)") +var dbn = flag.Int("n", 0, "ledisdb database number(default 0)") func main() { flag.Parse() @@ -35,8 +35,12 @@ func main() { prompt := "" for { - if *dbnum != 0 { - prompt = fmt.Sprintf("%s[%d]>", cfg.Addr, *dbnum) + if *dbn >= 16 { + fmt.Printf("ERR invalid db index %d. Auto switch back db 0.\n", *dbn) + *dbn = 0 + continue + } else if *dbn > 0 && *dbn < 16 { + prompt = fmt.Sprintf("%s[%d]>", cfg.Addr, *dbn) } else { prompt = fmt.Sprintf("%s>", cfg.Addr) } @@ -58,11 +62,12 @@ func main() { args[i] = strings.Trim(string(cmds[1+i]), "\"'") } r, err := c.Do(cmds[0], args...) - if strings.ToLower(cmds[0]) == "select" { - *dbnum, _ = strconv.Atoi(cmds[1]) - } + if err != nil { fmt.Printf("%s", err.Error()) + } else if nb, _ := strconv.Atoi(cmds[1]); strings.ToLower(cmds[0]) == "select" && nb < 16 { + *dbn = nb + printReply(cmd, r) } else { printReply(cmd, r) } From feb3c68f61c0bc6f791d6329726e9103209a384e Mon Sep 17 00:00:00 2001 From: holys Date: Fri, 18 Jul 2014 12:20:25 +0800 Subject: [PATCH 21/27] add help support in CLI --- cmd/ledis-cli/const.go | 84 ++++++++++++++++++++++++++++++++++++++++++ cmd/ledis-cli/main.go | 55 +++++++++++++++++++++++---- 2 files changed, 131 insertions(+), 8 deletions(-) create mode 100644 cmd/ledis-cli/const.go diff --git a/cmd/ledis-cli/const.go b/cmd/ledis-cli/const.go new file mode 100644 index 0000000..1cdff8f --- /dev/null +++ b/cmd/ledis-cli/const.go @@ -0,0 +1,84 @@ +package main + +var helpCommands = [][]string{ + {"DECR", "key", "KV"}, + {"DECRBY", "key decrement", "KV"}, + {"DEL", "key [key ...]", "KV"}, + {"EXISTS", "key", "KV"}, + {"GET", "key", "KV"}, + {"GETSET", " key value", "KV"}, + {"INCR", "key", "KV"}, + {"INCRBY", "key increment", "KV"}, + {"MGET", "key [key ...]", "KV"}, + {"MSET", "key value [key value ...]", "KV"}, + {"SET", "key value", "KV"}, + {"SETNX", "key value", "KV"}, + {"EXPIRE", "key seconds", "KV"}, + {"EXPIREAT", "key timestamp", "KV"}, + {"TTL", "PERSIST", "KV"}, + {"HDEL", "key field [field ...]", "Hash"}, + {"HEXISTS", "key field", "Hash"}, + {"HGET", "key field", "Hash"}, + {"HGETALL", "key", "Hash"}, + {"HINCRBY", "key field increment", "Hash"}, + {"HKEYS", "key", "Hash"}, + {"HLEN", "key", "Hash"}, + {"HMGET", "key field [field ...]", "Hash"}, + {"HMSET", "key field value [field value ...]", "Hash"}, + {"HSET", "key field value", "Hash"}, + {"HVALS", "key", "Hash"}, + {"HCLEAR", "key", "Hash"}, + {"HMCLEAR", "key [key ...]", "Hash"}, + {"HEXPIRE", "key seconds", "Hash"}, + {"HEXPIREAT", "key timestamp", "Hash"}, + {"HTTL", "key", "Hash"}, + {"HPERSIST", "key", "Hash"}, + {"LINDEX", "key index", "List"}, + {"LLEN", "key", "List"}, + {"LPOP", "key", "List"}, + {"LPUSH", "key value [value ...]", "List"}, + {"LRANGE", "key start stop", "List"}, + {"RPOP", "key", "List"}, + {"RPUSH", "key value [value ...]", "List"}, + {"LCLEAR", "key", "List"}, + {"LMCLEAR", "key [key ...]", "List"}, + {"LEXPIRE", "key seconds", "List"}, + {"LEXPIREAT", "key timestamp", "List"}, + {"LTTL", "key", "List"}, + {"LPERSIST", "key", "List"}, + {"ZADD", "key score member [score member ...]", "ZSet"}, + {"ZCARD", "key", "ZSet"}, + {"ZCOUNT", "key min max", "ZSet"}, + {"ZINCRBY", "key increment member", "ZSet"}, + {"ZRANGE", "key start stop [WITHSCORES]", "ZSet"}, + {"ZRANGEBYSCORE", "key min max [WITHSCORES] [LIMIT offset count]", "ZSet"}, + {"ZRANK", "key member", "ZSet"}, + {"ZREM", "key member [member ...]", "ZSet"}, + {"ZREMRANGEBYRANK", "key start stop", "ZSet"}, + {"ZREMRANGEBYSCORE", "key min max", "ZSet"}, + {"ZREVRANGEBYSCORE", "key max min [WITHSCORES][LIMIT offset count]", "ZSet"}, + {"ZREVRANK", "key member", "ZSet"}, + {"ZSCORE", "key member", "ZSet"}, + {"ZCLEAR", "key", "ZSet"}, + {"ZMCLEAR", "key [key ...]", "ZSet"}, + {"ZEXPIRE", "key seconds", "ZSet"}, + {"ZEXPIREAT", "key timestamp", "ZSet"}, + {"ZTTL", "key", "ZSet"}, + {"ZPERSIST", "key", "ZSet"}, + {"BGET", "key", "Bitmap"}, + {"BGETBIT", "key offset", "Bitmap"}, + {"BSETBIT", "key offset value", "Bitmap"}, + {"BMSETBIT", "key offset value [offset value ...]", "Bitmap"}, + {"BOPT", "operation destkey key [key ...]", "Bitmap"}, + {"BCOUNT", "key [start end]", "Bitmap"}, + {"BEXPIRE", "key seconds", "Bitmap"}, + {"BEXPIREAT", "key timestamp", "Bitmap"}, + {"BTTL", "key", "Bitmap"}, + {"BPERSIST", "key", "Bitmap"}, + {"SLAVEOF", "host port", "Replication"}, + {"FULLSYNC", "-", "Replication"}, + {"SYNC", "index offset", "Replication"}, + {"PING", "-", "Server"}, + {"ECHO", "message", "Server"}, + {"SELECT", "index", "Server"}, +} diff --git a/cmd/ledis-cli/main.go b/cmd/ledis-cli/main.go index e26af0b..bcd42ac 100644 --- a/cmd/ledis-cli/main.go +++ b/cmd/ledis-cli/main.go @@ -58,21 +58,29 @@ func main() { addHistory(cmd) args := make([]interface{}, len(cmds[1:])) + for i := range args { args[i] = strings.Trim(string(cmds[1+i]), "\"'") } - r, err := c.Do(cmds[0], args...) - if err != nil { - fmt.Printf("%s", err.Error()) - } else if nb, _ := strconv.Atoi(cmds[1]); strings.ToLower(cmds[0]) == "select" && nb < 16 { - *dbn = nb - printReply(cmd, r) + cmd := cmds[0] + if strings.ToLower(cmd) == "help" || cmd == "?" { + printHelp(cmds) } else { - printReply(cmd, r) + r, err := c.Do(cmds[0], args...) + + if err != nil { + fmt.Printf("%s", err.Error()) + } else if nb, _ := strconv.Atoi(cmds[1]); strings.ToLower(cmds[0]) == "select" && nb < 16 { + *dbn = nb + printReply(cmd, r) + } else { + printReply(cmd, r) + } + + fmt.Printf("\n") } - fmt.Printf("\n") } } } @@ -105,3 +113,34 @@ func printReply(cmd string, reply interface{}) { fmt.Printf("invalid ledis reply") } } + +func printGenericHelp() { + msg := + `ledis-cli +Type: "help " for help on + ` + fmt.Println(msg) +} + +func printCommandHelp(arr []string) { + fmt.Println() + fmt.Printf("\t%s %s \n", arr[0], arr[1]) + fmt.Printf("\tGroup: %s \n", arr[2]) + fmt.Println() +} + +func printHelp(cmds []string) { + args := cmds[1:] + if len(args) == 0 { + printGenericHelp() + } else if len(args) > 1 { + fmt.Println() + } else { + cmd := strings.ToUpper(args[0]) + for i := 0; i < len(helpCommands); i++ { + if helpCommands[i][0] == cmd { + printCommandHelp(helpCommands[i]) + } + } + } +} From 39e28b7f170858ffc6a98ce6bc3aaa86ae72b5c3 Mon Sep 17 00:00:00 2001 From: siddontang Date: Sun, 20 Jul 2014 16:15:23 +0800 Subject: [PATCH 22/27] change ledis-load refer mysql load dump although dump file may include master bin log information, ledis-load will not change in slave but to output. --- cmd/ledis-load/main.go | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/cmd/ledis-load/main.go b/cmd/ledis-load/main.go index da1fad3..6d7ef6e 100644 --- a/cmd/ledis-load/main.go +++ b/cmd/ledis-load/main.go @@ -3,16 +3,13 @@ package main import ( "encoding/json" "flag" + "fmt" "github.com/siddontang/ledisdb/ledis" - "github.com/siddontang/ledisdb/server" "io/ioutil" - "path" ) var configPath = flag.String("config", "/etc/ledis.json", "ledisdb config file") var dumpPath = flag.String("dump_file", "", "ledisdb dump file") -var masterAddr = flag.String("master_addr", "", - "master addr to set where dump file comes from, if not set correctly, next slaveof may cause fullsync") func main() { flag.Parse() @@ -74,13 +71,11 @@ func loadDump(cfg *ledis.Config, ldb *ledis.Ledis) error { return err } - info := new(server.MasterInfo) + //master enable binlog, here output this like mysql + if head.LogFileIndex != 0 && head.LogPos != 0 { + format := "-- CHANGE MASTER TO MASTER_LOG_FILE='binlog.%07d', MASTER_LOG_POS=%d;\n" + fmt.Printf(format, head.LogFileIndex, head.LogPos) + } - info.Addr = *masterAddr - info.LogFileIndex = head.LogFileIndex - info.LogPos = head.LogPos - - infoFile := path.Join(cfg.DataDir, "master.info") - - return info.Save(infoFile) + return nil } From e8e3f6324bc16ae9bd95b75c01833c47f99b3666 Mon Sep 17 00:00:00 2001 From: siddontang Date: Sun, 20 Jul 2014 16:16:12 +0800 Subject: [PATCH 23/27] ledis-load update --- cmd/ledis-load/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/ledis-load/main.go b/cmd/ledis-load/main.go index 6d7ef6e..0995b29 100644 --- a/cmd/ledis-load/main.go +++ b/cmd/ledis-load/main.go @@ -73,7 +73,7 @@ func loadDump(cfg *ledis.Config, ldb *ledis.Ledis) error { //master enable binlog, here output this like mysql if head.LogFileIndex != 0 && head.LogPos != 0 { - format := "-- CHANGE MASTER TO MASTER_LOG_FILE='binlog.%07d', MASTER_LOG_POS=%d;\n" + format := "MASTER_LOG_FILE='binlog.%07d', MASTER_LOG_POS=%d;\n" fmt.Printf(format, head.LogFileIndex, head.LogPos) } From b04aa337c4b5fb5c15e77d5179d833000c8bca5d Mon Sep 17 00:00:00 2001 From: siddontang Date: Sun, 20 Jul 2014 19:03:00 +0800 Subject: [PATCH 24/27] add makefile --- Makefile | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1dfe2f3 --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +all: build + +build: + go install ./... + +clean: + go clean -i ./... + +test: + go test ./... + go test -race ./... \ No newline at end of file From 6e295c59a94c7ba1065b687f13b27e1c26f22393 Mon Sep 17 00:00:00 2001 From: holys Date: Mon, 21 Jul 2014 09:52:53 +0800 Subject: [PATCH 25/27] add missing test for bitmap --- client/ledis-py/tests/test_cmd_bit.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/client/ledis-py/tests/test_cmd_bit.py b/client/ledis-py/tests/test_cmd_bit.py index 67b5bf9..7863d21 100644 --- a/client/ledis-py/tests/test_cmd_bit.py +++ b/client/ledis-py/tests/test_cmd_bit.py @@ -62,15 +62,28 @@ class TestCmdBit(unittest.TestCase): assert l.bcount('a') == 1 l.bmsetbit('a', 10, 1, 20, 1, 30, 1, 40, 1) assert l.bcount('a') == 5 - #TODO test bcount with start/end option + assert l.bcount('a', 0, 10) == 2 + assert l.bcount('a', 20, 30) == 2 + assert l.bcount('a', 10, 10) == 1 def test_bopt_not_empty_string(self): l.bopt('not', 'r', 'a') assert l.bget('r') is None - def test_bopt_not(self): - #TODO - pass + def test_bopt(self): + l.bmsetbit('a1', 10, 1, 30, 1, 50, 1, 70, 1, 90, 1) + l.bmsetbit('a2', 20, 1, 40, 1, 60, 1, 80, 1, 100, 1) + assert l.bopt('and', 'res1', 'a1', 'a2') == 101 + assert l.bcount('res1') == 0 + + assert l.bopt('or', 'res2', 'a1', 'a2') == 101 + assert l.bcount('res2') == 10 + + assert l.bopt('xor', 'res3', 'a1', 'a2') == 101 + assert l.bcount('res3') == 10 + + assert l.bopt('not', 'res4', 'a1') == 91 + assert l.bcount('res4') == 86 def test_bexpire(self): assert not l.bexpire('a', 100) From 1e28c6be6d163699b5baf30c7fb4f12bc02dbe55 Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 21 Jul 2014 10:46:11 +0800 Subject: [PATCH 26/27] update read me --- README.md | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 4473536..070e5b3 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,9 @@ Ledisdb is a high performance NoSQL like Redis based on LevelDB written by go. I sh build_leveldb.sh - It will default install leveldb at /usr/local/leveldb and snappy at /usr/local/snappy + It will default install leveldb at /usr/local/leveldb and snappy at /usr/local/snappy. + + LedisDB use the modification LevelDB for better performance, see [here](https://github.com/siddontang/ledisdb/wiki/leveldb-source-modification). + Set LEVELDB_DIR and SNAPPY_DIR to the actual install path in dev.sh. @@ -40,7 +42,7 @@ Ledisdb is a high performance NoSQL like Redis based on LevelDB written by go. I go install ./... -## Run +## Server Example ./ledis-server -config=/etc/ledis.json @@ -52,7 +54,7 @@ Ledisdb is a high performance NoSQL like Redis based on LevelDB written by go. I ledis 127.0.0.1:6380> get a "1" -## Lib +## Package Example import "github.com/siddontang/ledisdb/ledis" l, _ := ledis.Open(cfg) @@ -63,7 +65,7 @@ Ledisdb is a high performance NoSQL like Redis based on LevelDB written by go. I db.Get(key) -## Replication +## Replication Example Set slaveof in config or dynamiclly @@ -78,7 +80,7 @@ See benchmark.md for more. ## Todo -+ Admin +See [Issues todo](https://github.com/siddontang/ledisdb/issues?labels=todo&page=1&state=open) ## Links @@ -95,7 +97,12 @@ Gmail: cenqichao@gmail.com Gmail: chendahui007@gmail.com -Gamil: tiaotiaoyly@gmail.com +Gmail: cppgohan@gmail.com + +Gmail: tiaotiaoyly@gmail.com + +Gmail: wyk4true@gmail.com + ## Feedback From 5d5e0f4d58944fdb2bebb8cb584c7871281a9ea2 Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 21 Jul 2014 10:47:25 +0800 Subject: [PATCH 27/27] update read me --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 070e5b3..846ac17 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Ledisdb is a high performance NoSQL like Redis based on LevelDB written by go. I It will default install leveldb at /usr/local/leveldb and snappy at /usr/local/snappy. - LedisDB use the modification LevelDB for better performance, see [here](https://github.com/siddontang/ledisdb/wiki/leveldb-source-modification). + LedisDB use the modified LevelDB for better performance, see [here](https://github.com/siddontang/ledisdb/wiki/leveldb-source-modification). + Set LEVELDB_DIR and SNAPPY_DIR to the actual install path in dev.sh.