Merge remote-tracking branch 'origin/tool-feature' into develop

tools/redis_import/README.md

@@ -0,0 +1,19 @@
+## Notice
+
+1. The tool doesn't support `set` data type.
+2. The tool doesn't support `bitmap` data type.
+2. Our `zset` use integer instead of double, so the zset float score in Redis 
+   will be **converted to integer**.
+3. Only Support Redis version greater than  `2.8.0`, because we use `scan` command to scan data.
+   Also, you need `redis-py` greater than `2.9.0`.
+
+
+
+## Usage
+
+
+       $ python redis_import.py redis_host redis_port redis_db ledis_host ledis_port [True]
+
+The option `True` means convert `set` to `zset` or not, if not, set it to `False`.
+
+We will use the same db index as redis. That's to say, data in redis[0] will be transfer to ledisdb[0].

tools/redis_import/redis_import.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# refer: https://github.com/ideawu/ssdb/blob/master/tools/redis-import.php
+
+# Notice: for zset, float score will be converted to integer.
+
+import sys
+import os
+from collections import OrderedDict as od
+
+import redis
+import ledis
+
+total = 0
+entries = 0
+
+
+def scan_available(redis_client):
+    """"Scan Command is available since redis-server 2.8.0"""
+
+    if "scan" in dir(redis_client):
+        info = redis_client.info()
+        server_version = info["redis_version"]
+        version_list = server_version.split(".")
+        if len(version_list) > 2:
+            n = int(version_list[0]) * 10 + int(version_list[1])
+            if n >= 28:
+                return True
+    return False
+
+
+def set_ttl(redis_client, ledis_client, key, k_type):
+    k_types = {
+        "string": ledis_client.expire,
+        "list": ledis_client.lexpire,
+        "hash": ledis_client.hexpire,
+        "set": ledis_client.zexpire,
+        "zset": ledis_client.zexpire
+    }
+    timeout = redis_client.ttl(key)
+    if timeout > 0:
+        k_types[k_type](key, timeout)
+
+
+def copy_key(redis_client, ledis_client, key, convert=False):
+    global entries
+    k_type = redis_client.type(key)
+    if k_type == "string":
+        value = redis_client.get(key)
+        ledis_client.set(key, value)
+        set_ttl(redis_client, ledis_client, key, k_type)
+        entries += 1
+
+    elif k_type == "list":
+        _list = redis_client.lrange(key, 0, -1)
+        for value in _list:
+            ledis_client.rpush(key, value)
+        set_ttl(redis_client, ledis_client, key, k_type)
+        entries += 1
+
+    elif k_type == "hash":
+        mapping = od(redis_client.hgetall(key))
+        ledis_client.hmset(key, mapping)
+        set_ttl(redis_client, ledis_client, key, k_type)
+        entries += 1
+
+    elif k_type == "zset":
+        out = redis_client.zrange(key, 0, -1, withscores=True)
+        pieces = od()
+        for i in od(out).iteritems():
+            pieces[i[0]] = int(i[1])
+        ledis_client.zadd(key, **pieces)
+        set_ttl(redis_client, ledis_client, key, k_type)
+        entries += 1
+
+    elif k_type == "set":
+        if convert:
+            print "Convert set %s to zset\n" % key
+            members = redis_client.smembers(key)
+            set_to_zset(ledis_client, key, members)
+            entries += 1
+        else:
+            print "KEY %s of TYPE %s will not be converted to Zset" % (key, k_type)
+
+    else:
+        print "KEY %s of TYPE %s is not supported by LedisDB." % (key, k_type)
+
+
+def copy_keys(redis_client, ledis_client, keys, convert=False):
+    for key in keys:
+        copy_key(redis_client, ledis_client, key, convert=convert)
+
+
+def scan(redis_client, count=1000):
+    keys = []
+    total = redis_client.dbsize()
+    if total > 1000:
+        print "It may take a while, be patient please."
+
+    first = True
+    cursor = 0
+    while cursor != 0 or first:
+        cursor, data = redis_client.scan(cursor, count=count)
+        keys.extend(data)
+        first = False
+    assert len(keys) == total
+    return keys, total
+
+
+def copy(redis_client, ledis_client, count=1000, convert=False):
+    if scan_available(redis_client):
+        print "\nTransfer begin ...\n"
+        keys, total = scan(redis_client, count=count)
+        copy_keys(redis_client, ledis_client, keys, convert=convert)
+
+    else:
+        msg = """We do not support Redis version less than 2.8.0.
+            Please check both your redis server version and redis-py
+            version.
+              """
+        print msg
+        sys.exit()
+    print "%d keys, %d entries copied" % (total, entries)
+
+
+def set_to_zset(ledis_client, key, members):
+    d = {}
+    for m in members:
+        d[m] = int(0)
+    ledis_client.zadd(key, **d)
+
+
+def usage():
+    usage = """
+        Usage:
+        python %s redis_host redis_port redis_db ledis_host ledis_port [True]
+        """
+    print usage % os.path.basename(sys.argv[0])
+
+
+def get_prompt(choice):
+    yes = set(['yes', 'ye', 'y', ''])
+    no = set(['no', 'n'])
+
+    if choice in yes:
+        return True
+    elif choice in no:
+        return False
+    else:
+        sys.stdout.write("Please respond with 'yes' or 'no'")
+
+
+def main():
+    if len(sys.argv) < 6:
+        usage()
+        sys.exit()
+    convert = False
+    if len(sys.argv) >= 6:
+        (redis_host, redis_port, redis_db, ledis_host, ledis_port) = sys.argv[1:6]
+        if len(sys.argv) == 7 and sys.argv[-1] == "True" or sys.argv[-1] == "true":
+            convert = True
+
+    choice = raw_input("[y/N]").lower()
+    if not get_prompt(choice):
+        sys.exit("No proceed")
+
+    redis_c = redis.Redis(host=redis_host, port=int(redis_port), db=int(redis_db))
+    ledis_c = ledis.Ledis(host=ledis_host, port=int(ledis_port), db=int(redis_db))
+    try:
+        redis_c.ping()
+    except redis.ConnectionError:
+        print "Could not connect to Redis Server"
+        sys.exit()
+
+    try:
+        ledis_c.ping()
+    except redis.ConnectionError:
+        print "Could not connect to LedisDB Server"
+        sys.exit()
+
+    copy(redis_c, ledis_c, convert=convert)
+    print "Done\n"
+
+
+if __name__ == "__main__":
+    main()

tools/redis_import/test.py

@@ -0,0 +1,133 @@
+#coding: utf-8
+
+import random, string
+
+import redis
+import ledis
+
+from redis_import import copy, scan, set_ttl
+
+rds = redis.Redis()
+lds = ledis.Ledis(port=6380)
+
+
+def random_word(words, length):
+    return ''.join(random.choice(words) for i in range(length))
+
+
+def get_words():
+    word_file = "/usr/share/dict/words"
+    words = open(word_file).read().splitlines()
+    return words[:1000]
+
+
+def get_mapping(words, length=1000):
+    d = {}
+    for word in words:
+        d[word] = random.randint(1, length)
+    return d
+
+
+def random_string(client, words, length=1000):
+    d = get_mapping(words, length)
+    client.mset(d)
+
+
+def random_hash(client, words, length=1000):
+    d = get_mapping(words, length)
+    client.hmset("hashName", d)
+
+
+def random_list(client, words, length=1000):
+    client.lpush("listName", *words)
+
+
+def random_zset(client, words, length=1000):
+    d = get_mapping(words, length)
+    client.zadd("zsetName", **d)
+
+
+def random_set(client, words, length=1000):
+    client.sadd("setName", *words)
+
+
+def test():
+    words = get_words()
+    print "Flush all redis data before insert new."
+    rds.flushall()
+
+    random_string(rds, words)
+    print "random_string done"
+
+    random_hash(rds, words)
+    print "random_hash done"
+    
+    random_list(rds, words)
+    print "random_list done"
+
+    random_zset(rds, words)
+    print "random_zset done"
+
+    random_set(rds, words)
+    print "random_set done"
+
+    lds.lclear("listName")
+    lds.hclear("hashName")
+    lds.zclear("zsetName")
+    lds.zclear("setName")
+    copy(rds, lds, convert=True)
+
+    # for all keys
+    keys = scan(rds, 1000)
+    for key in keys:
+        if rds.type(key) == "string" and not lds.exists(key):
+            print key
+            print "String data not consistent"
+
+    # for list
+    l1 = rds.lrange("listName", 0, -1)
+    l2 = lds.lrange("listName", 0, -1)
+    assert l1 == l2
+   
+    #for hash
+    for key in keys:
+        if rds.type(key) == "hash":
+            assert rds.hgetall(key) == lds.hgetall(key)
+            assert sorted(rds.hkeys(key)) == sorted(lds.hkeys(key))
+            assert sorted(rds.hvals(key)) == sorted(lds.hvals(key))
+
+    # for zset
+    z1 = rds.zrange("zsetName", 0, -1, withscores=True)
+    z2 = lds.zrange("zsetName", 0, -1, withscores=True)
+    assert z1 == z2
+
+    # fo set
+    assert set(rds.smembers("setName")) == set(lds.zrange("setName", 0, -1))
+    for key in lds.zrange("setName", 0, -1):
+        assert int(lds.zscore("setName", key)) == 0
+
+
+def ledis_ttl(ledis_client, key, k_type):
+    ttls = {
+        "string": lds.ttl,
+        "list": lds.lttl,
+        "hash": lds.httl,
+        "zset": lds.zttl,
+        "set": lds.zttl
+    }
+    return ttls[k_type](key)
+
+
+def test_ttl():
+    keys, total = scan(rds, 1000)
+    for key in keys:
+        k_type = rds.type(key)
+        rds.expire(key, (60 * 60 * 24))
+        set_ttl(rds, lds, key, k_type)
+        if rds.ttl(key):
+            assert ledis_ttl(lds, key, k_type) > 0
+
+if __name__ == "__main__":
+    test()
+    test_ttl()
+    print "Test passed."