From 54ed8e0d383f7803ad444bdb7e492e1c8ecb89fb Mon Sep 17 00:00:00 2001 From: wenyekui Date: Thu, 14 Aug 2014 15:24:41 +0800 Subject: [PATCH] refactor scan --- ledis/scan.go | 86 +++++++++++++++++++++++++++++++++++++++++++ ledis/scan_test.go | 83 +++++++++++++++++++++++++++++++++++++++++ ledis/t_hash.go | 42 +-------------------- ledis/t_hash_test.go | 44 ---------------------- ledis/t_kv.go | 37 +------------------ ledis/t_kv_test.go | 34 ----------------- ledis/t_list.go | 14 +++++++ ledis/t_zset.go | 4 ++ server/client_http.go | 7 ++-- 9 files changed, 193 insertions(+), 158 deletions(-) create mode 100644 ledis/scan.go create mode 100644 ledis/scan_test.go diff --git a/ledis/scan.go b/ledis/scan.go new file mode 100644 index 0000000..a1f5fe2 --- /dev/null +++ b/ledis/scan.go @@ -0,0 +1,86 @@ +package ledis + +import ( + "errors" + "github.com/siddontang/ledisdb/store" +) + +var errDataType = errors.New("error data type") +var errMetaKey = errors.New("error meta key") + +func (db *DB) scan(dataType byte, key []byte, count int, inclusive bool) ([][]byte, error) { + var minKey, maxKey []byte + var err error + if key != nil { + if err = db.checkKeySize(key); err != nil { + return nil, err + } + if minKey, err = db.encodeOffsetKey(dataType, key); err != nil { + return nil, err + } + + } else { + if minKey, err = db.encodeMinKey(dataType); err != nil { + return nil, err + } + } + + if maxKey, err = db.encodeMaxKey(dataType); err != nil { + return nil, err + } + + if count <= 0 { + count = defaultScanCount + } + + v := make([][]byte, 0, count) + + rangeType := store.RangeROpen + if !inclusive { + rangeType = store.RangeOpen + } + + it := db.db.RangeLimitIterator(minKey, maxKey, rangeType, 0, count) + for ; it.Valid(); it.Next() { + if k, err := db.decodeMetaKey(dataType, it.Key()); err != nil { + continue + } else { + v = append(v, k) + } + } + it.Close() + return v, nil +} + +func (db *DB) encodeMinKey(dataType byte) ([]byte, error) { + return db.encodeOffsetKey(dataType, nil) +} + +func (db *DB) encodeMaxKey(dataType byte) ([]byte, error) { + k := db.encodeOffsetKey(dataType, nil) + k[len(k)-1] = dataType + 1 + return k, nil +} + +func (db *DB) encodeMetaKey(dataType byte, key []byte) ([]byte, error) { + switch dataType { + case KVType: + return db.encodeKVKey(key), nil + case LMetaType: + return db.lEncodeMetaKey(key), nil + case HSizeType: + return db.hEncodeSizeKey(key), nil + case ZSizeType: + return db.zEncodeSizeKey(key), nil + case BitMetaType: + return db.bEncodeMetaKey(key), nil + default: + return nil, errInvalidDataType + } +} +func (db *DB) decodeMetaKey(dataType, byte, ek []byte) ([]byte, error) { + if len(ek) < 2 || ek[0] != db.index || ek[1] != dataType { + return nil, errMetaKey + } + return ek[2:], nil +} diff --git a/ledis/scan_test.go b/ledis/scan_test.go new file mode 100644 index 0000000..86ac846 --- /dev/null +++ b/ledis/scan_test.go @@ -0,0 +1,83 @@ +package ledis + +import ( + "testing" +) + +func TestDBScan(t *testing.T) { + db := getTestDB() + + db.FlushAll() + + if v, err := db.Scan(nil, 10, true); err != nil { + t.Fatal(err) + } else if len(v) != 0 { + t.Fatal(len(v)) + } + + db.Set([]byte("a"), []byte{}) + db.Set([]byte("b"), []byte{}) + db.Set([]byte("c"), []byte{}) + + if v, err := db.Scan(nil, 1, true); err != nil { + t.Fatal(err) + } else if len(v) != 1 { + t.Fatal(len(v)) + } + + if v, err := db.Scan([]byte("a"), 2, false); err != nil { + t.Fatal(err) + } else if len(v) != 2 { + t.Fatal(len(v)) + } + + if v, err := db.Scan(nil, 3, true); err != nil { + t.Fatal(err) + } else if len(v) != 3 { + t.Fatal(len(v)) + } +} + +func TestDBHScan(t *testing.T) { + db := getTestDB() + + db.hFlush() + + k1 := []byte("k1") + db.HSet(k1, []byte("1"), []byte{}) + + k2 := []byte("k2") + db.HSet(k2, []byte("2"), []byte{}) + + k3 := []byte("k3") + db.HSet(k3, []byte("3"), []byte{}) + + if v, err := db.HScan(nil, 1, true); err != nil { + t.Fatal(err) + } else if len(v) != 1 { + t.Fatal("invalid length ", len(v)) + } else if string(v[0]) != "k1" { + t.Fatal("invalid value ", string(v[0])) + } + + if v, err := db.HScan(k1, 2, true); err != nil { + t.Fatal(err) + } else if len(v) != 2 { + t.Fatal("invalid length ", len(v)) + } else if string(v[0]) != "k1" { + t.Fatal("invalid value ", string(v[0])) + } else if string(v[1]) != "k2" { + t.Fatal("invalid value ", string(v[1])) + } + + if v, err := db.HScan(k1, 2, false); err != nil { + t.Fatal(err) + } else if len(v) != 2 { + t.Fatal("invalid length ", len(v)) + } else if string(v[0]) != "k2" { + t.Fatal("invalid value ", string(v[0])) + } else if string(v[1]) != "k3" { + t.Fatal("invalid value ", string(v[1])) + } + +} diff --git a/ledis/t_hash.go b/ledis/t_hash.go index 41c8ccb..eb6bbb1 100644 --- a/ledis/t_hash.go +++ b/ledis/t_hash.go @@ -472,48 +472,8 @@ func (db *DB) hFlush() (drop int64, err error) { return } -func (db *DB) hEncodeMinKey() []byte { - return db.hEncodeSizeKey(nil) -} - -func (db *DB) hEncodeMaxKey() []byte { - ek := db.hEncodeSizeKey(nil) - ek[len(ek)-1] = HSizeType + 1 - return ek -} - func (db *DB) HScan(key []byte, count int, inclusive bool) ([][]byte, error) { - var minKey []byte - if key != nil { - minKey = db.hEncodeSizeKey(key) - - } else { - minKey = db.hEncodeMinKey() - } - - maxKey := db.hEncodeMaxKey() - - if count <= 0 { - count = defaultScanCount - } - - v := make([][]byte, 0, count) - - rangeType := store.RangeROpen - if !inclusive { - rangeType = store.RangeOpen - } - - it := db.db.RangeLimitIterator(minKey, maxKey, rangeType, 0, count) - for ; it.Valid(); it.Next() { - if k, err := db.hDecodeSizeKey(it.Key()); err != nil { - continue - } else { - v = append(v, k) - } - } - it.Close() - return v, nil + return db.scan(HSizeType, key, count, inclusive) } func (db *DB) HExpire(key []byte, duration int64) (int64, error) { diff --git a/ledis/t_hash_test.go b/ledis/t_hash_test.go index ff8e6a0..bf7cc7c 100644 --- a/ledis/t_hash_test.go +++ b/ledis/t_hash_test.go @@ -56,50 +56,6 @@ func TestDBHash(t *testing.T) { } -func TestDBHScan(t *testing.T) { - db := getTestDB() - - db.hFlush() - - k1 := []byte("k1") - db.HSet(k1, []byte("1"), []byte{}) - - k2 := []byte("k2") - db.HSet(k2, []byte("2"), []byte{}) - - k3 := []byte("k3") - db.HSet(k3, []byte("3"), []byte{}) - - if v, err := db.HScan(nil, 1, true); err != nil { - t.Fatal(err) - } else if len(v) != 1 { - t.Fatal("invalid length ", len(v)) - } else if string(v[0]) != "k1" { - t.Fatal("invalid value ", string(v[0])) - } - - if v, err := db.HScan(k1, 2, true); err != nil { - t.Fatal(err) - } else if len(v) != 2 { - t.Fatal("invalid length ", len(v)) - } else if string(v[0]) != "k1" { - t.Fatal("invalid value ", string(v[0])) - } else if string(v[1]) != "k2" { - t.Fatal("invalid value ", string(v[1])) - } - - if v, err := db.HScan(k1, 2, false); err != nil { - t.Fatal(err) - } else if len(v) != 2 { - t.Fatal("invalid length ", len(v)) - } else if string(v[0]) != "k2" { - t.Fatal("invalid value ", string(v[0])) - } else if string(v[1]) != "k3" { - t.Fatal("invalid value ", string(v[1])) - } - -} - func TestHashPersist(t *testing.T) { db := getTestDB() diff --git a/ledis/t_kv.go b/ledis/t_kv.go index f11f525..b58a678 100644 --- a/ledis/t_kv.go +++ b/ledis/t_kv.go @@ -324,41 +324,8 @@ func (db *DB) flush() (drop int64, err error) { } //if inclusive is true, scan range [key, inf) else (key, inf) -func (db *DB) Scan(key []byte, count int, inclusive bool) ([]KVPair, error) { - var minKey []byte - if key != nil { - if err := checkKeySize(key); err != nil { - return nil, err - } - minKey = db.encodeKVKey(key) - } else { - minKey = db.encodeKVMinKey() - } - - maxKey := db.encodeKVMaxKey() - - if count <= 0 { - count = defaultScanCount - } - - v := make([]KVPair, 0, 2*count) - - rangeType := store.RangeROpen - if !inclusive { - rangeType = store.RangeOpen - } - - it := db.db.RangeLimitIterator(minKey, maxKey, rangeType, 0, count) - for ; it.Valid(); it.Next() { - if key, err := db.decodeKVKey(it.Key()); err != nil { - continue - } else { - v = append(v, KVPair{Key: key, Value: it.Value()}) - } - } - it.Close() - - return v, nil +func (db *DB) Scan(key []byte, count int, inclusive bool) ([][]byte, error) { + return db.scan(KVType, key, count, inclusive) } func (db *DB) Expire(key []byte, duration int64) (int64, error) { diff --git a/ledis/t_kv_test.go b/ledis/t_kv_test.go index 680aa5b..714e104 100644 --- a/ledis/t_kv_test.go +++ b/ledis/t_kv_test.go @@ -43,40 +43,6 @@ func TestDBKV(t *testing.T) { } -func TestDBScan(t *testing.T) { - db := getTestDB() - - db.FlushAll() - - if v, err := db.Scan(nil, 10, true); err != nil { - t.Fatal(err) - } else if len(v) != 0 { - t.Fatal(len(v)) - } - - db.Set([]byte("a"), []byte{}) - db.Set([]byte("b"), []byte{}) - db.Set([]byte("c"), []byte{}) - - if v, err := db.Scan(nil, 1, true); err != nil { - t.Fatal(err) - } else if len(v) != 1 { - t.Fatal(len(v)) - } - - if v, err := db.Scan([]byte("a"), 2, false); err != nil { - t.Fatal(err) - } else if len(v) != 2 { - t.Fatal(len(v)) - } - - if v, err := db.Scan(nil, 3, true); err != nil { - t.Fatal(err) - } else if len(v) != 3 { - t.Fatal(len(v)) - } -} - func TestKVPersist(t *testing.T) { db := getTestDB() diff --git a/ledis/t_list.go b/ledis/t_list.go index 079498e..f851013 100644 --- a/ledis/t_list.go +++ b/ledis/t_list.go @@ -484,3 +484,17 @@ func (db *DB) LPersist(key []byte) (int64, error) { err = t.Commit() return n, err } + +func (db *DB) LScan(key []byte, count int, inclusive bool) ([][]byte, error) { + return db.scan(LMetaType, key, count, inclusive) +} + +func (db *DB) lEncodeMinKey() []byte { + return db.lEncodeMetaKey(nil) +} + +func (db *DB) lEncodeMaxKey() []byte { + ek := db.lEncodeMetaKey(nil) + ek[len(ek)-1] = LMetaType + 1 + return ek +} diff --git a/ledis/t_zset.go b/ledis/t_zset.go index 964361e..f039487 100644 --- a/ledis/t_zset.go +++ b/ledis/t_zset.go @@ -1004,3 +1004,7 @@ func (db *DB) ZInterStore(destKey []byte, srcKeys [][]byte, weights []int64, agg } return int64(len(keptMembers)), nil } + +func (db *DB) ZScan(key []byte, count int, inclusive bool) ([][]byte, error) { + return db.scan(ZSizeType, key, count, inclusive) +} diff --git a/server/client_http.go b/server/client_http.go index 16f3b9a..01272d5 100644 --- a/server/client_http.go +++ b/server/client_http.go @@ -1,17 +1,16 @@ package server import ( + "encoding/json" "fmt" + "github.com/siddontang/go-bson/bson" "github.com/siddontang/go-log/log" "github.com/siddontang/ledisdb/ledis" + "github.com/ugorji/go/codec" "io" "net/http" "strconv" "strings" - - "encoding/json" - "github.com/siddontang/go-bson/bson" - "github.com/ugorji/go/codec" ) var allowedContentTypes = map[string]struct{}{