From acabb99368f090ff832588163963efd5851afd67 Mon Sep 17 00:00:00 2001 From: siddontang Date: Sat, 7 Mar 2015 10:19:15 +0800 Subject: [PATCH] move rdb package --- Godeps/Godeps.json | 4 + .../src/github.com/siddontang/rdb/LICENSE | 21 + .../src/github.com/siddontang/rdb/README.md | 3 + .../src/github.com/siddontang}/rdb/decode.go | 6 +- .../src/github.com/siddontang/rdb/digest.go | 106 +++++ .../src/github.com/siddontang}/rdb/encode.go | 2 +- .../src/github.com/siddontang/rdb/loader.go | 112 ++++++ .../github.com/siddontang/rdb/loader_test.go | 373 ++++++++++++++++++ .../github.com/siddontang}/rdb/rdb_test.go | 0 .../src/github.com/siddontang/rdb/reader.go | 332 ++++++++++++++++ .../siddontang}/rdb/wandoujia-license | 0 ledis/migrate.go | 6 +- 12 files changed, 958 insertions(+), 7 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/siddontang/rdb/LICENSE create mode 100644 Godeps/_workspace/src/github.com/siddontang/rdb/README.md rename {ledis => Godeps/_workspace/src/github.com/siddontang}/rdb/decode.go (97%) create mode 100644 Godeps/_workspace/src/github.com/siddontang/rdb/digest.go rename {ledis => Godeps/_workspace/src/github.com/siddontang}/rdb/encode.go (98%) create mode 100644 Godeps/_workspace/src/github.com/siddontang/rdb/loader.go create mode 100644 Godeps/_workspace/src/github.com/siddontang/rdb/loader_test.go rename {ledis => Godeps/_workspace/src/github.com/siddontang}/rdb/rdb_test.go (100%) create mode 100644 Godeps/_workspace/src/github.com/siddontang/rdb/reader.go rename {ledis => Godeps/_workspace/src/github.com/siddontang}/rdb/wandoujia-license (100%) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index a52e0e5..eb407f5 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -58,6 +58,10 @@ "ImportPath": "github.com/siddontang/go/sync2", "Rev": "c2b33271306fcb7c6532efceac33ec45ee2439e0" }, + { + "ImportPath": "github.com/siddontang/rdb", + "Rev": "fc89ed2e418d27e3ea76e708e54276d2b44ae9cf" + }, { "ImportPath": "github.com/syndtr/goleveldb/leveldb", "Rev": "e9e2c8f6d3b9c313fb4acaac5ab06285bcf30b04" diff --git a/Godeps/_workspace/src/github.com/siddontang/rdb/LICENSE b/Godeps/_workspace/src/github.com/siddontang/rdb/LICENSE new file mode 100644 index 0000000..c16e3af --- /dev/null +++ b/Godeps/_workspace/src/github.com/siddontang/rdb/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 siddontang + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/Godeps/_workspace/src/github.com/siddontang/rdb/README.md b/Godeps/_workspace/src/github.com/siddontang/rdb/README.md new file mode 100644 index 0000000..aa1582e --- /dev/null +++ b/Godeps/_workspace/src/github.com/siddontang/rdb/README.md @@ -0,0 +1,3 @@ +# rdb + +Handling Redis RDB format. diff --git a/ledis/rdb/decode.go b/Godeps/_workspace/src/github.com/siddontang/rdb/decode.go similarity index 97% rename from ledis/rdb/decode.go rename to Godeps/_workspace/src/github.com/siddontang/rdb/decode.go index 06cd392..865d241 100644 --- a/ledis/rdb/decode.go +++ b/Godeps/_workspace/src/github.com/siddontang/rdb/decode.go @@ -40,7 +40,7 @@ func (d *decoder) Set(key, value []byte, expiry int64) { } func (d *decoder) StartHash(key []byte, length, expiry int64) { - d.initObject(HashMap(nil)) + d.initObject(Hash(nil)) } func (d *decoder) Hset(key, field, value []byte) { @@ -50,7 +50,7 @@ func (d *decoder) Hset(key, field, value []byte) { switch h := d.obj.(type) { default: d.err = fmt.Errorf("invalid object, not a hashmap") - case HashMap: + case Hash: v := struct { Field, Value []byte }{ @@ -118,7 +118,7 @@ func (d *decoder) Zadd(key []byte, score float64, member []byte) { type String []byte type List [][]byte -type HashMap []struct { +type Hash []struct { Field, Value []byte } type Set [][]byte diff --git a/Godeps/_workspace/src/github.com/siddontang/rdb/digest.go b/Godeps/_workspace/src/github.com/siddontang/rdb/digest.go new file mode 100644 index 0000000..b59e4df --- /dev/null +++ b/Godeps/_workspace/src/github.com/siddontang/rdb/digest.go @@ -0,0 +1,106 @@ +// Copyright 2014 Wandoujia Inc. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package rdb + +import ( + "encoding/binary" + "hash" +) + +var crc64_table = [256]uint64{ + 0x0000000000000000, 0x7ad870c830358979, 0xf5b0e190606b12f2, 0x8f689158505e9b8b, + 0xc038e5739841b68f, 0xbae095bba8743ff6, 0x358804e3f82aa47d, 0x4f50742bc81f2d04, + 0xab28ecb46814fe75, 0xd1f09c7c5821770c, 0x5e980d24087fec87, 0x24407dec384a65fe, + 0x6b1009c7f05548fa, 0x11c8790fc060c183, 0x9ea0e857903e5a08, 0xe478989fa00bd371, + 0x7d08ff3b88be6f81, 0x07d08ff3b88be6f8, 0x88b81eabe8d57d73, 0xf2606e63d8e0f40a, + 0xbd301a4810ffd90e, 0xc7e86a8020ca5077, 0x4880fbd87094cbfc, 0x32588b1040a14285, + 0xd620138fe0aa91f4, 0xacf86347d09f188d, 0x2390f21f80c18306, 0x594882d7b0f40a7f, + 0x1618f6fc78eb277b, 0x6cc0863448deae02, 0xe3a8176c18803589, 0x997067a428b5bcf0, + 0xfa11fe77117cdf02, 0x80c98ebf2149567b, 0x0fa11fe77117cdf0, 0x75796f2f41224489, + 0x3a291b04893d698d, 0x40f16bccb908e0f4, 0xcf99fa94e9567b7f, 0xb5418a5cd963f206, + 0x513912c379682177, 0x2be1620b495da80e, 0xa489f35319033385, 0xde51839b2936bafc, + 0x9101f7b0e12997f8, 0xebd98778d11c1e81, 0x64b116208142850a, 0x1e6966e8b1770c73, + 0x8719014c99c2b083, 0xfdc17184a9f739fa, 0x72a9e0dcf9a9a271, 0x08719014c99c2b08, + 0x4721e43f0183060c, 0x3df994f731b68f75, 0xb29105af61e814fe, 0xc849756751dd9d87, + 0x2c31edf8f1d64ef6, 0x56e99d30c1e3c78f, 0xd9810c6891bd5c04, 0xa3597ca0a188d57d, + 0xec09088b6997f879, 0x96d1784359a27100, 0x19b9e91b09fcea8b, 0x636199d339c963f2, + 0xdf7adabd7a6e2d6f, 0xa5a2aa754a5ba416, 0x2aca3b2d1a053f9d, 0x50124be52a30b6e4, + 0x1f423fcee22f9be0, 0x659a4f06d21a1299, 0xeaf2de5e82448912, 0x902aae96b271006b, + 0x74523609127ad31a, 0x0e8a46c1224f5a63, 0x81e2d7997211c1e8, 0xfb3aa75142244891, + 0xb46ad37a8a3b6595, 0xceb2a3b2ba0eecec, 0x41da32eaea507767, 0x3b024222da65fe1e, + 0xa2722586f2d042ee, 0xd8aa554ec2e5cb97, 0x57c2c41692bb501c, 0x2d1ab4dea28ed965, + 0x624ac0f56a91f461, 0x1892b03d5aa47d18, 0x97fa21650afae693, 0xed2251ad3acf6fea, + 0x095ac9329ac4bc9b, 0x7382b9faaaf135e2, 0xfcea28a2faafae69, 0x8632586aca9a2710, + 0xc9622c4102850a14, 0xb3ba5c8932b0836d, 0x3cd2cdd162ee18e6, 0x460abd1952db919f, + 0x256b24ca6b12f26d, 0x5fb354025b277b14, 0xd0dbc55a0b79e09f, 0xaa03b5923b4c69e6, + 0xe553c1b9f35344e2, 0x9f8bb171c366cd9b, 0x10e3202993385610, 0x6a3b50e1a30ddf69, + 0x8e43c87e03060c18, 0xf49bb8b633338561, 0x7bf329ee636d1eea, 0x012b592653589793, + 0x4e7b2d0d9b47ba97, 0x34a35dc5ab7233ee, 0xbbcbcc9dfb2ca865, 0xc113bc55cb19211c, + 0x5863dbf1e3ac9dec, 0x22bbab39d3991495, 0xadd33a6183c78f1e, 0xd70b4aa9b3f20667, + 0x985b3e827bed2b63, 0xe2834e4a4bd8a21a, 0x6debdf121b863991, 0x1733afda2bb3b0e8, + 0xf34b37458bb86399, 0x8993478dbb8deae0, 0x06fbd6d5ebd3716b, 0x7c23a61ddbe6f812, + 0x3373d23613f9d516, 0x49aba2fe23cc5c6f, 0xc6c333a67392c7e4, 0xbc1b436e43a74e9d, + 0x95ac9329ac4bc9b5, 0xef74e3e19c7e40cc, 0x601c72b9cc20db47, 0x1ac40271fc15523e, + 0x5594765a340a7f3a, 0x2f4c0692043ff643, 0xa02497ca54616dc8, 0xdafce7026454e4b1, + 0x3e847f9dc45f37c0, 0x445c0f55f46abeb9, 0xcb349e0da4342532, 0xb1eceec59401ac4b, + 0xfebc9aee5c1e814f, 0x8464ea266c2b0836, 0x0b0c7b7e3c7593bd, 0x71d40bb60c401ac4, + 0xe8a46c1224f5a634, 0x927c1cda14c02f4d, 0x1d148d82449eb4c6, 0x67ccfd4a74ab3dbf, + 0x289c8961bcb410bb, 0x5244f9a98c8199c2, 0xdd2c68f1dcdf0249, 0xa7f41839ecea8b30, + 0x438c80a64ce15841, 0x3954f06e7cd4d138, 0xb63c61362c8a4ab3, 0xcce411fe1cbfc3ca, + 0x83b465d5d4a0eece, 0xf96c151de49567b7, 0x76048445b4cbfc3c, 0x0cdcf48d84fe7545, + 0x6fbd6d5ebd3716b7, 0x15651d968d029fce, 0x9a0d8ccedd5c0445, 0xe0d5fc06ed698d3c, + 0xaf85882d2576a038, 0xd55df8e515432941, 0x5a3569bd451db2ca, 0x20ed197575283bb3, + 0xc49581ead523e8c2, 0xbe4df122e51661bb, 0x3125607ab548fa30, 0x4bfd10b2857d7349, + 0x04ad64994d625e4d, 0x7e7514517d57d734, 0xf11d85092d094cbf, 0x8bc5f5c11d3cc5c6, + 0x12b5926535897936, 0x686de2ad05bcf04f, 0xe70573f555e26bc4, 0x9ddd033d65d7e2bd, + 0xd28d7716adc8cfb9, 0xa85507de9dfd46c0, 0x273d9686cda3dd4b, 0x5de5e64efd965432, + 0xb99d7ed15d9d8743, 0xc3450e196da80e3a, 0x4c2d9f413df695b1, 0x36f5ef890dc31cc8, + 0x79a59ba2c5dc31cc, 0x037deb6af5e9b8b5, 0x8c157a32a5b7233e, 0xf6cd0afa9582aa47, + 0x4ad64994d625e4da, 0x300e395ce6106da3, 0xbf66a804b64ef628, 0xc5bed8cc867b7f51, + 0x8aeeace74e645255, 0xf036dc2f7e51db2c, 0x7f5e4d772e0f40a7, 0x05863dbf1e3ac9de, + 0xe1fea520be311aaf, 0x9b26d5e88e0493d6, 0x144e44b0de5a085d, 0x6e963478ee6f8124, + 0x21c640532670ac20, 0x5b1e309b16452559, 0xd476a1c3461bbed2, 0xaeaed10b762e37ab, + 0x37deb6af5e9b8b5b, 0x4d06c6676eae0222, 0xc26e573f3ef099a9, 0xb8b627f70ec510d0, + 0xf7e653dcc6da3dd4, 0x8d3e2314f6efb4ad, 0x0256b24ca6b12f26, 0x788ec2849684a65f, + 0x9cf65a1b368f752e, 0xe62e2ad306bafc57, 0x6946bb8b56e467dc, 0x139ecb4366d1eea5, + 0x5ccebf68aecec3a1, 0x2616cfa09efb4ad8, 0xa97e5ef8cea5d153, 0xd3a62e30fe90582a, + 0xb0c7b7e3c7593bd8, 0xca1fc72bf76cb2a1, 0x45775673a732292a, 0x3faf26bb9707a053, + 0x70ff52905f188d57, 0x0a2722586f2d042e, 0x854fb3003f739fa5, 0xff97c3c80f4616dc, + 0x1bef5b57af4dc5ad, 0x61372b9f9f784cd4, 0xee5fbac7cf26d75f, 0x9487ca0fff135e26, + 0xdbd7be24370c7322, 0xa10fceec0739fa5b, 0x2e675fb4576761d0, 0x54bf2f7c6752e8a9, + 0xcdcf48d84fe75459, 0xb71738107fd2dd20, 0x387fa9482f8c46ab, 0x42a7d9801fb9cfd2, + 0x0df7adabd7a6e2d6, 0x772fdd63e7936baf, 0xf8474c3bb7cdf024, 0x829f3cf387f8795d, + 0x66e7a46c27f3aa2c, 0x1c3fd4a417c62355, 0x935745fc4798b8de, 0xe98f353477ad31a7, + 0xa6df411fbfb21ca3, 0xdc0731d78f8795da, 0x536fa08fdfd90e51, 0x29b7d047efec8728} + +type digest struct { + crc uint64 +} + +func (d *digest) update(p []byte) { + for _, b := range p { + d.crc = crc64_table[byte(d.crc)^b] ^ (d.crc >> 8) + } +} + +func newDigest() hash.Hash64 { + d := &digest{} + return d +} + +func (d *digest) Write(p []byte) (int, error) { + d.update(p) + return len(p), nil +} + +func (d *digest) Sum(in []byte) []byte { + buf := make([]byte, 8) + binary.LittleEndian.PutUint64(buf, d.crc) + return append(in, buf...) +} + +func (d *digest) Sum64() uint64 { return d.crc } +func (d *digest) BlockSize() int { return 1 } +func (d *digest) Size() int { return 8 } +func (d *digest) Reset() { d.crc = 0 } diff --git a/ledis/rdb/encode.go b/Godeps/_workspace/src/github.com/siddontang/rdb/encode.go similarity index 98% rename from ledis/rdb/encode.go rename to Godeps/_workspace/src/github.com/siddontang/rdb/encode.go index 1c33cb0..19c48c3 100644 --- a/ledis/rdb/encode.go +++ b/Godeps/_workspace/src/github.com/siddontang/rdb/encode.go @@ -15,7 +15,7 @@ func Dump(obj interface{}) ([]byte, error) { case String: e.EncodeType(rdb.TypeString) e.EncodeString(v) - case HashMap: + case Hash: e.EncodeType(rdb.TypeHash) e.EncodeLength(uint32(len(v))) diff --git a/Godeps/_workspace/src/github.com/siddontang/rdb/loader.go b/Godeps/_workspace/src/github.com/siddontang/rdb/loader.go new file mode 100644 index 0000000..22743cb --- /dev/null +++ b/Godeps/_workspace/src/github.com/siddontang/rdb/loader.go @@ -0,0 +1,112 @@ +// Copyright 2014 Wandoujia Inc. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package rdb + +import ( + "bytes" + "encoding/binary" + "fmt" + "hash" + "io" + "strconv" +) + +type Loader struct { + *rdbReader + crc hash.Hash64 + db uint32 +} + +func NewLoader(r io.Reader) *Loader { + l := &Loader{} + l.crc = newDigest() + l.rdbReader = newRdbReader(io.TeeReader(r, l.crc)) + return l +} + +func (l *Loader) LoadHeader() error { + header := make([]byte, 9) + if err := l.readFull(header); err != nil { + return err + } + if !bytes.Equal(header[:5], []byte("REDIS")) { + return fmt.Errorf("verify magic string, invalid file format") + } + if version, err := strconv.ParseInt(string(header[5:]), 10, 64); err != nil { + return err + } else if version <= 0 || version > Version { + return fmt.Errorf("verify version, invalid RDB version number %d", version) + } + return nil +} + +func (l *Loader) LoadChecksum() error { + crc1 := l.crc.Sum64() + if crc2, err := l.readUint64(); err != nil { + return err + } else if crc1 != crc2 { + return fmt.Errorf("checksum validation failed") + } + return nil +} + +type Entry struct { + DB uint32 + Key []byte + ValDump []byte + ExpireAt uint64 +} + +func (l *Loader) LoadEntry() (entry *Entry, err error) { + var expireat uint64 + for { + var otype byte + if otype, err = l.readByte(); err != nil { + return + } + switch otype { + case rdbFlagExpiryMS: + if expireat, err = l.readUint64(); err != nil { + return + } + case rdbFlagExpiry: + var sec uint32 + if sec, err = l.readUint32(); err != nil { + return + } + expireat = uint64(sec) * 1000 + case rdbFlagSelectDB: + if l.db, err = l.readLength(); err != nil { + return + } + case rdbFlagEOF: + return + default: + var key, obj []byte + if key, err = l.readString(); err != nil { + return + } + if obj, err = l.readObject(otype); err != nil { + return + } + entry = &Entry{} + entry.DB = l.db + entry.Key = key + entry.ValDump = createValDump(otype, obj) + entry.ExpireAt = expireat + return + } + } +} + +func createValDump(otype byte, obj []byte) []byte { + var b bytes.Buffer + c := newDigest() + w := io.MultiWriter(&b, c) + w.Write([]byte{otype}) + w.Write(obj) + binary.Write(w, binary.LittleEndian, uint16(Version)) + binary.Write(w, binary.LittleEndian, c.Sum64()) + return b.Bytes() +} diff --git a/Godeps/_workspace/src/github.com/siddontang/rdb/loader_test.go b/Godeps/_workspace/src/github.com/siddontang/rdb/loader_test.go new file mode 100644 index 0000000..db8e2ee --- /dev/null +++ b/Godeps/_workspace/src/github.com/siddontang/rdb/loader_test.go @@ -0,0 +1,373 @@ +// Copyright 2014 Wandoujia Inc. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package rdb + +import ( + "bytes" + "encoding/hex" + "fmt" + "math" + "strconv" + "strings" + "testing" +) + +func AssertNoError(t *testing.T, err error) { + if err == nil { + return + } + + t.Fatal(err) +} + +func Assert(t *testing.T, b bool) { + if b { + return + } + t.Fatal("assertion failed") +} + +func DecodeHexRdb(t *testing.T, s string, n int) map[string]*Entry { + p, err := hex.DecodeString(strings.NewReplacer("\t", "", "\r", "", "\n", "", " ", "").Replace(s)) + AssertNoError(t, err) + r := bytes.NewReader(p) + l := NewLoader(r) + AssertNoError(t, l.LoadHeader()) + entries := make(map[string]*Entry) + var i int = 0 + for { + e, err := l.LoadEntry() + AssertNoError(t, err) + if e == nil { + break + } + Assert(t, e.DB == 0) + entries[string(e.Key)] = e + i++ + } + AssertNoError(t, l.LoadChecksum()) + Assert(t, r.Len() == 0) + Assert(t, len(entries) == i && i == n) + return entries +} + +func getobj(t *testing.T, entries map[string]*Entry, key string) (*Entry, interface{}) { + e := entries[key] + Assert(t, e != nil) + val, err := DecodeDump(e.ValDump) + AssertNoError(t, err) + return e, val +} + +/* +#!/bin/bash +./redis-cli flushall +for i in 1 255 256 65535 65536 2147483647 2147483648 4294967295 4294967296 -2147483648; do + ./redis-cli set string_${i} ${i} +done +./redis-cli save && xxd -p -c 32 dump.rdb +*/ +func TestLoadIntString(t *testing.T) { + s := ` + 524544495330303036fe00000a737472696e675f323535c1ff00000873747269 + 6e675f31c0010011737472696e675f343239343936373239360a343239343936 + 373239360011737472696e675f343239343936373239350a3432393439363732 + 39350012737472696e675f2d32313437343833363438c200000080000c737472 + 696e675f3635353335c2ffff00000011737472696e675f323134373438333634 + 380a32313437343833363438000c737472696e675f3635353336c20000010000 + 0a737472696e675f323536c100010011737472696e675f323134373438333634 + 37c2ffffff7fffe49d9f131fb5c3b5 + ` + values := []int{1, 255, 256, 65535, 65536, 2147483647, 2147483648, 4294967295, 4294967296, -2147483648} + entries := DecodeHexRdb(t, s, len(values)) + for _, value := range values { + key := fmt.Sprintf("string_%d", value) + _, obj := getobj(t, entries, key) + val := obj.(String) + Assert(t, bytes.Equal([]byte(val), []byte(strconv.Itoa(value)))) + } +} + +/* +#!/bin/bash +./redis-cli flushall +./redis-cli set string_ttls string_ttls +./redis-cli expireat string_ttls 1500000000 +./redis-cli set string_ttlms string_ttlms +./redis-cli pexpireat string_ttlms 1500000000000 +./redis-cli save && xxd -p -c 32 dump.rdb +*/ +func TestLoadStringTTL(t *testing.T) { + s := ` + 524544495330303036fe00fc0098f73e5d010000000c737472696e675f74746c + 6d730c737472696e675f74746c6d73fc0098f73e5d010000000b737472696e67 + 5f74746c730b737472696e675f74746c73ffd15acd935a3fe949 + ` + expireat := uint64(1500000000000) + entries := DecodeHexRdb(t, s, 2) + keys := []string{"string_ttls", "string_ttlms"} + for _, key := range keys { + e, obj := getobj(t, entries, key) + val := obj.(String) + Assert(t, bytes.Equal([]byte(val), []byte(key))) + Assert(t, e.ExpireAt == expireat) + } +} + +/* +#!/bin/bash +s="01" +for ((i=0;i<15;i++)); do + s=$s$s +done +./redis-cli flushall +./redis-cli set string_long $s +./redis-cli save && xxd -p -c 32 dump.rdb +*/ +func TestLoadLongString(t *testing.T) { + s := ` + 524544495330303036fe00000b737472696e675f6c6f6e67c342f28000010000 + 02303130e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0 + ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01 + e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff + 01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0 + ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01 + e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff + 01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0 + ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01 + e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff + 01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0 + ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01 + e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff + 01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0 + ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01 + e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff + 01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0 + ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01 + e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff + 01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0 + ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01 + e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff + 01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0 + ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01e0ff01 + e0ff01e0ff01e0ff01e0ff01e03201013031ffdfdb02bd6d5da5e6 + ` + entries := DecodeHexRdb(t, s, 1) + _, obj := getobj(t, entries, "string_long") + val := []byte(obj.(String)) + for i := 0; i < (1 << 15); i++ { + var c uint8 = '0' + if i%2 != 0 { + c = '1' + } + Assert(t, val[i] == c) + } +} + +/* +#!/bin/bash +./redis-cli flushall +for ((i=0;i<256;i++)); do + ./redis-cli rpush list_lzf 0 + ./redis-cli rpush list_lzf 1 +done +./redis-cli save && xxd -p -c 32 dump.rdb +*/ +func TestLoadListZipmap(t *testing.T) { + s := ` + 524544495330303036fe000a086c6973745f6c7a66c31f440b040b0400000820 + 0306000200f102f202e0ff03e1ff07e1ff07e1d90701f2ffff6a1c2d51c02301 + 16 + ` + entries := DecodeHexRdb(t, s, 1) + _, obj := getobj(t, entries, "list_lzf") + val := obj.(List) + Assert(t, len(val) == 512) + for i := 0; i < 256; i++ { + var s string = "0" + if i%2 != 0 { + s = "1" + } + Assert(t, string(val[i]) == s) + } +} + +/* +#!/bin/bash +./redis-cli flushall +for ((i=0;i<32;i++)); do + ./redis-cli rpush list ${i} +done +./redis-cli save && xxd -p -c 32 dump.rdb +*/ +func TestLoadList(t *testing.T) { + s := ` + 524544495330303036fe0001046c69737420c000c001c002c003c004c005c006 + c007c008c009c00ac00bc00cc00dc00ec00fc010c011c012c013c014c015c016 + c017c018c019c01ac01bc01cc01dc01ec01fff756ea1fa90adefe3 + ` + entries := DecodeHexRdb(t, s, 1) + _, obj := getobj(t, entries, "list") + val := obj.(List) + Assert(t, len(val) == 32) + for i := 0; i < 32; i++ { + Assert(t, string(val[i]) == strconv.Itoa(i)) + } +} + +/* +#!/bin/bash +./redis-cli flushall +for ((i=0;i<16;i++)); do + ./redis-cli sadd set1 ${i} +done +for ((i=0;i<32;i++)); do + ./redis-cli sadd set2 ${i} +done +./redis-cli save && xxd -p -c 32 dump.rdb +*/ +func TestLoadSetAndSetIntset(t *testing.T) { + s := ` + 524544495330303036fe0002047365743220c016c00dc01bc012c01ac004c014 + c002c017c01dc01cc013c019c01ec008c006c000c001c007c00fc009c01fc00e + c003c00ac015c010c00bc018c011c00cc0050b04736574312802000000100000 + 0000000100020003000400050006000700080009000a000b000c000d000e000f + 00ff3a0a9697324d19c3 + ` + entries := DecodeHexRdb(t, s, 2) + + _, obj1 := getobj(t, entries, "set1") + val1 := obj1.(Set) + set1 := make(map[string]bool) + for _, mem := range val1 { + set1[string(mem)] = true + } + Assert(t, len(set1) == 16) + Assert(t, len(set1) == len(val1)) + for i := 0; i < 16; i++ { + _, ok := set1[strconv.Itoa(i)] + Assert(t, ok) + } + + _, obj2 := getobj(t, entries, "set2") + val2 := obj2.(Set) + set2 := make(map[string]bool) + for _, mem := range val2 { + set2[string(mem)] = true + } + Assert(t, len(set2) == 32) + Assert(t, len(set2) == len(val2)) + for i := 0; i < 32; i++ { + _, ok := set2[strconv.Itoa(i)] + Assert(t, ok) + } +} + +/* +#!/bin/bash +./redis-cli flushall +for ((i=0;i<16;i++)); do + ./redis-cli hset hash1 ${i} +done +for ((i=-16;i<16;i++)); do + ./redis-cli hset hash2 ${i} +done +./redis-cli save && xxd -p -c 32 dump.rdb +*/ +func TestLoadHashAndHashZiplist(t *testing.T) { + s := ` + 524544495330303036fe000405686173683220c00dc00dc0fcc0fcc0ffc0ffc0 + 04c004c002c002c0fbc0fbc0f0c0f0c0f9c0f9c008c008c0fac0fac006c006c0 + 00c000c001c001c0fec0fec007c007c0f6c0f6c00fc00fc009c009c0f7c0f7c0 + fdc0fdc0f1c0f1c0f2c0f2c0f3c0f3c00ec00ec003c003c00ac00ac00bc00bc0 + f8c0f8c00cc00cc0f5c0f5c0f4c0f4c005c0050d056861736831405151000000 + 4d000000200000f102f102f202f202f302f302f402f402f502f502f602f602f7 + 02f702f802f802f902f902fa02fa02fb02fb02fc02fc02fd02fd02fe0d03fe0d + 03fe0e03fe0e03fe0f03fe0fffffa423d3036c15e534 + ` + entries := DecodeHexRdb(t, s, 2) + + _, obj1 := getobj(t, entries, "hash1") + val1 := obj1.(Hash) + hash1 := make(map[string]string) + for _, ent := range val1 { + hash1[string(ent.Field)] = string(ent.Value) + } + Assert(t, len(hash1) == 16) + Assert(t, len(hash1) == len(val1)) + for i := 0; i < 16; i++ { + s := strconv.Itoa(i) + Assert(t, hash1[s] == s) + } + + _, obj2 := getobj(t, entries, "hash2") + val2 := obj2.(Hash) + hash2 := make(map[string]string) + for _, ent := range val2 { + hash2[string(ent.Field)] = string(ent.Value) + } + Assert(t, len(hash2) == 32) + Assert(t, len(hash2) == len(val2)) + for i := -16; i < 16; i++ { + s := strconv.Itoa(i) + Assert(t, hash2[s] == s) + } +} + +/* +#!/bin/bash +./redis-cli flushall +for ((i=0;i<16;i++)); do + ./redis-cli zadd zset1 ${i} ${i} +done +for ((i=0;i<32;i++)); do + ./redis-cli zadd zset2 -${i} ${i} +done +./redis-cli save && xxd -p -c 32 dump.rdb +*/ +func TestLoadZSetAndZSetZiplist(t *testing.T) { + s := ` + 524544495330303036fe0003057a7365743220c016032d3232c00d032d3133c0 + 1b032d3237c012032d3138c01a032d3236c004022d34c014032d3230c002022d + 32c017032d3233c01d032d3239c01c032d3238c013032d3139c019032d3235c0 + 1e032d3330c008022d38c006022d36c000022d30c001022d31c007022d37c009 + 022d39c00f032d3135c01f032d3331c00e032d3134c003022d33c00a032d3130 + c015032d3231c010032d3136c00b032d3131c018032d3234c011032d3137c00c + 032d3132c005022d350c057a736574314051510000004d000000200000f102f1 + 02f202f202f302f302f402f402f502f502f602f602f702f702f802f802f902f9 + 02fa02fa02fb02fb02fc02fc02fd02fd02fe0d03fe0d03fe0e03fe0e03fe0f03 + fe0fffff2addedbf4f5a8f93 + ` + entries := DecodeHexRdb(t, s, 2) + + _, obj1 := getobj(t, entries, "zset1") + val1 := obj1.(ZSet) + zset1 := make(map[string]float64) + for _, ent := range val1 { + zset1[string(ent.Member)] = ent.Score + } + Assert(t, len(zset1) == 16) + Assert(t, len(zset1) == len(val1)) + for i := 0; i < 16; i++ { + s := strconv.Itoa(i) + score, ok := zset1[s] + Assert(t, ok) + Assert(t, math.Abs(score-float64(i)) < 1e-10) + } + + _, obj2 := getobj(t, entries, "zset2") + val2 := obj2.(ZSet) + zset2 := make(map[string]float64) + for _, ent := range val2 { + zset2[string(ent.Member)] = ent.Score + } + Assert(t, len(zset2) == 32) + Assert(t, len(zset2) == len(val2)) + for i := 0; i < 32; i++ { + s := strconv.Itoa(i) + score, ok := zset2[s] + Assert(t, ok) + Assert(t, math.Abs(score+float64(i)) < 1e-10) + } +} diff --git a/ledis/rdb/rdb_test.go b/Godeps/_workspace/src/github.com/siddontang/rdb/rdb_test.go similarity index 100% rename from ledis/rdb/rdb_test.go rename to Godeps/_workspace/src/github.com/siddontang/rdb/rdb_test.go diff --git a/Godeps/_workspace/src/github.com/siddontang/rdb/reader.go b/Godeps/_workspace/src/github.com/siddontang/rdb/reader.go new file mode 100644 index 0000000..89ae9ed --- /dev/null +++ b/Godeps/_workspace/src/github.com/siddontang/rdb/reader.go @@ -0,0 +1,332 @@ +// Copyright 2014 Wandoujia Inc. All Rights Reserved. +// Licensed under the MIT (MIT-LICENSE.txt) license. + +package rdb + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "math" + "strconv" +) + +const ( + Version = 6 +) + +const ( + rdbTypeString = 0 + rdbTypeList = 1 + rdbTypeSet = 2 + rdbTypeZSet = 3 + rdbTypeHash = 4 + + rdbTypeHashZipmap = 9 + rdbTypeListZiplist = 10 + rdbTypeSetIntset = 11 + rdbTypeZSetZiplist = 12 + rdbTypeHashZiplist = 13 + + rdbFlagExpiryMS = 0xfc + rdbFlagExpiry = 0xfd + rdbFlagSelectDB = 0xfe + rdbFlagEOF = 0xff +) + +const ( + rdb6bitLen = 0 + rdb14bitLen = 1 + rdb32bitLen = 2 + rdbEncVal = 3 + + rdbEncInt8 = 0 + rdbEncInt16 = 1 + rdbEncInt32 = 2 + rdbEncLZF = 3 + + rdbZiplist6bitlenString = 0 + rdbZiplist14bitlenString = 1 + rdbZiplist32bitlenString = 2 + + rdbZiplistInt16 = 0xc0 + rdbZiplistInt32 = 0xd0 + rdbZiplistInt64 = 0xe0 + rdbZiplistInt24 = 0xf0 + rdbZiplistInt8 = 0xfe + rdbZiplistInt4 = 15 +) + +type rdbReader struct { + raw io.Reader + buf [8]byte + nread int64 +} + +func newRdbReader(r io.Reader) *rdbReader { + return &rdbReader{raw: r} +} + +func (r *rdbReader) Read(p []byte) (int, error) { + n, err := r.raw.Read(p) + r.nread += int64(n) + return n, err +} + +func (r *rdbReader) offset() int64 { + return r.nread +} + +func (r *rdbReader) readObject(otype byte) ([]byte, error) { + var b bytes.Buffer + r = newRdbReader(io.TeeReader(r, &b)) + switch otype { + default: + return nil, fmt.Errorf("unknown object-type %02x", otype) + case rdbTypeHashZipmap: + fallthrough + case rdbTypeListZiplist: + fallthrough + case rdbTypeSetIntset: + fallthrough + case rdbTypeZSetZiplist: + fallthrough + case rdbTypeHashZiplist: + fallthrough + case rdbTypeString: + if _, err := r.readString(); err != nil { + return nil, err + } + case rdbTypeList, rdbTypeSet: + if n, err := r.readLength(); err != nil { + return nil, err + } else { + for i := 0; i < int(n); i++ { + if _, err := r.readString(); err != nil { + return nil, err + } + } + } + case rdbTypeZSet: + if n, err := r.readLength(); err != nil { + return nil, err + } else { + for i := 0; i < int(n); i++ { + if _, err := r.readString(); err != nil { + return nil, err + } + if _, err := r.readFloat(); err != nil { + return nil, err + } + } + } + case rdbTypeHash: + if n, err := r.readLength(); err != nil { + return nil, err + } else { + for i := 0; i < int(n); i++ { + if _, err := r.readString(); err != nil { + return nil, err + } + if _, err := r.readString(); err != nil { + return nil, err + } + } + } + } + return b.Bytes(), nil +} + +func (r *rdbReader) readString() ([]byte, error) { + length, encoded, err := r.readEncodedLength() + if err != nil { + return nil, err + } + if !encoded { + return r.readBytes(int(length)) + } + switch t := uint8(length); t { + default: + return nil, fmt.Errorf("invalid encoded-string %02x", t) + case rdbEncInt8: + i, err := r.readInt8() + return []byte(strconv.FormatInt(int64(i), 10)), err + case rdbEncInt16: + i, err := r.readInt16() + return []byte(strconv.FormatInt(int64(i), 10)), err + case rdbEncInt32: + i, err := r.readInt32() + return []byte(strconv.FormatInt(int64(i), 10)), err + case rdbEncLZF: + var inlen, outlen uint32 + if inlen, err = r.readLength(); err != nil { + return nil, err + } + if outlen, err = r.readLength(); err != nil { + return nil, err + } + if in, err := r.readBytes(int(inlen)); err != nil { + return nil, err + } else { + return lzfDecompress(in, int(outlen)) + } + } +} + +func (r *rdbReader) readEncodedLength() (length uint32, encoded bool, err error) { + var u uint8 + if u, err = r.readUint8(); err != nil { + return + } + length = uint32(u & 0x3f) + switch u >> 6 { + case rdb6bitLen: + case rdb14bitLen: + u, err = r.readUint8() + length = (length << 8) + uint32(u) + case rdbEncVal: + encoded = true + default: + length, err = r.readUint32BigEndian() + } + return +} + +func (r *rdbReader) readLength() (uint32, error) { + length, encoded, err := r.readEncodedLength() + if err == nil && encoded { + err = fmt.Errorf("encoded-length") + } + return length, err +} + +func (r *rdbReader) readFloat() (float64, error) { + u, err := r.readUint8() + if err != nil { + return 0, err + } + switch u { + case 253: + return math.NaN(), nil + case 254: + return math.Inf(0), nil + case 255: + return math.Inf(-1), nil + default: + if b, err := r.readBytes(int(u)); err != nil { + return 0, err + } else { + v, err := strconv.ParseFloat(string(b), 64) + return v, err + } + } +} + +func (r *rdbReader) readByte() (byte, error) { + b := r.buf[:1] + _, err := r.Read(b) + return b[0], err +} + +func (r *rdbReader) readFull(p []byte) error { + _, err := io.ReadFull(r, p) + return err +} + +func (r *rdbReader) readBytes(n int) ([]byte, error) { + p := make([]byte, n) + return p, r.readFull(p) +} + +func (r *rdbReader) readUint8() (uint8, error) { + b, err := r.readByte() + return uint8(b), err +} + +func (r *rdbReader) readUint16() (uint16, error) { + b := r.buf[:2] + err := r.readFull(b) + return binary.LittleEndian.Uint16(b), err +} + +func (r *rdbReader) readUint32() (uint32, error) { + b := r.buf[:4] + err := r.readFull(b) + return binary.LittleEndian.Uint32(b), err +} + +func (r *rdbReader) readUint64() (uint64, error) { + b := r.buf[:8] + err := r.readFull(b) + return binary.LittleEndian.Uint64(b), err +} + +func (r *rdbReader) readUint32BigEndian() (uint32, error) { + b := r.buf[:4] + err := r.readFull(b) + return binary.BigEndian.Uint32(b), err +} + +func (r *rdbReader) readInt8() (int8, error) { + u, err := r.readUint8() + return int8(u), err +} + +func (r *rdbReader) readInt16() (int16, error) { + u, err := r.readUint16() + return int16(u), err +} + +func (r *rdbReader) readInt32() (int32, error) { + u, err := r.readUint32() + return int32(u), err +} + +func (r *rdbReader) readInt64() (int64, error) { + u, err := r.readUint64() + return int64(u), err +} + +func (r *rdbReader) readInt32BigEndian() (int32, error) { + u, err := r.readUint32BigEndian() + return int32(u), err +} + +func lzfDecompress(in []byte, outlen int) (out []byte, err error) { + defer func() { + if x := recover(); x != nil { + err = fmt.Errorf("decompress exception: %v", x) + } + }() + out = make([]byte, outlen) + i, o := 0, 0 + for i < len(in) { + ctrl := int(in[i]) + i++ + if ctrl < 32 { + for x := 0; x <= ctrl; x++ { + out[o] = in[i] + i++ + o++ + } + } else { + length := ctrl >> 5 + if length == 7 { + length = length + int(in[i]) + i++ + } + ref := o - ((ctrl & 0x1f) << 8) - int(in[i]) - 1 + i++ + for x := 0; x <= length+1; x++ { + out[o] = out[ref] + ref++ + o++ + } + } + } + if o != outlen { + return nil, fmt.Errorf("decompress length is %d != expected %d", o, outlen) + } + return out, nil +} diff --git a/ledis/rdb/wandoujia-license b/Godeps/_workspace/src/github.com/siddontang/rdb/wandoujia-license similarity index 100% rename from ledis/rdb/wandoujia-license rename to Godeps/_workspace/src/github.com/siddontang/rdb/wandoujia-license diff --git a/ledis/migrate.go b/ledis/migrate.go index b6df026..0db22b4 100644 --- a/ledis/migrate.go +++ b/ledis/migrate.go @@ -2,7 +2,7 @@ package ledis import ( "fmt" - "github.com/siddontang/ledisdb/ledis/rdb" + "github.com/siddontang/rdb" ) /* @@ -43,7 +43,7 @@ func (db *DB) HDump(key []byte) ([]byte, error) { return nil, err } - o := make(rdb.HashMap, len(v)) + o := make(rdb.Hash, len(v)) for i := 0; i < len(v); i++ { o[i].Field = v[i].Field o[i].Value = v[i].Value @@ -110,7 +110,7 @@ func (db *DB) Restore(key []byte, ttl int64, data []byte) error { return err } } - case rdb.HashMap: + case rdb.Hash: //first clear old key if _, err = db.HClear(key); err != nil { return err