From 990b2ee69dc68a6de582fcb34ecd79fe9226547f Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 14 May 2014 16:35:49 +0800 Subject: [PATCH] refactor, use go-leveldb, you must re execute bootstrap.sh to go get go-leveldb you must remove your old leveldb path because hash size format changed --- bootstrap.sh | 3 +- ledis/app.go | 2 +- ledis/client.go | 19 ++++++------ ledis/cmd_hash.go | 7 ++--- ledis/cmd_kv.go | 9 ++---- ledis/cmd_list.go | 11 +++---- ledis/cmd_zset.go | 27 ++++++++--------- ledis/config.go | 2 +- ledis/t_hash.go | 76 ++++++++++++++++++++++------------------------- ledis/t_kv.go | 6 ++-- ledis/t_list.go | 2 +- ledis/t_zset.go | 9 +++--- ledis/tx.go | 2 +- ledis/util.go | 46 +++++++++++++++++++++++++--- 14 files changed, 119 insertions(+), 102 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index fffd283..b121d0d 100644 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -3,7 +3,6 @@ . ./dev.sh go get -u github.com/jmhodges/levigo -go get -u github.com/siddontang/golib/leveldb +go get -u github.com/siddontang/go-leveldb/leveldb go get -u github.com/siddontang/golib/log -go get -u github.com/siddontang/golib/hack go get -u github.com/garyburd/redigo/redis diff --git a/ledis/app.go b/ledis/app.go index 50f0481..a0a6757 100644 --- a/ledis/app.go +++ b/ledis/app.go @@ -1,7 +1,7 @@ package ledis import ( - "github.com/siddontang/golib/leveldb" + "github.com/siddontang/go-leveldb/leveldb" "net" "strings" ) diff --git a/ledis/client.go b/ledis/client.go index 6a49429..c71d16a 100644 --- a/ledis/client.go +++ b/ledis/client.go @@ -3,7 +3,6 @@ package ledis import ( "bufio" "errors" - "github.com/siddontang/golib/hack" "github.com/siddontang/golib/log" "io" "net" @@ -92,7 +91,7 @@ func (c *client) readRequest() ([][]byte, error) { } var nparams int - if nparams, err = strconv.Atoi(hack.String(l[1:])); err != nil { + if nparams, err = strconv.Atoi(String(l[1:])); err != nil { return nil, err } else if nparams <= 0 { return nil, errReadRequest @@ -109,7 +108,7 @@ func (c *client) readRequest() ([][]byte, error) { return nil, errReadRequest } else if l[0] == '$' { //handle resp string - if n, err = strconv.Atoi(hack.String(l[1:])); err != nil { + if n, err = strconv.Atoi(String(l[1:])); err != nil { return nil, err } else if n == -1 { req = append(req, nil) @@ -139,7 +138,7 @@ func (c *client) handleRequest(req [][]byte) { if len(req) == 0 { err = ErrEmptyCommand } else { - c.cmd = strings.ToLower(hack.String(req[0])) + c.cmd = strings.ToLower(String(req[0])) c.args = req[1:] f, ok := regCmds[c.cmd] @@ -161,23 +160,23 @@ func (c *client) handleRequest(req [][]byte) { } func (c *client) writeError(err error) { - c.wb.Write(hack.Slice("-ERR")) + c.wb.Write(Slice("-ERR")) if err != nil { c.wb.WriteByte(' ') - c.wb.Write(hack.Slice(err.Error())) + c.wb.Write(Slice(err.Error())) } c.wb.Write(Delims) } func (c *client) writeStatus(status string) { c.wb.WriteByte('+') - c.wb.Write(hack.Slice(status)) + c.wb.Write(Slice(status)) c.wb.Write(Delims) } func (c *client) writeInteger(n int64) { c.wb.WriteByte(':') - c.wb.Write(hack.Slice(strconv.FormatInt(n, 10))) + c.wb.Write(Slice(strconv.FormatInt(n, 10))) c.wb.Write(Delims) } @@ -186,7 +185,7 @@ func (c *client) writeBulk(b []byte) { if b == nil { c.wb.Write(NullBulk) } else { - c.wb.Write(hack.Slice(strconv.Itoa(len(b)))) + c.wb.Write(Slice(strconv.Itoa(len(b)))) c.wb.Write(Delims) c.wb.Write(b) } @@ -200,7 +199,7 @@ func (c *client) writeArray(ay []interface{}) { c.wb.Write(NullArray) c.wb.Write(Delims) } else { - c.wb.Write(hack.Slice(strconv.Itoa(len(ay)))) + c.wb.Write(Slice(strconv.Itoa(len(ay)))) c.wb.Write(Delims) for i := 0; i < len(ay); i++ { diff --git a/ledis/cmd_hash.go b/ledis/cmd_hash.go index 91cc192..86b424f 100644 --- a/ledis/cmd_hash.go +++ b/ledis/cmd_hash.go @@ -1,9 +1,6 @@ package ledis -import ( - "github.com/siddontang/golib/hack" - "strconv" -) +import () func hsetCommand(c *client) error { args := c.args @@ -90,7 +87,7 @@ func hincrbyCommand(c *client) error { return ErrCmdParams } - delta, err := strconv.ParseInt(hack.String(args[2]), 10, 64) + delta, err := StrInt64(args[2], nil) if err != nil { return err } diff --git a/ledis/cmd_kv.go b/ledis/cmd_kv.go index d7a08b5..381ec8c 100644 --- a/ledis/cmd_kv.go +++ b/ledis/cmd_kv.go @@ -1,9 +1,6 @@ package ledis -import ( - "github.com/siddontang/golib/hack" - "strconv" -) +import () func getCommand(c *client) error { args := c.args @@ -115,7 +112,7 @@ func incrbyCommand(c *client) error { return ErrCmdParams } - delta, err := strconv.ParseInt(hack.String(args[1]), 10, 64) + delta, err := StrInt64(args[1], nil) if err != nil { return err } @@ -135,7 +132,7 @@ func decrbyCommand(c *client) error { return ErrCmdParams } - delta, err := strconv.ParseInt(hack.String(args[1]), 10, 64) + delta, err := StrInt64(args[1], nil) if err != nil { return err } diff --git a/ledis/cmd_list.go b/ledis/cmd_list.go index 09ed7a0..e41f520 100644 --- a/ledis/cmd_list.go +++ b/ledis/cmd_list.go @@ -1,9 +1,6 @@ package ledis -import ( - "github.com/siddontang/golib/hack" - "strconv" -) +import () func lpushCommand(c *client) error { args := c.args @@ -86,7 +83,7 @@ func lindexCommand(c *client) error { return ErrCmdParams } - index, err := strconv.ParseInt(hack.String(args[1]), 10, 64) + index, err := StrInt64(args[1], nil) if err != nil { return err } @@ -110,12 +107,12 @@ func lrangeCommand(c *client) error { var stop int64 var err error - start, err = strconv.ParseInt(hack.String(args[1]), 10, 64) + start, err = StrInt64(args[1], nil) if err != nil { return err } - stop, err = strconv.ParseInt(hack.String(args[2]), 10, 64) + stop, err = StrInt64(args[2], nil) if err != nil { return err } diff --git a/ledis/cmd_zset.go b/ledis/cmd_zset.go index 4035629..4f84e24 100644 --- a/ledis/cmd_zset.go +++ b/ledis/cmd_zset.go @@ -2,7 +2,6 @@ package ledis import ( "errors" - "github.com/siddontang/golib/hack" "math" "strconv" "strings" @@ -31,7 +30,7 @@ func zaddCommand(c *client) error { args = args[1:] params := make([]interface{}, len(args)) for i := 0; i < len(params); i += 2 { - score, err := strconv.ParseInt(hack.String(args[i]), 10, 64) + score, err := StrInt64(args[i], nil) if err != nil { return err } @@ -102,7 +101,7 @@ func zincrbyCommand(c *client) error { key := args[0] - delta, err := strconv.ParseInt(hack.String(args[1]), 10, 64) + delta, err := StrInt64(args[1], nil) if err != nil { return err } @@ -117,7 +116,7 @@ func zincrbyCommand(c *client) error { } func zparseScoreRange(minBuf []byte, maxBuf []byte) (min int64, max int64, err error) { - if strings.ToLower(hack.String(minBuf)) == "-inf" { + if strings.ToLower(String(minBuf)) == "-inf" { min = math.MinInt64 } else { var lopen bool = false @@ -131,7 +130,7 @@ func zparseScoreRange(minBuf []byte, maxBuf []byte) (min int64, max int64, err e return } - min, err = strconv.ParseInt(hack.String(minBuf), 10, 64) + min, err = StrInt64(minBuf, nil) if err != nil { return } @@ -146,7 +145,7 @@ func zparseScoreRange(minBuf []byte, maxBuf []byte) (min int64, max int64, err e } } - if strings.ToLower(hack.String(maxBuf)) == "+inf" { + if strings.ToLower(String(maxBuf)) == "+inf" { max = math.MaxInt64 } else { var ropen = false @@ -160,7 +159,7 @@ func zparseScoreRange(minBuf []byte, maxBuf []byte) (min int64, max int64, err e return } - max, err = strconv.ParseInt(hack.String(maxBuf), 10, 64) + max, err = StrInt64(maxBuf, nil) if err != nil { return } @@ -288,11 +287,11 @@ func zremrangebyscoreCommand(c *client) error { func zparseRange(c *client, key []byte, startBuf []byte, stopBuf []byte) (offset int, limit int, err error) { var start int var stop int - if start, err = strconv.Atoi(hack.String(startBuf)); err != nil { + if start, err = strconv.Atoi(String(startBuf)); err != nil { return } - if stop, err = strconv.Atoi(hack.String(stopBuf)); err != nil { + if stop, err = strconv.Atoi(String(stopBuf)); err != nil { return } @@ -354,7 +353,7 @@ func zrangeGeneric(c *client, reverse bool) error { args = args[3:] var withScores bool = false - if len(args) > 0 && strings.ToLower(hack.String(args[0])) == "withscores" { + if len(args) > 0 && strings.ToLower(String(args[0])) == "withscores" { withScores = true } @@ -390,7 +389,7 @@ func zrangebyscoreGeneric(c *client, reverse bool) error { var withScores bool = false - if len(args) > 0 && strings.ToLower(hack.String(args[0])) == "withscores" { + if len(args) > 0 && strings.ToLower(String(args[0])) == "withscores" { withScores = true args = args[1:] } @@ -403,15 +402,15 @@ func zrangebyscoreGeneric(c *client, reverse bool) error { return ErrCmdParams } - if strings.ToLower(hack.String(args[0])) != "limit" { + if strings.ToLower(String(args[0])) != "limit" { return ErrCmdParams } - if offset, err = strconv.Atoi(hack.String(args[1])); err != nil { + if offset, err = strconv.Atoi(String(args[1])); err != nil { return ErrCmdParams } - if limit, err = strconv.Atoi(hack.String(args[2])); err != nil { + if limit, err = strconv.Atoi(String(args[2])); err != nil { return ErrCmdParams } } diff --git a/ledis/config.go b/ledis/config.go index a303a73..9172e77 100644 --- a/ledis/config.go +++ b/ledis/config.go @@ -2,7 +2,7 @@ package ledis import ( "encoding/json" - "github.com/siddontang/golib/leveldb" + "github.com/siddontang/go-leveldb/leveldb" "io/ioutil" ) diff --git a/ledis/t_hash.go b/ledis/t_hash.go index 2c4d488..55a2568 100644 --- a/ledis/t_hash.go +++ b/ledis/t_hash.go @@ -3,9 +3,7 @@ package ledis import ( "encoding/binary" "errors" - "github.com/siddontang/golib/hack" - "github.com/siddontang/golib/leveldb" - "strconv" + "github.com/siddontang/go-leveldb/leveldb" ) var errHashKey = errors.New("invalid hash key") @@ -90,26 +88,21 @@ func decode_hash_key(ek []byte) ([]byte, []byte, error) { } func (a *App) hash_len(key []byte) (int64, error) { - return a.db.GetInt(encode_hsize_key(key)) + return Int64(a.db.Get(encode_hsize_key(key))) } func (a *App) hash_setItem(key []byte, field []byte, value []byte) (int64, error) { t := a.hashTx ek := encode_hash_key(key, field) - sk := encode_hsize_key(key) - - size, err := a.db.GetInt(sk) - if err != nil { - return 0, err - } var n int64 = 1 if v, _ := a.db.Get(ek); v != nil { n = 0 } else { - size++ - t.Put(sk, hack.Slice(strconv.FormatInt(size, 10))) + if _, err := a.hash_incrSize(key, 1); err != nil { + return 0, err + } } t.Put(ek, value) @@ -137,30 +130,26 @@ func (a *App) hash_get(key []byte, field []byte) ([]byte, error) { } func (a *App) hash_mset(key []byte, args [][]byte) error { - sk := encode_hsize_key(key) - t := a.hashTx t.Lock() defer t.Unlock() - size, err := a.db.GetInt(sk) - if err != nil { - return err - } - + var num int64 = 0 for i := 0; i < len(args); i += 2 { ek := encode_hash_key(key, args[i]) if v, _ := a.db.Get(ek); v == nil { - size++ + num++ } t.Put(ek, args[i+1]) } - t.Put(sk, hack.Slice(strconv.FormatInt(size, 10))) + if _, err := a.hash_incrSize(key, num); err != nil { + return err + } //todo add binglog - err = t.Commit() + err := t.Commit() return err } @@ -179,17 +168,10 @@ func (a *App) hash_mget(key []byte, args [][]byte) ([]interface{}, error) { } func (a *App) hash_del(key []byte, args [][]byte) (int64, error) { - sk := encode_hsize_key(key) - t := a.hashTx t.Lock() defer t.Unlock() - size, err := a.db.GetInt(sk) - if err != nil { - return 0, err - } - var num int64 = 0 for i := 0; i < len(args); i++ { ek := encode_hash_key(key, args[i]) @@ -199,22 +181,38 @@ func (a *App) hash_del(key []byte, args [][]byte) (int64, error) { continue } else { num++ - size-- t.Delete(ek) } } - if size <= 0 { - t.Delete(sk) - } else { - t.Put(sk, hack.Slice(strconv.FormatInt(size, 10))) + if _, err := a.hash_incrSize(key, -num); err != nil { + return 0, err } - err = t.Commit() + err := t.Commit() return num, err } +func (a *App) hash_incrSize(key []byte, delta int64) (int64, error) { + t := a.hashTx + sk := encode_hsize_key(key) + size, err := Int64(a.db.Get(sk)) + if err != nil { + return 0, err + } else { + size += delta + if size <= 0 { + size = 0 + t.Delete(sk) + } else { + t.Put(sk, PutInt64(size)) + } + } + + return size, nil +} + func (a *App) hash_incrby(key []byte, field []byte, delta int64) (int64, error) { t := a.hashTx t.Lock() @@ -223,18 +221,14 @@ func (a *App) hash_incrby(key []byte, field []byte, delta int64) (int64, error) ek := encode_hash_key(key, field) var n int64 = 0 - v, err := a.db.Get(ek) + n, err := StrInt64(a.db.Get(ek)) if err != nil { return 0, err - } else if v != nil { - if n, err = strconv.ParseInt(hack.String(v), 10, 64); err != nil { - return 0, err - } } n += delta - _, err = a.hash_setItem(key, field, hack.Slice(strconv.FormatInt(n, 10))) + _, err = a.hash_setItem(key, field, StrPutInt64(n)) if err != nil { return 0, err } diff --git a/ledis/t_kv.go b/ledis/t_kv.go index f575e6d..2fd146f 100644 --- a/ledis/t_kv.go +++ b/ledis/t_kv.go @@ -2,8 +2,6 @@ package ledis import ( "errors" - "github.com/siddontang/golib/hack" - "strconv" ) var errKVKey = errors.New("invalid encode kv key") @@ -117,14 +115,14 @@ func (a *App) kv_incr(key []byte, delta int64) (int64, error) { defer t.Unlock() var n int64 - n, err = a.db.GetInt(key) + n, err = StrInt64(a.db.Get(key)) if err != nil { return 0, err } n += delta - t.Put(key, hack.Slice(strconv.FormatInt(n, 10))) + t.Put(key, StrPutInt64(n)) //todo binlog diff --git a/ledis/t_list.go b/ledis/t_list.go index 8515b8d..af6f484 100644 --- a/ledis/t_list.go +++ b/ledis/t_list.go @@ -3,7 +3,7 @@ package ledis import ( "encoding/binary" "errors" - "github.com/siddontang/golib/leveldb" + "github.com/siddontang/go-leveldb/leveldb" ) const ( diff --git a/ledis/t_zset.go b/ledis/t_zset.go index 6635e1d..ea0fae0 100644 --- a/ledis/t_zset.go +++ b/ledis/t_zset.go @@ -4,8 +4,7 @@ import ( "bytes" "encoding/binary" "errors" - "github.com/siddontang/golib/hack" - "github.com/siddontang/golib/leveldb" + "github.com/siddontang/go-leveldb/leveldb" "strconv" ) @@ -262,7 +261,7 @@ func (a *App) zset_score(key []byte, member []byte) ([]byte, error) { return nil, err } - return hack.Slice(strconv.FormatInt(score, 10)), nil + return Slice(strconv.FormatInt(score, 10)), nil } func (a *App) zset_rem(key []byte, args [][]byte) (int64, error) { @@ -319,7 +318,7 @@ func (a *App) zset_incrby(key []byte, delta int64, member []byte) ([]byte, error t.Put(encode_zscore_key(key, member, score), []byte{}) err = t.Commit() - return hack.Slice(strconv.FormatInt(score, 10)), err + return Slice(strconv.FormatInt(score, 10)), err } func (a *App) zset_count(key []byte, min int64, max int64) (int64, error) { @@ -471,7 +470,7 @@ func (a *App) zset_range(key []byte, min int64, max int64, withScores bool, offs v = append(v, m) if withScores { - v = append(v, hack.Slice(strconv.FormatInt(s, 10))) + v = append(v, Slice(strconv.FormatInt(s, 10))) } } diff --git a/ledis/tx.go b/ledis/tx.go index 35c5180..41d0133 100644 --- a/ledis/tx.go +++ b/ledis/tx.go @@ -1,7 +1,7 @@ package ledis import ( - "github.com/siddontang/golib/leveldb" + "github.com/siddontang/go-leveldb/leveldb" "sync" ) diff --git a/ledis/util.go b/ledis/util.go index e5d6efd..a5a1289 100644 --- a/ledis/util.go +++ b/ledis/util.go @@ -3,11 +3,34 @@ package ledis import ( "encoding/binary" "errors" - "github.com/siddontang/golib/hack" + "reflect" + "strconv" + "unsafe" ) var errIntNumber = errors.New("invalid integer") +// no copy to change slice to string +// use your own risk +func String(b []byte) (s string) { + pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + pstring := (*reflect.StringHeader)(unsafe.Pointer(&s)) + pstring.Data = pbytes.Data + pstring.Len = pbytes.Len + return +} + +// no copy to change string to slice +// use your own risk +func Slice(s string) (b []byte) { + pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + pstring := (*reflect.StringHeader)(unsafe.Pointer(&s)) + pbytes.Data = pstring.Data + pbytes.Len = pstring.Len + pbytes.Cap = pstring.Len + return +} + func Int64(v []byte, err error) (int64, error) { if err != nil { return 0, err @@ -21,9 +44,24 @@ func Int64(v []byte, err error) (int64, error) { } func PutInt64(v int64) []byte { - return hack.Int64Slice(v) + var b []byte + pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b)) + pbytes.Data = uintptr(unsafe.Pointer(&v)) + pbytes.Len = 8 + pbytes.Cap = 8 + return b } -func PutInt32(v int32) []byte { - return hack.Int32Slice(v) +func StrInt64(v []byte, err error) (int64, error) { + if err != nil { + return 0, err + } else if v == nil { + return 0, nil + } else { + return strconv.ParseInt(String(v), 10, 64) + } +} + +func StrPutInt64(v int64) []byte { + return Slice(strconv.FormatInt(v, 10)) }