From b245fa417210e329d11e056251a295e3eaa02eaa Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 9 May 2020 21:30:50 +0800 Subject: [PATCH] Add a new command ledis and move all commands as the sub commands (#382) * Add a new command ledis and move all commands as the sub commands --- cmd/benchmark.go | 335 +++++++++++++++++++++++++++++ cmd/{ledis-cli/const.go => cli.go} | 211 +++++++++++++++++- cmd/dump.go | 57 +++++ cmd/ledis-benchmark/main.go | 329 +--------------------------- cmd/ledis-cli/main.go | 204 +----------------- cmd/ledis-dump/main.go | 54 +---- cmd/ledis-load/main.go | 59 +---- cmd/ledis-repair/main.go | 32 +-- cmd/ledis-server/main.go | 115 +--------- cmd/ledis/main.go | 55 +++++ cmd/load.go | 62 ++++++ cmd/repair.go | 35 +++ cmd/server.go | 121 +++++++++++ 13 files changed, 886 insertions(+), 783 deletions(-) create mode 100644 cmd/benchmark.go rename cmd/{ledis-cli/const.go => cli.go} (60%) create mode 100644 cmd/dump.go create mode 100644 cmd/ledis/main.go create mode 100644 cmd/load.go create mode 100644 cmd/repair.go create mode 100644 cmd/server.go diff --git a/cmd/benchmark.go b/cmd/benchmark.go new file mode 100644 index 0000000..088cb79 --- /dev/null +++ b/cmd/benchmark.go @@ -0,0 +1,335 @@ +package cmd + +import ( + "flag" + "fmt" + "math/rand" + "runtime" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/siddontang/goredis" +) + +var wg sync.WaitGroup +var client *goredis.Client +var loop int = 0 + +func waitBench(c *goredis.PoolConn, cmd string, args ...interface{}) { + _, err := c.Do(strings.ToUpper(cmd), args...) + if err != nil { + fmt.Printf("do %s error %s\n", cmd, err.Error()) + } +} + +func bench(cmd string, f func(c *goredis.PoolConn)) { + wg.Add(*clients) + + t1 := time.Now() + for i := 0; i < *clients; i++ { + go func() { + c, _ := client.Get() + for j := 0; j < loop; j++ { + f(c) + } + c.Close() + wg.Done() + }() + } + + wg.Wait() + + t2 := time.Now() + + d := t2.Sub(t1) + + fmt.Printf("%s: %s %0.3f micros/op, %0.2fop/s\n", + cmd, + d.String(), + float64(d.Nanoseconds()/1e3)/float64(*number), + float64(*number)/d.Seconds()) +} + +var kvSetBase int64 = 0 +var kvGetBase int64 = 0 +var kvIncrBase int64 = 0 +var kvDelBase int64 = 0 + +func benchSet() { + f := func(c *goredis.PoolConn) { + value := make([]byte, *valueSize) + n := atomic.AddInt64(&kvSetBase, 1) + waitBench(c, "SET", n, value) + } + + bench("set", f) +} + +func benchGet() { + f := func(c *goredis.PoolConn) { + n := atomic.AddInt64(&kvGetBase, 1) + waitBench(c, "GET", n) + } + + bench("get", f) +} + +func benchRandGet() { + f := func(c *goredis.PoolConn) { + n := rand.Int() % *number + waitBench(c, "GET", n) + } + + bench("randget", f) +} + +func benchDel() { + f := func(c *goredis.PoolConn) { + n := atomic.AddInt64(&kvDelBase, 1) + waitBench(c, "DEL", n) + } + + bench("del", f) +} + +func benchPushList() { + f := func(c *goredis.PoolConn) { + value := make([]byte, 100) + waitBench(c, "RPUSH", "mytestlist", value) + } + + bench("rpush", f) +} + +func benchRangeList10() { + f := func(c *goredis.PoolConn) { + waitBench(c, "LRANGE", "mytestlist", 0, 10) + } + + bench("lrange10", f) +} + +func benchRangeList50() { + f := func(c *goredis.PoolConn) { + waitBench(c, "LRANGE", "mytestlist", 0, 50) + } + + bench("lrange50", f) +} + +func benchRangeList100() { + f := func(c *goredis.PoolConn) { + waitBench(c, "LRANGE", "mytestlist", 0, 100) + } + + bench("lrange100", f) +} + +func benchPopList() { + f := func(c *goredis.PoolConn) { + waitBench(c, "LPOP", "mytestlist") + } + + bench("lpop", f) +} + +var hashSetBase int64 = 0 +var hashIncrBase int64 = 0 +var hashGetBase int64 = 0 +var hashDelBase int64 = 0 + +func benchHset() { + f := func(c *goredis.PoolConn) { + value := make([]byte, 100) + + n := atomic.AddInt64(&hashSetBase, 1) + waitBench(c, "HSET", "myhashkey", n, value) + } + + bench("hset", f) +} + +func benchHGet() { + f := func(c *goredis.PoolConn) { + n := atomic.AddInt64(&hashGetBase, 1) + waitBench(c, "HGET", "myhashkey", n) + } + + bench("hget", f) +} + +func benchHRandGet() { + f := func(c *goredis.PoolConn) { + n := rand.Int() % *number + waitBench(c, "HGET", "myhashkey", n) + } + + bench("hrandget", f) +} + +func benchHDel() { + f := func(c *goredis.PoolConn) { + n := atomic.AddInt64(&hashDelBase, 1) + waitBench(c, "HDEL", "myhashkey", n) + } + + bench("hdel", f) +} + +var zsetAddBase int64 = 0 +var zsetDelBase int64 = 0 +var zsetIncrBase int64 = 0 + +func benchZAdd() { + f := func(c *goredis.PoolConn) { + member := make([]byte, 16) + n := atomic.AddInt64(&zsetAddBase, 1) + waitBench(c, "ZADD", "myzsetkey", n, member) + } + + bench("zadd", f) +} + +func benchZDel() { + f := func(c *goredis.PoolConn) { + n := atomic.AddInt64(&zsetDelBase, 1) + waitBench(c, "ZREM", "myzsetkey", n) + } + + bench("zrem", f) +} + +func benchZIncr() { + f := func(c *goredis.PoolConn) { + n := atomic.AddInt64(&zsetIncrBase, 1) + waitBench(c, "ZINCRBY", "myzsetkey", 1, n) + } + + bench("zincrby", f) +} + +func benchZRangeByScore() { + f := func(c *goredis.PoolConn) { + waitBench(c, "ZRANGEBYSCORE", "myzsetkey", 0, rand.Int(), "withscores", "limit", rand.Int()%100, 100) + } + + bench("zrangebyscore", f) +} + +func benchZRangeByRank() { + f := func(c *goredis.PoolConn) { + waitBench(c, "ZRANGE", "myzsetkey", 0, rand.Int()%100) + } + + bench("zrange", f) +} + +func benchZRevRangeByScore() { + f := func(c *goredis.PoolConn) { + waitBench(c, "ZREVRANGEBYSCORE", "myzsetkey", 0, rand.Int(), "withscores", "limit", rand.Int()%100, 100) + } + + bench("zrevrangebyscore", f) +} + +func benchZRevRangeByRank() { + f := func(c *goredis.PoolConn) { + waitBench(c, "ZREVRANGE", "myzsetkey", 0, rand.Int()%100) + } + + bench("zrevrange", f) +} + +var benchIP *string +var benchPort *int +var number *int +var clients = flag.Int("c", 50, "number of clients") +var round = flag.Int("r", 1, "benchmark round number") +var valueSize = flag.Int("vsize", 100, "kv value size") +var tests = flag.String("t", "set,get,randget,del,lpush,lrange,lpop,hset,hget,hdel,zadd,zincr,zrange,zrevrange,zdel", "only run the comma separated list of tests") + +func CmdBenchmark() { + benchIP = flag.String("ip", "127.0.0.1", "redis/ledis/ssdb server ip") + number = flag.Int("n", 1000, "request number") + benchPort = flag.Int("port", 6380, "redis/ledis/ssdb server port") + + runtime.GOMAXPROCS(runtime.NumCPU()) + + flag.Parse() + + if *number <= 0 { + panic("invalid number") + } + + if *clients <= 0 || *number < *clients { + panic("invalid client number") + } + + loop = *number / *clients + + addr := fmt.Sprintf("%s:%d", *benchIP, *benchPort) + + client = goredis.NewClient(addr, "") + client.SetReadBufferSize(10240) + client.SetWriteBufferSize(10240) + client.SetMaxIdleConns(16) + + for i := 0; i < *clients; i++ { + c, _ := client.Get() + c.Close() + } + + if *round <= 0 { + *round = 1 + } + + ts := strings.Split(*tests, ",") + + for i := 0; i < *round; i++ { + for _, s := range ts { + switch strings.ToLower(s) { + case "set": + benchSet() + case "get": + benchGet() + case "randget": + benchRandGet() + case "del": + benchDel() + case "lpush": + benchPushList() + case "lrange": + benchRangeList10() + benchRangeList50() + benchRangeList100() + case "lpop": + benchPopList() + case "hset": + benchHset() + case "hget": + benchHGet() + benchHRandGet() + case "hdel": + benchHDel() + case "zadd": + benchZAdd() + case "zincr": + benchZIncr() + case "zrange": + benchZRangeByRank() + benchZRangeByScore() + case "zrevrange": + //rev is too slow in leveldb, rocksdb or other + //maybe disable for huge data benchmark + benchZRevRangeByRank() + benchZRevRangeByScore() + case "zdel": + benchZDel() + } + } + + println("") + } +} diff --git a/cmd/ledis-cli/const.go b/cmd/cli.go similarity index 60% rename from cmd/ledis-cli/const.go rename to cmd/cli.go index 66c05bb..ae28c6b 100644 --- a/cmd/ledis-cli/const.go +++ b/cmd/cli.go @@ -1,5 +1,17 @@ -//This file was generated by .tools/generate_commands.py on Sat Oct 28 2017 18:15:49 -0500 -package main +//This file was generated by .tools/generate_commands.py on Sat Oct 28 2017 18:15:49 -0500 +package cmd + +import ( + "flag" + "fmt" + "os" + "regexp" + "strconv" + "strings" + + "github.com/peterh/liner" + "github.com/siddontang/goredis" +) var helpCommands = [][]string{ {"APPEND", "key value", "KV"}, @@ -141,3 +153,198 @@ var helpCommands = [][]string{ {"ZTTL", "key", "ZSet"}, {"ZUNIONSTORE", "destkey numkeys key [key ...] [WEIGHTS weight [weight ...]] [AGGREGATE SUM|MIN|MAX]", "ZSet"}, } + +var ip = flag.String("h", "127.0.0.1", "ledisdb server ip (default 127.0.0.1)") +var port = flag.Int("p", 6380, "ledisdb server port (default 6380)") +var socket = flag.String("s", "", "ledisdb server socket, overwrite ip and port") + +var ( + line *liner.State + historyPath = "/tmp/ledis-cli" +) + +func CmdCli() { + var dbn = flag.Int("n", 0, "ledisdb database number(default 0)") + + flag.Parse() + + line = liner.NewLiner() + defer line.Close() + + line.SetCtrlCAborts(true) + + setCompletionHandler() + loadHisotry() + + defer saveHisotry() + + var addr string + if len(*socket) > 0 { + addr = *socket + } else { + addr = fmt.Sprintf("%s:%d", *ip, *port) + } + + c := goredis.NewClient(addr, "") + c.SetMaxIdleConns(1) + sendSelect(c, *dbn) + + reg, _ := regexp.Compile(`'.*?'|".*?"|\S+`) + + prompt := "" + + for { + if *dbn > 0 && *dbn < 16 { + prompt = fmt.Sprintf("%s[%d]> ", addr, *dbn) + } else { + prompt = fmt.Sprintf("%s> ", addr) + } + + cmd, err := line.Prompt(prompt) + if err != nil { + fmt.Printf("%s\n", err.Error()) + return + } + + cmds := reg.FindAllString(cmd, -1) + if len(cmds) == 0 { + continue + } else { + line.AppendHistory(cmd) + + args := make([]interface{}, len(cmds[1:])) + + for i := range args { + args[i] = strings.Trim(string(cmds[1+i]), "\"'") + } + + cmd := strings.ToLower(cmds[0]) + if cmd == "help" || cmd == "?" { + printHelp(cmds) + } else { + r, err := c.Do(cmds[0], args...) + + if err == nil && strings.ToLower(cmds[0]) == "select" { + *dbn, _ = strconv.Atoi(cmds[1]) + } + + if err != nil { + fmt.Printf("%s", err.Error()) + } else { + if cmd == "info" { + printInfo(r.([]byte)) + } else { + printReply(0, r) + } + } + + fmt.Printf("\n") + } + + } + } +} + +func printInfo(s []byte) { + fmt.Printf("%s", s) +} + +func printReply(level int, reply interface{}) { + switch reply := reply.(type) { + case int64: + fmt.Printf("(integer) %d", reply) + case string: + fmt.Printf("%s", reply) + case []byte: + fmt.Printf("%q", reply) + case nil: + fmt.Printf("(nil)") + case goredis.Error: + fmt.Printf("%s", string(reply)) + case []interface{}: + for i, v := range reply { + if i != 0 { + fmt.Printf("%s", strings.Repeat(" ", level*4)) + } + + s := fmt.Sprintf("%d) ", i+1) + fmt.Printf("%-4s", s) + + printReply(level+1, v) + if i != len(reply)-1 { + fmt.Printf("\n") + } + } + default: + fmt.Printf("invalid ledis reply") + } +} + +func printGenericHelp() { + msg := + `ledis-cli +Type: "help " for help on + ` + fmt.Println(msg) +} + +func printCommandHelp(arr []string) { + fmt.Println() + fmt.Printf("\t%s %s \n", arr[0], arr[1]) + fmt.Printf("\tGroup: %s \n", arr[2]) + fmt.Println() +} + +func printHelp(cmds []string) { + args := cmds[1:] + if len(args) == 0 { + printGenericHelp() + } else if len(args) > 1 { + fmt.Println() + } else { + cmd := strings.ToUpper(args[0]) + for i := 0; i < len(helpCommands); i++ { + if helpCommands[i][0] == cmd { + printCommandHelp(helpCommands[i]) + } + } + } +} + +func sendSelect(client *goredis.Client, index int) { + if index > 16 || index < 0 { + index = 0 + fmt.Println("index out of range, should less than 16") + } + _, err := client.Do("select", index) + if err != nil { + fmt.Printf("%s\n", err.Error()) + } +} + +func setCompletionHandler() { + line.SetCompleter(func(line string) (c []string) { + for _, i := range helpCommands { + if strings.HasPrefix(i[0], strings.ToLower(line)) { + c = append(c, i[0]) + } + } + return + }) +} + +func loadHisotry() { + if f, err := os.Open(historyPath); err == nil { + line.ReadHistory(f) + f.Close() + } +} + +func saveHisotry() { + if f, err := os.Create(historyPath); err != nil { + fmt.Printf("Error writing history file, err: %v", err) + } else { + line.WriteHistory(f) + f.Close() + } +} diff --git a/cmd/dump.go b/cmd/dump.go new file mode 100644 index 0000000..97bccb0 --- /dev/null +++ b/cmd/dump.go @@ -0,0 +1,57 @@ +package cmd + +import ( + "flag" + "fmt" + "os" + + "github.com/siddontang/goredis" +) + +func CmdDump() { + var host = flag.String("host", "127.0.0.1", "ledis server host") + var dumpPort = flag.Int("port", 6380, "ledis server port") + var sock = flag.String("sock", "", "ledis unix socket domain") + var dumpFile = flag.String("o", "./ledis.dump", "dump file to save") + + flag.Parse() + + var err error + var f *os.File + + if f, err = os.OpenFile(*dumpFile, os.O_CREATE|os.O_WRONLY, 0644); err != nil { + println(err.Error()) + return + } + + defer f.Close() + + var addr string + if len(*sock) != 0 { + addr = *sock + } else { + addr = fmt.Sprintf("%s:%d", *host, *dumpPort) + } + + c, err := goredis.ConnectWithSize(addr, 16*1024, 4096) + if err != nil { + println(err.Error()) + return + } + + defer c.Close() + + println("dump begin") + + if err = c.Send("fullsync"); err != nil { + println(err.Error()) + return + } + + if err = c.ReceiveBulkTo(f); err != nil { + println(err.Error()) + return + } + + println("dump end") +} diff --git a/cmd/ledis-benchmark/main.go b/cmd/ledis-benchmark/main.go index 8b14ee0..8e0379e 100644 --- a/cmd/ledis-benchmark/main.go +++ b/cmd/ledis-benchmark/main.go @@ -1,332 +1,7 @@ package main -import ( - "flag" - "fmt" - "math/rand" - "runtime" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/siddontang/goredis" -) - -var ip = flag.String("ip", "127.0.0.1", "redis/ledis/ssdb server ip") -var port = flag.Int("port", 6380, "redis/ledis/ssdb server port") -var number = flag.Int("n", 1000, "request number") -var clients = flag.Int("c", 50, "number of clients") -var round = flag.Int("r", 1, "benchmark round number") -var valueSize = flag.Int("vsize", 100, "kv value size") -var tests = flag.String("t", "set,get,randget,del,lpush,lrange,lpop,hset,hget,hdel,zadd,zincr,zrange,zrevrange,zdel", "only run the comma separated list of tests") -var wg sync.WaitGroup - -var client *goredis.Client -var loop int = 0 - -func waitBench(c *goredis.PoolConn, cmd string, args ...interface{}) { - _, err := c.Do(strings.ToUpper(cmd), args...) - if err != nil { - fmt.Printf("do %s error %s\n", cmd, err.Error()) - } - -} - -func bench(cmd string, f func(c *goredis.PoolConn)) { - wg.Add(*clients) - - t1 := time.Now() - for i := 0; i < *clients; i++ { - go func() { - c, _ := client.Get() - for j := 0; j < loop; j++ { - f(c) - } - c.Close() - wg.Done() - }() - } - - wg.Wait() - - t2 := time.Now() - - d := t2.Sub(t1) - - fmt.Printf("%s: %s %0.3f micros/op, %0.2fop/s\n", - cmd, - d.String(), - float64(d.Nanoseconds()/1e3)/float64(*number), - float64(*number)/d.Seconds()) -} - -var kvSetBase int64 = 0 -var kvGetBase int64 = 0 -var kvIncrBase int64 = 0 -var kvDelBase int64 = 0 - -func benchSet() { - f := func(c *goredis.PoolConn) { - value := make([]byte, *valueSize) - n := atomic.AddInt64(&kvSetBase, 1) - waitBench(c, "SET", n, value) - } - - bench("set", f) -} - -func benchGet() { - f := func(c *goredis.PoolConn) { - n := atomic.AddInt64(&kvGetBase, 1) - waitBench(c, "GET", n) - } - - bench("get", f) -} - -func benchRandGet() { - f := func(c *goredis.PoolConn) { - n := rand.Int() % *number - waitBench(c, "GET", n) - } - - bench("randget", f) -} - -func benchDel() { - f := func(c *goredis.PoolConn) { - n := atomic.AddInt64(&kvDelBase, 1) - waitBench(c, "DEL", n) - } - - bench("del", f) -} - -func benchPushList() { - f := func(c *goredis.PoolConn) { - value := make([]byte, 100) - waitBench(c, "RPUSH", "mytestlist", value) - } - - bench("rpush", f) -} - -func benchRangeList10() { - f := func(c *goredis.PoolConn) { - waitBench(c, "LRANGE", "mytestlist", 0, 10) - } - - bench("lrange10", f) -} - -func benchRangeList50() { - f := func(c *goredis.PoolConn) { - waitBench(c, "LRANGE", "mytestlist", 0, 50) - } - - bench("lrange50", f) -} - -func benchRangeList100() { - f := func(c *goredis.PoolConn) { - waitBench(c, "LRANGE", "mytestlist", 0, 100) - } - - bench("lrange100", f) -} - -func benchPopList() { - f := func(c *goredis.PoolConn) { - waitBench(c, "LPOP", "mytestlist") - } - - bench("lpop", f) -} - -var hashSetBase int64 = 0 -var hashIncrBase int64 = 0 -var hashGetBase int64 = 0 -var hashDelBase int64 = 0 - -func benchHset() { - f := func(c *goredis.PoolConn) { - value := make([]byte, 100) - - n := atomic.AddInt64(&hashSetBase, 1) - waitBench(c, "HSET", "myhashkey", n, value) - } - - bench("hset", f) -} - -func benchHGet() { - f := func(c *goredis.PoolConn) { - n := atomic.AddInt64(&hashGetBase, 1) - waitBench(c, "HGET", "myhashkey", n) - } - - bench("hget", f) -} - -func benchHRandGet() { - f := func(c *goredis.PoolConn) { - n := rand.Int() % *number - waitBench(c, "HGET", "myhashkey", n) - } - - bench("hrandget", f) -} - -func benchHDel() { - f := func(c *goredis.PoolConn) { - n := atomic.AddInt64(&hashDelBase, 1) - waitBench(c, "HDEL", "myhashkey", n) - } - - bench("hdel", f) -} - -var zsetAddBase int64 = 0 -var zsetDelBase int64 = 0 -var zsetIncrBase int64 = 0 - -func benchZAdd() { - f := func(c *goredis.PoolConn) { - member := make([]byte, 16) - n := atomic.AddInt64(&zsetAddBase, 1) - waitBench(c, "ZADD", "myzsetkey", n, member) - } - - bench("zadd", f) -} - -func benchZDel() { - f := func(c *goredis.PoolConn) { - n := atomic.AddInt64(&zsetDelBase, 1) - waitBench(c, "ZREM", "myzsetkey", n) - } - - bench("zrem", f) -} - -func benchZIncr() { - f := func(c *goredis.PoolConn) { - n := atomic.AddInt64(&zsetIncrBase, 1) - waitBench(c, "ZINCRBY", "myzsetkey", 1, n) - } - - bench("zincrby", f) -} - -func benchZRangeByScore() { - f := func(c *goredis.PoolConn) { - waitBench(c, "ZRANGEBYSCORE", "myzsetkey", 0, rand.Int(), "withscores", "limit", rand.Int()%100, 100) - } - - bench("zrangebyscore", f) -} - -func benchZRangeByRank() { - f := func(c *goredis.PoolConn) { - waitBench(c, "ZRANGE", "myzsetkey", 0, rand.Int()%100) - } - - bench("zrange", f) -} - -func benchZRevRangeByScore() { - f := func(c *goredis.PoolConn) { - waitBench(c, "ZREVRANGEBYSCORE", "myzsetkey", 0, rand.Int(), "withscores", "limit", rand.Int()%100, 100) - } - - bench("zrevrangebyscore", f) -} - -func benchZRevRangeByRank() { - f := func(c *goredis.PoolConn) { - waitBench(c, "ZREVRANGE", "myzsetkey", 0, rand.Int()%100) - } - - bench("zrevrange", f) -} +import "github.com/ledisdb/ledisdb/cmd" func main() { - runtime.GOMAXPROCS(runtime.NumCPU()) - - flag.Parse() - - if *number <= 0 { - panic("invalid number") - } - - if *clients <= 0 || *number < *clients { - panic("invalid client number") - } - - loop = *number / *clients - - addr := fmt.Sprintf("%s:%d", *ip, *port) - - client = goredis.NewClient(addr, "") - client.SetReadBufferSize(10240) - client.SetWriteBufferSize(10240) - client.SetMaxIdleConns(16) - - for i := 0; i < *clients; i++ { - c, _ := client.Get() - c.Close() - } - - if *round <= 0 { - *round = 1 - } - - ts := strings.Split(*tests, ",") - - for i := 0; i < *round; i++ { - for _, s := range ts { - switch strings.ToLower(s) { - case "set": - benchSet() - case "get": - benchGet() - case "randget": - benchRandGet() - case "del": - benchDel() - case "lpush": - benchPushList() - case "lrange": - benchRangeList10() - benchRangeList50() - benchRangeList100() - case "lpop": - benchPopList() - case "hset": - benchHset() - case "hget": - benchHGet() - benchHRandGet() - case "hdel": - benchHDel() - case "zadd": - benchZAdd() - case "zincr": - benchZIncr() - case "zrange": - benchZRangeByRank() - benchZRangeByScore() - case "zrevrange": - //rev is too slow in leveldb, rocksdb or other - //maybe disable for huge data benchmark - benchZRevRangeByRank() - benchZRevRangeByScore() - case "zdel": - benchZDel() - } - } - - println("") - } + cmd.CmdBenchmark() } diff --git a/cmd/ledis-cli/main.go b/cmd/ledis-cli/main.go index 71ce579..22a2e8e 100644 --- a/cmd/ledis-cli/main.go +++ b/cmd/ledis-cli/main.go @@ -1,207 +1,7 @@ package main -import ( - "flag" - "fmt" - "os" - "regexp" - "strconv" - "strings" - - "github.com/peterh/liner" - "github.com/siddontang/goredis" -) - -var ip = flag.String("h", "127.0.0.1", "ledisdb server ip (default 127.0.0.1)") -var port = flag.Int("p", 6380, "ledisdb server port (default 6380)") -var socket = flag.String("s", "", "ledisdb server socket, overwrite ip and port") -var dbn = flag.Int("n", 0, "ledisdb database number(default 0)") - -var ( - line *liner.State - historyPath = "/tmp/ledis-cli" -) +import "github.com/ledisdb/ledisdb/cmd" func main() { - flag.Parse() - - line = liner.NewLiner() - defer line.Close() - - line.SetCtrlCAborts(true) - - setCompletionHandler() - loadHisotry() - - defer saveHisotry() - - var addr string - if len(*socket) > 0 { - addr = *socket - } else { - addr = fmt.Sprintf("%s:%d", *ip, *port) - } - - c := goredis.NewClient(addr, "") - c.SetMaxIdleConns(1) - sendSelect(c, *dbn) - - reg, _ := regexp.Compile(`'.*?'|".*?"|\S+`) - - prompt := "" - - for { - if *dbn > 0 && *dbn < 16 { - prompt = fmt.Sprintf("%s[%d]> ", addr, *dbn) - } else { - prompt = fmt.Sprintf("%s> ", addr) - } - - cmd, err := line.Prompt(prompt) - if err != nil { - fmt.Printf("%s\n", err.Error()) - return - } - - cmds := reg.FindAllString(cmd, -1) - if len(cmds) == 0 { - continue - } else { - line.AppendHistory(cmd) - - args := make([]interface{}, len(cmds[1:])) - - for i := range args { - args[i] = strings.Trim(string(cmds[1+i]), "\"'") - } - - cmd := strings.ToLower(cmds[0]) - if cmd == "help" || cmd == "?" { - printHelp(cmds) - } else { - r, err := c.Do(cmds[0], args...) - - if err == nil && strings.ToLower(cmds[0]) == "select" { - *dbn, _ = strconv.Atoi(cmds[1]) - } - - if err != nil { - fmt.Printf("%s", err.Error()) - } else { - if cmd == "info" { - printInfo(r.([]byte)) - } else { - printReply(0, r) - } - } - - fmt.Printf("\n") - } - - } - } -} - -func printInfo(s []byte) { - fmt.Printf("%s", s) -} - -func printReply(level int, reply interface{}) { - switch reply := reply.(type) { - case int64: - fmt.Printf("(integer) %d", reply) - case string: - fmt.Printf("%s", reply) - case []byte: - fmt.Printf("%q", reply) - case nil: - fmt.Printf("(nil)") - case goredis.Error: - fmt.Printf("%s", string(reply)) - case []interface{}: - for i, v := range reply { - if i != 0 { - fmt.Printf("%s", strings.Repeat(" ", level*4)) - } - - s := fmt.Sprintf("%d) ", i+1) - fmt.Printf("%-4s", s) - - printReply(level+1, v) - if i != len(reply)-1 { - fmt.Printf("\n") - } - } - default: - fmt.Printf("invalid ledis reply") - } -} - -func printGenericHelp() { - msg := - `ledis-cli -Type: "help " for help on - ` - fmt.Println(msg) -} - -func printCommandHelp(arr []string) { - fmt.Println() - fmt.Printf("\t%s %s \n", arr[0], arr[1]) - fmt.Printf("\tGroup: %s \n", arr[2]) - fmt.Println() -} - -func printHelp(cmds []string) { - args := cmds[1:] - if len(args) == 0 { - printGenericHelp() - } else if len(args) > 1 { - fmt.Println() - } else { - cmd := strings.ToUpper(args[0]) - for i := 0; i < len(helpCommands); i++ { - if helpCommands[i][0] == cmd { - printCommandHelp(helpCommands[i]) - } - } - } -} - -func sendSelect(client *goredis.Client, index int) { - if index > 16 || index < 0 { - index = 0 - fmt.Println("index out of range, should less than 16") - } - _, err := client.Do("select", index) - if err != nil { - fmt.Printf("%s\n", err.Error()) - } -} - -func setCompletionHandler() { - line.SetCompleter(func(line string) (c []string) { - for _, i := range helpCommands { - if strings.HasPrefix(i[0], strings.ToLower(line)) { - c = append(c, i[0]) - } - } - return - }) -} - -func loadHisotry() { - if f, err := os.Open(historyPath); err == nil { - line.ReadHistory(f) - f.Close() - } -} - -func saveHisotry() { - if f, err := os.Create(historyPath); err != nil { - fmt.Printf("Error writing history file, err: %v", err) - } else { - line.WriteHistory(f) - f.Close() - } + cmd.CmdCli() } diff --git a/cmd/ledis-dump/main.go b/cmd/ledis-dump/main.go index 8bf49d5..855f826 100644 --- a/cmd/ledis-dump/main.go +++ b/cmd/ledis-dump/main.go @@ -1,57 +1,7 @@ package main -import ( - "flag" - "fmt" - "os" - - "github.com/siddontang/goredis" -) - -var host = flag.String("host", "127.0.0.1", "ledis server host") -var port = flag.Int("port", 6380, "ledis server port") -var sock = flag.String("sock", "", "ledis unix socket domain") -var dumpFile = flag.String("o", "./ledis.dump", "dump file to save") +import "github.com/ledisdb/ledisdb/cmd" func main() { - flag.Parse() - - var err error - var f *os.File - - if f, err = os.OpenFile(*dumpFile, os.O_CREATE|os.O_WRONLY, 0644); err != nil { - println(err.Error()) - return - } - - defer f.Close() - - var addr string - if len(*sock) != 0 { - addr = *sock - } else { - addr = fmt.Sprintf("%s:%d", *host, *port) - } - - c, err := goredis.ConnectWithSize(addr, 16*1024, 4096) - if err != nil { - println(err.Error()) - return - } - - defer c.Close() - - println("dump begin") - - if err = c.Send("fullsync"); err != nil { - println(err.Error()) - return - } - - if err = c.ReceiveBulkTo(f); err != nil { - println(err.Error()) - return - } - - println("dump end") + cmd.CmdDump() } diff --git a/cmd/ledis-load/main.go b/cmd/ledis-load/main.go index b55a362..08ceb6f 100644 --- a/cmd/ledis-load/main.go +++ b/cmd/ledis-load/main.go @@ -1,62 +1,7 @@ package main -import ( - "flag" - - "github.com/ledisdb/ledisdb/config" - "github.com/ledisdb/ledisdb/ledis" -) - -var configPath = flag.String("config", "", "ledisdb config file") -var dumpPath = flag.String("dump_file", "", "ledisdb dump file") +import "github.com/ledisdb/ledisdb/cmd" func main() { - flag.Parse() - - if len(*configPath) == 0 { - println("need ledis config file") - return - } - - cfg, err := config.NewConfigWithFile(*configPath) - if err != nil { - println(err.Error()) - return - } - - if len(*dumpPath) == 0 { - println("need dump file") - return - } - - if len(cfg.DataDir) == 0 { - println("must set data dir") - return - } - - ldb, err := ledis.Open(cfg) - if err != nil { - println("ledis open error ", err.Error()) - return - } - - err = loadDump(cfg, ldb) - ldb.Close() - - if err != nil { - println(err.Error()) - return - } - - println("Load OK") -} - -func loadDump(cfg *config.Config, ldb *ledis.Ledis) error { - var err error - if err = ldb.FlushAll(); err != nil { - return err - } - - _, err = ldb.LoadDumpFile(*dumpPath) - return err + cmd.CmdLoad() } diff --git a/cmd/ledis-repair/main.go b/cmd/ledis-repair/main.go index f8c16ab..a69d45a 100644 --- a/cmd/ledis-repair/main.go +++ b/cmd/ledis-repair/main.go @@ -1,35 +1,7 @@ package main -import ( - "flag" - - "github.com/ledisdb/ledisdb/config" - "github.com/ledisdb/ledisdb/store" -) - -var fileName = flag.String("config", "", "ledisdb config file") +import "github.com/ledisdb/ledisdb/cmd" func main() { - flag.Parse() - - if len(*fileName) == 0 { - println("need ledis config file") - return - } - - cfg, err := config.NewConfigWithFile(*fileName) - - if err != nil { - println(err.Error()) - return - } - - if len(cfg.DataDir) == 0 { - println("must set data dir") - return - } - - if err = store.Repair(cfg); err != nil { - println("repair error: ", err.Error()) - } + cmd.CmdRepair() } diff --git a/cmd/ledis-server/main.go b/cmd/ledis-server/main.go index 4fbd90b..4493858 100644 --- a/cmd/ledis-server/main.go +++ b/cmd/ledis-server/main.go @@ -1,120 +1,9 @@ package main import ( - "flag" - "fmt" - "log" - "net/http" - _ "net/http/pprof" - "os" - "os/signal" - "runtime" - "strings" - "syscall" - - "github.com/ledisdb/ledisdb/config" - "github.com/ledisdb/ledisdb/server" + "github.com/ledisdb/ledisdb/cmd" ) -var configFile = flag.String("config", "", "ledisdb config file") -var addr = flag.String("addr", "", "ledisdb listen address") -var dataDir = flag.String("data_dir", "", "ledisdb base data dir") -var dbName = flag.String("db_name", "", "select a db to use, it will overwrite the config's db name") -var usePprof = flag.Bool("pprof", false, "enable pprof") -var pprofPort = flag.Int("pprof_port", 6060, "pprof http port") -var slaveof = flag.String("slaveof", "", "make the server a slave of another instance") -var readonly = flag.Bool("readonly", false, "set readonly mode, slave server is always readonly") -var rpl = flag.Bool("rpl", false, "enable replication or not, slave server is always enabled") -var rplSync = flag.Bool("rpl_sync", false, "enable sync replication or not") -var ttlCheck = flag.Int("ttl_check", 0, "TTL check interval") -var databases = flag.Int("databases", 0, "ledisdb maximum database number") - func main() { - runtime.GOMAXPROCS(runtime.NumCPU()) - - flag.Parse() - - var cfg *config.Config - var err error - - if len(*configFile) == 0 { - println("no config set, using default config") - cfg = config.NewConfigDefault() - } else { - cfg, err = config.NewConfigWithFile(*configFile) - } - - if err != nil { - println(err.Error()) - return - } - - if len(*addr) > 0 { - cfg.Addr = *addr - } - - if len(*dataDir) > 0 { - cfg.DataDir = *dataDir - } - - if len(*dbName) > 0 { - cfg.DBName = *dbName - } - - if *databases > 0 { - cfg.Databases = *databases - } - - // check bool flag, use it. - for _, arg := range os.Args { - arg := strings.ToLower(arg) - switch arg { - case "-rpl", "-rpl=true", "-rpl=false": - cfg.UseReplication = *rpl - case "-readonly", "-readonly=true", "-readonly=false": - cfg.Readonly = *readonly - case "-rpl_sync", "-rpl_sync=true", "-rpl_sync=false": - cfg.Replication.Sync = *rplSync - } - } - - if len(*slaveof) > 0 { - cfg.SlaveOf = *slaveof - cfg.Readonly = true - cfg.UseReplication = true - } - - if *ttlCheck > 0 { - cfg.TTLCheckInterval = *ttlCheck - } - - var app *server.App - app, err = server.NewApp(cfg) - if err != nil { - println(err.Error()) - return - } - - sc := make(chan os.Signal, 1) - signal.Notify(sc, - os.Kill, - os.Interrupt, - syscall.SIGHUP, - syscall.SIGINT, - syscall.SIGTERM, - syscall.SIGQUIT) - - if *usePprof { - go func() { - log.Println(http.ListenAndServe(fmt.Sprintf(":%d", *pprofPort), nil)) - }() - } - - go app.Run() - - <-sc - - println("ledis-server is closing") - app.Close() - println("ledis-server is closed") + cmd.CmdServer() } diff --git a/cmd/ledis/main.go b/cmd/ledis/main.go new file mode 100644 index 0000000..5c86c8b --- /dev/null +++ b/cmd/ledis/main.go @@ -0,0 +1,55 @@ +package main + +import ( + "fmt" + "os" + + "github.com/ledisdb/ledisdb/cmd" +) + +var ( + cmds = [][]string{ + {"server", "run ledis server"}, + {"cli", "run ledis client"}, + {"repair", "repair ledis storage directory"}, + {"dump", "create a snapshort of ledis"}, + {"load", "load data from a snapshort"}, + {"benchmark", "run the benchmarks with ledis"}, + } +) + +func printSubCmds() { + for _, cmd := range cmds { + printCmd(cmd[0], cmd[1]) + } +} + +func printCmd(cmd, description string) { + fmt.Printf("%s\t- %s\n", cmd, description) +} + +func main() { + var subCmd string + if len(os.Args) == 1 { + subCmd = "server" + } else { + subCmd = os.Args[1] + } + + switch subCmd { + case "repair": + cmd.CmdRepair() + case "benchmark": + cmd.CmdBenchmark() + case "cli": + cmd.CmdCli() + case "dump": + cmd.CmdDump() + case "help": + printSubCmds() + case "server": + fallthrough + default: + cmd.CmdServer() + } +} diff --git a/cmd/load.go b/cmd/load.go new file mode 100644 index 0000000..dc50925 --- /dev/null +++ b/cmd/load.go @@ -0,0 +1,62 @@ +package cmd + +import ( + "flag" + + "github.com/ledisdb/ledisdb/config" + "github.com/ledisdb/ledisdb/ledis" +) + +func CmdLoad() { + var configPath = flag.String("config", "", "ledisdb config file") + var dumpPath = flag.String("dump_file", "", "ledisdb dump file") + + flag.Parse() + + if len(*configPath) == 0 { + println("need ledis config file") + return + } + + cfg, err := config.NewConfigWithFile(*configPath) + if err != nil { + println(err.Error()) + return + } + + if len(*dumpPath) == 0 { + println("need dump file") + return + } + + if len(cfg.DataDir) == 0 { + println("must set data dir") + return + } + + ldb, err := ledis.Open(cfg) + if err != nil { + println("ledis open error ", err.Error()) + return + } + + err = loadDump(cfg, ldb, dumpPath) + ldb.Close() + + if err != nil { + println(err.Error()) + return + } + + println("Load OK") +} + +func loadDump(cfg *config.Config, ldb *ledis.Ledis, dumpPath *string) error { + var err error + if err = ldb.FlushAll(); err != nil { + return err + } + + _, err = ldb.LoadDumpFile(*dumpPath) + return err +} diff --git a/cmd/repair.go b/cmd/repair.go new file mode 100644 index 0000000..b2cd760 --- /dev/null +++ b/cmd/repair.go @@ -0,0 +1,35 @@ +package cmd + +import ( + "flag" + + "github.com/ledisdb/ledisdb/config" + "github.com/ledisdb/ledisdb/store" +) + +func CmdRepair() { + var fileName = flag.String("config", "", "ledisdb config file") + + flag.Parse() + + if len(*fileName) == 0 { + println("need ledis config file") + return + } + + cfg, err := config.NewConfigWithFile(*fileName) + + if err != nil { + println(err.Error()) + return + } + + if len(cfg.DataDir) == 0 { + println("must set data dir") + return + } + + if err = store.Repair(cfg); err != nil { + println("repair error: ", err.Error()) + } +} diff --git a/cmd/server.go b/cmd/server.go new file mode 100644 index 0000000..0d1321f --- /dev/null +++ b/cmd/server.go @@ -0,0 +1,121 @@ +package cmd + +import ( + "flag" + "fmt" + "log" + "net/http" + _ "net/http/pprof" + "os" + "os/signal" + "runtime" + "strings" + "syscall" + + "github.com/ledisdb/ledisdb/config" + "github.com/ledisdb/ledisdb/server" +) + +var addr = flag.String("addr", "", "ledisdb listen address") +var dataDir = flag.String("data_dir", "", "ledisdb base data dir") +var dbName = flag.String("db_name", "", "select a db to use, it will overwrite the config's db name") +var usePprof = flag.Bool("pprof", false, "enable pprof") +var pprofPort = flag.Int("pprof_port", 6060, "pprof http port") +var slaveof = flag.String("slaveof", "", "make the server a slave of another instance") +var readonly = flag.Bool("readonly", false, "set readonly mode, slave server is always readonly") +var rpl = flag.Bool("rpl", false, "enable replication or not, slave server is always enabled") +var rplSync = flag.Bool("rpl_sync", false, "enable sync replication or not") +var ttlCheck = flag.Int("ttl_check", 0, "TTL check interval") +var databases = flag.Int("databases", 0, "ledisdb maximum database number") + +func CmdServer() { + var configFile = flag.String("config", "", "ledisdb config file") + + runtime.GOMAXPROCS(runtime.NumCPU()) + + flag.Parse() + + var cfg *config.Config + var err error + + if len(*configFile) == 0 { + println("no config set, using default config") + cfg = config.NewConfigDefault() + } else { + cfg, err = config.NewConfigWithFile(*configFile) + } + + if err != nil { + println(err.Error()) + return + } + + if len(*addr) > 0 { + cfg.Addr = *addr + } + + if len(*dataDir) > 0 { + cfg.DataDir = *dataDir + } + + if len(*dbName) > 0 { + cfg.DBName = *dbName + } + + if *databases > 0 { + cfg.Databases = *databases + } + + // check bool flag, use it. + for _, arg := range os.Args { + arg := strings.ToLower(arg) + switch arg { + case "-rpl", "-rpl=true", "-rpl=false": + cfg.UseReplication = *rpl + case "-readonly", "-readonly=true", "-readonly=false": + cfg.Readonly = *readonly + case "-rpl_sync", "-rpl_sync=true", "-rpl_sync=false": + cfg.Replication.Sync = *rplSync + } + } + + if len(*slaveof) > 0 { + cfg.SlaveOf = *slaveof + cfg.Readonly = true + cfg.UseReplication = true + } + + if *ttlCheck > 0 { + cfg.TTLCheckInterval = *ttlCheck + } + + var app *server.App + app, err = server.NewApp(cfg) + if err != nil { + println(err.Error()) + return + } + + sc := make(chan os.Signal, 1) + signal.Notify(sc, + os.Kill, + os.Interrupt, + syscall.SIGHUP, + syscall.SIGINT, + syscall.SIGTERM, + syscall.SIGQUIT) + + if *usePprof { + go func() { + log.Println(http.ListenAndServe(fmt.Sprintf(":%d", *pprofPort), nil)) + }() + } + + go app.Run() + + <-sc + + println("ledis-server is closing") + app.Close() + println("ledis-server is closed") +}