diff --git a/.gitignore b/.gitignore index 65a501e..c795b05 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -ssdb/testdb \ No newline at end of file +build \ No newline at end of file diff --git a/README.md b/README.md index 1f2230a..e363dd8 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,47 @@ -go-ssdb -======= +# ledisdb -a implementation of ssdb in go +ledisdb是一个用go实现的类似redis的高性能nosql数据库,底层基于leveldb实现。提供了kv,list,hash以及zset几种数据结构的实现。 + +最开始源于[ssdb](https://github.com/ideawu/ssdb),在使用了一段时间之后,因为兴趣的原因,决定用go实现一个。 + +## 编译 + ++ 创建一个工作目录,并check ledisdb源码 + + mkdir $WORKSPACE + cd $WORKSPACE + git clone git@github.com:siddontang/ledisdb.git src/github.com/siddontang/ledisdb + + cd src/github.com/siddontang/ledisdb + ++ 安装leveldb以及snappy,如果你已经安装,忽略 + + 我提供了一个简单的脚本进行leveldb的安装,你可以直接在shell中输入: + + sh build_leveldb.sh + + 默认该脚本会将leveldb以及snappy安装到/usr/local/leveldb以及/usr/local/snappy目录 + ++ 在dev.sh里面设置LEVELDB_DIR以及SNAPPY_DIR为实际的安装路径,默认为usr/local/leveldb以及/usr/local/snappy + ++ 运行bootstrap.sh构建ledisdb go的依赖库 + + . ./bootstap.sh 或者 source ./bootstrap.sh + ++ 运行dev.sh + + . ./dev.sh 或者 source ./dev.sh + ++ 编译安装ledisdb + + go install ./... + +## 运行 + + ./ledis-server -config=/etc/ledis.json + +## Todo + ++ Binlog ++ Replication ++ Admin \ No newline at end of file diff --git a/benchmark.md b/benchmark.md new file mode 100644 index 0000000..566c056 --- /dev/null +++ b/benchmark.md @@ -0,0 +1,76 @@ +## 测试环境搭建 + ++ Darwin karamatoMacBook-Air.local 13.1.0 Darwin Kernel Version 13.1.0: Thu Jan 16 19:40:37 PST 2014; root:xnu-2422.90.20~2/RELEASE_X86_64 x86_64 ++ 2 CPU Intel Core i5 1.7GHz ++ 4GB ++ SSD 120G + +使用redis-benchmark命令进行压力测试,测试语句 + + redis-benchmark -n 10000 -t set,incr,get,lpush,lpop,lrange,mset -q + +redis只用内存模式,关闭持久化 +ssdb使用默认的配置,手动更改代码关闭了binlog的功能: + + void BinlogQueue::add_log(char type, char cmd, const leveldb::Slice &key){ + tran_seq ++; + //Binlog log(tran_seq, type, cmd, key); + //batch.Put(encode_seq_key(tran_seq), log.repr()); + } + +ledisdb使用跟ssdb相同的leveldb配置 + +leveldb配置: + + compression = false + block_size = 32KB + write_buffer_size = 64MB + cache_size = 500MB + + +## redis + + SET: 42735.04 requests per second + GET: 45871.56 requests per second + INCR: 45248.87 requests per second + LPUSH: 45045.04 requests per second + LPOP: 43103.45 requests per second + LPUSH (needed to benchmark LRANGE): 44843.05 requests per second + LRANGE_100 (first 100 elements): 14727.54 requests per second + LRANGE_300 (first 300 elements): 6915.63 requests per second + LRANGE_500 (first 450 elements): 5042.86 requests per second + LRANGE_600 (first 600 elements): 3960.40 requests per second + MSET (10 keys): 33003.30 requests per second + +## ssdb + + SET: 35971.22 requests per second + GET: 47393.37 requests per second + INCR: 36630.04 requests per second + LPUSH: 37174.72 requests per second + LPOP: 38167.94 requests per second + LPUSH (needed to benchmark LRANGE): 37593.98 requests per second + LRANGE_100 (first 100 elements): 905.55 requests per second + LRANGE_300 (first 300 elements): 327.78 requests per second + LRANGE_500 (first 450 elements): 222.36 requests per second + LRANGE_600 (first 600 elements): 165.30 requests per second + MSET (10 keys): 33112.59 requests per second + +## ledisdb + + SET: 38759.69 requests per second + GET: 40160.64 requests per second + INCR: 36101.08 requests per second + LPUSH: 33003.30 requests per second + LPOP: 27624.31 requests per second + LPUSH (needed to benchmark LRANGE): 32894.74 requests per second + LRANGE_100 (first 100 elements): 7352.94 requests per second + LRANGE_300 (first 300 elements): 2867.79 requests per second + LRANGE_500 (first 450 elements): 1778.41 requests per second + LRANGE_600 (first 600 elements): 1590.33 requests per second + MSET (10 keys): 21881.84 requests per second + +可以看到,ledisdb的性能比redis以及ssdb略差,(至于为啥ssdb lrange性能这样差我感到比较困惑),但还不至于不可用,我觉得可能如下原因: + ++ go语言自身没有c,c++快 ++ ledisdb使用的cgo调用的leveldb相关函数,这块开销其实蛮大的,后续是优化重点。 diff --git a/bootstrap.sh b/bootstrap.sh new file mode 100644 index 0000000..fffd283 --- /dev/null +++ b/bootstrap.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +. ./dev.sh + +go get -u github.com/jmhodges/levigo +go get -u github.com/siddontang/golib/leveldb +go get -u github.com/siddontang/golib/log +go get -u github.com/siddontang/golib/hack +go get -u github.com/garyburd/redigo/redis diff --git a/build_leveldb.sh b/build_leveldb.sh new file mode 100644 index 0000000..a361be9 --- /dev/null +++ b/build_leveldb.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +SNAPPY_DIR=/usr/local/snappy +LEVELDB_DIR=/usr/local/leveldb + +mkdir -p ./build + +cd ./build + +if [ ! -f $SNAPPY_DIR/lib/libsnappy.a ]; then + (git clone git@github.com:siddontang/snappy.git && \ + cd ./snappy && \ + ./configure --prefix=$SNAPPY_DIR && \ + make && \ + make install) +else + echo "skip install snappy" +fi + +if [ ! -f $LEVELDB_DIR/lib/libleveldb.a ]; then + (git clone git@github.com:siddontang/leveldb.git && \ + cd ./leveldb && \ + echo "echo \"PLATFORM_CFLAGS+=-I$SNAPPY_DIR/include\" >> build_config.mk" >> build_detect_platform && + echo "echo \"PLATFORM_CXXFLAGS+=-I$SNAPPY_DIR/include\" >> build_config.mk" >> build_detect_platform && + echo "echo \"PLATFORM_LDFLAGS+=-L $SNAPPY_DIR/lib -lsnappy\" >> build_config.mk" >> build_detect_platform && + make SNAPPY=1 && \ + make && \ + mkdir -p $LEVELDB_DIR/include/leveldb && \ + install include/leveldb/*.h $LEVELDB_DIR/include/leveldb && \ + mkdir -p $LEVELDB_DIR/lib && \ + cp -f libleveldb.* $LEVELDB_DIR/lib) +else + echo "skip install leveldb" +fi \ No newline at end of file diff --git a/cmd/ssdb/main.go b/cmd/ledis-server/main.go similarity index 69% rename from cmd/ssdb/main.go rename to cmd/ledis-server/main.go index 92ba933..5fda7b3 100644 --- a/cmd/ssdb/main.go +++ b/cmd/ledis-server/main.go @@ -2,14 +2,14 @@ package main import ( "flag" - "github.com/siddontang/go-ssdb/ssdb" + "github.com/siddontang/ledisdb/ledis" "os" "os/signal" "runtime" "syscall" ) -var configFile = flag.String("config", "", "ssdb config file") +var configFile = flag.String("config", "", "ledisdb config file") func main() { runtime.GOMAXPROCS(runtime.NumCPU()) @@ -20,13 +20,13 @@ func main() { panic("must use a config file") } - cfg, err := ssdb.NewConfigWithFile(*configFile) + cfg, err := ledis.NewConfigWithFile(*configFile) if err != nil { panic(err) } - var app *ssdb.App - app, err = ssdb.NewApp(cfg) + var app *ledis.App + app, err = ledis.NewApp(cfg) if err != nil { panic(err) } diff --git a/dev.sh b/dev.sh new file mode 100644 index 0000000..fbf63bd --- /dev/null +++ b/dev.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +export VTTOP=$(pwd) +export VTROOT="${VTROOT:-${VTTOP/\/src\/github.com\/siddontang\/ledisdb/}}" +# VTTOP sanity check +if [[ "$VTTOP" == "${VTTOP/\/src\/github.com\/siddontang\/ledisdb/}" ]]; then + echo "WARNING: VTTOP($VTTOP) does not contain src/github.com/siddontang/ledisdb" + exit 1 +fi + + +#default snappy and leveldb install path +#you may change yourself + +SNAPPY_DIR=/usr/local/snappy +LEVELDB_DIR=/usr/local/leveldb + +function add_path() +{ + # $1 path variable + # $2 path to add + if [ -d "$2" ] && [[ ":$1:" != *":$2:"* ]]; then + echo "$1:$2" + else + echo "$1" + fi +} + +export CGO_CFLAGS="-I$LEVELDB_DIR/include -I$SNAPPY_DIR/include" +export CGO_LDFLAGS="-L$LEVELDB_DIR/lib -L$SNAPPY_DIR/lib -lsnappy" +export LD_LIBRARY_PATH=$(add_path $LD_LIBRARY_PATH $SNAPPY_DIR/lib) +export LD_LIBRARY_PATH=$(add_path $LD_LIBRARY_PATH $LEVELDB_DIR/lib) + diff --git a/etc/ssdb.json b/etc/ledis.json similarity index 84% rename from etc/ssdb.json rename to etc/ledis.json index ca79fdf..613426a 100644 --- a/etc/ssdb.json +++ b/etc/ledis.json @@ -1,7 +1,7 @@ { "addr": "127.0.0.1:6380", "leveldb": { - "path": "/tmp/ssdb", + "path": "/tmp/ledisdb", "compression": false, "block_size": 32768, "write_buffer_size": 67108864, diff --git a/ssdb/app.go b/ledis/app.go similarity index 98% rename from ssdb/app.go rename to ledis/app.go index 00ee1d3..50f0481 100644 --- a/ssdb/app.go +++ b/ledis/app.go @@ -1,4 +1,4 @@ -package ssdb +package ledis import ( "github.com/siddontang/golib/leveldb" diff --git a/ssdb/app_test.go b/ledis/app_test.go similarity index 98% rename from ssdb/app_test.go rename to ledis/app_test.go index 47360ad..66f48ce 100644 --- a/ssdb/app_test.go +++ b/ledis/app_test.go @@ -1,4 +1,4 @@ -package ssdb +package ledis import ( "github.com/garyburd/redigo/redis" diff --git a/ssdb/client.go b/ledis/client.go similarity index 99% rename from ssdb/client.go rename to ledis/client.go index 999aac9..6a49429 100644 --- a/ssdb/client.go +++ b/ledis/client.go @@ -1,4 +1,4 @@ -package ssdb +package ledis import ( "bufio" diff --git a/ssdb/cmd_hash.go b/ledis/cmd_hash.go similarity index 99% rename from ssdb/cmd_hash.go rename to ledis/cmd_hash.go index 4b77d02..297d469 100644 --- a/ssdb/cmd_hash.go +++ b/ledis/cmd_hash.go @@ -1,4 +1,4 @@ -package ssdb +package ledis import ( "github.com/siddontang/golib/hack" diff --git a/ssdb/cmd_hash_test.go b/ledis/cmd_hash_test.go similarity index 99% rename from ssdb/cmd_hash_test.go rename to ledis/cmd_hash_test.go index 3597ee3..47a004e 100644 --- a/ssdb/cmd_hash_test.go +++ b/ledis/cmd_hash_test.go @@ -1,4 +1,4 @@ -package ssdb +package ledis import ( "fmt" diff --git a/ssdb/cmd_kv.go b/ledis/cmd_kv.go similarity index 99% rename from ssdb/cmd_kv.go rename to ledis/cmd_kv.go index 5750d58..d7a08b5 100644 --- a/ssdb/cmd_kv.go +++ b/ledis/cmd_kv.go @@ -1,4 +1,4 @@ -package ssdb +package ledis import ( "github.com/siddontang/golib/hack" diff --git a/ssdb/cmd_kv_test.go b/ledis/cmd_kv_test.go similarity index 99% rename from ssdb/cmd_kv_test.go rename to ledis/cmd_kv_test.go index 49fc92f..7a636ae 100644 --- a/ssdb/cmd_kv_test.go +++ b/ledis/cmd_kv_test.go @@ -1,4 +1,4 @@ -package ssdb +package ledis import ( "github.com/garyburd/redigo/redis" diff --git a/ssdb/cmd_list.go b/ledis/cmd_list.go similarity index 99% rename from ssdb/cmd_list.go rename to ledis/cmd_list.go index a680a79..6b39cc6 100644 --- a/ssdb/cmd_list.go +++ b/ledis/cmd_list.go @@ -1,4 +1,4 @@ -package ssdb +package ledis import ( "github.com/siddontang/golib/hack" diff --git a/ssdb/cmd_list_test.go b/ledis/cmd_list_test.go similarity index 99% rename from ssdb/cmd_list_test.go rename to ledis/cmd_list_test.go index 4f14731..77a9a74 100644 --- a/ssdb/cmd_list_test.go +++ b/ledis/cmd_list_test.go @@ -1,4 +1,4 @@ -package ssdb +package ledis import ( "fmt" diff --git a/ssdb/cmd_zset.go b/ledis/cmd_zset.go similarity index 99% rename from ssdb/cmd_zset.go rename to ledis/cmd_zset.go index cd05958..e580f51 100644 --- a/ssdb/cmd_zset.go +++ b/ledis/cmd_zset.go @@ -1,4 +1,4 @@ -package ssdb +package ledis import ( "errors" diff --git a/ssdb/cmd_zset_test.go b/ledis/cmd_zset_test.go similarity index 99% rename from ssdb/cmd_zset_test.go rename to ledis/cmd_zset_test.go index d2aeea6..c8a05fc 100644 --- a/ssdb/cmd_zset_test.go +++ b/ledis/cmd_zset_test.go @@ -1,4 +1,4 @@ -package ssdb +package ledis import ( "bytes" diff --git a/ssdb/command.go b/ledis/command.go similarity index 97% rename from ssdb/command.go rename to ledis/command.go index 56ac7e9..39a855f 100644 --- a/ssdb/command.go +++ b/ledis/command.go @@ -1,4 +1,4 @@ -package ssdb +package ledis import ( "fmt" diff --git a/ssdb/config.go b/ledis/config.go similarity index 97% rename from ssdb/config.go rename to ledis/config.go index 307e2f2..a303a73 100644 --- a/ssdb/config.go +++ b/ledis/config.go @@ -1,4 +1,4 @@ -package ssdb +package ledis import ( "encoding/json" diff --git a/ssdb/const.go b/ledis/const.go similarity index 96% rename from ssdb/const.go rename to ledis/const.go index c2cd1ba..235053f 100644 --- a/ssdb/const.go +++ b/ledis/const.go @@ -1,4 +1,4 @@ -package ssdb +package ledis import ( "errors" diff --git a/ssdb/lock.go b/ledis/lock.go similarity index 96% rename from ssdb/lock.go rename to ledis/lock.go index 02c2768..8dfd4cd 100644 --- a/ssdb/lock.go +++ b/ledis/lock.go @@ -1,4 +1,4 @@ -package ssdb +package ledis import ( "hash/crc32" diff --git a/ssdb/t_hash.go b/ledis/t_hash.go similarity index 99% rename from ssdb/t_hash.go rename to ledis/t_hash.go index 5529fb6..e539389 100644 --- a/ssdb/t_hash.go +++ b/ledis/t_hash.go @@ -1,4 +1,4 @@ -package ssdb +package ledis import ( "encoding/binary" diff --git a/ssdb/t_kv.go b/ledis/t_kv.go similarity index 99% rename from ssdb/t_kv.go rename to ledis/t_kv.go index fcae406..7496af5 100644 --- a/ssdb/t_kv.go +++ b/ledis/t_kv.go @@ -1,4 +1,4 @@ -package ssdb +package ledis import ( "errors" diff --git a/ssdb/t_list.go b/ledis/t_list.go similarity index 99% rename from ssdb/t_list.go rename to ledis/t_list.go index 7264cb6..335d2f7 100644 --- a/ssdb/t_list.go +++ b/ledis/t_list.go @@ -1,4 +1,4 @@ -package ssdb +package ledis import ( "encoding/binary" diff --git a/ssdb/t_zset.go b/ledis/t_zset.go similarity index 99% rename from ssdb/t_zset.go rename to ledis/t_zset.go index 290370a..4613269 100644 --- a/ssdb/t_zset.go +++ b/ledis/t_zset.go @@ -1,4 +1,4 @@ -package ssdb +package ledis import ( "bytes" diff --git a/ssdb/tx.go b/ledis/tx.go similarity index 97% rename from ssdb/tx.go rename to ledis/tx.go index afe7d40..35c5180 100644 --- a/ssdb/tx.go +++ b/ledis/tx.go @@ -1,4 +1,4 @@ -package ssdb +package ledis import ( "github.com/siddontang/golib/leveldb" diff --git a/ssdb/util.go b/ledis/util.go similarity index 97% rename from ssdb/util.go rename to ledis/util.go index 61c2b83..e5d6efd 100644 --- a/ssdb/util.go +++ b/ledis/util.go @@ -1,4 +1,4 @@ -package ssdb +package ledis import ( "encoding/binary"