From f66ffb18dc107f7c4e4a55a7f29854675a4ede04 Mon Sep 17 00:00:00 2001 From: siddontang Date: Tue, 27 May 2014 16:05:24 +0800 Subject: [PATCH 01/31] add base bin log support --- ledis/binlog.go | 273 +++++++++++++++++++++++++++++++++++++++++++ ledis/binlog_test.go | 36 ++++++ ledis/const.go | 17 +++ ledis/ledis.go | 34 +++++- ledis/ledis_test.go | 8 +- ledis/tx.go | 57 ++++++++- 6 files changed, 419 insertions(+), 6 deletions(-) create mode 100644 ledis/binlog.go create mode 100644 ledis/binlog_test.go diff --git a/ledis/binlog.go b/ledis/binlog.go new file mode 100644 index 0000000..f7f2961 --- /dev/null +++ b/ledis/binlog.go @@ -0,0 +1,273 @@ +package ledis + +import ( + "bufio" + "encoding/binary" + "encoding/json" + "fmt" + "github.com/siddontang/go-log/log" + "io" + "os" + "path" + "strconv" + "strings" + "sync" + "time" +) + +/* +index file format: +ledis-bin.00001 +ledis-bin.00002 +ledis-bin.00003 + +log file format + +timestamp(bigendian uint32, seconds)|PayloadLen(bigendian uint32)|PayloadData|LogId + +*/ + +type BinLogConfig struct { + Path string `json:"path"` + MaxFileSize int `json:"max_file_size"` + MaxFileNum int `json:"max_file_num"` +} + +func (cfg *BinLogConfig) adjust() { + if cfg.MaxFileSize <= 0 { + cfg.MaxFileSize = DefaultBinLogFileSize + } else if cfg.MaxFileSize > MaxBinLogFileSize { + cfg.MaxFileSize = MaxBinLogFileSize + } + + if cfg.MaxFileNum <= 0 { + cfg.MaxFileNum = DefaultBinLogFileNum + } else if cfg.MaxFileNum > MaxBinLogFileNum { + cfg.MaxFileNum = MaxBinLogFileNum + } +} + +type BinLog struct { + sync.Mutex + + cfg *BinLogConfig + + logFile *os.File + + logWb *bufio.Writer + + indexName string + logNames []string + lastLogIndex int +} + +func NewBinLog(data json.RawMessage) (*BinLog, error) { + var cfg BinLogConfig + + if err := json.Unmarshal(data, &cfg); err != nil { + return nil, err + } + + return NewBinLogWithConfig(&cfg) +} + +func NewBinLogWithConfig(cfg *BinLogConfig) (*BinLog, error) { + b := new(BinLog) + + cfg.adjust() + + b.cfg = cfg + + if err := os.MkdirAll(cfg.Path, os.ModePerm); err != nil { + return nil, err + } + + b.logNames = make([]string, 0, b.cfg.MaxFileNum) + + if err := b.loadIndex(); err != nil { + return nil, err + } + + return b, nil +} + +func (b *BinLog) Close() { + if b.logFile != nil { + b.logFile.Close() + } +} + +func (b *BinLog) deleteOldest() { + logPath := path.Join(b.cfg.Path, b.logNames[0]) + os.Remove(logPath) + + copy(b.logNames[0:], b.logNames[1:]) + b.logNames = b.logNames[0 : len(b.logNames)-1] +} + +func (b *BinLog) flushIndex() error { + data := strings.Join(b.logNames, "\n") + + bakName := fmt.Sprintf("%s.bak", b.indexName) + f, err := os.OpenFile(bakName, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + log.Error("create binlog bak index error %s", err.Error()) + return err + } + + if _, err := f.WriteString(data); err != nil { + log.Error("write binlog index error %s", err.Error()) + f.Close() + return err + } + + f.Close() + + if err := os.Rename(bakName, b.indexName); err != nil { + log.Error("rename binlog bak index error %s", err.Error()) + return err + } + + return nil +} + +func (b *BinLog) loadIndex() error { + b.indexName = path.Join(b.cfg.Path, BinLogIndexFile) + fd, err := os.OpenFile(b.indexName, os.O_CREATE|os.O_RDWR, 0666) + if err != nil { + return err + } + + //maybe we will check valid later? + rb := bufio.NewReader(fd) + for { + line, err := rb.ReadString('\n') + if err != nil && err != io.EOF { + fd.Close() + return err + } + + line = strings.Trim(line, "\r\n ") + + if len(line) > 0 { + b.logNames = append(b.logNames, line) + } + + if len(b.logNames) == b.cfg.MaxFileNum { + //remove oldest logfile + b.deleteOldest() + } + + if err == io.EOF { + break + } + } + + fd.Close() + + if err := b.flushIndex(); err != nil { + return err + } + + if len(b.logNames) == 0 { + b.lastLogIndex = 1 + } else { + lastName := b.logNames[len(b.logNames)-1] + + if b.lastLogIndex, err = strconv.Atoi(path.Ext(lastName)[1:]); err != nil { + log.Error("invalid logfile name %s", err.Error()) + return err + } + + //like mysql, if server restart, a new binlog will create + b.lastLogIndex++ + } + + return nil +} + +func (b *BinLog) getLogName() string { + return fmt.Sprintf("%s.%05d", BinLogBaseName, b.lastLogIndex) +} + +func (b *BinLog) openNewLogFile() error { + var err error + lastName := b.getLogName() + + logPath := path.Join(b.cfg.Path, lastName) + if b.logFile, err = os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY, 0666); err != nil { + log.Error("open new logfile error %s", err.Error()) + return err + } + + if len(b.logNames) == b.cfg.MaxFileNum { + b.deleteOldest() + } + + b.logNames = append(b.logNames, lastName) + + if b.logWb == nil { + b.logWb = bufio.NewWriterSize(b.logFile, 1024) + } else { + b.logWb.Reset(b.logFile) + } + + if err = b.flushIndex(); err != nil { + return err + } + + return nil +} + +func (b *BinLog) openLogFile() error { + if b.logFile == nil { + return b.openNewLogFile() + } else { + //check file size + st, _ := b.logFile.Stat() + if st.Size() >= int64(b.cfg.MaxFileSize) { + //must use new file + b.lastLogIndex++ + + b.logFile.Close() + + return b.openNewLogFile() + } + } + + return nil +} + +func (b *BinLog) Log(args ...[]byte) error { + var err error + + for _, data := range args { + createTime := uint32(time.Now().Unix()) + payLoadLen := len(data) + + if err = b.openLogFile(); err != nil { + return err + } + + binary.Write(b.logWb, binary.BigEndian, createTime) + binary.Write(b.logWb, binary.BigEndian, payLoadLen) + + b.logWb.Write(data) + + if err = b.logWb.Flush(); err != nil { + log.Error("write log error %s", err.Error()) + return err + } + } + + return nil +} + +func (b *BinLog) SavePoint() (string, int64) { + if b.logFile == nil { + return "", 0 + } else { + st, _ := b.logFile.Stat() + return b.logNames[len(b.logNames)-1], st.Size() + } +} diff --git a/ledis/binlog_test.go b/ledis/binlog_test.go new file mode 100644 index 0000000..7fc89b4 --- /dev/null +++ b/ledis/binlog_test.go @@ -0,0 +1,36 @@ +package ledis + +import ( + "io/ioutil" + "os" + "testing" +) + +func TestBinLog(t *testing.T) { + cfg := new(BinLogConfig) + + cfg.MaxFileNum = 1 + cfg.MaxFileSize = 1024 + cfg.Path = "/tmp/ledis_binlog" + + os.RemoveAll(cfg.Path) + + b, err := NewBinLogWithConfig(cfg) + if err != nil { + t.Fatal(err) + } + + if err := b.Log(make([]byte, 1024)); err != nil { + t.Fatal(err) + } + + if err := b.Log(make([]byte, 1024)); err != nil { + t.Fatal(err) + } + + if fs, err := ioutil.ReadDir(cfg.Path); err != nil { + t.Fatal(err) + } else if len(fs) != 2 { + t.Fatal(len(fs)) + } +} diff --git a/ledis/const.go b/ledis/const.go index 9e4d0cf..9fecc29 100644 --- a/ledis/const.go +++ b/ledis/const.go @@ -38,3 +38,20 @@ var ( ErrHashFieldSize = errors.New("invalid hash field size") ErrZSetMemberSize = errors.New("invalid zset member size") ) + +const BinLogBaseName = "ledis-bin" +const BinLogIndexFile = "ledis-bin.index" + +const ( + MaxBinLogFileSize int = 1024 * 1024 * 1024 + MaxBinLogFileNum int = 10000 + + DefaultBinLogFileSize int = MaxBinLogFileSize + DefaultBinLogFileNum int = 10 +) + +//like leveldb +const ( + BinLogTypeDeletion uint8 = 0x0 + BinLogTypeValue uint8 = 0x1 +) diff --git a/ledis/ledis.go b/ledis/ledis.go index 330f24c..886e69a 100644 --- a/ledis/ledis.go +++ b/ledis/ledis.go @@ -8,6 +8,8 @@ import ( type Config struct { DataDB leveldb.Config `json:"data_db"` + + BinLog BinLogConfig `json:"binlog"` } type DB struct { @@ -26,6 +28,8 @@ type Ledis struct { ldb *leveldb.DB dbs [MaxDBNumber]*DB + + binlog *BinLog } func Open(configJson json.RawMessage) (*Ledis, error) { @@ -47,6 +51,15 @@ func OpenWithConfig(cfg *Config) (*Ledis, error) { l := new(Ledis) l.ldb = ldb + if len(cfg.BinLog.Path) > 0 { + l.binlog, err = NewBinLogWithConfig(&cfg.BinLog) + if err != nil { + return nil, err + } + } else { + l.binlog = nil + } + for i := uint8(0); i < MaxDBNumber; i++ { l.dbs[i] = newDB(l, i) } @@ -61,10 +74,10 @@ func newDB(l *Ledis, index uint8) *DB { d.index = index - d.kvTx = &tx{wb: d.db.NewWriteBatch()} - d.listTx = &tx{wb: d.db.NewWriteBatch()} - d.hashTx = &tx{wb: d.db.NewWriteBatch()} - d.zsetTx = &tx{wb: d.db.NewWriteBatch()} + d.kvTx = newTx(l) + d.listTx = newTx(l) + d.hashTx = newTx(l) + d.zsetTx = newTx(l) return d } @@ -80,3 +93,16 @@ func (l *Ledis) Select(index int) (*DB, error) { return l.dbs[index], nil } + +func (l *Ledis) Snapshot() (*leveldb.Snapshot, string, int64) { + if l.binlog == nil { + return l.ldb.NewSnapshot(), "", 0 + } else { + l.binlog.Lock() + s := l.ldb.NewSnapshot() + fileName, offset := l.binlog.SavePoint() + l.binlog.Unlock() + + return s, fileName, offset + } +} diff --git a/ledis/ledis_test.go b/ledis/ledis_test.go index 2512632..6967e31 100644 --- a/ledis/ledis_test.go +++ b/ledis/ledis_test.go @@ -18,7 +18,13 @@ func getTestDB() *DB { "block_size" : 32768, "write_buffer_size" : 2097152, "cache_size" : 20971520 - } + }, + + "binlog" : { + "path" : "/tmp/testdb_binlog", + "max_file_size" : 1073741824, + "max_file_num" : 3 + } } `) var err error diff --git a/ledis/tx.go b/ledis/tx.go index 38b6e01..f5fa7f3 100644 --- a/ledis/tx.go +++ b/ledis/tx.go @@ -1,6 +1,7 @@ package ledis import ( + "encoding/binary" "github.com/siddontang/go-leveldb/leveldb" "sync" ) @@ -9,6 +10,19 @@ type tx struct { m sync.Mutex wb *leveldb.WriteBatch + + binlog *BinLog + batch [][]byte +} + +func newTx(l *Ledis) *tx { + t := new(tx) + + t.wb = l.ldb.NewWriteBatch() + + t.batch = make([][]byte, 0, 4) + t.binlog = l.binlog + return t } func (t *tx) Close() { @@ -17,10 +31,36 @@ func (t *tx) Close() { func (t *tx) Put(key []byte, value []byte) { t.wb.Put(key, value) + + if t.binlog != nil { + buf := make([]byte, 9+len(key)+len(value)) + buf[0] = BinLogTypeValue + pos := 1 + binary.BigEndian.PutUint32(buf[pos:], uint32(len(key))) + pos += 4 + copy(buf[pos:], key) + pos += len(key) + binary.BigEndian.PutUint32(buf[pos:], uint32(len(value))) + pos += 4 + copy(buf[pos:], value) + + t.batch = append(t.batch, buf) + } } func (t *tx) Delete(key []byte) { t.wb.Delete(key) + + if t.binlog != nil { + buf := make([]byte, 5+len(key)) + buf[0] = BinLogTypeDeletion + pos := 1 + binary.BigEndian.PutUint32(buf[pos:], uint32(len(key))) + pos += 4 + copy(buf[pos:], key) + + t.batch = append(t.batch, buf) + } } func (t *tx) Lock() { @@ -28,12 +68,27 @@ func (t *tx) Lock() { } func (t *tx) Unlock() { + t.batch = t.batch[0:0] t.wb.Rollback() t.m.Unlock() } func (t *tx) Commit() error { - err := t.wb.Commit() + var err error + if t.binlog != nil { + t.binlog.Lock() + err = t.wb.Commit() + if err != nil { + t.binlog.Unlock() + return err + } + + err = t.binlog.Log(t.batch...) + + t.binlog.Unlock() + } else { + err = t.wb.Commit() + } return err } From d9dcab1d6b9a1b3dcab38912276e8645f4671684 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 28 May 2014 14:20:34 +0800 Subject: [PATCH 02/31] adjust key/value max size, change step commit in flush --- ledis/binlog.go | 18 +++++++++++++----- ledis/const.go | 10 +++++++--- ledis/ledis.go | 13 ------------- ledis/t_hash.go | 9 +++++++++ ledis/t_kv.go | 22 ++++++++++++++++++++++ ledis/t_list.go | 6 ++++++ ledis/t_zset.go | 5 +++++ ledis/tx.go | 12 ++++++------ 8 files changed, 68 insertions(+), 27 deletions(-) diff --git a/ledis/binlog.go b/ledis/binlog.go index f7f2961..716b11b 100644 --- a/ledis/binlog.go +++ b/ledis/binlog.go @@ -263,11 +263,19 @@ func (b *BinLog) Log(args ...[]byte) error { return nil } -func (b *BinLog) SavePoint() (string, int64) { - if b.logFile == nil { - return "", 0 +func (b *BinLog) LogFileName() string { + if len(b.logNames) == 0 { + return "" } else { - st, _ := b.logFile.Stat() - return b.logNames[len(b.logNames)-1], st.Size() + return b.logNames[len(b.logNames)-1] + } +} + +func (b *BinLog) LogFilePos() int64 { + if b.logFile == nil { + return 0 + } else { + st, _ := b.logFile.Stat() + return st.Size() } } diff --git a/ledis/const.go b/ledis/const.go index 9fecc29..cfe3cd9 100644 --- a/ledis/const.go +++ b/ledis/const.go @@ -24,17 +24,21 @@ const ( MaxDBNumber uint8 = 16 //max key size - MaxKeySize int = 1<<16 - 1 + MaxKeySize int = 1024 //max hash field size - MaxHashFieldSize int = 1<<16 - 1 + MaxHashFieldSize int = 1024 //max zset member size - MaxZSetMemberSize int = 1<<16 - 1 + MaxZSetMemberSize int = 1024 + + //max value size + MaxValueSize int = 10 * 1024 * 1024 ) var ( ErrKeySize = errors.New("invalid key size") + ErrValueSize = errors.New("invalid value size") ErrHashFieldSize = errors.New("invalid hash field size") ErrZSetMemberSize = errors.New("invalid zset member size") ) diff --git a/ledis/ledis.go b/ledis/ledis.go index 886e69a..8d4d950 100644 --- a/ledis/ledis.go +++ b/ledis/ledis.go @@ -93,16 +93,3 @@ func (l *Ledis) Select(index int) (*DB, error) { return l.dbs[index], nil } - -func (l *Ledis) Snapshot() (*leveldb.Snapshot, string, int64) { - if l.binlog == nil { - return l.ldb.NewSnapshot(), "", 0 - } else { - l.binlog.Lock() - s := l.ldb.NewSnapshot() - fileName, offset := l.binlog.SavePoint() - l.binlog.Unlock() - - return s, fileName, offset - } -} diff --git a/ledis/t_hash.go b/ledis/t_hash.go index 2d6550f..bb700c2 100644 --- a/ledis/t_hash.go +++ b/ledis/t_hash.go @@ -130,6 +130,8 @@ func (db *DB) hSetItem(key []byte, field []byte, value []byte) (int64, error) { func (db *DB) HSet(key []byte, field []byte, value []byte) (int64, error) { if err := checkHashKFSize(key, field); err != nil { return 0, err + } else if err := checkValueSize(value); err != nil { + return 0, err } t := db.hashTx @@ -166,6 +168,8 @@ func (db *DB) HMset(key []byte, args ...FVPair) error { for i := 0; i < len(args); i++ { if err := checkHashKFSize(key, args[i].Field); err != nil { return err + } else if err := checkValueSize(args[i].Value); err != nil { + return err } ek = db.hEncodeHashKey(key, args[i].Field) @@ -414,6 +418,11 @@ func (db *DB) HFlush() (drop int64, err error) { for ; it.Valid(); it.Next() { t.Delete(it.Key()) drop++ + if drop%1000 == 0 { + if err = t.Commit(); err != nil { + return + } + } } err = t.Commit() diff --git a/ledis/t_kv.go b/ledis/t_kv.go index c7e228e..9469604 100644 --- a/ledis/t_kv.go +++ b/ledis/t_kv.go @@ -19,6 +19,14 @@ func checkKeySize(key []byte) error { return nil } +func checkValueSize(value []byte) error { + if len(value) > MaxValueSize { + return ErrValueSize + } + + return nil +} + func (db *DB) encodeKVKey(key []byte) []byte { ek := make([]byte, len(key)+2) ek[0] = db.index @@ -137,6 +145,8 @@ func (db *DB) Get(key []byte) ([]byte, error) { func (db *DB) GetSet(key []byte, value []byte) ([]byte, error) { if err := checkKeySize(key); err != nil { return nil, err + } else if err := checkValueSize(value); err != nil { + return nil, err } key = db.encodeKVKey(key) @@ -204,6 +214,8 @@ func (db *DB) MSet(args ...KVPair) error { for i := 0; i < len(args); i++ { if err := checkKeySize(args[i].Key); err != nil { return err + } else if err := checkValueSize(args[i].Value); err != nil { + return err } key = db.encodeKVKey(args[i].Key) @@ -222,6 +234,8 @@ func (db *DB) MSet(args ...KVPair) error { func (db *DB) Set(key []byte, value []byte) error { if err := checkKeySize(key); err != nil { return err + } else if err := checkValueSize(value); err != nil { + return err } var err error @@ -244,6 +258,8 @@ func (db *DB) Set(key []byte, value []byte) error { func (db *DB) SetNX(key []byte, value []byte) (int64, error) { if err := checkKeySize(key); err != nil { return 0, err + } else if err := checkValueSize(value); err != nil { + return 0, err } var err error @@ -283,6 +299,12 @@ func (db *DB) KvFlush() (drop int64, err error) { for ; it.Valid(); it.Next() { t.Delete(it.Key()) drop++ + + if drop%1000 == 0 { + if err = t.Commit(); err != nil { + return + } + } } err = t.Commit() diff --git a/ledis/t_list.go b/ledis/t_list.go index 34cbb66..eac2769 100644 --- a/ledis/t_list.go +++ b/ledis/t_list.go @@ -398,6 +398,12 @@ func (db *DB) LFlush() (drop int64, err error) { for ; it.Valid(); it.Next() { t.Delete(it.Key()) drop++ + if drop%1000 == 0 { + if err = t.Commit(); err != nil { + return + } + } + } err = t.Commit() diff --git a/ledis/t_zset.go b/ledis/t_zset.go index 222b4ae..1c86a9e 100644 --- a/ledis/t_zset.go +++ b/ledis/t_zset.go @@ -697,6 +697,11 @@ func (db *DB) ZFlush() (drop int64, err error) { for ; it.Valid(); it.Next() { t.Delete(it.Key()) drop++ + if drop%1000 == 0 { + if err = t.Commit(); err != nil { + return + } + } } err = t.Commit() diff --git a/ledis/tx.go b/ledis/tx.go index f5fa7f3..c5aca2d 100644 --- a/ledis/tx.go +++ b/ledis/tx.go @@ -33,11 +33,11 @@ func (t *tx) Put(key []byte, value []byte) { t.wb.Put(key, value) if t.binlog != nil { - buf := make([]byte, 9+len(key)+len(value)) + buf := make([]byte, 7+len(key)+len(value)) buf[0] = BinLogTypeValue pos := 1 - binary.BigEndian.PutUint32(buf[pos:], uint32(len(key))) - pos += 4 + binary.BigEndian.PutUint16(buf[pos:], uint16(len(key))) + pos += 2 copy(buf[pos:], key) pos += len(key) binary.BigEndian.PutUint32(buf[pos:], uint32(len(value))) @@ -52,11 +52,11 @@ func (t *tx) Delete(key []byte) { t.wb.Delete(key) if t.binlog != nil { - buf := make([]byte, 5+len(key)) + buf := make([]byte, 3+len(key)) buf[0] = BinLogTypeDeletion pos := 1 - binary.BigEndian.PutUint32(buf[pos:], uint32(len(key))) - pos += 4 + binary.BigEndian.PutUint16(buf[pos:], uint16(len(key))) + pos += 2 copy(buf[pos:], key) t.batch = append(t.batch, buf) From 93491791d599d77cdd34c12541e8a598d0bfe863 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 28 May 2014 14:20:45 +0800 Subject: [PATCH 03/31] add dump and load --- ledis/dump.go | 158 +++++++++++++++++++++++++++++++++++++++++++++ ledis/dump_test.go | 74 +++++++++++++++++++++ 2 files changed, 232 insertions(+) create mode 100644 ledis/dump.go create mode 100644 ledis/dump_test.go diff --git a/ledis/dump.go b/ledis/dump.go new file mode 100644 index 0000000..546ef31 --- /dev/null +++ b/ledis/dump.go @@ -0,0 +1,158 @@ +package ledis + +import ( + "bufio" + "bytes" + "encoding/binary" + "encoding/json" + "github.com/siddontang/go-leveldb/leveldb" + "io" + "os" +) + +//dump format +// head len(bigendian int32)|head(json format) +// |keylen(bigendian int32)|key|valuelen(bigendian int32)|value...... + +type DumpHead struct { + LogFile string `json:"bin_log_file"` + LogPos int64 `json:"bin_log_pos"` +} + +func (l *Ledis) DumpFile(path string) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + return l.Dump(f) +} + +func (l *Ledis) Dump(w io.Writer) error { + var sp *leveldb.Snapshot + var logFileName string + var logPos int64 + if l.binlog == nil { + sp = l.ldb.NewSnapshot() + } else { + l.binlog.Lock() + sp = l.ldb.NewSnapshot() + logFileName = l.binlog.LogFileName() + logPos = l.binlog.LogFilePos() + l.binlog.Unlock() + } + + var head = DumpHead{ + LogFile: logFileName, + LogPos: logPos, + } + + data, err := json.Marshal(&head) + if err != nil { + return err + } + + wb := bufio.NewWriterSize(w, 4096) + if err = binary.Write(wb, binary.BigEndian, uint32(len(data))); err != nil { + return err + } + + if _, err = wb.Write(data); err != nil { + return err + } + + it := sp.Iterator(nil, nil, leveldb.RangeClose, 0, -1) + var key []byte + var value []byte + for ; it.Valid(); it.Next() { + key = it.Key() + value = it.Value() + + if err = binary.Write(wb, binary.BigEndian, uint16(len(key))); err != nil { + return err + } + + if _, err = wb.Write(key); err != nil { + return err + } + + if err = binary.Write(wb, binary.BigEndian, uint32(len(value))); err != nil { + return err + } + + if _, err = wb.Write(value); err != nil { + return err + } + } + + if err = wb.Flush(); err != nil { + return err + } + + return nil +} + +func (l *Ledis) LoadDumpFile(path string) (*DumpHead, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + return l.LoadDump(f) +} + +func (l *Ledis) LoadDump(r io.Reader) (*DumpHead, error) { + rb := bufio.NewReaderSize(r, 4096) + + var headLen uint32 + err := binary.Read(rb, binary.BigEndian, &headLen) + if err != nil { + return nil, err + } + + buf := make([]byte, headLen) + if _, err = io.ReadFull(rb, buf); err != nil { + return nil, err + } + + var head DumpHead + if err = json.Unmarshal(buf, &head); err != nil { + return nil, err + } + + var keyLen uint16 + var valueLen uint32 + + var keyBuf bytes.Buffer + var valueBuf bytes.Buffer + for { + if err = binary.Read(rb, binary.BigEndian, &keyLen); err != nil && err != io.EOF { + return nil, err + } else if err == io.EOF { + break + } + + if _, err = io.CopyN(&keyBuf, rb, int64(keyLen)); err != nil { + return nil, err + } + + if err = binary.Read(rb, binary.BigEndian, &valueLen); err != nil { + return nil, err + } + + if _, err = io.CopyN(&valueBuf, rb, int64(valueLen)); err != nil { + return nil, err + } + + if err = l.ldb.Put(keyBuf.Bytes(), valueBuf.Bytes()); err != nil { + return nil, err + } + + keyBuf.Reset() + valueBuf.Reset() + } + + return &head, nil +} diff --git a/ledis/dump_test.go b/ledis/dump_test.go new file mode 100644 index 0000000..b27d8f0 --- /dev/null +++ b/ledis/dump_test.go @@ -0,0 +1,74 @@ +package ledis + +import ( + "bytes" + "github.com/siddontang/go-leveldb/leveldb" + "os" + "testing" +) + +func TestDump(t *testing.T) { + os.RemoveAll("/tmp/testdb_master") + os.RemoveAll("/tmp/testdb_slave") + os.Remove("/tmp/testdb.dump") + + var masterConfig = []byte(` + { + "data_db" : { + "path" : "/tmp/testdb_master", + "compression":true, + "block_size" : 32768, + "write_buffer_size" : 2097152, + "cache_size" : 20971520 + } + } + `) + + master, err := Open(masterConfig) + if err != nil { + t.Fatal(err) + } + + var slaveConfig = []byte(` + { + "data_db" : { + "path" : "/tmp/testdb_slave", + "compression":true, + "block_size" : 32768, + "write_buffer_size" : 2097152, + "cache_size" : 20971520 + } + } + `) + + var slave *Ledis + if slave, err = Open(slaveConfig); err != nil { + t.Fatal(err) + } + + db, _ := master.Select(0) + + db.Set([]byte("a"), []byte("1")) + db.Set([]byte("b"), []byte("2")) + db.Set([]byte("c"), []byte("3")) + + if err := master.DumpFile("/tmp/testdb.dump"); err != nil { + t.Fatal(err) + } + + if _, err := slave.LoadDumpFile("/tmp/testdb.dump"); err != nil { + t.Fatal(err) + } + + it := master.ldb.Iterator(nil, nil, leveldb.RangeClose, 0, -1) + for ; it.Valid(); it.Next() { + key := it.Key() + value := it.Value() + + if v, err := slave.ldb.Get(key); err != nil { + t.Fatal(err) + } else if !bytes.Equal(v, value) { + t.Fatal("load dump error") + } + } +} From 2b106981baa9b8c3783a3504a3fbe97d01ac5a4f Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 29 May 2014 15:07:14 +0800 Subject: [PATCH 04/31] add global lock to handle replication --- ledis/binlog.go | 24 +++++++++++++++--------- ledis/binlog_util.go | 34 ++++++++++++++++++++++++++++++++++ ledis/const.go | 7 ++----- ledis/dump.go | 11 +++++++++-- ledis/ledis.go | 3 +++ ledis/tx.go | 31 +++++++++---------------------- 6 files changed, 72 insertions(+), 38 deletions(-) create mode 100644 ledis/binlog_util.go diff --git a/ledis/binlog.go b/ledis/binlog.go index 716b11b..258af09 100644 --- a/ledis/binlog.go +++ b/ledis/binlog.go @@ -11,7 +11,6 @@ import ( "path" "strconv" "strings" - "sync" "time" ) @@ -28,6 +27,8 @@ timestamp(bigendian uint32, seconds)|PayloadLen(bigendian uint32)|PayloadData|Lo */ type BinLogConfig struct { + BaseName string `json:"base_name"` + IndexName string `json:"index_name"` Path string `json:"path"` MaxFileSize int `json:"max_file_size"` MaxFileNum int `json:"max_file_num"` @@ -45,11 +46,16 @@ func (cfg *BinLogConfig) adjust() { } else if cfg.MaxFileNum > MaxBinLogFileNum { cfg.MaxFileNum = MaxBinLogFileNum } + + if len(cfg.BaseName) == 0 { + cfg.BaseName = "ledis" + } + if len(cfg.IndexName) == 0 { + cfg.IndexName = "ledis" + } } type BinLog struct { - sync.Mutex - cfg *BinLogConfig logFile *os.File @@ -132,7 +138,7 @@ func (b *BinLog) flushIndex() error { } func (b *BinLog) loadIndex() error { - b.indexName = path.Join(b.cfg.Path, BinLogIndexFile) + b.indexName = path.Join(b.cfg.Path, fmt.Sprintf("%s-bin.index", b.cfg.IndexName)) fd, err := os.OpenFile(b.indexName, os.O_CREATE|os.O_RDWR, 0666) if err != nil { return err @@ -187,7 +193,7 @@ func (b *BinLog) loadIndex() error { } func (b *BinLog) getLogName() string { - return fmt.Sprintf("%s.%05d", BinLogBaseName, b.lastLogIndex) + return fmt.Sprintf("%s-bin.%05d", b.cfg.BaseName, b.lastLogIndex) } func (b *BinLog) openNewLogFile() error { @@ -241,14 +247,14 @@ func (b *BinLog) openLogFile() error { func (b *BinLog) Log(args ...[]byte) error { var err error + if err = b.openLogFile(); err != nil { + return err + } + for _, data := range args { createTime := uint32(time.Now().Unix()) payLoadLen := len(data) - if err = b.openLogFile(); err != nil { - return err - } - binary.Write(b.logWb, binary.BigEndian, createTime) binary.Write(b.logWb, binary.BigEndian, payLoadLen) diff --git a/ledis/binlog_util.go b/ledis/binlog_util.go new file mode 100644 index 0000000..b0abd4e --- /dev/null +++ b/ledis/binlog_util.go @@ -0,0 +1,34 @@ +package ledis + +import ( + "encoding/binary" +) + +func encodeBinLogDelete(key []byte) []byte { + buf := make([]byte, 3+len(key)) + buf[0] = BinLogTypeDeletion + pos := 1 + binary.BigEndian.PutUint16(buf[pos:], uint16(len(key))) + pos += 2 + copy(buf[pos:], key) + return buf +} + +func encodeBinLogPut(key []byte, value []byte) []byte { + buf := make([]byte, 7+len(key)+len(value)) + buf[0] = BinLogTypePut + pos := 1 + binary.BigEndian.PutUint16(buf[pos:], uint16(len(key))) + pos += 2 + copy(buf[pos:], key) + pos += len(key) + binary.BigEndian.PutUint32(buf[pos:], uint32(len(value))) + pos += 4 + copy(buf[pos:], value) + return buf +} + +func encodeBinLogCommand(commandType uint8, args []byte) []byte { + //to do + return nil +} diff --git a/ledis/const.go b/ledis/const.go index cfe3cd9..042cf45 100644 --- a/ledis/const.go +++ b/ledis/const.go @@ -43,9 +43,6 @@ var ( ErrZSetMemberSize = errors.New("invalid zset member size") ) -const BinLogBaseName = "ledis-bin" -const BinLogIndexFile = "ledis-bin.index" - const ( MaxBinLogFileSize int = 1024 * 1024 * 1024 MaxBinLogFileNum int = 10000 @@ -54,8 +51,8 @@ const ( DefaultBinLogFileNum int = 10 ) -//like leveldb const ( BinLogTypeDeletion uint8 = 0x0 - BinLogTypeValue uint8 = 0x1 + BinLogTypePut uint8 = 0x1 + BinLogTypeCommand uint8 = 0x2 ) diff --git a/ledis/dump.go b/ledis/dump.go index 546ef31..e7ff8a0 100644 --- a/ledis/dump.go +++ b/ledis/dump.go @@ -36,11 +36,11 @@ func (l *Ledis) Dump(w io.Writer) error { if l.binlog == nil { sp = l.ldb.NewSnapshot() } else { - l.binlog.Lock() + l.Lock() sp = l.ldb.NewSnapshot() logFileName = l.binlog.LogFileName() logPos = l.binlog.LogFilePos() - l.binlog.Unlock() + l.Unlock() } var head = DumpHead{ @@ -104,6 +104,9 @@ func (l *Ledis) LoadDumpFile(path string) (*DumpHead, error) { } func (l *Ledis) LoadDump(r io.Reader) (*DumpHead, error) { + l.Lock() + defer l.Unlock() + rb := bufio.NewReaderSize(r, 4096) var headLen uint32 @@ -150,6 +153,10 @@ func (l *Ledis) LoadDump(r io.Reader) (*DumpHead, error) { return nil, err } + if l.binlog != nil { + err = l.binlog.Log(encodeBinLogPut(keyBuf.Bytes(), valueBuf.Bytes())) + } + keyBuf.Reset() valueBuf.Reset() } diff --git a/ledis/ledis.go b/ledis/ledis.go index 8d4d950..7a3a0d5 100644 --- a/ledis/ledis.go +++ b/ledis/ledis.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "github.com/siddontang/go-leveldb/leveldb" + "sync" ) type Config struct { @@ -24,6 +25,8 @@ type DB struct { } type Ledis struct { + sync.Mutex + cfg *Config ldb *leveldb.DB diff --git a/ledis/tx.go b/ledis/tx.go index c5aca2d..fa7379b 100644 --- a/ledis/tx.go +++ b/ledis/tx.go @@ -1,7 +1,6 @@ package ledis import ( - "encoding/binary" "github.com/siddontang/go-leveldb/leveldb" "sync" ) @@ -9,6 +8,7 @@ import ( type tx struct { m sync.Mutex + l *Ledis wb *leveldb.WriteBatch binlog *BinLog @@ -18,6 +18,7 @@ type tx struct { func newTx(l *Ledis) *tx { t := new(tx) + t.l = l t.wb = l.ldb.NewWriteBatch() t.batch = make([][]byte, 0, 4) @@ -33,17 +34,7 @@ func (t *tx) Put(key []byte, value []byte) { t.wb.Put(key, value) if t.binlog != nil { - buf := make([]byte, 7+len(key)+len(value)) - buf[0] = BinLogTypeValue - pos := 1 - binary.BigEndian.PutUint16(buf[pos:], uint16(len(key))) - pos += 2 - copy(buf[pos:], key) - pos += len(key) - binary.BigEndian.PutUint32(buf[pos:], uint32(len(value))) - pos += 4 - copy(buf[pos:], value) - + buf := encodeBinLogPut(key, value) t.batch = append(t.batch, buf) } } @@ -52,13 +43,7 @@ func (t *tx) Delete(key []byte) { t.wb.Delete(key) if t.binlog != nil { - buf := make([]byte, 3+len(key)) - buf[0] = BinLogTypeDeletion - pos := 1 - binary.BigEndian.PutUint16(buf[pos:], uint16(len(key))) - pos += 2 - copy(buf[pos:], key) - + buf := encodeBinLogDelete(key) t.batch = append(t.batch, buf) } } @@ -76,18 +61,20 @@ func (t *tx) Unlock() { func (t *tx) Commit() error { var err error if t.binlog != nil { - t.binlog.Lock() + t.l.Lock() err = t.wb.Commit() if err != nil { - t.binlog.Unlock() + t.l.Unlock() return err } err = t.binlog.Log(t.batch...) - t.binlog.Unlock() + t.l.Unlock() } else { + t.l.Lock() err = t.wb.Commit() + t.l.Unlock() } return err } From 8cc62ad89fe428808288b5d785424e87e1d04620 Mon Sep 17 00:00:00 2001 From: siddontang Date: Tue, 3 Jun 2014 09:51:29 +0800 Subject: [PATCH 05/31] move bin log to replication package --- ledis/const.go | 8 -------- ledis/ledis.go | 7 ++++--- ledis/tx.go | 3 ++- {ledis => replication}/binlog.go | 10 +++++++++- {ledis => replication}/binlog_test.go | 2 +- replication/relaylog.go | 4 ++++ replication/relaylog_test.go | 9 +++++++++ 7 files changed, 29 insertions(+), 14 deletions(-) rename {ledis => replication}/binlog.go (96%) rename {ledis => replication}/binlog_test.go (96%) create mode 100644 replication/relaylog.go create mode 100644 replication/relaylog_test.go diff --git a/ledis/const.go b/ledis/const.go index 042cf45..9fc697d 100644 --- a/ledis/const.go +++ b/ledis/const.go @@ -43,14 +43,6 @@ var ( ErrZSetMemberSize = errors.New("invalid zset member size") ) -const ( - MaxBinLogFileSize int = 1024 * 1024 * 1024 - MaxBinLogFileNum int = 10000 - - DefaultBinLogFileSize int = MaxBinLogFileSize - DefaultBinLogFileNum int = 10 -) - const ( BinLogTypeDeletion uint8 = 0x0 BinLogTypePut uint8 = 0x1 diff --git a/ledis/ledis.go b/ledis/ledis.go index 7a3a0d5..fb668fe 100644 --- a/ledis/ledis.go +++ b/ledis/ledis.go @@ -4,13 +4,14 @@ import ( "encoding/json" "fmt" "github.com/siddontang/go-leveldb/leveldb" + "github.com/siddontang/ledisdb/replication" "sync" ) type Config struct { DataDB leveldb.Config `json:"data_db"` - BinLog BinLogConfig `json:"binlog"` + BinLog replication.BinLogConfig `json:"binlog"` } type DB struct { @@ -32,7 +33,7 @@ type Ledis struct { ldb *leveldb.DB dbs [MaxDBNumber]*DB - binlog *BinLog + binlog *replication.BinLog } func Open(configJson json.RawMessage) (*Ledis, error) { @@ -55,7 +56,7 @@ func OpenWithConfig(cfg *Config) (*Ledis, error) { l.ldb = ldb if len(cfg.BinLog.Path) > 0 { - l.binlog, err = NewBinLogWithConfig(&cfg.BinLog) + l.binlog, err = replication.NewBinLogWithConfig(&cfg.BinLog) if err != nil { return nil, err } diff --git a/ledis/tx.go b/ledis/tx.go index fa7379b..fc4357d 100644 --- a/ledis/tx.go +++ b/ledis/tx.go @@ -2,6 +2,7 @@ package ledis import ( "github.com/siddontang/go-leveldb/leveldb" + "github.com/siddontang/ledisdb/replication" "sync" ) @@ -11,7 +12,7 @@ type tx struct { l *Ledis wb *leveldb.WriteBatch - binlog *BinLog + binlog *replication.BinLog batch [][]byte } diff --git a/ledis/binlog.go b/replication/binlog.go similarity index 96% rename from ledis/binlog.go rename to replication/binlog.go index 258af09..435e297 100644 --- a/ledis/binlog.go +++ b/replication/binlog.go @@ -1,4 +1,4 @@ -package ledis +package replication import ( "bufio" @@ -14,6 +14,14 @@ import ( "time" ) +const ( + MaxBinLogFileSize int = 1024 * 1024 * 1024 + MaxBinLogFileNum int = 10000 + + DefaultBinLogFileSize int = MaxBinLogFileSize + DefaultBinLogFileNum int = 10 +) + /* index file format: ledis-bin.00001 diff --git a/ledis/binlog_test.go b/replication/binlog_test.go similarity index 96% rename from ledis/binlog_test.go rename to replication/binlog_test.go index 7fc89b4..a66ea11 100644 --- a/ledis/binlog_test.go +++ b/replication/binlog_test.go @@ -1,4 +1,4 @@ -package ledis +package replication import ( "io/ioutil" diff --git a/replication/relaylog.go b/replication/relaylog.go new file mode 100644 index 0000000..1043568 --- /dev/null +++ b/replication/relaylog.go @@ -0,0 +1,4 @@ +package replication + +type RelayLog struct { +} diff --git a/replication/relaylog_test.go b/replication/relaylog_test.go new file mode 100644 index 0000000..d29eac0 --- /dev/null +++ b/replication/relaylog_test.go @@ -0,0 +1,9 @@ +package replication + +import ( + "testing" +) + +func TestRelayLog(t *testing.T) { + +} From 263a89b9aaf764a5ff6f5d98a3e72ca32925e995 Mon Sep 17 00:00:00 2001 From: siddontang Date: Tue, 3 Jun 2014 14:08:57 +0800 Subject: [PATCH 06/31] bin log and relay log use same log --- ledis/ledis.go | 2 +- ledis/tx.go | 2 +- replication/binlog.go | 256 ++++--------------------------- replication/log.go | 287 +++++++++++++++++++++++++++++++++++ replication/relaylog.go | 54 ++++++- replication/relaylog_test.go | 31 ++++ 6 files changed, 401 insertions(+), 231 deletions(-) create mode 100644 replication/log.go diff --git a/ledis/ledis.go b/ledis/ledis.go index fb668fe..f0bdf09 100644 --- a/ledis/ledis.go +++ b/ledis/ledis.go @@ -33,7 +33,7 @@ type Ledis struct { ldb *leveldb.DB dbs [MaxDBNumber]*DB - binlog *replication.BinLog + binlog *replication.Log } func Open(configJson json.RawMessage) (*Ledis, error) { diff --git a/ledis/tx.go b/ledis/tx.go index fc4357d..6c4c023 100644 --- a/ledis/tx.go +++ b/ledis/tx.go @@ -12,7 +12,7 @@ type tx struct { l *Ledis wb *leveldb.WriteBatch - binlog *replication.BinLog + binlog *replication.Log batch [][]byte } diff --git a/replication/binlog.go b/replication/binlog.go index 435e297..677e087 100644 --- a/replication/binlog.go +++ b/replication/binlog.go @@ -4,13 +4,6 @@ import ( "bufio" "encoding/binary" "encoding/json" - "fmt" - "github.com/siddontang/go-log/log" - "io" - "os" - "path" - "strconv" - "strings" "time" ) @@ -35,11 +28,7 @@ timestamp(bigendian uint32, seconds)|PayloadLen(bigendian uint32)|PayloadData|Lo */ type BinLogConfig struct { - BaseName string `json:"base_name"` - IndexName string `json:"index_name"` - Path string `json:"path"` - MaxFileSize int `json:"max_file_size"` - MaxFileNum int `json:"max_file_num"` + LogConfig } func (cfg *BinLogConfig) adjust() { @@ -61,21 +50,35 @@ func (cfg *BinLogConfig) adjust() { if len(cfg.IndexName) == 0 { cfg.IndexName = "ledis" } + + cfg.SpaceLimit = -1 + + cfg.LogType = "bin" } -type BinLog struct { - cfg *BinLogConfig - - logFile *os.File - - logWb *bufio.Writer - - indexName string - logNames []string - lastLogIndex int +type binlogHandler struct { } -func NewBinLog(data json.RawMessage) (*BinLog, error) { +func (h *binlogHandler) Write(wb *bufio.Writer, data []byte) (int, error) { + createTime := uint32(time.Now().Unix()) + payLoadLen := uint32(len(data)) + + if err := binary.Write(wb, binary.BigEndian, createTime); err != nil { + return 0, err + } + + if err := binary.Write(wb, binary.BigEndian, payLoadLen); err != nil { + return 0, err + } + + if _, err := wb.Write(data); err != nil { + return 0, err + } + + return 8 + len(data), nil +} + +func NewBinLog(data json.RawMessage) (*Log, error) { var cfg BinLogConfig if err := json.Unmarshal(data, &cfg); err != nil { @@ -85,211 +88,8 @@ func NewBinLog(data json.RawMessage) (*BinLog, error) { return NewBinLogWithConfig(&cfg) } -func NewBinLogWithConfig(cfg *BinLogConfig) (*BinLog, error) { - b := new(BinLog) - +func NewBinLogWithConfig(cfg *BinLogConfig) (*Log, error) { cfg.adjust() - b.cfg = cfg - - if err := os.MkdirAll(cfg.Path, os.ModePerm); err != nil { - return nil, err - } - - b.logNames = make([]string, 0, b.cfg.MaxFileNum) - - if err := b.loadIndex(); err != nil { - return nil, err - } - - return b, nil -} - -func (b *BinLog) Close() { - if b.logFile != nil { - b.logFile.Close() - } -} - -func (b *BinLog) deleteOldest() { - logPath := path.Join(b.cfg.Path, b.logNames[0]) - os.Remove(logPath) - - copy(b.logNames[0:], b.logNames[1:]) - b.logNames = b.logNames[0 : len(b.logNames)-1] -} - -func (b *BinLog) flushIndex() error { - data := strings.Join(b.logNames, "\n") - - bakName := fmt.Sprintf("%s.bak", b.indexName) - f, err := os.OpenFile(bakName, os.O_WRONLY|os.O_CREATE, 0666) - if err != nil { - log.Error("create binlog bak index error %s", err.Error()) - return err - } - - if _, err := f.WriteString(data); err != nil { - log.Error("write binlog index error %s", err.Error()) - f.Close() - return err - } - - f.Close() - - if err := os.Rename(bakName, b.indexName); err != nil { - log.Error("rename binlog bak index error %s", err.Error()) - return err - } - - return nil -} - -func (b *BinLog) loadIndex() error { - b.indexName = path.Join(b.cfg.Path, fmt.Sprintf("%s-bin.index", b.cfg.IndexName)) - fd, err := os.OpenFile(b.indexName, os.O_CREATE|os.O_RDWR, 0666) - if err != nil { - return err - } - - //maybe we will check valid later? - rb := bufio.NewReader(fd) - for { - line, err := rb.ReadString('\n') - if err != nil && err != io.EOF { - fd.Close() - return err - } - - line = strings.Trim(line, "\r\n ") - - if len(line) > 0 { - b.logNames = append(b.logNames, line) - } - - if len(b.logNames) == b.cfg.MaxFileNum { - //remove oldest logfile - b.deleteOldest() - } - - if err == io.EOF { - break - } - } - - fd.Close() - - if err := b.flushIndex(); err != nil { - return err - } - - if len(b.logNames) == 0 { - b.lastLogIndex = 1 - } else { - lastName := b.logNames[len(b.logNames)-1] - - if b.lastLogIndex, err = strconv.Atoi(path.Ext(lastName)[1:]); err != nil { - log.Error("invalid logfile name %s", err.Error()) - return err - } - - //like mysql, if server restart, a new binlog will create - b.lastLogIndex++ - } - - return nil -} - -func (b *BinLog) getLogName() string { - return fmt.Sprintf("%s-bin.%05d", b.cfg.BaseName, b.lastLogIndex) -} - -func (b *BinLog) openNewLogFile() error { - var err error - lastName := b.getLogName() - - logPath := path.Join(b.cfg.Path, lastName) - if b.logFile, err = os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY, 0666); err != nil { - log.Error("open new logfile error %s", err.Error()) - return err - } - - if len(b.logNames) == b.cfg.MaxFileNum { - b.deleteOldest() - } - - b.logNames = append(b.logNames, lastName) - - if b.logWb == nil { - b.logWb = bufio.NewWriterSize(b.logFile, 1024) - } else { - b.logWb.Reset(b.logFile) - } - - if err = b.flushIndex(); err != nil { - return err - } - - return nil -} - -func (b *BinLog) openLogFile() error { - if b.logFile == nil { - return b.openNewLogFile() - } else { - //check file size - st, _ := b.logFile.Stat() - if st.Size() >= int64(b.cfg.MaxFileSize) { - //must use new file - b.lastLogIndex++ - - b.logFile.Close() - - return b.openNewLogFile() - } - } - - return nil -} - -func (b *BinLog) Log(args ...[]byte) error { - var err error - - if err = b.openLogFile(); err != nil { - return err - } - - for _, data := range args { - createTime := uint32(time.Now().Unix()) - payLoadLen := len(data) - - binary.Write(b.logWb, binary.BigEndian, createTime) - binary.Write(b.logWb, binary.BigEndian, payLoadLen) - - b.logWb.Write(data) - - if err = b.logWb.Flush(); err != nil { - log.Error("write log error %s", err.Error()) - return err - } - } - - return nil -} - -func (b *BinLog) LogFileName() string { - if len(b.logNames) == 0 { - return "" - } else { - return b.logNames[len(b.logNames)-1] - } -} - -func (b *BinLog) LogFilePos() int64 { - if b.logFile == nil { - return 0 - } else { - st, _ := b.logFile.Stat() - return st.Size() - } + return newLog(new(binlogHandler), &cfg.LogConfig) } diff --git a/replication/log.go b/replication/log.go new file mode 100644 index 0000000..e61e59d --- /dev/null +++ b/replication/log.go @@ -0,0 +1,287 @@ +package replication + +import ( + "bufio" + "errors" + "fmt" + "github.com/siddontang/go-log/log" + "io/ioutil" + "os" + "path" + "strconv" + "strings" +) + +var ( + ErrOverSpaceLimit = errors.New("total log files exceed space limit") +) + +type logHandler interface { + Write(wb *bufio.Writer, data []byte) (int, error) +} + +type LogConfig struct { + BaseName string `json:"base_name"` + IndexName string `json:"index_name"` + LogType string `json:"log_type"` + Path string `json:"path"` + MaxFileSize int `json:"max_file_size"` + MaxFileNum int `json:"max_file_num"` + SpaceLimit int64 `json:"space_limit"` +} + +type Log struct { + cfg *LogConfig + + logFile *os.File + + logWb *bufio.Writer + + indexName string + logNames []string + lastLogIndex int + + space int64 + + handler logHandler +} + +func newLog(handler logHandler, cfg *LogConfig) (*Log, error) { + l := new(Log) + + l.cfg = cfg + l.handler = handler + + if err := os.MkdirAll(cfg.Path, os.ModePerm); err != nil { + return nil, err + } + + l.logNames = make([]string, 0, 16) + l.space = 0 + + if err := l.loadIndex(); err != nil { + return nil, err + } + + return l, nil +} + +func (l *Log) flushIndex() error { + data := strings.Join(l.logNames, "\n") + + bakName := fmt.Sprintf("%s.bak", l.indexName) + f, err := os.OpenFile(bakName, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + log.Error("create binlog bak index error %s", err.Error()) + return err + } + + if _, err := f.WriteString(data); err != nil { + log.Error("write binlog index error %s", err.Error()) + f.Close() + return err + } + + f.Close() + + if err := os.Rename(bakName, l.indexName); err != nil { + log.Error("rename binlog bak index error %s", err.Error()) + return err + } + + return nil +} + +func (l *Log) loadIndex() error { + l.indexName = path.Join(l.cfg.Path, fmt.Sprintf("%s-%s.index", l.cfg.IndexName, l.cfg.LogType)) + if _, err := os.Stat(l.indexName); os.IsNotExist(err) { + //no index file, nothing to do + } else { + indexData, err := ioutil.ReadFile(l.indexName) + if err != nil { + return err + } + + lines := strings.Split(string(indexData), "\n") + for _, line := range lines { + line = strings.Trim(line, "\r\n ") + if len(line) == 0 { + continue + } + + if st, err := os.Stat(path.Join(l.cfg.Path, line)); err != nil { + log.Error("load index line %s error %s", line, err.Error()) + return err + } else { + l.space += st.Size() + + l.logNames = append(l.logNames, line) + } + } + } + if l.cfg.MaxFileNum > 0 && len(l.logNames) > l.cfg.MaxFileNum { + //remove oldest logfile + if err := l.Purge(len(l.logNames) - l.cfg.MaxFileNum); err != nil { + return err + } + } + + var err error + if len(l.logNames) == 0 { + l.lastLogIndex = 1 + } else { + lastName := l.logNames[len(l.logNames)-1] + + if l.lastLogIndex, err = strconv.Atoi(path.Ext(lastName)[1:]); err != nil { + log.Error("invalid logfile name %s", err.Error()) + return err + } + + //like mysql, if server restart, a new binlog will create + l.lastLogIndex++ + } + + return nil +} + +func (l *Log) getLogFile() string { + return fmt.Sprintf("%s-%s.%05d", l.cfg.BaseName, l.cfg.LogType, l.lastLogIndex) +} + +func (l *Log) openNewLogFile() error { + var err error + lastName := l.getLogFile() + + logPath := path.Join(l.cfg.Path, lastName) + if l.logFile, err = os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY, 0666); err != nil { + log.Error("open new logfile error %s", err.Error()) + return err + } + + if l.cfg.MaxFileNum > 0 && len(l.logNames) == l.cfg.MaxFileNum { + l.purge(1) + } + + l.logNames = append(l.logNames, lastName) + + if l.logWb == nil { + l.logWb = bufio.NewWriterSize(l.logFile, 1024) + } else { + l.logWb.Reset(l.logFile) + } + + if err = l.flushIndex(); err != nil { + return err + } + + return nil +} + +func (l *Log) checkLogFileSize() bool { + if l.logFile == nil { + return false + } + + st, _ := l.logFile.Stat() + if st.Size() >= int64(l.cfg.MaxFileSize) { + l.lastLogIndex++ + + l.logFile.Close() + l.logFile = nil + return true + } + + return false +} + +func (l *Log) purge(n int) { + for i := 0; i < n; i++ { + logPath := path.Join(l.cfg.Path, l.logNames[i]) + if st, err := os.Stat(logPath); err != nil { + log.Error("purge %s error %s", logPath, err.Error()) + } else { + l.space -= st.Size() + } + + os.Remove(logPath) + } + + copy(l.logNames[0:], l.logNames[n:]) + l.logNames = l.logNames[0 : len(l.logNames)-n] +} + +func (l *Log) Close() { + if l.logFile != nil { + l.logFile.Close() + l.logFile = nil + } +} + +func (l *Log) LogFileName() string { + return l.getLogFile() +} + +func (l *Log) LogFilePos() int64 { + if l.logFile == nil { + return 0 + } else { + st, _ := l.logFile.Stat() + return st.Size() + } +} + +func (l *Log) Purge(n int) error { + if len(l.logNames) == 0 { + return nil + } + + if n >= len(l.logNames) { + n = len(l.logNames) + //can not purge current log file + if l.logNames[n-1] == l.getLogFile() { + n = n - 1 + } + } + + l.purge(n) + + return l.flushIndex() +} + +func (l *Log) Log(args ...[]byte) error { + if l.cfg.SpaceLimit > 0 && l.space >= l.cfg.SpaceLimit { + return ErrOverSpaceLimit + } + + var err error + + if l.logFile == nil { + if err = l.openNewLogFile(); err != nil { + return err + } + } + + totalSize := 0 + + var n int = 0 + for _, data := range args { + if n, err = l.handler.Write(l.logWb, data); err != nil { + log.Error("write log error %s", err.Error()) + return err + } else { + totalSize += n + } + + } + + if err = l.logWb.Flush(); err != nil { + log.Error("write log error %s", err.Error()) + return err + } + + l.space += int64(totalSize) + + l.checkLogFileSize() + + return nil +} diff --git a/replication/relaylog.go b/replication/relaylog.go index 1043568..9f2b7a4 100644 --- a/replication/relaylog.go +++ b/replication/relaylog.go @@ -1,4 +1,56 @@ package replication -type RelayLog struct { +import ( + "bufio" + "encoding/json" +) + +const ( + MaxRelayLogFileSize int = 1024 * 1024 * 1024 + DefaultRelayLogFileSize int = MaxRelayLogFileSize +) + +type RelayLogConfig struct { + LogConfig +} + +func (cfg *RelayLogConfig) adjust() { + if cfg.MaxFileSize <= 0 { + cfg.MaxFileSize = DefaultRelayLogFileSize + } else if cfg.MaxFileSize > MaxRelayLogFileSize { + cfg.MaxFileSize = MaxRelayLogFileSize + } + + if len(cfg.BaseName) == 0 { + cfg.BaseName = "ledis" + } + if len(cfg.IndexName) == 0 { + cfg.IndexName = "ledis" + } + + cfg.MaxFileNum = -1 + cfg.LogType = "relay" +} + +type relayLogHandler struct { +} + +func (h *relayLogHandler) Write(wb *bufio.Writer, data []byte) (int, error) { + return wb.Write(data) +} + +func NewRelayLog(data json.RawMessage) (*Log, error) { + var cfg RelayLogConfig + + if err := json.Unmarshal(data, &cfg); err != nil { + return nil, err + } + + return NewRelayLogWithConfig(&cfg) +} + +func NewRelayLogWithConfig(cfg *RelayLogConfig) (*Log, error) { + cfg.adjust() + + return newLog(new(relayLogHandler), &cfg.LogConfig) } diff --git a/replication/relaylog_test.go b/replication/relaylog_test.go index d29eac0..2e37af8 100644 --- a/replication/relaylog_test.go +++ b/replication/relaylog_test.go @@ -1,9 +1,40 @@ package replication import ( + "os" "testing" ) func TestRelayLog(t *testing.T) { + cfg := new(RelayLogConfig) + + cfg.MaxFileSize = 1024 + cfg.SpaceLimit = 1024 + cfg.Path = "/tmp/ledis_relaylog" + + os.RemoveAll(cfg.Path) + + b, err := NewRelayLogWithConfig(cfg) + if err != nil { + t.Fatal(err) + } + + if err := b.Log(make([]byte, 1024)); err != nil { + t.Fatal(err) + } + + if err := b.Log(make([]byte, 1)); err == nil { + t.Fatal("must not nil") + } else if err != ErrOverSpaceLimit { + t.Fatal(err) + } + + if err := b.Purge(1); err != nil { + t.Fatal(err) + } + + if err := b.Log(make([]byte, 1)); err != nil { + t.Fatal(err) + } } From 5c7d88516800b3adfb1245c0264851a5fa8828b4 Mon Sep 17 00:00:00 2001 From: silentsai Date: Tue, 3 Jun 2014 15:40:10 +0800 Subject: [PATCH 07/31] add expire for the kv type; simplify the listMetaKey format --- ledis/const.go | 10 +++ ledis/ledis.go | 2 + ledis/ledis_db.go | 21 +++++- ledis/ledis_test.go | 2 +- ledis/t_hash.go | 2 + ledis/t_kv.go | 78 +++++++++++++++++--- ledis/t_kv_test.go | 2 +- ledis/t_list.go | 170 ++++++++++++++++++++----------------------- ledis/t_list_test.go | 32 +++++++- ledis/t_zset.go | 4 + 10 files changed, 218 insertions(+), 105 deletions(-) diff --git a/ledis/const.go b/ledis/const.go index 042cf45..98c6b18 100644 --- a/ledis/const.go +++ b/ledis/const.go @@ -13,6 +13,15 @@ const ( zsetType zSizeType zScoreType + + kvExpType + kvExpMetaType + lExpType + lExpMetaType + hExpType + hExpMetaType + zExpType + zExpMetaType ) const ( @@ -41,6 +50,7 @@ var ( ErrValueSize = errors.New("invalid value size") ErrHashFieldSize = errors.New("invalid hash field size") ErrZSetMemberSize = errors.New("invalid zset member size") + ErrExpireValue = errors.New("invalid expire value") ) const ( diff --git a/ledis/ledis.go b/ledis/ledis.go index 7a3a0d5..b1bb54c 100644 --- a/ledis/ledis.go +++ b/ledis/ledis.go @@ -82,6 +82,8 @@ func newDB(l *Ledis, index uint8) *DB { d.hashTx = newTx(l) d.zsetTx = newTx(l) + d.activeExpireCycle() + return d } diff --git a/ledis/ledis_db.go b/ledis/ledis_db.go index af583c1..a45f59b 100644 --- a/ledis/ledis_db.go +++ b/ledis/ledis_db.go @@ -1,8 +1,12 @@ package ledis -func (db *DB) Flush() (drop int64, err error) { +import ( + "time" +) + +func (db *DB) FlushAll() (drop int64, err error) { all := [...](func() (int64, error)){ - db.KvFlush, + db.Flush, db.LFlush, db.HFlush, db.ZFlush} @@ -15,5 +19,18 @@ func (db *DB) Flush() (drop int64, err error) { drop += n } } + return } + +func (db *DB) activeExpireCycle() { + eliminator := newEliminator(db) + eliminator.regRetireContext(kvExpType, db.kvTx, db.delete) + + go func() { + for { + eliminator.active() + time.Sleep(1 * time.Second) + } + }() +} diff --git a/ledis/ledis_test.go b/ledis/ledis_test.go index 6967e31..e0f812d 100644 --- a/ledis/ledis_test.go +++ b/ledis/ledis_test.go @@ -81,7 +81,7 @@ func TestFlush(t *testing.T) { db1.LPush([]byte("lst"), []byte("a1"), []byte("b2")) db1.ZAdd([]byte("zset_0"), ScorePair{int64(3), []byte("mc")}) - db1.Flush() + db1.FlushAll() // 0 - existing if exists, _ := db0.Exists([]byte("a")); exists <= 0 { diff --git a/ledis/t_hash.go b/ledis/t_hash.go index bb700c2..8a57fc2 100644 --- a/ledis/t_hash.go +++ b/ledis/t_hash.go @@ -424,6 +424,7 @@ func (db *DB) HFlush() (drop int64, err error) { } } } + it.Close() err = t.Commit() return @@ -461,6 +462,7 @@ func (db *DB) HScan(key []byte, field []byte, count int, inclusive bool) ([]FVPa v = append(v, FVPair{Field: f, Value: it.Value()}) } } + it.Close() return v, nil } diff --git a/ledis/t_kv.go b/ledis/t_kv.go index 9469604..e1e0dba 100644 --- a/ledis/t_kv.go +++ b/ledis/t_kv.go @@ -3,6 +3,7 @@ package ledis import ( "errors" "github.com/siddontang/go-leveldb/leveldb" + "time" ) type KVPair struct { @@ -83,6 +84,12 @@ func (db *DB) incr(key []byte, delta int64) (int64, error) { return n, err } +func (db *DB) delete(t *tx, key []byte) int64 { + key = db.encodeKVKey(key) + t.Delete(key) + return 1 +} + func (db *DB) Decr(key []byte) (int64, error) { return db.incr(key, -1) } @@ -96,22 +103,22 @@ func (db *DB) Del(keys ...[]byte) (int64, error) { return 0, nil } - var err error - for i := range keys { - keys[i] = db.encodeKVKey(keys[i]) + codedKeys := make([][]byte, len(keys)) + for i, k := range keys { + codedKeys[i] = db.encodeKVKey(k) } t := db.kvTx - t.Lock() defer t.Unlock() - for i := range keys { - t.Delete(keys[i]) + for i, k := range keys { + t.Delete(codedKeys[i]) + db.rmExpire(t, kvExpType, k) //todo binlog } - err = t.Commit() + err := t.Commit() return int64(len(keys)), err } @@ -287,7 +294,7 @@ func (db *DB) SetNX(key []byte, value []byte) (int64, error) { return n, err } -func (db *DB) KvFlush() (drop int64, err error) { +func (db *DB) Flush() (drop int64, err error) { t := db.kvTx t.Lock() defer t.Unlock() @@ -300,14 +307,16 @@ func (db *DB) KvFlush() (drop int64, err error) { t.Delete(it.Key()) drop++ - if drop%1000 == 0 { + if drop&1023 == 0 { if err = t.Commit(); err != nil { return } } } + it.Close() err = t.Commit() + err = db.expFlush(t, kvExpType) return } @@ -344,6 +353,57 @@ func (db *DB) Scan(key []byte, count int, inclusive bool) ([]KVPair, error) { v = append(v, KVPair{Key: key, Value: it.Value()}) } } + it.Close() return v, nil } + +func (db *DB) Expire(key []byte, duration int64) (int64, error) { + if duration <= 0 { + return 0, ErrExpireValue + } + + t := db.kvTx + t.Lock() + defer t.Unlock() + + if exist, err := db.Exists(key); err != nil || exist == 0 { + return 0, err + } else { + db.expire(t, kvExpType, key, duration) + if err := t.Commit(); err != nil { + return 0, err + } else { + return 1, nil + } + } +} + +func (db *DB) ExpireAt(key []byte, when int64) (int64, error) { + if when <= time.Now().Unix() { + return 0, ErrExpireValue + } + + t := db.kvTx + t.Lock() + defer t.Unlock() + + if exist, err := db.Exists(key); err != nil || exist == 0 { + return 0, err + } else { + db.expireAt(t, kvExpType, key, when) + if err := t.Commit(); err != nil { + return 0, err + } else { + return 1, nil + } + } +} + +func (db *DB) Ttl(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return -1, err + } + + return db.ttl(kvExpType, key) +} diff --git a/ledis/t_kv_test.go b/ledis/t_kv_test.go index 967eab6..0252421 100644 --- a/ledis/t_kv_test.go +++ b/ledis/t_kv_test.go @@ -29,7 +29,7 @@ func TestDBKV(t *testing.T) { func TestDBScan(t *testing.T) { db := getTestDB() - db.Flush() + db.FlushAll() if v, err := db.Scan(nil, 10, true); err != nil { t.Fatal(err) diff --git a/ledis/t_list.go b/ledis/t_list.go index eac2769..5d1355b 100644 --- a/ledis/t_list.go +++ b/ledis/t_list.go @@ -84,62 +84,53 @@ func (db *DB) lpush(key []byte, whereSeq int32, args ...[]byte) (int64, error) { var err error metaKey := db.lEncodeMetaKey(key) + headSeq, tailSeq, size, err = db.lGetMeta(metaKey) + if err != nil { + return 0, err + } - if len(args) == 0 { - _, _, size, err := db.lGetMeta(metaKey) - return int64(size), err + var pushCnt int = len(args) + if pushCnt == 0 { + return int64(size), nil + } + + var seq int32 = headSeq + var delta int32 = -1 + if whereSeq == listTailSeq { + seq = tailSeq + delta = 1 } t := db.listTx t.Lock() defer t.Unlock() - if headSeq, tailSeq, size, err = db.lGetMeta(metaKey); err != nil { - return 0, err - } - - var delta int32 = 1 - var seq int32 = 0 - if whereSeq == listHeadSeq { - delta = -1 - seq = headSeq - } else { - seq = tailSeq - } - - if size == 0 { - headSeq = listInitialSeq - tailSeq = listInitialSeq - seq = headSeq - } else { + // append elements + if size > 0 { seq += delta } - for i := 0; i < len(args); i++ { + for i := 0; i < pushCnt; i++ { ek := db.lEncodeListKey(key, seq+int32(i)*delta) t.Put(ek, args[i]) - //to do add binlog } - seq += int32(len(args)-1) * delta - + seq += int32(pushCnt-1) * delta if seq <= listMinSeq || seq >= listMaxSeq { return 0, errListSeq } - size += int32(len(args)) - + // set meta info if whereSeq == listHeadSeq { headSeq = seq } else { tailSeq = seq } - db.lSetMeta(metaKey, headSeq, tailSeq, size) + db.lSetMeta(metaKey, headSeq, tailSeq) err = t.Commit() - - return int64(size), err + return int64(size) + int64(pushCnt), err } func (db *DB) lpop(key []byte, whereSeq int32) ([]byte, error) { @@ -153,54 +144,68 @@ func (db *DB) lpop(key []byte, whereSeq int32) ([]byte, error) { var headSeq int32 var tailSeq int32 - var size int32 var err error metaKey := db.lEncodeMetaKey(key) - - headSeq, tailSeq, size, err = db.lGetMeta(metaKey) - + headSeq, tailSeq, _, err = db.lGetMeta(metaKey) if err != nil { return nil, err } - var seq int32 = 0 - var delta int32 = 1 - if whereSeq == listHeadSeq { - seq = headSeq - } else { - delta = -1 + var value []byte + + var seq int32 = headSeq + if whereSeq == listTailSeq { seq = tailSeq } itemKey := db.lEncodeListKey(key, seq) - var value []byte value, err = db.db.Get(itemKey) if err != nil { return nil, err } - t.Delete(itemKey) - seq += delta - - size-- - if size <= 0 { - t.Delete(metaKey) + if whereSeq == listHeadSeq { + headSeq += 1 } else { - if whereSeq == listHeadSeq { - headSeq = seq - } else { - tailSeq = seq - } - - db.lSetMeta(metaKey, headSeq, tailSeq, size) + tailSeq -= 1 } - //todo add binlog + t.Delete(itemKey) + db.lSetMeta(metaKey, headSeq, tailSeq) + err = t.Commit() return value, err } +func (db *DB) lDelete(t *tx, key []byte) int64 { + mk := db.lEncodeMetaKey(key) + + var headSeq int32 + var tailSeq int32 + var err error + + headSeq, tailSeq, _, err = db.lGetMeta(mk) + if err != nil { + return 0 + } + + var num int64 = 0 + startKey := db.lEncodeListKey(key, headSeq) + stopKey := db.lEncodeListKey(key, tailSeq) + + it := db.db.Iterator(startKey, stopKey, leveldb.RangeClose, 0, -1) + for ; it.Valid(); it.Next() { + t.Delete(it.Key()) + num++ + } + it.Close() + + t.Delete(mk) + + return num +} + func (db *DB) lGetSeq(key []byte, whereSeq int32) (int64, error) { ek := db.lEncodeListKey(key, whereSeq) @@ -213,26 +218,35 @@ func (db *DB) lGetMeta(ek []byte) (headSeq int32, tailSeq int32, size int32, err if err != nil { return } else if v == nil { + headSeq = listInitialSeq + tailSeq = listInitialSeq size = 0 return } else { headSeq = int32(binary.LittleEndian.Uint32(v[0:4])) tailSeq = int32(binary.LittleEndian.Uint32(v[4:8])) - size = int32(binary.LittleEndian.Uint32(v[8:])) + size = tailSeq - headSeq + 1 } return } -func (db *DB) lSetMeta(ek []byte, headSeq int32, tailSeq int32, size int32) { +func (db *DB) lSetMeta(ek []byte, headSeq int32, tailSeq int32) { t := db.listTx - buf := make([]byte, 12) + var size int32 = tailSeq - headSeq + 1 + if size < 0 { + // todo : log error + panic + } else if size == 0 { + t.Delete(ek) + } else { + buf := make([]byte, 8) - binary.LittleEndian.PutUint32(buf[0:4], uint32(headSeq)) - binary.LittleEndian.PutUint32(buf[4:8], uint32(tailSeq)) - binary.LittleEndian.PutUint32(buf[8:], uint32(size)) + binary.LittleEndian.PutUint32(buf[0:4], uint32(headSeq)) + binary.LittleEndian.PutUint32(buf[4:8], uint32(tailSeq)) + //binary.LittleEndian.PutUint32(buf[8:], uint32(size)) - t.Put(ek, buf) + t.Put(ek, buf) + } } func (db *DB) LIndex(key []byte, index int32) ([]byte, error) { @@ -347,37 +361,13 @@ func (db *DB) LClear(key []byte) (int64, error) { return 0, err } - mk := db.lEncodeMetaKey(key) - t := db.listTx t.Lock() defer t.Unlock() - var headSeq int32 - var tailSeq int32 - var err error + num := db.lDelete(t, key) - headSeq, tailSeq, _, err = db.lGetMeta(mk) - - if err != nil { - return 0, err - } - - var num int64 = 0 - startKey := db.lEncodeListKey(key, headSeq) - stopKey := db.lEncodeListKey(key, tailSeq) - - it := db.db.Iterator(startKey, stopKey, leveldb.RangeClose, 0, -1) - for ; it.Valid(); it.Next() { - t.Delete(it.Key()) - num++ - } - - it.Close() - - t.Delete(mk) - - err = t.Commit() + err := t.Commit() return num, err } @@ -398,13 +388,13 @@ func (db *DB) LFlush() (drop int64, err error) { for ; it.Valid(); it.Next() { t.Delete(it.Key()) drop++ - if drop%1000 == 0 { + if drop&1023 == 0 { if err = t.Commit(); err != nil { return } } - } + it.Close() err = t.Commit() return diff --git a/ledis/t_list_test.go b/ledis/t_list_test.go index fe5d24d..b7cb660 100644 --- a/ledis/t_list_test.go +++ b/ledis/t_list_test.go @@ -31,9 +31,37 @@ func TestDBList(t *testing.T) { key := []byte("testdb_list_a") - if n, err := db.RPush(key, []byte("1"), []byte("2")); err != nil { + if n, err := db.RPush(key, []byte("1"), []byte("2"), []byte("3")); err != nil { t.Fatal(err) - } else if n != 2 { + } else if n != 3 { t.Fatal(n) } + + if k, err := db.RPop(key); err != nil { + t.Fatal(err) + } else if string(k) != "3" { + t.Fatal(string(k)) + } + + if k, err := db.LPop(key); err != nil { + t.Fatal(err) + } else if string(k) != "1" { + t.Fatal(string(k)) + } + + if llen, err := db.LLen(key); err != nil { + t.Fatal(err) + } else if llen != 1 { + t.Fatal(llen) + } + + if num, err := db.LClear(key); err != nil { + t.Fatal(err) + } else if num != 1 { + t.Fatal(num) + } + + if llen, _ := db.LLen(key); llen != 0 { + t.Fatal(llen) + } } diff --git a/ledis/t_zset.go b/ledis/t_zset.go index 1c86a9e..f548ca9 100644 --- a/ledis/t_zset.go +++ b/ledis/t_zset.go @@ -502,6 +502,7 @@ func (db *DB) zRemRange(key []byte, min int64, max int64, offset int, limit int) t.Delete(k) } + it.Close() if _, err := db.zIncrSize(key, -num); err != nil { return 0, err @@ -569,6 +570,7 @@ func (db *DB) zRange(key []byte, min int64, max int64, withScores bool, offset i v = append(v, StrPutInt64(s)) } } + it.Close() if reverse && (offset == 0 && limit < 0) { v = db.zReverse(v, withScores) @@ -703,6 +705,7 @@ func (db *DB) ZFlush() (drop int64, err error) { } } } + it.Close() err = t.Commit() // to do : binlog @@ -743,6 +746,7 @@ func (db *DB) ZScan(key []byte, member []byte, count int, inclusive bool) ([]Sco v = append(v, ScorePair{Member: m, Score: score}) } } + it.Close() return v, nil } From 654a0a4704a44d4112b3817f972fd5a5a83043a8 Mon Sep 17 00:00:00 2001 From: silentsai Date: Tue, 3 Jun 2014 15:41:17 +0800 Subject: [PATCH 08/31] add expire for the kv type; simplify the listMetaKey format --- ledis/t_ttl.go | 208 ++++++++++++++++++++++++++++++++++++++++++++ ledis/t_ttl_test.go | 172 ++++++++++++++++++++++++++++++++++++ 2 files changed, 380 insertions(+) create mode 100644 ledis/t_ttl.go create mode 100644 ledis/t_ttl_test.go diff --git a/ledis/t_ttl.go b/ledis/t_ttl.go new file mode 100644 index 0000000..167acce --- /dev/null +++ b/ledis/t_ttl.go @@ -0,0 +1,208 @@ +package ledis + +import ( + "encoding/binary" + "errors" + "github.com/siddontang/go-leveldb/leveldb" + "time" +) + +var mapExpMetaType = map[byte]byte{ + kvExpType: kvExpMetaType, + lExpType: lExpMetaType, + hExpType: hExpMetaType, + zExpType: zExpMetaType} + +type retireCallback func(*tx, []byte) int64 + +type Elimination struct { + db *DB + exp2Tx map[byte]*tx + exp2Retire map[byte]retireCallback +} + +var errExpType = errors.New("invalid expire type") + +func (db *DB) expEncodeTimeKey(expType byte, key []byte, when int64) []byte { + // format : db[8] / expType[8] / when[64] / key[...] + buf := make([]byte, len(key)+10) + + buf[0] = db.index + buf[1] = expType + pos := 2 + + binary.BigEndian.PutUint64(buf[pos:], uint64(when)) + pos += 8 + + copy(buf[pos:], key) + + return buf +} + +func (db *DB) expEncodeMetaKey(expType byte, key []byte) []byte { + // format : db[8] / expType[8] / key[...] + buf := make([]byte, len(key)+2) + + buf[0] = db.index + buf[1] = expType + pos := 2 + + copy(buf[pos:], key) + + return buf +} + +// usage : separate out the original key +func (db *DB) expDecodeMetaKey(mk []byte) []byte { + if len(mk) <= 2 { + // check db ? check type ? + return nil + } + + return mk[2:] +} + +func (db *DB) expire(t *tx, expType byte, key []byte, duration int64) { + db.expireAt(t, expType, key, time.Now().Unix()+duration) +} + +func (db *DB) expireAt(t *tx, expType byte, key []byte, when int64) { + mk := db.expEncodeMetaKey(expType+1, key) + tk := db.expEncodeTimeKey(expType, key, when) + + t.Put(tk, mk) + t.Put(mk, PutInt64(when)) +} + +func (db *DB) ttl(expType byte, key []byte) (t int64, err error) { + mk := db.expEncodeMetaKey(expType+1, key) + + if t, err = Int64(db.db.Get(mk)); err != nil || t == 0 { + t = -1 + } else { + t -= time.Now().Unix() + if t <= 0 { + t = -1 + } + // if t == -1 : to remove ???? + } + + return t, err +} + +func (db *DB) rmExpire(t *tx, expType byte, key []byte) { + mk := db.expEncodeMetaKey(expType+1, key) + when, err := Int64(db.db.Get(mk)) + if err == nil && when > 0 { + tk := db.expEncodeTimeKey(expType, key, when) + + t.Delete(mk) + t.Delete(tk) + } +} + +func (db *DB) expFlush(t *tx, expType byte) (err error) { + expMetaType, ok := mapExpMetaType[expType] + if !ok { + return errExpType + } + + drop := 0 + + minKey := make([]byte, 2) + minKey[0] = db.index + minKey[1] = expType + + maxKey := make([]byte, 2) + maxKey[0] = db.index + maxKey[1] = expMetaType + 1 + + it := db.db.Iterator(minKey, maxKey, leveldb.RangeROpen, 0, -1) + for ; it.Valid(); it.Next() { + t.Delete(it.Key()) + drop++ + if drop&1023 == 0 { + if err = t.Commit(); err != nil { + return + } + } + } + + err = t.Commit() + return +} + +////////////////////////////////////////////////////////// +// +////////////////////////////////////////////////////////// + +func newEliminator(db *DB) *Elimination { + eli := new(Elimination) + eli.db = db + eli.exp2Tx = make(map[byte]*tx) + eli.exp2Retire = make(map[byte]retireCallback) + return eli +} + +func (eli *Elimination) regRetireContext(expType byte, t *tx, onRetire retireCallback) { + eli.exp2Tx[expType] = t + eli.exp2Retire[expType] = onRetire +} + +// call by outside ... (from *db to another *db) +func (eli *Elimination) active() { + now := time.Now().Unix() + db := eli.db + dbGet := db.db.Get + expKeys := make([][]byte, 0, 1024) + expTypes := [...]byte{kvExpType, lExpType, hExpType, zExpType} + + for _, et := range expTypes { + // search those keys' which expire till the moment + minKey := db.expEncodeTimeKey(et, nil, 0) + maxKey := db.expEncodeTimeKey(et, nil, now+1) + expKeys = expKeys[0:0] + + t, _ := eli.exp2Tx[et] + onRetire, _ := eli.exp2Retire[et] + if t == nil || onRetire == nil { + // todo : log error + continue + } + + it := db.db.Iterator(minKey, maxKey, leveldb.RangeROpen, 0, -1) + for it.Valid() { + for i := 1; i < 512 && it.Valid(); i++ { + expKeys = append(expKeys, it.Key(), it.Value()) + it.Next() + } + + var cnt int = len(expKeys) + if cnt == 0 { + continue + } + + t.Lock() + var mk, ek, k []byte + for i := 0; i < cnt; i += 2 { + ek, mk = expKeys[i], expKeys[i+1] + if exp, err := Int64(dbGet(mk)); err == nil { + // check expire again + if exp > now { + continue + } + + // delete keys + k = db.expDecodeMetaKey(mk) + onRetire(t, k) + t.Delete(ek) + t.Delete(mk) + } + } + t.Commit() + t.Unlock() + } // end : it + } // end : expType + + return +} diff --git a/ledis/t_ttl_test.go b/ledis/t_ttl_test.go new file mode 100644 index 0000000..0483ae7 --- /dev/null +++ b/ledis/t_ttl_test.go @@ -0,0 +1,172 @@ +package ledis + +import ( + "fmt" + "sync" + "testing" + "time" +) + +var m sync.Mutex + +func TestKvExpire(t *testing.T) { + db := getTestDB() + m.Lock() + defer m.Unlock() + + k := []byte("ttl_a") + ek := []byte("ttl_b") + db.Set(k, []byte("1")) + + if ok, _ := db.Expire(k, 10); ok != 1 { + t.Fatal(ok) + } + + // err - expire on an inexisting key + if ok, _ := db.Expire(ek, 10); ok != 0 { + t.Fatal(ok) + } + + // err - duration is zero + if ok, err := db.Expire(k, 0); err == nil || ok != 0 { + t.Fatal(fmt.Sprintf("res = %d, err = %s", ok, err)) + } + + // err - duration is negative + if ok, err := db.Expire(k, -10); err == nil || ok != 0 { + t.Fatal(fmt.Sprintf("res = %d, err = %s", ok, err)) + } +} + +func TestKvExpireAt(t *testing.T) { + db := getTestDB() + m.Lock() + defer m.Unlock() + + k := []byte("ttl_a") + ek := []byte("ttl_b") + db.Set(k, []byte("1")) + + now := time.Now().Unix() + + if ok, _ := db.ExpireAt(k, now+5); ok != 1 { + t.Fatal(ok) + } + + // err - expire on an inexisting key + if ok, _ := db.ExpireAt(ek, now+5); ok != 0 { + t.Fatal(ok) + } + + // err - expire with the current time + if ok, err := db.ExpireAt(k, now); err == nil || ok != 0 { + t.Fatal(fmt.Sprintf("res = %d, err = %s", ok, err)) + } + + // err - expire with the time before + if ok, err := db.ExpireAt(k, now-5); err == nil || ok != 0 { + t.Fatal(fmt.Sprintf("res = %d, err = %s", ok, err)) + } +} + +func TestKvTtl(t *testing.T) { + db := getTestDB() + m.Lock() + defer m.Unlock() + + k := []byte("ttl_a") + ek := []byte("ttl_b") + + db.Set(k, []byte("1")) + db.Expire(k, 2) + + if tRemain, _ := db.Ttl(k); tRemain != 2 { + t.Fatal(tRemain) + } + + // err - check ttl on an inexisting key + if tRemain, _ := db.Ttl(ek); tRemain != -1 { + t.Fatal(tRemain) + } + + db.Del(k) + if tRemain, _ := db.Ttl(k); tRemain != -1 { + t.Fatal(tRemain) + } +} + +func TestKvExpCompose(t *testing.T) { + db := getTestDB() + m.Lock() + defer m.Unlock() + + k0 := []byte("ttl_a") + k1 := []byte("ttl_b") + k2 := []byte("ttl_c") + + db.Set(k0, k0) + db.Set(k1, k1) + db.Set(k2, k2) + + db.Expire(k0, 5) + db.Expire(k1, 2) + db.Expire(k2, 60) + + if tRemain, _ := db.Ttl(k0); tRemain != 5 { + t.Fatal(tRemain) + } + if tRemain, _ := db.Ttl(k1); tRemain != 2 { + t.Fatal(tRemain) + } + if tRemain, _ := db.Ttl(k2); tRemain != 60 { + t.Fatal(tRemain) + } + + // after 1 sec + time.Sleep(1 * time.Second) + if tRemain, _ := db.Ttl(k0); tRemain != 4 { + t.Fatal(tRemain) + } + if tRemain, _ := db.Ttl(k1); tRemain != 1 { + t.Fatal(tRemain) + } + + // after 2 sec + time.Sleep(2 * time.Second) + if tRemain, _ := db.Ttl(k1); tRemain != -1 { + t.Fatal(tRemain) + } + if v, _ := db.Get(k1); v != nil { + t.Fatal(v) + } + + if tRemain, _ := db.Ttl(k0); tRemain != 2 { + t.Fatal(tRemain) + } + if v, _ := db.Get(k0); v == nil { + t.Fatal(v) + } + + // refresh the expiration of key + if tRemain, _ := db.Ttl(k2); !(0 < tRemain && tRemain < 60) { + t.Fatal(tRemain) + } + + if ok, _ := db.Expire(k2, 100); ok != 1 { + t.Fatal(false) + } + + if tRemain, _ := db.Ttl(k2); tRemain != 100 { + t.Fatal(tRemain) + } + + // expire an inexisting key + if ok, _ := db.Expire(k1, 10); ok == 1 { + t.Fatal(false) + } + if tRemain, _ := db.Ttl(k1); tRemain != -1 { + t.Fatal(tRemain) + } + + return +} From 0d4949f060222e310c4fc7b448a15ee1ec8f6ca5 Mon Sep 17 00:00:00 2001 From: siddontang Date: Tue, 3 Jun 2014 16:19:39 +0800 Subject: [PATCH 09/31] change log name suffix format --- replication/binlog.go | 1 + replication/log.go | 2 +- replication/relaylog.go | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/replication/binlog.go b/replication/binlog.go index 677e087..1500b27 100644 --- a/replication/binlog.go +++ b/replication/binlog.go @@ -51,6 +51,7 @@ func (cfg *BinLogConfig) adjust() { cfg.IndexName = "ledis" } + //binlog not care space limit cfg.SpaceLimit = -1 cfg.LogType = "bin" diff --git a/replication/log.go b/replication/log.go index e61e59d..175099d 100644 --- a/replication/log.go +++ b/replication/log.go @@ -145,7 +145,7 @@ func (l *Log) loadIndex() error { } func (l *Log) getLogFile() string { - return fmt.Sprintf("%s-%s.%05d", l.cfg.BaseName, l.cfg.LogType, l.lastLogIndex) + return fmt.Sprintf("%s-%s.%07d", l.cfg.BaseName, l.cfg.LogType, l.lastLogIndex) } func (l *Log) openNewLogFile() error { diff --git a/replication/relaylog.go b/replication/relaylog.go index 9f2b7a4..6b4894d 100644 --- a/replication/relaylog.go +++ b/replication/relaylog.go @@ -28,6 +28,7 @@ func (cfg *RelayLogConfig) adjust() { cfg.IndexName = "ledis" } + //relaylog not care file num cfg.MaxFileNum = -1 cfg.LogType = "relay" } From a7eb1e18847148ec5b6a3e9ae6ee6ee0efdc3853 Mon Sep 17 00:00:00 2001 From: silentsai Date: Wed, 4 Jun 2014 14:29:21 +0800 Subject: [PATCH 10/31] for test cases : do remove db file instead of call function db.clear at the very start --- ledis/ledis_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ledis/ledis_test.go b/ledis/ledis_test.go index e0f812d..902d545 100644 --- a/ledis/ledis_test.go +++ b/ledis/ledis_test.go @@ -1,6 +1,7 @@ package ledis import ( + "os" "sync" "testing" ) @@ -27,14 +28,16 @@ func getTestDB() *DB { } } `) + + os.RemoveAll("/tmp/testdb") + os.RemoveAll("/tmp/testdb_binlog") + var err error testLedis, err = Open(d) if err != nil { println(err.Error()) panic(err) } - - testLedis.ldb.Clear() } testLedisOnce.Do(f) From 7a0c5bcd1744b8032c301ac0055f6bdc14be3ff7 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 4 Jun 2014 14:42:02 +0800 Subject: [PATCH 11/31] add replicate for relay log --- ledis/binlog_util.go | 46 ++++++++++--- ledis/dump.go | 4 +- ledis/replication.go | 132 ++++++++++++++++++++++++++++++++++++++ ledis/replication_test.go | 78 ++++++++++++++++++++++ replication/binlog.go | 2 +- 5 files changed, 250 insertions(+), 12 deletions(-) create mode 100644 ledis/replication.go create mode 100644 ledis/replication_test.go diff --git a/ledis/binlog_util.go b/ledis/binlog_util.go index b0abd4e..bc1cd63 100644 --- a/ledis/binlog_util.go +++ b/ledis/binlog_util.go @@ -2,33 +2,61 @@ package ledis import ( "encoding/binary" + "errors" +) + +var ( + errBinLogDeleteType = errors.New("invalid bin log delete type") + errBinLogPutType = errors.New("invalid bin log put type") + errBinLogCommandType = errors.New("invalid bin log command type") ) func encodeBinLogDelete(key []byte) []byte { - buf := make([]byte, 3+len(key)) + buf := make([]byte, 1+len(key)) buf[0] = BinLogTypeDeletion - pos := 1 - binary.BigEndian.PutUint16(buf[pos:], uint16(len(key))) - pos += 2 - copy(buf[pos:], key) + copy(buf[1:], key) return buf } +func decodeBinLogDelete(sz []byte) ([]byte, error) { + if len(sz) < 1 || sz[0] != BinLogTypeDeletion { + return nil, errBinLogDeleteType + } + + return sz[1:], nil +} + func encodeBinLogPut(key []byte, value []byte) []byte { - buf := make([]byte, 7+len(key)+len(value)) + buf := make([]byte, 3+len(key)+len(value)) buf[0] = BinLogTypePut pos := 1 binary.BigEndian.PutUint16(buf[pos:], uint16(len(key))) pos += 2 copy(buf[pos:], key) pos += len(key) - binary.BigEndian.PutUint32(buf[pos:], uint32(len(value))) - pos += 4 copy(buf[pos:], value) + return buf } -func encodeBinLogCommand(commandType uint8, args []byte) []byte { +func decodeBinLogPut(sz []byte) ([]byte, []byte, error) { + if len(sz) < 3 || sz[0] != BinLogTypePut { + return nil, nil, errBinLogPutType + } + + keyLen := int(binary.BigEndian.Uint16(sz[1:])) + if 3+keyLen > len(sz) { + return nil, nil, errBinLogPutType + } + + return sz[3 : 3+keyLen], sz[3+keyLen:], nil +} + +func encodeBinLogCommand(commandType uint8, args ...[]byte) []byte { //to do return nil } + +func decodeBinLogCommand(sz []byte) (uint8, [][]byte, error) { + return 0, nil, errBinLogCommandType +} diff --git a/ledis/dump.go b/ledis/dump.go index e7ff8a0..2cb08af 100644 --- a/ledis/dump.go +++ b/ledis/dump.go @@ -15,8 +15,8 @@ import ( // |keylen(bigendian int32)|key|valuelen(bigendian int32)|value...... type DumpHead struct { - LogFile string `json:"bin_log_file"` - LogPos int64 `json:"bin_log_pos"` + LogFile string `json:"log_file"` + LogPos int64 `json:"log_pos"` } func (l *Ledis) DumpFile(path string) error { diff --git a/ledis/replication.go b/ledis/replication.go new file mode 100644 index 0000000..8de0bc9 --- /dev/null +++ b/ledis/replication.go @@ -0,0 +1,132 @@ +package ledis + +import ( + "bufio" + "bytes" + "encoding/binary" + "errors" + "github.com/siddontang/go-log/log" + "io" + "os" +) + +var ( + errInvalidBinLogEvent = errors.New("invalid binglog event") +) + +func (l *Ledis) replicateEvent(event []byte) error { + if len(event) == 0 { + return errInvalidBinLogEvent + } + + logType := uint8(event[0]) + switch logType { + case BinLogTypePut: + return l.replicatePutEvent(event) + case BinLogTypeDeletion: + return l.replicateDeleteEvent(event) + case BinLogTypeCommand: + return l.replicateCommandEvent(event) + default: + return errInvalidBinLogEvent + } +} + +func (l *Ledis) replicatePutEvent(event []byte) error { + key, value, err := decodeBinLogPut(event) + if err != nil { + return err + } + + if err = l.ldb.Put(key, value); err != nil { + return err + } + + if l.binlog != nil { + err = l.binlog.Log(event) + } + + return err +} + +func (l *Ledis) replicateDeleteEvent(event []byte) error { + key, err := decodeBinLogDelete(event) + if err != nil { + return err + } + + if err = l.ldb.Delete(key); err != nil { + return err + } + + if l.binlog != nil { + err = l.binlog.Log(event) + } + + return err +} + +func (l *Ledis) replicateCommandEvent(event []byte) error { + return errors.New("command event not supported now") +} + +func (l *Ledis) RepliateRelayLog(relayLog string, offset int64) (int64, error) { + f, err := os.Open(relayLog) + if err != nil { + return 0, err + } + + defer f.Close() + + st, _ := f.Stat() + totalSize := st.Size() + + if _, err = f.Seek(offset, os.SEEK_SET); err != nil { + return 0, err + } + + rb := bufio.NewReaderSize(f, 4096) + + var createTime uint32 + var dataLen uint32 + var dataBuf bytes.Buffer + + for { + if offset+8 > totalSize { + //event may not sync completely + return f.Seek(offset, os.SEEK_SET) + } + + if err = binary.Read(rb, binary.BigEndian, &createTime); err != nil { + return 0, err + } + + if err = binary.Read(rb, binary.BigEndian, &dataLen); err != nil { + return 0, err + } + + if offset+8+int64(dataLen) > totalSize { + //event may not sync completely + return f.Seek(offset, os.SEEK_SET) + } else { + if _, err = io.CopyN(&dataBuf, rb, int64(dataLen)); err != nil { + return 0, err + } + + l.Lock() + err = l.replicateEvent(dataBuf.Bytes()) + l.Unlock() + if err != nil { + log.Fatal("replication error %s, skip to next", err.Error()) + } + + dataBuf.Reset() + + offset += (8 + int64(dataLen)) + } + } + + //can not go here??? + log.Error("can not go here") + return offset, nil +} diff --git a/ledis/replication_test.go b/ledis/replication_test.go new file mode 100644 index 0000000..5f2243a --- /dev/null +++ b/ledis/replication_test.go @@ -0,0 +1,78 @@ +package ledis + +import ( + "bytes" + "github.com/siddontang/go-leveldb/leveldb" + "os" + "testing" +) + +func TestReplication(t *testing.T) { + var master *Ledis + var slave *Ledis + var err error + + os.RemoveAll("/tmp/repl") + os.MkdirAll("/tmp/repl", os.ModePerm) + + master, err = Open([]byte(` + { + "data_db" : { + "path" : "/tmp/repl/master_db" + }, + + "binlog" : { + "path" : "/tmp/repl/master_binlog" + } + } + `)) + if err != nil { + t.Fatal(err) + } + + slave, err = Open([]byte(` + { + "data_db" : { + "path" : "/tmp/repl/slave_db" + }, + + "binlog" : { + "path" : "/tmp/repl/slave_binlog" + } + } + `)) + if err != nil { + t.Fatal(err) + } + + db, _ := master.Select(0) + db.Set([]byte("a"), []byte("1")) + db.Set([]byte("b"), []byte("2")) + db.Set([]byte("c"), []byte("3")) + + relayLog := "/tmp/repl/master_binlog/ledis-bin.0000001" + + var offset int64 + offset, err = slave.RepliateRelayLog(relayLog, 0) + if err != nil { + t.Fatal(err) + } else { + if st, err := os.Stat(relayLog); err != nil { + t.Fatal(err) + } else if st.Size() != offset { + t.Fatal(st.Size(), offset) + } + } + + it := master.ldb.Iterator(nil, nil, leveldb.RangeClose, 0, -1) + for ; it.Valid(); it.Next() { + key := it.Key() + value := it.Value() + + if v, err := slave.ldb.Get(key); err != nil { + t.Fatal(err) + } else if !bytes.Equal(v, value) { + t.Fatal("replication error", len(v), len(value)) + } + } +} diff --git a/replication/binlog.go b/replication/binlog.go index 1500b27..349880c 100644 --- a/replication/binlog.go +++ b/replication/binlog.go @@ -23,7 +23,7 @@ ledis-bin.00003 log file format -timestamp(bigendian uint32, seconds)|PayloadLen(bigendian uint32)|PayloadData|LogId +timestamp(bigendian uint32, seconds)|PayloadLen(bigendian uint32)|PayloadData */ From 4a5f1054492526ebaa60eb08e963107af4ef3810 Mon Sep 17 00:00:00 2001 From: silentsai Date: Wed, 4 Jun 2014 18:32:23 +0800 Subject: [PATCH 12/31] change the type of return score --- ledis/t_zset.go | 45 +++++++++++++++++++++++++------------------- ledis/t_zset_test.go | 27 +++++++++++++++++++++++++- server/cmd_zset.go | 4 ++-- 3 files changed, 54 insertions(+), 22 deletions(-) diff --git a/ledis/t_zset.go b/ledis/t_zset.go index 222b4ae..372368e 100644 --- a/ledis/t_zset.go +++ b/ledis/t_zset.go @@ -8,8 +8,9 @@ import ( ) const ( - MinScore int64 = -1<<63 + 1 - MaxScore int64 = 1<<63 - 1 + MinScore int64 = -1<<63 + 1 + MaxScore int64 = 1<<63 - 1 + InvalidScore int64 = -1 << 63 ) type ScorePair struct { @@ -21,6 +22,7 @@ var errZSizeKey = errors.New("invalid zsize key") var errZSetKey = errors.New("invalid zset key") var errZScoreKey = errors.New("invalid zscore key") var errScoreOverflow = errors.New("zset score overflow") +var errScoreMiss = errors.New("zset score miss") const ( zsetNScoreSep byte = '<' @@ -311,19 +313,25 @@ func (db *DB) ZCard(key []byte) (int64, error) { return Int64(db.db.Get(sk)) } -func (db *DB) ZScore(key []byte, member []byte) ([]byte, error) { +func (db *DB) ZScore(key []byte, member []byte) (int64, error) { if err := checkZSetKMSize(key, member); err != nil { - return nil, err + return InvalidScore, err } + var score int64 = InvalidScore + k := db.zEncodeSetKey(key, member) - - score, err := Int64(db.db.Get(k)) - if err != nil { - return nil, err + if v, err := db.db.Get(k); err != nil { + return InvalidScore, err + } else if v == nil { + return InvalidScore, errScoreMiss + } else { + if score, err = Int64(v, nil); err != nil { + return InvalidScore, err + } } - return StrPutInt64(score), nil + return score, nil } func (db *DB) ZRem(key []byte, members ...[]byte) (int64, error) { @@ -356,9 +364,9 @@ func (db *DB) ZRem(key []byte, members ...[]byte) (int64, error) { return num, err } -func (db *DB) ZIncrBy(key []byte, delta int64, member []byte) ([]byte, error) { +func (db *DB) ZIncrBy(key []byte, delta int64, member []byte) (int64, error) { if err := checkZSetKMSize(key, member); err != nil { - return nil, err + return InvalidScore, err } t := db.zsetTx @@ -371,18 +379,17 @@ func (db *DB) ZIncrBy(key []byte, delta int64, member []byte) ([]byte, error) { v, err := db.db.Get(ek) if err != nil { - return nil, err + return InvalidScore, err } else if v != nil { if s, err := Int64(v, err); err != nil { - return nil, err + return InvalidScore, err } else { sk := db.zEncodeScoreKey(key, member, s) t.Delete(sk) score = s + delta - if score >= MaxScore || score <= MinScore { - return nil, errScoreOverflow + return InvalidScore, errScoreOverflow } } } else { @@ -395,7 +402,7 @@ func (db *DB) ZIncrBy(key []byte, delta int64, member []byte) ([]byte, error) { t.Put(sk, []byte{}) err = t.Commit() - return StrPutInt64(score), err + return score, err } func (db *DB) ZCount(key []byte, min int64, max int64) (int64, error) { @@ -563,10 +570,10 @@ func (db *DB) zRange(key []byte, min int64, max int64, withScores bool, offset i continue } - v = append(v, m) - if withScores { - v = append(v, StrPutInt64(s)) + v = append(v, m, s) + } else { + v = append(v, m) } } diff --git a/ledis/t_zset_test.go b/ledis/t_zset_test.go index fa2bb45..92c0a3d 100644 --- a/ledis/t_zset_test.go +++ b/ledis/t_zset_test.go @@ -71,6 +71,16 @@ func TestDBZSet(t *testing.T) { t.Fatal(n) } + if s, err := db.ZScore(key, bin("d")); err != nil { + t.Fatal(err) + } else if s != 3 { + t.Fatal(s) + } + + if s, err := db.ZScore(key, bin("zzz")); err != errScoreMiss || s != InvalidScore { + t.Fatal(fmt.Sprintf("s=[%d] err=[%s]", s, err)) + } + // {c':2, 'd':3} if n, err := db.ZRem(key, bin("a"), bin("b")); err != nil { t.Fatal(err) @@ -179,8 +189,10 @@ func TestZSetOrder(t *testing.T) { } // {'a':0, 'b':1, 'c':2, 'd':999, 'e':6, 'f':5} - if _, err := db.ZIncrBy(key, 2, bin("e")); err != nil { + if s, err := db.ZIncrBy(key, 2, bin("e")); err != nil { t.Fatal(err) + } else if s != 6 { + t.Fatal(s) } if pos, _ := db.ZRank(key, bin("e")); int(pos) != 4 { @@ -190,6 +202,19 @@ func TestZSetOrder(t *testing.T) { if pos, _ := db.ZRevRank(key, bin("e")); int(pos) != 1 { t.Fatal(pos) } + + if datas, _ := db.ZRange(key, 0, endPos, true); len(datas) != 12 { + t.Fatal(len(datas)) + } else { + scores := []int64{0, 1, 2, 5, 6, 999} + for i := 1; i < len(datas); i += 2 { + if s, ok := datas[i].(int64); !ok || s != scores[(i-1)/2] { + t.Fatal(fmt.Sprintf("[%d]=%d", i, datas[i])) + } + } + } + + return } func TestDBZScan(t *testing.T) { diff --git a/server/cmd_zset.go b/server/cmd_zset.go index c7f26b5..c68f71b 100644 --- a/server/cmd_zset.go +++ b/server/cmd_zset.go @@ -66,10 +66,10 @@ func zscoreCommand(c *client) error { return ErrCmdParams } - if v, err := c.db.ZScore(args[0], args[1]); err != nil { + if s, err := c.db.ZScore(args[0], args[1]); err != nil { return err } else { - c.writeBulk(v) + c.writeInteger(s) } return nil From dee78beef18bc2cfdbf7e8cab51c472687ca1533 Mon Sep 17 00:00:00 2001 From: silentsai Date: Thu, 5 Jun 2014 10:20:50 +0800 Subject: [PATCH 13/31] fix the type of return value from zscore command --- ledis/const.go | 1 + ledis/t_zset.go | 3 +-- server/cmd_zset.go | 8 ++++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ledis/const.go b/ledis/const.go index 9e4d0cf..ba0cf05 100644 --- a/ledis/const.go +++ b/ledis/const.go @@ -37,4 +37,5 @@ var ( ErrKeySize = errors.New("invalid key size") ErrHashFieldSize = errors.New("invalid hash field size") ErrZSetMemberSize = errors.New("invalid zset member size") + ErrScoreMiss = errors.New("zset score miss") ) diff --git a/ledis/t_zset.go b/ledis/t_zset.go index 372368e..8be54b7 100644 --- a/ledis/t_zset.go +++ b/ledis/t_zset.go @@ -22,7 +22,6 @@ var errZSizeKey = errors.New("invalid zsize key") var errZSetKey = errors.New("invalid zset key") var errZScoreKey = errors.New("invalid zscore key") var errScoreOverflow = errors.New("zset score overflow") -var errScoreMiss = errors.New("zset score miss") const ( zsetNScoreSep byte = '<' @@ -324,7 +323,7 @@ func (db *DB) ZScore(key []byte, member []byte) (int64, error) { if v, err := db.db.Get(k); err != nil { return InvalidScore, err } else if v == nil { - return InvalidScore, errScoreMiss + return InvalidScore, ErrScoreMiss } else { if score, err = Int64(v, nil); err != nil { return InvalidScore, err diff --git a/server/cmd_zset.go b/server/cmd_zset.go index c68f71b..d9cb78a 100644 --- a/server/cmd_zset.go +++ b/server/cmd_zset.go @@ -67,9 +67,13 @@ func zscoreCommand(c *client) error { } if s, err := c.db.ZScore(args[0], args[1]); err != nil { - return err + if err == ledis.ErrScoreMiss { + c.writeBulk(nil) + } else { + return err + } } else { - c.writeInteger(s) + c.writeBulk(ledis.StrPutInt64(s)) } return nil From d0e15d4c1de4dfa206bc0bacbdd233e7eee032fb Mon Sep 17 00:00:00 2001 From: silentsai Date: Thu, 5 Jun 2014 11:49:12 +0800 Subject: [PATCH 14/31] fix bug - unify the return score type to bluk while call rangeXXX --- ledis/t_zset_test.go | 2 +- server/cmd_zset.go | 22 +++++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/ledis/t_zset_test.go b/ledis/t_zset_test.go index 92c0a3d..202da70 100644 --- a/ledis/t_zset_test.go +++ b/ledis/t_zset_test.go @@ -77,7 +77,7 @@ func TestDBZSet(t *testing.T) { t.Fatal(s) } - if s, err := db.ZScore(key, bin("zzz")); err != errScoreMiss || s != InvalidScore { + if s, err := db.ZScore(key, bin("zzz")); err != ErrScoreMiss || s != InvalidScore { t.Fatal(fmt.Sprintf("s=[%d] err=[%s]", s, err)) } diff --git a/server/cmd_zset.go b/server/cmd_zset.go index d9cb78a..c72fa71 100644 --- a/server/cmd_zset.go +++ b/server/cmd_zset.go @@ -110,7 +110,7 @@ func zincrbyCommand(c *client) error { if v, err := c.db.ZIncrBy(key, delta, args[2]); err != nil { return err } else { - c.writeBulk(v) + c.writeBulk(ledis.StrPutInt64(v)) } return nil @@ -312,10 +312,16 @@ func zrangeGeneric(c *client, reverse bool) error { withScores = true } - if v, err := c.db.ZRangeGeneric(key, start, stop, withScores, reverse); err != nil { + if datas, err := c.db.ZRangeGeneric(key, start, stop, withScores, reverse); err != nil { return err } else { - c.writeArray(v) + if withScores { + for i := len(datas) - 1; i > 0; i -= 2 { + v, _ := datas[i].(int64) + datas[i] = ledis.StrPutInt64(v) + } + } + c.writeArray(datas) } return nil } @@ -377,10 +383,16 @@ func zrangebyscoreGeneric(c *client, reverse bool) error { return nil } - if v, err := c.db.ZRangeByScoreGeneric(key, min, max, withScores, offset, count, reverse); err != nil { + if datas, err := c.db.ZRangeByScoreGeneric(key, min, max, withScores, offset, count, reverse); err != nil { return err } else { - c.writeArray(v) + if withScores { + for i := len(datas) - 1; i > 0; i -= 2 { + v, _ := datas[i].(int64) + datas[i] = ledis.StrPutInt64(v) + } + } + c.writeArray(datas) } return nil From 51349ea12a11c1a32aa697695fdde70fe681f4e8 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 5 Jun 2014 15:46:38 +0800 Subject: [PATCH 15/31] don't export some error --- ledis/const.go | 15 +++++++++------ ledis/replication.go | 4 ++++ ledis/t_hash.go | 4 ++-- ledis/t_kv.go | 8 ++++---- ledis/t_zset.go | 8 ++++---- 5 files changed, 23 insertions(+), 16 deletions(-) diff --git a/ledis/const.go b/ledis/const.go index 82ce267..577464d 100644 --- a/ledis/const.go +++ b/ledis/const.go @@ -28,6 +28,14 @@ const ( defaultScanCount int = 10 ) +var ( + errKeySize = errors.New("invalid key size") + errValueSize = errors.New("invalid value size") + errHashFieldSize = errors.New("invalid hash field size") + errZSetMemberSize = errors.New("invalid zset member size") + errExpireValue = errors.New("invalid expire value") +) + const ( //we don't support too many databases MaxDBNumber uint8 = 16 @@ -46,12 +54,7 @@ const ( ) var ( - ErrKeySize = errors.New("invalid key size") - ErrValueSize = errors.New("invalid value size") - ErrHashFieldSize = errors.New("invalid hash field size") - ErrZSetMemberSize = errors.New("invalid zset member size") - ErrExpireValue = errors.New("invalid expire value") - ErrScoreMiss = errors.New("zset score miss") + ErrScoreMiss = errors.New("zset score miss") ) const ( diff --git a/ledis/replication.go b/ledis/replication.go index 8de0bc9..2559dba 100644 --- a/ledis/replication.go +++ b/ledis/replication.go @@ -130,3 +130,7 @@ func (l *Ledis) RepliateRelayLog(relayLog string, offset int64) (int64, error) { log.Error("can not go here") return offset, nil } + +func (l *Ledis) WriteRelayLog(data []byte) error { + return nil +} diff --git a/ledis/t_hash.go b/ledis/t_hash.go index 8a57fc2..68bdc49 100644 --- a/ledis/t_hash.go +++ b/ledis/t_hash.go @@ -21,9 +21,9 @@ const ( func checkHashKFSize(key []byte, field []byte) error { if len(key) > MaxKeySize || len(key) == 0 { - return ErrKeySize + return errKeySize } else if len(field) > MaxHashFieldSize || len(field) == 0 { - return ErrHashFieldSize + return errHashFieldSize } return nil } diff --git a/ledis/t_kv.go b/ledis/t_kv.go index e1e0dba..caa26dd 100644 --- a/ledis/t_kv.go +++ b/ledis/t_kv.go @@ -15,14 +15,14 @@ var errKVKey = errors.New("invalid encode kv key") func checkKeySize(key []byte) error { if len(key) > MaxKeySize || len(key) == 0 { - return ErrKeySize + return errKeySize } return nil } func checkValueSize(value []byte) error { if len(value) > MaxValueSize { - return ErrValueSize + return errValueSize } return nil @@ -360,7 +360,7 @@ func (db *DB) Scan(key []byte, count int, inclusive bool) ([]KVPair, error) { func (db *DB) Expire(key []byte, duration int64) (int64, error) { if duration <= 0 { - return 0, ErrExpireValue + return 0, errExpireValue } t := db.kvTx @@ -381,7 +381,7 @@ func (db *DB) Expire(key []byte, duration int64) (int64, error) { func (db *DB) ExpireAt(key []byte, when int64) (int64, error) { if when <= time.Now().Unix() { - return 0, ErrExpireValue + return 0, errExpireValue } t := db.kvTx diff --git a/ledis/t_zset.go b/ledis/t_zset.go index 2244806..37bbc61 100644 --- a/ledis/t_zset.go +++ b/ledis/t_zset.go @@ -34,9 +34,9 @@ const ( func checkZSetKMSize(key []byte, member []byte) error { if len(key) > MaxKeySize || len(key) == 0 { - return ErrKeySize + return errKeySize } else if len(member) > MaxZSetMemberSize || len(member) == 0 { - return ErrZSetMemberSize + return errZSetMemberSize } return nil } @@ -484,7 +484,7 @@ func (db *DB) zIterator(key []byte, min int64, max int64, offset int, limit int, func (db *DB) zRemRange(key []byte, min int64, max int64, offset int, limit int) (int64, error) { if len(key) > MaxKeySize { - return 0, ErrKeySize + return 0, errKeySize } t := db.zsetTx @@ -537,7 +537,7 @@ func (db *DB) zReverse(s []interface{}, withScores bool) []interface{} { func (db *DB) zRange(key []byte, min int64, max int64, withScores bool, offset int, limit int, reverse bool) ([]interface{}, error) { if len(key) > MaxKeySize { - return nil, ErrKeySize + return nil, errKeySize } if offset < 0 { From be0464a21c16cde542b0092f74158ebf4b6d9c0a Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 5 Jun 2014 15:59:10 +0800 Subject: [PATCH 16/31] add relay log, add quit channel --- ledis/ledis.go | 36 ++++++++++++++++++++++++++++++++++-- ledis/ledis_db.go | 11 +++++++++-- ledis/replication.go | 11 +++++++++-- 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/ledis/ledis.go b/ledis/ledis.go index e132ab4..040f362 100644 --- a/ledis/ledis.go +++ b/ledis/ledis.go @@ -11,10 +11,13 @@ import ( type Config struct { DataDB leveldb.Config `json:"data_db"` - BinLog replication.BinLogConfig `json:"binlog"` + BinLog replication.BinLogConfig `json:"binlog"` + RelayLog replication.RelayLogConfig `json:"relaylog"` } type DB struct { + l *Ledis + db *leveldb.DB index uint8 @@ -33,7 +36,10 @@ type Ledis struct { ldb *leveldb.DB dbs [MaxDBNumber]*DB - binlog *replication.Log + binlog *replication.Log + relaylog *replication.Log + + quit chan struct{} } func Open(configJson json.RawMessage) (*Ledis, error) { @@ -53,6 +59,9 @@ func OpenWithConfig(cfg *Config) (*Ledis, error) { } l := new(Ledis) + + l.quit = make(chan struct{}) + l.ldb = ldb if len(cfg.BinLog.Path) > 0 { @@ -64,6 +73,15 @@ func OpenWithConfig(cfg *Config) (*Ledis, error) { l.binlog = nil } + if len(cfg.RelayLog.Path) > 0 { + l.relaylog, err = replication.NewRelayLogWithConfig(&cfg.RelayLog) + if err != nil { + return nil, err + } + } else { + l.relaylog = nil + } + for i := uint8(0); i < MaxDBNumber; i++ { l.dbs[i] = newDB(l, i) } @@ -74,6 +92,8 @@ func OpenWithConfig(cfg *Config) (*Ledis, error) { func newDB(l *Ledis, index uint8) *DB { d := new(DB) + d.l = l + d.db = l.ldb d.index = index @@ -89,7 +109,19 @@ func newDB(l *Ledis, index uint8) *DB { } func (l *Ledis) Close() { + close(l.quit) + l.ldb.Close() + + if l.binlog != nil { + l.binlog.Close() + l.binlog = nil + } + + if l.relaylog != nil { + l.relaylog.Close() + l.relaylog = nil + } } func (l *Ledis) Select(index int) (*DB, error) { diff --git a/ledis/ledis_db.go b/ledis/ledis_db.go index a45f59b..cae79f9 100644 --- a/ledis/ledis_db.go +++ b/ledis/ledis_db.go @@ -28,9 +28,16 @@ func (db *DB) activeExpireCycle() { eliminator.regRetireContext(kvExpType, db.kvTx, db.delete) go func() { + tick := time.NewTicker(1 * time.Second) for { - eliminator.active() - time.Sleep(1 * time.Second) + select { + case <-tick.C: + eliminator.active() + case <-db.l.quit: + break + } } + + tick.Stop() }() } diff --git a/ledis/replication.go b/ledis/replication.go index 2559dba..111f96b 100644 --- a/ledis/replication.go +++ b/ledis/replication.go @@ -11,7 +11,8 @@ import ( ) var ( - errInvalidBinLogEvent = errors.New("invalid binglog event") + errInvalidBinLogEvent = errors.New("invalid binglog event") + errRelayLogNotSupported = errors.New("write relay log not supported") ) func (l *Ledis) replicateEvent(event []byte) error { @@ -132,5 +133,11 @@ func (l *Ledis) RepliateRelayLog(relayLog string, offset int64) (int64, error) { } func (l *Ledis) WriteRelayLog(data []byte) error { - return nil + if l.relaylog == nil { + return errRelayLogNotSupported + } + + err := l.relaylog.Log(data) + + return err } From 8d76a58e5e933051ea0809555cf899771e131ea6 Mon Sep 17 00:00:00 2001 From: silentsai Date: Thu, 5 Jun 2014 16:13:35 +0800 Subject: [PATCH 17/31] supply function of expire/ttl for the other data types - list, hash, zset fix tiny mistaks --- ledis/ledis_db.go | 3 + ledis/t_hash.go | 93 +++++++++++++++++++----- ledis/t_kv.go | 51 ++++++-------- ledis/t_list.go | 56 ++++++++++++++- ledis/t_ttl.go | 20 +++--- ledis/t_ttl_test.go | 28 ++++---- ledis/t_zset.go | 167 +++++++++++++++++++++++++++++++------------- 7 files changed, 295 insertions(+), 123 deletions(-) diff --git a/ledis/ledis_db.go b/ledis/ledis_db.go index cae79f9..2390464 100644 --- a/ledis/ledis_db.go +++ b/ledis/ledis_db.go @@ -26,6 +26,9 @@ func (db *DB) FlushAll() (drop int64, err error) { func (db *DB) activeExpireCycle() { eliminator := newEliminator(db) eliminator.regRetireContext(kvExpType, db.kvTx, db.delete) + eliminator.regRetireContext(lExpType, db.listTx, db.lDelete) + eliminator.regRetireContext(hExpType, db.hashTx, db.hDelete) + eliminator.regRetireContext(zExpType, db.zsetTx, db.zDelete) go func() { tick := time.NewTicker(1 * time.Second) diff --git a/ledis/t_hash.go b/ledis/t_hash.go index 68bdc49..ac5d5cf 100644 --- a/ledis/t_hash.go +++ b/ledis/t_hash.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "errors" "github.com/siddontang/go-leveldb/leveldb" + "time" ) type FVPair struct { @@ -105,10 +106,6 @@ func (db *DB) hEncodeStopKey(key []byte) []byte { return k } -func (db *DB) HLen(key []byte) (int64, error) { - return Int64(db.db.Get(db.hEncodeSizeKey(key))) -} - func (db *DB) hSetItem(key []byte, field []byte, value []byte) (int64, error) { t := db.hashTx @@ -127,6 +124,49 @@ func (db *DB) hSetItem(key []byte, field []byte, value []byte) (int64, error) { return n, nil } +// ps : here just focus on deleting the hash data, +// any other likes expire is ignore. +func (db *DB) hDelete(t *tx, key []byte) int64 { + sk := db.hEncodeSizeKey(key) + start := db.hEncodeStartKey(key) + stop := db.hEncodeStopKey(key) + + var num int64 = 0 + it := db.db.Iterator(start, stop, leveldb.RangeROpen, 0, -1) + for ; it.Valid(); it.Next() { + t.Delete(it.Key()) + num++ + } + it.Close() + + t.Delete(sk) + return num +} + +func (db *DB) hExpireAt(key []byte, when int64) (int64, error) { + t := db.hashTx + t.Lock() + defer t.Unlock() + + if hlen, err := db.HLen(key); err != nil || hlen == 0 { + return 0, err + } else { + db.expireAt(t, hExpType, key, when) + if err := t.Commit(); err != nil { + return 0, err + } + } + return 1, nil +} + +func (db *DB) HLen(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return 0, err + } + + return Int64(db.db.Get(db.hEncodeSizeKey(key))) +} + func (db *DB) HSet(key []byte, field []byte, value []byte) (int64, error) { if err := checkHashKFSize(key, field); err != nil { return 0, err @@ -265,6 +305,7 @@ func (db *DB) hIncrSize(key []byte, delta int64) (int64, error) { if size <= 0 { size = 0 t.Delete(sk) + db.rmExpire(t, hExpType, key) } else { t.Put(sk, PutInt64(size)) } @@ -378,24 +419,12 @@ func (db *DB) HClear(key []byte) (int64, error) { return 0, err } - sk := db.hEncodeSizeKey(key) - start := db.hEncodeStartKey(key) - stop := db.hEncodeStopKey(key) - t := db.hashTx t.Lock() defer t.Unlock() - var num int64 = 0 - it := db.db.Iterator(start, stop, leveldb.RangeROpen, 0, -1) - for ; it.Valid(); it.Next() { - t.Delete(it.Key()) - num++ - } - - it.Close() - - t.Delete(sk) + num := db.hDelete(t, key) + db.rmExpire(t, hExpType, key) err := t.Commit() return num, err @@ -418,7 +447,7 @@ func (db *DB) HFlush() (drop int64, err error) { for ; it.Valid(); it.Next() { t.Delete(it.Key()) drop++ - if drop%1000 == 0 { + if drop&1023 == 0 { if err = t.Commit(); err != nil { return } @@ -426,6 +455,8 @@ func (db *DB) HFlush() (drop int64, err error) { } it.Close() + db.expFlush(t, hExpType) + err = t.Commit() return } @@ -466,3 +497,27 @@ func (db *DB) HScan(key []byte, field []byte, count int, inclusive bool) ([]FVPa return v, nil } + +func (db *DB) HExpire(key []byte, duration int64) (int64, error) { + if duration <= 0 { + return 0, errExpireValue + } + + return db.hExpireAt(key, time.Now().Unix()+duration) +} + +func (db *DB) HExpireAt(key []byte, when int64) (int64, error) { + if when <= time.Now().Unix() { + return 0, errExpireValue + } + + return db.hExpireAt(key, when) +} + +func (db *DB) HTTL(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return -1, err + } + + return db.ttl(hExpType, key) +} diff --git a/ledis/t_kv.go b/ledis/t_kv.go index caa26dd..3208761 100644 --- a/ledis/t_kv.go +++ b/ledis/t_kv.go @@ -84,12 +84,30 @@ func (db *DB) incr(key []byte, delta int64) (int64, error) { return n, err } +// ps : here just focus on deleting the key-value data, +// any other likes expire is ignore. func (db *DB) delete(t *tx, key []byte) int64 { key = db.encodeKVKey(key) t.Delete(key) return 1 } +func (db *DB) setExpireAt(key []byte, when int64) (int64, error) { + t := db.kvTx + t.Lock() + defer t.Unlock() + + if exist, err := db.Exists(key); err != nil || exist == 0 { + return 0, err + } else { + db.expireAt(t, kvExpType, key, when) + if err := t.Commit(); err != nil { + return 0, err + } + } + return 1, nil +} + func (db *DB) Decr(key []byte) (int64, error) { return db.incr(key, -1) } @@ -115,7 +133,6 @@ func (db *DB) Del(keys ...[]byte) (int64, error) { for i, k := range keys { t.Delete(codedKeys[i]) db.rmExpire(t, kvExpType, k) - //todo binlog } err := t.Commit() @@ -363,20 +380,7 @@ func (db *DB) Expire(key []byte, duration int64) (int64, error) { return 0, errExpireValue } - t := db.kvTx - t.Lock() - defer t.Unlock() - - if exist, err := db.Exists(key); err != nil || exist == 0 { - return 0, err - } else { - db.expire(t, kvExpType, key, duration) - if err := t.Commit(); err != nil { - return 0, err - } else { - return 1, nil - } - } + return db.setExpireAt(key, time.Now().Unix()+duration) } func (db *DB) ExpireAt(key []byte, when int64) (int64, error) { @@ -384,23 +388,10 @@ func (db *DB) ExpireAt(key []byte, when int64) (int64, error) { return 0, errExpireValue } - t := db.kvTx - t.Lock() - defer t.Unlock() - - if exist, err := db.Exists(key); err != nil || exist == 0 { - return 0, err - } else { - db.expireAt(t, kvExpType, key, when) - if err := t.Commit(); err != nil { - return 0, err - } else { - return 1, nil - } - } + return db.setExpireAt(key, when) } -func (db *DB) Ttl(key []byte) (int64, error) { +func (db *DB) TTL(key []byte) (int64, error) { if err := checkKeySize(key); err != nil { return -1, err } diff --git a/ledis/t_list.go b/ledis/t_list.go index 5d1355b..1532dd8 100644 --- a/ledis/t_list.go +++ b/ledis/t_list.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "errors" "github.com/siddontang/go-leveldb/leveldb" + "time" ) const ( @@ -172,12 +173,17 @@ func (db *DB) lpop(key []byte, whereSeq int32) ([]byte, error) { } t.Delete(itemKey) - db.lSetMeta(metaKey, headSeq, tailSeq) + size := db.lSetMeta(metaKey, headSeq, tailSeq) + if size == 0 { + db.rmExpire(t, hExpType, key) + } err = t.Commit() return value, err } +// ps : here just focus on deleting the list data, +// any other likes expire is ignore. func (db *DB) lDelete(t *tx, key []byte) int64 { mk := db.lEncodeMetaKey(key) @@ -230,7 +236,7 @@ func (db *DB) lGetMeta(ek []byte) (headSeq int32, tailSeq int32, size int32, err return } -func (db *DB) lSetMeta(ek []byte, headSeq int32, tailSeq int32) { +func (db *DB) lSetMeta(ek []byte, headSeq int32, tailSeq int32) int32 { t := db.listTx var size int32 = tailSeq - headSeq + 1 @@ -243,10 +249,27 @@ func (db *DB) lSetMeta(ek []byte, headSeq int32, tailSeq int32) { binary.LittleEndian.PutUint32(buf[0:4], uint32(headSeq)) binary.LittleEndian.PutUint32(buf[4:8], uint32(tailSeq)) - //binary.LittleEndian.PutUint32(buf[8:], uint32(size)) t.Put(ek, buf) } + + return size +} + +func (db *DB) lExpireAt(key []byte, when int64) (int64, error) { + t := db.listTx + t.Lock() + defer t.Unlock() + + if llen, err := db.LLen(key); err != nil || llen == 0 { + return 0, err + } else { + db.expireAt(t, lExpType, key, when) + if err := t.Commit(); err != nil { + return 0, err + } + } + return 1, nil } func (db *DB) LIndex(key []byte, index int32) ([]byte, error) { @@ -366,6 +389,7 @@ func (db *DB) LClear(key []byte) (int64, error) { defer t.Unlock() num := db.lDelete(t, key) + db.rmExpire(t, lExpType, key) err := t.Commit() return num, err @@ -396,6 +420,32 @@ func (db *DB) LFlush() (drop int64, err error) { } it.Close() + db.expFlush(t, lExpType) + err = t.Commit() return } + +func (db *DB) LExpire(key []byte, duration int64) (int64, error) { + if duration <= 0 { + return 0, errExpireValue + } + + return db.lExpireAt(key, time.Now().Unix()+duration) +} + +func (db *DB) LExpireAt(key []byte, when int64) (int64, error) { + if when <= time.Now().Unix() { + return 0, errExpireValue + } + + return db.lExpireAt(key, when) +} + +func (db *DB) LTTL(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return -1, err + } + + return db.ttl(lExpType, key) +} diff --git a/ledis/t_ttl.go b/ledis/t_ttl.go index 167acce..b353ce7 100644 --- a/ledis/t_ttl.go +++ b/ledis/t_ttl.go @@ -15,7 +15,7 @@ var mapExpMetaType = map[byte]byte{ type retireCallback func(*tx, []byte) int64 -type Elimination struct { +type elimination struct { db *DB exp2Tx map[byte]*tx exp2Retire map[byte]retireCallback @@ -92,10 +92,12 @@ func (db *DB) ttl(expType byte, key []byte) (t int64, err error) { func (db *DB) rmExpire(t *tx, expType byte, key []byte) { mk := db.expEncodeMetaKey(expType+1, key) - when, err := Int64(db.db.Get(mk)) - if err == nil && when > 0 { + if v, err := db.db.Get(mk); err != nil || v == nil { + return + } else if when, err2 := Int64(v, nil); err2 != nil { + return + } else { tk := db.expEncodeTimeKey(expType, key, when) - t.Delete(mk) t.Delete(tk) } @@ -127,6 +129,7 @@ func (db *DB) expFlush(t *tx, expType byte) (err error) { } } } + it.Close() err = t.Commit() return @@ -136,21 +139,21 @@ func (db *DB) expFlush(t *tx, expType byte) (err error) { // ////////////////////////////////////////////////////////// -func newEliminator(db *DB) *Elimination { - eli := new(Elimination) +func newEliminator(db *DB) *elimination { + eli := new(elimination) eli.db = db eli.exp2Tx = make(map[byte]*tx) eli.exp2Retire = make(map[byte]retireCallback) return eli } -func (eli *Elimination) regRetireContext(expType byte, t *tx, onRetire retireCallback) { +func (eli *elimination) regRetireContext(expType byte, t *tx, onRetire retireCallback) { eli.exp2Tx[expType] = t eli.exp2Retire[expType] = onRetire } // call by outside ... (from *db to another *db) -func (eli *Elimination) active() { +func (eli *elimination) active() { now := time.Now().Unix() db := eli.db dbGet := db.db.Get @@ -202,6 +205,7 @@ func (eli *Elimination) active() { t.Commit() t.Unlock() } // end : it + it.Close() } // end : expType return diff --git a/ledis/t_ttl_test.go b/ledis/t_ttl_test.go index 0483ae7..0f2cdac 100644 --- a/ledis/t_ttl_test.go +++ b/ledis/t_ttl_test.go @@ -69,7 +69,7 @@ func TestKvExpireAt(t *testing.T) { } } -func TestKvTtl(t *testing.T) { +func TestKvTTL(t *testing.T) { db := getTestDB() m.Lock() defer m.Unlock() @@ -80,17 +80,17 @@ func TestKvTtl(t *testing.T) { db.Set(k, []byte("1")) db.Expire(k, 2) - if tRemain, _ := db.Ttl(k); tRemain != 2 { + if tRemain, _ := db.TTL(k); tRemain != 2 { t.Fatal(tRemain) } // err - check ttl on an inexisting key - if tRemain, _ := db.Ttl(ek); tRemain != -1 { + if tRemain, _ := db.TTL(ek); tRemain != -1 { t.Fatal(tRemain) } db.Del(k) - if tRemain, _ := db.Ttl(k); tRemain != -1 { + if tRemain, _ := db.TTL(k); tRemain != -1 { t.Fatal(tRemain) } } @@ -112,35 +112,35 @@ func TestKvExpCompose(t *testing.T) { db.Expire(k1, 2) db.Expire(k2, 60) - if tRemain, _ := db.Ttl(k0); tRemain != 5 { + if tRemain, _ := db.TTL(k0); tRemain != 5 { t.Fatal(tRemain) } - if tRemain, _ := db.Ttl(k1); tRemain != 2 { + if tRemain, _ := db.TTL(k1); tRemain != 2 { t.Fatal(tRemain) } - if tRemain, _ := db.Ttl(k2); tRemain != 60 { + if tRemain, _ := db.TTL(k2); tRemain != 60 { t.Fatal(tRemain) } // after 1 sec time.Sleep(1 * time.Second) - if tRemain, _ := db.Ttl(k0); tRemain != 4 { + if tRemain, _ := db.TTL(k0); tRemain != 4 { t.Fatal(tRemain) } - if tRemain, _ := db.Ttl(k1); tRemain != 1 { + if tRemain, _ := db.TTL(k1); tRemain != 1 { t.Fatal(tRemain) } // after 2 sec time.Sleep(2 * time.Second) - if tRemain, _ := db.Ttl(k1); tRemain != -1 { + if tRemain, _ := db.TTL(k1); tRemain != -1 { t.Fatal(tRemain) } if v, _ := db.Get(k1); v != nil { t.Fatal(v) } - if tRemain, _ := db.Ttl(k0); tRemain != 2 { + if tRemain, _ := db.TTL(k0); tRemain != 2 { t.Fatal(tRemain) } if v, _ := db.Get(k0); v == nil { @@ -148,7 +148,7 @@ func TestKvExpCompose(t *testing.T) { } // refresh the expiration of key - if tRemain, _ := db.Ttl(k2); !(0 < tRemain && tRemain < 60) { + if tRemain, _ := db.TTL(k2); !(0 < tRemain && tRemain < 60) { t.Fatal(tRemain) } @@ -156,7 +156,7 @@ func TestKvExpCompose(t *testing.T) { t.Fatal(false) } - if tRemain, _ := db.Ttl(k2); tRemain != 100 { + if tRemain, _ := db.TTL(k2); tRemain != 100 { t.Fatal(tRemain) } @@ -164,7 +164,7 @@ func TestKvExpCompose(t *testing.T) { if ok, _ := db.Expire(k1, 10); ok == 1 { t.Fatal(false) } - if tRemain, _ := db.Ttl(k1); tRemain != -1 { + if tRemain, _ := db.TTL(k1); tRemain != -1 { t.Fatal(tRemain) } diff --git a/ledis/t_zset.go b/ledis/t_zset.go index 37bbc61..f10ce3f 100644 --- a/ledis/t_zset.go +++ b/ledis/t_zset.go @@ -5,6 +5,7 @@ import ( "encoding/binary" "errors" "github.com/siddontang/go-leveldb/leveldb" + "time" ) const ( @@ -191,13 +192,11 @@ func (db *DB) zDecodeScoreKey(ek []byte) (key []byte, member []byte, score int64 return } -func (db *DB) zSetItem(key []byte, score int64, member []byte) (int64, error) { +func (db *DB) zSetItem(t *tx, key []byte, score int64, member []byte) (int64, error) { if score <= MinScore || score >= MaxScore { return 0, errScoreOverflow } - t := db.zsetTx - var exists int64 = 0 ek := db.zEncodeSetKey(key, member) @@ -222,9 +221,7 @@ func (db *DB) zSetItem(key []byte, score int64, member []byte) (int64, error) { return exists, nil } -func (db *DB) zDelItem(key []byte, member []byte, skipDelScore bool) (int64, error) { - t := db.zsetTx - +func (db *DB) zDelItem(t *tx, key []byte, member []byte, skipDelScore bool) (int64, error) { ek := db.zEncodeSetKey(key, member) if v, err := db.db.Get(ek); err != nil { return 0, err @@ -245,6 +242,29 @@ func (db *DB) zDelItem(key []byte, member []byte, skipDelScore bool) (int64, err } t.Delete(ek) + + return 1, nil +} + +func (db *DB) zDelete(t *tx, key []byte) int64 { + delMembCnt, _ := db.zRemRange(t, key, MinScore, MaxScore, 0, -1) + // todo : log err + return delMembCnt +} + +func (db *DB) zExpireAt(key []byte, when int64) (int64, error) { + t := db.zsetTx + t.Lock() + defer t.Unlock() + + if zcnt, err := db.ZCard(key); err != nil || zcnt == 0 { + return 0, err + } else { + db.expireAt(t, zExpType, key, when) + if err := t.Commit(); err != nil { + return 0, err + } + } return 1, nil } @@ -266,7 +286,7 @@ func (db *DB) ZAdd(key []byte, args ...ScorePair) (int64, error) { return 0, err } - if n, err := db.zSetItem(key, score, member); err != nil { + if n, err := db.zSetItem(t, key, score, member); err != nil { return 0, err } else if n == 0 { //add new @@ -274,7 +294,7 @@ func (db *DB) ZAdd(key []byte, args ...ScorePair) (int64, error) { } } - if _, err := db.zIncrSize(key, num); err != nil { + if _, err := db.zIncrSize(t, key, num); err != nil { return 0, err } @@ -283,8 +303,7 @@ func (db *DB) ZAdd(key []byte, args ...ScorePair) (int64, error) { return num, err } -func (db *DB) zIncrSize(key []byte, delta int64) (int64, error) { - t := db.zsetTx +func (db *DB) zIncrSize(t *tx, key []byte, delta int64) (int64, error) { sk := db.zEncodeSizeKey(key) size, err := Int64(db.db.Get(sk)) @@ -295,6 +314,7 @@ func (db *DB) zIncrSize(key []byte, delta int64) (int64, error) { if size <= 0 { size = 0 t.Delete(sk) + db.rmExpire(t, zExpType, key) } else { t.Put(sk, PutInt64(size)) } @@ -348,14 +368,14 @@ func (db *DB) ZRem(key []byte, members ...[]byte) (int64, error) { return 0, err } - if n, err := db.zDelItem(key, members[i], false); err != nil { + if n, err := db.zDelItem(t, key, members[i], false); err != nil { return 0, err } else if n == 1 { num++ } } - if _, err := db.zIncrSize(key, -num); err != nil { + if _, err := db.zIncrSize(t, key, -num); err != nil { return 0, err } @@ -374,34 +394,35 @@ func (db *DB) ZIncrBy(key []byte, delta int64, member []byte) (int64, error) { ek := db.zEncodeSetKey(key, member) - var score int64 = delta - + var oldScore int64 = 0 v, err := db.db.Get(ek) if err != nil { return InvalidScore, err - } else if v != nil { - if s, err := Int64(v, err); err != nil { - return InvalidScore, err - } else { - sk := db.zEncodeScoreKey(key, member, s) - t.Delete(sk) - - score = s + delta - if score >= MaxScore || score <= MinScore { - return InvalidScore, errScoreOverflow - } - } + } else if v == nil { + db.zIncrSize(t, key, 1) } else { - db.zIncrSize(key, 1) + if oldScore, err = Int64(v, err); err != nil { + return InvalidScore, err + } } - t.Put(ek, PutInt64(score)) + newScore := oldScore + delta + if newScore >= MaxScore || newScore <= MinScore { + return InvalidScore, errScoreOverflow + } - sk := db.zEncodeScoreKey(key, member, score) + sk := db.zEncodeScoreKey(key, member, newScore) t.Put(sk, []byte{}) + t.Put(ek, PutInt64(newScore)) + + if v != nil { + // so as to update score, we must delete the old one + oldSk := db.zEncodeScoreKey(key, member, oldScore) + t.Delete(oldSk) + } err = t.Commit() - return score, err + return newScore, err } func (db *DB) ZCount(key []byte, min int64, max int64) (int64, error) { @@ -482,42 +503,35 @@ func (db *DB) zIterator(key []byte, min int64, max int64, offset int, limit int, } } -func (db *DB) zRemRange(key []byte, min int64, max int64, offset int, limit int) (int64, error) { +func (db *DB) zRemRange(t *tx, key []byte, min int64, max int64, offset int, limit int) (int64, error) { if len(key) > MaxKeySize { return 0, errKeySize } - t := db.zsetTx - t.Lock() - defer t.Unlock() - it := db.zIterator(key, min, max, offset, limit, false) var num int64 = 0 for ; it.Valid(); it.Next() { - k := it.Key() - _, m, _, err := db.zDecodeScoreKey(k) + sk := it.Key() + _, m, _, err := db.zDecodeScoreKey(sk) if err != nil { continue } - if n, err := db.zDelItem(key, m, true); err != nil { + if n, err := db.zDelItem(t, key, m, true); err != nil { return 0, err } else if n == 1 { num++ } - t.Delete(k) + t.Delete(sk) } it.Close() - if _, err := db.zIncrSize(key, -num); err != nil { + if _, err := db.zIncrSize(t, key, -num); err != nil { return 0, err } - //todo add binlog - - err := t.Commit() - return num, err + return num, nil } func (db *DB) zReverse(s []interface{}, withScores bool) []interface{} { @@ -624,7 +638,16 @@ func (db *DB) zParseLimit(key []byte, start int, stop int) (offset int, limit in } func (db *DB) ZClear(key []byte) (int64, error) { - return db.zRemRange(key, MinScore, MaxScore, 0, -1) + t := db.zsetTx + t.Lock() + defer t.Unlock() + + rmCnt, err := db.zRemRange(t, key, MinScore, MaxScore, 0, -1) + if err == nil { + err = t.Commit() + } + + return rmCnt, err } func (db *DB) ZRange(key []byte, start int, stop int, withScores bool) ([]interface{}, error) { @@ -647,12 +670,33 @@ func (db *DB) ZRemRangeByRank(key []byte, start int, stop int) (int64, error) { if err != nil { return 0, err } - return db.zRemRange(key, MinScore, MaxScore, offset, limit) + + var rmCnt int64 + + t := db.zsetTx + t.Lock() + defer t.Unlock() + + rmCnt, err = db.zRemRange(t, key, MinScore, MaxScore, offset, limit) + if err == nil { + err = t.Commit() + } + + return rmCnt, err } //min and max must be inclusive func (db *DB) ZRemRangeByScore(key []byte, min int64, max int64) (int64, error) { - return db.zRemRange(key, min, max, 0, -1) + t := db.zsetTx + t.Lock() + defer t.Unlock() + + rmCnt, err := db.zRemRange(t, key, min, max, 0, -1) + if err == nil { + err = t.Commit() + } + + return rmCnt, err } func (db *DB) ZRevRange(key []byte, start int, stop int, withScores bool) ([]interface{}, error) { @@ -705,7 +749,7 @@ func (db *DB) ZFlush() (drop int64, err error) { for ; it.Valid(); it.Next() { t.Delete(it.Key()) drop++ - if drop%1000 == 0 { + if drop&1023 == 0 { if err = t.Commit(); err != nil { return } @@ -713,8 +757,9 @@ func (db *DB) ZFlush() (drop int64, err error) { } it.Close() + db.expFlush(t, zExpType) + err = t.Commit() - // to do : binlog return } @@ -756,3 +801,27 @@ func (db *DB) ZScan(key []byte, member []byte, count int, inclusive bool) ([]Sco return v, nil } + +func (db *DB) ZExpire(key []byte, duration int64) (int64, error) { + if duration <= 0 { + return 0, errExpireValue + } + + return db.zExpireAt(key, time.Now().Unix()+duration) +} + +func (db *DB) ZExpireAt(key []byte, when int64) (int64, error) { + if when <= time.Now().Unix() { + return 0, errExpireValue + } + + return db.zExpireAt(key, when) +} + +func (db *DB) ZTTL(key []byte) (int64, error) { + if err := checkKeySize(key); err != nil { + return -1, err + } + + return db.ttl(zExpType, key) +} From c05a66a47ccd94eed6490b3b92524ee8096aa44c Mon Sep 17 00:00:00 2001 From: siddontang Date: Fri, 6 Jun 2014 08:30:10 +0800 Subject: [PATCH 18/31] remove relay log --- ledis/ledis.go | 20 ++------------------ ledis/replication.go | 13 +------------ 2 files changed, 3 insertions(+), 30 deletions(-) diff --git a/ledis/ledis.go b/ledis/ledis.go index 040f362..50214ad 100644 --- a/ledis/ledis.go +++ b/ledis/ledis.go @@ -11,8 +11,7 @@ import ( type Config struct { DataDB leveldb.Config `json:"data_db"` - BinLog replication.BinLogConfig `json:"binlog"` - RelayLog replication.RelayLogConfig `json:"relaylog"` + BinLog replication.BinLogConfig `json:"binlog"` } type DB struct { @@ -36,8 +35,7 @@ type Ledis struct { ldb *leveldb.DB dbs [MaxDBNumber]*DB - binlog *replication.Log - relaylog *replication.Log + binlog *replication.Log quit chan struct{} } @@ -73,15 +71,6 @@ func OpenWithConfig(cfg *Config) (*Ledis, error) { l.binlog = nil } - if len(cfg.RelayLog.Path) > 0 { - l.relaylog, err = replication.NewRelayLogWithConfig(&cfg.RelayLog) - if err != nil { - return nil, err - } - } else { - l.relaylog = nil - } - for i := uint8(0); i < MaxDBNumber; i++ { l.dbs[i] = newDB(l, i) } @@ -117,11 +106,6 @@ func (l *Ledis) Close() { l.binlog.Close() l.binlog = nil } - - if l.relaylog != nil { - l.relaylog.Close() - l.relaylog = nil - } } func (l *Ledis) Select(index int) (*DB, error) { diff --git a/ledis/replication.go b/ledis/replication.go index 111f96b..8de0bc9 100644 --- a/ledis/replication.go +++ b/ledis/replication.go @@ -11,8 +11,7 @@ import ( ) var ( - errInvalidBinLogEvent = errors.New("invalid binglog event") - errRelayLogNotSupported = errors.New("write relay log not supported") + errInvalidBinLogEvent = errors.New("invalid binglog event") ) func (l *Ledis) replicateEvent(event []byte) error { @@ -131,13 +130,3 @@ func (l *Ledis) RepliateRelayLog(relayLog string, offset int64) (int64, error) { log.Error("can not go here") return offset, nil } - -func (l *Ledis) WriteRelayLog(data []byte) error { - if l.relaylog == nil { - return errRelayLogNotSupported - } - - err := l.relaylog.Log(data) - - return err -} From d9f81233d62c2182ac9b4cfd041814828ff95a77 Mon Sep 17 00:00:00 2001 From: siddontang Date: Fri, 6 Jun 2014 08:30:17 +0800 Subject: [PATCH 19/31] get log names --- replication/log.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/replication/log.go b/replication/log.go index 175099d..6648b40 100644 --- a/replication/log.go +++ b/replication/log.go @@ -217,6 +217,10 @@ func (l *Log) Close() { } } +func (l *Log) LogNames() []string { + return l.logNames +} + func (l *Log) LogFileName() string { return l.getLogFile() } From 5d466fb82fb8abcfa7951656ef8a9ed7b42bf4f6 Mon Sep 17 00:00:00 2001 From: siddontang Date: Fri, 6 Jun 2014 10:34:57 +0800 Subject: [PATCH 20/31] add data_dir, like mysql to store all data --- ledis/dump_test.go | 9 ++++----- ledis/ledis.go | 14 +++++++++++++- ledis/ledis_test.go | 6 ++---- ledis/replication_test.go | 24 +++++++----------------- replication/binlog.go | 7 ------- replication/binlog_test.go | 1 + replication/log.go | 11 +++++++---- replication/relaylog.go | 7 ------- replication/relaylog_test.go | 1 + server/app_test.go | 2 +- 10 files changed, 36 insertions(+), 46 deletions(-) diff --git a/ledis/dump_test.go b/ledis/dump_test.go index b27d8f0..f15f8f4 100644 --- a/ledis/dump_test.go +++ b/ledis/dump_test.go @@ -8,14 +8,13 @@ import ( ) func TestDump(t *testing.T) { - os.RemoveAll("/tmp/testdb_master") - os.RemoveAll("/tmp/testdb_slave") - os.Remove("/tmp/testdb.dump") + os.RemoveAll("/tmp/test_ledis_master") + os.RemoveAll("/tmp/test_ledis_slave") var masterConfig = []byte(` { + "data_dir" : "/tmp/test_ledis_master", "data_db" : { - "path" : "/tmp/testdb_master", "compression":true, "block_size" : 32768, "write_buffer_size" : 2097152, @@ -31,8 +30,8 @@ func TestDump(t *testing.T) { var slaveConfig = []byte(` { + "data_dir" : "/tmp/test_ledis_slave", "data_db" : { - "path" : "/tmp/testdb_slave", "compression":true, "block_size" : 32768, "write_buffer_size" : 2097152, diff --git a/ledis/ledis.go b/ledis/ledis.go index 50214ad..8ba83b2 100644 --- a/ledis/ledis.go +++ b/ledis/ledis.go @@ -5,12 +5,18 @@ import ( "fmt" "github.com/siddontang/go-leveldb/leveldb" "github.com/siddontang/ledisdb/replication" + "path" "sync" ) type Config struct { + DataDir string `json:"data_dir"` + + //data_db path is data_dir/data DataDB leveldb.Config `json:"data_db"` + //binlog path is data_dir/binlog + //you muse set binlog name to enable binlog BinLog replication.BinLogConfig `json:"binlog"` } @@ -51,6 +57,11 @@ func Open(configJson json.RawMessage) (*Ledis, error) { } func OpenWithConfig(cfg *Config) (*Ledis, error) { + if len(cfg.DataDir) == 0 { + return nil, fmt.Errorf("must set correct data_dir") + } + + cfg.DataDB.Path = path.Join(cfg.DataDir, "data") ldb, err := leveldb.OpenWithConfig(&cfg.DataDB) if err != nil { return nil, err @@ -62,7 +73,8 @@ func OpenWithConfig(cfg *Config) (*Ledis, error) { l.ldb = ldb - if len(cfg.BinLog.Path) > 0 { + if len(cfg.BinLog.Name) > 0 { + cfg.BinLog.Path = path.Join(cfg.DataDir, "binlog") l.binlog, err = replication.NewBinLogWithConfig(&cfg.BinLog) if err != nil { return nil, err diff --git a/ledis/ledis_test.go b/ledis/ledis_test.go index 902d545..dc28b5c 100644 --- a/ledis/ledis_test.go +++ b/ledis/ledis_test.go @@ -13,8 +13,8 @@ func getTestDB() *DB { f := func() { var d = []byte(` { + "data_dir" : "/tmp/test_ledis", "data_db" : { - "path" : "/tmp/testdb", "compression":true, "block_size" : 32768, "write_buffer_size" : 2097152, @@ -22,15 +22,13 @@ func getTestDB() *DB { }, "binlog" : { - "path" : "/tmp/testdb_binlog", "max_file_size" : 1073741824, "max_file_num" : 3 } } `) - os.RemoveAll("/tmp/testdb") - os.RemoveAll("/tmp/testdb_binlog") + os.RemoveAll("/tmp/test_ledis") var err error testLedis, err = Open(d) diff --git a/ledis/replication_test.go b/ledis/replication_test.go index 5f2243a..f22617e 100644 --- a/ledis/replication_test.go +++ b/ledis/replication_test.go @@ -12,18 +12,14 @@ func TestReplication(t *testing.T) { var slave *Ledis var err error - os.RemoveAll("/tmp/repl") - os.MkdirAll("/tmp/repl", os.ModePerm) + os.RemoveAll("/tmp/repl_repl") master, err = Open([]byte(` { - "data_db" : { - "path" : "/tmp/repl/master_db" - }, - - "binlog" : { - "path" : "/tmp/repl/master_binlog" - } + "data_dir" : "/tmp/test_repl/master", + "binlog": { + "name" : "ledis" + } } `)) if err != nil { @@ -32,13 +28,7 @@ func TestReplication(t *testing.T) { slave, err = Open([]byte(` { - "data_db" : { - "path" : "/tmp/repl/slave_db" - }, - - "binlog" : { - "path" : "/tmp/repl/slave_binlog" - } + "data_dir" : "/tmp/test_repl/slave" } `)) if err != nil { @@ -50,7 +40,7 @@ func TestReplication(t *testing.T) { db.Set([]byte("b"), []byte("2")) db.Set([]byte("c"), []byte("3")) - relayLog := "/tmp/repl/master_binlog/ledis-bin.0000001" + relayLog := "/tmp/test_repl/master/binlog/ledis-bin.0000001" var offset int64 offset, err = slave.RepliateRelayLog(relayLog, 0) diff --git a/replication/binlog.go b/replication/binlog.go index 349880c..ad9163c 100644 --- a/replication/binlog.go +++ b/replication/binlog.go @@ -44,13 +44,6 @@ func (cfg *BinLogConfig) adjust() { cfg.MaxFileNum = MaxBinLogFileNum } - if len(cfg.BaseName) == 0 { - cfg.BaseName = "ledis" - } - if len(cfg.IndexName) == 0 { - cfg.IndexName = "ledis" - } - //binlog not care space limit cfg.SpaceLimit = -1 diff --git a/replication/binlog_test.go b/replication/binlog_test.go index a66ea11..48263ae 100644 --- a/replication/binlog_test.go +++ b/replication/binlog_test.go @@ -12,6 +12,7 @@ func TestBinLog(t *testing.T) { cfg.MaxFileNum = 1 cfg.MaxFileSize = 1024 cfg.Path = "/tmp/ledis_binlog" + cfg.Name = "ledis" os.RemoveAll(cfg.Path) diff --git a/replication/log.go b/replication/log.go index 6648b40..ad5c166 100644 --- a/replication/log.go +++ b/replication/log.go @@ -21,8 +21,7 @@ type logHandler interface { } type LogConfig struct { - BaseName string `json:"base_name"` - IndexName string `json:"index_name"` + Name string `json:"name"` LogType string `json:"log_type"` Path string `json:"path"` MaxFileSize int `json:"max_file_size"` @@ -52,6 +51,10 @@ func newLog(handler logHandler, cfg *LogConfig) (*Log, error) { l.cfg = cfg l.handler = handler + if len(l.cfg.Name) == 0 { + return nil, fmt.Errorf("you must set log name first") + } + if err := os.MkdirAll(cfg.Path, os.ModePerm); err != nil { return nil, err } @@ -93,7 +96,7 @@ func (l *Log) flushIndex() error { } func (l *Log) loadIndex() error { - l.indexName = path.Join(l.cfg.Path, fmt.Sprintf("%s-%s.index", l.cfg.IndexName, l.cfg.LogType)) + l.indexName = path.Join(l.cfg.Path, fmt.Sprintf("%s-%s.index", l.cfg.Name, l.cfg.LogType)) if _, err := os.Stat(l.indexName); os.IsNotExist(err) { //no index file, nothing to do } else { @@ -145,7 +148,7 @@ func (l *Log) loadIndex() error { } func (l *Log) getLogFile() string { - return fmt.Sprintf("%s-%s.%07d", l.cfg.BaseName, l.cfg.LogType, l.lastLogIndex) + return fmt.Sprintf("%s-%s.%07d", l.cfg.Name, l.cfg.LogType, l.lastLogIndex) } func (l *Log) openNewLogFile() error { diff --git a/replication/relaylog.go b/replication/relaylog.go index 6b4894d..4ee2a4a 100644 --- a/replication/relaylog.go +++ b/replication/relaylog.go @@ -21,13 +21,6 @@ func (cfg *RelayLogConfig) adjust() { cfg.MaxFileSize = MaxRelayLogFileSize } - if len(cfg.BaseName) == 0 { - cfg.BaseName = "ledis" - } - if len(cfg.IndexName) == 0 { - cfg.IndexName = "ledis" - } - //relaylog not care file num cfg.MaxFileNum = -1 cfg.LogType = "relay" diff --git a/replication/relaylog_test.go b/replication/relaylog_test.go index 2e37af8..0cde67b 100644 --- a/replication/relaylog_test.go +++ b/replication/relaylog_test.go @@ -11,6 +11,7 @@ func TestRelayLog(t *testing.T) { cfg.MaxFileSize = 1024 cfg.SpaceLimit = 1024 cfg.Path = "/tmp/ledis_relaylog" + cfg.Name = "ledis" os.RemoveAll(cfg.Path) diff --git a/server/app_test.go b/server/app_test.go index 2293ce2..efaf08e 100644 --- a/server/app_test.go +++ b/server/app_test.go @@ -40,8 +40,8 @@ func startTestApp() { { "addr" : "127.0.0.1:16380", "db" : { + "data_dir" : "/tmp/testdb", "data_db" : { - "path" : "/tmp/testdb", "compression":true, "block_size" : 32768, "write_buffer_size" : 2097152, From 68c63c416a4d1ca1dbb8e283d275838bc1df2f8d Mon Sep 17 00:00:00 2001 From: siddontang Date: Fri, 6 Jun 2014 11:25:13 +0800 Subject: [PATCH 21/31] adjust config --- etc/ledis.json | 2 +- ledis/ledis.go | 13 ++++++++++--- ledis/replication_test.go | 2 +- server/app.go | 9 +++++++++ server/app_test.go | 2 +- server/config.go | 7 +++++++ 6 files changed, 29 insertions(+), 6 deletions(-) diff --git a/etc/ledis.json b/etc/ledis.json index 92d9f93..dcf2ea5 100644 --- a/etc/ledis.json +++ b/etc/ledis.json @@ -1,8 +1,8 @@ { "addr": "127.0.0.1:6380", + "data_dir": "/tmp/ledis_server", "db": { "data_db" : { - "path": "/tmp/ledisdb", "compression": false, "block_size": 32768, "write_buffer_size": 67108864, diff --git a/ledis/ledis.go b/ledis/ledis.go index 8ba83b2..ad2bc76 100644 --- a/ledis/ledis.go +++ b/ledis/ledis.go @@ -13,11 +13,13 @@ type Config struct { DataDir string `json:"data_dir"` //data_db path is data_dir/data + //if you not set leveldb path, use data_dir/data DataDB leveldb.Config `json:"data_db"` //binlog path is data_dir/binlog //you muse set binlog name to enable binlog - BinLog replication.BinLogConfig `json:"binlog"` + //if you not set bin log path, use data_dir/bin_log + BinLog replication.BinLogConfig `json:"bin_log"` } type DB struct { @@ -61,7 +63,10 @@ func OpenWithConfig(cfg *Config) (*Ledis, error) { return nil, fmt.Errorf("must set correct data_dir") } - cfg.DataDB.Path = path.Join(cfg.DataDir, "data") + if len(cfg.DataDB.Path) == 0 { + cfg.DataDB.Path = path.Join(cfg.DataDir, "data") + } + ldb, err := leveldb.OpenWithConfig(&cfg.DataDB) if err != nil { return nil, err @@ -74,7 +79,9 @@ func OpenWithConfig(cfg *Config) (*Ledis, error) { l.ldb = ldb if len(cfg.BinLog.Name) > 0 { - cfg.BinLog.Path = path.Join(cfg.DataDir, "binlog") + if len(cfg.BinLog.Path) == 0 { + cfg.BinLog.Path = path.Join(cfg.DataDir, "bin_log") + } l.binlog, err = replication.NewBinLogWithConfig(&cfg.BinLog) if err != nil { return nil, err diff --git a/ledis/replication_test.go b/ledis/replication_test.go index f22617e..a3a2d6f 100644 --- a/ledis/replication_test.go +++ b/ledis/replication_test.go @@ -17,7 +17,7 @@ func TestReplication(t *testing.T) { master, err = Open([]byte(` { "data_dir" : "/tmp/test_repl/master", - "binlog": { + "bin_log": { "name" : "ledis" } } diff --git a/server/app.go b/server/app.go index 9e7e029..5e414ee 100644 --- a/server/app.go +++ b/server/app.go @@ -1,6 +1,7 @@ package server import ( + "fmt" "github.com/siddontang/ledisdb/ledis" "net" "strings" @@ -17,6 +18,14 @@ type App struct { } func NewApp(cfg *Config) (*App, error) { + if len(cfg.DataDir) == 0 { + return nil, fmt.Errorf("must set data_dir first") + } + + if len(cfg.DB.DataDir) == 0 { + cfg.DB.DataDir = cfg.DataDir + } + app := new(App) app.closed = false diff --git a/server/app_test.go b/server/app_test.go index efaf08e..3357ddd 100644 --- a/server/app_test.go +++ b/server/app_test.go @@ -38,9 +38,9 @@ func startTestApp() { var d = []byte(` { + "data_dir" : "/tmp/testdb", "addr" : "127.0.0.1:16380", "db" : { - "data_dir" : "/tmp/testdb", "data_db" : { "compression":true, "block_size" : 32768, diff --git a/server/config.go b/server/config.go index 5e98694..592b51d 100644 --- a/server/config.go +++ b/server/config.go @@ -3,13 +3,20 @@ package server import ( "encoding/json" "github.com/siddontang/ledisdb/ledis" + "github.com/siddontang/ledisdb/replication" "io/ioutil" ) type Config struct { Addr string `json:"addr"` + DataDir string `json:"data_dir"` + + //if you not set db path, use data_dir DB ledis.Config `json:"db"` + + //if you not set relay log path, use data_dir/realy_log + RelayLog replication.RelayLogConfig `json:"relay_log"` } func NewConfig(data json.RawMessage) (*Config, error) { From 778f8f4a2be76ff89786ad08a58f301066a894f6 Mon Sep 17 00:00:00 2001 From: silentsai Date: Fri, 6 Jun 2014 14:38:32 +0800 Subject: [PATCH 22/31] fullfill test case for various data type --- ledis/t_ttl_test.go | 410 ++++++++++++++++++++++++++++++++------------ 1 file changed, 300 insertions(+), 110 deletions(-) diff --git a/ledis/t_ttl_test.go b/ledis/t_ttl_test.go index 0f2cdac..e7ec920 100644 --- a/ledis/t_ttl_test.go +++ b/ledis/t_ttl_test.go @@ -9,67 +9,165 @@ import ( var m sync.Mutex -func TestKvExpire(t *testing.T) { - db := getTestDB() - m.Lock() - defer m.Unlock() +type adaptor struct { + set func([]byte, []byte) (int64, error) + del func([]byte) (int64, error) + exists func([]byte) (int64, error) - k := []byte("ttl_a") - ek := []byte("ttl_b") - db.Set(k, []byte("1")) + expire func([]byte, int64) (int64, error) + expireAt func([]byte, int64) (int64, error) + ttl func([]byte) (int64, error) - if ok, _ := db.Expire(k, 10); ok != 1 { - t.Fatal(ok) - } - - // err - expire on an inexisting key - if ok, _ := db.Expire(ek, 10); ok != 0 { - t.Fatal(ok) - } - - // err - duration is zero - if ok, err := db.Expire(k, 0); err == nil || ok != 0 { - t.Fatal(fmt.Sprintf("res = %d, err = %s", ok, err)) - } - - // err - duration is negative - if ok, err := db.Expire(k, -10); err == nil || ok != 0 { - t.Fatal(fmt.Sprintf("res = %d, err = %s", ok, err)) - } + showIdent func() string } -func TestKvExpireAt(t *testing.T) { - db := getTestDB() - m.Lock() - defer m.Unlock() - - k := []byte("ttl_a") - ek := []byte("ttl_b") - db.Set(k, []byte("1")) - - now := time.Now().Unix() - - if ok, _ := db.ExpireAt(k, now+5); ok != 1 { - t.Fatal(ok) +func kvAdaptor(db *DB) *adaptor { + adp := new(adaptor) + adp.showIdent = func() string { + return "kv-adptor" } - // err - expire on an inexisting key - if ok, _ := db.ExpireAt(ek, now+5); ok != 0 { - t.Fatal(ok) + adp.set = db.SetNX + adp.exists = db.Exists + adp.del = func(k []byte) (int64, error) { + return db.Del(k) } - // err - expire with the current time - if ok, err := db.ExpireAt(k, now); err == nil || ok != 0 { - t.Fatal(fmt.Sprintf("res = %d, err = %s", ok, err)) - } + adp.expire = db.Expire + adp.expireAt = db.ExpireAt + adp.ttl = db.TTL - // err - expire with the time before - if ok, err := db.ExpireAt(k, now-5); err == nil || ok != 0 { - t.Fatal(fmt.Sprintf("res = %d, err = %s", ok, err)) - } + return adp } -func TestKvTTL(t *testing.T) { +func listAdaptor(db *DB) *adaptor { + adp := new(adaptor) + adp.showIdent = func() string { + return "list-adptor" + } + + adp.set = func(k []byte, v []byte) (int64, error) { + eles := make([][]byte, 0) + for i := 0; i < 3; i++ { + e := []byte(String(v) + fmt.Sprintf("_%d", i)) + eles = append(eles, e) + } + + if n, err := db.LPush(k, eles...); err != nil { + return 0, err + } else { + return n, nil + } + } + + adp.exists = func(k []byte) (int64, error) { + if llen, err := db.LLen(k); err != nil || llen <= 0 { + return 0, err + } else { + return 1, nil + } + } + + adp.del = db.LClear + adp.expire = db.LExpire + adp.expireAt = db.LExpireAt + adp.ttl = db.LTTL + + return adp +} + +func hashAdaptor(db *DB) *adaptor { + adp := new(adaptor) + adp.showIdent = func() string { + return "hash-adptor" + } + + adp.set = func(k []byte, v []byte) (int64, error) { + datas := make([]FVPair, 0) + for i := 0; i < 3; i++ { + suffix := fmt.Sprintf("_%d", i) + pair := FVPair{ + Field: []byte(String(k) + suffix), + Value: []byte(String(v) + suffix)} + + datas = append(datas, pair) + } + + if err := db.HMset(k, datas...); err != nil { + return 0, err + } else { + return int64(len(datas)), nil + } + } + + adp.exists = func(k []byte) (int64, error) { + if hlen, err := db.HLen(k); err != nil || hlen <= 0 { + return 0, err + } else { + return 1, nil + } + } + + adp.del = db.HClear + adp.expire = db.HExpire + adp.expireAt = db.HExpireAt + adp.ttl = db.HTTL + + return adp +} + +func zsetAdaptor(db *DB) *adaptor { + adp := new(adaptor) + adp.showIdent = func() string { + return "zset-adptor" + } + + adp.set = func(k []byte, v []byte) (int64, error) { + datas := make([]ScorePair, 0) + for i := 0; i < 3; i++ { + memb := []byte(String(k) + fmt.Sprintf("_%d", i)) + pair := ScorePair{ + Score: int64(i), + Member: memb} + + datas = append(datas, pair) + } + + if n, err := db.ZAdd(k, datas...); err != nil { + return 0, err + } else { + return n, nil + } + } + + adp.exists = func(k []byte) (int64, error) { + if cnt, err := db.ZCard(k); err != nil || cnt <= 0 { + return 0, err + } else { + return 1, nil + } + } + + adp.del = db.ZClear + adp.expire = db.ZExpire + adp.expireAt = db.ZExpireAt + adp.ttl = db.ZTTL + + return adp +} + +func allAdaptors(db *DB) []*adaptor { + adps := make([]*adaptor, 4) + adps[0] = kvAdaptor(db) + adps[1] = listAdaptor(db) + adps[2] = hashAdaptor(db) + adps[3] = zsetAdaptor(db) + return adps +} + +/////////////////////////////////////////////////////// + +func TestExpire(t *testing.T) { db := getTestDB() m.Lock() defer m.Unlock() @@ -77,25 +175,101 @@ func TestKvTTL(t *testing.T) { k := []byte("ttl_a") ek := []byte("ttl_b") - db.Set(k, []byte("1")) - db.Expire(k, 2) + dbEntrys := allAdaptors(db) + for _, entry := range dbEntrys { + ident := entry.showIdent() - if tRemain, _ := db.TTL(k); tRemain != 2 { - t.Fatal(tRemain) - } + entry.set(k, []byte("1")) - // err - check ttl on an inexisting key - if tRemain, _ := db.TTL(ek); tRemain != -1 { - t.Fatal(tRemain) - } + if ok, _ := entry.expire(k, 10); ok != 1 { + t.Fatal(ident, ok) + } - db.Del(k) - if tRemain, _ := db.TTL(k); tRemain != -1 { - t.Fatal(tRemain) + // err - expire on an inexisting key + if ok, _ := entry.expire(ek, 10); ok != 0 { + t.Fatal(ident, ok) + } + + // err - duration is zero + if ok, err := entry.expire(k, 0); err == nil || ok != 0 { + t.Fatal(ident, fmt.Sprintf("res = %d, err = %s", ok, err)) + } + + // err - duration is negative + if ok, err := entry.expire(k, -10); err == nil || ok != 0 { + t.Fatal(ident, fmt.Sprintf("res = %d, err = %s", ok, err)) + } } } -func TestKvExpCompose(t *testing.T) { +func TestExpireAt(t *testing.T) { + db := getTestDB() + m.Lock() + defer m.Unlock() + + k := []byte("ttl_a") + ek := []byte("ttl_b") + + dbEntrys := allAdaptors(db) + for _, entry := range dbEntrys { + ident := entry.showIdent() + now := time.Now().Unix() + + entry.set(k, []byte("1")) + + if ok, _ := entry.expireAt(k, now+5); ok != 1 { + t.Fatal(ident, ok) + } + + // err - expire on an inexisting key + if ok, _ := entry.expireAt(ek, now+5); ok != 0 { + t.Fatal(ident, ok) + } + + // err - expire with the current time + if ok, err := entry.expireAt(k, now); err == nil || ok != 0 { + t.Fatal(ident, fmt.Sprintf("res = %d, err = %s", ok, err)) + } + + // err - expire with the time before + if ok, err := entry.expireAt(k, now-5); err == nil || ok != 0 { + t.Fatal(ident, fmt.Sprintf("res = %d, err = %s", ok, err)) + } + } +} + +func TestTTL(t *testing.T) { + db := getTestDB() + m.Lock() + defer m.Unlock() + + k := []byte("ttl_a") + ek := []byte("ttl_b") + + dbEntrys := allAdaptors(db) + for _, entry := range dbEntrys { + ident := entry.showIdent() + + entry.set(k, []byte("1")) + entry.expire(k, 2) + + if tRemain, _ := entry.ttl(k); tRemain != 2 { + t.Fatal(ident, tRemain) + } + + // err - check ttl on an inexisting key + if tRemain, _ := entry.ttl(ek); tRemain != -1 { + t.Fatal(ident, tRemain) + } + + entry.del(k) + if tRemain, _ := entry.ttl(k); tRemain != -1 { + t.Fatal(ident, tRemain) + } + } +} + +func TestExpCompose(t *testing.T) { db := getTestDB() m.Lock() defer m.Unlock() @@ -104,68 +278,84 @@ func TestKvExpCompose(t *testing.T) { k1 := []byte("ttl_b") k2 := []byte("ttl_c") - db.Set(k0, k0) - db.Set(k1, k1) - db.Set(k2, k2) + dbEntrys := allAdaptors(db) - db.Expire(k0, 5) - db.Expire(k1, 2) - db.Expire(k2, 60) + for _, entry := range dbEntrys { + ident := entry.showIdent() - if tRemain, _ := db.TTL(k0); tRemain != 5 { - t.Fatal(tRemain) - } - if tRemain, _ := db.TTL(k1); tRemain != 2 { - t.Fatal(tRemain) - } - if tRemain, _ := db.TTL(k2); tRemain != 60 { - t.Fatal(tRemain) + entry.set(k0, k0) + entry.set(k1, k1) + entry.set(k2, k2) + + entry.expire(k0, 5) + entry.expire(k1, 2) + entry.expire(k2, 60) + + if tRemain, _ := entry.ttl(k0); tRemain != 5 { + t.Fatal(ident, tRemain) + } + if tRemain, _ := entry.ttl(k1); tRemain != 2 { + t.Fatal(ident, tRemain) + } + if tRemain, _ := entry.ttl(k2); tRemain != 60 { + t.Fatal(ident, tRemain) + } } // after 1 sec time.Sleep(1 * time.Second) - if tRemain, _ := db.TTL(k0); tRemain != 4 { - t.Fatal(tRemain) - } - if tRemain, _ := db.TTL(k1); tRemain != 1 { - t.Fatal(tRemain) + + for _, entry := range dbEntrys { + ident := entry.showIdent() + + if tRemain, _ := entry.ttl(k0); tRemain != 4 { + t.Fatal(ident, tRemain) + } + if tRemain, _ := entry.ttl(k1); tRemain != 1 { + t.Fatal(ident, tRemain) + } } // after 2 sec time.Sleep(2 * time.Second) - if tRemain, _ := db.TTL(k1); tRemain != -1 { - t.Fatal(tRemain) - } - if v, _ := db.Get(k1); v != nil { - t.Fatal(v) - } - if tRemain, _ := db.TTL(k0); tRemain != 2 { - t.Fatal(tRemain) - } - if v, _ := db.Get(k0); v == nil { - t.Fatal(v) - } + for _, entry := range dbEntrys { + ident := entry.showIdent() - // refresh the expiration of key - if tRemain, _ := db.TTL(k2); !(0 < tRemain && tRemain < 60) { - t.Fatal(tRemain) - } + if tRemain, _ := entry.ttl(k1); tRemain != -1 { + t.Fatal(ident, tRemain) + } + if exist, _ := entry.exists(k1); exist > 0 { + t.Fatal(ident, false) + } - if ok, _ := db.Expire(k2, 100); ok != 1 { - t.Fatal(false) - } + if tRemain, _ := entry.ttl(k0); tRemain != 2 { + t.Fatal(ident, tRemain) + } + if exist, _ := entry.exists(k0); exist <= 0 { + t.Fatal(ident, false) + } - if tRemain, _ := db.TTL(k2); tRemain != 100 { - t.Fatal(tRemain) - } + // refresh the expiration of key + if tRemain, _ := entry.ttl(k2); !(0 < tRemain && tRemain < 60) { + t.Fatal(ident, tRemain) + } - // expire an inexisting key - if ok, _ := db.Expire(k1, 10); ok == 1 { - t.Fatal(false) - } - if tRemain, _ := db.TTL(k1); tRemain != -1 { - t.Fatal(tRemain) + if ok, _ := entry.expire(k2, 100); ok != 1 { + t.Fatal(ident, false) + } + + if tRemain, _ := entry.ttl(k2); tRemain != 100 { + t.Fatal(ident, tRemain) + } + + // expire an inexisting key + if ok, _ := entry.expire(k1, 10); ok == 1 { + t.Fatal(ident, false) + } + if tRemain, _ := entry.ttl(k1); tRemain != -1 { + t.Fatal(ident, tRemain) + } } return From 8fab454223a18c8ef51e74a0588a253f3d003324 Mon Sep 17 00:00:00 2001 From: siddontang Date: Fri, 6 Jun 2014 14:57:18 +0800 Subject: [PATCH 23/31] adjust config, add some replication fund --- ledis/ledis.go | 7 +++---- ledis/replication_test.go | 6 ++---- replication/log.go | 2 +- server/app.go | 34 +++++++++++++++++++++++++++++----- server/client.go | 9 ++++++--- server/cmd_replication.go | 20 ++++++++++++++++++++ server/config.go | 4 ++++ server/replication.go | 9 +++++++++ 8 files changed, 74 insertions(+), 17 deletions(-) create mode 100644 server/cmd_replication.go create mode 100644 server/replication.go diff --git a/ledis/ledis.go b/ledis/ledis.go index ad2bc76..d9425b5 100644 --- a/ledis/ledis.go +++ b/ledis/ledis.go @@ -12,12 +12,11 @@ import ( type Config struct { DataDir string `json:"data_dir"` - //data_db path is data_dir/data //if you not set leveldb path, use data_dir/data DataDB leveldb.Config `json:"data_db"` - //binlog path is data_dir/binlog - //you muse set binlog name to enable binlog + UseBinLog bool `json:"use_bin_log"` + //if you not set bin log path, use data_dir/bin_log BinLog replication.BinLogConfig `json:"bin_log"` } @@ -78,7 +77,7 @@ func OpenWithConfig(cfg *Config) (*Ledis, error) { l.ldb = ldb - if len(cfg.BinLog.Name) > 0 { + if cfg.UseBinLog { if len(cfg.BinLog.Path) == 0 { cfg.BinLog.Path = path.Join(cfg.DataDir, "bin_log") } diff --git a/ledis/replication_test.go b/ledis/replication_test.go index a3a2d6f..2fc02a4 100644 --- a/ledis/replication_test.go +++ b/ledis/replication_test.go @@ -17,9 +17,7 @@ func TestReplication(t *testing.T) { master, err = Open([]byte(` { "data_dir" : "/tmp/test_repl/master", - "bin_log": { - "name" : "ledis" - } + "use_bin_log" : true } `)) if err != nil { @@ -40,7 +38,7 @@ func TestReplication(t *testing.T) { db.Set([]byte("b"), []byte("2")) db.Set([]byte("c"), []byte("3")) - relayLog := "/tmp/test_repl/master/binlog/ledis-bin.0000001" + relayLog := "/tmp/test_repl/master/bin_log/ledis-bin.0000001" var offset int64 offset, err = slave.RepliateRelayLog(relayLog, 0) diff --git a/replication/log.go b/replication/log.go index ad5c166..4cdc460 100644 --- a/replication/log.go +++ b/replication/log.go @@ -52,7 +52,7 @@ func newLog(handler logHandler, cfg *LogConfig) (*Log, error) { l.handler = handler if len(l.cfg.Name) == 0 { - return nil, fmt.Errorf("you must set log name first") + l.cfg.Name = "ledis" } if err := os.MkdirAll(cfg.Path, os.ModePerm); err != nil { diff --git a/server/app.go b/server/app.go index 5e414ee..e293a14 100644 --- a/server/app.go +++ b/server/app.go @@ -3,6 +3,7 @@ package server import ( "fmt" "github.com/siddontang/ledisdb/ledis" + "github.com/siddontang/ledisdb/replication" "net" "strings" ) @@ -15,6 +16,12 @@ type App struct { ldb *ledis.Ledis closed bool + + slaveMode bool + + relayLog *replication.Log + + quit chan struct{} } func NewApp(cfg *Config) (*App, error) { @@ -28,6 +35,8 @@ func NewApp(cfg *Config) (*App, error) { app := new(App) + app.quit = make(chan struct{}) + app.closed = false app.cfg = cfg @@ -44,8 +53,17 @@ func NewApp(cfg *Config) (*App, error) { return nil, err } - app.ldb, err = ledis.OpenWithConfig(&cfg.DB) - if err != nil { + app.slaveMode = false + + if len(app.cfg.SlaveOf) > 0 { + app.slaveMode = true + + if app.relayLog, err = replication.NewRelayLogWithConfig(&cfg.RelayLog); err != nil { + return nil, err + } + } + + if app.ldb, err = ledis.OpenWithConfig(&cfg.DB); err != nil { return nil, err } @@ -57,20 +75,26 @@ func (app *App) Close() { return } + app.closed = true + + close(app.quit) + app.listener.Close() app.ldb.Close() - - app.closed = true } func (app *App) Run() { + if app.slaveMode { + app.runReplication() + } + for !app.closed { conn, err := app.listener.Accept() if err != nil { continue } - newClient(conn, app.ldb) + newClient(conn, app) } } diff --git a/server/client.go b/server/client.go index bd558fd..40533b0 100644 --- a/server/client.go +++ b/server/client.go @@ -15,6 +15,7 @@ import ( var errReadRequest = errors.New("invalid request protocol") type client struct { + app *App ldb *ledis.Ledis db *ledis.DB @@ -29,11 +30,13 @@ type client struct { reqC chan error } -func newClient(c net.Conn, ldb *ledis.Ledis) { +func newClient(c net.Conn, app *App) { co := new(client) - co.ldb = ldb + + co.app = app + co.ldb = app.ldb //use default db - co.db, _ = ldb.Select(0) + co.db, _ = app.ldb.Select(0) co.c = c co.rb = bufio.NewReaderSize(c, 256) diff --git a/server/cmd_replication.go b/server/cmd_replication.go new file mode 100644 index 0000000..634d57f --- /dev/null +++ b/server/cmd_replication.go @@ -0,0 +1,20 @@ +package server + +func slaveofCommand(c *client) error { + if len(c.args) > 1 { + return ErrCmdParams + } + + master := "" + if len(c.args) == 1 { + master = string(c.args[0]) + } + + if err := c.app.slaveof(master); err != nil { + return err + } + + c.writeStatus(OK) + + return nil +} diff --git a/server/config.go b/server/config.go index 592b51d..518a083 100644 --- a/server/config.go +++ b/server/config.go @@ -15,6 +15,10 @@ type Config struct { //if you not set db path, use data_dir DB ledis.Config `json:"db"` + //set slaveof to enable replication from master + //empty, no replication + SlaveOf string `json:"slaveof"` + //if you not set relay log path, use data_dir/realy_log RelayLog replication.RelayLogConfig `json:"relay_log"` } diff --git a/server/replication.go b/server/replication.go new file mode 100644 index 0000000..d1a3749 --- /dev/null +++ b/server/replication.go @@ -0,0 +1,9 @@ +package server + +func (app *App) slaveof(master string) error { + return nil +} + +func (app *App) runReplication() { + +} From 664a082c068e4dd35201cd1da93adb1a877104e7 Mon Sep 17 00:00:00 2001 From: siddontang Date: Sat, 7 Jun 2014 16:56:22 +0800 Subject: [PATCH 24/31] refactor replication, deprecate relaylog --- replication/log.go => ledis/binlog.go | 142 +++++++++++++++----------- {replication => ledis}/binlog_test.go | 2 +- ledis/ledis.go | 7 +- ledis/replication.go | 8 +- ledis/replication_test.go | 6 +- ledis/tx.go | 3 +- replication/binlog.go | 89 ---------------- replication/relaylog.go | 50 --------- replication/relaylog_test.go | 41 -------- server/app.go | 7 -- server/config.go | 4 - 11 files changed, 95 insertions(+), 264 deletions(-) rename replication/log.go => ledis/binlog.go (64%) rename {replication => ledis}/binlog_test.go (96%) delete mode 100644 replication/binlog.go delete mode 100644 replication/relaylog.go delete mode 100644 replication/relaylog_test.go diff --git a/replication/log.go b/ledis/binlog.go similarity index 64% rename from replication/log.go rename to ledis/binlog.go index 4cdc460..c5df748 100644 --- a/replication/log.go +++ b/ledis/binlog.go @@ -1,8 +1,9 @@ -package replication +package ledis import ( "bufio" - "errors" + "encoding/binary" + "encoding/json" "fmt" "github.com/siddontang/go-log/log" "io/ioutil" @@ -10,27 +11,56 @@ import ( "path" "strconv" "strings" + "time" ) -var ( - ErrOverSpaceLimit = errors.New("total log files exceed space limit") +const ( + MaxBinLogFileSize int = 1024 * 1024 * 1024 + MaxBinLogFileNum int = 10000 + + DefaultBinLogFileSize int = MaxBinLogFileSize + DefaultBinLogFileNum int = 10 ) -type logHandler interface { - Write(wb *bufio.Writer, data []byte) (int, error) -} +/* +index file format: +ledis-bin.00001 +ledis-bin.00002 +ledis-bin.00003 -type LogConfig struct { +log file format + +timestamp(bigendian uint32, seconds)|PayloadLen(bigendian uint32)|PayloadData + +*/ + +type BinLogConfig struct { Name string `json:"name"` - LogType string `json:"log_type"` Path string `json:"path"` MaxFileSize int `json:"max_file_size"` MaxFileNum int `json:"max_file_num"` - SpaceLimit int64 `json:"space_limit"` } -type Log struct { - cfg *LogConfig +func (cfg *BinLogConfig) adjust() { + if cfg.MaxFileSize <= 0 { + cfg.MaxFileSize = DefaultBinLogFileSize + } else if cfg.MaxFileSize > MaxBinLogFileSize { + cfg.MaxFileSize = MaxBinLogFileSize + } + + if cfg.MaxFileNum <= 0 { + cfg.MaxFileNum = DefaultBinLogFileNum + } else if cfg.MaxFileNum > MaxBinLogFileNum { + cfg.MaxFileNum = MaxBinLogFileNum + } + + if len(cfg.Name) == 0 { + cfg.Name = "ledis" + } +} + +type BinLog struct { + cfg *BinLogConfig logFile *os.File @@ -39,28 +69,30 @@ type Log struct { indexName string logNames []string lastLogIndex int - - space int64 - - handler logHandler } -func newLog(handler logHandler, cfg *LogConfig) (*Log, error) { - l := new(Log) +func NewBinLog(data json.RawMessage) (*BinLog, error) { + var cfg BinLogConfig + + if err := json.Unmarshal(data, &cfg); err != nil { + return nil, err + } + + return NewBinLogWithConfig(&cfg) +} + +func NewBinLogWithConfig(cfg *BinLogConfig) (*BinLog, error) { + cfg.adjust() + + l := new(BinLog) l.cfg = cfg - l.handler = handler - - if len(l.cfg.Name) == 0 { - l.cfg.Name = "ledis" - } if err := os.MkdirAll(cfg.Path, os.ModePerm); err != nil { return nil, err } l.logNames = make([]string, 0, 16) - l.space = 0 if err := l.loadIndex(); err != nil { return nil, err @@ -69,7 +101,7 @@ func newLog(handler logHandler, cfg *LogConfig) (*Log, error) { return l, nil } -func (l *Log) flushIndex() error { +func (l *BinLog) flushIndex() error { data := strings.Join(l.logNames, "\n") bakName := fmt.Sprintf("%s.bak", l.indexName) @@ -95,8 +127,8 @@ func (l *Log) flushIndex() error { return nil } -func (l *Log) loadIndex() error { - l.indexName = path.Join(l.cfg.Path, fmt.Sprintf("%s-%s.index", l.cfg.Name, l.cfg.LogType)) +func (l *BinLog) loadIndex() error { + l.indexName = path.Join(l.cfg.Path, fmt.Sprintf("%s-bin.index", l.cfg.Name)) if _, err := os.Stat(l.indexName); os.IsNotExist(err) { //no index file, nothing to do } else { @@ -112,12 +144,10 @@ func (l *Log) loadIndex() error { continue } - if st, err := os.Stat(path.Join(l.cfg.Path, line)); err != nil { + if _, err := os.Stat(path.Join(l.cfg.Path, line)); err != nil { log.Error("load index line %s error %s", line, err.Error()) return err } else { - l.space += st.Size() - l.logNames = append(l.logNames, line) } } @@ -147,11 +177,11 @@ func (l *Log) loadIndex() error { return nil } -func (l *Log) getLogFile() string { - return fmt.Sprintf("%s-%s.%07d", l.cfg.Name, l.cfg.LogType, l.lastLogIndex) +func (l *BinLog) getLogFile() string { + return fmt.Sprintf("%s-bin.%07d", l.cfg.Name, l.lastLogIndex) } -func (l *Log) openNewLogFile() error { +func (l *BinLog) openNewLogFile() error { var err error lastName := l.getLogFile() @@ -180,7 +210,7 @@ func (l *Log) openNewLogFile() error { return nil } -func (l *Log) checkLogFileSize() bool { +func (l *BinLog) checkLogFileSize() bool { if l.logFile == nil { return false } @@ -197,15 +227,9 @@ func (l *Log) checkLogFileSize() bool { return false } -func (l *Log) purge(n int) { +func (l *BinLog) purge(n int) { for i := 0; i < n; i++ { logPath := path.Join(l.cfg.Path, l.logNames[i]) - if st, err := os.Stat(logPath); err != nil { - log.Error("purge %s error %s", logPath, err.Error()) - } else { - l.space -= st.Size() - } - os.Remove(logPath) } @@ -213,22 +237,22 @@ func (l *Log) purge(n int) { l.logNames = l.logNames[0 : len(l.logNames)-n] } -func (l *Log) Close() { +func (l *BinLog) Close() { if l.logFile != nil { l.logFile.Close() l.logFile = nil } } -func (l *Log) LogNames() []string { +func (l *BinLog) LogNames() []string { return l.logNames } -func (l *Log) LogFileName() string { +func (l *BinLog) LogFileName() string { return l.getLogFile() } -func (l *Log) LogFilePos() int64 { +func (l *BinLog) LogFilePos() int64 { if l.logFile == nil { return 0 } else { @@ -237,7 +261,7 @@ func (l *Log) LogFilePos() int64 { } } -func (l *Log) Purge(n int) error { +func (l *BinLog) Purge(n int) error { if len(l.logNames) == 0 { return nil } @@ -255,11 +279,7 @@ func (l *Log) Purge(n int) error { return l.flushIndex() } -func (l *Log) Log(args ...[]byte) error { - if l.cfg.SpaceLimit > 0 && l.space >= l.cfg.SpaceLimit { - return ErrOverSpaceLimit - } - +func (l *BinLog) Log(args ...[]byte) error { var err error if l.logFile == nil { @@ -268,17 +288,23 @@ func (l *Log) Log(args ...[]byte) error { } } - totalSize := 0 + //we treat log many args as a batch, so use same createTime + createTime := uint32(time.Now().Unix()) - var n int = 0 for _, data := range args { - if n, err = l.handler.Write(l.logWb, data); err != nil { - log.Error("write log error %s", err.Error()) + payLoadLen := uint32(len(data)) + + if err := binary.Write(l.logWb, binary.BigEndian, createTime); err != nil { return err - } else { - totalSize += n } + if err := binary.Write(l.logWb, binary.BigEndian, payLoadLen); err != nil { + return err + } + + if _, err := l.logWb.Write(data); err != nil { + return err + } } if err = l.logWb.Flush(); err != nil { @@ -286,8 +312,6 @@ func (l *Log) Log(args ...[]byte) error { return err } - l.space += int64(totalSize) - l.checkLogFileSize() return nil diff --git a/replication/binlog_test.go b/ledis/binlog_test.go similarity index 96% rename from replication/binlog_test.go rename to ledis/binlog_test.go index 48263ae..995a8d1 100644 --- a/replication/binlog_test.go +++ b/ledis/binlog_test.go @@ -1,4 +1,4 @@ -package replication +package ledis import ( "io/ioutil" diff --git a/ledis/ledis.go b/ledis/ledis.go index d9425b5..0cf108a 100644 --- a/ledis/ledis.go +++ b/ledis/ledis.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "github.com/siddontang/go-leveldb/leveldb" - "github.com/siddontang/ledisdb/replication" "path" "sync" ) @@ -18,7 +17,7 @@ type Config struct { UseBinLog bool `json:"use_bin_log"` //if you not set bin log path, use data_dir/bin_log - BinLog replication.BinLogConfig `json:"bin_log"` + BinLog BinLogConfig `json:"bin_log"` } type DB struct { @@ -42,7 +41,7 @@ type Ledis struct { ldb *leveldb.DB dbs [MaxDBNumber]*DB - binlog *replication.Log + binlog *BinLog quit chan struct{} } @@ -81,7 +80,7 @@ func OpenWithConfig(cfg *Config) (*Ledis, error) { if len(cfg.BinLog.Path) == 0 { cfg.BinLog.Path = path.Join(cfg.DataDir, "bin_log") } - l.binlog, err = replication.NewBinLogWithConfig(&cfg.BinLog) + l.binlog, err = NewBinLogWithConfig(&cfg.BinLog) if err != nil { return nil, err } diff --git a/ledis/replication.go b/ledis/replication.go index 8de0bc9..9a2cb1b 100644 --- a/ledis/replication.go +++ b/ledis/replication.go @@ -14,7 +14,7 @@ var ( errInvalidBinLogEvent = errors.New("invalid binglog event") ) -func (l *Ledis) replicateEvent(event []byte) error { +func (l *Ledis) ReplicateEvent(event []byte) error { if len(event) == 0 { return errInvalidBinLogEvent } @@ -70,8 +70,8 @@ func (l *Ledis) replicateCommandEvent(event []byte) error { return errors.New("command event not supported now") } -func (l *Ledis) RepliateRelayLog(relayLog string, offset int64) (int64, error) { - f, err := os.Open(relayLog) +func (l *Ledis) RepliateFromBinLog(filePath string, offset int64) (int64, error) { + f, err := os.Open(filePath) if err != nil { return 0, err } @@ -114,7 +114,7 @@ func (l *Ledis) RepliateRelayLog(relayLog string, offset int64) (int64, error) { } l.Lock() - err = l.replicateEvent(dataBuf.Bytes()) + err = l.ReplicateEvent(dataBuf.Bytes()) l.Unlock() if err != nil { log.Fatal("replication error %s, skip to next", err.Error()) diff --git a/ledis/replication_test.go b/ledis/replication_test.go index 2fc02a4..2e4b977 100644 --- a/ledis/replication_test.go +++ b/ledis/replication_test.go @@ -38,14 +38,14 @@ func TestReplication(t *testing.T) { db.Set([]byte("b"), []byte("2")) db.Set([]byte("c"), []byte("3")) - relayLog := "/tmp/test_repl/master/bin_log/ledis-bin.0000001" + binLogName := "/tmp/test_repl/master/bin_log/ledis-bin.0000001" var offset int64 - offset, err = slave.RepliateRelayLog(relayLog, 0) + offset, err = slave.RepliateFromBinLog(binLogName, 0) if err != nil { t.Fatal(err) } else { - if st, err := os.Stat(relayLog); err != nil { + if st, err := os.Stat(binLogName); err != nil { t.Fatal(err) } else if st.Size() != offset { t.Fatal(st.Size(), offset) diff --git a/ledis/tx.go b/ledis/tx.go index 6c4c023..fa7379b 100644 --- a/ledis/tx.go +++ b/ledis/tx.go @@ -2,7 +2,6 @@ package ledis import ( "github.com/siddontang/go-leveldb/leveldb" - "github.com/siddontang/ledisdb/replication" "sync" ) @@ -12,7 +11,7 @@ type tx struct { l *Ledis wb *leveldb.WriteBatch - binlog *replication.Log + binlog *BinLog batch [][]byte } diff --git a/replication/binlog.go b/replication/binlog.go deleted file mode 100644 index ad9163c..0000000 --- a/replication/binlog.go +++ /dev/null @@ -1,89 +0,0 @@ -package replication - -import ( - "bufio" - "encoding/binary" - "encoding/json" - "time" -) - -const ( - MaxBinLogFileSize int = 1024 * 1024 * 1024 - MaxBinLogFileNum int = 10000 - - DefaultBinLogFileSize int = MaxBinLogFileSize - DefaultBinLogFileNum int = 10 -) - -/* -index file format: -ledis-bin.00001 -ledis-bin.00002 -ledis-bin.00003 - -log file format - -timestamp(bigendian uint32, seconds)|PayloadLen(bigendian uint32)|PayloadData - -*/ - -type BinLogConfig struct { - LogConfig -} - -func (cfg *BinLogConfig) adjust() { - if cfg.MaxFileSize <= 0 { - cfg.MaxFileSize = DefaultBinLogFileSize - } else if cfg.MaxFileSize > MaxBinLogFileSize { - cfg.MaxFileSize = MaxBinLogFileSize - } - - if cfg.MaxFileNum <= 0 { - cfg.MaxFileNum = DefaultBinLogFileNum - } else if cfg.MaxFileNum > MaxBinLogFileNum { - cfg.MaxFileNum = MaxBinLogFileNum - } - - //binlog not care space limit - cfg.SpaceLimit = -1 - - cfg.LogType = "bin" -} - -type binlogHandler struct { -} - -func (h *binlogHandler) Write(wb *bufio.Writer, data []byte) (int, error) { - createTime := uint32(time.Now().Unix()) - payLoadLen := uint32(len(data)) - - if err := binary.Write(wb, binary.BigEndian, createTime); err != nil { - return 0, err - } - - if err := binary.Write(wb, binary.BigEndian, payLoadLen); err != nil { - return 0, err - } - - if _, err := wb.Write(data); err != nil { - return 0, err - } - - return 8 + len(data), nil -} - -func NewBinLog(data json.RawMessage) (*Log, error) { - var cfg BinLogConfig - - if err := json.Unmarshal(data, &cfg); err != nil { - return nil, err - } - - return NewBinLogWithConfig(&cfg) -} - -func NewBinLogWithConfig(cfg *BinLogConfig) (*Log, error) { - cfg.adjust() - - return newLog(new(binlogHandler), &cfg.LogConfig) -} diff --git a/replication/relaylog.go b/replication/relaylog.go deleted file mode 100644 index 4ee2a4a..0000000 --- a/replication/relaylog.go +++ /dev/null @@ -1,50 +0,0 @@ -package replication - -import ( - "bufio" - "encoding/json" -) - -const ( - MaxRelayLogFileSize int = 1024 * 1024 * 1024 - DefaultRelayLogFileSize int = MaxRelayLogFileSize -) - -type RelayLogConfig struct { - LogConfig -} - -func (cfg *RelayLogConfig) adjust() { - if cfg.MaxFileSize <= 0 { - cfg.MaxFileSize = DefaultRelayLogFileSize - } else if cfg.MaxFileSize > MaxRelayLogFileSize { - cfg.MaxFileSize = MaxRelayLogFileSize - } - - //relaylog not care file num - cfg.MaxFileNum = -1 - cfg.LogType = "relay" -} - -type relayLogHandler struct { -} - -func (h *relayLogHandler) Write(wb *bufio.Writer, data []byte) (int, error) { - return wb.Write(data) -} - -func NewRelayLog(data json.RawMessage) (*Log, error) { - var cfg RelayLogConfig - - if err := json.Unmarshal(data, &cfg); err != nil { - return nil, err - } - - return NewRelayLogWithConfig(&cfg) -} - -func NewRelayLogWithConfig(cfg *RelayLogConfig) (*Log, error) { - cfg.adjust() - - return newLog(new(relayLogHandler), &cfg.LogConfig) -} diff --git a/replication/relaylog_test.go b/replication/relaylog_test.go deleted file mode 100644 index 0cde67b..0000000 --- a/replication/relaylog_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package replication - -import ( - "os" - "testing" -) - -func TestRelayLog(t *testing.T) { - cfg := new(RelayLogConfig) - - cfg.MaxFileSize = 1024 - cfg.SpaceLimit = 1024 - cfg.Path = "/tmp/ledis_relaylog" - cfg.Name = "ledis" - - os.RemoveAll(cfg.Path) - - b, err := NewRelayLogWithConfig(cfg) - if err != nil { - t.Fatal(err) - } - - if err := b.Log(make([]byte, 1024)); err != nil { - t.Fatal(err) - } - - if err := b.Log(make([]byte, 1)); err == nil { - t.Fatal("must not nil") - } else if err != ErrOverSpaceLimit { - t.Fatal(err) - } - - if err := b.Purge(1); err != nil { - t.Fatal(err) - } - - if err := b.Log(make([]byte, 1)); err != nil { - t.Fatal(err) - } - -} diff --git a/server/app.go b/server/app.go index e293a14..cf76049 100644 --- a/server/app.go +++ b/server/app.go @@ -3,7 +3,6 @@ package server import ( "fmt" "github.com/siddontang/ledisdb/ledis" - "github.com/siddontang/ledisdb/replication" "net" "strings" ) @@ -19,8 +18,6 @@ type App struct { slaveMode bool - relayLog *replication.Log - quit chan struct{} } @@ -57,10 +54,6 @@ func NewApp(cfg *Config) (*App, error) { if len(app.cfg.SlaveOf) > 0 { app.slaveMode = true - - if app.relayLog, err = replication.NewRelayLogWithConfig(&cfg.RelayLog); err != nil { - return nil, err - } } if app.ldb, err = ledis.OpenWithConfig(&cfg.DB); err != nil { diff --git a/server/config.go b/server/config.go index 518a083..80214c3 100644 --- a/server/config.go +++ b/server/config.go @@ -3,7 +3,6 @@ package server import ( "encoding/json" "github.com/siddontang/ledisdb/ledis" - "github.com/siddontang/ledisdb/replication" "io/ioutil" ) @@ -18,9 +17,6 @@ type Config struct { //set slaveof to enable replication from master //empty, no replication SlaveOf string `json:"slaveof"` - - //if you not set relay log path, use data_dir/realy_log - RelayLog replication.RelayLogConfig `json:"relay_log"` } func NewConfig(data json.RawMessage) (*Config, error) { From 993ccdd052b53c8b9766607e65f7aca3dfad5cf9 Mon Sep 17 00:00:00 2001 From: siddontang Date: Sun, 8 Jun 2014 16:43:59 +0800 Subject: [PATCH 25/31] add flushall, some replication --- ledis/ledis.go | 11 ++++++ server/app.go | 13 ++----- server/cmd_replication.go | 27 ++++++++++--- server/replication.go | 82 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 117 insertions(+), 16 deletions(-) diff --git a/ledis/ledis.go b/ledis/ledis.go index 0cf108a..3afa0f5 100644 --- a/ledis/ledis.go +++ b/ledis/ledis.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "github.com/siddontang/go-leveldb/leveldb" + "github.com/siddontang/go-log/log" "path" "sync" ) @@ -132,3 +133,13 @@ func (l *Ledis) Select(index int) (*DB, error) { return l.dbs[index], nil } + +func (l *Ledis) FlushAll() error { + for index, db := range l.dbs { + if _, err := db.FlushAll(); err != nil { + log.Error("flush db %d error %s", index, err.Error()) + } + } + + return nil +} diff --git a/server/app.go b/server/app.go index cf76049..1b745b6 100644 --- a/server/app.go +++ b/server/app.go @@ -16,9 +16,10 @@ type App struct { closed bool - slaveMode bool - quit chan struct{} + + //for slave replication + master masterInfo } func NewApp(cfg *Config) (*App, error) { @@ -50,12 +51,6 @@ func NewApp(cfg *Config) (*App, error) { return nil, err } - app.slaveMode = false - - if len(app.cfg.SlaveOf) > 0 { - app.slaveMode = true - } - if app.ldb, err = ledis.OpenWithConfig(&cfg.DB); err != nil { return nil, err } @@ -78,7 +73,7 @@ func (app *App) Close() { } func (app *App) Run() { - if app.slaveMode { + if len(app.cfg.SlaveOf) > 0 { app.runReplication() } diff --git a/server/cmd_replication.go b/server/cmd_replication.go index 634d57f..8588877 100644 --- a/server/cmd_replication.go +++ b/server/cmd_replication.go @@ -1,16 +1,33 @@ package server +import ( + "fmt" + "github.com/siddontang/ledisdb/ledis" + "strconv" + "strings" +) + func slaveofCommand(c *client) error { - if len(c.args) > 1 { + args := c.args + + if len(args) != 2 { return ErrCmdParams } - master := "" - if len(c.args) == 1 { - master = string(c.args[0]) + masterAddr := "" + + if strings.ToLower(ledis.String(args[0])) == "no" && + strings.ToLower(ledis.String(args[1])) == "one" { + //stop replication, use master = "" + } else { + if _, err := strconv.ParseInt(ledis.String(args[1]), 10, 16); err != nil { + return err + } + + masterAddr = fmt.Sprintf("%s:%s", args[0], args[1]) } - if err := c.app.slaveof(master); err != nil { + if err := c.app.slaveof(masterAddr); err != nil { return err } diff --git a/server/replication.go b/server/replication.go index d1a3749..81ba85e 100644 --- a/server/replication.go +++ b/server/replication.go @@ -1,9 +1,87 @@ package server -func (app *App) slaveof(master string) error { +import ( + "encoding/json" + "github.com/siddontang/go-log/log" + "io/ioutil" + "os" + "path" +) + +type masterInfo struct { + Addr string `json:"addr"` + LogFile string `json:"log_name"` + LogPos int64 `json:"log_pos"` +} + +func (app *App) getMasterInfoName() string { + return path.Join(app.cfg.DataDir, "master.info") +} + +func (app *App) loadMasterInfo() error { + data, err := ioutil.ReadFile(app.getMasterInfoName()) + if err != nil { + if os.IsNotExist(err) { + return nil + } else { + return err + } + } + + if err = json.Unmarshal(data, &app.master); err != nil { + return err + } + + return nil +} + +func (app *App) saveMasterInfo() error { + bakName := path.Join(app.cfg.DataDir, "master.info.bak") + + data, err := json.Marshal(&app.master) + if err != nil { + return err + } + + var fd *os.File + fd, err = os.OpenFile(bakName, os.O_CREATE|os.O_WRONLY, os.ModePerm) + if err != nil { + return err + } + + if _, err = fd.Write(data); err != nil { + fd.Close() + return err + } + + fd.Close() + return os.Rename(bakName, app.getMasterInfoName()) +} + +func (app *App) slaveof(masterAddr string) error { + if len(masterAddr) == 0 { + //stop replication + } else { + } + return nil } func (app *App) runReplication() { - +} + +func (app *App) startReplication(masterAddr string) error { + if err := app.loadMasterInfo(); err != nil { + log.Error("load master.info error %s, use fullsync", err.Error()) + app.master = masterInfo{masterAddr, "", 0} + } else if app.master.Addr != masterAddr { + if err := app.ldb.FlushAll(); err != nil { + log.Error("replication flush old data error %s", err.Error()) + return err + } + + app.master = masterInfo{masterAddr, "", 0} + } + + return nil } From ef22d7000d563decfe8212396cdd4950c71ca7ce Mon Sep 17 00:00:00 2001 From: siddontang Date: Mon, 9 Jun 2014 17:23:32 +0800 Subject: [PATCH 26/31] add replication, test later --- ledis/binlog.go | 29 ++-- ledis/binlog_test.go | 1 - ledis/dump.go | 78 +++++----- ledis/replication.go | 180 +++++++++++++++++------ ledis/replication_test.go | 9 +- server/app.go | 8 +- server/client.go | 45 +++--- server/cmd_replication.go | 77 ++++++++++ server/replication.go | 296 +++++++++++++++++++++++++++++++++----- server/util.go | 57 ++++++++ 10 files changed, 621 insertions(+), 159 deletions(-) create mode 100644 server/util.go diff --git a/ledis/binlog.go b/ledis/binlog.go index c5df748..c7f534c 100644 --- a/ledis/binlog.go +++ b/ledis/binlog.go @@ -35,7 +35,6 @@ timestamp(bigendian uint32, seconds)|PayloadLen(bigendian uint32)|PayloadData */ type BinLogConfig struct { - Name string `json:"name"` Path string `json:"path"` MaxFileSize int `json:"max_file_size"` MaxFileNum int `json:"max_file_num"` @@ -53,10 +52,6 @@ func (cfg *BinLogConfig) adjust() { } else if cfg.MaxFileNum > MaxBinLogFileNum { cfg.MaxFileNum = MaxBinLogFileNum } - - if len(cfg.Name) == 0 { - cfg.Name = "ledis" - } } type BinLog struct { @@ -68,7 +63,7 @@ type BinLog struct { indexName string logNames []string - lastLogIndex int + lastLogIndex int64 } func NewBinLog(data json.RawMessage) (*BinLog, error) { @@ -128,7 +123,7 @@ func (l *BinLog) flushIndex() error { } func (l *BinLog) loadIndex() error { - l.indexName = path.Join(l.cfg.Path, fmt.Sprintf("%s-bin.index", l.cfg.Name)) + l.indexName = path.Join(l.cfg.Path, fmt.Sprintf("ledis-bin.index")) if _, err := os.Stat(l.indexName); os.IsNotExist(err) { //no index file, nothing to do } else { @@ -165,7 +160,7 @@ func (l *BinLog) loadIndex() error { } else { lastName := l.logNames[len(l.logNames)-1] - if l.lastLogIndex, err = strconv.Atoi(path.Ext(lastName)[1:]); err != nil { + if l.lastLogIndex, err = strconv.ParseInt(path.Ext(lastName)[1:], 10, 64); err != nil { log.Error("invalid logfile name %s", err.Error()) return err } @@ -178,7 +173,7 @@ func (l *BinLog) loadIndex() error { } func (l *BinLog) getLogFile() string { - return fmt.Sprintf("%s-bin.%07d", l.cfg.Name, l.lastLogIndex) + return l.FormatLogFileName(l.lastLogIndex) } func (l *BinLog) openNewLogFile() error { @@ -261,6 +256,22 @@ func (l *BinLog) LogFilePos() int64 { } } +func (l *BinLog) LogFileIndex() int64 { + return l.lastLogIndex +} + +func (l *BinLog) FormatLogFileName(index int64) string { + return fmt.Sprintf("ledis-bin.%07d", index) +} + +func (l *BinLog) FormatLogFilePath(index int64) string { + return path.Join(l.cfg.Path, fmt.Sprintf("ledis-bin.%07d", index)) +} + +func (l *BinLog) LogPath() string { + return l.cfg.Path +} + func (l *BinLog) Purge(n int) error { if len(l.logNames) == 0 { return nil diff --git a/ledis/binlog_test.go b/ledis/binlog_test.go index 995a8d1..7fc89b4 100644 --- a/ledis/binlog_test.go +++ b/ledis/binlog_test.go @@ -12,7 +12,6 @@ func TestBinLog(t *testing.T) { cfg.MaxFileNum = 1 cfg.MaxFileSize = 1024 cfg.Path = "/tmp/ledis_binlog" - cfg.Name = "ledis" os.RemoveAll(cfg.Path) diff --git a/ledis/dump.go b/ledis/dump.go index 2cb08af..47bca19 100644 --- a/ledis/dump.go +++ b/ledis/dump.go @@ -4,19 +4,43 @@ import ( "bufio" "bytes" "encoding/binary" - "encoding/json" "github.com/siddontang/go-leveldb/leveldb" "io" "os" ) //dump format -// head len(bigendian int32)|head(json format) +// fileIndex(bigendian int64)|filePos(bigendian int64) // |keylen(bigendian int32)|key|valuelen(bigendian int32)|value...... -type DumpHead struct { - LogFile string `json:"log_file"` - LogPos int64 `json:"log_pos"` +type MasterInfo struct { + LogFileIndex int64 + LogPos int64 +} + +func (m *MasterInfo) WriteTo(w io.Writer) error { + if err := binary.Write(w, binary.BigEndian, m.LogFileIndex); err != nil { + return err + } + + if err := binary.Write(w, binary.BigEndian, m.LogPos); err != nil { + return err + } + return nil +} + +func (m *MasterInfo) ReadFrom(r io.Reader) error { + err := binary.Read(r, binary.BigEndian, &m.LogFileIndex) + if err != nil { + return err + } + + err = binary.Read(r, binary.BigEndian, &m.LogPos) + if err != nil { + return err + } + + return nil } func (l *Ledis) DumpFile(path string) error { @@ -31,34 +55,21 @@ func (l *Ledis) DumpFile(path string) error { func (l *Ledis) Dump(w io.Writer) error { var sp *leveldb.Snapshot - var logFileName string - var logPos int64 + var m *MasterInfo = new(MasterInfo) if l.binlog == nil { sp = l.ldb.NewSnapshot() } else { l.Lock() sp = l.ldb.NewSnapshot() - logFileName = l.binlog.LogFileName() - logPos = l.binlog.LogFilePos() + m.LogFileIndex = l.binlog.LogFileIndex() + m.LogPos = l.binlog.LogFilePos() l.Unlock() } - var head = DumpHead{ - LogFile: logFileName, - LogPos: logPos, - } - - data, err := json.Marshal(&head) - if err != nil { - return err - } + var err error wb := bufio.NewWriterSize(w, 4096) - if err = binary.Write(wb, binary.BigEndian, uint32(len(data))); err != nil { - return err - } - - if _, err = wb.Write(data); err != nil { + if err = m.WriteTo(wb); err != nil { return err } @@ -93,7 +104,7 @@ func (l *Ledis) Dump(w io.Writer) error { return nil } -func (l *Ledis) LoadDumpFile(path string) (*DumpHead, error) { +func (l *Ledis) LoadDumpFile(path string) (*MasterInfo, error) { f, err := os.Open(path) if err != nil { return nil, err @@ -103,28 +114,19 @@ func (l *Ledis) LoadDumpFile(path string) (*DumpHead, error) { return l.LoadDump(f) } -func (l *Ledis) LoadDump(r io.Reader) (*DumpHead, error) { +func (l *Ledis) LoadDump(r io.Reader) (*MasterInfo, error) { l.Lock() defer l.Unlock() + info := new(MasterInfo) + rb := bufio.NewReaderSize(r, 4096) - var headLen uint32 - err := binary.Read(rb, binary.BigEndian, &headLen) + err := info.ReadFrom(rb) if err != nil { return nil, err } - buf := make([]byte, headLen) - if _, err = io.ReadFull(rb, buf); err != nil { - return nil, err - } - - var head DumpHead - if err = json.Unmarshal(buf, &head); err != nil { - return nil, err - } - var keyLen uint16 var valueLen uint32 @@ -161,5 +163,5 @@ func (l *Ledis) LoadDump(r io.Reader) (*DumpHead, error) { valueBuf.Reset() } - return &head, nil + return info, nil } diff --git a/ledis/replication.go b/ledis/replication.go index 9a2cb1b..045ee01 100644 --- a/ledis/replication.go +++ b/ledis/replication.go @@ -12,6 +12,7 @@ import ( var ( errInvalidBinLogEvent = errors.New("invalid binglog event") + errInvalidBinLogFile = errors.New("invalid binlog file") ) func (l *Ledis) ReplicateEvent(event []byte) error { @@ -70,63 +71,154 @@ func (l *Ledis) replicateCommandEvent(event []byte) error { return errors.New("command event not supported now") } -func (l *Ledis) RepliateFromBinLog(filePath string, offset int64) (int64, error) { - f, err := os.Open(filePath) - if err != nil { - return 0, err +func (l *Ledis) ReplicateFromReader(rb io.Reader) error { + var createTime uint32 + var dataLen uint32 + var dataBuf bytes.Buffer + var err error + + for { + if err = binary.Read(rb, binary.BigEndian, &createTime); err != nil { + if err == io.EOF { + break + } else { + return err + } + } + + if err = binary.Read(rb, binary.BigEndian, &dataLen); err != nil { + return err + } + + if _, err = io.CopyN(&dataBuf, rb, int64(dataLen)); err != nil { + return err + } + + err = l.ReplicateEvent(dataBuf.Bytes()) + if err != nil { + log.Fatal("replication error %s, skip to next", err.Error()) + } + + dataBuf.Reset() } - defer f.Close() + return nil +} - st, _ := f.Stat() - totalSize := st.Size() +func (l *Ledis) ReplicateFromData(data []byte) error { + rb := bytes.NewReader(data) - if _, err = f.Seek(offset, os.SEEK_SET); err != nil { - return 0, err + l.Lock() + err := l.ReplicateFromReader(rb) + l.Unlock() + + return err +} + +func (l *Ledis) ReplicateFromBinLog(filePath string) error { + f, err := os.Open(filePath) + if err != nil { + return err } rb := bufio.NewReaderSize(f, 4096) - var createTime uint32 - var dataLen uint32 - var dataBuf bytes.Buffer + err = l.ReplicateFromReader(rb) - for { - if offset+8 > totalSize { - //event may not sync completely - return f.Seek(offset, os.SEEK_SET) - } + f.Close() - if err = binary.Read(rb, binary.BigEndian, &createTime); err != nil { - return 0, err - } + return err +} - if err = binary.Read(rb, binary.BigEndian, &dataLen); err != nil { - return 0, err - } +const maxSyncEvents = 16 - if offset+8+int64(dataLen) > totalSize { - //event may not sync completely - return f.Seek(offset, os.SEEK_SET) - } else { - if _, err = io.CopyN(&dataBuf, rb, int64(dataLen)); err != nil { - return 0, err - } +//events data format +// nextfileIndex(bigendian int64)|nextfilePos(bigendian int64)|binlogevents +func (l *Ledis) ReadEventsTo(index int64, offset int64, w io.Writer) (info *MasterInfo, err error) { + info = new(MasterInfo) - l.Lock() - err = l.ReplicateEvent(dataBuf.Bytes()) - l.Unlock() - if err != nil { - log.Fatal("replication error %s, skip to next", err.Error()) - } - - dataBuf.Reset() - - offset += (8 + int64(dataLen)) - } + if l.binlog == nil { + //binlog not supported + info.LogFileIndex = 0 + return } - //can not go here??? - log.Error("can not go here") - return offset, nil + info.LogFileIndex = index + info.LogPos = offset + + filePath := l.binlog.FormatLogFilePath(index) + + var f *os.File + f, err = os.Open(filePath) + if err != nil && !os.IsNotExist(err) { + return + } else if os.IsNotExist(err) { + l.Lock() + lastIndex := l.binlog.LogFileIndex() + if index == lastIndex { + //no binlog at all + l.Unlock() + return + } + l.Unlock() + + //slave binlog info had lost + info.LogFileIndex = -1 + } + + defer f.Close() + + if _, err = f.Seek(offset, os.SEEK_SET); err != nil { + //may be invliad seek offset + return + } + + var lastCreateTime uint32 = 0 + var createTime uint32 + var dataLen uint32 + + var n int = 0 + + for { + if err = binary.Read(f, binary.BigEndian, &createTime); err != nil { + if err == io.EOF { + //we will try to use next binlog + info.LogFileIndex = index + 1 + info.LogPos = 0 + + return + } else { + return + } + } + + n++ + if lastCreateTime == 0 { + lastCreateTime = createTime + } else if lastCreateTime != createTime { + return + } else if n > maxSyncEvents { + return + } + + if err = binary.Read(f, binary.BigEndian, &dataLen); err != nil { + return + } + + if err = binary.Write(w, binary.BigEndian, createTime); err != nil { + return + } + + if err = binary.Write(w, binary.BigEndian, dataLen); err != nil { + return + } + + if _, err = io.CopyN(w, f, int64(dataLen)); err != nil { + return + } + + info.LogPos = info.LogPos + 8 + int64(dataLen) + } + + return } diff --git a/ledis/replication_test.go b/ledis/replication_test.go index 2e4b977..f722bc0 100644 --- a/ledis/replication_test.go +++ b/ledis/replication_test.go @@ -40,16 +40,9 @@ func TestReplication(t *testing.T) { binLogName := "/tmp/test_repl/master/bin_log/ledis-bin.0000001" - var offset int64 - offset, err = slave.RepliateFromBinLog(binLogName, 0) + err = slave.ReplicateFromBinLog(binLogName) if err != nil { t.Fatal(err) - } else { - if st, err := os.Stat(binLogName); err != nil { - t.Fatal(err) - } else if st.Size() != offset { - t.Fatal(st.Size(), offset) - } } it := master.ldb.Iterator(nil, nil, leveldb.RangeClose, 0, -1) diff --git a/server/app.go b/server/app.go index 1b745b6..3b28e7d 100644 --- a/server/app.go +++ b/server/app.go @@ -19,7 +19,7 @@ type App struct { quit chan struct{} //for slave replication - master masterInfo + m *master } func NewApp(cfg *Config) (*App, error) { @@ -55,6 +55,8 @@ func NewApp(cfg *Config) (*App, error) { return nil, err } + app.m = newMaster(app) + return app, nil } @@ -69,12 +71,14 @@ func (app *App) Close() { app.listener.Close() + app.m.Close() + app.ldb.Close() } func (app *App) Run() { if len(app.cfg.SlaveOf) > 0 { - app.runReplication() + app.slaveof(app.cfg.SlaveOf) } for !app.closed { diff --git a/server/client.go b/server/client.go index 40533b0..8a7ed6e 100644 --- a/server/client.go +++ b/server/client.go @@ -2,6 +2,7 @@ package server import ( "bufio" + "bytes" "errors" "github.com/siddontang/go-log/log" "github.com/siddontang/ledisdb/ledis" @@ -28,6 +29,8 @@ type client struct { args [][]byte reqC chan error + + syncBuf bytes.Buffer } func newClient(c net.Conn, app *App) { @@ -71,22 +74,7 @@ func (c *client) run() { } func (c *client) readLine() ([]byte, error) { - var line []byte - for { - l, more, err := c.rb.ReadLine() - if err != nil { - return nil, err - } - - if line == nil && !more { - return l, nil - } - line = append(line, l...) - if !more { - break - } - } - return line, nil + return readLine(c.rb) } //A client sends to the Redis server a RESP Array consisting of just Bulk Strings. @@ -121,15 +109,19 @@ func (c *client) readRequest() ([][]byte, error) { } else if n == -1 { req = append(req, nil) } else { - buf := make([]byte, n+2) + buf := make([]byte, n) if _, err = io.ReadFull(c.rb, buf); err != nil { return nil, err - } else if buf[len(buf)-2] != '\r' || buf[len(buf)-1] != '\n' { - return nil, errReadRequest - - } else { - req = append(req, buf[0:len(buf)-2]) } + + if l, err = c.readLine(); err != nil { + return nil, err + } else if len(l) != 0 { + return nil, errors.New("bad bulk string format") + } + + req = append(req, buf) + } } else { @@ -226,3 +218,12 @@ func (c *client) writeArray(ay []interface{}) { } } } + +func (c *client) writeBulkFrom(n int64, rb io.Reader) { + c.wb.WriteByte('$') + c.wb.Write(ledis.Slice(strconv.FormatInt(n, 10))) + c.wb.Write(Delims) + + io.Copy(c.wb, rb) + c.wb.Write(Delims) +} diff --git a/server/cmd_replication.go b/server/cmd_replication.go index 8588877..7e1c881 100644 --- a/server/cmd_replication.go +++ b/server/cmd_replication.go @@ -1,8 +1,11 @@ package server import ( + "encoding/binary" "fmt" "github.com/siddontang/ledisdb/ledis" + "io/ioutil" + "os" "strconv" "strings" ) @@ -35,3 +38,77 @@ func slaveofCommand(c *client) error { return nil } + +func fullsyncCommand(c *client) error { + //todo, multi fullsync may use same dump file + dumpFile, err := ioutil.TempFile(c.app.cfg.DataDir, "dump_") + if err != nil { + return err + } + + if err = c.app.ldb.Dump(dumpFile); err != nil { + return err + } + + st, _ := dumpFile.Stat() + n := st.Size() + + dumpFile.Seek(0, os.SEEK_SET) + + c.writeBulkFrom(n, dumpFile) + + name := dumpFile.Name() + dumpFile.Close() + + os.Remove(name) + + return nil +} + +var reserveInfoSpace = make([]byte, 16) + +func syncCommand(c *client) error { + args := c.args + if len(args) != 2 { + return ErrCmdParams + } + + var logIndex int64 + var logPos int64 + var err error + logIndex, err = ledis.StrInt64(args[0], nil) + if err != nil { + return ErrCmdParams + } + + logPos, err = ledis.StrInt64(args[1], nil) + if err != nil { + return ErrCmdParams + } + + c.syncBuf.Reset() + + //reserve space to write master info + if _, err := c.syncBuf.Write(reserveInfoSpace); err != nil { + return err + } + + if m, err := c.app.ldb.ReadEventsTo(logIndex, logPos, &c.syncBuf); err != nil { + return err + } else { + buf := c.syncBuf.Bytes() + + binary.BigEndian.PutUint64(buf[0:], uint64(m.LogFileIndex)) + binary.BigEndian.PutUint64(buf[8:], uint64(m.LogPos)) + + c.writeBulk(buf) + } + + return nil +} + +func init() { + register("slaveof", slaveofCommand) + register("fullsync", fullsyncCommand) + register("sync", syncCommand) +} diff --git a/server/replication.go b/server/replication.go index 81ba85e..bc64496 100644 --- a/server/replication.go +++ b/server/replication.go @@ -1,25 +1,77 @@ package server import ( + "bufio" + "bytes" + "encoding/binary" "encoding/json" + "errors" + "fmt" "github.com/siddontang/go-log/log" + "github.com/siddontang/ledisdb/ledis" "io/ioutil" + "net" "os" "path" + "strconv" + "sync" + "time" ) -type masterInfo struct { - Addr string `json:"addr"` - LogFile string `json:"log_name"` - LogPos int64 `json:"log_pos"` +var ( + errConnectMaster = errors.New("connect master error") +) + +type master struct { + sync.Mutex + + addr string `json:"addr"` + logFileIndex int64 `json:"log_file_index"` + logPos int64 `json:"log_pos"` + + c net.Conn + rb *bufio.Reader + + app *App + + quit chan struct{} + + infoName string + infoNameBak string + + wg sync.WaitGroup + + syncBuf bytes.Buffer } -func (app *App) getMasterInfoName() string { - return path.Join(app.cfg.DataDir, "master.info") +func newMaster(app *App) *master { + m := new(master) + m.app = app + + m.infoName = path.Join(m.app.cfg.DataDir, "master.info") + m.infoNameBak = fmt.Sprintf("%s.bak", m.infoName) + + m.quit = make(chan struct{}) + + //if load error, we will start a fullsync later + m.loadInfo() + + return m } -func (app *App) loadMasterInfo() error { - data, err := ioutil.ReadFile(app.getMasterInfoName()) +func (m *master) Close() { + close(m.quit) + + if m.c != nil { + m.c.Close() + m.c = nil + } + + m.wg.Wait() +} + +func (m *master) loadInfo() error { + data, err := ioutil.ReadFile(m.infoName) if err != nil { if os.IsNotExist(err) { return nil @@ -28,23 +80,21 @@ func (app *App) loadMasterInfo() error { } } - if err = json.Unmarshal(data, &app.master); err != nil { + if err = json.Unmarshal(data, m); err != nil { return err } return nil } -func (app *App) saveMasterInfo() error { - bakName := path.Join(app.cfg.DataDir, "master.info.bak") - - data, err := json.Marshal(&app.master) +func (m *master) saveInfo() error { + data, err := json.Marshal(m) if err != nil { return err } var fd *os.File - fd, err = os.OpenFile(bakName, os.O_CREATE|os.O_WRONLY, os.ModePerm) + fd, err = os.OpenFile(m.infoNameBak, os.O_CREATE|os.O_WRONLY, os.ModePerm) if err != nil { return err } @@ -55,32 +105,208 @@ func (app *App) saveMasterInfo() error { } fd.Close() - return os.Rename(bakName, app.getMasterInfoName()) + return os.Rename(m.infoNameBak, m.infoName) +} + +func (m *master) connect() error { + if len(m.addr) == 0 { + return fmt.Errorf("no assign master addr") + } + + if m.c != nil { + m.c.Close() + m.c = nil + } + + if c, err := net.Dial("tcp", m.addr); err != nil { + return err + } else { + m.c = c + + m.rb = bufio.NewReaderSize(m.c, 4096) + } + return nil +} + +func (m *master) resetInfo(addr string) { + m.addr = addr + m.logFileIndex = 0 + m.logPos = 0 +} + +func (m *master) stopReplication() error { + m.Close() + + if err := m.saveInfo(); err != nil { + log.Error("save master info error %s", err.Error()) + return err + } + + return nil +} + +func (m *master) startReplication(masterAddr string) error { + //stop last replcation, if avaliable + m.Close() + + if masterAddr != m.addr { + m.resetInfo(masterAddr) + if err := m.saveInfo(); err != nil { + log.Error("save master info error %s", err.Error()) + return err + } + } + + m.quit = make(chan struct{}) + + go m.runReplication() + return nil +} + +func (m *master) runReplication() { + m.wg.Add(1) + defer m.wg.Done() + + for { + select { + case <-m.quit: + return + default: + if err := m.connect(); err != nil { + log.Error("connect master %s error %s, try 2s later", m.addr, err.Error()) + time.Sleep(2 * time.Second) + continue + } + } + + if m.logFileIndex == 0 { + //try a fullsync + if err := m.fullSync(); err != nil { + log.Warn("full sync error %s", err.Error()) + return + } + + if m.logFileIndex == 0 { + //master not support binlog, we cannot sync, so stop replication + m.stopReplication() + return + } + } + + t := time.NewTicker(1 * time.Second) + + //then we will try sync every 1 seconds + for { + select { + case <-t.C: + if err := m.sync(); err != nil { + log.Warn("sync error %s", err.Error()) + return + } + case <-m.quit: + return + } + } + } + + return +} + +var ( + fullSyncCmd = []byte("*1\r\n$8\r\nfullsync\r\n") //fullsync + syncCmdFormat = "*3\r\n$4\r\nsync\r\n$%d\r\n%s\r\n%d\r\n%s\r\n" //sync file pos +) + +func (m *master) fullSync() error { + if _, err := m.c.Write(fullSyncCmd); err != nil { + return err + } + + dumpPath := path.Join(m.app.cfg.DataDir, "master.dump") + f, err := os.OpenFile(dumpPath, os.O_CREATE|os.O_WRONLY, os.ModePerm) + if err != nil { + return err + } + + defer os.Remove(dumpPath) + + err = readBulkTo(m.rb, f) + f.Close() + if err != nil { + log.Error("read dump data error %s", err.Error()) + return err + } + + if err = m.app.ldb.FlushAll(); err != nil { + return err + } + + var head *ledis.MasterInfo + head, err = m.app.ldb.LoadDumpFile(dumpPath) + + if err != nil { + log.Error("load dump file error %s", err.Error()) + return err + } + + m.logFileIndex = head.LogFileIndex + m.logPos = head.LogPos + + return nil +} + +func (m *master) sync() error { + logIndexStr := strconv.FormatInt(m.logFileIndex, 10) + logPosStr := strconv.FormatInt(m.logPos, 10) + + if _, err := m.c.Write(ledis.Slice(fmt.Sprintf(syncCmdFormat, len(logIndexStr), + logIndexStr, len(logPosStr), logPosStr))); err != nil { + return err + } + + m.syncBuf.Reset() + + err := readBulkTo(m.rb, &m.syncBuf) + if err != nil { + return err + } + + err = binary.Read(&m.syncBuf, binary.BigEndian, &m.logFileIndex) + if err != nil { + return err + } + + err = binary.Read(&m.syncBuf, binary.BigEndian, &m.logPos) + if err != nil { + return err + } + + if m.logFileIndex == 0 { + //master now not support binlog, stop replication + m.stopReplication() + return nil + } else if m.logFileIndex == -1 { + //-1 means than binlog index and pos are lost, we must start a full sync instead + return m.fullSync() + } + + err = m.app.ldb.ReplicateFromReader(&m.syncBuf) + if err != nil { + return err + } + + return nil + } func (app *App) slaveof(masterAddr string) error { + app.m.Lock() + defer app.m.Unlock() + if len(masterAddr) == 0 { - //stop replication + return app.m.stopReplication() } else { - } - - return nil -} - -func (app *App) runReplication() { -} - -func (app *App) startReplication(masterAddr string) error { - if err := app.loadMasterInfo(); err != nil { - log.Error("load master.info error %s, use fullsync", err.Error()) - app.master = masterInfo{masterAddr, "", 0} - } else if app.master.Addr != masterAddr { - if err := app.ldb.FlushAll(); err != nil { - log.Error("replication flush old data error %s", err.Error()) - return err - } - - app.master = masterInfo{masterAddr, "", 0} + return app.m.startReplication(masterAddr) } return nil diff --git a/server/util.go b/server/util.go new file mode 100644 index 0000000..e2a6738 --- /dev/null +++ b/server/util.go @@ -0,0 +1,57 @@ +package server + +import ( + "bufio" + "errors" + "github.com/siddontang/ledisdb/ledis" + "io" + "strconv" +) + +var ( + errArrayFormat = errors.New("bad array format") + errBulkFormat = errors.New("bad bulk string format") + errLineFormat = errors.New("bad response line format") +) + +func readLine(rb *bufio.Reader) ([]byte, error) { + p, err := rb.ReadSlice('\n') + + if err != nil { + return nil, err + } + i := len(p) - 2 + if i < 0 || p[i] != '\r' { + return nil, errLineFormat + } + return p[:i], nil +} + +func readBulkTo(rb *bufio.Reader, w io.Writer) error { + l, err := readLine(rb) + if len(l) == 0 { + return errArrayFormat + } else if l[0] == '$' { + var n int + //handle resp string + if n, err = strconv.Atoi(ledis.String(l[1:])); err != nil { + return err + } else if n == -1 { + return nil + } else { + if _, err = io.CopyN(w, rb, int64(n)); err != nil { + return err + } + + if l, err = readLine(rb); err != nil { + return err + } else if len(l) != 0 { + return errBulkFormat + } + } + } else { + return errArrayFormat + } + + return nil +} From ebd15e5738877dde49e70af8e0da62b2febd05e3 Mon Sep 17 00:00:00 2001 From: siddontang Date: Tue, 10 Jun 2014 10:41:50 +0800 Subject: [PATCH 27/31] add replication test --- ledis/binlog.go | 2 +- ledis/ledis.go | 5 ++ ledis/replication.go | 58 +++++++++++------- ledis/replication_test.go | 90 ++++++++++++++++++++++----- server/cmd_replication.go | 4 +- server/cmd_replication_test.go | 107 +++++++++++++++++++++++++++++++++ server/replication.go | 36 +++++++---- server/util.go | 4 +- 8 files changed, 251 insertions(+), 55 deletions(-) create mode 100644 server/cmd_replication_test.go diff --git a/ledis/binlog.go b/ledis/binlog.go index c7f534c..d6e99f0 100644 --- a/ledis/binlog.go +++ b/ledis/binlog.go @@ -265,7 +265,7 @@ func (l *BinLog) FormatLogFileName(index int64) string { } func (l *BinLog) FormatLogFilePath(index int64) string { - return path.Join(l.cfg.Path, fmt.Sprintf("ledis-bin.%07d", index)) + return path.Join(l.cfg.Path, l.FormatLogFileName(index)) } func (l *BinLog) LogPath() string { diff --git a/ledis/ledis.go b/ledis/ledis.go index 3afa0f5..193832a 100644 --- a/ledis/ledis.go +++ b/ledis/ledis.go @@ -143,3 +143,8 @@ func (l *Ledis) FlushAll() error { return nil } + +//very dangerous to use +func (l *Ledis) DataDB() *leveldb.DB { + return l.ldb +} diff --git a/ledis/replication.go b/ledis/replication.go index 045ee01..7cfc75d 100644 --- a/ledis/replication.go +++ b/ledis/replication.go @@ -130,44 +130,53 @@ func (l *Ledis) ReplicateFromBinLog(filePath string) error { return err } -const maxSyncEvents = 16 - -//events data format -// nextfileIndex(bigendian int64)|nextfilePos(bigendian int64)|binlogevents -func (l *Ledis) ReadEventsTo(index int64, offset int64, w io.Writer) (info *MasterInfo, err error) { - info = new(MasterInfo) +const maxSyncEvents = 64 +func (l *Ledis) ReadEventsTo(info *MasterInfo, w io.Writer) (n int, err error) { + n = 0 if l.binlog == nil { //binlog not supported info.LogFileIndex = 0 + info.LogPos = 0 return } - info.LogFileIndex = index - info.LogPos = offset + index := info.LogFileIndex + offset := info.LogPos filePath := l.binlog.FormatLogFilePath(index) var f *os.File f, err = os.Open(filePath) - if err != nil && !os.IsNotExist(err) { - return - } else if os.IsNotExist(err) { - l.Lock() + if os.IsNotExist(err) { lastIndex := l.binlog.LogFileIndex() + if index == lastIndex { //no binlog at all - l.Unlock() - return + info.LogPos = 0 + } else { + //slave binlog info had lost + info.LogFileIndex = -1 } - l.Unlock() + } - //slave binlog info had lost - info.LogFileIndex = -1 + if err != nil { + if os.IsNotExist(err) { + err = nil + } + return } defer f.Close() + var fileSize int64 + st, _ := f.Stat() + fileSize = st.Size() + + if fileSize == info.LogPos { + return + } + if _, err = f.Seek(offset, os.SEEK_SET); err != nil { //may be invliad seek offset return @@ -177,27 +186,29 @@ func (l *Ledis) ReadEventsTo(index int64, offset int64, w io.Writer) (info *Mast var createTime uint32 var dataLen uint32 - var n int = 0 + var eventsNum int = 0 for { if err = binary.Read(f, binary.BigEndian, &createTime); err != nil { if err == io.EOF { //we will try to use next binlog - info.LogFileIndex = index + 1 - info.LogPos = 0 - + if index < l.binlog.LogFileIndex() { + info.LogFileIndex += 1 + info.LogPos = 0 + } + err = nil return } else { return } } - n++ + eventsNum++ if lastCreateTime == 0 { lastCreateTime = createTime } else if lastCreateTime != createTime { return - } else if n > maxSyncEvents { + } else if eventsNum > maxSyncEvents { return } @@ -217,6 +228,7 @@ func (l *Ledis) ReadEventsTo(index int64, offset int64, w io.Writer) (info *Mast return } + n += (8 + int(dataLen)) info.LogPos = info.LogPos + 8 + int64(dataLen) } diff --git a/ledis/replication_test.go b/ledis/replication_test.go index f722bc0..21d4dbc 100644 --- a/ledis/replication_test.go +++ b/ledis/replication_test.go @@ -2,23 +2,44 @@ package ledis import ( "bytes" + "fmt" "github.com/siddontang/go-leveldb/leveldb" "os" + "path" "testing" ) +func checkLedisEqual(master *Ledis, slave *Ledis) error { + it := master.ldb.Iterator(nil, nil, leveldb.RangeClose, 0, -1) + for ; it.Valid(); it.Next() { + key := it.Key() + value := it.Value() + + if v, err := slave.ldb.Get(key); err != nil { + return err + } else if !bytes.Equal(v, value) { + return fmt.Errorf("replication error %d != %d", len(v), len(value)) + } + } + + return nil +} + func TestReplication(t *testing.T) { var master *Ledis var slave *Ledis var err error - os.RemoveAll("/tmp/repl_repl") + os.RemoveAll("/tmp/test_repl") master, err = Open([]byte(` { "data_dir" : "/tmp/test_repl/master", - "use_bin_log" : true - } + "use_bin_log" : true, + "bin_log" : { + "max_file_size" : 50 + } + } `)) if err != nil { t.Fatal(err) @@ -34,26 +55,63 @@ func TestReplication(t *testing.T) { } db, _ := master.Select(0) - db.Set([]byte("a"), []byte("1")) - db.Set([]byte("b"), []byte("2")) - db.Set([]byte("c"), []byte("3")) + db.Set([]byte("a"), []byte("value")) + db.Set([]byte("b"), []byte("value")) + db.Set([]byte("c"), []byte("value")) - binLogName := "/tmp/test_repl/master/bin_log/ledis-bin.0000001" + db.HSet([]byte("a"), []byte("1"), []byte("value")) + db.HSet([]byte("b"), []byte("2"), []byte("value")) + db.HSet([]byte("c"), []byte("3"), []byte("value")) - err = slave.ReplicateFromBinLog(binLogName) - if err != nil { + for _, name := range master.binlog.LogNames() { + p := path.Join(master.binlog.cfg.Path, name) + + err = slave.ReplicateFromBinLog(p) + if err != nil { + t.Fatal(err) + } + } + + if err = checkLedisEqual(master, slave); err != nil { t.Fatal(err) } - it := master.ldb.Iterator(nil, nil, leveldb.RangeClose, 0, -1) - for ; it.Valid(); it.Next() { - key := it.Key() - value := it.Value() + slave.FlushAll() - if v, err := slave.ldb.Get(key); err != nil { + db.Set([]byte("a1"), []byte("1")) + db.Set([]byte("b1"), []byte("2")) + db.Set([]byte("c1"), []byte("3")) + + db.HSet([]byte("a1"), []byte("1"), []byte("value")) + db.HSet([]byte("b1"), []byte("2"), []byte("value")) + db.HSet([]byte("c1"), []byte("3"), []byte("value")) + + info := new(MasterInfo) + info.LogFileIndex = 1 + info.LogPos = 0 + var buf bytes.Buffer + var n int + + for { + buf.Reset() + n, err = master.ReadEventsTo(info, &buf) + if err != nil { t.Fatal(err) - } else if !bytes.Equal(v, value) { - t.Fatal("replication error", len(v), len(value)) + } else if info.LogFileIndex == -1 { + t.Fatal("invalid log file index -1") + } else if info.LogFileIndex == 0 { + t.Fatal("invalid log file index 0") + } else { + if err = slave.ReplicateFromReader(&buf); err != nil { + t.Fatal(err) + } + if n == 0 { + break + } } } + + if err = checkLedisEqual(master, slave); err != nil { + t.Fatal(err) + } } diff --git a/server/cmd_replication.go b/server/cmd_replication.go index 7e1c881..82803dd 100644 --- a/server/cmd_replication.go +++ b/server/cmd_replication.go @@ -93,7 +93,9 @@ func syncCommand(c *client) error { return err } - if m, err := c.app.ldb.ReadEventsTo(logIndex, logPos, &c.syncBuf); err != nil { + m := &ledis.MasterInfo{logIndex, logPos} + + if _, err := c.app.ldb.ReadEventsTo(m, &c.syncBuf); err != nil { return err } else { buf := c.syncBuf.Bytes() diff --git a/server/cmd_replication_test.go b/server/cmd_replication_test.go new file mode 100644 index 0000000..645f4f7 --- /dev/null +++ b/server/cmd_replication_test.go @@ -0,0 +1,107 @@ +package server + +import ( + "bytes" + "fmt" + "github.com/siddontang/go-leveldb/leveldb" + "os" + "testing" + "time" +) + +func checkDataEqual(master *App, slave *App) error { + it := master.ldb.DataDB().Iterator(nil, nil, leveldb.RangeClose, 0, -1) + for ; it.Valid(); it.Next() { + key := it.Key() + value := it.Value() + + if v, err := slave.ldb.DataDB().Get(key); err != nil { + return err + } else if !bytes.Equal(v, value) { + return fmt.Errorf("replication error %d != %d", len(v), len(value)) + } + } + + return nil +} + +func TestReplication(t *testing.T) { + data_dir := "/tmp/test_replication" + os.RemoveAll(data_dir) + + masterCfg := new(Config) + masterCfg.DataDir = fmt.Sprintf("%s/master", data_dir) + masterCfg.Addr = "127.0.0.1:11182" + masterCfg.DB.UseBinLog = true + + var master *App + var slave *App + var err error + master, err = NewApp(masterCfg) + if err != nil { + t.Fatal(err) + } + + slaveCfg := new(Config) + slaveCfg.DataDir = fmt.Sprintf("%s/slave", data_dir) + slaveCfg.Addr = "127.0.0.1:11183" + slaveCfg.SlaveOf = masterCfg.Addr + + slave, err = NewApp(slaveCfg) + if err != nil { + t.Fatal(err) + } + + go master.Run() + + db, _ := master.ldb.Select(0) + + value := make([]byte, 10) + + db.Set([]byte("a"), value) + db.Set([]byte("b"), value) + db.HSet([]byte("a"), []byte("1"), value) + db.HSet([]byte("b"), []byte("2"), value) + + go slave.Run() + + time.Sleep(1 * time.Second) + + if err = checkDataEqual(master, slave); err != nil { + t.Fatal(err) + } + + db.Set([]byte("a1"), value) + db.Set([]byte("b1"), value) + db.HSet([]byte("a1"), []byte("1"), value) + db.HSet([]byte("b1"), []byte("2"), value) + + time.Sleep(1 * time.Second) + if err = checkDataEqual(master, slave); err != nil { + t.Fatal(err) + } + + slave.slaveof("") + + db.Set([]byte("a2"), value) + db.Set([]byte("b2"), value) + db.HSet([]byte("a2"), []byte("1"), value) + db.HSet([]byte("b2"), []byte("2"), value) + + db.Set([]byte("a3"), value) + db.Set([]byte("b3"), value) + db.HSet([]byte("a3"), []byte("1"), value) + db.HSet([]byte("b3"), []byte("2"), value) + + if err = checkDataEqual(master, slave); err == nil { + t.Fatal("must error") + } + + slave.slaveof(masterCfg.Addr) + time.Sleep(1 * time.Second) + + if err = checkDataEqual(master, slave); err != nil { + t.Fatal(err) + } + +} diff --git a/server/replication.go b/server/replication.go index bc64496..2ee0af4 100644 --- a/server/replication.go +++ b/server/replication.go @@ -51,7 +51,7 @@ func newMaster(app *App) *master { m.infoName = path.Join(m.app.cfg.DataDir, "master.info") m.infoNameBak = fmt.Sprintf("%s.bak", m.infoName) - m.quit = make(chan struct{}) + m.quit = make(chan struct{}, 1) //if load error, we will start a fullsync later m.loadInfo() @@ -60,7 +60,10 @@ func newMaster(app *App) *master { } func (m *master) Close() { - close(m.quit) + select { + case m.quit <- struct{}{}: + default: + } if m.c != nil { m.c.Close() @@ -157,7 +160,7 @@ func (m *master) startReplication(masterAddr string) error { } } - m.quit = make(chan struct{}) + m.quit = make(chan struct{}, 1) go m.runReplication() return nil @@ -193,18 +196,26 @@ func (m *master) runReplication() { } } - t := time.NewTicker(1 * time.Second) - - //then we will try sync every 1 seconds for { - select { - case <-t.C: + for { + lastIndex := m.logFileIndex + lastPos := m.logPos if err := m.sync(); err != nil { log.Warn("sync error %s", err.Error()) return } + + if m.logFileIndex == lastIndex && m.logPos == lastPos { + //sync no data, wait 1s and retry + break + } + } + + select { case <-m.quit: return + case <-time.After(1 * time.Second): + break } } } @@ -213,8 +224,8 @@ func (m *master) runReplication() { } var ( - fullSyncCmd = []byte("*1\r\n$8\r\nfullsync\r\n") //fullsync - syncCmdFormat = "*3\r\n$4\r\nsync\r\n$%d\r\n%s\r\n%d\r\n%s\r\n" //sync file pos + fullSyncCmd = []byte("*1\r\n$8\r\nfullsync\r\n") //fullsync + syncCmdFormat = "*3\r\n$4\r\nsync\r\n$%d\r\n%s\r\n$%d\r\n%s\r\n" //sync index pos ) func (m *master) fullSync() error { @@ -259,8 +270,9 @@ func (m *master) sync() error { logIndexStr := strconv.FormatInt(m.logFileIndex, 10) logPosStr := strconv.FormatInt(m.logPos, 10) - if _, err := m.c.Write(ledis.Slice(fmt.Sprintf(syncCmdFormat, len(logIndexStr), - logIndexStr, len(logPosStr), logPosStr))); err != nil { + cmd := ledis.Slice(fmt.Sprintf(syncCmdFormat, len(logIndexStr), + logIndexStr, len(logPosStr), logPosStr)) + if _, err := m.c.Write(cmd); err != nil { return err } diff --git a/server/util.go b/server/util.go index e2a6738..9afee4e 100644 --- a/server/util.go +++ b/server/util.go @@ -30,7 +30,7 @@ func readLine(rb *bufio.Reader) ([]byte, error) { func readBulkTo(rb *bufio.Reader, w io.Writer) error { l, err := readLine(rb) if len(l) == 0 { - return errArrayFormat + return errBulkFormat } else if l[0] == '$' { var n int //handle resp string @@ -50,7 +50,7 @@ func readBulkTo(rb *bufio.Reader, w io.Writer) error { } } } else { - return errArrayFormat + return errBulkFormat } return nil From 00b5fb4b24359f0fa68e568e4773b99abee17b6a Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 11 Jun 2014 14:52:34 +0800 Subject: [PATCH 28/31] a liitle change --- cmd/ledis-server/main.go | 2 +- ledis/replication.go | 2 ++ server/app.go | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/ledis-server/main.go b/cmd/ledis-server/main.go index 8a76984..8513559 100644 --- a/cmd/ledis-server/main.go +++ b/cmd/ledis-server/main.go @@ -9,7 +9,7 @@ import ( "syscall" ) -var configFile = flag.String("config", "", "ledisdb config file") +var configFile = flag.String("config", "/etc/ledis.json", "ledisdb config file") func main() { runtime.GOMAXPROCS(runtime.NumCPU()) diff --git a/ledis/replication.go b/ledis/replication.go index 7cfc75d..e19da6a 100644 --- a/ledis/replication.go +++ b/ledis/replication.go @@ -123,7 +123,9 @@ func (l *Ledis) ReplicateFromBinLog(filePath string) error { rb := bufio.NewReaderSize(f, 4096) + l.Lock() err = l.ReplicateFromReader(rb) + l.Unlock() f.Close() diff --git a/server/app.go b/server/app.go index 3b28e7d..a1ba7f4 100644 --- a/server/app.go +++ b/server/app.go @@ -90,3 +90,7 @@ func (app *App) Run() { newClient(conn, app) } } + +func (app *App) Ledis() *ledis.DB { + return app.ldb +} From 655db076044050281411edffb50f098c50ac62e9 Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 11 Jun 2014 16:48:11 +0800 Subject: [PATCH 29/31] add access log --- etc/ledis.json | 4 +++- server/accesslog.go | 51 +++++++++++++++++++++++++++++++++++++++++++++ server/app.go | 21 ++++++++++++++++++- server/client.go | 9 ++++++++ server/config.go | 2 ++ 5 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 server/accesslog.go diff --git a/etc/ledis.json b/etc/ledis.json index dcf2ea5..8f230c3 100644 --- a/etc/ledis.json +++ b/etc/ledis.json @@ -8,5 +8,7 @@ "write_buffer_size": 67108864, "cache_size": 524288000 } - } + }, + + "access_log" : "access.log" } \ No newline at end of file diff --git a/server/accesslog.go b/server/accesslog.go new file mode 100644 index 0000000..58b34b7 --- /dev/null +++ b/server/accesslog.go @@ -0,0 +1,51 @@ +package server + +import ( + "fmt" + "github.com/siddontang/go-log/log" + "strings" +) + +const ( + accessTimeFormat = "2006/01/02 15:04:05" +) + +type accessLog struct { + l *log.Logger +} + +func newAcessLog(baseName string) (*accessLog, error) { + l := new(accessLog) + + h, err := log.NewTimeRotatingFileHandler(baseName, log.WhenDay, 1) + if err != nil { + return nil, err + } + + l.l = log.New(h, log.Ltime) + + return l, nil +} + +func (l *accessLog) Close() { + l.l.Close() +} + +func (l *accessLog) Log(remoteAddr string, usedTime int64, cmd string, args [][]byte, err error) { + argsFormat := make([]string, len(args)) + argsI := make([]interface{}, len(args)) + for i := range args { + argsFormat[i] = " %.24q" + argsI[i] = args[i] + } + + argsStr := fmt.Sprintf(strings.Join(argsFormat, ""), argsI...) + + format := `%s [%s%s] %d [%s]` + + if err == nil { + l.l.Info(format, remoteAddr, cmd, argsStr, usedTime, "OK") + } else { + l.l.Info(format, remoteAddr, cmd, argsStr, usedTime, err.Error()) + } +} diff --git a/server/app.go b/server/app.go index a1ba7f4..5a9bb87 100644 --- a/server/app.go +++ b/server/app.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/siddontang/ledisdb/ledis" "net" + "path" "strings" ) @@ -18,6 +19,8 @@ type App struct { quit chan struct{} + access *accessLog + //for slave replication m *master } @@ -51,6 +54,18 @@ func NewApp(cfg *Config) (*App, error) { return nil, err } + if len(cfg.AccessLog) > 0 { + if path.Dir(cfg.AccessLog) == "." { + app.access, err = newAcessLog(path.Join(cfg.DataDir, cfg.AccessLog)) + } else { + app.access, err = newAcessLog(cfg.AccessLog) + } + + if err != nil { + return nil, err + } + } + if app.ldb, err = ledis.OpenWithConfig(&cfg.DB); err != nil { return nil, err } @@ -73,6 +88,10 @@ func (app *App) Close() { app.m.Close() + if app.access != nil { + app.access.Close() + } + app.ldb.Close() } @@ -91,6 +110,6 @@ func (app *App) Run() { } } -func (app *App) Ledis() *ledis.DB { +func (app *App) Ledis() *ledis.Ledis { return app.ldb } diff --git a/server/client.go b/server/client.go index 8a7ed6e..ca3d72d 100644 --- a/server/client.go +++ b/server/client.go @@ -11,6 +11,7 @@ import ( "runtime" "strconv" "strings" + "time" ) var errReadRequest = errors.New("invalid request protocol") @@ -135,6 +136,8 @@ func (c *client) readRequest() ([][]byte, error) { func (c *client) handleRequest(req [][]byte) { var err error + start := time.Now() + if len(req) == 0 { err = ErrEmptyCommand } else { @@ -152,6 +155,12 @@ func (c *client) handleRequest(req [][]byte) { } } + duration := time.Since(start) + + if c.app.access != nil { + c.app.access.Log(c.c.RemoteAddr().String(), duration.Nanoseconds()/1000000, c.cmd, c.args, err) + } + if err != nil { c.writeError(err) } diff --git a/server/config.go b/server/config.go index 80214c3..89aba2c 100644 --- a/server/config.go +++ b/server/config.go @@ -17,6 +17,8 @@ type Config struct { //set slaveof to enable replication from master //empty, no replication SlaveOf string `json:"slaveof"` + + AccessLog string `json:"access_log"` } func NewConfig(data json.RawMessage) (*Config, error) { From 5a902fc9d1f17590ccc2a577d39cc02e9ebaf5dc Mon Sep 17 00:00:00 2001 From: siddontang Date: Wed, 11 Jun 2014 16:49:17 +0800 Subject: [PATCH 30/31] set args must be 2 --- server/cmd_kv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/cmd_kv.go b/server/cmd_kv.go index a6bdcbd..1c42cec 100644 --- a/server/cmd_kv.go +++ b/server/cmd_kv.go @@ -20,7 +20,7 @@ func getCommand(c *client) error { func setCommand(c *client) error { args := c.args - if len(args) < 2 { + if len(args) != 2 { return ErrCmdParams } From 52c55988de2cf453e36d2967e67e60e5a12a866b Mon Sep 17 00:00:00 2001 From: silentsai Date: Thu, 12 Jun 2014 10:51:36 +0800 Subject: [PATCH 31/31] unify all expire cycle activity into one routine; edit func name about flush --- ledis/ledis.go | 28 ++++++++++++++++++++++++++-- ledis/ledis_db.go | 28 ++++++---------------------- ledis/t_hash.go | 2 +- ledis/t_hash_test.go | 2 +- ledis/t_kv.go | 2 +- ledis/t_list.go | 2 +- ledis/t_zset.go | 2 +- ledis/t_zset_test.go | 2 +- 8 files changed, 38 insertions(+), 30 deletions(-) diff --git a/ledis/ledis.go b/ledis/ledis.go index 193832a..baf152b 100644 --- a/ledis/ledis.go +++ b/ledis/ledis.go @@ -7,6 +7,7 @@ import ( "github.com/siddontang/go-log/log" "path" "sync" + "time" ) type Config struct { @@ -93,6 +94,8 @@ func OpenWithConfig(cfg *Config) (*Ledis, error) { l.dbs[i] = newDB(l, i) } + l.activeExpireCycle() + return l, nil } @@ -110,8 +113,6 @@ func newDB(l *Ledis, index uint8) *DB { d.hashTx = newTx(l) d.zsetTx = newTx(l) - d.activeExpireCycle() - return d } @@ -148,3 +149,26 @@ func (l *Ledis) FlushAll() error { func (l *Ledis) DataDB() *leveldb.DB { return l.ldb } + +func (l *Ledis) activeExpireCycle() { + var executors []*elimination = make([]*elimination, len(l.dbs)) + for i, db := range l.dbs { + executors[i] = db.newEliminator() + } + + go func() { + tick := time.NewTicker(1 * time.Second) + for { + select { + case <-tick.C: + for _, eli := range executors { + eli.active() + } + case <-l.quit: + break + } + } + + tick.Stop() + }() +} diff --git a/ledis/ledis_db.go b/ledis/ledis_db.go index 2390464..0e76d17 100644 --- a/ledis/ledis_db.go +++ b/ledis/ledis_db.go @@ -1,15 +1,11 @@ package ledis -import ( - "time" -) - func (db *DB) FlushAll() (drop int64, err error) { all := [...](func() (int64, error)){ - db.Flush, - db.LFlush, - db.HFlush, - db.ZFlush} + db.flush, + db.lFlush, + db.hFlush, + db.zFlush} for _, flush := range all { if n, e := flush(); e != nil { @@ -23,24 +19,12 @@ func (db *DB) FlushAll() (drop int64, err error) { return } -func (db *DB) activeExpireCycle() { +func (db *DB) newEliminator() *elimination { eliminator := newEliminator(db) eliminator.regRetireContext(kvExpType, db.kvTx, db.delete) eliminator.regRetireContext(lExpType, db.listTx, db.lDelete) eliminator.regRetireContext(hExpType, db.hashTx, db.hDelete) eliminator.regRetireContext(zExpType, db.zsetTx, db.zDelete) - go func() { - tick := time.NewTicker(1 * time.Second) - for { - select { - case <-tick.C: - eliminator.active() - case <-db.l.quit: - break - } - } - - tick.Stop() - }() + return eliminator } diff --git a/ledis/t_hash.go b/ledis/t_hash.go index ac5d5cf..d64a1f6 100644 --- a/ledis/t_hash.go +++ b/ledis/t_hash.go @@ -430,7 +430,7 @@ func (db *DB) HClear(key []byte) (int64, error) { return num, err } -func (db *DB) HFlush() (drop int64, err error) { +func (db *DB) hFlush() (drop int64, err error) { t := db.kvTx t.Lock() defer t.Unlock() diff --git a/ledis/t_hash_test.go b/ledis/t_hash_test.go index e753361..4648c09 100644 --- a/ledis/t_hash_test.go +++ b/ledis/t_hash_test.go @@ -42,7 +42,7 @@ func TestDBHash(t *testing.T) { func TestDBHScan(t *testing.T) { db := getTestDB() - db.HFlush() + db.hFlush() key := []byte("a") db.HSet(key, []byte("1"), []byte{}) diff --git a/ledis/t_kv.go b/ledis/t_kv.go index 3208761..2ae2a63 100644 --- a/ledis/t_kv.go +++ b/ledis/t_kv.go @@ -311,7 +311,7 @@ func (db *DB) SetNX(key []byte, value []byte) (int64, error) { return n, err } -func (db *DB) Flush() (drop int64, err error) { +func (db *DB) flush() (drop int64, err error) { t := db.kvTx t.Lock() defer t.Unlock() diff --git a/ledis/t_list.go b/ledis/t_list.go index 1532dd8..df5f576 100644 --- a/ledis/t_list.go +++ b/ledis/t_list.go @@ -395,7 +395,7 @@ func (db *DB) LClear(key []byte) (int64, error) { return num, err } -func (db *DB) LFlush() (drop int64, err error) { +func (db *DB) lFlush() (drop int64, err error) { t := db.listTx t.Lock() defer t.Unlock() diff --git a/ledis/t_zset.go b/ledis/t_zset.go index f10ce3f..45f8cfa 100644 --- a/ledis/t_zset.go +++ b/ledis/t_zset.go @@ -732,7 +732,7 @@ func (db *DB) ZRangeByScoreGeneric(key []byte, min int64, max int64, return db.zRange(key, min, max, withScores, offset, count, reverse) } -func (db *DB) ZFlush() (drop int64, err error) { +func (db *DB) zFlush() (drop int64, err error) { t := db.zsetTx t.Lock() defer t.Unlock() diff --git a/ledis/t_zset_test.go b/ledis/t_zset_test.go index 202da70..2b637e6 100644 --- a/ledis/t_zset_test.go +++ b/ledis/t_zset_test.go @@ -220,7 +220,7 @@ func TestZSetOrder(t *testing.T) { func TestDBZScan(t *testing.T) { db := getTestDB() - db.ZFlush() + db.zFlush() key := []byte("key") db.ZAdd(key, pair("a", 0), pair("b", 1), pair("c", 2))