From c5af770387bc493183855b51057faf0582f0d238 Mon Sep 17 00:00:00 2001 From: siddontang Date: Thu, 19 Jun 2014 17:50:27 +0800 Subject: [PATCH] use my own log --- bootstrap.sh | 2 +- cmd/ledis-benchmark/main.go | 2 +- ledis/binlog.go | 2 +- ledis/ledis.go | 2 +- ledis/replication.go | 2 +- log/filehandler.go | 193 ++++++++++++++++++++++++++++++ log/handler.go | 45 +++++++ log/log.go | 226 ++++++++++++++++++++++++++++++++++++ log/log_test.go | 52 +++++++++ log/sockethandler.go | 62 ++++++++++ server/accesslog.go | 2 +- server/client.go | 2 +- server/replication.go | 2 +- 13 files changed, 586 insertions(+), 8 deletions(-) create mode 100644 log/filehandler.go create mode 100644 log/handler.go create mode 100644 log/log.go create mode 100644 log/log_test.go create mode 100644 log/sockethandler.go diff --git a/bootstrap.sh b/bootstrap.sh index c05f3db..29344e6 100644 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -2,4 +2,4 @@ . ./dev.sh -go get -u github.com/siddontang/go-log/log +#nothing to do now \ No newline at end of file diff --git a/cmd/ledis-benchmark/main.go b/cmd/ledis-benchmark/main.go index ac67411..9209dd7 100644 --- a/cmd/ledis-benchmark/main.go +++ b/cmd/ledis-benchmark/main.go @@ -3,7 +3,7 @@ package main import ( "flag" "fmt" - "github.com/garyburd/redigo/redis" + "github.com/siddontang/ledisdb/client/go/redis" "math/rand" "sync" "time" diff --git a/ledis/binlog.go b/ledis/binlog.go index d6e99f0..4785370 100644 --- a/ledis/binlog.go +++ b/ledis/binlog.go @@ -5,7 +5,7 @@ import ( "encoding/binary" "encoding/json" "fmt" - "github.com/siddontang/go-log/log" + "github.com/siddontang/ledisdb/log" "io/ioutil" "os" "path" diff --git a/ledis/ledis.go b/ledis/ledis.go index 668098c..6bf333a 100644 --- a/ledis/ledis.go +++ b/ledis/ledis.go @@ -3,8 +3,8 @@ package ledis import ( "encoding/json" "fmt" - "github.com/siddontang/go-log/log" "github.com/siddontang/ledisdb/leveldb" + "github.com/siddontang/ledisdb/log" "path" "sync" "time" diff --git a/ledis/replication.go b/ledis/replication.go index e19da6a..fce8fd4 100644 --- a/ledis/replication.go +++ b/ledis/replication.go @@ -5,7 +5,7 @@ import ( "bytes" "encoding/binary" "errors" - "github.com/siddontang/go-log/log" + "github.com/siddontang/ledisdb/log" "io" "os" ) diff --git a/log/filehandler.go b/log/filehandler.go new file mode 100644 index 0000000..e77eefe --- /dev/null +++ b/log/filehandler.go @@ -0,0 +1,193 @@ +package log + +import ( + "fmt" + "os" + "path" + "time" +) + +type FileHandler struct { + fd *os.File +} + +func NewFileHandler(fileName string, flag int) (*FileHandler, error) { + dir := path.Dir(fileName) + os.Mkdir(dir, 0777) + + f, err := os.OpenFile(fileName, flag, 0) + if err != nil { + return nil, err + } + + h := new(FileHandler) + + h.fd = f + + return h, nil +} + +func (h *FileHandler) Write(b []byte) (n int, err error) { + return h.fd.Write(b) +} + +func (h *FileHandler) Close() error { + return h.fd.Close() +} + +type RotatingFileHandler struct { + fd *os.File + + fileName string + maxBytes int + backupCount int +} + +func NewRotatingFileHandler(fileName string, maxBytes int, backupCount int) (*RotatingFileHandler, error) { + dir := path.Dir(fileName) + os.Mkdir(dir, 0777) + + h := new(RotatingFileHandler) + + if maxBytes <= 0 { + return nil, fmt.Errorf("invalid max bytes") + } + + h.fileName = fileName + h.maxBytes = maxBytes + h.backupCount = backupCount + + var err error + h.fd, err = os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + return nil, err + } + + return h, nil +} + +func (h *RotatingFileHandler) Write(p []byte) (n int, err error) { + h.doRollover() + return h.fd.Write(p) +} + +func (h *RotatingFileHandler) Close() error { + if h.fd != nil { + return h.fd.Close() + } + return nil +} + +func (h *RotatingFileHandler) doRollover() { + f, err := h.fd.Stat() + if err != nil { + return + } + + if h.maxBytes <= 0 { + return + } else if f.Size() < int64(h.maxBytes) { + return + } + + if h.backupCount > 0 { + h.fd.Close() + + for i := h.backupCount - 1; i > 0; i-- { + sfn := fmt.Sprintf("%s.%d", h.fileName, i) + dfn := fmt.Sprintf("%s.%d", h.fileName, i+1) + + os.Rename(sfn, dfn) + } + + dfn := fmt.Sprintf("%s.1", h.fileName) + os.Rename(h.fileName, dfn) + + h.fd, _ = os.OpenFile(h.fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + } +} + +//refer: http://docs.python.org/2/library/logging.handlers.html +//same like python TimedRotatingFileHandler + +type TimeRotatingFileHandler struct { + fd *os.File + + baseName string + interval int64 + suffix string + rolloverAt int64 +} + +const ( + WhenSecond = iota + WhenMinute + WhenHour + WhenDay +) + +func NewTimeRotatingFileHandler(baseName string, when int8, interval int) (*TimeRotatingFileHandler, error) { + dir := path.Dir(baseName) + os.Mkdir(dir, 0777) + + h := new(TimeRotatingFileHandler) + + h.baseName = baseName + + switch when { + case WhenSecond: + h.interval = 1 + h.suffix = "2006-01-02_15-04-05" + case WhenMinute: + h.interval = 60 + h.suffix = "2006-01-02_15-04" + case WhenHour: + h.interval = 3600 + h.suffix = "2006-01-02_15" + case WhenDay: + h.interval = 3600 * 24 + h.suffix = "2006-01-02" + default: + return nil, fmt.Errorf("invalid when_rotate: %d", when) + } + + h.interval = h.interval * int64(interval) + + var err error + h.fd, err = os.OpenFile(h.baseName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + return nil, err + } + + fInfo, _ := h.fd.Stat() + h.rolloverAt = fInfo.ModTime().Unix() + h.interval + + return h, nil +} + +func (h *TimeRotatingFileHandler) doRollover() { + //refer http://hg.python.org/cpython/file/2.7/Lib/logging/handlers.py + now := time.Now() + + if h.rolloverAt <= now.Unix() { + fName := h.baseName + now.Format(h.suffix) + h.fd.Close() + e := os.Rename(h.baseName, fName) + if e != nil { + panic(e) + } + + h.fd, _ = os.OpenFile(h.baseName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + + h.rolloverAt = time.Now().Unix() + h.interval + } +} + +func (h *TimeRotatingFileHandler) Write(b []byte) (n int, err error) { + h.doRollover() + return h.fd.Write(b) +} + +func (h *TimeRotatingFileHandler) Close() error { + return h.fd.Close() +} diff --git a/log/handler.go b/log/handler.go new file mode 100644 index 0000000..66257ad --- /dev/null +++ b/log/handler.go @@ -0,0 +1,45 @@ +package log + +import ( + "io" +) + +type Handler interface { + Write(p []byte) (n int, err error) + Close() error +} + +type StreamHandler struct { + w io.Writer +} + +func NewStreamHandler(w io.Writer) (*StreamHandler, error) { + h := new(StreamHandler) + + h.w = w + + return h, nil +} + +func (h *StreamHandler) Write(b []byte) (n int, err error) { + return h.w.Write(b) +} + +func (h *StreamHandler) Close() error { + return nil +} + +type NullHandler struct { +} + +func NewNullHandler() (*NullHandler, error) { + return new(NullHandler), nil +} + +func (h *NullHandler) Write(b []byte) (n int, err error) { + return len(b), nil +} + +func (h *NullHandler) Close() { + +} diff --git a/log/log.go b/log/log.go new file mode 100644 index 0000000..3a59df8 --- /dev/null +++ b/log/log.go @@ -0,0 +1,226 @@ +package log + +import ( + "fmt" + "os" + "runtime" + "strconv" + "sync" + "time" +) + +const ( + LevelTrace = iota + LevelDebug + LevelInfo + LevelWarn + LevelError + LevelFatal +) + +const ( + Ltime = 1 << iota //time format "2006/01/02 15:04:05" + Lfile //file.go:123 + Llevel //[Trace|Debug|Info...] +) + +var LevelName [6]string = [6]string{"Trace", "Debug", "Info", "Warn", "Error", "Fatal"} + +const TimeFormat = "2006/01/02 15:04:05" + +const maxBufPoolSize = 16 + +type Logger struct { + sync.Mutex + + level int + flag int + + handler Handler + + quit chan struct{} + msg chan []byte + + bufs [][]byte +} + +func New(handler Handler, flag int) *Logger { + var l = new(Logger) + + l.level = LevelInfo + l.handler = handler + + l.flag = flag + + l.quit = make(chan struct{}) + + l.msg = make(chan []byte, 1024) + + l.bufs = make([][]byte, 0, 16) + + go l.run() + + return l +} + +func NewDefault(handler Handler) *Logger { + return New(handler, Ltime|Lfile|Llevel) +} + +func newStdHandler() *StreamHandler { + h, _ := NewStreamHandler(os.Stdout) + return h +} + +var std = NewDefault(newStdHandler()) + +func (l *Logger) run() { + for { + select { + case msg := <-l.msg: + l.handler.Write(msg) + l.putBuf(msg) + case <-l.quit: + l.handler.Close() + } + } +} + +func (l *Logger) popBuf() []byte { + l.Lock() + var buf []byte + if len(l.bufs) == 0 { + buf = make([]byte, 0, 1024) + } else { + buf = l.bufs[len(l.bufs)-1] + l.bufs = l.bufs[0 : len(l.bufs)-1] + } + l.Unlock() + + return buf +} + +func (l *Logger) putBuf(buf []byte) { + l.Lock() + if len(l.bufs) < maxBufPoolSize { + buf = buf[0:0] + l.bufs = append(l.bufs, buf) + } + l.Unlock() +} + +func (l *Logger) Close() { + if l.quit == nil { + return + } + + close(l.quit) + l.quit = nil +} + +func (l *Logger) SetLevel(level int) { + l.level = level +} + +func (l *Logger) Output(callDepth int, level int, format string, v ...interface{}) { + if l.level > level { + return + } + + buf := l.popBuf() + + if l.flag&Ltime > 0 { + now := time.Now().Format(TimeFormat) + buf = append(buf, '[') + buf = append(buf, now...) + buf = append(buf, "] "...) + } + + if l.flag&Lfile > 0 { + _, file, line, ok := runtime.Caller(callDepth) + if !ok { + file = "???" + line = 0 + } else { + for i := len(file) - 1; i > 0; i-- { + if file[i] == '/' { + file = file[i+1:] + break + } + } + } + + buf = append(buf, file...) + buf = append(buf, ':') + + strconv.AppendInt(buf, int64(line), 10) + } + + if l.flag&Llevel > 0 { + buf = append(buf, '[') + buf = append(buf, LevelName[level]...) + buf = append(buf, "] "...) + } + + s := fmt.Sprintf(format, v...) + + buf = append(buf, s...) + + if s[len(s)-1] != '\n' { + buf = append(buf, '\n') + } + + l.msg <- buf +} + +func (l *Logger) Trace(format string, v ...interface{}) { + l.Output(2, LevelTrace, format, v...) +} + +func (l *Logger) Debug(format string, v ...interface{}) { + l.Output(2, LevelDebug, format, v...) +} + +func (l *Logger) Info(format string, v ...interface{}) { + l.Output(2, LevelInfo, format, v...) +} + +func (l *Logger) Warn(format string, v ...interface{}) { + l.Output(2, LevelWarn, format, v...) +} + +func (l *Logger) Error(format string, v ...interface{}) { + l.Output(2, LevelError, format, v...) +} + +func (l *Logger) Fatal(format string, v ...interface{}) { + l.Output(2, LevelFatal, format, v...) +} + +func SetLevel(level int) { + std.SetLevel(level) +} + +func Trace(format string, v ...interface{}) { + std.Output(2, LevelTrace, format, v...) +} + +func Debug(format string, v ...interface{}) { + std.Output(2, LevelDebug, format, v...) +} + +func Info(format string, v ...interface{}) { + std.Output(2, LevelInfo, format, v...) +} + +func Warn(format string, v ...interface{}) { + std.Output(2, LevelWarn, format, v...) +} + +func Error(format string, v ...interface{}) { + std.Output(2, LevelError, format, v...) +} + +func Fatal(format string, v ...interface{}) { + std.Output(2, LevelFatal, format, v...) +} diff --git a/log/log_test.go b/log/log_test.go new file mode 100644 index 0000000..67d3b0b --- /dev/null +++ b/log/log_test.go @@ -0,0 +1,52 @@ +package log + +import ( + "os" + "testing" +) + +func TestStdStreamLog(t *testing.T) { + h, _ := NewStreamHandler(os.Stdout) + s := NewDefault(h) + s.Info("hello world") + + s.Close() + + Info("hello world") +} + +func TestRotatingFileLog(t *testing.T) { + path := "./test_log" + os.RemoveAll(path) + + os.Mkdir(path, 0777) + fileName := path + "/test" + + h, err := NewRotatingFileHandler(fileName, 10, 2) + if err != nil { + t.Fatal(err) + } + + buf := make([]byte, 10) + + h.Write(buf) + + h.Write(buf) + + if _, err := os.Stat(fileName + ".1"); err != nil { + t.Fatal(err) + } + + if _, err := os.Stat(fileName + ".2"); err == nil { + t.Fatal(err) + } + + h.Write(buf) + if _, err := os.Stat(fileName + ".2"); err != nil { + t.Fatal(err) + } + + h.Close() + + os.RemoveAll(path) +} diff --git a/log/sockethandler.go b/log/sockethandler.go new file mode 100644 index 0000000..f19db05 --- /dev/null +++ b/log/sockethandler.go @@ -0,0 +1,62 @@ +package log + +import ( + "encoding/binary" + "net" + "time" +) + +type SocketHandler struct { + c net.Conn + protocol string + addr string +} + +func NewSocketHandler(protocol string, addr string) (*SocketHandler, error) { + s := new(SocketHandler) + + s.protocol = protocol + s.addr = addr + + return s, nil +} + +func (h *SocketHandler) Write(p []byte) (n int, err error) { + if err = h.connect(); err != nil { + return + } + + buf := make([]byte, len(p)+4) + + binary.BigEndian.PutUint32(buf, uint32(len(p))) + + copy(buf[4:], p) + + n, err = h.c.Write(buf) + if err != nil { + h.c.Close() + h.c = nil + } + return +} + +func (h *SocketHandler) Close() error { + if h.c != nil { + h.c.Close() + } + return nil +} + +func (h *SocketHandler) connect() error { + if h.c != nil { + return nil + } + + var err error + h.c, err = net.DialTimeout(h.protocol, h.addr, 20*time.Second) + if err != nil { + return err + } + + return nil +} diff --git a/server/accesslog.go b/server/accesslog.go index 9e517a8..2190899 100644 --- a/server/accesslog.go +++ b/server/accesslog.go @@ -1,7 +1,7 @@ package server import ( - "github.com/siddontang/go-log/log" + "github.com/siddontang/ledisdb/log" ) const ( diff --git a/server/client.go b/server/client.go index 3f5c43c..21412a3 100644 --- a/server/client.go +++ b/server/client.go @@ -4,8 +4,8 @@ import ( "bufio" "bytes" "errors" - "github.com/siddontang/go-log/log" "github.com/siddontang/ledisdb/ledis" + "github.com/siddontang/ledisdb/log" "io" "net" "runtime" diff --git a/server/replication.go b/server/replication.go index 383a244..bb3cc49 100644 --- a/server/replication.go +++ b/server/replication.go @@ -7,8 +7,8 @@ import ( "encoding/json" "errors" "fmt" - "github.com/siddontang/go-log/log" "github.com/siddontang/ledisdb/ledis" + "github.com/siddontang/ledisdb/log" "io/ioutil" "net" "os"