From eb13bf9e9ba90bc08889c2af5b66865d2f98817f Mon Sep 17 00:00:00 2001 From: siddontang Date: Tue, 29 Apr 2014 16:17:25 +0800 Subject: [PATCH] log refactor, add socket handler --- log/filehandler.go | 179 +++++++++++++++++++++++++++++++++++++++++++ log/handler.go | 175 ------------------------------------------ log/log.go | 5 +- log/loggingd/main.go | 21 +++++ log/server.go | 95 +++++++++++++++++++++++ log/socket_test.go | 56 ++++++++++++++ log/sockethandler.go | 62 +++++++++++++++ 7 files changed, 417 insertions(+), 176 deletions(-) create mode 100644 log/filehandler.go create mode 100644 log/loggingd/main.go create mode 100644 log/server.go create mode 100644 log/socket_test.go create mode 100644 log/sockethandler.go diff --git a/log/filehandler.go b/log/filehandler.go new file mode 100644 index 0000000..bf7aebb --- /dev/null +++ b/log/filehandler.go @@ -0,0 +1,179 @@ +package log + +import ( + "fmt" + "os" + "time" +) + +type FileHandler struct { + fd *os.File +} + +func NewFileHandler(fileName string, flag int) (*FileHandler, error) { + f, err := os.OpenFile(fileName, flag, 0) + if err != nil { + return nil, err + } + + h := new(FileHandler) + + h.fd = f + + return h, nil +} + +func (h *FileHandler) Write(b []byte) (n int, err error) { + return h.fd.Write(b) +} + +func (h *FileHandler) Close() error { + return h.fd.Close() +} + +type RotatingFileHandler struct { + fd *os.File + + fileName string + maxBytes int + backupCount int +} + +func NewRotatingFileHandler(fileName string, maxBytes int, backupCount int) (*RotatingFileHandler, error) { + h := new(RotatingFileHandler) + + h.fileName = fileName + h.maxBytes = maxBytes + h.backupCount = backupCount + + var err error + h.fd, err = os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + return nil, err + } + + return h, nil +} + +func (h *RotatingFileHandler) Write(p []byte) (n int, err error) { + h.doRollover() + return h.fd.Write(p) +} + +func (h *RotatingFileHandler) Close() error { + if h.fd != nil { + return h.fd.Close() + } + return nil +} + +func (h *RotatingFileHandler) doRollover() { + f, err := h.fd.Stat() + if err != nil { + return + } + + if h.maxBytes <= 0 { + return + } else if f.Size() < int64(h.maxBytes) { + return + } + + if h.backupCount > 0 { + h.fd.Close() + + for i := h.backupCount - 1; i > 0; i-- { + sfn := fmt.Sprintf("%s.%d", h.fileName, i) + dfn := fmt.Sprintf("%s.%d", h.fileName, i+1) + + os.Rename(sfn, dfn) + } + + dfn := fmt.Sprintf("%s.1", h.fileName) + os.Rename(h.fileName, dfn) + + h.fd, _ = os.OpenFile(h.fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + } +} + +//refer: http://docs.python.org/2/library/logging.handlers.html +//same like python TimedRotatingFileHandler + +type TimeRotatingFileHandler struct { + fd *os.File + + baseName string + interval int64 + suffix string + rolloverAt int64 +} + +const ( + WhenSecond = iota + WhenMinute + WhenHour + WhenDay +) + +func NewTimeRotatingFileHandler(baseName string, when int8, interval int) (*TimeRotatingFileHandler, error) { + h := new(TimeRotatingFileHandler) + + h.baseName = baseName + + switch when { + case WhenSecond: + h.interval = 1 + h.suffix = "2006-01-02_15-04-05" + case WhenMinute: + h.interval = 60 + h.suffix = "2006-01-02_15-04" + case WhenHour: + h.interval = 3600 + h.suffix = "2006-01-02_15" + case WhenDay: + h.interval = 3600 * 24 + h.suffix = "2006-01-02" + default: + return nil, fmt.Errorf("invalid when_rotate: %d", when) + } + + h.interval = h.interval * int64(interval) + + var err error + h.fd, err = os.OpenFile(h.baseName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + return nil, err + } + + fInfo, _ := h.fd.Stat() + h.rolloverAt = fInfo.ModTime().Unix() + h.interval + + return h, nil +} + +func (h *TimeRotatingFileHandler) doRollover() { + //refer http://hg.python.org/cpython/file/2.7/Lib/logging/handlers.py + now := time.Now() + + if h.rolloverAt <= now.Unix() { + fName := h.baseName + now.Format(h.suffix) + h.fd.Close() + e := os.Rename(h.baseName, fName) + if e != nil { + panic(e) + } + + h.fd, _ = os.OpenFile(h.baseName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + + h.rolloverAt = time.Now().Unix() + h.interval + } +} + +func (h *TimeRotatingFileHandler) Write(b []byte) (n int, err error) { + h.doRollover() + return h.fd.Write(b) +} + +func (h *TimeRotatingFileHandler) Close() error { + return h.fd.Close() +} diff --git a/log/handler.go b/log/handler.go index af4acb4..66257ad 100644 --- a/log/handler.go +++ b/log/handler.go @@ -1,10 +1,7 @@ package log import ( - "fmt" "io" - "os" - "time" ) type Handler interface { @@ -46,175 +43,3 @@ func (h *NullHandler) Write(b []byte) (n int, err error) { func (h *NullHandler) Close() { } - -type FileHandler struct { - fd *os.File -} - -func NewFileHandler(fileName string, flag int) (*FileHandler, error) { - f, err := os.OpenFile(fileName, flag, 0) - if err != nil { - return nil, err - } - - h := new(FileHandler) - - h.fd = f - - return h, nil -} - -func (h *FileHandler) Write(b []byte) (n int, err error) { - return h.fd.Write(b) -} - -func (h *FileHandler) Close() error { - return h.fd.Close() -} - -type RotatingFileHandler struct { - fd *os.File - - fileName string - maxBytes int - backupCount int -} - -func NewRotatingFileHandler(fileName string, maxBytes int, backupCount int) (*RotatingFileHandler, error) { - h := new(RotatingFileHandler) - - h.fileName = fileName - h.maxBytes = maxBytes - h.backupCount = backupCount - - var err error - h.fd, err = os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - if err != nil { - return nil, err - } - - return h, nil -} - -func (h *RotatingFileHandler) Write(p []byte) (n int, err error) { - h.doRollover() - return h.fd.Write(p) -} - -func (h *RotatingFileHandler) Close() error { - if h.fd != nil { - return h.fd.Close() - } - return nil -} - -func (h *RotatingFileHandler) doRollover() { - f, err := h.fd.Stat() - if err != nil { - return - } - - if h.maxBytes <= 0 { - return - } else if f.Size() < int64(h.maxBytes) { - return - } - - if h.backupCount > 0 { - h.fd.Close() - - for i := h.backupCount - 1; i > 0; i-- { - sfn := fmt.Sprintf("%s.%d", h.fileName, i) - dfn := fmt.Sprintf("%s.%d", h.fileName, i+1) - - os.Rename(sfn, dfn) - } - - dfn := fmt.Sprintf("%s.1", h.fileName) - os.Rename(h.fileName, dfn) - - h.fd, _ = os.OpenFile(h.fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - } -} - -//refer: http://docs.python.org/2/library/logging.handlers.html -//same like python TimedRotatingFileHandler - -type TimeRotatingFileHandler struct { - fd *os.File - - baseName string - interval int64 - suffix string - rolloverAt int64 -} - -const ( - WhenSecond = iota - WhenMinute - WhenHour - WhenDay -) - -func NewTimeRotatingFileHandler(baseName string, when int8, interval int) (*TimeRotatingFileHandler, error) { - h := new(TimeRotatingFileHandler) - - h.baseName = baseName - - switch when { - case WhenSecond: - h.interval = 1 - h.suffix = "2006-01-02_15-04-05" - case WhenMinute: - h.interval = 60 - h.suffix = "2006-01-02_15-04" - case WhenHour: - h.interval = 3600 - h.suffix = "2006-01-02_15" - case WhenDay: - h.interval = 3600 * 24 - h.suffix = "2006-01-02" - default: - return nil, fmt.Errorf("invalid when_rotate: %d", when) - } - - h.interval = h.interval * int64(interval) - - var err error - h.fd, err = os.OpenFile(h.baseName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - if err != nil { - return nil, err - } - - fInfo, _ := h.fd.Stat() - h.rolloverAt = fInfo.ModTime().Unix() + h.interval - - return h, nil -} - -func (h *TimeRotatingFileHandler) doRollover() { - //refer http://hg.python.org/cpython/file/2.7/Lib/logging/handlers.py - now := time.Now() - - if h.rolloverAt <= now.Unix() { - fName := h.baseName + now.Format(h.suffix) - h.fd.Close() - e := os.Rename(h.baseName, fName) - if e != nil { - panic(e) - } - - h.fd, _ = os.OpenFile(h.baseName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - - h.rolloverAt = time.Now().Unix() + h.interval - } -} - -func (h *TimeRotatingFileHandler) Write(b []byte) (n int, err error) { - h.doRollover() - return h.fd.Write(b) -} - -func (h *TimeRotatingFileHandler) Close() error { - return h.fd.Close() -} diff --git a/log/log.go b/log/log.go index e1c8440..9e8ed41 100644 --- a/log/log.go +++ b/log/log.go @@ -125,7 +125,10 @@ func (l *Logger) Output(callDepth int, level int, format string, v ...interface{ s := fmt.Sprintf(format, v...) buf = append(buf, s...) - buf = append(buf, "\n"...) + + if s[len(s)-1] != '\n' { + buf = append(buf, "\n"...) + } l.msg <- buf } diff --git a/log/loggingd/main.go b/log/loggingd/main.go new file mode 100644 index 0000000..8c50bb4 --- /dev/null +++ b/log/loggingd/main.go @@ -0,0 +1,21 @@ +package main + +import ( + "flag" + "github.com/siddontang/golib/log" +) + +var logFile = flag.String("logfile", "./logd.log", "file to log") +var net = flag.String("net", "tcp", "server listen protocol, like tcp, udp or unix") +var addr = flag.String("addr", "127.0.0.1:11183", "server listen address") + +func main() { + flag.Parse() + + s, err := log.NewServer(*logFile, *net, *addr) + if err != nil { + panic(err) + } + + s.Run() +} diff --git a/log/server.go b/log/server.go new file mode 100644 index 0000000..db3bf6b --- /dev/null +++ b/log/server.go @@ -0,0 +1,95 @@ +package log + +import ( + "bufio" + "encoding/binary" + "io" + "net" + "os" + "path" +) + +//a log server for handling SocketHandler send log + +type Server struct { + closed bool + listener net.Listener + fd *os.File +} + +func NewServer(fileName string, protocol string, addr string) (*Server, error) { + s := new(Server) + + s.closed = false + + var err error + + dir := path.Dir(fileName) + os.Mkdir(dir, 0777) + + s.fd, err = os.OpenFile(fileName, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) + if err != nil { + return nil, err + } + + s.listener, err = net.Listen(protocol, addr) + if err != nil { + return nil, err + } + + return s, nil +} + +func (s *Server) Close() error { + if s.closed { + return nil + } + + s.closed = true + + s.fd.Close() + + s.listener.Close() + return nil +} + +func (s *Server) Run() { + for { + conn, err := s.listener.Accept() + if err != nil { + continue + } + + go s.onRead(conn) + } +} + +func (s *Server) onRead(c net.Conn) { + br := bufio.NewReaderSize(c, 1024) + + var bufLen uint32 + + for { + if err := binary.Read(br, binary.BigEndian, &bufLen); err != nil { + c.Close() + return + } + + buf := make([]byte, bufLen, bufLen+1) + + if _, err := io.ReadFull(br, buf); err != nil && err != io.ErrUnexpectedEOF { + c.Close() + return + } else { + if len(buf) == 0 { + continue + } + if buf[len(buf)-1] != '\n' { + buf = append(buf, '\n') + } + + s.fd.Write(buf) + } + + } +} diff --git a/log/socket_test.go b/log/socket_test.go new file mode 100644 index 0000000..6fd7d35 --- /dev/null +++ b/log/socket_test.go @@ -0,0 +1,56 @@ +package log + +import ( + "io" + "os" + "testing" + "time" +) + +func TestSocket(t *testing.T) { + fileName := "./test_server.log" + + os.Remove(fileName) + + s, err := NewServer(fileName, "tcp", "127.0.0.1:11183") + if err != nil { + t.Fatal(err) + } + go s.Run() + defer s.Close() + + var h *SocketHandler + h, err = NewSocketHandler("tcp", "127.0.0.1:11183") + + _, err = h.Write([]byte("hello world")) + if err != nil { + t.Fatal(err) + } + + time.Sleep(1 * time.Second) + + s.Close() + + var f *os.File + f, err = os.Open(fileName) + if err != nil { + t.Fatal(err) + } + + defer f.Close() + + buf := make([]byte, 64) + var n int + n, err = f.Read(buf) + if err != nil && err != io.EOF { + t.Fatal(err) + } + + buf = buf[0:n] + + if string(buf) != "hello world\n" { + t.Fatal(string(buf)) + } + + os.Remove(fileName) +} diff --git a/log/sockethandler.go b/log/sockethandler.go new file mode 100644 index 0000000..f19db05 --- /dev/null +++ b/log/sockethandler.go @@ -0,0 +1,62 @@ +package log + +import ( + "encoding/binary" + "net" + "time" +) + +type SocketHandler struct { + c net.Conn + protocol string + addr string +} + +func NewSocketHandler(protocol string, addr string) (*SocketHandler, error) { + s := new(SocketHandler) + + s.protocol = protocol + s.addr = addr + + return s, nil +} + +func (h *SocketHandler) Write(p []byte) (n int, err error) { + if err = h.connect(); err != nil { + return + } + + buf := make([]byte, len(p)+4) + + binary.BigEndian.PutUint32(buf, uint32(len(p))) + + copy(buf[4:], p) + + n, err = h.c.Write(buf) + if err != nil { + h.c.Close() + h.c = nil + } + return +} + +func (h *SocketHandler) Close() error { + if h.c != nil { + h.c.Close() + } + return nil +} + +func (h *SocketHandler) connect() error { + if h.c != nil { + return nil + } + + var err error + h.c, err = net.DialTimeout(h.protocol, h.addr, 20*time.Second) + if err != nil { + return err + } + + return nil +}