diff --git a/leveldb/batch.go b/leveldb/batch.go deleted file mode 100644 index f65226a..0000000 --- a/leveldb/batch.go +++ /dev/null @@ -1,35 +0,0 @@ -package leveldb - -import ( - "github.com/jmhodges/levigo" -) - -type WriteBatch struct { - db *DB - wb *levigo.WriteBatch -} - -func (wb *WriteBatch) Put(key, value []byte) { - wb.wb.Put(key, value) -} - -func (wb *WriteBatch) Delete(key []byte) { - wb.wb.Delete(key) -} - -func (wb *WriteBatch) Commit() error { - return wb.db.db.Write(wb.db.writeOpts, wb.wb) -} - -func (wb *WriteBatch) Rollback() { - wb.wb.Clear() -} - -func (wb *WriteBatch) Close() { - if wb.wb == nil { - return - } - - wb.wb.Close() - wb.wb = nil -} diff --git a/leveldb/db.go b/leveldb/db.go deleted file mode 100644 index e3ad332..0000000 --- a/leveldb/db.go +++ /dev/null @@ -1,178 +0,0 @@ -package leveldb - -import ( - "encoding/json" - "github.com/jmhodges/levigo" -) - -const defaultFilterBits int = 10 - -type Config struct { - Path string `json:"path"` - - Compression bool `json:"compression"` - - BlockSize int `json:"block_size"` - WriteBufferSize int `json:"write_buffer_size"` - CacheSize int `json:"cache_size"` -} - -type DB struct { - cfg *Config - db *levigo.DB - - opts *levigo.Options - - //for default read and write options - readOpts *levigo.ReadOptions - writeOpts *levigo.WriteOptions - iteratorOpts *levigo.ReadOptions - - cache *levigo.Cache - - filter *levigo.FilterPolicy -} - -func OpenWithConfig(cfg *Config) (*DB, error) { - db := new(DB) - db.cfg = cfg - - db.opts = db.initOptions(cfg) - - db.readOpts = levigo.NewReadOptions() - db.writeOpts = levigo.NewWriteOptions() - db.iteratorOpts = levigo.NewReadOptions() - db.iteratorOpts.SetFillCache(false) - - var err error - db.db, err = levigo.Open(cfg.Path, db.opts) - return db, err -} - -func Open(configJson json.RawMessage) (*DB, error) { - cfg := new(Config) - err := json.Unmarshal(configJson, cfg) - if err != nil { - return nil, err - } - - return OpenWithConfig(cfg) -} - -func (db *DB) initOptions(cfg *Config) *levigo.Options { - opts := levigo.NewOptions() - - opts.SetCreateIfMissing(true) - - if cfg.CacheSize > 0 { - db.cache = levigo.NewLRUCache(cfg.CacheSize) - opts.SetCache(db.cache) - } - - //we must use bloomfilter - db.filter = levigo.NewBloomFilter(defaultFilterBits) - opts.SetFilterPolicy(db.filter) - - if !cfg.Compression { - opts.SetCompression(levigo.NoCompression) - } - - if cfg.BlockSize > 0 { - opts.SetBlockSize(cfg.BlockSize) - } - - if cfg.WriteBufferSize > 0 { - opts.SetWriteBufferSize(cfg.WriteBufferSize) - } - - return opts -} - -func (db *DB) Close() { - db.opts.Close() - - if db.cache != nil { - db.cache.Close() - } - - if db.filter != nil { - db.filter.Close() - } - - db.readOpts.Close() - db.writeOpts.Close() - db.iteratorOpts.Close() - - db.db.Close() - db.db = nil -} - -func (db *DB) Destroy() { - db.Close() - opts := levigo.NewOptions() - defer opts.Close() - - levigo.DestroyDatabase(db.cfg.Path, opts) -} - -func (db *DB) Clear() { - it := db.Iterator(nil, nil, 0, 0, -1) - for ; it.Valid(); it.Next() { - db.Delete(it.Key()) - } -} - -func (db *DB) Put(key, value []byte) error { - return db.db.Put(db.writeOpts, key, value) -} - -func (db *DB) Get(key []byte) ([]byte, error) { - return db.db.Get(db.readOpts, key) -} - -func (db *DB) Delete(key []byte) error { - return db.db.Delete(db.writeOpts, key) -} - -func (db *DB) NewWriteBatch() *WriteBatch { - wb := new(WriteBatch) - wb.wb = levigo.NewWriteBatch() - wb.db = db - return wb -} - -//limit < 0, unlimit -//offset must >= 0, if < 0, will get nothing -func (db *DB) Iterator(min []byte, max []byte, rangeType uint8, offset int, limit int) *Iterator { - return newIterator(db, db.iteratorOpts, NewRange(min, max, rangeType), offset, limit, IteratorForward) -} - -//limit < 0, unlimit -//offset must >= 0, if < 0, will get nothing -func (db *DB) RevIterator(min []byte, max []byte, rangeType uint8, offset int, limit int) *Iterator { - return newIterator(db, db.iteratorOpts, NewRange(min, max, rangeType), offset, limit, IteratorBackward) -} - -func (db *DB) NewSnapshot() *Snapshot { - return newSnapshot(db) -} - -func (db *DB) GetInt(key []byte) (int64, error) { - return Int(db.Get(key)) -} - -func (db *DB) GetUInt(key []byte) (uint64, error) { - return Uint(db.Get(key)) -} - -func (db *DB) GetFloat(key []byte) (float64, error) { - return Float(db.Get(key)) -} - -func (db *DB) GetString(key []byte) (string, error) { - return String(db.Get(key)) -} - -func (db *DB) GetSlice(key []byte) ([]byte, error) { - return Slice(db.Get(key)) -} diff --git a/leveldb/dev.sh b/leveldb/dev.sh deleted file mode 100644 index cdd826d..0000000 --- a/leveldb/dev.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -SNAPPY_DIR=/usr/local/snappy -LEVELDB_DIR=/usr/local/leveldb - -export CGO_CFLAGS="-I$LEVELDB_DIR/include -I$SNAPPY_DIR/include" -export CGO_CXXFLAGS="-I$LEVELDB_DIR/include -I$SNAPPY_DIR/include" -export CGO_LDFLAGS="-L$LEVELDB_DIR/lib -L$SNAPPY_DIR/lib -lsnappy" -export LD_LIBRARY_PATH=$(add_path $LD_LIBRARY_PATH $SNAPPY_DIR/lib) -export LD_LIBRARY_PATH=$(add_path $LD_LIBRARY_PATH $LEVELDB_DIR/lib) - -go get github.com/jmhodges/levigo diff --git a/leveldb/iterator.go b/leveldb/iterator.go deleted file mode 100644 index 599b04e..0000000 --- a/leveldb/iterator.go +++ /dev/null @@ -1,189 +0,0 @@ -package leveldb - -import ( - "bytes" - "github.com/jmhodges/levigo" -) - -const ( - IteratorForward uint8 = 0 - IteratorBackward uint8 = 1 -) - -const ( - RangeClose uint8 = 0x00 - RangeLOpen uint8 = 0x01 - RangeROpen uint8 = 0x10 - RangeOpen uint8 = 0x11 -) - -//min must less or equal than max -//range type: -//close: [min, max] -//open: (min, max) -//lopen: (min, max] -//ropen: [min, max) -type Range struct { - Min []byte - Max []byte - - Type uint8 -} - -func NewRange(min []byte, max []byte, tp uint8) *Range { - return &Range{min, max, tp} -} - -type Iterator struct { - it *levigo.Iterator - - r *Range - - offset int - limit int - - step int - - //0 for IteratorForward, 1 for IteratorBackward - direction uint8 -} - -func newIterator(db *DB, opts *levigo.ReadOptions, r *Range, offset int, limit int, direction uint8) *Iterator { - it := new(Iterator) - it.it = db.db.NewIterator(opts) - - it.r = r - it.offset = offset - it.limit = limit - it.direction = direction - - it.step = 0 - - if offset < 0 { - return it - } - - if direction == IteratorForward { - if r.Min == nil { - it.it.SeekToFirst() - } else { - it.it.Seek(r.Min) - - if r.Type&RangeLOpen > 0 { - if it.it.Valid() && bytes.Equal(it.it.Key(), r.Min) { - it.it.Next() - } - } - } - } else { - if r.Max == nil { - it.it.SeekToLast() - } else { - it.it.Seek(r.Max) - - if !it.it.Valid() { - it.it.SeekToLast() - } else { - if !bytes.Equal(it.it.Key(), r.Max) { - it.it.Prev() - } - } - - if r.Type&RangeROpen > 0 { - if it.Valid() && bytes.Equal(it.Key(), r.Max) { - it.it.Prev() - } - } - } - } - - for i := 0; i < offset; i++ { - if it.it.Valid() { - if it.direction == IteratorForward { - it.it.Next() - } else { - it.it.Prev() - } - } - } - - return it -} - -func (it *Iterator) Valid() bool { - if it.offset < 0 { - return false - } else if !it.it.Valid() { - return false - } else if it.limit >= 0 && it.step >= it.limit { - return false - } - - if it.direction == IteratorForward { - if it.r.Max != nil { - r := bytes.Compare(it.Key(), it.r.Max) - if it.r.Type&RangeROpen > 0 { - return !(r >= 0) - } else { - return !(r > 0) - } - } - } else { - if it.r.Min != nil { - r := bytes.Compare(it.Key(), it.r.Min) - if it.r.Type&RangeLOpen > 0 { - return !(r <= 0) - } else { - return !(r < 0) - } - } - } - - return true -} - -func (it *Iterator) GetError() error { - return it.it.GetError() -} - -func (it *Iterator) Next() { - it.step++ - - if it.direction == IteratorForward { - it.it.Next() - } else { - it.it.Prev() - } -} - -func (it *Iterator) Key() []byte { - return it.it.Key() -} - -func (it *Iterator) Value() []byte { - return it.it.Value() -} - -func (it *Iterator) Close() { - it.it.Close() -} - -func (it *Iterator) IntValue() (int64, error) { - return Int(it.Value(), nil) -} - -func (it *Iterator) UintValue() (uint64, error) { - return Uint(it.Value(), nil) -} - -func (it *Iterator) FloatValue() (float64, error) { - return Float(it.Value(), nil) -} - -func (it *Iterator) StringValue() (string, error) { - return String(it.Value(), nil) -} - -func (it *Iterator) SliceValue() ([]byte, error) { - return Slice(it.Value(), nil) -} diff --git a/leveldb/leveldb_test.go b/leveldb/leveldb_test.go deleted file mode 100644 index 9f68781..0000000 --- a/leveldb/leveldb_test.go +++ /dev/null @@ -1,228 +0,0 @@ -package leveldb - -import ( - "bytes" - "fmt" - "os" - "sync" - "testing" -) - -var testConfigJson = []byte(` - { - "path" : "./testdb", - "compression":true, - "block_size" : 32768, - "write_buffer_size" : 2097152, - "cache_size" : 20971520 - } - `) - -var testOnce sync.Once -var testDB *DB - -func getTestDB() *DB { - f := func() { - var err error - testDB, err = Open(testConfigJson) - if err != nil { - println(err.Error()) - panic(err) - } - } - - testOnce.Do(f) - return testDB -} - -func TestSimple(t *testing.T) { - db := getTestDB() - - key := []byte("key") - value := []byte("hello world") - if err := db.Put(key, value); err != nil { - t.Fatal(err) - } - - if v, err := db.Get(key); err != nil { - t.Fatal(err) - } else if !bytes.Equal(v, value) { - t.Fatal("not equal") - } - - if err := db.Delete(key); err != nil { - t.Fatal(err) - } - - if v, err := db.Get(key); err != nil { - t.Fatal(err) - } else if v != nil { - t.Fatal("must nil") - } -} - -func TestBatch(t *testing.T) { - db := getTestDB() - - key1 := []byte("key1") - key2 := []byte("key2") - - value := []byte("hello world") - - db.Put(key1, value) - db.Put(key2, value) - - wb := db.NewWriteBatch() - defer wb.Close() - - wb.Delete(key2) - wb.Put(key1, []byte("hello world2")) - - if err := wb.Commit(); err != nil { - t.Fatal(err) - } - - if v, err := db.Get(key2); err != nil { - t.Fatal(err) - } else if v != nil { - t.Fatal("must nil") - } - - if v, err := db.Get(key1); err != nil { - t.Fatal(err) - } else if string(v) != "hello world2" { - t.Fatal(string(v)) - } - - wb.Delete(key1) - - wb.Rollback() - - if v, err := db.Get(key1); err != nil { - t.Fatal(err) - } else if string(v) != "hello world2" { - t.Fatal(string(v)) - } - - db.Delete(key1) -} - -func checkIterator(it *Iterator, cv ...int) error { - v := make([]string, 0, len(cv)) - for ; it.Valid(); it.Next() { - k := it.Key() - v = append(v, string(k)) - } - - it.Close() - - if len(v) != len(cv) { - return fmt.Errorf("len error %d != %d", len(v), len(cv)) - } - - for k, i := range cv { - if fmt.Sprintf("key_%d", i) != v[k] { - return fmt.Errorf("%s, %d", v[k], i) - } - } - - return nil -} - -func TestIterator(t *testing.T) { - db := getTestDB() - - db.Clear() - - for i := 0; i < 10; i++ { - key := []byte(fmt.Sprintf("key_%d", i)) - value := []byte("") - db.Put(key, value) - } - - var it *Iterator - - k := func(i int) []byte { - return []byte(fmt.Sprintf("key_%d", i)) - } - - it = db.Iterator(k(1), k(5), RangeClose, 0, -1) - if err := checkIterator(it, 1, 2, 3, 4, 5); err != nil { - t.Fatal(err) - } - - it = db.Iterator(k(1), k(5), RangeClose, 1, 3) - if err := checkIterator(it, 2, 3, 4); err != nil { - t.Fatal(err) - } - - it = db.Iterator(k(1), k(5), RangeLOpen, 0, -1) - if err := checkIterator(it, 2, 3, 4, 5); err != nil { - t.Fatal(err) - } - - it = db.Iterator(k(1), k(5), RangeROpen, 0, -1) - if err := checkIterator(it, 1, 2, 3, 4); err != nil { - t.Fatal(err) - } - - it = db.Iterator(k(1), k(5), RangeOpen, 0, -1) - if err := checkIterator(it, 2, 3, 4); err != nil { - t.Fatal(err) - } - - it = db.RevIterator(k(1), k(5), RangeClose, 0, -1) - if err := checkIterator(it, 5, 4, 3, 2, 1); err != nil { - t.Fatal(err) - } - - it = db.RevIterator(k(1), k(5), RangeClose, 1, 3) - if err := checkIterator(it, 4, 3, 2); err != nil { - t.Fatal(err) - } - - it = db.RevIterator(k(1), k(5), RangeLOpen, 0, -1) - if err := checkIterator(it, 5, 4, 3, 2); err != nil { - t.Fatal(err) - } - - it = db.RevIterator(k(1), k(5), RangeROpen, 0, -1) - if err := checkIterator(it, 4, 3, 2, 1); err != nil { - t.Fatal(err) - } - - it = db.RevIterator(k(1), k(5), RangeOpen, 0, -1) - if err := checkIterator(it, 4, 3, 2); err != nil { - t.Fatal(err) - } -} - -func TestSnapshot(t *testing.T) { - db := getTestDB() - - key := []byte("key") - value := []byte("hello world") - - db.Put(key, value) - - s := db.NewSnapshot() - defer s.Close() - - db.Put(key, []byte("hello world2")) - - if v, err := s.Get(key); err != nil { - t.Fatal(err) - } else if string(v) != string(value) { - t.Fatal(string(v)) - } -} - -func TestDestroy(t *testing.T) { - db := getTestDB() - - db.Destroy() - - if _, err := os.Stat(db.cfg.Path); !os.IsNotExist(err) { - t.Fatal("must not exist") - } -} diff --git a/leveldb/readme.md b/leveldb/readme.md deleted file mode 100644 index 0389620..0000000 --- a/leveldb/readme.md +++ /dev/null @@ -1,19 +0,0 @@ -a leveldb wrapper for levigo - -simplify use leveldb in go - -# Build leveldb - -see [https://gist.github.com/siddontang/dfbc835e06e47d0f6297](https://gist.github.com/siddontang/dfbc835e06e47d0f6297) for build leveldb - -# Install - -you must first set CGO_CFLAGS, CGO_LDFLAGS to your leveldb and snappy directory. - -dev.sh may help you: - - . ./dev.sh - -# Notice - -I have changed this package to [https://github.com/siddontang/go-leveldb](https://github.com/siddontang/go-leveldb) and will not maintain here anymore. \ No newline at end of file diff --git a/leveldb/snapshot.go b/leveldb/snapshot.go deleted file mode 100644 index eca2041..0000000 --- a/leveldb/snapshot.go +++ /dev/null @@ -1,66 +0,0 @@ -package leveldb - -import ( - "github.com/jmhodges/levigo" -) - -type Snapshot struct { - db *DB - s *levigo.Snapshot - readOpts *levigo.ReadOptions - iteratorOpts *levigo.ReadOptions -} - -func newSnapshot(db *DB) *Snapshot { - s := new(Snapshot) - s.db = db - s.s = db.db.NewSnapshot() - - s.readOpts = levigo.NewReadOptions() - s.readOpts.SetSnapshot(s.s) - - s.iteratorOpts = levigo.NewReadOptions() - s.iteratorOpts.SetSnapshot(s.s) - s.iteratorOpts.SetFillCache(false) - - return s -} - -func (s *Snapshot) Close() { - s.db.db.ReleaseSnapshot(s.s) - - s.iteratorOpts.Close() - s.readOpts.Close() -} - -func (s *Snapshot) Get(key []byte) ([]byte, error) { - return s.db.db.Get(s.readOpts, key) -} - -func (s *Snapshot) Iterator(min []byte, max []byte, rangeType uint8, offset int, limit int) *Iterator { - return newIterator(s.db, s.iteratorOpts, NewRange(min, max, rangeType), offset, limit, IteratorForward) -} - -func (s *Snapshot) RevIterator(min []byte, max []byte, rangeType uint8, offset int, limit int) *Iterator { - return newIterator(s.db, s.iteratorOpts, NewRange(min, max, rangeType), offset, limit, IteratorBackward) -} - -func (s *Snapshot) GetInt(key []byte) (int64, error) { - return Int(s.Get(key)) -} - -func (s *Snapshot) GetUInt(key []byte) (uint64, error) { - return Uint(s.Get(key)) -} - -func (s *Snapshot) GetFloat(key []byte) (float64, error) { - return Float(s.Get(key)) -} - -func (s *Snapshot) GetString(key []byte) (string, error) { - return String(s.Get(key)) -} - -func (s *Snapshot) GetSlice(key []byte) ([]byte, error) { - return Slice(s.Get(key)) -} diff --git a/leveldb/util.go b/leveldb/util.go deleted file mode 100644 index 66b870b..0000000 --- a/leveldb/util.go +++ /dev/null @@ -1,56 +0,0 @@ -package leveldb - -import ( - "github.com/siddontang/golib/hack" - "strconv" -) - -func Int(v []byte, err error) (int64, error) { - if err != nil { - return 0, err - } else if v == nil { - return 0, nil - } - - return strconv.ParseInt(hack.String(v), 10, 64) -} - -func Uint(v []byte, err error) (uint64, error) { - if err != nil { - return 0, err - } else if v == nil { - return 0, nil - } - - return strconv.ParseUint(hack.String(v), 10, 64) -} - -func Float(v []byte, err error) (float64, error) { - if err != nil { - return 0, err - } else if v == nil { - return 0, nil - } - - return strconv.ParseFloat(hack.String(v), 64) -} - -func String(v []byte, err error) (string, error) { - if err != nil { - return "", err - } else if v == nil { - return "", nil - } - - return hack.String(v), nil -} - -func Slice(v []byte, err error) ([]byte, error) { - if err != nil { - return nil, err - } else if v == nil { - return []byte{}, nil - } - - return v, nil -} diff --git a/log/filehandler.go b/log/filehandler.go deleted file mode 100644 index bf7aebb..0000000 --- a/log/filehandler.go +++ /dev/null @@ -1,179 +0,0 @@ -package log - -import ( - "fmt" - "os" - "time" -) - -type FileHandler struct { - fd *os.File -} - -func NewFileHandler(fileName string, flag int) (*FileHandler, error) { - f, err := os.OpenFile(fileName, flag, 0) - if err != nil { - return nil, err - } - - h := new(FileHandler) - - h.fd = f - - return h, nil -} - -func (h *FileHandler) Write(b []byte) (n int, err error) { - return h.fd.Write(b) -} - -func (h *FileHandler) Close() error { - return h.fd.Close() -} - -type RotatingFileHandler struct { - fd *os.File - - fileName string - maxBytes int - backupCount int -} - -func NewRotatingFileHandler(fileName string, maxBytes int, backupCount int) (*RotatingFileHandler, error) { - h := new(RotatingFileHandler) - - h.fileName = fileName - h.maxBytes = maxBytes - h.backupCount = backupCount - - var err error - h.fd, err = os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - if err != nil { - return nil, err - } - - return h, nil -} - -func (h *RotatingFileHandler) Write(p []byte) (n int, err error) { - h.doRollover() - return h.fd.Write(p) -} - -func (h *RotatingFileHandler) Close() error { - if h.fd != nil { - return h.fd.Close() - } - return nil -} - -func (h *RotatingFileHandler) doRollover() { - f, err := h.fd.Stat() - if err != nil { - return - } - - if h.maxBytes <= 0 { - return - } else if f.Size() < int64(h.maxBytes) { - return - } - - if h.backupCount > 0 { - h.fd.Close() - - for i := h.backupCount - 1; i > 0; i-- { - sfn := fmt.Sprintf("%s.%d", h.fileName, i) - dfn := fmt.Sprintf("%s.%d", h.fileName, i+1) - - os.Rename(sfn, dfn) - } - - dfn := fmt.Sprintf("%s.1", h.fileName) - os.Rename(h.fileName, dfn) - - h.fd, _ = os.OpenFile(h.fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - } -} - -//refer: http://docs.python.org/2/library/logging.handlers.html -//same like python TimedRotatingFileHandler - -type TimeRotatingFileHandler struct { - fd *os.File - - baseName string - interval int64 - suffix string - rolloverAt int64 -} - -const ( - WhenSecond = iota - WhenMinute - WhenHour - WhenDay -) - -func NewTimeRotatingFileHandler(baseName string, when int8, interval int) (*TimeRotatingFileHandler, error) { - h := new(TimeRotatingFileHandler) - - h.baseName = baseName - - switch when { - case WhenSecond: - h.interval = 1 - h.suffix = "2006-01-02_15-04-05" - case WhenMinute: - h.interval = 60 - h.suffix = "2006-01-02_15-04" - case WhenHour: - h.interval = 3600 - h.suffix = "2006-01-02_15" - case WhenDay: - h.interval = 3600 * 24 - h.suffix = "2006-01-02" - default: - return nil, fmt.Errorf("invalid when_rotate: %d", when) - } - - h.interval = h.interval * int64(interval) - - var err error - h.fd, err = os.OpenFile(h.baseName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - if err != nil { - return nil, err - } - - fInfo, _ := h.fd.Stat() - h.rolloverAt = fInfo.ModTime().Unix() + h.interval - - return h, nil -} - -func (h *TimeRotatingFileHandler) doRollover() { - //refer http://hg.python.org/cpython/file/2.7/Lib/logging/handlers.py - now := time.Now() - - if h.rolloverAt <= now.Unix() { - fName := h.baseName + now.Format(h.suffix) - h.fd.Close() - e := os.Rename(h.baseName, fName) - if e != nil { - panic(e) - } - - h.fd, _ = os.OpenFile(h.baseName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - - h.rolloverAt = time.Now().Unix() + h.interval - } -} - -func (h *TimeRotatingFileHandler) Write(b []byte) (n int, err error) { - h.doRollover() - return h.fd.Write(b) -} - -func (h *TimeRotatingFileHandler) Close() error { - return h.fd.Close() -} diff --git a/log/handler.go b/log/handler.go deleted file mode 100644 index 66257ad..0000000 --- a/log/handler.go +++ /dev/null @@ -1,45 +0,0 @@ -package log - -import ( - "io" -) - -type Handler interface { - Write(p []byte) (n int, err error) - Close() error -} - -type StreamHandler struct { - w io.Writer -} - -func NewStreamHandler(w io.Writer) (*StreamHandler, error) { - h := new(StreamHandler) - - h.w = w - - return h, nil -} - -func (h *StreamHandler) Write(b []byte) (n int, err error) { - return h.w.Write(b) -} - -func (h *StreamHandler) Close() error { - return nil -} - -type NullHandler struct { -} - -func NewNullHandler() (*NullHandler, error) { - return new(NullHandler), nil -} - -func (h *NullHandler) Write(b []byte) (n int, err error) { - return len(b), nil -} - -func (h *NullHandler) Close() { - -} diff --git a/log/log.go b/log/log.go deleted file mode 100644 index 9e8ed41..0000000 --- a/log/log.go +++ /dev/null @@ -1,186 +0,0 @@ -package log - -import ( - "fmt" - "os" - "runtime" - "time" -) - -const ( - LevelTrace = iota - LevelDebug - LevelInfo - LevelWarn - LevelError - LevelFatal -) - -const ( - Ltime = iota << 1 //time format "2006/01/02 15:04:05" - Lfile //file.go:123 - Llevel //[Trace|Debug|Info...] -) - -var LevelName [6]string = [6]string{"Trace", "Debug", "Info", "Warn", "Error", "Fatal"} - -const TimeFormat = "2006/01/02 15:04:05" - -type Logger struct { - level int - flag int - - handler Handler - - quit chan struct{} - msg chan []byte -} - -func New(handler Handler, flag int) *Logger { - var l = new(Logger) - - l.level = LevelInfo - l.handler = handler - - l.flag = flag - - l.quit = make(chan struct{}) - - l.msg = make(chan []byte, 1024) - - go l.run() - - return l -} - -func NewDefault(handler Handler) *Logger { - return New(handler, Ltime|Lfile|Llevel) -} - -func newStdHandler() *StreamHandler { - h, _ := NewStreamHandler(os.Stdout) - return h -} - -var std = NewDefault(newStdHandler()) - -func (l *Logger) run() { - for { - select { - case msg := <-l.msg: - l.handler.Write(msg) - case <-l.quit: - l.handler.Close() - } - } -} - -func (l *Logger) Close() { - if l.quit == nil { - return - } - - close(l.quit) - l.quit = nil -} - -func (l *Logger) SetLevel(level int) { - l.level = level -} - -func (l *Logger) Output(callDepth int, level int, format string, v ...interface{}) { - if l.level > level { - return - } - - buf := make([]byte, 0, 1024) - - if l.flag&Ltime > 0 { - now := time.Now().Format(TimeFormat) - buf = append(buf, now...) - buf = append(buf, " "...) - } - - if l.flag&Lfile > 0 { - _, file, line, ok := runtime.Caller(callDepth) - if !ok { - file = "???" - line = 0 - } else { - for i := len(file) - 1; i > 0; i-- { - if file[i] == '/' { - file = file[i+1:] - break - } - } - } - - buf = append(buf, fmt.Sprintf("%s:%d ", file, line)...) - } - - if l.flag&Llevel > 0 { - buf = append(buf, fmt.Sprintf("[%s] ", LevelName[level])...) - } - - s := fmt.Sprintf(format, v...) - - buf = append(buf, s...) - - if s[len(s)-1] != '\n' { - buf = append(buf, "\n"...) - } - - l.msg <- buf -} - -func (l *Logger) Trace(format string, v ...interface{}) { - l.Output(2, LevelTrace, format, v...) -} - -func (l *Logger) Debug(format string, v ...interface{}) { - l.Output(2, LevelDebug, format, v...) -} - -func (l *Logger) Info(format string, v ...interface{}) { - l.Output(2, LevelInfo, format, v...) -} - -func (l *Logger) Warn(format string, v ...interface{}) { - l.Output(2, LevelWarn, format, v...) -} - -func (l *Logger) Error(format string, v ...interface{}) { - l.Output(2, LevelError, format, v...) -} - -func (l *Logger) Fatal(format string, v ...interface{}) { - l.Output(2, LevelFatal, format, v...) -} - -func SetLevel(level int) { - std.SetLevel(level) -} - -func Trace(format string, v ...interface{}) { - std.Output(2, LevelTrace, format, v...) -} - -func Debug(format string, v ...interface{}) { - std.Output(2, LevelDebug, format, v...) -} - -func Info(format string, v ...interface{}) { - std.Output(2, LevelInfo, format, v...) -} - -func Warn(format string, v ...interface{}) { - std.Output(2, LevelWarn, format, v...) -} - -func Error(format string, v ...interface{}) { - std.Output(2, LevelError, format, v...) -} - -func Fatal(format string, v ...interface{}) { - std.Output(2, LevelFatal, format, v...) -} diff --git a/log/log_test.go b/log/log_test.go deleted file mode 100644 index 67d3b0b..0000000 --- a/log/log_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package log - -import ( - "os" - "testing" -) - -func TestStdStreamLog(t *testing.T) { - h, _ := NewStreamHandler(os.Stdout) - s := NewDefault(h) - s.Info("hello world") - - s.Close() - - Info("hello world") -} - -func TestRotatingFileLog(t *testing.T) { - path := "./test_log" - os.RemoveAll(path) - - os.Mkdir(path, 0777) - fileName := path + "/test" - - h, err := NewRotatingFileHandler(fileName, 10, 2) - if err != nil { - t.Fatal(err) - } - - buf := make([]byte, 10) - - h.Write(buf) - - h.Write(buf) - - if _, err := os.Stat(fileName + ".1"); err != nil { - t.Fatal(err) - } - - if _, err := os.Stat(fileName + ".2"); err == nil { - t.Fatal(err) - } - - h.Write(buf) - if _, err := os.Stat(fileName + ".2"); err != nil { - t.Fatal(err) - } - - h.Close() - - os.RemoveAll(path) -} diff --git a/log/loggingd/main.go b/log/loggingd/main.go deleted file mode 100644 index 8c50bb4..0000000 --- a/log/loggingd/main.go +++ /dev/null @@ -1,21 +0,0 @@ -package main - -import ( - "flag" - "github.com/siddontang/golib/log" -) - -var logFile = flag.String("logfile", "./logd.log", "file to log") -var net = flag.String("net", "tcp", "server listen protocol, like tcp, udp or unix") -var addr = flag.String("addr", "127.0.0.1:11183", "server listen address") - -func main() { - flag.Parse() - - s, err := log.NewServer(*logFile, *net, *addr) - if err != nil { - panic(err) - } - - s.Run() -} diff --git a/log/readme.md b/log/readme.md deleted file mode 100644 index cded1c5..0000000 --- a/log/readme.md +++ /dev/null @@ -1,3 +0,0 @@ -# Notice - -I have changed this package to [https://github.com/siddontang/go-log](https://github.com/siddontang/go-log) and will not maintain here anymore. \ No newline at end of file diff --git a/log/server.go b/log/server.go deleted file mode 100644 index db3bf6b..0000000 --- a/log/server.go +++ /dev/null @@ -1,95 +0,0 @@ -package log - -import ( - "bufio" - "encoding/binary" - "io" - "net" - "os" - "path" -) - -//a log server for handling SocketHandler send log - -type Server struct { - closed bool - listener net.Listener - fd *os.File -} - -func NewServer(fileName string, protocol string, addr string) (*Server, error) { - s := new(Server) - - s.closed = false - - var err error - - dir := path.Dir(fileName) - os.Mkdir(dir, 0777) - - s.fd, err = os.OpenFile(fileName, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) - if err != nil { - return nil, err - } - - s.listener, err = net.Listen(protocol, addr) - if err != nil { - return nil, err - } - - return s, nil -} - -func (s *Server) Close() error { - if s.closed { - return nil - } - - s.closed = true - - s.fd.Close() - - s.listener.Close() - return nil -} - -func (s *Server) Run() { - for { - conn, err := s.listener.Accept() - if err != nil { - continue - } - - go s.onRead(conn) - } -} - -func (s *Server) onRead(c net.Conn) { - br := bufio.NewReaderSize(c, 1024) - - var bufLen uint32 - - for { - if err := binary.Read(br, binary.BigEndian, &bufLen); err != nil { - c.Close() - return - } - - buf := make([]byte, bufLen, bufLen+1) - - if _, err := io.ReadFull(br, buf); err != nil && err != io.ErrUnexpectedEOF { - c.Close() - return - } else { - if len(buf) == 0 { - continue - } - if buf[len(buf)-1] != '\n' { - buf = append(buf, '\n') - } - - s.fd.Write(buf) - } - - } -} diff --git a/log/socket_test.go b/log/socket_test.go deleted file mode 100644 index 6fd7d35..0000000 --- a/log/socket_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package log - -import ( - "io" - "os" - "testing" - "time" -) - -func TestSocket(t *testing.T) { - fileName := "./test_server.log" - - os.Remove(fileName) - - s, err := NewServer(fileName, "tcp", "127.0.0.1:11183") - if err != nil { - t.Fatal(err) - } - go s.Run() - defer s.Close() - - var h *SocketHandler - h, err = NewSocketHandler("tcp", "127.0.0.1:11183") - - _, err = h.Write([]byte("hello world")) - if err != nil { - t.Fatal(err) - } - - time.Sleep(1 * time.Second) - - s.Close() - - var f *os.File - f, err = os.Open(fileName) - if err != nil { - t.Fatal(err) - } - - defer f.Close() - - buf := make([]byte, 64) - var n int - n, err = f.Read(buf) - if err != nil && err != io.EOF { - t.Fatal(err) - } - - buf = buf[0:n] - - if string(buf) != "hello world\n" { - t.Fatal(string(buf)) - } - - os.Remove(fileName) -} diff --git a/log/sockethandler.go b/log/sockethandler.go deleted file mode 100644 index f19db05..0000000 --- a/log/sockethandler.go +++ /dev/null @@ -1,62 +0,0 @@ -package log - -import ( - "encoding/binary" - "net" - "time" -) - -type SocketHandler struct { - c net.Conn - protocol string - addr string -} - -func NewSocketHandler(protocol string, addr string) (*SocketHandler, error) { - s := new(SocketHandler) - - s.protocol = protocol - s.addr = addr - - return s, nil -} - -func (h *SocketHandler) Write(p []byte) (n int, err error) { - if err = h.connect(); err != nil { - return - } - - buf := make([]byte, len(p)+4) - - binary.BigEndian.PutUint32(buf, uint32(len(p))) - - copy(buf[4:], p) - - n, err = h.c.Write(buf) - if err != nil { - h.c.Close() - h.c = nil - } - return -} - -func (h *SocketHandler) Close() error { - if h.c != nil { - h.c.Close() - } - return nil -} - -func (h *SocketHandler) connect() error { - if h.c != nil { - return nil - } - - var err error - h.c, err = net.DialTimeout(h.protocol, h.addr, 20*time.Second) - if err != nil { - return err - } - - return nil -} diff --git a/websocket/client.go b/websocket/client.go deleted file mode 100644 index 6860fa8..0000000 --- a/websocket/client.go +++ /dev/null @@ -1,60 +0,0 @@ -package websocket - -import ( - "bytes" - "errors" - "net" - "net/http" - "net/url" - "strings" -) - -var ( - ErrBadHandshake = errors.New("bad handshake") -) - -func NewClient(netConn net.Conn, u *url.URL, requestHeader http.Header) (c *Conn, response *http.Response, err error) { - key, err := calcKey() - if err != nil { - return nil, nil, err - } - acceptKey := calcAcceptKey(key) - - c = NewConn(netConn, false) - - buf := bytes.NewBufferString("GET ") - buf.WriteString(u.RequestURI()) - buf.WriteString(" HTTP/1.1\r\nHost: ") - buf.WriteString(u.Host) - buf.WriteString("\r\nUpgrade: websocket\r\nConnection: upgrade\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: ") - buf.WriteString(key) - buf.WriteString("\r\n") - - for k, vs := range requestHeader { - for _, v := range vs { - buf.WriteString(k) - buf.WriteString(": ") - buf.WriteString(v) - buf.WriteString("\r\n") - } - } - - buf.WriteString("\r\n") - p := buf.Bytes() - if _, err := netConn.Write(p); err != nil { - return nil, nil, err - } - - resp, err := http.ReadResponse(c.br, &http.Request{Method: "GET", URL: u}) - if err != nil { - return nil, nil, err - } - - if resp.StatusCode != 101 || - !strings.EqualFold(resp.Header.Get("Upgrade"), "websocket") || - !strings.EqualFold(resp.Header.Get("Connection"), "upgrade") || - resp.Header.Get("Sec-Websocket-Accept") != acceptKey { - return nil, resp, ErrBadHandshake - } - return c, resp, nil -} diff --git a/websocket/conn.go b/websocket/conn.go deleted file mode 100644 index 7519e23..0000000 --- a/websocket/conn.go +++ /dev/null @@ -1,323 +0,0 @@ -package websocket - -import ( - "bufio" - "encoding/binary" - "errors" - "io" - "lib/log" - "math/rand" - "net" - "time" -) - -//refer RFC6455 - -const ( - TextMessage byte = 1 - BinaryMessage byte = 2 - CloseMessage byte = 8 - PingMessage byte = 9 - PongMessage byte = 10 -) - -var ( - ErrControlTooLong = errors.New("control message too long") - ErrRSVNotSupport = errors.New("reserved bit not support") - ErrPayloadError = errors.New("payload length error") - ErrControlFragmented = errors.New("control message can not be fragmented") - ErrNotTCPConn = errors.New("not a tcp connection") - ErrWriteError = errors.New("write error") -) - -type Conn struct { - conn net.Conn - - br *bufio.Reader - - isServer bool -} - -func NewConn(conn net.Conn, isServer bool) *Conn { - c := new(Conn) - - c.conn = conn - - c.br = bufio.NewReader(conn) - - c.isServer = isServer - - return c -} - -func (c *Conn) ReadMessage() (messageType byte, message []byte, err error) { - return c.Read() -} - -func (c *Conn) Read() (messageType byte, message []byte, err error) { - buf := make([]byte, 8, 8) - - message = []byte{} - - messageType = 0 - - for { - opcode, data, err := c.readFrame(buf) - - if err != nil { - return messageType, message, err - } - - message = append(message, data...) - - if opcode&0x80 != 0 { - //final - if opcode&0x0F > 0 { - //not continue frame - messageType = opcode & 0x0F - } - return messageType, message, nil - - } else { - if opcode&0x0F > 0 { - //first continue frame - messageType = opcode & 0x0F - } - } - } - - return -} - -func (c *Conn) Write(message []byte, binary bool) error { - if binary { - return c.sendFrame(BinaryMessage, message) - } else { - return c.sendFrame(TextMessage, message) - } -} - -func (c *Conn) WriteMessage(messageType byte, message []byte) error { - return c.sendFrame(messageType, message) -} - -//write utf-8 text message -func (c *Conn) WriteString(message []byte) error { - return c.Write(message, false) -} - -//write binary message -func (c *Conn) WriteBinary(message []byte) error { - return c.Write(message, true) -} - -func (c *Conn) Ping(message []byte) error { - return c.sendFrame(PingMessage, message) -} - -func (c *Conn) Pong(message []byte) error { - return c.sendFrame(PongMessage, message) -} - -//close socket, not send websocket close message -func (c *Conn) Close() error { - return c.conn.Close() -} - -func (c *Conn) LocalAddr() net.Addr { - return c.conn.LocalAddr() -} - -func (c *Conn) RemoteAddr() net.Addr { - return c.conn.RemoteAddr() -} - -func (c *Conn) SetReadDeadline(t time.Time) error { - return c.conn.SetReadDeadline(t) -} - -func (c *Conn) SetWriteDeadline(t time.Time) error { - return c.conn.SetWriteDeadline(t) -} - -func (c *Conn) SetReadBuffer(bytes int) error { - if tcpConn, ok := c.conn.(*net.TCPConn); ok { - return tcpConn.SetReadBuffer(bytes) - } else { - return ErrNotTCPConn - } -} - -func (c *Conn) SetWriteBuffer(bytes int) error { - if tcpConn, ok := c.conn.(*net.TCPConn); ok { - return tcpConn.SetWriteBuffer(bytes) - } else { - return ErrNotTCPConn - } -} - -func (c *Conn) readPayloadLen(length byte, buf []byte) (payloadLen uint64, err error) { - if length < 126 { - payloadLen = uint64(length) - } else if length == 126 { - err = c.read(buf[:2]) - if err != nil { - return - } - payloadLen = uint64(binary.BigEndian.Uint16(buf[:2])) - } else if length == 127 { - err = c.read(buf[:8]) - if err != nil { - return - } - payloadLen = uint64(binary.BigEndian.Uint16(buf[:8])) - } - - return -} - -func (c *Conn) readFrame(buf []byte) (opcode byte, messsage []byte, err error) { - //minimum head may 2 byte - - err = c.read(buf[:2]) - if err != nil { - return - } - - opcode = buf[0] - - if opcode&0x70 > 0 { - err = ErrRSVNotSupport - return - } - - //isMasking := (0x80 & buf[1]) > 0 - isMasking := (0x80 & buf[1]) > 0 - - var payloadLen uint64 - payloadLen, err = c.readPayloadLen(buf[1]&0x7F, buf) - if err != nil { - return - } - - if opcode&0x08 > 0 && payloadLen > 125 { - err = ErrControlTooLong - return - } - - var masking []byte - - if isMasking { - err = c.read(buf[:4]) - if err != nil { - return - } - - masking = buf[:4] - } - - messsage = make([]byte, payloadLen) - err = c.read(messsage) - - if err != nil { - return - } - - if isMasking { - //maskingKey := c.newMaskingKey() - c.maskingData(messsage, masking) - } - - return -} - -func (c *Conn) sendFrame(opcode byte, message []byte) error { - //max frame header may 14 length - buf := make([]byte, 0, len(message)+14) - //here we don not support continue frame, all are final - opcode |= 0x80 - - if opcode&0x08 > 0 && len(message) >= 126 { - return ErrControlTooLong - } - - buf = append(buf, opcode) - - //no mask, because chrome may not support - var mask byte = 0x00 - - if !c.isServer { - //for client, we will mask data - mask = 0x80 - } - - payloadLen := len(message) - - if payloadLen < 126 { - buf = append(buf, mask|byte(payloadLen)) - } else if payloadLen <= 0xFFFF { - buf = append(buf, mask|byte(126), 0, 0) - - binary.BigEndian.PutUint16(buf[len(buf)-2:], uint16(payloadLen)) - } else { - buf = append(buf, mask|byte(127), 0, 0, 0, 0, 0, 0, 0, 0) - - binary.BigEndian.PutUint64(buf[len(buf)-8:], uint64(payloadLen)) - } - - if !c.isServer { - maskingKey := c.newMaskingKey() - buf = append(buf, maskingKey...) - - pos := len(buf) - buf = append(buf, message...) - - c.maskingData(buf[pos:], maskingKey) - - } else { - buf = append(buf, message...) - } - - tmpBuf := buf - for i := 0; i < 3; i++ { - n, err := c.conn.Write(tmpBuf) - if err != nil { - return err - } - if n == len(tmpBuf) { - return nil - } else { - log.Warn("[conn write] buffer_size=%d return_size=%s", len(tmpBuf), n) - tmpBuf = tmpBuf[n:] - } - } - return ErrWriteError -} - -func (c *Conn) read(buf []byte) error { - var err error - for len(buf) > 0 && err == nil { - var nn int - nn, err = c.br.Read(buf) - buf = buf[nn:] - } - if err == io.EOF { - if len(buf) == 0 { - err = nil - } else { - err = io.ErrUnexpectedEOF - } - } - return err -} - -func (c *Conn) maskingData(data []byte, maskingKey []byte) { - for i := range data { - data[i] ^= maskingKey[i%4] - } -} - -func (c *Conn) newMaskingKey() []byte { - n := rand.Uint32() - return []byte{byte(n), byte(n >> 8), byte(n >> 16), byte(n >> 32)} -} diff --git a/websocket/server.go b/websocket/server.go deleted file mode 100644 index ab09539..0000000 --- a/websocket/server.go +++ /dev/null @@ -1,105 +0,0 @@ -package websocket - -import ( - "bufio" - "bytes" - "errors" - "net" - "net/http" - "strings" -) - -var ( - ErrInvalidMethod = errors.New("Only GET Supported") - ErrInvalidVersion = errors.New("Sec-Websocket-Version: 13") - ErrInvalidUpgrade = errors.New("Can \"Upgrade\" only to \"WebSocket\"") - ErrInvalidConnection = errors.New("\"Connection\" must be \"Upgrade\"") - ErrMissingKey = errors.New("Missing Key") - ErrHijacker = errors.New("Not implement http.Hijacker") - ErrNoEmptyConn = errors.New("Conn ReadBuf must be empty") -) - -func Upgrade(w http.ResponseWriter, r *http.Request, responseHeader http.Header) (*Conn, error) { - if r.Method != "GET" { - return nil, ErrInvalidMethod - } - - if r.Header.Get("Sec-Websocket-Version") != "13" { - return nil, ErrInvalidVersion - } - - if strings.ToLower(r.Header.Get("Upgrade")) != "websocket" { - return nil, ErrInvalidUpgrade - } - - if strings.ToLower(r.Header.Get("Connection")) != "upgrade" { - return nil, ErrInvalidConnection - } - - var acceptKey string - - if key := r.Header.Get("Sec-Websocket-key"); len(key) == 0 { - return nil, ErrMissingKey - } else { - acceptKey = calcAcceptKey(key) - } - - var ( - netConn net.Conn - br *bufio.Reader - err error - ) - - h, ok := w.(http.Hijacker) - if !ok { - return nil, ErrHijacker - } - - var rw *bufio.ReadWriter - netConn, rw, err = h.Hijack() - br = rw.Reader - - if br.Buffered() > 0 { - netConn.Close() - return nil, ErrNoEmptyConn - } - - c := NewConn(netConn, true) - - buf := bytes.NewBufferString("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ") - - buf.WriteString(acceptKey) - buf.WriteString("\r\n") - - subProtol := selectSubProtocol(r) - if len(subProtol) > 0 { - buf.WriteString("Sec-Websocket-Protocol: ") - buf.WriteString(subProtol) - buf.WriteString("\r\n") - } - - for k, vs := range responseHeader { - for _, v := range vs { - buf.WriteString(k) - buf.WriteString(": ") - buf.WriteString(v) - buf.WriteString("\r\n") - } - } - buf.WriteString("\r\n") - - if _, err = netConn.Write(buf.Bytes()); err != nil { - netConn.Close() - return nil, err - } - - return c, nil -} - -func selectSubProtocol(r *http.Request) string { - h := r.Header.Get("Sec-Websocket-Protocol") - if len(h) == 0 { - return "" - } - return strings.Split(h, ",")[0] -} diff --git a/websocket/util.go b/websocket/util.go deleted file mode 100644 index 5e156cb..0000000 --- a/websocket/util.go +++ /dev/null @@ -1,26 +0,0 @@ -package websocket - -import ( - "crypto/rand" - "crypto/sha1" - "encoding/base64" - "errors" - "io" -) - -var keyGUID = []byte("258EAFA5-E914-47DA-95CA-C5AB0DC85B11") - -func calcAcceptKey(key string) string { - h := sha1.New() - h.Write([]byte(key)) - h.Write(keyGUID) - return base64.StdEncoding.EncodeToString(h.Sum(nil)) -} - -func calcKey() (string, error) { - p := make([]byte, 16) - if _, err := io.ReadFull(rand.Reader, p); err != nil { - return "", err - } - return base64.StdEncoding.EncodeToString(p), nil -}