mirror
/
go
mirror of https://github.com/siddontang/go.git
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

remove unnecessary package

This commit is contained in:
siddontang 2014-09-04 21:42:05 +08:00
parent cde2e20315
commit 7c4b752908
21 changed files with 0 additions and 1996 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
leveldb/batch.go
View File

@ -1,35 +0,0 @@
package leveldb
import (
"github.com/jmhodges/levigo"
)
type WriteBatch struct {
db *DB
wb *levigo.WriteBatch
}
func (wb *WriteBatch) Put(key, value []byte) {
wb.wb.Put(key, value)
}
func (wb *WriteBatch) Delete(key []byte) {
wb.wb.Delete(key)
}
func (wb *WriteBatch) Commit() error {
return wb.db.db.Write(wb.db.writeOpts, wb.wb)
}
func (wb *WriteBatch) Rollback() {
wb.wb.Clear()
}
func (wb *WriteBatch) Close() {
if wb.wb == nil {
return
}
wb.wb.Close()
wb.wb = nil
}

178
leveldb/db.go
View File

@ -1,178 +0,0 @@
package leveldb
import (
"encoding/json"
"github.com/jmhodges/levigo"
)
const defaultFilterBits int = 10
type Config struct {
Path string `json:"path"`
Compression bool `json:"compression"`
BlockSize int `json:"block_size"`
WriteBufferSize int `json:"write_buffer_size"`
CacheSize int `json:"cache_size"`
}
type DB struct {
cfg *Config
db *levigo.DB
opts *levigo.Options
//for default read and write options
readOpts *levigo.ReadOptions
writeOpts *levigo.WriteOptions
iteratorOpts *levigo.ReadOptions
cache *levigo.Cache
filter *levigo.FilterPolicy
}
func OpenWithConfig(cfg *Config) (*DB, error) {
db := new(DB)
db.cfg = cfg
db.opts = db.initOptions(cfg)
db.readOpts = levigo.NewReadOptions()
db.writeOpts = levigo.NewWriteOptions()
db.iteratorOpts = levigo.NewReadOptions()
db.iteratorOpts.SetFillCache(false)
var err error
db.db, err = levigo.Open(cfg.Path, db.opts)
return db, err
}
func Open(configJson json.RawMessage) (*DB, error) {
cfg := new(Config)
err := json.Unmarshal(configJson, cfg)
if err != nil {
return nil, err
}
return OpenWithConfig(cfg)
}
func (db *DB) initOptions(cfg *Config) *levigo.Options {
opts := levigo.NewOptions()
opts.SetCreateIfMissing(true)
if cfg.CacheSize > 0 {
db.cache = levigo.NewLRUCache(cfg.CacheSize)
opts.SetCache(db.cache)
}
//we must use bloomfilter
db.filter = levigo.NewBloomFilter(defaultFilterBits)
opts.SetFilterPolicy(db.filter)
if !cfg.Compression {
opts.SetCompression(levigo.NoCompression)
}
if cfg.BlockSize > 0 {
opts.SetBlockSize(cfg.BlockSize)
}
if cfg.WriteBufferSize > 0 {
opts.SetWriteBufferSize(cfg.WriteBufferSize)
}
return opts
}
func (db *DB) Close() {
db.opts.Close()
if db.cache != nil {
db.cache.Close()
}
if db.filter != nil {
db.filter.Close()
}
db.readOpts.Close()
db.writeOpts.Close()
db.iteratorOpts.Close()
db.db.Close()
db.db = nil
}
func (db *DB) Destroy() {
db.Close()
opts := levigo.NewOptions()
defer opts.Close()
levigo.DestroyDatabase(db.cfg.Path, opts)
}
func (db *DB) Clear() {
it := db.Iterator(nil, nil, 0, 0, -1)
for ; it.Valid(); it.Next() {
db.Delete(it.Key())
}
}
func (db *DB) Put(key, value []byte) error {
return db.db.Put(db.writeOpts, key, value)
}
func (db *DB) Get(key []byte) ([]byte, error) {
return db.db.Get(db.readOpts, key)
}
func (db *DB) Delete(key []byte) error {
return db.db.Delete(db.writeOpts, key)
}
func (db *DB) NewWriteBatch() *WriteBatch {
wb := new(WriteBatch)
wb.wb = levigo.NewWriteBatch()
wb.db = db
return wb
}
//limit < 0, unlimit
//offset must >= 0, if < 0, will get nothing
func (db *DB) Iterator(min []byte, max []byte, rangeType uint8, offset int, limit int) *Iterator {
return newIterator(db, db.iteratorOpts, NewRange(min, max, rangeType), offset, limit, IteratorForward)
}
//limit < 0, unlimit
//offset must >= 0, if < 0, will get nothing
func (db *DB) RevIterator(min []byte, max []byte, rangeType uint8, offset int, limit int) *Iterator {
return newIterator(db, db.iteratorOpts, NewRange(min, max, rangeType), offset, limit, IteratorBackward)
}
func (db *DB) NewSnapshot() *Snapshot {
return newSnapshot(db)
}
func (db *DB) GetInt(key []byte) (int64, error) {
return Int(db.Get(key))
}
func (db *DB) GetUInt(key []byte) (uint64, error) {
return Uint(db.Get(key))
}
func (db *DB) GetFloat(key []byte) (float64, error) {
return Float(db.Get(key))
}
func (db *DB) GetString(key []byte) (string, error) {
return String(db.Get(key))
}
func (db *DB) GetSlice(key []byte) ([]byte, error) {
return Slice(db.Get(key))
}

12
leveldb/dev.sh
View File

@ -1,12 +0,0 @@
#!/bin/bash
SNAPPY_DIR=/usr/local/snappy
LEVELDB_DIR=/usr/local/leveldb
export CGO_CFLAGS="-I$LEVELDB_DIR/include -I$SNAPPY_DIR/include"
export CGO_CXXFLAGS="-I$LEVELDB_DIR/include -I$SNAPPY_DIR/include"
export CGO_LDFLAGS="-L$LEVELDB_DIR/lib -L$SNAPPY_DIR/lib -lsnappy"
export LD_LIBRARY_PATH=$(add_path $LD_LIBRARY_PATH $SNAPPY_DIR/lib)
export LD_LIBRARY_PATH=$(add_path $LD_LIBRARY_PATH $LEVELDB_DIR/lib)
go get github.com/jmhodges/levigo

189
leveldb/iterator.go
View File

@ -1,189 +0,0 @@
package leveldb
import (
"bytes"
"github.com/jmhodges/levigo"
)
const (
IteratorForward uint8 = 0
IteratorBackward uint8 = 1
)
const (
RangeClose uint8 = 0x00
RangeLOpen uint8 = 0x01
RangeROpen uint8 = 0x10
RangeOpen uint8 = 0x11
)
//min must less or equal than max
//range type:
//close: [min, max]
//open: (min, max)
//lopen: (min, max]
//ropen: [min, max)
type Range struct {
Min []byte
Max []byte
Type uint8
}
func NewRange(min []byte, max []byte, tp uint8) *Range {
return &Range{min, max, tp}
}
type Iterator struct {
it *levigo.Iterator
r *Range
offset int
limit int
step int
//0 for IteratorForward, 1 for IteratorBackward
direction uint8
}
func newIterator(db *DB, opts *levigo.ReadOptions, r *Range, offset int, limit int, direction uint8) *Iterator {
it := new(Iterator)
it.it = db.db.NewIterator(opts)
it.r = r
it.offset = offset
it.limit = limit
it.direction = direction
it.step = 0
if offset < 0 {
return it
}
if direction == IteratorForward {
if r.Min == nil {
it.it.SeekToFirst()
} else {
it.it.Seek(r.Min)
if r.Type&RangeLOpen > 0 {
if it.it.Valid() && bytes.Equal(it.it.Key(), r.Min) {
it.it.Next()
}
}
}
} else {
if r.Max == nil {
it.it.SeekToLast()
} else {
it.it.Seek(r.Max)
if !it.it.Valid() {
it.it.SeekToLast()
} else {
if !bytes.Equal(it.it.Key(), r.Max) {
it.it.Prev()
}
}
if r.Type&RangeROpen > 0 {
if it.Valid() && bytes.Equal(it.Key(), r.Max) {
it.it.Prev()
}
}
}
}
for i := 0; i < offset; i++ {
if it.it.Valid() {
if it.direction == IteratorForward {
it.it.Next()
} else {
it.it.Prev()
}
}
}
return it
}
func (it *Iterator) Valid() bool {
if it.offset < 0 {
return false
} else if !it.it.Valid() {
return false
} else if it.limit >= 0 && it.step >= it.limit {
return false
}
if it.direction == IteratorForward {
if it.r.Max != nil {
r := bytes.Compare(it.Key(), it.r.Max)
if it.r.Type&RangeROpen > 0 {
return !(r >= 0)
} else {
return !(r > 0)
}
}
} else {
if it.r.Min != nil {
r := bytes.Compare(it.Key(), it.r.Min)
if it.r.Type&RangeLOpen > 0 {
return !(r <= 0)
} else {
return !(r < 0)
}
}
}
return true
}
func (it *Iterator) GetError() error {
return it.it.GetError()
}
func (it *Iterator) Next() {
it.step++
if it.direction == IteratorForward {
it.it.Next()
} else {
it.it.Prev()
}
}
func (it *Iterator) Key() []byte {
return it.it.Key()
}
func (it *Iterator) Value() []byte {
return it.it.Value()
}
func (it *Iterator) Close() {
it.it.Close()
}
func (it *Iterator) IntValue() (int64, error) {
return Int(it.Value(), nil)
}
func (it *Iterator) UintValue() (uint64, error) {
return Uint(it.Value(), nil)
}
func (it *Iterator) FloatValue() (float64, error) {
return Float(it.Value(), nil)
}
func (it *Iterator) StringValue() (string, error) {
return String(it.Value(), nil)
}
func (it *Iterator) SliceValue() ([]byte, error) {
return Slice(it.Value(), nil)
}

228
leveldb/leveldb_test.go
View File

@ -1,228 +0,0 @@
package leveldb
import (
"bytes"
"fmt"
"os"
"sync"
"testing"
)
var testConfigJson = []byte(`
{
"path" : "./testdb",
"compression":true,
"block_size" : 32768,
"write_buffer_size" : 2097152,
"cache_size" : 20971520
}
`)
var testOnce sync.Once
var testDB *DB
func getTestDB() *DB {
f := func() {
var err error
testDB, err = Open(testConfigJson)
if err != nil {
println(err.Error())
panic(err)
}
}
testOnce.Do(f)
return testDB
}
func TestSimple(t *testing.T) {
db := getTestDB()
key := []byte("key")
value := []byte("hello world")
if err := db.Put(key, value); err != nil {
t.Fatal(err)
}
if v, err := db.Get(key); err != nil {
t.Fatal(err)
} else if !bytes.Equal(v, value) {
t.Fatal("not equal")
}
if err := db.Delete(key); err != nil {
t.Fatal(err)
}
if v, err := db.Get(key); err != nil {
t.Fatal(err)
} else if v != nil {
t.Fatal("must nil")
}
}
func TestBatch(t *testing.T) {
db := getTestDB()
key1 := []byte("key1")
key2 := []byte("key2")
value := []byte("hello world")
db.Put(key1, value)
db.Put(key2, value)
wb := db.NewWriteBatch()
defer wb.Close()
wb.Delete(key2)
wb.Put(key1, []byte("hello world2"))
if err := wb.Commit(); err != nil {
t.Fatal(err)
}
if v, err := db.Get(key2); err != nil {
t.Fatal(err)
} else if v != nil {
t.Fatal("must nil")
}
if v, err := db.Get(key1); err != nil {
t.Fatal(err)
} else if string(v) != "hello world2" {
t.Fatal(string(v))
}
wb.Delete(key1)
wb.Rollback()
if v, err := db.Get(key1); err != nil {
t.Fatal(err)
} else if string(v) != "hello world2" {
t.Fatal(string(v))
}
db.Delete(key1)
}
func checkIterator(it *Iterator, cv ...int) error {
v := make([]string, 0, len(cv))
for ; it.Valid(); it.Next() {
k := it.Key()
v = append(v, string(k))
}
it.Close()
if len(v) != len(cv) {
return fmt.Errorf("len error %d != %d", len(v), len(cv))
}
for k, i := range cv {
if fmt.Sprintf("key_%d", i) != v[k] {
return fmt.Errorf("%s, %d", v[k], i)
}
}
return nil
}
func TestIterator(t *testing.T) {
db := getTestDB()
db.Clear()
for i := 0; i < 10; i++ {
key := []byte(fmt.Sprintf("key_%d", i))
value := []byte("")
db.Put(key, value)
}
var it *Iterator
k := func(i int) []byte {
return []byte(fmt.Sprintf("key_%d", i))
}
it = db.Iterator(k(1), k(5), RangeClose, 0, -1)
if err := checkIterator(it, 1, 2, 3, 4, 5); err != nil {
t.Fatal(err)
}
it = db.Iterator(k(1), k(5), RangeClose, 1, 3)
if err := checkIterator(it, 2, 3, 4); err != nil {
t.Fatal(err)
}
it = db.Iterator(k(1), k(5), RangeLOpen, 0, -1)
if err := checkIterator(it, 2, 3, 4, 5); err != nil {
t.Fatal(err)
}
it = db.Iterator(k(1), k(5), RangeROpen, 0, -1)
if err := checkIterator(it, 1, 2, 3, 4); err != nil {
t.Fatal(err)
}
it = db.Iterator(k(1), k(5), RangeOpen, 0, -1)
if err := checkIterator(it, 2, 3, 4); err != nil {
t.Fatal(err)
}
it = db.RevIterator(k(1), k(5), RangeClose, 0, -1)
if err := checkIterator(it, 5, 4, 3, 2, 1); err != nil {
t.Fatal(err)
}
it = db.RevIterator(k(1), k(5), RangeClose, 1, 3)
if err := checkIterator(it, 4, 3, 2); err != nil {
t.Fatal(err)
}
it = db.RevIterator(k(1), k(5), RangeLOpen, 0, -1)
if err := checkIterator(it, 5, 4, 3, 2); err != nil {
t.Fatal(err)
}
it = db.RevIterator(k(1), k(5), RangeROpen, 0, -1)
if err := checkIterator(it, 4, 3, 2, 1); err != nil {
t.Fatal(err)
}
it = db.RevIterator(k(1), k(5), RangeOpen, 0, -1)
if err := checkIterator(it, 4, 3, 2); err != nil {
t.Fatal(err)
}
}
func TestSnapshot(t *testing.T) {
db := getTestDB()
key := []byte("key")
value := []byte("hello world")
db.Put(key, value)
s := db.NewSnapshot()
defer s.Close()
db.Put(key, []byte("hello world2"))
if v, err := s.Get(key); err != nil {
t.Fatal(err)
} else if string(v) != string(value) {
t.Fatal(string(v))
}
}
func TestDestroy(t *testing.T) {
db := getTestDB()
db.Destroy()
if _, err := os.Stat(db.cfg.Path); !os.IsNotExist(err) {
t.Fatal("must not exist")
}
}

19
leveldb/readme.md
View File

@ -1,19 +0,0 @@
a leveldb wrapper for levigo
simplify use leveldb in go
# Build leveldb
see [https://gist.github.com/siddontang/dfbc835e06e47d0f6297](https://gist.github.com/siddontang/dfbc835e06e47d0f6297) for build leveldb
# Install
you must first set CGO_CFLAGS, CGO_LDFLAGS to your leveldb and snappy directory.
dev.sh may help you:
. ./dev.sh
# Notice
I have changed this package to [https://github.com/siddontang/go-leveldb](https://github.com/siddontang/go-leveldb) and will not maintain here anymore.

66
leveldb/snapshot.go
View File

@ -1,66 +0,0 @@
package leveldb
import (
"github.com/jmhodges/levigo"
)
type Snapshot struct {
db *DB
s *levigo.Snapshot
readOpts *levigo.ReadOptions
iteratorOpts *levigo.ReadOptions
}
func newSnapshot(db *DB) *Snapshot {
s := new(Snapshot)
s.db = db
s.s = db.db.NewSnapshot()
s.readOpts = levigo.NewReadOptions()
s.readOpts.SetSnapshot(s.s)
s.iteratorOpts = levigo.NewReadOptions()
s.iteratorOpts.SetSnapshot(s.s)
s.iteratorOpts.SetFillCache(false)
return s
}
func (s *Snapshot) Close() {
s.db.db.ReleaseSnapshot(s.s)
s.iteratorOpts.Close()
s.readOpts.Close()
}
func (s *Snapshot) Get(key []byte) ([]byte, error) {
return s.db.db.Get(s.readOpts, key)
}
func (s *Snapshot) Iterator(min []byte, max []byte, rangeType uint8, offset int, limit int) *Iterator {
return newIterator(s.db, s.iteratorOpts, NewRange(min, max, rangeType), offset, limit, IteratorForward)
}
func (s *Snapshot) RevIterator(min []byte, max []byte, rangeType uint8, offset int, limit int) *Iterator {
return newIterator(s.db, s.iteratorOpts, NewRange(min, max, rangeType), offset, limit, IteratorBackward)
}
func (s *Snapshot) GetInt(key []byte) (int64, error) {
return Int(s.Get(key))
}
func (s *Snapshot) GetUInt(key []byte) (uint64, error) {
return Uint(s.Get(key))
}
func (s *Snapshot) GetFloat(key []byte) (float64, error) {
return Float(s.Get(key))
}
func (s *Snapshot) GetString(key []byte) (string, error) {
return String(s.Get(key))
}
func (s *Snapshot) GetSlice(key []byte) ([]byte, error) {
return Slice(s.Get(key))
}

56
leveldb/util.go
View File

@ -1,56 +0,0 @@
package leveldb
import (
"github.com/siddontang/golib/hack"
"strconv"
)
func Int(v []byte, err error) (int64, error) {
if err != nil {
return 0, err
} else if v == nil {
return 0, nil
}
return strconv.ParseInt(hack.String(v), 10, 64)
}
func Uint(v []byte, err error) (uint64, error) {
if err != nil {
return 0, err
} else if v == nil {
return 0, nil
}
return strconv.ParseUint(hack.String(v), 10, 64)
}
func Float(v []byte, err error) (float64, error) {
if err != nil {
return 0, err
} else if v == nil {
return 0, nil
}
return strconv.ParseFloat(hack.String(v), 64)
}
func String(v []byte, err error) (string, error) {
if err != nil {
return "", err
} else if v == nil {
return "", nil
}
return hack.String(v), nil
}
func Slice(v []byte, err error) ([]byte, error) {
if err != nil {
return nil, err
} else if v == nil {
return []byte{}, nil
}
return v, nil
}

179
log/filehandler.go
View File

@ -1,179 +0,0 @@
package log
import (
"fmt"
"os"
"time"
)
type FileHandler struct {
fd *os.File
}
func NewFileHandler(fileName string, flag int) (*FileHandler, error) {
f, err := os.OpenFile(fileName, flag, 0)
if err != nil {
return nil, err
}
h := new(FileHandler)
h.fd = f
return h, nil
}
func (h *FileHandler) Write(b []byte) (n int, err error) {
return h.fd.Write(b)
}
func (h *FileHandler) Close() error {
return h.fd.Close()
}
type RotatingFileHandler struct {
fd *os.File
fileName string
maxBytes int
backupCount int
}
func NewRotatingFileHandler(fileName string, maxBytes int, backupCount int) (*RotatingFileHandler, error) {
h := new(RotatingFileHandler)
h.fileName = fileName
h.maxBytes = maxBytes
h.backupCount = backupCount
var err error
h.fd, err = os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
if err != nil {
return nil, err
}
return h, nil
}
func (h *RotatingFileHandler) Write(p []byte) (n int, err error) {
h.doRollover()
return h.fd.Write(p)
}
func (h *RotatingFileHandler) Close() error {
if h.fd != nil {
return h.fd.Close()
}
return nil
}
func (h *RotatingFileHandler) doRollover() {
f, err := h.fd.Stat()
if err != nil {
return
}
if h.maxBytes <= 0 {
return
} else if f.Size() < int64(h.maxBytes) {
return
}
if h.backupCount > 0 {
h.fd.Close()
for i := h.backupCount - 1; i > 0; i-- {
sfn := fmt.Sprintf("%s.%d", h.fileName, i)
dfn := fmt.Sprintf("%s.%d", h.fileName, i+1)
os.Rename(sfn, dfn)
}
dfn := fmt.Sprintf("%s.1", h.fileName)
os.Rename(h.fileName, dfn)
h.fd, _ = os.OpenFile(h.fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
}
}
//refer: http://docs.python.org/2/library/logging.handlers.html
//same like python TimedRotatingFileHandler
type TimeRotatingFileHandler struct {
fd *os.File
baseName string
interval int64
suffix string
rolloverAt int64
}
const (
WhenSecond = iota
WhenMinute
WhenHour
WhenDay
)
func NewTimeRotatingFileHandler(baseName string, when int8, interval int) (*TimeRotatingFileHandler, error) {
h := new(TimeRotatingFileHandler)
h.baseName = baseName
switch when {
case WhenSecond:
h.interval = 1
h.suffix = "2006-01-02_15-04-05"
case WhenMinute:
h.interval = 60
h.suffix = "2006-01-02_15-04"
case WhenHour:
h.interval = 3600
h.suffix = "2006-01-02_15"
case WhenDay:
h.interval = 3600 * 24
h.suffix = "2006-01-02"
default:
return nil, fmt.Errorf("invalid when_rotate: %d", when)
}
h.interval = h.interval * int64(interval)
var err error
h.fd, err = os.OpenFile(h.baseName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
if err != nil {
return nil, err
}
fInfo, _ := h.fd.Stat()
h.rolloverAt = fInfo.ModTime().Unix() + h.interval
return h, nil
}
func (h *TimeRotatingFileHandler) doRollover() {
//refer http://hg.python.org/cpython/file/2.7/Lib/logging/handlers.py
now := time.Now()
if h.rolloverAt <= now.Unix() {
fName := h.baseName + now.Format(h.suffix)
h.fd.Close()
e := os.Rename(h.baseName, fName)
if e != nil {
panic(e)
}
h.fd, _ = os.OpenFile(h.baseName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
h.rolloverAt = time.Now().Unix() + h.interval
}
}
func (h *TimeRotatingFileHandler) Write(b []byte) (n int, err error) {
h.doRollover()
return h.fd.Write(b)
}
func (h *TimeRotatingFileHandler) Close() error {
return h.fd.Close()
}

45
log/handler.go
View File

@ -1,45 +0,0 @@
package log
import (
"io"
)
type Handler interface {
Write(p []byte) (n int, err error)
Close() error
}
type StreamHandler struct {
w io.Writer
}
func NewStreamHandler(w io.Writer) (*StreamHandler, error) {
h := new(StreamHandler)
h.w = w
return h, nil
}
func (h *StreamHandler) Write(b []byte) (n int, err error) {
return h.w.Write(b)
}
func (h *StreamHandler) Close() error {
return nil
}
type NullHandler struct {
}
func NewNullHandler() (*NullHandler, error) {
return new(NullHandler), nil
}
func (h *NullHandler) Write(b []byte) (n int, err error) {
return len(b), nil
}
func (h *NullHandler) Close() {
}

186
log/log.go
View File

@ -1,186 +0,0 @@
package log
import (
"fmt"
"os"
"runtime"
"time"
)
const (
LevelTrace = iota
LevelDebug
LevelInfo
LevelWarn
LevelError
LevelFatal
)
const (
Ltime = iota << 1 //time format "2006/01/02 15:04:05"
Lfile //file.go:123
Llevel //[Trace|Debug|Info...]
)
var LevelName [6]string = [6]string{"Trace", "Debug", "Info", "Warn", "Error", "Fatal"}
const TimeFormat = "2006/01/02 15:04:05"
type Logger struct {
level int
flag int
handler Handler
quit chan struct{}
msg chan []byte
}
func New(handler Handler, flag int) *Logger {
var l = new(Logger)
l.level = LevelInfo
l.handler = handler
l.flag = flag
l.quit = make(chan struct{})
l.msg = make(chan []byte, 1024)
go l.run()
return l
}
func NewDefault(handler Handler) *Logger {
return New(handler, Ltime|Lfile|Llevel)
}
func newStdHandler() *StreamHandler {
h, _ := NewStreamHandler(os.Stdout)
return h
}
var std = NewDefault(newStdHandler())
func (l *Logger) run() {
for {
select {
case msg := <-l.msg:
l.handler.Write(msg)
case <-l.quit:
l.handler.Close()
}
}
}
func (l *Logger) Close() {
if l.quit == nil {
return
}
close(l.quit)
l.quit = nil
}
func (l *Logger) SetLevel(level int) {
l.level = level
}
func (l *Logger) Output(callDepth int, level int, format string, v ...interface{}) {
if l.level > level {
return
}
buf := make([]byte, 0, 1024)
if l.flag&Ltime > 0 {
now := time.Now().Format(TimeFormat)
buf = append(buf, now...)
buf = append(buf, " "...)
}
if l.flag&Lfile > 0 {
_, file, line, ok := runtime.Caller(callDepth)
if !ok {
file = "???"
line = 0
} else {
for i := len(file) - 1; i > 0; i-- {
if file[i] == '/' {
file = file[i+1:]
break
}
}
}
buf = append(buf, fmt.Sprintf("%s:%d ", file, line)...)
}
if l.flag&Llevel > 0 {
buf = append(buf, fmt.Sprintf("[%s] ", LevelName[level])...)
}
s := fmt.Sprintf(format, v...)
buf = append(buf, s...)
if s[len(s)-1] != '\n' {
buf = append(buf, "\n"...)
}
l.msg <- buf
}
func (l *Logger) Trace(format string, v ...interface{}) {
l.Output(2, LevelTrace, format, v...)
}
func (l *Logger) Debug(format string, v ...interface{}) {
l.Output(2, LevelDebug, format, v...)
}
func (l *Logger) Info(format string, v ...interface{}) {
l.Output(2, LevelInfo, format, v...)
}
func (l *Logger) Warn(format string, v ...interface{}) {
l.Output(2, LevelWarn, format, v...)
}
func (l *Logger) Error(format string, v ...interface{}) {
l.Output(2, LevelError, format, v...)
}
func (l *Logger) Fatal(format string, v ...interface{}) {
l.Output(2, LevelFatal, format, v...)
}
func SetLevel(level int) {
std.SetLevel(level)
}
func Trace(format string, v ...interface{}) {
std.Output(2, LevelTrace, format, v...)
}
func Debug(format string, v ...interface{}) {
std.Output(2, LevelDebug, format, v...)
}
func Info(format string, v ...interface{}) {
std.Output(2, LevelInfo, format, v...)
}
func Warn(format string, v ...interface{}) {
std.Output(2, LevelWarn, format, v...)
}
func Error(format string, v ...interface{}) {
std.Output(2, LevelError, format, v...)
}
func Fatal(format string, v ...interface{}) {
std.Output(2, LevelFatal, format, v...)
}

52
log/log_test.go
View File

@ -1,52 +0,0 @@
package log
import (
"os"
"testing"
)
func TestStdStreamLog(t *testing.T) {
h, _ := NewStreamHandler(os.Stdout)
s := NewDefault(h)
s.Info("hello world")
s.Close()
Info("hello world")
}
func TestRotatingFileLog(t *testing.T) {
path := "./test_log"
os.RemoveAll(path)
os.Mkdir(path, 0777)
fileName := path + "/test"
h, err := NewRotatingFileHandler(fileName, 10, 2)
if err != nil {
t.Fatal(err)
}
buf := make([]byte, 10)
h.Write(buf)
h.Write(buf)
if _, err := os.Stat(fileName + ".1"); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(fileName + ".2"); err == nil {
t.Fatal(err)
}
h.Write(buf)
if _, err := os.Stat(fileName + ".2"); err != nil {
t.Fatal(err)
}
h.Close()
os.RemoveAll(path)
}

21
log/loggingd/main.go
View File

@ -1,21 +0,0 @@
package main
import (
"flag"
"github.com/siddontang/golib/log"
)
var logFile = flag.String("logfile", "./logd.log", "file to log")
var net = flag.String("net", "tcp", "server listen protocol, like tcp, udp or unix")
var addr = flag.String("addr", "127.0.0.1:11183", "server listen address")
func main() {
flag.Parse()
s, err := log.NewServer(*logFile, *net, *addr)
if err != nil {
panic(err)
}
s.Run()
}

3
log/readme.md
View File

@ -1,3 +0,0 @@
# Notice
I have changed this package to [https://github.com/siddontang/go-log](https://github.com/siddontang/go-log) and will not maintain here anymore.

95
log/server.go
View File

@ -1,95 +0,0 @@
package log
import (
"bufio"
"encoding/binary"
"io"
"net"
"os"
"path"
)
//a log server for handling SocketHandler send log
type Server struct {
closed bool
listener net.Listener
fd *os.File
}
func NewServer(fileName string, protocol string, addr string) (*Server, error) {
s := new(Server)
s.closed = false
var err error
dir := path.Dir(fileName)
os.Mkdir(dir, 0777)
s.fd, err = os.OpenFile(fileName, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
if err != nil {
return nil, err
}
s.listener, err = net.Listen(protocol, addr)
if err != nil {
return nil, err
}
return s, nil
}
func (s *Server) Close() error {
if s.closed {
return nil
}
s.closed = true
s.fd.Close()
s.listener.Close()
return nil
}
func (s *Server) Run() {
for {
conn, err := s.listener.Accept()
if err != nil {
continue
}
go s.onRead(conn)
}
}
func (s *Server) onRead(c net.Conn) {
br := bufio.NewReaderSize(c, 1024)
var bufLen uint32
for {
if err := binary.Read(br, binary.BigEndian, &bufLen); err != nil {
c.Close()
return
}
buf := make([]byte, bufLen, bufLen+1)
if _, err := io.ReadFull(br, buf); err != nil && err != io.ErrUnexpectedEOF {
c.Close()
return
} else {
if len(buf) == 0 {
continue
}
if buf[len(buf)-1] != '\n' {
buf = append(buf, '\n')
}
s.fd.Write(buf)
}
}
}

56
log/socket_test.go
View File

@ -1,56 +0,0 @@
package log
import (
"io"
"os"
"testing"
"time"
)
func TestSocket(t *testing.T) {
fileName := "./test_server.log"
os.Remove(fileName)
s, err := NewServer(fileName, "tcp", "127.0.0.1:11183")
if err != nil {
t.Fatal(err)
}
go s.Run()
defer s.Close()
var h *SocketHandler
h, err = NewSocketHandler("tcp", "127.0.0.1:11183")
_, err = h.Write([]byte("hello world"))
if err != nil {
t.Fatal(err)
}
time.Sleep(1 * time.Second)
s.Close()
var f *os.File
f, err = os.Open(fileName)
if err != nil {
t.Fatal(err)
}
defer f.Close()
buf := make([]byte, 64)
var n int
n, err = f.Read(buf)
if err != nil && err != io.EOF {
t.Fatal(err)
}
buf = buf[0:n]
if string(buf) != "hello world\n" {
t.Fatal(string(buf))
}
os.Remove(fileName)
}

62
log/sockethandler.go
View File

@ -1,62 +0,0 @@
package log
import (
"encoding/binary"
"net"
"time"
)
type SocketHandler struct {
c net.Conn
protocol string
addr string
}
func NewSocketHandler(protocol string, addr string) (*SocketHandler, error) {
s := new(SocketHandler)
s.protocol = protocol
s.addr = addr
return s, nil
}
func (h *SocketHandler) Write(p []byte) (n int, err error) {
if err = h.connect(); err != nil {
return
}
buf := make([]byte, len(p)+4)
binary.BigEndian.PutUint32(buf, uint32(len(p)))
copy(buf[4:], p)
n, err = h.c.Write(buf)
if err != nil {
h.c.Close()
h.c = nil
}
return
}
func (h *SocketHandler) Close() error {
if h.c != nil {
h.c.Close()
}
return nil
}
func (h *SocketHandler) connect() error {
if h.c != nil {
return nil
}
var err error
h.c, err = net.DialTimeout(h.protocol, h.addr, 20*time.Second)
if err != nil {
return err
}
return nil
}

60
websocket/client.go
View File

@ -1,60 +0,0 @@
package websocket
import (
"bytes"
"errors"
"net"
"net/http"
"net/url"
"strings"
)
var (
ErrBadHandshake = errors.New("bad handshake")
)
func NewClient(netConn net.Conn, u *url.URL, requestHeader http.Header) (c *Conn, response *http.Response, err error) {
key, err := calcKey()
if err != nil {
return nil, nil, err
}
acceptKey := calcAcceptKey(key)
c = NewConn(netConn, false)
buf := bytes.NewBufferString("GET ")
buf.WriteString(u.RequestURI())
buf.WriteString(" HTTP/1.1\r\nHost: ")
buf.WriteString(u.Host)
buf.WriteString("\r\nUpgrade: websocket\r\nConnection: upgrade\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: ")
buf.WriteString(key)
buf.WriteString("\r\n")
for k, vs := range requestHeader {
for _, v := range vs {
buf.WriteString(k)
buf.WriteString(": ")
buf.WriteString(v)
buf.WriteString("\r\n")
}
}
buf.WriteString("\r\n")
p := buf.Bytes()
if _, err := netConn.Write(p); err != nil {
return nil, nil, err
}
resp, err := http.ReadResponse(c.br, &http.Request{Method: "GET", URL: u})
if err != nil {
return nil, nil, err
}
if resp.StatusCode != 101 ||
!strings.EqualFold(resp.Header.Get("Upgrade"), "websocket") ||
!strings.EqualFold(resp.Header.Get("Connection"), "upgrade") ||
resp.Header.Get("Sec-Websocket-Accept") != acceptKey {
return nil, resp, ErrBadHandshake
}
return c, resp, nil
}

323
websocket/conn.go
View File

@ -1,323 +0,0 @@
package websocket
import (
"bufio"
"encoding/binary"
"errors"
"io"
"lib/log"
"math/rand"
"net"
"time"
)
//refer RFC6455
const (
TextMessage byte = 1
BinaryMessage byte = 2
CloseMessage byte = 8
PingMessage byte = 9
PongMessage byte = 10
)
var (
ErrControlTooLong = errors.New("control message too long")
ErrRSVNotSupport = errors.New("reserved bit not support")
ErrPayloadError = errors.New("payload length error")
ErrControlFragmented = errors.New("control message can not be fragmented")
ErrNotTCPConn = errors.New("not a tcp connection")
ErrWriteError = errors.New("write error")
)
type Conn struct {
conn net.Conn
br *bufio.Reader
isServer bool
}
func NewConn(conn net.Conn, isServer bool) *Conn {
c := new(Conn)
c.conn = conn
c.br = bufio.NewReader(conn)
c.isServer = isServer
return c
}
func (c *Conn) ReadMessage() (messageType byte, message []byte, err error) {
return c.Read()
}
func (c *Conn) Read() (messageType byte, message []byte, err error) {
buf := make([]byte, 8, 8)
message = []byte{}
messageType = 0
for {
opcode, data, err := c.readFrame(buf)
if err != nil {
return messageType, message, err
}
message = append(message, data...)
if opcode&0x80 != 0 {
//final
if opcode&0x0F > 0 {
//not continue frame
messageType = opcode & 0x0F
}
return messageType, message, nil
} else {
if opcode&0x0F > 0 {
//first continue frame
messageType = opcode & 0x0F
}
}
}
return
}
func (c *Conn) Write(message []byte, binary bool) error {
if binary {
return c.sendFrame(BinaryMessage, message)
} else {
return c.sendFrame(TextMessage, message)
}
}
func (c *Conn) WriteMessage(messageType byte, message []byte) error {
return c.sendFrame(messageType, message)
}
//write utf-8 text message
func (c *Conn) WriteString(message []byte) error {
return c.Write(message, false)
}
//write binary message
func (c *Conn) WriteBinary(message []byte) error {
return c.Write(message, true)
}
func (c *Conn) Ping(message []byte) error {
return c.sendFrame(PingMessage, message)
}
func (c *Conn) Pong(message []byte) error {
return c.sendFrame(PongMessage, message)
}
//close socket, not send websocket close message
func (c *Conn) Close() error {
return c.conn.Close()
}
func (c *Conn) LocalAddr() net.Addr {
return c.conn.LocalAddr()
}
func (c *Conn) RemoteAddr() net.Addr {
return c.conn.RemoteAddr()
}
func (c *Conn) SetReadDeadline(t time.Time) error {
return c.conn.SetReadDeadline(t)
}
func (c *Conn) SetWriteDeadline(t time.Time) error {
return c.conn.SetWriteDeadline(t)
}
func (c *Conn) SetReadBuffer(bytes int) error {
if tcpConn, ok := c.conn.(*net.TCPConn); ok {
return tcpConn.SetReadBuffer(bytes)
} else {
return ErrNotTCPConn
}
}
func (c *Conn) SetWriteBuffer(bytes int) error {
if tcpConn, ok := c.conn.(*net.TCPConn); ok {
return tcpConn.SetWriteBuffer(bytes)
} else {
return ErrNotTCPConn
}
}
func (c *Conn) readPayloadLen(length byte, buf []byte) (payloadLen uint64, err error) {
if length < 126 {
payloadLen = uint64(length)
} else if length == 126 {
err = c.read(buf[:2])
if err != nil {
return
}
payloadLen = uint64(binary.BigEndian.Uint16(buf[:2]))
} else if length == 127 {
err = c.read(buf[:8])
if err != nil {
return
}
payloadLen = uint64(binary.BigEndian.Uint16(buf[:8]))
}
return
}
func (c *Conn) readFrame(buf []byte) (opcode byte, messsage []byte, err error) {
//minimum head may 2 byte
err = c.read(buf[:2])
if err != nil {
return
}
opcode = buf[0]
if opcode&0x70 > 0 {
err = ErrRSVNotSupport
return
}
//isMasking := (0x80 & buf[1]) > 0
isMasking := (0x80 & buf[1]) > 0
var payloadLen uint64
payloadLen, err = c.readPayloadLen(buf[1]&0x7F, buf)
if err != nil {
return
}
if opcode&0x08 > 0 && payloadLen > 125 {
err = ErrControlTooLong
return
}
var masking []byte
if isMasking {
err = c.read(buf[:4])
if err != nil {
return
}
masking = buf[:4]
}
messsage = make([]byte, payloadLen)
err = c.read(messsage)
if err != nil {
return
}
if isMasking {
//maskingKey := c.newMaskingKey()
c.maskingData(messsage, masking)
}
return
}
func (c *Conn) sendFrame(opcode byte, message []byte) error {
//max frame header may 14 length
buf := make([]byte, 0, len(message)+14)
//here we don not support continue frame, all are final
opcode |= 0x80
if opcode&0x08 > 0 && len(message) >= 126 {
return ErrControlTooLong
}
buf = append(buf, opcode)
//no mask, because chrome may not support
var mask byte = 0x00
if !c.isServer {
//for client, we will mask data
mask = 0x80
}
payloadLen := len(message)
if payloadLen < 126 {
buf = append(buf, mask|byte(payloadLen))
} else if payloadLen <= 0xFFFF {
buf = append(buf, mask|byte(126), 0, 0)
binary.BigEndian.PutUint16(buf[len(buf)-2:], uint16(payloadLen))
} else {
buf = append(buf, mask|byte(127), 0, 0, 0, 0, 0, 0, 0, 0)
binary.BigEndian.PutUint64(buf[len(buf)-8:], uint64(payloadLen))
}
if !c.isServer {
maskingKey := c.newMaskingKey()
buf = append(buf, maskingKey...)
pos := len(buf)
buf = append(buf, message...)
c.maskingData(buf[pos:], maskingKey)
} else {
buf = append(buf, message...)
}
tmpBuf := buf
for i := 0; i < 3; i++ {
n, err := c.conn.Write(tmpBuf)
if err != nil {
return err
}
if n == len(tmpBuf) {
return nil
} else {
log.Warn("[conn write] buffer_size=%d return_size=%s", len(tmpBuf), n)
tmpBuf = tmpBuf[n:]
}
}
return ErrWriteError
}
func (c *Conn) read(buf []byte) error {
var err error
for len(buf) > 0 && err == nil {
var nn int
nn, err = c.br.Read(buf)
buf = buf[nn:]
}
if err == io.EOF {
if len(buf) == 0 {
err = nil
} else {
err = io.ErrUnexpectedEOF
}
}
return err
}
func (c *Conn) maskingData(data []byte, maskingKey []byte) {
for i := range data {
data[i] ^= maskingKey[i%4]
}
}
func (c *Conn) newMaskingKey() []byte {
n := rand.Uint32()
return []byte{byte(n), byte(n >> 8), byte(n >> 16), byte(n >> 32)}
}

105
websocket/server.go
View File

@ -1,105 +0,0 @@
package websocket
import (
"bufio"
"bytes"
"errors"
"net"
"net/http"
"strings"
)
var (
ErrInvalidMethod = errors.New("Only GET Supported")
ErrInvalidVersion = errors.New("Sec-Websocket-Version: 13")
ErrInvalidUpgrade = errors.New("Can \"Upgrade\" only to \"WebSocket\"")
ErrInvalidConnection = errors.New("\"Connection\" must be \"Upgrade\"")
ErrMissingKey = errors.New("Missing Key")
ErrHijacker = errors.New("Not implement http.Hijacker")
ErrNoEmptyConn = errors.New("Conn ReadBuf must be empty")
)
func Upgrade(w http.ResponseWriter, r *http.Request, responseHeader http.Header) (*Conn, error) {
if r.Method != "GET" {
return nil, ErrInvalidMethod
}
if r.Header.Get("Sec-Websocket-Version") != "13" {
return nil, ErrInvalidVersion
}
if strings.ToLower(r.Header.Get("Upgrade")) != "websocket" {
return nil, ErrInvalidUpgrade
}
if strings.ToLower(r.Header.Get("Connection")) != "upgrade" {
return nil, ErrInvalidConnection
}
var acceptKey string
if key := r.Header.Get("Sec-Websocket-key"); len(key) == 0 {
return nil, ErrMissingKey
} else {
acceptKey = calcAcceptKey(key)
}
var (
netConn net.Conn
br *bufio.Reader
err error
)
h, ok := w.(http.Hijacker)
if !ok {
return nil, ErrHijacker
}
var rw *bufio.ReadWriter
netConn, rw, err = h.Hijack()
br = rw.Reader
if br.Buffered() > 0 {
netConn.Close()
return nil, ErrNoEmptyConn
}
c := NewConn(netConn, true)
buf := bytes.NewBufferString("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ")
buf.WriteString(acceptKey)
buf.WriteString("\r\n")
subProtol := selectSubProtocol(r)
if len(subProtol) > 0 {
buf.WriteString("Sec-Websocket-Protocol: ")
buf.WriteString(subProtol)
buf.WriteString("\r\n")
}
for k, vs := range responseHeader {
for _, v := range vs {
buf.WriteString(k)
buf.WriteString(": ")
buf.WriteString(v)
buf.WriteString("\r\n")
}
}
buf.WriteString("\r\n")
if _, err = netConn.Write(buf.Bytes()); err != nil {
netConn.Close()
return nil, err
}
return c, nil
}
func selectSubProtocol(r *http.Request) string {
h := r.Header.Get("Sec-Websocket-Protocol")
if len(h) == 0 {
return ""
}
return strings.Split(h, ",")[0]
}

26
websocket/util.go
View File

@ -1,26 +0,0 @@
package websocket
import (
"crypto/rand"
"crypto/sha1"
"encoding/base64"
"errors"
"io"
)
var keyGUID = []byte("258EAFA5-E914-47DA-95CA-C5AB0DC85B11")
func calcAcceptKey(key string) string {
h := sha1.New()
h.Write([]byte(key))
h.Write(keyGUID)
return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
func calcKey() (string, error) {
p := make([]byte, 16)
if _, err := io.ReadFull(rand.Reader, p); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(p), nil
}