mirror
/
buntdb
mirror of https://github.com/tidwall/buntdb.git
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

Using generics tidwall/btree

This commit is contained in:
tidwall 2021-12-24 04:45:15 -07:00
parent f5d7b17e75
commit 4ccd490a10
3 changed files with 76 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
buntdb.go
View File

@ -63,19 +63,19 @@ var (
// DB represents a collection of key-value pairs that persist on disk. // DB represents a collection of key-value pairs that persist on disk.
// Transactions are used for all forms of data access to the DB. // Transactions are used for all forms of data access to the DB.
type DB struct { type DB struct {
mu sync.RWMutex // the gatekeeper for all fields mu sync.RWMutex // the gatekeeper for all fields
file *os.File // the underlying file file *os.File // the underlying file
buf []byte // a buffer to write to buf []byte // a buffer to write to
keys *btree.BTree // a tree of all item ordered by key keys *btree.BTree[*dbItem] // a tree of all item ordered by key
exps *btree.BTree // a tree of items ordered by expiration exps *btree.BTree[*dbItem] // a tree of items ordered by expiration
idxs map[string]*index // the index trees. idxs map[string]*index // the index trees.
insIdxs []*index // a reuse buffer for gathering indexes insIdxs []*index // a reuse buffer for gathering indexes
flushes int // a count of the number of disk flushes flushes int // a count of the number of disk flushes
closed bool // set when the database has been closed closed bool // set when the database has been closed
config Config // the database configuration config Config // the database configuration
persist bool // do we write to disk persist bool // do we write to disk
shrinking bool // when an aof shrink is in-process. shrinking bool // when an aof shrink is in-process.
lastaofsz int // the size of the last shrink aof size lastaofsz int // the size of the last shrink aof size
} }
// SyncPolicy represents how often data is synced to disk. // SyncPolicy represents how often data is synced to disk.
@ -203,8 +203,7 @@ func (db *DB) Save(wr io.Writer) error {
var buf []byte var buf []byte
now := time.Now() now := time.Now()
// iterated through every item in the database and write to the buffer // iterated through every item in the database and write to the buffer
btreeAscend(db.keys, func(item interface{}) bool { btreeAscend(db.keys, func(dbi *dbItem) bool {
dbi := item.(*dbItem)
buf = dbi.writeSetTo(buf, now) buf = dbi.writeSetTo(buf, now)
if len(buf) > 1024*1024*4 { if len(buf) > 1024*1024*4 {
// flush when buffer is over 4MB // flush when buffer is over 4MB
@ -246,7 +245,7 @@ func (db *DB) Load(rd io.Reader) error {
// index represents a b-tree or r-tree index and also acts as the // index represents a b-tree or r-tree index and also acts as the
// b-tree/r-tree context for itself. // b-tree/r-tree context for itself.
type index struct { type index struct {
btr *btree.BTree // contains the items btr *btree.BTree[*dbItem] // contains the items
rtr *rtred.RTree // contains the items rtr *rtred.RTree // contains the items
name string // name of the index name string // name of the index
pattern string // a required key pattern pattern string // a required key pattern
@ -303,8 +302,7 @@ func (idx *index) rebuild() {
idx.rtr = rtred.New(idx) idx.rtr = rtred.New(idx)
} }
// iterate through all keys and fill the index // iterate through all keys and fill the index
btreeAscend(idx.db.keys, func(item interface{}) bool { btreeAscend(idx.db.keys, func(dbi *dbItem) bool {
dbi := item.(*dbItem)
if !idx.match(dbi.key) { if !idx.match(dbi.key) {
// does not match the pattern, continue // does not match the pattern, continue
return true return true
@ -463,11 +461,10 @@ func (db *DB) insertIntoDatabase(item *dbItem) *dbItem {
idxs = append(idxs, idx) idxs = append(idxs, idx)
} }
} }
prev := db.keys.Set(item) pdbi, replaced := db.keys.Set(item)
if prev != nil { if replaced {
// A previous item was removed from the keys tree. Let's // A previous item was removed from the keys tree. Let's
// fully delete this item from all indexes. // fully delete this item from all indexes.
pdbi = prev.(*dbItem)
if pdbi.opts != nil && pdbi.opts.ex { if pdbi.opts != nil && pdbi.opts.ex {
// Remove it from the expires tree. // Remove it from the expires tree.
db.exps.Delete(pdbi) db.exps.Delete(pdbi)
@ -513,10 +510,8 @@ func (db *DB) insertIntoDatabase(item *dbItem) *dbItem {
// returned to the caller. A nil return value means that the item was not // returned to the caller. A nil return value means that the item was not
// found in the database // found in the database
func (db *DB) deleteFromDatabase(item *dbItem) *dbItem { func (db *DB) deleteFromDatabase(item *dbItem) *dbItem {
var pdbi *dbItem pdbi, deleted := db.keys.Delete(item)
prev := db.keys.Delete(item) if deleted {
if prev != nil {
pdbi = prev.(*dbItem)
if pdbi.opts != nil && pdbi.opts.ex { if pdbi.opts != nil && pdbi.opts.ex {
// Remove it from the exipres tree. // Remove it from the exipres tree.
db.exps.Delete(pdbi) db.exps.Delete(pdbi)
@ -570,8 +565,8 @@ func (db *DB) backgroundManager() {
// produce a list of expired items that need removing // produce a list of expired items that need removing
btreeAscendLessThan(db.exps, &dbItem{ btreeAscendLessThan(db.exps, &dbItem{
opts: &dbItemOpts{ex: true, exat: time.Now()}, opts: &dbItemOpts{ex: true, exat: time.Now()},
}, func(item interface{}) bool { }, func(item *dbItem) bool {
expired = append(expired, item.(*dbItem)) expired = append(expired, item)
return true return true
}) })
if onExpired == nil && onExpiredSync == nil { if onExpired == nil && onExpiredSync == nil {
@ -687,8 +682,7 @@ func (db *DB) Shrink() error {
var n int var n int
now := time.Now() now := time.Now()
btreeAscendGreaterOrEqual(db.keys, &dbItem{key: pivot}, btreeAscendGreaterOrEqual(db.keys, &dbItem{key: pivot},
func(item interface{}) bool { func(dbi *dbItem) bool {
dbi := item.(*dbItem)
// 1000 items or 64MB buffer // 1000 items or 64MB buffer
if n > 1000 || len(buf) > 64*1024*1024 { if n > 1000 || len(buf) > 64*1024*1024 {
pivot = dbi.key pivot = dbi.key
@ -959,10 +953,11 @@ func (db *DB) load() error {
return err return err
} }
var estaofsz int var estaofsz int
db.keys.Walk(func(items []interface{}) { db.keys.Walk(func(items []*dbItem) bool {
for _, v := range items { for _, v := range items {
estaofsz += v.(*dbItem).estAOFSetSize() estaofsz += v.estAOFSetSize()
} }
return true
}) })
db.lastaofsz += estaofsz db.lastaofsz += estaofsz
return nil return nil
@ -1022,11 +1017,9 @@ func (db *DB) Update(fn func(tx *Tx) error) error {
// get return an item or nil if not found. // get return an item or nil if not found.
func (db *DB) get(key string) *dbItem { func (db *DB) get(key string) *dbItem {
item := db.keys.Get(&dbItem{key: key}) // nil is the default item type
if item != nil { item, _ := db.keys.Get(&dbItem{key: key})
return item.(*dbItem) return item
}
return nil
} }
// Tx represents a transaction on the database. This transaction can either be // Tx represents a transaction on the database. This transaction can either be
@ -1044,9 +1037,9 @@ type Tx struct {
type txWriteContext struct { type txWriteContext struct {
// rollback when deleteAll is called // rollback when deleteAll is called
rbkeys *btree.BTree // a tree of all item ordered by key rbkeys *btree.BTree[*dbItem] // a tree of all item ordered by key
rbexps *btree.BTree // a tree of items ordered by expiration rbexps *btree.BTree[*dbItem] // a tree of items ordered by expiration
rbidxs map[string]*index // the index trees. rbidxs map[string]*index // the index trees.
rollbackItems map[string]*dbItem // details for rolling back tx. rollbackItems map[string]*dbItem // details for rolling back tx.
commitItems map[string]*dbItem // details for committing tx. commitItems map[string]*dbItem // details for committing tx.
@ -1404,9 +1397,9 @@ func (dbi *dbItem) Less(dbi2 *dbItem, ctx interface{}) bool {
return dbi.key < dbi2.key return dbi.key < dbi2.key
} }
func lessCtx(ctx interface{}) func(a, b interface{}) bool { func lessCtx(ctx interface{}) func(a, b *dbItem) bool {
return func(a, b interface{}) bool { return func(a, b *dbItem) bool {
return a.(*dbItem).Less(b.(*dbItem), ctx) return a.Less(b, ctx)
} }
} }
@ -1615,11 +1608,10 @@ func (tx *Tx) scan(desc, gt, lt bool, index, start, stop string,
return ErrTxClosed return ErrTxClosed
} }
// wrap a btree specific iterator around the user-defined iterator. // wrap a btree specific iterator around the user-defined iterator.
iter := func(item interface{}) bool { iter := func(dbi *dbItem) bool {
dbi := item.(*dbItem)
return iterator(dbi.key, dbi.val) return iterator(dbi.key, dbi.val)
} }
var tr *btree.BTree var tr *btree.BTree[*dbItem]
if index == "" { if index == "" {
// empty index means we will use the keys tree. // empty index means we will use the keys tree.
tr = tx.db.keys tr = tx.db.keys
@ -2302,69 +2294,72 @@ func Desc(less func(a, b string) bool) func(a, b string) bool {
//// Wrappers around btree Ascend/Descend //// Wrappers around btree Ascend/Descend
func bLT(tr *btree.BTree, a, b interface{}) bool { return tr.Less(a, b) } func bLT(tr *btree.BTree[*dbItem], a, b *dbItem) bool {
func bGT(tr *btree.BTree, a, b interface{}) bool { return tr.Less(b, a) } return tr.Less(a, b)
}
func bGT(tr *btree.BTree[*dbItem], a, b *dbItem) bool {
return tr.Less(b, a)
}
// func bLTE(tr *btree.BTree, a, b interface{}) bool { return !tr.Less(b, a) } // func bLTE(tr *btree.BTree, a, b *dbItem) bool { return !tr.Less(b, a) }
// func bGTE(tr *btree.BTree, a, b interface{}) bool { return !tr.Less(a, b) } // func bGTE(tr *btree.BTree, a, b *dbItem) bool { return !tr.Less(a, b) }
// Ascend // Ascend
func btreeAscend(tr *btree.BTree, iter func(item interface{}) bool) { func btreeAscend(tr *btree.BTree[*dbItem], iter func(item *dbItem) bool) {
tr.Ascend(nil, iter) tr.Scan(iter)
} }
func btreeAscendLessThan(tr *btree.BTree, pivot interface{}, func btreeAscendLessThan(tr *btree.BTree[*dbItem], pivot *dbItem,
iter func(item interface{}) bool, iter func(item *dbItem) bool,
) { ) {
tr.Ascend(nil, func(item interface{}) bool { tr.Scan(func(item *dbItem) bool {
return bLT(tr, item, pivot) && iter(item) return bLT(tr, item, pivot) && iter(item)
}) })
} }
func btreeAscendGreaterOrEqual(tr *btree.BTree, pivot interface{}, func btreeAscendGreaterOrEqual(tr *btree.BTree[*dbItem], pivot *dbItem,
iter func(item interface{}) bool, iter func(item *dbItem) bool,
) { ) {
tr.Ascend(pivot, iter) tr.Ascend(pivot, iter)
} }
func btreeAscendRange(tr *btree.BTree, greaterOrEqual, lessThan interface{}, func btreeAscendRange(tr *btree.BTree[*dbItem], greaterOrEqual,
iter func(item interface{}) bool, lessThan *dbItem, iter func(item *dbItem) bool,
) { ) {
tr.Ascend(greaterOrEqual, func(item interface{}) bool { tr.Ascend(greaterOrEqual, func(item *dbItem) bool {
return bLT(tr, item, lessThan) && iter(item) return bLT(tr, item, lessThan) && iter(item)
}) })
} }
// Descend // Descend
func btreeDescend(tr *btree.BTree, iter func(item interface{}) bool) { func btreeDescend(tr *btree.BTree[*dbItem], iter func(item *dbItem) bool) {
tr.Descend(nil, iter) tr.Reverse(iter)
} }
func btreeDescendGreaterThan(tr *btree.BTree, pivot interface{}, func btreeDescendGreaterThan(tr *btree.BTree[*dbItem], pivot *dbItem,
iter func(item interface{}) bool, iter func(item *dbItem) bool,
) { ) {
tr.Descend(nil, func(item interface{}) bool { tr.Reverse(func(item *dbItem) bool {
return bGT(tr, item, pivot) && iter(item) return bGT(tr, item, pivot) && iter(item)
}) })
} }
func btreeDescendRange(tr *btree.BTree, lessOrEqual, greaterThan interface{}, func btreeDescendRange(tr *btree.BTree[*dbItem], lessOrEqual,
iter func(item interface{}) bool, greaterThan *dbItem, iter func(item *dbItem) bool,
) { ) {
tr.Descend(lessOrEqual, func(item interface{}) bool { tr.Descend(lessOrEqual, func(item *dbItem) bool {
return bGT(tr, item, greaterThan) && iter(item) return bGT(tr, item, greaterThan) && iter(item)
}) })
} }
func btreeDescendLessOrEqual(tr *btree.BTree, pivot interface{}, func btreeDescendLessOrEqual(tr *btree.BTree[*dbItem], pivot *dbItem,
iter func(item interface{}) bool, iter func(item *dbItem) bool,
) { ) {
tr.Descend(pivot, iter) tr.Descend(pivot, iter)
} }
func btreeNew(less func(a, b interface{}) bool) *btree.BTree { func btreeNew(less func(a, b *dbItem) bool) *btree.BTree[*dbItem] {
// Using NewNonConcurrent because we're managing our own locks. return btree.NewOptions(less, btree.Options{NoLocks: true})
return btree.NewNonConcurrent(less)
} }

9
go.mod
View File

@ -1,13 +1,18 @@
module github.com/tidwall/buntdb module github.com/tidwall/buntdb
go 1.16 go 1.18
require ( require (
github.com/tidwall/assert v0.1.0 github.com/tidwall/assert v0.1.0
github.com/tidwall/btree v0.7.1 github.com/tidwall/btree v0.7.2-0.20211218005449-cbb03286d2f2
github.com/tidwall/gjson v1.12.1 github.com/tidwall/gjson v1.12.1
github.com/tidwall/grect v0.1.4 github.com/tidwall/grect v0.1.4
github.com/tidwall/lotsa v1.0.2 github.com/tidwall/lotsa v1.0.2
github.com/tidwall/match v1.1.1 github.com/tidwall/match v1.1.1
github.com/tidwall/rtred v0.1.2 github.com/tidwall/rtred v0.1.2
) )
require (
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tidwall/tinyqueue v0.1.1 // indirect
)

4
go.sum
View File

@ -1,7 +1,7 @@
github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI=
github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8=
github.com/tidwall/btree v0.7.1 h1:LPXN3VRIxsdMwyfbtPgOA60jLuj/eEmMpDjOh2szRPw= github.com/tidwall/btree v0.7.2-0.20211218005449-cbb03286d2f2 h1:ehSxDCH+l3iIecsbPBzed/grEc5R24bhLbJdU+QMbHQ=
github.com/tidwall/btree v0.7.1/go.mod h1:TzIRzen6yHbibdSfK6t8QimqbUnoxUSrZfeW7Uob0q4= github.com/tidwall/btree v0.7.2-0.20211218005449-cbb03286d2f2/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYmsObdKE=
github.com/tidwall/gjson v1.12.1 h1:ikuZsLdhr8Ws0IdROXUS1Gi4v9Z4pGqpX/CvJkxvfpo= github.com/tidwall/gjson v1.12.1 h1:ikuZsLdhr8Ws0IdROXUS1Gi4v9Z4pGqpX/CvJkxvfpo=
github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg=