mirror
/
buntdb
mirror of https://github.com/tidwall/buntdb.git
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

new shrink algorithm. better error checks.

This commit is contained in:
Josh Baker 2016-07-28 19:38:51 -07:00
parent 3a53e89dbc
commit ff93c0d8e9
3 changed files with 467 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
README.md
View File

@ -341,11 +341,14 @@ del key:2
...
```
When the database opens again, it will read back the aof file and process each command in exact order. This read process happens one time when the database opens. From there on the file is only appended.
When the database opens again, it will read back the aof file and process each command in exact order.
This read process happens one time when the database opens.
From there on the file is only appended.
As you may guess this log file can grow large over time. There is a `Shrink()` function which will rewrite the aof file so that it contains only the items in the database. The shrink operation does not lock up the database so read and write transactions can continue while shrinking is in process.
Also there's the database config setting `Config.AutoShrink` which is used to allow for shrinking to be self-managed. This value is set to a multiple that represents how many more entries in the aof file versus the number of items in memory. For example; If this value is set to 10 and the number of item in memory is 150,000, then the database will automatically shrink when the aof file has 1,500,000 lines in it. Currently default value is 50, but this may change in a future release. Autoshink can be disabled by setting this value to zero.
As you may guess this log file can grow large over time.
There's a background routine that automatically shrinks the log file when it gets too large.
There is also a `Shrink()` function which will rewrite the aof file so that it contains only the items in the database.
The shrink operation does not lock up the database so read and write transactions can continue while shrinking is in process.
### Durability and fsync

101
buntdb.go
View File

@ -45,9 +45,6 @@ var (
// ErrInvalidOperation is returned when an operation cannot be completed.
ErrInvalidOperation = errors.New("invalid operation")
// ErrInvalidAutoShrink is returned for an invalid AutoShrink value.
ErrInvalidAutoShrink = errors.New("invalid autoshink")
// ErrInvalidSyncPolicy is returned for an invalid SyncPolicy value.
ErrInvalidSyncPolicy = errors.New("invalid sync policy")
@ -73,10 +70,10 @@ type DB struct {
exmgr bool // indicates that expires manager is running.
flushes int // a count of the number of disk flushes
closed bool // set when the database has been closed
aoflen int // the number of lines in the aof file
config Config // the database configuration
persist bool // do we write to disk
shrinking bool // when an aof shrink is in-process.
lastaofsz int // the size of the last shrink aof size
}
// SyncPolicy represents how often data is synced to disk.
@ -103,19 +100,19 @@ type Config struct {
// This value can be Never, EverySecond, or Always.
// The default is EverySecond.
SyncPolicy SyncPolicy
// AutoShrink will automatically resize the database file on disk
// when it gets too large. The value represents a multiple of
// the maximum number of entries that can exist on disk before
// an automatic resize occurs.
// For example, a value of 2 means that the number of entries on
// disk may be up to 2x the number of items on disk.
// A zero value will disable auto shrinking.
// A negative value or 1 are invalid.
// The default is value is 50, but may change in the future.
AutoShrink int
}
const defaultAutoShrinkMultiplier = 50
// AutoShrinkPercentage is used by the background process to trigger
// a shrink of the aof file when the size of the file is larger than the
// percentage of the result of the previous shrunk file.
// For example, if this value is 100, and the last shrink process
// resulted in a 100mb file, then the new aof file must be 200mb before
// a shrink is triggered.
AutoShrinkPercentage int
// AutoShrinkMinSize defines the minimum size of the aof file before
// an automatic shrink can occur.
AutoShrinkMinSize int
}
// exctx is a simple b-tree context for ordering by expiration.
type exctx struct {
@ -130,8 +127,9 @@ func Open(path string) (*DB, error) {
db.exps = btree.New(16, &exctx{db})
db.idxs = make(map[string]*index)
db.config = Config{
SyncPolicy: EverySecond,
AutoShrink: defaultAutoShrinkMultiplier,
SyncPolicy: EverySecond,
AutoShrinkPercentage: 100,
AutoShrinkMinSize: 32 * 1024 * 1024,
}
db.persist = path != ":memory:"
if db.persist {
@ -142,7 +140,7 @@ func Open(path string) (*DB, error) {
return nil, err
}
if err := db.load(); err != nil {
db.file.Close()
_ = db.file.Close()
return nil, err
}
db.bufw = bufio.NewWriter(db.file)
@ -349,9 +347,6 @@ func (db *DB) SetConfig(config Config) error {
if db.closed {
return ErrDatabaseClosed
}
if config.AutoShrink < 0 || config.AutoShrink == 1 {
return ErrInvalidAutoShrink
}
switch config.SyncPolicy {
default:
return ErrInvalidSyncPolicy
@ -444,14 +439,20 @@ func (db *DB) backgroundManager() {
t := time.NewTicker(time.Second)
defer t.Stop()
for range t.C {
multiple := 0
autoshink := 0
var shrink bool
// Open a standard view. This will take a full lock of the
// database thus allowing for access to anything we need.
err := db.Update(func(tx *Tx) error {
autoshink = db.config.AutoShrink
if db.keys.Len() > 0 {
multiple = db.aoflen / db.keys.Len()
if db.persist {
pos, err := db.file.Seek(0, 1)
if err != nil {
return err
}
aofsz := int(pos)
if aofsz > db.config.AutoShrinkMinSize {
perc := float64(db.config.AutoShrinkPercentage) / 100.0
shrink = aofsz > db.lastaofsz+int(float64(db.lastaofsz)*perc)
}
}
// produce a list of expired items that need removing
var remove []*dbItem
@ -475,7 +476,7 @@ func (db *DB) backgroundManager() {
// execute a disk sync.
if db.persist && db.config.SyncPolicy == EverySecond &&
flushes != db.flushes {
db.file.Sync()
_ = db.file.Sync()
flushes = db.flushes
}
return nil
@ -483,7 +484,7 @@ func (db *DB) backgroundManager() {
if err == ErrDatabaseClosed {
break
}
if multiple >= autoshink && autoshink > 1 {
if shrink {
if err = db.Shrink(); err != nil {
if err == ErrDatabaseClosed {
break
@ -493,7 +494,7 @@ func (db *DB) backgroundManager() {
}
}
// Shrink will make the database file smaller by removing redundent
// Shrink will make the database file smaller by removing redundant
// log entries. This operation does not block the database.
func (db *DB) Shrink() error {
db.mu.Lock()
@ -522,7 +523,6 @@ func (db *DB) Shrink() error {
tmpname := fname + ".tmp"
// the endpos is used to return to the end of the file when we are
// finished writing all of the current items.
aoflen := db.aoflen
endpos, err := db.file.Seek(0, 2)
if err != nil {
return err
@ -533,13 +533,12 @@ func (db *DB) Shrink() error {
return err
}
defer func() {
f.Close()
os.RemoveAll(tmpname)
_ = f.Close()
_ = os.RemoveAll(tmpname)
}()
// we are going to read items in as chunks as to not hold up the database
// for too long.
naoflen := 0
wr := bufio.NewWriter(f)
pivot := ""
done := false
@ -561,7 +560,6 @@ func (db *DB) Shrink() error {
return false
}
dbi.writeSetTo(wr)
naoflen++
n++
return true
},
@ -590,7 +588,7 @@ func (db *DB) Shrink() error {
if err != nil {
return err
}
defer aof.Close()
defer func() { _ = aof.Close() }()
if _, err := aof.Seek(endpos, 0); err != nil {
return err
}
@ -617,13 +615,13 @@ func (db *DB) Shrink() error {
if err != nil {
panic(err)
}
if _, err := db.file.Seek(0, 2); err != nil {
pos, err := db.file.Seek(0, 2)
if err != nil {
return err
}
// reset the bufio writer
db.bufw = bufio.NewWriter(db.file)
// finally update the aoflen
db.aoflen = naoflen + (db.aoflen - aoflen)
db.lastaofsz = int(pos)
return nil
}()
return err
@ -718,7 +716,6 @@ func (db *DB) load() error {
if len(parts) == 0 {
continue
}
db.aoflen++
switch strings.ToLower(parts[0]) {
default:
return ErrInvalid
@ -750,6 +747,11 @@ func (db *DB) load() error {
db.deleteFromDatabase(item)
}
}
pos, err := db.file.Seek(0, 2)
if err != nil {
return err
}
db.lastaofsz = int(pos)
return nil
}
@ -764,7 +766,7 @@ func (db *DB) managed(writable bool, fn func(tx *Tx) error) (err error) {
defer func() {
if err != nil {
// The caller returned an error. We must rollback.
tx.rollback()
_ = tx.rollback()
return
}
if writable {
@ -914,11 +916,10 @@ func (tx *Tx) commit() error {
tx.rollbackInner()
}
if tx.db.config.SyncPolicy == Always {
tx.db.file.Sync()
_ = tx.db.file.Sync()
}
// Increment the number of flushes. The background syncing uses this.
tx.db.flushes++
tx.db.aoflen += len(tx.commits)
}
// Unlock the database and allow for another writable transaction.
@ -1405,23 +1406,23 @@ func Rect(min, max []float64) string {
}
}
var b bytes.Buffer
b.WriteByte('[')
_ = b.WriteByte('[')
for i, v := range min {
if i > 0 {
b.WriteByte(' ')
_ = b.WriteByte(' ')
}
b.WriteString(strconv.FormatFloat(v, 'f', -1, 64))
_, _ = b.WriteString(strconv.FormatFloat(v, 'f', -1, 64))
}
if diff {
b.WriteString("],[")
_, _ = b.WriteString("],[")
for i, v := range max {
if i > 0 {
b.WriteByte(' ')
_ = b.WriteByte(' ')
}
b.WriteString(strconv.FormatFloat(v, 'f', -1, 64))
_, _ = b.WriteString(strconv.FormatFloat(v, 'f', -1, 64))
}
}
b.WriteByte(']')
_ = b.WriteByte(']')
return b.String()
}

565
buntdb_test.go
View File

@ -15,75 +15,94 @@ import (
)
func TestBackgroudOperations(t *testing.T) {
os.RemoveAll("data.db")
if err := os.RemoveAll("data.db"); err != nil {
t.Fatal(err)
}
db, err := Open("data.db")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll("data.db")
defer db.Close()
defer func() { _ = os.RemoveAll("data.db") }()
defer func() { _ = db.Close() }()
for i := 0; i < 1000; i++ {
if err := db.Update(func(tx *Tx) error {
for j := 0; j < 200; j++ {
tx.Set(fmt.Sprintf("hello%d", j), "planet", nil)
if _, _, err := tx.Set(fmt.Sprintf("hello%d", j), "planet", nil); err != nil {
return err
}
}
if _, _, err := tx.Set("hi", "world", &SetOptions{Expires: true, TTL: time.Second / 2}); err != nil {
return err
}
tx.Set("hi", "world", &SetOptions{Expires: true, TTL: time.Second / 2})
return nil
}); err != nil {
t.Fatal(err)
}
}
n := 0
db.View(func(tx *Tx) error {
n, _ = tx.Len()
return nil
err = db.View(func(tx *Tx) error {
var err error
n, err = tx.Len()
return err
})
if err != nil {
t.Fatal(err)
}
if n != 201 {
t.Fatalf("expecting '%v', got '%v'", 201, n)
}
time.Sleep(time.Millisecond * 1500)
db.Close()
if err := db.Close(); err != nil {
t.Fatal(err)
}
db, err = Open("data.db")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll("data.db")
defer db.Close()
defer func() { _ = os.RemoveAll("data.db") }()
defer func() { _ = db.Close() }()
n = 0
db.View(func(tx *Tx) error {
n, _ = tx.Len()
return nil
err = db.View(func(tx *Tx) error {
var err error
n, err = tx.Len()
return err
})
if n != 200 {
t.Fatalf("expecting '%v', got '%v'", 200, n)
}
}
func TestVariousTx(t *testing.T) {
os.RemoveAll("data.db")
if err := os.RemoveAll("data.db"); err != nil {
t.Fatal(err)
}
db, err := Open("data.db")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll("data.db")
defer db.Close()
defer func() { _ = os.RemoveAll("data.db") }()
defer func() { _ = db.Close() }()
if err := db.Update(func(tx *Tx) error {
tx.Set("hello", "planet", nil)
return nil
_, _, err := tx.Set("hello", "planet", nil)
return err
}); err != nil {
t.Fatal(err)
}
errBroken := errors.New("broken")
if err := db.Update(func(tx *Tx) error {
tx.Set("hello", "world", nil)
_, _, _ = tx.Set("hello", "world", nil)
return errBroken
}); err != errBroken {
t.Fatalf("did not correctly receive the user-defined transaction error.")
}
var val string
db.View(func(tx *Tx) error {
val, _ = tx.Get("hello")
return nil
err = db.View(func(tx *Tx) error {
var err error
val, err = tx.Get("hello")
return err
})
if err != nil {
t.Fatal(err)
}
if val == "world" {
t.Fatal("a rollbacked transaction got through")
}
@ -116,7 +135,9 @@ func TestVariousTx(t *testing.T) {
if _, err := tx.Delete("something"); err != ErrNotFound {
t.Fatalf("expecting not found error")
}
tx.Set("var", "val", &SetOptions{Expires: true, TTL: 0})
if _, _, err := tx.Set("var", "val", &SetOptions{Expires: true, TTL: 0}); err != nil {
t.Fatal(err)
}
if _, err := tx.Get("var"); err != ErrNotFound {
t.Fatalf("expecting not found error")
}
@ -139,8 +160,7 @@ func TestVariousTx(t *testing.T) {
}
}
}()
tx.commit()
return nil
return tx.commit()
}); err != nil {
t.Fatal(err)
}
@ -155,8 +175,7 @@ func TestVariousTx(t *testing.T) {
}
}
}()
tx.rollback()
return nil
return tx.rollback()
}); err != nil {
t.Fatal(err)
}
@ -188,9 +207,11 @@ func TestVariousTx(t *testing.T) {
db.mu.RUnlock()
// flush to unwritable file
if err := db.Update(func(tx *Tx) error {
tx.Set("var1", "val1", nil)
tx.db.file.Close()
return nil
_, _, err := tx.Set("var1", "val1", nil)
if err != nil {
t.Fatal(err)
}
return tx.db.file.Close()
}); err == nil {
t.Fatal("should not be able to commit when the file is closed")
}
@ -202,20 +223,25 @@ func TestVariousTx(t *testing.T) {
t.Fatal(err)
}
db.bufw = bufio.NewWriter(db.file)
db.CreateIndex("blank", "*", nil)
if err := db.CreateIndex("blank", "*", nil); err != nil {
t.Fatal(err)
}
// test scanning
if err := db.Update(func(tx *Tx) error {
tx.Set("nothing", "here", nil)
return nil
_, _, err := tx.Set("nothing", "here", nil)
return err
}); err != nil {
t.Fatal(err)
}
if err := db.View(func(tx *Tx) error {
s := ""
tx.Ascend("", func(key, val string) bool {
err := tx.Ascend("", func(key, val string) bool {
s += key + ":" + val + "\n"
return true
})
if err != nil {
return err
}
if s != "hello:planet\nnothing:here\n" {
t.Fatal("invalid scan")
}
@ -235,45 +261,62 @@ func TestVariousTx(t *testing.T) {
t.Fatal(err)
}
s = ""
tx.AscendLessThan("", "liger", func(key, val string) bool {
err = tx.AscendLessThan("", "liger", func(key, val string) bool {
s += key + ":" + val + "\n"
return true
})
if err != nil {
return err
}
if s != "hello:planet\n" {
t.Fatal("invalid scan")
}
s = ""
tx.Descend("", func(key, val string) bool {
err = tx.Descend("", func(key, val string) bool {
s += key + ":" + val + "\n"
return true
})
if err != nil {
return err
}
if s != "nothing:here\nhello:planet\n" {
t.Fatal("invalid scan")
}
s = ""
tx.DescendLessOrEqual("", "liger", func(key, val string) bool {
err = tx.DescendLessOrEqual("", "liger", func(key, val string) bool {
s += key + ":" + val + "\n"
return true
})
if err != nil {
return err
}
if s != "hello:planet\n" {
t.Fatal("invalid scan")
}
s = ""
tx.DescendGreaterThan("", "liger", func(key, val string) bool {
err = tx.DescendGreaterThan("", "liger", func(key, val string) bool {
s += key + ":" + val + "\n"
return true
})
if err != nil {
return err
}
if s != "nothing:here\n" {
t.Fatal("invalid scan")
}
s = ""
tx.DescendRange("", "liger", "apple", func(key, val string) bool {
err = tx.DescendRange("", "liger", "apple", func(key, val string) bool {
s += key + ":" + val + "\n"
return true
})
if err != nil {
return err
}
if s != "hello:planet\n" {
t.Fatal("invalid scan")
}
@ -283,22 +326,35 @@ func TestVariousTx(t *testing.T) {
}
// test some spatial stuff
db.CreateSpatialIndex("spat", "rect:*", IndexRect)
db.CreateSpatialIndex("junk", "rect:*", nil)
db.Update(func(tx *Tx) error {
tx.Set("rect:1", "[10 10],[20 20]", nil)
tx.Set("rect:2", "[15 15],[25 25]", nil)
tx.Set("shape:1", "[12 12],[25 25]", nil)
if err := db.CreateSpatialIndex("spat", "rect:*", IndexRect); err != nil {
t.Fatal(err)
}
if err := db.CreateSpatialIndex("junk", "rect:*", nil); err != nil {
t.Fatal(err)
}
err = db.Update(func(tx *Tx) error {
if _, _, err := tx.Set("rect:1", "[10 10],[20 20]", nil); err != nil {
return err
}
if _, _, err := tx.Set("rect:2", "[15 15],[25 25]", nil); err != nil {
return err
}
if _, _, err := tx.Set("shape:1", "[12 12],[25 25]", nil); err != nil {
return err
}
s := ""
tx.Intersects("spat", "[5 5],[13 13]", func(key, val string) bool {
err := tx.Intersects("spat", "[5 5],[13 13]", func(key, val string) bool {
s += key + ":" + val + "\n"
return true
})
if err != nil {
return err
}
if s != "rect:1:[10 10],[20 20]\n" {
t.Fatal("invalid scan")
}
tx.db = nil
err := tx.Intersects("spat", "[5 5],[13 13]", func(key, val string) bool {
err = tx.Intersects("spat", "[5 5],[13 13]", func(key, val string) bool {
return true
})
if err != ErrTxClosed {
@ -338,9 +394,14 @@ func TestVariousTx(t *testing.T) {
tx.db = db
return nil
})
if err != nil {
t.Fatal(err)
}
// test after closing
db.Close()
if err := db.Close(); err != nil {
t.Fatal(err)
}
if err := db.Update(func(tx *Tx) error { return nil }); err != ErrDatabaseClosed {
t.Fatalf("should not be able to perform transactionso on a closed database.")
}
@ -354,6 +415,72 @@ func TestNoExpiringItem(t *testing.T) {
t.Fatal("item min,max should both be nil")
}
}
func TestAutoShrink(t *testing.T) {
if err := os.RemoveAll("data.db"); err != nil {
t.Fatal(err)
}
db, err := Open("data.db")
if err != nil {
t.Fatal(err)
}
defer func() { _ = os.RemoveAll("data.db") }()
defer func() { _ = db.Close() }()
for i := 0; i < 1000; i++ {
err = db.Update(func(tx *Tx) error {
for i := 0; i < 20; i++ {
if _, _, err := tx.Set(fmt.Sprintf("HELLO:%d", i), "WORLD", nil); err != nil {
return err
}
}
return nil
})
if err != nil {
t.Fatal(err)
}
}
if err := db.Close(); err != nil {
t.Fatal(err)
}
db, err = Open("data.db")
if err != nil {
t.Fatal(err)
}
db.config.AutoShrinkMinSize = 64 * 1024 // 64K
for i := 0; i < 2000; i++ {
err = db.Update(func(tx *Tx) error {
for i := 0; i < 20; i++ {
if _, _, err := tx.Set(fmt.Sprintf("HELLO:%d", i), "WORLD", nil); err != nil {
return err
}
}
return nil
})
if err != nil {
t.Fatal(err)
}
}
time.Sleep(time.Second * 3)
if err := db.Close(); err != nil {
t.Fatal(err)
}
db, err = Open("data.db")
if err != nil {
t.Fatal(err)
}
err = db.View(func(tx *Tx) error {
n, err := tx.Len()
if err != nil {
return err
}
if n != 20 {
t.Fatalf("expecting 20, got %v", n)
}
return nil
})
if err != nil {
t.Fatal(err)
}
}
// test database format loading
func TestDatabaseFormat(t *testing.T) {
@ -365,22 +492,34 @@ func TestDatabaseFormat(t *testing.T) {
"*2\r\n$3\r\ndel\r\n$4\r\nvar1\r\n",
"*5\r\n$3\r\nset\r\n$3\r\nvar\r\n$3\r\nval\r\n$2\r\nex\r\n$2\r\n10\r\n",
}, "")
os.RemoveAll("data.db")
ioutil.WriteFile("data.db", []byte(resp), 0666)
if err := os.RemoveAll("data.db"); err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile("data.db", []byte(resp), 0666); err != nil {
t.Fatal(err)
}
db, err := Open("data.db")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll("data.db")
defer db.Close()
defer func() { _ = os.RemoveAll("data.db") }()
defer func() { _ = db.Close() }()
}()
testBadFormat := func(resp string) {
os.RemoveAll("data.db")
ioutil.WriteFile("data.db", []byte(resp), 0666)
if err := os.RemoveAll("data.db"); err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile("data.db", []byte(resp), 0666); err != nil {
t.Fatal(err)
}
db, err := Open("data.db")
if err == nil {
db.Close()
os.RemoveAll("data.db")
if err := db.Close(); err != nil {
t.Fatal(err)
}
if err := os.RemoveAll("data.db"); err != nil {
t.Fatal(err)
}
t.Fatalf("invalid database should not be allowed")
}
}
@ -402,19 +541,31 @@ func TestDatabaseFormat(t *testing.T) {
}
func TestInsertsAndDeleted(t *testing.T) {
os.RemoveAll("data.db")
if err := os.RemoveAll("data.db"); err != nil {
t.Fatal(err)
}
db, err := Open("data.db")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll("data.db")
defer db.Close()
db.CreateIndex("any", "*", IndexString)
db.CreateSpatialIndex("rect", "*", IndexRect)
defer func() { _ = os.RemoveAll("data.db") }()
defer func() { _ = db.Close() }()
if err := db.CreateIndex("any", "*", IndexString); err != nil {
t.Fatal(err)
}
if err := db.CreateSpatialIndex("rect", "*", IndexRect); err != nil {
t.Fatal(err)
}
if err := db.Update(func(tx *Tx) error {
tx.Set("item1", "value1", &SetOptions{Expires: true, TTL: time.Second})
tx.Set("item2", "value2", nil)
tx.Set("item3", "value3", &SetOptions{Expires: true, TTL: time.Second})
if _, _, err := tx.Set("item1", "value1", &SetOptions{Expires: true, TTL: time.Second}); err != nil {
return err
}
if _, _, err := tx.Set("item2", "value2", nil); err != nil {
return err
}
if _, _, err := tx.Set("item3", "value3", &SetOptions{Expires: true, TTL: time.Second}); err != nil {
return err
}
return nil
}); err != nil {
t.Fatal(err)
@ -482,36 +633,50 @@ func TestIndexCompare(t *testing.T) {
// test opening a folder.
func TestOpeningAFolder(t *testing.T) {
os.RemoveAll("dir.tmp")
os.Mkdir("dir.tmp", 0700)
defer os.RemoveAll("dir.tmp")
if err := os.RemoveAll("dir.tmp"); err != nil {
t.Fatal(err)
}
if err := os.Mkdir("dir.tmp", 0700); err != nil {
t.Fatal(err)
}
defer func() { _ = os.RemoveAll("dir.tmp") }()
db, err := Open("dir.tmp")
if err == nil {
db.Close()
if err := db.Close(); err != nil {
t.Fatal(err)
}
t.Fatalf("opening a directory should not be allowed")
}
}
// test opening an invalid resp file.
func TestOpeningInvalidDatabaseFile(t *testing.T) {
os.RemoveAll("data.db")
ioutil.WriteFile("data.db", []byte("invalid\r\nfile"), 0666)
defer os.RemoveAll("data.db")
if err := os.RemoveAll("data.db"); err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile("data.db", []byte("invalid\r\nfile"), 0666); err != nil {
t.Fatal(err)
}
defer func() { _ = os.RemoveAll("data.db") }()
db, err := Open("data.db")
if err == nil {
db.Close()
if err := db.Close(); err != nil {
t.Fatal(err)
}
t.Fatalf("invalid database should not be allowed")
}
}
// test closing a closed database.
func TestOpeningClosedDatabase(t *testing.T) {
os.RemoveAll("data.db")
if err := os.RemoveAll("data.db"); err != nil {
t.Fatal(err)
}
db, err := Open("data.db")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll("data.db")
defer func() { _ = os.RemoveAll("data.db") }()
if err := db.Close(); err != nil {
t.Fatal(err)
}
@ -532,13 +697,15 @@ func TestOpeningClosedDatabase(t *testing.T) {
// test shrinking a database.
func TestShrink(t *testing.T) {
os.RemoveAll("data.db")
if err := os.RemoveAll("data.db"); err != nil {
t.Fatal(err)
}
db, err := Open("data.db")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll("data.db")
defer db.Close()
defer func() { _ = os.RemoveAll("data.db") }()
defer func() { _ = db.Close() }()
if err := db.Shrink(); err != nil {
t.Fatal(err)
}
@ -550,20 +717,30 @@ func TestShrink(t *testing.T) {
t.Fatalf("expected %v, got %v", 0, fi.Size())
}
// add 10 items
db.Update(func(tx *Tx) error {
err = db.Update(func(tx *Tx) error {
for i := 0; i < 10; i++ {
tx.Set(fmt.Sprintf("key%d", i), fmt.Sprintf("val%d", i), nil)
if _, _, err := tx.Set(fmt.Sprintf("key%d", i), fmt.Sprintf("val%d", i), nil); err != nil {
return err
}
}
return nil
})
if err != nil {
t.Fatal(err)
}
// add the same 10 items
// this will create 10 duplicate log entries
db.Update(func(tx *Tx) error {
err = db.Update(func(tx *Tx) error {
for i := 0; i < 10; i++ {
tx.Set(fmt.Sprintf("key%d", i), fmt.Sprintf("val%d", i), nil)
if _, _, err := tx.Set(fmt.Sprintf("key%d", i), fmt.Sprintf("val%d", i), nil); err != nil {
return err
}
}
return nil
})
if err != nil {
t.Fatal(err)
}
fi, err = os.Stat("data.db")
if err != nil {
t.Fatal(err)
@ -583,7 +760,9 @@ func TestShrink(t *testing.T) {
if sz2 >= sz1 {
t.Fatalf("expected < %v, got %v", sz1, sz2)
}
db.Close()
if err := db.Close(); err != nil {
t.Fatal(err)
}
if err := db.Shrink(); err != ErrDatabaseClosed {
t.Fatal("shrink on a closed databse should not be allowed")
}
@ -592,24 +771,33 @@ func TestShrink(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer db.Close()
defer func() { _ = db.Close() }()
// add 10 items
db.Update(func(tx *Tx) error {
err = db.Update(func(tx *Tx) error {
for i := 0; i < 10; i++ {
tx.Set(fmt.Sprintf("key%d", i), fmt.Sprintf("val%d", i), nil)
if _, _, err := tx.Set(fmt.Sprintf("key%d", i), fmt.Sprintf("val%d", i), nil); err != nil {
return err
}
}
return nil
})
if err != nil {
t.Fatal(err)
}
// add the same 10 items
// this will create 10 duplicate log entries
db.Update(func(tx *Tx) error {
err = db.Update(func(tx *Tx) error {
for i := 0; i < 10; i++ {
tx.Set(fmt.Sprintf("key%d", i), fmt.Sprintf("val%d", i), nil)
if _, _, err := tx.Set(fmt.Sprintf("key%d", i), fmt.Sprintf("val%d", i), nil); err != nil {
return err
}
}
return nil
})
db.View(func(tx *Tx) error {
if err != nil {
t.Fatal(err)
}
err = db.View(func(tx *Tx) error {
n, err := tx.Len()
if err != nil {
t.Fatal(err)
@ -619,6 +807,9 @@ func TestShrink(t *testing.T) {
}
return nil
})
if err != nil {
t.Fatal(err)
}
// this should succeed even though it's basically a noop.
if err := db.Shrink(); err != nil {
t.Fatal(err)
@ -626,13 +817,15 @@ func TestShrink(t *testing.T) {
}
func TestVariousIndexOperations(t *testing.T) {
os.RemoveAll("data.db")
if err := os.RemoveAll("data.db"); err != nil {
t.Fatal(err)
}
db, err := Open("data.db")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll("data.db")
defer db.Close()
defer func() { _ = os.RemoveAll("data.db") }()
defer func() { _ = db.Close() }()
// test creating an index with no index name.
err = db.CreateIndex("", "", nil)
if err == nil {
@ -647,15 +840,31 @@ func TestVariousIndexOperations(t *testing.T) {
if err == nil {
t.Fatal("should not be able to create a duplicate index")
}
db.Update(func(tx *Tx) error {
tx.Set("user:1", "tom", nil)
tx.Set("user:2", "janet", nil)
tx.Set("alt:1", "from", nil)
tx.Set("alt:2", "there", nil)
tx.Set("rect:1", "[1 2],[3 4]", nil)
tx.Set("rect:2", "[5 6],[7 8]", nil)
err = db.Update(func(tx *Tx) error {
if _, _, err := tx.Set("user:1", "tom", nil); err != nil {
return err
}
if _, _, err := tx.Set("user:2", "janet", nil); err != nil {
return err
}
if _, _, err := tx.Set("alt:1", "from", nil); err != nil {
return err
}
if _, _, err := tx.Set("alt:2", "there", nil); err != nil {
return err
}
if _, _, err := tx.Set("rect:1", "[1 2],[3 4]", nil); err != nil {
return err
}
if _, _, err := tx.Set("rect:2", "[5 6],[7 8]", nil); err != nil {
return err
}
return nil
})
if err != nil {
t.Fatal(err)
}
// test creating an index after adding items. use pattern matching. have some items in the match and some not.
if err := db.CreateIndex("string", "user:*", IndexString); err != nil {
t.Fatal(err)
@ -727,35 +936,60 @@ func TestPatternMatching(t *testing.T) {
func TestBasic(t *testing.T) {
rand.Seed(time.Now().UnixNano())
os.RemoveAll("data.db")
if err := os.RemoveAll("data.db"); err != nil {
t.Fatal(err)
}
db, err := Open("data.db")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll("data.db")
defer db.Close()
defer func() { _ = os.RemoveAll("data.db") }()
defer func() { _ = db.Close() }()
// create a simple index
db.CreateIndex("users", "fun:user:*", IndexString)
if err := db.CreateIndex("users", "fun:user:*", IndexString); err != nil {
t.Fatal(err)
}
// create a spatial index
db.CreateSpatialIndex("rects", "rect:*", IndexRect)
if err := db.CreateSpatialIndex("rects", "rect:*", IndexRect); err != nil {
t.Fatal(err)
}
if true {
db.Update(func(tx *Tx) error {
tx.Set("fun:user:0", "tom", nil)
tx.Set("fun:user:1", "Randi", nil)
tx.Set("fun:user:2", "jane", nil)
tx.Set("fun:user:4", "Janet", nil)
tx.Set("fun:user:5", "Paula", nil)
tx.Set("fun:user:6", "peter", nil)
tx.Set("fun:user:7", "Terri", nil)
err := db.Update(func(tx *Tx) error {
if _, _, err := tx.Set("fun:user:0", "tom", nil); err != nil {
return err
}
if _, _, err := tx.Set("fun:user:1", "Randi", nil); err != nil {
return err
}
if _, _, err := tx.Set("fun:user:2", "jane", nil); err != nil {
return err
}
if _, _, err := tx.Set("fun:user:4", "Janet", nil); err != nil {
return err
}
if _, _, err := tx.Set("fun:user:5", "Paula", nil); err != nil {
return err
}
if _, _, err := tx.Set("fun:user:6", "peter", nil); err != nil {
return err
}
if _, _, err := tx.Set("fun:user:7", "Terri", nil); err != nil {
return err
}
return nil
})
if err != nil {
t.Fatal(err)
}
// add some random items
start := time.Now()
if err := db.Update(func(tx *Tx) error {
for _, i := range rand.Perm(100) {
tx.Set(fmt.Sprintf("tag:%d", i+100), fmt.Sprintf("val:%d", rand.Int()%100+100), nil)
if _, _, err := tx.Set(fmt.Sprintf("tag:%d", i+100), fmt.Sprintf("val:%d", rand.Int()%100+100), nil); err != nil {
return err
}
}
return nil
}); err != nil {
@ -766,9 +1000,15 @@ func TestBasic(t *testing.T) {
}
// add some random rects
if err := db.Update(func(tx *Tx) error {
tx.Set("rect:1", Rect([]float64{10, 10}, []float64{20, 20}), nil)
tx.Set("rect:2", Rect([]float64{15, 15}, []float64{24, 24}), nil)
tx.Set("rect:3", Rect([]float64{17, 17}, []float64{27, 27}), nil)
if _, _, err := tx.Set("rect:1", Rect([]float64{10, 10}, []float64{20, 20}), nil); err != nil {
return err
}
if _, _, err := tx.Set("rect:2", Rect([]float64{15, 15}, []float64{24, 24}), nil); err != nil {
return err
}
if _, _, err := tx.Set("rect:3", Rect([]float64{17, 17}, []float64{27, 27}), nil); err != nil {
return err
}
return nil
}); err != nil {
t.Fatal(err)
@ -776,11 +1016,14 @@ func TestBasic(t *testing.T) {
}
// verify the data has been created
buf := &bytes.Buffer{}
db.View(func(tx *Tx) error {
tx.Ascend("users", func(key, val string) bool {
err = db.View(func(tx *Tx) error {
err = tx.Ascend("users", func(key, val string) bool {
fmt.Fprintf(buf, "%s %s\n", key, val)
return true
})
if err != nil {
t.Fatal(err)
}
err = tx.AscendRange("", "tag:170", "tag:172", func(key, val string) bool {
fmt.Fprintf(buf, "%s\n", key)
return true
@ -805,7 +1048,7 @@ func TestBasic(t *testing.T) {
})
expect := make([]string, 2)
n := 0
tx.Intersects("rects", "[0 0],[15 15]", func(key, val string) bool {
err = tx.Intersects("rects", "[0 0],[15 15]", func(key, val string) bool {
if n == 2 {
t.Fatalf("too many rects where received, expecting only two")
}
@ -819,11 +1062,19 @@ func TestBasic(t *testing.T) {
n++
return true
})
if err != nil {
t.Fatal(err)
}
for _, s := range expect {
buf.WriteString(s)
if _, err := buf.WriteString(s); err != nil {
return err
}
}
return nil
})
if err != nil {
t.Fatal(err)
}
res := `
fun:user:2 jane
fun:user:4 Janet
@ -897,19 +1148,28 @@ func TestRectStrings(t *testing.T) {
}
func TestTTL(t *testing.T) {
os.RemoveAll("data.db")
if err := os.RemoveAll("data.db"); err != nil {
t.Fatal(err)
}
db, err := Open("data.db")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll("data.db")
defer db.Close()
db.Update(func(tx *Tx) error {
tx.Set("key1", "val1", &SetOptions{Expires: true, TTL: time.Second})
tx.Set("key2", "val2", nil)
defer func() { _ = os.RemoveAll("data.db") }()
defer func() { _ = db.Close() }()
err = db.Update(func(tx *Tx) error {
if _, _, err := tx.Set("key1", "val1", &SetOptions{Expires: true, TTL: time.Second}); err != nil {
return err
}
if _, _, err := tx.Set("key2", "val2", nil); err != nil {
return err
}
return nil
})
db.View(func(tx *Tx) error {
if err != nil {
t.Fatal(err)
}
err = db.View(func(tx *Tx) error {
dur1, err := tx.TTL("key1")
if err != nil {
t.Fatal(err)
@ -926,35 +1186,22 @@ func TestTTL(t *testing.T) {
}
return nil
})
if err != nil {
t.Fatal(err)
}
}
func TestConfig(t *testing.T) {
os.RemoveAll("data.db")
if err := os.RemoveAll("data.db"); err != nil {
t.Fatal(err)
}
db, err := Open("data.db")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll("data.db")
defer db.Close()
defer func() { _ = os.RemoveAll("data.db") }()
defer func() { _ = db.Close() }()
err = db.SetConfig(Config{AutoShrink: -1})
if err == nil {
t.Fatal("expecting a config autoshrink error")
}
err = db.SetConfig(Config{AutoShrink: 1})
if err == nil {
t.Fatal("expecting a config autoshrink error")
}
err = db.SetConfig(Config{AutoShrink: 0})
if err != nil {
t.Fatal(err)
}
for i := 2; i < 50; i++ {
err = db.SetConfig(Config{AutoShrink: i})
if err != nil {
t.Fatal(err)
}
}
err = db.SetConfig(Config{SyncPolicy: SyncPolicy(-1)})
if err == nil {
t.Fatal("expecting a config syncpolicy error")
@ -971,7 +1218,7 @@ func TestConfig(t *testing.T) {
if err != nil {
t.Fatal(err)
}
err = db.SetConfig(Config{AutoShrink: 6, SyncPolicy: Always})
err = db.SetConfig(Config{AutoShrinkMinSize: 100, AutoShrinkPercentage: 200, SyncPolicy: Always})
if err != nil {
t.Fatal(err)
}
@ -980,8 +1227,8 @@ func TestConfig(t *testing.T) {
if err := db.ReadConfig(&c); err != nil {
t.Fatal(err)
}
if c.AutoShrink != 6 || c.SyncPolicy != Always {
t.Fatalf("expecting %v and %v, got %v and %v", 6, Always, c.AutoShrink, c.SyncPolicy)
if c.AutoShrinkMinSize != 100 || c.AutoShrinkPercentage != 200 && c.SyncPolicy != Always {
t.Fatalf("expecting %v, %v, and %v, got %v, %v, and %v", 100, 200, Always, c.AutoShrinkMinSize, c.AutoShrinkPercentage, c.SyncPolicy)
}
}
func testUint64Hex(n uint64) string {
@ -995,9 +1242,13 @@ func textHexUint64(s string) uint64 {
}
func benchClose(t *testing.B, persist bool, db *DB) {
if persist {
os.RemoveAll("data.db")
if err := os.RemoveAll("data.db"); err != nil {
t.Fatal(err)
}
}
if err := db.Close(); err != nil {
t.Fatal(err)
}
db.Close()
}
func benchOpenFillData(t *testing.B, N int,
@ -1009,7 +1260,9 @@ func benchOpenFillData(t *testing.B, N int,
rand.Seed(time.Now().UnixNano())
var err error
if persist {
os.RemoveAll("data.db")
if err := os.RemoveAll("data.db"); err != nil {
t.Fatal(err)
}
db, err = Open("data.db")
} else {
db, err = Open(":memory:")