From 73bb1186170c22c6c77a5dd4f9a0c626c3479577 Mon Sep 17 00:00:00 2001 From: Josh Baker Date: Tue, 5 Mar 2019 09:07:36 -0700 Subject: [PATCH 1/4] Fixed link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 13a48c37..b8db37b0 100644 --- a/README.md +++ b/README.md @@ -272,7 +272,7 @@ set city tempe object {"type":"Polygon","coordinates":[[[0,0],[10,10],[10,0],[0, #### XYZ Tile An XYZ tile is rectangle bounding area on earth that is represented by an X, Y coordinate and a Z (zoom) level. -Check out [maptiler.org](https://www.maptiler.org/google-maps-coordinates-tile-bounds-projection/) for an interactive example. +Check out [maptiler.org](http://www.maptiler.org/google-maps-coordinates-tile-bounds-projection/) for an interactive example. #### QuadKey A QuadKey used the same coordinate system as an XYZ tile except that the string representation is a string characters composed of 0, 1, 2, or 3. For a detailed explanation checkout [The Bing Maps Tile System](https://msdn.microsoft.com/en-us/library/bb259689.aspx). From 95a5556d6191f32322d45493c1408630927e474e Mon Sep 17 00:00:00 2001 From: tidwall Date: Tue, 5 Mar 2019 11:33:37 -0700 Subject: [PATCH 2/4] Added periodic yielding to iterators --- internal/collection/collection.go | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/internal/collection/collection.go b/internal/collection/collection.go index 821f0ff4..ad107e38 100644 --- a/internal/collection/collection.go +++ b/internal/collection/collection.go @@ -1,6 +1,8 @@ package collection import ( + "runtime" + "github.com/tidwall/boxtree/d2" "github.com/tidwall/btree" "github.com/tidwall/geojson" @@ -9,6 +11,9 @@ import ( "github.com/tidwall/tinybtree" ) +// yieldStep forces the iterator to yield goroutine every N steps. +const yieldStep = 0xFF + // Cursor allows for quickly paging through Scan, Within, Intersects, and Nearby type Cursor interface { Offset() uint64 @@ -330,6 +335,9 @@ func (c *Collection) Scan(desc bool, cursor Cursor, if count <= offset { return true } + if count&yieldStep == yieldStep { + runtime.Gosched() + } if cursor != nil { cursor.Step(1) } @@ -361,6 +369,9 @@ func (c *Collection) ScanRange(start, end string, desc bool, cursor Cursor, if count <= offset { return true } + if count&yieldStep == yieldStep { + runtime.Gosched() + } if cursor != nil { cursor.Step(1) } @@ -402,6 +413,9 @@ func (c *Collection) SearchValues(desc bool, cursor Cursor, if count <= offset { return true } + if count&yieldStep == yieldStep { + runtime.Gosched() + } if cursor != nil { cursor.Step(1) } @@ -434,6 +448,9 @@ func (c *Collection) SearchValuesRange(start, end string, desc bool, if count <= offset { return true } + if count&yieldStep == yieldStep { + runtime.Gosched() + } if cursor != nil { cursor.Step(1) } @@ -468,6 +485,9 @@ func (c *Collection) ScanGreaterOrEqual(id string, desc bool, if count <= offset { return true } + if count&yieldStep == yieldStep { + runtime.Gosched() + } if cursor != nil { cursor.Step(1) } @@ -591,6 +611,9 @@ func (c *Collection) Within( if count <= offset { return false, true } + if count&yieldStep == yieldStep { + runtime.Gosched() + } if cursor != nil { cursor.Step(1) } @@ -607,6 +630,9 @@ func (c *Collection) Within( if count <= offset { return true } + if count&yieldStep == yieldStep { + runtime.Gosched() + } if cursor != nil { cursor.Step(1) } @@ -641,6 +667,9 @@ func (c *Collection) Intersects( if count <= offset { return false, true } + if count&yieldStep == yieldStep { + runtime.Gosched() + } if cursor != nil { cursor.Step(1) } @@ -657,6 +686,9 @@ func (c *Collection) Intersects( if count <= offset { return true } + if count&yieldStep == yieldStep { + runtime.Gosched() + } if cursor != nil { cursor.Step(1) } @@ -714,6 +746,9 @@ func (c *Collection) Nearby( if count <= offset { return true } + if count&yieldStep == yieldStep { + runtime.Gosched() + } if cursor != nil { cursor.Step(1) } From 2d255b177d897c3f548268bf6837919b5a354035 Mon Sep 17 00:00:00 2001 From: 0xflotus <0xflotus@gmail.com> Date: Wed, 6 Mar 2019 00:05:38 +0100 Subject: [PATCH 3/4] fixed representation --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b8db37b0..774db1cd 100644 --- a/README.md +++ b/README.md @@ -254,9 +254,9 @@ A bounding box consists of two points. The first being the southwestern most poi set fleet truck1 bounds 30 -110 40 -100 ``` #### Geohash -A [geohash](https://en.wikipedia.org/wiki/Geohash) is a string respresentation of a point. With the length of the string indicating the precision of the point. +A [geohash](https://en.wikipedia.org/wiki/Geohash) is a string representation of a point. With the length of the string indicating the precision of the point. ``` -set fleet truck1 hash 9tbnthxzr # this would be equivlent to 'point 33.5123 -112.2693' +set fleet truck1 hash 9tbnthxzr # this would be equivalent to 'point 33.5123 -112.2693' ``` #### GeoJSON From 5333fab870aac54e547801ac6ad080bc2d522b80 Mon Sep 17 00:00:00 2001 From: tidwall Date: Sun, 10 Mar 2019 10:48:14 -0700 Subject: [PATCH 4/4] Recycle aof buffer --- internal/server/aof.go | 7 ++++++- internal/server/aofshrink.go | 2 +- internal/server/follow.go | 4 ++-- internal/server/server.go | 26 +++++++++++++++++++++++--- 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/internal/server/aof.go b/internal/server/aof.go index c190d128..42a42946 100644 --- a/internal/server/aof.go +++ b/internal/server/aof.go @@ -115,12 +115,17 @@ func commandErrIsFatal(err error) bool { return true } -func (server *Server) flushAOF() { +func (server *Server) flushAOF(sync bool) { if len(server.aofbuf) > 0 { _, err := server.aof.Write(server.aofbuf) if err != nil { panic(err) } + if sync { + if err := server.aof.Sync(); err != nil { + panic(err) + } + } server.aofbuf = server.aofbuf[:0] } } diff --git a/internal/server/aofshrink.go b/internal/server/aofshrink.go index efc1c1c1..095cad79 100644 --- a/internal/server/aofshrink.go +++ b/internal/server/aofshrink.go @@ -236,7 +236,7 @@ func (server *Server) aofshrink() { defer server.mu.Unlock() // flush the aof buffer - server.flushAOF() + server.flushAOF(false) aofbuf = aofbuf[:0] for _, values := range server.shrinklog { diff --git a/internal/server/follow.go b/internal/server/follow.go index 42d27dc5..6d12dd26 100644 --- a/internal/server/follow.go +++ b/internal/server/follow.go @@ -165,7 +165,7 @@ func (c *Server) followHandleCommand(args []string, followc int, w io.Writer) (i return c.aofsz, err } if len(c.aofbuf) > 10240 { - c.flushAOF() + c.flushAOF(false) } return c.aofsz, nil } @@ -291,7 +291,7 @@ func (c *Server) followStep(host string, port int, followc int) error { if aofsz >= int(aofSize) { caughtUp = true c.mu.Lock() - c.flushAOF() + c.flushAOF(false) c.fcup = true c.fcuponce = true c.mu.Unlock() diff --git a/internal/server/server.go b/internal/server/server.go index 94f38922..59f80c4f 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -250,7 +250,7 @@ func Serve(host string, port int, dir string, http bool) error { return err } defer func() { - server.flushAOF() + server.flushAOF(false) server.aof.Sync() }() } @@ -266,6 +266,7 @@ func Serve(host string, port int, dir string, http bool) error { go server.watchLuaStatePool() go server.watchAutoGC() go server.backgroundExpiring() + go server.backgroundSyncAOF() defer func() { // Stop background routines server.followc.add(1) // this will force any follow communication to die @@ -487,7 +488,7 @@ func (server *Server) evioServe() error { if atomic.LoadInt32(&server.aofdirty) != 0 { server.mu.Lock() defer server.mu.Unlock() - server.flushAOF() + server.flushAOF(false) atomic.StoreInt32(&server.aofdirty, 1) } } @@ -665,7 +666,7 @@ func (server *Server) netServe() error { // prewrite server.mu.Lock() defer server.mu.Unlock() - server.flushAOF() + server.flushAOF(false) }() atomic.StoreInt32(&server.aofdirty, 0) } @@ -783,6 +784,25 @@ func (server *Server) watchLuaStatePool() { } } +// backgroundSyncAOF ensures that the aof buffer is does not grow too big. +func (server *Server) backgroundSyncAOF() { + t := time.NewTicker(time.Second) + defer t.Stop() + for range t.C { + if server.stopServer.on() { + return + } + func() { + server.mu.Lock() + defer server.mu.Unlock() + if len(server.aofbuf) > 0 { + server.flushAOF(true) + } + server.aofbuf = nil + }() + } +} + func (server *Server) setCol(key string, col *collection.Collection) { server.cols.Set(key, col) }