wip: MVT output

2022-04-21 15:00:17 -07:00 · 2022-04-21 15:00:17 -07:00 · f5efc40d48
parent 036017db4f
commit f5efc40d48
8 changed files with 209 additions and 153 deletions
--- a/go.mod
+++ b/go.mod
@ -85,6 +85,7 @@ require (
 	github.com/tidwall/cities v0.1.0 // indirect
 	github.com/tidwall/grect v0.1.4 // indirect
 	github.com/tidwall/lotsa v1.0.2 // indirect
+	github.com/tidwall/mvt v0.1.2 // indirect
 	github.com/tidwall/rtred v0.1.2 // indirect
 	github.com/tidwall/tinyqueue v0.1.1 // indirect
 	github.com/xdg/stringprep v1.0.3 // indirect
--- a/go.sum
+++ b/go.sum
@ -348,6 +348,8 @@ github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8=
 github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/mvt v0.1.2 h1:hRNN7ZybDmfksMEF/H61F7v65uBmG9uFDr+xTOu2Opw=
+github.com/tidwall/mvt v0.1.2/go.mod h1:UDWI77bePzGClhFHsrPWM9SyzYr9NMZy/uB7BcHXymQ=
 github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tidwall/redbench v0.1.0 h1:UZYUMhwMMObQRq5xU4SA3lmlJRztXzqtushDii+AmPo=
--- a/internal/server/hooks.go
+++ b/internal/server/hooks.go
@ -145,7 +145,7 @@ func (s *Server) cmdSetHook(msg *Message) (
 	hook.ScanWriter, err = s.newScanWriter(
 		&wr, cmsg, args.key, args.output, args.precision, args.glob, false,
 		args.cursor, args.limit, args.wheres, args.whereins, args.whereevals,
-		args.nofields)
+		args.nofields, args.mvt)
 	if err != nil {

 		return NOMessage, d, err
--- a/internal/server/live.go
+++ b/internal/server/live.go
@ -107,7 +107,8 @@ func (s *Server) goLive(
 	s.mu.RLock()
 	sw, err = s.newScanWriter(
 		&wr, msg, lfs.key, lfs.output, lfs.precision, lfs.glob, false,
-		lfs.cursor, lfs.limit, lfs.wheres, lfs.whereins, lfs.whereevals, lfs.nofields)
+		lfs.cursor, lfs.limit, lfs.wheres, lfs.whereins, lfs.whereevals,
+		lfs.nofields, lfs.mvt)
 	s.mu.RUnlock()

 	// everything below if for live SCAN, NEARBY, WITHIN, INTERSECTS
--- a/internal/server/scan.go
+++ b/internal/server/scan.go
@ -48,7 +48,7 @@ func (s *Server) cmdScan(msg *Message) (res resp.Value, err error) {
 	sw, err := s.newScanWriter(
 		wr, msg, args.key, args.output, args.precision, args.glob, false,
 		args.cursor, args.limit, args.wheres, args.whereins, args.whereevals,
-		args.nofields)
+		args.nofields, args.mvt)
 	if err != nil {
 		return NOMessage, err
 	}
--- a/internal/server/scanner.go
+++ b/internal/server/scanner.go
@ -2,6 +2,7 @@ package server

 import (
 	"bytes"
+	"encoding/base64"
 	"errors"
 	"math"
 	"strconv"
@ -9,6 +10,7 @@ import (

 	"github.com/mmcloughlin/geohash"
 	"github.com/tidwall/geojson"
+	"github.com/tidwall/mvt"
 	"github.com/tidwall/resp"
 	"github.com/tidwall/tile38/internal/clip"
 	"github.com/tidwall/tile38/internal/collection"
@ -56,7 +58,9 @@ type scanWriter struct {
 	globSingle     bool
 	fullFields     bool
 	values         []resp.Value
+	mvtObjs        []geojson.Object
 	matchValues    bool
+	mvt            bool
 	respOut        resp.Value
 }

@ -77,7 +81,7 @@ func (s *Server) newScanWriter(
 	wr *bytes.Buffer, msg *Message, key string, output outputT,
 	precision uint64, globPattern string, matchValues bool,
 	cursor, limit uint64, wheres []whereT, whereins []whereinT,
-	whereevals []whereevalT, nofields bool,
+	whereevals []whereevalT, nofields, mvt bool,
 ) (
 	*scanWriter, error,
 ) {
@ -97,6 +101,7 @@ func (s *Server) newScanWriter(
 		s:           s,
 		wr:          wr,
 		msg:         msg,
+		mvt:         mvt,
 		limit:       limit,
 		cursor:      cursor,
 		output:      output,
@ -155,36 +160,60 @@ func (sw *scanWriter) hasFieldsOutput() bool {
 func (sw *scanWriter) writeHead() {
 	sw.mu.Lock()
 	defer sw.mu.Unlock()
-	switch sw.msg.OutputType {
-	case JSON:
-		if len(sw.farr) > 0 && sw.hasFieldsOutput() {
-			sw.wr.WriteString(`,"fields":[`)
-			for i, field := range sw.farr {
-				if i > 0 {
-					sw.wr.WriteByte(',')
+	if sw.mvt {
+		sw.wr.WriteString(`,"mvt":"`)
+	} else {
+		switch sw.msg.OutputType {
+		case JSON:
+			if len(sw.farr) > 0 && sw.hasFieldsOutput() {
+				sw.wr.WriteString(`,"fields":[`)
+				for i, field := range sw.farr {
+					if i > 0 {
+						sw.wr.WriteByte(',')
+					}
+					sw.wr.WriteString(jsonString(field))
 				}
-				sw.wr.WriteString(jsonString(field))
+				sw.wr.WriteByte(']')
 			}
-			sw.wr.WriteByte(']')
-		}
-		switch sw.output {
-		case outputIDs:
-			sw.wr.WriteString(`,"ids":[`)
-		case outputObjects:
-			sw.wr.WriteString(`,"objects":[`)
-		case outputPoints:
-			sw.wr.WriteString(`,"points":[`)
-		case outputBounds:
-			sw.wr.WriteString(`,"bounds":[`)
-		case outputHashes:
-			sw.wr.WriteString(`,"hashes":[`)
-		case outputCount:
+			switch sw.output {
+			case outputIDs:
+				sw.wr.WriteString(`,"ids":[`)
+			case outputObjects:
+				sw.wr.WriteString(`,"objects":[`)
+			case outputPoints:
+				sw.wr.WriteString(`,"points":[`)
+			case outputBounds:
+				sw.wr.WriteString(`,"bounds":[`)
+			case outputHashes:
+				sw.wr.WriteString(`,"hashes":[`)
+			case outputCount:

+			}
+		case RESP:
 		}
-	case RESP:
 	}
 }

+func (sw *scanWriter) compileMVT() []byte {
+	var tile mvt.Tile
+	l := tile.AddLayer("default")
+	l.SetExtent(4096)
+
+	for _, g := range sw.mvtObjs {
+		_ = g
+		f := l.AddFeature(mvt.Polygon)
+		// f.MoveTo(128, 96)
+		// f.LineTo(148, 128)
+		// f.LineTo(108, 128)
+		// f.LineTo(128, 96)
+		f.ClosePath()
+	}
+
+	// println(sw.mvtObjs)
+
+	return tile.Render()
+}
+
 func (sw *scanWriter) writeFoot() {
 	sw.mu.Lock()
 	defer sw.mu.Unlock()
@ -192,13 +221,22 @@ func (sw *scanWriter) writeFoot() {
 	if !sw.hitLimit {
 		cursor = 0
 	}
+
+	var mvtTile []byte
+	if sw.mvt {
+		mvtTile = sw.compileMVT()
+	}
 	switch sw.msg.OutputType {
 	case JSON:
-		switch sw.output {
-		default:
-			sw.wr.WriteByte(']')
-		case outputCount:
-
+		if sw.mvt {
+			sw.wr.WriteString(base64.RawStdEncoding.EncodeToString(mvtTile))
+			sw.wr.WriteByte('"')
+		} else {
+			switch sw.output {
+			default:
+				sw.wr.WriteByte(']')
+			case outputCount:
+			}
 		}
 		sw.wr.WriteString(`,"count":` + strconv.FormatUint(sw.count, 10))
 		sw.wr.WriteString(`,"cursor":` + strconv.FormatUint(cursor, 10))
@ -206,9 +244,11 @@ func (sw *scanWriter) writeFoot() {
 		if sw.output == outputCount {
 			sw.respOut = resp.IntegerValue(int(sw.count))
 		} else {
-			values := []resp.Value{
-				resp.IntegerValue(int(cursor)),
-				resp.ArrayValue(sw.values),
+			values := []resp.Value{resp.IntegerValue(int(cursor))}
+			if sw.mvt {
+				values = append(values, resp.BytesValue(mvtTile))
+			} else {
+				values = append(values, resp.ArrayValue(sw.values))
 			}
 			sw.respOut = resp.ArrayValue(values)
 		}
@ -384,134 +424,138 @@ func (sw *scanWriter) writeObject(opts ScanWriterParams) bool {
 	if opts.clip != nil {
 		opts.o = clip.Clip(opts.o, opts.clip, &sw.s.geomIndexOpts)
 	}
-	switch sw.msg.OutputType {
-	case JSON:
-		var wr bytes.Buffer
-		var jsfields string
-		if sw.once {
-			wr.WriteByte(',')
-		} else {
-			sw.once = true
-		}
-		if sw.hasFieldsOutput() {
-			if sw.fullFields {
-				if len(sw.fmap) > 0 {
-					jsfields = `,"fields":{`
-					var i int
-					for field, idx := range sw.fmap {
-						if len(opts.fields) > idx {
-							if opts.fields[idx] != 0 {
-								if i > 0 {
-									jsfields += `,`
+	if sw.mvt {
+		sw.mvtObjs = append(sw.mvtObjs, opts.o)
+	} else {
+		switch sw.msg.OutputType {
+		case JSON:
+			var wr bytes.Buffer
+			var jsfields string
+			if sw.once {
+				wr.WriteByte(',')
+			} else {
+				sw.once = true
+			}
+			if sw.hasFieldsOutput() {
+				if sw.fullFields {
+					if len(sw.fmap) > 0 {
+						jsfields = `,"fields":{`
+						var i int
+						for field, idx := range sw.fmap {
+							if len(opts.fields) > idx {
+								if opts.fields[idx] != 0 {
+									if i > 0 {
+										jsfields += `,`
+									}
+									jsfields += jsonString(field) + ":" + strconv.FormatFloat(opts.fields[idx], 'f', -1, 64)
+									i++
 								}
-								jsfields += jsonString(field) + ":" + strconv.FormatFloat(opts.fields[idx], 'f', -1, 64)
-								i++
 							}
 						}
+						jsfields += `}`
 					}
-					jsfields += `}`
+
+				} else if len(sw.farr) > 0 {
+					jsfields = `,"fields":[`
+					for i, name := range sw.farr {
+						if i > 0 {
+							jsfields += `,`
+						}
+						j := sw.fmap[name]
+						if j < len(opts.fields) {
+							jsfields += strconv.FormatFloat(opts.fields[j], 'f', -1, 64)
+						} else {
+							jsfields += "0"
+						}
+					}
+					jsfields += `]`
+				}
+			}
+			if sw.output == outputIDs {
+				wr.WriteString(jsonString(opts.id))
+			} else {
+				wr.WriteString(`{"id":` + jsonString(opts.id))
+				switch sw.output {
+				case outputObjects:
+					wr.WriteString(`,"object":` + string(opts.o.AppendJSON(nil)))
+				case outputPoints:
+					wr.WriteString(`,"point":` + string(appendJSONSimplePoint(nil, opts.o)))
+				case outputHashes:
+					center := opts.o.Center()
+					p := geohash.EncodeWithPrecision(center.Y, center.X, uint(sw.precision))
+					wr.WriteString(`,"hash":"` + p + `"`)
+				case outputBounds:
+					wr.WriteString(`,"bounds":` + string(appendJSONSimpleBounds(nil, opts.o)))
 				}

-			} else if len(sw.farr) > 0 {
-				jsfields = `,"fields":[`
-				for i, name := range sw.farr {
-					if i > 0 {
-						jsfields += `,`
-					}
-					j := sw.fmap[name]
-					if j < len(opts.fields) {
-						jsfields += strconv.FormatFloat(opts.fields[j], 'f', -1, 64)
+				wr.WriteString(jsfields)
+
+				if opts.distOutput || opts.distance > 0 {
+					wr.WriteString(`,"distance":` + strconv.FormatFloat(opts.distance, 'f', -1, 64))
+				}
+
+				wr.WriteString(`}`)
+			}
+			sw.wr.Write(wr.Bytes())
+		case RESP:
+			vals := make([]resp.Value, 1, 3)
+			vals[0] = resp.StringValue(opts.id)
+			if sw.output == outputIDs {
+				sw.values = append(sw.values, vals[0])
+			} else {
+				switch sw.output {
+				case outputObjects:
+					vals = append(vals, resp.StringValue(opts.o.String()))
+				case outputPoints:
+					point := opts.o.Center()
+					z := extractZCoordinate(opts.o)
+					if z != 0 {
+						vals = append(vals, resp.ArrayValue([]resp.Value{
+							resp.FloatValue(point.Y),
+							resp.FloatValue(point.X),
+							resp.FloatValue(z),
+						}))
 					} else {
-						jsfields += "0"
+						vals = append(vals, resp.ArrayValue([]resp.Value{
+							resp.FloatValue(point.Y),
+							resp.FloatValue(point.X),
+						}))
 					}
-				}
-				jsfields += `]`
-			}
-		}
-		if sw.output == outputIDs {
-			wr.WriteString(jsonString(opts.id))
-		} else {
-			wr.WriteString(`{"id":` + jsonString(opts.id))
-			switch sw.output {
-			case outputObjects:
-				wr.WriteString(`,"object":` + string(opts.o.AppendJSON(nil)))
-			case outputPoints:
-				wr.WriteString(`,"point":` + string(appendJSONSimplePoint(nil, opts.o)))
-			case outputHashes:
-				center := opts.o.Center()
-				p := geohash.EncodeWithPrecision(center.Y, center.X, uint(sw.precision))
-				wr.WriteString(`,"hash":"` + p + `"`)
-			case outputBounds:
-				wr.WriteString(`,"bounds":` + string(appendJSONSimpleBounds(nil, opts.o)))
-			}
-
-			wr.WriteString(jsfields)
-
-			if opts.distOutput || opts.distance > 0 {
-				wr.WriteString(`,"distance":` + strconv.FormatFloat(opts.distance, 'f', -1, 64))
-			}
-
-			wr.WriteString(`}`)
-		}
-		sw.wr.Write(wr.Bytes())
-	case RESP:
-		vals := make([]resp.Value, 1, 3)
-		vals[0] = resp.StringValue(opts.id)
-		if sw.output == outputIDs {
-			sw.values = append(sw.values, vals[0])
-		} else {
-			switch sw.output {
-			case outputObjects:
-				vals = append(vals, resp.StringValue(opts.o.String()))
-			case outputPoints:
-				point := opts.o.Center()
-				z := extractZCoordinate(opts.o)
-				if z != 0 {
+				case outputHashes:
+					center := opts.o.Center()
+					p := geohash.EncodeWithPrecision(center.Y, center.X, uint(sw.precision))
+					vals = append(vals, resp.StringValue(p))
+				case outputBounds:
+					bbox := opts.o.Rect()
 					vals = append(vals, resp.ArrayValue([]resp.Value{
-						resp.FloatValue(point.Y),
-						resp.FloatValue(point.X),
-						resp.FloatValue(z),
-					}))
-				} else {
-					vals = append(vals, resp.ArrayValue([]resp.Value{
-						resp.FloatValue(point.Y),
-						resp.FloatValue(point.X),
+						resp.ArrayValue([]resp.Value{
+							resp.FloatValue(bbox.Min.Y),
+							resp.FloatValue(bbox.Min.X),
+						}),
+						resp.ArrayValue([]resp.Value{
+							resp.FloatValue(bbox.Max.Y),
+							resp.FloatValue(bbox.Max.X),
+						}),
 					}))
 				}
-			case outputHashes:
-				center := opts.o.Center()
-				p := geohash.EncodeWithPrecision(center.Y, center.X, uint(sw.precision))
-				vals = append(vals, resp.StringValue(p))
-			case outputBounds:
-				bbox := opts.o.Rect()
-				vals = append(vals, resp.ArrayValue([]resp.Value{
-					resp.ArrayValue([]resp.Value{
-						resp.FloatValue(bbox.Min.Y),
-						resp.FloatValue(bbox.Min.X),
-					}),
-					resp.ArrayValue([]resp.Value{
-						resp.FloatValue(bbox.Max.Y),
-						resp.FloatValue(bbox.Max.X),
-					}),
-				}))
-			}

-			if sw.hasFieldsOutput() {
-				fvs := orderFields(sw.fmap, sw.farr, opts.fields)
-				if len(fvs) > 0 {
-					fvals := make([]resp.Value, 0, len(fvs)*2)
-					for i, fv := range fvs {
-						fvals = append(fvals, resp.StringValue(fv.field), resp.StringValue(strconv.FormatFloat(fv.value, 'f', -1, 64)))
-						i++
+				if sw.hasFieldsOutput() {
+					fvs := orderFields(sw.fmap, sw.farr, opts.fields)
+					if len(fvs) > 0 {
+						fvals := make([]resp.Value, 0, len(fvs)*2)
+						for i, fv := range fvs {
+							fvals = append(fvals, resp.StringValue(fv.field), resp.StringValue(strconv.FormatFloat(fv.value, 'f', -1, 64)))
+							i++
+						}
+						vals = append(vals, resp.ArrayValue(fvals))
 					}
-					vals = append(vals, resp.ArrayValue(fvals))
 				}
-			}
-			if opts.distOutput || opts.distance > 0 {
-				vals = append(vals, resp.FloatValue(opts.distance))
-			}
+				if opts.distOutput || opts.distance > 0 {
+					vals = append(vals, resp.FloatValue(opts.distance))
+				}

-			sw.values = append(sw.values, resp.ArrayValue(vals))
+				sw.values = append(sw.values, resp.ArrayValue(vals))
+			}
 		}
 	}
 	sw.numberItems++
--- a/internal/server/search.go
+++ b/internal/server/search.go
@ -131,7 +131,7 @@ func parseRectArea(ltyp string, vs []string) (nvs []string, rect *geojson.Rect,
 			Min: geometry.Point{X: minLon, Y: minLat},
 			Max: geometry.Point{X: maxLon, Y: maxLat},
 		})
-	case "tile":
+	case "tile", "mvt":
 		var sx, sy, sz string
 		if vs, sx, ok = tokenval(vs); !ok || sx == "" {
 			err = errInvalidNumberOfArguments
@ -188,6 +188,7 @@ func (s *Server) cmdSearchArgs(
 		err = errInvalidNumberOfArguments
 		return
 	}
+
 	if lfs.searchScanBaseTokens.output == outputBounds {
 		if cmd == "within" || cmd == "intersects" {
 			if _, err := strconv.ParseFloat(typ, 64); err == nil {
@ -208,6 +209,7 @@ func (s *Server) cmdSearchArgs(
 		err = errInvalidArgument(typ)
 		return
 	}
+
 	switch ltyp {
 	case "point":
 		var slat, slon, smeters string
@ -352,11 +354,12 @@ func (s *Server) cmdSearchArgs(
 		if err != nil {
 			return
 		}
-	case "bounds", "hash", "tile", "quadkey":
+	case "bounds", "hash", "tile", "mvt", "quadkey":
 		vs, lfs.obj, err = parseRectArea(ltyp, vs)
 		if err != nil {
 			return
 		}
+		lfs.mvt = ltyp == "mvt"
 	case "get":
 		if lfs.clip {
 			err = errInvalidArgument("cannot clip with get")
@ -463,6 +466,7 @@ var nearbyTypes = map[string]bool{
 var withinOrIntersectsTypes = map[string]bool{
 	"geo": true, "bounds": true, "hash": true, "tile": true, "quadkey": true,
 	"get": true, "object": true, "circle": true, "point": true, "sector": true,
+	"mvt": true,
 }

 func (s *Server) cmdNearby(msg *Message) (res resp.Value, err error) {
@ -489,7 +493,8 @@ func (s *Server) cmdNearby(msg *Message) (res resp.Value, err error) {
 	}
 	sw, err := s.newScanWriter(
 		wr, msg, sargs.key, sargs.output, sargs.precision, sargs.glob, false,
-		sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, sargs.whereevals, sargs.nofields)
+		sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins,
+		sargs.whereevals, sargs.nofields, sargs.mvt)
 	if err != nil {
 		return NOMessage, err
 	}
@ -581,7 +586,8 @@ func (s *Server) cmdWithinOrIntersects(cmd string, msg *Message) (res resp.Value
 	}
 	sw, err := s.newScanWriter(
 		wr, msg, sargs.key, sargs.output, sargs.precision, sargs.glob, false,
-		sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, sargs.whereevals, sargs.nofields)
+		sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins,
+		sargs.whereevals, sargs.nofields, sargs.mvt)
 	if err != nil {
 		return NOMessage, err
 	}
@ -665,7 +671,8 @@ func (s *Server) cmdSearch(msg *Message) (res resp.Value, err error) {
 	}
 	sw, err := s.newScanWriter(
 		wr, msg, sargs.key, sargs.output, sargs.precision, sargs.glob, true,
-		sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, sargs.whereevals, sargs.nofields)
+		sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins,
+		sargs.whereevals, sargs.nofields, sargs.mvt)
 	if err != nil {
 		return NOMessage, err
 	}
--- a/internal/server/token.go
+++ b/internal/server/token.go
@ -213,6 +213,7 @@ type searchScanBaseTokens struct {
 	clip       bool
 	buffer     float64
 	hasbuffer  bool
+	mvt        bool
 }

 func (s *Server) parseSearchScanBaseTokens(