From f5efc40d48833a54f9919ec8aaa0bea5841d89e8 Mon Sep 17 00:00:00 2001 From: tidwall Date: Thu, 21 Apr 2022 15:00:17 -0700 Subject: [PATCH] wip: MVT output --- go.mod | 1 + go.sum | 2 + internal/server/hooks.go | 2 +- internal/server/live.go | 3 +- internal/server/scan.go | 2 +- internal/server/scanner.go | 334 +++++++++++++++++++++---------------- internal/server/search.go | 17 +- internal/server/token.go | 1 + 8 files changed, 209 insertions(+), 153 deletions(-) diff --git a/go.mod b/go.mod index c956d681..7692d8ff 100644 --- a/go.mod +++ b/go.mod @@ -85,6 +85,7 @@ require ( github.com/tidwall/cities v0.1.0 // indirect github.com/tidwall/grect v0.1.4 // indirect github.com/tidwall/lotsa v1.0.2 // indirect + github.com/tidwall/mvt v0.1.2 // indirect github.com/tidwall/rtred v0.1.2 // indirect github.com/tidwall/tinyqueue v0.1.1 // indirect github.com/xdg/stringprep v1.0.3 // indirect diff --git a/go.sum b/go.sum index ec250d65..82c2d904 100644 --- a/go.sum +++ b/go.sum @@ -348,6 +348,8 @@ github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/mvt v0.1.2 h1:hRNN7ZybDmfksMEF/H61F7v65uBmG9uFDr+xTOu2Opw= +github.com/tidwall/mvt v0.1.2/go.mod h1:UDWI77bePzGClhFHsrPWM9SyzYr9NMZy/uB7BcHXymQ= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/redbench v0.1.0 h1:UZYUMhwMMObQRq5xU4SA3lmlJRztXzqtushDii+AmPo= diff --git a/internal/server/hooks.go b/internal/server/hooks.go index e2c22505..6ed66c32 100644 --- a/internal/server/hooks.go +++ b/internal/server/hooks.go @@ -145,7 +145,7 @@ func (s *Server) cmdSetHook(msg *Message) ( hook.ScanWriter, err = s.newScanWriter( &wr, cmsg, args.key, args.output, args.precision, args.glob, false, args.cursor, args.limit, args.wheres, args.whereins, args.whereevals, - args.nofields) + args.nofields, args.mvt) if err != nil { return NOMessage, d, err diff --git a/internal/server/live.go b/internal/server/live.go index 90d0a116..68844115 100644 --- a/internal/server/live.go +++ b/internal/server/live.go @@ -107,7 +107,8 @@ func (s *Server) goLive( s.mu.RLock() sw, err = s.newScanWriter( &wr, msg, lfs.key, lfs.output, lfs.precision, lfs.glob, false, - lfs.cursor, lfs.limit, lfs.wheres, lfs.whereins, lfs.whereevals, lfs.nofields) + lfs.cursor, lfs.limit, lfs.wheres, lfs.whereins, lfs.whereevals, + lfs.nofields, lfs.mvt) s.mu.RUnlock() // everything below if for live SCAN, NEARBY, WITHIN, INTERSECTS diff --git a/internal/server/scan.go b/internal/server/scan.go index ad595a12..8c2ddebe 100644 --- a/internal/server/scan.go +++ b/internal/server/scan.go @@ -48,7 +48,7 @@ func (s *Server) cmdScan(msg *Message) (res resp.Value, err error) { sw, err := s.newScanWriter( wr, msg, args.key, args.output, args.precision, args.glob, false, args.cursor, args.limit, args.wheres, args.whereins, args.whereevals, - args.nofields) + args.nofields, args.mvt) if err != nil { return NOMessage, err } diff --git a/internal/server/scanner.go b/internal/server/scanner.go index 7f73a649..20d13532 100644 --- a/internal/server/scanner.go +++ b/internal/server/scanner.go @@ -2,6 +2,7 @@ package server import ( "bytes" + "encoding/base64" "errors" "math" "strconv" @@ -9,6 +10,7 @@ import ( "github.com/mmcloughlin/geohash" "github.com/tidwall/geojson" + "github.com/tidwall/mvt" "github.com/tidwall/resp" "github.com/tidwall/tile38/internal/clip" "github.com/tidwall/tile38/internal/collection" @@ -56,7 +58,9 @@ type scanWriter struct { globSingle bool fullFields bool values []resp.Value + mvtObjs []geojson.Object matchValues bool + mvt bool respOut resp.Value } @@ -77,7 +81,7 @@ func (s *Server) newScanWriter( wr *bytes.Buffer, msg *Message, key string, output outputT, precision uint64, globPattern string, matchValues bool, cursor, limit uint64, wheres []whereT, whereins []whereinT, - whereevals []whereevalT, nofields bool, + whereevals []whereevalT, nofields, mvt bool, ) ( *scanWriter, error, ) { @@ -97,6 +101,7 @@ func (s *Server) newScanWriter( s: s, wr: wr, msg: msg, + mvt: mvt, limit: limit, cursor: cursor, output: output, @@ -155,36 +160,60 @@ func (sw *scanWriter) hasFieldsOutput() bool { func (sw *scanWriter) writeHead() { sw.mu.Lock() defer sw.mu.Unlock() - switch sw.msg.OutputType { - case JSON: - if len(sw.farr) > 0 && sw.hasFieldsOutput() { - sw.wr.WriteString(`,"fields":[`) - for i, field := range sw.farr { - if i > 0 { - sw.wr.WriteByte(',') + if sw.mvt { + sw.wr.WriteString(`,"mvt":"`) + } else { + switch sw.msg.OutputType { + case JSON: + if len(sw.farr) > 0 && sw.hasFieldsOutput() { + sw.wr.WriteString(`,"fields":[`) + for i, field := range sw.farr { + if i > 0 { + sw.wr.WriteByte(',') + } + sw.wr.WriteString(jsonString(field)) } - sw.wr.WriteString(jsonString(field)) + sw.wr.WriteByte(']') } - sw.wr.WriteByte(']') - } - switch sw.output { - case outputIDs: - sw.wr.WriteString(`,"ids":[`) - case outputObjects: - sw.wr.WriteString(`,"objects":[`) - case outputPoints: - sw.wr.WriteString(`,"points":[`) - case outputBounds: - sw.wr.WriteString(`,"bounds":[`) - case outputHashes: - sw.wr.WriteString(`,"hashes":[`) - case outputCount: + switch sw.output { + case outputIDs: + sw.wr.WriteString(`,"ids":[`) + case outputObjects: + sw.wr.WriteString(`,"objects":[`) + case outputPoints: + sw.wr.WriteString(`,"points":[`) + case outputBounds: + sw.wr.WriteString(`,"bounds":[`) + case outputHashes: + sw.wr.WriteString(`,"hashes":[`) + case outputCount: + } + case RESP: } - case RESP: } } +func (sw *scanWriter) compileMVT() []byte { + var tile mvt.Tile + l := tile.AddLayer("default") + l.SetExtent(4096) + + for _, g := range sw.mvtObjs { + _ = g + f := l.AddFeature(mvt.Polygon) + // f.MoveTo(128, 96) + // f.LineTo(148, 128) + // f.LineTo(108, 128) + // f.LineTo(128, 96) + f.ClosePath() + } + + // println(sw.mvtObjs) + + return tile.Render() +} + func (sw *scanWriter) writeFoot() { sw.mu.Lock() defer sw.mu.Unlock() @@ -192,13 +221,22 @@ func (sw *scanWriter) writeFoot() { if !sw.hitLimit { cursor = 0 } + + var mvtTile []byte + if sw.mvt { + mvtTile = sw.compileMVT() + } switch sw.msg.OutputType { case JSON: - switch sw.output { - default: - sw.wr.WriteByte(']') - case outputCount: - + if sw.mvt { + sw.wr.WriteString(base64.RawStdEncoding.EncodeToString(mvtTile)) + sw.wr.WriteByte('"') + } else { + switch sw.output { + default: + sw.wr.WriteByte(']') + case outputCount: + } } sw.wr.WriteString(`,"count":` + strconv.FormatUint(sw.count, 10)) sw.wr.WriteString(`,"cursor":` + strconv.FormatUint(cursor, 10)) @@ -206,9 +244,11 @@ func (sw *scanWriter) writeFoot() { if sw.output == outputCount { sw.respOut = resp.IntegerValue(int(sw.count)) } else { - values := []resp.Value{ - resp.IntegerValue(int(cursor)), - resp.ArrayValue(sw.values), + values := []resp.Value{resp.IntegerValue(int(cursor))} + if sw.mvt { + values = append(values, resp.BytesValue(mvtTile)) + } else { + values = append(values, resp.ArrayValue(sw.values)) } sw.respOut = resp.ArrayValue(values) } @@ -384,134 +424,138 @@ func (sw *scanWriter) writeObject(opts ScanWriterParams) bool { if opts.clip != nil { opts.o = clip.Clip(opts.o, opts.clip, &sw.s.geomIndexOpts) } - switch sw.msg.OutputType { - case JSON: - var wr bytes.Buffer - var jsfields string - if sw.once { - wr.WriteByte(',') - } else { - sw.once = true - } - if sw.hasFieldsOutput() { - if sw.fullFields { - if len(sw.fmap) > 0 { - jsfields = `,"fields":{` - var i int - for field, idx := range sw.fmap { - if len(opts.fields) > idx { - if opts.fields[idx] != 0 { - if i > 0 { - jsfields += `,` + if sw.mvt { + sw.mvtObjs = append(sw.mvtObjs, opts.o) + } else { + switch sw.msg.OutputType { + case JSON: + var wr bytes.Buffer + var jsfields string + if sw.once { + wr.WriteByte(',') + } else { + sw.once = true + } + if sw.hasFieldsOutput() { + if sw.fullFields { + if len(sw.fmap) > 0 { + jsfields = `,"fields":{` + var i int + for field, idx := range sw.fmap { + if len(opts.fields) > idx { + if opts.fields[idx] != 0 { + if i > 0 { + jsfields += `,` + } + jsfields += jsonString(field) + ":" + strconv.FormatFloat(opts.fields[idx], 'f', -1, 64) + i++ } - jsfields += jsonString(field) + ":" + strconv.FormatFloat(opts.fields[idx], 'f', -1, 64) - i++ } } + jsfields += `}` } - jsfields += `}` + + } else if len(sw.farr) > 0 { + jsfields = `,"fields":[` + for i, name := range sw.farr { + if i > 0 { + jsfields += `,` + } + j := sw.fmap[name] + if j < len(opts.fields) { + jsfields += strconv.FormatFloat(opts.fields[j], 'f', -1, 64) + } else { + jsfields += "0" + } + } + jsfields += `]` + } + } + if sw.output == outputIDs { + wr.WriteString(jsonString(opts.id)) + } else { + wr.WriteString(`{"id":` + jsonString(opts.id)) + switch sw.output { + case outputObjects: + wr.WriteString(`,"object":` + string(opts.o.AppendJSON(nil))) + case outputPoints: + wr.WriteString(`,"point":` + string(appendJSONSimplePoint(nil, opts.o))) + case outputHashes: + center := opts.o.Center() + p := geohash.EncodeWithPrecision(center.Y, center.X, uint(sw.precision)) + wr.WriteString(`,"hash":"` + p + `"`) + case outputBounds: + wr.WriteString(`,"bounds":` + string(appendJSONSimpleBounds(nil, opts.o))) } - } else if len(sw.farr) > 0 { - jsfields = `,"fields":[` - for i, name := range sw.farr { - if i > 0 { - jsfields += `,` - } - j := sw.fmap[name] - if j < len(opts.fields) { - jsfields += strconv.FormatFloat(opts.fields[j], 'f', -1, 64) + wr.WriteString(jsfields) + + if opts.distOutput || opts.distance > 0 { + wr.WriteString(`,"distance":` + strconv.FormatFloat(opts.distance, 'f', -1, 64)) + } + + wr.WriteString(`}`) + } + sw.wr.Write(wr.Bytes()) + case RESP: + vals := make([]resp.Value, 1, 3) + vals[0] = resp.StringValue(opts.id) + if sw.output == outputIDs { + sw.values = append(sw.values, vals[0]) + } else { + switch sw.output { + case outputObjects: + vals = append(vals, resp.StringValue(opts.o.String())) + case outputPoints: + point := opts.o.Center() + z := extractZCoordinate(opts.o) + if z != 0 { + vals = append(vals, resp.ArrayValue([]resp.Value{ + resp.FloatValue(point.Y), + resp.FloatValue(point.X), + resp.FloatValue(z), + })) } else { - jsfields += "0" + vals = append(vals, resp.ArrayValue([]resp.Value{ + resp.FloatValue(point.Y), + resp.FloatValue(point.X), + })) } - } - jsfields += `]` - } - } - if sw.output == outputIDs { - wr.WriteString(jsonString(opts.id)) - } else { - wr.WriteString(`{"id":` + jsonString(opts.id)) - switch sw.output { - case outputObjects: - wr.WriteString(`,"object":` + string(opts.o.AppendJSON(nil))) - case outputPoints: - wr.WriteString(`,"point":` + string(appendJSONSimplePoint(nil, opts.o))) - case outputHashes: - center := opts.o.Center() - p := geohash.EncodeWithPrecision(center.Y, center.X, uint(sw.precision)) - wr.WriteString(`,"hash":"` + p + `"`) - case outputBounds: - wr.WriteString(`,"bounds":` + string(appendJSONSimpleBounds(nil, opts.o))) - } - - wr.WriteString(jsfields) - - if opts.distOutput || opts.distance > 0 { - wr.WriteString(`,"distance":` + strconv.FormatFloat(opts.distance, 'f', -1, 64)) - } - - wr.WriteString(`}`) - } - sw.wr.Write(wr.Bytes()) - case RESP: - vals := make([]resp.Value, 1, 3) - vals[0] = resp.StringValue(opts.id) - if sw.output == outputIDs { - sw.values = append(sw.values, vals[0]) - } else { - switch sw.output { - case outputObjects: - vals = append(vals, resp.StringValue(opts.o.String())) - case outputPoints: - point := opts.o.Center() - z := extractZCoordinate(opts.o) - if z != 0 { + case outputHashes: + center := opts.o.Center() + p := geohash.EncodeWithPrecision(center.Y, center.X, uint(sw.precision)) + vals = append(vals, resp.StringValue(p)) + case outputBounds: + bbox := opts.o.Rect() vals = append(vals, resp.ArrayValue([]resp.Value{ - resp.FloatValue(point.Y), - resp.FloatValue(point.X), - resp.FloatValue(z), - })) - } else { - vals = append(vals, resp.ArrayValue([]resp.Value{ - resp.FloatValue(point.Y), - resp.FloatValue(point.X), + resp.ArrayValue([]resp.Value{ + resp.FloatValue(bbox.Min.Y), + resp.FloatValue(bbox.Min.X), + }), + resp.ArrayValue([]resp.Value{ + resp.FloatValue(bbox.Max.Y), + resp.FloatValue(bbox.Max.X), + }), })) } - case outputHashes: - center := opts.o.Center() - p := geohash.EncodeWithPrecision(center.Y, center.X, uint(sw.precision)) - vals = append(vals, resp.StringValue(p)) - case outputBounds: - bbox := opts.o.Rect() - vals = append(vals, resp.ArrayValue([]resp.Value{ - resp.ArrayValue([]resp.Value{ - resp.FloatValue(bbox.Min.Y), - resp.FloatValue(bbox.Min.X), - }), - resp.ArrayValue([]resp.Value{ - resp.FloatValue(bbox.Max.Y), - resp.FloatValue(bbox.Max.X), - }), - })) - } - if sw.hasFieldsOutput() { - fvs := orderFields(sw.fmap, sw.farr, opts.fields) - if len(fvs) > 0 { - fvals := make([]resp.Value, 0, len(fvs)*2) - for i, fv := range fvs { - fvals = append(fvals, resp.StringValue(fv.field), resp.StringValue(strconv.FormatFloat(fv.value, 'f', -1, 64))) - i++ + if sw.hasFieldsOutput() { + fvs := orderFields(sw.fmap, sw.farr, opts.fields) + if len(fvs) > 0 { + fvals := make([]resp.Value, 0, len(fvs)*2) + for i, fv := range fvs { + fvals = append(fvals, resp.StringValue(fv.field), resp.StringValue(strconv.FormatFloat(fv.value, 'f', -1, 64))) + i++ + } + vals = append(vals, resp.ArrayValue(fvals)) } - vals = append(vals, resp.ArrayValue(fvals)) } - } - if opts.distOutput || opts.distance > 0 { - vals = append(vals, resp.FloatValue(opts.distance)) - } + if opts.distOutput || opts.distance > 0 { + vals = append(vals, resp.FloatValue(opts.distance)) + } - sw.values = append(sw.values, resp.ArrayValue(vals)) + sw.values = append(sw.values, resp.ArrayValue(vals)) + } } } sw.numberItems++ diff --git a/internal/server/search.go b/internal/server/search.go index 925683a5..1735befc 100644 --- a/internal/server/search.go +++ b/internal/server/search.go @@ -131,7 +131,7 @@ func parseRectArea(ltyp string, vs []string) (nvs []string, rect *geojson.Rect, Min: geometry.Point{X: minLon, Y: minLat}, Max: geometry.Point{X: maxLon, Y: maxLat}, }) - case "tile": + case "tile", "mvt": var sx, sy, sz string if vs, sx, ok = tokenval(vs); !ok || sx == "" { err = errInvalidNumberOfArguments @@ -188,6 +188,7 @@ func (s *Server) cmdSearchArgs( err = errInvalidNumberOfArguments return } + if lfs.searchScanBaseTokens.output == outputBounds { if cmd == "within" || cmd == "intersects" { if _, err := strconv.ParseFloat(typ, 64); err == nil { @@ -208,6 +209,7 @@ func (s *Server) cmdSearchArgs( err = errInvalidArgument(typ) return } + switch ltyp { case "point": var slat, slon, smeters string @@ -352,11 +354,12 @@ func (s *Server) cmdSearchArgs( if err != nil { return } - case "bounds", "hash", "tile", "quadkey": + case "bounds", "hash", "tile", "mvt", "quadkey": vs, lfs.obj, err = parseRectArea(ltyp, vs) if err != nil { return } + lfs.mvt = ltyp == "mvt" case "get": if lfs.clip { err = errInvalidArgument("cannot clip with get") @@ -463,6 +466,7 @@ var nearbyTypes = map[string]bool{ var withinOrIntersectsTypes = map[string]bool{ "geo": true, "bounds": true, "hash": true, "tile": true, "quadkey": true, "get": true, "object": true, "circle": true, "point": true, "sector": true, + "mvt": true, } func (s *Server) cmdNearby(msg *Message) (res resp.Value, err error) { @@ -489,7 +493,8 @@ func (s *Server) cmdNearby(msg *Message) (res resp.Value, err error) { } sw, err := s.newScanWriter( wr, msg, sargs.key, sargs.output, sargs.precision, sargs.glob, false, - sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, sargs.whereevals, sargs.nofields) + sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, + sargs.whereevals, sargs.nofields, sargs.mvt) if err != nil { return NOMessage, err } @@ -581,7 +586,8 @@ func (s *Server) cmdWithinOrIntersects(cmd string, msg *Message) (res resp.Value } sw, err := s.newScanWriter( wr, msg, sargs.key, sargs.output, sargs.precision, sargs.glob, false, - sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, sargs.whereevals, sargs.nofields) + sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, + sargs.whereevals, sargs.nofields, sargs.mvt) if err != nil { return NOMessage, err } @@ -665,7 +671,8 @@ func (s *Server) cmdSearch(msg *Message) (res resp.Value, err error) { } sw, err := s.newScanWriter( wr, msg, sargs.key, sargs.output, sargs.precision, sargs.glob, true, - sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, sargs.whereevals, sargs.nofields) + sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, + sargs.whereevals, sargs.nofields, sargs.mvt) if err != nil { return NOMessage, err } diff --git a/internal/server/token.go b/internal/server/token.go index 50925626..c6f33664 100644 --- a/internal/server/token.go +++ b/internal/server/token.go @@ -213,6 +213,7 @@ type searchScanBaseTokens struct { clip bool buffer float64 hasbuffer bool + mvt bool } func (s *Server) parseSearchScanBaseTokens(