diff --git a/cmd/tile38-server/main.go b/cmd/tile38-server/main.go index cbc3d241..d6b36dab 100644 --- a/cmd/tile38-server/main.go +++ b/cmd/tile38-server/main.go @@ -90,6 +90,7 @@ Advanced Options: --http-transport yes/no : HTTP transport (default: yes) --protected-mode yes/no : protected mode (default: yes) --nohup : do not exit on SIGHUP + --metrics-addr addr : The listening addr for Prometheus metrics. Developer Options: --dev : enable developer mode diff --git a/internal/server/hooks.go b/internal/server/hooks.go index b19042b5..583b3fbb 100644 --- a/internal/server/hooks.go +++ b/internal/server/hooks.go @@ -143,7 +143,7 @@ func (s *Server) cmdSetHook(msg *Message) ( hook.ScanWriter, err = s.newScanWriter( &wr, cmsg, args.key, args.output, args.precision, args.globs, false, args.cursor, args.limit, args.wheres, args.whereins, args.whereevals, - args.nofields, args.mvt) + args.nofields, args.mvt, args.tileX, args.tileY, args.tileZ) if err != nil { return NOMessage, d, err diff --git a/internal/server/live.go b/internal/server/live.go index 3be4f761..d9f3d4ce 100644 --- a/internal/server/live.go +++ b/internal/server/live.go @@ -116,7 +116,7 @@ func (s *Server) goLive( sw, err = s.newScanWriter( &wr, msg, lfs.key, lfs.output, lfs.precision, lfs.globs, false, lfs.cursor, lfs.limit, lfs.wheres, lfs.whereins, lfs.whereevals, - lfs.nofields, lfs.mvt) + lfs.nofields, lfs.mvt, lfs.tileX, lfs.tileY, lfs.tileZ) s.mu.RUnlock() // everything below if for live SCAN, NEARBY, WITHIN, INTERSECTS diff --git a/internal/server/scan.go b/internal/server/scan.go index 92c4e757..6ce62b6d 100644 --- a/internal/server/scan.go +++ b/internal/server/scan.go @@ -47,7 +47,7 @@ func (s *Server) cmdScan(msg *Message) (res resp.Value, err error) { sw, err := s.newScanWriter( wr, msg, args.key, args.output, args.precision, args.globs, false, args.cursor, args.limit, args.wheres, args.whereins, args.whereevals, - args.nofields, args.mvt) + args.nofields, args.mvt, args.tileX, args.tileY, args.tileZ) if err != nil { return NOMessage, err } diff --git a/internal/server/scanner.go b/internal/server/scanner.go index 422df7ba..6e187f31 100644 --- a/internal/server/scanner.go +++ b/internal/server/scanner.go @@ -10,6 +10,7 @@ import ( "github.com/mmcloughlin/geohash" "github.com/tidwall/geojson" + "github.com/tidwall/geojson/geometry" "github.com/tidwall/mvt" "github.com/tidwall/resp" "github.com/tidwall/tile38/internal/clip" @@ -58,12 +59,15 @@ type scanWriter struct { globEverything bool fullFields bool values []resp.Value - mvtObjs []geojson.Object + mvtObjs []objPair matchValues bool - mvt bool respOut resp.Value orgWheres []whereT orgWhereins []whereinT + mvt bool + tileX int + tileY int + tileZ int } // ScanWriterParams ... @@ -84,7 +88,7 @@ func (s *Server) newScanWriter( wr *bytes.Buffer, msg *Message, key string, output outputT, precision uint64, globs []string, matchValues bool, cursor, limit uint64, wheres []whereT, whereins []whereinT, - whereevals []whereevalT, nofields, mvt bool, + whereevals []whereevalT, nofields, mvt bool, tileX, tileY, tileZ int, ) ( *scanWriter, error, ) { @@ -105,7 +109,6 @@ func (s *Server) newScanWriter( wr: wr, key: key, msg: msg, - mvt: mvt, globs: globs, limit: limit, cursor: cursor, @@ -116,6 +119,11 @@ func (s *Server) newScanWriter( matchValues: matchValues, } + sw.mvt = mvt + sw.tileX = tileX + sw.tileY = tileY + sw.tileZ = tileZ + if len(globs) == 0 || (len(globs) == 1 && globs[0] == "*") { sw.globEverything = true } @@ -209,23 +217,87 @@ func (sw *scanWriter) writeHead() { } } -func (sw *scanWriter) compileMVT() []byte { - var tile mvt.Tile - l := tile.AddLayer("default") - l.SetExtent(4096) - - for _, g := range sw.mvtObjs { - _ = g - f := l.AddFeature(mvt.Polygon) - // f.MoveTo(128, 96) - // f.LineTo(148, 128) - // f.LineTo(108, 128) - // f.LineTo(128, 96) - f.ClosePath() +func mvtDrawRing(f *mvt.Feature, tileX, tileY, tileZ int, ring geometry.Series, hole bool) { + npoints := ring.NumPoints() + if npoints < 3 { + return } + cw := ring.Clockwise() + reverse := (cw && hole) || (!cw && !hole) + if reverse { + p := ring.PointAt(npoints - 1) + f.MoveTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ)) + for i := npoints - 2; i >= 0; i-- { + p := ring.PointAt(i) + f.LineTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ)) + } + } else { + p := ring.PointAt(0) + f.MoveTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ)) + for i := 1; i < npoints; i++ { + p := ring.PointAt(i) + f.LineTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ)) + } + } + f.ClosePath() +} - // println(sw.mvtObjs) +func mvtAddFeature(l *mvt.Layer, tileX, tileY, tileZ int, p objPair) { + var f *mvt.Feature + switch g := p.obj.(type) { + case *geojson.Point: + f = l.AddFeature(mvt.Point) + p := g.Base() + f.MoveTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ)) + case *geojson.SimplePoint: + f = l.AddFeature(mvt.Point) + p := g + f.MoveTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ)) + case *geojson.LineString: + f = l.AddFeature(mvt.LineString) + line := g.Base() + npoints := line.NumPoints() + if npoints > 0 { + p := line.PointAt(0) + f.MoveTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ)) + for i := 1; i < npoints; i++ { + p := line.PointAt(0) + f.LineTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ)) + } + } + case *geojson.Polygon: + f = l.AddFeature(mvt.Polygon) + poly := g.Base() + mvtDrawRing(f, tileX, tileY, tileZ, poly.Exterior, false) + for _, hole := range poly.Holes { + mvtDrawRing(f, tileX, tileY, tileZ, hole, true) + } + case *geojson.Feature: + mvtAddFeature(l, tileX, tileY, tileZ, objPair{p.id, g.Base()}) + return + default: + if g, ok := g.(geojson.Collection); ok { + for _, g := range g.Children() { + mvtAddFeature(l, tileX, tileY, tileZ, objPair{p.id, g}) + } + } + return + } + f.AddTag("id", p.id) +} +type objPair struct { + id string + obj geojson.Object +} + +func objsToMVT(tileX, tileY, tileZ int, pairs []objPair) []byte { + var tile mvt.Tile + l := tile.AddLayer("tile38") + l.SetExtent(4096) + for _, p := range pairs { + mvtAddFeature(l, tileX, tileY, tileZ, p) + } return tile.Render() } @@ -239,7 +311,7 @@ func (sw *scanWriter) writeFoot() { var mvtTile []byte if sw.mvt { - mvtTile = sw.compileMVT() + mvtTile = objsToMVT(sw.tileX, sw.tileY, sw.tileZ, sw.mvtObjs) } switch sw.msg.OutputType { case JSON: @@ -443,7 +515,7 @@ func (sw *scanWriter) writeObject(opts ScanWriterParams) bool { opts.o = clip.Clip(opts.o, opts.clip, &sw.s.geomIndexOpts) } if sw.mvt { - sw.mvtObjs = append(sw.mvtObjs, opts.o) + sw.mvtObjs = append(sw.mvtObjs, objPair{opts.id, opts.o}) } else { switch sw.msg.OutputType { case JSON: diff --git a/internal/server/search.go b/internal/server/search.go index cce1642f..d20b47b8 100644 --- a/internal/server/search.go +++ b/internal/server/search.go @@ -57,7 +57,9 @@ func (lfs liveFenceSwitches) usingLua() bool { return len(lfs.whereevals) > 0 } -func parseRectArea(ltyp string, vs []string) (nvs []string, rect *geojson.Rect, err error) { +func parseRectArea(ltyp string, vs []string) ( + nvs []string, rect *geojson.Rect, tileX, tileY, tileZ int, err error, +) { var ok bool @@ -145,26 +147,29 @@ func parseRectArea(ltyp string, vs []string) (nvs []string, rect *geojson.Rect, err = errInvalidNumberOfArguments return } - var x, y int64 - var z uint64 - if x, err = strconv.ParseInt(sx, 10, 64); err != nil { + var x, y, z int + if x, err = strconv.Atoi(sx); err != nil || x < 0 { err = errInvalidArgument(sx) return } - if y, err = strconv.ParseInt(sy, 10, 64); err != nil { + if y, err = strconv.Atoi(sy); err != nil || y < 0 { err = errInvalidArgument(sy) return } - if z, err = strconv.ParseUint(sz, 10, 64); err != nil { + if z, err = strconv.Atoi(sz); err != nil || z < 0 || z > 23 { err = errInvalidArgument(sz) return } var minLat, minLon, maxLat, maxLon float64 - minLat, minLon, maxLat, maxLon = bing.TileXYToBounds(x, y, z) + minLat, minLon, maxLat, maxLon = + bing.TileXYToBounds(int64(x), int64(y), uint64(z)) rect = geojson.NewRect(geometry.Rect{ Min: geometry.Point{X: minLon, Y: minLat}, Max: geometry.Point{X: maxLon, Y: maxLat}, }) + tileX = x + tileY = y + tileZ = z } nvs = vs return @@ -355,7 +360,8 @@ func (s *Server) cmdSearchArgs( return } case "bounds", "hash", "tile", "mvt", "quadkey": - vs, lfs.obj, err = parseRectArea(ltyp, vs) + vs, lfs.obj, lfs.tileX, lfs.tileY, lfs.tileZ, err = + parseRectArea(ltyp, vs) if err != nil { return } @@ -435,7 +441,8 @@ func (s *Server) cmdSearchArgs( ltok = strings.ToLower(tok) switch ltok { case "bounds", "hash", "tile", "quadkey": - vs, clip_rect, err = parseRectArea(ltok, vs) + vs, clip_rect, lfs.tileX, lfs.tileY, lfs.tileZ, err = + parseRectArea(ltok, vs) if err == errNotRectangle { err = errInvalidArgument("cannot clipby " + ltok) return @@ -494,7 +501,8 @@ func (s *Server) cmdNearby(msg *Message) (res resp.Value, err error) { sw, err := s.newScanWriter( wr, msg, sargs.key, sargs.output, sargs.precision, sargs.globs, false, sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, - sargs.whereevals, sargs.nofields, sargs.mvt) + sargs.whereevals, sargs.nofields, + sargs.mvt, sargs.tileX, sargs.tileY, sargs.tileZ) if err != nil { return NOMessage, err } @@ -587,7 +595,8 @@ func (s *Server) cmdWithinOrIntersects(cmd string, msg *Message) (res resp.Value sw, err := s.newScanWriter( wr, msg, sargs.key, sargs.output, sargs.precision, sargs.globs, false, sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, - sargs.whereevals, sargs.nofields, sargs.mvt) + sargs.whereevals, sargs.nofields, + sargs.mvt, sargs.tileX, sargs.tileY, sargs.tileZ) if err != nil { return NOMessage, err } @@ -701,7 +710,8 @@ func (s *Server) cmdSearch(msg *Message) (res resp.Value, err error) { sw, err := s.newScanWriter( wr, msg, sargs.key, sargs.output, sargs.precision, sargs.globs, true, sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, - sargs.whereevals, sargs.nofields, sargs.mvt) + sargs.whereevals, sargs.nofields, + sargs.mvt, sargs.tileX, sargs.tileY, sargs.tileZ) if err != nil { return NOMessage, err } diff --git a/internal/server/server.go b/internal/server/server.go index 68ad06c9..080d64cf 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -702,8 +702,40 @@ func rewriteTimeoutMsg(msg *Message) (err error) { return } +func mvtFilterHTTPArgs(msg *Message, query string) (modified bool) { + path := msg.Args[0] + parts := strings.Split(path, "/") + if len(parts) != 4 { + return false + } + parts[3] = parts[3][:len(parts[3])-4] + for i := 0; i < len(parts); i++ { + var err error + parts[i], err = url.PathUnescape(parts[i]) + if err != nil { + return false + } + } + msg._command = "" + msg.Args = []string{"intersects", parts[0], "LIMIT", "1000000000", + "MVT", parts[2], parts[3], parts[1]} + log.HTTPf("%s", path) + return true +} + func (s *Server) handleInputCommand(client *Client, msg *Message) error { start := time.Now() + var mvt bool + if msg.ConnType == HTTP && len(msg.Args) == 1 { + var query string + if i := strings.IndexByte(msg.Args[0], '?'); i != -1 { + query = msg.Args[0][i+1:] + msg.Args[0] = msg.Args[0][:i] + } + if strings.HasSuffix(msg.Args[0], ".mvt") { + mvt = mvtFilterHTTPArgs(msg, query) + } + } serializeOutput := func(res resp.Value) (string, error) { var resStr string var err error @@ -726,16 +758,37 @@ func (s *Server) handleInputCommand(client *Client, msg *Message) error { case WebSocket: return WriteWebSocketMessage(client, []byte(res)) case HTTP: + origin := "" + extraNL := 2 + contentType := "application/json; charset=utf-8" status := "200 OK" if (s.http500Errors || msg._command == "healthz") && !gjson.Get(res, "ok").Bool() { status = "500 Internal Server Error" + } else if mvt { + v := gjson.Get(res, "mvt") + if !v.Exists() { + status = "500 Internal Server Error" + } else { + res = v.String() + out, err := base64.RawStdEncoding.DecodeString(res) + if err != nil { + status = "500 Internal Server Error" + } else { + res = string(out) + origin = "Access-Control-Allow-Origin: *\r\n" + contentType = "application/vnd.mapbox-vector-tile" + } + } } - _, err := fmt.Fprintf(client, "HTTP/1.1 %s\r\n"+ + + _, err := fmt.Fprintf(client, ""+ + "HTTP/1.1 %s\r\n"+ "Connection: close\r\n"+ "Content-Length: %d\r\n"+ - "Content-Type: application/json; charset=utf-8\r\n"+ - "\r\n", status, len(res)+2) + "Content-Type: %s\r\n"+ + "%s"+ + "\r\n", status, len(res)+extraNL, contentType, origin) if err != nil { return err } @@ -743,8 +796,13 @@ func (s *Server) handleInputCommand(client *Client, msg *Message) error { if err != nil { return err } - _, err = io.WriteString(client, "\r\n") - return err + if extraNL == 2 { + _, err = io.WriteString(client, "\r\n") + if err != nil { + return err + } + } + return nil case RESP: var err error if msg.OutputType == JSON { diff --git a/internal/server/token.go b/internal/server/token.go index ca8330f9..094e2f9a 100644 --- a/internal/server/token.go +++ b/internal/server/token.go @@ -214,6 +214,9 @@ type searchScanBaseTokens struct { buffer float64 hasbuffer bool mvt bool + tileX int + tileY int + tileZ int } func (s *Server) parseSearchScanBaseTokens(