mirror
/
tile38
mirror of https://github.com/tidwall/tile38.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Vector tiles over http 

Basic debugger support serving standard mapbox vector.
  INTERSECTS ... MVT x y z
or
  http://localhost:9851/{col}/{z}/{x}/{y}.mvt
This commit is contained in:
tidwall 2022-09-13 17:39:13 -07:00
parent 059dcd5e9a
commit c8ea2fc741
8 changed files with 184 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/tile38-server/main.go
View File

@ -90,6 +90,7 @@ Advanced Options:
--http-transport yes/no : HTTP transport (default: yes)
--protected-mode yes/no : protected mode (default: yes)
--nohup : do not exit on SIGHUP
--metrics-addr addr : The listening addr for Prometheus metrics.
Developer Options:
--dev : enable developer mode

2
internal/server/hooks.go
View File

@ -143,7 +143,7 @@ func (s *Server) cmdSetHook(msg *Message) (
hook.ScanWriter, err = s.newScanWriter(
&wr, cmsg, args.key, args.output, args.precision, args.globs, false,
args.cursor, args.limit, args.wheres, args.whereins, args.whereevals,
args.nofields, args.mvt)
args.nofields, args.mvt, args.tileX, args.tileY, args.tileZ)
if err != nil {
return NOMessage, d, err

2
internal/server/live.go
View File

@ -116,7 +116,7 @@ func (s *Server) goLive(
sw, err = s.newScanWriter(
&wr, msg, lfs.key, lfs.output, lfs.precision, lfs.globs, false,
lfs.cursor, lfs.limit, lfs.wheres, lfs.whereins, lfs.whereevals,
lfs.nofields, lfs.mvt)
lfs.nofields, lfs.mvt, lfs.tileX, lfs.tileY, lfs.tileZ)
s.mu.RUnlock()
// everything below if for live SCAN, NEARBY, WITHIN, INTERSECTS

2
internal/server/scan.go
View File

@ -47,7 +47,7 @@ func (s *Server) cmdScan(msg *Message) (res resp.Value, err error) {
sw, err := s.newScanWriter(
wr, msg, args.key, args.output, args.precision, args.globs, false,
args.cursor, args.limit, args.wheres, args.whereins, args.whereevals,
args.nofields, args.mvt)
args.nofields, args.mvt, args.tileX, args.tileY, args.tileZ)
if err != nil {
return NOMessage, err
}

112
internal/server/scanner.go
View File

@ -10,6 +10,7 @@ import (
"github.com/mmcloughlin/geohash"
"github.com/tidwall/geojson"
"github.com/tidwall/geojson/geometry"
"github.com/tidwall/mvt"
"github.com/tidwall/resp"
"github.com/tidwall/tile38/internal/clip"
@ -58,12 +59,15 @@ type scanWriter struct {
globEverything bool
fullFields bool
values []resp.Value
mvtObjs []geojson.Object
mvtObjs []objPair
matchValues bool
mvt bool
respOut resp.Value
orgWheres []whereT
orgWhereins []whereinT
mvt bool
tileX int
tileY int
tileZ int
}
// ScanWriterParams ...
@ -84,7 +88,7 @@ func (s *Server) newScanWriter(
wr *bytes.Buffer, msg *Message, key string, output outputT,
precision uint64, globs []string, matchValues bool,
cursor, limit uint64, wheres []whereT, whereins []whereinT,
whereevals []whereevalT, nofields, mvt bool,
whereevals []whereevalT, nofields, mvt bool, tileX, tileY, tileZ int,
) (
*scanWriter, error,
) {
@ -105,7 +109,6 @@ func (s *Server) newScanWriter(
wr: wr,
key: key,
msg: msg,
mvt: mvt,
globs: globs,
limit: limit,
cursor: cursor,
@ -116,6 +119,11 @@ func (s *Server) newScanWriter(
matchValues: matchValues,
}
sw.mvt = mvt
sw.tileX = tileX
sw.tileY = tileY
sw.tileZ = tileZ
if len(globs) == 0 || (len(globs) == 1 && globs[0] == "*") {
sw.globEverything = true
}
@ -209,23 +217,87 @@ func (sw *scanWriter) writeHead() {
}
}
func (sw *scanWriter) compileMVT() []byte {
var tile mvt.Tile
l := tile.AddLayer("default")
l.SetExtent(4096)
for _, g := range sw.mvtObjs {
_ = g
f := l.AddFeature(mvt.Polygon)
// f.MoveTo(128, 96)
// f.LineTo(148, 128)
// f.LineTo(108, 128)
// f.LineTo(128, 96)
f.ClosePath()
func mvtDrawRing(f *mvt.Feature, tileX, tileY, tileZ int, ring geometry.Series, hole bool) {
npoints := ring.NumPoints()
if npoints < 3 {
return
}
cw := ring.Clockwise()
reverse := (cw && hole) || (!cw && !hole)
if reverse {
p := ring.PointAt(npoints - 1)
f.MoveTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ))
for i := npoints - 2; i >= 0; i-- {
p := ring.PointAt(i)
f.LineTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ))
}
} else {
p := ring.PointAt(0)
f.MoveTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ))
for i := 1; i < npoints; i++ {
p := ring.PointAt(i)
f.LineTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ))
}
}
f.ClosePath()
}
// println(sw.mvtObjs)
func mvtAddFeature(l *mvt.Layer, tileX, tileY, tileZ int, p objPair) {
var f *mvt.Feature
switch g := p.obj.(type) {
case *geojson.Point:
f = l.AddFeature(mvt.Point)
p := g.Base()
f.MoveTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ))
case *geojson.SimplePoint:
f = l.AddFeature(mvt.Point)
p := g
f.MoveTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ))
case *geojson.LineString:
f = l.AddFeature(mvt.LineString)
line := g.Base()
npoints := line.NumPoints()
if npoints > 0 {
p := line.PointAt(0)
f.MoveTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ))
for i := 1; i < npoints; i++ {
p := line.PointAt(0)
f.LineTo(mvt.LatLonXY(p.Y, p.X, tileX, tileY, tileZ))
}
}
case *geojson.Polygon:
f = l.AddFeature(mvt.Polygon)
poly := g.Base()
mvtDrawRing(f, tileX, tileY, tileZ, poly.Exterior, false)
for _, hole := range poly.Holes {
mvtDrawRing(f, tileX, tileY, tileZ, hole, true)
}
case *geojson.Feature:
mvtAddFeature(l, tileX, tileY, tileZ, objPair{p.id, g.Base()})
return
default:
if g, ok := g.(geojson.Collection); ok {
for _, g := range g.Children() {
mvtAddFeature(l, tileX, tileY, tileZ, objPair{p.id, g})
}
}
return
}
f.AddTag("id", p.id)
}
type objPair struct {
id string
obj geojson.Object
}
func objsToMVT(tileX, tileY, tileZ int, pairs []objPair) []byte {
var tile mvt.Tile
l := tile.AddLayer("tile38")
l.SetExtent(4096)
for _, p := range pairs {
mvtAddFeature(l, tileX, tileY, tileZ, p)
}
return tile.Render()
}
@ -239,7 +311,7 @@ func (sw *scanWriter) writeFoot() {
var mvtTile []byte
if sw.mvt {
mvtTile = sw.compileMVT()
mvtTile = objsToMVT(sw.tileX, sw.tileY, sw.tileZ, sw.mvtObjs)
}
switch sw.msg.OutputType {
case JSON:
@ -443,7 +515,7 @@ func (sw *scanWriter) writeObject(opts ScanWriterParams) bool {
opts.o = clip.Clip(opts.o, opts.clip, &sw.s.geomIndexOpts)
}
if sw.mvt {
sw.mvtObjs = append(sw.mvtObjs, opts.o)
sw.mvtObjs = append(sw.mvtObjs, objPair{opts.id, opts.o})
} else {
switch sw.msg.OutputType {
case JSON:

34
internal/server/search.go
View File

@ -57,7 +57,9 @@ func (lfs liveFenceSwitches) usingLua() bool {
return len(lfs.whereevals) > 0
}
func parseRectArea(ltyp string, vs []string) (nvs []string, rect *geojson.Rect, err error) {
func parseRectArea(ltyp string, vs []string) (
nvs []string, rect *geojson.Rect, tileX, tileY, tileZ int, err error,
) {
var ok bool
@ -145,26 +147,29 @@ func parseRectArea(ltyp string, vs []string) (nvs []string, rect *geojson.Rect,
err = errInvalidNumberOfArguments
return
}
var x, y int64
var z uint64
if x, err = strconv.ParseInt(sx, 10, 64); err != nil {
var x, y, z int
if x, err = strconv.Atoi(sx); err != nil || x < 0 {
err = errInvalidArgument(sx)
return
}
if y, err = strconv.ParseInt(sy, 10, 64); err != nil {
if y, err = strconv.Atoi(sy); err != nil || y < 0 {
err = errInvalidArgument(sy)
return
}
if z, err = strconv.ParseUint(sz, 10, 64); err != nil {
if z, err = strconv.Atoi(sz); err != nil || z < 0 || z > 23 {
err = errInvalidArgument(sz)
return
}
var minLat, minLon, maxLat, maxLon float64
minLat, minLon, maxLat, maxLon = bing.TileXYToBounds(x, y, z)
minLat, minLon, maxLat, maxLon =
bing.TileXYToBounds(int64(x), int64(y), uint64(z))
rect = geojson.NewRect(geometry.Rect{
Min: geometry.Point{X: minLon, Y: minLat},
Max: geometry.Point{X: maxLon, Y: maxLat},
})
tileX = x
tileY = y
tileZ = z
}
nvs = vs
return
@ -355,7 +360,8 @@ func (s *Server) cmdSearchArgs(
return
}
case "bounds", "hash", "tile", "mvt", "quadkey":
vs, lfs.obj, err = parseRectArea(ltyp, vs)
vs, lfs.obj, lfs.tileX, lfs.tileY, lfs.tileZ, err =
parseRectArea(ltyp, vs)
if err != nil {
return
}
@ -435,7 +441,8 @@ func (s *Server) cmdSearchArgs(
ltok = strings.ToLower(tok)
switch ltok {
case "bounds", "hash", "tile", "quadkey":
vs, clip_rect, err = parseRectArea(ltok, vs)
vs, clip_rect, lfs.tileX, lfs.tileY, lfs.tileZ, err =
parseRectArea(ltok, vs)
if err == errNotRectangle {
err = errInvalidArgument("cannot clipby " + ltok)
return
@ -494,7 +501,8 @@ func (s *Server) cmdNearby(msg *Message) (res resp.Value, err error) {
sw, err := s.newScanWriter(
wr, msg, sargs.key, sargs.output, sargs.precision, sargs.globs, false,
sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins,
sargs.whereevals, sargs.nofields, sargs.mvt)
sargs.whereevals, sargs.nofields,
sargs.mvt, sargs.tileX, sargs.tileY, sargs.tileZ)
if err != nil {
return NOMessage, err
}
@ -587,7 +595,8 @@ func (s *Server) cmdWithinOrIntersects(cmd string, msg *Message) (res resp.Value
sw, err := s.newScanWriter(
wr, msg, sargs.key, sargs.output, sargs.precision, sargs.globs, false,
sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins,
sargs.whereevals, sargs.nofields, sargs.mvt)
sargs.whereevals, sargs.nofields,
sargs.mvt, sargs.tileX, sargs.tileY, sargs.tileZ)
if err != nil {
return NOMessage, err
}
@ -701,7 +710,8 @@ func (s *Server) cmdSearch(msg *Message) (res resp.Value, err error) {
sw, err := s.newScanWriter(
wr, msg, sargs.key, sargs.output, sargs.precision, sargs.globs, true,
sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins,
sargs.whereevals, sargs.nofields, sargs.mvt)
sargs.whereevals, sargs.nofields,
sargs.mvt, sargs.tileX, sargs.tileY, sargs.tileZ)
if err != nil {
return NOMessage, err
}

64
internal/server/server.go
View File

@ -702,8 +702,40 @@ func rewriteTimeoutMsg(msg *Message) (err error) {
return
}
func mvtFilterHTTPArgs(msg *Message, query string) (modified bool) {
path := msg.Args[0]
parts := strings.Split(path, "/")
if len(parts) != 4 {
return false
}
parts[3] = parts[3][:len(parts[3])-4]
for i := 0; i < len(parts); i++ {
var err error
parts[i], err = url.PathUnescape(parts[i])
if err != nil {
return false
}
}
msg._command = ""
msg.Args = []string{"intersects", parts[0], "LIMIT", "1000000000",
"MVT", parts[2], parts[3], parts[1]}
log.HTTPf("%s", path)
return true
}
func (s *Server) handleInputCommand(client *Client, msg *Message) error {
start := time.Now()
var mvt bool
if msg.ConnType == HTTP && len(msg.Args) == 1 {
var query string
if i := strings.IndexByte(msg.Args[0], '?'); i != -1 {
query = msg.Args[0][i+1:]
msg.Args[0] = msg.Args[0][:i]
}
if strings.HasSuffix(msg.Args[0], ".mvt") {
mvt = mvtFilterHTTPArgs(msg, query)
}
}
serializeOutput := func(res resp.Value) (string, error) {
var resStr string
var err error
@ -726,16 +758,37 @@ func (s *Server) handleInputCommand(client *Client, msg *Message) error {
case WebSocket:
return WriteWebSocketMessage(client, []byte(res))
case HTTP:
origin := ""
extraNL := 2
contentType := "application/json; charset=utf-8"
status := "200 OK"
if (s.http500Errors || msg._command == "healthz") &&
!gjson.Get(res, "ok").Bool() {
status = "500 Internal Server Error"
} else if mvt {
v := gjson.Get(res, "mvt")
if !v.Exists() {
status = "500 Internal Server Error"
} else {
res = v.String()
out, err := base64.RawStdEncoding.DecodeString(res)
if err != nil {
status = "500 Internal Server Error"
} else {
res = string(out)
origin = "Access-Control-Allow-Origin: *\r\n"
contentType = "application/vnd.mapbox-vector-tile"
}
_, err := fmt.Fprintf(client, "HTTP/1.1 %s\r\n"+
}
}
_, err := fmt.Fprintf(client, ""+
"HTTP/1.1 %s\r\n"+
"Connection: close\r\n"+
"Content-Length: %d\r\n"+
"Content-Type: application/json; charset=utf-8\r\n"+
"\r\n", status, len(res)+2)
"Content-Type: %s\r\n"+
"%s"+
"\r\n", status, len(res)+extraNL, contentType, origin)
if err != nil {
return err
}
@ -743,8 +796,13 @@ func (s *Server) handleInputCommand(client *Client, msg *Message) error {
if err != nil {
return err
}
if extraNL == 2 {
_, err = io.WriteString(client, "\r\n")
if err != nil {
return err
}
}
return nil
case RESP:
var err error
if msg.OutputType == JSON {

3
internal/server/token.go
View File

@ -214,6 +214,9 @@ type searchScanBaseTokens struct {
buffer float64
hasbuffer bool
mvt bool
tileX int
tileY int
tileZ int
}
func (s *Server) parseSearchScanBaseTokens(