tile38/internal/collection/collection.go

package collection

import (
	ifbtree "github.com/tidwall/btree"
	"github.com/tidwall/geojson"
	"github.com/tidwall/geojson/geo"
	"github.com/tidwall/geojson/geometry"
	"github.com/tidwall/tile38/internal/collection/btree"
	"github.com/tidwall/tile38/internal/collection/item"
	"github.com/tidwall/tile38/internal/collection/rtree"
)

// Cursor allows for quickly paging through Scan, Within, Intersects, and Nearby
type Cursor interface {
	Offset() uint64
	Step(count uint64)
}

// Collection represents a collection of geojson objects.
type Collection struct {
	items    btree.BTree    // items sorted by keys
	index    rtree.BoxTree  // items geospatially indexed
	values   *ifbtree.BTree // items sorted by value+key
	packed   bool
	fieldMap map[string]int
	weight   int
	points   int
	objects  int // geometry count
	nobjects int // non-geometry count
}

var counter uint64

// New creates an empty collection
func New(packed bool) *Collection {
	col := &Collection{
		packed:   packed,
		values:   ifbtree.New(16, nil),
		fieldMap: make(map[string]int),
	}
	return col
}

// Count returns the number of objects in collection.
func (c *Collection) Count() int {
	return c.objects + c.nobjects
}

// StringCount returns the number of string values.
func (c *Collection) StringCount() int {
	return c.nobjects
}

// PointCount returns the number of points (lat/lon coordinates) in collection.
func (c *Collection) PointCount() int {
	return c.points
}

// TotalWeight calculates the in-memory cost of the collection in bytes.
func (c *Collection) TotalWeight() int {
	return c.weight
}

// Bounds returns the bounds of all the items in the collection.
func (c *Collection) Bounds() (minX, minY, maxX, maxY float64) {
	min, max := c.index.Bounds()
	if len(min) >= 2 && len(max) >= 2 {
		return min[0], min[1], max[0], max[1]
	}
	return
}

func objIsSpatial(obj geojson.Object) bool {
	_, ok := obj.(geojson.Spatial)
	return ok
}

func (c *Collection) addItem(item *item.Item) {
	if objIsSpatial(item.Obj()) {
		if !item.Obj().Empty() {
			rect := item.Obj().Rect()
			c.index.Insert(
				[]float64{rect.Min.X, rect.Min.Y},
				[]float64{rect.Max.X, rect.Max.Y},
				item)
		}
		c.objects++
	} else {
		c.values.ReplaceOrInsert(item)
		c.nobjects++
	}
	weight, points := item.WeightAndPoints()
	c.weight += weight
	c.points += points
}

func (c *Collection) delItem(item *item.Item) {
	if objIsSpatial(item.Obj()) {
		if !item.Obj().Empty() {
			rect := item.Obj().Rect()
			c.index.Delete(
				[]float64{rect.Min.X, rect.Min.Y},
				[]float64{rect.Max.X, rect.Max.Y},
				item)
		}
		c.objects--
	} else {
		c.values.Delete(item)
		c.nobjects--
	}
	weight, points := item.WeightAndPoints()
	c.weight -= weight
	c.points -= points
}

// Set adds or replaces an object in the collection and returns the fields
// array. If an item with the same id is already in the collection then the
// new item will adopt the old item's fields.
// The fields argument is optional.
// The return values are the old object, the old fields, and the new fields
func (c *Collection) Set(
	id string, obj geojson.Object, fields []string, values []float64,
) (
	oldObj geojson.Object, oldFields *Fields, newFields *Fields,
) {
	// create the new item
	newItem := item.New(id, obj, c.packed)

	// add the new item to main btree and remove the old one if needed
	var oldItem *item.Item
	oldItemV, ok := c.items.Set(newItem)
	if ok {
		oldItem = oldItemV
		oldObj = oldItem.Obj()

		// remove old item from indexes
		c.delItem(oldItem)
		if oldItem.HasFields() {
			// merge old and new fields
			newItem.CopyOverFields(oldItem)
		}
	}

	if fields == nil && len(values) > 0 {
		// directly set the field values, from copy
		newItem.CopyOverFields(values)
	} else if len(fields) > 0 {
		// add new field to new item
		c.setFields(newItem, fields, values, false)
	}

	// add new item to indexes
	c.addItem(newItem)
	// fmt.Printf("!!! %#v\n", oldObj)

	return oldObj, itemFields(oldItem), itemFields(newItem)
}

func (c *Collection) setFields(
	item *item.Item, fieldNames []string, fieldValues []float64, updateWeight bool,
) (updatedCount int) {
	for i, fieldName := range fieldNames {
		var fieldValue float64
		if i < len(fieldValues) {
			fieldValue = fieldValues[i]
		}
		if c.setField(item, fieldName, fieldValue, updateWeight) {
			updatedCount++
		}
	}
	return updatedCount
}

func (c *Collection) setField(
	item *item.Item, fieldName string, fieldValue float64, updateWeight bool,
) (updated bool) {
	idx, ok := c.fieldMap[fieldName]
	if !ok {
		idx = len(c.fieldMap)
		c.fieldMap[fieldName] = idx
	}
	var pweight int
	if updateWeight {
		pweight, _ = item.WeightAndPoints()
	}
	updated = item.SetField(idx, fieldValue)
	if updateWeight && updated {
		nweight, _ := item.WeightAndPoints()
		c.weight = c.weight - pweight + nweight
	}
	return updated
}

// Delete removes an object and returns it.
// If the object does not exist then the 'ok' return value will be false.
func (c *Collection) Delete(id string) (
	obj geojson.Object, fields *Fields, ok bool,
) {
	oldItemV, ok := c.items.Delete(id)
	if !ok {
		return nil, nil, false
	}
	oldItem := oldItemV

	c.delItem(oldItem)

	return oldItem.Obj(), itemFields(oldItem), true
}

// Get returns an object.
// If the object does not exist then the 'ok' return value will be false.
func (c *Collection) Get(id string) (
	obj geojson.Object, fields *Fields, ok bool,
) {
	itemV, ok := c.items.Get(id)
	if !ok {
		return nil, nil, false
	}
	item := itemV

	return item.Obj(), itemFields(item), true
}

// SetField set a field value for an object and returns that object.
// If the object does not exist then the 'ok' return value will be false.
func (c *Collection) SetField(id, fieldName string, fieldValue float64) (
	obj geojson.Object, fields *Fields, updated bool, ok bool,
) {
	itemV, ok := c.items.Get(id)
	if !ok {
		return nil, nil, false, false
	}
	item := itemV
	updated = c.setField(item, fieldName, fieldValue, true)
	return item.Obj(), itemFields(item), updated, true
}

// SetFields is similar to SetField, just setting multiple fields at once
func (c *Collection) SetFields(
	id string, fieldNames []string, fieldValues []float64,
) (obj geojson.Object, fields *Fields, updatedCount int, ok bool) {
	itemV, ok := c.items.Get(id)
	if !ok {
		return nil, nil, 0, false
	}
	item := itemV

	updatedCount = c.setFields(item, fieldNames, fieldValues, true)

	return item.Obj(), itemFields(item), updatedCount, true
}

// FieldMap return a maps of the field names.
func (c *Collection) FieldMap() map[string]int {
	return c.fieldMap
}

// FieldArr return an array representation of the field names.
func (c *Collection) FieldArr() []string {
	arr := make([]string, len(c.fieldMap))
	for field, i := range c.fieldMap {
		arr[i] = field
	}
	return arr
}

// Scan iterates though the collection ids.
func (c *Collection) Scan(desc bool, cursor Cursor,
	iterator func(id string, obj geojson.Object, fields *Fields) bool,
) bool {
	var keepon = true
	var count uint64
	var offset uint64
	if cursor != nil {
		offset = cursor.Offset()
		cursor.Step(offset)
	}
	iter := func(item *item.Item) bool {
		count++
		if count <= offset {
			return true
		}
		if cursor != nil {
			cursor.Step(1)
		}
		keepon = iterator(item.ID(), item.Obj(), itemFields(item))
		return keepon
	}
	if desc {
		c.items.Reverse(iter)
	} else {
		c.items.Scan(iter)
	}
	return keepon
}

// ScanRange iterates though the collection starting with specified id.
func (c *Collection) ScanRange(start, end string, desc bool, cursor Cursor,
	iterator func(id string, obj geojson.Object, fields *Fields) bool,
) bool {
	var keepon = true
	var count uint64
	var offset uint64
	if cursor != nil {
		offset = cursor.Offset()
		cursor.Step(offset)
	}
	iter := func(item *item.Item) bool {
		count++
		if count <= offset {
			return true
		}
		if cursor != nil {
			cursor.Step(1)
		}
		if !desc {
			if item.ID() >= end {
				return false
			}
		} else {
			if item.ID() <= end {
				return false
			}
		}
		keepon = iterator(item.ID(), item.Obj(), itemFields(item))
		return keepon
	}

	if desc {
		c.items.Descend(start, iter)
	} else {
		c.items.Ascend(start, iter)
	}
	return keepon
}

// SearchValues iterates though the collection values.
func (c *Collection) SearchValues(desc bool, cursor Cursor,
	iterator func(id string, obj geojson.Object, fields *Fields) bool,
) bool {
	var keepon = true
	var count uint64
	var offset uint64
	if cursor != nil {
		offset = cursor.Offset()
		cursor.Step(offset)
	}
	iter := func(v ifbtree.Item) bool {
		count++
		if count <= offset {
			return true
		}
		if cursor != nil {
			cursor.Step(1)
		}
		iitm := v.(*item.Item)
		keepon = iterator(iitm.ID(), iitm.Obj(), itemFields(iitm))
		return keepon
	}
	if desc {
		c.values.Descend(iter)
	} else {
		c.values.Ascend(iter)
	}
	return keepon
}

// SearchValuesRange iterates though the collection values.
func (c *Collection) SearchValuesRange(start, end string, desc bool,
	cursor Cursor,
	iterator func(id string, obj geojson.Object, fields *Fields) bool,
) bool {
	var keepon = true
	var count uint64
	var offset uint64
	if cursor != nil {
		offset = cursor.Offset()
		cursor.Step(offset)
	}
	iter := func(v ifbtree.Item) bool {
		count++
		if count <= offset {
			return true
		}
		if cursor != nil {
			cursor.Step(1)
		}
		iitm := v.(*item.Item)
		keepon = iterator(iitm.ID(), iitm.Obj(), itemFields(iitm))
		return keepon
	}
	if desc {
		c.values.DescendRange(
			item.New("", String(start), false),
			item.New("", String(end), false),
			iter,
		)
	} else {
		c.values.AscendRange(
			item.New("", String(start), false),
			item.New("", String(end), false),
			iter,
		)
	}
	return keepon
}

// ScanGreaterOrEqual iterates though the collection starting with specified id.
func (c *Collection) ScanGreaterOrEqual(id string, desc bool,
	cursor Cursor,
	iterator func(id string, obj geojson.Object, fields *Fields) bool,
) bool {
	var keepon = true
	var count uint64
	var offset uint64
	if cursor != nil {
		offset = cursor.Offset()
		cursor.Step(offset)
	}
	iter := func(item *item.Item) bool {
		count++
		if count <= offset {
			return true
		}
		if cursor != nil {
			cursor.Step(1)
		}
		keepon = iterator(item.ID(), item.Obj(), itemFields(item))
		return keepon
	}
	if desc {
		c.items.Descend(id, iter)
	} else {
		c.items.Ascend(id, iter)
	}
	return keepon
}

func (c *Collection) geoSearch(
	rect geometry.Rect,
	iter func(id string, obj geojson.Object, fields *Fields) bool,
) bool {
	alive := true
	c.index.Search(
		[]float64{rect.Min.X, rect.Min.Y},
		[]float64{rect.Max.X, rect.Max.Y},
		func(_, _ []float64, itemv *item.Item) bool {
			item := itemv
			alive = iter(item.ID(), item.Obj(), itemFields(item))
			return alive
		},
	)
	return alive
}

func (c *Collection) geoSparse(
	obj geojson.Object, sparse uint8,
	iter func(id string, obj geojson.Object, fields *Fields) (match, ok bool),
) bool {
	matches := make(map[string]bool)
	alive := true
	c.geoSparseInner(obj.Rect(), sparse,
		func(id string, o geojson.Object, fields *Fields) (
			match, ok bool,
		) {
			ok = true
			if !matches[id] {
				match, ok = iter(id, o, fields)
				if match {
					matches[id] = true
				}
			}
			return match, ok
		},
	)
	return alive
}
func (c *Collection) geoSparseInner(
	rect geometry.Rect, sparse uint8,
	iter func(id string, obj geojson.Object, fields *Fields) (match, ok bool),
) bool {
	if sparse > 0 {
		w := rect.Max.X - rect.Min.X
		h := rect.Max.Y - rect.Min.Y
		quads := [4]geometry.Rect{
			geometry.Rect{
				Min: geometry.Point{X: rect.Min.X, Y: rect.Min.Y + h/2},
				Max: geometry.Point{X: rect.Min.X + w/2, Y: rect.Max.Y},
			},
			geometry.Rect{
				Min: geometry.Point{X: rect.Min.X + w/2, Y: rect.Min.Y + h/2},
				Max: geometry.Point{X: rect.Max.X, Y: rect.Max.Y},
			},
			geometry.Rect{
				Min: geometry.Point{X: rect.Min.X, Y: rect.Min.Y},
				Max: geometry.Point{X: rect.Min.X + w/2, Y: rect.Min.Y + h/2},
			},
			geometry.Rect{
				Min: geometry.Point{X: rect.Min.X + w/2, Y: rect.Min.Y},
				Max: geometry.Point{X: rect.Max.X, Y: rect.Min.Y + h/2},
			},
		}
		for _, quad := range quads {
			if !c.geoSparseInner(quad, sparse-1, iter) {
				return false
			}
		}
		return true
	}
	alive := true
	c.geoSearch(rect,
		func(id string, obj geojson.Object, fields *Fields) bool {
			match, ok := iter(id, obj, fields)
			if !ok {
				alive = false
				return false
			}
			return !match
		},
	)
	return alive
}

// Within returns all object that are fully contained within an object or
// bounding box. Set obj to nil in order to use the bounding box.
func (c *Collection) Within(
	obj geojson.Object,
	sparse uint8,
	cursor Cursor,
	iter func(id string, obj geojson.Object, fields *Fields) bool,
) bool {
	var count uint64
	var offset uint64
	if cursor != nil {
		offset = cursor.Offset()
		cursor.Step(offset)
	}
	if sparse > 0 {
		return c.geoSparse(obj, sparse,
			func(id string, o geojson.Object, fields *Fields) (
				match, ok bool,
			) {
				count++
				if count <= offset {
					return false, true
				}
				if cursor != nil {
					cursor.Step(1)
				}
				if match = o.Within(obj); match {
					ok = iter(id, o, fields)
				}
				return match, ok
			},
		)
	}
	return c.geoSearch(obj.Rect(),
		func(id string, o geojson.Object, fields *Fields) bool {
			count++
			if count <= offset {
				return true
			}
			if cursor != nil {
				cursor.Step(1)
			}
			if o.Within(obj) {
				return iter(id, o, fields)
			}
			return true
		},
	)
}

// Intersects returns all object that are intersect an object or bounding box.
// Set obj to nil in order to use the bounding box.
func (c *Collection) Intersects(
	obj geojson.Object,
	sparse uint8,
	cursor Cursor,
	iter func(id string, obj geojson.Object, fields *Fields) bool,
) bool {
	var count uint64
	var offset uint64
	if cursor != nil {
		offset = cursor.Offset()
		cursor.Step(offset)
	}
	if sparse > 0 {
		return c.geoSparse(obj, sparse,
			func(id string, o geojson.Object, fields *Fields) (
				match, ok bool,
			) {
				count++
				if count <= offset {
					return false, true
				}
				if cursor != nil {
					cursor.Step(1)
				}
				if match = o.Intersects(obj); match {
					ok = iter(id, o, fields)
				}
				return match, ok
			},
		)
	}
	return c.geoSearch(obj.Rect(),
		func(id string, o geojson.Object, fields *Fields) bool {
			count++
			if count <= offset {
				return true
			}
			if cursor != nil {
				cursor.Step(1)
			}
			if o.Intersects(obj) {
				return iter(id, o, fields)
			}
			return true
		},
	)
}

// Nearby returns the nearest neighbors
func (c *Collection) Nearby(
	target geojson.Object,
	cursor Cursor,
	iter func(id string, obj geojson.Object, fields *Fields) bool,
) bool {
	// First look to see if there's at least one candidate in the circle's
	// outer rectangle. This is a fast-fail operation.
	if circle, ok := target.(*geojson.Circle); ok {
		meters := circle.Meters()
		if meters > 0 {
			center := circle.Center()
			minLat, minLon, maxLat, maxLon :=
				geo.RectFromCenter(center.Y, center.X, meters)
			var exists bool
			c.index.Search(
				[]float64{minLon, minLat},
				[]float64{maxLon, maxLat},
				func(_, _ []float64, itemv *item.Item) bool {
					exists = true
					return false
				},
			)
			if !exists {
				// no candidates
				return true
			}
		}
	}
	// do the kNN operation
	alive := true
	center := target.Center()
	var count uint64
	var offset uint64
	if cursor != nil {
		offset = cursor.Offset()
		cursor.Step(offset)
	}
	c.index.Nearby(
		[]float64{center.X, center.Y},
		[]float64{center.X, center.Y},
		func(_, _ []float64, itemv *item.Item) bool {
			count++
			if count <= offset {
				return true
			}
			if cursor != nil {
				cursor.Step(1)
			}
			item := itemv
			alive = iter(item.ID(), item.Obj(), itemFields(item))
			return alive
		},
	)
	return alive
}