tile38/controller/hooks.go

580 lines
13 KiB
Go

package controller
import (
"bytes"
"encoding/json"
"errors"
"sort"
"strings"
"sync"
"time"
"github.com/tidwall/buntdb"
"github.com/tidwall/resp"
"github.com/tidwall/tile38/controller/endpoint"
"github.com/tidwall/tile38/controller/glob"
"github.com/tidwall/tile38/controller/log"
"github.com/tidwall/tile38/controller/server"
)
const hookLogTTL = time.Second * 30
func hookLogSetDefaults() *buntdb.SetOptions {
if hookLogTTL > 0 {
return &buntdb.SetOptions{
Expires: true, // automatically delete after 30 seconds
TTL: hookLogTTL,
}
}
return nil
}
type hooksByName []*Hook
func (a hooksByName) Len() int {
return len(a)
}
func (a hooksByName) Less(i, j int) bool {
return a[i].Name < a[j].Name
}
func (a hooksByName) Swap(i, j int) {
a[i], a[j] = a[j], a[i]
}
func (c *Controller) cmdSetHook(msg *server.Message) (res resp.Value, d commandDetailsT, err error) {
start := time.Now()
vs := msg.Values[1:]
var name, urls, cmd string
var ok bool
if vs, name, ok = tokenval(vs); !ok || name == "" {
return server.NOMessage, d, errInvalidNumberOfArguments
}
if vs, urls, ok = tokenval(vs); !ok || urls == "" {
return server.NOMessage, d, errInvalidNumberOfArguments
}
var endpoints []string
for _, url := range strings.Split(urls, ",") {
url = strings.TrimSpace(url)
err := c.epc.Validate(url)
if err != nil {
log.Errorf("sethook: %v", err)
return resp.SimpleStringValue(""), d, errInvalidArgument(url)
}
endpoints = append(endpoints, url)
}
var commandvs []resp.Value
var cmdlc string
var types []string
metaMap := make(map[string]string)
for {
commandvs = vs
if vs, cmd, ok = tokenval(vs); !ok || cmd == "" {
return server.NOMessage, d, errInvalidNumberOfArguments
}
cmdlc = strings.ToLower(cmd)
switch cmdlc {
default:
return server.NOMessage, d, errInvalidArgument(cmd)
case "meta":
var metakey string
var metaval string
if vs, metakey, ok = tokenval(vs); !ok || metakey == "" {
return server.NOMessage, d, errInvalidNumberOfArguments
}
if vs, metaval, ok = tokenval(vs); !ok || metaval == "" {
return server.NOMessage, d, errInvalidNumberOfArguments
}
metaMap[metakey] = metaval
continue
case "nearby":
types = nearbyTypes
case "within", "intersects":
types = withinOrIntersectsTypes
}
break
}
s, err := c.cmdSearchArgs(cmdlc, vs, types)
defer s.Close()
if err != nil {
return server.NOMessage, d, err
}
if !s.fence {
return server.NOMessage, d, errors.New("missing FENCE argument")
}
s.cmd = cmdlc
cmsg := &server.Message{}
*cmsg = *msg
cmsg.Values = make([]resp.Value, len(commandvs))
for i := 0; i < len(commandvs); i++ {
cmsg.Values[i] = commandvs[i]
}
cmsg.Command = strings.ToLower(cmsg.Values[0].String())
metas := make([]FenceMeta, 0, len(metaMap))
for key, val := range metaMap {
metas = append(metas, FenceMeta{key, val})
}
sort.Sort(hookMetaByName(metas))
hook := &Hook{
Key: s.key,
Name: name,
Endpoints: endpoints,
Fence: &s,
Message: cmsg,
db: c.qdb,
epm: c.epc,
Metas: metas,
}
hook.cond = sync.NewCond(&hook.mu)
var wr bytes.Buffer
hook.ScanWriter, err = c.newScanWriter(
&wr, cmsg, s.key, s.output, s.precision, s.glob, false,
s.cursor, s.limit, s.wheres, s.whereins, s.whereevals, s.nofields)
if err != nil {
return server.NOMessage, d, err
}
if h, ok := c.hooks[name]; ok {
if h.Equals(hook) {
// it was a match so we do nothing. But let's signal just
// for good measure.
h.Signal()
switch msg.OutputType {
case server.JSON:
return server.OKMessage(msg, start), d, nil
case server.RESP:
return resp.IntegerValue(0), d, nil
}
}
h.Close()
// delete the previous hook
if hm, ok := c.hookcols[h.Key]; ok {
delete(hm, h.Name)
}
delete(c.hooks, h.Name)
}
d.updated = true
d.timestamp = time.Now()
c.hooks[name] = hook
hm, ok := c.hookcols[hook.Key]
if !ok {
hm = make(map[string]*Hook)
c.hookcols[hook.Key] = hm
}
hm[name] = hook
hook.Open()
switch msg.OutputType {
case server.JSON:
return server.OKMessage(msg, start), d, nil
case server.RESP:
return resp.IntegerValue(1), d, nil
}
return server.NOMessage, d, nil
}
func (c *Controller) cmdDelHook(msg *server.Message) (res resp.Value, d commandDetailsT, err error) {
start := time.Now()
vs := msg.Values[1:]
var name string
var ok bool
if vs, name, ok = tokenval(vs); !ok || name == "" {
return server.NOMessage, d, errInvalidNumberOfArguments
}
if len(vs) != 0 {
return server.NOMessage, d, errInvalidNumberOfArguments
}
if h, ok := c.hooks[name]; ok {
h.Close()
if hm, ok := c.hookcols[h.Key]; ok {
delete(hm, h.Name)
}
delete(c.hooks, h.Name)
d.updated = true
}
d.timestamp = time.Now()
switch msg.OutputType {
case server.JSON:
return server.OKMessage(msg, start), d, nil
case server.RESP:
if d.updated {
return resp.IntegerValue(1), d, nil
}
return resp.IntegerValue(0), d, nil
}
return
}
func (c *Controller) cmdPDelHook(msg *server.Message) (res resp.Value, d commandDetailsT, err error) {
start := time.Now()
vs := msg.Values[1:]
var pattern string
var ok bool
if vs, pattern, ok = tokenval(vs); !ok || pattern == "" {
return server.NOMessage, d, errInvalidNumberOfArguments
}
if len(vs) != 0 {
return server.NOMessage, d, errInvalidNumberOfArguments
}
count := 0
for name := range c.hooks {
match, _ := glob.Match(pattern, name)
if match {
if h, ok := c.hooks[name]; ok {
h.Close()
if hm, ok := c.hookcols[h.Key]; ok {
delete(hm, h.Name)
}
delete(c.hooks, h.Name)
d.updated = true
count++
}
}
}
d.timestamp = time.Now()
switch msg.OutputType {
case server.JSON:
return server.OKMessage(msg, start), d, nil
case server.RESP:
return resp.IntegerValue(count), d, nil
}
return
}
func (c *Controller) cmdHooks(msg *server.Message) (res resp.Value, err error) {
start := time.Now()
vs := msg.Values[1:]
var pattern string
var ok bool
if vs, pattern, ok = tokenval(vs); !ok || pattern == "" {
return server.NOMessage, errInvalidNumberOfArguments
}
if len(vs) != 0 {
return server.NOMessage, errInvalidNumberOfArguments
}
var hooks []*Hook
for name, hook := range c.hooks {
match, _ := glob.Match(pattern, name)
if match {
hooks = append(hooks, hook)
}
}
sort.Sort(hooksByName(hooks))
switch msg.OutputType {
case server.JSON:
buf := &bytes.Buffer{}
buf.WriteString(`{"ok":true,"hooks":[`)
for i, hook := range hooks {
if i > 0 {
buf.WriteByte(',')
}
buf.WriteString(`{`)
buf.WriteString(`"name":` + jsonString(hook.Name))
buf.WriteString(`,"key":` + jsonString(hook.Key))
buf.WriteString(`,"endpoints":[`)
for i, endpoint := range hook.Endpoints {
if i > 0 {
buf.WriteByte(',')
}
buf.WriteString(jsonString(endpoint))
}
buf.WriteString(`],"command":[`)
for i, v := range hook.Message.Values {
if i > 0 {
buf.WriteString(`,`)
}
buf.WriteString(jsonString(v.String()))
}
buf.WriteString(`],"meta":{`)
for i, meta := range hook.Metas {
if i > 0 {
buf.WriteString(`,`)
}
buf.WriteString(jsonString(meta.Name))
buf.WriteString(`:`)
buf.WriteString(jsonString(meta.Value))
}
buf.WriteString(`}}`)
}
buf.WriteString(`],"elapsed":"` + time.Now().Sub(start).String() + "\"}")
return resp.StringValue(buf.String()), nil
case server.RESP:
var vals []resp.Value
for _, hook := range hooks {
var hvals []resp.Value
hvals = append(hvals, resp.StringValue(hook.Name))
hvals = append(hvals, resp.StringValue(hook.Key))
var evals []resp.Value
for _, endpoint := range hook.Endpoints {
evals = append(evals, resp.StringValue(endpoint))
}
hvals = append(hvals, resp.ArrayValue(evals))
hvals = append(hvals, resp.ArrayValue(hook.Message.Values))
var metas []resp.Value
for _, meta := range hook.Metas {
metas = append(metas, resp.StringValue(meta.Name))
metas = append(metas, resp.StringValue(meta.Value))
}
hvals = append(hvals, resp.ArrayValue(metas))
vals = append(vals, resp.ArrayValue(hvals))
}
return resp.ArrayValue(vals), nil
}
return resp.SimpleStringValue(""), nil
}
// Hook represents a hook.
type Hook struct {
mu sync.Mutex
cond *sync.Cond
Key string
Name string
Endpoints []string
Message *server.Message
Fence *liveFenceSwitches
ScanWriter *scanWriter
Metas []FenceMeta
db *buntdb.DB
closed bool
opened bool
query string
epm *endpoint.EndpointManager
}
func (h *Hook) Equals(hook *Hook) bool {
if h.Key != hook.Key ||
h.Name != hook.Name ||
len(h.Endpoints) != len(hook.Endpoints) ||
len(h.Metas) != len(hook.Metas) {
return false
}
for i, endpoint := range h.Endpoints {
if endpoint != hook.Endpoints[i] {
return false
}
}
for i, meta := range h.Metas {
if meta.Name != hook.Metas[i].Name ||
meta.Value != hook.Metas[i].Value {
return false
}
}
return resp.ArrayValue(h.Message.Values).Equals(
resp.ArrayValue(hook.Message.Values))
}
type FenceMeta struct {
Name, Value string
}
type hookMetaByName []FenceMeta
func (arr hookMetaByName) Len() int {
return len(arr)
}
func (arr hookMetaByName) Less(a, b int) bool {
return arr[a].Name < arr[b].Name
}
func (arr hookMetaByName) Swap(a, b int) {
arr[a], arr[b] = arr[b], arr[a]
}
// Open is called when a hook is first created. It calls the manager
// function in a goroutine
func (h *Hook) Open() {
h.mu.Lock()
defer h.mu.Unlock()
if h.opened {
return
}
h.opened = true
b, _ := json.Marshal(h.Name)
h.query = `{"hook":` + string(b) + `}`
go h.manager()
}
// Close closed the hook and stop the manager function
func (h *Hook) Close() {
h.mu.Lock()
defer h.mu.Unlock()
if h.closed {
return
}
h.closed = true
h.cond.Broadcast()
}
// Signal can be called at any point to wake up the hook and
// notify the manager that there may be something new in the queue.
func (h *Hook) Signal() {
h.mu.Lock()
defer h.mu.Unlock()
h.cond.Broadcast()
}
// the manager is a forever loop that calls proc whenever there's a signal.
// it ends when the "closed" flag is set.
func (h *Hook) manager() {
for {
h.mu.Lock()
for {
if h.closed {
h.mu.Unlock()
return
}
if h.proc() {
break
}
h.mu.Unlock()
time.Sleep(time.Second / 4)
h.mu.Lock()
}
h.cond.Wait()
h.mu.Unlock()
}
}
// proc processes queued hook logs.
// returning true will indicate that all log entries have been
// successfully handled.
func (h *Hook) proc() (ok bool) {
var keys, vals []string
var ttls []time.Duration
err := h.db.Update(func(tx *buntdb.Tx) error {
// get keys and vals
err := tx.AscendGreaterOrEqual("hooks", h.query, func(key, val string) bool {
if strings.HasPrefix(key, hookLogPrefix) {
keys = append(keys, key)
vals = append(vals, val)
}
return true
})
if err != nil {
return err
}
// delete the keys
for _, key := range keys {
if hookLogTTL > 0 {
ttl, err := tx.TTL(key)
if err != nil {
if err != buntdb.ErrNotFound {
return err
}
}
ttls = append(ttls, ttl)
}
_, err = tx.Delete(key)
if err != nil {
if err != buntdb.ErrNotFound {
return err
}
}
}
return nil
})
if err != nil {
log.Error(err)
return false
}
// send each val. on failure reinsert that one and all of the following
for i, key := range keys {
val := vals[i]
idx := stringToUint64(key[len(hookLogPrefix):])
var sent bool
for _, endpoint := range h.Endpoints {
err := h.epm.Send(endpoint, val)
if err != nil {
log.Debugf("Endpoint connect/send error: %v: %v: %v", idx, endpoint, err)
continue
}
log.Debugf("Endpoint send ok: %v: %v: %v", idx, endpoint, err)
sent = true
break
}
if !sent {
// failed to send. try to reinsert the remaining. if this fails we lose log entries.
keys = keys[i:]
vals = vals[i:]
if hookLogTTL > 0 {
ttls = ttls[i:]
}
h.db.Update(func(tx *buntdb.Tx) error {
for i, key := range keys {
val := vals[i]
var opts *buntdb.SetOptions
if hookLogTTL > 0 {
opts = &buntdb.SetOptions{
Expires: true,
TTL: ttls[i],
}
}
_, _, err := tx.Set(key, val, opts)
if err != nil {
return err
}
}
return nil
})
return false
}
}
return true
}
/*
// Do performs a hook.
func (hook *Hook) Do(details *commandDetailsT) error {
var lerrs []error
msgs := FenceMatch(hook.Name, hook.ScanWriter, hook.Fence, details)
nextMessage:
for _, msg := range msgs {
nextEndpoint:
for _, endpoint := range hook.Endpoints {
switch endpoint.Protocol {
case HTTP:
if err := sendHTTPMessage(endpoint, []byte(msg)); err != nil {
lerrs = append(lerrs, err)
continue nextEndpoint
}
continue nextMessage // sent
case Disque:
if err := sendDisqueMessage(endpoint, []byte(msg)); err != nil {
lerrs = append(lerrs, err)
continue nextEndpoint
}
continue nextMessage // sent
}
}
}
if len(lerrs) == 0 {
// log.Notice("YAY")
return nil
}
var errmsgs []string
for _, err := range lerrs {
errmsgs = append(errmsgs, err.Error())
}
err := errors.New("not sent: " + strings.Join(errmsgs, ","))
log.Error(err)
return err
}*/