tile38/internal/server/aof.go

539 lines
12 KiB
Go
Raw Normal View History

package server
2016-03-05 02:08:16 +03:00
import (
"errors"
"fmt"
"io"
"math"
2016-03-05 02:08:16 +03:00
"net"
"os"
2019-01-09 10:23:53 +03:00
"sort"
2016-03-05 02:08:16 +03:00
"strconv"
"strings"
"sync"
"time"
2016-09-11 17:49:48 +03:00
"github.com/tidwall/buntdb"
2019-01-09 10:23:53 +03:00
"github.com/tidwall/gjson"
2017-10-05 02:15:20 +03:00
"github.com/tidwall/redcon"
2016-03-28 18:57:41 +03:00
"github.com/tidwall/resp"
"github.com/tidwall/tile38/internal/log"
2016-03-05 02:08:16 +03:00
)
2016-03-19 17:16:19 +03:00
type errAOFHook struct {
err error
}
func (err errAOFHook) Error() string {
return fmt.Sprintf("hook: %v", err.err)
}
func (s *Server) loadAOF() (err error) {
fi, err := s.aof.Stat()
2016-04-01 02:26:36 +03:00
if err != nil {
return err
}
2016-03-05 02:08:16 +03:00
start := time.Now()
var count int
defer func() {
d := time.Since(start)
2016-03-05 02:08:16 +03:00
ps := float64(count) / (float64(d) / float64(time.Second))
2016-04-01 02:26:36 +03:00
suf := []string{"bytes/s", "KB/s", "MB/s", "GB/s", "TB/s"}
bps := float64(fi.Size()) / (float64(d) / float64(time.Second))
2022-09-25 13:54:22 +03:00
for i := 0; bps > 1024 && len(suf) > 1; i++ {
2016-04-01 02:26:36 +03:00
bps /= 1024
suf = suf[1:]
}
byteSpeed := fmt.Sprintf("%.0f %s", bps, suf[0])
log.Infof("AOF loaded %d commands: %.2fs, %.0f/s, %s",
count, float64(d)/float64(time.Second), ps, byteSpeed)
2016-03-05 02:08:16 +03:00
}()
2017-10-05 02:15:20 +03:00
var buf []byte
var args [][]byte
var packet [0xFFFF]byte
2016-03-05 02:08:16 +03:00
for {
n, err := s.aof.Read(packet[:])
2016-03-05 02:08:16 +03:00
if err != nil {
if err != io.EOF {
return err
}
if len(buf) > 0 {
// There was an incomplete command or other data at the end of
// the AOF file. Attempt to recover the file by truncating the
// file at the end position of the last complete command.
log.Warnf("Truncating %d bytes due to an incomplete command\n",
len(buf))
s.aofsz -= len(buf)
if err := s.aof.Truncate(int64(s.aofsz)); err != nil {
return err
2017-10-05 02:15:20 +03:00
}
if _, err := s.aof.Seek(int64(s.aofsz), 0); err != nil {
return err
}
2016-03-05 02:08:16 +03:00
}
return nil
2016-03-05 02:08:16 +03:00
}
s.aofsz += n
2017-10-05 02:15:20 +03:00
data := packet[:n]
if len(buf) > 0 {
data = append(buf, data...)
}
2017-10-05 02:15:20 +03:00
var complete bool
for {
if len(data) > 0 && data[0] == 0 {
// Zeros found in AOF file (issue #230).
// Just ignore it and move the next byte.
data = data[1:]
continue
}
2017-10-05 02:15:20 +03:00
complete, args, _, data, err = redcon.ReadNextCommand(data, args[:0])
if err != nil {
return err
}
2017-10-05 02:15:20 +03:00
if !complete {
break
}
2017-10-05 02:15:20 +03:00
if len(args) > 0 {
2018-11-15 19:15:39 +03:00
var msg Message
msg.Args = msg.Args[:0]
2017-10-05 02:15:20 +03:00
for _, arg := range args {
msg.Args = append(msg.Args, string(arg))
2017-10-05 02:15:20 +03:00
}
if _, _, err := s.command(&msg, nil); err != nil {
2017-10-05 02:15:20 +03:00
if commandErrIsFatal(err) {
return err
}
}
count++
}
2016-03-19 17:16:19 +03:00
}
2017-10-05 02:15:20 +03:00
if len(data) > 0 {
buf = append(buf[:0], data...)
} else if len(buf) > 0 {
buf = buf[:0]
2016-03-05 02:08:16 +03:00
}
}
}
2017-10-05 02:15:20 +03:00
2016-03-30 19:32:38 +03:00
func commandErrIsFatal(err error) bool {
// FSET (and other writable commands) may return errors that we need
// to ignore during the loading process. These errors may occur (though unlikely)
// due to the aof rewrite operation.
2022-09-25 13:54:22 +03:00
return !(err == errKeyNotFound || err == errIDNotFound)
2016-03-30 19:32:38 +03:00
}
2019-09-04 03:01:26 +03:00
// flushAOF flushes all aof buffer data to disk. Set sync to true to sync the
// fsync the file.
func (s *Server) flushAOF(sync bool) {
if len(s.aofbuf) > 0 {
_, err := s.aof.Write(s.aofbuf)
if err != nil {
panic(err)
}
// send a broadcast to all sleeping followers
s.fcond.Broadcast()
2019-03-10 20:48:14 +03:00
if sync {
if err := s.aof.Sync(); err != nil {
2019-03-10 20:48:14 +03:00
panic(err)
}
}
if cap(s.aofbuf) > 1024*1024*32 {
s.aofbuf = make([]byte, 0, 1024*1024*32)
2019-09-04 03:01:26 +03:00
} else {
s.aofbuf = s.aofbuf[:0]
2019-09-04 03:01:26 +03:00
}
}
}
func (s *Server) writeAOF(args []string, d *commandDetails) error {
2018-08-14 03:05:30 +03:00
if d != nil && !d.updated {
// just ignore writes if the command did not update
return nil
2016-03-19 17:16:19 +03:00
}
if s.shrinking {
nargs := make([]string, len(args))
copy(nargs, args)
s.shrinklog = append(s.shrinklog, nargs)
2016-03-28 18:57:41 +03:00
}
if s.aof != nil {
2022-11-03 20:07:17 +03:00
s.aofdirty.Store(true) // prewrite optimization flag
n := len(s.aofbuf)
s.aofbuf = redcon.AppendArray(s.aofbuf, len(args))
for _, arg := range args {
s.aofbuf = redcon.AppendBulkString(s.aofbuf, arg)
}
s.aofsz += len(s.aofbuf) - n
2016-03-20 04:31:59 +03:00
}
2016-03-05 02:08:16 +03:00
2018-08-14 03:05:30 +03:00
// process geofences
2016-03-05 02:08:16 +03:00
if d != nil {
2018-08-14 03:05:30 +03:00
// webhook geofences
if s.config.followHost() == "" {
2018-08-14 03:05:30 +03:00
// for leader only
if d.parent {
// queue children
for _, d := range d.children {
if err := s.queueHooks(d); err != nil {
2018-08-14 03:05:30 +03:00
return err
}
}
} else {
// queue parent
if err := s.queueHooks(d); err != nil {
2018-08-14 03:05:30 +03:00
return err
}
}
}
2018-11-23 11:39:04 +03:00
2018-08-14 03:05:30 +03:00
// live geofences
s.lcond.L.Lock()
if len(s.lives) > 0 {
2018-11-24 01:38:49 +03:00
if d.parent {
// queue children
s.lstack = append(s.lstack, d.children...)
2018-11-24 01:38:49 +03:00
} else {
// queue parent
s.lstack = append(s.lstack, d)
}
s.lcond.Broadcast()
}
s.lcond.L.Unlock()
2016-03-05 02:08:16 +03:00
}
2016-04-02 17:20:30 +03:00
return nil
}
2016-03-19 17:16:19 +03:00
func (s *Server) getQueueCandidates(d *commandDetails) []*Hook {
candidates := make(map[*Hook]bool)
2018-11-24 04:15:14 +03:00
// add the hooks with "outside" detection
s.hooksOut.Ascend(nil, func(v interface{}) bool {
hook := v.(*Hook)
if hook.Key == d.key {
candidates[hook] = true
2018-11-24 04:15:14 +03:00
}
return true
})
// look for candidates that might "cross" geofences
2022-09-21 00:20:53 +03:00
if d.old != nil && d.obj != nil && s.hookCross.Len() > 0 {
r1, r2 := d.old.Rect(), d.obj.Rect()
s.hookCross.Search(
[2]float64{
math.Min(r1.Min.X, r2.Min.X),
math.Min(r1.Min.Y, r2.Min.Y),
},
[2]float64{
math.Max(r1.Max.X, r2.Max.X),
math.Max(r1.Max.Y, r2.Max.Y),
},
func(min, max [2]float64, value interface{}) bool {
hook := value.(*Hook)
if hook.Key == d.key {
candidates[hook] = true
}
return true
})
2018-11-24 04:15:14 +03:00
}
// look for candidates that overlap the old object
2022-09-21 00:20:53 +03:00
if d.old != nil {
r1 := d.old.Rect()
s.hookTree.Search(
[2]float64{r1.Min.X, r1.Min.Y},
[2]float64{r1.Max.X, r1.Max.Y},
func(min, max [2]float64, value interface{}) bool {
hook := value.(*Hook)
if hook.Key == d.key {
candidates[hook] = true
}
return true
})
}
// look for candidates that overlap the new object
if d.obj != nil {
r1 := d.obj.Rect()
s.hookTree.Search(
[2]float64{r1.Min.X, r1.Min.Y},
[2]float64{r1.Max.X, r1.Max.Y},
func(min, max [2]float64, value interface{}) bool {
hook := value.(*Hook)
if hook.Key == d.key {
candidates[hook] = true
}
return true
})
}
if len(candidates) == 0 {
return nil
}
// return the candidates as a slice
ret := make([]*Hook, 0, len(candidates))
for hook := range candidates {
ret = append(ret, hook)
}
return ret
2018-11-24 04:15:14 +03:00
}
func (s *Server) queueHooks(d *commandDetails) error {
2019-01-09 10:23:53 +03:00
// Create the slices that will store all messages and hooks
var cmsgs, wmsgs []string
var whooks []*Hook
2018-11-24 04:15:14 +03:00
2019-01-09 10:23:53 +03:00
// Compile a slice of potential hook recipients
candidates := s.getQueueCandidates(d)
2018-11-24 04:15:14 +03:00
for _, hook := range candidates {
2019-01-09 10:23:53 +03:00
// Calculate all matching fence messages for all candidates and append
// them to the appropriate message slice
2018-11-24 04:15:14 +03:00
msgs := FenceMatch(hook.Name, hook.ScanWriter, hook.Fence, hook.Metas, d)
if len(msgs) > 0 {
if hook.channel {
2019-01-09 10:23:53 +03:00
cmsgs = append(cmsgs, msgs...)
2018-11-24 04:15:14 +03:00
} else {
2019-01-09 10:23:53 +03:00
wmsgs = append(wmsgs, msgs...)
whooks = append(whooks, hook)
2016-09-11 17:49:48 +03:00
}
2016-04-02 17:20:30 +03:00
}
}
2019-01-09 10:23:53 +03:00
// Return nil if there are no messages to be sent
if len(cmsgs)+len(wmsgs) == 0 {
2016-09-11 17:49:48 +03:00
return nil
}
2019-01-09 10:23:53 +03:00
// Sort both message channel and webhook message slices
2019-01-09 20:35:50 +03:00
if len(cmsgs) > 1 {
sortMsgs(cmsgs)
}
if len(wmsgs) > 1 {
sortMsgs(wmsgs)
}
2019-01-09 10:23:53 +03:00
// Publish all channel messages if any exist
if len(cmsgs) > 0 {
for _, m := range cmsgs {
s.Publish(gjson.Get(m, "hook").String(), m)
2019-01-09 10:23:53 +03:00
}
}
// Queue the webhook messages in the buntdb database
err := s.qdb.Update(func(tx *buntdb.Tx) error {
2019-01-09 10:23:53 +03:00
for _, msg := range wmsgs {
s.qidx++ // increment the log id
key := hookLogPrefix + uint64ToString(s.qidx)
2019-01-09 10:23:53 +03:00
_, _, err := tx.Set(key, msg, hookLogSetDefaults)
2016-09-11 17:49:48 +03:00
if err != nil {
return err
}
log.Debugf("queued hook: %d", s.qidx)
2016-09-11 17:49:48 +03:00
}
_, _, err := tx.Set("hook:idx", uint64ToString(s.qidx), nil)
2016-09-11 17:49:48 +03:00
if err != nil {
return err
}
return nil
})
if err != nil {
return err
}
// all the messages have been queued.
// notify the hooks
2019-01-09 10:23:53 +03:00
for _, hook := range whooks {
2016-09-11 17:49:48 +03:00
hook.Signal()
}
2016-03-05 02:08:16 +03:00
return nil
}
2019-01-09 10:23:53 +03:00
2019-01-09 20:35:50 +03:00
// sortMsgs sorts passed notification messages by their detect and hook fields
2019-01-09 10:23:53 +03:00
func sortMsgs(msgs []string) {
2019-01-09 20:35:50 +03:00
sort.SliceStable(msgs, func(i, j int) bool {
2019-01-09 10:23:53 +03:00
detectI := msgDetectCode(gjson.Get(msgs[i], "detect").String())
detectJ := msgDetectCode(gjson.Get(msgs[j], "detect").String())
if detectI < detectJ {
return true
}
if detectI > detectJ {
return false
}
hookI := gjson.Get(msgs[i], "hook").String()
hookJ := gjson.Get(msgs[j], "hook").String()
return hookI < hookJ
})
}
2019-01-09 20:35:50 +03:00
// msgDetectCode returns a weight value for the passed detect value
func msgDetectCode(detect string) int {
switch detect {
2019-01-09 10:23:53 +03:00
case "exit":
return 1
case "outside":
return 2
case "enter":
return 3
case "inside":
return 4
default:
return 0
}
}
2016-03-05 02:08:16 +03:00
2016-09-11 17:49:48 +03:00
// Converts string to an integer
func stringToUint64(s string) uint64 {
n, _ := strconv.ParseUint(s, 10, 64)
return n
}
// Converts a uint to a string
func uint64ToString(u uint64) string {
s := strings.Repeat("0", 20) + strconv.FormatUint(u, 10)
return s[len(s)-20:]
}
2016-03-05 02:08:16 +03:00
type liveAOFSwitches struct {
pos int64
}
func (s liveAOFSwitches) Error() string {
2018-08-14 03:05:30 +03:00
return goingLive
2016-03-05 02:08:16 +03:00
}
2022-09-27 01:43:14 +03:00
// AOFMD5 pos size
func (s *Server) cmdAOFMD5(msg *Message) (resp.Value, error) {
2016-03-05 02:08:16 +03:00
start := time.Now()
2017-10-05 18:20:40 +03:00
2022-09-27 01:43:14 +03:00
// >> Args
args := msg.Args
if len(args) != 3 {
return retrerr(errInvalidNumberOfArguments)
2016-03-05 02:08:16 +03:00
}
2022-09-27 01:43:14 +03:00
pos, err := strconv.ParseInt(args[1], 10, 64)
2016-03-05 02:08:16 +03:00
if err != nil || pos < 0 {
2022-09-27 01:43:14 +03:00
return retrerr(errInvalidArgument(args[1]))
2016-03-05 02:08:16 +03:00
}
2022-09-27 01:43:14 +03:00
size, err := strconv.ParseInt(args[2], 10, 64)
2016-03-05 02:08:16 +03:00
if err != nil || size < 0 {
2022-09-27 01:43:14 +03:00
return retrerr(errInvalidArgument(args[2]))
2016-03-05 02:08:16 +03:00
}
2022-09-27 01:43:14 +03:00
// >> Operation
sum, err := s.checksum(pos, size)
2016-03-05 02:08:16 +03:00
if err != nil {
2022-09-27 01:43:14 +03:00
return retrerr(err)
2016-03-05 02:08:16 +03:00
}
2022-09-27 01:43:14 +03:00
// >> Response
if msg.OutputType == JSON {
return resp.StringValue(fmt.Sprintf(
`{"ok":true,"md5":"%s","elapsed":"%s"}`,
sum, time.Since(start))), nil
2016-04-01 02:26:36 +03:00
}
2022-09-27 01:43:14 +03:00
return resp.SimpleStringValue(sum), nil
2016-03-05 02:08:16 +03:00
}
2022-09-27 01:43:14 +03:00
// AOF pos
func (s *Server) cmdAOF(msg *Message) (resp.Value, error) {
if s.aof == nil {
2022-09-27 01:43:14 +03:00
return retrerr(errors.New("aof disabled"))
}
2017-10-05 18:20:40 +03:00
2022-09-27 01:43:14 +03:00
// >> Args
args := msg.Args
if len(args) != 2 {
return retrerr(errInvalidNumberOfArguments)
2016-03-05 02:08:16 +03:00
}
2022-09-27 01:43:14 +03:00
pos, err := strconv.ParseInt(args[1], 10, 64)
2016-03-05 02:08:16 +03:00
if err != nil || pos < 0 {
2022-09-27 01:43:14 +03:00
return retrerr(errInvalidArgument(args[1]))
2016-03-05 02:08:16 +03:00
}
2022-09-27 01:43:14 +03:00
// >> Operation
f, err := os.Open(s.aof.Name())
2016-03-05 02:08:16 +03:00
if err != nil {
2022-09-27 01:43:14 +03:00
return retrerr(err)
2016-03-05 02:08:16 +03:00
}
defer f.Close()
2022-09-27 01:43:14 +03:00
2016-03-05 02:08:16 +03:00
n, err := f.Seek(0, 2)
if err != nil {
2022-09-27 01:43:14 +03:00
return retrerr(err)
2016-03-05 02:08:16 +03:00
}
2022-09-27 01:43:14 +03:00
2016-03-05 02:08:16 +03:00
if n < pos {
2022-09-27 01:43:14 +03:00
return retrerr(errors.New(
"pos is too big, must be less that the aof_size of leader"))
2016-03-05 02:08:16 +03:00
}
2022-09-27 01:43:14 +03:00
// >> Response
var ls liveAOFSwitches
ls.pos = pos
return NOMessage, ls
2016-03-05 02:08:16 +03:00
}
func (s *Server) liveAOF(pos int64, conn net.Conn, rd *PipelineReader, msg *Message) error {
s.mu.RLock()
f, err := os.Open(s.aof.Name())
s.mu.RUnlock()
if err != nil {
return err
}
s.mu.Lock()
s.aofconnM[conn] = f
s.mu.Unlock()
2016-04-01 04:20:42 +03:00
defer func() {
s.mu.Lock()
delete(s.aofconnM, conn)
s.mu.Unlock()
2016-04-01 04:20:42 +03:00
conn.Close()
2022-09-27 01:43:14 +03:00
f.Close()
2016-04-01 04:20:42 +03:00
}()
2016-04-01 02:26:36 +03:00
if _, err := conn.Write([]byte("+OK\r\n")); err != nil {
return err
2016-03-05 02:08:16 +03:00
}
if _, err := f.Seek(pos, 0); err != nil {
return err
}
2022-09-27 01:43:14 +03:00
var wg sync.WaitGroup
wg.Add(1)
2016-03-05 02:08:16 +03:00
go func() {
defer func() {
2022-09-27 01:43:14 +03:00
f.Close()
conn.Close()
wg.Done()
2016-03-05 02:08:16 +03:00
}()
2022-09-27 01:43:14 +03:00
// Any incoming message should end the connection
rd.ReadMessages()
2016-03-05 02:08:16 +03:00
}()
2022-09-27 01:43:14 +03:00
_, err = io.Copy(conn, f)
if err != nil {
return err
}
2022-09-27 01:43:14 +03:00
b := make([]byte, 4096*2)
for {
n, err := f.Read(b)
if n > 0 {
if _, err := conn.Write(b[:n]); err != nil {
2016-03-05 02:08:16 +03:00
return err
}
}
2022-09-27 01:43:14 +03:00
if err == io.EOF {
s.fcond.L.Lock()
s.fcond.Wait()
s.fcond.L.Unlock()
2022-09-27 01:43:14 +03:00
} else if err != nil {
if errors.Is(err, os.ErrClosed) {
// The live aof file can be closed when a client (follower) has
// closed their connection or following an AOFSHRINK operation.
err = nil
}
2022-09-27 01:43:14 +03:00
return err
2016-03-05 02:08:16 +03:00
}
}
}