271 lines
7.2 KiB
Go
271 lines
7.2 KiB
Go
package matchfinder
|
|
|
|
import (
|
|
"encoding/binary"
|
|
"math/bits"
|
|
"runtime"
|
|
)
|
|
|
|
const (
|
|
ssapBits = 17
|
|
ssapMask = (1 << ssapBits) - 1
|
|
)
|
|
|
|
// M4 is an implementation of the MatchFinder
|
|
// interface that uses a simple hash table to find matches,
|
|
// but the advanced parsing technique from
|
|
// https://fastcompression.blogspot.com/2011/12/advanced-parsing-strategies.html,
|
|
// except that it looks for matches at every input position.
|
|
type M4 struct {
|
|
// MaxDistance is the maximum distance (in bytes) to look back for
|
|
// a match. The default is 65535.
|
|
MaxDistance int
|
|
|
|
// MinLength is the length of the shortest match to return.
|
|
// The default is 4.
|
|
MinLength int
|
|
|
|
// HashLen is the number of bytes to use to calculate the hashes.
|
|
// The maximum is 8 and the default is 6.
|
|
HashLen int
|
|
|
|
table [1 << ssapBits]uint32
|
|
|
|
history []byte
|
|
}
|
|
|
|
func (q *M4) Reset() {
|
|
q.table = [1 << ssapBits]uint32{}
|
|
q.history = q.history[:0]
|
|
}
|
|
|
|
func (q *M4) FindMatches(dst []Match, src []byte) []Match {
|
|
if q.MaxDistance == 0 {
|
|
q.MaxDistance = 65535
|
|
}
|
|
if q.MinLength == 0 {
|
|
q.MinLength = 4
|
|
}
|
|
if q.HashLen == 0 {
|
|
q.HashLen = 6
|
|
}
|
|
var nextEmit int
|
|
|
|
if len(q.history) > q.MaxDistance*2 {
|
|
// Trim down the history buffer.
|
|
delta := len(q.history) - q.MaxDistance
|
|
copy(q.history, q.history[delta:])
|
|
q.history = q.history[:q.MaxDistance]
|
|
|
|
for i, v := range q.table {
|
|
newV := int(v) - delta
|
|
if newV < 0 {
|
|
newV = 0
|
|
}
|
|
q.table[i] = uint32(newV)
|
|
}
|
|
}
|
|
|
|
// Append src to the history buffer.
|
|
nextEmit = len(q.history)
|
|
q.history = append(q.history, src...)
|
|
src = q.history
|
|
|
|
// matches stores the matches that have been found but not emitted,
|
|
// in reverse order. (matches[0] is the most recent one.)
|
|
var matches [3]absoluteMatch
|
|
for i := nextEmit; i < len(src)-7; i++ {
|
|
if matches[0] != (absoluteMatch{}) && i >= matches[0].End {
|
|
// We have found some matches, and we're far enough along that we probably
|
|
// won't find overlapping matches, so we might as well emit them.
|
|
if matches[1] != (absoluteMatch{}) {
|
|
if matches[1].End > matches[0].Start {
|
|
matches[1].End = matches[0].Start
|
|
}
|
|
if matches[1].End-matches[1].Start >= q.MinLength {
|
|
dst = append(dst, Match{
|
|
Unmatched: matches[1].Start - nextEmit,
|
|
Length: matches[1].End - matches[1].Start,
|
|
Distance: matches[1].Start - matches[1].Match,
|
|
})
|
|
nextEmit = matches[1].End
|
|
}
|
|
}
|
|
dst = append(dst, Match{
|
|
Unmatched: matches[0].Start - nextEmit,
|
|
Length: matches[0].End - matches[0].Start,
|
|
Distance: matches[0].Start - matches[0].Match,
|
|
})
|
|
nextEmit = matches[0].End
|
|
matches = [3]absoluteMatch{}
|
|
}
|
|
|
|
// Now look for a match.
|
|
h := ((binary.LittleEndian.Uint64(src[i:]) & (1<<(8*q.HashLen) - 1)) * hashMul64) >> (64 - ssapBits)
|
|
candidate := int(q.table[h&ssapMask])
|
|
q.table[h&ssapMask] = uint32(i)
|
|
|
|
if candidate == 0 || i-candidate > q.MaxDistance || i-candidate == matches[0].Start-matches[0].Match {
|
|
continue
|
|
}
|
|
if binary.LittleEndian.Uint32(src[candidate:]) != binary.LittleEndian.Uint32(src[i:]) {
|
|
continue
|
|
}
|
|
|
|
// We have a 4-byte match now.
|
|
|
|
start := i
|
|
match := candidate
|
|
end := extendMatch(src, match+4, start+4)
|
|
for start > nextEmit && match > 0 && src[start-1] == src[match-1] {
|
|
start--
|
|
match--
|
|
}
|
|
if end-start <= matches[0].End-matches[0].Start {
|
|
continue
|
|
}
|
|
|
|
matches = [3]absoluteMatch{
|
|
absoluteMatch{
|
|
Start: start,
|
|
End: end,
|
|
Match: match,
|
|
},
|
|
matches[0],
|
|
matches[1],
|
|
}
|
|
|
|
if matches[2] == (absoluteMatch{}) {
|
|
continue
|
|
}
|
|
|
|
// We have three matches, so it's time to emit one and/or eliminate one.
|
|
switch {
|
|
case matches[0].Start < matches[2].End:
|
|
// The first and third matches overlap; discard the one in between.
|
|
matches = [3]absoluteMatch{
|
|
matches[0],
|
|
matches[2],
|
|
absoluteMatch{},
|
|
}
|
|
|
|
case matches[0].Start < matches[2].End+q.MinLength:
|
|
// The first and third matches don't overlap, but there's no room for
|
|
// another match between them. Emit the first match and discard the second.
|
|
dst = append(dst, Match{
|
|
Unmatched: matches[2].Start - nextEmit,
|
|
Length: matches[2].End - matches[2].Start,
|
|
Distance: matches[2].Start - matches[2].Match,
|
|
})
|
|
nextEmit = matches[2].End
|
|
matches = [3]absoluteMatch{
|
|
matches[0],
|
|
absoluteMatch{},
|
|
absoluteMatch{},
|
|
}
|
|
|
|
default:
|
|
// Emit the first match, shortening it if necessary to avoid overlap with the second.
|
|
if matches[2].End > matches[1].Start {
|
|
matches[2].End = matches[1].Start
|
|
}
|
|
if matches[2].End-matches[2].Start >= q.MinLength {
|
|
dst = append(dst, Match{
|
|
Unmatched: matches[2].Start - nextEmit,
|
|
Length: matches[2].End - matches[2].Start,
|
|
Distance: matches[2].Start - matches[2].Match,
|
|
})
|
|
nextEmit = matches[2].End
|
|
}
|
|
matches[2] = absoluteMatch{}
|
|
}
|
|
}
|
|
|
|
// We've found all the matches now; emit the remaining ones.
|
|
if matches[1] != (absoluteMatch{}) {
|
|
if matches[1].End > matches[0].Start {
|
|
matches[1].End = matches[0].Start
|
|
}
|
|
if matches[1].End-matches[1].Start >= q.MinLength {
|
|
dst = append(dst, Match{
|
|
Unmatched: matches[1].Start - nextEmit,
|
|
Length: matches[1].End - matches[1].Start,
|
|
Distance: matches[1].Start - matches[1].Match,
|
|
})
|
|
nextEmit = matches[1].End
|
|
}
|
|
}
|
|
if matches[0] != (absoluteMatch{}) {
|
|
dst = append(dst, Match{
|
|
Unmatched: matches[0].Start - nextEmit,
|
|
Length: matches[0].End - matches[0].Start,
|
|
Distance: matches[0].Start - matches[0].Match,
|
|
})
|
|
nextEmit = matches[0].End
|
|
}
|
|
|
|
if nextEmit < len(src) {
|
|
dst = append(dst, Match{
|
|
Unmatched: len(src) - nextEmit,
|
|
})
|
|
}
|
|
|
|
return dst
|
|
}
|
|
|
|
const hashMul64 = 0x1E35A7BD1E35A7BD
|
|
|
|
// An absoluteMatch is like a Match, but it stores indexes into the byte
|
|
// stream instead of lengths.
|
|
type absoluteMatch struct {
|
|
// Start is the index of the first byte.
|
|
Start int
|
|
|
|
// End is the index of the byte after the last byte
|
|
// (so that End - Start = Length).
|
|
End int
|
|
|
|
// Match is the index of the previous data that matches
|
|
// (Start - Match = Distance).
|
|
Match int
|
|
}
|
|
|
|
// extendMatch returns the largest k such that k <= len(src) and that
|
|
// src[i:i+k-j] and src[j:k] have the same contents.
|
|
//
|
|
// It assumes that:
|
|
//
|
|
// 0 <= i && i < j && j <= len(src)
|
|
func extendMatch(src []byte, i, j int) int {
|
|
switch runtime.GOARCH {
|
|
case "amd64":
|
|
// As long as we are 8 or more bytes before the end of src, we can load and
|
|
// compare 8 bytes at a time. If those 8 bytes are equal, repeat.
|
|
for j+8 < len(src) {
|
|
iBytes := binary.LittleEndian.Uint64(src[i:])
|
|
jBytes := binary.LittleEndian.Uint64(src[j:])
|
|
if iBytes != jBytes {
|
|
// If those 8 bytes were not equal, XOR the two 8 byte values, and return
|
|
// the index of the first byte that differs. The BSF instruction finds the
|
|
// least significant 1 bit, the amd64 architecture is little-endian, and
|
|
// the shift by 3 converts a bit index to a byte index.
|
|
return j + bits.TrailingZeros64(iBytes^jBytes)>>3
|
|
}
|
|
i, j = i+8, j+8
|
|
}
|
|
case "386":
|
|
// On a 32-bit CPU, we do it 4 bytes at a time.
|
|
for j+4 < len(src) {
|
|
iBytes := binary.LittleEndian.Uint32(src[i:])
|
|
jBytes := binary.LittleEndian.Uint32(src[j:])
|
|
if iBytes != jBytes {
|
|
return j + bits.TrailingZeros32(iBytes^jBytes)>>3
|
|
}
|
|
i, j = i+4, j+4
|
|
}
|
|
}
|
|
for ; j < len(src) && src[i] == src[j]; i, j = i+1, j+1 {
|
|
}
|
|
return j
|
|
}
|