package matchfinder import ( "encoding/binary" "math/bits" "runtime" ) // M4 is an implementation of the MatchFinder // interface that uses a simple hash table to find matches, // but the advanced parsing technique from // https://fastcompression.blogspot.com/2011/12/advanced-parsing-strategies.html, // except that it normally looks for matches at every input position. type M4 struct { // MaxDistance is the maximum distance (in bytes) to look back for // a match. The default is 65535. MaxDistance int // MinLength is the length of the shortest match to return. // The default is 4. MinLength int // HashLen is the number of bytes to use to calculate the hashes. // The maximum is 8 and the default is 6. HashLen int // TableBits is the number of bits in the hash table indexes. // The default is 17 (128K entries). TableBits int // When LimitedSearch is true, it only looks for matches at certain // points in the input rather than at every byte. // (This makes compression faster, but hurts the compression ratio.) LimitedSearch bool table []uint32 history []byte } func (q *M4) Reset() { for i := range q.table { q.table[i] = 0 } q.history = q.history[:0] } func (q *M4) FindMatches(dst []Match, src []byte) []Match { if q.MaxDistance == 0 { q.MaxDistance = 65535 } if q.MinLength == 0 { q.MinLength = 4 } if q.HashLen == 0 { q.HashLen = 6 } if q.TableBits == 0 { q.TableBits = 17 } if len(q.table) < 1< q.MaxDistance*2 { // Trim down the history buffer. delta := len(q.history) - q.MaxDistance copy(q.history, q.history[delta:]) q.history = q.history[:q.MaxDistance] for i, v := range q.table { newV := int(v) - delta if newV < 0 { newV = 0 } q.table[i] = uint32(newV) } } // Append src to the history buffer. e.NextEmit = len(q.history) q.history = append(q.history, src...) src = q.history // matches stores the matches that have been found but not emitted, // in reverse order. (matches[0] is the most recent one.) var matches [3]absoluteMatch for i := e.NextEmit; i < len(src)-7; i++ { if matches[0] != (absoluteMatch{}) && i >= matches[0].End { // We have found some matches, and we're far enough along that we probably // won't find overlapping matches, so we might as well emit them. if matches[1] != (absoluteMatch{}) { e.trim(matches[1], matches[0].Start, q.MinLength) } e.emit(matches[0]) matches = [3]absoluteMatch{} } // Now look for a match. h := ((binary.LittleEndian.Uint64(src[i:]) & (1<<(8*q.HashLen) - 1)) * hashMul64) >> (64 - q.TableBits) candidate := int(q.table[h]) q.table[h] = uint32(i) if q.LimitedSearch && i < matches[0].End && i != matches[0].End+2-q.HashLen { continue } if candidate == 0 || i-candidate > q.MaxDistance || i-candidate == matches[0].Start-matches[0].Match { continue } if binary.LittleEndian.Uint32(src[candidate:]) != binary.LittleEndian.Uint32(src[i:]) { continue } m := extendMatch2(src, i, candidate, e.NextEmit) if m.End-m.Start <= matches[0].End-matches[0].Start { continue } matches = [3]absoluteMatch{ m, matches[0], matches[1], } if matches[2] == (absoluteMatch{}) { continue } // We have three matches, so it's time to emit one and/or eliminate one. switch { case matches[0].Start < matches[2].End: // The first and third matches overlap; discard the one in between. matches = [3]absoluteMatch{ matches[0], matches[2], absoluteMatch{}, } case matches[0].Start < matches[2].End+q.MinLength: // The first and third matches don't overlap, but there's no room for // another match between them. Emit the first match and discard the second. e.emit(matches[2]) matches = [3]absoluteMatch{ matches[0], absoluteMatch{}, absoluteMatch{}, } default: // Emit the first match, shortening it if necessary to avoid overlap with the second. e.trim(matches[2], matches[1].Start, q.MinLength) matches[2] = absoluteMatch{} } } // We've found all the matches now; emit the remaining ones. if matches[1] != (absoluteMatch{}) { e.trim(matches[1], matches[0].Start, q.MinLength) } if matches[0] != (absoluteMatch{}) { e.emit(matches[0]) } dst = e.Dst if e.NextEmit < len(src) { dst = append(dst, Match{ Unmatched: len(src) - e.NextEmit, }) } return dst } const hashMul64 = 0x1E35A7BD1E35A7BD // extendMatch returns the largest k such that k <= len(src) and that // src[i:i+k-j] and src[j:k] have the same contents. // // It assumes that: // // 0 <= i && i < j && j <= len(src) func extendMatch(src []byte, i, j int) int { switch runtime.GOARCH { case "amd64": // As long as we are 8 or more bytes before the end of src, we can load and // compare 8 bytes at a time. If those 8 bytes are equal, repeat. for j+8 < len(src) { iBytes := binary.LittleEndian.Uint64(src[i:]) jBytes := binary.LittleEndian.Uint64(src[j:]) if iBytes != jBytes { // If those 8 bytes were not equal, XOR the two 8 byte values, and return // the index of the first byte that differs. The BSF instruction finds the // least significant 1 bit, the amd64 architecture is little-endian, and // the shift by 3 converts a bit index to a byte index. return j + bits.TrailingZeros64(iBytes^jBytes)>>3 } i, j = i+8, j+8 } case "386": // On a 32-bit CPU, we do it 4 bytes at a time. for j+4 < len(src) { iBytes := binary.LittleEndian.Uint32(src[i:]) jBytes := binary.LittleEndian.Uint32(src[j:]) if iBytes != jBytes { return j + bits.TrailingZeros32(iBytes^jBytes)>>3 } i, j = i+4, j+4 } } for ; j < len(src) && src[i] == src[j]; i, j = i+1, j+1 { } return j } // Given a 4-byte match at src[start] and src[candidate], extendMatch2 extends it // upward as far as possible, and downward no farther than to min. func extendMatch2(src []byte, start, candidate, min int) absoluteMatch { end := extendMatch(src, candidate+4, start+4) for start > min && candidate > 0 && src[start-1] == src[candidate-1] { start-- candidate-- } return absoluteMatch{ Start: start, End: end, Match: candidate, } }