brotli/hash.go

package brotli

import (
	"encoding/binary"
	"fmt"
)

/* Matches data against static dictionary words, and for each length l,
   for which a match is found, updates matches[l] to be the minimum possible
     (distance << 5) + len_code.
   Returns 1 if matches have been found, otherwise 0.
   Prerequisites:
     matches array is at least BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN + 1 long
     all elements are initialized to kInvalidMatch */
/* Pointer to hasher data.
 *
 * Excluding initialization and destruction, hasher can be passed as
 * HasherHandle by value.
 *
 * Typically hasher data consists of 3 sections:
 * * hasherCommon structure
 * * private structured hasher data, depending on hasher type
 * * private dynamic hasher data, depending on hasher type and parameters
 *
 */
type hasherCommon struct {
	params           hasherParams
	is_prepared_     bool
	dict_num_lookups uint
	dict_num_matches uint
}

func (h *hasherCommon) Common() *hasherCommon {
	return h
}

type hasherHandle interface {
	Common() *hasherCommon
	Initialize(params *encoderParams)
	Prepare(one_shot bool, input_size uint, data []byte)
	StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint)
	HashTypeLength() uint
	StoreLookahead() uint
	PrepareDistanceCache(distance_cache []int)
	FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult)
	StoreRange(data []byte, mask uint, ix_start uint, ix_end uint)
	Store(data []byte, mask uint, ix uint)
}

type score_t uint

var kCutoffTransformsCount uint32 = 10

/*   0,  12,   27,    23,    42,    63,    56,    48,    59,    64 */
/* 0+0, 4+8, 8+19, 12+11, 16+26, 20+43, 24+32, 28+20, 32+27, 36+28 */
var kCutoffTransforms uint64 = 0x071B520ADA2D3200

type hasherSearchResult struct {
	len            uint
	distance       uint
	score          uint
	len_code_delta int
}

/* kHashMul32 multiplier has these properties:
   * The multiplier must be odd. Otherwise we may lose the highest bit.
   * No long streaks of ones or zeros.
   * There is no effort to ensure that it is a prime, the oddity is enough
     for this use.
   * The number has been tuned heuristically against compression benchmarks. */
var kHashMul32 uint32 = 0x1E35A7BD

var kHashMul64 uint64 = 0x1E35A7BD1E35A7BD

var kHashMul64Long uint64 = 0x1FE35A7BD3579BD3

func hash14(data []byte) uint32 {
	var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32

	/* The higher bits contain more mixture from the multiplication,
	   so we take our results from there. */
	return h >> (32 - 14)
}

func prepareDistanceCache(distance_cache []int, num_distances int) {
	if num_distances > 4 {
		var last_distance int = distance_cache[0]
		distance_cache[4] = last_distance - 1
		distance_cache[5] = last_distance + 1
		distance_cache[6] = last_distance - 2
		distance_cache[7] = last_distance + 2
		distance_cache[8] = last_distance - 3
		distance_cache[9] = last_distance + 3
		if num_distances > 10 {
			var next_last_distance int = distance_cache[1]
			distance_cache[10] = next_last_distance - 1
			distance_cache[11] = next_last_distance + 1
			distance_cache[12] = next_last_distance - 2
			distance_cache[13] = next_last_distance + 2
			distance_cache[14] = next_last_distance - 3
			distance_cache[15] = next_last_distance + 3
		}
	}
}

const literalByteScore = 135

const distanceBitPenalty = 30

/* Score must be positive after applying maximal penalty. */
const scoreBase = (distanceBitPenalty * 8 * 8)

/* Usually, we always choose the longest backward reference. This function
   allows for the exception of that rule.

   If we choose a backward reference that is further away, it will
   usually be coded with more bits. We approximate this by assuming
   log2(distance). If the distance can be expressed in terms of the
   last four distances, we use some heuristic constants to estimate
   the bits cost. For the first up to four literals we use the bit
   cost of the literals from the literal cost model, after that we
   use the average bit cost of the cost model.

   This function is used to sometimes discard a longer backward reference
   when it is not much longer and the bit cost for encoding it is more
   than the saved literals.

   backward_reference_offset MUST be positive. */
func backwardReferenceScore(copy_length uint, backward_reference_offset uint) uint {
	return scoreBase + literalByteScore*uint(copy_length) - distanceBitPenalty*uint(log2FloorNonZero(backward_reference_offset))
}

func backwardReferenceScoreUsingLastDistance(copy_length uint) uint {
	return literalByteScore*uint(copy_length) + scoreBase + 15
}

func backwardReferencePenaltyUsingLastDistance(distance_short_code uint) uint {
	return uint(39) + ((0x1CA10 >> (distance_short_code & 0xE)) & 0xE)
}

func testStaticDictionaryItem(dictionary *encoderDictionary, item uint, data []byte, max_length uint, max_backward uint, max_distance uint, out *hasherSearchResult) bool {
	var len uint
	var word_idx uint
	var offset uint
	var matchlen uint
	var backward uint
	var score uint
	len = item & 0x1F
	word_idx = item >> 5
	offset = uint(dictionary.words.offsets_by_length[len]) + len*word_idx
	if len > max_length {
		return false
	}

	matchlen = findMatchLengthWithLimit(data, dictionary.words.data[offset:], uint(len))
	if matchlen+uint(dictionary.cutoffTransformsCount) <= len || matchlen == 0 {
		return false
	}
	{
		var cut uint = len - matchlen
		var transform_id uint = (cut << 2) + uint((dictionary.cutoffTransforms>>(cut*6))&0x3F)
		backward = max_backward + 1 + word_idx + (transform_id << dictionary.words.size_bits_by_length[len])
	}

	if backward > max_distance {
		return false
	}

	score = backwardReferenceScore(matchlen, backward)
	if score < out.score {
		return false
	}

	out.len = matchlen
	out.len_code_delta = int(len) - int(matchlen)
	out.distance = backward
	out.score = score
	return true
}

func searchInStaticDictionary(dictionary *encoderDictionary, handle hasherHandle, data []byte, max_length uint, max_backward uint, max_distance uint, out *hasherSearchResult, shallow bool) {
	var key uint
	var i uint
	var self *hasherCommon = handle.Common()
	if self.dict_num_matches < self.dict_num_lookups>>7 {
		return
	}

	key = uint(hash14(data) << 1)
	for i = 0; ; (func() { i++; key++ })() {
		var tmp uint
		if shallow {
			tmp = 1
		} else {
			tmp = 2
		}
		if i >= tmp {
			break
		}
		var item uint = uint(dictionary.hash_table[key])
		self.dict_num_lookups++
		if item != 0 {
			var item_matches bool = testStaticDictionaryItem(dictionary, item, data, max_length, max_backward, max_distance, out)
			if item_matches {
				self.dict_num_matches++
			}
		}
	}
}

type backwardMatch struct {
	distance        uint32
	length_and_code uint32
}

func initBackwardMatch(self *backwardMatch, dist uint, len uint) {
	self.distance = uint32(dist)
	self.length_and_code = uint32(len << 5)
}

func initDictionaryBackwardMatch(self *backwardMatch, dist uint, len uint, len_code uint) {
	self.distance = uint32(dist)
	var tmp uint
	if len == len_code {
		tmp = 0
	} else {
		tmp = len_code
	}
	self.length_and_code = uint32(len<<5 | tmp)
}

func backwardMatchLength(self *backwardMatch) uint {
	return uint(self.length_and_code >> 5)
}

func backwardMatchLengthCode(self *backwardMatch) uint {
	var code uint = uint(self.length_and_code) & 31
	if code != 0 {
		return code
	} else {
		return backwardMatchLength(self)
	}
}

func hasherReset(handle hasherHandle) {
	if handle == nil {
		return
	}
	handle.Common().is_prepared_ = false
}

func newHasher(typ int) hasherHandle {
	switch typ {
	case 2:
		return &hashLongestMatchQuickly{
			bucketBits:    16,
			bucketSweep:   1,
			hashLen:       5,
			useDictionary: true,
		}
	case 3:
		return &hashLongestMatchQuickly{
			bucketBits:    16,
			bucketSweep:   2,
			hashLen:       5,
			useDictionary: false,
		}
	case 4:
		return &hashLongestMatchQuickly{
			bucketBits:    17,
			bucketSweep:   4,
			hashLen:       5,
			useDictionary: true,
		}
	case 5:
		return new(h5)
	case 6:
		return new(h6)
	case 10:
		return new(h10)
	case 35:
		return &hashComposite{
			ha: newHasher(3),
			hb: &hashRolling{jump: 4},
		}
	case 40:
		return &hashForgetfulChain{
			bucketBits:              15,
			numBanks:                1,
			bankBits:                16,
			numLastDistancesToCheck: 4,
		}
	case 41:
		return &hashForgetfulChain{
			bucketBits:              15,
			numBanks:                1,
			bankBits:                16,
			numLastDistancesToCheck: 10,
		}
	case 42:
		return &hashForgetfulChain{
			bucketBits:              15,
			numBanks:                512,
			bankBits:                9,
			numLastDistancesToCheck: 16,
		}
	case 54:
		return &hashLongestMatchQuickly{
			bucketBits:    20,
			bucketSweep:   4,
			hashLen:       7,
			useDictionary: false,
		}
	case 55:
		return &hashComposite{
			ha: newHasher(54),
			hb: &hashRolling{jump: 4},
		}
	case 65:
		return &hashComposite{
			ha: newHasher(6),
			hb: &hashRolling{jump: 1},
		}
	}

	panic(fmt.Sprintf("unknown hasher type: %d", typ))
}

func hasherSetup(handle *hasherHandle, params *encoderParams, data []byte, position uint, input_size uint, is_last bool) {
	var self hasherHandle = nil
	var common *hasherCommon = nil
	var one_shot bool = (position == 0 && is_last)
	if *handle == nil {
		ChooseHasher(params, &params.hasher)
		self = newHasher(params.hasher.type_)

		*handle = self
		common = self.Common()
		common.params = params.hasher
		self.Initialize(params)
	}

	self = *handle
	common = self.Common()
	if !common.is_prepared_ {
		self.Prepare(one_shot, input_size, data)

		if position == 0 {
			common.dict_num_lookups = 0
			common.dict_num_matches = 0
		}

		common.is_prepared_ = true
	}
}

func initOrStitchToPreviousBlock(handle *hasherHandle, data []byte, mask uint, params *encoderParams, position uint, input_size uint, is_last bool) {
	var self hasherHandle
	hasherSetup(handle, params, data, position, input_size, is_last)
	self = *handle
	self.StitchToPreviousBlock(input_size, position, data, mask)
}