re
/
brotli
forked from mirror/brotli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Un-export some more symbols

This commit is contained in:
Andy Balholm 2019-03-15 17:24:40 -07:00
parent 906c822ef2
commit 7b66c0a2f2
50 changed files with 1476 additions and 1850 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
backward_references.go
View File

@ -42,8 +42,8 @@ func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uin
return distance + numDistanceShortCodes - 1
}
func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *BrotliEncoderParams, hasher HasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
var max_backward_limit uint = BROTLI_MAX_BACKWARD_LIMIT(params.lgwin)
func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
var orig_commands []command = commands
var insert_length uint = *last_insert_len
var pos_end uint = position + num_bytes
@ -58,7 +58,7 @@ func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte,
var gap uint = 0
/* Set maximum distance, see section 9.1. of the spec. */
var kMinScore uint = BROTLI_SCORE_BASE + 100
var kMinScore uint = scoreBase + 100
/* For speed up heuristics for random data. */
@ -68,7 +68,7 @@ func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte,
for position+hasher.HashTypeLength() < pos_end {
var max_length uint = pos_end - position
var max_distance uint = brotli_min_size_t(position, max_backward_limit)
var sr HasherSearchResult
var sr hasherSearchResult
sr.len = 0
sr.len_code_delta = 0
sr.distance = 0
@ -80,7 +80,7 @@ func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte,
max_length--
for ; ; max_length-- {
var cost_diff_lazy uint = 175
var sr2 HasherSearchResult
var sr2 hasherSearchResult
if params.quality < MIN_QUALITY_FOR_EXTENSIVE_REFERENCE_SEARCH {
sr2.len = brotli_min_size_t(sr.len-1, max_length)
} else {

76
backward_references_hq.go
View File

@ -80,7 +80,7 @@ type zopfliCostModel struct {
num_bytes_ uint
}
func initZopfliCostModel(self *zopfliCostModel, dist *BrotliDistanceParams, num_bytes uint) {
func initZopfliCostModel(self *zopfliCostModel, dist *distanceParams, num_bytes uint) {
var distance_histogram_size uint32 = dist.alphabet_size
if distance_histogram_size > maxEffectiveDistanceAlphabetSize {
distance_histogram_size = maxEffectiveDistanceAlphabetSize
@ -107,7 +107,7 @@ func setCost(histogram []uint32, histogram_size uint, literal_histogram bool, co
sum += uint(histogram[i])
}
log2sum = float32(FastLog2(sum))
log2sum = float32(fastLog2(sum))
missing_symbol_sum = sum
if !literal_histogram {
for i = 0; i < histogram_size; i++ {
@ -117,7 +117,7 @@ func setCost(histogram []uint32, histogram_size uint, literal_histogram bool, co
}
}
missing_symbol_cost = float32(FastLog2(missing_symbol_sum)) + 2
missing_symbol_cost = float32(fastLog2(missing_symbol_sum)) + 2
for i = 0; i < histogram_size; i++ {
if histogram[i] == 0 {
cost[i] = missing_symbol_cost
@ -125,7 +125,7 @@ func setCost(histogram []uint32, histogram_size uint, literal_histogram bool, co
}
/* Shannon bits for this symbol. */
cost[i] = log2sum - float32(FastLog2(uint(histogram[i])))
cost[i] = log2sum - float32(fastLog2(uint(histogram[i])))
/* Cannot be coded with less than 1 bit */
if cost[i] < 1 {
@ -197,7 +197,7 @@ func zopfliCostModelSetFromLiteralCosts(self *zopfliCostModel, position uint, ri
var cost_cmd []float32 = self.cost_cmd_[:]
var num_bytes uint = self.num_bytes_
var i uint
BrotliEstimateBitCostsForLiterals(position, num_bytes, ringbuffer_mask, ringbuffer, literal_costs[1:])
estimateBitCostsForLiterals(position, num_bytes, ringbuffer_mask, ringbuffer, literal_costs[1:])
literal_costs[0] = 0.0
for i = 0; i < num_bytes; i++ {
literal_carry += literal_costs[i+1]
@ -206,14 +206,14 @@ func zopfliCostModelSetFromLiteralCosts(self *zopfliCostModel, position uint, ri
}
for i = 0; i < numCommandSymbols; i++ {
cost_cmd[i] = float32(FastLog2(uint(11 + uint32(i))))
cost_cmd[i] = float32(fastLog2(uint(11 + uint32(i))))
}
for i = 0; uint32(i) < self.distance_histogram_size; i++ {
cost_dist[i] = float32(FastLog2(uint(20 + uint32(i))))
cost_dist[i] = float32(fastLog2(uint(20 + uint32(i))))
}
self.min_cost_cmd_ = float32(FastLog2(11))
self.min_cost_cmd_ = float32(fastLog2(11))
}
func zopfliCostModelGetCommandCost(self *zopfliCostModel, cmdcode uint16) float32 {
@ -382,7 +382,7 @@ func evaluateNode(block_start uint, pos uint, max_backward_limit uint, gap uint,
}
/* Returns longest copy length. */
func updateNodes(num_bytes uint, block_start uint, pos uint, ringbuffer []byte, ringbuffer_mask uint, params *BrotliEncoderParams, max_backward_limit uint, starting_dist_cache []int, num_matches uint, matches []BackwardMatch, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) uint {
func updateNodes(num_bytes uint, block_start uint, pos uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, max_backward_limit uint, starting_dist_cache []int, num_matches uint, matches []backwardMatch, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) uint {
var cur_ix uint = block_start + pos
var cur_ix_masked uint = cur_ix & ringbuffer_mask
var max_distance uint = brotli_min_size_t(cur_ix, max_backward_limit)
@ -439,7 +439,7 @@ func updateNodes(num_bytes uint, block_start uint, pos uint, ringbuffer []byte,
continue
}
len = FindMatchLengthWithLimit(ringbuffer[prev_ix:], ringbuffer[cur_ix_masked:], max_len)
len = findMatchLengthWithLimit(ringbuffer[prev_ix:], ringbuffer[cur_ix_masked:], max_len)
} else {
continue
}
@ -476,7 +476,7 @@ func updateNodes(num_bytes uint, block_start uint, pos uint, ringbuffer []byte,
/* Loop through all possible copy lengths at this position. */
var len uint = min_len
for j = 0; j < num_matches; j++ {
var match BackwardMatch = matches[j]
var match backwardMatch = matches[j]
var dist uint = uint(match.distance)
var is_dictionary_match bool = (dist > max_distance+gap)
var dist_code uint = dist + numDistanceShortCodes - 1
@ -495,7 +495,7 @@ func updateNodes(num_bytes uint, block_start uint, pos uint, ringbuffer []byte,
/* Try all copy lengths up until the maximum copy length corresponding
to this distance. If the distance refers to the static dictionary, or
the maximum length is long enough, try only one maximum length. */
max_match_len = BackwardMatchLength(&match)
max_match_len = backwardMatchLength(&match)
if len < max_match_len && (is_dictionary_match || max_match_len > max_zopfli_len) {
len = max_match_len
@ -504,7 +504,7 @@ func updateNodes(num_bytes uint, block_start uint, pos uint, ringbuffer []byte,
for ; len <= max_match_len; len++ {
var len_code uint
if is_dictionary_match {
len_code = BackwardMatchLengthCode(&match)
len_code = backwardMatchLengthCode(&match)
} else {
len_code = len
}
@ -541,8 +541,8 @@ func computeShortestPathFromNodes(num_bytes uint, nodes []zopfliNode) uint {
}
/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *BrotliEncoderParams, commands []command, num_literals *uint) {
var max_backward_limit uint = BROTLI_MAX_BACKWARD_LIMIT(params.lgwin)
func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands []command, num_literals *uint) {
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
var pos uint = 0
var offset uint32 = nodes[0].u.next
var i uint
@ -580,8 +580,8 @@ func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode,
*last_insert_len += num_bytes - pos
}
func zopfliIterate(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *BrotliEncoderParams, gap uint, dist_cache []int, model *zopfliCostModel, num_matches []uint32, matches []BackwardMatch, nodes []zopfliNode) uint {
var max_backward_limit uint = BROTLI_MAX_BACKWARD_LIMIT(params.lgwin)
func zopfliIterate(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, gap uint, dist_cache []int, model *zopfliCostModel, num_matches []uint32, matches []backwardMatch, nodes []zopfliNode) uint {
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
var max_zopfli_len uint = MaxZopfliLen(params)
var queue startPosQueue
var cur_match_pos uint = 0
@ -595,8 +595,8 @@ func zopfliIterate(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_
skip = 0
}
cur_match_pos += uint(num_matches[i])
if num_matches[i] == 1 && BackwardMatchLength(&matches[cur_match_pos-1]) > max_zopfli_len {
skip = brotli_max_size_t(BackwardMatchLength(&matches[cur_match_pos-1]), skip)
if num_matches[i] == 1 && backwardMatchLength(&matches[cur_match_pos-1]) > max_zopfli_len {
skip = brotli_max_size_t(backwardMatchLength(&matches[cur_match_pos-1]), skip)
}
if skip > 1 {
@ -617,12 +617,12 @@ func zopfliIterate(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_
}
/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *BrotliEncoderParams, dist_cache []int, hasher *H10, nodes []zopfliNode) uint {
var max_backward_limit uint = BROTLI_MAX_BACKWARD_LIMIT(params.lgwin)
func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, dist_cache []int, hasher *h10, nodes []zopfliNode) uint {
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
var max_zopfli_len uint = MaxZopfliLen(params)
var model zopfliCostModel
var queue startPosQueue
var matches [2 * (MAX_NUM_MATCHES_H10 + 64)]BackwardMatch
var matches [2 * (maxNumMatchesH10 + 64)]backwardMatch
var store_end uint
if num_bytes >= hasher.StoreLookahead() {
store_end = position + num_bytes - hasher.StoreLookahead() + 1
@ -642,8 +642,8 @@ func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte,
var max_distance uint = brotli_min_size_t(pos, max_backward_limit)
var skip uint
var num_matches uint
num_matches = FindAllMatchesH10(hasher, &params.dictionary, ringbuffer, ringbuffer_mask, pos, num_bytes-i, max_distance, gap, params, matches[lz_matches_offset:])
if num_matches > 0 && BackwardMatchLength(&matches[num_matches-1]) > max_zopfli_len {
num_matches = findAllMatchesH10(hasher, &params.dictionary, ringbuffer, ringbuffer_mask, pos, num_bytes-i, max_distance, gap, params, matches[lz_matches_offset:])
if num_matches > 0 && backwardMatchLength(&matches[num_matches-1]) > max_zopfli_len {
matches[0] = matches[num_matches-1]
num_matches = 1
}
@ -652,8 +652,8 @@ func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte,
if skip < BROTLI_LONG_COPY_QUICK_STEP {
skip = 0
}
if num_matches == 1 && BackwardMatchLength(&matches[0]) > max_zopfli_len {
skip = brotli_max_size_t(BackwardMatchLength(&matches[0]), skip)
if num_matches == 1 && backwardMatchLength(&matches[0]) > max_zopfli_len {
skip = brotli_max_size_t(backwardMatchLength(&matches[0]), skip)
}
if skip > 1 {
@ -676,7 +676,7 @@ func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte,
return computeShortestPathFromNodes(num_bytes, nodes)
}
func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *BrotliEncoderParams, hasher *H10, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
var nodes []zopfliNode
nodes = make([]zopfliNode, (num_bytes + 1))
initZopfliNodes(nodes, num_bytes+1)
@ -685,8 +685,8 @@ func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []
nodes = nil
}
func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *BrotliEncoderParams, hasher HasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
var max_backward_limit uint = BROTLI_MAX_BACKWARD_LIMIT(params.lgwin)
func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
var num_matches []uint32 = make([]uint32, num_bytes)
var matches_size uint = 4 * num_bytes
var store_end uint
@ -703,10 +703,10 @@ func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer
var orig_num_commands uint
var model zopfliCostModel
var nodes []zopfliNode
var matches []BackwardMatch = make([]BackwardMatch, matches_size)
var matches []backwardMatch = make([]backwardMatch, matches_size)
var gap uint = 0
var shadow_matches uint = 0
var new_array []BackwardMatch
var new_array []backwardMatch
for i = 0; i+hasher.HashTypeLength()-1 < num_bytes; i++ {
var pos uint = position + i
var max_distance uint = brotli_min_size_t(pos, max_backward_limit)
@ -716,17 +716,17 @@ func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer
var j uint
/* Ensure that we have enough free slots. */
if matches_size < cur_match_pos+MAX_NUM_MATCHES_H10+shadow_matches {
if matches_size < cur_match_pos+maxNumMatchesH10+shadow_matches {
var new_size uint = matches_size
if new_size == 0 {
new_size = cur_match_pos + MAX_NUM_MATCHES_H10 + shadow_matches
new_size = cur_match_pos + maxNumMatchesH10 + shadow_matches
}
for new_size < cur_match_pos+MAX_NUM_MATCHES_H10+shadow_matches {
for new_size < cur_match_pos+maxNumMatchesH10+shadow_matches {
new_size *= 2
}
new_array = make([]BackwardMatch, new_size)
new_array = make([]backwardMatch, new_size)
if matches_size != 0 {
copy(new_array, matches[:matches_size])
}
@ -735,15 +735,15 @@ func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer
matches_size = new_size
}
num_found_matches = FindAllMatchesH10(hasher, &params.dictionary, ringbuffer, ringbuffer_mask, pos, max_length, max_distance, gap, params, matches[cur_match_pos+shadow_matches:])
num_found_matches = findAllMatchesH10(hasher.(*h10), &params.dictionary, ringbuffer, ringbuffer_mask, pos, max_length, max_distance, gap, params, matches[cur_match_pos+shadow_matches:])
cur_match_end = cur_match_pos + num_found_matches
for j = cur_match_pos; j+1 < cur_match_end; j++ {
assert(BackwardMatchLength(&matches[j]) <= BackwardMatchLength(&matches[j+1]))
assert(backwardMatchLength(&matches[j]) <= backwardMatchLength(&matches[j+1]))
}
num_matches[i] = uint32(num_found_matches)
if num_found_matches > 0 {
var match_len uint = BackwardMatchLength(&matches[cur_match_end-1])
var match_len uint = backwardMatchLength(&matches[cur_match_end-1])
if match_len > MAX_ZOPFLI_LEN_QUALITY_11 {
var skip uint = match_len - 1
matches[cur_match_pos] = matches[cur_match_end-1]

28
bit_cost.go
View File

@ -23,11 +23,11 @@ func shannonEntropy(population []uint32, size uint, total *uint) float64 {
p = uint(population[0])
population = population[1:]
sum += p
retval -= float64(p) * FastLog2(p)
retval -= float64(p) * fastLog2(p)
}
if sum != 0 {
retval += float64(sum) * FastLog2(sum)
retval += float64(sum) * fastLog2(sum)
}
*total = sum
return retval
@ -49,8 +49,8 @@ const kTwoSymbolHistogramCost float64 = 20
const kThreeSymbolHistogramCost float64 = 28
const kFourSymbolHistogramCost float64 = 37
func populationCostLiteral(histogram *HistogramLiteral) float64 {
var data_size uint = HistogramDataSizeLiteral()
func populationCostLiteral(histogram *histogramLiteral) float64 {
var data_size uint = histogramDataSizeLiteral()
var count int = 0
var s [5]uint
var bits float64 = 0.0
@ -116,10 +116,10 @@ func populationCostLiteral(histogram *HistogramLiteral) float64 {
build a simplified histogram of the code length codes where we use the
zero repeat code 17, but we don't use the non-zero repeat code 16. */
var log2total float64 = FastLog2(histogram.total_count_)
var log2total float64 = fastLog2(histogram.total_count_)
for i = 0; i < data_size; {
if histogram.data_[i] > 0 {
var log2p float64 = log2total - FastLog2(uint(histogram.data_[i]))
var log2p float64 = log2total - fastLog2(uint(histogram.data_[i]))
/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
= log2(total_count) - log2(count(symbol)) */
@ -180,8 +180,8 @@ func populationCostLiteral(histogram *HistogramLiteral) float64 {
return bits
}
func populationCostCommand(histogram *HistogramCommand) float64 {
var data_size uint = HistogramDataSizeCommand()
func populationCostCommand(histogram *histogramCommand) float64 {
var data_size uint = histogramDataSizeCommand()
var count int = 0
var s [5]uint
var bits float64 = 0.0
@ -247,10 +247,10 @@ func populationCostCommand(histogram *HistogramCommand) float64 {
build a simplified histogram of the code length codes where we use the
zero repeat code 17, but we don't use the non-zero repeat code 16. */
var log2total float64 = FastLog2(histogram.total_count_)
var log2total float64 = fastLog2(histogram.total_count_)
for i = 0; i < data_size; {
if histogram.data_[i] > 0 {
var log2p float64 = log2total - FastLog2(uint(histogram.data_[i]))
var log2p float64 = log2total - fastLog2(uint(histogram.data_[i]))
/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
= log2(total_count) - log2(count(symbol)) */
@ -311,8 +311,8 @@ func populationCostCommand(histogram *HistogramCommand) float64 {
return bits
}
func populationCostDistance(histogram *HistogramDistance) float64 {
var data_size uint = HistogramDataSizeDistance()
func populationCostDistance(histogram *histogramDistance) float64 {
var data_size uint = histogramDataSizeDistance()
var count int = 0
var s [5]uint
var bits float64 = 0.0
@ -378,10 +378,10 @@ func populationCostDistance(histogram *HistogramDistance) float64 {
build a simplified histogram of the code length codes where we use the
zero repeat code 17, but we don't use the non-zero repeat code 16. */
var log2total float64 = FastLog2(histogram.total_count_)
var log2total float64 = fastLog2(histogram.total_count_)
for i = 0; i < data_size; {
if histogram.data_[i] > 0 {
var log2p float64 = log2total - FastLog2(uint(histogram.data_[i]))
var log2p float64 = log2total - fastLog2(uint(histogram.data_[i]))
/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) =
= log2(total_count) - log2(count(symbol)) */

4
block_splitter.go
View File

@ -102,7 +102,7 @@ func bitCost(count uint) float64 {
if count == 0 {
return -2.0
} else {
return FastLog2(count)
return fastLog2(count)
}
}
@ -124,7 +124,7 @@ func destroyBlockSplit(self *blockSplit) {
self.lengths = nil
}
func splitBlock(cmds []command, num_commands uint, data []byte, pos uint, mask uint, params *BrotliEncoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) {
func splitBlock(cmds []command, num_commands uint, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) {
{
var literals_count uint = countLiterals(cmds, num_commands)
var literals []byte = make([]byte, literals_count)

56
block_splitter_command.go
View File

@ -6,11 +6,11 @@ package brotli
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
func initialEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []HistogramCommand) {
func initialEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) {
var seed uint32 = 7
var block_length uint = length / num_histograms
var i uint
ClearHistogramsCommand(histograms, num_histograms)
clearHistogramsCommand(histograms, num_histograms)
for i = 0; i < num_histograms; i++ {
var pos uint = length * i / num_histograms
if i != 0 {
@ -21,11 +21,11 @@ func initialEntropyCodesCommand(data []uint16, length uint, stride uint, num_his
pos = length - stride - 1
}
HistogramAddVectorCommand(&histograms[i], data[pos:], stride)
histogramAddVectorCommand(&histograms[i], data[pos:], stride)
}
}
func randomSampleCommand(seed *uint32, data []uint16, length uint, stride uint, sample *HistogramCommand) {
func randomSampleCommand(seed *uint32, data []uint16, length uint, stride uint, sample *histogramCommand) {
var pos uint = 0
if stride >= length {
stride = length
@ -33,27 +33,27 @@ func randomSampleCommand(seed *uint32, data []uint16, length uint, stride uint,
pos = uint(myRand(seed) % uint32(length-stride+1))
}
HistogramAddVectorCommand(sample, data[pos:], stride)
histogramAddVectorCommand(sample, data[pos:], stride)
}
func refineEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []HistogramCommand) {
func refineEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) {
var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining
var seed uint32 = 7
var iter uint
iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms
for iter = 0; iter < iters; iter++ {
var sample HistogramCommand
HistogramClearCommand(&sample)
var sample histogramCommand
histogramClearCommand(&sample)
randomSampleCommand(&seed, data, length, stride, &sample)
HistogramAddHistogramCommand(&histograms[iter%num_histograms], &sample)
histogramAddHistogramCommand(&histograms[iter%num_histograms], &sample)
}
}
/* Assigns a block id from the range [0, num_histograms) to each data element
in data[0..length) and fills in block_id[0..length) with the assigned values.
Returns the number of blocks, i.e. one plus the number of block switches. */
func findBlocksCommand(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []HistogramCommand, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint {
var data_size uint = HistogramDataSizeCommand()
func findBlocksCommand(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramCommand, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint {
var data_size uint = histogramDataSizeCommand()
var bitmaplen uint = (num_histograms + 7) >> 3
var num_blocks uint = 1
var i uint
@ -71,7 +71,7 @@ func findBlocksCommand(data []uint16, length uint, block_switch_bitcost float64,
insert_cost[i] = 0
}
for i = 0; i < num_histograms; i++ {
insert_cost[i] = FastLog2(uint(uint32(histograms[i].total_count_)))
insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_)))
}
for i = data_size; i != 0; {
@ -176,11 +176,11 @@ func remapBlockIdsCommand(block_ids []byte, length uint, new_id []uint16, num_hi
return uint(next_id)
}
func buildBlockHistogramsCommand(data []uint16, length uint, block_ids []byte, num_histograms uint, histograms []HistogramCommand) {
func buildBlockHistogramsCommand(data []uint16, length uint, block_ids []byte, num_histograms uint, histograms []histogramCommand) {
var i uint
ClearHistogramsCommand(histograms, num_histograms)
clearHistogramsCommand(histograms, num_histograms)
for i = 0; i < length; i++ {
HistogramAddCommand(&histograms[block_ids[i]], uint(data[i]))
histogramAddCommand(&histograms[block_ids[i]], uint(data[i]))
}
}
@ -192,12 +192,12 @@ func clusterBlocksCommand(data []uint16, length uint, num_blocks uint, block_ids
var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch
var all_histograms_size uint = 0
var all_histograms_capacity uint = expected_num_clusters
var all_histograms []HistogramCommand = make([]HistogramCommand, all_histograms_capacity)
var all_histograms []histogramCommand = make([]histogramCommand, all_histograms_capacity)
var cluster_size_size uint = 0
var cluster_size_capacity uint = expected_num_clusters
var cluster_size []uint32 = make([]uint32, cluster_size_capacity)
var num_clusters uint = 0
var histograms []HistogramCommand = make([]HistogramCommand, brotli_min_size_t(num_blocks, histogramsPerBatch))
var histograms []histogramCommand = make([]histogramCommand, brotli_min_size_t(num_blocks, histogramsPerBatch))
var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2
var pairs_capacity uint = max_num_pairs + 1
var pairs []histogramPair = make([]histogramPair, pairs_capacity)
@ -233,9 +233,9 @@ func clusterBlocksCommand(data []uint16, length uint, num_blocks uint, block_ids
var j uint
for j = 0; j < num_to_combine; j++ {
var k uint
HistogramClearCommand(&histograms[j])
histogramClearCommand(&histograms[j])
for k = 0; uint32(k) < block_lengths[i+j]; k++ {
HistogramAddCommand(&histograms[j], uint(data[pos]))
histogramAddCommand(&histograms[j], uint(data[pos]))
pos++
}
@ -253,11 +253,11 @@ func clusterBlocksCommand(data []uint16, length uint, num_blocks uint, block_ids
} else {
_new_size = all_histograms_capacity
}
var new_array []HistogramCommand
var new_array []histogramCommand
for _new_size < (all_histograms_size + num_new_clusters) {
_new_size *= 2
}
new_array = make([]HistogramCommand, _new_size)
new_array = make([]histogramCommand, _new_size)
if all_histograms_capacity != 0 {
copy(new_array, all_histograms[:all_histograms_capacity])
}
@ -309,13 +309,13 @@ func clusterBlocksCommand(data []uint16, length uint, num_blocks uint, block_ids
{
var next_index uint32 = 0
for i = 0; i < num_blocks; i++ {
var histo HistogramCommand
var histo histogramCommand
var j uint
var best_out uint32
var best_bits float64
HistogramClearCommand(&histo)
histogramClearCommand(&histo)
for j = 0; uint32(j) < block_lengths[i]; j++ {
HistogramAddCommand(&histo, uint(data[pos]))
histogramAddCommand(&histo, uint(data[pos]))
pos++
}
@ -370,10 +370,10 @@ func clusterBlocksCommand(data []uint16, length uint, num_blocks uint, block_ids
histogram_symbols = nil
}
func splitByteVectorCommand(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *BrotliEncoderParams, split *blockSplit) {
var data_size uint = HistogramDataSizeCommand()
func splitByteVectorCommand(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
var data_size uint = histogramDataSizeCommand()
var num_histograms uint = length/literals_per_histogram + 1
var histograms []HistogramCommand
var histograms []histogramCommand
if num_histograms > max_histograms {
num_histograms = max_histograms
}
@ -391,7 +391,7 @@ func splitByteVectorCommand(data []uint16, length uint, literals_per_histogram u
return
}
histograms = make([]HistogramCommand, num_histograms)
histograms = make([]histogramCommand, num_histograms)
/* Find good entropy codes. */
initialEntropyCodesCommand(data, length, sampling_stride_length, num_histograms, histograms)

56
block_splitter_distance.go
View File

@ -6,11 +6,11 @@ package brotli
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
func initialEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []HistogramDistance) {
func initialEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) {
var seed uint32 = 7
var block_length uint = length / num_histograms
var i uint
ClearHistogramsDistance(histograms, num_histograms)
clearHistogramsDistance(histograms, num_histograms)
for i = 0; i < num_histograms; i++ {
var pos uint = length * i / num_histograms
if i != 0 {
@ -21,11 +21,11 @@ func initialEntropyCodesDistance(data []uint16, length uint, stride uint, num_hi
pos = length - stride - 1
}
HistogramAddVectorDistance(&histograms[i], data[pos:], stride)
histogramAddVectorDistance(&histograms[i], data[pos:], stride)
}
}
func randomSampleDistance(seed *uint32, data []uint16, length uint, stride uint, sample *HistogramDistance) {
func randomSampleDistance(seed *uint32, data []uint16, length uint, stride uint, sample *histogramDistance) {
var pos uint = 0
if stride >= length {
stride = length
@ -33,27 +33,27 @@ func randomSampleDistance(seed *uint32, data []uint16, length uint, stride uint,
pos = uint(myRand(seed) % uint32(length-stride+1))
}
HistogramAddVectorDistance(sample, data[pos:], stride)
histogramAddVectorDistance(sample, data[pos:], stride)
}
func refineEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []HistogramDistance) {
func refineEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) {
var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining
var seed uint32 = 7
var iter uint
iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms
for iter = 0; iter < iters; iter++ {
var sample HistogramDistance
HistogramClearDistance(&sample)
var sample histogramDistance
histogramClearDistance(&sample)
randomSampleDistance(&seed, data, length, stride, &sample)
HistogramAddHistogramDistance(&histograms[iter%num_histograms], &sample)
histogramAddHistogramDistance(&histograms[iter%num_histograms], &sample)
}
}
/* Assigns a block id from the range [0, num_histograms) to each data element
in data[0..length) and fills in block_id[0..length) with the assigned values.
Returns the number of blocks, i.e. one plus the number of block switches. */
func findBlocksDistance(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []HistogramDistance, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint {
var data_size uint = HistogramDataSizeDistance()
func findBlocksDistance(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramDistance, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint {
var data_size uint = histogramDataSizeDistance()
var bitmaplen uint = (num_histograms + 7) >> 3
var num_blocks uint = 1
var i uint
@ -71,7 +71,7 @@ func findBlocksDistance(data []uint16, length uint, block_switch_bitcost float64
insert_cost[i] = 0
}
for i = 0; i < num_histograms; i++ {
insert_cost[i] = FastLog2(uint(uint32(histograms[i].total_count_)))
insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_)))
}
for i = data_size; i != 0; {
@ -176,11 +176,11 @@ func remapBlockIdsDistance(block_ids []byte, length uint, new_id []uint16, num_h
return uint(next_id)
}
func buildBlockHistogramsDistance(data []uint16, length uint, block_ids []byte, num_histograms uint, histograms []HistogramDistance) {
func buildBlockHistogramsDistance(data []uint16, length uint, block_ids []byte, num_histograms uint, histograms []histogramDistance) {
var i uint
ClearHistogramsDistance(histograms, num_histograms)
clearHistogramsDistance(histograms, num_histograms)
for i = 0; i < length; i++ {
HistogramAddDistance(&histograms[block_ids[i]], uint(data[i]))
histogramAddDistance(&histograms[block_ids[i]], uint(data[i]))
}
}
@ -192,12 +192,12 @@ func clusterBlocksDistance(data []uint16, length uint, num_blocks uint, block_id
var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch
var all_histograms_size uint = 0
var all_histograms_capacity uint = expected_num_clusters
var all_histograms []HistogramDistance = make([]HistogramDistance, all_histograms_capacity)
var all_histograms []histogramDistance = make([]histogramDistance, all_histograms_capacity)
var cluster_size_size uint = 0
var cluster_size_capacity uint = expected_num_clusters
var cluster_size []uint32 = make([]uint32, cluster_size_capacity)
var num_clusters uint = 0
var histograms []HistogramDistance = make([]HistogramDistance, brotli_min_size_t(num_blocks, histogramsPerBatch))
var histograms []histogramDistance = make([]histogramDistance, brotli_min_size_t(num_blocks, histogramsPerBatch))
var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2
var pairs_capacity uint = max_num_pairs + 1
var pairs []histogramPair = make([]histogramPair, pairs_capacity)
@ -233,9 +233,9 @@ func clusterBlocksDistance(data []uint16, length uint, num_blocks uint, block_id
var j uint
for j = 0; j < num_to_combine; j++ {
var k uint
HistogramClearDistance(&histograms[j])
histogramClearDistance(&histograms[j])
for k = 0; uint32(k) < block_lengths[i+j]; k++ {
HistogramAddDistance(&histograms[j], uint(data[pos]))
histogramAddDistance(&histograms[j], uint(data[pos]))
pos++
}
@ -253,11 +253,11 @@ func clusterBlocksDistance(data []uint16, length uint, num_blocks uint, block_id
} else {
_new_size = all_histograms_capacity
}
var new_array []HistogramDistance
var new_array []histogramDistance
for _new_size < (all_histograms_size + num_new_clusters) {
_new_size *= 2
}
new_array = make([]HistogramDistance, _new_size)
new_array = make([]histogramDistance, _new_size)
if all_histograms_capacity != 0 {
copy(new_array, all_histograms[:all_histograms_capacity])
}
@ -309,13 +309,13 @@ func clusterBlocksDistance(data []uint16, length uint, num_blocks uint, block_id
{
var next_index uint32 = 0
for i = 0; i < num_blocks; i++ {
var histo HistogramDistance
var histo histogramDistance
var j uint
var best_out uint32
var best_bits float64
HistogramClearDistance(&histo)
histogramClearDistance(&histo)
for j = 0; uint32(j) < block_lengths[i]; j++ {
HistogramAddDistance(&histo, uint(data[pos]))
histogramAddDistance(&histo, uint(data[pos]))
pos++
}
@ -370,10 +370,10 @@ func clusterBlocksDistance(data []uint16, length uint, num_blocks uint, block_id
histogram_symbols = nil
}
func splitByteVectorDistance(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *BrotliEncoderParams, split *blockSplit) {
var data_size uint = HistogramDataSizeDistance()
func splitByteVectorDistance(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
var data_size uint = histogramDataSizeDistance()
var num_histograms uint = length/literals_per_histogram + 1
var histograms []HistogramDistance
var histograms []histogramDistance
if num_histograms > max_histograms {
num_histograms = max_histograms
}
@ -391,7 +391,7 @@ func splitByteVectorDistance(data []uint16, length uint, literals_per_histogram
return
}
histograms = make([]HistogramDistance, num_histograms)
histograms = make([]histogramDistance, num_histograms)
/* Find good entropy codes. */
initialEntropyCodesDistance(data, length, sampling_stride_length, num_histograms, histograms)

56
block_splitter_literal.go
View File

@ -6,11 +6,11 @@ package brotli
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
func initialEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []HistogramLiteral) {
func initialEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) {
var seed uint32 = 7
var block_length uint = length / num_histograms
var i uint
ClearHistogramsLiteral(histograms, num_histograms)
clearHistogramsLiteral(histograms, num_histograms)
for i = 0; i < num_histograms; i++ {
var pos uint = length * i / num_histograms
if i != 0 {
@ -21,11 +21,11 @@ func initialEntropyCodesLiteral(data []byte, length uint, stride uint, num_histo
pos = length - stride - 1
}
HistogramAddVectorLiteral(&histograms[i], data[pos:], stride)
histogramAddVectorLiteral(&histograms[i], data[pos:], stride)
}
}
func randomSampleLiteral(seed *uint32, data []byte, length uint, stride uint, sample *HistogramLiteral) {
func randomSampleLiteral(seed *uint32, data []byte, length uint, stride uint, sample *histogramLiteral) {
var pos uint = 0
if stride >= length {
stride = length
@ -33,27 +33,27 @@ func randomSampleLiteral(seed *uint32, data []byte, length uint, stride uint, sa
pos = uint(myRand(seed) % uint32(length-stride+1))
}
HistogramAddVectorLiteral(sample, data[pos:], stride)
histogramAddVectorLiteral(sample, data[pos:], stride)
}
func refineEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []HistogramLiteral) {
func refineEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) {
var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining
var seed uint32 = 7
var iter uint
iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms
for iter = 0; iter < iters; iter++ {
var sample HistogramLiteral
HistogramClearLiteral(&sample)
var sample histogramLiteral
histogramClearLiteral(&sample)
randomSampleLiteral(&seed, data, length, stride, &sample)
HistogramAddHistogramLiteral(&histograms[iter%num_histograms], &sample)
histogramAddHistogramLiteral(&histograms[iter%num_histograms], &sample)
}
}
/* Assigns a block id from the range [0, num_histograms) to each data element
in data[0..length) and fills in block_id[0..length) with the assigned values.
Returns the number of blocks, i.e. one plus the number of block switches. */
func findBlocksLiteral(data []byte, length uint, block_switch_bitcost float64, num_histograms uint, histograms []HistogramLiteral, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint {
var data_size uint = HistogramDataSizeLiteral()
func findBlocksLiteral(data []byte, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramLiteral, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint {
var data_size uint = histogramDataSizeLiteral()
var bitmaplen uint = (num_histograms + 7) >> 3
var num_blocks uint = 1
var i uint
@ -71,7 +71,7 @@ func findBlocksLiteral(data []byte, length uint, block_switch_bitcost float64, n
insert_cost[i] = 0
}
for i = 0; i < num_histograms; i++ {
insert_cost[i] = FastLog2(uint(uint32(histograms[i].total_count_)))
insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_)))
}
for i = data_size; i != 0; {
@ -176,11 +176,11 @@ func remapBlockIdsLiteral(block_ids []byte, length uint, new_id []uint16, num_hi
return uint(next_id)
}
func buildBlockHistogramsLiteral(data []byte, length uint, block_ids []byte, num_histograms uint, histograms []HistogramLiteral) {
func buildBlockHistogramsLiteral(data []byte, length uint, block_ids []byte, num_histograms uint, histograms []histogramLiteral) {
var i uint
ClearHistogramsLiteral(histograms, num_histograms)
clearHistogramsLiteral(histograms, num_histograms)
for i = 0; i < length; i++ {
HistogramAddLiteral(&histograms[block_ids[i]], uint(data[i]))
histogramAddLiteral(&histograms[block_ids[i]], uint(data[i]))
}
}
@ -192,12 +192,12 @@ func clusterBlocksLiteral(data []byte, length uint, num_blocks uint, block_ids [
var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch
var all_histograms_size uint = 0
var all_histograms_capacity uint = expected_num_clusters
var all_histograms []HistogramLiteral = make([]HistogramLiteral, all_histograms_capacity)
var all_histograms []histogramLiteral = make([]histogramLiteral, all_histograms_capacity)
var cluster_size_size uint = 0
var cluster_size_capacity uint = expected_num_clusters
var cluster_size []uint32 = make([]uint32, cluster_size_capacity)
var num_clusters uint = 0
var histograms []HistogramLiteral = make([]HistogramLiteral, brotli_min_size_t(num_blocks, histogramsPerBatch))
var histograms []histogramLiteral = make([]histogramLiteral, brotli_min_size_t(num_blocks, histogramsPerBatch))
var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2
var pairs_capacity uint = max_num_pairs + 1
var pairs []histogramPair = make([]histogramPair, pairs_capacity)
@ -233,9 +233,9 @@ func clusterBlocksLiteral(data []byte, length uint, num_blocks uint, block_ids [
var j uint
for j = 0; j < num_to_combine; j++ {
var k uint
HistogramClearLiteral(&histograms[j])
histogramClearLiteral(&histograms[j])
for k = 0; uint32(k) < block_lengths[i+j]; k++ {
HistogramAddLiteral(&histograms[j], uint(data[pos]))
histogramAddLiteral(&histograms[j], uint(data[pos]))
pos++
}
@ -253,11 +253,11 @@ func clusterBlocksLiteral(data []byte, length uint, num_blocks uint, block_ids [
} else {
_new_size = all_histograms_capacity
}
var new_array []HistogramLiteral
var new_array []histogramLiteral
for _new_size < (all_histograms_size + num_new_clusters) {
_new_size *= 2
}
new_array = make([]HistogramLiteral, _new_size)
new_array = make([]histogramLiteral, _new_size)
if all_histograms_capacity != 0 {
copy(new_array, all_histograms[:all_histograms_capacity])
}
@ -309,13 +309,13 @@ func clusterBlocksLiteral(data []byte, length uint, num_blocks uint, block_ids [
{
var next_index uint32 = 0
for i = 0; i < num_blocks; i++ {
var histo HistogramLiteral
var histo histogramLiteral
var j uint
var best_out uint32
var best_bits float64
HistogramClearLiteral(&histo)
histogramClearLiteral(&histo)
for j = 0; uint32(j) < block_lengths[i]; j++ {
HistogramAddLiteral(&histo, uint(data[pos]))
histogramAddLiteral(&histo, uint(data[pos]))
pos++
}
@ -370,10 +370,10 @@ func clusterBlocksLiteral(data []byte, length uint, num_blocks uint, block_ids [
histogram_symbols = nil
}
func splitByteVectorLiteral(data []byte, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *BrotliEncoderParams, split *blockSplit) {
var data_size uint = HistogramDataSizeLiteral()
func splitByteVectorLiteral(data []byte, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
var data_size uint = histogramDataSizeLiteral()
var num_histograms uint = length/literals_per_histogram + 1
var histograms []HistogramLiteral
var histograms []histogramLiteral
if num_histograms > max_histograms {
num_histograms = max_histograms
}
@ -391,7 +391,7 @@ func splitByteVectorLiteral(data []byte, length uint, literals_per_histogram uin
return
}
histograms = make([]HistogramLiteral, num_histograms)
histograms = make([]histogramLiteral, num_histograms)
/* Find good entropy codes. */
initialEntropyCodesLiteral(data, length, sampling_stride_length, num_histograms, histograms)

124
brotli_bit_stream.go
View File

@ -99,7 +99,7 @@ func encodeMlen(length uint, bits *uint64, numbits *uint, nibblesbits *uint64) {
if length == 1 {
lg = 1
} else {
lg = uint(Log2FloorNonZero(uint(uint32(length-1)))) + 1
lg = uint(log2FloorNonZero(uint(uint32(length-1)))) + 1
}
var tmp uint
if lg < 16 {
@ -142,7 +142,7 @@ func storeVarLenUint8(n uint, storage_ix *uint, storage []byte) {
if n == 0 {
BrotliWriteBits(1, 0, storage_ix, storage)
} else {
var nbits uint = uint(Log2FloorNonZero(n))
var nbits uint = uint(log2FloorNonZero(n))
BrotliWriteBits(1, 1, storage_ix, storage)
BrotliWriteBits(3, uint64(nbits), storage_ix, storage)
BrotliWriteBits(nbits, uint64(n)-(uint64(uint(1))<<nbits), storage_ix, storage)
@ -309,7 +309,7 @@ func storeSimpleHuffmanTree(depths []byte, symbols []uint, num_symbols uint, max
/* num = alphabet size
depths = symbol depths */
func storeHuffmanTree(depths []byte, num uint, tree []HuffmanTree, storage_ix *uint, storage []byte) {
func storeHuffmanTree(depths []byte, num uint, tree []huffmanTree, storage_ix *uint, storage []byte) {
var huffman_tree [numCommandSymbols]byte
var huffman_tree_extra_bits [numCommandSymbols]byte
var huffman_tree_size uint = 0
@ -326,7 +326,7 @@ func storeHuffmanTree(depths []byte, num uint, tree []HuffmanTree, storage_ix *u
assert(num <= numCommandSymbols)
BrotliWriteHuffmanTree(depths, num, &huffman_tree_size, huffman_tree[:], huffman_tree_extra_bits[:])
writeHuffmanTree(depths, num, &huffman_tree_size, huffman_tree[:], huffman_tree_extra_bits[:])
/* Calculate the statistics of the Huffman tree in brotli-representation. */
for i = 0; i < huffman_tree_size; i++ {
@ -347,9 +347,9 @@ func storeHuffmanTree(depths []byte, num uint, tree []HuffmanTree, storage_ix *u
/* Calculate another Huffman tree to use for compressing both the
earlier Huffman tree with. */
BrotliCreateHuffmanTree(huffman_tree_histogram[:], codeLengthCodes, 5, tree, code_length_bitdepth[:])
createHuffmanTree(huffman_tree_histogram[:], codeLengthCodes, 5, tree, code_length_bitdepth[:])
BrotliConvertBitDepthsToSymbols(code_length_bitdepth[:], codeLengthCodes, code_length_bitdepth_symbols[:])
convertBitDepthsToSymbols(code_length_bitdepth[:], codeLengthCodes, code_length_bitdepth_symbols[:])
/* Now, we have all the data, let's start storing it */
storeHuffmanTreeOfHuffmanTreeToBitMask(num_codes, code_length_bitdepth[:], storage_ix, storage)
@ -364,7 +364,7 @@ func storeHuffmanTree(depths []byte, num uint, tree []HuffmanTree, storage_ix *u
/* Builds a Huffman tree from histogram[0:length] into depth[0:length] and
bits[0:length] and stores the encoded tree to the bit stream. */
func buildAndStoreHuffmanTree(histogram []uint32, histogram_length uint, alphabet_size uint, tree []HuffmanTree, depth []byte, bits []uint16, storage_ix *uint, storage []byte) {
func buildAndStoreHuffmanTree(histogram []uint32, histogram_length uint, alphabet_size uint, tree []huffmanTree, depth []byte, bits []uint16, storage_ix *uint, storage []byte) {
var count uint = 0
var s4 = [4]uint{0}
var i uint
@ -399,8 +399,8 @@ func buildAndStoreHuffmanTree(histogram []uint32, histogram_length uint, alphabe
for i := 0; i < int(histogram_length); i++ {
depth[i] = 0
}
BrotliCreateHuffmanTree(histogram, histogram_length, 15, tree, depth)
BrotliConvertBitDepthsToSymbols(depth, histogram_length, bits)
createHuffmanTree(histogram, histogram_length, 15, tree, depth)
convertBitDepthsToSymbols(depth, histogram_length, bits)
if count <= 4 {
storeSimpleHuffmanTree(depth, s4[:], count, max_bits, storage_ix, storage)
@ -409,7 +409,7 @@ func buildAndStoreHuffmanTree(histogram []uint32, histogram_length uint, alphabe
}
}
func sortHuffmanTree1(v0 *HuffmanTree, v1 *HuffmanTree) bool {
func sortHuffmanTree1(v0 *huffmanTree, v1 *huffmanTree) bool {
return v0.total_count_ < v1.total_count_
}
@ -444,7 +444,7 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_
}
{
var max_tree_size uint = 2*length + 1
var tree []HuffmanTree = make([]HuffmanTree, max_tree_size)
var tree []huffmanTree = make([]huffmanTree, max_tree_size)
var count_limit uint32
for count_limit = 1; ; count_limit *= 2 {
var node int = 0
@ -453,9 +453,9 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_
l--
if histogram[l] != 0 {
if histogram[l] >= count_limit {
InitHuffmanTree(&tree[node:][0], histogram[l], -1, int16(l))
initHuffmanTree(&tree[node:][0], histogram[l], -1, int16(l))
} else {
InitHuffmanTree(&tree[node:][0], count_limit, -1, int16(l))
initHuffmanTree(&tree[node:][0], count_limit, -1, int16(l))
}
node++
@ -464,12 +464,12 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_
{
var n int = node
/* Points to the next leaf node. */ /* Points to the next non-leaf node. */
var sentinel HuffmanTree
var sentinel huffmanTree
var i int = 0
var j int = n + 1
var k int
SortHuffmanTreeItems(tree, uint(n), HuffmanTreeComparator(sortHuffmanTree1))
sortHuffmanTreeItems(tree, uint(n), huffmanTreeComparator(sortHuffmanTree1))
/* The nodes are:
[0, n): the sorted leaf nodes that we start with.
@ -478,7 +478,7 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_
(n+1). These are naturally in ascending order.
[2n]: we add a sentinel at the end as well.
There will be (2n+1) elements at the end. */
InitHuffmanTree(&sentinel, BROTLI_UINT32_MAX, -1, -1)
initHuffmanTree(&sentinel, BROTLI_UINT32_MAX, -1, -1)
tree[node] = sentinel
node++
@ -515,7 +515,7 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_
node++
}
if BrotliSetDepth(2*n-1, tree, depth, 14) {
if setDepth(2*n-1, tree, depth, 14) {
/* We need to pack the Huffman tree in 14 bits. If this was not
successful, add fake entities to the lowest values and retry. */
break
@ -526,7 +526,7 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_
tree = nil
}
BrotliConvertBitDepthsToSymbols(depth, length, bits)
convertBitDepthsToSymbols(depth, length, bits)
if count <= 4 {
var i uint
@ -683,7 +683,7 @@ func runLengthCodeZeros(in_size uint, v []uint32, out_size *uint, max_run_length
}
if max_reps > 0 {
max_prefix = Log2FloorNonZero(uint(max_reps))
max_prefix = log2FloorNonZero(uint(max_reps))
} else {
max_prefix = 0
}
@ -706,7 +706,7 @@ func runLengthCodeZeros(in_size uint, v []uint32, out_size *uint, max_run_length
i += uint(reps)
for reps != 0 {
if reps < 2<<max_prefix {
var run_length_prefix uint32 = Log2FloorNonZero(uint(reps))
var run_length_prefix uint32 = log2FloorNonZero(uint(reps))
var extra_bits uint32 = reps - (1 << run_length_prefix)
v[*out_size] = run_length_prefix + (extra_bits << 9)
(*out_size)++
@ -726,7 +726,7 @@ const symbolBits = 9
var encodeContextMap_kSymbolMask uint32 = (1 << symbolBits) - 1
func encodeContextMap(context_map []uint32, context_map_size uint, num_clusters uint, tree []HuffmanTree, storage_ix *uint, storage []byte) {
func encodeContextMap(context_map []uint32, context_map_size uint, num_clusters uint, tree []huffmanTree, storage_ix *uint, storage []byte) {
var i uint
var rle_symbols []uint32
var max_run_length_prefix uint32 = 6
@ -788,7 +788,7 @@ func storeBlockSwitch(code *blockSplitCode, block_len uint32, block_type byte, i
/* Builds a BlockSplitCode data structure from the block split given by the
vector of block types and block lengths and stores it to the bit stream. */
func buildAndStoreBlockSplitCode(types []byte, lengths []uint32, num_blocks uint, num_types uint, tree []HuffmanTree, code *blockSplitCode, storage_ix *uint, storage []byte) {
func buildAndStoreBlockSplitCode(types []byte, lengths []uint32, num_blocks uint, num_types uint, tree []huffmanTree, code *blockSplitCode, storage_ix *uint, storage []byte) {
var type_histo [maxBlockTypeSymbols]uint32
var length_histo [numBlockLenSymbols]uint32
var i uint
@ -815,7 +815,7 @@ func buildAndStoreBlockSplitCode(types []byte, lengths []uint32, num_blocks uint
}
/* Stores a context map where the histogram type is always the block type. */
func storeTrivialContextMap(num_types uint, context_bits uint, tree []HuffmanTree, storage_ix *uint, storage []byte) {
func storeTrivialContextMap(num_types uint, context_bits uint, tree []huffmanTree, storage_ix *uint, storage []byte) {
storeVarLenUint8(num_types-1, storage_ix, storage)
if num_types > 1 {
var repeat_code uint = context_bits - 1
@ -898,7 +898,7 @@ func cleanupBlockEncoder(self *blockEncoder) {
/* Creates entropy codes of block lengths and block types and stores them
to the bit stream. */
func buildAndStoreBlockSwitchEntropyCodes(self *blockEncoder, tree []HuffmanTree, storage_ix *uint, storage []byte) {
func buildAndStoreBlockSwitchEntropyCodes(self *blockEncoder, tree []huffmanTree, storage_ix *uint, storage []byte) {
buildAndStoreBlockSplitCode(self.block_types_, self.block_lengths_, self.num_blocks_, self.num_block_types_, tree, &self.block_split_code_, storage_ix, storage)
}
@ -944,7 +944,7 @@ func storeSymbolWithContext(self *blockEncoder, symbol uint, context uint, conte
}
}
func buildAndStoreEntropyCodesLiteral(self *blockEncoder, histograms []HistogramLiteral, histograms_size uint, alphabet_size uint, tree []HuffmanTree, storage_ix *uint, storage []byte) {
func buildAndStoreEntropyCodesLiteral(self *blockEncoder, histograms []histogramLiteral, histograms_size uint, alphabet_size uint, tree []huffmanTree, storage_ix *uint, storage []byte) {
var table_size uint = histograms_size * self.histogram_length_
self.depths_ = make([]byte, table_size)
self.bits_ = make([]uint16, table_size)
@ -957,7 +957,7 @@ func buildAndStoreEntropyCodesLiteral(self *blockEncoder, histograms []Histogram
}
}
func buildAndStoreEntropyCodesCommand(self *blockEncoder, histograms []HistogramCommand, histograms_size uint, alphabet_size uint, tree []HuffmanTree, storage_ix *uint, storage []byte) {
func buildAndStoreEntropyCodesCommand(self *blockEncoder, histograms []histogramCommand, histograms_size uint, alphabet_size uint, tree []huffmanTree, storage_ix *uint, storage []byte) {
var table_size uint = histograms_size * self.histogram_length_
self.depths_ = make([]byte, table_size)
self.bits_ = make([]uint16, table_size)
@ -970,7 +970,7 @@ func buildAndStoreEntropyCodesCommand(self *blockEncoder, histograms []Histogram
}
}
func buildAndStoreEntropyCodesDistance(self *blockEncoder, histograms []HistogramDistance, histograms_size uint, alphabet_size uint, tree []HuffmanTree, storage_ix *uint, storage []byte) {
func buildAndStoreEntropyCodesDistance(self *blockEncoder, histograms []histogramDistance, histograms_size uint, alphabet_size uint, tree []huffmanTree, storage_ix *uint, storage []byte) {
var table_size uint = histograms_size * self.histogram_length_
self.depths_ = make([]byte, table_size)
self.bits_ = make([]uint16, table_size)
@ -988,24 +988,24 @@ func jumpToByteBoundary(storage_ix *uint, storage []byte) {
storage[*storage_ix>>3] = 0
}
func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_byte byte, prev_byte2 byte, is_last bool, params *BrotliEncoderParams, literal_context_mode int, commands []command, n_commands uint, mb *MetaBlockSplit, storage_ix *uint, storage []byte) {
func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_byte byte, prev_byte2 byte, is_last bool, params *encoderParams, literal_context_mode int, commands []command, n_commands uint, mb *metaBlockSplit, storage_ix *uint, storage []byte) {
var pos uint = start_pos
var i uint
var num_distance_symbols uint32 = params.dist.alphabet_size
var num_effective_distance_symbols uint32 = num_distance_symbols
var tree []HuffmanTree
var literal_context_lut ContextLut = BROTLI_CONTEXT_LUT(literal_context_mode)
var tree []huffmanTree
var literal_context_lut contextLUT = getContextLUT(literal_context_mode)
var literal_enc blockEncoder
var command_enc blockEncoder
var distance_enc blockEncoder
var dist *BrotliDistanceParams = &params.dist
if params.large_window && num_effective_distance_symbols > BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS {
num_effective_distance_symbols = BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS
var dist *distanceParams = &params.dist
if params.large_window && num_effective_distance_symbols > numHistogramDistanceSymbols {
num_effective_distance_symbols = numHistogramDistanceSymbols
}
storeCompressedMetaBlockHeader(is_last, length, storage_ix, storage)
tree = make([]HuffmanTree, maxHuffmanTreeSize)
tree = make([]huffmanTree, maxHuffmanTreeSize)
initBlockEncoder(&literal_enc, numLiteralSymbols, mb.literal_split.num_types, mb.literal_split.types, mb.literal_split.lengths, mb.literal_split.num_blocks)
initBlockEncoder(&command_enc, numCommandSymbols, mb.command_split.num_types, mb.command_split.types, mb.command_split.lengths, mb.command_split.num_blocks)
initBlockEncoder(&distance_enc, uint(num_effective_distance_symbols), mb.distance_split.num_types, mb.distance_split.types, mb.distance_split.lengths, mb.distance_split.num_blocks)
@ -1021,13 +1021,13 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b
}
if mb.literal_context_map_size == 0 {
storeTrivialContextMap(mb.literal_histograms_size, BROTLI_LITERAL_CONTEXT_BITS, tree, storage_ix, storage)
storeTrivialContextMap(mb.literal_histograms_size, literalContextBits, tree, storage_ix, storage)
} else {
encodeContextMap(mb.literal_context_map, mb.literal_context_map_size, mb.literal_histograms_size, tree, storage_ix, storage)
}
if mb.distance_context_map_size == 0 {
storeTrivialContextMap(mb.distance_histograms_size, BROTLI_DISTANCE_CONTEXT_BITS, tree, storage_ix, storage)
storeTrivialContextMap(mb.distance_histograms_size, distanceContextBits, tree, storage_ix, storage)
} else {
encodeContextMap(mb.distance_context_map, mb.distance_context_map_size, mb.distance_histograms_size, tree, storage_ix, storage)
}
@ -1051,9 +1051,9 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b
} else {
var j uint
for j = uint(cmd.insert_len_); j != 0; j-- {
var context uint = uint(BROTLI_CONTEXT(prev_byte, prev_byte2, literal_context_lut))
var context uint = uint(getContext(prev_byte, prev_byte2, literal_context_lut))
var literal byte = input[pos&mask]
storeSymbolWithContext(&literal_enc, uint(literal), context, mb.literal_context_map, storage_ix, storage, BROTLI_LITERAL_CONTEXT_BITS)
storeSymbolWithContext(&literal_enc, uint(literal), context, mb.literal_context_map, storage_ix, storage, literalContextBits)
prev_byte2 = prev_byte
prev_byte = literal
pos++
@ -1072,7 +1072,7 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b
storeSymbol(&distance_enc, dist_code, storage_ix, storage)
} else {
var context uint = uint(commandDistanceContext(&cmd))
storeSymbolWithContext(&distance_enc, dist_code, context, mb.distance_context_map, storage_ix, storage, BROTLI_DISTANCE_CONTEXT_BITS)
storeSymbolWithContext(&distance_enc, dist_code, context, mb.distance_context_map, storage_ix, storage, distanceContextBits)
}
BrotliWriteBits(uint(distnumextra), distextra, storage_ix, storage)
@ -1088,21 +1088,21 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b
}
}
func buildHistograms(input []byte, start_pos uint, mask uint, commands []command, n_commands uint, lit_histo *HistogramLiteral, cmd_histo *HistogramCommand, dist_histo *HistogramDistance) {
func buildHistograms(input []byte, start_pos uint, mask uint, commands []command, n_commands uint, lit_histo *histogramLiteral, cmd_histo *histogramCommand, dist_histo *histogramDistance) {
var pos uint = start_pos
var i uint
for i = 0; i < n_commands; i++ {
var cmd command = commands[i]
var j uint
HistogramAddCommand(cmd_histo, uint(cmd.cmd_prefix_))
histogramAddCommand(cmd_histo, uint(cmd.cmd_prefix_))
for j = uint(cmd.insert_len_); j != 0; j-- {
HistogramAddLiteral(lit_histo, uint(input[pos&mask]))
histogramAddLiteral(lit_histo, uint(input[pos&mask]))
pos++
}
pos += uint(commandCopyLen(&cmd))
if commandCopyLen(&cmd) != 0 && cmd.cmd_prefix_ >= 128 {
HistogramAddDistance(dist_histo, uint(cmd.dist_prefix_)&0x3FF)
histogramAddDistance(dist_histo, uint(cmd.dist_prefix_)&0x3FF)
}
}
}
@ -1133,30 +1133,30 @@ func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands
}
}
func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *BrotliEncoderParams, commands []command, n_commands uint, storage_ix *uint, storage []byte) {
var lit_histo HistogramLiteral
var cmd_histo HistogramCommand
var dist_histo HistogramDistance
func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, n_commands uint, storage_ix *uint, storage []byte) {
var lit_histo histogramLiteral
var cmd_histo histogramCommand
var dist_histo histogramDistance
var lit_depth [numLiteralSymbols]byte
var lit_bits [numLiteralSymbols]uint16
var cmd_depth [numCommandSymbols]byte
var cmd_bits [numCommandSymbols]uint16
var dist_depth [maxSimpleDistanceAlphabetSize]byte
var dist_bits [maxSimpleDistanceAlphabetSize]uint16
var tree []HuffmanTree
var tree []huffmanTree
var num_distance_symbols uint32 = params.dist.alphabet_size
storeCompressedMetaBlockHeader(is_last, length, storage_ix, storage)
HistogramClearLiteral(&lit_histo)
HistogramClearCommand(&cmd_histo)
HistogramClearDistance(&dist_histo)
histogramClearLiteral(&lit_histo)
histogramClearCommand(&cmd_histo)
histogramClearDistance(&dist_histo)
buildHistograms(input, start_pos, mask, commands, n_commands, &lit_histo, &cmd_histo, &dist_histo)
BrotliWriteBits(13, 0, storage_ix, storage)
tree = make([]HuffmanTree, maxHuffmanTreeSize)
tree = make([]huffmanTree, maxHuffmanTreeSize)
buildAndStoreHuffmanTree(lit_histo.data_[:], numLiteralSymbols, numLiteralSymbols, tree, lit_depth[:], lit_bits[:], storage_ix, storage)
buildAndStoreHuffmanTree(cmd_histo.data_[:], numCommandSymbols, numCommandSymbols, tree, cmd_depth[:], cmd_bits[:], storage_ix, storage)
buildAndStoreHuffmanTree(dist_histo.data_[:], maxSimpleDistanceAlphabetSize, uint(num_distance_symbols), tree, dist_depth[:], dist_bits[:], storage_ix, storage)
@ -1167,9 +1167,9 @@ func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint,
}
}
func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *BrotliEncoderParams, commands []command, n_commands uint, storage_ix *uint, storage []byte) {
func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, n_commands uint, storage_ix *uint, storage []byte) {
var num_distance_symbols uint32 = params.dist.alphabet_size
var distance_alphabet_bits uint32 = Log2FloorNonZero(uint(num_distance_symbols-1)) + 1
var distance_alphabet_bits uint32 = log2FloorNonZero(uint(num_distance_symbols-1)) + 1
storeCompressedMetaBlockHeader(is_last, length, storage_ix, storage)
@ -1197,22 +1197,22 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is
buildAndStoreHuffmanTreeFast(histogram[:], num_literals, /* max_bits = */
8, lit_depth[:], lit_bits[:], storage_ix, storage)
StoreStaticCommandHuffmanTree(storage_ix, storage)
StoreStaticDistanceHuffmanTree(storage_ix, storage)
storeStaticCommandHuffmanTree(storage_ix, storage)
storeStaticDistanceHuffmanTree(storage_ix, storage)
storeDataWithHuffmanCodes(input, start_pos, mask, commands, n_commands, lit_depth[:], lit_bits[:], kStaticCommandCodeDepth[:], kStaticCommandCodeBits[:], kStaticDistanceCodeDepth[:], kStaticDistanceCodeBits[:], storage_ix, storage)
} else {
var lit_histo HistogramLiteral
var cmd_histo HistogramCommand
var dist_histo HistogramDistance
var lit_histo histogramLiteral
var cmd_histo histogramCommand
var dist_histo histogramDistance
var lit_depth [numLiteralSymbols]byte
var lit_bits [numLiteralSymbols]uint16
var cmd_depth [numCommandSymbols]byte
var cmd_bits [numCommandSymbols]uint16
var dist_depth [maxSimpleDistanceAlphabetSize]byte
var dist_bits [maxSimpleDistanceAlphabetSize]uint16
HistogramClearLiteral(&lit_histo)
HistogramClearCommand(&cmd_histo)
HistogramClearDistance(&dist_histo)
histogramClearLiteral(&lit_histo)
histogramClearCommand(&cmd_histo)
histogramClearDistance(&dist_histo)
buildHistograms(input, start_pos, mask, commands, n_commands, &lit_histo, &cmd_histo, &dist_histo)
buildAndStoreHuffmanTreeFast(lit_histo.data_[:], lit_histo.total_count_, /* max_bits = */
8, lit_depth[:], lit_bits[:], storage_ix, storage)

2
cluster.go
View File

@ -39,5 +39,5 @@ func histogramPairIsLess(p1 *histogramPair, p2 *histogramPair) bool {
/* Returns entropy reduction of the context map when we combine two clusters. */
func clusterCostDiff(size_a uint, size_b uint) float64 {
var size_c uint = size_a + size_b
return float64(size_a)*FastLog2(size_a) + float64(size_b)*FastLog2(size_b) - float64(size_c)*FastLog2(size_c)
return float64(size_a)*fastLog2(size_a) + float64(size_b)*fastLog2(size_b) - float64(size_c)*fastLog2(size_c)
}

30
cluster_command.go
View File

@ -9,7 +9,7 @@ package brotli
/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if
it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */
func compareAndPushToQueueCommand(out []HistogramCommand, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) {
func compareAndPushToQueueCommand(out []histogramCommand, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) {
var is_good_pair bool = false
var p histogramPair
p.idx2 = 0
@ -45,9 +45,9 @@ func compareAndPushToQueueCommand(out []HistogramCommand, cluster_size []uint32,
} else {
threshold = brotli_max_double(0.0, pairs[0].cost_diff)
}
var combo HistogramCommand = out[idx1]
var combo histogramCommand = out[idx1]
var cost_combo float64
HistogramAddHistogramCommand(&combo, &out[idx2])
histogramAddHistogramCommand(&combo, &out[idx2])
cost_combo = populationCostCommand(&combo)
if cost_combo < threshold-p.cost_diff {
p.cost_combo = cost_combo
@ -72,7 +72,7 @@ func compareAndPushToQueueCommand(out []HistogramCommand, cluster_size []uint32,
}
}
func histogramCombineCommand(out []HistogramCommand, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint {
func histogramCombineCommand(out []histogramCommand, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint {
var cost_diff_threshold float64 = 0.0
var min_cluster_size uint = 1
var num_pairs uint = 0
@ -102,7 +102,7 @@ func histogramCombineCommand(out []HistogramCommand, cluster_size []uint32, symb
best_idx1 = pairs[0].idx1
best_idx2 = pairs[0].idx2
HistogramAddHistogramCommand(&out[best_idx1], &out[best_idx2])
histogramAddHistogramCommand(&out[best_idx1], &out[best_idx2])
out[best_idx1].bit_cost_ = pairs[0].cost_combo
cluster_size[best_idx1] += cluster_size[best_idx2]
for i = 0; i < symbols_size; i++ {
@ -154,12 +154,12 @@ func histogramCombineCommand(out []HistogramCommand, cluster_size []uint32, symb
}
/* What is the bit cost of moving histogram from cur_symbol to candidate. */
func histogramBitCostDistanceCommand(histogram *HistogramCommand, candidate *HistogramCommand) float64 {
func histogramBitCostDistanceCommand(histogram *histogramCommand, candidate *histogramCommand) float64 {
if histogram.total_count_ == 0 {
return 0.0
} else {
var tmp HistogramCommand = *histogram
HistogramAddHistogramCommand(&tmp, candidate)
var tmp histogramCommand = *histogram
histogramAddHistogramCommand(&tmp, candidate)
return populationCostCommand(&tmp) - candidate.bit_cost_
}
}
@ -168,7 +168,7 @@ func histogramBitCostDistanceCommand(histogram *HistogramCommand, candidate *His
When called, clusters[0..num_clusters) contains the unique values from
symbols[0..in_size), but this property is not preserved in this function.
Note: we assume that out[]->bit_cost_ is already up-to-date. */
func histogramRemapCommand(in []HistogramCommand, in_size uint, clusters []uint32, num_clusters uint, out []HistogramCommand, symbols []uint32) {
func histogramRemapCommand(in []histogramCommand, in_size uint, clusters []uint32, num_clusters uint, out []histogramCommand, symbols []uint32) {
var i uint
for i = 0; i < in_size; i++ {
var best_out uint32
@ -192,11 +192,11 @@ func histogramRemapCommand(in []HistogramCommand, in_size uint, clusters []uint3
/* Recompute each out based on raw and symbols. */
for i = 0; i < num_clusters; i++ {
HistogramClearCommand(&out[clusters[i]])
histogramClearCommand(&out[clusters[i]])
}
for i = 0; i < in_size; i++ {
HistogramAddHistogramCommand(&out[symbols[i]], &in[i])
histogramAddHistogramCommand(&out[symbols[i]], &in[i])
}
}
@ -213,10 +213,10 @@ func histogramRemapCommand(in []HistogramCommand, in_size uint, clusters []uint3
var histogramReindexCommand_kInvalidIndex uint32 = BROTLI_UINT32_MAX
func histogramReindexCommand(out []HistogramCommand, symbols []uint32, length uint) uint {
func histogramReindexCommand(out []histogramCommand, symbols []uint32, length uint) uint {
var new_index []uint32 = make([]uint32, length)
var next_index uint32
var tmp []HistogramCommand
var tmp []histogramCommand
var i uint
for i = 0; i < length; i++ {
new_index[i] = histogramReindexCommand_kInvalidIndex
@ -232,7 +232,7 @@ func histogramReindexCommand(out []HistogramCommand, symbols []uint32, length ui
/* TODO: by using idea of "cycle-sort" we can avoid allocation of
tmp and reduce the number of copying by the factor of 2. */
tmp = make([]HistogramCommand, next_index)
tmp = make([]histogramCommand, next_index)
next_index = 0
for i = 0; i < length; i++ {
@ -253,7 +253,7 @@ func histogramReindexCommand(out []HistogramCommand, symbols []uint32, length ui
return uint(next_index)
}
func clusterHistogramsCommand(in []HistogramCommand, in_size uint, max_histograms uint, out []HistogramCommand, out_size *uint, histogram_symbols []uint32) {
func clusterHistogramsCommand(in []histogramCommand, in_size uint, max_histograms uint, out []histogramCommand, out_size *uint, histogram_symbols []uint32) {
var cluster_size []uint32 = make([]uint32, in_size)
var clusters []uint32 = make([]uint32, in_size)
var num_clusters uint = 0

30
cluster_distance.go
View File

@ -9,7 +9,7 @@ package brotli
/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if
it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */
func compareAndPushToQueueDistance(out []HistogramDistance, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) {
func compareAndPushToQueueDistance(out []histogramDistance, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) {
var is_good_pair bool = false
var p histogramPair
p.idx2 = 0
@ -45,9 +45,9 @@ func compareAndPushToQueueDistance(out []HistogramDistance, cluster_size []uint3
} else {
threshold = brotli_max_double(0.0, pairs[0].cost_diff)
}
var combo HistogramDistance = out[idx1]
var combo histogramDistance = out[idx1]
var cost_combo float64
HistogramAddHistogramDistance(&combo, &out[idx2])
histogramAddHistogramDistance(&combo, &out[idx2])
cost_combo = populationCostDistance(&combo)
if cost_combo < threshold-p.cost_diff {
p.cost_combo = cost_combo
@ -72,7 +72,7 @@ func compareAndPushToQueueDistance(out []HistogramDistance, cluster_size []uint3
}
}
func histogramCombineDistance(out []HistogramDistance, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint {
func histogramCombineDistance(out []histogramDistance, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint {
var cost_diff_threshold float64 = 0.0
var min_cluster_size uint = 1
var num_pairs uint = 0
@ -102,7 +102,7 @@ func histogramCombineDistance(out []HistogramDistance, cluster_size []uint32, sy
best_idx1 = pairs[0].idx1
best_idx2 = pairs[0].idx2
HistogramAddHistogramDistance(&out[best_idx1], &out[best_idx2])
histogramAddHistogramDistance(&out[best_idx1], &out[best_idx2])
out[best_idx1].bit_cost_ = pairs[0].cost_combo
cluster_size[best_idx1] += cluster_size[best_idx2]
for i = 0; i < symbols_size; i++ {
@ -154,12 +154,12 @@ func histogramCombineDistance(out []HistogramDistance, cluster_size []uint32, sy
}
/* What is the bit cost of moving histogram from cur_symbol to candidate. */
func histogramBitCostDistanceDistance(histogram *HistogramDistance, candidate *HistogramDistance) float64 {
func histogramBitCostDistanceDistance(histogram *histogramDistance, candidate *histogramDistance) float64 {
if histogram.total_count_ == 0 {
return 0.0
} else {
var tmp HistogramDistance = *histogram
HistogramAddHistogramDistance(&tmp, candidate)
var tmp histogramDistance = *histogram
histogramAddHistogramDistance(&tmp, candidate)
return populationCostDistance(&tmp) - candidate.bit_cost_
}
}
@ -168,7 +168,7 @@ func histogramBitCostDistanceDistance(histogram *HistogramDistance, candidate *H
When called, clusters[0..num_clusters) contains the unique values from
symbols[0..in_size), but this property is not preserved in this function.
Note: we assume that out[]->bit_cost_ is already up-to-date. */
func histogramRemapDistance(in []HistogramDistance, in_size uint, clusters []uint32, num_clusters uint, out []HistogramDistance, symbols []uint32) {
func histogramRemapDistance(in []histogramDistance, in_size uint, clusters []uint32, num_clusters uint, out []histogramDistance, symbols []uint32) {
var i uint
for i = 0; i < in_size; i++ {
var best_out uint32
@ -192,11 +192,11 @@ func histogramRemapDistance(in []HistogramDistance, in_size uint, clusters []uin
/* Recompute each out based on raw and symbols. */
for i = 0; i < num_clusters; i++ {
HistogramClearDistance(&out[clusters[i]])
histogramClearDistance(&out[clusters[i]])
}
for i = 0; i < in_size; i++ {
HistogramAddHistogramDistance(&out[symbols[i]], &in[i])
histogramAddHistogramDistance(&out[symbols[i]], &in[i])
}
}
@ -213,10 +213,10 @@ func histogramRemapDistance(in []HistogramDistance, in_size uint, clusters []uin
var histogramReindexDistance_kInvalidIndex uint32 = BROTLI_UINT32_MAX
func histogramReindexDistance(out []HistogramDistance, symbols []uint32, length uint) uint {
func histogramReindexDistance(out []histogramDistance, symbols []uint32, length uint) uint {
var new_index []uint32 = make([]uint32, length)
var next_index uint32
var tmp []HistogramDistance
var tmp []histogramDistance
var i uint
for i = 0; i < length; i++ {
new_index[i] = histogramReindexDistance_kInvalidIndex
@ -232,7 +232,7 @@ func histogramReindexDistance(out []HistogramDistance, symbols []uint32, length
/* TODO: by using idea of "cycle-sort" we can avoid allocation of
tmp and reduce the number of copying by the factor of 2. */
tmp = make([]HistogramDistance, next_index)
tmp = make([]histogramDistance, next_index)
next_index = 0
for i = 0; i < length; i++ {
@ -253,7 +253,7 @@ func histogramReindexDistance(out []HistogramDistance, symbols []uint32, length
return uint(next_index)
}
func clusterHistogramsDistance(in []HistogramDistance, in_size uint, max_histograms uint, out []HistogramDistance, out_size *uint, histogram_symbols []uint32) {
func clusterHistogramsDistance(in []histogramDistance, in_size uint, max_histograms uint, out []histogramDistance, out_size *uint, histogram_symbols []uint32) {
var cluster_size []uint32 = make([]uint32, in_size)
var clusters []uint32 = make([]uint32, in_size)
var num_clusters uint = 0

30
cluster_literal.go
View File

@ -9,7 +9,7 @@ package brotli
/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if
it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */
func compareAndPushToQueueLiteral(out []HistogramLiteral, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) {
func compareAndPushToQueueLiteral(out []histogramLiteral, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) {
var is_good_pair bool = false
var p histogramPair
p.idx2 = 0
@ -45,9 +45,9 @@ func compareAndPushToQueueLiteral(out []HistogramLiteral, cluster_size []uint32,
} else {
threshold = brotli_max_double(0.0, pairs[0].cost_diff)
}
var combo HistogramLiteral = out[idx1]
var combo histogramLiteral = out[idx1]
var cost_combo float64
HistogramAddHistogramLiteral(&combo, &out[idx2])
histogramAddHistogramLiteral(&combo, &out[idx2])
cost_combo = populationCostLiteral(&combo)
if cost_combo < threshold-p.cost_diff {
p.cost_combo = cost_combo
@ -72,7 +72,7 @@ func compareAndPushToQueueLiteral(out []HistogramLiteral, cluster_size []uint32,
}
}
func histogramCombineLiteral(out []HistogramLiteral, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint {
func histogramCombineLiteral(out []histogramLiteral, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint {
var cost_diff_threshold float64 = 0.0
var min_cluster_size uint = 1
var num_pairs uint = 0
@ -102,7 +102,7 @@ func histogramCombineLiteral(out []HistogramLiteral, cluster_size []uint32, symb
best_idx1 = pairs[0].idx1
best_idx2 = pairs[0].idx2
HistogramAddHistogramLiteral(&out[best_idx1], &out[best_idx2])
histogramAddHistogramLiteral(&out[best_idx1], &out[best_idx2])
out[best_idx1].bit_cost_ = pairs[0].cost_combo
cluster_size[best_idx1] += cluster_size[best_idx2]
for i = 0; i < symbols_size; i++ {
@ -154,12 +154,12 @@ func histogramCombineLiteral(out []HistogramLiteral, cluster_size []uint32, symb
}
/* What is the bit cost of moving histogram from cur_symbol to candidate. */
func histogramBitCostDistanceLiteral(histogram *HistogramLiteral, candidate *HistogramLiteral) float64 {
func histogramBitCostDistanceLiteral(histogram *histogramLiteral, candidate *histogramLiteral) float64 {
if histogram.total_count_ == 0 {
return 0.0
} else {
var tmp HistogramLiteral = *histogram
HistogramAddHistogramLiteral(&tmp, candidate)
var tmp histogramLiteral = *histogram
histogramAddHistogramLiteral(&tmp, candidate)
return populationCostLiteral(&tmp) - candidate.bit_cost_
}
}
@ -168,7 +168,7 @@ func histogramBitCostDistanceLiteral(histogram *HistogramLiteral, candidate *His
When called, clusters[0..num_clusters) contains the unique values from
symbols[0..in_size), but this property is not preserved in this function.
Note: we assume that out[]->bit_cost_ is already up-to-date. */
func histogramRemapLiteral(in []HistogramLiteral, in_size uint, clusters []uint32, num_clusters uint, out []HistogramLiteral, symbols []uint32) {
func histogramRemapLiteral(in []histogramLiteral, in_size uint, clusters []uint32, num_clusters uint, out []histogramLiteral, symbols []uint32) {
var i uint
for i = 0; i < in_size; i++ {
var best_out uint32
@ -192,11 +192,11 @@ func histogramRemapLiteral(in []HistogramLiteral, in_size uint, clusters []uint3
/* Recompute each out based on raw and symbols. */
for i = 0; i < num_clusters; i++ {
HistogramClearLiteral(&out[clusters[i]])
histogramClearLiteral(&out[clusters[i]])
}
for i = 0; i < in_size; i++ {
HistogramAddHistogramLiteral(&out[symbols[i]], &in[i])
histogramAddHistogramLiteral(&out[symbols[i]], &in[i])
}
}
@ -213,10 +213,10 @@ func histogramRemapLiteral(in []HistogramLiteral, in_size uint, clusters []uint3
var histogramReindexLiteral_kInvalidIndex uint32 = BROTLI_UINT32_MAX
func histogramReindexLiteral(out []HistogramLiteral, symbols []uint32, length uint) uint {
func histogramReindexLiteral(out []histogramLiteral, symbols []uint32, length uint) uint {
var new_index []uint32 = make([]uint32, length)
var next_index uint32
var tmp []HistogramLiteral
var tmp []histogramLiteral
var i uint
for i = 0; i < length; i++ {
new_index[i] = histogramReindexLiteral_kInvalidIndex
@ -232,7 +232,7 @@ func histogramReindexLiteral(out []HistogramLiteral, symbols []uint32, length ui
/* TODO: by using idea of "cycle-sort" we can avoid allocation of
tmp and reduce the number of copying by the factor of 2. */
tmp = make([]HistogramLiteral, next_index)
tmp = make([]histogramLiteral, next_index)
next_index = 0
for i = 0; i < length; i++ {
@ -253,7 +253,7 @@ func histogramReindexLiteral(out []HistogramLiteral, symbols []uint32, length ui
return uint(next_index)
}
func clusterHistogramsLiteral(in []HistogramLiteral, in_size uint, max_histograms uint, out []HistogramLiteral, out_size *uint, histogram_symbols []uint32) {
func clusterHistogramsLiteral(in []histogramLiteral, in_size uint, max_histograms uint, out []histogramLiteral, out_size *uint, histogram_symbols []uint32) {
var cluster_size []uint32 = make([]uint32, in_size)
var clusters []uint32 = make([]uint32, in_size)
var num_clusters uint = 0

12
command.go
View File

@ -112,10 +112,10 @@ func getInsertLengthCode(insertlen uint) uint16 {
if insertlen < 6 {
return uint16(insertlen)
} else if insertlen < 130 {
var nbits uint32 = Log2FloorNonZero(insertlen-2) - 1
var nbits uint32 = log2FloorNonZero(insertlen-2) - 1
return uint16((nbits << 1) + uint32((insertlen-2)>>nbits) + 2)
} else if insertlen < 2114 {
return uint16(Log2FloorNonZero(insertlen-66) + 10)
return uint16(log2FloorNonZero(insertlen-66) + 10)
} else if insertlen < 6210 {
return 21
} else if insertlen < 22594 {
@ -129,10 +129,10 @@ func getCopyLengthCode(copylen uint) uint16 {
if copylen < 10 {
return uint16(copylen - 2)
} else if copylen < 134 {
var nbits uint32 = Log2FloorNonZero(copylen-6) - 1
var nbits uint32 = log2FloorNonZero(copylen-6) - 1
return uint16((nbits << 1) + uint32((copylen-6)>>nbits) + 4)
} else if copylen < 2118 {
return uint16(Log2FloorNonZero(copylen-70) + 12)
return uint16(log2FloorNonZero(copylen-70) + 12)
} else {
return 23
}
@ -194,7 +194,7 @@ type command struct {
}
/* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */
func initCommand(self *command, dist *BrotliDistanceParams, insertlen uint, copylen uint, copylen_code_delta int, distance_code uint) {
func initCommand(self *command, dist *distanceParams, insertlen uint, copylen uint, copylen_code_delta int, distance_code uint) {
/* Don't rely on signed int representation, use honest casts. */
var delta uint32 = uint32(byte(int8(copylen_code_delta)))
self.insert_len_ = uint32(insertlen)
@ -216,7 +216,7 @@ func initInsertCommand(self *command, insertlen uint) {
getLengthCode(insertlen, 4, false, &self.cmd_prefix_)
}
func commandRestoreDistanceCode(self *command, dist *BrotliDistanceParams) uint32 {
func commandRestoreDistanceCode(self *command, dist *distanceParams) uint32 {
if uint32(self.dist_prefix_&0x3FF) < numDistanceShortCodes+dist.num_direct_distance_codes {
return uint32(self.dist_prefix_) & 0x3FF
} else {

40
compress_fragment.go
View File

@ -135,14 +135,14 @@ func buildAndStoreLiteralPrefixCode(input []byte, input_size uint, depths []byte
/* Builds a command and distance prefix code (each 64 symbols) into "depth" and
"bits" based on "histogram" and stores it into the bit stream. */
func buildAndStoreCommandPrefixCode1(histogram []uint32, depth []byte, bits []uint16, storage_ix *uint, storage []byte) {
var tree [129]HuffmanTree
var tree [129]huffmanTree
var cmd_depth = [numCommandSymbols]byte{0}
/* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */
var cmd_bits [64]uint16
BrotliCreateHuffmanTree(histogram, 64, 15, tree[:], depth)
BrotliCreateHuffmanTree(histogram[64:], 64, 14, tree[:], depth[64:])
createHuffmanTree(histogram, 64, 15, tree[:], depth)
createHuffmanTree(histogram[64:], 64, 14, tree[:], depth[64:])
/* We have to jump through a few hoops here in order to compute
the command bits because the symbols are in a different order than in
@ -156,14 +156,14 @@ func buildAndStoreCommandPrefixCode1(histogram []uint32, depth []byte, bits []ui
copy(cmd_depth[40:][:], depth[48:][:8])
copy(cmd_depth[48:][:], depth[32:][:8])
copy(cmd_depth[56:][:], depth[56:][:8])
BrotliConvertBitDepthsToSymbols(cmd_depth[:], 64, cmd_bits[:])
convertBitDepthsToSymbols(cmd_depth[:], 64, cmd_bits[:])
copy(bits, cmd_bits[:24])
copy(bits[24:], cmd_bits[32:][:8])
copy(bits[32:], cmd_bits[48:][:8])
copy(bits[40:], cmd_bits[24:][:8])
copy(bits[48:], cmd_bits[40:][:8])
copy(bits[56:], cmd_bits[56:][:8])
BrotliConvertBitDepthsToSymbols(depth[64:], 64, bits[64:])
convertBitDepthsToSymbols(depth[64:], 64, bits[64:])
{
/* Create the bit length array for the full command alphabet. */
var i uint
@ -195,7 +195,7 @@ func emitInsertLen1(insertlen uint, depth []byte, bits []uint16, histo []uint32,
histo[code]++
} else if insertlen < 130 {
var tail uint = insertlen - 2
var nbits uint32 = Log2FloorNonZero(tail) - 1
var nbits uint32 = log2FloorNonZero(tail) - 1
var prefix uint = tail >> nbits
var inscode uint = uint((nbits << 1) + uint32(prefix) + 42)
BrotliWriteBits(uint(depth[inscode]), uint64(bits[inscode]), storage_ix, storage)
@ -203,7 +203,7 @@ func emitInsertLen1(insertlen uint, depth []byte, bits []uint16, histo []uint32,
histo[inscode]++
} else if insertlen < 2114 {
var tail uint = insertlen - 66
var nbits uint32 = Log2FloorNonZero(tail)
var nbits uint32 = log2FloorNonZero(tail)
var code uint = uint(nbits + 50)
BrotliWriteBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
BrotliWriteBits(uint(nbits), uint64(tail)-(uint64(uint(1))<<nbits), storage_ix, storage)
@ -233,7 +233,7 @@ func emitCopyLen1(copylen uint, depth []byte, bits []uint16, histo []uint32, sto
histo[copylen+14]++
} else if copylen < 134 {
var tail uint = copylen - 6
var nbits uint32 = Log2FloorNonZero(tail) - 1
var nbits uint32 = log2FloorNonZero(tail) - 1
var prefix uint = tail >> nbits
var code uint = uint((nbits << 1) + uint32(prefix) + 20)
BrotliWriteBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
@ -241,7 +241,7 @@ func emitCopyLen1(copylen uint, depth []byte, bits []uint16, histo []uint32, sto
histo[code]++
} else if copylen < 2118 {
var tail uint = copylen - 70
var nbits uint32 = Log2FloorNonZero(tail)
var nbits uint32 = log2FloorNonZero(tail)
var code uint = uint(nbits + 28)
BrotliWriteBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
BrotliWriteBits(uint(nbits), uint64(tail)-(uint64(uint(1))<<nbits), storage_ix, storage)
@ -259,7 +259,7 @@ func emitCopyLenLastDistance1(copylen uint, depth []byte, bits []uint16, histo [
histo[copylen-4]++
} else if copylen < 72 {
var tail uint = copylen - 8
var nbits uint32 = Log2FloorNonZero(tail) - 1
var nbits uint32 = log2FloorNonZero(tail) - 1
var prefix uint = tail >> nbits
var code uint = uint((nbits << 1) + uint32(prefix) + 4)
BrotliWriteBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
@ -275,7 +275,7 @@ func emitCopyLenLastDistance1(copylen uint, depth []byte, bits []uint16, histo [
histo[64]++
} else if copylen < 2120 {
var tail uint = copylen - 72
var nbits uint32 = Log2FloorNonZero(tail)
var nbits uint32 = log2FloorNonZero(tail)
var code uint = uint(nbits + 28)
BrotliWriteBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage)
BrotliWriteBits(uint(nbits), uint64(tail)-(uint64(uint(1))<<nbits), storage_ix, storage)
@ -293,7 +293,7 @@ func emitCopyLenLastDistance1(copylen uint, depth []byte, bits []uint16, histo [
func emitDistance1(distance uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) {
var d uint = distance + 3
var nbits uint32 = Log2FloorNonZero(d) - 1
var nbits uint32 = log2FloorNonZero(d) - 1
var prefix uint = (d >> nbits) & 1
var offset uint = (2 + prefix) << nbits
var distcode uint = uint(2*(nbits-1) + uint32(prefix) + 80)
@ -363,9 +363,9 @@ func shouldMergeBlock(data []byte, len uint, depths []byte) bool {
}
{
var total uint = (len + shouldMergeBlock_kSampleRate - 1) / shouldMergeBlock_kSampleRate
var r float64 = (FastLog2(total)+0.5)*float64(total) + 200
var r float64 = (fastLog2(total)+0.5)*float64(total) + 200
for i = 0; i < 256; i++ {
r -= float64(histo[i]) * (float64(depths[i]) + FastLog2(histo[i]))
r -= float64(histo[i]) * (float64(depths[i]) + fastLog2(histo[i]))
}
return r >= 0.0
@ -531,7 +531,7 @@ func compressFragmentFastImpl(in []byte, input_size uint, is_last bool, table []
var next_emit int = 0
var base_ip int = 0
var input int = 0
var kInputMarginBytes uint = BROTLI_WINDOW_GAP
var kInputMarginBytes uint = windowGap
var kMinMatchLen uint = 5
var metablock_start int = input
var block_size uint = brotli_min_size_t(input_size, compressFragmentFastImpl_kFirstBlockSize)
@ -647,7 +647,7 @@ emit_commands:
/* Check copy distance. If candidate is not feasible, continue search.
Checking is done outside of hot loop to reduce overhead. */
if ip-candidate > maxDistance {
if ip-candidate > maxDistance_compress_fragment {
goto trawl
}
@ -658,7 +658,7 @@ emit_commands:
{
var base int = ip
/* > 0 */
var matched uint = 5 + FindMatchLengthWithLimit(in[candidate+5:], in[ip+5:], uint(ip_end-ip)-5)
var matched uint = 5 + findMatchLengthWithLimit(in[candidate+5:], in[ip+5:], uint(ip_end-ip)-5)
var distance int = int(base - candidate)
/* We have a 5-byte match at ip, and we need to emit bytes in
[next_emit, ip). */
@ -716,8 +716,8 @@ emit_commands:
/* We have a 5-byte match at ip, and no need to emit any literal bytes
prior to ip. */
var matched uint = 5 + FindMatchLengthWithLimit(in[candidate+5:], in[ip+5:], uint(ip_end-ip)-5)
if ip-candidate > maxDistance {
var matched uint = 5 + findMatchLengthWithLimit(in[candidate+5:], in[ip+5:], uint(ip_end-ip)-5)
if ip-candidate > maxDistance_compress_fragment {
break
}
ip += int(matched)
@ -823,7 +823,7 @@ next_block:
func compressFragmentFast(input []byte, input_size uint, is_last bool, table []int, table_size uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, storage_ix *uint, storage []byte) {
var initial_storage_ix uint = *storage_ix
var table_bits uint = uint(Log2FloorNonZero(table_size))
var table_bits uint = uint(log2FloorNonZero(table_size))
if input_size == 0 {
assert(is_last)

2
compress_fragment_common.go
View File

@ -13,7 +13,7 @@ package brotli
REQUIRES: "table_size" is a power of two
OUTPUT: maximal copy distance <= |input_size|
OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */
const maxDistance = 262128
const maxDistance_compress_fragment = 262128
/* kHashMul32_a multiplier has these properties:
* The multiplier must be odd. Otherwise we may lose the highest bit.

36
compress_fragment_two_pass.go
View File

@ -53,13 +53,13 @@ func isMatch1(p1 []byte, p2 []byte, length uint) bool {
/* Builds a command and distance prefix code (each 64 symbols) into "depth" and
"bits" based on "histogram" and stores it into the bit stream. */
func buildAndStoreCommandPrefixCode(histogram []uint32, depth []byte, bits []uint16, storage_ix *uint, storage []byte) {
var tree [129]HuffmanTree
var tree [129]huffmanTree
var cmd_depth = [numCommandSymbols]byte{0}
/* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */
var cmd_bits [64]uint16
BrotliCreateHuffmanTree(histogram, 64, 15, tree[:], depth)
BrotliCreateHuffmanTree(histogram[64:], 64, 14, tree[:], depth[64:])
createHuffmanTree(histogram, 64, 15, tree[:], depth)
createHuffmanTree(histogram[64:], 64, 14, tree[:], depth[64:])
/* We have to jump through a few hoops here in order to compute
the command bits because the symbols are in a different order than in
@ -73,14 +73,14 @@ func buildAndStoreCommandPrefixCode(histogram []uint32, depth []byte, bits []uin
copy(cmd_depth[40:][:], depth[8:][:8])
copy(cmd_depth[48:][:], depth[56:][:8])
copy(cmd_depth[56:][:], depth[16:][:8])
BrotliConvertBitDepthsToSymbols(cmd_depth[:], 64, cmd_bits[:])
convertBitDepthsToSymbols(cmd_depth[:], 64, cmd_bits[:])
copy(bits, cmd_bits[24:][:8])
copy(bits[8:], cmd_bits[40:][:8])
copy(bits[16:], cmd_bits[56:][:8])
copy(bits[24:], cmd_bits[:24])
copy(bits[48:], cmd_bits[32:][:8])
copy(bits[56:], cmd_bits[48:][:8])
BrotliConvertBitDepthsToSymbols(depth[64:], 64, bits[64:])
convertBitDepthsToSymbols(depth[64:], 64, bits[64:])
{
/* Create the bit length array for the full command alphabet. */
var i uint
@ -109,14 +109,14 @@ func emitInsertLen(insertlen uint32, commands *[]uint32) {
(*commands)[0] = insertlen
} else if insertlen < 130 {
var tail uint32 = insertlen - 2
var nbits uint32 = Log2FloorNonZero(uint(tail)) - 1
var nbits uint32 = log2FloorNonZero(uint(tail)) - 1
var prefix uint32 = tail >> nbits
var inscode uint32 = (nbits << 1) + prefix + 2
var extra uint32 = tail - (prefix << nbits)
(*commands)[0] = inscode | extra<<8
} else if insertlen < 2114 {
var tail uint32 = insertlen - 66
var nbits uint32 = Log2FloorNonZero(uint(tail))
var nbits uint32 = log2FloorNonZero(uint(tail))
var code uint32 = nbits + 10
var extra uint32 = tail - (1 << nbits)
(*commands)[0] = code | extra<<8
@ -139,14 +139,14 @@ func emitCopyLen(copylen uint, commands *[]uint32) {
(*commands)[0] = uint32(copylen + 38)
} else if copylen < 134 {
var tail uint = copylen - 6
var nbits uint = uint(Log2FloorNonZero(tail) - 1)
var nbits uint = uint(log2FloorNonZero(tail) - 1)
var prefix uint = tail >> nbits
var code uint = (nbits << 1) + prefix + 44
var extra uint = tail - (prefix << nbits)
(*commands)[0] = uint32(code | extra<<8)
} else if copylen < 2118 {
var tail uint = copylen - 70
var nbits uint = uint(Log2FloorNonZero(tail))
var nbits uint = uint(log2FloorNonZero(tail))
var code uint = nbits + 52
var extra uint = tail - (uint(1) << nbits)
(*commands)[0] = uint32(code | extra<<8)
@ -164,7 +164,7 @@ func emitCopyLenLastDistance(copylen uint, commands *[]uint32) {
*commands = (*commands)[1:]
} else if copylen < 72 {
var tail uint = copylen - 8
var nbits uint = uint(Log2FloorNonZero(tail) - 1)
var nbits uint = uint(log2FloorNonZero(tail) - 1)
var prefix uint = tail >> nbits
var code uint = (nbits << 1) + prefix + 28
var extra uint = tail - (prefix << nbits)
@ -180,7 +180,7 @@ func emitCopyLenLastDistance(copylen uint, commands *[]uint32) {
*commands = (*commands)[1:]
} else if copylen < 2120 {
var tail uint = copylen - 72
var nbits uint = uint(Log2FloorNonZero(tail))
var nbits uint = uint(log2FloorNonZero(tail))
var code uint = nbits + 52
var extra uint = tail - (uint(1) << nbits)
(*commands)[0] = uint32(code | extra<<8)
@ -198,7 +198,7 @@ func emitCopyLenLastDistance(copylen uint, commands *[]uint32) {
func emitDistance(distance uint32, commands *[]uint32) {
var d uint32 = distance + 3
var nbits uint32 = Log2FloorNonZero(uint(d)) - 1
var nbits uint32 = log2FloorNonZero(uint(d)) - 1
var prefix uint32 = (d >> nbits) & 1
var offset uint32 = (2 + prefix) << nbits
var distcode uint32 = 2*(nbits-1) + prefix + 80
@ -236,7 +236,7 @@ func createCommands(input []byte, block_size uint, input_size uint, base_ip_ptr
var last_distance int = -1
/* "ip" is the input pointer. */
var kInputMarginBytes uint = BROTLI_WINDOW_GAP
var kInputMarginBytes uint = windowGap
/* "next_emit" is a pointer to the first byte that is not covered by a
previous copy. Bytes between "next_emit" and the start of the next copy or
@ -307,7 +307,7 @@ func createCommands(input []byte, block_size uint, input_size uint, base_ip_ptr
/* Check copy distance. If candidate is not feasible, continue search.
Checking is done outside of hot loop to reduce overhead. */
if ip-candidate > maxDistance {
if ip-candidate > maxDistance_compress_fragment {
goto trawl
}
@ -318,7 +318,7 @@ func createCommands(input []byte, block_size uint, input_size uint, base_ip_ptr
{
var base int = ip
/* > 0 */
var matched uint = min_match + FindMatchLengthWithLimit(input[uint(candidate)+min_match:], input[uint(ip)+min_match:], uint(ip_end-ip)-min_match)
var matched uint = min_match + findMatchLengthWithLimit(input[uint(candidate)+min_match:], input[uint(ip)+min_match:], uint(ip_end-ip)-min_match)
var distance int = int(base - candidate)
/* We have a 6-byte match at ip, and we need to emit bytes in
[next_emit, ip). */
@ -380,12 +380,12 @@ func createCommands(input []byte, block_size uint, input_size uint, base_ip_ptr
}
}
for ip-candidate <= maxDistance && isMatch1(input[ip:], input[candidate:], min_match) {
for ip-candidate <= maxDistance_compress_fragment && isMatch1(input[ip:], input[candidate:], min_match) {
var base int = ip
/* We have a 6-byte match at ip, and no need to emit any
literal bytes prior to ip. */
var matched uint = min_match + FindMatchLengthWithLimit(input[uint(candidate)+min_match:], input[uint(ip)+min_match:], uint(ip_end-ip)-min_match)
var matched uint = min_match + findMatchLengthWithLimit(input[uint(candidate)+min_match:], input[uint(ip)+min_match:], uint(ip_end-ip)-min_match)
ip += int(matched)
last_distance = int(base - candidate) /* > 0 */
emitCopyLen(matched, commands)
@ -723,7 +723,7 @@ func compressFragmentTwoPassImpl(input []byte, input_size uint, is_last bool, co
func compressFragmentTwoPass(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_size uint, storage_ix *uint, storage []byte) {
var initial_storage_ix uint = *storage_ix
var table_bits uint = uint(Log2FloorNonZero(table_size))
var table_bits uint = uint(log2FloorNonZero(table_size))
var min_match uint
if table_bits <= 15 {
min_match = 4

14
constants.go
View File

@ -56,22 +56,22 @@ func distanceAlphabetSize(NPOSTFIX uint, NDIRECT uint, MAXNBITS uint) uint {
/* numDistanceSymbols == 1128 */
const numDistanceSymbols = 1128
const BROTLI_MAX_DISTANCE = 0x3FFFFFC
const maxDistance = 0x3FFFFFC
const BROTLI_MAX_ALLOWED_DISTANCE = 0x7FFFFFFC
const maxAllowedDistance = 0x7FFFFFFC
/* 7.1. Context modes and context ID lookup for literals */
/* "context IDs for literals are in the range of 0..63" */
const BROTLI_LITERAL_CONTEXT_BITS = 6
const literalContextBits = 6
/* 7.2. Context ID for distances */
const BROTLI_DISTANCE_CONTEXT_BITS = 2
const distanceContextBits = 2
/* 9.1. Format of the Stream Header */
/* Number of slack bytes for window size. Don't confuse
with BROTLI_NUM_DISTANCE_SHORT_CODES. */
const BROTLI_WINDOW_GAP = 16
const windowGap = 16
func BROTLI_MAX_BACKWARD_LIMIT(W uint) uint {
return (uint(1) << W) - BROTLI_WINDOW_GAP
func maxBackwardLimit(W uint) uint {
return (uint(1) << W) - windowGap
}

24
context.go
View File

@ -179,17 +179,17 @@ package brotli
/* Lookup table to map the previous two bytes to a context id.
There are four different context modeling modes defined here:
CONTEXT_LSB6: context id is the least significant 6 bits of the last byte,
CONTEXT_MSB6: context id is the most significant 6 bits of the last byte,
CONTEXT_UTF8: second-order context model tuned for UTF8-encoded text,
CONTEXT_SIGNED: second-order context model tuned for signed integers.
contextLSB6: context id is the least significant 6 bits of the last byte,
contextMSB6: context id is the most significant 6 bits of the last byte,
contextUTF8: second-order context model tuned for UTF8-encoded text,
contextSigned: second-order context model tuned for signed integers.
If |p1| and |p2| are the previous two bytes, and |mode| is current context
mode, we calculate the context as:
context = ContextLut(mode)[p1] | ContextLut(mode)[p2 + 256].
For CONTEXT_UTF8 mode, if the previous two bytes are ASCII characters
For contextUTF8 mode, if the previous two bytes are ASCII characters
(i.e. < 128), this will be equivalent to
context = 4 * context1(p1) + context2(p2),
@ -257,10 +257,10 @@ context ids and the type of the next byte is summarized in the table below:
|-------------|-------------------|---------------------|------------------|
*/
const (
CONTEXT_LSB6 = 0
CONTEXT_MSB6 = 1
CONTEXT_UTF8 = 2
CONTEXT_SIGNED = 3
contextLSB6 = 0
contextMSB6 = 1
contextUTF8 = 2
contextSigned = 3
)
/* Common context lookup table for all context modes. */
@ -2340,12 +2340,12 @@ var kContextLookup = [2048]byte{
7,
}
type ContextLut []byte
type contextLUT []byte
func BROTLI_CONTEXT_LUT(mode int) ContextLut {
func getContextLUT(mode int) contextLUT {
return kContextLookup[mode<<9:]
}
func BROTLI_CONTEXT(p1 byte, p2 byte, lut ContextLut) byte {
func getContext(p1 byte, p2 byte, lut contextLUT) byte {
return lut[p1] | lut[256+int(p2)]
}

898
decode.go
View File

File diff suppressed because it is too large Load Diff

16
dictionary.go
View File

@ -12,7 +12,7 @@ package brotli
*/
/* Collection of static dictionary words. */
type BrotliDictionary struct {
type dictionary struct {
size_bits_by_length [32]byte
offsets_by_length [32]uint32
data_size uint
@ -28,9 +28,9 @@ type BrotliDictionary struct {
* This method is used ONLY in multi-client environment (e.g. C + Java),
* to reduce storage by sharing single dictionary between implementations.
*/
const BROTLI_MIN_DICTIONARY_WORD_LENGTH = 4
const minDictionaryWordLength = 4
const BROTLI_MAX_DICTIONARY_WORD_LENGTH = 24
const maxDictionaryWordLength = 24
var kBrotliDictionaryData = []byte{
116,
@ -122819,7 +122819,7 @@ var kBrotliDictionaryData = []byte{
190,
}
var kBrotliDictionary = BrotliDictionary{
var kBrotliDictionary = dictionary{
/* size_bits_by_length */
[32]byte{
0,
@ -122899,12 +122899,6 @@ var kBrotliDictionary = BrotliDictionary{
kBrotliDictionaryData,
}
func BrotliGetDictionary() *BrotliDictionary {
func getDictionary() *dictionary {
return &kBrotliDictionary
}
func BrotliSetDictionaryData(data []byte) {
if !(data == nil) && kBrotliDictionary.data == nil {
kBrotliDictionary.data = data
}
}

623
encode.go
View File

File diff suppressed because it is too large Load Diff

8
encoder_dict.go
View File

@ -1,8 +1,8 @@
package brotli
/* Dictionary data (words and transforms) for 1 possible context */
type BrotliEncoderDictionary struct {
words *BrotliDictionary
type encoderDictionary struct {
words *dictionary
cutoffTransformsCount uint32
cutoffTransforms uint64
hash_table []uint16
@ -10,8 +10,8 @@ type BrotliEncoderDictionary struct {
dict_words []DictWord
}
func BrotliInitEncoderDictionary(dict *BrotliEncoderDictionary) {
dict.words = BrotliGetDictionary()
func initEncoderDictionary(dict *encoderDictionary) {
dict.words = getDictionary()
dict.hash_table = kStaticDictionaryHash[:]
dict.buckets = kStaticDictionaryBuckets[:]

78
entropy_encode.go
View File

@ -16,13 +16,13 @@ package brotli
/* Entropy encoding (Huffman) utilities. */
/* A node of a Huffman tree. */
type HuffmanTree struct {
type huffmanTree struct {
total_count_ uint32
index_left_ int16
index_right_or_value_ int16
}
func InitHuffmanTree(self *HuffmanTree, count uint32, left int16, right int16) {
func initHuffmanTree(self *huffmanTree, count uint32, left int16, right int16) {
self.total_count_ = count
self.index_left_ = left
self.index_right_or_value_ = right
@ -58,16 +58,16 @@ func InitHuffmanTree(self *HuffmanTree, count uint32, left int16, right int16) {
/* Get the actual bit values for a tree of bit depths. */
/* Input size optimized Shell sort. */
type HuffmanTreeComparator func(*HuffmanTree, *HuffmanTree) bool
type huffmanTreeComparator func(*huffmanTree, *huffmanTree) bool
var SortHuffmanTreeItems_gaps = []uint{132, 57, 23, 10, 4, 1}
var sortHuffmanTreeItems_gaps = []uint{132, 57, 23, 10, 4, 1}
func SortHuffmanTreeItems(items []HuffmanTree, n uint, comparator HuffmanTreeComparator) {
func sortHuffmanTreeItems(items []huffmanTree, n uint, comparator huffmanTreeComparator) {
if n < 13 {
/* Insertion sort. */
var i uint
for i = 1; i < n; i++ {
var tmp HuffmanTree = items[i]
var tmp huffmanTree = items[i]
var k uint = i
var j uint = i - 1
for comparator(&tmp, &items[j]) {
@ -92,11 +92,11 @@ func SortHuffmanTreeItems(items []HuffmanTree, n uint, comparator HuffmanTreeCom
g = 0
}
for ; g < 6; g++ {
var gap uint = SortHuffmanTreeItems_gaps[g]
var gap uint = sortHuffmanTreeItems_gaps[g]
var i uint
for i = gap; i < n; i++ {
var j uint = i
var tmp HuffmanTree = items[i]
var tmp huffmanTree = items[i]
for ; j >= gap && comparator(&tmp, &items[j-gap]); j -= gap {
items[j] = items[j-gap]
}
@ -107,7 +107,7 @@ func SortHuffmanTreeItems(items []HuffmanTree, n uint, comparator HuffmanTreeCom
}
}
func BrotliSetDepth(p0 int, pool []HuffmanTree, depth []byte, max_depth int) bool {
func setDepth(p0 int, pool []huffmanTree, depth []byte, max_depth int) bool {
var stack [16]int
var level int = 0
var p int = p0
@ -138,7 +138,7 @@ func BrotliSetDepth(p0 int, pool []HuffmanTree, depth []byte, max_depth int) boo
}
/* Sort the root nodes, least popular first. */
func SortHuffmanTree(v0 *HuffmanTree, v1 *HuffmanTree) bool {
func sortHuffmanTree(v0 *huffmanTree, v1 *huffmanTree) bool {
if v0.total_count_ != v1.total_count_ {
return v0.total_count_ < v1.total_count_
}
@ -161,10 +161,10 @@ func SortHuffmanTree(v0 *HuffmanTree, v1 *HuffmanTree) bool {
we are not planning to use this with extremely long blocks.
See http://en.wikipedia.org/wiki/Huffman_coding */
func BrotliCreateHuffmanTree(data []uint32, length uint, tree_limit int, tree []HuffmanTree, depth []byte) {
func createHuffmanTree(data []uint32, length uint, tree_limit int, tree []huffmanTree, depth []byte) {
var count_limit uint32
var sentinel HuffmanTree
InitHuffmanTree(&sentinel, BROTLI_UINT32_MAX, -1, -1)
var sentinel huffmanTree
initHuffmanTree(&sentinel, BROTLI_UINT32_MAX, -1, -1)
/* For block sizes below 64 kB, we never need to do a second iteration
of this loop. Probably all of our block sizes will be smaller than
@ -179,7 +179,7 @@ func BrotliCreateHuffmanTree(data []uint32, length uint, tree_limit int, tree []
i--
if data[i] != 0 {
var count uint32 = brotli_max_uint32_t(data[i], count_limit)
InitHuffmanTree(&tree[n], count, -1, int16(i))
initHuffmanTree(&tree[n], count, -1, int16(i))
n++
}
}
@ -189,7 +189,7 @@ func BrotliCreateHuffmanTree(data []uint32, length uint, tree_limit int, tree []
break
}
SortHuffmanTreeItems(tree, n, HuffmanTreeComparator(SortHuffmanTree))
sortHuffmanTreeItems(tree, n, huffmanTreeComparator(sortHuffmanTree))
/* The nodes are:
[0, n): the sorted leaf nodes that we start with.
@ -234,7 +234,7 @@ func BrotliCreateHuffmanTree(data []uint32, length uint, tree_limit int, tree []
}
}
if BrotliSetDepth(int(2*n-1), tree[0:], depth, tree_limit) {
if setDepth(int(2*n-1), tree[0:], depth, tree_limit) {
/* We need to pack the Huffman tree in tree_limit bits. If this was not
successful, add fake entities to the lowest values and retry. */
break
@ -242,7 +242,7 @@ func BrotliCreateHuffmanTree(data []uint32, length uint, tree_limit int, tree []
}
}
func Reverse(v []byte, start uint, end uint) {
func reverse(v []byte, start uint, end uint) {
end--
for start < end {
var tmp byte = v[start]
@ -253,7 +253,7 @@ func Reverse(v []byte, start uint, end uint) {
}
}
func BrotliWriteHuffmanTreeRepetitions(previous_value byte, value byte, repetitions uint, tree_size *uint, tree []byte, extra_bits_data []byte) {
func writeHuffmanTreeRepetitions(previous_value byte, value byte, repetitions uint, tree_size *uint, tree []byte, extra_bits_data []byte) {
assert(repetitions > 0)
if previous_value != value {
tree[*tree_size] = value
@ -291,12 +291,12 @@ func BrotliWriteHuffmanTreeRepetitions(previous_value byte, value byte, repetiti
repetitions--
}
Reverse(tree, start, *tree_size)
Reverse(extra_bits_data, start, *tree_size)
reverse(tree, start, *tree_size)
reverse(extra_bits_data, start, *tree_size)
}
}
func BrotliWriteHuffmanTreeRepetitionsZeros(repetitions uint, tree_size *uint, tree []byte, extra_bits_data []byte) {
func writeHuffmanTreeRepetitionsZeros(repetitions uint, tree_size *uint, tree []byte, extra_bits_data []byte) {
if repetitions == 11 {
tree[*tree_size] = 0
extra_bits_data[*tree_size] = 0
@ -326,12 +326,12 @@ func BrotliWriteHuffmanTreeRepetitionsZeros(repetitions uint, tree_size *uint, t
repetitions--
}
Reverse(tree, start, *tree_size)
Reverse(extra_bits_data, start, *tree_size)
reverse(tree, start, *tree_size)
reverse(extra_bits_data, start, *tree_size)
}
}
func BrotliOptimizeHuffmanCountsForRle(length uint, counts []uint32, good_for_rle []byte) {
func optimizeHuffmanCountsForRLE(length uint, counts []uint32, good_for_rle []byte) {
var nonzero_count uint = 0
var stride uint
var limit uint
@ -477,7 +477,7 @@ func BrotliOptimizeHuffmanCountsForRle(length uint, counts []uint32, good_for_rl
}
}
func DecideOverRleUse(depth []byte, length uint, use_rle_for_non_zero *bool, use_rle_for_zero *bool) {
func decideOverRLEUse(depth []byte, length uint, use_rle_for_non_zero *bool, use_rle_for_zero *bool) {
var total_reps_zero uint = 0
var total_reps_non_zero uint = 0
var count_reps_zero uint = 1
@ -508,7 +508,7 @@ func DecideOverRleUse(depth []byte, length uint, use_rle_for_non_zero *bool, use
*use_rle_for_zero = total_reps_zero > count_reps_zero*2
}
func BrotliWriteHuffmanTree(depth []byte, length uint, tree_size *uint, tree []byte, extra_bits_data []byte) {
func writeHuffmanTree(depth []byte, length uint, tree_size *uint, tree []byte, extra_bits_data []byte) {
var previous_value byte = initialRepeatedCodeLength
var i uint
var use_rle_for_non_zero bool = false
@ -527,7 +527,7 @@ func BrotliWriteHuffmanTree(depth []byte, length uint, tree_size *uint, tree []b
if length > 50 {
/* Find RLE coding for longer codes.
Shorter codes seem not to benefit from RLE. */
DecideOverRleUse(depth, new_length, &use_rle_for_non_zero, &use_rle_for_zero)
decideOverRLEUse(depth, new_length, &use_rle_for_non_zero, &use_rle_for_zero)
}
/* Actual RLE coding. */
@ -542,9 +542,9 @@ func BrotliWriteHuffmanTree(depth []byte, length uint, tree_size *uint, tree []b
}
if value == 0 {
BrotliWriteHuffmanTreeRepetitionsZeros(reps, tree_size, tree, extra_bits_data)
writeHuffmanTreeRepetitionsZeros(reps, tree_size, tree, extra_bits_data)
} else {
BrotliWriteHuffmanTreeRepetitions(previous_value, value, reps, tree_size, tree, extra_bits_data)
writeHuffmanTreeRepetitions(previous_value, value, reps, tree_size, tree, extra_bits_data)
previous_value = value
}
@ -552,7 +552,7 @@ func BrotliWriteHuffmanTree(depth []byte, length uint, tree_size *uint, tree []b
}
}
var BrotliReverseBits_kLut = [16]uint{
var reverseBits_kLut = [16]uint{
0x00,
0x08,
0x04,
@ -571,13 +571,13 @@ var BrotliReverseBits_kLut = [16]uint{
0x0F,
}
func BrotliReverseBits(num_bits uint, bits uint16) uint16 {
var retval uint = BrotliReverseBits_kLut[bits&0x0F]
func reverseBits(num_bits uint, bits uint16) uint16 {
var retval uint = reverseBits_kLut[bits&0x0F]
var i uint
for i = 4; i < num_bits; i += 4 {
retval <<= 4
bits = uint16(bits >> 4)
retval |= BrotliReverseBits_kLut[bits&0x0F]
retval |= reverseBits_kLut[bits&0x0F]
}
retval >>= ((0 - num_bits) & 0x03)
@ -585,11 +585,11 @@ func BrotliReverseBits(num_bits uint, bits uint16) uint16 {
}
/* 0..15 are values for bits */
const MAX_HUFFMAN_BITS = 16
const maxHuffmanBits = 16
func BrotliConvertBitDepthsToSymbols(depth []byte, len uint, bits []uint16) {
var bl_count = [MAX_HUFFMAN_BITS]uint16{0}
var next_code [MAX_HUFFMAN_BITS]uint16
func convertBitDepthsToSymbols(depth []byte, len uint, bits []uint16) {
var bl_count = [maxHuffmanBits]uint16{0}
var next_code [maxHuffmanBits]uint16
var i uint
/* In Brotli, all bit depths are [1..15]
0 bit depth means that the symbol does not exist. */
@ -601,14 +601,14 @@ func BrotliConvertBitDepthsToSymbols(depth []byte, len uint, bits []uint16) {
bl_count[0] = 0
next_code[0] = 0
for i = 1; i < MAX_HUFFMAN_BITS; i++ {
for i = 1; i < maxHuffmanBits; i++ {
code = (code + int(bl_count[i-1])) << 1
next_code[i] = uint16(code)
}
for i = 0; i < len; i++ {
if depth[i] != 0 {
bits[i] = BrotliReverseBits(uint(depth[i]), next_code[depth[i]])
bits[i] = reverseBits(uint(depth[i]), next_code[depth[i]])
next_code[depth[i]]++
}
}

4
entropy_encode_static.go
View File

@ -4317,7 +4317,7 @@ var kStaticCommandCodeBits = [numCommandSymbols]uint16{
2047,
}
func StoreStaticCommandHuffmanTree(storage_ix *uint, storage []byte) {
func storeStaticCommandHuffmanTree(storage_ix *uint, storage []byte) {
BrotliWriteBits(56, 0x92624416307003, storage_ix, storage)
BrotliWriteBits(3, 0x00000000, storage_ix, storage)
}
@ -4389,6 +4389,6 @@ var kStaticDistanceCodeBits = [64]uint16{
63,
}
func StoreStaticDistanceHuffmanTree(storage_ix *uint, storage []byte) {
func storeStaticDistanceHuffmanTree(storage_ix *uint, storage []byte) {
BrotliWriteBits(28, 0x0369DC03, storage_ix, storage)
}

6
fast_log.go
View File

@ -29,7 +29,7 @@ import "math"
*/
/* Utilities for fast computation of logarithms. */
func Log2FloorNonZero(n uint) uint32 {
func log2FloorNonZero(n uint) uint32 {
/* TODO: generalize and move to platform.h */
var result uint32 = 0
for {
@ -305,10 +305,8 @@ var kLog2Table = []float32{
7.9943534368588578,
}
const LOG_2_INV = 1.4426950408889634
/* Faster logarithm for small integers, with the property of log2(0) == 0. */
func FastLog2(v uint) float64 {
func fastLog2(v uint) float64 {
if v < uint(len(kLog2Table)) {
return float64(kLog2Table[v])
}

2
find_match_length.go
View File

@ -15,7 +15,7 @@ package brotli
*/
/* Function to find maximal matching prefixes of strings. */
func FindMatchLengthWithLimit(s1 []byte, s2 []byte, limit uint) uint {
func findMatchLengthWithLimit(s1 []byte, s2 []byte, limit uint) uint {
var matched uint = 0
for matched < limit && s1[matched] == s2[matched] {
matched++

80
h10.go
View File

@ -15,15 +15,15 @@ import "encoding/binary"
position in the input data. The binary tree is sorted by the lexicographic
order of the sequences, and it is also a max-heap with respect to the
starting positions. */
func (*H10) HashTypeLength() uint {
func (*h10) HashTypeLength() uint {
return 4
}
func (*H10) StoreLookahead() uint {
func (*h10) StoreLookahead() uint {
return 128
}
func HashBytesH10(data []byte) uint32 {
func hashBytesH10(data []byte) uint32 {
var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32
/* The higher bits contain more mixture from the multiplication,
@ -31,30 +31,22 @@ func HashBytesH10(data []byte) uint32 {
return h >> (32 - 17)
}
type H10 struct {
HasherCommon
type h10 struct {
hasherCommon
window_mask_ uint
buckets_ [1 << 17]uint32
invalid_pos_ uint32
forest []uint32
}
func SelfH10(handle HasherHandle) *H10 {
return handle.(*H10)
}
func ForestH10(self *H10) []uint32 {
return []uint32(self.forest)
}
func (h *H10) Initialize(params *BrotliEncoderParams) {
func (h *h10) Initialize(params *encoderParams) {
h.window_mask_ = (1 << params.lgwin) - 1
h.invalid_pos_ = uint32(0 - h.window_mask_)
var num_nodes uint = uint(1) << params.lgwin
h.forest = make([]uint32, 2*num_nodes)
}
func (h *H10) Prepare(one_shot bool, input_size uint, data []byte) {
func (h *h10) Prepare(one_shot bool, input_size uint, data []byte) {
var invalid_pos uint32 = h.invalid_pos_
var i uint32
for i = 0; i < 1<<17; i++ {
@ -62,11 +54,11 @@ func (h *H10) Prepare(one_shot bool, input_size uint, data []byte) {
}
}
func LeftChildIndexH10(self *H10, pos uint) uint {
func leftChildIndexH10(self *h10, pos uint) uint {
return 2 * (pos & self.window_mask_)
}
func RightChildIndexH10(self *H10, pos uint) uint {
func rightChildIndexH10(self *h10, pos uint) uint {
return 2*(pos&self.window_mask_) + 1
}
@ -80,15 +72,15 @@ func RightChildIndexH10(self *H10, pos uint) uint {
current (incomplete) sequence.
This function must be called with increasing cur_ix positions. */
func StoreAndFindMatchesH10(self *H10, data []byte, cur_ix uint, ring_buffer_mask uint, max_length uint, max_backward uint, best_len *uint, matches []BackwardMatch) []BackwardMatch {
func storeAndFindMatchesH10(self *h10, data []byte, cur_ix uint, ring_buffer_mask uint, max_length uint, max_backward uint, best_len *uint, matches []backwardMatch) []backwardMatch {
var cur_ix_masked uint = cur_ix & ring_buffer_mask
var max_comp_len uint = brotli_min_size_t(max_length, 128)
var should_reroot_tree bool = (max_length >= 128)
var key uint32 = HashBytesH10(data[cur_ix_masked:])
var forest []uint32 = ForestH10(self)
var key uint32 = hashBytesH10(data[cur_ix_masked:])
var forest []uint32 = self.forest
var prev_ix uint = uint(self.buckets_[key])
var node_left uint = LeftChildIndexH10(self, cur_ix)
var node_right uint = RightChildIndexH10(self, cur_ix)
var node_left uint = leftChildIndexH10(self, cur_ix)
var node_right uint = rightChildIndexH10(self, cur_ix)
var best_len_left uint = 0
var best_len_right uint = 0
var depth_remaining uint
@ -122,17 +114,17 @@ func StoreAndFindMatchesH10(self *H10, data []byte, cur_ix uint, ring_buffer_mas
var cur_len uint = brotli_min_size_t(best_len_left, best_len_right)
var len uint
assert(cur_len <= 128)
len = cur_len + FindMatchLengthWithLimit(data[cur_ix_masked+cur_len:], data[prev_ix_masked+cur_len:], max_length-cur_len)
len = cur_len + findMatchLengthWithLimit(data[cur_ix_masked+cur_len:], data[prev_ix_masked+cur_len:], max_length-cur_len)
if matches != nil && len > *best_len {
*best_len = uint(len)
InitBackwardMatch(&matches[0], backward, uint(len))
initBackwardMatch(&matches[0], backward, uint(len))
matches = matches[1:]
}
if len >= max_comp_len {
if should_reroot_tree {
forest[node_left] = forest[LeftChildIndexH10(self, prev_ix)]
forest[node_right] = forest[RightChildIndexH10(self, prev_ix)]
forest[node_left] = forest[leftChildIndexH10(self, prev_ix)]
forest[node_right] = forest[rightChildIndexH10(self, prev_ix)]
}
break
@ -144,7 +136,7 @@ func StoreAndFindMatchesH10(self *H10, data []byte, cur_ix uint, ring_buffer_mas
forest[node_left] = uint32(prev_ix)
}
node_left = RightChildIndexH10(self, prev_ix)
node_left = rightChildIndexH10(self, prev_ix)
prev_ix = uint(forest[node_left])
} else {
best_len_right = uint(len)
@ -152,7 +144,7 @@ func StoreAndFindMatchesH10(self *H10, data []byte, cur_ix uint, ring_buffer_mas
forest[node_right] = uint32(prev_ix)
}
node_right = LeftChildIndexH10(self, prev_ix)
node_right = leftChildIndexH10(self, prev_ix)
prev_ix = uint(forest[node_right])
}
}
@ -168,8 +160,8 @@ func StoreAndFindMatchesH10(self *H10, data []byte, cur_ix uint, ring_buffer_mas
matches in matches[0] to matches[*num_matches - 1]. The matches will be
sorted by strictly increasing length and (non-strictly) increasing
distance. */
func FindAllMatchesH10(handle HasherHandle, dictionary *BrotliEncoderDictionary, data []byte, ring_buffer_mask uint, cur_ix uint, max_length uint, max_backward uint, gap uint, params *BrotliEncoderParams, matches []BackwardMatch) uint {
var orig_matches []BackwardMatch = matches
func findAllMatchesH10(handle *h10, dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, cur_ix uint, max_length uint, max_backward uint, gap uint, params *encoderParams, matches []backwardMatch) uint {
var orig_matches []backwardMatch = matches
var cur_ix_masked uint = cur_ix & ring_buffer_mask
var best_len uint = 1
var short_match_max_backward uint
@ -196,17 +188,17 @@ func FindAllMatchesH10(handle HasherHandle, dictionary *BrotliEncoderDictionary,
continue
}
{
var len uint = FindMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
if len > best_len {
best_len = uint(len)
InitBackwardMatch(&matches[0], backward, uint(len))
initBackwardMatch(&matches[0], backward, uint(len))
matches = matches[1:]
}
}
}
if best_len < max_length {
matches = StoreAndFindMatchesH10(SelfH10(handle), data, cur_ix, ring_buffer_mask, max_length, max_backward, &best_len, matches)
matches = storeAndFindMatchesH10(handle, data, cur_ix, ring_buffer_mask, max_length, max_backward, &best_len, matches)
}
for i = 0; i <= BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN; i++ {
@ -222,7 +214,7 @@ func FindAllMatchesH10(handle HasherHandle, dictionary *BrotliEncoderDictionary,
if dict_id < kInvalidMatch {
var distance uint = max_backward + gap + uint(dict_id>>5) + 1
if distance <= params.dist.max_distance {
InitDictionaryBackwardMatch(&matches[0], distance, l, uint(dict_id&31))
initDictionaryBackwardMatch(&matches[0], distance, l, uint(dict_id&31))
matches = matches[1:]
}
}
@ -236,13 +228,13 @@ func FindAllMatchesH10(handle HasherHandle, dictionary *BrotliEncoderDictionary,
/* Stores the hash of the next 4 bytes and re-roots the binary tree at the
current sequence, without returning any matches.
REQUIRES: ix + 128 <= end-of-current-block */
func (h *H10) Store(data []byte, mask uint, ix uint) {
var max_backward uint = h.window_mask_ - BROTLI_WINDOW_GAP + 1
func (h *h10) Store(data []byte, mask uint, ix uint) {
var max_backward uint = h.window_mask_ - windowGap + 1
/* Maximum distance is window size - 16, see section 9.1. of the spec. */
StoreAndFindMatchesH10(h, data, ix, mask, 128, max_backward, nil, nil)
storeAndFindMatchesH10(h, data, ix, mask, 128, max_backward, nil, nil)
}
func (h *H10) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
func (h *h10) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
var i uint = ix_start
var j uint = ix_start
if ix_start+63 <= ix_end {
@ -260,7 +252,7 @@ func (h *H10) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
}
}
func (h *H10) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) {
func (h *h10) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) {
if num_bytes >= h.HashTypeLength()-1 && position >= 128 {
var i_start uint = position - 128 + 1
var i_end uint = brotli_min_size_t(position, i_start+num_bytes)
@ -274,23 +266,23 @@ func (h *H10) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []
Furthermore, we have to make sure that we don't look further back
from the start of the next block than the window size, otherwise we
could access already overwritten areas of the ring-buffer. */
var max_backward uint = h.window_mask_ - brotli_max_size_t(BROTLI_WINDOW_GAP-1, position-i)
var max_backward uint = h.window_mask_ - brotli_max_size_t(windowGap-1, position-i)
/* We know that i + 128 <= position + num_bytes, i.e. the
end of the current block and that we have at least
128 tail in the ring-buffer. */
StoreAndFindMatchesH10(h, ringbuffer, i, ringbuffer_mask, 128, max_backward, nil, nil)
storeAndFindMatchesH10(h, ringbuffer, i, ringbuffer_mask, 128, max_backward, nil, nil)
}
}
}
/* MAX_NUM_MATCHES == 64 + MAX_TREE_SEARCH_DEPTH */
const MAX_NUM_MATCHES_H10 = 128
const maxNumMatchesH10 = 128
func (*H10) FindLongestMatch(dictionary *BrotliEncoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *HasherSearchResult) {
func (*h10) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
panic("unimplemented")
}
func (*H10) PrepareDistanceCache(distance_cache []int) {
func (*h10) PrepareDistanceCache(distance_cache []int) {
panic("unimplemented")
}

56
h5.go
View File

@ -15,16 +15,16 @@ import "encoding/binary"
This is a hash map of fixed size (bucket_size_) to a ring buffer of
fixed size (block_size_). The ring buffer contains the last block_size_
index positions of the given hash key in the compressed data. */
func (*H5) HashTypeLength() uint {
func (*h5) HashTypeLength() uint {
return 4
}
func (*H5) StoreLookahead() uint {
func (*h5) StoreLookahead() uint {
return 4
}
/* HashBytes is the function that chooses the bucket to place the address in. */
func HashBytesH5(data []byte, shift int) uint32 {
func hashBytesH5(data []byte, shift int) uint32 {
var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32
/* The higher bits contain more mixture from the multiplication,
@ -32,8 +32,8 @@ func HashBytesH5(data []byte, shift int) uint32 {
return uint32(h >> uint(shift))
}
type H5 struct {
HasherCommon
type h5 struct {
hasherCommon
bucket_size_ uint
block_size_ uint
hash_shift_ int
@ -42,19 +42,7 @@ type H5 struct {
buckets []uint32
}
func SelfH5(handle HasherHandle) *H5 {
return handle.(*H5)
}
func NumH5(self *H5) []uint16 {
return []uint16(self.num)
}
func BucketsH5(self *H5) []uint32 {
return []uint32(self.buckets)
}
func (h *H5) Initialize(params *BrotliEncoderParams) {
func (h *h5) Initialize(params *encoderParams) {
h.hash_shift_ = 32 - h.params.bucket_bits
h.bucket_size_ = uint(1) << uint(h.params.bucket_bits)
h.block_size_ = uint(1) << uint(h.params.block_bits)
@ -63,14 +51,14 @@ func (h *H5) Initialize(params *BrotliEncoderParams) {
h.buckets = make([]uint32, h.block_size_*h.bucket_size_)
}
func (h *H5) Prepare(one_shot bool, input_size uint, data []byte) {
func (h *h5) Prepare(one_shot bool, input_size uint, data []byte) {
var num []uint16 = h.num
var partial_prepare_threshold uint = h.bucket_size_ >> 6
/* Partial preparation is 100 times slower (per socket). */
if one_shot && input_size <= partial_prepare_threshold {
var i uint
for i = 0; i < input_size; i++ {
var key uint32 = HashBytesH5(data[i:], h.hash_shift_)
var key uint32 = hashBytesH5(data[i:], h.hash_shift_)
num[key] = 0
}
} else {
@ -82,23 +70,23 @@ func (h *H5) Prepare(one_shot bool, input_size uint, data []byte) {
/* Look at 4 bytes at &data[ix & mask].
Compute a hash from these, and store the value of ix at that position. */
func (h *H5) Store(data []byte, mask uint, ix uint) {
func (h *h5) Store(data []byte, mask uint, ix uint) {
var num []uint16 = h.num
var key uint32 = HashBytesH5(data[ix&mask:], h.hash_shift_)
var key uint32 = hashBytesH5(data[ix&mask:], h.hash_shift_)
var minor_ix uint = uint(num[key]) & uint(h.block_mask_)
var offset uint = minor_ix + uint(key<<uint(h.params.block_bits))
h.buckets[offset] = uint32(ix)
num[key]++
}
func (h *H5) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
func (h *h5) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
var i uint
for i = ix_start; i < ix_end; i++ {
h.Store(data, mask, i)
}
}
func (h *H5) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) {
func (h *h5) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) {
if num_bytes >= h.HashTypeLength()-1 && position >= 3 {
/* Prepare the hashes for three last bytes of the last write.
These could not be calculated before, since they require knowledge
@ -109,8 +97,8 @@ func (h *H5) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []b
}
}
func (h *H5) PrepareDistanceCache(distance_cache []int) {
PrepareDistanceCache(distance_cache, h.params.num_last_distances_to_check)
func (h *h5) PrepareDistanceCache(distance_cache []int) {
prepareDistanceCache(distance_cache, h.params.num_last_distances_to_check)
}
/* Find a longest backward match of &data[cur_ix] up to the length of
@ -124,7 +112,7 @@ func (h *H5) PrepareDistanceCache(distance_cache []int) {
Does not look for matches further away than max_backward.
Writes the best match into |out|.
|out|->score is updated only if a better match is found. */
func (h *H5) FindLongestMatch(dictionary *BrotliEncoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *HasherSearchResult) {
func (h *h5) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
var num []uint16 = h.num
var buckets []uint32 = h.buckets
var cur_ix_masked uint = cur_ix & ring_buffer_mask
@ -156,15 +144,15 @@ func (h *H5) FindLongestMatch(dictionary *BrotliEncoderDictionary, data []byte,
continue
}
{
var len uint = FindMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
if len >= 3 || (len == 2 && i < 2) {
/* Comparing for >= 2 does not change the semantics, but just saves for
a few unnecessary binary logarithms in backward reference score,
since we are not interested in such short matches. */
var score uint = BackwardReferenceScoreUsingLastDistance(uint(len))
var score uint = backwardReferenceScoreUsingLastDistance(uint(len))
if best_score < score {
if i != 0 {
score -= BackwardReferencePenaltyUsingLastDistance(i)
score -= backwardReferencePenaltyUsingLastDistance(i)
}
if best_score < score {
best_score = score
@ -178,7 +166,7 @@ func (h *H5) FindLongestMatch(dictionary *BrotliEncoderDictionary, data []byte,
}
}
{
var key uint32 = HashBytesH5(data[cur_ix_masked:], h.hash_shift_)
var key uint32 = hashBytesH5(data[cur_ix_masked:], h.hash_shift_)
bucket = buckets[key<<uint(h.params.block_bits):]
var down uint
if uint(num[key]) > h.block_size_ {
@ -200,12 +188,12 @@ func (h *H5) FindLongestMatch(dictionary *BrotliEncoderDictionary, data []byte,
continue
}
{
var len uint = FindMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
if len >= 4 {
/* Comparing for >= 3 does not change the semantics, but just saves
for a few unnecessary binary logarithms in backward reference
score, since we are not interested in such short matches. */
var score uint = BackwardReferenceScore(uint(len), backward)
var score uint = backwardReferenceScore(uint(len), backward)
if best_score < score {
best_score = score
best_len = uint(len)
@ -222,6 +210,6 @@ func (h *H5) FindLongestMatch(dictionary *BrotliEncoderDictionary, data []byte,
}
if min_score == out.score {
SearchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false)
searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false)
}
}

56
h6.go
View File

@ -15,16 +15,16 @@ import "encoding/binary"
This is a hash map of fixed size (bucket_size_) to a ring buffer of
fixed size (block_size_). The ring buffer contains the last block_size_
index positions of the given hash key in the compressed data. */
func (*H6) HashTypeLength() uint {
func (*h6) HashTypeLength() uint {
return 8
}
func (*H6) StoreLookahead() uint {
func (*h6) StoreLookahead() uint {
return 8
}
/* HashBytes is the function that chooses the bucket to place the address in. */
func HashBytesH6(data []byte, mask uint64, shift int) uint32 {
func hashBytesH6(data []byte, mask uint64, shift int) uint32 {
var h uint64 = (binary.LittleEndian.Uint64(data) & mask) * kHashMul64Long
/* The higher bits contain more mixture from the multiplication,
@ -32,8 +32,8 @@ func HashBytesH6(data []byte, mask uint64, shift int) uint32 {
return uint32(h >> uint(shift))
}
type H6 struct {
HasherCommon
type h6 struct {
hasherCommon
bucket_size_ uint
block_size_ uint
hash_shift_ int
@ -43,19 +43,7 @@ type H6 struct {
buckets []uint32
}
func SelfH6(handle HasherHandle) *H6 {
return handle.(*H6)
}
func NumH6(self *H6) []uint16 {
return []uint16(self.num)
}
func BucketsH6(self *H6) []uint32 {
return []uint32(self.buckets)
}
func (h *H6) Initialize(params *BrotliEncoderParams) {
func (h *h6) Initialize(params *encoderParams) {
h.hash_shift_ = 64 - h.params.bucket_bits
h.hash_mask_ = (^(uint64(0))) >> uint(64-8*h.params.hash_len)
h.bucket_size_ = uint(1) << uint(h.params.bucket_bits)
@ -65,14 +53,14 @@ func (h *H6) Initialize(params *BrotliEncoderParams) {
h.buckets = make([]uint32, h.block_size_*h.bucket_size_)
}
func (h *H6) Prepare(one_shot bool, input_size uint, data []byte) {
func (h *h6) Prepare(one_shot bool, input_size uint, data []byte) {
var num []uint16 = h.num
var partial_prepare_threshold uint = h.bucket_size_ >> 6
/* Partial preparation is 100 times slower (per socket). */
if one_shot && input_size <= partial_prepare_threshold {
var i uint
for i = 0; i < input_size; i++ {
var key uint32 = HashBytesH6(data[i:], h.hash_mask_, h.hash_shift_)
var key uint32 = hashBytesH6(data[i:], h.hash_mask_, h.hash_shift_)
num[key] = 0
}
} else {
@ -84,23 +72,23 @@ func (h *H6) Prepare(one_shot bool, input_size uint, data []byte) {
/* Look at 4 bytes at &data[ix & mask].
Compute a hash from these, and store the value of ix at that position. */
func (h *H6) Store(data []byte, mask uint, ix uint) {
func (h *h6) Store(data []byte, mask uint, ix uint) {
var num []uint16 = h.num
var key uint32 = HashBytesH6(data[ix&mask:], h.hash_mask_, h.hash_shift_)
var key uint32 = hashBytesH6(data[ix&mask:], h.hash_mask_, h.hash_shift_)
var minor_ix uint = uint(num[key]) & uint(h.block_mask_)
var offset uint = minor_ix + uint(key<<uint(h.params.block_bits))
h.buckets[offset] = uint32(ix)
num[key]++
}
func (h *H6) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
func (h *h6) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) {
var i uint
for i = ix_start; i < ix_end; i++ {
h.Store(data, mask, i)
}
}
func (h *H6) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) {
func (h *h6) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) {
if num_bytes >= h.HashTypeLength()-1 && position >= 3 {
/* Prepare the hashes for three last bytes of the last write.
These could not be calculated before, since they require knowledge
@ -111,8 +99,8 @@ func (h *H6) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []b
}
}
func (h *H6) PrepareDistanceCache(distance_cache []int) {
PrepareDistanceCache(distance_cache, h.params.num_last_distances_to_check)
func (h *h6) PrepareDistanceCache(distance_cache []int) {
prepareDistanceCache(distance_cache, h.params.num_last_distances_to_check)
}
/* Find a longest backward match of &data[cur_ix] up to the length of
@ -126,7 +114,7 @@ func (h *H6) PrepareDistanceCache(distance_cache []int) {
Does not look for matches further away than max_backward.
Writes the best match into |out|.
|out|->score is updated only if a better match is found. */
func (h *H6) FindLongestMatch(dictionary *BrotliEncoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *HasherSearchResult) {
func (h *h6) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
var num []uint16 = h.num
var buckets []uint32 = h.buckets
var cur_ix_masked uint = cur_ix & ring_buffer_mask
@ -158,15 +146,15 @@ func (h *H6) FindLongestMatch(dictionary *BrotliEncoderDictionary, data []byte,
continue
}
{
var len uint = FindMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
if len >= 3 || (len == 2 && i < 2) {
/* Comparing for >= 2 does not change the semantics, but just saves for
a few unnecessary binary logarithms in backward reference score,
since we are not interested in such short matches. */
var score uint = BackwardReferenceScoreUsingLastDistance(uint(len))
var score uint = backwardReferenceScoreUsingLastDistance(uint(len))
if best_score < score {
if i != 0 {
score -= BackwardReferencePenaltyUsingLastDistance(i)
score -= backwardReferencePenaltyUsingLastDistance(i)
}
if best_score < score {
best_score = score
@ -180,7 +168,7 @@ func (h *H6) FindLongestMatch(dictionary *BrotliEncoderDictionary, data []byte,
}
}
{
var key uint32 = HashBytesH6(data[cur_ix_masked:], h.hash_mask_, h.hash_shift_)
var key uint32 = hashBytesH6(data[cur_ix_masked:], h.hash_mask_, h.hash_shift_)
bucket = buckets[key<<uint(h.params.block_bits):]
var down uint
if uint(num[key]) > h.block_size_ {
@ -202,12 +190,12 @@ func (h *H6) FindLongestMatch(dictionary *BrotliEncoderDictionary, data []byte,
continue
}
{
var len uint = FindMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
if len >= 4 {
/* Comparing for >= 3 does not change the semantics, but just saves
for a few unnecessary binary logarithms in backward reference
score, since we are not interested in such short matches. */
var score uint = BackwardReferenceScore(uint(len), backward)
var score uint = backwardReferenceScore(uint(len), backward)
if best_score < score {
best_score = score
best_len = uint(len)
@ -224,6 +212,6 @@ func (h *H6) FindLongestMatch(dictionary *BrotliEncoderDictionary, data []byte,
}
if min_score == out.score {
SearchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false)
searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false)
}
}

93
hash.go
View File

@ -18,31 +18,31 @@ import (
* HasherHandle by value.
*
* Typically hasher data consists of 3 sections:
* * HasherCommon structure
* * hasherCommon structure
* * private structured hasher data, depending on hasher type
* * private dynamic hasher data, depending on hasher type and parameters
*
*/
type HasherCommon struct {
params BrotliHasherParams
type hasherCommon struct {
params hasherParams
is_prepared_ bool
dict_num_lookups uint
dict_num_matches uint
}
func (h *HasherCommon) Common() *HasherCommon {
func (h *hasherCommon) Common() *hasherCommon {
return h
}
type HasherHandle interface {
Common() *HasherCommon
Initialize(params *BrotliEncoderParams)
type hasherHandle interface {
Common() *hasherCommon
Initialize(params *encoderParams)
Prepare(one_shot bool, input_size uint, data []byte)
StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint)
HashTypeLength() uint
StoreLookahead() uint
PrepareDistanceCache(distance_cache []int)
FindLongestMatch(dictionary *BrotliEncoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *HasherSearchResult)
FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult)
StoreRange(data []byte, mask uint, ix_start uint, ix_end uint)
Store(data []byte, mask uint, ix uint)
}
@ -55,7 +55,7 @@ var kCutoffTransformsCount uint32 = 10
/* 0+0, 4+8, 8+19, 12+11, 16+26, 20+43, 24+32, 28+20, 32+27, 36+28 */
var kCutoffTransforms uint64 = 0x071B520ADA2D3200
type HasherSearchResult struct {
type hasherSearchResult struct {
len uint
distance uint
score uint
@ -74,7 +74,7 @@ var kHashMul64 uint64 = 0x1E35A7BD1E35A7BD
var kHashMul64Long uint64 = 0x1FE35A7BD3579BD3
func Hash14(data []byte) uint32 {
func hash14(data []byte) uint32 {
var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32
/* The higher bits contain more mixture from the multiplication,
@ -82,7 +82,7 @@ func Hash14(data []byte) uint32 {
return h >> (32 - 14)
}
func PrepareDistanceCache(distance_cache []int, num_distances int) {
func prepareDistanceCache(distance_cache []int, num_distances int) {
if num_distances > 4 {
var last_distance int = distance_cache[0]
distance_cache[4] = last_distance - 1
@ -103,12 +103,12 @@ func PrepareDistanceCache(distance_cache []int, num_distances int) {
}
}
const BROTLI_LITERAL_BYTE_SCORE = 135
const literalByteScore = 135
const BROTLI_DISTANCE_BIT_PENALTY = 30
const distanceBitPenalty = 30
/* Score must be positive after applying maximal penalty. */
const BROTLI_SCORE_BASE = (BROTLI_DISTANCE_BIT_PENALTY * 8 * 8)
const scoreBase = (distanceBitPenalty * 8 * 8)
/* Usually, we always choose the longest backward reference. This function
allows for the exception of that rule.
@ -126,19 +126,19 @@ const BROTLI_SCORE_BASE = (BROTLI_DISTANCE_BIT_PENALTY * 8 * 8)
than the saved literals.
backward_reference_offset MUST be positive. */
func BackwardReferenceScore(copy_length uint, backward_reference_offset uint) uint {
return BROTLI_SCORE_BASE + BROTLI_LITERAL_BYTE_SCORE*uint(copy_length) - BROTLI_DISTANCE_BIT_PENALTY*uint(Log2FloorNonZero(backward_reference_offset))
func backwardReferenceScore(copy_length uint, backward_reference_offset uint) uint {
return scoreBase + literalByteScore*uint(copy_length) - distanceBitPenalty*uint(log2FloorNonZero(backward_reference_offset))
}
func BackwardReferenceScoreUsingLastDistance(copy_length uint) uint {
return BROTLI_LITERAL_BYTE_SCORE*uint(copy_length) + BROTLI_SCORE_BASE + 15
func backwardReferenceScoreUsingLastDistance(copy_length uint) uint {
return literalByteScore*uint(copy_length) + scoreBase + 15
}
func BackwardReferencePenaltyUsingLastDistance(distance_short_code uint) uint {
func backwardReferencePenaltyUsingLastDistance(distance_short_code uint) uint {
return uint(39) + ((0x1CA10 >> (distance_short_code & 0xE)) & 0xE)
}
func TestStaticDictionaryItem(dictionary *BrotliEncoderDictionary, item uint, data []byte, max_length uint, max_backward uint, max_distance uint, out *HasherSearchResult) bool {
func testStaticDictionaryItem(dictionary *encoderDictionary, item uint, data []byte, max_length uint, max_backward uint, max_distance uint, out *hasherSearchResult) bool {
var len uint
var word_idx uint
var offset uint
@ -152,7 +152,7 @@ func TestStaticDictionaryItem(dictionary *BrotliEncoderDictionary, item uint, da
return false
}
matchlen = FindMatchLengthWithLimit(data, dictionary.words.data[offset:], uint(len))
matchlen = findMatchLengthWithLimit(data, dictionary.words.data[offset:], uint(len))
if matchlen+uint(dictionary.cutoffTransformsCount) <= len || matchlen == 0 {
return false
}
@ -166,7 +166,7 @@ func TestStaticDictionaryItem(dictionary *BrotliEncoderDictionary, item uint, da
return false
}
score = BackwardReferenceScore(matchlen, backward)
score = backwardReferenceScore(matchlen, backward)
if score < out.score {
return false
}
@ -178,15 +178,15 @@ func TestStaticDictionaryItem(dictionary *BrotliEncoderDictionary, item uint, da
return true
}
func SearchInStaticDictionary(dictionary *BrotliEncoderDictionary, handle HasherHandle, data []byte, max_length uint, max_backward uint, max_distance uint, out *HasherSearchResult, shallow bool) {
func searchInStaticDictionary(dictionary *encoderDictionary, handle hasherHandle, data []byte, max_length uint, max_backward uint, max_distance uint, out *hasherSearchResult, shallow bool) {
var key uint
var i uint
var self *HasherCommon = handle.Common()
var self *hasherCommon = handle.Common()
if self.dict_num_matches < self.dict_num_lookups>>7 {
return
}
key = uint(Hash14(data) << 1)
key = uint(hash14(data) << 1)
for i = 0; ; (func() { i++; key++ })() {
var tmp uint
if shallow {
@ -200,7 +200,7 @@ func SearchInStaticDictionary(dictionary *BrotliEncoderDictionary, handle Hasher
var item uint = uint(dictionary.hash_table[key])
self.dict_num_lookups++
if item != 0 {
var item_matches bool = TestStaticDictionaryItem(dictionary, item, data, max_length, max_backward, max_distance, out)
var item_matches bool = testStaticDictionaryItem(dictionary, item, data, max_length, max_backward, max_distance, out)
if item_matches {
self.dict_num_matches++
}
@ -208,17 +208,17 @@ func SearchInStaticDictionary(dictionary *BrotliEncoderDictionary, handle Hasher
}
}
type BackwardMatch struct {
type backwardMatch struct {
distance uint32
length_and_code uint32
}
func InitBackwardMatch(self *BackwardMatch, dist uint, len uint) {
func initBackwardMatch(self *backwardMatch, dist uint, len uint) {
self.distance = uint32(dist)
self.length_and_code = uint32(len << 5)
}
func InitDictionaryBackwardMatch(self *BackwardMatch, dist uint, len uint, len_code uint) {
func initDictionaryBackwardMatch(self *backwardMatch, dist uint, len uint, len_code uint) {
self.distance = uint32(dist)
var tmp uint
if len == len_code {
@ -229,34 +229,27 @@ func InitDictionaryBackwardMatch(self *BackwardMatch, dist uint, len uint, len_c
self.length_and_code = uint32(len<<5 | tmp)
}
func BackwardMatchLength(self *BackwardMatch) uint {
func backwardMatchLength(self *backwardMatch) uint {
return uint(self.length_and_code >> 5)
}
func BackwardMatchLengthCode(self *BackwardMatch) uint {
func backwardMatchLengthCode(self *backwardMatch) uint {
var code uint = uint(self.length_and_code) & 31
if code != 0 {
return code
} else {
return BackwardMatchLength(self)
return backwardMatchLength(self)
}
}
func DestroyHasher(handle *HasherHandle) {
if *handle == nil {
return
}
*handle = nil
}
func HasherReset(handle HasherHandle) {
func hasherReset(handle hasherHandle) {
if handle == nil {
return
}
handle.Common().is_prepared_ = false
}
func newHasher(typ int) HasherHandle {
func newHasher(typ int) hasherHandle {
switch typ {
case 2:
return &hashLongestMatchQuickly{
@ -280,11 +273,11 @@ func newHasher(typ int) HasherHandle {
useDictionary: true,
}
case 5:
return new(H5)
return new(h5)
case 6:
return new(H6)
return new(h6)
case 10:
return new(H10)
return new(h10)
case 35:
return &hashComposite{
ha: newHasher(3),
@ -333,9 +326,9 @@ func newHasher(typ int) HasherHandle {
panic(fmt.Sprintf("unknown hasher type: %d", typ))
}
func HasherSetup(handle *HasherHandle, params *BrotliEncoderParams, data []byte, position uint, input_size uint, is_last bool) {
var self HasherHandle = nil
var common *HasherCommon = nil
func hasherSetup(handle *hasherHandle, params *encoderParams, data []byte, position uint, input_size uint, is_last bool) {
var self hasherHandle = nil
var common *hasherCommon = nil
var one_shot bool = (position == 0 && is_last)
if *handle == nil {
ChooseHasher(params, &params.hasher)
@ -361,9 +354,9 @@ func HasherSetup(handle *HasherHandle, params *BrotliEncoderParams, data []byte,
}
}
func InitOrStitchToPreviousBlock(handle *HasherHandle, data []byte, mask uint, params *BrotliEncoderParams, position uint, input_size uint, is_last bool) {
var self HasherHandle
HasherSetup(handle, params, data, position, input_size, is_last)
func initOrStitchToPreviousBlock(handle *hasherHandle, data []byte, mask uint, params *encoderParams, position uint, input_size uint, is_last bool) {
var self hasherHandle
hasherSetup(handle, params, data, position, input_size, is_last)
self = *handle
self.StitchToPreviousBlock(input_size, position, data, mask)
}

16
hash_composite.go
View File

@ -32,13 +32,13 @@ func (h *hashComposite) StoreLookahead() uint {
}
type hashComposite struct {
HasherCommon
ha HasherHandle
hb HasherHandle
params *BrotliEncoderParams
hasherCommon
ha hasherHandle
hb hasherHandle
params *encoderParams
}
func (h *hashComposite) Initialize(params *BrotliEncoderParams) {
func (h *hashComposite) Initialize(params *encoderParams) {
h.params = params
}
@ -48,8 +48,8 @@ func (h *hashComposite) Initialize(params *BrotliEncoderParams) {
those params to all hashers InitializehashComposite */
func (h *hashComposite) Prepare(one_shot bool, input_size uint, data []byte) {
if h.ha == nil {
var common_a *HasherCommon
var common_b *HasherCommon
var common_a *hasherCommon
var common_b *hasherCommon
common_a = h.ha.Common()
common_a.params = h.params.hasher
@ -90,7 +90,7 @@ func (h *hashComposite) PrepareDistanceCache(distance_cache []int) {
h.hb.PrepareDistanceCache(distance_cache)
}
func (h *hashComposite) FindLongestMatch(dictionary *BrotliEncoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *HasherSearchResult) {
func (h *hashComposite) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
h.ha.FindLongestMatch(dictionary, data, ring_buffer_mask, distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out)
h.hb.FindLongestMatch(dictionary, data, ring_buffer_mask, distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out)
}

20
hash_forgetful_chain.go
View File

@ -38,7 +38,7 @@ type slot struct {
}
type hashForgetfulChain struct {
HasherCommon
hasherCommon
bucketBits uint
numBanks uint
@ -53,7 +53,7 @@ type hashForgetfulChain struct {
max_hops uint
}
func (h *hashForgetfulChain) Initialize(params *BrotliEncoderParams) {
func (h *hashForgetfulChain) Initialize(params *encoderParams) {
var q uint
if params.quality > 6 {
q = 7
@ -144,7 +144,7 @@ func (h *hashForgetfulChain) StitchToPreviousBlock(num_bytes uint, position uint
}
func (h *hashForgetfulChain) PrepareDistanceCache(distance_cache []int) {
PrepareDistanceCache(distance_cache, h.numLastDistancesToCheck)
prepareDistanceCache(distance_cache, h.numLastDistancesToCheck)
}
/* Find a longest backward match of &data[cur_ix] up to the length of
@ -158,7 +158,7 @@ func (h *hashForgetfulChain) PrepareDistanceCache(distance_cache []int) {
Does not look for matches further away than max_backward.
Writes the best match into |out|.
|out|->score is updated only if a better match is found. */
func (h *hashForgetfulChain) FindLongestMatch(dictionary *BrotliEncoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *HasherSearchResult) {
func (h *hashForgetfulChain) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
var cur_ix_masked uint = cur_ix & ring_buffer_mask
var min_score uint = out.score
var best_score uint = out.score
@ -185,12 +185,12 @@ func (h *hashForgetfulChain) FindLongestMatch(dictionary *BrotliEncoderDictionar
prev_ix &= ring_buffer_mask
{
var len uint = FindMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
if len >= 2 {
var score uint = BackwardReferenceScoreUsingLastDistance(uint(len))
var score uint = backwardReferenceScoreUsingLastDistance(uint(len))
if best_score < score {
if i != 0 {
score -= BackwardReferencePenaltyUsingLastDistance(uint(i))
score -= backwardReferencePenaltyUsingLastDistance(uint(i))
}
if best_score < score {
best_score = score
@ -228,12 +228,12 @@ func (h *hashForgetfulChain) FindLongestMatch(dictionary *BrotliEncoderDictionar
continue
}
{
var len uint = FindMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
if len >= 4 {
/* Comparing for >= 3 does not change the semantics, but just saves
for a few unnecessary binary logarithms in backward reference
score, since we are not interested in such short matches. */
var score uint = BackwardReferenceScore(uint(len), backward)
var score uint = backwardReferenceScore(uint(len), backward)
if best_score < score {
best_score = score
best_len = uint(len)
@ -249,6 +249,6 @@ func (h *hashForgetfulChain) FindLongestMatch(dictionary *BrotliEncoderDictionar
}
if out.score == min_score {
SearchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false)
searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false)
}
}

20
hash_longest_match_quickly.go
View File

@ -37,7 +37,7 @@ func (h *hashLongestMatchQuickly) HashBytes(data []byte) uint32 {
This is a hash map of fixed size (1 << 16). Starting from the
given index, 1 buckets are used to store values of a key. */
type hashLongestMatchQuickly struct {
HasherCommon
hasherCommon
bucketBits uint
bucketSweep int
@ -47,7 +47,7 @@ type hashLongestMatchQuickly struct {
buckets []uint32
}
func (h *hashLongestMatchQuickly) Initialize(params *BrotliEncoderParams) {
func (h *hashLongestMatchQuickly) Initialize(params *encoderParams) {
h.buckets = make([]uint32, 1<<h.bucketBits+h.bucketSweep)
}
@ -112,7 +112,7 @@ func (*hashLongestMatchQuickly) PrepareDistanceCache(distance_cache []int) {
Does not look for matches further away than max_backward.
Writes the best match into |out|.
|out|->score is updated only if a better match is found. */
func (h *hashLongestMatchQuickly) FindLongestMatch(dictionary *BrotliEncoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *HasherSearchResult) {
func (h *hashLongestMatchQuickly) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
var best_len_in uint = out.len
var cur_ix_masked uint = cur_ix & ring_buffer_mask
var key uint32 = h.HashBytes(data[cur_ix_masked:])
@ -127,9 +127,9 @@ func (h *hashLongestMatchQuickly) FindLongestMatch(dictionary *BrotliEncoderDict
if prev_ix < cur_ix {
prev_ix &= uint(uint32(ring_buffer_mask))
if compare_char == int(data[prev_ix+best_len]) {
var len uint = FindMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
if len >= 4 {
var score uint = BackwardReferenceScoreUsingLastDistance(uint(len))
var score uint = backwardReferenceScoreUsingLastDistance(uint(len))
if best_score < score {
best_score = score
best_len = uint(len)
@ -164,9 +164,9 @@ func (h *hashLongestMatchQuickly) FindLongestMatch(dictionary *BrotliEncoderDict
return
}
len = FindMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
len = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
if len >= 4 {
var score uint = BackwardReferenceScore(uint(len), backward)
var score uint = backwardReferenceScore(uint(len), backward)
if best_score < score {
out.len = uint(len)
out.distance = backward
@ -191,9 +191,9 @@ func (h *hashLongestMatchQuickly) FindLongestMatch(dictionary *BrotliEncoderDict
continue
}
len = FindMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
len = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length)
if len >= 4 {
var score uint = BackwardReferenceScore(uint(len), backward)
var score uint = backwardReferenceScore(uint(len), backward)
if best_score < score {
best_score = score
best_len = uint(len)
@ -207,7 +207,7 @@ func (h *hashLongestMatchQuickly) FindLongestMatch(dictionary *BrotliEncoderDict
}
if h.useDictionary && min_score == out.score {
SearchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, true)
searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, true)
}
h.buckets[key+uint32((cur_ix>>3)%uint(h.bucketSweep))] = uint32(cur_ix)

10
hash_rolling.go
View File

@ -41,7 +41,7 @@ func (h *hashRolling) HashRollingFunction(state uint32, add byte, rem byte, fact
}
type hashRolling struct {
HasherCommon
hasherCommon
jump int
@ -53,7 +53,7 @@ type hashRolling struct {
factor_remove uint32
}
func (h *hashRolling) Initialize(params *BrotliEncoderParams) {
func (h *hashRolling) Initialize(params *encoderParams) {
h.state = 0
h.next_ix = 0
@ -120,7 +120,7 @@ func (h *hashRolling) StitchToPreviousBlock(num_bytes uint, position uint, ringb
func (*hashRolling) PrepareDistanceCache(distance_cache []int) {
}
func (h *hashRolling) FindLongestMatch(dictionary *BrotliEncoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *HasherSearchResult) {
func (h *hashRolling) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) {
var cur_ix_masked uint = cur_ix & ring_buffer_mask
var pos uint = h.next_ix
@ -150,9 +150,9 @@ func (h *hashRolling) FindLongestMatch(dictionary *BrotliEncoderDictionary, data
var backward uint = uint(uint32(cur_ix - found_ix))
if backward <= max_backward {
var found_ix_masked uint = found_ix & ring_buffer_mask
var len uint = FindMatchLengthWithLimit(data[found_ix_masked:], data[cur_ix_masked:], max_length)
var len uint = findMatchLengthWithLimit(data[found_ix_masked:], data[cur_ix_masked:], max_length)
if len >= 4 && len > out.len {
var score uint = BackwardReferenceScore(uint(len), backward)
var score uint = backwardReferenceScore(uint(len), backward)
if score > out.score {
out.len = uint(len)
out.distance = backward

88
histogram.go
View File

@ -3,33 +3,33 @@ package brotli
import "math"
/* The distance symbols effectively used by "Large Window Brotli" (32-bit). */
const BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS = 544
const numHistogramDistanceSymbols = 544
type HistogramLiteral struct {
type histogramLiteral struct {
data_ [numLiteralSymbols]uint32
total_count_ uint
bit_cost_ float64
}
func HistogramClearLiteral(self *HistogramLiteral) {
func histogramClearLiteral(self *histogramLiteral) {
self.data_ = [numLiteralSymbols]uint32{}
self.total_count_ = 0
self.bit_cost_ = math.MaxFloat64
}
func ClearHistogramsLiteral(array []HistogramLiteral, length uint) {
func clearHistogramsLiteral(array []histogramLiteral, length uint) {
var i uint
for i = 0; i < length; i++ {
HistogramClearLiteral(&array[i:][0])
histogramClearLiteral(&array[i:][0])
}
}
func HistogramAddLiteral(self *HistogramLiteral, val uint) {
func histogramAddLiteral(self *histogramLiteral, val uint) {
self.data_[val]++
self.total_count_++
}
func HistogramAddVectorLiteral(self *HistogramLiteral, p []byte, n uint) {
func histogramAddVectorLiteral(self *histogramLiteral, p []byte, n uint) {
self.total_count_ += n
n += 1
for {
@ -42,7 +42,7 @@ func HistogramAddVectorLiteral(self *HistogramLiteral, p []byte, n uint) {
}
}
func HistogramAddHistogramLiteral(self *HistogramLiteral, v *HistogramLiteral) {
func histogramAddHistogramLiteral(self *histogramLiteral, v *histogramLiteral) {
var i uint
self.total_count_ += v.total_count_
for i = 0; i < numLiteralSymbols; i++ {
@ -50,35 +50,35 @@ func HistogramAddHistogramLiteral(self *HistogramLiteral, v *HistogramLiteral) {
}
}
func HistogramDataSizeLiteral() uint {
func histogramDataSizeLiteral() uint {
return numLiteralSymbols
}
type HistogramCommand struct {
type histogramCommand struct {
data_ [numCommandSymbols]uint32
total_count_ uint
bit_cost_ float64
}
func HistogramClearCommand(self *HistogramCommand) {
func histogramClearCommand(self *histogramCommand) {
self.data_ = [numCommandSymbols]uint32{}
self.total_count_ = 0
self.bit_cost_ = math.MaxFloat64
}
func ClearHistogramsCommand(array []HistogramCommand, length uint) {
func clearHistogramsCommand(array []histogramCommand, length uint) {
var i uint
for i = 0; i < length; i++ {
HistogramClearCommand(&array[i:][0])
histogramClearCommand(&array[i:][0])
}
}
func HistogramAddCommand(self *HistogramCommand, val uint) {
func histogramAddCommand(self *histogramCommand, val uint) {
self.data_[val]++
self.total_count_++
}
func HistogramAddVectorCommand(self *HistogramCommand, p []uint16, n uint) {
func histogramAddVectorCommand(self *histogramCommand, p []uint16, n uint) {
self.total_count_ += n
n += 1
for {
@ -91,7 +91,7 @@ func HistogramAddVectorCommand(self *HistogramCommand, p []uint16, n uint) {
}
}
func HistogramAddHistogramCommand(self *HistogramCommand, v *HistogramCommand) {
func histogramAddHistogramCommand(self *histogramCommand, v *histogramCommand) {
var i uint
self.total_count_ += v.total_count_
for i = 0; i < numCommandSymbols; i++ {
@ -99,35 +99,35 @@ func HistogramAddHistogramCommand(self *HistogramCommand, v *HistogramCommand) {
}
}
func HistogramDataSizeCommand() uint {
func histogramDataSizeCommand() uint {
return numCommandSymbols
}
type HistogramDistance struct {
type histogramDistance struct {
data_ [numDistanceSymbols]uint32
total_count_ uint
bit_cost_ float64
}
func HistogramClearDistance(self *HistogramDistance) {
func histogramClearDistance(self *histogramDistance) {
self.data_ = [numDistanceSymbols]uint32{}
self.total_count_ = 0
self.bit_cost_ = math.MaxFloat64
}
func ClearHistogramsDistance(array []HistogramDistance, length uint) {
func clearHistogramsDistance(array []histogramDistance, length uint) {
var i uint
for i = 0; i < length; i++ {
HistogramClearDistance(&array[i:][0])
histogramClearDistance(&array[i:][0])
}
}
func HistogramAddDistance(self *HistogramDistance, val uint) {
func histogramAddDistance(self *histogramDistance, val uint) {
self.data_[val]++
self.total_count_++
}
func HistogramAddVectorDistance(self *HistogramDistance, p []uint16, n uint) {
func histogramAddVectorDistance(self *histogramDistance, p []uint16, n uint) {
self.total_count_ += n
n += 1
for {
@ -140,7 +140,7 @@ func HistogramAddVectorDistance(self *HistogramDistance, p []uint16, n uint) {
}
}
func HistogramAddHistogramDistance(self *HistogramDistance, v *HistogramDistance) {
func histogramAddHistogramDistance(self *histogramDistance, v *histogramDistance) {
var i uint
self.total_count_ += v.total_count_
for i = 0; i < numDistanceSymbols; i++ {
@ -148,18 +148,18 @@ func HistogramAddHistogramDistance(self *HistogramDistance, v *HistogramDistance
}
}
func HistogramDataSizeDistance() uint {
func histogramDataSizeDistance() uint {
return numDistanceSymbols
}
type BlockSplitIterator struct {
type blockSplitIterator struct {
split_ *blockSplit
idx_ uint
type_ uint
length_ uint
}
func InitBlockSplitIterator(self *BlockSplitIterator, split *blockSplit) {
func initBlockSplitIterator(self *blockSplitIterator, split *blockSplit) {
self.split_ = split
self.idx_ = 0
self.type_ = 0
@ -170,7 +170,7 @@ func InitBlockSplitIterator(self *BlockSplitIterator, split *blockSplit) {
}
}
func BlockSplitIteratorNext(self *BlockSplitIterator) {
func blockSplitIteratorNext(self *blockSplitIterator) {
if self.length_ == 0 {
self.idx_++
self.type_ = uint(self.split_.types[self.idx_])
@ -180,33 +180,33 @@ func BlockSplitIteratorNext(self *BlockSplitIterator) {
self.length_--
}
func BrotliBuildHistogramsWithContext(cmds []command, num_commands uint, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit, ringbuffer []byte, start_pos uint, mask uint, prev_byte byte, prev_byte2 byte, context_modes []int, literal_histograms []HistogramLiteral, insert_and_copy_histograms []HistogramCommand, copy_dist_histograms []HistogramDistance) {
func buildHistogramsWithContext(cmds []command, num_commands uint, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit, ringbuffer []byte, start_pos uint, mask uint, prev_byte byte, prev_byte2 byte, context_modes []int, literal_histograms []histogramLiteral, insert_and_copy_histograms []histogramCommand, copy_dist_histograms []histogramDistance) {
var pos uint = start_pos
var literal_it BlockSplitIterator
var insert_and_copy_it BlockSplitIterator
var dist_it BlockSplitIterator
var literal_it blockSplitIterator
var insert_and_copy_it blockSplitIterator
var dist_it blockSplitIterator
var i uint
InitBlockSplitIterator(&literal_it, literal_split)
InitBlockSplitIterator(&insert_and_copy_it, insert_and_copy_split)
InitBlockSplitIterator(&dist_it, dist_split)
initBlockSplitIterator(&literal_it, literal_split)
initBlockSplitIterator(&insert_and_copy_it, insert_and_copy_split)
initBlockSplitIterator(&dist_it, dist_split)
for i = 0; i < num_commands; i++ {
var cmd *command = &cmds[i]
var j uint
BlockSplitIteratorNext(&insert_and_copy_it)
HistogramAddCommand(&insert_and_copy_histograms[insert_and_copy_it.type_], uint(cmd.cmd_prefix_))
blockSplitIteratorNext(&insert_and_copy_it)
histogramAddCommand(&insert_and_copy_histograms[insert_and_copy_it.type_], uint(cmd.cmd_prefix_))
/* TODO: unwrap iterator blocks. */
for j = uint(cmd.insert_len_); j != 0; j-- {
var context uint
BlockSplitIteratorNext(&literal_it)
blockSplitIteratorNext(&literal_it)
context = literal_it.type_
if context_modes != nil {
var lut ContextLut = BROTLI_CONTEXT_LUT(context_modes[context])
context = (context << BROTLI_LITERAL_CONTEXT_BITS) + uint(BROTLI_CONTEXT(prev_byte, prev_byte2, lut))
var lut contextLUT = getContextLUT(context_modes[context])
context = (context << literalContextBits) + uint(getContext(prev_byte, prev_byte2, lut))
}
HistogramAddLiteral(&literal_histograms[context], uint(ringbuffer[pos&mask]))
histogramAddLiteral(&literal_histograms[context], uint(ringbuffer[pos&mask]))
prev_byte2 = prev_byte
prev_byte = ringbuffer[pos&mask]
pos++
@ -218,9 +218,9 @@ func BrotliBuildHistogramsWithContext(cmds []command, num_commands uint, literal
prev_byte = ringbuffer[(pos-1)&mask]
if cmd.cmd_prefix_ >= 128 {
var context uint
BlockSplitIteratorNext(&dist_it)
context = uint(uint32(dist_it.type_<<BROTLI_DISTANCE_CONTEXT_BITS) + commandDistanceContext(cmd))
HistogramAddDistance(&copy_dist_histograms[context], uint(cmd.dist_prefix_)&0x3FF)
blockSplitIteratorNext(&dist_it)
context = uint(uint32(dist_it.type_<<distanceContextBits) + commandDistanceContext(cmd))
histogramAddDistance(&copy_dist_histograms[context], uint(cmd.dist_prefix_)&0x3FF)
}
}
}

148
huffman.go
View File

@ -7,7 +7,7 @@ package brotli
*/
/* Utilities for building Huffman decoding tables. */
const BROTLI_HUFFMAN_MAX_CODE_LENGTH = 15
const huffmanMaxCodeLength = 15
/* Maximum possible Huffman table size for an alphabet size of (index * 32),
max code length 15 and root table bits 8. */
@ -52,25 +52,25 @@ var kMaxHuffmanTableSize = []uint16{
}
/* BROTLI_NUM_BLOCK_LEN_SYMBOLS == 26 */
const BROTLI_HUFFMAN_MAX_SIZE_26 = 396
const huffmanMaxSize26 = 396
/* BROTLI_MAX_BLOCK_TYPE_SYMBOLS == 258 */
const BROTLI_HUFFMAN_MAX_SIZE_258 = 632
const huffmanMaxSize258 = 632
/* BROTLI_MAX_CONTEXT_MAP_SYMBOLS == 272 */
const BROTLI_HUFFMAN_MAX_SIZE_272 = 646
const huffmanMaxSize272 = 646
const BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH = 5
const huffmanMaxCodeLengthCodeLength = 5
/* Do not create this struct directly - use the ConstructHuffmanCode
* constructor below! */
type HuffmanCode struct {
type huffmanCode struct {
bits byte
value uint16
}
func ConstructHuffmanCode(bits byte, value uint16) HuffmanCode {
var h HuffmanCode
func constructHuffmanCode(bits byte, value uint16) huffmanCode {
var h huffmanCode
h.bits = bits
h.value = value
return h
@ -89,19 +89,19 @@ func ConstructHuffmanCode(bits byte, value uint16) HuffmanCode {
/* Contains a collection of Huffman trees with the same alphabet size. */
/* max_symbol is needed due to simple codes since log2(alphabet_size) could be
greater than log2(max_symbol). */
type HuffmanTreeGroup struct {
htrees [][]HuffmanCode
codes []HuffmanCode
type huffmanTreeGroup struct {
htrees [][]huffmanCode
codes []huffmanCode
alphabet_size uint16
max_symbol uint16
num_htrees uint16
}
const BROTLI_REVERSE_BITS_MAX = 8
const reverseBitsMax = 8
const BROTLI_REVERSE_BITS_BASE = 0
const reverseBitsBase = 0
var kReverseBits = [1 << BROTLI_REVERSE_BITS_MAX]byte{
var kReverseBits = [1 << reverseBitsMax]byte{
0x00,
0x80,
0x40,
@ -360,18 +360,18 @@ var kReverseBits = [1 << BROTLI_REVERSE_BITS_MAX]byte{
0xFF,
}
const BROTLI_REVERSE_BITS_LOWEST = (uint64(1) << (BROTLI_REVERSE_BITS_MAX - 1 + BROTLI_REVERSE_BITS_BASE))
const reverseBitsLowest = (uint64(1) << (reverseBitsMax - 1 + reverseBitsBase))
/* Returns reverse(num >> BROTLI_REVERSE_BITS_BASE, BROTLI_REVERSE_BITS_MAX),
where reverse(value, len) is the bit-wise reversal of the len least
significant bits of value. */
func BrotliReverseBits8(num uint64) uint64 {
func reverseBits8(num uint64) uint64 {
return uint64(kReverseBits[num])
}
/* Stores code in table[0], table[step], table[2*step], ..., table[end] */
/* Assumes that end is an integer multiple of step */
func ReplicateValue(table []HuffmanCode, step int, end int, code HuffmanCode) {
func replicateValue(table []huffmanCode, step int, end int, code huffmanCode) {
for {
end -= step
table[end] = code
@ -384,9 +384,9 @@ func ReplicateValue(table []HuffmanCode, step int, end int, code HuffmanCode) {
/* Returns the table width of the next 2nd level table. |count| is the histogram
of bit lengths for the remaining symbols, |len| is the code length of the
next processed symbol. */
func NextTableBitSize(count []uint16, len int, root_bits int) int {
func nextTableBitSize(count []uint16, len int, root_bits int) int {
var left int = 1 << uint(len-root_bits)
for len < BROTLI_HUFFMAN_MAX_CODE_LENGTH {
for len < huffmanMaxCodeLength {
left -= int(count[len])
if left <= 0 {
break
@ -398,26 +398,26 @@ func NextTableBitSize(count []uint16, len int, root_bits int) int {
return len - root_bits
}
func BrotliBuildCodeLengthsHuffmanTable(table []HuffmanCode, code_lengths []byte, count []uint16) {
var code HuffmanCode /* current table entry */ /* symbol index in original or sorted table */ /* prefix code */ /* prefix code addend */ /* step size to replicate values in current table */ /* size of current table */ /* symbols sorted by code length */
func buildCodeLengthsHuffmanTable(table []huffmanCode, code_lengths []byte, count []uint16) {
var code huffmanCode /* current table entry */ /* symbol index in original or sorted table */ /* prefix code */ /* prefix code addend */ /* step size to replicate values in current table */ /* size of current table */ /* symbols sorted by code length */
var symbol int
var key uint64
var key_step uint64
var step int
var table_size int
var sorted [codeLengthCodes]int
var offset [BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH + 1]int
var offset [huffmanMaxCodeLengthCodeLength + 1]int
var bits int
var bits_count int
/* offsets in sorted table for each length */
assert(BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH <= BROTLI_REVERSE_BITS_MAX)
assert(huffmanMaxCodeLengthCodeLength <= reverseBitsMax)
/* Generate offsets into sorted symbol table by code length. */
symbol = -1
bits = 1
var i int
for i = 0; i < BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH; i++ {
for i = 0; i < huffmanMaxCodeLengthCodeLength; i++ {
symbol += int(count[bits])
offset[bits] = symbol
bits++
@ -441,11 +441,11 @@ func BrotliBuildCodeLengthsHuffmanTable(table []HuffmanCode, code_lengths []byte
}
}
table_size = 1 << BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH
table_size = 1 << huffmanMaxCodeLengthCodeLength
/* Special case: all symbols but one have 0 code length. */
if offset[0] == 0 {
code = ConstructHuffmanCode(0, uint16(sorted[0]))
code = constructHuffmanCode(0, uint16(sorted[0]))
for key = 0; key < uint64(table_size); key++ {
table[key] = code
}
@ -456,30 +456,30 @@ func BrotliBuildCodeLengthsHuffmanTable(table []HuffmanCode, code_lengths []byte
/* Fill in table. */
key = 0
key_step = BROTLI_REVERSE_BITS_LOWEST
key_step = reverseBitsLowest
symbol = 0
bits = 1
step = 2
for {
for bits_count = int(count[bits]); bits_count != 0; bits_count-- {
code = ConstructHuffmanCode(byte(bits), uint16(sorted[symbol]))
code = constructHuffmanCode(byte(bits), uint16(sorted[symbol]))
symbol++
ReplicateValue(table[BrotliReverseBits8(key):], step, table_size, code)
replicateValue(table[reverseBits8(key):], step, table_size, code)
key += key_step
}
step <<= 1
key_step >>= 1
bits++
if bits > BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH {
if bits > huffmanMaxCodeLengthCodeLength {
break
}
}
}
func BrotliBuildHuffmanTable(root_table []HuffmanCode, root_bits int, symbol_lists SymbolList, count []uint16) uint32 {
var code HuffmanCode /* current table entry */ /* next available space in table */ /* current code length */ /* symbol index in original or sorted table */ /* prefix code */ /* prefix code addend */ /* 2nd level table prefix code */ /* 2nd level table prefix code addend */ /* step size to replicate values in current table */ /* key length of current table */ /* size of current table */ /* sum of root table size and 2nd level table sizes */
var table []HuffmanCode
func buildHuffmanTable(root_table []huffmanCode, root_bits int, symbol_lists SymbolList, count []uint16) uint32 {
var code huffmanCode /* current table entry */ /* next available space in table */ /* current code length */ /* symbol index in original or sorted table */ /* prefix code */ /* prefix code addend */ /* 2nd level table prefix code */ /* 2nd level table prefix code addend */ /* step size to replicate values in current table */ /* key length of current table */ /* size of current table */ /* sum of root table size and 2nd level table sizes */
var table []huffmanCode
var len int
var symbol int
var key uint64
@ -494,13 +494,13 @@ func BrotliBuildHuffmanTable(root_table []HuffmanCode, root_bits int, symbol_lis
var bits int
var bits_count int
assert(root_bits <= BROTLI_REVERSE_BITS_MAX)
assert(BROTLI_HUFFMAN_MAX_CODE_LENGTH-root_bits <= BROTLI_REVERSE_BITS_MAX)
assert(root_bits <= reverseBitsMax)
assert(huffmanMaxCodeLength-root_bits <= reverseBitsMax)
for SymbolListGet(symbol_lists, max_length) == 0xFFFF {
max_length--
}
max_length += BROTLI_HUFFMAN_MAX_CODE_LENGTH + 1
max_length += huffmanMaxCodeLength + 1
table = root_table
table_bits = root_bits
@ -515,15 +515,15 @@ func BrotliBuildHuffmanTable(root_table []HuffmanCode, root_bits int, symbol_lis
}
key = 0
key_step = BROTLI_REVERSE_BITS_LOWEST
key_step = reverseBitsLowest
bits = 1
step = 2
for {
symbol = bits - (BROTLI_HUFFMAN_MAX_CODE_LENGTH + 1)
symbol = bits - (huffmanMaxCodeLength + 1)
for bits_count = int(count[bits]); bits_count != 0; bits_count-- {
symbol = int(SymbolListGet(symbol_lists, symbol))
code = ConstructHuffmanCode(byte(bits), uint16(symbol))
ReplicateValue(table[BrotliReverseBits8(key):], step, table_size, code)
code = constructHuffmanCode(byte(bits), uint16(symbol))
replicateValue(table[reverseBits8(key):], step, table_size, code)
key += key_step
}
@ -542,29 +542,29 @@ func BrotliBuildHuffmanTable(root_table []HuffmanCode, root_bits int, symbol_lis
}
/* Fill in 2nd level tables and add pointers to root table. */
key_step = BROTLI_REVERSE_BITS_LOWEST >> uint(root_bits-1)
key_step = reverseBitsLowest >> uint(root_bits-1)
sub_key = BROTLI_REVERSE_BITS_LOWEST << 1
sub_key_step = BROTLI_REVERSE_BITS_LOWEST
sub_key = reverseBitsLowest << 1
sub_key_step = reverseBitsLowest
len = root_bits + 1
step = 2
for ; len <= max_length; len++ {
symbol = len - (BROTLI_HUFFMAN_MAX_CODE_LENGTH + 1)
symbol = len - (huffmanMaxCodeLength + 1)
for ; count[len] != 0; count[len]-- {
if sub_key == BROTLI_REVERSE_BITS_LOWEST<<1 {
if sub_key == reverseBitsLowest<<1 {
table = table[table_size:]
table_bits = NextTableBitSize(count, int(len), root_bits)
table_bits = nextTableBitSize(count, int(len), root_bits)
table_size = 1 << uint(table_bits)
total_size += table_size
sub_key = BrotliReverseBits8(key)
sub_key = reverseBits8(key)
key += key_step
root_table[sub_key] = ConstructHuffmanCode(byte(table_bits+root_bits), uint16(uint64(uint(-cap(table)+cap(root_table)))-sub_key))
root_table[sub_key] = constructHuffmanCode(byte(table_bits+root_bits), uint16(uint64(uint(-cap(table)+cap(root_table)))-sub_key))
sub_key = 0
}
symbol = int(SymbolListGet(symbol_lists, symbol))
code = ConstructHuffmanCode(byte(len-root_bits), uint16(symbol))
ReplicateValue(table[BrotliReverseBits8(sub_key):], step, table_size, code)
code = constructHuffmanCode(byte(len-root_bits), uint16(symbol))
replicateValue(table[reverseBits8(sub_key):], step, table_size, code)
sub_key += sub_key_step
}
@ -575,33 +575,33 @@ func BrotliBuildHuffmanTable(root_table []HuffmanCode, root_bits int, symbol_lis
return uint32(total_size)
}
func BrotliBuildSimpleHuffmanTable(table []HuffmanCode, root_bits int, val []uint16, num_symbols uint32) uint32 {
func buildSimpleHuffmanTable(table []huffmanCode, root_bits int, val []uint16, num_symbols uint32) uint32 {
var table_size uint32 = 1
var goal_size uint32 = 1 << uint(root_bits)
switch num_symbols {
case 0:
table[0] = ConstructHuffmanCode(0, val[0])
table[0] = constructHuffmanCode(0, val[0])
case 1:
if val[1] > val[0] {
table[0] = ConstructHuffmanCode(1, val[0])
table[1] = ConstructHuffmanCode(1, val[1])
table[0] = constructHuffmanCode(1, val[0])
table[1] = constructHuffmanCode(1, val[1])
} else {
table[0] = ConstructHuffmanCode(1, val[1])
table[1] = ConstructHuffmanCode(1, val[0])
table[0] = constructHuffmanCode(1, val[1])
table[1] = constructHuffmanCode(1, val[0])
}
table_size = 2
case 2:
table[0] = ConstructHuffmanCode(1, val[0])
table[2] = ConstructHuffmanCode(1, val[0])
table[0] = constructHuffmanCode(1, val[0])
table[2] = constructHuffmanCode(1, val[0])
if val[2] > val[1] {
table[1] = ConstructHuffmanCode(2, val[1])
table[3] = ConstructHuffmanCode(2, val[2])
table[1] = constructHuffmanCode(2, val[1])
table[3] = constructHuffmanCode(2, val[2])
} else {
table[1] = ConstructHuffmanCode(2, val[2])
table[3] = ConstructHuffmanCode(2, val[1])
table[1] = constructHuffmanCode(2, val[2])
table[3] = constructHuffmanCode(2, val[1])
}
table_size = 4
@ -619,10 +619,10 @@ func BrotliBuildSimpleHuffmanTable(table []HuffmanCode, root_bits int, val []uin
}
}
table[0] = ConstructHuffmanCode(2, val[0])
table[2] = ConstructHuffmanCode(2, val[1])
table[1] = ConstructHuffmanCode(2, val[2])
table[3] = ConstructHuffmanCode(2, val[3])
table[0] = constructHuffmanCode(2, val[0])
table[2] = constructHuffmanCode(2, val[1])
table[1] = constructHuffmanCode(2, val[2])
table[3] = constructHuffmanCode(2, val[3])
table_size = 4
break
}
@ -635,14 +635,14 @@ func BrotliBuildSimpleHuffmanTable(table []HuffmanCode, root_bits int, val []uin
val[2] = t
}
table[0] = ConstructHuffmanCode(1, val[0])
table[1] = ConstructHuffmanCode(2, val[1])
table[2] = ConstructHuffmanCode(1, val[0])
table[3] = ConstructHuffmanCode(3, val[2])
table[4] = ConstructHuffmanCode(1, val[0])
table[5] = ConstructHuffmanCode(2, val[1])
table[6] = ConstructHuffmanCode(1, val[0])
table[7] = ConstructHuffmanCode(3, val[3])
table[0] = constructHuffmanCode(1, val[0])
table[1] = constructHuffmanCode(2, val[1])
table[2] = constructHuffmanCode(1, val[0])
table[3] = constructHuffmanCode(3, val[2])
table[4] = constructHuffmanCode(1, val[0])
table[5] = constructHuffmanCode(2, val[1])
table[6] = constructHuffmanCode(1, val[0])
table[7] = constructHuffmanCode(3, val[3])
table_size = 8
break
}

26
literal_cost.go
View File

@ -1,6 +1,6 @@
package brotli
func UTF8Position(last uint, c uint, clamp uint) uint {
func utf8Position(last uint, c uint, clamp uint) uint {
if c < 128 {
return 0 /* Next one is the 'Byte 1' again. */
} else if c >= 192 { /* Next one is the 'Byte 2' of utf-8 encoding. */
@ -15,14 +15,14 @@ func UTF8Position(last uint, c uint, clamp uint) uint {
}
}
func DecideMultiByteStatsLevel(pos uint, len uint, mask uint, data []byte) uint {
func decideMultiByteStatsLevel(pos uint, len uint, mask uint, data []byte) uint {
var counts = [3]uint{0} /* should be 2, but 1 compresses better. */
var max_utf8 uint = 1
var last_c uint = 0
var i uint
for i = 0; i < len; i++ {
var c uint = uint(data[(pos+i)&mask])
counts[UTF8Position(last_c, c, 2)]++
counts[utf8Position(last_c, c, 2)]++
last_c = c
}
@ -37,8 +37,8 @@ func DecideMultiByteStatsLevel(pos uint, len uint, mask uint, data []byte) uint
return max_utf8
}
func EstimateBitCostsForLiteralsUTF8(pos uint, len uint, mask uint, data []byte, cost []float32) {
var max_utf8 uint = DecideMultiByteStatsLevel(pos, uint(len), mask, data)
func estimateBitCostsForLiteralsUTF8(pos uint, len uint, mask uint, data []byte, cost []float32) {
var max_utf8 uint = decideMultiByteStatsLevel(pos, uint(len), mask, data)
/* Bootstrap histograms. */
var histogram = [3][256]uint{[256]uint{0}}
var window_half uint = 495
@ -55,7 +55,7 @@ func EstimateBitCostsForLiteralsUTF8(pos uint, len uint, mask uint, data []byte,
var c uint = uint(data[(pos+i)&mask])
histogram[utf8_pos][c]++
in_window_utf8[utf8_pos]++
utf8_pos = UTF8Position(last_c, c, max_utf8)
utf8_pos = utf8Position(last_c, c, max_utf8)
last_c = c
}
}
@ -77,7 +77,7 @@ func EstimateBitCostsForLiteralsUTF8(pos uint, len uint, mask uint, data []byte,
}
/* Remove a byte in the past. */
var utf8_pos2 uint = UTF8Position(last_c, c, max_utf8)
var utf8_pos2 uint = utf8Position(last_c, c, max_utf8)
histogram[utf8_pos2][data[(pos+i-window_half)&mask]]--
in_window_utf8[utf8_pos2]--
}
@ -87,7 +87,7 @@ func EstimateBitCostsForLiteralsUTF8(pos uint, len uint, mask uint, data []byte,
var last_c uint = uint(data[(pos+i+window_half-2)&mask])
/* Add a byte in the future. */
var utf8_pos2 uint = UTF8Position(last_c, c, max_utf8)
var utf8_pos2 uint = utf8Position(last_c, c, max_utf8)
histogram[utf8_pos2][data[(pos+i+window_half)&mask]]++
in_window_utf8[utf8_pos2]++
}
@ -104,7 +104,7 @@ func EstimateBitCostsForLiteralsUTF8(pos uint, len uint, mask uint, data []byte,
} else {
last_c = uint(data[(pos+i-2)&mask])
}
var utf8_pos uint = UTF8Position(last_c, c, max_utf8)
var utf8_pos uint = utf8Position(last_c, c, max_utf8)
var masked_pos uint = (pos + i) & mask
var histo uint = histogram[utf8_pos][data[masked_pos]]
var lit_cost float64
@ -112,7 +112,7 @@ func EstimateBitCostsForLiteralsUTF8(pos uint, len uint, mask uint, data []byte,
histo = 1
}
lit_cost = FastLog2(in_window_utf8[utf8_pos]) - FastLog2(histo)
lit_cost = fastLog2(in_window_utf8[utf8_pos]) - fastLog2(histo)
lit_cost += 0.02905
if lit_cost < 1.0 {
lit_cost *= 0.5
@ -132,9 +132,9 @@ func EstimateBitCostsForLiteralsUTF8(pos uint, len uint, mask uint, data []byte,
}
}
func BrotliEstimateBitCostsForLiterals(pos uint, len uint, mask uint, data []byte, cost []float32) {
func estimateBitCostsForLiterals(pos uint, len uint, mask uint, data []byte, cost []float32) {
if BrotliIsMostlyUTF8(data, pos, mask, uint(len), kMinUTF8Ratio) {
EstimateBitCostsForLiteralsUTF8(pos, uint(len), mask, data, cost)
estimateBitCostsForLiteralsUTF8(pos, uint(len), mask, data, cost)
return
} else {
var histogram = [256]uint{0}
@ -168,7 +168,7 @@ func BrotliEstimateBitCostsForLiterals(pos uint, len uint, mask uint, data []byt
histo = 1
}
{
var lit_cost float64 = FastLog2(in_window) - FastLog2(histo)
var lit_cost float64 = fastLog2(in_window) - fastLog2(histo)
lit_cost += 0.029
if lit_cost < 1.0 {
lit_cost *= 0.5

186
metablock.go
View File

@ -31,7 +31,7 @@ package brotli
/* Algorithms for distributing the literals and commands of a metablock between
block types and contexts. */
type MetaBlockSplit struct {
type metaBlockSplit struct {
literal_split blockSplit
command_split blockSplit
distance_split blockSplit
@ -39,15 +39,15 @@ type MetaBlockSplit struct {
literal_context_map_size uint
distance_context_map []uint32
distance_context_map_size uint
literal_histograms []HistogramLiteral
literal_histograms []histogramLiteral
literal_histograms_size uint
command_histograms []HistogramCommand
command_histograms []histogramCommand
command_histograms_size uint
distance_histograms []HistogramDistance
distance_histograms []histogramDistance
distance_histograms_size uint
}
func InitMetaBlockSplit(mb *MetaBlockSplit) {
func initMetaBlockSplit(mb *metaBlockSplit) {
initBlockSplit(&mb.literal_split)
initBlockSplit(&mb.command_split)
initBlockSplit(&mb.distance_split)
@ -63,7 +63,7 @@ func InitMetaBlockSplit(mb *MetaBlockSplit) {
mb.distance_histograms_size = 0
}
func DestroyMetaBlockSplit(mb *MetaBlockSplit) {
func destroyMetaBlockSplit(mb *metaBlockSplit) {
destroyBlockSplit(&mb.literal_split)
destroyBlockSplit(&mb.command_split)
destroyBlockSplit(&mb.distance_split)
@ -82,8 +82,8 @@ func DestroyMetaBlockSplit(mb *MetaBlockSplit) {
/* Algorithms for distributing the literals and commands of a metablock between
block types and contexts. */
func BrotliInitDistanceParams(params *BrotliEncoderParams, npostfix uint32, ndirect uint32) {
var dist_params *BrotliDistanceParams = &params.dist
func initDistanceParams(params *encoderParams, npostfix uint32, ndirect uint32) {
var dist_params *distanceParams = &params.dist
var alphabet_size uint32
var max_distance uint32
@ -102,11 +102,11 @@ func BrotliInitDistanceParams(params *BrotliEncoderParams, npostfix uint32, ndir
a distance larger than BROTLI_MAX_ALLOWED_DISTANCE with all
its extra bits set. */
if ndirect < bound[npostfix] {
max_distance = BROTLI_MAX_ALLOWED_DISTANCE - (bound[npostfix] - ndirect)
max_distance = maxAllowedDistance - (bound[npostfix] - ndirect)
} else if ndirect >= bound[npostfix]+postfix {
max_distance = (3 << 29) - 4 + (ndirect - bound[npostfix])
} else {
max_distance = BROTLI_MAX_ALLOWED_DISTANCE
max_distance = maxAllowedDistance
}
}
@ -114,7 +114,7 @@ func BrotliInitDistanceParams(params *BrotliEncoderParams, npostfix uint32, ndir
dist_params.max_distance = uint(max_distance)
}
func RecomputeDistancePrefixes(cmds []command, num_commands uint, orig_params *BrotliDistanceParams, new_params *BrotliDistanceParams) {
func recomputeDistancePrefixes(cmds []command, num_commands uint, orig_params *distanceParams, new_params *distanceParams) {
var i uint
if orig_params.distance_postfix_bits == new_params.distance_postfix_bits && orig_params.num_direct_distance_codes == new_params.num_direct_distance_codes {
@ -129,14 +129,14 @@ func RecomputeDistancePrefixes(cmds []command, num_commands uint, orig_params *B
}
}
func ComputeDistanceCost(cmds []command, num_commands uint, orig_params *BrotliDistanceParams, new_params *BrotliDistanceParams, cost *float64) bool {
func computeDistanceCost(cmds []command, num_commands uint, orig_params *distanceParams, new_params *distanceParams, cost *float64) bool {
var i uint
var equal_params bool = false
var dist_prefix uint16
var dist_extra uint32
var extra_bits float64 = 0.0
var histo HistogramDistance
HistogramClearDistance(&histo)
var histo histogramDistance
histogramClearDistance(&histo)
if orig_params.distance_postfix_bits == new_params.distance_postfix_bits && orig_params.num_direct_distance_codes == new_params.num_direct_distance_codes {
equal_params = true
@ -156,7 +156,7 @@ func ComputeDistanceCost(cmds []command, num_commands uint, orig_params *BrotliD
PrefixEncodeCopyDistance(uint(distance), uint(new_params.num_direct_distance_codes), uint(new_params.distance_postfix_bits), &dist_prefix, &dist_extra)
}
HistogramAddDistance(&histo, uint(dist_prefix)&0x3FF)
histogramAddDistance(&histo, uint(dist_prefix)&0x3FF)
extra_bits += float64(dist_prefix >> 10)
}
}
@ -165,11 +165,11 @@ func ComputeDistanceCost(cmds []command, num_commands uint, orig_params *BrotliD
return true
}
var BrotliBuildMetaBlock_kMaxNumberOfHistograms uint = 256
var buildMetaBlock_kMaxNumberOfHistograms uint = 256
func BrotliBuildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *BrotliEncoderParams, prev_byte byte, prev_byte2 byte, cmds []command, num_commands uint, literal_context_mode int, mb *MetaBlockSplit) {
var distance_histograms []HistogramDistance
var literal_histograms []HistogramLiteral
func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParams, prev_byte byte, prev_byte2 byte, cmds []command, num_commands uint, literal_context_mode int, mb *metaBlockSplit) {
var distance_histograms []histogramDistance
var literal_histograms []histogramLiteral
var literal_context_modes []int = nil
var literal_histograms_size uint
var distance_histograms_size uint
@ -179,22 +179,22 @@ func BrotliBuildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *Brotli
var ndirect_msb uint32 = 0
var check_orig bool = true
var best_dist_cost float64 = 1e99
var orig_params BrotliEncoderParams = *params
var orig_params encoderParams = *params
/* Histogram ids need to fit in one byte. */
var new_params BrotliEncoderParams = *params
var new_params encoderParams = *params
for npostfix = 0; npostfix <= maxNpostfix; npostfix++ {
for ; ndirect_msb < 16; ndirect_msb++ {
var ndirect uint32 = ndirect_msb << npostfix
var skip bool
var dist_cost float64
BrotliInitDistanceParams(&new_params, npostfix, ndirect)
initDistanceParams(&new_params, npostfix, ndirect)
if npostfix == orig_params.dist.distance_postfix_bits && ndirect == orig_params.dist.num_direct_distance_codes {
check_orig = false
}
skip = !ComputeDistanceCost(cmds, num_commands, &orig_params.dist, &new_params.dist, &dist_cost)
skip = !computeDistanceCost(cmds, num_commands, &orig_params.dist, &new_params.dist, &dist_cost)
if skip || (dist_cost > best_dist_cost) {
break
}
@ -211,7 +211,7 @@ func BrotliBuildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *Brotli
if check_orig {
var dist_cost float64
ComputeDistanceCost(cmds, num_commands, &orig_params.dist, &orig_params.dist, &dist_cost)
computeDistanceCost(cmds, num_commands, &orig_params.dist, &orig_params.dist, &dist_cost)
if dist_cost < best_dist_cost {
/* NB: currently unused; uncomment when more param tuning is added. */
/* best_dist_cost = dist_cost; */
@ -219,12 +219,12 @@ func BrotliBuildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *Brotli
}
}
RecomputeDistancePrefixes(cmds, num_commands, &orig_params.dist, &params.dist)
recomputeDistancePrefixes(cmds, num_commands, &orig_params.dist, &params.dist)
splitBlock(cmds, num_commands, ringbuffer, pos, mask, params, &mb.literal_split, &mb.command_split, &mb.distance_split)
if !params.disable_literal_context_modeling {
literal_context_multiplier = 1 << BROTLI_LITERAL_CONTEXT_BITS
literal_context_multiplier = 1 << literalContextBits
literal_context_modes = make([]int, (mb.literal_split.num_types))
for i = 0; i < mb.literal_split.num_types; i++ {
literal_context_modes[i] = literal_context_mode
@ -232,30 +232,30 @@ func BrotliBuildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *Brotli
}
literal_histograms_size = mb.literal_split.num_types * literal_context_multiplier
literal_histograms = make([]HistogramLiteral, literal_histograms_size)
ClearHistogramsLiteral(literal_histograms, literal_histograms_size)
literal_histograms = make([]histogramLiteral, literal_histograms_size)
clearHistogramsLiteral(literal_histograms, literal_histograms_size)
distance_histograms_size = mb.distance_split.num_types << BROTLI_DISTANCE_CONTEXT_BITS
distance_histograms = make([]HistogramDistance, distance_histograms_size)
ClearHistogramsDistance(distance_histograms, distance_histograms_size)
distance_histograms_size = mb.distance_split.num_types << distanceContextBits
distance_histograms = make([]histogramDistance, distance_histograms_size)
clearHistogramsDistance(distance_histograms, distance_histograms_size)
assert(mb.command_histograms == nil)
mb.command_histograms_size = mb.command_split.num_types
mb.command_histograms = make([]HistogramCommand, (mb.command_histograms_size))
ClearHistogramsCommand(mb.command_histograms, mb.command_histograms_size)
mb.command_histograms = make([]histogramCommand, (mb.command_histograms_size))
clearHistogramsCommand(mb.command_histograms, mb.command_histograms_size)
BrotliBuildHistogramsWithContext(cmds, num_commands, &mb.literal_split, &mb.command_split, &mb.distance_split, ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_modes, literal_histograms, mb.command_histograms, distance_histograms)
buildHistogramsWithContext(cmds, num_commands, &mb.literal_split, &mb.command_split, &mb.distance_split, ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_modes, literal_histograms, mb.command_histograms, distance_histograms)
literal_context_modes = nil
assert(mb.literal_context_map == nil)
mb.literal_context_map_size = mb.literal_split.num_types << BROTLI_LITERAL_CONTEXT_BITS
mb.literal_context_map_size = mb.literal_split.num_types << literalContextBits
mb.literal_context_map = make([]uint32, (mb.literal_context_map_size))
assert(mb.literal_histograms == nil)
mb.literal_histograms_size = mb.literal_context_map_size
mb.literal_histograms = make([]HistogramLiteral, (mb.literal_histograms_size))
mb.literal_histograms = make([]histogramLiteral, (mb.literal_histograms_size))
clusterHistogramsLiteral(literal_histograms, literal_histograms_size, BrotliBuildMetaBlock_kMaxNumberOfHistograms, mb.literal_histograms, &mb.literal_histograms_size, mb.literal_context_map)
clusterHistogramsLiteral(literal_histograms, literal_histograms_size, buildMetaBlock_kMaxNumberOfHistograms, mb.literal_histograms, &mb.literal_histograms_size, mb.literal_context_map)
literal_histograms = nil
if params.disable_literal_context_modeling {
@ -263,29 +263,29 @@ func BrotliBuildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *Brotli
for i = mb.literal_split.num_types; i != 0; {
var j uint = 0
i--
for ; j < 1<<BROTLI_LITERAL_CONTEXT_BITS; j++ {
mb.literal_context_map[(i<<BROTLI_LITERAL_CONTEXT_BITS)+j] = mb.literal_context_map[i]
for ; j < 1<<literalContextBits; j++ {
mb.literal_context_map[(i<<literalContextBits)+j] = mb.literal_context_map[i]
}
}
}
assert(mb.distance_context_map == nil)
mb.distance_context_map_size = mb.distance_split.num_types << BROTLI_DISTANCE_CONTEXT_BITS
mb.distance_context_map_size = mb.distance_split.num_types << distanceContextBits
mb.distance_context_map = make([]uint32, (mb.distance_context_map_size))
assert(mb.distance_histograms == nil)
mb.distance_histograms_size = mb.distance_context_map_size
mb.distance_histograms = make([]HistogramDistance, (mb.distance_histograms_size))
mb.distance_histograms = make([]histogramDistance, (mb.distance_histograms_size))
clusterHistogramsDistance(distance_histograms, mb.distance_context_map_size, BrotliBuildMetaBlock_kMaxNumberOfHistograms, mb.distance_histograms, &mb.distance_histograms_size, mb.distance_context_map)
clusterHistogramsDistance(distance_histograms, mb.distance_context_map_size, buildMetaBlock_kMaxNumberOfHistograms, mb.distance_histograms, &mb.distance_histograms_size, mb.distance_context_map)
distance_histograms = nil
}
const BROTLI_MAX_STATIC_CONTEXTS = 13
const maxStaticContexts = 13
/* Greedy block splitter for one block category (literal, command or distance).
Gathers histograms for all context buckets. */
type ContextBlockSplitter struct {
type contextBlockSplitter struct {
alphabet_size_ uint
num_contexts_ uint
max_block_types_ uint
@ -293,20 +293,20 @@ type ContextBlockSplitter struct {
split_threshold_ float64
num_blocks_ uint
split_ *blockSplit
histograms_ []HistogramLiteral
histograms_ []histogramLiteral
histograms_size_ *uint
target_block_size_ uint
block_size_ uint
curr_histogram_ix_ uint
last_histogram_ix_ [2]uint
last_entropy_ [2 * BROTLI_MAX_STATIC_CONTEXTS]float64
last_entropy_ [2 * maxStaticContexts]float64
merge_last_count_ uint
}
func InitContextBlockSplitter(self *ContextBlockSplitter, alphabet_size uint, num_contexts uint, min_block_size uint, split_threshold float64, num_symbols uint, split *blockSplit, histograms *[]HistogramLiteral, histograms_size *uint) {
func initContextBlockSplitter(self *contextBlockSplitter, alphabet_size uint, num_contexts uint, min_block_size uint, split_threshold float64, num_symbols uint, split *blockSplit, histograms *[]histogramLiteral, histograms_size *uint) {
var max_num_blocks uint = num_symbols/min_block_size + 1
var max_num_types uint
assert(num_contexts <= BROTLI_MAX_STATIC_CONTEXTS)
assert(num_contexts <= maxStaticContexts)
self.alphabet_size_ = alphabet_size
self.num_contexts_ = num_contexts
@ -330,11 +330,11 @@ func InitContextBlockSplitter(self *ContextBlockSplitter, alphabet_size uint, nu
split.num_blocks = max_num_blocks
assert(*histograms == nil)
*histograms_size = max_num_types * num_contexts
*histograms = make([]HistogramLiteral, (*histograms_size))
*histograms = make([]histogramLiteral, (*histograms_size))
self.histograms_ = *histograms
/* Clear only current histogram. */
ClearHistogramsLiteral(self.histograms_[0:], num_contexts)
clearHistogramsLiteral(self.histograms_[0:], num_contexts)
self.last_histogram_ix_[1] = 0
self.last_histogram_ix_[0] = self.last_histogram_ix_[1]
@ -344,11 +344,11 @@ func InitContextBlockSplitter(self *ContextBlockSplitter, alphabet_size uint, nu
(1) emits the current block with a new block type;
(2) emits the current block with the type of the second last block;
(3) merges the current block with the last block. */
func ContextBlockSplitterFinishBlock(self *ContextBlockSplitter, is_final bool) {
func contextBlockSplitterFinishBlock(self *contextBlockSplitter, is_final bool) {
var split *blockSplit = self.split_
var num_contexts uint = self.num_contexts_
var last_entropy []float64 = self.last_entropy_[:]
var histograms []HistogramLiteral = self.histograms_
var histograms []histogramLiteral = self.histograms_
if self.block_size_ < self.min_block_size_ {
self.block_size_ = self.min_block_size_
@ -371,14 +371,14 @@ func ContextBlockSplitterFinishBlock(self *ContextBlockSplitter, is_final bool)
split.num_types++
self.curr_histogram_ix_ += num_contexts
if self.curr_histogram_ix_ < *self.histograms_size_ {
ClearHistogramsLiteral(self.histograms_[self.curr_histogram_ix_:], self.num_contexts_)
clearHistogramsLiteral(self.histograms_[self.curr_histogram_ix_:], self.num_contexts_)
}
self.block_size_ = 0
} else if self.block_size_ > 0 {
var entropy [BROTLI_MAX_STATIC_CONTEXTS]float64
var combined_histo []HistogramLiteral = make([]HistogramLiteral, (2 * num_contexts))
var combined_entropy [2 * BROTLI_MAX_STATIC_CONTEXTS]float64
var entropy [maxStaticContexts]float64
var combined_histo []histogramLiteral = make([]histogramLiteral, (2 * num_contexts))
var combined_entropy [2 * maxStaticContexts]float64
var diff = [2]float64{0.0}
/* Try merging the set of histograms for the current block type with the
respective set of histograms for the last and second last block types.
@ -394,7 +394,7 @@ func ContextBlockSplitterFinishBlock(self *ContextBlockSplitter, is_final bool)
var jx uint = j*num_contexts + i
var last_histogram_ix uint = self.last_histogram_ix_[j] + i
combined_histo[jx] = histograms[curr_histo_ix]
HistogramAddHistogramLiteral(&combined_histo[jx], &histograms[last_histogram_ix])
histogramAddHistogramLiteral(&combined_histo[jx], &histograms[last_histogram_ix])
combined_entropy[jx] = bitsEntropy(combined_histo[jx].data_[0:], self.alphabet_size_)
diff[j] += combined_entropy[jx] - entropy[i] - last_entropy[jx]
}
@ -416,7 +416,7 @@ func ContextBlockSplitterFinishBlock(self *ContextBlockSplitter, is_final bool)
split.num_types++
self.curr_histogram_ix_ += num_contexts
if self.curr_histogram_ix_ < *self.histograms_size_ {
ClearHistogramsLiteral(self.histograms_[self.curr_histogram_ix_:], self.num_contexts_)
clearHistogramsLiteral(self.histograms_[self.curr_histogram_ix_:], self.num_contexts_)
}
self.block_size_ = 0
@ -434,7 +434,7 @@ func ContextBlockSplitterFinishBlock(self *ContextBlockSplitter, is_final bool)
histograms[self.last_histogram_ix_[0]+i] = combined_histo[num_contexts+i]
last_entropy[num_contexts+i] = last_entropy[i]
last_entropy[i] = combined_entropy[num_contexts+i]
HistogramClearLiteral(&histograms[self.curr_histogram_ix_+i])
histogramClearLiteral(&histograms[self.curr_histogram_ix_+i])
}
self.num_blocks_++
@ -452,7 +452,7 @@ func ContextBlockSplitterFinishBlock(self *ContextBlockSplitter, is_final bool)
last_entropy[num_contexts+i] = last_entropy[i]
}
HistogramClearLiteral(&histograms[self.curr_histogram_ix_+i])
histogramClearLiteral(&histograms[self.curr_histogram_ix_+i])
}
self.block_size_ = 0
@ -473,36 +473,36 @@ func ContextBlockSplitterFinishBlock(self *ContextBlockSplitter, is_final bool)
/* Adds the next symbol to the current block type and context. When the
current block reaches the target size, decides on merging the block. */
func ContextBlockSplitterAddSymbol(self *ContextBlockSplitter, symbol uint, context uint) {
HistogramAddLiteral(&self.histograms_[self.curr_histogram_ix_+context], symbol)
func contextBlockSplitterAddSymbol(self *contextBlockSplitter, symbol uint, context uint) {
histogramAddLiteral(&self.histograms_[self.curr_histogram_ix_+context], symbol)
self.block_size_++
if self.block_size_ == self.target_block_size_ {
ContextBlockSplitterFinishBlock(self, false) /* is_final = */
contextBlockSplitterFinishBlock(self, false) /* is_final = */
}
}
func MapStaticContexts(num_contexts uint, static_context_map []uint32, mb *MetaBlockSplit) {
func mapStaticContexts(num_contexts uint, static_context_map []uint32, mb *metaBlockSplit) {
var i uint
assert(mb.literal_context_map == nil)
mb.literal_context_map_size = mb.literal_split.num_types << BROTLI_LITERAL_CONTEXT_BITS
mb.literal_context_map_size = mb.literal_split.num_types << literalContextBits
mb.literal_context_map = make([]uint32, (mb.literal_context_map_size))
for i = 0; i < mb.literal_split.num_types; i++ {
var offset uint32 = uint32(i * num_contexts)
var j uint
for j = 0; j < 1<<BROTLI_LITERAL_CONTEXT_BITS; j++ {
mb.literal_context_map[(i<<BROTLI_LITERAL_CONTEXT_BITS)+j] = offset + static_context_map[j]
for j = 0; j < 1<<literalContextBits; j++ {
mb.literal_context_map[(i<<literalContextBits)+j] = offset + static_context_map[j]
}
}
}
func BrotliBuildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut ContextLut, num_contexts uint, static_context_map []uint32, commands []command, n_commands uint, mb *MetaBlockSplit) {
func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, n_commands uint, mb *metaBlockSplit) {
var lit_blocks struct {
plain BlockSplitterLiteral
ctx ContextBlockSplitter
plain blockSplitterLiteral
ctx contextBlockSplitter
}
var cmd_blocks BlockSplitterCommand
var dist_blocks BlockSplitterDistance
var cmd_blocks blockSplitterCommand
var dist_blocks blockSplitterDistance
var num_literals uint = 0
var i uint
for i = 0; i < n_commands; i++ {
@ -510,25 +510,25 @@ func BrotliBuildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint,
}
if num_contexts == 1 {
InitBlockSplitterLiteral(&lit_blocks.plain, 256, 512, 400.0, num_literals, &mb.literal_split, &mb.literal_histograms, &mb.literal_histograms_size)
initBlockSplitterLiteral(&lit_blocks.plain, 256, 512, 400.0, num_literals, &mb.literal_split, &mb.literal_histograms, &mb.literal_histograms_size)
} else {
InitContextBlockSplitter(&lit_blocks.ctx, 256, num_contexts, 512, 400.0, num_literals, &mb.literal_split, &mb.literal_histograms, &mb.literal_histograms_size)
initContextBlockSplitter(&lit_blocks.ctx, 256, num_contexts, 512, 400.0, num_literals, &mb.literal_split, &mb.literal_histograms, &mb.literal_histograms_size)
}
InitBlockSplitterCommand(&cmd_blocks, numCommandSymbols, 1024, 500.0, n_commands, &mb.command_split, &mb.command_histograms, &mb.command_histograms_size)
InitBlockSplitterDistance(&dist_blocks, 64, 512, 100.0, n_commands, &mb.distance_split, &mb.distance_histograms, &mb.distance_histograms_size)
initBlockSplitterCommand(&cmd_blocks, numCommandSymbols, 1024, 500.0, n_commands, &mb.command_split, &mb.command_histograms, &mb.command_histograms_size)
initBlockSplitterDistance(&dist_blocks, 64, 512, 100.0, n_commands, &mb.distance_split, &mb.distance_histograms, &mb.distance_histograms_size)
for i = 0; i < n_commands; i++ {
var cmd command = commands[i]
var j uint
BlockSplitterAddSymbolCommand(&cmd_blocks, uint(cmd.cmd_prefix_))
blockSplitterAddSymbolCommand(&cmd_blocks, uint(cmd.cmd_prefix_))
for j = uint(cmd.insert_len_); j != 0; j-- {
var literal byte = ringbuffer[pos&mask]
if num_contexts == 1 {
BlockSplitterAddSymbolLiteral(&lit_blocks.plain, uint(literal))
blockSplitterAddSymbolLiteral(&lit_blocks.plain, uint(literal))
} else {
var context uint = uint(BROTLI_CONTEXT(prev_byte, prev_byte2, literal_context_lut))
ContextBlockSplitterAddSymbol(&lit_blocks.ctx, uint(literal), uint(static_context_map[context]))
var context uint = uint(getContext(prev_byte, prev_byte2, literal_context_lut))
contextBlockSplitterAddSymbol(&lit_blocks.ctx, uint(literal), uint(static_context_map[context]))
}
prev_byte2 = prev_byte
@ -541,45 +541,45 @@ func BrotliBuildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint,
prev_byte2 = ringbuffer[(pos-2)&mask]
prev_byte = ringbuffer[(pos-1)&mask]
if cmd.cmd_prefix_ >= 128 {
BlockSplitterAddSymbolDistance(&dist_blocks, uint(cmd.dist_prefix_)&0x3FF)
blockSplitterAddSymbolDistance(&dist_blocks, uint(cmd.dist_prefix_)&0x3FF)
}
}
}
if num_contexts == 1 {
BlockSplitterFinishBlockLiteral(&lit_blocks.plain, true) /* is_final = */
blockSplitterFinishBlockLiteral(&lit_blocks.plain, true) /* is_final = */
} else {
ContextBlockSplitterFinishBlock(&lit_blocks.ctx, true) /* is_final = */
contextBlockSplitterFinishBlock(&lit_blocks.ctx, true) /* is_final = */
}
BlockSplitterFinishBlockCommand(&cmd_blocks, true) /* is_final = */
BlockSplitterFinishBlockDistance(&dist_blocks, true) /* is_final = */
blockSplitterFinishBlockCommand(&cmd_blocks, true) /* is_final = */
blockSplitterFinishBlockDistance(&dist_blocks, true) /* is_final = */
if num_contexts > 1 {
MapStaticContexts(num_contexts, static_context_map, mb)
mapStaticContexts(num_contexts, static_context_map, mb)
}
}
func BrotliBuildMetaBlockGreedy(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut ContextLut, num_contexts uint, static_context_map []uint32, commands []command, n_commands uint, mb *MetaBlockSplit) {
func buildMetaBlockGreedy(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, n_commands uint, mb *metaBlockSplit) {
if num_contexts == 1 {
BrotliBuildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, 1, nil, commands, n_commands, mb)
buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, 1, nil, commands, n_commands, mb)
} else {
BrotliBuildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, num_contexts, static_context_map, commands, n_commands, mb)
buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, num_contexts, static_context_map, commands, n_commands, mb)
}
}
func BrotliOptimizeHistograms(num_distance_codes uint32, mb *MetaBlockSplit) {
func optimizeHistograms(num_distance_codes uint32, mb *metaBlockSplit) {
var good_for_rle [numCommandSymbols]byte
var i uint
for i = 0; i < mb.literal_histograms_size; i++ {
BrotliOptimizeHuffmanCountsForRle(256, mb.literal_histograms[i].data_[:], good_for_rle[:])
optimizeHuffmanCountsForRLE(256, mb.literal_histograms[i].data_[:], good_for_rle[:])
}
for i = 0; i < mb.command_histograms_size; i++ {
BrotliOptimizeHuffmanCountsForRle(numCommandSymbols, mb.command_histograms[i].data_[:], good_for_rle[:])
optimizeHuffmanCountsForRLE(numCommandSymbols, mb.command_histograms[i].data_[:], good_for_rle[:])
}
for i = 0; i < mb.distance_histograms_size; i++ {
BrotliOptimizeHuffmanCountsForRle(uint(num_distance_codes), mb.distance_histograms[i].data_[:], good_for_rle[:])
optimizeHuffmanCountsForRLE(uint(num_distance_codes), mb.distance_histograms[i].data_[:], good_for_rle[:])
}
}

32
metablock_command.go
View File

@ -9,13 +9,13 @@ package brotli
/* Greedy block splitter for one block category (literal, command or distance).
*/
type BlockSplitterCommand struct {
type blockSplitterCommand struct {
alphabet_size_ uint
min_block_size_ uint
split_threshold_ float64
num_blocks_ uint
split_ *blockSplit
histograms_ []HistogramCommand
histograms_ []histogramCommand
histograms_size_ *uint
target_block_size_ uint
block_size_ uint
@ -25,7 +25,7 @@ type BlockSplitterCommand struct {
merge_last_count_ uint
}
func InitBlockSplitterCommand(self *BlockSplitterCommand, alphabet_size uint, min_block_size uint, split_threshold float64, num_symbols uint, split *blockSplit, histograms *[]HistogramCommand, histograms_size *uint) {
func initBlockSplitterCommand(self *blockSplitterCommand, alphabet_size uint, min_block_size uint, split_threshold float64, num_symbols uint, split *blockSplit, histograms *[]histogramCommand, histograms_size *uint) {
var max_num_blocks uint = num_symbols/min_block_size + 1
var max_num_types uint = brotli_min_size_t(max_num_blocks, maxNumberOfBlockTypes+1)
/* We have to allocate one more histogram than the maximum number of block
@ -46,11 +46,11 @@ func InitBlockSplitterCommand(self *BlockSplitterCommand, alphabet_size uint, mi
self.split_.num_blocks = max_num_blocks
assert(*histograms == nil)
*histograms_size = max_num_types
*histograms = make([]HistogramCommand, (*histograms_size))
*histograms = make([]histogramCommand, (*histograms_size))
self.histograms_ = *histograms
/* Clear only current histogram. */
HistogramClearCommand(&self.histograms_[0])
histogramClearCommand(&self.histograms_[0])
self.last_histogram_ix_[1] = 0
self.last_histogram_ix_[0] = self.last_histogram_ix_[1]
@ -60,10 +60,10 @@ func InitBlockSplitterCommand(self *BlockSplitterCommand, alphabet_size uint, mi
(1) emits the current block with a new block type;
(2) emits the current block with the type of the second last block;
(3) merges the current block with the last block. */
func BlockSplitterFinishBlockCommand(self *BlockSplitterCommand, is_final bool) {
func blockSplitterFinishBlockCommand(self *blockSplitterCommand, is_final bool) {
var split *blockSplit = self.split_
var last_entropy []float64 = self.last_entropy_[:]
var histograms []HistogramCommand = self.histograms_
var histograms []histogramCommand = self.histograms_
self.block_size_ = brotli_max_size_t(self.block_size_, self.min_block_size_)
if self.num_blocks_ == 0 {
/* Create first block. */
@ -76,19 +76,19 @@ func BlockSplitterFinishBlockCommand(self *BlockSplitterCommand, is_final bool)
split.num_types++
self.curr_histogram_ix_++
if self.curr_histogram_ix_ < *self.histograms_size_ {
HistogramClearCommand(&histograms[self.curr_histogram_ix_])
histogramClearCommand(&histograms[self.curr_histogram_ix_])
}
self.block_size_ = 0
} else if self.block_size_ > 0 {
var entropy float64 = bitsEntropy(histograms[self.curr_histogram_ix_].data_[:], self.alphabet_size_)
var combined_histo [2]HistogramCommand
var combined_histo [2]histogramCommand
var combined_entropy [2]float64
var diff [2]float64
var j uint
for j = 0; j < 2; j++ {
var last_histogram_ix uint = self.last_histogram_ix_[j]
combined_histo[j] = histograms[self.curr_histogram_ix_]
HistogramAddHistogramCommand(&combined_histo[j], &histograms[last_histogram_ix])
histogramAddHistogramCommand(&combined_histo[j], &histograms[last_histogram_ix])
combined_entropy[j] = bitsEntropy(combined_histo[j].data_[0:], self.alphabet_size_)
diff[j] = combined_entropy[j] - entropy - last_entropy[j]
}
@ -106,7 +106,7 @@ func BlockSplitterFinishBlockCommand(self *BlockSplitterCommand, is_final bool)
split.num_types++
self.curr_histogram_ix_++
if self.curr_histogram_ix_ < *self.histograms_size_ {
HistogramClearCommand(&histograms[self.curr_histogram_ix_])
histogramClearCommand(&histograms[self.curr_histogram_ix_])
}
self.block_size_ = 0
self.merge_last_count_ = 0
@ -124,7 +124,7 @@ func BlockSplitterFinishBlockCommand(self *BlockSplitterCommand, is_final bool)
last_entropy[0] = combined_entropy[1]
self.num_blocks_++
self.block_size_ = 0
HistogramClearCommand(&histograms[self.curr_histogram_ix_])
histogramClearCommand(&histograms[self.curr_histogram_ix_])
self.merge_last_count_ = 0
self.target_block_size_ = self.min_block_size_
} else {
@ -138,7 +138,7 @@ func BlockSplitterFinishBlockCommand(self *BlockSplitterCommand, is_final bool)
}
self.block_size_ = 0
HistogramClearCommand(&histograms[self.curr_histogram_ix_])
histogramClearCommand(&histograms[self.curr_histogram_ix_])
self.merge_last_count_++
if self.merge_last_count_ > 1 {
self.target_block_size_ += self.min_block_size_
@ -154,10 +154,10 @@ func BlockSplitterFinishBlockCommand(self *BlockSplitterCommand, is_final bool)
/* Adds the next symbol to the current histogram. When the current histogram
reaches the target size, decides on merging the block. */
func BlockSplitterAddSymbolCommand(self *BlockSplitterCommand, symbol uint) {
HistogramAddCommand(&self.histograms_[self.curr_histogram_ix_], symbol)
func blockSplitterAddSymbolCommand(self *blockSplitterCommand, symbol uint) {
histogramAddCommand(&self.histograms_[self.curr_histogram_ix_], symbol)
self.block_size_++
if self.block_size_ == self.target_block_size_ {
BlockSplitterFinishBlockCommand(self, false) /* is_final = */
blockSplitterFinishBlockCommand(self, false) /* is_final = */
}
}

32
metablock_distance.go
View File

@ -9,13 +9,13 @@ package brotli
/* Greedy block splitter for one block category (literal, command or distance).
*/
type BlockSplitterDistance struct {
type blockSplitterDistance struct {
alphabet_size_ uint
min_block_size_ uint
split_threshold_ float64
num_blocks_ uint
split_ *blockSplit
histograms_ []HistogramDistance
histograms_ []histogramDistance
histograms_size_ *uint
target_block_size_ uint
block_size_ uint
@ -25,7 +25,7 @@ type BlockSplitterDistance struct {
merge_last_count_ uint
}
func InitBlockSplitterDistance(self *BlockSplitterDistance, alphabet_size uint, min_block_size uint, split_threshold float64, num_symbols uint, split *blockSplit, histograms *[]HistogramDistance, histograms_size *uint) {
func initBlockSplitterDistance(self *blockSplitterDistance, alphabet_size uint, min_block_size uint, split_threshold float64, num_symbols uint, split *blockSplit, histograms *[]histogramDistance, histograms_size *uint) {
var max_num_blocks uint = num_symbols/min_block_size + 1
var max_num_types uint = brotli_min_size_t(max_num_blocks, maxNumberOfBlockTypes+1)
/* We have to allocate one more histogram than the maximum number of block
@ -46,11 +46,11 @@ func InitBlockSplitterDistance(self *BlockSplitterDistance, alphabet_size uint,
self.split_.num_blocks = max_num_blocks
assert(*histograms == nil)
*histograms_size = max_num_types
*histograms = make([]HistogramDistance, (*histograms_size))
*histograms = make([]histogramDistance, (*histograms_size))
self.histograms_ = *histograms
/* Clear only current histogram. */
HistogramClearDistance(&self.histograms_[0])
histogramClearDistance(&self.histograms_[0])
self.last_histogram_ix_[1] = 0
self.last_histogram_ix_[0] = self.last_histogram_ix_[1]
@ -60,10 +60,10 @@ func InitBlockSplitterDistance(self *BlockSplitterDistance, alphabet_size uint,
(1) emits the current block with a new block type;
(2) emits the current block with the type of the second last block;
(3) merges the current block with the last block. */
func BlockSplitterFinishBlockDistance(self *BlockSplitterDistance, is_final bool) {
func blockSplitterFinishBlockDistance(self *blockSplitterDistance, is_final bool) {
var split *blockSplit = self.split_
var last_entropy []float64 = self.last_entropy_[:]
var histograms []HistogramDistance = self.histograms_
var histograms []histogramDistance = self.histograms_
self.block_size_ = brotli_max_size_t(self.block_size_, self.min_block_size_)
if self.num_blocks_ == 0 {
/* Create first block. */
@ -76,19 +76,19 @@ func BlockSplitterFinishBlockDistance(self *BlockSplitterDistance, is_final bool
split.num_types++
self.curr_histogram_ix_++
if self.curr_histogram_ix_ < *self.histograms_size_ {
HistogramClearDistance(&histograms[self.curr_histogram_ix_])
histogramClearDistance(&histograms[self.curr_histogram_ix_])
}
self.block_size_ = 0
} else if self.block_size_ > 0 {
var entropy float64 = bitsEntropy(histograms[self.curr_histogram_ix_].data_[:], self.alphabet_size_)
var combined_histo [2]HistogramDistance
var combined_histo [2]histogramDistance
var combined_entropy [2]float64
var diff [2]float64
var j uint
for j = 0; j < 2; j++ {
var last_histogram_ix uint = self.last_histogram_ix_[j]
combined_histo[j] = histograms[self.curr_histogram_ix_]
HistogramAddHistogramDistance(&combined_histo[j], &histograms[last_histogram_ix])
histogramAddHistogramDistance(&combined_histo[j], &histograms[last_histogram_ix])
combined_entropy[j] = bitsEntropy(combined_histo[j].data_[0:], self.alphabet_size_)
diff[j] = combined_entropy[j] - entropy - last_entropy[j]
}
@ -106,7 +106,7 @@ func BlockSplitterFinishBlockDistance(self *BlockSplitterDistance, is_final bool
split.num_types++
self.curr_histogram_ix_++
if self.curr_histogram_ix_ < *self.histograms_size_ {
HistogramClearDistance(&histograms[self.curr_histogram_ix_])
histogramClearDistance(&histograms[self.curr_histogram_ix_])
}
self.block_size_ = 0
self.merge_last_count_ = 0
@ -124,7 +124,7 @@ func BlockSplitterFinishBlockDistance(self *BlockSplitterDistance, is_final bool
last_entropy[0] = combined_entropy[1]
self.num_blocks_++
self.block_size_ = 0
HistogramClearDistance(&histograms[self.curr_histogram_ix_])
histogramClearDistance(&histograms[self.curr_histogram_ix_])
self.merge_last_count_ = 0
self.target_block_size_ = self.min_block_size_
} else {
@ -138,7 +138,7 @@ func BlockSplitterFinishBlockDistance(self *BlockSplitterDistance, is_final bool
}
self.block_size_ = 0
HistogramClearDistance(&histograms[self.curr_histogram_ix_])
histogramClearDistance(&histograms[self.curr_histogram_ix_])
self.merge_last_count_++
if self.merge_last_count_ > 1 {
self.target_block_size_ += self.min_block_size_
@ -154,10 +154,10 @@ func BlockSplitterFinishBlockDistance(self *BlockSplitterDistance, is_final bool
/* Adds the next symbol to the current histogram. When the current histogram
reaches the target size, decides on merging the block. */
func BlockSplitterAddSymbolDistance(self *BlockSplitterDistance, symbol uint) {
HistogramAddDistance(&self.histograms_[self.curr_histogram_ix_], symbol)
func blockSplitterAddSymbolDistance(self *blockSplitterDistance, symbol uint) {
histogramAddDistance(&self.histograms_[self.curr_histogram_ix_], symbol)
self.block_size_++
if self.block_size_ == self.target_block_size_ {
BlockSplitterFinishBlockDistance(self, false) /* is_final = */
blockSplitterFinishBlockDistance(self, false) /* is_final = */
}
}

32
metablock_literal.go
View File

@ -9,13 +9,13 @@ package brotli
/* Greedy block splitter for one block category (literal, command or distance).
*/
type BlockSplitterLiteral struct {
type blockSplitterLiteral struct {
alphabet_size_ uint
min_block_size_ uint
split_threshold_ float64
num_blocks_ uint
split_ *blockSplit
histograms_ []HistogramLiteral
histograms_ []histogramLiteral
histograms_size_ *uint
target_block_size_ uint
block_size_ uint
@ -25,7 +25,7 @@ type BlockSplitterLiteral struct {
merge_last_count_ uint
}
func InitBlockSplitterLiteral(self *BlockSplitterLiteral, alphabet_size uint, min_block_size uint, split_threshold float64, num_symbols uint, split *blockSplit, histograms *[]HistogramLiteral, histograms_size *uint) {
func initBlockSplitterLiteral(self *blockSplitterLiteral, alphabet_size uint, min_block_size uint, split_threshold float64, num_symbols uint, split *blockSplit, histograms *[]histogramLiteral, histograms_size *uint) {
var max_num_blocks uint = num_symbols/min_block_size + 1
var max_num_types uint = brotli_min_size_t(max_num_blocks, maxNumberOfBlockTypes+1)
/* We have to allocate one more histogram than the maximum number of block
@ -46,11 +46,11 @@ func InitBlockSplitterLiteral(self *BlockSplitterLiteral, alphabet_size uint, mi
self.split_.num_blocks = max_num_blocks
assert(*histograms == nil)
*histograms_size = max_num_types
*histograms = make([]HistogramLiteral, (*histograms_size))
*histograms = make([]histogramLiteral, (*histograms_size))
self.histograms_ = *histograms
/* Clear only current histogram. */
HistogramClearLiteral(&self.histograms_[0])
histogramClearLiteral(&self.histograms_[0])
self.last_histogram_ix_[1] = 0
self.last_histogram_ix_[0] = self.last_histogram_ix_[1]
@ -60,10 +60,10 @@ func InitBlockSplitterLiteral(self *BlockSplitterLiteral, alphabet_size uint, mi
(1) emits the current block with a new block type;
(2) emits the current block with the type of the second last block;
(3) merges the current block with the last block. */
func BlockSplitterFinishBlockLiteral(self *BlockSplitterLiteral, is_final bool) {
func blockSplitterFinishBlockLiteral(self *blockSplitterLiteral, is_final bool) {
var split *blockSplit = self.split_
var last_entropy []float64 = self.last_entropy_[:]
var histograms []HistogramLiteral = self.histograms_
var histograms []histogramLiteral = self.histograms_
self.block_size_ = brotli_max_size_t(self.block_size_, self.min_block_size_)
if self.num_blocks_ == 0 {
/* Create first block. */
@ -76,19 +76,19 @@ func BlockSplitterFinishBlockLiteral(self *BlockSplitterLiteral, is_final bool)
split.num_types++
self.curr_histogram_ix_++
if self.curr_histogram_ix_ < *self.histograms_size_ {
HistogramClearLiteral(&histograms[self.curr_histogram_ix_])
histogramClearLiteral(&histograms[self.curr_histogram_ix_])
}
self.block_size_ = 0
} else if self.block_size_ > 0 {
var entropy float64 = bitsEntropy(histograms[self.curr_histogram_ix_].data_[:], self.alphabet_size_)
var combined_histo [2]HistogramLiteral
var combined_histo [2]histogramLiteral
var combined_entropy [2]float64
var diff [2]float64
var j uint
for j = 0; j < 2; j++ {
var last_histogram_ix uint = self.last_histogram_ix_[j]
combined_histo[j] = histograms[self.curr_histogram_ix_]
HistogramAddHistogramLiteral(&combined_histo[j], &histograms[last_histogram_ix])
histogramAddHistogramLiteral(&combined_histo[j], &histograms[last_histogram_ix])
combined_entropy[j] = bitsEntropy(combined_histo[j].data_[0:], self.alphabet_size_)
diff[j] = combined_entropy[j] - entropy - last_entropy[j]
}
@ -106,7 +106,7 @@ func BlockSplitterFinishBlockLiteral(self *BlockSplitterLiteral, is_final bool)
split.num_types++
self.curr_histogram_ix_++
if self.curr_histogram_ix_ < *self.histograms_size_ {
HistogramClearLiteral(&histograms[self.curr_histogram_ix_])
histogramClearLiteral(&histograms[self.curr_histogram_ix_])
}
self.block_size_ = 0
self.merge_last_count_ = 0
@ -124,7 +124,7 @@ func BlockSplitterFinishBlockLiteral(self *BlockSplitterLiteral, is_final bool)
last_entropy[0] = combined_entropy[1]
self.num_blocks_++
self.block_size_ = 0
HistogramClearLiteral(&histograms[self.curr_histogram_ix_])
histogramClearLiteral(&histograms[self.curr_histogram_ix_])
self.merge_last_count_ = 0
self.target_block_size_ = self.min_block_size_
} else {
@ -138,7 +138,7 @@ func BlockSplitterFinishBlockLiteral(self *BlockSplitterLiteral, is_final bool)
}
self.block_size_ = 0
HistogramClearLiteral(&histograms[self.curr_histogram_ix_])
histogramClearLiteral(&histograms[self.curr_histogram_ix_])
self.merge_last_count_++
if self.merge_last_count_ > 1 {
self.target_block_size_ += self.min_block_size_
@ -154,10 +154,10 @@ func BlockSplitterFinishBlockLiteral(self *BlockSplitterLiteral, is_final bool)
/* Adds the next symbol to the current histogram. When the current histogram
reaches the target size, decides on merging the block. */
func BlockSplitterAddSymbolLiteral(self *BlockSplitterLiteral, symbol uint) {
HistogramAddLiteral(&self.histograms_[self.curr_histogram_ix_], symbol)
func blockSplitterAddSymbolLiteral(self *blockSplitterLiteral, symbol uint) {
histogramAddLiteral(&self.histograms_[self.curr_histogram_ix_], symbol)
self.block_size_++
if self.block_size_ == self.target_block_size_ {
BlockSplitterFinishBlockLiteral(self, false) /* is_final = */
blockSplitterFinishBlockLiteral(self, false) /* is_final = */
}
}

12
params.go
View File

@ -199,7 +199,7 @@ package brotli
*/
/* Parameters for the Brotli encoder with chosen quality levels. */
type BrotliHasherParams struct {
type hasherParams struct {
type_ int
bucket_bits int
block_bits int
@ -207,7 +207,7 @@ type BrotliHasherParams struct {
num_last_distances_to_check int
}
type BrotliDistanceParams struct {
type distanceParams struct {
distance_postfix_bits uint32
num_direct_distance_codes uint32
alphabet_size uint32
@ -215,7 +215,7 @@ type BrotliDistanceParams struct {
}
/* Encoding parameters */
type BrotliEncoderParams struct {
type encoderParams struct {
mode int
quality int
lgwin uint
@ -223,7 +223,7 @@ type BrotliEncoderParams struct {
size_hint uint
disable_literal_context_modeling bool
large_window bool
hasher BrotliHasherParams
dist BrotliDistanceParams
dictionary BrotliEncoderDictionary
hasher hasherParams
dist distanceParams
dictionary encoderDictionary
}

2
prefix.go
View File

@ -39,7 +39,7 @@ func PrefixEncodeCopyDistance(distance_code uint, num_direct_codes uint, postfix
return
} else {
var dist uint = (uint(1) << (postfix_bits + 2)) + (distance_code - numDistanceShortCodes - num_direct_codes)
var bucket uint = uint(Log2FloorNonZero(dist) - 1)
var bucket uint = uint(log2FloorNonZero(dist) - 1)
var postfix_mask uint = (1 << postfix_bits) - 1
var postfix uint = dist & postfix_mask
var prefix uint = (dist >> bucket) & 1

2
prefix_dec.go
View File

@ -11,7 +11,7 @@ package brotli
/* Represents the range of values belonging to a prefix code:
[offset, offset + 2^nbits) */
type PrefixCodeRange1 struct {
type prefixCodeRange1 struct {
offset uint16
nbits byte
}

30
quality.go
View File

@ -45,7 +45,7 @@ const MAX_ZOPFLI_LEN_QUALITY_11 = 325
/* Do not thoroughly search when a long copy is found. */
const BROTLI_LONG_COPY_QUICK_STEP = 16384
func MaxZopfliLen(params *BrotliEncoderParams) uint {
func MaxZopfliLen(params *encoderParams) uint {
if params.quality <= 10 {
return MAX_ZOPFLI_LEN_QUALITY_10
} else {
@ -54,7 +54,7 @@ func MaxZopfliLen(params *BrotliEncoderParams) uint {
}
/* Number of best candidates to evaluate to expand Zopfli chain. */
func MaxZopfliCandidates(params *BrotliEncoderParams) uint {
func MaxZopfliCandidates(params *encoderParams) uint {
if params.quality <= 10 {
return 1
} else {
@ -62,20 +62,20 @@ func MaxZopfliCandidates(params *BrotliEncoderParams) uint {
}
}
func SanitizeParams(params *BrotliEncoderParams) {
params.quality = brotli_min_int(BROTLI_MAX_QUALITY, brotli_max_int(BROTLI_MIN_QUALITY, params.quality))
func SanitizeParams(params *encoderParams) {
params.quality = brotli_min_int(maxQuality, brotli_max_int(minQuality, params.quality))
if params.quality <= MAX_QUALITY_FOR_STATIC_ENTROPY_CODES {
params.large_window = false
}
if params.lgwin < BROTLI_MIN_WINDOW_BITS {
params.lgwin = BROTLI_MIN_WINDOW_BITS
if params.lgwin < minWindowBits {
params.lgwin = minWindowBits
} else {
var max_lgwin int
if params.large_window {
max_lgwin = BROTLI_LARGE_MAX_WINDOW_BITS
max_lgwin = largeMaxWindowBits
} else {
max_lgwin = BROTLI_MAX_WINDOW_BITS
max_lgwin = maxWindowBits
}
if params.lgwin > uint(max_lgwin) {
params.lgwin = uint(max_lgwin)
@ -84,7 +84,7 @@ func SanitizeParams(params *BrotliEncoderParams) {
}
/* Returns optimized lg_block value. */
func ComputeLgBlock(params *BrotliEncoderParams) int {
func ComputeLgBlock(params *encoderParams) int {
var lgblock int = params.lgblock
if params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY || params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY {
lgblock = int(params.lgwin)
@ -96,7 +96,7 @@ func ComputeLgBlock(params *BrotliEncoderParams) int {
lgblock = brotli_min_int(18, int(params.lgwin))
}
} else {
lgblock = brotli_min_int(BROTLI_MAX_INPUT_BLOCK_BITS, brotli_max_int(BROTLI_MIN_INPUT_BLOCK_BITS, lgblock))
lgblock = brotli_min_int(maxInputBlockBits, brotli_max_int(minInputBlockBits, lgblock))
}
return lgblock
@ -107,12 +107,12 @@ func ComputeLgBlock(params *BrotliEncoderParams) int {
added block fits there completely and we still get lgwin bits and at least
read_block_size_bits + 1 bits because the copy tail length needs to be
smaller than ring-buffer size. */
func ComputeRbBits(params *BrotliEncoderParams) int {
func ComputeRbBits(params *encoderParams) int {
return 1 + brotli_max_int(int(params.lgwin), params.lgblock)
}
func MaxMetablockSize(params *BrotliEncoderParams) uint {
var bits int = brotli_min_int(ComputeRbBits(params), BROTLI_MAX_INPUT_BLOCK_BITS)
func MaxMetablockSize(params *encoderParams) uint {
var bits int = brotli_min_int(ComputeRbBits(params), maxInputBlockBits)
return uint(1) << uint(bits)
}
@ -122,7 +122,7 @@ func MaxMetablockSize(params *BrotliEncoderParams) uint {
At first 8 byte strides are taken and every second byte is put to hasher.
After 4x more literals stride by 16 bytes, every put 4-th byte to hasher.
Applied only to qualities 2 to 9. */
func LiteralSpreeLengthForSparseSearch(params *BrotliEncoderParams) uint {
func LiteralSpreeLengthForSparseSearch(params *encoderParams) uint {
if params.quality < 9 {
return 64
} else {
@ -130,7 +130,7 @@ func LiteralSpreeLengthForSparseSearch(params *BrotliEncoderParams) uint {
}
}
func ChooseHasher(params *BrotliEncoderParams, hparams *BrotliHasherParams) {
func ChooseHasher(params *encoderParams, hparams *hasherParams) {
if params.quality > 9 {
hparams.type_ = 10
} else if params.quality == 4 && params.size_hint >= 1<<20 {

16
reader.go
View File

@ -8,7 +8,7 @@ import (
type decodeError int
func (err decodeError) Error() string {
return "brotli: " + string(BrotliDecoderErrorString(int(err)))
return "brotli: " + string(decoderErrorString(int(err)))
}
var errExcessiveInput = errors.New("brotli: excessive input")
@ -30,7 +30,7 @@ func NewReader(src io.Reader) *Reader {
}
func (r *Reader) Read(p []byte) (n int, err error) {
if !BrotliDecoderHasMoreOutput(r) && len(r.in) == 0 {
if !decoderHasMoreOutput(r) && len(r.in) == 0 {
m, readErr := r.src.Read(r.buf)
if m == 0 {
// If readErr is `nil`, we just proxy underlying stream behavior.
@ -49,24 +49,24 @@ func (r *Reader) Read(p []byte) (n int, err error) {
out_len := uint(len(p))
in_remaining := in_len
out_remaining := out_len
result := BrotliDecoderDecompressStream(r, &in_remaining, &r.in, &out_remaining, &p)
result := decoderDecompressStream(r, &in_remaining, &r.in, &out_remaining, &p)
written = out_len - out_remaining
n = int(written)
switch result {
case BROTLI_DECODER_RESULT_SUCCESS:
case decoderResultSuccess:
if len(r.in) > 0 {
return n, errExcessiveInput
}
return n, nil
case BROTLI_DECODER_RESULT_ERROR:
return n, decodeError(BrotliDecoderGetErrorCode(r))
case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
case decoderResultError:
return n, decodeError(decoderGetErrorCode(r))
case decoderResultNeedsMoreOutput:
if n == 0 {
return 0, io.ErrShortBuffer
}
return n, nil
case BROTLI_DECODER_NEEDS_MORE_INPUT:
case decoderNeedsMoreInput:
}
if len(r.in) != 0 {

2
ringbuffer.go
View File

@ -42,7 +42,7 @@ func RingBufferInit(rb *RingBuffer) {
rb.buffer_ = nil
}
func RingBufferSetup(params *BrotliEncoderParams, rb *RingBuffer) {
func RingBufferSetup(params *encoderParams, rb *RingBuffer) {
var window_bits int = ComputeRbBits(params)
var tail_bits int = params.lgblock
*(*uint32)(&rb.size_) = 1 << uint(window_bits)

34
state.go
View File

@ -115,15 +115,15 @@ type Reader struct {
sub_loop_counter uint32
ringbuffer []byte
ringbuffer_end []byte
htree_command []HuffmanCode
htree_command []huffmanCode
context_lookup []byte
context_map_slice []byte
dist_context_map_slice []byte
literal_hgroup HuffmanTreeGroup
insert_copy_hgroup HuffmanTreeGroup
distance_hgroup HuffmanTreeGroup
block_type_trees []HuffmanCode
block_len_trees []HuffmanCode
literal_hgroup huffmanTreeGroup
insert_copy_hgroup huffmanTreeGroup
distance_hgroup huffmanTreeGroup
block_type_trees []huffmanCode
block_len_trees []huffmanCode
trivial_literal_context int
distance_context int
meta_block_remaining_len int
@ -136,7 +136,7 @@ type Reader struct {
distance_postfix_mask int
num_dist_htrees uint32
dist_context_map []byte
literal_htree []HuffmanCode
literal_htree []huffmanCode
dist_htree_index byte
repeat_code_len uint32
prev_code_len uint32
@ -147,18 +147,18 @@ type Reader struct {
symbol uint32
repeat uint32
space uint32
table [32]HuffmanCode
table [32]huffmanCode
symbol_lists SymbolList
symbols_lists_array [BROTLI_HUFFMAN_MAX_CODE_LENGTH + 1 + numCommandSymbols]uint16
symbols_lists_array [huffmanMaxCodeLength + 1 + numCommandSymbols]uint16
next_symbol [32]int
code_length_code_lengths [codeLengthCodes]byte
code_length_histo [16]uint16
htree_index int
next []HuffmanCode
next []huffmanCode
context_index uint32
max_run_length_prefix uint32
code uint32
context_map_table [BROTLI_HUFFMAN_MAX_SIZE_272]HuffmanCode
context_map_table [huffmanMaxSize272]huffmanCode
substate_metablock_header int
substate_tree_group int
substate_context_map int
@ -178,7 +178,7 @@ type Reader struct {
num_literal_htrees uint32
context_map []byte
context_modes []byte
dictionary *BrotliDictionary
dictionary *dictionary
transforms *BrotliTransforms
trivial_literal_contexts [8]uint32
}
@ -242,9 +242,9 @@ func BrotliDecoderStateInit(s *Reader) bool {
s.block_len_trees = nil
s.symbol_lists.storage = s.symbols_lists_array[:]
s.symbol_lists.offset = BROTLI_HUFFMAN_MAX_CODE_LENGTH + 1
s.symbol_lists.offset = huffmanMaxCodeLength + 1
s.dictionary = BrotliGetDictionary()
s.dictionary = getDictionary()
s.transforms = BrotliGetTransforms()
return true
@ -296,12 +296,12 @@ func BrotliDecoderStateCleanup(s *Reader) {
s.block_type_trees = nil
}
func BrotliDecoderHuffmanTreeGroupInit(s *Reader, group *HuffmanTreeGroup, alphabet_size uint32, max_symbol uint32, ntrees uint32) bool {
func BrotliDecoderHuffmanTreeGroupInit(s *Reader, group *huffmanTreeGroup, alphabet_size uint32, max_symbol uint32, ntrees uint32) bool {
var max_table_size uint = uint(kMaxHuffmanTableSize[(alphabet_size+31)>>5])
group.alphabet_size = uint16(alphabet_size)
group.max_symbol = uint16(max_symbol)
group.num_htrees = uint16(ntrees)
group.htrees = make([][]HuffmanCode, ntrees)
group.codes = make([]HuffmanCode, (uint(ntrees) * max_table_size))
group.htrees = make([][]huffmanCode, ntrees)
group.codes = make([]huffmanCode, (uint(ntrees) * max_table_size))
return !(group.codes == nil)
}

56
static_dict.go
View File

@ -31,24 +31,24 @@ func AddMatch(distance uint, len uint, len_code uint, matches []uint32) {
matches[len] = brotli_min_uint32_t(matches[len], match)
}
func DictMatchLength(dictionary *BrotliDictionary, data []byte, id uint, len uint, maxlen uint) uint {
var offset uint = uint(dictionary.offsets_by_length[len]) + len*id
return FindMatchLengthWithLimit(dictionary.data[offset:], data, brotli_min_size_t(uint(len), maxlen))
func DictMatchLength(dict *dictionary, data []byte, id uint, len uint, maxlen uint) uint {
var offset uint = uint(dict.offsets_by_length[len]) + len*id
return findMatchLengthWithLimit(dict.data[offset:], data, brotli_min_size_t(uint(len), maxlen))
}
func IsMatch(dictionary *BrotliDictionary, w DictWord, data []byte, max_length uint) bool {
func IsMatch(d *dictionary, w DictWord, data []byte, max_length uint) bool {
if uint(w.len) > max_length {
return false
} else {
var offset uint = uint(dictionary.offsets_by_length[w.len]) + uint(w.len)*uint(w.idx)
var dict []byte = dictionary.data[offset:]
var offset uint = uint(d.offsets_by_length[w.len]) + uint(w.len)*uint(w.idx)
var dict []byte = d.data[offset:]
if w.transform == 0 {
/* Match against base dictionary word. */
return FindMatchLengthWithLimit(dict, data, uint(w.len)) == uint(w.len)
return findMatchLengthWithLimit(dict, data, uint(w.len)) == uint(w.len)
} else if w.transform == 10 {
/* Match against uppercase first transform.
Note that there are only ASCII uppercase words in the lookup table. */
return dict[0] >= 'a' && dict[0] <= 'z' && (dict[0]^32) == data[0] && FindMatchLengthWithLimit(dict[1:], data[1:], uint(w.len)-1) == uint(w.len-1)
return dict[0] >= 'a' && dict[0] <= 'z' && (dict[0]^32) == data[0] && findMatchLengthWithLimit(dict[1:], data[1:], uint(w.len)-1) == uint(w.len-1)
} else {
/* Match against uppercase all transform.
Note that there are only ASCII uppercase words in the lookup table. */
@ -70,22 +70,22 @@ func IsMatch(dictionary *BrotliDictionary, w DictWord, data []byte, max_length u
}
}
func BrotliFindAllStaticDictionaryMatches(dictionary *BrotliEncoderDictionary, data []byte, min_length uint, max_length uint, matches []uint32) bool {
func BrotliFindAllStaticDictionaryMatches(dict *encoderDictionary, data []byte, min_length uint, max_length uint, matches []uint32) bool {
var has_found_match bool = false
{
var offset uint = uint(dictionary.buckets[Hash(data)])
var offset uint = uint(dict.buckets[Hash(data)])
var end bool = offset == 0
for !end {
var w DictWord
w = dictionary.dict_words[offset]
w = dict.dict_words[offset]
offset++
var l uint = uint(w.len) & 0x1F
var n uint = uint(1) << dictionary.words.size_bits_by_length[l]
var n uint = uint(1) << dict.words.size_bits_by_length[l]
var id uint = uint(w.idx)
end = !(w.len&0x80 == 0)
w.len = byte(l)
if w.transform == 0 {
var matchlen uint = DictMatchLength(dictionary.words, data, id, l, max_length)
var matchlen uint = DictMatchLength(dict.words, data, id, l, max_length)
var s []byte
var minlen uint
var maxlen uint
@ -117,7 +117,7 @@ func BrotliFindAllStaticDictionaryMatches(dictionary *BrotliEncoderDictionary, d
maxlen = brotli_min_size_t(matchlen, l-2)
for len = minlen; len <= maxlen; len++ {
var cut uint = l - len
var transform_id uint = (cut << 2) + uint((dictionary.cutoffTransforms>>(cut*6))&0x3F)
var transform_id uint = (cut << 2) + uint((dict.cutoffTransforms>>(cut*6))&0x3F)
AddMatch(id+transform_id*n, uint(len), l, matches)
has_found_match = true
}
@ -305,7 +305,7 @@ func BrotliFindAllStaticDictionaryMatches(dictionary *BrotliEncoderDictionary, d
transform. */
var s []byte
if !IsMatch(dictionary.words, w, data, max_length) {
if !IsMatch(dict.words, w, data, max_length) {
continue
}
@ -427,20 +427,20 @@ func BrotliFindAllStaticDictionaryMatches(dictionary *BrotliEncoderDictionary, d
/* Transforms with prefixes " " and "." */
if max_length >= 5 && (data[0] == ' ' || data[0] == '.') {
var is_space bool = (data[0] == ' ')
var offset uint = uint(dictionary.buckets[Hash(data[1:])])
var offset uint = uint(dict.buckets[Hash(data[1:])])
var end bool = offset == 0
for !end {
var w DictWord
w = dictionary.dict_words[offset]
w = dict.dict_words[offset]
offset++
var l uint = uint(w.len) & 0x1F
var n uint = uint(1) << dictionary.words.size_bits_by_length[l]
var n uint = uint(1) << dict.words.size_bits_by_length[l]
var id uint = uint(w.idx)
end = !(w.len&0x80 == 0)
w.len = byte(l)
if w.transform == 0 {
var s []byte
if !IsMatch(dictionary.words, w, data[1:], max_length-1) {
if !IsMatch(dict.words, w, data[1:], max_length-1) {
continue
}
@ -506,7 +506,7 @@ func BrotliFindAllStaticDictionaryMatches(dictionary *BrotliEncoderDictionary, d
transform. */
var s []byte
if !IsMatch(dictionary.words, w, data[1:], max_length-1) {
if !IsMatch(dict.words, w, data[1:], max_length-1) {
continue
}
@ -592,18 +592,18 @@ func BrotliFindAllStaticDictionaryMatches(dictionary *BrotliEncoderDictionary, d
if max_length >= 6 {
/* Transforms with prefixes "e ", "s ", ", " and "\xC2\xA0" */
if (data[1] == ' ' && (data[0] == 'e' || data[0] == 's' || data[0] == ',')) || (data[0] == 0xC2 && data[1] == 0xA0) {
var offset uint = uint(dictionary.buckets[Hash(data[2:])])
var offset uint = uint(dict.buckets[Hash(data[2:])])
var end bool = offset == 0
for !end {
var w DictWord
w = dictionary.dict_words[offset]
w = dict.dict_words[offset]
offset++
var l uint = uint(w.len) & 0x1F
var n uint = uint(1) << dictionary.words.size_bits_by_length[l]
var n uint = uint(1) << dict.words.size_bits_by_length[l]
var id uint = uint(w.idx)
end = !(w.len&0x80 == 0)
w.len = byte(l)
if w.transform == 0 && IsMatch(dictionary.words, w, data[2:], max_length-2) {
if w.transform == 0 && IsMatch(dict.words, w, data[2:], max_length-2) {
if data[0] == 0xC2 {
AddMatch(id+102*n, l+2, l, matches)
has_found_match = true
@ -625,18 +625,18 @@ func BrotliFindAllStaticDictionaryMatches(dictionary *BrotliEncoderDictionary, d
if max_length >= 9 {
/* Transforms with prefixes " the " and ".com/" */
if (data[0] == ' ' && data[1] == 't' && data[2] == 'h' && data[3] == 'e' && data[4] == ' ') || (data[0] == '.' && data[1] == 'c' && data[2] == 'o' && data[3] == 'm' && data[4] == '/') {
var offset uint = uint(dictionary.buckets[Hash(data[5:])])
var offset uint = uint(dict.buckets[Hash(data[5:])])
var end bool = offset == 0
for !end {
var w DictWord
w = dictionary.dict_words[offset]
w = dict.dict_words[offset]
offset++
var l uint = uint(w.len) & 0x1F
var n uint = uint(1) << dictionary.words.size_bits_by_length[l]
var n uint = uint(1) << dict.words.size_bits_by_length[l]
var id uint = uint(w.idx)
end = !(w.len&0x80 == 0)
w.len = byte(l)
if w.transform == 0 && IsMatch(dictionary.words, w, data[5:], max_length-5) {
if w.transform == 0 && IsMatch(dict.words, w, data[5:], max_length-5) {
var tmp int
if data[0] == ' ' {
tmp = 41

12
writer.go
View File

@ -23,7 +23,7 @@ var (
// NewWriter initializes new Writer instance.
func NewWriter(dst io.Writer, options WriterOptions) *Writer {
w := new(Writer)
BrotliEncoderInitState(w)
encoderInitState(w)
w.params.quality = options.Quality
if options.LGWin > 0 {
w.params.lgwin = uint(options.LGWin)
@ -40,7 +40,7 @@ func (w *Writer) writeChunk(p []byte, op int) (n int, err error) {
for {
availableIn := uint(len(p))
nextIn := p
success := BrotliEncoderCompressStream(w, op, &availableIn, &nextIn)
success := encoderCompressStream(w, op, &availableIn, &nextIn)
bytesConsumed := len(p) - int(availableIn)
p = p[bytesConsumed:]
n += bytesConsumed
@ -48,7 +48,7 @@ func (w *Writer) writeChunk(p []byte, op int) (n int, err error) {
return n, errEncode
}
outputData := BrotliEncoderTakeOutput(w)
outputData := encoderTakeOutput(w)
if len(outputData) > 0 {
_, err = w.dst.Write(outputData)
@ -67,14 +67,14 @@ func (w *Writer) writeChunk(p []byte, op int) (n int, err error) {
// not yet complete until after Close.
// Flush has a negative impact on compression.
func (w *Writer) Flush() error {
_, err := w.writeChunk(nil, BROTLI_OPERATION_FLUSH)
_, err := w.writeChunk(nil, operationFlush)
return err
}
// Close flushes remaining data to the decorated writer.
func (w *Writer) Close() error {
// If stream is already closed, it is reported by `writeChunk`.
_, err := w.writeChunk(nil, BROTLI_OPERATION_FINISH)
_, err := w.writeChunk(nil, operationFinish)
w.dst = nil
return err
}
@ -82,5 +82,5 @@ func (w *Writer) Close() error {
// Write implements io.Writer. Flush or Close must be called to ensure that the
// encoded bytes are actually flushed to the underlying Writer.
func (w *Writer) Write(p []byte) (n int, err error) {
return w.writeChunk(p, BROTLI_OPERATION_PROCESS)
return w.writeChunk(p, operationProcess)
}