Use len and cap instead of num_commands_ and cmd_alloc_size_.

This commit is contained in:
Andy Balholm 2020-05-08 16:48:16 -07:00
parent 4b2775ea5e
commit e2c5f2109f
9 changed files with 108 additions and 137 deletions

View File

@ -31,13 +31,8 @@ func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uin
return distance + numDistanceShortCodes - 1 return distance + numDistanceShortCodes - 1
} }
/* "commands" points to the next output command to write to, "*num_commands" is func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) {
initially the total amount of commands output by previous
CreateBackwardReferences calls, and must be incremented by the amount written
by this call. */
func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
var max_backward_limit uint = maxBackwardLimit(params.lgwin) var max_backward_limit uint = maxBackwardLimit(params.lgwin)
var orig_commands []command = commands
var insert_length uint = *last_insert_len var insert_length uint = *last_insert_len
var pos_end uint = position + num_bytes var pos_end uint = position + num_bytes
var store_end uint var store_end uint
@ -114,8 +109,7 @@ func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte,
hasher.PrepareDistanceCache(dist_cache) hasher.PrepareDistanceCache(dist_cache)
} }
initCommand(&commands[0], &params.dist, insert_length, sr.len, sr.len_code_delta, distance_code) *commands = append(*commands, makeCommand(&params.dist, insert_length, sr.len, sr.len_code_delta, distance_code))
commands = commands[1:]
} }
*num_literals += insert_length *num_literals += insert_length
@ -173,5 +167,4 @@ func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte,
insert_length += pos_end - position insert_length += pos_end - position
*last_insert_len = insert_length *last_insert_len = insert_length
*num_commands += uint(-cap(commands) + cap(orig_commands))
} }

View File

@ -123,14 +123,13 @@ func setCost(histogram []uint32, histogram_size uint, literal_histogram bool, co
} }
} }
func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint, commands []command, num_commands uint, last_insert_len uint) { func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint, commands []command, last_insert_len uint) {
var histogram_literal [numLiteralSymbols]uint32 var histogram_literal [numLiteralSymbols]uint32
var histogram_cmd [numCommandSymbols]uint32 var histogram_cmd [numCommandSymbols]uint32
var histogram_dist [maxEffectiveDistanceAlphabetSize]uint32 var histogram_dist [maxEffectiveDistanceAlphabetSize]uint32
var cost_literal [numLiteralSymbols]float32 var cost_literal [numLiteralSymbols]float32
var pos uint = position - last_insert_len var pos uint = position - last_insert_len
var min_cost_cmd float32 = kInfinity var min_cost_cmd float32 = kInfinity
var i uint
var cost_cmd []float32 = self.cost_cmd_[:] var cost_cmd []float32 = self.cost_cmd_[:]
var literal_costs []float32 var literal_costs []float32
@ -138,7 +137,7 @@ func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbu
histogram_cmd = [numCommandSymbols]uint32{} histogram_cmd = [numCommandSymbols]uint32{}
histogram_dist = [maxEffectiveDistanceAlphabetSize]uint32{} histogram_dist = [maxEffectiveDistanceAlphabetSize]uint32{}
for i = 0; i < num_commands; i++ { for i := range commands {
var inslength uint = uint(commands[i].insert_len_) var inslength uint = uint(commands[i].insert_len_)
var copylength uint = uint(commandCopyLen(&commands[i])) var copylength uint = uint(commandCopyLen(&commands[i]))
var distcode uint = uint(commands[i].dist_prefix_) & 0x3FF var distcode uint = uint(commands[i].dist_prefix_) & 0x3FF
@ -161,7 +160,7 @@ func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbu
setCost(histogram_cmd[:], numCommandSymbols, false, cost_cmd) setCost(histogram_cmd[:], numCommandSymbols, false, cost_cmd)
setCost(histogram_dist[:], uint(self.distance_histogram_size), false, self.cost_dist_) setCost(histogram_dist[:], uint(self.distance_histogram_size), false, self.cost_dist_)
for i = 0; i < numCommandSymbols; i++ { for i := 0; i < numCommandSymbols; i++ {
min_cost_cmd = brotli_min_float(min_cost_cmd, cost_cmd[i]) min_cost_cmd = brotli_min_float(min_cost_cmd, cost_cmd[i])
} }
@ -169,10 +168,10 @@ func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbu
{ {
literal_costs = self.literal_costs_ literal_costs = self.literal_costs_
var literal_carry float32 = 0.0 var literal_carry float32 = 0.0
var num_bytes uint = self.num_bytes_ num_bytes := int(self.num_bytes_)
literal_costs[0] = 0.0 literal_costs[0] = 0.0
for i = 0; i < num_bytes; i++ { for i := 0; i < num_bytes; i++ {
literal_carry += cost_literal[ringbuffer[(position+i)&ringbuffer_mask]] literal_carry += cost_literal[ringbuffer[(position+uint(i))&ringbuffer_mask]]
literal_costs[i+1] = literal_costs[i] + literal_carry literal_costs[i+1] = literal_costs[i] + literal_carry
literal_carry -= literal_costs[i+1] - literal_costs[i] literal_carry -= literal_costs[i+1] - literal_costs[i]
} }
@ -532,7 +531,7 @@ func computeShortestPathFromNodes(num_bytes uint, nodes []zopfliNode) uint {
} }
/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */ /* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands []command, num_literals *uint) { func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands *[]command, num_literals *uint) {
var max_backward_limit uint = maxBackwardLimit(params.lgwin) var max_backward_limit uint = maxBackwardLimit(params.lgwin)
var pos uint = 0 var pos uint = 0
var offset uint32 = nodes[0].u.next var offset uint32 = nodes[0].u.next
@ -554,7 +553,7 @@ func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode,
var max_distance uint = brotli_min_size_t(block_start+pos, max_backward_limit) var max_distance uint = brotli_min_size_t(block_start+pos, max_backward_limit)
var is_dictionary bool = (distance > max_distance+gap) var is_dictionary bool = (distance > max_distance+gap)
var dist_code uint = uint(zopfliNodeDistanceCode(next)) var dist_code uint = uint(zopfliNodeDistanceCode(next))
initCommand(&commands[i], &params.dist, insert_length, copy_length, int(len_code)-int(copy_length), dist_code) *commands = append(*commands, makeCommand(&params.dist, insert_length, copy_length, int(len_code)-int(copy_length), dist_code))
if !is_dictionary && dist_code > 0 { if !is_dictionary && dist_code > 0 {
dist_cache[3] = dist_cache[2] dist_cache[3] = dist_cache[2]
@ -681,16 +680,16 @@ func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte,
return computeShortestPathFromNodes(num_bytes, nodes) return computeShortestPathFromNodes(num_bytes, nodes)
} }
func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) { func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) {
var nodes []zopfliNode var nodes []zopfliNode
nodes = make([]zopfliNode, (num_bytes + 1)) nodes = make([]zopfliNode, (num_bytes + 1))
initZopfliNodes(nodes, num_bytes+1) initZopfliNodes(nodes, num_bytes+1)
*num_commands += zopfliComputeShortestPath(num_bytes, position, ringbuffer, ringbuffer_mask, params, dist_cache, hasher, nodes) zopfliComputeShortestPath(num_bytes, position, ringbuffer, ringbuffer_mask, params, dist_cache, hasher, nodes)
zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals) zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals)
nodes = nil nodes = nil
} }
func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) { func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) {
var max_backward_limit uint = maxBackwardLimit(params.lgwin) var max_backward_limit uint = maxBackwardLimit(params.lgwin)
var num_matches []uint32 = make([]uint32, num_bytes) var num_matches []uint32 = make([]uint32, num_bytes)
var matches_size uint = 4 * num_bytes var matches_size uint = 4 * num_bytes
@ -705,7 +704,7 @@ func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer
var orig_num_literals uint var orig_num_literals uint
var orig_last_insert_len uint var orig_last_insert_len uint
var orig_dist_cache [4]int var orig_dist_cache [4]int
var orig_num_commands uint var orig_num_commands int
var model zopfliCostModel var model zopfliCostModel
var nodes []zopfliNode var nodes []zopfliNode
var matches []backwardMatch = make([]backwardMatch, matches_size) var matches []backwardMatch = make([]backwardMatch, matches_size)
@ -771,7 +770,7 @@ func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer
orig_num_literals = *num_literals orig_num_literals = *num_literals
orig_last_insert_len = *last_insert_len orig_last_insert_len = *last_insert_len
copy(orig_dist_cache[:], dist_cache[:4]) copy(orig_dist_cache[:], dist_cache[:4])
orig_num_commands = *num_commands orig_num_commands = len(*commands)
nodes = make([]zopfliNode, (num_bytes + 1)) nodes = make([]zopfliNode, (num_bytes + 1))
initZopfliCostModel(&model, &params.dist, num_bytes) initZopfliCostModel(&model, &params.dist, num_bytes)
for i = 0; i < 2; i++ { for i = 0; i < 2; i++ {
@ -779,14 +778,14 @@ func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer
if i == 0 { if i == 0 {
zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask) zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask)
} else { } else {
zopfliCostModelSetFromCommands(&model, position, ringbuffer, ringbuffer_mask, commands, *num_commands-orig_num_commands, orig_last_insert_len) zopfliCostModelSetFromCommands(&model, position, ringbuffer, ringbuffer_mask, (*commands)[orig_num_commands:], orig_last_insert_len)
} }
*num_commands = orig_num_commands *commands = (*commands)[:orig_num_commands]
*num_literals = orig_num_literals *num_literals = orig_num_literals
*last_insert_len = orig_last_insert_len *last_insert_len = orig_last_insert_len
copy(dist_cache, orig_dist_cache[:4]) copy(dist_cache, orig_dist_cache[:4])
*num_commands += zopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, nodes) zopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, nodes)
zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals) zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals)
} }

View File

@ -33,23 +33,21 @@ const (
kMinItersForRefining uint = 100 kMinItersForRefining uint = 100
) )
func countLiterals(cmds []command, num_commands uint) uint { func countLiterals(cmds []command) uint {
var total_length uint = 0 var total_length uint = 0
/* Count how many we have. */ /* Count how many we have. */
var i uint for i := range cmds {
for i = 0; i < num_commands; i++ {
total_length += uint(cmds[i].insert_len_) total_length += uint(cmds[i].insert_len_)
} }
return total_length return total_length
} }
func copyLiteralsToByteArray(cmds []command, num_commands uint, data []byte, offset uint, mask uint, literals []byte) { func copyLiteralsToByteArray(cmds []command, data []byte, offset uint, mask uint, literals []byte) {
var pos uint = 0 var pos uint = 0
var from_pos uint = offset & mask var from_pos uint = offset & mask
var i uint for i := range cmds {
for i = 0; i < num_commands; i++ {
var insert_len uint = uint(cmds[i].insert_len_) var insert_len uint = uint(cmds[i].insert_len_)
if from_pos+insert_len > mask { if from_pos+insert_len > mask {
var head_size uint = mask + 1 - from_pos var head_size uint = mask + 1 - from_pos
@ -101,13 +99,13 @@ func destroyBlockSplit(self *blockSplit) {
self.lengths = nil self.lengths = nil
} }
func splitBlock(cmds []command, num_commands uint, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) { func splitBlock(cmds []command, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) {
{ {
var literals_count uint = countLiterals(cmds, num_commands) var literals_count uint = countLiterals(cmds)
var literals []byte = make([]byte, literals_count) var literals []byte = make([]byte, literals_count)
/* Create a continuous array of literals. */ /* Create a continuous array of literals. */
copyLiteralsToByteArray(cmds, num_commands, data, pos, mask, literals) copyLiteralsToByteArray(cmds, data, pos, mask, literals)
/* Create the block split on the array of literals. /* Create the block split on the array of literals.
Literal histograms have alphabet size 256. */ Literal histograms have alphabet size 256. */
@ -116,28 +114,26 @@ func splitBlock(cmds []command, num_commands uint, data []byte, pos uint, mask u
literals = nil literals = nil
} }
{ {
var insert_and_copy_codes []uint16 = make([]uint16, num_commands) var insert_and_copy_codes []uint16 = make([]uint16, len(cmds))
/* Compute prefix codes for commands. */ /* Compute prefix codes for commands. */
var i uint for i := range cmds {
for i = 0; i < num_commands; i++ {
insert_and_copy_codes[i] = cmds[i].cmd_prefix_ insert_and_copy_codes[i] = cmds[i].cmd_prefix_
} }
/* Create the block split on the array of command prefixes. */ /* Create the block split on the array of command prefixes. */
splitByteVectorCommand(insert_and_copy_codes, num_commands, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params, insert_and_copy_split) splitByteVectorCommand(insert_and_copy_codes, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params, insert_and_copy_split)
/* TODO: reuse for distances? */ /* TODO: reuse for distances? */
insert_and_copy_codes = nil insert_and_copy_codes = nil
} }
{ {
var distance_prefixes []uint16 = make([]uint16, num_commands) var distance_prefixes []uint16 = make([]uint16, len(cmds))
var j uint = 0 var j uint = 0
/* Create a continuous array of distance prefixes. */ /* Create a continuous array of distance prefixes. */
var i uint for i := range cmds {
for i = 0; i < num_commands; i++ {
var cmd *command = &cmds[i] var cmd *command = &cmds[i]
if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 { if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 {
distance_prefixes[j] = cmd.dist_prefix_ & 0x3FF distance_prefixes[j] = cmd.dist_prefix_ & 0x3FF

View File

@ -372,7 +372,8 @@ func clusterBlocksCommand(data []uint16, length uint, num_blocks uint, block_ids
histogram_symbols = nil histogram_symbols = nil
} }
func splitByteVectorCommand(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) { func splitByteVectorCommand(data []uint16, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
length := uint(len(data))
var data_size uint = histogramDataSizeCommand() var data_size uint = histogramDataSizeCommand()
var num_histograms uint = length/literals_per_histogram + 1 var num_histograms uint = length/literals_per_histogram + 1
var histograms []histogramCommand var histograms []histogramCommand

View File

@ -990,7 +990,7 @@ func jumpToByteBoundary(storage_ix *uint, storage []byte) {
storage[*storage_ix>>3] = 0 storage[*storage_ix>>3] = 0
} }
func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_byte byte, prev_byte2 byte, is_last bool, params *encoderParams, literal_context_mode int, commands []command, n_commands uint, mb *metaBlockSplit, storage_ix *uint, storage []byte) { func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_byte byte, prev_byte2 byte, is_last bool, params *encoderParams, literal_context_mode int, commands []command, mb *metaBlockSplit, storage_ix *uint, storage []byte) {
var pos uint = start_pos var pos uint = start_pos
var i uint var i uint
var num_distance_symbols uint32 = params.dist.alphabet_size var num_distance_symbols uint32 = params.dist.alphabet_size
@ -1039,8 +1039,7 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b
buildAndStoreEntropyCodesDistance(&distance_enc, mb.distance_histograms, mb.distance_histograms_size, uint(num_distance_symbols), tree, storage_ix, storage) buildAndStoreEntropyCodesDistance(&distance_enc, mb.distance_histograms, mb.distance_histograms_size, uint(num_distance_symbols), tree, storage_ix, storage)
tree = nil tree = nil
for i = 0; i < n_commands; i++ { for _, cmd := range commands {
var cmd command = commands[i]
var cmd_code uint = uint(cmd.cmd_prefix_) var cmd_code uint = uint(cmd.cmd_prefix_)
storeSymbol(&command_enc, cmd_code, storage_ix, storage) storeSymbol(&command_enc, cmd_code, storage_ix, storage)
storeCommandExtra(&cmd, storage_ix, storage) storeCommandExtra(&cmd, storage_ix, storage)
@ -1090,11 +1089,9 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b
} }
} }
func buildHistograms(input []byte, start_pos uint, mask uint, commands []command, n_commands uint, lit_histo *histogramLiteral, cmd_histo *histogramCommand, dist_histo *histogramDistance) { func buildHistograms(input []byte, start_pos uint, mask uint, commands []command, lit_histo *histogramLiteral, cmd_histo *histogramCommand, dist_histo *histogramDistance) {
var pos uint = start_pos var pos uint = start_pos
var i uint for _, cmd := range commands {
for i = 0; i < n_commands; i++ {
var cmd command = commands[i]
var j uint var j uint
histogramAddCommand(cmd_histo, uint(cmd.cmd_prefix_)) histogramAddCommand(cmd_histo, uint(cmd.cmd_prefix_))
for j = uint(cmd.insert_len_); j != 0; j-- { for j = uint(cmd.insert_len_); j != 0; j-- {
@ -1109,11 +1106,9 @@ func buildHistograms(input []byte, start_pos uint, mask uint, commands []command
} }
} }
func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands []command, n_commands uint, lit_depth []byte, lit_bits []uint16, cmd_depth []byte, cmd_bits []uint16, dist_depth []byte, dist_bits []uint16, storage_ix *uint, storage []byte) { func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands []command, lit_depth []byte, lit_bits []uint16, cmd_depth []byte, cmd_bits []uint16, dist_depth []byte, dist_bits []uint16, storage_ix *uint, storage []byte) {
var pos uint = start_pos var pos uint = start_pos
var i uint for _, cmd := range commands {
for i = 0; i < n_commands; i++ {
var cmd command = commands[i]
var cmd_code uint = uint(cmd.cmd_prefix_) var cmd_code uint = uint(cmd.cmd_prefix_)
var j uint var j uint
writeBits(uint(cmd_depth[cmd_code]), uint64(cmd_bits[cmd_code]), storage_ix, storage) writeBits(uint(cmd_depth[cmd_code]), uint64(cmd_bits[cmd_code]), storage_ix, storage)
@ -1135,7 +1130,7 @@ func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands
} }
} }
func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, n_commands uint, storage_ix *uint, storage []byte) { func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, storage_ix *uint, storage []byte) {
var lit_histo histogramLiteral var lit_histo histogramLiteral
var cmd_histo histogramCommand var cmd_histo histogramCommand
var dist_histo histogramDistance var dist_histo histogramDistance
@ -1154,7 +1149,7 @@ func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint,
histogramClearCommand(&cmd_histo) histogramClearCommand(&cmd_histo)
histogramClearDistance(&dist_histo) histogramClearDistance(&dist_histo)
buildHistograms(input, start_pos, mask, commands, n_commands, &lit_histo, &cmd_histo, &dist_histo) buildHistograms(input, start_pos, mask, commands, &lit_histo, &cmd_histo, &dist_histo)
writeBits(13, 0, storage_ix, storage) writeBits(13, 0, storage_ix, storage)
@ -1163,13 +1158,13 @@ func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint,
buildAndStoreHuffmanTree(cmd_histo.data_[:], numCommandSymbols, numCommandSymbols, tree, cmd_depth[:], cmd_bits[:], storage_ix, storage) buildAndStoreHuffmanTree(cmd_histo.data_[:], numCommandSymbols, numCommandSymbols, tree, cmd_depth[:], cmd_bits[:], storage_ix, storage)
buildAndStoreHuffmanTree(dist_histo.data_[:], maxSimpleDistanceAlphabetSize, uint(num_distance_symbols), tree, dist_depth[:], dist_bits[:], storage_ix, storage) buildAndStoreHuffmanTree(dist_histo.data_[:], maxSimpleDistanceAlphabetSize, uint(num_distance_symbols), tree, dist_depth[:], dist_bits[:], storage_ix, storage)
tree = nil tree = nil
storeDataWithHuffmanCodes(input, start_pos, mask, commands, n_commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage) storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage)
if is_last { if is_last {
jumpToByteBoundary(storage_ix, storage) jumpToByteBoundary(storage_ix, storage)
} }
} }
func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, n_commands uint, storage_ix *uint, storage []byte) { func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, storage_ix *uint, storage []byte) {
var num_distance_symbols uint32 = params.dist.alphabet_size var num_distance_symbols uint32 = params.dist.alphabet_size
var distance_alphabet_bits uint32 = log2FloorNonZero(uint(num_distance_symbols-1)) + 1 var distance_alphabet_bits uint32 = log2FloorNonZero(uint(num_distance_symbols-1)) + 1
@ -1177,15 +1172,13 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is
writeBits(13, 0, storage_ix, storage) writeBits(13, 0, storage_ix, storage)
if n_commands <= 128 { if len(commands) <= 128 {
var histogram = [numLiteralSymbols]uint32{0} var histogram = [numLiteralSymbols]uint32{0}
var pos uint = start_pos var pos uint = start_pos
var num_literals uint = 0 var num_literals uint = 0
var i uint
var lit_depth [numLiteralSymbols]byte var lit_depth [numLiteralSymbols]byte
var lit_bits [numLiteralSymbols]uint16 var lit_bits [numLiteralSymbols]uint16
for i = 0; i < n_commands; i++ { for _, cmd := range commands {
var cmd command = commands[i]
var j uint var j uint
for j = uint(cmd.insert_len_); j != 0; j-- { for j = uint(cmd.insert_len_); j != 0; j-- {
histogram[input[pos&mask]]++ histogram[input[pos&mask]]++
@ -1201,7 +1194,7 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is
storeStaticCommandHuffmanTree(storage_ix, storage) storeStaticCommandHuffmanTree(storage_ix, storage)
storeStaticDistanceHuffmanTree(storage_ix, storage) storeStaticDistanceHuffmanTree(storage_ix, storage)
storeDataWithHuffmanCodes(input, start_pos, mask, commands, n_commands, lit_depth[:], lit_bits[:], kStaticCommandCodeDepth[:], kStaticCommandCodeBits[:], kStaticDistanceCodeDepth[:], kStaticDistanceCodeBits[:], storage_ix, storage) storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], kStaticCommandCodeDepth[:], kStaticCommandCodeBits[:], kStaticDistanceCodeDepth[:], kStaticDistanceCodeBits[:], storage_ix, storage)
} else { } else {
var lit_histo histogramLiteral var lit_histo histogramLiteral
var cmd_histo histogramCommand var cmd_histo histogramCommand
@ -1215,7 +1208,7 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is
histogramClearLiteral(&lit_histo) histogramClearLiteral(&lit_histo)
histogramClearCommand(&cmd_histo) histogramClearCommand(&cmd_histo)
histogramClearDistance(&dist_histo) histogramClearDistance(&dist_histo)
buildHistograms(input, start_pos, mask, commands, n_commands, &lit_histo, &cmd_histo, &dist_histo) buildHistograms(input, start_pos, mask, commands, &lit_histo, &cmd_histo, &dist_histo)
buildAndStoreHuffmanTreeFast(lit_histo.data_[:], lit_histo.total_count_, /* max_bits = */ buildAndStoreHuffmanTreeFast(lit_histo.data_[:], lit_histo.total_count_, /* max_bits = */
8, lit_depth[:], lit_bits[:], storage_ix, storage) 8, lit_depth[:], lit_bits[:], storage_ix, storage)
@ -1225,7 +1218,7 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is
buildAndStoreHuffmanTreeFast(dist_histo.data_[:], dist_histo.total_count_, /* max_bits = */ buildAndStoreHuffmanTreeFast(dist_histo.data_[:], dist_histo.total_count_, /* max_bits = */
uint(distance_alphabet_bits), dist_depth[:], dist_bits[:], storage_ix, storage) uint(distance_alphabet_bits), dist_depth[:], dist_bits[:], storage_ix, storage)
storeDataWithHuffmanCodes(input, start_pos, mask, commands, n_commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage) storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage)
} }
if is_last { if is_last {

View File

@ -194,26 +194,28 @@ type command struct {
} }
/* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */ /* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */
func initCommand(self *command, dist *distanceParams, insertlen uint, copylen uint, copylen_code_delta int, distance_code uint) { func makeCommand(dist *distanceParams, insertlen uint, copylen uint, copylen_code_delta int, distance_code uint) (cmd command) {
/* Don't rely on signed int representation, use honest casts. */ /* Don't rely on signed int representation, use honest casts. */
var delta uint32 = uint32(byte(int8(copylen_code_delta))) var delta uint32 = uint32(byte(int8(copylen_code_delta)))
self.insert_len_ = uint32(insertlen) cmd.insert_len_ = uint32(insertlen)
self.copy_len_ = uint32(uint32(copylen) | delta<<25) cmd.copy_len_ = uint32(uint32(copylen) | delta<<25)
/* The distance prefix and extra bits are stored in this Command as if /* The distance prefix and extra bits are stored in this Command as if
npostfix and ndirect were 0, they are only recomputed later after the npostfix and ndirect were 0, they are only recomputed later after the
clustering if needed. */ clustering if needed. */
prefixEncodeCopyDistance(distance_code, uint(dist.num_direct_distance_codes), uint(dist.distance_postfix_bits), &self.dist_prefix_, &self.dist_extra_) prefixEncodeCopyDistance(distance_code, uint(dist.num_direct_distance_codes), uint(dist.distance_postfix_bits), &cmd.dist_prefix_, &cmd.dist_extra_)
getLengthCode(insertlen, uint(int(copylen)+copylen_code_delta), (cmd.dist_prefix_&0x3FF == 0), &cmd.cmd_prefix_)
getLengthCode(insertlen, uint(int(copylen)+copylen_code_delta), (self.dist_prefix_&0x3FF == 0), &self.cmd_prefix_) return cmd
} }
func initInsertCommand(self *command, insertlen uint) { func makeInsertCommand(insertlen uint) (cmd command) {
self.insert_len_ = uint32(insertlen) cmd.insert_len_ = uint32(insertlen)
self.copy_len_ = 4 << 25 cmd.copy_len_ = 4 << 25
self.dist_extra_ = 0 cmd.dist_extra_ = 0
self.dist_prefix_ = numDistanceShortCodes cmd.dist_prefix_ = numDistanceShortCodes
getLengthCode(insertlen, 4, false, &self.cmd_prefix_) getLengthCode(insertlen, 4, false, &cmd.cmd_prefix_)
return cmd
} }
func commandRestoreDistanceCode(self *command, dist *distanceParams) uint32 { func commandRestoreDistanceCode(self *command, dist *distanceParams) uint32 {

View File

@ -80,9 +80,7 @@ type Writer struct {
hasher_ hasherHandle hasher_ hasherHandle
input_pos_ uint64 input_pos_ uint64
ringbuffer_ ringBuffer ringbuffer_ ringBuffer
cmd_alloc_size_ uint commands []command
commands_ []command
num_commands_ uint
num_literals_ uint num_literals_ uint
last_insert_len_ uint last_insert_len_ uint
last_flush_pos_ uint64 last_flush_pos_ uint64
@ -434,7 +432,7 @@ func chooseContextMode(params *encoderParams, data []byte, pos uint, mask uint,
return contextUTF8 return contextUTF8
} }
func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes uint, is_last bool, literal_context_mode int, params *encoderParams, prev_byte byte, prev_byte2 byte, num_literals uint, num_commands uint, commands []command, saved_dist_cache []int, dist_cache []int, storage_ix *uint, storage []byte) { func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes uint, is_last bool, literal_context_mode int, params *encoderParams, prev_byte byte, prev_byte2 byte, num_literals uint, commands []command, saved_dist_cache []int, dist_cache []int, storage_ix *uint, storage []byte) {
var wrapped_last_flush_pos uint32 = wrapPosition(last_flush_pos) var wrapped_last_flush_pos uint32 = wrapPosition(last_flush_pos)
var last_bytes uint16 var last_bytes uint16
var last_bytes_bits byte var last_bytes_bits byte
@ -449,7 +447,7 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes
return return
} }
if !shouldCompress_encode(data, mask, last_flush_pos, bytes, num_literals, num_commands) { if !shouldCompress_encode(data, mask, last_flush_pos, bytes, num_literals, uint(len(commands))) {
/* Restore the distance cache, as its last update by /* Restore the distance cache, as its last update by
CreateBackwardReferences is now unused. */ CreateBackwardReferences is now unused. */
copy(dist_cache, saved_dist_cache[:4]) copy(dist_cache, saved_dist_cache[:4])
@ -462,9 +460,9 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes
last_bytes = uint16(storage[1])<<8 | uint16(storage[0]) last_bytes = uint16(storage[1])<<8 | uint16(storage[0])
last_bytes_bits = byte(*storage_ix) last_bytes_bits = byte(*storage_ix)
if params.quality <= maxQualityForStaticEntropyCodes { if params.quality <= maxQualityForStaticEntropyCodes {
storeMetaBlockFast(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, num_commands, storage_ix, storage) storeMetaBlockFast(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, storage_ix, storage)
} else if params.quality < minQualityForBlockSplit { } else if params.quality < minQualityForBlockSplit {
storeMetaBlockTrivial(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, num_commands, storage_ix, storage) storeMetaBlockTrivial(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, storage_ix, storage)
} else { } else {
var mb metaBlockSplit var mb metaBlockSplit
initMetaBlockSplit(&mb) initMetaBlockSplit(&mb)
@ -475,9 +473,9 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes
decideOverLiteralContextModeling(data, uint(wrapped_last_flush_pos), bytes, mask, params.quality, params.size_hint, &num_literal_contexts, &literal_context_map) decideOverLiteralContextModeling(data, uint(wrapped_last_flush_pos), bytes, mask, params.quality, params.size_hint, &num_literal_contexts, &literal_context_map)
} }
buildMetaBlockGreedy(data, uint(wrapped_last_flush_pos), mask, prev_byte, prev_byte2, literal_context_lut, num_literal_contexts, literal_context_map, commands, num_commands, &mb) buildMetaBlockGreedy(data, uint(wrapped_last_flush_pos), mask, prev_byte, prev_byte2, literal_context_lut, num_literal_contexts, literal_context_map, commands, &mb)
} else { } else {
buildMetaBlock(data, uint(wrapped_last_flush_pos), mask, &block_params, prev_byte, prev_byte2, commands, num_commands, literal_context_mode, &mb) buildMetaBlock(data, uint(wrapped_last_flush_pos), mask, &block_params, prev_byte, prev_byte2, commands, literal_context_mode, &mb)
} }
if params.quality >= minQualityForOptimizeHistograms { if params.quality >= minQualityForOptimizeHistograms {
@ -492,7 +490,7 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes
optimizeHistograms(num_effective_dist_codes, &mb) optimizeHistograms(num_effective_dist_codes, &mb)
} }
storeMetaBlock(data, uint(wrapped_last_flush_pos), bytes, mask, prev_byte, prev_byte2, is_last, &block_params, literal_context_mode, commands, num_commands, &mb, storage_ix, storage) storeMetaBlock(data, uint(wrapped_last_flush_pos), bytes, mask, prev_byte, prev_byte2, is_last, &block_params, literal_context_mode, commands, &mb, storage_ix, storage)
destroyMetaBlockSplit(&mb) destroyMetaBlockSplit(&mb)
} }
@ -613,7 +611,7 @@ func encoderInitParams(params *encoderParams) {
func encoderInitState(s *Writer) { func encoderInitState(s *Writer) {
encoderInitParams(&s.params) encoderInitParams(&s.params)
s.input_pos_ = 0 s.input_pos_ = 0
s.num_commands_ = 0 s.commands = s.commands[:0]
s.num_literals_ = 0 s.num_literals_ = 0
s.last_insert_len_ = 0 s.last_insert_len_ = 0
s.last_flush_pos_ = 0 s.last_flush_pos_ = 0
@ -712,7 +710,7 @@ func updateLastProcessedPos(s *Writer) bool {
} }
func extendLastCommand(s *Writer, bytes *uint32, wrapped_last_processed_pos *uint32) { func extendLastCommand(s *Writer, bytes *uint32, wrapped_last_processed_pos *uint32) {
var last_command *command = &s.commands_[s.num_commands_-1] var last_command *command = &s.commands[len(s.commands)-1]
var data []byte = s.ringbuffer_.buffer_ var data []byte = s.ringbuffer_.buffer_
var mask uint32 = s.ringbuffer_.mask_ var mask uint32 = s.ringbuffer_.mask_
var max_backward_distance uint64 = ((uint64(1)) << s.params.lgwin) - windowGap var max_backward_distance uint64 = ((uint64(1)) << s.params.lgwin) - windowGap
@ -809,22 +807,18 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
} }
{ {
/* Theoretical max number of commands is 1 per 2 bytes. */ /* Theoretical max number of commands is 1 per 2 bytes. */
var newsize uint = uint(uint32(s.num_commands_) + bytes/2 + 1) newsize := len(s.commands) + int(bytes)/2 + 1
if newsize > s.cmd_alloc_size_ { if newsize > cap(s.commands) {
var new_commands []command
/* Reserve a bit more memory to allow merging with a next block /* Reserve a bit more memory to allow merging with a next block
without reallocation: that would impact speed. */ without reallocation: that would impact speed. */
newsize += uint((bytes / 4) + 16) newsize += int(bytes/4) + 16
s.cmd_alloc_size_ = newsize new_commands := make([]command, len(s.commands), newsize)
new_commands = make([]command, newsize) if s.commands != nil {
if s.commands_ != nil { copy(new_commands, s.commands)
copy(new_commands, s.commands_[:s.num_commands_])
s.commands_ = nil
} }
s.commands_ = new_commands s.commands = new_commands
} }
} }
@ -832,32 +826,32 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
literal_context_mode = chooseContextMode(&s.params, data, uint(wrapPosition(s.last_flush_pos_)), uint(mask), uint(s.input_pos_-s.last_flush_pos_)) literal_context_mode = chooseContextMode(&s.params, data, uint(wrapPosition(s.last_flush_pos_)), uint(mask), uint(s.input_pos_-s.last_flush_pos_))
if s.num_commands_ != 0 && s.last_insert_len_ == 0 { if len(s.commands) != 0 && s.last_insert_len_ == 0 {
extendLastCommand(s, &bytes, &wrapped_last_processed_pos) extendLastCommand(s, &bytes, &wrapped_last_processed_pos)
} }
if s.params.quality == zopflificationQuality { if s.params.quality == zopflificationQuality {
assert(s.params.hasher.type_ == 10) assert(s.params.hasher.type_ == 10)
createZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_.(*h10), s.dist_cache_[:], &s.last_insert_len_, s.commands_[s.num_commands_:], &s.num_commands_, &s.num_literals_) createZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_.(*h10), s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_)
} else if s.params.quality == hqZopflificationQuality { } else if s.params.quality == hqZopflificationQuality {
assert(s.params.hasher.type_ == 10) assert(s.params.hasher.type_ == 10)
createHqZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, s.commands_[s.num_commands_:], &s.num_commands_, &s.num_literals_) createHqZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_)
} else { } else {
createBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, s.commands_[s.num_commands_:], &s.num_commands_, &s.num_literals_) createBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_)
} }
{ {
var max_length uint = maxMetablockSize(&s.params) var max_length uint = maxMetablockSize(&s.params)
var max_literals uint = max_length / 8 var max_literals uint = max_length / 8
var max_commands uint = max_length / 8 max_commands := int(max_length / 8)
var processed_bytes uint = uint(s.input_pos_ - s.last_flush_pos_) var processed_bytes uint = uint(s.input_pos_ - s.last_flush_pos_)
var next_input_fits_metablock bool = (processed_bytes+inputBlockSize(s) <= max_length) var next_input_fits_metablock bool = (processed_bytes+inputBlockSize(s) <= max_length)
var should_flush bool = (s.params.quality < minQualityForBlockSplit && s.num_literals_+s.num_commands_ >= maxNumDelayedSymbols) var should_flush bool = (s.params.quality < minQualityForBlockSplit && s.num_literals_+uint(len(s.commands)) >= maxNumDelayedSymbols)
/* If maximal possible additional block doesn't fit metablock, flush now. */ /* If maximal possible additional block doesn't fit metablock, flush now. */
/* TODO: Postpone decision until next block arrives? */ /* TODO: Postpone decision until next block arrives? */
/* If block splitting is not used, then flush as soon as there is some /* If block splitting is not used, then flush as soon as there is some
amount of commands / literals produced. */ amount of commands / literals produced. */
if !is_last && !force_flush && !should_flush && next_input_fits_metablock && s.num_literals_ < max_literals && s.num_commands_ < max_commands { if !is_last && !force_flush && !should_flush && next_input_fits_metablock && s.num_literals_ < max_literals && len(s.commands) < max_commands {
/* Merge with next input block. Everything will happen later. */ /* Merge with next input block. Everything will happen later. */
if updateLastProcessedPos(s) { if updateLastProcessedPos(s) {
hasherReset(s.hasher_) hasherReset(s.hasher_)
@ -869,8 +863,7 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
/* Create the last insert-only command. */ /* Create the last insert-only command. */
if s.last_insert_len_ > 0 { if s.last_insert_len_ > 0 {
initInsertCommand(&s.commands_[s.num_commands_], s.last_insert_len_) s.commands = append(s.commands, makeInsertCommand(s.last_insert_len_))
s.num_commands_++
s.num_literals_ += s.last_insert_len_ s.num_literals_ += s.last_insert_len_
s.last_insert_len_ = 0 s.last_insert_len_ = 0
} }
@ -890,7 +883,7 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
var storage_ix uint = uint(s.last_bytes_bits_) var storage_ix uint = uint(s.last_bytes_bits_)
storage[0] = byte(s.last_bytes_) storage[0] = byte(s.last_bytes_)
storage[1] = byte(s.last_bytes_ >> 8) storage[1] = byte(s.last_bytes_ >> 8)
writeMetaBlockInternal(data, uint(mask), s.last_flush_pos_, uint(metablock_size), is_last, literal_context_mode, &s.params, s.prev_byte_, s.prev_byte2_, s.num_literals_, s.num_commands_, s.commands_, s.saved_dist_cache_[:], s.dist_cache_[:], &storage_ix, storage) writeMetaBlockInternal(data, uint(mask), s.last_flush_pos_, uint(metablock_size), is_last, literal_context_mode, &s.params, s.prev_byte_, s.prev_byte2_, s.num_literals_, s.commands, s.saved_dist_cache_[:], s.dist_cache_[:], &storage_ix, storage)
s.last_bytes_ = uint16(storage[storage_ix>>3]) s.last_bytes_ = uint16(storage[storage_ix>>3])
s.last_bytes_bits_ = byte(storage_ix & 7) s.last_bytes_bits_ = byte(storage_ix & 7)
s.last_flush_pos_ = s.input_pos_ s.last_flush_pos_ = s.input_pos_
@ -906,7 +899,7 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
s.prev_byte2_ = data[uint32(s.last_flush_pos_-2)&mask] s.prev_byte2_ = data[uint32(s.last_flush_pos_-2)&mask]
} }
s.num_commands_ = 0 s.commands = s.commands[:0]
s.num_literals_ = 0 s.num_literals_ = 0
/* Save the state of the distance cache in case we need to restore it for /* Save the state of the distance cache in case we need to restore it for

View File

@ -180,17 +180,16 @@ func blockSplitIteratorNext(self *blockSplitIterator) {
self.length_-- self.length_--
} }
func buildHistogramsWithContext(cmds []command, num_commands uint, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit, ringbuffer []byte, start_pos uint, mask uint, prev_byte byte, prev_byte2 byte, context_modes []int, literal_histograms []histogramLiteral, insert_and_copy_histograms []histogramCommand, copy_dist_histograms []histogramDistance) { func buildHistogramsWithContext(cmds []command, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit, ringbuffer []byte, start_pos uint, mask uint, prev_byte byte, prev_byte2 byte, context_modes []int, literal_histograms []histogramLiteral, insert_and_copy_histograms []histogramCommand, copy_dist_histograms []histogramDistance) {
var pos uint = start_pos var pos uint = start_pos
var literal_it blockSplitIterator var literal_it blockSplitIterator
var insert_and_copy_it blockSplitIterator var insert_and_copy_it blockSplitIterator
var dist_it blockSplitIterator var dist_it blockSplitIterator
var i uint
initBlockSplitIterator(&literal_it, literal_split) initBlockSplitIterator(&literal_it, literal_split)
initBlockSplitIterator(&insert_and_copy_it, insert_and_copy_split) initBlockSplitIterator(&insert_and_copy_it, insert_and_copy_split)
initBlockSplitIterator(&dist_it, dist_split) initBlockSplitIterator(&dist_it, dist_split)
for i = 0; i < num_commands; i++ { for i := range cmds {
var cmd *command = &cmds[i] var cmd *command = &cmds[i]
var j uint var j uint
blockSplitIteratorNext(&insert_and_copy_it) blockSplitIteratorNext(&insert_and_copy_it)

View File

@ -84,14 +84,12 @@ func initDistanceParams(params *encoderParams, npostfix uint32, ndirect uint32)
dist_params.max_distance = uint(max_distance) dist_params.max_distance = uint(max_distance)
} }
func recomputeDistancePrefixes(cmds []command, num_commands uint, orig_params *distanceParams, new_params *distanceParams) { func recomputeDistancePrefixes(cmds []command, orig_params *distanceParams, new_params *distanceParams) {
var i uint
if orig_params.distance_postfix_bits == new_params.distance_postfix_bits && orig_params.num_direct_distance_codes == new_params.num_direct_distance_codes { if orig_params.distance_postfix_bits == new_params.distance_postfix_bits && orig_params.num_direct_distance_codes == new_params.num_direct_distance_codes {
return return
} }
for i = 0; i < num_commands; i++ { for i := range cmds {
var cmd *command = &cmds[i] var cmd *command = &cmds[i]
if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 { if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 {
prefixEncodeCopyDistance(uint(commandRestoreDistanceCode(cmd, orig_params)), uint(new_params.num_direct_distance_codes), uint(new_params.distance_postfix_bits), &cmd.dist_prefix_, &cmd.dist_extra_) prefixEncodeCopyDistance(uint(commandRestoreDistanceCode(cmd, orig_params)), uint(new_params.num_direct_distance_codes), uint(new_params.distance_postfix_bits), &cmd.dist_prefix_, &cmd.dist_extra_)
@ -99,8 +97,7 @@ func recomputeDistancePrefixes(cmds []command, num_commands uint, orig_params *d
} }
} }
func computeDistanceCost(cmds []command, num_commands uint, orig_params *distanceParams, new_params *distanceParams, cost *float64) bool { func computeDistanceCost(cmds []command, orig_params *distanceParams, new_params *distanceParams, cost *float64) bool {
var i uint
var equal_params bool = false var equal_params bool = false
var dist_prefix uint16 var dist_prefix uint16
var dist_extra uint32 var dist_extra uint32
@ -112,8 +109,8 @@ func computeDistanceCost(cmds []command, num_commands uint, orig_params *distanc
equal_params = true equal_params = true
} }
for i = 0; i < num_commands; i++ { for i := range cmds {
var cmd *command = &cmds[i] cmd := &cmds[i]
if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 { if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 {
if equal_params { if equal_params {
dist_prefix = cmd.dist_prefix_ dist_prefix = cmd.dist_prefix_
@ -137,7 +134,7 @@ func computeDistanceCost(cmds []command, num_commands uint, orig_params *distanc
var buildMetaBlock_kMaxNumberOfHistograms uint = 256 var buildMetaBlock_kMaxNumberOfHistograms uint = 256
func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParams, prev_byte byte, prev_byte2 byte, cmds []command, num_commands uint, literal_context_mode int, mb *metaBlockSplit) { func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParams, prev_byte byte, prev_byte2 byte, cmds []command, literal_context_mode int, mb *metaBlockSplit) {
var distance_histograms []histogramDistance var distance_histograms []histogramDistance
var literal_histograms []histogramLiteral var literal_histograms []histogramLiteral
var literal_context_modes []int = nil var literal_context_modes []int = nil
@ -164,7 +161,7 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam
check_orig = false check_orig = false
} }
skip = !computeDistanceCost(cmds, num_commands, &orig_params.dist, &new_params.dist, &dist_cost) skip = !computeDistanceCost(cmds, &orig_params.dist, &new_params.dist, &dist_cost)
if skip || (dist_cost > best_dist_cost) { if skip || (dist_cost > best_dist_cost) {
break break
} }
@ -181,7 +178,7 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam
if check_orig { if check_orig {
var dist_cost float64 var dist_cost float64
computeDistanceCost(cmds, num_commands, &orig_params.dist, &orig_params.dist, &dist_cost) computeDistanceCost(cmds, &orig_params.dist, &orig_params.dist, &dist_cost)
if dist_cost < best_dist_cost { if dist_cost < best_dist_cost {
/* NB: currently unused; uncomment when more param tuning is added. */ /* NB: currently unused; uncomment when more param tuning is added. */
/* best_dist_cost = dist_cost; */ /* best_dist_cost = dist_cost; */
@ -189,9 +186,9 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam
} }
} }
recomputeDistancePrefixes(cmds, num_commands, &orig_params.dist, &params.dist) recomputeDistancePrefixes(cmds, &orig_params.dist, &params.dist)
splitBlock(cmds, num_commands, ringbuffer, pos, mask, params, &mb.literal_split, &mb.command_split, &mb.distance_split) splitBlock(cmds, ringbuffer, pos, mask, params, &mb.literal_split, &mb.command_split, &mb.distance_split)
if !params.disable_literal_context_modeling { if !params.disable_literal_context_modeling {
literal_context_multiplier = 1 << literalContextBits literal_context_multiplier = 1 << literalContextBits
@ -214,7 +211,7 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam
mb.command_histograms = make([]histogramCommand, (mb.command_histograms_size)) mb.command_histograms = make([]histogramCommand, (mb.command_histograms_size))
clearHistogramsCommand(mb.command_histograms, mb.command_histograms_size) clearHistogramsCommand(mb.command_histograms, mb.command_histograms_size)
buildHistogramsWithContext(cmds, num_commands, &mb.literal_split, &mb.command_split, &mb.distance_split, ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_modes, literal_histograms, mb.command_histograms, distance_histograms) buildHistogramsWithContext(cmds, &mb.literal_split, &mb.command_split, &mb.distance_split, ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_modes, literal_histograms, mb.command_histograms, distance_histograms)
literal_context_modes = nil literal_context_modes = nil
assert(mb.literal_context_map == nil) assert(mb.literal_context_map == nil)
@ -466,7 +463,7 @@ func mapStaticContexts(num_contexts uint, static_context_map []uint32, mb *metaB
} }
} }
func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, n_commands uint, mb *metaBlockSplit) { func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, mb *metaBlockSplit) {
var lit_blocks struct { var lit_blocks struct {
plain blockSplitterLiteral plain blockSplitterLiteral
ctx contextBlockSplitter ctx contextBlockSplitter
@ -474,8 +471,7 @@ func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_b
var cmd_blocks blockSplitterCommand var cmd_blocks blockSplitterCommand
var dist_blocks blockSplitterDistance var dist_blocks blockSplitterDistance
var num_literals uint = 0 var num_literals uint = 0
var i uint for i := range commands {
for i = 0; i < n_commands; i++ {
num_literals += uint(commands[i].insert_len_) num_literals += uint(commands[i].insert_len_)
} }
@ -485,11 +481,10 @@ func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_b
initContextBlockSplitter(&lit_blocks.ctx, 256, num_contexts, 512, 400.0, num_literals, &mb.literal_split, &mb.literal_histograms, &mb.literal_histograms_size) initContextBlockSplitter(&lit_blocks.ctx, 256, num_contexts, 512, 400.0, num_literals, &mb.literal_split, &mb.literal_histograms, &mb.literal_histograms_size)
} }
initBlockSplitterCommand(&cmd_blocks, numCommandSymbols, 1024, 500.0, n_commands, &mb.command_split, &mb.command_histograms, &mb.command_histograms_size) initBlockSplitterCommand(&cmd_blocks, numCommandSymbols, 1024, 500.0, uint(len(commands)), &mb.command_split, &mb.command_histograms, &mb.command_histograms_size)
initBlockSplitterDistance(&dist_blocks, 64, 512, 100.0, n_commands, &mb.distance_split, &mb.distance_histograms, &mb.distance_histograms_size) initBlockSplitterDistance(&dist_blocks, 64, 512, 100.0, uint(len(commands)), &mb.distance_split, &mb.distance_histograms, &mb.distance_histograms_size)
for i = 0; i < n_commands; i++ { for _, cmd := range commands {
var cmd command = commands[i]
var j uint var j uint
blockSplitterAddSymbolCommand(&cmd_blocks, uint(cmd.cmd_prefix_)) blockSplitterAddSymbolCommand(&cmd_blocks, uint(cmd.cmd_prefix_))
for j = uint(cmd.insert_len_); j != 0; j-- { for j = uint(cmd.insert_len_); j != 0; j-- {
@ -530,11 +525,11 @@ func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_b
} }
} }
func buildMetaBlockGreedy(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, n_commands uint, mb *metaBlockSplit) { func buildMetaBlockGreedy(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, mb *metaBlockSplit) {
if num_contexts == 1 { if num_contexts == 1 {
buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, 1, nil, commands, n_commands, mb) buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, 1, nil, commands, mb)
} else { } else {
buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, num_contexts, static_context_map, commands, n_commands, mb) buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, num_contexts, static_context_map, commands, mb)
} }
} }