From e2c5f2109f20e3972f65281c8e30eb636b410581 Mon Sep 17 00:00:00 2001 From: Andy Balholm Date: Fri, 8 May 2020 16:48:16 -0700 Subject: [PATCH] Use len and cap instead of num_commands_ and cmd_alloc_size_. --- backward_references.go | 11 ++----- backward_references_hq.go | 33 ++++++++++----------- block_splitter.go | 28 ++++++++---------- block_splitter_command.go | 3 +- brotli_bit_stream.go | 37 ++++++++++-------------- command.go | 24 ++++++++------- encode.go | 61 +++++++++++++++++---------------------- histogram.go | 5 ++-- metablock.go | 43 ++++++++++++--------------- 9 files changed, 108 insertions(+), 137 deletions(-) diff --git a/backward_references.go b/backward_references.go index 1642e47..f7b164f 100644 --- a/backward_references.go +++ b/backward_references.go @@ -31,13 +31,8 @@ func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uin return distance + numDistanceShortCodes - 1 } -/* "commands" points to the next output command to write to, "*num_commands" is - initially the total amount of commands output by previous - CreateBackwardReferences calls, and must be incremented by the amount written - by this call. */ -func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) { +func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) { var max_backward_limit uint = maxBackwardLimit(params.lgwin) - var orig_commands []command = commands var insert_length uint = *last_insert_len var pos_end uint = position + num_bytes var store_end uint @@ -114,8 +109,7 @@ func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, hasher.PrepareDistanceCache(dist_cache) } - initCommand(&commands[0], ¶ms.dist, insert_length, sr.len, sr.len_code_delta, distance_code) - commands = commands[1:] + *commands = append(*commands, makeCommand(¶ms.dist, insert_length, sr.len, sr.len_code_delta, distance_code)) } *num_literals += insert_length @@ -173,5 +167,4 @@ func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, insert_length += pos_end - position *last_insert_len = insert_length - *num_commands += uint(-cap(commands) + cap(orig_commands)) } diff --git a/backward_references_hq.go b/backward_references_hq.go index bd2de48..21629c1 100644 --- a/backward_references_hq.go +++ b/backward_references_hq.go @@ -123,14 +123,13 @@ func setCost(histogram []uint32, histogram_size uint, literal_histogram bool, co } } -func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint, commands []command, num_commands uint, last_insert_len uint) { +func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint, commands []command, last_insert_len uint) { var histogram_literal [numLiteralSymbols]uint32 var histogram_cmd [numCommandSymbols]uint32 var histogram_dist [maxEffectiveDistanceAlphabetSize]uint32 var cost_literal [numLiteralSymbols]float32 var pos uint = position - last_insert_len var min_cost_cmd float32 = kInfinity - var i uint var cost_cmd []float32 = self.cost_cmd_[:] var literal_costs []float32 @@ -138,7 +137,7 @@ func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbu histogram_cmd = [numCommandSymbols]uint32{} histogram_dist = [maxEffectiveDistanceAlphabetSize]uint32{} - for i = 0; i < num_commands; i++ { + for i := range commands { var inslength uint = uint(commands[i].insert_len_) var copylength uint = uint(commandCopyLen(&commands[i])) var distcode uint = uint(commands[i].dist_prefix_) & 0x3FF @@ -161,7 +160,7 @@ func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbu setCost(histogram_cmd[:], numCommandSymbols, false, cost_cmd) setCost(histogram_dist[:], uint(self.distance_histogram_size), false, self.cost_dist_) - for i = 0; i < numCommandSymbols; i++ { + for i := 0; i < numCommandSymbols; i++ { min_cost_cmd = brotli_min_float(min_cost_cmd, cost_cmd[i]) } @@ -169,10 +168,10 @@ func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbu { literal_costs = self.literal_costs_ var literal_carry float32 = 0.0 - var num_bytes uint = self.num_bytes_ + num_bytes := int(self.num_bytes_) literal_costs[0] = 0.0 - for i = 0; i < num_bytes; i++ { - literal_carry += cost_literal[ringbuffer[(position+i)&ringbuffer_mask]] + for i := 0; i < num_bytes; i++ { + literal_carry += cost_literal[ringbuffer[(position+uint(i))&ringbuffer_mask]] literal_costs[i+1] = literal_costs[i] + literal_carry literal_carry -= literal_costs[i+1] - literal_costs[i] } @@ -532,7 +531,7 @@ func computeShortestPathFromNodes(num_bytes uint, nodes []zopfliNode) uint { } /* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */ -func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands []command, num_literals *uint) { +func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands *[]command, num_literals *uint) { var max_backward_limit uint = maxBackwardLimit(params.lgwin) var pos uint = 0 var offset uint32 = nodes[0].u.next @@ -554,7 +553,7 @@ func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, var max_distance uint = brotli_min_size_t(block_start+pos, max_backward_limit) var is_dictionary bool = (distance > max_distance+gap) var dist_code uint = uint(zopfliNodeDistanceCode(next)) - initCommand(&commands[i], ¶ms.dist, insert_length, copy_length, int(len_code)-int(copy_length), dist_code) + *commands = append(*commands, makeCommand(¶ms.dist, insert_length, copy_length, int(len_code)-int(copy_length), dist_code)) if !is_dictionary && dist_code > 0 { dist_cache[3] = dist_cache[2] @@ -681,16 +680,16 @@ func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte, return computeShortestPathFromNodes(num_bytes, nodes) } -func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) { +func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) { var nodes []zopfliNode nodes = make([]zopfliNode, (num_bytes + 1)) initZopfliNodes(nodes, num_bytes+1) - *num_commands += zopfliComputeShortestPath(num_bytes, position, ringbuffer, ringbuffer_mask, params, dist_cache, hasher, nodes) + zopfliComputeShortestPath(num_bytes, position, ringbuffer, ringbuffer_mask, params, dist_cache, hasher, nodes) zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals) nodes = nil } -func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) { +func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) { var max_backward_limit uint = maxBackwardLimit(params.lgwin) var num_matches []uint32 = make([]uint32, num_bytes) var matches_size uint = 4 * num_bytes @@ -705,7 +704,7 @@ func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer var orig_num_literals uint var orig_last_insert_len uint var orig_dist_cache [4]int - var orig_num_commands uint + var orig_num_commands int var model zopfliCostModel var nodes []zopfliNode var matches []backwardMatch = make([]backwardMatch, matches_size) @@ -771,7 +770,7 @@ func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer orig_num_literals = *num_literals orig_last_insert_len = *last_insert_len copy(orig_dist_cache[:], dist_cache[:4]) - orig_num_commands = *num_commands + orig_num_commands = len(*commands) nodes = make([]zopfliNode, (num_bytes + 1)) initZopfliCostModel(&model, ¶ms.dist, num_bytes) for i = 0; i < 2; i++ { @@ -779,14 +778,14 @@ func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer if i == 0 { zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask) } else { - zopfliCostModelSetFromCommands(&model, position, ringbuffer, ringbuffer_mask, commands, *num_commands-orig_num_commands, orig_last_insert_len) + zopfliCostModelSetFromCommands(&model, position, ringbuffer, ringbuffer_mask, (*commands)[orig_num_commands:], orig_last_insert_len) } - *num_commands = orig_num_commands + *commands = (*commands)[:orig_num_commands] *num_literals = orig_num_literals *last_insert_len = orig_last_insert_len copy(dist_cache, orig_dist_cache[:4]) - *num_commands += zopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, nodes) + zopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, nodes) zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals) } diff --git a/block_splitter.go b/block_splitter.go index 2ccff45..f0063c1 100644 --- a/block_splitter.go +++ b/block_splitter.go @@ -33,23 +33,21 @@ const ( kMinItersForRefining uint = 100 ) -func countLiterals(cmds []command, num_commands uint) uint { +func countLiterals(cmds []command) uint { var total_length uint = 0 /* Count how many we have. */ - var i uint - for i = 0; i < num_commands; i++ { + for i := range cmds { total_length += uint(cmds[i].insert_len_) } return total_length } -func copyLiteralsToByteArray(cmds []command, num_commands uint, data []byte, offset uint, mask uint, literals []byte) { +func copyLiteralsToByteArray(cmds []command, data []byte, offset uint, mask uint, literals []byte) { var pos uint = 0 var from_pos uint = offset & mask - var i uint - for i = 0; i < num_commands; i++ { + for i := range cmds { var insert_len uint = uint(cmds[i].insert_len_) if from_pos+insert_len > mask { var head_size uint = mask + 1 - from_pos @@ -101,13 +99,13 @@ func destroyBlockSplit(self *blockSplit) { self.lengths = nil } -func splitBlock(cmds []command, num_commands uint, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) { +func splitBlock(cmds []command, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) { { - var literals_count uint = countLiterals(cmds, num_commands) + var literals_count uint = countLiterals(cmds) var literals []byte = make([]byte, literals_count) /* Create a continuous array of literals. */ - copyLiteralsToByteArray(cmds, num_commands, data, pos, mask, literals) + copyLiteralsToByteArray(cmds, data, pos, mask, literals) /* Create the block split on the array of literals. Literal histograms have alphabet size 256. */ @@ -116,28 +114,26 @@ func splitBlock(cmds []command, num_commands uint, data []byte, pos uint, mask u literals = nil } { - var insert_and_copy_codes []uint16 = make([]uint16, num_commands) + var insert_and_copy_codes []uint16 = make([]uint16, len(cmds)) /* Compute prefix codes for commands. */ - var i uint - for i = 0; i < num_commands; i++ { + for i := range cmds { insert_and_copy_codes[i] = cmds[i].cmd_prefix_ } /* Create the block split on the array of command prefixes. */ - splitByteVectorCommand(insert_and_copy_codes, num_commands, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params, insert_and_copy_split) + splitByteVectorCommand(insert_and_copy_codes, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params, insert_and_copy_split) /* TODO: reuse for distances? */ insert_and_copy_codes = nil } { - var distance_prefixes []uint16 = make([]uint16, num_commands) + var distance_prefixes []uint16 = make([]uint16, len(cmds)) var j uint = 0 /* Create a continuous array of distance prefixes. */ - var i uint - for i = 0; i < num_commands; i++ { + for i := range cmds { var cmd *command = &cmds[i] if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 { distance_prefixes[j] = cmd.dist_prefix_ & 0x3FF diff --git a/block_splitter_command.go b/block_splitter_command.go index e505fe1..9dec13e 100644 --- a/block_splitter_command.go +++ b/block_splitter_command.go @@ -372,7 +372,8 @@ func clusterBlocksCommand(data []uint16, length uint, num_blocks uint, block_ids histogram_symbols = nil } -func splitByteVectorCommand(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) { +func splitByteVectorCommand(data []uint16, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) { + length := uint(len(data)) var data_size uint = histogramDataSizeCommand() var num_histograms uint = length/literals_per_histogram + 1 var histograms []histogramCommand diff --git a/brotli_bit_stream.go b/brotli_bit_stream.go index 23d63f1..137ef36 100644 --- a/brotli_bit_stream.go +++ b/brotli_bit_stream.go @@ -990,7 +990,7 @@ func jumpToByteBoundary(storage_ix *uint, storage []byte) { storage[*storage_ix>>3] = 0 } -func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_byte byte, prev_byte2 byte, is_last bool, params *encoderParams, literal_context_mode int, commands []command, n_commands uint, mb *metaBlockSplit, storage_ix *uint, storage []byte) { +func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_byte byte, prev_byte2 byte, is_last bool, params *encoderParams, literal_context_mode int, commands []command, mb *metaBlockSplit, storage_ix *uint, storage []byte) { var pos uint = start_pos var i uint var num_distance_symbols uint32 = params.dist.alphabet_size @@ -1039,8 +1039,7 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b buildAndStoreEntropyCodesDistance(&distance_enc, mb.distance_histograms, mb.distance_histograms_size, uint(num_distance_symbols), tree, storage_ix, storage) tree = nil - for i = 0; i < n_commands; i++ { - var cmd command = commands[i] + for _, cmd := range commands { var cmd_code uint = uint(cmd.cmd_prefix_) storeSymbol(&command_enc, cmd_code, storage_ix, storage) storeCommandExtra(&cmd, storage_ix, storage) @@ -1090,11 +1089,9 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b } } -func buildHistograms(input []byte, start_pos uint, mask uint, commands []command, n_commands uint, lit_histo *histogramLiteral, cmd_histo *histogramCommand, dist_histo *histogramDistance) { +func buildHistograms(input []byte, start_pos uint, mask uint, commands []command, lit_histo *histogramLiteral, cmd_histo *histogramCommand, dist_histo *histogramDistance) { var pos uint = start_pos - var i uint - for i = 0; i < n_commands; i++ { - var cmd command = commands[i] + for _, cmd := range commands { var j uint histogramAddCommand(cmd_histo, uint(cmd.cmd_prefix_)) for j = uint(cmd.insert_len_); j != 0; j-- { @@ -1109,11 +1106,9 @@ func buildHistograms(input []byte, start_pos uint, mask uint, commands []command } } -func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands []command, n_commands uint, lit_depth []byte, lit_bits []uint16, cmd_depth []byte, cmd_bits []uint16, dist_depth []byte, dist_bits []uint16, storage_ix *uint, storage []byte) { +func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands []command, lit_depth []byte, lit_bits []uint16, cmd_depth []byte, cmd_bits []uint16, dist_depth []byte, dist_bits []uint16, storage_ix *uint, storage []byte) { var pos uint = start_pos - var i uint - for i = 0; i < n_commands; i++ { - var cmd command = commands[i] + for _, cmd := range commands { var cmd_code uint = uint(cmd.cmd_prefix_) var j uint writeBits(uint(cmd_depth[cmd_code]), uint64(cmd_bits[cmd_code]), storage_ix, storage) @@ -1135,7 +1130,7 @@ func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands } } -func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, n_commands uint, storage_ix *uint, storage []byte) { +func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, storage_ix *uint, storage []byte) { var lit_histo histogramLiteral var cmd_histo histogramCommand var dist_histo histogramDistance @@ -1154,7 +1149,7 @@ func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, histogramClearCommand(&cmd_histo) histogramClearDistance(&dist_histo) - buildHistograms(input, start_pos, mask, commands, n_commands, &lit_histo, &cmd_histo, &dist_histo) + buildHistograms(input, start_pos, mask, commands, &lit_histo, &cmd_histo, &dist_histo) writeBits(13, 0, storage_ix, storage) @@ -1163,13 +1158,13 @@ func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, buildAndStoreHuffmanTree(cmd_histo.data_[:], numCommandSymbols, numCommandSymbols, tree, cmd_depth[:], cmd_bits[:], storage_ix, storage) buildAndStoreHuffmanTree(dist_histo.data_[:], maxSimpleDistanceAlphabetSize, uint(num_distance_symbols), tree, dist_depth[:], dist_bits[:], storage_ix, storage) tree = nil - storeDataWithHuffmanCodes(input, start_pos, mask, commands, n_commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage) + storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage) if is_last { jumpToByteBoundary(storage_ix, storage) } } -func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, n_commands uint, storage_ix *uint, storage []byte) { +func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, storage_ix *uint, storage []byte) { var num_distance_symbols uint32 = params.dist.alphabet_size var distance_alphabet_bits uint32 = log2FloorNonZero(uint(num_distance_symbols-1)) + 1 @@ -1177,15 +1172,13 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is writeBits(13, 0, storage_ix, storage) - if n_commands <= 128 { + if len(commands) <= 128 { var histogram = [numLiteralSymbols]uint32{0} var pos uint = start_pos var num_literals uint = 0 - var i uint var lit_depth [numLiteralSymbols]byte var lit_bits [numLiteralSymbols]uint16 - for i = 0; i < n_commands; i++ { - var cmd command = commands[i] + for _, cmd := range commands { var j uint for j = uint(cmd.insert_len_); j != 0; j-- { histogram[input[pos&mask]]++ @@ -1201,7 +1194,7 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is storeStaticCommandHuffmanTree(storage_ix, storage) storeStaticDistanceHuffmanTree(storage_ix, storage) - storeDataWithHuffmanCodes(input, start_pos, mask, commands, n_commands, lit_depth[:], lit_bits[:], kStaticCommandCodeDepth[:], kStaticCommandCodeBits[:], kStaticDistanceCodeDepth[:], kStaticDistanceCodeBits[:], storage_ix, storage) + storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], kStaticCommandCodeDepth[:], kStaticCommandCodeBits[:], kStaticDistanceCodeDepth[:], kStaticDistanceCodeBits[:], storage_ix, storage) } else { var lit_histo histogramLiteral var cmd_histo histogramCommand @@ -1215,7 +1208,7 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is histogramClearLiteral(&lit_histo) histogramClearCommand(&cmd_histo) histogramClearDistance(&dist_histo) - buildHistograms(input, start_pos, mask, commands, n_commands, &lit_histo, &cmd_histo, &dist_histo) + buildHistograms(input, start_pos, mask, commands, &lit_histo, &cmd_histo, &dist_histo) buildAndStoreHuffmanTreeFast(lit_histo.data_[:], lit_histo.total_count_, /* max_bits = */ 8, lit_depth[:], lit_bits[:], storage_ix, storage) @@ -1225,7 +1218,7 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is buildAndStoreHuffmanTreeFast(dist_histo.data_[:], dist_histo.total_count_, /* max_bits = */ uint(distance_alphabet_bits), dist_depth[:], dist_bits[:], storage_ix, storage) - storeDataWithHuffmanCodes(input, start_pos, mask, commands, n_commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage) + storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage) } if is_last { diff --git a/command.go b/command.go index e93ccdf..b1662a5 100644 --- a/command.go +++ b/command.go @@ -194,26 +194,28 @@ type command struct { } /* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */ -func initCommand(self *command, dist *distanceParams, insertlen uint, copylen uint, copylen_code_delta int, distance_code uint) { +func makeCommand(dist *distanceParams, insertlen uint, copylen uint, copylen_code_delta int, distance_code uint) (cmd command) { /* Don't rely on signed int representation, use honest casts. */ var delta uint32 = uint32(byte(int8(copylen_code_delta))) - self.insert_len_ = uint32(insertlen) - self.copy_len_ = uint32(uint32(copylen) | delta<<25) + cmd.insert_len_ = uint32(insertlen) + cmd.copy_len_ = uint32(uint32(copylen) | delta<<25) /* The distance prefix and extra bits are stored in this Command as if npostfix and ndirect were 0, they are only recomputed later after the clustering if needed. */ - prefixEncodeCopyDistance(distance_code, uint(dist.num_direct_distance_codes), uint(dist.distance_postfix_bits), &self.dist_prefix_, &self.dist_extra_) + prefixEncodeCopyDistance(distance_code, uint(dist.num_direct_distance_codes), uint(dist.distance_postfix_bits), &cmd.dist_prefix_, &cmd.dist_extra_) + getLengthCode(insertlen, uint(int(copylen)+copylen_code_delta), (cmd.dist_prefix_&0x3FF == 0), &cmd.cmd_prefix_) - getLengthCode(insertlen, uint(int(copylen)+copylen_code_delta), (self.dist_prefix_&0x3FF == 0), &self.cmd_prefix_) + return cmd } -func initInsertCommand(self *command, insertlen uint) { - self.insert_len_ = uint32(insertlen) - self.copy_len_ = 4 << 25 - self.dist_extra_ = 0 - self.dist_prefix_ = numDistanceShortCodes - getLengthCode(insertlen, 4, false, &self.cmd_prefix_) +func makeInsertCommand(insertlen uint) (cmd command) { + cmd.insert_len_ = uint32(insertlen) + cmd.copy_len_ = 4 << 25 + cmd.dist_extra_ = 0 + cmd.dist_prefix_ = numDistanceShortCodes + getLengthCode(insertlen, 4, false, &cmd.cmd_prefix_) + return cmd } func commandRestoreDistanceCode(self *command, dist *distanceParams) uint32 { diff --git a/encode.go b/encode.go index c9e9763..dbd9db8 100644 --- a/encode.go +++ b/encode.go @@ -80,9 +80,7 @@ type Writer struct { hasher_ hasherHandle input_pos_ uint64 ringbuffer_ ringBuffer - cmd_alloc_size_ uint - commands_ []command - num_commands_ uint + commands []command num_literals_ uint last_insert_len_ uint last_flush_pos_ uint64 @@ -434,7 +432,7 @@ func chooseContextMode(params *encoderParams, data []byte, pos uint, mask uint, return contextUTF8 } -func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes uint, is_last bool, literal_context_mode int, params *encoderParams, prev_byte byte, prev_byte2 byte, num_literals uint, num_commands uint, commands []command, saved_dist_cache []int, dist_cache []int, storage_ix *uint, storage []byte) { +func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes uint, is_last bool, literal_context_mode int, params *encoderParams, prev_byte byte, prev_byte2 byte, num_literals uint, commands []command, saved_dist_cache []int, dist_cache []int, storage_ix *uint, storage []byte) { var wrapped_last_flush_pos uint32 = wrapPosition(last_flush_pos) var last_bytes uint16 var last_bytes_bits byte @@ -449,7 +447,7 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes return } - if !shouldCompress_encode(data, mask, last_flush_pos, bytes, num_literals, num_commands) { + if !shouldCompress_encode(data, mask, last_flush_pos, bytes, num_literals, uint(len(commands))) { /* Restore the distance cache, as its last update by CreateBackwardReferences is now unused. */ copy(dist_cache, saved_dist_cache[:4]) @@ -462,9 +460,9 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes last_bytes = uint16(storage[1])<<8 | uint16(storage[0]) last_bytes_bits = byte(*storage_ix) if params.quality <= maxQualityForStaticEntropyCodes { - storeMetaBlockFast(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, num_commands, storage_ix, storage) + storeMetaBlockFast(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, storage_ix, storage) } else if params.quality < minQualityForBlockSplit { - storeMetaBlockTrivial(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, num_commands, storage_ix, storage) + storeMetaBlockTrivial(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, storage_ix, storage) } else { var mb metaBlockSplit initMetaBlockSplit(&mb) @@ -475,9 +473,9 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes decideOverLiteralContextModeling(data, uint(wrapped_last_flush_pos), bytes, mask, params.quality, params.size_hint, &num_literal_contexts, &literal_context_map) } - buildMetaBlockGreedy(data, uint(wrapped_last_flush_pos), mask, prev_byte, prev_byte2, literal_context_lut, num_literal_contexts, literal_context_map, commands, num_commands, &mb) + buildMetaBlockGreedy(data, uint(wrapped_last_flush_pos), mask, prev_byte, prev_byte2, literal_context_lut, num_literal_contexts, literal_context_map, commands, &mb) } else { - buildMetaBlock(data, uint(wrapped_last_flush_pos), mask, &block_params, prev_byte, prev_byte2, commands, num_commands, literal_context_mode, &mb) + buildMetaBlock(data, uint(wrapped_last_flush_pos), mask, &block_params, prev_byte, prev_byte2, commands, literal_context_mode, &mb) } if params.quality >= minQualityForOptimizeHistograms { @@ -492,7 +490,7 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes optimizeHistograms(num_effective_dist_codes, &mb) } - storeMetaBlock(data, uint(wrapped_last_flush_pos), bytes, mask, prev_byte, prev_byte2, is_last, &block_params, literal_context_mode, commands, num_commands, &mb, storage_ix, storage) + storeMetaBlock(data, uint(wrapped_last_flush_pos), bytes, mask, prev_byte, prev_byte2, is_last, &block_params, literal_context_mode, commands, &mb, storage_ix, storage) destroyMetaBlockSplit(&mb) } @@ -613,7 +611,7 @@ func encoderInitParams(params *encoderParams) { func encoderInitState(s *Writer) { encoderInitParams(&s.params) s.input_pos_ = 0 - s.num_commands_ = 0 + s.commands = s.commands[:0] s.num_literals_ = 0 s.last_insert_len_ = 0 s.last_flush_pos_ = 0 @@ -712,7 +710,7 @@ func updateLastProcessedPos(s *Writer) bool { } func extendLastCommand(s *Writer, bytes *uint32, wrapped_last_processed_pos *uint32) { - var last_command *command = &s.commands_[s.num_commands_-1] + var last_command *command = &s.commands[len(s.commands)-1] var data []byte = s.ringbuffer_.buffer_ var mask uint32 = s.ringbuffer_.mask_ var max_backward_distance uint64 = ((uint64(1)) << s.params.lgwin) - windowGap @@ -809,22 +807,18 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool { } { /* Theoretical max number of commands is 1 per 2 bytes. */ - var newsize uint = uint(uint32(s.num_commands_) + bytes/2 + 1) - if newsize > s.cmd_alloc_size_ { - var new_commands []command - + newsize := len(s.commands) + int(bytes)/2 + 1 + if newsize > cap(s.commands) { /* Reserve a bit more memory to allow merging with a next block without reallocation: that would impact speed. */ - newsize += uint((bytes / 4) + 16) + newsize += int(bytes/4) + 16 - s.cmd_alloc_size_ = newsize - new_commands = make([]command, newsize) - if s.commands_ != nil { - copy(new_commands, s.commands_[:s.num_commands_]) - s.commands_ = nil + new_commands := make([]command, len(s.commands), newsize) + if s.commands != nil { + copy(new_commands, s.commands) } - s.commands_ = new_commands + s.commands = new_commands } } @@ -832,32 +826,32 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool { literal_context_mode = chooseContextMode(&s.params, data, uint(wrapPosition(s.last_flush_pos_)), uint(mask), uint(s.input_pos_-s.last_flush_pos_)) - if s.num_commands_ != 0 && s.last_insert_len_ == 0 { + if len(s.commands) != 0 && s.last_insert_len_ == 0 { extendLastCommand(s, &bytes, &wrapped_last_processed_pos) } if s.params.quality == zopflificationQuality { assert(s.params.hasher.type_ == 10) - createZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_.(*h10), s.dist_cache_[:], &s.last_insert_len_, s.commands_[s.num_commands_:], &s.num_commands_, &s.num_literals_) + createZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_.(*h10), s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_) } else if s.params.quality == hqZopflificationQuality { assert(s.params.hasher.type_ == 10) - createHqZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, s.commands_[s.num_commands_:], &s.num_commands_, &s.num_literals_) + createHqZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_) } else { - createBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, s.commands_[s.num_commands_:], &s.num_commands_, &s.num_literals_) + createBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_) } { var max_length uint = maxMetablockSize(&s.params) var max_literals uint = max_length / 8 - var max_commands uint = max_length / 8 + max_commands := int(max_length / 8) var processed_bytes uint = uint(s.input_pos_ - s.last_flush_pos_) var next_input_fits_metablock bool = (processed_bytes+inputBlockSize(s) <= max_length) - var should_flush bool = (s.params.quality < minQualityForBlockSplit && s.num_literals_+s.num_commands_ >= maxNumDelayedSymbols) + var should_flush bool = (s.params.quality < minQualityForBlockSplit && s.num_literals_+uint(len(s.commands)) >= maxNumDelayedSymbols) /* If maximal possible additional block doesn't fit metablock, flush now. */ /* TODO: Postpone decision until next block arrives? */ /* If block splitting is not used, then flush as soon as there is some amount of commands / literals produced. */ - if !is_last && !force_flush && !should_flush && next_input_fits_metablock && s.num_literals_ < max_literals && s.num_commands_ < max_commands { + if !is_last && !force_flush && !should_flush && next_input_fits_metablock && s.num_literals_ < max_literals && len(s.commands) < max_commands { /* Merge with next input block. Everything will happen later. */ if updateLastProcessedPos(s) { hasherReset(s.hasher_) @@ -869,8 +863,7 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool { /* Create the last insert-only command. */ if s.last_insert_len_ > 0 { - initInsertCommand(&s.commands_[s.num_commands_], s.last_insert_len_) - s.num_commands_++ + s.commands = append(s.commands, makeInsertCommand(s.last_insert_len_)) s.num_literals_ += s.last_insert_len_ s.last_insert_len_ = 0 } @@ -890,7 +883,7 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool { var storage_ix uint = uint(s.last_bytes_bits_) storage[0] = byte(s.last_bytes_) storage[1] = byte(s.last_bytes_ >> 8) - writeMetaBlockInternal(data, uint(mask), s.last_flush_pos_, uint(metablock_size), is_last, literal_context_mode, &s.params, s.prev_byte_, s.prev_byte2_, s.num_literals_, s.num_commands_, s.commands_, s.saved_dist_cache_[:], s.dist_cache_[:], &storage_ix, storage) + writeMetaBlockInternal(data, uint(mask), s.last_flush_pos_, uint(metablock_size), is_last, literal_context_mode, &s.params, s.prev_byte_, s.prev_byte2_, s.num_literals_, s.commands, s.saved_dist_cache_[:], s.dist_cache_[:], &storage_ix, storage) s.last_bytes_ = uint16(storage[storage_ix>>3]) s.last_bytes_bits_ = byte(storage_ix & 7) s.last_flush_pos_ = s.input_pos_ @@ -906,7 +899,7 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool { s.prev_byte2_ = data[uint32(s.last_flush_pos_-2)&mask] } - s.num_commands_ = 0 + s.commands = s.commands[:0] s.num_literals_ = 0 /* Save the state of the distance cache in case we need to restore it for diff --git a/histogram.go b/histogram.go index f208ff7..44db4f3 100644 --- a/histogram.go +++ b/histogram.go @@ -180,17 +180,16 @@ func blockSplitIteratorNext(self *blockSplitIterator) { self.length_-- } -func buildHistogramsWithContext(cmds []command, num_commands uint, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit, ringbuffer []byte, start_pos uint, mask uint, prev_byte byte, prev_byte2 byte, context_modes []int, literal_histograms []histogramLiteral, insert_and_copy_histograms []histogramCommand, copy_dist_histograms []histogramDistance) { +func buildHistogramsWithContext(cmds []command, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit, ringbuffer []byte, start_pos uint, mask uint, prev_byte byte, prev_byte2 byte, context_modes []int, literal_histograms []histogramLiteral, insert_and_copy_histograms []histogramCommand, copy_dist_histograms []histogramDistance) { var pos uint = start_pos var literal_it blockSplitIterator var insert_and_copy_it blockSplitIterator var dist_it blockSplitIterator - var i uint initBlockSplitIterator(&literal_it, literal_split) initBlockSplitIterator(&insert_and_copy_it, insert_and_copy_split) initBlockSplitIterator(&dist_it, dist_split) - for i = 0; i < num_commands; i++ { + for i := range cmds { var cmd *command = &cmds[i] var j uint blockSplitIteratorNext(&insert_and_copy_it) diff --git a/metablock.go b/metablock.go index 4a412cf..d7e9eee 100644 --- a/metablock.go +++ b/metablock.go @@ -84,14 +84,12 @@ func initDistanceParams(params *encoderParams, npostfix uint32, ndirect uint32) dist_params.max_distance = uint(max_distance) } -func recomputeDistancePrefixes(cmds []command, num_commands uint, orig_params *distanceParams, new_params *distanceParams) { - var i uint - +func recomputeDistancePrefixes(cmds []command, orig_params *distanceParams, new_params *distanceParams) { if orig_params.distance_postfix_bits == new_params.distance_postfix_bits && orig_params.num_direct_distance_codes == new_params.num_direct_distance_codes { return } - for i = 0; i < num_commands; i++ { + for i := range cmds { var cmd *command = &cmds[i] if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 { prefixEncodeCopyDistance(uint(commandRestoreDistanceCode(cmd, orig_params)), uint(new_params.num_direct_distance_codes), uint(new_params.distance_postfix_bits), &cmd.dist_prefix_, &cmd.dist_extra_) @@ -99,8 +97,7 @@ func recomputeDistancePrefixes(cmds []command, num_commands uint, orig_params *d } } -func computeDistanceCost(cmds []command, num_commands uint, orig_params *distanceParams, new_params *distanceParams, cost *float64) bool { - var i uint +func computeDistanceCost(cmds []command, orig_params *distanceParams, new_params *distanceParams, cost *float64) bool { var equal_params bool = false var dist_prefix uint16 var dist_extra uint32 @@ -112,8 +109,8 @@ func computeDistanceCost(cmds []command, num_commands uint, orig_params *distanc equal_params = true } - for i = 0; i < num_commands; i++ { - var cmd *command = &cmds[i] + for i := range cmds { + cmd := &cmds[i] if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 { if equal_params { dist_prefix = cmd.dist_prefix_ @@ -137,7 +134,7 @@ func computeDistanceCost(cmds []command, num_commands uint, orig_params *distanc var buildMetaBlock_kMaxNumberOfHistograms uint = 256 -func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParams, prev_byte byte, prev_byte2 byte, cmds []command, num_commands uint, literal_context_mode int, mb *metaBlockSplit) { +func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParams, prev_byte byte, prev_byte2 byte, cmds []command, literal_context_mode int, mb *metaBlockSplit) { var distance_histograms []histogramDistance var literal_histograms []histogramLiteral var literal_context_modes []int = nil @@ -164,7 +161,7 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam check_orig = false } - skip = !computeDistanceCost(cmds, num_commands, &orig_params.dist, &new_params.dist, &dist_cost) + skip = !computeDistanceCost(cmds, &orig_params.dist, &new_params.dist, &dist_cost) if skip || (dist_cost > best_dist_cost) { break } @@ -181,7 +178,7 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam if check_orig { var dist_cost float64 - computeDistanceCost(cmds, num_commands, &orig_params.dist, &orig_params.dist, &dist_cost) + computeDistanceCost(cmds, &orig_params.dist, &orig_params.dist, &dist_cost) if dist_cost < best_dist_cost { /* NB: currently unused; uncomment when more param tuning is added. */ /* best_dist_cost = dist_cost; */ @@ -189,9 +186,9 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam } } - recomputeDistancePrefixes(cmds, num_commands, &orig_params.dist, ¶ms.dist) + recomputeDistancePrefixes(cmds, &orig_params.dist, ¶ms.dist) - splitBlock(cmds, num_commands, ringbuffer, pos, mask, params, &mb.literal_split, &mb.command_split, &mb.distance_split) + splitBlock(cmds, ringbuffer, pos, mask, params, &mb.literal_split, &mb.command_split, &mb.distance_split) if !params.disable_literal_context_modeling { literal_context_multiplier = 1 << literalContextBits @@ -214,7 +211,7 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam mb.command_histograms = make([]histogramCommand, (mb.command_histograms_size)) clearHistogramsCommand(mb.command_histograms, mb.command_histograms_size) - buildHistogramsWithContext(cmds, num_commands, &mb.literal_split, &mb.command_split, &mb.distance_split, ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_modes, literal_histograms, mb.command_histograms, distance_histograms) + buildHistogramsWithContext(cmds, &mb.literal_split, &mb.command_split, &mb.distance_split, ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_modes, literal_histograms, mb.command_histograms, distance_histograms) literal_context_modes = nil assert(mb.literal_context_map == nil) @@ -466,7 +463,7 @@ func mapStaticContexts(num_contexts uint, static_context_map []uint32, mb *metaB } } -func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, n_commands uint, mb *metaBlockSplit) { +func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, mb *metaBlockSplit) { var lit_blocks struct { plain blockSplitterLiteral ctx contextBlockSplitter @@ -474,8 +471,7 @@ func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_b var cmd_blocks blockSplitterCommand var dist_blocks blockSplitterDistance var num_literals uint = 0 - var i uint - for i = 0; i < n_commands; i++ { + for i := range commands { num_literals += uint(commands[i].insert_len_) } @@ -485,11 +481,10 @@ func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_b initContextBlockSplitter(&lit_blocks.ctx, 256, num_contexts, 512, 400.0, num_literals, &mb.literal_split, &mb.literal_histograms, &mb.literal_histograms_size) } - initBlockSplitterCommand(&cmd_blocks, numCommandSymbols, 1024, 500.0, n_commands, &mb.command_split, &mb.command_histograms, &mb.command_histograms_size) - initBlockSplitterDistance(&dist_blocks, 64, 512, 100.0, n_commands, &mb.distance_split, &mb.distance_histograms, &mb.distance_histograms_size) + initBlockSplitterCommand(&cmd_blocks, numCommandSymbols, 1024, 500.0, uint(len(commands)), &mb.command_split, &mb.command_histograms, &mb.command_histograms_size) + initBlockSplitterDistance(&dist_blocks, 64, 512, 100.0, uint(len(commands)), &mb.distance_split, &mb.distance_histograms, &mb.distance_histograms_size) - for i = 0; i < n_commands; i++ { - var cmd command = commands[i] + for _, cmd := range commands { var j uint blockSplitterAddSymbolCommand(&cmd_blocks, uint(cmd.cmd_prefix_)) for j = uint(cmd.insert_len_); j != 0; j-- { @@ -530,11 +525,11 @@ func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_b } } -func buildMetaBlockGreedy(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, n_commands uint, mb *metaBlockSplit) { +func buildMetaBlockGreedy(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, mb *metaBlockSplit) { if num_contexts == 1 { - buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, 1, nil, commands, n_commands, mb) + buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, 1, nil, commands, mb) } else { - buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, num_contexts, static_context_map, commands, n_commands, mb) + buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, num_contexts, static_context_map, commands, mb) } }