re
/
brotli
forked from mirror/brotli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use len and cap instead of num_commands_ and cmd_alloc_size_.

This commit is contained in:
Andy Balholm 2020-05-08 16:48:16 -07:00
parent 4b2775ea5e
commit e2c5f2109f
9 changed files with 108 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
backward_references.go
View File

@ -31,13 +31,8 @@ func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uin
return distance + numDistanceShortCodes - 1
}
/* "commands" points to the next output command to write to, "*num_commands" is
initially the total amount of commands output by previous
CreateBackwardReferences calls, and must be incremented by the amount written
by this call. */
func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) {
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
var orig_commands []command = commands
var insert_length uint = *last_insert_len
var pos_end uint = position + num_bytes
var store_end uint
@ -114,8 +109,7 @@ func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte,
hasher.PrepareDistanceCache(dist_cache)
}
initCommand(&commands[0], &params.dist, insert_length, sr.len, sr.len_code_delta, distance_code)
commands = commands[1:]
*commands = append(*commands, makeCommand(&params.dist, insert_length, sr.len, sr.len_code_delta, distance_code))
}
*num_literals += insert_length
@ -173,5 +167,4 @@ func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte,
insert_length += pos_end - position
*last_insert_len = insert_length
*num_commands += uint(-cap(commands) + cap(orig_commands))
}

33
backward_references_hq.go
View File

@ -123,14 +123,13 @@ func setCost(histogram []uint32, histogram_size uint, literal_histogram bool, co
}
}
func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint, commands []command, num_commands uint, last_insert_len uint) {
func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint, commands []command, last_insert_len uint) {
var histogram_literal [numLiteralSymbols]uint32
var histogram_cmd [numCommandSymbols]uint32
var histogram_dist [maxEffectiveDistanceAlphabetSize]uint32
var cost_literal [numLiteralSymbols]float32
var pos uint = position - last_insert_len
var min_cost_cmd float32 = kInfinity
var i uint
var cost_cmd []float32 = self.cost_cmd_[:]
var literal_costs []float32
@ -138,7 +137,7 @@ func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbu
histogram_cmd = [numCommandSymbols]uint32{}
histogram_dist = [maxEffectiveDistanceAlphabetSize]uint32{}
for i = 0; i < num_commands; i++ {
for i := range commands {
var inslength uint = uint(commands[i].insert_len_)
var copylength uint = uint(commandCopyLen(&commands[i]))
var distcode uint = uint(commands[i].dist_prefix_) & 0x3FF
@ -161,7 +160,7 @@ func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbu
setCost(histogram_cmd[:], numCommandSymbols, false, cost_cmd)
setCost(histogram_dist[:], uint(self.distance_histogram_size), false, self.cost_dist_)
for i = 0; i < numCommandSymbols; i++ {
for i := 0; i < numCommandSymbols; i++ {
min_cost_cmd = brotli_min_float(min_cost_cmd, cost_cmd[i])
}
@ -169,10 +168,10 @@ func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbu
{
literal_costs = self.literal_costs_
var literal_carry float32 = 0.0
var num_bytes uint = self.num_bytes_
num_bytes := int(self.num_bytes_)
literal_costs[0] = 0.0
for i = 0; i < num_bytes; i++ {
literal_carry += cost_literal[ringbuffer[(position+i)&ringbuffer_mask]]
for i := 0; i < num_bytes; i++ {
literal_carry += cost_literal[ringbuffer[(position+uint(i))&ringbuffer_mask]]
literal_costs[i+1] = literal_costs[i] + literal_carry
literal_carry -= literal_costs[i+1] - literal_costs[i]
}
@ -532,7 +531,7 @@ func computeShortestPathFromNodes(num_bytes uint, nodes []zopfliNode) uint {
}
/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands []command, num_literals *uint) {
func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands *[]command, num_literals *uint) {
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
var pos uint = 0
var offset uint32 = nodes[0].u.next
@ -554,7 +553,7 @@ func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode,
var max_distance uint = brotli_min_size_t(block_start+pos, max_backward_limit)
var is_dictionary bool = (distance > max_distance+gap)
var dist_code uint = uint(zopfliNodeDistanceCode(next))
initCommand(&commands[i], &params.dist, insert_length, copy_length, int(len_code)-int(copy_length), dist_code)
*commands = append(*commands, makeCommand(&params.dist, insert_length, copy_length, int(len_code)-int(copy_length), dist_code))
if !is_dictionary && dist_code > 0 {
dist_cache[3] = dist_cache[2]
@ -681,16 +680,16 @@ func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte,
return computeShortestPathFromNodes(num_bytes, nodes)
}
func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) {
var nodes []zopfliNode
nodes = make([]zopfliNode, (num_bytes + 1))
initZopfliNodes(nodes, num_bytes+1)
*num_commands += zopfliComputeShortestPath(num_bytes, position, ringbuffer, ringbuffer_mask, params, dist_cache, hasher, nodes)
zopfliComputeShortestPath(num_bytes, position, ringbuffer, ringbuffer_mask, params, dist_cache, hasher, nodes)
zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals)
nodes = nil
}
func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) {
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
var num_matches []uint32 = make([]uint32, num_bytes)
var matches_size uint = 4 * num_bytes
@ -705,7 +704,7 @@ func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer
var orig_num_literals uint
var orig_last_insert_len uint
var orig_dist_cache [4]int
var orig_num_commands uint
var orig_num_commands int
var model zopfliCostModel
var nodes []zopfliNode
var matches []backwardMatch = make([]backwardMatch, matches_size)
@ -771,7 +770,7 @@ func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer
orig_num_literals = *num_literals
orig_last_insert_len = *last_insert_len
copy(orig_dist_cache[:], dist_cache[:4])
orig_num_commands = *num_commands
orig_num_commands = len(*commands)
nodes = make([]zopfliNode, (num_bytes + 1))
initZopfliCostModel(&model, &params.dist, num_bytes)
for i = 0; i < 2; i++ {
@ -779,14 +778,14 @@ func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer
if i == 0 {
zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask)
} else {
zopfliCostModelSetFromCommands(&model, position, ringbuffer, ringbuffer_mask, commands, *num_commands-orig_num_commands, orig_last_insert_len)
zopfliCostModelSetFromCommands(&model, position, ringbuffer, ringbuffer_mask, (*commands)[orig_num_commands:], orig_last_insert_len)
}
*num_commands = orig_num_commands
*commands = (*commands)[:orig_num_commands]
*num_literals = orig_num_literals
*last_insert_len = orig_last_insert_len
copy(dist_cache, orig_dist_cache[:4])
*num_commands += zopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, nodes)
zopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, nodes)
zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals)
}

28
block_splitter.go
View File

@ -33,23 +33,21 @@ const (
kMinItersForRefining uint = 100
)
func countLiterals(cmds []command, num_commands uint) uint {
func countLiterals(cmds []command) uint {
var total_length uint = 0
/* Count how many we have. */
var i uint
for i = 0; i < num_commands; i++ {
for i := range cmds {
total_length += uint(cmds[i].insert_len_)
}
return total_length
}
func copyLiteralsToByteArray(cmds []command, num_commands uint, data []byte, offset uint, mask uint, literals []byte) {
func copyLiteralsToByteArray(cmds []command, data []byte, offset uint, mask uint, literals []byte) {
var pos uint = 0
var from_pos uint = offset & mask
var i uint
for i = 0; i < num_commands; i++ {
for i := range cmds {
var insert_len uint = uint(cmds[i].insert_len_)
if from_pos+insert_len > mask {
var head_size uint = mask + 1 - from_pos
@ -101,13 +99,13 @@ func destroyBlockSplit(self *blockSplit) {
self.lengths = nil
}
func splitBlock(cmds []command, num_commands uint, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) {
func splitBlock(cmds []command, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) {
{
var literals_count uint = countLiterals(cmds, num_commands)
var literals_count uint = countLiterals(cmds)
var literals []byte = make([]byte, literals_count)
/* Create a continuous array of literals. */
copyLiteralsToByteArray(cmds, num_commands, data, pos, mask, literals)
copyLiteralsToByteArray(cmds, data, pos, mask, literals)
/* Create the block split on the array of literals.
Literal histograms have alphabet size 256. */
@ -116,28 +114,26 @@ func splitBlock(cmds []command, num_commands uint, data []byte, pos uint, mask u
literals = nil
}
{
var insert_and_copy_codes []uint16 = make([]uint16, num_commands)
var insert_and_copy_codes []uint16 = make([]uint16, len(cmds))
/* Compute prefix codes for commands. */
var i uint
for i = 0; i < num_commands; i++ {
for i := range cmds {
insert_and_copy_codes[i] = cmds[i].cmd_prefix_
}
/* Create the block split on the array of command prefixes. */
splitByteVectorCommand(insert_and_copy_codes, num_commands, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params, insert_and_copy_split)
splitByteVectorCommand(insert_and_copy_codes, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params, insert_and_copy_split)
/* TODO: reuse for distances? */
insert_and_copy_codes = nil
}
{
var distance_prefixes []uint16 = make([]uint16, num_commands)
var distance_prefixes []uint16 = make([]uint16, len(cmds))
var j uint = 0
/* Create a continuous array of distance prefixes. */
var i uint
for i = 0; i < num_commands; i++ {
for i := range cmds {
var cmd *command = &cmds[i]
if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 {
distance_prefixes[j] = cmd.dist_prefix_ & 0x3FF

3
block_splitter_command.go
View File

@ -372,7 +372,8 @@ func clusterBlocksCommand(data []uint16, length uint, num_blocks uint, block_ids
histogram_symbols = nil
}
func splitByteVectorCommand(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
func splitByteVectorCommand(data []uint16, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
length := uint(len(data))
var data_size uint = histogramDataSizeCommand()
var num_histograms uint = length/literals_per_histogram + 1
var histograms []histogramCommand

37
brotli_bit_stream.go
View File

@ -990,7 +990,7 @@ func jumpToByteBoundary(storage_ix *uint, storage []byte) {
storage[*storage_ix>>3] = 0
}
func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_byte byte, prev_byte2 byte, is_last bool, params *encoderParams, literal_context_mode int, commands []command, n_commands uint, mb *metaBlockSplit, storage_ix *uint, storage []byte) {
func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_byte byte, prev_byte2 byte, is_last bool, params *encoderParams, literal_context_mode int, commands []command, mb *metaBlockSplit, storage_ix *uint, storage []byte) {
var pos uint = start_pos
var i uint
var num_distance_symbols uint32 = params.dist.alphabet_size
@ -1039,8 +1039,7 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b
buildAndStoreEntropyCodesDistance(&distance_enc, mb.distance_histograms, mb.distance_histograms_size, uint(num_distance_symbols), tree, storage_ix, storage)
tree = nil
for i = 0; i < n_commands; i++ {
var cmd command = commands[i]
for _, cmd := range commands {
var cmd_code uint = uint(cmd.cmd_prefix_)
storeSymbol(&command_enc, cmd_code, storage_ix, storage)
storeCommandExtra(&cmd, storage_ix, storage)
@ -1090,11 +1089,9 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b
}
}
func buildHistograms(input []byte, start_pos uint, mask uint, commands []command, n_commands uint, lit_histo *histogramLiteral, cmd_histo *histogramCommand, dist_histo *histogramDistance) {
func buildHistograms(input []byte, start_pos uint, mask uint, commands []command, lit_histo *histogramLiteral, cmd_histo *histogramCommand, dist_histo *histogramDistance) {
var pos uint = start_pos
var i uint
for i = 0; i < n_commands; i++ {
var cmd command = commands[i]
for _, cmd := range commands {
var j uint
histogramAddCommand(cmd_histo, uint(cmd.cmd_prefix_))
for j = uint(cmd.insert_len_); j != 0; j-- {
@ -1109,11 +1106,9 @@ func buildHistograms(input []byte, start_pos uint, mask uint, commands []command
}
}
func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands []command, n_commands uint, lit_depth []byte, lit_bits []uint16, cmd_depth []byte, cmd_bits []uint16, dist_depth []byte, dist_bits []uint16, storage_ix *uint, storage []byte) {
func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands []command, lit_depth []byte, lit_bits []uint16, cmd_depth []byte, cmd_bits []uint16, dist_depth []byte, dist_bits []uint16, storage_ix *uint, storage []byte) {
var pos uint = start_pos
var i uint
for i = 0; i < n_commands; i++ {
var cmd command = commands[i]
for _, cmd := range commands {
var cmd_code uint = uint(cmd.cmd_prefix_)
var j uint
writeBits(uint(cmd_depth[cmd_code]), uint64(cmd_bits[cmd_code]), storage_ix, storage)
@ -1135,7 +1130,7 @@ func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands
}
}
func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, n_commands uint, storage_ix *uint, storage []byte) {
func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, storage_ix *uint, storage []byte) {
var lit_histo histogramLiteral
var cmd_histo histogramCommand
var dist_histo histogramDistance
@ -1154,7 +1149,7 @@ func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint,
histogramClearCommand(&cmd_histo)
histogramClearDistance(&dist_histo)
buildHistograms(input, start_pos, mask, commands, n_commands, &lit_histo, &cmd_histo, &dist_histo)
buildHistograms(input, start_pos, mask, commands, &lit_histo, &cmd_histo, &dist_histo)
writeBits(13, 0, storage_ix, storage)
@ -1163,13 +1158,13 @@ func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint,
buildAndStoreHuffmanTree(cmd_histo.data_[:], numCommandSymbols, numCommandSymbols, tree, cmd_depth[:], cmd_bits[:], storage_ix, storage)
buildAndStoreHuffmanTree(dist_histo.data_[:], maxSimpleDistanceAlphabetSize, uint(num_distance_symbols), tree, dist_depth[:], dist_bits[:], storage_ix, storage)
tree = nil
storeDataWithHuffmanCodes(input, start_pos, mask, commands, n_commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage)
storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage)
if is_last {
jumpToByteBoundary(storage_ix, storage)
}
}
func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, n_commands uint, storage_ix *uint, storage []byte) {
func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, storage_ix *uint, storage []byte) {
var num_distance_symbols uint32 = params.dist.alphabet_size
var distance_alphabet_bits uint32 = log2FloorNonZero(uint(num_distance_symbols-1)) + 1
@ -1177,15 +1172,13 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is
writeBits(13, 0, storage_ix, storage)
if n_commands <= 128 {
if len(commands) <= 128 {
var histogram = [numLiteralSymbols]uint32{0}
var pos uint = start_pos
var num_literals uint = 0
var i uint
var lit_depth [numLiteralSymbols]byte
var lit_bits [numLiteralSymbols]uint16
for i = 0; i < n_commands; i++ {
var cmd command = commands[i]
for _, cmd := range commands {
var j uint
for j = uint(cmd.insert_len_); j != 0; j-- {
histogram[input[pos&mask]]++
@ -1201,7 +1194,7 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is
storeStaticCommandHuffmanTree(storage_ix, storage)
storeStaticDistanceHuffmanTree(storage_ix, storage)
storeDataWithHuffmanCodes(input, start_pos, mask, commands, n_commands, lit_depth[:], lit_bits[:], kStaticCommandCodeDepth[:], kStaticCommandCodeBits[:], kStaticDistanceCodeDepth[:], kStaticDistanceCodeBits[:], storage_ix, storage)
storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], kStaticCommandCodeDepth[:], kStaticCommandCodeBits[:], kStaticDistanceCodeDepth[:], kStaticDistanceCodeBits[:], storage_ix, storage)
} else {
var lit_histo histogramLiteral
var cmd_histo histogramCommand
@ -1215,7 +1208,7 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is
histogramClearLiteral(&lit_histo)
histogramClearCommand(&cmd_histo)
histogramClearDistance(&dist_histo)
buildHistograms(input, start_pos, mask, commands, n_commands, &lit_histo, &cmd_histo, &dist_histo)
buildHistograms(input, start_pos, mask, commands, &lit_histo, &cmd_histo, &dist_histo)
buildAndStoreHuffmanTreeFast(lit_histo.data_[:], lit_histo.total_count_, /* max_bits = */
8, lit_depth[:], lit_bits[:], storage_ix, storage)
@ -1225,7 +1218,7 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is
buildAndStoreHuffmanTreeFast(dist_histo.data_[:], dist_histo.total_count_, /* max_bits = */
uint(distance_alphabet_bits), dist_depth[:], dist_bits[:], storage_ix, storage)
storeDataWithHuffmanCodes(input, start_pos, mask, commands, n_commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage)
storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage)
}
if is_last {

24
command.go
View File

@ -194,26 +194,28 @@ type command struct {
}
/* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */
func initCommand(self *command, dist *distanceParams, insertlen uint, copylen uint, copylen_code_delta int, distance_code uint) {
func makeCommand(dist *distanceParams, insertlen uint, copylen uint, copylen_code_delta int, distance_code uint) (cmd command) {
/* Don't rely on signed int representation, use honest casts. */
var delta uint32 = uint32(byte(int8(copylen_code_delta)))
self.insert_len_ = uint32(insertlen)
self.copy_len_ = uint32(uint32(copylen) | delta<<25)
cmd.insert_len_ = uint32(insertlen)
cmd.copy_len_ = uint32(uint32(copylen) | delta<<25)
/* The distance prefix and extra bits are stored in this Command as if
npostfix and ndirect were 0, they are only recomputed later after the
clustering if needed. */
prefixEncodeCopyDistance(distance_code, uint(dist.num_direct_distance_codes), uint(dist.distance_postfix_bits), &self.dist_prefix_, &self.dist_extra_)
prefixEncodeCopyDistance(distance_code, uint(dist.num_direct_distance_codes), uint(dist.distance_postfix_bits), &cmd.dist_prefix_, &cmd.dist_extra_)
getLengthCode(insertlen, uint(int(copylen)+copylen_code_delta), (cmd.dist_prefix_&0x3FF == 0), &cmd.cmd_prefix_)
getLengthCode(insertlen, uint(int(copylen)+copylen_code_delta), (self.dist_prefix_&0x3FF == 0), &self.cmd_prefix_)
return cmd
}
func initInsertCommand(self *command, insertlen uint) {
self.insert_len_ = uint32(insertlen)
self.copy_len_ = 4 << 25
self.dist_extra_ = 0
self.dist_prefix_ = numDistanceShortCodes
getLengthCode(insertlen, 4, false, &self.cmd_prefix_)
func makeInsertCommand(insertlen uint) (cmd command) {
cmd.insert_len_ = uint32(insertlen)
cmd.copy_len_ = 4 << 25
cmd.dist_extra_ = 0
cmd.dist_prefix_ = numDistanceShortCodes
getLengthCode(insertlen, 4, false, &cmd.cmd_prefix_)
return cmd
}
func commandRestoreDistanceCode(self *command, dist *distanceParams) uint32 {

61
encode.go
View File

@ -80,9 +80,7 @@ type Writer struct {
hasher_ hasherHandle
input_pos_ uint64
ringbuffer_ ringBuffer
cmd_alloc_size_ uint
commands_ []command
num_commands_ uint
commands []command
num_literals_ uint
last_insert_len_ uint
last_flush_pos_ uint64
@ -434,7 +432,7 @@ func chooseContextMode(params *encoderParams, data []byte, pos uint, mask uint,
return contextUTF8
}
func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes uint, is_last bool, literal_context_mode int, params *encoderParams, prev_byte byte, prev_byte2 byte, num_literals uint, num_commands uint, commands []command, saved_dist_cache []int, dist_cache []int, storage_ix *uint, storage []byte) {
func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes uint, is_last bool, literal_context_mode int, params *encoderParams, prev_byte byte, prev_byte2 byte, num_literals uint, commands []command, saved_dist_cache []int, dist_cache []int, storage_ix *uint, storage []byte) {
var wrapped_last_flush_pos uint32 = wrapPosition(last_flush_pos)
var last_bytes uint16
var last_bytes_bits byte
@ -449,7 +447,7 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes
return
}
if !shouldCompress_encode(data, mask, last_flush_pos, bytes, num_literals, num_commands) {
if !shouldCompress_encode(data, mask, last_flush_pos, bytes, num_literals, uint(len(commands))) {
/* Restore the distance cache, as its last update by
CreateBackwardReferences is now unused. */
copy(dist_cache, saved_dist_cache[:4])
@ -462,9 +460,9 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes
last_bytes = uint16(storage[1])<<8 | uint16(storage[0])
last_bytes_bits = byte(*storage_ix)
if params.quality <= maxQualityForStaticEntropyCodes {
storeMetaBlockFast(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, num_commands, storage_ix, storage)
storeMetaBlockFast(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, storage_ix, storage)
} else if params.quality < minQualityForBlockSplit {
storeMetaBlockTrivial(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, num_commands, storage_ix, storage)
storeMetaBlockTrivial(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, storage_ix, storage)
} else {
var mb metaBlockSplit
initMetaBlockSplit(&mb)
@ -475,9 +473,9 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes
decideOverLiteralContextModeling(data, uint(wrapped_last_flush_pos), bytes, mask, params.quality, params.size_hint, &num_literal_contexts, &literal_context_map)
}
buildMetaBlockGreedy(data, uint(wrapped_last_flush_pos), mask, prev_byte, prev_byte2, literal_context_lut, num_literal_contexts, literal_context_map, commands, num_commands, &mb)
buildMetaBlockGreedy(data, uint(wrapped_last_flush_pos), mask, prev_byte, prev_byte2, literal_context_lut, num_literal_contexts, literal_context_map, commands, &mb)
} else {
buildMetaBlock(data, uint(wrapped_last_flush_pos), mask, &block_params, prev_byte, prev_byte2, commands, num_commands, literal_context_mode, &mb)
buildMetaBlock(data, uint(wrapped_last_flush_pos), mask, &block_params, prev_byte, prev_byte2, commands, literal_context_mode, &mb)
}
if params.quality >= minQualityForOptimizeHistograms {
@ -492,7 +490,7 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes
optimizeHistograms(num_effective_dist_codes, &mb)
}
storeMetaBlock(data, uint(wrapped_last_flush_pos), bytes, mask, prev_byte, prev_byte2, is_last, &block_params, literal_context_mode, commands, num_commands, &mb, storage_ix, storage)
storeMetaBlock(data, uint(wrapped_last_flush_pos), bytes, mask, prev_byte, prev_byte2, is_last, &block_params, literal_context_mode, commands, &mb, storage_ix, storage)
destroyMetaBlockSplit(&mb)
}
@ -613,7 +611,7 @@ func encoderInitParams(params *encoderParams) {
func encoderInitState(s *Writer) {
encoderInitParams(&s.params)
s.input_pos_ = 0
s.num_commands_ = 0
s.commands = s.commands[:0]
s.num_literals_ = 0
s.last_insert_len_ = 0
s.last_flush_pos_ = 0
@ -712,7 +710,7 @@ func updateLastProcessedPos(s *Writer) bool {
}
func extendLastCommand(s *Writer, bytes *uint32, wrapped_last_processed_pos *uint32) {
var last_command *command = &s.commands_[s.num_commands_-1]
var last_command *command = &s.commands[len(s.commands)-1]
var data []byte = s.ringbuffer_.buffer_
var mask uint32 = s.ringbuffer_.mask_
var max_backward_distance uint64 = ((uint64(1)) << s.params.lgwin) - windowGap
@ -809,22 +807,18 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
}
{
/* Theoretical max number of commands is 1 per 2 bytes. */
var newsize uint = uint(uint32(s.num_commands_) + bytes/2 + 1)
if newsize > s.cmd_alloc_size_ {
var new_commands []command
newsize := len(s.commands) + int(bytes)/2 + 1
if newsize > cap(s.commands) {
/* Reserve a bit more memory to allow merging with a next block
without reallocation: that would impact speed. */
newsize += uint((bytes / 4) + 16)
newsize += int(bytes/4) + 16
s.cmd_alloc_size_ = newsize
new_commands = make([]command, newsize)
if s.commands_ != nil {
copy(new_commands, s.commands_[:s.num_commands_])
s.commands_ = nil
new_commands := make([]command, len(s.commands), newsize)
if s.commands != nil {
copy(new_commands, s.commands)
}
s.commands_ = new_commands
s.commands = new_commands
}
}
@ -832,32 +826,32 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
literal_context_mode = chooseContextMode(&s.params, data, uint(wrapPosition(s.last_flush_pos_)), uint(mask), uint(s.input_pos_-s.last_flush_pos_))
if s.num_commands_ != 0 && s.last_insert_len_ == 0 {
if len(s.commands) != 0 && s.last_insert_len_ == 0 {
extendLastCommand(s, &bytes, &wrapped_last_processed_pos)
}
if s.params.quality == zopflificationQuality {
assert(s.params.hasher.type_ == 10)
createZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_.(*h10), s.dist_cache_[:], &s.last_insert_len_, s.commands_[s.num_commands_:], &s.num_commands_, &s.num_literals_)
createZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_.(*h10), s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_)
} else if s.params.quality == hqZopflificationQuality {
assert(s.params.hasher.type_ == 10)
createHqZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, s.commands_[s.num_commands_:], &s.num_commands_, &s.num_literals_)
createHqZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_)
} else {
createBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, s.commands_[s.num_commands_:], &s.num_commands_, &s.num_literals_)
createBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_)
}
{
var max_length uint = maxMetablockSize(&s.params)
var max_literals uint = max_length / 8
var max_commands uint = max_length / 8
max_commands := int(max_length / 8)
var processed_bytes uint = uint(s.input_pos_ - s.last_flush_pos_)
var next_input_fits_metablock bool = (processed_bytes+inputBlockSize(s) <= max_length)
var should_flush bool = (s.params.quality < minQualityForBlockSplit && s.num_literals_+s.num_commands_ >= maxNumDelayedSymbols)
var should_flush bool = (s.params.quality < minQualityForBlockSplit && s.num_literals_+uint(len(s.commands)) >= maxNumDelayedSymbols)
/* If maximal possible additional block doesn't fit metablock, flush now. */
/* TODO: Postpone decision until next block arrives? */
/* If block splitting is not used, then flush as soon as there is some
amount of commands / literals produced. */
if !is_last && !force_flush && !should_flush && next_input_fits_metablock && s.num_literals_ < max_literals && s.num_commands_ < max_commands {
if !is_last && !force_flush && !should_flush && next_input_fits_metablock && s.num_literals_ < max_literals && len(s.commands) < max_commands {
/* Merge with next input block. Everything will happen later. */
if updateLastProcessedPos(s) {
hasherReset(s.hasher_)
@ -869,8 +863,7 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
/* Create the last insert-only command. */
if s.last_insert_len_ > 0 {
initInsertCommand(&s.commands_[s.num_commands_], s.last_insert_len_)
s.num_commands_++
s.commands = append(s.commands, makeInsertCommand(s.last_insert_len_))
s.num_literals_ += s.last_insert_len_
s.last_insert_len_ = 0
}
@ -890,7 +883,7 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
var storage_ix uint = uint(s.last_bytes_bits_)
storage[0] = byte(s.last_bytes_)
storage[1] = byte(s.last_bytes_ >> 8)
writeMetaBlockInternal(data, uint(mask), s.last_flush_pos_, uint(metablock_size), is_last, literal_context_mode, &s.params, s.prev_byte_, s.prev_byte2_, s.num_literals_, s.num_commands_, s.commands_, s.saved_dist_cache_[:], s.dist_cache_[:], &storage_ix, storage)
writeMetaBlockInternal(data, uint(mask), s.last_flush_pos_, uint(metablock_size), is_last, literal_context_mode, &s.params, s.prev_byte_, s.prev_byte2_, s.num_literals_, s.commands, s.saved_dist_cache_[:], s.dist_cache_[:], &storage_ix, storage)
s.last_bytes_ = uint16(storage[storage_ix>>3])
s.last_bytes_bits_ = byte(storage_ix & 7)
s.last_flush_pos_ = s.input_pos_
@ -906,7 +899,7 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
s.prev_byte2_ = data[uint32(s.last_flush_pos_-2)&mask]
}
s.num_commands_ = 0
s.commands = s.commands[:0]
s.num_literals_ = 0
/* Save the state of the distance cache in case we need to restore it for

5
histogram.go
View File

@ -180,17 +180,16 @@ func blockSplitIteratorNext(self *blockSplitIterator) {
self.length_--
}
func buildHistogramsWithContext(cmds []command, num_commands uint, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit, ringbuffer []byte, start_pos uint, mask uint, prev_byte byte, prev_byte2 byte, context_modes []int, literal_histograms []histogramLiteral, insert_and_copy_histograms []histogramCommand, copy_dist_histograms []histogramDistance) {
func buildHistogramsWithContext(cmds []command, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit, ringbuffer []byte, start_pos uint, mask uint, prev_byte byte, prev_byte2 byte, context_modes []int, literal_histograms []histogramLiteral, insert_and_copy_histograms []histogramCommand, copy_dist_histograms []histogramDistance) {
var pos uint = start_pos
var literal_it blockSplitIterator
var insert_and_copy_it blockSplitIterator
var dist_it blockSplitIterator
var i uint
initBlockSplitIterator(&literal_it, literal_split)
initBlockSplitIterator(&insert_and_copy_it, insert_and_copy_split)
initBlockSplitIterator(&dist_it, dist_split)
for i = 0; i < num_commands; i++ {
for i := range cmds {
var cmd *command = &cmds[i]
var j uint
blockSplitIteratorNext(&insert_and_copy_it)

43
metablock.go
View File

@ -84,14 +84,12 @@ func initDistanceParams(params *encoderParams, npostfix uint32, ndirect uint32)
dist_params.max_distance = uint(max_distance)
}
func recomputeDistancePrefixes(cmds []command, num_commands uint, orig_params *distanceParams, new_params *distanceParams) {
var i uint
func recomputeDistancePrefixes(cmds []command, orig_params *distanceParams, new_params *distanceParams) {
if orig_params.distance_postfix_bits == new_params.distance_postfix_bits && orig_params.num_direct_distance_codes == new_params.num_direct_distance_codes {
return
}
for i = 0; i < num_commands; i++ {
for i := range cmds {
var cmd *command = &cmds[i]
if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 {
prefixEncodeCopyDistance(uint(commandRestoreDistanceCode(cmd, orig_params)), uint(new_params.num_direct_distance_codes), uint(new_params.distance_postfix_bits), &cmd.dist_prefix_, &cmd.dist_extra_)
@ -99,8 +97,7 @@ func recomputeDistancePrefixes(cmds []command, num_commands uint, orig_params *d
}
}
func computeDistanceCost(cmds []command, num_commands uint, orig_params *distanceParams, new_params *distanceParams, cost *float64) bool {
var i uint
func computeDistanceCost(cmds []command, orig_params *distanceParams, new_params *distanceParams, cost *float64) bool {
var equal_params bool = false
var dist_prefix uint16
var dist_extra uint32
@ -112,8 +109,8 @@ func computeDistanceCost(cmds []command, num_commands uint, orig_params *distanc
equal_params = true
}
for i = 0; i < num_commands; i++ {
var cmd *command = &cmds[i]
for i := range cmds {
cmd := &cmds[i]
if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 {
if equal_params {
dist_prefix = cmd.dist_prefix_
@ -137,7 +134,7 @@ func computeDistanceCost(cmds []command, num_commands uint, orig_params *distanc
var buildMetaBlock_kMaxNumberOfHistograms uint = 256
func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParams, prev_byte byte, prev_byte2 byte, cmds []command, num_commands uint, literal_context_mode int, mb *metaBlockSplit) {
func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParams, prev_byte byte, prev_byte2 byte, cmds []command, literal_context_mode int, mb *metaBlockSplit) {
var distance_histograms []histogramDistance
var literal_histograms []histogramLiteral
var literal_context_modes []int = nil
@ -164,7 +161,7 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam
check_orig = false
}
skip = !computeDistanceCost(cmds, num_commands, &orig_params.dist, &new_params.dist, &dist_cost)
skip = !computeDistanceCost(cmds, &orig_params.dist, &new_params.dist, &dist_cost)
if skip || (dist_cost > best_dist_cost) {
break
}
@ -181,7 +178,7 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam
if check_orig {
var dist_cost float64
computeDistanceCost(cmds, num_commands, &orig_params.dist, &orig_params.dist, &dist_cost)
computeDistanceCost(cmds, &orig_params.dist, &orig_params.dist, &dist_cost)
if dist_cost < best_dist_cost {
/* NB: currently unused; uncomment when more param tuning is added. */
/* best_dist_cost = dist_cost; */
@ -189,9 +186,9 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam
}
}
recomputeDistancePrefixes(cmds, num_commands, &orig_params.dist, &params.dist)
recomputeDistancePrefixes(cmds, &orig_params.dist, &params.dist)
splitBlock(cmds, num_commands, ringbuffer, pos, mask, params, &mb.literal_split, &mb.command_split, &mb.distance_split)
splitBlock(cmds, ringbuffer, pos, mask, params, &mb.literal_split, &mb.command_split, &mb.distance_split)
if !params.disable_literal_context_modeling {
literal_context_multiplier = 1 << literalContextBits
@ -214,7 +211,7 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam
mb.command_histograms = make([]histogramCommand, (mb.command_histograms_size))
clearHistogramsCommand(mb.command_histograms, mb.command_histograms_size)
buildHistogramsWithContext(cmds, num_commands, &mb.literal_split, &mb.command_split, &mb.distance_split, ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_modes, literal_histograms, mb.command_histograms, distance_histograms)
buildHistogramsWithContext(cmds, &mb.literal_split, &mb.command_split, &mb.distance_split, ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_modes, literal_histograms, mb.command_histograms, distance_histograms)
literal_context_modes = nil
assert(mb.literal_context_map == nil)
@ -466,7 +463,7 @@ func mapStaticContexts(num_contexts uint, static_context_map []uint32, mb *metaB
}
}
func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, n_commands uint, mb *metaBlockSplit) {
func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, mb *metaBlockSplit) {
var lit_blocks struct {
plain blockSplitterLiteral
ctx contextBlockSplitter
@ -474,8 +471,7 @@ func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_b
var cmd_blocks blockSplitterCommand
var dist_blocks blockSplitterDistance
var num_literals uint = 0
var i uint
for i = 0; i < n_commands; i++ {
for i := range commands {
num_literals += uint(commands[i].insert_len_)
}
@ -485,11 +481,10 @@ func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_b
initContextBlockSplitter(&lit_blocks.ctx, 256, num_contexts, 512, 400.0, num_literals, &mb.literal_split, &mb.literal_histograms, &mb.literal_histograms_size)
}
initBlockSplitterCommand(&cmd_blocks, numCommandSymbols, 1024, 500.0, n_commands, &mb.command_split, &mb.command_histograms, &mb.command_histograms_size)
initBlockSplitterDistance(&dist_blocks, 64, 512, 100.0, n_commands, &mb.distance_split, &mb.distance_histograms, &mb.distance_histograms_size)
initBlockSplitterCommand(&cmd_blocks, numCommandSymbols, 1024, 500.0, uint(len(commands)), &mb.command_split, &mb.command_histograms, &mb.command_histograms_size)
initBlockSplitterDistance(&dist_blocks, 64, 512, 100.0, uint(len(commands)), &mb.distance_split, &mb.distance_histograms, &mb.distance_histograms_size)
for i = 0; i < n_commands; i++ {
var cmd command = commands[i]
for _, cmd := range commands {
var j uint
blockSplitterAddSymbolCommand(&cmd_blocks, uint(cmd.cmd_prefix_))
for j = uint(cmd.insert_len_); j != 0; j-- {
@ -530,11 +525,11 @@ func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_b
}
}
func buildMetaBlockGreedy(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, n_commands uint, mb *metaBlockSplit) {
func buildMetaBlockGreedy(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, mb *metaBlockSplit) {
if num_contexts == 1 {
buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, 1, nil, commands, n_commands, mb)
buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, 1, nil, commands, mb)
} else {
buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, num_contexts, static_context_map, commands, n_commands, mb)
buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, num_contexts, static_context_map, commands, mb)
}
}