forked from mirror/brotli
Use len and cap instead of num_commands_ and cmd_alloc_size_.
This commit is contained in:
parent
4b2775ea5e
commit
e2c5f2109f
|
@ -31,13 +31,8 @@ func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uin
|
||||||
return distance + numDistanceShortCodes - 1
|
return distance + numDistanceShortCodes - 1
|
||||||
}
|
}
|
||||||
|
|
||||||
/* "commands" points to the next output command to write to, "*num_commands" is
|
func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) {
|
||||||
initially the total amount of commands output by previous
|
|
||||||
CreateBackwardReferences calls, and must be incremented by the amount written
|
|
||||||
by this call. */
|
|
||||||
func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
|
|
||||||
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
|
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
|
||||||
var orig_commands []command = commands
|
|
||||||
var insert_length uint = *last_insert_len
|
var insert_length uint = *last_insert_len
|
||||||
var pos_end uint = position + num_bytes
|
var pos_end uint = position + num_bytes
|
||||||
var store_end uint
|
var store_end uint
|
||||||
|
@ -114,8 +109,7 @@ func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte,
|
||||||
hasher.PrepareDistanceCache(dist_cache)
|
hasher.PrepareDistanceCache(dist_cache)
|
||||||
}
|
}
|
||||||
|
|
||||||
initCommand(&commands[0], ¶ms.dist, insert_length, sr.len, sr.len_code_delta, distance_code)
|
*commands = append(*commands, makeCommand(¶ms.dist, insert_length, sr.len, sr.len_code_delta, distance_code))
|
||||||
commands = commands[1:]
|
|
||||||
}
|
}
|
||||||
|
|
||||||
*num_literals += insert_length
|
*num_literals += insert_length
|
||||||
|
@ -173,5 +167,4 @@ func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte,
|
||||||
|
|
||||||
insert_length += pos_end - position
|
insert_length += pos_end - position
|
||||||
*last_insert_len = insert_length
|
*last_insert_len = insert_length
|
||||||
*num_commands += uint(-cap(commands) + cap(orig_commands))
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -123,14 +123,13 @@ func setCost(histogram []uint32, histogram_size uint, literal_histogram bool, co
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint, commands []command, num_commands uint, last_insert_len uint) {
|
func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint, commands []command, last_insert_len uint) {
|
||||||
var histogram_literal [numLiteralSymbols]uint32
|
var histogram_literal [numLiteralSymbols]uint32
|
||||||
var histogram_cmd [numCommandSymbols]uint32
|
var histogram_cmd [numCommandSymbols]uint32
|
||||||
var histogram_dist [maxEffectiveDistanceAlphabetSize]uint32
|
var histogram_dist [maxEffectiveDistanceAlphabetSize]uint32
|
||||||
var cost_literal [numLiteralSymbols]float32
|
var cost_literal [numLiteralSymbols]float32
|
||||||
var pos uint = position - last_insert_len
|
var pos uint = position - last_insert_len
|
||||||
var min_cost_cmd float32 = kInfinity
|
var min_cost_cmd float32 = kInfinity
|
||||||
var i uint
|
|
||||||
var cost_cmd []float32 = self.cost_cmd_[:]
|
var cost_cmd []float32 = self.cost_cmd_[:]
|
||||||
var literal_costs []float32
|
var literal_costs []float32
|
||||||
|
|
||||||
|
@ -138,7 +137,7 @@ func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbu
|
||||||
histogram_cmd = [numCommandSymbols]uint32{}
|
histogram_cmd = [numCommandSymbols]uint32{}
|
||||||
histogram_dist = [maxEffectiveDistanceAlphabetSize]uint32{}
|
histogram_dist = [maxEffectiveDistanceAlphabetSize]uint32{}
|
||||||
|
|
||||||
for i = 0; i < num_commands; i++ {
|
for i := range commands {
|
||||||
var inslength uint = uint(commands[i].insert_len_)
|
var inslength uint = uint(commands[i].insert_len_)
|
||||||
var copylength uint = uint(commandCopyLen(&commands[i]))
|
var copylength uint = uint(commandCopyLen(&commands[i]))
|
||||||
var distcode uint = uint(commands[i].dist_prefix_) & 0x3FF
|
var distcode uint = uint(commands[i].dist_prefix_) & 0x3FF
|
||||||
|
@ -161,7 +160,7 @@ func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbu
|
||||||
setCost(histogram_cmd[:], numCommandSymbols, false, cost_cmd)
|
setCost(histogram_cmd[:], numCommandSymbols, false, cost_cmd)
|
||||||
setCost(histogram_dist[:], uint(self.distance_histogram_size), false, self.cost_dist_)
|
setCost(histogram_dist[:], uint(self.distance_histogram_size), false, self.cost_dist_)
|
||||||
|
|
||||||
for i = 0; i < numCommandSymbols; i++ {
|
for i := 0; i < numCommandSymbols; i++ {
|
||||||
min_cost_cmd = brotli_min_float(min_cost_cmd, cost_cmd[i])
|
min_cost_cmd = brotli_min_float(min_cost_cmd, cost_cmd[i])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -169,10 +168,10 @@ func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbu
|
||||||
{
|
{
|
||||||
literal_costs = self.literal_costs_
|
literal_costs = self.literal_costs_
|
||||||
var literal_carry float32 = 0.0
|
var literal_carry float32 = 0.0
|
||||||
var num_bytes uint = self.num_bytes_
|
num_bytes := int(self.num_bytes_)
|
||||||
literal_costs[0] = 0.0
|
literal_costs[0] = 0.0
|
||||||
for i = 0; i < num_bytes; i++ {
|
for i := 0; i < num_bytes; i++ {
|
||||||
literal_carry += cost_literal[ringbuffer[(position+i)&ringbuffer_mask]]
|
literal_carry += cost_literal[ringbuffer[(position+uint(i))&ringbuffer_mask]]
|
||||||
literal_costs[i+1] = literal_costs[i] + literal_carry
|
literal_costs[i+1] = literal_costs[i] + literal_carry
|
||||||
literal_carry -= literal_costs[i+1] - literal_costs[i]
|
literal_carry -= literal_costs[i+1] - literal_costs[i]
|
||||||
}
|
}
|
||||||
|
@ -532,7 +531,7 @@ func computeShortestPathFromNodes(num_bytes uint, nodes []zopfliNode) uint {
|
||||||
}
|
}
|
||||||
|
|
||||||
/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
|
/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
|
||||||
func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands []command, num_literals *uint) {
|
func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands *[]command, num_literals *uint) {
|
||||||
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
|
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
|
||||||
var pos uint = 0
|
var pos uint = 0
|
||||||
var offset uint32 = nodes[0].u.next
|
var offset uint32 = nodes[0].u.next
|
||||||
|
@ -554,7 +553,7 @@ func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode,
|
||||||
var max_distance uint = brotli_min_size_t(block_start+pos, max_backward_limit)
|
var max_distance uint = brotli_min_size_t(block_start+pos, max_backward_limit)
|
||||||
var is_dictionary bool = (distance > max_distance+gap)
|
var is_dictionary bool = (distance > max_distance+gap)
|
||||||
var dist_code uint = uint(zopfliNodeDistanceCode(next))
|
var dist_code uint = uint(zopfliNodeDistanceCode(next))
|
||||||
initCommand(&commands[i], ¶ms.dist, insert_length, copy_length, int(len_code)-int(copy_length), dist_code)
|
*commands = append(*commands, makeCommand(¶ms.dist, insert_length, copy_length, int(len_code)-int(copy_length), dist_code))
|
||||||
|
|
||||||
if !is_dictionary && dist_code > 0 {
|
if !is_dictionary && dist_code > 0 {
|
||||||
dist_cache[3] = dist_cache[2]
|
dist_cache[3] = dist_cache[2]
|
||||||
|
@ -681,16 +680,16 @@ func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte,
|
||||||
return computeShortestPathFromNodes(num_bytes, nodes)
|
return computeShortestPathFromNodes(num_bytes, nodes)
|
||||||
}
|
}
|
||||||
|
|
||||||
func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
|
func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) {
|
||||||
var nodes []zopfliNode
|
var nodes []zopfliNode
|
||||||
nodes = make([]zopfliNode, (num_bytes + 1))
|
nodes = make([]zopfliNode, (num_bytes + 1))
|
||||||
initZopfliNodes(nodes, num_bytes+1)
|
initZopfliNodes(nodes, num_bytes+1)
|
||||||
*num_commands += zopfliComputeShortestPath(num_bytes, position, ringbuffer, ringbuffer_mask, params, dist_cache, hasher, nodes)
|
zopfliComputeShortestPath(num_bytes, position, ringbuffer, ringbuffer_mask, params, dist_cache, hasher, nodes)
|
||||||
zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals)
|
zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals)
|
||||||
nodes = nil
|
nodes = nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
|
func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands *[]command, num_literals *uint) {
|
||||||
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
|
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
|
||||||
var num_matches []uint32 = make([]uint32, num_bytes)
|
var num_matches []uint32 = make([]uint32, num_bytes)
|
||||||
var matches_size uint = 4 * num_bytes
|
var matches_size uint = 4 * num_bytes
|
||||||
|
@ -705,7 +704,7 @@ func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer
|
||||||
var orig_num_literals uint
|
var orig_num_literals uint
|
||||||
var orig_last_insert_len uint
|
var orig_last_insert_len uint
|
||||||
var orig_dist_cache [4]int
|
var orig_dist_cache [4]int
|
||||||
var orig_num_commands uint
|
var orig_num_commands int
|
||||||
var model zopfliCostModel
|
var model zopfliCostModel
|
||||||
var nodes []zopfliNode
|
var nodes []zopfliNode
|
||||||
var matches []backwardMatch = make([]backwardMatch, matches_size)
|
var matches []backwardMatch = make([]backwardMatch, matches_size)
|
||||||
|
@ -771,7 +770,7 @@ func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer
|
||||||
orig_num_literals = *num_literals
|
orig_num_literals = *num_literals
|
||||||
orig_last_insert_len = *last_insert_len
|
orig_last_insert_len = *last_insert_len
|
||||||
copy(orig_dist_cache[:], dist_cache[:4])
|
copy(orig_dist_cache[:], dist_cache[:4])
|
||||||
orig_num_commands = *num_commands
|
orig_num_commands = len(*commands)
|
||||||
nodes = make([]zopfliNode, (num_bytes + 1))
|
nodes = make([]zopfliNode, (num_bytes + 1))
|
||||||
initZopfliCostModel(&model, ¶ms.dist, num_bytes)
|
initZopfliCostModel(&model, ¶ms.dist, num_bytes)
|
||||||
for i = 0; i < 2; i++ {
|
for i = 0; i < 2; i++ {
|
||||||
|
@ -779,14 +778,14 @@ func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer
|
||||||
if i == 0 {
|
if i == 0 {
|
||||||
zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask)
|
zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask)
|
||||||
} else {
|
} else {
|
||||||
zopfliCostModelSetFromCommands(&model, position, ringbuffer, ringbuffer_mask, commands, *num_commands-orig_num_commands, orig_last_insert_len)
|
zopfliCostModelSetFromCommands(&model, position, ringbuffer, ringbuffer_mask, (*commands)[orig_num_commands:], orig_last_insert_len)
|
||||||
}
|
}
|
||||||
|
|
||||||
*num_commands = orig_num_commands
|
*commands = (*commands)[:orig_num_commands]
|
||||||
*num_literals = orig_num_literals
|
*num_literals = orig_num_literals
|
||||||
*last_insert_len = orig_last_insert_len
|
*last_insert_len = orig_last_insert_len
|
||||||
copy(dist_cache, orig_dist_cache[:4])
|
copy(dist_cache, orig_dist_cache[:4])
|
||||||
*num_commands += zopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, nodes)
|
zopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, nodes)
|
||||||
zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals)
|
zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -33,23 +33,21 @@ const (
|
||||||
kMinItersForRefining uint = 100
|
kMinItersForRefining uint = 100
|
||||||
)
|
)
|
||||||
|
|
||||||
func countLiterals(cmds []command, num_commands uint) uint {
|
func countLiterals(cmds []command) uint {
|
||||||
var total_length uint = 0
|
var total_length uint = 0
|
||||||
/* Count how many we have. */
|
/* Count how many we have. */
|
||||||
|
|
||||||
var i uint
|
for i := range cmds {
|
||||||
for i = 0; i < num_commands; i++ {
|
|
||||||
total_length += uint(cmds[i].insert_len_)
|
total_length += uint(cmds[i].insert_len_)
|
||||||
}
|
}
|
||||||
|
|
||||||
return total_length
|
return total_length
|
||||||
}
|
}
|
||||||
|
|
||||||
func copyLiteralsToByteArray(cmds []command, num_commands uint, data []byte, offset uint, mask uint, literals []byte) {
|
func copyLiteralsToByteArray(cmds []command, data []byte, offset uint, mask uint, literals []byte) {
|
||||||
var pos uint = 0
|
var pos uint = 0
|
||||||
var from_pos uint = offset & mask
|
var from_pos uint = offset & mask
|
||||||
var i uint
|
for i := range cmds {
|
||||||
for i = 0; i < num_commands; i++ {
|
|
||||||
var insert_len uint = uint(cmds[i].insert_len_)
|
var insert_len uint = uint(cmds[i].insert_len_)
|
||||||
if from_pos+insert_len > mask {
|
if from_pos+insert_len > mask {
|
||||||
var head_size uint = mask + 1 - from_pos
|
var head_size uint = mask + 1 - from_pos
|
||||||
|
@ -101,13 +99,13 @@ func destroyBlockSplit(self *blockSplit) {
|
||||||
self.lengths = nil
|
self.lengths = nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func splitBlock(cmds []command, num_commands uint, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) {
|
func splitBlock(cmds []command, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) {
|
||||||
{
|
{
|
||||||
var literals_count uint = countLiterals(cmds, num_commands)
|
var literals_count uint = countLiterals(cmds)
|
||||||
var literals []byte = make([]byte, literals_count)
|
var literals []byte = make([]byte, literals_count)
|
||||||
|
|
||||||
/* Create a continuous array of literals. */
|
/* Create a continuous array of literals. */
|
||||||
copyLiteralsToByteArray(cmds, num_commands, data, pos, mask, literals)
|
copyLiteralsToByteArray(cmds, data, pos, mask, literals)
|
||||||
|
|
||||||
/* Create the block split on the array of literals.
|
/* Create the block split on the array of literals.
|
||||||
Literal histograms have alphabet size 256. */
|
Literal histograms have alphabet size 256. */
|
||||||
|
@ -116,28 +114,26 @@ func splitBlock(cmds []command, num_commands uint, data []byte, pos uint, mask u
|
||||||
literals = nil
|
literals = nil
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
var insert_and_copy_codes []uint16 = make([]uint16, num_commands)
|
var insert_and_copy_codes []uint16 = make([]uint16, len(cmds))
|
||||||
/* Compute prefix codes for commands. */
|
/* Compute prefix codes for commands. */
|
||||||
|
|
||||||
var i uint
|
for i := range cmds {
|
||||||
for i = 0; i < num_commands; i++ {
|
|
||||||
insert_and_copy_codes[i] = cmds[i].cmd_prefix_
|
insert_and_copy_codes[i] = cmds[i].cmd_prefix_
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Create the block split on the array of command prefixes. */
|
/* Create the block split on the array of command prefixes. */
|
||||||
splitByteVectorCommand(insert_and_copy_codes, num_commands, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params, insert_and_copy_split)
|
splitByteVectorCommand(insert_and_copy_codes, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params, insert_and_copy_split)
|
||||||
|
|
||||||
/* TODO: reuse for distances? */
|
/* TODO: reuse for distances? */
|
||||||
|
|
||||||
insert_and_copy_codes = nil
|
insert_and_copy_codes = nil
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
var distance_prefixes []uint16 = make([]uint16, num_commands)
|
var distance_prefixes []uint16 = make([]uint16, len(cmds))
|
||||||
var j uint = 0
|
var j uint = 0
|
||||||
/* Create a continuous array of distance prefixes. */
|
/* Create a continuous array of distance prefixes. */
|
||||||
|
|
||||||
var i uint
|
for i := range cmds {
|
||||||
for i = 0; i < num_commands; i++ {
|
|
||||||
var cmd *command = &cmds[i]
|
var cmd *command = &cmds[i]
|
||||||
if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 {
|
if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 {
|
||||||
distance_prefixes[j] = cmd.dist_prefix_ & 0x3FF
|
distance_prefixes[j] = cmd.dist_prefix_ & 0x3FF
|
||||||
|
|
|
@ -372,7 +372,8 @@ func clusterBlocksCommand(data []uint16, length uint, num_blocks uint, block_ids
|
||||||
histogram_symbols = nil
|
histogram_symbols = nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func splitByteVectorCommand(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
|
func splitByteVectorCommand(data []uint16, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) {
|
||||||
|
length := uint(len(data))
|
||||||
var data_size uint = histogramDataSizeCommand()
|
var data_size uint = histogramDataSizeCommand()
|
||||||
var num_histograms uint = length/literals_per_histogram + 1
|
var num_histograms uint = length/literals_per_histogram + 1
|
||||||
var histograms []histogramCommand
|
var histograms []histogramCommand
|
||||||
|
|
|
@ -990,7 +990,7 @@ func jumpToByteBoundary(storage_ix *uint, storage []byte) {
|
||||||
storage[*storage_ix>>3] = 0
|
storage[*storage_ix>>3] = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_byte byte, prev_byte2 byte, is_last bool, params *encoderParams, literal_context_mode int, commands []command, n_commands uint, mb *metaBlockSplit, storage_ix *uint, storage []byte) {
|
func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_byte byte, prev_byte2 byte, is_last bool, params *encoderParams, literal_context_mode int, commands []command, mb *metaBlockSplit, storage_ix *uint, storage []byte) {
|
||||||
var pos uint = start_pos
|
var pos uint = start_pos
|
||||||
var i uint
|
var i uint
|
||||||
var num_distance_symbols uint32 = params.dist.alphabet_size
|
var num_distance_symbols uint32 = params.dist.alphabet_size
|
||||||
|
@ -1039,8 +1039,7 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b
|
||||||
buildAndStoreEntropyCodesDistance(&distance_enc, mb.distance_histograms, mb.distance_histograms_size, uint(num_distance_symbols), tree, storage_ix, storage)
|
buildAndStoreEntropyCodesDistance(&distance_enc, mb.distance_histograms, mb.distance_histograms_size, uint(num_distance_symbols), tree, storage_ix, storage)
|
||||||
tree = nil
|
tree = nil
|
||||||
|
|
||||||
for i = 0; i < n_commands; i++ {
|
for _, cmd := range commands {
|
||||||
var cmd command = commands[i]
|
|
||||||
var cmd_code uint = uint(cmd.cmd_prefix_)
|
var cmd_code uint = uint(cmd.cmd_prefix_)
|
||||||
storeSymbol(&command_enc, cmd_code, storage_ix, storage)
|
storeSymbol(&command_enc, cmd_code, storage_ix, storage)
|
||||||
storeCommandExtra(&cmd, storage_ix, storage)
|
storeCommandExtra(&cmd, storage_ix, storage)
|
||||||
|
@ -1090,11 +1089,9 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildHistograms(input []byte, start_pos uint, mask uint, commands []command, n_commands uint, lit_histo *histogramLiteral, cmd_histo *histogramCommand, dist_histo *histogramDistance) {
|
func buildHistograms(input []byte, start_pos uint, mask uint, commands []command, lit_histo *histogramLiteral, cmd_histo *histogramCommand, dist_histo *histogramDistance) {
|
||||||
var pos uint = start_pos
|
var pos uint = start_pos
|
||||||
var i uint
|
for _, cmd := range commands {
|
||||||
for i = 0; i < n_commands; i++ {
|
|
||||||
var cmd command = commands[i]
|
|
||||||
var j uint
|
var j uint
|
||||||
histogramAddCommand(cmd_histo, uint(cmd.cmd_prefix_))
|
histogramAddCommand(cmd_histo, uint(cmd.cmd_prefix_))
|
||||||
for j = uint(cmd.insert_len_); j != 0; j-- {
|
for j = uint(cmd.insert_len_); j != 0; j-- {
|
||||||
|
@ -1109,11 +1106,9 @@ func buildHistograms(input []byte, start_pos uint, mask uint, commands []command
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands []command, n_commands uint, lit_depth []byte, lit_bits []uint16, cmd_depth []byte, cmd_bits []uint16, dist_depth []byte, dist_bits []uint16, storage_ix *uint, storage []byte) {
|
func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands []command, lit_depth []byte, lit_bits []uint16, cmd_depth []byte, cmd_bits []uint16, dist_depth []byte, dist_bits []uint16, storage_ix *uint, storage []byte) {
|
||||||
var pos uint = start_pos
|
var pos uint = start_pos
|
||||||
var i uint
|
for _, cmd := range commands {
|
||||||
for i = 0; i < n_commands; i++ {
|
|
||||||
var cmd command = commands[i]
|
|
||||||
var cmd_code uint = uint(cmd.cmd_prefix_)
|
var cmd_code uint = uint(cmd.cmd_prefix_)
|
||||||
var j uint
|
var j uint
|
||||||
writeBits(uint(cmd_depth[cmd_code]), uint64(cmd_bits[cmd_code]), storage_ix, storage)
|
writeBits(uint(cmd_depth[cmd_code]), uint64(cmd_bits[cmd_code]), storage_ix, storage)
|
||||||
|
@ -1135,7 +1130,7 @@ func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, n_commands uint, storage_ix *uint, storage []byte) {
|
func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, storage_ix *uint, storage []byte) {
|
||||||
var lit_histo histogramLiteral
|
var lit_histo histogramLiteral
|
||||||
var cmd_histo histogramCommand
|
var cmd_histo histogramCommand
|
||||||
var dist_histo histogramDistance
|
var dist_histo histogramDistance
|
||||||
|
@ -1154,7 +1149,7 @@ func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint,
|
||||||
histogramClearCommand(&cmd_histo)
|
histogramClearCommand(&cmd_histo)
|
||||||
histogramClearDistance(&dist_histo)
|
histogramClearDistance(&dist_histo)
|
||||||
|
|
||||||
buildHistograms(input, start_pos, mask, commands, n_commands, &lit_histo, &cmd_histo, &dist_histo)
|
buildHistograms(input, start_pos, mask, commands, &lit_histo, &cmd_histo, &dist_histo)
|
||||||
|
|
||||||
writeBits(13, 0, storage_ix, storage)
|
writeBits(13, 0, storage_ix, storage)
|
||||||
|
|
||||||
|
@ -1163,13 +1158,13 @@ func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint,
|
||||||
buildAndStoreHuffmanTree(cmd_histo.data_[:], numCommandSymbols, numCommandSymbols, tree, cmd_depth[:], cmd_bits[:], storage_ix, storage)
|
buildAndStoreHuffmanTree(cmd_histo.data_[:], numCommandSymbols, numCommandSymbols, tree, cmd_depth[:], cmd_bits[:], storage_ix, storage)
|
||||||
buildAndStoreHuffmanTree(dist_histo.data_[:], maxSimpleDistanceAlphabetSize, uint(num_distance_symbols), tree, dist_depth[:], dist_bits[:], storage_ix, storage)
|
buildAndStoreHuffmanTree(dist_histo.data_[:], maxSimpleDistanceAlphabetSize, uint(num_distance_symbols), tree, dist_depth[:], dist_bits[:], storage_ix, storage)
|
||||||
tree = nil
|
tree = nil
|
||||||
storeDataWithHuffmanCodes(input, start_pos, mask, commands, n_commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage)
|
storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage)
|
||||||
if is_last {
|
if is_last {
|
||||||
jumpToByteBoundary(storage_ix, storage)
|
jumpToByteBoundary(storage_ix, storage)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, n_commands uint, storage_ix *uint, storage []byte) {
|
func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, storage_ix *uint, storage []byte) {
|
||||||
var num_distance_symbols uint32 = params.dist.alphabet_size
|
var num_distance_symbols uint32 = params.dist.alphabet_size
|
||||||
var distance_alphabet_bits uint32 = log2FloorNonZero(uint(num_distance_symbols-1)) + 1
|
var distance_alphabet_bits uint32 = log2FloorNonZero(uint(num_distance_symbols-1)) + 1
|
||||||
|
|
||||||
|
@ -1177,15 +1172,13 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is
|
||||||
|
|
||||||
writeBits(13, 0, storage_ix, storage)
|
writeBits(13, 0, storage_ix, storage)
|
||||||
|
|
||||||
if n_commands <= 128 {
|
if len(commands) <= 128 {
|
||||||
var histogram = [numLiteralSymbols]uint32{0}
|
var histogram = [numLiteralSymbols]uint32{0}
|
||||||
var pos uint = start_pos
|
var pos uint = start_pos
|
||||||
var num_literals uint = 0
|
var num_literals uint = 0
|
||||||
var i uint
|
|
||||||
var lit_depth [numLiteralSymbols]byte
|
var lit_depth [numLiteralSymbols]byte
|
||||||
var lit_bits [numLiteralSymbols]uint16
|
var lit_bits [numLiteralSymbols]uint16
|
||||||
for i = 0; i < n_commands; i++ {
|
for _, cmd := range commands {
|
||||||
var cmd command = commands[i]
|
|
||||||
var j uint
|
var j uint
|
||||||
for j = uint(cmd.insert_len_); j != 0; j-- {
|
for j = uint(cmd.insert_len_); j != 0; j-- {
|
||||||
histogram[input[pos&mask]]++
|
histogram[input[pos&mask]]++
|
||||||
|
@ -1201,7 +1194,7 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is
|
||||||
|
|
||||||
storeStaticCommandHuffmanTree(storage_ix, storage)
|
storeStaticCommandHuffmanTree(storage_ix, storage)
|
||||||
storeStaticDistanceHuffmanTree(storage_ix, storage)
|
storeStaticDistanceHuffmanTree(storage_ix, storage)
|
||||||
storeDataWithHuffmanCodes(input, start_pos, mask, commands, n_commands, lit_depth[:], lit_bits[:], kStaticCommandCodeDepth[:], kStaticCommandCodeBits[:], kStaticDistanceCodeDepth[:], kStaticDistanceCodeBits[:], storage_ix, storage)
|
storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], kStaticCommandCodeDepth[:], kStaticCommandCodeBits[:], kStaticDistanceCodeDepth[:], kStaticDistanceCodeBits[:], storage_ix, storage)
|
||||||
} else {
|
} else {
|
||||||
var lit_histo histogramLiteral
|
var lit_histo histogramLiteral
|
||||||
var cmd_histo histogramCommand
|
var cmd_histo histogramCommand
|
||||||
|
@ -1215,7 +1208,7 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is
|
||||||
histogramClearLiteral(&lit_histo)
|
histogramClearLiteral(&lit_histo)
|
||||||
histogramClearCommand(&cmd_histo)
|
histogramClearCommand(&cmd_histo)
|
||||||
histogramClearDistance(&dist_histo)
|
histogramClearDistance(&dist_histo)
|
||||||
buildHistograms(input, start_pos, mask, commands, n_commands, &lit_histo, &cmd_histo, &dist_histo)
|
buildHistograms(input, start_pos, mask, commands, &lit_histo, &cmd_histo, &dist_histo)
|
||||||
buildAndStoreHuffmanTreeFast(lit_histo.data_[:], lit_histo.total_count_, /* max_bits = */
|
buildAndStoreHuffmanTreeFast(lit_histo.data_[:], lit_histo.total_count_, /* max_bits = */
|
||||||
8, lit_depth[:], lit_bits[:], storage_ix, storage)
|
8, lit_depth[:], lit_bits[:], storage_ix, storage)
|
||||||
|
|
||||||
|
@ -1225,7 +1218,7 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is
|
||||||
buildAndStoreHuffmanTreeFast(dist_histo.data_[:], dist_histo.total_count_, /* max_bits = */
|
buildAndStoreHuffmanTreeFast(dist_histo.data_[:], dist_histo.total_count_, /* max_bits = */
|
||||||
uint(distance_alphabet_bits), dist_depth[:], dist_bits[:], storage_ix, storage)
|
uint(distance_alphabet_bits), dist_depth[:], dist_bits[:], storage_ix, storage)
|
||||||
|
|
||||||
storeDataWithHuffmanCodes(input, start_pos, mask, commands, n_commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage)
|
storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage)
|
||||||
}
|
}
|
||||||
|
|
||||||
if is_last {
|
if is_last {
|
||||||
|
|
24
command.go
24
command.go
|
@ -194,26 +194,28 @@ type command struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
/* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */
|
/* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */
|
||||||
func initCommand(self *command, dist *distanceParams, insertlen uint, copylen uint, copylen_code_delta int, distance_code uint) {
|
func makeCommand(dist *distanceParams, insertlen uint, copylen uint, copylen_code_delta int, distance_code uint) (cmd command) {
|
||||||
/* Don't rely on signed int representation, use honest casts. */
|
/* Don't rely on signed int representation, use honest casts. */
|
||||||
var delta uint32 = uint32(byte(int8(copylen_code_delta)))
|
var delta uint32 = uint32(byte(int8(copylen_code_delta)))
|
||||||
self.insert_len_ = uint32(insertlen)
|
cmd.insert_len_ = uint32(insertlen)
|
||||||
self.copy_len_ = uint32(uint32(copylen) | delta<<25)
|
cmd.copy_len_ = uint32(uint32(copylen) | delta<<25)
|
||||||
|
|
||||||
/* The distance prefix and extra bits are stored in this Command as if
|
/* The distance prefix and extra bits are stored in this Command as if
|
||||||
npostfix and ndirect were 0, they are only recomputed later after the
|
npostfix and ndirect were 0, they are only recomputed later after the
|
||||||
clustering if needed. */
|
clustering if needed. */
|
||||||
prefixEncodeCopyDistance(distance_code, uint(dist.num_direct_distance_codes), uint(dist.distance_postfix_bits), &self.dist_prefix_, &self.dist_extra_)
|
prefixEncodeCopyDistance(distance_code, uint(dist.num_direct_distance_codes), uint(dist.distance_postfix_bits), &cmd.dist_prefix_, &cmd.dist_extra_)
|
||||||
|
getLengthCode(insertlen, uint(int(copylen)+copylen_code_delta), (cmd.dist_prefix_&0x3FF == 0), &cmd.cmd_prefix_)
|
||||||
|
|
||||||
getLengthCode(insertlen, uint(int(copylen)+copylen_code_delta), (self.dist_prefix_&0x3FF == 0), &self.cmd_prefix_)
|
return cmd
|
||||||
}
|
}
|
||||||
|
|
||||||
func initInsertCommand(self *command, insertlen uint) {
|
func makeInsertCommand(insertlen uint) (cmd command) {
|
||||||
self.insert_len_ = uint32(insertlen)
|
cmd.insert_len_ = uint32(insertlen)
|
||||||
self.copy_len_ = 4 << 25
|
cmd.copy_len_ = 4 << 25
|
||||||
self.dist_extra_ = 0
|
cmd.dist_extra_ = 0
|
||||||
self.dist_prefix_ = numDistanceShortCodes
|
cmd.dist_prefix_ = numDistanceShortCodes
|
||||||
getLengthCode(insertlen, 4, false, &self.cmd_prefix_)
|
getLengthCode(insertlen, 4, false, &cmd.cmd_prefix_)
|
||||||
|
return cmd
|
||||||
}
|
}
|
||||||
|
|
||||||
func commandRestoreDistanceCode(self *command, dist *distanceParams) uint32 {
|
func commandRestoreDistanceCode(self *command, dist *distanceParams) uint32 {
|
||||||
|
|
61
encode.go
61
encode.go
|
@ -80,9 +80,7 @@ type Writer struct {
|
||||||
hasher_ hasherHandle
|
hasher_ hasherHandle
|
||||||
input_pos_ uint64
|
input_pos_ uint64
|
||||||
ringbuffer_ ringBuffer
|
ringbuffer_ ringBuffer
|
||||||
cmd_alloc_size_ uint
|
commands []command
|
||||||
commands_ []command
|
|
||||||
num_commands_ uint
|
|
||||||
num_literals_ uint
|
num_literals_ uint
|
||||||
last_insert_len_ uint
|
last_insert_len_ uint
|
||||||
last_flush_pos_ uint64
|
last_flush_pos_ uint64
|
||||||
|
@ -434,7 +432,7 @@ func chooseContextMode(params *encoderParams, data []byte, pos uint, mask uint,
|
||||||
return contextUTF8
|
return contextUTF8
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes uint, is_last bool, literal_context_mode int, params *encoderParams, prev_byte byte, prev_byte2 byte, num_literals uint, num_commands uint, commands []command, saved_dist_cache []int, dist_cache []int, storage_ix *uint, storage []byte) {
|
func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes uint, is_last bool, literal_context_mode int, params *encoderParams, prev_byte byte, prev_byte2 byte, num_literals uint, commands []command, saved_dist_cache []int, dist_cache []int, storage_ix *uint, storage []byte) {
|
||||||
var wrapped_last_flush_pos uint32 = wrapPosition(last_flush_pos)
|
var wrapped_last_flush_pos uint32 = wrapPosition(last_flush_pos)
|
||||||
var last_bytes uint16
|
var last_bytes uint16
|
||||||
var last_bytes_bits byte
|
var last_bytes_bits byte
|
||||||
|
@ -449,7 +447,7 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if !shouldCompress_encode(data, mask, last_flush_pos, bytes, num_literals, num_commands) {
|
if !shouldCompress_encode(data, mask, last_flush_pos, bytes, num_literals, uint(len(commands))) {
|
||||||
/* Restore the distance cache, as its last update by
|
/* Restore the distance cache, as its last update by
|
||||||
CreateBackwardReferences is now unused. */
|
CreateBackwardReferences is now unused. */
|
||||||
copy(dist_cache, saved_dist_cache[:4])
|
copy(dist_cache, saved_dist_cache[:4])
|
||||||
|
@ -462,9 +460,9 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes
|
||||||
last_bytes = uint16(storage[1])<<8 | uint16(storage[0])
|
last_bytes = uint16(storage[1])<<8 | uint16(storage[0])
|
||||||
last_bytes_bits = byte(*storage_ix)
|
last_bytes_bits = byte(*storage_ix)
|
||||||
if params.quality <= maxQualityForStaticEntropyCodes {
|
if params.quality <= maxQualityForStaticEntropyCodes {
|
||||||
storeMetaBlockFast(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, num_commands, storage_ix, storage)
|
storeMetaBlockFast(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, storage_ix, storage)
|
||||||
} else if params.quality < minQualityForBlockSplit {
|
} else if params.quality < minQualityForBlockSplit {
|
||||||
storeMetaBlockTrivial(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, num_commands, storage_ix, storage)
|
storeMetaBlockTrivial(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, storage_ix, storage)
|
||||||
} else {
|
} else {
|
||||||
var mb metaBlockSplit
|
var mb metaBlockSplit
|
||||||
initMetaBlockSplit(&mb)
|
initMetaBlockSplit(&mb)
|
||||||
|
@ -475,9 +473,9 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes
|
||||||
decideOverLiteralContextModeling(data, uint(wrapped_last_flush_pos), bytes, mask, params.quality, params.size_hint, &num_literal_contexts, &literal_context_map)
|
decideOverLiteralContextModeling(data, uint(wrapped_last_flush_pos), bytes, mask, params.quality, params.size_hint, &num_literal_contexts, &literal_context_map)
|
||||||
}
|
}
|
||||||
|
|
||||||
buildMetaBlockGreedy(data, uint(wrapped_last_flush_pos), mask, prev_byte, prev_byte2, literal_context_lut, num_literal_contexts, literal_context_map, commands, num_commands, &mb)
|
buildMetaBlockGreedy(data, uint(wrapped_last_flush_pos), mask, prev_byte, prev_byte2, literal_context_lut, num_literal_contexts, literal_context_map, commands, &mb)
|
||||||
} else {
|
} else {
|
||||||
buildMetaBlock(data, uint(wrapped_last_flush_pos), mask, &block_params, prev_byte, prev_byte2, commands, num_commands, literal_context_mode, &mb)
|
buildMetaBlock(data, uint(wrapped_last_flush_pos), mask, &block_params, prev_byte, prev_byte2, commands, literal_context_mode, &mb)
|
||||||
}
|
}
|
||||||
|
|
||||||
if params.quality >= minQualityForOptimizeHistograms {
|
if params.quality >= minQualityForOptimizeHistograms {
|
||||||
|
@ -492,7 +490,7 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes
|
||||||
optimizeHistograms(num_effective_dist_codes, &mb)
|
optimizeHistograms(num_effective_dist_codes, &mb)
|
||||||
}
|
}
|
||||||
|
|
||||||
storeMetaBlock(data, uint(wrapped_last_flush_pos), bytes, mask, prev_byte, prev_byte2, is_last, &block_params, literal_context_mode, commands, num_commands, &mb, storage_ix, storage)
|
storeMetaBlock(data, uint(wrapped_last_flush_pos), bytes, mask, prev_byte, prev_byte2, is_last, &block_params, literal_context_mode, commands, &mb, storage_ix, storage)
|
||||||
destroyMetaBlockSplit(&mb)
|
destroyMetaBlockSplit(&mb)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -613,7 +611,7 @@ func encoderInitParams(params *encoderParams) {
|
||||||
func encoderInitState(s *Writer) {
|
func encoderInitState(s *Writer) {
|
||||||
encoderInitParams(&s.params)
|
encoderInitParams(&s.params)
|
||||||
s.input_pos_ = 0
|
s.input_pos_ = 0
|
||||||
s.num_commands_ = 0
|
s.commands = s.commands[:0]
|
||||||
s.num_literals_ = 0
|
s.num_literals_ = 0
|
||||||
s.last_insert_len_ = 0
|
s.last_insert_len_ = 0
|
||||||
s.last_flush_pos_ = 0
|
s.last_flush_pos_ = 0
|
||||||
|
@ -712,7 +710,7 @@ func updateLastProcessedPos(s *Writer) bool {
|
||||||
}
|
}
|
||||||
|
|
||||||
func extendLastCommand(s *Writer, bytes *uint32, wrapped_last_processed_pos *uint32) {
|
func extendLastCommand(s *Writer, bytes *uint32, wrapped_last_processed_pos *uint32) {
|
||||||
var last_command *command = &s.commands_[s.num_commands_-1]
|
var last_command *command = &s.commands[len(s.commands)-1]
|
||||||
var data []byte = s.ringbuffer_.buffer_
|
var data []byte = s.ringbuffer_.buffer_
|
||||||
var mask uint32 = s.ringbuffer_.mask_
|
var mask uint32 = s.ringbuffer_.mask_
|
||||||
var max_backward_distance uint64 = ((uint64(1)) << s.params.lgwin) - windowGap
|
var max_backward_distance uint64 = ((uint64(1)) << s.params.lgwin) - windowGap
|
||||||
|
@ -809,22 +807,18 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
/* Theoretical max number of commands is 1 per 2 bytes. */
|
/* Theoretical max number of commands is 1 per 2 bytes. */
|
||||||
var newsize uint = uint(uint32(s.num_commands_) + bytes/2 + 1)
|
newsize := len(s.commands) + int(bytes)/2 + 1
|
||||||
if newsize > s.cmd_alloc_size_ {
|
if newsize > cap(s.commands) {
|
||||||
var new_commands []command
|
|
||||||
|
|
||||||
/* Reserve a bit more memory to allow merging with a next block
|
/* Reserve a bit more memory to allow merging with a next block
|
||||||
without reallocation: that would impact speed. */
|
without reallocation: that would impact speed. */
|
||||||
newsize += uint((bytes / 4) + 16)
|
newsize += int(bytes/4) + 16
|
||||||
|
|
||||||
s.cmd_alloc_size_ = newsize
|
new_commands := make([]command, len(s.commands), newsize)
|
||||||
new_commands = make([]command, newsize)
|
if s.commands != nil {
|
||||||
if s.commands_ != nil {
|
copy(new_commands, s.commands)
|
||||||
copy(new_commands, s.commands_[:s.num_commands_])
|
|
||||||
s.commands_ = nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
s.commands_ = new_commands
|
s.commands = new_commands
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -832,32 +826,32 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
|
||||||
|
|
||||||
literal_context_mode = chooseContextMode(&s.params, data, uint(wrapPosition(s.last_flush_pos_)), uint(mask), uint(s.input_pos_-s.last_flush_pos_))
|
literal_context_mode = chooseContextMode(&s.params, data, uint(wrapPosition(s.last_flush_pos_)), uint(mask), uint(s.input_pos_-s.last_flush_pos_))
|
||||||
|
|
||||||
if s.num_commands_ != 0 && s.last_insert_len_ == 0 {
|
if len(s.commands) != 0 && s.last_insert_len_ == 0 {
|
||||||
extendLastCommand(s, &bytes, &wrapped_last_processed_pos)
|
extendLastCommand(s, &bytes, &wrapped_last_processed_pos)
|
||||||
}
|
}
|
||||||
|
|
||||||
if s.params.quality == zopflificationQuality {
|
if s.params.quality == zopflificationQuality {
|
||||||
assert(s.params.hasher.type_ == 10)
|
assert(s.params.hasher.type_ == 10)
|
||||||
createZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_.(*h10), s.dist_cache_[:], &s.last_insert_len_, s.commands_[s.num_commands_:], &s.num_commands_, &s.num_literals_)
|
createZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_.(*h10), s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_)
|
||||||
} else if s.params.quality == hqZopflificationQuality {
|
} else if s.params.quality == hqZopflificationQuality {
|
||||||
assert(s.params.hasher.type_ == 10)
|
assert(s.params.hasher.type_ == 10)
|
||||||
createHqZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, s.commands_[s.num_commands_:], &s.num_commands_, &s.num_literals_)
|
createHqZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_)
|
||||||
} else {
|
} else {
|
||||||
createBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, s.commands_[s.num_commands_:], &s.num_commands_, &s.num_literals_)
|
createBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, &s.commands, &s.num_literals_)
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
var max_length uint = maxMetablockSize(&s.params)
|
var max_length uint = maxMetablockSize(&s.params)
|
||||||
var max_literals uint = max_length / 8
|
var max_literals uint = max_length / 8
|
||||||
var max_commands uint = max_length / 8
|
max_commands := int(max_length / 8)
|
||||||
var processed_bytes uint = uint(s.input_pos_ - s.last_flush_pos_)
|
var processed_bytes uint = uint(s.input_pos_ - s.last_flush_pos_)
|
||||||
var next_input_fits_metablock bool = (processed_bytes+inputBlockSize(s) <= max_length)
|
var next_input_fits_metablock bool = (processed_bytes+inputBlockSize(s) <= max_length)
|
||||||
var should_flush bool = (s.params.quality < minQualityForBlockSplit && s.num_literals_+s.num_commands_ >= maxNumDelayedSymbols)
|
var should_flush bool = (s.params.quality < minQualityForBlockSplit && s.num_literals_+uint(len(s.commands)) >= maxNumDelayedSymbols)
|
||||||
/* If maximal possible additional block doesn't fit metablock, flush now. */
|
/* If maximal possible additional block doesn't fit metablock, flush now. */
|
||||||
/* TODO: Postpone decision until next block arrives? */
|
/* TODO: Postpone decision until next block arrives? */
|
||||||
|
|
||||||
/* If block splitting is not used, then flush as soon as there is some
|
/* If block splitting is not used, then flush as soon as there is some
|
||||||
amount of commands / literals produced. */
|
amount of commands / literals produced. */
|
||||||
if !is_last && !force_flush && !should_flush && next_input_fits_metablock && s.num_literals_ < max_literals && s.num_commands_ < max_commands {
|
if !is_last && !force_flush && !should_flush && next_input_fits_metablock && s.num_literals_ < max_literals && len(s.commands) < max_commands {
|
||||||
/* Merge with next input block. Everything will happen later. */
|
/* Merge with next input block. Everything will happen later. */
|
||||||
if updateLastProcessedPos(s) {
|
if updateLastProcessedPos(s) {
|
||||||
hasherReset(s.hasher_)
|
hasherReset(s.hasher_)
|
||||||
|
@ -869,8 +863,7 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
|
||||||
|
|
||||||
/* Create the last insert-only command. */
|
/* Create the last insert-only command. */
|
||||||
if s.last_insert_len_ > 0 {
|
if s.last_insert_len_ > 0 {
|
||||||
initInsertCommand(&s.commands_[s.num_commands_], s.last_insert_len_)
|
s.commands = append(s.commands, makeInsertCommand(s.last_insert_len_))
|
||||||
s.num_commands_++
|
|
||||||
s.num_literals_ += s.last_insert_len_
|
s.num_literals_ += s.last_insert_len_
|
||||||
s.last_insert_len_ = 0
|
s.last_insert_len_ = 0
|
||||||
}
|
}
|
||||||
|
@ -890,7 +883,7 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
|
||||||
var storage_ix uint = uint(s.last_bytes_bits_)
|
var storage_ix uint = uint(s.last_bytes_bits_)
|
||||||
storage[0] = byte(s.last_bytes_)
|
storage[0] = byte(s.last_bytes_)
|
||||||
storage[1] = byte(s.last_bytes_ >> 8)
|
storage[1] = byte(s.last_bytes_ >> 8)
|
||||||
writeMetaBlockInternal(data, uint(mask), s.last_flush_pos_, uint(metablock_size), is_last, literal_context_mode, &s.params, s.prev_byte_, s.prev_byte2_, s.num_literals_, s.num_commands_, s.commands_, s.saved_dist_cache_[:], s.dist_cache_[:], &storage_ix, storage)
|
writeMetaBlockInternal(data, uint(mask), s.last_flush_pos_, uint(metablock_size), is_last, literal_context_mode, &s.params, s.prev_byte_, s.prev_byte2_, s.num_literals_, s.commands, s.saved_dist_cache_[:], s.dist_cache_[:], &storage_ix, storage)
|
||||||
s.last_bytes_ = uint16(storage[storage_ix>>3])
|
s.last_bytes_ = uint16(storage[storage_ix>>3])
|
||||||
s.last_bytes_bits_ = byte(storage_ix & 7)
|
s.last_bytes_bits_ = byte(storage_ix & 7)
|
||||||
s.last_flush_pos_ = s.input_pos_
|
s.last_flush_pos_ = s.input_pos_
|
||||||
|
@ -906,7 +899,7 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
|
||||||
s.prev_byte2_ = data[uint32(s.last_flush_pos_-2)&mask]
|
s.prev_byte2_ = data[uint32(s.last_flush_pos_-2)&mask]
|
||||||
}
|
}
|
||||||
|
|
||||||
s.num_commands_ = 0
|
s.commands = s.commands[:0]
|
||||||
s.num_literals_ = 0
|
s.num_literals_ = 0
|
||||||
|
|
||||||
/* Save the state of the distance cache in case we need to restore it for
|
/* Save the state of the distance cache in case we need to restore it for
|
||||||
|
|
|
@ -180,17 +180,16 @@ func blockSplitIteratorNext(self *blockSplitIterator) {
|
||||||
self.length_--
|
self.length_--
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildHistogramsWithContext(cmds []command, num_commands uint, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit, ringbuffer []byte, start_pos uint, mask uint, prev_byte byte, prev_byte2 byte, context_modes []int, literal_histograms []histogramLiteral, insert_and_copy_histograms []histogramCommand, copy_dist_histograms []histogramDistance) {
|
func buildHistogramsWithContext(cmds []command, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit, ringbuffer []byte, start_pos uint, mask uint, prev_byte byte, prev_byte2 byte, context_modes []int, literal_histograms []histogramLiteral, insert_and_copy_histograms []histogramCommand, copy_dist_histograms []histogramDistance) {
|
||||||
var pos uint = start_pos
|
var pos uint = start_pos
|
||||||
var literal_it blockSplitIterator
|
var literal_it blockSplitIterator
|
||||||
var insert_and_copy_it blockSplitIterator
|
var insert_and_copy_it blockSplitIterator
|
||||||
var dist_it blockSplitIterator
|
var dist_it blockSplitIterator
|
||||||
var i uint
|
|
||||||
|
|
||||||
initBlockSplitIterator(&literal_it, literal_split)
|
initBlockSplitIterator(&literal_it, literal_split)
|
||||||
initBlockSplitIterator(&insert_and_copy_it, insert_and_copy_split)
|
initBlockSplitIterator(&insert_and_copy_it, insert_and_copy_split)
|
||||||
initBlockSplitIterator(&dist_it, dist_split)
|
initBlockSplitIterator(&dist_it, dist_split)
|
||||||
for i = 0; i < num_commands; i++ {
|
for i := range cmds {
|
||||||
var cmd *command = &cmds[i]
|
var cmd *command = &cmds[i]
|
||||||
var j uint
|
var j uint
|
||||||
blockSplitIteratorNext(&insert_and_copy_it)
|
blockSplitIteratorNext(&insert_and_copy_it)
|
||||||
|
|
43
metablock.go
43
metablock.go
|
@ -84,14 +84,12 @@ func initDistanceParams(params *encoderParams, npostfix uint32, ndirect uint32)
|
||||||
dist_params.max_distance = uint(max_distance)
|
dist_params.max_distance = uint(max_distance)
|
||||||
}
|
}
|
||||||
|
|
||||||
func recomputeDistancePrefixes(cmds []command, num_commands uint, orig_params *distanceParams, new_params *distanceParams) {
|
func recomputeDistancePrefixes(cmds []command, orig_params *distanceParams, new_params *distanceParams) {
|
||||||
var i uint
|
|
||||||
|
|
||||||
if orig_params.distance_postfix_bits == new_params.distance_postfix_bits && orig_params.num_direct_distance_codes == new_params.num_direct_distance_codes {
|
if orig_params.distance_postfix_bits == new_params.distance_postfix_bits && orig_params.num_direct_distance_codes == new_params.num_direct_distance_codes {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
for i = 0; i < num_commands; i++ {
|
for i := range cmds {
|
||||||
var cmd *command = &cmds[i]
|
var cmd *command = &cmds[i]
|
||||||
if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 {
|
if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 {
|
||||||
prefixEncodeCopyDistance(uint(commandRestoreDistanceCode(cmd, orig_params)), uint(new_params.num_direct_distance_codes), uint(new_params.distance_postfix_bits), &cmd.dist_prefix_, &cmd.dist_extra_)
|
prefixEncodeCopyDistance(uint(commandRestoreDistanceCode(cmd, orig_params)), uint(new_params.num_direct_distance_codes), uint(new_params.distance_postfix_bits), &cmd.dist_prefix_, &cmd.dist_extra_)
|
||||||
|
@ -99,8 +97,7 @@ func recomputeDistancePrefixes(cmds []command, num_commands uint, orig_params *d
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func computeDistanceCost(cmds []command, num_commands uint, orig_params *distanceParams, new_params *distanceParams, cost *float64) bool {
|
func computeDistanceCost(cmds []command, orig_params *distanceParams, new_params *distanceParams, cost *float64) bool {
|
||||||
var i uint
|
|
||||||
var equal_params bool = false
|
var equal_params bool = false
|
||||||
var dist_prefix uint16
|
var dist_prefix uint16
|
||||||
var dist_extra uint32
|
var dist_extra uint32
|
||||||
|
@ -112,8 +109,8 @@ func computeDistanceCost(cmds []command, num_commands uint, orig_params *distanc
|
||||||
equal_params = true
|
equal_params = true
|
||||||
}
|
}
|
||||||
|
|
||||||
for i = 0; i < num_commands; i++ {
|
for i := range cmds {
|
||||||
var cmd *command = &cmds[i]
|
cmd := &cmds[i]
|
||||||
if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 {
|
if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 {
|
||||||
if equal_params {
|
if equal_params {
|
||||||
dist_prefix = cmd.dist_prefix_
|
dist_prefix = cmd.dist_prefix_
|
||||||
|
@ -137,7 +134,7 @@ func computeDistanceCost(cmds []command, num_commands uint, orig_params *distanc
|
||||||
|
|
||||||
var buildMetaBlock_kMaxNumberOfHistograms uint = 256
|
var buildMetaBlock_kMaxNumberOfHistograms uint = 256
|
||||||
|
|
||||||
func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParams, prev_byte byte, prev_byte2 byte, cmds []command, num_commands uint, literal_context_mode int, mb *metaBlockSplit) {
|
func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParams, prev_byte byte, prev_byte2 byte, cmds []command, literal_context_mode int, mb *metaBlockSplit) {
|
||||||
var distance_histograms []histogramDistance
|
var distance_histograms []histogramDistance
|
||||||
var literal_histograms []histogramLiteral
|
var literal_histograms []histogramLiteral
|
||||||
var literal_context_modes []int = nil
|
var literal_context_modes []int = nil
|
||||||
|
@ -164,7 +161,7 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam
|
||||||
check_orig = false
|
check_orig = false
|
||||||
}
|
}
|
||||||
|
|
||||||
skip = !computeDistanceCost(cmds, num_commands, &orig_params.dist, &new_params.dist, &dist_cost)
|
skip = !computeDistanceCost(cmds, &orig_params.dist, &new_params.dist, &dist_cost)
|
||||||
if skip || (dist_cost > best_dist_cost) {
|
if skip || (dist_cost > best_dist_cost) {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
@ -181,7 +178,7 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam
|
||||||
|
|
||||||
if check_orig {
|
if check_orig {
|
||||||
var dist_cost float64
|
var dist_cost float64
|
||||||
computeDistanceCost(cmds, num_commands, &orig_params.dist, &orig_params.dist, &dist_cost)
|
computeDistanceCost(cmds, &orig_params.dist, &orig_params.dist, &dist_cost)
|
||||||
if dist_cost < best_dist_cost {
|
if dist_cost < best_dist_cost {
|
||||||
/* NB: currently unused; uncomment when more param tuning is added. */
|
/* NB: currently unused; uncomment when more param tuning is added. */
|
||||||
/* best_dist_cost = dist_cost; */
|
/* best_dist_cost = dist_cost; */
|
||||||
|
@ -189,9 +186,9 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
recomputeDistancePrefixes(cmds, num_commands, &orig_params.dist, ¶ms.dist)
|
recomputeDistancePrefixes(cmds, &orig_params.dist, ¶ms.dist)
|
||||||
|
|
||||||
splitBlock(cmds, num_commands, ringbuffer, pos, mask, params, &mb.literal_split, &mb.command_split, &mb.distance_split)
|
splitBlock(cmds, ringbuffer, pos, mask, params, &mb.literal_split, &mb.command_split, &mb.distance_split)
|
||||||
|
|
||||||
if !params.disable_literal_context_modeling {
|
if !params.disable_literal_context_modeling {
|
||||||
literal_context_multiplier = 1 << literalContextBits
|
literal_context_multiplier = 1 << literalContextBits
|
||||||
|
@ -214,7 +211,7 @@ func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParam
|
||||||
mb.command_histograms = make([]histogramCommand, (mb.command_histograms_size))
|
mb.command_histograms = make([]histogramCommand, (mb.command_histograms_size))
|
||||||
clearHistogramsCommand(mb.command_histograms, mb.command_histograms_size)
|
clearHistogramsCommand(mb.command_histograms, mb.command_histograms_size)
|
||||||
|
|
||||||
buildHistogramsWithContext(cmds, num_commands, &mb.literal_split, &mb.command_split, &mb.distance_split, ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_modes, literal_histograms, mb.command_histograms, distance_histograms)
|
buildHistogramsWithContext(cmds, &mb.literal_split, &mb.command_split, &mb.distance_split, ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_modes, literal_histograms, mb.command_histograms, distance_histograms)
|
||||||
literal_context_modes = nil
|
literal_context_modes = nil
|
||||||
|
|
||||||
assert(mb.literal_context_map == nil)
|
assert(mb.literal_context_map == nil)
|
||||||
|
@ -466,7 +463,7 @@ func mapStaticContexts(num_contexts uint, static_context_map []uint32, mb *metaB
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, n_commands uint, mb *metaBlockSplit) {
|
func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, mb *metaBlockSplit) {
|
||||||
var lit_blocks struct {
|
var lit_blocks struct {
|
||||||
plain blockSplitterLiteral
|
plain blockSplitterLiteral
|
||||||
ctx contextBlockSplitter
|
ctx contextBlockSplitter
|
||||||
|
@ -474,8 +471,7 @@ func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_b
|
||||||
var cmd_blocks blockSplitterCommand
|
var cmd_blocks blockSplitterCommand
|
||||||
var dist_blocks blockSplitterDistance
|
var dist_blocks blockSplitterDistance
|
||||||
var num_literals uint = 0
|
var num_literals uint = 0
|
||||||
var i uint
|
for i := range commands {
|
||||||
for i = 0; i < n_commands; i++ {
|
|
||||||
num_literals += uint(commands[i].insert_len_)
|
num_literals += uint(commands[i].insert_len_)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -485,11 +481,10 @@ func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_b
|
||||||
initContextBlockSplitter(&lit_blocks.ctx, 256, num_contexts, 512, 400.0, num_literals, &mb.literal_split, &mb.literal_histograms, &mb.literal_histograms_size)
|
initContextBlockSplitter(&lit_blocks.ctx, 256, num_contexts, 512, 400.0, num_literals, &mb.literal_split, &mb.literal_histograms, &mb.literal_histograms_size)
|
||||||
}
|
}
|
||||||
|
|
||||||
initBlockSplitterCommand(&cmd_blocks, numCommandSymbols, 1024, 500.0, n_commands, &mb.command_split, &mb.command_histograms, &mb.command_histograms_size)
|
initBlockSplitterCommand(&cmd_blocks, numCommandSymbols, 1024, 500.0, uint(len(commands)), &mb.command_split, &mb.command_histograms, &mb.command_histograms_size)
|
||||||
initBlockSplitterDistance(&dist_blocks, 64, 512, 100.0, n_commands, &mb.distance_split, &mb.distance_histograms, &mb.distance_histograms_size)
|
initBlockSplitterDistance(&dist_blocks, 64, 512, 100.0, uint(len(commands)), &mb.distance_split, &mb.distance_histograms, &mb.distance_histograms_size)
|
||||||
|
|
||||||
for i = 0; i < n_commands; i++ {
|
for _, cmd := range commands {
|
||||||
var cmd command = commands[i]
|
|
||||||
var j uint
|
var j uint
|
||||||
blockSplitterAddSymbolCommand(&cmd_blocks, uint(cmd.cmd_prefix_))
|
blockSplitterAddSymbolCommand(&cmd_blocks, uint(cmd.cmd_prefix_))
|
||||||
for j = uint(cmd.insert_len_); j != 0; j-- {
|
for j = uint(cmd.insert_len_); j != 0; j-- {
|
||||||
|
@ -530,11 +525,11 @@ func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_b
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildMetaBlockGreedy(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, n_commands uint, mb *metaBlockSplit) {
|
func buildMetaBlockGreedy(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, mb *metaBlockSplit) {
|
||||||
if num_contexts == 1 {
|
if num_contexts == 1 {
|
||||||
buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, 1, nil, commands, n_commands, mb)
|
buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, 1, nil, commands, mb)
|
||||||
} else {
|
} else {
|
||||||
buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, num_contexts, static_context_map, commands, n_commands, mb)
|
buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, num_contexts, static_context_map, commands, mb)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue