package brotli /* Copyright 2014 Google Inc. All Rights Reserved. Distributed under MIT license. See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ /* Brotli bit stream functions to support the low level format. There are no compression algorithms here, just the right ordering of bits to match the specs. */ /* Copyright 2014 Google Inc. All Rights Reserved. Distributed under MIT license. See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ /* Functions to convert brotli-related data structures into the brotli bit stream. The functions here operate under assumption that there is enough space in the storage, i.e., there are no out-of-range checks anywhere. These functions do bit addressing into a byte array. The byte array is called "storage" and the index to the bit is called storage_ix in function arguments. */ /* Copyright 2015 Google Inc. All Rights Reserved. Distributed under MIT license. See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ /* Algorithms for distributing the literals and commands of a metablock between block types and contexts. */ type MetaBlockSplit struct { literal_split BlockSplit command_split BlockSplit distance_split BlockSplit literal_context_map []uint32 literal_context_map_size uint distance_context_map []uint32 distance_context_map_size uint literal_histograms []HistogramLiteral literal_histograms_size uint command_histograms []HistogramCommand command_histograms_size uint distance_histograms []HistogramDistance distance_histograms_size uint } func InitMetaBlockSplit(mb *MetaBlockSplit) { BrotliInitBlockSplit(&mb.literal_split) BrotliInitBlockSplit(&mb.command_split) BrotliInitBlockSplit(&mb.distance_split) mb.literal_context_map = nil mb.literal_context_map_size = 0 mb.distance_context_map = nil mb.distance_context_map_size = 0 mb.literal_histograms = nil mb.literal_histograms_size = 0 mb.command_histograms = nil mb.command_histograms_size = 0 mb.distance_histograms = nil mb.distance_histograms_size = 0 } func DestroyMetaBlockSplit(mb *MetaBlockSplit) { BrotliDestroyBlockSplit(&mb.literal_split) BrotliDestroyBlockSplit(&mb.command_split) BrotliDestroyBlockSplit(&mb.distance_split) mb.literal_context_map = nil mb.distance_context_map = nil mb.literal_histograms = nil mb.command_histograms = nil mb.distance_histograms = nil } /* Copyright 2015 Google Inc. All Rights Reserved. Distributed under MIT license. See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ /* Algorithms for distributing the literals and commands of a metablock between block types and contexts. */ func BrotliInitDistanceParams(params *BrotliEncoderParams, npostfix uint32, ndirect uint32) { var dist_params *BrotliDistanceParams = ¶ms.dist var alphabet_size uint32 var max_distance uint32 dist_params.distance_postfix_bits = npostfix dist_params.num_direct_distance_codes = ndirect alphabet_size = uint32(BROTLI_DISTANCE_ALPHABET_SIZE(uint(npostfix), uint(ndirect), BROTLI_MAX_DISTANCE_BITS)) max_distance = ndirect + (1 << (BROTLI_MAX_DISTANCE_BITS + npostfix + 2)) - (1 << (npostfix + 2)) if params.large_window { var bound = [BROTLI_MAX_NPOSTFIX + 1]uint32{0, 4, 12, 28} var postfix uint32 = 1 << npostfix alphabet_size = uint32(BROTLI_DISTANCE_ALPHABET_SIZE(uint(npostfix), uint(ndirect), BROTLI_LARGE_MAX_DISTANCE_BITS)) /* The maximum distance is set so that no distance symbol used can encode a distance larger than BROTLI_MAX_ALLOWED_DISTANCE with all its extra bits set. */ if ndirect < bound[npostfix] { max_distance = BROTLI_MAX_ALLOWED_DISTANCE - (bound[npostfix] - ndirect) } else if ndirect >= bound[npostfix]+postfix { max_distance = (3 << 29) - 4 + (ndirect - bound[npostfix]) } else { max_distance = BROTLI_MAX_ALLOWED_DISTANCE } } dist_params.alphabet_size = alphabet_size dist_params.max_distance = uint(max_distance) } func RecomputeDistancePrefixes(cmds []Command, num_commands uint, orig_params *BrotliDistanceParams, new_params *BrotliDistanceParams) { var i uint if orig_params.distance_postfix_bits == new_params.distance_postfix_bits && orig_params.num_direct_distance_codes == new_params.num_direct_distance_codes { return } for i = 0; i < num_commands; i++ { var cmd *Command = &cmds[i] if CommandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 { PrefixEncodeCopyDistance(uint(CommandRestoreDistanceCode(cmd, orig_params)), uint(new_params.num_direct_distance_codes), uint(new_params.distance_postfix_bits), &cmd.dist_prefix_, &cmd.dist_extra_) } } } func ComputeDistanceCost(cmds []Command, num_commands uint, orig_params *BrotliDistanceParams, new_params *BrotliDistanceParams, cost *float64) bool { var i uint var equal_params bool = false var dist_prefix uint16 var dist_extra uint32 var extra_bits float64 = 0.0 var histo HistogramDistance HistogramClearDistance(&histo) if orig_params.distance_postfix_bits == new_params.distance_postfix_bits && orig_params.num_direct_distance_codes == new_params.num_direct_distance_codes { equal_params = true } for i = 0; i < num_commands; i++ { var cmd *Command = &cmds[i] if CommandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 { if equal_params { dist_prefix = cmd.dist_prefix_ } else { var distance uint32 = CommandRestoreDistanceCode(cmd, orig_params) if distance > uint32(new_params.max_distance) { return false } PrefixEncodeCopyDistance(uint(distance), uint(new_params.num_direct_distance_codes), uint(new_params.distance_postfix_bits), &dist_prefix, &dist_extra) } HistogramAddDistance(&histo, uint(dist_prefix)&0x3FF) extra_bits += float64(dist_prefix >> 10) } } *cost = BrotliPopulationCostDistance(&histo) + extra_bits return true } var BrotliBuildMetaBlock_kMaxNumberOfHistograms uint = 256 func BrotliBuildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *BrotliEncoderParams, prev_byte byte, prev_byte2 byte, cmds []Command, num_commands uint, literal_context_mode int, mb *MetaBlockSplit) { var distance_histograms []HistogramDistance var literal_histograms []HistogramLiteral var literal_context_modes []int = nil var literal_histograms_size uint var distance_histograms_size uint var i uint var literal_context_multiplier uint = 1 var npostfix uint32 var ndirect_msb uint32 = 0 var check_orig bool = true var best_dist_cost float64 = 1e99 var orig_params BrotliEncoderParams = *params /* Histogram ids need to fit in one byte. */ var new_params BrotliEncoderParams = *params for npostfix = 0; npostfix <= BROTLI_MAX_NPOSTFIX; npostfix++ { for ; ndirect_msb < 16; ndirect_msb++ { var ndirect uint32 = ndirect_msb << npostfix var skip bool var dist_cost float64 BrotliInitDistanceParams(&new_params, npostfix, ndirect) if npostfix == orig_params.dist.distance_postfix_bits && ndirect == orig_params.dist.num_direct_distance_codes { check_orig = false } skip = !ComputeDistanceCost(cmds, num_commands, &orig_params.dist, &new_params.dist, &dist_cost) if skip || (dist_cost > best_dist_cost) { break } best_dist_cost = dist_cost params.dist = new_params.dist } if ndirect_msb > 0 { ndirect_msb-- } ndirect_msb /= 2 } if check_orig { var dist_cost float64 ComputeDistanceCost(cmds, num_commands, &orig_params.dist, &orig_params.dist, &dist_cost) if dist_cost < best_dist_cost { /* NB: currently unused; uncomment when more param tuning is added. */ /* best_dist_cost = dist_cost; */ params.dist = orig_params.dist } } RecomputeDistancePrefixes(cmds, num_commands, &orig_params.dist, ¶ms.dist) BrotliSplitBlock(cmds, num_commands, ringbuffer, pos, mask, params, &mb.literal_split, &mb.command_split, &mb.distance_split) if !params.disable_literal_context_modeling { literal_context_multiplier = 1 << BROTLI_LITERAL_CONTEXT_BITS literal_context_modes = make([]int, (mb.literal_split.num_types)) for i = 0; i < mb.literal_split.num_types; i++ { literal_context_modes[i] = literal_context_mode } } literal_histograms_size = mb.literal_split.num_types * literal_context_multiplier literal_histograms = make([]HistogramLiteral, literal_histograms_size) ClearHistogramsLiteral(literal_histograms, literal_histograms_size) distance_histograms_size = mb.distance_split.num_types << BROTLI_DISTANCE_CONTEXT_BITS distance_histograms = make([]HistogramDistance, distance_histograms_size) ClearHistogramsDistance(distance_histograms, distance_histograms_size) assert(mb.command_histograms == nil) mb.command_histograms_size = mb.command_split.num_types mb.command_histograms = make([]HistogramCommand, (mb.command_histograms_size)) ClearHistogramsCommand(mb.command_histograms, mb.command_histograms_size) BrotliBuildHistogramsWithContext(cmds, num_commands, &mb.literal_split, &mb.command_split, &mb.distance_split, ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_modes, literal_histograms, mb.command_histograms, distance_histograms) literal_context_modes = nil assert(mb.literal_context_map == nil) mb.literal_context_map_size = mb.literal_split.num_types << BROTLI_LITERAL_CONTEXT_BITS mb.literal_context_map = make([]uint32, (mb.literal_context_map_size)) assert(mb.literal_histograms == nil) mb.literal_histograms_size = mb.literal_context_map_size mb.literal_histograms = make([]HistogramLiteral, (mb.literal_histograms_size)) BrotliClusterHistogramsLiteral(literal_histograms, literal_histograms_size, BrotliBuildMetaBlock_kMaxNumberOfHistograms, mb.literal_histograms, &mb.literal_histograms_size, mb.literal_context_map) literal_histograms = nil if params.disable_literal_context_modeling { /* Distribute assignment to all contexts. */ for i = mb.literal_split.num_types; i != 0; { var j uint = 0 i-- for ; j < 1< 0 { var entropy [BROTLI_MAX_STATIC_CONTEXTS]float64 var combined_histo []HistogramLiteral = make([]HistogramLiteral, (2 * num_contexts)) var combined_entropy [2 * BROTLI_MAX_STATIC_CONTEXTS]float64 var diff = [2]float64{0.0} /* Try merging the set of histograms for the current block type with the respective set of histograms for the last and second last block types. Decide over the split based on the total reduction of entropy across all contexts. */ var i uint for i = 0; i < num_contexts; i++ { var curr_histo_ix uint = self.curr_histogram_ix_ + i var j uint entropy[i] = BitsEntropy(histograms[curr_histo_ix].data_[:], self.alphabet_size_) for j = 0; j < 2; j++ { var jx uint = j*num_contexts + i var last_histogram_ix uint = self.last_histogram_ix_[j] + i combined_histo[jx] = histograms[curr_histo_ix] HistogramAddHistogramLiteral(&combined_histo[jx], &histograms[last_histogram_ix]) combined_entropy[jx] = BitsEntropy(combined_histo[jx].data_[0:], self.alphabet_size_) diff[j] += combined_entropy[jx] - entropy[i] - last_entropy[jx] } } if split.num_types < self.max_block_types_ && diff[0] > self.split_threshold_ && diff[1] > self.split_threshold_ { /* Create new block. */ split.lengths[self.num_blocks_] = uint32(self.block_size_) split.types[self.num_blocks_] = byte(split.num_types) self.last_histogram_ix_[1] = self.last_histogram_ix_[0] self.last_histogram_ix_[0] = split.num_types * num_contexts for i = 0; i < num_contexts; i++ { last_entropy[num_contexts+i] = last_entropy[i] last_entropy[i] = entropy[i] } self.num_blocks_++ split.num_types++ self.curr_histogram_ix_ += num_contexts if self.curr_histogram_ix_ < *self.histograms_size_ { ClearHistogramsLiteral(self.histograms_[self.curr_histogram_ix_:], self.num_contexts_) } self.block_size_ = 0 self.merge_last_count_ = 0 self.target_block_size_ = self.min_block_size_ } else if diff[1] < diff[0]-20.0 { split.lengths[self.num_blocks_] = uint32(self.block_size_) split.types[self.num_blocks_] = split.types[self.num_blocks_-2] /* Combine this block with second last block. */ var tmp uint = self.last_histogram_ix_[0] self.last_histogram_ix_[0] = self.last_histogram_ix_[1] self.last_histogram_ix_[1] = tmp for i = 0; i < num_contexts; i++ { histograms[self.last_histogram_ix_[0]+i] = combined_histo[num_contexts+i] last_entropy[num_contexts+i] = last_entropy[i] last_entropy[i] = combined_entropy[num_contexts+i] HistogramClearLiteral(&histograms[self.curr_histogram_ix_+i]) } self.num_blocks_++ self.block_size_ = 0 self.merge_last_count_ = 0 self.target_block_size_ = self.min_block_size_ } else { /* Combine this block with last block. */ split.lengths[self.num_blocks_-1] += uint32(self.block_size_) for i = 0; i < num_contexts; i++ { histograms[self.last_histogram_ix_[0]+i] = combined_histo[i] last_entropy[i] = combined_entropy[i] if split.num_types == 1 { last_entropy[num_contexts+i] = last_entropy[i] } HistogramClearLiteral(&histograms[self.curr_histogram_ix_+i]) } self.block_size_ = 0 self.merge_last_count_++ if self.merge_last_count_ > 1 { self.target_block_size_ += self.min_block_size_ } } combined_histo = nil } if is_final { *self.histograms_size_ = split.num_types * num_contexts split.num_blocks = self.num_blocks_ } } /* Adds the next symbol to the current block type and context. When the current block reaches the target size, decides on merging the block. */ func ContextBlockSplitterAddSymbol(self *ContextBlockSplitter, symbol uint, context uint) { HistogramAddLiteral(&self.histograms_[self.curr_histogram_ix_+context], symbol) self.block_size_++ if self.block_size_ == self.target_block_size_ { ContextBlockSplitterFinishBlock(self, false) /* is_final = */ } } func MapStaticContexts(num_contexts uint, static_context_map []uint32, mb *MetaBlockSplit) { var i uint assert(mb.literal_context_map == nil) mb.literal_context_map_size = mb.literal_split.num_types << BROTLI_LITERAL_CONTEXT_BITS mb.literal_context_map = make([]uint32, (mb.literal_context_map_size)) for i = 0; i < mb.literal_split.num_types; i++ { var offset uint32 = uint32(i * num_contexts) var j uint for j = 0; j < 1<= 128 { BlockSplitterAddSymbolDistance(&dist_blocks, uint(cmd.dist_prefix_)&0x3FF) } } } if num_contexts == 1 { BlockSplitterFinishBlockLiteral(&lit_blocks.plain, true) /* is_final = */ } else { ContextBlockSplitterFinishBlock(&lit_blocks.ctx, true) /* is_final = */ } BlockSplitterFinishBlockCommand(&cmd_blocks, true) /* is_final = */ BlockSplitterFinishBlockDistance(&dist_blocks, true) /* is_final = */ if num_contexts > 1 { MapStaticContexts(num_contexts, static_context_map, mb) } } func BrotliBuildMetaBlockGreedy(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut ContextLut, num_contexts uint, static_context_map []uint32, commands []Command, n_commands uint, mb *MetaBlockSplit) { if num_contexts == 1 { BrotliBuildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, 1, nil, commands, n_commands, mb) } else { BrotliBuildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, num_contexts, static_context_map, commands, n_commands, mb) } } func BrotliOptimizeHistograms(num_distance_codes uint32, mb *MetaBlockSplit) { var good_for_rle [BROTLI_NUM_COMMAND_SYMBOLS]byte var i uint for i = 0; i < mb.literal_histograms_size; i++ { BrotliOptimizeHuffmanCountsForRle(256, mb.literal_histograms[i].data_[:], good_for_rle[:]) } for i = 0; i < mb.command_histograms_size; i++ { BrotliOptimizeHuffmanCountsForRle(BROTLI_NUM_COMMAND_SYMBOLS, mb.command_histograms[i].data_[:], good_for_rle[:]) } for i = 0; i < mb.distance_histograms_size; i++ { BrotliOptimizeHuffmanCountsForRle(uint(num_distance_codes), mb.distance_histograms[i].data_[:], good_for_rle[:]) } }