diff --git a/brotli_bit_stream.go b/brotli_bit_stream.go index 2470f84..7acfb18 100644 --- a/brotli_bit_stream.go +++ b/brotli_bit_stream.go @@ -121,7 +121,7 @@ func encodeMlen(length uint, bits *uint64, numbits *uint, nibblesbits *uint64) { *bits = uint64(length) - 1 } -func storeCommandExtra(cmd *command, bw *bitWriter) { +func storeCommandExtra(cmd *command, storage_ix *uint, storage []byte) { var copylen_code uint32 = commandCopyLenCode(cmd) var inscode uint16 = getInsertLengthCode(uint(cmd.insert_len_)) var copycode uint16 = getCopyLengthCode(uint(copylen_code)) @@ -129,7 +129,7 @@ func storeCommandExtra(cmd *command, bw *bitWriter) { var insextraval uint64 = uint64(cmd.insert_len_) - uint64(getInsertBase(inscode)) var copyextraval uint64 = uint64(copylen_code) - uint64(getCopyBase(copycode)) var bits uint64 = copyextraval< 0 REQUIRES: length <= (1 << 24) */ -func storeCompressedMetaBlockHeader(is_final_block bool, length uint, bw *bitWriter) { +func storeCompressedMetaBlockHeader(is_final_block bool, length uint, storage_ix *uint, storage []byte) { var lenbits uint64 var nlenbits uint var nibblesbits uint64 @@ -169,41 +169,41 @@ func storeCompressedMetaBlockHeader(is_final_block bool, length uint, bw *bitWri } /* Write ISLAST bit. */ - bw.writeBits(1, is_final) + writeBits(1, is_final, storage_ix, storage) /* Write ISEMPTY bit. */ if is_final_block { - bw.writeBits(1, 0) + writeBits(1, 0, storage_ix, storage) } encodeMlen(length, &lenbits, &nlenbits, &nibblesbits) - bw.writeBits(2, nibblesbits) - bw.writeBits(nlenbits, lenbits) + writeBits(2, nibblesbits, storage_ix, storage) + writeBits(nlenbits, lenbits, storage_ix, storage) if !is_final_block { /* Write ISUNCOMPRESSED bit. */ - bw.writeBits(1, 0) + writeBits(1, 0, storage_ix, storage) } } /* Stores the uncompressed meta-block header. REQUIRES: length > 0 REQUIRES: length <= (1 << 24) */ -func storeUncompressedMetaBlockHeader(length uint, bw *bitWriter) { +func storeUncompressedMetaBlockHeader(length uint, storage_ix *uint, storage []byte) { var lenbits uint64 var nlenbits uint var nibblesbits uint64 /* Write ISLAST bit. Uncompressed block cannot be the last one, so set to 0. */ - bw.writeBits(1, 0) + writeBits(1, 0, storage_ix, storage) encodeMlen(length, &lenbits, &nlenbits, &nibblesbits) - bw.writeBits(2, nibblesbits) - bw.writeBits(nlenbits, lenbits) + writeBits(2, nibblesbits, storage_ix, storage) + writeBits(nlenbits, lenbits, storage_ix, storage) /* Write ISUNCOMPRESSED bit. */ - bw.writeBits(1, 1) + writeBits(1, 1, storage_ix, storage) } var storeHuffmanTreeOfHuffmanTreeToBitMask_kStorageOrder = [codeLengthCodes]byte{1, 2, 3, 4, 0, 5, 17, 6, 16, 7, 8, 9, 10, 11, 12, 13, 14, 15} @@ -211,7 +211,7 @@ var storeHuffmanTreeOfHuffmanTreeToBitMask_kStorageOrder = [codeLengthCodes]byte var storeHuffmanTreeOfHuffmanTreeToBitMask_kHuffmanBitLengthHuffmanCodeSymbols = [6]byte{0, 7, 3, 2, 1, 15} var storeHuffmanTreeOfHuffmanTreeToBitMask_kHuffmanBitLengthHuffmanCodeBitLengths = [6]byte{2, 4, 3, 2, 2, 4} -func storeHuffmanTreeOfHuffmanTreeToBitMask(num_codes int, code_length_bitdepth []byte, bw *bitWriter) { +func storeHuffmanTreeOfHuffmanTreeToBitMask(num_codes int, code_length_bitdepth []byte, storage_ix *uint, storage []byte) { var skip_some uint = 0 var codes_to_store uint = codeLengthCodes /* The bit lengths of the Huffman code over the code length alphabet @@ -241,38 +241,38 @@ func storeHuffmanTreeOfHuffmanTreeToBitMask(num_codes int, code_length_bitdepth } } - bw.writeBits(2, uint64(skip_some)) + writeBits(2, uint64(skip_some), storage_ix, storage) { var i uint for i = skip_some; i < codes_to_store; i++ { var l uint = uint(code_length_bitdepth[storeHuffmanTreeOfHuffmanTreeToBitMask_kStorageOrder[i]]) - bw.writeBits(uint(storeHuffmanTreeOfHuffmanTreeToBitMask_kHuffmanBitLengthHuffmanCodeBitLengths[l]), uint64(storeHuffmanTreeOfHuffmanTreeToBitMask_kHuffmanBitLengthHuffmanCodeSymbols[l])) + writeBits(uint(storeHuffmanTreeOfHuffmanTreeToBitMask_kHuffmanBitLengthHuffmanCodeBitLengths[l]), uint64(storeHuffmanTreeOfHuffmanTreeToBitMask_kHuffmanBitLengthHuffmanCodeSymbols[l]), storage_ix, storage) } } } -func storeHuffmanTreeToBitMask(huffman_tree_size uint, huffman_tree []byte, huffman_tree_extra_bits []byte, code_length_bitdepth []byte, code_length_bitdepth_symbols []uint16, bw *bitWriter) { +func storeHuffmanTreeToBitMask(huffman_tree_size uint, huffman_tree []byte, huffman_tree_extra_bits []byte, code_length_bitdepth []byte, code_length_bitdepth_symbols []uint16, storage_ix *uint, storage []byte) { var i uint for i = 0; i < huffman_tree_size; i++ { var ix uint = uint(huffman_tree[i]) - bw.writeBits(uint(code_length_bitdepth[ix]), uint64(code_length_bitdepth_symbols[ix])) + writeBits(uint(code_length_bitdepth[ix]), uint64(code_length_bitdepth_symbols[ix]), storage_ix, storage) /* Extra bits */ switch ix { case repeatPreviousCodeLength: - bw.writeBits(2, uint64(huffman_tree_extra_bits[i])) + writeBits(2, uint64(huffman_tree_extra_bits[i]), storage_ix, storage) case repeatZeroCodeLength: - bw.writeBits(3, uint64(huffman_tree_extra_bits[i])) + writeBits(3, uint64(huffman_tree_extra_bits[i]), storage_ix, storage) } } } -func storeSimpleHuffmanTree(depths []byte, symbols []uint, num_symbols uint, max_bits uint, bw *bitWriter) { +func storeSimpleHuffmanTree(depths []byte, symbols []uint, num_symbols uint, max_bits uint, storage_ix *uint, storage []byte) { /* value of 1 indicates a simple Huffman code */ - bw.writeBits(2, 1) + writeBits(2, 1, storage_ix, storage) - bw.writeBits(2, uint64(num_symbols)-1) /* NSYM - 1 */ + writeBits(2, uint64(num_symbols)-1, storage_ix, storage) /* NSYM - 1 */ { /* Sort */ var i uint @@ -289,17 +289,17 @@ func storeSimpleHuffmanTree(depths []byte, symbols []uint, num_symbols uint, max } if num_symbols == 2 { - bw.writeBits(max_bits, uint64(symbols[0])) - bw.writeBits(max_bits, uint64(symbols[1])) + writeBits(max_bits, uint64(symbols[0]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[1]), storage_ix, storage) } else if num_symbols == 3 { - bw.writeBits(max_bits, uint64(symbols[0])) - bw.writeBits(max_bits, uint64(symbols[1])) - bw.writeBits(max_bits, uint64(symbols[2])) + writeBits(max_bits, uint64(symbols[0]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[1]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[2]), storage_ix, storage) } else { - bw.writeBits(max_bits, uint64(symbols[0])) - bw.writeBits(max_bits, uint64(symbols[1])) - bw.writeBits(max_bits, uint64(symbols[2])) - bw.writeBits(max_bits, uint64(symbols[3])) + writeBits(max_bits, uint64(symbols[0]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[1]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[2]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[3]), storage_ix, storage) /* tree-select */ var tmp int @@ -308,13 +308,13 @@ func storeSimpleHuffmanTree(depths []byte, symbols []uint, num_symbols uint, max } else { tmp = 0 } - bw.writeBits(1, uint64(tmp)) + writeBits(1, uint64(tmp), storage_ix, storage) } } /* num = alphabet size depths = symbol depths */ -func storeHuffmanTree(depths []byte, num uint, tree []huffmanTree, bw *bitWriter) { +func storeHuffmanTree(depths []byte, num uint, tree []huffmanTree, storage_ix *uint, storage []byte) { var huffman_tree [numCommandSymbols]byte var huffman_tree_extra_bits [numCommandSymbols]byte var huffman_tree_size uint = 0 @@ -357,19 +357,19 @@ func storeHuffmanTree(depths []byte, num uint, tree []huffmanTree, bw *bitWriter convertBitDepthsToSymbols(code_length_bitdepth[:], codeLengthCodes, code_length_bitdepth_symbols[:]) /* Now, we have all the data, let's start storing it */ - storeHuffmanTreeOfHuffmanTreeToBitMask(num_codes, code_length_bitdepth[:], bw) + storeHuffmanTreeOfHuffmanTreeToBitMask(num_codes, code_length_bitdepth[:], storage_ix, storage) if num_codes == 1 { code_length_bitdepth[code] = 0 } /* Store the real Huffman tree now. */ - storeHuffmanTreeToBitMask(huffman_tree_size, huffman_tree[:], huffman_tree_extra_bits[:], code_length_bitdepth[:], code_length_bitdepth_symbols[:], bw) + storeHuffmanTreeToBitMask(huffman_tree_size, huffman_tree[:], huffman_tree_extra_bits[:], code_length_bitdepth[:], code_length_bitdepth_symbols[:], storage_ix, storage) } /* Builds a Huffman tree from histogram[0:length] into depth[0:length] and bits[0:length] and stores the encoded tree to the bit stream. */ -func buildAndStoreHuffmanTree(histogram []uint32, histogram_length uint, alphabet_size uint, tree []huffmanTree, depth []byte, bits []uint16, bw *bitWriter) { +func buildAndStoreHuffmanTree(histogram []uint32, histogram_length uint, alphabet_size uint, tree []huffmanTree, depth []byte, bits []uint16, storage_ix *uint, storage []byte) { var count uint = 0 var s4 = [4]uint{0} var i uint @@ -394,8 +394,8 @@ func buildAndStoreHuffmanTree(histogram []uint32, histogram_length uint, alphabe } if count <= 1 { - bw.writeBits(4, 1) - bw.writeBits(max_bits, uint64(s4[0])) + writeBits(4, 1, storage_ix, storage) + writeBits(max_bits, uint64(s4[0]), storage_ix, storage) depth[s4[0]] = 0 bits[s4[0]] = 0 return @@ -408,9 +408,9 @@ func buildAndStoreHuffmanTree(histogram []uint32, histogram_length uint, alphabe convertBitDepthsToSymbols(depth, histogram_length, bits) if count <= 4 { - storeSimpleHuffmanTree(depth, s4[:], count, max_bits, bw) + storeSimpleHuffmanTree(depth, s4[:], count, max_bits, storage_ix, storage) } else { - storeHuffmanTree(depth, histogram_length, tree, bw) + storeHuffmanTree(depth, histogram_length, tree, storage_ix, storage) } } @@ -420,7 +420,7 @@ func sortHuffmanTree1(v0 huffmanTree, v1 huffmanTree) bool { var huffmanTreePool sync.Pool -func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_bits uint, depth []byte, bits []uint16, bw *bitWriter) { +func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_bits uint, depth []byte, bits []uint16, storage_ix *uint, storage []byte) { var count uint = 0 var symbols = [4]uint{0} var length uint = 0 @@ -439,8 +439,8 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ } if count <= 1 { - bw.writeBits(4, 1) - bw.writeBits(max_bits, uint64(symbols[0])) + writeBits(4, 1, storage_ix, storage) + writeBits(max_bits, uint64(symbols[0]), storage_ix, storage) depth[symbols[0]] = 0 bits[symbols[0]] = 0 return @@ -544,9 +544,9 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ var i uint /* value of 1 indicates a simple Huffman code */ - bw.writeBits(2, 1) + writeBits(2, 1, storage_ix, storage) - bw.writeBits(2, uint64(count)-1) /* NSYM - 1 */ + writeBits(2, uint64(count)-1, storage_ix, storage) /* NSYM - 1 */ /* Sort */ for i = 0; i < count; i++ { @@ -561,27 +561,33 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ } if count == 2 { - bw.writeBits(max_bits, uint64(symbols[0])) - bw.writeBits(max_bits, uint64(symbols[1])) + writeBits(max_bits, uint64(symbols[0]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[1]), storage_ix, storage) } else if count == 3 { - bw.writeBits(max_bits, uint64(symbols[0])) - bw.writeBits(max_bits, uint64(symbols[1])) - bw.writeBits(max_bits, uint64(symbols[2])) + writeBits(max_bits, uint64(symbols[0]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[1]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[2]), storage_ix, storage) } else { - bw.writeBits(max_bits, uint64(symbols[0])) - bw.writeBits(max_bits, uint64(symbols[1])) - bw.writeBits(max_bits, uint64(symbols[2])) - bw.writeBits(max_bits, uint64(symbols[3])) + writeBits(max_bits, uint64(symbols[0]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[1]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[2]), storage_ix, storage) + writeBits(max_bits, uint64(symbols[3]), storage_ix, storage) /* tree-select */ - bw.writeSingleBit(depth[symbols[0]] == 1) + var tmp int + if depth[symbols[0]] == 1 { + tmp = 1 + } else { + tmp = 0 + } + writeBits(1, uint64(tmp), storage_ix, storage) } } else { var previous_value byte = 8 var i uint /* Complex Huffman Tree */ - storeStaticCodeLengthCode(bw) + storeStaticCodeLengthCode(storage_ix, storage) /* Actual RLE coding. */ for i = 0; i < length; { @@ -594,21 +600,21 @@ func buildAndStoreHuffmanTreeFast(histogram []uint32, histogram_total uint, max_ i += reps if value == 0 { - bw.writeBits(uint(kZeroRepsDepth[reps]), kZeroRepsBits[reps]) + writeBits(uint(kZeroRepsDepth[reps]), kZeroRepsBits[reps], storage_ix, storage) } else { if previous_value != value { - bw.writeBits(uint(kCodeLengthDepth[value]), uint64(kCodeLengthBits[value])) + writeBits(uint(kCodeLengthDepth[value]), uint64(kCodeLengthBits[value]), storage_ix, storage) reps-- } if reps < 3 { for reps != 0 { reps-- - bw.writeBits(uint(kCodeLengthDepth[value]), uint64(kCodeLengthBits[value])) + writeBits(uint(kCodeLengthDepth[value]), uint64(kCodeLengthBits[value]), storage_ix, storage) } } else { reps -= 3 - bw.writeBits(uint(kNonZeroRepsDepth[reps]), kNonZeroRepsBits[reps]) + writeBits(uint(kNonZeroRepsDepth[reps]), kNonZeroRepsBits[reps], storage_ix, storage) } previous_value = value @@ -733,7 +739,7 @@ const symbolBits = 9 var encodeContextMap_kSymbolMask uint32 = (1 << symbolBits) - 1 -func encodeContextMap(context_map []uint32, context_map_size uint, num_clusters uint, tree []huffmanTree, bw *bitWriter) { +func encodeContextMap(context_map []uint32, context_map_size uint, num_clusters uint, tree []huffmanTree, storage_ix *uint, storage []byte) { var i uint var rle_symbols []uint32 var max_run_length_prefix uint32 = 6 @@ -742,7 +748,7 @@ func encodeContextMap(context_map []uint32, context_map_size uint, num_clusters var depths [maxContextMapSymbols]byte var bits [maxContextMapSymbols]uint16 - storeVarLenUint8(num_clusters-1, bw) + storeVarLenUint8(num_clusters-1, storage_ix, storage) if num_clusters == 1 { return @@ -757,45 +763,45 @@ func encodeContextMap(context_map []uint32, context_map_size uint, num_clusters } { var use_rle bool = (max_run_length_prefix > 0) - bw.writeSingleBit(use_rle) + writeSingleBit(use_rle, storage_ix, storage) if use_rle { - bw.writeBits(4, uint64(max_run_length_prefix)-1) + writeBits(4, uint64(max_run_length_prefix)-1, storage_ix, storage) } } - buildAndStoreHuffmanTree(histogram[:], uint(uint32(num_clusters)+max_run_length_prefix), uint(uint32(num_clusters)+max_run_length_prefix), tree, depths[:], bits[:], bw) + buildAndStoreHuffmanTree(histogram[:], uint(uint32(num_clusters)+max_run_length_prefix), uint(uint32(num_clusters)+max_run_length_prefix), tree, depths[:], bits[:], storage_ix, storage) for i = 0; i < num_rle_symbols; i++ { var rle_symbol uint32 = rle_symbols[i] & encodeContextMap_kSymbolMask var extra_bits_val uint32 = rle_symbols[i] >> symbolBits - bw.writeBits(uint(depths[rle_symbol]), uint64(bits[rle_symbol])) + writeBits(uint(depths[rle_symbol]), uint64(bits[rle_symbol]), storage_ix, storage) if rle_symbol > 0 && rle_symbol <= max_run_length_prefix { - bw.writeBits(uint(rle_symbol), uint64(extra_bits_val)) + writeBits(uint(rle_symbol), uint64(extra_bits_val), storage_ix, storage) } } - bw.writeBits(1, 1) /* use move-to-front */ + writeBits(1, 1, storage_ix, storage) /* use move-to-front */ rle_symbols = nil } /* Stores the block switch command with index block_ix to the bit stream. */ -func storeBlockSwitch(code *blockSplitCode, block_len uint32, block_type byte, is_first_block bool, bw *bitWriter) { +func storeBlockSwitch(code *blockSplitCode, block_len uint32, block_type byte, is_first_block bool, storage_ix *uint, storage []byte) { var typecode uint = nextBlockTypeCode(&code.type_code_calculator, block_type) var lencode uint var len_nextra uint32 var len_extra uint32 if !is_first_block { - bw.writeBits(uint(code.type_depths[typecode]), uint64(code.type_bits[typecode])) + writeBits(uint(code.type_depths[typecode]), uint64(code.type_bits[typecode]), storage_ix, storage) } getBlockLengthPrefixCode(block_len, &lencode, &len_nextra, &len_extra) - bw.writeBits(uint(code.length_depths[lencode]), uint64(code.length_bits[lencode])) - bw.writeBits(uint(len_nextra), uint64(len_extra)) + writeBits(uint(code.length_depths[lencode]), uint64(code.length_bits[lencode]), storage_ix, storage) + writeBits(uint(len_nextra), uint64(len_extra), storage_ix, storage) } /* Builds a BlockSplitCode data structure from the block split given by the vector of block types and block lengths and stores it to the bit stream. */ -func buildAndStoreBlockSplitCode(types []byte, lengths []uint32, num_blocks uint, num_types uint, tree []huffmanTree, code *blockSplitCode, bw *bitWriter) { +func buildAndStoreBlockSplitCode(types []byte, lengths []uint32, num_blocks uint, num_types uint, tree []huffmanTree, code *blockSplitCode, storage_ix *uint, storage []byte) { var type_histo [maxBlockTypeSymbols]uint32 var length_histo [numBlockLenSymbols]uint32 var i uint @@ -813,17 +819,17 @@ func buildAndStoreBlockSplitCode(types []byte, lengths []uint32, num_blocks uint length_histo[blockLengthPrefixCode(lengths[i])]++ } - storeVarLenUint8(num_types-1, bw) + storeVarLenUint8(num_types-1, storage_ix, storage) if num_types > 1 { /* TODO: else? could StoreBlockSwitch occur? */ - buildAndStoreHuffmanTree(type_histo[0:], num_types+2, num_types+2, tree, code.type_depths[0:], code.type_bits[0:], bw) - buildAndStoreHuffmanTree(length_histo[0:], numBlockLenSymbols, numBlockLenSymbols, tree, code.length_depths[0:], code.length_bits[0:], bw) - storeBlockSwitch(code, lengths[0], types[0], true, bw) + buildAndStoreHuffmanTree(type_histo[0:], num_types+2, num_types+2, tree, code.type_depths[0:], code.type_bits[0:], storage_ix, storage) + buildAndStoreHuffmanTree(length_histo[0:], numBlockLenSymbols, numBlockLenSymbols, tree, code.length_depths[0:], code.length_bits[0:], storage_ix, storage) + storeBlockSwitch(code, lengths[0], types[0], true, storage_ix, storage) } } /* Stores a context map where the histogram type is always the block type. */ -func storeTrivialContextMap(num_types uint, context_bits uint, tree []huffmanTree, bw *bitWriter) { - storeVarLenUint8(num_types-1, bw) +func storeTrivialContextMap(num_types uint, context_bits uint, tree []huffmanTree, storage_ix *uint, storage []byte) { + storeVarLenUint8(num_types-1, storage_ix, storage) if num_types > 1 { var repeat_code uint = context_bits - 1 var repeat_bits uint = (1 << repeat_code) - 1 @@ -837,16 +843,16 @@ func storeTrivialContextMap(num_types uint, context_bits uint, tree []huffmanTre } /* Write RLEMAX. */ - bw.writeBits(1, 1) + writeBits(1, 1, storage_ix, storage) - bw.writeBits(4, uint64(repeat_code)-1) + writeBits(4, uint64(repeat_code)-1, storage_ix, storage) histogram[repeat_code] = uint32(num_types) histogram[0] = 1 for i = context_bits; i < alphabet_size; i++ { histogram[i] = 1 } - buildAndStoreHuffmanTree(histogram[:], alphabet_size, alphabet_size, tree, depths[:], bits[:], bw) + buildAndStoreHuffmanTree(histogram[:], alphabet_size, alphabet_size, tree, depths[:], bits[:], storage_ix, storage) for i = 0; i < num_types; i++ { var tmp uint if i == 0 { @@ -855,13 +861,13 @@ func storeTrivialContextMap(num_types uint, context_bits uint, tree []huffmanTre tmp = i + context_bits - 1 } var code uint = tmp - bw.writeBits(uint(depths[code]), uint64(bits[code])) - bw.writeBits(uint(depths[repeat_code]), uint64(bits[repeat_code])) - bw.writeBits(repeat_code, uint64(repeat_bits)) + writeBits(uint(depths[code]), uint64(bits[code]), storage_ix, storage) + writeBits(uint(depths[repeat_code]), uint64(bits[repeat_code]), storage_ix, storage) + writeBits(repeat_code, uint64(repeat_bits), storage_ix, storage) } /* Write IMTF (inverse-move-to-front) bit. */ - bw.writeBits(1, 1) + writeBits(1, 1, storage_ix, storage) } } @@ -915,13 +921,13 @@ func cleanupBlockEncoder(self *blockEncoder) { /* Creates entropy codes of block lengths and block types and stores them to the bit stream. */ -func buildAndStoreBlockSwitchEntropyCodes(self *blockEncoder, tree []huffmanTree, bw *bitWriter) { - buildAndStoreBlockSplitCode(self.block_types_, self.block_lengths_, self.num_blocks_, self.num_block_types_, tree, &self.block_split_code_, bw) +func buildAndStoreBlockSwitchEntropyCodes(self *blockEncoder, tree []huffmanTree, storage_ix *uint, storage []byte) { + buildAndStoreBlockSplitCode(self.block_types_, self.block_lengths_, self.num_blocks_, self.num_block_types_, tree, &self.block_split_code_, storage_ix, storage) } /* Stores the next symbol with the entropy code of the current block type. Updates the block type and block length at block boundaries. */ -func storeSymbol(self *blockEncoder, symbol uint, bw *bitWriter) { +func storeSymbol(self *blockEncoder, symbol uint, storage_ix *uint, storage []byte) { if self.block_len_ == 0 { self.block_ix_++ var block_ix uint = self.block_ix_ @@ -929,20 +935,20 @@ func storeSymbol(self *blockEncoder, symbol uint, bw *bitWriter) { var block_type byte = self.block_types_[block_ix] self.block_len_ = uint(block_len) self.entropy_ix_ = uint(block_type) * self.histogram_length_ - storeBlockSwitch(&self.block_split_code_, block_len, block_type, false, bw) + storeBlockSwitch(&self.block_split_code_, block_len, block_type, false, storage_ix, storage) } self.block_len_-- { var ix uint = self.entropy_ix_ + symbol - bw.writeBits(uint(self.depths_[ix]), uint64(self.bits_[ix])) + writeBits(uint(self.depths_[ix]), uint64(self.bits_[ix]), storage_ix, storage) } } /* Stores the next symbol with the entropy code of the current block type and context value. Updates the block type and block length at block boundaries. */ -func storeSymbolWithContext(self *blockEncoder, symbol uint, context uint, context_map []uint32, bw *bitWriter, context_bits uint) { +func storeSymbolWithContext(self *blockEncoder, symbol uint, context uint, context_map []uint32, storage_ix *uint, storage []byte, context_bits uint) { if self.block_len_ == 0 { self.block_ix_++ var block_ix uint = self.block_ix_ @@ -950,18 +956,18 @@ func storeSymbolWithContext(self *blockEncoder, symbol uint, context uint, conte var block_type byte = self.block_types_[block_ix] self.block_len_ = uint(block_len) self.entropy_ix_ = uint(block_type) << context_bits - storeBlockSwitch(&self.block_split_code_, block_len, block_type, false, bw) + storeBlockSwitch(&self.block_split_code_, block_len, block_type, false, storage_ix, storage) } self.block_len_-- { var histo_ix uint = uint(context_map[self.entropy_ix_+context]) var ix uint = histo_ix*self.histogram_length_ + symbol - bw.writeBits(uint(self.depths_[ix]), uint64(self.bits_[ix])) + writeBits(uint(self.depths_[ix]), uint64(self.bits_[ix]), storage_ix, storage) } } -func buildAndStoreEntropyCodesLiteral(self *blockEncoder, histograms []histogramLiteral, histograms_size uint, alphabet_size uint, tree []huffmanTree, bw *bitWriter) { +func buildAndStoreEntropyCodesLiteral(self *blockEncoder, histograms []histogramLiteral, histograms_size uint, alphabet_size uint, tree []huffmanTree, storage_ix *uint, storage []byte) { var table_size uint = histograms_size * self.histogram_length_ if cap(self.depths_) < int(table_size) { self.depths_ = make([]byte, table_size) @@ -977,12 +983,12 @@ func buildAndStoreEntropyCodesLiteral(self *blockEncoder, histograms []histogram var i uint for i = 0; i < histograms_size; i++ { var ix uint = i * self.histogram_length_ - buildAndStoreHuffmanTree(histograms[i].data_[0:], self.histogram_length_, alphabet_size, tree, self.depths_[ix:], self.bits_[ix:], bw) + buildAndStoreHuffmanTree(histograms[i].data_[0:], self.histogram_length_, alphabet_size, tree, self.depths_[ix:], self.bits_[ix:], storage_ix, storage) } } } -func buildAndStoreEntropyCodesCommand(self *blockEncoder, histograms []histogramCommand, histograms_size uint, alphabet_size uint, tree []huffmanTree, bw *bitWriter) { +func buildAndStoreEntropyCodesCommand(self *blockEncoder, histograms []histogramCommand, histograms_size uint, alphabet_size uint, tree []huffmanTree, storage_ix *uint, storage []byte) { var table_size uint = histograms_size * self.histogram_length_ if cap(self.depths_) < int(table_size) { self.depths_ = make([]byte, table_size) @@ -998,12 +1004,12 @@ func buildAndStoreEntropyCodesCommand(self *blockEncoder, histograms []histogram var i uint for i = 0; i < histograms_size; i++ { var ix uint = i * self.histogram_length_ - buildAndStoreHuffmanTree(histograms[i].data_[0:], self.histogram_length_, alphabet_size, tree, self.depths_[ix:], self.bits_[ix:], bw) + buildAndStoreHuffmanTree(histograms[i].data_[0:], self.histogram_length_, alphabet_size, tree, self.depths_[ix:], self.bits_[ix:], storage_ix, storage) } } } -func buildAndStoreEntropyCodesDistance(self *blockEncoder, histograms []histogramDistance, histograms_size uint, alphabet_size uint, tree []huffmanTree, bw *bitWriter) { +func buildAndStoreEntropyCodesDistance(self *blockEncoder, histograms []histogramDistance, histograms_size uint, alphabet_size uint, tree []huffmanTree, storage_ix *uint, storage []byte) { var table_size uint = histograms_size * self.histogram_length_ if cap(self.depths_) < int(table_size) { self.depths_ = make([]byte, table_size) @@ -1019,12 +1025,17 @@ func buildAndStoreEntropyCodesDistance(self *blockEncoder, histograms []histogra var i uint for i = 0; i < histograms_size; i++ { var ix uint = i * self.histogram_length_ - buildAndStoreHuffmanTree(histograms[i].data_[0:], self.histogram_length_, alphabet_size, tree, self.depths_[ix:], self.bits_[ix:], bw) + buildAndStoreHuffmanTree(histograms[i].data_[0:], self.histogram_length_, alphabet_size, tree, self.depths_[ix:], self.bits_[ix:], storage_ix, storage) } } } -func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_byte byte, prev_byte2 byte, is_last bool, params *encoderParams, literal_context_mode int, commands []command, mb *metaBlockSplit, bw *bitWriter) { +func jumpToByteBoundary(storage_ix *uint, storage []byte) { + *storage_ix = (*storage_ix + 7) &^ 7 + storage[*storage_ix>>3] = 0 +} + +func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_byte byte, prev_byte2 byte, is_last bool, params *encoderParams, literal_context_mode int, commands []command, mb *metaBlockSplit, storage_ix *uint, storage []byte) { var pos uint = start_pos var i uint var num_distance_symbols uint32 = params.dist.alphabet_size @@ -1036,48 +1047,48 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b num_effective_distance_symbols = numHistogramDistanceSymbols } - storeCompressedMetaBlockHeader(is_last, length, bw) + storeCompressedMetaBlockHeader(is_last, length, storage_ix, storage) tree = make([]huffmanTree, maxHuffmanTreeSize) literal_enc := getBlockEncoder(numLiteralSymbols, mb.literal_split.num_types, mb.literal_split.types, mb.literal_split.lengths, mb.literal_split.num_blocks) command_enc := getBlockEncoder(numCommandSymbols, mb.command_split.num_types, mb.command_split.types, mb.command_split.lengths, mb.command_split.num_blocks) distance_enc := getBlockEncoder(uint(num_effective_distance_symbols), mb.distance_split.num_types, mb.distance_split.types, mb.distance_split.lengths, mb.distance_split.num_blocks) - buildAndStoreBlockSwitchEntropyCodes(literal_enc, tree, bw) - buildAndStoreBlockSwitchEntropyCodes(command_enc, tree, bw) - buildAndStoreBlockSwitchEntropyCodes(distance_enc, tree, bw) + buildAndStoreBlockSwitchEntropyCodes(literal_enc, tree, storage_ix, storage) + buildAndStoreBlockSwitchEntropyCodes(command_enc, tree, storage_ix, storage) + buildAndStoreBlockSwitchEntropyCodes(distance_enc, tree, storage_ix, storage) - bw.writeBits(2, uint64(dist.distance_postfix_bits)) - bw.writeBits(4, uint64(dist.num_direct_distance_codes)>>dist.distance_postfix_bits) + writeBits(2, uint64(dist.distance_postfix_bits), storage_ix, storage) + writeBits(4, uint64(dist.num_direct_distance_codes)>>dist.distance_postfix_bits, storage_ix, storage) for i = 0; i < mb.literal_split.num_types; i++ { - bw.writeBits(2, uint64(literal_context_mode)) + writeBits(2, uint64(literal_context_mode), storage_ix, storage) } if mb.literal_context_map_size == 0 { - storeTrivialContextMap(mb.literal_histograms_size, literalContextBits, tree, bw) + storeTrivialContextMap(mb.literal_histograms_size, literalContextBits, tree, storage_ix, storage) } else { - encodeContextMap(mb.literal_context_map, mb.literal_context_map_size, mb.literal_histograms_size, tree, bw) + encodeContextMap(mb.literal_context_map, mb.literal_context_map_size, mb.literal_histograms_size, tree, storage_ix, storage) } if mb.distance_context_map_size == 0 { - storeTrivialContextMap(mb.distance_histograms_size, distanceContextBits, tree, bw) + storeTrivialContextMap(mb.distance_histograms_size, distanceContextBits, tree, storage_ix, storage) } else { - encodeContextMap(mb.distance_context_map, mb.distance_context_map_size, mb.distance_histograms_size, tree, bw) + encodeContextMap(mb.distance_context_map, mb.distance_context_map_size, mb.distance_histograms_size, tree, storage_ix, storage) } - buildAndStoreEntropyCodesLiteral(literal_enc, mb.literal_histograms, mb.literal_histograms_size, numLiteralSymbols, tree, bw) - buildAndStoreEntropyCodesCommand(command_enc, mb.command_histograms, mb.command_histograms_size, numCommandSymbols, tree, bw) - buildAndStoreEntropyCodesDistance(distance_enc, mb.distance_histograms, mb.distance_histograms_size, uint(num_distance_symbols), tree, bw) + buildAndStoreEntropyCodesLiteral(literal_enc, mb.literal_histograms, mb.literal_histograms_size, numLiteralSymbols, tree, storage_ix, storage) + buildAndStoreEntropyCodesCommand(command_enc, mb.command_histograms, mb.command_histograms_size, numCommandSymbols, tree, storage_ix, storage) + buildAndStoreEntropyCodesDistance(distance_enc, mb.distance_histograms, mb.distance_histograms_size, uint(num_distance_symbols), tree, storage_ix, storage) tree = nil for _, cmd := range commands { var cmd_code uint = uint(cmd.cmd_prefix_) - storeSymbol(command_enc, cmd_code, bw) - storeCommandExtra(&cmd, bw) + storeSymbol(command_enc, cmd_code, storage_ix, storage) + storeCommandExtra(&cmd, storage_ix, storage) if mb.literal_context_map_size == 0 { var j uint for j = uint(cmd.insert_len_); j != 0; j-- { - storeSymbol(literal_enc, uint(input[pos&mask]), bw) + storeSymbol(literal_enc, uint(input[pos&mask]), storage_ix, storage) pos++ } } else { @@ -1085,7 +1096,7 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b for j = uint(cmd.insert_len_); j != 0; j-- { var context uint = uint(getContext(prev_byte, prev_byte2, literal_context_lut)) var literal byte = input[pos&mask] - storeSymbolWithContext(literal_enc, uint(literal), context, mb.literal_context_map, bw, literalContextBits) + storeSymbolWithContext(literal_enc, uint(literal), context, mb.literal_context_map, storage_ix, storage, literalContextBits) prev_byte2 = prev_byte prev_byte = literal pos++ @@ -1101,13 +1112,13 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b var distnumextra uint32 = uint32(cmd.dist_prefix_) >> 10 var distextra uint64 = uint64(cmd.dist_extra_) if mb.distance_context_map_size == 0 { - storeSymbol(distance_enc, dist_code, bw) + storeSymbol(distance_enc, dist_code, storage_ix, storage) } else { var context uint = uint(commandDistanceContext(&cmd)) - storeSymbolWithContext(distance_enc, dist_code, context, mb.distance_context_map, bw, distanceContextBits) + storeSymbolWithContext(distance_enc, dist_code, context, mb.distance_context_map, storage_ix, storage, distanceContextBits) } - bw.writeBits(uint(distnumextra), distextra) + writeBits(uint(distnumextra), distextra, storage_ix, storage) } } } @@ -1116,7 +1127,7 @@ func storeMetaBlock(input []byte, start_pos uint, length uint, mask uint, prev_b cleanupBlockEncoder(command_enc) cleanupBlockEncoder(literal_enc) if is_last { - bw.jumpToByteBoundary() + jumpToByteBoundary(storage_ix, storage) } } @@ -1137,16 +1148,16 @@ func buildHistograms(input []byte, start_pos uint, mask uint, commands []command } } -func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands []command, lit_depth []byte, lit_bits []uint16, cmd_depth []byte, cmd_bits []uint16, dist_depth []byte, dist_bits []uint16, bw *bitWriter) { +func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands []command, lit_depth []byte, lit_bits []uint16, cmd_depth []byte, cmd_bits []uint16, dist_depth []byte, dist_bits []uint16, storage_ix *uint, storage []byte) { var pos uint = start_pos for _, cmd := range commands { var cmd_code uint = uint(cmd.cmd_prefix_) var j uint - bw.writeBits(uint(cmd_depth[cmd_code]), uint64(cmd_bits[cmd_code])) - storeCommandExtra(&cmd, bw) + writeBits(uint(cmd_depth[cmd_code]), uint64(cmd_bits[cmd_code]), storage_ix, storage) + storeCommandExtra(&cmd, storage_ix, storage) for j = uint(cmd.insert_len_); j != 0; j-- { var literal byte = input[pos&mask] - bw.writeBits(uint(lit_depth[literal]), uint64(lit_bits[literal])) + writeBits(uint(lit_depth[literal]), uint64(lit_bits[literal]), storage_ix, storage) pos++ } @@ -1155,13 +1166,13 @@ func storeDataWithHuffmanCodes(input []byte, start_pos uint, mask uint, commands var dist_code uint = uint(cmd.dist_prefix_) & 0x3FF var distnumextra uint32 = uint32(cmd.dist_prefix_) >> 10 var distextra uint32 = cmd.dist_extra_ - bw.writeBits(uint(dist_depth[dist_code]), uint64(dist_bits[dist_code])) - bw.writeBits(uint(distnumextra), uint64(distextra)) + writeBits(uint(dist_depth[dist_code]), uint64(dist_bits[dist_code]), storage_ix, storage) + writeBits(uint(distnumextra), uint64(distextra), storage_ix, storage) } } } -func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, bw *bitWriter) { +func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, storage_ix *uint, storage []byte) { var lit_histo histogramLiteral var cmd_histo histogramCommand var dist_histo histogramDistance @@ -1174,7 +1185,7 @@ func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, var tree []huffmanTree var num_distance_symbols uint32 = params.dist.alphabet_size - storeCompressedMetaBlockHeader(is_last, length, bw) + storeCompressedMetaBlockHeader(is_last, length, storage_ix, storage) histogramClearLiteral(&lit_histo) histogramClearCommand(&cmd_histo) @@ -1182,26 +1193,26 @@ func storeMetaBlockTrivial(input []byte, start_pos uint, length uint, mask uint, buildHistograms(input, start_pos, mask, commands, &lit_histo, &cmd_histo, &dist_histo) - bw.writeBits(13, 0) + writeBits(13, 0, storage_ix, storage) tree = make([]huffmanTree, maxHuffmanTreeSize) - buildAndStoreHuffmanTree(lit_histo.data_[:], numLiteralSymbols, numLiteralSymbols, tree, lit_depth[:], lit_bits[:], bw) - buildAndStoreHuffmanTree(cmd_histo.data_[:], numCommandSymbols, numCommandSymbols, tree, cmd_depth[:], cmd_bits[:], bw) - buildAndStoreHuffmanTree(dist_histo.data_[:], maxSimpleDistanceAlphabetSize, uint(num_distance_symbols), tree, dist_depth[:], dist_bits[:], bw) + buildAndStoreHuffmanTree(lit_histo.data_[:], numLiteralSymbols, numLiteralSymbols, tree, lit_depth[:], lit_bits[:], storage_ix, storage) + buildAndStoreHuffmanTree(cmd_histo.data_[:], numCommandSymbols, numCommandSymbols, tree, cmd_depth[:], cmd_bits[:], storage_ix, storage) + buildAndStoreHuffmanTree(dist_histo.data_[:], maxSimpleDistanceAlphabetSize, uint(num_distance_symbols), tree, dist_depth[:], dist_bits[:], storage_ix, storage) tree = nil - storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], bw) + storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage) if is_last { - bw.jumpToByteBoundary() + jumpToByteBoundary(storage_ix, storage) } } -func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, bw *bitWriter) { +func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is_last bool, params *encoderParams, commands []command, storage_ix *uint, storage []byte) { var num_distance_symbols uint32 = params.dist.alphabet_size var distance_alphabet_bits uint32 = log2FloorNonZero(uint(num_distance_symbols-1)) + 1 - storeCompressedMetaBlockHeader(is_last, length, bw) + storeCompressedMetaBlockHeader(is_last, length, storage_ix, storage) - bw.writeBits(13, 0) + writeBits(13, 0, storage_ix, storage) if len(commands) <= 128 { var histogram = [numLiteralSymbols]uint32{0} @@ -1221,11 +1232,11 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is } buildAndStoreHuffmanTreeFast(histogram[:], num_literals, /* max_bits = */ - 8, lit_depth[:], lit_bits[:], bw) + 8, lit_depth[:], lit_bits[:], storage_ix, storage) - storeStaticCommandHuffmanTree(bw) - storeStaticDistanceHuffmanTree(bw) - storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], kStaticCommandCodeDepth[:], kStaticCommandCodeBits[:], kStaticDistanceCodeDepth[:], kStaticDistanceCodeBits[:], bw) + storeStaticCommandHuffmanTree(storage_ix, storage) + storeStaticDistanceHuffmanTree(storage_ix, storage) + storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], kStaticCommandCodeDepth[:], kStaticCommandCodeBits[:], kStaticDistanceCodeDepth[:], kStaticDistanceCodeBits[:], storage_ix, storage) } else { var lit_histo histogramLiteral var cmd_histo histogramCommand @@ -1241,43 +1252,49 @@ func storeMetaBlockFast(input []byte, start_pos uint, length uint, mask uint, is histogramClearDistance(&dist_histo) buildHistograms(input, start_pos, mask, commands, &lit_histo, &cmd_histo, &dist_histo) buildAndStoreHuffmanTreeFast(lit_histo.data_[:], lit_histo.total_count_, /* max_bits = */ - 8, lit_depth[:], lit_bits[:], bw) + 8, lit_depth[:], lit_bits[:], storage_ix, storage) buildAndStoreHuffmanTreeFast(cmd_histo.data_[:], cmd_histo.total_count_, /* max_bits = */ - 10, cmd_depth[:], cmd_bits[:], bw) + 10, cmd_depth[:], cmd_bits[:], storage_ix, storage) buildAndStoreHuffmanTreeFast(dist_histo.data_[:], dist_histo.total_count_, /* max_bits = */ - uint(distance_alphabet_bits), dist_depth[:], dist_bits[:], bw) + uint(distance_alphabet_bits), dist_depth[:], dist_bits[:], storage_ix, storage) - storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], bw) + storeDataWithHuffmanCodes(input, start_pos, mask, commands, lit_depth[:], lit_bits[:], cmd_depth[:], cmd_bits[:], dist_depth[:], dist_bits[:], storage_ix, storage) } if is_last { - bw.jumpToByteBoundary() + jumpToByteBoundary(storage_ix, storage) } } /* This is for storing uncompressed blocks (simple raw storage of bytes-as-bytes). */ -func storeUncompressedMetaBlock(is_final_block bool, input []byte, position uint, mask uint, len uint, bw *bitWriter) { +func storeUncompressedMetaBlock(is_final_block bool, input []byte, position uint, mask uint, len uint, storage_ix *uint, storage []byte) { var masked_pos uint = position & mask - storeUncompressedMetaBlockHeader(uint(len), bw) - bw.jumpToByteBoundary() + storeUncompressedMetaBlockHeader(uint(len), storage_ix, storage) + jumpToByteBoundary(storage_ix, storage) if masked_pos+len > mask+1 { var len1 uint = mask + 1 - masked_pos - bw.writeBytes(input[masked_pos:][:len1]) + copy(storage[*storage_ix>>3:], input[masked_pos:][:len1]) + *storage_ix += len1 << 3 len -= len1 masked_pos = 0 } - bw.writeBytes(input[masked_pos:][:len]) + copy(storage[*storage_ix>>3:], input[masked_pos:][:len]) + *storage_ix += uint(len << 3) + + /* We need to clear the next 4 bytes to continue to be + compatible with BrotliWriteBits. */ + writeBitsPrepareStorage(*storage_ix, storage) /* Since the uncompressed block itself may not be the final block, add an empty one after this. */ if is_final_block { - bw.writeBits(1, 1) /* islast */ - bw.writeBits(1, 1) /* isempty */ - bw.jumpToByteBoundary() + writeBits(1, 1, storage_ix, storage) /* islast */ + writeBits(1, 1, storage_ix, storage) /* isempty */ + jumpToByteBoundary(storage_ix, storage) } } diff --git a/compress_fragment.go b/compress_fragment.go index dbf0c43..2fc2df1 100644 --- a/compress_fragment.go +++ b/compress_fragment.go @@ -45,7 +45,7 @@ func isMatch5(p1 []byte, p2 []byte) bool { and thus have to assign a non-zero depth for each literal. Returns estimated compression ratio millibytes/char for encoding given input with generated code. */ -func buildAndStoreLiteralPrefixCode(input []byte, input_size uint, depths []byte, bits []uint16, bw *bitWriter) uint { +func buildAndStoreLiteralPrefixCode(input []byte, input_size uint, depths []byte, bits []uint16, storage_ix *uint, storage []byte) uint { var histogram = [256]uint32{0} var histogram_total uint var i uint @@ -82,7 +82,7 @@ func buildAndStoreLiteralPrefixCode(input []byte, input_size uint, depths []byte } buildAndStoreHuffmanTreeFast(histogram[:], histogram_total, /* max_bits = */ - 8, depths, bits, bw) + 8, depths, bits, storage_ix, storage) { var literal_ratio uint = 0 for i = 0; i < 256; i++ { @@ -98,7 +98,7 @@ func buildAndStoreLiteralPrefixCode(input []byte, input_size uint, depths []byte /* Builds a command and distance prefix code (each 64 symbols) into "depth" and "bits" based on "histogram" and stores it into the bit stream. */ -func buildAndStoreCommandPrefixCode1(histogram []uint32, depth []byte, bits []uint16, bw *bitWriter) { +func buildAndStoreCommandPrefixCode1(histogram []uint32, depth []byte, bits []uint16, storage_ix *uint, storage []byte) { var tree [129]huffmanTree var cmd_depth = [numCommandSymbols]byte{0} /* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */ @@ -145,141 +145,141 @@ func buildAndStoreCommandPrefixCode1(histogram []uint32, depth []byte, bits []ui cmd_depth[448+8*i] = depth[56+i] } - storeHuffmanTree(cmd_depth[:], numCommandSymbols, tree[:], bw) + storeHuffmanTree(cmd_depth[:], numCommandSymbols, tree[:], storage_ix, storage) } - storeHuffmanTree(depth[64:], 64, tree[:], bw) + storeHuffmanTree(depth[64:], 64, tree[:], storage_ix, storage) } /* REQUIRES: insertlen < 6210 */ -func emitInsertLen1(insertlen uint, depth []byte, bits []uint16, histo []uint32, bw *bitWriter) { +func emitInsertLen1(insertlen uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) { if insertlen < 6 { var code uint = insertlen + 40 - bw.writeBits(uint(depth[code]), uint64(bits[code])) + writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage) histo[code]++ } else if insertlen < 130 { var tail uint = insertlen - 2 var nbits uint32 = log2FloorNonZero(tail) - 1 var prefix uint = tail >> nbits var inscode uint = uint((nbits << 1) + uint32(prefix) + 42) - bw.writeBits(uint(depth[inscode]), uint64(bits[inscode])) - bw.writeBits(uint(nbits), uint64(tail)-(uint64(prefix)<> nbits var code uint = uint((nbits << 1) + uint32(prefix) + 20) - bw.writeBits(uint(depth[code]), uint64(bits[code])) - bw.writeBits(uint(nbits), uint64(tail)-(uint64(prefix)<> nbits var code uint = uint((nbits << 1) + uint32(prefix) + 4) - bw.writeBits(uint(depth[code]), uint64(bits[code])) - bw.writeBits(uint(nbits), uint64(tail)-(uint64(prefix)<> 5) + 30 - bw.writeBits(uint(depth[code]), uint64(bits[code])) - bw.writeBits(5, uint64(tail)&31) - bw.writeBits(uint(depth[64]), uint64(bits[64])) + writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage) + writeBits(5, uint64(tail)&31, storage_ix, storage) + writeBits(uint(depth[64]), uint64(bits[64]), storage_ix, storage) histo[code]++ histo[64]++ } else if copylen < 2120 { var tail uint = copylen - 72 var nbits uint32 = log2FloorNonZero(tail) var code uint = uint(nbits + 28) - bw.writeBits(uint(depth[code]), uint64(bits[code])) - bw.writeBits(uint(nbits), uint64(tail)-(uint64(uint(1))<> nbits) & 1 var offset uint = (2 + prefix) << nbits var distcode uint = uint(2*(nbits-1) + uint32(prefix) + 80) - bw.writeBits(uint(depth[distcode]), uint64(bits[distcode])) - bw.writeBits(uint(nbits), uint64(d)-uint64(offset)) + writeBits(uint(depth[distcode]), uint64(bits[distcode]), storage_ix, storage) + writeBits(uint(nbits), uint64(d)-uint64(offset), storage_ix, storage) histo[distcode]++ } -func emitLiterals(input []byte, len uint, depth []byte, bits []uint16, bw *bitWriter) { +func emitLiterals(input []byte, len uint, depth []byte, bits []uint16, storage_ix *uint, storage []byte) { var j uint for j = 0; j < len; j++ { var lit byte = input[j] - bw.writeBits(uint(depth[lit]), uint64(bits[lit])) + writeBits(uint(depth[lit]), uint64(bits[lit]), storage_ix, storage) } } /* REQUIRES: len <= 1 << 24. */ -func storeMetaBlockHeader1(len uint, is_uncompressed bool, bw *bitWriter) { +func storeMetaBlockHeader1(len uint, is_uncompressed bool, storage_ix *uint, storage []byte) { var nibbles uint = 6 /* ISLAST */ - bw.writeBits(1, 0) + writeBits(1, 0, storage_ix, storage) if len <= 1<<16 { nibbles = 4 @@ -287,11 +287,34 @@ func storeMetaBlockHeader1(len uint, is_uncompressed bool, bw *bitWriter) { nibbles = 5 } - bw.writeBits(2, uint64(nibbles)-4) - bw.writeBits(nibbles*4, uint64(len)-1) + writeBits(2, uint64(nibbles)-4, storage_ix, storage) + writeBits(nibbles*4, uint64(len)-1, storage_ix, storage) /* ISUNCOMPRESSED */ - bw.writeSingleBit(is_uncompressed) + writeSingleBit(is_uncompressed, storage_ix, storage) +} + +func updateBits(n_bits uint, bits uint32, pos uint, array []byte) { + for n_bits > 0 { + var byte_pos uint = pos >> 3 + var n_unchanged_bits uint = pos & 7 + var n_changed_bits uint = brotli_min_size_t(n_bits, 8-n_unchanged_bits) + var total_bits uint = n_unchanged_bits + n_changed_bits + var mask uint32 = (^((1 << total_bits) - 1)) | ((1 << n_unchanged_bits) - 1) + var unchanged_bits uint32 = uint32(array[byte_pos]) & mask + var changed_bits uint32 = bits & ((1 << n_changed_bits) - 1) + array[byte_pos] = byte(changed_bits<>= n_changed_bits + pos += n_changed_bits + } +} + +func rewindBitPosition1(new_storage_ix uint, storage_ix *uint, storage []byte) { + var bitpos uint = new_storage_ix & 7 + var mask uint = (1 << bitpos) - 1 + storage[new_storage_ix>>3] &= byte(mask) + *storage_ix = new_storage_ix } var shouldMergeBlock_kSampleRate uint = 43 @@ -322,26 +345,151 @@ func shouldUseUncompressedMode(metablock_start []byte, next_emit []byte, insertl } } -func emitUncompressedMetaBlock1(data []byte, storage_ix_start uint, bw *bitWriter) { - bw.rewind(storage_ix_start) - storeMetaBlockHeader1(uint(len(data)), true, bw) - bw.jumpToByteBoundary() - bw.writeBytes(data) +func emitUncompressedMetaBlock1(begin []byte, end []byte, storage_ix_start uint, storage_ix *uint, storage []byte) { + var len uint = uint(-cap(end) + cap(begin)) + rewindBitPosition1(storage_ix_start, storage_ix, storage) + storeMetaBlockHeader1(uint(len), true, storage_ix, storage) + *storage_ix = (*storage_ix + 7) &^ 7 + copy(storage[*storage_ix>>3:], begin[:len]) + *storage_ix += uint(len << 3) + storage[*storage_ix>>3] = 0 } var kCmdHistoSeed = [128]uint32{ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 0, 0, 0, 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + 0, } var compressFragmentFastImpl_kFirstBlockSize uint = 3 << 15 var compressFragmentFastImpl_kMergeBlockSize uint = 1 << 16 -func compressFragmentFastImpl(in []byte, input_size uint, is_last bool, table []int, table_bits uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, bw *bitWriter) { +func compressFragmentFastImpl(in []byte, input_size uint, is_last bool, table []int, table_bits uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, storage_ix *uint, storage []byte) { var cmd_histo [128]uint32 var ip_end int var next_emit int = 0 @@ -352,7 +500,7 @@ func compressFragmentFastImpl(in []byte, input_size uint, is_last bool, table [] var metablock_start int = input var block_size uint = brotli_min_size_t(input_size, compressFragmentFastImpl_kFirstBlockSize) var total_block_size uint = block_size - var mlen_storage_ix uint = bw.getPos() + 3 + var mlen_storage_ix uint = *storage_ix + 3 var lit_depth [256]byte var lit_bits [256]uint16 var literal_ratio uint @@ -369,21 +517,21 @@ func compressFragmentFastImpl(in []byte, input_size uint, is_last bool, table [] /* Save the bit position of the MLEN field of the meta-block header, so that we can update it later if we decide to extend this meta-block. */ - storeMetaBlockHeader1(block_size, false, bw) + storeMetaBlockHeader1(block_size, false, storage_ix, storage) /* No block splits, no contexts. */ - bw.writeBits(13, 0) + writeBits(13, 0, storage_ix, storage) - literal_ratio = buildAndStoreLiteralPrefixCode(in[input:], block_size, lit_depth[:], lit_bits[:], bw) + literal_ratio = buildAndStoreLiteralPrefixCode(in[input:], block_size, lit_depth[:], lit_bits[:], storage_ix, storage) { /* Store the pre-compressed command and distance prefix codes. */ var i uint for i = 0; i+7 < *cmd_code_numbits; i += 8 { - bw.writeBits(8, uint64(cmd_code[i>>3])) + writeBits(8, uint64(cmd_code[i>>3]), storage_ix, storage) } } - bw.writeBits(*cmd_code_numbits&7, uint64(cmd_code[*cmd_code_numbits>>3])) + writeBits(*cmd_code_numbits&7, uint64(cmd_code[*cmd_code_numbits>>3]), storage_ix, storage) /* Initialize the command and distance histograms. We will gather statistics of command and distance codes during the processing @@ -482,27 +630,27 @@ emit_commands: var insert uint = uint(base - next_emit) ip += int(matched) if insert < 6210 { - emitInsertLen1(insert, cmd_depth, cmd_bits, cmd_histo[:], bw) + emitInsertLen1(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) } else if shouldUseUncompressedMode(in[metablock_start:], in[next_emit:], insert, literal_ratio) { - emitUncompressedMetaBlock1(in[metablock_start:base], mlen_storage_ix-3, bw) + emitUncompressedMetaBlock1(in[metablock_start:], in[base:], mlen_storage_ix-3, storage_ix, storage) input_size -= uint(base - input) input = base next_emit = input goto next_block } else { - emitLongInsertLen(insert, cmd_depth, cmd_bits, cmd_histo[:], bw) + emitLongInsertLen(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) } - emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], bw) + emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], storage_ix, storage) if distance == last_distance { - bw.writeBits(uint(cmd_depth[64]), uint64(cmd_bits[64])) + writeBits(uint(cmd_depth[64]), uint64(cmd_bits[64]), storage_ix, storage) cmd_histo[64]++ } else { - emitDistance1(uint(distance), cmd_depth, cmd_bits, cmd_histo[:], bw) + emitDistance1(uint(distance), cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) last_distance = distance } - emitCopyLenLastDistance1(matched, cmd_depth, cmd_bits, cmd_histo[:], bw) + emitCopyLenLastDistance1(matched, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) next_emit = ip if ip >= ip_limit { @@ -538,8 +686,8 @@ emit_commands: } ip += int(matched) last_distance = int(base - candidate) /* > 0 */ - emitCopyLen1(matched, cmd_depth, cmd_bits, cmd_histo[:], bw) - emitDistance1(uint(last_distance), cmd_depth, cmd_bits, cmd_histo[:], bw) + emitCopyLen1(matched, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) + emitDistance1(uint(last_distance), cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) next_emit = ip if ip >= ip_limit { @@ -585,7 +733,7 @@ emit_remainder: nibbles. */ total_block_size += block_size - bw.updateBits(20, uint32(total_block_size-1), mlen_storage_ix) + updateBits(20, uint32(total_block_size-1), mlen_storage_ix, storage) goto emit_commands } @@ -593,13 +741,13 @@ emit_remainder: if next_emit < ip_end { var insert uint = uint(ip_end - next_emit) if insert < 6210 { - emitInsertLen1(insert, cmd_depth, cmd_bits, cmd_histo[:], bw) - emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], bw) + emitInsertLen1(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) + emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], storage_ix, storage) } else if shouldUseUncompressedMode(in[metablock_start:], in[next_emit:], insert, literal_ratio) { - emitUncompressedMetaBlock1(in[metablock_start:ip_end], mlen_storage_ix-3, bw) + emitUncompressedMetaBlock1(in[metablock_start:], in[ip_end:], mlen_storage_ix-3, storage_ix, storage) } else { - emitLongInsertLen(insert, cmd_depth, cmd_bits, cmd_histo[:], bw) - emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], bw) + emitLongInsertLen(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) + emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], storage_ix, storage) } } @@ -615,29 +763,30 @@ next_block: /* Save the bit position of the MLEN field of the meta-block header, so that we can update it later if we decide to extend this meta-block. */ - mlen_storage_ix = bw.getPos() + 3 + mlen_storage_ix = *storage_ix + 3 - storeMetaBlockHeader1(block_size, false, bw) + storeMetaBlockHeader1(block_size, false, storage_ix, storage) /* No block splits, no contexts. */ - bw.writeBits(13, 0) + writeBits(13, 0, storage_ix, storage) - literal_ratio = buildAndStoreLiteralPrefixCode(in[input:], block_size, lit_depth[:], lit_bits[:], bw) - buildAndStoreCommandPrefixCode1(cmd_histo[:], cmd_depth, cmd_bits, bw) + literal_ratio = buildAndStoreLiteralPrefixCode(in[input:], block_size, lit_depth[:], lit_bits[:], storage_ix, storage) + buildAndStoreCommandPrefixCode1(cmd_histo[:], cmd_depth, cmd_bits, storage_ix, storage) goto emit_commands } if !is_last { /* If this is not the last block, update the command and distance prefix codes for the next block and store the compressed forms. */ - var bw bitWriter - bw.dst = cmd_code - buildAndStoreCommandPrefixCode1(cmd_histo[:], cmd_depth, cmd_bits, &bw) - *cmd_code_numbits = bw.getPos() + cmd_code[0] = 0 + + *cmd_code_numbits = 0 + buildAndStoreCommandPrefixCode1(cmd_histo[:], cmd_depth, cmd_bits, cmd_code_numbits, cmd_code) } } -/* Compresses "input" string to bw as one or more complete meta-blocks. +/* Compresses "input" string to the "*storage" buffer as one or more complete + meta-blocks, and updates the "*storage_ix" bit position. If "is_last" is 1, emits an additional empty last meta-block. @@ -658,28 +807,28 @@ next_block: REQUIRES: "table_size" is an odd (9, 11, 13, 15) power of two OUTPUT: maximal copy distance <= |input_size| OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */ -func compressFragmentFast(input []byte, input_size uint, is_last bool, table []int, table_size uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, bw *bitWriter) { - var initial_storage_ix uint = bw.getPos() +func compressFragmentFast(input []byte, input_size uint, is_last bool, table []int, table_size uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, storage_ix *uint, storage []byte) { + var initial_storage_ix uint = *storage_ix var table_bits uint = uint(log2FloorNonZero(table_size)) if input_size == 0 { assert(is_last) - bw.writeBits(1, 1) /* islast */ - bw.writeBits(1, 1) /* isempty */ - bw.jumpToByteBoundary() + writeBits(1, 1, storage_ix, storage) /* islast */ + writeBits(1, 1, storage_ix, storage) /* isempty */ + *storage_ix = (*storage_ix + 7) &^ 7 return } - compressFragmentFastImpl(input, input_size, is_last, table, table_bits, cmd_depth, cmd_bits, cmd_code_numbits, cmd_code, bw) + compressFragmentFastImpl(input, input_size, is_last, table, table_bits, cmd_depth, cmd_bits, cmd_code_numbits, cmd_code, storage_ix, storage) /* If output is larger than single uncompressed block, rewrite it. */ - if bw.getPos()-initial_storage_ix > 31+(input_size<<3) { - emitUncompressedMetaBlock1(input[:input_size], initial_storage_ix, bw) + if *storage_ix-initial_storage_ix > 31+(input_size<<3) { + emitUncompressedMetaBlock1(input, input[input_size:], initial_storage_ix, storage_ix, storage) } if is_last { - bw.writeBits(1, 1) /* islast */ - bw.writeBits(1, 1) /* isempty */ - bw.jumpToByteBoundary() + writeBits(1, 1, storage_ix, storage) /* islast */ + writeBits(1, 1, storage_ix, storage) /* isempty */ + *storage_ix = (*storage_ix + 7) &^ 7 } } diff --git a/compress_fragment_two_pass.go b/compress_fragment_two_pass.go index 2473aca..172dc7f 100644 --- a/compress_fragment_two_pass.go +++ b/compress_fragment_two_pass.go @@ -41,7 +41,7 @@ func isMatch1(p1 []byte, p2 []byte, length uint) bool { /* Builds a command and distance prefix code (each 64 symbols) into "depth" and "bits" based on "histogram" and stores it into the bit stream. */ -func buildAndStoreCommandPrefixCode(histogram []uint32, depth []byte, bits []uint16, bw *bitWriter) { +func buildAndStoreCommandPrefixCode(histogram []uint32, depth []byte, bits []uint16, storage_ix *uint, storage []byte) { var tree [129]huffmanTree var cmd_depth = [numCommandSymbols]byte{0} /* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */ @@ -87,10 +87,10 @@ func buildAndStoreCommandPrefixCode(histogram []uint32, depth []byte, bits []uin cmd_depth[448+8*i] = depth[16+i] } - storeHuffmanTree(cmd_depth[:], numCommandSymbols, tree[:], bw) + storeHuffmanTree(cmd_depth[:], numCommandSymbols, tree[:], storage_ix, storage) } - storeHuffmanTree(depth[64:], 64, tree[:], bw) + storeHuffmanTree(depth[64:], 64, tree[:], storage_ix, storage) } func emitInsertLen(insertlen uint32, commands *[]uint32) { @@ -197,11 +197,11 @@ func emitDistance(distance uint32, commands *[]uint32) { } /* REQUIRES: len <= 1 << 24. */ -func storeMetaBlockHeader(len uint, is_uncompressed bool, bw *bitWriter) { +func storeMetaBlockHeader(len uint, is_uncompressed bool, storage_ix *uint, storage []byte) { var nibbles uint = 6 /* ISLAST */ - bw.writeBits(1, 0) + writeBits(1, 0, storage_ix, storage) if len <= 1<<16 { nibbles = 4 @@ -209,11 +209,11 @@ func storeMetaBlockHeader(len uint, is_uncompressed bool, bw *bitWriter) { nibbles = 5 } - bw.writeBits(2, uint64(nibbles)-4) - bw.writeBits(nibbles*4, uint64(len)-1) + writeBits(2, uint64(nibbles)-4, storage_ix, storage) + writeBits(nibbles*4, uint64(len)-1, storage_ix, storage) /* ISUNCOMPRESSED */ - bw.writeSingleBit(is_uncompressed) + writeSingleBit(is_uncompressed, storage_ix, storage) } func createCommands(input []byte, block_size uint, input_size uint, base_ip_ptr []byte, table []int, table_bits uint, min_match uint, literals *[]byte, commands *[]uint32) { @@ -440,20 +440,163 @@ emit_remainder: } var storeCommands_kNumExtraBits = [128]uint32{ - 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 12, 14, 24, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, - 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 24, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, - 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16, - 17, 17, 18, 18, 19, 19, 20, 20, 21, 21, 22, 22, 23, 23, 24, 24, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 2, + 2, + 3, + 3, + 4, + 4, + 5, + 5, + 6, + 7, + 8, + 9, + 10, + 12, + 14, + 24, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 2, + 2, + 3, + 3, + 4, + 4, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 2, + 2, + 3, + 3, + 4, + 4, + 5, + 5, + 6, + 7, + 8, + 9, + 10, + 24, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + 2, + 2, + 3, + 3, + 4, + 4, + 5, + 5, + 6, + 6, + 7, + 7, + 8, + 8, + 9, + 9, + 10, + 10, + 11, + 11, + 12, + 12, + 13, + 13, + 14, + 14, + 15, + 15, + 16, + 16, + 17, + 17, + 18, + 18, + 19, + 19, + 20, + 20, + 21, + 21, + 22, + 22, + 23, + 23, + 24, + 24, } var storeCommands_kInsertOffset = [24]uint32{ - 0, 1, 2, 3, 4, 5, 6, 8, 10, 14, 18, 26, 34, 50, 66, 98, 130, 194, 322, 578, - 1090, 2114, 6210, 22594, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 8, + 10, + 14, + 18, + 26, + 34, + 50, + 66, + 98, + 130, + 194, + 322, + 578, + 1090, + 2114, + 6210, + 22594, } -func storeCommands(literals []byte, num_literals uint, commands []uint32, num_commands uint, bw *bitWriter) { +func storeCommands(literals []byte, num_literals uint, commands []uint32, num_commands uint, storage_ix *uint, storage []byte) { var lit_depths [256]byte var lit_bits [256]uint16 var lit_histo = [256]uint32{0} @@ -466,7 +609,7 @@ func storeCommands(literals []byte, num_literals uint, commands []uint32, num_co } buildAndStoreHuffmanTreeFast(lit_histo[:], num_literals, /* max_bits = */ - 8, lit_depths[:], lit_bits[:], bw) + 8, lit_depths[:], lit_bits[:], storage_ix, storage) for i = 0; i < num_commands; i++ { var code uint32 = commands[i] & 0xFF @@ -478,21 +621,21 @@ func storeCommands(literals []byte, num_literals uint, commands []uint32, num_co cmd_histo[2] += 1 cmd_histo[64] += 1 cmd_histo[84] += 1 - buildAndStoreCommandPrefixCode(cmd_histo[:], cmd_depths[:], cmd_bits[:], bw) + buildAndStoreCommandPrefixCode(cmd_histo[:], cmd_depths[:], cmd_bits[:], storage_ix, storage) for i = 0; i < num_commands; i++ { var cmd uint32 = commands[i] var code uint32 = cmd & 0xFF var extra uint32 = cmd >> 8 assert(code < 128) - bw.writeBits(uint(cmd_depths[code]), uint64(cmd_bits[code])) - bw.writeBits(uint(storeCommands_kNumExtraBits[code]), uint64(extra)) + writeBits(uint(cmd_depths[code]), uint64(cmd_bits[code]), storage_ix, storage) + writeBits(uint(storeCommands_kNumExtraBits[code]), uint64(extra), storage_ix, storage) if code < 24 { var insert uint32 = storeCommands_kInsertOffset[code] + extra var j uint32 for j = 0; j < insert; j++ { var lit byte = literals[0] - bw.writeBits(uint(lit_depths[lit]), uint64(lit_bits[lit])) + writeBits(uint(lit_depths[lit]), uint64(lit_bits[lit]), storage_ix, storage) literals = literals[1:] } } @@ -520,13 +663,22 @@ func shouldCompress(input []byte, input_size uint, num_literals uint) bool { } } -func emitUncompressedMetaBlock(input []byte, input_size uint, bw *bitWriter) { - storeMetaBlockHeader(input_size, true, bw) - bw.jumpToByteBoundary() - bw.writeBytes(input[:input_size]) +func rewindBitPosition(new_storage_ix uint, storage_ix *uint, storage []byte) { + var bitpos uint = new_storage_ix & 7 + var mask uint = (1 << bitpos) - 1 + storage[new_storage_ix>>3] &= byte(mask) + *storage_ix = new_storage_ix } -func compressFragmentTwoPassImpl(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_bits uint, min_match uint, bw *bitWriter) { +func emitUncompressedMetaBlock(input []byte, input_size uint, storage_ix *uint, storage []byte) { + storeMetaBlockHeader(input_size, true, storage_ix, storage) + *storage_ix = (*storage_ix + 7) &^ 7 + copy(storage[*storage_ix>>3:], input[:input_size]) + *storage_ix += input_size << 3 + storage[*storage_ix>>3] = 0 +} + +func compressFragmentTwoPassImpl(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_bits uint, min_match uint, storage_ix *uint, storage []byte) { /* Save the start of the first block for position and distance computations. */ var base_ip []byte = input @@ -540,17 +692,17 @@ func compressFragmentTwoPassImpl(input []byte, input_size uint, is_last bool, co num_literals = uint(-cap(literals) + cap(literal_buf)) if shouldCompress(input, block_size, num_literals) { var num_commands uint = uint(-cap(commands) + cap(command_buf)) - storeMetaBlockHeader(block_size, false, bw) + storeMetaBlockHeader(block_size, false, storage_ix, storage) /* No block splits, no contexts. */ - bw.writeBits(13, 0) + writeBits(13, 0, storage_ix, storage) - storeCommands(literal_buf, num_literals, command_buf, num_commands, bw) + storeCommands(literal_buf, num_literals, command_buf, num_commands, storage_ix, storage) } else { /* Since we did not find many backward references and the entropy of the data is close to 8 bits, we can simply emit an uncompressed block. This makes compression speed of uncompressible data about 3x faster. */ - emitUncompressedMetaBlock(input, block_size, bw) + emitUncompressedMetaBlock(input, block_size, storage_ix, storage) } input = input[block_size:] @@ -558,7 +710,8 @@ func compressFragmentTwoPassImpl(input []byte, input_size uint, is_last bool, co } } -/* Compresses "input" string to bw as one or more complete meta-blocks. +/* Compresses "input" string to the "*storage" buffer as one or more complete + meta-blocks, and updates the "*storage_ix" bit position. If "is_last" is 1, emits an additional empty last meta-block. @@ -570,8 +723,8 @@ func compressFragmentTwoPassImpl(input []byte, input_size uint, is_last bool, co REQUIRES: "table_size" is a power of two OUTPUT: maximal copy distance <= |input_size| OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */ -func compressFragmentTwoPass(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_size uint, bw *bitWriter) { - var initial_storage_ix uint = bw.getPos() +func compressFragmentTwoPass(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_size uint, storage_ix *uint, storage []byte) { + var initial_storage_ix uint = *storage_ix var table_bits uint = uint(log2FloorNonZero(table_size)) var min_match uint if table_bits <= 15 { @@ -579,17 +732,17 @@ func compressFragmentTwoPass(input []byte, input_size uint, is_last bool, comman } else { min_match = 6 } - compressFragmentTwoPassImpl(input, input_size, is_last, command_buf, literal_buf, table, table_bits, min_match, bw) + compressFragmentTwoPassImpl(input, input_size, is_last, command_buf, literal_buf, table, table_bits, min_match, storage_ix, storage) /* If output is larger than single uncompressed block, rewrite it. */ - if bw.getPos()-initial_storage_ix > 31+(input_size<<3) { - bw.rewind(initial_storage_ix) - emitUncompressedMetaBlock(input, input_size, bw) + if *storage_ix-initial_storage_ix > 31+(input_size<<3) { + rewindBitPosition(initial_storage_ix, storage_ix, storage) + emitUncompressedMetaBlock(input, input_size, storage_ix, storage) } if is_last { - bw.writeBits(1, 1) /* islast */ - bw.writeBits(1, 1) /* isempty */ - bw.jumpToByteBoundary() + writeBits(1, 1, storage_ix, storage) /* islast */ + writeBits(1, 1, storage_ix, storage) /* isempty */ + *storage_ix = (*storage_ix + 7) &^ 7 } } diff --git a/encode.go b/encode.go index 3abaf57..8f9eaa4 100644 --- a/encode.go +++ b/encode.go @@ -87,9 +87,11 @@ type Writer struct { last_processed_pos_ uint64 dist_cache_ [numDistanceShortCodes]int saved_dist_cache_ [4]int + last_bytes_ uint16 + last_bytes_bits_ byte prev_byte_ byte prev_byte2_ byte - bw bitWriter + storage []byte small_table_ [1 << 10]int large_table_ []int large_table_size_ uint @@ -139,6 +141,14 @@ func wrapPosition(position uint64) uint32 { return result } +func (s *Writer) getStorage(size int) []byte { + if len(s.storage) < size { + s.storage = make([]byte, size) + } + + return s.storage +} + func hashTableSize(max_table_size uint, input_size uint) uint { var htsize uint = 256 for htsize < max_table_size && htsize < input_size { @@ -184,18 +194,23 @@ func getHashTable(s *Writer, quality int, input_size uint, table_size *uint) []i return table } -func encodeWindowBits(lgwin int, large_window bool, bw *bitWriter) { +func encodeWindowBits(lgwin int, large_window bool, last_bytes *uint16, last_bytes_bits *byte) { if large_window { - bw.writeBits(14, uint64((lgwin&0x3F)<<8|0x11)) + *last_bytes = uint16((lgwin&0x3F)<<8 | 0x11) + *last_bytes_bits = 14 } else { if lgwin == 16 { - bw.writeBits(1, 0) + *last_bytes = 0 + *last_bytes_bits = 1 } else if lgwin == 17 { - bw.writeBits(7, 1) + *last_bytes = 1 + *last_bytes_bits = 7 } else if lgwin > 17 { - bw.writeBits(4, uint64((lgwin-17)<<1|0x01)) + *last_bytes = uint16((lgwin-17)<<1 | 0x01) + *last_bytes_bits = 4 } else { - bw.writeBits(7, uint64((lgwin-8)<<4|0x01)) + *last_bytes = uint16((lgwin-8)<<4 | 0x01) + *last_bytes_bits = 7 } } } @@ -417,15 +432,18 @@ func chooseContextMode(params *encoderParams, data []byte, pos uint, mask uint, return contextUTF8 } -func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes uint, is_last bool, literal_context_mode int, params *encoderParams, prev_byte byte, prev_byte2 byte, num_literals uint, commands []command, saved_dist_cache []int, dist_cache []int, bw *bitWriter) { +func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes uint, is_last bool, literal_context_mode int, params *encoderParams, prev_byte byte, prev_byte2 byte, num_literals uint, commands []command, saved_dist_cache []int, dist_cache []int, storage_ix *uint, storage []byte) { var wrapped_last_flush_pos uint32 = wrapPosition(last_flush_pos) + var last_bytes uint16 + var last_bytes_bits byte var literal_context_lut contextLUT = getContextLUT(literal_context_mode) var block_params encoderParams = *params if bytes == 0 { /* Write the ISLAST and ISEMPTY bits. */ - bw.writeBits(2, 3) - bw.jumpToByteBoundary() + writeBits(2, 3, storage_ix, storage) + + *storage_ix = (*storage_ix + 7) &^ 7 return } @@ -434,15 +452,17 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes CreateBackwardReferences is now unused. */ copy(dist_cache, saved_dist_cache[:4]) - storeUncompressedMetaBlock(is_last, data, uint(wrapped_last_flush_pos), mask, bytes, bw) + storeUncompressedMetaBlock(is_last, data, uint(wrapped_last_flush_pos), mask, bytes, storage_ix, storage) return } - savedPos := bw.getPos() + assert(*storage_ix <= 14) + last_bytes = uint16(storage[1])<<8 | uint16(storage[0]) + last_bytes_bits = byte(*storage_ix) if params.quality <= maxQualityForStaticEntropyCodes { - storeMetaBlockFast(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, bw) + storeMetaBlockFast(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, storage_ix, storage) } else if params.quality < minQualityForBlockSplit { - storeMetaBlockTrivial(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, bw) + storeMetaBlockTrivial(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, storage_ix, storage) } else { mb := getMetaBlockSplit() if params.quality < minQualityForHqBlockSplitting { @@ -469,15 +489,18 @@ func writeMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes optimizeHistograms(num_effective_dist_codes, mb) } - storeMetaBlock(data, uint(wrapped_last_flush_pos), bytes, mask, prev_byte, prev_byte2, is_last, &block_params, literal_context_mode, commands, mb, bw) + storeMetaBlock(data, uint(wrapped_last_flush_pos), bytes, mask, prev_byte, prev_byte2, is_last, &block_params, literal_context_mode, commands, mb, storage_ix, storage) freeMetaBlockSplit(mb) } - if bytes+4 < bw.getPos()>>3 { + if bytes+4 < *storage_ix>>3 { /* Restore the distance cache and last byte. */ copy(dist_cache, saved_dist_cache[:4]) - bw.rewind(savedPos) - storeUncompressedMetaBlock(is_last, data, uint(wrapped_last_flush_pos), mask, bytes, bw) + + storage[0] = byte(last_bytes) + storage[1] = byte(last_bytes >> 8) + *storage_ix = uint(last_bytes_bits) + storeUncompressedMetaBlock(is_last, data, uint(wrapped_last_flush_pos), mask, bytes, storage_ix, storage) } } @@ -510,10 +533,8 @@ func ensureInitialized(s *Writer) bool { return true } - s.bw.bits = 0 - s.bw.nbits = 0 - s.bw.dst = s.bw.dst[:0] - + s.last_bytes_bits_ = 0 + s.last_bytes_ = 0 s.remaining_metadata_bytes_ = math.MaxUint32 sanitizeParams(&s.params) @@ -529,7 +550,7 @@ func ensureInitialized(s *Writer) bool { lgwin = brotli_max_int(lgwin, 18) } - encodeWindowBits(lgwin, s.params.large_window, &s.bw) + encodeWindowBits(lgwin, s.params.large_window, &s.last_bytes_, &s.last_bytes_bits_) } if s.params.quality == fastOnePassCompressionQuality { @@ -761,6 +782,8 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool { } if s.params.quality == fastOnePassCompressionQuality || s.params.quality == fastTwoPassCompressionQuality { + var storage []byte + var storage_ix uint = uint(s.last_bytes_bits_) var table_size uint var table []int @@ -770,16 +793,20 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool { return true } + storage = s.getStorage(int(2*bytes + 503)) + storage[0] = byte(s.last_bytes_) + storage[1] = byte(s.last_bytes_ >> 8) table = getHashTable(s, s.params.quality, uint(bytes), &table_size) if s.params.quality == fastOnePassCompressionQuality { - compressFragmentFast(data[wrapped_last_processed_pos&mask:], uint(bytes), is_last, table, table_size, s.cmd_depths_[:], s.cmd_bits_[:], &s.cmd_code_numbits_, s.cmd_code_[:], &s.bw) + compressFragmentFast(data[wrapped_last_processed_pos&mask:], uint(bytes), is_last, table, table_size, s.cmd_depths_[:], s.cmd_bits_[:], &s.cmd_code_numbits_, s.cmd_code_[:], &storage_ix, storage) } else { - compressFragmentTwoPass(data[wrapped_last_processed_pos&mask:], uint(bytes), is_last, s.command_buf_, s.literal_buf_, table, table_size, &s.bw) + compressFragmentTwoPass(data[wrapped_last_processed_pos&mask:], uint(bytes), is_last, s.command_buf_, s.literal_buf_, table, table_size, &storage_ix, storage) } + s.last_bytes_ = uint16(storage[storage_ix>>3]) + s.last_bytes_bits_ = byte(storage_ix & 7) updateLastProcessedPos(s) - s.writeOutput(s.bw.dst) - s.bw.dst = s.bw.dst[:0] + s.writeOutput(storage[:storage_ix>>3]) return true } { @@ -856,7 +883,13 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool { assert(s.input_pos_-s.last_flush_pos_ <= 1<<24) { var metablock_size uint32 = uint32(s.input_pos_ - s.last_flush_pos_) - writeMetaBlockInternal(data, uint(mask), s.last_flush_pos_, uint(metablock_size), is_last, literal_context_mode, &s.params, s.prev_byte_, s.prev_byte2_, s.num_literals_, s.commands, s.saved_dist_cache_[:], s.dist_cache_[:], &s.bw) + var storage []byte = s.getStorage(int(2*metablock_size + 503)) + var storage_ix uint = uint(s.last_bytes_bits_) + storage[0] = byte(s.last_bytes_) + storage[1] = byte(s.last_bytes_ >> 8) + writeMetaBlockInternal(data, uint(mask), s.last_flush_pos_, uint(metablock_size), is_last, literal_context_mode, &s.params, s.prev_byte_, s.prev_byte2_, s.num_literals_, s.commands, s.saved_dist_cache_[:], s.dist_cache_[:], &storage_ix, storage) + s.last_bytes_ = uint16(storage[storage_ix>>3]) + s.last_bytes_bits_ = byte(storage_ix & 7) s.last_flush_pos_ = s.input_pos_ if updateLastProcessedPos(s) { hasherReset(s.hasher_) @@ -877,22 +910,28 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool { emitting an uncompressed block. */ copy(s.saved_dist_cache_[:], s.dist_cache_[:]) - s.writeOutput(s.bw.dst) - s.bw.dst = s.bw.dst[:0] + s.writeOutput(storage[:storage_ix>>3]) return true } } -/* Dumps remaining output bits and metadata header to s.bw. +/* Dumps remaining output bits and metadata header to |header|. + Returns number of produced bytes. + REQUIRED: |header| should be 8-byte aligned and at least 16 bytes long. REQUIRED: |block_size| <= (1 << 24). */ -func writeMetadataHeader(s *Writer, block_size uint) { - bw := &s.bw +func writeMetadataHeader(s *Writer, block_size uint, header []byte) uint { + var storage_ix uint + storage_ix = uint(s.last_bytes_bits_) + header[0] = byte(s.last_bytes_) + header[1] = byte(s.last_bytes_ >> 8) + s.last_bytes_ = 0 + s.last_bytes_bits_ = 0 - bw.writeBits(1, 0) - bw.writeBits(2, 3) - bw.writeBits(1, 0) + writeBits(1, 0, &storage_ix, header) + writeBits(2, 3, &storage_ix, header) + writeBits(1, 0, &storage_ix, header) if block_size == 0 { - bw.writeBits(2, 0) + writeBits(2, 0, &storage_ix, header) } else { var nbits uint32 if block_size == 1 { @@ -901,19 +940,34 @@ func writeMetadataHeader(s *Writer, block_size uint) { nbits = log2FloorNonZero(uint(uint32(block_size)-1)) + 1 } var nbytes uint32 = (nbits + 7) / 8 - bw.writeBits(2, uint64(nbytes)) - bw.writeBits(uint(8*nbytes), uint64(block_size)-1) + writeBits(2, uint64(nbytes), &storage_ix, header) + writeBits(uint(8*nbytes), uint64(block_size)-1, &storage_ix, header) } - bw.jumpToByteBoundary() + return (storage_ix + 7) >> 3 } func injectBytePaddingBlock(s *Writer) { + var seal uint32 = uint32(s.last_bytes_) + var seal_bits uint = uint(s.last_bytes_bits_) + s.last_bytes_ = 0 + s.last_bytes_bits_ = 0 + /* is_last = 0, data_nibbles = 11, reserved = 0, meta_nibbles = 00 */ - s.bw.writeBits(6, 0x6) - s.bw.jumpToByteBoundary() - s.writeOutput(s.bw.dst) - s.bw.dst = s.bw.dst[:0] + seal |= 0x6 << seal_bits + + seal_bits += 6 + + destination := s.tiny_buf_.u8[:] + + destination[0] = byte(seal) + if seal_bits > 8 { + destination[1] = byte(seal >> 8) + } + if seal_bits > 16 { + destination[2] = byte(seal >> 16) + } + s.writeOutput(destination[:(seal_bits+7)>>3]) } func checkFlushComplete(s *Writer) { @@ -945,7 +999,7 @@ func encoderCompressStreamFast(s *Writer, op int, available_in *uint, next_in *[ } for { - if s.stream_state_ == streamFlushRequested && s.bw.nbits&7 != 0 { + if s.stream_state_ == streamFlushRequested && s.last_bytes_bits_ != 0 { injectBytePaddingBlock(s) continue } @@ -957,6 +1011,9 @@ func encoderCompressStreamFast(s *Writer, op int, available_in *uint, next_in *[ var block_size uint = brotli_min_size_t(block_size_limit, *available_in) var is_last bool = (*available_in == block_size) && (op == int(operationFinish)) var force_flush bool = (*available_in == block_size) && (op == int(operationFlush)) + var max_out_size uint = 2*block_size + 503 + var storage []byte = nil + var storage_ix uint = uint(s.last_bytes_bits_) var table_size uint var table []int @@ -965,18 +1022,25 @@ func encoderCompressStreamFast(s *Writer, op int, available_in *uint, next_in *[ continue } + storage = s.getStorage(int(max_out_size)) + + storage[0] = byte(s.last_bytes_) + storage[1] = byte(s.last_bytes_ >> 8) table = getHashTable(s, s.params.quality, block_size, &table_size) if s.params.quality == fastOnePassCompressionQuality { - compressFragmentFast(*next_in, block_size, is_last, table, table_size, s.cmd_depths_[:], s.cmd_bits_[:], &s.cmd_code_numbits_, s.cmd_code_[:], &s.bw) + compressFragmentFast(*next_in, block_size, is_last, table, table_size, s.cmd_depths_[:], s.cmd_bits_[:], &s.cmd_code_numbits_, s.cmd_code_[:], &storage_ix, storage) } else { - compressFragmentTwoPass(*next_in, block_size, is_last, command_buf, literal_buf, table, table_size, &s.bw) + compressFragmentTwoPass(*next_in, block_size, is_last, command_buf, literal_buf, table, table_size, &storage_ix, storage) } *next_in = (*next_in)[block_size:] *available_in -= block_size - s.writeOutput(s.bw.dst) - s.bw.dst = s.bw.dst[:0] + var out_bytes uint = storage_ix >> 3 + s.writeOutput(storage[:out_bytes]) + + s.last_bytes_ = uint16(storage[storage_ix>>3]) + s.last_bytes_bits_ = byte(storage_ix & 7) if force_flush { s.stream_state_ = streamFlushRequested @@ -1010,7 +1074,7 @@ func processMetadata(s *Writer, available_in *uint, next_in *[]byte) bool { } for { - if s.stream_state_ == streamFlushRequested && s.bw.nbits&7 != 0 { + if s.stream_state_ == streamFlushRequested && s.last_bytes_bits_ != 0 { injectBytePaddingBlock(s) continue } @@ -1024,9 +1088,8 @@ func processMetadata(s *Writer, available_in *uint, next_in *[]byte) bool { } if s.stream_state_ == streamMetadataHead { - writeMetadataHeader(s, uint(s.remaining_metadata_bytes_)) - s.writeOutput(s.bw.dst) - s.bw.dst = s.bw.dst[:0] + n := writeMetadataHeader(s, uint(s.remaining_metadata_bytes_), s.tiny_buf_.u8[:]) + s.writeOutput(s.tiny_buf_.u8[:n]) s.stream_state_ = streamMetadataBody continue } else { @@ -1112,7 +1175,7 @@ func encoderCompressStream(s *Writer, op int, available_in *uint, next_in *[]byt continue } - if s.stream_state_ == streamFlushRequested && s.bw.nbits&7 != 0 { + if s.stream_state_ == streamFlushRequested && s.last_bytes_bits_ != 0 { injectBytePaddingBlock(s) continue } diff --git a/entropy_encode_static.go b/entropy_encode_static.go index 2543f8f..5ddf3fc 100644 --- a/entropy_encode_static.go +++ b/entropy_encode_static.go @@ -778,9 +778,8 @@ var kStaticDistanceCodeDepth = [64]byte{ var kCodeLengthBits = [18]uint32{0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 15, 31, 0, 11, 7} -func storeStaticCodeLengthCode(bw *bitWriter) { - bw.writeBits(32, 0x55555554) - bw.writeBits(8, 0xFF) +func storeStaticCodeLengthCode(storage_ix *uint, storage []byte) { + writeBits(40, 0x0000FF55555554, storage_ix, storage) } var kZeroRepsBits = [numCommandSymbols]uint64{ @@ -4318,10 +4317,9 @@ var kStaticCommandCodeBits = [numCommandSymbols]uint16{ 2047, } -func storeStaticCommandHuffmanTree(bw *bitWriter) { - bw.writeBits(32, 0x16307003) - bw.writeBits(24, 0x926244) - bw.writeBits(3, 0x00000000) +func storeStaticCommandHuffmanTree(storage_ix *uint, storage []byte) { + writeBits(56, 0x92624416307003, storage_ix, storage) + writeBits(3, 0x00000000, storage_ix, storage) } var kStaticDistanceCodeBits = [64]uint16{ @@ -4391,6 +4389,6 @@ var kStaticDistanceCodeBits = [64]uint16{ 63, } -func storeStaticDistanceHuffmanTree(bw *bitWriter) { - bw.writeBits(28, 0x0369DC03) +func storeStaticDistanceHuffmanTree(storage_ix *uint, storage []byte) { + writeBits(28, 0x0369DC03, storage_ix, storage) } diff --git a/write_bits.go b/write_bits.go index 2d216d7..8729901 100644 --- a/write_bits.go +++ b/write_bits.go @@ -1,5 +1,7 @@ package brotli +import "encoding/binary" + /* Copyright 2010 Google Inc. All Rights Reserved. Distributed under MIT license. @@ -8,87 +10,43 @@ package brotli /* Write bits into a byte array. */ -type bitWriter struct { - dst []byte +/* This function writes bits into bytes in increasing addresses, and within + a byte least-significant-bit first. - // Data waiting to be written is the low nbits of bits. - bits uint64 - nbits uint + The function can write up to 56 bits in one go with WriteBits + Example: let's assume that 3 bits (Rs below) have been written already: + + BYTE-0 BYTE+1 BYTE+2 + + 0000 0RRR 0000 0000 0000 0000 + + Now, we could write 5 or less bits in MSB by just sifting by 3 + and OR'ing to BYTE-0. + + For n bits, we take the last 5 bits, OR that with high bits in BYTE-0, + and locate the rest in BYTE+1, BYTE+2, etc. */ +func writeBits(n_bits uint, bits uint64, pos *uint, array []byte) { + /* This branch of the code can write up to 56 bits at a time, + 7 bits are lost by being perhaps already in *p and at least + 1 bit is needed to initialize the bit-stream ahead (i.e. if 7 + bits are in *p and we write 57 bits, then the next write will + access a byte that was never initialized). */ + p := array[*pos>>3:] + v := uint64(p[0]) + v |= bits << (*pos & 7) + binary.LittleEndian.PutUint64(p, v) + *pos += n_bits } -func (w *bitWriter) writeBits(nb uint, b uint64) { - w.bits |= b << w.nbits - w.nbits += nb - if w.nbits >= 32 { - bits := w.bits - w.bits >>= 32 - w.nbits -= 32 - w.dst = append(w.dst, - byte(bits), - byte(bits>>8), - byte(bits>>16), - byte(bits>>24), - ) - } -} - -func (w *bitWriter) writeSingleBit(bit bool) { +func writeSingleBit(bit bool, pos *uint, array []byte) { if bit { - w.writeBits(1, 1) + writeBits(1, 1, pos, array) } else { - w.writeBits(1, 0) + writeBits(1, 0, pos, array) } } -func (w *bitWriter) jumpToByteBoundary() { - dst := w.dst - for w.nbits != 0 { - dst = append(dst, byte(w.bits)) - w.bits >>= 8 - if w.nbits > 8 { // Avoid underflow - w.nbits -= 8 - } else { - w.nbits = 0 - } - } - w.bits = 0 - w.dst = dst -} - -func (w *bitWriter) writeBytes(b []byte) { - if w.nbits&7 != 0 { - panic("writeBytes with unfinished bits") - } - for w.nbits != 0 { - w.dst = append(w.dst, byte(w.bits)) - w.bits >>= 8 - w.nbits -= 8 - } - w.dst = append(w.dst, b...) -} - -func (w *bitWriter) getPos() uint { - return uint(len(w.dst)<<3) + w.nbits -} - -func (w *bitWriter) rewind(p uint) { - w.bits = uint64(w.dst[p>>3] & byte((1<<(p&7))-1)) - w.nbits = p & 7 - w.dst = w.dst[:p>>3] -} - -func (w *bitWriter) updateBits(n_bits uint, bits uint32, pos uint) { - for n_bits > 0 { - var byte_pos uint = pos >> 3 - var n_unchanged_bits uint = pos & 7 - var n_changed_bits uint = brotli_min_size_t(n_bits, 8-n_unchanged_bits) - var total_bits uint = n_unchanged_bits + n_changed_bits - var mask uint32 = (^((1 << total_bits) - 1)) | ((1 << n_unchanged_bits) - 1) - var unchanged_bits uint32 = uint32(w.dst[byte_pos]) & mask - var changed_bits uint32 = bits & ((1 << n_changed_bits) - 1) - w.dst[byte_pos] = byte(changed_bits<>= n_changed_bits - pos += n_changed_bits - } +func writeBitsPrepareStorage(pos uint, array []byte) { + assert(pos&7 == 0) + array[pos>>3] = 0 }