From 52165d4974b047fa58ce73396e65ea60040a865e Mon Sep 17 00:00:00 2001 From: Andy Balholm Date: Thu, 21 Mar 2019 15:34:30 -0700 Subject: [PATCH] Clean up duplicate and misplaced comments. --- backward_references.go | 15 +-- backward_references_hq.go | 29 ++--- bit_cost.go | 7 -- bit_reader.go | 1 + block_splitter.go | 15 +-- block_splitter_command.go | 2 +- block_splitter_distance.go | 2 +- block_splitter_literal.go | 2 +- cluster.go | 15 +-- cluster_command.go | 1 - cluster_distance.go | 1 - cluster_literal.go | 1 - compress_fragment.go | 58 +++++----- compress_fragment_common.go | 24 ----- compress_fragment_two_pass.go | 27 ++--- context.go | 177 +------------------------------ decode.go | 33 ------ dictionary.go | 14 --- dictionary_hash.go | 7 -- encode.go | 13 --- entropy_encode.go | 48 +++------ fast_log.go | 21 +--- find_match_length.go | 8 -- h10.go | 13 ++- h5.go | 1 - h6.go | 1 - hash.go | 24 +---- hash_composite.go | 7 +- hash_forgetful_chain.go | 13 ++- hash_longest_match_quickly.go | 2 +- hash_rolling.go | 10 +- huffman.go | 1 + memory.go | 2 - metablock.go | 32 +----- metablock_command.go | 1 - metablock_distance.go | 1 - metablock_literal.go | 1 - params.go | 192 ---------------------------------- platform.go | 24 ----- prefix.go | 21 ---- ringbuffer.go | 9 -- static_dict.go | 1 + static_dict_lut.go | 11 +- symbol_list.go | 1 + transform.go | 21 ---- write_bits.go | 37 ------- 46 files changed, 107 insertions(+), 840 deletions(-) delete mode 100644 compress_fragment_common.go diff --git a/backward_references.go b/backward_references.go index 74d47ed..1d4ce86 100644 --- a/backward_references.go +++ b/backward_references.go @@ -7,18 +7,7 @@ package brotli */ /* Function to find backward reference copies. */ -/* Copyright 2013 Google Inc. All Rights Reserved. - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - -/* Function to find backward reference copies. */ - -/* "commands" points to the next output command to write to, "*num_commands" is - initially the total amount of commands output by previous - CreateBackwardReferences calls, and must be incremented by the amount written - by this call. */ func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uint { if distance <= max_distance { var distance_plus_3 uint = distance + 3 @@ -42,6 +31,10 @@ func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uin return distance + numDistanceShortCodes - 1 } +/* "commands" points to the next output command to write to, "*num_commands" is + initially the total amount of commands output by previous + CreateBackwardReferences calls, and must be incremented by the amount written + by this call. */ func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) { var max_backward_limit uint = maxBackwardLimit(params.lgwin) var orig_commands []command = commands diff --git a/backward_references_hq.go b/backward_references_hq.go index 7f598a5..2323285 100644 --- a/backward_references_hq.go +++ b/backward_references_hq.go @@ -13,19 +13,6 @@ type zopfliNode struct { } } -/* Computes the shortest path of commands from position to at most - position + num_bytes. - - On return, path->size() is the number of commands found and path[i] is the - length of the i-th command (copy length plus insert length). - Note that the sum of the lengths of all commands can be less than num_bytes. - - On return, the nodes[0..num_bytes] array will have the following - "ZopfliNode array invariant": - For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then - (1) nodes[i].copy_length() >= 2 - (2) nodes[i].command_length() <= i and - (3) nodes[i - nodes[i].command_length()].cost < kInfinity */ const maxEffectiveDistanceAlphabetSize = 544 var kInfinity float32 = 1.7e38 /* ~= 2 ^ 127 */ @@ -618,7 +605,21 @@ func zopfliIterate(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_ return computeShortestPathFromNodes(num_bytes, nodes) } -/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */ +/* Computes the shortest path of commands from position to at most + position + num_bytes. + + On return, path->size() is the number of commands found and path[i] is the + length of the i-th command (copy length plus insert length). + Note that the sum of the lengths of all commands can be less than num_bytes. + + On return, the nodes[0..num_bytes] array will have the following + "ZopfliNode array invariant": + For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then + (1) nodes[i].copy_length() >= 2 + (2) nodes[i].command_length() <= i and + (3) nodes[i - nodes[i].command_length()].cost < kInfinity + + REQUIRES: nodes != nil and len(nodes) >= num_bytes + 1 */ func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, dist_cache []int, hasher *h10, nodes []zopfliNode) uint { var max_backward_limit uint = maxBackwardLimit(params.lgwin) var max_zopfli_len uint = maxZopfliLen(params) diff --git a/bit_cost.go b/bit_cost.go index c6eaad7..0005fc1 100644 --- a/bit_cost.go +++ b/bit_cost.go @@ -6,13 +6,6 @@ package brotli See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* Functions to estimate the bit cost of Huffman trees. */ -/* Copyright 2013 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - /* Functions to estimate the bit cost of Huffman trees. */ func shannonEntropy(population []uint32, size uint, total *uint) float64 { var sum uint = 0 diff --git a/bit_reader.go b/bit_reader.go index f69b12d..fba8687 100644 --- a/bit_reader.go +++ b/bit_reader.go @@ -9,6 +9,7 @@ import "encoding/binary" */ /* Bit reading helpers */ + const shortFillBitWindowRead = (8 >> 1) var kBitMask = [33]uint32{ diff --git a/block_splitter.go b/block_splitter.go index ab37ff4..1fc1cd8 100644 --- a/block_splitter.go +++ b/block_splitter.go @@ -6,21 +6,8 @@ package brotli See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* Build per-context histograms of literals, commands and distance codes. */ -/* Copyright 2013 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - -/* Models the histograms of literals, commands and distance codes. */ -/* Copyright 2013 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - /* Block split point selection utilities. */ + type blockSplit struct { num_types uint num_blocks uint diff --git a/block_splitter_command.go b/block_splitter_command.go index 7c34239..e505fe1 100644 --- a/block_splitter_command.go +++ b/block_splitter_command.go @@ -2,12 +2,12 @@ package brotli import "math" -/* NOLINT(build/header_guard) */ /* Copyright 2013 Google Inc. All Rights Reserved. Distributed under MIT license. See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ + func initialEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) { var seed uint32 = 7 var block_length uint = length / num_histograms diff --git a/block_splitter_distance.go b/block_splitter_distance.go index 5eb1ce1..953530d 100644 --- a/block_splitter_distance.go +++ b/block_splitter_distance.go @@ -2,12 +2,12 @@ package brotli import "math" -/* NOLINT(build/header_guard) */ /* Copyright 2013 Google Inc. All Rights Reserved. Distributed under MIT license. See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ + func initialEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) { var seed uint32 = 7 var block_length uint = length / num_histograms diff --git a/block_splitter_literal.go b/block_splitter_literal.go index 89ca883..1c895cf 100644 --- a/block_splitter_literal.go +++ b/block_splitter_literal.go @@ -2,12 +2,12 @@ package brotli import "math" -/* NOLINT(build/header_guard) */ /* Copyright 2013 Google Inc. All Rights Reserved. Distributed under MIT license. See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ + func initialEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) { var seed uint32 = 7 var block_length uint = length / num_histograms diff --git a/cluster.go b/cluster.go index 1cd56d6..df8a328 100644 --- a/cluster.go +++ b/cluster.go @@ -6,14 +6,8 @@ package brotli See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* Block split point selection utilities. */ -/* Copyright 2013 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - /* Functions for clustering similar histograms together. */ + type histogramPair struct { idx1 uint32 idx2 uint32 @@ -21,13 +15,6 @@ type histogramPair struct { cost_diff float64 } -/* Copyright 2013 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - -/* Functions for clustering similar histograms together. */ func histogramPairIsLess(p1 *histogramPair, p2 *histogramPair) bool { if p1.cost_diff != p2.cost_diff { return p1.cost_diff > p2.cost_diff diff --git a/cluster_command.go b/cluster_command.go index e9b24b0..7449751 100644 --- a/cluster_command.go +++ b/cluster_command.go @@ -2,7 +2,6 @@ package brotli import "math" -/* NOLINT(build/header_guard) */ /* Copyright 2013 Google Inc. All Rights Reserved. Distributed under MIT license. diff --git a/cluster_distance.go b/cluster_distance.go index b987da6..1aaa86e 100644 --- a/cluster_distance.go +++ b/cluster_distance.go @@ -2,7 +2,6 @@ package brotli import "math" -/* NOLINT(build/header_guard) */ /* Copyright 2013 Google Inc. All Rights Reserved. Distributed under MIT license. diff --git a/cluster_literal.go b/cluster_literal.go index dc4b4db..6ba66f3 100644 --- a/cluster_literal.go +++ b/cluster_literal.go @@ -2,7 +2,6 @@ package brotli import "math" -/* NOLINT(build/header_guard) */ /* Copyright 2013 Google Inc. All Rights Reserved. Distributed under MIT license. diff --git a/compress_fragment.go b/compress_fragment.go index 6952007..1f6520a 100644 --- a/compress_fragment.go +++ b/compress_fragment.go @@ -15,41 +15,11 @@ import "encoding/binary" Adapted from the CompressFragment() function in https://github.com/google/snappy/blob/master/snappy.cc */ -/* Copyright 2015 Google Inc. All Rights Reserved. - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ +const maxDistance_compress_fragment = 262128 -/* Function for fast encoding of an input fragment, independently from the input - history. This function uses one-pass processing: when we find a backward - match, we immediately emit the corresponding command and literal codes to - the bit stream. */ - -/* Compresses "input" string to the "*storage" buffer as one or more complete - meta-blocks, and updates the "*storage_ix" bit position. - - If "is_last" is 1, emits an additional empty last meta-block. - - "cmd_depth" and "cmd_bits" contain the command and distance prefix codes - (see comment in encode.h) used for the encoding of this input fragment. - If "is_last" is 0, they are updated to reflect the statistics - of this input fragment, to be used for the encoding of the next fragment. - - "*cmd_code_numbits" is the number of bits of the compressed representation - of the command and distance prefix codes, and "cmd_code" is an array of - at least "(*cmd_code_numbits + 7) >> 3" size that contains the compressed - command and distance prefix codes. If "is_last" is 0, these are also - updated to represent the updated "cmd_depth" and "cmd_bits". - - REQUIRES: "input_size" is greater than zero, or "is_last" is 1. - REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24). - REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero. - REQUIRES: "table_size" is an odd (9, 11, 13, 15) power of two - OUTPUT: maximal copy distance <= |input_size| - OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */ func hash5(p []byte, shift uint) uint32 { - var h uint64 = (binary.LittleEndian.Uint64(p) << 24) * uint64(kHashMul32_a) + var h uint64 = (binary.LittleEndian.Uint64(p) << 24) * uint64(kHashMul32) return uint32(h >> shift) } @@ -57,7 +27,7 @@ func hashBytesAtOffset5(v uint64, offset int, shift uint) uint32 { assert(offset >= 0) assert(offset <= 3) { - var h uint64 = ((v >> uint(8*offset)) << 24) * uint64(kHashMul32_a) + var h uint64 = ((v >> uint(8*offset)) << 24) * uint64(kHashMul32) return uint32(h >> shift) } } @@ -821,6 +791,28 @@ next_block: } } +/* Compresses "input" string to the "*storage" buffer as one or more complete + meta-blocks, and updates the "*storage_ix" bit position. + + If "is_last" is 1, emits an additional empty last meta-block. + + "cmd_depth" and "cmd_bits" contain the command and distance prefix codes + (see comment in encode.h) used for the encoding of this input fragment. + If "is_last" is 0, they are updated to reflect the statistics + of this input fragment, to be used for the encoding of the next fragment. + + "*cmd_code_numbits" is the number of bits of the compressed representation + of the command and distance prefix codes, and "cmd_code" is an array of + at least "(*cmd_code_numbits + 7) >> 3" size that contains the compressed + command and distance prefix codes. If "is_last" is 0, these are also + updated to represent the updated "cmd_depth" and "cmd_bits". + + REQUIRES: "input_size" is greater than zero, or "is_last" is 1. + REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24). + REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero. + REQUIRES: "table_size" is an odd (9, 11, 13, 15) power of two + OUTPUT: maximal copy distance <= |input_size| + OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */ func compressFragmentFast(input []byte, input_size uint, is_last bool, table []int, table_size uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, storage_ix *uint, storage []byte) { var initial_storage_ix uint = *storage_ix var table_bits uint = uint(log2FloorNonZero(table_size)) diff --git a/compress_fragment_common.go b/compress_fragment_common.go deleted file mode 100644 index 1d74fe1..0000000 --- a/compress_fragment_common.go +++ /dev/null @@ -1,24 +0,0 @@ -package brotli - -/* Compresses "input" string to the "*storage" buffer as one or more complete - meta-blocks, and updates the "*storage_ix" bit position. - - If "is_last" is 1, emits an additional empty last meta-block. - - REQUIRES: "input_size" is greater than zero, or "is_last" is 1. - REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24). - REQUIRES: "command_buf" and "literal_buf" point to at least - kCompressFragmentTwoPassBlockSize long arrays. - REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero. - REQUIRES: "table_size" is a power of two - OUTPUT: maximal copy distance <= |input_size| - OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */ -const maxDistance_compress_fragment = 262128 - -/* kHashMul32_a multiplier has these properties: - * The multiplier must be odd. Otherwise we may lose the highest bit. - * No long streaks of ones or zeros. - * There is no effort to ensure that it is a prime, the oddity is enough - for this use. - * The number has been tuned heuristically against compression benchmarks. */ -var kHashMul32_a uint32 = 0x1E35A7BD diff --git a/compress_fragment_two_pass.go b/compress_fragment_two_pass.go index dca1412..5c03d39 100644 --- a/compress_fragment_two_pass.go +++ b/compress_fragment_two_pass.go @@ -13,28 +13,18 @@ import "encoding/binary" the found backward matches and literal bytes into a buffer, and in the second pass we emit them into the bit stream using prefix codes built based on the actual command and literal byte histograms. */ -/* Copyright 2015 Google Inc. All Rights Reserved. - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - -/* Function for fast encoding of an input fragment, independently from the input - history. This function uses two-pass processing: in the first pass we save - the found backward matches and literal bytes into a buffer, and in the - second pass we emit them into the bit stream using prefix codes built based - on the actual command and literal byte histograms. */ var kCompressFragmentTwoPassBlockSize uint = 1 << 17 func hash1(p []byte, shift uint, length uint) uint32 { - var h uint64 = (binary.LittleEndian.Uint64(p) << ((8 - length) * 8)) * uint64(kHashMul32_a) + var h uint64 = (binary.LittleEndian.Uint64(p) << ((8 - length) * 8)) * uint64(kHashMul32) return uint32(h >> shift) } func hashBytesAtOffset(v uint64, offset uint, shift uint, length uint) uint32 { assert(offset <= 8-length) { - var h uint64 = ((v >> (8 * offset)) << ((8 - length) * 8)) * uint64(kHashMul32_a) + var h uint64 = ((v >> (8 * offset)) << ((8 - length) * 8)) * uint64(kHashMul32) return uint32(h >> shift) } } @@ -721,6 +711,19 @@ func compressFragmentTwoPassImpl(input []byte, input_size uint, is_last bool, co } } +/* Compresses "input" string to the "*storage" buffer as one or more complete + meta-blocks, and updates the "*storage_ix" bit position. + + If "is_last" is 1, emits an additional empty last meta-block. + + REQUIRES: "input_size" is greater than zero, or "is_last" is 1. + REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24). + REQUIRES: "command_buf" and "literal_buf" point to at least + kCompressFragmentTwoPassBlockSize long arrays. + REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero. + REQUIRES: "table_size" is a power of two + OUTPUT: maximal copy distance <= |input_size| + OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */ func compressFragmentTwoPass(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_size uint, storage_ix *uint, storage []byte) { var initial_storage_ix uint = *storage_ix var table_bits uint = uint(log2FloorNonZero(table_size)) diff --git a/context.go b/context.go index f4a1317..884ff8a 100644 --- a/context.go +++ b/context.go @@ -1,181 +1,5 @@ package brotli -/** - * Sets the specified parameter to the given decoder instance. - * - * @param state decoder instance - * @param param parameter to set - * @param value new parameter value - * @returns ::false if parameter is unrecognized, or value is invalid - * @returns ::true if value is accepted - */ - -/** - * Creates an instance of ::BrotliDecoderState and initializes it. - * - * The instance can be used once for decoding and should then be destroyed with - * ::BrotliDecoderDestroyInstance, it cannot be reused for a new decoding - * session. - * - * @p alloc_func and @p free_func @b MUST be both zero or both non-zero. In the - * case they are both zero, default memory allocators are used. @p opaque is - * passed to @p alloc_func and @p free_func when they are called. @p free_func - * has to return without doing anything when asked to free a NULL pointer. - * - * @param alloc_func custom memory allocation function - * @param free_func custom memory free function - * @param opaque custom memory manager handle - * @returns @c 0 if instance can not be allocated or initialized - * @returns pointer to initialized ::BrotliDecoderState otherwise - */ - -/** - * Deinitializes and frees ::BrotliDecoderState instance. - * - * @param state decoder instance to be cleaned up and deallocated - */ - -/** - * Performs one-shot memory-to-memory decompression. - * - * Decompresses the data in @p encoded_buffer into @p decoded_buffer, and sets - * @p *decoded_size to the decompressed length. - * - * @param encoded_size size of @p encoded_buffer - * @param encoded_buffer compressed data buffer with at least @p encoded_size - * addressable bytes - * @param[in, out] decoded_size @b in: size of @p decoded_buffer; \n - * @b out: length of decompressed data written to - * @p decoded_buffer - * @param decoded_buffer decompressed data destination buffer - * @returns ::BROTLI_DECODER_RESULT_ERROR if input is corrupted, memory - * allocation failed, or @p decoded_buffer is not large enough; - * @returns ::BROTLI_DECODER_RESULT_SUCCESS otherwise - */ - -/** - * Decompresses the input stream to the output stream. - * - * The values @p *available_in and @p *available_out must specify the number of - * bytes addressable at @p *next_in and @p *next_out respectively. - * When @p *available_out is @c 0, @p next_out is allowed to be @c NULL. - * - * After each call, @p *available_in will be decremented by the amount of input - * bytes consumed, and the @p *next_in pointer will be incremented by that - * amount. Similarly, @p *available_out will be decremented by the amount of - * output bytes written, and the @p *next_out pointer will be incremented by - * that amount. - * - * @p total_out, if it is not a null-pointer, will be set to the number - * of bytes decompressed since the last @p state initialization. - * - * @note Input is never overconsumed, so @p next_in and @p available_in could be - * passed to the next consumer after decoding is complete. - * - * @param state decoder instance - * @param[in, out] available_in @b in: amount of available input; \n - * @b out: amount of unused input - * @param[in, out] next_in pointer to the next compressed byte - * @param[in, out] available_out @b in: length of output buffer; \n - * @b out: remaining size of output buffer - * @param[in, out] next_out output buffer cursor; - * can be @c NULL if @p available_out is @c 0 - * @param[out] total_out number of bytes decompressed so far; can be @c NULL - * @returns ::BROTLI_DECODER_RESULT_ERROR if input is corrupted, memory - * allocation failed, arguments were invalid, etc.; - * use ::BrotliDecoderGetErrorCode to get detailed error code - * @returns ::BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT decoding is blocked until - * more input data is provided - * @returns ::BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT decoding is blocked until - * more output space is provided - * @returns ::BROTLI_DECODER_RESULT_SUCCESS decoding is finished, no more - * input might be consumed and no more output will be produced - */ - -/** - * Checks if decoder has more output. - * - * @param state decoder instance - * @returns ::true, if decoder has some unconsumed output - * @returns ::false otherwise - */ - -/** - * Acquires pointer to internal output buffer. - * - * This method is used to make language bindings easier and more efficient: - * -# push data to ::BrotliDecoderDecompressStream, - * until ::BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT is reported - * -# use ::BrotliDecoderTakeOutput to peek bytes and copy to language-specific - * entity - * - * Also this could be useful if there is an output stream that is able to - * consume all the provided data (e.g. when data is saved to file system). - * - * @attention After every call to ::BrotliDecoderTakeOutput @p *size bytes of - * output are considered consumed for all consecutive calls to the - * instance methods; returned pointer becomes invalidated as well. - * - * @note Decoder output is not guaranteed to be contiguous. This means that - * after the size-unrestricted call to ::BrotliDecoderTakeOutput, - * immediate next call to ::BrotliDecoderTakeOutput may return more data. - * - * @param state decoder instance - * @param[in, out] size @b in: number of bytes caller is ready to take, @c 0 if - * any amount could be handled; \n - * @b out: amount of data pointed by returned pointer and - * considered consumed; \n - * out value is never greater than in value, unless it is @c 0 - * @returns pointer to output data - */ - -/** - * Checks if instance has already consumed input. - * - * Instance that returns ::false is considered "fresh" and could be - * reused. - * - * @param state decoder instance - * @returns ::true if decoder has already used some input bytes - * @returns ::false otherwise - */ - -/** - * Checks if decoder instance reached the final state. - * - * @param state decoder instance - * @returns ::true if decoder is in a state where it reached the end of - * the input and produced all of the output - * @returns ::false otherwise - */ - -/** - * Acquires a detailed error code. - * - * Should be used only after ::BrotliDecoderDecompressStream returns - * ::BROTLI_DECODER_RESULT_ERROR. - * - * See also ::BrotliDecoderErrorString - * - * @param state decoder instance - * @returns last saved error code - */ - -/** - * Converts error code to a c-string. - */ - -/** - * Gets a decoder library version. - * - * Look at BROTLI_VERSION for more information. - */ -/* Copyright 2013 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - /* Lookup table to map the previous two bytes to a context id. There are four different context modeling modes defined here: @@ -256,6 +80,7 @@ context ids and the type of the next byte is summarized in the table below: | (208-) | | context: 2 - 3 | | |-------------|-------------------|---------------------|------------------| */ + const ( contextLSB6 = 0 contextMSB6 = 1 diff --git a/decode.go b/decode.go index c580011..1086110 100644 --- a/decode.go +++ b/decode.go @@ -5,21 +5,7 @@ package brotli Distributed under MIT license. See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* Copyright 2013 Google Inc. All Rights Reserved. - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - -/** - * @file - * API for Brotli decompression. - */ - -/** - * Result type for ::BrotliDecoderDecompress and - * ::BrotliDecoderDecompressStream functions. - */ const ( decoderResultError = 0 decoderResultSuccess = 1 @@ -27,25 +13,6 @@ const ( decoderResultNeedsMoreOutput = 3 ) -/** - * Template that evaluates items of ::BrotliDecoderErrorCode. - * - * Example: @code {.cpp} - * // Log Brotli error code. - * switch (brotliDecoderErrorCode) { - * #define CASE_(PREFIX, NAME, CODE) \ - * case BROTLI_DECODER ## PREFIX ## NAME: \ - * LOG(INFO) << "error code:" << #NAME; \ - * break; - * #define NEWLINE_ - * BROTLI_DECODER_ERROR_CODES_LIST(CASE_, NEWLINE_) - * #undef CASE_ - * #undef NEWLINE_ - * default: LOG(FATAL) << "unknown brotli error code"; - * } - * @endcode - */ - /** * Error code for detailed logging / production debugging. * diff --git a/dictionary.go b/dictionary.go index ebb8830..79148c9 100644 --- a/dictionary.go +++ b/dictionary.go @@ -1,10 +1,5 @@ package brotli -/* Copyright 2013 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ /* Copyright 2013 Google Inc. All Rights Reserved. Distributed under MIT license. @@ -19,15 +14,6 @@ type dictionary struct { data []byte } -/** - * Sets dictionary data. - * - * When dictionary data is already set / present, this method is no-op. - * - * Dictionary data MUST be provided before BrotliGetDictionary is invoked. - * This method is used ONLY in multi-client environment (e.g. C + Java), - * to reduce storage by sharing single dictionary between implementations. - */ const minDictionaryWordLength = 4 const maxDictionaryWordLength = 24 diff --git a/dictionary_hash.go b/dictionary_hash.go index c5101b2..a062030 100644 --- a/dictionary_hash.go +++ b/dictionary_hash.go @@ -6,13 +6,6 @@ package brotli See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* Hash table on the 4-byte prefixes of static dictionary words. */ -/* Copyright 2015 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - /* Hash table on the 4-byte prefixes of static dictionary words. */ var kStaticDictionaryHash = [32768]uint16{ 32072, diff --git a/encode.go b/encode.go index 0706662..4e26efb 100644 --- a/encode.go +++ b/encode.go @@ -11,19 +11,6 @@ import ( See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* Constants and formulas that affect speed-ratio trade-offs and thus define - quality levels. */ -/* Copyright 2013 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - -/** - * @file - * API for Brotli compression. - */ - /** Minimal value for ::BROTLI_PARAM_LGWIN parameter. */ const minWindowBits = 10 diff --git a/entropy_encode.go b/entropy_encode.go index a9bfcc9..d0c1dca 100644 --- a/entropy_encode.go +++ b/entropy_encode.go @@ -8,13 +8,6 @@ import "math" See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* Entropy encoding (Huffman) utilities. */ -/* Copyright 2010 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - /* Entropy encoding (Huffman) utilities. */ /* A node of a Huffman tree. */ @@ -30,35 +23,6 @@ func initHuffmanTree(self *huffmanTree, count uint32, left int16, right int16) { self.index_right_or_value_ = right } -/* Returns 1 is assignment of depths succeeded, otherwise 0. */ - -/* This function will create a Huffman tree. - - The (data,length) contains the population counts. - The tree_limit is the maximum bit depth of the Huffman codes. - - The depth contains the tree, i.e., how many bits are used for - the symbol. - - The actual Huffman tree is constructed in the tree[] array, which has to - be at least 2 * length + 1 long. - - See http://en.wikipedia.org/wiki/Huffman_coding */ - -/* Change the population counts in a way that the consequent - Huffman tree compression, especially its RLE-part will be more - likely to compress this data more efficiently. - - length contains the size of the histogram. - counts contains the population counts. - good_for_rle is a buffer of at least length size */ - -/* Write a Huffman tree from bit depths into the bit-stream representation - of a Huffman tree. The generated Huffman tree is to be compressed once - more using a Huffman tree */ - -/* Get the actual bit values for a tree of bit depths. */ - /* Input size optimized Shell sort. */ type huffmanTreeComparator func(*huffmanTree, *huffmanTree) bool @@ -109,6 +73,7 @@ func sortHuffmanTreeItems(items []huffmanTree, n uint, comparator huffmanTreeCom } } +/* Returns 1 if assignment of depths succeeded, otherwise 0. */ func setDepth(p0 int, pool []huffmanTree, depth []byte, max_depth int) bool { var stack [16]int var level int = 0 @@ -333,6 +298,13 @@ func writeHuffmanTreeRepetitionsZeros(repetitions uint, tree_size *uint, tree [] } } +/* Change the population counts in a way that the consequent + Huffman tree compression, especially its RLE-part will be more + likely to compress this data more efficiently. + + length contains the size of the histogram. + counts contains the population counts. + good_for_rle is a buffer of at least length size */ func optimizeHuffmanCountsForRLE(length uint, counts []uint32, good_for_rle []byte) { var nonzero_count uint = 0 var stride uint @@ -510,6 +482,9 @@ func decideOverRLEUse(depth []byte, length uint, use_rle_for_non_zero *bool, use *use_rle_for_zero = total_reps_zero > count_reps_zero*2 } +/* Write a Huffman tree from bit depths into the bit-stream representation + of a Huffman tree. The generated Huffman tree is to be compressed once + more using a Huffman tree */ func writeHuffmanTree(depth []byte, length uint, tree_size *uint, tree []byte, extra_bits_data []byte) { var previous_value byte = initialRepeatedCodeLength var i uint @@ -589,6 +564,7 @@ func reverseBits(num_bits uint, bits uint16) uint16 { /* 0..15 are values for bits */ const maxHuffmanBits = 16 +/* Get the actual bit values for a tree of bit depths. */ func convertBitDepthsToSymbols(depth []byte, len uint, bits []uint16) { var bl_count = [maxHuffmanBits]uint16{0} var next_code [maxHuffmanBits]uint16 diff --git a/fast_log.go b/fast_log.go index e8f0e63..bbae300 100644 --- a/fast_log.go +++ b/fast_log.go @@ -8,27 +8,8 @@ import "math" See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* Literal cost model to allow backward reference replacement to be efficient. - */ -/* Copyright 2013 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - -/* Literal cost model to allow backward reference replacement to be efficient. - */ - -/* Estimates how many bits the literals in the interval [pos, pos + len) in the - ring-buffer (data, mask) will take entropy coded and writes these estimates - to the cost[0..len) array. */ -/* Copyright 2013 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - /* Utilities for fast computation of logarithms. */ + func log2FloorNonZero(n uint) uint32 { /* TODO: generalize and move to platform.h */ var result uint32 = 0 diff --git a/find_match_length.go b/find_match_length.go index 855c896..14d350a 100644 --- a/find_match_length.go +++ b/find_match_length.go @@ -6,14 +6,6 @@ package brotli See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* A (forgetful) hash table to the data seen by the compressor, to - help create backward references to previous data. */ -/* Copyright 2010 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - /* Function to find maximal matching prefixes of strings. */ func findMatchLengthWithLimit(s1 []byte, s2 []byte, limit uint) uint { var matched uint = 0 diff --git a/h10.go b/h10.go index 4a093b1..5662fbb 100644 --- a/h10.go +++ b/h10.go @@ -2,19 +2,12 @@ package brotli import "encoding/binary" -/* NOLINT(build/header_guard) */ /* Copyright 2016 Google Inc. All Rights Reserved. Distributed under MIT license. See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* A (forgetful) hash table where each hash bucket contains a binary tree of - sequences whose first 4 bytes share the same hash code. - Each sequence is 128 long and is identified by its starting - position in the input data. The binary tree is sorted by the lexicographic - order of the sequences, and it is also a max-heap with respect to the - starting positions. */ func (*h10) HashTypeLength() uint { return 4 } @@ -31,6 +24,12 @@ func hashBytesH10(data []byte) uint32 { return h >> (32 - 17) } +/* A (forgetful) hash table where each hash bucket contains a binary tree of + sequences whose first 4 bytes share the same hash code. + Each sequence is 128 long and is identified by its starting + position in the input data. The binary tree is sorted by the lexicographic + order of the sequences, and it is also a max-heap with respect to the + starting positions. */ type h10 struct { hasherCommon window_mask_ uint diff --git a/h5.go b/h5.go index 13761ba..f391b73 100644 --- a/h5.go +++ b/h5.go @@ -2,7 +2,6 @@ package brotli import "encoding/binary" -/* NOLINT(build/header_guard) */ /* Copyright 2010 Google Inc. All Rights Reserved. Distributed under MIT license. diff --git a/h6.go b/h6.go index cab8768..80bb224 100644 --- a/h6.go +++ b/h6.go @@ -2,7 +2,6 @@ package brotli import "encoding/binary" -/* NOLINT(build/header_guard) */ /* Copyright 2010 Google Inc. All Rights Reserved. Distributed under MIT license. diff --git a/hash.go b/hash.go index 3468680..63d61f1 100644 --- a/hash.go +++ b/hash.go @@ -5,24 +5,6 @@ import ( "fmt" ) -/* Matches data against static dictionary words, and for each length l, - for which a match is found, updates matches[l] to be the minimum possible - (distance << 5) + len_code. - Returns 1 if matches have been found, otherwise 0. - Prerequisites: - matches array is at least BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN + 1 long - all elements are initialized to kInvalidMatch */ -/* Pointer to hasher data. - * - * Excluding initialization and destruction, hasher can be passed as - * HasherHandle by value. - * - * Typically hasher data consists of 3 sections: - * * hasherCommon structure - * * private structured hasher data, depending on hasher type - * * private dynamic hasher data, depending on hasher type and parameters - * - */ type hasherCommon struct { params hasherParams is_prepared_ bool @@ -68,11 +50,11 @@ type hasherSearchResult struct { * There is no effort to ensure that it is a prime, the oddity is enough for this use. * The number has been tuned heuristically against compression benchmarks. */ -var kHashMul32 uint32 = 0x1E35A7BD +const kHashMul32 uint32 = 0x1E35A7BD -var kHashMul64 uint64 = 0x1E35A7BD1E35A7BD +const kHashMul64 uint64 = 0x1E35A7BD1E35A7BD -var kHashMul64Long uint64 = 0x1FE35A7BD3579BD3 +const kHashMul64Long uint64 = 0x1FE35A7BD3579BD3 func hash14(data []byte) uint32 { var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32 diff --git a/hash_composite.go b/hash_composite.go index 7abe926..a65fe2e 100644 --- a/hash_composite.go +++ b/hash_composite.go @@ -1,16 +1,11 @@ package brotli -/* NOTE: this hasher does not search in the dictionary. It is used as - backup-hasher, the main hasher already searches in it. */ -/* NOLINT(build/header_guard) */ /* Copyright 2018 Google Inc. All Rights Reserved. Distributed under MIT license. See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* Composite hasher: This hasher allows to combine two other hashers, HASHER_A - and HASHER_B. */ func (h *hashComposite) HashTypeLength() uint { var a uint = h.ha.HashTypeLength() var b uint = h.hb.HashTypeLength() @@ -31,6 +26,8 @@ func (h *hashComposite) StoreLookahead() uint { } } +/* Composite hasher: This hasher allows to combine two other hashers, HASHER_A + and HASHER_B. */ type hashComposite struct { hasherCommon ha hasherHandle diff --git a/hash_forgetful_chain.go b/hash_forgetful_chain.go index 456c728..3364c44 100644 --- a/hash_forgetful_chain.go +++ b/hash_forgetful_chain.go @@ -2,19 +2,12 @@ package brotli import "encoding/binary" -/* NOLINT(build/header_guard) */ /* Copyright 2016 Google Inc. All Rights Reserved. Distributed under MIT license. See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* A (forgetful) hash table to the data seen by the compressor, to - help create backward references to previous data. - - Hashes are stored in chains which are bucketed to groups. Group of chains - share a storage "bank". When more than "bank size" chain nodes are added, - oldest nodes are replaced; this way several chains may share a tail. */ func (*hashForgetfulChain) HashTypeLength() uint { return 4 } @@ -37,6 +30,12 @@ type slot struct { next uint16 } +/* A (forgetful) hash table to the data seen by the compressor, to + help create backward references to previous data. + + Hashes are stored in chains which are bucketed to groups. Group of chains + share a storage "bank". When more than "bank size" chain nodes are added, + oldest nodes are replaced; this way several chains may share a tail. */ type hashForgetfulChain struct { hasherCommon diff --git a/hash_longest_match_quickly.go b/hash_longest_match_quickly.go index 025f5e3..9375dc1 100644 --- a/hash_longest_match_quickly.go +++ b/hash_longest_match_quickly.go @@ -2,7 +2,6 @@ package brotli import "encoding/binary" -/* NOLINT(build/header_guard) */ /* Copyright 2010 Google Inc. All Rights Reserved. Distributed under MIT license. @@ -12,6 +11,7 @@ import "encoding/binary" /* For BUCKET_SWEEP == 1, enabling the dictionary lookup makes compression a little faster (0.5% - 1%) and it compresses 0.15% better on small text and HTML inputs. */ + func (*hashLongestMatchQuickly) HashTypeLength() uint { return 8 } diff --git a/hash_rolling.go b/hash_rolling.go index 692ce11..569fbda 100644 --- a/hash_rolling.go +++ b/hash_rolling.go @@ -1,16 +1,14 @@ package brotli -/* NOTE: this hasher does not search in the dictionary. It is used as - backup-hasher, the main hasher already searches in it. */ -/* NOLINT(build/header_guard) */ /* Copyright 2018 Google Inc. All Rights Reserved. Distributed under MIT license. See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* Rolling hash for long distance long string matches. Stores one position - per bucket, bucket key is computed over a long region. */ +/* NOTE: this hasher does not search in the dictionary. It is used as + backup-hasher, the main hasher already searches in it. */ + var kRollingHashMul32hashRolling uint32 = 69069 var kInvalidPosHashRolling uint32 = 0xffffffff @@ -40,6 +38,8 @@ func (h *hashRolling) HashRollingFunction(state uint32, add byte, rem byte, fact return uint32(factor*state + h.HashByte(add) - factor_remove*h.HashByte(rem)) } +/* Rolling hash for long distance long string matches. Stores one position + per bucket, bucket key is computed over a long region. */ type hashRolling struct { hasherCommon diff --git a/huffman.go b/huffman.go index a8d5cd7..9f7a0f0 100644 --- a/huffman.go +++ b/huffman.go @@ -7,6 +7,7 @@ package brotli */ /* Utilities for building Huffman decoding tables. */ + const huffmanMaxCodeLength = 15 /* Maximum possible Huffman table size for an alphabet size of (index * 32), diff --git a/memory.go b/memory.go index affa390..7208a3b 100644 --- a/memory.go +++ b/memory.go @@ -6,8 +6,6 @@ package brotli See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* Macros for memory management. */ - /* Dynamically grows array capacity to at least the requested size T: data type diff --git a/metablock.go b/metablock.go index 7595352..4a412cf 100644 --- a/metablock.go +++ b/metablock.go @@ -6,31 +6,9 @@ package brotli See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* Brotli bit stream functions to support the low level format. There are no - compression algorithms here, just the right ordering of bits to match the - specs. */ -/* Copyright 2014 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - -/* Functions to convert brotli-related data structures into the - brotli bit stream. The functions here operate under - assumption that there is enough space in the storage, i.e., there are - no out-of-range checks anywhere. - - These functions do bit addressing into a byte array. The byte array - is called "storage" and the index to the bit is called storage_ix - in function arguments. */ -/* Copyright 2015 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - /* Algorithms for distributing the literals and commands of a metablock between block types and contexts. */ + type metaBlockSplit struct { literal_split blockSplit command_split blockSplit @@ -74,14 +52,6 @@ func destroyMetaBlockSplit(mb *metaBlockSplit) { mb.distance_histograms = nil } -/* Copyright 2015 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - -/* Algorithms for distributing the literals and commands of a metablock between - block types and contexts. */ func initDistanceParams(params *encoderParams, npostfix uint32, ndirect uint32) { var dist_params *distanceParams = ¶ms.dist var alphabet_size uint32 diff --git a/metablock_command.go b/metablock_command.go index dfa6a1c..d47541c 100644 --- a/metablock_command.go +++ b/metablock_command.go @@ -1,6 +1,5 @@ package brotli -/* NOLINT(build/header_guard) */ /* Copyright 2015 Google Inc. All Rights Reserved. Distributed under MIT license. diff --git a/metablock_distance.go b/metablock_distance.go index e2b2ddc..9592312 100644 --- a/metablock_distance.go +++ b/metablock_distance.go @@ -1,6 +1,5 @@ package brotli -/* NOLINT(build/header_guard) */ /* Copyright 2015 Google Inc. All Rights Reserved. Distributed under MIT license. diff --git a/metablock_literal.go b/metablock_literal.go index 1b6ef18..d7e8a7c 100644 --- a/metablock_literal.go +++ b/metablock_literal.go @@ -1,6 +1,5 @@ package brotli -/* NOLINT(build/header_guard) */ /* Copyright 2015 Google Inc. All Rights Reserved. Distributed under MIT license. diff --git a/params.go b/params.go index f577d1c..0a4c687 100644 --- a/params.go +++ b/params.go @@ -1,197 +1,5 @@ package brotli -/** - * Opaque structure that holds encoder state. - * - * Allocated and initialized with ::BrotliEncoderCreateInstance. - * Cleaned up and deallocated with ::BrotliEncoderDestroyInstance. - */ - -/** - * Sets the specified parameter to the given encoder instance. - * - * @param state encoder instance - * @param param parameter to set - * @param value new parameter value - * @returns ::false if parameter is unrecognized, or value is invalid - * @returns ::false if value of parameter can not be changed at current - * encoder state (e.g. when encoding is started, window size might be - * already encoded and therefore it is impossible to change it) - * @returns ::true if value is accepted - * @warning invalid values might be accepted in case they would not break - * encoding process. - */ - -/** - * Creates an instance of ::BrotliEncoderState and initializes it. - * - * @p alloc_func and @p free_func @b MUST be both zero or both non-zero. In the - * case they are both zero, default memory allocators are used. @p opaque is - * passed to @p alloc_func and @p free_func when they are called. @p free_func - * has to return without doing anything when asked to free a NULL pointer. - * - * @param alloc_func custom memory allocation function - * @param free_func custom memory free function - * @param opaque custom memory manager handle - * @returns @c 0 if instance can not be allocated or initialized - * @returns pointer to initialized ::BrotliEncoderState otherwise - */ - -/** - * Deinitializes and frees ::BrotliEncoderState instance. - * - * @param state decoder instance to be cleaned up and deallocated - */ - -/** - * Calculates the output size bound for the given @p input_size. - * - * @warning Result is only valid if quality is at least @c 2 and, in - * case ::BrotliEncoderCompressStream was used, no flushes - * (::BROTLI_OPERATION_FLUSH) were performed. - * - * @param input_size size of projected input - * @returns @c 0 if result does not fit @c size_t - */ - -/** - * Performs one-shot memory-to-memory compression. - * - * Compresses the data in @p input_buffer into @p encoded_buffer, and sets - * @p *encoded_size to the compressed length. - * - * @note If ::BrotliEncoderMaxCompressedSize(@p input_size) returns non-zero - * value, then output is guaranteed to be no longer than that. - * - * @param quality quality parameter value, e.g. ::BROTLI_DEFAULT_QUALITY - * @param lgwin lgwin parameter value, e.g. ::BROTLI_DEFAULT_WINDOW - * @param mode mode parameter value, e.g. ::BROTLI_DEFAULT_MODE - * @param input_size size of @p input_buffer - * @param input_buffer input data buffer with at least @p input_size - * addressable bytes - * @param[in, out] encoded_size @b in: size of @p encoded_buffer; \n - * @b out: length of compressed data written to - * @p encoded_buffer, or @c 0 if compression fails - * @param encoded_buffer compressed data destination buffer - * @returns ::false in case of compression error - * @returns ::false if output buffer is too small - * @returns ::true otherwise - */ - -/** - * Compresses input stream to output stream. - * - * The values @p *available_in and @p *available_out must specify the number of - * bytes addressable at @p *next_in and @p *next_out respectively. - * When @p *available_out is @c 0, @p next_out is allowed to be @c NULL. - * - * After each call, @p *available_in will be decremented by the amount of input - * bytes consumed, and the @p *next_in pointer will be incremented by that - * amount. Similarly, @p *available_out will be decremented by the amount of - * output bytes written, and the @p *next_out pointer will be incremented by - * that amount. - * - * @p total_out, if it is not a null-pointer, will be set to the number - * of bytes compressed since the last @p state initialization. - * - * - * - * Internally workflow consists of 3 tasks: - * -# (optionally) copy input data to internal buffer - * -# actually compress data and (optionally) store it to internal buffer - * -# (optionally) copy compressed bytes from internal buffer to output stream - * - * Whenever all 3 tasks can't move forward anymore, or error occurs, this - * method returns the control flow to caller. - * - * @p op is used to perform flush, finish the stream, or inject metadata block. - * See ::BrotliEncoderOperation for more information. - * - * Flushing the stream means forcing encoding of all input passed to encoder and - * completing the current output block, so it could be fully decoded by stream - * decoder. To perform flush set @p op to ::BROTLI_OPERATION_FLUSH. - * Under some circumstances (e.g. lack of output stream capacity) this operation - * would require several calls to ::BrotliEncoderCompressStream. The method must - * be called again until both input stream is depleted and encoder has no more - * output (see ::BrotliEncoderHasMoreOutput) after the method is called. - * - * Finishing the stream means encoding of all input passed to encoder and - * adding specific "final" marks, so stream decoder could determine that stream - * is complete. To perform finish set @p op to ::BROTLI_OPERATION_FINISH. - * Under some circumstances (e.g. lack of output stream capacity) this operation - * would require several calls to ::BrotliEncoderCompressStream. The method must - * be called again until both input stream is depleted and encoder has no more - * output (see ::BrotliEncoderHasMoreOutput) after the method is called. - * - * @warning When flushing and finishing, @p op should not change until operation - * is complete; input stream should not be swapped, reduced or - * extended as well. - * - * @param state encoder instance - * @param op requested operation - * @param[in, out] available_in @b in: amount of available input; \n - * @b out: amount of unused input - * @param[in, out] next_in pointer to the next input byte - * @param[in, out] available_out @b in: length of output buffer; \n - * @b out: remaining size of output buffer - * @param[in, out] next_out compressed output buffer cursor; - * can be @c NULL if @p available_out is @c 0 - * @param[out] total_out number of bytes produced so far; can be @c NULL - * @returns ::false if there was an error - * @returns ::true otherwise - */ - -/** - * Checks if encoder instance reached the final state. - * - * @param state encoder instance - * @returns ::true if encoder is in a state where it reached the end of - * the input and produced all of the output - * @returns ::false otherwise - */ - -/** - * Checks if encoder has more output. - * - * @param state encoder instance - * @returns ::true, if encoder has some unconsumed output - * @returns ::false otherwise - */ - -/** - * Acquires pointer to internal output buffer. - * - * This method is used to make language bindings easier and more efficient: - * -# push data to ::BrotliEncoderCompressStream, - * until ::BrotliEncoderHasMoreOutput returns BROTL_TRUE - * -# use ::BrotliEncoderTakeOutput to peek bytes and copy to language-specific - * entity - * - * Also this could be useful if there is an output stream that is able to - * consume all the provided data (e.g. when data is saved to file system). - * - * @attention After every call to ::BrotliEncoderTakeOutput @p *size bytes of - * output are considered consumed for all consecutive calls to the - * instance methods; returned pointer becomes invalidated as well. - * - * @note Encoder output is not guaranteed to be contiguous. This means that - * after the size-unrestricted call to ::BrotliEncoderTakeOutput, - * immediate next call to ::BrotliEncoderTakeOutput may return more data. - * - * @param state encoder instance - * @param[in, out] size @b in: number of bytes caller is ready to take, @c 0 if - * any amount could be handled; \n - * @b out: amount of data pointed by returned pointer and - * considered consumed; \n - * out value is never greater than in value, unless it is @c 0 - * @returns pointer to output data - */ - -/** - * Gets an encoder library version. - * - * Look at BROTLI_VERSION for more information. - */ /* Copyright 2017 Google Inc. All Rights Reserved. Distributed under MIT license. diff --git a/platform.go b/platform.go index 237947b..4ebfb15 100644 --- a/platform.go +++ b/platform.go @@ -6,30 +6,6 @@ package brotli See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* Bit reading helpers */ -/* Copyright 2016 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - -/* Macros for compiler / platform specific features and build options. - - Build options are: - * BROTLI_BUILD_32_BIT disables 64-bit optimizations - * BROTLI_BUILD_64_BIT forces to use 64-bit optimizations - * BROTLI_BUILD_BIG_ENDIAN forces to use big-endian optimizations - * BROTLI_BUILD_ENDIAN_NEUTRAL disables endian-aware optimizations - * BROTLI_BUILD_LITTLE_ENDIAN forces to use little-endian optimizations - * BROTLI_BUILD_PORTABLE disables dangerous optimizations, like unaligned - read and overlapping memcpy; this reduces decompression speed by 5% - * BROTLI_BUILD_NO_RBIT disables "rbit" optimization for ARM CPUs - * BROTLI_DEBUG dumps file name and line number when decoder detects stream - or memory error - * BROTLI_ENABLE_LOG enables asserts and dumps various state information -*/ -type brotli_reg_t uint64 - func brotli_min_double(a float64, b float64) float64 { if a < b { return a diff --git a/prefix.go b/prefix.go index c0164e8..484df0d 100644 --- a/prefix.go +++ b/prefix.go @@ -6,27 +6,6 @@ package brotli See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* Function to find backward reference copies. */ -/* Copyright 2013 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - -/* Function to find backward reference copies. */ -/* Copyright 2013 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - -/* This class models a sequence of literals and a backward reference copy. */ -/* Copyright 2013 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - /* Functions for encoding of integers into prefix codes the amount of extra bits, and the actual values of the extra bits. */ diff --git a/ringbuffer.go b/ringbuffer.go index e2336e9..6ea3254 100644 --- a/ringbuffer.go +++ b/ringbuffer.go @@ -6,15 +6,6 @@ package brotli See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* Implementation of Brotli compressor. */ -/* Copyright 2013 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - -/* Sliding window over the input data. */ - /* A ringBuffer(window_bits, tail_bits) contains `1 << window_bits' bytes of data in a circular manner: writing a byte writes it to: `position() % (1 << window_bits)'. diff --git a/static_dict.go b/static_dict.go index 1698f78..181337f 100644 --- a/static_dict.go +++ b/static_dict.go @@ -9,6 +9,7 @@ import "encoding/binary" */ /* Class to model the static dictionary. */ + const maxStaticDictionaryMatchLen = 37 var kInvalidMatch uint32 = 0xFFFFFFF diff --git a/static_dict_lut.go b/static_dict_lut.go index a723fb1..997ced4 100644 --- a/static_dict_lut.go +++ b/static_dict_lut.go @@ -5,18 +5,9 @@ package brotli Distributed under MIT license. See file LICENSE for detail or copy at https://opensource.org/licenses/MIT */ -/* Copyright 2017 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ -/* Copyright 2015 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ /* Lookup table for static dictionary and transforms. */ + type dictWord struct { len byte transform byte diff --git a/symbol_list.go b/symbol_list.go index 05c4c11..c5cb49e 100644 --- a/symbol_list.go +++ b/symbol_list.go @@ -7,6 +7,7 @@ package brotli */ /* Utilities for building Huffman decoding tables. */ + type symbolList struct { storage []uint16 offset int diff --git a/transform.go b/transform.go index 77a83fb..0bb00d7 100644 --- a/transform.go +++ b/transform.go @@ -1,11 +1,5 @@ package brotli -/* transforms is a part of ABI, but not API. - - It means that there are some functions that are supposed to be in "common" - library, but header itself is not placed into include/brotli. This way, - aforementioned functions will be available only to brotli internals. -*/ const ( transformIdentity = 0 transformOmitLast1 = 1 @@ -45,7 +39,6 @@ type transforms struct { cutOffTransforms [transformsMaxCutOff + 1]int16 } -/* T is BrotliTransforms*; result is uint8_t. */ func transformPrefixId(t *transforms, I int) byte { return t.transforms[(I*3)+0] } @@ -58,7 +51,6 @@ func transformSuffixId(t *transforms, I int) byte { return t.transforms[(I*3)+2] } -/* T is BrotliTransforms*; result is const uint8_t*. */ func transformPrefix(t *transforms, I int) []byte { return t.prefix_suffix[t.prefix_suffix_map[transformPrefixId(t, I)]:] } @@ -70,19 +62,6 @@ func transformSuffix(t *transforms, I int) []byte { /* RFC 7932 transforms string data */ var kPrefixSuffix string = "\001 \002, \010 of the \004 of \002s \001.\005 and \004 " + "in \001\"\004 to \002\">\001\n\002. \001]\005 for \003 a \006 " + "that \001'\006 with \006 from \004 by \001(\006. T" + "he \004 on \004 as \004 is \004ing \002\n\t\001:\003ed " + "\002=\"\004 at \003ly \001,\002='\005.com/\007. This \005" + " not \003er \003al \004ful \004ive \005less \004es" + "t \004ize \002\xc2\xa0\004ous \005 the \002e \000" -/* 0x _0 _2 __5 _E _3 _6 _8 _E */ - -/* 2x _3_ _5 _A_ _D_ _F _2 _4 _A _E */ - -/* 4x _5_ _7 _E _5 _A _C */ - -/* 6x _3 _8 _D _2 _7_ _ _A _C */ - -/* 8x _0 _ _3 _8 _C _E _ _1 _7 _F */ - -/* Ax _5 _9 _D _2 _7 _D */ - -/* Cx _2 _7___ ___ _A _F _5 _8 */ var kPrefixSuffixMap = [50]uint16{ 0x00, 0x02, diff --git a/write_bits.go b/write_bits.go index 1b081a2..8f15c20 100644 --- a/write_bits.go +++ b/write_bits.go @@ -1,40 +1,5 @@ package brotli -/* Uses the slow shortest-path block splitter and does context clustering. - The distance parameters are dynamically selected based on the commands - which get recomputed under the new distance parameters. The new distance - parameters are stored into *params. */ - -/* Uses a fast greedy block splitter that tries to merge current block with the - last or the second last block and uses a static context clustering which - is the same for all block types. */ -/* All Store functions here will use a storage_ix, which is always the bit - position for the current storage. */ - -/* REQUIRES: length > 0 */ -/* REQUIRES: length <= (1 << 24) */ - -/* Stores the meta-block without doing any block splitting, just collects - one histogram per block category and uses that for entropy coding. - REQUIRES: length > 0 - REQUIRES: length <= (1 << 24) */ - -/* Same as above, but uses static prefix codes for histograms with a only a few - symbols, and uses static code length prefix codes for all other histograms. - REQUIRES: length > 0 - REQUIRES: length <= (1 << 24) */ - -/* This is for storing uncompressed blocks (simple raw storage of - bytes-as-bytes). - REQUIRES: length > 0 - REQUIRES: length <= (1 << 24) */ -/* Copyright 2015 Google Inc. All Rights Reserved. - - Distributed under MIT license. - See file LICENSE for detail or copy at https://opensource.org/licenses/MIT -*/ - -/* Static entropy codes used for faster meta-block encoding. */ /* Copyright 2010 Google Inc. All Rights Reserved. Distributed under MIT license. @@ -43,8 +8,6 @@ package brotli /* Write bits into a byte array. */ -/*#define BIT_WRITER_DEBUG */ - /* This function writes bits into bytes in increasing addresses, and within a byte least-significant-bit first.