Clean up duplicate and misplaced comments.

This commit is contained in:
Andy Balholm 2019-03-21 15:34:30 -07:00
parent 55e85f3f75
commit 52165d4974
46 changed files with 107 additions and 840 deletions

View File

@ -7,18 +7,7 @@ package brotli
*/ */
/* Function to find backward reference copies. */ /* Function to find backward reference copies. */
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Function to find backward reference copies. */
/* "commands" points to the next output command to write to, "*num_commands" is
initially the total amount of commands output by previous
CreateBackwardReferences calls, and must be incremented by the amount written
by this call. */
func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uint { func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uint {
if distance <= max_distance { if distance <= max_distance {
var distance_plus_3 uint = distance + 3 var distance_plus_3 uint = distance + 3
@ -42,6 +31,10 @@ func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uin
return distance + numDistanceShortCodes - 1 return distance + numDistanceShortCodes - 1
} }
/* "commands" points to the next output command to write to, "*num_commands" is
initially the total amount of commands output by previous
CreateBackwardReferences calls, and must be incremented by the amount written
by this call. */
func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) { func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
var max_backward_limit uint = maxBackwardLimit(params.lgwin) var max_backward_limit uint = maxBackwardLimit(params.lgwin)
var orig_commands []command = commands var orig_commands []command = commands

View File

@ -13,19 +13,6 @@ type zopfliNode struct {
} }
} }
/* Computes the shortest path of commands from position to at most
position + num_bytes.
On return, path->size() is the number of commands found and path[i] is the
length of the i-th command (copy length plus insert length).
Note that the sum of the lengths of all commands can be less than num_bytes.
On return, the nodes[0..num_bytes] array will have the following
"ZopfliNode array invariant":
For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then
(1) nodes[i].copy_length() >= 2
(2) nodes[i].command_length() <= i and
(3) nodes[i - nodes[i].command_length()].cost < kInfinity */
const maxEffectiveDistanceAlphabetSize = 544 const maxEffectiveDistanceAlphabetSize = 544
var kInfinity float32 = 1.7e38 /* ~= 2 ^ 127 */ var kInfinity float32 = 1.7e38 /* ~= 2 ^ 127 */
@ -618,7 +605,21 @@ func zopfliIterate(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_
return computeShortestPathFromNodes(num_bytes, nodes) return computeShortestPathFromNodes(num_bytes, nodes)
} }
/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */ /* Computes the shortest path of commands from position to at most
position + num_bytes.
On return, path->size() is the number of commands found and path[i] is the
length of the i-th command (copy length plus insert length).
Note that the sum of the lengths of all commands can be less than num_bytes.
On return, the nodes[0..num_bytes] array will have the following
"ZopfliNode array invariant":
For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then
(1) nodes[i].copy_length() >= 2
(2) nodes[i].command_length() <= i and
(3) nodes[i - nodes[i].command_length()].cost < kInfinity
REQUIRES: nodes != nil and len(nodes) >= num_bytes + 1 */
func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, dist_cache []int, hasher *h10, nodes []zopfliNode) uint { func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, dist_cache []int, hasher *h10, nodes []zopfliNode) uint {
var max_backward_limit uint = maxBackwardLimit(params.lgwin) var max_backward_limit uint = maxBackwardLimit(params.lgwin)
var max_zopfli_len uint = maxZopfliLen(params) var max_zopfli_len uint = maxZopfliLen(params)

View File

@ -6,13 +6,6 @@ package brotli
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
/* Functions to estimate the bit cost of Huffman trees. */
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Functions to estimate the bit cost of Huffman trees. */ /* Functions to estimate the bit cost of Huffman trees. */
func shannonEntropy(population []uint32, size uint, total *uint) float64 { func shannonEntropy(population []uint32, size uint, total *uint) float64 {
var sum uint = 0 var sum uint = 0

View File

@ -9,6 +9,7 @@ import "encoding/binary"
*/ */
/* Bit reading helpers */ /* Bit reading helpers */
const shortFillBitWindowRead = (8 >> 1) const shortFillBitWindowRead = (8 >> 1)
var kBitMask = [33]uint32{ var kBitMask = [33]uint32{

View File

@ -6,21 +6,8 @@ package brotli
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
/* Build per-context histograms of literals, commands and distance codes. */
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Models the histograms of literals, commands and distance codes. */
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Block split point selection utilities. */ /* Block split point selection utilities. */
type blockSplit struct { type blockSplit struct {
num_types uint num_types uint
num_blocks uint num_blocks uint

View File

@ -2,12 +2,12 @@ package brotli
import "math" import "math"
/* NOLINT(build/header_guard) */
/* Copyright 2013 Google Inc. All Rights Reserved. /* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license. Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
func initialEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) { func initialEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) {
var seed uint32 = 7 var seed uint32 = 7
var block_length uint = length / num_histograms var block_length uint = length / num_histograms

View File

@ -2,12 +2,12 @@ package brotli
import "math" import "math"
/* NOLINT(build/header_guard) */
/* Copyright 2013 Google Inc. All Rights Reserved. /* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license. Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
func initialEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) { func initialEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) {
var seed uint32 = 7 var seed uint32 = 7
var block_length uint = length / num_histograms var block_length uint = length / num_histograms

View File

@ -2,12 +2,12 @@ package brotli
import "math" import "math"
/* NOLINT(build/header_guard) */
/* Copyright 2013 Google Inc. All Rights Reserved. /* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license. Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
func initialEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) { func initialEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) {
var seed uint32 = 7 var seed uint32 = 7
var block_length uint = length / num_histograms var block_length uint = length / num_histograms

View File

@ -6,14 +6,8 @@ package brotli
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
/* Block split point selection utilities. */
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Functions for clustering similar histograms together. */ /* Functions for clustering similar histograms together. */
type histogramPair struct { type histogramPair struct {
idx1 uint32 idx1 uint32
idx2 uint32 idx2 uint32
@ -21,13 +15,6 @@ type histogramPair struct {
cost_diff float64 cost_diff float64
} }
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Functions for clustering similar histograms together. */
func histogramPairIsLess(p1 *histogramPair, p2 *histogramPair) bool { func histogramPairIsLess(p1 *histogramPair, p2 *histogramPair) bool {
if p1.cost_diff != p2.cost_diff { if p1.cost_diff != p2.cost_diff {
return p1.cost_diff > p2.cost_diff return p1.cost_diff > p2.cost_diff

View File

@ -2,7 +2,6 @@ package brotli
import "math" import "math"
/* NOLINT(build/header_guard) */
/* Copyright 2013 Google Inc. All Rights Reserved. /* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license. Distributed under MIT license.

View File

@ -2,7 +2,6 @@ package brotli
import "math" import "math"
/* NOLINT(build/header_guard) */
/* Copyright 2013 Google Inc. All Rights Reserved. /* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license. Distributed under MIT license.

View File

@ -2,7 +2,6 @@ package brotli
import "math" import "math"
/* NOLINT(build/header_guard) */
/* Copyright 2013 Google Inc. All Rights Reserved. /* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license. Distributed under MIT license.

View File

@ -15,41 +15,11 @@ import "encoding/binary"
Adapted from the CompressFragment() function in Adapted from the CompressFragment() function in
https://github.com/google/snappy/blob/master/snappy.cc */ https://github.com/google/snappy/blob/master/snappy.cc */
/* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license. const maxDistance_compress_fragment = 262128
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Function for fast encoding of an input fragment, independently from the input
history. This function uses one-pass processing: when we find a backward
match, we immediately emit the corresponding command and literal codes to
the bit stream. */
/* Compresses "input" string to the "*storage" buffer as one or more complete
meta-blocks, and updates the "*storage_ix" bit position.
If "is_last" is 1, emits an additional empty last meta-block.
"cmd_depth" and "cmd_bits" contain the command and distance prefix codes
(see comment in encode.h) used for the encoding of this input fragment.
If "is_last" is 0, they are updated to reflect the statistics
of this input fragment, to be used for the encoding of the next fragment.
"*cmd_code_numbits" is the number of bits of the compressed representation
of the command and distance prefix codes, and "cmd_code" is an array of
at least "(*cmd_code_numbits + 7) >> 3" size that contains the compressed
command and distance prefix codes. If "is_last" is 0, these are also
updated to represent the updated "cmd_depth" and "cmd_bits".
REQUIRES: "input_size" is greater than zero, or "is_last" is 1.
REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24).
REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero.
REQUIRES: "table_size" is an odd (9, 11, 13, 15) power of two
OUTPUT: maximal copy distance <= |input_size|
OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */
func hash5(p []byte, shift uint) uint32 { func hash5(p []byte, shift uint) uint32 {
var h uint64 = (binary.LittleEndian.Uint64(p) << 24) * uint64(kHashMul32_a) var h uint64 = (binary.LittleEndian.Uint64(p) << 24) * uint64(kHashMul32)
return uint32(h >> shift) return uint32(h >> shift)
} }
@ -57,7 +27,7 @@ func hashBytesAtOffset5(v uint64, offset int, shift uint) uint32 {
assert(offset >= 0) assert(offset >= 0)
assert(offset <= 3) assert(offset <= 3)
{ {
var h uint64 = ((v >> uint(8*offset)) << 24) * uint64(kHashMul32_a) var h uint64 = ((v >> uint(8*offset)) << 24) * uint64(kHashMul32)
return uint32(h >> shift) return uint32(h >> shift)
} }
} }
@ -821,6 +791,28 @@ next_block:
} }
} }
/* Compresses "input" string to the "*storage" buffer as one or more complete
meta-blocks, and updates the "*storage_ix" bit position.
If "is_last" is 1, emits an additional empty last meta-block.
"cmd_depth" and "cmd_bits" contain the command and distance prefix codes
(see comment in encode.h) used for the encoding of this input fragment.
If "is_last" is 0, they are updated to reflect the statistics
of this input fragment, to be used for the encoding of the next fragment.
"*cmd_code_numbits" is the number of bits of the compressed representation
of the command and distance prefix codes, and "cmd_code" is an array of
at least "(*cmd_code_numbits + 7) >> 3" size that contains the compressed
command and distance prefix codes. If "is_last" is 0, these are also
updated to represent the updated "cmd_depth" and "cmd_bits".
REQUIRES: "input_size" is greater than zero, or "is_last" is 1.
REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24).
REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero.
REQUIRES: "table_size" is an odd (9, 11, 13, 15) power of two
OUTPUT: maximal copy distance <= |input_size|
OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */
func compressFragmentFast(input []byte, input_size uint, is_last bool, table []int, table_size uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, storage_ix *uint, storage []byte) { func compressFragmentFast(input []byte, input_size uint, is_last bool, table []int, table_size uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, storage_ix *uint, storage []byte) {
var initial_storage_ix uint = *storage_ix var initial_storage_ix uint = *storage_ix
var table_bits uint = uint(log2FloorNonZero(table_size)) var table_bits uint = uint(log2FloorNonZero(table_size))

View File

@ -1,24 +0,0 @@
package brotli
/* Compresses "input" string to the "*storage" buffer as one or more complete
meta-blocks, and updates the "*storage_ix" bit position.
If "is_last" is 1, emits an additional empty last meta-block.
REQUIRES: "input_size" is greater than zero, or "is_last" is 1.
REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24).
REQUIRES: "command_buf" and "literal_buf" point to at least
kCompressFragmentTwoPassBlockSize long arrays.
REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero.
REQUIRES: "table_size" is a power of two
OUTPUT: maximal copy distance <= |input_size|
OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */
const maxDistance_compress_fragment = 262128
/* kHashMul32_a multiplier has these properties:
* The multiplier must be odd. Otherwise we may lose the highest bit.
* No long streaks of ones or zeros.
* There is no effort to ensure that it is a prime, the oddity is enough
for this use.
* The number has been tuned heuristically against compression benchmarks. */
var kHashMul32_a uint32 = 0x1E35A7BD

View File

@ -13,28 +13,18 @@ import "encoding/binary"
the found backward matches and literal bytes into a buffer, and in the the found backward matches and literal bytes into a buffer, and in the
second pass we emit them into the bit stream using prefix codes built based second pass we emit them into the bit stream using prefix codes built based
on the actual command and literal byte histograms. */ on the actual command and literal byte histograms. */
/* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Function for fast encoding of an input fragment, independently from the input
history. This function uses two-pass processing: in the first pass we save
the found backward matches and literal bytes into a buffer, and in the
second pass we emit them into the bit stream using prefix codes built based
on the actual command and literal byte histograms. */
var kCompressFragmentTwoPassBlockSize uint = 1 << 17 var kCompressFragmentTwoPassBlockSize uint = 1 << 17
func hash1(p []byte, shift uint, length uint) uint32 { func hash1(p []byte, shift uint, length uint) uint32 {
var h uint64 = (binary.LittleEndian.Uint64(p) << ((8 - length) * 8)) * uint64(kHashMul32_a) var h uint64 = (binary.LittleEndian.Uint64(p) << ((8 - length) * 8)) * uint64(kHashMul32)
return uint32(h >> shift) return uint32(h >> shift)
} }
func hashBytesAtOffset(v uint64, offset uint, shift uint, length uint) uint32 { func hashBytesAtOffset(v uint64, offset uint, shift uint, length uint) uint32 {
assert(offset <= 8-length) assert(offset <= 8-length)
{ {
var h uint64 = ((v >> (8 * offset)) << ((8 - length) * 8)) * uint64(kHashMul32_a) var h uint64 = ((v >> (8 * offset)) << ((8 - length) * 8)) * uint64(kHashMul32)
return uint32(h >> shift) return uint32(h >> shift)
} }
} }
@ -721,6 +711,19 @@ func compressFragmentTwoPassImpl(input []byte, input_size uint, is_last bool, co
} }
} }
/* Compresses "input" string to the "*storage" buffer as one or more complete
meta-blocks, and updates the "*storage_ix" bit position.
If "is_last" is 1, emits an additional empty last meta-block.
REQUIRES: "input_size" is greater than zero, or "is_last" is 1.
REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24).
REQUIRES: "command_buf" and "literal_buf" point to at least
kCompressFragmentTwoPassBlockSize long arrays.
REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero.
REQUIRES: "table_size" is a power of two
OUTPUT: maximal copy distance <= |input_size|
OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */
func compressFragmentTwoPass(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_size uint, storage_ix *uint, storage []byte) { func compressFragmentTwoPass(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_size uint, storage_ix *uint, storage []byte) {
var initial_storage_ix uint = *storage_ix var initial_storage_ix uint = *storage_ix
var table_bits uint = uint(log2FloorNonZero(table_size)) var table_bits uint = uint(log2FloorNonZero(table_size))

View File

@ -1,181 +1,5 @@
package brotli package brotli
/**
* Sets the specified parameter to the given decoder instance.
*
* @param state decoder instance
* @param param parameter to set
* @param value new parameter value
* @returns ::false if parameter is unrecognized, or value is invalid
* @returns ::true if value is accepted
*/
/**
* Creates an instance of ::BrotliDecoderState and initializes it.
*
* The instance can be used once for decoding and should then be destroyed with
* ::BrotliDecoderDestroyInstance, it cannot be reused for a new decoding
* session.
*
* @p alloc_func and @p free_func @b MUST be both zero or both non-zero. In the
* case they are both zero, default memory allocators are used. @p opaque is
* passed to @p alloc_func and @p free_func when they are called. @p free_func
* has to return without doing anything when asked to free a NULL pointer.
*
* @param alloc_func custom memory allocation function
* @param free_func custom memory free function
* @param opaque custom memory manager handle
* @returns @c 0 if instance can not be allocated or initialized
* @returns pointer to initialized ::BrotliDecoderState otherwise
*/
/**
* Deinitializes and frees ::BrotliDecoderState instance.
*
* @param state decoder instance to be cleaned up and deallocated
*/
/**
* Performs one-shot memory-to-memory decompression.
*
* Decompresses the data in @p encoded_buffer into @p decoded_buffer, and sets
* @p *decoded_size to the decompressed length.
*
* @param encoded_size size of @p encoded_buffer
* @param encoded_buffer compressed data buffer with at least @p encoded_size
* addressable bytes
* @param[in, out] decoded_size @b in: size of @p decoded_buffer; \n
* @b out: length of decompressed data written to
* @p decoded_buffer
* @param decoded_buffer decompressed data destination buffer
* @returns ::BROTLI_DECODER_RESULT_ERROR if input is corrupted, memory
* allocation failed, or @p decoded_buffer is not large enough;
* @returns ::BROTLI_DECODER_RESULT_SUCCESS otherwise
*/
/**
* Decompresses the input stream to the output stream.
*
* The values @p *available_in and @p *available_out must specify the number of
* bytes addressable at @p *next_in and @p *next_out respectively.
* When @p *available_out is @c 0, @p next_out is allowed to be @c NULL.
*
* After each call, @p *available_in will be decremented by the amount of input
* bytes consumed, and the @p *next_in pointer will be incremented by that
* amount. Similarly, @p *available_out will be decremented by the amount of
* output bytes written, and the @p *next_out pointer will be incremented by
* that amount.
*
* @p total_out, if it is not a null-pointer, will be set to the number
* of bytes decompressed since the last @p state initialization.
*
* @note Input is never overconsumed, so @p next_in and @p available_in could be
* passed to the next consumer after decoding is complete.
*
* @param state decoder instance
* @param[in, out] available_in @b in: amount of available input; \n
* @b out: amount of unused input
* @param[in, out] next_in pointer to the next compressed byte
* @param[in, out] available_out @b in: length of output buffer; \n
* @b out: remaining size of output buffer
* @param[in, out] next_out output buffer cursor;
* can be @c NULL if @p available_out is @c 0
* @param[out] total_out number of bytes decompressed so far; can be @c NULL
* @returns ::BROTLI_DECODER_RESULT_ERROR if input is corrupted, memory
* allocation failed, arguments were invalid, etc.;
* use ::BrotliDecoderGetErrorCode to get detailed error code
* @returns ::BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT decoding is blocked until
* more input data is provided
* @returns ::BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT decoding is blocked until
* more output space is provided
* @returns ::BROTLI_DECODER_RESULT_SUCCESS decoding is finished, no more
* input might be consumed and no more output will be produced
*/
/**
* Checks if decoder has more output.
*
* @param state decoder instance
* @returns ::true, if decoder has some unconsumed output
* @returns ::false otherwise
*/
/**
* Acquires pointer to internal output buffer.
*
* This method is used to make language bindings easier and more efficient:
* -# push data to ::BrotliDecoderDecompressStream,
* until ::BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT is reported
* -# use ::BrotliDecoderTakeOutput to peek bytes and copy to language-specific
* entity
*
* Also this could be useful if there is an output stream that is able to
* consume all the provided data (e.g. when data is saved to file system).
*
* @attention After every call to ::BrotliDecoderTakeOutput @p *size bytes of
* output are considered consumed for all consecutive calls to the
* instance methods; returned pointer becomes invalidated as well.
*
* @note Decoder output is not guaranteed to be contiguous. This means that
* after the size-unrestricted call to ::BrotliDecoderTakeOutput,
* immediate next call to ::BrotliDecoderTakeOutput may return more data.
*
* @param state decoder instance
* @param[in, out] size @b in: number of bytes caller is ready to take, @c 0 if
* any amount could be handled; \n
* @b out: amount of data pointed by returned pointer and
* considered consumed; \n
* out value is never greater than in value, unless it is @c 0
* @returns pointer to output data
*/
/**
* Checks if instance has already consumed input.
*
* Instance that returns ::false is considered "fresh" and could be
* reused.
*
* @param state decoder instance
* @returns ::true if decoder has already used some input bytes
* @returns ::false otherwise
*/
/**
* Checks if decoder instance reached the final state.
*
* @param state decoder instance
* @returns ::true if decoder is in a state where it reached the end of
* the input and produced all of the output
* @returns ::false otherwise
*/
/**
* Acquires a detailed error code.
*
* Should be used only after ::BrotliDecoderDecompressStream returns
* ::BROTLI_DECODER_RESULT_ERROR.
*
* See also ::BrotliDecoderErrorString
*
* @param state decoder instance
* @returns last saved error code
*/
/**
* Converts error code to a c-string.
*/
/**
* Gets a decoder library version.
*
* Look at BROTLI_VERSION for more information.
*/
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Lookup table to map the previous two bytes to a context id. /* Lookup table to map the previous two bytes to a context id.
There are four different context modeling modes defined here: There are four different context modeling modes defined here:
@ -256,6 +80,7 @@ context ids and the type of the next byte is summarized in the table below:
| (208-) | | context: 2 - 3 | | | (208-) | | context: 2 - 3 | |
|-------------|-------------------|---------------------|------------------| |-------------|-------------------|---------------------|------------------|
*/ */
const ( const (
contextLSB6 = 0 contextLSB6 = 0
contextMSB6 = 1 contextMSB6 = 1

View File

@ -5,21 +5,7 @@ package brotli
Distributed under MIT license. Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/**
* @file
* API for Brotli decompression.
*/
/**
* Result type for ::BrotliDecoderDecompress and
* ::BrotliDecoderDecompressStream functions.
*/
const ( const (
decoderResultError = 0 decoderResultError = 0
decoderResultSuccess = 1 decoderResultSuccess = 1
@ -27,25 +13,6 @@ const (
decoderResultNeedsMoreOutput = 3 decoderResultNeedsMoreOutput = 3
) )
/**
* Template that evaluates items of ::BrotliDecoderErrorCode.
*
* Example: @code {.cpp}
* // Log Brotli error code.
* switch (brotliDecoderErrorCode) {
* #define CASE_(PREFIX, NAME, CODE) \
* case BROTLI_DECODER ## PREFIX ## NAME: \
* LOG(INFO) << "error code:" << #NAME; \
* break;
* #define NEWLINE_
* BROTLI_DECODER_ERROR_CODES_LIST(CASE_, NEWLINE_)
* #undef CASE_
* #undef NEWLINE_
* default: LOG(FATAL) << "unknown brotli error code";
* }
* @endcode
*/
/** /**
* Error code for detailed logging / production debugging. * Error code for detailed logging / production debugging.
* *

View File

@ -1,10 +1,5 @@
package brotli package brotli
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Copyright 2013 Google Inc. All Rights Reserved. /* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license. Distributed under MIT license.
@ -19,15 +14,6 @@ type dictionary struct {
data []byte data []byte
} }
/**
* Sets dictionary data.
*
* When dictionary data is already set / present, this method is no-op.
*
* Dictionary data MUST be provided before BrotliGetDictionary is invoked.
* This method is used ONLY in multi-client environment (e.g. C + Java),
* to reduce storage by sharing single dictionary between implementations.
*/
const minDictionaryWordLength = 4 const minDictionaryWordLength = 4
const maxDictionaryWordLength = 24 const maxDictionaryWordLength = 24

View File

@ -6,13 +6,6 @@ package brotli
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
/* Hash table on the 4-byte prefixes of static dictionary words. */
/* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Hash table on the 4-byte prefixes of static dictionary words. */ /* Hash table on the 4-byte prefixes of static dictionary words. */
var kStaticDictionaryHash = [32768]uint16{ var kStaticDictionaryHash = [32768]uint16{
32072, 32072,

View File

@ -11,19 +11,6 @@ import (
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
/* Constants and formulas that affect speed-ratio trade-offs and thus define
quality levels. */
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/**
* @file
* API for Brotli compression.
*/
/** Minimal value for ::BROTLI_PARAM_LGWIN parameter. */ /** Minimal value for ::BROTLI_PARAM_LGWIN parameter. */
const minWindowBits = 10 const minWindowBits = 10

View File

@ -8,13 +8,6 @@ import "math"
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
/* Entropy encoding (Huffman) utilities. */
/* Copyright 2010 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Entropy encoding (Huffman) utilities. */ /* Entropy encoding (Huffman) utilities. */
/* A node of a Huffman tree. */ /* A node of a Huffman tree. */
@ -30,35 +23,6 @@ func initHuffmanTree(self *huffmanTree, count uint32, left int16, right int16) {
self.index_right_or_value_ = right self.index_right_or_value_ = right
} }
/* Returns 1 is assignment of depths succeeded, otherwise 0. */
/* This function will create a Huffman tree.
The (data,length) contains the population counts.
The tree_limit is the maximum bit depth of the Huffman codes.
The depth contains the tree, i.e., how many bits are used for
the symbol.
The actual Huffman tree is constructed in the tree[] array, which has to
be at least 2 * length + 1 long.
See http://en.wikipedia.org/wiki/Huffman_coding */
/* Change the population counts in a way that the consequent
Huffman tree compression, especially its RLE-part will be more
likely to compress this data more efficiently.
length contains the size of the histogram.
counts contains the population counts.
good_for_rle is a buffer of at least length size */
/* Write a Huffman tree from bit depths into the bit-stream representation
of a Huffman tree. The generated Huffman tree is to be compressed once
more using a Huffman tree */
/* Get the actual bit values for a tree of bit depths. */
/* Input size optimized Shell sort. */ /* Input size optimized Shell sort. */
type huffmanTreeComparator func(*huffmanTree, *huffmanTree) bool type huffmanTreeComparator func(*huffmanTree, *huffmanTree) bool
@ -109,6 +73,7 @@ func sortHuffmanTreeItems(items []huffmanTree, n uint, comparator huffmanTreeCom
} }
} }
/* Returns 1 if assignment of depths succeeded, otherwise 0. */
func setDepth(p0 int, pool []huffmanTree, depth []byte, max_depth int) bool { func setDepth(p0 int, pool []huffmanTree, depth []byte, max_depth int) bool {
var stack [16]int var stack [16]int
var level int = 0 var level int = 0
@ -333,6 +298,13 @@ func writeHuffmanTreeRepetitionsZeros(repetitions uint, tree_size *uint, tree []
} }
} }
/* Change the population counts in a way that the consequent
Huffman tree compression, especially its RLE-part will be more
likely to compress this data more efficiently.
length contains the size of the histogram.
counts contains the population counts.
good_for_rle is a buffer of at least length size */
func optimizeHuffmanCountsForRLE(length uint, counts []uint32, good_for_rle []byte) { func optimizeHuffmanCountsForRLE(length uint, counts []uint32, good_for_rle []byte) {
var nonzero_count uint = 0 var nonzero_count uint = 0
var stride uint var stride uint
@ -510,6 +482,9 @@ func decideOverRLEUse(depth []byte, length uint, use_rle_for_non_zero *bool, use
*use_rle_for_zero = total_reps_zero > count_reps_zero*2 *use_rle_for_zero = total_reps_zero > count_reps_zero*2
} }
/* Write a Huffman tree from bit depths into the bit-stream representation
of a Huffman tree. The generated Huffman tree is to be compressed once
more using a Huffman tree */
func writeHuffmanTree(depth []byte, length uint, tree_size *uint, tree []byte, extra_bits_data []byte) { func writeHuffmanTree(depth []byte, length uint, tree_size *uint, tree []byte, extra_bits_data []byte) {
var previous_value byte = initialRepeatedCodeLength var previous_value byte = initialRepeatedCodeLength
var i uint var i uint
@ -589,6 +564,7 @@ func reverseBits(num_bits uint, bits uint16) uint16 {
/* 0..15 are values for bits */ /* 0..15 are values for bits */
const maxHuffmanBits = 16 const maxHuffmanBits = 16
/* Get the actual bit values for a tree of bit depths. */
func convertBitDepthsToSymbols(depth []byte, len uint, bits []uint16) { func convertBitDepthsToSymbols(depth []byte, len uint, bits []uint16) {
var bl_count = [maxHuffmanBits]uint16{0} var bl_count = [maxHuffmanBits]uint16{0}
var next_code [maxHuffmanBits]uint16 var next_code [maxHuffmanBits]uint16

View File

@ -8,27 +8,8 @@ import "math"
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
/* Literal cost model to allow backward reference replacement to be efficient.
*/
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Literal cost model to allow backward reference replacement to be efficient.
*/
/* Estimates how many bits the literals in the interval [pos, pos + len) in the
ring-buffer (data, mask) will take entropy coded and writes these estimates
to the cost[0..len) array. */
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Utilities for fast computation of logarithms. */ /* Utilities for fast computation of logarithms. */
func log2FloorNonZero(n uint) uint32 { func log2FloorNonZero(n uint) uint32 {
/* TODO: generalize and move to platform.h */ /* TODO: generalize and move to platform.h */
var result uint32 = 0 var result uint32 = 0

View File

@ -6,14 +6,6 @@ package brotli
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
/* A (forgetful) hash table to the data seen by the compressor, to
help create backward references to previous data. */
/* Copyright 2010 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Function to find maximal matching prefixes of strings. */ /* Function to find maximal matching prefixes of strings. */
func findMatchLengthWithLimit(s1 []byte, s2 []byte, limit uint) uint { func findMatchLengthWithLimit(s1 []byte, s2 []byte, limit uint) uint {
var matched uint = 0 var matched uint = 0

13
h10.go
View File

@ -2,19 +2,12 @@ package brotli
import "encoding/binary" import "encoding/binary"
/* NOLINT(build/header_guard) */
/* Copyright 2016 Google Inc. All Rights Reserved. /* Copyright 2016 Google Inc. All Rights Reserved.
Distributed under MIT license. Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
/* A (forgetful) hash table where each hash bucket contains a binary tree of
sequences whose first 4 bytes share the same hash code.
Each sequence is 128 long and is identified by its starting
position in the input data. The binary tree is sorted by the lexicographic
order of the sequences, and it is also a max-heap with respect to the
starting positions. */
func (*h10) HashTypeLength() uint { func (*h10) HashTypeLength() uint {
return 4 return 4
} }
@ -31,6 +24,12 @@ func hashBytesH10(data []byte) uint32 {
return h >> (32 - 17) return h >> (32 - 17)
} }
/* A (forgetful) hash table where each hash bucket contains a binary tree of
sequences whose first 4 bytes share the same hash code.
Each sequence is 128 long and is identified by its starting
position in the input data. The binary tree is sorted by the lexicographic
order of the sequences, and it is also a max-heap with respect to the
starting positions. */
type h10 struct { type h10 struct {
hasherCommon hasherCommon
window_mask_ uint window_mask_ uint

1
h5.go
View File

@ -2,7 +2,6 @@ package brotli
import "encoding/binary" import "encoding/binary"
/* NOLINT(build/header_guard) */
/* Copyright 2010 Google Inc. All Rights Reserved. /* Copyright 2010 Google Inc. All Rights Reserved.
Distributed under MIT license. Distributed under MIT license.

1
h6.go
View File

@ -2,7 +2,6 @@ package brotli
import "encoding/binary" import "encoding/binary"
/* NOLINT(build/header_guard) */
/* Copyright 2010 Google Inc. All Rights Reserved. /* Copyright 2010 Google Inc. All Rights Reserved.
Distributed under MIT license. Distributed under MIT license.

24
hash.go
View File

@ -5,24 +5,6 @@ import (
"fmt" "fmt"
) )
/* Matches data against static dictionary words, and for each length l,
for which a match is found, updates matches[l] to be the minimum possible
(distance << 5) + len_code.
Returns 1 if matches have been found, otherwise 0.
Prerequisites:
matches array is at least BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN + 1 long
all elements are initialized to kInvalidMatch */
/* Pointer to hasher data.
*
* Excluding initialization and destruction, hasher can be passed as
* HasherHandle by value.
*
* Typically hasher data consists of 3 sections:
* * hasherCommon structure
* * private structured hasher data, depending on hasher type
* * private dynamic hasher data, depending on hasher type and parameters
*
*/
type hasherCommon struct { type hasherCommon struct {
params hasherParams params hasherParams
is_prepared_ bool is_prepared_ bool
@ -68,11 +50,11 @@ type hasherSearchResult struct {
* There is no effort to ensure that it is a prime, the oddity is enough * There is no effort to ensure that it is a prime, the oddity is enough
for this use. for this use.
* The number has been tuned heuristically against compression benchmarks. */ * The number has been tuned heuristically against compression benchmarks. */
var kHashMul32 uint32 = 0x1E35A7BD const kHashMul32 uint32 = 0x1E35A7BD
var kHashMul64 uint64 = 0x1E35A7BD1E35A7BD const kHashMul64 uint64 = 0x1E35A7BD1E35A7BD
var kHashMul64Long uint64 = 0x1FE35A7BD3579BD3 const kHashMul64Long uint64 = 0x1FE35A7BD3579BD3
func hash14(data []byte) uint32 { func hash14(data []byte) uint32 {
var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32 var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32

View File

@ -1,16 +1,11 @@
package brotli package brotli
/* NOTE: this hasher does not search in the dictionary. It is used as
backup-hasher, the main hasher already searches in it. */
/* NOLINT(build/header_guard) */
/* Copyright 2018 Google Inc. All Rights Reserved. /* Copyright 2018 Google Inc. All Rights Reserved.
Distributed under MIT license. Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
/* Composite hasher: This hasher allows to combine two other hashers, HASHER_A
and HASHER_B. */
func (h *hashComposite) HashTypeLength() uint { func (h *hashComposite) HashTypeLength() uint {
var a uint = h.ha.HashTypeLength() var a uint = h.ha.HashTypeLength()
var b uint = h.hb.HashTypeLength() var b uint = h.hb.HashTypeLength()
@ -31,6 +26,8 @@ func (h *hashComposite) StoreLookahead() uint {
} }
} }
/* Composite hasher: This hasher allows to combine two other hashers, HASHER_A
and HASHER_B. */
type hashComposite struct { type hashComposite struct {
hasherCommon hasherCommon
ha hasherHandle ha hasherHandle

View File

@ -2,19 +2,12 @@ package brotli
import "encoding/binary" import "encoding/binary"
/* NOLINT(build/header_guard) */
/* Copyright 2016 Google Inc. All Rights Reserved. /* Copyright 2016 Google Inc. All Rights Reserved.
Distributed under MIT license. Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
/* A (forgetful) hash table to the data seen by the compressor, to
help create backward references to previous data.
Hashes are stored in chains which are bucketed to groups. Group of chains
share a storage "bank". When more than "bank size" chain nodes are added,
oldest nodes are replaced; this way several chains may share a tail. */
func (*hashForgetfulChain) HashTypeLength() uint { func (*hashForgetfulChain) HashTypeLength() uint {
return 4 return 4
} }
@ -37,6 +30,12 @@ type slot struct {
next uint16 next uint16
} }
/* A (forgetful) hash table to the data seen by the compressor, to
help create backward references to previous data.
Hashes are stored in chains which are bucketed to groups. Group of chains
share a storage "bank". When more than "bank size" chain nodes are added,
oldest nodes are replaced; this way several chains may share a tail. */
type hashForgetfulChain struct { type hashForgetfulChain struct {
hasherCommon hasherCommon

View File

@ -2,7 +2,6 @@ package brotli
import "encoding/binary" import "encoding/binary"
/* NOLINT(build/header_guard) */
/* Copyright 2010 Google Inc. All Rights Reserved. /* Copyright 2010 Google Inc. All Rights Reserved.
Distributed under MIT license. Distributed under MIT license.
@ -12,6 +11,7 @@ import "encoding/binary"
/* For BUCKET_SWEEP == 1, enabling the dictionary lookup makes compression /* For BUCKET_SWEEP == 1, enabling the dictionary lookup makes compression
a little faster (0.5% - 1%) and it compresses 0.15% better on small text a little faster (0.5% - 1%) and it compresses 0.15% better on small text
and HTML inputs. */ and HTML inputs. */
func (*hashLongestMatchQuickly) HashTypeLength() uint { func (*hashLongestMatchQuickly) HashTypeLength() uint {
return 8 return 8
} }

View File

@ -1,16 +1,14 @@
package brotli package brotli
/* NOTE: this hasher does not search in the dictionary. It is used as
backup-hasher, the main hasher already searches in it. */
/* NOLINT(build/header_guard) */
/* Copyright 2018 Google Inc. All Rights Reserved. /* Copyright 2018 Google Inc. All Rights Reserved.
Distributed under MIT license. Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
/* Rolling hash for long distance long string matches. Stores one position /* NOTE: this hasher does not search in the dictionary. It is used as
per bucket, bucket key is computed over a long region. */ backup-hasher, the main hasher already searches in it. */
var kRollingHashMul32hashRolling uint32 = 69069 var kRollingHashMul32hashRolling uint32 = 69069
var kInvalidPosHashRolling uint32 = 0xffffffff var kInvalidPosHashRolling uint32 = 0xffffffff
@ -40,6 +38,8 @@ func (h *hashRolling) HashRollingFunction(state uint32, add byte, rem byte, fact
return uint32(factor*state + h.HashByte(add) - factor_remove*h.HashByte(rem)) return uint32(factor*state + h.HashByte(add) - factor_remove*h.HashByte(rem))
} }
/* Rolling hash for long distance long string matches. Stores one position
per bucket, bucket key is computed over a long region. */
type hashRolling struct { type hashRolling struct {
hasherCommon hasherCommon

View File

@ -7,6 +7,7 @@ package brotli
*/ */
/* Utilities for building Huffman decoding tables. */ /* Utilities for building Huffman decoding tables. */
const huffmanMaxCodeLength = 15 const huffmanMaxCodeLength = 15
/* Maximum possible Huffman table size for an alphabet size of (index * 32), /* Maximum possible Huffman table size for an alphabet size of (index * 32),

View File

@ -6,8 +6,6 @@ package brotli
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
/* Macros for memory management. */
/* /*
Dynamically grows array capacity to at least the requested size Dynamically grows array capacity to at least the requested size
T: data type T: data type

View File

@ -6,31 +6,9 @@ package brotli
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
/* Brotli bit stream functions to support the low level format. There are no
compression algorithms here, just the right ordering of bits to match the
specs. */
/* Copyright 2014 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Functions to convert brotli-related data structures into the
brotli bit stream. The functions here operate under
assumption that there is enough space in the storage, i.e., there are
no out-of-range checks anywhere.
These functions do bit addressing into a byte array. The byte array
is called "storage" and the index to the bit is called storage_ix
in function arguments. */
/* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Algorithms for distributing the literals and commands of a metablock between /* Algorithms for distributing the literals and commands of a metablock between
block types and contexts. */ block types and contexts. */
type metaBlockSplit struct { type metaBlockSplit struct {
literal_split blockSplit literal_split blockSplit
command_split blockSplit command_split blockSplit
@ -74,14 +52,6 @@ func destroyMetaBlockSplit(mb *metaBlockSplit) {
mb.distance_histograms = nil mb.distance_histograms = nil
} }
/* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Algorithms for distributing the literals and commands of a metablock between
block types and contexts. */
func initDistanceParams(params *encoderParams, npostfix uint32, ndirect uint32) { func initDistanceParams(params *encoderParams, npostfix uint32, ndirect uint32) {
var dist_params *distanceParams = &params.dist var dist_params *distanceParams = &params.dist
var alphabet_size uint32 var alphabet_size uint32

View File

@ -1,6 +1,5 @@
package brotli package brotli
/* NOLINT(build/header_guard) */
/* Copyright 2015 Google Inc. All Rights Reserved. /* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license. Distributed under MIT license.

View File

@ -1,6 +1,5 @@
package brotli package brotli
/* NOLINT(build/header_guard) */
/* Copyright 2015 Google Inc. All Rights Reserved. /* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license. Distributed under MIT license.

View File

@ -1,6 +1,5 @@
package brotli package brotli
/* NOLINT(build/header_guard) */
/* Copyright 2015 Google Inc. All Rights Reserved. /* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license. Distributed under MIT license.

192
params.go
View File

@ -1,197 +1,5 @@
package brotli package brotli
/**
* Opaque structure that holds encoder state.
*
* Allocated and initialized with ::BrotliEncoderCreateInstance.
* Cleaned up and deallocated with ::BrotliEncoderDestroyInstance.
*/
/**
* Sets the specified parameter to the given encoder instance.
*
* @param state encoder instance
* @param param parameter to set
* @param value new parameter value
* @returns ::false if parameter is unrecognized, or value is invalid
* @returns ::false if value of parameter can not be changed at current
* encoder state (e.g. when encoding is started, window size might be
* already encoded and therefore it is impossible to change it)
* @returns ::true if value is accepted
* @warning invalid values might be accepted in case they would not break
* encoding process.
*/
/**
* Creates an instance of ::BrotliEncoderState and initializes it.
*
* @p alloc_func and @p free_func @b MUST be both zero or both non-zero. In the
* case they are both zero, default memory allocators are used. @p opaque is
* passed to @p alloc_func and @p free_func when they are called. @p free_func
* has to return without doing anything when asked to free a NULL pointer.
*
* @param alloc_func custom memory allocation function
* @param free_func custom memory free function
* @param opaque custom memory manager handle
* @returns @c 0 if instance can not be allocated or initialized
* @returns pointer to initialized ::BrotliEncoderState otherwise
*/
/**
* Deinitializes and frees ::BrotliEncoderState instance.
*
* @param state decoder instance to be cleaned up and deallocated
*/
/**
* Calculates the output size bound for the given @p input_size.
*
* @warning Result is only valid if quality is at least @c 2 and, in
* case ::BrotliEncoderCompressStream was used, no flushes
* (::BROTLI_OPERATION_FLUSH) were performed.
*
* @param input_size size of projected input
* @returns @c 0 if result does not fit @c size_t
*/
/**
* Performs one-shot memory-to-memory compression.
*
* Compresses the data in @p input_buffer into @p encoded_buffer, and sets
* @p *encoded_size to the compressed length.
*
* @note If ::BrotliEncoderMaxCompressedSize(@p input_size) returns non-zero
* value, then output is guaranteed to be no longer than that.
*
* @param quality quality parameter value, e.g. ::BROTLI_DEFAULT_QUALITY
* @param lgwin lgwin parameter value, e.g. ::BROTLI_DEFAULT_WINDOW
* @param mode mode parameter value, e.g. ::BROTLI_DEFAULT_MODE
* @param input_size size of @p input_buffer
* @param input_buffer input data buffer with at least @p input_size
* addressable bytes
* @param[in, out] encoded_size @b in: size of @p encoded_buffer; \n
* @b out: length of compressed data written to
* @p encoded_buffer, or @c 0 if compression fails
* @param encoded_buffer compressed data destination buffer
* @returns ::false in case of compression error
* @returns ::false if output buffer is too small
* @returns ::true otherwise
*/
/**
* Compresses input stream to output stream.
*
* The values @p *available_in and @p *available_out must specify the number of
* bytes addressable at @p *next_in and @p *next_out respectively.
* When @p *available_out is @c 0, @p next_out is allowed to be @c NULL.
*
* After each call, @p *available_in will be decremented by the amount of input
* bytes consumed, and the @p *next_in pointer will be incremented by that
* amount. Similarly, @p *available_out will be decremented by the amount of
* output bytes written, and the @p *next_out pointer will be incremented by
* that amount.
*
* @p total_out, if it is not a null-pointer, will be set to the number
* of bytes compressed since the last @p state initialization.
*
*
*
* Internally workflow consists of 3 tasks:
* -# (optionally) copy input data to internal buffer
* -# actually compress data and (optionally) store it to internal buffer
* -# (optionally) copy compressed bytes from internal buffer to output stream
*
* Whenever all 3 tasks can't move forward anymore, or error occurs, this
* method returns the control flow to caller.
*
* @p op is used to perform flush, finish the stream, or inject metadata block.
* See ::BrotliEncoderOperation for more information.
*
* Flushing the stream means forcing encoding of all input passed to encoder and
* completing the current output block, so it could be fully decoded by stream
* decoder. To perform flush set @p op to ::BROTLI_OPERATION_FLUSH.
* Under some circumstances (e.g. lack of output stream capacity) this operation
* would require several calls to ::BrotliEncoderCompressStream. The method must
* be called again until both input stream is depleted and encoder has no more
* output (see ::BrotliEncoderHasMoreOutput) after the method is called.
*
* Finishing the stream means encoding of all input passed to encoder and
* adding specific "final" marks, so stream decoder could determine that stream
* is complete. To perform finish set @p op to ::BROTLI_OPERATION_FINISH.
* Under some circumstances (e.g. lack of output stream capacity) this operation
* would require several calls to ::BrotliEncoderCompressStream. The method must
* be called again until both input stream is depleted and encoder has no more
* output (see ::BrotliEncoderHasMoreOutput) after the method is called.
*
* @warning When flushing and finishing, @p op should not change until operation
* is complete; input stream should not be swapped, reduced or
* extended as well.
*
* @param state encoder instance
* @param op requested operation
* @param[in, out] available_in @b in: amount of available input; \n
* @b out: amount of unused input
* @param[in, out] next_in pointer to the next input byte
* @param[in, out] available_out @b in: length of output buffer; \n
* @b out: remaining size of output buffer
* @param[in, out] next_out compressed output buffer cursor;
* can be @c NULL if @p available_out is @c 0
* @param[out] total_out number of bytes produced so far; can be @c NULL
* @returns ::false if there was an error
* @returns ::true otherwise
*/
/**
* Checks if encoder instance reached the final state.
*
* @param state encoder instance
* @returns ::true if encoder is in a state where it reached the end of
* the input and produced all of the output
* @returns ::false otherwise
*/
/**
* Checks if encoder has more output.
*
* @param state encoder instance
* @returns ::true, if encoder has some unconsumed output
* @returns ::false otherwise
*/
/**
* Acquires pointer to internal output buffer.
*
* This method is used to make language bindings easier and more efficient:
* -# push data to ::BrotliEncoderCompressStream,
* until ::BrotliEncoderHasMoreOutput returns BROTL_TRUE
* -# use ::BrotliEncoderTakeOutput to peek bytes and copy to language-specific
* entity
*
* Also this could be useful if there is an output stream that is able to
* consume all the provided data (e.g. when data is saved to file system).
*
* @attention After every call to ::BrotliEncoderTakeOutput @p *size bytes of
* output are considered consumed for all consecutive calls to the
* instance methods; returned pointer becomes invalidated as well.
*
* @note Encoder output is not guaranteed to be contiguous. This means that
* after the size-unrestricted call to ::BrotliEncoderTakeOutput,
* immediate next call to ::BrotliEncoderTakeOutput may return more data.
*
* @param state encoder instance
* @param[in, out] size @b in: number of bytes caller is ready to take, @c 0 if
* any amount could be handled; \n
* @b out: amount of data pointed by returned pointer and
* considered consumed; \n
* out value is never greater than in value, unless it is @c 0
* @returns pointer to output data
*/
/**
* Gets an encoder library version.
*
* Look at BROTLI_VERSION for more information.
*/
/* Copyright 2017 Google Inc. All Rights Reserved. /* Copyright 2017 Google Inc. All Rights Reserved.
Distributed under MIT license. Distributed under MIT license.

View File

@ -6,30 +6,6 @@ package brotli
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
/* Bit reading helpers */
/* Copyright 2016 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Macros for compiler / platform specific features and build options.
Build options are:
* BROTLI_BUILD_32_BIT disables 64-bit optimizations
* BROTLI_BUILD_64_BIT forces to use 64-bit optimizations
* BROTLI_BUILD_BIG_ENDIAN forces to use big-endian optimizations
* BROTLI_BUILD_ENDIAN_NEUTRAL disables endian-aware optimizations
* BROTLI_BUILD_LITTLE_ENDIAN forces to use little-endian optimizations
* BROTLI_BUILD_PORTABLE disables dangerous optimizations, like unaligned
read and overlapping memcpy; this reduces decompression speed by 5%
* BROTLI_BUILD_NO_RBIT disables "rbit" optimization for ARM CPUs
* BROTLI_DEBUG dumps file name and line number when decoder detects stream
or memory error
* BROTLI_ENABLE_LOG enables asserts and dumps various state information
*/
type brotli_reg_t uint64
func brotli_min_double(a float64, b float64) float64 { func brotli_min_double(a float64, b float64) float64 {
if a < b { if a < b {
return a return a

View File

@ -6,27 +6,6 @@ package brotli
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
/* Function to find backward reference copies. */
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Function to find backward reference copies. */
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* This class models a sequence of literals and a backward reference copy. */
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Functions for encoding of integers into prefix codes the amount of extra /* Functions for encoding of integers into prefix codes the amount of extra
bits, and the actual values of the extra bits. */ bits, and the actual values of the extra bits. */

View File

@ -6,15 +6,6 @@ package brotli
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
/* Implementation of Brotli compressor. */
/* Copyright 2013 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Sliding window over the input data. */
/* A ringBuffer(window_bits, tail_bits) contains `1 << window_bits' bytes of /* A ringBuffer(window_bits, tail_bits) contains `1 << window_bits' bytes of
data in a circular manner: writing a byte writes it to: data in a circular manner: writing a byte writes it to:
`position() % (1 << window_bits)'. `position() % (1 << window_bits)'.

View File

@ -9,6 +9,7 @@ import "encoding/binary"
*/ */
/* Class to model the static dictionary. */ /* Class to model the static dictionary. */
const maxStaticDictionaryMatchLen = 37 const maxStaticDictionaryMatchLen = 37
var kInvalidMatch uint32 = 0xFFFFFFF var kInvalidMatch uint32 = 0xFFFFFFF

View File

@ -5,18 +5,9 @@ package brotli
Distributed under MIT license. Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/ */
/* Copyright 2017 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Lookup table for static dictionary and transforms. */ /* Lookup table for static dictionary and transforms. */
type dictWord struct { type dictWord struct {
len byte len byte
transform byte transform byte

View File

@ -7,6 +7,7 @@ package brotli
*/ */
/* Utilities for building Huffman decoding tables. */ /* Utilities for building Huffman decoding tables. */
type symbolList struct { type symbolList struct {
storage []uint16 storage []uint16
offset int offset int

View File

@ -1,11 +1,5 @@
package brotli package brotli
/* transforms is a part of ABI, but not API.
It means that there are some functions that are supposed to be in "common"
library, but header itself is not placed into include/brotli. This way,
aforementioned functions will be available only to brotli internals.
*/
const ( const (
transformIdentity = 0 transformIdentity = 0
transformOmitLast1 = 1 transformOmitLast1 = 1
@ -45,7 +39,6 @@ type transforms struct {
cutOffTransforms [transformsMaxCutOff + 1]int16 cutOffTransforms [transformsMaxCutOff + 1]int16
} }
/* T is BrotliTransforms*; result is uint8_t. */
func transformPrefixId(t *transforms, I int) byte { func transformPrefixId(t *transforms, I int) byte {
return t.transforms[(I*3)+0] return t.transforms[(I*3)+0]
} }
@ -58,7 +51,6 @@ func transformSuffixId(t *transforms, I int) byte {
return t.transforms[(I*3)+2] return t.transforms[(I*3)+2]
} }
/* T is BrotliTransforms*; result is const uint8_t*. */
func transformPrefix(t *transforms, I int) []byte { func transformPrefix(t *transforms, I int) []byte {
return t.prefix_suffix[t.prefix_suffix_map[transformPrefixId(t, I)]:] return t.prefix_suffix[t.prefix_suffix_map[transformPrefixId(t, I)]:]
} }
@ -70,19 +62,6 @@ func transformSuffix(t *transforms, I int) []byte {
/* RFC 7932 transforms string data */ /* RFC 7932 transforms string data */
var kPrefixSuffix string = "\001 \002, \010 of the \004 of \002s \001.\005 and \004 " + "in \001\"\004 to \002\">\001\n\002. \001]\005 for \003 a \006 " + "that \001'\006 with \006 from \004 by \001(\006. T" + "he \004 on \004 as \004 is \004ing \002\n\t\001:\003ed " + "\002=\"\004 at \003ly \001,\002='\005.com/\007. This \005" + " not \003er \003al \004ful \004ive \005less \004es" + "t \004ize \002\xc2\xa0\004ous \005 the \002e \000" var kPrefixSuffix string = "\001 \002, \010 of the \004 of \002s \001.\005 and \004 " + "in \001\"\004 to \002\">\001\n\002. \001]\005 for \003 a \006 " + "that \001'\006 with \006 from \004 by \001(\006. T" + "he \004 on \004 as \004 is \004ing \002\n\t\001:\003ed " + "\002=\"\004 at \003ly \001,\002='\005.com/\007. This \005" + " not \003er \003al \004ful \004ive \005less \004es" + "t \004ize \002\xc2\xa0\004ous \005 the \002e \000"
/* 0x _0 _2 __5 _E _3 _6 _8 _E */
/* 2x _3_ _5 _A_ _D_ _F _2 _4 _A _E */
/* 4x _5_ _7 _E _5 _A _C */
/* 6x _3 _8 _D _2 _7_ _ _A _C */
/* 8x _0 _ _3 _8 _C _E _ _1 _7 _F */
/* Ax _5 _9 _D _2 _7 _D */
/* Cx _2 _7___ ___ _A _F _5 _8 */
var kPrefixSuffixMap = [50]uint16{ var kPrefixSuffixMap = [50]uint16{
0x00, 0x00,
0x02, 0x02,

View File

@ -1,40 +1,5 @@
package brotli package brotli
/* Uses the slow shortest-path block splitter and does context clustering.
The distance parameters are dynamically selected based on the commands
which get recomputed under the new distance parameters. The new distance
parameters are stored into *params. */
/* Uses a fast greedy block splitter that tries to merge current block with the
last or the second last block and uses a static context clustering which
is the same for all block types. */
/* All Store functions here will use a storage_ix, which is always the bit
position for the current storage. */
/* REQUIRES: length > 0 */
/* REQUIRES: length <= (1 << 24) */
/* Stores the meta-block without doing any block splitting, just collects
one histogram per block category and uses that for entropy coding.
REQUIRES: length > 0
REQUIRES: length <= (1 << 24) */
/* Same as above, but uses static prefix codes for histograms with a only a few
symbols, and uses static code length prefix codes for all other histograms.
REQUIRES: length > 0
REQUIRES: length <= (1 << 24) */
/* This is for storing uncompressed blocks (simple raw storage of
bytes-as-bytes).
REQUIRES: length > 0
REQUIRES: length <= (1 << 24) */
/* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license.
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
*/
/* Static entropy codes used for faster meta-block encoding. */
/* Copyright 2010 Google Inc. All Rights Reserved. /* Copyright 2010 Google Inc. All Rights Reserved.
Distributed under MIT license. Distributed under MIT license.
@ -43,8 +8,6 @@ package brotli
/* Write bits into a byte array. */ /* Write bits into a byte array. */
/*#define BIT_WRITER_DEBUG */
/* This function writes bits into bytes in increasing addresses, and within /* This function writes bits into bytes in increasing addresses, and within
a byte least-significant-bit first. a byte least-significant-bit first.