forked from mirror/brotli
Clean up duplicate and misplaced comments.
This commit is contained in:
parent
55e85f3f75
commit
52165d4974
|
@ -7,18 +7,7 @@ package brotli
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Function to find backward reference copies. */
|
/* Function to find backward reference copies. */
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Function to find backward reference copies. */
|
|
||||||
|
|
||||||
/* "commands" points to the next output command to write to, "*num_commands" is
|
|
||||||
initially the total amount of commands output by previous
|
|
||||||
CreateBackwardReferences calls, and must be incremented by the amount written
|
|
||||||
by this call. */
|
|
||||||
func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uint {
|
func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uint {
|
||||||
if distance <= max_distance {
|
if distance <= max_distance {
|
||||||
var distance_plus_3 uint = distance + 3
|
var distance_plus_3 uint = distance + 3
|
||||||
|
@ -42,6 +31,10 @@ func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uin
|
||||||
return distance + numDistanceShortCodes - 1
|
return distance + numDistanceShortCodes - 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* "commands" points to the next output command to write to, "*num_commands" is
|
||||||
|
initially the total amount of commands output by previous
|
||||||
|
CreateBackwardReferences calls, and must be incremented by the amount written
|
||||||
|
by this call. */
|
||||||
func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
|
func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) {
|
||||||
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
|
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
|
||||||
var orig_commands []command = commands
|
var orig_commands []command = commands
|
||||||
|
|
|
@ -13,19 +13,6 @@ type zopfliNode struct {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Computes the shortest path of commands from position to at most
|
|
||||||
position + num_bytes.
|
|
||||||
|
|
||||||
On return, path->size() is the number of commands found and path[i] is the
|
|
||||||
length of the i-th command (copy length plus insert length).
|
|
||||||
Note that the sum of the lengths of all commands can be less than num_bytes.
|
|
||||||
|
|
||||||
On return, the nodes[0..num_bytes] array will have the following
|
|
||||||
"ZopfliNode array invariant":
|
|
||||||
For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then
|
|
||||||
(1) nodes[i].copy_length() >= 2
|
|
||||||
(2) nodes[i].command_length() <= i and
|
|
||||||
(3) nodes[i - nodes[i].command_length()].cost < kInfinity */
|
|
||||||
const maxEffectiveDistanceAlphabetSize = 544
|
const maxEffectiveDistanceAlphabetSize = 544
|
||||||
|
|
||||||
var kInfinity float32 = 1.7e38 /* ~= 2 ^ 127 */
|
var kInfinity float32 = 1.7e38 /* ~= 2 ^ 127 */
|
||||||
|
@ -618,7 +605,21 @@ func zopfliIterate(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_
|
||||||
return computeShortestPathFromNodes(num_bytes, nodes)
|
return computeShortestPathFromNodes(num_bytes, nodes)
|
||||||
}
|
}
|
||||||
|
|
||||||
/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
|
/* Computes the shortest path of commands from position to at most
|
||||||
|
position + num_bytes.
|
||||||
|
|
||||||
|
On return, path->size() is the number of commands found and path[i] is the
|
||||||
|
length of the i-th command (copy length plus insert length).
|
||||||
|
Note that the sum of the lengths of all commands can be less than num_bytes.
|
||||||
|
|
||||||
|
On return, the nodes[0..num_bytes] array will have the following
|
||||||
|
"ZopfliNode array invariant":
|
||||||
|
For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then
|
||||||
|
(1) nodes[i].copy_length() >= 2
|
||||||
|
(2) nodes[i].command_length() <= i and
|
||||||
|
(3) nodes[i - nodes[i].command_length()].cost < kInfinity
|
||||||
|
|
||||||
|
REQUIRES: nodes != nil and len(nodes) >= num_bytes + 1 */
|
||||||
func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, dist_cache []int, hasher *h10, nodes []zopfliNode) uint {
|
func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, dist_cache []int, hasher *h10, nodes []zopfliNode) uint {
|
||||||
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
|
var max_backward_limit uint = maxBackwardLimit(params.lgwin)
|
||||||
var max_zopfli_len uint = maxZopfliLen(params)
|
var max_zopfli_len uint = maxZopfliLen(params)
|
||||||
|
|
|
@ -6,13 +6,6 @@ package brotli
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Functions to estimate the bit cost of Huffman trees. */
|
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Functions to estimate the bit cost of Huffman trees. */
|
/* Functions to estimate the bit cost of Huffman trees. */
|
||||||
func shannonEntropy(population []uint32, size uint, total *uint) float64 {
|
func shannonEntropy(population []uint32, size uint, total *uint) float64 {
|
||||||
var sum uint = 0
|
var sum uint = 0
|
||||||
|
|
|
@ -9,6 +9,7 @@ import "encoding/binary"
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Bit reading helpers */
|
/* Bit reading helpers */
|
||||||
|
|
||||||
const shortFillBitWindowRead = (8 >> 1)
|
const shortFillBitWindowRead = (8 >> 1)
|
||||||
|
|
||||||
var kBitMask = [33]uint32{
|
var kBitMask = [33]uint32{
|
||||||
|
|
|
@ -6,21 +6,8 @@ package brotli
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Build per-context histograms of literals, commands and distance codes. */
|
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Models the histograms of literals, commands and distance codes. */
|
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Block split point selection utilities. */
|
/* Block split point selection utilities. */
|
||||||
|
|
||||||
type blockSplit struct {
|
type blockSplit struct {
|
||||||
num_types uint
|
num_types uint
|
||||||
num_blocks uint
|
num_blocks uint
|
||||||
|
|
|
@ -2,12 +2,12 @@ package brotli
|
||||||
|
|
||||||
import "math"
|
import "math"
|
||||||
|
|
||||||
/* NOLINT(build/header_guard) */
|
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||||
|
|
||||||
Distributed under MIT license.
|
Distributed under MIT license.
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
func initialEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) {
|
func initialEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) {
|
||||||
var seed uint32 = 7
|
var seed uint32 = 7
|
||||||
var block_length uint = length / num_histograms
|
var block_length uint = length / num_histograms
|
||||||
|
|
|
@ -2,12 +2,12 @@ package brotli
|
||||||
|
|
||||||
import "math"
|
import "math"
|
||||||
|
|
||||||
/* NOLINT(build/header_guard) */
|
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||||
|
|
||||||
Distributed under MIT license.
|
Distributed under MIT license.
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
func initialEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) {
|
func initialEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) {
|
||||||
var seed uint32 = 7
|
var seed uint32 = 7
|
||||||
var block_length uint = length / num_histograms
|
var block_length uint = length / num_histograms
|
||||||
|
|
|
@ -2,12 +2,12 @@ package brotli
|
||||||
|
|
||||||
import "math"
|
import "math"
|
||||||
|
|
||||||
/* NOLINT(build/header_guard) */
|
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||||
|
|
||||||
Distributed under MIT license.
|
Distributed under MIT license.
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
func initialEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) {
|
func initialEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) {
|
||||||
var seed uint32 = 7
|
var seed uint32 = 7
|
||||||
var block_length uint = length / num_histograms
|
var block_length uint = length / num_histograms
|
||||||
|
|
15
cluster.go
15
cluster.go
|
@ -6,14 +6,8 @@ package brotli
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Block split point selection utilities. */
|
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Functions for clustering similar histograms together. */
|
/* Functions for clustering similar histograms together. */
|
||||||
|
|
||||||
type histogramPair struct {
|
type histogramPair struct {
|
||||||
idx1 uint32
|
idx1 uint32
|
||||||
idx2 uint32
|
idx2 uint32
|
||||||
|
@ -21,13 +15,6 @@ type histogramPair struct {
|
||||||
cost_diff float64
|
cost_diff float64
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Functions for clustering similar histograms together. */
|
|
||||||
func histogramPairIsLess(p1 *histogramPair, p2 *histogramPair) bool {
|
func histogramPairIsLess(p1 *histogramPair, p2 *histogramPair) bool {
|
||||||
if p1.cost_diff != p2.cost_diff {
|
if p1.cost_diff != p2.cost_diff {
|
||||||
return p1.cost_diff > p2.cost_diff
|
return p1.cost_diff > p2.cost_diff
|
||||||
|
|
|
@ -2,7 +2,6 @@ package brotli
|
||||||
|
|
||||||
import "math"
|
import "math"
|
||||||
|
|
||||||
/* NOLINT(build/header_guard) */
|
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||||
|
|
||||||
Distributed under MIT license.
|
Distributed under MIT license.
|
||||||
|
|
|
@ -2,7 +2,6 @@ package brotli
|
||||||
|
|
||||||
import "math"
|
import "math"
|
||||||
|
|
||||||
/* NOLINT(build/header_guard) */
|
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||||
|
|
||||||
Distributed under MIT license.
|
Distributed under MIT license.
|
||||||
|
|
|
@ -2,7 +2,6 @@ package brotli
|
||||||
|
|
||||||
import "math"
|
import "math"
|
||||||
|
|
||||||
/* NOLINT(build/header_guard) */
|
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||||
|
|
||||||
Distributed under MIT license.
|
Distributed under MIT license.
|
||||||
|
|
|
@ -15,41 +15,11 @@ import "encoding/binary"
|
||||||
|
|
||||||
Adapted from the CompressFragment() function in
|
Adapted from the CompressFragment() function in
|
||||||
https://github.com/google/snappy/blob/master/snappy.cc */
|
https://github.com/google/snappy/blob/master/snappy.cc */
|
||||||
/* Copyright 2015 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
const maxDistance_compress_fragment = 262128
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Function for fast encoding of an input fragment, independently from the input
|
|
||||||
history. This function uses one-pass processing: when we find a backward
|
|
||||||
match, we immediately emit the corresponding command and literal codes to
|
|
||||||
the bit stream. */
|
|
||||||
|
|
||||||
/* Compresses "input" string to the "*storage" buffer as one or more complete
|
|
||||||
meta-blocks, and updates the "*storage_ix" bit position.
|
|
||||||
|
|
||||||
If "is_last" is 1, emits an additional empty last meta-block.
|
|
||||||
|
|
||||||
"cmd_depth" and "cmd_bits" contain the command and distance prefix codes
|
|
||||||
(see comment in encode.h) used for the encoding of this input fragment.
|
|
||||||
If "is_last" is 0, they are updated to reflect the statistics
|
|
||||||
of this input fragment, to be used for the encoding of the next fragment.
|
|
||||||
|
|
||||||
"*cmd_code_numbits" is the number of bits of the compressed representation
|
|
||||||
of the command and distance prefix codes, and "cmd_code" is an array of
|
|
||||||
at least "(*cmd_code_numbits + 7) >> 3" size that contains the compressed
|
|
||||||
command and distance prefix codes. If "is_last" is 0, these are also
|
|
||||||
updated to represent the updated "cmd_depth" and "cmd_bits".
|
|
||||||
|
|
||||||
REQUIRES: "input_size" is greater than zero, or "is_last" is 1.
|
|
||||||
REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24).
|
|
||||||
REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero.
|
|
||||||
REQUIRES: "table_size" is an odd (9, 11, 13, 15) power of two
|
|
||||||
OUTPUT: maximal copy distance <= |input_size|
|
|
||||||
OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */
|
|
||||||
func hash5(p []byte, shift uint) uint32 {
|
func hash5(p []byte, shift uint) uint32 {
|
||||||
var h uint64 = (binary.LittleEndian.Uint64(p) << 24) * uint64(kHashMul32_a)
|
var h uint64 = (binary.LittleEndian.Uint64(p) << 24) * uint64(kHashMul32)
|
||||||
return uint32(h >> shift)
|
return uint32(h >> shift)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -57,7 +27,7 @@ func hashBytesAtOffset5(v uint64, offset int, shift uint) uint32 {
|
||||||
assert(offset >= 0)
|
assert(offset >= 0)
|
||||||
assert(offset <= 3)
|
assert(offset <= 3)
|
||||||
{
|
{
|
||||||
var h uint64 = ((v >> uint(8*offset)) << 24) * uint64(kHashMul32_a)
|
var h uint64 = ((v >> uint(8*offset)) << 24) * uint64(kHashMul32)
|
||||||
return uint32(h >> shift)
|
return uint32(h >> shift)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -821,6 +791,28 @@ next_block:
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Compresses "input" string to the "*storage" buffer as one or more complete
|
||||||
|
meta-blocks, and updates the "*storage_ix" bit position.
|
||||||
|
|
||||||
|
If "is_last" is 1, emits an additional empty last meta-block.
|
||||||
|
|
||||||
|
"cmd_depth" and "cmd_bits" contain the command and distance prefix codes
|
||||||
|
(see comment in encode.h) used for the encoding of this input fragment.
|
||||||
|
If "is_last" is 0, they are updated to reflect the statistics
|
||||||
|
of this input fragment, to be used for the encoding of the next fragment.
|
||||||
|
|
||||||
|
"*cmd_code_numbits" is the number of bits of the compressed representation
|
||||||
|
of the command and distance prefix codes, and "cmd_code" is an array of
|
||||||
|
at least "(*cmd_code_numbits + 7) >> 3" size that contains the compressed
|
||||||
|
command and distance prefix codes. If "is_last" is 0, these are also
|
||||||
|
updated to represent the updated "cmd_depth" and "cmd_bits".
|
||||||
|
|
||||||
|
REQUIRES: "input_size" is greater than zero, or "is_last" is 1.
|
||||||
|
REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24).
|
||||||
|
REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero.
|
||||||
|
REQUIRES: "table_size" is an odd (9, 11, 13, 15) power of two
|
||||||
|
OUTPUT: maximal copy distance <= |input_size|
|
||||||
|
OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */
|
||||||
func compressFragmentFast(input []byte, input_size uint, is_last bool, table []int, table_size uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, storage_ix *uint, storage []byte) {
|
func compressFragmentFast(input []byte, input_size uint, is_last bool, table []int, table_size uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, storage_ix *uint, storage []byte) {
|
||||||
var initial_storage_ix uint = *storage_ix
|
var initial_storage_ix uint = *storage_ix
|
||||||
var table_bits uint = uint(log2FloorNonZero(table_size))
|
var table_bits uint = uint(log2FloorNonZero(table_size))
|
||||||
|
|
|
@ -1,24 +0,0 @@
|
||||||
package brotli
|
|
||||||
|
|
||||||
/* Compresses "input" string to the "*storage" buffer as one or more complete
|
|
||||||
meta-blocks, and updates the "*storage_ix" bit position.
|
|
||||||
|
|
||||||
If "is_last" is 1, emits an additional empty last meta-block.
|
|
||||||
|
|
||||||
REQUIRES: "input_size" is greater than zero, or "is_last" is 1.
|
|
||||||
REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24).
|
|
||||||
REQUIRES: "command_buf" and "literal_buf" point to at least
|
|
||||||
kCompressFragmentTwoPassBlockSize long arrays.
|
|
||||||
REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero.
|
|
||||||
REQUIRES: "table_size" is a power of two
|
|
||||||
OUTPUT: maximal copy distance <= |input_size|
|
|
||||||
OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */
|
|
||||||
const maxDistance_compress_fragment = 262128
|
|
||||||
|
|
||||||
/* kHashMul32_a multiplier has these properties:
|
|
||||||
* The multiplier must be odd. Otherwise we may lose the highest bit.
|
|
||||||
* No long streaks of ones or zeros.
|
|
||||||
* There is no effort to ensure that it is a prime, the oddity is enough
|
|
||||||
for this use.
|
|
||||||
* The number has been tuned heuristically against compression benchmarks. */
|
|
||||||
var kHashMul32_a uint32 = 0x1E35A7BD
|
|
|
@ -13,28 +13,18 @@ import "encoding/binary"
|
||||||
the found backward matches and literal bytes into a buffer, and in the
|
the found backward matches and literal bytes into a buffer, and in the
|
||||||
second pass we emit them into the bit stream using prefix codes built based
|
second pass we emit them into the bit stream using prefix codes built based
|
||||||
on the actual command and literal byte histograms. */
|
on the actual command and literal byte histograms. */
|
||||||
/* Copyright 2015 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Function for fast encoding of an input fragment, independently from the input
|
|
||||||
history. This function uses two-pass processing: in the first pass we save
|
|
||||||
the found backward matches and literal bytes into a buffer, and in the
|
|
||||||
second pass we emit them into the bit stream using prefix codes built based
|
|
||||||
on the actual command and literal byte histograms. */
|
|
||||||
var kCompressFragmentTwoPassBlockSize uint = 1 << 17
|
var kCompressFragmentTwoPassBlockSize uint = 1 << 17
|
||||||
|
|
||||||
func hash1(p []byte, shift uint, length uint) uint32 {
|
func hash1(p []byte, shift uint, length uint) uint32 {
|
||||||
var h uint64 = (binary.LittleEndian.Uint64(p) << ((8 - length) * 8)) * uint64(kHashMul32_a)
|
var h uint64 = (binary.LittleEndian.Uint64(p) << ((8 - length) * 8)) * uint64(kHashMul32)
|
||||||
return uint32(h >> shift)
|
return uint32(h >> shift)
|
||||||
}
|
}
|
||||||
|
|
||||||
func hashBytesAtOffset(v uint64, offset uint, shift uint, length uint) uint32 {
|
func hashBytesAtOffset(v uint64, offset uint, shift uint, length uint) uint32 {
|
||||||
assert(offset <= 8-length)
|
assert(offset <= 8-length)
|
||||||
{
|
{
|
||||||
var h uint64 = ((v >> (8 * offset)) << ((8 - length) * 8)) * uint64(kHashMul32_a)
|
var h uint64 = ((v >> (8 * offset)) << ((8 - length) * 8)) * uint64(kHashMul32)
|
||||||
return uint32(h >> shift)
|
return uint32(h >> shift)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -721,6 +711,19 @@ func compressFragmentTwoPassImpl(input []byte, input_size uint, is_last bool, co
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Compresses "input" string to the "*storage" buffer as one or more complete
|
||||||
|
meta-blocks, and updates the "*storage_ix" bit position.
|
||||||
|
|
||||||
|
If "is_last" is 1, emits an additional empty last meta-block.
|
||||||
|
|
||||||
|
REQUIRES: "input_size" is greater than zero, or "is_last" is 1.
|
||||||
|
REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24).
|
||||||
|
REQUIRES: "command_buf" and "literal_buf" point to at least
|
||||||
|
kCompressFragmentTwoPassBlockSize long arrays.
|
||||||
|
REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero.
|
||||||
|
REQUIRES: "table_size" is a power of two
|
||||||
|
OUTPUT: maximal copy distance <= |input_size|
|
||||||
|
OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */
|
||||||
func compressFragmentTwoPass(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_size uint, storage_ix *uint, storage []byte) {
|
func compressFragmentTwoPass(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_size uint, storage_ix *uint, storage []byte) {
|
||||||
var initial_storage_ix uint = *storage_ix
|
var initial_storage_ix uint = *storage_ix
|
||||||
var table_bits uint = uint(log2FloorNonZero(table_size))
|
var table_bits uint = uint(log2FloorNonZero(table_size))
|
||||||
|
|
177
context.go
177
context.go
|
@ -1,181 +1,5 @@
|
||||||
package brotli
|
package brotli
|
||||||
|
|
||||||
/**
|
|
||||||
* Sets the specified parameter to the given decoder instance.
|
|
||||||
*
|
|
||||||
* @param state decoder instance
|
|
||||||
* @param param parameter to set
|
|
||||||
* @param value new parameter value
|
|
||||||
* @returns ::false if parameter is unrecognized, or value is invalid
|
|
||||||
* @returns ::true if value is accepted
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates an instance of ::BrotliDecoderState and initializes it.
|
|
||||||
*
|
|
||||||
* The instance can be used once for decoding and should then be destroyed with
|
|
||||||
* ::BrotliDecoderDestroyInstance, it cannot be reused for a new decoding
|
|
||||||
* session.
|
|
||||||
*
|
|
||||||
* @p alloc_func and @p free_func @b MUST be both zero or both non-zero. In the
|
|
||||||
* case they are both zero, default memory allocators are used. @p opaque is
|
|
||||||
* passed to @p alloc_func and @p free_func when they are called. @p free_func
|
|
||||||
* has to return without doing anything when asked to free a NULL pointer.
|
|
||||||
*
|
|
||||||
* @param alloc_func custom memory allocation function
|
|
||||||
* @param free_func custom memory free function
|
|
||||||
* @param opaque custom memory manager handle
|
|
||||||
* @returns @c 0 if instance can not be allocated or initialized
|
|
||||||
* @returns pointer to initialized ::BrotliDecoderState otherwise
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Deinitializes and frees ::BrotliDecoderState instance.
|
|
||||||
*
|
|
||||||
* @param state decoder instance to be cleaned up and deallocated
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Performs one-shot memory-to-memory decompression.
|
|
||||||
*
|
|
||||||
* Decompresses the data in @p encoded_buffer into @p decoded_buffer, and sets
|
|
||||||
* @p *decoded_size to the decompressed length.
|
|
||||||
*
|
|
||||||
* @param encoded_size size of @p encoded_buffer
|
|
||||||
* @param encoded_buffer compressed data buffer with at least @p encoded_size
|
|
||||||
* addressable bytes
|
|
||||||
* @param[in, out] decoded_size @b in: size of @p decoded_buffer; \n
|
|
||||||
* @b out: length of decompressed data written to
|
|
||||||
* @p decoded_buffer
|
|
||||||
* @param decoded_buffer decompressed data destination buffer
|
|
||||||
* @returns ::BROTLI_DECODER_RESULT_ERROR if input is corrupted, memory
|
|
||||||
* allocation failed, or @p decoded_buffer is not large enough;
|
|
||||||
* @returns ::BROTLI_DECODER_RESULT_SUCCESS otherwise
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Decompresses the input stream to the output stream.
|
|
||||||
*
|
|
||||||
* The values @p *available_in and @p *available_out must specify the number of
|
|
||||||
* bytes addressable at @p *next_in and @p *next_out respectively.
|
|
||||||
* When @p *available_out is @c 0, @p next_out is allowed to be @c NULL.
|
|
||||||
*
|
|
||||||
* After each call, @p *available_in will be decremented by the amount of input
|
|
||||||
* bytes consumed, and the @p *next_in pointer will be incremented by that
|
|
||||||
* amount. Similarly, @p *available_out will be decremented by the amount of
|
|
||||||
* output bytes written, and the @p *next_out pointer will be incremented by
|
|
||||||
* that amount.
|
|
||||||
*
|
|
||||||
* @p total_out, if it is not a null-pointer, will be set to the number
|
|
||||||
* of bytes decompressed since the last @p state initialization.
|
|
||||||
*
|
|
||||||
* @note Input is never overconsumed, so @p next_in and @p available_in could be
|
|
||||||
* passed to the next consumer after decoding is complete.
|
|
||||||
*
|
|
||||||
* @param state decoder instance
|
|
||||||
* @param[in, out] available_in @b in: amount of available input; \n
|
|
||||||
* @b out: amount of unused input
|
|
||||||
* @param[in, out] next_in pointer to the next compressed byte
|
|
||||||
* @param[in, out] available_out @b in: length of output buffer; \n
|
|
||||||
* @b out: remaining size of output buffer
|
|
||||||
* @param[in, out] next_out output buffer cursor;
|
|
||||||
* can be @c NULL if @p available_out is @c 0
|
|
||||||
* @param[out] total_out number of bytes decompressed so far; can be @c NULL
|
|
||||||
* @returns ::BROTLI_DECODER_RESULT_ERROR if input is corrupted, memory
|
|
||||||
* allocation failed, arguments were invalid, etc.;
|
|
||||||
* use ::BrotliDecoderGetErrorCode to get detailed error code
|
|
||||||
* @returns ::BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT decoding is blocked until
|
|
||||||
* more input data is provided
|
|
||||||
* @returns ::BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT decoding is blocked until
|
|
||||||
* more output space is provided
|
|
||||||
* @returns ::BROTLI_DECODER_RESULT_SUCCESS decoding is finished, no more
|
|
||||||
* input might be consumed and no more output will be produced
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Checks if decoder has more output.
|
|
||||||
*
|
|
||||||
* @param state decoder instance
|
|
||||||
* @returns ::true, if decoder has some unconsumed output
|
|
||||||
* @returns ::false otherwise
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Acquires pointer to internal output buffer.
|
|
||||||
*
|
|
||||||
* This method is used to make language bindings easier and more efficient:
|
|
||||||
* -# push data to ::BrotliDecoderDecompressStream,
|
|
||||||
* until ::BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT is reported
|
|
||||||
* -# use ::BrotliDecoderTakeOutput to peek bytes and copy to language-specific
|
|
||||||
* entity
|
|
||||||
*
|
|
||||||
* Also this could be useful if there is an output stream that is able to
|
|
||||||
* consume all the provided data (e.g. when data is saved to file system).
|
|
||||||
*
|
|
||||||
* @attention After every call to ::BrotliDecoderTakeOutput @p *size bytes of
|
|
||||||
* output are considered consumed for all consecutive calls to the
|
|
||||||
* instance methods; returned pointer becomes invalidated as well.
|
|
||||||
*
|
|
||||||
* @note Decoder output is not guaranteed to be contiguous. This means that
|
|
||||||
* after the size-unrestricted call to ::BrotliDecoderTakeOutput,
|
|
||||||
* immediate next call to ::BrotliDecoderTakeOutput may return more data.
|
|
||||||
*
|
|
||||||
* @param state decoder instance
|
|
||||||
* @param[in, out] size @b in: number of bytes caller is ready to take, @c 0 if
|
|
||||||
* any amount could be handled; \n
|
|
||||||
* @b out: amount of data pointed by returned pointer and
|
|
||||||
* considered consumed; \n
|
|
||||||
* out value is never greater than in value, unless it is @c 0
|
|
||||||
* @returns pointer to output data
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Checks if instance has already consumed input.
|
|
||||||
*
|
|
||||||
* Instance that returns ::false is considered "fresh" and could be
|
|
||||||
* reused.
|
|
||||||
*
|
|
||||||
* @param state decoder instance
|
|
||||||
* @returns ::true if decoder has already used some input bytes
|
|
||||||
* @returns ::false otherwise
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Checks if decoder instance reached the final state.
|
|
||||||
*
|
|
||||||
* @param state decoder instance
|
|
||||||
* @returns ::true if decoder is in a state where it reached the end of
|
|
||||||
* the input and produced all of the output
|
|
||||||
* @returns ::false otherwise
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Acquires a detailed error code.
|
|
||||||
*
|
|
||||||
* Should be used only after ::BrotliDecoderDecompressStream returns
|
|
||||||
* ::BROTLI_DECODER_RESULT_ERROR.
|
|
||||||
*
|
|
||||||
* See also ::BrotliDecoderErrorString
|
|
||||||
*
|
|
||||||
* @param state decoder instance
|
|
||||||
* @returns last saved error code
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Converts error code to a c-string.
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Gets a decoder library version.
|
|
||||||
*
|
|
||||||
* Look at BROTLI_VERSION for more information.
|
|
||||||
*/
|
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Lookup table to map the previous two bytes to a context id.
|
/* Lookup table to map the previous two bytes to a context id.
|
||||||
|
|
||||||
There are four different context modeling modes defined here:
|
There are four different context modeling modes defined here:
|
||||||
|
@ -256,6 +80,7 @@ context ids and the type of the next byte is summarized in the table below:
|
||||||
| (208-) | | context: 2 - 3 | |
|
| (208-) | | context: 2 - 3 | |
|
||||||
|-------------|-------------------|---------------------|------------------|
|
|-------------|-------------------|---------------------|------------------|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
const (
|
const (
|
||||||
contextLSB6 = 0
|
contextLSB6 = 0
|
||||||
contextMSB6 = 1
|
contextMSB6 = 1
|
||||||
|
|
33
decode.go
33
decode.go
|
@ -5,21 +5,7 @@ package brotli
|
||||||
Distributed under MIT license.
|
Distributed under MIT license.
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||||
*/
|
*/
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @file
|
|
||||||
* API for Brotli decompression.
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Result type for ::BrotliDecoderDecompress and
|
|
||||||
* ::BrotliDecoderDecompressStream functions.
|
|
||||||
*/
|
|
||||||
const (
|
const (
|
||||||
decoderResultError = 0
|
decoderResultError = 0
|
||||||
decoderResultSuccess = 1
|
decoderResultSuccess = 1
|
||||||
|
@ -27,25 +13,6 @@ const (
|
||||||
decoderResultNeedsMoreOutput = 3
|
decoderResultNeedsMoreOutput = 3
|
||||||
)
|
)
|
||||||
|
|
||||||
/**
|
|
||||||
* Template that evaluates items of ::BrotliDecoderErrorCode.
|
|
||||||
*
|
|
||||||
* Example: @code {.cpp}
|
|
||||||
* // Log Brotli error code.
|
|
||||||
* switch (brotliDecoderErrorCode) {
|
|
||||||
* #define CASE_(PREFIX, NAME, CODE) \
|
|
||||||
* case BROTLI_DECODER ## PREFIX ## NAME: \
|
|
||||||
* LOG(INFO) << "error code:" << #NAME; \
|
|
||||||
* break;
|
|
||||||
* #define NEWLINE_
|
|
||||||
* BROTLI_DECODER_ERROR_CODES_LIST(CASE_, NEWLINE_)
|
|
||||||
* #undef CASE_
|
|
||||||
* #undef NEWLINE_
|
|
||||||
* default: LOG(FATAL) << "unknown brotli error code";
|
|
||||||
* }
|
|
||||||
* @endcode
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Error code for detailed logging / production debugging.
|
* Error code for detailed logging / production debugging.
|
||||||
*
|
*
|
||||||
|
|
|
@ -1,10 +1,5 @@
|
||||||
package brotli
|
package brotli
|
||||||
|
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
/* Copyright 2013 Google Inc. All Rights Reserved.
|
||||||
|
|
||||||
Distributed under MIT license.
|
Distributed under MIT license.
|
||||||
|
@ -19,15 +14,6 @@ type dictionary struct {
|
||||||
data []byte
|
data []byte
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Sets dictionary data.
|
|
||||||
*
|
|
||||||
* When dictionary data is already set / present, this method is no-op.
|
|
||||||
*
|
|
||||||
* Dictionary data MUST be provided before BrotliGetDictionary is invoked.
|
|
||||||
* This method is used ONLY in multi-client environment (e.g. C + Java),
|
|
||||||
* to reduce storage by sharing single dictionary between implementations.
|
|
||||||
*/
|
|
||||||
const minDictionaryWordLength = 4
|
const minDictionaryWordLength = 4
|
||||||
|
|
||||||
const maxDictionaryWordLength = 24
|
const maxDictionaryWordLength = 24
|
||||||
|
|
|
@ -6,13 +6,6 @@ package brotli
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Hash table on the 4-byte prefixes of static dictionary words. */
|
|
||||||
/* Copyright 2015 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Hash table on the 4-byte prefixes of static dictionary words. */
|
/* Hash table on the 4-byte prefixes of static dictionary words. */
|
||||||
var kStaticDictionaryHash = [32768]uint16{
|
var kStaticDictionaryHash = [32768]uint16{
|
||||||
32072,
|
32072,
|
||||||
|
|
13
encode.go
13
encode.go
|
@ -11,19 +11,6 @@ import (
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Constants and formulas that affect speed-ratio trade-offs and thus define
|
|
||||||
quality levels. */
|
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @file
|
|
||||||
* API for Brotli compression.
|
|
||||||
*/
|
|
||||||
|
|
||||||
/** Minimal value for ::BROTLI_PARAM_LGWIN parameter. */
|
/** Minimal value for ::BROTLI_PARAM_LGWIN parameter. */
|
||||||
const minWindowBits = 10
|
const minWindowBits = 10
|
||||||
|
|
||||||
|
|
|
@ -8,13 +8,6 @@ import "math"
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Entropy encoding (Huffman) utilities. */
|
|
||||||
/* Copyright 2010 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Entropy encoding (Huffman) utilities. */
|
/* Entropy encoding (Huffman) utilities. */
|
||||||
|
|
||||||
/* A node of a Huffman tree. */
|
/* A node of a Huffman tree. */
|
||||||
|
@ -30,35 +23,6 @@ func initHuffmanTree(self *huffmanTree, count uint32, left int16, right int16) {
|
||||||
self.index_right_or_value_ = right
|
self.index_right_or_value_ = right
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Returns 1 is assignment of depths succeeded, otherwise 0. */
|
|
||||||
|
|
||||||
/* This function will create a Huffman tree.
|
|
||||||
|
|
||||||
The (data,length) contains the population counts.
|
|
||||||
The tree_limit is the maximum bit depth of the Huffman codes.
|
|
||||||
|
|
||||||
The depth contains the tree, i.e., how many bits are used for
|
|
||||||
the symbol.
|
|
||||||
|
|
||||||
The actual Huffman tree is constructed in the tree[] array, which has to
|
|
||||||
be at least 2 * length + 1 long.
|
|
||||||
|
|
||||||
See http://en.wikipedia.org/wiki/Huffman_coding */
|
|
||||||
|
|
||||||
/* Change the population counts in a way that the consequent
|
|
||||||
Huffman tree compression, especially its RLE-part will be more
|
|
||||||
likely to compress this data more efficiently.
|
|
||||||
|
|
||||||
length contains the size of the histogram.
|
|
||||||
counts contains the population counts.
|
|
||||||
good_for_rle is a buffer of at least length size */
|
|
||||||
|
|
||||||
/* Write a Huffman tree from bit depths into the bit-stream representation
|
|
||||||
of a Huffman tree. The generated Huffman tree is to be compressed once
|
|
||||||
more using a Huffman tree */
|
|
||||||
|
|
||||||
/* Get the actual bit values for a tree of bit depths. */
|
|
||||||
|
|
||||||
/* Input size optimized Shell sort. */
|
/* Input size optimized Shell sort. */
|
||||||
type huffmanTreeComparator func(*huffmanTree, *huffmanTree) bool
|
type huffmanTreeComparator func(*huffmanTree, *huffmanTree) bool
|
||||||
|
|
||||||
|
@ -109,6 +73,7 @@ func sortHuffmanTreeItems(items []huffmanTree, n uint, comparator huffmanTreeCom
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Returns 1 if assignment of depths succeeded, otherwise 0. */
|
||||||
func setDepth(p0 int, pool []huffmanTree, depth []byte, max_depth int) bool {
|
func setDepth(p0 int, pool []huffmanTree, depth []byte, max_depth int) bool {
|
||||||
var stack [16]int
|
var stack [16]int
|
||||||
var level int = 0
|
var level int = 0
|
||||||
|
@ -333,6 +298,13 @@ func writeHuffmanTreeRepetitionsZeros(repetitions uint, tree_size *uint, tree []
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Change the population counts in a way that the consequent
|
||||||
|
Huffman tree compression, especially its RLE-part will be more
|
||||||
|
likely to compress this data more efficiently.
|
||||||
|
|
||||||
|
length contains the size of the histogram.
|
||||||
|
counts contains the population counts.
|
||||||
|
good_for_rle is a buffer of at least length size */
|
||||||
func optimizeHuffmanCountsForRLE(length uint, counts []uint32, good_for_rle []byte) {
|
func optimizeHuffmanCountsForRLE(length uint, counts []uint32, good_for_rle []byte) {
|
||||||
var nonzero_count uint = 0
|
var nonzero_count uint = 0
|
||||||
var stride uint
|
var stride uint
|
||||||
|
@ -510,6 +482,9 @@ func decideOverRLEUse(depth []byte, length uint, use_rle_for_non_zero *bool, use
|
||||||
*use_rle_for_zero = total_reps_zero > count_reps_zero*2
|
*use_rle_for_zero = total_reps_zero > count_reps_zero*2
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Write a Huffman tree from bit depths into the bit-stream representation
|
||||||
|
of a Huffman tree. The generated Huffman tree is to be compressed once
|
||||||
|
more using a Huffman tree */
|
||||||
func writeHuffmanTree(depth []byte, length uint, tree_size *uint, tree []byte, extra_bits_data []byte) {
|
func writeHuffmanTree(depth []byte, length uint, tree_size *uint, tree []byte, extra_bits_data []byte) {
|
||||||
var previous_value byte = initialRepeatedCodeLength
|
var previous_value byte = initialRepeatedCodeLength
|
||||||
var i uint
|
var i uint
|
||||||
|
@ -589,6 +564,7 @@ func reverseBits(num_bits uint, bits uint16) uint16 {
|
||||||
/* 0..15 are values for bits */
|
/* 0..15 are values for bits */
|
||||||
const maxHuffmanBits = 16
|
const maxHuffmanBits = 16
|
||||||
|
|
||||||
|
/* Get the actual bit values for a tree of bit depths. */
|
||||||
func convertBitDepthsToSymbols(depth []byte, len uint, bits []uint16) {
|
func convertBitDepthsToSymbols(depth []byte, len uint, bits []uint16) {
|
||||||
var bl_count = [maxHuffmanBits]uint16{0}
|
var bl_count = [maxHuffmanBits]uint16{0}
|
||||||
var next_code [maxHuffmanBits]uint16
|
var next_code [maxHuffmanBits]uint16
|
||||||
|
|
21
fast_log.go
21
fast_log.go
|
@ -8,27 +8,8 @@ import "math"
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Literal cost model to allow backward reference replacement to be efficient.
|
|
||||||
*/
|
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Literal cost model to allow backward reference replacement to be efficient.
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Estimates how many bits the literals in the interval [pos, pos + len) in the
|
|
||||||
ring-buffer (data, mask) will take entropy coded and writes these estimates
|
|
||||||
to the cost[0..len) array. */
|
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Utilities for fast computation of logarithms. */
|
/* Utilities for fast computation of logarithms. */
|
||||||
|
|
||||||
func log2FloorNonZero(n uint) uint32 {
|
func log2FloorNonZero(n uint) uint32 {
|
||||||
/* TODO: generalize and move to platform.h */
|
/* TODO: generalize and move to platform.h */
|
||||||
var result uint32 = 0
|
var result uint32 = 0
|
||||||
|
|
|
@ -6,14 +6,6 @@ package brotli
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* A (forgetful) hash table to the data seen by the compressor, to
|
|
||||||
help create backward references to previous data. */
|
|
||||||
/* Copyright 2010 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Function to find maximal matching prefixes of strings. */
|
/* Function to find maximal matching prefixes of strings. */
|
||||||
func findMatchLengthWithLimit(s1 []byte, s2 []byte, limit uint) uint {
|
func findMatchLengthWithLimit(s1 []byte, s2 []byte, limit uint) uint {
|
||||||
var matched uint = 0
|
var matched uint = 0
|
||||||
|
|
13
h10.go
13
h10.go
|
@ -2,19 +2,12 @@ package brotli
|
||||||
|
|
||||||
import "encoding/binary"
|
import "encoding/binary"
|
||||||
|
|
||||||
/* NOLINT(build/header_guard) */
|
|
||||||
/* Copyright 2016 Google Inc. All Rights Reserved.
|
/* Copyright 2016 Google Inc. All Rights Reserved.
|
||||||
|
|
||||||
Distributed under MIT license.
|
Distributed under MIT license.
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* A (forgetful) hash table where each hash bucket contains a binary tree of
|
|
||||||
sequences whose first 4 bytes share the same hash code.
|
|
||||||
Each sequence is 128 long and is identified by its starting
|
|
||||||
position in the input data. The binary tree is sorted by the lexicographic
|
|
||||||
order of the sequences, and it is also a max-heap with respect to the
|
|
||||||
starting positions. */
|
|
||||||
func (*h10) HashTypeLength() uint {
|
func (*h10) HashTypeLength() uint {
|
||||||
return 4
|
return 4
|
||||||
}
|
}
|
||||||
|
@ -31,6 +24,12 @@ func hashBytesH10(data []byte) uint32 {
|
||||||
return h >> (32 - 17)
|
return h >> (32 - 17)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* A (forgetful) hash table where each hash bucket contains a binary tree of
|
||||||
|
sequences whose first 4 bytes share the same hash code.
|
||||||
|
Each sequence is 128 long and is identified by its starting
|
||||||
|
position in the input data. The binary tree is sorted by the lexicographic
|
||||||
|
order of the sequences, and it is also a max-heap with respect to the
|
||||||
|
starting positions. */
|
||||||
type h10 struct {
|
type h10 struct {
|
||||||
hasherCommon
|
hasherCommon
|
||||||
window_mask_ uint
|
window_mask_ uint
|
||||||
|
|
1
h5.go
1
h5.go
|
@ -2,7 +2,6 @@ package brotli
|
||||||
|
|
||||||
import "encoding/binary"
|
import "encoding/binary"
|
||||||
|
|
||||||
/* NOLINT(build/header_guard) */
|
|
||||||
/* Copyright 2010 Google Inc. All Rights Reserved.
|
/* Copyright 2010 Google Inc. All Rights Reserved.
|
||||||
|
|
||||||
Distributed under MIT license.
|
Distributed under MIT license.
|
||||||
|
|
1
h6.go
1
h6.go
|
@ -2,7 +2,6 @@ package brotli
|
||||||
|
|
||||||
import "encoding/binary"
|
import "encoding/binary"
|
||||||
|
|
||||||
/* NOLINT(build/header_guard) */
|
|
||||||
/* Copyright 2010 Google Inc. All Rights Reserved.
|
/* Copyright 2010 Google Inc. All Rights Reserved.
|
||||||
|
|
||||||
Distributed under MIT license.
|
Distributed under MIT license.
|
||||||
|
|
24
hash.go
24
hash.go
|
@ -5,24 +5,6 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
)
|
)
|
||||||
|
|
||||||
/* Matches data against static dictionary words, and for each length l,
|
|
||||||
for which a match is found, updates matches[l] to be the minimum possible
|
|
||||||
(distance << 5) + len_code.
|
|
||||||
Returns 1 if matches have been found, otherwise 0.
|
|
||||||
Prerequisites:
|
|
||||||
matches array is at least BROTLI_MAX_STATIC_DICTIONARY_MATCH_LEN + 1 long
|
|
||||||
all elements are initialized to kInvalidMatch */
|
|
||||||
/* Pointer to hasher data.
|
|
||||||
*
|
|
||||||
* Excluding initialization and destruction, hasher can be passed as
|
|
||||||
* HasherHandle by value.
|
|
||||||
*
|
|
||||||
* Typically hasher data consists of 3 sections:
|
|
||||||
* * hasherCommon structure
|
|
||||||
* * private structured hasher data, depending on hasher type
|
|
||||||
* * private dynamic hasher data, depending on hasher type and parameters
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
type hasherCommon struct {
|
type hasherCommon struct {
|
||||||
params hasherParams
|
params hasherParams
|
||||||
is_prepared_ bool
|
is_prepared_ bool
|
||||||
|
@ -68,11 +50,11 @@ type hasherSearchResult struct {
|
||||||
* There is no effort to ensure that it is a prime, the oddity is enough
|
* There is no effort to ensure that it is a prime, the oddity is enough
|
||||||
for this use.
|
for this use.
|
||||||
* The number has been tuned heuristically against compression benchmarks. */
|
* The number has been tuned heuristically against compression benchmarks. */
|
||||||
var kHashMul32 uint32 = 0x1E35A7BD
|
const kHashMul32 uint32 = 0x1E35A7BD
|
||||||
|
|
||||||
var kHashMul64 uint64 = 0x1E35A7BD1E35A7BD
|
const kHashMul64 uint64 = 0x1E35A7BD1E35A7BD
|
||||||
|
|
||||||
var kHashMul64Long uint64 = 0x1FE35A7BD3579BD3
|
const kHashMul64Long uint64 = 0x1FE35A7BD3579BD3
|
||||||
|
|
||||||
func hash14(data []byte) uint32 {
|
func hash14(data []byte) uint32 {
|
||||||
var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32
|
var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32
|
||||||
|
|
|
@ -1,16 +1,11 @@
|
||||||
package brotli
|
package brotli
|
||||||
|
|
||||||
/* NOTE: this hasher does not search in the dictionary. It is used as
|
|
||||||
backup-hasher, the main hasher already searches in it. */
|
|
||||||
/* NOLINT(build/header_guard) */
|
|
||||||
/* Copyright 2018 Google Inc. All Rights Reserved.
|
/* Copyright 2018 Google Inc. All Rights Reserved.
|
||||||
|
|
||||||
Distributed under MIT license.
|
Distributed under MIT license.
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Composite hasher: This hasher allows to combine two other hashers, HASHER_A
|
|
||||||
and HASHER_B. */
|
|
||||||
func (h *hashComposite) HashTypeLength() uint {
|
func (h *hashComposite) HashTypeLength() uint {
|
||||||
var a uint = h.ha.HashTypeLength()
|
var a uint = h.ha.HashTypeLength()
|
||||||
var b uint = h.hb.HashTypeLength()
|
var b uint = h.hb.HashTypeLength()
|
||||||
|
@ -31,6 +26,8 @@ func (h *hashComposite) StoreLookahead() uint {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Composite hasher: This hasher allows to combine two other hashers, HASHER_A
|
||||||
|
and HASHER_B. */
|
||||||
type hashComposite struct {
|
type hashComposite struct {
|
||||||
hasherCommon
|
hasherCommon
|
||||||
ha hasherHandle
|
ha hasherHandle
|
||||||
|
|
|
@ -2,19 +2,12 @@ package brotli
|
||||||
|
|
||||||
import "encoding/binary"
|
import "encoding/binary"
|
||||||
|
|
||||||
/* NOLINT(build/header_guard) */
|
|
||||||
/* Copyright 2016 Google Inc. All Rights Reserved.
|
/* Copyright 2016 Google Inc. All Rights Reserved.
|
||||||
|
|
||||||
Distributed under MIT license.
|
Distributed under MIT license.
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* A (forgetful) hash table to the data seen by the compressor, to
|
|
||||||
help create backward references to previous data.
|
|
||||||
|
|
||||||
Hashes are stored in chains which are bucketed to groups. Group of chains
|
|
||||||
share a storage "bank". When more than "bank size" chain nodes are added,
|
|
||||||
oldest nodes are replaced; this way several chains may share a tail. */
|
|
||||||
func (*hashForgetfulChain) HashTypeLength() uint {
|
func (*hashForgetfulChain) HashTypeLength() uint {
|
||||||
return 4
|
return 4
|
||||||
}
|
}
|
||||||
|
@ -37,6 +30,12 @@ type slot struct {
|
||||||
next uint16
|
next uint16
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* A (forgetful) hash table to the data seen by the compressor, to
|
||||||
|
help create backward references to previous data.
|
||||||
|
|
||||||
|
Hashes are stored in chains which are bucketed to groups. Group of chains
|
||||||
|
share a storage "bank". When more than "bank size" chain nodes are added,
|
||||||
|
oldest nodes are replaced; this way several chains may share a tail. */
|
||||||
type hashForgetfulChain struct {
|
type hashForgetfulChain struct {
|
||||||
hasherCommon
|
hasherCommon
|
||||||
|
|
||||||
|
|
|
@ -2,7 +2,6 @@ package brotli
|
||||||
|
|
||||||
import "encoding/binary"
|
import "encoding/binary"
|
||||||
|
|
||||||
/* NOLINT(build/header_guard) */
|
|
||||||
/* Copyright 2010 Google Inc. All Rights Reserved.
|
/* Copyright 2010 Google Inc. All Rights Reserved.
|
||||||
|
|
||||||
Distributed under MIT license.
|
Distributed under MIT license.
|
||||||
|
@ -12,6 +11,7 @@ import "encoding/binary"
|
||||||
/* For BUCKET_SWEEP == 1, enabling the dictionary lookup makes compression
|
/* For BUCKET_SWEEP == 1, enabling the dictionary lookup makes compression
|
||||||
a little faster (0.5% - 1%) and it compresses 0.15% better on small text
|
a little faster (0.5% - 1%) and it compresses 0.15% better on small text
|
||||||
and HTML inputs. */
|
and HTML inputs. */
|
||||||
|
|
||||||
func (*hashLongestMatchQuickly) HashTypeLength() uint {
|
func (*hashLongestMatchQuickly) HashTypeLength() uint {
|
||||||
return 8
|
return 8
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,16 +1,14 @@
|
||||||
package brotli
|
package brotli
|
||||||
|
|
||||||
/* NOTE: this hasher does not search in the dictionary. It is used as
|
|
||||||
backup-hasher, the main hasher already searches in it. */
|
|
||||||
/* NOLINT(build/header_guard) */
|
|
||||||
/* Copyright 2018 Google Inc. All Rights Reserved.
|
/* Copyright 2018 Google Inc. All Rights Reserved.
|
||||||
|
|
||||||
Distributed under MIT license.
|
Distributed under MIT license.
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Rolling hash for long distance long string matches. Stores one position
|
/* NOTE: this hasher does not search in the dictionary. It is used as
|
||||||
per bucket, bucket key is computed over a long region. */
|
backup-hasher, the main hasher already searches in it. */
|
||||||
|
|
||||||
var kRollingHashMul32hashRolling uint32 = 69069
|
var kRollingHashMul32hashRolling uint32 = 69069
|
||||||
|
|
||||||
var kInvalidPosHashRolling uint32 = 0xffffffff
|
var kInvalidPosHashRolling uint32 = 0xffffffff
|
||||||
|
@ -40,6 +38,8 @@ func (h *hashRolling) HashRollingFunction(state uint32, add byte, rem byte, fact
|
||||||
return uint32(factor*state + h.HashByte(add) - factor_remove*h.HashByte(rem))
|
return uint32(factor*state + h.HashByte(add) - factor_remove*h.HashByte(rem))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Rolling hash for long distance long string matches. Stores one position
|
||||||
|
per bucket, bucket key is computed over a long region. */
|
||||||
type hashRolling struct {
|
type hashRolling struct {
|
||||||
hasherCommon
|
hasherCommon
|
||||||
|
|
||||||
|
|
|
@ -7,6 +7,7 @@ package brotli
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Utilities for building Huffman decoding tables. */
|
/* Utilities for building Huffman decoding tables. */
|
||||||
|
|
||||||
const huffmanMaxCodeLength = 15
|
const huffmanMaxCodeLength = 15
|
||||||
|
|
||||||
/* Maximum possible Huffman table size for an alphabet size of (index * 32),
|
/* Maximum possible Huffman table size for an alphabet size of (index * 32),
|
||||||
|
|
|
@ -6,8 +6,6 @@ package brotli
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Macros for memory management. */
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
Dynamically grows array capacity to at least the requested size
|
Dynamically grows array capacity to at least the requested size
|
||||||
T: data type
|
T: data type
|
||||||
|
|
32
metablock.go
32
metablock.go
|
@ -6,31 +6,9 @@ package brotli
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Brotli bit stream functions to support the low level format. There are no
|
|
||||||
compression algorithms here, just the right ordering of bits to match the
|
|
||||||
specs. */
|
|
||||||
/* Copyright 2014 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Functions to convert brotli-related data structures into the
|
|
||||||
brotli bit stream. The functions here operate under
|
|
||||||
assumption that there is enough space in the storage, i.e., there are
|
|
||||||
no out-of-range checks anywhere.
|
|
||||||
|
|
||||||
These functions do bit addressing into a byte array. The byte array
|
|
||||||
is called "storage" and the index to the bit is called storage_ix
|
|
||||||
in function arguments. */
|
|
||||||
/* Copyright 2015 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Algorithms for distributing the literals and commands of a metablock between
|
/* Algorithms for distributing the literals and commands of a metablock between
|
||||||
block types and contexts. */
|
block types and contexts. */
|
||||||
|
|
||||||
type metaBlockSplit struct {
|
type metaBlockSplit struct {
|
||||||
literal_split blockSplit
|
literal_split blockSplit
|
||||||
command_split blockSplit
|
command_split blockSplit
|
||||||
|
@ -74,14 +52,6 @@ func destroyMetaBlockSplit(mb *metaBlockSplit) {
|
||||||
mb.distance_histograms = nil
|
mb.distance_histograms = nil
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Copyright 2015 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Algorithms for distributing the literals and commands of a metablock between
|
|
||||||
block types and contexts. */
|
|
||||||
func initDistanceParams(params *encoderParams, npostfix uint32, ndirect uint32) {
|
func initDistanceParams(params *encoderParams, npostfix uint32, ndirect uint32) {
|
||||||
var dist_params *distanceParams = ¶ms.dist
|
var dist_params *distanceParams = ¶ms.dist
|
||||||
var alphabet_size uint32
|
var alphabet_size uint32
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
package brotli
|
package brotli
|
||||||
|
|
||||||
/* NOLINT(build/header_guard) */
|
|
||||||
/* Copyright 2015 Google Inc. All Rights Reserved.
|
/* Copyright 2015 Google Inc. All Rights Reserved.
|
||||||
|
|
||||||
Distributed under MIT license.
|
Distributed under MIT license.
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
package brotli
|
package brotli
|
||||||
|
|
||||||
/* NOLINT(build/header_guard) */
|
|
||||||
/* Copyright 2015 Google Inc. All Rights Reserved.
|
/* Copyright 2015 Google Inc. All Rights Reserved.
|
||||||
|
|
||||||
Distributed under MIT license.
|
Distributed under MIT license.
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
package brotli
|
package brotli
|
||||||
|
|
||||||
/* NOLINT(build/header_guard) */
|
|
||||||
/* Copyright 2015 Google Inc. All Rights Reserved.
|
/* Copyright 2015 Google Inc. All Rights Reserved.
|
||||||
|
|
||||||
Distributed under MIT license.
|
Distributed under MIT license.
|
||||||
|
|
192
params.go
192
params.go
|
@ -1,197 +1,5 @@
|
||||||
package brotli
|
package brotli
|
||||||
|
|
||||||
/**
|
|
||||||
* Opaque structure that holds encoder state.
|
|
||||||
*
|
|
||||||
* Allocated and initialized with ::BrotliEncoderCreateInstance.
|
|
||||||
* Cleaned up and deallocated with ::BrotliEncoderDestroyInstance.
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Sets the specified parameter to the given encoder instance.
|
|
||||||
*
|
|
||||||
* @param state encoder instance
|
|
||||||
* @param param parameter to set
|
|
||||||
* @param value new parameter value
|
|
||||||
* @returns ::false if parameter is unrecognized, or value is invalid
|
|
||||||
* @returns ::false if value of parameter can not be changed at current
|
|
||||||
* encoder state (e.g. when encoding is started, window size might be
|
|
||||||
* already encoded and therefore it is impossible to change it)
|
|
||||||
* @returns ::true if value is accepted
|
|
||||||
* @warning invalid values might be accepted in case they would not break
|
|
||||||
* encoding process.
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates an instance of ::BrotliEncoderState and initializes it.
|
|
||||||
*
|
|
||||||
* @p alloc_func and @p free_func @b MUST be both zero or both non-zero. In the
|
|
||||||
* case they are both zero, default memory allocators are used. @p opaque is
|
|
||||||
* passed to @p alloc_func and @p free_func when they are called. @p free_func
|
|
||||||
* has to return without doing anything when asked to free a NULL pointer.
|
|
||||||
*
|
|
||||||
* @param alloc_func custom memory allocation function
|
|
||||||
* @param free_func custom memory free function
|
|
||||||
* @param opaque custom memory manager handle
|
|
||||||
* @returns @c 0 if instance can not be allocated or initialized
|
|
||||||
* @returns pointer to initialized ::BrotliEncoderState otherwise
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Deinitializes and frees ::BrotliEncoderState instance.
|
|
||||||
*
|
|
||||||
* @param state decoder instance to be cleaned up and deallocated
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Calculates the output size bound for the given @p input_size.
|
|
||||||
*
|
|
||||||
* @warning Result is only valid if quality is at least @c 2 and, in
|
|
||||||
* case ::BrotliEncoderCompressStream was used, no flushes
|
|
||||||
* (::BROTLI_OPERATION_FLUSH) were performed.
|
|
||||||
*
|
|
||||||
* @param input_size size of projected input
|
|
||||||
* @returns @c 0 if result does not fit @c size_t
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Performs one-shot memory-to-memory compression.
|
|
||||||
*
|
|
||||||
* Compresses the data in @p input_buffer into @p encoded_buffer, and sets
|
|
||||||
* @p *encoded_size to the compressed length.
|
|
||||||
*
|
|
||||||
* @note If ::BrotliEncoderMaxCompressedSize(@p input_size) returns non-zero
|
|
||||||
* value, then output is guaranteed to be no longer than that.
|
|
||||||
*
|
|
||||||
* @param quality quality parameter value, e.g. ::BROTLI_DEFAULT_QUALITY
|
|
||||||
* @param lgwin lgwin parameter value, e.g. ::BROTLI_DEFAULT_WINDOW
|
|
||||||
* @param mode mode parameter value, e.g. ::BROTLI_DEFAULT_MODE
|
|
||||||
* @param input_size size of @p input_buffer
|
|
||||||
* @param input_buffer input data buffer with at least @p input_size
|
|
||||||
* addressable bytes
|
|
||||||
* @param[in, out] encoded_size @b in: size of @p encoded_buffer; \n
|
|
||||||
* @b out: length of compressed data written to
|
|
||||||
* @p encoded_buffer, or @c 0 if compression fails
|
|
||||||
* @param encoded_buffer compressed data destination buffer
|
|
||||||
* @returns ::false in case of compression error
|
|
||||||
* @returns ::false if output buffer is too small
|
|
||||||
* @returns ::true otherwise
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Compresses input stream to output stream.
|
|
||||||
*
|
|
||||||
* The values @p *available_in and @p *available_out must specify the number of
|
|
||||||
* bytes addressable at @p *next_in and @p *next_out respectively.
|
|
||||||
* When @p *available_out is @c 0, @p next_out is allowed to be @c NULL.
|
|
||||||
*
|
|
||||||
* After each call, @p *available_in will be decremented by the amount of input
|
|
||||||
* bytes consumed, and the @p *next_in pointer will be incremented by that
|
|
||||||
* amount. Similarly, @p *available_out will be decremented by the amount of
|
|
||||||
* output bytes written, and the @p *next_out pointer will be incremented by
|
|
||||||
* that amount.
|
|
||||||
*
|
|
||||||
* @p total_out, if it is not a null-pointer, will be set to the number
|
|
||||||
* of bytes compressed since the last @p state initialization.
|
|
||||||
*
|
|
||||||
*
|
|
||||||
*
|
|
||||||
* Internally workflow consists of 3 tasks:
|
|
||||||
* -# (optionally) copy input data to internal buffer
|
|
||||||
* -# actually compress data and (optionally) store it to internal buffer
|
|
||||||
* -# (optionally) copy compressed bytes from internal buffer to output stream
|
|
||||||
*
|
|
||||||
* Whenever all 3 tasks can't move forward anymore, or error occurs, this
|
|
||||||
* method returns the control flow to caller.
|
|
||||||
*
|
|
||||||
* @p op is used to perform flush, finish the stream, or inject metadata block.
|
|
||||||
* See ::BrotliEncoderOperation for more information.
|
|
||||||
*
|
|
||||||
* Flushing the stream means forcing encoding of all input passed to encoder and
|
|
||||||
* completing the current output block, so it could be fully decoded by stream
|
|
||||||
* decoder. To perform flush set @p op to ::BROTLI_OPERATION_FLUSH.
|
|
||||||
* Under some circumstances (e.g. lack of output stream capacity) this operation
|
|
||||||
* would require several calls to ::BrotliEncoderCompressStream. The method must
|
|
||||||
* be called again until both input stream is depleted and encoder has no more
|
|
||||||
* output (see ::BrotliEncoderHasMoreOutput) after the method is called.
|
|
||||||
*
|
|
||||||
* Finishing the stream means encoding of all input passed to encoder and
|
|
||||||
* adding specific "final" marks, so stream decoder could determine that stream
|
|
||||||
* is complete. To perform finish set @p op to ::BROTLI_OPERATION_FINISH.
|
|
||||||
* Under some circumstances (e.g. lack of output stream capacity) this operation
|
|
||||||
* would require several calls to ::BrotliEncoderCompressStream. The method must
|
|
||||||
* be called again until both input stream is depleted and encoder has no more
|
|
||||||
* output (see ::BrotliEncoderHasMoreOutput) after the method is called.
|
|
||||||
*
|
|
||||||
* @warning When flushing and finishing, @p op should not change until operation
|
|
||||||
* is complete; input stream should not be swapped, reduced or
|
|
||||||
* extended as well.
|
|
||||||
*
|
|
||||||
* @param state encoder instance
|
|
||||||
* @param op requested operation
|
|
||||||
* @param[in, out] available_in @b in: amount of available input; \n
|
|
||||||
* @b out: amount of unused input
|
|
||||||
* @param[in, out] next_in pointer to the next input byte
|
|
||||||
* @param[in, out] available_out @b in: length of output buffer; \n
|
|
||||||
* @b out: remaining size of output buffer
|
|
||||||
* @param[in, out] next_out compressed output buffer cursor;
|
|
||||||
* can be @c NULL if @p available_out is @c 0
|
|
||||||
* @param[out] total_out number of bytes produced so far; can be @c NULL
|
|
||||||
* @returns ::false if there was an error
|
|
||||||
* @returns ::true otherwise
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Checks if encoder instance reached the final state.
|
|
||||||
*
|
|
||||||
* @param state encoder instance
|
|
||||||
* @returns ::true if encoder is in a state where it reached the end of
|
|
||||||
* the input and produced all of the output
|
|
||||||
* @returns ::false otherwise
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Checks if encoder has more output.
|
|
||||||
*
|
|
||||||
* @param state encoder instance
|
|
||||||
* @returns ::true, if encoder has some unconsumed output
|
|
||||||
* @returns ::false otherwise
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Acquires pointer to internal output buffer.
|
|
||||||
*
|
|
||||||
* This method is used to make language bindings easier and more efficient:
|
|
||||||
* -# push data to ::BrotliEncoderCompressStream,
|
|
||||||
* until ::BrotliEncoderHasMoreOutput returns BROTL_TRUE
|
|
||||||
* -# use ::BrotliEncoderTakeOutput to peek bytes and copy to language-specific
|
|
||||||
* entity
|
|
||||||
*
|
|
||||||
* Also this could be useful if there is an output stream that is able to
|
|
||||||
* consume all the provided data (e.g. when data is saved to file system).
|
|
||||||
*
|
|
||||||
* @attention After every call to ::BrotliEncoderTakeOutput @p *size bytes of
|
|
||||||
* output are considered consumed for all consecutive calls to the
|
|
||||||
* instance methods; returned pointer becomes invalidated as well.
|
|
||||||
*
|
|
||||||
* @note Encoder output is not guaranteed to be contiguous. This means that
|
|
||||||
* after the size-unrestricted call to ::BrotliEncoderTakeOutput,
|
|
||||||
* immediate next call to ::BrotliEncoderTakeOutput may return more data.
|
|
||||||
*
|
|
||||||
* @param state encoder instance
|
|
||||||
* @param[in, out] size @b in: number of bytes caller is ready to take, @c 0 if
|
|
||||||
* any amount could be handled; \n
|
|
||||||
* @b out: amount of data pointed by returned pointer and
|
|
||||||
* considered consumed; \n
|
|
||||||
* out value is never greater than in value, unless it is @c 0
|
|
||||||
* @returns pointer to output data
|
|
||||||
*/
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Gets an encoder library version.
|
|
||||||
*
|
|
||||||
* Look at BROTLI_VERSION for more information.
|
|
||||||
*/
|
|
||||||
/* Copyright 2017 Google Inc. All Rights Reserved.
|
/* Copyright 2017 Google Inc. All Rights Reserved.
|
||||||
|
|
||||||
Distributed under MIT license.
|
Distributed under MIT license.
|
||||||
|
|
24
platform.go
24
platform.go
|
@ -6,30 +6,6 @@ package brotli
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Bit reading helpers */
|
|
||||||
/* Copyright 2016 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Macros for compiler / platform specific features and build options.
|
|
||||||
|
|
||||||
Build options are:
|
|
||||||
* BROTLI_BUILD_32_BIT disables 64-bit optimizations
|
|
||||||
* BROTLI_BUILD_64_BIT forces to use 64-bit optimizations
|
|
||||||
* BROTLI_BUILD_BIG_ENDIAN forces to use big-endian optimizations
|
|
||||||
* BROTLI_BUILD_ENDIAN_NEUTRAL disables endian-aware optimizations
|
|
||||||
* BROTLI_BUILD_LITTLE_ENDIAN forces to use little-endian optimizations
|
|
||||||
* BROTLI_BUILD_PORTABLE disables dangerous optimizations, like unaligned
|
|
||||||
read and overlapping memcpy; this reduces decompression speed by 5%
|
|
||||||
* BROTLI_BUILD_NO_RBIT disables "rbit" optimization for ARM CPUs
|
|
||||||
* BROTLI_DEBUG dumps file name and line number when decoder detects stream
|
|
||||||
or memory error
|
|
||||||
* BROTLI_ENABLE_LOG enables asserts and dumps various state information
|
|
||||||
*/
|
|
||||||
type brotli_reg_t uint64
|
|
||||||
|
|
||||||
func brotli_min_double(a float64, b float64) float64 {
|
func brotli_min_double(a float64, b float64) float64 {
|
||||||
if a < b {
|
if a < b {
|
||||||
return a
|
return a
|
||||||
|
|
21
prefix.go
21
prefix.go
|
@ -6,27 +6,6 @@ package brotli
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Function to find backward reference copies. */
|
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Function to find backward reference copies. */
|
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* This class models a sequence of literals and a backward reference copy. */
|
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Functions for encoding of integers into prefix codes the amount of extra
|
/* Functions for encoding of integers into prefix codes the amount of extra
|
||||||
bits, and the actual values of the extra bits. */
|
bits, and the actual values of the extra bits. */
|
||||||
|
|
||||||
|
|
|
@ -6,15 +6,6 @@ package brotli
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Implementation of Brotli compressor. */
|
|
||||||
/* Copyright 2013 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Sliding window over the input data. */
|
|
||||||
|
|
||||||
/* A ringBuffer(window_bits, tail_bits) contains `1 << window_bits' bytes of
|
/* A ringBuffer(window_bits, tail_bits) contains `1 << window_bits' bytes of
|
||||||
data in a circular manner: writing a byte writes it to:
|
data in a circular manner: writing a byte writes it to:
|
||||||
`position() % (1 << window_bits)'.
|
`position() % (1 << window_bits)'.
|
||||||
|
|
|
@ -9,6 +9,7 @@ import "encoding/binary"
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Class to model the static dictionary. */
|
/* Class to model the static dictionary. */
|
||||||
|
|
||||||
const maxStaticDictionaryMatchLen = 37
|
const maxStaticDictionaryMatchLen = 37
|
||||||
|
|
||||||
var kInvalidMatch uint32 = 0xFFFFFFF
|
var kInvalidMatch uint32 = 0xFFFFFFF
|
||||||
|
|
|
@ -5,18 +5,9 @@ package brotli
|
||||||
Distributed under MIT license.
|
Distributed under MIT license.
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||||
*/
|
*/
|
||||||
/* Copyright 2017 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
/* Copyright 2015 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Lookup table for static dictionary and transforms. */
|
/* Lookup table for static dictionary and transforms. */
|
||||||
|
|
||||||
type dictWord struct {
|
type dictWord struct {
|
||||||
len byte
|
len byte
|
||||||
transform byte
|
transform byte
|
||||||
|
|
|
@ -7,6 +7,7 @@ package brotli
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Utilities for building Huffman decoding tables. */
|
/* Utilities for building Huffman decoding tables. */
|
||||||
|
|
||||||
type symbolList struct {
|
type symbolList struct {
|
||||||
storage []uint16
|
storage []uint16
|
||||||
offset int
|
offset int
|
||||||
|
|
21
transform.go
21
transform.go
|
@ -1,11 +1,5 @@
|
||||||
package brotli
|
package brotli
|
||||||
|
|
||||||
/* transforms is a part of ABI, but not API.
|
|
||||||
|
|
||||||
It means that there are some functions that are supposed to be in "common"
|
|
||||||
library, but header itself is not placed into include/brotli. This way,
|
|
||||||
aforementioned functions will be available only to brotli internals.
|
|
||||||
*/
|
|
||||||
const (
|
const (
|
||||||
transformIdentity = 0
|
transformIdentity = 0
|
||||||
transformOmitLast1 = 1
|
transformOmitLast1 = 1
|
||||||
|
@ -45,7 +39,6 @@ type transforms struct {
|
||||||
cutOffTransforms [transformsMaxCutOff + 1]int16
|
cutOffTransforms [transformsMaxCutOff + 1]int16
|
||||||
}
|
}
|
||||||
|
|
||||||
/* T is BrotliTransforms*; result is uint8_t. */
|
|
||||||
func transformPrefixId(t *transforms, I int) byte {
|
func transformPrefixId(t *transforms, I int) byte {
|
||||||
return t.transforms[(I*3)+0]
|
return t.transforms[(I*3)+0]
|
||||||
}
|
}
|
||||||
|
@ -58,7 +51,6 @@ func transformSuffixId(t *transforms, I int) byte {
|
||||||
return t.transforms[(I*3)+2]
|
return t.transforms[(I*3)+2]
|
||||||
}
|
}
|
||||||
|
|
||||||
/* T is BrotliTransforms*; result is const uint8_t*. */
|
|
||||||
func transformPrefix(t *transforms, I int) []byte {
|
func transformPrefix(t *transforms, I int) []byte {
|
||||||
return t.prefix_suffix[t.prefix_suffix_map[transformPrefixId(t, I)]:]
|
return t.prefix_suffix[t.prefix_suffix_map[transformPrefixId(t, I)]:]
|
||||||
}
|
}
|
||||||
|
@ -70,19 +62,6 @@ func transformSuffix(t *transforms, I int) []byte {
|
||||||
/* RFC 7932 transforms string data */
|
/* RFC 7932 transforms string data */
|
||||||
var kPrefixSuffix string = "\001 \002, \010 of the \004 of \002s \001.\005 and \004 " + "in \001\"\004 to \002\">\001\n\002. \001]\005 for \003 a \006 " + "that \001'\006 with \006 from \004 by \001(\006. T" + "he \004 on \004 as \004 is \004ing \002\n\t\001:\003ed " + "\002=\"\004 at \003ly \001,\002='\005.com/\007. This \005" + " not \003er \003al \004ful \004ive \005less \004es" + "t \004ize \002\xc2\xa0\004ous \005 the \002e \000"
|
var kPrefixSuffix string = "\001 \002, \010 of the \004 of \002s \001.\005 and \004 " + "in \001\"\004 to \002\">\001\n\002. \001]\005 for \003 a \006 " + "that \001'\006 with \006 from \004 by \001(\006. T" + "he \004 on \004 as \004 is \004ing \002\n\t\001:\003ed " + "\002=\"\004 at \003ly \001,\002='\005.com/\007. This \005" + " not \003er \003al \004ful \004ive \005less \004es" + "t \004ize \002\xc2\xa0\004ous \005 the \002e \000"
|
||||||
|
|
||||||
/* 0x _0 _2 __5 _E _3 _6 _8 _E */
|
|
||||||
|
|
||||||
/* 2x _3_ _5 _A_ _D_ _F _2 _4 _A _E */
|
|
||||||
|
|
||||||
/* 4x _5_ _7 _E _5 _A _C */
|
|
||||||
|
|
||||||
/* 6x _3 _8 _D _2 _7_ _ _A _C */
|
|
||||||
|
|
||||||
/* 8x _0 _ _3 _8 _C _E _ _1 _7 _F */
|
|
||||||
|
|
||||||
/* Ax _5 _9 _D _2 _7 _D */
|
|
||||||
|
|
||||||
/* Cx _2 _7___ ___ _A _F _5 _8 */
|
|
||||||
var kPrefixSuffixMap = [50]uint16{
|
var kPrefixSuffixMap = [50]uint16{
|
||||||
0x00,
|
0x00,
|
||||||
0x02,
|
0x02,
|
||||||
|
|
|
@ -1,40 +1,5 @@
|
||||||
package brotli
|
package brotli
|
||||||
|
|
||||||
/* Uses the slow shortest-path block splitter and does context clustering.
|
|
||||||
The distance parameters are dynamically selected based on the commands
|
|
||||||
which get recomputed under the new distance parameters. The new distance
|
|
||||||
parameters are stored into *params. */
|
|
||||||
|
|
||||||
/* Uses a fast greedy block splitter that tries to merge current block with the
|
|
||||||
last or the second last block and uses a static context clustering which
|
|
||||||
is the same for all block types. */
|
|
||||||
/* All Store functions here will use a storage_ix, which is always the bit
|
|
||||||
position for the current storage. */
|
|
||||||
|
|
||||||
/* REQUIRES: length > 0 */
|
|
||||||
/* REQUIRES: length <= (1 << 24) */
|
|
||||||
|
|
||||||
/* Stores the meta-block without doing any block splitting, just collects
|
|
||||||
one histogram per block category and uses that for entropy coding.
|
|
||||||
REQUIRES: length > 0
|
|
||||||
REQUIRES: length <= (1 << 24) */
|
|
||||||
|
|
||||||
/* Same as above, but uses static prefix codes for histograms with a only a few
|
|
||||||
symbols, and uses static code length prefix codes for all other histograms.
|
|
||||||
REQUIRES: length > 0
|
|
||||||
REQUIRES: length <= (1 << 24) */
|
|
||||||
|
|
||||||
/* This is for storing uncompressed blocks (simple raw storage of
|
|
||||||
bytes-as-bytes).
|
|
||||||
REQUIRES: length > 0
|
|
||||||
REQUIRES: length <= (1 << 24) */
|
|
||||||
/* Copyright 2015 Google Inc. All Rights Reserved.
|
|
||||||
|
|
||||||
Distributed under MIT license.
|
|
||||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Static entropy codes used for faster meta-block encoding. */
|
|
||||||
/* Copyright 2010 Google Inc. All Rights Reserved.
|
/* Copyright 2010 Google Inc. All Rights Reserved.
|
||||||
|
|
||||||
Distributed under MIT license.
|
Distributed under MIT license.
|
||||||
|
@ -43,8 +8,6 @@ package brotli
|
||||||
|
|
||||||
/* Write bits into a byte array. */
|
/* Write bits into a byte array. */
|
||||||
|
|
||||||
/*#define BIT_WRITER_DEBUG */
|
|
||||||
|
|
||||||
/* This function writes bits into bytes in increasing addresses, and within
|
/* This function writes bits into bytes in increasing addresses, and within
|
||||||
a byte least-significant-bit first.
|
a byte least-significant-bit first.
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue