forked from mirror/brotli
2241 lines
59 KiB
Go
2241 lines
59 KiB
Go
package brotli
|
|
|
|
import "io"
|
|
|
|
/* Copyright 2016 Google Inc. All Rights Reserved.
|
|
|
|
Distributed under MIT license.
|
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
*/
|
|
|
|
/* Constants and formulas that affect speed-ratio trade-offs and thus define
|
|
quality levels. */
|
|
/* Copyright 2013 Google Inc. All Rights Reserved.
|
|
|
|
Distributed under MIT license.
|
|
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
|
*/
|
|
|
|
/**
|
|
* @file
|
|
* API for Brotli compression.
|
|
*/
|
|
|
|
/** Minimal value for ::BROTLI_PARAM_LGWIN parameter. */
|
|
const BROTLI_MIN_WINDOW_BITS = 10
|
|
|
|
/**
|
|
* Maximal value for ::BROTLI_PARAM_LGWIN parameter.
|
|
*
|
|
* @note equal to @c BROTLI_MAX_DISTANCE_BITS constant.
|
|
*/
|
|
const BROTLI_MAX_WINDOW_BITS = 24
|
|
|
|
/**
|
|
* Maximal value for ::BROTLI_PARAM_LGWIN parameter
|
|
* in "Large Window Brotli" (32-bit).
|
|
*/
|
|
const BROTLI_LARGE_MAX_WINDOW_BITS = 30
|
|
|
|
/** Minimal value for ::BROTLI_PARAM_LGBLOCK parameter. */
|
|
const BROTLI_MIN_INPUT_BLOCK_BITS = 16
|
|
|
|
/** Maximal value for ::BROTLI_PARAM_LGBLOCK parameter. */
|
|
const BROTLI_MAX_INPUT_BLOCK_BITS = 24
|
|
|
|
/** Minimal value for ::BROTLI_PARAM_QUALITY parameter. */
|
|
const BROTLI_MIN_QUALITY = 0
|
|
|
|
/** Maximal value for ::BROTLI_PARAM_QUALITY parameter. */
|
|
const BROTLI_MAX_QUALITY = 11
|
|
|
|
/** Options for ::BROTLI_PARAM_MODE parameter. */
|
|
const (
|
|
BROTLI_MODE_GENERIC = 0
|
|
BROTLI_MODE_TEXT = 1
|
|
BROTLI_MODE_FONT = 2
|
|
)
|
|
|
|
/** Default value for ::BROTLI_PARAM_QUALITY parameter. */
|
|
const BROTLI_DEFAULT_QUALITY = 11
|
|
|
|
/** Default value for ::BROTLI_PARAM_LGWIN parameter. */
|
|
const BROTLI_DEFAULT_WINDOW = 22
|
|
|
|
/** Default value for ::BROTLI_PARAM_MODE parameter. */
|
|
const BROTLI_DEFAULT_MODE = BROTLI_MODE_GENERIC
|
|
|
|
/** Operations that can be performed by streaming encoder. */
|
|
const (
|
|
BROTLI_OPERATION_PROCESS = 0
|
|
BROTLI_OPERATION_FLUSH = 1
|
|
BROTLI_OPERATION_FINISH = 2
|
|
BROTLI_OPERATION_EMIT_METADATA = 3
|
|
)
|
|
|
|
/** Options to be used with ::BrotliEncoderSetParameter. */
|
|
const (
|
|
BROTLI_PARAM_MODE = 0
|
|
BROTLI_PARAM_QUALITY = 1
|
|
BROTLI_PARAM_LGWIN = 2
|
|
BROTLI_PARAM_LGBLOCK = 3
|
|
BROTLI_PARAM_DISABLE_LITERAL_CONTEXT_MODELING = 4
|
|
BROTLI_PARAM_SIZE_HINT = 5
|
|
BROTLI_PARAM_LARGE_WINDOW = 6
|
|
BROTLI_PARAM_NPOSTFIX = 7
|
|
BROTLI_PARAM_NDIRECT = 8
|
|
)
|
|
|
|
const (
|
|
BROTLI_STREAM_PROCESSING = 0
|
|
BROTLI_STREAM_FLUSH_REQUESTED = 1
|
|
BROTLI_STREAM_FINISHED = 2
|
|
BROTLI_STREAM_METADATA_HEAD = 3
|
|
BROTLI_STREAM_METADATA_BODY = 4
|
|
)
|
|
|
|
type Writer struct {
|
|
dst io.Writer
|
|
|
|
params BrotliEncoderParams
|
|
hasher_ HasherHandle
|
|
input_pos_ uint64
|
|
ringbuffer_ RingBuffer
|
|
cmd_alloc_size_ uint
|
|
commands_ []Command
|
|
num_commands_ uint
|
|
num_literals_ uint
|
|
last_insert_len_ uint
|
|
last_flush_pos_ uint64
|
|
last_processed_pos_ uint64
|
|
dist_cache_ [BROTLI_NUM_DISTANCE_SHORT_CODES]int
|
|
saved_dist_cache_ [4]int
|
|
last_bytes_ uint16
|
|
last_bytes_bits_ byte
|
|
prev_byte_ byte
|
|
prev_byte2_ byte
|
|
storage_size_ uint
|
|
storage_ []byte
|
|
small_table_ [1 << 10]int
|
|
large_table_ []int
|
|
large_table_size_ uint
|
|
cmd_depths_ [128]byte
|
|
cmd_bits_ [128]uint16
|
|
cmd_code_ [512]byte
|
|
cmd_code_numbits_ uint
|
|
command_buf_ []uint32
|
|
literal_buf_ []byte
|
|
next_out_ []byte
|
|
available_out_ uint
|
|
total_out_ uint
|
|
tiny_buf_ struct {
|
|
u64 [2]uint64
|
|
u8 [16]byte
|
|
}
|
|
remaining_metadata_bytes_ uint32
|
|
stream_state_ int
|
|
is_last_block_emitted_ bool
|
|
is_initialized_ bool
|
|
}
|
|
|
|
func InputBlockSize(s *Writer) uint {
|
|
return uint(1) << uint(s.params.lgblock)
|
|
}
|
|
|
|
func UnprocessedInputSize(s *Writer) uint64 {
|
|
return s.input_pos_ - s.last_processed_pos_
|
|
}
|
|
|
|
func RemainingInputBlockSize(s *Writer) uint {
|
|
var delta uint64 = UnprocessedInputSize(s)
|
|
var block_size uint = InputBlockSize(s)
|
|
if delta >= uint64(block_size) {
|
|
return 0
|
|
}
|
|
return block_size - uint(delta)
|
|
}
|
|
|
|
func BrotliEncoderSetParameter(state *Writer, p int, value uint32) bool {
|
|
/* Changing parameters on the fly is not implemented yet. */
|
|
if state.is_initialized_ {
|
|
return false
|
|
}
|
|
|
|
/* TODO: Validate/clamp parameters here. */
|
|
switch p {
|
|
case BROTLI_PARAM_MODE:
|
|
state.params.mode = int(value)
|
|
return true
|
|
|
|
case BROTLI_PARAM_QUALITY:
|
|
state.params.quality = int(value)
|
|
return true
|
|
|
|
case BROTLI_PARAM_LGWIN:
|
|
state.params.lgwin = uint(int(value))
|
|
return true
|
|
|
|
case BROTLI_PARAM_LGBLOCK:
|
|
state.params.lgblock = int(value)
|
|
return true
|
|
|
|
case BROTLI_PARAM_DISABLE_LITERAL_CONTEXT_MODELING:
|
|
if (value != 0) && (value != 1) {
|
|
return false
|
|
}
|
|
state.params.disable_literal_context_modeling = (!(value == 0))
|
|
return true
|
|
|
|
case BROTLI_PARAM_SIZE_HINT:
|
|
state.params.size_hint = uint(value)
|
|
return true
|
|
|
|
case BROTLI_PARAM_LARGE_WINDOW:
|
|
state.params.large_window = (!(value == 0))
|
|
return true
|
|
|
|
case BROTLI_PARAM_NPOSTFIX:
|
|
state.params.dist.distance_postfix_bits = value
|
|
return true
|
|
|
|
case BROTLI_PARAM_NDIRECT:
|
|
state.params.dist.num_direct_distance_codes = value
|
|
return true
|
|
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
/* Wraps 64-bit input position to 32-bit ring-buffer position preserving
|
|
"not-a-first-lap" feature. */
|
|
func WrapPosition(position uint64) uint32 {
|
|
var result uint32 = uint32(position)
|
|
var gb uint64 = position >> 30
|
|
if gb > 2 {
|
|
/* Wrap every 2GiB; The first 3GB are continuous. */
|
|
result = result&((1<<30)-1) | (uint32((gb-1)&1)+1)<<30
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
func GetBrotliStorage(s *Writer, size uint) []byte {
|
|
if s.storage_size_ < size {
|
|
s.storage_ = nil
|
|
s.storage_ = make([]byte, size)
|
|
s.storage_size_ = size
|
|
}
|
|
|
|
return s.storage_
|
|
}
|
|
|
|
func HashTableSize(max_table_size uint, input_size uint) uint {
|
|
var htsize uint = 256
|
|
for htsize < max_table_size && htsize < input_size {
|
|
htsize <<= 1
|
|
}
|
|
|
|
return htsize
|
|
}
|
|
|
|
func GetHashTable(s *Writer, quality int, input_size uint, table_size *uint) []int {
|
|
var max_table_size uint = MaxHashTableSize(quality)
|
|
var htsize uint = HashTableSize(max_table_size, input_size)
|
|
/* Use smaller hash table when input.size() is smaller, since we
|
|
fill the table, incurring O(hash table size) overhead for
|
|
compression, and if the input is short, we won't need that
|
|
many hash table entries anyway. */
|
|
|
|
var table []int
|
|
assert(max_table_size >= 256)
|
|
if quality == FAST_ONE_PASS_COMPRESSION_QUALITY {
|
|
/* Only odd shifts are supported by fast-one-pass. */
|
|
if htsize&0xAAAAA == 0 {
|
|
htsize <<= 1
|
|
}
|
|
}
|
|
|
|
if htsize <= uint(len(s.small_table_)) {
|
|
table = s.small_table_[:]
|
|
} else {
|
|
if htsize > s.large_table_size_ {
|
|
s.large_table_size_ = htsize
|
|
s.large_table_ = nil
|
|
s.large_table_ = make([]int, htsize)
|
|
}
|
|
|
|
table = s.large_table_
|
|
}
|
|
|
|
*table_size = htsize
|
|
for i := 0; i < int(htsize); i++ {
|
|
table[i] = 0
|
|
}
|
|
return table
|
|
}
|
|
|
|
func EncodeWindowBits(lgwin int, large_window bool, last_bytes *uint16, last_bytes_bits *byte) {
|
|
if large_window {
|
|
*last_bytes = uint16((lgwin&0x3F)<<8 | 0x11)
|
|
*last_bytes_bits = 14
|
|
} else {
|
|
if lgwin == 16 {
|
|
*last_bytes = 0
|
|
*last_bytes_bits = 1
|
|
} else if lgwin == 17 {
|
|
*last_bytes = 1
|
|
*last_bytes_bits = 7
|
|
} else if lgwin > 17 {
|
|
*last_bytes = uint16((lgwin-17)<<1 | 0x01)
|
|
*last_bytes_bits = 4
|
|
} else {
|
|
*last_bytes = uint16((lgwin-8)<<4 | 0x01)
|
|
*last_bytes_bits = 7
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Initializes the command and distance prefix codes for the first block. */
|
|
|
|
var InitCommandPrefixCodes_kDefaultCommandDepths = [128]byte{
|
|
0,
|
|
4,
|
|
4,
|
|
5,
|
|
6,
|
|
6,
|
|
7,
|
|
7,
|
|
7,
|
|
7,
|
|
7,
|
|
8,
|
|
8,
|
|
8,
|
|
8,
|
|
8,
|
|
0,
|
|
0,
|
|
0,
|
|
4,
|
|
4,
|
|
4,
|
|
4,
|
|
4,
|
|
5,
|
|
5,
|
|
6,
|
|
6,
|
|
6,
|
|
6,
|
|
7,
|
|
7,
|
|
7,
|
|
7,
|
|
10,
|
|
10,
|
|
10,
|
|
10,
|
|
10,
|
|
10,
|
|
0,
|
|
4,
|
|
4,
|
|
5,
|
|
5,
|
|
5,
|
|
6,
|
|
6,
|
|
7,
|
|
8,
|
|
8,
|
|
9,
|
|
10,
|
|
10,
|
|
10,
|
|
10,
|
|
10,
|
|
10,
|
|
10,
|
|
10,
|
|
10,
|
|
10,
|
|
10,
|
|
10,
|
|
5,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
6,
|
|
6,
|
|
6,
|
|
6,
|
|
6,
|
|
6,
|
|
5,
|
|
5,
|
|
5,
|
|
5,
|
|
5,
|
|
5,
|
|
4,
|
|
4,
|
|
4,
|
|
4,
|
|
4,
|
|
4,
|
|
4,
|
|
5,
|
|
5,
|
|
5,
|
|
5,
|
|
5,
|
|
5,
|
|
6,
|
|
6,
|
|
7,
|
|
7,
|
|
7,
|
|
8,
|
|
10,
|
|
12,
|
|
12,
|
|
12,
|
|
12,
|
|
12,
|
|
12,
|
|
12,
|
|
12,
|
|
12,
|
|
12,
|
|
12,
|
|
12,
|
|
}
|
|
var InitCommandPrefixCodes_kDefaultCommandBits = [128]uint16{
|
|
0,
|
|
0,
|
|
8,
|
|
9,
|
|
3,
|
|
35,
|
|
7,
|
|
71,
|
|
39,
|
|
103,
|
|
23,
|
|
47,
|
|
175,
|
|
111,
|
|
239,
|
|
31,
|
|
0,
|
|
0,
|
|
0,
|
|
4,
|
|
12,
|
|
2,
|
|
10,
|
|
6,
|
|
13,
|
|
29,
|
|
11,
|
|
43,
|
|
27,
|
|
59,
|
|
87,
|
|
55,
|
|
15,
|
|
79,
|
|
319,
|
|
831,
|
|
191,
|
|
703,
|
|
447,
|
|
959,
|
|
0,
|
|
14,
|
|
1,
|
|
25,
|
|
5,
|
|
21,
|
|
19,
|
|
51,
|
|
119,
|
|
159,
|
|
95,
|
|
223,
|
|
479,
|
|
991,
|
|
63,
|
|
575,
|
|
127,
|
|
639,
|
|
383,
|
|
895,
|
|
255,
|
|
767,
|
|
511,
|
|
1023,
|
|
14,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
27,
|
|
59,
|
|
7,
|
|
39,
|
|
23,
|
|
55,
|
|
30,
|
|
1,
|
|
17,
|
|
9,
|
|
25,
|
|
5,
|
|
0,
|
|
8,
|
|
4,
|
|
12,
|
|
2,
|
|
10,
|
|
6,
|
|
21,
|
|
13,
|
|
29,
|
|
3,
|
|
19,
|
|
11,
|
|
15,
|
|
47,
|
|
31,
|
|
95,
|
|
63,
|
|
127,
|
|
255,
|
|
767,
|
|
2815,
|
|
1791,
|
|
3839,
|
|
511,
|
|
2559,
|
|
1535,
|
|
3583,
|
|
1023,
|
|
3071,
|
|
2047,
|
|
4095,
|
|
}
|
|
var InitCommandPrefixCodes_kDefaultCommandCode = []byte{
|
|
0xff,
|
|
0x77,
|
|
0xd5,
|
|
0xbf,
|
|
0xe7,
|
|
0xde,
|
|
0xea,
|
|
0x9e,
|
|
0x51,
|
|
0x5d,
|
|
0xde,
|
|
0xc6,
|
|
0x70,
|
|
0x57,
|
|
0xbc,
|
|
0x58,
|
|
0x58,
|
|
0x58,
|
|
0xd8,
|
|
0xd8,
|
|
0x58,
|
|
0xd5,
|
|
0xcb,
|
|
0x8c,
|
|
0xea,
|
|
0xe0,
|
|
0xc3,
|
|
0x87,
|
|
0x1f,
|
|
0x83,
|
|
0xc1,
|
|
0x60,
|
|
0x1c,
|
|
0x67,
|
|
0xb2,
|
|
0xaa,
|
|
0x06,
|
|
0x83,
|
|
0xc1,
|
|
0x60,
|
|
0x30,
|
|
0x18,
|
|
0xcc,
|
|
0xa1,
|
|
0xce,
|
|
0x88,
|
|
0x54,
|
|
0x94,
|
|
0x46,
|
|
0xe1,
|
|
0xb0,
|
|
0xd0,
|
|
0x4e,
|
|
0xb2,
|
|
0xf7,
|
|
0x04,
|
|
0x00,
|
|
}
|
|
var InitCommandPrefixCodes_kDefaultCommandCodeNumBits uint = 448
|
|
|
|
func InitCommandPrefixCodes(cmd_depths []byte, cmd_bits []uint16, cmd_code []byte, cmd_code_numbits *uint) {
|
|
copy(cmd_depths, InitCommandPrefixCodes_kDefaultCommandDepths[:])
|
|
copy(cmd_bits, InitCommandPrefixCodes_kDefaultCommandBits[:])
|
|
|
|
/* Initialize the pre-compressed form of the command and distance prefix
|
|
codes. */
|
|
copy(cmd_code, InitCommandPrefixCodes_kDefaultCommandCode)
|
|
|
|
*cmd_code_numbits = InitCommandPrefixCodes_kDefaultCommandCodeNumBits
|
|
}
|
|
|
|
/* Decide about the context map based on the ability of the prediction
|
|
ability of the previous byte UTF8-prefix on the next byte. The
|
|
prediction ability is calculated as Shannon entropy. Here we need
|
|
Shannon entropy instead of 'BitsEntropy' since the prefix will be
|
|
encoded with the remaining 6 bits of the following byte, and
|
|
BitsEntropy will assume that symbol to be stored alone using Huffman
|
|
coding. */
|
|
|
|
var ChooseContextMap_kStaticContextMapContinuation = [64]uint32{
|
|
1,
|
|
1,
|
|
2,
|
|
2,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
}
|
|
var ChooseContextMap_kStaticContextMapSimpleUTF8 = [64]uint32{
|
|
0,
|
|
0,
|
|
1,
|
|
1,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
}
|
|
|
|
func ChooseContextMap(quality int, bigram_histo []uint32, num_literal_contexts *uint, literal_context_map *[]uint32) {
|
|
var monogram_histo = [3]uint32{0}
|
|
var two_prefix_histo = [6]uint32{0}
|
|
var total uint
|
|
var i uint
|
|
var dummy uint
|
|
var entropy [4]float64
|
|
for i = 0; i < 9; i++ {
|
|
monogram_histo[i%3] += bigram_histo[i]
|
|
two_prefix_histo[i%6] += bigram_histo[i]
|
|
}
|
|
|
|
entropy[1] = ShannonEntropy(monogram_histo[:], 3, &dummy)
|
|
entropy[2] = (ShannonEntropy(two_prefix_histo[:], 3, &dummy) + ShannonEntropy(two_prefix_histo[3:], 3, &dummy))
|
|
entropy[3] = 0
|
|
for i = 0; i < 3; i++ {
|
|
entropy[3] += ShannonEntropy(bigram_histo[3*i:], 3, &dummy)
|
|
}
|
|
|
|
total = uint(monogram_histo[0] + monogram_histo[1] + monogram_histo[2])
|
|
assert(total != 0)
|
|
entropy[0] = 1.0 / float64(total)
|
|
entropy[1] *= entropy[0]
|
|
entropy[2] *= entropy[0]
|
|
entropy[3] *= entropy[0]
|
|
|
|
if quality < MIN_QUALITY_FOR_HQ_CONTEXT_MODELING {
|
|
/* 3 context models is a bit slower, don't use it at lower qualities. */
|
|
entropy[3] = entropy[1] * 10
|
|
}
|
|
|
|
/* If expected savings by symbol are less than 0.2 bits, skip the
|
|
context modeling -- in exchange for faster decoding speed. */
|
|
if entropy[1]-entropy[2] < 0.2 && entropy[1]-entropy[3] < 0.2 {
|
|
*num_literal_contexts = 1
|
|
} else if entropy[2]-entropy[3] < 0.02 {
|
|
*num_literal_contexts = 2
|
|
*literal_context_map = ChooseContextMap_kStaticContextMapSimpleUTF8[:]
|
|
} else {
|
|
*num_literal_contexts = 3
|
|
*literal_context_map = ChooseContextMap_kStaticContextMapContinuation[:]
|
|
}
|
|
}
|
|
|
|
/* Decide if we want to use a more complex static context map containing 13
|
|
context values, based on the entropy reduction of histograms over the
|
|
first 5 bits of literals. */
|
|
|
|
var ShouldUseComplexStaticContextMap_kStaticContextMapComplexUTF8 = [64]uint32{
|
|
11,
|
|
11,
|
|
12,
|
|
12,
|
|
0,
|
|
0,
|
|
0,
|
|
0,
|
|
1,
|
|
1,
|
|
9,
|
|
9,
|
|
2,
|
|
2,
|
|
2,
|
|
2,
|
|
1,
|
|
1,
|
|
1,
|
|
1,
|
|
8,
|
|
3,
|
|
3,
|
|
3,
|
|
1,
|
|
1,
|
|
1,
|
|
1,
|
|
2,
|
|
2,
|
|
2,
|
|
2,
|
|
8,
|
|
4,
|
|
4,
|
|
4,
|
|
8,
|
|
7,
|
|
4,
|
|
4,
|
|
8,
|
|
0,
|
|
0,
|
|
0,
|
|
3,
|
|
3,
|
|
3,
|
|
3,
|
|
5,
|
|
5,
|
|
10,
|
|
5,
|
|
5,
|
|
5,
|
|
10,
|
|
5,
|
|
6,
|
|
6,
|
|
6,
|
|
6,
|
|
6,
|
|
6,
|
|
6,
|
|
6,
|
|
}
|
|
|
|
func ShouldUseComplexStaticContextMap(input []byte, start_pos uint, length uint, mask uint, quality int, size_hint uint, num_literal_contexts *uint, literal_context_map *[]uint32) bool {
|
|
/* Try the more complex static context map only for long data. */
|
|
if size_hint < 1<<20 {
|
|
return false
|
|
} else {
|
|
var end_pos uint = start_pos + length
|
|
var combined_histo = [32]uint32{0}
|
|
var context_histo = [13][32]uint32{[32]uint32{0}}
|
|
var total uint32 = 0
|
|
var entropy [3]float64
|
|
var dummy uint
|
|
var i uint
|
|
var utf8_lut ContextLut = BROTLI_CONTEXT_LUT(CONTEXT_UTF8)
|
|
/* To make entropy calculations faster and to fit on the stack, we collect
|
|
histograms over the 5 most significant bits of literals. One histogram
|
|
without context and 13 additional histograms for each context value. */
|
|
for ; start_pos+64 <= end_pos; start_pos += 4096 {
|
|
var stride_end_pos uint = start_pos + 64
|
|
var prev2 byte = input[start_pos&mask]
|
|
var prev1 byte = input[(start_pos+1)&mask]
|
|
var pos uint
|
|
|
|
/* To make the analysis of the data faster we only examine 64 byte long
|
|
strides at every 4kB intervals. */
|
|
for pos = start_pos + 2; pos < stride_end_pos; pos++ {
|
|
var literal byte = input[pos&mask]
|
|
var context byte = byte(ShouldUseComplexStaticContextMap_kStaticContextMapComplexUTF8[BROTLI_CONTEXT(prev1, prev2, utf8_lut)])
|
|
total++
|
|
combined_histo[literal>>3]++
|
|
context_histo[context][literal>>3]++
|
|
prev2 = prev1
|
|
prev1 = literal
|
|
}
|
|
}
|
|
|
|
entropy[1] = ShannonEntropy(combined_histo[:], 32, &dummy)
|
|
entropy[2] = 0
|
|
for i = 0; i < 13; i++ {
|
|
entropy[2] += ShannonEntropy(context_histo[i][0:], 32, &dummy)
|
|
}
|
|
|
|
entropy[0] = 1.0 / float64(total)
|
|
entropy[1] *= entropy[0]
|
|
entropy[2] *= entropy[0]
|
|
|
|
/* The triggering heuristics below were tuned by compressing the individual
|
|
files of the silesia corpus. If we skip this kind of context modeling
|
|
for not very well compressible input (i.e. entropy using context modeling
|
|
is 60% of maximal entropy) or if expected savings by symbol are less
|
|
than 0.2 bits, then in every case when it triggers, the final compression
|
|
ratio is improved. Note however that this heuristics might be too strict
|
|
for some cases and could be tuned further. */
|
|
if entropy[2] > 3.0 || entropy[1]-entropy[2] < 0.2 {
|
|
return false
|
|
} else {
|
|
*num_literal_contexts = 13
|
|
*literal_context_map = ShouldUseComplexStaticContextMap_kStaticContextMapComplexUTF8[:]
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
|
|
func DecideOverLiteralContextModeling(input []byte, start_pos uint, length uint, mask uint, quality int, size_hint uint, num_literal_contexts *uint, literal_context_map *[]uint32) {
|
|
if quality < MIN_QUALITY_FOR_CONTEXT_MODELING || length < 64 {
|
|
return
|
|
} else if ShouldUseComplexStaticContextMap(input, start_pos, length, mask, quality, size_hint, num_literal_contexts, literal_context_map) {
|
|
} else /* Context map was already set, nothing else to do. */
|
|
{
|
|
var end_pos uint = start_pos + length
|
|
/* Gather bi-gram data of the UTF8 byte prefixes. To make the analysis of
|
|
UTF8 data faster we only examine 64 byte long strides at every 4kB
|
|
intervals. */
|
|
|
|
var bigram_prefix_histo = [9]uint32{0}
|
|
for ; start_pos+64 <= end_pos; start_pos += 4096 {
|
|
var lut = [4]int{0, 0, 1, 2}
|
|
var stride_end_pos uint = start_pos + 64
|
|
var prev int = lut[input[start_pos&mask]>>6] * 3
|
|
var pos uint
|
|
for pos = start_pos + 1; pos < stride_end_pos; pos++ {
|
|
var literal byte = input[pos&mask]
|
|
bigram_prefix_histo[prev+lut[literal>>6]]++
|
|
prev = lut[literal>>6] * 3
|
|
}
|
|
}
|
|
|
|
ChooseContextMap(quality, bigram_prefix_histo[0:], num_literal_contexts, literal_context_map)
|
|
}
|
|
}
|
|
|
|
func ShouldCompress_encode(data []byte, mask uint, last_flush_pos uint64, bytes uint, num_literals uint, num_commands uint) bool {
|
|
/* TODO: find more precise minimal block overhead. */
|
|
if bytes <= 2 {
|
|
return false
|
|
}
|
|
if num_commands < (bytes>>8)+2 {
|
|
if float64(num_literals) > 0.99*float64(bytes) {
|
|
var literal_histo = [256]uint32{0}
|
|
var kSampleRate uint32 = 13
|
|
var kMinEntropy float64 = 7.92
|
|
var bit_cost_threshold float64 = float64(bytes) * kMinEntropy / float64(kSampleRate)
|
|
var t uint = uint((uint32(bytes) + kSampleRate - 1) / kSampleRate)
|
|
var pos uint32 = uint32(last_flush_pos)
|
|
var i uint
|
|
for i = 0; i < t; i++ {
|
|
literal_histo[data[pos&uint32(mask)]]++
|
|
pos += kSampleRate
|
|
}
|
|
|
|
if BitsEntropy(literal_histo[:], 256) > bit_cost_threshold {
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
/* Chooses the literal context mode for a metablock */
|
|
func ChooseContextMode(params *BrotliEncoderParams, data []byte, pos uint, mask uint, length uint) int {
|
|
/* We only do the computation for the option of something else than
|
|
CONTEXT_UTF8 for the highest qualities */
|
|
if params.quality >= MIN_QUALITY_FOR_HQ_BLOCK_SPLITTING && !BrotliIsMostlyUTF8(data, pos, mask, length, kMinUTF8Ratio) {
|
|
return CONTEXT_SIGNED
|
|
}
|
|
|
|
return CONTEXT_UTF8
|
|
}
|
|
|
|
func WriteMetaBlockInternal(data []byte, mask uint, last_flush_pos uint64, bytes uint, is_last bool, literal_context_mode int, params *BrotliEncoderParams, prev_byte byte, prev_byte2 byte, num_literals uint, num_commands uint, commands []Command, saved_dist_cache []int, dist_cache []int, storage_ix *uint, storage []byte) {
|
|
var wrapped_last_flush_pos uint32 = WrapPosition(last_flush_pos)
|
|
var last_bytes uint16
|
|
var last_bytes_bits byte
|
|
var literal_context_lut ContextLut = BROTLI_CONTEXT_LUT(literal_context_mode)
|
|
var block_params BrotliEncoderParams = *params
|
|
|
|
if bytes == 0 {
|
|
/* Write the ISLAST and ISEMPTY bits. */
|
|
BrotliWriteBits(2, 3, storage_ix, storage)
|
|
|
|
*storage_ix = (*storage_ix + 7) &^ 7
|
|
return
|
|
}
|
|
|
|
if !ShouldCompress_encode(data, mask, last_flush_pos, bytes, num_literals, num_commands) {
|
|
/* Restore the distance cache, as its last update by
|
|
CreateBackwardReferences is now unused. */
|
|
copy(dist_cache, saved_dist_cache[:4])
|
|
|
|
BrotliStoreUncompressedMetaBlock(is_last, data, uint(wrapped_last_flush_pos), mask, bytes, storage_ix, storage)
|
|
return
|
|
}
|
|
|
|
assert(*storage_ix <= 14)
|
|
last_bytes = uint16(storage[1]<<8 | storage[0])
|
|
last_bytes_bits = byte(*storage_ix)
|
|
if params.quality <= MAX_QUALITY_FOR_STATIC_ENTROPY_CODES {
|
|
BrotliStoreMetaBlockFast(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, num_commands, storage_ix, storage)
|
|
} else if params.quality < MIN_QUALITY_FOR_BLOCK_SPLIT {
|
|
BrotliStoreMetaBlockTrivial(data, uint(wrapped_last_flush_pos), bytes, mask, is_last, params, commands, num_commands, storage_ix, storage)
|
|
} else {
|
|
var mb MetaBlockSplit
|
|
InitMetaBlockSplit(&mb)
|
|
if params.quality < MIN_QUALITY_FOR_HQ_BLOCK_SPLITTING {
|
|
var num_literal_contexts uint = 1
|
|
var literal_context_map []uint32 = nil
|
|
if !params.disable_literal_context_modeling {
|
|
DecideOverLiteralContextModeling(data, uint(wrapped_last_flush_pos), bytes, mask, params.quality, params.size_hint, &num_literal_contexts, &literal_context_map)
|
|
}
|
|
|
|
BrotliBuildMetaBlockGreedy(data, uint(wrapped_last_flush_pos), mask, prev_byte, prev_byte2, literal_context_lut, num_literal_contexts, literal_context_map, commands, num_commands, &mb)
|
|
} else {
|
|
BrotliBuildMetaBlock(data, uint(wrapped_last_flush_pos), mask, &block_params, prev_byte, prev_byte2, commands, num_commands, literal_context_mode, &mb)
|
|
}
|
|
|
|
if params.quality >= MIN_QUALITY_FOR_OPTIMIZE_HISTOGRAMS {
|
|
/* The number of distance symbols effectively used for distance
|
|
histograms. It might be less than distance alphabet size
|
|
for "Large Window Brotli" (32-bit). */
|
|
var num_effective_dist_codes uint32 = block_params.dist.alphabet_size
|
|
if num_effective_dist_codes > BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS {
|
|
num_effective_dist_codes = BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS
|
|
}
|
|
|
|
BrotliOptimizeHistograms(num_effective_dist_codes, &mb)
|
|
}
|
|
|
|
BrotliStoreMetaBlock(data, uint(wrapped_last_flush_pos), bytes, mask, prev_byte, prev_byte2, is_last, &block_params, literal_context_mode, commands, num_commands, &mb, storage_ix, storage)
|
|
DestroyMetaBlockSplit(&mb)
|
|
}
|
|
|
|
if bytes+4 < *storage_ix>>3 {
|
|
/* Restore the distance cache and last byte. */
|
|
copy(dist_cache, saved_dist_cache[:4])
|
|
|
|
storage[0] = byte(last_bytes)
|
|
storage[1] = byte(last_bytes >> 8)
|
|
*storage_ix = uint(last_bytes_bits)
|
|
BrotliStoreUncompressedMetaBlock(is_last, data, uint(wrapped_last_flush_pos), mask, bytes, storage_ix, storage)
|
|
}
|
|
}
|
|
|
|
func ChooseDistanceParams(params *BrotliEncoderParams) {
|
|
var distance_postfix_bits uint32 = 0
|
|
var num_direct_distance_codes uint32 = 0
|
|
|
|
if params.quality >= MIN_QUALITY_FOR_NONZERO_DISTANCE_PARAMS {
|
|
var ndirect_msb uint32
|
|
if params.mode == BROTLI_MODE_FONT {
|
|
distance_postfix_bits = 1
|
|
num_direct_distance_codes = 12
|
|
} else {
|
|
distance_postfix_bits = params.dist.distance_postfix_bits
|
|
num_direct_distance_codes = params.dist.num_direct_distance_codes
|
|
}
|
|
|
|
ndirect_msb = (num_direct_distance_codes >> distance_postfix_bits) & 0x0F
|
|
if distance_postfix_bits > BROTLI_MAX_NPOSTFIX || num_direct_distance_codes > BROTLI_MAX_NDIRECT || ndirect_msb<<distance_postfix_bits != num_direct_distance_codes {
|
|
distance_postfix_bits = 0
|
|
num_direct_distance_codes = 0
|
|
}
|
|
}
|
|
|
|
BrotliInitDistanceParams(params, distance_postfix_bits, num_direct_distance_codes)
|
|
}
|
|
|
|
func EnsureInitialized(s *Writer) bool {
|
|
if s.is_initialized_ {
|
|
return true
|
|
}
|
|
|
|
s.last_bytes_bits_ = 0
|
|
s.last_bytes_ = 0
|
|
s.remaining_metadata_bytes_ = BROTLI_UINT32_MAX
|
|
|
|
SanitizeParams(&s.params)
|
|
s.params.lgblock = ComputeLgBlock(&s.params)
|
|
ChooseDistanceParams(&s.params)
|
|
|
|
RingBufferSetup(&s.params, &s.ringbuffer_)
|
|
|
|
/* Initialize last byte with stream header. */
|
|
{
|
|
var lgwin int = int(s.params.lgwin)
|
|
if s.params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY || s.params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY {
|
|
lgwin = brotli_max_int(lgwin, 18)
|
|
}
|
|
|
|
EncodeWindowBits(lgwin, s.params.large_window, &s.last_bytes_, &s.last_bytes_bits_)
|
|
}
|
|
|
|
if s.params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY {
|
|
InitCommandPrefixCodes(s.cmd_depths_[:], s.cmd_bits_[:], s.cmd_code_[:], &s.cmd_code_numbits_)
|
|
}
|
|
|
|
s.is_initialized_ = true
|
|
return true
|
|
}
|
|
|
|
func BrotliEncoderInitParams(params *BrotliEncoderParams) {
|
|
params.mode = BROTLI_DEFAULT_MODE
|
|
params.large_window = false
|
|
params.quality = BROTLI_DEFAULT_QUALITY
|
|
params.lgwin = BROTLI_DEFAULT_WINDOW
|
|
params.lgblock = 0
|
|
params.size_hint = 0
|
|
params.disable_literal_context_modeling = false
|
|
BrotliInitEncoderDictionary(¶ms.dictionary)
|
|
params.dist.distance_postfix_bits = 0
|
|
params.dist.num_direct_distance_codes = 0
|
|
params.dist.alphabet_size = uint32(BROTLI_DISTANCE_ALPHABET_SIZE(0, 0, BROTLI_MAX_DISTANCE_BITS))
|
|
params.dist.max_distance = BROTLI_MAX_DISTANCE
|
|
}
|
|
|
|
func BrotliEncoderInitState(s *Writer) {
|
|
BrotliEncoderInitParams(&s.params)
|
|
s.input_pos_ = 0
|
|
s.num_commands_ = 0
|
|
s.num_literals_ = 0
|
|
s.last_insert_len_ = 0
|
|
s.last_flush_pos_ = 0
|
|
s.last_processed_pos_ = 0
|
|
s.prev_byte_ = 0
|
|
s.prev_byte2_ = 0
|
|
s.storage_size_ = 0
|
|
s.storage_ = nil
|
|
s.hasher_ = nil
|
|
s.large_table_ = nil
|
|
s.large_table_size_ = 0
|
|
s.cmd_code_numbits_ = 0
|
|
s.command_buf_ = nil
|
|
s.literal_buf_ = nil
|
|
s.next_out_ = nil
|
|
s.available_out_ = 0
|
|
s.total_out_ = 0
|
|
s.stream_state_ = BROTLI_STREAM_PROCESSING
|
|
s.is_last_block_emitted_ = false
|
|
s.is_initialized_ = false
|
|
|
|
RingBufferInit(&s.ringbuffer_)
|
|
|
|
s.commands_ = nil
|
|
s.cmd_alloc_size_ = 0
|
|
|
|
/* Initialize distance cache. */
|
|
s.dist_cache_[0] = 4
|
|
|
|
s.dist_cache_[1] = 11
|
|
s.dist_cache_[2] = 15
|
|
s.dist_cache_[3] = 16
|
|
|
|
/* Save the state of the distance cache in case we need to restore it for
|
|
emitting an uncompressed block. */
|
|
copy(s.saved_dist_cache_[:], s.dist_cache_[:])
|
|
}
|
|
|
|
func BrotliEncoderCreateInstance() *Writer {
|
|
var state *Writer = nil
|
|
state = new(Writer)
|
|
if state == nil {
|
|
/* BROTLI_DUMP(); */
|
|
return nil
|
|
}
|
|
|
|
BrotliEncoderInitState(state)
|
|
return state
|
|
}
|
|
|
|
func BrotliEncoderCleanupState(s *Writer) {
|
|
s.storage_ = nil
|
|
s.commands_ = nil
|
|
RingBufferFree(&s.ringbuffer_)
|
|
DestroyHasher(&s.hasher_)
|
|
s.large_table_ = nil
|
|
s.command_buf_ = nil
|
|
s.literal_buf_ = nil
|
|
}
|
|
|
|
/* Deinitializes and frees BrotliEncoderState instance. */
|
|
func BrotliEncoderDestroyInstance(state *Writer) {
|
|
if state == nil {
|
|
return
|
|
} else {
|
|
BrotliEncoderCleanupState(state)
|
|
}
|
|
}
|
|
|
|
/*
|
|
Copies the given input data to the internal ring buffer of the compressor.
|
|
No processing of the data occurs at this time and this function can be
|
|
called multiple times before calling WriteBrotliData() to process the
|
|
accumulated input. At most input_block_size() bytes of input data can be
|
|
copied to the ring buffer, otherwise the next WriteBrotliData() will fail.
|
|
*/
|
|
func CopyInputToRingBuffer(s *Writer, input_size uint, input_buffer []byte) {
|
|
var ringbuffer_ *RingBuffer = &s.ringbuffer_
|
|
RingBufferWrite(input_buffer, input_size, ringbuffer_)
|
|
s.input_pos_ += uint64(input_size)
|
|
|
|
/* TL;DR: If needed, initialize 7 more bytes in the ring buffer to make the
|
|
hashing not depend on uninitialized data. This makes compression
|
|
deterministic and it prevents uninitialized memory warnings in Valgrind.
|
|
Even without erasing, the output would be valid (but nondeterministic).
|
|
|
|
Background information: The compressor stores short (at most 8 bytes)
|
|
substrings of the input already read in a hash table, and detects
|
|
repetitions by looking up such substrings in the hash table. If it
|
|
can find a substring, it checks whether the substring is really there
|
|
in the ring buffer (or it's just a hash collision). Should the hash
|
|
table become corrupt, this check makes sure that the output is
|
|
still valid, albeit the compression ratio would be bad.
|
|
|
|
The compressor populates the hash table from the ring buffer as it's
|
|
reading new bytes from the input. However, at the last few indexes of
|
|
the ring buffer, there are not enough bytes to build full-length
|
|
substrings from. Since the hash table always contains full-length
|
|
substrings, we erase with dummy zeros here to make sure that those
|
|
substrings will contain zeros at the end instead of uninitialized
|
|
data.
|
|
|
|
Please note that erasing is not necessary (because the
|
|
memory region is already initialized since he ring buffer
|
|
has a `tail' that holds a copy of the beginning,) so we
|
|
skip erasing if we have already gone around at least once in
|
|
the ring buffer.
|
|
|
|
Only clear during the first round of ring-buffer writes. On
|
|
subsequent rounds data in the ring-buffer would be affected. */
|
|
if ringbuffer_.pos_ <= ringbuffer_.mask_ {
|
|
/* This is the first time when the ring buffer is being written.
|
|
We clear 7 bytes just after the bytes that have been copied from
|
|
the input buffer.
|
|
|
|
The ring-buffer has a "tail" that holds a copy of the beginning,
|
|
but only once the ring buffer has been fully written once, i.e.,
|
|
pos <= mask. For the first time, we need to write values
|
|
in this tail (where index may be larger than mask), so that
|
|
we have exactly defined behavior and don't read uninitialized
|
|
memory. Due to performance reasons, hashing reads data using a
|
|
LOAD64, which can go 7 bytes beyond the bytes written in the
|
|
ring-buffer. */
|
|
for i := 0; i < int(7); i++ {
|
|
ringbuffer_.buffer_[ringbuffer_.pos_:][i] = 0
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Marks all input as processed.
|
|
Returns true if position wrapping occurs. */
|
|
func UpdateLastProcessedPos(s *Writer) bool {
|
|
var wrapped_last_processed_pos uint32 = WrapPosition(s.last_processed_pos_)
|
|
var wrapped_input_pos uint32 = WrapPosition(s.input_pos_)
|
|
s.last_processed_pos_ = s.input_pos_
|
|
return wrapped_input_pos < wrapped_last_processed_pos
|
|
}
|
|
|
|
func ExtendLastCommand(s *Writer, bytes *uint32, wrapped_last_processed_pos *uint32) {
|
|
var last_command *Command = &s.commands_[s.num_commands_-1]
|
|
var data []byte = s.ringbuffer_.buffer_
|
|
var mask uint32 = s.ringbuffer_.mask_
|
|
var max_backward_distance uint64 = ((uint64(1)) << s.params.lgwin) - BROTLI_WINDOW_GAP
|
|
var last_copy_len uint64 = uint64(last_command.copy_len_) & 0x1FFFFFF
|
|
var last_processed_pos uint64 = s.last_processed_pos_ - last_copy_len
|
|
var max_distance uint64
|
|
if last_processed_pos < max_backward_distance {
|
|
max_distance = last_processed_pos
|
|
} else {
|
|
max_distance = max_backward_distance
|
|
}
|
|
var cmd_dist uint64 = uint64(s.dist_cache_[0])
|
|
var distance_code uint32 = CommandRestoreDistanceCode(last_command, &s.params.dist)
|
|
if distance_code < BROTLI_NUM_DISTANCE_SHORT_CODES || uint64(distance_code-(BROTLI_NUM_DISTANCE_SHORT_CODES-1)) == cmd_dist {
|
|
if cmd_dist <= max_distance {
|
|
for *bytes != 0 && data[*wrapped_last_processed_pos&mask] == data[(uint64(*wrapped_last_processed_pos)-cmd_dist)&uint64(mask)] {
|
|
last_command.copy_len_++
|
|
(*bytes)--
|
|
(*wrapped_last_processed_pos)++
|
|
}
|
|
}
|
|
|
|
/* The copy length is at most the metablock size, and thus expressible. */
|
|
GetLengthCode(uint(last_command.insert_len_), uint(int(last_command.copy_len_&0x1FFFFFF)+int(last_command.copy_len_>>25)), (last_command.dist_prefix_&0x3FF == 0), &last_command.cmd_prefix_)
|
|
}
|
|
}
|
|
|
|
/*
|
|
Processes the accumulated input data and sets |*out_size| to the length of
|
|
the new output meta-block, or to zero if no new output meta-block has been
|
|
created (in this case the processed input data is buffered internally).
|
|
If |*out_size| is positive, |*output| points to the start of the output
|
|
data. If |is_last| or |force_flush| is true, an output meta-block is
|
|
always created. However, until |is_last| is true encoder may retain up
|
|
to 7 bits of the last byte of output. To force encoder to dump the remaining
|
|
bits use WriteMetadata() to append an empty meta-data block.
|
|
Returns false if the size of the input data is larger than
|
|
input_block_size().
|
|
*/
|
|
func EncodeData(s *Writer, is_last bool, force_flush bool, out_size *uint, output *[]byte) bool {
|
|
var delta uint64 = UnprocessedInputSize(s)
|
|
var bytes uint32 = uint32(delta)
|
|
var wrapped_last_processed_pos uint32 = WrapPosition(s.last_processed_pos_)
|
|
var data []byte
|
|
var mask uint32
|
|
var literal_context_mode int
|
|
|
|
data = s.ringbuffer_.buffer_
|
|
mask = s.ringbuffer_.mask_
|
|
|
|
/* Adding more blocks after "last" block is forbidden. */
|
|
if s.is_last_block_emitted_ {
|
|
return false
|
|
}
|
|
if is_last {
|
|
s.is_last_block_emitted_ = true
|
|
}
|
|
|
|
if delta > uint64(InputBlockSize(s)) {
|
|
return false
|
|
}
|
|
|
|
if s.params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY && s.command_buf_ == nil {
|
|
s.command_buf_ = make([]uint32, kCompressFragmentTwoPassBlockSize)
|
|
s.literal_buf_ = make([]byte, kCompressFragmentTwoPassBlockSize)
|
|
}
|
|
|
|
if s.params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY || s.params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY {
|
|
var storage []byte
|
|
var storage_ix uint = uint(s.last_bytes_bits_)
|
|
var table_size uint
|
|
var table []int
|
|
|
|
if delta == 0 && !is_last {
|
|
/* We have no new input data and we don't have to finish the stream, so
|
|
nothing to do. */
|
|
*out_size = 0
|
|
|
|
return true
|
|
}
|
|
|
|
storage = GetBrotliStorage(s, uint(2*bytes+503))
|
|
storage[0] = byte(s.last_bytes_)
|
|
storage[1] = byte(s.last_bytes_ >> 8)
|
|
table = GetHashTable(s, s.params.quality, uint(bytes), &table_size)
|
|
if s.params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY {
|
|
BrotliCompressFragmentFast(data[wrapped_last_processed_pos&mask:], uint(bytes), is_last, table, table_size, s.cmd_depths_[:], s.cmd_bits_[:], &s.cmd_code_numbits_, s.cmd_code_[:], &storage_ix, storage)
|
|
} else {
|
|
BrotliCompressFragmentTwoPass(data[wrapped_last_processed_pos&mask:], uint(bytes), is_last, s.command_buf_, s.literal_buf_, table, table_size, &storage_ix, storage)
|
|
}
|
|
|
|
s.last_bytes_ = uint16(storage[storage_ix>>3])
|
|
s.last_bytes_bits_ = byte(storage_ix & 7)
|
|
UpdateLastProcessedPos(s)
|
|
*output = storage[0:]
|
|
*out_size = storage_ix >> 3
|
|
return true
|
|
}
|
|
{
|
|
/* Theoretical max number of commands is 1 per 2 bytes. */
|
|
var newsize uint = uint(uint32(s.num_commands_) + bytes/2 + 1)
|
|
if newsize > s.cmd_alloc_size_ {
|
|
var new_commands []Command
|
|
|
|
/* Reserve a bit more memory to allow merging with a next block
|
|
without reallocation: that would impact speed. */
|
|
newsize += uint((bytes / 4) + 16)
|
|
|
|
s.cmd_alloc_size_ = newsize
|
|
new_commands = make([]Command, newsize)
|
|
if s.commands_ != nil {
|
|
copy(new_commands, s.commands_[:s.num_commands_])
|
|
s.commands_ = nil
|
|
}
|
|
|
|
s.commands_ = new_commands
|
|
}
|
|
}
|
|
|
|
InitOrStitchToPreviousBlock(&s.hasher_, data, uint(mask), &s.params, uint(wrapped_last_processed_pos), uint(bytes), is_last)
|
|
|
|
literal_context_mode = ChooseContextMode(&s.params, data, uint(WrapPosition(s.last_flush_pos_)), uint(mask), uint(s.input_pos_-s.last_flush_pos_))
|
|
|
|
if s.num_commands_ != 0 && s.last_insert_len_ == 0 {
|
|
ExtendLastCommand(s, &bytes, &wrapped_last_processed_pos)
|
|
}
|
|
|
|
if s.params.quality == ZOPFLIFICATION_QUALITY {
|
|
assert(s.params.hasher.type_ == 10)
|
|
BrotliCreateZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_.(*H10), s.dist_cache_[:], &s.last_insert_len_, s.commands_[s.num_commands_:], &s.num_commands_, &s.num_literals_)
|
|
} else if s.params.quality == HQ_ZOPFLIFICATION_QUALITY {
|
|
assert(s.params.hasher.type_ == 10)
|
|
BrotliCreateHqZopfliBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, s.commands_[s.num_commands_:], &s.num_commands_, &s.num_literals_)
|
|
} else {
|
|
BrotliCreateBackwardReferences(uint(bytes), uint(wrapped_last_processed_pos), data, uint(mask), &s.params, s.hasher_, s.dist_cache_[:], &s.last_insert_len_, s.commands_[s.num_commands_:], &s.num_commands_, &s.num_literals_)
|
|
}
|
|
{
|
|
var max_length uint = MaxMetablockSize(&s.params)
|
|
var max_literals uint = max_length / 8
|
|
var max_commands uint = max_length / 8
|
|
var processed_bytes uint = uint(s.input_pos_ - s.last_flush_pos_)
|
|
var next_input_fits_metablock bool = (processed_bytes+InputBlockSize(s) <= max_length)
|
|
var should_flush bool = (s.params.quality < MIN_QUALITY_FOR_BLOCK_SPLIT && s.num_literals_+s.num_commands_ >= MAX_NUM_DELAYED_SYMBOLS)
|
|
/* If maximal possible additional block doesn't fit metablock, flush now. */
|
|
/* TODO: Postpone decision until next block arrives? */
|
|
|
|
/* If block splitting is not used, then flush as soon as there is some
|
|
amount of commands / literals produced. */
|
|
if !is_last && !force_flush && !should_flush && next_input_fits_metablock && s.num_literals_ < max_literals && s.num_commands_ < max_commands {
|
|
/* Merge with next input block. Everything will happen later. */
|
|
if UpdateLastProcessedPos(s) {
|
|
HasherReset(s.hasher_)
|
|
}
|
|
|
|
*out_size = 0
|
|
return true
|
|
}
|
|
}
|
|
|
|
/* Create the last insert-only command. */
|
|
if s.last_insert_len_ > 0 {
|
|
InitInsertCommand(&s.commands_[s.num_commands_], s.last_insert_len_)
|
|
s.num_commands_++
|
|
s.num_literals_ += s.last_insert_len_
|
|
s.last_insert_len_ = 0
|
|
}
|
|
|
|
if !is_last && s.input_pos_ == s.last_flush_pos_ {
|
|
/* We have no new input data and we don't have to finish the stream, so
|
|
nothing to do. */
|
|
*out_size = 0
|
|
|
|
return true
|
|
}
|
|
|
|
assert(s.input_pos_ >= s.last_flush_pos_)
|
|
assert(s.input_pos_ > s.last_flush_pos_ || is_last)
|
|
assert(s.input_pos_-s.last_flush_pos_ <= 1<<24)
|
|
{
|
|
var metablock_size uint32 = uint32(s.input_pos_ - s.last_flush_pos_)
|
|
var storage []byte = GetBrotliStorage(s, uint(2*metablock_size+503))
|
|
var storage_ix uint = uint(s.last_bytes_bits_)
|
|
storage[0] = byte(s.last_bytes_)
|
|
storage[1] = byte(s.last_bytes_ >> 8)
|
|
WriteMetaBlockInternal(data, uint(mask), s.last_flush_pos_, uint(metablock_size), is_last, literal_context_mode, &s.params, s.prev_byte_, s.prev_byte2_, s.num_literals_, s.num_commands_, s.commands_, s.saved_dist_cache_[:], s.dist_cache_[:], &storage_ix, storage)
|
|
s.last_bytes_ = uint16(storage[storage_ix>>3])
|
|
s.last_bytes_bits_ = byte(storage_ix & 7)
|
|
s.last_flush_pos_ = s.input_pos_
|
|
if UpdateLastProcessedPos(s) {
|
|
HasherReset(s.hasher_)
|
|
}
|
|
|
|
if s.last_flush_pos_ > 0 {
|
|
s.prev_byte_ = data[(uint32(s.last_flush_pos_)-1)&mask]
|
|
}
|
|
|
|
if s.last_flush_pos_ > 1 {
|
|
s.prev_byte2_ = data[uint32(s.last_flush_pos_-2)&mask]
|
|
}
|
|
|
|
s.num_commands_ = 0
|
|
s.num_literals_ = 0
|
|
|
|
/* Save the state of the distance cache in case we need to restore it for
|
|
emitting an uncompressed block. */
|
|
copy(s.saved_dist_cache_[:], s.dist_cache_[:])
|
|
|
|
*output = storage[0:]
|
|
*out_size = storage_ix >> 3
|
|
return true
|
|
}
|
|
}
|
|
|
|
/* Dumps remaining output bits and metadata header to |header|.
|
|
Returns number of produced bytes.
|
|
REQUIRED: |header| should be 8-byte aligned and at least 16 bytes long.
|
|
REQUIRED: |block_size| <= (1 << 24). */
|
|
func WriteMetadataHeader(s *Writer, block_size uint, header []byte) uint {
|
|
var storage_ix uint
|
|
storage_ix = uint(s.last_bytes_bits_)
|
|
header[0] = byte(s.last_bytes_)
|
|
header[1] = byte(s.last_bytes_ >> 8)
|
|
s.last_bytes_ = 0
|
|
s.last_bytes_bits_ = 0
|
|
|
|
BrotliWriteBits(1, 0, &storage_ix, header)
|
|
BrotliWriteBits(2, 3, &storage_ix, header)
|
|
BrotliWriteBits(1, 0, &storage_ix, header)
|
|
if block_size == 0 {
|
|
BrotliWriteBits(2, 0, &storage_ix, header)
|
|
} else {
|
|
var nbits uint32
|
|
if block_size == 1 {
|
|
nbits = 0
|
|
} else {
|
|
nbits = Log2FloorNonZero(uint(uint32(block_size)-1)) + 1
|
|
}
|
|
var nbytes uint32 = (nbits + 7) / 8
|
|
BrotliWriteBits(2, uint64(nbytes), &storage_ix, header)
|
|
BrotliWriteBits(uint(8*nbytes), uint64(block_size)-1, &storage_ix, header)
|
|
}
|
|
|
|
return (storage_ix + 7) >> 3
|
|
}
|
|
|
|
func BrotliCompressBufferQuality10(lgwin int, input_size uint, input_buffer []byte, encoded_size *uint, encoded_buffer []byte) bool {
|
|
var mask uint = BROTLI_SIZE_MAX >> 1
|
|
var dist_cache = [4]int{4, 11, 15, 16}
|
|
var saved_dist_cache = [4]int{4, 11, 15, 16}
|
|
var ok bool = true
|
|
var max_out_size uint = *encoded_size
|
|
var total_out_size uint = 0
|
|
var last_bytes uint16
|
|
var last_bytes_bits byte
|
|
var hasher HasherHandle = nil
|
|
var hasher_eff_size uint = brotli_min_size_t(input_size, BROTLI_MAX_BACKWARD_LIMIT(uint(lgwin))+BROTLI_WINDOW_GAP)
|
|
var params BrotliEncoderParams
|
|
var lgmetablock int = brotli_min_int(24, lgwin+1)
|
|
var max_block_size uint
|
|
var max_metablock_size uint = uint(1) << uint(lgmetablock)
|
|
var max_literals_per_metablock uint = max_metablock_size / 8
|
|
var max_commands_per_metablock uint = max_metablock_size / 8
|
|
var metablock_start uint = 0
|
|
var prev_byte byte = 0
|
|
var prev_byte2 byte = 0
|
|
|
|
BrotliEncoderInitParams(¶ms)
|
|
params.quality = 10
|
|
params.lgwin = uint(lgwin)
|
|
if lgwin > BROTLI_MAX_WINDOW_BITS {
|
|
params.large_window = true
|
|
}
|
|
|
|
SanitizeParams(¶ms)
|
|
params.lgblock = ComputeLgBlock(¶ms)
|
|
ChooseDistanceParams(¶ms)
|
|
max_block_size = uint(1) << uint(params.lgblock)
|
|
|
|
assert(input_size <= mask+1)
|
|
EncodeWindowBits(lgwin, params.large_window, &last_bytes, &last_bytes_bits)
|
|
InitOrStitchToPreviousBlock(&hasher, input_buffer, mask, ¶ms, 0, hasher_eff_size, true)
|
|
|
|
for ok && metablock_start < input_size {
|
|
var metablock_end uint = brotli_min_size_t(input_size, metablock_start+max_metablock_size)
|
|
var expected_num_commands uint = (metablock_end-metablock_start)/12 + 16
|
|
var commands []Command = nil
|
|
var num_commands uint = 0
|
|
var last_insert_len uint = 0
|
|
var num_literals uint = 0
|
|
var metablock_size uint = 0
|
|
var cmd_alloc_size uint = 0
|
|
var is_last bool
|
|
var storage []byte
|
|
var storage_ix uint
|
|
var literal_context_mode int = ChooseContextMode(¶ms, input_buffer, metablock_start, mask, metablock_end-metablock_start)
|
|
var block_start uint
|
|
for block_start = metablock_start; block_start < metablock_end; {
|
|
var block_size uint = brotli_min_size_t(metablock_end-block_start, max_block_size)
|
|
var nodes []ZopfliNode = make([]ZopfliNode, (block_size + 1))
|
|
var path_size uint
|
|
var new_cmd_alloc_size uint
|
|
BrotliInitZopfliNodes(nodes, block_size+1)
|
|
hasher.StitchToPreviousBlock(block_size, block_start, input_buffer, mask)
|
|
path_size = BrotliZopfliComputeShortestPath(block_size, block_start, input_buffer, mask, ¶ms, dist_cache[:], hasher.(*H10), nodes)
|
|
|
|
/* We allocate a command buffer in the first iteration of this loop that
|
|
will be likely big enough for the whole metablock, so that for most
|
|
inputs we will not have to reallocate in later iterations. We do the
|
|
allocation here and not before the loop, because if the input is small,
|
|
this will be allocated after the Zopfli cost model is freed, so this
|
|
will not increase peak memory usage.
|
|
TODO: If the first allocation is too small, increase command
|
|
buffer size exponentially. */
|
|
new_cmd_alloc_size = brotli_max_size_t(expected_num_commands, num_commands+path_size+1)
|
|
|
|
if cmd_alloc_size != new_cmd_alloc_size {
|
|
var new_commands []Command = make([]Command, new_cmd_alloc_size)
|
|
cmd_alloc_size = new_cmd_alloc_size
|
|
if commands != nil {
|
|
copy(new_commands, commands[:num_commands])
|
|
commands = nil
|
|
}
|
|
|
|
commands = new_commands
|
|
}
|
|
|
|
BrotliZopfliCreateCommands(block_size, block_start, nodes[0:], dist_cache[:], &last_insert_len, ¶ms, commands[num_commands:], &num_literals)
|
|
num_commands += path_size
|
|
block_start += block_size
|
|
metablock_size += block_size
|
|
nodes = nil
|
|
if num_literals > max_literals_per_metablock || num_commands > max_commands_per_metablock {
|
|
break
|
|
}
|
|
}
|
|
|
|
if last_insert_len > 0 {
|
|
InitInsertCommand(&commands[num_commands], last_insert_len)
|
|
num_commands++
|
|
num_literals += last_insert_len
|
|
}
|
|
|
|
is_last = (metablock_start+metablock_size == input_size)
|
|
storage = nil
|
|
storage_ix = uint(last_bytes_bits)
|
|
|
|
if metablock_size == 0 {
|
|
/* Write the ISLAST and ISEMPTY bits. */
|
|
storage = make([]byte, 16)
|
|
|
|
storage[0] = byte(last_bytes)
|
|
storage[1] = byte(last_bytes >> 8)
|
|
BrotliWriteBits(2, 3, &storage_ix, storage)
|
|
storage_ix = (storage_ix + 7) &^ 7
|
|
} else if !ShouldCompress_encode(input_buffer, mask, uint64(metablock_start), metablock_size, num_literals, num_commands) {
|
|
/* Restore the distance cache, as its last update by
|
|
CreateBackwardReferences is now unused. */
|
|
copy(dist_cache[:], saved_dist_cache[:4])
|
|
|
|
storage = make([]byte, (metablock_size + 16))
|
|
storage[0] = byte(last_bytes)
|
|
storage[1] = byte(last_bytes >> 8)
|
|
BrotliStoreUncompressedMetaBlock(is_last, input_buffer, metablock_start, mask, metablock_size, &storage_ix, storage)
|
|
} else {
|
|
var mb MetaBlockSplit
|
|
var block_params BrotliEncoderParams = params
|
|
InitMetaBlockSplit(&mb)
|
|
BrotliBuildMetaBlock(input_buffer, metablock_start, mask, &block_params, prev_byte, prev_byte2, commands, num_commands, literal_context_mode, &mb)
|
|
{
|
|
/* The number of distance symbols effectively used for distance
|
|
histograms. It might be less than distance alphabet size
|
|
for "Large Window Brotli" (32-bit). */
|
|
var num_effective_dist_codes uint32 = block_params.dist.alphabet_size
|
|
if num_effective_dist_codes > BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS {
|
|
num_effective_dist_codes = BROTLI_NUM_HISTOGRAM_DISTANCE_SYMBOLS
|
|
}
|
|
|
|
BrotliOptimizeHistograms(num_effective_dist_codes, &mb)
|
|
}
|
|
|
|
storage = make([]byte, (2*metablock_size + 503))
|
|
storage[0] = byte(last_bytes)
|
|
storage[1] = byte(last_bytes >> 8)
|
|
BrotliStoreMetaBlock(input_buffer, metablock_start, metablock_size, mask, prev_byte, prev_byte2, is_last, &block_params, literal_context_mode, commands, num_commands, &mb, &storage_ix, storage)
|
|
if metablock_size+4 < storage_ix>>3 {
|
|
/* Restore the distance cache and last byte. */
|
|
copy(dist_cache[:], saved_dist_cache[:4])
|
|
|
|
storage[0] = byte(last_bytes)
|
|
storage[1] = byte(last_bytes >> 8)
|
|
storage_ix = uint(last_bytes_bits)
|
|
BrotliStoreUncompressedMetaBlock(is_last, input_buffer, metablock_start, mask, metablock_size, &storage_ix, storage)
|
|
}
|
|
|
|
DestroyMetaBlockSplit(&mb)
|
|
}
|
|
|
|
last_bytes = uint16(storage[storage_ix>>3])
|
|
last_bytes_bits = byte(storage_ix & 7)
|
|
metablock_start += metablock_size
|
|
if metablock_start < input_size {
|
|
prev_byte = input_buffer[metablock_start-1]
|
|
prev_byte2 = input_buffer[metablock_start-2]
|
|
}
|
|
|
|
/* Save the state of the distance cache in case we need to restore it for
|
|
emitting an uncompressed block. */
|
|
copy(saved_dist_cache[:], dist_cache[:4])
|
|
{
|
|
var out_size uint = storage_ix >> 3
|
|
total_out_size += out_size
|
|
if total_out_size <= max_out_size {
|
|
copy(encoded_buffer, storage[:out_size])
|
|
encoded_buffer = encoded_buffer[out_size:]
|
|
} else {
|
|
ok = false
|
|
}
|
|
}
|
|
|
|
storage = nil
|
|
commands = nil
|
|
}
|
|
|
|
*encoded_size = total_out_size
|
|
DestroyHasher(&hasher)
|
|
return ok
|
|
}
|
|
|
|
func BrotliEncoderMaxCompressedSize(input_size uint) uint {
|
|
var num_large_blocks uint = input_size >> 14
|
|
var overhead uint = 2 + (4 * num_large_blocks) + 3 + 1
|
|
/* [window bits / empty metadata] + N * [uncompressed] + [last empty] */
|
|
|
|
var result uint = input_size + overhead
|
|
if input_size == 0 {
|
|
return 2
|
|
}
|
|
if result < input_size {
|
|
return 0
|
|
} else {
|
|
return result
|
|
}
|
|
}
|
|
|
|
/* Wraps data to uncompressed brotli stream with minimal window size.
|
|
|output| should point at region with at least BrotliEncoderMaxCompressedSize
|
|
addressable bytes.
|
|
Returns the length of stream. */
|
|
func MakeUncompressedStream(input []byte, input_size uint, output []byte) uint {
|
|
var size uint = input_size
|
|
var result uint = 0
|
|
var offset uint = 0
|
|
if input_size == 0 {
|
|
output[0] = 6
|
|
return 1
|
|
}
|
|
|
|
output[result] = 0x21
|
|
result++ /* window bits = 10, is_last = false */
|
|
output[result] = 0x03
|
|
result++ /* empty metadata, padding */
|
|
for size > 0 {
|
|
var nibbles uint32 = 0
|
|
var chunk_size uint32
|
|
var bits uint32
|
|
if size > 1<<24 {
|
|
chunk_size = 1 << 24
|
|
} else {
|
|
chunk_size = uint32(size)
|
|
}
|
|
if chunk_size > 1<<16 {
|
|
if chunk_size > 1<<20 {
|
|
nibbles = 2
|
|
} else {
|
|
nibbles = 1
|
|
}
|
|
}
|
|
bits = nibbles<<1 | (chunk_size-1)<<3 | 1<<(19+4*nibbles)
|
|
output[result] = byte(bits)
|
|
result++
|
|
output[result] = byte(bits >> 8)
|
|
result++
|
|
output[result] = byte(bits >> 16)
|
|
result++
|
|
if nibbles == 2 {
|
|
output[result] = byte(bits >> 24)
|
|
result++
|
|
}
|
|
copy(output[result:], input[offset:][:chunk_size])
|
|
result += uint(chunk_size)
|
|
offset += uint(chunk_size)
|
|
size -= uint(chunk_size)
|
|
}
|
|
|
|
output[result] = 3
|
|
result++
|
|
return result
|
|
}
|
|
|
|
func BrotliEncoderCompress(quality int, lgwin int, mode int, input_size uint, input_buffer []byte, encoded_size *uint, encoded_buffer []byte) bool {
|
|
var s *Writer
|
|
var out_size uint = *encoded_size
|
|
var input_start []byte = input_buffer
|
|
var output_start []byte = encoded_buffer
|
|
var max_out_size uint = BrotliEncoderMaxCompressedSize(input_size)
|
|
if out_size == 0 {
|
|
/* Output buffer needs at least one byte. */
|
|
return false
|
|
}
|
|
|
|
if input_size == 0 {
|
|
/* Handle the special case of empty input. */
|
|
*encoded_size = 1
|
|
|
|
encoded_buffer[0] = 6
|
|
return true
|
|
}
|
|
|
|
if quality == 10 {
|
|
var lg_win int = brotli_min_int(BROTLI_LARGE_MAX_WINDOW_BITS, brotli_max_int(16, lgwin))
|
|
/* TODO: Implement this direct path for all quality levels. */
|
|
|
|
var ok bool = BrotliCompressBufferQuality10(lg_win, input_size, input_buffer, encoded_size, encoded_buffer)
|
|
if !ok || (max_out_size != 0 && *encoded_size > max_out_size) {
|
|
goto fallback
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
s = BrotliEncoderCreateInstance()
|
|
if s == nil {
|
|
return false
|
|
} else {
|
|
var available_in uint = input_size
|
|
var next_in []byte = input_buffer
|
|
var available_out uint = *encoded_size
|
|
var next_out []byte = encoded_buffer
|
|
var total_out uint = 0
|
|
var result bool = false
|
|
BrotliEncoderSetParameter(s, BROTLI_PARAM_QUALITY, uint32(quality))
|
|
BrotliEncoderSetParameter(s, BROTLI_PARAM_LGWIN, uint32(lgwin))
|
|
BrotliEncoderSetParameter(s, BROTLI_PARAM_MODE, uint32(mode))
|
|
BrotliEncoderSetParameter(s, BROTLI_PARAM_SIZE_HINT, uint32(input_size))
|
|
if lgwin > BROTLI_MAX_WINDOW_BITS {
|
|
BrotliEncoderSetParameter(s, BROTLI_PARAM_LARGE_WINDOW, 1)
|
|
}
|
|
|
|
result = BrotliEncoderCompressStream(s, int(BROTLI_OPERATION_FINISH), &available_in, &next_in, &available_out, &next_out, &total_out)
|
|
if !BrotliEncoderIsFinished(s) {
|
|
result = false
|
|
}
|
|
*encoded_size = total_out
|
|
BrotliEncoderDestroyInstance(s)
|
|
if !result || (max_out_size != 0 && *encoded_size > max_out_size) {
|
|
goto fallback
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
fallback:
|
|
*encoded_size = 0
|
|
if max_out_size == 0 {
|
|
return false
|
|
}
|
|
if out_size >= max_out_size {
|
|
*encoded_size = MakeUncompressedStream(input_start, input_size, output_start)
|
|
return true
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
func InjectBytePaddingBlock(s *Writer) {
|
|
var seal uint32 = uint32(s.last_bytes_)
|
|
var seal_bits uint = uint(s.last_bytes_bits_)
|
|
var destination []byte
|
|
s.last_bytes_ = 0
|
|
s.last_bytes_bits_ = 0
|
|
|
|
/* is_last = 0, data_nibbles = 11, reserved = 0, meta_nibbles = 00 */
|
|
seal |= 0x6 << seal_bits
|
|
|
|
seal_bits += 6
|
|
|
|
/* If we have already created storage, then append to it.
|
|
Storage is valid until next block is being compressed. */
|
|
if s.next_out_ != nil {
|
|
destination = s.next_out_[s.available_out_:]
|
|
} else {
|
|
destination = s.tiny_buf_.u8[:]
|
|
s.next_out_ = destination
|
|
}
|
|
|
|
destination[0] = byte(seal)
|
|
if seal_bits > 8 {
|
|
destination[1] = byte(seal >> 8)
|
|
}
|
|
if seal_bits > 16 {
|
|
destination[2] = byte(seal >> 16)
|
|
}
|
|
s.available_out_ += (seal_bits + 7) >> 3
|
|
}
|
|
|
|
/* Injects padding bits or pushes compressed data to output.
|
|
Returns false if nothing is done. */
|
|
func InjectFlushOrPushOutput(s *Writer, available_out *uint, next_out *[]byte, total_out *uint) bool {
|
|
if s.stream_state_ == BROTLI_STREAM_FLUSH_REQUESTED && s.last_bytes_bits_ != 0 {
|
|
InjectBytePaddingBlock(s)
|
|
return true
|
|
}
|
|
|
|
if s.available_out_ != 0 && *available_out != 0 {
|
|
var copy_output_size uint = brotli_min_size_t(s.available_out_, *available_out)
|
|
copy(*next_out, s.next_out_[:copy_output_size])
|
|
*next_out = (*next_out)[copy_output_size:]
|
|
*available_out -= copy_output_size
|
|
s.next_out_ = s.next_out_[copy_output_size:]
|
|
s.available_out_ -= copy_output_size
|
|
s.total_out_ += copy_output_size
|
|
if total_out != nil {
|
|
*total_out = s.total_out_
|
|
}
|
|
return true
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
func CheckFlushComplete(s *Writer) {
|
|
if s.stream_state_ == BROTLI_STREAM_FLUSH_REQUESTED && s.available_out_ == 0 {
|
|
s.stream_state_ = BROTLI_STREAM_PROCESSING
|
|
s.next_out_ = nil
|
|
}
|
|
}
|
|
|
|
func BrotliEncoderCompressStreamFast(s *Writer, op int, available_in *uint, next_in *[]byte, available_out *uint, next_out *[]byte, total_out *uint) bool {
|
|
var block_size_limit uint = uint(1) << s.params.lgwin
|
|
var buf_size uint = brotli_min_size_t(kCompressFragmentTwoPassBlockSize, brotli_min_size_t(*available_in, block_size_limit))
|
|
var tmp_command_buf []uint32 = nil
|
|
var command_buf []uint32 = nil
|
|
var tmp_literal_buf []byte = nil
|
|
var literal_buf []byte = nil
|
|
if s.params.quality != FAST_ONE_PASS_COMPRESSION_QUALITY && s.params.quality != FAST_TWO_PASS_COMPRESSION_QUALITY {
|
|
return false
|
|
}
|
|
|
|
if s.params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY {
|
|
if s.command_buf_ == nil && buf_size == kCompressFragmentTwoPassBlockSize {
|
|
s.command_buf_ = make([]uint32, kCompressFragmentTwoPassBlockSize)
|
|
s.literal_buf_ = make([]byte, kCompressFragmentTwoPassBlockSize)
|
|
}
|
|
|
|
if s.command_buf_ != nil {
|
|
command_buf = s.command_buf_
|
|
literal_buf = s.literal_buf_
|
|
} else {
|
|
tmp_command_buf = make([]uint32, buf_size)
|
|
tmp_literal_buf = make([]byte, buf_size)
|
|
command_buf = tmp_command_buf
|
|
literal_buf = tmp_literal_buf
|
|
}
|
|
}
|
|
|
|
for {
|
|
if InjectFlushOrPushOutput(s, available_out, next_out, total_out) {
|
|
continue
|
|
}
|
|
|
|
/* Compress block only when internal output buffer is empty, stream is not
|
|
finished, there is no pending flush request, and there is either
|
|
additional input or pending operation. */
|
|
if s.available_out_ == 0 && s.stream_state_ == BROTLI_STREAM_PROCESSING && (*available_in != 0 || op != int(BROTLI_OPERATION_PROCESS)) {
|
|
var block_size uint = brotli_min_size_t(block_size_limit, *available_in)
|
|
var is_last bool = (*available_in == block_size) && (op == int(BROTLI_OPERATION_FINISH))
|
|
var force_flush bool = (*available_in == block_size) && (op == int(BROTLI_OPERATION_FLUSH))
|
|
var max_out_size uint = 2*block_size + 503
|
|
var inplace bool = true
|
|
var storage []byte = nil
|
|
var storage_ix uint = uint(s.last_bytes_bits_)
|
|
var table_size uint
|
|
var table []int
|
|
|
|
if force_flush && block_size == 0 {
|
|
s.stream_state_ = BROTLI_STREAM_FLUSH_REQUESTED
|
|
continue
|
|
}
|
|
|
|
if max_out_size <= *available_out {
|
|
storage = *next_out
|
|
} else {
|
|
inplace = false
|
|
storage = GetBrotliStorage(s, max_out_size)
|
|
}
|
|
|
|
storage[0] = byte(s.last_bytes_)
|
|
storage[1] = byte(s.last_bytes_ >> 8)
|
|
table = GetHashTable(s, s.params.quality, block_size, &table_size)
|
|
|
|
if s.params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY {
|
|
BrotliCompressFragmentFast(*next_in, block_size, is_last, table, table_size, s.cmd_depths_[:], s.cmd_bits_[:], &s.cmd_code_numbits_, s.cmd_code_[:], &storage_ix, storage)
|
|
} else {
|
|
BrotliCompressFragmentTwoPass(*next_in, block_size, is_last, command_buf, literal_buf, table, table_size, &storage_ix, storage)
|
|
}
|
|
|
|
*next_in = (*next_in)[block_size:]
|
|
*available_in -= block_size
|
|
if inplace {
|
|
var out_bytes uint = storage_ix >> 3
|
|
assert(out_bytes <= *available_out)
|
|
assert(storage_ix&7 == 0 || out_bytes < *available_out)
|
|
*next_out = (*next_out)[out_bytes:]
|
|
*available_out -= out_bytes
|
|
s.total_out_ += out_bytes
|
|
if total_out != nil {
|
|
*total_out = s.total_out_
|
|
}
|
|
} else {
|
|
var out_bytes uint = storage_ix >> 3
|
|
s.next_out_ = storage
|
|
s.available_out_ = out_bytes
|
|
}
|
|
|
|
s.last_bytes_ = uint16(storage[storage_ix>>3])
|
|
s.last_bytes_bits_ = byte(storage_ix & 7)
|
|
|
|
if force_flush {
|
|
s.stream_state_ = BROTLI_STREAM_FLUSH_REQUESTED
|
|
}
|
|
if is_last {
|
|
s.stream_state_ = BROTLI_STREAM_FINISHED
|
|
}
|
|
continue
|
|
}
|
|
|
|
break
|
|
}
|
|
|
|
tmp_command_buf = nil
|
|
tmp_literal_buf = nil
|
|
CheckFlushComplete(s)
|
|
return true
|
|
}
|
|
|
|
func ProcessMetadata(s *Writer, available_in *uint, next_in *[]byte, available_out *uint, next_out *[]byte, total_out *uint) bool {
|
|
if *available_in > 1<<24 {
|
|
return false
|
|
}
|
|
|
|
/* Switch to metadata block workflow, if required. */
|
|
if s.stream_state_ == BROTLI_STREAM_PROCESSING {
|
|
s.remaining_metadata_bytes_ = uint32(*available_in)
|
|
s.stream_state_ = BROTLI_STREAM_METADATA_HEAD
|
|
}
|
|
|
|
if s.stream_state_ != BROTLI_STREAM_METADATA_HEAD && s.stream_state_ != BROTLI_STREAM_METADATA_BODY {
|
|
return false
|
|
}
|
|
|
|
for {
|
|
if InjectFlushOrPushOutput(s, available_out, next_out, total_out) {
|
|
continue
|
|
}
|
|
|
|
if s.available_out_ != 0 {
|
|
break
|
|
}
|
|
|
|
if s.input_pos_ != s.last_flush_pos_ {
|
|
var result bool = EncodeData(s, false, true, &s.available_out_, &s.next_out_)
|
|
if !result {
|
|
return false
|
|
}
|
|
continue
|
|
}
|
|
|
|
if s.stream_state_ == BROTLI_STREAM_METADATA_HEAD {
|
|
s.next_out_ = s.tiny_buf_.u8[:]
|
|
s.available_out_ = WriteMetadataHeader(s, uint(s.remaining_metadata_bytes_), s.next_out_)
|
|
s.stream_state_ = BROTLI_STREAM_METADATA_BODY
|
|
continue
|
|
} else {
|
|
/* Exit workflow only when there is no more input and no more output.
|
|
Otherwise client may continue producing empty metadata blocks. */
|
|
if s.remaining_metadata_bytes_ == 0 {
|
|
s.remaining_metadata_bytes_ = BROTLI_UINT32_MAX
|
|
s.stream_state_ = BROTLI_STREAM_PROCESSING
|
|
break
|
|
}
|
|
|
|
if *available_out != 0 {
|
|
/* Directly copy input to output. */
|
|
var c uint32 = uint32(brotli_min_size_t(uint(s.remaining_metadata_bytes_), *available_out))
|
|
copy(*next_out, (*next_in)[:c])
|
|
*next_in = (*next_in)[c:]
|
|
*available_in -= uint(c)
|
|
s.remaining_metadata_bytes_ -= c
|
|
*next_out = (*next_out)[c:]
|
|
*available_out -= uint(c)
|
|
} else {
|
|
/* This guarantees progress in "TakeOutput" workflow. */
|
|
var c uint32 = brotli_min_uint32_t(s.remaining_metadata_bytes_, 16)
|
|
s.next_out_ = s.tiny_buf_.u8[:]
|
|
copy(s.next_out_, (*next_in)[:c])
|
|
*next_in = (*next_in)[c:]
|
|
*available_in -= uint(c)
|
|
s.remaining_metadata_bytes_ -= c
|
|
s.available_out_ = uint(c)
|
|
}
|
|
|
|
continue
|
|
}
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
func UpdateSizeHint(s *Writer, available_in uint) {
|
|
if s.params.size_hint == 0 {
|
|
var delta uint64 = UnprocessedInputSize(s)
|
|
var tail uint64 = uint64(available_in)
|
|
var limit uint32 = 1 << 30
|
|
var total uint32
|
|
if (delta >= uint64(limit)) || (tail >= uint64(limit)) || ((delta + tail) >= uint64(limit)) {
|
|
total = limit
|
|
} else {
|
|
total = uint32(delta + tail)
|
|
}
|
|
|
|
s.params.size_hint = uint(total)
|
|
}
|
|
}
|
|
|
|
func BrotliEncoderCompressStream(s *Writer, op int, available_in *uint, next_in *[]byte, available_out *uint, next_out *[]byte, total_out *uint) bool {
|
|
if !EnsureInitialized(s) {
|
|
return false
|
|
}
|
|
|
|
/* Unfinished metadata block; check requirements. */
|
|
if s.remaining_metadata_bytes_ != BROTLI_UINT32_MAX {
|
|
if uint32(*available_in) != s.remaining_metadata_bytes_ {
|
|
return false
|
|
}
|
|
if op != int(BROTLI_OPERATION_EMIT_METADATA) {
|
|
return false
|
|
}
|
|
}
|
|
|
|
if op == int(BROTLI_OPERATION_EMIT_METADATA) {
|
|
UpdateSizeHint(s, 0) /* First data metablock might be emitted here. */
|
|
return ProcessMetadata(s, available_in, next_in, available_out, next_out, total_out)
|
|
}
|
|
|
|
if s.stream_state_ == BROTLI_STREAM_METADATA_HEAD || s.stream_state_ == BROTLI_STREAM_METADATA_BODY {
|
|
return false
|
|
}
|
|
|
|
if s.stream_state_ != BROTLI_STREAM_PROCESSING && *available_in != 0 {
|
|
return false
|
|
}
|
|
|
|
if s.params.quality == FAST_ONE_PASS_COMPRESSION_QUALITY || s.params.quality == FAST_TWO_PASS_COMPRESSION_QUALITY {
|
|
return BrotliEncoderCompressStreamFast(s, op, available_in, next_in, available_out, next_out, total_out)
|
|
}
|
|
|
|
for {
|
|
var remaining_block_size uint = RemainingInputBlockSize(s)
|
|
|
|
if remaining_block_size != 0 && *available_in != 0 {
|
|
var copy_input_size uint = brotli_min_size_t(remaining_block_size, *available_in)
|
|
CopyInputToRingBuffer(s, copy_input_size, *next_in)
|
|
*next_in = (*next_in)[copy_input_size:]
|
|
*available_in -= copy_input_size
|
|
continue
|
|
}
|
|
|
|
if InjectFlushOrPushOutput(s, available_out, next_out, total_out) {
|
|
continue
|
|
}
|
|
|
|
/* Compress data only when internal output buffer is empty, stream is not
|
|
finished and there is no pending flush request. */
|
|
if s.available_out_ == 0 && s.stream_state_ == BROTLI_STREAM_PROCESSING {
|
|
if remaining_block_size == 0 || op != int(BROTLI_OPERATION_PROCESS) {
|
|
var is_last bool = ((*available_in == 0) && op == int(BROTLI_OPERATION_FINISH))
|
|
var force_flush bool = ((*available_in == 0) && op == int(BROTLI_OPERATION_FLUSH))
|
|
var result bool
|
|
UpdateSizeHint(s, *available_in)
|
|
result = EncodeData(s, is_last, force_flush, &s.available_out_, &s.next_out_)
|
|
if !result {
|
|
return false
|
|
}
|
|
if force_flush {
|
|
s.stream_state_ = BROTLI_STREAM_FLUSH_REQUESTED
|
|
}
|
|
if is_last {
|
|
s.stream_state_ = BROTLI_STREAM_FINISHED
|
|
}
|
|
continue
|
|
}
|
|
}
|
|
|
|
break
|
|
}
|
|
|
|
CheckFlushComplete(s)
|
|
return true
|
|
}
|
|
|
|
func BrotliEncoderIsFinished(s *Writer) bool {
|
|
return s.stream_state_ == BROTLI_STREAM_FINISHED && !BrotliEncoderHasMoreOutput(s)
|
|
}
|
|
|
|
func BrotliEncoderHasMoreOutput(s *Writer) bool {
|
|
return s.available_out_ != 0
|
|
}
|
|
|
|
func BrotliEncoderTakeOutput(s *Writer, size *uint) []byte {
|
|
var consumed_size uint = s.available_out_
|
|
var result []byte = s.next_out_
|
|
if *size != 0 {
|
|
consumed_size = brotli_min_size_t(*size, s.available_out_)
|
|
}
|
|
|
|
if consumed_size != 0 {
|
|
s.next_out_ = s.next_out_[consumed_size:]
|
|
s.available_out_ -= consumed_size
|
|
s.total_out_ += consumed_size
|
|
CheckFlushComplete(s)
|
|
*size = consumed_size
|
|
} else {
|
|
*size = 0
|
|
result = nil
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
func BrotliEncoderVersion() uint32 {
|
|
return BROTLI_VERSION
|
|
}
|