re
/
brotli
forked from mirror/brotli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make BrotliDecoderState into a Reader.

This commit is contained in:
Andy Balholm 2019-03-06 17:08:24 -08:00
parent f036cd04c1
commit 0da14edee1
4 changed files with 556 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

396
brotli_test.go Normal file
View File

@ -0,0 +1,396 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Distributed under MIT license.
// See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
package brotli
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"math"
"math/rand"
"testing"
"time"
)
func checkCompressedData(compressedData, wantOriginalData []byte) error {
uncompressed, err := Decode(compressedData)
if err != nil {
return fmt.Errorf("brotli decompress failed: %v", err)
}
if !bytes.Equal(uncompressed, wantOriginalData) {
if len(wantOriginalData) != len(uncompressed) {
return fmt.Errorf(""+
"Data doesn't uncompress to the original value.\n"+
"Length of original: %v\n"+
"Length of uncompressed: %v",
len(wantOriginalData), len(uncompressed))
}
for i := range wantOriginalData {
if wantOriginalData[i] != uncompressed[i] {
return fmt.Errorf(""+
"Data doesn't uncompress to the original value.\n"+
"Original at %v is %v\n"+
"Uncompressed at %v is %v",
i, wantOriginalData[i], i, uncompressed[i])
}
}
}
return nil
}
func TestEncoderNoWrite(t *testing.T) {
out := bytes.Buffer{}
e := NewWriter(&out, WriterOptions{Quality: 5})
if err := e.Close(); err != nil {
t.Errorf("Close()=%v, want nil", err)
}
// Check Write after close.
if _, err := e.Write([]byte("hi")); err == nil {
t.Errorf("No error after Close() + Write()")
}
}
func TestEncoderEmptyWrite(t *testing.T) {
out := bytes.Buffer{}
e := NewWriter(&out, WriterOptions{Quality: 5})
n, err := e.Write([]byte(""))
if n != 0 || err != nil {
t.Errorf("Write()=%v,%v, want 0, nil", n, err)
}
if err := e.Close(); err != nil {
t.Errorf("Close()=%v, want nil", err)
}
}
func TestWriter(t *testing.T) {
// Test basic encoder usage.
input := []byte("<html><body><H1>Hello world</H1></body></html>")
out := bytes.Buffer{}
e := NewWriter(&out, WriterOptions{Quality: 1})
in := bytes.NewReader([]byte(input))
n, err := io.Copy(e, in)
if err != nil {
t.Errorf("Copy Error: %v", err)
}
if int(n) != len(input) {
t.Errorf("Copy() n=%v, want %v", n, len(input))
}
if err := e.Close(); err != nil {
t.Errorf("Close Error after copied %d bytes: %v", n, err)
}
if err := checkCompressedData(out.Bytes(), input); err != nil {
t.Error(err)
}
}
func TestEncoderStreams(t *testing.T) {
// Test that output is streamed.
// Adjust window size to ensure the encoder outputs at least enough bytes
// to fill the window.
const lgWin = 16
windowSize := int(math.Pow(2, lgWin))
input := make([]byte, 8*windowSize)
rand.Read(input)
out := bytes.Buffer{}
e := NewWriter(&out, WriterOptions{Quality: 11, LGWin: lgWin})
halfInput := input[:len(input)/2]
in := bytes.NewReader(halfInput)
n, err := io.Copy(e, in)
if err != nil {
t.Errorf("Copy Error: %v", err)
}
// We've fed more data than the sliding window size. Check that some
// compressed data has been output.
if out.Len() == 0 {
t.Errorf("Output length is 0 after %d bytes written", n)
}
if err := e.Close(); err != nil {
t.Errorf("Close Error after copied %d bytes: %v", n, err)
}
if err := checkCompressedData(out.Bytes(), halfInput); err != nil {
t.Error(err)
}
}
func TestEncoderLargeInput(t *testing.T) {
input := make([]byte, 1000000)
rand.Read(input)
out := bytes.Buffer{}
e := NewWriter(&out, WriterOptions{Quality: 5})
in := bytes.NewReader(input)
n, err := io.Copy(e, in)
if err != nil {
t.Errorf("Copy Error: %v", err)
}
if int(n) != len(input) {
t.Errorf("Copy() n=%v, want %v", n, len(input))
}
if err := e.Close(); err != nil {
t.Errorf("Close Error after copied %d bytes: %v", n, err)
}
if err := checkCompressedData(out.Bytes(), input); err != nil {
t.Error(err)
}
}
func TestEncoderFlush(t *testing.T) {
input := make([]byte, 1000)
rand.Read(input)
out := bytes.Buffer{}
e := NewWriter(&out, WriterOptions{Quality: 5})
in := bytes.NewReader(input)
_, err := io.Copy(e, in)
if err != nil {
t.Fatalf("Copy Error: %v", err)
}
if err := e.Flush(); err != nil {
t.Fatalf("Flush(): %v", err)
}
if out.Len() == 0 {
t.Fatalf("0 bytes written after Flush()")
}
decompressed := make([]byte, 1000)
reader := NewReader(bytes.NewReader(out.Bytes()))
n, err := reader.Read(decompressed)
if n != len(decompressed) || err != nil {
t.Errorf("Expected <%v, nil>, but <%v, %v>", len(decompressed), n, err)
}
if !bytes.Equal(decompressed, input) {
t.Errorf(""+
"Decompress after flush: %v\n"+
"%q\n"+
"want:\n%q",
err, decompressed, input)
}
if err := e.Close(); err != nil {
t.Errorf("Close(): %v", err)
}
}
type readerWithTimeout struct {
io.Reader
}
func (r readerWithTimeout) Read(p []byte) (int, error) {
type result struct {
n int
err error
}
ch := make(chan result)
go func() {
n, err := r.Reader.Read(p)
ch <- result{n, err}
}()
select {
case result := <-ch:
return result.n, result.err
case <-time.After(5 * time.Second):
return 0, fmt.Errorf("read timed out")
}
}
func TestDecoderStreaming(t *testing.T) {
pr, pw := io.Pipe()
writer := NewWriter(pw, WriterOptions{Quality: 5, LGWin: 20})
reader := readerWithTimeout{NewReader(pr)}
defer func() {
go ioutil.ReadAll(pr) // swallow the "EOF" token from writer.Close
if err := writer.Close(); err != nil {
t.Errorf("writer.Close: %v", err)
}
}()
ch := make(chan []byte)
errch := make(chan error)
go func() {
for {
segment, ok := <-ch
if !ok {
return
}
if n, err := writer.Write(segment); err != nil || n != len(segment) {
errch <- fmt.Errorf("write=%v,%v, want %v,%v", n, err, len(segment), nil)
return
}
if err := writer.Flush(); err != nil {
errch <- fmt.Errorf("flush: %v", err)
return
}
}
}()
defer close(ch)
segments := [...][]byte{
[]byte("first"),
[]byte("second"),
[]byte("third"),
}
for k, segment := range segments {
t.Run(fmt.Sprintf("Segment%d", k), func(t *testing.T) {
select {
case ch <- segment:
case err := <-errch:
t.Fatalf("write: %v", err)
case <-time.After(5 * time.Second):
t.Fatalf("timed out")
}
wantLen := len(segment)
got := make([]byte, wantLen)
if n, err := reader.Read(got); err != nil || n != wantLen || !bytes.Equal(got, segment) {
t.Fatalf("read[%d]=%q,%v,%v, want %q,%v,%v", k, got, n, err, segment, wantLen, nil)
}
})
}
}
func TestReader(t *testing.T) {
content := bytes.Repeat([]byte("hello world!"), 10000)
encoded, _ := Encode(content, WriterOptions{Quality: 5})
r := NewReader(bytes.NewReader(encoded))
var decodedOutput bytes.Buffer
n, err := io.Copy(&decodedOutput, r)
if err != nil {
t.Fatalf("Copy(): n=%v, err=%v", n, err)
}
if got := decodedOutput.Bytes(); !bytes.Equal(got, content) {
t.Errorf(""+
"Reader output:\n"+
"%q\n"+
"want:\n"+
"<%d bytes>",
got, len(content))
}
}
func TestDecode(t *testing.T) {
content := bytes.Repeat([]byte("hello world!"), 10000)
encoded, _ := Encode(content, WriterOptions{Quality: 5})
decoded, err := Decode(encoded)
if err != nil {
t.Errorf("Decode: %v", err)
}
if !bytes.Equal(decoded, content) {
t.Errorf(""+
"Decode content:\n"+
"%q\n"+
"want:\n"+
"<%d bytes>",
decoded, len(content))
}
}
func TestQuality(t *testing.T) {
content := bytes.Repeat([]byte("hello world!"), 10000)
for q := 0; q < 12; q++ {
encoded, _ := Encode(content, WriterOptions{Quality: q})
decoded, err := Decode(encoded)
if err != nil {
t.Errorf("Decode: %v", err)
}
if !bytes.Equal(decoded, content) {
t.Errorf(""+
"Decode content:\n"+
"%q\n"+
"want:\n"+
"<%d bytes>",
decoded, len(content))
}
}
}
func TestDecodeFuzz(t *testing.T) {
// Test that the decoder terminates with corrupted input.
content := bytes.Repeat([]byte("hello world!"), 100)
src := rand.NewSource(0)
encoded, err := Encode(content, WriterOptions{Quality: 5})
if err != nil {
t.Fatalf("Encode(<%d bytes>, _) = _, %s", len(content), err)
}
if len(encoded) == 0 {
t.Fatalf("Encode(<%d bytes>, _) produced empty output", len(content))
}
for i := 0; i < 100; i++ {
enc := append([]byte{}, encoded...)
for j := 0; j < 5; j++ {
enc[int(src.Int63())%len(enc)] = byte(src.Int63() % 256)
}
Decode(enc)
}
}
func TestDecodeTrailingData(t *testing.T) {
content := bytes.Repeat([]byte("hello world!"), 100)
encoded, _ := Encode(content, WriterOptions{Quality: 5})
_, err := Decode(append(encoded, 0))
if err == nil {
t.Errorf("Expected 'excessive input' error")
}
}
func TestEncodeDecode(t *testing.T) {
for _, test := range []struct {
data []byte
repeats int
}{
{nil, 0},
{[]byte("A"), 1},
{[]byte("<html><body><H1>Hello world</H1></body></html>"), 10},
{[]byte("<html><body><H1>Hello world</H1></body></html>"), 1000},
} {
t.Logf("case %q x %d", test.data, test.repeats)
input := bytes.Repeat(test.data, test.repeats)
encoded, err := Encode(input, WriterOptions{Quality: 5})
if err != nil {
t.Errorf("Encode: %v", err)
}
// Inputs are compressible, but may be too small to compress.
if maxSize := len(input)/2 + 20; len(encoded) >= maxSize {
t.Errorf(""+
"Encode returned %d bytes, want <%d\n"+
"Encoded=%q",
len(encoded), maxSize, encoded)
}
decoded, err := Decode(encoded)
if err != nil {
t.Errorf("Decode: %v", err)
}
if !bytes.Equal(decoded, input) {
var want string
if len(input) > 320 {
want = fmt.Sprintf("<%d bytes>", len(input))
} else {
want = fmt.Sprintf("%q", input)
}
t.Errorf(""+
"Decode content:\n"+
"%q\n"+
"want:\n"+
"%s",
decoded, want)
}
}
}
// Encode returns content encoded with Brotli.
func Encode(content []byte, options WriterOptions) ([]byte, error) {
var buf bytes.Buffer
writer := NewWriter(&buf, options)
_, err := writer.Write(content)
if closeErr := writer.Close(); err == nil {
err = closeErr
}
return buf.Bytes(), err
}
// Decode decodes Brotli encoded data.
func Decode(encodedData []byte) ([]byte, error) {
r := NewReader(bytes.NewReader(encodedData))
return ioutil.ReadAll(r)
}

108
decode.go
View File

@ -114,7 +114,7 @@ var kCodeLengthPrefixLength = [16]byte{2, 2, 2, 3, 2, 2, 2, 4, 2, 2, 2, 3, 2, 2,
var kCodeLengthPrefixValue = [16]byte{0, 4, 3, 2, 0, 4, 3, 1, 0, 4, 3, 2, 0, 4, 3, 5} var kCodeLengthPrefixValue = [16]byte{0, 4, 3, 2, 0, 4, 3, 1, 0, 4, 3, 2, 0, 4, 3, 5}
func BrotliDecoderSetParameter(state *BrotliDecoderState, p int, value uint32) bool { func BrotliDecoderSetParameter(state *Reader, p int, value uint32) bool {
if state.state != BROTLI_STATE_UNINITED { if state.state != BROTLI_STATE_UNINITED {
return false return false
} }
@ -136,9 +136,9 @@ func BrotliDecoderSetParameter(state *BrotliDecoderState, p int, value uint32) b
} }
} }
func BrotliDecoderCreateInstance() *BrotliDecoderState { func BrotliDecoderCreateInstance() *Reader {
var state *BrotliDecoderState var state *Reader
state = new(BrotliDecoderState) state = new(Reader)
if state == nil { if state == nil {
return nil return nil
} }
@ -151,7 +151,7 @@ func BrotliDecoderCreateInstance() *BrotliDecoderState {
} }
/* Deinitializes and frees BrotliDecoderState instance. */ /* Deinitializes and frees BrotliDecoderState instance. */
func BrotliDecoderDestroyInstance(state *BrotliDecoderState) { func BrotliDecoderDestroyInstance(state *Reader) {
if state == nil { if state == nil {
return return
} else { } else {
@ -160,7 +160,7 @@ func BrotliDecoderDestroyInstance(state *BrotliDecoderState) {
} }
/* Saves error code and converts it to BrotliDecoderResult. */ /* Saves error code and converts it to BrotliDecoderResult. */
func SaveErrorCode(s *BrotliDecoderState, e int) int { func SaveErrorCode(s *Reader, e int) int {
s.error_code = int(e) s.error_code = int(e)
switch e { switch e {
case BROTLI_DECODER_SUCCESS: case BROTLI_DECODER_SUCCESS:
@ -179,7 +179,7 @@ func SaveErrorCode(s *BrotliDecoderState, e int) int {
/* Decodes WBITS by reading 1 - 7 bits, or 0x11 for "Large Window Brotli". /* Decodes WBITS by reading 1 - 7 bits, or 0x11 for "Large Window Brotli".
Precondition: bit-reader accumulator has at least 8 bits. */ Precondition: bit-reader accumulator has at least 8 bits. */
func DecodeWindowBits(s *BrotliDecoderState, br *BrotliBitReader) int { func DecodeWindowBits(s *Reader, br *BrotliBitReader) int {
var n uint32 var n uint32
var large_window bool = s.large_window var large_window bool = s.large_window
s.large_window = false s.large_window = false
@ -220,7 +220,7 @@ func DecodeWindowBits(s *BrotliDecoderState, br *BrotliBitReader) int {
} }
/* Decodes a number in the range [0..255], by reading 1 - 11 bits. */ /* Decodes a number in the range [0..255], by reading 1 - 11 bits. */
func DecodeVarLenUint8(s *BrotliDecoderState, br *BrotliBitReader, value *uint32) int { func DecodeVarLenUint8(s *Reader, br *BrotliBitReader, value *uint32) int {
var bits uint32 var bits uint32
switch s.substate_decode_uint8 { switch s.substate_decode_uint8 {
case BROTLI_STATE_DECODE_UINT8_NONE: case BROTLI_STATE_DECODE_UINT8_NONE:
@ -268,7 +268,7 @@ func DecodeVarLenUint8(s *BrotliDecoderState, br *BrotliBitReader, value *uint32
} }
/* Decodes a metablock length and flags by reading 2 - 31 bits. */ /* Decodes a metablock length and flags by reading 2 - 31 bits. */
func DecodeMetaBlockLength(s *BrotliDecoderState, br *BrotliBitReader) int { func DecodeMetaBlockLength(s *Reader, br *BrotliBitReader) int {
var bits uint32 var bits uint32
var i int var i int
for { for {
@ -538,7 +538,7 @@ func Log2Floor(x uint32) uint32 {
/* Reads (s->symbol + 1) symbols. /* Reads (s->symbol + 1) symbols.
Totally 1..4 symbols are read, 1..11 bits each. Totally 1..4 symbols are read, 1..11 bits each.
The list of symbols MUST NOT contain duplicates. */ The list of symbols MUST NOT contain duplicates. */
func ReadSimpleHuffmanSymbols(alphabet_size uint32, max_symbol uint32, s *BrotliDecoderState) int { func ReadSimpleHuffmanSymbols(alphabet_size uint32, max_symbol uint32, s *Reader) int {
var br *BrotliBitReader = &s.br var br *BrotliBitReader = &s.br
var max_bits uint32 = Log2Floor(alphabet_size - 1) var max_bits uint32 = Log2Floor(alphabet_size - 1)
var i uint32 = s.sub_loop_counter var i uint32 = s.sub_loop_counter
@ -651,7 +651,7 @@ func ProcessRepeatedCodeLength(code_len uint32, repeat_delta uint32, alphabet_si
} }
/* Reads and decodes symbol codelengths. */ /* Reads and decodes symbol codelengths. */
func ReadSymbolCodeLengths(alphabet_size uint32, s *BrotliDecoderState) int { func ReadSymbolCodeLengths(alphabet_size uint32, s *Reader) int {
var br *BrotliBitReader = &s.br var br *BrotliBitReader = &s.br
var symbol uint32 = s.symbol var symbol uint32 = s.symbol
var repeat uint32 = s.repeat var repeat uint32 = s.repeat
@ -700,7 +700,7 @@ func ReadSymbolCodeLengths(alphabet_size uint32, s *BrotliDecoderState) int {
return BROTLI_DECODER_SUCCESS return BROTLI_DECODER_SUCCESS
} }
func SafeReadSymbolCodeLengths(alphabet_size uint32, s *BrotliDecoderState) int { func SafeReadSymbolCodeLengths(alphabet_size uint32, s *Reader) int {
var br *BrotliBitReader = &s.br var br *BrotliBitReader = &s.br
var get_byte bool = false var get_byte bool = false
var p []HuffmanCode var p []HuffmanCode
@ -746,7 +746,7 @@ func SafeReadSymbolCodeLengths(alphabet_size uint32, s *BrotliDecoderState) int
/* Reads and decodes 15..18 codes using static prefix code. /* Reads and decodes 15..18 codes using static prefix code.
Each code is 2..4 bits long. In total 30..72 bits are used. */ Each code is 2..4 bits long. In total 30..72 bits are used. */
func ReadCodeLengthCodeLengths(s *BrotliDecoderState) int { func ReadCodeLengthCodeLengths(s *Reader) int {
var br *BrotliBitReader = &s.br var br *BrotliBitReader = &s.br
var num_codes uint32 = s.repeat var num_codes uint32 = s.repeat
var space uint32 = s.space var space uint32 = s.space
@ -804,7 +804,7 @@ func ReadCodeLengthCodeLengths(s *BrotliDecoderState) int {
encoded with predefined entropy code. 32 - 74 bits are used. encoded with predefined entropy code. 32 - 74 bits are used.
B.2) Decoded table is used to decode code lengths of symbols in resulting B.2) Decoded table is used to decode code lengths of symbols in resulting
Huffman table. In worst case 3520 bits are read. */ Huffman table. In worst case 3520 bits are read. */
func ReadHuffmanCode(alphabet_size uint32, max_symbol uint32, table []HuffmanCode, opt_table_size *uint32, s *BrotliDecoderState) int { func ReadHuffmanCode(alphabet_size uint32, max_symbol uint32, table []HuffmanCode, opt_table_size *uint32, s *Reader) int {
var br *BrotliBitReader = &s.br var br *BrotliBitReader = &s.br
/* Unnecessary masking, but might be good for safety. */ /* Unnecessary masking, but might be good for safety. */
@ -954,7 +954,7 @@ func ReadBlockLength(table []HuffmanCode, br *BrotliBitReader) uint32 {
/* WARNING: if state is not BROTLI_STATE_READ_BLOCK_LENGTH_NONE, then /* WARNING: if state is not BROTLI_STATE_READ_BLOCK_LENGTH_NONE, then
reading can't be continued with ReadBlockLength. */ reading can't be continued with ReadBlockLength. */
func SafeReadBlockLength(s *BrotliDecoderState, result *uint32, table []HuffmanCode, br *BrotliBitReader) bool { func SafeReadBlockLength(s *Reader, result *uint32, table []HuffmanCode, br *BrotliBitReader) bool {
var index uint32 var index uint32
if s.substate_read_block_length == BROTLI_STATE_READ_BLOCK_LENGTH_NONE { if s.substate_read_block_length == BROTLI_STATE_READ_BLOCK_LENGTH_NONE {
if !SafeReadSymbol(table, br, &index) { if !SafeReadSymbol(table, br, &index) {
@ -992,7 +992,7 @@ func SafeReadBlockLength(s *BrotliDecoderState, result *uint32, table []HuffmanC
Most of input values are 0 and 1. To reduce number of branches, we replace Most of input values are 0 and 1. To reduce number of branches, we replace
inner for loop with do-while. */ inner for loop with do-while. */
func InverseMoveToFrontTransform(v []byte, v_len uint32, state *BrotliDecoderState) { func InverseMoveToFrontTransform(v []byte, v_len uint32, state *Reader) {
var mtf [256]byte var mtf [256]byte
var i int var i int
for i = 1; i < 256; i++ { for i = 1; i < 256; i++ {
@ -1016,7 +1016,7 @@ func InverseMoveToFrontTransform(v []byte, v_len uint32, state *BrotliDecoderSta
} }
/* Decodes a series of Huffman table using ReadHuffmanCode function. */ /* Decodes a series of Huffman table using ReadHuffmanCode function. */
func HuffmanTreeGroupDecode(group *HuffmanTreeGroup, s *BrotliDecoderState) int { func HuffmanTreeGroupDecode(group *HuffmanTreeGroup, s *Reader) int {
if s.substate_tree_group != BROTLI_STATE_TREE_GROUP_LOOP { if s.substate_tree_group != BROTLI_STATE_TREE_GROUP_LOOP {
s.next = group.codes s.next = group.codes
s.htree_index = 0 s.htree_index = 0
@ -1046,7 +1046,7 @@ func HuffmanTreeGroupDecode(group *HuffmanTreeGroup, s *BrotliDecoderState) int
This table will be used for reading context map items. This table will be used for reading context map items.
3) Read context map items; "0" values could be run-length encoded. 3) Read context map items; "0" values could be run-length encoded.
4) Optionally, apply InverseMoveToFront transform to the resulting map. */ 4) Optionally, apply InverseMoveToFront transform to the resulting map. */
func DecodeContextMap(context_map_size uint32, num_htrees *uint32, context_map_arg *[]byte, s *BrotliDecoderState) int { func DecodeContextMap(context_map_size uint32, num_htrees *uint32, context_map_arg *[]byte, s *Reader) int {
var br *BrotliBitReader = &s.br var br *BrotliBitReader = &s.br
var result int = BROTLI_DECODER_SUCCESS var result int = BROTLI_DECODER_SUCCESS
@ -1192,7 +1192,7 @@ func DecodeContextMap(context_map_size uint32, num_htrees *uint32, context_map_a
/* Decodes a command or literal and updates block type ring-buffer. /* Decodes a command or literal and updates block type ring-buffer.
Reads 3..54 bits. */ Reads 3..54 bits. */
func DecodeBlockTypeAndLength(safe int, s *BrotliDecoderState, tree_type int) bool { func DecodeBlockTypeAndLength(safe int, s *Reader, tree_type int) bool {
var max_block_type uint32 = s.num_block_types[tree_type] var max_block_type uint32 = s.num_block_types[tree_type]
var type_tree []HuffmanCode var type_tree []HuffmanCode
type_tree = s.block_type_trees[tree_type*BROTLI_HUFFMAN_MAX_SIZE_258:] type_tree = s.block_type_trees[tree_type*BROTLI_HUFFMAN_MAX_SIZE_258:]
@ -1239,7 +1239,7 @@ func DecodeBlockTypeAndLength(safe int, s *BrotliDecoderState, tree_type int) bo
return true return true
} }
func DetectTrivialLiteralBlockTypes(s *BrotliDecoderState) { func DetectTrivialLiteralBlockTypes(s *Reader) {
var i uint var i uint
for i = 0; i < 8; i++ { for i = 0; i < 8; i++ {
s.trivial_literal_contexts[i] = 0 s.trivial_literal_contexts[i] = 0
@ -1263,7 +1263,7 @@ func DetectTrivialLiteralBlockTypes(s *BrotliDecoderState) {
} }
} }
func PrepareLiteralDecoding(s *BrotliDecoderState) { func PrepareLiteralDecoding(s *Reader) {
var context_mode byte var context_mode byte
var trivial uint var trivial uint
var block_type uint32 = s.block_type_rb[1] var block_type uint32 = s.block_type_rb[1]
@ -1278,7 +1278,7 @@ func PrepareLiteralDecoding(s *BrotliDecoderState) {
/* Decodes the block type and updates the state for literal context. /* Decodes the block type and updates the state for literal context.
Reads 3..54 bits. */ Reads 3..54 bits. */
func DecodeLiteralBlockSwitchInternal(safe int, s *BrotliDecoderState) bool { func DecodeLiteralBlockSwitchInternal(safe int, s *Reader) bool {
if !DecodeBlockTypeAndLength(safe, s, 0) { if !DecodeBlockTypeAndLength(safe, s, 0) {
return false return false
} }
@ -1287,17 +1287,17 @@ func DecodeLiteralBlockSwitchInternal(safe int, s *BrotliDecoderState) bool {
return true return true
} }
func DecodeLiteralBlockSwitch(s *BrotliDecoderState) { func DecodeLiteralBlockSwitch(s *Reader) {
DecodeLiteralBlockSwitchInternal(0, s) DecodeLiteralBlockSwitchInternal(0, s)
} }
func SafeDecodeLiteralBlockSwitch(s *BrotliDecoderState) bool { func SafeDecodeLiteralBlockSwitch(s *Reader) bool {
return DecodeLiteralBlockSwitchInternal(1, s) return DecodeLiteralBlockSwitchInternal(1, s)
} }
/* Block switch for insert/copy length. /* Block switch for insert/copy length.
Reads 3..54 bits. */ Reads 3..54 bits. */
func DecodeCommandBlockSwitchInternal(safe int, s *BrotliDecoderState) bool { func DecodeCommandBlockSwitchInternal(safe int, s *Reader) bool {
if !DecodeBlockTypeAndLength(safe, s, 1) { if !DecodeBlockTypeAndLength(safe, s, 1) {
return false return false
} }
@ -1306,17 +1306,17 @@ func DecodeCommandBlockSwitchInternal(safe int, s *BrotliDecoderState) bool {
return true return true
} }
func DecodeCommandBlockSwitch(s *BrotliDecoderState) { func DecodeCommandBlockSwitch(s *Reader) {
DecodeCommandBlockSwitchInternal(0, s) DecodeCommandBlockSwitchInternal(0, s)
} }
func SafeDecodeCommandBlockSwitch(s *BrotliDecoderState) bool { func SafeDecodeCommandBlockSwitch(s *Reader) bool {
return DecodeCommandBlockSwitchInternal(1, s) return DecodeCommandBlockSwitchInternal(1, s)
} }
/* Block switch for distance codes. /* Block switch for distance codes.
Reads 3..54 bits. */ Reads 3..54 bits. */
func DecodeDistanceBlockSwitchInternal(safe int, s *BrotliDecoderState) bool { func DecodeDistanceBlockSwitchInternal(safe int, s *Reader) bool {
if !DecodeBlockTypeAndLength(safe, s, 2) { if !DecodeBlockTypeAndLength(safe, s, 2) {
return false return false
} }
@ -1326,15 +1326,15 @@ func DecodeDistanceBlockSwitchInternal(safe int, s *BrotliDecoderState) bool {
return true return true
} }
func DecodeDistanceBlockSwitch(s *BrotliDecoderState) { func DecodeDistanceBlockSwitch(s *Reader) {
DecodeDistanceBlockSwitchInternal(0, s) DecodeDistanceBlockSwitchInternal(0, s)
} }
func SafeDecodeDistanceBlockSwitch(s *BrotliDecoderState) bool { func SafeDecodeDistanceBlockSwitch(s *Reader) bool {
return DecodeDistanceBlockSwitchInternal(1, s) return DecodeDistanceBlockSwitchInternal(1, s)
} }
func UnwrittenBytes(s *BrotliDecoderState, wrap bool) uint { func UnwrittenBytes(s *Reader, wrap bool) uint {
var pos uint var pos uint
if wrap && s.pos > s.ringbuffer_size { if wrap && s.pos > s.ringbuffer_size {
pos = uint(s.ringbuffer_size) pos = uint(s.ringbuffer_size)
@ -1348,7 +1348,7 @@ func UnwrittenBytes(s *BrotliDecoderState, wrap bool) uint {
/* Dumps output. /* Dumps output.
Returns BROTLI_DECODER_NEEDS_MORE_OUTPUT only if there is more output to push Returns BROTLI_DECODER_NEEDS_MORE_OUTPUT only if there is more output to push
and either ring-buffer is as big as window size, or |force| is true. */ and either ring-buffer is as big as window size, or |force| is true. */
func WriteRingBuffer(s *BrotliDecoderState, available_out *uint, next_out *[]byte, total_out *uint, force bool) int { func WriteRingBuffer(s *Reader, available_out *uint, next_out *[]byte, total_out *uint, force bool) int {
var start []byte var start []byte
start = s.ringbuffer[s.partial_pos_out&uint(s.ringbuffer_mask):] start = s.ringbuffer[s.partial_pos_out&uint(s.ringbuffer_mask):]
var to_write uint = UnwrittenBytes(s, true) var to_write uint = UnwrittenBytes(s, true)
@ -1398,7 +1398,7 @@ func WriteRingBuffer(s *BrotliDecoderState, available_out *uint, next_out *[]byt
return BROTLI_DECODER_SUCCESS return BROTLI_DECODER_SUCCESS
} }
func WrapRingBuffer(s *BrotliDecoderState) { func WrapRingBuffer(s *Reader) {
if s.should_wrap_ringbuffer != 0 { if s.should_wrap_ringbuffer != 0 {
copy(s.ringbuffer, s.ringbuffer_end[:uint(s.pos)]) copy(s.ringbuffer, s.ringbuffer_end[:uint(s.pos)])
s.should_wrap_ringbuffer = 0 s.should_wrap_ringbuffer = 0
@ -1412,7 +1412,7 @@ func WrapRingBuffer(s *BrotliDecoderState) {
Last two bytes of ring-buffer are initialized to 0, so context calculation Last two bytes of ring-buffer are initialized to 0, so context calculation
could be done uniformly for the first two and all other positions. */ could be done uniformly for the first two and all other positions. */
func BrotliEnsureRingBuffer(s *BrotliDecoderState) bool { func BrotliEnsureRingBuffer(s *Reader) bool {
var old_ringbuffer []byte = s.ringbuffer var old_ringbuffer []byte = s.ringbuffer
if s.ringbuffer_size == s.new_ringbuffer_size { if s.ringbuffer_size == s.new_ringbuffer_size {
return true return true
@ -1442,7 +1442,7 @@ func BrotliEnsureRingBuffer(s *BrotliDecoderState) bool {
return true return true
} }
func CopyUncompressedBlockToOutput(available_out *uint, next_out *[]byte, total_out *uint, s *BrotliDecoderState) int { func CopyUncompressedBlockToOutput(available_out *uint, next_out *[]byte, total_out *uint, s *Reader) int {
/* TODO: avoid allocation for single uncompressed block. */ /* TODO: avoid allocation for single uncompressed block. */
if !BrotliEnsureRingBuffer(s) { if !BrotliEnsureRingBuffer(s) {
return BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_1 return BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_1
@ -1508,7 +1508,7 @@ func CopyUncompressedBlockToOutput(available_out *uint, next_out *[]byte, total_
size than needed to reduce memory usage. size than needed to reduce memory usage.
When this method is called, metablock size and flags MUST be decoded. */ When this method is called, metablock size and flags MUST be decoded. */
func BrotliCalculateRingBufferSize(s *BrotliDecoderState) { func BrotliCalculateRingBufferSize(s *Reader) {
var window_size int = 1 << s.window_bits var window_size int = 1 << s.window_bits
var new_ringbuffer_size int = window_size var new_ringbuffer_size int = window_size
var min_size int var min_size int
@ -1557,7 +1557,7 @@ func BrotliCalculateRingBufferSize(s *BrotliDecoderState) {
} }
/* Reads 1..256 2-bit context modes. */ /* Reads 1..256 2-bit context modes. */
func ReadContextModes(s *BrotliDecoderState) int { func ReadContextModes(s *Reader) int {
var br *BrotliBitReader = &s.br var br *BrotliBitReader = &s.br
var i int = s.loop_counter var i int = s.loop_counter
@ -1575,7 +1575,7 @@ func ReadContextModes(s *BrotliDecoderState) int {
return BROTLI_DECODER_SUCCESS return BROTLI_DECODER_SUCCESS
} }
func TakeDistanceFromRingBuffer(s *BrotliDecoderState) { func TakeDistanceFromRingBuffer(s *Reader) {
if s.distance_code == 0 { if s.distance_code == 0 {
s.dist_rb_idx-- s.dist_rb_idx--
s.distance_code = s.dist_rb[s.dist_rb_idx&3] s.distance_code = s.dist_rb[s.dist_rb_idx&3]
@ -1618,7 +1618,7 @@ func SafeReadBits(br *BrotliBitReader, n_bits uint32, val *uint32) bool {
} }
/* Precondition: s->distance_code < 0. */ /* Precondition: s->distance_code < 0. */
func ReadDistanceInternal(safe int, s *BrotliDecoderState, br *BrotliBitReader) bool { func ReadDistanceInternal(safe int, s *Reader, br *BrotliBitReader) bool {
var distval int var distval int
var memento BrotliBitReaderState var memento BrotliBitReaderState
var distance_tree []HuffmanCode = []HuffmanCode(s.distance_hgroup.htrees[s.dist_htree_index]) var distance_tree []HuffmanCode = []HuffmanCode(s.distance_hgroup.htrees[s.dist_htree_index])
@ -1679,15 +1679,15 @@ func ReadDistanceInternal(safe int, s *BrotliDecoderState, br *BrotliBitReader)
return true return true
} }
func ReadDistance(s *BrotliDecoderState, br *BrotliBitReader) { func ReadDistance(s *Reader, br *BrotliBitReader) {
ReadDistanceInternal(0, s, br) ReadDistanceInternal(0, s, br)
} }
func SafeReadDistance(s *BrotliDecoderState, br *BrotliBitReader) bool { func SafeReadDistance(s *Reader, br *BrotliBitReader) bool {
return ReadDistanceInternal(1, s, br) return ReadDistanceInternal(1, s, br)
} }
func ReadCommandInternal(safe int, s *BrotliDecoderState, br *BrotliBitReader, insert_length *int) bool { func ReadCommandInternal(safe int, s *Reader, br *BrotliBitReader, insert_length *int) bool {
var cmd_code uint32 var cmd_code uint32
var insert_len_extra uint32 = 0 var insert_len_extra uint32 = 0
var copy_length uint32 var copy_length uint32
@ -1726,11 +1726,11 @@ func ReadCommandInternal(safe int, s *BrotliDecoderState, br *BrotliBitReader, i
return true return true
} }
func ReadCommand(s *BrotliDecoderState, br *BrotliBitReader, insert_length *int) { func ReadCommand(s *Reader, br *BrotliBitReader, insert_length *int) {
ReadCommandInternal(0, s, br, insert_length) ReadCommandInternal(0, s, br, insert_length)
} }
func SafeReadCommand(s *BrotliDecoderState, br *BrotliBitReader, insert_length *int) bool { func SafeReadCommand(s *Reader, br *BrotliBitReader, insert_length *int) bool {
return ReadCommandInternal(1, s, br, insert_length) return ReadCommandInternal(1, s, br, insert_length)
} }
@ -1742,7 +1742,7 @@ func CheckInputAmount(safe int, br *BrotliBitReader, num uint) bool {
return BrotliCheckInputAmount(br, num) return BrotliCheckInputAmount(br, num)
} }
func ProcessCommandsInternal(safe int, s *BrotliDecoderState) int { func ProcessCommandsInternal(safe int, s *Reader) int {
var pos int = s.pos var pos int = s.pos
var i int = s.loop_counter var i int = s.loop_counter
var result int = BROTLI_DECODER_SUCCESS var result int = BROTLI_DECODER_SUCCESS
@ -2110,11 +2110,11 @@ saveStateAndReturn:
return result return result
} }
func ProcessCommands(s *BrotliDecoderState) int { func ProcessCommands(s *Reader) int {
return ProcessCommandsInternal(0, s) return ProcessCommandsInternal(0, s)
} }
func SafeProcessCommands(s *BrotliDecoderState) int { func SafeProcessCommands(s *Reader) int {
return ProcessCommandsInternal(1, s) return ProcessCommandsInternal(1, s)
} }
@ -2136,7 +2136,7 @@ func BrotliMaxDistanceSymbol(ndirect uint32, npostfix uint32) uint32 {
} }
func BrotliDecoderDecompress(encoded_size uint, encoded_buffer []byte, decoded_size *uint, decoded_buffer []byte) int { func BrotliDecoderDecompress(encoded_size uint, encoded_buffer []byte, decoded_size *uint, decoded_buffer []byte) int {
var s BrotliDecoderState var s Reader
var result int var result int
var total_out uint = 0 var total_out uint = 0
var available_in uint = encoded_size var available_in uint = encoded_size
@ -2168,7 +2168,7 @@ func BrotliDecoderDecompress(encoded_size uint, encoded_buffer []byte, decoded_s
buffer ahead of time buffer ahead of time
- when result is "success" decoder MUST return all unused data back to input - when result is "success" decoder MUST return all unused data back to input
buffer; this is possible because the invariant is held on enter */ buffer; this is possible because the invariant is held on enter */
func BrotliDecoderDecompressStream(s *BrotliDecoderState, available_in *uint, next_in *[]byte, available_out *uint, next_out *[]byte, total_out *uint) int { func BrotliDecoderDecompressStream(s *Reader, available_in *uint, next_in *[]byte, available_out *uint, next_out *[]byte, total_out *uint) int {
var result int = BROTLI_DECODER_SUCCESS var result int = BROTLI_DECODER_SUCCESS
var br *BrotliBitReader = &s.br var br *BrotliBitReader = &s.br
@ -2687,7 +2687,7 @@ func BrotliDecoderDecompressStream(s *BrotliDecoderState, available_in *uint, ne
return SaveErrorCode(s, result) return SaveErrorCode(s, result)
} }
func BrotliDecoderHasMoreOutput(s *BrotliDecoderState) bool { func BrotliDecoderHasMoreOutput(s *Reader) bool {
/* After unrecoverable error remaining output is considered nonsensical. */ /* After unrecoverable error remaining output is considered nonsensical. */
if int(s.error_code) < 0 { if int(s.error_code) < 0 {
return false return false
@ -2696,7 +2696,7 @@ func BrotliDecoderHasMoreOutput(s *BrotliDecoderState) bool {
return s.ringbuffer != nil && UnwrittenBytes(s, false) != 0 return s.ringbuffer != nil && UnwrittenBytes(s, false) != 0
} }
func BrotliDecoderTakeOutput(s *BrotliDecoderState, size *uint) []byte { func BrotliDecoderTakeOutput(s *Reader, size *uint) []byte {
var result []byte = nil var result []byte = nil
var available_out uint var available_out uint
if *size != 0 { if *size != 0 {
@ -2730,15 +2730,15 @@ func BrotliDecoderTakeOutput(s *BrotliDecoderState, size *uint) []byte {
return result return result
} }
func BrotliDecoderIsUsed(s *BrotliDecoderState) bool { func BrotliDecoderIsUsed(s *Reader) bool {
return s.state != BROTLI_STATE_UNINITED || BrotliGetAvailableBits(&s.br) != 0 return s.state != BROTLI_STATE_UNINITED || BrotliGetAvailableBits(&s.br) != 0
} }
func BrotliDecoderIsFinished(s *BrotliDecoderState) bool { func BrotliDecoderIsFinished(s *Reader) bool {
return (s.state == BROTLI_STATE_DONE) && !BrotliDecoderHasMoreOutput(s) return (s.state == BROTLI_STATE_DONE) && !BrotliDecoderHasMoreOutput(s)
} }
func BrotliDecoderGetErrorCode(s *BrotliDecoderState) int { func BrotliDecoderGetErrorCode(s *Reader) int {
return int(s.error_code) return int(s.error_code)
} }

94
reader.go Normal file
View File

@ -0,0 +1,94 @@
package brotli
import (
"errors"
"io"
)
type decodeError int
func (err decodeError) Error() string {
return "brotli: " + string(BrotliDecoderErrorString(int(err)))
}
var errExcessiveInput = errors.New("brotli: excessive input")
var errInvalidState = errors.New("brotli: invalid state")
var errReaderClosed = errors.New("brotli: Reader is closed")
// readBufSize is a "good" buffer size that avoids excessive round-trips
// between C and Go but doesn't waste too much memory on buffering.
// It is arbitrarily chosen to be equal to the constant used in io.Copy.
const readBufSize = 32 * 1024
// NewReader initializes new Reader instance.
func NewReader(src io.Reader) *Reader {
r := new(Reader)
BrotliDecoderStateInit(r)
r.src = src
r.buf = make([]byte, readBufSize)
return r
}
func (r *Reader) Read(p []byte) (n int, err error) {
if !BrotliDecoderHasMoreOutput(r) && len(r.in) == 0 {
m, readErr := r.src.Read(r.buf)
if m == 0 {
// If readErr is `nil`, we just proxy underlying stream behavior.
return 0, readErr
}
r.in = r.buf[:m]
}
if len(p) == 0 {
return 0, nil
}
for {
var written uint
in_len := uint(len(r.in))
out_len := uint(len(p))
in_remaining := in_len
out_remaining := out_len
result := BrotliDecoderDecompressStream(r, &in_remaining, &r.in, &out_remaining, &p, nil)
written = out_len - out_remaining
n = int(written)
switch result {
case BROTLI_DECODER_RESULT_SUCCESS:
if len(r.in) > 0 {
return n, errExcessiveInput
}
return n, nil
case BROTLI_DECODER_RESULT_ERROR:
return n, decodeError(BrotliDecoderGetErrorCode(r))
case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
if n == 0 {
return 0, io.ErrShortBuffer
}
return n, nil
case BROTLI_DECODER_NEEDS_MORE_INPUT:
}
if len(r.in) != 0 {
return 0, errInvalidState
}
// Calling r.src.Read may block. Don't block if we have data to return.
if n > 0 {
return n, nil
}
// Top off the buffer.
encN, err := r.src.Read(r.buf)
if encN == 0 {
// Not enough data to complete decoding.
if err == io.EOF {
return 0, io.ErrUnexpectedEOF
}
return 0, err
}
r.in = r.buf[:encN]
}
return n, nil
}

18
state.go
View File

@ -1,5 +1,7 @@
package brotli package brotli
import "io"
/* Copyright 2015 Google Inc. All Rights Reserved. /* Copyright 2015 Google Inc. All Rights Reserved.
Distributed under MIT license. Distributed under MIT license.
@ -89,7 +91,11 @@ const (
BROTLI_STATE_READ_BLOCK_LENGTH_SUFFIX BROTLI_STATE_READ_BLOCK_LENGTH_SUFFIX
) )
type BrotliDecoderState struct { type Reader struct {
src io.Reader
buf []byte // scratch space for reading from src
in []byte // current chunk to decode; usually aliases buf
state int state int
loop_counter int loop_counter int
br BrotliBitReader br BrotliBitReader
@ -177,7 +183,7 @@ type BrotliDecoderState struct {
trivial_literal_contexts [8]uint32 trivial_literal_contexts [8]uint32
} }
func BrotliDecoderStateInit(s *BrotliDecoderState) bool { func BrotliDecoderStateInit(s *Reader) bool {
s.error_code = 0 /* BROTLI_DECODER_NO_ERROR */ s.error_code = 0 /* BROTLI_DECODER_NO_ERROR */
BrotliInitBitReader(&s.br) BrotliInitBitReader(&s.br)
@ -244,7 +250,7 @@ func BrotliDecoderStateInit(s *BrotliDecoderState) bool {
return true return true
} }
func BrotliDecoderStateMetablockBegin(s *BrotliDecoderState) { func BrotliDecoderStateMetablockBegin(s *Reader) {
s.meta_block_remaining_len = 0 s.meta_block_remaining_len = 0
s.block_length[0] = 1 << 24 s.block_length[0] = 1 << 24
s.block_length[1] = 1 << 24 s.block_length[1] = 1 << 24
@ -274,7 +280,7 @@ func BrotliDecoderStateMetablockBegin(s *BrotliDecoderState) {
s.distance_hgroup.htrees = nil s.distance_hgroup.htrees = nil
} }
func BrotliDecoderStateCleanupAfterMetablock(s *BrotliDecoderState) { func BrotliDecoderStateCleanupAfterMetablock(s *Reader) {
s.context_modes = nil s.context_modes = nil
s.context_map = nil s.context_map = nil
s.dist_context_map = nil s.dist_context_map = nil
@ -283,14 +289,14 @@ func BrotliDecoderStateCleanupAfterMetablock(s *BrotliDecoderState) {
s.distance_hgroup.htrees = nil s.distance_hgroup.htrees = nil
} }
func BrotliDecoderStateCleanup(s *BrotliDecoderState) { func BrotliDecoderStateCleanup(s *Reader) {
BrotliDecoderStateCleanupAfterMetablock(s) BrotliDecoderStateCleanupAfterMetablock(s)
s.ringbuffer = nil s.ringbuffer = nil
s.block_type_trees = nil s.block_type_trees = nil
} }
func BrotliDecoderHuffmanTreeGroupInit(s *BrotliDecoderState, group *HuffmanTreeGroup, alphabet_size uint32, max_symbol uint32, ntrees uint32) bool { func BrotliDecoderHuffmanTreeGroupInit(s *Reader, group *HuffmanTreeGroup, alphabet_size uint32, max_symbol uint32, ntrees uint32) bool {
var max_table_size uint = uint(kMaxHuffmanTableSize[(alphabet_size+31)>>5]) var max_table_size uint = uint(kMaxHuffmanTableSize[(alphabet_size+31)>>5])
group.alphabet_size = uint16(alphabet_size) group.alphabet_size = uint16(alphabet_size)
group.max_symbol = uint16(max_symbol) group.max_symbol = uint16(max_symbol)