mirror
/
brotli
mirror of https://github.com/andybalholm/brotli.git
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

Push output directly to dst instead of buffering.

This commit is contained in:
Andy Balholm 2020-05-07 17:27:37 -07:00
parent 625cbb6f92
commit 7c7a5a10ef
2 changed files with 35 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
encode.go
View File

@ -74,6 +74,7 @@ const (
type Writer struct { type Writer struct {
dst io.Writer dst io.Writer
options WriterOptions options WriterOptions
err error
params encoderParams params encoderParams
hasher_ hasherHandle hasher_ hasherHandle
@ -102,9 +103,6 @@ type Writer struct {
cmd_code_numbits_ uint cmd_code_numbits_ uint
command_buf_ []uint32 command_buf_ []uint32
literal_buf_ []byte literal_buf_ []byte
next_out_ []byte
available_out_ uint
total_out_ uint
tiny_buf_ struct { tiny_buf_ struct {
u64 [2]uint64 u64 [2]uint64
u8 [16]byte u8 [16]byte
@ -145,7 +143,7 @@ func wrapPosition(position uint64) uint32 {
return result return result
} }
func getBrotliStorage(s *Writer, size int) []byte { func (s *Writer) getStorage(size int) []byte {
if len(s.storage) < size { if len(s.storage) < size {
s.storage = make([]byte, size) s.storage = make([]byte, size)
} }
@ -1089,9 +1087,6 @@ func encoderInitState(s *Writer) {
s.hasher_.Common().is_prepared_ = false s.hasher_.Common().is_prepared_ = false
} }
s.cmd_code_numbits_ = 0 s.cmd_code_numbits_ = 0
s.next_out_ = nil
s.available_out_ = 0
s.total_out_ = 0
s.stream_state_ = streamProcessing s.stream_state_ = streamProcessing
s.is_last_block_emitted_ = false s.is_last_block_emitted_ = false
s.is_initialized_ = false s.is_initialized_ = false
@ -1209,11 +1204,10 @@ func extendLastCommand(s *Writer, bytes *uint32, wrapped_last_processed_pos *uin
} }
/* /*
Processes the accumulated input data and sets |s.available_out_| to the length of Processes the accumulated input data and writes
the new output meta-block, or to zero if no new output meta-block has been the new output meta-block to s.dest, if one has been
created (in this case the processed input data is buffered internally). created (otherwise the processed input data is buffered internally).
If |s.available_out_| is positive, |s.next_out_| points to the start of the output If |is_last| or |force_flush| is true, an output meta-block is
data. If |is_last| or |force_flush| is true, an output meta-block is
always created. However, until |is_last| is true encoder may retain up always created. However, until |is_last| is true encoder may retain up
to 7 bits of the last byte of output. To force encoder to dump the remaining to 7 bits of the last byte of output. To force encoder to dump the remaining
bits use WriteMetadata() to append an empty meta-data block. bits use WriteMetadata() to append an empty meta-data block.
@ -1257,11 +1251,10 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
if delta == 0 && !is_last { if delta == 0 && !is_last {
/* We have no new input data and we don't have to finish the stream, so /* We have no new input data and we don't have to finish the stream, so
nothing to do. */ nothing to do. */
s.available_out_ = 0
return true return true
} }
storage = getBrotliStorage(s, int(2*bytes+503)) storage = s.getStorage(int(2*bytes + 503))
storage[0] = byte(s.last_bytes_) storage[0] = byte(s.last_bytes_)
storage[1] = byte(s.last_bytes_ >> 8) storage[1] = byte(s.last_bytes_ >> 8)
table = getHashTable(s, s.params.quality, uint(bytes), &table_size) table = getHashTable(s, s.params.quality, uint(bytes), &table_size)
@ -1274,8 +1267,7 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
s.last_bytes_ = uint16(storage[storage_ix>>3]) s.last_bytes_ = uint16(storage[storage_ix>>3])
s.last_bytes_bits_ = byte(storage_ix & 7) s.last_bytes_bits_ = byte(storage_ix & 7)
updateLastProcessedPos(s) updateLastProcessedPos(s)
s.next_out_ = storage[0:] s.writeOutput(storage[:storage_ix>>3])
s.available_out_ = storage_ix >> 3
return true return true
} }
{ {
@ -1334,7 +1326,6 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
hasherReset(s.hasher_) hasherReset(s.hasher_)
} }
s.available_out_ = 0
return true return true
} }
} }
@ -1350,8 +1341,6 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
if !is_last && s.input_pos_ == s.last_flush_pos_ { if !is_last && s.input_pos_ == s.last_flush_pos_ {
/* We have no new input data and we don't have to finish the stream, so /* We have no new input data and we don't have to finish the stream, so
nothing to do. */ nothing to do. */
s.available_out_ = 0
return true return true
} }
@ -1360,7 +1349,7 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
assert(s.input_pos_-s.last_flush_pos_ <= 1<<24) assert(s.input_pos_-s.last_flush_pos_ <= 1<<24)
{ {
var metablock_size uint32 = uint32(s.input_pos_ - s.last_flush_pos_) var metablock_size uint32 = uint32(s.input_pos_ - s.last_flush_pos_)
var storage []byte = getBrotliStorage(s, int(2*metablock_size+503)) var storage []byte = s.getStorage(int(2*metablock_size + 503))
var storage_ix uint = uint(s.last_bytes_bits_) var storage_ix uint = uint(s.last_bytes_bits_)
storage[0] = byte(s.last_bytes_) storage[0] = byte(s.last_bytes_)
storage[1] = byte(s.last_bytes_ >> 8) storage[1] = byte(s.last_bytes_ >> 8)
@ -1387,8 +1376,7 @@ func encodeData(s *Writer, is_last bool, force_flush bool) bool {
emitting an uncompressed block. */ emitting an uncompressed block. */
copy(s.saved_dist_cache_[:], s.dist_cache_[:]) copy(s.saved_dist_cache_[:], s.dist_cache_[:])
s.next_out_ = storage[0:] s.writeOutput(storage[:storage_ix>>3])
s.available_out_ = storage_ix >> 3
return true return true
} }
} }
@ -1428,7 +1416,6 @@ func writeMetadataHeader(s *Writer, block_size uint, header []byte) uint {
func injectBytePaddingBlock(s *Writer) { func injectBytePaddingBlock(s *Writer) {
var seal uint32 = uint32(s.last_bytes_) var seal uint32 = uint32(s.last_bytes_)
var seal_bits uint = uint(s.last_bytes_bits_) var seal_bits uint = uint(s.last_bytes_bits_)
var destination []byte
s.last_bytes_ = 0 s.last_bytes_ = 0
s.last_bytes_bits_ = 0 s.last_bytes_bits_ = 0
@ -1437,14 +1424,7 @@ func injectBytePaddingBlock(s *Writer) {
seal_bits += 6 seal_bits += 6
/* If we have already created storage, then append to it. destination := s.tiny_buf_.u8[:]
Storage is valid until next block is being compressed. */
if s.next_out_ != nil {
destination = s.next_out_[s.available_out_:]
} else {
destination = s.tiny_buf_.u8[:]
s.next_out_ = destination
}
destination[0] = byte(seal) destination[0] = byte(seal)
if seal_bits > 8 { if seal_bits > 8 {
@ -1453,13 +1433,12 @@ func injectBytePaddingBlock(s *Writer) {
if seal_bits > 16 { if seal_bits > 16 {
destination[2] = byte(seal >> 16) destination[2] = byte(seal >> 16)
} }
s.available_out_ += (seal_bits + 7) >> 3 s.writeOutput(destination[:(seal_bits+7)>>3])
} }
func checkFlushComplete(s *Writer) { func checkFlushComplete(s *Writer) {
if s.stream_state_ == streamFlushRequested && s.available_out_ == 0 { if s.stream_state_ == streamFlushRequested && s.err == nil {
s.stream_state_ = streamProcessing s.stream_state_ = streamProcessing
s.next_out_ = nil
} }
} }
@ -1497,10 +1476,10 @@ func encoderCompressStreamFast(s *Writer, op int, available_in *uint, next_in *[
continue continue
} }
/* Compress block only when internal output buffer is empty, stream is not /* Compress block only when stream is not
finished, there is no pending flush request, and there is either finished, there is no pending flush request, and there is either
additional input or pending operation. */ additional input or pending operation. */
if s.available_out_ == 0 && s.stream_state_ == streamProcessing && (*available_in != 0 || op != int(operationProcess)) { if s.stream_state_ == streamProcessing && (*available_in != 0 || op != int(operationProcess)) {
var block_size uint = brotli_min_size_t(block_size_limit, *available_in) var block_size uint = brotli_min_size_t(block_size_limit, *available_in)
var is_last bool = (*available_in == block_size) && (op == int(operationFinish)) var is_last bool = (*available_in == block_size) && (op == int(operationFinish))
var force_flush bool = (*available_in == block_size) && (op == int(operationFlush)) var force_flush bool = (*available_in == block_size) && (op == int(operationFlush))
@ -1515,7 +1494,7 @@ func encoderCompressStreamFast(s *Writer, op int, available_in *uint, next_in *[
continue continue
} }
storage = getBrotliStorage(s, int(max_out_size)) storage = s.getStorage(int(max_out_size))
storage[0] = byte(s.last_bytes_) storage[0] = byte(s.last_bytes_)
storage[1] = byte(s.last_bytes_ >> 8) storage[1] = byte(s.last_bytes_ >> 8)
@ -1530,8 +1509,7 @@ func encoderCompressStreamFast(s *Writer, op int, available_in *uint, next_in *[
*next_in = (*next_in)[block_size:] *next_in = (*next_in)[block_size:]
*available_in -= block_size *available_in -= block_size
var out_bytes uint = storage_ix >> 3 var out_bytes uint = storage_ix >> 3
s.next_out_ = storage s.writeOutput(storage[:out_bytes])
s.available_out_ = out_bytes
s.last_bytes_ = uint16(storage[storage_ix>>3]) s.last_bytes_ = uint16(storage[storage_ix>>3])
s.last_bytes_bits_ = byte(storage_ix & 7) s.last_bytes_bits_ = byte(storage_ix & 7)
@ -1575,10 +1553,6 @@ func processMetadata(s *Writer, available_in *uint, next_in *[]byte) bool {
continue continue
} }
if s.available_out_ != 0 {
break
}
if s.input_pos_ != s.last_flush_pos_ { if s.input_pos_ != s.last_flush_pos_ {
var result bool = encodeData(s, false, true) var result bool = encodeData(s, false, true)
if !result { if !result {
@ -1588,8 +1562,8 @@ func processMetadata(s *Writer, available_in *uint, next_in *[]byte) bool {
} }
if s.stream_state_ == streamMetadataHead { if s.stream_state_ == streamMetadataHead {
s.next_out_ = s.tiny_buf_.u8[:] n := writeMetadataHeader(s, uint(s.remaining_metadata_bytes_), s.tiny_buf_.u8[:])
s.available_out_ = writeMetadataHeader(s, uint(s.remaining_metadata_bytes_), s.next_out_) s.writeOutput(s.tiny_buf_.u8[:n])
s.stream_state_ = streamMetadataBody s.stream_state_ = streamMetadataBody
continue continue
} else { } else {
@ -1603,12 +1577,11 @@ func processMetadata(s *Writer, available_in *uint, next_in *[]byte) bool {
/* This guarantees progress in "TakeOutput" workflow. */ /* This guarantees progress in "TakeOutput" workflow. */
var c uint32 = brotli_min_uint32_t(s.remaining_metadata_bytes_, 16) var c uint32 = brotli_min_uint32_t(s.remaining_metadata_bytes_, 16)
s.next_out_ = s.tiny_buf_.u8[:] copy(s.tiny_buf_.u8[:], (*next_in)[:c])
copy(s.next_out_, (*next_in)[:c])
*next_in = (*next_in)[c:] *next_in = (*next_in)[c:]
*available_in -= uint(c) *available_in -= uint(c)
s.remaining_metadata_bytes_ -= c s.remaining_metadata_bytes_ -= c
s.available_out_ = uint(c) s.writeOutput(s.tiny_buf_.u8[:c])
continue continue
} }
@ -1681,9 +1654,9 @@ func encoderCompressStream(s *Writer, op int, available_in *uint, next_in *[]byt
continue continue
} }
/* Compress data only when internal output buffer is empty, stream is not /* Compress data only when stream is not
finished and there is no pending flush request. */ finished and there is no pending flush request. */
if s.available_out_ == 0 && s.stream_state_ == streamProcessing { if s.stream_state_ == streamProcessing {
if remaining_block_size == 0 || op != int(operationProcess) { if remaining_block_size == 0 || op != int(operationProcess) {
var is_last bool = ((*available_in == 0) && op == int(operationFinish)) var is_last bool = ((*available_in == 0) && op == int(operationFinish))
var force_flush bool = ((*available_in == 0) && op == int(operationFlush)) var force_flush bool = ((*available_in == 0) && op == int(operationFlush))
@ -1710,18 +1683,13 @@ func encoderCompressStream(s *Writer, op int, available_in *uint, next_in *[]byt
return true return true
} }
func encoderHasMoreOutput(s *Writer) bool { func (w *Writer) writeOutput(data []byte) {
return s.available_out_ != 0 if w.err != nil {
return
} }
func encoderTakeOutput(s *Writer) []byte { _, w.err = w.dst.Write(data)
if s.available_out_ == 0 { if w.err == nil {
return nil checkFlushComplete(w)
} }
result := s.next_out_[:s.available_out_]
s.total_out_ += s.available_out_
s.available_out_ = 0
checkFlushComplete(s)
return result
} }

15
writer.go
View File

@ -67,6 +67,9 @@ func (w *Writer) writeChunk(p []byte, op int) (n int, err error) {
if w.dst == nil { if w.dst == nil {
return 0, errWriterClosed return 0, errWriterClosed
} }
if w.err != nil {
return 0, w.err
}
for { for {
availableIn := uint(len(p)) availableIn := uint(len(p))
@ -79,16 +82,8 @@ func (w *Writer) writeChunk(p []byte, op int) (n int, err error) {
return n, errEncode return n, errEncode
} }
outputData := encoderTakeOutput(w) if len(p) == 0 || w.err != nil {
return n, w.err
if len(outputData) > 0 {
_, err = w.dst.Write(outputData)
if err != nil {
return n, err
}
}
if len(p) == 0 {
return n, nil
} }
} }
} }