parse: new package to replace parser

This package provides stateless parsing, simplifying inclusing in
concurrent use and reducing synchronisation.

Use depends on a chan writer type until the generator package is
simplified.
This commit is contained in:
Dan Kortschak 2018-06-25 23:22:33 +09:30
parent 1383fd8b91
commit 62147e160d
2 changed files with 476 additions and 0 deletions

159
parse/parse.go Normal file
View File

@ -0,0 +1,159 @@
/*
NAME
parse.go
DESCRIPTION
See Readme.md
AUTHOR
Dan Kortschak <dan@ausocean.org>
LICENSE
parse.go is Copyright (C) 2017 the Australian Ocean Lab (AusOcean)
It is free software: you can redistribute it and/or modify them
under the terms of the GNU General Public License as published by the
Free Software Foundation, either version 3 of the License, or (at your
option) any later version.
It is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with revid in gpl.txt. If not, see http://www.gnu.org/licenses.
*/
// Package parse provides parsers for video encodings.
package parse
import (
"bufio"
"bytes"
"fmt"
"io"
"time"
)
var noDelay = make(chan time.Time)
func init() {
close(noDelay)
}
var h264Prefix = [...]byte{0x00, 0x00, 0x01, 0x09, 0xf0}
// H264 parses H.264 NAL units read from src into separate writes to dst with
// successive writes being performed not earlier than the specified delay.
// NAL units are split at type 1 (Coded slice of a non-IDR picture), 5
// (Coded slice of a IDR picture) and 8 (Picture parameter set).
func H264(dst io.Writer, src io.Reader, delay time.Duration) error {
var tick <-chan time.Time
if delay == 0 {
tick = noDelay
} else {
ticker := time.NewTicker(delay)
defer ticker.Stop()
tick = ticker.C
}
r := bufio.NewReader(src)
buf := make([]byte, len(h264Prefix), 4<<10)
copy(buf, h264Prefix[:])
var zeroes int
for {
b, err := r.ReadByte()
if err != nil {
if err != io.EOF {
return err
}
if len(buf) == len(h264Prefix) {
return nil
}
<-tick
_, err = dst.Write(buf)
return err
}
if b == 0 {
zeroes++
}
if (zeroes == 2 || zeroes == 3) && b == 1 && len(buf)-len(h264Prefix) > zeroes {
b, err = r.ReadByte()
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return err
}
const (
nonIdrPic = 1
idrPic = 5
paramSet = 8
)
switch nalTyp := b & 0x1f; nalTyp {
case nonIdrPic, idrPic, paramSet:
<-tick
_, err = dst.Write(buf[:len(buf)-zeroes])
if err != nil {
return err
}
buf = make([]byte, zeroes+len(h264Prefix), 4<<10)
copy(buf, h264Prefix[:])
zeroes = 0
}
buf = append(buf, 1, b)
continue
}
if b != 0 {
zeroes = 0
}
buf = append(buf, b)
}
}
// MJPEG parses MJPEG frames read from src into separate writes to dst with
// successive writes being performed not earlier than the specified delay.
func MJPEG(dst io.Writer, src io.Reader, delay time.Duration) error {
var tick <-chan time.Time
if delay == 0 {
tick = noDelay
} else {
ticker := time.NewTicker(delay)
defer ticker.Stop()
tick = ticker.C
}
r := bufio.NewReader(src)
for {
buf := make([]byte, 2, 4<<10)
n, err := r.Read(buf)
if n < 2 {
return nil
}
if err != nil {
return err
}
if !bytes.Equal(buf, []byte{0xff, 0xd8}) {
return fmt.Errorf("parser: not MJPEG frame start: %#v", buf)
}
var last byte
for {
b, err := r.ReadByte()
if err != nil {
return err
}
buf = append(buf, b)
if last == 0xff && b == 0xd9 {
break
}
last = b
}
<-tick
_, err = dst.Write(buf)
if err != nil {
return err
}
}
}

317
parse/parse_test.go Normal file
View File

@ -0,0 +1,317 @@
/*
NAME
parse_test.go
DESCRIPTION
See Readme.md
AUTHOR
Dan Kortschak <dan@ausocean.org>
LICENSE
parse_test.go is Copyright (C) 2017 the Australian Ocean Lab (AusOcean)
It is free software: you can redistribute it and/or modify them
under the terms of the GNU General Public License as published by the
Free Software Foundation, either version 3 of the License, or (at your
option) any later version.
It is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with revid in gpl.txt. If not, see http://www.gnu.org/licenses.
*/
package parse
import (
"bytes"
"fmt"
"reflect"
"testing"
"time"
)
var h264Tests = []struct {
name string
input []byte
delay time.Duration
want [][]byte
err error
}{
{
name: "empty",
},
{
name: "null short",
input: []byte{0x00, 0x00, 0x01},
delay: 0,
want: [][]byte{{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x01}},
},
{
name: "null short delayed",
input: []byte{0x00, 0x00, 0x01},
delay: time.Millisecond,
want: [][]byte{{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x01}},
},
{
name: "full short type 1",
input: []byte{
0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l',
0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e',
0x00, 0x00, 0x01, 0x41, 'w', 'i', 't', 'h',
0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h',
0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd',
},
delay: 0,
want: [][]byte{
{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l',
0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e'},
{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x01, 0x41, 'w', 'i', 't', 'h',
0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h',
0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd'},
},
},
{
name: "full short type 5",
input: []byte{
0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l',
0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e',
0x00, 0x00, 0x01, 0x45, 'w', 'i', 't', 'h',
0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h',
0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd',
},
delay: 0,
want: [][]byte{
{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l',
0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e'},
{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x01, 0x45, 'w', 'i', 't', 'h',
0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h',
0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd'},
},
},
{
name: "full short type 8",
input: []byte{
0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l',
0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e',
0x00, 0x00, 0x01, 0x48, 'w', 'i', 't', 'h',
0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h',
0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd',
},
delay: 0,
want: [][]byte{
{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l',
0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e'},
{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x01, 0x48, 'w', 'i', 't', 'h',
0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h',
0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd'},
},
},
{
name: "full short delayed",
input: []byte{
0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l',
0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e',
0x00, 0x00, 0x01, 0x41, 'w', 'i', 't', 'h',
0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h',
0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd',
},
delay: time.Millisecond,
want: [][]byte{
{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l',
0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e'},
{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x01, 0x41, 'w', 'i', 't', 'h',
0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h',
0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd'},
},
},
{
name: "null long",
input: []byte{0x00, 0x00, 0x00, 0x01},
delay: 0,
want: [][]byte{{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x00, 0x01}},
},
{
name: "null long delayed",
input: []byte{0x00, 0x00, 0x00, 0x01},
delay: time.Millisecond,
want: [][]byte{{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x00, 0x01}},
},
{
name: "full long type 1",
input: []byte{
0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l',
0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e',
0x00, 0x00, 0x00, 0x01, 0x41, 'w', 'i', 't', 'h',
0x00, 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h',
0x00, 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd',
},
delay: 0,
want: [][]byte{
{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l',
0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e'},
{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x00, 0x01, 0x41, 'w', 'i', 't', 'h',
0x00, 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h',
0x00, 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd'},
},
},
{
name: "full long type 5",
input: []byte{
0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l',
0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e',
0x00, 0x00, 0x00, 0x01, 0x45, 'w', 'i', 't', 'h',
0x00, 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h',
0x00, 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd',
},
delay: 0,
want: [][]byte{
{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l',
0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e'},
{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x00, 0x01, 0x45, 'w', 'i', 't', 'h',
0x00, 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h',
0x00, 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd'},
},
},
{
name: "full long type 8",
input: []byte{
0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l',
0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e',
0x00, 0x00, 0x00, 0x01, 0x48, 'w', 'i', 't', 'h',
0x00, 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h',
0x00, 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd',
},
delay: 0,
want: [][]byte{
{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l',
0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e'},
{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x00, 0x01, 0x48, 'w', 'i', 't', 'h',
0x00, 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h',
0x00, 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd'},
},
},
{
name: "full long delayed",
input: []byte{
0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l',
0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e',
0x00, 0x00, 0x00, 0x01, 0x41, 'w', 'i', 't', 'h',
0x00, 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h',
0x00, 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd',
},
delay: time.Millisecond,
want: [][]byte{
{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l',
0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e'},
{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x00, 0x01, 0x41, 'w', 'i', 't', 'h',
0x00, 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h',
0x00, 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd'},
},
},
}
func TestH264(t *testing.T) {
for _, test := range h264Tests {
var buf chunkWriter
err := H264(&buf, bytes.NewReader(test.input), test.delay)
if fmt.Sprint(err) != fmt.Sprint(test.err) {
t.Errorf("unexpected error for %q: got:%v want:%v", test.name, err, test.err)
}
if err != nil {
continue
}
got := [][]byte(buf)
if !reflect.DeepEqual(got, test.want) {
t.Errorf("unexpected result for %q:\ngot :%#v\nwant:%#v", test.name, got, test.want)
}
}
}
var mjpegTests = []struct {
name string
input []byte
delay time.Duration
want [][]byte
err error
}{
{
name: "empty",
},
{
name: "null",
input: []byte{0xff, 0xd8, 0xff, 0xd9},
delay: 0,
want: [][]byte{{0xff, 0xd8, 0xff, 0xd9}},
},
{
name: "null delayed",
input: []byte{0xff, 0xd8, 0xff, 0xd9},
delay: time.Millisecond,
want: [][]byte{{0xff, 0xd8, 0xff, 0xd9}},
},
{
name: "full",
input: []byte{
0xff, 0xd8, 'f', 'u', 'l', 'l', 0xff, 0xd9,
0xff, 0xd8, 'f', 'r', 'a', 'm', 'e', 0xff, 0xd9,
0xff, 0xd8, 'w', 'i', 't', 'h', 0xff, 0xd9,
0xff, 0xd8, 'l', 'e', 'n', 'g', 't', 'h', 0xff, 0xd9,
0xff, 0xd8, 's', 'p', 'r', 'e', 'a', 'd', 0xff, 0xd9,
},
delay: 0,
want: [][]byte{
{0xff, 0xd8, 'f', 'u', 'l', 'l', 0xff, 0xd9},
{0xff, 0xd8, 'f', 'r', 'a', 'm', 'e', 0xff, 0xd9},
{0xff, 0xd8, 'w', 'i', 't', 'h', 0xff, 0xd9},
{0xff, 0xd8, 'l', 'e', 'n', 'g', 't', 'h', 0xff, 0xd9},
{0xff, 0xd8, 's', 'p', 'r', 'e', 'a', 'd', 0xff, 0xd9},
},
},
{
name: "full delayed",
input: []byte{
0xff, 0xd8, 'f', 'u', 'l', 'l', 0xff, 0xd9,
0xff, 0xd8, 'f', 'r', 'a', 'm', 'e', 0xff, 0xd9,
0xff, 0xd8, 'w', 'i', 't', 'h', 0xff, 0xd9,
0xff, 0xd8, 'l', 'e', 'n', 'g', 't', 'h', 0xff, 0xd9,
0xff, 0xd8, 's', 'p', 'r', 'e', 'a', 'd', 0xff, 0xd9,
},
delay: time.Millisecond,
want: [][]byte{
{0xff, 0xd8, 'f', 'u', 'l', 'l', 0xff, 0xd9},
{0xff, 0xd8, 'f', 'r', 'a', 'm', 'e', 0xff, 0xd9},
{0xff, 0xd8, 'w', 'i', 't', 'h', 0xff, 0xd9},
{0xff, 0xd8, 'l', 'e', 'n', 'g', 't', 'h', 0xff, 0xd9},
{0xff, 0xd8, 's', 'p', 'r', 'e', 'a', 'd', 0xff, 0xd9},
},
},
}
func TestMJEG(t *testing.T) {
for _, test := range mjpegTests {
var buf chunkWriter
err := MJPEG(&buf, bytes.NewReader(test.input), test.delay)
if fmt.Sprint(err) != fmt.Sprint(test.err) {
t.Errorf("unexpected error for %q: got:%v want:%v", test.name, err, test.err)
}
if err != nil {
continue
}
got := [][]byte(buf)
if !reflect.DeepEqual(got, test.want) {
t.Errorf("unexpected result for %q:\ngot :%#v\nwant:%#v", test.name, got, test.want)
}
}
}
type chunkWriter [][]byte
func (w *chunkWriter) Write(b []byte) (int, error) {
*w = append(*w, b)
return len(b), nil
}