From b42e7f9b1310ba359eb1b42d11cfdb7e1658f1eb Mon Sep 17 00:00:00 2001 From: Saxon Date: Fri, 26 Apr 2019 20:31:12 +0930 Subject: [PATCH] codec: created h264 and mjpeg packages to house lexers --- codec/{lex => h264}/lex.go | 52 +--------- codec/{lex => h264}/lex_test.go | 133 +------------------------- codec/mjpeg/lex.go | 155 ++++++++++++++++++++++++++++++ codec/mjpeg/lex_test.go | 163 ++++++++++++++++++++++++++++++++ 4 files changed, 320 insertions(+), 183 deletions(-) rename codec/{lex => h264}/lex.go (81%) rename codec/{lex => h264}/lex_test.go (67%) create mode 100644 codec/mjpeg/lex.go create mode 100644 codec/mjpeg/lex_test.go diff --git a/codec/lex/lex.go b/codec/h264/lex.go similarity index 81% rename from codec/lex/lex.go rename to codec/h264/lex.go index da0dd1b6..c9de1196 100644 --- a/codec/lex/lex.go +++ b/codec/h264/lex.go @@ -24,14 +24,9 @@ LICENSE You should have received a copy of the GNU General Public License along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. */ - -// Package lex provides lexers for video encodings. -package lex +package h264 import ( - "bufio" - "bytes" - "fmt" "io" "time" ) @@ -200,48 +195,3 @@ func (c *scanner) reload() error { c.off = 0 return nil } - -// MJPEG parses MJPEG frames read from src into separate writes to dst with -// successive writes being performed not earlier than the specified delay. -func MJPEG(dst io.Writer, src io.Reader, delay time.Duration) error { - var tick <-chan time.Time - if delay == 0 { - tick = noDelay - } else { - ticker := time.NewTicker(delay) - defer ticker.Stop() - tick = ticker.C - } - - r := bufio.NewReader(src) - for { - buf := make([]byte, 2, 4<<10) - n, err := r.Read(buf) - if n < 2 { - return nil - } - if err != nil { - return err - } - if !bytes.Equal(buf, []byte{0xff, 0xd8}) { - return fmt.Errorf("parser: not MJPEG frame start: %#v", buf) - } - var last byte - for { - b, err := r.ReadByte() - if err != nil { - return err - } - buf = append(buf, b) - if last == 0xff && b == 0xd9 { - break - } - last = b - } - <-tick - _, err = dst.Write(buf) - if err != nil { - return err - } - } -} diff --git a/codec/lex/lex_test.go b/codec/h264/lex_test.go similarity index 67% rename from codec/lex/lex_test.go rename to codec/h264/lex_test.go index a107b253..9912d751 100644 --- a/codec/lex/lex_test.go +++ b/codec/h264/lex_test.go @@ -25,12 +25,9 @@ LICENSE along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. */ -package lex +package h264 import ( - "bytes" - "reflect" - "testing" "time" ) @@ -221,131 +218,3 @@ func TestH264(t *testing.T) { } } */ - -var mjpegTests = []struct { - name string - input []byte - delay time.Duration - want [][]byte - err error -}{ - { - name: "empty", - }, - { - name: "null", - input: []byte{0xff, 0xd8, 0xff, 0xd9}, - delay: 0, - want: [][]byte{{0xff, 0xd8, 0xff, 0xd9}}, - }, - { - name: "null delayed", - input: []byte{0xff, 0xd8, 0xff, 0xd9}, - delay: time.Millisecond, - want: [][]byte{{0xff, 0xd8, 0xff, 0xd9}}, - }, - { - name: "full", - input: []byte{ - 0xff, 0xd8, 'f', 'u', 'l', 'l', 0xff, 0xd9, - 0xff, 0xd8, 'f', 'r', 'a', 'm', 'e', 0xff, 0xd9, - 0xff, 0xd8, 'w', 'i', 't', 'h', 0xff, 0xd9, - 0xff, 0xd8, 'l', 'e', 'n', 'g', 't', 'h', 0xff, 0xd9, - 0xff, 0xd8, 's', 'p', 'r', 'e', 'a', 'd', 0xff, 0xd9, - }, - delay: 0, - want: [][]byte{ - {0xff, 0xd8, 'f', 'u', 'l', 'l', 0xff, 0xd9}, - {0xff, 0xd8, 'f', 'r', 'a', 'm', 'e', 0xff, 0xd9}, - {0xff, 0xd8, 'w', 'i', 't', 'h', 0xff, 0xd9}, - {0xff, 0xd8, 'l', 'e', 'n', 'g', 't', 'h', 0xff, 0xd9}, - {0xff, 0xd8, 's', 'p', 'r', 'e', 'a', 'd', 0xff, 0xd9}, - }, - }, - { - name: "full delayed", - input: []byte{ - 0xff, 0xd8, 'f', 'u', 'l', 'l', 0xff, 0xd9, - 0xff, 0xd8, 'f', 'r', 'a', 'm', 'e', 0xff, 0xd9, - 0xff, 0xd8, 'w', 'i', 't', 'h', 0xff, 0xd9, - 0xff, 0xd8, 'l', 'e', 'n', 'g', 't', 'h', 0xff, 0xd9, - 0xff, 0xd8, 's', 'p', 'r', 'e', 'a', 'd', 0xff, 0xd9, - }, - delay: time.Millisecond, - want: [][]byte{ - {0xff, 0xd8, 'f', 'u', 'l', 'l', 0xff, 0xd9}, - {0xff, 0xd8, 'f', 'r', 'a', 'm', 'e', 0xff, 0xd9}, - {0xff, 0xd8, 'w', 'i', 't', 'h', 0xff, 0xd9}, - {0xff, 0xd8, 'l', 'e', 'n', 'g', 't', 'h', 0xff, 0xd9}, - {0xff, 0xd8, 's', 'p', 'r', 'e', 'a', 'd', 0xff, 0xd9}, - }, - }, -} - -// FIXME this needs to be adapted -/* -func TestMJEG(t *testing.T) { - for _, test := range mjpegTests { - var buf chunkEncoder - err := MJPEG(&buf, bytes.NewReader(test.input), test.delay) - if fmt.Sprint(err) != fmt.Sprint(test.err) { - t.Errorf("unexpected error for %q: got:%v want:%v", test.name, err, test.err) - } - if err != nil { - continue - } - got := [][]byte(buf) - if !reflect.DeepEqual(got, test.want) { - t.Errorf("unexpected result for %q:\ngot :%#v\nwant:%#v", test.name, got, test.want) - } - } -} -*/ - -type chunkEncoder [][]byte - -func (e *chunkEncoder) Encode(b []byte) error { - *e = append(*e, b) - return nil -} - -func (*chunkEncoder) Stream() <-chan []byte { panic("INVALID USE") } - -func TestScannerReadByte(t *testing.T) { - data := []byte("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.") - - for _, size := range []int{1, 2, 8, 1 << 10} { - r := newScanner(bytes.NewReader(data), make([]byte, size)) - var got []byte - for { - b, err := r.readByte() - if err != nil { - break - } - got = append(got, b) - } - if !bytes.Equal(got, data) { - t.Errorf("unexpected result for buffer size %d:\ngot :%q\nwant:%q", size, got, data) - } - } -} - -func TestScannerScanUntilZero(t *testing.T) { - data := []byte("Lorem ipsum dolor sit amet, consectetur adipiscing elit,\x00 sed do eiusmod tempor incididunt ut \x00labore et dolore magna aliqua.") - - for _, size := range []int{1, 2, 8, 1 << 10} { - r := newScanner(bytes.NewReader(data), make([]byte, size)) - var got [][]byte - for { - buf, _, err := r.scanUntilZeroInto(nil) - got = append(got, buf) - if err != nil { - break - } - } - want := bytes.SplitAfter(data, []byte{0}) - if !reflect.DeepEqual(got, want) { - t.Errorf("unexpected result for buffer zie %d:\ngot :%q\nwant:%q", size, got, want) - } - } -} diff --git a/codec/mjpeg/lex.go b/codec/mjpeg/lex.go new file mode 100644 index 00000000..07d0d50c --- /dev/null +++ b/codec/mjpeg/lex.go @@ -0,0 +1,155 @@ +/* +NAME + lex.go + +DESCRIPTION + See Readme.md + +AUTHOR + Dan Kortschak + +LICENSE + lex.go is Copyright (C) 2017 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ + +// Package lex provides lexers for video encodings. +package mjpeg + +import ( + "bufio" + "bytes" + "fmt" + "io" + "time" +) + +var noDelay = make(chan time.Time) + +func init() { + close(noDelay) +} + +// MJPEG parses MJPEG frames read from src into separate writes to dst with +// successive writes being performed not earlier than the specified delay. +func MJPEG(dst io.Writer, src io.Reader, delay time.Duration) error { + var tick <-chan time.Time + if delay == 0 { + tick = noDelay + } else { + ticker := time.NewTicker(delay) + defer ticker.Stop() + tick = ticker.C + } + + r := bufio.NewReader(src) + for { + buf := make([]byte, 2, 4<<10) + n, err := r.Read(buf) + if n < 2 { + return nil + } + if err != nil { + return err + } + if !bytes.Equal(buf, []byte{0xff, 0xd8}) { + return fmt.Errorf("parser: not MJPEG frame start: %#v", buf) + } + var last byte + for { + b, err := r.ReadByte() + if err != nil { + return err + } + buf = append(buf, b) + if last == 0xff && b == 0xd9 { + break + } + last = b + } + <-tick + _, err = dst.Write(buf) + if err != nil { + return err + } + } +} + +// scanner is a byte scanner. +type scanner struct { + buf []byte + off int + + // r is the source of data for the scanner. + r io.Reader +} + +// newScanner returns a scanner initialised with an io.Reader and a read buffer. +func newScanner(r io.Reader, buf []byte) *scanner { + return &scanner{r: r, buf: buf[:0]} +} + +// scanUntilZeroInto scans the scanner's underlying io.Reader until a zero byte +// has been read, appending all read bytes to dst. The resulting appended data, +// the last read byte and whether the last read byte was zero are returned. +func (c *scanner) scanUntilZeroInto(dst []byte) (res []byte, b byte, err error) { +outer: + for { + var i int + for i, b = range c.buf[c.off:] { + if b != 0x0 { + continue + } + dst = append(dst, c.buf[c.off:c.off+i+1]...) + c.off += i + 1 + break outer + } + dst = append(dst, c.buf[c.off:]...) + err = c.reload() + if err != nil { + break + } + } + return dst, b, err +} + +// readByte is an unexported ReadByte. +func (c *scanner) readByte() (byte, error) { + if c.off >= len(c.buf) { + err := c.reload() + if err != nil { + return 0, err + } + } + b := c.buf[c.off] + c.off++ + return b, nil +} + +// reload re-fills the scanner's buffer. +func (c *scanner) reload() error { + n, err := c.r.Read(c.buf[:cap(c.buf)]) + c.buf = c.buf[:n] + if err != nil { + if err != io.EOF { + return err + } + if n == 0 { + return io.EOF + } + } + c.off = 0 + return nil +} diff --git a/codec/mjpeg/lex_test.go b/codec/mjpeg/lex_test.go new file mode 100644 index 00000000..216658d8 --- /dev/null +++ b/codec/mjpeg/lex_test.go @@ -0,0 +1,163 @@ +/* +NAME + lex_test.go + +DESCRIPTION + See Readme.md + +AUTHOR + Dan Kortschak + +LICENSE + lex_test.go is Copyright (C) 2017 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ + +package mjpeg + +import ( + "bytes" + "reflect" + "testing" + "time" +) + +var mjpegTests = []struct { + name string + input []byte + delay time.Duration + want [][]byte + err error +}{ + { + name: "empty", + }, + { + name: "null", + input: []byte{0xff, 0xd8, 0xff, 0xd9}, + delay: 0, + want: [][]byte{{0xff, 0xd8, 0xff, 0xd9}}, + }, + { + name: "null delayed", + input: []byte{0xff, 0xd8, 0xff, 0xd9}, + delay: time.Millisecond, + want: [][]byte{{0xff, 0xd8, 0xff, 0xd9}}, + }, + { + name: "full", + input: []byte{ + 0xff, 0xd8, 'f', 'u', 'l', 'l', 0xff, 0xd9, + 0xff, 0xd8, 'f', 'r', 'a', 'm', 'e', 0xff, 0xd9, + 0xff, 0xd8, 'w', 'i', 't', 'h', 0xff, 0xd9, + 0xff, 0xd8, 'l', 'e', 'n', 'g', 't', 'h', 0xff, 0xd9, + 0xff, 0xd8, 's', 'p', 'r', 'e', 'a', 'd', 0xff, 0xd9, + }, + delay: 0, + want: [][]byte{ + {0xff, 0xd8, 'f', 'u', 'l', 'l', 0xff, 0xd9}, + {0xff, 0xd8, 'f', 'r', 'a', 'm', 'e', 0xff, 0xd9}, + {0xff, 0xd8, 'w', 'i', 't', 'h', 0xff, 0xd9}, + {0xff, 0xd8, 'l', 'e', 'n', 'g', 't', 'h', 0xff, 0xd9}, + {0xff, 0xd8, 's', 'p', 'r', 'e', 'a', 'd', 0xff, 0xd9}, + }, + }, + { + name: "full delayed", + input: []byte{ + 0xff, 0xd8, 'f', 'u', 'l', 'l', 0xff, 0xd9, + 0xff, 0xd8, 'f', 'r', 'a', 'm', 'e', 0xff, 0xd9, + 0xff, 0xd8, 'w', 'i', 't', 'h', 0xff, 0xd9, + 0xff, 0xd8, 'l', 'e', 'n', 'g', 't', 'h', 0xff, 0xd9, + 0xff, 0xd8, 's', 'p', 'r', 'e', 'a', 'd', 0xff, 0xd9, + }, + delay: time.Millisecond, + want: [][]byte{ + {0xff, 0xd8, 'f', 'u', 'l', 'l', 0xff, 0xd9}, + {0xff, 0xd8, 'f', 'r', 'a', 'm', 'e', 0xff, 0xd9}, + {0xff, 0xd8, 'w', 'i', 't', 'h', 0xff, 0xd9}, + {0xff, 0xd8, 'l', 'e', 'n', 'g', 't', 'h', 0xff, 0xd9}, + {0xff, 0xd8, 's', 'p', 'r', 'e', 'a', 'd', 0xff, 0xd9}, + }, + }, +} + +// FIXME this needs to be adapted +/* +func TestMJEG(t *testing.T) { + for _, test := range mjpegTests { + var buf chunkEncoder + err := MJPEG(&buf, bytes.NewReader(test.input), test.delay) + if fmt.Sprint(err) != fmt.Sprint(test.err) { + t.Errorf("unexpected error for %q: got:%v want:%v", test.name, err, test.err) + } + if err != nil { + continue + } + got := [][]byte(buf) + if !reflect.DeepEqual(got, test.want) { + t.Errorf("unexpected result for %q:\ngot :%#v\nwant:%#v", test.name, got, test.want) + } + } +} +*/ + +type chunkEncoder [][]byte + +func (e *chunkEncoder) Encode(b []byte) error { + *e = append(*e, b) + return nil +} + +func (*chunkEncoder) Stream() <-chan []byte { panic("INVALID USE") } + +func TestScannerReadByte(t *testing.T) { + data := []byte("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.") + + for _, size := range []int{1, 2, 8, 1 << 10} { + r := newScanner(bytes.NewReader(data), make([]byte, size)) + var got []byte + for { + b, err := r.readByte() + if err != nil { + break + } + got = append(got, b) + } + if !bytes.Equal(got, data) { + t.Errorf("unexpected result for buffer size %d:\ngot :%q\nwant:%q", size, got, data) + } + } +} + +func TestScannerScanUntilZero(t *testing.T) { + data := []byte("Lorem ipsum dolor sit amet, consectetur adipiscing elit,\x00 sed do eiusmod tempor incididunt ut \x00labore et dolore magna aliqua.") + + for _, size := range []int{1, 2, 8, 1 << 10} { + r := newScanner(bytes.NewReader(data), make([]byte, size)) + var got [][]byte + for { + buf, _, err := r.scanUntilZeroInto(nil) + got = append(got, buf) + if err != nil { + break + } + } + want := bytes.SplitAfter(data, []byte{0}) + if !reflect.DeepEqual(got, want) { + t.Errorf("unexpected result for buffer zie %d:\ngot :%q\nwant:%q", size, got, want) + } + } +}