From 62147e160d8481675b842da3e7ebeca4fda883d8 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Mon, 25 Jun 2018 23:22:33 +0930 Subject: [PATCH] parse: new package to replace parser This package provides stateless parsing, simplifying inclusing in concurrent use and reducing synchronisation. Use depends on a chan writer type until the generator package is simplified. --- parse/parse.go | 159 ++++++++++++++++++++++ parse/parse_test.go | 317 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 476 insertions(+) create mode 100644 parse/parse.go create mode 100644 parse/parse_test.go diff --git a/parse/parse.go b/parse/parse.go new file mode 100644 index 00000000..bcad48d5 --- /dev/null +++ b/parse/parse.go @@ -0,0 +1,159 @@ +/* +NAME + parse.go + +DESCRIPTION + See Readme.md + +AUTHOR + Dan Kortschak + +LICENSE + parse.go is Copyright (C) 2017 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ + +// Package parse provides parsers for video encodings. +package parse + +import ( + "bufio" + "bytes" + "fmt" + "io" + "time" +) + +var noDelay = make(chan time.Time) + +func init() { + close(noDelay) +} + +var h264Prefix = [...]byte{0x00, 0x00, 0x01, 0x09, 0xf0} + +// H264 parses H.264 NAL units read from src into separate writes to dst with +// successive writes being performed not earlier than the specified delay. +// NAL units are split at type 1 (Coded slice of a non-IDR picture), 5 +// (Coded slice of a IDR picture) and 8 (Picture parameter set). +func H264(dst io.Writer, src io.Reader, delay time.Duration) error { + var tick <-chan time.Time + if delay == 0 { + tick = noDelay + } else { + ticker := time.NewTicker(delay) + defer ticker.Stop() + tick = ticker.C + } + + r := bufio.NewReader(src) + + buf := make([]byte, len(h264Prefix), 4<<10) + copy(buf, h264Prefix[:]) + var zeroes int + for { + b, err := r.ReadByte() + if err != nil { + if err != io.EOF { + return err + } + if len(buf) == len(h264Prefix) { + return nil + } + <-tick + _, err = dst.Write(buf) + return err + } + if b == 0 { + zeroes++ + } + if (zeroes == 2 || zeroes == 3) && b == 1 && len(buf)-len(h264Prefix) > zeroes { + b, err = r.ReadByte() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return err + } + const ( + nonIdrPic = 1 + idrPic = 5 + paramSet = 8 + ) + switch nalTyp := b & 0x1f; nalTyp { + case nonIdrPic, idrPic, paramSet: + <-tick + _, err = dst.Write(buf[:len(buf)-zeroes]) + if err != nil { + return err + } + buf = make([]byte, zeroes+len(h264Prefix), 4<<10) + copy(buf, h264Prefix[:]) + zeroes = 0 + } + buf = append(buf, 1, b) + continue + } + if b != 0 { + zeroes = 0 + } + buf = append(buf, b) + } +} + +// MJPEG parses MJPEG frames read from src into separate writes to dst with +// successive writes being performed not earlier than the specified delay. +func MJPEG(dst io.Writer, src io.Reader, delay time.Duration) error { + var tick <-chan time.Time + if delay == 0 { + tick = noDelay + } else { + ticker := time.NewTicker(delay) + defer ticker.Stop() + tick = ticker.C + } + + r := bufio.NewReader(src) + for { + buf := make([]byte, 2, 4<<10) + n, err := r.Read(buf) + if n < 2 { + return nil + } + if err != nil { + return err + } + if !bytes.Equal(buf, []byte{0xff, 0xd8}) { + return fmt.Errorf("parser: not MJPEG frame start: %#v", buf) + } + var last byte + for { + b, err := r.ReadByte() + if err != nil { + return err + } + buf = append(buf, b) + if last == 0xff && b == 0xd9 { + break + } + last = b + } + <-tick + _, err = dst.Write(buf) + if err != nil { + return err + } + } +} diff --git a/parse/parse_test.go b/parse/parse_test.go new file mode 100644 index 00000000..2b635eaa --- /dev/null +++ b/parse/parse_test.go @@ -0,0 +1,317 @@ +/* +NAME + parse_test.go + +DESCRIPTION + See Readme.md + +AUTHOR + Dan Kortschak + +LICENSE + parse_test.go is Copyright (C) 2017 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ + +package parse + +import ( + "bytes" + "fmt" + "reflect" + "testing" + "time" +) + +var h264Tests = []struct { + name string + input []byte + delay time.Duration + want [][]byte + err error +}{ + { + name: "empty", + }, + { + name: "null short", + input: []byte{0x00, 0x00, 0x01}, + delay: 0, + want: [][]byte{{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x01}}, + }, + { + name: "null short delayed", + input: []byte{0x00, 0x00, 0x01}, + delay: time.Millisecond, + want: [][]byte{{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x01}}, + }, + { + name: "full short type 1", + input: []byte{ + 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l', + 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e', + 0x00, 0x00, 0x01, 0x41, 'w', 'i', 't', 'h', + 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h', + 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd', + }, + delay: 0, + want: [][]byte{ + {0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l', + 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e'}, + {0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x01, 0x41, 'w', 'i', 't', 'h', + 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h', + 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd'}, + }, + }, + { + name: "full short type 5", + input: []byte{ + 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l', + 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e', + 0x00, 0x00, 0x01, 0x45, 'w', 'i', 't', 'h', + 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h', + 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd', + }, + delay: 0, + want: [][]byte{ + {0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l', + 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e'}, + {0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x01, 0x45, 'w', 'i', 't', 'h', + 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h', + 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd'}, + }, + }, + { + name: "full short type 8", + input: []byte{ + 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l', + 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e', + 0x00, 0x00, 0x01, 0x48, 'w', 'i', 't', 'h', + 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h', + 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd', + }, + delay: 0, + want: [][]byte{ + {0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l', + 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e'}, + {0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x01, 0x48, 'w', 'i', 't', 'h', + 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h', + 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd'}, + }, + }, + { + name: "full short delayed", + input: []byte{ + 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l', + 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e', + 0x00, 0x00, 0x01, 0x41, 'w', 'i', 't', 'h', + 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h', + 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd', + }, + delay: time.Millisecond, + want: [][]byte{ + {0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l', + 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e'}, + {0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x01, 0x41, 'w', 'i', 't', 'h', + 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h', + 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd'}, + }, + }, + { + name: "null long", + input: []byte{0x00, 0x00, 0x00, 0x01}, + delay: 0, + want: [][]byte{{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x00, 0x01}}, + }, + { + name: "null long delayed", + input: []byte{0x00, 0x00, 0x00, 0x01}, + delay: time.Millisecond, + want: [][]byte{{0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x00, 0x01}}, + }, + { + name: "full long type 1", + input: []byte{ + 0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l', + 0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e', + 0x00, 0x00, 0x00, 0x01, 0x41, 'w', 'i', 't', 'h', + 0x00, 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h', + 0x00, 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd', + }, + delay: 0, + want: [][]byte{ + {0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l', + 0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e'}, + {0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x00, 0x01, 0x41, 'w', 'i', 't', 'h', + 0x00, 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h', + 0x00, 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd'}, + }, + }, + { + name: "full long type 5", + input: []byte{ + 0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l', + 0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e', + 0x00, 0x00, 0x00, 0x01, 0x45, 'w', 'i', 't', 'h', + 0x00, 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h', + 0x00, 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd', + }, + delay: 0, + want: [][]byte{ + {0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l', + 0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e'}, + {0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x00, 0x01, 0x45, 'w', 'i', 't', 'h', + 0x00, 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h', + 0x00, 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd'}, + }, + }, + { + name: "full long type 8", + input: []byte{ + 0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l', + 0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e', + 0x00, 0x00, 0x00, 0x01, 0x48, 'w', 'i', 't', 'h', + 0x00, 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h', + 0x00, 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd', + }, + delay: 0, + want: [][]byte{ + {0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l', + 0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e'}, + {0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x00, 0x01, 0x48, 'w', 'i', 't', 'h', + 0x00, 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h', + 0x00, 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd'}, + }, + }, + { + name: "full long delayed", + input: []byte{ + 0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l', + 0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e', + 0x00, 0x00, 0x00, 0x01, 0x41, 'w', 'i', 't', 'h', + 0x00, 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h', + 0x00, 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd', + }, + delay: time.Millisecond, + want: [][]byte{ + {0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'u', 'l', 'l', + 0x00, 0x00, 0x00, 0x01, 0x00, 'f', 'r', 'a', 'm', 'e'}, + {0x0, 0x0, 0x1, 0x9, 0xf0, 0x00, 0x00, 0x00, 0x01, 0x41, 'w', 'i', 't', 'h', + 0x00, 0x00, 0x00, 0x01, 0x00, 'l', 'e', 'n', 'g', 't', 'h', + 0x00, 0x00, 0x00, 0x01, 0x00, 's', 'p', 'r', 'e', 'a', 'd'}, + }, + }, +} + +func TestH264(t *testing.T) { + for _, test := range h264Tests { + var buf chunkWriter + err := H264(&buf, bytes.NewReader(test.input), test.delay) + if fmt.Sprint(err) != fmt.Sprint(test.err) { + t.Errorf("unexpected error for %q: got:%v want:%v", test.name, err, test.err) + } + if err != nil { + continue + } + got := [][]byte(buf) + if !reflect.DeepEqual(got, test.want) { + t.Errorf("unexpected result for %q:\ngot :%#v\nwant:%#v", test.name, got, test.want) + } + } +} + +var mjpegTests = []struct { + name string + input []byte + delay time.Duration + want [][]byte + err error +}{ + { + name: "empty", + }, + { + name: "null", + input: []byte{0xff, 0xd8, 0xff, 0xd9}, + delay: 0, + want: [][]byte{{0xff, 0xd8, 0xff, 0xd9}}, + }, + { + name: "null delayed", + input: []byte{0xff, 0xd8, 0xff, 0xd9}, + delay: time.Millisecond, + want: [][]byte{{0xff, 0xd8, 0xff, 0xd9}}, + }, + { + name: "full", + input: []byte{ + 0xff, 0xd8, 'f', 'u', 'l', 'l', 0xff, 0xd9, + 0xff, 0xd8, 'f', 'r', 'a', 'm', 'e', 0xff, 0xd9, + 0xff, 0xd8, 'w', 'i', 't', 'h', 0xff, 0xd9, + 0xff, 0xd8, 'l', 'e', 'n', 'g', 't', 'h', 0xff, 0xd9, + 0xff, 0xd8, 's', 'p', 'r', 'e', 'a', 'd', 0xff, 0xd9, + }, + delay: 0, + want: [][]byte{ + {0xff, 0xd8, 'f', 'u', 'l', 'l', 0xff, 0xd9}, + {0xff, 0xd8, 'f', 'r', 'a', 'm', 'e', 0xff, 0xd9}, + {0xff, 0xd8, 'w', 'i', 't', 'h', 0xff, 0xd9}, + {0xff, 0xd8, 'l', 'e', 'n', 'g', 't', 'h', 0xff, 0xd9}, + {0xff, 0xd8, 's', 'p', 'r', 'e', 'a', 'd', 0xff, 0xd9}, + }, + }, + { + name: "full delayed", + input: []byte{ + 0xff, 0xd8, 'f', 'u', 'l', 'l', 0xff, 0xd9, + 0xff, 0xd8, 'f', 'r', 'a', 'm', 'e', 0xff, 0xd9, + 0xff, 0xd8, 'w', 'i', 't', 'h', 0xff, 0xd9, + 0xff, 0xd8, 'l', 'e', 'n', 'g', 't', 'h', 0xff, 0xd9, + 0xff, 0xd8, 's', 'p', 'r', 'e', 'a', 'd', 0xff, 0xd9, + }, + delay: time.Millisecond, + want: [][]byte{ + {0xff, 0xd8, 'f', 'u', 'l', 'l', 0xff, 0xd9}, + {0xff, 0xd8, 'f', 'r', 'a', 'm', 'e', 0xff, 0xd9}, + {0xff, 0xd8, 'w', 'i', 't', 'h', 0xff, 0xd9}, + {0xff, 0xd8, 'l', 'e', 'n', 'g', 't', 'h', 0xff, 0xd9}, + {0xff, 0xd8, 's', 'p', 'r', 'e', 'a', 'd', 0xff, 0xd9}, + }, + }, +} + +func TestMJEG(t *testing.T) { + for _, test := range mjpegTests { + var buf chunkWriter + err := MJPEG(&buf, bytes.NewReader(test.input), test.delay) + if fmt.Sprint(err) != fmt.Sprint(test.err) { + t.Errorf("unexpected error for %q: got:%v want:%v", test.name, err, test.err) + } + if err != nil { + continue + } + got := [][]byte(buf) + if !reflect.DeepEqual(got, test.want) { + t.Errorf("unexpected result for %q:\ngot :%#v\nwant:%#v", test.name, got, test.want) + } + } +} + +type chunkWriter [][]byte + +func (w *chunkWriter) Write(b []byte) (int, error) { + *w = append(*w, b) + return len(b), nil +}