Merge pull request #279 from orisano/fix-utf-8-handling

fix: fixed invalid utf8 on stream decoder

internal/decoder/string.go

@@ -239,6 +239,14 @@ func stringBytes(s *Stream) ([]byte, error) {
 			fallthrough
 		default:
 			// multi bytes character
+			if !utf8.FullRune(s.buf[cursor : len(s.buf)-1]) {
+				s.cursor = cursor
+				if s.read() {
+					_, cursor, p = s.stat()
+					continue
+				}
+				goto ERROR
+			}
 			r, _ := utf8.DecodeRune(s.buf[cursor:])
 			b := []byte(string(r))
 			if r == utf8.RuneError {

stream_test.go

@@ -14,7 +14,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"reflect"
-
+	"strconv"
 	"strings"
 	"testing"
 
@@ -502,3 +502,16 @@ func TestGzipStreaming(t *testing.T) {
 		t.Fatalf("Unexpected error: %v", err)
 	}
 }
+
+func TestLongUTF8(t *testing.T) {
+	want := strings.Repeat("あ", 342)
+	r := strings.NewReader(strconv.Quote(want))
+
+	var got string
+	if err := json.NewDecoder(r).Decode(&got); err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	if got != want {
+		t.Errorf("string %q; want = %q", got, want)
+	}
+}