diff --git a/decode_test.go b/decode_test.go
index 8319b0f..f52069b 100644
--- a/decode_test.go
+++ b/decode_test.go
@@ -1270,7 +1270,7 @@ var unmarshalTests = []unmarshalTest{
 		in:  `invalid`, // 143
 		ptr: new(json.Number),
 		err: json.NewSyntaxError(
-			`json: json.Number unexpected end of JSON input`,
+			`invalid character 'i' looking for beginning of value`,
 			1,
 		),
 	},
@@ -1998,8 +1998,8 @@ type wrongStringTest struct {
 }
 
 var wrongStringTests = []wrongStringTest{
-	{`{"result":"x"}`, `not at beginning of value`},
-	{`{"result":"foo"}`, `not at beginning of value`},
+	{`{"result":"x"}`, `invalid character 'x' looking for beginning of value`},
+	{`{"result":"foo"}`, `invalid character 'f' looking for beginning of value`},
 	{`{"result":"123"}`, `json: cannot unmarshal number into Go struct field WrongString.Message of type string`},
 	{`{"result":123}`, `json: cannot unmarshal number into Go struct field WrongString.Message of type string`},
 	{`{"result":"\""}`, `json: string unexpected end of JSON input`},
@@ -2691,10 +2691,10 @@ func TestUnmarshalErrorAfterMultipleJSON(t *testing.T) {
 		err error
 	}{{
 		in:  `1 false null :`,
-		err: json.NewSyntaxError("not at beginning of value", 14),
+		err: json.NewSyntaxError("invalid character '\x00' looking for beginning of value", 14),
 	}, {
 		in:  `1 [] [,]`,
-		err: json.NewSyntaxError("not at beginning of value", 6),
+		err: json.NewSyntaxError("invalid character ',' looking for beginning of value", 6),
 	}, {
 		in:  `1 [] [true:]`,
 		err: json.NewSyntaxError("json: slice unexpected end of JSON input", 10),
diff --git a/internal/decoder/func.go b/internal/decoder/func.go
index 75afe75..ee35637 100644
--- a/internal/decoder/func.go
+++ b/internal/decoder/func.go
@@ -76,7 +76,7 @@ func (d *funcDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) err
 			}
 		}
 	}
-	return errors.ErrNotAtBeginningOfValue(start)
+	return errors.ErrInvalidBeginningOfValue(s.buf[s.cursor], s.totalOffset())
 }
 
 func (d *funcDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.Pointer) (int64, error) {
@@ -137,5 +137,5 @@ func (d *funcDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.
 			}
 		}
 	}
-	return 0, errors.ErrNotAtBeginningOfValue(start)
+	return cursor, errors.ErrInvalidBeginningOfValue(buf[cursor], cursor)
 }
diff --git a/internal/decoder/interface.go b/internal/decoder/interface.go
index ea1b4aa..4dbb4be 100644
--- a/internal/decoder/interface.go
+++ b/internal/decoder/interface.go
@@ -277,7 +277,7 @@ func (d *interfaceDecoder) decodeStreamEmptyInterface(s *Stream, depth int64, p
 		}
 		break
 	}
-	return errors.ErrNotAtBeginningOfValue(s.totalOffset())
+	return errors.ErrInvalidBeginningOfValue(c, s.totalOffset())
 }
 
 type emptyInterface struct {
@@ -454,5 +454,5 @@ func (d *interfaceDecoder) decodeEmptyInterface(ctx *RuntimeContext, cursor, dep
 		**(**interface{})(unsafe.Pointer(&p)) = nil
 		return cursor, nil
 	}
-	return cursor, errors.ErrNotAtBeginningOfValue(cursor)
+	return cursor, errors.ErrInvalidBeginningOfValue(buf[cursor], cursor)
 }
diff --git a/internal/decoder/number.go b/internal/decoder/number.go
index c50d62b..bf63773 100644
--- a/internal/decoder/number.go
+++ b/internal/decoder/number.go
@@ -52,6 +52,7 @@ func (d *numberDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsaf
 }
 
 func (d *numberDecoder) decodeStreamByte(s *Stream) ([]byte, error) {
+	start := s.cursor
 	for {
 		switch s.char() {
 		case ' ', '\n', '\t', '\r':
@@ -76,6 +77,9 @@ func (d *numberDecoder) decodeStreamByte(s *Stream) ([]byte, error) {
 		}
 	}
 ERROR:
+	if s.cursor == start {
+		return nil, errors.ErrInvalidBeginningOfValue(s.char(), s.totalOffset())
+	}
 	return nil, errors.ErrUnexpectedEndOfJSON("json.Number", s.totalOffset())
 }
 
diff --git a/internal/decoder/string.go b/internal/decoder/string.go
index 5b1a00b..50e8835 100644
--- a/internal/decoder/string.go
+++ b/internal/decoder/string.go
@@ -281,7 +281,7 @@ func (d *stringDecoder) decodeStreamByte(s *Stream) ([]byte, error) {
 		}
 		break
 	}
-	return nil, errors.ErrNotAtBeginningOfValue(s.totalOffset())
+	return nil, errors.ErrInvalidBeginningOfValue(s.char(), s.totalOffset())
 }
 
 func (d *stringDecoder) decodeByte(buf []byte, cursor int64) ([]byte, int64, error) {
@@ -356,7 +356,7 @@ func (d *stringDecoder) decodeByte(buf []byte, cursor int64) ([]byte, int64, err
 			cursor += 4
 			return nil, cursor, nil
 		default:
-			return nil, 0, errors.ErrNotAtBeginningOfValue(cursor)
+			return nil, 0, errors.ErrInvalidBeginningOfValue(buf[cursor], cursor)
 		}
 	}
 }
diff --git a/internal/decoder/struct.go b/internal/decoder/struct.go
index d467b0d..2c64680 100644
--- a/internal/decoder/struct.go
+++ b/internal/decoder/struct.go
@@ -261,7 +261,7 @@ func decodeKeyByBitmapUint8(d *structDecoder, buf []byte, cursor int64) (int64,
 				cursor++
 			}
 		default:
-			return cursor, nil, errors.ErrNotAtBeginningOfValue(cursor)
+			return cursor, nil, errors.ErrInvalidBeginningOfValue(char(b, cursor), cursor)
 		}
 	}
 }
@@ -324,7 +324,7 @@ func decodeKeyByBitmapUint16(d *structDecoder, buf []byte, cursor int64) (int64,
 				cursor++
 			}
 		default:
-			return cursor, nil, errors.ErrNotAtBeginningOfValue(cursor)
+			return cursor, nil, errors.ErrInvalidBeginningOfValue(char(b, cursor), cursor)
 		}
 	}
 }
@@ -376,7 +376,7 @@ func decodeKeyByBitmapUint8Stream(d *structDecoder, s *Stream) (*structFieldSet,
 				_, cursor, p = s.stat()
 				continue
 			}
-			return nil, "", errors.ErrNotAtBeginningOfValue(s.totalOffset())
+			return nil, "", errors.ErrInvalidBeginningOfValue(char(p, cursor), s.totalOffset())
 		case '"':
 			cursor++
 		FIRST_CHAR:
@@ -443,7 +443,7 @@ func decodeKeyByBitmapUint8Stream(d *structDecoder, s *Stream) (*structFieldSet,
 				cursor++
 			}
 		default:
-			return nil, "", errors.ErrNotAtBeginningOfValue(s.totalOffset())
+			return nil, "", errors.ErrInvalidBeginningOfValue(char(p, cursor), s.totalOffset())
 		}
 	}
 }
@@ -463,7 +463,7 @@ func decodeKeyByBitmapUint16Stream(d *structDecoder, s *Stream) (*structFieldSet
 				_, cursor, p = s.stat()
 				continue
 			}
-			return nil, "", errors.ErrNotAtBeginningOfValue(s.totalOffset())
+			return nil, "", errors.ErrInvalidBeginningOfValue(char(p, cursor), s.totalOffset())
 		case '"':
 			cursor++
 		FIRST_CHAR:
@@ -530,7 +530,7 @@ func decodeKeyByBitmapUint16Stream(d *structDecoder, s *Stream) (*structFieldSet
 				cursor++
 			}
 		default:
-			return nil, "", errors.ErrNotAtBeginningOfValue(s.totalOffset())
+			return nil, "", errors.ErrInvalidBeginningOfValue(char(p, cursor), s.totalOffset())
 		}
 	}
 }
@@ -653,7 +653,7 @@ func (d *structDecoder) DecodeStream(s *Stream, depth int64, p unsafe.Pointer) e
 		return nil
 	default:
 		if s.char() != '{' {
-			return errors.ErrNotAtBeginningOfValue(s.totalOffset())
+			return errors.ErrInvalidBeginningOfValue(s.char(), s.totalOffset())
 		}
 	}
 	s.cursor++
@@ -740,7 +740,7 @@ func (d *structDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsaf
 		return cursor, nil
 	case '{':
 	default:
-		return 0, errors.ErrNotAtBeginningOfValue(cursor)
+		return 0, errors.ErrInvalidBeginningOfValue(char(b, cursor), cursor)
 	}
 	cursor++
 	cursor = skipWhiteSpace(buf, cursor)
diff --git a/internal/errors/error.go b/internal/errors/error.go
index 329e2f1..d58e39f 100644
--- a/internal/errors/error.go
+++ b/internal/errors/error.go
@@ -155,3 +155,10 @@ func ErrInvalidCharacter(c byte, context string, cursor int64) *SyntaxError {
 		Offset: cursor,
 	}
 }
+
+func ErrInvalidBeginningOfValue(c byte, cursor int64) *SyntaxError {
+	return &SyntaxError{
+		msg:    fmt.Sprintf("invalid character '%c' looking for beginning of value", c),
+		Offset: cursor,
+	}
+}