commit 83efa1992ee4590a67ef36821854b169147c341f
parent 0daebb40c873255a06be8cc7dabef7b5bc1d6c0a
Author: francoispqt <francois@parquet.ninja>
Date: Thu, 24 May 2018 23:39:07 +0800
change way we get exponent to return an error, fix crashers found in number parsing
Diffstat:
11 files changed, 217 insertions(+), 80 deletions(-)
diff --git a/decode_number.go b/decode_number.go
@@ -79,7 +79,7 @@ func (dec *Decoder) skipNumber() (int, error) {
return end, nil
}
-func (dec *Decoder) getExponent() int64 {
+func (dec *Decoder) getExponent() (int64, error) {
start := dec.cursor
end := dec.cursor
for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
@@ -88,7 +88,8 @@ func (dec *Decoder) getExponent() int64 {
end = dec.cursor + 1
case '-':
dec.cursor++
- return -dec.getExponent()
+ exp, err := dec.getExponent()
+ return -exp, err
case '+':
dec.cursor++
return dec.getExponent()
@@ -96,15 +97,14 @@ func (dec *Decoder) getExponent() int64 {
// if nothing return 0
// could raise error
if start == end {
- dec.raiseInvalidJSONErr(dec.cursor)
- return 0
+ return 0, dec.raiseInvalidJSONErr(dec.cursor)
}
- return dec.atoi64(start, end-1)
+ return dec.atoi64(start, end-1), nil
}
}
if start == end {
- dec.raiseInvalidJSONErr(dec.cursor)
- return 0
+
+ return 0, dec.raiseInvalidJSONErr(dec.cursor)
}
- return dec.atoi64(start, end-1)
+ return dec.atoi64(start, end-1), nil
}
diff --git a/decode_number_float.go b/decode_number_float.go
@@ -92,9 +92,12 @@ func (dec *Decoder) getFloat() (float64, error) {
}
pow := pow10uint64[expI]
floatVal := float64(beforeDecimal+afterDecimal) / float64(pow)
- exp := dec.getExponent()
+ exp, err := dec.getExponent()
+ if err != nil {
+ return 0, err
+ }
pExp := (exp + (exp >> 31)) ^ (exp >> 31) + 1 // abs
- if pExp >= int64(len(pow10uint64)) {
+ if pExp >= int64(len(pow10uint64)) || pExp < 0 {
return 0, dec.raiseInvalidJSONErr(dec.cursor)
}
// if exponent is negative
@@ -123,9 +126,12 @@ func (dec *Decoder) getFloat() (float64, error) {
// we get part before decimal as integer
beforeDecimal := uint64(dec.atoi64(start, end))
// get exponent
- exp := dec.getExponent()
+ exp, err := dec.getExponent()
+ if err != nil {
+ return 0, err
+ }
pExp := (exp + (exp >> 31)) ^ (exp >> 31) + 1 // abs
- if pExp >= int64(len(pow10uint64)) {
+ if pExp >= int64(len(pow10uint64)) || pExp < 0 {
return 0, dec.raiseInvalidJSONErr(dec.cursor)
}
// if exponent is negative
@@ -235,9 +241,12 @@ func (dec *Decoder) getFloat32() (float32, error) {
}
pow := pow10uint64[expI]
floatVal := float32(beforeDecimal+afterDecimal) / float32(pow)
- exp := dec.getExponent()
+ exp, err := dec.getExponent()
+ if err != nil {
+ return 0, err
+ }
pExp := (exp + (exp >> 31)) ^ (exp >> 31) + 1 // abs
- if pExp >= int64(len(pow10uint64)) {
+ if pExp >= int64(len(pow10uint64)) || pExp < 0 {
return 0, dec.raiseInvalidJSONErr(dec.cursor)
}
// if exponent is negative
@@ -266,10 +275,13 @@ func (dec *Decoder) getFloat32() (float32, error) {
// we get part before decimal as integer
beforeDecimal := uint32(dec.atoi32(start, end))
// get exponent
- exp := dec.getExponent()
+ exp, err := dec.getExponent()
+ if err != nil {
+ return 0, err
+ }
pExp := (exp + (exp >> 31)) ^ (exp >> 31) + 1
// log.Print(exp, " after")
- if pExp >= int64(len(pow10uint64)) {
+ if pExp >= int64(len(pow10uint64)) || pExp < 0 {
return 0, dec.raiseInvalidJSONErr(dec.cursor)
}
// if exponent is negative
diff --git a/decode_number_float_test.go b/decode_number_float_test.go
@@ -80,6 +80,12 @@ func TestDecoderFloat64(t *testing.T) {
err: true,
},
{
+ name: "exponent-err-",
+ json: "0.1e",
+ expectedResult: 0,
+ err: true,
+ },
+ {
name: "exp-err",
json: "0e-20",
expectedResult: 0,
@@ -197,6 +203,12 @@ func TestDecoderFloat64(t *testing.T) {
},
{
name: "basic-exp-too-big",
+ json: "0e9223372036000000000 ",
+ expectedResult: 0,
+ err: true,
+ },
+ {
+ name: "basic-exp-too-big",
json: "1.00232492420002423545849009",
expectedResult: 0,
err: true,
@@ -215,6 +227,18 @@ func TestDecoderFloat64(t *testing.T) {
errType: InvalidJSONError(""),
},
{
+ name: "exponent-err",
+ json: "0.1e",
+ expectedResult: 0,
+ err: true,
+ },
+ {
+ name: "exponent-err",
+ json: "0e",
+ expectedResult: 0,
+ err: true,
+ },
+ {
name: "error",
json: "-83zez4",
expectedResult: 0,
@@ -382,6 +406,12 @@ func TestDecoderFloat32(t *testing.T) {
errType: InvalidJSONError(""),
},
{
+ name: "exponent-err-",
+ json: "0.1e",
+ expectedResult: 0,
+ err: true,
+ },
+ {
name: "basic-negative-err",
json: "-q",
expectedResult: 0,
@@ -513,6 +543,12 @@ func TestDecoderFloat32(t *testing.T) {
errType: InvalidJSONError(""),
},
{
+ name: "exponent-err",
+ json: "0e",
+ expectedResult: 0,
+ err: true,
+ },
+ {
name: "invalid-type",
json: `"string"`,
expectedResult: 0,
diff --git a/decode_number_int.go b/decode_number_int.go
@@ -170,11 +170,15 @@ func (dec *Decoder) getInt16() (int16, error) {
pow := pow10uint64[expI]
floatVal := float64(beforeDecimal+afterDecimal) / float64(pow)
// we have the floating value, now multiply by the exponent
- exp := dec.getExponent()
- if +exp+1 >= int64(len(pow10uint64)) {
+ exp, err := dec.getExponent()
+ if err != nil {
+ return 0, err
+ }
+ pExp := (exp + (exp >> 31)) ^ (exp >> 31) + 1 // abs
+ if pExp >= int64(len(pow10uint64)) || pExp < 0 {
return 0, dec.raiseInvalidJSONErr(dec.cursor)
}
- val := floatVal * float64(pow10uint64[exp+1])
+ val := floatVal * float64(pow10uint64[pExp])
return int16(val), nil
case ' ', '\t', '\n', ',', ']', '}':
dec.cursor = j
@@ -218,24 +222,26 @@ func (dec *Decoder) getInt16WithExp(init int16) (int16, error) {
uintv := uint16(digits[dec.data[dec.cursor]])
exp = (exp << 3) + (exp << 1) + uintv
case ' ', '\t', '\n', '}', ',', ']':
- if exp+1 >= uint16(len(pow10uint64)) {
+ exp = exp + 1
+ if exp >= uint16(len(pow10uint64)) {
return 0, dec.raiseInvalidJSONErr(dec.cursor)
}
if sign == -1 {
- return init * (1 / int16(pow10uint64[exp+1])), nil
+ return init * (1 / int16(pow10uint64[exp])), nil
}
- return init * int16(pow10uint64[exp+1]), nil
+ return init * int16(pow10uint64[exp]), nil
default:
return 0, dec.raiseInvalidJSONErr(dec.cursor)
}
}
- if exp+1 >= uint16(len(pow10uint64)) {
+ exp = exp + 1
+ if exp >= uint16(len(pow10uint64)) {
return 0, dec.raiseInvalidJSONErr(dec.cursor)
}
if sign == -1 {
- return init * (1 / int16(pow10uint64[exp+1])), nil
+ return init * (1 / int16(pow10uint64[exp])), nil
}
- return init * int16(pow10uint64[exp+1]), nil
+ return init * int16(pow10uint64[exp]), nil
default:
return 0, dec.raiseInvalidJSONErr(dec.cursor)
}
@@ -352,11 +358,15 @@ func (dec *Decoder) getInt8() (int8, error) {
pow := pow10uint64[expI]
floatVal := float64(beforeDecimal+afterDecimal) / float64(pow)
// we have the floating value, now multiply by the exponent
- exp := dec.getExponent()
- if +exp+1 >= int64(len(pow10uint64)) {
+ exp, err := dec.getExponent()
+ if err != nil {
+ return 0, err
+ }
+ pExp := (exp + (exp >> 31)) ^ (exp >> 31) + 1 // abs
+ if pExp >= int64(len(pow10uint64)) || pExp < 0 {
return 0, dec.raiseInvalidJSONErr(dec.cursor)
}
- val := floatVal * float64(pow10uint64[exp+1])
+ val := floatVal * float64(pow10uint64[pExp])
return int8(val), nil
case ' ', '\t', '\n', ',', ']', '}':
dec.cursor = j
@@ -534,11 +544,15 @@ func (dec *Decoder) getInt32() (int32, error) {
pow := pow10uint64[expI]
floatVal := float64(beforeDecimal+afterDecimal) / float64(pow)
// we have the floating value, now multiply by the exponent
- exp := dec.getExponent()
- if +exp+1 >= int64(len(pow10uint64)) {
+ exp, err := dec.getExponent()
+ if err != nil {
+ return 0, err
+ }
+ pExp := (exp + (exp >> 31)) ^ (exp >> 31) + 1 // abs
+ if pExp >= int64(len(pow10uint64)) || pExp < 0 {
return 0, dec.raiseInvalidJSONErr(dec.cursor)
}
- val := floatVal * float64(pow10uint64[exp+1])
+ val := floatVal * float64(pow10uint64[pExp])
return int32(val), nil
case ' ', '\t', '\n', ',', ']', '}':
dec.cursor = j
@@ -720,9 +734,12 @@ func (dec *Decoder) getInt64() (int64, error) {
pow := pow10uint64[expI]
floatVal := float64(beforeDecimal+afterDecimal) / float64(pow)
// we have the floating value, now multiply by the exponent
- exp := dec.getExponent()
+ exp, err := dec.getExponent()
+ if err != nil {
+ return 0, err
+ }
pExp := (exp + (exp >> 31)) ^ (exp >> 31) + 1 // abs
- if pExp >= int64(len(pow10uint64)) {
+ if pExp >= int64(len(pow10uint64)) || pExp < 0 {
return 0, dec.raiseInvalidJSONErr(dec.cursor)
}
val := floatVal * float64(pow10uint64[pExp])
diff --git a/decode_number_int_test.go b/decode_number_int_test.go
@@ -216,6 +216,12 @@ func TestDecoderInt(t *testing.T) {
errType: InvalidJSONError(""),
},
{
+ name: "exponent-err-",
+ json: "0.1e",
+ expectedResult: 0,
+ err: true,
+ },
+ {
name: "error5",
json: "0E40",
expectedResult: 0,
@@ -229,6 +235,7 @@ func TestDecoderInt(t *testing.T) {
err: true,
errType: InvalidJSONError(""),
},
+
{
name: "error7",
json: "-5.e-2",
@@ -539,6 +546,12 @@ func TestDecoderInt64(t *testing.T) {
errType: InvalidJSONError(""),
},
{
+ name: "exponent-err-",
+ json: "0.1e",
+ expectedResult: 0,
+ err: true,
+ },
+ {
name: "error6",
json: "0.e-9",
expectedResult: 0,
@@ -802,6 +815,12 @@ func TestDecoderInt32(t *testing.T) {
expectedResult: -800000,
},
{
+ name: "exponent-err-",
+ json: "0.1e",
+ expectedResult: 0,
+ err: true,
+ },
+ {
name: "exponent-err-too-big",
json: "0.1e10000000000000000000",
expectedResult: 0,
@@ -1127,6 +1146,12 @@ func TestDecoderInt16(t *testing.T) {
expectedResult: -100,
},
{
+ name: "exponent-err-",
+ json: "0.1e",
+ expectedResult: 0,
+ err: true,
+ },
+ {
name: "exponent-err-too-big",
json: "0.1e10000000000000000000",
expectedResult: 0,
@@ -1485,6 +1510,12 @@ func TestDecoderInt8(t *testing.T) {
errType: InvalidJSONError(""),
},
{
+ name: "exponent-err-",
+ json: "0.1e",
+ expectedResult: 0,
+ err: true,
+ },
+ {
name: "exponent-err-too-big",
json: "0.1e10000000000000000000",
expectedResult: 0,
diff --git a/decode_number_uint.go b/decode_number_uint.go
@@ -20,20 +20,18 @@ func (dec *Decoder) decodeUint8(v *uint8) error {
case ' ', '\n', '\t', '\r', ',':
continue
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
- val, err := dec.getUint8(c)
+ val, err := dec.getUint8()
if err != nil {
return err
}
*v = val
return nil
- case '-':
- dec.cursor = dec.cursor + 1
- val, err := dec.getUint8(dec.data[dec.cursor])
+ case '-': // if negative, we just set it to 0
+ err := dec.skipData()
if err != nil {
return err
}
- // unsigned int so we don't bother with the sign
- *v = val
+ *v = 0
return nil
case 'n':
dec.cursor++
@@ -54,7 +52,7 @@ func (dec *Decoder) decodeUint8(v *uint8) error {
return dec.raiseInvalidJSONErr(dec.cursor)
}
-func (dec *Decoder) getUint8(b byte) (uint8, error) {
+func (dec *Decoder) getUint8() (uint8, error) {
var end = dec.cursor
var start = dec.cursor
// look for following numbers
@@ -91,20 +89,18 @@ func (dec *Decoder) decodeUint16(v *uint16) error {
case ' ', '\n', '\t', '\r', ',':
continue
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
- val, err := dec.getUint16(c)
+ val, err := dec.getUint16()
if err != nil {
return err
}
*v = val
return nil
case '-':
- dec.cursor = dec.cursor + 1
- val, err := dec.getUint16(dec.data[dec.cursor])
+ err := dec.skipData()
if err != nil {
return err
}
- // unsigned int so we don't bother with the sign
- *v = val
+ *v = 0
return nil
case 'n':
dec.cursor++
@@ -125,7 +121,7 @@ func (dec *Decoder) decodeUint16(v *uint16) error {
return dec.raiseInvalidJSONErr(dec.cursor)
}
-func (dec *Decoder) getUint16(b byte) (uint16, error) {
+func (dec *Decoder) getUint16() (uint16, error) {
var end = dec.cursor
var start = dec.cursor
// look for following numbers
@@ -162,20 +158,18 @@ func (dec *Decoder) decodeUint32(v *uint32) error {
case ' ', '\n', '\t', '\r', ',':
continue
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
- val, err := dec.getUint32(c)
+ val, err := dec.getUint32()
if err != nil {
return err
}
*v = val
return nil
case '-':
- dec.cursor = dec.cursor + 1
- val, err := dec.getUint32(dec.data[dec.cursor])
+ err := dec.skipData()
if err != nil {
return err
}
- // unsigned int so we don't bother with the sign
- *v = val
+ *v = 0
return nil
case 'n':
dec.cursor++
@@ -196,7 +190,7 @@ func (dec *Decoder) decodeUint32(v *uint32) error {
return dec.raiseInvalidJSONErr(dec.cursor)
}
-func (dec *Decoder) getUint32(b byte) (uint32, error) {
+func (dec *Decoder) getUint32() (uint32, error) {
var end = dec.cursor
var start = dec.cursor
// look for following numbers
@@ -232,20 +226,18 @@ func (dec *Decoder) decodeUint64(v *uint64) error {
case ' ', '\n', '\t', '\r', ',':
continue
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
- val, err := dec.getUint64(c)
+ val, err := dec.getUint64()
if err != nil {
return err
}
*v = val
return nil
case '-':
- dec.cursor = dec.cursor + 1
- val, err := dec.getUint64(dec.data[dec.cursor])
+ err := dec.skipData()
if err != nil {
return err
}
- // unsigned int so we don't bother with the sign
- *v = val
+ *v = 0
return nil
case 'n':
dec.cursor++
@@ -266,7 +258,7 @@ func (dec *Decoder) decodeUint64(v *uint64) error {
return dec.raiseInvalidJSONErr(dec.cursor)
}
-func (dec *Decoder) getUint64(b byte) (uint64, error) {
+func (dec *Decoder) getUint64() (uint64, error) {
var end = dec.cursor
var start = dec.cursor
// look for following numbers
diff --git a/decode_number_uint_test.go b/decode_number_uint_test.go
@@ -30,7 +30,7 @@ func TestDecoderUint64(t *testing.T) {
{
name: "basic-negative",
json: "-2",
- expectedResult: 2,
+ expectedResult: 0,
},
{
name: "basic-null",
@@ -77,7 +77,7 @@ func TestDecoderUint64(t *testing.T) {
{
name: "basic-negative2",
json: "-2349557",
- expectedResult: 2349557,
+ expectedResult: 0,
},
{
name: "basic-float",
@@ -87,7 +87,7 @@ func TestDecoderUint64(t *testing.T) {
{
name: "basic-float2",
json: "-7.8876",
- expectedResult: 7,
+ expectedResult: 0,
},
{
name: "error1",
@@ -182,7 +182,7 @@ func TestDecoderUint32(t *testing.T) {
{
name: "basic-negative",
json: "-2",
- expectedResult: 2,
+ expectedResult: 0,
},
{
name: "basic-null",
@@ -206,7 +206,7 @@ func TestDecoderUint32(t *testing.T) {
{
name: "basic-negative2",
json: "-2349557",
- expectedResult: 2349557,
+ expectedResult: 0,
},
{
name: "basic-big",
@@ -239,7 +239,7 @@ func TestDecoderUint32(t *testing.T) {
{
name: "basic-float2",
json: "-7.8876",
- expectedResult: 7,
+ expectedResult: 0,
},
{
name: "error",
@@ -342,7 +342,7 @@ func TestDecoderUint16(t *testing.T) {
{
name: "basic-negative",
json: "-2",
- expectedResult: 2,
+ expectedResult: 0,
},
{
name: "basic-null",
@@ -380,7 +380,7 @@ func TestDecoderUint16(t *testing.T) {
{
name: "basic-negative2",
json: "-24467",
- expectedResult: 24467,
+ expectedResult: 0,
},
{
name: "basic-big",
@@ -419,7 +419,7 @@ func TestDecoderUint16(t *testing.T) {
{
name: "basic-float2",
json: "-7.8876",
- expectedResult: 7,
+ expectedResult: 0,
},
{
name: "error",
@@ -530,7 +530,7 @@ func TestDecoderUint8(t *testing.T) {
{
name: "basic-negative",
json: "-2",
- expectedResult: 2,
+ expectedResult: 0,
},
{
name: "basic-null",
@@ -554,7 +554,7 @@ func TestDecoderUint8(t *testing.T) {
{
name: "basic-negative2",
json: "-234",
- expectedResult: 234,
+ expectedResult: 0,
},
{
name: "basic-big",
@@ -595,7 +595,7 @@ func TestDecoderUint8(t *testing.T) {
{
name: "basic-float2",
json: "-7.8876",
- expectedResult: 7,
+ expectedResult: 0,
},
{
name: "error",
diff --git a/decode_object_test.go b/decode_object_test.go
@@ -410,6 +410,48 @@ func TestDecodeObjectBasic(t *testing.T) {
},
err: false,
},
+ {
+ name: "basic-skip-data",
+ json: `{
+ "testStr": "hello world!",
+ "testInt": 4535,
+ "testBool": true,
+ "testFloat32": 2.345,
+ "testFloat64": 123.677,
+ "testInt8": 23,
+ "skipObject": {
+ "escapedString": "string with escaped \\n new line"
+ },
+ "testInt16": 1245,
+ "testInt32": 456778,
+ "testInt64": 1446685358,
+ "testUint8": -255,
+ "skipArray": [[],[],{}],
+ "testUint16": 3455,
+ "skipBool": true,
+ "skipNull": null,
+ "testUint32": 343443,
+ "testUint64": 545665757,
+ "skipString": "skipping string with escaped \\n new line",
+ "skipInt": 3,
+ }`,
+ expectedResult: testObject{
+ testStr: "hello world!",
+ testInt: 4535,
+ testBool: true,
+ testFloat32: 2.345,
+ testFloat64: 123.677,
+ testInt8: 23,
+ testInt16: 1245,
+ testInt32: 456778,
+ testInt64: 1446685358,
+ testUint8: 0,
+ testUint16: 3455,
+ testUint32: 343443,
+ testUint64: 545665757,
+ },
+ err: false,
+ },
}
for _, testCase := range testCases {
diff --git a/decode_test.go b/decode_test.go
@@ -100,7 +100,7 @@ func TestUnmarshalAllTypes(t *testing.T) {
expectations: func(err error, v interface{}, t *testing.T) {
vt := v.(*uint64)
assert.Nil(t, err, "err must be nil")
- assert.Equal(t, uint64(1), *vt, "v must be equal to 1")
+ assert.Equal(t, uint64(0), *vt, "v must be equal to 1")
},
},
{
@@ -130,7 +130,7 @@ func TestUnmarshalAllTypes(t *testing.T) {
expectations: func(err error, v interface{}, t *testing.T) {
vt := v.(*uint32)
assert.Nil(t, err, "err must be nil")
- assert.Equal(t, uint32(1), *vt, "v must be equal to 1")
+ assert.Equal(t, uint32(0), *vt, "v must be equal to 1")
},
},
{
@@ -338,7 +338,7 @@ func TestDecodeAllTypes(t *testing.T) {
expectations: func(err error, v interface{}, t *testing.T) {
vt := v.(*uint64)
assert.Nil(t, err, "err must be nil")
- assert.Equal(t, uint64(1), *vt, "v must be equal to 1")
+ assert.Equal(t, uint64(0), *vt, "v must be equal to 1")
},
},
{
@@ -368,7 +368,7 @@ func TestDecodeAllTypes(t *testing.T) {
expectations: func(err error, v interface{}, t *testing.T) {
vt := v.(*uint32)
assert.Nil(t, err, "err must be nil")
- assert.Equal(t, uint32(1), *vt, "v must be equal to 1")
+ assert.Equal(t, uint32(0), *vt, "v must be equal to 1")
},
},
{
diff --git a/decode_unsafe_test.go b/decode_unsafe_test.go
@@ -102,7 +102,7 @@ func TestUnmarshalUnsafeAllTypes(t *testing.T) {
expectations: func(err error, v interface{}, t *testing.T) {
vt := v.(*uint64)
assert.Nil(t, err, "err must be nil")
- assert.Equal(t, uint64(1), *vt, "v must be equal to 1")
+ assert.Equal(t, uint64(0), *vt, "v must be equal to 1")
},
},
{
@@ -132,7 +132,7 @@ func TestUnmarshalUnsafeAllTypes(t *testing.T) {
expectations: func(err error, v interface{}, t *testing.T) {
vt := v.(*uint32)
assert.Nil(t, err, "err must be nil")
- assert.Equal(t, uint32(1), *vt, "v must be equal to 1")
+ assert.Equal(t, uint32(0), *vt, "v must be equal to 1")
},
},
{
diff --git a/examples/fuzz/main.go b/examples/fuzz/main.go
@@ -5,10 +5,12 @@ import (
)
type user struct {
- id int
- age float64
- name string
- email string
+ id int
+ created uint64
+ age float64
+ name string
+ email string
+ friend *user
}
// implement gojay.UnmarshalerJSONObject
@@ -16,12 +18,17 @@ func (u *user) UnmarshalJSONObject(dec *gojay.Decoder, key string) error {
switch key {
case "id":
return dec.Int(&u.id)
+ case "created":
+ return dec.Uint64(&u.created)
case "age":
return dec.Float(&u.age)
case "name":
return dec.String(&u.name)
case "email":
return dec.String(&u.email)
+ case "friend":
+ uu := &user{}
+ return dec.Object(uu)
}
return nil
}