commit 0daebb40c873255a06be8cc7dabef7b5bc1d6c0a
parent 7166fd659b3e557c8be1b04c95527add7a0f5f67
Author: francoispqt <francois@parquet.ninja>
Date: Thu, 24 May 2018 00:01:48 +0800
fix cases where malformed numbers can cause panic
Diffstat:
6 files changed, 304 insertions(+), 29 deletions(-)
diff --git a/decode_number_float.go b/decode_number_float.go
@@ -83,7 +83,7 @@ func (dec *Decoder) getFloat() (float64, error) {
end = i
beforeDecimal = (beforeDecimal << 3) + (beforeDecimal << 1)
continue
- } else if c == 'e' || c == 'E' {
+ } else if (c == 'e' || c == 'E') && j < i-1 {
afterDecimal := dec.atoi64(start, end)
dec.cursor = i + 1
expI := end - start + 2
@@ -93,18 +93,22 @@ func (dec *Decoder) getFloat() (float64, error) {
pow := pow10uint64[expI]
floatVal := float64(beforeDecimal+afterDecimal) / float64(pow)
exp := dec.getExponent()
- if +exp+1 >= int64(len(pow10uint64)) {
+ pExp := (exp + (exp >> 31)) ^ (exp >> 31) + 1 // abs
+ if pExp >= int64(len(pow10uint64)) {
return 0, dec.raiseInvalidJSONErr(dec.cursor)
}
// if exponent is negative
if exp < 0 {
- return float64(floatVal) * (1 / float64(pow10uint64[exp*-1+1])), nil
+ return float64(floatVal) * (1 / float64(pow10uint64[pExp])), nil
}
- return float64(floatVal) * float64(pow10uint64[exp+1]), nil
+ return float64(floatVal) * float64(pow10uint64[pExp]), nil
}
dec.cursor = i
break
}
+ if end >= dec.length || end <= start {
+ return 0, dec.raiseInvalidJSONErr(dec.cursor)
+ }
// then we add both integers
// then we divide the number by the power found
afterDecimal := dec.atoi64(start, end)
@@ -120,14 +124,15 @@ func (dec *Decoder) getFloat() (float64, error) {
beforeDecimal := uint64(dec.atoi64(start, end))
// get exponent
exp := dec.getExponent()
- if +exp+1 >= int64(len(pow10uint64)) {
+ pExp := (exp + (exp >> 31)) ^ (exp >> 31) + 1 // abs
+ if pExp >= int64(len(pow10uint64)) {
return 0, dec.raiseInvalidJSONErr(dec.cursor)
}
// if exponent is negative
if exp < 0 {
- return float64(beforeDecimal) * (1 / float64(pow10uint64[exp*-1+1])), nil
+ return float64(beforeDecimal) * (1 / float64(pow10uint64[pExp])), nil
}
- return float64(beforeDecimal) * float64(pow10uint64[exp+1]), nil
+ return float64(beforeDecimal) * float64(pow10uint64[pExp]), nil
case ' ', '\n', '\t', '\r', ',', '}', ']': // does not have decimal
dec.cursor = j
return float64(dec.atoi64(start, end)), nil
@@ -221,7 +226,7 @@ func (dec *Decoder) getFloat32() (float32, error) {
end = i
beforeDecimal = (beforeDecimal << 3) + (beforeDecimal << 1)
continue
- } else if c == 'e' || c == 'E' {
+ } else if (c == 'e' || c == 'E') && j < i-1 {
afterDecimal := dec.atoi32(start, end)
dec.cursor = i + 1
expI := end - start + 2
@@ -231,18 +236,22 @@ func (dec *Decoder) getFloat32() (float32, error) {
pow := pow10uint64[expI]
floatVal := float32(beforeDecimal+afterDecimal) / float32(pow)
exp := dec.getExponent()
- if +exp+1 >= int64(len(pow10uint64)) {
+ pExp := (exp + (exp >> 31)) ^ (exp >> 31) + 1 // abs
+ if pExp >= int64(len(pow10uint64)) {
return 0, dec.raiseInvalidJSONErr(dec.cursor)
}
// if exponent is negative
if exp < 0 {
- return float32(floatVal) * (1 / float32(pow10uint64[exp*-1+1])), nil
+ return float32(floatVal) * (1 / float32(pow10uint64[pExp])), nil
}
- return float32(floatVal) * float32(pow10uint64[exp+1]), nil
+ return float32(floatVal) * float32(pow10uint64[pExp]), nil
}
dec.cursor = i
break
}
+ if end >= dec.length || end <= start {
+ return 0, dec.raiseInvalidJSONErr(dec.cursor)
+ }
// then we add both integers
// then we divide the number by the power found
afterDecimal := dec.atoi32(start, end)
@@ -258,14 +267,16 @@ func (dec *Decoder) getFloat32() (float32, error) {
beforeDecimal := uint32(dec.atoi32(start, end))
// get exponent
exp := dec.getExponent()
- if +exp+1 >= int64(len(pow10uint64)) {
+ pExp := (exp + (exp >> 31)) ^ (exp >> 31) + 1
+ // log.Print(exp, " after")
+ if pExp >= int64(len(pow10uint64)) {
return 0, dec.raiseInvalidJSONErr(dec.cursor)
}
// if exponent is negative
if exp < 0 {
- return float32(beforeDecimal) * (1 / float32(pow10uint64[exp*-1+1])), nil
+ return float32(beforeDecimal) * (1 / float32(pow10uint64[pExp])), nil
}
- return float32(beforeDecimal) * float32(pow10uint64[exp+1]), nil
+ return float32(beforeDecimal) * float32(pow10uint64[pExp]), nil
case ' ', '\n', '\t', '\r', ',', '}', ']': // does not have decimal
dec.cursor = j
return float32(dec.atoi32(start, end)), nil
diff --git a/decode_number_float_test.go b/decode_number_float_test.go
@@ -15,6 +15,7 @@ func TestDecoderFloat64(t *testing.T) {
name string
json string
expectedResult float64
+ skipResult bool
err bool
errType interface{}
}{
@@ -67,6 +68,42 @@ func TestDecoderFloat64(t *testing.T) {
errType: InvalidJSONError(""),
},
{
+ name: "basic-err1",
+ json: "0.",
+ expectedResult: 0,
+ err: true,
+ },
+ {
+ name: "basic-err2",
+ json: "-1.",
+ expectedResult: 0,
+ err: true,
+ },
+ {
+ name: "exp-err",
+ json: "0e-20",
+ expectedResult: 0,
+ err: true,
+ },
+ {
+ name: "exp-err3",
+ json: "-9e-60",
+ expectedResult: 0,
+ err: true,
+ },
+ {
+ name: "exp-err4",
+ json: "0.e-2",
+ skipResult: true,
+ err: true,
+ },
+ {
+ name: "exp-err5",
+ json: "-5.E-2",
+ skipResult: true,
+ err: true,
+ },
+ {
name: "basic-exponent-positive-positive-exp4",
json: "8e+005",
expectedResult: 800000,
@@ -210,7 +247,9 @@ func TestDecoderFloat64(t *testing.T) {
} else {
assert.Nil(t, err, "Err must be nil")
}
- assert.Equal(t, testCase.expectedResult*1000000, math.Round(v*1000000), fmt.Sprintf("v must be equal to %f", testCase.expectedResult))
+ if !testCase.skipResult {
+ assert.Equal(t, testCase.expectedResult*1000000, math.Round(v*1000000), fmt.Sprintf("v must be equal to %f", testCase.expectedResult))
+ }
})
}
t.Run("pool-error", func(t *testing.T) {
@@ -256,6 +295,7 @@ func TestDecoderFloat32(t *testing.T) {
name string
json string
expectedResult float32
+ skipResult bool
err bool
errType interface{}
}{
@@ -280,6 +320,47 @@ func TestDecoderFloat32(t *testing.T) {
expectedResult: 0,
},
{
+ name: "basic-err1",
+ json: "0.",
+ expectedResult: 0,
+ err: true,
+ },
+ {
+ name: "basic-err2",
+ json: "-1.",
+ expectedResult: 0,
+ err: true,
+ },
+ {
+ name: "exp-err",
+ json: "0e-20",
+ expectedResult: 0,
+ err: true,
+ },
+ {
+ name: "exp-err3",
+ json: "-9e-60",
+ expectedResult: 0,
+ err: true,
+ },
+ {
+ name: "exp-err4",
+ json: "0.e-2",
+ skipResult: true,
+ err: true,
+ },
+ {
+ name: "exp-err5",
+ json: "-5.E-2",
+ skipResult: true,
+ err: true,
+ },
+ {
+ name: "basic-null",
+ json: "null",
+ expectedResult: 0,
+ },
+ {
name: "basic-null-err",
json: "trua",
expectedResult: 0,
@@ -457,7 +538,13 @@ func TestDecoderFloat32(t *testing.T) {
} else {
assert.Nil(t, err, "Err must be nil")
}
- assert.Equal(t, float64(testCase.expectedResult*1000000), math.Round(float64(v*1000000)), fmt.Sprintf("v must be equal to %f", testCase.expectedResult))
+ if !testCase.skipResult {
+ assert.Equal(
+ t,
+ float64(testCase.expectedResult*1000000), math.Round(float64(v*1000000)),
+ fmt.Sprintf("v must be equal to %f", testCase.expectedResult),
+ )
+ }
})
}
t.Run("pool-error", func(t *testing.T) {
diff --git a/decode_number_int.go b/decode_number_int.go
@@ -138,15 +138,18 @@ func (dec *Decoder) getInt16() (int16, error) {
// look for exponent (e,E) as exponent can change the
// way number should be parsed to int.
// if no exponent found, just unmarshal the number before decimal point
- startDecimal := j + 1
- endDecimal := j + 1
j++
+ startDecimal := j
+ endDecimal := j - 1
for ; j < dec.length || dec.read(); j++ {
switch dec.data[j] {
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
endDecimal = j
continue
case 'e', 'E':
+ if startDecimal > endDecimal {
+ return 0, dec.raiseInvalidJSONErr(dec.cursor)
+ }
dec.cursor = j + 1
// can try unmarshalling to int as Exponent might change decimal number to non decimal
// let's get the float value first
@@ -317,15 +320,18 @@ func (dec *Decoder) getInt8() (int8, error) {
// look for exponent (e,E) as exponent can change the
// way number should be parsed to int.
// if no exponent found, just unmarshal the number before decimal point
- startDecimal := j + 1
- endDecimal := j + 1
j++
+ startDecimal := j
+ endDecimal := j - 1
for ; j < dec.length || dec.read(); j++ {
switch dec.data[j] {
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
endDecimal = j
continue
case 'e', 'E':
+ if startDecimal > endDecimal {
+ return 0, dec.raiseInvalidJSONErr(dec.cursor)
+ }
dec.cursor = j + 1
// can try unmarshalling to int as Exponent might change decimal number to non decimal
// let's get the float value first
@@ -495,15 +501,19 @@ func (dec *Decoder) getInt32() (int32, error) {
// look for exponent (e,E) as exponent can change the
// way number should be parsed to int.
// if no exponent found, just unmarshal the number before decimal point
- startDecimal := j + 1
- endDecimal := j + 1
j++
+ startDecimal := j
+ endDecimal := j - 1
for ; j < dec.length || dec.read(); j++ {
switch dec.data[j] {
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
endDecimal = j
continue
case 'e', 'E':
+ // if eg 1.E
+ if startDecimal > endDecimal {
+ return 0, dec.raiseInvalidJSONErr(dec.cursor)
+ }
dec.cursor = j + 1
// can try unmarshalling to int as Exponent might change decimal number to non decimal
// let's get the float value first
@@ -677,15 +687,19 @@ func (dec *Decoder) getInt64() (int64, error) {
// look for exponent (e,E) as exponent can change the
// way number should be parsed to int.
// if no exponent found, just unmarshal the number before decimal point
- startDecimal := j + 1
- endDecimal := j + 1
j++
+ startDecimal := j
+ endDecimal := j - 1
for ; j < dec.length || dec.read(); j++ {
switch dec.data[j] {
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
endDecimal = j
continue
case 'e', 'E':
+ // if eg 1.E
+ if startDecimal > endDecimal {
+ return 0, dec.raiseInvalidJSONErr(dec.cursor)
+ }
dec.cursor = j + 1
// can try unmarshalling to int as Exponent might change decimal number to non decimal
// let's get the float value first
@@ -707,10 +721,11 @@ func (dec *Decoder) getInt64() (int64, error) {
floatVal := float64(beforeDecimal+afterDecimal) / float64(pow)
// we have the floating value, now multiply by the exponent
exp := dec.getExponent()
- if +exp+1 >= int64(len(pow10uint64)) {
+ pExp := (exp + (exp >> 31)) ^ (exp >> 31) + 1 // abs
+ if pExp >= int64(len(pow10uint64)) {
return 0, dec.raiseInvalidJSONErr(dec.cursor)
}
- val := floatVal * float64(pow10uint64[exp+1])
+ val := floatVal * float64(pow10uint64[pExp])
return int64(val), nil
case ' ', '\t', '\n', ',', ']', '}':
dec.cursor = j
diff --git a/decode_number_int_test.go b/decode_number_int_test.go
@@ -209,20 +209,34 @@ func TestDecoderInt(t *testing.T) {
errType: InvalidJSONError(""),
},
{
- name: "error3",
+ name: "error4",
json: "0.E----",
expectedResult: 0,
err: true,
errType: InvalidJSONError(""),
},
{
- name: "error3",
+ name: "error5",
json: "0E40",
expectedResult: 0,
err: true,
errType: InvalidJSONError(""),
},
{
+ name: "error6",
+ json: "0.e-9",
+ expectedResult: 0,
+ err: true,
+ errType: InvalidJSONError(""),
+ },
+ {
+ name: "error7",
+ json: "-5.e-2",
+ expectedResult: 0,
+ err: true,
+ errType: InvalidJSONError(""),
+ },
+ {
name: "invalid-type",
json: `"string"`,
expectedResult: 0,
@@ -237,7 +251,7 @@ func TestDecoderInt(t *testing.T) {
err := Unmarshal(json, &v)
if testCase.err {
assert.NotNil(t, err, "Err must not be nil")
- if testCase.errType != nil {
+ if testCase.errType != nil && err != nil {
assert.IsType(
t,
testCase.errType,
@@ -511,6 +525,34 @@ func TestDecoderInt64(t *testing.T) {
err: true,
},
{
+ name: "error4",
+ json: "0.E----",
+ expectedResult: 0,
+ err: true,
+ errType: InvalidJSONError(""),
+ },
+ {
+ name: "error5",
+ json: "0E40",
+ expectedResult: 0,
+ err: true,
+ errType: InvalidJSONError(""),
+ },
+ {
+ name: "error6",
+ json: "0.e-9",
+ expectedResult: 0,
+ err: true,
+ errType: InvalidJSONError(""),
+ },
+ {
+ name: "error7",
+ json: "-5.e-2",
+ expectedResult: 0,
+ err: true,
+ errType: InvalidJSONError(""),
+ },
+ {
name: "invalid-type",
json: `"string"`,
expectedResult: 0,
@@ -803,6 +845,34 @@ func TestDecoderInt32(t *testing.T) {
errType: InvalidJSONError(""),
},
{
+ name: "error4",
+ json: "0.E----",
+ expectedResult: 0,
+ err: true,
+ errType: InvalidJSONError(""),
+ },
+ {
+ name: "error5",
+ json: "0E40",
+ expectedResult: 0,
+ err: true,
+ errType: InvalidJSONError(""),
+ },
+ {
+ name: "error6",
+ json: "0.e-9",
+ expectedResult: 0,
+ err: true,
+ errType: InvalidJSONError(""),
+ },
+ {
+ name: "error7",
+ json: "-5.e-2",
+ expectedResult: 0,
+ err: true,
+ errType: InvalidJSONError(""),
+ },
+ {
name: "basic-float",
json: "8.32 ",
expectedResult: 8,
@@ -1134,6 +1204,41 @@ func TestDecoderInt16(t *testing.T) {
errType: InvalidJSONError(""),
},
{
+ name: "error4",
+ json: "0.E----",
+ expectedResult: 0,
+ err: true,
+ errType: InvalidJSONError(""),
+ },
+ {
+ name: "error5",
+ json: "0E40",
+ expectedResult: 0,
+ err: true,
+ errType: InvalidJSONError(""),
+ },
+ {
+ name: "error6",
+ json: "0.e-9",
+ expectedResult: 0,
+ err: true,
+ errType: InvalidJSONError(""),
+ },
+ {
+ name: "error7",
+ json: "0.e",
+ expectedResult: 0,
+ err: true,
+ errType: InvalidJSONError(""),
+ },
+ {
+ name: "error8",
+ json: "-5.e-2",
+ expectedResult: 0,
+ err: true,
+ errType: InvalidJSONError(""),
+ },
+ {
name: "invalid-type",
json: `"string"`,
expectedResult: 0,
@@ -1440,6 +1545,48 @@ func TestDecoderInt8(t *testing.T) {
err: true,
},
{
+ name: "error4",
+ json: "0.E----",
+ expectedResult: 0,
+ err: true,
+ errType: InvalidJSONError(""),
+ },
+ {
+ name: "error5",
+ json: "0E40",
+ expectedResult: 0,
+ err: true,
+ errType: InvalidJSONError(""),
+ },
+ {
+ name: "error6",
+ json: "0.e-9",
+ expectedResult: 0,
+ err: true,
+ errType: InvalidJSONError(""),
+ },
+ {
+ name: "error7",
+ json: "0.e",
+ expectedResult: 0,
+ err: true,
+ errType: InvalidJSONError(""),
+ },
+ {
+ name: "error8",
+ json: "-5.e-2",
+ expectedResult: 0,
+ err: true,
+ errType: InvalidJSONError(""),
+ },
+ {
+ name: "error8",
+ json: "-5.01e",
+ expectedResult: 0,
+ err: true,
+ errType: InvalidJSONError(""),
+ },
+ {
name: "invalid-type",
json: `"string"`,
expectedResult: 0,
diff --git a/examples/fuzz/Makefile b/examples/fuzz/Makefile
@@ -0,0 +1,11 @@
+.PHONY: gofuzzbuild
+gofuzzbuild:
+ go-fuzz-build github.com/francoispqt/gojay/examples/fuzz
+
+.PHONY: gofuzz
+gofuzz:
+ go-fuzz -bin=fuzz-fuzz.zip -workdir=.
+
+.PHONY: gofuzzclean
+gofuzzclean:
+ rm -rf corpus crashers suppressions fuzz-fuzz.zip
+\ No newline at end of file
diff --git a/examples/fuzz/main.go b/examples/fuzz/main.go
@@ -6,6 +6,7 @@ import (
type user struct {
id int
+ age float64
name string
email string
}
@@ -15,6 +16,8 @@ func (u *user) UnmarshalJSONObject(dec *gojay.Decoder, key string) error {
switch key {
case "id":
return dec.Int(&u.id)
+ case "age":
+ return dec.Float(&u.age)
case "name":
return dec.String(&u.name)
case "email":
