gojay

commit 90d7e8221793880a06e933749224ae2810271e35
parent 688c5d008625b62011496858a3e55f852dccd40f
Author: francoispqt <francois@parquet.ninja>
Date:   Sat, 27 Oct 2018 23:59:24 +0800

make parsing of big floats possible by truncating them at high precision

Diffstat:
M
decode_number.go
 | 
3
++-
M
decode_number_float.go
 | 
17
++++++++++++-----
M
decode_number_float_test.go
 | 
26
+++++++++++++++++---------

3 files changed, 31 insertions(+), 15 deletions(-)
diff --git a/decode_number.go b/decode_number.go
@@ -24,7 +24,7 @@ const maxInt16Length = 5
 const maxInt8Length = 3
 const invalidNumber = int8(-1)
 
-var pow10uint64 = [20]uint64{
+var pow10uint64 = [21]uint64{
 	0,
 	1,
 	10,
@@ -45,6 +45,7 @@ var pow10uint64 = [20]uint64{
 	10000000000000000,
 	100000000000000000,
 	1000000000000000000,
+	10000000000000000000,
 }
 
 var skipNumberEndCursorIncrement [256]int
diff --git a/decode_number_float.go b/decode_number_float.go
@@ -125,7 +125,9 @@ func (dec *Decoder) getFloat() (float64, error) {
 				c := dec.data[i]
 				if isDigit(c) {
 					end = i
-					beforeDecimal = (beforeDecimal << 3) + (beforeDecimal << 1)
+					if v := (beforeDecimal << 3) + (beforeDecimal << 1); v >= beforeDecimal {
+						beforeDecimal = v
+					}
 					continue
 				} else if (c == 'e' || c == 'E') && j < i-1 {
 					afterDecimal := dec.atoi64(start, end)
@@ -156,13 +158,18 @@ func (dec *Decoder) getFloat() (float64, error) {
 			if end >= dec.length || end < start {
 				return 0, dec.raiseInvalidJSONErr(dec.cursor)
 			}
-			// then we add both integers
-			// then we divide the number by the power found
-			afterDecimal := dec.atoi64(start, end)
+			var afterDecimal int64
 			expI := end - start + 2
+			// if exp is too long, just cut the number
 			if expI >= len(pow10uint64) || expI < 0 {
-				return 0, dec.raiseInvalidJSONErr(dec.cursor)
+				expI = len(pow10uint64) - 2
+				afterDecimal = dec.atoi64(start, start+expI-2)
+			} else {
+				// then we add both integers
+				// then we divide the number by the power found
+				afterDecimal = dec.atoi64(start, end)
 			}
+
 			pow := pow10uint64[expI]
 			return float64(beforeDecimal+afterDecimal) / float64(pow), nil
 		case 'e', 'E':
diff --git a/decode_number_float_test.go b/decode_number_float_test.go
@@ -218,10 +218,14 @@ func TestDecoderFloat64(t *testing.T) {
 			err:            true,
 		},
 		{
-			name:           "basic-exp-too-big",
+			name:           "big float",
 			json:           "1.00232492420002423545849009",
-			expectedResult: 0,
-			err:            true,
+			expectedResult: 1.002325,
+		},
+		{
+			name:           "big float",
+			json:           "5620.1400000000003",
+			expectedResult: 5620.14,
 		},
 		{
 			name:           "basic-exp-too-big",
@@ -262,6 +266,11 @@ func TestDecoderFloat64(t *testing.T) {
 			err:            true,
 			errType:        InvalidUnmarshalError(""),
 		},
+		{
+			name:           "big float",
+			json:           "5620.1400000000003",
+			expectedResult: 5620.1400000000003,
+		},
 	}
 	for _, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
@@ -282,7 +291,7 @@ func TestDecoderFloat64(t *testing.T) {
 				assert.Nil(t, err, "Err must be nil")
 			}
 			if !testCase.skipResult {
-				assert.Equal(t, testCase.expectedResult*1000000, math.Round(v*1000000), fmt.Sprintf("v must be equal to %f", testCase.expectedResult))
+				assert.Equal(t, math.Round(testCase.expectedResult*1000000), math.Round(v*1000000), fmt.Sprintf("v must be equal to %f", testCase.expectedResult))
 			}
 		})
 	}
@@ -540,16 +549,15 @@ func TestDecoderFloat64Null(t *testing.T) {
 		{
 			name:           "basic-exp-too-big",
 			json:           "0e9223372036000000000 ",
-			expectedResult: 0,
+			expectedResult: 1,
 			err:            true,
 			resultIsNil:    true,
 		},
 		{
 			name:           "basic-exp-too-big",
 			json:           "1.00232492420002423545849009",
-			expectedResult: 0,
-			err:            true,
-			resultIsNil:    true,
+			expectedResult: 1.002325,
+			resultIsNil:    false,
 		},
 		{
 			name:           "basic-exp-too-big",
@@ -618,7 +626,7 @@ func TestDecoderFloat64Null(t *testing.T) {
 			if testCase.resultIsNil {
 				assert.Nil(t, v)
 			} else {
-				assert.Equal(t, testCase.expectedResult*1000000, math.Round(*v*1000000), fmt.Sprintf("v must be equal to %f", testCase.expectedResult))
+				assert.Equal(t, math.Round(testCase.expectedResult*1000000), math.Round(*v*1000000), fmt.Sprintf("v must be equal to %f", testCase.expectedResult))
 			}
 		})
 	}