commit 002972f789c10d5804f875d7a803c245542ebc9d
parent c135fe6615f52b48d922c0a49c88a78bca7253a3
Author: francoispqt <francois@parquet.ninja>
Date: Sun, 13 May 2018 23:16:42 +0800
update escaping sequences for both decoding and encoding
Diffstat:
8 files changed, 150 insertions(+), 60 deletions(-)
diff --git a/README.md b/README.md
@@ -681,11 +681,12 @@ cd $GOPATH/src/github.com/francoispqt/gojay/benchmarks/encoder && make bench
| | ns/op | bytes/op | allocs/op |
|-------------|-------|--------------|-----------|
-| Std Library | 4661 | 496 | 12 |
-| JsonParser | 1313 | 0 | 0 |
-| JsonIter | 899 | 192 | 5 |
+| Std Library | 2547 | 496 | 4 |
+| JsonIter | 2046 | 312 | 12 |
+| JsonParser | 1408 | 0 | 0 |
| EasyJson | 929 | 240 | 2 |
-| GoJay | 662 | 112 | 1 |
+| GoJay | 807 | 256 | 2 |
+| GoJay-unsafe| 712 | 112 | 1 |
### Medium Payload
[benchmark code is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/decoder/decoder_bench_medium_test.go)
@@ -695,10 +696,11 @@ cd $GOPATH/src/github.com/francoispqt/gojay/benchmarks/encoder && make bench
| | ns/op | bytes/op | allocs/op |
|-------------|-------|--------------|-----------|
| Std Library | 30148 | 2152 | 496 |
+| JsonIter | 16309 | 2976 | 80 |
| JsonParser | 7793 | 0 | 0 |
| EasyJson | 7957 | 232 | 6 |
-| JsonIter | 5967 | 496 | 44 |
-| GoJay | 3914 | 128 | 7 |
+| GoJay | 4984 | 2448 | 8 |
+| GoJay-unsafe| 4809 | 144 | 7 |
### Large Payload
[benchmark code is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/decoder/decoder_bench_large_test.go)
@@ -707,10 +709,11 @@ cd $GOPATH/src/github.com/francoispqt/gojay/benchmarks/encoder && make bench
| | ns/op | bytes/op | allocs/op |
|-------------|-------|--------------|-----------|
+| JsonIter | 210078| 41712 | 1136 |
| EasyJson | 106626| 160 | 2 |
| JsonParser | 66813 | 0 | 0 |
-| JsonIter | 87994 | 6738 | 329 |
-| GoJay | 43402 | 1408 | 76 |
+| GoJay | 52153 | 31241 | 77 |
+| GoJay-unsafe| 48277 | 2561 | 76 |
## Encode
@@ -726,7 +729,8 @@ cd $GOPATH/src/github.com/francoispqt/gojay/benchmarks/encoder && make bench
| Std Library | 1280 | 464 | 3 |
| EasyJson | 871 | 944 | 6 |
| JsonIter | 866 | 272 | 3 |
-| GoJay | 484 | 320 | 2 |
+| GoJay | 543 | 112 | 1 |
+| GoJay-func | 347 | 0 | 0 |
### Medium Struct
[benchmark code is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/encoder/encoder_bench_medium_test.go)
@@ -735,10 +739,10 @@ cd $GOPATH/src/github.com/francoispqt/gojay/benchmarks/encoder && make bench
| | ns/op | bytes/op | allocs/op |
|-------------|-------|--------------|-----------|
-| Std Library | 3325 | 1496 | 18 |
-| EasyJson | 1997 | 1320 | 19 |
-| JsonIter | 1939 | 648 | 16 |
-| GoJay | 1196 | 936 | 16 |
+| Std Library | 5006 | 1496 | 25 |
+| JsonIter | 2232 | 1544 | 20 |
+| EasyJson | 1997 | 1544 | 19 |
+| GoJay | 1522 | 312 | 14 |
### Large Struct
[benchmark code is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/encoder/encoder_bench_large_test.go)
@@ -747,10 +751,10 @@ cd $GOPATH/src/github.com/francoispqt/gojay/benchmarks/encoder && make bench
| | ns/op | bytes/op | allocs/op |
|-------------|-------|--------------|-----------|
-| Std Library | 51317 | 28704 | 326 |
-| JsonIter | 35247 | 14608 | 320 |
+| Std Library | 66441 | 20576 | 332 |
+| JsonIter | 35247 | 20255 | 328 |
| EasyJson | 32053 | 15474 | 327 |
-| GoJay | 27847 | 27888 | 326 |
+| GoJay | 27847 | 9802 | 318 |
# Contributing
diff --git a/decode_object_test.go b/decode_object_test.go
@@ -158,7 +158,7 @@ func TestDecodeObjectNull(t *testing.T) {
var jsonComplex = []byte(`{
"test": "{\"test\":\"1\",\"test1\":2}",
- "test2\\n": "\\\\\\\\\\\n",
+ "test2\\n": "\\\\\\\\\\n",
"testArrSkip": ["testString with escaped \\\" quotes"],
"testSkipString": "skip \\ string with \\n escaped char \" ",
"testSkipObject": {
@@ -200,7 +200,7 @@ func (j *jsonObjectComplex) UnmarshalObject(dec *Decoder, key string) error {
switch key {
case "test":
return dec.AddString(&j.Test)
- case `test2\n`:
+ case "test2\n":
return dec.AddString(&j.Test2)
case "test3":
return dec.AddInt(&j.Test3)
@@ -224,9 +224,9 @@ func TestDecodeObjComplex(t *testing.T) {
result := jsonObjectComplex{}
err := UnmarshalObject(jsonComplex, &result)
assert.NotNil(t, err, "err should not be as invalid type as been encountered nil")
- assert.Equal(t, `Cannot unmarshal to struct, wrong char '"' found at pos 643`, err.Error(), "err should not be as invalid type as been encountered nil")
+ assert.Equal(t, `Cannot unmarshal to struct, wrong char '"' found at pos 639`, err.Error(), "err should not be as invalid type as been encountered nil")
assert.Equal(t, `{"test":"1","test1":2}`, result.Test, "result.Test is not expected value")
- assert.Equal(t, `\\\\\\n`, result.Test2, "result.Test2 is not expected value")
+ assert.Equal(t, "\\\\\\\\\n", result.Test2, "result.Test2 is not expected value")
assert.Equal(t, 1, result.Test3, "result.test3 is not expected value")
assert.Equal(t, `{"test":"1","test1":2}`, result.testSub.Test, "result.testSub.test is not expected value")
assert.Equal(t, `[1,2,3]`, result.testSub.Test2, "result.testSub.test2 is not expected value")
diff --git a/decode_string.go b/decode_string.go
@@ -77,21 +77,65 @@ func (dec *Decoder) parseEscapedString() error {
dec.length = len(dec.data)
dec.cursor -= nSlash - diff
return nil
- case 'n', 'r', 't':
+ case 'b':
// number of slash must be even
// if is odd number of slashes
// divide nSlash - 1 by 2 and leave last one
// else divide nSlash by 2 and leave the letter
var diff int
- if nSlash&1 == 1 {
- diff = (nSlash - 1) >> 1
- dec.data = append(dec.data[:start+diff], dec.data[dec.cursor-1:]...)
+ if nSlash&1 != 0 {
+ return InvalidJSONError("Invalid JSON unescaped character")
} else {
diff = nSlash >> 1
- dec.data = append(dec.data[:start+diff-1], dec.data[dec.cursor-1:]...)
+ dec.data = append(append(dec.data[:start+diff-2], '\b'), dec.data[dec.cursor:]...)
}
dec.length = len(dec.data)
- dec.cursor -= nSlash - diff
+ dec.cursor -= nSlash - diff + 1
+ return nil
+ case 'n':
+ // number of slash must be even
+ // if is odd number of slashes
+ // divide nSlash - 1 by 2 and leave last one
+ // else divide nSlash by 2 and leave the letter
+ var diff int
+ if nSlash&1 != 0 {
+ return InvalidJSONError("Invalid JSON unescaped character")
+ } else {
+ diff = nSlash >> 1
+ dec.data = append(append(dec.data[:start+diff-2], '\n'), dec.data[dec.cursor:]...)
+ }
+ dec.length = len(dec.data)
+ dec.cursor -= nSlash - diff + 1
+ return nil
+ case 'r':
+ // number of slash must be even
+ // if is odd number of slashes
+ // divide nSlash - 1 by 2 and leave last one
+ // else divide nSlash by 2 and leave the letter
+ var diff int
+ if nSlash&1 != 0 {
+ return InvalidJSONError("Invalid JSON unescaped character")
+ } else {
+ diff = nSlash >> 1
+ dec.data = append(append(dec.data[:start+diff-2], '\r'), dec.data[dec.cursor:]...)
+ }
+ dec.length = len(dec.data)
+ dec.cursor -= nSlash - diff + 1
+ return nil
+ case 't':
+ // number of slash must be even
+ // if is odd number of slashes
+ // divide nSlash - 1 by 2 and leave last one
+ // else divide nSlash by 2 and leave the letter
+ var diff int
+ if nSlash&1 != 0 {
+ return InvalidJSONError("Invalid JSON unescaped character")
+ } else {
+ diff = nSlash >> 1
+ dec.data = append(append(dec.data[:start+diff-2], '\t'), dec.data[dec.cursor:]...)
+ }
+ dec.length = len(dec.data)
+ dec.cursor -= nSlash - diff + 1
return nil
default:
// nSlash must be even
diff --git a/decode_string_test.go b/decode_string_test.go
@@ -40,7 +40,7 @@ func TestDecoderStringComplex(t *testing.T) {
var v string
err := Unmarshal(json, &v)
assert.Nil(t, err, "Err must be nil")
- assert.Equal(t, "string with spaces and \"escape\"d \"quotes\" and escaped line returns \\n and escaped \\\\ escaped char", v, "v is not equal to the value expected")
+ assert.Equal(t, "string with spaces and \"escape\"d \"quotes\" and escaped line returns \n and escaped \\\\ escaped char", v, "v is not equal to the value expected")
}
func TestDecoderStringNull(t *testing.T) {
@@ -137,6 +137,36 @@ func TestParseEscapedString(t *testing.T) {
}{
{
name: "escape quote err",
+ json: `"test string \" escaped"`,
+ expectedResult: `test string " escaped`,
+ err: false,
+ },
+ {
+ name: "escape quote err2",
+ json: `"test string \\t escaped"`,
+ expectedResult: "test string \t escaped",
+ err: false,
+ },
+ {
+ name: "escape quote err2",
+ json: `"test string \\r escaped"`,
+ expectedResult: "test string \r escaped",
+ err: false,
+ },
+ {
+ name: "escape quote err2",
+ json: `"test string \\b escaped"`,
+ expectedResult: "test string \b escaped",
+ err: false,
+ },
+ {
+ name: "escape quote err",
+ json: `"test string \\n escaped"`,
+ expectedResult: "test string \n escaped",
+ err: false,
+ },
+ {
+ name: "escape quote err",
json: `"test string \\" escaped"`,
expectedResult: ``,
err: true,
@@ -152,20 +182,23 @@ func TestParseEscapedString(t *testing.T) {
}
for _, testCase := range testCases {
- str := ""
- dec := NewDecoder(strings.NewReader(testCase.json))
- err := dec.Decode(&str)
- if testCase.err {
- assert.NotNil(t, err, "err should not be nil")
- if testCase.errType != nil {
- assert.IsType(t, testCase.errType, err, "err should be of expected type")
+ t.Run(testCase.name, func(t *testing.T) {
+ str := ""
+ dec := NewDecoder(strings.NewReader(testCase.json))
+ err := dec.Decode(&str)
+ if testCase.err {
+ assert.NotNil(t, err, "err should not be nil")
+ if testCase.errType != nil {
+ assert.IsType(t, testCase.errType, err, "err should be of expected type")
+ }
+ log.Print(err)
+ } else {
+ assert.Nil(t, err, "err should be nil")
}
- log.Print(err)
- } else {
- assert.Nil(t, err, "err should be nil")
- }
- assert.Equal(t, testCase.expectedResult, str, fmt.Sprintf("str should be equal to '%s'", testCase.expectedResult))
+ assert.Equal(t, testCase.expectedResult, str, fmt.Sprintf("str should be equal to '%s'", testCase.expectedResult))
+ })
}
+
}
func TestSkipString(t *testing.T) {
diff --git a/encode.go b/encode.go
@@ -29,7 +29,8 @@ import (
// fmt.Println(b) // {"id":123456}
// }
func MarshalObject(v MarshalerObject) ([]byte, error) {
- enc := newEncoder()
+ enc := BorrowEncoder(nil)
+ enc.grow(512)
defer enc.Release()
return enc.encodeObject(v)
}
@@ -56,8 +57,8 @@ func MarshalObject(v MarshalerObject) ([]byte, error) {
// fmt.Println(b) // [{"id":123456},{"id":7890}]
// }
func MarshalArray(v MarshalerArray) ([]byte, error) {
- enc := newEncoder()
- enc.grow(200)
+ enc := BorrowEncoder(nil)
+ enc.grow(512)
enc.writeByte('[')
v.(MarshalerArray).MarshalArray(enc)
enc.writeByte(']')
diff --git a/encode_builder.go b/encode_builder.go
@@ -17,6 +17,10 @@ func (enc *Encoder) writeBytes(p []byte) {
enc.buf = append(enc.buf, p...)
}
+func (enc *Encoder) writeTwoBytes(b1 byte, b2 byte) {
+ enc.buf = append(enc.buf, b1, b2)
+}
+
// WriteByte appends the byte c to b's Buffer.
// The returned error is always nil.
func (enc *Encoder) writeByte(c byte) {
@@ -34,17 +38,17 @@ func (enc *Encoder) writeStringEscape(s string) {
for i := 0; i < l; i++ {
switch s[i] {
case '\\', '"':
- enc.writeByte('\\')
- enc.writeByte(s[i])
+ enc.writeTwoBytes('\\', s[i])
case '\n':
- enc.writeByte('\\')
- enc.writeByte('n')
+ enc.writeTwoBytes('\\', 'n')
+ case '\f':
+ enc.writeTwoBytes('\\', 'f')
+ case '\b':
+ enc.writeTwoBytes('\\', 'b')
case '\r':
- enc.writeByte('\\')
- enc.writeByte('r')
+ enc.writeTwoBytes('\\', 'r')
case '\t':
- enc.writeByte('\\')
- enc.writeByte('t')
+ enc.writeTwoBytes('\\', 't')
default:
enc.writeByte(s[i])
}
diff --git a/encode_string.go b/encode_string.go
@@ -35,9 +35,10 @@ func (enc *Encoder) AddString(v string) {
enc.grow(len(v) + 4)
r := enc.getPreviousRune()
if r != '[' {
- enc.writeByte(',')
+ enc.writeTwoBytes(',', '"')
+ } else {
+ enc.writeByte('"')
}
- enc.writeByte('"')
enc.writeStringEscape(v)
enc.writeByte('"')
}
@@ -50,9 +51,10 @@ func (enc *Encoder) AddStringOmitEmpty(v string) {
}
r := enc.getPreviousRune()
if r != '[' {
- enc.writeByte(',')
+ enc.writeTwoBytes(',', '"')
+ } else {
+ enc.writeByte('"')
}
- enc.writeByte('"')
enc.writeStringEscape(v)
enc.writeByte('"')
}
@@ -62,9 +64,10 @@ func (enc *Encoder) AddStringKey(key, v string) {
enc.grow(len(key) + len(v) + 5)
r := enc.getPreviousRune()
if r != '{' {
- enc.writeByte(',')
+ enc.writeTwoBytes(',', '"')
+ } else {
+ enc.writeByte('"')
}
- enc.writeByte('"')
enc.writeStringEscape(key)
enc.writeBytes(objKeyStr)
enc.writeStringEscape(v)
@@ -80,9 +83,10 @@ func (enc *Encoder) AddStringKeyOmitEmpty(key, v string) {
enc.grow(len(key) + len(v) + 5)
r := enc.getPreviousRune()
if r != '{' {
- enc.writeByte(',')
+ enc.writeTwoBytes(',', '"')
+ } else {
+ enc.writeByte('"')
}
- enc.writeByte('"')
enc.writeStringEscape(key)
enc.writeBytes(objKeyStr)
enc.writeStringEscape(v)
diff --git a/encode_string_test.go b/encode_string_test.go
@@ -22,11 +22,11 @@ func TestEncoderStringEncodeAPI(t *testing.T) {
t.Run("utf8", func(t *testing.T) {
builder := &strings.Builder{}
enc := NewEncoder(builder)
- err := enc.EncodeString("漢字")
+ err := enc.EncodeString("漢字𩸽")
assert.Nil(t, err, "Error should be nil")
assert.Equal(
t,
- `"漢字"`,
+ `"漢字𩸽"`,
builder.String(),
"Result of marshalling is different as the one expected")
})
