gojay

high performance JSON encoder/decoder with stream API for Golang
git clone git://git.lair.cx/gojay
Log | Files | Refs | README | LICENSE

commit b9a557e24cfc5fe028d884ea7bfe78e3ed7cee93
parent 2896c17ffd3c0033fc2279f66759c9224d7d6edb
Author: francoispqt <francois@parquet.ninja>
Date:   Sat, 28 Apr 2018 10:15:14 +0800

improve pooling, make it safer and expose Release and BorrowDecoder funcs

Diffstat:
Mdecode.go | 70+++++++++++++++++++++++++++++++++++++---------------------------------
Mdecode_array.go | 6++++++
Mdecode_bool.go | 6++++++
Mdecode_number.go | 38++++++++++++++++++++++++++++++++++++++
Mdecode_object.go | 9+++++++++
Mdecode_pool.go | 29+++++++++++++++++++++++------
Mdecode_stream.go | 16+++-------------
Adecode_stream_pool.go | 49+++++++++++++++++++++++++++++++++++++++++++++++++
Mdecode_string.go | 6++++++
Mdecode_unsafe.go | 30+++++++++++++++---------------
Mencode.go | 34+++++++++++++++++-----------------
Mencode_pool.go | 2+-
Merrors.go | 8++++++++
13 files changed, 218 insertions(+), 85 deletions(-)

diff --git a/decode.go b/decode.go @@ -13,12 +13,12 @@ import ( // If a JSON value is not appropriate for a given target type, or if a JSON number // overflows the target type, UnmarshalArray skips that field and completes the unmarshaling as best it can. func UnmarshalArray(data []byte, v UnmarshalerArray) error { - dec := newDecoder(nil, 0) + dec := BorrowDecoder(nil, 0) dec.data = make([]byte, len(data)) copy(dec.data, data) dec.length = len(data) - _, err := dec.DecodeArray(v) - dec.addToPool() + _, err := dec.decodeArray(v) + defer dec.Release() if err != nil { return err } @@ -35,12 +35,12 @@ func UnmarshalArray(data []byte, v UnmarshalerArray) error { // If a JSON value is not appropriate for a given target type, or if a JSON number // overflows the target type, UnmarshalObject skips that field and completes the unmarshaling as best it can. func UnmarshalObject(data []byte, v UnmarshalerObject) error { - dec := newDecoder(nil, 0) + dec := BorrowDecoder(nil, 0) dec.data = make([]byte, len(data)) copy(dec.data, data) dec.length = len(data) - _, err := dec.DecodeObject(v) - dec.addToPool() + _, err := dec.decodeObject(v) + defer dec.Release() if err != nil { return err } @@ -73,61 +73,61 @@ func Unmarshal(data []byte, v interface{}) error { var dec *Decoder switch vt := v.(type) { case *string: - dec = newDecoder(nil, 0) + dec = BorrowDecoder(nil, 0) dec.length = len(data) dec.data = data - err = dec.DecodeString(vt) + err = dec.decodeString(vt) case *int: - dec = newDecoder(nil, 0) + dec = BorrowDecoder(nil, 0) dec.length = len(data) dec.data = data - err = dec.DecodeInt(vt) + err = dec.decodeInt(vt) case *int32: - dec = newDecoder(nil, 0) + dec = BorrowDecoder(nil, 0) dec.length = len(data) dec.data = data - err = dec.DecodeInt32(vt) + err = dec.decodeInt32(vt) case *uint32: - dec = newDecoder(nil, 0) + dec = BorrowDecoder(nil, 0) dec.length = len(data) dec.data = data - err = dec.DecodeUint32(vt) + err = dec.decodeUint32(vt) case *int64: - dec = newDecoder(nil, 0) + dec = BorrowDecoder(nil, 0) dec.length = len(data) dec.data = data - err = dec.DecodeInt64(vt) + err = dec.decodeInt64(vt) case *uint64: - dec = newDecoder(nil, 0) + dec = BorrowDecoder(nil, 0) dec.length = len(data) dec.data = data - err = dec.DecodeUint64(vt) + err = dec.decodeUint64(vt) case *float64: - dec = newDecoder(nil, 0) + dec = BorrowDecoder(nil, 0) dec.length = len(data) dec.data = data - err = dec.DecodeFloat64(vt) + err = dec.decodeFloat64(vt) case *bool: - dec = newDecoder(nil, 0) + dec = BorrowDecoder(nil, 0) dec.length = len(data) dec.data = data - err = dec.DecodeBool(vt) + err = dec.decodeBool(vt) case UnmarshalerObject: - dec = newDecoder(nil, 0) + dec = BorrowDecoder(nil, 0) dec.length = len(data) dec.data = make([]byte, len(data)) copy(dec.data, data) - _, err = dec.DecodeObject(vt) + _, err = dec.decodeObject(vt) case UnmarshalerArray: - dec = newDecoder(nil, 0) + dec = BorrowDecoder(nil, 0) dec.length = len(data) dec.data = make([]byte, len(data)) copy(dec.data, data) - _, err = dec.DecodeArray(vt) + _, err = dec.decodeArray(vt) default: return InvalidUnmarshalError(fmt.Sprintf(invalidUnmarshalErrorMsg, reflect.TypeOf(vt).String())) } - defer dec.addToPool() + defer dec.Release() if err != nil { return err } @@ -163,12 +163,16 @@ type Decoder struct { child byte err error r io.Reader + isPooled byte } // Decode reads the next JSON-encoded value from its input and stores it in the value pointed to by v. // // See the documentation for Unmarshal for details about the conversion of JSON into a Go value. func (dec *Decoder) Decode(v interface{}) error { + if dec.isPooled == 1 { + panic(InvalidUsagePooledDecoderError("Invalid usagee of pooled decoder")) + } switch vt := v.(type) { case *string: return dec.DecodeString(vt) @@ -202,7 +206,7 @@ func (dec *Decoder) Decode(v interface{}) error { // AddInt decodes the next key to an *int. // If next key value overflows int, an InvalidTypeError error will be returned. func (dec *Decoder) AddInt(v *int) error { - err := dec.DecodeInt(v) + err := dec.decodeInt(v) if err != nil { return err } @@ -213,7 +217,7 @@ func (dec *Decoder) AddInt(v *int) error { // AddFloat decodes the next key to a *float64. // If next key value overflows float64, an InvalidTypeError error will be returned. func (dec *Decoder) AddFloat(v *float64) error { - err := dec.DecodeFloat64(v) + err := dec.decodeFloat64(v) if err != nil { return err } @@ -225,7 +229,7 @@ func (dec *Decoder) AddFloat(v *float64) error { // If next key is neither null nor a JSON boolean, an InvalidTypeError will be returned. // If next key is null, bool will be false. func (dec *Decoder) AddBool(v *bool) error { - err := dec.DecodeBool(v) + err := dec.decodeBool(v) if err != nil { return err } @@ -236,7 +240,7 @@ func (dec *Decoder) AddBool(v *bool) error { // AddString decodes the next key to a *string. // If next key is not a JSON string nor null, InvalidTypeError will be returned. func (dec *Decoder) AddString(v *string) error { - err := dec.DecodeString(v) + err := dec.decodeString(v) if err != nil { return err } @@ -251,7 +255,7 @@ func (dec *Decoder) AddObject(value UnmarshalerObject) error { dec.keysDone = 0 dec.called = 0 dec.child |= 1 - newCursor, err := dec.DecodeObject(value) + newCursor, err := dec.decodeObject(value) if err != nil { return err } @@ -264,7 +268,7 @@ func (dec *Decoder) AddObject(value UnmarshalerObject) error { // AddArray decodes the next key to a UnmarshalerArray. func (dec *Decoder) AddArray(value UnmarshalerArray) error { - newCursor, err := dec.DecodeArray(value) + newCursor, err := dec.decodeArray(value) if err != nil { return err } diff --git a/decode_array.go b/decode_array.go @@ -10,6 +10,12 @@ import ( // // See the documentation for Unmarshal for details about the conversion of JSON into a Go value. func (dec *Decoder) DecodeArray(arr UnmarshalerArray) (int, error) { + if dec.isPooled == 1 { + panic(InvalidUsagePooledDecoderError("Invalid usagee of pooled decoder")) + } + return dec.decodeArray(arr) +} +func (dec *Decoder) decodeArray(arr UnmarshalerArray) (int, error) { // not an array not an error, but do not know what to do // do not check syntax for ; dec.cursor < dec.length || dec.read(); dec.cursor++ { diff --git a/decode_bool.go b/decode_bool.go @@ -6,6 +6,12 @@ import "fmt" // // See the documentation for Unmarshal for details about the conversion of JSON into a Go value. func (dec *Decoder) DecodeBool(v *bool) error { + if dec.isPooled == 1 { + panic(InvalidUsagePooledDecoderError("Invalid usagee of pooled decoder")) + } + return dec.decodeBool(v) +} +func (dec *Decoder) decodeBool(v *bool) error { for ; dec.cursor < dec.length || dec.read(); dec.cursor++ { switch dec.data[dec.cursor] { case ' ', '\n', '\t', '\r', ',': diff --git a/decode_number.go b/decode_number.go @@ -57,6 +57,12 @@ func init() { // // See the documentation for Unmarshal for details about the conversion of JSON into a Go value. func (dec *Decoder) DecodeInt(v *int) error { + if dec.isPooled == 1 { + panic(InvalidUsagePooledDecoderError("Invalid usagee of pooled decoder")) + } + return dec.decodeInt(v) +} +func (dec *Decoder) decodeInt(v *int) error { for ; dec.cursor < dec.length || dec.read(); dec.cursor++ { switch c := dec.data[dec.cursor]; c { case ' ', '\n', '\t', '\r', ',': @@ -101,6 +107,12 @@ func (dec *Decoder) DecodeInt(v *int) error { // // See the documentation for Unmarshal for details about the conversion of JSON into a Go value. func (dec *Decoder) DecodeInt32(v *int32) error { + if dec.isPooled == 1 { + panic(InvalidUsagePooledDecoderError("Invalid usagee of pooled decoder")) + } + return dec.decodeInt32(v) +} +func (dec *Decoder) decodeInt32(v *int32) error { for ; dec.cursor < dec.length || dec.read(); dec.cursor++ { switch c := dec.data[dec.cursor]; c { case ' ', '\n', '\t', '\r', ',': @@ -145,6 +157,13 @@ func (dec *Decoder) DecodeInt32(v *int32) error { // // See the documentation for Unmarshal for details about the conversion of JSON into a Go value. func (dec *Decoder) DecodeUint32(v *uint32) error { + if dec.isPooled == 1 { + panic(InvalidUsagePooledDecoderError("Invalid usagee of pooled decoder")) + } + return dec.decodeUint32(v) +} + +func (dec *Decoder) decodeUint32(v *uint32) error { for ; dec.cursor < dec.length || dec.read(); dec.cursor++ { switch c := dec.data[dec.cursor]; c { case ' ', '\n', '\t', '\r', ',': @@ -190,6 +209,13 @@ func (dec *Decoder) DecodeUint32(v *uint32) error { // // See the documentation for Unmarshal for details about the conversion of JSON into a Go value. func (dec *Decoder) DecodeInt64(v *int64) error { + if dec.isPooled == 1 { + panic(InvalidUsagePooledDecoderError("Invalid usagee of pooled decoder")) + } + return dec.decodeInt64(v) +} + +func (dec *Decoder) decodeInt64(v *int64) error { for ; dec.cursor < dec.length || dec.read(); dec.cursor++ { switch c := dec.data[dec.cursor]; c { case ' ', '\n', '\t', '\r', ',': @@ -234,6 +260,12 @@ func (dec *Decoder) DecodeInt64(v *int64) error { // // See the documentation for Unmarshal for details about the conversion of JSON into a Go value. func (dec *Decoder) DecodeUint64(v *uint64) error { + if dec.isPooled == 1 { + panic(InvalidUsagePooledDecoderError("Invalid usagee of pooled decoder")) + } + return dec.decodeUint64(v) +} +func (dec *Decoder) decodeUint64(v *uint64) error { for ; dec.cursor < dec.length || dec.read(); dec.cursor++ { switch c := dec.data[dec.cursor]; c { case ' ', '\n', '\t', '\r', ',': @@ -279,6 +311,12 @@ func (dec *Decoder) DecodeUint64(v *uint64) error { // // See the documentation for Unmarshal for details about the conversion of JSON into a Go value. func (dec *Decoder) DecodeFloat64(v *float64) error { + if dec.isPooled == 1 { + panic(InvalidUsagePooledDecoderError("Invalid usagee of pooled decoder")) + } + return dec.decodeFloat64(v) +} +func (dec *Decoder) decodeFloat64(v *float64) error { for ; dec.cursor < dec.length || dec.read(); dec.cursor++ { switch c := dec.data[dec.cursor]; c { case ' ', '\n', '\t', '\r', ',': diff --git a/decode_object.go b/decode_object.go @@ -11,6 +11,15 @@ import ( // // See the documentation for Unmarshal for details about the conversion of JSON into a Go value. func (dec *Decoder) DecodeObject(j UnmarshalerObject) (int, error) { + if dec.isPooled == 1 { + panic(InvalidUsagePooledDecoderError("Invalid usagee of pooled decoder")) + } + return dec.decodeObject(j) +} +func (dec *Decoder) decodeObject(j UnmarshalerObject) (int, error) { + if dec.isPooled == 1 { + panic(InvalidUsagePooledDecoderError("Invalid usagee of pooled decoder")) + } keys := j.NKeys() for ; dec.cursor < dec.length || dec.read(); dec.cursor++ { switch dec.data[dec.cursor] { diff --git a/decode_pool.go b/decode_pool.go @@ -4,13 +4,25 @@ import "io" var decPool = make(chan *Decoder, 16) -// NewDecoder returns a new decoder or borrows one from the pool -// it takes an io.Reader implementation as data input +// NewDecoder returns a new decoder. +// It takes an io.Reader implementation as data input. func NewDecoder(r io.Reader) *Decoder { - return newDecoder(r, 512) + return &Decoder{ + called: 0, + cursor: 0, + keysDone: 0, + err: nil, + r: r, + data: make([]byte, 512), + length: 0, + isPooled: 0, + } } -func newDecoder(r io.Reader, bufSize int) *Decoder { +// BorrowDecoder borrows a Decoder a decoder from the pool. +// It takes an io.Reader implementation as data input. +// It initiates the done channel returned by Done(). +func BorrowDecoder(r io.Reader, bufSize int) *Decoder { select { case dec := <-decPool: dec.called = 0 @@ -19,6 +31,7 @@ func newDecoder(r io.Reader, bufSize int) *Decoder { dec.err = nil dec.r = r dec.length = 0 + dec.isPooled = 0 if bufSize > 0 { dec.data = make([]byte, bufSize) } @@ -30,16 +43,20 @@ func newDecoder(r io.Reader, bufSize int) *Decoder { keysDone: 0, err: nil, r: r, + isPooled: 0, } if bufSize > 0 { dec.data = make([]byte, bufSize) - dec.length = 0 } return dec } } -func (dec *Decoder) addToPool() { +// Release sends back a Decoder to the pool. +// If a decoder is used after calling Release +// a panic will be raised with an InvalidUsagePooledDecoderError error. +func (dec *Decoder) Release() { + dec.isPooled = 1 select { case decPool <- dec: default: diff --git a/decode_stream.go b/decode_stream.go @@ -1,7 +1,6 @@ package gojay import ( - "io" "time" ) @@ -19,24 +18,15 @@ type StreamDecoder struct { deadline *time.Time } -// NewDecoder returns a new decoder or borrows one from the pool. -// It takes an io.Reader implementation as data input. -// It initiates the done channel returned by Done(). -func (s stream) NewDecoder(r io.Reader) *StreamDecoder { - dec := newDecoder(r, 512) - streamDec := &StreamDecoder{ - Decoder: dec, - done: make(chan struct{}, 1), - } - return streamDec -} - // DecodeStream reads the next line delimited JSON-encoded value from its input and stores it in the value pointed to by c. // // c must implement UnmarshalerStream. Ideally c is a channel. See example for implementation. // // See the documentation for Unmarshal for details about the conversion of JSON into a Go value. func (dec *StreamDecoder) DecodeStream(c UnmarshalerStream) error { + if dec.isPooled == 1 { + panic(InvalidUsagePooledDecoderError("Invalid usagee of pooled decoder")) + } if dec.r == nil { dec.err = NoReaderError("No reader given to decode stream") close(dec.done) diff --git a/decode_stream_pool.go b/decode_stream_pool.go @@ -0,0 +1,49 @@ +package gojay + +import "io" + +var streamDecPool = make(chan *StreamDecoder, 16) + +// NewDecoder returns a new decoder. +// It takes an io.Reader implementation as data input. +// It initiates the done channel returned by Done(). +func (s stream) NewDecoder(r io.Reader) *StreamDecoder { + dec := NewDecoder(r) + streamDec := &StreamDecoder{ + Decoder: dec, + done: make(chan struct{}, 1), + } + return streamDec +} + +// BorrowDecoder borrows a StreamDecoder a decoder from the pool. +// It takes an io.Reader implementation as data input. +// It initiates the done channel returned by Done(). +func (s stream) BorrowDecoder(r io.Reader, bufSize int) *StreamDecoder { + select { + case streamDec := <-streamDecPool: + streamDec.called = 0 + streamDec.keysDone = 0 + streamDec.cursor = 0 + streamDec.err = nil + streamDec.r = r + streamDec.length = 0 + streamDec.isPooled = 0 + streamDec.done = make(chan struct{}, 1) + if bufSize > 0 { + streamDec.data = make([]byte, bufSize) + } + return streamDec + default: + dec := NewDecoder(r) + if bufSize > 0 { + dec.data = make([]byte, bufSize) + dec.length = 0 + } + streamDec := &StreamDecoder{ + Decoder: dec, + done: make(chan struct{}, 1), + } + return streamDec + } +} diff --git a/decode_string.go b/decode_string.go @@ -9,6 +9,12 @@ import ( // // See the documentation for Unmarshal for details about the conversion of JSON into a Go value. func (dec *Decoder) DecodeString(v *string) error { + if dec.isPooled == 1 { + panic(InvalidUsagePooledDecoderError("Invalid usagee of pooled decoder")) + } + return dec.decodeString(v) +} +func (dec *Decoder) decodeString(v *string) error { for ; dec.cursor < dec.length || dec.read(); dec.cursor++ { switch dec.data[dec.cursor] { case ' ', '\n', '\t', '\r', ',': diff --git a/decode_unsafe.go b/decode_unsafe.go @@ -14,11 +14,11 @@ var Unsafe = decUnsafe{} type decUnsafe struct{} func (u decUnsafe) UnmarshalArray(data []byte, v UnmarshalerArray) error { - dec := newDecoder(nil, 0) + dec := BorrowDecoder(nil, 0) dec.data = data dec.length = len(data) _, err := dec.DecodeArray(v) - dec.addToPool() + dec.Release() if err != nil { return err } @@ -29,11 +29,11 @@ func (u decUnsafe) UnmarshalArray(data []byte, v UnmarshalerArray) error { } func (u decUnsafe) UnmarshalObject(data []byte, v UnmarshalerObject) error { - dec := newDecoder(nil, 0) + dec := BorrowDecoder(nil, 0) dec.data = data dec.length = len(data) _, err := dec.DecodeObject(v) - dec.addToPool() + dec.Release() if err != nil { return err } @@ -48,59 +48,59 @@ func (u decUnsafe) Unmarshal(data []byte, v interface{}) error { var dec *Decoder switch vt := v.(type) { case *string: - dec = newDecoder(nil, 0) + dec = BorrowDecoder(nil, 0) dec.length = len(data) dec.data = data err = dec.DecodeString(vt) case *int: - dec = newDecoder(nil, 0) + dec = BorrowDecoder(nil, 0) dec.length = len(data) dec.data = data err = dec.DecodeInt(vt) case *int32: - dec = newDecoder(nil, 0) + dec = BorrowDecoder(nil, 0) dec.length = len(data) dec.data = data err = dec.DecodeInt32(vt) case *uint32: - dec = newDecoder(nil, 0) + dec = BorrowDecoder(nil, 0) dec.length = len(data) dec.data = data err = dec.DecodeUint32(vt) case *int64: - dec = newDecoder(nil, 0) + dec = BorrowDecoder(nil, 0) dec.length = len(data) dec.data = data err = dec.DecodeInt64(vt) case *uint64: - dec = newDecoder(nil, 0) + dec = BorrowDecoder(nil, 0) dec.length = len(data) dec.data = data err = dec.DecodeUint64(vt) case *float64: - dec = newDecoder(nil, 0) + dec = BorrowDecoder(nil, 0) dec.length = len(data) dec.data = data err = dec.DecodeFloat64(vt) case *bool: - dec = newDecoder(nil, 0) + dec = BorrowDecoder(nil, 0) dec.length = len(data) dec.data = data err = dec.DecodeBool(vt) case UnmarshalerObject: - dec = newDecoder(nil, 0) + dec = BorrowDecoder(nil, 0) dec.length = len(data) dec.data = data _, err = dec.DecodeObject(vt) case UnmarshalerArray: - dec = newDecoder(nil, 0) + dec = BorrowDecoder(nil, 0) dec.length = len(data) dec.data = data _, err = dec.DecodeArray(vt) default: return InvalidUnmarshalError(fmt.Sprintf(invalidUnmarshalErrorMsg, reflect.TypeOf(vt).String())) } - defer dec.addToPool() + defer dec.Release() if err != nil { return err } diff --git a/encode.go b/encode.go @@ -28,7 +28,7 @@ func MarshalObject(v MarshalerObject) ([]byte, error) { enc.writeByte('{') v.MarshalObject(enc) enc.writeByte('}') - defer enc.addToPool() + defer enc.Release() return enc.buf, nil } @@ -59,7 +59,7 @@ func MarshalArray(v MarshalerArray) ([]byte, error) { enc.writeByte('[') v.(MarshalerArray).MarshalArray(enc) enc.writeByte(']') - defer enc.addToPool() + defer enc.Release() return enc.buf, nil } @@ -103,7 +103,7 @@ func Marshal(v interface{}) ([]byte, error) { vt.MarshalObject(enc) enc.writeByte('}') b = enc.buf - defer enc.addToPool() + defer enc.Release() return b, nil case MarshalerArray: enc := NewEncoder() @@ -111,60 +111,60 @@ func Marshal(v interface{}) ([]byte, error) { vt.MarshalArray(enc) enc.writeByte(']') b = enc.buf - defer enc.addToPool() + defer enc.Release() return b, nil case string: enc := NewEncoder() b, err = enc.encodeString(vt) - defer enc.addToPool() + defer enc.Release() case bool: enc := NewEncoder() err = enc.AddBool(vt) b = enc.buf - defer enc.addToPool() + defer enc.Release() case int: enc := NewEncoder() b, err = enc.encodeInt(int64(vt)) - defer enc.addToPool() + defer enc.Release() case int64: enc := NewEncoder() - defer enc.addToPool() + defer enc.Release() return enc.encodeInt(vt) case int32: enc := NewEncoder() - defer enc.addToPool() + defer enc.Release() return enc.encodeInt(int64(vt)) case int16: enc := NewEncoder() - defer enc.addToPool() + defer enc.Release() return enc.encodeInt(int64(vt)) case int8: enc := NewEncoder() - defer enc.addToPool() + defer enc.Release() return enc.encodeInt(int64(vt)) case uint64: enc := NewEncoder() - defer enc.addToPool() + defer enc.Release() return enc.encodeInt(int64(vt)) case uint32: enc := NewEncoder() - defer enc.addToPool() + defer enc.Release() return enc.encodeInt(int64(vt)) case uint16: enc := NewEncoder() - defer enc.addToPool() + defer enc.Release() return enc.encodeInt(int64(vt)) case uint8: enc := NewEncoder() b, err = enc.encodeInt(int64(vt)) - defer enc.addToPool() + defer enc.Release() case float64: enc := NewEncoder() - defer enc.addToPool() + defer enc.Release() return enc.encodeFloat(vt) case float32: enc := NewEncoder() - defer enc.addToPool() + defer enc.Release() return enc.encodeFloat(float64(vt)) } return b, err diff --git a/encode_pool.go b/encode_pool.go @@ -12,7 +12,7 @@ func NewEncoder() *Encoder { } } -func (enc *Encoder) addToPool() { +func (enc *Encoder) Release() { enc.buf = nil select { case encObjPool <- enc: diff --git a/errors.go b/errors.go @@ -33,3 +33,11 @@ type NoReaderError string func (err NoReaderError) Error() string { return string(err) } + +// InvalidUsagePooledDecoderError is a type representing an error returned +// when decoding is called on a still pooled Decoder +type InvalidUsagePooledDecoderError string + +func (err InvalidUsagePooledDecoderError) Error() string { + return string(err) +}