commit fec80d1ff4e808acc8bd646a4631844345000c3e
parent 2f1addadc72bb7ba394d8cbdf4d7faaa14b25c2a
Author: Francois Parquet <francois.parquet@gmail.com>
Date: Sun, 6 May 2018 23:46:50 +0800
Merge pull request #20 from francoispqt/update/encode-optimisations
add buf management optimisation for encoding
Diffstat:
11 files changed, 182 insertions(+), 86 deletions(-)
diff --git a/decode_pool.go b/decode_pool.go
@@ -2,7 +2,26 @@ package gojay
import "io"
-var decPool = make(chan *Decoder, 16)
+var decPool = make(chan *Decoder, 32)
+
+func init() {
+initStreamDecPool:
+ for {
+ select {
+ case streamDecPool <- Stream.NewDecoder(nil):
+ default:
+ break initStreamDecPool
+ }
+ }
+initDecPool:
+ for {
+ select {
+ case decPool <- NewDecoder(nil):
+ default:
+ break initDecPool
+ }
+ }
+}
// NewDecoder returns a new decoder.
// It takes an io.Reader implementation as data input.
diff --git a/decode_stream_pool.go b/decode_stream_pool.go
@@ -2,7 +2,7 @@ package gojay
import "io"
-var streamDecPool = make(chan *StreamDecoder, 16)
+var streamDecPool = make(chan *StreamDecoder, 32)
// NewDecoder returns a new StreamDecoder.
// It takes an io.Reader implementation as data input.
diff --git a/encode.go b/encode.go
@@ -187,13 +187,29 @@ type Encoder struct {
err error
}
-func (enc *Encoder) getPreviousRune() (byte, bool) {
- last := len(enc.buf) - 1
- return enc.buf[last], true
+// AppendBytes allows a modular usage by appending bytes manually to the current state of the buffer.
+func (enc *Encoder) AppendBytes(b []byte) {
+ enc.writeBytes(b)
+}
+
+// AppendByte allows a modular usage by appending a single byte manually to the current state of the buffer.
+func (enc *Encoder) AppendByte(b byte) {
+ enc.writeByte(b)
+}
+
+// Buf returns the Encoder's buffer.
+func (enc *Encoder) Buf() []byte {
+ return enc.buf
}
-func (enc *Encoder) write() (int, error) {
+// Write writes to the io.Writer and resets the buffer.
+func (enc *Encoder) Write() (int, error) {
i, err := enc.w.Write(enc.buf)
- enc.buf = make([]byte, 0, 512)
+ enc.buf = enc.buf[:0]
return i, err
}
+
+func (enc *Encoder) getPreviousRune() byte {
+ last := len(enc.buf) - 1
+ return enc.buf[last]
+}
diff --git a/encode_array.go b/encode_array.go
@@ -6,7 +6,7 @@ func (enc *Encoder) EncodeArray(v MarshalerArray) error {
panic(InvalidUsagePooledEncoderError("Invalid usage of pooled encoder"))
}
_, _ = enc.encodeArray(v)
- _, err := enc.write()
+ _, err := enc.Write()
if err != nil {
enc.err = err
return err
@@ -26,8 +26,8 @@ func (enc *Encoder) encodeArray(v MarshalerArray) ([]byte, error) {
func (enc *Encoder) AddArray(v MarshalerArray) {
if v.IsNil() {
enc.grow(3)
- r, ok := enc.getPreviousRune()
- if ok && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '[' {
enc.writeByte(',')
}
enc.writeByte('[')
@@ -35,8 +35,8 @@ func (enc *Encoder) AddArray(v MarshalerArray) {
return
}
enc.grow(100)
- r, ok := enc.getPreviousRune()
- if ok && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '[' {
enc.writeByte(',')
}
enc.writeByte('[')
@@ -51,8 +51,8 @@ func (enc *Encoder) AddArrayOmitEmpty(v MarshalerArray) {
return
}
enc.grow(4)
- r, ok := enc.getPreviousRune()
- if ok && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '[' {
enc.writeByte(',')
}
enc.writeByte('[')
@@ -65,8 +65,8 @@ func (enc *Encoder) AddArrayOmitEmpty(v MarshalerArray) {
func (enc *Encoder) AddArrayKey(key string, v MarshalerArray) {
if v.IsNil() {
enc.grow(2 + len(key))
- r, ok := enc.getPreviousRune()
- if ok && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '[' {
enc.writeByte(',')
}
enc.writeByte('"')
@@ -76,8 +76,8 @@ func (enc *Encoder) AddArrayKey(key string, v MarshalerArray) {
return
}
enc.grow(5 + len(key))
- r, ok := enc.getPreviousRune()
- if ok && r != '[' && r != '{' {
+ r := enc.getPreviousRune()
+ if r != '[' && r != '{' {
enc.writeByte(',')
}
enc.writeByte('"')
@@ -94,8 +94,8 @@ func (enc *Encoder) AddArrayKeyOmitEmpty(key string, v MarshalerArray) {
return
}
enc.grow(5 + len(key))
- r, ok := enc.getPreviousRune()
- if ok && r != '[' && r != '{' {
+ r := enc.getPreviousRune()
+ if r != '[' && r != '{' {
enc.writeByte(',')
}
enc.writeByte('"')
diff --git a/encode_bool.go b/encode_bool.go
@@ -8,7 +8,7 @@ func (enc *Encoder) EncodeBool(v bool) error {
panic(InvalidUsagePooledEncoderError("Invalid usage of pooled encoder"))
}
_, _ = enc.encodeBool(v)
- _, err := enc.write()
+ _, err := enc.Write()
if err != nil {
enc.err = err
return err
@@ -30,8 +30,8 @@ func (enc *Encoder) encodeBool(v bool) ([]byte, error) {
// AddBool adds a bool to be encoded, must be used inside a slice or array encoding (does not encode a key)
func (enc *Encoder) AddBool(v bool) {
enc.grow(5)
- r, ok := enc.getPreviousRune()
- if ok && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '[' {
enc.writeByte(',')
}
if v {
@@ -47,8 +47,8 @@ func (enc *Encoder) AddBoolOmitEmpty(v bool) {
return
}
enc.grow(5)
- r, ok := enc.getPreviousRune()
- if ok && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '[' {
enc.writeByte(',')
}
enc.writeString("true")
@@ -57,8 +57,8 @@ func (enc *Encoder) AddBoolOmitEmpty(v bool) {
// AddBoolKey adds a bool to be encoded, must be used inside an object as it will encode a key.
func (enc *Encoder) AddBoolKey(key string, value bool) {
enc.grow(5 + len(key))
- r, ok := enc.getPreviousRune()
- if ok && r != '{' && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '{' && r != '[' {
enc.writeByte(',')
}
enc.writeByte('"')
@@ -74,8 +74,8 @@ func (enc *Encoder) AddBoolKeyOmitEmpty(key string, v bool) {
return
}
enc.grow(5 + len(key))
- r, ok := enc.getPreviousRune()
- if ok && r != '{' && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '{' && r != '[' {
enc.writeByte(',')
}
enc.writeByte('"')
diff --git a/encode_number.go b/encode_number.go
@@ -8,7 +8,7 @@ func (enc *Encoder) EncodeInt(n int) error {
panic(InvalidUsagePooledEncoderError("Invalid usage of pooled encoder"))
}
_, _ = enc.encodeInt(n)
- _, err := enc.write()
+ _, err := enc.Write()
if err != nil {
return err
}
@@ -27,7 +27,7 @@ func (enc *Encoder) EncodeInt64(n int64) error {
panic(InvalidUsagePooledEncoderError("Invalid usage of pooled encoder"))
}
_, _ = enc.encodeInt64(n)
- _, err := enc.write()
+ _, err := enc.Write()
if err != nil {
return err
}
@@ -46,7 +46,7 @@ func (enc *Encoder) EncodeFloat(n float64) error {
panic(InvalidUsagePooledEncoderError("Invalid usage of pooled encoder"))
}
_, _ = enc.encodeFloat(n)
- _, err := enc.write()
+ _, err := enc.Write()
if err != nil {
return err
}
@@ -65,7 +65,7 @@ func (enc *Encoder) EncodeFloat32(n float32) error {
panic(InvalidUsagePooledEncoderError("Invalid usage of pooled encoder"))
}
_, _ = enc.encodeFloat32(n)
- _, err := enc.write()
+ _, err := enc.Write()
if err != nil {
return err
}
@@ -80,8 +80,8 @@ func (enc *Encoder) encodeFloat32(n float32) ([]byte, error) {
// AddInt adds an int to be encoded, must be used inside a slice or array encoding (does not encode a key)
func (enc *Encoder) AddInt(v int) {
enc.grow(10)
- r, ok := enc.getPreviousRune()
- if ok && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '[' {
enc.writeByte(',')
}
enc.buf = strconv.AppendInt(enc.buf, int64(v), 10)
@@ -94,8 +94,8 @@ func (enc *Encoder) AddIntOmitEmpty(v int) {
return
}
enc.grow(10)
- r, ok := enc.getPreviousRune()
- if ok && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '[' {
enc.writeByte(',')
}
enc.buf = strconv.AppendInt(enc.buf, int64(v), 10)
@@ -104,8 +104,8 @@ func (enc *Encoder) AddIntOmitEmpty(v int) {
// AddFloat adds a float64 to be encoded, must be used inside a slice or array encoding (does not encode a key)
func (enc *Encoder) AddFloat(v float64) {
enc.grow(10)
- r, ok := enc.getPreviousRune()
- if ok && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '[' {
enc.writeByte(',')
}
enc.buf = strconv.AppendFloat(enc.buf, v, 'f', -1, 64)
@@ -118,8 +118,8 @@ func (enc *Encoder) AddFloatOmitEmpty(v float64) {
return
}
enc.grow(10)
- r, ok := enc.getPreviousRune()
- if ok && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '[' {
enc.writeByte(',')
}
enc.buf = strconv.AppendFloat(enc.buf, v, 'f', -1, 64)
@@ -127,8 +127,8 @@ func (enc *Encoder) AddFloatOmitEmpty(v float64) {
// AddFloat32 adds a float32 to be encoded, must be used inside a slice or array encoding (does not encode a key)
func (enc *Encoder) AddFloat32(v float32) {
- r, ok := enc.getPreviousRune()
- if ok && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '[' {
enc.writeByte(',')
}
enc.buf = strconv.AppendFloat(enc.buf, float64(v), 'f', -1, 32)
@@ -141,8 +141,8 @@ func (enc *Encoder) AddFloat32OmitEmpty(v float32) {
return
}
enc.grow(10)
- r, ok := enc.getPreviousRune()
- if ok && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '[' {
enc.writeByte(',')
}
enc.buf = strconv.AppendFloat(enc.buf, float64(v), 'f', -1, 32)
@@ -151,8 +151,8 @@ func (enc *Encoder) AddFloat32OmitEmpty(v float32) {
// AddIntKey adds an int to be encoded, must be used inside an object as it will encode a key
func (enc *Encoder) AddIntKey(key string, v int) {
enc.grow(10 + len(key))
- r, ok := enc.getPreviousRune()
- if ok && r != '{' && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '{' && r != '[' {
enc.writeByte(',')
}
enc.writeByte('"')
@@ -168,8 +168,8 @@ func (enc *Encoder) AddIntKeyOmitEmpty(key string, v int) {
return
}
enc.grow(10 + len(key))
- r, ok := enc.getPreviousRune()
- if ok && r != '{' && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '{' && r != '[' {
enc.writeByte(',')
}
enc.writeByte('"')
@@ -180,8 +180,8 @@ func (enc *Encoder) AddIntKeyOmitEmpty(key string, v int) {
// AddFloatKey adds a float64 to be encoded, must be used inside an object as it will encode a key
func (enc *Encoder) AddFloatKey(key string, value float64) {
- r, ok := enc.getPreviousRune()
- if ok && r != '{' && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '{' && r != '[' {
enc.writeByte(',')
}
enc.grow(10)
@@ -198,8 +198,8 @@ func (enc *Encoder) AddFloatKeyOmitEmpty(key string, v float64) {
return
}
enc.grow(10 + len(key))
- r, ok := enc.getPreviousRune()
- if ok && r != '{' && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '{' && r != '[' {
enc.writeByte(',')
}
enc.writeByte('"')
@@ -211,8 +211,8 @@ func (enc *Encoder) AddFloatKeyOmitEmpty(key string, v float64) {
// AddFloat32Key adds a float32 to be encoded, must be used inside an object as it will encode a key
func (enc *Encoder) AddFloat32Key(key string, v float32) {
enc.grow(10 + len(key))
- r, ok := enc.getPreviousRune()
- if ok && r != '{' && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '{' && r != '[' {
enc.writeByte(',')
}
enc.writeByte('"')
@@ -229,8 +229,8 @@ func (enc *Encoder) AddFloat32KeyOmitEmpty(key string, v float32) {
return
}
enc.grow(10 + len(key))
- r, ok := enc.getPreviousRune()
- if ok && r != '{' && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '{' && r != '[' {
enc.writeByte(',')
}
enc.writeByte('"')
diff --git a/encode_object.go b/encode_object.go
@@ -15,7 +15,7 @@ func (enc *Encoder) EncodeObject(v MarshalerObject) error {
enc.err = err
return err
}
- _, err = enc.write()
+ _, err = enc.Write()
if err != nil {
enc.err = err
return err
@@ -38,8 +38,8 @@ func (enc *Encoder) encodeObject(v MarshalerObject) ([]byte, error) {
func (enc *Encoder) AddObject(v MarshalerObject) {
if v.IsNil() {
enc.grow(2)
- r, ok := enc.getPreviousRune()
- if ok && r != '{' && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '{' && r != '[' {
enc.writeByte(',')
}
enc.writeByte('{')
@@ -47,8 +47,8 @@ func (enc *Encoder) AddObject(v MarshalerObject) {
return
}
enc.grow(4)
- r, ok := enc.getPreviousRune()
- if ok && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '[' {
enc.writeByte(',')
}
enc.writeByte('{')
@@ -64,8 +64,8 @@ func (enc *Encoder) AddObjectOmitEmpty(v MarshalerObject) {
return
}
enc.grow(2)
- r, ok := enc.getPreviousRune()
- if ok && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '[' {
enc.writeByte(',')
}
enc.writeByte('{')
@@ -78,8 +78,8 @@ func (enc *Encoder) AddObjectOmitEmpty(v MarshalerObject) {
func (enc *Encoder) AddObjectKey(key string, value MarshalerObject) {
if value.IsNil() {
enc.grow(2 + len(key))
- r, ok := enc.getPreviousRune()
- if ok && r != '{' {
+ r := enc.getPreviousRune()
+ if r != '{' {
enc.writeByte(',')
}
enc.writeByte('"')
@@ -89,8 +89,8 @@ func (enc *Encoder) AddObjectKey(key string, value MarshalerObject) {
return
}
enc.grow(5 + len(key))
- r, ok := enc.getPreviousRune()
- if ok && r != '{' {
+ r := enc.getPreviousRune()
+ if r != '{' {
enc.writeByte(',')
}
enc.writeByte('"')
@@ -108,8 +108,8 @@ func (enc *Encoder) AddObjectKeyOmitEmpty(key string, value MarshalerObject) {
return
}
enc.grow(5 + len(key))
- r, ok := enc.getPreviousRune()
- if ok && r != '{' {
+ r := enc.getPreviousRune()
+ if r != '{' {
enc.writeByte(',')
}
enc.writeByte('"')
diff --git a/encode_pool.go b/encode_pool.go
@@ -2,8 +2,27 @@ package gojay
import "io"
-var encPool = make(chan *Encoder, 16)
-var streamEncPool = make(chan *StreamEncoder, 16)
+var encPool = make(chan *Encoder, 32)
+var streamEncPool = make(chan *StreamEncoder, 32)
+
+func init() {
+initStreamEncPool:
+ for {
+ select {
+ case streamEncPool <- Stream.NewEncoder(nil):
+ default:
+ break initStreamEncPool
+ }
+ }
+initEncPool:
+ for {
+ select {
+ case encPool <- NewEncoder(nil):
+ default:
+ break initEncPool
+ }
+ }
+}
// NewEncoder returns a new encoder or borrows one from the pool
func NewEncoder(w io.Writer) *Encoder {
@@ -20,16 +39,15 @@ func BorrowEncoder(w io.Writer) *Encoder {
enc.isPooled = 0
enc.w = w
enc.err = nil
- enc.buf = make([]byte, 0)
return enc
default:
- return &Encoder{w: w}
+ return &Encoder{w: w, buf: make([]byte, 0, 512)}
}
}
// Release sends back a Encoder to the pool.
func (enc *Encoder) Release() {
- enc.buf = nil
+ enc.buf = enc.buf[:0]
enc.isPooled = 1
select {
case encPool <- enc:
diff --git a/encode_stream.go b/encode_stream.go
@@ -180,7 +180,7 @@ func consume(init *StreamEncoder, s *StreamEncoder, m MarshalerStream) {
init.Cancel(s.Encoder.err)
return
}
- i, err := s.Encoder.write()
+ i, err := s.Encoder.Write()
if err != nil || i == 0 {
init.Cancel(err)
return
diff --git a/encode_string.go b/encode_string.go
@@ -6,7 +6,7 @@ func (enc *Encoder) EncodeString(s string) error {
panic(InvalidUsagePooledEncoderError("Invalid usage of pooled encoder"))
}
_, _ = enc.encodeString(s)
- _, err := enc.write()
+ _, err := enc.Write()
if err != nil {
enc.err = err
return err
@@ -22,11 +22,18 @@ func (enc *Encoder) encodeString(v string) ([]byte, error) {
return enc.buf, nil
}
+func (enc *Encoder) AppendString(v string) {
+ enc.grow(len(v) + 2)
+ enc.writeByte('"')
+ enc.writeStringEscape(v)
+ enc.writeByte('"')
+}
+
// AddString adds a string to be encoded, must be used inside a slice or array encoding (does not encode a key)
func (enc *Encoder) AddString(v string) {
enc.grow(len(v) + 4)
- r, ok := enc.getPreviousRune()
- if ok && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '[' {
enc.writeByte(',')
}
enc.writeByte('"')
@@ -40,8 +47,8 @@ func (enc *Encoder) AddStringOmitEmpty(v string) {
if v == "" {
return
}
- r, ok := enc.getPreviousRune()
- if ok && r != '[' {
+ r := enc.getPreviousRune()
+ if r != '[' {
enc.writeByte(',')
}
enc.writeByte('"')
@@ -52,8 +59,8 @@ func (enc *Encoder) AddStringOmitEmpty(v string) {
// AddStringKey adds a string to be encoded, must be used inside an object as it will encode a key
func (enc *Encoder) AddStringKey(key, v string) {
enc.grow(len(key) + len(v) + 5)
- r, ok := enc.getPreviousRune()
- if ok && r != '{' {
+ r := enc.getPreviousRune()
+ if r != '{' {
enc.writeByte(',')
}
enc.writeByte('"')
@@ -70,8 +77,8 @@ func (enc *Encoder) AddStringKeyOmitEmpty(key, v string) {
return
}
enc.grow(len(key) + len(v) + 5)
- r, ok := enc.getPreviousRune()
- if ok && r != '{' {
+ r := enc.getPreviousRune()
+ if r != '{' {
enc.writeByte(',')
}
enc.writeByte('"')
diff --git a/encode_test.go b/encode_test.go
@@ -1,9 +1,45 @@
package gojay
-import "errors"
+import (
+ "errors"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
type TestWriterError string
func (t TestWriterError) Write(b []byte) (int, error) {
return 0, errors.New("Test Error")
}
+
+func TestAppendBytes(t *testing.T) {
+ b := []byte(``)
+ enc := NewEncoder(nil)
+ enc.buf = b
+ enc.AppendBytes([]byte(`true`))
+ assert.Equal(t, string(enc.buf), `true`, "string(enc.buf) should equal to true")
+}
+
+func TestAppendByte(t *testing.T) {
+ b := []byte(``)
+ enc := NewEncoder(nil)
+ enc.buf = b
+ enc.AppendByte(1)
+ assert.Equal(t, enc.buf[0], uint8(0x1), "b[0] should equal to 1")
+}
+
+func TestAppendString(t *testing.T) {
+ b := []byte(``)
+ enc := NewEncoder(nil)
+ enc.buf = b
+ enc.AppendString("true")
+ assert.Equal(t, string(enc.buf), `"true"`, "string(enc.buf) should equal to true")
+}
+
+func TestBuf(t *testing.T) {
+ b := []byte(`test`)
+ enc := NewEncoder(nil)
+ enc.buf = b
+ assert.Equal(t, b, enc.Buf(), "enc.Buf() should equal to b")
+}