From 8e8c2977da53c75361a9e23d90a43cc070fa0bbd Mon Sep 17 00:00:00 2001 From: Jessica Frazelle Date: Mon, 2 Nov 2015 10:26:42 -0800 Subject: [PATCH] rebased canonical/json off go 1.5.1 Signed-off-by: Jessica Frazelle --- Godeps/Godeps.json | 3 +- .../jfrazelle/go/canonical/json/bench_test.go | 189 --- .../jfrazelle/go/canonical/json/decode.go | 51 +- .../go/canonical/json/decode_test.go | 1373 ----------------- .../jfrazelle/go/canonical/json/encode.go | 27 +- .../go/canonical/json/encode_test.go | 605 -------- .../go/canonical/json/example_test.go | 161 -- .../jfrazelle/go/canonical/json/fold.go | 2 +- .../jfrazelle/go/canonical/json/fold_test.go | 116 -- .../jfrazelle/go/canonical/json/scanner.go | 9 +- .../go/canonical/json/scanner_test.go | 315 ---- .../jfrazelle/go/canonical/json/stream.go | 328 +++- .../go/canonical/json/stream_test.go | 206 --- .../go/canonical/json/tagkey_test.go | 115 -- .../jfrazelle/go/canonical/json/tags_test.go | 28 - 15 files changed, 364 insertions(+), 3164 deletions(-) delete mode 100644 Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/bench_test.go delete mode 100644 Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/decode_test.go delete mode 100644 Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/encode_test.go delete mode 100644 Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/example_test.go delete mode 100644 Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/fold_test.go delete mode 100644 Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/scanner_test.go delete mode 100644 Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/stream_test.go delete mode 100644 Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/tagkey_test.go delete mode 100644 Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/tags_test.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 010cab1c36..e853b1f265 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -125,7 +125,8 @@ }, { "ImportPath": "github.com/jfrazelle/go/canonical/json", - "Rev": "6e461eb70cb4187b41a84e9a567d7137bdbe0f16" + "Comment": "v1.5.1-1-1-gbaf439e", + "Rev": "baf439e6c161bd2106346fc8022b74ac2444e311" }, { "ImportPath": "github.com/jinzhu/gorm", diff --git a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/bench_test.go b/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/bench_test.go deleted file mode 100644 index 29dbc26d41..0000000000 --- a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/bench_test.go +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Large data benchmark. -// The JSON data is a summary of agl's changes in the -// go, webkit, and chromium open source projects. -// We benchmark converting between the JSON form -// and in-memory data structures. - -package json - -import ( - "bytes" - "compress/gzip" - "io/ioutil" - "os" - "testing" -) - -type codeResponse struct { - Tree *codeNode `json:"tree"` - Username string `json:"username"` -} - -type codeNode struct { - Name string `json:"name"` - Kids []*codeNode `json:"kids"` - CLWeight float64 `json:"cl_weight"` - Touches int `json:"touches"` - MinT int64 `json:"min_t"` - MaxT int64 `json:"max_t"` - MeanT int64 `json:"mean_t"` -} - -var codeJSON []byte -var codeStruct codeResponse - -func codeInit() { - f, err := os.Open("testdata/code.json.gz") - if err != nil { - panic(err) - } - defer f.Close() - gz, err := gzip.NewReader(f) - if err != nil { - panic(err) - } - data, err := ioutil.ReadAll(gz) - if err != nil { - panic(err) - } - - codeJSON = data - - if err := Unmarshal(codeJSON, &codeStruct); err != nil { - panic("unmarshal code.json: " + err.Error()) - } - - if data, err = Marshal(&codeStruct); err != nil { - panic("marshal code.json: " + err.Error()) - } - - if !bytes.Equal(data, codeJSON) { - println("different lengths", len(data), len(codeJSON)) - for i := 0; i < len(data) && i < len(codeJSON); i++ { - if data[i] != codeJSON[i] { - println("re-marshal: changed at byte", i) - println("orig: ", string(codeJSON[i-10:i+10])) - println("new: ", string(data[i-10:i+10])) - break - } - } - panic("re-marshal code.json: different result") - } -} - -func BenchmarkCodeEncoder(b *testing.B) { - if codeJSON == nil { - b.StopTimer() - codeInit() - b.StartTimer() - } - enc := NewEncoder(ioutil.Discard) - for i := 0; i < b.N; i++ { - if err := enc.Encode(&codeStruct); err != nil { - b.Fatal("Encode:", err) - } - } - b.SetBytes(int64(len(codeJSON))) -} - -func BenchmarkCodeMarshal(b *testing.B) { - if codeJSON == nil { - b.StopTimer() - codeInit() - b.StartTimer() - } - for i := 0; i < b.N; i++ { - if _, err := Marshal(&codeStruct); err != nil { - b.Fatal("Marshal:", err) - } - } - b.SetBytes(int64(len(codeJSON))) -} - -func BenchmarkCodeDecoder(b *testing.B) { - if codeJSON == nil { - b.StopTimer() - codeInit() - b.StartTimer() - } - var buf bytes.Buffer - dec := NewDecoder(&buf) - var r codeResponse - for i := 0; i < b.N; i++ { - buf.Write(codeJSON) - // hide EOF - buf.WriteByte('\n') - buf.WriteByte('\n') - buf.WriteByte('\n') - if err := dec.Decode(&r); err != nil { - b.Fatal("Decode:", err) - } - } - b.SetBytes(int64(len(codeJSON))) -} - -func BenchmarkCodeUnmarshal(b *testing.B) { - if codeJSON == nil { - b.StopTimer() - codeInit() - b.StartTimer() - } - for i := 0; i < b.N; i++ { - var r codeResponse - if err := Unmarshal(codeJSON, &r); err != nil { - b.Fatal("Unmmarshal:", err) - } - } - b.SetBytes(int64(len(codeJSON))) -} - -func BenchmarkCodeUnmarshalReuse(b *testing.B) { - if codeJSON == nil { - b.StopTimer() - codeInit() - b.StartTimer() - } - var r codeResponse - for i := 0; i < b.N; i++ { - if err := Unmarshal(codeJSON, &r); err != nil { - b.Fatal("Unmmarshal:", err) - } - } -} - -func BenchmarkUnmarshalString(b *testing.B) { - data := []byte(`"hello, world"`) - var s string - - for i := 0; i < b.N; i++ { - if err := Unmarshal(data, &s); err != nil { - b.Fatal("Unmarshal:", err) - } - } -} - -func BenchmarkUnmarshalFloat64(b *testing.B) { - var f float64 - data := []byte(`3.14`) - - for i := 0; i < b.N; i++ { - if err := Unmarshal(data, &f); err != nil { - b.Fatal("Unmarshal:", err) - } - } -} - -func BenchmarkUnmarshalInt64(b *testing.B) { - var x int64 - data := []byte(`3`) - - for i := 0; i < b.N; i++ { - if err := Unmarshal(data, &x); err != nil { - b.Fatal("Unmarshal:", err) - } - } -} diff --git a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/decode.go b/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/decode.go index 51b4ffea9d..35bac2e2b4 100644 --- a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/decode.go +++ b/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/decode.go @@ -48,6 +48,13 @@ import ( // map[string]interface{}, for JSON objects // nil for JSON null // +// To unmarshal a JSON array into a slice, Unmarshal resets the slice to nil +// and then appends each element to the slice. +// +// To unmarshal a JSON object into a map, Unmarshal replaces the map +// with an empty map and then adds key-value pairs from the object to +// the map. +// // If a JSON value is not appropriate for a given target type, // or if a JSON number overflows the target type, Unmarshal // skips that field and completes the unmarshalling as best it can. @@ -90,8 +97,9 @@ type Unmarshaler interface { // An UnmarshalTypeError describes a JSON value that was // not appropriate for a value of a specific Go type. type UnmarshalTypeError struct { - Value string // description of JSON value - "bool", "array", "number -5" - Type reflect.Type // type of Go value it could not be assigned to + Value string // description of JSON value - "bool", "array", "number -5" + Type reflect.Type // type of Go value it could not be assigned to + Offset int64 // error occurred after reading Offset bytes } func (e *UnmarshalTypeError) Error() string { @@ -378,7 +386,7 @@ func (d *decodeState) array(v reflect.Value) { return } if ut != nil { - d.saveError(&UnmarshalTypeError{"array", v.Type()}) + d.saveError(&UnmarshalTypeError{"array", v.Type(), int64(d.off)}) d.off-- d.next() return @@ -397,7 +405,7 @@ func (d *decodeState) array(v reflect.Value) { // Otherwise it's invalid. fallthrough default: - d.saveError(&UnmarshalTypeError{"array", v.Type()}) + d.saveError(&UnmarshalTypeError{"array", v.Type(), int64(d.off)}) d.off-- d.next() return @@ -486,7 +494,7 @@ func (d *decodeState) object(v reflect.Value) { return } if ut != nil { - d.saveError(&UnmarshalTypeError{"object", v.Type()}) + d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) d.off-- d.next() // skip over { } in input return @@ -505,7 +513,7 @@ func (d *decodeState) object(v reflect.Value) { // map must have string kind t := v.Type() if t.Key().Kind() != reflect.String { - d.saveError(&UnmarshalTypeError{"object", v.Type()}) + d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) d.off-- d.next() // skip over { } in input return @@ -516,7 +524,7 @@ func (d *decodeState) object(v reflect.Value) { case reflect.Struct: default: - d.saveError(&UnmarshalTypeError{"object", v.Type()}) + d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) d.off-- d.next() // skip over { } in input return @@ -600,7 +608,7 @@ func (d *decodeState) object(v reflect.Value) { case string: d.literalStore([]byte(qv), subv, true) default: - d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal unquoted value into %v", item, v.Type())) + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal unquoted value into %v", subv.Type())) } } else { d.value(subv) @@ -647,7 +655,7 @@ func (d *decodeState) convertNumber(s string) (interface{}, error) { } f, err := strconv.ParseFloat(s, 64) if err != nil { - return nil, &UnmarshalTypeError{"number " + s, reflect.TypeOf(0.0)} + return nil, &UnmarshalTypeError{"number " + s, reflect.TypeOf(0.0), int64(d.off)} } return f, nil } @@ -680,8 +688,9 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool if fromQuoted { d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) } else { - d.saveError(&UnmarshalTypeError{"string", v.Type()}) + d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) } + return } s, ok := unquoteBytes(item) if !ok { @@ -714,7 +723,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool if fromQuoted { d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) } else { - d.saveError(&UnmarshalTypeError{"bool", v.Type()}) + d.saveError(&UnmarshalTypeError{"bool", v.Type(), int64(d.off)}) } case reflect.Bool: v.SetBool(value) @@ -722,7 +731,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool if v.NumMethod() == 0 { v.Set(reflect.ValueOf(value)) } else { - d.saveError(&UnmarshalTypeError{"bool", v.Type()}) + d.saveError(&UnmarshalTypeError{"bool", v.Type(), int64(d.off)}) } } @@ -737,10 +746,10 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool } switch v.Kind() { default: - d.saveError(&UnmarshalTypeError{"string", v.Type()}) + d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) case reflect.Slice: - if v.Type() != byteSliceType { - d.saveError(&UnmarshalTypeError{"string", v.Type()}) + if v.Type().Elem().Kind() != reflect.Uint8 { + d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) break } b := make([]byte, base64.StdEncoding.DecodedLen(len(s))) @@ -756,7 +765,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool if v.NumMethod() == 0 { v.Set(reflect.ValueOf(string(s))) } else { - d.saveError(&UnmarshalTypeError{"string", v.Type()}) + d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) } } @@ -778,7 +787,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool if fromQuoted { d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) } else { - d.error(&UnmarshalTypeError{"number", v.Type()}) + d.error(&UnmarshalTypeError{"number", v.Type(), int64(d.off)}) } case reflect.Interface: n, err := d.convertNumber(s) @@ -787,7 +796,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool break } if v.NumMethod() != 0 { - d.saveError(&UnmarshalTypeError{"number", v.Type()}) + d.saveError(&UnmarshalTypeError{"number", v.Type(), int64(d.off)}) break } v.Set(reflect.ValueOf(n)) @@ -795,7 +804,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: n, err := strconv.ParseInt(s, 10, 64) if err != nil || v.OverflowInt(n) { - d.saveError(&UnmarshalTypeError{"number " + s, v.Type()}) + d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)}) break } v.SetInt(n) @@ -803,7 +812,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: n, err := strconv.ParseUint(s, 10, 64) if err != nil || v.OverflowUint(n) { - d.saveError(&UnmarshalTypeError{"number " + s, v.Type()}) + d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)}) break } v.SetUint(n) @@ -811,7 +820,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool case reflect.Float32, reflect.Float64: n, err := strconv.ParseFloat(s, v.Type().Bits()) if err != nil || v.OverflowFloat(n) { - d.saveError(&UnmarshalTypeError{"number " + s, v.Type()}) + d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)}) break } v.SetFloat(n) diff --git a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/decode_test.go b/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/decode_test.go deleted file mode 100644 index 7235969b9f..0000000000 --- a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/decode_test.go +++ /dev/null @@ -1,1373 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package json - -import ( - "bytes" - "encoding" - "fmt" - "image" - "reflect" - "strings" - "testing" - "time" -) - -type T struct { - X string - Y int - Z int `json:"-"` -} - -type U struct { - Alphabet string `json:"alpha"` -} - -type V struct { - F1 interface{} - F2 int32 - F3 Number -} - -// ifaceNumAsFloat64/ifaceNumAsNumber are used to test unmarshaling with and -// without UseNumber -var ifaceNumAsFloat64 = map[string]interface{}{ - "k1": float64(1), - "k2": "s", - "k3": []interface{}{float64(1), float64(2.0), float64(3e-3)}, - "k4": map[string]interface{}{"kk1": "s", "kk2": float64(2)}, -} - -var ifaceNumAsNumber = map[string]interface{}{ - "k1": Number("1"), - "k2": "s", - "k3": []interface{}{Number("1"), Number("2.0"), Number("3e-3")}, - "k4": map[string]interface{}{"kk1": "s", "kk2": Number("2")}, -} - -type tx struct { - x int -} - -// A type that can unmarshal itself. - -type unmarshaler struct { - T bool -} - -func (u *unmarshaler) UnmarshalJSON(b []byte) error { - *u = unmarshaler{true} // All we need to see that UnmarshalJSON is called. - return nil -} - -type ustruct struct { - M unmarshaler -} - -type unmarshalerText struct { - T bool -} - -// needed for re-marshaling tests -func (u *unmarshalerText) MarshalText() ([]byte, error) { - return []byte(""), nil -} - -func (u *unmarshalerText) UnmarshalText(b []byte) error { - *u = unmarshalerText{true} // All we need to see that UnmarshalText is called. - return nil -} - -var _ encoding.TextUnmarshaler = (*unmarshalerText)(nil) - -type ustructText struct { - M unmarshalerText -} - -var ( - um0, um1 unmarshaler // target2 of unmarshaling - ump = &um1 - umtrue = unmarshaler{true} - umslice = []unmarshaler{{true}} - umslicep = new([]unmarshaler) - umstruct = ustruct{unmarshaler{true}} - - um0T, um1T unmarshalerText // target2 of unmarshaling - umpT = &um1T - umtrueT = unmarshalerText{true} - umsliceT = []unmarshalerText{{true}} - umslicepT = new([]unmarshalerText) - umstructT = ustructText{unmarshalerText{true}} -) - -// Test data structures for anonymous fields. - -type Point struct { - Z int -} - -type Top struct { - Level0 int - Embed0 - *Embed0a - *Embed0b `json:"e,omitempty"` // treated as named - Embed0c `json:"-"` // ignored - Loop - Embed0p // has Point with X, Y, used - Embed0q // has Point with Z, used -} - -type Embed0 struct { - Level1a int // overridden by Embed0a's Level1a with json tag - Level1b int // used because Embed0a's Level1b is renamed - Level1c int // used because Embed0a's Level1c is ignored - Level1d int // annihilated by Embed0a's Level1d - Level1e int `json:"x"` // annihilated by Embed0a.Level1e -} - -type Embed0a struct { - Level1a int `json:"Level1a,omitempty"` - Level1b int `json:"LEVEL1B,omitempty"` - Level1c int `json:"-"` - Level1d int // annihilated by Embed0's Level1d - Level1f int `json:"x"` // annihilated by Embed0's Level1e -} - -type Embed0b Embed0 - -type Embed0c Embed0 - -type Embed0p struct { - image.Point -} - -type Embed0q struct { - Point -} - -type Loop struct { - Loop1 int `json:",omitempty"` - Loop2 int `json:",omitempty"` - *Loop -} - -// From reflect test: -// The X in S6 and S7 annihilate, but they also block the X in S8.S9. -type S5 struct { - S6 - S7 - S8 -} - -type S6 struct { - X int -} - -type S7 S6 - -type S8 struct { - S9 -} - -type S9 struct { - X int - Y int -} - -// From reflect test: -// The X in S11.S6 and S12.S6 annihilate, but they also block the X in S13.S8.S9. -type S10 struct { - S11 - S12 - S13 -} - -type S11 struct { - S6 -} - -type S12 struct { - S6 -} - -type S13 struct { - S8 -} - -type unmarshalTest struct { - in string - ptr interface{} - out interface{} - err error - useNumber bool -} - -type Ambig struct { - // Given "hello", the first match should win. - First int `json:"HELLO"` - Second int `json:"Hello"` -} - -type XYZ struct { - X interface{} - Y interface{} - Z interface{} -} - -var unmarshalTests = []unmarshalTest{ - // basic types - {in: `true`, ptr: new(bool), out: true}, - {in: `1`, ptr: new(int), out: 1}, - {in: `1.2`, ptr: new(float64), out: 1.2}, - {in: `-5`, ptr: new(int16), out: int16(-5)}, - {in: `2`, ptr: new(Number), out: Number("2"), useNumber: true}, - {in: `2`, ptr: new(Number), out: Number("2")}, - {in: `2`, ptr: new(interface{}), out: float64(2.0)}, - {in: `2`, ptr: new(interface{}), out: Number("2"), useNumber: true}, - {in: `"a\u1234"`, ptr: new(string), out: "a\u1234"}, - {in: `"http:\/\/"`, ptr: new(string), out: "http://"}, - {in: `"g-clef: \uD834\uDD1E"`, ptr: new(string), out: "g-clef: \U0001D11E"}, - {in: `"invalid: \uD834x\uDD1E"`, ptr: new(string), out: "invalid: \uFFFDx\uFFFD"}, - {in: "null", ptr: new(interface{}), out: nil}, - {in: `{"X": [1,2,3], "Y": 4}`, ptr: new(T), out: T{Y: 4}, err: &UnmarshalTypeError{"array", reflect.TypeOf("")}}, - {in: `{"x": 1}`, ptr: new(tx), out: tx{}}, - {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: float64(1), F2: int32(2), F3: Number("3")}}, - {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: Number("1"), F2: int32(2), F3: Number("3")}, useNumber: true}, - {in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsFloat64}, - {in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsNumber, useNumber: true}, - - // raw values with whitespace - {in: "\n true ", ptr: new(bool), out: true}, - {in: "\t 1 ", ptr: new(int), out: 1}, - {in: "\r 1.2 ", ptr: new(float64), out: 1.2}, - {in: "\t -5 \n", ptr: new(int16), out: int16(-5)}, - {in: "\t \"a\\u1234\" \n", ptr: new(string), out: "a\u1234"}, - - // Z has a "-" tag. - {in: `{"Y": 1, "Z": 2}`, ptr: new(T), out: T{Y: 1}}, - - {in: `{"alpha": "abc", "alphabet": "xyz"}`, ptr: new(U), out: U{Alphabet: "abc"}}, - {in: `{"alpha": "abc"}`, ptr: new(U), out: U{Alphabet: "abc"}}, - {in: `{"alphabet": "xyz"}`, ptr: new(U), out: U{}}, - - // syntax errors - {in: `{"X": "foo", "Y"}`, err: &SyntaxError{"invalid character '}' after object key", 17}}, - {in: `[1, 2, 3+]`, err: &SyntaxError{"invalid character '+' after array element", 9}}, - {in: `{"X":12x}`, err: &SyntaxError{"invalid character 'x' after object key:value pair", 8}, useNumber: true}, - - // raw value errors - {in: "\x01 42", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, - {in: " 42 \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 5}}, - {in: "\x01 true", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, - {in: " false \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 8}}, - {in: "\x01 1.2", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, - {in: " 3.4 \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 6}}, - {in: "\x01 \"string\"", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, - {in: " \"string\" \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 11}}, - - // array tests - {in: `[1, 2, 3]`, ptr: new([3]int), out: [3]int{1, 2, 3}}, - {in: `[1, 2, 3]`, ptr: new([1]int), out: [1]int{1}}, - {in: `[1, 2, 3]`, ptr: new([5]int), out: [5]int{1, 2, 3, 0, 0}}, - - // empty array to interface test - {in: `[]`, ptr: new([]interface{}), out: []interface{}{}}, - {in: `null`, ptr: new([]interface{}), out: []interface{}(nil)}, - {in: `{"T":[]}`, ptr: new(map[string]interface{}), out: map[string]interface{}{"T": []interface{}{}}}, - {in: `{"T":null}`, ptr: new(map[string]interface{}), out: map[string]interface{}{"T": interface{}(nil)}}, - - // composite tests - {in: allValueIndent, ptr: new(All), out: allValue}, - {in: allValueCompact, ptr: new(All), out: allValue}, - {in: allValueIndent, ptr: new(*All), out: &allValue}, - {in: allValueCompact, ptr: new(*All), out: &allValue}, - {in: pallValueIndent, ptr: new(All), out: pallValue}, - {in: pallValueCompact, ptr: new(All), out: pallValue}, - {in: pallValueIndent, ptr: new(*All), out: &pallValue}, - {in: pallValueCompact, ptr: new(*All), out: &pallValue}, - - // unmarshal interface test - {in: `{"T":false}`, ptr: &um0, out: umtrue}, // use "false" so test will fail if custom unmarshaler is not called - {in: `{"T":false}`, ptr: &ump, out: &umtrue}, - {in: `[{"T":false}]`, ptr: &umslice, out: umslice}, - {in: `[{"T":false}]`, ptr: &umslicep, out: &umslice}, - {in: `{"M":{"T":false}}`, ptr: &umstruct, out: umstruct}, - - // UnmarshalText interface test - {in: `"X"`, ptr: &um0T, out: umtrueT}, // use "false" so test will fail if custom unmarshaler is not called - {in: `"X"`, ptr: &umpT, out: &umtrueT}, - {in: `["X"]`, ptr: &umsliceT, out: umsliceT}, - {in: `["X"]`, ptr: &umslicepT, out: &umsliceT}, - {in: `{"M":"X"}`, ptr: &umstructT, out: umstructT}, - - { - in: `{ - "Level0": 1, - "Level1b": 2, - "Level1c": 3, - "x": 4, - "Level1a": 5, - "LEVEL1B": 6, - "e": { - "Level1a": 8, - "Level1b": 9, - "Level1c": 10, - "Level1d": 11, - "x": 12 - }, - "Loop1": 13, - "Loop2": 14, - "X": 15, - "Y": 16, - "Z": 17 - }`, - ptr: new(Top), - out: Top{ - Level0: 1, - Embed0: Embed0{ - Level1b: 2, - Level1c: 3, - }, - Embed0a: &Embed0a{ - Level1a: 5, - Level1b: 6, - }, - Embed0b: &Embed0b{ - Level1a: 8, - Level1b: 9, - Level1c: 10, - Level1d: 11, - Level1e: 12, - }, - Loop: Loop{ - Loop1: 13, - Loop2: 14, - }, - Embed0p: Embed0p{ - Point: image.Point{X: 15, Y: 16}, - }, - Embed0q: Embed0q{ - Point: Point{Z: 17}, - }, - }, - }, - { - in: `{"hello": 1}`, - ptr: new(Ambig), - out: Ambig{First: 1}, - }, - - { - in: `{"X": 1,"Y":2}`, - ptr: new(S5), - out: S5{S8: S8{S9: S9{Y: 2}}}, - }, - { - in: `{"X": 1,"Y":2}`, - ptr: new(S10), - out: S10{S13: S13{S8: S8{S9: S9{Y: 2}}}}, - }, - - // invalid UTF-8 is coerced to valid UTF-8. - { - in: "\"hello\xffworld\"", - ptr: new(string), - out: "hello\ufffdworld", - }, - { - in: "\"hello\xc2\xc2world\"", - ptr: new(string), - out: "hello\ufffd\ufffdworld", - }, - { - in: "\"hello\xc2\xffworld\"", - ptr: new(string), - out: "hello\ufffd\ufffdworld", - }, - { - in: "\"hello\\ud800world\"", - ptr: new(string), - out: "hello\ufffdworld", - }, - { - in: "\"hello\\ud800\\ud800world\"", - ptr: new(string), - out: "hello\ufffd\ufffdworld", - }, - { - in: "\"hello\\ud800\\ud800world\"", - ptr: new(string), - out: "hello\ufffd\ufffdworld", - }, - { - in: "\"hello\xed\xa0\x80\xed\xb0\x80world\"", - ptr: new(string), - out: "hello\ufffd\ufffd\ufffd\ufffd\ufffd\ufffdworld", - }, - - // issue 8305 - { - in: `{"2009-11-10T23:00:00Z": "hello world"}`, - ptr: &map[time.Time]string{}, - err: &UnmarshalTypeError{"object", reflect.TypeOf(map[time.Time]string{})}, - }, -} - -func TestMarshal(t *testing.T) { - b, err := Marshal(allValue) - if err != nil { - t.Fatalf("Marshal allValue: %v", err) - } - if string(b) != allValueCompact { - t.Errorf("Marshal allValueCompact") - diff(t, b, []byte(allValueCompact)) - return - } - - b, err = Marshal(pallValue) - if err != nil { - t.Fatalf("Marshal pallValue: %v", err) - } - if string(b) != pallValueCompact { - t.Errorf("Marshal pallValueCompact") - diff(t, b, []byte(pallValueCompact)) - return - } -} - -var badUTF8 = []struct { - in, out string -}{ - {"hello\xffworld", `"hello\ufffdworld"`}, - {"", `""`}, - {"\xff", `"\ufffd"`}, - {"\xff\xff", `"\ufffd\ufffd"`}, - {"a\xffb", `"a\ufffdb"`}, - {"\xe6\x97\xa5\xe6\x9c\xac\xff\xaa\x9e", `"日本\ufffd\ufffd\ufffd"`}, -} - -func TestMarshalBadUTF8(t *testing.T) { - for _, tt := range badUTF8 { - b, err := Marshal(tt.in) - if string(b) != tt.out || err != nil { - t.Errorf("Marshal(%q) = %#q, %v, want %#q, nil", tt.in, b, err, tt.out) - } - } -} - -func TestMarshalNumberZeroVal(t *testing.T) { - var n Number - out, err := Marshal(n) - if err != nil { - t.Fatal(err) - } - outStr := string(out) - if outStr != "0" { - t.Fatalf("Invalid zero val for Number: %q", outStr) - } -} - -func TestMarshalEmbeds(t *testing.T) { - top := &Top{ - Level0: 1, - Embed0: Embed0{ - Level1b: 2, - Level1c: 3, - }, - Embed0a: &Embed0a{ - Level1a: 5, - Level1b: 6, - }, - Embed0b: &Embed0b{ - Level1a: 8, - Level1b: 9, - Level1c: 10, - Level1d: 11, - Level1e: 12, - }, - Loop: Loop{ - Loop1: 13, - Loop2: 14, - }, - Embed0p: Embed0p{ - Point: image.Point{X: 15, Y: 16}, - }, - Embed0q: Embed0q{ - Point: Point{Z: 17}, - }, - } - b, err := Marshal(top) - if err != nil { - t.Fatal(err) - } - want := "{\"Level0\":1,\"Level1b\":2,\"Level1c\":3,\"Level1a\":5,\"LEVEL1B\":6,\"e\":{\"Level1a\":8,\"Level1b\":9,\"Level1c\":10,\"Level1d\":11,\"x\":12},\"Loop1\":13,\"Loop2\":14,\"X\":15,\"Y\":16,\"Z\":17}" - if string(b) != want { - t.Errorf("Wrong marshal result.\n got: %q\nwant: %q", b, want) - } -} - -func TestUnmarshal(t *testing.T) { - for i, tt := range unmarshalTests { - var scan scanner - in := []byte(tt.in) - if err := checkValid(in, &scan); err != nil { - if !reflect.DeepEqual(err, tt.err) { - t.Errorf("#%d: checkValid: %#v", i, err) - continue - } - } - if tt.ptr == nil { - continue - } - - // v = new(right-type) - v := reflect.New(reflect.TypeOf(tt.ptr).Elem()) - dec := NewDecoder(bytes.NewReader(in)) - if tt.useNumber { - dec.UseNumber() - } - if err := dec.Decode(v.Interface()); !reflect.DeepEqual(err, tt.err) { - t.Errorf("#%d: %v, want %v", i, err, tt.err) - continue - } else if err != nil { - continue - } - if !reflect.DeepEqual(v.Elem().Interface(), tt.out) { - t.Errorf("#%d: mismatch\nhave: %#+v\nwant: %#+v", i, v.Elem().Interface(), tt.out) - data, _ := Marshal(v.Elem().Interface()) - println(string(data)) - data, _ = Marshal(tt.out) - println(string(data)) - continue - } - - // Check round trip. - if tt.err == nil { - enc, err := Marshal(v.Interface()) - if err != nil { - t.Errorf("#%d: error re-marshaling: %v", i, err) - continue - } - vv := reflect.New(reflect.TypeOf(tt.ptr).Elem()) - dec = NewDecoder(bytes.NewReader(enc)) - if tt.useNumber { - dec.UseNumber() - } - if err := dec.Decode(vv.Interface()); err != nil { - t.Errorf("#%d: error re-unmarshaling %#q: %v", i, enc, err) - continue - } - if !reflect.DeepEqual(v.Elem().Interface(), vv.Elem().Interface()) { - t.Errorf("#%d: mismatch\nhave: %#+v\nwant: %#+v", i, v.Elem().Interface(), vv.Elem().Interface()) - t.Errorf(" In: %q", strings.Map(noSpace, string(in))) - t.Errorf("Marshal: %q", strings.Map(noSpace, string(enc))) - continue - } - } - } -} - -func TestUnmarshalMarshal(t *testing.T) { - initBig() - var v interface{} - if err := Unmarshal(jsonBig, &v); err != nil { - t.Fatalf("Unmarshal: %v", err) - } - b, err := Marshal(v) - if err != nil { - t.Fatalf("Marshal: %v", err) - } - if !bytes.Equal(jsonBig, b) { - t.Errorf("Marshal jsonBig") - diff(t, b, jsonBig) - return - } -} - -var numberTests = []struct { - in string - i int64 - intErr string - f float64 - floatErr string -}{ - {in: "-1.23e1", intErr: "strconv.ParseInt: parsing \"-1.23e1\": invalid syntax", f: -1.23e1}, - {in: "-12", i: -12, f: -12.0}, - {in: "1e1000", intErr: "strconv.ParseInt: parsing \"1e1000\": invalid syntax", floatErr: "strconv.ParseFloat: parsing \"1e1000\": value out of range"}, -} - -// Independent of Decode, basic coverage of the accessors in Number -func TestNumberAccessors(t *testing.T) { - for _, tt := range numberTests { - n := Number(tt.in) - if s := n.String(); s != tt.in { - t.Errorf("Number(%q).String() is %q", tt.in, s) - } - if i, err := n.Int64(); err == nil && tt.intErr == "" && i != tt.i { - t.Errorf("Number(%q).Int64() is %d", tt.in, i) - } else if (err == nil && tt.intErr != "") || (err != nil && err.Error() != tt.intErr) { - t.Errorf("Number(%q).Int64() wanted error %q but got: %v", tt.in, tt.intErr, err) - } - if f, err := n.Float64(); err == nil && tt.floatErr == "" && f != tt.f { - t.Errorf("Number(%q).Float64() is %g", tt.in, f) - } else if (err == nil && tt.floatErr != "") || (err != nil && err.Error() != tt.floatErr) { - t.Errorf("Number(%q).Float64() wanted error %q but got: %v", tt.in, tt.floatErr, err) - } - } -} - -func TestLargeByteSlice(t *testing.T) { - s0 := make([]byte, 2000) - for i := range s0 { - s0[i] = byte(i) - } - b, err := Marshal(s0) - if err != nil { - t.Fatalf("Marshal: %v", err) - } - var s1 []byte - if err := Unmarshal(b, &s1); err != nil { - t.Fatalf("Unmarshal: %v", err) - } - if !bytes.Equal(s0, s1) { - t.Errorf("Marshal large byte slice") - diff(t, s0, s1) - } -} - -type Xint struct { - X int -} - -func TestUnmarshalInterface(t *testing.T) { - var xint Xint - var i interface{} = &xint - if err := Unmarshal([]byte(`{"X":1}`), &i); err != nil { - t.Fatalf("Unmarshal: %v", err) - } - if xint.X != 1 { - t.Fatalf("Did not write to xint") - } -} - -func TestUnmarshalPtrPtr(t *testing.T) { - var xint Xint - pxint := &xint - if err := Unmarshal([]byte(`{"X":1}`), &pxint); err != nil { - t.Fatalf("Unmarshal: %v", err) - } - if xint.X != 1 { - t.Fatalf("Did not write to xint") - } -} - -func TestEscape(t *testing.T) { - const input = `"foobar"` + " [\u2028 \u2029]" - const expected = `"\"foobar\"\u003chtml\u003e [\u2028 \u2029]"` - b, err := Marshal(input) - if err != nil { - t.Fatalf("Marshal error: %v", err) - } - if s := string(b); s != expected { - t.Errorf("Encoding of [%s]:\n got [%s]\nwant [%s]", input, s, expected) - } -} - -// WrongString is a struct that's misusing the ,string modifier. -type WrongString struct { - Message string `json:"result,string"` -} - -type wrongStringTest struct { - in, err string -} - -var wrongStringTests = []wrongStringTest{ - {`{"result":"x"}`, `json: invalid use of ,string struct tag, trying to unmarshal "x" into string`}, - {`{"result":"foo"}`, `json: invalid use of ,string struct tag, trying to unmarshal "foo" into string`}, - {`{"result":"123"}`, `json: invalid use of ,string struct tag, trying to unmarshal "123" into string`}, -} - -// If people misuse the ,string modifier, the error message should be -// helpful, telling the user that they're doing it wrong. -func TestErrorMessageFromMisusedString(t *testing.T) { - for n, tt := range wrongStringTests { - r := strings.NewReader(tt.in) - var s WrongString - err := NewDecoder(r).Decode(&s) - got := fmt.Sprintf("%v", err) - if got != tt.err { - t.Errorf("%d. got err = %q, want %q", n, got, tt.err) - } - } -} - -func noSpace(c rune) rune { - if isSpace(c) { - return -1 - } - return c -} - -type All struct { - Bool bool - Int int - Int8 int8 - Int16 int16 - Int32 int32 - Int64 int64 - Uint uint - Uint8 uint8 - Uint16 uint16 - Uint32 uint32 - Uint64 uint64 - Uintptr uintptr - Float32 float32 - Float64 float64 - - Foo string `json:"bar"` - Foo2 string `json:"bar2,dummyopt"` - - IntStr int64 `json:",string"` - - PBool *bool - PInt *int - PInt8 *int8 - PInt16 *int16 - PInt32 *int32 - PInt64 *int64 - PUint *uint - PUint8 *uint8 - PUint16 *uint16 - PUint32 *uint32 - PUint64 *uint64 - PUintptr *uintptr - PFloat32 *float32 - PFloat64 *float64 - - String string - PString *string - - Map map[string]Small - MapP map[string]*Small - PMap *map[string]Small - PMapP *map[string]*Small - - EmptyMap map[string]Small - NilMap map[string]Small - - Slice []Small - SliceP []*Small - PSlice *[]Small - PSliceP *[]*Small - - EmptySlice []Small - NilSlice []Small - - StringSlice []string - ByteSlice []byte - - Small Small - PSmall *Small - PPSmall **Small - - Interface interface{} - PInterface *interface{} - - unexported int -} - -type Small struct { - Tag string -} - -var allValue = All{ - Bool: true, - Int: 2, - Int8: 3, - Int16: 4, - Int32: 5, - Int64: 6, - Uint: 7, - Uint8: 8, - Uint16: 9, - Uint32: 10, - Uint64: 11, - Uintptr: 12, - Float32: 14.1, - Float64: 15.1, - Foo: "foo", - Foo2: "foo2", - IntStr: 42, - String: "16", - Map: map[string]Small{ - "17": {Tag: "tag17"}, - "18": {Tag: "tag18"}, - }, - MapP: map[string]*Small{ - "19": {Tag: "tag19"}, - "20": nil, - }, - EmptyMap: map[string]Small{}, - Slice: []Small{{Tag: "tag20"}, {Tag: "tag21"}}, - SliceP: []*Small{{Tag: "tag22"}, nil, {Tag: "tag23"}}, - EmptySlice: []Small{}, - StringSlice: []string{"str24", "str25", "str26"}, - ByteSlice: []byte{27, 28, 29}, - Small: Small{Tag: "tag30"}, - PSmall: &Small{Tag: "tag31"}, - Interface: 5.2, -} - -var pallValue = All{ - PBool: &allValue.Bool, - PInt: &allValue.Int, - PInt8: &allValue.Int8, - PInt16: &allValue.Int16, - PInt32: &allValue.Int32, - PInt64: &allValue.Int64, - PUint: &allValue.Uint, - PUint8: &allValue.Uint8, - PUint16: &allValue.Uint16, - PUint32: &allValue.Uint32, - PUint64: &allValue.Uint64, - PUintptr: &allValue.Uintptr, - PFloat32: &allValue.Float32, - PFloat64: &allValue.Float64, - PString: &allValue.String, - PMap: &allValue.Map, - PMapP: &allValue.MapP, - PSlice: &allValue.Slice, - PSliceP: &allValue.SliceP, - PPSmall: &allValue.PSmall, - PInterface: &allValue.Interface, -} - -var allValueIndent = `{ - "Bool": true, - "Int": 2, - "Int8": 3, - "Int16": 4, - "Int32": 5, - "Int64": 6, - "Uint": 7, - "Uint8": 8, - "Uint16": 9, - "Uint32": 10, - "Uint64": 11, - "Uintptr": 12, - "Float32": 14.1, - "Float64": 15.1, - "bar": "foo", - "bar2": "foo2", - "IntStr": "42", - "PBool": null, - "PInt": null, - "PInt8": null, - "PInt16": null, - "PInt32": null, - "PInt64": null, - "PUint": null, - "PUint8": null, - "PUint16": null, - "PUint32": null, - "PUint64": null, - "PUintptr": null, - "PFloat32": null, - "PFloat64": null, - "String": "16", - "PString": null, - "Map": { - "17": { - "Tag": "tag17" - }, - "18": { - "Tag": "tag18" - } - }, - "MapP": { - "19": { - "Tag": "tag19" - }, - "20": null - }, - "PMap": null, - "PMapP": null, - "EmptyMap": {}, - "NilMap": null, - "Slice": [ - { - "Tag": "tag20" - }, - { - "Tag": "tag21" - } - ], - "SliceP": [ - { - "Tag": "tag22" - }, - null, - { - "Tag": "tag23" - } - ], - "PSlice": null, - "PSliceP": null, - "EmptySlice": [], - "NilSlice": null, - "StringSlice": [ - "str24", - "str25", - "str26" - ], - "ByteSlice": "Gxwd", - "Small": { - "Tag": "tag30" - }, - "PSmall": { - "Tag": "tag31" - }, - "PPSmall": null, - "Interface": 5.2, - "PInterface": null -}` - -var allValueCompact = strings.Map(noSpace, allValueIndent) - -var pallValueIndent = `{ - "Bool": false, - "Int": 0, - "Int8": 0, - "Int16": 0, - "Int32": 0, - "Int64": 0, - "Uint": 0, - "Uint8": 0, - "Uint16": 0, - "Uint32": 0, - "Uint64": 0, - "Uintptr": 0, - "Float32": 0, - "Float64": 0, - "bar": "", - "bar2": "", - "IntStr": "0", - "PBool": true, - "PInt": 2, - "PInt8": 3, - "PInt16": 4, - "PInt32": 5, - "PInt64": 6, - "PUint": 7, - "PUint8": 8, - "PUint16": 9, - "PUint32": 10, - "PUint64": 11, - "PUintptr": 12, - "PFloat32": 14.1, - "PFloat64": 15.1, - "String": "", - "PString": "16", - "Map": null, - "MapP": null, - "PMap": { - "17": { - "Tag": "tag17" - }, - "18": { - "Tag": "tag18" - } - }, - "PMapP": { - "19": { - "Tag": "tag19" - }, - "20": null - }, - "EmptyMap": null, - "NilMap": null, - "Slice": null, - "SliceP": null, - "PSlice": [ - { - "Tag": "tag20" - }, - { - "Tag": "tag21" - } - ], - "PSliceP": [ - { - "Tag": "tag22" - }, - null, - { - "Tag": "tag23" - } - ], - "EmptySlice": null, - "NilSlice": null, - "StringSlice": null, - "ByteSlice": null, - "Small": { - "Tag": "" - }, - "PSmall": null, - "PPSmall": { - "Tag": "tag31" - }, - "Interface": null, - "PInterface": 5.2 -}` - -var pallValueCompact = strings.Map(noSpace, pallValueIndent) - -func TestRefUnmarshal(t *testing.T) { - type S struct { - // Ref is defined in encode_test.go. - R0 Ref - R1 *Ref - R2 RefText - R3 *RefText - } - want := S{ - R0: 12, - R1: new(Ref), - R2: 13, - R3: new(RefText), - } - *want.R1 = 12 - *want.R3 = 13 - - var got S - if err := Unmarshal([]byte(`{"R0":"ref","R1":"ref","R2":"ref","R3":"ref"}`), &got); err != nil { - t.Fatalf("Unmarshal: %v", err) - } - if !reflect.DeepEqual(got, want) { - t.Errorf("got %+v, want %+v", got, want) - } -} - -// Test that the empty string doesn't panic decoding when ,string is specified -// Issue 3450 -func TestEmptyString(t *testing.T) { - type T2 struct { - Number1 int `json:",string"` - Number2 int `json:",string"` - } - data := `{"Number1":"1", "Number2":""}` - dec := NewDecoder(strings.NewReader(data)) - var t2 T2 - err := dec.Decode(&t2) - if err == nil { - t.Fatal("Decode: did not return error") - } - if t2.Number1 != 1 { - t.Fatal("Decode: did not set Number1") - } -} - -// Test that a null for ,string is not replaced with the previous quoted string (issue 7046). -// It should also not be an error (issue 2540, issue 8587). -func TestNullString(t *testing.T) { - type T struct { - A int `json:",string"` - B int `json:",string"` - C *int `json:",string"` - } - data := []byte(`{"A": "1", "B": null, "C": null}`) - var s T - s.B = 1 - s.C = new(int) - *s.C = 2 - err := Unmarshal(data, &s) - if err != nil { - t.Fatalf("Unmarshal: %v") - } - if s.B != 1 || s.C != nil { - t.Fatalf("after Unmarshal, s.B=%d, s.C=%p, want 1, nil", s.B, s.C) - } -} - -func intp(x int) *int { - p := new(int) - *p = x - return p -} - -func intpp(x *int) **int { - pp := new(*int) - *pp = x - return pp -} - -var interfaceSetTests = []struct { - pre interface{} - json string - post interface{} -}{ - {"foo", `"bar"`, "bar"}, - {"foo", `2`, 2.0}, - {"foo", `true`, true}, - {"foo", `null`, nil}, - - {nil, `null`, nil}, - {new(int), `null`, nil}, - {(*int)(nil), `null`, nil}, - {new(*int), `null`, new(*int)}, - {(**int)(nil), `null`, nil}, - {intp(1), `null`, nil}, - {intpp(nil), `null`, intpp(nil)}, - {intpp(intp(1)), `null`, intpp(nil)}, -} - -func TestInterfaceSet(t *testing.T) { - for _, tt := range interfaceSetTests { - b := struct{ X interface{} }{tt.pre} - blob := `{"X":` + tt.json + `}` - if err := Unmarshal([]byte(blob), &b); err != nil { - t.Errorf("Unmarshal %#q: %v", blob, err) - continue - } - if !reflect.DeepEqual(b.X, tt.post) { - t.Errorf("Unmarshal %#q into %#v: X=%#v, want %#v", blob, tt.pre, b.X, tt.post) - } - } -} - -// JSON null values should be ignored for primitives and string values instead of resulting in an error. -// Issue 2540 -func TestUnmarshalNulls(t *testing.T) { - jsonData := []byte(`{ - "Bool" : null, - "Int" : null, - "Int8" : null, - "Int16" : null, - "Int32" : null, - "Int64" : null, - "Uint" : null, - "Uint8" : null, - "Uint16" : null, - "Uint32" : null, - "Uint64" : null, - "Float32" : null, - "Float64" : null, - "String" : null}`) - - nulls := All{ - Bool: true, - Int: 2, - Int8: 3, - Int16: 4, - Int32: 5, - Int64: 6, - Uint: 7, - Uint8: 8, - Uint16: 9, - Uint32: 10, - Uint64: 11, - Float32: 12.1, - Float64: 13.1, - String: "14"} - - err := Unmarshal(jsonData, &nulls) - if err != nil { - t.Errorf("Unmarshal of null values failed: %v", err) - } - if !nulls.Bool || nulls.Int != 2 || nulls.Int8 != 3 || nulls.Int16 != 4 || nulls.Int32 != 5 || nulls.Int64 != 6 || - nulls.Uint != 7 || nulls.Uint8 != 8 || nulls.Uint16 != 9 || nulls.Uint32 != 10 || nulls.Uint64 != 11 || - nulls.Float32 != 12.1 || nulls.Float64 != 13.1 || nulls.String != "14" { - - t.Errorf("Unmarshal of null values affected primitives") - } -} - -func TestStringKind(t *testing.T) { - type stringKind string - - var m1, m2 map[stringKind]int - m1 = map[stringKind]int{ - "foo": 42, - } - - data, err := Marshal(m1) - if err != nil { - t.Errorf("Unexpected error marshalling: %v", err) - } - - err = Unmarshal(data, &m2) - if err != nil { - t.Errorf("Unexpected error unmarshalling: %v", err) - } - - if !reflect.DeepEqual(m1, m2) { - t.Error("Items should be equal after encoding and then decoding") - } - -} - -var decodeTypeErrorTests = []struct { - dest interface{} - src string -}{ - {new(string), `{"user": "name"}`}, // issue 4628. - {new(error), `{}`}, // issue 4222 - {new(error), `[]`}, - {new(error), `""`}, - {new(error), `123`}, - {new(error), `true`}, -} - -func TestUnmarshalTypeError(t *testing.T) { - for _, item := range decodeTypeErrorTests { - err := Unmarshal([]byte(item.src), item.dest) - if _, ok := err.(*UnmarshalTypeError); !ok { - t.Errorf("expected type error for Unmarshal(%q, type %T): got %T", - item.src, item.dest, err) - } - } -} - -var unmarshalSyntaxTests = []string{ - "tru", - "fals", - "nul", - "123e", - `"hello`, - `[1,2,3`, - `{"key":1`, - `{"key":1,`, -} - -func TestUnmarshalSyntax(t *testing.T) { - var x interface{} - for _, src := range unmarshalSyntaxTests { - err := Unmarshal([]byte(src), &x) - if _, ok := err.(*SyntaxError); !ok { - t.Errorf("expected syntax error for Unmarshal(%q): got %T", src, err) - } - } -} - -// Test handling of unexported fields that should be ignored. -// Issue 4660 -type unexportedFields struct { - Name string - m map[string]interface{} `json:"-"` - m2 map[string]interface{} `json:"abcd"` -} - -func TestUnmarshalUnexported(t *testing.T) { - input := `{"Name": "Bob", "m": {"x": 123}, "m2": {"y": 456}, "abcd": {"z": 789}}` - want := &unexportedFields{Name: "Bob"} - - out := &unexportedFields{} - err := Unmarshal([]byte(input), out) - if err != nil { - t.Errorf("got error %v, expected nil", err) - } - if !reflect.DeepEqual(out, want) { - t.Errorf("got %q, want %q", out, want) - } -} - -// Time3339 is a time.Time which encodes to and from JSON -// as an RFC 3339 time in UTC. -type Time3339 time.Time - -func (t *Time3339) UnmarshalJSON(b []byte) error { - if len(b) < 2 || b[0] != '"' || b[len(b)-1] != '"' { - return fmt.Errorf("types: failed to unmarshal non-string value %q as an RFC 3339 time", b) - } - tm, err := time.Parse(time.RFC3339, string(b[1:len(b)-1])) - if err != nil { - return err - } - *t = Time3339(tm) - return nil -} - -func TestUnmarshalJSONLiteralError(t *testing.T) { - var t3 Time3339 - err := Unmarshal([]byte(`"0000-00-00T00:00:00Z"`), &t3) - if err == nil { - t.Fatalf("expected error; got time %v", time.Time(t3)) - } - if !strings.Contains(err.Error(), "range") { - t.Errorf("got err = %v; want out of range error", err) - } -} - -// Test that extra object elements in an array do not result in a -// "data changing underfoot" error. -// Issue 3717 -func TestSkipArrayObjects(t *testing.T) { - json := `[{}]` - var dest [0]interface{} - - err := Unmarshal([]byte(json), &dest) - if err != nil { - t.Errorf("got error %q, want nil", err) - } -} - -// Test semantics of pre-filled struct fields and pre-filled map fields. -// Issue 4900. -func TestPrefilled(t *testing.T) { - ptrToMap := func(m map[string]interface{}) *map[string]interface{} { return &m } - - // Values here change, cannot reuse table across runs. - var prefillTests = []struct { - in string - ptr interface{} - out interface{} - }{ - { - in: `{"X": 1, "Y": 2}`, - ptr: &XYZ{X: float32(3), Y: int16(4), Z: 1.5}, - out: &XYZ{X: float64(1), Y: float64(2), Z: 1.5}, - }, - { - in: `{"X": 1, "Y": 2}`, - ptr: ptrToMap(map[string]interface{}{"X": float32(3), "Y": int16(4), "Z": 1.5}), - out: ptrToMap(map[string]interface{}{"X": float64(1), "Y": float64(2), "Z": 1.5}), - }, - } - - for _, tt := range prefillTests { - ptrstr := fmt.Sprintf("%v", tt.ptr) - err := Unmarshal([]byte(tt.in), tt.ptr) // tt.ptr edited here - if err != nil { - t.Errorf("Unmarshal: %v", err) - } - if !reflect.DeepEqual(tt.ptr, tt.out) { - t.Errorf("Unmarshal(%#q, %s): have %v, want %v", tt.in, ptrstr, tt.ptr, tt.out) - } - } -} - -var invalidUnmarshalTests = []struct { - v interface{} - want string -}{ - {nil, "json: Unmarshal(nil)"}, - {struct{}{}, "json: Unmarshal(non-pointer struct {})"}, - {(*int)(nil), "json: Unmarshal(nil *int)"}, -} - -func TestInvalidUnmarshal(t *testing.T) { - buf := []byte(`{"a":"1"}`) - for _, tt := range invalidUnmarshalTests { - err := Unmarshal(buf, tt.v) - if err == nil { - t.Errorf("Unmarshal expecting error, got nil") - continue - } - if got := err.Error(); got != tt.want { - t.Errorf("Unmarshal = %q; want %q", got, tt.want) - } - } -} diff --git a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/encode.go b/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/encode.go index 655f1a6661..0fab020e2c 100644 --- a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/encode.go +++ b/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/encode.go @@ -7,7 +7,7 @@ // in the documentation for the Marshal and Unmarshal functions. // // See "JSON and Go" for an introduction to this package: -// http://golang.org/doc/articles/json_and_go.html +// https://golang.org/doc/articles/json_and_go.html package json import ( @@ -79,8 +79,8 @@ import ( // // The "string" option signals that a field is stored as JSON inside a // JSON-encoded string. It applies only to fields of string, floating point, -// or integer types. This extra level of encoding is sometimes used when -// communicating with JavaScript programs: +// integer, or boolean types. This extra level of encoding is sometimes used +// when communicating with JavaScript programs: // // Int64String int64 `json:",string"` // @@ -113,8 +113,8 @@ import ( // a JSON tag of "-". // // Map values encode as JSON objects. -// The map's key type must be string; the object keys are used directly -// as map keys. +// The map's key type must be string; the map keys are used as JSON object +// keys, subject to the UTF-8 coercion described for string values above. // // Pointer values encode as the value pointed to. // A nil pointer encodes as the null JSON object. @@ -287,8 +287,6 @@ func (e *encodeState) error(err error) { panic(err) } -var byteSliceType = reflect.TypeOf([]byte(nil)) - func isEmptyValue(v reflect.Value) bool { switch v.Kind() { case reflect.Array, reflect.Map, reflect.Slice, reflect.String: @@ -1075,6 +1073,19 @@ func typeFields(t reflect.Type) []field { ft = ft.Elem() } + // Only strings, floats, integers, and booleans can be quoted. + quoted := false + if opts.Contains("string") { + switch ft.Kind() { + case reflect.Bool, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + reflect.Float32, reflect.Float64, + reflect.String: + quoted = true + } + } + // Record found field and index sequence. if name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { tagged := name != "" @@ -1087,7 +1098,7 @@ func typeFields(t reflect.Type) []field { index: index, typ: ft, omitEmpty: opts.Contains("omitempty"), - quoted: opts.Contains("string"), + quoted: quoted, })) if count[f.typ] > 1 { // If there were multiple instances, add a second, diff --git a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/encode_test.go b/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/encode_test.go deleted file mode 100644 index 2e42b4e722..0000000000 --- a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/encode_test.go +++ /dev/null @@ -1,605 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package json - -import ( - "bytes" - "fmt" - "math" - "reflect" - "testing" - "unicode" -) - -type Optionals struct { - Sr string `json:"sr"` - So string `json:"so,omitempty"` - Sw string `json:"-"` - - Ir int `json:"omitempty"` // actually named omitempty, not an option - Io int `json:"io,omitempty"` - - Slr []string `json:"slr,random"` - Slo []string `json:"slo,omitempty"` - - Mr map[string]interface{} `json:"mr"` - Mo map[string]interface{} `json:",omitempty"` - - Fr float64 `json:"fr"` - Fo float64 `json:"fo,omitempty"` - - Br bool `json:"br"` - Bo bool `json:"bo,omitempty"` - - Ur uint `json:"ur"` - Uo uint `json:"uo,omitempty"` - - Str struct{} `json:"str"` - Sto struct{} `json:"sto,omitempty"` -} - -var optionalsExpected = `{ - "sr": "", - "omitempty": 0, - "slr": null, - "mr": {}, - "fr": 0, - "br": false, - "ur": 0, - "str": {}, - "sto": {} -}` - -func TestOmitEmpty(t *testing.T) { - var o Optionals - o.Sw = "something" - o.Mr = map[string]interface{}{} - o.Mo = map[string]interface{}{} - - got, err := MarshalIndent(&o, "", " ") - if err != nil { - t.Fatal(err) - } - if got := string(got); got != optionalsExpected { - t.Errorf(" got: %s\nwant: %s\n", got, optionalsExpected) - } -} - -type StringTag struct { - BoolStr bool `json:",string"` - IntStr int64 `json:",string"` - StrStr string `json:",string"` -} - -var stringTagExpected = `{ - "BoolStr": "true", - "IntStr": "42", - "StrStr": "\"xzbit\"" -}` - -func TestStringTag(t *testing.T) { - var s StringTag - s.BoolStr = true - s.IntStr = 42 - s.StrStr = "xzbit" - got, err := MarshalIndent(&s, "", " ") - if err != nil { - t.Fatal(err) - } - if got := string(got); got != stringTagExpected { - t.Fatalf(" got: %s\nwant: %s\n", got, stringTagExpected) - } - - // Verify that it round-trips. - var s2 StringTag - err = NewDecoder(bytes.NewReader(got)).Decode(&s2) - if err != nil { - t.Fatalf("Decode: %v", err) - } - if !reflect.DeepEqual(s, s2) { - t.Fatalf("decode didn't match.\nsource: %#v\nEncoded as:\n%s\ndecode: %#v", s, string(got), s2) - } -} - -// byte slices are special even if they're renamed types. -type renamedByte byte -type renamedByteSlice []byte -type renamedRenamedByteSlice []renamedByte - -func TestEncodeRenamedByteSlice(t *testing.T) { - s := renamedByteSlice("abc") - result, err := Marshal(s) - if err != nil { - t.Fatal(err) - } - expect := `"YWJj"` - if string(result) != expect { - t.Errorf(" got %s want %s", result, expect) - } - r := renamedRenamedByteSlice("abc") - result, err = Marshal(r) - if err != nil { - t.Fatal(err) - } - if string(result) != expect { - t.Errorf(" got %s want %s", result, expect) - } -} - -var unsupportedValues = []interface{}{ - math.NaN(), - math.Inf(-1), - math.Inf(1), -} - -func TestUnsupportedValues(t *testing.T) { - for _, v := range unsupportedValues { - if _, err := Marshal(v); err != nil { - if _, ok := err.(*UnsupportedValueError); !ok { - t.Errorf("for %v, got %T want UnsupportedValueError", v, err) - } - } else { - t.Errorf("for %v, expected error", v) - } - } -} - -// Ref has Marshaler and Unmarshaler methods with pointer receiver. -type Ref int - -func (*Ref) MarshalJSON() ([]byte, error) { - return []byte(`"ref"`), nil -} - -func (r *Ref) UnmarshalJSON([]byte) error { - *r = 12 - return nil -} - -// Val has Marshaler methods with value receiver. -type Val int - -func (Val) MarshalJSON() ([]byte, error) { - return []byte(`"val"`), nil -} - -// RefText has Marshaler and Unmarshaler methods with pointer receiver. -type RefText int - -func (*RefText) MarshalText() ([]byte, error) { - return []byte(`"ref"`), nil -} - -func (r *RefText) UnmarshalText([]byte) error { - *r = 13 - return nil -} - -// ValText has Marshaler methods with value receiver. -type ValText int - -func (ValText) MarshalText() ([]byte, error) { - return []byte(`"val"`), nil -} - -func TestRefValMarshal(t *testing.T) { - var s = struct { - R0 Ref - R1 *Ref - R2 RefText - R3 *RefText - V0 Val - V1 *Val - V2 ValText - V3 *ValText - }{ - R0: 12, - R1: new(Ref), - R2: 14, - R3: new(RefText), - V0: 13, - V1: new(Val), - V2: 15, - V3: new(ValText), - } - const want = `{"R0":"ref","R1":"ref","R2":"\"ref\"","R3":"\"ref\"","V0":"val","V1":"val","V2":"\"val\"","V3":"\"val\""}` - b, err := Marshal(&s) - if err != nil { - t.Fatalf("Marshal: %v", err) - } - if got := string(b); got != want { - t.Errorf("got %q, want %q", got, want) - } -} - -// C implements Marshaler and returns unescaped JSON. -type C int - -func (C) MarshalJSON() ([]byte, error) { - return []byte(`"<&>"`), nil -} - -// CText implements Marshaler and returns unescaped text. -type CText int - -func (CText) MarshalText() ([]byte, error) { - return []byte(`"<&>"`), nil -} - -func TestMarshalerEscaping(t *testing.T) { - var c C - want := `"\u003c\u0026\u003e"` - b, err := Marshal(c) - if err != nil { - t.Fatalf("Marshal(c): %v", err) - } - if got := string(b); got != want { - t.Errorf("Marshal(c) = %#q, want %#q", got, want) - } - - var ct CText - want = `"\"\u003c\u0026\u003e\""` - b, err = Marshal(ct) - if err != nil { - t.Fatalf("Marshal(ct): %v", err) - } - if got := string(b); got != want { - t.Errorf("Marshal(ct) = %#q, want %#q", got, want) - } -} - -type IntType int - -type MyStruct struct { - IntType -} - -func TestAnonymousNonstruct(t *testing.T) { - var i IntType = 11 - a := MyStruct{i} - const want = `{"IntType":11}` - - b, err := Marshal(a) - if err != nil { - t.Fatalf("Marshal: %v", err) - } - if got := string(b); got != want { - t.Errorf("got %q, want %q", got, want) - } -} - -type BugA struct { - S string -} - -type BugB struct { - BugA - S string -} - -type BugC struct { - S string -} - -// Legal Go: We never use the repeated embedded field (S). -type BugX struct { - A int - BugA - BugB -} - -// Issue 5245. -func TestEmbeddedBug(t *testing.T) { - v := BugB{ - BugA{"A"}, - "B", - } - b, err := Marshal(v) - if err != nil { - t.Fatal("Marshal:", err) - } - want := `{"S":"B"}` - got := string(b) - if got != want { - t.Fatalf("Marshal: got %s want %s", got, want) - } - // Now check that the duplicate field, S, does not appear. - x := BugX{ - A: 23, - } - b, err = Marshal(x) - if err != nil { - t.Fatal("Marshal:", err) - } - want = `{"A":23}` - got = string(b) - if got != want { - t.Fatalf("Marshal: got %s want %s", got, want) - } -} - -type BugD struct { // Same as BugA after tagging. - XXX string `json:"S"` -} - -// BugD's tagged S field should dominate BugA's. -type BugY struct { - BugA - BugD -} - -// Test that a field with a tag dominates untagged fields. -func TestTaggedFieldDominates(t *testing.T) { - v := BugY{ - BugA{"BugA"}, - BugD{"BugD"}, - } - b, err := Marshal(v) - if err != nil { - t.Fatal("Marshal:", err) - } - want := `{"S":"BugD"}` - got := string(b) - if got != want { - t.Fatalf("Marshal: got %s want %s", got, want) - } -} - -// There are no tags here, so S should not appear. -type BugZ struct { - BugA - BugC - BugY // Contains a tagged S field through BugD; should not dominate. -} - -func TestDuplicatedFieldDisappears(t *testing.T) { - v := BugZ{ - BugA{"BugA"}, - BugC{"BugC"}, - BugY{ - BugA{"nested BugA"}, - BugD{"nested BugD"}, - }, - } - b, err := Marshal(v) - if err != nil { - t.Fatal("Marshal:", err) - } - want := `{}` - got := string(b) - if got != want { - t.Fatalf("Marshal: got %s want %s", got, want) - } -} - -func TestStringBytes(t *testing.T) { - // Test that encodeState.stringBytes and encodeState.string use the same encoding. - es := &encodeState{} - var r []rune - for i := '\u0000'; i <= unicode.MaxRune; i++ { - r = append(r, i) - } - s := string(r) + "\xff\xff\xffhello" // some invalid UTF-8 too - _, err := es.string(s) - if err != nil { - t.Fatal(err) - } - - esBytes := &encodeState{} - _, err = esBytes.stringBytes([]byte(s)) - if err != nil { - t.Fatal(err) - } - - enc := es.Buffer.String() - encBytes := esBytes.Buffer.String() - if enc != encBytes { - i := 0 - for i < len(enc) && i < len(encBytes) && enc[i] == encBytes[i] { - i++ - } - enc = enc[i:] - encBytes = encBytes[i:] - i = 0 - for i < len(enc) && i < len(encBytes) && enc[len(enc)-i-1] == encBytes[len(encBytes)-i-1] { - i++ - } - enc = enc[:len(enc)-i] - encBytes = encBytes[:len(encBytes)-i] - - if len(enc) > 20 { - enc = enc[:20] + "..." - } - if len(encBytes) > 20 { - encBytes = encBytes[:20] + "..." - } - - t.Errorf("encodings differ at %#q vs %#q", enc, encBytes) - } -} - -func TestIssue6458(t *testing.T) { - type Foo struct { - M RawMessage - } - x := Foo{RawMessage(`"foo"`)} - - b, err := Marshal(&x) - if err != nil { - t.Fatal(err) - } - if want := `{"M":"foo"}`; string(b) != want { - t.Errorf("Marshal(&x) = %#q; want %#q", b, want) - } - - b, err = Marshal(x) - if err != nil { - t.Fatal(err) - } - - if want := `{"M":"ImZvbyI="}`; string(b) != want { - t.Errorf("Marshal(x) = %#q; want %#q", b, want) - } -} - -func TestHTMLEscape(t *testing.T) { - var b, want bytes.Buffer - m := `{"M":"foo &` + "\xe2\x80\xa8 \xe2\x80\xa9" + `"}` - want.Write([]byte(`{"M":"\u003chtml\u003efoo \u0026\u2028 \u2029\u003c/html\u003e"}`)) - HTMLEscape(&b, []byte(m)) - if !bytes.Equal(b.Bytes(), want.Bytes()) { - t.Errorf("HTMLEscape(&b, []byte(m)) = %s; want %s", b.Bytes(), want.Bytes()) - } -} - -// golang.org/issue/8582 -func TestEncodePointerString(t *testing.T) { - type stringPointer struct { - N *int64 `json:"n,string"` - } - var n int64 = 42 - b, err := Marshal(stringPointer{N: &n}) - if err != nil { - t.Fatalf("Marshal: %v", err) - } - if got, want := string(b), `{"n":"42"}`; got != want { - t.Errorf("Marshal = %s, want %s", got, want) - } - var back stringPointer - err = Unmarshal(b, &back) - if err != nil { - t.Fatalf("Unmarshal: %v", err) - } - if back.N == nil { - t.Fatalf("Unmarshalled nil N field") - } - if *back.N != 42 { - t.Fatalf("*N = %d; want 42", *back.N) - } -} - -var encodeStringTests = []struct { - in string - out string -}{ - {"\x00", `"\u0000"`}, - {"\x01", `"\u0001"`}, - {"\x02", `"\u0002"`}, - {"\x03", `"\u0003"`}, - {"\x04", `"\u0004"`}, - {"\x05", `"\u0005"`}, - {"\x06", `"\u0006"`}, - {"\x07", `"\u0007"`}, - {"\x08", `"\u0008"`}, - {"\x09", `"\t"`}, - {"\x0a", `"\n"`}, - {"\x0b", `"\u000b"`}, - {"\x0c", `"\u000c"`}, - {"\x0d", `"\r"`}, - {"\x0e", `"\u000e"`}, - {"\x0f", `"\u000f"`}, - {"\x10", `"\u0010"`}, - {"\x11", `"\u0011"`}, - {"\x12", `"\u0012"`}, - {"\x13", `"\u0013"`}, - {"\x14", `"\u0014"`}, - {"\x15", `"\u0015"`}, - {"\x16", `"\u0016"`}, - {"\x17", `"\u0017"`}, - {"\x18", `"\u0018"`}, - {"\x19", `"\u0019"`}, - {"\x1a", `"\u001a"`}, - {"\x1b", `"\u001b"`}, - {"\x1c", `"\u001c"`}, - {"\x1d", `"\u001d"`}, - {"\x1e", `"\u001e"`}, - {"\x1f", `"\u001f"`}, -} - -func TestEncodeString(t *testing.T) { - for _, tt := range encodeStringTests { - b, err := Marshal(tt.in) - if err != nil { - t.Errorf("Marshal(%q): %v", tt.in, err) - continue - } - out := string(b) - if out != tt.out { - t.Errorf("Marshal(%q) = %#q, want %#q", tt.in, out, tt.out) - } - } -} - -type CanonicalTestStruct struct { - S string - F float64 - I int - E *CanonicalTestStruct -} - -func (s *CanonicalTestStruct) String() string { - var e interface{} = s.E - if s.E == nil { - e = "nil" - } - return fmt.Sprintf("{S:%q F:%v I:%v E:%v}", s.S, s.F, s.I, e) -} - -var encodeCanonicalTests = []struct { - in interface{} - out string - expectErr bool -}{ - {nil, `null`, false}, - {&CanonicalTestStruct{}, `{"E":null,"F":0,"I":0,"S":""}`, false}, - {&CanonicalTestStruct{F: 1.0}, `{"E":null,"F":1,"I":0,"S":""}`, false}, - // error out on floating numbers - {&CanonicalTestStruct{F: 1.2}, ``, true}, - {&CanonicalTestStruct{S: "foo", E: &CanonicalTestStruct{I: 42}}, `{"E":{"E":null,"F":0,"I":42,"S":""},"F":0,"I":0,"S":"foo"}`, false}, - // only escape \ and " and keep any other character as-is - {"\u0090 \t \\ \n \"", "\"\u0090 \t \\\\ \n \\\"\"", false}, -} - -func TestEncodeCanonicalStruct(t *testing.T) { - for _, tt := range encodeCanonicalTests { - b, err := MarshalCanonical(tt.in) - if err != nil { - if !tt.expectErr { - t.Errorf("MarshalCanonical(%#v) = error(%v), want %s", tt.in, err, tt.out) - } - continue - } else if tt.expectErr { - t.Errorf("MarshalCanonical(%#v) expects an error", tt.in) - continue - } - out := string(b) - if out != tt.out { - t.Errorf("MarshalCanonical(%#v) = %q, want %q", tt.in, out, tt.out) - } - } -} - -func TestCanonicalFloatError(t *testing.T) { - input := struct{ A float64 }{1.1} - - _, err := MarshalCanonical(input) - if err == nil { - t.Errorf("want float error, got nil") - } -} - -func TestCanonicalFloatAsInt(t *testing.T) { - in := struct{ A float64 }{1234567} - - b, err := MarshalCanonical(in) - if err != nil { - t.Fatalf("Marshal(%q): %v", in, err) - } - out := string(b) - expected := `{"A":1234567}` - if out != expected { - t.Errorf("Marshal(%q) = %#q, want %#q", in, out, expected) - } -} diff --git a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/example_test.go b/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/example_test.go deleted file mode 100644 index ca4e5ae68d..0000000000 --- a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/example_test.go +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package json_test - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "log" - "os" - "strings" -) - -func ExampleMarshal() { - type ColorGroup struct { - ID int - Name string - Colors []string - } - group := ColorGroup{ - ID: 1, - Name: "Reds", - Colors: []string{"Crimson", "Red", "Ruby", "Maroon"}, - } - b, err := json.Marshal(group) - if err != nil { - fmt.Println("error:", err) - } - os.Stdout.Write(b) - // Output: - // {"ID":1,"Name":"Reds","Colors":["Crimson","Red","Ruby","Maroon"]} -} - -func ExampleUnmarshal() { - var jsonBlob = []byte(`[ - {"Name": "Platypus", "Order": "Monotremata"}, - {"Name": "Quoll", "Order": "Dasyuromorphia"} - ]`) - type Animal struct { - Name string - Order string - } - var animals []Animal - err := json.Unmarshal(jsonBlob, &animals) - if err != nil { - fmt.Println("error:", err) - } - fmt.Printf("%+v", animals) - // Output: - // [{Name:Platypus Order:Monotremata} {Name:Quoll Order:Dasyuromorphia}] -} - -// This example uses a Decoder to decode a stream of distinct JSON values. -func ExampleDecoder() { - const jsonStream = ` - {"Name": "Ed", "Text": "Knock knock."} - {"Name": "Sam", "Text": "Who's there?"} - {"Name": "Ed", "Text": "Go fmt."} - {"Name": "Sam", "Text": "Go fmt who?"} - {"Name": "Ed", "Text": "Go fmt yourself!"} - ` - type Message struct { - Name, Text string - } - dec := json.NewDecoder(strings.NewReader(jsonStream)) - for { - var m Message - if err := dec.Decode(&m); err == io.EOF { - break - } else if err != nil { - log.Fatal(err) - } - fmt.Printf("%s: %s\n", m.Name, m.Text) - } - // Output: - // Ed: Knock knock. - // Sam: Who's there? - // Ed: Go fmt. - // Sam: Go fmt who? - // Ed: Go fmt yourself! -} - -// This example uses RawMessage to delay parsing part of a JSON message. -func ExampleRawMessage() { - type Color struct { - Space string - Point json.RawMessage // delay parsing until we know the color space - } - type RGB struct { - R uint8 - G uint8 - B uint8 - } - type YCbCr struct { - Y uint8 - Cb int8 - Cr int8 - } - - var j = []byte(`[ - {"Space": "YCbCr", "Point": {"Y": 255, "Cb": 0, "Cr": -10}}, - {"Space": "RGB", "Point": {"R": 98, "G": 218, "B": 255}} - ]`) - var colors []Color - err := json.Unmarshal(j, &colors) - if err != nil { - log.Fatalln("error:", err) - } - - for _, c := range colors { - var dst interface{} - switch c.Space { - case "RGB": - dst = new(RGB) - case "YCbCr": - dst = new(YCbCr) - } - err := json.Unmarshal(c.Point, dst) - if err != nil { - log.Fatalln("error:", err) - } - fmt.Println(c.Space, dst) - } - // Output: - // YCbCr &{255 0 -10} - // RGB &{98 218 255} -} - -func ExampleIndent() { - type Road struct { - Name string - Number int - } - roads := []Road{ - {"Diamond Fork", 29}, - {"Sheep Creek", 51}, - } - - b, err := json.Marshal(roads) - if err != nil { - log.Fatal(err) - } - - var out bytes.Buffer - json.Indent(&out, b, "=", "\t") - out.WriteTo(os.Stdout) - // Output: - // [ - // = { - // = "Name": "Diamond Fork", - // = "Number": 29 - // = }, - // = { - // = "Name": "Sheep Creek", - // = "Number": 51 - // = } - // =] -} diff --git a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/fold.go b/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/fold.go index d6f77c93e5..9e170127db 100644 --- a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/fold.go +++ b/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/fold.go @@ -26,7 +26,7 @@ const ( // The letters S and K are special because they map to 3 runes, not just 2: // * S maps to s and to U+017F 'ſ' Latin small letter long s // * k maps to K and to U+212A 'K' Kelvin sign -// See http://play.golang.org/p/tTxjOc0OGo +// See https://play.golang.org/p/tTxjOc0OGo // // The returned function is specialized for matching against s and // should only be given s. It's not curried for performance reasons. diff --git a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/fold_test.go b/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/fold_test.go deleted file mode 100644 index 9fb94646a8..0000000000 --- a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/fold_test.go +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package json - -import ( - "bytes" - "strings" - "testing" - "unicode/utf8" -) - -var foldTests = []struct { - fn func(s, t []byte) bool - s, t string - want bool -}{ - {equalFoldRight, "", "", true}, - {equalFoldRight, "a", "a", true}, - {equalFoldRight, "", "a", false}, - {equalFoldRight, "a", "", false}, - {equalFoldRight, "a", "A", true}, - {equalFoldRight, "AB", "ab", true}, - {equalFoldRight, "AB", "ac", false}, - {equalFoldRight, "sbkKc", "ſbKKc", true}, - {equalFoldRight, "SbKkc", "ſbKKc", true}, - {equalFoldRight, "SbKkc", "ſbKK", false}, - {equalFoldRight, "e", "é", false}, - {equalFoldRight, "s", "S", true}, - - {simpleLetterEqualFold, "", "", true}, - {simpleLetterEqualFold, "abc", "abc", true}, - {simpleLetterEqualFold, "abc", "ABC", true}, - {simpleLetterEqualFold, "abc", "ABCD", false}, - {simpleLetterEqualFold, "abc", "xxx", false}, - - {asciiEqualFold, "a_B", "A_b", true}, - {asciiEqualFold, "aa@", "aa`", false}, // verify 0x40 and 0x60 aren't case-equivalent -} - -func TestFold(t *testing.T) { - for i, tt := range foldTests { - if got := tt.fn([]byte(tt.s), []byte(tt.t)); got != tt.want { - t.Errorf("%d. %q, %q = %v; want %v", i, tt.s, tt.t, got, tt.want) - } - truth := strings.EqualFold(tt.s, tt.t) - if truth != tt.want { - t.Errorf("strings.EqualFold doesn't agree with case %d", i) - } - } -} - -func TestFoldAgainstUnicode(t *testing.T) { - const bufSize = 5 - buf1 := make([]byte, 0, bufSize) - buf2 := make([]byte, 0, bufSize) - var runes []rune - for i := 0x20; i <= 0x7f; i++ { - runes = append(runes, rune(i)) - } - runes = append(runes, kelvin, smallLongEss) - - funcs := []struct { - name string - fold func(s, t []byte) bool - letter bool // must be ASCII letter - simple bool // must be simple ASCII letter (not 'S' or 'K') - }{ - { - name: "equalFoldRight", - fold: equalFoldRight, - }, - { - name: "asciiEqualFold", - fold: asciiEqualFold, - simple: true, - }, - { - name: "simpleLetterEqualFold", - fold: simpleLetterEqualFold, - simple: true, - letter: true, - }, - } - - for _, ff := range funcs { - for _, r := range runes { - if r >= utf8.RuneSelf { - continue - } - if ff.letter && !isASCIILetter(byte(r)) { - continue - } - if ff.simple && (r == 's' || r == 'S' || r == 'k' || r == 'K') { - continue - } - for _, r2 := range runes { - buf1 := append(buf1[:0], 'x') - buf2 := append(buf2[:0], 'x') - buf1 = buf1[:1+utf8.EncodeRune(buf1[1:bufSize], r)] - buf2 = buf2[:1+utf8.EncodeRune(buf2[1:bufSize], r2)] - buf1 = append(buf1, 'x') - buf2 = append(buf2, 'x') - want := bytes.EqualFold(buf1, buf2) - if got := ff.fold(buf1, buf2); got != want { - t.Errorf("%s(%q, %q) = %v; want %v", ff.name, buf1, buf2, got, want) - } - } - } - } -} - -func isASCIILetter(b byte) bool { - return ('A' <= b && b <= 'Z') || ('a' <= b && b <= 'z') -} diff --git a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/scanner.go b/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/scanner.go index a4609c8950..38d0b0802b 100644 --- a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/scanner.go +++ b/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/scanner.go @@ -38,8 +38,15 @@ func nextValue(data []byte, scan *scanner) (value, rest []byte, err error) { scan.reset() for i, c := range data { v := scan.step(scan, int(c)) - if v >= scanEnd { + if v >= scanEndObject { switch v { + // probe the scanner with a space to determine whether we will + // get scanEnd on the next character. Otherwise, if the next character + // is not a space, scanEndTop allocates a needless error. + case scanEndObject, scanEndArray: + if scan.step(scan, ' ') == scanEnd { + return data[:i+1], data[i+1:], nil + } case scanError: return nil, nil, scan.err case scanEnd: diff --git a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/scanner_test.go b/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/scanner_test.go deleted file mode 100644 index 7880342902..0000000000 --- a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/scanner_test.go +++ /dev/null @@ -1,315 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package json - -import ( - "bytes" - "math" - "math/rand" - "reflect" - "testing" -) - -// Tests of simple examples. - -type example struct { - compact string - indent string -} - -var examples = []example{ - {`1`, `1`}, - {`{}`, `{}`}, - {`[]`, `[]`}, - {`{"":2}`, "{\n\t\"\": 2\n}"}, - {`[3]`, "[\n\t3\n]"}, - {`[1,2,3]`, "[\n\t1,\n\t2,\n\t3\n]"}, - {`{"x":1}`, "{\n\t\"x\": 1\n}"}, - {ex1, ex1i}, -} - -var ex1 = `[true,false,null,"x",1,1.5,0,-5e+2]` - -var ex1i = `[ - true, - false, - null, - "x", - 1, - 1.5, - 0, - -5e+2 -]` - -func TestCompact(t *testing.T) { - var buf bytes.Buffer - for _, tt := range examples { - buf.Reset() - if err := Compact(&buf, []byte(tt.compact)); err != nil { - t.Errorf("Compact(%#q): %v", tt.compact, err) - } else if s := buf.String(); s != tt.compact { - t.Errorf("Compact(%#q) = %#q, want original", tt.compact, s) - } - - buf.Reset() - if err := Compact(&buf, []byte(tt.indent)); err != nil { - t.Errorf("Compact(%#q): %v", tt.indent, err) - continue - } else if s := buf.String(); s != tt.compact { - t.Errorf("Compact(%#q) = %#q, want %#q", tt.indent, s, tt.compact) - } - } -} - -func TestCompactSeparators(t *testing.T) { - // U+2028 and U+2029 should be escaped inside strings. - // They should not appear outside strings. - tests := []struct { - in, compact string - }{ - {"{\"\u2028\": 1}", `{"\u2028":1}`}, - {"{\"\u2029\" :2}", `{"\u2029":2}`}, - } - for _, tt := range tests { - var buf bytes.Buffer - if err := Compact(&buf, []byte(tt.in)); err != nil { - t.Errorf("Compact(%q): %v", tt.in, err) - } else if s := buf.String(); s != tt.compact { - t.Errorf("Compact(%q) = %q, want %q", tt.in, s, tt.compact) - } - } -} - -func TestIndent(t *testing.T) { - var buf bytes.Buffer - for _, tt := range examples { - buf.Reset() - if err := Indent(&buf, []byte(tt.indent), "", "\t"); err != nil { - t.Errorf("Indent(%#q): %v", tt.indent, err) - } else if s := buf.String(); s != tt.indent { - t.Errorf("Indent(%#q) = %#q, want original", tt.indent, s) - } - - buf.Reset() - if err := Indent(&buf, []byte(tt.compact), "", "\t"); err != nil { - t.Errorf("Indent(%#q): %v", tt.compact, err) - continue - } else if s := buf.String(); s != tt.indent { - t.Errorf("Indent(%#q) = %#q, want %#q", tt.compact, s, tt.indent) - } - } -} - -// Tests of a large random structure. - -func TestCompactBig(t *testing.T) { - initBig() - var buf bytes.Buffer - if err := Compact(&buf, jsonBig); err != nil { - t.Fatalf("Compact: %v", err) - } - b := buf.Bytes() - if !bytes.Equal(b, jsonBig) { - t.Error("Compact(jsonBig) != jsonBig") - diff(t, b, jsonBig) - return - } -} - -func TestIndentBig(t *testing.T) { - initBig() - var buf bytes.Buffer - if err := Indent(&buf, jsonBig, "", "\t"); err != nil { - t.Fatalf("Indent1: %v", err) - } - b := buf.Bytes() - if len(b) == len(jsonBig) { - // jsonBig is compact (no unnecessary spaces); - // indenting should make it bigger - t.Fatalf("Indent(jsonBig) did not get bigger") - } - - // should be idempotent - var buf1 bytes.Buffer - if err := Indent(&buf1, b, "", "\t"); err != nil { - t.Fatalf("Indent2: %v", err) - } - b1 := buf1.Bytes() - if !bytes.Equal(b1, b) { - t.Error("Indent(Indent(jsonBig)) != Indent(jsonBig)") - diff(t, b1, b) - return - } - - // should get back to original - buf1.Reset() - if err := Compact(&buf1, b); err != nil { - t.Fatalf("Compact: %v", err) - } - b1 = buf1.Bytes() - if !bytes.Equal(b1, jsonBig) { - t.Error("Compact(Indent(jsonBig)) != jsonBig") - diff(t, b1, jsonBig) - return - } -} - -type indentErrorTest struct { - in string - err error -} - -var indentErrorTests = []indentErrorTest{ - {`{"X": "foo", "Y"}`, &SyntaxError{"invalid character '}' after object key", 17}}, - {`{"X": "foo" "Y": "bar"}`, &SyntaxError{"invalid character '\"' after object key:value pair", 13}}, -} - -func TestIndentErrors(t *testing.T) { - for i, tt := range indentErrorTests { - slice := make([]uint8, 0) - buf := bytes.NewBuffer(slice) - if err := Indent(buf, []uint8(tt.in), "", ""); err != nil { - if !reflect.DeepEqual(err, tt.err) { - t.Errorf("#%d: Indent: %#v", i, err) - continue - } - } - } -} - -func TestNextValueBig(t *testing.T) { - initBig() - var scan scanner - item, rest, err := nextValue(jsonBig, &scan) - if err != nil { - t.Fatalf("nextValue: %s", err) - } - if len(item) != len(jsonBig) || &item[0] != &jsonBig[0] { - t.Errorf("invalid item: %d %d", len(item), len(jsonBig)) - } - if len(rest) != 0 { - t.Errorf("invalid rest: %d", len(rest)) - } - - item, rest, err = nextValue(append(jsonBig, "HELLO WORLD"...), &scan) - if err != nil { - t.Fatalf("nextValue extra: %s", err) - } - if len(item) != len(jsonBig) { - t.Errorf("invalid item: %d %d", len(item), len(jsonBig)) - } - if string(rest) != "HELLO WORLD" { - t.Errorf("invalid rest: %d", len(rest)) - } -} - -var benchScan scanner - -func BenchmarkSkipValue(b *testing.B) { - initBig() - for i := 0; i < b.N; i++ { - nextValue(jsonBig, &benchScan) - } - b.SetBytes(int64(len(jsonBig))) -} - -func diff(t *testing.T, a, b []byte) { - for i := 0; ; i++ { - if i >= len(a) || i >= len(b) || a[i] != b[i] { - j := i - 10 - if j < 0 { - j = 0 - } - t.Errorf("diverge at %d: «%s» vs «%s»", i, trim(a[j:]), trim(b[j:])) - return - } - } -} - -func trim(b []byte) []byte { - if len(b) > 20 { - return b[0:20] - } - return b -} - -// Generate a random JSON object. - -var jsonBig []byte - -func initBig() { - n := 10000 - if testing.Short() { - n = 100 - } - b, err := Marshal(genValue(n)) - if err != nil { - panic(err) - } - jsonBig = b -} - -func genValue(n int) interface{} { - if n > 1 { - switch rand.Intn(2) { - case 0: - return genArray(n) - case 1: - return genMap(n) - } - } - switch rand.Intn(3) { - case 0: - return rand.Intn(2) == 0 - case 1: - return rand.NormFloat64() - case 2: - return genString(30) - } - panic("unreachable") -} - -func genString(stddev float64) string { - n := int(math.Abs(rand.NormFloat64()*stddev + stddev/2)) - c := make([]rune, n) - for i := range c { - f := math.Abs(rand.NormFloat64()*64 + 32) - if f > 0x10ffff { - f = 0x10ffff - } - c[i] = rune(f) - } - return string(c) -} - -func genArray(n int) []interface{} { - f := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2))) - if f > n { - f = n - } - if f < 1 { - f = 1 - } - x := make([]interface{}, f) - for i := range x { - x[i] = genValue(((i+1)*n)/f - (i*n)/f) - } - return x -} - -func genMap(n int) map[string]interface{} { - f := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2))) - if f > n { - f = n - } - if n > 0 && f == 0 { - f = 1 - } - x := make(map[string]interface{}) - for i := 0; i < f; i++ { - x[genString(10)] = genValue(((i+1)*n)/f - (i*n)/f) - } - return x -} diff --git a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/stream.go b/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/stream.go index 890555010b..b78d5920f9 100644 --- a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/stream.go +++ b/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/stream.go @@ -12,11 +12,15 @@ import ( // A Decoder reads and decodes JSON objects from an input stream. type Decoder struct { - r io.Reader - buf []byte - d decodeState - scan scanner - err error + r io.Reader + buf []byte + d decodeState + scanp int // start of unread data in buf + scan scanner + err error + + tokenState int + tokenStack []int } // NewDecoder returns a new decoder that reads from r. @@ -41,20 +45,29 @@ func (dec *Decoder) Decode(v interface{}) error { return dec.err } + if err := dec.tokenPrepareForDecode(); err != nil { + return err + } + + if !dec.tokenValueAllowed() { + return &SyntaxError{msg: "not at beginning of value"} + } + + // Read whole value into buffer. n, err := dec.readValue() if err != nil { return err } + dec.d.init(dec.buf[dec.scanp : dec.scanp+n]) + dec.scanp += n // Don't save err from unmarshal into dec.err: // the connection is still usable since we read a complete JSON // object from it before the error happened. - dec.d.init(dec.buf[0:n]) err = dec.d.unmarshal(v) - // Slide rest of data down. - rest := copy(dec.buf, dec.buf[n:]) - dec.buf = dec.buf[0:rest] + // fixup token streaming state + dec.tokenValueEnd() return err } @@ -62,7 +75,7 @@ func (dec *Decoder) Decode(v interface{}) error { // Buffered returns a reader of the data remaining in the Decoder's // buffer. The reader is valid until the next call to Decode. func (dec *Decoder) Buffered() io.Reader { - return bytes.NewReader(dec.buf) + return bytes.NewReader(dec.buf[dec.scanp:]) } // readValue reads a JSON value into dec.buf. @@ -70,7 +83,7 @@ func (dec *Decoder) Buffered() io.Reader { func (dec *Decoder) readValue() (int, error) { dec.scan.reset() - scanp := 0 + scanp := dec.scanp var err error Input: for { @@ -111,20 +124,35 @@ Input: return 0, err } - // Make room to read more into the buffer. - const minRead = 512 - if cap(dec.buf)-len(dec.buf) < minRead { - newBuf := make([]byte, len(dec.buf), 2*cap(dec.buf)+minRead) - copy(newBuf, dec.buf) - dec.buf = newBuf - } - - // Read. Delay error for next iteration (after scan). - var n int - n, err = dec.r.Read(dec.buf[len(dec.buf):cap(dec.buf)]) - dec.buf = dec.buf[0 : len(dec.buf)+n] + n := scanp - dec.scanp + err = dec.refill() + scanp = dec.scanp + n } - return scanp, nil + return scanp - dec.scanp, nil +} + +func (dec *Decoder) refill() error { + // Make room to read more into the buffer. + // First slide down data already consumed. + if dec.scanp > 0 { + n := copy(dec.buf, dec.buf[dec.scanp:]) + dec.buf = dec.buf[:n] + dec.scanp = 0 + } + + // Grow buffer if not large enough. + const minRead = 512 + if cap(dec.buf)-len(dec.buf) < minRead { + newBuf := make([]byte, len(dec.buf), 2*cap(dec.buf)+minRead) + copy(newBuf, dec.buf) + dec.buf = newBuf + } + + // Read. Delay error for next iteration (after scan). + n, err := dec.r.Read(dec.buf[len(dec.buf):cap(dec.buf)]) + dec.buf = dec.buf[0 : len(dec.buf)+n] + + return err } func nonSpace(b []byte) bool { @@ -205,3 +233,255 @@ func (m *RawMessage) UnmarshalJSON(data []byte) error { var _ Marshaler = (*RawMessage)(nil) var _ Unmarshaler = (*RawMessage)(nil) + +// A Token holds a value of one of these types: +// +// Delim, for the four JSON delimiters [ ] { } +// bool, for JSON booleans +// float64, for JSON numbers +// Number, for JSON numbers +// string, for JSON string literals +// nil, for JSON null +// +type Token interface{} + +const ( + tokenTopValue = iota + tokenArrayStart + tokenArrayValue + tokenArrayComma + tokenObjectStart + tokenObjectKey + tokenObjectColon + tokenObjectValue + tokenObjectComma +) + +// advance tokenstate from a separator state to a value state +func (dec *Decoder) tokenPrepareForDecode() error { + // Note: Not calling peek before switch, to avoid + // putting peek into the standard Decode path. + // peek is only called when using the Token API. + switch dec.tokenState { + case tokenArrayComma: + c, err := dec.peek() + if err != nil { + return err + } + if c != ',' { + return &SyntaxError{"expected comma after array element", 0} + } + dec.scanp++ + dec.tokenState = tokenArrayValue + case tokenObjectColon: + c, err := dec.peek() + if err != nil { + return err + } + if c != ':' { + return &SyntaxError{"expected colon after object key", 0} + } + dec.scanp++ + dec.tokenState = tokenObjectValue + } + return nil +} + +func (dec *Decoder) tokenValueAllowed() bool { + switch dec.tokenState { + case tokenTopValue, tokenArrayStart, tokenArrayValue, tokenObjectValue: + return true + } + return false +} + +func (dec *Decoder) tokenValueEnd() { + switch dec.tokenState { + case tokenArrayStart, tokenArrayValue: + dec.tokenState = tokenArrayComma + case tokenObjectValue: + dec.tokenState = tokenObjectComma + } +} + +// A Delim is a JSON array or object delimiter, one of [ ] { or }. +type Delim rune + +func (d Delim) String() string { + return string(d) +} + +// Token returns the next JSON token in the input stream. +// At the end of the input stream, Token returns nil, io.EOF. +// +// Token guarantees that the delimiters [ ] { } it returns are +// properly nested and matched: if Token encounters an unexpected +// delimiter in the input, it will return an error. +// +// The input stream consists of basic JSON values—bool, string, +// number, and null—along with delimiters [ ] { } of type Delim +// to mark the start and end of arrays and objects. +// Commas and colons are elided. +func (dec *Decoder) Token() (Token, error) { + for { + c, err := dec.peek() + if err != nil { + return nil, err + } + switch c { + case '[': + if !dec.tokenValueAllowed() { + return dec.tokenError(c) + } + dec.scanp++ + dec.tokenStack = append(dec.tokenStack, dec.tokenState) + dec.tokenState = tokenArrayStart + return Delim('['), nil + + case ']': + if dec.tokenState != tokenArrayStart && dec.tokenState != tokenArrayComma { + return dec.tokenError(c) + } + dec.scanp++ + dec.tokenState = dec.tokenStack[len(dec.tokenStack)-1] + dec.tokenStack = dec.tokenStack[:len(dec.tokenStack)-1] + dec.tokenValueEnd() + return Delim(']'), nil + + case '{': + if !dec.tokenValueAllowed() { + return dec.tokenError(c) + } + dec.scanp++ + dec.tokenStack = append(dec.tokenStack, dec.tokenState) + dec.tokenState = tokenObjectStart + return Delim('{'), nil + + case '}': + if dec.tokenState != tokenObjectStart && dec.tokenState != tokenObjectComma { + return dec.tokenError(c) + } + dec.scanp++ + dec.tokenState = dec.tokenStack[len(dec.tokenStack)-1] + dec.tokenStack = dec.tokenStack[:len(dec.tokenStack)-1] + dec.tokenValueEnd() + return Delim('}'), nil + + case ':': + if dec.tokenState != tokenObjectColon { + return dec.tokenError(c) + } + dec.scanp++ + dec.tokenState = tokenObjectValue + continue + + case ',': + if dec.tokenState == tokenArrayComma { + dec.scanp++ + dec.tokenState = tokenArrayValue + continue + } + if dec.tokenState == tokenObjectComma { + dec.scanp++ + dec.tokenState = tokenObjectKey + continue + } + return dec.tokenError(c) + + case '"': + if dec.tokenState == tokenObjectStart || dec.tokenState == tokenObjectKey { + var x string + old := dec.tokenState + dec.tokenState = tokenTopValue + err := dec.Decode(&x) + dec.tokenState = old + if err != nil { + clearOffset(err) + return nil, err + } + dec.tokenState = tokenObjectColon + return x, nil + } + fallthrough + + default: + if !dec.tokenValueAllowed() { + return dec.tokenError(c) + } + var x interface{} + if err := dec.Decode(&x); err != nil { + clearOffset(err) + return nil, err + } + return x, nil + } + } +} + +func clearOffset(err error) { + if s, ok := err.(*SyntaxError); ok { + s.Offset = 0 + } +} + +func (dec *Decoder) tokenError(c byte) (Token, error) { + var context string + switch dec.tokenState { + case tokenTopValue: + context = " looking for beginning of value" + case tokenArrayStart, tokenArrayValue, tokenObjectValue: + context = " looking for beginning of value" + case tokenArrayComma: + context = " after array element" + case tokenObjectKey: + context = " looking for beginning of object key string" + case tokenObjectColon: + context = " after object key" + case tokenObjectComma: + context = " after object key:value pair" + } + return nil, &SyntaxError{"invalid character " + quoteChar(int(c)) + " " + context, 0} +} + +// More reports whether there is another element in the +// current array or object being parsed. +func (dec *Decoder) More() bool { + c, err := dec.peek() + return err == nil && c != ']' && c != '}' +} + +func (dec *Decoder) peek() (byte, error) { + var err error + for { + for i := dec.scanp; i < len(dec.buf); i++ { + c := dec.buf[i] + if isSpace(rune(c)) { + continue + } + dec.scanp = i + return c, nil + } + // buffer has been scanned, now report any error + if err != nil { + return 0, err + } + err = dec.refill() + } +} + +/* +TODO + +// EncodeToken writes the given JSON token to the stream. +// It returns an error if the delimiters [ ] { } are not properly used. +// +// EncodeToken does not call Flush, because usually it is part of +// a larger operation such as Encode, and those will call Flush when finished. +// Callers that create an Encoder and then invoke EncodeToken directly, +// without using Encode, need to call Flush when finished to ensure that +// the JSON is written to the underlying writer. +func (e *Encoder) EncodeToken(t Token) error { + ... +} + +*/ diff --git a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/stream_test.go b/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/stream_test.go deleted file mode 100644 index b562e87690..0000000000 --- a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/stream_test.go +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package json - -import ( - "bytes" - "io/ioutil" - "net" - "reflect" - "strings" - "testing" -) - -// Test values for the stream test. -// One of each JSON kind. -var streamTest = []interface{}{ - 0.1, - "hello", - nil, - true, - false, - []interface{}{"a", "b", "c"}, - map[string]interface{}{"K": "Kelvin", "ß": "long s"}, - 3.14, // another value to make sure something can follow map -} - -var streamEncoded = `0.1 -"hello" -null -true -false -["a","b","c"] -{"ß":"long s","K":"Kelvin"} -3.14 -` - -func TestEncoder(t *testing.T) { - for i := 0; i <= len(streamTest); i++ { - var buf bytes.Buffer - enc := NewEncoder(&buf) - for j, v := range streamTest[0:i] { - if err := enc.Encode(v); err != nil { - t.Fatalf("encode #%d: %v", j, err) - } - } - if have, want := buf.String(), nlines(streamEncoded, i); have != want { - t.Errorf("encoding %d items: mismatch", i) - diff(t, []byte(have), []byte(want)) - break - } - } -} - -func TestDecoder(t *testing.T) { - for i := 0; i <= len(streamTest); i++ { - // Use stream without newlines as input, - // just to stress the decoder even more. - // Our test input does not include back-to-back numbers. - // Otherwise stripping the newlines would - // merge two adjacent JSON values. - var buf bytes.Buffer - for _, c := range nlines(streamEncoded, i) { - if c != '\n' { - buf.WriteRune(c) - } - } - out := make([]interface{}, i) - dec := NewDecoder(&buf) - for j := range out { - if err := dec.Decode(&out[j]); err != nil { - t.Fatalf("decode #%d/%d: %v", j, i, err) - } - } - if !reflect.DeepEqual(out, streamTest[0:i]) { - t.Errorf("decoding %d items: mismatch", i) - for j := range out { - if !reflect.DeepEqual(out[j], streamTest[j]) { - t.Errorf("#%d: have %v want %v", j, out[j], streamTest[j]) - } - } - break - } - } -} - -func TestDecoderBuffered(t *testing.T) { - r := strings.NewReader(`{"Name": "Gopher"} extra `) - var m struct { - Name string - } - d := NewDecoder(r) - err := d.Decode(&m) - if err != nil { - t.Fatal(err) - } - if m.Name != "Gopher" { - t.Errorf("Name = %q; want Gopher", m.Name) - } - rest, err := ioutil.ReadAll(d.Buffered()) - if err != nil { - t.Fatal(err) - } - if g, w := string(rest), " extra "; g != w { - t.Errorf("Remaining = %q; want %q", g, w) - } -} - -func nlines(s string, n int) string { - if n <= 0 { - return "" - } - for i, c := range s { - if c == '\n' { - if n--; n == 0 { - return s[0 : i+1] - } - } - } - return s -} - -func TestRawMessage(t *testing.T) { - // TODO(rsc): Should not need the * in *RawMessage - var data struct { - X float64 - Id *RawMessage - Y float32 - } - const raw = `["\u0056",null]` - const msg = `{"X":0.1,"Id":["\u0056",null],"Y":0.2}` - err := Unmarshal([]byte(msg), &data) - if err != nil { - t.Fatalf("Unmarshal: %v", err) - } - if string([]byte(*data.Id)) != raw { - t.Fatalf("Raw mismatch: have %#q want %#q", []byte(*data.Id), raw) - } - b, err := Marshal(&data) - if err != nil { - t.Fatalf("Marshal: %v", err) - } - if string(b) != msg { - t.Fatalf("Marshal: have %#q want %#q", b, msg) - } -} - -func TestNullRawMessage(t *testing.T) { - // TODO(rsc): Should not need the * in *RawMessage - var data struct { - X float64 - Id *RawMessage - Y float32 - } - data.Id = new(RawMessage) - const msg = `{"X":0.1,"Id":null,"Y":0.2}` - err := Unmarshal([]byte(msg), &data) - if err != nil { - t.Fatalf("Unmarshal: %v", err) - } - if data.Id != nil { - t.Fatalf("Raw mismatch: have non-nil, want nil") - } - b, err := Marshal(&data) - if err != nil { - t.Fatalf("Marshal: %v", err) - } - if string(b) != msg { - t.Fatalf("Marshal: have %#q want %#q", b, msg) - } -} - -var blockingTests = []string{ - `{"x": 1}`, - `[1, 2, 3]`, -} - -func TestBlocking(t *testing.T) { - for _, enc := range blockingTests { - r, w := net.Pipe() - go w.Write([]byte(enc)) - var val interface{} - - // If Decode reads beyond what w.Write writes above, - // it will block, and the test will deadlock. - if err := NewDecoder(r).Decode(&val); err != nil { - t.Errorf("decoding %s: %v", enc, err) - } - r.Close() - w.Close() - } -} - -func BenchmarkEncoderEncode(b *testing.B) { - b.ReportAllocs() - type T struct { - X, Y string - } - v := &T{"foo", "bar"} - for i := 0; i < b.N; i++ { - if err := NewEncoder(ioutil.Discard).Encode(v); err != nil { - b.Fatal(err) - } - } -} diff --git a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/tagkey_test.go b/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/tagkey_test.go deleted file mode 100644 index 23e71c7525..0000000000 --- a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/tagkey_test.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package json - -import ( - "testing" -) - -type basicLatin2xTag struct { - V string `json:"$%-/"` -} - -type basicLatin3xTag struct { - V string `json:"0123456789"` -} - -type basicLatin4xTag struct { - V string `json:"ABCDEFGHIJKLMO"` -} - -type basicLatin5xTag struct { - V string `json:"PQRSTUVWXYZ_"` -} - -type basicLatin6xTag struct { - V string `json:"abcdefghijklmno"` -} - -type basicLatin7xTag struct { - V string `json:"pqrstuvwxyz"` -} - -type miscPlaneTag struct { - V string `json:"色は匂へど"` -} - -type percentSlashTag struct { - V string `json:"text/html%"` // http://golang.org/issue/2718 -} - -type punctuationTag struct { - V string `json:"!#$%&()*+-./:<=>?@[]^_{|}~"` // http://golang.org/issue/3546 -} - -type emptyTag struct { - W string -} - -type misnamedTag struct { - X string `jsom:"Misnamed"` -} - -type badFormatTag struct { - Y string `:"BadFormat"` -} - -type badCodeTag struct { - Z string `json:" !\"#&'()*+,."` -} - -type spaceTag struct { - Q string `json:"With space"` -} - -type unicodeTag struct { - W string `json:"Ελλάδα"` -} - -var structTagObjectKeyTests = []struct { - raw interface{} - value string - key string -}{ - {basicLatin2xTag{"2x"}, "2x", "$%-/"}, - {basicLatin3xTag{"3x"}, "3x", "0123456789"}, - {basicLatin4xTag{"4x"}, "4x", "ABCDEFGHIJKLMO"}, - {basicLatin5xTag{"5x"}, "5x", "PQRSTUVWXYZ_"}, - {basicLatin6xTag{"6x"}, "6x", "abcdefghijklmno"}, - {basicLatin7xTag{"7x"}, "7x", "pqrstuvwxyz"}, - {miscPlaneTag{"いろはにほへと"}, "いろはにほへと", "色は匂へど"}, - {emptyTag{"Pour Moi"}, "Pour Moi", "W"}, - {misnamedTag{"Animal Kingdom"}, "Animal Kingdom", "X"}, - {badFormatTag{"Orfevre"}, "Orfevre", "Y"}, - {badCodeTag{"Reliable Man"}, "Reliable Man", "Z"}, - {percentSlashTag{"brut"}, "brut", "text/html%"}, - {punctuationTag{"Union Rags"}, "Union Rags", "!#$%&()*+-./:<=>?@[]^_{|}~"}, - {spaceTag{"Perreddu"}, "Perreddu", "With space"}, - {unicodeTag{"Loukanikos"}, "Loukanikos", "Ελλάδα"}, -} - -func TestStructTagObjectKey(t *testing.T) { - for _, tt := range structTagObjectKeyTests { - b, err := Marshal(tt.raw) - if err != nil { - t.Fatalf("Marshal(%#q) failed: %v", tt.raw, err) - } - var f interface{} - err = Unmarshal(b, &f) - if err != nil { - t.Fatalf("Unmarshal(%#q) failed: %v", b, err) - } - for i, v := range f.(map[string]interface{}) { - switch i { - case tt.key: - if s, ok := v.(string); !ok || s != tt.value { - t.Fatalf("Unexpected value: %#q, want %v", s, tt.value) - } - default: - t.Fatalf("Unexpected key: %#q, from %#q", i, b) - } - } - } -} diff --git a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/tags_test.go b/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/tags_test.go deleted file mode 100644 index 91fb18831e..0000000000 --- a/Godeps/_workspace/src/github.com/jfrazelle/go/canonical/json/tags_test.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package json - -import ( - "testing" -) - -func TestTagParsing(t *testing.T) { - name, opts := parseTag("field,foobar,foo") - if name != "field" { - t.Fatalf("name = %q, want field", name) - } - for _, tt := range []struct { - opt string - want bool - }{ - {"foobar", true}, - {"foo", true}, - {"bar", false}, - } { - if opts.Contains(tt.opt) != tt.want { - t.Errorf("Contains(%q) = %v", tt.opt, !tt.want) - } - } -}