Skip to content

Commit 837dfaa

Browse files
florenfloren
committed
Implement fixes for EachKey when you have lots of keys, or big arrays.
1 parent cb835d4 commit 837dfaa

File tree

3 files changed

+156
-13
lines changed

3 files changed

+156
-13
lines changed

‎go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
module github.com/buger/jsonparser
22

33
go 1.13
4+

‎parser.go

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -380,7 +380,8 @@ func sameTree(p1, p2 []string) bool {
380380
}
381381

382382
func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]string) int {
383-
var pathFlags int64
383+
var x struct{}
384+
pathFlags := make([]bool, len(paths))
384385
var level, pathsMatched, i int
385386
ln := len(data)
386387

@@ -439,15 +440,15 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str
439440

440441
pathsBuf[level-1] = bytesToString(&keyUnesc)
441442
for pi, p := range paths {
442-
if len(p) != level || pathFlags&bitwiseFlags[pi+1] != 0 || !equalStr(&keyUnesc, p[level-1]) || !sameTree(p, pathsBuf[:level]) {
443+
if len(p) != level || pathFlags[pi] || !equalStr(&keyUnesc, p[level-1]) || !sameTree(p, pathsBuf[:level]) {
443444
continue
444445
}
445446

446447
match = pi
447448

448449
i++
449450
pathsMatched++
450-
pathFlags |= bitwiseFlags[pi+1]
451+
pathFlags[pi] = true
451452

452453
v, dt, _, e := Get(data[i:])
453454
cb(pi, v, dt, e)
@@ -485,40 +486,41 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str
485486
case '}':
486487
level--
487488
case '[':
488-
var arrIdxFlags int64
489-
var pIdxFlags int64
489+
var ok bool
490+
arrIdxFlags := make(map[int]struct{})
491+
pIdxFlags := make([]bool, len(paths))
490492

491493
if level < 0 {
492494
cb(-1, nil, Unknown, MalformedJsonError)
493495
return -1
494496
}
495497

496498
for pi, p := range paths {
497-
if len(p) < level+1 || pathFlags&bitwiseFlags[pi+1] != 0 || p[level][0] != '[' || !sameTree(p, pathsBuf[:level]) {
499+
if len(p) < level+1 || pathFlags[pi] || p[level][0] != '[' || !sameTree(p, pathsBuf[:level]) {
498500
continue
499501
}
500502
if len(p[level]) >= 2 {
501503
aIdx, _ := strconv.Atoi(p[level][1 : len(p[level])-1])
502-
arrIdxFlags |= bitwiseFlags[aIdx+1]
503-
pIdxFlags |= bitwiseFlags[pi+1]
504+
arrIdxFlags[aIdx] = x
505+
pIdxFlags[pi] = true
504506
}
505507
}
506508

507-
if arrIdxFlags > 0 {
509+
if len(arrIdxFlags) > 0 {
508510
level++
509511

510512
var curIdx int
511513
arrOff, _ := ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) {
512-
if arrIdxFlags&bitwiseFlags[curIdx+1] != 0 {
514+
if _, ok = arrIdxFlags[curIdx]; ok {
513515
for pi, p := range paths {
514-
if pIdxFlags&bitwiseFlags[pi+1] != 0 {
516+
if pIdxFlags[pi] {
515517
aIdx, _ := strconv.Atoi(p[level-1][1 : len(p[level-1])-1])
516518

517519
if curIdx == aIdx {
518520
of := searchKeys(value, p[level:]...)
519521

520522
pathsMatched++
521-
pathFlags |= bitwiseFlags[pi+1]
523+
pathFlags[pi] = true
522524

523525
if of != -1 {
524526
v, dt, _, e := Get(value[of:])
@@ -930,7 +932,7 @@ func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int
930932
return -1, MalformedJsonError
931933
}
932934

933-
offset = nT+1
935+
offset = nT + 1
934936

935937
if len(keys) > 0 {
936938
if offset = searchKeys(data, keys...); offset == -1 {

‎parser_error_test.go

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package jsonparser
22

33
import (
44
"fmt"
5+
"strings"
56
"testing"
67
)
78

@@ -38,3 +39,142 @@ func TestPanickingErrors(t *testing.T) {
3839
t.Error("Expected error...")
3940
}
4041
}
42+
43+
// check having a very deep key depth
44+
func TestKeyDepth(t *testing.T) {
45+
var sb strings.Builder
46+
var keys []string
47+
//build data
48+
sb.WriteString("{")
49+
for i := 0; i < 128; i++ {
50+
fmt.Fprintf(&sb, `"key%d": %dx,`, i, i)
51+
keys = append(keys, fmt.Sprintf("key%d", i))
52+
}
53+
sb.WriteString("}")
54+
55+
data := []byte(sb.String())
56+
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
57+
return
58+
}, keys)
59+
}
60+
61+
// check having a bunch of keys in a call to EachKey
62+
func TestKeyCount(t *testing.T) {
63+
var sb strings.Builder
64+
var keys [][]string
65+
//build data
66+
sb.WriteString("{")
67+
for i := 0; i < 128; i++ {
68+
fmt.Fprintf(&sb, `"key%d":"%d"`, i, i)
69+
if i < 127 {
70+
sb.WriteString(",")
71+
}
72+
keys = append(keys, []string{fmt.Sprintf("key%d", i)})
73+
}
74+
sb.WriteString("}")
75+
76+
data := []byte(sb.String())
77+
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
78+
return
79+
}, keys...)
80+
}
81+
82+
// try pulling lots of keys out of a big array
83+
func TestKeyDepthArray(t *testing.T) {
84+
var sb strings.Builder
85+
var keys []string
86+
//build data
87+
sb.WriteString("[")
88+
for i := 0; i < 128; i++ {
89+
fmt.Fprintf(&sb, `{"key": %d},`, i)
90+
keys = append(keys, fmt.Sprintf("[%d].key", i))
91+
}
92+
sb.WriteString("]")
93+
94+
data := []byte(sb.String())
95+
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
96+
return
97+
}, keys)
98+
}
99+
100+
// check having a bunch of keys
101+
func TestKeyCountArray(t *testing.T) {
102+
var sb strings.Builder
103+
var keys [][]string
104+
//build data
105+
sb.WriteString("[")
106+
for i := 0; i < 128; i++ {
107+
fmt.Fprintf(&sb, `{"key":"%d"}`, i)
108+
if i < 127 {
109+
sb.WriteString(",")
110+
}
111+
keys = append(keys, []string{fmt.Sprintf("[%d].key", i)})
112+
}
113+
sb.WriteString("]")
114+
115+
data := []byte(sb.String())
116+
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
117+
return
118+
}, keys...)
119+
}
120+
121+
// check having a bunch of keys in a super deep array
122+
func TestEachKeyArray(t *testing.T) {
123+
var sb strings.Builder
124+
var keys [][]string
125+
//build data
126+
sb.WriteString(`[`)
127+
for i := 0; i < 127; i++ {
128+
fmt.Fprintf(&sb, `%d`, i)
129+
if i < 127 {
130+
sb.WriteString(",")
131+
}
132+
if i < 32 {
133+
keys = append(keys, []string{fmt.Sprintf("[%d]", 128+i)})
134+
}
135+
}
136+
sb.WriteString(`]`)
137+
138+
data := []byte(sb.String())
139+
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
140+
return
141+
}, keys...)
142+
}
143+
144+
func TestLargeArray(t *testing.T) {
145+
var sb strings.Builder
146+
//build data
147+
sb.WriteString(`[`)
148+
for i := 0; i < 127; i++ {
149+
fmt.Fprintf(&sb, `%d`, i)
150+
if i < 127 {
151+
sb.WriteString(",")
152+
}
153+
}
154+
sb.WriteString(`]`)
155+
keys := [][]string{[]string{`[1]`}}
156+
157+
data := []byte(sb.String())
158+
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
159+
return
160+
}, keys...)
161+
}
162+
163+
func TestArrayOutOfBounds(t *testing.T) {
164+
var sb strings.Builder
165+
//build data
166+
sb.WriteString(`[`)
167+
for i := 0; i < 61; i++ {
168+
fmt.Fprintf(&sb, `%d`, i)
169+
if i < 61 {
170+
sb.WriteString(",")
171+
}
172+
}
173+
sb.WriteString(`]`)
174+
keys := [][]string{[]string{`[128]`}}
175+
176+
data := []byte(sb.String())
177+
EachKey(data, func(offset int, value []byte, dt ValueType, err error) {
178+
return
179+
}, keys...)
180+
}

0 commit comments

Comments
 (0)