json: fix find_object_tokens
James Prestwood
PATCH v2 1/9 json: fix find_object_tokens
First, this was renamed to 'count_tokens_in_container' to be more general purpose (i.e. include future array counting).
The way the tokens are counted also changed to be more intuitive. While the previous way was correct, it was somewhat convoluted in how it worked (finding the next parent of the objects parent).
Instead we can use the container token itself as the parent and begin counting tokens. When we find a token with a parent index less than the target we have reached the end of this container. This also works for nested containers, including arrays since we no longer rely on a key (which an array element would not have).
For example::
{
"first":{"foo":"bar"},
"second":{"foo2":"bar2"}
}index 0 <overall object>
index 1 "first" with parent 0
index 2 {"foo":"bar"} with parent 1
Counting tokens inside "first"'s object we have:
index 3 "foo" with parent 2
index 4 "bar" with parent 3
If we continue counting we reach:
index 5 "second" with parent 0
This terminates the counting loop since the parent index is less than '2' (the index of {"foo":"bar"} object).
--- src/json.c | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-)
src/json.c +8 -12
@@ -69,20 +69,16 @@ static jsmntok_t *next_key_in_parent(struct json_iter *iter, jsmntok_t *current)return NULL ; } -/* -* 'object' is expected to be a value, so object - 1 is its key. Find -* the next key who's parent matches the parent of object - 1. The -* token preceeding this next key will mark the end of 'object'. -*/ -static int find_object_tokens (struct json_iter *iter,jsmntok_t *object) +static int count_tokens_in_container (struct json_iter *iter,jsmntok_t *container) { -jsmntok_t *next =next_key_in_parent (iter, object -1 ); +int idx = container - iter->contents->tokens; +jsmntok_t *contents; -/* End of token list */ -if (!next) - next =ITER_END (iter); +for (contents = ++container; contents <ITER_END (iter); contents++) +if (contents->parent < idx) +break ; -return next - object -1 ; +return contents - container; } static void iter_recurse(struct json_iter *iter, jsmntok_t *object,
@@ -92,7 +88,7 @@ static void iter_recurse(struct json_iter *iter, jsmntok_t *object, child->contents = c; child->start = object - c->tokens; - child->count =find_object_tokens (iter, object); + child->count =count_tokens_in_container (iter, object); }struct json_contents *json_contents_new (const char *json,size_t json_len)
Signature · 1 line
2.31.1
James Prestwood
PATCH v2 2/9 json: add support for primitive types
This adds support for boolean, (unsigned) integers, and null types. JSON_PRIMITIVE should be used as the type when parsing and the value should be struct json_iter.
Once parsed the actual value can be obtained using one of the primitive getters. If the type does not match they will return false.
If using JSON_OPTIONAL with JSON_PRIMITIVE the resulting iterator can be checked with json_iter_is_valid. If false the key/value was not found or the type was not matching.
--- src/json.c | 120 +++++++++++++++++++++++++++++++++++++++++++++++++---- src/json.h | 13 +++++- 2 files changed, 125 insertions(+), 8 deletions(-)
src/json.c +113 -7
@@ -81,14 +81,14 @@ static int count_tokens_in_container(struct json_iter *iter, jsmntok_t *containereturn contents - container; } -static void iter_recurse (struct json_iter *iter,jsmntok_t *object, +static void iter_recurse (struct json_iter *iter,jsmntok_t *token,struct json_iter *child) {struct json_contents *c = iter->contents; child->contents = c; - child->start = object - c->tokens; - child->count =count_tokens_in_container (iter, object); + child->start = token - c->tokens; + child->count =count_tokens_in_container (iter, token); }struct json_contents *json_contents_new (const char *json,size_t json_len)
@@ -149,7 +149,7 @@ static void assign_arg(void *data, void *user_data)struct json_arg *arg = data;struct json_contents *c = iter->contents;char **sval; -struct json_iter *oval; +struct json_iter *iter_val;switch (arg->type) {case JSON_STRING :
@@ -159,12 +159,13 @@ static void assign_arg(void *data, void *user_data)break ;case JSON_OBJECT : - oval = arg->value; +case JSON_PRIMITIVE : + iter_val = arg->value;if (!arg->v) - oval->start =-1 ; + iter_val->start =-1 ;else -iter_recurse (iter, arg->v, oval); +iter_recurse (iter, arg->v, iter_val);break ;default :
@@ -205,6 +206,7 @@ bool json_iter_parse(struct json_iter *iter, enum json_type type, ...) goto done;case JSON_STRING :case JSON_OBJECT : +case JSON_PRIMITIVE :break ;default : goto error;
@@ -264,3 +266,107 @@ error:l_queue_destroy (args, l_free);return false; } + +static bool iter_get_primitive_data (struct json_iter *iter,void **ptr, +size_t *len) +{ +struct json_contents *c = iter->contents; +jsmntok_t *t = c->tokens + iter->start; + +if (t->type !=JSMN_PRIMITIVE ) +return false; + + *ptr =TOK_PTR (c->json, t); + *len =TOK_LEN (t); + +return true; +} + +bool json_iter_get_int (struct json_iter *iter,int *i) +{ +void *ptr; +size_t len; +long int r; +int t; +char *endp; + +if (!iter_get_primitive_data (iter, &ptr, &len)) +return false; + + errno =0 ; + + t = r =strtol (ptr, &endp,10 ); +if (endp != ptr + len) +return false; + +if (errno ==ERANGE || r != t) +return false; + +if (i) + *i = r; + +return true; +} + +bool json_iter_get_uint (struct json_iter *iter,unsigned int *i) +{ +void *ptr; +size_t len; +unsigned long int r; +unsigned int t; +char *endp; + +if (!iter_get_primitive_data (iter, &ptr, &len)) +return false; + + errno =0 ; + + t = r =strtoul (ptr, &endp,10 ); +if (endp != ptr + len) +return false; + +if (errno ==ERANGE || r != t) +return false; + +if (i) + *i = r; + +return true; +} + +bool json_iter_get_boolean (struct json_iter *iter,bool *b) +{ +void *ptr; +size_t len; + +if (!iter_get_primitive_data (iter, &ptr, &len)) +return false; + +if (len ==4 && !memcmp (ptr,"true" ,4 )) { +if (b) + *b = true; + +return true; + }else if (len ==5 && !memcmp (ptr,"false" ,5 )) { +if (b) + *b = false; + +return true; + } + +return false; +} + +bool json_iter_get_null (struct json_iter *iter) +{ +void *ptr; +size_t len; + +if (!iter_get_primitive_data (iter, &ptr, &len)) +return false; + +if (len ==4 && !memcmp (ptr,"null" ,4 )) +return true; + +return false; +}
src/json.h +12 -1
@@ -73,11 +73,22 @@ void json_iter_init(struct json_iter *iter, struct json_contents *c);* * String values should be of type char ** and must be freed * Object values should be of type struct json_iter * +* Primitive types (numbers, booleans, null) should be of type +* struct json_iter *. This is to allow the caller to distinguish +* between the actual value type after parsing using a getter for +* the expected type (get_uint/get_int/get_boolean etc.). In +* addition this lets the caller use JSON_OPTIONAL and check post +* json_iter_parse if the iterator is valid (json_iter_is_valid). * * No other types are supported at this time, and json_iter_parse will fail if * other types are encountered. * * JSON_OPTIONAL string values will point to NULL if not found -* JSON_OPTIONAL objects can be checked with json_object_not_found. +* JSON_OPTIONAL objects/primitives can be checked with json_object_is_valid. */ bool json_iter_parse (struct json_iter *iter,enum json_type type, ...); + +bool json_iter_get_int (struct json_iter *iter,int *i); +bool json_iter_get_uint (struct json_iter *iter,unsigned int *i); +bool json_iter_get_boolean (struct json_iter *iter,bool *b); +bool json_iter_get_null (struct json_iter *iter);
Signature · 1 line
2.31.1
James Prestwood
PATCH v2 3/9 unit: add primitive json test
In reply to James Prestwood--- unit/test-json.c | 111 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 95 insertions(+), 16 deletions(-)
unit/test-json.c +95 -16
@@ -84,8 +84,6 @@ static void test_json_unsupported_types(const void *data)* Valid JSON objects but currently unsupported types */ char arrays[] ="{\"test\":[1, 2, 3, 4]}" ; -char integers[] ="{\"test\": 10 }" ; -char bools[] ="{\"test\": true}" ;struct json_iter iter;struct json_contents *c =json_contents_new (arrays,strlen (arrays));
@@ -95,20 +93,6 @@ static void test_json_unsupported_types(const void *data)JSON_MANDATORY ("test" ,JSON_ARRAY ,NULL ), JSON_UNDEFINED));json_contents_free (c); - - c =json_contents_new (integers,strlen (integers)); -json_iter_init (&iter, c); -assert (!json_iter_parse (&iter, -JSON_MANDATORY ("test" ,JSON_PRIMITIVE ,NULL ), -JSON_UNDEFINED )); -json_contents_free (c); - - c =json_contents_new (bools,strlen (bools)); -json_iter_init (&iter, c); -assert (!json_iter_parse (&iter, -JSON_MANDATORY ("test" ,JSON_PRIMITIVE ,NULL ), -JSON_UNDEFINED )); -json_contents_free (c); }/*
@@ -248,6 +232,100 @@ static void test_json_larger_object(const void *data)json_contents_free (c); } +static void check_primitives (struct json_iter *i,struct json_iter *ui, +struct json_iter *t,struct json_iter *f, +struct json_iter *null,struct json_iter *obj) +{ + +int i_val; +unsigned int ui_val; +bool b_val; + +assert (json_iter_is_valid (i)); +assert (!json_iter_get_uint (i,NULL )); +assert (!json_iter_get_boolean (i,NULL )); +assert (!json_iter_get_null (i)); +assert (json_iter_get_int (i, &i_val)); +assert (i_val ==-10 ); + +assert (json_iter_is_valid (ui)); +assert (!json_iter_get_boolean (ui,NULL )); +assert (!json_iter_get_null (ui)); +assert (json_iter_get_int (ui, &i_val)); +assert (json_iter_get_uint (ui, &ui_val)); +assert (i_val ==10 && ui_val ==10 ); + + +assert (json_iter_is_valid (t)); +assert (!json_iter_get_null (t)); +assert (!json_iter_get_int (t,NULL )); +assert (!json_iter_get_uint (t,NULL )); +assert (json_iter_get_boolean (t, &b_val)); +assert (b_val == true); + +assert (json_iter_is_valid (f)); +assert (!json_iter_get_null (f)); +assert (!json_iter_get_int (f,NULL )); +assert (!json_iter_get_uint (f,NULL )); +assert (json_iter_get_boolean (f, &b_val)); +assert (b_val == false); + +assert (json_iter_is_valid (null)); +assert (!json_iter_get_int (null,NULL )); +assert (!json_iter_get_uint (null,NULL )); +assert (!json_iter_get_boolean (null,NULL )); +assert (json_iter_get_null (null)); + +if (obj) { +assert (json_iter_is_valid (obj)); +assert (json_iter_parse (obj, +JSON_MANDATORY ("null_val" ,JSON_PRIMITIVE , null), +JSON_MANDATORY ("false_val" ,JSON_PRIMITIVE , f), +JSON_MANDATORY ("true_val" ,JSON_PRIMITIVE , t), +JSON_MANDATORY ("int_val" ,JSON_PRIMITIVE , i), +JSON_MANDATORY ("uint_val" ,JSON_PRIMITIVE , ui), +JSON_UNDEFINED )); + +check_primitives (i, ui, t, f, null,NULL ); + } +} + +static void test_json_primitives (const void *data) +{ +char json[] ="{\"int_val\": -10," +"\"uint_val\": 10," +"\"true_val\": true," +"\"false_val\": false," +"\"null_val\": null," +"\"obj_val\":{" +"\"int_val\": -10," +"\"uint_val\": 10," +"\"true_val\": true," +"\"false_val\": false," +"\"null_val\": null}}" ; +struct json_contents *c =json_contents_new (json,strlen (json)); +struct json_iter outer, inner, null, f, t, i, ui; +struct json_iter not_found; + +json_iter_init (&outer, c); +assert (json_iter_parse (&outer, +JSON_MANDATORY ("obj_val" ,JSON_OBJECT , &inner), +JSON_MANDATORY ("null_val" ,JSON_PRIMITIVE , &null), +JSON_MANDATORY ("false_val" ,JSON_PRIMITIVE , &f), +JSON_MANDATORY ("true_val" ,JSON_PRIMITIVE , &t), +JSON_MANDATORY ("int_val" ,JSON_PRIMITIVE , &i), +JSON_MANDATORY ("uint_val" ,JSON_PRIMITIVE , &ui), +JSON_OPTIONAL ("not_found" ,JSON_PRIMITIVE , ¬_found), +JSON_UNDEFINED )); + +assert (!json_iter_is_valid (¬_found)); + +check_primitives (&i, &ui, &t, &f, &null, &inner); + +json_contents_free (c); +} + +int main (int argc,char *argv[]) {l_test_init (&argc, &argv);
@@ -259,6 +337,7 @@ int main(int argc, char *argv[])l_test_add ("json empty objects" , test_json_empty_objects,NULL );l_test_add ("json parse out of order" , test_json_out_of_order,NULL );l_test_add ("json larger object" , test_json_larger_object,NULL ); +l_test_add ("json test primitives" , test_json_primitives,NULL );return l_test_run (); }
Signature · 1 line
2.31.1
Denis Kenzior
Re: [PATCH v2 3/9] unit: add primitive json test
Hi James,
Quoted 4 lines · James Prestwood
--- unit/test-json.c | 111 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 95 insertions(+), 16 deletions(-)
I removed some double empty lines in this commit. Patches 1-3 applied, thanks.
Regards,
-Denis
James Prestwood
PATCH v2 4/9 json: add support for array iteration
In reply to James PrestwoodArrays can now be parsed using the JSON_ARRAY type (stored in a struct json_iter) then iterated using json_iter_next. When iterating the type can be checked with json_iter_get_type. For each iteration the value can be obtained using any of the type getters (int/uint/boolean/null).
--- src/json.c | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++- src/json.h | 4 ++++ 2 files changed, 59 insertions(+), 1 deletion(-)
src/json.c +55 -1
@@ -88,7 +88,17 @@ static void iter_recurse(struct json_iter *iter, jsmntok_t *token, child->contents = c; child->start = token - c->tokens; + child->current = child->start; child->count =count_tokens_in_container (iter, token); + +/* +* Add one to include the object/array token itself. This is required +* since 'current' points to the container initially. Only after a call +* to json_iter_next() will 'current' point to the first token in the +* container. +*/ +if (token->type ==JSMN_OBJECT || token->type ==JSMN_ARRAY ) + child->count++; }struct json_contents *json_contents_new (const char *json,size_t json_len)
@@ -160,6 +170,7 @@ static void assign_arg(void *data, void *user_data)break ;case JSON_OBJECT :case JSON_PRIMITIVE : +case JSON_ARRAY : iter_val = arg->value;if (!arg->v)
@@ -207,6 +218,7 @@ bool json_iter_parse(struct json_iter *iter, enum json_type type, ...)case JSON_STRING :case JSON_OBJECT :case JSON_PRIMITIVE : +case JSON_ARRAY :break ;default : goto error;
@@ -271,7 +283,7 @@ static bool iter_get_primitive_data(struct json_iter *iter, void **ptr, size_t *len) {struct json_contents *c = iter->contents; -jsmntok_t *t = c->tokens + iter->start; +jsmntok_t *t = c->tokens + iter->current;if (t->type !=JSMN_PRIMITIVE )return false;
@@ -370,3 +382,45 @@ bool json_iter_get_null(struct json_iter *iter)return false; } + +enum json_type json_iter_get_type (struct json_iter *iter) +{ +struct json_contents *c = iter->contents; +jsmntok_t *t = c->tokens + iter->current; + +return (enum json_type ) t->type; +} + +bool json_iter_next (struct json_iter *iter) +{ +struct json_contents *c = iter->contents; +jsmntok_t *t = c->tokens + iter->current; +int inc =1 ; + +/* +* If this is the initial iteration skip this and just increment +* current by 1 since this iterator points to a container which needs to +* be advanced to the first token.. +* +* In addition primitive types and empty containers will have a size +* of 1, so no special handling is needed there. +* +* For non-empty containers 'current' needs to be advanced by all the +* containers child tokens, plus the container itself. +* +* This check ensures: +* 1. It is not the initial iteration +* 2. This is a container +* 3. The container is not empty +*/ +if (iter->current != iter->start && ((t->type ==JSMN_OBJECT || + t->type ==JSMN_ARRAY ) && t->size)) + inc =count_tokens_in_container (iter, t) +1 ; + +if (c->tokens + iter->current + inc >=ITER_END (iter)) +return false; + + iter->current += inc; + +return true; +}
src/json.h +4 -0
@@ -42,6 +42,7 @@ struct json_iter {struct json_contents *contents;int start;int count; +int current; };#define JSON_MANDATORY (key, type, out) \
@@ -92,3 +93,6 @@ bool json_iter_get_int(struct json_iter *iter, int *i);bool json_iter_get_uint (struct json_iter *iter,unsigned int *i);bool json_iter_get_boolean (struct json_iter *iter,bool *b);bool json_iter_get_null (struct json_iter *iter); + +enum json_type json_iter_get_type (struct json_iter *iter); +bool json_iter_next (struct json_iter *iter);
Signature · 1 line
2.31.1
Denis Kenzior
Re: [PATCH v2 4/9] json: add support for array iteration
Hi James,
Quoted 22 lines · James Prestwood
Arrays can now be parsed using the
JSON_ARRAYtype (stored in a structjson_iter) then iterated usingjson_iter_next. When iterating the type can be checked withjson_iter_get_type. For each iteration the value can be obtained using any of the type getters (int/uint/boolean/null).--- src/json.c | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++- src/json.h | 4 ++++ 2 files changed, 59 insertions(+), 1 deletion(-)src/json.c +1
-0@@ -88,7 +88,17 @@ static void iter_recurse(struct json_iter *iter, jsmntok_t *token, child->contents = c; child->start = token - c->tokens; + child->current = child->start; child->count =count_tokens_in_container (iter, token);
src/json.c static void iter_recurse(struct json_iter *iter, jsmntok_t *token, · line 102
Quoted 10 lines
src/json.c +9
-0+ +/* +* Add one to include the object/array token itself. This is required +* since 'current' points to the container initially. Only after a call +* to json_iter_next() will 'current' point to the first token in the +* container. +*/ +if (token->type ==JSMN_OBJECT || token->type ==JSMN_ARRAY ) + child->count++; }
This seems a bit dangerous since count is set based on ITER_END in patch 1. Is there a chance this ends up allowing going past the allocated tokens?
I'm guessing you are trying to account for a slight difference in behavior of json_iter depending on what type it points to:
- if json_iter points to JSMN_PRIMITIVE,
then json_iter_next should not be called (and should fail)
- if json_iter points to JSMN_OBJECT/ARRAY, then json_iter_next has to be called prior to obtaining the first element
Am I right so far?
Quoted 2 lines
src/json.c +0
-0struct json_contents *json_contents_new (const char *json,size_t json_len)
<snip>
Quoted 19 lines
src/json.c +15
-0@@ -370,3 +382,45 @@ bool json_iter_get_null(struct json_iter *iter)return false; } + +enum json_type json_iter_get_type (struct json_iter *iter) +{ +struct json_contents *c = iter->contents; +jsmntok_t *t = c->tokens + iter->current; + +return (enum json_type ) t->type; +} + +bool json_iter_next (struct json_iter *iter) +{ +struct json_contents *c = iter->contents; +jsmntok_t *t = c->tokens + iter->current; +int inc =1 ; +
src/json.c bool json_iter_get_null(struct json_iter *iter) · line 409
Quoted 10 lines
src/json.c +10
-0+/* +* If this is the initial iteration skip this and just increment +* current by 1 since this iterator points to a container which needs to +* be advanced to the first token.. +* +* In addition primitive types and empty containers will have a size +* of 1, so no special handling is needed there. +* +* For non-empty containers 'current' needs to be advanced by all the +* containers child tokens, plus the container itself.
One idea might be to use the 'size' variable of the start token to know whether the iterator can be advanced or not. Alternatively, look up the base type of the iterator at the start token and modify the behavior here accordingly.
Quoted 7 lines
src/json.c +7
-0+* +* This check ensures: +* 1. It is not the initial iteration +* 2. This is a container +* 3. The container is not empty +*/ + if (iter->current != iter->start && ((t->type == JSMN_OBJECT ||
src/json.c bool json_iter_get_null(struct json_iter *iter) · line 426
Quoted 10 lines
src/json.c +10
-0+ t->type == JSMN_ARRAY) && t->size)) + inc =count_tokens_in_container (iter, t) +1 ; + +if (c->tokens + iter->current + inc >=ITER_END (iter)) +return false; + + iter->current += inc; + +return true; +}
Regards,
-Denis
James Prestwood
Re: [PATCH v2 4/9] json: add support for array iteration
Quoted 55 lines · Denis Kenzior
Hi James,
Quoted 30 lines · James Prestwood
Arrays can now be parsed using the
JSON_ARRAYtype (stored in a structjson_iter) then iterated usingjson_iter_next. When iterating the type can be checked withjson_iter_get_type. For each iteration the value can be obtained using any of the type getters (int/uint/boolean/null).
---
src/json.c| 56
+++++++++++++++++++++++++++++++++++++++++++++++++++++-
src/json.h| 4 ++++2 files changed, 59 insertions(+), 1 deletion(-)src/json.c +0
-0@@ -88,7 +88,17 @@ static void iter_recurse(struct json_iter *iter, jsmntok_t *token,
child->contents = c;
child->start = token -c->tokens;
+ child->current =child->start;
child->count =count_tokens_in_container(iter, token); +
+ /*
+ * Add one to include the object/array token itself. This is required
+ * since 'current' points to the container initially. Only after a callQuoted 8 lines
+ * to
json_iter_next()will 'current' point to the first token in the
+ * container.
+ */
+ if (token->type==JSMN_OBJECT||token->type==JSMN_ARRAY)
+ child->count++;
}This seems a bit dangerous since count is set based on
ITER_ENDin patch 1. Is
there a chance this ends up allowing going past the allocated tokens?I'm guessing you are trying to account for a slight difference in behavior of
json_iterdepending on what type it points to:
- ifjson_iterpoints toJSMN_PRIMITIVE,
thenjson_iter_nextshould not be called (and should fail)
- ifjson_iterpoints to JSMN_OBJECT/ARRAY, thenjson_iter_nexthas to be called
prior to obtaining the first element
Quoted 1 line
Am I right so far?
Yes that is exactly it. Consider this array:
["foo", "bar"]
count_tokens_in_container() will return 2 tokens. count++ gives us 3 and 'start' points to the array itself.
Call json_iter_next():
'start' is advanced by 1, pointing to "foo"
Call json_iter_next():
'start' is advanced by 1, pointing to "bar"
Call json_iter_next():
'start' is advanced by 1, pointing to some token after "bar". We then check: if (c->tokens + iter->current + inc >= ITER_END(iter)) return false; And in this case iter->current + inc will equal ITER_END
I do see your point, and accessing a token at [count] would indeed be bad. But that never actually happens, we bail out once we see current equal count.
Quoted 40 lines
Quoted 4 lines
structjson_contents*json_contents_new(const char *json,size_tjson_len)<snip>
Quoted 33 lines
src/json.c +0
-0@@ -370,3 +382,45 @@ bool json_iter_get_null(struct json_iter *iter)
return false;
}src/json.c +17
-0+ +enum json_type json_iter_get_type (struct json_iter *iter) +{ + struct json_contents *c = iter->contents; + jsmntok_t *t = c->tokens + iter->current; + + return (enum json_type) t->type; +} + +bool json_iter_next (struct json_iter *iter) +{ + struct json_contents *c = iter->contents; + jsmntok_t *t = c->tokens + iter->current; + int inc = 1; + + /* +* If this is the initial iteration skip this and just increment
+ * current by 1 since this iterator points to a container which needs to
+ * be advanced to the first token..
+ *
+ * In addition primitive types and empty containers will have a size
+ * of 1, so no special handling is needed there. + *
+ * For non-empty containers 'current' needs to be advanced by all the
Quoted 8 lines
Quoted 1 line
+ * containers child tokens, plus the container itself.
One idea might be to use the 'size' variable of the start token to know whether
the iterator can be advanced or not. Alternatively, look up the base type of
the iterator at the start token and modify the behavior here accordingly.
Are you talking about preventing primitive iteration? Or just altertering how iteration is done in general?
The check below ensures the token is an iterable container, and size is used in that check if only to detect empty objects/arrays, which would then be treated exactly like a primitive/string.
I'm not sure 'size' can be trusted soley to know if its an iterable object. For example a key's size is 1. I'm not sure how an iterator could end up pointing there, but checking OBJECT/ARRAY explicitly seems like the safest bet to me.
Quoted 23 lines
Quoted 12 lines
+ *
+ * This check ensures:
+ * 1. It is not the initial iteration
+ * 2. This is a container
+ * 3. The container is not empty
+ */
+ if (iter->current!=iter->start&& ((t->type==JSMN_OBJECT||
+ t->type ==JSMN_ARRAY) &&t->size))
+ inc =count_tokens_in_container(iter, t)+ 1;src/json.c bool json_iter_get_null(struct json_iter *iter) · line 426
Quoted 8 lines
src/json.c +8
-0+ + if (c->tokens + iter->current + inc >= ITER_END(iter)) + return false; + + iter->current += inc; + + return true; +}Regards,
-Denis
Denis Kenzior
Re: [PATCH v2 4/9] json: add support for array iteration
Hi James,
Quoted 9 lines
Quoted 1 line
Am I right so far?
Yes that is exactly it. Consider this array:
["foo", "bar"]
count_tokens_in_container()will return 2 tokens. count++ gives us 3 and 'start' points to the array itself.
Ok, I think I managed to convince myself this is indeed okay, particularly since it is consistent with json_iter_init() behavior.
One thing you may to do is add a check to json_iter_parse to make sure it is operating on iterators of type JSMN_OBJECT and nothing else. That function's use of ITER_END is what made me suspicious.
Quoted 6 lines
Call
json_iter_next():
'start' is advanced by 1, pointing to "foo"Call
json_iter_next():
'start' is advanced by 1, pointing to "bar"
So in theory, you could use 'size' as a limit counter. So for primitives, set pos to 1 and have iter_next check whether pos + 1 <= size.
For arrays/objects, set size to 0 and don't allow getters to work if pos is 0. Or something to that effect. Not sure if this makes things easier or not.
Quoted 10 lines
Call
json_iter_next():'start' is advanced by 1, pointing to some token after "bar". We then check: if (c->tokens + iter->current + inc >= ITER_END(iter)) return false; And in this case iter->current + inc will equal ITER_END
Yes, this all makes sense. Why do we increment by count_tokens_in_container() + 1 though?
<snip>
Quoted 9 lines
Quoted 6 lines
One idea might be to use the 'size' variable of the start token to know whether
the iterator can be advanced or not. Alternatively, look up the base type of
the iterator at the start token and modify the behavior here accordingly.Are you talking about preventing primitive iteration? Or just altertering how iteration is done in general?
I think both.
Quoted 9 lines
The check below ensures the token is an iterable container, and size is used in that check if only to detect empty objects/arrays, which would then be treated exactly like a primitive/string.
I'm not sure 'size' can be trusted soley to know if its an iterable object. For example a key's size is 1. I'm not sure how an iterator could end up pointing there, but checking OBJECT/ARRAY explicitly seems like the safest bet to me.
I would assume (hope?) that size can be trusted. I could be wrong, but seems to me like it would make some checks easier.
Are empty objects/arrays really of size 1? How does one distinguish between an array with 1 element and an empty array?
Regards,
-Denis
Denis Kenzior
Re: [PATCH v2 4/9] json: add support for array iteration
Hi James,
Quoted 2 lines
Yes, this all makes sense. Why do we increment by
count_tokens_in_container()+ 1 though?
Ah nevermind, I gotcha.
Anyhow, I applied the rest of this series.
Regards,
-Denis
James Prestwood
PATCH v2 5/9 json: increase default token size to 60
In reply to James Prestwood--- src/json.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
src/json.c +1 -1
@@ -33,7 +33,7 @@#include "shared/jsmn.h" /* Max number of tokens supported. Increase if larger objects are expected */ -#define JSON_DEFAULT_TOKENS 30 +#define JSON_DEFAULT_TOKENS 60#define TOK_LEN (token) (token)->end - (token)->start#define TOK_PTR (json, token) (void *)((json) + (token)->start)
Signature · 1 line
2.31.1
Denis Kenzior
AppliedRe: [PATCH v2 5/9] json: increase default token size to 60
Hi James,
Quoted 4 lines · James Prestwood
--- src/json.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
Applied, thanks.
Regards,
-Denis
James Prestwood
PATCH v2 6/9 unit: add test for json arrays
In reply to James Prestwood--- unit/test-json.c | 113 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 92 insertions(+), 21 deletions(-)
unit/test-json.c +92 -21
@@ -75,26 +75,6 @@ static void test_json_escaped_unicode(const void *data)json_contents_free (c); } -/* -* Tests that unsupported types will not parse -*/ -static void test_json_unsupported_types (const void *data) -{ -/* -* Valid JSON objects but currently unsupported types -*/ -char arrays[] ="{\"test\":[1, 2, 3, 4]}" ; - -struct json_iter iter; -struct json_contents *c =json_contents_new (arrays,strlen (arrays)); - -json_iter_init (&iter, c); -assert (!json_iter_parse (&iter, -JSON_MANDATORY ("test" ,JSON_ARRAY ,NULL ), -JSON_UNDEFINED )); -json_contents_free (c); -} -/* * Basic test string values and nested objects */
@@ -325,6 +305,97 @@ static void test_json_primitives(const void *data)json_contents_free (c); } +static void test_json_arrays (const void *data) +{ +unsigned int ui; +int i; +bool b; +int count; +char json[] ="{\"uint_array\":[1, 2, 3, 4, 5, 6]," +"\"int_array\":[-1, -2, -3, -4, -5, -6]," +"\"bool_array\":[true, false, true, false]," +"\"null_array\":[null, null, null, null]," +"\"mixed_array\":[1, -1, true, false, null]}" ; + +struct json_iter iter; +struct json_iter i_array, ui_array, b_array, n_array, m_array; +struct json_contents *c =json_contents_new (json,strlen (json)); + +json_iter_init (&iter, c); +assert (json_iter_parse (&iter, +JSON_MANDATORY ("mixed_array" ,JSON_ARRAY , &m_array), +JSON_MANDATORY ("null_array" ,JSON_ARRAY , &n_array), +JSON_MANDATORY ("bool_array" ,JSON_ARRAY , &b_array), +JSON_MANDATORY ("int_array" ,JSON_ARRAY , &i_array), +JSON_MANDATORY ("uint_array" ,JSON_ARRAY , &ui_array), +JSON_UNDEFINED )); + + count =1 ; + +while (json_iter_next (&ui_array)) { +assert (json_iter_get_type (&ui_array) ==JSON_PRIMITIVE ); +assert (json_iter_get_uint (&ui_array, &ui)); +assert (ui == (unsigned int ) count); + count++; + } + + count =-1 ; + +while (json_iter_next (&i_array)) { +assert (json_iter_get_type (&i_array) ==JSON_PRIMITIVE ); +assert (json_iter_get_int (&i_array, &i)); +assert (i == count); + count--; + } + + count =0 ; + +while (json_iter_next (&b_array)) { +assert (json_iter_get_type (&b_array) ==JSON_PRIMITIVE ); +assert (json_iter_get_boolean (&b_array, &b)); +assert (b == count %2 ? false : true); + count++; + } + + count =0 ; + +while (json_iter_next (&n_array)) { +assert (json_iter_get_type (&n_array) ==JSON_PRIMITIVE ); +assert (json_iter_get_null (&n_array)); + count++; + } + +assert (count ==4 ); + + count =0 ; + +while (json_iter_next (&m_array)) { +assert (json_iter_get_type (&m_array) ==JSON_PRIMITIVE ); + +switch (count) { +case 0 : +assert (json_iter_get_uint (&m_array, &ui)); +assert (ui ==1 ); +break ; +case 1 : +assert (json_iter_get_int (&m_array, &i)); +assert (i ==-1 ); +break ; +case 2 : +case 3 : +assert (json_iter_get_boolean (&m_array, &b)); +assert (b == count %2 ? false : true); +break ; +case 4 : +assert (json_iter_get_null (&m_array)); +break ; + } + + count++; + } + +json_contents_free (c); +}int main (int argc,char *argv[]) {
@@ -333,11 +404,11 @@ int main(int argc, char *argv[])l_test_add ("json unicode" , test_json_unicode,NULL );l_test_add ("json escaped unicode" , test_json_escaped_unicode,NULL );l_test_add ("json nested objects" , test_json,NULL ); -l_test_add ("json unsupported types" , test_json_unsupported_types,NULL );l_test_add ("json empty objects" , test_json_empty_objects,NULL );l_test_add ("json parse out of order" , test_json_out_of_order,NULL );l_test_add ("json larger object" , test_json_larger_object,NULL );l_test_add ("json test primitives" , test_json_primitives,NULL ); +l_test_add ("json test arrays" , test_json_arrays,NULL );return l_test_run (); }
Signature · 1 line
2.31.1
James Prestwood
PATCH v2 7/9 json: add json_iter_get_container
In reply to James PrestwoodThis allows getting an iterator for a container nested inside an array.
--- src/json.c | 14 ++++++++++++++ src/json.h | 2 ++ 2 files changed, 16 insertions(+)
src/json.c +14 -0
@@ -383,6 +383,20 @@ bool json_iter_get_null(struct json_iter *iter)return false; } +bool json_iter_get_container (struct json_iter *iter, +struct json_iter *container) +{ +struct json_contents *c = iter->contents; +jsmntok_t *t = c->tokens + iter->current; + +if (t->type !=JSMN_OBJECT && t->type !=JSMN_ARRAY ) +return false; + +iter_recurse (iter, t, container); + +return true; +} +enum json_type json_iter_get_type (struct json_iter *iter) {struct json_contents *c = iter->contents;
src/json.h +2 -0
@@ -93,6 +93,8 @@ bool json_iter_get_int(struct json_iter *iter, int *i);bool json_iter_get_uint (struct json_iter *iter,unsigned int *i);bool json_iter_get_boolean (struct json_iter *iter,bool *b);bool json_iter_get_null (struct json_iter *iter); +bool json_iter_get_container (struct json_iter *iter, +struct json_iter *container);enum json_type json_iter_get_type (struct json_iter *iter);bool json_iter_next (struct json_iter *iter);
Signature · 1 line
2.31.1
James Prestwood
PATCH v2 8/9 unit: add arrays of objects test
In reply to James Prestwood--- unit/test-json.c | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-)
unit/test-json.c +28 -1
@@ -315,10 +315,12 @@ static void test_json_arrays(const void *data)"\"int_array\":[-1, -2, -3, -4, -5, -6]," "\"bool_array\":[true, false, true, false]," "\"null_array\":[null, null, null, null]," +"\"obj_array\":[{}, {\"key\":\"value\", \"key2\":\"value2\"}]," "\"mixed_array\":[1, -1, true, false, null]}" ;struct json_iter iter; -struct json_iter i_array, ui_array, b_array, n_array, m_array; +struct json_iter i_array, ui_array, b_array, n_array, + m_array, obj_array;struct json_contents *c =json_contents_new (json,strlen (json));json_iter_init (&iter, c);
@@ -328,6 +330,7 @@ static void test_json_arrays(const void *data)JSON_MANDATORY ("bool_array" ,JSON_ARRAY , &b_array),JSON_MANDATORY ("int_array" ,JSON_ARRAY , &i_array),JSON_MANDATORY ("uint_array" ,JSON_ARRAY , &ui_array), +JSON_MANDATORY ("obj_array" ,JSON_ARRAY , &obj_array), JSON_UNDEFINED)); count =1 ;
@@ -394,6 +397,30 @@ static void test_json_arrays(const void *data) count++; } + count =0 ; + +while (json_iter_next (&obj_array)) { +struct json_iter object; + +assert (json_iter_get_type (&obj_array) ==JSON_OBJECT ); +assert (json_iter_get_container (&obj_array, &object)); + +switch (count) { +case 0 : +assert (json_iter_parse (&object,JSON_UNDEFINED )); +break ; +case 1 : +assert (json_iter_parse (&object, +JSON_MANDATORY ("key" ,JSON_STRING ,NULL ), +JSON_UNDEFINED )); +break ; + } + + count++; + } + +assert (count ==2 ); +json_contents_free (c); }
Signature · 1 line
2.31.1
James Prestwood
PATCH v2 9/9 unit: add test for nested arrays
In reply to James Prestwood--- unit/test-json.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+)
unit/test-json.c +46 -0
@@ -424,6 +424,51 @@ static void test_json_arrays(const void *data)json_contents_free (c); } +static void test_json_nested_arrays (const void *data) +{ +char json[] ="{\"array\":[[], {}, [1, 2], {\"key\":\"value\"}]}" ; +int count =0 ; +struct json_iter iter; +struct json_iter array; +struct json_iter inner; +struct json_contents *c =json_contents_new (json,strlen (json)); + +json_iter_init (&iter, c); +assert (json_iter_parse (&iter, +JSON_MANDATORY ("array" ,JSON_ARRAY , &array), +JSON_UNDEFINED )); + +while (json_iter_next (&array)) { +int count2 =0 ; + +assert (json_iter_get_container (&array, &inner)); + +while (json_iter_next (&inner)) + count2++; + +/* +* TODO: add checks for object iteration. Currently these will +* just count all tokens in the object. +*/ +switch (count) { +case 0 : +assert (count2 ==0 ); +break ; +case 2 : +assert (count2 ==2 ); +break ; +default : +break ; + } + + count++; + } + +assert (count ==4 ); + +json_contents_free (c); +} +int main (int argc,char *argv[]) {l_test_init (&argc, &argv);
@@ -436,6 +481,7 @@ int main(int argc, char *argv[])l_test_add ("json larger object" , test_json_larger_object,NULL );l_test_add ("json test primitives" , test_json_primitives,NULL );l_test_add ("json test arrays" , test_json_arrays,NULL ); +l_test_add ("json test nested arrays" , test_json_nested_arrays,NULL );return l_test_run (); }
Signature · 1 line
2.31.1