Skip to contentSkip to conversation

json: fix find_object_tokens

PATCH Applied James Prestwood 2 participants 15 messages Jan 6, 2022

lore
Share & tools
lore

James Prestwood

author
#0 lore raw

PATCH v2 1/9 json: fix find_object_tokens

First, this was renamed to 'count_tokens_in_container' to be more general purpose (i.e. include future array counting).

The way the tokens are counted also changed to be more intuitive. While the previous way was correct, it was somewhat convoluted in how it worked (finding the next parent of the objects parent).

Instead we can use the container token itself as the parent and begin counting tokens. When we find a token with a parent index less than the target we have reached the end of this container. This also works for nested containers, including arrays since we no longer rely on a key (which an array element would not have).

For example::

{
	"first":{"foo":"bar"},
	"second":{"foo2":"bar2"}
}

index 0 <overall object>
index 1 "first" with parent 0
index 2 {"foo":"bar"} with parent 1

Counting tokens inside "first"'s object we have:

index 3 "foo" with parent 2
index 4 "bar" with parent 3

If we continue counting we reach:

index 5 "second" with parent 0

This terminates the counting loop since the parent index is less than '2' (the index of {"foo":"bar"} object).

---
 src/json.c | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)
src/json.c +8 -12
index 39570e7e..a917fb89 100644
@@ -69,20 +69,16 @@ static jsmntok_t *next_key_in_parent(struct json_iter *iter, jsmntok_t *current)
 	return NULL;
 }
 
-/*
- * 'object' is expected to be a value, so object - 1 is its key. Find
- * the next key who's parent matches the parent of object - 1. The
- * token preceeding this next key will mark the end of 'object'.
- */
-static int find_object_tokens(struct json_iter *iter, jsmntok_t *object)
+static int count_tokens_in_container(struct json_iter *iter, jsmntok_t *container)
 {
-	jsmntok_t *next = next_key_in_parent(iter, object - 1);
+	int idx = container - iter->contents->tokens;
+	jsmntok_t *contents;
 
-	/* End of token list */
-	if (!next)
-		next = ITER_END(iter);
+	for (contents = ++container; contents < ITER_END(iter); contents++)
+		if (contents->parent < idx)
+			break;
 
-	return next - object - 1;
+	return contents - container;
 }
 
 static void iter_recurse(struct json_iter *iter, jsmntok_t *object,
@@ -92,7 +88,7 @@ static void iter_recurse(struct json_iter *iter, jsmntok_t *object,
 
 	child->contents = c;
 	child->start = object - c->tokens;
-	child->count = find_object_tokens(iter, object);
+	child->count = count_tokens_in_container(iter, object);
 }
 
 struct json_contents *json_contents_new(const char *json, size_t json_len)
Signature · 1 line
2.31.1
author
#1 lore raw

PATCH v2 2/9 json: add support for primitive types

This adds support for boolean, (unsigned) integers, and null types. JSON_PRIMITIVE should be used as the type when parsing and the value should be struct json_iter.

Once parsed the actual value can be obtained using one of the primitive getters. If the type does not match they will return false.

If using JSON_OPTIONAL with JSON_PRIMITIVE the resulting iterator can be checked with json_iter_is_valid. If false the key/value was not found or the type was not matching.

---
 src/json.c | 120 +++++++++++++++++++++++++++++++++++++++++++++++++----
 src/json.h |  13 +++++-
 2 files changed, 125 insertions(+), 8 deletions(-)
src/json.c +113 -7
index a917fb89..6c88ed3a 100644
@@ -81,14 +81,14 @@ static int count_tokens_in_container(struct json_iter *iter, jsmntok_t *containe
 	return contents - container;
 }
 
-static void iter_recurse(struct json_iter *iter, jsmntok_t *object,
+static void iter_recurse(struct json_iter *iter, jsmntok_t *token,
 				struct json_iter *child)
 {
 	struct json_contents *c = iter->contents;
 
 	child->contents = c;
-	child->start = object - c->tokens;
-	child->count = count_tokens_in_container(iter, object);
+	child->start = token - c->tokens;
+	child->count = count_tokens_in_container(iter, token);
 }
 
 struct json_contents *json_contents_new(const char *json, size_t json_len)
@@ -149,7 +149,7 @@ static void assign_arg(void *data, void *user_data)
 	struct json_arg *arg = data;
 	struct json_contents *c = iter->contents;
 	char **sval;
-	struct json_iter *oval;
+	struct json_iter *iter_val;
 
 	switch (arg->type) {
 	case JSON_STRING:
@@ -159,12 +159,13 @@ static void assign_arg(void *data, void *user_data)
 
 		break;
 	case JSON_OBJECT:
-		oval = arg->value;
+	case JSON_PRIMITIVE:
+		iter_val = arg->value;
 
 		if (!arg->v)
-			oval->start = -1;
+			iter_val->start = -1;
 		else
-			iter_recurse(iter, arg->v, oval);
+			iter_recurse(iter, arg->v, iter_val);
 
 		break;
 	default:
@@ -205,6 +206,7 @@ bool json_iter_parse(struct json_iter *iter, enum json_type type, ...)
 			goto done;
 		case JSON_STRING:
 		case JSON_OBJECT:
+		case JSON_PRIMITIVE:
 			break;
 		default:
 			goto error;
@@ -264,3 +266,107 @@ error:
 	l_queue_destroy(args, l_free);
 	return false;
 }
+
+static bool iter_get_primitive_data(struct json_iter *iter, void **ptr,
+					size_t *len)
+{
+	struct json_contents *c = iter->contents;
+	jsmntok_t *t = c->tokens + iter->start;
+
+	if (t->type != JSMN_PRIMITIVE)
+		return false;
+
+	*ptr = TOK_PTR(c->json, t);
+	*len = TOK_LEN(t);
+
+	return true;
+}
+
+bool json_iter_get_int(struct json_iter *iter, int *i)
+{
+	void *ptr;
+	size_t len;
+	long int r;
+	int t;
+	char *endp;
+
+	if (!iter_get_primitive_data(iter, &ptr, &len))
+		return false;
+
+	errno = 0;
+
+	t = r = strtol(ptr, &endp, 10);
+	if (endp != ptr + len)
+		return false;
+
+	if (errno == ERANGE || r != t)
+		return false;
+
+	if (i)
+		*i = r;
+
+	return true;
+}
+
+bool json_iter_get_uint(struct json_iter *iter, unsigned int *i)
+{
+	void *ptr;
+	size_t len;
+	unsigned long int r;
+	unsigned int t;
+	char *endp;
+
+	if (!iter_get_primitive_data(iter, &ptr, &len))
+		return false;
+
+	errno = 0;
+
+	t = r = strtoul(ptr, &endp, 10);
+	if (endp != ptr + len)
+		return false;
+
+	if (errno == ERANGE || r != t)
+		return false;
+
+	if (i)
+		*i = r;
+
+	return true;
+}
+
+bool json_iter_get_boolean(struct json_iter *iter, bool *b)
+{
+	void *ptr;
+	size_t len;
+
+	if (!iter_get_primitive_data(iter, &ptr, &len))
+		return false;
+
+	if (len == 4 && !memcmp(ptr, "true", 4)) {
+		if (b)
+			*b = true;
+
+		return true;
+	} else if (len == 5 && !memcmp(ptr, "false", 5)) {
+		if (b)
+			*b = false;
+
+		return true;
+	}
+
+	return false;
+}
+
+bool json_iter_get_null(struct json_iter *iter)
+{
+	void *ptr;
+	size_t len;
+
+	if (!iter_get_primitive_data(iter, &ptr, &len))
+		return false;
+
+	if (len == 4 && !memcmp(ptr, "null", 4))
+		return true;
+
+	return false;
+}
src/json.h +12 -1
index 915b675b..9f00a9d5 100644
@@ -73,11 +73,22 @@ void json_iter_init(struct json_iter *iter, struct json_contents *c);
  *
  * String values should be of type char ** and must be freed
  * Object values should be of type struct json_iter *
+ * Primitive types (numbers, booleans, null) should be of type
+ * 		struct json_iter *. This is to allow the caller to distinguish
+ * 		between the actual value type after parsing using a getter for
+ * 		the expected type (get_uint/get_int/get_boolean etc.). In
+ * 		addition this lets the caller use JSON_OPTIONAL and check post
+ * 		json_iter_parse if the iterator is valid (json_iter_is_valid).
  *
  * No other types are supported at this time, and json_iter_parse will fail if
  * other types are encountered.
  *
  * JSON_OPTIONAL string values will point to NULL if not found
- * JSON_OPTIONAL objects can be checked with json_object_not_found.
+ * JSON_OPTIONAL objects/primitives can be checked with json_object_is_valid.
  */
 bool json_iter_parse(struct json_iter *iter, enum json_type type, ...);
+
+bool json_iter_get_int(struct json_iter *iter, int *i);
+bool json_iter_get_uint(struct json_iter *iter, unsigned int *i);
+bool json_iter_get_boolean(struct json_iter *iter, bool *b);
+bool json_iter_get_null(struct json_iter *iter);
Signature · 1 line
2.31.1
author
#2 lore raw

PATCH v2 3/9 unit: add primitive json test

In reply to James Prestwood
---
 unit/test-json.c | 111 ++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 95 insertions(+), 16 deletions(-)
unit/test-json.c +95 -16
index 854b92dd..fe8f756f 100644
@@ -84,8 +84,6 @@ static void test_json_unsupported_types(const void *data)
 	 * Valid JSON objects but currently unsupported types
 	 */
 	char arrays[] = "{\"test\":[1, 2, 3, 4]}";
-	char integers[] = "{\"test\": 10 }";
-	char bools[] = "{\"test\": true}";
 
 	struct json_iter iter;
 	struct json_contents *c = json_contents_new(arrays, strlen(arrays));
@@ -95,20 +93,6 @@ static void test_json_unsupported_types(const void *data)
 				JSON_MANDATORY("test", JSON_ARRAY, NULL),
 				JSON_UNDEFINED));
 	json_contents_free(c);
-
-	c = json_contents_new(integers, strlen(integers));
-	json_iter_init(&iter, c);
-	assert(!json_iter_parse(&iter,
-				JSON_MANDATORY("test", JSON_PRIMITIVE, NULL),
-				JSON_UNDEFINED));
-	json_contents_free(c);
-
-	c = json_contents_new(bools, strlen(bools));
-	json_iter_init(&iter, c);
-	assert(!json_iter_parse(&iter,
-				JSON_MANDATORY("test", JSON_PRIMITIVE, NULL),
-				JSON_UNDEFINED));
-	json_contents_free(c);
 }
 
 /*
@@ -248,6 +232,100 @@ static void test_json_larger_object(const void *data)
 	json_contents_free(c);
 }
 
+static void check_primitives(struct json_iter *i, struct json_iter *ui,
+				struct json_iter *t, struct json_iter *f,
+				struct json_iter *null, struct json_iter *obj)
+{
+
+	int i_val;
+	unsigned int ui_val;
+	bool b_val;
+
+	assert(json_iter_is_valid(i));
+	assert(!json_iter_get_uint(i, NULL));
+	assert(!json_iter_get_boolean(i, NULL));
+	assert(!json_iter_get_null(i));
+	assert(json_iter_get_int(i, &i_val));
+	assert(i_val == -10);
+
+	assert(json_iter_is_valid(ui));
+	assert(!json_iter_get_boolean(ui, NULL));
+	assert(!json_iter_get_null(ui));
+	assert(json_iter_get_int(ui, &i_val));
+	assert(json_iter_get_uint(ui, &ui_val));
+	assert(i_val == 10 && ui_val == 10);
+
+
+	assert(json_iter_is_valid(t));
+	assert(!json_iter_get_null(t));
+	assert(!json_iter_get_int(t, NULL));
+	assert(!json_iter_get_uint(t, NULL));
+	assert(json_iter_get_boolean(t, &b_val));
+	assert(b_val == true);
+
+	assert(json_iter_is_valid(f));
+	assert(!json_iter_get_null(f));
+	assert(!json_iter_get_int(f, NULL));
+	assert(!json_iter_get_uint(f, NULL));
+	assert(json_iter_get_boolean(f, &b_val));
+	assert(b_val == false);
+
+	assert(json_iter_is_valid(null));
+	assert(!json_iter_get_int(null, NULL));
+	assert(!json_iter_get_uint(null, NULL));
+	assert(!json_iter_get_boolean(null, NULL));
+	assert(json_iter_get_null(null));
+
+	if (obj) {
+		assert(json_iter_is_valid(obj));
+		assert(json_iter_parse(obj,
+			JSON_MANDATORY("null_val", JSON_PRIMITIVE, null),
+			JSON_MANDATORY("false_val", JSON_PRIMITIVE, f),
+			JSON_MANDATORY("true_val", JSON_PRIMITIVE, t),
+			JSON_MANDATORY("int_val", JSON_PRIMITIVE, i),
+			JSON_MANDATORY("uint_val", JSON_PRIMITIVE, ui),
+			JSON_UNDEFINED));
+
+		check_primitives(i, ui, t, f, null, NULL);
+	}
+}
+
+static void test_json_primitives(const void *data)
+{
+	char json[] = "{\"int_val\": -10,"
+			"\"uint_val\": 10,"
+			"\"true_val\": true,"
+			"\"false_val\": false,"
+			"\"null_val\": null,"
+			"\"obj_val\":{"
+				"\"int_val\": -10,"
+				"\"uint_val\": 10,"
+				"\"true_val\": true,"
+				"\"false_val\": false,"
+				"\"null_val\": null}}";
+	struct json_contents *c = json_contents_new(json, strlen(json));
+	struct json_iter outer, inner, null, f, t, i, ui;
+	struct json_iter not_found;
+
+	json_iter_init(&outer, c);
+	assert(json_iter_parse(&outer,
+			JSON_MANDATORY("obj_val", JSON_OBJECT, &inner),
+			JSON_MANDATORY("null_val", JSON_PRIMITIVE, &null),
+			JSON_MANDATORY("false_val", JSON_PRIMITIVE, &f),
+			JSON_MANDATORY("true_val", JSON_PRIMITIVE, &t),
+			JSON_MANDATORY("int_val", JSON_PRIMITIVE, &i),
+			JSON_MANDATORY("uint_val", JSON_PRIMITIVE, &ui),
+			JSON_OPTIONAL("not_found", JSON_PRIMITIVE, &not_found),
+			JSON_UNDEFINED));
+
+	assert(!json_iter_is_valid(&not_found));
+
+	check_primitives(&i, &ui, &t, &f, &null, &inner);
+
+	json_contents_free(c);
+}
+
+
 int main(int argc, char *argv[])
 {
 	l_test_init(&argc, &argv);
@@ -259,6 +337,7 @@ int main(int argc, char *argv[])
 	l_test_add("json empty objects", test_json_empty_objects, NULL);
 	l_test_add("json parse out of order", test_json_out_of_order, NULL);
 	l_test_add("json larger object", test_json_larger_object, NULL);
+	l_test_add("json test primitives", test_json_primitives, NULL);
 
 	return l_test_run();
 }
Signature · 1 line
2.31.1
#3 lore raw

Re: [PATCH v2 3/9] unit: add primitive json test

Hi James,

Quoted 4 lines · James Prestwood
---
  unit/test-json.c | 111 ++++++++++++++++++++++++++++++++++++++++-------
  1 file changed, 95 insertions(+), 16 deletions(-)

I removed some double empty lines in this commit. Patches 1-3 applied, thanks.

Regards,
-Denis

author
#4 lore raw

PATCH v2 4/9 json: add support for array iteration

In reply to James Prestwood

Arrays can now be parsed using the JSON_ARRAY type (stored in a struct json_iter) then iterated using json_iter_next. When iterating the type can be checked with json_iter_get_type. For each iteration the value can be obtained using any of the type getters (int/uint/boolean/null).

---
 src/json.c | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
 src/json.h |  4 ++++
 2 files changed, 59 insertions(+), 1 deletion(-)
src/json.c +55 -1
index 6c88ed3a..cf233e56 100644
@@ -88,7 +88,17 @@ static void iter_recurse(struct json_iter *iter, jsmntok_t *token,
 
 	child->contents = c;
 	child->start = token - c->tokens;
+	child->current = child->start;
 	child->count = count_tokens_in_container(iter, token);
+
+	/*
+	 * Add one to include the object/array token itself. This is required
+	 * since 'current' points to the container initially. Only after a call
+	 * to json_iter_next() will 'current' point to the first token in the
+	 * container.
+	 */
+	if (token->type == JSMN_OBJECT || token->type == JSMN_ARRAY)
+		child->count++;
 }
 
 struct json_contents *json_contents_new(const char *json, size_t json_len)
@@ -160,6 +170,7 @@ static void assign_arg(void *data, void *user_data)
 		break;
 	case JSON_OBJECT:
 	case JSON_PRIMITIVE:
+	case JSON_ARRAY:
 		iter_val = arg->value;
 
 		if (!arg->v)
@@ -207,6 +218,7 @@ bool json_iter_parse(struct json_iter *iter, enum json_type type, ...)
 		case JSON_STRING:
 		case JSON_OBJECT:
 		case JSON_PRIMITIVE:
+		case JSON_ARRAY:
 			break;
 		default:
 			goto error;
@@ -271,7 +283,7 @@ static bool iter_get_primitive_data(struct json_iter *iter, void **ptr,
 					size_t *len)
 {
 	struct json_contents *c = iter->contents;
-	jsmntok_t *t = c->tokens + iter->start;
+	jsmntok_t *t = c->tokens + iter->current;
 
 	if (t->type != JSMN_PRIMITIVE)
 		return false;
@@ -370,3 +382,45 @@ bool json_iter_get_null(struct json_iter *iter)
 
 	return false;
 }
+
+enum json_type json_iter_get_type(struct json_iter *iter)
+{
+	struct json_contents *c = iter->contents;
+	jsmntok_t *t = c->tokens + iter->current;
+
+	return (enum json_type) t->type;
+}
+
+bool json_iter_next(struct json_iter *iter)
+{
+	struct json_contents *c = iter->contents;
+	jsmntok_t *t = c->tokens + iter->current;
+	int inc = 1;
+
+	/*
+	 * If this is the initial iteration skip this and just increment
+	 * current by 1 since this iterator points to a container which needs to
+	 * be advanced to the first token..
+	 *
+	 * In addition primitive types and empty containers will have a size
+	 * of 1, so no special handling is needed there.
+	 *
+	 * For non-empty containers 'current' needs to be advanced by all the
+	 * containers child tokens, plus the container itself.
+	 *
+	 * This check ensures:
+	 *    1. It is not the initial iteration
+	 *    2. This is a container
+	 *    3. The container is not empty
+	 */
+	if (iter->current != iter->start && ((t->type == JSMN_OBJECT ||
+					t->type == JSMN_ARRAY) && t->size))
+		inc = count_tokens_in_container(iter, t) + 1;
+
+	if (c->tokens + iter->current + inc >= ITER_END(iter))
+		return false;
+
+	iter->current += inc;
+
+	return true;
+}
src/json.h +4 -0
index 9f00a9d5..458deb93 100644
@@ -42,6 +42,7 @@ struct json_iter {
 	struct json_contents *contents;
 	int start;
 	int count;
+	int current;
 };
 
 #define JSON_MANDATORY(key, type, out) \
@@ -92,3 +93,6 @@ bool json_iter_get_int(struct json_iter *iter, int *i);
 bool json_iter_get_uint(struct json_iter *iter, unsigned int *i);
 bool json_iter_get_boolean(struct json_iter *iter, bool *b);
 bool json_iter_get_null(struct json_iter *iter);
+
+enum json_type json_iter_get_type(struct json_iter *iter);
+bool json_iter_next(struct json_iter *iter);
Signature · 1 line
2.31.1
#5 lore raw

Re: [PATCH v2 4/9] json: add support for array iteration

Hi James,

Quoted 22 lines · James Prestwood

Arrays can now be parsed using the JSON_ARRAY type (stored in a struct json_iter) then iterated using json_iter_next. When iterating the type can be checked with json_iter_get_type. For each iteration the value can be obtained using any of the type getters (int/uint/boolean/null).

---
  src/json.c | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
  src/json.h |  4 ++++
  2 files changed, 59 insertions(+), 1 deletion(-)
src/json.c +1 -0
index 6c88ed3a..cf233e56 100644
@@ -88,7 +88,17 @@ static void iter_recurse(struct json_iter *iter, jsmntok_t *token,
 
  	child->contents = c;
  	child->start = token - c->tokens;
+	child->current = child->start;
  	child->count = count_tokens_in_container(iter, token);

src/json.c static void iter_recurse(struct json_iter *iter, jsmntok_t *token, · line 102

Quoted 10 lines
src/json.c +9 -0
+
+	/*
+	 * Add one to include the object/array token itself. This is required
+	 * since 'current' points to the container initially. Only after a call
+	 * to json_iter_next() will 'current' point to the first token in the
+	 * container.
+	 */
+	if (token->type == JSMN_OBJECT || token->type == JSMN_ARRAY)
+		child->count++;
  }

This seems a bit dangerous since count is set based on ITER_END in patch 1. Is there a chance this ends up allowing going past the allocated tokens?

I'm guessing you are trying to account for a slight difference in behavior of json_iter depending on what type it points to:
- if json_iter points to JSMN_PRIMITIVE,
then json_iter_next should not be called (and should fail)
- if json_iter points to JSMN_OBJECT/ARRAY, then json_iter_next has to be called prior to obtaining the first element

Am I right so far?

Quoted 2 lines
src/json.c +0 -0
  struct json_contents *json_contents_new(const char *json, size_t json_len)

<snip>

Quoted 19 lines
src/json.c +15 -0
@@ -370,3 +382,45 @@ bool json_iter_get_null(struct json_iter *iter)
 
  	return false;
  }
+
+enum json_type json_iter_get_type(struct json_iter *iter)
+{
+	struct json_contents *c = iter->contents;
+	jsmntok_t *t = c->tokens + iter->current;
+
+	return (enum json_type) t->type;
+}
+
+bool json_iter_next(struct json_iter *iter)
+{
+	struct json_contents *c = iter->contents;
+	jsmntok_t *t = c->tokens + iter->current;
+	int inc = 1;
+

src/json.c bool json_iter_get_null(struct json_iter *iter) · line 409

Quoted 10 lines
src/json.c +10 -0
+	/*
+	 * If this is the initial iteration skip this and just increment
+	 * current by 1 since this iterator points to a container which needs to
+	 * be advanced to the first token..
+	 *
+	 * In addition primitive types and empty containers will have a size
+	 * of 1, so no special handling is needed there.
+	 *
+	 * For non-empty containers 'current' needs to be advanced by all the
+	 * containers child tokens, plus the container itself.

One idea might be to use the 'size' variable of the start token to know whether the iterator can be advanced or not. Alternatively, look up the base type of the iterator at the start token and modify the behavior here accordingly.

Quoted 7 lines
src/json.c +7 -0
+	 *
+	 * This check ensures:
+	 *    1. It is not the initial iteration
+	 *    2. This is a container
+	 *    3. The container is not empty
+	 */
+	if (iter->current != iter->start && ((t->type == JSMN_OBJECT ||

src/json.c bool json_iter_get_null(struct json_iter *iter) · line 426

Quoted 10 lines
src/json.c +10 -0
+					t->type == JSMN_ARRAY) && t->size))
+		inc = count_tokens_in_container(iter, t) + 1;
+
+	if (c->tokens + iter->current + inc >= ITER_END(iter))
+		return false;
+
+	iter->current += inc;
+
+	return true;
+}

Regards,
-Denis

author
#6 lore raw

Re: [PATCH v2 4/9] json: add support for array iteration

Quoted 55 lines · Denis Kenzior

Hi James,

Quoted 30 lines · James Prestwood

Arrays can now be parsed using the JSON_ARRAY type (stored in a struct json_iter) then iterated using json_iter_next. When iterating the type can be checked with json_iter_get_type. For each iteration the value can be obtained using any of the type getters (int/uint/boolean/null).
---
  src/json.c | 56
+++++++++++++++++++++++++++++++++++++++++++++++++++++-
  src/json.h |  4 ++++

  2 files changed, 59 insertions(+), 1 deletion(-)
src/json.c +0 -0
index 6c88ed3a..cf233e56 100644
@@ -88,7 +88,17 @@ static void iter_recurse(struct json_iter *iter,
 jsmntok_t *token,


        child->contents = c;
        child->start = token - c->tokens;
+       child->current = child->start;
        child->count = count_tokens_in_container(iter, token); +
+       /*
+        * Add one to include the object/array token itself. This is required
+        * since 'current' points to the container initially. Only after a call

Quoted 8 lines

+        * to json_iter_next() will 'current' point to the first token in the
+        * container.
+        */
+       if (token->type == JSMN_OBJECT || token->type == JSMN_ARRAY)
+               child->count++;
  }

This seems a bit dangerous since count is set based on ITER_END in patch 1.  Is
there a chance this ends up allowing going past the allocated tokens?

I'm guessing you are trying to account for a slight difference in behavior of
json_iter depending on what type it points to:
- if json_iter points to JSMN_PRIMITIVE,
then json_iter_next should not be called (and should fail)
- if json_iter points to JSMN_OBJECT/ARRAY, then json_iter_next has to be called
prior to obtaining the first element

Quoted 1 line

Am I right so far?

Yes that is exactly it. Consider this array:

["foo", "bar"]

count_tokens_in_container() will return 2 tokens. count++ gives us 3 and 'start' points to the array itself.

Call json_iter_next():
'start' is advanced by 1, pointing to "foo"

Call json_iter_next():
'start' is advanced by 1, pointing to "bar"

Call json_iter_next():

	'start' is advanced by 1, pointing to some token after "bar".

	We then check:

	if (c->tokens + iter->current + inc >= ITER_END(iter))
		return false;

	And in this case iter->current + inc will equal ITER_END

I do see your point, and accessing a token at [count] would indeed be bad. But that never actually happens, we bail out once we see current equal count.

Quoted 40 lines
Quoted 4 lines


  struct json_contents *json_contents_new(const char *json, size_t json_len)

<snip>

Quoted 33 lines
src/json.c +0 -0
@@ -370,3 +382,45 @@ bool json_iter_get_null(struct json_iter
 *iter)


        return false;
  }

src/json.c +17 -0
+
+enum json_type json_iter_get_type(struct json_iter *iter)
+{
+       struct json_contents *c = iter->contents;
+       jsmntok_t *t = c->tokens + iter->current;
+
+       return (enum json_type) t->type;
+}
+
+bool json_iter_next(struct json_iter *iter)
+{
+       struct json_contents *c = iter->contents;
+       jsmntok_t *t = c->tokens + iter->current;
+       int inc = 1;
+
+       /*
+        * If this is the initial iteration skip this and just

increment
+        * current by 1 since this iterator points to a container which needs to
+        * be advanced to the first token..
+        *
+        * In addition primitive types and empty containers will have a size
+        * of 1, so no special handling is needed there. +        *
+        * For non-empty containers 'current' needs to be advanced by all the

Quoted 8 lines
Quoted 1 line

+        * containers child tokens, plus the container itself.

One idea might be to use the 'size' variable of the start token to know whether
the iterator can be advanced or not.  Alternatively, look up the base type of
the iterator at the start token and modify the behavior here accordingly.

Are you talking about preventing primitive iteration? Or just altertering how iteration is done in general?

The check below ensures the token is an iterable container, and size is used in that check if only to detect empty objects/arrays, which would then be treated exactly like a primitive/string.

I'm not sure 'size' can be trusted soley to know if its an iterable object. For example a key's size is 1. I'm not sure how an iterator could end up pointing there, but checking OBJECT/ARRAY explicitly seems like the safest bet to me.

Quoted 23 lines
Quoted 12 lines

+        *
+        * This check ensures:
+        *    1. It is not the initial iteration
+        *    2. This is a container
+        *    3. The container is not empty
+        */
+       if (iter->current != iter->start && ((t->type == JSMN_OBJECT ||
+                                       t->type == JSMN_ARRAY) && t->size))
+               inc = count_tokens_in_container(iter, t) + 1;

src/json.c bool json_iter_get_null(struct json_iter *iter) · line 426

Quoted 8 lines
src/json.c +8 -0
+
+       if (c->tokens + iter->current + inc >= ITER_END(iter))
+               return false;
+
+       iter->current += inc;
+
+       return true;
+}

Regards,
-Denis

#7 lore raw

Re: [PATCH v2 4/9] json: add support for array iteration

Hi James,

Quoted 9 lines
Quoted 1 line

Am I right so far?

Yes that is exactly it. Consider this array:

["foo", "bar"]

count_tokens_in_container() will return 2 tokens. count++ gives us 3 and 'start' points to the array itself.

Ok, I think I managed to convince myself this is indeed okay, particularly since it is consistent with json_iter_init() behavior.

One thing you may to do is add a check to json_iter_parse to make sure it is operating on iterators of type JSMN_OBJECT and nothing else. That function's use of ITER_END is what made me suspicious.

Quoted 6 lines

Call json_iter_next():
'start' is advanced by 1, pointing to "foo"

Call json_iter_next():
'start' is advanced by 1, pointing to "bar"

So in theory, you could use 'size' as a limit counter. So for primitives, set pos to 1 and have iter_next check whether pos + 1 <= size.

For arrays/objects, set size to 0 and don't allow getters to work if pos is 0. Or something to that effect. Not sure if this makes things easier or not.

Quoted 10 lines

Call json_iter_next():

	'start' is advanced by 1, pointing to some token after "bar".

	We then check:

	if (c->tokens + iter->current + inc >= ITER_END(iter))
		return false;

	And in this case iter->current + inc will equal ITER_END

Yes, this all makes sense. Why do we increment by count_tokens_in_container() + 1 though?

<snip>

Quoted 9 lines
Quoted 6 lines

One idea might be to use the 'size' variable of the start token to know whether
the iterator can be advanced or not.  Alternatively, look up the base type of
the iterator at the start token and modify the behavior here accordingly.

Are you talking about preventing primitive iteration? Or just altertering how iteration is done in general?

I think both.

Quoted 9 lines

The check below ensures the token is an iterable container, and size is used in that check if only to detect empty objects/arrays, which would then be treated exactly like a primitive/string.

I'm not sure 'size' can be trusted soley to know if its an iterable object. For example a key's size is 1. I'm not sure how an iterator could end up pointing there, but checking OBJECT/ARRAY explicitly seems like the safest bet to me.

I would assume (hope?) that size can be trusted. I could be wrong, but seems to me like it would make some checks easier.

Are empty objects/arrays really of size 1? How does one distinguish between an array with 1 element and an empty array?

Regards,
-Denis

#8 lore raw

Re: [PATCH v2 4/9] json: add support for array iteration

Hi James,

Quoted 2 lines

Yes, this all makes sense.  Why do we increment by count_tokens_in_container() + 1 though?

Ah nevermind, I gotcha.

Anyhow, I applied the rest of this series.

Regards,
-Denis

author
#9 lore raw

PATCH v2 5/9 json: increase default token size to 60

In reply to James Prestwood
---
 src/json.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
src/json.c +1 -1
index cf233e56..aec5fe5c 100644
@@ -33,7 +33,7 @@
 #include "shared/jsmn.h"
 
 /* Max number of tokens supported. Increase if larger objects are expected */
-#define JSON_DEFAULT_TOKENS 30
+#define JSON_DEFAULT_TOKENS 60
 
 #define TOK_LEN(token) (token)->end - (token)->start
 #define TOK_PTR(json, token) (void *)((json) + (token)->start)
Signature · 1 line
2.31.1
Applied
#10 lore raw

Re: [PATCH v2 5/9] json: increase default token size to 60

Hi James,

Quoted 4 lines · James Prestwood
---
  src/json.c | 2 +-
  1 file changed, 1 insertion(+), 1 deletion(-)

Applied, thanks.

Regards,
-Denis

author
#11 lore raw

PATCH v2 6/9 unit: add test for json arrays

In reply to James Prestwood
---
 unit/test-json.c | 113 ++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 92 insertions(+), 21 deletions(-)
unit/test-json.c +92 -21
index fe8f756f..1a7daaff 100644
@@ -75,26 +75,6 @@ static void test_json_escaped_unicode(const void *data)
 	json_contents_free(c);
 }
 
-/*
- * Tests that unsupported types will not parse
- */
-static void test_json_unsupported_types(const void *data)
-{
-	/*
-	 * Valid JSON objects but currently unsupported types
-	 */
-	char arrays[] = "{\"test\":[1, 2, 3, 4]}";
-
-	struct json_iter iter;
-	struct json_contents *c = json_contents_new(arrays, strlen(arrays));
-
-	json_iter_init(&iter, c);
-	assert(!json_iter_parse(&iter,
-				JSON_MANDATORY("test", JSON_ARRAY, NULL),
-				JSON_UNDEFINED));
-	json_contents_free(c);
-}
-
 /*
  * Basic test string values and nested objects
  */
@@ -325,6 +305,97 @@ static void test_json_primitives(const void *data)
 	json_contents_free(c);
 }
 
+static void test_json_arrays(const void *data)
+{
+	unsigned int ui;
+	int i;
+	bool b;
+	int count;
+	char json[] = "{\"uint_array\":[1, 2, 3, 4, 5, 6],"
+			"\"int_array\":[-1, -2, -3, -4, -5, -6],"
+			"\"bool_array\":[true, false, true, false],"
+			"\"null_array\":[null, null, null, null],"
+			"\"mixed_array\":[1, -1, true, false, null]}";
+
+	struct json_iter iter;
+	struct json_iter i_array, ui_array, b_array, n_array, m_array;
+	struct json_contents *c = json_contents_new(json, strlen(json));
+
+	json_iter_init(&iter, c);
+	assert(json_iter_parse(&iter,
+			JSON_MANDATORY("mixed_array", JSON_ARRAY, &m_array),
+			JSON_MANDATORY("null_array", JSON_ARRAY, &n_array),
+			JSON_MANDATORY("bool_array", JSON_ARRAY, &b_array),
+			JSON_MANDATORY("int_array", JSON_ARRAY, &i_array),
+			JSON_MANDATORY("uint_array", JSON_ARRAY, &ui_array),
+			JSON_UNDEFINED));
+
+	count = 1;
+
+	while (json_iter_next(&ui_array)) {
+		assert(json_iter_get_type(&ui_array) == JSON_PRIMITIVE);
+		assert(json_iter_get_uint(&ui_array, &ui));
+		assert(ui == (unsigned int) count);
+		count++;
+	}
+
+	count = -1;
+
+	while (json_iter_next(&i_array)) {
+		assert(json_iter_get_type(&i_array) == JSON_PRIMITIVE);
+		assert(json_iter_get_int(&i_array, &i));
+		assert(i == count);
+		count--;
+	}
+
+	count = 0;
+
+	while (json_iter_next(&b_array)) {
+		assert(json_iter_get_type(&b_array) == JSON_PRIMITIVE);
+		assert(json_iter_get_boolean(&b_array, &b));
+		assert(b == count % 2 ? false : true);
+		count++;
+	}
+
+	count = 0;
+
+	while (json_iter_next(&n_array)) {
+		assert(json_iter_get_type(&n_array) == JSON_PRIMITIVE);
+		assert(json_iter_get_null(&n_array));
+		count++;
+	}
+
+	assert(count == 4);
+
+	count = 0;
+
+	while (json_iter_next(&m_array)) {
+		assert(json_iter_get_type(&m_array) == JSON_PRIMITIVE);
+
+		switch (count) {
+		case 0:
+			assert(json_iter_get_uint(&m_array, &ui));
+			assert(ui == 1);
+			break;
+		case 1:
+			assert(json_iter_get_int(&m_array, &i));
+			assert(i == -1);
+			break;
+		case 2:
+		case 3:
+			assert(json_iter_get_boolean(&m_array, &b));
+			assert(b == count % 2 ? false : true);
+			break;
+		case 4:
+			assert(json_iter_get_null(&m_array));
+			break;
+		}
+
+		count++;
+	}
+
+	json_contents_free(c);
+}
 
 int main(int argc, char *argv[])
 {
@@ -333,11 +404,11 @@ int main(int argc, char *argv[])
 	l_test_add("json unicode", test_json_unicode, NULL);
 	l_test_add("json escaped unicode", test_json_escaped_unicode, NULL);
 	l_test_add("json nested objects", test_json, NULL);
-	l_test_add("json unsupported types", test_json_unsupported_types, NULL);
 	l_test_add("json empty objects", test_json_empty_objects, NULL);
 	l_test_add("json parse out of order", test_json_out_of_order, NULL);
 	l_test_add("json larger object", test_json_larger_object, NULL);
 	l_test_add("json test primitives", test_json_primitives, NULL);
+	l_test_add("json test arrays", test_json_arrays, NULL);
 
 	return l_test_run();
 }
Signature · 1 line
2.31.1
author
#12 lore raw

PATCH v2 7/9 json: add json_iter_get_container

In reply to James Prestwood

This allows getting an iterator for a container nested inside an array.

---
 src/json.c | 14 ++++++++++++++
 src/json.h |  2 ++
 2 files changed, 16 insertions(+)
src/json.c +14 -0
index aec5fe5c..6e40629d 100644
@@ -383,6 +383,20 @@ bool json_iter_get_null(struct json_iter *iter)
 	return false;
 }
 
+bool json_iter_get_container(struct json_iter *iter,
+				struct json_iter *container)
+{
+	struct json_contents *c = iter->contents;
+	jsmntok_t *t = c->tokens + iter->current;
+
+	if (t->type != JSMN_OBJECT && t->type != JSMN_ARRAY)
+		return false;
+
+	iter_recurse(iter, t, container);
+
+	return true;
+}
+
 enum json_type json_iter_get_type(struct json_iter *iter)
 {
 	struct json_contents *c = iter->contents;
src/json.h +2 -0
index 458deb93..518d778c 100644
@@ -93,6 +93,8 @@ bool json_iter_get_int(struct json_iter *iter, int *i);
 bool json_iter_get_uint(struct json_iter *iter, unsigned int *i);
 bool json_iter_get_boolean(struct json_iter *iter, bool *b);
 bool json_iter_get_null(struct json_iter *iter);
+bool json_iter_get_container(struct json_iter *iter,
+				struct json_iter *container);
 
 enum json_type json_iter_get_type(struct json_iter *iter);
 bool json_iter_next(struct json_iter *iter);
Signature · 1 line
2.31.1
author
#13 lore raw

PATCH v2 8/9 unit: add arrays of objects test

In reply to James Prestwood
---
 unit/test-json.c | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)
unit/test-json.c +28 -1
index 1a7daaff..52e3f675 100644
@@ -315,10 +315,12 @@ static void test_json_arrays(const void *data)
 			"\"int_array\":[-1, -2, -3, -4, -5, -6],"
 			"\"bool_array\":[true, false, true, false],"
 			"\"null_array\":[null, null, null, null],"
+			"\"obj_array\":[{}, {\"key\":\"value\", \"key2\":\"value2\"}],"
 			"\"mixed_array\":[1, -1, true, false, null]}";
 
 	struct json_iter iter;
-	struct json_iter i_array, ui_array, b_array, n_array, m_array;
+	struct json_iter i_array, ui_array, b_array, n_array,
+				m_array, obj_array;
 	struct json_contents *c = json_contents_new(json, strlen(json));
 
 	json_iter_init(&iter, c);
@@ -328,6 +330,7 @@ static void test_json_arrays(const void *data)
 			JSON_MANDATORY("bool_array", JSON_ARRAY, &b_array),
 			JSON_MANDATORY("int_array", JSON_ARRAY, &i_array),
 			JSON_MANDATORY("uint_array", JSON_ARRAY, &ui_array),
+			JSON_MANDATORY("obj_array", JSON_ARRAY, &obj_array),
 			JSON_UNDEFINED));
 
 	count = 1;
@@ -394,6 +397,30 @@ static void test_json_arrays(const void *data)
 		count++;
 	}
 
+	count = 0;
+
+	while (json_iter_next(&obj_array)) {
+		struct json_iter object;
+
+		assert(json_iter_get_type(&obj_array) == JSON_OBJECT);
+		assert(json_iter_get_container(&obj_array, &object));
+
+		switch (count) {
+		case 0:
+			assert(json_iter_parse(&object, JSON_UNDEFINED));
+			break;
+		case 1:
+			assert(json_iter_parse(&object,
+				JSON_MANDATORY("key", JSON_STRING, NULL),
+				JSON_UNDEFINED));
+			break;
+		}
+
+		count++;
+	}
+
+	assert(count == 2);
+
 	json_contents_free(c);
 }
 
Signature · 1 line
2.31.1
author
#14 lore raw

PATCH v2 9/9 unit: add test for nested arrays

In reply to James Prestwood
---
 unit/test-json.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)
unit/test-json.c +46 -0
index 52e3f675..3d7e690b 100644
@@ -424,6 +424,51 @@ static void test_json_arrays(const void *data)
 	json_contents_free(c);
 }
 
+static void test_json_nested_arrays(const void *data)
+{
+	char json[] = "{\"array\":[[], {}, [1, 2], {\"key\":\"value\"}]}";
+	int count = 0;
+	struct json_iter iter;
+	struct json_iter array;
+	struct json_iter inner;
+	struct json_contents *c = json_contents_new(json, strlen(json));
+
+	json_iter_init(&iter, c);
+	assert(json_iter_parse(&iter,
+			JSON_MANDATORY("array", JSON_ARRAY, &array),
+			JSON_UNDEFINED));
+
+	while (json_iter_next(&array)) {
+		int count2 = 0;
+
+		assert(json_iter_get_container(&array, &inner));
+
+		while (json_iter_next(&inner))
+			count2++;
+
+		/*
+		 * TODO: add checks for object iteration. Currently these will
+		 * just count all tokens in the object.
+		 */
+		switch (count) {
+		case 0:
+			assert(count2 == 0);
+			break;
+		case 2:
+			assert(count2 == 2);
+			break;
+		default:
+			break;
+		}
+
+		count++;
+	}
+
+	assert(count == 4);
+
+	json_contents_free(c);
+}
+
 int main(int argc, char *argv[])
 {
 	l_test_init(&argc, &argv);
@@ -436,6 +481,7 @@ int main(int argc, char *argv[])
 	l_test_add("json larger object", test_json_larger_object, NULL);
 	l_test_add("json test primitives", test_json_primitives, NULL);
 	l_test_add("json test arrays", test_json_arrays, NULL);
+	l_test_add("json test nested arrays", test_json_nested_arrays, NULL);
 
 	return l_test_run();
 }
Signature · 1 line
2.31.1