shared: add jsmn.h header for JSON parser
James Prestwood
PATCH v2 1/3 shared: add jsmn.h header for JSON parser
--- shared/jsmn.h | 471 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 471 insertions(+) create mode 100644 shared/jsmn.h
shared/jsmn.h new file +471 -0
@@ -0,0 +1,471 @@ +/* +* MIT License +* +* Copyright (c) 2010 Serge Zaitsev +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +* copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +* SOFTWARE. +*/ +#ifndef JSMN_H +#define JSMN_H + +#include <stddef.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef JSMN_STATIC +#define JSMN_API static +#else +#define JSMN_API extern +#endif + +/** +* JSON type identifier. Basic types are: +* o Object +* o Array +* o String +* o Other primitive: number, boolean (true/false) or null +*/ +typedef enum { +JSMN_UNDEFINED =0 , +JSMN_OBJECT =1 <<0 , +JSMN_ARRAY =1 <<1 , +JSMN_STRING =1 <<2 , +JSMN_PRIMITIVE =1 <<3 +}jsmntype_t ; + +enum jsmnerr { +/* Not enough tokens were provided */ +JSMN_ERROR_NOMEM =-1 , +/* Invalid character inside JSON string */ +JSMN_ERROR_INVAL =-2 , +/* The string is not a full JSON packet, more bytes expected */ +JSMN_ERROR_PART =-3 +}; + +/** +* JSON token description. +* type type (object, array, string etc.) +* start start position in JSON data string +* end end position in JSON data string +*/ +typedef struct jsmntok { +jsmntype_t type; +int start; +int end; +int size; +#ifdef JSMN_PARENT_LINKS +int parent; +#endif +}jsmntok_t ; + +/** +* JSON parser. Contains an array of token blocks available. Also stores +* the string being parsed now and current position in that string. +*/ +typedef struct jsmn_parser { +unsigned int pos;/* offset in the JSON string */ +unsigned int toknext;/* next token to allocate */ +int toksuper;/* superior token node, e.g. parent object or array */ +}jsmn_parser ; + +/** +* Create JSON parser over an array of tokens +*/ +JSMN_API void jsmn_init(jsmn_parser *parser); + +/** +* Run JSON parser. It parses a JSON data string into and array of tokens, each +* describing +* a single JSON object. +*/ +JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, const size_t len, +jsmntok_t *tokens,const unsigned int num_tokens); + +#ifndef JSMN_HEADER +/** +* Allocates a fresh unused token from the token pool. +*/ +static jsmntok_t *jsmn_alloc_token (jsmn_parser *parser,jsmntok_t *tokens, +const size_t num_tokens) { +jsmntok_t *tok; +if (parser->toknext >= num_tokens) { +return NULL ; + } + tok = &tokens[parser->toknext++]; + tok->start = tok->end =-1 ; + tok->size =0 ; +#ifdef JSMN_PARENT_LINKS + tok->parent =-1 ; +#endif +return tok; +} + +/** +* Fills token type and boundaries. +*/ +static void jsmn_fill_token (jsmntok_t *token,const jsmntype_t type, +const int start,const int end) { + token->type = type; + token->start = start; + token->end = end; + token->size =0 ; +} + +/** +* Fills next available token with JSON primitive. +*/ +static int jsmn_parse_primitive (jsmn_parser *parser,const char *js, +const size_t len,jsmntok_t *tokens, +const size_t num_tokens) { +jsmntok_t *token; +int start; + + start = parser->pos; + +for (; parser->pos < len && js[parser->pos] !='\0' ; parser->pos++) { +switch (js[parser->pos]) { +#ifndef JSMN_STRICT +/* In strict mode primitive must be followed by "," or "}" or "]" */ +case ':' : +#endif +case '\t' : +case '\r' : +case '\n' : +case ' ' : +case ',' : +case ']' : +case '}' : + goto found; +default : +/* to quiet a warning from gcc*/ +break ; + } +if (js[parser->pos] <32 || js[parser->pos] >=127 ) { + parser->pos = start; +return JSMN_ERROR_INVAL ; + } + } +#ifdef JSMN_STRICT +/* In strict mode primitive must be followed by a comma/object/array */ + parser->pos = start; +return JSMN_ERROR_PART ; +#endif + +found: +if (tokens ==NULL ) { + parser->pos--; +return 0 ; + } + token =jsmn_alloc_token (parser, tokens, num_tokens); +if (token ==NULL ) { + parser->pos = start; +return JSMN_ERROR_NOMEM ; + } +jsmn_fill_token (token,JSMN_PRIMITIVE , start, parser->pos); +#ifdef JSMN_PARENT_LINKS + token->parent = parser->toksuper; +#endif + parser->pos--; +return 0 ; +} + +/** +* Fills next token with JSON string. +*/ +static int jsmn_parse_string (jsmn_parser *parser,const char *js, +const size_t len,jsmntok_t *tokens, +const size_t num_tokens) { +jsmntok_t *token; + +int start = parser->pos; + +/* Skip starting quote */ + parser->pos++; + +for (; parser->pos < len && js[parser->pos] !='\0' ; parser->pos++) { +char c = js[parser->pos]; + +/* Quote: end of string */ +if (c =='\"' ) { +if (tokens ==NULL ) { +return 0 ; + } + token =jsmn_alloc_token (parser, tokens, num_tokens); +if (token ==NULL ) { + parser->pos = start; +return JSMN_ERROR_NOMEM ; + } +jsmn_fill_token (token,JSMN_STRING , start +1 , parser->pos); +#ifdef JSMN_PARENT_LINKS + token->parent = parser->toksuper; +#endif +return 0 ; + } + +/* Backslash: Quoted symbol expected */ +if (c =='\\' && parser->pos +1 < len) { +int i; + parser->pos++; +switch (js[parser->pos]) { +/* Allowed escaped symbols */ +case '\"' : +case '/' : +case '\\' : +case 'b' : +case 'f' : +case 'r' : +case 'n' : +case 't' : +break ; +/* Allows escaped symbol \uXXXX */ +case 'u' : + parser->pos++; +for (i =0 ; i <4 && parser->pos < len && js[parser->pos] !='\0' ; + i++) { +/* If it isn't a hex character we have an error */ +if (!((js[parser->pos] >=48 && js[parser->pos] <=57 ) ||/* 0-9 */ + (js[parser->pos] >=65 && js[parser->pos] <=70 ) ||/* A-F */ + (js[parser->pos] >=97 && js[parser->pos] <=102 ))) {/* a-f */ + parser->pos = start; +return JSMN_ERROR_INVAL ; + } + parser->pos++; + } + parser->pos--; +break ; +/* Unexpected symbol */ +default : + parser->pos = start; +return JSMN_ERROR_INVAL ; + } + } + } + parser->pos = start; +return JSMN_ERROR_PART ; +} + +/** +* Parse JSON string and fill tokens. +*/ +JSMN_API int jsmn_parse(jsmn_parser *parser, const char *js, const size_t len, +jsmntok_t *tokens,const unsigned int num_tokens) { +int r; +int i; +jsmntok_t *token; +int count = parser->toknext; + +for (; parser->pos < len && js[parser->pos] !='\0' ; parser->pos++) { +char c; +jsmntype_t type; + + c = js[parser->pos]; +switch (c) { +case '{' : +case '[' : + count++; +if (tokens ==NULL ) { +break ; + } + token =jsmn_alloc_token (parser, tokens, num_tokens); +if (token ==NULL ) { +return JSMN_ERROR_NOMEM ; + } +if (parser->toksuper !=-1 ) { +jsmntok_t *t = &tokens[parser->toksuper]; +#ifdef JSMN_STRICT +/* In strict mode an object or array can't become a key */ +if (t->type ==JSMN_OBJECT ) { +return JSMN_ERROR_INVAL ; + } +#endif + t->size++; +#ifdef JSMN_PARENT_LINKS + token->parent = parser->toksuper; +#endif + } + token->type = (c =='{' ?JSMN_OBJECT :JSMN_ARRAY ); + token->start = parser->pos; + parser->toksuper = parser->toknext -1 ; +break ; +case '}' : +case ']' : +if (tokens ==NULL ) { +break ; + } + type = (c =='}' ?JSMN_OBJECT :JSMN_ARRAY ); +#ifdef JSMN_PARENT_LINKS +if (parser->toknext <1 ) { +return JSMN_ERROR_INVAL ; + } + token = &tokens[parser->toknext -1 ]; +for (;;) { +if (token->start !=-1 && token->end ==-1 ) { +if (token->type != type) { +return JSMN_ERROR_INVAL ; + } + token->end = parser->pos +1 ; + parser->toksuper = token->parent; +break ; + } +if (token->parent ==-1 ) { +if (token->type != type || parser->toksuper ==-1 ) { +return JSMN_ERROR_INVAL ; + } +break ; + } + token = &tokens[token->parent]; + } +#else +for (i = parser->toknext -1 ; i >=0 ; i--) { + token = &tokens[i]; +if (token->start !=-1 && token->end ==-1 ) { +if (token->type != type) { +return JSMN_ERROR_INVAL ; + } + parser->toksuper =-1 ; + token->end = parser->pos +1 ; +break ; + } + } +/* Error if unmatched closing bracket */ +if (i ==-1 ) { +return JSMN_ERROR_INVAL ; + } +for (; i >=0 ; i--) { + token = &tokens[i]; +if (token->start !=-1 && token->end ==-1 ) { + parser->toksuper = i; +break ; + } + } +#endif +break ; +case '\"' : + r =jsmn_parse_string (parser, js, len, tokens, num_tokens); +if (r <0 ) { +return r; + } + count++; +if (parser->toksuper !=-1 && tokens !=NULL ) { + tokens[parser->toksuper].size++; + } +break ; +case '\t' : +case '\r' : +case '\n' : +case ' ' : +break ; +case ':' : + parser->toksuper = parser->toknext -1 ; +break ; +case ',' : +if (tokens !=NULL && parser->toksuper !=-1 && + tokens[parser->toksuper].type !=JSMN_ARRAY && + tokens[parser->toksuper].type !=JSMN_OBJECT ) { +#ifdef JSMN_PARENT_LINKS + parser->toksuper = tokens[parser->toksuper].parent; +#else +for (i = parser->toknext -1 ; i >=0 ; i--) { +if (tokens[i].type ==JSMN_ARRAY || tokens[i].type ==JSMN_OBJECT ) { +if (tokens[i].start !=-1 && tokens[i].end ==-1 ) { + parser->toksuper = i; +break ; + } + } + } +#endif + } +break ; +#ifdef JSMN_STRICT +/* In strict mode primitives are: numbers and booleans */ +case '-' : +case '0' : +case '1' : +case '2' : +case '3' : +case '4' : +case '5' : +case '6' : +case '7' : +case '8' : +case '9' : +case 't' : +case 'f' : +case 'n' : +/* And they must not be keys of the object */ +if (tokens !=NULL && parser->toksuper !=-1 ) { +const jsmntok_t *t = &tokens[parser->toksuper]; +if (t->type ==JSMN_OBJECT || + (t->type ==JSMN_STRING && t->size !=0 )) { +return JSMN_ERROR_INVAL ; + } + } +#else +/* In non-strict mode every unquoted value is a primitive */ +default : +#endif + r =jsmn_parse_primitive (parser, js, len, tokens, num_tokens); +if (r <0 ) { +return r; + } + count++; +if (parser->toksuper !=-1 && tokens !=NULL ) { + tokens[parser->toksuper].size++; + } +break ; + +#ifdef JSMN_STRICT +/* Unexpected char in strict mode */ +default : +return JSMN_ERROR_INVAL ; +#endif + } + } + +if (tokens !=NULL ) { +for (i = parser->toknext -1 ; i >=0 ; i--) { +/* Unmatched opened object or array */ +if (tokens[i].start !=-1 && tokens[i].end ==-1 ) { +return JSMN_ERROR_PART ; + } + } + } + +return count; +} + +/** +* Creates a new parser based over a given buffer with an array of tokens +* available. +*/ +JSMN_API void jsmn_init(jsmn_parser *parser) { + parser->pos =0 ; + parser->toknext =0 ; + parser->toksuper =-1 ; +} + +#endif /* JSMN_HEADER */ + +#ifdef __cplusplus +} +#endif + +#endif /* JSMN_H */
Signature · 1 line
2.31.1
James Prestwood
PATCH v2 2/3 json: introduce JSON module
This is a minimal wrapper around jsmn.h to make things a bit easier for iterating through a JSON object.
An iterator can be created/destroyed with json_iter_{new,free} and values can be parsed from this iterator using json_iter_parse.
Arguments to json_iter_parse should use JSON_MANDATORY/JSON_OPTIONAL macros, and terminate the argument list with JSON_FLAG_INVALID.
The value pointer in the MANDATORY/OPTIONAL macros can be NULL, in which case the presence of the key/value is checked but no data is returned out.
Currently only JSON_STRING and JSON_OBJECT types are supported. String values should be of type struct iovec *, and objects should be of type struct json_iter **. These nested objects can be parsed again using json_iter_parse and should not be freed (the original iterator keeps track of nested object iterators).
--- Makefile.am | 1 + src/json.c | 222 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/json.h | 84 ++++++++++++++++++++ 3 files changed, 307 insertions(+) create mode 100644 src/json.c create mode 100644 src/json.h
v2:
* Re-wrote to remove callback style parsing. Instead the expected values are provided to the parser directly.
Makefile.am +1 -0
@@ -249,6 +249,7 @@ src_iwd_SOURCES = src/main.c linux/nl80211.h src/iwd.h src/missing.h \ src/sysfs.h src/sysfs.c \ src/offchannel.h src/offchannel.c \ src/dpp-util.h src/dpp-util.c \ + src/json.h src/json.c \ $(eap_sources) \ $(builtin_sources)
src/json.c new file +222 -0
222 lines not on this page · raw message
src/json.h new file +84 -0
84 lines not on this page · raw message
Signature · 1 line
2.31.1
Denis Kenzior
Re: [PATCH v2 2/3] json: introduce JSON module
Hi James,
Quoted 6 lines · James Prestwood
This is a minimal wrapper around
jsmn.hto make things a bit easier for iterating through a JSON object.An iterator can be created/destroyed with json_iter_{new,free} and values can be parsed from this iterator using
json_iter_parse.
Have you considered using an intermediate object to hold the bits that need to be opaque? That way json_iter objects could be public and held on the stack. Would make things easier to use / simpler to implement.
So roughly:
struct json_contents;
struct json_iter {
struct json_contents *contents;
int begin;
int end;
};also you could possibly track parent objects if needed.
Quoted 9 lines
Arguments to
json_iter_parseshould use JSON_MANDATORY/JSON_OPTIONAL macros, and terminate the argument list withJSON_FLAG_INVALID.The value pointer in the MANDATORY/OPTIONAL macros can be NULL, in which case the presence of the key/value is checked but no data is returned out.
Currently only
JSON_STRINGandJSON_OBJECTtypes are supported. String values should be of type struct iovec *, and objects should be of
Ok, I wonder if we could allocate the strings instead of having the caller do this.
Quoted 3 lines
type struct
json_iter**. These nested objects can be parsed again usingjson_iter_parseand should not be freed (the original iterator keeps track of nested object iterators).
It would be nice if these were simply on the stack to cut down the queue management.
Quoted 30 lines
--- Makefile.am | 1 + src/json.c | 222 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/json.h | 84 ++++++++++++++++++++ 3 files changed, 307 insertions(+)create mode 100644
src/json.c
create mode 100644src/json.hv2:
* Re-wrote to remove callback style parsing. Instead the expected values are provided to the parser directly.Makefile.am +1
-0@@ -249,6 +249,7 @@ src_iwd_SOURCES = src/main.c linux/nl80211.h src/iwd.h src/missing.h \ src/sysfs.h src/sysfs.c \ src/offchannel.h src/offchannel.c \ src/dpp-util.h src/dpp-util.c \ + src/json.h src/json.c \ $(eap_sources) \ $(builtin_sources)src/json.c new file +0
-0
src/json.c line 5
Quoted 5 lines
src/json.c new file +5
-0@@ -0,0 +1,222 @@ +/* +* +* Wireless daemon for Linux +* +* Copyright (C) 2013-2019 Intel Corporation. All rights reserved.
Please update this
Quoted 50 lines
src/json.c new file +50
-0+* +* This library is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* This library is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with this library; if not, write to the Free Software +* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +* +*/ + +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif + +#include <errno.h> + +#include <ell/ell.h> + +#include "src/json.h" + +#include "shared/jsmn.h" + +/* Max number of tokens supported. Increase if larger objects are expected */ +#define JSON_DEFAULT_TOKENS 30 + +#define TOK_LEN (token) (token)->end - (token)->start +#define TOK_PTR (json, token) (void *)((json) + (token)->start) + +struct json_iter { +const char *json; +size_t json_len; +jsmntok_t *tokens; +int tokens_len; +jsmn_parser *p; +struct l_queue *children; +bool root :1 ; +}; + +static struct json_iter *iter_recurse (struct json_iter *parent, +jsmntok_t *object) +{ + struct json_iter *iter = l_new(struct json_iter, 1); +
src/json.c line 65
Quoted 10 lines
src/json.c new file +10
-0+ iter->json = parent->json; + iter->json_len = parent->json_len; + iter->tokens = object; + iter->children =l_queue_new (); + +/* +* Since object->size only indicates the number of keys in an object +* we cannot easily calculate the number of tokens remaining (for this +* object) since each object may have nested objects. Set tokens_len to +* the number of tokens remaining in the overall list after 'object'
This seems brittle. Can we recursively count the number of tokens for each child?
Quoted 6 lines
src/json.c new file +6
-0+*/ + iter->tokens_len = parent->tokens_len - + ((object - parent->tokens) %sizeof (jsmntok_t)); + +l_queue_push_head (parent->children, iter); +
src/json.c line 81
Quoted 10 lines
src/json.c new file +10
-0+return iter; +} + +struct json_iter *json_iter_new (const char *json,size_t json_len) +{ + struct json_iter *iter = l_new(struct json_iter, 1); + + iter->json = json; + iter->json_len = json_len; + iter->p =l_new (jsmn_parser,1 );
Do you need jsmn_init here?
Quoted 21 lines
src/json.c new file +21
-0+ iter->tokens =l_new (jsmntok_t,JSON_DEFAULT_TOKENS ); + iter->root = true; + +jsmn_init (iter->p); + iter->tokens_len =jsmn_parse (iter->p, iter->json, iter->json_len, + iter->tokens,JSON_DEFAULT_TOKENS ); +if (iter->tokens_len <0 ) { +json_iter_free (iter); +return NULL ; + } + + iter->children =l_queue_new (); + +return iter; +} + +static void json_iter_destroy (void *data) +{ +struct json_iter *iter = data; + +json_iter_free (iter);
src/json.c line 112
Quoted 10 lines
src/json.c new file +10
-0+} + +void json_iter_free (struct json_iter *iter) +{ +if (iter->root) { +l_free (iter->p); +l_free (iter->tokens); + } + +l_queue_destroy (iter->children, json_iter_destroy);
You could do:
l_queue_destroy(iter->children,
(l_queue_destroy_func_t *) json_iter_free);
src/json.c line 122
Quoted 10 lines
src/json.c new file +10
-0+l_free (iter); +} + +static jsmntok_t *next_key (struct json_iter *iter,jsmntok_t *current) +{ +jsmntok_t *value; +int i; + +/* +* In theory JSMN should prevent any of these bounds checks from failing
Not very reassuring :)
src/json.c line 130
Quoted 8 lines
src/json.c new file +8
-0+*/ +if (current +1 > iter->tokens + iter->tokens_len) +return NULL ; + + value = current +1 ; + +if (value->size ==0 ) +return value +1 ;
So value->size == 0 implies a basic type?
src/json.c line 132
Quoted 2 lines
src/json.c new file +2
-0+ > + for (i = 0; i < value->size; i++) + value =next_key (iter, value);
So this should be recursively skipping tokens when value is non-basic. The naming is a bit confusing here
src/json.c line 134
Quoted 2 lines
src/json.c new file +2
-0+ +return value;
Since you're returning 'value', but the reader expects a 'key'.
Quoted 40 lines
src/json.c new file +40
-0+} + +/* +* Checks that the keys value (t + 1) is in bounds, the value type matches, +* key length matches, and key string match +*/ +#define TOKEN_MATCHES (iter, t, key, type) \ + (t) + 1 < iter->tokens + iter->tokens_len && \ + (jsmntype_t)(type) == ((t) + 1)->type && \ + TOK_LEN((t)) == (int)strlen((key)) && \ + !memcmp(TOK_PTR((iter)->json, (t)), (key), TOK_LEN((t))) + +bool json_iter_parse (struct json_iter *iter, ...) +{ +va_list va; +int i; +int num = iter->tokens->size; +jsmntok_t *next; +struct iovec *sval; +struct json_iter **oval; + +/* Empty object {} */ +if (iter->tokens_len <1 ) +return false; + +va_start (va, iter); + +while (true) { +enum json_flag flag; +char *key; +enum json_type type; +void *value; +jsmntok_t *v =NULL ; + + flag = va_arg(va, enum json_flag); + +if (flag ==JSON_FLAG_INVALID ) +break ; + + key = va_arg(va, char *);
src/json.c line 185
Quoted 10 lines
src/json.c new file +10
-0+ type = va_arg(va, enum json_type); + value = va_arg(va, void *); + + next = iter->tokens +1 ; + +/* Iterate over this objects keys */ +for (i =0 ; i < num; i++) { +if (TOKEN_MATCHES (iter, next, key, type)) { + v = next +1 ; +break ;
What if the key matches but the type doesn't? Shouldn't we bail early?
src/json.c line 190
Quoted 5 lines
src/json.c new file +5
-0+ } + + next =next_key (iter, next); +if (!next) +return false;
May not be as big of a deal in this particular case, but this approach is side-effecting on failure. You could parse the object part of the way through, then fail to find a key and bail, but you may have assigned stuff along the way.
Quoted 112 lines
src/json.c new file +32
-0+ } + +if (flag ==JSON_FLAG_MANDATORY && !v) +return false; + +switch (type) { +case JSON_STRING : +if (!value) +continue ; + + sval = value; + sval->iov_base = v ?TOK_PTR (iter->json, v) :NULL ; + sval->iov_len = v ?TOK_LEN (v) :0 ; + +break ; +case JSON_OBJECT : +if (!value) +continue ; + + oval = value; + *oval = v ?iter_recurse (iter, v) :NULL ; + +break ; +default : +return false; + } + } + +va_end (va); + +return true; +}src/json.h new file +74
-0@@ -0,0 +1,84 @@ +/* +* +* Wireless daemon for Linux +* +* Copyright (C) 2021 Intel Corporation. All rights reserved. +* +* This library is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version. +* +* This library is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with this library; if not, write to the Free Software +* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +* +*/ + +struct json_iter ; + +/* +* Identical to JSMN types +*/ +enum json_type { +JSON_UNDEFINED =0 , +JSON_OBJECT =1 <<0 , +JSON_ARRAY =1 <<1 , +JSON_STRING =1 <<2 , +JSON_PRIMITIVE =1 <<3 +}; + +enum json_flag { +JSON_FLAG_INVALID =0 , +JSON_FLAG_MANDATORY =1 , +JSON_FLAG_OPTIONAL =2 , +}; + +#define JSON_MANDATORY (key, type, out) \ + JSON_FLAG_MANDATORY, (key), (type), (out) + +#define JSON_OPTIONAL (key, type, out) \ + JSON_FLAG_OPTIONAL, (key), (type), (out) + +#define JSON_STR_EQ (iov, s) \ + ((iov)->iov_len == strlen((s)) && \ + !memcmp((iov)->iov_base, (s), (iov)->iov_len)) + +#define JSON_TO_STR (iov) \ +({ \ + char *_tmp = l_malloc((iov)->iov_len + 1); \ + memcpy(_tmp, (iov)->iov_base, (iov)->iov_len); \ + _tmp[(iov)->iov_len] = '\0'; \ + _tmp; \ +}) + +struct json_iter *json_iter_new (const char *json,size_t json_len); +void json_iter_free (struct json_iter *iter); + +/* +* Parse an arbitrary number of key/value pairs from a JSON iterator. Initially +* a new JSON iterator should be created with json_iter_new() and, when done, +* freed with json_iter_free. Nested objects are also parsed with this function +* but these iterators should not be freed and are tracked by the original +* iterator. +* +* Arguments should be specified using JSON_MANDATORY or JSON_OPTIONAL: +* +* r = json_iter_parse(iter, JSON_MANDATORY("mykey", JSON_STRING, &strvalue), +* JSON_OPTIONAL("optkey", JSON_STRING, &optvalue)); +*
src/json.h line 84
Quoted 10 lines
src/json.h new file +10
-0+* String values should be of type struct iovec * +* Object values should be of type struct json_iter ** +* +* No other types are supported at this time, and json_iter_parse will fail if +* other types are encountered. +* +* JSON_OPTIONAL string values will have a NULL/0 iov_base/iov_len if they were +* not found. JSON_OPTIONAL object value iterators will be NULL if not found. +*/ +bool json_iter_parse (struct json_iter *iter, ...);
Just a nit, but it is a bit clearer to start the '...' after either a printf-style string or an argument that could carry the sentintel value. For example:
int nl80211_parse_attrs(struct l_genl_msg *msg, int tag, ...);
It isn't much, but gives just a bit extra context for how the API is supposed to be used. So in the case of json_iter_parse it would be enum json_flag.
Though really, I wonder why json_iter_parse doesn't use the sequence of:
(json_type, key, value, flags) ?
Regards,
-Denis
James Prestwood
Re: [PATCH v2 2/3] json: introduce JSON module
Quoted 18 lines · Denis Kenzior
Hi James,
Quoted 6 lines · James Prestwood
This is a minimal wrapper around
jsmn.hto make things a bit easier for iterating through a JSON object.An iterator can be created/destroyed with json_iter_{new,free} and values can be parsed from this iterator using
json_iter_parse.Have you considered using an intermediate object to hold the bits that need to
be opaque? That wayjson_iterobjects could be public and held on the stack.
Would make things easier to use / simpler to implement.So roughly:
Quoted 7 lines
struct
json_contents;struct
json_iter{
structjson_contents*contents;
int begin;
int end;
};
This didn't feel consistent with any other iterators on cleanup:
struct json_iter iter;
json_iter_init(&iter, json, json_len);
...
json_iter_free(&iter);
Quoted 11 lines
also you could possibly track parent objects if needed.
Quoted 8 lines
Arguments to
json_iter_parseshould use JSON_MANDATORY/JSON_OPTIONAL
macros, and terminate the argument list withJSON_FLAG_INVALID.The value pointer in the MANDATORY/OPTIONAL macros can be NULL, in which case the presence of the key/value is checked but no data is returned out.
Quoted 6 lines
Quoted 3 lines
Currently only
JSON_STRINGandJSON_OBJECTtypes are supported. String
values should be of type struct iovec *, and objects should be ofOk, I wonder if we could allocate the strings instead of having the caller do this.
Sure. Initially this was supposed to be a minimal wrapper and the allocation seemed like too much, but this has evolved :)
Quoted 119 lines
Quoted 5 lines
type struct
json_iter**. These nested objects can be parsed again usingjson_iter_parseand should not be freed (the original iterator
keeps track of nested object iterators).It would be nice if these were simply on the stack to cut down the queue management.
Quoted 40 lines
---
Makefile.am | 1 +
src/json.c | 222
++++++++++++++++++++++++++++++++++++++++++++++++++++
src/json.h | 84 ++++++++++++++++++++3 files changed, 307 insertions(+)create mode 100644
src/json.c
create mode 100644src/json.hv2:
* Re-wrote to remove callback style parsing. Instead the expected values
are provided to the parser directly.Makefile.am +0
-0@@ -249,6 +249,7 @@ src_iwd_SOURCES = src/main.c linux/nl80211.h src/iwd.h src/missing.h \ src/sysfs.h src/sysfs.c \src/offchannel.h
src/offchannel.c\
src/dpp-util.h src/dpp-util.c\
+ src/json.hsrc/json.c\
$(eap_sources) \
$(builtin_sources)src/json.c new file +5
-0@@ -0,0 +1,222 @@ +/* +* +* Wireless daemon for Linux +* +* Copyright (C) 2013-2019 Intel Corporation. All rights Quoted 1 line
reserved.
Please update this
Quoted 66 lines
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ */src/json.c new file +14
-0+ +#ifdef HAVE_CONFIG_H +#include <config.h> +#endif + +#include <errno.h> + +#include <ell/ell.h> + +#include "src/json.h" + +#include "shared/jsmn.h" + +/* Max number of tokens supported. Increase if larger objects are expected */
src/json.c new file +27
-0+#define JSON_DEFAULT_TOKENS 30 + +#define TOK_LEN (token) (token)->end - (token)->start +#define TOK_PTR (json, token) (void *)((json) + (token)->start) + +struct json_iter { + const char *json; + size_t json_len; + jsmntok_t *tokens; + int tokens_len; + jsmn_parser *p; + struct l_queue *children; + bool root : 1; +}; + +static struct json_iter *iter_recurse (struct json_iter *parent, + jsmntok_t *object) +{ + struct json_iter *iter = l_new(struct json_iter, 1); + + iter->json = parent->json; + iter->json_len = parent->json_len; + iter->tokens = object; + iter->children = l_queue_new(); + + /* +* Since object->size only indicates the number of keys in an object
+ * we cannot easily calculate the number of tokens
Quoted 8 lines
Quoted 5 lines
remaining (for this
+ * object) since each object may have nested objects. Settokens_lento
+ * the number of tokens remaining in the overall list after 'object'This seems brittle. Can we recursively count the number of tokens for each child?
So this is really only for bounds checking, which may not actually be requred as that comment below suggests. I think so long as we trust JSMN we can guarantee its parsed tokens will always contain key/value pairs. Still, a maximum bounds probably doesn't hurt. So yes we can limit the bounds of these recursive iterators to only the child object.
Quoted 12 lines
Quoted 12 lines
+ */
+ iter->tokens_len =parent->tokens_len-
+ ((object -parent->tokens) %sizeof(jsmntok_t));src/json.c new file +7
-0+ + l_queue_push_head(parent->children, iter); + + return iter; +} + +struct json_iter *json_iter_new (const char *json,size_t json_len)
Quoted 8 lines
src/json.c line 81
Quoted 6 lines
src/json.c new file +6
-0+{ + struct json_iter *iter = l_new(struct json_iter, 1); + + iter->json = json; + iter->json_len = json_len; + iter->p = l_new(jsmn_parser, 1);Do you need
jsmn_inithere?
Yes, below.
Quoted 39 lines
Quoted 24 lines
+ iter->tokens =
l_new(jsmntok_t, JSON_DEFAULT_TOKENS); + iter->root = true;
+
+ jsmn_init(iter->p);
+ iter->tokens_len = jsmn_parse(iter->p,iter->json, iter-Quoted 1 line
json_len,+ iter->tokens,
JSON_DEFAULT_TOKENS);
+ if (iter->tokens_len< 0) {
+ json_iter_free(iter);
+ return NULL;
+ }src/json.c new file +11
-0+ + iter->children = l_queue_new(); + + return iter; +} + +static void json_iter_destroy (void *data) +{ + struct json_iter *iter = data; + + json_iter_free(iter);src/json.c line 112
Quoted 10 lines
src/json.c new file +10
-0+} + +void json_iter_free (struct json_iter *iter) +{ + if (iter->root) { + l_free(iter->p); + l_free(iter->tokens); + } + + l_queue_destroy(iter->children, json_iter_destroy);You could do:
l_queue_destroy(iter->children, (l_queue_destroy_func_t *) json_iter_free);
Quoted 14 lines
Quoted 11 lines
+ l_free(iter);
src/json.c new file +3
-0+} + +static jsmntok_t *next_key(struct json_iter *iter, jsmntok_t*current)
src/json.c new file +6
-0+{ + jsmntok_t *value; + int i; + + /* +* In theory JSMN should prevent any of these bounds checks Quoted 1 line
from failing
Not very reassuring :)
This was more paranoia than anything.
Quoted 11 lines
src/json.c line 130
Quoted 9 lines
+ */
+ if (current + 1 >iter->tokens+iter->tokens_len) + return NULL;src/json.c new file +5
-0+ + value = current + 1; + + if (value->size == 0) + return value + 1;So
value->size== 0 implies a basic type?
AFAIK any value that is not an object will have a size of zero, so this could include strings as well not just basic types.
Quoted 4 lines
Quoted 3 lines
+ > + for (i = 0; i <
value->size; i++)
+ value =next_key(iter, value);
Quoted 8 lines
So this should be recursively skipping tokens when value is non- basic. The
naming is a bit confusing hereQuoted 2 lines
+
+ return value;Since you're returning 'value', but the reader expects a 'key'.
Yes, some re-naming is in order. This is actually returning a key though, I just reused 'value' rather than a new variable. Something like 'token' or 'tmp' would be better.
Quoted 47 lines
Quoted 47 lines
src/json.c new file +4
-0+} + +/* +* Checks that the keys value (t + 1) is in bounds, the value type matches,
+ * key length matches, and key string match
+ */src/json.c new file +39
-0+#define TOKEN_MATCHES (iter, t, key, type) \ + (t) + 1 < iter->tokens + iter->tokens_len && \ + (jsmntype_t)(type) == ((t) + 1)->type && \ + TOK_LEN((t)) == (int)strlen((key)) && \ + !memcmp(TOK_PTR((iter)->json, (t)), (key), TOK_LEN((t))) + +bool json_iter_parse (struct json_iter *iter, ...) +{ + va_list va; + int i; + int num = iter->tokens->size; + jsmntok_t *next; + struct iovec *sval; + struct json_iter **oval; + + /* Empty object {} */ + if (iter->tokens_len < 1) + return false; + + va_start(va, iter); + + while (true) { + enum json_flag flag; + char *key; + enum json_type type; + void *value; + jsmntok_t *v = NULL; + + flag = va_arg(va, enum json_flag); + + if (flag == JSON_FLAG_INVALID) + break; + + key = va_arg(va, char *); + type = va_arg(va, enum json_type); + value = va_arg(va, void *); + + next = iter->tokens + 1; +
Quoted 8 lines
Quoted 5 lines
+ /* Iterate over this objects keys */
+ for (i = 0; i < num; i++) {
+ if (TOKEN_MATCHES(iter, next, key, type)) { + v = next + 1;
+ break;What if the key matches but the type doesn't? Shouldn't we bail early?
Yep, I'll add some extra checks.
Quoted 7 lines
Quoted 6 lines
+ }
+
+ next =next_key(iter, next); + if (!next)
+ return false;
Quoted 6 lines
May not be as big of a deal in this particular case, but this approach is
side-effecting on failure. You could parse the object part of the way through,
then fail to find a key and bail, but you may have assigned stuff along the way.
Yeah I could not think of a decent way to handle this without iterating twice (verification first, then assignment) since we lose reference to the previous set of arguments with each iteration.
Quoted 154 lines
Quoted 133 lines
+ }
src/json.c new file +11
-0+ + if (flag == JSON_FLAG_MANDATORY && !v) + return false; + + switch (type) { + case JSON_STRING: + if (!value) + continue; + + sval = value; + sval->iov_base = v ? TOK_PTR(iter->json, v): NULL;
+ sval->iov_len = v ?TOK_LEN(v): 0;src/json.c new file +19
-0+ + break; + case JSON_OBJECT: + if (!value) + continue; + + oval = value; + *oval = v ? iter_recurse(iter, v) : NULL; + + break; + default: + return false; + } + } + + va_end(va); + + return true; +}src/json.h new file +10
-0@@ -0,0 +1,84 @@ +/* +* +* Wireless daemon for Linux +* +* Copyright (C) 2021 Intel Corporation. All rights reserved. +* +* This library is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; either +* version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ */src/json.h new file +39
-0+ +struct json_iter ; + +/* +* Identical to JSMN types +*/ +enum json_type { + JSON_UNDEFINED = 0, + JSON_OBJECT = 1 << 0, + JSON_ARRAY = 1 << 1, + JSON_STRING = 1 << 2, + JSON_PRIMITIVE = 1 << 3 +}; + +enum json_flag { + JSON_FLAG_INVALID = 0, + JSON_FLAG_MANDATORY = 1, + JSON_FLAG_OPTIONAL = 2, +}; + +#define JSON_MANDATORY (key, type, out) \ + JSON_FLAG_MANDATORY, (key), (type), (out) + +#define JSON_OPTIONAL (key, type, out) \ + JSON_FLAG_OPTIONAL, (key), (type), (out) + +#define JSON_STR_EQ (iov, s) \ + ((iov)->iov_len == strlen((s)) && \ + !memcmp((iov)->iov_base, (s), (iov)->iov_len)) + +#define JSON_TO_STR (iov) \ +({ \ + char *_tmp = l_malloc((iov)->iov_len + 1); \ + memcpy(_tmp, (iov)->iov_base, (iov)->iov_len); \ + _tmp[(iov)->iov_len] = '\0'; \ + _tmp; \ +}) + +struct json_iter *json_iter_new(const char *json, size_t
json_len);src/json.h new file +4
-0+void json_iter_free (struct json_iter *iter); + +/* +* Parse an arbitrary number of key/value pairs from a JSON iterator. Initially
+ * a new JSON iterator should be created withjson_iter_new()and, when done,
+ * freed withjson_iter_free. Nested objects are also parsed with this function
+ * but these iterators should not be freed and are tracked by the original
+ * iterator.
+ *
+ * Arguments should be specified usingJSON_MANDATORYorJSON_OPTIONAL:
+ *
+ * r = json_iter_parse(iter,JSON_MANDATORY("mykey", JSON_STRING, &strvalue),
+ * JSON_OPTIONAL("optkey",JSON_STRING, &optvalue));
+ *
+ * String values should be of type struct iovec *
+ * Object values should be of type structjson_iter**
+ *
+ * No other types are supported at this time, andjson_iter_parsewill fail ifQuoted 8 lines
+ * other types are encountered.
+ *
+ *JSON_OPTIONALstring values will have a NULL/0 iov_base/iov_len if they were
+ * not found.JSON_OPTIONALobject value iterators will be NULL if not found.
+ */
+booljson_iter_parse(struct json_iter *iter, ...);Just a nit, but it is a bit clearer to start the '...' after either a printf-style string or an argument that could carry the sentintel value. For
example:int
nl80211_parse_attrs(struct l_genl_msg *msg, int tag, ...);It isn't much, but gives just a bit extra context for how the API is supposed to
be used. So in the case ofjson_iter_parseit would be enumjson_flag.
Quoted 4 lines
Though really, I wonder why
json_iter_parsedoesn't use the sequence of:(
json_type, key, value, flags) ?
Yeah this is fine, I can reorder them like this and add a new JSON_INVALID type to signal the end.
Quoted 3 lines
Regards,
-Denis
Denis Kenzior
Re: [PATCH v2 2/3] json: introduce JSON module
Hi James,
Quoted 18 lines
Quoted 8 lines
Have you considered using an intermediate object to hold the bits that need to
be opaque? That wayjson_iterobjects could be public and held on the stack.
Would make things easier to use / simpler to implement.So roughly:
Quoted 7 lines
struct
json_contents;struct
json_iter{
structjson_contents*contents;
int begin;
int end;
};This didn't feel consistent with any other iterators on cleanup:
Quoted 8 lines
struct
json_iteriter;
json_iter_init(&iter, json, json_len);...
json_iter_free(&iter);
Well, you would do something like:
struct json_contents *c;
struct json_iter iter;
struct json_iter nested;_auto_(l_free) char *foo = NULL;
c = json_contents_new(json, json_len);
json_iter_init(&iter, c);
json_iter_parse(&iter, JSON_STRING, "foo",
&foo, JSON_FLAG_MANDATORY, JSON_OBJECT, "bar", &nested, JSON_FLAG_OPTIONAL, JSON_UNDEFINED);
json_iter_parse(&nested, ...);
json_contents_free(c);
Quoted 4 lines
Quoted 2 lines
Do you need
jsmn_inithere?Yes, below.
Ah right, sorry for the noise.
Quoted 5 lines
Quoted 2 lines
So
value->size== 0 implies a basic type?AFAIK any value that is not an object will have a size of zero, so this could include strings as well not just basic types.
So essentially anything that is not a container? integers, strings, etc.
Quoted 8 lines
Quoted 8 lines
Quoted 3 lines
+ > + for (i = 0; i <
value->size; i++)
+ value =next_key(iter, value);So this should be recursively skipping tokens when value is non- basic. The
naming is a bit confusing here
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines
+
+ return value;Since you're returning 'value', but the reader expects a 'key'.
Yes, some re-naming is in order. This is actually returning a key though, I just reused 'value' rather than a new variable. Something like 'token' or 'tmp' would be better.
Right, this is doing the right thing, just nitpicking the variable naming.
Quoted 10 lines
Quoted 6 lines
May not be as big of a deal in this particular case, but this approach is
side-effecting on failure. You could parse the object part of the way through,
then fail to find a key and bail, but you may have assigned stuff along the way.Yeah I could not think of a decent way to handle this without iterating twice (verification first, then assignment) since we lose reference to the previous set of arguments with each iteration.
You could iterate over the va_arg list, build a temporary structure and perform the parse. If the parse succeeds, then assign the values in the final step.
Regards,
-Denis
James Prestwood
PATCH v2 3/3 unit: add JSON unit test
In reply to James Prestwood--- Makefile.am | 5 +- unit/test-json.c | 238 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 242 insertions(+), 1 deletion(-) create mode 100644 unit/test-json.c
Makefile.am +4 -1
@@ -396,7 +396,7 @@ unit_tests = unit/test-cmac-aes \ unit/test-ie unit/test-util unit/test-ssid-security \ unit/test-arc4 unit/test-wsc unit/test-eap-mschapv2 \ unit/test-eap-sim unit/test-sae unit/test-p2p unit/test-band \ - unit/test-dpp + unit/test-dpp unit/test-json if CLIENT unit_tests += unit/test-client
@@ -556,6 +556,9 @@ unit_test_dpp_SOURCES = unit/test-dpp.c src/dpp-util.h src/dpp-util.c \ unit_test_dpp_LDADD = $(ell_ldadd) +unit_test_json_SOURCES = unit/test-json.c src/json.h src/json.c shared/jsmn.h +unit_test_json_LDADD = $(ell_ldadd) + TESTS = $(unit_tests) EXTRA_DIST = src/genbuiltin src/iwd.service.in src/net.connman.iwd.service \
unit/test-json.c new file +238 -0
238 lines not on this page · raw message
Signature · 1 line
2.31.1
Marcel Holtmann
Hi James,
Quoted 4 lines
---
shared/jsmn.h| 471 ++++++++++++++++++++++++++++++++++++++++++++++++++1 file changed, 471 insertions(+)create mode 100644
shared/jsmn.h
the commit message needs to state what version you are adding here. So are you adding v1.1.0 or just a snapshot of some git tree?
Regards
Marcel