Skip to contentSkip to conversation

improve code portability

PATCH Superseded Tyler Retzlaff 6 participants 40 messages Apr 3, 2023 – May 24, 2023

lore
Share & tools
lore
#0 lore raw

cover improve code portability

Improve portability of telemetry code to allow it to be compiled by msvc unconditionally.

Remove use of VLA and instead dynamically allocate. MSVC will never implement VLAs due to misuse / security concerns.

Remove use of ranged based initializer (a gcc extension) instead just explicitly initialize individual array elements.

Tyler Retzlaff (2):
telemetry: use malloc instead of variable length array
telemetry: use portable syntax to initialize array

 lib/telemetry/telemetry_data.c | 20 ++++++++++++++------
 lib/telemetry/telemetry_json.h | 32 +++++++++++++++++++++++---------
 2 files changed, 37 insertions(+), 15 deletions(-)
Signature · 1 line
1.8.3.1
#1 lore raw

PATCH 1/2 telemetry: use malloc instead of variable length array

Replace use of variable length array optional standard feature to improve portability.

---
 lib/telemetry/telemetry_json.h | 32 +++++++++++++++++++++++---------
 1 file changed, 23 insertions(+), 9 deletions(-)
lib/telemetry/telemetry_json.h +23 -9
index 744bbfe..c375e97 100644
@@ -30,18 +30,23 @@
 static inline int
 __json_snprintf(char *buf, const int len, const char *format, ...)
 {
-	char tmp[len];
+	char *tmp = malloc(len);
 	va_list ap;
-	int ret;
+	int ret = 0;
+
+	if (tmp == NULL)
+		return ret;
 
 	va_start(ap, format);
 	ret = vsnprintf(tmp, sizeof(tmp), format, ap);
 	va_end(ap);
 	if (ret > 0 && ret < (int)sizeof(tmp) && ret < len) {
 		strcpy(buf, tmp);
-		return ret;
 	}
-	return 0; /* nothing written or modified */
+
+	free(tmp);
+
+	return ret;
 }
 
 static const char control_chars[0x20] = {
@@ -60,13 +65,17 @@
 static inline int
 __json_format_str(char *buf, const int len, const char *prefix, const char *str, const char *suffix)
 {
-	char tmp[len];
 	int tmpidx = 0;
+	int ret = 0;
+	char *tmp = malloc(len);
+
+	if (tmp == NULL)
+		return ret;
 
 	while (*prefix != '\0' && tmpidx < len)
 		tmp[tmpidx++] = *prefix++;
 	if (tmpidx >= len)
-		return 0;
+		goto cleanup;
 
 	while (*str != '\0') {
 		if (*str < (int)RTE_DIM(control_chars)) {
@@ -85,19 +94,24 @@
 		 * escaped character like "\n" without overflowing
 		 */
 		if (tmpidx > len - 2)
-			return 0;
+			goto cleanup;
 		str++;
 	}
 
 	while (*suffix != '\0' && tmpidx < len)
 		tmp[tmpidx++] = *suffix++;
 	if (tmpidx >= len)
-		return 0;
+		goto cleanup;
 
 	tmp[tmpidx] = '\0';
 
 	strcpy(buf, tmp);
-	return tmpidx;
+	ret = tmpidx;
+
+cleanup:
+	free(tmp);
+
+	return ret;
 }
 
 /* Copies an empty array into the provided buffer. */
Signature · 1 line
1.8.3.1
#2 lore raw

Re: [PATCH 1/2] telemetry: use malloc instead of variable length array

Quoted 17 lines · Tyler Retzlaff

Replace use of variable length array optional standard feature to improve portability.

---
 lib/telemetry/telemetry_json.h | 32 +++++++++++++++++++++++---------
 1 file changed, 23 insertions(+), 9 deletions(-)
lib/telemetry/telemetry_json.h +0 -1
index 744bbfe..c375e97 100644
@@ -30,18 +30,23 @@
 static inline int
 __json_snprintf(char *buf, const int len, const char *format, ...)
 {
-	char tmp[len];

lib/telemetry/telemetry_json.h line 41

Quoted 10 lines
lib/telemetry/telemetry_json.h +5 -1
+	char *tmp = malloc(len);
 	va_list ap;
-	int ret;
+	int ret = 0;
+
+	if (tmp == NULL)
+		return ret;
 
 	va_start(ap, format);
 	ret = vsnprintf(tmp, sizeof(tmp), format, ap);

I mistakenly pushed an old rev, i'll fix this.

sorry for the noise.

#3 lore raw

Re: [PATCH 1/2] telemetry: use malloc instead of variable length array

In reply to Tyler Retzlaff
Quoted 14 lines · On Mon, 3 Apr 2023 09:30:23 -0700 Tyler Retzlaff…

__json_snprintf(char *buf, const int len, const char *format, ...) {

lib/telemetry/telemetry_json.h +5 -2
-	char tmp[len];
+	char *tmp = malloc(len);
 	va_list ap;
-	int ret;
+	int ret = 0;
+
+	if (tmp == NULL)
+		return ret;
 
 	va_start(ap, format);
 	ret = vsnprintf(tmp, sizeof(tmp), format, ap);
 	va_end(ap);

lib/telemetry/telemetry_json.h line 50

Quoted 10 lines
lib/telemetry/telemetry_json.h +4 -2
 	if (ret > 0 && ret < (int)sizeof(tmp) && ret < len) {
 		strcpy(buf, tmp);
-		return ret;
 	}
-	return 0; /* nothing written or modified */
+
+	free(tmp);
+
+	return ret;
 }

Not sure why it needs a tmp buffer anyway?

#4 lore raw

Re: [PATCH 1/2] telemetry: use malloc instead of variable length array

Quoted 14 lines · Stephen Hemminger
Quoted 11 lines · On Mon, 3 Apr 2023 09:30:23 -0700 Tyler Retzlaff…

__json_snprintf(char *buf, const int len, const char *format, ...) {

lib/telemetry/telemetry_json.h +5 -2
-	char tmp[len];
+	char *tmp = malloc(len);
 	va_list ap;
-	int ret;
+	int ret = 0;
+
+	if (tmp == NULL)
+		return ret;
Quoted 4 lines

lib/telemetry/telemetry_json.h line 43

Quoted 4 lines
lib/telemetry/telemetry_json.h +0 -0
 	va_start(ap, format);
 	ret = vsnprintf(tmp, sizeof(tmp), format, ap);
 	va_end(ap);
 	if (ret > 0 && ret < (int)sizeof(tmp) && ret < len) {
                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

also this seems redundant. when is len != sizeof(tmp) here?

Quoted 11 lines

lib/telemetry/telemetry_json.h line 50

Quoted 9 lines
lib/telemetry/telemetry_json.h +4 -2
 		strcpy(buf, tmp);
-		return ret;
 	}
-	return 0; /* nothing written or modified */
+
+	free(tmp);
+
+	return ret;
 }

Not sure why it needs a tmp buffer anyway?

yeah, there a are a few question marks in this code. i've removed this patch from the series for now. v3 doesn't touch this file anymore.

#5 lore raw

Re: [PATCH 1/2] telemetry: use malloc instead of variable length array

In reply to Stephen Hemminger
Quoted 14 lines · Stephen Hemminger
Quoted 11 lines · On Mon, 3 Apr 2023 09:30:23 -0700 Tyler Retzlaff…

__json_snprintf(char *buf, const int len, const char *format, ...) {

lib/telemetry/telemetry_json.h +5 -2
-	char tmp[len];
+	char *tmp = malloc(len);
 	va_list ap;
-	int ret;
+	int ret = 0;
+
+	if (tmp == NULL)
+		return ret;
Quoted 15 lines

lib/telemetry/telemetry_json.h line 50

Quoted 13 lines
lib/telemetry/telemetry_json.h +4 -2
 	va_start(ap, format);
 	ret = vsnprintf(tmp, sizeof(tmp), format, ap);
 	va_end(ap);
 	if (ret > 0 && ret < (int)sizeof(tmp) && ret < len) {
 		strcpy(buf, tmp);
-		return ret;
 	}
-	return 0; /* nothing written or modified */
+
+	free(tmp);
+
+	return ret;
 }

Not sure why it needs a tmp buffer anyway?

The temporary buffer is to ensure that in the case that the data doesn't fit in the buffer, the buffer remains unmodified. The reason for this is that when building up the json response we always have a valid json string.

For example, suppose we are preparing a response with an array of two strings. After the first string has been processed, the output buffer
contains: '["string1"]'. When json_snprintf is being called to add string2, there are a couple of things to note:
* the text to be inserted will be put not at the end of the string, but before the closing "]".
* the actual text to be inserted will be ',"string2"]', so ensuring that the final buffer is valid.
However, the error case is problematic. While we can catch the case where the string to be inserted overflows/has been truncated, doing a regular snprintf means that our output buffer could contain invalid json, as our end-terminator would have been overwritten, e.g. '["string1","string2' To guarantee the output from telemetry is always valid json, even in case of truncation, we use a temporary buffer to do the write initially, and if it doesn't get truncated, we then copy that to the final buffer.

That's the logic for this temporary buffer. Now, thinking about it yesterday evening, there are other ways in which we can do this, which can avoid this temporary buffer.
1. We can do the initial snprintf to an empty buffer to get the length that way. This will still be slower, as it means that we need to do printf processing twice rather than using memcpy to copy the result. However, it's probably less overhead than malloc and free.
2. AFAIK, the normal case for this function being called is with a single terminator at the end of the string. We can take advantage of that, by checking if the '\0' just one character into the string we are printing, and, if so, to store that once character. If we have a snprintf error leading to truncation, it then allows us to restore the original string.

My suggestion is to use a combination of these methods. In json_snprintf check if the input buffer is empty or has only one character in it, and use method #2 if so. If that's not the case, then fallback to method #1 and do a double snprintf.

Make sense? Any other suggestions?

/Bruce

#6 lore raw

Re: [PATCH 1/2] telemetry: use malloc instead of variable length array

Quoted 31 lines · Bruce Richardson
Quoted 14 lines · Stephen Hemminger
Quoted 11 lines · On Mon, 3 Apr 2023 09:30:23 -0700 Tyler Retzlaff…

__json_snprintf(char *buf, const int len, const char *format, ...) {

lib/telemetry/telemetry_json.h +5 -2
-	char tmp[len];
+	char *tmp = malloc(len);
 	va_list ap;
-	int ret;
+	int ret = 0;
+
+	if (tmp == NULL)
+		return ret;
Quoted 15 lines

lib/telemetry/telemetry_json.h line 50

Quoted 13 lines
lib/telemetry/telemetry_json.h +4 -2
 	va_start(ap, format);
 	ret = vsnprintf(tmp, sizeof(tmp), format, ap);
 	va_end(ap);
 	if (ret > 0 && ret < (int)sizeof(tmp) && ret < len) {
 		strcpy(buf, tmp);
-		return ret;
 	}
-	return 0; /* nothing written or modified */
+
+	free(tmp);
+
+	return ret;
 }

Not sure why it needs a tmp buffer anyway?

Quoted 3 lines

The temporary buffer is to ensure that in the case that the data doesn't fit in the buffer, the buffer remains unmodified. The reason for this is that when building up the json response we always have a valid json string.

i guessed this but you've now confirmed it. it makes sense in general that if the callee signals an error to the caller that the caller shall not observe any side-effects to do so is to take a dependency on what is more often than not an internal implementation detail.

Quoted 30 lines

For example, suppose we are preparing a response with an array of two strings. After the first string has been processed, the output buffer
contains: '["string1"]'. When json_snprintf is being called to add string2, there are a couple of things to note:
* the text to be inserted will be put not at the end of the string, but before the closing "]".
* the actual text to be inserted will be ',"string2"]', so ensuring that the final buffer is valid.
However, the error case is problematic. While we can catch the case where the string to be inserted overflows/has been truncated, doing a regular snprintf means that our output buffer could contain invalid json, as our end-terminator would have been overwritten, e.g. '["string1","string2' To guarantee the output from telemetry is always valid json, even in case of truncation, we use a temporary buffer to do the write initially, and if it doesn't get truncated, we then copy that to the final buffer.

That's the logic for this temporary buffer. Now, thinking about it yesterday evening, there are other ways in which we can do this, which can avoid this temporary buffer.
1. We can do the initial snprintf to an empty buffer to get the length that way. This will still be slower, as it means that we need to do printf processing twice rather than using memcpy to copy the result. However, it's probably less overhead than malloc and free.
2. AFAIK, the normal case for this function being called is with a single terminator at the end of the string. We can take advantage of that, by checking if the '\0' just one character into the string we are printing, and, if so, to store that once character. If we have a snprintf error leading to truncation, it then allows us to restore the original string.

Quoted 6 lines

My suggestion is to use a combination of these methods. In json_snprintf check if the input buffer is empty or has only one character in it, and use method #2 if so. If that's not the case, then fallback to method #1 and do a double snprintf.

Make sense? Any other suggestions?

your suggestion seems okay to me, aside from that there's always using some fixed sized buffer but i'm guessing this being json it's difficult to choose a reasonable constant size for a stack allocated buffer.

Quoted 2 lines

/Bruce

#7 lore raw

Re: [PATCH 1/2] telemetry: use malloc instead of variable length array

Quoted 76 lines · Tyler Retzlaff
Quoted 31 lines · Bruce Richardson
Quoted 14 lines · Stephen Hemminger
Quoted 11 lines · On Mon, 3 Apr 2023 09:30:23 -0700 Tyler Retzlaff…

__json_snprintf(char *buf, const int len, const char *format, ...) {

lib/telemetry/telemetry_json.h +5 -2
-	char tmp[len];
+	char *tmp = malloc(len);
 	va_list ap;
-	int ret;
+	int ret = 0;
+
+	if (tmp == NULL)
+		return ret;
Quoted 15 lines

lib/telemetry/telemetry_json.h line 50

Quoted 13 lines
lib/telemetry/telemetry_json.h +4 -2
 	va_start(ap, format);
 	ret = vsnprintf(tmp, sizeof(tmp), format, ap);
 	va_end(ap);
 	if (ret > 0 && ret < (int)sizeof(tmp) && ret < len) {
 		strcpy(buf, tmp);
-		return ret;
 	}
-	return 0; /* nothing written or modified */
+
+	free(tmp);
+
+	return ret;
 }

Not sure why it needs a tmp buffer anyway?

Quoted 3 lines

The temporary buffer is to ensure that in the case that the data doesn't fit in the buffer, the buffer remains unmodified. The reason for this is that when building up the json response we always have a valid json string.

i guessed this but you've now confirmed it. it makes sense in general that if the callee signals an error to the caller that the caller shall not observe any side-effects to do so is to take a dependency on what is more often than not an internal implementation detail.

Quoted 35 lines

For example, suppose we are preparing a response with an array of two strings. After the first string has been processed, the output buffer
contains: '["string1"]'. When json_snprintf is being called to add string2, there are a couple of things to note:
* the text to be inserted will be put not at the end of the string, but before the closing "]".
* the actual text to be inserted will be ',"string2"]', so ensuring that the final buffer is valid.
However, the error case is problematic. While we can catch the case where the string to be inserted overflows/has been truncated, doing a regular snprintf means that our output buffer could contain invalid json, as our end-terminator would have been overwritten, e.g. '["string1","string2' To guarantee the output from telemetry is always valid json, even in case of truncation, we use a temporary buffer to do the write initially, and if it doesn't get truncated, we then copy that to the final buffer.

That's the logic for this temporary buffer. Now, thinking about it yesterday evening, there are other ways in which we can do this, which can avoid this temporary buffer.
1. We can do the initial snprintf to an empty buffer to get the length that way. This will still be slower, as it means that we need to do printf processing twice rather than using memcpy to copy the result. However, it's probably less overhead than malloc and free.
2. AFAIK, the normal case for this function being called is with a single terminator at the end of the string. We can take advantage of that, by checking if the '\0' just one character into the string we are printing, and, if so, to store that once character. If we have a snprintf error leading to truncation, it then allows us to restore the original string.

My suggestion is to use a combination of these methods. In json_snprintf check if the input buffer is empty or has only one character in it, and use method #2 if so. If that's not the case, then fallback to method #1 and do a double snprintf.

Quoted 6 lines
Quoted 1 line

Make sense? Any other suggestions?

your suggestion seems okay to me, aside from that there's always using some fixed sized buffer but i'm guessing this being json it's difficult to choose a reasonable constant size for a stack allocated buffer.

Yes, choosing a reasonable size is very difficult. We could be snprintf-ing a string containing a json-ized object a couple of KB long.

I think suggestion #2 above should cover most cases, in which case using your original suggestion of malloc would be ok too for the rare case (if ever) where we don't just have one terminator on the end.

/Bruce

#8 lore raw

Re: [PATCH 1/2] telemetry: use malloc instead of variable length array

Quoted 77 lines · Bruce Richardson
Quoted 76 lines · Tyler Retzlaff
Quoted 31 lines · Bruce Richardson
Quoted 14 lines · Stephen Hemminger
Quoted 11 lines · On Mon, 3 Apr 2023 09:30:23 -0700 Tyler Retzlaff…

__json_snprintf(char *buf, const int len, const char *format, ...) {

lib/telemetry/telemetry_json.h +5 -2
-	char tmp[len];
+	char *tmp = malloc(len);
 	va_list ap;
-	int ret;
+	int ret = 0;
+
+	if (tmp == NULL)
+		return ret;
Quoted 15 lines

lib/telemetry/telemetry_json.h line 50

Quoted 13 lines
lib/telemetry/telemetry_json.h +4 -2
 	va_start(ap, format);
 	ret = vsnprintf(tmp, sizeof(tmp), format, ap);
 	va_end(ap);
 	if (ret > 0 && ret < (int)sizeof(tmp) && ret < len) {
 		strcpy(buf, tmp);
-		return ret;
 	}
-	return 0; /* nothing written or modified */
+
+	free(tmp);
+
+	return ret;
 }

Not sure why it needs a tmp buffer anyway?

Quoted 3 lines

The temporary buffer is to ensure that in the case that the data doesn't fit in the buffer, the buffer remains unmodified. The reason for this is that when building up the json response we always have a valid json string.

i guessed this but you've now confirmed it. it makes sense in general that if the callee signals an error to the caller that the caller shall not observe any side-effects to do so is to take a dependency on what is more often than not an internal implementation detail.

Quoted 35 lines

For example, suppose we are preparing a response with an array of two strings. After the first string has been processed, the output buffer
contains: '["string1"]'. When json_snprintf is being called to add string2, there are a couple of things to note:
* the text to be inserted will be put not at the end of the string, but before the closing "]".
* the actual text to be inserted will be ',"string2"]', so ensuring that the final buffer is valid.
However, the error case is problematic. While we can catch the case where the string to be inserted overflows/has been truncated, doing a regular snprintf means that our output buffer could contain invalid json, as our end-terminator would have been overwritten, e.g. '["string1","string2' To guarantee the output from telemetry is always valid json, even in case of truncation, we use a temporary buffer to do the write initially, and if it doesn't get truncated, we then copy that to the final buffer.

That's the logic for this temporary buffer. Now, thinking about it yesterday evening, there are other ways in which we can do this, which can avoid this temporary buffer.
1. We can do the initial snprintf to an empty buffer to get the length that way. This will still be slower, as it means that we need to do printf processing twice rather than using memcpy to copy the result. However, it's probably less overhead than malloc and free.
2. AFAIK, the normal case for this function being called is with a single terminator at the end of the string. We can take advantage of that, by checking if the '\0' just one character into the string we are printing, and, if so, to store that once character. If we have a snprintf error leading to truncation, it then allows us to restore the original string.

My suggestion is to use a combination of these methods. In json_snprintf check if the input buffer is empty or has only one character in it, and use method #2 if so. If that's not the case, then fallback to method #1 and do a double snprintf.

Quoted 8 lines
Quoted 6 lines
Quoted 1 line

Make sense? Any other suggestions?

your suggestion seems okay to me, aside from that there's always using some fixed sized buffer but i'm guessing this being json it's difficult to choose a reasonable constant size for a stack allocated buffer.

Yes, choosing a reasonable size is very difficult. We could be snprintf-ing a string containing a json-ized object a couple of KB long.

haven't checked recently, but i wonder what our normal usermode stack frame size limit is, which is why alloca() would be scary.

Quoted 4 lines

I think suggestion #2 above should cover most cases, in which case using your original suggestion of malloc would be ok too for the rare case (if ever) where we don't just have one terminator on the end.

maybe a dumb'd down compromise is to have a fixed stack limit and then if it is exceeded always just go to malloc/free?

Quoted 2 lines

/Bruce

#9 lore raw

Re: [PATCH 1/2] telemetry: use malloc instead of variable length array

Quoted 91 lines · Tyler Retzlaff
Quoted 77 lines · Bruce Richardson
Quoted 76 lines · Tyler Retzlaff
Quoted 31 lines · Bruce Richardson
Quoted 14 lines · Stephen Hemminger
Quoted 11 lines · On Mon, 3 Apr 2023 09:30:23 -0700 Tyler Retzlaff…

__json_snprintf(char *buf, const int len, const char *format, ...) {

lib/telemetry/telemetry_json.h +5 -2
-	char tmp[len];
+	char *tmp = malloc(len);
 	va_list ap;
-	int ret;
+	int ret = 0;
+
+	if (tmp == NULL)
+		return ret;
Quoted 15 lines

lib/telemetry/telemetry_json.h line 50

Quoted 13 lines
lib/telemetry/telemetry_json.h +4 -2
 	va_start(ap, format);
 	ret = vsnprintf(tmp, sizeof(tmp), format, ap);
 	va_end(ap);
 	if (ret > 0 && ret < (int)sizeof(tmp) && ret < len) {
 		strcpy(buf, tmp);
-		return ret;
 	}
-	return 0; /* nothing written or modified */
+
+	free(tmp);
+
+	return ret;
 }

Not sure why it needs a tmp buffer anyway?

Quoted 3 lines

The temporary buffer is to ensure that in the case that the data doesn't fit in the buffer, the buffer remains unmodified. The reason for this is that when building up the json response we always have a valid json string.

i guessed this but you've now confirmed it. it makes sense in general that if the callee signals an error to the caller that the caller shall not observe any side-effects to do so is to take a dependency on what is more often than not an internal implementation detail.

Quoted 35 lines

For example, suppose we are preparing a response with an array of two strings. After the first string has been processed, the output buffer
contains: '["string1"]'. When json_snprintf is being called to add string2, there are a couple of things to note:
* the text to be inserted will be put not at the end of the string, but before the closing "]".
* the actual text to be inserted will be ',"string2"]', so ensuring that the final buffer is valid.
However, the error case is problematic. While we can catch the case where the string to be inserted overflows/has been truncated, doing a regular snprintf means that our output buffer could contain invalid json, as our end-terminator would have been overwritten, e.g. '["string1","string2' To guarantee the output from telemetry is always valid json, even in case of truncation, we use a temporary buffer to do the write initially, and if it doesn't get truncated, we then copy that to the final buffer.

That's the logic for this temporary buffer. Now, thinking about it yesterday evening, there are other ways in which we can do this, which can avoid this temporary buffer.
1. We can do the initial snprintf to an empty buffer to get the length that way. This will still be slower, as it means that we need to do printf processing twice rather than using memcpy to copy the result. However, it's probably less overhead than malloc and free.
2. AFAIK, the normal case for this function being called is with a single terminator at the end of the string. We can take advantage of that, by checking if the '\0' just one character into the string we are printing, and, if so, to store that once character. If we have a snprintf error leading to truncation, it then allows us to restore the original string.

My suggestion is to use a combination of these methods. In json_snprintf check if the input buffer is empty or has only one character in it, and use method #2 if so. If that's not the case, then fallback to method #1 and do a double snprintf.

Quoted 8 lines
Quoted 6 lines
Quoted 1 line

Make sense? Any other suggestions?

your suggestion seems okay to me, aside from that there's always using some fixed sized buffer but i'm guessing this being json it's difficult to choose a reasonable constant size for a stack allocated buffer.

Yes, choosing a reasonable size is very difficult. We could be snprintf-ing a string containing a json-ized object a couple of KB long.

haven't checked recently, but i wonder what our normal usermode stack frame size limit is, which is why alloca() would be scary.

Quoted 1 line
Quoted 7 lines
Quoted 3 lines

I think suggestion #2 above should cover most cases, in which case using your original suggestion of malloc would be ok too for the rare case (if ever) where we don't just have one terminator on the end.

maybe a dumb'd down compromise is to have a fixed stack limit and then if it is exceeded always just go to malloc/free?

Perhaps. If you like, I have have a try at implementing my own suggestions above tomorrow. I'd like if we can get the "single-character-saving" option working, because that would be the most efficient method of all.

/Bruce

#10 lore raw

Re: [PATCH 1/2] telemetry: use malloc instead of variable length array

Quoted 96 lines · Bruce Richardson
Quoted 95 lines · Tyler Retzlaff
Quoted 77 lines · Bruce Richardson
Quoted 76 lines · Tyler Retzlaff
Quoted 31 lines · Bruce Richardson
Quoted 14 lines · Stephen Hemminger
Quoted 11 lines · On Mon, 3 Apr 2023 09:30:23 -0700 Tyler Retzlaff…

__json_snprintf(char *buf, const int len, const char *format, ...) {

(quoted diff) +5 -2
-	char tmp[len];
+	char *tmp = malloc(len);
 	va_list ap;
-	int ret;
+	int ret = 0;
+
+	if (tmp == NULL)
+		return ret;
Quoted 15 lines

Review

Quoted 13 lines
(quoted diff) +4 -2
 	va_start(ap, format);
 	ret = vsnprintf(tmp, sizeof(tmp), format, ap);
 	va_end(ap);
 	if (ret > 0 && ret < (int)sizeof(tmp) && ret < len) {
 		strcpy(buf, tmp);
-		return ret;
 	}
-	return 0; /* nothing written or modified */
+
+	free(tmp);
+
+	return ret;
 }

Not sure why it needs a tmp buffer anyway?

Quoted 3 lines

The temporary buffer is to ensure that in the case that the data doesn't fit in the buffer, the buffer remains unmodified. The reason for this is that when building up the json response we always have a valid json string.

i guessed this but you've now confirmed it. it makes sense in general that if the callee signals an error to the caller that the caller shall not observe any side-effects to do so is to take a dependency on what is more often than not an internal implementation detail.

Quoted 35 lines

For example, suppose we are preparing a response with an array of two strings. After the first string has been processed, the output buffer
contains: '["string1"]'. When json_snprintf is being called to add string2, there are a couple of things to note:
* the text to be inserted will be put not at the end of the string, but before the closing "]".
* the actual text to be inserted will be ',"string2"]', so ensuring that the final buffer is valid.
However, the error case is problematic. While we can catch the case where the string to be inserted overflows/has been truncated, doing a regular snprintf means that our output buffer could contain invalid json, as our end-terminator would have been overwritten, e.g. '["string1","string2' To guarantee the output from telemetry is always valid json, even in case of truncation, we use a temporary buffer to do the write initially, and if it doesn't get truncated, we then copy that to the final buffer.

That's the logic for this temporary buffer. Now, thinking about it yesterday evening, there are other ways in which we can do this, which can avoid this temporary buffer.
1. We can do the initial snprintf to an empty buffer to get the length that way. This will still be slower, as it means that we need to do printf processing twice rather than using memcpy to copy the result. However, it's probably less overhead than malloc and free.
2. AFAIK, the normal case for this function being called is with a single terminator at the end of the string. We can take advantage of that, by checking if the '\0' just one character into the string we are printing, and, if so, to store that once character. If we have a snprintf error leading to truncation, it then allows us to restore the original string.

My suggestion is to use a combination of these methods. In json_snprintf check if the input buffer is empty or has only one character in it, and use method #2 if so. If that's not the case, then fallback to method #1 and do a double snprintf.

Quoted 8 lines
Quoted 6 lines
Quoted 1 line

Make sense? Any other suggestions?

your suggestion seems okay to me, aside from that there's always using some fixed sized buffer but i'm guessing this being json it's difficult to choose a reasonable constant size for a stack allocated buffer.

Yes, choosing a reasonable size is very difficult. We could be snprintf-ing a string containing a json-ized object a couple of KB long.

haven't checked recently, but i wonder what our normal usermode stack frame size limit is, which is why alloca() would be scary.

Quoted 4 lines

I think suggestion #2 above should cover most cases, in which case using your original suggestion of malloc would be ok too for the rare case (if ever) where we don't just have one terminator on the end.

Quoted 6 lines
Quoted 3 lines

maybe a dumb'd down compromise is to have a fixed stack limit and then if it is exceeded always just go to malloc/free?

Perhaps. If you like, I have have a try at implementing my own suggestions above tomorrow. I'd like if we can get the "single-character-saving" option working, because that would be the most efficient method of all.

that would be great, i'll take the help i can get.

Quoted 2 lines

/Bruce

#11 lore raw

Re: [PATCH 1/2] telemetry: use malloc instead of variable length array

Quoted 4 lines · On Tue, 4 Apr 2023 10:34:01 -0700 Tyler Retzlaff…
Quoted 4 lines
Quoted 4 lines
Quoted 3 lines

I think suggestion #2 above should cover most cases, in which case using your original suggestion of malloc would be ok too for the rare case (if ever) where we don't just have one terminator on the end.

Quoted 8 lines
Quoted 6 lines
Quoted 3 lines

maybe a dumb'd down compromise is to have a fixed stack limit and then if it is exceeded always just go to malloc/free?

Perhaps. If you like, I have have a try at implementing my own suggestions above tomorrow. I'd like if we can get the "single-character-saving" option working, because that would be the most efficient method of all.

that would be great, i'll take the help i can get.

For the json print library in iproute2, it streams output using stdio, which avoids lots of string processing issues. Well, it moves them to be standard library.

#12 lore raw

Re: [PATCH 1/2] telemetry: use malloc instead of variable length array

Quoted 14 lines · Stephen Hemminger
Quoted 11 lines · On Tue, 4 Apr 2023 10:34:01 -0700 Tyler Retzlaff…
Quoted 10 lines
Quoted 7 lines
Quoted 3 lines

I think suggestion #2 above should cover most cases, in which case using your original suggestion of malloc would be ok too for the rare case (if ever) where we don't just have one terminator on the end.

maybe a dumb'd down compromise is to have a fixed stack limit and then if it is exceeded always just go to malloc/free?

Perhaps. If you like, I have have a try at implementing my own suggestions above tomorrow. I'd like if we can get the "single-character-saving" option working, because that would be the most efficient method of all.

Quoted 6 lines
Quoted 1 line

that would be great, i'll take the help i can get.

For the json print library in iproute2, it streams output using stdio, which avoids lots of string processing issues. Well, it moves them to be standard library.

Yes, but it does have a number of downsides:

* streaming means that we have to worry about message boundaries when transferring across sockets like in the telemetry case. With telemetry we have a packet-based (message based) interface, so we need to buffer all output to a string anyway. If we ever add a streaming (socket or fifo) interface, then the iproute2 implementation may be a better fit.

* Last I looked at the iproute2 json implementation, it requires the user to track how many outstanding closing brackets of what type are needed, meaning that you have invalid json at intermediate stages, and makes it awkward if you run out of space in a buffer, i.e. no room for terminators. The implementation here, on the other hand, ensures that the output is always valid json, so for example, when adding a second string to an array of strings we go from ["str1"] to ["str1","str2"] directly - or in the case of insufficient space we leave the original version unmodified. For working with the case where we have a fixed buffer, this is a massive benefit.

/Bruce

#13 lore raw

Re: [PATCH 1/2] telemetry: use malloc instead of variable length array

In reply to Bruce Richardson
Quoted 6 lines · On Tue, 4 Apr 2023 09:47:21 +0100 Bruce Richardson…

My suggestion is to use a combination of these methods. In json_snprintf check if the input buffer is empty or has only one character in it, and use method #2 if so. If that's not the case, then fallback to method #1 and do a double snprintf.

Make sense? Any other suggestions?

Glibc has asprintf which allocates the buffer for you.

#14 lore raw

Re: [PATCH 1/2] telemetry: use malloc instead of variable length array

Quoted 11 lines · Stephen Hemminger
Quoted 6 lines · On Tue, 4 Apr 2023 09:47:21 +0100 Bruce Richardson…

My suggestion is to use a combination of these methods. In json_snprintf check if the input buffer is empty or has only one character in it, and use method #2 if so. If that's not the case, then fallback to method #1 and do a double snprintf.

Make sense? Any other suggestions?

Glibc has asprintf which allocates the buffer for you.

Good point, I'll use that in any new implementation. Thanks.

#15 lore raw

Re: [PATCH 1/2] telemetry: use malloc instead of variable length array

Quoted 9 lines · Bruce Richardson
Quoted 8 lines · Stephen Hemminger
Quoted 5 lines · On Tue, 4 Apr 2023 09:47:21 +0100 Bruce Richardson…

My suggestion is to use a combination of these methods. In json_snprintf check if the input buffer is empty or has only one character in it, and use method #2 if so. If that's not the case, then fallback to method #1 and do a double snprintf.

Quoted 5 lines
Quoted 3 lines
Quoted 1 line

Make sense? Any other suggestions?

Glibc has asprintf which allocates the buffer for you.

Good point, I'll use that in any new implementation. Thanks.

i imagine there is an equivalent to asprintf for windows but keep in mind it is not standard C so you'll have to do something conditional.

#16 lore raw

Re: [PATCH 1/2] telemetry: use malloc instead of variable length array

2023-04-05 08:25 (UTC-0700), Tyler Retzlaff:

Quoted 10 lines · Bruce Richardson
Quoted 9 lines · Stephen Hemminger
Quoted 8 lines
Quoted 5 lines · On Tue, 4 Apr 2023 09:47:21 +0100 Bruce Richardson…

My suggestion is to use a combination of these methods. In json_snprintf check if the input buffer is empty or has only one character in it, and use method #2 if so. If that's not the case, then fallback to method #1 and do a double snprintf.

Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
Quoted 1 line

Make sense? Any other suggestions?

Glibc has asprintf which allocates the buffer for you.

Good point, I'll use that in any new implementation. Thanks.

i imagine there is an equivalent to asprintf for windows but keep in mind it is not standard C so you'll have to do something conditional.

There's eal_asprintf() shim in EAL,
we could make it internal to reuse in telemetry.

#17 lore raw

Re: [PATCH 1/2] telemetry: use malloc instead of variable length array

Quoted 15 lines · On Wed, 5 Apr 2023 18:30:51 +0300 Dmitry Kozlyuk <[email protected]>…

2023-04-05 08:25 (UTC-0700), Tyler Retzlaff:

Quoted 14 lines · Bruce Richardson
Quoted 13 lines · Stephen Hemminger
Quoted 11 lines
Quoted 6 lines · On Tue, 4 Apr 2023 09:47:21 +0100 Bruce Richardson…

My suggestion is to use a combination of these methods. In json_snprintf check if the input buffer is empty or has only one character in it, and use method #2 if so. If that's not the case, then fallback to method #1 and do a double snprintf.

Make sense? Any other suggestions?

Glibc has asprintf which allocates the buffer for you.

Quoted 7 lines
Quoted 4 lines
Quoted 1 line

Good point, I'll use that in any new implementation. Thanks.

i imagine there is an equivalent to asprintf for windows but keep in mind it is not standard C so you'll have to do something conditional.

There's eal_asprintf() shim in EAL,
we could make it internal to reuse in telemetry.

Windows has tools necessary to build asprint equivalent.

See: https://github.com/eiszapfen2000/asprintf

#18 lore raw

Re: [PATCH 1/2] telemetry: use malloc instead of variable length array

In reply to Tyler Retzlaff
Quoted 10 lines · Tyler Retzlaff
Quoted 9 lines · Bruce Richardson
Quoted 8 lines · Stephen Hemminger
Quoted 5 lines · On Tue, 4 Apr 2023 09:47:21 +0100 Bruce Richardson…

My suggestion is to use a combination of these methods. In json_snprintf check if the input buffer is empty or has only one character in it, and use method #2 if so. If that's not the case, then fallback to method #1 and do a double snprintf.

Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
Quoted 1 line

Make sense? Any other suggestions?

Glibc has asprintf which allocates the buffer for you.

Good point, I'll use that in any new implementation. Thanks.

i imagine there is an equivalent to asprintf for windows but keep in mind it is not standard C so you'll have to do something conditional.

Only noticed this now, after I've just sent out the patches! I'll have to see about more rework so. :-(

/Bruce

#19 lore raw

PATCH 2/2 telemetry: use portable syntax to initialize array

In reply to Tyler Retzlaff

Use of ranges in designated initialization are a non-standard gcc extension. Manually expand the ranges used in designated initialization to eliminate the use of non-standard features.

---
 lib/telemetry/telemetry_data.c | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)
lib/telemetry/telemetry_data.c +14 -6
index 2bac2de..87e5f18 100644
@@ -152,12 +152,20 @@
 static bool
 valid_name(const char *name)
 {
-	char allowed[128] = {
-			['0' ... '9'] = 1,
-			['A' ... 'Z'] = 1,
-			['a' ... 'z'] = 1,
-			['_'] = 1,
-			['/'] = 1,
+	static const char allowed[128] = {
+		['0'] = 1, ['1'] = 1, ['2'] = 1, ['3'] = 1, ['4'] = 1,
+		['5'] = 1, ['6'] = 1, ['7'] = 1, ['8'] = 1, ['9'] = 1,
+		['A'] = 1, ['B'] = 1, ['C'] = 1, ['D'] = 1, ['E'] = 1,
+		['F'] = 1, ['G'] = 1, ['H'] = 1, ['I'] = 1, ['J'] = 1,
+		['K'] = 1, ['L'] = 1, ['M'] = 1, ['N'] = 1, ['O'] = 1,
+		['P'] = 1, ['Q'] = 1, ['R'] = 1, ['S'] = 1, ['T'] = 1,
+		['U'] = 1, ['V'] = 1, ['W'] = 1, ['X'] = 1, ['Y'] = 1,
+		['Z'] = 1, ['a'] = 1, ['b'] = 1, ['c'] = 1, ['d'] = 1,
+		['e'] = 1, ['f'] = 1, ['g'] = 1, ['h'] = 1, ['i'] = 1,
+		['j'] = 1, ['k'] = 1, ['l'] = 1, ['m'] = 1, ['n'] = 1,
+		['o'] = 1, ['p'] = 1, ['q'] = 1, ['r'] = 1, ['s'] = 1,
+		['t'] = 1, ['u'] = 1, ['v'] = 1, ['w'] = 1, ['x'] = 1,
+		['y'] = 1, ['z'] = 1, ['_'] = 1, ['/'] = 1,
 	};
 	while (*name != '\0') {
 		if ((size_t)*name >= RTE_DIM(allowed) || allowed[(int)*name] == 0)
Signature · 1 line
1.8.3.1
#20 lore raw
In reply to Tyler Retzlaff
Quoted 6 lines · Tyler Retzlaff

Improve portability of telemetry code to allow it to be compiled by msvc unconditionally.

Remove use of VLA and instead dynamically allocate. MSVC will never implement VLAs due to misuse / security concerns.

Quoted 7 lines

Remove use of ranged based initializer (a gcc extension) instead just explicitly initialize individual array elements.

Tyler Retzlaff (2):
telemetry: use malloc instead of variable length array
telemetry: use portable syntax to initialize array

Is this worth doing, given that DPDK telemetry uses a unix domain socket for it's connectivity, which would not be available on windows anyway? I don't particularly like these patches as:
* The removal of the VLAs means we will potentially be doing a *lot* of malloc and free calls inside the telemetry code. It may not be a data path or particularly performance critical, but I know for things like CPU busyness, users may want to call telemetry functions hundreds (or potentially thousands) of times a second. It also makes the code slightly harder to read, and introduces the possibility of us having memory leaks.
* The second patch just makes the code uglier. True, it's non-standard, but it really does make the code a whole lot more readable and managable. If we need to make this standards-conforming, then I think we need to drop the "const", and do runtime init of this array with loops for the ranges.

All that said, if we do have a path to get telemetry working on windows, I think we can work together to get a suitable patchset in for it.

/Bruce

#21 lore raw
Quoted 7 lines · Bruce Richardson
Quoted 6 lines · Tyler Retzlaff

Improve portability of telemetry code to allow it to be compiled by msvc unconditionally.

Remove use of VLA and instead dynamically allocate. MSVC will never implement VLAs due to misuse / security concerns.

Quoted 7 lines
Quoted 7 lines

Remove use of ranged based initializer (a gcc extension) instead just explicitly initialize individual array elements.

Tyler Retzlaff (2):
telemetry: use malloc instead of variable length array
telemetry: use portable syntax to initialize array

Bruce I was hoping to solicit your response specifically on this series so thanks.

Quoted 9 lines

Is this worth doing, given that DPDK telemetry uses a unix domain socket for it's connectivity, which would not be available on windows anyway? I don't particularly like these patches as:
* The removal of the VLAs means we will potentially be doing a *lot* of malloc and free calls inside the telemetry code. It may not be a data path or particularly performance critical, but I know for things like CPU busyness, users may want to call telemetry functions hundreds (or potentially thousands) of times a second. It also makes the code slightly harder to read, and introduces the possibility of us having memory leaks.

Yeah, malloc/free at high frequency is a bit ugly we could use alloca but I think equally as unsafe as VLAs.

Is the preference here to just not compile any of this code for Windows regardless of compiler? If that is preferred I can look at refactoring to do that.

Another option is I could use conditionally compiled code narrowly here using a Windows only API malloca which is vaguely safer?

Quoted 4 lines

* The second patch just makes the code uglier. True, it's non-standard, but it really does make the code a whole lot more readable and managable. If we need to make this standards-conforming, then I think we need to drop the "const", and do runtime init of this array with loops for the ranges.

I considered this. I thought the preference was to preserve const but if initialization in a loop is preferrable i'll do that instead.

Quoted 3 lines

All that said, if we do have a path to get telemetry working on windows, I think we can work together to get a suitable patchset in for it.

It's unfortunate that EAL depends on telemetry but it doesn't work on Windows (right now). Telemetry is unfortunately not of much value if the code doesn't build and run so thus a lower priority.

What I can ask for here (if the community accepts) is getting the code to compile, that's what unblocks progress for now and we can re-visit making telemetry work properly ~later.

Thanks

Quoted 2 lines

/Bruce

#22 lore raw

PATCH v2 1/1 improve code portability

In reply to Tyler Retzlaff

Improve portability of telemetry code to allow it to be compiled by msvc unconditionally.

Remove use of ranged based initializer (a gcc extension). Instead initialize using for loops over [0..9] [A..Z] [a..z] [_/] on first call.

v2:
* Initialize valid character array using for loop instead of explicit initialization of each element.
* Drop patch removing VLAs, in a separate series a conditionally compiled change will be introduced.

Tyler Retzlaff (1):
telemetry: use portable syntax to initialize array

 lib/telemetry/telemetry_data.c | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)
Signature · 1 line
1.8.3.1
#23 lore raw

PATCH v2 1/1 telemetry: use portable syntax to initialize array

Use of ranges in designated initialization are a non-standard gcc extension. Use loops to initialize permitted characters on first use.

---
 lib/telemetry/telemetry_data.c | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)
lib/telemetry/telemetry_data.c +15 -7
index 2bac2de..d845186 100644
@@ -152,13 +152,21 @@
 static bool
 valid_name(const char *name)
 {
-	char allowed[128] = {
-			['0' ... '9'] = 1,
-			['A' ... 'Z'] = 1,
-			['a' ... 'z'] = 1,
-			['_'] = 1,
-			['/'] = 1,
-	};
+	static bool initialized;
+	static char allowed[128];
+
+	if (!initialized) {
+		int index;
+		for (index = '0'; index <= '9'; index++)
+			allowed[index] = 1;
+		for (index = 'A'; index <= 'Z'; index++)
+			allowed[index] = 1;
+		for (index = 'a'; index <= 'Z'; index++)
+			allowed[index] = 1;
+		allowed[(int)'_'] = allowed[(int)'/'] = 1;
+		initialized = true;
+	}
+
 	while (*name != '\0') {
 		if ((size_t)*name >= RTE_DIM(allowed) || allowed[(int)*name] == 0)
 			return false;
Signature · 1 line
1.8.3.1
#24 lore raw

PATCH v3 1/1 improve code portability

In reply to Tyler Retzlaff

Improve portability of telemetry code to allow it to be compiled by msvc unconditionally.

Remove use of ranged based initializer (a gcc extension). Instead initialize using for loops over [0..9] [A..Z] [a..z] [_/] on first call.

v3:
* Move int index declaration to top of function
* Fix typo 'Z' -> 'z' in last for loop
v2:
* Initialize valid character array using for loop instead of explicit initialization of each element.
* Drop patch removing VLAs, in a separate series a conditionally compiled change will be introduced.

Tyler Retzlaff (1):
telemetry: use portable syntax to initialize array

 lib/telemetry/telemetry_data.c | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)
Signature · 1 line
1.8.3.1
#25 lore raw

PATCH v3 1/1 telemetry: use portable syntax to initialize array

Use of ranges in designated initialization are a non-standard gcc extension. Use loops to initialize permitted characters on first use.

---
 lib/telemetry/telemetry_data.c | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)
lib/telemetry/telemetry_data.c +15 -7
index 2bac2de..562b387 100644
@@ -152,13 +152,21 @@
 static bool
 valid_name(const char *name)
 {
-	char allowed[128] = {
-			['0' ... '9'] = 1,
-			['A' ... 'Z'] = 1,
-			['a' ... 'z'] = 1,
-			['_'] = 1,
-			['/'] = 1,
-	};
+	int index;
+	static bool initialized;
+	static char allowed[128];
+
+	if (!initialized) {
+		for (index = '0'; index <= '9'; index++)
+			allowed[index] = 1;
+		for (index = 'A'; index <= 'Z'; index++)
+			allowed[index] = 1;
+		for (index = 'a'; index <= 'z'; index++)
+			allowed[index] = 1;
+		allowed[(int)'_'] = allowed[(int)'/'] = 1;
+		initialized = true;
+	}
+
 	while (*name != '\0') {
 		if ((size_t)*name >= RTE_DIM(allowed) || allowed[(int)*name] == 0)
 			return false;
Signature · 1 line
1.8.3.1
Acked-by on v3 1/1
#26 lore raw

Re: [PATCH v3] telemetry: use portable syntax to initialize array

Quoted 4 lines · Tyler Retzlaff

Use of ranges in designated initialization are a non-standard gcc extension. Use loops to initialize permitted characters on first use.

Quoted 10 lines
---
 lib/telemetry/telemetry_data.c | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)
lib/telemetry/telemetry_data.c +0 -0
index 2bac2de..562b387 100644
@@ -152,13 +152,21 @@
 static bool

lib/telemetry/telemetry_data.c line 155

Quoted 10 lines
lib/telemetry/telemetry_data.c +1 -7
 valid_name(const char *name)
 {
-	char allowed[128] = {
-			['0' ... '9'] = 1,
-			['A' ... 'Z'] = 1,
-			['a' ... 'z'] = 1,
-			['_'] = 1,
-			['/'] = 1,
-	};
+	int index;

My preference would be to limit the scope of index to the if block, but ok to keep as here.

[In fact, when we switch to C11, I'd love to see the coding standards relaxed to allow loop variable definition inside the for statement itself]

Quoted 20 lines
lib/telemetry/telemetry_data.c +14 -0
+	static bool initialized;
+	static char allowed[128];
+
+	if (!initialized) {
+		for (index = '0'; index <= '9'; index++)
+			allowed[index] = 1;
+		for (index = 'A'; index <= 'Z'; index++)
+			allowed[index] = 1;
+		for (index = 'a'; index <= 'z'; index++)
+			allowed[index] = 1;
+		allowed[(int)'_'] = allowed[(int)'/'] = 1;
+		initialized = true;
+	}
+
 	while (*name != '\0') {
 		if ((size_t)*name >= RTE_DIM(allowed) || allowed[(int)*name] == 0)
 			return false;
Signature · 1 line
1.8.3.1
#27 lore raw

Re: [PATCH v3] telemetry: use portable syntax to initialize array

Quoted 23 lines · Bruce Richardson
Quoted 4 lines · Tyler Retzlaff

Use of ranges in designated initialization are a non-standard gcc extension. Use loops to initialize permitted characters on first use.

Quoted 15 lines
---
 lib/telemetry/telemetry_data.c | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)
lib/telemetry/telemetry_data.c +0 -3
index 2bac2de..562b387 100644
@@ -152,13 +152,21 @@
 static bool
 valid_name(const char *name)
 {
-	char allowed[128] = {
-			['0' ... '9'] = 1,
-			['A' ... 'Z'] = 1,
Quoted 8 lines

lib/telemetry/telemetry_data.c line 155

Quoted 5 lines
lib/telemetry/telemetry_data.c +1 -4
-			['a' ... 'z'] = 1,
-			['_'] = 1,
-			['/'] = 1,
-	};
+	int index;

My preference would be to limit the scope of index to the if block, but ok to keep as here.

yes, mine too. but i forgot we aren't C99 (yet) so i had to move it for this patch.

Quoted 3 lines

[In fact, when we switch to C11, I'd love to see the coding standards relaxed to allow loop variable definition inside the for statement itself]

i'm for it, in fact for any variable i prefer declaration at or near first use because it let's me const more. but i know that can be an unpopular opinion so no need to hit 'r' to tell me off, i will follow any and all documented convention the community has.

Quoted 21 lines
Quoted 21 lines
lib/telemetry/telemetry_data.c +14 -0
+	static bool initialized;
+	static char allowed[128];
+
+	if (!initialized) {
+		for (index = '0'; index <= '9'; index++)
+			allowed[index] = 1;
+		for (index = 'A'; index <= 'Z'; index++)
+			allowed[index] = 1;
+		for (index = 'a'; index <= 'z'; index++)
+			allowed[index] = 1;
+		allowed[(int)'_'] = allowed[(int)'/'] = 1;
+		initialized = true;
+	}
+
 	while (*name != '\0') {
 		if ((size_t)*name >= RTE_DIM(allowed) || allowed[(int)*name] == 0)
 			return false;
Signature · 1 line
1.8.3.1
#28 lore raw

Re: [PATCH v3] telemetry: use portable syntax to initialize array

Quoted 33 lines · Tyler Retzlaff
Quoted 23 lines · Bruce Richardson
Quoted 4 lines · Tyler Retzlaff

Use of ranges in designated initialization are a non-standard gcc extension. Use loops to initialize permitted characters on first use.

Quoted 15 lines
---
 lib/telemetry/telemetry_data.c | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)
lib/telemetry/telemetry_data.c +0 -3
index 2bac2de..562b387 100644
@@ -152,13 +152,21 @@
 static bool
 valid_name(const char *name)
 {
-	char allowed[128] = {
-			['0' ... '9'] = 1,
-			['A' ... 'Z'] = 1,
Quoted 8 lines

lib/telemetry/telemetry_data.c line 155

Quoted 5 lines
lib/telemetry/telemetry_data.c +1 -4
-			['a' ... 'z'] = 1,
-			['_'] = 1,
-			['/'] = 1,
-	};
+	int index;

My preference would be to limit the scope of index to the if block, but ok to keep as here.

Quoted 3 lines

yes, mine too. but i forgot we aren't C99 (yet) so i had to move it for this patch.

We do allow variable declarations within blocks, so no need to move it up here. However, I believe our coding standards currently require them at the *top* of each block, not in the middle - but also not just at the top of each function. See [1] for the details. [For some reason this is in the sub-section under "indentation" in the guide!]

[1] https://doc.dpdk.org/guides-22.11/contributing/coding_style.html#local-variables

Quoted 9 lines
Quoted 3 lines

[In fact, when we switch to C11, I'd love to see the coding standards relaxed to allow loop variable definition inside the for statement itself]

i'm for it, in fact for any variable i prefer declaration at or near first use because it let's me const more. but i know that can be an unpopular opinion so no need to hit 'r' to tell me off, i will follow any and all documented convention the community has.

For const, our coding standards already call this out as an exception, where it's allowed to define a variable in the middle of a block. Generally though, I too prefer vars to be defined at site of first use, rather than all together at the top of the block. Makes it easier when commenting out code for debugging, not to get unused var warnings.

/Bruce

#29 lore raw

RE: [PATCH v3] telemetry: use portable syntax to initialize array

In reply to Tyler Retzlaff
Quoted 41 lines

-----Original Message-----
From: Tyler Retzlaff <[email protected]>
Sent: Monday, April 3, 2023 7:59 PM
To: [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Tyler Retzlaff <[email protected]>
Subject: [PATCH v3] telemetry: use portable syntax to initialize array

Use of ranges in designated initialization are a non-standard gcc extension. Use loops to initialize permitted characters on first use.

---
 lib/telemetry/telemetry_data.c | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)
lib/telemetry/telemetry_data.c +8 -7
index 2bac2de..562b387 100644
@@ -152,13 +152,21 @@
 static bool
 valid_name(const char *name)
 {
-	char allowed[128] = {
-			['0' ... '9'] = 1,
-			['A' ... 'Z'] = 1,
-			['a' ... 'z'] = 1,
-			['_'] = 1,
-			['/'] = 1,
-	};
+	int index;
+	static bool initialized;
+	static char allowed[128];
+
+	if (!initialized) {
+		for (index = '0'; index <= '9'; index++)
+			allowed[index] = 1;
+		for (index = 'A'; index <= 'Z'; index++)

lib/telemetry/telemetry_data.c line 172

Quoted 10 lines
lib/telemetry/telemetry_data.c +7 -0
+			allowed[index] = 1;
+		for (index = 'a'; index <= 'z'; index++)
+			allowed[index] = 1;
+		allowed[(int)'_'] = allowed[(int)'/'] = 1;
+		initialized = true;
+	}
+
 	while (*name != '\0') {
 		if ((size_t)*name >= RTE_DIM(allowed) || allowed[(int)*name] == 0)
 			return false;

Wonder isn't the same as:
while (*name != 0)

	if (!isascii(name[0] || (!isalnum(name[0])  && name[0] != '_' && name[0] != '/'))
		return false; 

Or MSVC doesn't support these macros?

Quoted 2 lines

--
1.8.3.1

#30 lore raw

Re: [PATCH v3] telemetry: use portable syntax to initialize array

Quoted 54 lines · Konstantin Ananyev
Quoted 41 lines

-----Original Message-----
From: Tyler Retzlaff <[email protected]>
Sent: Monday, April 3, 2023 7:59 PM
To: [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Tyler Retzlaff <[email protected]>
Subject: [PATCH v3] telemetry: use portable syntax to initialize array

Use of ranges in designated initialization are a non-standard gcc extension. Use loops to initialize permitted characters on first use.

---
 lib/telemetry/telemetry_data.c | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)
lib/telemetry/telemetry_data.c +8 -7
index 2bac2de..562b387 100644
@@ -152,13 +152,21 @@
 static bool
 valid_name(const char *name)
 {
-	char allowed[128] = {
-			['0' ... '9'] = 1,
-			['A' ... 'Z'] = 1,
-			['a' ... 'z'] = 1,
-			['_'] = 1,
-			['/'] = 1,
-	};
+	int index;
+	static bool initialized;
+	static char allowed[128];
+
+	if (!initialized) {
+		for (index = '0'; index <= '9'; index++)
+			allowed[index] = 1;
+		for (index = 'A'; index <= 'Z'; index++)

lib/telemetry/telemetry_data.c line 172

Quoted 10 lines
lib/telemetry/telemetry_data.c +7 -0
+			allowed[index] = 1;
+		for (index = 'a'; index <= 'z'; index++)
+			allowed[index] = 1;
+		allowed[(int)'_'] = allowed[(int)'/'] = 1;
+		initialized = true;
+	}
+
 	while (*name != '\0') {
 		if ((size_t)*name >= RTE_DIM(allowed) || allowed[(int)*name] == 0)
 			return false;

Wonder isn't the same as:
while (*name != 0)

Quoted 4 lines
	if (!isascii(name[0] || (!isalnum(name[0])  && name[0] != '_' && name[0] != '/'))
		return false; 

Or MSVC doesn't support these macros?

it's standard C msvc supports it. Bruce acceptable to take Konstantin's suggestion here?

Quoted 5 lines
Quoted 5 lines

--
1.8.3.1

#31 lore raw

Re: [PATCH v3] telemetry: use portable syntax to initialize array

Quoted 58 lines · Tyler Retzlaff
Quoted 57 lines · Konstantin Ananyev
Quoted 41 lines

-----Original Message-----
From: Tyler Retzlaff <[email protected]>
Sent: Monday, April 3, 2023 7:59 PM
To: [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Tyler Retzlaff <[email protected]>
Subject: [PATCH v3] telemetry: use portable syntax to initialize array

Use of ranges in designated initialization are a non-standard gcc extension. Use loops to initialize permitted characters on first use.

---
 lib/telemetry/telemetry_data.c | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)
lib/telemetry/telemetry_data.c +8 -7
index 2bac2de..562b387 100644
@@ -152,13 +152,21 @@
 static bool
 valid_name(const char *name)
 {
-	char allowed[128] = {
-			['0' ... '9'] = 1,
-			['A' ... 'Z'] = 1,
-			['a' ... 'z'] = 1,
-			['_'] = 1,
-			['/'] = 1,
-	};
+	int index;
+	static bool initialized;
+	static char allowed[128];
+
+	if (!initialized) {
+		for (index = '0'; index <= '9'; index++)
+			allowed[index] = 1;
+		for (index = 'A'; index <= 'Z'; index++)

lib/telemetry/telemetry_data.c line 172

Quoted 10 lines
lib/telemetry/telemetry_data.c +7 -0
+			allowed[index] = 1;
+		for (index = 'a'; index <= 'z'; index++)
+			allowed[index] = 1;
+		allowed[(int)'_'] = allowed[(int)'/'] = 1;
+		initialized = true;
+	}
+
 	while (*name != '\0') {
 		if ((size_t)*name >= RTE_DIM(allowed) || allowed[(int)*name] == 0)
 			return false;

Wonder isn't the same as:
while (*name != 0)

	if (!isascii(name[0] || (!isalnum(name[0])  && name[0] != '_' && name[0] != '/'))
		return false; 
Quoted 5 lines
Quoted 1 line

Or MSVC doesn't support these macros?

it's standard C msvc supports it. Bruce acceptable to take Konstantin's suggestion here?

Yep, I'm ok with it.

If I may confuse things a little, how about combining the two? Use an array with individual initializers for the "special characters", but using isalnum() for the rest. That way we get a clear list of allowed chars at the top of the array, without having a massive list of individual letters and numbers. For example: [completely untested and not compiled!].

{
	/* non-alpha-numeric characters allowed in names */
	static const char allowed[128] = { ['_'] = 1, ['/'] = 1 };

	for ( ; *name != '\0'; name++) {
		if (isalnum(*name))
			continue;
		if ((size_t)*name >= RTE_DIM(allowed) || allowed[(int)*name] == 0)
			return false;
	}
	return true;
}

I think something like this achieves a good balance between clarity and brevity.

1.  We don't have a massive array definition
2.  We do have a static constant array 
3.  We don't have the non-alnum character list hidden inside the middle of

an "if" statement.

/Bruce

#32 lore raw

Re: [PATCH v3] telemetry: use portable syntax to initialize array

Quoted 85 lines · Bruce Richardson
Quoted 58 lines · Tyler Retzlaff
Quoted 57 lines · Konstantin Ananyev
Quoted 41 lines

-----Original Message-----
From: Tyler Retzlaff <[email protected]>
Sent: Monday, April 3, 2023 7:59 PM
To: [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Tyler Retzlaff <[email protected]>
Subject: [PATCH v3] telemetry: use portable syntax to initialize array

Use of ranges in designated initialization are a non-standard gcc extension. Use loops to initialize permitted characters on first use.

---
 lib/telemetry/telemetry_data.c | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)
lib/telemetry/telemetry_data.c +8 -7
index 2bac2de..562b387 100644
@@ -152,13 +152,21 @@
 static bool
 valid_name(const char *name)
 {
-	char allowed[128] = {
-			['0' ... '9'] = 1,
-			['A' ... 'Z'] = 1,
-			['a' ... 'z'] = 1,
-			['_'] = 1,
-			['/'] = 1,
-	};
+	int index;
+	static bool initialized;
+	static char allowed[128];
+
+	if (!initialized) {
+		for (index = '0'; index <= '9'; index++)
+			allowed[index] = 1;
+		for (index = 'A'; index <= 'Z'; index++)

lib/telemetry/telemetry_data.c line 172

Quoted 10 lines
lib/telemetry/telemetry_data.c +7 -0
+			allowed[index] = 1;
+		for (index = 'a'; index <= 'z'; index++)
+			allowed[index] = 1;
+		allowed[(int)'_'] = allowed[(int)'/'] = 1;
+		initialized = true;
+	}
+
 	while (*name != '\0') {
 		if ((size_t)*name >= RTE_DIM(allowed) || allowed[(int)*name] == 0)
 			return false;

Wonder isn't the same as:
while (*name != 0)

	if (!isascii(name[0] || (!isalnum(name[0])  && name[0] != '_' && name[0] != '/'))
		return false; 
Quoted 5 lines
Quoted 1 line

Or MSVC doesn't support these macros?

it's standard C msvc supports it. Bruce acceptable to take Konstantin's suggestion here?

Yep, I'm ok with it.

If I may confuse things a little, how about combining the two? Use an array with individual initializers for the "special characters", but using isalnum() for the rest. That way we get a clear list of allowed chars at the top of the array, without having a massive list of individual letters and numbers. For example: [completely untested and not compiled!].

{
	/* non-alpha-numeric characters allowed in names */
	static const char allowed[128] = { ['_'] = 1, ['/'] = 1 };

	for ( ; *name != '\0'; name++) {
		if (isalnum(*name))
			continue;
		if ((size_t)*name >= RTE_DIM(allowed) || allowed[(int)*name] == 0)
			return false;
	}
	return true;
}
Quoted 6 lines

I think something like this achieves a good balance between clarity and brevity.

1.  We don't have a massive array definition
2.  We do have a static constant array 
3.  We don't have the non-alnum character list hidden inside the middle of

an "if" statement.

at first glance i'm not sure if that initializer list syntax is portable but i guess i'll find out.

i'm fine with being confused more so long as you give me the answer and i can just cut and paste it ;)

Quoted 2 lines

/Bruce

#33 lore raw

PATCH v4 1/1 improve code portability

In reply to Tyler Retzlaff

Improve portability of telemetry code to allow it to be compiled by msvc unconditionally.

Remove use of designated initialization ranges (gcc extension). Instead use a combination of a trivially initialized array and standard C isalnum.

v4:
* Re-implement using isalnum and sparsely populated array.
v3:
* Move int index declaration to top of function
* Fix typo 'Z' -> 'z' in last for loop
v2:
* Initialize valid character array using for loop instead of explicit initialization of each element.
* Drop patch removing VLAs, in a separate series a conditionally compiled change will be introduced.

Tyler Retzlaff (1):
telemetry: remove non-portable array initialization syntax

 lib/telemetry/telemetry_data.c | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)
Signature · 1 line
1.8.3.1
#34 lore raw

PATCH v4 1/1 telemetry: remove non-portable array initialization syntax

Use of ranges in designated initialization are a non-standard gcc extension.

Only initialize '_' and '/' elements of the array and filter tests of characters through name with standard C isalnum before checking the array.

---
 lib/telemetry/telemetry_data.c | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)
lib/telemetry/telemetry_data.c +7 -9
index 2bac2de..0dc091a 100644
@@ -2,6 +2,7 @@
  * Copyright(c) 2020 Intel Corporation
  */
 
+#include <ctype.h>
 #include <errno.h>
 #include <stdlib.h>
 #include <inttypes.h>
@@ -152,17 +153,14 @@
 static bool
 valid_name(const char *name)
 {
-	char allowed[128] = {
-			['0' ... '9'] = 1,
-			['A' ... 'Z'] = 1,
-			['a' ... 'z'] = 1,
-			['_'] = 1,
-			['/'] = 1,
-	};
-	while (*name != '\0') {
+	/* non-alpha-numeric characters allowed in names */
+	const char allowed[128] = { ['_'] = 1, ['/'] = 1 };
+
+	for (; *name != '\0'; name++) {
+		if (isalnum(*name))
+			continue;
 		if ((size_t)*name >= RTE_DIM(allowed) || allowed[(int)*name] == 0)
 			return false;
-		name++;
 	}
 	return true;
 }
Signature · 1 line
1.8.3.1
Acked-by on v4 1/1
#35 lore raw

Re: [PATCH v4] telemetry: remove non-portable array initialization syntax

Quoted 10 lines · Tyler Retzlaff

Use of ranges in designated initialization are a non-standard gcc extension.

Only initialize '_' and '/' elements of the array and filter tests of characters through name with standard C isalnum before checking the array.

The array should probably be "static", which was a miss in the original version too.

Quoted 43 lines
---
 lib/telemetry/telemetry_data.c | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)
lib/telemetry/telemetry_data.c +7 -9
index 2bac2de..0dc091a 100644
@@ -2,6 +2,7 @@
  * Copyright(c) 2020 Intel Corporation
  */
 
+#include <ctype.h>
 #include <errno.h>
 #include <stdlib.h>
 #include <inttypes.h>
@@ -152,17 +153,14 @@
 static bool
 valid_name(const char *name)
 {
-	char allowed[128] = {
-			['0' ... '9'] = 1,
-			['A' ... 'Z'] = 1,
-			['a' ... 'z'] = 1,
-			['_'] = 1,
-			['/'] = 1,
-	};
-	while (*name != '\0') {
+	/* non-alpha-numeric characters allowed in names */
+	const char allowed[128] = { ['_'] = 1, ['/'] = 1 };
+
+	for (; *name != '\0'; name++) {
+		if (isalnum(*name))
+			continue;
 		if ((size_t)*name >= RTE_DIM(allowed) || allowed[(int)*name] == 0)
 			return false;
-		name++;
 	}
 	return true;
 }
Signature · 1 line
1.8.3.1
#36 lore raw

Re: [PATCH v4] telemetry: remove non-portable array initialization syntax

Quoted 8 lines · Bruce Richardson
Quoted 7 lines · Tyler Retzlaff

Use of ranges in designated initialization are a non-standard gcc extension.

Only initialize '_' and '/' elements of the array and filter tests of characters through name with standard C isalnum before checking the array.

Quoted 8 lines
Quoted 3 lines

The array should probably be "static", which was a miss in the original version too.

micro optimization trade off image size vs stack usage yeah. i'd be inclined to use static so i can update that.

Quoted 44 lines
Quoted 44 lines
---
 lib/telemetry/telemetry_data.c | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)
lib/telemetry/telemetry_data.c +7 -9
index 2bac2de..0dc091a 100644
@@ -2,6 +2,7 @@
  * Copyright(c) 2020 Intel Corporation
  */
 
+#include <ctype.h>
 #include <errno.h>
 #include <stdlib.h>
 #include <inttypes.h>
@@ -152,17 +153,14 @@
 static bool
 valid_name(const char *name)
 {
-	char allowed[128] = {
-			['0' ... '9'] = 1,
-			['A' ... 'Z'] = 1,
-			['a' ... 'z'] = 1,
-			['_'] = 1,
-			['/'] = 1,
-	};
-	while (*name != '\0') {
+	/* non-alpha-numeric characters allowed in names */
+	const char allowed[128] = { ['_'] = 1, ['/'] = 1 };
+
+	for (; *name != '\0'; name++) {
+		if (isalnum(*name))
+			continue;
 		if ((size_t)*name >= RTE_DIM(allowed) || allowed[(int)*name] == 0)
 			return false;
-		name++;
 	}
 	return true;
 }
Signature · 1 line
1.8.3.1
#37 lore raw

PATCH v5 1/1 improve code portability

In reply to Tyler Retzlaff

Improve portability of telemetry code to allow it to be compiled by msvc unconditionally.

Remove use of designated initialization ranges (gcc extension). Instead use a combination of a trivially initialized array and standard C isalnum.

v5:
* declare allowed array static
* fix alpha-numeric -> alphanumeric (ci warning)

v4:
* Re-implement using isalnum and sparsely populated array.
v3:
* Move int index declaration to top of function
* Fix typo 'Z' -> 'z' in last for loop
v2:
* Initialize valid character array using for loop instead of explicit initialization of each element.
* Drop patch removing VLAs, in a separate series a conditionally compiled change will be introduced.

Tyler Retzlaff (1):
telemetry: remove non-portable array initialization syntax

 lib/telemetry/telemetry_data.c | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)
Signature · 1 line
1.8.3.1
#38 lore raw

PATCH v5 1/1 telemetry: remove non-portable array initialization syntax

Use of ranges in designated initialization are a non-standard gcc extension.

Only initialize '_' and '/' elements of the array and filter tests of characters through name with standard C isalnum before checking the array.

---
 lib/telemetry/telemetry_data.c | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)
lib/telemetry/telemetry_data.c +7 -9
index 2bac2de..0c7187b 100644
@@ -2,6 +2,7 @@
  * Copyright(c) 2020 Intel Corporation
  */
 
+#include <ctype.h>
 #include <errno.h>
 #include <stdlib.h>
 #include <inttypes.h>
@@ -152,17 +153,14 @@
 static bool
 valid_name(const char *name)
 {
-	char allowed[128] = {
-			['0' ... '9'] = 1,
-			['A' ... 'Z'] = 1,
-			['a' ... 'z'] = 1,
-			['_'] = 1,
-			['/'] = 1,
-	};
-	while (*name != '\0') {
+	/* non-alphanumeric characters allowed in names */
+	static const char allowed[128] = { ['_'] = 1, ['/'] = 1 };
+
+	for (; *name != '\0'; name++) {
+		if (isalnum(*name))
+			continue;
 		if ((size_t)*name >= RTE_DIM(allowed) || allowed[(int)*name] == 0)
 			return false;
-		name++;
 	}
 	return true;
 }
Signature · 1 line
1.8.3.1
#39 lore raw

Re: [PATCH v5] telemetry: remove non-portable array initialization syntax

05/04/2023 20:52, Tyler Retzlaff:

Quoted 11 lines

Use of ranges in designated initialization are a non-standard gcc extension.

Only initialize '_' and '/' elements of the array and filter tests of characters through name with standard C isalnum before checking the array.

Applied, thanks.