Skip to contentSkip to conversation

Fix various v4lconvert functions assuming stride == width

PATCH In review Hans de Goede 2 participants 6 messages Oct 16, 2022 – Nov 14, 2022 Media (V4L/DVB)

lore
Share & tools
lore
#0 lore raw

cover Fix various v4lconvert functions assuming stride == width

Hi All,

I (finally) have my videobuf2 conversion of the atomisp2 driver working (I'll post the kernel patches real soon now). This means working MMAP mode, which means I can use e.g. camorama with libv4l2 to do the conversion.

By hacking libv4lconvert to prefer certain formats I have been able to test yuv420, yuyv, rgb565, nv12 and nv16 support in the atomisp2 code. Which has resulted in me finding a bunch of issues inside v4lconvert because the atomisp2 driver generally creates frames where stride != width.

This patch series fixes this. All patches have been tested on actual hw, at least all the input -> rgb24 paths.

Regards,

Hans

Hans de Goede (4):
libv4lconvert: Fix v4lconvert_yuv420_to_rgb/bgr24() not taking stride
into account
libv4lconvert: Fix v4lconvert_rgb565_to_rgb/bgr24() not taking stride
into account
libv4lconvert: Fix v4lconvert_nv12_*() not taking stride into account
libv4lconvert: Fix v4lconvert_nv16_to_yuyv() not taking stride into
account

 lib/libv4lconvert/libv4lconvert-priv.h | 14 +++---
 lib/libv4lconvert/libv4lconvert.c      | 32 ++++++------
 lib/libv4lconvert/rgbyuv.c             | 70 +++++++++++++++++---------
 3 files changed, 69 insertions(+), 47 deletions(-)
Signature · 1 line
2.37.3
#1 lore raw

PATCH 1/4 libv4lconvert: Fix v4lconvert_yuv420_to_rgb/bgr24() not taking stride into account

The atomisp driver can generate V4L2_PIX_FMT_YUV420 buffers where stride != width. Where as v4lconvert_yuv420_to_rgb/bgr24() assumed that stride == width is always true.

Add a stride argument to v4lconvert_yuv420_to_rgb/bgr24() to fix this.

---
 lib/libv4lconvert/libv4lconvert-priv.h |  4 ++--
 lib/libv4lconvert/libv4lconvert.c      | 12 +++++------
 lib/libv4lconvert/rgbyuv.c             | 30 ++++++++++++++++----------
 3 files changed, 27 insertions(+), 19 deletions(-)
lib/libv4lconvert/libv4lconvert-priv.h +2 -2
index 6b9128ce..495f726d 100644
@@ -118,10 +118,10 @@ void v4lconvert_rgb24_to_yuv420(const unsigned char *src, unsigned char *dest,
 		const struct v4l2_format *src_fmt, int bgr, int yvu, int bpp);
 
 void v4lconvert_yuv420_to_rgb24(const unsigned char *src, unsigned char *dst,
-		int width, int height, int yvu);
+		int width, int height, int stride, int yvu);
 
 void v4lconvert_yuv420_to_bgr24(const unsigned char *src, unsigned char *dst,
-		int width, int height, int yvu);
+		int width, int height, int stride, int yvu);
 
 void v4lconvert_yuyv_to_rgb24(const unsigned char *src, unsigned char *dst,
 		int width, int height, int stride);
lib/libv4lconvert/libv4lconvert.c +6 -6
index e794ec00..e5d5ddde 100644
@@ -905,11 +905,11 @@ static int v4lconvert_convert_pixfmt(struct v4lconvert_data *data,
 		switch (dest_pix_fmt) {
 		case V4L2_PIX_FMT_RGB24:
 			v4lconvert_yuv420_to_rgb24(data->convert_pixfmt_buf, dest, width,
-					height, yvu);
+					height, bytesperline, yvu);
 			break;
 		case V4L2_PIX_FMT_BGR24:
 			v4lconvert_yuv420_to_bgr24(data->convert_pixfmt_buf, dest, width,
-					height, yvu);
+					height, bytesperline, yvu);
 			break;
 		}
 		break;
@@ -1398,11 +1398,11 @@ static int v4lconvert_convert_pixfmt(struct v4lconvert_data *data,
 		switch (dest_pix_fmt) {
 		case V4L2_PIX_FMT_RGB24:
 			v4lconvert_yuv420_to_rgb24(src, dest, width,
-					height, 0);
+					height, bytesperline, 0);
 			break;
 		case V4L2_PIX_FMT_BGR24:
 			v4lconvert_yuv420_to_bgr24(src, dest, width,
-					height, 0);
+					height, bytesperline, 0);
 			break;
 		case V4L2_PIX_FMT_YUV420:
 			memcpy(dest, src, width * height * 3 / 2);
@@ -1422,11 +1422,11 @@ static int v4lconvert_convert_pixfmt(struct v4lconvert_data *data,
 		switch (dest_pix_fmt) {
 		case V4L2_PIX_FMT_RGB24:
 			v4lconvert_yuv420_to_rgb24(src, dest, width,
-					height, 1);
+					height, bytesperline, 1);
 			break;
 		case V4L2_PIX_FMT_BGR24:
 			v4lconvert_yuv420_to_bgr24(src, dest, width,
-					height, 1);
+					height, bytesperline, 1);
 			break;
 		case V4L2_PIX_FMT_YUV420:
 			v4lconvert_swap_uv(src, dest, fmt);
lib/libv4lconvert/rgbyuv.c +19 -11
index b54b4577..1ca821ab 100644
@@ -93,7 +93,7 @@ void v4lconvert_rgb24_to_yuv420(const unsigned char *src, unsigned char *dest,
 #define CLIP(color) (unsigned char)(((color) > 0xFF) ? 0xff : (((color) < 0) ? 0 : (color)))
 
 void v4lconvert_yuv420_to_bgr24(const unsigned char *src, unsigned char *dest,
-		int width, int height, int yvu)
+		int width, int height, int stride, int yvu)
 {
 	int i, j;
 
@@ -101,11 +101,11 @@ void v4lconvert_yuv420_to_bgr24(const unsigned char *src, unsigned char *dest,
 	const unsigned char *usrc, *vsrc;
 
 	if (yvu) {
-		vsrc = src + width * height;
-		usrc = vsrc + (width * height) / 4;
+		vsrc = src + stride * height;
+		usrc = vsrc + (stride * height) / 4;
 	} else {
-		usrc = src + width * height;
-		vsrc = usrc + (width * height) / 4;
+		usrc = src + stride * height;
+		vsrc = usrc + (stride * height) / 4;
 	}
 
 	for (i = 0; i < height; i++) {
@@ -138,16 +138,20 @@ void v4lconvert_yuv420_to_bgr24(const unsigned char *src, unsigned char *dest,
 			usrc++;
 			vsrc++;
 		}
+		ysrc += stride - width;
 		/* Rewind u and v for next line */
 		if (!(i & 1)) {
 			usrc -= width / 2;
 			vsrc -= width / 2;
+		} else {
+			usrc += (stride - width) / 2;
+			vsrc += (stride - width) / 2;
 		}
 	}
 }
 
 void v4lconvert_yuv420_to_rgb24(const unsigned char *src, unsigned char *dest,
-		int width, int height, int yvu)
+		int width, int height, int stride, int yvu)
 {
 	int i, j;
 
@@ -155,11 +159,11 @@ void v4lconvert_yuv420_to_rgb24(const unsigned char *src, unsigned char *dest,
 	const unsigned char *usrc, *vsrc;
 
 	if (yvu) {
-		vsrc = src + width * height;
-		usrc = vsrc + (width * height) / 4;
+		vsrc = src + stride * height;
+		usrc = vsrc + (stride * height) / 4;
 	} else {
-		usrc = src + width * height;
-		vsrc = usrc + (width * height) / 4;
+		usrc = src + stride * height;
+		vsrc = usrc + (stride * height) / 4;
 	}
 
 	for (i = 0; i < height; i++) {
@@ -192,10 +196,14 @@ void v4lconvert_yuv420_to_rgb24(const unsigned char *src, unsigned char *dest,
 			usrc++;
 			vsrc++;
 		}
+		ysrc += stride - width;
 		/* Rewind u and v for next line */
-		if (!(i&1)) {
+		if (!(i & 1)) {
 			usrc -= width / 2;
 			vsrc -= width / 2;
+		} else {
+			usrc += (stride - width) / 2;
+			vsrc += (stride - width) / 2;
 		}
 	}
 }
Signature · 1 line
2.37.3
#2 lore raw

PATCH 2/4 libv4lconvert: Fix v4lconvert_rgb565_to_rgb/bgr24() not taking stride into account

In reply to Hans de Goede

The atomisp driver can generate V4L2_PIX_FMT_RGB565 buffers where stride != width. Where as v4lconvert_rgb565_to_rgb/bgr24() assumed that stride == width is always true.

Add a stride argument to v4lconvert_rgb565_to_rgb/bgr24() to fix this.

---
 lib/libv4lconvert/libv4lconvert-priv.h | 4 ++--
 lib/libv4lconvert/libv4lconvert.c      | 4 ++--
 lib/libv4lconvert/rgbyuv.c             | 6 ++++--
 3 files changed, 8 insertions(+), 6 deletions(-)
lib/libv4lconvert/libv4lconvert-priv.h +2 -2
index 495f726d..f87a43a4 100644
@@ -178,10 +178,10 @@ int v4lconvert_y10b_to_yuv420(struct v4lconvert_data *data,
 	const unsigned char *src, unsigned char *dest, int width, int height);
 
 void v4lconvert_rgb565_to_rgb24(const unsigned char *src, unsigned char *dest,
-		int width, int height);
+		int width, int height, int stride);
 
 void v4lconvert_rgb565_to_bgr24(const unsigned char *src, unsigned char *dest,
-		int width, int height);
+		int width, int height, int stride);
 
 void v4lconvert_rgb565_to_yuv420(const unsigned char *src, unsigned char *dest,
 		const struct v4l2_format *src_fmt, int yvu);
lib/libv4lconvert/libv4lconvert.c +2 -2
index e5d5ddde..77f9eca5 100644
@@ -1282,10 +1282,10 @@ static int v4lconvert_convert_pixfmt(struct v4lconvert_data *data,
 		}
 		switch (dest_pix_fmt) {
 		case V4L2_PIX_FMT_RGB24:
-			v4lconvert_rgb565_to_rgb24(src, dest, width, height);
+			v4lconvert_rgb565_to_rgb24(src, dest, width, height, bytesperline);
 			break;
 		case V4L2_PIX_FMT_BGR24:
-			v4lconvert_rgb565_to_bgr24(src, dest, width, height);
+			v4lconvert_rgb565_to_bgr24(src, dest, width, height, bytesperline);
 			break;
 		case V4L2_PIX_FMT_YUV420:
 			v4lconvert_rgb565_to_yuv420(src, dest, fmt, 0);
lib/libv4lconvert/rgbyuv.c +4 -2
index 1ca821ab..f9017016 100644
@@ -511,7 +511,7 @@ void v4lconvert_swap_uv(const unsigned char *src, unsigned char *dest,
 }
 
 void v4lconvert_rgb565_to_rgb24(const unsigned char *src, unsigned char *dest,
-		int width, int height)
+		int width, int height, int stride)
 {
 	int j;
 	while (--height >= 0) {
@@ -525,11 +525,12 @@ void v4lconvert_rgb565_to_rgb24(const unsigned char *src, unsigned char *dest,
 
 			src += 2;
 		}
+		src += stride - 2 * width;
 	}
 }
 
 void v4lconvert_rgb565_to_bgr24(const unsigned char *src, unsigned char *dest,
-		int width, int height)
+		int width, int height, int stride)
 {
 	int j;
 	while (--height >= 0) {
@@ -543,6 +544,7 @@ void v4lconvert_rgb565_to_bgr24(const unsigned char *src, unsigned char *dest,
 
 			src += 2;
 		}
+		src += stride - 2 * width;
 	}
 }
 
Signature · 1 line
2.37.3
#3 lore raw

PATCH 3/4 libv4lconvert: Fix v4lconvert_nv12_*() not taking stride into account

In reply to Hans de Goede

The atomisp driver can generate V4L2_PIX_FMT_NV12 buffers where stride != width. Where as v4lconvert_nv12_*() assumed that stride == width is always true.

Add a stride argument to v4lconvert_nv12_*() to fix this.

---
 lib/libv4lconvert/libv4lconvert-priv.h |  4 ++--
 lib/libv4lconvert/libv4lconvert.c      |  8 ++++----
 lib/libv4lconvert/rgbyuv.c             | 18 +++++++++++++-----
 3 files changed, 19 insertions(+), 11 deletions(-)
lib/libv4lconvert/libv4lconvert-priv.h +2 -2
index f87a43a4..f361f2a0 100644
@@ -287,10 +287,10 @@ void v4lconvert_hsv_to_rgb24(const unsigned char *src, unsigned char *dest,
 		int width, int height, int bgr, int Xin, unsigned char hsv_enc);
 
 void v4lconvert_nv12_to_rgb24(const unsigned char *src, unsigned char *dest,
-		int width, int height, int bgr);
+		int width, int height, int stride, int bgr);
 
 void v4lconvert_nv12_to_yuv420(const unsigned char *src, unsigned char *dest,
-		int width, int height, int yvu);
+		int width, int height, int stride, int yvu);
 
 void v4lconvert_rotate90(unsigned char *src, unsigned char *dest,
 		struct v4l2_format *fmt);
lib/libv4lconvert/libv4lconvert.c +4 -4
index 77f9eca5..d0d38286 100644
@@ -937,16 +937,16 @@ static int v4lconvert_convert_pixfmt(struct v4lconvert_data *data,
 	case V4L2_PIX_FMT_NV12:
 		switch (dest_pix_fmt) {
 		case V4L2_PIX_FMT_RGB24:
-			v4lconvert_nv12_to_rgb24(src, dest, width, height, 0);
+			v4lconvert_nv12_to_rgb24(src, dest, width, height, bytesperline, 0);
 			break;
 		case V4L2_PIX_FMT_BGR24:
-			v4lconvert_nv12_to_rgb24(src, dest, width, height, 1);
+			v4lconvert_nv12_to_rgb24(src, dest, width, height, bytesperline, 1);
 			break;
 		case V4L2_PIX_FMT_YUV420:
-			v4lconvert_nv12_to_yuv420(src, dest, width, height, 0);
+			v4lconvert_nv12_to_yuv420(src, dest, width, height, bytesperline, 0);
 			break;
 		case V4L2_PIX_FMT_YVU420:
-			v4lconvert_nv12_to_yuv420(src, dest, width, height, 1);
+			v4lconvert_nv12_to_yuv420(src, dest, width, height, bytesperline, 1);
 			break;
 		}
 		break;
lib/libv4lconvert/rgbyuv.c +13 -5
index f9017016..e9fe6df9 100644
@@ -857,11 +857,11 @@ void v4lconvert_hsv_to_rgb24(const unsigned char *src, unsigned char *dest,
 }
 
 void v4lconvert_nv12_to_rgb24(const unsigned char *src, unsigned char *dest,
-		int width, int height, int bgr)
+		int width, int height, int stride, int bgr)
 {
 	int i, j;
 	const unsigned char *ysrc = src;
-	const unsigned char *uvsrc = src + width * height;
+	const unsigned char *uvsrc = src + stride * height;
 
 	for (i = 0; i < height; i++) {
 		for (j = 0; j < width; j ++) {
@@ -879,18 +879,21 @@ void v4lconvert_nv12_to_rgb24(const unsigned char *src, unsigned char *dest,
 				uvsrc += 2;
 		}
 
+		ysrc += stride - width;
 		/* Rewind u and v for next line */
 		if (!(i&1))
 			uvsrc -= width;
+		else
+			uvsrc += stride - width;
 	}
 }
 
 void v4lconvert_nv12_to_yuv420(const unsigned char *src, unsigned char *dest,
-		int width, int height, int yvu)
+		int width, int height, int stride, int yvu)
 {
 	int i, j;
 	const unsigned char *ysrc = src;
-	const unsigned char *uvsrc = src + width * height;
+	const unsigned char *uvsrc = src + stride * height;
 	unsigned char *ydst = dest;
 	unsigned char *udst, *vdst;
 
@@ -902,7 +905,7 @@ void v4lconvert_nv12_to_yuv420(const unsigned char *src, unsigned char *dest,
 		vdst = udst + ((width / 2) * (height / 2));
 	}
 
-	for (i = 0; i < height; i++)
+	for (i = 0; i < height; i++) {
 		for (j = 0; j < width; j++) {
 			*ydst++ = *ysrc++;
 			if (((i % 2) == 0) && ((j % 2) == 0)) {
@@ -910,4 +913,9 @@ void v4lconvert_nv12_to_yuv420(const unsigned char *src, unsigned char *dest,
 				*vdst++ = *uvsrc++;
 			}
 		}
+
+		ysrc += stride - width;
+		if ((i % 2) == 0)
+			uvsrc += stride - width;
+	}
 }
Signature · 1 line
2.37.3
#4 lore raw

PATCH 4/4 libv4lconvert: Fix v4lconvert_nv16_to_yuyv() not taking stride into account

In reply to Hans de Goede

The atomisp driver can generate V4L2_PIX_FMT_NV16 buffers where stride != width. Where as v4lconvert_nv16_to_yuyv() assumed that stride == width is always true.

Add a stride argument to v4lconvert_nv16_to_yuyv() to fix this.

---
 lib/libv4lconvert/libv4lconvert-priv.h |  2 +-
 lib/libv4lconvert/libv4lconvert.c      |  8 ++++----
 lib/libv4lconvert/rgbyuv.c             | 16 ++++++++++------
 3 files changed, 15 insertions(+), 11 deletions(-)
lib/libv4lconvert/libv4lconvert-priv.h +1 -1
index f361f2a0..00a03f9e 100644
@@ -133,7 +133,7 @@ void v4lconvert_yuyv_to_yuv420(const unsigned char *src, unsigned char *dst,
 		int width, int height, int stride, int yvu);
 
 void v4lconvert_nv16_to_yuyv(const unsigned char *src, unsigned char *dest,
-		int width, int height);
+		int width, int height, int stride);
 
 void v4lconvert_yvyu_to_rgb24(const unsigned char *src, unsigned char *dst,
 		int width, int height, int stride);
lib/libv4lconvert/libv4lconvert.c +4 -4
index d0d38286..b07bf3ba 100644
@@ -1445,10 +1445,10 @@ static int v4lconvert_convert_pixfmt(struct v4lconvert_data *data,
 		if (!tmpbuf)
 			return v4lconvert_oom_error(data);
 
-		v4lconvert_nv16_to_yuyv(src, tmpbuf, width, height);
+		v4lconvert_nv16_to_yuyv(src, tmpbuf, width, height, bytesperline);
 		src_pix_fmt = V4L2_PIX_FMT_YUYV;
 		src = tmpbuf;
-		bytesperline = bytesperline * 2;
+		bytesperline = width * 2;
 		/* fall through */
 	}
 	case V4L2_PIX_FMT_YUYV:
@@ -1482,10 +1482,10 @@ static int v4lconvert_convert_pixfmt(struct v4lconvert_data *data,
 			return v4lconvert_oom_error(data);
 
 		/* Note NV61 is NV16 with U and V swapped so this becomes yvyu. */
-		v4lconvert_nv16_to_yuyv(src, tmpbuf, width, height);
+		v4lconvert_nv16_to_yuyv(src, tmpbuf, width, height, bytesperline);
 		src_pix_fmt = V4L2_PIX_FMT_YVYU;
 		src = tmpbuf;
-		bytesperline = bytesperline * 2;
+		bytesperline = width * 2;
 		/* fall through */
 	}
 	case V4L2_PIX_FMT_YVYU:
lib/libv4lconvert/rgbyuv.c +10 -6
index e9fe6df9..ce31a1ba 100644
@@ -304,17 +304,21 @@ void v4lconvert_yuyv_to_yuv420(const unsigned char *src, unsigned char *dest,
 }
 
 void v4lconvert_nv16_to_yuyv(const unsigned char *src, unsigned char *dest,
-		int width, int height)
+		int width, int height, int stride)
 {
 	const unsigned char *y, *cbcr;
-	int count = 0;
+	int i, j;
 
 	y = src;
-	cbcr = src + width*height;
+	cbcr = src + stride * height;
 
-	while (count++ < width*height) {
-		*dest++ = *y++;
-		*dest++ = *cbcr++;
+	for (i = 0; i < height; i++) {
+		for (j = 0; j < width; j++) {
+			*dest++ = *y++;
+			*dest++ = *cbcr++;
+		}
+		y += stride - width;
+		cbcr += stride - width;
 	}
 }
 
Signature · 1 line
2.37.3
#5 lore raw
In reply to Hans de Goede

Em Sun, 16 Oct 2022 19:57:25 +0200
Hans de Goede <[email protected]> escreveu:

Quoted 19 lines

Hi All,

I (finally) have my videobuf2 conversion of the atomisp2 driver working (I'll post the kernel patches real soon now). This means working MMAP mode, which means I can use e.g. camorama with libv4l2 to do the conversion.

By hacking libv4lconvert to prefer certain formats I have been able to test yuv420, yuyv, rgb565, nv12 and nv16 support in the atomisp2 code. Which has resulted in me finding a bunch of issues inside v4lconvert because the atomisp2 driver generally creates frames where stride != width.

This patch series fixes this. All patches have been tested on actual hw, at least all the input -> rgb24 paths.

Regards,

Hans

Quoted 8 lines

Hans de Goede (4):
libv4lconvert: Fix v4lconvert_yuv420_to_rgb/bgr24() not taking stride
into account
libv4lconvert: Fix v4lconvert_rgb565_to_rgb/bgr24() not taking stride
into account
libv4lconvert: Fix v4lconvert_nv12_*() not taking stride into account
libv4lconvert: Fix v4lconvert_nv16_to_yuyv() not taking stride into
account

Merged those on master:

0c96e7ca0266 (HEAD -> master, origin/master, origin/HEAD) libv4lconvert: Fix v4lconvert_nv16_to_yuyv() not taking stride into account e8b224d4be7e libv4lconvert: Fix v4lconvert_nv12_*() not taking stride into account
aecfcfccfc2f libv4lconvert: Fix v4lconvert_rgb565_to_rgb/bgr24() not taking stride into account
6269f88db0d3 libv4lconvert: Fix v4lconvert_yuv420_to_rgb/bgr24() not taking stride into account

I also merged them on 1.18, 1.20 and 1.22, as they're bug fixes and affects a real hardware. I was able to test only on 1.22 and master, as older versions don't build on Fedora 36.

Regards,
Mauro