Intel IPU3 ImgU patchset

Add IPU3-specific meta formats for parameter
processing and 3A, DVS statistics:

V4L2_META_FMT_IPU3_PARAMS
V4L2_META_FMT_IPU3_STAT_3A

Signed-off-by Yong Zhi <[email protected]>

---
 drivers/media/v4l2-core/v4l2-ioctl.c | 2 ++
 include/uapi/linux/videodev2.h       | 4 ++++
 2 files changed, 6 insertions(+)

drivers/media/v4l2-core/v4l2-ioctl.c +2 -0

index 6489f25..abff64b 100644
@@ -1299,6 +1299,8 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt)
 	case V4L2_META_FMT_VSP1_HGO:	descr = "R-Car VSP1 1-D Histogram"; break;
 	case V4L2_META_FMT_VSP1_HGT:	descr = "R-Car VSP1 2-D Histogram"; break;
 	case V4L2_META_FMT_UVC:		descr = "UVC payload header metadata"; break;
+	case V4L2_META_FMT_IPU3_PARAMS:	descr = "IPU3 processing parameters"; break;
+	case V4L2_META_FMT_IPU3_STAT_3A:	descr = "IPU3 3A statistics"; break;
 
 	default:
 		/* Compressed formats */

include/uapi/linux/videodev2.h +0 -0

index f0a968a..bdccd7a 100644

Hi Zhi-san,

Em Mon, 29 Oct 2018 15:22:55 -0700
Yong Zhi <[email protected]> escreveu:

Quoted 33 lines

Add IPU3-specific meta formats for parameter
processing and 3A, DVS statistics:

V4L2_META_FMT_IPU3_PARAMS
V4L2_META_FMT_IPU3_STAT_3A

Signed-off-by Yong Zhi <[email protected]>

---
 drivers/media/v4l2-core/v4l2-ioctl.c | 2 ++
 include/uapi/linux/videodev2.h       | 4 ++++
 2 files changed, 6 insertions(+)

drivers/media/v4l2-core/v4l2-ioctl.c +2 -0

index 6489f25..abff64b 100644
@@ -1299,6 +1299,8 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt)
 	case V4L2_META_FMT_VSP1_HGO:	descr = "R-Car VSP1 1-D Histogram"; break;
 	case V4L2_META_FMT_VSP1_HGT:	descr = "R-Car VSP1 2-D Histogram"; break;
 	case V4L2_META_FMT_UVC:		descr = "UVC payload header metadata"; break;
+	case V4L2_META_FMT_IPU3_PARAMS:	descr = "IPU3 processing parameters"; break;
+	case V4L2_META_FMT_IPU3_STAT_3A:	descr = "IPU3 3A statistics"; break;
 
 	default:
 		/* Compressed formats */

include/uapi/linux/videodev2.h +0 -0

index f0a968a..bdccd7a 100644
@@ -718,6 +718,10 @@ struct v4l2_pix_format {
 #define V4L2_META_FMT_UVC         v4l2_fourcc('U', 'V', 'C', 'H') /* UVC Payload Header metadata */
 #define V4L2_META_FMT_D4XX        v4l2_fourcc('D', '4', 'X', 'X') /* D4XX Payload Header metadata */
 

include/uapi/linux/videodev2.h struct v4l2_pix_format { · line 723

Quoted 3 lines

include/uapi/linux/videodev2.h +3 -0

+/* Vendor specific - used for IPU3 camera sub-system */
+#define V4L2_META_FMT_IPU3_PARAMS	v4l2_fourcc('i', 'p', '3', 'p') /* IPU3 params */
+#define V4L2_META_FMT_IPU3_STAT_3A	v4l2_fourcc('i', 'p', '3', 's') /* IPU3 3A statistics */

Where's the documentation for those two new formats? The best is to always add the documentation bits for V4L2 uAPI stuff at the same patch, as it makes easier for us to review.

include/uapi/linux/videodev2.h struct v4l2_pix_format { · line 727

Quoted 5 lines

include/uapi/linux/videodev2.h +1 -0

+
 /* priv field value to indicates that subsequent fields are valid. */
 #define V4L2_PIX_FMT_PRIV_MAGIC		0xfeedcafe
 

Thanks,
Mauro

Add IPU3-specific meta formats for parameter
processing and 3A, DVS statistics:

V4L2_META_FMT_IPU3_PARAMS
V4L2_META_FMT_IPU3_STAT_3A

Signed-off-by Yong Zhi <[email protected]>

---
 drivers/media/v4l2-core/v4l2-ioctl.c | 2 ++
 include/uapi/linux/videodev2.h       | 4 ++++
 2 files changed, 6 insertions(+)

(quoted diff) +0 -0

b/drivers/media/v4l2-core/v4l2-ioctl.c index 6489f25..abff64b 100644

drivers/media/v4l2-core/v4l2-ioctl.c +2 -0

@@ -1299,6 +1299,8 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt)
 case V4L2_META_FMT_VSP1_HGO:	descr = "R-Car VSP1 1-D Histogram"; break;
 case V4L2_META_FMT_VSP1_HGT:	descr = "R-Car VSP1 2-D Histogram"; break;
 case V4L2_META_FMT_UVC:		descr = "UVC payload header metadata"; break;
+	case V4L2_META_FMT_IPU3_PARAMS:	descr = "IPU3 processing parameters";
 break;
+	case V4L2_META_FMT_IPU3_STAT_3A:	descr = "IPU3 3A statistics"; break;
 
 	default:

drivers/media/v4l2-core/v4l2-ioctl.c +0 -0

 		/* Compressed formats */

include/uapi/linux/videodev2.h +0 -0

index f0a968a..bdccd7a 100644
@@ -718,6 +718,10 @@ struct v4l2_pix_format {
 #define V4L2_META_FMT_UVC         v4l2_fourcc('U', 'V', 'C', 'H') /* UVC

Payload Header metadata */ #define V4L2_META_FMT_D4XX v4l2_fourcc('D', '4', 'X', 'X') /* D4XX Payload Header metadata */

include/uapi/linux/videodev2.h +2 -0

+/* Vendor specific - used for IPU3 camera sub-system */
+#define V4L2_META_FMT_IPU3_PARAMS	v4l2_fourcc('i', 'p', '3', 'p') /* IPU3

params */

+#define V4L2_META_FMT_IPU3_STAT_3A v4l2_fourcc('i', 'p', '3', 's') /* IPU3
3A statistics */
+
/* priv field value to indicates that subsequent fields are valid. */
#define V4L2_PIX_FMT_PRIV_MAGIC 0xfeedcafe

-----Original Message-----
From: Laurent Pinchart [mailto:[email protected]]
Sent: Thursday, November 29, 2018 1:17 PM
To: Zhi, Yong <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; Mani, Rajmohan <[email protected]>; Zheng, Jian Xu <[email protected]>; Hu, Jerry W <[email protected]>; Toivonen, Tuukka <[email protected]>; Qiu, Tian Shu <[email protected]>; Cao, Bingbu <[email protected]>
Subject: Re: [PATCH v7 01/16] v4l: Add Intel IPU3 meta buffer formats

Hello Yong,

Thank you for the patch.

Quoted 2 lines · On Tuesday, 30 October 2018 00:22:55 EET Yong Zhi wrote:

Add IPU3-specific meta formats for parameter processing and 3A, DVS statistics:

Unless I'm mistaken DVS support has been removed. You can write this as

Add IPU3-specific meta formats for processing parameters and 3A statistics.

Quoted 4 lines

V4L2_META_FMT_IPU3_PARAMS
V4L2_META_FMT_IPU3_STAT_3A

Quoted 5 lines

Signed-off-by Yong Zhi <[email protected]>

---
 drivers/media/v4l2-core/v4l2-ioctl.c | 2 ++
 include/uapi/linux/videodev2.h       | 4 ++++
 2 files changed, 6 insertions(+)

I would squash this with patch 03/16.

drivers/media/v4l2-core/v4l2-ioctl.c static void v4l_fill_fmtdesc(struct v4l2_fmtdesc

Quoted 5 lines

(quoted diff) +0 -0

b/drivers/media/v4l2-core/v4l2-ioctl.c index 6489f25..abff64b 100644

drivers/media/v4l2-core/v4l2-ioctl.c +0 -0

@@ -1299,6 +1299,8 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc

*fmt)

drivers/media/v4l2-core/v4l2-ioctl.c static void v4l_fill_fmtdesc(struct v4l2_fmtdesc · line 1299

Quoted 1 line

drivers/media/v4l2-core/v4l2-ioctl.c +0 -0

 case V4L2_META_FMT_VSP1_HGO:	descr = "R-Car VSP1 1-D Histogram";

break;

drivers/media/v4l2-core/v4l2-ioctl.c static void v4l_fill_fmtdesc(struct v4l2_fmtdesc · line 1300

Quoted 1 line

drivers/media/v4l2-core/v4l2-ioctl.c +0 -0

 case V4L2_META_FMT_VSP1_HGT:	descr = "R-Car VSP1 2-D Histogram";

break;

Quoted 1 line

case V4L2_META_FMT_UVC: descr = "UVC payload header

metadata"; break;

drivers/media/v4l2-core/v4l2-ioctl.c

Quoted 1 line

drivers/media/v4l2-core/v4l2-ioctl.c +1 -0

+	case V4L2_META_FMT_IPU3_PARAMS:	descr = "IPU3 processing

parameters";

drivers/media/v4l2-core/v4l2-ioctl.c

Quoted 2 lines

break;

drivers/media/v4l2-core/v4l2-ioctl.c +1 -0

+	case V4L2_META_FMT_IPU3_STAT_3A:	descr = "IPU3 3A statistics";

break;

Quoted 12 lines

drivers/media/v4l2-core/v4l2-ioctl.c +0 -0

 	default:
 		/* Compressed formats */

(quoted diff) +0 -0

b/include/uapi/linux/videodev2.h index f0a968a..bdccd7a 100644

include/uapi/linux/videodev2.h +0 -0

@@ -718,6 +718,10 @@ struct v4l2_pix_format {
 #define V4L2_META_FMT_UVC         v4l2_fourcc('U', 'V', 'C', 'H') /* UVC

Payload Header metadata */ #define V4L2_META_FMT_D4XX v4l2_fourcc('D', '4', 'X', 'X') /* D4XX Payload Header metadata */

include/uapi/linux/videodev2.h

Quoted 2 lines

include/uapi/linux/videodev2.h +2 -0

+/* Vendor specific - used for IPU3 camera sub-system */
+#define V4L2_META_FMT_IPU3_PARAMS	v4l2_fourcc('i', 'p', '3', 'p') /*

IPU3

Quoted 1 line

params */

Maybe "IPU3 processing parameters" in full ?

Apart from that the patch looks good to me.

Reviewed-by Laurent Pinchart <[email protected]>

Quoted 1 line

+#define V4L2_META_FMT_IPU3_STAT_3A v4l2_fourcc('i', 'p', '3', 's') /*

IPU3

Quoted 4 lines

3A statistics */
+
/* priv field value to indicates that subsequent fields are valid. */
#define V4L2_PIX_FMT_PRIV_MAGIC 0xfeedcafe

--
Regards,

Laurent Pinchart

From: Rajmohan Mani <[email protected]>

This patch adds the details about the IPU3 Imaging Unit driver.

Change-Id: I560cecf673df2dcc3ec72767cf8077708d649656

Signed-off-by Rajmohan Mani <[email protected]>

---
 Documentation/media/v4l-drivers/index.rst |   1 +
 Documentation/media/v4l-drivers/ipu3.rst  | 326 +++++++++++++++++++++++++++
 2 files changed, 327 insertions(+)
 create mode 100644 Documentation/media/v4l-drivers/ipu3.rst

(quoted diff) +0 -0

b/Documentation/media/v4l-drivers/index.rst index 679238e..179a393 100644

Documentation/media/v4l-drivers/index.rst +0 -0

@@ -44,6 +44,7 @@ For more details see the file COPYING in the source

distribution of Linux. davinci-vpbe

Documentation/media/v4l-drivers/index.rst +1 -0

 	fimc
 	imx
+	ipu3
 	ivtv
 	max2175
 	meye

(quoted diff) +0 -0

b/Documentation/media/v4l-drivers/ipu3.rst new file mode 100644 index 0000000..045bf42

Documentation/media/v4l-drivers/ipu3.rst new file +12 -0

@@ -0,0 +1,326 @@
+.. include:: <isonum.txt>
+
+===============================================================
+Intel Image Processing Unit 3 (IPU3) Imaging Unit (ImgU) driver
+===============================================================
+
+Copyright |copy| 2018 Intel Corporation
+
+Introduction
+============
+
+This file documents Intel IPU3 (3rd generation Image Processing Unit)

Imaging

Documentation/media/v4l-drivers/ipu3.rst new file +2 -0

+
+The CIO2 device receives the raw bayer data from the sensors and outputs

the +frames in a format that is specific to IPU3 (for consumption by IPU3 ImgU).

+CIO2 driver is available as drivers/media/pci/intel/ipu3/ipu3-cio2* and is +enabled through the CONFIG_VIDEO_IPU3_CIO2 config option.

+through IPU3 CIO2 device. The ImgU driver sources can be found under

+The driver implements V4L2, Media controller and V4L2 sub-device interfaces.

Support for lens and flash drivers depends on the
+above sensors.

Documentation/media/v4l-drivers/ipu3.rst new file +4 -0

+CIO2 device
+===========
+
+The CIO2 is represented as a single V4L2 subdev, which provides a V4L2

subdev +interface to the user space. There is a video node for each CSI-2 receiver, +with a single media controller interface for the entire device.

Documentation/media/v4l-drivers/ipu3.rst new file +4 -0

+Media controller
+----------------
+
+The media device interface allows to configure the ImgU links, which

defines +the behavior of the IPU3 firmware.

+format), IPU3 ISP starts processing the buffer and produces the video

output +in YUV format and statistics output on respective output nodes. The driver +is expected to have buffers ready for all of parameter, output and +statistics nodes, when input video node is queued with buffer.

+input video node is processed by the IPU3 Imaging Unit and is output to 2

+video nodes, with each targeting different purpose (main output and

viewfinder +output).

+Documentation/media/uapi/v4l/pixfmt-meta-intel-ipu3.rst

Documentation/media/v4l-drivers/ipu3.rst new file +7 -0

+Configuring the Intel IPU3
+==========================
+
+The Intel IPU3 ImgU driver supports V4L2 interface. Using V4L2 ioctl calls,
+the ISP can be configured and enabled.
+
+The IPU3 ImgU pipelines can be configured using media controller APIs,

+media-ctl -d $MDEV -l "ipu3-csi2 0":1 -> "ipu3-cio2 0":0[1]

Documentation/media/v4l-drivers/ipu3.rst new file +16 -0

+media-ctl -d $MDEV -V "ipu3-csi2 0":0 [fmt:SGRBG10/2592x1944]
+
+media-ctl -d $MDEV -V "ipu3-csi2 0":1 [fmt:SGRBG10/2592x1944]
+
+Once the media pipeline is configured, desired sensor specific settings
+(such as exposure and gain settings) can be set, using the yavta tool.
+
+e.g
+
+yavta -w 0x009e0903 444 $(media-ctl -d $MDEV -e "$SDEV")
+
+yavta -w 0x009e0913 1024 $(media-ctl -d $MDEV -e "$SDEV")
+
+yavta -w 0x009e0911 2046 $(media-ctl -d $MDEV -e "$SDEV")
+
+Once the desired sensor settings are set, frame captures can be done as

below.

+-f IPU3_GRBG10 media-ctl -d $MDEV -e ipu3-cio2 0

Documentation/media/v4l-drivers/ipu3.rst new file +2 -0

+
+With the above command, 10 frames are captured at 2592x1944 resolution,

with +sGRBG10 format and output as IPU3_GRBG10 format.

Documentation/media/v4l-drivers/ipu3.rst new file +10 -0

+Processing the image in raw bayer format
+----------------------------------------
+
+Configuring ImgU V4L2 subdev for image processing
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ImgU V4L2 subdevs have to be configured with media controller APIs to
+have all the video nodes setup correctly.
+
+Let us take "ipu3-imgu 0" subdev as an example.

Documentation/media/v4l-drivers/ipu3.rst new file +4 -0

+The following steps prepare the ImgU ISP pipeline for the image processing.
+
+1. The ImgU V4L2 subdev data format should be set by using the
+VIDIOC_SUBDEV_S_FMT on pad 0, using the GDC width and height obtained

above.

Documentation/media/v4l-drivers/ipu3.rst new file +7 -0

+
+2. The ImgU V4L2 subdev cropping should be set by using the
+VIDIOC_SUBDEV_S_SELECTION on pad 0, with V4L2_SEL_TGT_CROP as the target,
+using the input feeder height and width.
+
+3. The ImgU V4L2 subdev composing should be set by using the
+VIDIOC_SUBDEV_S_SELECTION on pad 0, with V4L2_SEL_TGT_COMPOSE as the

target, +using the BDS height and width.

Documentation/media/v4l-drivers/ipu3.rst new file +5 -0

+Once this is done, the received raw bayer frames can be input to the ImgU
+V4L2 subdev as below, using the open source application v4l2n.
+
+For an image captured with 2592x1944 [#f4]_ resolution, with desired output
+resolution as 2560x1920 and viewfinder resolution as 2560x1920, the

following +v4l2n command can be used. This helps process the raw bayer frames and +produces the desired results for the main output image and the viewfinder +output, in NV12 format.

Documentation/media/v4l-drivers/ipu3.rst new file +3 -0

+
+v4l2n --pipe=4 --load=/tmp/frame-#.bin --open=/dev/video4
+--fmt=type:VIDEO_OUTPUT_MPLANE,width=2592,height=1944,pixelformat=0X4733706

9 +--reqbufs=type:VIDEO_OUTPUT_MPLANE,count:1 --pipe=1 --output=/tmp/frames.out +--open=/dev/video5

+--fmt=type:VIDEO_CAPTURE_MPLANE,width=2560,height=1920,pixelformat=NV12
+--reqbufs=type:VIDEO_CAPTURE_MPLANE,count:1 --pipe=2
--output=/tmp/frames.vf +--open=/dev/video6
+--fmt=type:VIDEO_CAPTURE_MPLANE,width=2560,height=1920,pixelformat=NV12
+--reqbufs=type:VIDEO_CAPTURE_MPLANE,count:1 --pipe=3 --open=/dev/video7
+--output=/tmp/frames.3A --fmt=type:META_CAPTURE,?
+--reqbufs=count:1,type:META_CAPTURE --pipe=1,2,3,4 --stream=5

Documentation/media/v4l-drivers/ipu3.rst new file +32 -0

+where 2560x1920 is output resolution, NV12 is the video format, followed
+by input frame and output PNM file.
+
+Viewfinder output frames
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+raw2pnm -x2560 -y1920 -fNV12 /tmp/frames.vf /tmp/frames.vf.pnm
+
+where 2560x1920 is output resolution, NV12 is the video format, followed
+by input frame and output PNM file.
+
+Example user space code for IPU3
+================================
+
+User space code that configures and uses IPU3 is available here.
+
+https://chromium.googlesource.com/chromiumos/platform/arc-camera/+/master/
+
+The source can be located under hal/intel directory.
+
+References
+==========
+
+include/uapi/linux/intel-ipu3.h
+
+.. [#f1] https://github.com/intel/nvt
+
+.. [#f2] http://git.ideasonboard.org/yavta.git
+
+.. [#f3] http://git.ideasonboard.org/?p=media-ctl.git;a=summary
+
+.. [#f4] ImgU limitation requires an additional 16x16 for all input

resolutions

Hello Yong,

Thank you for the patch.

Quoted 2 lines · On Tuesday, 30 October 2018 00:22:56 EET Yong Zhi wrote:

From: Rajmohan Mani <[email protected]>

Quoted 1 line

This patch adds the details about the IPU3 Imaging Unit driver.

Strictly speaking this documents both the CIO2 and the IMGU. As they're handled by two separate drivers, should they be split in two separate files ? If you prefer keeping them together you should update the commit message accordingly. I would in that case also split the documentation in a CIO2 and a IMGU section in the file, instead of mixing them.

Quoted 2 lines

Change-Id: I560cecf673df2dcc3ec72767cf8077708d649656

The Change-Id: tag isn't suitable for mainline, you can drop it.

Quoted 33 lines

Signed-off-by Rajmohan Mani <[email protected]>

---
 Documentation/media/v4l-drivers/index.rst |   1 +
 Documentation/media/v4l-drivers/ipu3.rst  | 326 +++++++++++++++++++++++++++
 2 files changed, 327 insertions(+)
 create mode 100644 Documentation/media/v4l-drivers/ipu3.rst

(quoted diff) +0 -0

b/Documentation/media/v4l-drivers/index.rst index 679238e..179a393 100644

Documentation/media/v4l-drivers/index.rst +0 -0

@@ -44,6 +44,7 @@ For more details see the file COPYING in the source

distribution of Linux. davinci-vpbe

Documentation/media/v4l-drivers/index.rst +1 -0

 	fimc
 	imx
+	ipu3
 	ivtv
 	max2175
 	meye

(quoted diff) +0 -0

b/Documentation/media/v4l-drivers/ipu3.rst new file mode 100644 index 0000000..045bf42

Documentation/media/v4l-drivers/ipu3.rst new file +7 -0

@@ -0,0 +1,326 @@
+.. include:: <isonum.txt>
+
+===============================================================
+Intel Image Processing Unit 3 (IPU3) Imaging Unit (ImgU) driver
+===============================================================
+
+Copyright |copy| 2018 Intel Corporation

Quoted 6 lines

Documentation/media/v4l-drivers/ipu3.rst new file +5 -0

+
+Introduction
+============
+
+This file documents Intel IPU3 (3rd generation Image Processing Unit)

Imaging

s/documents Intel/documents the Intel/

Quoted 2 lines

+Unit driver located under drivers/media/pci/intel/ipu3.

Documentation/media/v4l-drivers/ipu3.rst line 17

Quoted 4 lines

Documentation/media/v4l-drivers/ipu3.rst new file +4 -0

+
+The Intel IPU3 found in certain Kaby Lake (as well as certain Sky Lake)
+platforms (U/Y processor lines) is made up of two parts namely Imaging Unit
+(ImgU) and CIO2 device (MIPI CSI2 receiver).

s/namely Imaging Unit/, namely the Imaging Unit/
s/and CIO2/and the CIO2/

Quoted 5 lines

Documentation/media/v4l-drivers/ipu3.rst new file +2 -0

+
+The CIO2 device receives the raw bayer data from the sensors and outputs

the +frames in a format that is specific to IPU3 (for consumption by IPU3 ImgU).

s/to IPU3/to the IPU3/
s/by IPU3/by the IPU3/

Quoted 3 lines

+CIO2 driver is available as drivers/media/pci/intel/ipu3/ipu3-cio2* and is +enabled through the CONFIG_VIDEO_IPU3_CIO2 config option.

s/CIO2 driver/The CIO2 driver/

Documentation/media/v4l-drivers/ipu3.rst

Quoted 3 lines

Documentation/media/v4l-drivers/ipu3.rst new file +2 -0

+
+The Imaging Unit (ImgU) is responsible for processing images captured

s/images/the images/

Quoted 2 lines

+through IPU3 CIO2 device. The ImgU driver sources can be found under

s/IPU3 CIO2/the IPU3 CIO2/

Documentation/media/v4l-drivers/ipu3.rst line 29

Quoted 5 lines

Documentation/media/v4l-drivers/ipu3.rst new file +4 -0

+drivers/media/pci/intel/ipu3 directory. The driver is enabled through the
+CONFIG_VIDEO_IPU3_IMGU config option.
+
+The two driver modules are named ipu3-csi2 and ipu3-imgu, respectively.

Documentation/media/v4l-drivers/ipu3.rst

Quoted 2 lines

Documentation/media/v4l-drivers/ipu3.rst new file +2 -0

+
+The driver has been tested on Kaby Lake platforms (U/Y processor lines).

I assume both drivers have been tested, so I would write

s/The driver has/The drivers have/

Quoted 3 lines

+The driver implements V4L2, Media controller and V4L2 sub-device interfaces.

As this is true for both drivers,

s/The driver implements V4L2/Both drivers implement the V4L2/ s/Media controller/Media Controller/

Documentation/media/v4l-drivers/ipu3.rst

Quoted 3 lines

Documentation/media/v4l-drivers/ipu3.rst new file +2 -0

+Camera sensors that have CSI-2 bus, which are connected to the IPU3 CIO2
+device are supported.

I would rephrase this slightly, as "The IPU3 CIO2 driver supports camera sensors connected to the CIO2 MIPI CSI-2 interfaces through V4L2 sub-device sensor drivers."

Quoted 3 lines

Support for lens and flash drivers depends on the
+above sensors.

That's a very good introduction !

I would follow with two sections, "IPU3 CIO2" followed by "IPU3 ImgU".

Documentation/media/v4l-drivers/ipu3.rst line 42

Quoted 6 lines

Documentation/media/v4l-drivers/ipu3.rst new file +5 -0

+ImgU device nodes
+=================
+
+The ImgU is represented as two V4L2 subdevs, each of which provides a V4L2
+subdev interface to the user space.

Not just subdevs, but video nodes too. I would rephrase as follows (please note that this might not be valid .rst content, I haven't tried compiling it, it might need to be reworked slightly) :

"The ImgU contains two independent pipes, each modelled as a V4L2 sub-device exposed to userspace as a V4L2 sub-device node.

Each pipe has two input pads and three output pads for the following purpose:

Pad 0 (input): Input raw video stream
Pad 1 (input): Processing parameters
Pad 2 (output): Output processed video stream
Pad 3 (output): Output viewfinder video stream
Pad 4 (output): 3A statistics

Each pad is connected to a corresponding V4L2 video interface, exposed to userspace as a V4L2 video device node."

Documentation/media/v4l-drivers/ipu3.rst line 49

Quoted 7 lines

Documentation/media/v4l-drivers/ipu3.rst new file +6 -0

+Each V4L2 subdev represents a pipe, which can support a maximum of 2
+streams. A private ioctl can be used to configure the mode (video or still)
+of the pipe.
+
+This helps to support advanced camera features like Continuous View Finder
+(CVF) and Snapshot During Video(SDV).

As far as I know there's no private ioctl anymore, at least I can't find it in the source code. Could you thus rephrase that sentence and the next one to explain the current method to implement CVF and SDV ?

Quoted 7 lines

Documentation/media/v4l-drivers/ipu3.rst new file +4 -0

+CIO2 device
+===========
+
+The CIO2 is represented as a single V4L2 subdev, which provides a V4L2

subdev +interface to the user space. There is a video node for each CSI-2 receiver, +with a single media controller interface for the entire device.

Similarly, I think we should explain here that there are four channels :

"The CIO2 contains four independent capture channel, each with its own MIPI CSI-2 receiver and DMA engine. Each channel is modelled as a V4L2 sub-device exposed to userspace as a V4L2 sub-device node and has two pads:

Pad 0 (input): MIPI CSI-2 input, connected to the sensor subdev Pad 1 (output): Raw video capture, connected to the V4L2 video interface

The V4L2 video interfaces model the DMA engines. They are exposed to userspace as V4L2 video device nodes."

The section should be moved above the IPU3 ImgU section.

Quoted 6 lines

Documentation/media/v4l-drivers/ipu3.rst new file +4 -0

+Media controller
+----------------
+
+The media device interface allows to configure the ImgU links, which

defines +the behavior of the IPU3 firmware.

s/defines/define/ or possibly better s/defines/control/

Documentation/media/v4l-drivers/ipu3.rst line 68

Quoted 7 lines

Documentation/media/v4l-drivers/ipu3.rst new file +6 -0

+
+Device operation
+----------------
+
+With IPU3, once the input video node ("ipu3-imgu 0/1":0,
+in <entity>:<pad-number> format) is queued with buffer (in packed raw bayer

s/bayer/Bayer/

Quoted 1 line

+format), IPU3 ISP starts processing the buffer and produces the video

s/IPU3 ISP/the IPU3 ISP/

This is the first time you mention an ISP. Should the term ISP be replaced by ImgU here and below ? I'm fine keeping it, but it should then be defined in the introduction, in particular with an explanation of the difference between ImgU and ISP.

Quoted 4 lines

output +in YUV format and statistics output on respective output nodes. The driver +is expected to have buffers ready for all of parameter, output and +statistics nodes, when input video node is queued with buffer.

Why is that, shouldn't the driver wait for all necessary buffers to be ready before processing ?

Documentation/media/v4l-drivers/ipu3.rst line 75

Quoted 3 lines

Documentation/media/v4l-drivers/ipu3.rst new file +2 -0

+At a minimum, all of input, main output, 3A statistics and viewfinder
+video nodes should be enabled for IPU3 to start image processing.

If they all need to be enabled, shouldn't the respective links be ENABLED and IMMUTABLE ?

Documentation/media/v4l-drivers/ipu3.rst line 82

Quoted 7 lines

Documentation/media/v4l-drivers/ipu3.rst new file +6 -0

+Each ImgU V4L2 subdev has the following set of video nodes.
+
+input, output and viewfinder video nodes
+----------------------------------------
+
+The frames (in packed raw bayer format specific to IPU3) received by the

s/bayer/Bayer/

(same in a few other locations below)

Quoted 1 line

+input video node is processed by the IPU3 Imaging Unit and is output to 2

s/is output/are output/

Quoted 2 lines

+video nodes, with each targeting different purpose (main output and

s/different purpose/a different purpose/

Quoted 2 lines

viewfinder +output).

We will eventually need more information about this, but that could come as a separate patch.

Documentation/media/v4l-drivers/ipu3.rst

Quoted 2 lines

Documentation/media/v4l-drivers/ipu3.rst new file +2 -0

+Details on raw bayer format specific to IPU3 can be found as below.
+Documentation/media/uapi/v4l/pixfmt-meta-intel-ipu3.rst

How about linking to the document using :ref: instead of adding the file path ?

Documentation/media/v4l-drivers/ipu3.rst

Quoted 12 lines

+The driver supports V4L2 Video Capture Interface as defined at :ref:devices.

Documentation/media/v4l-drivers/ipu3.rst new file +9 -0

+
+Only the multi-planar API is supported. More details can be found at
+:ref:`planar-apis`.
+
+
+parameters video node
+---------------------
+
+The parameter video node receives the ISP algorithm parameters that are

s/parameters/parameter/

Documentation/media/v4l-drivers/ipu3.rst

Quoted 4 lines

used +to configure how the ISP algorithms process the image.

Documentation/media/v4l-drivers/ipu3.rst new file +2 -0

+
+Details on raw bayer format specific to IPU3 can be found as below.

I assume you meant processing parameters, not raw Bayer format.

Quoted 2 lines

+Documentation/media/uapi/v4l/pixfmt-meta-intel-ipu3.rst

:ref: here too.

Documentation/media/v4l-drivers/ipu3.rst line 112

Quoted 9 lines

+3A statistics video node
+------------------------

Documentation/media/v4l-drivers/ipu3.rst new file +6 -0

+
+3A statistics video node is used by the ImgU driver to output the 3A (auto
+focus, auto exposure and auto white balance) statistics for the frames that
+are being processed by the ISP to user space applications. User space
+applications can use this statistics data to arrive at desired algorithm
+parameters for ISP.

s/arrive at/compute the/
s/ISP/the ISP/

Documentation/media/v4l-drivers/ipu3.rst line 118

Quoted 7 lines

Documentation/media/v4l-drivers/ipu3.rst new file +6 -0

+
+CIO2 device nodes
+=================
+
+CIO2 is represented as a single V4L2 sub-device with a video node for each
+CSI-2 receiver. The video node represents the DMA engine.

I think you can remove this section, it's already explained above. If you want to keep it, please move it with the rest of the CIO2 documentation above the ImgU documentation.

Quoted 14 lines

Documentation/media/v4l-drivers/ipu3.rst new file +13 -0

+Configuring the Intel IPU3
+==========================
+
+The Intel IPU3 ImgU driver supports V4L2 interface. Using V4L2 ioctl calls,
+the ISP can be configured and enabled.
+
+The IPU3 ImgU pipelines can be configured using media controller APIs,
+defined at :ref:`media_controller`.
+
+Capturing frames in raw bayer format
+------------------------------------
+
+IPU3 MIPI CSI2 receiver is used to capture frames (in packed raw bayer

Documentation/media/v4l-drivers/ipu3.rst line 136

Quoted 4 lines

Documentation/media/v4l-drivers/ipu3.rst new file +4 -0

+format) from the raw sensors connected to the CSI2 ports. The captured
+frames are used as input to the ImgU driver.
+
+Image processing using IPU3 ImgU requires tools such as v4l2n [#f1]_,

I would drop v4l2n from the documentation as it's not maintained and is not functional (in particular it doesn't implement MPLANE support which the driver requires).

Documentation/media/v4l-drivers/ipu3.rst line 141

Quoted 6 lines

Documentation/media/v4l-drivers/ipu3.rst new file +5 -0

+raw2pnm [#f1]_, and yavta [#f2]_ due to the following unique requirements
+and / or features specific to IPU3.
+
+-- The IPU3 CSI2 receiver outputs the captured frames from the sensor in
+packed raw bayer format that is specific to IPU3

s/to IPU3/to the IPU3/.

Documentation/media/v4l-drivers/ipu3.rst line 151

Quoted 11 lines

Documentation/media/v4l-drivers/ipu3.rst new file +10 -0

+
+-- Multiple video nodes have to be operated simultaneously
+
+Let us take the example of ov5670 sensor connected to CSI2 port 0, for a
+2592x1944 image capture.
+
+Using the media contorller APIs, the ov5670 sensor is configured to send
+frames in packed raw bayer format to IPU3 CSI2 receiver.
+
+# This example assumes /dev/media0 as the ImgU media device

Shouldn't that be CIO2 ?

Documentation/media/v4l-drivers/ipu3.rst line 160

Quoted 10 lines

Documentation/media/v4l-drivers/ipu3.rst new file +9 -0

+
+export MDEV=/dev/media0
+
+# and that ov5670 sensor is connected to i2c bus 10 with address 0x36
+
+export SDEV="ov5670 10-0036"
+
+# Establish the link for the media devices using media-ctl [#f3]_
+media-ctl -d $MDEV -l "ov5670 ":0 -> "ipu3-csi2 0":0[1]

You're missing quotes around the link description, and the entity name is incorrect. Please test all the commands listed here with the upstream driver.

Quoted 1 line

+media-ctl -d $MDEV -l "ipu3-csi2 0":1 -> "ipu3-cio2 0":0[1]

Shouldn't this link be immutable ?

Documentation/media/v4l-drivers/ipu3.rst line 165

Quoted 3 lines

Documentation/media/v4l-drivers/ipu3.rst new file +2 -0

+# Set the format for the media devices
+media-ctl -d $MDEV -V "ov5670 ":0 [fmt:SGRBG10/2592x1944]

The entity name is incorrect.

Quoted 5 lines

Documentation/media/v4l-drivers/ipu3.rst new file +4 -0

+media-ctl -d $MDEV -V "ipu3-csi2 0":0 [fmt:SGRBG10/2592x1944]
+
+media-ctl -d $MDEV -V "ipu3-csi2 0":1 [fmt:SGRBG10/2592x1944]
+

Documentation/media/v4l-drivers/ipu3.rst line 180

Quoted 10 lines

Documentation/media/v4l-drivers/ipu3.rst new file +10 -0

+Once the media pipeline is configured, desired sensor specific settings
+(such as exposure and gain settings) can be set, using the yavta tool.
+
+e.g
+
+yavta -w 0x009e0903 444 $(media-ctl -d $MDEV -e "$SDEV")
+
+yavta -w 0x009e0913 1024 $(media-ctl -d $MDEV -e "$SDEV")
+
+yavta -w 0x009e0911 2046 $(media-ctl -d $MDEV -e "$SDEV")

Documentation/media/v4l-drivers/ipu3.rst line 186

Quoted 7 lines

Documentation/media/v4l-drivers/ipu3.rst new file +2 -0

+
+Once the desired sensor settings are set, frame captures can be done as

below.

Documentation/media/v4l-drivers/ipu3.rst new file +4 -0

+
+e.g
+
+yavta --data-prefix -u -c10 -n5 -I -s2592x1944 --file=/tmp/frame-#.bin

Do you need --data-prefix ?

Quoted 1 line

+-f IPU3_GRBG10 media-ctl -d $MDEV -e ipu3-cio2 0

The upstream yavta names the format IPU3_SGRBG10.

You're missing $(...) around the media-ctl command, and double quotes around the entity name.

Quoted 4 lines

Documentation/media/v4l-drivers/ipu3.rst new file +2 -0

+
+With the above command, 10 frames are captured at 2592x1944 resolution,

with +sGRBG10 format and output as IPU3_GRBG10 format.

IPU3_SGRBG10 here too.

Documentation/media/v4l-drivers/ipu3.rst line 192

Quoted 3 lines

Documentation/media/v4l-drivers/ipu3.rst new file +2 -0

+
+The captured frames are available as /tmp/frame-#.bin files.

All this should be moved to the CIO2 section.

Quoted 15 lines

Documentation/media/v4l-drivers/ipu3.rst new file +14 -0

+Processing the image in raw bayer format
+----------------------------------------
+
+Configuring ImgU V4L2 subdev for image processing
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The ImgU V4L2 subdevs have to be configured with media controller APIs to
+have all the video nodes setup correctly.
+
+Let us take "ipu3-imgu 0" subdev as an example.
+
+media-ctl -d $MDEV -r
+
+media-ctl -d $MDEV -l "ipu3-imgu 0 input":0 -> "ipu3-imgu 0":0[1]

Documentation/media/v4l-drivers/ipu3.rst line 213

Quoted 6 lines

Documentation/media/v4l-drivers/ipu3.rst new file +6 -0

+
+media-ctl -d $MDEV -l "ipu3-imgu 0":2 -> "output":0[1]
+
+media-ctl -d $MDEV -l "ipu3-imgu 0":3 -> "viewfinder":0[1]
+
+media-ctl -d $MDEV -l "ipu3-imgu 0":4 -> "3a stat":0[1]

Entity names are incorrect here too.

Documentation/media/v4l-drivers/ipu3.rst line 221

Quoted 8 lines

Documentation/media/v4l-drivers/ipu3.rst new file +7 -0

+Also the pipe mode of the corresponding V4L2 subdev should be set as
+desired (e.g 0 for video mode or 1 for still mode) through the
+control id 0x009819a1 as below.
+
+e.g
+
+v4l2n -d /dev/v4l-subdev7 --ctrl=0x009819A1=1

You can use yavta instead of v4l2n. Another option would be v4l2-ctl, but I'd avoid adding a dependency to another tool if yavta is already used in other places.

Documentation/media/v4l-drivers/ipu3.rst line 227

Quoted 6 lines

Documentation/media/v4l-drivers/ipu3.rst new file +5 -0

+RAW bayer frames go through the following ISP pipeline HW blocks to
+have the processed image output to the DDR memory.
+
+RAW bayer frame -> Input Feeder -> Bayer Down Scaling (BDS) -> Geometric
+Distortion Correction (GDC) -> DDR

A more detailed block diagram, with the other blocks included, should be added to the ImgU description above. Each block should have a short description of its purpose.

Documentation/media/v4l-drivers/ipu3.rst line 235

Quoted 8 lines

Documentation/media/v4l-drivers/ipu3.rst new file +7 -0

+The ImgU V4L2 subdev has to be configured with the supported resolutions
+in all the above HW blocks, for a given input resolution.
+
+For a given supported resolution for an input frame, the Input Feeder,
+Bayer Down Scaling and GDC blocks should be configured with the supported
+resolutions. This information can be obtained by looking at the following
+IPU3 ISP configuration table.

Does this mean that the ImgU will not operate properly when exercised through the MC and V4L2 only without configuration of the internal blocks through the processing parameters device node ?

Documentation/media/v4l-drivers/ipu3.rst line 240

Quoted 6 lines

+https://chromium.googlesource.com/chromiumos/overlays/board-overlays/+/mast er

Documentation/media/v4l-drivers/ipu3.rst new file +3 -0

+
+Under baseboard-poppy/media-libs/arc-camera3-hal-configs-poppy/files/gcss
+directory, graph_settings_ov5670.xml can be used as an example.

The directory name is incorrect.

Quoted 13 lines

Documentation/media/v4l-drivers/ipu3.rst new file +4 -0

+The following steps prepare the ImgU ISP pipeline for the image processing.
+
+1. The ImgU V4L2 subdev data format should be set by using the
+VIDIOC_SUBDEV_S_FMT on pad 0, using the GDC width and height obtained

above.

Documentation/media/v4l-drivers/ipu3.rst new file +7 -0

+
+2. The ImgU V4L2 subdev cropping should be set by using the
+VIDIOC_SUBDEV_S_SELECTION on pad 0, with V4L2_SEL_TGT_CROP as the target,
+using the input feeder height and width.
+
+3. The ImgU V4L2 subdev composing should be set by using the
+VIDIOC_SUBDEV_S_SELECTION on pad 0, with V4L2_SEL_TGT_COMPOSE as the

Quoted 1 line

target, +using the BDS height and width.

How the format and selection rectangles related to the internal processing blocks should also be explained in more details.

Documentation/media/v4l-drivers/ipu3.rst line 258

Quoted 5 lines

Documentation/media/v4l-drivers/ipu3.rst new file +4 -0

+For the ov5670 example, for an input frame with a resolution of 2592x1944
+(which is input to the ImgU subdev pad 0), the corresponding resolutions
+for input feeder, BDS and GDC are 2592x1944, 2592x1944 and 2560x1920
+respectively.

Why ? How is that computed ? If my input resolution was different, how would I compute the other resolutions ?

Quoted 15 lines

Documentation/media/v4l-drivers/ipu3.rst new file +5 -0

+Once this is done, the received raw bayer frames can be input to the ImgU
+V4L2 subdev as below, using the open source application v4l2n.
+
+For an image captured with 2592x1944 [#f4]_ resolution, with desired output
+resolution as 2560x1920 and viewfinder resolution as 2560x1920, the

following +v4l2n command can be used. This helps process the raw bayer frames and +produces the desired results for the main output image and the viewfinder +output, in NV12 format.

Documentation/media/v4l-drivers/ipu3.rst new file +3 -0

+
+v4l2n --pipe=4 --load=/tmp/frame-#.bin --open=/dev/video4
+--fmt=type:VIDEO_OUTPUT_MPLANE,width=2592,height=1944,pixelformat=0X4733706

9 +--reqbufs=type:VIDEO_OUTPUT_MPLANE,count:1 --pipe=1 --output=/tmp/frames.out +--open=/dev/video5 +--fmt=type:VIDEO_CAPTURE_MPLANE,width=2560,height=1920,pixelformat=NV12

Quoted 6 lines

+--reqbufs=type:VIDEO_CAPTURE_MPLANE,count:1 --pipe=2 --output=/tmp/frames.vf +--open=/dev/video6

+--fmt=type:VIDEO_CAPTURE_MPLANE,width=2560,height=1920,pixelformat=NV12
+--reqbufs=type:VIDEO_CAPTURE_MPLANE,count:1 --pipe=3 --open=/dev/video7
+--output=/tmp/frames.3A --fmt=type:META_CAPTURE,?
+--reqbufs=count:1,type:META_CAPTURE --pipe=1,2,3,4 --stream=5

You can replace this with four yavta commands.

Quoted 10 lines

+where /dev/video4, /dev/video5, /dev/video6 and /dev/video7 devices point to +input, output, viewfinder and 3A statistics video nodes respectively. +

Documentation/media/v4l-drivers/ipu3.rst new file +7 -0

+Converting the raw bayer image into YUV domain
+----------------------------------------------
+
+The processed images after the above step, can be converted to YUV domain
+as below.
+
+Main output frames

Documentation/media/v4l-drivers/ipu3.rst line 293

Quoted 3 lines

+~~~~~~~~~~~~~~~~~~

Documentation/media/v4l-drivers/ipu3.rst new file +2 -0

+
+raw2pnm -x2560 -y1920 -fNV12 /tmp/frames.out /tmp/frames.out.pnm

PNM is an umbrella term that refers to any of the PBM, PGM or PPM format. As the raw2pnm tool outputs PPM files, let's name them .ppm here and below. This helps with some image viewers that use file extensions to identify the format, and have trouble handling .pnm files.

Quoted 34 lines

Documentation/media/v4l-drivers/ipu3.rst new file +32 -0

+where 2560x1920 is output resolution, NV12 is the video format, followed
+by input frame and output PNM file.
+
+Viewfinder output frames
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+raw2pnm -x2560 -y1920 -fNV12 /tmp/frames.vf /tmp/frames.vf.pnm
+
+where 2560x1920 is output resolution, NV12 is the video format, followed
+by input frame and output PNM file.
+
+Example user space code for IPU3
+================================
+
+User space code that configures and uses IPU3 is available here.
+
+https://chromium.googlesource.com/chromiumos/platform/arc-camera/+/master/
+
+The source can be located under hal/intel directory.
+
+References
+==========
+
+include/uapi/linux/intel-ipu3.h
+
+.. [#f1] https://github.com/intel/nvt
+
+.. [#f2] http://git.ideasonboard.org/yavta.git
+
+.. [#f3] http://git.ideasonboard.org/?p=media-ctl.git;a=summary
+
+.. [#f4] ImgU limitation requires an additional 16x16 for all input

resolutions

Signature · 3 lines

Regards,

Laurent Pinchart

Hi Laurent,

I'm sending a separate patch to address the comments.

Quoted 5 lines · Laurent Pinchart

Quoted 3 lines · On Tuesday, 30 October 2018 00:22:56 EET Yong Zhi wrote:

From: Rajmohan Mani <[email protected]>

This patch adds the details about the IPU3 Imaging Unit driver.

Quoted 5 lines

Strictly speaking this documents both the CIO2 and the IMGU. As they're handled by two separate drivers, should they be split in two separate files ? If you prefer keeping them together you should update the commit message accordingly. I would in that case also split the documentation in a CIO2 and a IMGU section in the file, instead of mixing them.

I'm keeping it in a single document for now. In practice these devices always come together as neither is really usable without the other.

Quoted 3 lines

Quoted 1 line

Change-Id: I560cecf673df2dcc3ec72767cf8077708d649656

The Change-Id: tag isn't suitable for mainline, you can drop it.

Fixed.

Quoted 7 lines

Quoted 7 lines

Signed-off-by Rajmohan Mani <[email protected]>

---

 Documentation/media/v4l-drivers/index.rst |   1 +
 Documentation/media/v4l-drivers/ipu3.rst  | 326 ++++++++++++++++++++++++
 2 files changed, 327 insertions(+)
 create mode 100644 Documentation/media/v4l-drivers/ipu3.rst

Quoted 5 lines

Quoted 5 lines

(quoted diff) +0 -0

b/Documentation/media/v4l-drivers/ipu3.rst new file mode 100644 index 0000000..045bf42

Documentation/media/v4l-drivers/ipu3.rst new file +0 -0

Quoted 7 lines

Quoted 5 lines

Documentation/media/v4l-drivers/ipu3.rst new file +4 -0

+Media controller
+----------------
+
+The media device interface allows to configure the ImgU links, which

defines +the behavior of the IPU3 firmware.

s/defines/define/ or possibly better s/defines/control/

I'm removing the section as the binary is selected using a control in the current version. I'm adding this instead:

Firmware binary selection
-------------------------

The firmware binary is selected using the V4L2_CID_INTEL_IPU3_MODE, currently defined in drivers/staging/media/ipu3/include/intel-ipu3.h . "VIDEO" and "STILL" modes are available.

Quoted 9 lines

Quoted 7 lines

Documentation/media/v4l-drivers/ipu3.rst new file +6 -0

+
+Device operation
+----------------
+
+With IPU3, once the input video node ("ipu3-imgu 0/1":0,
+in <entity>:<pad-number> format) is queued with buffer (in packed raw

bayer

s/bayer/Bayer/

Fixed in the entire file.

Quoted 8 lines

Quoted 1 line

+format), IPU3 ISP starts processing the buffer and produces the video

s/IPU3 ISP/the IPU3 ISP/

This is the first time you mention an ISP. Should the term ISP be replaced by ImgU here and below ? I'm fine keeping it, but it should then be defined in the introduction, in particular with an explanation of the difference between ImgU and ISP.

I replaced IPU3 with ImgU in this section.

Quoted 5 lines

Quoted 4 lines

output +in YUV format and statistics output on respective output nodes. The driver +is expected to have buffers ready for all of parameter, output and +statistics nodes, when input video node is queued with buffer.

Quoted 2 lines

Why is that, shouldn't the driver wait for all necessary buffers to be ready before processing ?

Not all are mandatory.

Quoted 5 lines

Documentation/media/v4l-drivers/ipu3.rst line 75

Quoted 2 lines

Documentation/media/v4l-drivers/ipu3.rst new file +2 -0

+At a minimum, all of input, main output, 3A statistics and viewfinder
+video nodes should be enabled for IPU3 to start image processing.

If they all need to be enabled, shouldn't the respective links be ENABLED and IMMUTABLE ?

Yes.

Quoted 13 lines

Quoted 13 lines

Documentation/media/v4l-drivers/ipu3.rst new file +4 -0

+Configuring the Intel IPU3
+==========================
+
+The Intel IPU3 ImgU driver supports V4L2 interface. Using V4L2 ioctl

calls, +the ISP can be configured and enabled.

Documentation/media/v4l-drivers/ipu3.rst new file +8 -0

+
+The IPU3 ImgU pipelines can be configured using media controller APIs,
+defined at :ref:`media_controller`.
+
+Capturing frames in raw bayer format
+------------------------------------
+
+IPU3 MIPI CSI2 receiver is used to capture frames (in packed raw bayer

Quoted 8 lines

Documentation/media/v4l-drivers/ipu3.rst line 136

Quoted 4 lines

Documentation/media/v4l-drivers/ipu3.rst new file +4 -0

+format) from the raw sensors connected to the CSI2 ports. The captured
+frames are used as input to the ImgU driver.
+
+Image processing using IPU3 ImgU requires tools such as v4l2n [#f1]_,

I would drop v4l2n from the documentation as it's not maintained and is not functional (in particular it doesn't implement MPLANE support which the driver requires).

Ack. I'm leaving the command plus the reference to v4l2n, this might contain information useful still.

Quoted 3 lines

Quoted 3 lines

+raw2pnm [#f1]_, and yavta [#f2]_ due to the following unique requirements
+and / or features specific to IPU3.

Quoted 5 lines

Quoted 5 lines

Documentation/media/v4l-drivers/ipu3.rst new file +5 -0

+Processing the image in raw bayer format
+----------------------------------------
+
+Configuring ImgU V4L2 subdev for image processing
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Quoted 9 lines

Quoted 5 lines

Documentation/media/v4l-drivers/ipu3.rst new file +4 -0

+RAW bayer frames go through the following ISP pipeline HW blocks to
+have the processed image output to the DDR memory.
+
+RAW bayer frame -> Input Feeder -> Bayer Down Scaling (BDS) ->

Geometric +Distortion Correction (GDC) -> DDR

A more detailed block diagram, with the other blocks included, should be added to the ImgU description above. Each block should have a short description of its purpose.

I think we have one in conjunction with the parameter format description.

Quoted 11 lines

Quoted 7 lines

+The ImgU V4L2 subdev has to be configured with the supported resolutions +in all the above HW blocks, for a given input resolution.

Documentation/media/v4l-drivers/ipu3.rst new file +3 -0

+
+For a given supported resolution for an input frame, the Input Feeder,
+Bayer Down Scaling and GDC blocks should be configured with the

supported +resolutions. This information can be obtained by looking at the following +IPU3 ISP configuration table.

Does this mean that the ImgU will not operate properly when exercised through the MC and V4L2 only without configuration of the internal blocks through the processing parameters device node ?

I need to let Raj and Yong to answer that. My understanding is the default parameters should be usable. If they're not, that needs to be addressed.

Quoted 8 lines

Quoted 6 lines

+https://chromium.googlesource.com/chromiumos/overlays/board-overlays/+/ master

Documentation/media/v4l-drivers/ipu3.rst new file +2 -0

+
+Under

baseboard-poppy/media-libs/arc-camera3-hal-configs-poppy/files/gcss +directory, graph_settings_ov5670.xml can be used as an example.

The directory name is incorrect.

Fixed.

Quoted 10 lines

Quoted 10 lines

+The following steps prepare the ImgU ISP pipeline for the image processing.

Documentation/media/v4l-drivers/ipu3.rst new file +3 -0

+
+1. The ImgU V4L2 subdev data format should be set by using the
+VIDIOC_SUBDEV_S_FMT on pad 0, using the GDC width and height obtained

above.

Documentation/media/v4l-drivers/ipu3.rst new file +3 -0

+
+2. The ImgU V4L2 subdev cropping should be set by using the
+VIDIOC_SUBDEV_S_SELECTION on pad 0, with V4L2_SEL_TGT_CROP as the

target,

Quoted 8 lines

Quoted 5 lines

Documentation/media/v4l-drivers/ipu3.rst new file +4 -0

+using the input feeder height and width.
+
+3. The ImgU V4L2 subdev composing should be set by using the
+VIDIOC_SUBDEV_S_SELECTION on pad 0, with V4L2_SEL_TGT_COMPOSE as the

target, +using the BDS height and width.

How the format and selection rectangles related to the internal processing blocks should also be explained in more details.

To be added later.

Quoted 9 lines

Documentation/media/v4l-drivers/ipu3.rst line 258

Quoted 6 lines

+For the ov5670 example, for an input frame with a resolution of 2592x1944
+(which is input to the ImgU subdev pad 0), the corresponding resolutions

Documentation/media/v4l-drivers/ipu3.rst new file +2 -0

+for input feeder, BDS and GDC are 2592x1944, 2592x1944 and 2560x1920
+respectively.

Why ? How is that computed ? If my input resolution was different, how would I compute the other resolutions ?

Ditto.

Quoted 15 lines

Quoted 15 lines

+Once this is done, the received raw bayer frames can be input to the ImgU

Documentation/media/v4l-drivers/ipu3.rst new file +3 -0

+V4L2 subdev as below, using the open source application v4l2n.
+
+For an image captured with 2592x1944 [#f4]_ resolution, with desired

output +resolution as 2560x1920 and viewfinder resolution as 2560x1920, the following +v4l2n command can be used. This helps process the raw bayer frames and +produces the desired results for the main output image and the viewfinder +output, in NV12 format.

Documentation/media/v4l-drivers/ipu3.rst new file +3 -0

+
+v4l2n --pipe=4 --load=/tmp/frame-#.bin --open=/dev/video4
+--fmt=type:VIDEO_OUTPUT_MPLANE,width=2592,height=1944,pixelformat=0X473

3706 9 +--reqbufs=type:VIDEO_OUTPUT_MPLANE,count:1 --pipe=1 --output=/tmp/frames.out +--open=/dev/video5 +--fmt=type:VIDEO_CAPTURE_MPLANE,width=2560,height=1920,pixelformat=NV12

Quoted 8 lines

Quoted 6 lines

+--reqbufs=type:VIDEO_CAPTURE_MPLANE,count:1 --pipe=2 --output=/tmp/frames.vf +--open=/dev/video6

+--fmt=type:VIDEO_CAPTURE_MPLANE,width=2560,height=1920,pixelformat=NV12
+--reqbufs=type:VIDEO_CAPTURE_MPLANE,count:1 --pipe=3 --open=/dev/video7
+--output=/tmp/frames.3A --fmt=type:META_CAPTURE,?
+--reqbufs=count:1,type:META_CAPTURE --pipe=1,2,3,4 --stream=5

You can replace this with four yavta commands.

Ditto.

Hi Sakari,

Quoted 16 lines · On Thursday, 13 December 2018 11:38:26 EET Sakari Ailus wrote:

Hi Laurent,

I'm sending a separate patch to address the comments.

Quoted 10 lines · Laurent Pinchart

Quoted 3 lines · On Tuesday, 30 October 2018 00:22:56 EET Yong Zhi wrote:

From: Rajmohan Mani <[email protected]>

This patch adds the details about the IPU3 Imaging Unit driver.

Strictly speaking this documents both the CIO2 and the IMGU. As they're handled by two separate drivers, should they be split in two separate files ? If you prefer keeping them together you should update the commit message accordingly. I would in that case also split the documentation in a CIO2 and a IMGU section in the file, instead of mixing them.

Quoted 2 lines

I'm keeping it in a single document for now. In practice these devices always come together as neither is really usable without the other.

But they're still two separate devices. Splitting the documentation would clarify which part is associated with each device. CIO2 and ImgU instructions are currently interleaved and that's very confusing. If you really want to keep everything in one file, the CIO2 and ImgU parts should be deinterleaved, and the CIO2 should come first.

Quoted 14 lines

Quoted 4 lines

Quoted 2 lines

Change-Id: I560cecf673df2dcc3ec72767cf8077708d649656

The Change-Id: tag isn't suitable for mainline, you can drop it.

Fixed.

Quoted 7 lines

Quoted 7 lines

Signed-off-by Rajmohan Mani <[email protected]>

---

 Documentation/media/v4l-drivers/index.rst |   1 +
 Documentation/media/v4l-drivers/ipu3.rst  | 326 ++++++++++++++++++++++++
 2 files changed, 327 insertions(+)
 create mode 100644 Documentation/media/v4l-drivers/ipu3.rst

[snip]

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

(quoted diff) +0 -0

b/Documentation/media/v4l-drivers/ipu3.rst new file mode 100644 index 0000000..045bf42

Documentation/media/v4l-drivers/ipu3.rst new file +0 -0

[snip]

Quoted 14 lines

Quoted 7 lines

Quoted 5 lines

Documentation/media/v4l-drivers/ipu3.rst new file +4 -0

+Media controller
+----------------
+
+The media device interface allows to configure the ImgU links, which

defines +the behavior of the IPU3 firmware.

s/defines/define/ or possibly better s/defines/control/

I'm removing the section as the binary is selected using a control in the current version. I'm adding this instead:

Firmware binary selection
-------------------------

Quoted 3 lines

The firmware binary is selected using the V4L2_CID_INTEL_IPU3_MODE, currently defined in drivers/staging/media/ipu3/include/intel-ipu3.h . "VIDEO" and "STILL" modes are available.

Do we need to expose the fact that they use different firmwares ? Or should we instead document the two modes of operation, and explain that they need to be selected before anything else ? In any case modes need to be documented.

Interestingly enough, the driver code includes

enum ipu3_css_pipe_id {
        IPU3_CSS_PIPE_ID_PREVIEW,
        IPU3_CSS_PIPE_ID_COPY,
        IPU3_CSS_PIPE_ID_VIDEO,
        IPU3_CSS_PIPE_ID_CAPTURE,
        IPU3_CSS_PIPE_ID_YUVPP,
        IPU3_CSS_PIPE_ID_ACC,
        IPU3_CSS_PIPE_ID_NUM
};

but seems to only support the video and capture modes.

Quoted 32 lines

Quoted 10 lines

Quoted 8 lines

Documentation/media/v4l-drivers/ipu3.rst new file +6 -0

+
+Device operation
+----------------
+
+With IPU3, once the input video node ("ipu3-imgu 0/1":0,
+in <entity>:<pad-number> format) is queued with buffer (in packed raw

bayer

s/bayer/Bayer/

Fixed in the entire file.

Quoted 8 lines

Quoted 1 line

+format), IPU3 ISP starts processing the buffer and produces the video

s/IPU3 ISP/the IPU3 ISP/

This is the first time you mention an ISP. Should the term ISP be replaced by ImgU here and below ? I'm fine keeping it, but it should then be defined in the introduction, in particular with an explanation of the difference between ImgU and ISP.

I replaced IPU3 with ImgU in this section.

Quoted 7 lines

Quoted 4 lines

output +in YUV format and statistics output on respective output nodes. The driver +is expected to have buffers ready for all of parameter, output and +statistics nodes, when input video node is queued with buffer.

Why is that, shouldn't the driver wait for all necessary buffers to be ready before processing ?

Quoted 1 line

Not all are mandatory.

Which ones are mandatory, and which ones are not ? V4L2 doesn't enforce buffer queuing order for M2M devices, it's a very bad idea to do so here. It would make usage of the driver impossible with separate unsynchronized processes for different queues (which is typically the case when using command line test tools).

Quoted 8 lines

Quoted 6 lines

Documentation/media/v4l-drivers/ipu3.rst line 75

Quoted 3 lines

Documentation/media/v4l-drivers/ipu3.rst new file +2 -0

+At a minimum, all of input, main output, 3A statistics and viewfinder
+video nodes should be enabled for IPU3 to start image processing.

If they all need to be enabled, shouldn't the respective links be ENABLED and IMMUTABLE ?

Yes.

Could you please capture this in the TODO file ?

[snip]

Quoted 22 lines

Quoted 21 lines

Quoted 7 lines

Documentation/media/v4l-drivers/ipu3.rst new file +4 -0

+Configuring the Intel IPU3
+==========================
+
+The Intel IPU3 ImgU driver supports V4L2 interface. Using V4L2 ioctl

calls, +the ISP can be configured and enabled.

Documentation/media/v4l-drivers/ipu3.rst new file +2 -0

+
+The IPU3 ImgU pipelines can be configured using media controller APIs,

Documentation/media/v4l-drivers/ipu3.rst line 136

Quoted 10 lines

Documentation/media/v4l-drivers/ipu3.rst new file +10 -0

+defined at :ref:`media_controller`.
+
+Capturing frames in raw bayer format
+------------------------------------
+
+IPU3 MIPI CSI2 receiver is used to capture frames (in packed raw bayer
+format) from the raw sensors connected to the CSI2 ports. The captured
+frames are used as input to the ImgU driver.
+
+Image processing using IPU3 ImgU requires tools such as v4l2n [#f1]_,

I would drop v4l2n from the documentation as it's not maintained and is not functional (in particular it doesn't implement MPLANE support which the driver requires).

Quoted 2 lines

Ack. I'm leaving the command plus the reference to v4l2n, this might contain information useful still.

Who will address this TODO item ? :-)

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

+raw2pnm [#f1]_, and yavta [#f2]_ due to the following unique requirements
+and / or features specific to IPU3.

[snip]

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Documentation/media/v4l-drivers/ipu3.rst new file +5 -0

+Processing the image in raw bayer format
+----------------------------------------
+
+Configuring ImgU V4L2 subdev for image processing
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

[snip]

Quoted 10 lines

Quoted 9 lines

Quoted 5 lines

Documentation/media/v4l-drivers/ipu3.rst new file +4 -0

+RAW bayer frames go through the following ISP pipeline HW blocks to
+have the processed image output to the DDR memory.
+
+RAW bayer frame -> Input Feeder -> Bayer Down Scaling (BDS) ->

Geometric +Distortion Correction (GDC) -> DDR

A more detailed block diagram, with the other blocks included, should be added to the ImgU description above. Each block should have a short description of its purpose.

Quoted 1 line

I think we have one in conjunction with the parameter format description.

You're right. As mentioned in my review for that patch, I think it should be moved to this document.

These meta formats are used on Intel IPU3 ImgU video queues to carry 3A statistics and ISP pipeline parameters.

V4L2_META_FMT_IPU3_3A
V4L2_META_FMT_IPU3_PARAMS

Signed-off-by Yong Zhi <[email protected]>

Signed-off-by Chao C Li <[email protected]>

Signed-off-by Rajmohan Mani <[email protected]>

---
 Documentation/media/uapi/v4l/meta-formats.rst      |    1 +
 .../media/uapi/v4l/pixfmt-meta-intel-ipu3.rst      |  181 ++
 include/uapi/linux/intel-ipu3.h                    | 2819 ++++++++++++++++++++
 3 files changed, 3001 insertions(+)
 create mode 100644 Documentation/media/uapi/v4l/pixfmt-meta-intel-ipu3.rst
 create mode 100644 include/uapi/linux/intel-ipu3.h

Documentation/media/uapi/v4l/meta-formats.rst +1 -0

index cf971d5..eafc534 100644
@@ -12,6 +12,7 @@ These formats are used for the :ref:`metadata` interface only.
 .. toctree::
     :maxdepth: 1
 
+    pixfmt-meta-intel-ipu3
     pixfmt-meta-d4xx
     pixfmt-meta-uvc
     pixfmt-meta-vsp1-hgo

Documentation/media/uapi/v4l/pixfmt-meta-intel-ipu3.rst new file +0 -0

new file mode 100644
index 0000000..23b945b

Documentation/media/uapi/v4l/pixfmt-meta-intel-ipu3.rst new file +7 -0

+
+******************************************************************
+V4L2_META_FMT_IPU3_PARAMS ('ip3p'), V4L2_META_FMT_IPU3_3A ('ip3s')
+******************************************************************
+
+.. c:type:: ipu3_uapi_stats_3a
+

Documentation/media/uapi/v4l/pixfmt-meta-intel-ipu3.rst new file +11 -0

+Saturation measure) cells, AWB filter response, AF (Auto-focus) filter response,
+and AE (Auto-exposure) histogram.
+
+struct :c:type:`ipu3_uapi_4a_config` saves configurable parameters for all above.
+
+
+.. code-block:: c
+
+
+     struct ipu3_uapi_stats_3a {
+	struct ipu3_uapi_awb_raw_buffer awb_raw_buffer

Documentation/media/uapi/v4l/pixfmt-meta-intel-ipu3.rst new file +90 -0

+	struct ipu3_uapi_ff_status stats_3a_status;
+     } __packed;
+
+
+.. c:type:: ipu3_uapi_params
+
+Pipeline parameters
+===================
+
+IPU3 pipeline has a number of image processing stages, each of which takes a
+set of parameters as input. The major stages of pipelines are shown here:
+
+Raw pixels -> Bayer Downscaling -> Optical Black Correction ->
+
+Linearization -> Lens Shading Correction -> White Balance / Exposure /
+
+Focus Apply -> Bayer Noise Reduction -> ANR -> Demosaicing -> Color
+
+Correction Matrix -> Gamma correction -> Color Space Conversion ->
+
+Chroma Down Scaling -> Chromatic Noise Reduction -> Total Color
+
+Correction -> XNR3 -> TNR -> DDR
+
+The table below presents a description of the above algorithms.
+
+======================== =======================================================
+Name			 Description
+======================== =======================================================
+Optical Black Correction Optical Black Correction block subtracts a pre-defined
+			 value from the respective pixel values to obtain better
+			 image quality.
+			 Defined in :c:type:`ipu3_uapi_obgrid_param`.
+Linearization		 This algo block uses linearization parameters to
+			 address non-linearity sensor effects. The Lookup table
+			 table is defined in
+			 :c:type:`ipu3_uapi_isp_lin_vmem_params`.
+SHD			 Lens shading correction is used to correct spatial
+			 non-uniformity of the pixel response due to optical
+			 lens shading. This is done by applying a different gain
+			 for each pixel. The gain, black level etc are
+			 configured in :c:type:`ipu3_uapi_shd_config_static`.
+BNR			 Bayer noise reduction block removes image noise by
+			 applying a bilateral filter.
+			 See :c:type:`ipu3_uapi_bnr_static_config` for details.
+ANR			 Advanced Noise Reduction is a block based algorithm
+			 that performs noise reduction in the Bayer domain. The
+			 convolution matrix etc can be found in
+			 :c:type:`ipu3_uapi_anr_config`.
+Demosaicing		 Demosaicing converts raw sensor data in Bayer format
+			 into RGB (Red, Green, Blue) presentation. Then add
+			 outputs of estimation of Y channel for following stream
+			 processing by Firmware. The struct is defined as
+			 :c:type:`ipu3_uapi_dm_config`.
+Color Correction	 Color Correction algo transforms sensor specific color
+			 space to the standard "sRGB" color space. This is done
+			 by applying 3x3 matrix defined in
+			 :c:type:`ipu3_uapi_ccm_mat_config`.
+Gamma correction	 Gamma correction :c:type:`ipu3_uapi_gamma_config` is a
+			 basic non-linear tone mapping correction that is
+			 applied per pixel for each pixel component.
+CSC			 Color space conversion transforms each pixel from the
+			 RGB primary presentation to YUV (Y - brightness,
+			 UV - Luminance) presentation. This is done by applying
+			 a 3x3 matrix defined in
+			 :c:type:`ipu3_uapi_csc_mat_config`
+CDS			 Chroma down sampling
+			 After the CSC is performed, the Chroma Down Sampling
+			 is applied for a UV plane down sampling by a factor
+			 of 2 in each direction for YUV 4:2:0 using a 4x2
+			 configurable filter :c:type:`ipu3_uapi_cds_params`.
+CHNR			 Chroma noise reduction
+			 This block processes only the chrominance pixels and
+			 performs noise reduction by cleaning the high
+			 frequency noise.
+			 See struct :c:type:`ipu3_uapi_yuvp1_chnr_config`.
+TCC			 Total color correction as defined in struct
+			 :c:type:`ipu3_uapi_yuvp2_tcc_static_config`.
+XNR3			 eXtreme Noise Reduction V3 is the third revision of
+			 noise reduction algorithm used to improve image
+			 quality. This removes the low frequency noise in the
+			 captured image. Two related structs are  being defined,
+			 :c:type:`ipu3_uapi_isp_xnr3_params` for ISP data memory
+			 and :c:type:`ipu3_uapi_isp_xnr3_vmem_params` for vector
+			 memory.
+TNR			 Temporal Noise Reduction block compares successive
+			 frames in time to remove anomalies / noise in pixel
+			 values. :c:type:`ipu3_uapi_isp_tnr3_vmem_params` and
+			 :c:type:`ipu3_uapi_isp_tnr3_params` are defined for ISP
+			 vector and data memory respectively.

Documentation/media/uapi/v4l/pixfmt-meta-intel-ipu3.rst new file +40 -0

+
+Both 3A statistics and pipeline parameters described here are closely tied to
+the underlying camera sub-system (CSS) APIs. They are usually consumed and
+produced by dedicated user space libraries that comprise the important tuning
+tools, thus freeing the developers from being bothered with the low level
+hardware and algorithm details.
+
+It should be noted that IPU3 DMA operations require the addresses of all data
+structures (that includes both input and output) to be aligned on 32 byte
+boundaries.
+
+The meta data :c:type:`ipu3_uapi_params` will be sent to "ipu3-imgu parameters"
+video node in ``V4L2_BUF_TYPE_META_CAPTURE`` format.
+
+.. code-block:: c
+
+    struct ipu3_uapi_params {
+	/* Flags which of the settings below are to be applied */
+	struct ipu3_uapi_flags use __attribute__((aligned(32)));
+
+	/* Accelerator cluster parameters */
+	struct ipu3_uapi_acc_param acc_param;
+
+	/* ISP vector address space parameters */
+	struct ipu3_uapi_isp_lin_vmem_params lin_vmem_params;
+	struct ipu3_uapi_isp_tnr3_vmem_params tnr3_vmem_params;
+	struct ipu3_uapi_isp_xnr3_vmem_params xnr3_vmem_params;
+
+	/* ISP data memory (DMEM) parameters */
+	struct ipu3_uapi_isp_tnr3_params tnr3_dmem_params;
+	struct ipu3_uapi_isp_xnr3_params xnr3_dmem_params;
+
+	/* Optical black level compensation */
+	struct ipu3_uapi_obgrid_param obgrid_param;
+    } __packed;
+
+Intel IPU3 ImgU uAPI data types
+===============================
+
+.. kernel-doc:: include/uapi/linux/intel-ipu3.h

include/uapi/linux/intel-ipu3.h new file +143 -0

new file mode 100644
index 0000000..c2608b6
@@ -0,0 +1,2819 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (C) 2017 - 2018 Intel Corporation */
+
+#ifndef __IPU3_UAPI_H
+#define __IPU3_UAPI_H
+
+#include <linux/types.h>
+
+/********************* Key Acronyms *************************/
+/*
+ * ACC - Accelerator cluster
+ * ANR - Adaptive noise reduction
+ * AWB_FR- Auto white balance filter response statistics
+ * BNR - Bayer noise reduction parameters
+ * BDS - Bayer downscaler parameters
+ * CCM - Color correction matrix coefficients
+ * CDS - Chroma down sample
+ * CHNR - Chroma noise reduction
+ * CSC - Color space conversion
+ * DM - De-mosaic
+ * IEFd - Image enhancement filter directed
+ * Obgrid - Optical black level compensation
+ * OSYS - Output system configuration
+ * ROI - Region of interest
+ * SHD - Lens shading correction table
+ * TCC - Total color correction
+ * YDS - Y down sampling
+ * YTM - Y-tone mapping
+ */
+
+/*
+ * IPU3 DMA operations require buffers to be aligned at
+ * 32 byte boundaries
+ */
+
+/******************* ipu3_uapi_stats_3a *******************/
+
+#define IPU3_UAPI_MAX_STRIPES				2
+#define IPU3_UAPI_MAX_BUBBLE_SIZE			10
+
+#define IPU3_UAPI_GRID_START_MASK			((1 << 12) - 1)
+#define IPU3_UAPI_GRID_Y_START_EN			(1 << 15)
+
+/* controls generation of meta_data (like FF enable/disable) */
+#define IPU3_UAPI_AWB_RGBS_THR_B_EN			(1 << 14)
+#define IPU3_UAPI_AWB_RGBS_THR_B_INCL_SAT		(1 << 15)
+
+/**
+ * struct ipu3_uapi_grid_config - Grid plane config
+ *
+ * @width:	Grid horizontal dimensions, in number of grid blocks(cells).
+ * @height:	Grid vertical dimensions, in number of grid cells.
+ * @block_width_log2:	Log2 of the width of each cell in pixels.
+ *			for (2^3, 2^4, 2^5, 2^6, 2^7), values [3, 7].
+ * @block_height_log2:	Log2 of the height of each cell in pixels.
+ *			for (2^3, 2^4, 2^5, 2^6, 2^7), values [3, 7].
+ * @height_per_slice:	The number of blocks in vertical axis per slice.
+ *			Default 2.
+ * @x_start: X value of top left corner of Region of Interest(ROI).
+ * @y_start: Y value of top left corner of ROI
+ * @x_end: X value of bottom right corner of ROI
+ * @y_end: Y value of bottom right corner of ROI
+ *
+ * Due to the size of total amount of collected data, most statistics
+ * create a grid-based output, and the data is then divided into "slices".
+ */
+struct ipu3_uapi_grid_config {
+	__u8 width;
+	__u8 height;
+	__u16 block_width_log2:3;
+	__u16 block_height_log2:3;
+	__u16 height_per_slice:8;
+	__u16 x_start;
+	__u16 y_start;
+	__u16 x_end;
+	__u16 y_end;
+} __packed;
+
+/*
+ * The grid based data is divided into "slices" called set, each slice of setX
+ * refers to ipu3_uapi_grid_config width * height_per_slice.
+ */
+#define IPU3_UAPI_AWB_MAX_SETS				60
+/* Based on grid size 80 * 60 and cell size 16 x 16 */
+#define IPU3_UAPI_AWB_SET_SIZE				1280
+#define IPU3_UAPI_AWB_MD_ITEM_SIZE			8
+#define IPU3_UAPI_AWB_SPARE_FOR_BUBBLES \
+	(IPU3_UAPI_MAX_BUBBLE_SIZE * IPU3_UAPI_MAX_STRIPES * \
+	 IPU3_UAPI_AWB_MD_ITEM_SIZE)
+#define IPU3_UAPI_AWB_MAX_BUFFER_SIZE \
+	(IPU3_UAPI_AWB_MAX_SETS * \
+	 (IPU3_UAPI_AWB_SET_SIZE + IPU3_UAPI_AWB_SPARE_FOR_BUBBLES))
+/**
+ * struct ipu3_uapi_awb_meta_data - AWB meta data
+ *
+ * @meta_data_buffer:	Average values for each color channel
+ */
+struct ipu3_uapi_awb_meta_data {
+	__u8 meta_data_buffer[IPU3_UAPI_AWB_MAX_BUFFER_SIZE];
+} __packed;
+
+/**
+ * struct ipu3_uapi_awb_raw_buffer - AWB raw buffer
+ *
+ * @meta_data: buffer to hold auto white balance meta data.
+ */
+struct ipu3_uapi_awb_raw_buffer {
+	struct ipu3_uapi_awb_meta_data meta_data;
+} __packed;
+
+/**
+ * struct ipu3_uapi_awb_config_s - AWB config
+ *
+ * @rgbs_thr_gr: gr threshold value.
+ * @rgbs_thr_r: Red threshold value.
+ * @rgbs_thr_gb: gb threshold value.
+ * @rgbs_thr_b: Blue threshold value.
+ * @grid: &ipu3_uapi_grid_config, the default grid resolution is 16x16 cells.
+ *
+ * The threshold is a saturation measure range [0, 8191], 8191 is default.
+ * Values over threshold may be optionally rejected for averaging.
+ */
+struct ipu3_uapi_awb_config_s {
+	__u16 rgbs_thr_gr;
+	__u16 rgbs_thr_r;
+	__u16 rgbs_thr_gb;
+	__u16 rgbs_thr_b;
+	struct ipu3_uapi_grid_config grid;
+} __attribute__((aligned(32))) __packed;
+
+/**
+ * struct ipu3_uapi_awb_config - AWB config wrapper
+ *
+ * @config: config for auto white balance as defined by &ipu3_uapi_awb_config_s
+ */
+struct ipu3_uapi_awb_config {
+	struct ipu3_uapi_awb_config_s config __attribute__((aligned(32)));
+} __packed;
+
+#define IPU3_UAPI_AE_COLORS				4	/* R, G, B, Y */
+#define IPU3_UAPI_AE_BINS				256
+#define IPU3_UAPI_AE_WEIGHTS				96
+

include/uapi/linux/intel-ipu3.h new file +85 -0

+ * @rst_hist_array: write 1 to trigger histogram array reset.
+ * @done_rst_hist_array: flag for histogram array reset done.
+ * @x_start: X value of top left corner of ROI, default 0.
+ * @y_start: Y value of top left corner of ROI, default 0.
+ * @x_end: X value of bottom right corner of ROI
+ * @y_end: Y value of bottom right corner of ROI
+ *
+ * The AE block accumulates 4 global weighted histograms(R, G, B, Y) over
+ * a defined ROI within the frame. The contribution of each pixel into the
+ * histogram, defined by &ipu3_uapi_ae_weight_elem LUT, is indexed by a grid.
+ */
+struct ipu3_uapi_ae_grid_config {
+	__u8 width;
+	__u8 height;
+	__u8 block_width_log2:4;
+	__u8 block_height_log2:4;
+	__u8 __reserved0:5;
+	__u8 ae_en:1;
+	__u8 rst_hist_array:1;
+	__u8 done_rst_hist_array:1;
+	__u16 x_start;
+	__u16 y_start;
+	__u16 x_end;
+	__u16 y_end;
+} __packed;
+
+/**
+ * struct ipu3_uapi_ae_weight_elem - AE weights LUT
+ *
+ * @cell0: weighted histogram grid value.
+ * @cell1: weighted histogram grid value.
+ * @cell2: weighted histogram grid value.
+ * @cell3: weighted histogram grid value.
+ * @cell4: weighted histogram grid value.
+ * @cell5: weighted histogram grid value.
+ * @cell6: weighted histogram grid value.
+ * @cell7: weighted histogram grid value.
+ *
+ * Use weighted grid value to give a different contribution factor to each cell.
+ * Precision u4, range [0, 15].
+ */
+struct ipu3_uapi_ae_weight_elem {
+	__u32 cell0:4;
+	__u32 cell1:4;
+	__u32 cell2:4;
+	__u32 cell3:4;
+	__u32 cell4:4;
+	__u32 cell5:4;
+	__u32 cell6:4;
+	__u32 cell7:4;
+} __packed;
+
+/**
+ * struct ipu3_uapi_ae_ccm - AE coefficients for WB and CCM
+ *
+ * @gain_gr: WB gain factor for the gr channels. Default 256.
+ * @gain_r: WB gain factor for the r channel. Default 256.
+ * @gain_b: WB gain factor for the b channel. Default 256.
+ * @gain_gb: WB gain factor for the gb channels. Default 256.
+ * @mat: 4x4 matrix that transforms Bayer quad output from WB to RGB+Y.
+ *
+ * Default:
+ *	128, 0, 0, 0,
+ *	0, 128, 0, 0,
+ *	0, 0, 128, 0,
+ *	0, 0, 0, 128,
+ *
+ * As part of the raw frame pre-process stage, the WB and color conversion need
+ * to be applied to expose the impact of these gain operations.
+ */
+struct ipu3_uapi_ae_ccm {
+	__u16 gain_gr;
+	__u16 gain_r;
+	__u16 gain_b;
+	__u16 gain_gb;
+	__s16 mat[16];
+} __packed;
+
+/**
+ * struct ipu3_uapi_ae_config - AE config
+ *
+ * @grid_cfg:	config for auto exposure statistics grid. See struct
+ *		&ipu3_uapi_ae_grid_config
+ * @weights:	&IPU3_UAPI_AE_WEIGHTS is based on 32x24 blocks in the grid.
+ *		Each grid cell has a corresponding value in weights LUT called

include/uapi/linux/intel-ipu3.h new file +142 -0

+	struct ipu3_uapi_ae_ccm ae_ccm __attribute__((aligned(32)));
+} __packed;
+
+/**
+ * struct ipu3_uapi_af_filter_config - AF 2D filter for contrast measurements
+ *
+ * @y1_coeff_0:	filter Y1, structure: 3x11, support both symmetry and
+ *		anti-symmetry type. A12 is center, A1-A11 are neighbours.
+ *		for analyzing low frequency content, used to calculate sum
+ *		of gradients in x direction.
+ * @y1_coeff_0.a1:	filter1 coefficients A1, u8, default 0.
+ * @y1_coeff_0.a2:	filter1 coefficients A2, u8, default 0.
+ * @y1_coeff_0.a3:	filter1 coefficients A3, u8, default 0.
+ * @y1_coeff_0.a4:	filter1 coefficients A4, u8, default 0.
+ * @y1_coeff_1:		Struct
+ * @y1_coeff_1.a5:	filter1 coefficients A5, u8, default 0.
+ * @y1_coeff_1.a6:	filter1 coefficients A6, u8, default 0.
+ * @y1_coeff_1.a7:	filter1 coefficients A7, u8, default 0.
+ * @y1_coeff_1.a8:	filter1 coefficients A8, u8, default 0.
+ * @y1_coeff_2:		Struct
+ * @y1_coeff_2.a9:	filter1 coefficients A9, u8, default 0.
+ * @y1_coeff_2.a10:	filter1 coefficients A10, u8, default 0.
+ * @y1_coeff_2.a11:	filter1 coefficients A11, u8, default 0.
+ * @y1_coeff_2.a12:	filter1 coefficients A12, u8, default 128.
+ * @y1_sign_vec:	Each bit corresponds to one coefficient sign bit,
+ *			0: positive, 1: negative, default 0.
+ * @y2_coeff_0:	Y2, same structure as Y1. For analyzing high frequency content.
+ * @y2_coeff_0.a1:	filter2 coefficients A1, u8, default 0.
+ * @y2_coeff_0.a2:	filter2 coefficients A2, u8, default 0.
+ * @y2_coeff_0.a3:	filter2 coefficients A3, u8, default 0.
+ * @y2_coeff_0.a4:	filter2 coefficients A4, u8, default 0.
+ * @y2_coeff_1:	Struct
+ * @y2_coeff_1.a5:	filter2 coefficients A5, u8, default 0.
+ * @y2_coeff_1.a6:	filter2 coefficients A6, u8, default 0.
+ * @y2_coeff_1.a7:	filter2 coefficients A7, u8, default 0.
+ * @y2_coeff_1.a8:	filter2 coefficients A8, u8, default 0.
+ * @y2_coeff_2:	Struct
+ * @y2_coeff_2.a9:	filter1 coefficients A9, u8, default 0.
+ * @y2_coeff_2.a10:	filter1 coefficients A10, u8, default 0.
+ * @y2_coeff_2.a11:	filter1 coefficients A11, u8, default 0.
+ * @y2_coeff_2.a12:	filter1 coefficients A12, u8, default 128.
+ * @y2_sign_vec:	Each bit corresponds to one coefficient sign bit,
+ *			0: positive, 1: negative, default 0.
+ * @y_calc:	Pre-processing that converts Bayer quad to RGB+Y values to be
+ *		used for building histogram. Range [0, 32], default 8.
+ * Rule:
+ *		y_gen_rate_gr + y_gen_rate_r + y_gen_rate_b + y_gen_rate_gb = 32
+ *		A single Y is calculated based on sum of Gr/R/B/Gb based on
+ *		their contribution ratio.
+ * @y_calc.y_gen_rate_gr:	Contribution ratio Gr for Y
+ * @y_calc.y_gen_rate_r:	Contribution ratio R for Y
+ * @y_calc.y_gen_rate_b:	Contribution ratio B for Y
+ * @y_calc.y_gen_rate_gb:	Contribution ratio Gb for Y
+ * @nf:	The shift right value that should be applied during the Y1/Y2 filter to
+ *	make sure the total memory needed is 2 bytes per grid cell.
+ * @nf.__reserved0:	reserved
+ * @nf.y1_nf:	Normalization factor for the convolution coeffs of y1,
+ *		should be log2 of the sum of the abs values of the filter
+ *		coeffs, default 7 (2^7 = 128).
+ * @nf.__reserved1:	reserved
+ * @nf.y2_nf:	Normalization factor for y2, should be log2 of the sum of the
+ *		abs values of the filter coeffs.
+ * @nf.__reserved2:	reserved
+ */
+struct ipu3_uapi_af_filter_config {
+	struct {
+		__u8 a1;
+		__u8 a2;
+		__u8 a3;
+		__u8 a4;
+	} y1_coeff_0;
+	struct {
+		__u8 a5;
+		__u8 a6;
+		__u8 a7;
+		__u8 a8;
+	} y1_coeff_1;
+	struct {
+		__u8 a9;
+		__u8 a10;
+		__u8 a11;
+		__u8 a12;
+	} y1_coeff_2;
+
+	__u32 y1_sign_vec;
+
+	struct {
+		__u8 a1;
+		__u8 a2;
+		__u8 a3;
+		__u8 a4;
+	} y2_coeff_0;
+	struct {
+		__u8 a5;
+		__u8 a6;
+		__u8 a7;
+		__u8 a8;
+	} y2_coeff_1;
+	struct {
+		__u8 a9;
+		__u8 a10;
+		__u8 a11;
+		__u8 a12;
+	} y2_coeff_2;
+
+	__u32 y2_sign_vec;
+
+	struct {
+		__u8 y_gen_rate_gr;
+		__u8 y_gen_rate_r;
+		__u8 y_gen_rate_b;
+		__u8 y_gen_rate_gb;
+	} y_calc;
+
+	struct {
+		__u32 __reserved0:8;
+		__u32 y1_nf:4;
+		__u32 __reserved1:4;
+		__u32 y2_nf:4;
+		__u32 __reserved2:12;
+	} nf;
+} __packed;
+
+#define IPU3_UAPI_AF_MAX_SETS				24
+#define IPU3_UAPI_AF_MD_ITEM_SIZE			4
+#define IPU3_UAPI_AF_SPARE_FOR_BUBBLES \
+	(IPU3_UAPI_MAX_BUBBLE_SIZE * IPU3_UAPI_MAX_STRIPES * \
+	 IPU3_UAPI_AF_MD_ITEM_SIZE)
+#define IPU3_UAPI_AF_Y_TABLE_SET_SIZE			128
+#define IPU3_UAPI_AF_Y_TABLE_MAX_SIZE \
+	(IPU3_UAPI_AF_MAX_SETS * \
+	 (IPU3_UAPI_AF_Y_TABLE_SET_SIZE + IPU3_UAPI_AF_SPARE_FOR_BUBBLES) * \
+	 IPU3_UAPI_MAX_STRIPES)
+
+/**
+ * struct ipu3_uapi_af_meta_data - AF meta data
+ *
+ * @y_table:	Each color component will be convolved separately with filter1
+ *		and filter2 and the result will be summed out and averaged for
+ *		each cell.
+ */
+struct ipu3_uapi_af_meta_data {

include/uapi/linux/intel-ipu3.h new file +13 -0

+} __packed;
+
+/**
+ * struct ipu3_uapi_af_config_s - AF config
+ *
+ * @filter_config: AF uses Y1 and Y2 filters as configured in
+ *		   &ipu3_uapi_af_filter_config
+ * @padding: paddings
+ * @grid_cfg: See &ipu3_uapi_grid_config, default resolution 16x16. Use large
+ *	      grid size for large image and vice versa.
+ */
+struct ipu3_uapi_af_config_s {
+	struct ipu3_uapi_af_filter_config filter_config __attribute__((aligned(32)));

include/uapi/linux/intel-ipu3.h new file +19 -0

+	struct ipu3_uapi_af_config_s config;
+} __packed;
+
+#define IPU3_UAPI_AWB_FR_MAX_SETS			24
+#define IPU3_UAPI_AWB_FR_MD_ITEM_SIZE			8
+#define IPU3_UAPI_AWB_FR_BAYER_TBL_SIZE			256
+#define IPU3_UAPI_AWB_FR_SPARE_FOR_BUBBLES \
+	(IPU3_UAPI_MAX_BUBBLE_SIZE * IPU3_UAPI_MAX_STRIPES * \
+	 IPU3_UAPI_AWB_FR_MD_ITEM_SIZE)
+#define IPU3_UAPI_AWB_FR_BAYER_TABLE_MAX_SIZE \
+	(IPU3_UAPI_AWB_FR_MAX_SETS * \
+	(IPU3_UAPI_AWB_FR_BAYER_TBL_SIZE + \
+	 IPU3_UAPI_AWB_FR_SPARE_FOR_BUBBLES) * IPU3_UAPI_MAX_STRIPES)
+
+/**
+ * struct ipu3_uapi_awb_fr_meta_data - AWB filter response meta data
+ *
+ * @bayer_table: Statistics output on the grid after convolving with 1D filter.
+ */

include/uapi/linux/intel-ipu3.h new file +24 -0

+	struct ipu3_uapi_awb_fr_meta_data meta_data;
+} __packed;
+
+/**
+ * struct ipu3_uapi_awb_fr_config_s - AWB filter response config
+ *
+ * @grid_cfg:	grid config, default 16x16.
+ * @bayer_coeff:	1D Filter 1x11 center symmetry/anti-symmetry.
+ *			coeffcients defaults { 0, 0, 0, 0, 0, 128 }.
+ *			Applied on whole image for each Bayer channel separately
+ *			by a weighted sum of its 11x1 neighbors.
+ * @__reserved1:	reserved
+ * @bayer_sign:	sign of filter coeffcients, default 0.
+ * @bayer_nf:	normalization factor for the convolution coeffs, to make sure
+ *		total memory needed is within pre-determined range.
+ *		NF should be the log2 of the sum of the abs values of the
+ *		filter coeffs, range [7, 14], default 7.
+ * @__reserved2:	reserved
+ */
+struct ipu3_uapi_awb_fr_config_s {
+	struct ipu3_uapi_grid_config grid_cfg;
+	__u8 bayer_coeff[6];
+	__u16 __reserved1;
+	__u32 bayer_sign;

include/uapi/linux/intel-ipu3.h new file +133 -0

+	struct ipu3_uapi_awb_fr_config_s config;
+} __packed;
+
+/**
+ * struct ipu3_uapi_4a_config - 4A config
+ *
+ * @awb_config: &ipu3_uapi_awb_config_s, default resolution 16x16
+ * @ae_grd_config: auto exposure statistics &ipu3_uapi_ae_grid_config
+ * @padding: paddings
+ * @af_config: auto focus config &ipu3_uapi_af_config_s
+ * @awb_fr_config: &ipu3_uapi_awb_fr_config_s, default resolution 16x16
+ */
+struct ipu3_uapi_4a_config {
+	struct ipu3_uapi_awb_config_s awb_config __attribute__((aligned(32)));
+	struct ipu3_uapi_ae_grid_config ae_grd_config;
+	__u8 padding[20];
+	struct ipu3_uapi_af_config_s af_config;
+	struct ipu3_uapi_awb_fr_config_s awb_fr_config;
+} __packed;
+
+/**
+ * struct ipu3_uapi_bubble_info - Bubble info for host side debugging
+ *
+ * @num_of_stripes: A single frame is divided into several parts called stripes
+ *		    due to limitation on line buffer memory.
+ *		    The separation between the stripes is vertical. Each such
+ *		    stripe is processed as a single frame by the ISP pipe.
+ * @padding: padding bytes.
+ * @num_sets: number of sets.
+ * @padding1: padding bytes.
+ * @size_of_set: set size.
+ * @padding2: padding bytes.
+ * @bubble_size: is the amount of padding in the bubble expressed in "sets".
+ * @padding3: padding bytes.
+ */
+struct ipu3_uapi_bubble_info {
+	__u32 num_of_stripes __attribute__((aligned(32)));
+	__u8 padding[28];
+	__u32 num_sets;
+	__u8 padding1[28];
+	__u32 size_of_set;
+	__u8 padding2[28];
+	__u32 bubble_size;
+	__u8 padding3[28];
+} __packed;
+
+/*
+ * struct ipu3_uapi_stats_3a_bubble_info_per_stripe
+ */
+struct ipu3_uapi_stats_3a_bubble_info_per_stripe {
+	struct ipu3_uapi_bubble_info awb[IPU3_UAPI_MAX_STRIPES];
+	struct ipu3_uapi_bubble_info af[IPU3_UAPI_MAX_STRIPES];
+	struct ipu3_uapi_bubble_info awb_fr[IPU3_UAPI_MAX_STRIPES];
+} __packed;
+
+/**
+ * struct ipu3_uapi_ff_status - Enable bits for each 3A fixed function
+ *
+ * @awb_en: auto white balance enable
+ * @padding: padding config
+ * @ae_en: auto exposure enable
+ * @padding1: padding config
+ * @af_en: auto focus enable
+ * @padding2: padding config
+ * @awb_fr_en: awb filter response enable bit
+ * @padding3: padding config
+ */
+struct ipu3_uapi_ff_status {
+	__u32 awb_en __attribute__((aligned(32)));
+	__u8 padding[28];
+	__u32 ae_en;
+	__u8 padding1[28];
+	__u32 af_en;
+	__u8 padding2[28];
+	__u32 awb_fr_en;
+	__u8 padding3[28];
+} __packed;
+
+/**
+ * struct ipu3_uapi_stats_3a - 3A statistics
+ *
+ * @awb_raw_buffer: auto white balance meta data &ipu3_uapi_awb_raw_buffer
+ * @ae_raw_buffer: auto exposure raw data &ipu3_uapi_ae_raw_buffer_aligned
+ * @af_raw_buffer: &ipu3_uapi_af_raw_buffer for auto focus meta data
+ * @awb_fr_raw_buffer: value as specified by &ipu3_uapi_awb_fr_raw_buffer
+ * @stats_4a_config: 4a statistics config as defined by &ipu3_uapi_4a_config.
+ * @ae_join_buffers: 1 to use ae_raw_buffer.
+ * @padding: padding config
+ * @stats_3a_bubble_per_stripe: a &ipu3_uapi_stats_3a_bubble_info_per_stripe
+ * @stats_3a_status: 3a statistics status set in &ipu3_uapi_ff_status
+ */
+struct ipu3_uapi_stats_3a {
+	struct ipu3_uapi_awb_raw_buffer awb_raw_buffer __attribute__((aligned(32)));
+	struct ipu3_uapi_ae_raw_buffer_aligned
+			ae_raw_buffer[IPU3_UAPI_MAX_STRIPES];
+	struct ipu3_uapi_af_raw_buffer af_raw_buffer;
+	struct ipu3_uapi_awb_fr_raw_buffer awb_fr_raw_buffer;
+	struct ipu3_uapi_4a_config stats_4a_config;
+	__u32 ae_join_buffers;
+	__u8 padding[28];
+	struct ipu3_uapi_stats_3a_bubble_info_per_stripe
+			stats_3a_bubble_per_stripe;
+	struct ipu3_uapi_ff_status stats_3a_status;
+} __packed;
+
+/******************* ipu3_uapi_acc_param *******************/
+
+#define IPU3_UAPI_ISP_VEC_ELEMS				64
+#define IPU3_UAPI_ISP_TNR3_VMEM_LEN			9
+
+#define IPU3_UAPI_BNR_LUT_SIZE				32
+
+/* number of elements in gamma correction LUT */
+#define IPU3_UAPI_GAMMA_CORR_LUT_ENTRIES		256
+
+/* largest grid is 73x56, for grid_height_per_slice of 2, 73x2 = 146 */
+#define IPU3_UAPI_SHD_MAX_CELLS_PER_SET			146
+#define IPU3_UAPI_SHD_MAX_CFG_SETS			28
+/* Normalization shift aka nf */
+#define IPU3_UAPI_SHD_BLGR_NF_SHIFT			13
+#define IPU3_UAPI_SHD_BLGR_NF_MASK			7
+
+#define IPU3_UAPI_YUVP2_TCC_MACC_TABLE_ELEMENTS		16
+#define IPU3_UAPI_YUVP2_TCC_INV_Y_LUT_ELEMENTS		14
+#define IPU3_UAPI_YUVP2_TCC_GAIN_PCWL_LUT_ELEMENTS	258
+#define IPU3_UAPI_YUVP2_TCC_R_SQR_LUT_ELEMENTS		24
+
+#define IPU3_UAPI_ANR_LUT_SIZE				26
+#define IPU3_UAPI_ANR_PYRAMID_SIZE			22
+
+#define IPU3_UAPI_LIN_LUT_SIZE				64
+
+/* Bayer Noise Reduction related structs */

include/uapi/linux/intel-ipu3.h new file +395 -0

+ * a multiplicative gain to each color channels prior to BNR.
+ */
+struct ipu3_uapi_bnr_static_config_wb_gains_config {
+	__u16 gr;
+	__u16 r;
+	__u16 b;
+	__u16 gb;
+} __packed;
+
+/**
+ * struct ipu3_uapi_bnr_static_config_wb_gains_thr_config - Threshold config
+ *
+ * @gr:	white balance threshold gain for Gr channel.
+ * @r:	white balance threshold gain for R channel.
+ * @b:	white balance threshold gain for B channel.
+ * @gb:	white balance threshold gain for Gb channel.
+ *
+ * Defines the threshold that specifies how different a defect pixel can be from
+ * its neighbors.(used by dynamic defect pixel correction sub block)
+ * Precision u4.4 range [0, 8].
+ */
+struct ipu3_uapi_bnr_static_config_wb_gains_thr_config {
+	__u8 gr;
+	__u8 r;
+	__u8 b;
+	__u8 gb;
+} __packed;
+
+/**
+ * struct ipu3_uapi_bnr_static_config_thr_coeffs_config - Noise model
+ *				coefficients that controls noise threshold
+ *
+ * @cf:	Free coefficient for threshold calculation, range [0, 8191], default 0.
+ * @__reserved0:	reserved
+ * @cg:	Gain coefficient for threshold calculation, [0, 31], default 8.
+ * @ci:	Intensity coefficient for threshold calculation. range [0, 0x1f]
+ *	default 6.
+ * 	format: u3.2 (3 most significant bits represent whole number,
+ *	2 least significant bits represent the fractional part
+ *	with each count representing 0.25)
+ *	e.g 6 in binary format is 00110, that translates to 1.5
+ * @__reserved1:	reserved
+ * @r_nf:	Normalization shift value for r^2 calculation, range [12, 20]
+ *		where r is a radius of pixel [row, col] from centor of sensor.
+ *		default 14.
+ *
+ * Threshold used to distinguish between noise and details.
+ */
+struct ipu3_uapi_bnr_static_config_thr_coeffs_config {
+	__u32 cf:13;
+	__u32 __reserved0:3;
+	__u32 cg:5;
+	__u32 ci:5;
+	__u32 __reserved1:1;
+	__u32 r_nf:5;
+} __packed;
+
+/**
+ * struct ipu3_uapi_bnr_static_config_thr_ctrl_shd_config - Shading config
+ *
+ * @gr:	Coefficient defines lens shading gain approximation for gr channel
+ * @r:	Coefficient defines lens shading gain approximation for r channel
+ * @b:	Coefficient defines lens shading gain approximation for b channel
+ * @gb:	Coefficient defines lens shading gain approximation for gb channel
+ *
+ * Parameters for noise model (NM) adaptation of BNR due to shading correction.
+ * All above have precision of u3.3, default to 0.
+ */
+struct ipu3_uapi_bnr_static_config_thr_ctrl_shd_config {
+	__u8 gr;
+	__u8 r;
+	__u8 b;
+	__u8 gb;
+} __packed;
+
+/**
+ * struct ipu3_uapi_bnr_static_config_opt_center_config - Optical center config
+ *
+ * @x_reset:	Reset value of X (col start - X center). Precision s12.0.
+ * @__reserved0:	reserved
+ * @y_reset:	Reset value of Y (row start - Y center). Precision s12.0.
+ * @__reserved2:	reserved
+ *
+ * Distance from corner to optical center for NM adaptation due to shading
+ * correction (should be calculated based on shading tables)
+ */
+struct ipu3_uapi_bnr_static_config_opt_center_config {
+	__s32 x_reset:13;
+	__u32 __reserved0:3;
+	__s32 y_reset:13;
+	__u32 __reserved2:3;
+} __packed;
+
+/**
+ * struct ipu3_uapi_bnr_static_config_lut_config - BNR square root lookup table
+ *
+ * @values: pre-calculated values of square root function.
+ *
+ * LUT implementation of square root operation.
+ */
+struct ipu3_uapi_bnr_static_config_lut_config {
+	__u8 values[IPU3_UAPI_BNR_LUT_SIZE];
+} __packed;
+
+/**
+ * struct ipu3_uapi_bnr_static_config_bp_ctrl_config - Detect bad pixels (bp)
+ *
+ * @bp_thr_gain:	Defines the threshold that specifies how different a
+ *			defect pixel can be from its neighbors. Threshold is
+ *			dependent on de-noise threshold calculated by algorithm.
+ *			Range [4, 31], default 4.
+ * @__reserved0:	reserved
+ * @defect_mode:	Mode of addressed defect pixels,
+ *			0 - single defect pixel is expected,
+ *			1 - 2 adjacent defect pixels are expected, default 1.
+ * @bp_gain:	Defines how 2nd derivation that passes through a defect pixel
+ *		is different from 2nd derivations that pass through
+ *		neighbor pixels. u4.2, range [0, 256], default 8.
+ * @__reserved1:	reserved
+ * @w0_coeff:	Blending coefficient of defect pixel correction.
+ *		Precision u4, range [0, 8], default 8.
+ * @__reserved2:	reserved
+ * @w1_coeff:	Enable influence of incorrect defect pixel correction to be
+ *		avoided. Precision u4, range [1, 8], default 8.
+ * @__reserved3:	reserved
+ */
+struct ipu3_uapi_bnr_static_config_bp_ctrl_config {
+	__u32 bp_thr_gain:5;
+	__u32 __reserved0:2;
+	__u32 defect_mode:1;
+	__u32 bp_gain:6;
+	__u32 __reserved1:18;
+	__u32 w0_coeff:4;
+	__u32 __reserved2:4;
+	__u32 w1_coeff:4;
+	__u32 __reserved3:20;
+} __packed;
+
+/**
+ * struct ipu3_uapi_bnr_static_config_dn_detect_ctrl_config - Denoising config
+ *
+ * @alpha:	Weight of central element of smoothing filter.
+ * @beta:	Weight of peripheral elements of smoothing filter, default 4.
+ * @gamma:	Weight of diagonal elements of smoothing filter, default 4.
+ *
+ * beta and gamma parameter define the strength of the noise removal filter.
+ *		All above has precision u0.4, range [0, 0xf]
+ *		format: u0.4 (no / zero bits represent whole number,
+ *		4 bits represent the fractional part
+ *		with each count representing 0.0625)
+ *		e.g 0xf translates to 0.0625x15 = 0.9375
+ *
+ * @__reserved0:	reserved
+ * @max_inf:	Maximum increase of peripheral or diagonal element influence
+ *		relative to the pre-defined value range: [0x5, 0xa]
+ * @__reserved1:	reserved
+ * @gd_enable:	Green disparity enable control, 0 - disable, 1 - enable.
+ * @bpc_enable:	Bad pixel correction enable control, 0 - disable, 1 - enable.
+ * @bnr_enable:	Bayer noise removal enable control, 0 - disable, 1 - enable.
+ * @ff_enable:	Fixed function enable, 0 - disable, 1 - enable.
+ * @__reserved2:	reserved
+ */
+struct ipu3_uapi_bnr_static_config_dn_detect_ctrl_config {
+	__u32 alpha:4;
+	__u32 beta:4;
+	__u32 gamma:4;
+	__u32 __reserved0:4;
+	__u32 max_inf:4;
+	__u32 __reserved1:7;
+	__u32 gd_enable:1;
+	__u32 bpc_enable:1;
+	__u32 bnr_enable:1;
+	__u32 ff_enable:1;
+	__u32 __reserved2:1;
+} __packed;
+
+/**
+ * struct ipu3_uapi_bnr_static_config_opt_center_sqr_config - BNR optical square
+ *
+ * @x_sqr_reset: Reset value of X^2.
+ * @y_sqr_reset: Reset value of Y^2.
+ *
+ * Please note:
+ *
+ *    #. X and Y ref to
+ *       &ipu3_uapi_bnr_static_config_opt_center_config
+ *    #. Both structs are used in threshold formula to calculate r^2, where r
+ *       is a radius of pixel [row, col] from centor of sensor.
+ */
+struct ipu3_uapi_bnr_static_config_opt_center_sqr_config {
+	__u32 x_sqr_reset;
+	__u32 y_sqr_reset;
+} __packed;
+
+/**
+ * struct ipu3_uapi_bnr_static_config - BNR static config
+ *
+ * @wb_gains:	white balance gains &ipu3_uapi_bnr_static_config_wb_gains_config
+ * @wb_gains_thr:	white balance gains threshold as defined by
+ *			&ipu3_uapi_bnr_static_config_wb_gains_thr_config
+ * @thr_coeffs:	coefficients of threshold
+ *		&ipu3_uapi_bnr_static_config_thr_coeffs_config
+ * @thr_ctrl_shd:	control of shading threshold
+ *			&ipu3_uapi_bnr_static_config_thr_ctrl_shd_config
+ * @opt_center:	optical center &ipu3_uapi_bnr_static_config_opt_center_config
+ *
+ * Above parameters and opt_center_sqr are used for white balance and shading.
+ *
+ * @lut:	lookup table &ipu3_uapi_bnr_static_config_lut_config
+ * @bp_ctrl:	detect and remove bad pixels as defined in struct
+ *		&ipu3_uapi_bnr_static_config_bp_ctrl_config
+ * @dn_detect_ctrl:	detect and remove noise.
+ *			&ipu3_uapi_bnr_static_config_dn_detect_ctrl_config
+ * @column_size:	The number of pixels in column.
+ * @opt_center_sqr:	Reset value of r^2 to optical center, see
+ *			&ipu3_uapi_bnr_static_config_opt_center_sqr_config.
+ */
+struct ipu3_uapi_bnr_static_config {
+	struct ipu3_uapi_bnr_static_config_wb_gains_config wb_gains;
+	struct ipu3_uapi_bnr_static_config_wb_gains_thr_config wb_gains_thr;
+	struct ipu3_uapi_bnr_static_config_thr_coeffs_config thr_coeffs;
+	struct ipu3_uapi_bnr_static_config_thr_ctrl_shd_config thr_ctrl_shd;
+	struct ipu3_uapi_bnr_static_config_opt_center_config opt_center;
+	struct ipu3_uapi_bnr_static_config_lut_config lut;
+	struct ipu3_uapi_bnr_static_config_bp_ctrl_config bp_ctrl;
+	struct ipu3_uapi_bnr_static_config_dn_detect_ctrl_config dn_detect_ctrl;
+	__u32 column_size;
+	struct ipu3_uapi_bnr_static_config_opt_center_sqr_config opt_center_sqr;
+} __packed;
+
+/**
+ * struct ipu3_uapi_bnr_static_config_green_disparity - Correct green disparity
+ *
+ * @gd_red:	Shading gain coeff for gr disparity level in bright red region.
+ *		Precision u0.6, default 4(0.0625).
+ * @__reserved0:	reserved
+ * @gd_green:	Shading gain coeff for gr disparity level in bright green
+ *		region. Precision u0.6, default 4(0.0625).
+ * @__reserved1:	reserved
+ * @gd_blue:	Shading gain coeff for gr disparity level in bright blue region.
+ *		Precision u0.6, default 4(0.0625).
+ * @__reserved2:	reserved
+ * @gd_black:	Maximal green disparity level in dark region (stronger disparity
+ *		assumed to be image detail). Precision u14, default 80.
+ * @__reserved3:	reserved
+ * @gd_shading:	Change maximal green disparity level according to square
+ *		distance from image center.
+ * @__reserved4:	reserved
+ * @gd_support:	Lower bound for the number of second green color pixels in
+ *		current pixel neighborhood with less than threshold difference
+ *		from it.
+ *
+ * The shading gain coeff of red, green, blue and black are used to calculate
+ * threshold given a pixel's color value and its coordinates in the image.
+ *
+ * @__reserved5:	reserved
+ * @gd_clip:	Turn green disparity clip on/off, [0, 1], default 1.
+ * @gd_central_weight:	Central pixel weight in 9 pixels weighted sum.
+ */
+struct ipu3_uapi_bnr_static_config_green_disparity {
+	__u32 gd_red:6;
+	__u32 __reserved0:2;
+	__u32 gd_green:6;
+	__u32 __reserved1:2;
+	__u32 gd_blue:6;
+	__u32 __reserved2:10;
+	__u32 gd_black:14;
+	__u32 __reserved3:2;
+	__u32 gd_shading:7;
+	__u32 __reserved4:1;
+	__u32 gd_support:2;
+	__u32 __reserved5:1;
+	__u32 gd_clip:1;
+	__u32 gd_central_weight:4;
+} __packed;
+
+/**
+ * struct ipu3_uapi_dm_config - De-mosaic parameters
+ *
+ * @dm_en:	de-mosaic enable.
+ * @ch_ar_en:	Checker artifacts removal enable flag. Default 0.
+ * @fcc_en:	False color correction (FCC) enable flag. Default 0.
+ * @__reserved0:	reserved
+ * @frame_width:	do not care
+ * @gamma_sc:	Sharpening coefficient (coefficient of 2-d derivation of
+ *		complementary color in Hamilton-Adams interpolation).
+ *		u5, range [0, 31], default 8.
+ * @__reserved1:	reserved
+ * @lc_ctrl:	Parameter that controls weights of Chroma Homogeneity metric
+ *		in calculation of final homogeneity metric.
+ *		u5, range [0, 31], default 7.
+ * @__reserved2:	reserved
+ * @cr_param1:	First parameter that defines Checker artifact removal
+ *		feature gain.Precision u5, range [0, 31], default 8.
+ * @__reserved3:	reserved
+ * @cr_param2:	Second parameter that defines Checker artifact removal
+ *		feature gain. Precision u5, range [0, 31], default 8.
+ * @__reserved4:	reserved
+ * @coring_param:	Defines power of false color correction operation.
+ *			low for preserving edge colors, high for preserving gray
+ *			edge artifacts. u1.4, range [0, 1.9375], default 4(0.25).
+ * @__reserved5:	reserved
+ *
+ * The demosaic fixed function block is responsible to covert Bayer(mosaiced)
+ * images into color images based on demosaicing algorithm.
+ */
+struct ipu3_uapi_dm_config {
+	__u32 dm_en:1;
+	__u32 ch_ar_en:1;
+	__u32 fcc_en:1;
+	__u32 __reserved0:13;
+	__u32 frame_width:16;
+
+	__u32 gamma_sc:5;
+	__u32 __reserved1:3;
+	__u32 lc_ctrl:5;
+	__u32 __reserved2:3;
+	__u32 cr_param1:5;
+	__u32 __reserved3:3;
+	__u32 cr_param2:5;
+	__u32 __reserved4:3;
+
+	__u32 coring_param:5;
+	__u32 __reserved5:27;
+} __packed;
+
+/**
+ * struct ipu3_uapi_ccm_mat_config - Color correction matrix
+ *
+ * @coeff_m11: CCM 3x3 coefficient, range [-65536, 65535]
+ * @coeff_m12: CCM 3x3 coefficient, range [-8192, 8191]
+ * @coeff_m13: CCM 3x3 coefficient, range [-32768, 32767]
+ * @coeff_o_r: Bias 3x1 coefficient, range [-8191, 8181]
+ * @coeff_m21: CCM 3x3 coefficient, range [-32767, 32767]
+ * @coeff_m22: CCM 3x3 coefficient, range [-8192, 8191]
+ * @coeff_m23: CCM 3x3 coefficient, range [-32768, 32767]
+ * @coeff_o_g: Bias 3x1 coefficient, range [-8191, 8181]
+ * @coeff_m31: CCM 3x3 coefficient, range [-32768, 32767]
+ * @coeff_m32: CCM 3x3 coefficient, range [-8192, 8191]
+ * @coeff_m33: CCM 3x3 coefficient, range [-32768, 32767]
+ * @coeff_o_b: Bias 3x1 coefficient, range [-8191, 8181]
+ *
+ * Transform sensor specific color space to standard sRGB by applying 3x3 matrix
+ * and adding a bias vector O. The transformation is basically a rotation and
+ * translation in the 3-dimensional color spaces. Here are the defaults:
+ *
+ *	9775,	-2671,	1087,	0
+ *	-1071,	8303,	815,	0
+ *	-23,	-7887,	16103,	0
+ */
+struct ipu3_uapi_ccm_mat_config {
+	__s16 coeff_m11;
+	__s16 coeff_m12;
+	__s16 coeff_m13;
+	__s16 coeff_o_r;
+	__s16 coeff_m21;
+	__s16 coeff_m22;
+	__s16 coeff_m23;
+	__s16 coeff_o_g;
+	__s16 coeff_m31;
+	__s16 coeff_m32;
+	__s16 coeff_m33;
+	__s16 coeff_o_b;
+} __packed;
+
+/**
+ * struct ipu3_uapi_gamma_corr_ctrl - Gamma correction
+ *
+ * @enable: gamma correction enable.
+ * @__reserved: reserved
+ */
+struct ipu3_uapi_gamma_corr_ctrl {
+	__u32 enable:1;
+	__u32 __reserved:31;
+} __packed;
+
+/**
+ * struct ipu3_uapi_gamma_corr_lut - Per-pixel tone mapping implemented as LUT.
+ *
+ * @lut:	256 tabulated values of the gamma function. LUT[1].. LUT[256]
+ *		format u13.0, range [0, 8191].
+ *
+ * The tone mapping operation is done by a Piece wise linear graph
+ * that is implemented as a lookup table(LUT). The pixel component input
+ * intensity is the X-axis of the graph which is the table entry.
+ */
+struct ipu3_uapi_gamma_corr_lut {
+	__u16 lut[IPU3_UAPI_GAMMA_CORR_LUT_ENTRIES];
+} __packed;
+
+/**
+ * struct ipu3_uapi_gamma_config - Gamma config
+ *
+ * @gc_ctrl: control of gamma correction &ipu3_uapi_gamma_corr_ctrl
+ * @gc_lut: lookup table of gamma correction &ipu3_uapi_gamma_corr_lut