Skip to contentSkip to conversation

Allwinner A31/A83T MIPI CSI-2 and A31 ISP / ISP Driver

PATCH Superseded Paul Kocialkowski 3 participants 11 messages Jul 4, 2022 – Jul 19, 2022

Subsystems (6)

v5 of 8 Superseded Latest is v8 →

lore
Share & tools
lore
#0 lore raw

v5 cover Allwinner A31/A83T MIPI CSI-2 and A31 ISP / ISP Driver

This new version is an offspring from the big "Allwinner A31/A83T MIPI CSI-2 Support and A31 ISP Support" series, which was split into individual series for better clarity and handling.

This part only concerns the introduction of the new ISP driver.

Changes since v4:
- Fixed device-tree binding indent-align;
- Added collected tag;
- Rebased on latest media tree;

Changes since v3:
- Removed the v4l2 controls handler from the driver;
- Added variant structure for table sizes;
- Removed the info message about video device registration;
- Removed comments in uAPI header;
- Used '/schemas/graph.yaml#/properties/port' whenever possible in bindings;
- Added CSI patches dependent on the ISP driver;
- Rebased on the latest media tree;

Changes since all-in-one v2:
- Updated Kconfig to follow the latest media-wide changes;
- Reworked async subdev handling with a dedicated structure holding the corresponding source to avoid matching in the driver;
- Switched to clock-managed regmap mmio;
- Used helper to get a single enabled link for an entity's pad, to replace source selection at link_validate time and select the remote source at stream on time instead;
- Added mutex for mbus format serialization;
- Used endpoint-base instead of video-interface for "internal" endpoints in device-tree schema;
- Added TODO with unstaging requirements;
- Various cosmetic cleanups;
- Updated copyright years;

Paul Kocialkowski (6):
dt-bindings: media: Add Allwinner A31 ISP bindings documentation
dt-bindings: media: sun6i-a31-csi: Add ISP output port
staging: media: Add support for the Allwinner A31 ISP
MAINTAINERS: Add entry for the Allwinner A31 ISP driver
media: sun6i-csi: Detect the availability of the ISP
media: sun6i-csi: Add support for hooking to the isp devices

 .../media/allwinner,sun6i-a31-csi.yaml        |   4 +
 .../media/allwinner,sun6i-a31-isp.yaml        |  97 +++
 MAINTAINERS                                   |   9 +
 .../platform/sunxi/sun6i-csi/sun6i_csi.c      |  74 +-
 .../platform/sunxi/sun6i-csi/sun6i_csi.h      |  10 +
 .../sunxi/sun6i-csi/sun6i_csi_bridge.c        |  32 +-
 .../sunxi/sun6i-csi/sun6i_csi_capture.c       |  19 +-
 .../sunxi/sun6i-csi/sun6i_csi_capture.h       |   1 +
 drivers/staging/media/sunxi/Kconfig           |   1 +
 drivers/staging/media/sunxi/Makefile          |   1 +
 drivers/staging/media/sunxi/sun6i-isp/Kconfig |  15 +
 .../staging/media/sunxi/sun6i-isp/Makefile    |   4 +
 .../staging/media/sunxi/sun6i-isp/TODO.txt    |   6 +
 .../staging/media/sunxi/sun6i-isp/sun6i_isp.c | 555 +++++++++++++
 .../staging/media/sunxi/sun6i-isp/sun6i_isp.h |  90 +++
 .../media/sunxi/sun6i-isp/sun6i_isp_capture.c | 742 ++++++++++++++++++
 .../media/sunxi/sun6i-isp/sun6i_isp_capture.h |  78 ++
 .../media/sunxi/sun6i-isp/sun6i_isp_params.c  | 566 +++++++++++++
 .../media/sunxi/sun6i-isp/sun6i_isp_params.h  |  52 ++
 .../media/sunxi/sun6i-isp/sun6i_isp_proc.c    | 578 ++++++++++++++
 .../media/sunxi/sun6i-isp/sun6i_isp_proc.h    |  66 ++
 .../media/sunxi/sun6i-isp/sun6i_isp_reg.h     | 275 +++++++
 .../sunxi/sun6i-isp/uapi/sun6i-isp-config.h   |  43 +
 23 files changed, 3305 insertions(+), 13 deletions(-)
 create mode 100644 Documentation/devicetree/bindings/media/allwinner,sun6i-a31-isp.yaml
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/Kconfig
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/Makefile
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/TODO.txt
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp.c
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp.h
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_capture.c
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_capture.h
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_params.c
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_params.h
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_proc.c
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_proc.h
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_reg.h
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/uapi/sun6i-isp-config.h
Signature · 1 line
2.37.0
#1 lore raw

PATCH v5 1/6 dt-bindings: media: Add Allwinner A31 ISP bindings documentation

This introduces YAML bindings documentation for the Allwinner A31 Image Signal Processor (ISP).

---
 .../media/allwinner,sun6i-a31-isp.yaml        | 97 +++++++++++++++++++
 1 file changed, 97 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/media/allwinner,sun6i-a31-isp.yaml
Documentation/devicetree/bindings/media/allwinner,sun6i-a31-isp.yaml new file +97 -0
new file mode 100644
index 000000000000..2fda6e05e16c
@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/media/allwinner,sun6i-a31-isp.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Allwinner A31 Image Signal Processor Driver (ISP) Device Tree Bindings
+
+maintainers:
+  - Paul Kocialkowski <[email protected]>
+
+properties:
+  compatible:
+    enum:
+      - allwinner,sun6i-a31-isp
+      - allwinner,sun8i-v3s-isp
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  clocks:
+    items:
+      - description: Bus Clock
+      - description: Module Clock
+      - description: DRAM Clock
+
+  clock-names:
+    items:
+      - const: bus
+      - const: mod
+      - const: ram
+
+  resets:
+    maxItems: 1
+
+  ports:
+    $ref: /schemas/graph.yaml#/properties/ports
+
+    properties:
+      port@0:
+        $ref: /schemas/graph.yaml#/properties/port
+        description: CSI0 input port
+
+      port@1:
+        $ref: /schemas/graph.yaml#/properties/port
+        description: CSI1 input port
+
+    anyOf:
+      - required:
+          - port@0
+      - required:
+          - port@1
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - clocks
+  - clock-names
+  - resets
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/clock/sun8i-v3s-ccu.h>
+    #include <dt-bindings/reset/sun8i-v3s-ccu.h>
+
+    isp: isp@1cb8000 {
+        compatible = "allwinner,sun8i-v3s-isp";
+        reg = <0x01cb8000 0x1000>;
+        interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>;
+        clocks = <&ccu CLK_BUS_CSI>,
+             <&ccu CLK_CSI1_SCLK>,
+             <&ccu CLK_DRAM_CSI>;
+        clock-names = "bus", "mod", "ram";
+        resets = <&ccu RST_BUS_CSI>;
+
+        ports {
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            port@0 {
+                reg = <0>;
+
+                isp_in_csi0: endpoint {
+                    remote-endpoint = <&csi0_out_isp>;
+                };
+            };
+        };
+    };
+
+...
Signature · 1 line
2.37.0
#2 lore raw

Re: [PATCH v5 1/6] dt-bindings: media: Add Allwinner A31 ISP bindings documentation

Hi Paul,

Quoted 55 lines · Paul Kocialkowski

This introduces YAML bindings documentation for the Allwinner A31 Image Signal Processor (ISP).

---
 .../media/allwinner,sun6i-a31-isp.yaml        | 97 +++++++++++++++++++
 1 file changed, 97 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/media/allwinner,sun6i-a31-isp.yaml
Documentation/devicetree/bindings/media/allwinner,sun6i-a31-isp.yaml new file +39 -0
new file mode 100644
index 000000000000..2fda6e05e16c
@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/media/allwinner,sun6i-a31-isp.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Allwinner A31 Image Signal Processor Driver (ISP) Device Tree Bindings
+
+maintainers:
+  - Paul Kocialkowski <[email protected]>
+
+properties:
+  compatible:
+    enum:
+      - allwinner,sun6i-a31-isp
+      - allwinner,sun8i-v3s-isp
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  clocks:
+    items:
+      - description: Bus Clock
+      - description: Module Clock
+      - description: DRAM Clock
+
+  clock-names:
+    items:
+      - const: bus
+      - const: mod
+      - const: ram
+
+  resets:
+    maxItems: 1
+
+  ports:

Documentation/devicetree/bindings/media/allwinner,sun6i-a31-isp.yaml line 49

Quoted 10 lines
Documentation/devicetree/bindings/media/allwinner,sun6i-a31-isp.yaml new file +10 -0
+    $ref: /schemas/graph.yaml#/properties/ports
+
+    properties:
+      port@0:
+        $ref: /schemas/graph.yaml#/properties/port
+        description: CSI0 input port
+
+      port@1:
+        $ref: /schemas/graph.yaml#/properties/port
+        description: CSI1 input port

Do both support a single PHY with a single data only? If multiple data lanes are supported, please require data-lanes property (on endpoint).

Quoted 48 lines
Documentation/devicetree/bindings/media/allwinner,sun6i-a31-isp.yaml new file +48 -0
+
+    anyOf:
+      - required:
+          - port@0
+      - required:
+          - port@1
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - clocks
+  - clock-names
+  - resets
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/clock/sun8i-v3s-ccu.h>
+    #include <dt-bindings/reset/sun8i-v3s-ccu.h>
+
+    isp: isp@1cb8000 {
+        compatible = "allwinner,sun8i-v3s-isp";
+        reg = <0x01cb8000 0x1000>;
+        interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>;
+        clocks = <&ccu CLK_BUS_CSI>,
+             <&ccu CLK_CSI1_SCLK>,
+             <&ccu CLK_DRAM_CSI>;
+        clock-names = "bus", "mod", "ram";
+        resets = <&ccu RST_BUS_CSI>;
+
+        ports {
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            port@0 {
+                reg = <0>;
+
+                isp_in_csi0: endpoint {
+                    remote-endpoint = <&csi0_out_isp>;
+                };
+            };
+        };
+    };
+
+...
Signature · 1 line
Sakari Ailus
#3 lore raw

Re: [PATCH v5 1/6] dt-bindings: media: Add Allwinner A31 ISP bindings documentation

Hi Sakari,

Quoted 63 lines · Sakari Ailus

Hi Paul,

Quoted 60 lines · Paul Kocialkowski

This introduces YAML bindings documentation for the Allwinner A31 Image Signal Processor (ISP).

---
 .../media/allwinner,sun6i-a31-isp.yaml        | 97 +++++++++++++++++++
 1 file changed, 97 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/media/allwinner,sun6i-a31-isp.yaml
Documentation/devicetree/bindings/media/allwinner,sun6i-a31-isp.yaml new file +44 -0
new file mode 100644
index 000000000000..2fda6e05e16c
@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/media/allwinner,sun6i-a31-isp.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Allwinner A31 Image Signal Processor Driver (ISP) Device Tree Bindings
+
+maintainers:
+  - Paul Kocialkowski <[email protected]>
+
+properties:
+  compatible:
+    enum:
+      - allwinner,sun6i-a31-isp
+      - allwinner,sun8i-v3s-isp
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  clocks:
+    items:
+      - description: Bus Clock
+      - description: Module Clock
+      - description: DRAM Clock
+
+  clock-names:
+    items:
+      - const: bus
+      - const: mod
+      - const: ram
+
+  resets:
+    maxItems: 1
+
+  ports:
+    $ref: /schemas/graph.yaml#/properties/ports
+
+    properties:
+      port@0:
+        $ref: /schemas/graph.yaml#/properties/port
Quoted 8 lines

Documentation/devicetree/bindings/media/allwinner,sun6i-a31-isp.yaml line 49

Quoted 5 lines
Documentation/devicetree/bindings/media/allwinner,sun6i-a31-isp.yaml new file +5 -0
+        description: CSI0 input port
+
+      port@1:
+        $ref: /schemas/graph.yaml#/properties/port
+        description: CSI1 input port

Do both support a single PHY with a single data only? If multiple data lanes are supported, please require data-lanes property (on endpoint).

There's actually no PHY involved here: the ISP drivers gets its video stream from these CSI controllers which are the ones interfacing with a MIPI CSI-2 receiver (on A31/V3 it uses an external D-PHY, on A83T the D-PHY is collocated with the controller).

So I think the data-lanes property is not relevant here.

What do you think?

Cheers,

Paul

Quoted 51 lines
Quoted 48 lines
Documentation/devicetree/bindings/media/allwinner,sun6i-a31-isp.yaml new file +48 -0
+
+    anyOf:
+      - required:
+          - port@0
+      - required:
+          - port@1
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - clocks
+  - clock-names
+  - resets
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/clock/sun8i-v3s-ccu.h>
+    #include <dt-bindings/reset/sun8i-v3s-ccu.h>
+
+    isp: isp@1cb8000 {
+        compatible = "allwinner,sun8i-v3s-isp";
+        reg = <0x01cb8000 0x1000>;
+        interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>;
+        clocks = <&ccu CLK_BUS_CSI>,
+             <&ccu CLK_CSI1_SCLK>,
+             <&ccu CLK_DRAM_CSI>;
+        clock-names = "bus", "mod", "ram";
+        resets = <&ccu RST_BUS_CSI>;
+
+        ports {
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            port@0 {
+                reg = <0>;
+
+                isp_in_csi0: endpoint {
+                    remote-endpoint = <&csi0_out_isp>;
+                };
+            };
+        };
+    };
+
+...
Signature · 1 line
Sakari Ailus
Signature · 3 lines
Paul Kocialkowski, Bootlin
Embedded Linux and kernel engineering
https://bootlin.com
#4 lore raw

Re: [PATCH v5 1/6] dt-bindings: media: Add Allwinner A31 ISP bindings documentation

Hi Paul,

Quoted 75 lines · Paul Kocialkowski

Hi Sakari,

Quoted 63 lines · Sakari Ailus

Hi Paul,

Quoted 60 lines · Paul Kocialkowski

This introduces YAML bindings documentation for the Allwinner A31 Image Signal Processor (ISP).

---
 .../media/allwinner,sun6i-a31-isp.yaml        | 97 +++++++++++++++++++
 1 file changed, 97 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/media/allwinner,sun6i-a31-isp.yaml
Documentation/devicetree/bindings/media/allwinner,sun6i-a31-isp.yaml new file +44 -0
new file mode 100644
index 000000000000..2fda6e05e16c
@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/media/allwinner,sun6i-a31-isp.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Allwinner A31 Image Signal Processor Driver (ISP) Device Tree Bindings
+
+maintainers:
+  - Paul Kocialkowski <[email protected]>
+
+properties:
+  compatible:
+    enum:
+      - allwinner,sun6i-a31-isp
+      - allwinner,sun8i-v3s-isp
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  clocks:
+    items:
+      - description: Bus Clock
+      - description: Module Clock
+      - description: DRAM Clock
+
+  clock-names:
+    items:
+      - const: bus
+      - const: mod
+      - const: ram
+
+  resets:
+    maxItems: 1
+
+  ports:
+    $ref: /schemas/graph.yaml#/properties/ports
+
+    properties:
+      port@0:
+        $ref: /schemas/graph.yaml#/properties/port
Quoted 8 lines

Documentation/devicetree/bindings/media/allwinner,sun6i-a31-isp.yaml line 49

Quoted 5 lines
Documentation/devicetree/bindings/media/allwinner,sun6i-a31-isp.yaml new file +5 -0
+        description: CSI0 input port
+
+      port@1:
+        $ref: /schemas/graph.yaml#/properties/port
+        description: CSI1 input port

Do both support a single PHY with a single data only? If multiple data lanes are supported, please require data-lanes property (on endpoint).

Quoted 8 lines

There's actually no PHY involved here: the ISP drivers gets its video stream from these CSI controllers which are the ones interfacing with a MIPI CSI-2 receiver (on A31/V3 it uses an external D-PHY, on A83T the D-PHY is collocated with the controller).

So I think the data-lanes property is not relevant here.

What do you think?

Ah, indeed, if it's an internal bus, then data-lanes isn't relevant.

Signature · 1 line
Sakari Ailus
#5 lore raw

PATCH v5 2/6 dt-bindings: media: sun6i-a31-csi: Add ISP output port

In reply to Paul Kocialkowski

Some Allwinner devices come with an Image Signal Processor (ISP) that allows processing camera data to produce good-looking images, especially from raw bayer representations.

The ISP does not have a dedicated capture path: it is fed directly by one of the CSI controllers, which can be selected at run-time.

Represent this possibility as a graph connection between the CSI controller and the ISP in the device-tree bindings.

---
 .../devicetree/bindings/media/allwinner,sun6i-a31-csi.yaml    | 4 ++++
 1 file changed, 4 insertions(+)
Documentation/devicetree/bindings/media/allwinner,sun6i-a31-csi.yaml +4 -0
index 8551c4a711dc..54eb80e517e3 100644
@@ -73,6 +73,10 @@ properties:
         $ref: /schemas/graph.yaml#/properties/port
         description: MIPI CSI-2 bridge input port
 
+      port@2:
+        $ref: /schemas/graph.yaml#/properties/port
+        description: ISP output port
+
     anyOf:
       - required:
           - port@0
Signature · 1 line
2.37.0
maintainer Acked-by on v5 2/6
#6 lore raw

Re: [PATCH v5 2/6] dt-bindings: media: sun6i-a31-csi: Add ISP output port

Quoted 7 lines · Paul Kocialkowski

Some Allwinner devices come with an Image Signal Processor (ISP) that allows processing camera data to produce good-looking images, especially from raw bayer representations.

The ISP does not have a dedicated capture path: it is fed directly by one of the CSI controllers, which can be selected at run-time.

Quoted 8 lines

Represent this possibility as a graph connection between the CSI controller and the ISP in the device-tree bindings.

---
 .../devicetree/bindings/media/allwinner,sun6i-a31-csi.yaml    | 4 ++++
 1 file changed, 4 insertions(+)
#7 lore raw

PATCH v5 3/6 staging: media: Add support for the Allwinner A31 ISP

In reply to Paul Kocialkowski

Some Allwinner platforms come with an Image Signal Processor, which supports various features in order to enhance and transform data received by image sensors into good-looking pictures. In most cases, the data is raw bayer, which gets internally converted to RGB and finally YUV, which is what the hardware produces.

This driver supports ISPs that are similar to the A31 ISP, which was the first standalone ISP found in Allwinner platforms. Simpler ISP blocks were found in the A10 and A20, where they are tied to a CSI controller. Newer generations of Allwinner SoCs (starting with the H6, H616, etc) come with a new camera subsystem and revised ISP. Even though these previous and next-generation ISPs are somewhat similar to the A31 ISP, they have enough significant differences to be out of the scope of this driver.

While the ISP supports many features, including 3A and many enhancement blocks, this implementation is limited to the following:
- V3s (V3/S3) platform support;
- Bayer media bus formats as input;
- Semi-planar YUV (NV12/NV21) as output;
- Debayering with per-component gain and offset configuration;
- 2D noise filtering with configurable coefficients.

Since many features are missing from the associated uAPI, the driver is aimed to integrate staging until all features are properly described.

On the technical side, it uses the v4l2 and media controller APIs, with a video node for capture, a processor subdev and a video node for parameters submission. A specific uAPI structure and associated v4l2 meta format are used to configure parameters of the supported modules.

One particular thing about the hardware is that configuration for module registers needs to be stored in a DMA buffer and gets copied to actual registers by the hardware at the next vsync, when instructed by a flag. This is handled by the "state" mechanism in the driver.

---
 drivers/staging/media/sunxi/Kconfig           |   1 +
 drivers/staging/media/sunxi/Makefile          |   1 +
 drivers/staging/media/sunxi/sun6i-isp/Kconfig |  15 +
 .../staging/media/sunxi/sun6i-isp/Makefile    |   4 +
 .../staging/media/sunxi/sun6i-isp/TODO.txt    |   6 +
 .../staging/media/sunxi/sun6i-isp/sun6i_isp.c | 555 +++++++++++++
 .../staging/media/sunxi/sun6i-isp/sun6i_isp.h |  90 +++
 .../media/sunxi/sun6i-isp/sun6i_isp_capture.c | 742 ++++++++++++++++++
 .../media/sunxi/sun6i-isp/sun6i_isp_capture.h |  78 ++
 .../media/sunxi/sun6i-isp/sun6i_isp_params.c  | 566 +++++++++++++
 .../media/sunxi/sun6i-isp/sun6i_isp_params.h  |  52 ++
 .../media/sunxi/sun6i-isp/sun6i_isp_proc.c    | 578 ++++++++++++++
 .../media/sunxi/sun6i-isp/sun6i_isp_proc.h    |  66 ++
 .../media/sunxi/sun6i-isp/sun6i_isp_reg.h     | 275 +++++++
 .../sunxi/sun6i-isp/uapi/sun6i-isp-config.h   |  43 +
 15 files changed, 3072 insertions(+)
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/Kconfig
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/Makefile
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/TODO.txt
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp.c
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp.h
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_capture.c
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_capture.h
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_params.c
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_params.h
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_proc.c
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_proc.h
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_reg.h
 create mode 100644 drivers/staging/media/sunxi/sun6i-isp/uapi/sun6i-isp-config.h
drivers/staging/media/sunxi/Kconfig +1 -0
index 4549a135741f..62a486aba88b 100644
@@ -12,5 +12,6 @@ config VIDEO_SUNXI
 if VIDEO_SUNXI
 
 source "drivers/staging/media/sunxi/cedrus/Kconfig"
+source "drivers/staging/media/sunxi/sun6i-isp/Kconfig"
 
 endif
drivers/staging/media/sunxi/Makefile +1 -0
index b87140b0e15f..3d20b2f0e644 100644
@@ -1,2 +1,3 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-$(CONFIG_VIDEO_SUNXI_CEDRUS)	+= cedrus/
+obj-$(CONFIG_VIDEO_SUN6I_ISP)		+= sun6i-isp/
drivers/staging/media/sunxi/sun6i-isp/Kconfig new file +15 -0
new file mode 100644
index 000000000000..68dcae9cd7d7
@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config VIDEO_SUN6I_ISP
+	tristate "Allwinner A31 Image Signal Processor (ISP) Driver"
+	depends on V4L_PLATFORM_DRIVERS && VIDEO_DEV
+	depends on ARCH_SUNXI || COMPILE_TEST
+	depends on PM && COMMON_CLK && HAS_DMA
+	select MEDIA_CONTROLLER
+	select VIDEO_V4L2_SUBDEV_API
+	select VIDEOBUF2_DMA_CONTIG
+	select VIDEOBUF2_VMALLOC
+	select V4L2_FWNODE
+	select REGMAP_MMIO
+	help
+	   Support for the Allwinner A31 Image Signal Processor (ISP), also
+	   found on other platforms such as the A80, A83T or V3/V3s.
drivers/staging/media/sunxi/sun6i-isp/Makefile new file +4 -0
new file mode 100644
index 000000000000..da1034785144
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0-only
+sun6i-isp-y += sun6i_isp.o sun6i_isp_proc.o sun6i_isp_capture.o sun6i_isp_params.o
+
+obj-$(CONFIG_VIDEO_SUN6I_ISP) += sun6i-isp.o
drivers/staging/media/sunxi/sun6i-isp/TODO.txt new file +6 -0
new file mode 100644
index 000000000000..1e3236edc1ab
@@ -0,0 +1,6 @@
+Unstaging requirements:
+- Add uAPI support and documentation for the configuration of all the hardware
+  modules and description of the statistics data structures;
+- Add support for statistics reporting;
+- Add userspace support in libcamera which demonstrates the ability to receive
+  statistics and adapt hardware modules configuration accordingly;
drivers/staging/media/sunxi/sun6i-isp/sun6i_isp.c new file +555 -0

555 lines not on this page · raw message

drivers/staging/media/sunxi/sun6i-isp/sun6i_isp.h new file +90 -0

90 lines not on this page · raw message

drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_capture.c new file +742 -0

742 lines not on this page · raw message

drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_capture.h new file +78 -0

78 lines not on this page · raw message

drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_params.c new file +566 -0

566 lines not on this page · raw message

drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_params.h new file +52 -0

52 lines not on this page · raw message

drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_proc.c new file +578 -0

578 lines not on this page · raw message

drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_proc.h new file +66 -0

66 lines not on this page · raw message

drivers/staging/media/sunxi/sun6i-isp/sun6i_isp_reg.h new file +275 -0

239 lines not on this page · raw message

drivers/staging/media/sunxi/sun6i-isp/uapi/sun6i-isp-config.h new file +43 -0

43 lines not on this page · raw message

The rest of this diff is not shown. View the raw message.

Signature · 1 line
2.37.0

Part of the diff is not shown here. View the raw message.

#8 lore raw

PATCH v5 4/6 MAINTAINERS: Add entry for the Allwinner A31 ISP driver

In reply to Paul Kocialkowski

Add myself as maintainer of the Allwinner A31 ISP media driver.

---
 MAINTAINERS | 9 +++++++++
 1 file changed, 9 insertions(+)
MAINTAINERS +9 -0
index 3af1f87caf3a..72e3d6abdf34 100644
@@ -771,6 +771,15 @@ T:	git git://linuxtv.org/media_tree.git
 F:	Documentation/devicetree/bindings/media/allwinner,sun6i-a31-csi.yaml
 F:	drivers/media/platform/sunxi/sun6i-csi/
 
+ALLWINNER A31 ISP DRIVER
+M:	Paul Kocialkowski <[email protected]>
+L:	[email protected]
+S:	Maintained
+T:	git git://linuxtv.org/media_tree.git
+F:	Documentation/devicetree/bindings/media/allwinner,sun6i-a31-isp.yaml
+F:	drivers/staging/media/sunxi/sun6i-isp/
+F:	drivers/staging/media/sunxi/sun6i-isp/uapi/sun6i-isp-config.h
+
 ALLWINNER A31 MIPI CSI-2 BRIDGE DRIVER
 M:	Paul Kocialkowski <[email protected]>
 L:	[email protected]
Signature · 1 line
2.37.0
#9 lore raw

PATCH v5 5/6 media: sun6i-csi: Detect the availability of the ISP

In reply to Paul Kocialkowski

Add a helper to detect whether the ISP is available and connected and store the indication in a driver-wide variable.

---
 .../platform/sunxi/sun6i-csi/sun6i_csi.c      | 33 +++++++++++++++++++
 .../platform/sunxi/sun6i-csi/sun6i_csi.h      |  3 ++
 2 files changed, 36 insertions(+)
drivers/media/platform/sunxi/sun6i-csi/sun6i_csi.c +33 -0
index 00521f966cee..b16166cba2ef 100644
@@ -24,6 +24,35 @@
 #include "sun6i_csi_capture.h"
 #include "sun6i_csi_reg.h"
 
+/* ISP */
+
+static bool sun6i_csi_isp_detect(struct sun6i_csi_device *csi_dev)
+{
+	struct device *dev = csi_dev->dev;
+	struct fwnode_handle *handle = NULL;
+
+	/* ISP is not available if disabled in kernel config. */
+	if (!IS_ENABLED(CONFIG_VIDEO_SUN6I_ISP))
+		return 0;
+
+	/*
+	 * ISP is not available if not connected via fwnode graph.
+	 * This weill also check that the remote parent node is available.
+	 */
+	handle = fwnode_graph_get_endpoint_by_id(dev_fwnode(dev),
+						 SUN6I_CSI_PORT_ISP, 0,
+						 FWNODE_GRAPH_ENDPOINT_NEXT);
+	if (!handle)
+		return 0;
+
+	fwnode_handle_put(handle);
+
+	dev_info(dev, "ISP link is available\n");
+	csi_dev->isp_available = true;
+
+	return 0;
+}
+
 /* Media */
 
 static const struct media_device_ops sun6i_csi_media_ops = {
@@ -290,6 +319,10 @@ static int sun6i_csi_probe(struct platform_device *platform_dev)
 	if (ret)
 		return ret;
 
+	ret = sun6i_csi_isp_detect(csi_dev);
+	if (ret)
+		goto error_resources;
+
 	ret = sun6i_csi_v4l2_setup(csi_dev);
 	if (ret)
 		goto error_resources;
drivers/media/platform/sunxi/sun6i-csi/sun6i_csi.h +3 -0
index e611bdd6e9b2..8e232cd91ebe 100644
@@ -21,6 +21,7 @@
 enum sun6i_csi_port {
 	SUN6I_CSI_PORT_PARALLEL		= 0,
 	SUN6I_CSI_PORT_MIPI_CSI2	= 1,
+	SUN6I_CSI_PORT_ISP		= 2,
 };
 
 struct sun6i_csi_buffer {
@@ -44,6 +45,8 @@ struct sun6i_csi_device {
 	struct clk			*clock_mod;
 	struct clk			*clock_ram;
 	struct reset_control		*reset;
+
+	bool				isp_available;
 };
 
 struct sun6i_csi_variant {
Signature · 1 line
2.37.0
#10 lore raw

PATCH v5 6/6 media: sun6i-csi: Add support for hooking to the isp devices

In reply to Paul Kocialkowski

In order to use the isp and csi together, both devices need to be parented to the same v4l2 and media devices. We use the isp as top-level device and let the csi code hook to its v4l2 and media devices when async subdev registration takes place.

As a result v4l2/media device setup is only called when the ISP is missing and the capture device is registered after the devices are hooked. The bridge subdev and its notifier are registered without any device when the ISP is available. Top-level pointers for the devices are introduced to either redirect to the hooked ones (isp available) or the registered ones (isp missing).

Also keep track of whether the capture node was setup or not to avoid cleaning up resources when it wasn't.

---
 .../platform/sunxi/sun6i-csi/sun6i_csi.c      | 45 +++++++++++++++----
 .../platform/sunxi/sun6i-csi/sun6i_csi.h      |  7 +++
 .../sunxi/sun6i-csi/sun6i_csi_bridge.c        | 32 +++++++++++--
 .../sunxi/sun6i-csi/sun6i_csi_capture.c       | 19 ++++++--
 .../sunxi/sun6i-csi/sun6i_csi_capture.h       |  1 +
 5 files changed, 89 insertions(+), 15 deletions(-)
drivers/media/platform/sunxi/sun6i-csi/sun6i_csi.c +37 -8
index b16166cba2ef..0bac89d8112f 100644
@@ -26,6 +26,18 @@
 
 /* ISP */
 
+int sun6i_csi_isp_complete(struct sun6i_csi_device *csi_dev,
+			   struct v4l2_device *v4l2_dev)
+{
+	if (csi_dev->v4l2_dev && csi_dev->v4l2_dev != v4l2_dev)
+		return -EINVAL;
+
+	csi_dev->v4l2_dev = v4l2_dev;
+	csi_dev->media_dev = v4l2_dev->mdev;
+
+	return sun6i_csi_capture_setup(csi_dev);
+}
+
 static bool sun6i_csi_isp_detect(struct sun6i_csi_device *csi_dev)
 {
 	struct device *dev = csi_dev->dev;
@@ -95,6 +107,9 @@ static int sun6i_csi_v4l2_setup(struct sun6i_csi_device *csi_dev)
 		goto error_media;
 	}
 
+	csi_dev->v4l2_dev = v4l2_dev;
+	csi_dev->media_dev = media_dev;
+
 	return 0;
 
 error_media:
@@ -323,17 +338,27 @@ static int sun6i_csi_probe(struct platform_device *platform_dev)
 	if (ret)
 		goto error_resources;
 
-	ret = sun6i_csi_v4l2_setup(csi_dev);
-	if (ret)
-		goto error_resources;
+	/*
+	 * Register our own v4l2 and media devices when there is no ISP around.
+	 * Otherwise the ISP will use async subdev registration with our bridge,
+	 * which will provide v4l2 and media devices that are used to register
+	 * the video interface.
+	 */
+	if (!csi_dev->isp_available) {
+		ret = sun6i_csi_v4l2_setup(csi_dev);
+		if (ret)
+			goto error_resources;
+	}
 
 	ret = sun6i_csi_bridge_setup(csi_dev);
 	if (ret)
 		goto error_v4l2;
 
-	ret = sun6i_csi_capture_setup(csi_dev);
-	if (ret)
-		goto error_bridge;
+	if (!csi_dev->isp_available) {
+		ret = sun6i_csi_capture_setup(csi_dev);
+		if (ret)
+			goto error_bridge;
+	}
 
 	return 0;
 
@@ -341,7 +366,8 @@ static int sun6i_csi_probe(struct platform_device *platform_dev)
 	sun6i_csi_bridge_cleanup(csi_dev);
 
 error_v4l2:
-	sun6i_csi_v4l2_cleanup(csi_dev);
+	if (!csi_dev->isp_available)
+		sun6i_csi_v4l2_cleanup(csi_dev);
 
 error_resources:
 	sun6i_csi_resources_cleanup(csi_dev);
@@ -355,7 +381,10 @@ static int sun6i_csi_remove(struct platform_device *pdev)
 
 	sun6i_csi_capture_cleanup(csi_dev);
 	sun6i_csi_bridge_cleanup(csi_dev);
-	sun6i_csi_v4l2_cleanup(csi_dev);
+
+	if (!csi_dev->isp_available)
+		sun6i_csi_v4l2_cleanup(csi_dev);
+
 	sun6i_csi_resources_cleanup(csi_dev);
 
 	return 0;
drivers/media/platform/sunxi/sun6i-csi/sun6i_csi.h +7 -0
index 8e232cd91ebe..bc3f0dae35df 100644
@@ -36,6 +36,8 @@ struct sun6i_csi_v4l2 {
 
 struct sun6i_csi_device {
 	struct device			*dev;
+	struct v4l2_device		*v4l2_dev;
+	struct media_device		*media_dev;
 
 	struct sun6i_csi_v4l2		v4l2;
 	struct sun6i_csi_bridge		bridge;
@@ -53,4 +55,9 @@ struct sun6i_csi_variant {
 	unsigned long	clock_mod_rate;
 };
 
+/* ISP */
+
+int sun6i_csi_isp_complete(struct sun6i_csi_device *csi_dev,
+			   struct v4l2_device *v4l2_dev);
+
 #endif
drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_bridge.c +28 -4
index 8e9eded0df28..9624c3f683ee 100644
@@ -654,6 +654,7 @@ sun6i_csi_bridge_notifier_bound(struct v4l2_async_notifier *notifier,
 	struct sun6i_csi_bridge *bridge = &csi_dev->bridge;
 	struct sun6i_csi_bridge_source *source = bridge_async_subdev->source;
 	bool enabled;
+	int ret;
 
 	switch (source->endpoint.base.port) {
 	case SUN6I_CSI_PORT_PARALLEL:
@@ -668,6 +669,16 @@ sun6i_csi_bridge_notifier_bound(struct v4l2_async_notifier *notifier,
 
 	source->subdev = remote_subdev;
 
+	if (csi_dev->isp_available) {
+		/*
+		 * Hook to the first available remote subdev to get v4l2 and
+		 * media devices and register the capture device then.
+		 */
+		ret = sun6i_csi_isp_complete(csi_dev, remote_subdev->v4l2_dev);
+		if (ret)
+			return ret;
+	}
+
 	return sun6i_csi_bridge_link(csi_dev, SUN6I_CSI_BRIDGE_PAD_SINK,
 				     remote_subdev, enabled);
 }
@@ -680,6 +691,9 @@ sun6i_csi_bridge_notifier_complete(struct v4l2_async_notifier *notifier)
 			     bridge.notifier);
 	struct v4l2_device *v4l2_dev = &csi_dev->v4l2.v4l2_dev;
 
+	if (csi_dev->isp_available)
+		return 0;
+
 	return v4l2_device_register_subdev_nodes(v4l2_dev);
 }
 
@@ -753,7 +767,7 @@ int sun6i_csi_bridge_setup(struct sun6i_csi_device *csi_dev)
 {
 	struct device *dev = csi_dev->dev;
 	struct sun6i_csi_bridge *bridge = &csi_dev->bridge;
-	struct v4l2_device *v4l2_dev = &csi_dev->v4l2.v4l2_dev;
+	struct v4l2_device *v4l2_dev = csi_dev->v4l2_dev;
 	struct v4l2_subdev *subdev = &bridge->subdev;
 	struct v4l2_async_notifier *notifier = &bridge->notifier;
 	struct media_pad *pads = bridge->pads;
@@ -794,7 +808,11 @@ int sun6i_csi_bridge_setup(struct sun6i_csi_device *csi_dev)
 
 	/* V4L2 Subdev */
 
-	ret = v4l2_device_register_subdev(v4l2_dev, subdev);
+	if (csi_dev->isp_available)
+		ret = v4l2_async_register_subdev(subdev);
+	else
+		ret = v4l2_device_register_subdev(v4l2_dev, subdev);
+
 	if (ret) {
 		dev_err(dev, "failed to register v4l2 subdev: %d\n", ret);
 		goto error_media_entity;
@@ -811,7 +829,10 @@ int sun6i_csi_bridge_setup(struct sun6i_csi_device *csi_dev)
 	sun6i_csi_bridge_source_setup(csi_dev, &bridge->source_mipi_csi2,
 				      SUN6I_CSI_PORT_MIPI_CSI2, NULL);
 
-	ret = v4l2_async_nf_register(v4l2_dev, notifier);
+	if (csi_dev->isp_available)
+		ret = v4l2_async_subdev_nf_register(subdev, notifier);
+	else
+		ret = v4l2_async_nf_register(v4l2_dev, notifier);
 	if (ret) {
 		dev_err(dev, "failed to register v4l2 async notifier: %d\n",
 			ret);
@@ -823,7 +844,10 @@ int sun6i_csi_bridge_setup(struct sun6i_csi_device *csi_dev)
 error_v4l2_async_notifier:
 	v4l2_async_nf_cleanup(notifier);
 
-	v4l2_device_unregister_subdev(subdev);
+	if (csi_dev->isp_available)
+		v4l2_async_unregister_subdev(subdev);
+	else
+		v4l2_device_unregister_subdev(subdev);
 
 error_media_entity:
 	media_entity_cleanup(&subdev->entity);
drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_capture.c +16 -3
index c9e7526b84c4..69ea1cbaea0c 100644
@@ -570,7 +570,7 @@ static int sun6i_csi_capture_buffer_prepare(struct vb2_buffer *buffer)
 {
 	struct sun6i_csi_device *csi_dev = vb2_get_drv_priv(buffer->vb2_queue);
 	struct sun6i_csi_capture *capture = &csi_dev->capture;
-	struct v4l2_device *v4l2_dev = &csi_dev->v4l2.v4l2_dev;
+	struct v4l2_device *v4l2_dev = csi_dev->v4l2_dev;
 	struct vb2_v4l2_buffer *v4l2_buffer = to_vb2_v4l2_buffer(buffer);
 	unsigned long size = capture->format.fmt.pix.sizeimage;
 
@@ -889,7 +889,7 @@ static int sun6i_csi_capture_link_validate(struct media_link *link)
 	struct video_device *video_dev =
 		media_entity_to_video_device(link->sink->entity);
 	struct sun6i_csi_device *csi_dev = video_get_drvdata(video_dev);
-	struct v4l2_device *v4l2_dev = &csi_dev->v4l2.v4l2_dev;
+	struct v4l2_device *v4l2_dev = csi_dev->v4l2_dev;
 	const struct sun6i_csi_capture_format *capture_format;
 	const struct sun6i_csi_bridge_format *bridge_format;
 	unsigned int capture_width, capture_height;
@@ -971,7 +971,7 @@ int sun6i_csi_capture_setup(struct sun6i_csi_device *csi_dev)
 {
 	struct sun6i_csi_capture *capture = &csi_dev->capture;
 	struct sun6i_csi_capture_state *state = &capture->state;
-	struct v4l2_device *v4l2_dev = &csi_dev->v4l2.v4l2_dev;
+	struct v4l2_device *v4l2_dev = csi_dev->v4l2_dev;
 	struct v4l2_subdev *bridge_subdev = &csi_dev->bridge.subdev;
 	struct video_device *video_dev = &capture->video_dev;
 	struct vb2_queue *queue = &capture->queue;
@@ -980,6 +980,10 @@ int sun6i_csi_capture_setup(struct sun6i_csi_device *csi_dev)
 	struct v4l2_pix_format *pix_format = &format->fmt.pix;
 	int ret;
 
+	/* This may happen with multiple bridge notifier bound calls. */
+	if (state->setup)
+		return 0;
+
 	/* State */
 
 	INIT_LIST_HEAD(&state->queue);
@@ -1055,6 +1059,7 @@ int sun6i_csi_capture_setup(struct sun6i_csi_device *csi_dev)
 	ret = media_create_pad_link(&bridge_subdev->entity,
 				    SUN6I_CSI_BRIDGE_PAD_SOURCE,
 				    &video_dev->entity, 0,
+				    csi_dev->isp_available ? 0 :
 				    MEDIA_LNK_FL_ENABLED |
 				    MEDIA_LNK_FL_IMMUTABLE);
 	if (ret < 0) {
@@ -1065,6 +1070,8 @@ int sun6i_csi_capture_setup(struct sun6i_csi_device *csi_dev)
 		goto error_video_device;
 	}
 
+	state->setup = true;
+
 	return 0;
 
 error_video_device:
@@ -1083,7 +1090,13 @@ void sun6i_csi_capture_cleanup(struct sun6i_csi_device *csi_dev)
 	struct sun6i_csi_capture *capture = &csi_dev->capture;
 	struct video_device *video_dev = &capture->video_dev;
 
+	/* This may happen if async registration failed to complete. */
+	if (!capture->state.setup)
+		return;
+
 	vb2_video_unregister_device(video_dev);
 	media_entity_cleanup(&video_dev->entity);
 	mutex_destroy(&capture->lock);
+
+	capture->state.setup = false;
 }
drivers/media/platform/sunxi/sun6i-csi/sun6i_csi_capture.h +1 -0
index 29893cf96f6b..3ee5ccefbd10 100644
@@ -45,6 +45,7 @@ struct sun6i_csi_capture_state {
 
 	unsigned int			sequence;
 	bool				streaming;
+	bool				setup;
 };
 
 struct sun6i_csi_capture {
Signature · 1 line
2.37.0