Skip to contentSkip to conversation

Intel Wired LAN Driver Updates 2019-12-09

GIT PULL Superseded Jeff Kirsher 8 participants 62 messages Dec 9, 2019 – Jan 2, 2020 Page 1 of 2

Subsystems (8)

v3 of 8 Superseded Latest is v8 →

lore
Share & tools
lore
#0 lore raw

This series contains the initial implementation of the Virtual Bus, virtbus_device, virtbus_driver, updates to 'ice' and 'i40e' to use the new Virtual Bus and the new RDMA driver 'irdma' for use with 'ice' and 'i40e'.

The primary purpose of the Virtual bus is to provide a matching service and to pass the data pointer contained in the virtbus_device to the virtbus_driver during its probe call. This will allow two separate kernel objects to match up and start communication.

The last 16 patches of the series adds a unified Intel Ethernet Protocol driver for RDMA that supports a new network device E810 (iWARP and RoCEv2 capable) and the existing X722 iWARP device. The driver architecture provides the extensibility for future generations of Intel hardware supporting RDMA.

The 'irdma' driver replaces the legacy X722 driver i40iw and extends the ABI already defined for i40iw. It is backward compatible with legacy X722 rdma-core provider (libi40iw).

This series currently builds against net-next tree AND the rdma "for-next" branch.

v1: Initial virtual bus submission
v2: Added example virtbus_dev and virtbus_drv in
tools/testing/sefltests/ to test the virtual bus and provide an example on how to implement
v3: Added ice and i40e driver changes to implement the virtual bus, also
added the new irdma driver which is the RDMA driver which communicates with the ice and i40e drivers

The following are changes since commit 4a63ef710cc3e79ce58b46b122118e415a44b3db:
Merge branch 'for-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/bluetooth/bluetooth-next
and are available in the git repository at:
git://git.kernel.org/pub/scm/linux/kernel/git/jkirsher/next-queue rdma

Dave Ertman (3):
virtual-bus: Implementation of Virtual Bus
ice: Initialize and register a virtual bus to provide RDMA
ice: Implement peer communications

Michael J. Ruhl (1):
RDMA/irdma: Add dynamic tracing for CM

Mustafa Ismail (13):
RDMA/irdma: Add driver framework definitions RDMA/irdma: Implement device initialization definitions RDMA/irdma: Implement HW Admin Queue OPs RDMA/irdma: Add HMC backing store setup functions RDMA/irdma: Add privileged UDA queue implementation RDMA/irdma: Add QoS definitions
RDMA/irdma: Add connection manager
RDMA/irdma: Add PBLE resource manager RDMA/irdma: Implement device supported verb APIs RDMA/irdma: Add RoCEv2 UD OP support RDMA/irdma: Add user/kernel shared libraries RDMA/irdma: Add miscellaneous utility definitions RDMA/irdma: Add ABI definitions

Shiraz Saleem (3):
i40e: Register a virtbus device to provide RDMA
RDMA: Add irdma Kconfig/Makefile and remove i40iw RDMA/irdma: Update MAINTAINERS file

 .../ABI/stable/sysfs-class-infiniband         |   18 -
 Documentation/driver-api/virtual_bus.rst      |   76 +
 MAINTAINERS                                   |    9 +-
 drivers/bus/Kconfig                           |   12 +
 drivers/bus/Makefile                          |    1 +
 drivers/bus/virtual_bus.c                     |  295 +
 drivers/infiniband/Kconfig                    |    2 +-
 drivers/infiniband/hw/Makefile                |    2 +-
 drivers/infiniband/hw/i40iw/Kconfig           |    9 -
 drivers/infiniband/hw/i40iw/Makefile          |   10 -
 drivers/infiniband/hw/i40iw/i40iw.h           |  602 --
 drivers/infiniband/hw/i40iw/i40iw_cm.c        | 4422 ------------
 drivers/infiniband/hw/i40iw/i40iw_cm.h        |  462 --
 drivers/infiniband/hw/i40iw/i40iw_ctrl.c      | 5198 --------------
 drivers/infiniband/hw/i40iw/i40iw_d.h         | 1737 -----
 drivers/infiniband/hw/i40iw/i40iw_hmc.c       |  821 ---
 drivers/infiniband/hw/i40iw/i40iw_hmc.h       |  241 -
 drivers/infiniband/hw/i40iw/i40iw_hw.c        |  852 ---
 drivers/infiniband/hw/i40iw/i40iw_main.c      | 2068 ------
 drivers/infiniband/hw/i40iw/i40iw_osdep.h     |  217 -
 drivers/infiniband/hw/i40iw/i40iw_p.h         |  128 -
 drivers/infiniband/hw/i40iw/i40iw_pble.c      |  612 --
 drivers/infiniband/hw/i40iw/i40iw_pble.h      |  131 -
 drivers/infiniband/hw/i40iw/i40iw_puda.c      | 1493 ----
 drivers/infiniband/hw/i40iw/i40iw_puda.h      |  188 -
 drivers/infiniband/hw/i40iw/i40iw_register.h  | 1030 ---
 drivers/infiniband/hw/i40iw/i40iw_status.h    |  101 -
 drivers/infiniband/hw/i40iw/i40iw_type.h      | 1363 ----
 drivers/infiniband/hw/i40iw/i40iw_uk.c        | 1232 ----
 drivers/infiniband/hw/i40iw/i40iw_user.h      |  430 --
 drivers/infiniband/hw/i40iw/i40iw_utils.c     | 1557 -----
 drivers/infiniband/hw/i40iw/i40iw_verbs.c     | 2788 --------
 drivers/infiniband/hw/i40iw/i40iw_verbs.h     |  179 -
 drivers/infiniband/hw/i40iw/i40iw_vf.c        |   85 -
 drivers/infiniband/hw/i40iw/i40iw_vf.h        |   62 -
 drivers/infiniband/hw/i40iw/i40iw_virtchnl.c  |  756 ---
 drivers/infiniband/hw/i40iw/i40iw_virtchnl.h  |  124 -
 drivers/infiniband/hw/irdma/Kconfig           |   11 +
 drivers/infiniband/hw/irdma/Makefile          |   28 +
 drivers/infiniband/hw/irdma/cm.c              | 4515 +++++++++++++
 drivers/infiniband/hw/irdma/cm.h              |  414 ++
 drivers/infiniband/hw/irdma/ctrl.c            | 5990 +++++++++++++++++
 drivers/infiniband/hw/irdma/defs.h            | 2129 ++++++
 drivers/infiniband/hw/irdma/hmc.c             |  705 ++
 drivers/infiniband/hw/irdma/hmc.h             |  218 +
 drivers/infiniband/hw/irdma/hw.c              | 2596 +++++++
 drivers/infiniband/hw/irdma/i40iw_hw.c        |  209 +
 drivers/infiniband/hw/irdma/i40iw_hw.h        |  162 +
 drivers/infiniband/hw/irdma/i40iw_if.c        |  222 +
 drivers/infiniband/hw/irdma/icrdma_hw.c       |   74 +
 drivers/infiniband/hw/irdma/icrdma_hw.h       |   62 +
 drivers/infiniband/hw/irdma/irdma.h           |  190 +
 drivers/infiniband/hw/irdma/irdma_if.c        |  463 ++
 drivers/infiniband/hw/irdma/main.c            |  630 ++
 drivers/infiniband/hw/irdma/main.h            |  652 ++
 drivers/infiniband/hw/irdma/osdep.h           |  106 +
 drivers/infiniband/hw/irdma/pble.c            |  510 ++
 drivers/infiniband/hw/irdma/pble.h            |  135 +
 drivers/infiniband/hw/irdma/protos.h          |   91 +
 drivers/infiniband/hw/irdma/puda.c            | 1690 +++++
 drivers/infiniband/hw/irdma/puda.h            |  186 +
 drivers/infiniband/hw/irdma/status.h          |   69 +
 drivers/infiniband/hw/irdma/trace.c           |  112 +
 drivers/infiniband/hw/irdma/trace.h           |    3 +
 drivers/infiniband/hw/irdma/trace_cm.h        |  458 ++
 drivers/infiniband/hw/irdma/type.h            | 1709 +++++
 drivers/infiniband/hw/irdma/uda.c             |  390 ++
 drivers/infiniband/hw/irdma/uda.h             |   64 +
 drivers/infiniband/hw/irdma/uda_d.h           |  382 ++
 drivers/infiniband/hw/irdma/uk.c              | 1744 +++++
 drivers/infiniband/hw/irdma/user.h            |  450 ++
 drivers/infiniband/hw/irdma/utils.c           | 2407 +++++++
 drivers/infiniband/hw/irdma/verbs.c           | 4358 ++++++++++++
 drivers/infiniband/hw/irdma/verbs.h           |  199 +
 drivers/infiniband/hw/irdma/ws.c              |  395 ++
 drivers/infiniband/hw/irdma/ws.h              |   39 +
 drivers/net/ethernet/intel/Kconfig            |    2 +
 drivers/net/ethernet/intel/i40e/i40e.h        |    3 +-
 drivers/net/ethernet/intel/i40e/i40e_client.c |  109 +-
 drivers/net/ethernet/intel/ice/Makefile       |    1 +
 drivers/net/ethernet/intel/ice/ice.h          |   15 +
 .../net/ethernet/intel/ice/ice_adminq_cmd.h   |   33 +
 drivers/net/ethernet/intel/ice/ice_common.c   |  203 +
 drivers/net/ethernet/intel/ice/ice_common.h   |    9 +
 drivers/net/ethernet/intel/ice/ice_dcb_lib.c  |   65 +
 drivers/net/ethernet/intel/ice/ice_dcb_lib.h  |    3 +
 .../net/ethernet/intel/ice/ice_hw_autogen.h   |    1 +
 drivers/net/ethernet/intel/ice/ice_idc.c      | 1289 ++++
 drivers/net/ethernet/intel/ice/ice_idc_int.h  |  124 +
 drivers/net/ethernet/intel/ice/ice_lib.c      |   50 +
 drivers/net/ethernet/intel/ice/ice_lib.h      |    3 +
 drivers/net/ethernet/intel/ice/ice_main.c     |  108 +-
 drivers/net/ethernet/intel/ice/ice_sched.c    |   69 +-
 drivers/net/ethernet/intel/ice/ice_switch.c   |   27 +
 drivers/net/ethernet/intel/ice/ice_switch.h   |    4 +
 drivers/net/ethernet/intel/ice/ice_type.h     |    4 +
 .../net/ethernet/intel/ice/ice_virtchnl_pf.c  |   25 -
 include/linux/mod_devicetable.h               |    8 +
 .../linux/net/intel}/i40e_client.h            |   20 +-
 include/linux/net/intel/iidc.h                |  336 +
 include/linux/virtual_bus.h                   |   45 +
 include/uapi/rdma/i40iw-abi.h                 |  107 -
 include/uapi/rdma/ib_user_ioctl_verbs.h       |    1 +
 include/uapi/rdma/irdma-abi.h                 |  131 +
 scripts/mod/devicetable-offsets.c             |    3 +
 scripts/mod/file2alias.c                      |    8 +
 .../virtual_bus/virtual_bus_dev/Makefile      |    7 +
 .../virtual_bus_dev/virtual_bus_dev.c         |   60 +
 .../virtual_bus/virtual_bus_drv/Makefile      |    7 +
 .../virtual_bus_drv/virtual_bus_drv.c         |  115 +
 110 files changed, 37985 insertions(+), 29090 deletions(-)
 create mode 100644 Documentation/driver-api/virtual_bus.rst
 create mode 100644 drivers/bus/virtual_bus.c
 delete mode 100644 drivers/infiniband/hw/i40iw/Kconfig
 delete mode 100644 drivers/infiniband/hw/i40iw/Makefile
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw.h
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_cm.c
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_cm.h
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_ctrl.c
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_d.h
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_hmc.c
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_hmc.h
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_hw.c
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_main.c
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_osdep.h
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_p.h
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_pble.c
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_pble.h
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_puda.c
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_puda.h
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_register.h
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_status.h
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_type.h
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_uk.c
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_user.h
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_utils.c
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_verbs.c
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_verbs.h
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_vf.c
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_vf.h
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_virtchnl.c
 delete mode 100644 drivers/infiniband/hw/i40iw/i40iw_virtchnl.h
 create mode 100644 drivers/infiniband/hw/irdma/Kconfig
 create mode 100644 drivers/infiniband/hw/irdma/Makefile
 create mode 100644 drivers/infiniband/hw/irdma/cm.c
 create mode 100644 drivers/infiniband/hw/irdma/cm.h
 create mode 100644 drivers/infiniband/hw/irdma/ctrl.c
 create mode 100644 drivers/infiniband/hw/irdma/defs.h
 create mode 100644 drivers/infiniband/hw/irdma/hmc.c
 create mode 100644 drivers/infiniband/hw/irdma/hmc.h
 create mode 100644 drivers/infiniband/hw/irdma/hw.c
 create mode 100644 drivers/infiniband/hw/irdma/i40iw_hw.c
 create mode 100644 drivers/infiniband/hw/irdma/i40iw_hw.h
 create mode 100644 drivers/infiniband/hw/irdma/i40iw_if.c
 create mode 100644 drivers/infiniband/hw/irdma/icrdma_hw.c
 create mode 100644 drivers/infiniband/hw/irdma/icrdma_hw.h
 create mode 100644 drivers/infiniband/hw/irdma/irdma.h
 create mode 100644 drivers/infiniband/hw/irdma/irdma_if.c
 create mode 100644 drivers/infiniband/hw/irdma/main.c
 create mode 100644 drivers/infiniband/hw/irdma/main.h
 create mode 100644 drivers/infiniband/hw/irdma/osdep.h
 create mode 100644 drivers/infiniband/hw/irdma/pble.c
 create mode 100644 drivers/infiniband/hw/irdma/pble.h
 create mode 100644 drivers/infiniband/hw/irdma/protos.h
 create mode 100644 drivers/infiniband/hw/irdma/puda.c
 create mode 100644 drivers/infiniband/hw/irdma/puda.h
 create mode 100644 drivers/infiniband/hw/irdma/status.h
 create mode 100644 drivers/infiniband/hw/irdma/trace.c
 create mode 100644 drivers/infiniband/hw/irdma/trace.h
 create mode 100644 drivers/infiniband/hw/irdma/trace_cm.h
 create mode 100644 drivers/infiniband/hw/irdma/type.h
 create mode 100644 drivers/infiniband/hw/irdma/uda.c
 create mode 100644 drivers/infiniband/hw/irdma/uda.h
 create mode 100644 drivers/infiniband/hw/irdma/uda_d.h
 create mode 100644 drivers/infiniband/hw/irdma/uk.c
 create mode 100644 drivers/infiniband/hw/irdma/user.h
 create mode 100644 drivers/infiniband/hw/irdma/utils.c
 create mode 100644 drivers/infiniband/hw/irdma/verbs.c
 create mode 100644 drivers/infiniband/hw/irdma/verbs.h
 create mode 100644 drivers/infiniband/hw/irdma/ws.c
 create mode 100644 drivers/infiniband/hw/irdma/ws.h
 create mode 100644 drivers/net/ethernet/intel/ice/ice_idc.c
 create mode 100644 drivers/net/ethernet/intel/ice/ice_idc_int.h
 rename {drivers/net/ethernet/intel/i40e => include/linux/net/intel}/i40e_client.h (92%)
 create mode 100644 include/linux/net/intel/iidc.h
 create mode 100644 include/linux/virtual_bus.h
 delete mode 100644 include/uapi/rdma/i40iw-abi.h
 create mode 100644 include/uapi/rdma/irdma-abi.h
 create mode 100644 tools/testing/selftests/virtual_bus/virtual_bus_dev/Makefile
 create mode 100644 tools/testing/selftests/virtual_bus/virtual_bus_dev/virtual_bus_dev.c
 create mode 100644 tools/testing/selftests/virtual_bus/virtual_bus_drv/Makefile
 create mode 100644 tools/testing/selftests/virtual_bus/virtual_bus_drv/virtual_bus_drv.c
Signature · 1 line
2.23.0
#1 lore raw

PATCH v3 1/20 virtual-bus: Implementation of Virtual Bus

From: Dave Ertman <[email protected]>

This is the initial implementation of the virtual bus, virtbus_device and virtbus_driver. The virtual bus is a software based bus intended to support registering virtbus_devices and virtbus_drivers and provide matching between them and probing of the registered drivers.

The primary purpose of the virtual bus is to provide matching services to allow the use of a container_of to get access to a piece of desired data. This will allow two separate kernel objects to match up and start communication.

The bus will support probe/remove shutdown and
suspend/resume callbacks.

Kconfig and Makefile alterations are included

---
 Documentation/driver-api/virtual_bus.rst      |  76 +++++
 drivers/bus/Kconfig                           |  12 +
 drivers/bus/Makefile                          |   1 +
 drivers/bus/virtual_bus.c                     | 295 ++++++++++++++++++
 include/linux/mod_devicetable.h               |   8 +
 include/linux/virtual_bus.h                   |  45 +++
 scripts/mod/devicetable-offsets.c             |   3 +
 scripts/mod/file2alias.c                      |   8 +
 .../virtual_bus/virtual_bus_dev/Makefile      |   7 +
 .../virtual_bus_dev/virtual_bus_dev.c         |  60 ++++
 .../virtual_bus/virtual_bus_drv/Makefile      |   7 +
 .../virtual_bus_drv/virtual_bus_drv.c         | 115 +++++++
 12 files changed, 637 insertions(+)
 create mode 100644 Documentation/driver-api/virtual_bus.rst
 create mode 100644 drivers/bus/virtual_bus.c
 create mode 100644 include/linux/virtual_bus.h
 create mode 100644 tools/testing/selftests/virtual_bus/virtual_bus_dev/Makefile
 create mode 100644 tools/testing/selftests/virtual_bus/virtual_bus_dev/virtual_bus_dev.c
 create mode 100644 tools/testing/selftests/virtual_bus/virtual_bus_drv/Makefile
 create mode 100644 tools/testing/selftests/virtual_bus/virtual_bus_drv/virtual_bus_drv.c
Documentation/driver-api/virtual_bus.rst new file +76 -0
new file mode 100644
index 000000000000..db8c34fcafe8
@@ -0,0 +1,76 @@
+===============================
+Virtual Bus Devices and Drivers
+===============================
+
+See <linux/virtual_bus.h> for the models for virtbus_device and virtbus_driver.
+This bus is meant to be a lightweight software based bus to attach generic
+devices and drivers to so that a chunk of data can be passed between them.
+
+One use case example is an rdma driver needing to connect with several
+different types of PCI LAN devices to be able to request resources from
+them (queue sets).  Each LAN driver that supports rdma will register a
+virtbus_device on the virtual bus for each physical function.  The rdma
+driver will register as a virtbus_driver on the virtual bus to be
+matched up with multiple virtbus_devices and receive a pointer to a
+struct containing the callbacks that the PCI LAN drivers support for
+registering with them.
+
+Sections in this document:
+        Virtbus devices
+        Virtbus drivers
+        Device Enumeration
+        Device naming and driver binding
+        Virtual Bus API entry points
+
+Virtbus devices
+~~~~~~~~~~~~~~~
+Virtbus_devices are lightweight objects that support the minimal device
+functionality.  Devices will accept a name, and then an automatically
+generated index is concatenated onto it for the virtbus_device->name.
+
+The virtbus_driver and virtbus_device creators need to both have access
+to a predefined virtbus_device_object struct that will look like the
+following:
+     struct virtbus_object {
+          struct virtbus_device vdev;
+          struct foo_type foo;
+     }
+
+Then when the virtbus_driver's probe is called with the virtbus_device
+as a parameter, it can do a container_of on the virtbus_device to get
+to the struct foo_type foo that it cares about.
+
+Virtbus drivers
+~~~~~~~~~~~~~~~
+Virtbus drivers register with the virtual bus to be matched with virtbus
+devices.  They expect to be registered with a probe and remove callback,
+and also support shutdown, suspend, and resume callbacks.  They otherwise
+follow the standard driver behavior of having discovery and enumeration
+handled in the bus infrastructure.
+
+Virtbus drivers register themselves with the API entry point virtbus_drv_reg
+and unregister with virtbus_drv_unreg.
+
+Device Enumeration
+~~~~~~~~~~~~~~~~~~
+Enumeration is handled automatically by the bus infrastructure via the
+ida_simple methods.
+
+Device naming and driver binding
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The virtbus_device.dev.name is the canonical name for the device. It is
+built from two other parts:
+
+        - virtbus_device.name (also used for matching).
+        - virtbus_device.id (generated automatically from ida_simple calls)
+
+This allows for multiple virtbus_devices with the same name, which will all
+be matched to the same virtbus_driver. Driver binding is performed by the
+driver core, invoking driver probe() after finding a match between device and driver.
+
+Virtual Bus API entry points
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+int virtbus_dev_register(struct virtbus_device *vdev)
+void virtbus_dev_unregister(struct virtbus_device *vdev)
+int virtbus_drv_register(struct virtbus_driver *vdrv, struct module *owner)
+void virtbus_drv_unregister(struct virtbus_driver *vdrv)
drivers/bus/Kconfig +12 -0
index 50200d1c06ea..770519d16d43 100644
@@ -203,4 +203,16 @@ config DA8XX_MSTPRI
 
 source "drivers/bus/fsl-mc/Kconfig"
 
+config VIRTUAL_BUS
+       tristate "lightweight Virtual Bus"
+       depends on PM
+       help
+         Provides a software bus for virtbus_devices to be added to it
+         and virtbus_drivers to be registered on it.  Will create a match
+         between the driver and device, then call the driver's probe with
+         the virtbus_device's struct.
+         One example is the irdma driver needing to connect with various
+         PCI LAN drivers to request resources (queues) to be able to perform
+         its function.
+
 endmenu
drivers/bus/Makefile +1 -0
index 1320bcf9fa9d..6721c77dc71b 100644
@@ -34,3 +34,4 @@ obj-$(CONFIG_UNIPHIER_SYSTEM_BUS)	+= uniphier-system-bus.o
 obj-$(CONFIG_VEXPRESS_CONFIG)	+= vexpress-config.o
 
 obj-$(CONFIG_DA8XX_MSTPRI)	+= da8xx-mstpri.o
+obj-$(CONFIG_VIRTUAL_BUS)	+= virtual_bus.o
drivers/bus/virtual_bus.c new file +295 -0

295 lines not on this page · raw message

include/linux/mod_devicetable.h +8 -0
index 5714fd35a83c..e19214f84c98 100644
@@ -821,4 +821,12 @@ struct wmi_device_id {
 	const void *context;
 };
 
+#define VIRTBUS_NAME_SIZE 20
+#define VIRTBUS_MODULE_PREFIX "virtbus:"
+
+struct virtbus_dev_id {
+	char name[VIRTBUS_NAME_SIZE];
+	kernel_ulong_t driver_data;
+};
+
 #endif /* LINUX_MOD_DEVICETABLE_H */
include/linux/virtual_bus.h new file +45 -0

45 lines not on this page · raw message

scripts/mod/devicetable-offsets.c +3 -0
index 054405b90ba4..9a6099bf90c8 100644
@@ -231,5 +231,8 @@ int main(void)
 	DEVID(wmi_device_id);
 	DEVID_FIELD(wmi_device_id, guid_string);
 
+	DEVID(virtbus_dev_id);
+	DEVID_FIELD(virtbus_dev_id, name);
+
 	return 0;
 }
scripts/mod/file2alias.c +8 -0
index c91eba751804..713fdfe010b0 100644
@@ -1335,6 +1335,13 @@ static int do_wmi_entry(const char *filename, void *symval, char *alias)
 	return 1;
 }
 
+static int do_virtbus_entry(const char *filename, void *symval, char *alias)
+{
+	DEF_FIELD_ADDR(symval, virtbus_dev_id, name);
+	sprintf(alias, VIRTBUS_MODULE_PREFIX "%s", *name);
+	return 1;
+}
+
 /* Does namelen bytes of name exactly match the symbol? */
 static bool sym_is(const char *name, unsigned namelen, const char *symbol)
 {
@@ -1407,6 +1414,7 @@ static const struct devtable devtable[] = {
 	{"typec", SIZE_typec_device_id, do_typec_entry},
 	{"tee", SIZE_tee_client_device_id, do_tee_entry},
 	{"wmi", SIZE_wmi_device_id, do_wmi_entry},
+	{"virtbus", SIZE_virtbus_dev_id, do_virtbus_entry},
 };
 
 /* Create MODULE_ALIAS() statements.
tools/testing/selftests/virtual_bus/virtual_bus_dev/Makefile new file +7 -0
new file mode 100644
index 000000000000..ddd5088eb26b
@@ -0,0 +1,7 @@
+obj-m += virtual_bus_dev.o
+
+all:
+	make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules
+
+clean:
+	make -C /lib/modules/$(shell uname -r)/build M=$(PWD) clean
tools/testing/selftests/virtual_bus/virtual_bus_dev/virtual_bus_dev.c new file +60 -0

60 lines not on this page · raw message

tools/testing/selftests/virtual_bus/virtual_bus_drv/Makefile new file +7 -0
new file mode 100644
index 000000000000..a4b7467f7878
@@ -0,0 +1,7 @@
+obj-m += virtual_bus_drv.o
+
+all:
+	make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules
+
+clean:
+	make -C /lib/modules/$(shell uname -r)/build M=$(PWD) clean
tools/testing/selftests/virtual_bus/virtual_bus_drv/virtual_bus_drv.c new file +115 -0

115 lines not on this page · raw message

Signature · 1 line
2.23.0
#2 lore raw

Re: [PATCH v3 01/20] virtual-bus: Implementation of Virtual Bus

Quoted 178 lines · Jeff Kirsher

From: Dave Ertman <[email protected]>

This is the initial implementation of the virtual bus, virtbus_device and virtbus_driver. The virtual bus is a software based bus intended to support registering virtbus_devices and virtbus_drivers and provide matching between them and probing of the registered drivers.

The primary purpose of the virtual bus is to provide matching services to allow the use of a container_of to get access to a piece of desired data. This will allow two separate kernel objects to match up and start communication.

The bus will support probe/remove shutdown and
suspend/resume callbacks.

Kconfig and Makefile alterations are included

---
 Documentation/driver-api/virtual_bus.rst      |  76 +++++
 drivers/bus/Kconfig                           |  12 +
 drivers/bus/Makefile                          |   1 +
 drivers/bus/virtual_bus.c                     | 295 ++++++++++++++++++
 include/linux/mod_devicetable.h               |   8 +
 include/linux/virtual_bus.h                   |  45 +++
 scripts/mod/devicetable-offsets.c             |   3 +
 scripts/mod/file2alias.c                      |   8 +
 .../virtual_bus/virtual_bus_dev/Makefile      |   7 +
 .../virtual_bus_dev/virtual_bus_dev.c         |  60 ++++
 .../virtual_bus/virtual_bus_drv/Makefile      |   7 +
 .../virtual_bus_drv/virtual_bus_drv.c         | 115 +++++++
 12 files changed, 637 insertions(+)
 create mode 100644 Documentation/driver-api/virtual_bus.rst
 create mode 100644 drivers/bus/virtual_bus.c
 create mode 100644 include/linux/virtual_bus.h
 create mode 100644 tools/testing/selftests/virtual_bus/virtual_bus_dev/Makefile
 create mode 100644 tools/testing/selftests/virtual_bus/virtual_bus_dev/virtual_bus_dev.c
 create mode 100644 tools/testing/selftests/virtual_bus/virtual_bus_drv/Makefile
 create mode 100644 tools/testing/selftests/virtual_bus/virtual_bus_drv/virtual_bus_drv.c
Documentation/driver-api/virtual_bus.rst new file +76 -0
new file mode 100644
index 000000000000..db8c34fcafe8
@@ -0,0 +1,76 @@
+===============================
+Virtual Bus Devices and Drivers
+===============================
+
+See <linux/virtual_bus.h> for the models for virtbus_device and virtbus_driver.
+This bus is meant to be a lightweight software based bus to attach generic
+devices and drivers to so that a chunk of data can be passed between them.
+
+One use case example is an rdma driver needing to connect with several
+different types of PCI LAN devices to be able to request resources from
+them (queue sets).  Each LAN driver that supports rdma will register a
+virtbus_device on the virtual bus for each physical function.  The rdma
+driver will register as a virtbus_driver on the virtual bus to be
+matched up with multiple virtbus_devices and receive a pointer to a
+struct containing the callbacks that the PCI LAN drivers support for
+registering with them.
+
+Sections in this document:
+        Virtbus devices
+        Virtbus drivers
+        Device Enumeration
+        Device naming and driver binding
+        Virtual Bus API entry points
+
+Virtbus devices
+~~~~~~~~~~~~~~~
+Virtbus_devices are lightweight objects that support the minimal device
+functionality.  Devices will accept a name, and then an automatically
+generated index is concatenated onto it for the virtbus_device->name.
+
+The virtbus_driver and virtbus_device creators need to both have access
+to a predefined virtbus_device_object struct that will look like the
+following:
+     struct virtbus_object {
+          struct virtbus_device vdev;
+          struct foo_type foo;
+     }
+
+Then when the virtbus_driver's probe is called with the virtbus_device
+as a parameter, it can do a container_of on the virtbus_device to get
+to the struct foo_type foo that it cares about.
+
+Virtbus drivers
+~~~~~~~~~~~~~~~
+Virtbus drivers register with the virtual bus to be matched with virtbus
+devices.  They expect to be registered with a probe and remove callback,
+and also support shutdown, suspend, and resume callbacks.  They otherwise
+follow the standard driver behavior of having discovery and enumeration
+handled in the bus infrastructure.
+
+Virtbus drivers register themselves with the API entry point virtbus_drv_reg
+and unregister with virtbus_drv_unreg.
+
+Device Enumeration
+~~~~~~~~~~~~~~~~~~
+Enumeration is handled automatically by the bus infrastructure via the
+ida_simple methods.
+
+Device naming and driver binding
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The virtbus_device.dev.name is the canonical name for the device. It is
+built from two other parts:
+
+        - virtbus_device.name (also used for matching).
+        - virtbus_device.id (generated automatically from ida_simple calls)
+
+This allows for multiple virtbus_devices with the same name, which will all
+be matched to the same virtbus_driver. Driver binding is performed by the
+driver core, invoking driver probe() after finding a match between device and driver.
+
+Virtual Bus API entry points
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+int virtbus_dev_register(struct virtbus_device *vdev)
+void virtbus_dev_unregister(struct virtbus_device *vdev)
+int virtbus_drv_register(struct virtbus_driver *vdrv, struct module *owner)
+void virtbus_drv_unregister(struct virtbus_driver *vdrv)
drivers/bus/Kconfig +12 -0
index 50200d1c06ea..770519d16d43 100644
@@ -203,4 +203,16 @@ config DA8XX_MSTPRI
 
 source "drivers/bus/fsl-mc/Kconfig"
 
+config VIRTUAL_BUS
+       tristate "lightweight Virtual Bus"
+       depends on PM
+       help
+         Provides a software bus for virtbus_devices to be added to it
+         and virtbus_drivers to be registered on it.  Will create a match
+         between the driver and device, then call the driver's probe with
+         the virtbus_device's struct.
+         One example is the irdma driver needing to connect with various
+         PCI LAN drivers to request resources (queues) to be able to perform
+         its function.
+
 endmenu
drivers/bus/Makefile +1 -0
index 1320bcf9fa9d..6721c77dc71b 100644
@@ -34,3 +34,4 @@ obj-$(CONFIG_UNIPHIER_SYSTEM_BUS)	+= uniphier-system-bus.o
 obj-$(CONFIG_VEXPRESS_CONFIG)	+= vexpress-config.o
 
 obj-$(CONFIG_DA8XX_MSTPRI)	+= da8xx-mstpri.o
+obj-$(CONFIG_VIRTUAL_BUS)	+= virtual_bus.o
drivers/bus/virtual_bus.c new file +16 -0
new file mode 100644
index 000000000000..6bc986659e4b
@@ -0,0 +1,295 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * virtual_bus.c - lightweight software based bus for virtual devices
+ *
+ * Copyright (c) 2019-20 Intel Corporation
+ *
+ * Please see Documentation/driver-api/virtual_bus.rst for
+ * more information
+ */
+
+#include <linux/string.h>
+#include <linux/virtual_bus.h>
+#include <linux/of_irq.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/pm_runtime.h>

drivers/bus/virtual_bus.c line 26

Quoted 10 lines
drivers/bus/virtual_bus.c new file +10 -0
+#include <linux/pm_domain.h>
+#include <linux/acpi.h>
+#include <linux/device.h>
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Lightweight Virtual Bus");
+MODULE_AUTHOR("David Ertman <[email protected]>");
+MODULE_AUTHOR("Kiran Patil <[email protected]>");
+
+static DEFINE_IDA(virtbus_dev_ida);

I was under impression that usage of IDA interface is discouraged in favor of direct calls to XArray.

Thanks

#3 lore raw

Re: [PATCH v3 01/20] virtual-bus: Implementation of Virtual Bus

Quoted 9 lines · Leon Romanovsky

drivers/bus/virtual_bus.c line 26

Quoted 6 lines
drivers/bus/virtual_bus.c new file +6 -0
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Lightweight Virtual Bus");
+MODULE_AUTHOR("David Ertman <[email protected]>");
+MODULE_AUTHOR("Kiran Patil <[email protected]>");
+
+static DEFINE_IDA(virtbus_dev_ida);

I was under impression that usage of IDA interface is discouraged in favor of direct calls to XArray.

IDA is OK, idr should be xarray

Jason

#4 lore raw

Re: [PATCH v3 01/20] virtual-bus: Implementation of Virtual Bus

In reply to Jeff Kirsher
Quoted 8 lines · Jeff Kirsher

From: Dave Ertman <[email protected]>

This is the initial implementation of the virtual bus, virtbus_device and virtbus_driver. The virtual bus is a software based bus intended to support registering virtbus_devices and virtbus_drivers and provide matching between them and probing of the registered drivers.

Quoted 5 lines

The primary purpose of the virtual bus is to provide matching services to allow the use of a container_of to get access to a piece of desired data. This will allow two separate kernel objects to match up and start communication.

What? That's not the job of a virtual bus, a virtual bus is there to put devices on it and hook them up to drivers.

What do you mean by "two separate kernel objects"? What are the objects here?

Quoted 68 lines

The bus will support probe/remove shutdown and
suspend/resume callbacks.

Kconfig and Makefile alterations are included

---
 Documentation/driver-api/virtual_bus.rst      |  76 +++++
 drivers/bus/Kconfig                           |  12 +
 drivers/bus/Makefile                          |   1 +
 drivers/bus/virtual_bus.c                     | 295 ++++++++++++++++++
 include/linux/mod_devicetable.h               |   8 +
 include/linux/virtual_bus.h                   |  45 +++
 scripts/mod/devicetable-offsets.c             |   3 +
 scripts/mod/file2alias.c                      |   8 +
 .../virtual_bus/virtual_bus_dev/Makefile      |   7 +
 .../virtual_bus_dev/virtual_bus_dev.c         |  60 ++++
 .../virtual_bus/virtual_bus_drv/Makefile      |   7 +
 .../virtual_bus_drv/virtual_bus_drv.c         | 115 +++++++
 12 files changed, 637 insertions(+)
 create mode 100644 Documentation/driver-api/virtual_bus.rst
 create mode 100644 drivers/bus/virtual_bus.c
 create mode 100644 include/linux/virtual_bus.h
 create mode 100644 tools/testing/selftests/virtual_bus/virtual_bus_dev/Makefile
 create mode 100644 tools/testing/selftests/virtual_bus/virtual_bus_dev/virtual_bus_dev.c
 create mode 100644 tools/testing/selftests/virtual_bus/virtual_bus_drv/Makefile
 create mode 100644 tools/testing/selftests/virtual_bus/virtual_bus_drv/virtual_bus_drv.c
Documentation/driver-api/virtual_bus.rst new file +31 -0
new file mode 100644
index 000000000000..db8c34fcafe8
@@ -0,0 +1,76 @@
+===============================
+Virtual Bus Devices and Drivers
+===============================
+
+See <linux/virtual_bus.h> for the models for virtbus_device and virtbus_driver.
+This bus is meant to be a lightweight software based bus to attach generic
+devices and drivers to so that a chunk of data can be passed between them.
+
+One use case example is an rdma driver needing to connect with several
+different types of PCI LAN devices to be able to request resources from
+them (queue sets).  Each LAN driver that supports rdma will register a
+virtbus_device on the virtual bus for each physical function.  The rdma
+driver will register as a virtbus_driver on the virtual bus to be
+matched up with multiple virtbus_devices and receive a pointer to a
+struct containing the callbacks that the PCI LAN drivers support for
+registering with them.
+
+Sections in this document:
+        Virtbus devices
+        Virtbus drivers
+        Device Enumeration
+        Device naming and driver binding
+        Virtual Bus API entry points
+
+Virtbus devices
+~~~~~~~~~~~~~~~
+Virtbus_devices are lightweight objects that support the minimal device
+functionality.  Devices will accept a name, and then an automatically
+generated index is concatenated onto it for the virtbus_device->name.
+
+The virtbus_driver and virtbus_device creators need to both have access

Documentation/driver-api/virtual_bus.rst line 41

Quoted 10 lines
Documentation/driver-api/virtual_bus.rst new file +10 -0
+to a predefined virtbus_device_object struct that will look like the
+following:
+     struct virtbus_object {
+          struct virtbus_device vdev;
+          struct foo_type foo;
+     }
+
+Then when the virtbus_driver's probe is called with the virtbus_device
+as a parameter, it can do a container_of on the virtbus_device to get
+to the struct foo_type foo that it cares about.

That's just how the driver model works, I'm not quite sure what you are trying to say here. What is this section for?

Quoted 24 lines
Documentation/driver-api/virtual_bus.rst new file +24 -0
+Virtbus drivers
+~~~~~~~~~~~~~~~
+Virtbus drivers register with the virtual bus to be matched with virtbus
+devices.  They expect to be registered with a probe and remove callback,
+and also support shutdown, suspend, and resume callbacks.  They otherwise
+follow the standard driver behavior of having discovery and enumeration
+handled in the bus infrastructure.
+
+Virtbus drivers register themselves with the API entry point virtbus_drv_reg
+and unregister with virtbus_drv_unreg.
+
+Device Enumeration
+~~~~~~~~~~~~~~~~~~
+Enumeration is handled automatically by the bus infrastructure via the
+ida_simple methods.
+
+Device naming and driver binding
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The virtbus_device.dev.name is the canonical name for the device. It is
+built from two other parts:
+
+        - virtbus_device.name (also used for matching).
+        - virtbus_device.id (generated automatically from ida_simple calls)
+

Documentation/driver-api/virtual_bus.rst line 76

Quoted 10 lines
Documentation/driver-api/virtual_bus.rst new file +10 -0
+This allows for multiple virtbus_devices with the same name, which will all
+be matched to the same virtbus_driver. Driver binding is performed by the
+driver core, invoking driver probe() after finding a match between device and driver.
+
+Virtual Bus API entry points
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+int virtbus_dev_register(struct virtbus_device *vdev)
+void virtbus_dev_unregister(struct virtbus_device *vdev)
+int virtbus_drv_register(struct virtbus_driver *vdrv, struct module *owner)
+void virtbus_drv_unregister(struct virtbus_driver *vdrv)

I don't understand this documentation at all. Can't you put what you need in the code itself and have it be auto-generated? This is really vague stuff that I can't find useful. And if I don't understand it, well... :)

drivers/bus/Kconfig config DA8XX_MSTPRI · line 207

Quoted 10 lines
drivers/bus/Kconfig +2 -0
index 50200d1c06ea..770519d16d43 100644
@@ -203,4 +203,16 @@ config DA8XX_MSTPRI
 
 source "drivers/bus/fsl-mc/Kconfig"
 
+config VIRTUAL_BUS
+       tristate "lightweight Virtual Bus"

Why "lightweight"? Where's the heavy one? :)

drivers/bus/Kconfig config DA8XX_MSTPRI · line 208

Quoted 1 line
drivers/bus/Kconfig +1 -0
+       depends on PM

Why does it depend on PM?

Quoted 70 lines
drivers/bus/Kconfig +9 -0
+       help
+         Provides a software bus for virtbus_devices to be added to it
+         and virtbus_drivers to be registered on it.  Will create a match
+         between the driver and device, then call the driver's probe with
+         the virtbus_device's struct.
+         One example is the irdma driver needing to connect with various
+         PCI LAN drivers to request resources (queues) to be able to perform
+         its function.
+
 endmenu
drivers/bus/Makefile +1 -0
index 1320bcf9fa9d..6721c77dc71b 100644
@@ -34,3 +34,4 @@ obj-$(CONFIG_UNIPHIER_SYSTEM_BUS)	+= uniphier-system-bus.o
 obj-$(CONFIG_VEXPRESS_CONFIG)	+= vexpress-config.o
 
 obj-$(CONFIG_DA8XX_MSTPRI)	+= da8xx-mstpri.o
+obj-$(CONFIG_VIRTUAL_BUS)	+= virtual_bus.o
drivers/bus/virtual_bus.c new file +45 -0
new file mode 100644
index 000000000000..6bc986659e4b
@@ -0,0 +1,295 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * virtual_bus.c - lightweight software based bus for virtual devices
+ *
+ * Copyright (c) 2019-20 Intel Corporation
+ *
+ * Please see Documentation/driver-api/virtual_bus.rst for
+ * more information
+ */
+
+#include <linux/string.h>
+#include <linux/virtual_bus.h>
+#include <linux/of_irq.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/pm_runtime.h>
+#include <linux/pm_domain.h>
+#include <linux/acpi.h>
+#include <linux/device.h>
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Lightweight Virtual Bus");
+MODULE_AUTHOR("David Ertman <[email protected]>");
+MODULE_AUTHOR("Kiran Patil <[email protected]>");
+
+static DEFINE_IDA(virtbus_dev_ida);
+
+static const
+struct virtbus_dev_id *virtbus_match_id(const struct virtbus_dev_id *id,
+					struct virtbus_device *vdev)
+{
+	while (id->name[0]) {
+		if (!strcmp(vdev->name, id->name)) {
+			vdev->matched_element = id;
+			return id;
+		}
+		id++;
+	}
+	return NULL;
+}
+
+#define to_virtbus_dev(x)	(container_of((x), struct virtbus_device, dev))
+#define to_virtbus_drv(x)	(container_of((x), struct virtbus_driver, \
+				 driver))
+

drivers/bus/virtual_bus.c line 55

Quoted 10 lines
drivers/bus/virtual_bus.c new file +10 -0
+/**
+ * virtbus_match - bind virtbus device to virtbus driver
+ * @dev: device
+ * @drv: driver
+ *
+ * Virtbus device IDs are always in "<name>.<instance>" format.
+ * Instances are automatically selected through an ida_simple_get so
+ * are positive integers. Names are taken from the device name field.
+ * Driver IDs are simple <name>.  Need to extract the name from the
+ * Virtual Device compare to name of the driver.

Why kerneldoc for static functions?

This naming scheme is great to have documented somewhere, why am I just finding it here? See, your .rst file was useless :(

Make the documentation be generated from this .c file please.

drivers/bus/virtual_bus.c line 65

Quoted 10 lines
drivers/bus/virtual_bus.c new file +10 -0
+ */
+static int virtbus_match(struct device *dev, struct device_driver *drv)
+{
+	struct virtbus_driver *vdrv = to_virtbus_drv(drv);
+	struct virtbus_device *vdev = to_virtbus_dev(dev);
+
+	if (vdrv->id_table)
+		return virtbus_match_id(vdrv->id_table, vdev) != NULL;
+
+	return !strcmp(vdev->name, drv->name);

Wait, why would a driver not have an id table? You are matching two different ways here, only one documented above?

drivers/bus/virtual_bus.c line 75

Quoted 10 lines
drivers/bus/virtual_bus.c new file +10 -0
+}
+
+/**
+ * virtbus_probe - call probe of the virtbus_drv
+ * @dev: device struct
+ */
+static int virtbus_probe(struct device *dev)
+{
+	if (dev->driver->probe)
+		return dev->driver->probe(dev);

How can we not have a probe? Test for that at registration.

drivers/bus/virtual_bus.c line 83

Quoted 8 lines
drivers/bus/virtual_bus.c new file +8 -0
+
+	return 0;
+}
+
+static int virtbus_remove(struct device *dev)
+{
+	if (dev->driver->remove)
+		return dev->driver->remove(dev);

Same as above.

drivers/bus/virtual_bus.c line 91

Quoted 8 lines
drivers/bus/virtual_bus.c new file +8 -0
+
+	return 0;
+}
+
+static void virtbus_shutdown(struct device *dev)
+{
+	if (dev->driver->shutdown)
+		dev->driver->shutdown(dev);

Same as above

Quoted 33 lines
drivers/bus/virtual_bus.c new file +33 -0
+}
+
+static int virtbus_suspend(struct device *dev, pm_message_t state)
+{
+	if (dev->driver->suspend)
+		return dev->driver->suspend(dev, state);
+
+	return 0;
+}
+
+static int virtbus_resume(struct device *dev)
+{
+	if (dev->driver->resume)
+		return dev->driver->resume(dev);
+
+	return 0;
+}
+
+struct bus_type virtual_bus_type = {
+	.name = "virtbus",
+	.match = virtbus_match,
+	.probe = virtbus_probe,
+	.remove = virtbus_remove,
+	.shutdown = virtbus_shutdown,
+	.suspend = virtbus_suspend,
+	.resume = virtbus_resume,
+};
+
+/**
+ * virtbus_dev_register - add a virtual bus device
+ * @vdev: virtual bus device to add
+ */
+int virtbus_dev_register(struct virtbus_device *vdev)

drivers/bus/virtual_bus.c line 134

Quoted 10 lines
drivers/bus/virtual_bus.c new file +10 -0
+{
+	int ret;
+
+	device_initialize(&vdev->dev);
+
+	vdev->dev.bus = &virtual_bus_type;
+	/* All device IDs are automatically allocated */
+	ret = ida_simple_get(&virtbus_dev_ida, 0, 0, GFP_KERNEL);
+	if (ret < 0)
+		return ret;

Ugh, you just lost the reference count logic for your call to device_initialize() above, right? Or did you document the heck out of this saying that if you call virtbus_dev_register, you MUST call put_device() to properly clean things up.

Nope, you did not :(

Quoted 5 lines
drivers/bus/virtual_bus.c new file +5 -0
+
+	vdev->id = ret;
+	dev_set_name(&vdev->dev, "%s.%d", vdev->name, vdev->id);
+
+	dev_dbg(&vdev->dev, "Registering virtbus device '%s'\n",

drivers/bus/virtual_bus.c line 149

Quoted 10 lines
drivers/bus/virtual_bus.c new file +10 -0
+		dev_name(&vdev->dev));
+
+	ret = device_add(&vdev->dev);
+	if (ret)
+		goto device_add_err;
+
+	return 0;
+
+device_add_err:
+	/* Error adding virtual device */

We know that, document things that are not obvious :)

Quoted 10 lines
drivers/bus/virtual_bus.c new file +10 -0
+	put_device(&vdev->dev);
+	ida_simple_remove(&virtbus_dev_ida, vdev->id);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(virtbus_dev_register);
+
+/**
+ * virtbus_dev_unregister - remove a virtual bus device
+ * vdev: virtual bus device we are removing

drivers/bus/virtual_bus.c line 169

Quoted 10 lines
drivers/bus/virtual_bus.c new file +10 -0
+ */
+void virtbus_dev_unregister(struct virtbus_device *vdev)
+{
+	put_device(&vdev->dev);
+}
+EXPORT_SYMBOL_GPL(virtbus_dev_unregister);
+
+struct virtbus_object {
+	struct virtbus_device vdev;
+	char name[];

Why an empty array? are you sure?

Quoted 158 lines
drivers/bus/virtual_bus.c new file +126 -0
+};
+
+/**
+ * virtbus_dev_release - Destroy a virtbus device
+ * @vdev: virtual device to release
+ *
+ * Note that the vdev->data which is separately allocated needs to be
+ * separately freed on it own.
+ */
+static void virtbus_dev_release(struct device *dev)
+{
+	struct virtbus_object *vo = container_of(dev, struct virtbus_object,
+						 vdev.dev);
+
+	ida_simple_remove(&virtbus_dev_ida, vo->vdev.id);
+	kfree(vo);
+}
+
+static int virtbus_drv_probe(struct device *_dev)
+{
+	struct virtbus_driver *vdrv = to_virtbus_drv(_dev->driver);
+	struct virtbus_device *vdev = to_virtbus_dev(_dev);
+	int ret;
+
+	ret = dev_pm_domain_attach(_dev, true);
+	if (ret) {
+		dev_warn(_dev, "Failed to attatch to PM Domain : %d\n", ret);
+		return ret;
+	}
+
+	if (vdrv->probe) {
+		ret = vdrv->probe(vdev);
+		if (ret)
+			dev_pm_domain_detach(_dev, true);
+	}
+
+	return ret;
+}
+
+static int virtbus_drv_remove(struct device *_dev)
+{
+	struct virtbus_driver *vdrv = to_virtbus_drv(_dev->driver);
+	struct virtbus_device *vdev = to_virtbus_dev(_dev);
+	int ret = 0;
+
+	if (vdrv->remove)
+		ret = vdrv->remove(vdev);
+
+	dev_pm_domain_detach(_dev, true);
+
+	return ret;
+}
+
+static void virtbus_drv_shutdown(struct device *_dev)
+{
+	struct virtbus_driver *vdrv = to_virtbus_drv(_dev->driver);
+	struct virtbus_device *vdev = to_virtbus_dev(_dev);
+
+	if (vdrv->shutdown)
+		vdrv->shutdown(vdev);
+}
+
+static int virtbus_drv_suspend(struct device *_dev, pm_message_t state)
+{
+	struct virtbus_driver *vdrv = to_virtbus_drv(_dev->driver);
+	struct virtbus_device *vdev = to_virtbus_dev(_dev);
+
+	if (vdrv->suspend)
+		return vdrv->suspend(vdev, state);
+
+	return 0;
+}
+
+static int virtbus_drv_resume(struct device *_dev)
+{
+	struct virtbus_driver *vdrv = to_virtbus_drv(_dev->driver);
+	struct virtbus_device *vdev = to_virtbus_dev(_dev);
+
+	if (vdrv->resume)
+		return vdrv->resume(vdev);
+
+	return 0;
+}
+
+/**
+ * __virtbus_drv_register - register a driver for virtual bus devices
+ * @vdrv: virtbus_driver structure
+ * @owner: owning module/driver
+ */
+int __virtbus_drv_register(struct virtbus_driver *vdrv, struct module *owner)
+{
+	vdrv->driver.owner = owner;
+	vdrv->driver.bus = &virtual_bus_type;
+	vdrv->driver.probe = virtbus_drv_probe;
+	vdrv->driver.remove = virtbus_drv_remove;
+	vdrv->driver.shutdown = virtbus_drv_shutdown;
+	vdrv->driver.suspend = virtbus_drv_suspend;
+	vdrv->driver.resume = virtbus_drv_resume;
+
+	return driver_register(&vdrv->driver);
+}
+EXPORT_SYMBOL_GPL(__virtbus_drv_register);
+
+/**
+ * virtbus_drv_unregister - unregister a driver for virtual bus devices
+ * @drv: virtbus_driver structure
+ */
+void virtbus_drv_unregister(struct virtbus_driver *vdrv)
+{
+	driver_unregister(&vdrv->driver);
+}
+EXPORT_SYMBOL_GPL(virtbus_drv_unregister);
+
+static int __init virtual_bus_init(void)
+{
+	return bus_register(&virtual_bus_type);
+}
+
+static void __exit virtual_bus_exit(void)
+{
+	bus_unregister(&virtual_bus_type);
+	ida_destroy(&virtbus_dev_ida);
+}
+
+module_init(virtual_bus_init);
+module_exit(virtual_bus_exit);
include/linux/mod_devicetable.h +8 -0
index 5714fd35a83c..e19214f84c98 100644
@@ -821,4 +821,12 @@ struct wmi_device_id {
 	const void *context;
 };
 
+#define VIRTBUS_NAME_SIZE 20
+#define VIRTBUS_MODULE_PREFIX "virtbus:"
+
+struct virtbus_dev_id {
+	char name[VIRTBUS_NAME_SIZE];
+	kernel_ulong_t driver_data;
+};
+
 #endif /* LINUX_MOD_DEVICETABLE_H */
include/linux/virtual_bus.h new file +9 -0
new file mode 100644
index 000000000000..fe4725b1e6b1
@@ -0,0 +1,45 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * virtual_bus.h - lightweight software bus
+ *
+ * Copyright (c) 2019-20 Intel Corporation
+ *
+ * Please see Documentation/driver-api/virtual_bus.rst for more information
+ */
+

include/linux/virtual_bus.h line 19

Quoted 10 lines
include/linux/virtual_bus.h new file +10 -0
+#ifndef _VIRTUAL_BUS_H_
+#define _VIRTUAL_BUS_H_
+
+#include <linux/device.h>
+
+struct virtbus_device {
+	const char *name;
+	int id;
+	const struct virtbus_dev_id *matched_element;
+	struct device dev;

Put dev at the front might align things a bit better in memory. Not a big issue...

include/linux/virtual_bus.h line 25

Quoted 6 lines
include/linux/virtual_bus.h new file +6 -0
+};
+
+/* If the driver uses a id_table to match with virtbus_devices, then the
+ * memory for the table is expected to remain allocated for the duration
+ * of the pairing between driver and device.  The pointer for the matching
+ * element will be copied to the dev_id field of the virtbus_device.

Again, id_table is optional? Huh????

Quoted 144 lines
include/linux/virtual_bus.h new file +20 -0
+ */
+struct virtbus_driver {
+	int (*probe)(struct virtbus_device *);
+	int (*remove)(struct virtbus_device *);
+	void (*shutdown)(struct virtbus_device *);
+	int (*suspend)(struct virtbus_device *, pm_message_t);
+	int (*resume)(struct virtbus_device *);
+	struct device_driver driver;
+	const struct virtbus_dev_id *id_table;
+};
+
+int virtbus_dev_register(struct virtbus_device *vdev);
+void virtbus_dev_unregister(struct virtbus_device *vdev);
+int __virtbus_drv_register(struct virtbus_driver *vdrv, struct module *owner);
+void virtbus_drv_unregister(struct virtbus_driver *vdrv);
+
+#define virtbus_drv_register(vdrv) \
+	__virtbus_drv_register(vdrv, THIS_MODULE)
+
+#endif /* _VIRTUAL_BUS_H_ */
scripts/mod/devicetable-offsets.c +3 -0
index 054405b90ba4..9a6099bf90c8 100644
@@ -231,5 +231,8 @@ int main(void)
 	DEVID(wmi_device_id);
 	DEVID_FIELD(wmi_device_id, guid_string);
 
+	DEVID(virtbus_dev_id);
+	DEVID_FIELD(virtbus_dev_id, name);
+
 	return 0;
 }
scripts/mod/file2alias.c +8 -0
index c91eba751804..713fdfe010b0 100644
@@ -1335,6 +1335,13 @@ static int do_wmi_entry(const char *filename, void *symval, char *alias)
 	return 1;
 }
 
+static int do_virtbus_entry(const char *filename, void *symval, char *alias)
+{
+	DEF_FIELD_ADDR(symval, virtbus_dev_id, name);
+	sprintf(alias, VIRTBUS_MODULE_PREFIX "%s", *name);
+	return 1;
+}
+
 /* Does namelen bytes of name exactly match the symbol? */
 static bool sym_is(const char *name, unsigned namelen, const char *symbol)
 {
@@ -1407,6 +1414,7 @@ static const struct devtable devtable[] = {
 	{"typec", SIZE_typec_device_id, do_typec_entry},
 	{"tee", SIZE_tee_client_device_id, do_tee_entry},
 	{"wmi", SIZE_wmi_device_id, do_wmi_entry},
+	{"virtbus", SIZE_virtbus_dev_id, do_virtbus_entry},
 };
 
 /* Create MODULE_ALIAS() statements.
tools/testing/selftests/virtual_bus/virtual_bus_dev/Makefile new file +7 -0
new file mode 100644
index 000000000000..ddd5088eb26b
@@ -0,0 +1,7 @@
+obj-m += virtual_bus_dev.o
+
+all:
+	make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules
+
+clean:
+	make -C /lib/modules/$(shell uname -r)/build M=$(PWD) clean
tools/testing/selftests/virtual_bus/virtual_bus_dev/virtual_bus_dev.c new file +60 -0
new file mode 100644
index 000000000000..966e882452d2
@@ -0,0 +1,60 @@
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/module.h>
+#include <linux/virtual_bus.h>
+#include <linux/string.h>
+#include <linux/slab.h>
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Dave Ertman");
+MODULE_DESCRIPTION("Test to create a device on virtual bus");
+MODULE_VERSION("1.0");
+
+struct virtbus_data {
+	int numb;
+};
+
+struct virtbus_object {
+	struct virtbus_device vdev;
+	struct virtbus_data vd;
+};
+
+static struct virtbus_object vo = {
+	.vdev = {
+		.name = "virtual_bus_dev",
+		},
+	.vd = {
+		.numb = 789,
+	},
+};
+
+static int __init test_dev_init(void)
+{
+	int ret = 0;
+
+	printk(KERN_INFO "Loading Virtual Bus Test Device\n");
+
+	printk(KERN_ERR "Virtbus Device values:\n\t%s\n\t%d\n", vo.vdev.name,
+	       vo.vd.numb);
+
+	ret = virtbus_dev_register(&vo.vdev);
+	if (ret) {
+		printk(KERN_ERR "FAILED TO ADD VIRTBUS DEVICE %d\n", ret);
+		return ret;
+	}
+
+	printk(KERN_INFO "Virtual Device created\n");
+	return ret;
+}
+
+static void __exit test_dev_exit(void)
+{
+	printk(KERN_INFO "Exiting Virtual Bus Test Device");
+
+	virtbus_dev_unregister(&vo.vdev);
+
+	printk(KERN_INFO "Virtual Bus Test Device removed\n");
+}
+
+module_init(test_dev_init);
+module_exit(test_dev_exit);
tools/testing/selftests/virtual_bus/virtual_bus_drv/Makefile new file +0 -0
new file mode 100644
index 000000000000..a4b7467f7878

tools/testing/selftests/virtual_bus/virtual_bus_drv/Makefile line 7

Quoted 7 lines
tools/testing/selftests/virtual_bus/virtual_bus_drv/Makefile new file +7 -0
@@ -0,0 +1,7 @@
+obj-m += virtual_bus_drv.o
+
+all:
+	make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules
+
+clean:
+	make -C /lib/modules/$(shell uname -r)/build M=$(PWD) clean

Add all the test stuff as a follow-on patch, not all in one big mess here.

Quoted 8 lines
tools/testing/selftests/virtual_bus/virtual_bus_drv/virtual_bus_drv.c new file +2 -0
new file mode 100644
index 000000000000..13e2e3d686ab
@@ -0,0 +1,115 @@
+#include <linux/init.h>
+#include <linux/module.h>

tools/testing/selftests/virtual_bus/virtual_bus_drv/virtual_bus_drv.c line 12

Quoted 10 lines
tools/testing/selftests/virtual_bus/virtual_bus_drv/virtual_bus_drv.c new file +10 -0
+#include <linux/module.h>
+#include <linux/virtual_bus.h>
+#include <linux/string.h>
+#include <linux/slab.h>
+#include <linux/mod_devicetable.h>
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Dave Ertman");
+MODULE_DESCRIPTION("Test to register a driver on virtual bus");
+MODULE_VERSION("1.0");

MODULE_VERSION? What is this, the 1990's? :(

Quoted 93 lines
tools/testing/selftests/virtual_bus/virtual_bus_drv/virtual_bus_drv.c new file +93 -0
+
+struct virtbus_data {
+	int numb;
+};
+
+struct virtbus_object {
+	struct virtbus_device vdev;
+	struct virtbus_data vd;
+};
+
+static int td_probe(struct virtbus_device *vdev)
+{
+	struct virtbus_object *vo;
+
+	printk(KERN_ERR "VIRTBUS DRIVER PROBED\n");
+
+	vo = container_of(vdev, struct virtbus_object, vdev);
+
+	if (vdev->matched_element) {
+		printk(KERN_ERR "DEV_ID->DATA 0x%08x\n",
+			(uint)vdev->matched_element->driver_data);
+		if (vdev->matched_element->driver_data == 1)
+			printk(KERN_ERR "NUMB: %d\n",
+			       vo->vd.numb);
+	} else {
+		printk(KERN_ERR "MATCHED_ELEMENT->DATA is NULL\n");
+	}
+
+	return 0;
+}
+
+static int td_remove(struct virtbus_device *vdev)
+{
+	printk(KERN_ERR "VIRTBUS DRIVER REMOVED\n");
+	return 0;
+}
+
+static void td_shutdown(struct virtbus_device *vdev)
+{
+	printk(KERN_ERR "VIRTBUS DRIVER SHUTDOWN\n");
+}
+
+static const struct virtbus_dev_id vdev_id_table[] = {
+
+	{
+		.name = "NOT THE NAME",
+		.driver_data = 0x00000000,
+	},
+	{
+		.name = "virtual_bus_dev",
+		.driver_data = 0x00000001,
+	},
+	{
+		.name = "ice_rdma",
+		.driver_data = 0x00000002,
+	},
+	{
+		.name = "YET AGAIN NOT NAME",
+		.driver_data = 0x00000003,
+	},
+};
+
+static struct virtbus_driver vdrv = {
+	.probe = td_probe,
+	.remove = td_remove,
+	.shutdown = td_shutdown,
+	.driver = {
+		.name = "virtual_bus_dev",
+	},
+};
+
+static int __init test_drv_init(void)
+{
+	int ret;
+
+	printk(KERN_INFO "Registering Virtual Bus Test Driver\n");
+
+	/* To do a simple match, leave the id_table as NULL */
+	vdrv.id_table = &vdev_id_table[0];
+
+	printk(KERN_ERR "name of 0 is %s\n", vdrv.id_table->name);
+
+	ret = virtbus_drv_register(&vdrv);
+
+	if (!ret)
+		printk(KERN_INFO "Virtual Driver registered\n");
+	else
+		printk(KERN_INFO "Virtual Driver FAILED!!\n");
+
+	return ret;
+}
+
+static void __exit test_drv_exit(void)

tools/testing/selftests/virtual_bus/virtual_bus_drv/virtual_bus_drv.c line 115

Quoted 10 lines
tools/testing/selftests/virtual_bus/virtual_bus_drv/virtual_bus_drv.c new file +10 -0
+{
+	printk(KERN_INFO "Exiting Virtual Bus Test Driver");
+
+	virtbus_drv_unregister(&vdrv);
+
+	printk(KERN_INFO "Virtual Bus Test Driver removed\n");
+}
+
+module_init(test_drv_init);
+module_exit(test_drv_exit);

Odd test, what is it supposed to do? If you use kunit, does that make it easier to test things out?

thanks,

greg k-h

#5 lore raw

Re: [PATCH v3 01/20] virtual-bus: Implementation of Virtual Bus

In reply to Jeff Kirsher

drivers/bus/virtual_bus.c line 44

Quoted 3 lines · Jeff Kirsher
drivers/bus/virtual_bus.c new file +3 -0
+#define to_virtbus_dev(x)	(container_of((x), struct virtbus_device, dev))
+#define to_virtbus_drv(x)	(container_of((x), struct virtbus_driver, \
+				 driver))

Please use static inlines for things like this, it makes the type system clearer

drivers/bus/virtual_bus.c line 128

Quoted 9 lines
drivers/bus/virtual_bus.c new file +9 -0
+/**
+ * virtbus_dev_register - add a virtual bus device
+ * @vdev: virtual bus device to add
+ */
+int virtbus_dev_register(struct virtbus_device *vdev)
+{
+	int ret;
+
+	device_initialize(&vdev->dev);

I generally try to discourage the pattern where the device_initialize is inside a function called register.

The kref inside the struct device should be the only kref for this memory, and the kref system should be setup close to allocating the memory. Any non-trivial user tends to require access to the kref before calling register (which should be done last)

drivers/bus/virtual_bus.c line 134

Quoted 4 lines
drivers/bus/virtual_bus.c new file +4 -0
+	/* All device IDs are automatically allocated */
+	ret = ida_simple_get(&virtbus_dev_ida, 0, 0, GFP_KERNEL);
+	if (ret < 0)
+		return ret;

Should this be a cyclic allocation?

drivers/bus/virtual_bus.c line 137

Quoted 3 lines
drivers/bus/virtual_bus.c new file +3 -0
+
+	vdev->id = ret;
+	dev_set_name(&vdev->dev, "%s.%d", vdev->name, vdev->id);

This is also stuff that can be useful to do early as the driver can then use functions like dev_warn/etc

drivers/bus/virtual_bus.c line 170

Quoted 4 lines
drivers/bus/virtual_bus.c new file +4 -0
+struct virtbus_object {
+	struct virtbus_device vdev;
+	char name[];
+};

This whole virtbus_object makes no sense to me

drivers/bus/virtual_bus.c line 177

Quoted 7 lines
drivers/bus/virtual_bus.c new file +7 -0
+
+/**
+ * virtbus_dev_release - Destroy a virtbus device
+ * @vdev: virtual device to release
+ *
+ * Note that the vdev->data which is separately allocated needs to be
+ * separately freed on it own.

How will that happen?

drivers/bus/virtual_bus.c line 186

Quoted 9 lines

+ */

drivers/bus/virtual_bus.c new file +8 -0
+static void virtbus_dev_release(struct device *dev)
+{
+	struct virtbus_object *vo = container_of(dev, struct virtbus_object,
+						 vdev.dev);
+
+	ida_simple_remove(&virtbus_dev_ida, vo->vdev.id);
+	kfree(vo);
+}

Is something wrong with my search? I couldn't find a user for this?

If the virtbus framework wants to provide a release function then it should also provide the alloc and require that the virtbus_device be at offset 0 in the caller's struct so that the above kfree can work. (ie like netdev does with the whole priv thing)

I have no idea what the virtbus_object is supposed to be doing here.

include/linux/virtual_bus.h line 17

Quoted 3 lines
include/linux/virtual_bus.h new file +3 -0
+struct virtbus_device {
+	const char *name;
+	int id;

id is always positive, should be unsigned

Jason

#6 lore raw

PATCH v3 2/20 ice: Initialize and register a virtual bus to provide RDMA

In reply to Jeff Kirsher

From: Dave Ertman <[email protected]>

The RDMA block does not have its own PCI function, instead it must utilize the ice driver to gain access to the PCI device. Create a virtual bus for the RDMA driver to bind to the device data. The device data contains all of the relevant information that the IRDMA peer will need to access this PF's IIDC API callbacks.

Note the header file iidc.h is located under include/linux/net/intel as this is a unified header file to be used by all consumers of the IIDC interface.

---
 MAINTAINERS                                   |   1 +
 drivers/net/ethernet/intel/Kconfig            |   1 +
 drivers/net/ethernet/intel/ice/Makefile       |   1 +
 drivers/net/ethernet/intel/ice/ice.h          |  14 +
 .../net/ethernet/intel/ice/ice_adminq_cmd.h   |   1 +
 drivers/net/ethernet/intel/ice/ice_common.c   |  15 +
 drivers/net/ethernet/intel/ice/ice_dcb_lib.c  |  31 ++
 drivers/net/ethernet/intel/ice/ice_dcb_lib.h  |   3 +
 .../net/ethernet/intel/ice/ice_hw_autogen.h   |   1 +
 drivers/net/ethernet/intel/ice/ice_idc.c      | 410 ++++++++++++++++++
 drivers/net/ethernet/intel/ice/ice_idc_int.h  |  79 ++++
 drivers/net/ethernet/intel/ice/ice_lib.c      |  11 +
 drivers/net/ethernet/intel/ice/ice_lib.h      |   1 +
 drivers/net/ethernet/intel/ice/ice_main.c     |  65 ++-
 drivers/net/ethernet/intel/ice/ice_type.h     |   1 +
 include/linux/net/intel/iidc.h                | 336 ++++++++++++++
 16 files changed, 969 insertions(+), 2 deletions(-)
 create mode 100644 drivers/net/ethernet/intel/ice/ice_idc.c
 create mode 100644 drivers/net/ethernet/intel/ice/ice_idc_int.h
 create mode 100644 include/linux/net/intel/iidc.h
MAINTAINERS +1 -0
index ce7fd2e7aa1a..1de817eef0ff 100644
@@ -8329,6 +8329,7 @@ F:	Documentation/networking/device_drivers/intel/ice.rst
 F:	drivers/net/ethernet/intel/
 F:	drivers/net/ethernet/intel/*/
 F:	include/linux/avf/virtchnl.h
+F:	include/linux/net/intel/iidc.h
 
 INTEL FRAMEBUFFER DRIVER (excluding 810 and 815)
 M:	Maik Broemme <[email protected]>
drivers/net/ethernet/intel/Kconfig +1 -0
index 154e2e818ec6..b88328fea1d0 100644
@@ -294,6 +294,7 @@ config ICE
 	tristate "Intel(R) Ethernet Connection E800 Series Support"
 	default n
 	depends on PCI_MSI
+	select VIRTUAL_BUS
 	---help---
 	  This driver supports Intel(R) Ethernet Connection E800 Series of
 	  devices.  For more information on how to identify your adapter, go
drivers/net/ethernet/intel/ice/Makefile +1 -0
index 7cb829132d28..b5ffe4ea6071 100644
@@ -18,6 +18,7 @@ ice-y := ice_main.o	\
 	 ice_txrx_lib.o	\
 	 ice_txrx.o	\
 	 ice_flex_pipe.o	\
+	 ice_idc.o	\
 	 ice_ethtool.o
 ice-$(CONFIG_PCI_IOV) += ice_virtchnl_pf.o ice_sriov.o
 ice-$(CONFIG_DCB) += ice_dcb.o ice_dcb_nl.o ice_dcb_lib.o
drivers/net/ethernet/intel/ice/ice.h +14 -0
index f972dce8aebb..1f09e9b53106 100644
@@ -33,6 +33,7 @@
 #include <linux/if_bridge.h>
 #include <linux/ctype.h>
 #include <linux/bpf.h>
+#include <linux/virtual_bus.h>
 #include <linux/avf/virtchnl.h>
 #include <net/ipv6.h>
 #include <net/xdp_sock.h>
@@ -43,6 +44,7 @@
 #include "ice_switch.h"
 #include "ice_common.h"
 #include "ice_sched.h"
+#include "ice_idc_int.h"
 #include "ice_virtchnl_pf.h"
 #include "ice_sriov.h"
 #include "ice_xsk.h"
@@ -73,6 +75,7 @@ extern const char ice_drv_ver[];
 #define ICE_MAX_SMALL_RSS_QS	8
 #define ICE_RES_VALID_BIT	0x8000
 #define ICE_RES_MISC_VEC_ID	(ICE_RES_VALID_BIT - 1)
+#define ICE_RES_RDMA_VEC_ID	(ICE_RES_MISC_VEC_ID - 1)
 #define ICE_INVAL_Q_INDEX	0xffff
 #define ICE_INVAL_VFID		256
 
@@ -326,11 +329,13 @@ struct ice_q_vector {
 
 enum ice_pf_flags {
 	ICE_FLAG_FLTR_SYNC,
+	ICE_FLAG_IWARP_ENA,
 	ICE_FLAG_RSS_ENA,
 	ICE_FLAG_SRIOV_ENA,
 	ICE_FLAG_SRIOV_CAPABLE,
 	ICE_FLAG_DCB_CAPABLE,
 	ICE_FLAG_DCB_ENA,
+	ICE_FLAG_PEER_ENA,
 	ICE_FLAG_ADV_FEATURES,
 	ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA,
 	ICE_FLAG_NO_MEDIA,
@@ -372,6 +377,9 @@ struct ice_pf {
 	struct mutex sw_mutex;		/* lock for protecting VSI alloc flow */
 	struct mutex tc_mutex;		/* lock to protect TC changes */
 	u32 msg_enable;
+	/* Total number of MSIX vectors reserved for base driver */
+	u32 num_rdma_msix;
+	u32 rdma_base_vector;
 	u32 hw_csum_rx_error;
 	u32 oicr_idx;		/* Other interrupt cause MSIX vector index */
 	u32 num_avail_sw_msix;	/* remaining MSIX SW vectors left unclaimed */
@@ -398,6 +406,8 @@ struct ice_pf {
 	unsigned long tx_timeout_last_recovery;
 	u32 tx_timeout_recovery_level;
 	char int_name[ICE_INT_NAME_STR_LEN];
+	struct ice_peer_dev_int **peers;
+	int peer_idx;
 	u32 sw_int_count;
 };
 
@@ -510,6 +520,10 @@ int ice_get_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size);
 void ice_fill_rss_lut(u8 *lut, u16 rss_table_size, u16 rss_size);
 int ice_schedule_reset(struct ice_pf *pf, enum ice_reset_req reset);
 void ice_print_link_msg(struct ice_vsi *vsi, bool isup);
+int ice_init_peer_devices(struct ice_pf *pf);
+int
+ice_for_each_peer(struct ice_pf *pf, void *data,
+		  int (*fn)(struct ice_peer_dev_int *, void *));
 int ice_open(struct net_device *netdev);
 int ice_stop(struct net_device *netdev);
 
drivers/net/ethernet/intel/ice/ice_adminq_cmd.h +1 -0
index 5421fc413f94..d413980f0370 100644
@@ -108,6 +108,7 @@ struct ice_aqc_list_caps_elem {
 #define ICE_AQC_CAPS_TXQS				0x0042
 #define ICE_AQC_CAPS_MSIX				0x0043
 #define ICE_AQC_CAPS_MAX_MTU				0x0047
+#define ICE_AQC_CAPS_IWARP				0x0051
 
 	u8 major_ver;
 	u8 minor_ver;
drivers/net/ethernet/intel/ice/ice_common.c +15 -0
index fb1d930470c7..42ecb0d7063c 100644
@@ -1771,6 +1771,11 @@ ice_parse_caps(struct ice_hw *hw, void *buf, u32 cap_count,
 				  "%s: msix_vector_first_id = %d\n", prefix,
 				  caps->msix_vector_first_id);
 			break;
+		case ICE_AQC_CAPS_IWARP:
+			caps->iwarp = (number == 1);
+			ice_debug(hw, ICE_DBG_INIT,
+				  "%s: iwarp = %d\n", prefix, caps->iwarp);
+			break;
 		case ICE_AQC_CAPS_MAX_MTU:
 			caps->max_mtu = number;
 			ice_debug(hw, ICE_DBG_INIT, "%s: max_mtu = %d\n",
@@ -1794,6 +1799,16 @@ ice_parse_caps(struct ice_hw *hw, void *buf, u32 cap_count,
 		ice_debug(hw, ICE_DBG_INIT,
 			  "%s: maxtc = %d (based on #ports)\n", prefix,
 			  caps->maxtc);
+		if (caps->iwarp) {
+			ice_debug(hw, ICE_DBG_INIT, "%s: forcing RDMA off\n",
+				  prefix);
+			caps->iwarp = 0;
+		}
+
+		/* print message only when processing device capabilities */
+		if (dev_p)
+			dev_info(ice_hw_to_dev(hw),
+				 "RDMA functionality is not available with the current device configuration.\n");
 	}
 }
 
drivers/net/ethernet/intel/ice/ice_dcb_lib.c +31 -0
index d3d3ec29def9..ec03038ea2ba 100644
@@ -720,6 +720,37 @@ ice_tx_prepare_vlan_flags_dcb(struct ice_ring *tx_ring,
 	return 0;
 }
 
+/**
+ * ice_setup_dcb_qos_info - Setup DCB QoS information
+ * @pf: ptr to ice_pf
+ * @qos_info: QoS param instance
+ */
+void ice_setup_dcb_qos_info(struct ice_pf *pf, struct iidc_qos_params *qos_info)
+{
+	struct ice_dcbx_cfg *dcbx_cfg;
+	u32 up2tc;
+	int i;
+
+	dcbx_cfg = &pf->hw.port_info->local_dcbx_cfg;
+	up2tc = rd32(&pf->hw, PRTDCB_TUP2TC);
+	qos_info->num_apps = dcbx_cfg->numapps;
+
+	qos_info->num_tc = ice_dcb_get_num_tc(dcbx_cfg);
+
+	for (i = 0; i < IIDC_MAX_USER_PRIORITY; i++)
+		qos_info->up2tc[i] = (up2tc >> (i * 3)) & 0x7;
+
+	for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++)
+		qos_info->tc_info[i].rel_bw =
+			dcbx_cfg->etscfg.tcbwtable[i];
+
+	for (i = 0; i < qos_info->num_apps; i++) {
+		qos_info->apps[i].priority = dcbx_cfg->app[i].priority;
+		qos_info->apps[i].prot_id = dcbx_cfg->app[i].prot_id;
+		qos_info->apps[i].selector = dcbx_cfg->app[i].selector;
+	}
+}
+
 /**
  * ice_dcb_process_lldp_set_mib_change - Process MIB change
  * @pf: ptr to ice_pf
drivers/net/ethernet/intel/ice/ice_dcb_lib.h +3 -0
index f15e5776f287..bb53edf462ba 100644
@@ -28,6 +28,8 @@ int
 ice_tx_prepare_vlan_flags_dcb(struct ice_ring *tx_ring,
 			      struct ice_tx_buf *first);
 void
+ice_setup_dcb_qos_info(struct ice_pf *pf, struct iidc_qos_params *qos_info);
+void
 ice_dcb_process_lldp_set_mib_change(struct ice_pf *pf,
 				    struct ice_rq_event_info *event);
 void ice_vsi_cfg_netdev_tc(struct ice_vsi *vsi, u8 ena_tc);
@@ -81,6 +83,7 @@ ice_tx_prepare_vlan_flags_dcb(struct ice_ring __always_unused *tx_ring,
 #define ice_update_dcb_stats(pf) do {} while (0)
 #define ice_pf_dcb_recfg(pf) do {} while (0)
 #define ice_vsi_cfg_dcb_rings(vsi) do {} while (0)
+#define ice_setup_dcb_qos_info(pf, qos_info) do {} while (0)
 #define ice_dcb_process_lldp_set_mib_change(pf, event) do {} while (0)
 #define ice_set_cgd_num(tlan_ctx, ring) do {} while (0)
 #define ice_vsi_cfg_netdev_tc(vsi, ena_tc) do {} while (0)
drivers/net/ethernet/intel/ice/ice_hw_autogen.h +1 -0
index e8f32350fed2..bf9743c970fe 100644
@@ -58,6 +58,7 @@
 #define PRTDCB_GENS				0x00083020
 #define PRTDCB_GENS_DCBX_STATUS_S		0
 #define PRTDCB_GENS_DCBX_STATUS_M		ICE_M(0x7, 0)
+#define PRTDCB_TUP2TC				0x001D26C0
 #define GL_PREEXT_L2_PMASK0(_i)			(0x0020F0FC + ((_i) * 4))
 #define GL_PREEXT_L2_PMASK1(_i)			(0x0020F108 + ((_i) * 4))
 #define GLFLXP_RXDID_FLAGS(_i, _j)		(0x0045D000 + ((_i) * 4 + (_j) * 256))
drivers/net/ethernet/intel/ice/ice_idc.c new file +410 -0

410 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_idc_int.h new file +79 -0

79 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_lib.c +11 -0
index e7449248fab4..1456eed1b3b9 100644
@@ -492,6 +492,17 @@ bool ice_is_safe_mode(struct ice_pf *pf)
 	return !test_bit(ICE_FLAG_ADV_FEATURES, pf->flags);
 }
 
+/**
+ * ice_is_peer_ena
+ * @pf: pointer to the PF struct
+ *
+ * returns true if peer devices/drivers are supported, false otherwise
+ */
+bool ice_is_peer_ena(struct ice_pf *pf)
+{
+	return test_bit(ICE_FLAG_PEER_ENA, pf->flags);
+}
+
 /**
  * ice_rss_clean - Delete RSS related VSI structures that hold user inputs
  * @vsi: the VSI being removed
drivers/net/ethernet/intel/ice/ice_lib.h +1 -0
index 6e31e30aba39..502c05c1e84b 100644
@@ -103,4 +103,5 @@ enum ice_status
 ice_vsi_cfg_mac_fltr(struct ice_vsi *vsi, const u8 *macaddr, bool set);
 
 bool ice_is_safe_mode(struct ice_pf *pf);
+bool ice_is_peer_ena(struct ice_pf *pf);
 #endif /* !_ICE_LIB_H_ */
drivers/net/ethernet/intel/ice/ice_main.c +63 -2

121 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_type.h +1 -0
index c4854a987130..e73784e093c5 100644
@@ -187,6 +187,7 @@ struct ice_hw_common_caps {
 	u8 rss_table_entry_width;	/* RSS Entry width in bits */
 
 	u8 dcb;
+	u8 iwarp;
 };
 
 /* Function specific capabilities */
include/linux/net/intel/iidc.h new file +336 -0

336 lines not on this page · raw message

Signature · 1 line
2.23.0
#7 lore raw

Re: [PATCH v3 02/20] ice: Initialize and register a virtual bus to provide RDMA

Quoted 28 lines · Jeff Kirsher

From: Dave Ertman <[email protected]>

The RDMA block does not have its own PCI function, instead it must utilize the ice driver to gain access to the PCI device. Create a virtual bus for the RDMA driver to bind to the device data. The device data contains all of the relevant information that the IRDMA peer will need to access this PF's IIDC API callbacks.

Note the header file iidc.h is located under include/linux/net/intel as this is a unified header file to be used by all consumers of the IIDC interface.

---
 MAINTAINERS                                   |   1 +
 drivers/net/ethernet/intel/Kconfig            |   1 +
 drivers/net/ethernet/intel/ice/Makefile       |   1 +
 drivers/net/ethernet/intel/ice/ice.h          |  14 +
 .../net/ethernet/intel/ice/ice_adminq_cmd.h   |   1 +
 drivers/net/ethernet/intel/ice/ice_common.c   |  15 +
 drivers/net/ethernet/intel/ice/ice_dcb_lib.c  |  31 ++
 drivers/net/ethernet/intel/ice/ice_dcb_lib.h  |   3 +
 .../net/ethernet/intel/ice/ice_hw_autogen.h   |   1 +
 drivers/net/ethernet/intel/ice/ice_idc.c      | 410 ++++++++++++++++++
 drivers/net/ethernet/intel/ice/ice_idc_int.h  |  79 ++++
 drivers/net/ethernet/intel/ice/ice_lib.c      |  11 +
Quoted 8 lines
 drivers/net/ethernet/intel/ice/ice_lib.h      |   1 +
 drivers/net/ethernet/intel/ice/ice_main.c     |  65 ++-
 drivers/net/ethernet/intel/ice/ice_type.h     |   1 +
 include/linux/net/intel/iidc.h                | 336 ++++++++++++++
 16 files changed, 969 insertions(+), 2 deletions(-)
 create mode 100644 drivers/net/ethernet/intel/ice/ice_idc.c
 create mode 100644 drivers/net/ethernet/intel/ice/ice_idc_int.h
 create mode 100644 include/linux/net/intel/iidc.h

You have a lot of different things all mushed together into one patch, so much so that this is pretty much impossible for me to review. How did you all handle it?

Can't this be broken up into smaller, logical, changes that make it obvious that what you are doing here is correct and makes sense? As it is, my few comments below show total confusion on my part...

I can think of a few ways that this can be handled better, but as you know the code best, I'm sure you can come up with a better way.

Quoted 29 lines
drivers/net/ethernet/intel/ice/ice_idc.c new file +29 -0
+int
+ice_unreg_peer_device(struct ice_peer_dev_int *peer_dev_int,
+		      void __always_unused *data)
+{
+	struct ice_peer_drv_int *peer_drv_int;
+	struct iidc_peer_dev *peer_dev;
+	struct pci_dev *pdev;
+	struct device *dev;
+	struct ice_pf *pf;
+
+	if (!peer_dev_int)
+		return 0;
+
+	peer_dev = ice_get_peer_dev(peer_dev_int);
+	pdev = peer_dev->pdev;
+	if (!pdev)
+		return 0;
+
+	pf = pci_get_drvdata(pdev);
+	if (!pf)
+		return 0;
+	dev = ice_pf_to_dev(pf);
+
+	virtbus_dev_unregister(&peer_dev_int->vobj.vdev);
+
+	peer_drv_int = peer_dev_int->peer_drv_int;
+
+	if (peer_dev_int->ice_peer_wq) {
+		if (peer_dev_int->peer_prep_task.func)

drivers/net/ethernet/intel/ice/ice_idc.c line 241

Quoted 10 lines
drivers/net/ethernet/intel/ice/ice_idc.c new file +10 -0
+			cancel_work_sync(&peer_dev_int->peer_prep_task);
+
+		if (peer_dev_int->peer_close_task.func)
+			cancel_work_sync(&peer_dev_int->peer_close_task);
+		destroy_workqueue(peer_dev_int->ice_peer_wq);
+	}
+
+	devm_kfree(dev, peer_drv_int);
+
+	devm_kfree(dev, peer_dev_int);

I had to stare at this multiple times to make sure this really is correct. Shouldn't the devm api handle this for you?

Quoted 90 lines
drivers/net/ethernet/intel/ice/ice_idc.c new file +90 -0
+/**
+ * ice_init_peer_devices - initializes peer devices
+ * @pf: ptr to ice_pf
+ *
+ * This function initializes peer devices on the virtual bus.
+ */
+int ice_init_peer_devices(struct ice_pf *pf)
+{
+	struct ice_vsi *vsi = pf->vsi[0];
+	struct pci_dev *pdev = pf->pdev;
+	struct device *dev = &pdev->dev;
+	int status = 0;
+	int i;
+
+	/* Reserve vector resources */
+	status = ice_reserve_peer_qvector(pf);
+	if (status < 0) {
+		dev_err(dev, "failed to reserve vectors for peer drivers\n");
+		return status;
+	}
+	for (i = 0; i < ARRAY_SIZE(ice_peers); i++) {
+		struct ice_peer_dev_int *peer_dev_int;
+		struct ice_peer_drv_int *peer_drv_int;
+		struct iidc_qos_params *qos_info;
+		struct msix_entry *entry = NULL;
+		struct iidc_peer_dev *peer_dev;
+		struct virtbus_device *vdev;
+		int j;
+
+		peer_dev_int = devm_kzalloc(dev, sizeof(*peer_dev_int),
+					    GFP_KERNEL);
+		if (!peer_dev_int)
+			return -ENOMEM;
+		pf->peers[i] = peer_dev_int;
+
+		peer_drv_int = devm_kzalloc(dev, sizeof(*peer_drv_int),
+					    GFP_KERNEL);
+		if (!peer_drv_int) {
+			devm_kfree(dev, peer_dev_int);
+			return -ENOMEM;
+		}
+
+		peer_dev_int->peer_drv_int = peer_drv_int;
+
+		/* Initialize driver values */
+		for (j = 0; j < IIDC_EVENT_NBITS; j++)
+			bitmap_zero(peer_drv_int->current_events[j].type,
+				    IIDC_EVENT_NBITS);
+
+		mutex_init(&peer_dev_int->peer_dev_state_mutex);
+
+		peer_dev = ice_get_peer_dev(peer_dev_int);
+		peer_dev->peer_ops = NULL;
+		peer_dev->hw_addr = (u8 __iomem *)pf->hw.hw_addr;
+		peer_dev->peer_dev_id = ice_peers[i].id;
+		peer_dev->pf_vsi_num = vsi->vsi_num;
+		peer_dev->netdev = vsi->netdev;
+
+		peer_dev_int->ice_peer_wq =
+			alloc_ordered_workqueue("ice_peer_wq_%d", WQ_UNBOUND,
+						i);
+		if (!peer_dev_int->ice_peer_wq)
+			return -ENOMEM;
+
+		peer_dev->pdev = pdev;
+		qos_info = &peer_dev->initial_qos_info;
+
+		/* setup qos_info fields with defaults */
+		qos_info->num_apps = 0;
+		qos_info->num_tc = 1;
+
+		for (j = 0; j < IIDC_MAX_USER_PRIORITY; j++)
+			qos_info->up2tc[j] = 0;
+
+		qos_info->tc_info[0].rel_bw = 100;
+		for (j = 1; j < IEEE_8021QAZ_MAX_TCS; j++)
+			qos_info->tc_info[j].rel_bw = 0;
+
+		/* for DCB, override the qos_info defaults. */
+		ice_setup_dcb_qos_info(pf, qos_info);
+
+		/* make sure peer specific resources such as msix_count and
+		 * msix_entries are initialized
+		 */
+		switch (ice_peers[i].id) {
+		case IIDC_PEER_RDMA_ID:
+			if (test_bit(ICE_FLAG_IWARP_ENA, pf->flags)) {
+				peer_dev->msix_count = pf->num_rdma_msix;
+				entry = &pf->msix_entries[pf->rdma_base_vector];
+			}

drivers/net/ethernet/intel/ice/ice_idc.c line 395

Quoted 10 lines
drivers/net/ethernet/intel/ice/ice_idc.c new file +10 -0
+			break;
+		default:
+			break;
+		}
+
+		peer_dev->msix_entries = entry;
+		ice_peer_state_change(peer_dev_int, ICE_PEER_DEV_STATE_INIT,
+				      false);
+
+		vdev = &peer_dev_int->vobj.vdev;

The virtual device is embedded in another structure that is embedded inside another one which is next to another object in the same structure? Am I reading this right?

Who controls the lifetime rules here? virtbus _should_ but it seems really really complex here and I can't figure it out.

Why so complex?

drivers/net/ethernet/intel/ice/ice_idc.c line 402

Quoted 7 lines
drivers/net/ethernet/intel/ice/ice_idc.c new file +7 -0
+		vdev->name = ice_peers[i].name;
+		vdev->dev.parent = &pdev->dev;
+
+		status = virtbus_dev_register(vdev);
+		if (status) {
+			dev_err(dev, "Failure adding peer dev %s %d\n",
+				ice_peers[i].name, status);

Shouldn't virtbus_dev_register() print the error message as to what went wrong? Callers should not be responsible for that.

Quoted 7 lines
drivers/net/ethernet/intel/ice/ice_idc_int.h new file +7 -0
+			virtbus_dev_unregister(vdev);
+			vdev = NULL;
+			return status;
+		}
+	}
+
+	return status;

drivers/net/ethernet/intel/ice/ice_idc_int.h line 78

Quoted 10 lines
drivers/net/ethernet/intel/ice/ice_idc_int.h new file +9 -0
+}
 
+static inline struct
+iidc_peer_dev *ice_get_peer_dev(struct ice_peer_dev_int *peer_dev_int)
+{
+	if (peer_dev_int)
+		return &peer_dev_int->vobj.peer_dev;
+	else
+		return NULL;
+}

get_/put_ functions usually grab a reference count. You didn't do that here :(

Quoted 4 lines
drivers/net/ethernet/intel/ice/ice_main.c +4 -0
+	/* init peers only if supported */
+	if (ice_is_peer_ena(pf)) {
+		pf->peers = devm_kcalloc(dev, IIDC_MAX_NUM_PEERS,
+					 sizeof(*pf->peers), GFP_KERNEL);

drivers/net/ethernet/intel/ice/ice_main.c ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent) · line 3391

Quoted 10 lines
drivers/net/ethernet/intel/ice/ice_main.c +10 -0
+		if (!pf->peers) {
+			err = -ENOMEM;
+			goto err_init_peer_unroll;
+		}
+
+		err = ice_init_peer_devices(pf);
+		if (err) {
+			dev_err(dev,
+				"Failed to initialize peer devices: 0x%x\n",
+				err);

Wait the memory for the virtual devices is allocated using devm_kcalloc()? Is that true? And then that memory is handled by the reference count logic of the child objects in that structure? That feels really wrong...

Or am I just still confused about the whole object hierarchy here?

drivers/net/ethernet/intel/ice/ice_main.c ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent) · line 3416

Quoted 5 lines
drivers/net/ethernet/intel/ice/ice_main.c +5 -0
+err_init_peer_unroll:
+	if (ice_is_peer_ena(pf)) {
+		ice_for_each_peer(pf, NULL, ice_unroll_peer);
+		if (pf->peers) {
+			devm_kfree(dev, pf->peers);

Isn't the joy of using devm* functions such that you never have to free the memory on your own? Why are you doing that here?

totally lost,

greg k-h

#8 lore raw

RE: [PATCH v3 02/20] ice: Initialize and register a virtual bus to provide RDMA

In reply to Jeff Kirsher
Quoted 335 lines

-----Original Message-----
From: [email protected] <[email protected]> On Behalf Of Jeff Kirsher
Sent: Monday, December 09, 2019 2:49 PM
To: [email protected]; [email protected]
Cc: Ertman, David M <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; Nguyen, Anthony L <[email protected]>; Kirsher, Jeffrey T <[email protected]>
Subject: [PATCH v3 02/20] ice: Initialize and register a virtual bus to provide RDMA

From: Dave Ertman <[email protected]>

The RDMA block does not have its own PCI function, instead it must utilize the ice driver to gain access to the PCI device. Create a virtual bus for the RDMA driver to bind to the device data. The device data contains all of the relevant information that the IRDMA peer will need to access this PF's IIDC API callbacks.

Note the header file iidc.h is located under include/linux/net/intel as this is a unified header file to be used by all consumers of the IIDC interface.

---
 MAINTAINERS                                   |   1 +
 drivers/net/ethernet/intel/Kconfig            |   1 +
 drivers/net/ethernet/intel/ice/Makefile       |   1 +
 drivers/net/ethernet/intel/ice/ice.h          |  14 +
 .../net/ethernet/intel/ice/ice_adminq_cmd.h   |   1 +
 drivers/net/ethernet/intel/ice/ice_common.c   |  15 +
 drivers/net/ethernet/intel/ice/ice_dcb_lib.c  |  31 ++
 drivers/net/ethernet/intel/ice/ice_dcb_lib.h  |   3 +
 .../net/ethernet/intel/ice/ice_hw_autogen.h   |   1 +
 drivers/net/ethernet/intel/ice/ice_idc.c      | 410 ++++++++++++++++++
 drivers/net/ethernet/intel/ice/ice_idc_int.h  |  79 ++++
 drivers/net/ethernet/intel/ice/ice_lib.c      |  11 +
 drivers/net/ethernet/intel/ice/ice_lib.h      |   1 +
 drivers/net/ethernet/intel/ice/ice_main.c     |  65 ++-
 drivers/net/ethernet/intel/ice/ice_type.h     |   1 +
 include/linux/net/intel/iidc.h                | 336 ++++++++++++++
 16 files changed, 969 insertions(+), 2 deletions(-)
 create mode 100644 drivers/net/ethernet/intel/ice/ice_idc.c
 create mode 100644 drivers/net/ethernet/intel/ice/ice_idc_int.h
 create mode 100644 include/linux/net/intel/iidc.h
MAINTAINERS +1 -0
index ce7fd2e7aa1a..1de817eef0ff 100644
@@ -8329,6 +8329,7 @@ F:
 	Documentation/networking/device_drivers/intel/ice.rst
 F:	drivers/net/ethernet/intel/
 F:	drivers/net/ethernet/intel/*/
 F:	include/linux/avf/virtchnl.h
+F:	include/linux/net/intel/iidc.h
 
 INTEL FRAMEBUFFER DRIVER (excluding 810 and 815)

M: Maik Broemme <[email protected]>

drivers/net/ethernet/intel/Kconfig +1 -0
index 154e2e818ec6..b88328fea1d0 100644
@@ -294,6 +294,7 @@ config ICE
 	tristate "Intel(R) Ethernet Connection E800 Series Support"
 	default n
 	depends on PCI_MSI
+	select VIRTUAL_BUS
 	---help---
 	  This driver supports Intel(R) Ethernet Connection E800 Series of
 	  devices.  For more information on how to identify your adapter, go
drivers/net/ethernet/intel/ice/Makefile +1 -0
index 7cb829132d28..b5ffe4ea6071 100644
@@ -18,6 +18,7 @@ ice-y := ice_main.o	\
 	 ice_txrx_lib.o	\
 	 ice_txrx.o	\
 	 ice_flex_pipe.o	\
+	 ice_idc.o	\
 	 ice_ethtool.o
 ice-$(CONFIG_PCI_IOV) += ice_virtchnl_pf.o ice_sriov.o
 ice-$(CONFIG_DCB) += ice_dcb.o ice_dcb_nl.o ice_dcb_lib.o
drivers/net/ethernet/intel/ice/ice.h +8 -0
index f972dce8aebb..1f09e9b53106 100644
@@ -33,6 +33,7 @@
 #include <linux/if_bridge.h>
 #include <linux/ctype.h>
 #include <linux/bpf.h>
+#include <linux/virtual_bus.h>
 #include <linux/avf/virtchnl.h>
 #include <net/ipv6.h>
 #include <net/xdp_sock.h>
@@ -43,6 +44,7 @@
 #include "ice_switch.h"
 #include "ice_common.h"
 #include "ice_sched.h"
+#include "ice_idc_int.h"
 #include "ice_virtchnl_pf.h"
 #include "ice_sriov.h"
 #include "ice_xsk.h"
@@ -73,6 +75,7 @@ extern const char ice_drv_ver[];
 #define ICE_MAX_SMALL_RSS_QS	8
 #define ICE_RES_VALID_BIT	0x8000
 #define ICE_RES_MISC_VEC_ID	(ICE_RES_VALID_BIT - 1)
+#define ICE_RES_RDMA_VEC_ID	(ICE_RES_MISC_VEC_ID - 1)
 #define ICE_INVAL_Q_INDEX	0xffff
 #define ICE_INVAL_VFID		256
 
@@ -326,11 +329,13 @@ struct ice_q_vector {
 
 enum ice_pf_flags {
 	ICE_FLAG_FLTR_SYNC,
+	ICE_FLAG_IWARP_ENA,
 	ICE_FLAG_RSS_ENA,
 	ICE_FLAG_SRIOV_ENA,
 	ICE_FLAG_SRIOV_CAPABLE,
 	ICE_FLAG_DCB_CAPABLE,
 	ICE_FLAG_DCB_ENA,
+	ICE_FLAG_PEER_ENA,
 	ICE_FLAG_ADV_FEATURES,
 	ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA,
 	ICE_FLAG_NO_MEDIA,
@@ -372,6 +377,9 @@ struct ice_pf {
 	struct mutex sw_mutex;		/* lock for protecting VSI alloc flow */
 	struct mutex tc_mutex;		/* lock to protect TC changes */
 	u32 msg_enable;
+	/* Total number of MSIX vectors reserved for base driver */
+	u32 num_rdma_msix;
+	u32 rdma_base_vector;
 	u32 hw_csum_rx_error;
 	u32 oicr_idx;		/* Other interrupt cause MSIX vector index */
 	u32 num_avail_sw_msix;	/* remaining MSIX SW vectors left

unclaimed */

drivers/net/ethernet/intel/ice/ice.h +6 -0
@@ -398,6 +406,8 @@ struct ice_pf {
 	unsigned long tx_timeout_last_recovery;
 	u32 tx_timeout_recovery_level;
 	char int_name[ICE_INT_NAME_STR_LEN];
+	struct ice_peer_dev_int **peers;
+	int peer_idx;
 	u32 sw_int_count;
 };
 
@@ -510,6 +520,10 @@ int ice_get_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut,
 u16 lut_size);
 void ice_fill_rss_lut(u8 *lut, u16 rss_table_size, u16 rss_size);
 int ice_schedule_reset(struct ice_pf *pf, enum ice_reset_req reset);
 void ice_print_link_msg(struct ice_vsi *vsi, bool isup);
+int ice_init_peer_devices(struct ice_pf *pf);
+int
+ice_for_each_peer(struct ice_pf *pf, void *data,
+		  int (*fn)(struct ice_peer_dev_int *, void *));
 int ice_open(struct net_device *netdev);
 int ice_stop(struct net_device *netdev);
drivers/net/ethernet/intel/ice/ice_adminq_cmd.h +1 -0
index 5421fc413f94..d413980f0370 100644
@@ -108,6 +108,7 @@ struct ice_aqc_list_caps_elem {
 #define ICE_AQC_CAPS_TXQS				0x0042
 #define ICE_AQC_CAPS_MSIX				0x0043
 #define ICE_AQC_CAPS_MAX_MTU				0x0047
+#define ICE_AQC_CAPS_IWARP				0x0051
 
 	u8 major_ver;
 	u8 minor_ver;
drivers/net/ethernet/intel/ice/ice_common.c +5 -0
index fb1d930470c7..42ecb0d7063c 100644
@@ -1771,6 +1771,11 @@ ice_parse_caps(struct ice_hw *hw, void *buf, u32
 cap_count,
 				  "%s: msix_vector_first_id = %d\n", prefix,
 				  caps->msix_vector_first_id);
 			break;
+		case ICE_AQC_CAPS_IWARP:
+			caps->iwarp = (number == 1);
+			ice_debug(hw, ICE_DBG_INIT,
+				  "%s: iwarp = %d\n", prefix, caps->iwarp);
+			break;
 		case ICE_AQC_CAPS_MAX_MTU:
 			caps->max_mtu = number;
drivers/net/ethernet/intel/ice/ice_common.c +10 -0
 			ice_debug(hw, ICE_DBG_INIT, "%s: max_mtu = %d\n",
@@ -1794,6 +1799,16 @@ ice_parse_caps(struct ice_hw *hw, void *buf, u32
 cap_count,
 		ice_debug(hw, ICE_DBG_INIT,
 			  "%s: maxtc = %d (based on #ports)\n", prefix,
 			  caps->maxtc);
+		if (caps->iwarp) {
+			ice_debug(hw, ICE_DBG_INIT, "%s: forcing RDMA
 off\n",
+				  prefix);
+			caps->iwarp = 0;
+		}
+
+		/* print message only when processing device capabilities */
+		if (dev_p)
+			dev_info(ice_hw_to_dev(hw),
+				 "RDMA functionality is not available with the
 current device configuration.\n");
drivers/net/ethernet/intel/ice/ice_common.c +0 -0
 	}
 }
drivers/net/ethernet/intel/ice/ice_dcb_lib.c +31 -0
index d3d3ec29def9..ec03038ea2ba 100644
@@ -720,6 +720,37 @@ ice_tx_prepare_vlan_flags_dcb(struct ice_ring
 *tx_ring,
 	return 0;
 }
 
+/**
+ * ice_setup_dcb_qos_info - Setup DCB QoS information
+ * @pf: ptr to ice_pf
+ * @qos_info: QoS param instance
+ */
+void ice_setup_dcb_qos_info(struct ice_pf *pf, struct iidc_qos_params
 *qos_info)
+{
+	struct ice_dcbx_cfg *dcbx_cfg;
+	u32 up2tc;
+	int i;
+
+	dcbx_cfg = &pf->hw.port_info->local_dcbx_cfg;
+	up2tc = rd32(&pf->hw, PRTDCB_TUP2TC);
+	qos_info->num_apps = dcbx_cfg->numapps;
+
+	qos_info->num_tc = ice_dcb_get_num_tc(dcbx_cfg);
+
+	for (i = 0; i < IIDC_MAX_USER_PRIORITY; i++)
+		qos_info->up2tc[i] = (up2tc >> (i * 3)) & 0x7;
+
+	for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++)
+		qos_info->tc_info[i].rel_bw =
+			dcbx_cfg->etscfg.tcbwtable[i];
+
+	for (i = 0; i < qos_info->num_apps; i++) {
+		qos_info->apps[i].priority = dcbx_cfg->app[i].priority;
+		qos_info->apps[i].prot_id = dcbx_cfg->app[i].prot_id;
+		qos_info->apps[i].selector = dcbx_cfg->app[i].selector;
+	}
+}
+
 /**

* ice_dcb_process_lldp_set_mib_change - Process MIB change
* @pf: ptr to ice_pf

drivers/net/ethernet/intel/ice/ice_dcb_lib.h +3 -0
index f15e5776f287..bb53edf462ba 100644
@@ -28,6 +28,8 @@ int
 ice_tx_prepare_vlan_flags_dcb(struct ice_ring *tx_ring,
 			      struct ice_tx_buf *first);
 void
+ice_setup_dcb_qos_info(struct ice_pf *pf, struct iidc_qos_params *qos_info);
+void
 ice_dcb_process_lldp_set_mib_change(struct ice_pf *pf,
 				    struct ice_rq_event_info *event);
 void ice_vsi_cfg_netdev_tc(struct ice_vsi *vsi, u8 ena_tc);
@@ -81,6 +83,7 @@ ice_tx_prepare_vlan_flags_dcb(struct ice_ring
 __always_unused *tx_ring,
 #define ice_update_dcb_stats(pf) do {} while (0)
 #define ice_pf_dcb_recfg(pf) do {} while (0)
 #define ice_vsi_cfg_dcb_rings(vsi) do {} while (0)
+#define ice_setup_dcb_qos_info(pf, qos_info) do {} while (0)
 #define ice_dcb_process_lldp_set_mib_change(pf, event) do {} while (0)
 #define ice_set_cgd_num(tlan_ctx, ring) do {} while (0)

#define ice_vsi_cfg_netdev_tc(vsi, ena_tc) do {} while (0)

drivers/net/ethernet/intel/ice/ice_hw_autogen.h +1 -0
index e8f32350fed2..bf9743c970fe 100644
@@ -58,6 +58,7 @@
 #define PRTDCB_GENS				0x00083020
 #define PRTDCB_GENS_DCBX_STATUS_S		0
 #define PRTDCB_GENS_DCBX_STATUS_M		ICE_M(0x7, 0)
+#define PRTDCB_TUP2TC				0x001D26C0
 #define GL_PREEXT_L2_PMASK0(_i)			(0x0020F0FC + ((_i) *
 4))
 #define GL_PREEXT_L2_PMASK1(_i)			(0x0020F108 + ((_i) *

4))
#define GLFLXP_RXDID_FLAGS(_i, _j) (0x0045D000 + ((_i) * 4 + (_j) *
256))

drivers/net/ethernet/intel/ice/ice_idc.c new file +29 -0
new file mode 100644
index 000000000000..54783ae6511c
@@ -0,0 +1,410 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2019, Intel Corporation. */
+
+/* Inter-Driver Communication */
+#include "ice.h"
+#include "ice_lib.h"
+#include "ice_dcb_lib.h"
+
+static struct peer_dev_id ice_peers[] = ASSIGN_PEER_INFO;
+
+/**
+ * ice_peer_state_change - manage state machine for peer
+ * @peer_dev: pointer to peer's configuration
+ * @new_state: the state requested to transition into
+ * @locked: boolean to determine if call made with mutex held
+ *
+ * This function handles all state transitions for peer devices.
+ * The state machine is as follows:
+ *
+ *     +<-----------------------+<-----------------------------+
+ *				|<-------+<----------+	       +
+ *				\/	 +	     +	       +
+ *    INIT  --------------> PROBED --> OPENING	  CLOSED --> REMOVED
+ *					 +           +
+ *				       OPENED --> CLOSING
+ *					 +	     +
+ *				       PREP_RST	     +
+ *					 +	     +
+ *				      PREPPED	     +

drivers/net/ethernet/intel/ice/ice_idc.c line 39

Quoted 10 lines
drivers/net/ethernet/intel/ice/ice_idc.c new file +10 -0
+ *					 +---------->+
+ */
+static void
+ice_peer_state_change(struct ice_peer_dev_int *peer_dev, long new_state,
+		      bool locked)
+{
+	struct device *dev = NULL;
+
+	if (peer_dev)
+		dev = &peer_dev->vobj.vdev.dev;

Why is this check for non-NULL peer_dev here? If peer_dev is ever NULL there will be NULL pointer de-references in the mutex_lock/test_and_clear_bit statements below.

Quoted 1,021 lines
drivers/net/ethernet/intel/ice/ice_idc.c new file +10 -0
+
+	if (!locked)
+		mutex_lock(&peer_dev->peer_dev_state_mutex);
+
+	switch (new_state) {
+	case ICE_PEER_DEV_STATE_INIT:
+		if (test_and_clear_bit(ICE_PEER_DEV_STATE_REMOVED,
+				       peer_dev->state)) {
+			set_bit(ICE_PEER_DEV_STATE_INIT, peer_dev->state);
+			dev_dbg(dev, "state transition from _REMOVED to

_INIT\n");

drivers/net/ethernet/intel/ice/ice_idc.c new file +10 -0
+		} else {
+			set_bit(ICE_PEER_DEV_STATE_INIT, peer_dev->state);
+			if (dev)
+				dev_dbg(dev, "state set to _INIT\n");
+		}
+		break;
+	case ICE_PEER_DEV_STATE_PROBED:
+		if (test_and_clear_bit(ICE_PEER_DEV_STATE_INIT,
+				       peer_dev->state)) {
+			set_bit(ICE_PEER_DEV_STATE_PROBED, peer_dev-
Quoted 1 line

state);

drivers/net/ethernet/intel/ice/ice_idc.c new file +1 -0
+			dev_dbg(dev, "state transition from _INIT to

_PROBED\n");

drivers/net/ethernet/intel/ice/ice_idc.c new file +3 -0
+		} else if (test_and_clear_bit(ICE_PEER_DEV_STATE_REMOVED,
+					      peer_dev->state)) {
+			set_bit(ICE_PEER_DEV_STATE_PROBED, peer_dev-
Quoted 1 line

state);

drivers/net/ethernet/intel/ice/ice_idc.c new file +1 -0
+			dev_dbg(dev, "state transition from _REMOVED to

_PROBED\n");

drivers/net/ethernet/intel/ice/ice_idc.c new file +3 -0
+		} else if (test_and_clear_bit(ICE_PEER_DEV_STATE_OPENING,
+					      peer_dev->state)) {
+			set_bit(ICE_PEER_DEV_STATE_PROBED, peer_dev-
Quoted 1 line

state);

drivers/net/ethernet/intel/ice/ice_idc.c new file +1 -0
+			dev_dbg(dev, "state transition from _OPENING to

_PROBED\n");

drivers/net/ethernet/intel/ice/ice_idc.c new file +6 -0
+		}
+		break;
+	case ICE_PEER_DEV_STATE_OPENING:
+		if (test_and_clear_bit(ICE_PEER_DEV_STATE_PROBED,
+				       peer_dev->state)) {
+			set_bit(ICE_PEER_DEV_STATE_OPENING, peer_dev-
Quoted 1 line

state);

drivers/net/ethernet/intel/ice/ice_idc.c new file +1 -0
+			dev_dbg(dev, "state transition from _PROBED to

_OPENING\n");

drivers/net/ethernet/intel/ice/ice_idc.c new file +3 -0
+		} else if (test_and_clear_bit(ICE_PEER_DEV_STATE_CLOSED,
+					      peer_dev->state)) {
+			set_bit(ICE_PEER_DEV_STATE_OPENING, peer_dev-
Quoted 1 line

state);

drivers/net/ethernet/intel/ice/ice_idc.c new file +1 -0
+			dev_dbg(dev, "state transition from _CLOSED to

_OPENING\n");

drivers/net/ethernet/intel/ice/ice_idc.c new file +6 -0
+		}
+		break;
+	case ICE_PEER_DEV_STATE_OPENED:
+		if (test_and_clear_bit(ICE_PEER_DEV_STATE_OPENING,
+				       peer_dev->state)) {
+			set_bit(ICE_PEER_DEV_STATE_OPENED, peer_dev-
Quoted 1 line

state);

drivers/net/ethernet/intel/ice/ice_idc.c new file +1 -0
+			dev_dbg(dev, "state transition from _OPENING to

_OPENED\n");

drivers/net/ethernet/intel/ice/ice_idc.c new file +6 -0
+		}
+		break;
+	case ICE_PEER_DEV_STATE_PREP_RST:
+		if (test_and_clear_bit(ICE_PEER_DEV_STATE_OPENED,
+				       peer_dev->state)) {
+			set_bit(ICE_PEER_DEV_STATE_PREP_RST, peer_dev-
Quoted 1 line

state);

drivers/net/ethernet/intel/ice/ice_idc.c new file +1 -0
+			dev_dbg(dev, "state transition from _OPENED to

_PREP_RST\n");

drivers/net/ethernet/intel/ice/ice_idc.c new file +6 -0
+		}
+		break;
+	case ICE_PEER_DEV_STATE_PREPPED:
+		if (test_and_clear_bit(ICE_PEER_DEV_STATE_PREP_RST,
+				       peer_dev->state)) {
+			set_bit(ICE_PEER_DEV_STATE_PREPPED, peer_dev-
Quoted 1 line

state);

drivers/net/ethernet/intel/ice/ice_idc.c new file +1 -0
+			dev_dbg(dev, "state transition _PREP_RST to

_PREPPED\n");

drivers/net/ethernet/intel/ice/ice_idc.c new file +6 -0
+		}
+		break;
+	case ICE_PEER_DEV_STATE_CLOSING:
+		if (test_and_clear_bit(ICE_PEER_DEV_STATE_OPENED,
+				       peer_dev->state)) {
+			set_bit(ICE_PEER_DEV_STATE_CLOSING, peer_dev-
Quoted 1 line

state);

drivers/net/ethernet/intel/ice/ice_idc.c new file +1 -0
+			dev_dbg(dev, "state transition from _OPENED to

_CLOSING\n");

drivers/net/ethernet/intel/ice/ice_idc.c new file +4 -0
+		}
+		if (test_and_clear_bit(ICE_PEER_DEV_STATE_PREPPED,
+				       peer_dev->state)) {
+			set_bit(ICE_PEER_DEV_STATE_CLOSING, peer_dev-
Quoted 1 line

state);

drivers/net/ethernet/intel/ice/ice_idc.c new file +1 -0
+			dev_dbg(dev, "state transition _PREPPED to

_CLOSING\n");

drivers/net/ethernet/intel/ice/ice_idc.c new file +5 -0
+		}
+		/* NOTE - up to peer to handle this situation correctly */
+		if (test_and_clear_bit(ICE_PEER_DEV_STATE_PREP_RST,
+				       peer_dev->state)) {
+			set_bit(ICE_PEER_DEV_STATE_CLOSING, peer_dev-
Quoted 1 line

state);

drivers/net/ethernet/intel/ice/ice_idc.c new file +1 -0
+			dev_warn(dev, "WARN: Peer state PREP_RST to

_CLOSING\n");

drivers/net/ethernet/intel/ice/ice_idc.c new file +6 -0
+		}
+		break;
+	case ICE_PEER_DEV_STATE_CLOSED:
+		if (test_and_clear_bit(ICE_PEER_DEV_STATE_CLOSING,
+				       peer_dev->state)) {
+			set_bit(ICE_PEER_DEV_STATE_CLOSED, peer_dev-
Quoted 1 line

state);

drivers/net/ethernet/intel/ice/ice_idc.c new file +1 -0
+			dev_dbg(dev, "state transition from _CLOSING to

_CLOSED\n");

drivers/net/ethernet/intel/ice/ice_idc.c new file +8 -0
+		}
+		break;
+	case ICE_PEER_DEV_STATE_REMOVED:
+		if (test_and_clear_bit(ICE_PEER_DEV_STATE_OPENED,
+				       peer_dev->state) ||
+		    test_and_clear_bit(ICE_PEER_DEV_STATE_CLOSED,
+				       peer_dev->state)) {
+			set_bit(ICE_PEER_DEV_STATE_REMOVED, peer_dev-
Quoted 1 line

state);

drivers/net/ethernet/intel/ice/ice_idc.c new file +1 -0
+			dev_dbg(dev, "state from _OPENED/_CLOSED to

_REMOVED\n");

drivers/net/ethernet/intel/ice/ice_idc.c new file +2 -0
+			/* Clear registration for events when peer removed */
+			bitmap_zero(peer_dev->events,

ICE_PEER_DEV_STATE_NBITS);

drivers/net/ethernet/intel/ice/ice_idc.c new file +219 -0
+		}
+		break;
+	default:
+		break;
+	}
+
+	if (!locked)
+		mutex_unlock(&peer_dev->peer_dev_state_mutex);
+}
+
+/**
+ * ice_peer_update_vsi - update the pf_vsi info in peer_dev struct
+ * @peer_dev_int: pointer to peer dev internal struct
+ * @data: opaque pointer - VSI to be updated
+ */
+int ice_peer_update_vsi(struct ice_peer_dev_int *peer_dev_int, void *data)
+{
+	struct ice_vsi *vsi = (struct ice_vsi *)data;
+	struct iidc_peer_dev *peer_dev;
+
+	peer_dev = ice_get_peer_dev(peer_dev_int);
+	if (!peer_dev)
+		return 0;
+
+	peer_dev->pf_vsi_num = vsi->vsi_num;
+	return 0;
+}
+
+/**
+ * ice_for_each_peer - iterate across and call function for each peer dev
+ * @pf: pointer to private board struct
+ * @data: data to pass to function on each call
+ * @fn: pointer to function to call for each peer
+ */
+int
+ice_for_each_peer(struct ice_pf *pf, void *data,
+		  int (*fn)(struct ice_peer_dev_int *, void *))
+{
+	int i;
+
+	if (!pf->peers)
+		return 0;
+
+	for (i = 0; i < ARRAY_SIZE(ice_peers); i++) {
+		struct ice_peer_dev_int *peer_dev_int;
+
+		peer_dev_int = pf->peers[i];
+		if (peer_dev_int) {
+			int ret = fn(peer_dev_int, data);
+
+			if (ret)
+				return ret;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * ice_unreg_peer_device - unregister specified device
+ * @peer_dev_int: ptr to peer device internal
+ * @data: ptr to opaque data
+ *
+ * This function invokes device unregistration, removes ID associated with
+ * the specified device.
+ */
+int
+ice_unreg_peer_device(struct ice_peer_dev_int *peer_dev_int,
+		      void __always_unused *data)
+{
+	struct ice_peer_drv_int *peer_drv_int;
+	struct iidc_peer_dev *peer_dev;
+	struct pci_dev *pdev;
+	struct device *dev;
+	struct ice_pf *pf;
+
+	if (!peer_dev_int)
+		return 0;
+
+	peer_dev = ice_get_peer_dev(peer_dev_int);
+	pdev = peer_dev->pdev;
+	if (!pdev)
+		return 0;
+
+	pf = pci_get_drvdata(pdev);
+	if (!pf)
+		return 0;
+	dev = ice_pf_to_dev(pf);
+
+	virtbus_dev_unregister(&peer_dev_int->vobj.vdev);
+
+	peer_drv_int = peer_dev_int->peer_drv_int;
+
+	if (peer_dev_int->ice_peer_wq) {
+		if (peer_dev_int->peer_prep_task.func)
+			cancel_work_sync(&peer_dev_int->peer_prep_task);
+
+		if (peer_dev_int->peer_close_task.func)
+			cancel_work_sync(&peer_dev_int->peer_close_task);
+		destroy_workqueue(peer_dev_int->ice_peer_wq);
+	}
+
+	devm_kfree(dev, peer_drv_int);
+
+	devm_kfree(dev, peer_dev_int);
+
+	return 0;
+}
+
+/**
+ * ice_unroll_peer - destroy peers and peer_wq in case of error
+ * @peer_dev_int: ptr to peer device internal struct
+ * @data: ptr to opaque data
+ *
+ * This function releases resources in the event of a failure in creating
+ * peer devices or their individual work_queues. Meant to be called from
+ * a ice_for_each_peer invocation
+ */
+int
+ice_unroll_peer(struct ice_peer_dev_int *peer_dev_int,
+		void __always_unused *data)
+{
+	struct iidc_peer_dev *peer_dev;
+	struct ice_pf *pf;
+
+	peer_dev = ice_get_peer_dev(peer_dev_int);
+	if (!peer_dev)
+		return 0;
+
+	pf = pci_get_drvdata(peer_dev->pdev);
+	if (!pf)
+		return 0;
+
+	if (peer_dev_int->ice_peer_wq)
+		destroy_workqueue(peer_dev_int->ice_peer_wq);
+	devm_kfree(ice_pf_to_dev(pf), peer_dev_int);
+
+	return 0;
+}
+
+/**
+ * ice_reserve_peer_qvector - Reserve vector resources for peer drivers
+ * @pf: board private structure to initialize
+ */
+static int ice_reserve_peer_qvector(struct ice_pf *pf)
+{
+	if (test_bit(ICE_FLAG_IWARP_ENA, pf->flags)) {
+		int index;
+
+		index = ice_get_res(pf, pf->irq_tracker, pf->num_rdma_msix,
+				    ICE_RES_RDMA_VEC_ID);
+		if (index < 0)
+			return index;
+		pf->num_avail_sw_msix -= pf->num_rdma_msix;
+		pf->rdma_base_vector = index;
+	}
+	return 0;
+}
+
+/**
+ * ice_init_peer_devices - initializes peer devices
+ * @pf: ptr to ice_pf
+ *
+ * This function initializes peer devices on the virtual bus.
+ */
+int ice_init_peer_devices(struct ice_pf *pf)
+{
+	struct ice_vsi *vsi = pf->vsi[0];
+	struct pci_dev *pdev = pf->pdev;
+	struct device *dev = &pdev->dev;
+	int status = 0;
+	int i;
+
+	/* Reserve vector resources */
+	status = ice_reserve_peer_qvector(pf);
+	if (status < 0) {
+		dev_err(dev, "failed to reserve vectors for peer drivers\n");
+		return status;
+	}
+	for (i = 0; i < ARRAY_SIZE(ice_peers); i++) {
+		struct ice_peer_dev_int *peer_dev_int;
+		struct ice_peer_drv_int *peer_drv_int;
+		struct iidc_qos_params *qos_info;
+		struct msix_entry *entry = NULL;
+		struct iidc_peer_dev *peer_dev;
+		struct virtbus_device *vdev;
+		int j;
+
+		peer_dev_int = devm_kzalloc(dev, sizeof(*peer_dev_int),
+					    GFP_KERNEL);
+		if (!peer_dev_int)
+			return -ENOMEM;
+		pf->peers[i] = peer_dev_int;
+
+		peer_drv_int = devm_kzalloc(dev, sizeof(*peer_drv_int),
+					    GFP_KERNEL);
+		if (!peer_drv_int) {
+			devm_kfree(dev, peer_dev_int);
+			return -ENOMEM;
+		}
+
+		peer_dev_int->peer_drv_int = peer_drv_int;
+
+		/* Initialize driver values */
+		for (j = 0; j < IIDC_EVENT_NBITS; j++)
+			bitmap_zero(peer_drv_int->current_events[j].type,
+				    IIDC_EVENT_NBITS);
+
+		mutex_init(&peer_dev_int->peer_dev_state_mutex);
+
+		peer_dev = ice_get_peer_dev(peer_dev_int);
+		peer_dev->peer_ops = NULL;
+		peer_dev->hw_addr = (u8 __iomem *)pf->hw.hw_addr;
+		peer_dev->peer_dev_id = ice_peers[i].id;
+		peer_dev->pf_vsi_num = vsi->vsi_num;
+		peer_dev->netdev = vsi->netdev;
+
+		peer_dev_int->ice_peer_wq =
+			alloc_ordered_workqueue("ice_peer_wq_%d",

WQ_UNBOUND,

drivers/net/ethernet/intel/ice/ice_idc.c new file +29 -0
+						i);
+		if (!peer_dev_int->ice_peer_wq)
+			return -ENOMEM;
+
+		peer_dev->pdev = pdev;
+		qos_info = &peer_dev->initial_qos_info;
+
+		/* setup qos_info fields with defaults */
+		qos_info->num_apps = 0;
+		qos_info->num_tc = 1;
+
+		for (j = 0; j < IIDC_MAX_USER_PRIORITY; j++)
+			qos_info->up2tc[j] = 0;
+
+		qos_info->tc_info[0].rel_bw = 100;
+		for (j = 1; j < IEEE_8021QAZ_MAX_TCS; j++)
+			qos_info->tc_info[j].rel_bw = 0;
+
+		/* for DCB, override the qos_info defaults. */
+		ice_setup_dcb_qos_info(pf, qos_info);
+
+		/* make sure peer specific resources such as msix_count and
+		 * msix_entries are initialized
+		 */
+		switch (ice_peers[i].id) {
+		case IIDC_PEER_RDMA_ID:
+			if (test_bit(ICE_FLAG_IWARP_ENA, pf->flags)) {
+				peer_dev->msix_count = pf->num_rdma_msix;
+				entry = &pf->msix_entries[pf-
Quoted 1 line

rdma_base_vector];

drivers/net/ethernet/intel/ice/ice_idc.c new file +8 -0
+			}
+			break;
+		default:
+			break;
+		}
+
+		peer_dev->msix_entries = entry;
+		ice_peer_state_change(peer_dev_int,

ICE_PEER_DEV_STATE_INIT,

drivers/net/ethernet/intel/ice/ice_idc.c new file +18 -0
+				      false);
+
+		vdev = &peer_dev_int->vobj.vdev;
+		vdev->name = ice_peers[i].name;
+		vdev->dev.parent = &pdev->dev;
+
+		status = virtbus_dev_register(vdev);
+		if (status) {
+			dev_err(dev, "Failure adding peer dev %s %d\n",
+				ice_peers[i].name, status);
+			virtbus_dev_unregister(vdev);
+			vdev = NULL;
+			return status;
+		}
+	}
+
+	return status;
+}
drivers/net/ethernet/intel/ice/ice_idc_int.h new file +69 -0
new file mode 100644
index 000000000000..bba5a925aefb
@@ -0,0 +1,79 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2019, Intel Corporation. */
+
+#ifndef _ICE_IDC_INT_H_
+#define _ICE_IDC_INT_H_
+
+#include <linux/net/intel/iidc.h>
+#include "ice.h"
+
+enum ice_peer_dev_state {
+	ICE_PEER_DEV_STATE_INIT,
+	ICE_PEER_DEV_STATE_PROBED,
+	ICE_PEER_DEV_STATE_OPENING,
+	ICE_PEER_DEV_STATE_OPENED,
+	ICE_PEER_DEV_STATE_PREP_RST,
+	ICE_PEER_DEV_STATE_PREPPED,
+	ICE_PEER_DEV_STATE_CLOSING,
+	ICE_PEER_DEV_STATE_CLOSED,
+	ICE_PEER_DEV_STATE_REMOVED,
+	ICE_PEER_DEV_STATE_API_RDY,
+	ICE_PEER_DEV_STATE_NBITS,               /* must be last */
+};
+
+enum ice_peer_drv_state {
+	ICE_PEER_DRV_STATE_MBX_RDY,
+	ICE_PEER_DRV_STATE_NBITS,               /* must be last */
+};
+
+struct ice_peer_drv_int {
+	struct iidc_peer_drv *peer_drv;
+
+	/* States associated with peer driver */
+	DECLARE_BITMAP(state, ICE_PEER_DRV_STATE_NBITS);
+
+	/* if this peer_dev is the originator of an event, these are the
+	 * most recent events of each type
+	 */
+	struct iidc_event current_events[IIDC_EVENT_NBITS];
+};
+
+#define ICE_MAX_PEER_NAME 64
+
+struct ice_peer_dev_int {
+	struct ice_peer_drv_int *peer_drv_int; /* driver private structure */
+	struct iidc_virtbus_object vobj;
+
+	/* if this peer_dev is the originator of an event, these are the
+	 * most recent events of each type
+	 */
+	struct iidc_event current_events[IIDC_EVENT_NBITS];
+	/* Events a peer has registered to be notified about */
+	DECLARE_BITMAP(events, IIDC_EVENT_NBITS);
+
+	/* States associated with peer device */
+	DECLARE_BITMAP(state, ICE_PEER_DEV_STATE_NBITS);
+	struct mutex peer_dev_state_mutex; /* peer_dev state mutex */
+
+	/* per peer workqueue */
+	struct workqueue_struct *ice_peer_wq;
+
+	struct work_struct peer_prep_task;
+	struct work_struct peer_close_task;
+
+	enum iidc_close_reason rst_type;
+};
+
+int ice_peer_update_vsi(struct ice_peer_dev_int *peer_dev_int, void *data);
+int ice_unroll_peer(struct ice_peer_dev_int *peer_dev_int, void *data);
+int ice_unreg_peer_device(struct ice_peer_dev_int *peer_dev_int, void

*data);

drivers/net/ethernet/intel/ice/ice_idc_int.h new file +10 -0
+
+static inline struct
+iidc_peer_dev *ice_get_peer_dev(struct ice_peer_dev_int *peer_dev_int)
+{
+	if (peer_dev_int)
+		return &peer_dev_int->vobj.peer_dev;
+	else
+		return NULL;
+}
+#endif /* !_ICE_IDC_INT_H_ */
drivers/net/ethernet/intel/ice/ice_lib.c +11 -0
index e7449248fab4..1456eed1b3b9 100644
@@ -492,6 +492,17 @@ bool ice_is_safe_mode(struct ice_pf *pf)
 	return !test_bit(ICE_FLAG_ADV_FEATURES, pf->flags);
 }
 
+/**
+ * ice_is_peer_ena
+ * @pf: pointer to the PF struct
+ *
+ * returns true if peer devices/drivers are supported, false otherwise
+ */
+bool ice_is_peer_ena(struct ice_pf *pf)
+{
+	return test_bit(ICE_FLAG_PEER_ENA, pf->flags);
+}
+
 /**
  * ice_rss_clean - Delete RSS related VSI structures that hold user inputs
  * @vsi: the VSI being removed
drivers/net/ethernet/intel/ice/ice_lib.h +1 -0
index 6e31e30aba39..502c05c1e84b 100644
@@ -103,4 +103,5 @@ enum ice_status
 ice_vsi_cfg_mac_fltr(struct ice_vsi *vsi, const u8 *macaddr, bool set);
 
 bool ice_is_safe_mode(struct ice_pf *pf);
+bool ice_is_peer_ena(struct ice_pf *pf);
 #endif /* !_ICE_LIB_H_ */
drivers/net/ethernet/intel/ice/ice_main.c +44 -2
index 69bff085acf7..950c213a8802 100644
@@ -2657,6 +2657,12 @@ static void ice_set_pf_caps(struct ice_pf *pf)
 {
 	struct ice_hw_func_caps *func_caps = &pf->hw.func_caps;
 
+	clear_bit(ICE_FLAG_IWARP_ENA, pf->flags);
+	clear_bit(ICE_FLAG_PEER_ENA, pf->flags);
+	if (func_caps->common_cap.iwarp) {
+		set_bit(ICE_FLAG_IWARP_ENA, pf->flags);
+		set_bit(ICE_FLAG_PEER_ENA, pf->flags);
+	}
 	clear_bit(ICE_FLAG_DCB_CAPABLE, pf->flags);
 	if (func_caps->common_cap.dcb)
 		set_bit(ICE_FLAG_DCB_CAPABLE, pf->flags);
@@ -2738,6 +2744,17 @@ static int ice_ena_msix_range(struct ice_pf *pf)
 	v_budget += needed;
 	v_left -= needed;
 
+	/* reserve vectors for RDMA peer driver */
+	if (test_bit(ICE_FLAG_IWARP_ENA, pf->flags)) {
+		/* RDMA peer driver needs one extra to handle misc causes */
+		needed = min_t(int, num_online_cpus(), v_left) + 1;
+		if (v_left < needed)
+			goto no_hw_vecs_left_err;
+		pf->num_rdma_msix = needed;
+		v_budget += needed;
+		v_left -= needed;
+	}
+
 	pf->msix_entries = devm_kcalloc(dev, v_budget,
 					sizeof(*pf->msix_entries),
 GFP_KERNEL);
@@ -2763,16 +2780,19 @@ static int ice_ena_msix_range(struct ice_pf *pf)
 		dev_warn(dev,
 			 "not enough OS MSI-X vectors. requested = %d,
 obtained = %d\n",
 			 v_budget, v_actual);
-/* 2 vectors for LAN (traffic + OICR) */
+/* 2 vectors for LAN and RDMA (traffic + OICR) */
 #define ICE_MIN_LAN_VECS 2
+#define ICE_MIN_RDMA_VECS 2
+#define ICE_MIN_VECS (ICE_MIN_LAN_VECS + ICE_MIN_RDMA_VECS)
 
-		if (v_actual < ICE_MIN_LAN_VECS) {
+		if (v_actual < ICE_MIN_VECS) {
 			/* error if we can't get minimum vectors */
 			pci_disable_msix(pf->pdev);
 			err = -ERANGE;
 			goto msix_err;
 		} else {
 			pf->num_lan_msix = ICE_MIN_LAN_VECS;
+			pf->num_rdma_msix = ICE_MIN_RDMA_VECS;
 		}
 	}
@@ -2789,6 +2809,7 @@ static int ice_ena_msix_range(struct ice_pf *pf)
 	err = -ERANGE;
 exit_err:
 	pf->num_lan_msix = 0;
+	pf->num_rdma_msix = 0;
 	return err;
 }
 
@@ -3354,6 +3375,27 @@ ice_probe(struct pci_dev *pdev, const struct
 pci_device_id __always_unused *ent)
 
 	/* initialize DDP driven features */
 
+	/* init peers only if supported */
+	if (ice_is_peer_ena(pf)) {
+		pf->peers = devm_kcalloc(dev, IIDC_MAX_NUM_PEERS,
+					 sizeof(*pf->peers), GFP_KERNEL);
+		if (!pf->peers) {
+			err = -ENOMEM;
+			goto err_init_peer_unroll;
+		}
+
+		err = ice_init_peer_devices(pf);
+		if (err) {
+			dev_err(dev,
+				"Failed to initialize peer devices: 0x%x\n",
+				err);
+			err = -EIO;
+			goto err_init_peer_unroll;
+		}
+	} else {
+		dev_warn(dev, "RDMA is not supported on this device\n");
+	}
+
 	/* Note: DCB init failure is non-fatal to load */
 	if (ice_init_pf_dcb(pf, false)) {
drivers/net/ethernet/intel/ice/ice_main.c +8 -0
 		clear_bit(ICE_FLAG_DCB_CAPABLE, pf->flags);
@@ -3367,6 +3409,14 @@ ice_probe(struct pci_dev *pdev, const struct
 pci_device_id __always_unused *ent)
 
 	return 0;
 
+err_init_peer_unroll:
+	if (ice_is_peer_ena(pf)) {
+		ice_for_each_peer(pf, NULL, ice_unroll_peer);
+		if (pf->peers) {
+			devm_kfree(dev, pf->peers);
+			pf->peers = NULL;
+		}
+	}
 err_alloc_sw_unroll:
 	set_bit(__ICE_SERVICE_DIS, pf->state);
drivers/net/ethernet/intel/ice/ice_main.c +11 -0
 	set_bit(__ICE_DOWN, pf->state);
@@ -3408,6 +3458,7 @@ static void ice_remove(struct pci_dev *pdev)
 	if (test_bit(ICE_FLAG_SRIOV_ENA, pf->flags))
 		ice_free_vfs(pf);
 	ice_vsi_release_all(pf);
+	ice_for_each_peer(pf, NULL, ice_unreg_peer_device);
 	ice_free_irq_msix_misc(pf);
 	ice_for_each_vsi(pf, i) {
 		if (!pf->vsi[i])
@@ -4703,6 +4754,16 @@ static void ice_rebuild(struct ice_pf *pf, enum
 ice_reset_req reset_type)
 		goto err_vsi_rebuild;
 	}
 
+	if (ice_is_peer_ena(pf)) {
+		struct ice_vsi *vsi = ice_get_main_vsi(pf);
+
+		if (!vsi) {
+			dev_err(dev, "No PF_VSI to update peer\n");
+			goto err_vsi_rebuild;
+		}
+		ice_for_each_peer(pf, vsi, ice_peer_update_vsi);
+	}
+
 	if (test_bit(ICE_FLAG_SRIOV_ENA, pf->flags)) {
 		err = ice_vsi_rebuild_by_type(pf, ICE_VSI_VF);
drivers/net/ethernet/intel/ice/ice_main.c +0 -0
 		if (err) {
drivers/net/ethernet/intel/ice/ice_type.h +1 -0
index c4854a987130..e73784e093c5 100644
@@ -187,6 +187,7 @@ struct ice_hw_common_caps {
 	u8 rss_table_entry_width;	/* RSS Entry width in bits */
 
 	u8 dcb;
+	u8 iwarp;
 };
 
 /* Function specific capabilities */
include/linux/net/intel/iidc.h new file +48 -0
new file mode 100644
index 000000000000..a7d694276aa0
@@ -0,0 +1,336 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2019, Intel Corporation. */
+
+#ifndef _IIDC_H_
+#define _IIDC_H_
+
+#include <linux/dcbnl.h>
+#include <linux/device.h>
+#include <linux/if_ether.h>
+#include <linux/kernel.h>
+#include <linux/netdevice.h>
+#include <linux/virtual_bus.h>
+
+enum iidc_event_type {
+	IIDC_EVENT_LINK_CHANGE,
+	IIDC_EVENT_MTU_CHANGE,
+	IIDC_EVENT_TC_CHANGE,
+	IIDC_EVENT_API_CHANGE,
+	IIDC_EVENT_MBX_CHANGE,
+	IIDC_EVENT_NBITS		/* must be last */
+};
+
+enum iidc_res_type {
+	IIDC_INVAL_RES,
+	IIDC_VSI,
+	IIDC_VEB,
+	IIDC_EVENT_Q,
+	IIDC_EGRESS_CMPL_Q,
+	IIDC_CMPL_EVENT_Q,
+	IIDC_ASYNC_EVENT_Q,
+	IIDC_DOORBELL_Q,
+	IIDC_RDMA_QSETS_TXSCHED,
+};
+
+enum iidc_peer_reset_type {
+	IIDC_PEER_PFR,
+	IIDC_PEER_CORER,
+	IIDC_PEER_CORER_SW_CORE,
+	IIDC_PEER_CORER_SW_FULL,
+	IIDC_PEER_GLOBR,
+};
+
+/* reason notified to peer driver as part of event handling */
+enum iidc_close_reason {
+	IIDC_REASON_INVAL,
+	IIDC_REASON_HW_UNRESPONSIVE,
+	IIDC_REASON_INTERFACE_DOWN, /* Administrative down */
+	IIDC_REASON_PEER_DRV_UNREG, /* peer driver getting unregistered

*/

include/linux/net/intel/iidc.h new file +224 -0
+	IIDC_REASON_PEER_DEV_UNINIT,
+	IIDC_REASON_GLOBR_REQ,
+	IIDC_REASON_CORER_REQ,
+	IIDC_REASON_EMPR_REQ,
+	IIDC_REASON_PFR_REQ,
+	IIDC_REASON_HW_RESET_PENDING,
+	IIDC_REASON_RECOVERY_MODE,
+	IIDC_REASON_PARAM_CHANGE,
+};
+
+enum iidc_rdma_filter {
+	IIDC_RDMA_FILTER_INVAL,
+	IIDC_RDMA_FILTER_IWARP,
+	IIDC_RDMA_FILTER_ROCEV2,
+	IIDC_RDMA_FILTER_BOTH,
+};
+
+/* Struct to hold per DCB APP info */
+struct iidc_dcb_app_info {
+	u8  priority;
+	u8  selector;
+	u16 prot_id;
+};
+
+struct iidc_peer_dev;
+
+#define IIDC_MAX_USER_PRIORITY		8
+#define IIDC_MAX_APPS			8
+
+/* Struct to hold per RDMA Qset info */
+struct iidc_rdma_qset_params {
+	u32 teid;	/* qset TEID */
+	u16 qs_handle; /* RDMA driver provides this */
+	u16 vsi_id; /* VSI index */
+	u8 tc; /* TC branch the QSet should belong to */
+	u8 reserved[3];
+};
+
+struct iidc_res_base {
+	/* Union for future provision e.g. other res_type */
+	union {
+		struct iidc_rdma_qset_params qsets;
+	} res;
+};
+
+struct iidc_res {
+	/* Type of resource. Filled by peer driver */
+	enum iidc_res_type res_type;
+	/* Count requested by peer driver */
+	u16 cnt_req;
+
+	/* Number of resources allocated. Filled in by callee.
+	 * Based on this value, caller to fill up "resources"
+	 */
+	u16 res_allocated;
+
+	/* Unique handle to resources allocated. Zero if call fails.
+	 * Allocated by callee and for now used by caller for internal
+	 * tracking purpose.
+	 */
+	u32 res_handle;
+
+	/* Peer driver has to allocate sufficient memory, to accommodate
+	 * cnt_requested before calling this function.
+	 * Memory has to be zero initialized. It is input/output param.
+	 * As a result of alloc_res API, this structures will be populated.
+	 */
+	struct iidc_res_base res[1];
+};
+
+struct iidc_qos_info {
+	u64 tc_ctx;
+	u8 rel_bw;
+	u8 prio_type;
+	u8 egress_virt_up;
+	u8 ingress_virt_up;
+};
+
+/* Struct to hold QoS info */
+struct iidc_qos_params {
+	struct iidc_qos_info tc_info[IEEE_8021QAZ_MAX_TCS];
+	u8 up2tc[IIDC_MAX_USER_PRIORITY];
+	u8 vsi_relative_bw;
+	u8 vsi_priority_type;
+	u32 num_apps;
+	struct iidc_dcb_app_info apps[IIDC_MAX_APPS];
+	u8 num_tc;
+};
+
+union iidc_event_info {
+	/* IIDC_EVENT_LINK_CHANGE */
+	struct {
+		struct net_device *lwr_nd;
+		u16 vsi_num; /* HW index of VSI corresponding to lwr ndev */
+		u8 new_link_state;
+		u8 lport;
+	} link_info;
+	/* IIDC_EVENT_MTU_CHANGE */
+	u16 mtu;
+	/* IIDC_EVENT_TC_CHANGE */
+	struct iidc_qos_params port_qos;
+	/* IIDC_EVENT_API_CHANGE */
+	u8 api_rdy;
+	/* IIDC_EVENT_MBX_CHANGE */
+	u8 mbx_rdy;
+};
+
+/* iidc_event elements are to be passed back and forth between the device
+ * owner and the peer drivers. They are to be used to both register/unregister
+ * for event reporting and to report an event (events can be either device
+ * owner generated or peer generated).
+ *
+ * For (un)registering for events, the structure needs to be populated with:
+ *   reporter - pointer to the iidc_peer_dev struct of the peer (un)registering
+ *   type - bitmap with bits set for event types to (un)register for
+ *
+ * For reporting events, the structure needs to be populated with:
+ *   reporter - pointer to peer that generated the event (NULL for ice)
+ *   type - bitmap with single bit set for this event type
+ *   info - union containing data relevant to this event type
+ */
+struct iidc_event {
+	struct iidc_peer_dev *reporter;
+	DECLARE_BITMAP(type, IIDC_EVENT_NBITS);
+	union iidc_event_info info;
+};
+
+/* Following APIs are implemented by device owner and invoked by peer
+ * drivers
+ */
+struct iidc_ops {
+	/* APIs to allocate resources such as VEB, VSI, Doorbell queues,
+	 * completion queues, Tx/Rx queues, etc...
+	 */
+	int (*alloc_res)(struct iidc_peer_dev *peer_dev,
+			 struct iidc_res *res,
+			 int partial_acceptable);
+	int (*free_res)(struct iidc_peer_dev *peer_dev,
+			struct iidc_res *res);
+
+	int (*is_vsi_ready)(struct iidc_peer_dev *peer_dev);
+	int (*peer_register)(struct iidc_peer_dev *peer_dev);
+	int (*peer_unregister)(struct iidc_peer_dev *peer_dev);
+	int (*request_reset)(struct iidc_peer_dev *dev,
+			     enum iidc_peer_reset_type reset_type);
+
+	void (*notify_state_change)(struct iidc_peer_dev *dev,
+				    struct iidc_event *event);
+
+	/* Notification APIs */
+	void (*reg_for_notification)(struct iidc_peer_dev *dev,
+				     struct iidc_event *event);
+	void (*unreg_for_notification)(struct iidc_peer_dev *dev,
+				       struct iidc_event *event);
+	int (*update_vsi_filter)(struct iidc_peer_dev *peer_dev,
+				 enum iidc_rdma_filter filter, bool enable);
+	int (*vc_send)(struct iidc_peer_dev *peer_dev, u32 vf_id, u8 *msg,
+		       u16 len);
+};
+
+/* Following APIs are implemented by peer drivers and invoked by device
+ * owner
+ */
+struct iidc_peer_ops {
+	void (*event_handler)(struct iidc_peer_dev *peer_dev,
+			      struct iidc_event *event);
+
+	/* Why we have 'open' and when it is expected to be called:
+	 * 1. symmetric set of API w.r.t close
+	 * 2. To be invoked form driver initialization path
+	 *     - call peer_driver:open once device owner is fully
+	 *     initialized
+	 * 3. To be invoked upon RESET complete
+	 */
+	int (*open)(struct iidc_peer_dev *peer_dev);
+
+	/* Peer's close function is to be called when the peer needs to be
+	 * quiesced. This can be for a variety of reasons (enumerated in the
+	 * iidc_close_reason enum struct). A call to close will only be
+	 * followed by a call to either remove or open. No IDC calls from the
+	 * peer should be accepted until it is re-opened.
+	 *
+	 * The *reason* parameter is the reason for the call to close. This
+	 * can be for any reason enumerated in the iidc_close_reason struct.
+	 * It's primary reason is for the peer's bookkeeping and in case the
+	 * peer want to perform any different tasks dictated by the reason.
+	 */
+	void (*close)(struct iidc_peer_dev *peer_dev,
+		      enum iidc_close_reason reason);
+
+	int (*vc_receive)(struct iidc_peer_dev *peer_dev, u32 vf_id, u8 *msg,
+			  u16 len);
+	/* tell RDMA peer to prepare for TC change in a blocking call
+	 * that will directly precede the change event
+	 */
+	void (*prep_tc_change)(struct iidc_peer_dev *peer_dev);
+};
+
+#define IIDC_PEER_RDMA_NAME	"ice_rdma"
+#define IIDC_PEER_RDMA_ID	0x00000010
+#define IIDC_MAX_NUM_PEERS	4
+
+/* The const struct that instantiates peer_dev_id needs to be initialized
+ * in the .c with the macro ASSIGN_PEER_INFO.
+ * For example:
+ * static const struct peer_dev_id peer_dev_ids[] = ASSIGN_PEER_INFO;
+ */
+struct peer_dev_id {
+	char *name;
+	int id;
+};
+
+#define ASSIGN_PEER_INFO						\
+{									\
+	{ .name = IIDC_PEER_RDMA_NAME, .id = IIDC_PEER_RDMA_ID },	\
+}
+
+#define iidc_peer_priv(x) ((x)->peer_priv)
+
+/* Structure representing peer specific information, each peer using the IIDC
+ * interface will have an instance of this struct dedicated to it.
+ */
+struct iidc_peer_dev {
+	struct pci_dev *pdev; /* PCI device of corresponding to main function

*/

include/linux/net/intel/iidc.h new file +64 -1
+	/* KVA / Linear address corresponding to BAR0 of underlying
+	 * pci_device.
+	 */
+	u8 __iomem *hw_addr;
+	int peer_dev_id;
+
+	/* Opaque pointer for peer specific data tracking.  This memory will
+	 * be alloc'd and freed by the peer driver and used for private data
+	 * accessible only to the specific peer.  It is stored here so that
+	 * when this struct is passed to the peer via an IDC call, the data
+	 * can be accessed by the peer at that time.
+	 * The peers should only retrieve the pointer by the macro:
+	 *    iidc_peer_priv(struct iidc_peer_dev *)
+	 */
+	void *peer_priv;
+
+	u8 ftype;	/* PF(false) or VF (true) */
+
+	/* Data VSI created by driver */
+	u16 pf_vsi_num;
+
+	struct iidc_qos_params initial_qos_info;
+	struct net_device *netdev;
+
+	/* Based on peer driver type, this shall point to corresponding MSIx
+	 * entries in pf->msix_entries (which were allocated as part of driver
+	 * initialization) e.g. for RDMA driver, msix_entries reserved will be
+	 * num_online_cpus + 1.
+	 */
+	u16 msix_count; /* How many vectors are reserved for this device */
+	struct msix_entry *msix_entries;
+
+	/* Following struct contains function pointers to be initialized
+	 * by device owner and called by peer driver
+	 */
+	const struct iidc_ops *ops;
+
+	/* Following struct contains function pointers to be initialized
+	 * by peer driver and called by device owner
+	 */
+	const struct iidc_peer_ops *peer_ops;
+
+	/* Pointer to peer_drv struct to be populated by peer driver */
+	struct iidc_peer_drv *peer_drv;
+};
+
+struct iidc_virtbus_object {
+	struct virtbus_device vdev;
+	struct iidc_peer_dev peer_dev;
+};
+
+/* structure representing peer driver
+ * Peer driver to initialize those function ptrs and it will be invoked
+ * by device owner as part of driver_registration via bus infrastructure
+ */
+struct iidc_peer_drv {
+	u16 driver_id;
+#define IIDC_PEER_DEVICE_OWNER		0
+#define IIDC_PEER_RDMA_DRIVER		4
+
+	const char *name;
+
+};
+#endif /* _IIDC_H_*/
--

2.23.0

#9 lore raw

PATCH v3 3/20 ice: Implement peer communications

In reply to Jeff Kirsher

From: Dave Ertman <[email protected]>

Set and implement operations for the peer device and peer driver to communicate with each other, via iidc_ops and iidc_peer_ops, to request resources and manage event notification.

---
 drivers/net/ethernet/intel/ice/ice.h          |   1 +
 .../net/ethernet/intel/ice/ice_adminq_cmd.h   |  32 +
 drivers/net/ethernet/intel/ice/ice_common.c   | 188 ++++
 drivers/net/ethernet/intel/ice/ice_common.h   |   9 +
 drivers/net/ethernet/intel/ice/ice_dcb_lib.c  |  34 +
 drivers/net/ethernet/intel/ice/ice_idc.c      | 879 ++++++++++++++++++
 drivers/net/ethernet/intel/ice/ice_idc_int.h  |  45 +
 drivers/net/ethernet/intel/ice/ice_lib.c      |  39 +
 drivers/net/ethernet/intel/ice/ice_lib.h      |   2 +
 drivers/net/ethernet/intel/ice/ice_main.c     |  45 +-
 drivers/net/ethernet/intel/ice/ice_sched.c    |  69 +-
 drivers/net/ethernet/intel/ice/ice_switch.c   |  27 +
 drivers/net/ethernet/intel/ice/ice_switch.h   |   4 +
 drivers/net/ethernet/intel/ice/ice_type.h     |   3 +
 .../net/ethernet/intel/ice/ice_virtchnl_pf.c  |  25 -
 15 files changed, 1364 insertions(+), 38 deletions(-)
drivers/net/ethernet/intel/ice/ice.h +1 -0
index 1f09e9b53106..dc341ab1bd71 100644
@@ -292,6 +292,7 @@ struct ice_vsi {
 	u16 req_rxq;			 /* User requested Rx queues */
 	u16 num_rx_desc;
 	u16 num_tx_desc;
+	u16 qset_handle[ICE_MAX_TRAFFIC_CLASS];
 	struct ice_tc_cfg tc_cfg;
 	struct bpf_prog *xdp_prog;
 	struct ice_ring **xdp_rings;	 /* XDP ring array */
drivers/net/ethernet/intel/ice/ice_adminq_cmd.h +32 -0
index d413980f0370..2fe53f530517 100644
@@ -1526,6 +1526,36 @@ struct ice_aqc_dis_txq {
 	struct ice_aqc_dis_txq_item qgrps[1];
 };
 
+/* Add Tx RDMA Queue Set (indirect 0x0C33) */
+struct ice_aqc_add_rdma_qset {
+	u8 num_qset_grps;
+	u8 reserved[7];
+	__le32 addr_high;
+	__le32 addr_low;
+};
+
+/* This is the descriptor of each qset entry for the Add Tx RDMA Queue Set
+ * command (0x0C33). Only used within struct ice_aqc_add_rdma_qset.
+ */
+struct ice_aqc_add_tx_rdma_qset_entry {
+	__le16 tx_qset_id;
+	u8 rsvd[2];
+	__le32 qset_teid;
+	struct ice_aqc_txsched_elem info;
+};
+
+/* The format of the command buffer for Add Tx RDMA Queue Set(0x0C33)
+ * is an array of the following structs. Please note that the length of
+ * each struct ice_aqc_add_rdma_qset is variable due to the variable
+ * number of queues in each group!
+ */
+struct ice_aqc_add_rdma_qset_data {
+	__le32 parent_teid;
+	__le16 num_qsets;
+	u8 rsvd[2];
+	struct ice_aqc_add_tx_rdma_qset_entry rdma_qsets[1];
+};
+
 /* Configure Firmware Logging Command (indirect 0xFF09)
  * Logging Information Read Response (indirect 0xFF10)
  * Note: The 0xFF10 command has no input parameters.
@@ -1714,6 +1744,7 @@ struct ice_aq_desc {
 		struct ice_aqc_get_set_rss_key get_set_rss_key;
 		struct ice_aqc_add_txqs add_txqs;
 		struct ice_aqc_dis_txqs dis_txqs;
+		struct ice_aqc_add_rdma_qset add_rdma_qset;
 		struct ice_aqc_add_get_update_free_vsi vsi_cmd;
 		struct ice_aqc_add_update_free_vsi_resp add_update_free_vsi_res;
 		struct ice_aqc_fw_logging fw_logging;
@@ -1847,6 +1878,7 @@ enum ice_adminq_opc {
 	/* Tx queue handling commands/events */
 	ice_aqc_opc_add_txqs				= 0x0C30,
 	ice_aqc_opc_dis_txqs				= 0x0C31,
+	ice_aqc_opc_add_rdma_qset			= 0x0C33,
 
 	/* package commands */
 	ice_aqc_opc_download_pkg			= 0x0C40,
drivers/net/ethernet/intel/ice/ice_common.c +188 -0

200 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_common.h +9 -0
index b22aa561e253..a6beebff1379 100644
@@ -125,6 +125,15 @@ ice_aq_sff_eeprom(struct ice_hw *hw, u16 lport, u8 bus_addr,
 		  bool write, struct ice_sq_cd *cd);
 
 enum ice_status
+ice_cfg_vsi_rdma(struct ice_port_info *pi, u16 vsi_handle, u8 tc_bitmap,
+		 u16 *max_rdmaqs);
+enum ice_status
+ice_ena_vsi_rdma_qset(struct ice_port_info *pi, u16 vsi_handle, u8 tc,
+		      u16 *rdma_qset, u16 num_qsets, u32 *qset_teid);
+enum ice_status
+ice_dis_vsi_rdma_qset(struct ice_port_info *pi, u16 count, u32 *qset_teid,
+		      u16 *q_id);
+enum ice_status
 ice_dis_vsi_txq(struct ice_port_info *pi, u16 vsi_handle, u8 tc, u8 num_queues,
 		u16 *q_handle, u16 *q_ids, u32 *q_teids,
 		enum ice_disq_rst_src rst_src, u16 vmvf_num,
drivers/net/ethernet/intel/ice/ice_dcb_lib.c +34 -0
index ec03038ea2ba..f950deef524b 100644
@@ -148,6 +148,27 @@ void ice_vsi_cfg_dcb_rings(struct ice_vsi *vsi)
 	}
 }
 
+/**
+ * ice_peer_prep_tc_change - Pre-notify RDMA Peer in blocking call of TC change
+ * @peer_dev_int: ptr to peer device internal struct
+ * @data: ptr to opaque data
+ */
+static int
+ice_peer_prep_tc_change(struct ice_peer_dev_int *peer_dev_int,
+			void __always_unused *data)
+{
+	struct iidc_peer_dev *peer_dev;
+
+	peer_dev = ice_get_peer_dev(peer_dev_int);
+	if (!ice_validate_peer_dev(peer_dev))
+		return 0;
+
+	if (peer_dev->peer_ops && peer_dev->peer_ops->prep_tc_change)
+		peer_dev->peer_ops->prep_tc_change(peer_dev);
+
+	return 0;
+}
+
 /**
  * ice_pf_dcb_cfg - Apply new DCB configuration
  * @pf: pointer to the PF struct
@@ -182,6 +203,9 @@ int ice_pf_dcb_cfg(struct ice_pf *pf, struct ice_dcbx_cfg *new_cfg, bool locked)
 		return ret;
 	}
 
+	/* Notify capable peers about impending change to TCs */
+	ice_for_each_peer(pf, NULL, ice_peer_prep_tc_change);
+
 	/* Store old config in case FW config fails */
 	old_cfg = kmemdup(curr_cfg, sizeof(*old_cfg), GFP_KERNEL);
 	if (!old_cfg)
@@ -536,6 +560,7 @@ static int ice_dcb_noncontig_cfg(struct ice_pf *pf)
 void ice_pf_dcb_recfg(struct ice_pf *pf)
 {
 	struct ice_dcbx_cfg *dcbcfg = &pf->hw.port_info->local_dcbx_cfg;
+	struct iidc_event *event;
 	u8 tc_map = 0;
 	int v, ret;
 
@@ -571,6 +596,15 @@ void ice_pf_dcb_recfg(struct ice_pf *pf)
 		if (vsi->type == ICE_VSI_PF)
 			ice_dcbnl_set_all(vsi);
 	}
+	event = kzalloc(sizeof(*event), GFP_KERNEL);
+	if (!event)
+		return;
+
+	set_bit(IIDC_EVENT_TC_CHANGE, event->type);
+	event->reporter = NULL;
+	ice_setup_dcb_qos_info(pf, &event->info.port_qos);
+	ice_for_each_peer(pf, event, ice_peer_check_for_reg);
+	kfree(event);
 }
 
 /**
drivers/net/ethernet/intel/ice/ice_idc.c +879 -0

921 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_idc_int.h +45 -0

57 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_lib.c +39 -0
index 1456eed1b3b9..f4b4a1dcd643 100644
@@ -1119,6 +1119,30 @@ int ice_add_mac_to_list(struct ice_vsi *vsi, struct list_head *add_list,
 	return 0;
 }
 
+/**
+ * ice_pf_state_is_nominal - checks the PF for nominal state
+ * @pf: pointer to PF to check
+ *
+ * Check the PF's state for a collection of bits that would indicate
+ * the PF is in a state that would inhibit normal operation for
+ * driver functionality.
+ *
+ * Returns true if PF is in a nominal state, false otherwise
+ */
+bool ice_pf_state_is_nominal(struct ice_pf *pf)
+{
+	DECLARE_BITMAP(check_bits, __ICE_STATE_NBITS) = { 0 };
+
+	if (!pf)
+		return false;
+
+	bitmap_set(check_bits, 0, __ICE_STATE_NOMINAL_CHECK_BITS);
+	if (bitmap_intersects(pf->state, check_bits, __ICE_STATE_NBITS))
+		return false;
+
+	return true;
+}
+
 /**
  * ice_update_eth_stats - Update VSI-specific ethernet statistics counters
  * @vsi: the VSI to be updated
@@ -2109,6 +2133,21 @@ void ice_vsi_free_rx_rings(struct ice_vsi *vsi)
  */
 void ice_vsi_close(struct ice_vsi *vsi)
 {
+	enum iidc_close_reason reason = IIDC_REASON_INTERFACE_DOWN;
+
+	if (test_bit(__ICE_CORER_REQ, vsi->back->state))
+		reason = IIDC_REASON_CORER_REQ;
+	if (test_bit(__ICE_GLOBR_REQ, vsi->back->state))
+		reason = IIDC_REASON_GLOBR_REQ;
+	if (test_bit(__ICE_PFR_REQ, vsi->back->state))
+		reason = IIDC_REASON_PFR_REQ;
+	if (!ice_is_safe_mode(vsi->back) && vsi->type == ICE_VSI_PF) {
+		int ret = ice_for_each_peer(vsi->back, &reason, ice_peer_close);
+
+		if (ret)
+			dev_dbg(ice_pf_to_dev(vsi->back), "Peer device did not implement close function\n");
+	}
+
 	if (!test_and_set_bit(__ICE_DOWN, vsi->state))
 		ice_down(vsi);
 
drivers/net/ethernet/intel/ice/ice_lib.h +2 -0
index 502c05c1e84b..1cb8513fcf1f 100644
@@ -14,6 +14,8 @@ ice_add_mac_to_list(struct ice_vsi *vsi, struct list_head *add_list,
 
 void ice_free_fltr_list(struct device *dev, struct list_head *h);
 
+bool ice_pf_state_is_nominal(struct ice_pf *pf);
+
 void ice_update_eth_stats(struct ice_vsi *vsi);
 
 int ice_vsi_cfg_rxqs(struct ice_vsi *vsi);
drivers/net/ethernet/intel/ice/ice_main.c +37 -8

102 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_sched.c +64 -5

102 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_switch.c +27 -0
index b5a53f862a83..990b65aed039 100644
@@ -346,6 +346,10 @@ static void ice_clear_vsi_q_ctx(struct ice_hw *hw, u16 vsi_handle)
 			devm_kfree(ice_hw_to_dev(hw), vsi->lan_q_ctx[i]);
 			vsi->lan_q_ctx[i] = NULL;
 		}
+		if (vsi->rdma_q_ctx[i]) {
+			devm_kfree(ice_hw_to_dev(hw), vsi->rdma_q_ctx[i]);
+			vsi->rdma_q_ctx[i] = NULL;
+		}
 	}
 }
 
@@ -466,6 +470,29 @@ ice_update_vsi(struct ice_hw *hw, u16 vsi_handle, struct ice_vsi_ctx *vsi_ctx,
 	return ice_aq_update_vsi(hw, vsi_ctx, cd);
 }
 
+/**
+ * ice_cfg_iwarp_fltr - enable/disable iWARP filtering on VSI
+ * @hw: pointer to HW struct
+ * @vsi_handle: VSI SW index
+ * @enable: boolean for enable/disable
+ */
+enum ice_status
+ice_cfg_iwarp_fltr(struct ice_hw *hw, u16 vsi_handle, bool enable)
+{
+	struct ice_vsi_ctx *ctx;
+
+	ctx = ice_get_vsi_ctx(hw, vsi_handle);
+	if (!ctx)
+		return ICE_ERR_DOES_NOT_EXIST;
+
+	if (enable)
+		ctx->info.q_opt_flags |= ICE_AQ_VSI_Q_OPT_PE_FLTR_EN;
+	else
+		ctx->info.q_opt_flags &= ~ICE_AQ_VSI_Q_OPT_PE_FLTR_EN;
+
+	return ice_update_vsi(hw, vsi_handle, ctx, NULL);
+}
+
 /**
  * ice_aq_alloc_free_vsi_list
  * @hw: pointer to the HW struct
drivers/net/ethernet/intel/ice/ice_switch.h +4 -0
index fa14b9545dab..acd2f150c30b 100644
@@ -26,6 +26,8 @@ struct ice_vsi_ctx {
 	u8 vf_num;
 	u16 num_lan_q_entries[ICE_MAX_TRAFFIC_CLASS];
 	struct ice_q_ctx *lan_q_ctx[ICE_MAX_TRAFFIC_CLASS];
+	u16 num_rdma_q_entries[ICE_MAX_TRAFFIC_CLASS];
+	struct ice_q_ctx *rdma_q_ctx[ICE_MAX_TRAFFIC_CLASS];
 };
 
 enum ice_sw_fwd_act_type {
@@ -220,6 +222,8 @@ void ice_remove_vsi_fltr(struct ice_hw *hw, u16 vsi_handle);
 enum ice_status
 ice_add_vlan(struct ice_hw *hw, struct list_head *m_list);
 enum ice_status ice_remove_vlan(struct ice_hw *hw, struct list_head *v_list);
+enum ice_status
+ice_cfg_iwarp_fltr(struct ice_hw *hw, u16 vsi_handle, bool enable);
 
 /* Promisc/defport setup for VSIs */
 enum ice_status
drivers/net/ethernet/intel/ice/ice_type.h +3 -0
index e73784e093c5..060dc95b660d 100644
@@ -43,6 +43,7 @@ static inline u32 ice_round_to_num(u32 N, u32 R)
 #define ICE_DBG_LAN		BIT_ULL(8)
 #define ICE_DBG_SW		BIT_ULL(13)
 #define ICE_DBG_SCHED		BIT_ULL(14)
+#define ICE_DBG_RDMA		BIT_ULL(15)
 #define ICE_DBG_PKG		BIT_ULL(16)
 #define ICE_DBG_RES		BIT_ULL(17)
 #define ICE_DBG_AQ_MSG		BIT_ULL(24)
@@ -271,6 +272,7 @@ struct ice_sched_node {
 	u8 tc_num;
 	u8 owner;
 #define ICE_SCHED_NODE_OWNER_LAN	0
+#define ICE_SCHED_NODE_OWNER_RDMA	2
 };
 
 /* Access Macros for Tx Sched Elements data */
@@ -342,6 +344,7 @@ struct ice_sched_vsi_info {
 	struct ice_sched_node *ag_node[ICE_MAX_TRAFFIC_CLASS];
 	struct list_head list_entry;
 	u16 max_lanq[ICE_MAX_TRAFFIC_CLASS];
+	u16 max_rdmaq[ICE_MAX_TRAFFIC_CLASS];
 };
 
 /* driver defines the policy */
drivers/net/ethernet/intel/ice/ice_virtchnl_pf.c +0 -25
index edb374296d1f..0004117bc192 100644
@@ -1420,31 +1420,6 @@ static int ice_alloc_vfs(struct ice_pf *pf, u16 num_alloc_vfs)
 	return ret;
 }
 
-/**
- * ice_pf_state_is_nominal - checks the PF for nominal state
- * @pf: pointer to PF to check
- *
- * Check the PF's state for a collection of bits that would indicate
- * the PF is in a state that would inhibit normal operation for
- * driver functionality.
- *
- * Returns true if PF is in a nominal state.
- * Returns false otherwise
- */
-static bool ice_pf_state_is_nominal(struct ice_pf *pf)
-{
-	DECLARE_BITMAP(check_bits, __ICE_STATE_NBITS) = { 0 };
-
-	if (!pf)
-		return false;
-
-	bitmap_set(check_bits, 0, __ICE_STATE_NOMINAL_CHECK_BITS);
-	if (bitmap_intersects(pf->state, check_bits, __ICE_STATE_NBITS))
-		return false;
-
-	return true;
-}
-
 /**
  * ice_pci_sriov_ena - Enable or change number of VFs
  * @pf: pointer to the PF structure
Signature · 1 line
2.23.0
#10 lore raw

PATCH v3 4/20 i40e: Register a virtbus device to provide RDMA

In reply to Jeff Kirsher

From: Shiraz Saleem <[email protected]>

Register client virtbus device on the virtbus for the RDMA virtbus driver (irdma) to bind to. It allows to realize a single RDMA driver capable of working with multiple netdev drivers over multi-generation Intel HW supporting RDMA. There is also no load ordering dependencies between i40e and irdma.

Summary of changes:
* Support to add/remove virtbus devices
* Add 2 new client ops.
* i40e_client_device_register() which is called during RDMA probe() per PF. Validate client drv OPs and schedule service task to call open()
* i40e_client_device_unregister() called during RDMA remove() per PF. Call client close() and release_qvlist.
* The global register/unregister calls exported for i40iw are retained until i40iw is removed from the kernel.

---
 drivers/infiniband/hw/i40iw/Makefile          |   1 -
 drivers/infiniband/hw/i40iw/i40iw.h           |   2 +-
 drivers/net/ethernet/intel/Kconfig            |   1 +
 drivers/net/ethernet/intel/i40e/i40e.h        |   3 +-
 drivers/net/ethernet/intel/i40e/i40e_client.c | 109 +++++++++++++++---
 .../linux/net/intel}/i40e_client.h            |  20 +++-
 6 files changed, 112 insertions(+), 24 deletions(-)
 rename {drivers/net/ethernet/intel/i40e => include/linux/net/intel}/i40e_client.h (92%)
drivers/infiniband/hw/i40iw/Makefile +0 -1
index 8942f8229945..34da9eba8a7c 100644
@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0
-ccflags-y :=  -I $(srctree)/drivers/net/ethernet/intel/i40e
 
 obj-$(CONFIG_INFINIBAND_I40IW) += i40iw.o
 
drivers/infiniband/hw/i40iw/i40iw.h +1 -1
index 8feec35f95a7..3197e3536d5c 100644
@@ -57,7 +57,7 @@
 #include "i40iw_d.h"
 #include "i40iw_hmc.h"
 
-#include <i40e_client.h>
+#include <linux/net/intel/i40e_client.h>
 #include "i40iw_type.h"
 #include "i40iw_p.h"
 #include <rdma/i40iw-abi.h>
drivers/net/ethernet/intel/Kconfig +1 -0
index b88328fea1d0..8595f578fbe7 100644
@@ -241,6 +241,7 @@ config I40E
 	tristate "Intel(R) Ethernet Controller XL710 Family support"
 	imply PTP_1588_CLOCK
 	depends on PCI
+	select VIRTUAL_BUS
 	---help---
 	  This driver supports Intel(R) Ethernet Controller XL710 Family of
 	  devices.  For more information on how to identify your adapter, go
drivers/net/ethernet/intel/i40e/i40e.h +2 -1
index cb6367334ca7..4321e81d347c 100644
@@ -38,7 +38,7 @@
 #include <net/xdp_sock.h>
 #include "i40e_type.h"
 #include "i40e_prototype.h"
-#include "i40e_client.h"
+#include <linux/net/intel/i40e_client.h>
 #include <linux/avf/virtchnl.h>
 #include "i40e_virtchnl_pf.h"
 #include "i40e_txrx.h"
@@ -655,6 +655,7 @@ struct i40e_pf {
 	u16 last_sw_conf_valid_flags;
 	/* List to keep previous DDP profiles to be rolled back in the future */
 	struct list_head ddp_old_prof;
+	int peer_idx;
 };
 
 /**
drivers/net/ethernet/intel/i40e/i40e_client.c +93 -16

182 lines not on this page · raw message

include/linux/net/intel/i40e_client.h renamed from drivers/net/ethernet/intel/i40e/i40e_client.h +15 -5
similarity index 92%
rename from drivers/net/ethernet/intel/i40e/i40e_client.h
rename to include/linux/net/intel/i40e_client.h
index 72994baf4941..1ca9c1eae60e 100644
@@ -4,6 +4,9 @@
 #ifndef _I40E_CLIENT_H_
 #define _I40E_CLIENT_H_
 
+#include <linux/virtual_bus.h>
+
+#define I40E_PEER_RDMA_NAME	"i40e_rdma"
 #define I40E_CLIENT_STR_LENGTH 10
 
 /* Client interface version should be updated anytime there is a change in the
@@ -41,8 +44,8 @@ struct i40e_client;
  * In order for us to keep the interface simple, SW will define a
  * unique type value for AEQ.
  */
-#define I40E_QUEUE_TYPE_PE_AEQ  0x80
-#define I40E_QUEUE_INVALID_IDX	0xFFFF
+#define I40E_QUEUE_TYPE_PE_AEQ 0x80
+#define I40E_QUEUE_INVALID_IDX 0xFFFF
 
 struct i40e_qv_info {
 	u32 v_idx; /* msix_vector */
@@ -67,7 +70,7 @@ struct i40e_prio_qos_params {
 	u8 reserved;
 };
 
-#define I40E_CLIENT_MAX_USER_PRIORITY        8
+#define I40E_CLIENT_MAX_USER_PRIORITY 8
 /* Struct to hold Client QoS */
 struct i40e_qos_params {
 	struct i40e_prio_qos_params prio_qos[I40E_CLIENT_MAX_USER_PRIORITY];
@@ -80,12 +83,13 @@ struct i40e_params {
 
 /* Structure to hold Lan device info for a client device */
 struct i40e_info {
+	struct virtbus_device vdev;
 	struct i40e_client_version version;
 	u8 lanmac[6];
 	struct net_device *netdev;
 	struct pci_dev *pcidev;
 	u8 __iomem *hw_addr;
-	u8 fid;	/* function id, PF id or VF id */
+	u8 fid; /* function id, PF id or VF id */
 #define I40E_CLIENT_FTYPE_PF 0
 #define I40E_CLIENT_FTYPE_VF 1
 	u8 ftype; /* function type, PF or VF */
@@ -97,8 +101,9 @@ struct i40e_info {
 	struct i40e_qvlist_info *qvlist_info;
 	struct i40e_params params;
 	struct i40e_ops *ops;
+	struct i40e_client *client;
 
-	u16 msix_count;	 /* number of msix vectors*/
+	u16 msix_count;  /* number of msix vectors*/
 	/* Array down below will be dynamically allocated based on msix_count */
 	struct msix_entry *msix_entries;
 	u16 itr_index; /* Which ITR index the PE driver is suppose to use */
@@ -132,6 +137,11 @@ struct i40e_ops {
 			       struct i40e_client *client,
 			       bool is_vf, u32 vf_id,
 			       u32 flag, u32 valid_flag);
+
+	int (*client_device_register)(struct i40e_info *ldev);
+
+	void (*client_device_unregister)(struct i40e_info *ldev);
+
 };
 
 struct i40e_client_ops {
Signature · 1 line
2.23.0
#11 lore raw

Re: [PATCH v3 04/20] i40e: Register a virtbus device to provide RDMA

drivers/infiniband/hw/i40iw/Makefile line 2

Quoted 5 lines · Jeff Kirsher
drivers/infiniband/hw/i40iw/Makefile +0 -1
@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0
-ccflags-y :=  -I $(srctree)/drivers/net/ethernet/intel/i40e

Isn't dropping this cflags good for a nice simple separate patch to keep the #include mess from cluttering up a patch that is trying to add new functionality?

#12 lore raw

Re: [PATCH v3 04/20] i40e: Register a virtbus device to provide RDMA

In reply to Jeff Kirsher
Quoted 129 lines · Jeff Kirsher

From: Shiraz Saleem <[email protected]>

Register client virtbus device on the virtbus for the RDMA virtbus driver (irdma) to bind to. It allows to realize a single RDMA driver capable of working with multiple netdev drivers over multi-generation Intel HW supporting RDMA. There is also no load ordering dependencies between i40e and irdma.

Summary of changes:
* Support to add/remove virtbus devices
* Add 2 new client ops.
* i40e_client_device_register() which is called during RDMA probe() per PF. Validate client drv OPs and schedule service task to call open()
* i40e_client_device_unregister() called during RDMA remove() per PF. Call client close() and release_qvlist.
* The global register/unregister calls exported for i40iw are retained until i40iw is removed from the kernel.

---
 drivers/infiniband/hw/i40iw/Makefile          |   1 -
 drivers/infiniband/hw/i40iw/i40iw.h           |   2 +-
 drivers/net/ethernet/intel/Kconfig            |   1 +
 drivers/net/ethernet/intel/i40e/i40e.h        |   3 +-
 drivers/net/ethernet/intel/i40e/i40e_client.c | 109 +++++++++++++++---
 .../linux/net/intel}/i40e_client.h            |  20 +++-
 6 files changed, 112 insertions(+), 24 deletions(-)
 rename {drivers/net/ethernet/intel/i40e => include/linux/net/intel}/i40e_client.h (92%)
drivers/infiniband/hw/i40iw/Makefile +0 -1
index 8942f8229945..34da9eba8a7c 100644
@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0
-ccflags-y :=  -I $(srctree)/drivers/net/ethernet/intel/i40e
 
 obj-$(CONFIG_INFINIBAND_I40IW) += i40iw.o
 
drivers/infiniband/hw/i40iw/i40iw.h +1 -1
index 8feec35f95a7..3197e3536d5c 100644
@@ -57,7 +57,7 @@
 #include "i40iw_d.h"
 #include "i40iw_hmc.h"
 
-#include <i40e_client.h>
+#include <linux/net/intel/i40e_client.h>
 #include "i40iw_type.h"
 #include "i40iw_p.h"
 #include <rdma/i40iw-abi.h>
drivers/net/ethernet/intel/Kconfig +1 -0
index b88328fea1d0..8595f578fbe7 100644
@@ -241,6 +241,7 @@ config I40E
 	tristate "Intel(R) Ethernet Controller XL710 Family support"
 	imply PTP_1588_CLOCK
 	depends on PCI
+	select VIRTUAL_BUS
 	---help---
 	  This driver supports Intel(R) Ethernet Controller XL710 Family of
 	  devices.  For more information on how to identify your adapter, go
drivers/net/ethernet/intel/i40e/i40e.h +2 -1
index cb6367334ca7..4321e81d347c 100644
@@ -38,7 +38,7 @@
 #include <net/xdp_sock.h>
 #include "i40e_type.h"
 #include "i40e_prototype.h"
-#include "i40e_client.h"
+#include <linux/net/intel/i40e_client.h>
 #include <linux/avf/virtchnl.h>
 #include "i40e_virtchnl_pf.h"
 #include "i40e_txrx.h"
@@ -655,6 +655,7 @@ struct i40e_pf {
 	u16 last_sw_conf_valid_flags;
 	/* List to keep previous DDP profiles to be rolled back in the future */
 	struct list_head ddp_old_prof;
+	int peer_idx;
 };
 
 /**
drivers/net/ethernet/intel/i40e/i40e_client.c +7 -1
index e81530ca08d0..a3dee729719b 100644
@@ -1,12 +1,12 @@
 // SPDX-License-Identifier: GPL-2.0
 /* Copyright(c) 2013 - 2018 Intel Corporation. */
 
+#include <linux/net/intel/i40e_client.h>
 #include <linux/list.h>
 #include <linux/errno.h>
 
 #include "i40e.h"
 #include "i40e_prototype.h"
-#include "i40e_client.h"
 
 static const char i40e_client_interface_version_str[] = I40E_CLIENT_VERSION_STR;
 static struct i40e_client *registered_client;
@@ -30,11 +30,17 @@ static int i40e_client_update_vsi_ctxt(struct i40e_info *ldev,
 				       bool is_vf, u32 vf_id,
 				       u32 flag, u32 valid_flag);
 
+static int i40e_client_device_register(struct i40e_info *ldev);
+
+static void i40e_client_device_unregister(struct i40e_info *ldev);
+
 static struct i40e_ops i40e_lan_ops = {
 	.virtchnl_send = i40e_client_virtchnl_send,
 	.setup_qvlist = i40e_client_setup_qvlist,
 	.request_reset = i40e_client_request_reset,
 	.update_vsi_ctxt = i40e_client_update_vsi_ctxt,
+	.client_device_register = i40e_client_device_register,
+	.client_device_unregister = i40e_client_device_unregister,
 };
 
 /**
@@ -275,6 +281,27 @@ void i40e_client_update_msix_info(struct i40e_pf *pf)
 	cdev->lan_info.msix_entries = &pf->msix_entries[pf->iwarp_base_vector];
 }
 

drivers/net/ethernet/intel/i40e/i40e_client.c void i40e_client_update_msix_info(struct i40e_pf *pf) · line 293

Quoted 10 lines
drivers/net/ethernet/intel/i40e/i40e_client.c +10 -0
+static int i40e_init_client_virtdev(struct i40e_pf *pf)
+{
+	struct i40e_info *ldev = &pf->cinst->lan_info;
+	struct pci_dev *pdev = pf->pdev;
+	struct virtbus_device *vdev;
+	int ret;
+
+	vdev = &ldev->vdev;
+	vdev->name = I40E_PEER_RDMA_NAME;
+	vdev->dev.parent = &pf->pdev->dev;

What a total and complete mess of a tangled web you just wove here.

Ok, so you pass in a single pointer, that then dereferences 3 pointers deep to find the pointer to the virtbus_device structure, but then you point the parent of that device, back at the original structure's sub-pointer's device itself.

WTF?

And who owns the memory of this thing that is supposed to be dynamically controlled by something OUTSIDE of this driver? Who created that thing 3 pointers deep? What happens when you leak the memory below (hint, you did), and who is supposed to clean it up if you need to properly clean it up if something bad happens?

drivers/net/ethernet/intel/i40e/i40e_client.c void i40e_client_update_msix_info(struct i40e_pf *pf) · line 298

Quoted 5 lines
drivers/net/ethernet/intel/i40e/i40e_client.c +5 -0
+
+	ret = virtbus_dev_register(vdev);
+	if (ret) {
+		dev_err(&pdev->dev, "Failure adding client virtbus dev %s %d\n",
+			I40E_PEER_RDMA_NAME, ret);

Again, the core should handle this, right?

drivers/net/ethernet/intel/i40e/i40e_client.c void i40e_client_update_msix_info(struct i40e_pf *pf) · line 299

Quoted 1 line
drivers/net/ethernet/intel/i40e/i40e_client.c +1 -0
+		return ret;

Did you just leak memory?

Yup, you did, you never actually checked the return value of this function :(

Ugh.

I feel like the virtual bus code is getting better, but this use of the code, um, no, not ok.

Either way, this series is NOT ready to be merged anywhere, please do not try to rush things.

Also, what ever happened to my "YOU ALL MUST AGREE TO WORK TOGETHER" requirement between this group, and the other group trying to do the same thing? I want to see signed-off-by from EVERYONE involved before we are going to consider this thing.

thanks,

greg k-h

#13 lore raw

RE: [PATCH v3 04/20] i40e: Register a virtbus device to provide RDMA

Quoted 2 lines

Subject: Re: [PATCH v3 04/20] i40e: Register a virtbus device to provide RDMA

[....]

Quoted 18 lines
Quoted 5 lines

/**

drivers/net/ethernet/intel/i40e/i40e_client.c +0 -0
@@ -275,6 +281,27 @@ void i40e_client_update_msix_info(struct i40e_pf *pf)
 	cdev->lan_info.msix_entries =
 &pf->msix_entries[pf->iwarp_base_vector];
 }

drivers/net/ethernet/intel/i40e/i40e_client.c void i40e_client_update_msix_info(struct i40e_pf *pf) · line 293

Quoted 10 lines
drivers/net/ethernet/intel/i40e/i40e_client.c +9 -0
 
+static int i40e_init_client_virtdev(struct i40e_pf *pf) {
+	struct i40e_info *ldev = &pf->cinst->lan_info;
+	struct pci_dev *pdev = pf->pdev;
+	struct virtbus_device *vdev;
+	int ret;
+
+	vdev = &ldev->vdev;
+	vdev->name = I40E_PEER_RDMA_NAME;
+	vdev->dev.parent = &pf->pdev->dev;

What a total and complete mess of a tangled web you just wove here.

Quoted 5 lines

Ok, so you pass in a single pointer, that then dereferences 3 pointers deep to find the pointer to the virtbus_device structure, but then you point the parent of that device, back at the original structure's sub-pointer's device itself.

WTF?

OK. This is convoluted. Passing a pointer to the i40e_info object should suffice. So something like,

+static int i40e_init_client_virtdev(struct i40e_info *ldev) {

+       struct pci_dev *pdev = ldev->pcidev;
+       struct virtbus_device *vdev = &ldev->vdev;
+       int ret;
+
+       vdev->name = I40E_PEER_RDMA_NAME;
+       vdev->dev.parent = &pdev->dev;
+       ret = virtbus_dev_register(vdev);
+       if (ret)

+ return ret;
+
+ return 0;
+}
+

Quoted 6 lines

And who owns the memory of this thing that is supposed to be dynamically controlled by something OUTSIDE of this driver? Who created that thing 3 pointers deep? What happens when you leak the memory below (hint, you did), and who is supposed to clean it up if you need to properly clean it up if something bad happens?

The i40e_info object memory is tied to the PF driver.

The object hierarchy is,

i40e_pf: pointer to i40e_client_instance

	----- i40e_client_instance: i40e_info
		----- i40e_info: virtbus_device

For each PF, there is a client_instance object allocated.
The i40e_info object is populated and the virtbus_device hanging off this object is registered. In irdma probe(), we use the container_of macro to get to this i40e_info object from the virtbus_device. It contains all the ops/info which RDMA driver needs from the PCI function driver.

The lifetime of the i40e_info object (and the virtbus device) is tied to the PF. When PF goes away, virtbus_device is unregistered and the client_instance object memory is freed.

Quoted 8 lines

drivers/net/ethernet/intel/i40e/i40e_client.c void i40e_client_update_msix_info(struct i40e_pf *pf) · line 298

Quoted 6 lines
drivers/net/ethernet/intel/i40e/i40e_client.c +5 -0
+
+	ret = virtbus_dev_register(vdev);
+	if (ret) {
+		dev_err(&pdev->dev, "Failure adding client virtbus dev %s %d\n",
+			I40E_PEER_RDMA_NAME, ret);

Again, the core should handle this, right?

Right. Will fix.

Quoted 4 lines

drivers/net/ethernet/intel/i40e/i40e_client.c void i40e_client_update_msix_info(struct i40e_pf *pf) · line 299

Quoted 2 lines
drivers/net/ethernet/intel/i40e/i40e_client.c +1 -0
+		return ret;

Did you just leak memory?

Thanks! Will fix.

Quoted 5 lines

Also, what ever happened to my "YOU ALL MUST AGREE TO WORK TOGETHER" requirement between this group, and the other group trying to do the same thing? I want to see signed-off-by from EVERYONE involved before we are going to consider this thing.

We will have all parties cc'ed in the next submission. Would encourage folks to review and hopefully we can get some consensus.

#14 lore raw

Re: [PATCH v3 04/20] i40e: Register a virtbus device to provide RDMA

Quoted 47 lines · Shiraz
Quoted 2 lines

Subject: Re: [PATCH v3 04/20] i40e: Register a virtbus device to provide RDMA

[....]

Quoted 18 lines
Quoted 5 lines

/**

drivers/net/ethernet/intel/i40e/i40e_client.c +0 -0
@@ -275,6 +281,27 @@ void i40e_client_update_msix_info(struct i40e_pf *pf)
 	cdev->lan_info.msix_entries =
 &pf->msix_entries[pf->iwarp_base_vector];
 }

drivers/net/ethernet/intel/i40e/i40e_client.c void i40e_client_update_msix_info(struct i40e_pf *pf) · line 293

Quoted 10 lines
drivers/net/ethernet/intel/i40e/i40e_client.c +9 -0
 
+static int i40e_init_client_virtdev(struct i40e_pf *pf) {
+	struct i40e_info *ldev = &pf->cinst->lan_info;
+	struct pci_dev *pdev = pf->pdev;
+	struct virtbus_device *vdev;
+	int ret;
+
+	vdev = &ldev->vdev;
+	vdev->name = I40E_PEER_RDMA_NAME;
+	vdev->dev.parent = &pf->pdev->dev;

What a total and complete mess of a tangled web you just wove here.

Quoted 5 lines

Ok, so you pass in a single pointer, that then dereferences 3 pointers deep to find the pointer to the virtbus_device structure, but then you point the parent of that device, back at the original structure's sub-pointer's device itself.

WTF?

OK. This is convoluted. Passing a pointer to the i40e_info object should suffice. So something like,

drivers/net/ethernet/intel/i40e/i40e_client.c +14 -0
+static int i40e_init_client_virtdev(struct i40e_info *ldev) {
+       struct pci_dev *pdev = ldev->pcidev;
+       struct virtbus_device *vdev = &ldev->vdev;
+       int ret;
+
+       vdev->name = I40E_PEER_RDMA_NAME;
+       vdev->dev.parent = &pdev->dev;
+       ret = virtbus_dev_register(vdev);
+       if (ret)
+               return ret;
+
+       return 0;
+}
+
Quoted 2 lines
Quoted 7 lines
Quoted 5 lines

And who owns the memory of this thing that is supposed to be dynamically controlled by something OUTSIDE of this driver? Who created that thing 3 pointers deep? What happens when you leak the memory below (hint, you did), and who is supposed to clean it up if you need to properly clean it up if something bad happens?

The i40e_info object memory is tied to the PF driver.

What is a "PF"?

Quoted 5 lines

The object hierarchy is,

i40e_pf: pointer to i40e_client_instance

	----- i40e_client_instance: i40e_info
		----- i40e_info: virtbus_device

So you are 3 pointers deep to get a structure that is dynamically controlled? Why are those "3 pointers" not also represented in sysfs? You have a heiarchy within the kernel that is not being represented that way to userspace, why?

Hint, I think this is totally wrong, you need to rework this to be sane.

Quoted 1 line

For each PF, there is a client_instance object allocated.

Great, make it dynamic and in the device tree.

Quoted 1 line

The i40e_info object is populated and the virtbus_device hanging off this object is registered.

Great, make that dynamic and inthe device tree.

If you think this is too much, then your whole mess here is too much and needs to be made a lot simpler.

Quoted 2 lines

In irdma probe(), we use the container_of macro to get to this i40e_info object from the virtbus_device. It contains all the ops/info which RDMA driver needs from the PCI function driver.

Ok, that's what you are supposed to do, but why walk back 3 levels?

Quoted 1 line

The lifetime of the i40e_info object (and the virtbus device) is tied to the PF.

Careful, these are dynamic structures, you don't get to fully "tie" anything here.

Quoted 2 lines

When PF goes away, virtbus_device is unregistered and the client_instance object memory is freed.

Hopefully, if no one else has a reference (hint, you never know this.) If you are relying on this, then you are not using the driver model correctly.

Quoted 7 lines
Quoted 5 lines

Also, what ever happened to my "YOU ALL MUST AGREE TO WORK TOGETHER" requirement between this group, and the other group trying to do the same thing? I want to see signed-off-by from EVERYONE involved before we are going to consider this thing.

We will have all parties cc'ed in the next submission. Would encourage folks to review and hopefully we can get some consensus.

Then when you post your patches, do so to be obvious that is what you are asking for, don't try to do a "here's a pull request to take!" type request like you did here, that's not nice.

thanks,

greg k-h

#15 lore raw

RE: [PATCH v3 04/20] i40e: Register a virtbus device to provide RDMA

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

Subject: Re: [PATCH v3 04/20] i40e: Register a virtbus device to provide RDMA

[.....]

Quoted 10 lines
Quoted 8 lines
Quoted 6 lines

And who owns the memory of this thing that is supposed to be dynamically controlled by something OUTSIDE of this driver? Who created that thing 3 pointers deep? What happens when you leak the memory below (hint, you did), and who is supposed to clean it up if you need to properly clean it up if something bad happens?

The i40e_info object memory is tied to the PF driver.

What is a "PF"?

physical function.

Quoted 18 lines
Quoted 6 lines

The object hierarchy is,

i40e_pf: pointer to i40e_client_instance

	----- i40e_client_instance: i40e_info
		----- i40e_info: virtbus_device

So you are 3 pointers deep to get a structure that is dynamically controlled? Why are those "3 pointers" not also represented in sysfs?
You have a heiarchy within the kernel that is not being represented that way to userspace, why?

Hint, I think this is totally wrong, you need to rework this to be sane.

Quoted 1 line

For each PF, there is a client_instance object allocated.

Great, make it dynamic and in the device tree.

Quoted 8 lines
Quoted 1 line

The i40e_info object is populated and the virtbus_device hanging off this object

is registered.

Great, make that dynamic and inthe device tree.

If you think this is too much, then your whole mess here is too much and needs to be made a lot simpler.

I think we can decouple the virtbus_device object from i40e_info object.

Instead allocate a i40e_virtbus_device object which contains the virtbus_device and a pointer to i40e_info object for the RDMA driver to consume on probe(). Register it the virtbus, and provide a release callback to free up its memory.

Sending a patch snippet to hopefully make it clearer.

drivers/net/ethernet/intel/i40e/i40e_client.c +47 -6
index b80ffaf..c6cf2eb 100644
@@ -281,6 +281,42 @@ void i40e_client_update_msix_info(struct i40e_pf *pf)
 	cdev->lan_info.msix_entries = &pf->msix_entries[pf->iwarp_base_vector];
 }
 
+static void i40e_virtdev_release(struct device *dev)
+{
+	struct virtbus_device *vdev = container_of(dev, struct virtbus_device, dev);
+	struct i40e_virtbus_device *i40e_vdev =
+		container_of(vdev, struct i40e_virtbus_device, vdev);
+
+	kfree(i40e_vdev);
+}
+
+static int i40e_init_client_virtdev(struct i40e_info *ldev)
+{
+	struct pci_dev *pdev = ldev->pcidev;
+	struct i40e_virtbus_device *i40e_vdev;
+	int ret;
+
+	i40e_vdev = kzalloc(sizeof(*i40e_vdev), GFP_KERNEL);
+	if (!i40e_vdev)
+		return -ENOMEM;
+
+	i40e_vdev->vdev.name = I40E_PEER_RDMA_NAME;
+	i40e_vdev->vdev.dev.parent = &pdev->dev;
+	i40e_vdev->vdev.dev.release = i40e_virtdev_release;
+	i40e_vdev->ldev = ldev;
+
+	ret = virtbus_dev_register(&i40e_vdev->vdev);
+	if (ret)
+		/* TBD: Assuming virtbus_dev_register() does put_device
+		 * on err clean-up
+		 */
+		return ret;
+
+	ldev->vdev = &i40e_vdev->vdev;
+
+	return 0;
+}
+
 /**
  * i40e_client_add_instance - add a client instance struct to the instance list
  * @pf: pointer to the board struct
@@ -313,11 +349,8 @@ static void i40e_client_add_instance(struct i40e_pf *pf)
 	cdev->lan_info.fw_build = pf->hw.aq.fw_build;
 	set_bit(__I40E_CLIENT_INSTANCE_NONE, &cdev->state);
 
-	if (i40e_client_get_params(vsi, &cdev->lan_info.params)) {
-		kfree(cdev);
-		cdev = NULL;
-		return;
-	}
+	if (i40e_client_get_params(vsi, &cdev->lan_info.params))
+		goto free_cdev;
 
 	mac = list_first_entry(&cdev->lan_info.netdev->dev_addrs.list,
 			       struct netdev_hw_addr, list);
@@ -332,6 +365,14 @@ static void i40e_client_add_instance(struct i40e_pf *pf)
 	cdev->lan_info.msix_count = pf->num_iwarp_msix;
 	cdev->lan_info.msix_entries = &pf->msix_entries[pf->iwarp_base_vector];
 
+	if (i40e_init_client_virtdev(&cdev->lan_info))
+		goto free_cdev;
+
+	return;
+
+free_cdev:
+	kfree(cdev);
+	cdev = NULL;
 }
 
 /**
@@ -452,7 +493,7 @@ int i40e_lan_del_device(struct i40e_pf *pf)
 	struct i40e_device *ldev, *tmp;
 	int ret = -ENODEV;
 
-	virtbus_dev_unregister(&pf->cinst->lan_info.vdev);
+	virtbus_dev_unregister(pf->cinst->lan_info.vdev);
 
 	/* First, remove any client instance. */
 	i40e_client_del_instance(pf);
include/linux/net/intel/i40e_client.h +6 -0
index 7e147d3..5c81261 100644
@@ -83,11 +83,11 @@ struct i40e_params {
 
 /* Structure to hold Lan device info for a client device */
 struct i40e_info {
 	struct i40e_client_version version;
 	u8 lanmac[6];
 	struct net_device *netdev;
 	struct pci_dev *pcidev;
+	struct virtbus_device *vdev;
 	u8 __iomem *hw_addr;
 	u8 fid;	/* function id, PF id or VF id */
 #define I40E_CLIENT_FTYPE_PF 0
@@ -112,6 +112,11 @@ struct i40e_info {
 	u32 fw_build;                   /* firmware build number */
 };
 
+struct i40e_virtbus_device {
+	struct virtbus_device vdev;
+	struct i40e_info *ldev;
+};
+
 

Is this more in line with correct usage and what you're expecting?

Shiraz

#16 lore raw

Re: [PATCH v3 04/20] i40e: Register a virtbus device to provide RDMA

include/linux/net/intel/i40e_client.h struct i40e_params { · line 86

Quoted 8 lines · Shiraz
include/linux/net/intel/i40e_client.h +1 -0
index 7e147d3..5c81261 100644
+++ b/include/linux/net/intel/i40e_client.h
@@ -83,11 +83,11 @@ struct i40e_params {
 
 /* Structure to hold Lan device info for a client device */
 struct i40e_info {
 	struct i40e_client_version version;

I hope this isn't the inter-module versioning stuff we already Nak'd?

include/linux/net/intel/i40e_client.h struct i40e_params { · line 87

Quoted 1 line
include/linux/net/intel/i40e_client.h +0 -0
 	u8 lanmac[6];

Is this different from the mac reachable from the netdev?

include/linux/net/intel/i40e_client.h struct i40e_params { · line 90

Quoted 3 lines
include/linux/net/intel/i40e_client.h +1 -0
 	struct net_device *netdev;
 	struct pci_dev *pcidev;
+	struct virtbus_device *vdev;

If there is only one of these per virtbus_device then why do we need to split the structure?

include/linux/net/intel/i40e_client.h struct i40e_info { · line 117

Quoted 10 lines
include/linux/net/intel/i40e_client.h +3 -0
 	u8 __iomem *hw_addr;
 	u8 fid;	/* function id, PF id or VF id */
 #define I40E_CLIENT_FTYPE_PF 0
@@ -112,6 +112,11 @@ struct i40e_info {
 	u32 fw_build;                   /* firmware build number */
 };
 
+struct i40e_virtbus_device {
+	struct virtbus_device vdev;
+	struct i40e_info *ldev;

Is the lifetime actually any better? Will ldev be freed and left danling before virtbus_device is released?

Jason

#17 lore raw

RE: [PATCH v3 04/20] i40e: Register a virtbus device to provide RDMA

Quoted 9 lines

Subject: Re: [PATCH v3 04/20] i40e: Register a virtbus device to provide RDMA

Quoted 6 lines · Shiraz
include/linux/net/intel/i40e_client.h +1 -0
index 7e147d3..5c81261 100644
+++ b/include/linux/net/intel/i40e_client.h
@@ -83,11 +83,11 @@ struct i40e_params {
 
Quoted 5 lines

include/linux/net/intel/i40e_client.h struct i40e_params { · line 86

Quoted 3 lines
include/linux/net/intel/i40e_client.h +0 -0
 /* Structure to hold Lan device info for a client device */  struct
 i40e_info {
 	struct i40e_client_version version;

I hope this isn't the inter-module versioning stuff we already Nak'd?

This is not used in irdma for checking any versioning.

This and the version in struct i40e_client are not cleaned up because they are used between i40e <--> i40iw. And the thought was once i40iw is removed, we can send a follow on patch to remove this versioning + the global
exported functions i40e_register_client() and i40e_unregister_client() used by i40iw.

For the ice driver, we have already removed all versioning.

Quoted 4 lines

include/linux/net/intel/i40e_client.h struct i40e_params { · line 87

Quoted 2 lines
include/linux/net/intel/i40e_client.h +0 -0
 	u8 lanmac[6];

Is this different from the mac reachable from the netdev?

Hmmm... I think not. This should be possible to get from netdev. We can send a cleanup follow on patch.

Quoted 7 lines

include/linux/net/intel/i40e_client.h struct i40e_info { · line 117

Quoted 3 lines
include/linux/net/intel/i40e_client.h +3 -0
+struct i40e_virtbus_device {
+	struct virtbus_device vdev;
+	struct i40e_info *ldev;

Is the lifetime actually any better? Will ldev be freed and left danling before virtbus_device is released?

The consumer of the ldev (i.e the rdma driver) will have a valid ldev when drv->remove() is called as part of the virtbus device unregister.
It is probably good to set this ldev pointer inside the i40e_virtbus_device object to NULL in the netdev driver soon after virtbus device unregister.

#18 lore raw

Re: [PATCH v3 04/20] i40e: Register a virtbus device to provide RDMA

In reply to Saleem, Shiraz
Quoted 44 lines · Shiraz
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

Subject: Re: [PATCH v3 04/20] i40e: Register a virtbus device to provide RDMA

[.....]

Quoted 10 lines
Quoted 8 lines
Quoted 6 lines

And who owns the memory of this thing that is supposed to be dynamically controlled by something OUTSIDE of this driver? Who created that thing 3 pointers deep? What happens when you leak the memory below (hint, you did), and who is supposed to clean it up if you need to properly clean it up if something bad happens?

The i40e_info object memory is tied to the PF driver.

What is a "PF"?

physical function.

Quoted 18 lines
Quoted 6 lines

The object hierarchy is,

i40e_pf: pointer to i40e_client_instance

	----- i40e_client_instance: i40e_info
		----- i40e_info: virtbus_device

So you are 3 pointers deep to get a structure that is dynamically controlled? Why are those "3 pointers" not also represented in sysfs?
You have a heiarchy within the kernel that is not being represented that way to userspace, why?

Hint, I think this is totally wrong, you need to rework this to be sane.

Quoted 1 line

For each PF, there is a client_instance object allocated.

Great, make it dynamic and in the device tree.

Quoted 8 lines
Quoted 1 line

The i40e_info object is populated and the virtbus_device hanging off this object

is registered.

Great, make that dynamic and inthe device tree.

If you think this is too much, then your whole mess here is too much and needs to be made a lot simpler.

Quoted 8 lines

I think we can decouple the virtbus_device object from i40e_info object.

Instead allocate a i40e_virtbus_device object which contains the virtbus_device and a pointer to i40e_info object for the RDMA driver to consume on probe(). Register it the virtbus, and provide a release callback to free up its memory.

Sending a patch snippet to hopefully make it clearer.

Yes, this looks a little bit more sane.

greg k-h

#19 lore raw

Re: [PATCH v3 04/20] i40e: Register a virtbus device to provide RDMA

In reply to Greg KH

Hi Greg,

Quoted 176 lines · Greg KH
Quoted 129 lines · Jeff Kirsher

From: Shiraz Saleem <[email protected]>

Register client virtbus device on the virtbus for the RDMA virtbus driver (irdma) to bind to. It allows to realize a single RDMA driver capable of working with multiple netdev drivers over multi-generation Intel HW supporting RDMA. There is also no load ordering dependencies between i40e and irdma.

Summary of changes:
* Support to add/remove virtbus devices
* Add 2 new client ops.
* i40e_client_device_register() which is called during RDMA probe() per PF. Validate client drv OPs and schedule service task to call open()
* i40e_client_device_unregister() called during RDMA remove() per PF. Call client close() and release_qvlist.
* The global register/unregister calls exported for i40iw are retained until i40iw is removed from the kernel.

---
 drivers/infiniband/hw/i40iw/Makefile          |   1 -
 drivers/infiniband/hw/i40iw/i40iw.h           |   2 +-
 drivers/net/ethernet/intel/Kconfig            |   1 +
 drivers/net/ethernet/intel/i40e/i40e.h        |   3 +-
 drivers/net/ethernet/intel/i40e/i40e_client.c | 109 +++++++++++++++---
 .../linux/net/intel}/i40e_client.h            |  20 +++-
 6 files changed, 112 insertions(+), 24 deletions(-)
 rename {drivers/net/ethernet/intel/i40e => include/linux/net/intel}/i40e_client.h (92%)
drivers/infiniband/hw/i40iw/Makefile +0 -1
index 8942f8229945..34da9eba8a7c 100644
@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0
-ccflags-y :=  -I $(srctree)/drivers/net/ethernet/intel/i40e
 
 obj-$(CONFIG_INFINIBAND_I40IW) += i40iw.o
 
drivers/infiniband/hw/i40iw/i40iw.h +1 -1
index 8feec35f95a7..3197e3536d5c 100644
@@ -57,7 +57,7 @@
 #include "i40iw_d.h"
 #include "i40iw_hmc.h"
 
-#include <i40e_client.h>
+#include <linux/net/intel/i40e_client.h>
 #include "i40iw_type.h"
 #include "i40iw_p.h"
 #include <rdma/i40iw-abi.h>
drivers/net/ethernet/intel/Kconfig +1 -0
index b88328fea1d0..8595f578fbe7 100644
@@ -241,6 +241,7 @@ config I40E
 	tristate "Intel(R) Ethernet Controller XL710 Family support"
 	imply PTP_1588_CLOCK
 	depends on PCI
+	select VIRTUAL_BUS
 	---help---
 	  This driver supports Intel(R) Ethernet Controller XL710 Family of
 	  devices.  For more information on how to identify your adapter, go
drivers/net/ethernet/intel/i40e/i40e.h +2 -1
index cb6367334ca7..4321e81d347c 100644
@@ -38,7 +38,7 @@
 #include <net/xdp_sock.h>
 #include "i40e_type.h"
 #include "i40e_prototype.h"
-#include "i40e_client.h"
+#include <linux/net/intel/i40e_client.h>
 #include <linux/avf/virtchnl.h>
 #include "i40e_virtchnl_pf.h"
 #include "i40e_txrx.h"
@@ -655,6 +655,7 @@ struct i40e_pf {
 	u16 last_sw_conf_valid_flags;
 	/* List to keep previous DDP profiles to be rolled back in the future */
 	struct list_head ddp_old_prof;
+	int peer_idx;
 };
 
 /**
drivers/net/ethernet/intel/i40e/i40e_client.c +7 -1
index e81530ca08d0..a3dee729719b 100644
@@ -1,12 +1,12 @@
 // SPDX-License-Identifier: GPL-2.0
 /* Copyright(c) 2013 - 2018 Intel Corporation. */
 
+#include <linux/net/intel/i40e_client.h>
 #include <linux/list.h>
 #include <linux/errno.h>
 
 #include "i40e.h"
 #include "i40e_prototype.h"
-#include "i40e_client.h"
 
 static const char i40e_client_interface_version_str[] = I40E_CLIENT_VERSION_STR;
 static struct i40e_client *registered_client;
@@ -30,11 +30,17 @@ static int i40e_client_update_vsi_ctxt(struct i40e_info *ldev,
 				       bool is_vf, u32 vf_id,
 				       u32 flag, u32 valid_flag);
 
+static int i40e_client_device_register(struct i40e_info *ldev);
+
+static void i40e_client_device_unregister(struct i40e_info *ldev);
+
 static struct i40e_ops i40e_lan_ops = {
 	.virtchnl_send = i40e_client_virtchnl_send,
 	.setup_qvlist = i40e_client_setup_qvlist,
 	.request_reset = i40e_client_request_reset,
 	.update_vsi_ctxt = i40e_client_update_vsi_ctxt,
+	.client_device_register = i40e_client_device_register,
+	.client_device_unregister = i40e_client_device_unregister,
 };
 
 /**
@@ -275,6 +281,27 @@ void i40e_client_update_msix_info(struct i40e_pf *pf)
 	cdev->lan_info.msix_entries = &pf->msix_entries[pf->iwarp_base_vector];
 }
 

drivers/net/ethernet/intel/i40e/i40e_client.c void i40e_client_update_msix_info(struct i40e_pf *pf) · line 293

Quoted 10 lines
drivers/net/ethernet/intel/i40e/i40e_client.c +10 -0
+static int i40e_init_client_virtdev(struct i40e_pf *pf)
+{
+	struct i40e_info *ldev = &pf->cinst->lan_info;
+	struct pci_dev *pdev = pf->pdev;
+	struct virtbus_device *vdev;
+	int ret;
+
+	vdev = &ldev->vdev;
+	vdev->name = I40E_PEER_RDMA_NAME;
+	vdev->dev.parent = &pf->pdev->dev;

What a total and complete mess of a tangled web you just wove here.

Ok, so you pass in a single pointer, that then dereferences 3 pointers deep to find the pointer to the virtbus_device structure, but then you point the parent of that device, back at the original structure's sub-pointer's device itself.

WTF?

And who owns the memory of this thing that is supposed to be dynamically controlled by something OUTSIDE of this driver? Who created that thing 3 pointers deep? What happens when you leak the memory below (hint, you did), and who is supposed to clean it up if you need to properly clean it up if something bad happens?

drivers/net/ethernet/intel/i40e/i40e_client.c void i40e_client_update_msix_info(struct i40e_pf *pf) · line 298

Quoted 5 lines
drivers/net/ethernet/intel/i40e/i40e_client.c +5 -0
+
+	ret = virtbus_dev_register(vdev);
+	if (ret) {
+		dev_err(&pdev->dev, "Failure adding client virtbus dev %s %d\n",
+			I40E_PEER_RDMA_NAME, ret);

Again, the core should handle this, right?

drivers/net/ethernet/intel/i40e/i40e_client.c void i40e_client_update_msix_info(struct i40e_pf *pf) · line 299

Quoted 1 line
drivers/net/ethernet/intel/i40e/i40e_client.c +1 -0
+		return ret;

Did you just leak memory?

Yup, you did, you never actually checked the return value of this function :(

Ugh.

I feel like the virtual bus code is getting better, but this use of the code, um, no, not ok.

Quoted 7 lines

Either way, this series is NOT ready to be merged anywhere, please do not try to rush things.

Also, what ever happened to my "YOU ALL MUST AGREE TO WORK TOGETHER" requirement between this group, and the other group trying to do the same thing? I want to see signed-off-by from EVERYONE involved before we are going to consider this thing.

I am working on RFC where PCI device is sliced to create sub-functions. Each sub-function/slice is created dynamically by the user. User gives sf-number at creation time which will be used for plumbing by systemd/udev, devlink ports.

This sub-function will have sysfs attributes = sfnumber, irq vectors, PCI BAR resource files.
sfnumber as sysfs file will be used by systemd/udev to have deterministic names of netdev and rdma device created on top of sub-function's 'struct device'.

As opposed to that, matching service devices won't have such attributes.

We stayed away from using mdev bus for such dual purpose in past.

Should we have virtbus that holds 'struct device' created for different purpose and have different sysfs attributes? Is it ok?

Your opinion will help us do it correctly.

#20 lore raw

Re: [PATCH v3 04/20] i40e: Register a virtbus device to provide RDMA

Quoted 187 lines · Parav Pandit

Hi Greg,

Quoted 176 lines · Greg KH
Quoted 129 lines · Jeff Kirsher

From: Shiraz Saleem <[email protected]>

Register client virtbus device on the virtbus for the RDMA virtbus driver (irdma) to bind to. It allows to realize a single RDMA driver capable of working with multiple netdev drivers over multi-generation Intel HW supporting RDMA. There is also no load ordering dependencies between i40e and irdma.

Summary of changes:
* Support to add/remove virtbus devices
* Add 2 new client ops.
* i40e_client_device_register() which is called during RDMA probe() per PF. Validate client drv OPs and schedule service task to call open()
* i40e_client_device_unregister() called during RDMA remove() per PF. Call client close() and release_qvlist.
* The global register/unregister calls exported for i40iw are retained until i40iw is removed from the kernel.

---
 drivers/infiniband/hw/i40iw/Makefile          |   1 -
 drivers/infiniband/hw/i40iw/i40iw.h           |   2 +-
 drivers/net/ethernet/intel/Kconfig            |   1 +
 drivers/net/ethernet/intel/i40e/i40e.h        |   3 +-
 drivers/net/ethernet/intel/i40e/i40e_client.c | 109 +++++++++++++++---
 .../linux/net/intel}/i40e_client.h            |  20 +++-
 6 files changed, 112 insertions(+), 24 deletions(-)
 rename {drivers/net/ethernet/intel/i40e => include/linux/net/intel}/i40e_client.h (92%)
drivers/infiniband/hw/i40iw/Makefile +0 -1
index 8942f8229945..34da9eba8a7c 100644
@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0
-ccflags-y :=  -I $(srctree)/drivers/net/ethernet/intel/i40e
 
 obj-$(CONFIG_INFINIBAND_I40IW) += i40iw.o
 
drivers/infiniband/hw/i40iw/i40iw.h +1 -1
index 8feec35f95a7..3197e3536d5c 100644
@@ -57,7 +57,7 @@
 #include "i40iw_d.h"
 #include "i40iw_hmc.h"
 
-#include <i40e_client.h>
+#include <linux/net/intel/i40e_client.h>
 #include "i40iw_type.h"
 #include "i40iw_p.h"
 #include <rdma/i40iw-abi.h>
drivers/net/ethernet/intel/Kconfig +1 -0
index b88328fea1d0..8595f578fbe7 100644
@@ -241,6 +241,7 @@ config I40E
 	tristate "Intel(R) Ethernet Controller XL710 Family support"
 	imply PTP_1588_CLOCK
 	depends on PCI
+	select VIRTUAL_BUS
 	---help---
 	  This driver supports Intel(R) Ethernet Controller XL710 Family of
 	  devices.  For more information on how to identify your adapter, go
drivers/net/ethernet/intel/i40e/i40e.h +2 -1
index cb6367334ca7..4321e81d347c 100644
@@ -38,7 +38,7 @@
 #include <net/xdp_sock.h>
 #include "i40e_type.h"
 #include "i40e_prototype.h"
-#include "i40e_client.h"
+#include <linux/net/intel/i40e_client.h>
 #include <linux/avf/virtchnl.h>
 #include "i40e_virtchnl_pf.h"
 #include "i40e_txrx.h"
@@ -655,6 +655,7 @@ struct i40e_pf {
 	u16 last_sw_conf_valid_flags;
 	/* List to keep previous DDP profiles to be rolled back in the future */
 	struct list_head ddp_old_prof;
+	int peer_idx;
 };
 
 /**
drivers/net/ethernet/intel/i40e/i40e_client.c +7 -1
index e81530ca08d0..a3dee729719b 100644
@@ -1,12 +1,12 @@
 // SPDX-License-Identifier: GPL-2.0
 /* Copyright(c) 2013 - 2018 Intel Corporation. */
 
+#include <linux/net/intel/i40e_client.h>
 #include <linux/list.h>
 #include <linux/errno.h>
 
 #include "i40e.h"
 #include "i40e_prototype.h"
-#include "i40e_client.h"
 
 static const char i40e_client_interface_version_str[] = I40E_CLIENT_VERSION_STR;
 static struct i40e_client *registered_client;
@@ -30,11 +30,17 @@ static int i40e_client_update_vsi_ctxt(struct i40e_info *ldev,
 				       bool is_vf, u32 vf_id,
 				       u32 flag, u32 valid_flag);
 
+static int i40e_client_device_register(struct i40e_info *ldev);
+
+static void i40e_client_device_unregister(struct i40e_info *ldev);
+
 static struct i40e_ops i40e_lan_ops = {
 	.virtchnl_send = i40e_client_virtchnl_send,
 	.setup_qvlist = i40e_client_setup_qvlist,
 	.request_reset = i40e_client_request_reset,
 	.update_vsi_ctxt = i40e_client_update_vsi_ctxt,
+	.client_device_register = i40e_client_device_register,
+	.client_device_unregister = i40e_client_device_unregister,
 };
 
 /**
@@ -275,6 +281,27 @@ void i40e_client_update_msix_info(struct i40e_pf *pf)
 	cdev->lan_info.msix_entries = &pf->msix_entries[pf->iwarp_base_vector];
 }
 

drivers/net/ethernet/intel/i40e/i40e_client.c void i40e_client_update_msix_info(struct i40e_pf *pf) · line 293

Quoted 10 lines
drivers/net/ethernet/intel/i40e/i40e_client.c +10 -0
+static int i40e_init_client_virtdev(struct i40e_pf *pf)
+{
+	struct i40e_info *ldev = &pf->cinst->lan_info;
+	struct pci_dev *pdev = pf->pdev;
+	struct virtbus_device *vdev;
+	int ret;
+
+	vdev = &ldev->vdev;
+	vdev->name = I40E_PEER_RDMA_NAME;
+	vdev->dev.parent = &pf->pdev->dev;

What a total and complete mess of a tangled web you just wove here.

Ok, so you pass in a single pointer, that then dereferences 3 pointers deep to find the pointer to the virtbus_device structure, but then you point the parent of that device, back at the original structure's sub-pointer's device itself.

WTF?

And who owns the memory of this thing that is supposed to be dynamically controlled by something OUTSIDE of this driver? Who created that thing 3 pointers deep? What happens when you leak the memory below (hint, you did), and who is supposed to clean it up if you need to properly clean it up if something bad happens?

drivers/net/ethernet/intel/i40e/i40e_client.c void i40e_client_update_msix_info(struct i40e_pf *pf) · line 298

Quoted 5 lines
drivers/net/ethernet/intel/i40e/i40e_client.c +5 -0
+
+	ret = virtbus_dev_register(vdev);
+	if (ret) {
+		dev_err(&pdev->dev, "Failure adding client virtbus dev %s %d\n",
+			I40E_PEER_RDMA_NAME, ret);

Again, the core should handle this, right?

drivers/net/ethernet/intel/i40e/i40e_client.c void i40e_client_update_msix_info(struct i40e_pf *pf) · line 299

Quoted 1 line
drivers/net/ethernet/intel/i40e/i40e_client.c +1 -0
+		return ret;

Did you just leak memory?

Yup, you did, you never actually checked the return value of this function :(

Ugh.

I feel like the virtual bus code is getting better, but this use of the code, um, no, not ok.

Quoted 7 lines

Either way, this series is NOT ready to be merged anywhere, please do not try to rush things.

Also, what ever happened to my "YOU ALL MUST AGREE TO WORK TOGETHER" requirement between this group, and the other group trying to do the same thing? I want to see signed-off-by from EVERYONE involved before we are going to consider this thing.

Quoted 4 lines

I am working on RFC where PCI device is sliced to create sub-functions. Each sub-function/slice is created dynamically by the user. User gives sf-number at creation time which will be used for plumbing by systemd/udev, devlink ports.

That sounds exactly what is wanted here as well, right?

Quoted 9 lines

This sub-function will have sysfs attributes = sfnumber, irq vectors, PCI BAR resource files.
sfnumber as sysfs file will be used by systemd/udev to have deterministic names of netdev and rdma device created on top of sub-function's 'struct device'.

As opposed to that, matching service devices won't have such attributes.

We stayed away from using mdev bus for such dual purpose in past.

That is good.

Quoted 2 lines

Should we have virtbus that holds 'struct device' created for different purpose and have different sysfs attributes? Is it ok?

That's fine to do, I was expecting that to happen.

thanks,

greg k-h

#21 lore raw

Re: [PATCH v3 04/20] i40e: Register a virtbus device to provide RDMA

Quoted 1 line · Greg KH

On Mon, Dec 16, 2019 at 03:48:05AM +0000, Parav Pandit wrote:

[..]

Quoted 11 lines
Quoted 11 lines
Quoted 10 lines

I feel like the virtual bus code is getting better, but this use of the code, um, no, not ok.

Either way, this series is NOT ready to be merged anywhere, please do not try to rush things.

Also, what ever happened to my "YOU ALL MUST AGREE TO WORK TOGETHER" requirement between this group, and the other group trying to do the same thing? I want to see signed-off-by from EVERYONE involved before we are going to consider this thing.

Quoted 6 lines
Quoted 4 lines

I am working on RFC where PCI device is sliced to create sub-functions. Each sub-function/slice is created dynamically by the user. User gives sf-number at creation time which will be used for plumbing by systemd/udev, devlink ports.

That sounds exactly what is wanted here as well, right?

Not exactly.
Here, in i40 use case - there is a PCI function. This PCI function is used by two drivers:
(1) vendor_foo_netdev.ko creating Netdevice (class net)
(2) vendor_foo_rdma.ko creating RDMA device (class infiniband)

And both drivers are notified using matching service virtbus, which attempts to create to two virtbus_devices with different driver-id, one for each class of device.

However, devices of both class (net, infiniband) will have parent device as PCI device.

In case of sub-functions, created rdma and netdevice will have parent as the sub-function 'struct device'. This way those SFs gets their systemd/udev plumbing done rightly.

Quoted 9 lines
Quoted 9 lines

This sub-function will have sysfs attributes = sfnumber, irq vectors, PCI BAR resource files.
sfnumber as sysfs file will be used by systemd/udev to have deterministic names of netdev and rdma device created on top of sub-function's 'struct device'.

As opposed to that, matching service devices won't have such attributes.

Quoted 9 lines
Quoted 1 line

We stayed away from using mdev bus for such dual purpose in past.

That is good.

Quoted 2 lines

Should we have virtbus that holds 'struct device' created for different purpose and have different sysfs attributes? Is it ok?

That's fine to do, I was expecting that to happen.

ok. Thanks a lot.
Lets understand above additional (non sysfs) difference as well on how virtbus device is getting used differently between sub-functions and matching service purposes.

#22 lore raw

Re: [PATCH v3 04/20] i40e: Register a virtbus device to provide RDMA

Quoted 27 lines · Parav Pandit
Quoted 1 line · Greg KH

On Mon, Dec 16, 2019 at 03:48:05AM +0000, Parav Pandit wrote:

[..]

Quoted 11 lines
Quoted 11 lines
Quoted 10 lines

I feel like the virtual bus code is getting better, but this use of the code, um, no, not ok.

Either way, this series is NOT ready to be merged anywhere, please do not try to rush things.

Also, what ever happened to my "YOU ALL MUST AGREE TO WORK TOGETHER" requirement between this group, and the other group trying to do the same thing? I want to see signed-off-by from EVERYONE involved before we are going to consider this thing.

Quoted 6 lines
Quoted 4 lines

I am working on RFC where PCI device is sliced to create sub-functions. Each sub-function/slice is created dynamically by the user. User gives sf-number at creation time which will be used for plumbing by systemd/udev, devlink ports.

That sounds exactly what is wanted here as well, right?

Not exactly.
Here, in i40 use case - there is a PCI function. This PCI function is used by two drivers:
(1) vendor_foo_netdev.ko creating Netdevice (class net)
(2) vendor_foo_rdma.ko creating RDMA device (class infiniband)

Quoted 3 lines

And both drivers are notified using matching service virtbus, which attempts to create to two virtbus_devices with different driver-id, one for each class of device.

Yes, that is fine.

Quoted 2 lines

However, devices of both class (net, infiniband) will have parent device as PCI device.

That is fine.

Quoted 3 lines

In case of sub-functions, created rdma and netdevice will have parent as the sub-function 'struct device'. This way those SFs gets their systemd/udev plumbing done rightly.

huh? The rdma and netdevice will have as their parent device the virtdevice that is on the virtbus. Not the PCI device's 'struct device'.

thanks,

greg k-h

#23 lore raw

Re: [PATCH v3 04/20] i40e: Register a virtbus device to provide RDMA

Quoted 39 lines · Greg KH
Quoted 27 lines · Parav Pandit
Quoted 1 line · Greg KH

On Mon, Dec 16, 2019 at 03:48:05AM +0000, Parav Pandit wrote:

[..]

Quoted 11 lines
Quoted 11 lines
Quoted 10 lines

I feel like the virtual bus code is getting better, but this use of the code, um, no, not ok.

Either way, this series is NOT ready to be merged anywhere, please do not try to rush things.

Also, what ever happened to my "YOU ALL MUST AGREE TO WORK TOGETHER" requirement between this group, and the other group trying to do the same thing? I want to see signed-off-by from EVERYONE involved before we are going to consider this thing.

Quoted 6 lines
Quoted 4 lines

I am working on RFC where PCI device is sliced to create sub-functions. Each sub-function/slice is created dynamically by the user. User gives sf-number at creation time which will be used for plumbing by systemd/udev, devlink ports.

That sounds exactly what is wanted here as well, right?

Not exactly.
Here, in i40 use case - there is a PCI function. This PCI function is used by two drivers:
(1) vendor_foo_netdev.ko creating Netdevice (class net)
(2) vendor_foo_rdma.ko creating RDMA device (class infiniband)

Quoted 3 lines

And both drivers are notified using matching service virtbus, which attempts to create to two virtbus_devices with different driver-id, one for each class of device.

Yes, that is fine.

Quoted 2 lines

However, devices of both class (net, infiniband) will have parent device as PCI device.

That is fine.

Quoted 8 lines
Quoted 3 lines

In case of sub-functions, created rdma and netdevice will have parent as the sub-function 'struct device'. This way those SFs gets their systemd/udev plumbing done rightly.

huh? The rdma and netdevice will have as their parent device the virtdevice that is on the virtbus. Not the PCI device's 'struct device'.

Yes. I meant same when I said "sub-function 'struct device'", which is nothing bug a virtdevice.

ok. Great. We are on same page now.

As we discussed, if sub-functions uses the bus, it will use the virtbus and virtdevice device with above discussed differences.
Once I finish internal RFC review for sub-functions, will post it on the list.
I do not expect RFC to finish before Christmas holidays.

Thanks,
Parav

Quoted 4 lines

thanks,

greg k-h

#24 lore raw

PATCH v3 5/20 RDMA/irdma: Add driver framework definitions

In reply to Jeff Kirsher

From: Mustafa Ismail <[email protected]>

Register irdma as a virtbus driver binding to 'i40e' and 'ice' virtbus devices added from their respective netdev drivers for each PF. During irdma probe(), the gen-specific netdev peer device is obtained from virtbus device to establish an interface and initialize the HW.

---
 drivers/infiniband/hw/irdma/i40iw_if.c | 222 +++++++++
 drivers/infiniband/hw/irdma/irdma_if.c | 463 ++++++++++++++++++
 drivers/infiniband/hw/irdma/main.c     | 630 ++++++++++++++++++++++++
 drivers/infiniband/hw/irdma/main.h     | 652 +++++++++++++++++++++++++
 4 files changed, 1967 insertions(+)
 create mode 100644 drivers/infiniband/hw/irdma/i40iw_if.c
 create mode 100644 drivers/infiniband/hw/irdma/irdma_if.c
 create mode 100644 drivers/infiniband/hw/irdma/main.c
 create mode 100644 drivers/infiniband/hw/irdma/main.h
drivers/infiniband/hw/irdma/i40iw_if.c new file +222 -0
new file mode 100644
index 000000000000..debdbb9108bb
@@ -0,0 +1,222 @@
+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
+/* Copyright (c) 2015 - 2019 Intel Corporation */
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/net/intel/i40e_client.h>
+#include <net/addrconf.h>
+#include "main.h"
+#include "i40iw_hw.h"
+
+/**
+ * i40iw_request_reset - Request a reset
+ * @rf: RDMA PCI function
+ *
+ */
+static void i40iw_request_reset(struct irdma_pci_f *rf)
+{
+	struct i40e_info *ldev = (struct i40e_info *)rf->ldev.if_ldev;
+
+	ldev->ops->request_reset(ldev, rf->ldev.if_client, 1);
+}
+
+/**
+ * i40iw_open - client interface operation open for iwarp/uda device
+ * @ldev: LAN device information
+ * @client: iwarp client information, provided during registration
+ *
+ * Called by the LAN driver during the processing of client
+ * register Create device resources, set up queues, pble and hmc
+ * objects and register the device with the ib verbs interface
+ * Return 0 if successful, otherwise return error
+ */
+static int i40iw_open(struct i40e_info *ldev, struct i40e_client *client)
+{
+	struct irdma_device *iwdev = NULL;
+	struct irdma_handler *hdl = NULL;
+	struct irdma_priv_ldev *pldev;
+	struct irdma_sc_dev *dev;
+	struct irdma_pci_f *rf;
+	struct irdma_l2params l2params = {};
+	int err_code = -EIO;
+	int i;
+	u16 qset;
+	u16 last_qset = IRDMA_NO_QSET;
+
+	hdl = irdma_find_handler(ldev->pcidev);
+	if (hdl)
+		return 0;
+
+	hdl = kzalloc((sizeof(*hdl) + sizeof(*iwdev)), GFP_KERNEL);
+	if (!hdl)
+		return -ENOMEM;
+
+	iwdev = (struct irdma_device *)((u8 *)hdl + sizeof(*hdl));
+
+	iwdev->param_wq = alloc_ordered_workqueue("l2params", WQ_MEM_RECLAIM);
+	if (!iwdev->param_wq)
+		goto error;
+
+	rf = &hdl->rf;
+	rf->hdl = hdl;
+	dev = &rf->sc_dev;
+	dev->back_dev = rf;
+	rf->rdma_ver = IRDMA_GEN_1;
+	hdl->vdev = &ldev->vdev;
+	irdma_init_rf_config_params(rf);
+	rf->gen_ops.init_hw = i40iw_init_hw;
+	rf->gen_ops.request_reset = i40iw_request_reset;
+	rf->hw.hw_addr = ldev->hw_addr;
+	rf->pdev = ldev->pcidev;
+	rf->netdev = ldev->netdev;
+	dev->pci_rev = rf->pdev->revision;
+	iwdev->rf = rf;
+	iwdev->hdl = hdl;
+	iwdev->ldev = &rf->ldev;
+	iwdev->init_state = INITIAL_STATE;
+	iwdev->rcv_wnd = IRDMA_CM_DEFAULT_RCV_WND_SCALED;
+	iwdev->rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE;
+	iwdev->netdev = ldev->netdev;
+	iwdev->create_ilq = true;
+	iwdev->vsi_num = 0;
+
+	pldev = &rf->ldev;
+	hdl->ldev = pldev;
+	pldev->if_client = client;
+	pldev->if_ldev = ldev;
+	pldev->fn_num = ldev->fid;
+	pldev->ftype = ldev->ftype;
+	pldev->pf_vsi_num = 0;
+	pldev->msix_count = ldev->msix_count;
+	pldev->msix_entries = ldev->msix_entries;
+
+	if (irdma_ctrl_init_hw(rf))
+		goto error;
+
+	l2params.mtu =
+		(ldev->params.mtu) ? ldev->params.mtu : IRDMA_DEFAULT_MTU;
+	for (i = 0; i < I40E_CLIENT_MAX_USER_PRIORITY; i++) {
+		qset = ldev->params.qos.prio_qos[i].qs_handle;
+		l2params.up2tc[i] = ldev->params.qos.prio_qos[i].tc;
+		l2params.qs_handle_list[i] = qset;
+		if (last_qset == IRDMA_NO_QSET)
+			last_qset = qset;
+		else if ((qset != last_qset) && (qset != IRDMA_NO_QSET))
+			iwdev->dcb = true;
+	}
+
+	if (irdma_rt_init_hw(rf, iwdev, &l2params)) {
+		irdma_deinit_ctrl_hw(rf);
+		goto error;
+	}
+
+	irdma_add_handler(hdl);
+	return 0;
+error:
+	kfree(hdl);
+	return err_code;
+}
+
+/**
+ * i40iw_l2param_change - handle mss change
+ * @ldev: lan device information
+ * @client: client for parameter change
+ * @params: new parameters from L2
+ */
+static void i40iw_l2param_change(struct i40e_info *ldev,
+				 struct i40e_client *client,
+				 struct i40e_params *params)
+{
+	struct irdma_l2params l2params = {};
+	struct irdma_device *iwdev;
+
+	iwdev = irdma_get_device(ldev->netdev);
+	if (!iwdev)
+		return;
+
+	if (iwdev->vsi.mtu != params->mtu) {
+		l2params.mtu_changed = true;
+		l2params.mtu = params->mtu;
+	}
+	irdma_change_l2params(&iwdev->vsi, &l2params);
+	irdma_put_device(iwdev);
+}
+
+/**
+ * i40iw_close - client interface operation close for iwarp/uda device
+ * @ldev: lan device information
+ * @client: client to close
+ * @reset: flag to indicate close on reset
+ *
+ * Called by the lan driver during the processing of client unregister
+ * Destroy and clean up the driver resources
+ */
+static void i40iw_close(struct i40e_info *ldev, struct i40e_client *client,
+			bool reset)
+{
+	struct irdma_handler *hdl;
+	struct irdma_pci_f *rf;
+	struct irdma_device *iwdev;
+
+	hdl = irdma_find_handler(ldev->pcidev);
+	if (!hdl)
+		return;
+	rf = &hdl->rf;
+	iwdev = (struct irdma_device *)((u8 *)hdl + sizeof(*hdl));
+
+	if (iwdev->param_wq)
+		destroy_workqueue(iwdev->param_wq);
+
+	if (reset)
+		iwdev->reset = true;
+
+	irdma_deinit_rt_device(iwdev);
+	irdma_deinit_ctrl_hw(rf);
+	irdma_del_handler(irdma_find_handler(ldev->pcidev));
+	kfree(hdl);
+	pr_info("IRDMA hardware deinitialization complete\n");
+}
+
+/* client interface functions */
+static const struct i40e_client_ops i40e_ops = {
+	.open = i40iw_open,
+	.close = i40iw_close,
+	.l2_param_change = i40iw_l2param_change
+};
+
+static struct i40e_client i40iw_client = {
+	.name = "irdma",
+	.ops = &i40e_ops,
+	.type = I40E_CLIENT_IWARP,
+};
+
+int i40iw_probe(struct virtbus_device *vdev)
+{
+	struct i40e_info *ldev =
+		container_of(vdev, struct i40e_info, vdev);
+
+	if (!ldev)
+		return -EINVAL;
+
+	if (!ldev->ops->client_device_register)
+		return -EINVAL;
+
+	ldev->client = &i40iw_client;
+
+	return ldev->ops->client_device_register(ldev);
+}
+
+int i40iw_remove(struct virtbus_device *vdev)
+{
+	struct i40e_info *ldev =
+		container_of(vdev, struct i40e_info, vdev);
+
+	if (!ldev)
+		return -EINVAL;
+
+	if (ldev->ops->client_device_unregister)
+		ldev->ops->client_device_unregister(ldev);
+
+	return 0;
+}
drivers/infiniband/hw/irdma/irdma_if.c new file +463 -0

463 lines not on this page · raw message

drivers/infiniband/hw/irdma/main.c new file +630 -0

630 lines not on this page · raw message

drivers/infiniband/hw/irdma/main.h new file +652 -0

652 lines not on this page · raw message

Signature · 1 line
2.23.0
#25 lore raw

Re: [PATCH v3 05/20] RDMA/irdma: Add driver framework definitions

drivers/infiniband/hw/irdma/i40iw_if.c line 19

Quoted 2 lines · Jeff Kirsher
drivers/infiniband/hw/irdma/i40iw_if.c new file +2 -0
+{
+	struct i40e_info *ldev = (struct i40e_info *)rf->ldev.if_ldev;

Why are there so many casts in this file? Is this really container of?

drivers/infiniband/hw/irdma/i40iw_if.c line 55

Quoted 5 lines
drivers/infiniband/hw/irdma/i40iw_if.c new file +5 -0
+	hdl = kzalloc((sizeof(*hdl) + sizeof(*iwdev)), GFP_KERNEL);
+	if (!hdl)
+		return -ENOMEM;
+
+	iwdev = (struct irdma_device *)((u8 *)hdl + sizeof(*hdl));

Yikes, use structs and container of for things like this please.

drivers/infiniband/hw/irdma/i40iw_if.c line 59

Quoted 3 lines
drivers/infiniband/hw/irdma/i40iw_if.c new file +3 -0
+	iwdev->param_wq = alloc_ordered_workqueue("l2params", WQ_MEM_RECLAIM);
+	if (!iwdev->param_wq)
+		goto error;

Leon usually asks why another work queue at this point, at least have a comment justifying why. Shouldn't it have a better name?

drivers/infiniband/hw/irdma/i40iw_if.c line 186

Quoted 6 lines
drivers/infiniband/hw/irdma/i40iw_if.c new file +6 -0
+/* client interface functions */
+static const struct i40e_client_ops i40e_ops = {
+	.open = i40iw_open,
+	.close = i40iw_close,
+	.l2_param_change = i40iw_l2param_change
+};

Wasn't the whole point of virtual bus to avoid stuff like this? Why isn't a client the virtual bus object and this information extended into the driver ops?

drivers/infiniband/hw/irdma/i40iw_if.c line 200

Quoted 7 lines
drivers/infiniband/hw/irdma/i40iw_if.c new file +7 -0
+int i40iw_probe(struct virtbus_device *vdev)
+{
+	struct i40e_info *ldev =
+		container_of(vdev, struct i40e_info, vdev);
+
+	if (!ldev)
+		return -EINVAL;

eh? how can that happen

drivers/infiniband/hw/irdma/i40iw_if.c line 203

Quoted 3 lines
drivers/infiniband/hw/irdma/i40iw_if.c new file +3 -0
+
+	if (!ldev->ops->client_device_register)
+		return -EINVAL;

How can this happen too? If it doesn't support register then don't create a virtual device, surely?

I've really developed a strong distate to these random non-functional 'ifs' that seem to get into things.

If it is functional then fine, but if it is an assertion write it as if (WARN_ON()) to make it clear to readers it can't happen by design

drivers/infiniband/hw/irdma/irdma_if.c line 19

Quoted 10 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +10 -0
+/**
+ * irdma_lan_register_qset - Register qset with LAN driver
+ * @vsi: vsi structure
+ * @tc_node: Traffic class node
+ */
+static enum irdma_status_code irdma_lan_register_qset(struct irdma_sc_vsi *vsi,
+						      struct irdma_ws_node *tc_node)
+{
+	struct irdma_device *iwdev = vsi->back_vsi;
+	struct iidc_peer_dev *ldev = (struct iidc_peer_dev *)iwdev->ldev->if_ldev;

Again with the casts.. Please try to clean up the casting in this driver

drivers/infiniband/hw/irdma/irdma_if.c line 23

Quoted 4 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +4 -0
+	struct iidc_res rdma_qset_res = {};
+	int ret;
+
+	if (ldev->ops->alloc_res) {

Quite an abnormal coding style to put the entire function under an if, just if() return 0 ? Many examples of this

drivers/infiniband/hw/irdma/irdma_if.c line 81

Quoted 13 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +13 -0
+/**
+ * irdma_log_invalid_mtu: log warning on invalid mtu
+ * @mtu: maximum tranmission unit
+ */
+static void irdma_log_invalid_mtu(u16 mtu)
+{
+	if (mtu < IRDMA_MIN_MTU_IPV4)
+		pr_warn("Current MTU setting of %d is too low for RDMA traffic. Minimum MTU is 576 for IPv4 and 1280 for IPv6\n",
+			mtu);
+	else if (mtu < IRDMA_MIN_MTU_IPV6)
+		pr_warn("Current MTU setting of %d is too low for IPv6 RDMA traffic, the minimum is 1280\n",
+			mtu);
+}

Don't use pr_* stuff in drivers that have a struct device.

Quoted 7 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +7 -0
+/**
+ * irdma_event_handler - Called by LAN driver to notify events
+ * @ldev: Peer device structure
+ * @event: event from LAN driver
+ */
+static void irdma_event_handler(struct iidc_peer_dev *ldev,
+				struct iidc_event *event)

drivers/infiniband/hw/irdma/irdma_if.c line 126

Quoted 10 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +10 -0
+{
+	struct irdma_l2params l2params = {};
+	struct irdma_device *iwdev;
+	int i;
+
+	iwdev = irdma_get_device(ldev->netdev);
+	if (!iwdev)
+		return;
+
+	if (test_bit(IIDC_EVENT_LINK_CHANGE, event->type)) {

Is this atomic? Why using test_bit?

drivers/infiniband/hw/irdma/irdma_if.c line 245

Quoted 2 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +2 -0
+		ldev->ops->reg_for_notification(ldev, &events);
+	dev_info(rfdev_to_dev(dev), "IRDMA VSI Open Successful");

Lets not do this kind of logging..

Quoted 11 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +11 -0
+static void irdma_close(struct iidc_peer_dev *ldev, enum iidc_close_reason reason)
+{
+	struct irdma_device *iwdev;
+	struct irdma_pci_f *rf;
+
+	iwdev = irdma_get_device(ldev->netdev);
+	if (!iwdev)
+		return;
+
+	irdma_put_device(iwdev);
+	rf = iwdev->rf;

drivers/infiniband/hw/irdma/irdma_if.c line 279

Quoted 10 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +10 -0
+	if (reason == IIDC_REASON_GLOBR_REQ || reason == IIDC_REASON_CORER_REQ ||
+	    reason == IIDC_REASON_PFR_REQ || rf->reset) {
+		iwdev->reset = true;
+		rf->reset = true;
+	}
+
+	if (iwdev->init_state >= CEQ0_CREATED)
+		irdma_deinit_rt_device(iwdev);
+
+	kfree(iwdev);

Mixing put and kfree? So confusing. Why are there so many structs and so much indirection? Very hard to understand if this is right or not.

Quoted 18 lines

new file mode 100644
index 000000000000..b418e76a3302
+++ b/drivers/infiniband/hw/irdma/main.c

drivers/infiniband/hw/irdma/main.c new file +14 -0
@@ -0,0 +1,630 @@
+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
+/* Copyright (c) 2015 - 2019 Intel Corporation */
+#include "main.h"
+
+/* Legacy i40iw module parameters */
+static int resource_profile;
+module_param(resource_profile, int, 0644);
+MODULE_PARM_DESC(resource_profile, "Resource Profile: 0=PF only, 1=Weighted VF, 2=Even Distribution");
+
+static int max_rdma_vfs = 32;
+module_param(max_rdma_vfs, int, 0644);
+MODULE_PARM_DESC(max_rdma_vfs, "Maximum VF count: 0-32 32=default");
+
+static int mpa_version = 2;

drivers/infiniband/hw/irdma/main.c line 24

Quoted 10 lines
drivers/infiniband/hw/irdma/main.c new file +10 -0
+module_param(mpa_version, int, 0644);
+MODULE_PARM_DESC(mpa_version, "MPA version: deprecated parameter");
+
+static int push_mode;
+module_param(push_mode, int, 0644);
+MODULE_PARM_DESC(push_mode, "Low latency mode: deprecated parameter");
+
+static int debug;
+module_param(debug, int, 0644);
+MODULE_PARM_DESC(debug, "debug flags: deprecated parameter");

Generally no to module parameters

drivers/infiniband/hw/irdma/main.c line 27

Quoted 1 line
drivers/infiniband/hw/irdma/main.c new file +1 -0
+static struct workqueue_struct *irdma_wq;

Another wq already?

drivers/infiniband/hw/irdma/main.h line 264

Quoted 6 lines
drivers/infiniband/hw/irdma/main.h new file +6 -0
+struct irdma_pci_f {
+	bool ooo;
+	bool reset;
+	bool rsrc_created;
+	bool stop_cqp_thread;
+	bool msix_shared;

Linus has spoken poorly about lots of bools in a struct. Can this be a bitfield?

drivers/infiniband/hw/irdma/main.h line 385

Quoted 5 lines
drivers/infiniband/hw/irdma/main.h new file +5 -0
+/***********************************************************/
+/**
+ * to_iwdev - get device
+ * @ibdev: ib device
+ **/

Maybe some of these comment blocks are not so valuable :\

drivers/infiniband/hw/irdma/main.h line 516

Quoted 3 lines
drivers/infiniband/hw/irdma/main.h new file +3 -0
+	spin_lock_irqsave(&rf->rsrc_lock, flags);
+
+	bit_is_set = test_bit(rsrc_num, rsrc_array);

Again, are these atomics? Looks like no, why test_bit?

Jason

#26 lore raw

Re: [PATCH v3 05/20] RDMA/irdma: Add driver framework definitions

Quoted 14 lines · Jason Gunthorpe

drivers/infiniband/hw/irdma/i40iw_if.c line 19

Quoted 2 lines · Jeff Kirsher
drivers/infiniband/hw/irdma/i40iw_if.c new file +2 -0
+{
+	struct i40e_info *ldev = (struct i40e_info *)rf->ldev.if_ldev;

Why are there so many casts in this file? Is this really container of?

drivers/infiniband/hw/irdma/i40iw_if.c line 55

Quoted 5 lines
drivers/infiniband/hw/irdma/i40iw_if.c new file +5 -0
+	hdl = kzalloc((sizeof(*hdl) + sizeof(*iwdev)), GFP_KERNEL);
+	if (!hdl)
+		return -ENOMEM;
+
+	iwdev = (struct irdma_device *)((u8 *)hdl + sizeof(*hdl));

Yikes, use structs and container of for things like this please.

Quoted 6 lines

drivers/infiniband/hw/irdma/i40iw_if.c line 59

Quoted 3 lines
drivers/infiniband/hw/irdma/i40iw_if.c new file +3 -0
+	iwdev->param_wq = alloc_ordered_workqueue("l2params", WQ_MEM_RECLAIM);
+	if (!iwdev->param_wq)
+		goto error;

Leon usually asks why another work queue at this point, at least have a comment justifying why. Shouldn't it have a better name?

Yeah, combination of WQ_MEM_RECLAIM flag and "params" in the name raises eyebrows immediately.

Thanks

#27 lore raw

RE: [PATCH v3 05/20] RDMA/irdma: Add driver framework definitions

In reply to Jason Gunthorpe
Quoted 7 lines

Subject: Re: [PATCH v3 05/20] RDMA/irdma: Add driver framework definitions

drivers/infiniband/hw/irdma/i40iw_if.c line 19

Quoted 2 lines · Jeff Kirsher
drivers/infiniband/hw/irdma/i40iw_if.c new file +2 -0
+{
+	struct i40e_info *ldev = (struct i40e_info *)rf->ldev.if_ldev;

Why are there so many casts in this file? Is this really container of?

The casting here is redundant. Will clean up.

Quoted 8 lines

drivers/infiniband/hw/irdma/i40iw_if.c line 55

Quoted 6 lines
drivers/infiniband/hw/irdma/i40iw_if.c new file +5 -0
+	hdl = kzalloc((sizeof(*hdl) + sizeof(*iwdev)), GFP_KERNEL);
+	if (!hdl)
+		return -ENOMEM;
+
+	iwdev = (struct irdma_device *)((u8 *)hdl + sizeof(*hdl));

Yikes, use structs and container of for things like this please.

iwdev object alloc should be split here from hdl. Will fix.

Quoted 9 lines

Review

Quoted 2 lines
(quoted diff) +1 -0
+	iwdev->param_wq = alloc_ordered_workqueue("l2params",

WQ_MEM_RECLAIM);

Review

Quoted 2 lines
(quoted diff) +2 -0
+	if (!iwdev->param_wq)
+		goto error;

Leon usually asks why another work queue at this point, at least have a comment justifying why. Shouldn't it have a better name?

Ugh! The l2 param changes is made synchronous based on prior feedback from Leon. This wq should be removed.

Quoted 8 lines

drivers/infiniband/hw/irdma/i40iw_if.c

Quoted 5 lines
drivers/infiniband/hw/irdma/i40iw_if.c new file +5 -0
+/* client interface functions */
+static const struct i40e_client_ops i40e_ops = {
+	.open = i40iw_open,
+	.close = i40iw_close,
+	.l2_param_change = i40iw_l2param_change };

Wasn't the whole point of virtual bus to avoid stuff like this? Why isn't a client the virtual bus object and this information extended into the driver ops?

These are the private interface calls between lan and rdma. These ops are implemented by RDMA driver but invoked by netdev driver.

Quoted 17 lines

drivers/infiniband/hw/irdma/i40iw_if.c line 200

Quoted 7 lines
drivers/infiniband/hw/irdma/i40iw_if.c new file +6 -0
+int i40iw_probe(struct virtbus_device *vdev) {
+	struct i40e_info *ldev =
+		container_of(vdev, struct i40e_info, vdev);
+
+	if (!ldev)
+		return -EINVAL;

eh? how can that happen

drivers/infiniband/hw/irdma/i40iw_if.c line 203

Quoted 3 lines
drivers/infiniband/hw/irdma/i40iw_if.c new file +3 -0
+
+	if (!ldev->ops->client_device_register)
+		return -EINVAL;

How can this happen too? If it doesn't support register then don't create a virtual device, surely?

Quoted 6 lines

I've really developed a strong distate to these random non-functional 'ifs' that seem to get into things.

If it is functional then fine, but if it is an assertion write it as if (WARN_ON()) to make it clear to readers it can't happen by design

Yeah. These cant happen by design and should be treated as assertion. Will fix.

Quoted 8 lines

drivers/infiniband/hw/irdma/irdma_if.c

Quoted 7 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +6 -0
+/**
+ * irdma_lan_register_qset - Register qset with LAN driver
+ * @vsi: vsi structure
+ * @tc_node: Traffic class node
+ */
+static enum irdma_status_code irdma_lan_register_qset(struct irdma_sc_vsi

*vsi,

Quoted 8 lines

Review

Quoted 1 line
(quoted diff) +1 -0
+						      struct irdma_ws_node

*tc_node) {

Review

Quoted 3 lines
(quoted diff) +3 -0
+	struct irdma_device *iwdev = vsi->back_vsi;
+	struct iidc_peer_dev *ldev = (struct iidc_peer_dev
+*)iwdev->ldev->if_ldev;

Again with the casts.. Please try to clean up the casting in this driver

Ditto as my previous comment.

Quoted 7 lines

drivers/infiniband/hw/irdma/irdma_if.c line 23

Quoted 4 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +4 -0
+	struct iidc_res rdma_qset_res = {};
+	int ret;
+
+	if (ldev->ops->alloc_res) {

Quite an abnormal coding style to put the entire function under an if, just if() return 0 ? Many examples of this

Will fix.

Quoted 9 lines

drivers/infiniband/hw/irdma/irdma_if.c

Quoted 8 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +7 -0
+/**
+ * irdma_log_invalid_mtu: log warning on invalid mtu
+ * @mtu: maximum tranmission unit
+ */
+static void irdma_log_invalid_mtu(u16 mtu) {
+	if (mtu < IRDMA_MIN_MTU_IPV4)
+		pr_warn("Current MTU setting of %d is too low for RDMA traffic.

Minimum MTU is 576 for IPv4 and 1280 for IPv6\n",

Quoted 9 lines

drivers/infiniband/hw/irdma/irdma_if.c

Quoted 3 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +3 -0
+			mtu);
+	else if (mtu < IRDMA_MIN_MTU_IPV6)
+		pr_warn("Current MTU setting of %d is too low for IPv6 RDMA

traffic, the minimum is 1280\n",

Review

Quoted 2 lines
(quoted diff) +2 -0
+			mtu);
+}

Don't use pr_* stuff in drivers that have a struct device.

Will fix.

Quoted 11 lines
Quoted 11 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +11 -0
+/**
+ * irdma_event_handler - Called by LAN driver to notify events
+ * @ldev: Peer device structure
+ * @event: event from LAN driver
+ */
+static void irdma_event_handler(struct iidc_peer_dev *ldev,
+				struct iidc_event *event)
+{
+	struct irdma_l2params l2params = {};
+	struct irdma_device *iwdev;
+	int i;
Quoted 8 lines

drivers/infiniband/hw/irdma/irdma_if.c line 126

Quoted 6 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +6 -0
+
+	iwdev = irdma_get_device(ldev->netdev);
+	if (!iwdev)
+		return;
+
+	if (test_bit(IIDC_EVENT_LINK_CHANGE, event->type)) {

Is this atomic? Why using test_bit?

No its not. What do you suggest we use?

Quoted 6 lines

drivers/infiniband/hw/irdma/irdma_if.c line 245

Quoted 3 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +2 -0
+		ldev->ops->reg_for_notification(ldev, &events);
+	dev_info(rfdev_to_dev(dev), "IRDMA VSI Open Successful");

Lets not do this kind of logging..

There is some dev_info which should be cleaned up to dev_dbg.
But logging this info is useful to know that this functions VSI (and associated ibdev) is up and reading for RDMA traffic.
Is info logging to be avoided altogether?

Quoted 13 lines

drivers/infiniband/hw/irdma/irdma_if.c

Quoted 12 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +12 -0
+static void irdma_close(struct iidc_peer_dev *ldev, enum
+iidc_close_reason reason) {
+	struct irdma_device *iwdev;
+	struct irdma_pci_f *rf;
+
+	iwdev = irdma_get_device(ldev->netdev);
+	if (!iwdev)
+		return;
+
+	irdma_put_device(iwdev);
+	rf = iwdev->rf;
+	if (reason == IIDC_REASON_GLOBR_REQ || reason ==

IIDC_REASON_CORER_REQ ||

Quoted 12 lines

drivers/infiniband/hw/irdma/irdma_if.c line 279

Quoted 9 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +9 -0
+	    reason == IIDC_REASON_PFR_REQ || rf->reset) {
+		iwdev->reset = true;
+		rf->reset = true;
+	}
+
+	if (iwdev->init_state >= CEQ0_CREATED)
+		irdma_deinit_rt_device(iwdev);
+
+	kfree(iwdev);

Mixing put and kfree? So confusing. Why are there so many structs and so much indirection? Very hard to understand if this is right or not.

This does look weird. I think the irdma_get_device() was here just to get to iwdev. And put_device is releasing the refcnt immediately. Since we are in a VSI close(), we should not need to take refcnt on ibdev and just deregister it. Will fix this.

Quoted 17 lines
Quoted 6 lines

new file mode 100644
index 000000000000..b418e76a3302
+++ b/drivers/infiniband/hw/irdma/main.c

drivers/infiniband/hw/irdma/main.c new file +1 -0
@@ -0,0 +1,630 @@
+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB

drivers/infiniband/hw/irdma/main.c line 11

Quoted 10 lines
drivers/infiniband/hw/irdma/main.c new file +10 -0
+/* Copyright (c) 2015 - 2019 Intel Corporation */ #include "main.h"
+
+/* Legacy i40iw module parameters */
+static int resource_profile;
+module_param(resource_profile, int, 0644);
+MODULE_PARM_DESC(resource_profile, "Resource Profile: 0=PF only,
+1=Weighted VF, 2=Even Distribution");
+
+static int max_rdma_vfs = 32;
+module_param(max_rdma_vfs, int, 0644);

MODULE_PARM_DESC(max_rdma_vfs,

Quoted 16 lines
Quoted 4 lines
drivers/infiniband/hw/irdma/main.c new file +4 -0
+"Maximum VF count: 0-32 32=default");
+
+static int mpa_version = 2;
+module_param(mpa_version, int, 0644); MODULE_PARM_DESC(mpa_version,

drivers/infiniband/hw/irdma/main.c line 24

Quoted 10 lines
drivers/infiniband/hw/irdma/main.c new file +10 -0
+"MPA version: deprecated parameter");
+
+static int push_mode;
+module_param(push_mode, int, 0644);
+MODULE_PARM_DESC(push_mode, "Low latency mode: deprecated
+parameter");
+
+static int debug;
+module_param(debug, int, 0644);
+MODULE_PARM_DESC(debug, "debug flags: deprecated parameter");

Generally no to module parameters

Agree. But these are module params that existed in i40iw. And irdma replaces i40iw and has a module alias for it.

Quoted 4 lines

drivers/infiniband/hw/irdma/main.c line 27

Quoted 2 lines
drivers/infiniband/hw/irdma/main.c new file +1 -0
+static struct workqueue_struct *irdma_wq;

Another wq already?

This wq is used for deferred handling of irdma service tasks. Such as rebuild/recovery after reset.

Quoted 9 lines

drivers/infiniband/hw/irdma/main.h line 264

Quoted 7 lines
drivers/infiniband/hw/irdma/main.h new file +6 -0
+struct irdma_pci_f {
+	bool ooo;
+	bool reset;
+	bool rsrc_created;
+	bool stop_cqp_thread;
+	bool msix_shared;

Linus has spoken poorly about lots of bools in a struct. Can this be a bitfield?

Possibly. Yes. Will look into it.

Quoted 8 lines

drivers/infiniband/hw/irdma/main.h line 385

Quoted 6 lines
drivers/infiniband/hw/irdma/main.h new file +5 -0
+/***********************************************************/
+/**
+ * to_iwdev - get device
+ * @ibdev: ib device
+ **/

Maybe some of these comment blocks are not so valuable :\

Will fix.

Quoted 6 lines

drivers/infiniband/hw/irdma/main.h line 516

Quoted 4 lines
drivers/infiniband/hw/irdma/main.h new file +3 -0
+	spin_lock_irqsave(&rf->rsrc_lock, flags);
+
+	bit_is_set = test_bit(rsrc_num, rsrc_array);

Again, are these atomics? Looks like no, why test_bit?

This helper is not used and to be removed.

But, yes integrity needs to be assured while read/modify rsrc_array. irdma_alloc_rsrc() and irdma_free_rsrc() probably warrant using the non-atomic ver. of set/clear bit since its inside a lock.

Thanks for the feedback!

Shiraz

#28 lore raw

Re: [PATCH v3 05/20] RDMA/irdma: Add driver framework definitions

Quoted 1 line · Shiraz
Quoted 1 line

Subject: Re: [PATCH v3 05/20] RDMA/irdma: Add driver framework definitions

<...>

Quoted 11 lines
Quoted 6 lines

drivers/infiniband/hw/irdma/irdma_if.c line 245

Quoted 3 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +2 -0
+		ldev->ops->reg_for_notification(ldev, &events);
+	dev_info(rfdev_to_dev(dev), "IRDMA VSI Open Successful");

Lets not do this kind of logging..

There is some dev_info which should be cleaned up to dev_dbg.
But logging this info is useful to know that this functions VSI (and associated ibdev) is up and reading for RDMA traffic.
Is info logging to be avoided altogether?

Will function tracer (ftrace) output be sufficient here?
https://www.kernel.org/doc/html/latest/trace/ftrace.html

Quoted 66 lines
Quoted 14 lines

drivers/infiniband/hw/irdma/irdma_if.c

Quoted 13 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +12 -0
+static void irdma_close(struct iidc_peer_dev *ldev, enum
+iidc_close_reason reason) {
+	struct irdma_device *iwdev;
+	struct irdma_pci_f *rf;
+
+	iwdev = irdma_get_device(ldev->netdev);
+	if (!iwdev)
+		return;
+
+	irdma_put_device(iwdev);
+	rf = iwdev->rf;
+	if (reason == IIDC_REASON_GLOBR_REQ || reason ==

IIDC_REASON_CORER_REQ ||

Quoted 12 lines

drivers/infiniband/hw/irdma/irdma_if.c line 279

Quoted 9 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +9 -0
+	    reason == IIDC_REASON_PFR_REQ || rf->reset) {
+		iwdev->reset = true;
+		rf->reset = true;
+	}
+
+	if (iwdev->init_state >= CEQ0_CREATED)
+		irdma_deinit_rt_device(iwdev);
+
+	kfree(iwdev);

Mixing put and kfree? So confusing. Why are there so many structs and so much indirection? Very hard to understand if this is right or not.

This does look weird. I think the irdma_get_device() was here just to get to iwdev. And put_device is releasing the refcnt immediately. Since we are in a VSI close(), we should not need to take refcnt on ibdev and just deregister it. Will fix this.

Quoted 17 lines
Quoted 6 lines

new file mode 100644
index 000000000000..b418e76a3302
+++ b/drivers/infiniband/hw/irdma/main.c

drivers/infiniband/hw/irdma/main.c new file +1 -0
@@ -0,0 +1,630 @@
+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB

drivers/infiniband/hw/irdma/main.c line 11

Quoted 10 lines
drivers/infiniband/hw/irdma/main.c new file +10 -0
+/* Copyright (c) 2015 - 2019 Intel Corporation */ #include "main.h"
+
+/* Legacy i40iw module parameters */
+static int resource_profile;
+module_param(resource_profile, int, 0644);
+MODULE_PARM_DESC(resource_profile, "Resource Profile: 0=PF only,
+1=Weighted VF, 2=Even Distribution");
+
+static int max_rdma_vfs = 32;
+module_param(max_rdma_vfs, int, 0644);

MODULE_PARM_DESC(max_rdma_vfs,

Quoted 16 lines
Quoted 4 lines
drivers/infiniband/hw/irdma/main.c new file +4 -0
+"Maximum VF count: 0-32 32=default");
+
+static int mpa_version = 2;
+module_param(mpa_version, int, 0644); MODULE_PARM_DESC(mpa_version,

drivers/infiniband/hw/irdma/main.c line 24

Quoted 10 lines
drivers/infiniband/hw/irdma/main.c new file +10 -0
+"MPA version: deprecated parameter");
+
+static int push_mode;
+module_param(push_mode, int, 0644);
+MODULE_PARM_DESC(push_mode, "Low latency mode: deprecated
+parameter");
+
+static int debug;
+module_param(debug, int, 0644);
+MODULE_PARM_DESC(debug, "debug flags: deprecated parameter");

Generally no to module parameters

Quoted 3 lines

Agree. But these are module params that existed in i40iw. And irdma replaces i40iw and has a module alias for it.

Maybe use this opportunity and ditch "deprecated" module parameters?

Thanks

#29 lore raw

Re: [PATCH v3 05/20] RDMA/irdma: Add driver framework definitions

Quoted 12 lines · Leon Romanovsky
Quoted 1 line · Shiraz
Quoted 1 line

Subject: Re: [PATCH v3 05/20] RDMA/irdma: Add driver framework definitions

<...>

Quoted 7 lines
Quoted 6 lines

drivers/infiniband/hw/irdma/irdma_if.c line 245

Quoted 3 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +2 -0
+		ldev->ops->reg_for_notification(ldev, &events);
+	dev_info(rfdev_to_dev(dev), "IRDMA VSI Open Successful");

Lets not do this kind of logging..

Quoted 7 lines
Quoted 4 lines

There is some dev_info which should be cleaned up to dev_dbg.
But logging this info is useful to know that this functions VSI (and associated ibdev) is up and reading for RDMA traffic.
Is info logging to be avoided altogether?

Will function tracer (ftrace) output be sufficient here?
https://www.kernel.org/doc/html/latest/trace/ftrace.html

Yes it should.

#30 lore raw

Re: [PATCH v3 05/20] RDMA/irdma: Add driver framework definitions

In reply to Saleem, Shiraz
Quoted 9 lines · Shiraz
Quoted 8 lines

drivers/infiniband/hw/irdma/i40iw_if.c

Quoted 5 lines
drivers/infiniband/hw/irdma/i40iw_if.c new file +5 -0
+/* client interface functions */
+static const struct i40e_client_ops i40e_ops = {
+	.open = i40iw_open,
+	.close = i40iw_close,
+	.l2_param_change = i40iw_l2param_change };

Wasn't the whole point of virtual bus to avoid stuff like this? Why isn't a client the virtual bus object and this information extended into the driver ops?

Quoted 3 lines

These are the private interface calls between lan and rdma. These ops are implemented by RDMA driver but invoked by netdev driver.

AFAIK you are supposed to provide things like this as part of your device driver ops, ie open is probe, close is unbind, etc.

Quoted 19 lines
Quoted 11 lines
Quoted 11 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +11 -0
+/**
+ * irdma_event_handler - Called by LAN driver to notify events
+ * @ldev: Peer device structure
+ * @event: event from LAN driver
+ */
+static void irdma_event_handler(struct iidc_peer_dev *ldev,
+				struct iidc_event *event)
+{
+	struct irdma_l2params l2params = {};
+	struct irdma_device *iwdev;
+	int i;
Quoted 8 lines

drivers/infiniband/hw/irdma/irdma_if.c line 126

Quoted 6 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +6 -0
+
+	iwdev = irdma_get_device(ldev->netdev);
+	if (!iwdev)
+		return;
+
+	if (test_bit(IIDC_EVENT_LINK_CHANGE, event->type)) {

Is this atomic? Why using test_bit?

Quoted 1 line

No its not. What do you suggest we use?

Just test the bit?

if (event->type & BIT(IIDC_EVENT_LINK_CHANGE)) ?

test_bit is the atomic version of those that matches atomic set_bit/clear_bit

Quoted 10 lines
Quoted 5 lines

drivers/infiniband/hw/irdma/irdma_if.c line 245

Quoted 2 lines
drivers/infiniband/hw/irdma/irdma_if.c new file +2 -0
+		ldev->ops->reg_for_notification(ldev, &events);
+	dev_info(rfdev_to_dev(dev), "IRDMA VSI Open Successful");

Lets not do this kind of logging..

There is some dev_info which should be cleaned up to dev_dbg.
But logging this info is useful to know that this functions VSI (and associated ibdev) is up and reading for RDMA traffic.
Is info logging to be avoided altogether?

Certainly not at display-by-default level. If every driver printed when it binds we'd have a mess.

Quoted 33 lines
Quoted 16 lines
Quoted 5 lines

new file mode 100644
index 000000000000..b418e76a3302
+++ b/drivers/infiniband/hw/irdma/main.c

drivers/infiniband/hw/irdma/main.c new file +1 -0
@@ -0,0 +1,630 @@
+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB

drivers/infiniband/hw/irdma/main.c line 11

Quoted 10 lines
drivers/infiniband/hw/irdma/main.c new file +10 -0
+/* Copyright (c) 2015 - 2019 Intel Corporation */ #include "main.h"
+
+/* Legacy i40iw module parameters */
+static int resource_profile;
+module_param(resource_profile, int, 0644);
+MODULE_PARM_DESC(resource_profile, "Resource Profile: 0=PF only,
+1=Weighted VF, 2=Even Distribution");
+
+static int max_rdma_vfs = 32;
+module_param(max_rdma_vfs, int, 0644);

MODULE_PARM_DESC(max_rdma_vfs,

Quoted 16 lines
Quoted 4 lines
drivers/infiniband/hw/irdma/main.c new file +4 -0
+"Maximum VF count: 0-32 32=default");
+
+static int mpa_version = 2;
+module_param(mpa_version, int, 0644); MODULE_PARM_DESC(mpa_version,

drivers/infiniband/hw/irdma/main.c line 24

Quoted 10 lines
drivers/infiniband/hw/irdma/main.c new file +10 -0
+"MPA version: deprecated parameter");
+
+static int push_mode;
+module_param(push_mode, int, 0644);
+MODULE_PARM_DESC(push_mode, "Low latency mode: deprecated
+parameter");
+
+static int debug;
+module_param(debug, int, 0644);
+MODULE_PARM_DESC(debug, "debug flags: deprecated parameter");

Generally no to module parameters

Quoted 3 lines

Agree. But these are module params that existed in i40iw. And irdma replaces i40iw and has a module alias for it.

Provide non-module parameter alternatives for all of these, and it can be OK only because of the module alias, and only if Doug lets you remove i40iw.

Quoted 5 lines
Quoted 3 lines

drivers/infiniband/hw/irdma/main.c line 27

Quoted 1 line
drivers/infiniband/hw/irdma/main.c new file +1 -0
+static struct workqueue_struct *irdma_wq;

Another wq already?

This wq is used for deferred handling of irdma service tasks. Such as rebuild/recovery after reset.

We have system global wqs. Why are they not OK?

Jason

#31 lore raw

RE: [PATCH v3 05/20] RDMA/irdma: Add driver framework definitions

In reply to Jeff Kirsher
Quoted 46 lines

-----Original Message-----
From: [email protected] <[email protected]> On Behalf Of Jeff Kirsher
Sent: Monday, December 09, 2019 2:49 PM
To: [email protected]; [email protected]
Cc: Ismail, Mustafa <[email protected]>; [email protected]; linux- [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; Saleem, Shiraz <[email protected]>; Kirsher, Jeffrey T <[email protected]>
Subject: [PATCH v3 05/20] RDMA/irdma: Add driver framework definitions

From: Mustafa Ismail <[email protected]>

Register irdma as a virtbus driver binding to 'i40e' and 'ice' virtbus devices added from their respective netdev drivers for each PF. During irdma probe(), the gen-specific netdev peer device is obtained from virtbus device to establish an interface and initialize the HW.

(quoted diff) +2 -0
+enum irdma_dl_param_id {
+	IRDMA_DEVLINK_PARAM_ID_BASE =

DEVLINK_PARAM_GENERIC_ID_MAX,

drivers/infiniband/hw/irdma/main.c new file +8 -0
+	IRDMA_DEVLINK_PARAM_ID_LIMITS_SELECTOR,
+	IRDMA_DEVLINK_PARAM_ID_UPLOAD_CONTEXT,
+	IRDMA_DEVLINK_PARAM_ID_ROCE_ENABLE,
+};
+
+static const struct devlink_param irdma_devlink_params[] = {
+	/* Common */
+

DEVLINK_PARAM_DRIVER(IRDMA_DEVLINK_PARAM_ID_LIMITS_SELECT
OR,

(quoted diff) +1 -0
+			     "resource_limits_selector",

DEVLINK_PARAM_TYPE_U8,

drivers/infiniband/hw/irdma/main.c new file +3 -0
+			      BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
+			      NULL, NULL, irdma_devlink_rsrc_limits_validate),
+

DEVLINK_PARAM_DRIVER(IRDMA_DEVLINK_PARAM_ID_UPLOAD_CON
TEXT,

drivers/infiniband/hw/irdma/main.c new file +2 -0
+			     "upload_context", DEVLINK_PARAM_TYPE_BOOL,
+			     BIT(DEVLINK_PARAM_CMODE_RUNTIME),

drivers/infiniband/hw/irdma/main.c line 315

Quoted 10 lines
drivers/infiniband/hw/irdma/main.c new file +10 -0
+			     irdma_devlink_upload_ctx_get,
+			     irdma_devlink_upload_ctx_set, NULL),
+#define IRDMA_DL_COMMON_PARAMS_ARRAY_SZ 2
+	/* GEN_2 only */
+	DEVLINK_PARAM_DRIVER(IRDMA_DEVLINK_PARAM_ID_ROCE_ENABLE,
+			     "roce_enable", DEVLINK_PARAM_TYPE_BOOL,
+			      BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
+			      NULL, NULL, NULL),
+};
+

Instead of adding a driver specific "roce_enable", use the generic parameter "enable_roce".

Thanks,
Jake

#32 lore raw

PATCH v3 6/20 RDMA/irdma: Implement device initialization definitions

In reply to Jeff Kirsher

From: Mustafa Ismail <[email protected]>

Implement device initialization routines, interrupt set-up, and allocate object bit-map tracking structures. Also, add device specific attributes and register definitions.

---
 drivers/infiniband/hw/irdma/hw.c        | 2596 +++++++++++++++++++++++
 drivers/infiniband/hw/irdma/i40iw_hw.c  |  209 ++
 drivers/infiniband/hw/irdma/i40iw_hw.h  |  162 ++
 drivers/infiniband/hw/irdma/icrdma_hw.c |   74 +
 drivers/infiniband/hw/irdma/icrdma_hw.h |   62 +
 5 files changed, 3103 insertions(+)
 create mode 100644 drivers/infiniband/hw/irdma/hw.c
 create mode 100644 drivers/infiniband/hw/irdma/i40iw_hw.c
 create mode 100644 drivers/infiniband/hw/irdma/i40iw_hw.h
 create mode 100644 drivers/infiniband/hw/irdma/icrdma_hw.c
 create mode 100644 drivers/infiniband/hw/irdma/icrdma_hw.h
drivers/infiniband/hw/irdma/hw.c new file +2596 -0
new file mode 100644
index 000000000000..6938b6f660e1
@@ -0,0 +1,2596 @@
+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
+/* Copyright (c) 2015 - 2019 Intel Corporation */
+#include "main.h"
+
+static struct irdma_rsrc_limits rsrc_limits_table[] = {
+	[0] = {
+		.qplimit = 4096,
+	},
+	[1] = {
+		.qplimit = 128,
+	},
+	[2] = {
+		.qplimit = 1024,
+	},
+	[3] = {
+		.qplimit = 2048,
+	},
+	[4] = {
+		.qplimit = 16384,
+	},
+	[5] = {
+		.qplimit = 65536,
+	},
+};
+
+/* types of hmc objects */
+static enum irdma_hmc_rsrc_type iw_hmc_obj_types[] = {
+	IRDMA_HMC_IW_QP,
+	IRDMA_HMC_IW_CQ,
+	IRDMA_HMC_IW_HTE,
+	IRDMA_HMC_IW_ARP,
+	IRDMA_HMC_IW_APBVT_ENTRY,
+	IRDMA_HMC_IW_MR,
+	IRDMA_HMC_IW_XF,
+	IRDMA_HMC_IW_XFFL,
+	IRDMA_HMC_IW_Q1,
+	IRDMA_HMC_IW_Q1FL,
+	IRDMA_HMC_IW_TIMER,
+	IRDMA_HMC_IW_FSIMC,
+	IRDMA_HMC_IW_FSIAV,
+	IRDMA_HMC_IW_RRF,
+	IRDMA_HMC_IW_RRFFL,
+	IRDMA_HMC_IW_HDR,
+	IRDMA_HMC_IW_MD,
+	IRDMA_HMC_IW_OOISC,
+	IRDMA_HMC_IW_OOISCFFL,
+};
+
+/**
+ * irdma_iwarp_ce_handler - handle iwarp completions
+ * @iwcq: iwarp cq receiving event
+ */
+static void irdma_iwarp_ce_handler(struct irdma_sc_cq *iwcq)
+{
+	struct irdma_cq *cq = iwcq->back_cq;
+
+	if (cq->ibcq.comp_handler)
+		cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
+}
+
+/**
+ * irdma_puda_ce_handler - handle puda completion events
+ * @rf: RDMA PCI function
+ * @cq: puda completion q for event
+ */
+static void irdma_puda_ce_handler(struct irdma_pci_f *rf,
+				  struct irdma_sc_cq *cq)
+{
+	struct irdma_sc_dev *dev = (struct irdma_sc_dev *)&rf->sc_dev;
+	enum irdma_status_code status;
+	u32 compl_error;
+
+	do {
+		status = irdma_puda_poll_cmpl(dev, cq, &compl_error);
+		if (status == IRDMA_ERR_Q_EMPTY)
+			break;
+		if (status) {
+			dev_dbg(rfdev_to_dev(dev), "ERR: puda status = %d\n",
+				status);
+			break;
+		}
+		if (compl_error) {
+			dev_dbg(rfdev_to_dev(dev),
+				"ERR: puda compl_err  =0x%x\n", compl_error);
+			break;
+		}
+	} while (1);
+
+	dev->ccq_ops->ccq_arm(cq);
+}
+
+/**
+ * irdma_process_ceq - handle ceq for completions
+ * @rf: RDMA PCI function
+ * @ceq: ceq having cq for completion
+ */
+static void irdma_process_ceq(struct irdma_pci_f *rf, struct irdma_ceq *ceq)
+{
+	struct irdma_sc_dev *dev = &rf->sc_dev;
+	struct irdma_sc_ceq *sc_ceq;
+	struct irdma_sc_cq *cq;
+
+	sc_ceq = &ceq->sc_ceq;
+	do {
+		cq = dev->ceq_ops->process_ceq(dev, sc_ceq);
+		if (!cq)
+			break;
+
+		if (cq->cq_type == IRDMA_CQ_TYPE_CQP)
+			up(&rf->cqp.cqp_compl_sem);
+		else if (cq->cq_type == IRDMA_CQ_TYPE_IWARP)
+			irdma_iwarp_ce_handler(cq);
+		else if (cq->cq_type == IRDMA_CQ_TYPE_ILQ ||
+			 cq->cq_type == IRDMA_CQ_TYPE_IEQ)
+			irdma_puda_ce_handler(rf, cq);
+	} while (1);
+}
+
+/**
+ * irdma_process_aeq - handle aeq events
+ * @rf: RDMA PCI function
+ */
+static void irdma_process_aeq(struct irdma_pci_f *rf)
+{
+	struct irdma_sc_dev *dev = &rf->sc_dev;
+	struct irdma_aeq *aeq = &rf->aeq;
+	struct irdma_sc_aeq *sc_aeq = &aeq->sc_aeq;
+	struct irdma_aeqe_info aeinfo;
+	struct irdma_aeqe_info *info = &aeinfo;
+	int ret;
+	struct irdma_qp *iwqp = NULL;
+	struct irdma_sc_cq *cq = NULL;
+	struct irdma_cq *iwcq = NULL;
+	struct irdma_sc_qp *qp = NULL;
+	struct irdma_qp_host_ctx_info *ctx_info = NULL;
+	unsigned long flags;
+
+	u32 aeqcnt = 0;
+
+	if (!sc_aeq->size)
+		return;
+
+	do {
+		memset(info, 0, sizeof(*info));
+		ret = dev->aeq_ops->get_next_aeqe(sc_aeq, info);
+		if (ret)
+			break;
+
+		aeqcnt++;
+		dev_dbg(rfdev_to_dev(dev),
+			"AEQ: ae_id = 0x%x bool qp=%d qp_id = %d\n",
+			info->ae_id, info->qp, info->qp_cq_id);
+		if (info->qp) {
+			spin_lock_irqsave(&rf->qptable_lock, flags);
+			iwqp = rf->qp_table[info->qp_cq_id];
+			if (!iwqp) {
+				spin_unlock_irqrestore(&rf->qptable_lock,
+						       flags);
+				dev_dbg(rfdev_to_dev(dev),
+					"AEQ: qp_id %d is already freed\n",
+					info->qp_cq_id);
+				continue;
+			}
+			irdma_add_ref(&iwqp->ibqp);
+			spin_unlock_irqrestore(&rf->qptable_lock, flags);
+			qp = &iwqp->sc_qp;
+			spin_lock_irqsave(&iwqp->lock, flags);
+			iwqp->hw_tcp_state = info->tcp_state;
+			iwqp->hw_iwarp_state = info->iwarp_state;
+			iwqp->last_aeq = info->ae_id;
+			spin_unlock_irqrestore(&iwqp->lock, flags);
+			ctx_info = &iwqp->ctx_info;
+			if (rdma_protocol_roce(&iwqp->iwdev->iwibdev->ibdev, 1))
+				ctx_info->roce_info->err_rq_idx_valid = true;
+			else
+				ctx_info->iwarp_info->err_rq_idx_valid = true;
+		} else {
+			if (info->ae_id != IRDMA_AE_CQ_OPERATION_ERROR)
+				continue;
+		}
+
+		switch (info->ae_id) {
+			struct irdma_cm_node *cm_node;
+		case IRDMA_AE_LLP_CONNECTION_ESTABLISHED:
+			cm_node = iwqp->cm_node;
+			if (cm_node->accept_pend) {
+				atomic_dec(&cm_node->listener->pend_accepts_cnt);
+				cm_node->accept_pend = 0;
+			}
+			iwqp->rts_ae_rcvd = 1;
+			wake_up_interruptible(&iwqp->waitq);
+			break;
+		case IRDMA_AE_LLP_FIN_RECEIVED:
+		case IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE:
+			if (qp->term_flags)
+				break;
+			if (atomic_inc_return(&iwqp->close_timer_started) == 1) {
+				iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSE_WAIT;
+				if (iwqp->hw_tcp_state == IRDMA_TCP_STATE_CLOSE_WAIT &&
+				    iwqp->ibqp_state == IB_QPS_RTS) {
+					irdma_next_iw_state(iwqp,
+							    IRDMA_QP_STATE_CLOSING,
+							    0, 0, 0);
+					irdma_cm_disconn(iwqp);
+				}
+				iwqp->cm_id->add_ref(iwqp->cm_id);
+				irdma_schedule_cm_timer(iwqp->cm_node,
+							(struct irdma_puda_buf *)iwqp,
+							IRDMA_TIMER_TYPE_CLOSE,
+							1, 0);
+			}
+			break;
+		case IRDMA_AE_LLP_CLOSE_COMPLETE:
+			if (qp->term_flags)
+				irdma_terminate_done(qp, 0);
+			else
+				irdma_cm_disconn(iwqp);
+			break;
+		case IRDMA_AE_BAD_CLOSE:
+			/* fall through */
+		case IRDMA_AE_RESET_SENT:
+			irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, 1, 0,
+					    0);
+			irdma_cm_disconn(iwqp);
+			break;
+		case IRDMA_AE_LLP_CONNECTION_RESET:
+			if (atomic_read(&iwqp->close_timer_started))
+				break;
+			irdma_cm_disconn(iwqp);
+			break;
+		case IRDMA_AE_QP_SUSPEND_COMPLETE:
+			atomic_dec(&iwqp->sc_qp.vsi->qp_suspend_reqs);
+			wake_up(&iwqp->iwdev->suspend_wq);
+			break;
+		case IRDMA_AE_TERMINATE_SENT:
+			irdma_terminate_send_fin(qp);
+			break;
+		case IRDMA_AE_LLP_TERMINATE_RECEIVED:
+			irdma_terminate_received(qp, info);
+			break;
+		case IRDMA_AE_CQ_OPERATION_ERROR:
+			dev_err(rfdev_to_dev(dev),
+				"Processing an iWARP related AE for CQ misc = 0x%04X\n",
+				info->ae_id);
+			cq = (struct irdma_sc_cq *)(unsigned long)
+			     info->compl_ctx;
+
+			iwcq = (struct irdma_cq *)cq->back_cq;
+
+			if (iwcq->ibcq.event_handler) {
+				struct ib_event ibevent;
+
+				ibevent.device = iwcq->ibcq.device;
+				ibevent.event = IB_EVENT_CQ_ERR;
+				ibevent.element.cq = &iwcq->ibcq;
+				iwcq->ibcq.event_handler(&ibevent,
+							 iwcq->ibcq.cq_context);
+			}
+			break;
+		case IRDMA_AE_LLP_DOUBT_REACHABILITY:
+		case IRDMA_AE_RESOURCE_EXHAUSTION:
+			break;
+		case IRDMA_AE_PRIV_OPERATION_DENIED:
+		case IRDMA_AE_STAG_ZERO_INVALID:
+		case IRDMA_AE_IB_RREQ_AND_Q1_FULL:
+		case IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION:
+		case IRDMA_AE_DDP_UBE_INVALID_MO:
+		case IRDMA_AE_DDP_UBE_INVALID_QN:
+		case IRDMA_AE_DDP_NO_L_BIT:
+		case IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
+		case IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE:
+		case IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST:
+		case IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
+		case IRDMA_AE_INVALID_ARP_ENTRY:
+		case IRDMA_AE_INVALID_TCP_OPTION_RCVD:
+		case IRDMA_AE_STALE_ARP_ENTRY:
+		case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
+		case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
+		case IRDMA_AE_LLP_SYN_RECEIVED:
+		case IRDMA_AE_LLP_TOO_MANY_RETRIES:
+		case IRDMA_AE_LCE_QP_CATASTROPHIC:
+		case IRDMA_AE_LCE_FUNCTION_CATASTROPHIC:
+		case IRDMA_AE_LCE_CQ_CATASTROPHIC:
+		case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:
+			if (rdma_protocol_roce(&iwqp->iwdev->iwibdev->ibdev, 1))
+				ctx_info->roce_info->err_rq_idx_valid = false;
+			else
+				ctx_info->iwarp_info->err_rq_idx_valid = false;
+			/* fall through */
+		default:
+			dev_err(rfdev_to_dev(dev),
+				"abnormal ae_id = 0x%x bool qp=%d qp_id = %d\n",
+				info->ae_id, info->qp, info->qp_cq_id);
+			if (rdma_protocol_roce(&iwqp->iwdev->iwibdev->ibdev, 1)) {
+				if (!info->sq && ctx_info->roce_info->err_rq_idx_valid) {
+					ctx_info->roce_info->err_rq_idx = info->wqe_idx;
+					ret = dev->iw_priv_qp_ops->qp_setctx_roce(&iwqp->sc_qp,
+										  iwqp->host_ctx.va,
+										  ctx_info);
+				}
+				irdma_cm_disconn(iwqp);
+				break;
+			}
+			if (!info->sq && ctx_info->iwarp_info->err_rq_idx_valid) {
+				ctx_info->iwarp_info->err_rq_idx = info->wqe_idx;
+				ctx_info->tcp_info_valid = false;
+				ctx_info->iwarp_info_valid = false;
+				ret = dev->iw_priv_qp_ops->qp_setctx(&iwqp->sc_qp,
+								     iwqp->host_ctx.va,
+								     ctx_info);
+			}
+			if (iwqp->hw_iwarp_state != IRDMA_QP_STATE_RTS &&
+			    iwqp->hw_iwarp_state != IRDMA_QP_STATE_TERMINATE) {
+				irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, 1, 0, 0);
+				irdma_cm_disconn(iwqp);
+			} else {
+				iwqp->sc_qp.term_flags = 1;
+				irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, 1, 0,
+						    0);
+				irdma_cm_disconn(iwqp);
+			}
+			break;
+		}
+		if (info->qp)
+			irdma_rem_ref(&iwqp->ibqp);
+	} while (1);
+
+	if (aeqcnt)
+		dev->aeq_ops->repost_aeq_entries(dev, aeqcnt);
+}
+
+/**
+ * irdma_enable_intr - set up device interrupts
+ * @dev: hardware control device structure
+ * @msix_id: id of the interrupt to be enabled
+ */
+static void irdma_ena_intr(struct irdma_sc_dev *dev, u32 msix_id)
+{
+	dev->irq_ops->irdma_en_irq(dev, msix_id);
+}
+
+/**
+ * irdma_dpc - tasklet for aeq and ceq 0
+ * @data: RDMA PCI function
+ */
+static void irdma_dpc(unsigned long data)
+{
+	struct irdma_pci_f *rf = (struct irdma_pci_f *)data;
+
+	if (rf->msix_shared)
+		irdma_process_ceq(rf, rf->ceqlist);
+	irdma_process_aeq(rf);
+	irdma_ena_intr(&rf->sc_dev, rf->iw_msixtbl[0].idx);
+}
+
+/**
+ * irdma_ceq_dpc - dpc handler for CEQ
+ * @data: data points to CEQ
+ */
+static void irdma_ceq_dpc(unsigned long data)
+{
+	struct irdma_ceq *iwceq = (struct irdma_ceq *)data;
+	struct irdma_pci_f *rf = iwceq->rf;
+
+	irdma_process_ceq(rf, iwceq);
+	irdma_ena_intr(&rf->sc_dev, iwceq->msix_idx);
+}
+
+/**
+ * irdma_save_msix_info - copy msix vector information to iwarp device
+ * @rf: RDMA PCI function
+ *
+ * Allocate iwdev msix table and copy the ldev msix info to the table
+ * Return 0 if successful, otherwise return error
+ */
+static enum irdma_status_code irdma_save_msix_info(struct irdma_pci_f *rf)
+{
+	struct irdma_priv_ldev *ldev = &rf->ldev;
+	struct irdma_qvlist_info *iw_qvlist;
+	struct irdma_qv_info *iw_qvinfo;
+	struct msix_entry *pmsix;
+	u32 ceq_idx;
+	u32 i;
+	u32 size;
+
+	if (!ldev->msix_count) {
+		pr_err("No MSI-X vectors for RDMA\n");
+		return IRDMA_ERR_CFG;
+	}
+
+	rf->msix_count = ldev->msix_count;
+	size = sizeof(struct irdma_msix_vector) * rf->msix_count;
+	size += sizeof(struct irdma_qvlist_info);
+	size += sizeof(struct irdma_qv_info) * rf->msix_count - 1;
+	rf->iw_msixtbl = kzalloc(size, GFP_KERNEL);
+	if (!rf->iw_msixtbl)
+		return IRDMA_ERR_NO_MEMORY;
+
+	rf->iw_qvlist = (struct irdma_qvlist_info *)
+			(&rf->iw_msixtbl[rf->msix_count]);
+	iw_qvlist = rf->iw_qvlist;
+	iw_qvinfo = iw_qvlist->qv_info;
+	iw_qvlist->num_vectors = rf->msix_count;
+	if (rf->msix_count <= num_online_cpus())
+		rf->msix_shared = true;
+
+	for (i = 0, ceq_idx = 0, pmsix = ldev->msix_entries; i < rf->msix_count;
+	     i++, iw_qvinfo++, pmsix++) {
+		rf->iw_msixtbl[i].idx = pmsix->entry;
+		rf->iw_msixtbl[i].irq = pmsix->vector;
+		rf->iw_msixtbl[i].cpu_affinity = ceq_idx;
+		if (!i) {
+			iw_qvinfo->aeq_idx = 0;
+			if (rf->msix_shared)
+				iw_qvinfo->ceq_idx = ceq_idx++;
+			else
+				iw_qvinfo->ceq_idx = IRDMA_Q_INVALID_IDX;
+		} else {
+			iw_qvinfo->aeq_idx = IRDMA_Q_INVALID_IDX;
+			iw_qvinfo->ceq_idx = ceq_idx++;
+		}
+		iw_qvinfo->itr_idx = 3;
+		iw_qvinfo->v_idx = rf->iw_msixtbl[i].idx;
+	}
+
+	return 0;
+}
+
+/**
+ * irdma_irq_handler - interrupt handler for aeq and ceq0
+ * @irq: Interrupt request number
+ * @data: RDMA PCI function
+ */
+static irqreturn_t irdma_irq_handler(int irq, void *data)
+{
+	struct irdma_pci_f *rf = data;
+
+	tasklet_schedule(&rf->dpc_tasklet);
+
+	return IRQ_HANDLED;
+}
+
+/**
+ * irdma_ceq_handler - interrupt handler for ceq
+ * @irq: interrupt request number
+ * @data: ceq pointer
+ */
+static irqreturn_t irdma_ceq_handler(int irq, void *data)
+{
+	struct irdma_ceq *iwceq = data;
+
+	if (iwceq->irq != irq)
+		dev_err(rfdev_to_dev(&iwceq->rf->sc_dev),
+			"expected irq = %d received irq = %d\n", iwceq->irq,
+			irq);
+	tasklet_schedule(&iwceq->dpc_tasklet);
+
+	return IRQ_HANDLED;
+}
+
+/**
+ * irdma_destroy_irq - destroy device interrupts
+ * @rf: RDMA PCI function
+ * @msix_vec: msix vector to disable irq
+ * @dev_id: parameter to pass to free_irq (used during irq setup)
+ *
+ * The function is called when destroying aeq/ceq
+ */
+static void irdma_destroy_irq(struct irdma_pci_f *rf,
+			      struct irdma_msix_vector *msix_vec, void *dev_id)
+{
+	struct irdma_sc_dev *dev = &rf->sc_dev;
+
+	dev->irq_ops->irdma_dis_irq(dev, msix_vec->idx);
+	irq_set_affinity_hint(msix_vec->irq, NULL);
+	free_irq(msix_vec->irq, dev_id);
+}
+
+/**
+ * irdma_destroy_cqp  - destroy control qp
+ * @rf: RDMA PCI function
+ * @free_hwcqp: 1 if hw cqp should be freed
+ *
+ * Issue destroy cqp request and
+ * free the resources associated with the cqp
+ */
+static void irdma_destroy_cqp(struct irdma_pci_f *rf, bool free_hwcqp)
+{
+	enum irdma_status_code status = 0;
+	struct irdma_sc_dev *dev = &rf->sc_dev;
+	struct irdma_cqp *cqp = &rf->cqp;
+
+	if (free_hwcqp && dev->cqp_ops->cqp_destroy)
+		status = dev->cqp_ops->cqp_destroy(dev->cqp);
+	if (status)
+		dev_dbg(rfdev_to_dev(dev), "ERR: Destroy CQP failed %d\n",
+			status);
+
+	irdma_cleanup_pending_cqp_op(rf);
+	dma_free_coherent(hw_to_dev(dev->hw), cqp->sq.size, cqp->sq.va,
+			  cqp->sq.pa);
+	cqp->sq.va = NULL;
+	kfree(cqp->scratch_array);
+	cqp->scratch_array = NULL;
+	kfree(cqp->cqp_requests);
+	cqp->cqp_requests = NULL;
+}
+
+/**
+ * irdma_destroy_aeq - destroy aeq
+ * @rf: RDMA PCI function
+ *
+ * Issue a destroy aeq request and
+ * free the resources associated with the aeq
+ * The function is called during driver unload
+ */
+static void irdma_destroy_aeq(struct irdma_pci_f *rf)
+{
+	enum irdma_status_code status = IRDMA_ERR_NOT_READY;
+	struct irdma_sc_dev *dev = &rf->sc_dev;
+	struct irdma_aeq *aeq = &rf->aeq;
+
+	if (!rf->msix_shared)
+		irdma_destroy_irq(rf, rf->iw_msixtbl, (void *)rf);
+	if (rf->reset)
+		goto exit;
+
+	if (!dev->aeq_ops->aeq_destroy(&aeq->sc_aeq, 0, 1))
+		status = dev->aeq_ops->aeq_destroy_done(&aeq->sc_aeq);
+	if (status)
+		dev_dbg(rfdev_to_dev(dev), "ERR: Destroy AEQ failed %d\n",
+			status);
+
+exit:
+	dma_free_coherent(hw_to_dev(dev->hw), aeq->mem.size, aeq->mem.va,
+			  aeq->mem.pa);
+	aeq->mem.va = NULL;
+}
+
+/**
+ * irdma_destroy_ceq - destroy ceq
+ * @rf: RDMA PCI function
+ * @iwceq: ceq to be destroyed
+ *
+ * Issue a destroy ceq request and
+ * free the resources associated with the ceq
+ */
+static void irdma_destroy_ceq(struct irdma_pci_f *rf, struct irdma_ceq *iwceq)
+{
+	enum irdma_status_code status;
+	struct irdma_sc_dev *dev = &rf->sc_dev;
+
+	if (rf->reset)
+		goto exit;
+
+	status = dev->ceq_ops->ceq_destroy(&iwceq->sc_ceq, 0, 1);
+	if (status) {
+		dev_dbg(rfdev_to_dev(dev),
+			"ERR: CEQ destroy command failed %d\n", status);
+		goto exit;
+	}
+
+	status = dev->ceq_ops->cceq_destroy_done(&iwceq->sc_ceq);
+	if (status)
+		dev_dbg(rfdev_to_dev(dev),
+			"ERR: CEQ destroy completion failed %d\n", status);
+exit:
+	dma_free_coherent(hw_to_dev(dev->hw), iwceq->mem.size, iwceq->mem.va,
+			  iwceq->mem.pa);
+	iwceq->mem.va = NULL;
+}
+
+/**
+ * irdma_del_ceq_0 - destroy ceq 0
+ * @rf: RDMA PCI function
+ *
+ * Disable the ceq 0 interrupt and destroy the ceq 0
+ */
+static void irdma_del_ceq_0(struct irdma_pci_f *rf)
+{
+	struct irdma_ceq *iwceq = rf->ceqlist;
+	struct irdma_msix_vector *msix_vec;
+
+	if (rf->msix_shared) {
+		msix_vec = &rf->iw_msixtbl[0];
+		irdma_destroy_irq(rf, msix_vec, (void *)rf);
+	} else {
+		msix_vec = &rf->iw_msixtbl[1];
+		irdma_destroy_irq(rf, msix_vec, (void *)iwceq);
+	}
+	irdma_destroy_ceq(rf, iwceq);
+	rf->sc_dev.ceq_valid = false;
+	rf->ceqs_count = 0;
+}
+
+/**
+ * irdma_del_ceqs - destroy all ceq's except CEQ 0
+ * @rf: RDMA PCI function
+ *
+ * Go through all of the device ceq's, except 0, and for each
+ * ceq disable the ceq interrupt and destroy the ceq
+ */
+static void irdma_del_ceqs(struct irdma_pci_f *rf)
+{
+	struct irdma_ceq *iwceq = &rf->ceqlist[1];
+	struct irdma_msix_vector *msix_vec;
+	u32 i = 0;
+
+	if (rf->msix_shared)
+		msix_vec = &rf->iw_msixtbl[1];
+	else
+		msix_vec = &rf->iw_msixtbl[2];
+
+	for (i = 1; i < rf->ceqs_count; i++, msix_vec++, iwceq++) {
+		irdma_destroy_irq(rf, msix_vec, (void *)iwceq);
+		irdma_cqp_ceq_cmd(&rf->sc_dev, &iwceq->sc_ceq,
+				  IRDMA_OP_CEQ_DESTROY);
+		dma_free_coherent(hw_to_dev(rf->sc_dev.hw), iwceq->mem.size,
+				  iwceq->mem.va, iwceq->mem.pa);
+		iwceq->mem.va = NULL;
+	}
+	rf->ceqs_count = 1;
+}
+
+/**
+ * irdma_destroy_ccq - destroy control cq
+ * @rf: RDMA PCI function
+ *
+ * Issue destroy ccq request and
+ * free the resources associated with the ccq
+ */
+static void irdma_destroy_ccq(struct irdma_pci_f *rf)
+{
+	struct irdma_sc_dev *dev = &rf->sc_dev;
+	struct irdma_ccq *ccq = &rf->ccq;
+	enum irdma_status_code status = 0;
+
+	if (!rf->reset)
+		status = dev->ccq_ops->ccq_destroy(dev->ccq, 0, true);
+	if (status)
+		dev_dbg(rfdev_to_dev(dev), "ERR: CCQ destroy failed %d\n",
+			status);
+	dma_free_coherent(hw_to_dev(dev->hw), ccq->mem_cq.size,
+			  ccq->mem_cq.va, ccq->mem_cq.pa);
+	ccq->mem_cq.va = NULL;
+}
+
+/**
+ * irdma_close_hmc_objects_type - delete hmc objects of a given type
+ * @dev: iwarp device
+ * @obj_type: the hmc object type to be deleted
+ * @hmc_info: host memory info struct
+ * @privileged: permission to close HMC objects
+ * @reset: true if called before reset
+ */
+static void irdma_close_hmc_objects_type(struct irdma_sc_dev *dev,
+					 enum irdma_hmc_rsrc_type obj_type,
+					 struct irdma_hmc_info *hmc_info,
+					 bool privileged, bool reset)
+{
+	struct irdma_hmc_del_obj_info info = {};
+
+	info.hmc_info = hmc_info;
+	info.rsrc_type = obj_type;
+	info.count = hmc_info->hmc_obj[obj_type].cnt;
+	info.privileged = privileged;
+	if (dev->hmc_ops->del_hmc_object(dev, &info, reset))
+		dev_dbg(rfdev_to_dev(dev),
+			"ERR: del HMC obj of type %d failed\n", obj_type);
+}
+
+/**
+ * irdma_del_hmc_objects - remove all device hmc objects
+ * @dev: iwarp device
+ * @hmc_info: hmc_info to free
+ * @privileged: permission to delete HMC objects
+ * @reset: true if called before reset
+ * @vers: hardware version
+ */
+static void irdma_del_hmc_objects(struct irdma_sc_dev *dev,
+				  struct irdma_hmc_info *hmc_info, bool privileged,
+				  bool reset, enum irdma_vers vers)
+{
+	unsigned int i;
+
+	for (i = 0; i < IW_HMC_OBJ_TYPE_NUM; i++) {
+		if (dev->hmc_info->hmc_obj[iw_hmc_obj_types[i]].cnt)
+			irdma_close_hmc_objects_type(dev, iw_hmc_obj_types[i],
+						     hmc_info, privileged, reset);
+		if (vers == IRDMA_GEN_1 && i == IRDMA_HMC_IW_TIMER)
+			break;
+	}
+}
+
+/**
+ * irdma_create_hmc_obj_type - create hmc object of a given type
+ * @dev: hardware control device structure
+ * @info: information for the hmc object to create
+ */
+static enum irdma_status_code
+irdma_create_hmc_obj_type(struct irdma_sc_dev *dev,
+			  struct irdma_hmc_create_obj_info *info)
+{
+	return dev->hmc_ops->create_hmc_object(dev, info);
+}
+
+/**
+ * irdma_create_hmc_objs - create all hmc objects for the device
+ * @rf: RDMA PCI function
+ * @privileged: permission to create HMC objects
+ * @vers: HW version
+ *
+ * Create the device hmc objects and allocate hmc pages
+ * Return 0 if successful, otherwise clean up and return error
+ */
+static enum irdma_status_code
+irdma_create_hmc_objs(struct irdma_pci_f *rf, bool privileged, enum irdma_vers vers)
+{
+	struct irdma_sc_dev *dev = &rf->sc_dev;
+	struct irdma_hmc_create_obj_info info = {};
+	enum irdma_status_code status = 0;
+	int i;
+
+	info.hmc_info = dev->hmc_info;
+	info.privileged = privileged;
+	info.entry_type = rf->sd_type;
+
+	for (i = 0; i < IW_HMC_OBJ_TYPE_NUM; i++) {
+		if (dev->hmc_info->hmc_obj[iw_hmc_obj_types[i]].cnt) {
+			info.rsrc_type = iw_hmc_obj_types[i];
+			info.count = dev->hmc_info->hmc_obj[info.rsrc_type].cnt;
+			info.add_sd_cnt = 0;
+			status = irdma_create_hmc_obj_type(dev, &info);
+			if (status) {
+				dev_dbg(rfdev_to_dev(dev),
+					"ERR: create obj type %d status = %d\n",
+					iw_hmc_obj_types[i], status);
+				break;
+			}
+		}
+		if (vers == IRDMA_GEN_1 && i == IRDMA_HMC_IW_TIMER)
+			break;
+	}
+
+	if (!status)
+		return dev->hmc_ops->static_hmc_pages_allocated(dev->cqp, 0,
+								dev->hmc_fn_id,
+								true, true);
+
+	while (i) {
+		i--;
+		/* destroy the hmc objects of a given type */
+		irdma_close_hmc_objects_type(dev, iw_hmc_obj_types[i],
+					     dev->hmc_info, privileged, false);
+	}
+
+	return status;
+}
+
+/**
+ * irdma_obj_aligned_mem - get aligned memory from device allocated memory
+ * @rf: RDMA PCI function
+ * @memptr: points to the memory addresses
+ * @size: size of memory needed
+ * @mask: mask for the aligned memory
+ *
+ * Get aligned memory of the requested size and
+ * update the memptr to point to the new aligned memory
+ * Return 0 if successful, otherwise return no memory error
+ */
+static enum irdma_status_code
+irdma_obj_aligned_mem(struct irdma_pci_f *rf, struct irdma_dma_mem *memptr,
+		      u32 size, u32 mask)
+{
+	unsigned long va, newva;
+	unsigned long extra;
+
+	va = (unsigned long)rf->obj_next.va;
+	newva = va;
+	if (mask)
+		newva = ALIGN(va, (unsigned long)mask + 1ULL);
+	extra = newva - va;
+	memptr->va = (u8 *)va + extra;
+	memptr->pa = rf->obj_next.pa + extra;
+	memptr->size = size;
+	if ((memptr->va + size) > (rf->obj_mem.va + rf->obj_mem.size))
+		return IRDMA_ERR_NO_MEMORY;
+
+	rf->obj_next.va = memptr->va + size;
+	rf->obj_next.pa = memptr->pa + size;
+
+	return 0;
+}
+
+/**
+ * irdma_create_cqp - create control qp
+ * @rf: RDMA PCI function
+ *
+ * Return 0, if the cqp and all the resources associated with it
+ * are successfully created, otherwise return error
+ */
+static enum irdma_status_code irdma_create_cqp(struct irdma_pci_f *rf)
+{
+	enum irdma_status_code status;
+	u32 sqsize = IRDMA_CQP_SW_SQSIZE_2048;
+	struct irdma_dma_mem mem;
+	struct irdma_sc_dev *dev = &rf->sc_dev;
+	struct irdma_cqp_init_info cqp_init_info = {};
+	struct irdma_cqp *cqp = &rf->cqp;
+	u16 maj_err, min_err;
+	int i;
+
+	cqp->cqp_requests = kcalloc(sqsize, sizeof(*cqp->cqp_requests), GFP_KERNEL);
+	if (!cqp->cqp_requests)
+		return IRDMA_ERR_NO_MEMORY;
+
+	cqp->scratch_array = kcalloc(sqsize, sizeof(*cqp->scratch_array), GFP_KERNEL);
+	if (!cqp->scratch_array) {
+		kfree(cqp->cqp_requests);
+		return IRDMA_ERR_NO_MEMORY;
+	}
+
+	dev->cqp = &cqp->sc_cqp;
+	dev->cqp->dev = dev;
+	cqp->sq.size = ALIGN(sizeof(struct irdma_cqp_sq_wqe) * sqsize,
+			     IRDMA_CQP_ALIGNMENT);
+	cqp->sq.va = dma_alloc_coherent(hw_to_dev(dev->hw), cqp->sq.size,
+					&cqp->sq.pa, GFP_KERNEL);
+	if (!cqp->sq.va) {
+		kfree(cqp->scratch_array);
+		kfree(cqp->cqp_requests);
+		return IRDMA_ERR_NO_MEMORY;
+	}
+
+	status = irdma_obj_aligned_mem(rf, &mem, sizeof(struct irdma_cqp_ctx),
+				       IRDMA_HOST_CTX_ALIGNMENT_M);
+	if (status)
+		goto exit;
+
+	dev->cqp->host_ctx_pa = mem.pa;
+	dev->cqp->host_ctx = mem.va;
+	/* populate the cqp init info */
+	cqp_init_info.dev = dev;
+	cqp_init_info.sq_size = sqsize;
+	cqp_init_info.sq = cqp->sq.va;
+	cqp_init_info.sq_pa = cqp->sq.pa;
+	cqp_init_info.host_ctx_pa = mem.pa;
+	cqp_init_info.host_ctx = mem.va;
+	cqp_init_info.hmc_profile = rf->rsrc_profile;
+	cqp_init_info.ena_vf_count = rf->max_rdma_vfs;
+	cqp_init_info.scratch_array = cqp->scratch_array;
+	cqp_init_info.disable_packed = true;
+	cqp_init_info.protocol_used = rf->protocol_used;
+	status = dev->cqp_ops->cqp_init(dev->cqp, &cqp_init_info);
+	if (status) {
+		dev_dbg(rfdev_to_dev(dev), "ERR: cqp init status %d\n",
+			status);
+		goto exit;
+	}
+
+	status = dev->cqp_ops->cqp_create(dev->cqp, &maj_err, &min_err);
+	if (status) {
+		dev_dbg(rfdev_to_dev(dev),
+			"ERR: cqp create failed - status %d maj_err %d min_err %d\n",
+			status, maj_err, min_err);
+		goto exit;
+	}
+
+	spin_lock_init(&cqp->req_lock);
+	spin_lock_init(&cqp->compl_lock);
+	INIT_LIST_HEAD(&cqp->cqp_avail_reqs);
+	INIT_LIST_HEAD(&cqp->cqp_pending_reqs);
+	sema_init(&cqp->cqp_compl_sem, 0);
+
+	/* init the waitqueue of the cqp_requests and add them to the list */
+	for (i = 0; i < sqsize; i++) {
+		init_waitqueue_head(&cqp->cqp_requests[i].waitq);
+		list_add_tail(&cqp->cqp_requests[i].list, &cqp->cqp_avail_reqs);
+	}
+	init_waitqueue_head(&cqp->remove_wq);
+	return 0;
+
+exit:
+	irdma_destroy_cqp(rf, false);
+
+	return status;
+}
+
+/**
+ * irdma_create_ccq - create control cq
+ * @rf: RDMA PCI function
+ *
+ * Return 0, if the ccq and the resources associated with it
+ * are successfully created, otherwise return error
+ */
+static enum irdma_status_code irdma_create_ccq(struct irdma_pci_f *rf)
+{
+	struct irdma_sc_dev *dev = &rf->sc_dev;
+	enum irdma_status_code status;
+	struct irdma_ccq_init_info info = {};
+	struct irdma_ccq *ccq = &rf->ccq;
+
+	dev->ccq = &ccq->sc_cq;
+	dev->ccq->dev = dev;
+	info.dev = dev;
+	ccq->shadow_area.size = sizeof(struct irdma_cq_shadow_area);
+	ccq->mem_cq.size = ALIGN(sizeof(struct irdma_cqe) * IW_CCQ_SIZE,
+				 IRDMA_CQ0_ALIGNMENT);
+	ccq->mem_cq.va = dma_alloc_coherent(hw_to_dev(dev->hw),
+					    ccq->mem_cq.size, &ccq->mem_cq.pa,
+					    GFP_KERNEL);
+	if (!ccq->mem_cq.va)
+		return IRDMA_ERR_NO_MEMORY;
+
+	status = irdma_obj_aligned_mem(rf, &ccq->shadow_area,
+				       ccq->shadow_area.size,
+				       IRDMA_SHADOWAREA_M);
+	if (status)
+		goto exit;
+
+	ccq->sc_cq.back_cq = (void *)ccq;
+	/* populate the ccq init info */
+	info.cq_base = ccq->mem_cq.va;
+	info.cq_pa = ccq->mem_cq.pa;
+	info.num_elem = IW_CCQ_SIZE;
+	info.shadow_area = ccq->shadow_area.va;
+	info.shadow_area_pa = ccq->shadow_area.pa;
+	info.ceqe_mask = false;
+	info.ceq_id_valid = true;
+	info.shadow_read_threshold = 16;
+	info.vsi = &rf->default_vsi;
+	status = dev->ccq_ops->ccq_init(dev->ccq, &info);
+	if (!status)
+		status = dev->ccq_ops->ccq_create(dev->ccq, 0, true, true);
+exit:
+	if (status) {
+		dma_free_coherent(hw_to_dev(dev->hw), ccq->mem_cq.size,
+				  ccq->mem_cq.va, ccq->mem_cq.pa);
+		ccq->mem_cq.va = NULL;
+	}
+
+	return status;
+}
+
+/**
+ * irdma_alloc_set_mac - set up a mac address table entry
+ * @iwdev: device
+ *
+ * Allocate a mac ip entry and add it to the hw table Return 0
+ * if successful, otherwise return error
+ */
+static enum irdma_status_code irdma_alloc_set_mac(struct irdma_device *iwdev)
+{
+	enum irdma_status_code status;
+
+	status = irdma_alloc_local_mac_entry(iwdev->rf,
+					     &iwdev->mac_ip_table_idx);
+	if (!status) {
+		status = irdma_add_local_mac_entry(iwdev->rf,
+						   (u8 *)iwdev->netdev->dev_addr,
+						   (u8)iwdev->mac_ip_table_idx);
+		if (status)
+			irdma_del_local_mac_entry(iwdev->rf,
+						  (u8)iwdev->mac_ip_table_idx);
+	}
+	return status;
+}
+
+/**
+ * irdma_configure_ceq_vector - set up the msix interrupt vector for ceq
+ * @rf: RDMA PCI function
+ * @iwceq: ceq associated with the vector
+ * @ceq_id: the id number of the iwceq
+ * @msix_vec: interrupt vector information
+ *
+ * Allocate interrupt resources and enable irq handling
+ * Return 0 if successful, otherwise return error
+ */
+static enum irdma_status_code
+irdma_cfg_ceq_vector(struct irdma_pci_f *rf, struct irdma_ceq *iwceq,
+		     u32 ceq_id, struct irdma_msix_vector *msix_vec)
+{
+	int status;
+
+	if (rf->msix_shared && !ceq_id) {
+		tasklet_init(&rf->dpc_tasklet, irdma_dpc, (unsigned long)rf);
+		status = request_irq(msix_vec->irq, irdma_irq_handler, 0,
+				     "AEQCEQ", rf);
+	} else {
+		tasklet_init(&iwceq->dpc_tasklet, irdma_ceq_dpc,
+			     (unsigned long)iwceq);
+
+		status = request_irq(msix_vec->irq, irdma_ceq_handler, 0, "CEQ",
+				     iwceq);
+	}
+
+	cpumask_clear(&msix_vec->mask);
+	cpumask_set_cpu(msix_vec->cpu_affinity, &msix_vec->mask);
+	irq_set_affinity_hint(msix_vec->irq, &msix_vec->mask);
+	if (status) {
+		dev_dbg(rfdev_to_dev(&rf->sc_dev),
+			"ERR: ceq irq config fail\n");
+		return IRDMA_ERR_CFG;
+	}
+
+	msix_vec->ceq_id = ceq_id;
+	rf->sc_dev.irq_ops->irdma_cfg_ceq(&rf->sc_dev, ceq_id, msix_vec->idx);
+
+	return 0;
+}
+
+/**
+ * irdma_configure_aeq_vector - set up the msix vector for aeq
+ * @rf: RDMA PCI function
+ *
+ * Allocate interrupt resources and enable irq handling
+ * Return 0 if successful, otherwise return error
+ */
+static enum irdma_status_code irdma_cfg_aeq_vector(struct irdma_pci_f *rf)
+{
+	struct irdma_msix_vector *msix_vec = rf->iw_msixtbl;
+	u32 ret = 0;
+
+	if (!rf->msix_shared) {
+		tasklet_init(&rf->dpc_tasklet, irdma_dpc, (unsigned long)rf);
+		ret = request_irq(msix_vec->irq, irdma_irq_handler, 0, "irdma",
+				  rf);
+	}
+	if (ret) {
+		dev_dbg(rfdev_to_dev(&rf->sc_dev),
+			"ERR: aeq irq config fail\n");
+		return IRDMA_ERR_CFG;
+	}
+
+	rf->sc_dev.irq_ops->irdma_cfg_aeq(&rf->sc_dev, msix_vec->idx);
+
+	return 0;
+}
+
+/**
+ * irdma_create_ceq - create completion event queue
+ * @rf: RDMA PCI function
+ * @iwceq: pointer to the ceq resources to be created
+ * @ceq_id: the id number of the iwceq
+ * @vsi: SC vsi struct
+ *
+ * Return 0, if the ceq and the resources associated with it
+ * are successfully created, otherwise return error
+ */
+static enum irdma_status_code irdma_create_ceq(struct irdma_pci_f *rf,
+					       struct irdma_ceq *iwceq,
+					       u32 ceq_id,
+					       struct irdma_sc_vsi *vsi)
+{
+	enum irdma_status_code status;
+	struct irdma_ceq_init_info info = {};
+	struct irdma_sc_dev *dev = &rf->sc_dev;
+	u64 scratch;
+
+	info.ceq_id = ceq_id;
+	iwceq->rf = rf;
+	iwceq->mem.size = ALIGN(sizeof(struct irdma_ceqe) * rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt,
+				IRDMA_CEQ_ALIGNMENT);
+	iwceq->mem.va = dma_alloc_coherent(hw_to_dev(dev->hw),
+					   iwceq->mem.size, &iwceq->mem.pa,
+					   GFP_KERNEL);
+	if (!iwceq->mem.va)
+		return IRDMA_ERR_NO_MEMORY;
+
+	info.ceq_id = ceq_id;
+	info.ceqe_base = iwceq->mem.va;
+	info.ceqe_pa = iwceq->mem.pa;
+	info.elem_cnt = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt;
+	iwceq->sc_ceq.ceq_id = ceq_id;
+	info.dev = dev;
+	info.vsi = vsi;
+	scratch = (uintptr_t)&rf->cqp.sc_cqp;
+	status = dev->ceq_ops->ceq_init(&iwceq->sc_ceq, &info);
+	if (!status) {
+		if (dev->ceq_valid)
+			status = irdma_cqp_ceq_cmd(&rf->sc_dev, &iwceq->sc_ceq,
+						   IRDMA_OP_CEQ_CREATE);
+		else
+			status = dev->ceq_ops->cceq_create(&iwceq->sc_ceq,
+							   scratch);
+	}
+
+	if (status) {
+		dma_free_coherent(hw_to_dev(dev->hw), iwceq->mem.size,
+				  iwceq->mem.va, iwceq->mem.pa);
+		iwceq->mem.va = NULL;
+	}
+
+	return status;
+}
+
+/**
+ * irdma_setup_ceq_0 - create CEQ 0 and it's interrupt resource
+ * @rf: RDMA PCI function
+ *
+ * Allocate a list for all device completion event queues
+ * Create the ceq 0 and configure it's msix interrupt vector
+ * Return 0, if successfully set up, otherwise return error
+ */
+static enum irdma_status_code irdma_setup_ceq_0(struct irdma_pci_f *rf)
+{
+	u32 i;
+	struct irdma_ceq *iwceq;
+	struct irdma_msix_vector *msix_vec;
+	enum irdma_status_code status = 0;
+	u32 num_ceqs;
+
+	num_ceqs = min(rf->msix_count, rf->sc_dev.hmc_fpm_misc.max_ceqs);
+	rf->ceqlist = kcalloc(num_ceqs, sizeof(*rf->ceqlist), GFP_KERNEL);
+	if (!rf->ceqlist) {
+		status = IRDMA_ERR_NO_MEMORY;
+		goto exit;
+	}
+
+	i = rf->msix_shared ? 0 : 1;
+	iwceq = &rf->ceqlist[0];
+	status = irdma_create_ceq(rf, iwceq, 0, &rf->default_vsi);
+	if (status) {
+		dev_dbg(rfdev_to_dev(&rf->sc_dev),
+			"ERR: create ceq status = %d\n", status);
+		goto exit;
+	}
+
+	msix_vec = &rf->iw_msixtbl[i];
+	iwceq->irq = msix_vec->irq;
+	iwceq->msix_idx = msix_vec->idx;
+	status = irdma_cfg_ceq_vector(rf, iwceq, 0, msix_vec);
+	if (status) {
+		irdma_destroy_ceq(rf, iwceq);
+		goto exit;
+	}
+
+	irdma_ena_intr(&rf->sc_dev, msix_vec->idx);
+	rf->ceqs_count++;
+
+exit:
+	if (status && !rf->ceqs_count) {
+		kfree(rf->ceqlist);
+		rf->ceqlist = NULL;
+		return status;
+	}
+	rf->sc_dev.ceq_valid = true;
+
+	return 0;
+}
+
+/**
+ * irdma_setup_ceqs - manage the device ceq's and their interrupt resources
+ * @rf: RDMA PCI function
+ * @vsi: VSI structure for this CEQ
+ *
+ * Allocate a list for all device completion event queues
+ * Create the ceq's and configure their msix interrupt vectors
+ * Return 0, if at least one ceq is successfully set up, otherwise return error
+ */
+static enum irdma_status_code irdma_setup_ceqs(struct irdma_pci_f *rf,
+					       struct irdma_sc_vsi *vsi)
+{
+	u32 i;
+	u32 ceq_id;
+	struct irdma_ceq *iwceq;
+	struct irdma_msix_vector *msix_vec;
+	enum irdma_status_code status = 0;
+	u32 num_ceqs;
+
+	num_ceqs = min(rf->msix_count, rf->sc_dev.hmc_fpm_misc.max_ceqs);
+	i = (rf->msix_shared) ? 1 : 2;
+	for (ceq_id = 1; i < num_ceqs; i++, ceq_id++) {
+		iwceq = &rf->ceqlist[ceq_id];
+		status = irdma_create_ceq(rf, iwceq, ceq_id, vsi);
+		if (status) {
+			dev_dbg(rfdev_to_dev(&rf->sc_dev),
+				"ERR: create ceq status = %d\n", status);
+			break;
+		}
+		msix_vec = &rf->iw_msixtbl[i];
+		iwceq->irq = msix_vec->irq;
+		iwceq->msix_idx = msix_vec->idx;
+		status = irdma_cfg_ceq_vector(rf, iwceq, ceq_id, msix_vec);
+		if (status) {
+			irdma_destroy_ceq(rf, iwceq);
+			break;
+		}
+		irdma_ena_intr(&rf->sc_dev, msix_vec->idx);
+		rf->ceqs_count++;
+	}
+
+	return status;
+}
+
+/**
+ * irdma_create_aeq - create async event queue
+ * @rf: RDMA PCI function
+ *
+ * Return 0, if the aeq and the resources associated with it
+ * are successfully created, otherwise return error
+ */
+static enum irdma_status_code irdma_create_aeq(struct irdma_pci_f *rf)
+{
+	enum irdma_status_code status;
+	struct irdma_aeq_init_info info = {};
+	struct irdma_sc_dev *dev = &rf->sc_dev;
+	struct irdma_aeq *aeq = &rf->aeq;
+	struct irdma_hmc_info *hmc_info = rf->sc_dev.hmc_info;
+	u64 scratch = 0;
+	u32 aeq_size;
+
+	aeq_size = 2 * hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt +
+		   hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt;
+	aeq->mem.size = ALIGN(sizeof(struct irdma_sc_aeqe) * aeq_size,
+			      IRDMA_AEQ_ALIGNMENT);
+	aeq->mem.va = dma_alloc_coherent(hw_to_dev(dev->hw), aeq->mem.size,
+					 &aeq->mem.pa, GFP_KERNEL);
+	if (!aeq->mem.va)
+		return IRDMA_ERR_NO_MEMORY;
+
+	info.aeqe_base = aeq->mem.va;
+	info.aeq_elem_pa = aeq->mem.pa;
+	info.elem_cnt = aeq_size;
+	info.dev = dev;
+	status = dev->aeq_ops->aeq_init(&aeq->sc_aeq, &info);
+	if (status)
+		goto exit;
+
+	status = dev->aeq_ops->aeq_create(&aeq->sc_aeq, scratch, 1);
+	if (!status)
+		status = dev->aeq_ops->aeq_create_done(&aeq->sc_aeq);
+exit:
+	if (status) {
+		dma_free_coherent(hw_to_dev(dev->hw), aeq->mem.size,
+				  aeq->mem.va, aeq->mem.pa);
+		aeq->mem.va = NULL;
+	}
+
+	return status;
+}
+
+/**
+ * irdma_setup_aeq - set up the device aeq
+ * @rf: RDMA PCI function
+ *
+ * Create the aeq and configure its msix interrupt vector
+ * Return 0 if successful, otherwise return error
+ */
+static enum irdma_status_code irdma_setup_aeq(struct irdma_pci_f *rf)
+{
+	struct irdma_sc_dev *dev = &rf->sc_dev;
+	enum irdma_status_code status;
+
+	status = irdma_create_aeq(rf);
+	if (status)
+		return status;
+
+	status = irdma_cfg_aeq_vector(rf);
+	if (status) {
+		irdma_destroy_aeq(rf);
+		return status;
+	}
+
+	if (!rf->msix_shared)
+		irdma_ena_intr(dev, rf->iw_msixtbl[0].idx);
+
+	return 0;
+}
+
+/**
+ * irdma_initialize_ilq - create iwarp local queue for cm
+ * @iwdev: iwarp device
+ *
+ * Return 0 if successful, otherwise return error
+ */
+static enum irdma_status_code irdma_initialize_ilq(struct irdma_device *iwdev)
+{
+	struct irdma_puda_rsrc_info info = {};
+	enum irdma_status_code status;
+
+	info.type = IRDMA_PUDA_RSRC_TYPE_ILQ;
+	info.cq_id = 1;
+	info.qp_id = 1;
+	info.count = 1;
+	info.pd_id = 1;
+	info.sq_size = min(iwdev->rf->max_qp / 2, (u32)32768);
+	info.rq_size = info.sq_size;
+	info.buf_size = 1024;
+	info.tx_buf_cnt = 2 * info.sq_size;
+	info.receive = irdma_receive_ilq;
+	info.xmit_complete = irdma_free_sqbuf;
+	status = irdma_puda_create_rsrc(&iwdev->vsi, &info);
+	if (status)
+		dev_dbg(rfdev_to_dev(&iwdev->rf->sc_dev),
+			"ERR: ilq create fail\n");
+
+	return status;
+}
+
+/**
+ * irdma_initialize_ieq - create iwarp exception queue
+ * @iwdev: iwarp device
+ *
+ * Return 0 if successful, otherwise return error
+ */
+static enum irdma_status_code irdma_initialize_ieq(struct irdma_device *iwdev)
+{
+	struct irdma_puda_rsrc_info info = {};
+	enum irdma_status_code status;
+
+	info.type = IRDMA_PUDA_RSRC_TYPE_IEQ;
+	info.cq_id = 2;
+	info.qp_id = iwdev->vsi.exception_lan_q;
+	info.count = 1;
+	info.pd_id = 2;
+	info.sq_size = min(iwdev->rf->max_qp / 2, (u32)32768);
+	info.rq_size = info.sq_size;
+	info.buf_size = iwdev->vsi.mtu + IRDMA_IPV4_PAD;
+	info.tx_buf_cnt = 4096;
+	status = irdma_puda_create_rsrc(&iwdev->vsi, &info);
+	if (status)
+		dev_dbg(rfdev_to_dev(&iwdev->rf->sc_dev),
+			"ERR: ieq create fail\n");
+
+	return status;
+}
+
+/**
+ * irdma_reinitialize_ieq - destroy and re-create ieq
+ * @vsi: VSI structure
+ */
+void irdma_reinitialize_ieq(struct irdma_sc_vsi *vsi)
+{
+	struct irdma_device *iwdev = vsi->back_vsi;
+	struct irdma_pci_f *rf = iwdev->rf;
+
+	irdma_puda_dele_rsrc(vsi, IRDMA_PUDA_RSRC_TYPE_IEQ, false);
+	if (irdma_initialize_ieq(iwdev)) {
+		iwdev->reset = true;
+		rf->gen_ops.request_reset(rf);
+	}
+}
+
+/**
+ * irdma_hmc_setup - create hmc objects for the device
+ * @rf: RDMA PCI function
+ *
+ * Set up the device private memory space for the number and size of
+ * the hmc objects and create the objects
+ * Return 0 if successful, otherwise return error
+ */
+static enum irdma_status_code irdma_hmc_setup(struct irdma_pci_f *rf)
+{
+	enum irdma_status_code status;
+	u32 qpcnt;
+
+	if (rf->rdma_ver == IRDMA_GEN_1)
+		qpcnt = rsrc_limits_table[rf->limits_sel].qplimit * 2;
+	else
+		qpcnt = rsrc_limits_table[rf->limits_sel].qplimit;
+
+	rf->sd_type = IRDMA_SD_TYPE_DIRECT;
+	status = irdma_cfg_fpm_val(&rf->sc_dev, qpcnt);
+	if (status)
+		return status;
+
+	status = irdma_create_hmc_objs(rf, true, rf->rdma_ver);
+
+	return status;
+}
+
+/**
+ * irdma_del_init_mem - deallocate memory resources
+ * @rf: RDMA PCI function
+ */
+static void irdma_del_init_mem(struct irdma_pci_f *rf)
+{
+	struct irdma_sc_dev *dev = &rf->sc_dev;
+
+	kfree(dev->hmc_info->sd_table.sd_entry);
+	dev->hmc_info->sd_table.sd_entry = NULL;
+	kfree(rf->mem_rsrc);
+	rf->mem_rsrc = NULL;
+	dma_free_coherent(hw_to_dev(&rf->hw), rf->obj_mem.size,
+			  rf->obj_mem.va, rf->obj_mem.pa);
+	rf->obj_mem.va = NULL;
+	if (rf->rdma_ver != IRDMA_GEN_1) {
+		kfree(rf->allocated_ws_nodes);
+		rf->allocated_ws_nodes = NULL;
+	}
+	kfree(rf->ceqlist);
+	rf->ceqlist = NULL;
+	kfree(rf->iw_msixtbl);
+	rf->iw_msixtbl = NULL;
+	kfree(rf->hmc_info_mem);
+	rf->hmc_info_mem = NULL;
+}
+
+/**
+ * irdma_initialize_dev - initialize device
+ * @rf: RDMA PCI function
+ * @ldev: lan device information
+ *
+ * Allocate memory for the hmc objects and initialize iwdev
+ * Return 0 if successful, otherwise clean up the resources
+ * and return error
+ */
+static enum irdma_status_code irdma_initialize_dev(struct irdma_pci_f *rf,
+						   struct irdma_priv_ldev *ldev)
+{
+	enum irdma_status_code status;
+	struct irdma_sc_dev *dev = &rf->sc_dev;
+	struct irdma_device_init_info info = {};
+	struct irdma_dma_mem mem;
+	u32 size;
+
+	size = sizeof(struct irdma_hmc_pble_rsrc) +
+	       sizeof(struct irdma_hmc_info) +
+	       (sizeof(struct irdma_hmc_obj_info) * IRDMA_HMC_IW_MAX);
+
+	rf->hmc_info_mem = kzalloc(size, GFP_KERNEL);
+	if (!rf->hmc_info_mem)
+		return IRDMA_ERR_NO_MEMORY;
+
+	rf->pble_rsrc = (struct irdma_hmc_pble_rsrc *)rf->hmc_info_mem;
+	dev->hmc_info = &rf->hw.hmc;
+	dev->hmc_info->hmc_obj = (struct irdma_hmc_obj_info *)
+				 (rf->pble_rsrc + 1);
+
+	status = irdma_obj_aligned_mem(rf, &mem, IRDMA_QUERY_FPM_BUF_SIZE,
+				       IRDMA_FPM_QUERY_BUF_ALIGNMENT_M);
+	if (status)
+		goto error;
+
+	info.fpm_query_buf_pa = mem.pa;
+	info.fpm_query_buf = mem.va;
+	info.init_hw = rf->gen_ops.init_hw;
+
+	status = irdma_obj_aligned_mem(rf, &mem, IRDMA_COMMIT_FPM_BUF_SIZE,
+				       IRDMA_FPM_COMMIT_BUF_ALIGNMENT_M);
+	if (status)
+		goto error;
+
+	info.fpm_commit_buf_pa = mem.pa;
+	info.fpm_commit_buf = mem.va;
+
+	info.bar0 = rf->hw.hw_addr;
+	info.hmc_fn_id = (u8)ldev->fn_num;
+	info.privileged = !ldev->ftype;
+	info.hw = &rf->hw;
+	info.vchnl_send = NULL;
+	status = irdma_sc_ctrl_init(rf->rdma_ver, &rf->sc_dev, &info);
+	if (status)
+		goto error;
+
+	return status;
+error:
+	kfree(rf->hmc_info_mem);
+	rf->hmc_info_mem = NULL;
+
+	return status;
+}
+
+/**
+ * irdma_deinit_rt_device - clean up the device resources
+ * @iwdev: iwarp device
+ *
+ * Destroy the ib device interface, remove the mac ip entry and
+ * ipv4/ipv6 addresses, destroy the device queues and free the
+ * pble and the hmc objects
+ */
+void irdma_deinit_rt_device(struct irdma_device *iwdev)
+{
+	dev_info(rfdev_to_dev(&iwdev->rf->sc_dev), "state = %d\n",
+		 iwdev->init_state);
+
+	switch (iwdev->init_state) {
+	case RDMA_DEV_REGISTERED:
+		iwdev->iw_status = 0;
+		irdma_port_ibevent(iwdev);
+		irdma_destroy_rdma_device(iwdev->iwibdev);
+		/* fallthrough */
+	case IP_ADDR_REGISTERED:
+		if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
+			irdma_del_local_mac_entry(iwdev->rf,
+						  (u8)iwdev->mac_ip_table_idx);
+		/* fallthrough */
+	case PBLE_CHUNK_MEM:
+		/* fallthrough */
+	case CEQS_CREATED:
+		/* fallthrough */
+	case IEQ_CREATED:
+		irdma_puda_dele_rsrc(&iwdev->vsi, IRDMA_PUDA_RSRC_TYPE_IEQ,
+				     iwdev->reset);
+		/* fallthrough */
+	case ILQ_CREATED:
+		if (iwdev->create_ilq)
+			irdma_puda_dele_rsrc(&iwdev->vsi,
+					     IRDMA_PUDA_RSRC_TYPE_ILQ,
+					     iwdev->reset);
+		break;
+	default:
+		dev_warn(rfdev_to_dev(&iwdev->rf->sc_dev),
+			 "bad init_state = %d\n", iwdev->init_state);
+		break;
+	}
+
+	irdma_cleanup_cm_core(&iwdev->cm_core);
+	if (iwdev->vsi.pestat) {
+		irdma_vsi_stats_free(&iwdev->vsi);
+		kfree(iwdev->vsi.pestat);
+	}
+}
+
+/**
+ * irdma_setup_init_state - set up the initial device struct
+ * @rf: RDMA PCI function
+ *
+ * Initialize the iwarp device and its hdl information
+ * using the ldev and client information
+ * Return 0 if successful, otherwise return error
+ */
+static enum irdma_status_code irdma_setup_init_state(struct irdma_pci_f *rf)
+{
+	struct irdma_priv_ldev *ldev = &rf->ldev;
+	enum irdma_status_code status;
+
+	status = irdma_save_msix_info(rf);
+	if (status)
+		return status;
+
+	rf->hw.pdev = rf->pdev;
+	rf->obj_mem.size = ALIGN(8192, IRDMA_HW_PAGE_SIZE);
+	rf->obj_mem.va = dma_alloc_coherent(hw_to_dev(&rf->hw),
+					    rf->obj_mem.size, &rf->obj_mem.pa,
+					    GFP_KERNEL);
+	if (!rf->obj_mem.va) {
+		kfree(rf->iw_msixtbl);
+		rf->iw_msixtbl = NULL;
+		return IRDMA_ERR_NO_MEMORY;
+	}
+
+	rf->obj_next = rf->obj_mem;
+	rf->ooo = false;
+	init_waitqueue_head(&rf->vchnl_waitq);
+
+	status = irdma_initialize_dev(rf, ldev);
+	if (status) {
+		kfree(rf->iw_msixtbl);
+		dma_free_coherent(hw_to_dev(&rf->hw), rf->obj_mem.size,
+				  rf->obj_mem.va, rf->obj_mem.pa);
+		rf->obj_mem.va = NULL;
+		rf->iw_msixtbl = NULL;
+	}
+
+	return status;
+}
+
+/**
+ * irdma_get_used_rsrc - determine resources used internally
+ * @iwdev: iwarp device
+ *
+ * Called at the end of open to get all internal allocations
+ */
+static void irdma_get_used_rsrc(struct irdma_device *iwdev)
+{
+	iwdev->rf->used_pds = find_next_zero_bit(iwdev->rf->allocated_pds,
+						 iwdev->rf->max_pd, 0);
+	iwdev->rf->used_qps = find_next_zero_bit(iwdev->rf->allocated_qps,
+						 iwdev->rf->max_qp, 0);
+	iwdev->rf->used_cqs = find_next_zero_bit(iwdev->rf->allocated_cqs,
+						 iwdev->rf->max_cq, 0);
+	iwdev->rf->used_mrs = find_next_zero_bit(iwdev->rf->allocated_mrs,
+						 iwdev->rf->max_mr, 0);
+}
+
+/**
+ * irdma_deinit_hw - De-initializes RDMA HW
+ * @rf: RDMA device information
+ *
+ */
+void irdma_deinit_ctrl_hw(struct irdma_pci_f *rf)
+{
+	enum init_completion_state state = rf->init_state;
+
+	rf->init_state = INVALID_STATE;
+	if (rf->rsrc_created) {
+		irdma_destroy_pble_prm(rf->pble_rsrc);
+		irdma_del_ceqs(rf);
+		rf->rsrc_created = false;
+	}
+	switch (state) {
+	case CEQ0_CREATED:
+		irdma_del_ceq_0(rf);
+		/* fallthrough */
+	case AEQ_CREATED:
+		irdma_destroy_aeq(rf);
+		/* fallthrough */
+	case CCQ_CREATED:
+		irdma_destroy_ccq(rf);
+		/* fallthrough */
+	case HMC_OBJS_CREATED:
+		irdma_del_hmc_objects(&rf->sc_dev, rf->sc_dev.hmc_info, true,
+				      rf->reset, rf->rdma_ver);
+		/* fallthrough */
+	case CQP_CREATED:
+		if (rf->cqp.cqp_compl_thread) {
+			rf->stop_cqp_thread = true;
+			up(&rf->cqp.cqp_compl_sem);
+			kthread_stop(rf->cqp.cqp_compl_thread);
+		}
+		irdma_destroy_cqp(rf, true);
+		/* fallthrough */
+	case INITIAL_STATE:
+		irdma_del_init_mem(rf);
+		break;
+	case INVALID_STATE:
+		/* fallthrough */
+	default:
+		pr_warn("bad init_state = %d\n", rf->init_state);
+		break;
+	}
+}
+
+enum irdma_status_code irdma_rt_init_hw(struct irdma_pci_f *rf,
+					struct irdma_device *iwdev,
+					struct irdma_l2params *l2params)
+{
+	struct irdma_sc_dev *dev = &rf->sc_dev;
+	enum irdma_status_code status;
+	struct irdma_vsi_init_info vsi_info = {};
+	struct irdma_vsi_stats_info stats_info = {};
+
+	irdma_sc_rt_init(dev);
+	vsi_info.vm_vf_type = rf->ldev.ftype ? IRDMA_VF_TYPE : IRDMA_PF_TYPE;
+	vsi_info.dev = dev;
+	vsi_info.back_vsi = (void *)iwdev;
+	vsi_info.params = l2params;
+	vsi_info.pf_data_vsi_num = iwdev->vsi_num;
+	vsi_info.register_qset = rf->gen_ops.register_qset;
+	vsi_info.unregister_qset = rf->gen_ops.unregister_qset;
+	vsi_info.exception_lan_q = 2;
+	irdma_sc_vsi_init(&iwdev->vsi, &vsi_info);
+
+	status = irdma_setup_cm_core(iwdev, rf->rdma_ver);
+	if (status)
+		return status;
+
+	stats_info.pestat = kzalloc(sizeof(*stats_info.pestat), GFP_KERNEL);
+	if (!stats_info.pestat)
+		return IRDMA_ERR_NO_MEMORY;
+
+	stats_info.fcn_id = dev->hmc_fn_id;
+	status = irdma_vsi_stats_init(&iwdev->vsi, &stats_info);
+	if (status) {
+		kfree(stats_info.pestat);
+		return status;
+	}
+
+	do {
+		if (iwdev->create_ilq) {
+			status = irdma_initialize_ilq(iwdev);
+			if (status)
+				break;
+			iwdev->init_state = ILQ_CREATED;
+		}
+		status = irdma_initialize_ieq(iwdev);
+		if (status)
+			break;
+		iwdev->init_state = IEQ_CREATED;
+		if (!rf->rsrc_created) {
+			status = irdma_setup_ceqs(rf, &iwdev->vsi);
+			if (status)
+				break;
+			iwdev->init_state = CEQS_CREATED;
+
+			status = irdma_hmc_init_pble(&rf->sc_dev,
+						     rf->pble_rsrc);
+			if (status) {
+				irdma_del_ceqs(rf);
+				break;
+			}
+			spin_lock_init(&rf->pble_rsrc->pble_lock);
+			iwdev->init_state = PBLE_CHUNK_MEM;
+			rf->rsrc_created = true;
+		}
+
+		iwdev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
+					  IB_DEVICE_MEM_WINDOW |
+					  IB_DEVICE_MEM_MGT_EXTENSIONS;
+
+		if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
+			irdma_alloc_set_mac(iwdev);
+		irdma_add_ip(iwdev);
+		iwdev->init_state = IP_ADDR_REGISTERED;
+		status = irdma_register_rdma_device(iwdev);
+		if (status)
+			break;
+		iwdev->init_state = RDMA_DEV_REGISTERED;
+		irdma_port_ibevent(iwdev);
+		iwdev->iw_status = 1;
+		irdma_get_used_rsrc(iwdev);
+		init_waitqueue_head(&iwdev->suspend_wq);
+
+		return 0;
+	} while (0);
+
+	dev_err(rfdev_to_dev(dev), "VSI open FAIL status = %d last cmpl = %d\n",
+		status, iwdev->init_state);
+	irdma_deinit_rt_device(iwdev);
+
+	return status;
+}
+
+/**
+ * irdma_ctrl_init_hw - Initializes RDMA HW
+ * @rf: RDMA PCI function
+ *
+ */
+enum irdma_status_code irdma_ctrl_init_hw(struct irdma_pci_f *rf)
+{
+	struct irdma_sc_dev *dev = &rf->sc_dev;
+	enum irdma_status_code status;
+
+	do {
+		status = irdma_setup_init_state(rf);
+		if (status)
+			break;
+		rf->init_state = INITIAL_STATE;
+
+		status = irdma_create_cqp(rf);
+		if (status)
+			break;
+		rf->init_state = CQP_CREATED;
+
+		status = irdma_hmc_setup(rf);
+		if (status)
+			break;
+		rf->init_state = HMC_OBJS_CREATED;
+
+		status = irdma_initialize_hw_rsrc(rf);
+		if (status)
+			break;
+
+		status = irdma_create_ccq(rf);
+		if (status)
+			break;
+		rf->init_state = CCQ_CREATED;
+
+		status = irdma_setup_aeq(rf);
+		if (status)
+			break;
+		rf->init_state = AEQ_CREATED;
+		rf->sc_dev.feature_info[IRDMA_FEATURE_FW_INFO] = IRDMA_FW_VER_DEFAULT;
+
+		if (rf->rdma_ver != IRDMA_GEN_1)
+			status = irdma_get_rdma_features(&rf->sc_dev);
+		if (!status) {
+			u32 fw_ver = dev->feature_info[IRDMA_FEATURE_FW_INFO];
+			u8 hw_rev = dev->hw_attrs.uk_attrs.hw_rev;
+
+			if ((hw_rev == IRDMA_GEN_1 && fw_ver >= IRDMA_FW_VER_0x30010) ||
+			    (hw_rev != IRDMA_GEN_1 && fw_ver >= IRDMA_FW_VER_0x1000D))
+
+				dev->hw_attrs.uk_attrs.feature_flags |= IRDMA_FEATURE_RTS_AE |
+									IRDMA_FEATURE_CQ_RESIZE;
+		}
+		rf->cqp.cqp_compl_thread =
+			kthread_run(cqp_compl_thread, rf, "cqp_compl_thread");
+
+		status = irdma_setup_ceq_0(rf);
+		if (status)
+			break;
+		rf->init_state = CEQ0_CREATED;
+
+		rf->free_qp_wq =
+			alloc_ordered_workqueue("free_qp_wq", WQ_MEM_RECLAIM);
+		if (!rf->free_qp_wq) {
+			status = IRDMA_ERR_NO_MEMORY;
+			break;
+		}
+
+		rf->free_cqbuf_wq =
+			alloc_ordered_workqueue("free_cqbuf_wq", WQ_MEM_RECLAIM);
+		if (!rf->free_cqbuf_wq) {
+			status = IRDMA_ERR_NO_MEMORY;
+			break;
+		}
+		dev->ccq_ops->ccq_arm(dev->ccq);
+		dev_info(rfdev_to_dev(dev), "IRDMA hardware initialization successful\n");
+		return 0;
+	} while (0);
+
+	pr_err("IRDMA hardware initialization FAILED init_state=%d status=%d\n",
+	       rf->init_state, status);
+	irdma_deinit_ctrl_hw(rf);
+	return status;
+}
+
+/**
+ * irdma_initialize_hw_resources - initialize hw resource during open
+ * @rf: RDMA PCI function
+ */
+u32 irdma_initialize_hw_rsrc(struct irdma_pci_f *rf)
+{
+	unsigned long num_pds;
+	u32 rsrc_size;
+	u32 max_mr;
+	u32 max_qp;
+	u32 max_cq;
+	u32 arp_table_size;
+	u32 mrdrvbits;
+	void *rsrc_ptr;
+	u32 num_ahs;
+	u32 num_mcg;
+
+	if (rf->rdma_ver != IRDMA_GEN_1) {
+		rf->allocated_ws_nodes =
+			kcalloc(BITS_TO_LONGS(IRDMA_MAX_WS_NODES),
+				sizeof(unsigned long), GFP_KERNEL);
+		if (!rf->allocated_ws_nodes)
+			return -ENOMEM;
+
+		set_bit(0, rf->allocated_ws_nodes);
+		rf->max_ws_node_id = IRDMA_MAX_WS_NODES;
+	}
+	max_qp = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt;
+	max_cq = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt;
+	max_mr = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt;
+	arp_table_size = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].cnt;
+	rf->max_cqe = rf->sc_dev.hw_attrs.uk_attrs.max_hw_cq_size;
+	num_pds = rf->sc_dev.hw_attrs.max_hw_pds;
+	rsrc_size = sizeof(struct irdma_arp_entry) * arp_table_size;
+	rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
+	rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
+	rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
+	rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
+	rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
+	num_ahs = max_qp * 4;
+	rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(num_ahs);
+	num_mcg = max_qp;
+	rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(num_mcg);
+	rsrc_size += sizeof(struct irdma_qp **) * max_qp;
+
+	rf->mem_rsrc = kzalloc(rsrc_size, GFP_KERNEL);
+	if (!rf->mem_rsrc) {
+		kfree(rf->allocated_ws_nodes);
+		rf->allocated_ws_nodes = NULL;
+		return -ENOMEM;
+	}
+
+	rf->max_qp = max_qp;
+	rf->max_mr = max_mr;
+	rf->max_cq = max_cq;
+	rf->max_pd = num_pds;
+	rf->arp_table_size = arp_table_size;
+	rf->arp_table = (struct irdma_arp_entry *)rf->mem_rsrc;
+	rsrc_ptr = rf->mem_rsrc +
+		   (sizeof(struct irdma_arp_entry) * arp_table_size);
+	rf->max_ah = num_ahs;
+	rf->max_mcg = num_mcg;
+	rf->allocated_qps = rsrc_ptr;
+	rf->allocated_cqs = &rf->allocated_qps[BITS_TO_LONGS(max_qp)];
+	rf->allocated_mrs = &rf->allocated_cqs[BITS_TO_LONGS(max_cq)];
+	rf->allocated_pds = &rf->allocated_mrs[BITS_TO_LONGS(max_mr)];
+	rf->allocated_ahs = &rf->allocated_pds[BITS_TO_LONGS(num_pds)];
+	rf->allocated_mcgs = &rf->allocated_ahs[BITS_TO_LONGS(num_ahs)];
+	rf->allocated_arps = &rf->allocated_mcgs[BITS_TO_LONGS(num_mcg)];
+	rf->qp_table = (struct irdma_qp **)
+		       (&rf->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
+
+	set_bit(0, rf->allocated_mrs);
+	set_bit(0, rf->allocated_qps);
+	set_bit(0, rf->allocated_cqs);
+	set_bit(0, rf->allocated_pds);
+	set_bit(0, rf->allocated_arps);
+	set_bit(0, rf->allocated_ahs);
+	set_bit(0, rf->allocated_mcgs);
+	set_bit(2, rf->allocated_qps); /* qp 2 IEQ */
+	set_bit(1, rf->allocated_qps); /* qp 1 ILQ */
+	set_bit(1, rf->allocated_cqs);
+	set_bit(1, rf->allocated_pds);
+	set_bit(2, rf->allocated_cqs);
+	set_bit(2, rf->allocated_pds);
+
+	spin_lock_init(&rf->rsrc_lock);
+	spin_lock_init(&rf->arp_lock);
+	spin_lock_init(&rf->qptable_lock);
+	spin_lock_init(&rf->qh_list_lock);
+
+	INIT_LIST_HEAD(&rf->mc_qht_list.list);
+	/* stag index mask has a minimum of 14 bits */
+	mrdrvbits = 24 - max(get_count_order(rf->max_mr), 14);
+	rf->mr_stagmask = ~(((1 << mrdrvbits) - 1) << (32 - mrdrvbits));
+
+	return 0;
+}
+
+/**
+ * irdma_cqp_ce_handler - handle cqp completions
+ * @rf: RDMA PCI function
+ * @cq: cq for cqp completions
+ */
+void irdma_cqp_ce_handler(struct irdma_pci_f *rf, struct irdma_sc_cq *cq)
+{
+	struct irdma_cqp_request *cqp_request;
+	struct irdma_sc_dev *dev = &rf->sc_dev;
+	u32 cqe_count = 0;
+	struct irdma_ccq_cqe_info info;
+	unsigned long flags;
+	int ret;
+
+	do {
+		memset(&info, 0, sizeof(info));
+		spin_lock_irqsave(&rf->cqp.compl_lock, flags);
+		ret = dev->ccq_ops->ccq_get_cqe_info(cq, &info);
+		spin_unlock_irqrestore(&rf->cqp.compl_lock, flags);
+		if (ret)
+			break;
+
+		cqp_request = (struct irdma_cqp_request *)
+			      (unsigned long)info.scratch;
+		if (info.error)
+			dev_dbg(rfdev_to_dev(dev),
+				"ERR: opcode = 0x%x maj_err_code = 0x%x min_err_code = 0x%x\n",
+				info.op_code, info.maj_err_code,
+				info.min_err_code);
+		if (cqp_request) {
+			cqp_request->compl_info.maj_err_code = info.maj_err_code;
+			cqp_request->compl_info.min_err_code = info.min_err_code;
+			cqp_request->compl_info.op_ret_val = info.op_ret_val;
+			cqp_request->compl_info.error = info.error;
+
+			if (cqp_request->waiting) {
+				cqp_request->request_done = true;
+				wake_up(&cqp_request->waitq);
+				irdma_put_cqp_request(&rf->cqp, cqp_request);
+			} else {
+				if (cqp_request->callback_fcn)
+					cqp_request->callback_fcn(cqp_request);
+				irdma_put_cqp_request(&rf->cqp, cqp_request);
+			}
+		}
+
+		cqe_count++;
+	} while (1);
+
+	if (cqe_count) {
+		irdma_process_bh(dev);
+		dev->ccq_ops->ccq_arm(cq);
+	}
+}
+
+/**
+ * cqp_thread - Handle cqp completions
+ * @context: Pointer to RDMA PCI Function
+ */
+int cqp_compl_thread(void *context)
+{
+	struct irdma_pci_f *rf = context;
+	struct irdma_sc_cq *cq = &rf->ccq.sc_cq;
+
+	do {
+		if (down_interruptible(&rf->cqp.cqp_compl_sem) ||
+		    rf->stop_cqp_thread)
+			return 0;
+
+		irdma_cqp_ce_handler(rf, cq);
+	} while (!kthread_should_stop());
+
+	return 0;
+}
+
+/**
+ * irdma_next_iw_state - modify qp state
+ * @iwqp: iwarp qp to modify
+ * @state: next state for qp
+ * @del_hash: del hash
+ * @term: term message
+ * @termlen: length of term message
+ */
+void irdma_next_iw_state(struct irdma_qp *iwqp, u8 state, u8 del_hash, u8 term,
+			 u8 termlen)
+{
+	struct irdma_modify_qp_info info = {};
+
+	info.next_iwarp_state = state;
+	info.remove_hash_idx = del_hash;
+	info.cq_num_valid = true;
+	info.arp_cache_idx_valid = true;
+	info.dont_send_term = true;
+	info.dont_send_fin = true;
+	info.termlen = termlen;
+
+	if (term & IRDMAQP_TERM_SEND_TERM_ONLY)
+		info.dont_send_term = false;
+	if (term & IRDMAQP_TERM_SEND_FIN_ONLY)
+		info.dont_send_fin = false;
+	if (iwqp->sc_qp.term_flags && state == IRDMA_QP_STATE_ERROR)
+		info.reset_tcp_conn = true;
+	iwqp->hw_iwarp_state = state;
+	irdma_hw_modify_qp(iwqp->iwdev, iwqp, &info, 0);
+	iwqp->iwarp_state = info.next_iwarp_state;
+}
+
+/**
+ * irdma_del_mac_entry - remove a mac entry from the hw table
+ * @rf: RDMA PCI function
+ * @idx: the index of the mac ip address to delete
+ */
+void irdma_del_local_mac_entry(struct irdma_pci_f *rf, u16 idx)
+{
+	struct irdma_cqp *iwcqp = &rf->cqp;
+	struct irdma_cqp_request *cqp_request;
+	struct cqp_cmds_info *cqp_info;
+	enum irdma_status_code status = 0;
+
+	cqp_request = irdma_get_cqp_request(iwcqp, true);
+	if (!cqp_request) {
+		pr_err("cqp_request memory failed\n");
+		return;
+	}
+
+	cqp_info = &cqp_request->info;
+	cqp_info->cqp_cmd = IRDMA_OP_DELETE_LOCAL_MAC_ENTRY;
+	cqp_info->post_sq = 1;
+	cqp_info->in.u.del_local_mac_entry.cqp = &iwcqp->sc_cqp;
+	cqp_info->in.u.del_local_mac_entry.scratch = (uintptr_t)cqp_request;
+	cqp_info->in.u.del_local_mac_entry.entry_idx = idx;
+	cqp_info->in.u.del_local_mac_entry.ignore_ref_count = 0;
+	status = irdma_handle_cqp_op(rf, cqp_request);
+	if (status)
+		pr_err("CQP-OP Del MAC entry fail");
+}
+
+/**
+ * irdma_add_mac_entry - add a mac ip address entry to the hw table
+ * @rf: RDMA PCI function
+ * @mac_addr: pointer to mac address
+ * @idx: the index of the mac ip address to add
+ */
+int irdma_add_local_mac_entry(struct irdma_pci_f *rf, u8 *mac_addr, u16 idx)
+{
+	struct irdma_local_mac_entry_info *info;
+	struct irdma_cqp *iwcqp = &rf->cqp;
+	struct irdma_cqp_request *cqp_request;
+	struct cqp_cmds_info *cqp_info;
+	enum irdma_status_code status = 0;
+
+	cqp_request = irdma_get_cqp_request(iwcqp, true);
+	if (!cqp_request) {
+		pr_err("cqp_request memory failed\n");
+		return IRDMA_ERR_NO_MEMORY;
+	}
+
+	cqp_info = &cqp_request->info;
+	cqp_info->post_sq = 1;
+	info = &cqp_info->in.u.add_local_mac_entry.info;
+	ether_addr_copy(info->mac_addr, mac_addr);
+	info->entry_idx = idx;
+	cqp_info->in.u.add_local_mac_entry.scratch = (uintptr_t)cqp_request;
+	cqp_info->cqp_cmd = IRDMA_OP_ADD_LOCAL_MAC_ENTRY;
+	cqp_info->in.u.add_local_mac_entry.cqp = &iwcqp->sc_cqp;
+	cqp_info->in.u.add_local_mac_entry.scratch = (uintptr_t)cqp_request;
+	status = irdma_handle_cqp_op(rf, cqp_request);
+	if (status)
+		pr_err("CQP-OP Add MAC entry fail");
+
+	return status;
+}
+
+/**
+ * irdma_alloc_local_mac_entry - allocate a mac entry
+ * @rf: RDMA PCI function
+ * @mac_tbl_idx: the index of the new mac address
+ *
+ * Allocate a mac address entry and update the mac_tbl_idx
+ * to hold the index of the newly created mac address
+ * Return 0 if successful, otherwise return error
+ */
+int irdma_alloc_local_mac_entry(struct irdma_pci_f *rf, u16 *mac_tbl_idx)
+{
+	struct irdma_cqp *iwcqp = &rf->cqp;
+	struct irdma_cqp_request *cqp_request;
+	struct cqp_cmds_info *cqp_info;
+	enum irdma_status_code status = 0;
+
+	cqp_request = irdma_get_cqp_request(iwcqp, true);
+	if (!cqp_request) {
+		pr_err("cqp_request memory failed\n");
+		return IRDMA_ERR_NO_MEMORY;
+	}
+
+	/* increment refcount, because we need the cqp request ret value */
+	atomic_inc(&cqp_request->refcount);
+	cqp_info = &cqp_request->info;
+	cqp_info->cqp_cmd = IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY;
+	cqp_info->post_sq = 1;
+	cqp_info->in.u.alloc_local_mac_entry.cqp = &iwcqp->sc_cqp;
+	cqp_info->in.u.alloc_local_mac_entry.scratch = (uintptr_t)cqp_request;
+	status = irdma_handle_cqp_op(rf, cqp_request);
+	if (!status)
+		*mac_tbl_idx = (u16)cqp_request->compl_info.op_ret_val;
+	else
+		pr_err("CQP-OP Alloc MAC entry fail");
+	/* decrement refcount and free the cqp request, if no longer used */
+	irdma_put_cqp_request(iwcqp, cqp_request);
+
+	return status;
+}
+
+/**
+ * irdma_cqp_manage_apbvt_cmd - send cqp command manage apbvt
+ * @iwdev: iwarp device
+ * @accel_local_port: port for apbvt
+ * @add_port: add ordelete port
+ */
+static enum irdma_status_code
+irdma_cqp_manage_apbvt_cmd(struct irdma_device *iwdev, u16 accel_local_port,
+			   bool add_port)
+{
+	struct irdma_apbvt_info *info;
+	struct irdma_cqp_request *cqp_request;
+	struct cqp_cmds_info *cqp_info;
+	enum irdma_status_code status;
+
+	cqp_request = irdma_get_cqp_request(&iwdev->rf->cqp, add_port);
+	if (!cqp_request)
+		return IRDMA_ERR_NO_MEMORY;
+
+	cqp_info = &cqp_request->info;
+	info = &cqp_info->in.u.manage_apbvt_entry.info;
+	memset(info, 0, sizeof(*info));
+	info->add = add_port;
+	info->port = accel_local_port;
+	cqp_info->cqp_cmd = IRDMA_OP_MANAGE_APBVT_ENTRY;
+	cqp_info->post_sq = 1;
+	cqp_info->in.u.manage_apbvt_entry.cqp = &iwdev->rf->cqp.sc_cqp;
+	cqp_info->in.u.manage_apbvt_entry.scratch = (uintptr_t)cqp_request;
+	status = irdma_handle_cqp_op(iwdev->rf, cqp_request);
+	if (status)
+		dev_dbg(rfdev_to_dev(&iwdev->rf->sc_dev),
+			"ERR: CQP-OP Manage APBVT entry fail");
+
+	return status;
+}
+
+/**
+ * irdma_manage_apbvt - add or delete tcp port
+ * @iwdev: iwarp device
+ * @accel_local_port: port for apbvt
+ * @add_port: add or delete port
+ */
+enum irdma_status_code irdma_manage_apbvt(struct irdma_device *iwdev,
+					  u16 accel_local_port, bool add_port)
+{
+	struct irdma_cm_core *cm_core = &iwdev->cm_core;
+	enum irdma_status_code status = 0;
+	unsigned long flags;
+	bool in_use;
+
+	/* apbvt_lock is held across CQP delete APBVT OP (non-waiting) to
+	 * protect against race where add APBVT CQP can race ahead of the delete
+	 * APBVT for same port.
+	 */
+	if (add_port) {
+		spin_lock_irqsave(&cm_core->apbvt_lock, flags);
+		in_use = __test_and_set_bit(accel_local_port,
+					    cm_core->ports_in_use);
+		spin_unlock_irqrestore(&cm_core->apbvt_lock, flags);
+		if (in_use)
+			return 0;
+		return irdma_cqp_manage_apbvt_cmd(iwdev, accel_local_port,
+						  true);
+	} else {
+		spin_lock_irqsave(&cm_core->apbvt_lock, flags);
+		in_use = irdma_port_in_use(cm_core, accel_local_port);
+		if (in_use) {
+			spin_unlock_irqrestore(&cm_core->apbvt_lock, flags);
+			return 0;
+		}
+		__clear_bit(accel_local_port, cm_core->ports_in_use);
+		status = irdma_cqp_manage_apbvt_cmd(iwdev, accel_local_port,
+						    false);
+		spin_unlock_irqrestore(&cm_core->apbvt_lock, flags);
+		return status;
+	}
+}
+
+/**
+ * irdma_manage_arp_cache - manage hw arp cache
+ * @rf: RDMA PCI function
+ * @mac_addr: mac address ptr
+ * @ip_addr: ip addr for arp cache
+ * @ipv4: flag inicating IPv4
+ * @action: add, delete or modify
+ */
+void irdma_manage_arp_cache(struct irdma_pci_f *rf, unsigned char *mac_addr,
+			    u32 *ip_addr, bool ipv4, u32 action)
+{
+	struct irdma_add_arp_cache_entry_info *info;
+	struct irdma_cqp_request *cqp_request;
+	struct cqp_cmds_info *cqp_info;
+	int arp_index;
+
+	arp_index = irdma_arp_table(rf, ip_addr, ipv4, mac_addr, action);
+	if (arp_index == -1)
+		return;
+
+	cqp_request = irdma_get_cqp_request(&rf->cqp, false);
+	if (!cqp_request)
+		return;
+
+	cqp_info = &cqp_request->info;
+	if (action == IRDMA_ARP_ADD) {
+		cqp_info->cqp_cmd = IRDMA_OP_ADD_ARP_CACHE_ENTRY;
+		info = &cqp_info->in.u.add_arp_cache_entry.info;
+		memset(info, 0, sizeof(*info));
+		info->arp_index = (u16)arp_index;
+		info->permanent = true;
+		ether_addr_copy(info->mac_addr, mac_addr);
+		cqp_info->in.u.add_arp_cache_entry.scratch =
+			(uintptr_t)cqp_request;
+		cqp_info->in.u.add_arp_cache_entry.cqp = &rf->cqp.sc_cqp;
+	} else {
+		cqp_info->cqp_cmd = IRDMA_OP_DELETE_ARP_CACHE_ENTRY;
+		cqp_info->in.u.del_arp_cache_entry.scratch =
+			(uintptr_t)cqp_request;
+		cqp_info->in.u.del_arp_cache_entry.cqp = &rf->cqp.sc_cqp;
+		cqp_info->in.u.del_arp_cache_entry.arp_index = arp_index;
+	}
+
+	cqp_info->in.u.add_arp_cache_entry.cqp = &rf->cqp.sc_cqp;
+	cqp_info->in.u.add_arp_cache_entry.scratch = (uintptr_t)cqp_request;
+	cqp_info->post_sq = 1;
+	if (irdma_handle_cqp_op(rf, cqp_request))
+		dev_dbg(rfdev_to_dev(&rf->sc_dev),
+			"ERR: CQP-OP Add/Del Arp Cache entry fail");
+}
+
+/**
+ * irdma_send_syn_cqp_callback - do syn/ack after qhash
+ * @cqp_request: qhash cqp completion
+ */
+static void irdma_send_syn_cqp_callback(struct irdma_cqp_request *cqp_request)
+{
+	irdma_send_syn(cqp_request->param, 1);
+}
+
+/**
+ * irdma_manage_qhash - add or modify qhash
+ * @iwdev: iwarp device
+ * @cminfo: cm info for qhash
+ * @etype: type (syn or quad)
+ * @mtype: type of qhash
+ * @cmnode: cmnode associated with connection
+ * @wait: wait for completion
+ */
+enum irdma_status_code
+irdma_manage_qhash(struct irdma_device *iwdev, struct irdma_cm_info *cminfo,
+		   enum irdma_quad_entry_type etype,
+		   enum irdma_quad_hash_manage_type mtype, void *cmnode,
+		   bool wait)
+{
+	struct irdma_qhash_table_info *info;
+	struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
+	enum irdma_status_code status;
+	struct irdma_cqp *iwcqp = &iwdev->rf->cqp;
+	struct irdma_cqp_request *cqp_request;
+	struct cqp_cmds_info *cqp_info;
+
+	cqp_request = irdma_get_cqp_request(iwcqp, wait);
+	if (!cqp_request)
+		return IRDMA_ERR_NO_MEMORY;
+
+	cqp_info = &cqp_request->info;
+	info = &cqp_info->in.u.manage_qhash_table_entry.info;
+	memset(info, 0, sizeof(*info));
+	info->vsi = &iwdev->vsi;
+	info->manage = mtype;
+	info->entry_type = etype;
+	if (cminfo->vlan_id < VLAN_N_VID) {
+		info->vlan_valid = true;
+		info->vlan_id = cminfo->vlan_id;
+	} else {
+		info->vlan_valid = false;
+	}
+	info->ipv4_valid = cminfo->ipv4;
+	info->user_pri = cminfo->user_pri;
+	ether_addr_copy(info->mac_addr, iwdev->netdev->dev_addr);
+	info->qp_num = cminfo->qh_qpid;
+	info->dest_port = cminfo->loc_port;
+	info->dest_ip[0] = cminfo->loc_addr[0];
+	info->dest_ip[1] = cminfo->loc_addr[1];
+	info->dest_ip[2] = cminfo->loc_addr[2];
+	info->dest_ip[3] = cminfo->loc_addr[3];
+	if (etype == IRDMA_QHASH_TYPE_TCP_ESTABLISHED ||
+	    etype == IRDMA_QHASH_TYPE_UDP_UNICAST ||
+	    etype == IRDMA_QHASH_TYPE_UDP_MCAST ||
+	    etype == IRDMA_QHASH_TYPE_ROCE_MCAST ||
+	    etype == IRDMA_QHASH_TYPE_ROCEV2_HW) {
+		info->src_port = cminfo->rem_port;
+		info->src_ip[0] = cminfo->rem_addr[0];
+		info->src_ip[1] = cminfo->rem_addr[1];
+		info->src_ip[2] = cminfo->rem_addr[2];
+		info->src_ip[3] = cminfo->rem_addr[3];
+	}
+	if (cmnode) {
+		cqp_request->callback_fcn = irdma_send_syn_cqp_callback;
+		cqp_request->param = cmnode;
+	}
+	if (info->ipv4_valid)
+		dev_dbg(rfdev_to_dev(dev),
+			"CM: %s IP=%pI4, port=%d, mac=%pM, vlan_id=%d\n",
+			!mtype ? "DELETE" : "ADD", info->dest_ip,
+			info->dest_port, info->mac_addr, cminfo->vlan_id);
+	else
+		dev_dbg(rfdev_to_dev(dev),
+			"CM: %s IP=%pI6, port=%d, mac=%pM, vlan_id=%d\n",
+			!mtype ? "DELETE" : "ADD", info->dest_ip,
+			info->dest_port, info->mac_addr, cminfo->vlan_id);
+	cqp_info->in.u.manage_qhash_table_entry.cqp = &iwdev->rf->cqp.sc_cqp;
+	cqp_info->in.u.manage_qhash_table_entry.scratch = (uintptr_t)cqp_request;
+	cqp_info->cqp_cmd = IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY;
+	cqp_info->post_sq = 1;
+	status = irdma_handle_cqp_op(iwdev->rf, cqp_request);
+	if (status)
+		dev_dbg(rfdev_to_dev(dev),
+			"ERR: CQP-OP Manage Qhash Entry fail");
+
+	return status;
+}
+
+/**
+ * irdma_post_qp_fatal - Post QP_FATAL event associated with given QP
+ * @qp: QP associated with QP_FATL event
+ */
+static inline void irdma_post_qp_fatal(struct irdma_qp *qp)
+{
+	struct ib_event ibevent;
+
+	if (qp->ibqp.event_handler) {
+		ibevent.device = qp->ibqp.device;
+		ibevent.event = IB_EVENT_QP_FATAL;
+		ibevent.element.qp = &qp->ibqp;
+		qp->ibqp.event_handler(&ibevent, qp->ibqp.qp_context);
+	}
+}
+
+/**
+ * irdma_hw_flush_wqes_callback - Check return code after flush
+ * @cqp_request: qhash cqp completion
+ */
+static void irdma_hw_flush_wqes_callback(struct irdma_cqp_request *cqp_request)
+{
+	struct irdma_qp_flush_info *hw_info;
+	struct irdma_sc_qp *qp;
+	struct irdma_qp *iwqp;
+	struct cqp_cmds_info *cqp_info;
+
+	cqp_info = &cqp_request->info;
+	hw_info = &cqp_request->info.in.u.qp_flush_wqes.info;
+	qp = cqp_info->in.u.qp_flush_wqes.qp;
+	iwqp = qp->qp_uk.back_qp;
+
+	if (cqp_request->compl_info.maj_err_code)
+		return;
+	if (hw_info->rq &&
+	    (cqp_request->compl_info.min_err_code == IRDMA_CQP_COMPL_SQ_WQE_FLUSHED ||
+	     cqp_request->compl_info.min_err_code == 0)) {
+		/* RQ WQE flush was requested but did not happen */
+		qp->qp_uk.rq_flush_complete = true;
+		complete(&iwqp->rq_drained);
+	}
+	if (hw_info->sq &&
+	    (cqp_request->compl_info.min_err_code == IRDMA_CQP_COMPL_RQ_WQE_FLUSHED ||
+	     cqp_request->compl_info.min_err_code == 0)) {
+		qp->qp_uk.sq_flush_complete = true;
+		complete(&iwqp->sq_drained);
+	}
+}
+
+/**
+ * irdma_hw_flush_wqes - flush qp's wqe
+ * @rf: RDMA PCI function
+ * @qp: hardware control qp
+ * @info: info for flush
+ * @wait: flag wait for completion
+ */
+enum irdma_status_code irdma_hw_flush_wqes(struct irdma_pci_f *rf,
+					   struct irdma_sc_qp *qp,
+					   struct irdma_qp_flush_info *info,
+					   bool wait)
+{
+	enum irdma_status_code status;
+	struct irdma_qp_flush_info *hw_info;
+	struct irdma_cqp_request *cqp_request;
+	struct cqp_cmds_info *cqp_info;
+	struct irdma_qp *iwqp = qp->qp_uk.back_qp;
+	unsigned long flags = 0;
+
+	cqp_request = irdma_get_cqp_request(&rf->cqp, wait);
+	if (!cqp_request)
+		return IRDMA_ERR_NO_MEMORY;
+
+	cqp_info = &cqp_request->info;
+	if (!wait)
+		cqp_request->callback_fcn = irdma_hw_flush_wqes_callback;
+	hw_info = &cqp_request->info.in.u.qp_flush_wqes.info;
+	memcpy(hw_info, info, sizeof(*hw_info));
+	cqp_info->cqp_cmd = IRDMA_OP_QP_FLUSH_WQES;
+	cqp_info->post_sq = 1;
+	cqp_info->in.u.qp_flush_wqes.qp = qp;
+	cqp_info->in.u.qp_flush_wqes.scratch = (uintptr_t)cqp_request;
+	status = irdma_handle_cqp_op(rf, cqp_request);
+	if (status) {
+		dev_dbg(rfdev_to_dev(&rf->sc_dev),
+			"ERR: CQP-OP Flush WQE's fail");
+		complete(&iwqp->sq_drained);
+		complete(&iwqp->rq_drained);
+		qp->qp_uk.sq_flush_complete = true;
+		qp->qp_uk.rq_flush_complete = true;
+		return status;
+	}
+
+	if (!wait || cqp_request->compl_info.maj_err_code)
+		return 0;
+
+	if (info->rq) {
+		if (cqp_request->compl_info.min_err_code == IRDMA_CQP_COMPL_SQ_WQE_FLUSHED ||
+		    cqp_request->compl_info.min_err_code == 0) {
+			/* RQ WQE flush was requested but did not happen */
+			qp->qp_uk.rq_flush_complete = true;
+			complete(&iwqp->rq_drained);
+		}
+	}
+	if (info->sq) {
+		if (cqp_request->compl_info.min_err_code == IRDMA_CQP_COMPL_RQ_WQE_FLUSHED ||
+		    cqp_request->compl_info.min_err_code == 0) {
+			spin_lock_irqsave(&iwqp->lock, flags);
+			/*
+			 * Handling case where WQE is posted to empty SQ when
+			 * flush has not completed
+			 */
+			if (IRDMA_RING_MORE_WORK(qp->qp_uk.sq_ring)) {
+				struct irdma_cqp_request *new_req;
+
+				if (!qp->qp_uk.sq_flush_complete) {
+					spin_unlock_irqrestore(&iwqp->lock, flags);
+					return 0;
+				}
+				qp->qp_uk.sq_flush_complete = false;
+				qp->flush_sq = false;
+				spin_unlock_irqrestore(&iwqp->lock, flags);
+
+				info->rq = false;
+				info->sq = true;
+				new_req = irdma_get_cqp_request(&rf->cqp, true);
+				if (!new_req)
+					return IRDMA_ERR_NO_MEMORY;
+				cqp_info = &new_req->info;
+				hw_info = &new_req->info.in.u.qp_flush_wqes.info;
+				memcpy(hw_info, info, sizeof(*hw_info));
+				cqp_info->cqp_cmd = IRDMA_OP_QP_FLUSH_WQES;
+				cqp_info->post_sq = 1;
+				cqp_info->in.u.qp_flush_wqes.qp = qp;
+				cqp_info->in.u.qp_flush_wqes.scratch = (uintptr_t)new_req;
+
+				status = irdma_handle_cqp_op(rf, new_req);
+				if (new_req->compl_info.maj_err_code ||
+				    new_req->compl_info.min_err_code != IRDMA_CQP_COMPL_SQ_WQE_FLUSHED ||
+				    status) {
+					pr_err("SQ in error but not flushed");
+					qp->qp_uk.sq_flush_complete = false;
+					irdma_post_qp_fatal(iwqp);
+				}
+			} else {
+				/* SQ WQE flush was requested but did not happen */
+				qp->qp_uk.sq_flush_complete = true;
+				spin_unlock_irqrestore(&iwqp->lock, flags);
+				complete(&iwqp->sq_drained);
+			}
+		} else {
+			spin_lock_irqsave(&iwqp->lock, flags);
+			if (!IRDMA_RING_MORE_WORK(qp->qp_uk.sq_ring)) {
+				qp->qp_uk.sq_flush_complete = true;
+				spin_unlock_irqrestore(&iwqp->lock, flags);
+				complete(&iwqp->sq_drained);
+			} else {
+				spin_unlock_irqrestore(&iwqp->lock, flags);
+			}
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * irdma_gen_ae - generate AE
+ * @rf: RDMA PCI function
+ * @qp: qp associated with AE
+ * @info: info for ae
+ * @wait: wait for completion
+ */
+void irdma_gen_ae(struct irdma_pci_f *rf, struct irdma_sc_qp *qp,
+		  struct irdma_gen_ae_info *info, bool wait)
+{
+	struct irdma_gen_ae_info *ae_info;
+	struct irdma_cqp_request *cqp_request;
+	struct cqp_cmds_info *cqp_info;
+
+	cqp_request = irdma_get_cqp_request(&rf->cqp, wait);
+	if (!cqp_request)
+		return;
+
+	cqp_info = &cqp_request->info;
+	ae_info = &cqp_request->info.in.u.gen_ae.info;
+	memcpy(ae_info, info, sizeof(*ae_info));
+	cqp_info->cqp_cmd = IRDMA_OP_GEN_AE;
+	cqp_info->post_sq = 1;
+	cqp_info->in.u.gen_ae.qp = qp;
+	cqp_info->in.u.gen_ae.scratch = (uintptr_t)cqp_request;
+	if (irdma_handle_cqp_op(rf, cqp_request))
+		dev_dbg(rfdev_to_dev(&rf->sc_dev),
+			"ERR: CQP OP failed attempting to generate ae_code=0x%x\n",
+			info->ae_code);
+}
+
+/**
+ * irdma_get_ib_wc - return change flush code to IB's
+ * @opcode: iwarp flush code
+ */
+static enum ib_wc_status irdma_get_ib_wc(enum irdma_flush_opcode opcode)
+{
+	switch (opcode) {
+	case FLUSH_PROT_ERR:
+		return IB_WC_LOC_PROT_ERR;
+	case FLUSH_REM_ACCESS_ERR:
+		return IB_WC_REM_ACCESS_ERR;
+	case FLUSH_LOC_QP_OP_ERR:
+		return IB_WC_LOC_QP_OP_ERR;
+	case FLUSH_REM_OP_ERR:
+		return IB_WC_REM_OP_ERR;
+	case FLUSH_LOC_LEN_ERR:
+		return IB_WC_LOC_LEN_ERR;
+	case FLUSH_GENERAL_ERR:
+		return IB_WC_GENERAL_ERR;
+	case FLUSH_FATAL_ERR:
+	default:
+		return IB_WC_FATAL_ERR;
+	}
+}
+
+void irdma_flush_wqes(struct irdma_qp *iwqp, int flush_mask)
+{
+	struct irdma_qp_flush_info info = {};
+	struct irdma_pci_f *rf = iwqp->iwdev->rf;
+	u8 opcode = iwqp->sc_qp.flush_code;
+
+	if (!(flush_mask & IRDMA_FLUSH_SQ) && !(flush_mask & IRDMA_FLUSH_RQ))
+		return;
+
+	/* Set flush info fields*/
+	info.sq = flush_mask & IRDMA_FLUSH_SQ;
+	info.rq = flush_mask & IRDMA_FLUSH_RQ;
+
+	if (flush_mask & IRDMA_REFLUSH) {
+		if (info.sq)
+			iwqp->sc_qp.flush_sq = false;
+		if (info.rq)
+			iwqp->sc_qp.flush_rq = false;
+	}
+
+	/* Generate userflush errors in CQE */
+	if (opcode) {
+		if (info.sq) {
+			info.sq_minor_code = (u16)irdma_get_ib_wc(opcode);
+			info.sq_major_code = IRDMA_FLUSH_MAJOR_ERR;
+		}
+		if (info.rq) {
+			info.rq_minor_code = (u16)irdma_get_ib_wc(opcode);
+			info.rq_major_code = IRDMA_FLUSH_MAJOR_ERR;
+		}
+		info.userflushcode = true;
+	}
+
+	if (irdma_upload_context && !(flush_mask & IRDMA_REFLUSH) &&
+	    irdma_upload_qp_context(iwqp, 0, 1))
+		dev_warn(rfdev_to_dev(&rf->sc_dev),
+			 "failed to upload QP context\n");
+
+	/* Issue flush */
+	(void)irdma_hw_flush_wqes(rf, &iwqp->sc_qp, &info,
+				  flush_mask & IRDMA_FLUSH_WAIT);
+}
drivers/infiniband/hw/irdma/i40iw_hw.c new file +209 -0

209 lines not on this page · raw message

drivers/infiniband/hw/irdma/i40iw_hw.h new file +162 -0

162 lines not on this page · raw message

drivers/infiniband/hw/irdma/icrdma_hw.c new file +74 -0

33 lines not on this page · raw message

drivers/infiniband/hw/irdma/icrdma_hw.h new file +62 -0

62 lines not on this page · raw message

The rest of this diff is not shown. View the raw message.

Signature · 1 line
2.23.0

Part of the diff is not shown here. View the raw message.

#33 lore raw

PATCH v3 7/20 RDMA/irdma: Implement HW Admin Queue OPs

In reply to Jeff Kirsher

From: Mustafa Ismail <[email protected]>

The driver posts privileged commands to the HW Admin Queue (Control QP or CQP) to request administrative actions from the HW. Implement create/destroy of CQP and the supporting functions, data structures and headers to handle the different CQP commands

---
 drivers/infiniband/hw/irdma/ctrl.c  | 5990 +++++++++++++++++++++++++++
 drivers/infiniband/hw/irdma/defs.h  | 2129 ++++++++++
 drivers/infiniband/hw/irdma/irdma.h |  190 +
 drivers/infiniband/hw/irdma/type.h  | 1709 ++++++++
 4 files changed, 10018 insertions(+)
 create mode 100644 drivers/infiniband/hw/irdma/ctrl.c
 create mode 100644 drivers/infiniband/hw/irdma/defs.h
 create mode 100644 drivers/infiniband/hw/irdma/irdma.h
 create mode 100644 drivers/infiniband/hw/irdma/type.h
drivers/infiniband/hw/irdma/ctrl.c new file +5990 -0
new file mode 100644
index 000000000000..caa5717ca31a
@@ -0,0 +1,5990 @@
+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
+/* Copyright (c) 2015 - 2019 Intel Corporation */
+#include "osdep.h"
+#include "status.h"
+#include "hmc.h"
+#include "defs.h"
+#include "type.h"
+#include "ws.h"
+#include "protos.h"
+
+/**
+ * irdma_get_qp_from_list - get next qp from a list
+ * @head: Listhead of qp's
+ * @qp: current qp
+ */
+struct irdma_sc_qp *irdma_get_qp_from_list(struct list_head *head,
+					   struct irdma_sc_qp *qp)
+{
+	struct list_head *lastentry;
+	struct list_head *entry = NULL;
+
+	if (list_empty(head))
+		return NULL;
+
+	if (!qp) {
+		entry = head->next;
+	} else {
+		lastentry = &qp->list;
+		entry = lastentry->next;
+		if (entry == head)
+			return NULL;
+	}
+
+	return container_of(entry, struct irdma_sc_qp, list);
+}
+
+/**
+ * irdma_suspend_qps - suspend all qp's on VSI
+ * @vsi: the VSI struct pointer
+ */
+void irdma_suspend_qps(struct irdma_sc_vsi *vsi)
+{
+	struct irdma_sc_qp *qp = NULL;
+	unsigned long flags;
+	int i;
+
+	for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
+		spin_lock_irqsave(&vsi->qos[i].lock, flags);
+		qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);
+		while (qp) {
+			/* issue cqp suspend command */
+			if (!irdma_qp_suspend_resume(qp, true))
+				atomic_inc(&vsi->qp_suspend_reqs);
+			qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);
+		}
+		spin_unlock_irqrestore(&vsi->qos[i].lock, flags);
+	}
+}
+
+/**
+ * irdma_change_l2params - given the new l2 parameters, change all qp
+ * @vsi: RDMA VSI pointer
+ * @l2params: New parameters from l2
+ */
+void irdma_change_l2params(struct irdma_sc_vsi *vsi,
+			   struct irdma_l2params *l2params)
+{
+	struct irdma_sc_qp *qp = NULL;
+	u8 i;
+	unsigned long flags;
+
+	if (l2params->mtu_changed) {
+		vsi->mtu = l2params->mtu;
+		irdma_reinitialize_ieq(vsi);
+	}
+
+	if (!l2params->tc_changed)
+		return;
+
+	vsi->tc_change_pending = false;
+	for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
+		vsi->qos[i].traffic_class = l2params->up2tc[i];
+		spin_lock_irqsave(&vsi->qos[i].lock, flags);
+		qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);
+		while (qp) {
+			if (!irdma_ws_add(vsi, i)) {
+				qp->qs_handle = vsi->qos[qp->user_pri].qs_handle;
+				irdma_qp_suspend_resume(qp, false);
+			} else {
+				irdma_qp_suspend_resume(qp, false);
+				spin_unlock_irqrestore(&vsi->qos[i].lock, flags);
+				irdma_modify_qp_to_err(qp);
+				spin_lock_irqsave(&vsi->qos[i].lock, flags);
+			}
+			qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);
+		}
+		spin_unlock_irqrestore(&vsi->qos[i].lock, flags);
+	}
+}
+
+/**
+ * irdma_qp_rem_qos - remove qp from qos lists during destroy qp
+ * @qp: qp to be removed from qos
+ */
+void irdma_qp_rem_qos(struct irdma_sc_qp *qp)
+{
+	struct irdma_sc_vsi *vsi = qp->vsi;
+	unsigned long flags;
+
+	if (!qp->on_qoslist)
+		return;
+
+	spin_lock_irqsave(&vsi->qos[qp->user_pri].lock, flags);
+	qp->on_qoslist = false;
+	list_del(&qp->list);
+	spin_unlock_irqrestore(&vsi->qos[qp->user_pri].lock, flags);
+	dev_dbg(rfdev_to_dev(qp->dev),
+		"DCB: DCB: Remove qp[%d] UP[%d] qset[%d]\n", qp->qp_uk.qp_id,
+		qp->user_pri, qp->qs_handle);
+}
+
+/**
+ * irdma_qp_add_qos - called during setctx for qp to be added to qos
+ * @qp: qp to be added to qos
+ */
+void irdma_qp_add_qos(struct irdma_sc_qp *qp)
+{
+	struct irdma_sc_vsi *vsi = qp->vsi;
+	unsigned long flags;
+
+	if (qp->on_qoslist)
+		return;
+
+	spin_lock_irqsave(&vsi->qos[qp->user_pri].lock, flags);
+	list_add(&qp->list, &vsi->qos[qp->user_pri].qplist);
+	qp->on_qoslist = true;
+	qp->qs_handle = vsi->qos[qp->user_pri].qs_handle;
+	spin_unlock_irqrestore(&vsi->qos[qp->user_pri].lock, flags);
+	dev_dbg(rfdev_to_dev(qp->dev),
+		"DCB: DCB: Add qp[%d] UP[%d] qset[%d]\n", qp->qp_uk.qp_id,
+		qp->user_pri, qp->qs_handle);
+}
+
+/**
+ * irdma_sc_pd_init - initialize sc pd struct
+ * @dev: sc device struct
+ * @pd: sc pd ptr
+ * @pd_id: pd_id for allocated pd
+ * @abi_ver: ABI version from user context, -1 if not valid
+ */
+static void irdma_sc_pd_init(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd,
+			     u32 pd_id, int abi_ver)
+{
+	pd->pd_id = pd_id;
+	pd->abi_ver = abi_ver;
+	pd->dev = dev;
+}
+
+/**
+ * irdma_sc_add_arp_cache_entry - cqp wqe add arp cache entry
+ * @cqp: struct for cqp hw
+ * @info: arp entry information
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_add_arp_cache_entry(struct irdma_sc_cqp *cqp,
+			     struct irdma_add_arp_cache_entry_info *info,
+			     u64 scratch, bool post_sq)
+{
+	__le64 *wqe;
+	u64 temp, hdr;
+
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+	set_64bit_val(wqe, 8, info->reach_max);
+
+	temp = info->mac_addr[5] | LS_64_1(info->mac_addr[4], 8) |
+	       LS_64_1(info->mac_addr[3], 16) | LS_64_1(info->mac_addr[2], 24) |
+	       LS_64_1(info->mac_addr[1], 32) | LS_64_1(info->mac_addr[0], 40);
+	set_64bit_val(wqe, 16, temp);
+
+	hdr = info->arp_index |
+	      LS_64(IRDMA_CQP_OP_MANAGE_ARP, IRDMA_CQPSQ_OPCODE) |
+	      LS_64((info->permanent ? 1 : 0), IRDMA_CQPSQ_MAT_PERMANENT) |
+	      LS_64(1, IRDMA_CQPSQ_MAT_ENTRYVALID) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "ARP_CACHE_ENTRY WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	if (post_sq)
+		irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_del_arp_cache_entry - dele arp cache entry
+ * @cqp: struct for cqp hw
+ * @scratch: u64 saved to be used during cqp completion
+ * @arp_index: arp index to delete arp entry
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_del_arp_cache_entry(struct irdma_sc_cqp *cqp, u64 scratch,
+			     u16 arp_index, bool post_sq)
+{
+	__le64 *wqe;
+	u64 hdr;
+
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	hdr = arp_index | LS_64(IRDMA_CQP_OP_MANAGE_ARP, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "ARP_CACHE_DEL_ENTRY WQE",
+			wqe, IRDMA_CQP_WQE_SIZE * 8);
+	if (post_sq)
+		irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_query_arp_cache_entry - cqp wqe to query arp and arp index
+ * @cqp: struct for cqp hw
+ * @scratch: u64 saved to be used during cqp completion
+ * @arp_index: arp index to delete arp entry
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_query_arp_cache_entry(struct irdma_sc_cqp *cqp, u64 scratch,
+			       u16 arp_index, bool post_sq)
+{
+	__le64 *wqe;
+	u64 hdr;
+
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	hdr = arp_index | LS_64(IRDMA_CQP_OP_MANAGE_ARP, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(1, IRDMA_CQPSQ_MAT_QUERY) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QUERY_ARP_CACHE_ENTRY WQE",
+			wqe, IRDMA_CQP_WQE_SIZE * 8);
+	if (post_sq)
+		irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_manage_apbvt_entry - for adding and deleting apbvt entries
+ * @cqp: struct for cqp hw
+ * @info: info for apbvt entry to add or delete
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_manage_apbvt_entry(struct irdma_sc_cqp *cqp,
+			    struct irdma_apbvt_info *info, u64 scratch,
+			    bool post_sq)
+{
+	__le64 *wqe;
+	u64 hdr;
+
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	set_64bit_val(wqe, 16, info->port);
+
+	hdr = LS_64(IRDMA_CQP_OP_MANAGE_APBVT, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(info->add, IRDMA_CQPSQ_MAPT_ADDPORT) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_APBVT WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	if (post_sq)
+		irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_manage_qhash_table_entry - manage quad hash entries
+ * @cqp: struct for cqp hw
+ * @info: info for quad hash to manage
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ *
+ * This is called before connection establishment is started.
+ * For passive connections, when listener is created, it will
+ * call with entry type of  IRDMA_QHASH_TYPE_TCP_SYN with local
+ * ip address and tcp port. When SYN is received (passive
+ * connections) or sent (active connections), this routine is
+ * called with entry type of IRDMA_QHASH_TYPE_TCP_ESTABLISHED
+ * and quad is passed in info.
+ *
+ * When iwarp connection is done and its state moves to RTS, the
+ * quad hash entry in the hardware will point to iwarp's qp
+ * number and requires no calls from the driver.
+ */
+static enum irdma_status_code
+irdma_sc_manage_qhash_table_entry(struct irdma_sc_cqp *cqp,
+				  struct irdma_qhash_table_info *info,
+				  u64 scratch, bool post_sq)
+{
+	__le64 *wqe;
+	u64 qw1 = 0;
+	u64 qw2 = 0;
+	u64 temp;
+	struct irdma_sc_vsi *vsi = info->vsi;
+
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+	temp = info->mac_addr[5] | LS_64_1(info->mac_addr[4], 8) |
+	       LS_64_1(info->mac_addr[3], 16) | LS_64_1(info->mac_addr[2], 24) |
+	       LS_64_1(info->mac_addr[1], 32) | LS_64_1(info->mac_addr[0], 40);
+	set_64bit_val(wqe, 0, temp);
+
+	qw1 = LS_64(info->qp_num, IRDMA_CQPSQ_QHASH_QPN) |
+	      LS_64(info->dest_port, IRDMA_CQPSQ_QHASH_DEST_PORT);
+	if (info->ipv4_valid) {
+		set_64bit_val(wqe, 48,
+			      LS_64(info->dest_ip[0], IRDMA_CQPSQ_QHASH_ADDR3));
+	} else {
+		set_64bit_val(wqe, 56,
+			      LS_64(info->dest_ip[0], IRDMA_CQPSQ_QHASH_ADDR0) |
+			      LS_64(info->dest_ip[1], IRDMA_CQPSQ_QHASH_ADDR1));
+
+		set_64bit_val(wqe, 48,
+			      LS_64(info->dest_ip[2], IRDMA_CQPSQ_QHASH_ADDR2) |
+			      LS_64(info->dest_ip[3], IRDMA_CQPSQ_QHASH_ADDR3));
+	}
+	qw2 = LS_64(vsi->qos[info->user_pri].qs_handle,
+		    IRDMA_CQPSQ_QHASH_QS_HANDLE);
+	if (info->vlan_valid)
+		qw2 |= LS_64(info->vlan_id, IRDMA_CQPSQ_QHASH_VLANID);
+	set_64bit_val(wqe, 16, qw2);
+	if (info->entry_type == IRDMA_QHASH_TYPE_TCP_ESTABLISHED) {
+		qw1 |= LS_64(info->src_port, IRDMA_CQPSQ_QHASH_SRC_PORT);
+		if (!info->ipv4_valid) {
+			set_64bit_val(wqe, 40,
+				      LS_64(info->src_ip[0], IRDMA_CQPSQ_QHASH_ADDR0) |
+				      LS_64(info->src_ip[1], IRDMA_CQPSQ_QHASH_ADDR1));
+			set_64bit_val(wqe, 32,
+				      LS_64(info->src_ip[2], IRDMA_CQPSQ_QHASH_ADDR2) |
+				      LS_64(info->src_ip[3], IRDMA_CQPSQ_QHASH_ADDR3));
+		} else {
+			set_64bit_val(wqe, 32,
+				      LS_64(info->src_ip[0], IRDMA_CQPSQ_QHASH_ADDR3));
+		}
+	}
+
+	set_64bit_val(wqe, 8, qw1);
+	temp = LS_64(cqp->polarity, IRDMA_CQPSQ_QHASH_WQEVALID) |
+	       LS_64(IRDMA_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY,
+		     IRDMA_CQPSQ_QHASH_OPCODE) |
+	       LS_64(info->manage, IRDMA_CQPSQ_QHASH_MANAGE) |
+	       LS_64(info->ipv4_valid, IRDMA_CQPSQ_QHASH_IPV4VALID) |
+	       LS_64(info->vlan_valid, IRDMA_CQPSQ_QHASH_VLANVALID) |
+	       LS_64(info->entry_type, IRDMA_CQPSQ_QHASH_ENTRYTYPE);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, temp);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_QHASH WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	if (post_sq)
+		irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_cqp_nop - send a nop wqe
+ * @cqp: struct for cqp hw
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code irdma_sc_cqp_nop(struct irdma_sc_cqp *cqp,
+					       u64 scratch, bool post_sq)
+{
+	__le64 *wqe;
+	u64 hdr;
+
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+	hdr = LS_64(IRDMA_CQP_OP_NOP, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "NOP WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	if (post_sq)
+		irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_qp_init - initialize qp
+ * @qp: sc qp
+ * @info: initialization qp info
+ */
+static enum irdma_status_code irdma_sc_qp_init(struct irdma_sc_qp *qp,
+					       struct irdma_qp_init_info *info)
+{
+	enum irdma_status_code ret_code;
+	u32 pble_obj_cnt;
+	u16 wqe_size;
+
+	if (info->qp_uk_init_info.max_sq_frag_cnt >
+	    info->pd->dev->hw_attrs.uk_attrs.max_hw_wq_frags ||
+	    info->qp_uk_init_info.max_rq_frag_cnt >
+	    info->pd->dev->hw_attrs.uk_attrs.max_hw_wq_frags)
+		return IRDMA_ERR_INVALID_FRAG_COUNT;
+
+	qp->dev = info->pd->dev;
+	qp->vsi = info->vsi;
+	qp->ieq_qp = info->vsi->exception_lan_q;
+	qp->sq_pa = info->sq_pa;
+	qp->rq_pa = info->rq_pa;
+	qp->hw_host_ctx_pa = info->host_ctx_pa;
+	qp->q2_pa = info->q2_pa;
+	qp->shadow_area_pa = info->shadow_area_pa;
+	qp->q2_buf = info->q2;
+	qp->pd = info->pd;
+	qp->hw_host_ctx = info->host_ctx;
+	info->qp_uk_init_info.wqe_alloc_db = qp->pd->dev->wqe_alloc_db;
+	ret_code = irdma_qp_uk_init(&qp->qp_uk, &info->qp_uk_init_info);
+	if (ret_code)
+		return ret_code;
+
+	qp->virtual_map = info->virtual_map;
+	pble_obj_cnt = info->pd->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
+
+	if ((info->virtual_map && info->sq_pa >= pble_obj_cnt) ||
+	    (info->virtual_map && info->rq_pa >= pble_obj_cnt))
+		return IRDMA_ERR_INVALID_PBLE_INDEX;
+
+	qp->llp_stream_handle = (void *)(-1);
+	qp->qp_type = info->type ? info->type : IRDMA_QP_TYPE_IWARP;
+	qp->hw_sq_size = irdma_get_encoded_wqe_size(qp->qp_uk.sq_ring.size, false);
+	dev_dbg(rfdev_to_dev(qp->dev),
+		"WQE: hw_sq_size[%04d] sq_ring.size[%04d]\n", qp->hw_sq_size,
+		qp->qp_uk.sq_ring.size);
+	if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1 && qp->pd->abi_ver > 4)
+		wqe_size = IRDMA_WQE_SIZE_128;
+	else
+		ret_code = irdma_fragcnt_to_wqesize_rq(qp->qp_uk.max_rq_frag_cnt,
+						       &wqe_size);
+	if (ret_code)
+		return ret_code;
+
+	qp->hw_rq_size = irdma_get_encoded_wqe_size(qp->qp_uk.rq_size *
+				(wqe_size / IRDMA_QP_WQE_MIN_SIZE), false);
+	dev_dbg(rfdev_to_dev(qp->dev),
+		"WQE: hw_rq_size[%04d] qp_uk.rq_size[%04d] wqe_size[%04d]\n",
+		qp->hw_rq_size, qp->qp_uk.rq_size, wqe_size);
+	qp->sq_tph_val = info->sq_tph_val;
+	qp->rq_tph_val = info->rq_tph_val;
+	qp->sq_tph_en = info->sq_tph_en;
+	qp->rq_tph_en = info->rq_tph_en;
+	qp->rcv_tph_en = info->rcv_tph_en;
+	qp->xmit_tph_en = info->xmit_tph_en;
+	qp->qp_uk.first_sq_wq = info->qp_uk_init_info.first_sq_wq;
+	qp->qs_handle = qp->vsi->qos[qp->user_pri].qs_handle;
+
+	return 0;
+}
+
+/**
+ * irdma_sc_qp_create - create qp
+ * @qp: sc qp
+ * @info: qp create info
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_qp_create(struct irdma_sc_qp *qp, struct irdma_create_qp_info *info,
+		   u64 scratch, bool post_sq)
+{
+	struct irdma_sc_cqp *cqp;
+	__le64 *wqe;
+	u64 hdr;
+
+	cqp = qp->dev->cqp;
+	if (qp->qp_uk.qp_id < cqp->dev->hw_attrs.min_hw_qp_id ||
+	    qp->qp_uk.qp_id > (cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt - 1))
+		return IRDMA_ERR_INVALID_QP_ID;
+
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
+	set_64bit_val(wqe, 40, qp->shadow_area_pa);
+
+	hdr = qp->qp_uk.qp_id |
+	      LS_64(IRDMA_CQP_OP_CREATE_QP, IRDMA_CQPSQ_OPCODE) |
+	      LS_64((info->ord_valid ? 1 : 0), IRDMA_CQPSQ_QP_ORDVALID) |
+	      LS_64(info->tcp_ctx_valid, IRDMA_CQPSQ_QP_TOECTXVALID) |
+	      LS_64(info->mac_valid, IRDMA_CQPSQ_QP_MACVALID) |
+	      LS_64(qp->qp_type, IRDMA_CQPSQ_QP_QPTYPE) |
+	      LS_64(qp->virtual_map, IRDMA_CQPSQ_QP_VQ) |
+	      LS_64(info->force_lpb, IRDMA_CQPSQ_QP_FORCELOOPBACK) |
+	      LS_64(info->cq_num_valid, IRDMA_CQPSQ_QP_CQNUMVALID) |
+	      LS_64(info->arp_cache_idx_valid, IRDMA_CQPSQ_QP_ARPTABIDXVALID) |
+	      LS_64(info->next_iwarp_state, IRDMA_CQPSQ_QP_NEXTIWSTATE) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QP_CREATE WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	if (post_sq)
+		irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_qp_modify - modify qp cqp wqe
+ * @qp: sc qp
+ * @info: modify qp info
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_qp_modify(struct irdma_sc_qp *qp, struct irdma_modify_qp_info *info,
+		   u64 scratch, bool post_sq)
+{
+	__le64 *wqe;
+	struct irdma_sc_cqp *cqp;
+	u64 hdr;
+	u8 term_actions = 0;
+	u8 term_len = 0;
+
+	cqp = qp->dev->cqp;
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	if (info->next_iwarp_state == IRDMA_QP_STATE_TERMINATE) {
+		if (info->dont_send_fin)
+			term_actions += IRDMAQP_TERM_SEND_TERM_ONLY;
+		if (info->dont_send_term)
+			term_actions += IRDMAQP_TERM_SEND_FIN_ONLY;
+		if (term_actions == IRDMAQP_TERM_SEND_TERM_AND_FIN ||
+		    term_actions == IRDMAQP_TERM_SEND_TERM_ONLY)
+			term_len = info->termlen;
+	}
+
+	set_64bit_val(wqe, 8,
+		      LS_64(info->new_mss, IRDMA_CQPSQ_QP_NEWMSS) |
+		      LS_64(term_len, IRDMA_CQPSQ_QP_TERMLEN));
+	set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
+	set_64bit_val(wqe, 40, qp->shadow_area_pa);
+
+	hdr = qp->qp_uk.qp_id |
+	      LS_64(IRDMA_CQP_OP_MODIFY_QP, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(info->ord_valid, IRDMA_CQPSQ_QP_ORDVALID) |
+	      LS_64(info->tcp_ctx_valid, IRDMA_CQPSQ_QP_TOECTXVALID) |
+	      LS_64(info->cached_var_valid, IRDMA_CQPSQ_QP_CACHEDVARVALID) |
+	      LS_64(qp->virtual_map, IRDMA_CQPSQ_QP_VQ) |
+	      LS_64(info->force_lpb, IRDMA_CQPSQ_QP_FORCELOOPBACK) |
+	      LS_64(info->cq_num_valid, IRDMA_CQPSQ_QP_CQNUMVALID) |
+	      LS_64(info->force_lpb, IRDMA_CQPSQ_QP_FORCELOOPBACK) |
+	      LS_64(info->mac_valid, IRDMA_CQPSQ_QP_MACVALID) |
+	      LS_64(qp->qp_type, IRDMA_CQPSQ_QP_QPTYPE) |
+	      LS_64(info->mss_change, IRDMA_CQPSQ_QP_MSSCHANGE) |
+	      LS_64(info->remove_hash_idx, IRDMA_CQPSQ_QP_REMOVEHASHENTRY) |
+	      LS_64(term_actions, IRDMA_CQPSQ_QP_TERMACT) |
+	      LS_64(info->reset_tcp_conn, IRDMA_CQPSQ_QP_RESETCON) |
+	      LS_64(info->arp_cache_idx_valid, IRDMA_CQPSQ_QP_ARPTABIDXVALID) |
+	      LS_64(info->next_iwarp_state, IRDMA_CQPSQ_QP_NEXTIWSTATE) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QP_MODIFY WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	if (post_sq)
+		irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_qp_destroy - cqp destroy qp
+ * @qp: sc qp
+ * @scratch: u64 saved to be used during cqp completion
+ * @remove_hash_idx: flag if to remove hash idx
+ * @ignore_mw_bnd: memory window bind flag
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch, bool remove_hash_idx,
+		    bool ignore_mw_bnd, bool post_sq)
+{
+	__le64 *wqe;
+	struct irdma_sc_cqp *cqp;
+	u64 hdr;
+
+	irdma_qp_rem_qos(qp);
+	cqp = qp->dev->cqp;
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
+	set_64bit_val(wqe, 40, qp->shadow_area_pa);
+
+	hdr = qp->qp_uk.qp_id |
+	      LS_64(IRDMA_CQP_OP_DESTROY_QP, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(qp->qp_type, IRDMA_CQPSQ_QP_QPTYPE) |
+	      LS_64(ignore_mw_bnd, IRDMA_CQPSQ_QP_IGNOREMWBOUND) |
+	      LS_64(remove_hash_idx, IRDMA_CQPSQ_QP_REMOVEHASHENTRY) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QP_DESTROY WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	if (post_sq)
+		irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_qp_setctx_roce - set qp's context
+ * @qp: sc qp
+ * @qp_ctx: context ptr
+ * @info: ctx info
+ */
+static enum irdma_status_code
+irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 *qp_ctx,
+			struct irdma_qp_host_ctx_info *info)
+{
+	struct irdma_roce_offload_info *roce_info;
+	struct irdma_udp_offload_info *udp;
+	u8 push_mode_en;
+	u16 push_idx;
+	u64 mac;
+
+	roce_info = info->roce_info;
+	udp = info->udp_info;
+
+	mac = LS_64_1(roce_info->mac_addr[5], 16) |
+	      LS_64_1(roce_info->mac_addr[4], 24) |
+	      LS_64_1(roce_info->mac_addr[3], 32) |
+	      LS_64_1(roce_info->mac_addr[2], 40) |
+	      LS_64_1(roce_info->mac_addr[1], 48) |
+	      LS_64_1(roce_info->mac_addr[0], 56);
+
+	qp->user_pri = info->user_pri;
+	if (info->add_to_qoslist)
+		irdma_qp_add_qos(qp);
+	if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) {
+		push_mode_en = 0;
+		push_idx = 0;
+	} else {
+		push_mode_en = 1;
+		push_idx = qp->push_idx;
+	}
+	set_64bit_val(qp_ctx, 0,
+		      LS_64(qp->qp_uk.rq_wqe_size, IRDMAQPC_RQWQESIZE) |
+		      LS_64(qp->rcv_tph_en, IRDMAQPC_RCVTPHEN) |
+		      LS_64(qp->xmit_tph_en, IRDMAQPC_XMITTPHEN) |
+		      LS_64(qp->rq_tph_en, IRDMAQPC_RQTPHEN) |
+		      LS_64(qp->sq_tph_en, IRDMAQPC_SQTPHEN) |
+		      LS_64(push_idx, IRDMAQPC_PPIDX) |
+		      LS_64(push_mode_en, IRDMAQPC_PMENA) |
+		      LS_64(roce_info->pd_id >> 16, IRDMAQPC_PDIDXHI) |
+		      LS_64(roce_info->dctcp_en, IRDMAQPC_DC_TCP_EN) |
+		      LS_64(roce_info->err_rq_idx_valid, IRDMAQPC_ERR_RQ_IDX_VALID) |
+		      LS_64(roce_info->is_qp1, IRDMAQPC_ISQP1) |
+		      LS_64(roce_info->roce_tver, IRDMAQPC_ROCE_TVER) |
+		      LS_64(udp->ipv4, IRDMAQPC_IPV4) |
+		      LS_64(udp->insert_vlan_tag, IRDMAQPC_INSERTVLANTAG));
+	set_64bit_val(qp_ctx, 8, qp->sq_pa);
+	set_64bit_val(qp_ctx, 16, qp->rq_pa);
+	if ((roce_info->dcqcn_en || roce_info->dctcp_en) &&
+	    !(udp->tos & 0x03))
+		udp->tos |= ECN_CODE_PT_VAL;
+	set_64bit_val(qp_ctx, 24,
+		      LS_64(qp->hw_rq_size, IRDMAQPC_RQSIZE) |
+		      LS_64(qp->hw_sq_size, IRDMAQPC_SQSIZE) |
+		      LS_64(udp->ttl, IRDMAQPC_TTL) | LS_64(udp->tos, IRDMAQPC_TOS) |
+		      LS_64(udp->src_port, IRDMAQPC_SRCPORTNUM) |
+		      LS_64(udp->dst_port, IRDMAQPC_DESTPORTNUM));
+	set_64bit_val(qp_ctx, 32,
+		      LS_64(udp->dest_ip_addr2, IRDMAQPC_DESTIPADDR2) |
+		      LS_64(udp->dest_ip_addr3, IRDMAQPC_DESTIPADDR3));
+	set_64bit_val(qp_ctx, 40,
+		      LS_64(udp->dest_ip_addr0, IRDMAQPC_DESTIPADDR0) |
+		      LS_64(udp->dest_ip_addr1, IRDMAQPC_DESTIPADDR1));
+	set_64bit_val(qp_ctx, 48,
+		      LS_64(udp->snd_mss, IRDMAQPC_SNDMSS) |
+		      LS_64(udp->vlan_tag, IRDMAQPC_VLANTAG) |
+		      LS_64(udp->arp_idx, IRDMAQPC_ARPIDX));
+	set_64bit_val(qp_ctx, 56,
+		      LS_64(roce_info->p_key, IRDMAQPC_PKEY) |
+		      LS_64(roce_info->pd_id, IRDMAQPC_PDIDX) |
+		      LS_64(roce_info->ack_credits, IRDMAQPC_ACKCREDITS) |
+		      LS_64(udp->flow_label, IRDMAQPC_FLOWLABEL));
+	set_64bit_val(qp_ctx, 64,
+		      LS_64(roce_info->qkey, IRDMAQPC_QKEY) |
+		      LS_64(roce_info->dest_qp, IRDMAQPC_DESTQP));
+	set_64bit_val(qp_ctx, 80,
+		      LS_64(udp->psn_nxt, IRDMAQPC_PSNNXT) |
+		      LS_64(udp->lsn, IRDMAQPC_LSN));
+	set_64bit_val(qp_ctx, 88, LS_64(udp->epsn, IRDMAQPC_EPSN));
+	set_64bit_val(qp_ctx, 96,
+		      LS_64(udp->psn_max, IRDMAQPC_PSNMAX) |
+		      LS_64(udp->psn_una, IRDMAQPC_PSNUNA));
+	set_64bit_val(qp_ctx, 112,
+		      LS_64(udp->cwnd, IRDMAQPC_CWNDROCE));
+	set_64bit_val(qp_ctx, 128,
+		      LS_64(roce_info->err_rq_idx, IRDMAQPC_ERR_RQ_IDX) |
+		      LS_64(udp->rnr_nak_thresh, IRDMAQPC_RNRNAK_THRESH) |
+		      LS_64(udp->rexmit_thresh, IRDMAQPC_REXMIT_THRESH));
+	set_64bit_val(qp_ctx, 136,
+		      LS_64(info->send_cq_num, IRDMAQPC_TXCQNUM) |
+		      LS_64(info->rcv_cq_num, IRDMAQPC_RXCQNUM));
+	set_64bit_val(qp_ctx, 144,
+		      LS_64(info->stats_idx, IRDMAQPC_STAT_INDEX));
+	set_64bit_val(qp_ctx, 152, mac);
+	set_64bit_val(qp_ctx, 160,
+		      LS_64(roce_info->ord_size, IRDMAQPC_ORDSIZE) |
+		      LS_64(roce_info->ird_size, IRDMAQPC_IRDSIZE) |
+		      LS_64(roce_info->wr_rdresp_en, IRDMAQPC_WRRDRSPOK) |
+		      LS_64(roce_info->rd_en, IRDMAQPC_RDOK) |
+		      LS_64(info->stats_idx_valid, IRDMAQPC_USESTATSINSTANCE) |
+		      LS_64(roce_info->bind_en, IRDMAQPC_BINDEN) |
+		      LS_64(roce_info->fast_reg_en, IRDMAQPC_FASTREGEN) |
+		      LS_64(roce_info->dcqcn_en, IRDMAQPC_DCQCNENABLE) |
+		      LS_64(roce_info->rcv_no_icrc, IRDMAQPC_RCVNOICRC) |
+		      LS_64(roce_info->fw_cc_enable, IRDMAQPC_FW_CC_ENABLE) |
+		      LS_64(roce_info->udprivcq_en, IRDMAQPC_UDPRIVCQENABLE) |
+		      LS_64(roce_info->priv_mode_en, IRDMAQPC_PRIVEN) |
+		      LS_64(roce_info->timely_en, IRDMAQPC_TIMELYENABLE));
+	set_64bit_val(qp_ctx, 168,
+		      LS_64(info->qp_compl_ctx, IRDMAQPC_QPCOMPCTX));
+	set_64bit_val(qp_ctx, 176,
+		      LS_64(qp->sq_tph_val, IRDMAQPC_SQTPHVAL) |
+		      LS_64(qp->rq_tph_val, IRDMAQPC_RQTPHVAL) |
+		      LS_64(qp->qs_handle, IRDMAQPC_QSHANDLE));
+	set_64bit_val(qp_ctx, 184,
+		      LS_64(udp->local_ipaddr3, IRDMAQPC_LOCAL_IPADDR3) |
+		      LS_64(udp->local_ipaddr2, IRDMAQPC_LOCAL_IPADDR2));
+	set_64bit_val(qp_ctx, 192,
+		      LS_64(udp->local_ipaddr1, IRDMAQPC_LOCAL_IPADDR1) |
+		      LS_64(udp->local_ipaddr0, IRDMAQPC_LOCAL_IPADDR0));
+	set_64bit_val(qp_ctx, 200,
+		      LS_64(roce_info->t_high, IRDMAQPC_THIGH) |
+		      LS_64(roce_info->t_low, IRDMAQPC_TLOW));
+	set_64bit_val(qp_ctx, 208,
+		      LS_64(info->rem_endpoint_idx, IRDMAQPC_REMENDPOINTIDX));
+
+	irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "QP_HOST CTX WQE", qp_ctx,
+			IRDMA_QP_CTX_SIZE);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_alloc_local_mac_entry - allocate a mac entry
+ * @cqp: struct for cqp hw
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_alloc_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch,
+			       bool post_sq)
+{
+	__le64 *wqe;
+	u64 hdr;
+
+	wqe = cqp->dev->cqp_ops->cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	hdr = LS_64(IRDMA_CQP_OP_ALLOCATE_LOC_MAC_TABLE_ENTRY,
+		    IRDMA_CQPSQ_OPCODE) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "ALLOCATE_LOCAL_MAC WQE",
+			wqe, IRDMA_CQP_WQE_SIZE * 8);
+
+	if (post_sq)
+		cqp->dev->cqp_ops->cqp_post_sq(cqp);
+	return 0;
+}
+
+/**
+ * irdma_sc_add_local_mac_entry - add mac enry
+ * @cqp: struct for cqp hw
+ * @info:mac addr info
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_add_local_mac_entry(struct irdma_sc_cqp *cqp,
+			     struct irdma_local_mac_entry_info *info,
+			     u64 scratch, bool post_sq)
+{
+	__le64 *wqe;
+	u64 temp, header;
+
+	wqe = cqp->dev->cqp_ops->cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+	temp = info->mac_addr[5] | LS_64_1(info->mac_addr[4], 8) |
+	       LS_64_1(info->mac_addr[3], 16) | LS_64_1(info->mac_addr[2], 24) |
+	       LS_64_1(info->mac_addr[1], 32) | LS_64_1(info->mac_addr[0], 40);
+
+	set_64bit_val(wqe, 32, temp);
+
+	header = LS_64(info->entry_idx, IRDMA_CQPSQ_MLM_TABLEIDX) |
+		 LS_64(IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE, IRDMA_CQPSQ_OPCODE) |
+		 LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, header);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "ADD_LOCAL_MAC WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+
+	if (post_sq)
+		cqp->dev->cqp_ops->cqp_post_sq(cqp);
+	return 0;
+}
+
+/**
+ * irdma_sc_del_local_mac_entry - cqp wqe to dele local mac
+ * @cqp: struct for cqp hw
+ * @scratch: u64 saved to be used during cqp completion
+ * @entry_idx: index of mac entry
+ * @ignore_ref_count: to force mac adde delete
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_del_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch,
+			     u16 entry_idx, u8 ignore_ref_count, bool post_sq)
+{
+	__le64 *wqe;
+	u64 header;
+
+	wqe = cqp->dev->cqp_ops->cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+	header = LS_64(entry_idx, IRDMA_CQPSQ_MLM_TABLEIDX) |
+		 LS_64(IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE, IRDMA_CQPSQ_OPCODE) |
+		 LS_64(1, IRDMA_CQPSQ_MLM_FREEENTRY) |
+		 LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID) |
+		 LS_64(ignore_ref_count, IRDMA_CQPSQ_MLM_IGNORE_REF_CNT);
+
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, header);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "DEL_LOCAL_MAC_IPADDR WQE",
+			wqe, IRDMA_CQP_WQE_SIZE * 8);
+
+	if (post_sq)
+		cqp->dev->cqp_ops->cqp_post_sq(cqp);
+	return 0;
+}
+
+/**
+ * irdma_sc_qp_setctx - set qp's context
+ * @qp: sc qp
+ * @qp_ctx: context ptr
+ * @info: ctx info
+ */
+static enum irdma_status_code
+irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 *qp_ctx,
+		   struct irdma_qp_host_ctx_info *info)
+{
+	struct irdma_iwarp_offload_info *iw;
+	struct irdma_tcp_offload_info *tcp;
+	struct irdma_sc_dev *dev;
+	u8 push_mode_en;
+	u16 push_idx;
+	u64 qw0, qw3, qw7 = 0;
+	u64 mac = 0;
+
+	iw = info->iwarp_info;
+	tcp = info->tcp_info;
+	dev = qp->dev;
+
+	if (dev->hw_attrs.uk_attrs.hw_rev > IRDMA_GEN_1) {
+		mac = LS_64_1(iw->mac_addr[5], 16) |
+		      LS_64_1(iw->mac_addr[4], 24) |
+		      LS_64_1(iw->mac_addr[3], 32) |
+		      LS_64_1(iw->mac_addr[2], 40) |
+		      LS_64_1(iw->mac_addr[1], 48) |
+		      LS_64_1(iw->mac_addr[0], 56);
+	}
+
+	qp->user_pri = info->user_pri;
+	if (info->add_to_qoslist)
+		irdma_qp_add_qos(qp);
+
+	if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) {
+		push_mode_en = 0;
+		push_idx = 0;
+	} else {
+		push_mode_en = 1;
+		push_idx = qp->push_idx;
+	}
+	qw0 = LS_64(qp->qp_uk.rq_wqe_size, IRDMAQPC_RQWQESIZE) |
+	      LS_64(iw->err_rq_idx_valid, IRDMAQPC_ERR_RQ_IDX_VALID) |
+	      LS_64(qp->rcv_tph_en, IRDMAQPC_RCVTPHEN) |
+	      LS_64(qp->xmit_tph_en, IRDMAQPC_XMITTPHEN) |
+	      LS_64(qp->rq_tph_en, IRDMAQPC_RQTPHEN) |
+	      LS_64(qp->sq_tph_en, IRDMAQPC_SQTPHEN) |
+	      LS_64(push_idx, IRDMAQPC_PPIDX) |
+	      LS_64(push_mode_en, IRDMAQPC_PMENA) |
+	      LS_64(iw->ib_rd_en, IRDMAQPC_IBRDENABLE) |
+	      LS_64(iw->pd_id >> 16, IRDMAQPC_PDIDXHI);
+
+	set_64bit_val(qp_ctx, 8, qp->sq_pa);
+	set_64bit_val(qp_ctx, 16, qp->rq_pa);
+
+	qw3 = LS_64(qp->hw_rq_size, IRDMAQPC_RQSIZE) |
+	      LS_64(qp->hw_sq_size, IRDMAQPC_SQSIZE);
+	if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
+		qw3 |= LS_64(qp->src_mac_addr_idx, IRDMAQPC_GEN1_SRCMACADDRIDX);
+	set_64bit_val(qp_ctx, 128,
+		      LS_64(iw->err_rq_idx, IRDMAQPC_ERR_RQ_IDX));
+	set_64bit_val(qp_ctx, 136,
+		      LS_64(info->send_cq_num, IRDMAQPC_TXCQNUM) |
+		      LS_64(info->rcv_cq_num, IRDMAQPC_RXCQNUM));
+	set_64bit_val(qp_ctx, 168,
+		      LS_64(info->qp_compl_ctx, IRDMAQPC_QPCOMPCTX));
+	set_64bit_val(qp_ctx, 176,
+		      LS_64(qp->sq_tph_val, IRDMAQPC_SQTPHVAL) |
+		      LS_64(qp->rq_tph_val, IRDMAQPC_RQTPHVAL) |
+		      LS_64(qp->qs_handle, IRDMAQPC_QSHANDLE) |
+		      LS_64(qp->ieq_qp, IRDMAQPC_EXCEPTION_LAN_QUEUE));
+	if (info->iwarp_info_valid) {
+		qw0 |= LS_64(iw->ddp_ver, IRDMAQPC_DDP_VER) |
+		       LS_64(iw->rdmap_ver, IRDMAQPC_RDMAP_VER) |
+		       LS_64(iw->dctcp_en, IRDMAQPC_DC_TCP_EN) |
+		       LS_64(iw->ecn_en, IRDMAQPC_ECN_EN);
+		qw7 |= LS_64(iw->pd_id, IRDMAQPC_PDIDX);
+		set_64bit_val(qp_ctx, 144,
+			      LS_64(qp->q2_pa >> 8, IRDMAQPC_Q2ADDR) |
+			      LS_64(info->stats_idx, IRDMAQPC_STAT_INDEX));
+		set_64bit_val(qp_ctx, 152,
+			      mac | LS_64(iw->last_byte_sent, IRDMAQPC_LASTBYTESENT));
+		set_64bit_val(qp_ctx, 160,
+			      LS_64(iw->ord_size, IRDMAQPC_ORDSIZE) |
+			      LS_64(iw->ird_size, IRDMAQPC_IRDSIZE) |
+			      LS_64(iw->wr_rdresp_en, IRDMAQPC_WRRDRSPOK) |
+			      LS_64(iw->rd_en, IRDMAQPC_RDOK) |
+			      LS_64(iw->snd_mark_en, IRDMAQPC_SNDMARKERS) |
+			      LS_64(iw->bind_en, IRDMAQPC_BINDEN) |
+			      LS_64(iw->fast_reg_en, IRDMAQPC_FASTREGEN) |
+			      LS_64(iw->priv_mode_en, IRDMAQPC_PRIVEN) |
+			      LS_64(info->stats_idx_valid, IRDMAQPC_USESTATSINSTANCE) |
+			      LS_64(1, IRDMAQPC_IWARPMODE) |
+			      LS_64(iw->rcv_mark_en, IRDMAQPC_RCVMARKERS) |
+			      LS_64(iw->align_hdrs, IRDMAQPC_ALIGNHDRS) |
+			      LS_64(iw->rcv_no_mpa_crc, IRDMAQPC_RCVNOMPACRC) |
+			      LS_64(iw->rcv_mark_offset || !tcp ? iw->rcv_mark_offset : tcp->rcv_nxt, IRDMAQPC_RCVMARKOFFSET) |
+			      LS_64(iw->snd_mark_offset || !tcp ? iw->snd_mark_offset : tcp->snd_nxt, IRDMAQPC_SNDMARKOFFSET) |
+			      LS_64(iw->timely_en, IRDMAQPC_TIMELYENABLE));
+	}
+	if (info->tcp_info_valid) {
+		qw0 |= LS_64(tcp->ipv4, IRDMAQPC_IPV4) |
+		       LS_64(tcp->no_nagle, IRDMAQPC_NONAGLE) |
+		       LS_64(tcp->insert_vlan_tag, IRDMAQPC_INSERTVLANTAG) |
+		       LS_64(tcp->time_stamp, IRDMAQPC_TIMESTAMP) |
+		       LS_64(tcp->cwnd_inc_limit, IRDMAQPC_LIMIT) |
+		       LS_64(tcp->drop_ooo_seg, IRDMAQPC_DROPOOOSEG) |
+		       LS_64(tcp->dup_ack_thresh, IRDMAQPC_DUPACK_THRESH);
+
+		if ((iw->ecn_en || iw->dctcp_en) && !(tcp->tos & 0x03))
+			tcp->tos |= ECN_CODE_PT_VAL;
+
+		qw3 |= LS_64(tcp->ttl, IRDMAQPC_TTL) |
+		       LS_64(tcp->avoid_stretch_ack, IRDMAQPC_AVOIDSTRETCHACK) |
+		       LS_64(tcp->tos, IRDMAQPC_TOS) |
+		       LS_64(tcp->src_port, IRDMAQPC_SRCPORTNUM) |
+		       LS_64(tcp->dst_port, IRDMAQPC_DESTPORTNUM);
+		if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) {
+			qw3 |= LS_64(tcp->src_mac_addr_idx,
+				     IRDMAQPC_GEN1_SRCMACADDRIDX);
+
+			qp->src_mac_addr_idx = tcp->src_mac_addr_idx;
+		}
+		set_64bit_val(qp_ctx, 32,
+			      LS_64(tcp->dest_ip_addr2, IRDMAQPC_DESTIPADDR2) |
+			      LS_64(tcp->dest_ip_addr3, IRDMAQPC_DESTIPADDR3));
+		set_64bit_val(qp_ctx, 40,
+			      LS_64(tcp->dest_ip_addr0, IRDMAQPC_DESTIPADDR0) |
+			      LS_64(tcp->dest_ip_addr1, IRDMAQPC_DESTIPADDR1));
+		set_64bit_val(qp_ctx, 48,
+			      LS_64(tcp->snd_mss, IRDMAQPC_SNDMSS) |
+			      LS_64(tcp->syn_rst_handling, IRDMAQPC_SYN_RST_HANDLING) |
+			      LS_64(tcp->vlan_tag, IRDMAQPC_VLANTAG) |
+			      LS_64(tcp->arp_idx, IRDMAQPC_ARPIDX));
+		qw7 |= LS_64(tcp->flow_label, IRDMAQPC_FLOWLABEL) |
+		       LS_64(tcp->wscale, IRDMAQPC_WSCALE) |
+		       LS_64(tcp->ignore_tcp_opt, IRDMAQPC_IGNORE_TCP_OPT) |
+		       LS_64(tcp->ignore_tcp_uns_opt,
+			     IRDMAQPC_IGNORE_TCP_UNS_OPT) |
+		       LS_64(tcp->tcp_state, IRDMAQPC_TCPSTATE) |
+		       LS_64(tcp->rcv_wscale, IRDMAQPC_RCVSCALE) |
+		       LS_64(tcp->snd_wscale, IRDMAQPC_SNDSCALE);
+		set_64bit_val(qp_ctx, 72,
+			      LS_64(tcp->time_stamp_recent, IRDMAQPC_TIMESTAMP_RECENT) |
+			      LS_64(tcp->time_stamp_age, IRDMAQPC_TIMESTAMP_AGE));
+		set_64bit_val(qp_ctx, 80,
+			      LS_64(tcp->snd_nxt, IRDMAQPC_SNDNXT) |
+			      LS_64(tcp->snd_wnd, IRDMAQPC_SNDWND));
+		set_64bit_val(qp_ctx, 88,
+			      LS_64(tcp->rcv_nxt, IRDMAQPC_RCVNXT) |
+			      LS_64(tcp->rcv_wnd, IRDMAQPC_RCVWND));
+		set_64bit_val(qp_ctx, 96,
+			      LS_64(tcp->snd_max, IRDMAQPC_SNDMAX) |
+			      LS_64(tcp->snd_una, IRDMAQPC_SNDUNA));
+		set_64bit_val(qp_ctx, 104,
+			      LS_64(tcp->srtt, IRDMAQPC_SRTT) |
+			      LS_64(tcp->rtt_var, IRDMAQPC_RTTVAR));
+		set_64bit_val(qp_ctx, 112,
+			      LS_64(tcp->ss_thresh, IRDMAQPC_SSTHRESH) |
+			      LS_64(tcp->cwnd, IRDMAQPC_CWND));
+		set_64bit_val(qp_ctx, 120,
+			      LS_64(tcp->snd_wl1, IRDMAQPC_SNDWL1) |
+			      LS_64(tcp->snd_wl2, IRDMAQPC_SNDWL2));
+		set_64bit_val(qp_ctx, 128,
+			      LS_64(tcp->max_snd_window, IRDMAQPC_MAXSNDWND) |
+			      LS_64(tcp->rexmit_thresh, IRDMAQPC_REXMIT_THRESH));
+		set_64bit_val(qp_ctx, 184,
+			      LS_64(tcp->local_ipaddr3, IRDMAQPC_LOCAL_IPADDR3) |
+			      LS_64(tcp->local_ipaddr2, IRDMAQPC_LOCAL_IPADDR2));
+		set_64bit_val(qp_ctx, 192,
+			      LS_64(tcp->local_ipaddr1, IRDMAQPC_LOCAL_IPADDR1) |
+			      LS_64(tcp->local_ipaddr0, IRDMAQPC_LOCAL_IPADDR0));
+		set_64bit_val(qp_ctx, 200,
+			      LS_64(iw->t_high, IRDMAQPC_THIGH) |
+			      LS_64(iw->t_low, IRDMAQPC_TLOW));
+		set_64bit_val(qp_ctx, 208,
+			      LS_64(info->rem_endpoint_idx, IRDMAQPC_REMENDPOINTIDX));
+	}
+
+	set_64bit_val(qp_ctx, 0, qw0);
+	set_64bit_val(qp_ctx, 24, qw3);
+	set_64bit_val(qp_ctx, 56, qw7);
+
+	irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "QP_HOST CTX", qp_ctx,
+			IRDMA_QP_CTX_SIZE);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_alloc_stag - mr stag alloc
+ * @dev: sc device struct
+ * @info: stag info
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_alloc_stag(struct irdma_sc_dev *dev,
+		    struct irdma_allocate_stag_info *info, u64 scratch,
+		    bool post_sq)
+{
+	__le64 *wqe;
+	struct irdma_sc_cqp *cqp;
+	u64 hdr;
+	enum irdma_page_size page_size;
+
+	if (info->page_size == 0x40000000)
+		page_size = IRDMA_PAGE_SIZE_1G;
+	else if (info->page_size == 0x200000)
+		page_size = IRDMA_PAGE_SIZE_2M;
+	else
+		page_size = IRDMA_PAGE_SIZE_4K;
+
+	cqp = dev->cqp;
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	set_64bit_val(wqe, 8,
+		      FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID) |
+		      LS_64(info->total_len, IRDMA_CQPSQ_STAG_STAGLEN));
+	set_64bit_val(wqe, 16,
+		      LS_64(info->stag_idx, IRDMA_CQPSQ_STAG_IDX));
+	set_64bit_val(wqe, 40,
+		      LS_64(info->hmc_fcn_index, IRDMA_CQPSQ_STAG_HMCFNIDX));
+
+	if (info->chunk_size)
+		set_64bit_val(wqe, 48,
+			      LS_64(info->first_pm_pbl_idx, IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX));
+
+	hdr = LS_64(IRDMA_CQP_OP_ALLOC_STAG, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(1, IRDMA_CQPSQ_STAG_MR) |
+	      LS_64(info->access_rights, IRDMA_CQPSQ_STAG_ARIGHTS) |
+	      LS_64(info->chunk_size, IRDMA_CQPSQ_STAG_LPBLSIZE) |
+	      LS_64(page_size, IRDMA_CQPSQ_STAG_HPAGESIZE) |
+	      LS_64(info->remote_access, IRDMA_CQPSQ_STAG_REMACCENABLED) |
+	      LS_64(info->use_hmc_fcn_index, IRDMA_CQPSQ_STAG_USEHMCFNIDX) |
+	      LS_64(info->use_pf_rid, IRDMA_CQPSQ_STAG_USEPFRID) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "ALLOC_STAG WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	if (post_sq)
+		irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_mr_reg_non_shared - non-shared mr registration
+ * @dev: sc device struct
+ * @info: mr info
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_mr_reg_non_shared(struct irdma_sc_dev *dev,
+			   struct irdma_reg_ns_stag_info *info, u64 scratch,
+			   bool post_sq)
+{
+	__le64 *wqe;
+	u64 temp;
+	struct irdma_sc_cqp *cqp;
+	u64 hdr;
+	u32 pble_obj_cnt;
+	bool remote_access;
+	u8 addr_type;
+	enum irdma_page_size page_size;
+
+	if (info->page_size == 0x40000000)
+		page_size = IRDMA_PAGE_SIZE_1G;
+	else if (info->page_size == 0x200000)
+		page_size = IRDMA_PAGE_SIZE_2M;
+	else
+		page_size = IRDMA_PAGE_SIZE_4K;
+
+	if (info->access_rights & (IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY |
+				   IRDMA_ACCESS_FLAGS_REMOTEWRITE_ONLY))
+		remote_access = true;
+	else
+		remote_access = false;
+
+	pble_obj_cnt = dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
+	if (info->chunk_size && info->first_pm_pbl_index >= pble_obj_cnt)
+		return IRDMA_ERR_INVALID_PBLE_INDEX;
+
+	cqp = dev->cqp;
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	temp = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ?
+		(uintptr_t)info->va : info->fbo;
+
+	set_64bit_val(wqe, 0, temp);
+	set_64bit_val(wqe, 8,
+		      LS_64(info->total_len, IRDMA_CQPSQ_STAG_STAGLEN) |
+		      FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));
+	set_64bit_val(wqe, 16,
+		      LS_64(info->stag_key, IRDMA_CQPSQ_STAG_KEY) |
+		      LS_64(info->stag_idx, IRDMA_CQPSQ_STAG_IDX));
+	if (!info->chunk_size) {
+		set_64bit_val(wqe, 32, info->reg_addr_pa);
+		set_64bit_val(wqe, 48, 0);
+	} else {
+		set_64bit_val(wqe, 32, 0);
+		set_64bit_val(wqe, 48,
+			      LS_64(info->first_pm_pbl_index, IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX));
+	}
+	set_64bit_val(wqe, 40, info->hmc_fcn_index);
+	set_64bit_val(wqe, 56, 0);
+
+	addr_type = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? 1 : 0;
+	hdr = LS_64(IRDMA_CQP_OP_REG_MR, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(1, IRDMA_CQPSQ_STAG_MR) |
+	      LS_64(info->chunk_size, IRDMA_CQPSQ_STAG_LPBLSIZE) |
+	      LS_64(page_size, IRDMA_CQPSQ_STAG_HPAGESIZE) |
+	      LS_64(info->access_rights, IRDMA_CQPSQ_STAG_ARIGHTS) |
+	      LS_64(remote_access, IRDMA_CQPSQ_STAG_REMACCENABLED) |
+	      LS_64(addr_type, IRDMA_CQPSQ_STAG_VABASEDTO) |
+	      LS_64(info->use_hmc_fcn_index, IRDMA_CQPSQ_STAG_USEHMCFNIDX) |
+	      LS_64(info->use_pf_rid, IRDMA_CQPSQ_STAG_USEPFRID) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "MR_REG_NS WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	if (post_sq)
+		irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_mr_reg_shared - registered shared memory region
+ * @dev: sc device struct
+ * @info: info for shared memory registration
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_mr_reg_shared(struct irdma_sc_dev *dev,
+		       struct irdma_register_shared_stag *info, u64 scratch,
+		       bool post_sq)
+{
+	__le64 *wqe;
+	struct irdma_sc_cqp *cqp;
+	u64 temp, va64, fbo, hdr;
+	u32 va32;
+	bool remote_access;
+	u8 addr_type;
+
+	if (info->access_rights & (IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY |
+				   IRDMA_ACCESS_FLAGS_REMOTEWRITE_ONLY))
+		remote_access = true;
+	else
+		remote_access = false;
+	cqp = dev->cqp;
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+	va64 = (uintptr_t)(info->va);
+	va32 = (u32)(va64 & 0x00000000FFFFFFFF);
+	fbo = (u64)(va32 & (4096 - 1));
+
+	set_64bit_val(wqe, 0,
+		      (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED ?
+		       (uintptr_t)info->va : fbo));
+	set_64bit_val(wqe, 8,
+		      FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));
+	temp = LS_64(info->new_stag_key, IRDMA_CQPSQ_STAG_KEY) |
+	       LS_64(info->new_stag_idx, IRDMA_CQPSQ_STAG_IDX) |
+	       LS_64(info->parent_stag_idx, IRDMA_CQPSQ_STAG_PARENTSTAGIDX);
+	set_64bit_val(wqe, 16, temp);
+
+	addr_type = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? 1 : 0;
+	hdr = LS_64(IRDMA_CQP_OP_REG_SMR, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(1, IRDMA_CQPSQ_STAG_MR) |
+	      LS_64(info->access_rights, IRDMA_CQPSQ_STAG_ARIGHTS) |
+	      LS_64(remote_access, IRDMA_CQPSQ_STAG_REMACCENABLED) |
+	      LS_64(addr_type, IRDMA_CQPSQ_STAG_VABASEDTO) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "MR_REG_SHARED WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	if (post_sq)
+		irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_dealloc_stag - deallocate stag
+ * @dev: sc device struct
+ * @info: dealloc stag info
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_dealloc_stag(struct irdma_sc_dev *dev,
+		      struct irdma_dealloc_stag_info *info, u64 scratch,
+		      bool post_sq)
+{
+	u64 hdr;
+	__le64 *wqe;
+	struct irdma_sc_cqp *cqp;
+
+	cqp = dev->cqp;
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	set_64bit_val(wqe, 8,
+		      FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));
+	set_64bit_val(wqe, 16,
+		      LS_64(info->stag_idx, IRDMA_CQPSQ_STAG_IDX));
+
+	hdr = LS_64(IRDMA_CQP_OP_DEALLOC_STAG, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(info->mr, IRDMA_CQPSQ_STAG_MR) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "DEALLOC_STAG WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	if (post_sq)
+		irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_query_stag - query hardware for stag
+ * @dev: sc device struct
+ * @scratch: u64 saved to be used during cqp completion
+ * @stag_index: stag index for query
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code irdma_sc_query_stag(struct irdma_sc_dev *dev,
+						  u64 scratch, u32 stag_index,
+						  bool post_sq)
+{
+	u64 hdr;
+	__le64 *wqe;
+	struct irdma_sc_cqp *cqp;
+
+	cqp = dev->cqp;
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	set_64bit_val(wqe, 16,
+		      LS_64(stag_index, IRDMA_CQPSQ_QUERYSTAG_IDX));
+
+	hdr = LS_64(IRDMA_CQP_OP_QUERY_STAG, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "QUERY_STAG WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	if (post_sq)
+		irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_mw_alloc - mw allocate
+ * @dev: sc device struct
+ * @info: memory window allocation information
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_mw_alloc(struct irdma_sc_dev *dev, struct irdma_mw_alloc_info *info,
+		  u64 scratch, bool post_sq)
+{
+	u64 hdr;
+	struct irdma_sc_cqp *cqp;
+	__le64 *wqe;
+
+	cqp = dev->cqp;
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	set_64bit_val(wqe, 8,
+		      FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));
+	set_64bit_val(wqe, 16,
+		      LS_64(info->mw_stag_index, IRDMA_CQPSQ_STAG_IDX));
+
+	hdr = LS_64(IRDMA_CQP_OP_ALLOC_STAG, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(info->mw_wide, IRDMA_CQPSQ_STAG_MWTYPE) |
+	      LS_64(info->mw1_bind_dont_vldt_key,
+		    IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "MW_ALLOC WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	if (post_sq)
+		irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp
+ * @qp: sc qp struct
+ * @info: fast mr info
+ * @post_sq: flag for cqp db to ring
+ */
+enum irdma_status_code
+irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
+			  struct irdma_fast_reg_stag_info *info, bool post_sq)
+{
+	u64 temp, hdr;
+	__le64 *wqe;
+	u32 wqe_idx;
+	enum irdma_page_size page_size;
+	struct irdma_post_sq_info sq_info = {};
+
+	if (info->page_size == 0x40000000)
+		page_size = IRDMA_PAGE_SIZE_1G;
+	else if (info->page_size == 0x200000)
+		page_size = IRDMA_PAGE_SIZE_2M;
+	else
+		page_size = IRDMA_PAGE_SIZE_4K;
+
+	sq_info.wr_id = info->wr_id;
+	sq_info.signaled = info->signaled;
+	sq_info.push_wqe = info->push_wqe;
+
+	wqe = irdma_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx,
+					 IRDMA_QP_WQE_MIN_QUANTA, 0, &sq_info);
+	if (!wqe)
+		return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
+
+	irdma_clr_wqes(&qp->qp_uk, wqe_idx);
+
+	dev_dbg(rfdev_to_dev(qp->dev),
+		"MR: wr_id[%llxh] wqe_idx[%04d] location[%p]\n", info->wr_id,
+		wqe_idx, &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid);
+
+	temp = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ?
+		(uintptr_t)info->va : info->fbo;
+	set_64bit_val(wqe, 0, temp);
+
+	temp = RS_64(info->first_pm_pbl_index >> 16, IRDMAQPSQ_FIRSTPMPBLIDXHI);
+	set_64bit_val(wqe, 8,
+		      LS_64(temp, IRDMAQPSQ_FIRSTPMPBLIDXHI) |
+		      LS_64(info->reg_addr_pa >> IRDMAQPSQ_PBLADDR_S, IRDMAQPSQ_PBLADDR));
+	set_64bit_val(wqe, 16,
+		      info->total_len |
+		      LS_64(info->first_pm_pbl_index, IRDMAQPSQ_FIRSTPMPBLIDXLO));
+
+	hdr = LS_64(info->stag_key, IRDMAQPSQ_STAGKEY) |
+	      LS_64(info->stag_idx, IRDMAQPSQ_STAGINDEX) |
+	      LS_64(IRDMAQP_OP_FAST_REGISTER, IRDMAQPSQ_OPCODE) |
+	      LS_64(info->chunk_size, IRDMAQPSQ_LPBLSIZE) |
+	      LS_64(page_size, IRDMAQPSQ_HPAGESIZE) |
+	      LS_64(info->access_rights, IRDMAQPSQ_STAGRIGHTS) |
+	      LS_64(info->addr_type, IRDMAQPSQ_VABASEDTO) |
+	      LS_64((sq_info.push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) |
+	      LS_64(info->read_fence, IRDMAQPSQ_READFENCE) |
+	      LS_64(info->local_fence, IRDMAQPSQ_LOCALFENCE) |
+	      LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) |
+	      LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "FAST_REG WQE", wqe,
+			IRDMA_QP_WQE_MIN_SIZE);
+	if (sq_info.push_wqe) {
+		irdma_qp_push_wqe(&qp->qp_uk, wqe, IRDMA_QP_WQE_MIN_QUANTA,
+				  wqe_idx, post_sq);
+	} else {
+		if (post_sq)
+			irdma_qp_post_wr(&qp->qp_uk);
+	}
+
+	return 0;
+}
+
+/**
+ * irdma_sc_gen_rts_ae - request AE generated after RTS
+ * @qp: sc qp struct
+ */
+static void irdma_sc_gen_rts_ae(struct irdma_sc_qp *qp)
+{
+	__le64 *wqe;
+	u64 hdr;
+	struct irdma_qp_uk *qp_uk;
+
+	qp_uk = &qp->qp_uk;
+
+	wqe = qp_uk->sq_base[1].elem;
+
+	hdr = LS_64(IRDMAQP_OP_NOP, IRDMAQPSQ_OPCODE) |
+	      LS_64(1, IRDMAQPSQ_LOCALFENCE) |
+	      LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+	irdma_debug_buf(qp->dev, IRDMA_DEBUG_QP, "NOP W/LOCAL FENCE WQE", wqe,
+			IRDMA_QP_WQE_MIN_SIZE);
+
+	wqe = qp_uk->sq_base[2].elem;
+	hdr = LS_64(IRDMAQP_OP_GEN_RTS_AE, IRDMAQPSQ_OPCODE) |
+	      LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+	irdma_debug_buf(qp->dev, IRDMA_DEBUG_QP, "CONN EST WQE", wqe,
+			IRDMA_QP_WQE_MIN_SIZE);
+}
+
+/**
+ * irdma_sc_send_lsmm - send last streaming mode message
+ * @qp: sc qp struct
+ * @lsmm_buf: buffer with lsmm message
+ * @size: size of lsmm buffer
+ * @stag: stag of lsmm buffer
+ */
+static void irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size,
+			       irdma_stag stag)
+{
+	__le64 *wqe;
+	u64 hdr;
+	struct irdma_qp_uk *qp_uk;
+
+	qp_uk = &qp->qp_uk;
+	wqe = qp_uk->sq_base->elem;
+
+	set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);
+	if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {
+		set_64bit_val(wqe, 8,
+			      LS_64(size, IRDMAQPSQ_GEN1_FRAG_LEN) |
+			      LS_64(stag, IRDMAQPSQ_GEN1_FRAG_STAG));
+	} else {
+		set_64bit_val(wqe, 8,
+			      LS_64(size, IRDMAQPSQ_FRAG_LEN) |
+			      LS_64(stag, IRDMAQPSQ_FRAG_STAG) |
+			      LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID));
+	}
+	set_64bit_val(wqe, 16, 0);
+
+	hdr = LS_64(IRDMAQP_OP_RDMA_SEND, IRDMAQPSQ_OPCODE) |
+	      LS_64(1, IRDMAQPSQ_STREAMMODE) |
+	      LS_64(1, IRDMAQPSQ_WAITFORRCVPDU) |
+	      LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "SEND_LSMM WQE", wqe,
+			IRDMA_QP_WQE_MIN_SIZE);
+
+	if (qp->dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE)
+		irdma_sc_gen_rts_ae(qp);
+}
+
+/**
+ * irdma_sc_send_lsmm_nostag - for privilege qp
+ * @qp: sc qp struct
+ * @lsmm_buf: buffer with lsmm message
+ * @size: size of lsmm buffer
+ */
+static void irdma_sc_send_lsmm_nostag(struct irdma_sc_qp *qp, void *lsmm_buf,
+				      u32 size)
+{
+	__le64 *wqe;
+	u64 hdr;
+	struct irdma_qp_uk *qp_uk;
+
+	qp_uk = &qp->qp_uk;
+	wqe = qp_uk->sq_base->elem;
+
+	set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);
+
+	if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1)
+		set_64bit_val(wqe, 8,
+			      LS_64(size, IRDMAQPSQ_GEN1_FRAG_LEN));
+	else
+		set_64bit_val(wqe, 8,
+			      LS_64(size, IRDMAQPSQ_FRAG_LEN) |
+			      LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID));
+	set_64bit_val(wqe, 16, 0);
+
+	hdr = LS_64(IRDMAQP_OP_RDMA_SEND, IRDMAQPSQ_OPCODE) |
+	      LS_64(1, IRDMAQPSQ_STREAMMODE) |
+	      LS_64(1, IRDMAQPSQ_WAITFORRCVPDU) |
+	      LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "SEND_LSMM_NOSTAG WQE", wqe,
+			IRDMA_QP_WQE_MIN_SIZE);
+}
+
+/**
+ * irdma_sc_send_rtt - send last read0 or write0
+ * @qp: sc qp struct
+ * @read: Do read0 or write0
+ */
+static void irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read)
+{
+	__le64 *wqe;
+	u64 hdr;
+	struct irdma_qp_uk *qp_uk;
+
+	qp_uk = &qp->qp_uk;
+	wqe = qp_uk->sq_base->elem;
+
+	set_64bit_val(wqe, 0, 0);
+	set_64bit_val(wqe, 16, 0);
+	if (read) {
+		if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {
+			set_64bit_val(wqe, 8,
+				      LS_64(0xabcd, IRDMAQPSQ_GEN1_FRAG_STAG));
+		} else {
+			set_64bit_val(wqe, 8,
+				      (u64)0xabcd | LS_64(qp->qp_uk.swqe_polarity,
+				      IRDMAQPSQ_VALID));
+		}
+		hdr = LS_64(0x1234, IRDMAQPSQ_REMSTAG) |
+		      LS_64(IRDMAQP_OP_RDMA_READ, IRDMAQPSQ_OPCODE) |
+		      LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);
+
+	} else {
+		if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {
+			set_64bit_val(wqe, 8, 0);
+		} else {
+			set_64bit_val(wqe, 8,
+				      LS_64(qp->qp_uk.swqe_polarity,
+					    IRDMAQPSQ_VALID));
+		}
+		hdr = LS_64(IRDMAQP_OP_RDMA_WRITE, IRDMAQPSQ_OPCODE) |
+		      LS_64(qp->qp_uk.swqe_polarity, IRDMAQPSQ_VALID);
+	}
+
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "RTR WQE", wqe,
+			IRDMA_QP_WQE_MIN_SIZE);
+
+	if (qp->dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE)
+		irdma_sc_gen_rts_ae(qp);
+}
+
+/**
+ * irdma_iwarp_opcode - determine if incoming is rdma layer
+ * @info: aeq info for the packet
+ * @pkt: packet for error
+ */
+static u32 irdma_iwarp_opcode(struct irdma_aeqe_info *info, u8 *pkt)
+{
+	__be16 *mpa;
+	u32 opcode = 0xffffffff;
+
+	if (info->q2_data_written) {
+		mpa = (__be16 *)pkt;
+		opcode = ntohs(mpa[1]) & 0xf;
+	}
+
+	return opcode;
+}
+
+/**
+ * irdma_locate_mpa - return pointer to mpa in the pkt
+ * @pkt: packet with data
+ */
+static u8 *irdma_locate_mpa(u8 *pkt)
+{
+	/* skip over ethernet header */
+	pkt += IRDMA_MAC_HLEN;
+
+	/* Skip over IP and TCP headers */
+	pkt += 4 * (pkt[0] & 0x0f);
+	pkt += 4 * ((pkt[12] >> 4) & 0x0f);
+
+	return pkt;
+}
+
+/**
+ * irdma_bld_termhdr_ctrl - setup terminate hdr control fields
+ * @qp: sc qp ptr for pkt
+ * @hdr: term hdr
+ * @opcode: flush opcode for termhdr
+ * @layer_etype: error layer + error type
+ * @err: error cod ein the header
+ */
+static void irdma_bld_termhdr_ctrl(struct irdma_sc_qp *qp,
+				   struct irdma_terminate_hdr *hdr,
+				   enum irdma_flush_opcode opcode,
+				   u8 layer_etype, u8 err)
+{
+	qp->flush_code = opcode;
+	hdr->layer_etype = layer_etype;
+	hdr->error_code = err;
+}
+
+/**
+ * irdma_bld_termhdr_ddp_rdma - setup ddp and rdma hdrs in terminate hdr
+ * @pkt: ptr to mpa in offending pkt
+ * @hdr: term hdr
+ * @copy_len: offending pkt length to be copied to term hdr
+ * @is_tagged: DDP tagged or untagged
+ */
+static void irdma_bld_termhdr_ddp_rdma(u8 *pkt, struct irdma_terminate_hdr *hdr,
+				       int *copy_len, u8 *is_tagged)
+{
+	u16 ddp_seg_len;
+
+	ddp_seg_len = ntohs(*(__be16 *)pkt);
+	if (ddp_seg_len) {
+		*copy_len = 2;
+		hdr->hdrct = DDP_LEN_FLAG;
+		if (pkt[2] & 0x80) {
+			*is_tagged = 1;
+			if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
+				*copy_len += TERM_DDP_LEN_TAGGED;
+				hdr->hdrct |= DDP_HDR_FLAG;
+			}
+		} else {
+			if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
+				*copy_len += TERM_DDP_LEN_UNTAGGED;
+				hdr->hdrct |= DDP_HDR_FLAG;
+			}
+			if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN) &&
+			    ((pkt[3] & RDMA_OPCODE_M) == RDMA_READ_REQ_OPCODE)) {
+				*copy_len += TERM_RDMA_LEN;
+				hdr->hdrct |= RDMA_HDR_FLAG;
+			}
+		}
+	}
+}
+
+/**
+ * irdma_bld_terminate_hdr - build terminate message header
+ * @qp: qp associated with received terminate AE
+ * @info: the struct contiaing AE information
+ */
+static int irdma_bld_terminate_hdr(struct irdma_sc_qp *qp,
+				   struct irdma_aeqe_info *info)
+{
+	u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;
+	int copy_len = 0;
+	u8 is_tagged = 0;
+	u32 opcode;
+	struct irdma_terminate_hdr *termhdr;
+
+	termhdr = (struct irdma_terminate_hdr *)qp->q2_buf;
+	memset(termhdr, 0, Q2_BAD_FRAME_OFFSET);
+
+	if (info->q2_data_written) {
+		pkt = irdma_locate_mpa(pkt);
+		irdma_bld_termhdr_ddp_rdma(pkt, termhdr, &copy_len, &is_tagged);
+	}
+
+	opcode = irdma_iwarp_opcode(info, pkt);
+	qp->eventtype = TERM_EVENT_QP_FATAL;
+
+	switch (info->ae_id) {
+	case IRDMA_AE_AMP_UNALLOCATED_STAG:
+		qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
+		if (opcode == IRDMA_OP_TYPE_RDMA_WRITE)
+			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR,
+					       (LAYER_DDP << 4) | DDP_TAGGED_BUF,
+					       DDP_TAGGED_INV_STAG);
+		else
+			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
+					       (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
+					       RDMAP_INV_STAG);
+		break;
+	case IRDMA_AE_AMP_BOUNDS_VIOLATION:
+		qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
+		if (info->q2_data_written)
+			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR,
+					       (LAYER_DDP << 4) | DDP_TAGGED_BUF,
+					       DDP_TAGGED_BOUNDS);
+		else
+			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
+					       (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
+					       RDMAP_INV_BOUNDS);
+		break;
+	case IRDMA_AE_AMP_BAD_PD:
+		switch (opcode) {
+		case IRDMA_OP_TYPE_RDMA_WRITE:
+			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR,
+					       (LAYER_DDP << 4) | DDP_TAGGED_BUF,
+					       DDP_TAGGED_UNASSOC_STAG);
+			break;
+		case IRDMA_OP_TYPE_SEND_INV:
+		case IRDMA_OP_TYPE_SEND_SOL_INV:
+			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
+					       (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
+					       RDMAP_CANT_INV_STAG);
+			break;
+		default:
+			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
+					       (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
+					       RDMAP_UNASSOC_STAG);
+		}
+		break;
+	case IRDMA_AE_AMP_INVALID_STAG:
+		qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
+				       (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
+				       RDMAP_INV_STAG);
+		break;
+	case IRDMA_AE_AMP_BAD_QP:
+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_QP_OP_ERR,
+				       (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
+				       DDP_UNTAGGED_INV_QN);
+		break;
+	case IRDMA_AE_AMP_BAD_STAG_KEY:
+	case IRDMA_AE_AMP_BAD_STAG_INDEX:
+		qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
+		switch (opcode) {
+		case IRDMA_OP_TYPE_SEND_INV:
+		case IRDMA_OP_TYPE_SEND_SOL_INV:
+			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_OP_ERR,
+					       (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,
+					       RDMAP_CANT_INV_STAG);
+			break;
+		default:
+			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
+					       (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,
+					       RDMAP_INV_STAG);
+		}
+		break;
+	case IRDMA_AE_AMP_RIGHTS_VIOLATION:
+	case IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
+	case IRDMA_AE_PRIV_OPERATION_DENIED:
+		qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
+				       (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
+				       RDMAP_ACCESS);
+		break;
+	case IRDMA_AE_AMP_TO_WRAP:
+		qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
+				       (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
+				       RDMAP_TO_WRAP);
+		break;
+	case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
+				       (LAYER_MPA << 4) | DDP_LLP, MPA_CRC);
+		break;
+	case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_LEN_ERR,
+				       (LAYER_DDP << 4) | DDP_CATASTROPHIC,
+				       DDP_CATASTROPHIC_LOCAL);
+		break;
+	case IRDMA_AE_LCE_QP_CATASTROPHIC:
+	case IRDMA_AE_DDP_NO_L_BIT:
+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_FATAL_ERR,
+				       (LAYER_DDP << 4) | DDP_CATASTROPHIC,
+				       DDP_CATASTROPHIC_LOCAL);
+		break;
+	case IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN:
+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
+				       (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
+				       DDP_UNTAGGED_INV_MSN_RANGE);
+		break;
+	case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
+		qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_LEN_ERR,
+				       (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
+				       DDP_UNTAGGED_INV_TOO_LONG);
+		break;
+	case IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION:
+		if (is_tagged)
+			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
+					       (LAYER_DDP << 4) | DDP_TAGGED_BUF,
+					       DDP_TAGGED_INV_DDP_VER);
+		else
+			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
+					       (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
+					       DDP_UNTAGGED_INV_DDP_VER);
+		break;
+	case IRDMA_AE_DDP_UBE_INVALID_MO:
+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
+				       (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
+				       DDP_UNTAGGED_INV_MO);
+		break;
+	case IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_OP_ERR,
+				       (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
+				       DDP_UNTAGGED_INV_MSN_NO_BUF);
+		break;
+	case IRDMA_AE_DDP_UBE_INVALID_QN:
+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
+				       (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
+				       DDP_UNTAGGED_INV_QN);
+		break;
+	case IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
+				       (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,
+				       RDMAP_INV_RDMAP_VER);
+		break;
+	default:
+		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_FATAL_ERR,
+				       (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,
+				       RDMAP_UNSPECIFIED);
+		break;
+	}
+
+	if (copy_len)
+		memcpy(termhdr + 1, pkt, copy_len);
+
+	return sizeof(struct irdma_terminate_hdr) + copy_len;
+}
+
+/**
+ * irdma_terminate_send_fin() - Send fin for terminate message
+ * @qp: qp associated with received terminate AE
+ */
+void irdma_terminate_send_fin(struct irdma_sc_qp *qp)
+{
+	irdma_term_modify_qp(qp, IRDMA_QP_STATE_TERMINATE,
+			     IRDMAQP_TERM_SEND_FIN_ONLY, 0);
+}
+
+/**
+ * irdma_terminate_connection() - Bad AE and send terminate to remote QP
+ * @qp: qp associated with received terminate AE
+ * @info: the struct contiaing AE information
+ */
+void irdma_terminate_connection(struct irdma_sc_qp *qp,
+				struct irdma_aeqe_info *info)
+{
+	u8 termlen = 0;
+
+	if (qp->term_flags & IRDMA_TERM_SENT)
+		return;
+
+	termlen = irdma_bld_terminate_hdr(qp, info);
+	irdma_terminate_start_timer(qp);
+	qp->term_flags |= IRDMA_TERM_SENT;
+	irdma_term_modify_qp(qp, IRDMA_QP_STATE_TERMINATE,
+			     IRDMAQP_TERM_SEND_TERM_ONLY, termlen);
+}
+
+/**
+ * irdma_terminate_received - handle terminate received AE
+ * @qp: qp associated with received terminate AE
+ * @info: the struct contiaing AE information
+ */
+void irdma_terminate_received(struct irdma_sc_qp *qp,
+			      struct irdma_aeqe_info *info)
+{
+	u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;
+	__be32 *mpa;
+	u8 ddp_ctl;
+	u8 rdma_ctl;
+	u16 aeq_id = 0;
+	struct irdma_terminate_hdr *termhdr;
+
+	mpa = (__be32 *)irdma_locate_mpa(pkt);
+	if (info->q2_data_written) {
+		/* did not validate the frame - do it now */
+		ddp_ctl = (ntohl(mpa[0]) >> 8) & 0xff;
+		rdma_ctl = ntohl(mpa[0]) & 0xff;
+		if ((ddp_ctl & 0xc0) != 0x40)
+			aeq_id = IRDMA_AE_LCE_QP_CATASTROPHIC;
+		else if ((ddp_ctl & 0x03) != 1)
+			aeq_id = IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION;
+		else if (ntohl(mpa[2]) != 2)
+			aeq_id = IRDMA_AE_DDP_UBE_INVALID_QN;
+		else if (ntohl(mpa[3]) != 1)
+			aeq_id = IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN;
+		else if (ntohl(mpa[4]) != 0)
+			aeq_id = IRDMA_AE_DDP_UBE_INVALID_MO;
+		else if ((rdma_ctl & 0xc0) != 0x40)
+			aeq_id = IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION;
+
+		info->ae_id = aeq_id;
+		if (info->ae_id) {
+			/* Bad terminate recvd - send back a terminate */
+			irdma_terminate_connection(qp, info);
+			return;
+		}
+	}
+
+	qp->term_flags |= IRDMA_TERM_RCVD;
+	qp->eventtype = TERM_EVENT_QP_FATAL;
+	termhdr = (struct irdma_terminate_hdr *)&mpa[5];
+	if (termhdr->layer_etype == RDMAP_REMOTE_PROT ||
+	    termhdr->layer_etype == RDMAP_REMOTE_OP) {
+		irdma_terminate_done(qp, 0);
+	} else {
+		irdma_terminate_start_timer(qp);
+		irdma_terminate_send_fin(qp);
+	}
+}
+
+static enum irdma_status_code irdma_null_ws_add(struct irdma_sc_vsi *vsi,
+						u8 user_pri)
+{
+	return 0;
+}
+
+static void irdma_null_ws_remove(struct irdma_sc_vsi *vsi, u8 user_pri)
+{
+	/* do nothing */
+}
+
+static void irdma_null_ws_reset(struct irdma_sc_vsi *vsi)
+{
+	/* do nothing */
+}
+
+/**
+ * irdma_sc_vsi_init - Init the vsi structure
+ * @vsi: pointer to vsi structure to initialize
+ * @info: the info used to initialize the vsi struct
+ */
+void irdma_sc_vsi_init(struct irdma_sc_vsi *vsi,
+		       struct irdma_vsi_init_info *info)
+{
+	int i;
+	u32 reg_data = 0;
+	u32 reg_offset = 0;
+	struct irdma_l2params *l2p;
+
+	vsi->dev = info->dev;
+	vsi->back_vsi = info->back_vsi;
+	vsi->register_qset = info->register_qset;
+	vsi->unregister_qset = info->unregister_qset;
+	vsi->mtu = info->params->mtu;
+	vsi->exception_lan_q = info->exception_lan_q;
+	vsi->vsi_idx = info->pf_data_vsi_num;
+	vsi->vm_vf_type = info->vm_vf_type;
+	vsi->vm_id = info->vm_id;
+	if (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
+		vsi->fcn_id = info->dev->hmc_fn_id;
+
+	l2p = info->params;
+	vsi->qos_rel_bw = l2p->vsi_rel_bw;
+	vsi->qos_prio_type = l2p->vsi_prio_type;
+	for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
+		if (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
+			vsi->qos[i].qs_handle = l2p->qs_handle_list[i];
+		vsi->qos[i].traffic_class = info->params->up2tc[i];
+		vsi->qos[i].rel_bw =
+			l2p->tc_info[vsi->qos[i].traffic_class].rel_bw;
+		vsi->qos[i].prio_type =
+			l2p->tc_info[vsi->qos[i].traffic_class].prio_type;
+		spin_lock_init(&vsi->qos[i].lock);
+		INIT_LIST_HEAD(&vsi->qos[i].qplist);
+	}
+	if (vsi->register_qset) {
+		vsi->dev->ws_add = irdma_ws_add;
+		vsi->dev->ws_remove = irdma_ws_remove;
+		vsi->dev->ws_reset = irdma_ws_reset;
+	} else {
+		vsi->dev->ws_add = irdma_null_ws_add;
+		vsi->dev->ws_remove = irdma_null_ws_remove;
+		vsi->dev->ws_reset = irdma_null_ws_reset;
+	}
+	if (info->dev->privileged) {
+		reg_offset = info->dev->hw_regs[IRDMA_VSIQF_PE_CTL1] +
+			     4 * (vsi->vsi_idx);
+		if (vsi->dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) {
+			wr32(info->dev->hw, reg_offset, 0x1);
+		} else {
+			reg_data = rd32(info->dev->hw, reg_offset);
+			reg_data |= 0x2;
+			wr32(info->dev->hw, reg_offset, reg_data);
+		}
+	}
+}
+
+/**
+ * irdma_get_fcn_id - Return the function id
+ * @vsi: pointer to the vsi
+ */
+static u8 irdma_get_fcn_id(struct irdma_sc_vsi *vsi)
+{
+	struct irdma_stats_inst_info stats_info = {};
+	struct irdma_sc_dev *dev = vsi->dev;
+	u8 fcn_id = IRDMA_INVALID_FCN_ID;
+	u8 start_idx, max_stats, i;
+
+	if (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) {
+		if (!irdma_cqp_stats_inst_cmd(vsi, IRDMA_OP_STATS_ALLOCATE,
+					      &stats_info))
+			return stats_info.stats_idx;
+	}
+
+	start_idx = 1;
+	max_stats = 16;
+	for (i = start_idx; i < max_stats; i++)
+		if (!dev->fcn_id_array[i]) {
+			fcn_id = i;
+			dev->fcn_id_array[i] = true;
+			break;
+		}
+
+	return fcn_id;
+}
+
+/**
+ * irdma_vsi_stats_init - Initialize the vsi statistics
+ * @vsi: pointer to the vsi structure
+ * @info: The info structure used for initialization
+ */
+enum irdma_status_code irdma_vsi_stats_init(struct irdma_sc_vsi *vsi,
+					    struct irdma_vsi_stats_info *info)
+{
+	u8 fcn_id = info->fcn_id;
+	struct irdma_dma_mem *stats_buff_mem;
+
+	vsi->pestat = info->pestat;
+	vsi->pestat->hw = vsi->dev->hw;
+	vsi->pestat->vsi = vsi;
+	stats_buff_mem = &vsi->pestat->gather_info.stats_buff_mem;
+	stats_buff_mem->size = ALIGN(IRDMA_GATHER_STATS_BUF_SIZE * 2, 1);
+	stats_buff_mem->va = dma_alloc_coherent(hw_to_dev(vsi->pestat->hw),
+						stats_buff_mem->size,
+						&stats_buff_mem->pa,
+						GFP_KERNEL);
+	if (!stats_buff_mem->va)
+		return IRDMA_ERR_NO_MEMORY;
+
+	vsi->pestat->gather_info.gather_stats = stats_buff_mem->va;
+	vsi->pestat->gather_info.last_gather_stats =
+		(void *)((uintptr_t)stats_buff_mem->va +
+			 IRDMA_GATHER_STATS_BUF_SIZE);
+
+	if (vsi->dev->privileged)
+		irdma_hw_stats_start_timer(vsi);
+
+	if (info->alloc_fcn_id)
+		fcn_id = irdma_get_fcn_id(vsi);
+	if (fcn_id == IRDMA_INVALID_FCN_ID)
+		goto stats_error;
+
+	vsi->stats_fcn_id_alloc = info->alloc_fcn_id;
+	vsi->fcn_id = fcn_id;
+	if (info->alloc_fcn_id) {
+		vsi->pestat->gather_info.use_stats_inst = true;
+		vsi->pestat->gather_info.stats_inst_index = fcn_id;
+	}
+
+	return 0;
+
+stats_error:
+	dma_free_coherent(hw_to_dev(vsi->pestat->hw), stats_buff_mem->size,
+			  stats_buff_mem->va, stats_buff_mem->pa);
+	stats_buff_mem->va = NULL;
+
+	return IRDMA_ERR_CQP_COMPL_ERROR;
+}
+
+/**
+ * irdma_vsi_stats_free - Free the vsi stats
+ * @vsi: pointer to the vsi structure
+ */
+void irdma_vsi_stats_free(struct irdma_sc_vsi *vsi)
+{
+	struct irdma_stats_inst_info stats_info = {};
+	u8 fcn_id = vsi->fcn_id;
+	struct irdma_sc_dev *dev = vsi->dev;
+
+	if (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) {
+		if (vsi->stats_fcn_id_alloc) {
+			stats_info.stats_idx = vsi->fcn_id;
+			irdma_cqp_stats_inst_cmd(vsi, IRDMA_OP_STATS_FREE,
+						 &stats_info);
+		}
+	} else {
+		if (vsi->stats_fcn_id_alloc &&
+		    fcn_id < vsi->dev->hw_attrs.max_stat_inst)
+			vsi->dev->fcn_id_array[fcn_id] = false;
+	}
+
+	if (!vsi->pestat)
+		return;
+	if (vsi->dev->privileged)
+		irdma_hw_stats_stop_timer(vsi);
+	dma_free_coherent(hw_to_dev(vsi->pestat->hw),
+			  vsi->pestat->gather_info.stats_buff_mem.size,
+			  vsi->pestat->gather_info.stats_buff_mem.va,
+			  vsi->pestat->gather_info.stats_buff_mem.pa);
+	vsi->pestat->gather_info.stats_buff_mem.va = NULL;
+}
+
+/**
+ * irdma_get_encoded_wqe_size - given wq size, returns hardware encoded size
+ * @wqsize: size of the wq (sq, rq) to encoded_size
+ * @cqpsq: encoded size for sq for cqp as its encoded size is 1+ other wq's
+ */
+u8 irdma_get_encoded_wqe_size(u32 wqsize, bool cqpsq)
+{
+	u8 encoded_size = 0;
+
+	/* cqp sq's hw coded value starts from 1 for size of 4
+	 * while it starts from 0 for qp' wq's.
+	 */
+	if (cqpsq)
+		encoded_size = 1;
+	wqsize >>= 2;
+	while (wqsize >>= 1)
+		encoded_size++;
+
+	return encoded_size;
+}
+
+/**
+ * irdma_sc_gather_stats - collect the statistics
+ * @cqp: struct for cqp hw
+ * @info: gather stats info structure
+ * @scratch: u64 saved to be used during cqp completion
+ */
+static enum irdma_status_code
+irdma_sc_gather_stats(struct irdma_sc_cqp *cqp,
+		      struct irdma_stats_gather_info *info, u64 scratch)
+{
+	__le64 *wqe;
+	u64 temp;
+
+	if (info->stats_buff_mem.size < IRDMA_GATHER_STATS_BUF_SIZE)
+		return IRDMA_ERR_BUF_TOO_SHORT;
+
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	set_64bit_val(wqe, 40,
+		      LS_64(info->hmc_fcn_index, IRDMA_CQPSQ_STATS_HMC_FCN_INDEX));
+	set_64bit_val(wqe, 32, info->stats_buff_mem.pa);
+
+	temp = LS_64(cqp->polarity, IRDMA_CQPSQ_STATS_WQEVALID) |
+	       LS_64(info->use_stats_inst, IRDMA_CQPSQ_STATS_USE_INST) |
+	       LS_64(info->stats_inst_index, IRDMA_CQPSQ_STATS_INST_INDEX) |
+	       LS_64(info->use_hmc_fcn_index,
+		     IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX) |
+	       LS_64(IRDMA_CQP_OP_GATHER_STATS, IRDMA_CQPSQ_STATS_OP);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, temp);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_STATS, "GATHER_STATS WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	wr32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CQPDB],
+	     IRDMA_RING_CURRENT_HEAD(cqp->sq_ring));
+	dev_dbg(rfdev_to_dev(cqp->dev),
+		"STATS: CQP SQ head 0x%x tail 0x%x size 0x%x\n",
+		cqp->sq_ring.head, cqp->sq_ring.tail, cqp->sq_ring.size);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_manage_stats_inst - allocate or free stats instance
+ * @cqp: struct for cqp hw
+ * @info: stats info structure
+ * @alloc: alloc vs. delete flag
+ * @scratch: u64 saved to be used during cqp completion
+ */
+static enum irdma_status_code
+irdma_sc_manage_stats_inst(struct irdma_sc_cqp *cqp,
+			   struct irdma_stats_inst_info *info, bool alloc,
+			   u64 scratch)
+{
+	__le64 *wqe;
+	u64 temp;
+
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	set_64bit_val(wqe, 40,
+		      LS_64(info->hmc_fn_id, IRDMA_CQPSQ_STATS_HMC_FCN_INDEX));
+	temp = LS_64(cqp->polarity, IRDMA_CQPSQ_STATS_WQEVALID) |
+	       LS_64(alloc, IRDMA_CQPSQ_STATS_ALLOC_INST) |
+	       LS_64(info->use_hmc_fcn_index, IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX) |
+	       LS_64(info->stats_idx, IRDMA_CQPSQ_STATS_INST_INDEX) |
+	       LS_64(IRDMA_CQP_OP_MANAGE_STATS, IRDMA_CQPSQ_STATS_OP);
+
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, temp);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_STATS WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+
+	irdma_sc_cqp_post_sq(cqp);
+	return 0;
+}
+
+/**
+ * irdma_sc_set_up_mapping - set the up map table
+ * @cqp: struct for cqp hw
+ * @info: User priority map info
+ * @scratch: u64 saved to be used during cqp completion
+ */
+static enum irdma_status_code irdma_sc_set_up_map(struct irdma_sc_cqp *cqp,
+						  struct irdma_up_info *info,
+						  u64 scratch)
+{
+	__le64 *wqe;
+	u64 temp;
+
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	temp = info->map[0] | LS_64_1(info->map[1], 8) |
+	       LS_64_1(info->map[2], 16) | LS_64_1(info->map[3], 24) |
+	       LS_64_1(info->map[4], 32) | LS_64_1(info->map[5], 40) |
+	       LS_64_1(info->map[6], 48) | LS_64_1(info->map[7], 56);
+
+	set_64bit_val(wqe, 0, temp);
+	set_64bit_val(wqe, 40,
+		      LS_64(info->cnp_up_override, IRDMA_CQPSQ_UP_CNPOVERRIDE) |
+		      LS_64(info->hmc_fcn_idx, IRDMA_CQPSQ_UP_HMCFCNIDX));
+
+	temp = LS_64(cqp->polarity, IRDMA_CQPSQ_UP_WQEVALID) |
+	       LS_64(info->use_vlan, IRDMA_CQPSQ_UP_USEVLAN) |
+	       LS_64(info->use_cnp_up_override, IRDMA_CQPSQ_UP_USEOVERRIDE) |
+	       LS_64(IRDMA_CQP_OP_UP_MAP, IRDMA_CQPSQ_UP_OP);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, temp);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "UPMAP WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_manage_ws_node - create/modify/destroy WS node
+ * @cqp: struct for cqp hw
+ * @info: node info structure
+ * @node_op: 0 for add 1 for modify, 2 for delete
+ * @scratch: u64 saved to be used during cqp completion
+ */
+static enum irdma_status_code
+irdma_sc_manage_ws_node(struct irdma_sc_cqp *cqp,
+			struct irdma_ws_node_info *info,
+			enum irdma_ws_node_op node_op, u64 scratch)
+{
+	__le64 *wqe;
+	u64 temp = 0;
+
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	set_64bit_val(wqe, 32,
+		      LS_64(info->vsi, IRDMA_CQPSQ_WS_VSI) |
+		      LS_64(info->weight, IRDMA_CQPSQ_WS_WEIGHT));
+
+	temp = LS_64(cqp->polarity, IRDMA_CQPSQ_WS_WQEVALID) |
+	       LS_64(node_op, IRDMA_CQPSQ_WS_NODEOP) |
+	       LS_64(info->enable, IRDMA_CQPSQ_WS_ENABLENODE) |
+	       LS_64(info->type_leaf, IRDMA_CQPSQ_WS_NODETYPE) |
+	       LS_64(info->prio_type, IRDMA_CQPSQ_WS_PRIOTYPE) |
+	       LS_64(info->tc, IRDMA_CQPSQ_WS_TC) |
+	       LS_64(IRDMA_CQP_OP_WORK_SCHED_NODE, IRDMA_CQPSQ_WS_OP) |
+	       LS_64(info->parent_id, IRDMA_CQPSQ_WS_PARENTID) |
+	       LS_64(info->id, IRDMA_CQPSQ_WS_NODEID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, temp);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_WS WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_qp_flush_wqes - flush qp's wqe
+ * @qp: sc qp
+ * @info: dlush information
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp, struct irdma_qp_flush_info *info,
+		       u64 scratch, bool post_sq)
+{
+	u64 temp = 0;
+	__le64 *wqe;
+	struct irdma_sc_cqp *cqp;
+	u64 hdr;
+	bool flush_sq = false, flush_rq = false;
+
+	if (info->rq && !qp->flush_rq)
+		flush_rq = true;
+	if (info->sq && !qp->flush_sq)
+		flush_sq = true;
+	qp->flush_sq |= flush_sq;
+	qp->flush_rq |= flush_rq;
+
+	if (!flush_sq && !flush_rq) {
+		dev_dbg(rfdev_to_dev(qp->dev),
+			"CQP: Additional flush request ignored\n");
+		return 0;
+	}
+
+	cqp = qp->pd->dev->cqp;
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	if (info->userflushcode) {
+		if (flush_rq)
+			temp |= LS_64(info->rq_minor_code, IRDMA_CQPSQ_FWQE_RQMNERR) |
+				LS_64(info->rq_major_code, IRDMA_CQPSQ_FWQE_RQMJERR);
+		if (flush_sq)
+			temp |= LS_64(info->sq_minor_code, IRDMA_CQPSQ_FWQE_SQMNERR) |
+				LS_64(info->sq_major_code, IRDMA_CQPSQ_FWQE_SQMJERR);
+	}
+	set_64bit_val(wqe, 16, temp);
+
+	temp = (info->generate_ae) ?
+		info->ae_code | LS_64(info->ae_src, IRDMA_CQPSQ_FWQE_AESOURCE) : 0;
+	set_64bit_val(wqe, 8, temp);
+
+	hdr = qp->qp_uk.qp_id |
+	      LS_64(IRDMA_CQP_OP_FLUSH_WQES, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(info->generate_ae, IRDMA_CQPSQ_FWQE_GENERATE_AE) |
+	      LS_64(info->userflushcode, IRDMA_CQPSQ_FWQE_USERFLCODE) |
+	      LS_64(flush_sq, IRDMA_CQPSQ_FWQE_FLUSHSQ) |
+	      LS_64(flush_rq, IRDMA_CQPSQ_FWQE_FLUSHRQ) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "QP_FLUSH WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	if (post_sq)
+		irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_gen_ae - generate AE, uses flush WQE CQP OP
+ * @qp: sc qp
+ * @info: gen ae information
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code irdma_sc_gen_ae(struct irdma_sc_qp *qp,
+					      struct irdma_gen_ae_info *info,
+					      u64 scratch, bool post_sq)
+{
+	u64 temp;
+	__le64 *wqe;
+	struct irdma_sc_cqp *cqp;
+	u64 hdr;
+
+	cqp = qp->pd->dev->cqp;
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	temp = info->ae_code | LS_64(info->ae_src, IRDMA_CQPSQ_FWQE_AESOURCE);
+	set_64bit_val(wqe, 8, temp);
+
+	hdr = qp->qp_uk.qp_id | LS_64(IRDMA_CQP_OP_GEN_AE, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(1, IRDMA_CQPSQ_FWQE_GENERATE_AE) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "GEN_AE WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	if (post_sq)
+		irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/*** irdma_sc_qp_upload_context - upload qp's context
+ * @dev: sc device struct
+ * @info: upload context info ptr for return
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_qp_upload_context(struct irdma_sc_dev *dev,
+			   struct irdma_upload_context_info *info, u64 scratch,
+			   bool post_sq)
+{
+	__le64 *wqe;
+	struct irdma_sc_cqp *cqp;
+	u64 hdr;
+
+	cqp = dev->cqp;
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	set_64bit_val(wqe, 16, info->buf_pa);
+
+	hdr = LS_64(info->qp_id, IRDMA_CQPSQ_UCTX_QPID) |
+	      LS_64(IRDMA_CQP_OP_UPLOAD_CONTEXT, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(info->qp_type, IRDMA_CQPSQ_UCTX_QPTYPE) |
+	      LS_64(info->raw_format, IRDMA_CQPSQ_UCTX_RAWFORMAT) |
+	      LS_64(info->freeze_qp, IRDMA_CQPSQ_UCTX_FREEZEQP) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "QP_UPLOAD_CTX WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	if (post_sq)
+		irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_manage_push_page - Handle push page
+ * @cqp: struct for cqp hw
+ * @info: push page info
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code
+irdma_sc_manage_push_page(struct irdma_sc_cqp *cqp,
+			  struct irdma_cqp_manage_push_page_info *info,
+			  u64 scratch, bool post_sq)
+{
+	__le64 *wqe;
+	u64 hdr;
+
+	if (info->push_idx >= cqp->dev->hw_attrs.max_hw_device_pages)
+		return IRDMA_ERR_INVALID_PUSH_PAGE_INDEX;
+
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	set_64bit_val(wqe, 16, info->qs_handle);
+	hdr = LS_64(info->push_idx, IRDMA_CQPSQ_MPP_PPIDX) |
+	      LS_64(info->push_page_type, IRDMA_CQPSQ_MPP_PPTYPE) |
+	      LS_64(IRDMA_CQP_OP_MANAGE_PUSH_PAGES, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID) |
+	      LS_64(info->free_page, IRDMA_CQPSQ_MPP_FREE_PAGE);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "MANAGE_PUSH_PAGES WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	if (post_sq)
+		irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_suspend_qp - suspend qp for param change
+ * @cqp: struct for cqp hw
+ * @qp: sc qp struct
+ * @scratch: u64 saved to be used during cqp completion
+ */
+static enum irdma_status_code irdma_sc_suspend_qp(struct irdma_sc_cqp *cqp,
+						  struct irdma_sc_qp *qp,
+						  u64 scratch)
+{
+	u64 hdr;
+	__le64 *wqe;
+
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	hdr = LS_64(qp->qp_uk.qp_id, IRDMA_CQPSQ_SUSPENDQP_QPID) |
+	      LS_64(IRDMA_CQP_OP_SUSPEND_QP, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "SUSPEND_QP WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_resume_qp - resume qp after suspend
+ * @cqp: struct for cqp hw
+ * @qp: sc qp struct
+ * @scratch: u64 saved to be used during cqp completion
+ */
+static enum irdma_status_code irdma_sc_resume_qp(struct irdma_sc_cqp *cqp,
+						 struct irdma_sc_qp *qp,
+						 u64 scratch)
+{
+	u64 hdr;
+	__le64 *wqe;
+
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	set_64bit_val(wqe, 16,
+		      LS_64(qp->qs_handle, IRDMA_CQPSQ_RESUMEQP_QSHANDLE));
+
+	hdr = LS_64(qp->qp_uk.qp_id, IRDMA_CQPSQ_RESUMEQP_QPID) |
+	      LS_64(IRDMA_CQP_OP_RESUME_QP, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "RESUME_QP WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_cq_ack - acknowledge completion q
+ * @cq: cq struct
+ */
+static void irdma_sc_cq_ack(struct irdma_sc_cq *cq)
+{
+	writel(cq->cq_uk.cq_id, cq->cq_uk.cq_ack_db);
+}
+
+/**
+ * irdma_sc_cq_init - initialize completion q
+ * @cq: cq struct
+ * @info: cq initialization info
+ */
+static enum irdma_status_code irdma_sc_cq_init(struct irdma_sc_cq *cq,
+					       struct irdma_cq_init_info *info)
+{
+	enum irdma_status_code ret_code;
+	u32 pble_obj_cnt;
+
+	if (info->cq_uk_init_info.cq_size < info->dev->hw_attrs.uk_attrs.min_hw_cq_size ||
+	    info->cq_uk_init_info.cq_size > info->dev->hw_attrs.uk_attrs.max_hw_cq_size)
+		return IRDMA_ERR_INVALID_SIZE;
+
+	pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
+	if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)
+		return IRDMA_ERR_INVALID_PBLE_INDEX;
+
+	cq->cq_pa = info->cq_base_pa;
+	cq->dev = info->dev;
+	cq->ceq_id = info->ceq_id;
+	info->cq_uk_init_info.cqe_alloc_db = cq->dev->cq_arm_db;
+	info->cq_uk_init_info.cq_ack_db = cq->dev->cq_ack_db;
+	ret_code = irdma_cq_uk_init(&cq->cq_uk, &info->cq_uk_init_info);
+	if (ret_code)
+		return ret_code;
+
+	cq->virtual_map = info->virtual_map;
+	cq->pbl_chunk_size = info->pbl_chunk_size;
+	cq->ceqe_mask = info->ceqe_mask;
+	cq->cq_type = (info->type) ? info->type : IRDMA_CQ_TYPE_IWARP;
+	cq->shadow_area_pa = info->shadow_area_pa;
+	cq->shadow_read_threshold = info->shadow_read_threshold;
+	cq->ceq_id_valid = info->ceq_id_valid;
+	cq->tph_en = info->tph_en;
+	cq->tph_val = info->tph_val;
+	cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
+	cq->vsi = info->vsi;
+
+	return 0;
+}
+
+/**
+ * irdma_sc_cq_create - create completion q
+ * @cq: cq struct
+ * @scratch: u64 saved to be used during cqp completion
+ * @check_overflow: flag for overflow check
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code irdma_sc_cq_create(struct irdma_sc_cq *cq,
+						 u64 scratch,
+						 bool check_overflow,
+						 bool post_sq)
+{
+	__le64 *wqe;
+	struct irdma_sc_cqp *cqp;
+	u64 hdr;
+	struct irdma_sc_ceq *ceq;
+	enum irdma_status_code ret_code = 0;
+
+	cqp = cq->dev->cqp;
+	if (cq->cq_uk.cq_id > (cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt - 1))
+		return IRDMA_ERR_INVALID_CQ_ID;
+
+	if (cq->ceq_id > (cq->dev->hmc_fpm_misc.max_ceqs - 1))
+		return IRDMA_ERR_INVALID_CEQ_ID;
+
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	ceq = cq->dev->ceq[cq->ceq_id];
+	if (ceq && ceq->reg_cq)
+		ret_code = irdma_sc_add_cq_ctx(ceq, cq);
+
+	if (ret_code)
+		return ret_code;
+
+	set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
+	set_64bit_val(wqe, 8, RS_64_1(cq, 1));
+	set_64bit_val(wqe, 16,
+		      LS_64(cq->shadow_read_threshold,
+			    IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
+	set_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa));
+	set_64bit_val(wqe, 40, cq->shadow_area_pa);
+	set_64bit_val(wqe, 48,
+		      LS_64((cq->virtual_map ? cq->first_pm_pbl_idx : 0),
+			    IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX));
+	set_64bit_val(wqe, 56,
+		      LS_64(cq->tph_val, IRDMA_CQPSQ_TPHVAL) |
+		      LS_64(cq->vsi->vsi_idx, IRDMA_CQPSQ_VSIIDX));
+
+	hdr = FLD_LS_64(cq->dev, cq->cq_uk.cq_id, IRDMA_CQPSQ_CQ_CQID) |
+	      FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0),
+			IRDMA_CQPSQ_CQ_CEQID) |
+	      LS_64(IRDMA_CQP_OP_CREATE_CQ, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(cq->pbl_chunk_size, IRDMA_CQPSQ_CQ_LPBLSIZE) |
+	      LS_64(check_overflow, IRDMA_CQPSQ_CQ_CHKOVERFLOW) |
+	      LS_64(cq->virtual_map, IRDMA_CQPSQ_CQ_VIRTMAP) |
+	      LS_64(cq->ceqe_mask, IRDMA_CQPSQ_CQ_ENCEQEMASK) |
+	      LS_64(cq->ceq_id_valid, IRDMA_CQPSQ_CQ_CEQIDVALID) |
+	      LS_64(cq->tph_en, IRDMA_CQPSQ_TPHEN) |
+	      LS_64(cq->cq_uk.avoid_mem_cflct, IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CQ_CREATE WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	if (post_sq)
+		irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_cq_destroy - destroy completion q
+ * @cq: cq struct
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag for cqp db to ring
+ */
+static enum irdma_status_code irdma_sc_cq_destroy(struct irdma_sc_cq *cq,
+						  u64 scratch, bool post_sq)
+{
+	struct irdma_sc_cqp *cqp;
+	__le64 *wqe;
+	u64 hdr;
+	struct irdma_sc_ceq *ceq;
+
+	cqp = cq->dev->cqp;
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	ceq = cq->dev->ceq[cq->ceq_id];
+	if (ceq && ceq->reg_cq)
+		irdma_sc_remove_cq_ctx(ceq, cq);
+
+	set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
+	set_64bit_val(wqe, 8, RS_64_1(cq, 1));
+	set_64bit_val(wqe, 40, cq->shadow_area_pa);
+	set_64bit_val(wqe, 48,
+		      (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
+
+	hdr = cq->cq_uk.cq_id |
+	      FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0),
+			IRDMA_CQPSQ_CQ_CEQID) |
+	      LS_64(IRDMA_CQP_OP_DESTROY_CQ, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(cq->pbl_chunk_size, IRDMA_CQPSQ_CQ_LPBLSIZE) |
+	      LS_64(cq->virtual_map, IRDMA_CQPSQ_CQ_VIRTMAP) |
+	      LS_64(cq->ceqe_mask, IRDMA_CQPSQ_CQ_ENCEQEMASK) |
+	      LS_64(cq->ceq_id_valid, IRDMA_CQPSQ_CQ_CEQIDVALID) |
+	      LS_64(cq->tph_en, IRDMA_CQPSQ_TPHEN) |
+	      LS_64(cq->cq_uk.avoid_mem_cflct, IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CQ_DESTROY WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	if (post_sq)
+		irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_sc_cq_resize - set resized cq buffer info
+ * @cq: resized cq
+ * @info: resized cq buffer info
+ */
+static void irdma_sc_cq_resize(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info)
+{
+	cq->virtual_map = info->virtual_map;
+	cq->cq_pa = info->cq_pa;
+	cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
+	cq->pbl_chunk_size = info->pbl_chunk_size;
+	cq->cq_uk.ops.iw_cq_resize(&cq->cq_uk, info->cq_base, info->cq_size);
+}
+
+/**
+ * irdma_sc_cq_modify - modify a Completion Queue
+ * @cq: cq struct
+ * @info: modification info struct
+ * @scratch: u64 saved to be used during cqp completion
+ * @post_sq: flag to post to sq
+ */
+static enum irdma_status_code
+irdma_sc_cq_modify(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info,
+		   u64 scratch, bool post_sq)
+{
+	struct irdma_sc_cqp *cqp;
+	__le64 *wqe;
+	u64 hdr;
+	u32 pble_obj_cnt;
+
+	if (info->ceq_valid &&
+	    info->ceq_id > (cq->dev->hmc_fpm_misc.max_ceqs - 1))
+		return IRDMA_ERR_INVALID_CEQ_ID;
+
+	pble_obj_cnt = cq->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
+	if (info->cq_resize && info->virtual_map &&
+	    info->first_pm_pbl_idx >= pble_obj_cnt)
+		return IRDMA_ERR_INVALID_PBLE_INDEX;
+
+	cqp = cq->dev->cqp;
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	set_64bit_val(wqe, 0, info->cq_size);
+	set_64bit_val(wqe, 8, RS_64_1(cq, 1));
+	set_64bit_val(wqe, 16,
+		      LS_64(info->shadow_read_threshold,
+			    IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
+	set_64bit_val(wqe, 32, info->cq_pa);
+	set_64bit_val(wqe, 40, cq->shadow_area_pa);
+	set_64bit_val(wqe, 48, info->first_pm_pbl_idx);
+	set_64bit_val(wqe, 56,
+		      LS_64(cq->tph_val, IRDMA_CQPSQ_TPHVAL) |
+		      LS_64(cq->vsi->vsi_idx, IRDMA_CQPSQ_VSIIDX));
+
+	hdr = cq->cq_uk.cq_id |
+	      FLD_LS_64(cq->dev, (info->ceq_valid ? cq->ceq_id : 0),
+			IRDMA_CQPSQ_CQ_CEQID) |
+	      LS_64(IRDMA_CQP_OP_MODIFY_CQ, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(info->cq_resize, IRDMA_CQPSQ_CQ_CQRESIZE) |
+	      LS_64(info->pbl_chunk_size, IRDMA_CQPSQ_CQ_LPBLSIZE) |
+	      LS_64(info->check_overflow, IRDMA_CQPSQ_CQ_CHKOVERFLOW) |
+	      LS_64(info->virtual_map, IRDMA_CQPSQ_CQ_VIRTMAP) |
+	      LS_64(cq->ceqe_mask, IRDMA_CQPSQ_CQ_ENCEQEMASK) |
+	      LS_64(info->ceq_valid, IRDMA_CQPSQ_CQ_CEQIDVALID) |
+	      LS_64(cq->tph_en, IRDMA_CQPSQ_TPHEN) |
+	      LS_64(cq->cq_uk.avoid_mem_cflct, IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_WQE, "CQ_MODIFY WQE", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	if (post_sq)
+		irdma_sc_cqp_post_sq(cqp);
+
+	return 0;
+}
+
+/**
+ * irdma_check_cqp_progress - check cqp processing progress
+ * @timeout: timeout info struct
+ * @dev: sc device struct
+ */
+static void irdma_check_cqp_progress(struct irdma_cqp_timeout *timeout,
+				     struct irdma_sc_dev *dev)
+{
+	if (timeout->compl_cqp_cmds != dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]) {
+		timeout->compl_cqp_cmds = dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS];
+		timeout->count = 0;
+	} else {
+		if (dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS] !=
+		    timeout->compl_cqp_cmds)
+			timeout->count++;
+	}
+}
+
+/**
+ * irdma_get_cqp_reg_info - get head and tail for cqp using registers
+ * @cqp: struct for cqp hw
+ * @val: cqp tail register value
+ * @tail: wqtail register value
+ * @error: cqp processing err
+ */
+static inline void irdma_get_cqp_reg_info(struct irdma_sc_cqp *cqp, u32 *val,
+					  u32 *tail, u32 *error)
+{
+	*val = rd32(cqp->dev->hw, cqp->dev->hw_regs[IRDMA_CQPTAIL]);
+	*tail = RS_32(*val, IRDMA_CQPTAIL_WQTAIL);
+	*error = RS_32(*val, IRDMA_CQPTAIL_CQP_OP_ERR);
+}
+
+/**
+ * irdma_cqp_poll_registers - poll cqp registers
+ * @cqp: struct for cqp hw
+ * @tail: wqtail register value
+ * @count: how many times to try for completion
+ */
+static enum irdma_status_code irdma_cqp_poll_registers(struct irdma_sc_cqp *cqp,
+						       u32 tail, u32 count)
+{
+	u32 i = 0;
+	u32 newtail, error, val;
+
+	while (i++ < count) {
+		irdma_get_cqp_reg_info(cqp, &val, &newtail, &error);
+		if (error) {
+			error = rd32(cqp->dev->hw,
+				     cqp->dev->hw_regs[IRDMA_CQPERRCODES]);
+			dev_dbg(rfdev_to_dev(cqp->dev),
+				"CQP: CQPERRCODES error_code[x%08X]\n", error);
+			return IRDMA_ERR_CQP_COMPL_ERROR;
+		}
+		if (newtail != tail) {
+			/* SUCCESS */
+			IRDMA_RING_MOVE_TAIL(cqp->sq_ring);
+			cqp->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]++;
+			return 0;
+		}
+		udelay(cqp->dev->hw_attrs.max_sleep_count);
+	}
+
+	return IRDMA_ERR_TIMEOUT;
+}
+
+/**
+ * irdma_sc_decode_fpm_commit - decode a 64 bit value into count and base
+ * @buf: pointer to commit buffer
+ * @buf_idx: buffer index
+ * @obj_info: object info pointer
+ * @rsrc_idx: indexs of memory resource
+ */
+static u64 irdma_sc_decode_fpm_commit(__le64 *buf, u32 buf_idx,
+				      struct irdma_hmc_obj_info *obj_info,
+				      u32 rsrc_idx)
+{
+	u64 temp;
+
+	get_64bit_val(buf, buf_idx, &temp);
+
+	switch (rsrc_idx) {
+	case IRDMA_HMC_IW_QP:
+		obj_info[rsrc_idx].cnt = (u32)RS_64(temp, IRDMA_COMMIT_FPM_QPCNT);
+		break;
+	case IRDMA_HMC_IW_CQ:
+		obj_info[rsrc_idx].cnt = (u32)RS_64(temp, IRDMA_COMMIT_FPM_CQCNT);
+		break;
+	case IRDMA_HMC_IW_APBVT_ENTRY:
+		obj_info[rsrc_idx].cnt = 1;
[… more lines in the raw message]

Lines of this file are not shown. View the raw message.

drivers/infiniband/hw/irdma/defs.h new file +2129 -0

2,129 lines not on this page · raw message

drivers/infiniband/hw/irdma/irdma.h new file +190 -0

190 lines not on this page · raw message

drivers/infiniband/hw/irdma/type.h new file +1709 -0

1,709 lines not on this page · raw message

The rest of this diff is not shown. View the raw message.

Signature · 1 line
2.23.0

Part of the diff is not shown here. View the raw message.

#34 lore raw

PATCH v3 8/20 RDMA/irdma: Add HMC backing store setup functions

In reply to Jeff Kirsher

From: Mustafa Ismail <[email protected]>

HW uses host memory as a backing store for a number of protocol context objects and queue state tracking. The Host Memory Cache (HMC) is a component responsible for managing these objects stored in host memory.

Add the functions and data structures to manage the allocation of backing pages used by the HMC for the various objects

---
 drivers/infiniband/hw/irdma/hmc.c | 705 ++++++++++++++++++++++++++++++
 drivers/infiniband/hw/irdma/hmc.h | 218 +++++++++
 2 files changed, 923 insertions(+)
 create mode 100644 drivers/infiniband/hw/irdma/hmc.c
 create mode 100644 drivers/infiniband/hw/irdma/hmc.h
drivers/infiniband/hw/irdma/hmc.c new file +705 -0
new file mode 100644
index 000000000000..a5928030947c
@@ -0,0 +1,705 @@
+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
+/* Copyright (c) 2015 - 2019 Intel Corporation */
+#include "osdep.h"
+#include "status.h"
+#include "hmc.h"
+#include "defs.h"
+#include "type.h"
+#include "protos.h"
+
+/**
+ * irdma_find_sd_index_limit - finds segment descriptor index limit
+ * @hmc_info: pointer to the HMC configuration information structure
+ * @type: type of HMC resources we're searching
+ * @idx: starting index for the object
+ * @cnt: number of objects we're trying to create
+ * @sd_idx: pointer to return index of the segment descriptor in question
+ * @sd_limit: pointer to return the maximum number of segment descriptors
+ *
+ * This function calculates the segment descriptor index and index limit
+ * for the resource defined by irdma_hmc_rsrc_type.
+ */
+
+static void irdma_find_sd_index_limit(struct irdma_hmc_info *hmc_info, u32 type,
+				      u32 idx, u32 cnt, u32 *sd_idx,
+				      u32 *sd_limit)
+{
+	u64 fpm_addr, fpm_limit;
+
+	fpm_addr = hmc_info->hmc_obj[(type)].base +
+		   hmc_info->hmc_obj[type].size * idx;
+	fpm_limit = fpm_addr + hmc_info->hmc_obj[type].size * cnt;
+	*sd_idx = (u32)(fpm_addr / IRDMA_HMC_DIRECT_BP_SIZE);
+	*sd_limit = (u32)((fpm_limit - 1) / IRDMA_HMC_DIRECT_BP_SIZE);
+	*sd_limit += 1;
+}
+
+/**
+ * irdma_find_pd_index_limit - finds page descriptor index limit
+ * @hmc_info: pointer to the HMC configuration information struct
+ * @type: HMC resource type we're examining
+ * @idx: starting index for the object
+ * @cnt: number of objects we're trying to create
+ * @pd_idx: pointer to return page descriptor index
+ * @pd_limit: pointer to return page descriptor index limit
+ *
+ * Calculates the page descriptor index and index limit for the resource
+ * defined by irdma_hmc_rsrc_type.
+ */
+
+static void irdma_find_pd_index_limit(struct irdma_hmc_info *hmc_info, u32 type,
+				      u32 idx, u32 cnt, u32 *pd_idx,
+				      u32 *pd_limit)
+{
+	u64 fpm_adr, fpm_limit;
+
+	fpm_adr = hmc_info->hmc_obj[type].base +
+		  hmc_info->hmc_obj[type].size * idx;
+	fpm_limit = fpm_adr + (hmc_info)->hmc_obj[(type)].size * (cnt);
+	*pd_idx = (u32)(fpm_adr / IRDMA_HMC_PAGED_BP_SIZE);
+	*pd_limit = (u32)((fpm_limit - 1) / IRDMA_HMC_PAGED_BP_SIZE);
+	*pd_limit += 1;
+}
+
+/**
+ * irdma_set_sd_entry - setup entry for sd programming
+ * @pa: physical addr
+ * @idx: sd index
+ * @type: paged or direct sd
+ * @entry: sd entry ptr
+ */
+static void irdma_set_sd_entry(u64 pa, u32 idx, enum irdma_sd_entry_type type,
+			       struct irdma_update_sd_entry *entry)
+{
+	entry->data = pa | (IRDMA_HMC_MAX_BP_COUNT << IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT_S) |
+		      (((type == IRDMA_SD_TYPE_PAGED) ? 0 : 1) << IRDMA_PFHMC_SDDATALOW_PMSDTYPE_S) |
+		      (1 << IRDMA_PFHMC_SDDATALOW_PMSDVALID_S);
+	entry->cmd = (idx | (1 << IRDMA_PFHMC_SDCMD_PMSDWR_S) | (1 << 15));
+}
+
+/**
+ * irdma_clr_sd_entry - setup entry for sd clear
+ * @idx: sd index
+ * @type: paged or direct sd
+ * @entry: sd entry ptr
+ */
+static void irdma_clr_sd_entry(u32 idx, enum irdma_sd_entry_type type,
+			       struct irdma_update_sd_entry *entry)
+{
+	entry->data = (IRDMA_HMC_MAX_BP_COUNT << IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT_S) |
+		      (((type == IRDMA_SD_TYPE_PAGED) ? 0 : 1) << IRDMA_PFHMC_SDDATALOW_PMSDTYPE_S);
+	entry->cmd = (idx | (1 << IRDMA_PFHMC_SDCMD_PMSDWR_S) | (1 << 15));
+}
+
+/**
+ * irdma_hmc_sd_one - setup 1 sd entry for cqp
+ * @dev: pointer to the device structure
+ * @hmc_fn_id: hmc's function id
+ * @pa: physical addr
+ * @sd_idx: sd index
+ * @type: paged or direct sd
+ * @setsd: flag to set or clear sd
+ */
+enum irdma_status_code irdma_hmc_sd_one(struct irdma_sc_dev *dev, u8 hmc_fn_id,
+					u64 pa, u32 sd_idx,
+					enum irdma_sd_entry_type type,
+					bool setsd)
+{
+	struct irdma_update_sds_info sdinfo;
+
+	sdinfo.cnt = 1;
+	sdinfo.hmc_fn_id = hmc_fn_id;
+	if (setsd)
+		irdma_set_sd_entry(pa, sd_idx, type, sdinfo.entry);
+	else
+		irdma_clr_sd_entry(sd_idx, type, sdinfo.entry);
+	return dev->cqp->process_cqp_sds(dev, &sdinfo);
+}
+
+/**
+ * irdma_hmc_sd_grp - setup group of sd entries for cqp
+ * @dev: pointer to the device structure
+ * @hmc_info: pointer to the HMC configuration information struct
+ * @sd_index: sd index
+ * @sd_cnt: number of sd entries
+ * @setsd: flag to set or clear sd
+ */
+static enum irdma_status_code irdma_hmc_sd_grp(struct irdma_sc_dev *dev,
+					       struct irdma_hmc_info *hmc_info,
+					       u32 sd_index, u32 sd_cnt,
+					       bool setsd)
+{
+	struct irdma_hmc_sd_entry *sd_entry;
+	struct irdma_update_sds_info sdinfo = {};
+	u64 pa;
+	u32 i;
+	enum irdma_status_code ret_code = 0;
+
+	sdinfo.hmc_fn_id = hmc_info->hmc_fn_id;
+	for (i = sd_index; i < sd_index + sd_cnt; i++) {
+		sd_entry = &hmc_info->sd_table.sd_entry[i];
+		if (!sd_entry || (!sd_entry->valid && setsd) ||
+		    (sd_entry->valid && !setsd))
+			continue;
+		if (setsd) {
+			pa = (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED) ?
+				     sd_entry->u.pd_table.pd_page_addr.pa :
+				     sd_entry->u.bp.addr.pa;
+			irdma_set_sd_entry(pa, i, sd_entry->entry_type,
+					   &sdinfo.entry[sdinfo.cnt]);
+		} else {
+			irdma_clr_sd_entry(i, sd_entry->entry_type,
+					   &sdinfo.entry[sdinfo.cnt]);
+		}
+		sdinfo.cnt++;
+		if (sdinfo.cnt == IRDMA_MAX_SD_ENTRIES) {
+			ret_code = dev->cqp->process_cqp_sds(dev, &sdinfo);
+			if (ret_code) {
+				dev_dbg(rfdev_to_dev(dev),
+					"HMC: sd_programming failed err=%d\n",
+					ret_code);
+				return ret_code;
+			}
+
+			sdinfo.cnt = 0;
+		}
+	}
+	if (sdinfo.cnt)
+		ret_code = dev->cqp->process_cqp_sds(dev, &sdinfo);
+
+	return ret_code;
+}
+
+/**
+ * irdma_hmc_finish_add_sd_reg - program sd entries for objects
+ * @dev: pointer to the device structure
+ * @info: create obj info
+ */
+static enum irdma_status_code
+irdma_hmc_finish_add_sd_reg(struct irdma_sc_dev *dev,
+			    struct irdma_hmc_create_obj_info *info)
+{
+	if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt)
+		return IRDMA_ERR_INVALID_HMC_OBJ_INDEX;
+
+	if ((info->start_idx + info->count) >
+	    info->hmc_info->hmc_obj[info->rsrc_type].cnt)
+		return IRDMA_ERR_INVALID_HMC_OBJ_COUNT;
+
+	if (!info->add_sd_cnt)
+		return 0;
+	return irdma_hmc_sd_grp(dev, info->hmc_info,
+				info->hmc_info->sd_indexes[0], info->add_sd_cnt,
+				true);
+}
+
+/**
+ * irdma_sc_create_hmc_obj - allocate backing store for hmc objects
+ * @dev: pointer to the device structure
+ * @info: pointer to irdma_hmc_create_obj_info struct
+ *
+ * This will allocate memory for PDs and backing pages and populate
+ * the sd and pd entries.
+ */
+enum irdma_status_code
+irdma_sc_create_hmc_obj(struct irdma_sc_dev *dev,
+			struct irdma_hmc_create_obj_info *info)
+{
+	struct irdma_hmc_sd_entry *sd_entry;
+	u32 sd_idx, sd_lmt;
+	u32 pd_idx = 0, pd_lmt = 0;
+	u32 pd_idx1 = 0, pd_lmt1 = 0;
+	u32 i, j;
+	bool pd_error = false;
+	enum irdma_status_code ret_code = 0;
+
+	if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt)
+		return IRDMA_ERR_INVALID_HMC_OBJ_INDEX;
+
+	if ((info->start_idx + info->count) >
+	    info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
+		dev_dbg(rfdev_to_dev(dev),
+			"HMC: error type %u, start = %u, req cnt %u, cnt = %u\n",
+			info->rsrc_type, info->start_idx, info->count,
+			info->hmc_info->hmc_obj[info->rsrc_type].cnt);
+		return IRDMA_ERR_INVALID_HMC_OBJ_COUNT;
+	}
+
+	irdma_find_sd_index_limit(info->hmc_info, info->rsrc_type,
+				  info->start_idx, info->count, &sd_idx,
+				  &sd_lmt);
+	if (sd_idx >= info->hmc_info->sd_table.sd_cnt ||
+	    sd_lmt > info->hmc_info->sd_table.sd_cnt) {
+		return IRDMA_ERR_INVALID_SD_INDEX;
+	}
+
+	irdma_find_pd_index_limit(info->hmc_info, info->rsrc_type,
+				  info->start_idx, info->count, &pd_idx,
+				  &pd_lmt);
+
+	for (j = sd_idx; j < sd_lmt; j++) {
+		ret_code = irdma_add_sd_table_entry(dev->hw, info->hmc_info, j,
+						    info->entry_type,
+						    IRDMA_HMC_DIRECT_BP_SIZE);
+		if (ret_code)
+			goto exit_sd_error;
+
+		sd_entry = &info->hmc_info->sd_table.sd_entry[j];
+		if (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED &&
+		    (dev->hmc_info == info->hmc_info &&
+		     info->rsrc_type != IRDMA_HMC_IW_PBLE)) {
+			pd_idx1 = max(pd_idx, (j * IRDMA_HMC_MAX_BP_COUNT));
+			pd_lmt1 = min(pd_lmt, (j + 1) * IRDMA_HMC_MAX_BP_COUNT);
+			for (i = pd_idx1; i < pd_lmt1; i++) {
+				/* update the pd table entry */
+				ret_code = irdma_add_pd_table_entry(dev,
+								    info->hmc_info,
+								    i, NULL);
+				if (ret_code) {
+					pd_error = true;
+					break;
+				}
+			}
+			if (pd_error) {
+				while (i && (i > pd_idx1)) {
+					irdma_remove_pd_bp(dev, info->hmc_info,
+							   i - 1);
+					i--;
+				}
+			}
+		}
+		if (sd_entry->valid)
+			continue;
+
+		info->hmc_info->sd_indexes[info->add_sd_cnt] = (u16)j;
+		info->add_sd_cnt++;
+		sd_entry->valid = true;
+	}
+	return irdma_hmc_finish_add_sd_reg(dev, info);
+
+exit_sd_error:
+	while (j && (j > sd_idx)) {
+		sd_entry = &info->hmc_info->sd_table.sd_entry[j - 1];
+		switch (sd_entry->entry_type) {
+		case IRDMA_SD_TYPE_PAGED:
+			pd_idx1 = max(pd_idx, (j - 1) * IRDMA_HMC_MAX_BP_COUNT);
+			pd_lmt1 = min(pd_lmt, (j * IRDMA_HMC_MAX_BP_COUNT));
+			for (i = pd_idx1; i < pd_lmt1; i++)
+				irdma_prep_remove_pd_page(info->hmc_info, i);
+			break;
+		case IRDMA_SD_TYPE_DIRECT:
+			irdma_prep_remove_pd_page(info->hmc_info, (j - 1));
+			break;
+		default:
+			ret_code = IRDMA_ERR_INVALID_SD_TYPE;
+			break;
+		}
+		j--;
+	}
+
+	return ret_code;
+}
+
+/**
+ * irdma_finish_del_sd_reg - delete sd entries for objects
+ * @dev: pointer to the device structure
+ * @info: dele obj info
+ * @reset: true if called before reset
+ */
+static enum irdma_status_code
+irdma_finish_del_sd_reg(struct irdma_sc_dev *dev,
+			struct irdma_hmc_del_obj_info *info, bool reset)
+{
+	struct irdma_hmc_sd_entry *sd_entry;
+	enum irdma_status_code ret_code = 0;
+	u32 i, sd_idx;
+	struct irdma_dma_mem *mem;
+
+	if (dev->privileged && !reset)
+		ret_code = irdma_hmc_sd_grp(dev, info->hmc_info,
+					    info->hmc_info->sd_indexes[0],
+					    info->del_sd_cnt, false);
+
+	if (ret_code)
+		dev_dbg(rfdev_to_dev(dev), "HMC: error cqp sd sd_grp\n");
+	for (i = 0; i < info->del_sd_cnt; i++) {
+		sd_idx = info->hmc_info->sd_indexes[i];
+		sd_entry = &info->hmc_info->sd_table.sd_entry[sd_idx];
+		if (!sd_entry)
+			continue;
+		mem = (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED) ?
+			      &sd_entry->u.pd_table.pd_page_addr :
+			      &sd_entry->u.bp.addr;
+
+		if (!mem || !mem->va) {
+			dev_dbg(rfdev_to_dev(dev), "HMC: error cqp sd mem\n");
+		} else {
+			dma_free_coherent(hw_to_dev(dev->hw), mem->size,
+					  mem->va, mem->pa);
+			mem->va = NULL;
+		}
+	}
+
+	return ret_code;
+}
+
+/**
+ * irdma_sc_del_hmc_obj - remove pe hmc objects
+ * @dev: pointer to the device structure
+ * @info: pointer to irdma_hmc_del_obj_info struct
+ * @reset: true if called before reset
+ *
+ * This will de-populate the SDs and PDs.  It frees
+ * the memory for PDS and backing storage.  After this function is returned,
+ * caller should deallocate memory allocated previously for
+ * book-keeping information about PDs and backing storage.
+ */
+enum irdma_status_code irdma_sc_del_hmc_obj(struct irdma_sc_dev *dev,
+					    struct irdma_hmc_del_obj_info *info,
+					    bool reset)
+{
+	struct irdma_hmc_pd_table *pd_table;
+	u32 sd_idx, sd_lmt;
+	u32 pd_idx, pd_lmt, rel_pd_idx;
+	u32 i, j;
+	enum irdma_status_code ret_code = 0;
+
+	if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
+		dev_dbg(rfdev_to_dev(dev),
+			"HMC: error start_idx[%04d]  >= [type %04d].cnt[%04d]\n",
+			info->start_idx, info->rsrc_type,
+			info->hmc_info->hmc_obj[info->rsrc_type].cnt);
+		return IRDMA_ERR_INVALID_HMC_OBJ_INDEX;
+	}
+
+	if ((info->start_idx + info->count) >
+	    info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
+		dev_dbg(rfdev_to_dev(dev),
+			"HMC: error start_idx[%04d] + count %04d  >= [type %04d].cnt[%04d]\n",
+			info->start_idx, info->count, info->rsrc_type,
+			info->hmc_info->hmc_obj[info->rsrc_type].cnt);
+		return IRDMA_ERR_INVALID_HMC_OBJ_COUNT;
+	}
+
+	irdma_find_pd_index_limit(info->hmc_info, info->rsrc_type,
+				  info->start_idx, info->count, &pd_idx,
+				  &pd_lmt);
+
+	for (j = pd_idx; j < pd_lmt; j++) {
+		sd_idx = j / IRDMA_HMC_PD_CNT_IN_SD;
+
+		if (!info->hmc_info->sd_table.sd_entry[sd_idx].valid)
+			continue;
+
+		if (info->hmc_info->sd_table.sd_entry[sd_idx].entry_type !=
+		    IRDMA_SD_TYPE_PAGED)
+			continue;
+
+		rel_pd_idx = j % IRDMA_HMC_PD_CNT_IN_SD;
+		pd_table = &info->hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
+		if (pd_table->pd_entry &&
+		    pd_table->pd_entry[rel_pd_idx].valid) {
+			ret_code = irdma_remove_pd_bp(dev, info->hmc_info, j);
+			if (ret_code) {
+				dev_dbg(rfdev_to_dev(dev),
+					"HMC: remove_pd_bp error\n");
+				return ret_code;
+			}
+		}
+	}
+
+	irdma_find_sd_index_limit(info->hmc_info, info->rsrc_type,
+				  info->start_idx, info->count, &sd_idx,
+				  &sd_lmt);
+	if (sd_idx >= info->hmc_info->sd_table.sd_cnt ||
+	    sd_lmt > info->hmc_info->sd_table.sd_cnt) {
+		dev_dbg(rfdev_to_dev(dev), "HMC: invalid sd_idx\n");
+		return IRDMA_ERR_INVALID_SD_INDEX;
+	}
+
+	for (i = sd_idx; i < sd_lmt; i++) {
+		pd_table = &info->hmc_info->sd_table.sd_entry[i].u.pd_table;
+		if (!info->hmc_info->sd_table.sd_entry[i].valid)
+			continue;
+		switch (info->hmc_info->sd_table.sd_entry[i].entry_type) {
+		case IRDMA_SD_TYPE_DIRECT:
+			ret_code = irdma_prep_remove_sd_bp(info->hmc_info, i);
+			if (!ret_code) {
+				info->hmc_info->sd_indexes[info->del_sd_cnt] =
+					(u16)i;
+				info->del_sd_cnt++;
+			}
+			break;
+		case IRDMA_SD_TYPE_PAGED:
+			ret_code = irdma_prep_remove_pd_page(info->hmc_info, i);
+			if (ret_code)
+				break;
+			if (dev->hmc_info != info->hmc_info &&
+			    info->rsrc_type == IRDMA_HMC_IW_PBLE &&
+			    pd_table->pd_entry) {
+				kfree(pd_table->pd_entry_virt_mem.va);
+				pd_table->pd_entry = NULL;
+			}
+			info->hmc_info->sd_indexes[info->del_sd_cnt] = (u16)i;
+			info->del_sd_cnt++;
+			break;
+		default:
+			break;
+		}
+	}
+	return irdma_finish_del_sd_reg(dev, info, reset);
+}
+
+/**
+ * irdma_add_sd_table_entry - Adds a segment descriptor to the table
+ * @hw: pointer to our hw struct
+ * @hmc_info: pointer to the HMC configuration information struct
+ * @sd_index: segment descriptor index to manipulate
+ * @type: what type of segment descriptor we're manipulating
+ * @direct_mode_sz: size to alloc in direct mode
+ */
+enum irdma_status_code irdma_add_sd_table_entry(struct irdma_hw *hw,
+						struct irdma_hmc_info *hmc_info,
+						u32 sd_index,
+						enum irdma_sd_entry_type type,
+						u64 direct_mode_sz)
+{
+	struct irdma_hmc_sd_entry *sd_entry;
+	struct irdma_dma_mem dma_mem;
+	u64 alloc_len;
+
+	sd_entry = &hmc_info->sd_table.sd_entry[sd_index];
+	if (!sd_entry->valid) {
+		if (type == IRDMA_SD_TYPE_PAGED)
+			alloc_len = IRDMA_HMC_PAGED_BP_SIZE;
+		else
+			alloc_len = direct_mode_sz;
+
+		/* allocate a 4K pd page or 2M backing page */
+		dma_mem.size = ALIGN(alloc_len, IRDMA_HMC_PD_BP_BUF_ALIGNMENT);
+		dma_mem.va = dma_alloc_coherent(hw_to_dev(hw),
+						dma_mem.size, &dma_mem.pa,
+						GFP_ATOMIC);
+		if (!dma_mem.va)
+			return IRDMA_ERR_NO_MEMORY;
+
+		if (type == IRDMA_SD_TYPE_PAGED) {
+			struct irdma_virt_mem *vmem =
+				&sd_entry->u.pd_table.pd_entry_virt_mem;
+
+			vmem->size = sizeof(struct irdma_hmc_pd_entry) * 512;
+			vmem->va = kzalloc(vmem->size, GFP_ATOMIC);
+			if (!vmem->va) {
+				dma_free_coherent(hw_to_dev(hw), dma_mem.size,
+						  dma_mem.va, dma_mem.pa);
+				dma_mem.va = NULL;
+				return IRDMA_ERR_NO_MEMORY;
+			}
+			sd_entry->u.pd_table.pd_entry = vmem->va;
+
+			memcpy(&sd_entry->u.pd_table.pd_page_addr, &dma_mem,
+			       sizeof(sd_entry->u.pd_table.pd_page_addr));
+		} else {
+			memcpy(&sd_entry->u.bp.addr, &dma_mem,
+			       sizeof(sd_entry->u.bp.addr));
+
+			sd_entry->u.bp.sd_pd_index = sd_index;
+		}
+
+		hmc_info->sd_table.sd_entry[sd_index].entry_type = type;
+		IRDMA_INC_SD_REFCNT(&hmc_info->sd_table);
+	}
+	if (sd_entry->entry_type == IRDMA_SD_TYPE_DIRECT)
+		IRDMA_INC_BP_REFCNT(&sd_entry->u.bp);
+
+	return 0;
+}
+
+/**
+ * irdma_add_pd_table_entry - Adds page descriptor to the specified table
+ * @dev: pointer to our device structure
+ * @hmc_info: pointer to the HMC configuration information structure
+ * @pd_index: which page descriptor index to manipulate
+ * @rsrc_pg: if not NULL, use preallocated page instead of allocating new one.
+ *
+ * This function:
+ *	1. Initializes the pd entry
+ *	2. Adds pd_entry in the pd_table
+ *	3. Mark the entry valid in irdma_hmc_pd_entry structure
+ *	4. Initializes the pd_entry's ref count to 1
+ * assumptions:
+ *	1. The memory for pd should be pinned down, physically contiguous and
+ *	   aligned on 4K boundary and zeroed memory.
+ *	2. It should be 4K in size.
+ */
+enum irdma_status_code irdma_add_pd_table_entry(struct irdma_sc_dev *dev,
+						struct irdma_hmc_info *hmc_info,
+						u32 pd_index,
+						struct irdma_dma_mem *rsrc_pg)
+{
+	struct irdma_hmc_pd_table *pd_table;
+	struct irdma_hmc_pd_entry *pd_entry;
+	struct irdma_dma_mem mem;
+	struct irdma_dma_mem *page = &mem;
+	u32 sd_idx, rel_pd_idx;
+	u64 *pd_addr;
+	u64 page_desc;
+
+	if (pd_index / IRDMA_HMC_PD_CNT_IN_SD >= hmc_info->sd_table.sd_cnt)
+		return IRDMA_ERR_INVALID_PAGE_DESC_INDEX;
+
+	sd_idx = (pd_index / IRDMA_HMC_PD_CNT_IN_SD);
+	if (hmc_info->sd_table.sd_entry[sd_idx].entry_type !=
+	    IRDMA_SD_TYPE_PAGED)
+		return 0;
+
+	rel_pd_idx = (pd_index % IRDMA_HMC_PD_CNT_IN_SD);
+	pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
+	pd_entry = &pd_table->pd_entry[rel_pd_idx];
+	if (!pd_entry->valid) {
+		if (rsrc_pg) {
+			pd_entry->rsrc_pg = true;
+			page = rsrc_pg;
+		} else {
+			page->size = ALIGN(IRDMA_HMC_PAGED_BP_SIZE,
+					   IRDMA_HMC_PD_BP_BUF_ALIGNMENT);
+			page->va = dma_alloc_coherent(hw_to_dev(dev->hw),
+						      page->size, &page->pa,
+						      GFP_KERNEL);
+			if (!page->va)
+				return IRDMA_ERR_NO_MEMORY;
+
+			pd_entry->rsrc_pg = false;
+		}
+
+		memcpy(&pd_entry->bp.addr, page, sizeof(pd_entry->bp.addr));
+		pd_entry->bp.sd_pd_index = pd_index;
+		pd_entry->bp.entry_type = IRDMA_SD_TYPE_PAGED;
+		page_desc = page->pa | 0x1;
+		pd_addr = pd_table->pd_page_addr.va;
+		pd_addr += rel_pd_idx;
+		memcpy(pd_addr, &page_desc, sizeof(*pd_addr));
+		pd_entry->sd_index = sd_idx;
+		pd_entry->valid = true;
+		IRDMA_INC_PD_REFCNT(pd_table);
+		if (hmc_info->hmc_fn_id < dev->hw_attrs.first_hw_vf_fpm_id)
+			IRDMA_INVALIDATE_PF_HMC_PD(dev, sd_idx, rel_pd_idx);
+		else if (dev->hw->hmc.hmc_fn_id != hmc_info->hmc_fn_id)
+			IRDMA_INVALIDATE_VF_HMC_PD(dev, sd_idx, rel_pd_idx,
+						   hmc_info->hmc_fn_id);
+	}
+	IRDMA_INC_BP_REFCNT(&pd_entry->bp);
+
+	return 0;
+}
+
+/**
+ * irdma_remove_pd_bp - remove a backing page from a page descriptor
+ * @dev: pointer to our HW structure
+ * @hmc_info: pointer to the HMC configuration information structure
+ * @idx: the page index
+ *
+ * This function:
+ *	1. Marks the entry in pd table (for paged address mode) or in sd table
+ *	   (for direct address mode) invalid.
+ *	2. Write to register PMPDINV to invalidate the backing page in FV cache
+ *	3. Decrement the ref count for the pd _entry
+ * assumptions:
+ *	1. Caller can deallocate the memory used by backing storage after this
+ *	   function returns.
+ */
+enum irdma_status_code irdma_remove_pd_bp(struct irdma_sc_dev *dev,
+					  struct irdma_hmc_info *hmc_info,
+					  u32 idx)
+{
+	struct irdma_hmc_pd_entry *pd_entry;
+	struct irdma_hmc_pd_table *pd_table;
+	struct irdma_hmc_sd_entry *sd_entry;
+	u32 sd_idx, rel_pd_idx;
+	struct irdma_dma_mem *mem;
+	u64 *pd_addr;
+
+	sd_idx = idx / IRDMA_HMC_PD_CNT_IN_SD;
+	rel_pd_idx = idx % IRDMA_HMC_PD_CNT_IN_SD;
+	if (sd_idx >= hmc_info->sd_table.sd_cnt)
+		return IRDMA_ERR_INVALID_PAGE_DESC_INDEX;
+
+	sd_entry = &hmc_info->sd_table.sd_entry[sd_idx];
+	if (sd_entry->entry_type != IRDMA_SD_TYPE_PAGED)
+		return IRDMA_ERR_INVALID_SD_TYPE;
+
+	pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
+	pd_entry = &pd_table->pd_entry[rel_pd_idx];
+	IRDMA_DEC_BP_REFCNT(&pd_entry->bp);
+	if (pd_entry->bp.ref_cnt)
+		return 0;
+
+	pd_entry->valid = false;
+	IRDMA_DEC_PD_REFCNT(pd_table);
+	pd_addr = pd_table->pd_page_addr.va;
+	pd_addr += rel_pd_idx;
+	memset(pd_addr, 0, sizeof(u64));
+	if (dev->privileged) {
+		if (dev->hmc_fn_id == hmc_info->hmc_fn_id)
+			IRDMA_INVALIDATE_PF_HMC_PD(dev, sd_idx, idx);
+		else
+			IRDMA_INVALIDATE_VF_HMC_PD(dev, sd_idx, idx,
+						   hmc_info->hmc_fn_id);
+	}
+
+	if (!pd_entry->rsrc_pg) {
+		mem = &pd_entry->bp.addr;
+		if (!mem || !mem->va)
+			return IRDMA_ERR_PARAM;
+
+		dma_free_coherent(hw_to_dev(dev->hw), mem->size, mem->va,
+				  mem->pa);
+		mem->va = NULL;
+	}
+	if (!pd_table->ref_cnt)
+		kfree(pd_table->pd_entry_virt_mem.va);
+
+	return 0;
+}
+
+/**
+ * irdma_prep_remove_sd_bp - Prepares to remove a backing page from a sd entry
+ * @hmc_info: pointer to the HMC configuration information structure
+ * @idx: the page index
+ */
+enum irdma_status_code irdma_prep_remove_sd_bp(struct irdma_hmc_info *hmc_info,
+					       u32 idx)
+{
+	struct irdma_hmc_sd_entry *sd_entry;
+
+	sd_entry = &hmc_info->sd_table.sd_entry[idx];
+	IRDMA_DEC_BP_REFCNT(&sd_entry->u.bp);
+	if (sd_entry->u.bp.ref_cnt)
+		return IRDMA_ERR_NOT_READY;
+
+	IRDMA_DEC_SD_REFCNT(&hmc_info->sd_table);
+	sd_entry->valid = false;
+
+	return 0;
+}
+
+/**
+ * irdma_prep_remove_pd_page - Prepares to remove a PD page from sd entry.
+ * @hmc_info: pointer to the HMC configuration information structure
+ * @idx: segment descriptor index to find the relevant page descriptor
+ */
+enum irdma_status_code
+irdma_prep_remove_pd_page(struct irdma_hmc_info *hmc_info, u32 idx)
+{
+	struct irdma_hmc_sd_entry *sd_entry;
+
+	sd_entry = &hmc_info->sd_table.sd_entry[idx];
+
+	if (sd_entry->u.pd_table.ref_cnt)
+		return IRDMA_ERR_NOT_READY;
+
+	sd_entry->valid = false;
+	IRDMA_DEC_SD_REFCNT(&hmc_info->sd_table);
+
+	return 0;
+}
drivers/infiniband/hw/irdma/hmc.h new file +218 -0

218 lines not on this page · raw message

Signature · 1 line
2.23.0
#35 lore raw

PATCH v3 9/20 RDMA/irdma: Add privileged UDA queue implementation

In reply to Jeff Kirsher

From: Mustafa Ismail <[email protected]>

Implement privileged UDA queues to handle iWARP connection packets and receive exceptions.

---
 drivers/infiniband/hw/irdma/puda.c | 1690 ++++++++++++++++++++++++++++
 drivers/infiniband/hw/irdma/puda.h |  186 +++
 2 files changed, 1876 insertions(+)
 create mode 100644 drivers/infiniband/hw/irdma/puda.c
 create mode 100644 drivers/infiniband/hw/irdma/puda.h
drivers/infiniband/hw/irdma/puda.c new file +1690 -0
new file mode 100644
index 000000000000..1d5d3eb37885
@@ -0,0 +1,1690 @@
+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
+/* Copyright (c) 2015 - 2019 Intel Corporation */
+#include "osdep.h"
+#include "status.h"
+#include "hmc.h"
+#include "defs.h"
+#include "type.h"
+#include "protos.h"
+#include "puda.h"
+#include "ws.h"
+
+static void irdma_ieq_receive(struct irdma_sc_vsi *vsi,
+			      struct irdma_puda_buf *buf);
+static void irdma_ieq_tx_compl(struct irdma_sc_vsi *vsi, void *sqwrid);
+static void irdma_ilq_putback_rcvbuf(struct irdma_sc_qp *qp, u32 wqe_idx);
+/**
+ * irdma_puda_get_listbuf - get buffer from puda list
+ * @list: list to use for buffers (ILQ or IEQ)
+ */
+static struct irdma_puda_buf *irdma_puda_get_listbuf(struct list_head *list)
+{
+	struct irdma_puda_buf *buf = NULL;
+
+	if (!list_empty(list)) {
+		buf = (struct irdma_puda_buf *)list->next;
+		list_del((struct list_head *)&buf->list);
+	}
+
+	return buf;
+}
+
+/**
+ * irdma_puda_get_bufpool - return buffer from resource
+ * @rsrc: resource to use for buffer
+ */
+struct irdma_puda_buf *irdma_puda_get_bufpool(struct irdma_puda_rsrc *rsrc)
+{
+	struct irdma_puda_buf *buf = NULL;
+	struct list_head *list = &rsrc->bufpool;
+	unsigned long flags;
+
+	spin_lock_irqsave(&rsrc->bufpool_lock, flags);
+	buf = irdma_puda_get_listbuf(list);
+	if (buf) {
+		rsrc->avail_buf_count--;
+		buf->vsi = rsrc->vsi;
+	} else {
+		rsrc->stats_buf_alloc_fail++;
+	}
+	spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
+
+	return buf;
+}
+
+/**
+ * irdma_puda_ret_bufpool - return buffer to rsrc list
+ * @rsrc: resource to use for buffer
+ * @buf: buffer to return to resource
+ */
+void irdma_puda_ret_bufpool(struct irdma_puda_rsrc *rsrc,
+			    struct irdma_puda_buf *buf)
+{
+	unsigned long flags;
+
+	buf->do_lpb = false;
+	spin_lock_irqsave(&rsrc->bufpool_lock, flags);
+	list_add(&buf->list, &rsrc->bufpool);
+	spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
+	rsrc->avail_buf_count++;
+}
+
+/**
+ * irdma_puda_post_recvbuf - set wqe for rcv buffer
+ * @rsrc: resource ptr
+ * @wqe_idx: wqe index to use
+ * @buf: puda buffer for rcv q
+ * @initial: flag if during init time
+ */
+static void irdma_puda_post_recvbuf(struct irdma_puda_rsrc *rsrc, u32 wqe_idx,
+				    struct irdma_puda_buf *buf, bool initial)
+{
+	__le64 *wqe;
+	struct irdma_sc_qp *qp = &rsrc->qp;
+	u64 offset24 = 0;
+
+	qp->qp_uk.rq_wrid_array[wqe_idx] = (uintptr_t)buf;
+	wqe = qp->qp_uk.rq_base[wqe_idx].elem;
+	if (!initial)
+		get_64bit_val(wqe, 24, &offset24);
+
+	offset24 = (offset24) ? 0 : LS_64(1, IRDMAQPSQ_VALID);
+
+	set_64bit_val(wqe, 16, 0);
+	set_64bit_val(wqe, 0, buf->mem.pa);
+	if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {
+		set_64bit_val(wqe, 8,
+			      LS_64(buf->mem.size, IRDMAQPSQ_GEN1_FRAG_LEN));
+	} else {
+		set_64bit_val(wqe, 8,
+			      LS_64(buf->mem.size,
+				    IRDMAQPSQ_FRAG_LEN) | (offset24 & IRDMAQPSQ_VALID_M));
+	}
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, offset24);
+}
+
+/**
+ * irdma_puda_replenish_rq - post rcv buffers
+ * @rsrc: resource to use for buffer
+ * @initial: flag if during init time
+ */
+static enum irdma_status_code
+irdma_puda_replenish_rq(struct irdma_puda_rsrc *rsrc, bool initial)
+{
+	u32 i;
+	u32 invalid_cnt = rsrc->rxq_invalid_cnt;
+	struct irdma_puda_buf *buf = NULL;
+
+	for (i = 0; i < invalid_cnt; i++) {
+		buf = irdma_puda_get_bufpool(rsrc);
+		if (!buf)
+			return IRDMA_ERR_list_empty;
+		irdma_puda_post_recvbuf(rsrc, rsrc->rx_wqe_idx, buf, initial);
+		rsrc->rx_wqe_idx = ((rsrc->rx_wqe_idx + 1) % rsrc->rq_size);
+		rsrc->rxq_invalid_cnt--;
+	}
+
+	return 0;
+}
+
+/**
+ * irdma_puda_alloc_buf - allocate mem for buffer
+ * @dev: iwarp device
+ * @len: length of buffer
+ */
+static struct irdma_puda_buf *irdma_puda_alloc_buf(struct irdma_sc_dev *dev,
+						   u32 len)
+{
+	struct irdma_puda_buf *buf = NULL;
+	struct irdma_virt_mem buf_mem;
+
+	buf_mem.size = sizeof(struct irdma_puda_buf);
+	buf_mem.va = kzalloc(buf_mem.size, GFP_ATOMIC);
+	if (!buf_mem.va)
+		return NULL;
+
+	buf = buf_mem.va;
+	buf->mem.size = ALIGN(len, 1);
+	buf->mem.va = dma_alloc_coherent(hw_to_dev(dev->hw), buf->mem.size,
+					 &buf->mem.pa, GFP_KERNEL);
+	if (!buf->mem.va) {
+		kfree(buf_mem.va);
+		return NULL;
+	}
+
+	buf->buf_mem.va = buf_mem.va;
+	buf->buf_mem.size = buf_mem.size;
+
+	return buf;
+}
+
+/**
+ * irdma_puda_dele_buf - delete buffer back to system
+ * @dev: iwarp device
+ * @buf: buffer to free
+ */
+static void irdma_puda_dele_buf(struct irdma_sc_dev *dev,
+				struct irdma_puda_buf *buf)
+{
+	dma_free_coherent(hw_to_dev(dev->hw), buf->mem.size, buf->mem.va,
+			  buf->mem.pa);
+	buf->mem.va = NULL;
+	kfree(buf->buf_mem.va);
+}
+
+/**
+ * irdma_puda_get_next_send_wqe - return next wqe for processing
+ * @qp: puda qp for wqe
+ * @wqe_idx: wqe index for caller
+ */
+static __le64 *irdma_puda_get_next_send_wqe(struct irdma_qp_uk *qp,
+					    u32 *wqe_idx)
+{
+	__le64 *wqe = NULL;
+	enum irdma_status_code ret_code = 0;
+
+	*wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
+	if (!*wqe_idx)
+		qp->swqe_polarity = !qp->swqe_polarity;
+	IRDMA_RING_MOVE_HEAD(qp->sq_ring, ret_code);
+	if (ret_code)
+		return wqe;
+
+	wqe = qp->sq_base[*wqe_idx].elem;
+
+	return wqe;
+}
+
+/**
+ * irdma_puda_poll_info - poll cq for completion
+ * @cq: cq for poll
+ * @info: info return for successful completion
+ */
+static enum irdma_status_code
+irdma_puda_poll_info(struct irdma_sc_cq *cq, struct irdma_puda_cmpl_info *info)
+{
+	struct irdma_cq_uk *cq_uk = &cq->cq_uk;
+	u64 qword0, qword2, qword3, qword6;
+	__le64 *cqe;
+	__le64 *ext_cqe = NULL;
+	u64 qword7 = 0;
+	u64 comp_ctx;
+	bool valid_bit;
+	bool ext_valid = 0;
+	u32 major_err, minor_err;
+	u32 peek_head;
+	bool error;
+	u8 polarity;
+
+	cqe = IRDMA_GET_CURRENT_CQ_ELEM(&cq->cq_uk);
+	get_64bit_val(cqe, 24, &qword3);
+	valid_bit = (bool)RS_64(qword3, IRDMA_CQ_VALID);
+	if (valid_bit != cq_uk->polarity)
+		return IRDMA_ERR_Q_EMPTY;
+
+	if (cq->dev->hw_attrs.uk_attrs.hw_rev > IRDMA_GEN_1)
+		ext_valid = (bool)RS_64(qword3, IRDMA_CQ_EXTCQE);
+
+	if (ext_valid) {
+		peek_head = (cq_uk->cq_ring.head + 1) % cq_uk->cq_ring.size;
+		ext_cqe = cq_uk->cq_base[peek_head].buf;
+		get_64bit_val(ext_cqe, 24, &qword7);
+		polarity = (u8)RS_64(qword7, IRDMA_CQ_VALID);
+		if (!peek_head)
+			polarity ^= 1;
+		if (polarity != cq_uk->polarity)
+			return IRDMA_ERR_Q_EMPTY;
+
+		IRDMA_RING_MOVE_HEAD_NOCHECK(cq_uk->cq_ring);
+		if (!IRDMA_RING_CURRENT_HEAD(cq_uk->cq_ring))
+			cq_uk->polarity = !cq_uk->polarity;
+		/* update cq tail in cq shadow memory also */
+		IRDMA_RING_MOVE_TAIL(cq_uk->cq_ring);
+	}
+
+	irdma_debug_buf(cq->dev, IRDMA_DEBUG_PUDA, "PUDA CQE", cqe, 32);
+	if (ext_valid)
+		irdma_debug_buf(cq->dev, IRDMA_DEBUG_PUDA, "PUDA EXT-CQE",
+				ext_cqe, 32);
+
+	error = (bool)RS_64(qword3, IRDMA_CQ_ERROR);
+	if (error) {
+		dev_dbg(rfdev_to_dev(cq->dev), "PUDA: receive error\n");
+		major_err = (u32)(RS_64(qword3, IRDMA_CQ_MAJERR));
+		minor_err = (u32)(RS_64(qword3, IRDMA_CQ_MINERR));
+		info->compl_error = major_err << 16 | minor_err;
+		return IRDMA_ERR_CQ_COMPL_ERROR;
+	}
+
+	get_64bit_val(cqe, 0, &qword0);
+	get_64bit_val(cqe, 16, &qword2);
+
+	info->q_type = (u8)RS_64(qword3, IRDMA_CQ_SQ);
+	info->qp_id = (u32)RS_64(qword2, IRDMACQ_QPID);
+	if (cq->dev->hw_attrs.uk_attrs.hw_rev > IRDMA_GEN_1)
+		info->ipv4 = (bool)RS_64(qword3, IRDMACQ_IPV4);
+
+	get_64bit_val(cqe, 8, &comp_ctx);
+	info->qp = (struct irdma_qp_uk *)(unsigned long)comp_ctx;
+	info->wqe_idx = (u32)RS_64(qword3, IRDMA_CQ_WQEIDX);
+
+	if (info->q_type == IRDMA_CQE_QTYPE_RQ) {
+		if (ext_valid) {
+			info->vlan_valid = (bool)RS_64(qword7, IRDMA_CQ_UDVLANVALID);
+			if (info->vlan_valid) {
+				get_64bit_val(ext_cqe, 16, &qword6);
+				info->vlan = (u16)RS_64(qword6, IRDMA_CQ_UDVLAN);
+			}
+			info->smac_valid = (bool)RS_64(qword7, IRDMA_CQ_UDSMACVALID);
+			if (info->smac_valid) {
+				get_64bit_val(ext_cqe, 16, &qword6);
+				info->smac[0] = (u8)((qword6 >> 40) & 0xFF);
+				info->smac[1] = (u8)((qword6 >> 32) & 0xFF);
+				info->smac[2] = (u8)((qword6 >> 24) & 0xFF);
+				info->smac[3] = (u8)((qword6 >> 16) & 0xFF);
+				info->smac[4] = (u8)((qword6 >> 8) & 0xFF);
+				info->smac[5] = (u8)(qword6 & 0xFF);
+			}
+		}
+
+		if (cq->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) {
+			info->vlan_valid = (bool)RS_64(qword3, IRDMA_VLAN_TAG_VALID);
+			info->l4proto = (u8)RS_64(qword2, IRDMA_UDA_L4PROTO);
+			info->l3proto = (u8)RS_64(qword2, IRDMA_UDA_L3PROTO);
+		}
+
+		info->payload_len = (u32)RS_64(qword0, IRDMACQ_PAYLDLEN);
+	}
+
+	return 0;
+}
+
+/**
+ * irdma_puda_poll_completion - processes completion for cq
+ * @dev: iwarp device
+ * @cq: cq getting interrupt
+ * @compl_err: return any completion err
+ */
+enum irdma_status_code irdma_puda_poll_cmpl(struct irdma_sc_dev *dev,
+					    struct irdma_sc_cq *cq,
+					    u32 *compl_err)
+{
+	struct irdma_qp_uk *qp;
+	struct irdma_cq_uk *cq_uk = &cq->cq_uk;
+	struct irdma_puda_cmpl_info info = {};
+	enum irdma_status_code ret = 0;
+	struct irdma_puda_buf *buf;
+	struct irdma_puda_rsrc *rsrc;
+	void *sqwrid;
+	u8 cq_type = cq->cq_type;
+	unsigned long flags;
+
+	if (cq_type == IRDMA_CQ_TYPE_ILQ || cq_type == IRDMA_CQ_TYPE_IEQ) {
+		rsrc = (cq_type == IRDMA_CQ_TYPE_ILQ) ? cq->vsi->ilq :
+							cq->vsi->ieq;
+	} else {
+		dev_dbg(rfdev_to_dev(dev), "PUDA: qp_type error\n");
+		return IRDMA_ERR_BAD_PTR;
+	}
+
+	ret = irdma_puda_poll_info(cq, &info);
+	*compl_err = info.compl_error;
+	if (ret == IRDMA_ERR_Q_EMPTY)
+		return ret;
+	if (ret)
+		goto done;
+
+	qp = info.qp;
+	if (!qp || !rsrc) {
+		ret = IRDMA_ERR_BAD_PTR;
+		goto done;
+	}
+
+	if (qp->qp_id != rsrc->qp_id) {
+		ret = IRDMA_ERR_BAD_PTR;
+		goto done;
+	}
+
+	if (info.q_type == IRDMA_CQE_QTYPE_RQ) {
+		buf = (struct irdma_puda_buf *)(uintptr_t)
+			      qp->rq_wrid_array[info.wqe_idx];
+		/* Get all the tcpip information in the buf header */
+		ret = irdma_puda_get_tcpip_info(&info, buf);
+		if (ret) {
+			rsrc->stats_rcvd_pkt_err++;
+			if (cq_type == IRDMA_CQ_TYPE_ILQ) {
+				irdma_ilq_putback_rcvbuf(&rsrc->qp,
+							 info.wqe_idx);
+			} else {
+				irdma_puda_ret_bufpool(rsrc, buf);
+				irdma_puda_replenish_rq(rsrc, false);
+			}
+			goto done;
+		}
+
+		rsrc->stats_pkt_rcvd++;
+		rsrc->compl_rxwqe_idx = info.wqe_idx;
+		dev_dbg(rfdev_to_dev(dev), "PUDA: RQ completion\n");
+		rsrc->receive(rsrc->vsi, buf);
+		if (cq_type == IRDMA_CQ_TYPE_ILQ)
+			irdma_ilq_putback_rcvbuf(&rsrc->qp, info.wqe_idx);
+		else
+			irdma_puda_replenish_rq(rsrc, false);
+
+	} else {
+		dev_dbg(rfdev_to_dev(dev), "PUDA: SQ completion\n");
+		sqwrid = (void *)(uintptr_t)qp->sq_wrtrk_array[info.wqe_idx].wrid;
+		IRDMA_RING_SET_TAIL(qp->sq_ring, info.wqe_idx);
+		rsrc->xmit_complete(rsrc->vsi, sqwrid);
+		spin_lock_irqsave(&rsrc->bufpool_lock, flags);
+		rsrc->tx_wqe_avail_cnt++;
+		spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
+		if (!list_empty(&rsrc->txpend))
+			irdma_puda_send_buf(rsrc, NULL);
+	}
+
+done:
+	IRDMA_RING_MOVE_HEAD_NOCHECK(cq_uk->cq_ring);
+	if (!IRDMA_RING_CURRENT_HEAD(cq_uk->cq_ring))
+		cq_uk->polarity = !cq_uk->polarity;
+	/* update cq tail in cq shadow memory also */
+	IRDMA_RING_MOVE_TAIL(cq_uk->cq_ring);
+	set_64bit_val(cq_uk->shadow_area, 0,
+		      IRDMA_RING_CURRENT_HEAD(cq_uk->cq_ring));
+
+	return ret;
+}
+
+/**
+ * irdma_puda_send - complete send wqe for transmit
+ * @qp: puda qp for send
+ * @info: buffer information for transmit
+ */
+enum irdma_status_code irdma_puda_send(struct irdma_sc_qp *qp,
+				       struct irdma_puda_send_info *info)
+{
+	__le64 *wqe;
+	u32 iplen, l4len;
+	u64 hdr[2];
+	u32 wqe_idx;
+	u8 iipt;
+
+	/* number of 32 bits DWORDS in header */
+	l4len = info->tcplen >> 2;
+	if (info->ipv4) {
+		iipt = 3;
+		iplen = 5;
+	} else {
+		iipt = 1;
+		iplen = 10;
+	}
+
+	wqe = irdma_puda_get_next_send_wqe(&qp->qp_uk, &wqe_idx);
+	if (!wqe)
+		return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
+
+	qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid = (uintptr_t)info->scratch;
+	/* Third line of WQE descriptor */
+	/* maclen is in words */
+
+	if (qp->dev->hw_attrs.uk_attrs.hw_rev > IRDMA_GEN_1) {
+		hdr[0] = 0; /* Dest_QPN and Dest_QKey only for UD */
+		hdr[1] = LS_64(IRDMA_OP_TYPE_SEND, IRDMA_UDA_QPSQ_OPCODE) |
+			 LS_64(l4len, IRDMA_UDA_QPSQ_L4LEN) |
+			 LS_64(info->ah_id, IRDMAQPSQ_AHID) |
+			 LS_64(1, IRDMA_UDA_QPSQ_SIGCOMPL) |
+			 LS_64(qp->qp_uk.swqe_polarity, IRDMA_UDA_QPSQ_VALID);
+
+		/* Forth line of WQE descriptor */
+
+		set_64bit_val(wqe, 0, info->paddr);
+		set_64bit_val(wqe, 8,
+			      LS_64(info->len, IRDMAQPSQ_FRAG_LEN) |
+			      LS_64(qp->qp_uk.swqe_polarity, IRDMA_UDA_QPSQ_VALID));
+	} else {
+		hdr[0] = LS_64((info->maclen >> 1), IRDMA_UDA_QPSQ_MACLEN) |
+			 LS_64(iplen, IRDMA_UDA_QPSQ_IPLEN) |
+			 LS_64(1, IRDMA_UDA_QPSQ_L4T) |
+			 LS_64(iipt, IRDMA_UDA_QPSQ_IIPT) |
+			 LS_64(l4len, IRDMA_GEN1_UDA_QPSQ_L4LEN);
+
+		hdr[1] = LS_64(IRDMA_OP_TYPE_SEND, IRDMA_UDA_QPSQ_OPCODE) |
+			 LS_64(1, IRDMA_UDA_QPSQ_SIGCOMPL) |
+			 LS_64(info->do_lpb, IRDMA_UDA_QPSQ_DOLOOPBACK) |
+			 LS_64(qp->qp_uk.swqe_polarity, IRDMA_UDA_QPSQ_VALID);
+
+		/* Forth line of WQE descriptor */
+
+		set_64bit_val(wqe, 0, info->paddr);
+		set_64bit_val(wqe, 8,
+			      LS_64(info->len, IRDMAQPSQ_GEN1_FRAG_LEN));
+	}
+
+	set_64bit_val(wqe, 16, hdr[0]);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr[1]);
+
+	irdma_debug_buf(qp->dev, IRDMA_DEBUG_PUDA, "PUDA SEND WQE", wqe, 32);
+	irdma_qp_post_wr(&qp->qp_uk);
+	return 0;
+}
+
+/**
+ * irdma_puda_send_buf - transmit puda buffer
+ * @rsrc: resource to use for buffer
+ * @buf: puda buffer to transmit
+ */
+void irdma_puda_send_buf(struct irdma_puda_rsrc *rsrc,
+			 struct irdma_puda_buf *buf)
+{
+	struct irdma_puda_send_info info;
+	enum irdma_status_code ret = 0;
+	unsigned long flags;
+
+	spin_lock_irqsave(&rsrc->bufpool_lock, flags);
+	/* if no wqe available or not from a completion and we have
+	 * pending buffers, we must queue new buffer
+	 */
+	if (!rsrc->tx_wqe_avail_cnt || (buf && !list_empty(&rsrc->txpend))) {
+		list_add_tail(&buf->list, &rsrc->txpend);
+		spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
+		rsrc->stats_sent_pkt_q++;
+		if (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ)
+			dev_dbg(rfdev_to_dev(rsrc->dev),
+				"PUDA: adding to txpend\n");
+		return;
+	}
+	rsrc->tx_wqe_avail_cnt--;
+	/* if we are coming from a completion and have pending buffers
+	 * then Get one from pending list
+	 */
+	if (!buf) {
+		buf = irdma_puda_get_listbuf(&rsrc->txpend);
+		if (!buf)
+			goto done;
+	}
+
+	info.scratch = (void *)buf;
+	info.paddr = buf->mem.pa;
+	info.len = buf->totallen;
+	info.tcplen = buf->tcphlen;
+	info.ipv4 = buf->ipv4;
+
+	if (rsrc->dev->hw_attrs.uk_attrs.hw_rev > IRDMA_GEN_1) {
+		info.ah_id = buf->ah_id;
+	} else {
+		info.maclen = buf->maclen;
+		info.do_lpb = buf->do_lpb;
+	}
+
+	ret = irdma_puda_send(&rsrc->qp, &info);
+	if (ret) {
+		rsrc->tx_wqe_avail_cnt++;
+		rsrc->stats_sent_pkt_q++;
+		list_add(&buf->list, &rsrc->txpend);
+		if (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ)
+			dev_dbg(rfdev_to_dev(rsrc->dev),
+				"PUDA: adding to puda_send\n");
+	} else {
+		rsrc->stats_pkt_sent++;
+	}
+done:
+	spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
+}
+
+/**
+ * irdma_puda_qp_setctx - during init, set qp's context
+ * @rsrc: qp's resource
+ */
+static void irdma_puda_qp_setctx(struct irdma_puda_rsrc *rsrc)
+{
+	struct irdma_sc_qp *qp = &rsrc->qp;
+	__le64 *qp_ctx = qp->hw_host_ctx;
+
+	set_64bit_val(qp_ctx, 8, qp->sq_pa);
+	set_64bit_val(qp_ctx, 16, qp->rq_pa);
+	set_64bit_val(qp_ctx, 24,
+		      LS_64(qp->hw_rq_size, IRDMAQPC_RQSIZE) |
+		      LS_64(qp->hw_sq_size, IRDMAQPC_SQSIZE));
+	set_64bit_val(qp_ctx, 48,
+		      LS_64(rsrc->buf_size, IRDMAQPC_SNDMSS));
+	set_64bit_val(qp_ctx, 56, 0);
+	if (qp->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
+		set_64bit_val(qp_ctx, 64, 1);
+	set_64bit_val(qp_ctx, 136,
+		      LS_64(rsrc->cq_id, IRDMAQPC_TXCQNUM) |
+		      LS_64(rsrc->cq_id, IRDMAQPC_RXCQNUM));
+	set_64bit_val(qp_ctx, 144,
+		      LS_64(rsrc->stats_idx, IRDMAQPC_STAT_INDEX));
+	set_64bit_val(qp_ctx, 160,
+		      LS_64(1, IRDMAQPC_PRIVEN) |
+		      LS_64(rsrc->stats_idx_valid, IRDMAQPC_USESTATSINSTANCE));
+	set_64bit_val(qp_ctx, 168,
+		      LS_64((uintptr_t)qp, IRDMAQPC_QPCOMPCTX));
+	set_64bit_val(qp_ctx, 176,
+		      LS_64(qp->sq_tph_val, IRDMAQPC_SQTPHVAL) |
+		      LS_64(qp->rq_tph_val, IRDMAQPC_RQTPHVAL) |
+		      LS_64(qp->qs_handle, IRDMAQPC_QSHANDLE));
+
+	irdma_debug_buf(rsrc->dev, IRDMA_DEBUG_PUDA, "PUDA QP CONTEXT", qp_ctx,
+			IRDMA_QP_CTX_SIZE);
+}
+
+/**
+ * irdma_puda_qp_wqe - setup wqe for qp create
+ * @dev: Device
+ * @qp: Resource qp
+ */
+static enum irdma_status_code irdma_puda_qp_wqe(struct irdma_sc_dev *dev,
+						struct irdma_sc_qp *qp)
+{
+	struct irdma_sc_cqp *cqp;
+	__le64 *wqe;
+	u64 hdr;
+	struct irdma_ccq_cqe_info compl_info;
+	enum irdma_status_code status = 0;
+
+	cqp = dev->cqp;
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, 0);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
+	set_64bit_val(wqe, 40, qp->shadow_area_pa);
+
+	hdr = qp->qp_uk.qp_id |
+	      LS_64(IRDMA_CQP_OP_CREATE_QP, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(IRDMA_QP_TYPE_UDA, IRDMA_CQPSQ_QP_QPTYPE) |
+	      LS_64(1, IRDMA_CQPSQ_QP_CQNUMVALID) |
+	      LS_64(2, IRDMA_CQPSQ_QP_NEXTIWSTATE) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(cqp->dev, IRDMA_DEBUG_PUDA, "PUDA QP CREATE", wqe, 40);
+	irdma_sc_cqp_post_sq(cqp);
+	status = dev->cqp_ops->poll_for_cqp_op_done(dev->cqp,
+						    IRDMA_CQP_OP_CREATE_QP,
+						    &compl_info);
+
+	return status;
+}
+
+/**
+ * irdma_puda_qp_create - create qp for resource
+ * @rsrc: resource to use for buffer
+ */
+static enum irdma_status_code irdma_puda_qp_create(struct irdma_puda_rsrc *rsrc)
+{
+	struct irdma_sc_qp *qp = &rsrc->qp;
+	struct irdma_qp_uk *ukqp = &qp->qp_uk;
+	enum irdma_status_code ret = 0;
+	u32 sq_size, rq_size;
+	struct irdma_dma_mem *mem;
+
+	sq_size = rsrc->sq_size * IRDMA_QP_WQE_MIN_SIZE;
+	rq_size = rsrc->rq_size * IRDMA_QP_WQE_MIN_SIZE;
+	rsrc->qpmem.size = ALIGN((sq_size + rq_size + (IRDMA_SHADOW_AREA_SIZE << 3) + IRDMA_QP_CTX_SIZE),
+				 IRDMA_HW_PAGE_SIZE);
+	rsrc->qpmem.va = dma_alloc_coherent(hw_to_dev(rsrc->dev->hw),
+					    rsrc->qpmem.size, &rsrc->qpmem.pa,
+					    GFP_KERNEL);
+	if (!rsrc->qpmem.va)
+		return IRDMA_ERR_NO_MEMORY;
+
+	mem = &rsrc->qpmem;
+	memset(mem->va, 0, rsrc->qpmem.size);
+	qp->hw_sq_size = irdma_get_encoded_wqe_size(rsrc->sq_size, false);
+	qp->hw_rq_size = irdma_get_encoded_wqe_size(rsrc->rq_size, false);
+	qp->pd = &rsrc->sc_pd;
+	qp->qp_type = IRDMA_QP_TYPE_UDA;
+	qp->dev = rsrc->dev;
+	qp->qp_uk.back_qp = (void *)rsrc;
+	qp->sq_pa = mem->pa;
+	qp->rq_pa = qp->sq_pa + sq_size;
+	qp->vsi = rsrc->vsi;
+	ukqp->sq_base = mem->va;
+	ukqp->rq_base = &ukqp->sq_base[rsrc->sq_size];
+	ukqp->shadow_area = ukqp->rq_base[rsrc->rq_size].elem;
+	ukqp->uk_attrs = &qp->dev->hw_attrs.uk_attrs;
+	qp->shadow_area_pa = qp->rq_pa + rq_size;
+	qp->hw_host_ctx = ukqp->shadow_area + IRDMA_SHADOW_AREA_SIZE;
+	qp->hw_host_ctx_pa = qp->shadow_area_pa + (IRDMA_SHADOW_AREA_SIZE << 3);
+	qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;
+	ukqp->qp_id = rsrc->qp_id;
+	ukqp->sq_wrtrk_array = rsrc->sq_wrtrk_array;
+	ukqp->rq_wrid_array = rsrc->rq_wrid_array;
+	ukqp->sq_size = rsrc->sq_size;
+	ukqp->rq_size = rsrc->rq_size;
+
+	IRDMA_RING_INIT(ukqp->sq_ring, ukqp->sq_size);
+	IRDMA_RING_INIT(ukqp->initial_ring, ukqp->sq_size);
+	IRDMA_RING_INIT(ukqp->rq_ring, ukqp->rq_size);
+	ukqp->wqe_alloc_db = qp->pd->dev->wqe_alloc_db;
+
+	ret = rsrc->dev->ws_add(qp->vsi, qp->user_pri);
+	if (ret) {
+		dma_free_coherent(hw_to_dev(rsrc->dev->hw), rsrc->qpmem.size,
+				  rsrc->qpmem.va, rsrc->qpmem.pa);
+		rsrc->qpmem.va = NULL;
+		return ret;
+	}
+
+	irdma_qp_add_qos(qp);
+	irdma_puda_qp_setctx(rsrc);
+
+	if (rsrc->dev->ceq_valid)
+		ret = irdma_cqp_qp_create_cmd(rsrc->dev, qp);
+	else
+		ret = irdma_puda_qp_wqe(rsrc->dev, qp);
+	if (ret) {
+		irdma_qp_rem_qos(qp);
+		rsrc->dev->ws_remove(qp->vsi, qp->user_pri);
+		dma_free_coherent(hw_to_dev(rsrc->dev->hw), rsrc->qpmem.size,
+				  rsrc->qpmem.va, rsrc->qpmem.pa);
+		rsrc->qpmem.va = NULL;
+	}
+
+	return ret;
+}
+
+/**
+ * irdma_puda_cq_wqe - setup wqe for CQ create
+ * @dev: Device
+ * @cq: resource for cq
+ */
+static enum irdma_status_code irdma_puda_cq_wqe(struct irdma_sc_dev *dev,
+						struct irdma_sc_cq *cq)
+{
+	__le64 *wqe;
+	struct irdma_sc_cqp *cqp;
+	u64 hdr;
+	struct irdma_ccq_cqe_info compl_info;
+	enum irdma_status_code status = 0;
+
+	cqp = dev->cqp;
+	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, 0);
+	if (!wqe)
+		return IRDMA_ERR_RING_FULL;
+
+	set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
+	set_64bit_val(wqe, 8, RS_64_1(cq, 1));
+	set_64bit_val(wqe, 16,
+		      LS_64(cq->shadow_read_threshold,
+			    IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
+	set_64bit_val(wqe, 32, cq->cq_pa);
+	set_64bit_val(wqe, 40, cq->shadow_area_pa);
+	set_64bit_val(wqe, 56,
+		      LS_64(cq->tph_val, IRDMA_CQPSQ_TPHVAL) |
+		      LS_64(cq->vsi->vsi_idx, IRDMA_CQPSQ_VSIIDX));
+
+	hdr = cq->cq_uk.cq_id |
+	      LS_64(IRDMA_CQP_OP_CREATE_CQ, IRDMA_CQPSQ_OPCODE) |
+	      LS_64(1, IRDMA_CQPSQ_CQ_CHKOVERFLOW) |
+	      LS_64(1, IRDMA_CQPSQ_CQ_ENCEQEMASK) |
+	      LS_64(1, IRDMA_CQPSQ_CQ_CEQIDVALID) |
+	      LS_64(cqp->polarity, IRDMA_CQPSQ_WQEVALID);
+	dma_wmb(); /* make sure WQE is written before valid bit is set */
+
+	set_64bit_val(wqe, 24, hdr);
+
+	irdma_debug_buf(dev, IRDMA_DEBUG_PUDA, "PUDA CREATE CQ", wqe,
+			IRDMA_CQP_WQE_SIZE * 8);
+	irdma_sc_cqp_post_sq(dev->cqp);
+	status = dev->cqp_ops->poll_for_cqp_op_done(dev->cqp,
+						    IRDMA_CQP_OP_CREATE_CQ,
+						    &compl_info);
+
+	return status;
+}
+
+/**
+ * irdma_puda_cq_create - create cq for resource
+ * @rsrc: resource for which cq to create
+ */
+static enum irdma_status_code irdma_puda_cq_create(struct irdma_puda_rsrc *rsrc)
+{
+	struct irdma_sc_dev *dev = rsrc->dev;
+	struct irdma_sc_cq *cq = &rsrc->cq;
+	enum irdma_status_code ret = 0;
+	u32 cqsize;
+	struct irdma_dma_mem *mem;
+	struct irdma_cq_init_info info = {};
+	struct irdma_cq_uk_init_info *init_info = &info.cq_uk_init_info;
+
+	cq->vsi = rsrc->vsi;
+	cqsize = rsrc->cq_size * (sizeof(struct irdma_cqe));
+	rsrc->cqmem.size = ALIGN(cqsize + sizeof(struct irdma_cq_shadow_area),
+				 IRDMA_CQ0_ALIGNMENT);
+	rsrc->cqmem.va = dma_alloc_coherent(hw_to_dev(dev->hw),
+					    rsrc->cqmem.size, &rsrc->cqmem.pa,
+					    GFP_KERNEL);
+	if (!rsrc->cqmem.va)
+		return IRDMA_ERR_NO_MEMORY;
+
+	mem = &rsrc->cqmem;
+	info.dev = dev;
+	info.type = (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ) ?
+		    IRDMA_CQ_TYPE_ILQ : IRDMA_CQ_TYPE_IEQ;
+	info.shadow_read_threshold = rsrc->cq_size >> 2;
+	info.cq_base_pa = mem->pa;
+	info.shadow_area_pa = mem->pa + cqsize;
+	init_info->cq_base = mem->va;
+	init_info->shadow_area = (__le64 *)((u8 *)mem->va + cqsize);
+	init_info->cq_size = rsrc->cq_size;
+	init_info->cq_id = rsrc->cq_id;
+	info.ceqe_mask = true;
+	info.ceq_id_valid = true;
+	info.vsi = rsrc->vsi;
+
+	ret = dev->iw_priv_cq_ops->cq_init(cq, &info);
+	if (ret)
+		goto error;
+
+	if (rsrc->dev->ceq_valid)
+		ret = irdma_cqp_cq_create_cmd(dev, cq);
+	else
+		ret = irdma_puda_cq_wqe(dev, cq);
+error:
+	if (ret) {
+		dma_free_coherent(hw_to_dev(dev->hw), rsrc->cqmem.size,
+				  rsrc->cqmem.va, rsrc->cqmem.pa);
+		rsrc->cqmem.va = NULL;
+	}
+
+	return ret;
+}
+
+/**
+ * irdma_puda_free_qp - free qp for resource
+ * @rsrc: resource for which qp to free
+ */
+static void irdma_puda_free_qp(struct irdma_puda_rsrc *rsrc)
+{
+	enum irdma_status_code ret;
+	struct irdma_ccq_cqe_info compl_info;
+	struct irdma_sc_dev *dev = rsrc->dev;
+
+	if (rsrc->dev->ceq_valid) {
+		irdma_cqp_qp_destroy_cmd(dev, &rsrc->qp);
+		rsrc->dev->ws_remove(rsrc->qp.vsi, rsrc->qp.user_pri);
+		return;
+	}
+
+	ret = dev->iw_priv_qp_ops->qp_destroy(&rsrc->qp, 0, false, true, true);
+	if (ret)
+		dev_dbg(rfdev_to_dev(dev),
+			"PUDA: error puda qp destroy wqe, status = %d\n", ret);
+	if (!ret) {
+		ret = dev->cqp_ops->poll_for_cqp_op_done(dev->cqp,
+							 IRDMA_CQP_OP_DESTROY_QP,
+							 &compl_info);
+		if (ret)
+			dev_dbg(rfdev_to_dev(dev),
+				"PUDA: error puda qp destroy failed, status = %d\n",
+				ret);
+	}
+
+	rsrc->dev->ws_remove(rsrc->qp.vsi, rsrc->qp.user_pri);
+}
+
+/**
+ * irdma_puda_free_cq - free cq for resource
+ * @rsrc: resource for which cq to free
+ */
+static void irdma_puda_free_cq(struct irdma_puda_rsrc *rsrc)
+{
+	enum irdma_status_code ret;
+	struct irdma_ccq_cqe_info compl_info;
+	struct irdma_sc_dev *dev = rsrc->dev;
+
+	if (rsrc->dev->ceq_valid) {
+		irdma_cqp_cq_destroy_cmd(dev, &rsrc->cq);
+		return;
+	}
+
+	ret = dev->iw_priv_cq_ops->cq_destroy(&rsrc->cq, 0, true);
+	if (ret)
+		dev_dbg(rfdev_to_dev(dev), "PUDA: error ieq cq destroy\n");
+	if (!ret) {
+		ret = dev->cqp_ops->poll_for_cqp_op_done(dev->cqp,
+							 IRDMA_CQP_OP_DESTROY_CQ,
+							 &compl_info);
+		if (ret)
+			dev_dbg(rfdev_to_dev(dev),
+				"PUDA: error ieq qp destroy done\n");
+	}
+}
+
+/**
+ * irdma_puda_dele_rsrc - delete all resources during close
+ * @vsi: VSI structure of device
+ * @type: type of resource to dele
+ * @reset: true if reset chip
+ */
+void irdma_puda_dele_rsrc(struct irdma_sc_vsi *vsi, enum puda_rsrc_type type,
+			  bool reset)
+{
+	struct irdma_sc_dev *dev = vsi->dev;
+	struct irdma_puda_rsrc *rsrc;
+	struct irdma_puda_buf *buf = NULL;
+	struct irdma_puda_buf *nextbuf = NULL;
+	struct irdma_virt_mem *vmem;
+	struct irdma_sc_ceq *ceq;
+
+	ceq = vsi->dev->ceq[0];
+	switch (type) {
+	case IRDMA_PUDA_RSRC_TYPE_ILQ:
+		rsrc = vsi->ilq;
+		vmem = &vsi->ilq_mem;
+		vsi->ilq = NULL;
+		if (ceq && ceq->reg_cq)
+			irdma_sc_remove_cq_ctx(ceq, &rsrc->cq);
+		break;
+	case IRDMA_PUDA_RSRC_TYPE_IEQ:
+		rsrc = vsi->ieq;
+		vmem = &vsi->ieq_mem;
+		vsi->ieq = NULL;
+		if (ceq && ceq->reg_cq)
+			irdma_sc_remove_cq_ctx(ceq, &rsrc->cq);
+		break;
+	default:
+		dev_dbg(rfdev_to_dev(dev),
+			"PUDA: error resource type = 0x%x\n", type);
+		return;
+	}
+
+	switch (rsrc->cmpl) {
+	case PUDA_HASH_CRC_COMPLETE:
+		irdma_free_hash_desc(rsrc->hash_desc);
+		/* fallthrough */
+	case PUDA_QP_CREATED:
+		if (!reset)
+			irdma_puda_free_qp(rsrc);
+		else
+			irdma_qp_rem_qos(&rsrc->qp);
+
+		dma_free_coherent(hw_to_dev(dev->hw), rsrc->qpmem.size,
+				  rsrc->qpmem.va, rsrc->qpmem.pa);
+		rsrc->qpmem.va = NULL;
+		/* fallthrough */
+	case PUDA_CQ_CREATED:
+		if (!reset)
+			irdma_puda_free_cq(rsrc);
+
+		dma_free_coherent(hw_to_dev(dev->hw), rsrc->cqmem.size,
+				  rsrc->cqmem.va, rsrc->cqmem.pa);
+		rsrc->cqmem.va = NULL;
+		break;
+	default:
+		dev_dbg(rfdev_to_dev(rsrc->dev), "PUDA: error no resources\n");
+		break;
+	}
+	/* Free all allocated puda buffers for both tx and rx */
+	buf = rsrc->alloclist;
+	while (buf) {
+		nextbuf = buf->next;
+		irdma_puda_dele_buf(dev, buf);
+		buf = nextbuf;
+		rsrc->alloc_buf_count--;
+	}
+
+	kfree(vmem->va);
+}
+
+/**
+ * irdma_puda_allocbufs - allocate buffers for resource
+ * @rsrc: resource for buffer allocation
+ * @count: number of buffers to create
+ */
+static enum irdma_status_code irdma_puda_allocbufs(struct irdma_puda_rsrc *rsrc,
+						   u32 count)
+{
+	u32 i;
+	struct irdma_puda_buf *buf;
+	struct irdma_puda_buf *nextbuf;
+
+	for (i = 0; i < count; i++) {
+		buf = irdma_puda_alloc_buf(rsrc->dev, rsrc->buf_size);
+		if (!buf) {
+			rsrc->stats_buf_alloc_fail++;
+			return IRDMA_ERR_NO_MEMORY;
+		}
+		irdma_puda_ret_bufpool(rsrc, buf);
+		rsrc->alloc_buf_count++;
+		if (!rsrc->alloclist) {
+			rsrc->alloclist = buf;
+		} else {
+			nextbuf = rsrc->alloclist;
+			rsrc->alloclist = buf;
+			buf->next = nextbuf;
+		}
+	}
+
+	rsrc->avail_buf_count = rsrc->alloc_buf_count;
+
+	return 0;
+}
+
+/**
+ * irdma_puda_create_rsrc - create resource (ilq or ieq)
+ * @vsi: sc VSI struct
+ * @info: resource information
+ */
+enum irdma_status_code irdma_puda_create_rsrc(struct irdma_sc_vsi *vsi,
+					      struct irdma_puda_rsrc_info *info)
+{
+	struct irdma_sc_dev *dev = vsi->dev;
+	enum irdma_status_code ret = 0;
+	struct irdma_puda_rsrc *rsrc;
+	u32 pudasize;
+	u32 sqwridsize, rqwridsize;
+	struct irdma_virt_mem *vmem;
+	struct irdma_sc_ceq *ceq;
+
+	info->count = 1;
+	pudasize = sizeof(struct irdma_puda_rsrc);
+	sqwridsize = info->sq_size * sizeof(struct irdma_sq_uk_wr_trk_info);
+	rqwridsize = info->rq_size * 8;
+	switch (info->type) {
+	case IRDMA_PUDA_RSRC_TYPE_ILQ:
+		vmem = &vsi->ilq_mem;
+		break;
+	case IRDMA_PUDA_RSRC_TYPE_IEQ:
+		vmem = &vsi->ieq_mem;
+		break;
+	default:
+		return IRDMA_NOT_SUPPORTED;
+	}
+	vmem->size = pudasize + sqwridsize + rqwridsize;
+	vmem->va = kzalloc(vmem->size, GFP_ATOMIC);
+	if (!vmem->va)
+		return IRDMA_ERR_NO_MEMORY;
+
+	rsrc = vmem->va;
+	spin_lock_init(&rsrc->bufpool_lock);
+	switch (info->type) {
+	case IRDMA_PUDA_RSRC_TYPE_ILQ:
+		vsi->ilq = vmem->va;
+		vsi->ilq_count = info->count;
+		rsrc->receive = info->receive;
+		rsrc->xmit_complete = info->xmit_complete;
+		break;
+	case IRDMA_PUDA_RSRC_TYPE_IEQ:
+		vsi->ieq_count = info->count;
+		vsi->ieq = vmem->va;
+		rsrc->receive = irdma_ieq_receive;
+		rsrc->xmit_complete = irdma_ieq_tx_compl;
+		break;
+	default:
+		return IRDMA_NOT_SUPPORTED;
+	}
+
+	rsrc->type = info->type;
+	rsrc->sq_wrtrk_array = (struct irdma_sq_uk_wr_trk_info *)
+			       ((u8 *)vmem->va + pudasize);
+	rsrc->rq_wrid_array = (u64 *)((u8 *)vmem->va + pudasize + sqwridsize);
+	/* Initialize all ieq lists */
+	INIT_LIST_HEAD(&rsrc->bufpool);
+	INIT_LIST_HEAD(&rsrc->txpend);
+
+	rsrc->tx_wqe_avail_cnt = info->sq_size - 1;
+	dev->iw_pd_ops->pd_init(dev, &rsrc->sc_pd, info->pd_id, -1);
+	rsrc->qp_id = info->qp_id;
+	rsrc->cq_id = info->cq_id;
+	rsrc->sq_size = info->sq_size;
+	rsrc->rq_size = info->rq_size;
+	rsrc->cq_size = info->rq_size + info->sq_size;
+	if (dev->hw_attrs.uk_attrs.hw_rev > IRDMA_GEN_1) {
+		if (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ)
+			rsrc->cq_size += info->rq_size;
+	}
+	rsrc->buf_size = info->buf_size;
+	rsrc->dev = dev;
+	rsrc->vsi = vsi;
+	rsrc->stats_idx = info->stats_idx;
+	rsrc->stats_idx_valid = info->stats_idx_valid;
+
+	ret = irdma_puda_cq_create(rsrc);
+	if (!ret) {
+		rsrc->cmpl = PUDA_CQ_CREATED;
+		ret = irdma_puda_qp_create(rsrc);
+	}
+	if (ret) {
+		dev_dbg(rfdev_to_dev(dev),
+			"PUDA: error qp_create type=%d, status=%d\n",
+			rsrc->type, ret);
+		goto error;
+	}
+	rsrc->cmpl = PUDA_QP_CREATED;
+
+	ceq = vsi->dev->ceq[0];
+	if (ceq->reg_cq)
+		ret = irdma_sc_add_cq_ctx(ceq, &rsrc->cq);
+
+	if (ret) {
+		dev_dbg(rfdev_to_dev(dev),
+			"PUDA: error unable to add to cq_ctx\n");
+		goto error;
+	}
+
+	ret = irdma_puda_allocbufs(rsrc, info->tx_buf_cnt + info->rq_size);
+	if (ret) {
+		dev_dbg(rfdev_to_dev(dev), "PUDA: error alloc_buf\n");
+		goto error;
+	}
+
+	rsrc->rxq_invalid_cnt = info->rq_size;
+	ret = irdma_puda_replenish_rq(rsrc, true);
+	if (ret)
+		goto error;
+
+	if (info->type == IRDMA_PUDA_RSRC_TYPE_IEQ) {
+		if (!irdma_init_hash_desc(&rsrc->hash_desc)) {
+			rsrc->check_crc = true;
+			rsrc->cmpl = PUDA_HASH_CRC_COMPLETE;
+			ret = 0;
+		}
+	}
+
+	dev->ccq_ops->ccq_arm(&rsrc->cq);
+	return ret;
+
+error:
+	irdma_puda_dele_rsrc(vsi, info->type, false);
+
+	return ret;
+}
+
+/**
+ * irdma_ilq_putback_rcvbuf - ilq buffer to put back on rq
+ * @qp: ilq's qp resource
+ * @wqe_idx:  wqe index of completed rcvbuf
+ */
+static void irdma_ilq_putback_rcvbuf(struct irdma_sc_qp *qp, u32 wqe_idx)
+{
+	__le64 *wqe;
+	u64 offset8, offset24;
+
+	wqe = qp->qp_uk.rq_base[wqe_idx].elem;
+	get_64bit_val(wqe, 24, &offset24);
+	if (qp->dev->hw_attrs.uk_attrs.hw_rev > IRDMA_GEN_1) {
+		get_64bit_val(wqe, 8, &offset8);
+		if (offset24)
+			offset8 &= ~LS_64(1, IRDMAQPSQ_VALID);
+		else
+			offset8 |= LS_64(1, IRDMAQPSQ_VALID);
+		set_64bit_val(wqe, 8, offset8);
+		dma_wmb(); /* make sure WQE is written before valid bit is set */
+	}
+	if (offset24)
+		offset24 = 0;
+	else
+		offset24 = LS_64(1, IRDMAQPSQ_VALID);
+
+	set_64bit_val(wqe, 24, offset24);
+}
+
+/**
+ * irdma_ieq_get_fpdu_len - given length return fpdu length
+ * @len: length of fpdu
+ */
+static u16 irdma_ieq_get_fpdu_len(u16 len)
+{
+	u16 fpdu_len;
+
+	fpdu_len = len + IRDMA_IEQ_MPA_FRAMING;
+	fpdu_len = (fpdu_len + 3) & 0xfffffffc;
+
+	return fpdu_len;
+}
+
+/**
+ * irdma_ieq_copy_to_txbuf - copydata from rcv buf to tx buf
+ * @buf: rcv buffer with partial
+ * @txbuf: tx buffer for sending back
+ * @buf_offset: rcv buffer offset to copy from
+ * @txbuf_offset: at offset in tx buf to copy
+ * @len: length of data to copy
+ */
+static void irdma_ieq_copy_to_txbuf(struct irdma_puda_buf *buf,
+				    struct irdma_puda_buf *txbuf,
+				    u16 buf_offset, u32 txbuf_offset, u32 len)
+{
+	void *mem1 = (u8 *)buf->mem.va + buf_offset;
+	void *mem2 = (u8 *)txbuf->mem.va + txbuf_offset;
+
+	memcpy(mem2, mem1, len);
+}
+
+/**
+ * irdma_ieq_setup_tx_buf - setup tx buffer for partial handling
+ * @buf: reeive buffer with partial
+ * @txbuf: buffer to prepare
+ */
+static void irdma_ieq_setup_tx_buf(struct irdma_puda_buf *buf,
+				   struct irdma_puda_buf *txbuf)
+{
+	txbuf->tcphlen = buf->tcphlen;
+	txbuf->ipv4 = buf->ipv4;
+
+	if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev > IRDMA_GEN_1) {
+		txbuf->hdrlen = txbuf->tcphlen;
+		irdma_ieq_copy_to_txbuf(buf, txbuf, IRDMA_TCP_OFFSET, 0,
+					txbuf->hdrlen);
+	} else {
+		txbuf->maclen = buf->maclen;
+		txbuf->hdrlen = buf->hdrlen;
+		irdma_ieq_copy_to_txbuf(buf, txbuf, 0, 0, buf->hdrlen);
+	}
+}
+
+/**
+ * irdma_ieq_check_first_buf - check if rcv buffer's seq is in range
+ * @buf: receive exception buffer
+ * @fps: first partial sequence number
+ */
+static void irdma_ieq_check_first_buf(struct irdma_puda_buf *buf, u32 fps)
+{
+	u32 offset;
+
+	if (buf->seqnum < fps) {
+		offset = fps - buf->seqnum;
+		if (offset > buf->datalen)
+			return;
+		buf->data += offset;
+		buf->datalen -= (u16)offset;
+		buf->seqnum = fps;
+	}
+}
+
+/**
+ * irdma_ieq_compl_pfpdu - write txbuf with full fpdu
+ * @ieq: ieq resource
+ * @rxlist: ieq's received buffer list
+ * @pbufl: temporary list for buffers for fpddu
+ * @txbuf: tx buffer for fpdu
+ * @fpdu_len: total length of fpdu
+ */
+static void irdma_ieq_compl_pfpdu(struct irdma_puda_rsrc *ieq,
+				  struct list_head *rxlist,
+				  struct list_head *pbufl,
+				  struct irdma_puda_buf *txbuf, u16 fpdu_len)
+{
+	struct irdma_puda_buf *buf;
+	u32 nextseqnum;
+	u16 txoffset, bufoffset;
+
+	buf = irdma_puda_get_listbuf(pbufl);
+	if (!buf)
+		return;
+
+	nextseqnum = buf->seqnum + fpdu_len;
+	irdma_ieq_setup_tx_buf(buf, txbuf);
+	if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev > IRDMA_GEN_1) {
+		txoffset = txbuf->hdrlen;
+		txbuf->totallen = txbuf->hdrlen + fpdu_len;
+		txbuf->data = (u8 *)txbuf->mem.va + txoffset;
+	} else {
+		txoffset = buf->hdrlen;
+		txbuf->totallen = buf->hdrlen + fpdu_len;
+		txbuf->data = (u8 *)txbuf->mem.va + buf->hdrlen;
+	}
+	bufoffset = (u16)(buf->data - (u8 *)buf->mem.va);
+
+	do {
+		if (buf->datalen >= fpdu_len) {
+			/* copied full fpdu */
+			irdma_ieq_copy_to_txbuf(buf, txbuf, bufoffset, txoffset,
+						fpdu_len);
+			buf->datalen -= fpdu_len;
+			buf->data += fpdu_len;
+			buf->seqnum = nextseqnum;
+			break;
+		}
+		/* copy partial fpdu */
+		irdma_ieq_copy_to_txbuf(buf, txbuf, bufoffset, txoffset,
+					buf->datalen);
+		txoffset += buf->datalen;
+		fpdu_len -= buf->datalen;
+		irdma_puda_ret_bufpool(ieq, buf);
+		buf = irdma_puda_get_listbuf(pbufl);
+		if (!buf)
+			return;
+
+		bufoffset = (u16)(buf->data - (u8 *)buf->mem.va);
+	} while (1);
+
+	/* last buffer on the list*/
+	if (buf->datalen)
+		list_add(&buf->list, rxlist);
+	else
+		irdma_puda_ret_bufpool(ieq, buf);
+}
+
+/**
+ * irdma_ieq_create_pbufl - create buffer list for single fpdu
+ * @pfpdu: pointer to fpdu
+ * @rxlist: resource list for receive ieq buffes
+ * @pbufl: temp. list for buffers for fpddu
+ * @buf: first receive buffer
+ * @fpdu_len: total length of fpdu
+ */
+static enum irdma_status_code
+irdma_ieq_create_pbufl(struct irdma_pfpdu *pfpdu, struct list_head *rxlist,
+		       struct list_head *pbufl, struct irdma_puda_buf *buf,
+		       u16 fpdu_len)
+{
+	enum irdma_status_code status = 0;
+	struct irdma_puda_buf *nextbuf;
+	u32 nextseqnum;
+	u16 plen = fpdu_len - buf->datalen;
+	bool done = false;
+
+	nextseqnum = buf->seqnum + buf->datalen;
+	do {
+		nextbuf = irdma_puda_get_listbuf(rxlist);
+		if (!nextbuf) {
+			status = IRDMA_ERR_list_empty;
+			break;
+		}
+		list_add_tail(&nextbuf->list, pbufl);
+		if (nextbuf->seqnum != nextseqnum) {
+			pfpdu->bad_seq_num++;
+			status = IRDMA_ERR_SEQ_NUM;
+			break;
+		}
+		if (nextbuf->datalen >= plen) {
+			done = true;
+		} else {
+			plen -= nextbuf->datalen;
+			nextseqnum = nextbuf->seqnum + nextbuf->datalen;
+		}
+
+	} while (!done);
+
+	return status;
+}
+
+/**
+ * irdma_ieq_handle_partial - process partial fpdu buffer
+ * @ieq: ieq resource
+ * @pfpdu: partial management per user qp
+ * @buf: receive buffer
+ * @fpdu_len: fpdu len in the buffer
+ */
+static enum irdma_status_code
+irdma_ieq_handle_partial(struct irdma_puda_rsrc *ieq, struct irdma_pfpdu *pfpdu,
+			 struct irdma_puda_buf *buf, u16 fpdu_len)
+{
+	enum irdma_status_code status = 0;
+	u8 *crcptr;
+	u32 mpacrc;
+	u32 seqnum = buf->seqnum;
+	struct list_head pbufl; /* partial buffer list */
+	struct irdma_puda_buf *txbuf = NULL;
+	struct list_head *rxlist = &pfpdu->rxlist;
+
+	INIT_LIST_HEAD(&pbufl);
+	list_add(&buf->list, &pbufl);
+
+	status = irdma_ieq_create_pbufl(pfpdu, rxlist, &pbufl, buf, fpdu_len);
+	if (status)
+		goto error;
+
+	txbuf = irdma_puda_get_bufpool(ieq);
+	if (!txbuf) {
+		pfpdu->no_tx_bufs++;
+		status = IRDMA_ERR_NO_TXBUFS;
+		goto error;
+	}
+
+	irdma_ieq_compl_pfpdu(ieq, rxlist, &pbufl, txbuf, fpdu_len);
+	irdma_ieq_update_tcpip_info(txbuf, fpdu_len, seqnum);
+
+	crcptr = txbuf->data + fpdu_len - 4;
+	mpacrc = *(u32 *)crcptr;
+	if (ieq->check_crc) {
+		status = irdma_ieq_check_mpacrc(ieq->hash_desc, txbuf->data,
+						(fpdu_len - 4), mpacrc);
+		if (status) {
+			dev_dbg(rfdev_to_dev(ieq->dev),
+				"IEQ: error bad crc\n");
+			goto error;
+		}
+	}
+
+	irdma_debug_buf(ieq->dev, IRDMA_DEBUG_IEQ, "IEQ TX BUFFER",
+			txbuf->mem.va, txbuf->totallen);
+	if (ieq->dev->hw_attrs.uk_attrs.hw_rev > IRDMA_GEN_1)
+		txbuf->ah_id = pfpdu->ah->ah_info.ah_idx;
+	txbuf->do_lpb = true;
+	irdma_puda_send_buf(ieq, txbuf);
+	pfpdu->rcv_nxt = seqnum + fpdu_len;
+	return status;
+
+error:
+	while (!list_empty(&pbufl)) {
+		buf = (struct irdma_puda_buf *)(pbufl.prev);
+		list_del(&buf->list);
+		list_add(&buf->list, rxlist);
+	}
+	if (txbuf)
+		irdma_puda_ret_bufpool(ieq, txbuf);
+
+	return status;
+}
+
+/**
+ * irdma_ieq_process_buf - process buffer rcvd for ieq
+ * @ieq: ieq resource
+ * @pfpdu: partial management per user qp
+ * @buf: receive buffer
+ */
+static enum irdma_status_code irdma_ieq_process_buf(struct irdma_puda_rsrc *ieq,
+						    struct irdma_pfpdu *pfpdu,
+						    struct irdma_puda_buf *buf)
+{
+	u16 fpdu_len = 0;
+	u16 datalen = buf->datalen;
+	u8 *datap = buf->data;
+	u8 *crcptr;
+	u16 ioffset = 0;
+	u32 mpacrc;
+	u32 seqnum = buf->seqnum;
+	u16 len = 0;
+	u16 full = 0;
+	bool partial = false;
+	struct irdma_puda_buf *txbuf;
+	struct list_head *rxlist = &pfpdu->rxlist;
+	enum irdma_status_code ret = 0;
+
+	ioffset = (u16)(buf->data - (u8 *)buf->mem.va);
+	while (datalen) {
+		fpdu_len = irdma_ieq_get_fpdu_len(ntohs(*(__be16 *)datap));
+		if (fpdu_len > pfpdu->max_fpdu_data) {
+			dev_dbg(rfdev_to_dev(ieq->dev),
+				"IEQ: error bad fpdu_len\n");
+			list_add(&buf->list, rxlist);
+			return IRDMA_ERR_MPA_CRC;
+		}
+
+		if (datalen < fpdu_len) {
+			partial = true;
+			break;
+		}
+		crcptr = datap + fpdu_len - 4;
+		mpacrc = *(u32 *)crcptr;
+		if (ieq->check_crc)
+			ret = irdma_ieq_check_mpacrc(ieq->hash_desc, datap,
+						     fpdu_len - 4, mpacrc);
+		if (ret) {
+			list_add(&buf->list, rxlist);
+			dev_dbg(rfdev_to_dev(ieq->dev),
+				"ERR: IRDMA_ERR_MPA_CRC\n");
+			return IRDMA_ERR_MPA_CRC;
+		}
+		full++;
+		pfpdu->fpdu_processed++;
+		datap += fpdu_len;
+		len += fpdu_len;
+		datalen -= fpdu_len;
+	}
+	if (full) {
+		/* copy full pdu's in the txbuf and send them out */
+		txbuf = irdma_puda_get_bufpool(ieq);
+		if (!txbuf) {
+			pfpdu->no_tx_bufs++;
+			list_add(&buf->list, rxlist);
+			return IRDMA_ERR_NO_TXBUFS;
+		}
+		/* modify txbuf's buffer header */
+		irdma_ieq_setup_tx_buf(buf, txbuf);
+		/* copy full fpdu's to new buffer */
+		if (ieq->dev->hw_attrs.uk_attrs.hw_rev > IRDMA_GEN_1) {
+			irdma_ieq_copy_to_txbuf(buf, txbuf, ioffset,
+						txbuf->hdrlen, len);
+			txbuf->totallen = txbuf->hdrlen + len;
+			txbuf->ah_id = pfpdu->ah->ah_info.ah_idx;
+		} else {
+			irdma_ieq_copy_to_txbuf(buf, txbuf, ioffset,
+						buf->hdrlen, len);
+			txbuf->totallen = buf->hdrlen + len;
+		}
+		irdma_ieq_update_tcpip_info(txbuf, len, buf->seqnum);
+		irdma_debug_buf(ieq->dev, IRDMA_DEBUG_IEQ, "IEQ TX BUFFER",
+				txbuf->mem.va, txbuf->totallen);
+		txbuf->do_lpb = true;
+		irdma_puda_send_buf(ieq, txbuf);
+
+		if (!datalen) {
+			pfpdu->rcv_nxt = buf->seqnum + len;
+			irdma_puda_ret_bufpool(ieq, buf);
+			return 0;
+		}
+		buf->data = datap;
+		buf->seqnum = seqnum + len;
+		buf->datalen = datalen;
+		pfpdu->rcv_nxt = buf->seqnum;
+	}
+	if (partial)
+		return irdma_ieq_handle_partial(ieq, pfpdu, buf, fpdu_len);
+
+	return 0;
+}
+
+/**
+ * irdma_ieq_process_fpdus - process fpdu's buffers on its list
+ * @qp: qp for which partial fpdus
+ * @ieq: ieq resource
+ */
+void irdma_ieq_process_fpdus(struct irdma_sc_qp *qp,
+			     struct irdma_puda_rsrc *ieq)
+{
+	struct irdma_pfpdu *pfpdu = &qp->pfpdu;
+	struct list_head *rxlist = &pfpdu->rxlist;
+	struct irdma_puda_buf *buf;
+	enum irdma_status_code status;
+
+	do {
+		if (list_empty(rxlist))
+			break;
+		buf = irdma_puda_get_listbuf(rxlist);
+		if (!buf) {
+			dev_dbg(rfdev_to_dev(ieq->dev), "IEQ: error no buf\n");
+			break;
+		}
+		if (buf->seqnum != pfpdu->rcv_nxt) {
+			/* This could be out of order or missing packet */
+			pfpdu->out_of_order++;
+			list_add(&buf->list, rxlist);
+			break;
+		}
+		/* keep processing buffers from the head of the list */
+		status = irdma_ieq_process_buf(ieq, pfpdu, buf);
+		if (status == IRDMA_ERR_MPA_CRC) {
+			pfpdu->mpa_crc_err = true;
+			while (!list_empty(rxlist)) {
+				buf = irdma_puda_get_listbuf(rxlist);
+				irdma_puda_ret_bufpool(ieq, buf);
+				pfpdu->crc_err++;
+			}
+			/* create CQP for AE */
+			irdma_ieq_mpa_crc_ae(ieq->dev, qp);
+		}
+	} while (!status);
+}
+
+/**
+ * irdma_ieq_create_ah - create an address handle for IEQ
+ * @qp: qp pointer
+ * @buf: buf received on IEQ used to create AH
+ */
+static enum irdma_status_code irdma_ieq_create_ah(struct irdma_sc_qp *qp,
+						  struct irdma_puda_buf *buf)
+{
+	struct irdma_ah_info ah_info = {};
+
+	qp->pfpdu.ah_buf = buf;
+	irdma_puda_ieq_get_ah_info(qp, &ah_info);
+	return irdma_puda_create_ah(qp->vsi->dev, &ah_info, false,
+				    IRDMA_PUDA_RSRC_TYPE_IEQ, qp,
+				    &qp->pfpdu.ah);
+}
+
+/**
+ * irdma_ieq_handle_exception - handle qp's exception
+ * @ieq: ieq resource
+ * @qp: qp receiving excpetion
+ * @buf: receive buffer
+ */
+static void irdma_ieq_handle_exception(struct irdma_puda_rsrc *ieq,
+				       struct irdma_sc_qp *qp,
+				       struct irdma_puda_buf *buf)
+{
+	struct irdma_pfpdu *pfpdu = &qp->pfpdu;
+	u32 *hw_host_ctx = (u32 *)qp->hw_host_ctx;
+	u32 rcv_wnd = hw_host_ctx[23];
+	/* first partial seq # in q2 */
+	u32 fps = *(u32 *)(qp->q2_buf + Q2_FPSN_OFFSET);
+	struct list_head *rxlist = &pfpdu->rxlist;
+	unsigned long flags = 0;
+	u8 hw_rev = qp->dev->hw_attrs.uk_attrs.hw_rev;
+
+	irdma_debug_buf(ieq->dev, IRDMA_DEBUG_IEQ, "IEQ RX BUFFER", buf->mem.va,
+			buf->totallen);
+
+	spin_lock_irqsave(&pfpdu->lock, flags);
+	pfpdu->total_ieq_bufs++;
+	if (pfpdu->mpa_crc_err) {
+		pfpdu->crc_err++;
+		goto error;
+	}
+	if (pfpdu->mode && fps != pfpdu->fps) {
+		/* clean up qp as it is new partial sequence */
+		irdma_ieq_cleanup_qp(ieq, qp);
+		dev_dbg(rfdev_to_dev(ieq->dev),
+			"IEQ: restarting new partial\n");
+		pfpdu->mode = false;
+	}
+
+	if (!pfpdu->mode) {
+		irdma_debug_buf(ieq->dev, IRDMA_DEBUG_IEQ, "Q2 BUFFER",
+				(u64 *)qp->q2_buf, 128);
+		/* First_Partial_Sequence_Number check */
+		pfpdu->rcv_nxt = fps;
+		pfpdu->fps = fps;
+		pfpdu->mode = true;
+		pfpdu->max_fpdu_data = (buf->ipv4) ?
+				       (ieq->vsi->mtu - IRDMA_MTU_TO_MSS_IPV4) :
+				       (ieq->vsi->mtu - IRDMA_MTU_TO_MSS_IPV6);
+		pfpdu->pmode_count++;
+		INIT_LIST_HEAD(rxlist);
+		irdma_ieq_check_first_buf(buf, fps);
+	}
+
+	if (!(rcv_wnd >= (buf->seqnum - pfpdu->rcv_nxt))) {
+		pfpdu->bad_seq_num++;
+		goto error;
+	}
+
+	if (!list_empty(rxlist)) {
+		if (buf->seqnum != pfpdu->nextseqnum) {
+			irdma_send_ieq_ack(qp);
+			/* throw away out-of-order, duplicates*/
+			goto error;
+		}
+	}
+	/* Insert buf before head */
+	list_add_tail(&buf->list, rxlist);
+	pfpdu->nextseqnum = buf->seqnum + buf->datalen;
+	pfpdu->lastrcv_buf = buf;
+	if (hw_rev > IRDMA_GEN_1 && !pfpdu->ah) {
+		irdma_ieq_create_ah(qp, buf);
+		if (!pfpdu->ah)
+			goto error;
+		goto exit;
+	}
+	if (hw_rev == IRDMA_GEN_1)
+		irdma_ieq_process_fpdus(qp, ieq);
+	else if (pfpdu->ah && pfpdu->ah->ah_info.ah_valid)
+		irdma_ieq_process_fpdus(qp, ieq);
+exit:
+	spin_unlock_irqrestore(&pfpdu->lock, flags);
+	return;
+
+error:
+	irdma_puda_ret_bufpool(ieq, buf);
+	spin_unlock_irqrestore(&pfpdu->lock, flags);
+}
+
+/**
+ * irdma_ieq_receive - received exception buffer
+ * @vsi: VSI of device
+ * @buf: exception buffer received
+ */
+static void irdma_ieq_receive(struct irdma_sc_vsi *vsi,
+			      struct irdma_puda_buf *buf)
+{
+	struct irdma_puda_rsrc *ieq = vsi->ieq;
+	struct irdma_sc_qp *qp = NULL;
+	u32 wqe_idx = ieq->compl_rxwqe_idx;
+
+	qp = irdma_ieq_get_qp(vsi->dev, buf);
+	if (!qp) {
+		ieq->stats_bad_qp_id++;
+		irdma_puda_ret_bufpool(ieq, buf);
+	} else {
+		irdma_ieq_handle_exception(ieq, qp, buf);
+	}
+	/*
+	 * ieq->rx_wqe_idx is used by irdma_puda_replenish_rq()
+	 * on which wqe_idx to start replenish rq
+	 */
+	if (!ieq->rxq_invalid_cnt)
+		ieq->rx_wqe_idx = wqe_idx;
+	ieq->rxq_invalid_cnt++;
+}
+
+/**
+ * irdma_ieq_tx_compl - put back after sending completed exception buffer
+ * @vsi: sc VSI struct
+ * @sqwrid: pointer to puda buffer
+ */
+static void irdma_ieq_tx_compl(struct irdma_sc_vsi *vsi, void *sqwrid)
+{
+	struct irdma_puda_rsrc *ieq = vsi->ieq;
+	struct irdma_puda_buf *buf = sqwrid;
+
+	irdma_puda_ret_bufpool(ieq, buf);
+}
+
+/**
+ * irdma_ieq_cleanup_qp - qp is being destroyed
+ * @ieq: ieq resource
+ * @qp: all pending fpdu buffers
+ */
+void irdma_ieq_cleanup_qp(struct irdma_puda_rsrc *ieq, struct irdma_sc_qp *qp)
+{
+	struct irdma_puda_buf *buf;
+	struct irdma_pfpdu *pfpdu = &qp->pfpdu;
+	struct list_head *rxlist = &pfpdu->rxlist;
+
+	if (!pfpdu->mode)
+		return;
+
+	while (!list_empty(rxlist)) {
+		buf = irdma_puda_get_listbuf(rxlist);
+		irdma_puda_ret_bufpool(ieq, buf);
+	}
+
+	if (qp->pfpdu.ah) {
+		irdma_puda_free_ah(ieq->dev, qp->pfpdu.ah);
+		qp->pfpdu.ah = NULL;
+		qp->pfpdu.ah_buf = NULL;
+	}
+}
drivers/infiniband/hw/irdma/puda.h new file +186 -0

186 lines not on this page · raw message

Signature · 1 line
2.23.0
#36 lore raw

PATCH v3 10/20 RDMA/irdma: Add QoS definitions

In reply to Jeff Kirsher

From: Mustafa Ismail <[email protected]>

Add definitions for managing the RDMA HW work scheduler (WS) tree.

A WS node is created via a control QP operation with the bandwidth allocation, arbitration scheme, and traffic class of the QP specified. The Qset handle returned associates the QoS parameters for the QP. The Qset is registered with the LAN and a equivalent node is created in the LAN packet scheduler tree.

---
 drivers/infiniband/hw/irdma/ws.c | 395 +++++++++++++++++++++++++++++++
 drivers/infiniband/hw/irdma/ws.h |  39 +++
 2 files changed, 434 insertions(+)
 create mode 100644 drivers/infiniband/hw/irdma/ws.c
 create mode 100644 drivers/infiniband/hw/irdma/ws.h
drivers/infiniband/hw/irdma/ws.c new file +395 -0
new file mode 100644
index 000000000000..4828d7b0aee6
@@ -0,0 +1,395 @@
+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
+/* Copyright (c) 2019 Intel Corporation */
+#include "osdep.h"
+#include "status.h"
+#include "hmc.h"
+#include "defs.h"
+#include "type.h"
+#include "protos.h"
+
+#include "ws.h"
+
+/**
+ * irdma_alloc_node - Allocate a WS node and init
+ * @vsi: vsi pointer
+ * @user_pri: user priority
+ * @node_type: Type of node, leaf or parent
+ * @parent: parent node pointer
+ */
+static struct irdma_ws_node *irdma_alloc_node(struct irdma_sc_vsi *vsi,
+					      u8 user_pri,
+					      enum irdma_ws_node_type node_type,
+					      struct irdma_ws_node *parent)
+{
+	struct irdma_virt_mem ws_mem;
+	struct irdma_ws_node *node;
+	u16 node_index = 0;
+
+	ws_mem.size = sizeof(struct irdma_ws_node);
+	ws_mem.va = kzalloc(ws_mem.size, GFP_ATOMIC);
+	if (!ws_mem.va)
+		return NULL;
+
+	if (parent || vsi->vm_vf_type == IRDMA_VF_TYPE) {
+		node_index = irdma_alloc_ws_node_id(vsi->dev);
+		if (node_index == IRDMA_WS_NODE_INVALID) {
+			kfree(ws_mem.va);
+			return NULL;
+		}
+	}
+
+	node = ws_mem.va;
+	node->index = node_index;
+	node->vsi_index = vsi->vsi_idx;
+	INIT_LIST_HEAD(&node->child_list_head);
+	if (node_type == WS_NODE_TYPE_LEAF) {
+		node->type_leaf = true;
+		node->traffic_class = vsi->qos[user_pri].traffic_class;
+		node->user_pri = user_pri;
+		node->rel_bw = vsi->qos[user_pri].rel_bw;
+		if (!node->rel_bw)
+			node->rel_bw = 1;
+
+		node->lan_qs_handle = vsi->qos[user_pri].lan_qos_handle;
+		node->prio_type = IRDMA_PRIO_WEIGHTED_RR;
+	} else {
+		node->rel_bw = 1;
+		node->prio_type = IRDMA_PRIO_WEIGHTED_RR;
+	}
+
+	node->parent = parent;
+
+	return node;
+}
+
+/**
+ * irdma_free_node - Free a WS node
+ * @vsi: VSI stricture of device
+ * @node: Pointer to node to free
+ */
+static void irdma_free_node(struct irdma_sc_vsi *vsi,
+			    struct irdma_ws_node *node)
+{
+	struct irdma_virt_mem ws_mem;
+
+	if (node->index)
+		irdma_free_ws_node_id(vsi->dev, node->index);
+
+	ws_mem.va = node;
+	ws_mem.size = sizeof(struct irdma_ws_node);
+	kfree(ws_mem.va);
+}
+
+/**
+ * irdma_ws_cqp_cmd - Post CQP work scheduler node cmd
+ * @vsi: vsi pointer
+ * @node: pointer to node
+ * @cmd: add, remove or modify
+ */
+static enum irdma_status_code
+irdma_ws_cqp_cmd(struct irdma_sc_vsi *vsi, struct irdma_ws_node *node, u8 cmd)
+{
+	struct irdma_ws_node_info node_info = {};
+
+	node_info.id = node->index;
+	node_info.vsi = node->vsi_index;
+	if (node->parent)
+		node_info.parent_id = node->parent->index;
+	else
+		node_info.parent_id = node_info.id;
+
+	node_info.weight = node->rel_bw;
+	node_info.tc = node->traffic_class;
+	node_info.prio_type = node->prio_type;
+	node_info.type_leaf = node->type_leaf;
+	node_info.enable = node->enable;
+	if (irdma_cqp_ws_node_cmd(vsi->dev, cmd, &node_info)) {
+		dev_dbg(rfdev_to_dev(vsi->dev), "WS: CQP WS CMD failed\n");
+		return IRDMA_ERR_NO_MEMORY;
+	}
+
+	if (node->type_leaf && cmd == IRDMA_OP_WS_ADD_NODE) {
+		node->qs_handle = node_info.qs_handle;
+		vsi->qos[node->user_pri].qs_handle = node_info.qs_handle;
+	}
+
+	return 0;
+}
+
+/**
+ * ws_find_node - Find SC WS node based on VSI id or TC
+ * @parent: parent node of First VSI or TC node
+ * @match_val: value to match
+ * @type: match type VSI/TC
+ */
+static struct irdma_ws_node *ws_find_node(struct irdma_ws_node *parent,
+					  u16 match_val,
+					  enum irdma_ws_match_type type)
+{
+	struct irdma_ws_node *node;
+
+	switch (type) {
+	case WS_MATCH_TYPE_VSI:
+		list_for_each_entry(node, &parent->child_list_head, siblings) {
+			if (node->vsi_index == match_val)
+				return node;
+		}
+		break;
+	case WS_MATCH_TYPE_TC:
+		list_for_each_entry(node, &parent->child_list_head, siblings) {
+			if (node->traffic_class == match_val)
+				return node;
+		}
+		break;
+	default:
+		break;
+	}
+
+	return NULL;
+}
+
+/**
+ * irdma_tc_in_use - Checks to see if a leaf node is in use
+ * @vsi: vsi pointer
+ * @user_pri: user priority
+ */
+static bool irdma_tc_in_use(struct irdma_sc_vsi *vsi, u8 user_pri)
+{
+	unsigned long flags;
+	int i;
+
+	spin_lock_irqsave(&vsi->qos[user_pri].lock, flags);
+	if (!list_empty(&vsi->qos[user_pri].qplist)) {
+		spin_unlock_irqrestore(&vsi->qos[user_pri].lock, flags);
+		return true;
+	}
+
+	/* Check if the traffic class associated with the given user priority
+	 * is in use by any other user priority. If so, nothing left to do
+	 */
+	for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
+		if (vsi->qos[i].traffic_class == vsi->qos[user_pri].traffic_class &&
+		    !list_empty(&vsi->qos[i].qplist)) {
+			spin_unlock_irqrestore(&vsi->qos[user_pri].lock, flags);
+			return true;
+		}
+	}
+	spin_unlock_irqrestore(&vsi->qos[user_pri].lock, flags);
+
+	return false;
+}
+
+/**
+ * irdma_remove_leaf - Remove leaf node unconditionally
+ * @vsi: vsi pointer
+ * @user_pri: user priority
+ */
+static void irdma_remove_leaf(struct irdma_sc_vsi *vsi, u8 user_pri)
+{
+	struct irdma_ws_node *ws_tree_root, *vsi_node, *tc_node;
+
+	ws_tree_root = vsi->dev->ws_tree_root;
+	if (!ws_tree_root)
+		return;
+
+	vsi_node = ws_find_node(ws_tree_root, vsi->vsi_idx,
+				WS_MATCH_TYPE_VSI);
+	if (!vsi_node)
+		return;
+
+	tc_node = ws_find_node(vsi_node,
+			       vsi->qos[user_pri].traffic_class,
+			       WS_MATCH_TYPE_TC);
+	if (!tc_node)
+		return;
+
+	irdma_ws_cqp_cmd(vsi, tc_node, IRDMA_OP_WS_DELETE_NODE);
+	vsi->unregister_qset(vsi, tc_node);
+	list_del(&tc_node->siblings);
+	irdma_free_node(vsi, tc_node);
+	/* Check if VSI node can be freed */
+	if (list_empty(&vsi_node->child_list_head)) {
+		irdma_ws_cqp_cmd(vsi, vsi_node, IRDMA_OP_WS_DELETE_NODE);
+		list_del(&vsi_node->siblings);
+		irdma_free_node(vsi, vsi_node);
+		/* Free head node there are no remaining VSI nodes */
+		if (list_empty(&ws_tree_root->child_list_head)) {
+			irdma_ws_cqp_cmd(vsi, ws_tree_root,
+					 IRDMA_OP_WS_DELETE_NODE);
+			irdma_free_node(vsi, ws_tree_root);
+			vsi->dev->ws_tree_root = NULL;
+		}
+	}
+}
+
+/**
+ * irdma_ws_add - Build work scheduler tree, set RDMA qs_handle
+ * @vsi: vsi pointer
+ * @user_pri: user priority
+ */
+enum irdma_status_code irdma_ws_add(struct irdma_sc_vsi *vsi, u8 user_pri)
+{
+	struct irdma_ws_node *ws_tree_root;
+	struct irdma_ws_node *vsi_node;
+	struct irdma_ws_node *tc_node;
+	u16 traffic_class;
+	enum irdma_status_code ret = 0;
+	int i;
+
+	mutex_lock(&vsi->dev->ws_mutex);
+	if (vsi->tc_change_pending) {
+		ret = IRDMA_ERR_NOT_READY;
+		goto exit;
+	}
+
+	ws_tree_root = vsi->dev->ws_tree_root;
+	if (!ws_tree_root) {
+		dev_dbg(rfdev_to_dev(vsi->dev), "WS: Creating root node\n");
+		ws_tree_root = irdma_alloc_node(vsi, user_pri,
+						WS_NODE_TYPE_PARENT, NULL);
+		if (!ws_tree_root) {
+			ret = IRDMA_ERR_NO_MEMORY;
+			goto exit;
+		}
+
+		ret = irdma_ws_cqp_cmd(vsi, ws_tree_root, IRDMA_OP_WS_ADD_NODE);
+		if (ret) {
+			irdma_free_node(vsi, ws_tree_root);
+			goto exit;
+		}
+
+		vsi->dev->ws_tree_root = ws_tree_root;
+	}
+
+	/* Find a second tier node that matches the VSI */
+	vsi_node = ws_find_node(ws_tree_root, vsi->vsi_idx,
+				WS_MATCH_TYPE_VSI);
+
+	/* If VSI node doesn't exist, add one */
+	if (!vsi_node) {
+		dev_dbg(rfdev_to_dev(vsi->dev),
+			"WS: Node not found matching VSI %d\n", vsi->vsi_idx);
+		vsi_node = irdma_alloc_node(vsi, user_pri, WS_NODE_TYPE_PARENT,
+					    ws_tree_root);
+		if (!vsi_node) {
+			ret = IRDMA_ERR_NO_MEMORY;
+			goto vsi_add_err;
+		}
+
+		ret = irdma_ws_cqp_cmd(vsi, vsi_node, IRDMA_OP_WS_ADD_NODE);
+		if (ret) {
+			irdma_free_node(vsi, vsi_node);
+			goto vsi_add_err;
+		}
+
+		list_add(&vsi_node->siblings, &ws_tree_root->child_list_head);
+	}
+
+	dev_dbg(rfdev_to_dev(vsi->dev),
+		"WS: Using node %d which represents VSI %d\n",
+		vsi_node->index, vsi->vsi_idx);
+	traffic_class = vsi->qos[user_pri].traffic_class;
+	tc_node = ws_find_node(vsi_node, traffic_class,
+			       WS_MATCH_TYPE_TC);
+	if (!tc_node) {
+		/* Add leaf node */
+		dev_dbg(rfdev_to_dev(vsi->dev),
+			"WS: Node not found matching VSI %d and TC %d\n",
+			vsi->vsi_idx, traffic_class);
+		tc_node = irdma_alloc_node(vsi, user_pri, WS_NODE_TYPE_LEAF,
+					   vsi_node);
+		if (!tc_node) {
+			ret = IRDMA_ERR_NO_MEMORY;
+			goto leaf_add_err;
+		}
+
+		ret = irdma_ws_cqp_cmd(vsi, tc_node, IRDMA_OP_WS_ADD_NODE);
+		if (ret) {
+			irdma_free_node(vsi, tc_node);
+			goto leaf_add_err;
+		}
+
+		list_add(&tc_node->siblings, &vsi_node->child_list_head);
+		/*
+		 * callback to LAN to update the LAN tree with our node
+		 */
+		ret = vsi->register_qset(vsi, tc_node);
+		if (ret)
+			goto reg_err;
+
+		tc_node->enable = true;
+		ret = irdma_ws_cqp_cmd(vsi, tc_node, IRDMA_OP_WS_MODIFY_NODE);
+		if (ret)
+			goto reg_err;
+	}
+	dev_dbg(rfdev_to_dev(vsi->dev),
+		"WS: Using node %d which represents VSI %d TC %d\n",
+		tc_node->index, vsi->vsi_idx, traffic_class);
+	/*
+	 * Iterate through other UPs and update the QS handle if they have
+	 * a matching traffic class.
+	 */
+	for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
+		if (vsi->qos[i].traffic_class == traffic_class) {
+			vsi->qos[i].qs_handle = tc_node->qs_handle;
+			vsi->qos[i].lan_qos_handle = tc_node->lan_qs_handle;
+			vsi->qos[i].l2_sched_node_id = tc_node->l2_sched_node_id;
+		}
+	}
+	goto exit;
+
+leaf_add_err:
+	if (list_empty(&vsi_node->child_list_head)) {
+		if (irdma_ws_cqp_cmd(vsi, vsi_node, IRDMA_OP_WS_DELETE_NODE))
+			goto exit;
+		list_del(&vsi_node->siblings);
+		irdma_free_node(vsi, vsi_node);
+	}
+
+vsi_add_err:
+	/* Free head node there are no remaining VSI nodes */
+	if (list_empty(&ws_tree_root->child_list_head)) {
+		irdma_ws_cqp_cmd(vsi, ws_tree_root, IRDMA_OP_WS_DELETE_NODE);
+		vsi->dev->ws_tree_root = NULL;
+		irdma_free_node(vsi, ws_tree_root);
+	}
+
+exit:
+	mutex_unlock(&vsi->dev->ws_mutex);
+	return ret;
+
+reg_err:
+	mutex_unlock(&vsi->dev->ws_mutex);
+	irdma_ws_remove(vsi, user_pri);
+	return ret;
+}
+
+/**
+ * irdma_ws_remove - Free WS scheduler node, update WS tree
+ * @vsi: vsi pointer
+ * @user_pri: user priority
+ */
+void irdma_ws_remove(struct irdma_sc_vsi *vsi, u8 user_pri)
+{
+	mutex_lock(&vsi->dev->ws_mutex);
+	if (irdma_tc_in_use(vsi, user_pri))
+		goto exit;
+
+	irdma_remove_leaf(vsi, user_pri);
+exit:
+	mutex_unlock(&vsi->dev->ws_mutex);
+}
+
+/**
+ * irdma_ws_reset - Reset entire WS tree
+ * @vsi: vsi pointer
+ */
+void irdma_ws_reset(struct irdma_sc_vsi *vsi)
+{
+	u8 i;
+
+	mutex_lock(&vsi->dev->ws_mutex);
+	for (i = 0; i < IRDMA_MAX_USER_PRIORITY; ++i)
+		irdma_remove_leaf(vsi, i);
+	mutex_unlock(&vsi->dev->ws_mutex);
+}
drivers/infiniband/hw/irdma/ws.h new file +39 -0
new file mode 100644
index 000000000000..e05feb8a5f9e
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
+/* Copyright (c) 2015 - 2019 Intel Corporation */
+#ifndef IRDMA_WS_H
+#define IRDMA_WS_H
+
+#include "osdep.h"
+
+enum irdma_ws_node_type {
+	WS_NODE_TYPE_PARENT,
+	WS_NODE_TYPE_LEAF,
+};
+
+enum irdma_ws_match_type {
+	WS_MATCH_TYPE_VSI,
+	WS_MATCH_TYPE_TC,
+};
+
+struct irdma_ws_node {
+	struct list_head siblings;
+	struct list_head child_list_head;
+	struct irdma_ws_node *parent;
+	u64 lan_qs_handle; /* opaque handle used by LAN */
+	u32 l2_sched_node_id;
+	u16 index;
+	u16 qs_handle;
+	u16 vsi_index;
+	u8 traffic_class;
+	u8 user_pri;
+	u8 rel_bw;
+	u8 abstraction_layer; /* used for splitting a TC */
+	u8 prio_type;
+	bool type_leaf;
+	bool enable;
+};
+
+enum irdma_status_code irdma_ws_add(struct irdma_sc_vsi *vsi, u8 user_pri);
+void irdma_ws_remove(struct irdma_sc_vsi *vsi, u8 user_pri);
+void irdma_ws_reset(struct irdma_sc_vsi *vsi);
+#endif /* IRDMA_WS_H */
Signature · 1 line
2.23.0