Skip to contentSkip to conversation

Add Broadcom VK driver

PATCH Superseded Scott Branden 3 participants 16 messages Aug 25, 2020 – Sep 30, 2020 1 message not in the archive

Subsystems (2)

v3 of 10 Superseded Latest is v10 →

lore
Share & tools
lore

Scott Branden

author maintainer
#0 lore raw

v3 cover Add Broadcom VK driver

This patch series drops previous patches in [1] that were incorporated by Kees Cook into patch series "Introduce partial kernel_read_file() support" [2].

Remaining patches are contained in this series to add Broadcom VK driver. (which depends on request_firmware_into_buf API addition in other patch series [2] being applied first).

Please note this patch series will not compile without [2].

[1] https://lore.kernel.org/lkml/[email protected]/
[2] https://lore.kernel.org/lkml/[email protected]/

Changes from v2:
- open code BIT macro in uapi header
- A0/B0 boot improvements
Changes from v1:
- declare bcm_vk_intf_ver_chk as static

Scott Branden (3):
bcm-vk: add bcm_vk UAPI
misc: bcm-vk: add Broadcom VK driver
MAINTAINERS: bcm-vk: add maintainer for Broadcom VK Driver

 MAINTAINERS                      |    7 +
 drivers/misc/Kconfig             |    1 +
 drivers/misc/Makefile            |    1 +
 drivers/misc/bcm-vk/Kconfig      |   29 +
 drivers/misc/bcm-vk/Makefile     |   11 +
 drivers/misc/bcm-vk/bcm_vk.h     |  430 +++++++++
 drivers/misc/bcm-vk/bcm_vk_dev.c | 1475 +++++++++++++++++++++++++++++
 drivers/misc/bcm-vk/bcm_vk_msg.c | 1523 ++++++++++++++++++++++++++++++
 drivers/misc/bcm-vk/bcm_vk_msg.h |  211 +++++
 drivers/misc/bcm-vk/bcm_vk_sg.c  |  275 ++++++
 drivers/misc/bcm-vk/bcm_vk_sg.h  |   61 ++
 drivers/misc/bcm-vk/bcm_vk_tty.c |  365 +++++++
 include/uapi/linux/misc/bcm_vk.h |   99 ++
 13 files changed, 4488 insertions(+)
 create mode 100644 drivers/misc/bcm-vk/Kconfig
 create mode 100644 drivers/misc/bcm-vk/Makefile
 create mode 100644 drivers/misc/bcm-vk/bcm_vk.h
 create mode 100644 drivers/misc/bcm-vk/bcm_vk_dev.c
 create mode 100644 drivers/misc/bcm-vk/bcm_vk_msg.c
 create mode 100644 drivers/misc/bcm-vk/bcm_vk_msg.h
 create mode 100644 drivers/misc/bcm-vk/bcm_vk_sg.c
 create mode 100644 drivers/misc/bcm-vk/bcm_vk_sg.h
 create mode 100644 drivers/misc/bcm-vk/bcm_vk_tty.c
 create mode 100644 include/uapi/linux/misc/bcm_vk.h
Signature · 1 line
2.17.1
author maintainer
#1 lore raw

PATCH v3 1/3 bcm-vk: add bcm_vk UAPI

Add user space api for bcm-vk driver.

---
 include/uapi/linux/misc/bcm_vk.h | 99 ++++++++++++++++++++++++++++++++
 1 file changed, 99 insertions(+)
 create mode 100644 include/uapi/linux/misc/bcm_vk.h
include/uapi/linux/misc/bcm_vk.h new file +99 -0
new file mode 100644
index 000000000000..da7848e7c438
@@ -0,0 +1,99 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-2-Clause) */
+/*
+ * Copyright 2018-2020 Broadcom.
+ */
+
+#ifndef __UAPI_LINUX_MISC_BCM_VK_H
+#define __UAPI_LINUX_MISC_BCM_VK_H
+
+#include <linux/ioctl.h>
+#include <linux/types.h>
+
+#define BCM_VK_MAX_FILENAME 64
+
+struct vk_image {
+	__u32 type; /* Type of image */
+#define VK_IMAGE_TYPE_BOOT1 1 /* 1st stage (load to SRAM) */
+#define VK_IMAGE_TYPE_BOOT2 2 /* 2nd stage (load to DDR) */
+	char filename[BCM_VK_MAX_FILENAME]; /* Filename of image */
+};
+
+struct vk_reset {
+	__u32 arg1;
+	__u32 arg2;
+};
+
+#define VK_MAGIC		0x5e
+
+/* Load image to Valkyrie */
+#define VK_IOCTL_LOAD_IMAGE	_IOW(VK_MAGIC, 0x2, struct vk_image)
+
+/* Send Reset to Valkyrie */
+#define VK_IOCTL_RESET		_IOW(VK_MAGIC, 0x4, struct vk_reset)
+
+/*
+ * message block - basic unit in the message where a message's size is always
+ *		   N x sizeof(basic_block)
+ */
+struct vk_msg_blk {
+	__u8 function_id;
+#define VK_FID_TRANS_BUF	5
+#define VK_FID_SHUTDOWN		8
+	__u8 size;
+	__u16 trans_id; /* transport id, queue & msg_id */
+	__u32 context_id;
+	__u32 args[2];
+#define VK_CMD_PLANES_MASK	0x000f /* number of planes to up/download */
+#define VK_CMD_UPLOAD		0x0400 /* memory transfer to vk */
+#define VK_CMD_DOWNLOAD		0x0500 /* memory transfer from vk */
+#define VK_CMD_MASK		0x0f00 /* command mask */
+};
+
+#define VK_BAR_FWSTS			0x41c
+#define VK_BAR_COP_FWSTS		0x428
+/* VK_FWSTS definitions */
+#define VK_FWSTS_RELOCATION_ENTRY	(1ULL << 0)
+#define VK_FWSTS_RELOCATION_EXIT	(1ULL << 1)
+#define VK_FWSTS_INIT_START		(1ULL << 2)
+#define VK_FWSTS_ARCH_INIT_DONE		(1ULL << 3)
+#define VK_FWSTS_PRE_KNL1_INIT_DONE	(1ULL << 4)
+#define VK_FWSTS_PRE_KNL2_INIT_DONE	(1ULL << 5)
+#define VK_FWSTS_POST_KNL_INIT_DONE	(1ULL << 6)
+#define VK_FWSTS_INIT_DONE		(1ULL << 7)
+#define VK_FWSTS_APP_INIT_START		(1ULL << 8)
+#define VK_FWSTS_APP_INIT_DONE		(1ULL << 9)
+#define VK_FWSTS_MASK			0xffffffff
+#define VK_FWSTS_READY			(VK_FWSTS_INIT_START | \
+					 VK_FWSTS_ARCH_INIT_DONE | \
+					 VK_FWSTS_PRE_KNL1_INIT_DONE | \
+					 VK_FWSTS_PRE_KNL2_INIT_DONE | \
+					 VK_FWSTS_POST_KNL_INIT_DONE | \
+					 VK_FWSTS_INIT_DONE | \
+					 VK_FWSTS_APP_INIT_START | \
+					 VK_FWSTS_APP_INIT_DONE)
+/* Deinit */
+#define VK_FWSTS_APP_DEINIT_START	(1ULL << 23)
+#define VK_FWSTS_APP_DEINIT_DONE	(1ULL << 24)
+#define VK_FWSTS_DRV_DEINIT_START	(1ULL << 25)
+#define VK_FWSTS_DRV_DEINIT_DONE	(1ULL << 26)
+#define VK_FWSTS_RESET_DONE		(1ULL << 27)
+#define VK_FWSTS_DEINIT_TRIGGERED	(VK_FWSTS_APP_DEINIT_START | \
+					 VK_FWSTS_APP_DEINIT_DONE  | \
+					 VK_FWSTS_DRV_DEINIT_START | \
+					 VK_FWSTS_DRV_DEINIT_DONE)
+/* Last nibble for reboot reason */
+#define VK_FWSTS_RESET_REASON_SHIFT	28
+#define VK_FWSTS_RESET_REASON_MASK	(0xf << VK_FWSTS_RESET_REASON_SHIFT)
+#define VK_FWSTS_RESET_SYS_PWRUP	(0x0 << VK_FWSTS_RESET_REASON_SHIFT)
+#define VK_FWSTS_RESET_MBOX_DB		(0x1 << VK_FWSTS_RESET_REASON_SHIFT)
+#define VK_FWSTS_RESET_M7_WDOG		(0x2 << VK_FWSTS_RESET_REASON_SHIFT)
+#define VK_FWSTS_RESET_TEMP		(0x3 << VK_FWSTS_RESET_REASON_SHIFT)
+#define VK_FWSTS_RESET_PCI_FLR		(0x4 << VK_FWSTS_RESET_REASON_SHIFT)
+#define VK_FWSTS_RESET_PCI_HOT		(0x5 << VK_FWSTS_RESET_REASON_SHIFT)
+#define VK_FWSTS_RESET_PCI_WARM		(0x6 << VK_FWSTS_RESET_REASON_SHIFT)
+#define VK_FWSTS_RESET_PCI_COLD		(0x7 << VK_FWSTS_RESET_REASON_SHIFT)
+#define VK_FWSTS_RESET_L1		(0x8 << VK_FWSTS_RESET_REASON_SHIFT)
+#define VK_FWSTS_RESET_L0		(0x9 << VK_FWSTS_RESET_REASON_SHIFT)
+#define VK_FWSTS_RESET_UNKNOWN		(0xf << VK_FWSTS_RESET_REASON_SHIFT)
+
+#endif /* __UAPI_LINUX_MISC_BCM_VK_H */
Signature · 1 line
2.17.1
#2 lore raw

Re: [PATCH v3 1/3] bcm-vk: add bcm_vk UAPI

Quoted 22 lines · Scott Branden

Add user space api for bcm-vk driver.

---
 include/uapi/linux/misc/bcm_vk.h | 99 ++++++++++++++++++++++++++++++++
 1 file changed, 99 insertions(+)
 create mode 100644 include/uapi/linux/misc/bcm_vk.h
include/uapi/linux/misc/bcm_vk.h new file +8 -0
new file mode 100644
index 000000000000..da7848e7c438
@@ -0,0 +1,99 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-2-Clause) */
+/*
+ * Copyright 2018-2020 Broadcom.
+ */
+
+#ifndef __UAPI_LINUX_MISC_BCM_VK_H
+#define __UAPI_LINUX_MISC_BCM_VK_H
+

include/uapi/linux/misc/bcm_vk.h line 18

Quoted 10 lines
include/uapi/linux/misc/bcm_vk.h new file +10 -0
+#include <linux/ioctl.h>
+#include <linux/types.h>
+
+#define BCM_VK_MAX_FILENAME 64
+
+struct vk_image {
+	__u32 type; /* Type of image */
+#define VK_IMAGE_TYPE_BOOT1 1 /* 1st stage (load to SRAM) */
+#define VK_IMAGE_TYPE_BOOT2 2 /* 2nd stage (load to DDR) */
+	char filename[BCM_VK_MAX_FILENAME]; /* Filename of image */

nit, but this should really be __u8 filename[...
right?

Quoted 14 lines
include/uapi/linux/misc/bcm_vk.h new file +14 -0
+};
+
+struct vk_reset {
+	__u32 arg1;
+	__u32 arg2;
+};
+
+#define VK_MAGIC		0x5e
+
+/* Load image to Valkyrie */
+#define VK_IOCTL_LOAD_IMAGE	_IOW(VK_MAGIC, 0x2, struct vk_image)
+
+/* Send Reset to Valkyrie */
+#define VK_IOCTL_RESET		_IOW(VK_MAGIC, 0x4, struct vk_reset)

include/uapi/linux/misc/bcm_vk.h line 42

Quoted 10 lines
include/uapi/linux/misc/bcm_vk.h new file +10 -0
+
+/*
+ * message block - basic unit in the message where a message's size is always
+ *		   N x sizeof(basic_block)
+ */
+struct vk_msg_blk {
+	__u8 function_id;
+#define VK_FID_TRANS_BUF	5
+#define VK_FID_SHUTDOWN		8
+	__u8 size;

Size of what?

include/uapi/linux/misc/bcm_vk.h line 50

Quoted 8 lines
include/uapi/linux/misc/bcm_vk.h new file +8 -0
+	__u16 trans_id; /* transport id, queue & msg_id */
+	__u32 context_id;
+	__u32 args[2];
+#define VK_CMD_PLANES_MASK	0x000f /* number of planes to up/download */
+#define VK_CMD_UPLOAD		0x0400 /* memory transfer to vk */
+#define VK_CMD_DOWNLOAD		0x0500 /* memory transfer from vk */
+#define VK_CMD_MASK		0x0f00 /* command mask */
+};

What are these defines for? The args? Something else? It's not really obvious here...

thanks,

greg k-h

author maintainer
#3 lore raw

Re: [PATCH v3 1/3] bcm-vk: add bcm_vk UAPI

Hi Greg,

Quoted 27 lines · Greg Kroah-Hartman
Quoted 26 lines · Scott Branden

Add user space api for bcm-vk driver.

---
 include/uapi/linux/misc/bcm_vk.h | 99 ++++++++++++++++++++++++++++++++
 1 file changed, 99 insertions(+)
 create mode 100644 include/uapi/linux/misc/bcm_vk.h
include/uapi/linux/misc/bcm_vk.h new file +12 -0
new file mode 100644
index 000000000000..da7848e7c438
@@ -0,0 +1,99 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-2-Clause) */
+/*
+ * Copyright 2018-2020 Broadcom.
+ */
+
+#ifndef __UAPI_LINUX_MISC_BCM_VK_H
+#define __UAPI_LINUX_MISC_BCM_VK_H
+
+#include <linux/ioctl.h>
+#include <linux/types.h>
+
+#define BCM_VK_MAX_FILENAME 64
Quoted 8 lines

include/uapi/linux/misc/bcm_vk.h line 18

Quoted 6 lines
include/uapi/linux/misc/bcm_vk.h new file +6 -0
+
+struct vk_image {
+	__u32 type; /* Type of image */
+#define VK_IMAGE_TYPE_BOOT1 1 /* 1st stage (load to SRAM) */
+#define VK_IMAGE_TYPE_BOOT2 2 /* 2nd stage (load to DDR) */
+	char filename[BCM_VK_MAX_FILENAME]; /* Filename of image */

nit, but this should really be __u8 filename[...
right?

Changed to __u8.  Thanks.

Quoted 20 lines
Quoted 20 lines
include/uapi/linux/misc/bcm_vk.h new file +17 -0
+};
+
+struct vk_reset {
+	__u32 arg1;
+	__u32 arg2;
+};
+
+#define VK_MAGIC		0x5e
+
+/* Load image to Valkyrie */
+#define VK_IOCTL_LOAD_IMAGE	_IOW(VK_MAGIC, 0x2, struct vk_image)
+
+/* Send Reset to Valkyrie */
+#define VK_IOCTL_RESET		_IOW(VK_MAGIC, 0x4, struct vk_reset)
+
+/*
+ * message block - basic unit in the message where a message's size is always
Quoted 8 lines

include/uapi/linux/misc/bcm_vk.h line 42

Quoted 7 lines
include/uapi/linux/misc/bcm_vk.h new file +7 -0
+ *		   N x sizeof(basic_block)
+ */
+struct vk_msg_blk {
+	__u8 function_id;
+#define VK_FID_TRANS_BUF	5
+#define VK_FID_SHUTDOWN		8
+	__u8 size;

Size of what?

Size is number of vk_msg_blk's.  Comment is above the struct. Can add a comment on this line as well.

Quoted 11 lines

include/uapi/linux/misc/bcm_vk.h line 50

Quoted 9 lines
include/uapi/linux/misc/bcm_vk.h new file +8 -0
+	__u16 trans_id; /* transport id, queue & msg_id */
+	__u32 context_id;
+	__u32 args[2];
+#define VK_CMD_PLANES_MASK	0x000f /* number of planes to up/download */
+#define VK_CMD_UPLOAD		0x0400 /* memory transfer to vk */
+#define VK_CMD_DOWNLOAD		0x0500 /* memory transfer from vk */
+#define VK_CMD_MASK		0x0f00 /* command mask */
+};

What are these defines for? The args? Something else? It's not really obvious here...

Agree, it's not really obvious and is confusing.
When args[0] is a command the defines apply.
Will add comments.

Quoted 4 lines

thanks,

greg k-h

author maintainer
#4 lore raw

PATCH v3 2/3 misc: bcm-vk: add Broadcom VK driver

In reply to Scott Branden

Add Broadcom VK driver offload engine.
This driver interfaces to the VK PCIe offload engine to perform should offload functions as video transcoding on multiple streams in parallel. VK device is booted from files loaded using request_firmware_into_buf mechanism. After booted card status is updated and messages can then be sent to the card.
Such messages contain scatter gather list of addresses
to pull data from the host to perform operations on.

---
 drivers/misc/Kconfig             |    1 +
 drivers/misc/Makefile            |    1 +
 drivers/misc/bcm-vk/Kconfig      |   29 +
 drivers/misc/bcm-vk/Makefile     |   11 +
 drivers/misc/bcm-vk/bcm_vk.h     |  430 +++++++++
 drivers/misc/bcm-vk/bcm_vk_dev.c | 1475 +++++++++++++++++++++++++++++
 drivers/misc/bcm-vk/bcm_vk_msg.c | 1523 ++++++++++++++++++++++++++++++
 drivers/misc/bcm-vk/bcm_vk_msg.h |  211 +++++
 drivers/misc/bcm-vk/bcm_vk_sg.c  |  275 ++++++
 drivers/misc/bcm-vk/bcm_vk_sg.h  |   61 ++
 drivers/misc/bcm-vk/bcm_vk_tty.c |  365 +++++++
 11 files changed, 4382 insertions(+)
 create mode 100644 drivers/misc/bcm-vk/Kconfig
 create mode 100644 drivers/misc/bcm-vk/Makefile
 create mode 100644 drivers/misc/bcm-vk/bcm_vk.h
 create mode 100644 drivers/misc/bcm-vk/bcm_vk_dev.c
 create mode 100644 drivers/misc/bcm-vk/bcm_vk_msg.c
 create mode 100644 drivers/misc/bcm-vk/bcm_vk_msg.h
 create mode 100644 drivers/misc/bcm-vk/bcm_vk_sg.c
 create mode 100644 drivers/misc/bcm-vk/bcm_vk_sg.h
 create mode 100644 drivers/misc/bcm-vk/bcm_vk_tty.c
drivers/misc/Kconfig +1 -0
index ce136d685d14..9d42b5def81b 100644
@@ -469,6 +469,7 @@ source "drivers/misc/genwqe/Kconfig"
 source "drivers/misc/echo/Kconfig"
 source "drivers/misc/cxl/Kconfig"
 source "drivers/misc/ocxl/Kconfig"
+source "drivers/misc/bcm-vk/Kconfig"
 source "drivers/misc/cardreader/Kconfig"
 source "drivers/misc/habanalabs/Kconfig"
 source "drivers/misc/uacce/Kconfig"
drivers/misc/Makefile +1 -0
index c7bd01ac6291..766837e4b961 100644
@@ -52,6 +52,7 @@ obj-$(CONFIG_ECHO)		+= echo/
 obj-$(CONFIG_CXL_BASE)		+= cxl/
 obj-$(CONFIG_PCI_ENDPOINT_TEST)	+= pci_endpoint_test.o
 obj-$(CONFIG_OCXL)		+= ocxl/
+obj-$(CONFIG_BCM_VK)		+= bcm-vk/
 obj-y				+= cardreader/
 obj-$(CONFIG_PVPANIC)   	+= pvpanic.o
 obj-$(CONFIG_HABANA_AI)		+= habanalabs/
drivers/misc/bcm-vk/Kconfig new file +29 -0
new file mode 100644
index 000000000000..a3a020b19e3b
@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Broadcom VK device
+#
+config BCM_VK
+	tristate "Support for Broadcom VK Accelerators"
+	depends on PCI_MSI
+	help
+	  Select this option to enable support for Broadcom
+	  VK Accelerators.  VK is used for performing
+	  specific offload processing.
+	  This driver enables userspace programs to access these
+	  accelerators via /dev/bcm-vk.N devices.
+
+	  If unsure, say N.
+
+if BCM_VK
+
+config BCM_VK_QSTATS
+	bool "VK Queue Statistics"
+	help
+	  Turn on to enable Queue Statistics.
+	  These are useful for debugging purposes.
+	  Some performance loss by enabling this debug config.
+	  For properly operating PCIe hardware no need to enable this.
+
+	  If unsure, say N.
+
+endif
drivers/misc/bcm-vk/Makefile new file +11 -0
new file mode 100644
index 000000000000..05cb213ee826
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for Broadcom VK driver
+#
+
+obj-$(CONFIG_BCM_VK) += bcm_vk.o
+bcm_vk-objs := \
+	bcm_vk_dev.o \
+	bcm_vk_msg.o \
+	bcm_vk_sg.o \
+	bcm_vk_tty.o
drivers/misc/bcm-vk/bcm_vk.h new file +430 -0

430 lines not on this page · raw message

drivers/misc/bcm-vk/bcm_vk_dev.c new file +1475 -0

1,475 lines not on this page · raw message

drivers/misc/bcm-vk/bcm_vk_msg.c new file +1523 -0

1,041 lines not on this page · raw message

drivers/misc/bcm-vk/bcm_vk_msg.h new file +211 -0

211 lines not on this page · raw message

drivers/misc/bcm-vk/bcm_vk_sg.c new file +275 -0

275 lines not on this page · raw message

drivers/misc/bcm-vk/bcm_vk_sg.h new file +61 -0

61 lines not on this page · raw message

drivers/misc/bcm-vk/bcm_vk_tty.c new file +365 -0

365 lines not on this page · raw message

The rest of this diff is not shown. View the raw message.

Signature · 1 line
2.17.1

Part of the diff is not shown here. View the raw message.

#5 lore raw

Re: [PATCH v3 2/3] misc: bcm-vk: add Broadcom VK driver

I love your patch! Yet something to improve:

Hi Scott,

I love your patch! Yet something to improve:

[auto build test ERROR on char-misc/char-misc-testing]
[also build test ERROR on soc/for-next kees/for-next/pstore linus/master v5.9-rc2 next-20200825] [If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Scott-Branden/Add-Broadcom-VK-driver/20200826-034531
base:   https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git d162219c655c8cf8003128a13840d6c1e183fb80
config: ia64-allmodconfig (attached as .config)
compiler: ia64-linux-gcc (GCC) 9.3.0

reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=ia64

If you fix the issue, kindly add following tag as appropriate

All errors (new ones prefixed by >>):

drivers/misc/bcm-vk/bcm_vk_dev.c: In function 'bcm_vk_load_image_by_type':

>> drivers/misc/bcm-vk/bcm_vk_dev.c:592:8: error: implicit declaration of function 'request_partial_firmware_into_buf'; did you mean 'request_firmware_into_buf'? [-Werror=implicit-function-declaration]
     592 |  ret = request_partial_firmware_into_buf(&fw, filename, dev,
         |        ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
         |        request_firmware_into_buf

cc1: some warnings being treated as errors

# https://github.com/0day-ci/linux/commit/829bb182b6fcada015394adff99b914d9415c8b6
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Scott-Branden/Add-Broadcom-VK-driver/20200826-034531 git checkout 829bb182b6fcada015394adff99b914d9415c8b6
vim +592 drivers/misc/bcm-vk/bcm_vk_dev.c

514
515 static int bcm_vk_load_image_by_type(struct bcm_vk *vk, uint32_t load_type,
516 const char *filename)
517 {
518 struct device *dev = &vk->pdev->dev;
519 const struct firmware *fw = NULL;
520 void *bufp = NULL;
521 size_t max_buf, offset;
522 int ret;
523 uint64_t offset_codepush;
524 uint32_t codepush;
525 uint32_t value;
526 dma_addr_t boot_dma_addr;
527 bool is_stdalone;
528
529 if (load_type == VK_IMAGE_TYPE_BOOT1) {
530 /*
531 * After POR, enable VK soft BOOTSRC so bootrom do not clear
532 * the pushed image (the TCM memories).
533 */
534 value = vkread32(vk, BAR_0, BAR_BOOTSRC_SELECT);
535 value |= BOOTSRC_SOFT_ENABLE;
536 vkwrite32(vk, value, BAR_0, BAR_BOOTSRC_SELECT);
537
538 codepush = CODEPUSH_BOOTSTART + CODEPUSH_BOOT1_ENTRY;
539 offset_codepush = BAR_CODEPUSH_SBL;
540
541 /* Write a 1 to request SRAM open bit */
542 vkwrite32(vk, CODEPUSH_BOOTSTART, BAR_0, offset_codepush);
543
544 /* Wait for VK to respond */
545 ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS, SRAM_OPEN,
546 SRAM_OPEN, LOAD_IMAGE_TIMEOUT_MS);
547 if (ret < 0) {
548 dev_err(dev, "boot1 wait SRAM err - ret(%d)\n", ret);
549 goto err_buf_out;
550 }
551
552 max_buf = SZ_256K;
553 bufp = dma_alloc_coherent(dev,
554 max_buf,
555 &boot_dma_addr, GFP_KERNEL);
556 if (!bufp) {
557 dev_err(dev, "Error allocating 0x%zx\n", max_buf);
558 ret = -ENOMEM;
559 goto err_buf_out;
560 }
561 } else if (load_type == VK_IMAGE_TYPE_BOOT2) {
562 codepush = CODEPUSH_BOOT2_ENTRY;
563 offset_codepush = BAR_CODEPUSH_SBI;
564
565 /* Wait for VK to respond */
566 ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS, DDR_OPEN,
567 DDR_OPEN, LOAD_IMAGE_TIMEOUT_MS);
568 if (ret < 0) {
569 dev_err(dev, "boot2 wait DDR open error - ret(%d)\n",
570 ret);
571 goto err_buf_out;
572 }
573
574 max_buf = SZ_4M;
575 bufp = dma_alloc_coherent(dev,
576 max_buf,
577 &boot_dma_addr, GFP_KERNEL);
578 if (!bufp) {
579 dev_err(dev, "Error allocating 0x%zx\n", max_buf);
580 ret = -ENOMEM;
581 goto err_buf_out;
582 }
583
584 bcm_vk_buf_notify(vk, bufp, boot_dma_addr, max_buf);
585 } else {
586 dev_err(dev, "Error invalid image type 0x%x\n", load_type);
587 ret = -EINVAL;
588 goto err_buf_out;
589 }
590
591 offset = 0;

 > 592		ret = request_partial_firmware_into_buf(&fw, filename, dev,
   593							bufp, max_buf, offset);
   594		if (ret) {
   595			dev_err(dev, "Error %d requesting firmware file: %s\n",
   596				ret, filename);
   597			goto err_firmware_out;
   598		}
   599		dev_dbg(dev, "size=0x%zx\n", fw->size);
   600		if (load_type == VK_IMAGE_TYPE_BOOT1)
   601			memcpy_toio(vk->bar[BAR_1] + BAR1_CODEPUSH_BASE_BOOT1,
   602				    bufp,
   603				    fw->size);
   604	
   605		dev_dbg(dev, "Signaling 0x%x to 0x%llx\n", codepush, offset_codepush);
   606		vkwrite32(vk, codepush, BAR_0, offset_codepush);
   607	
   608		if (load_type == VK_IMAGE_TYPE_BOOT1) {
   609			/* wait until done */
   610			ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS,
   611					  BOOT1_RUNNING,
   612					  BOOT1_RUNNING,
   613					  BOOT1_STARTUP_TIMEOUT_MS);
   614	
   615			is_stdalone = vkread32(vk, BAR_0, BAR_BOOT_STATUS) &
   616				      BOOT_STDALONE_RUNNING;
   617			if (ret && !is_stdalone) {
   618				dev_err(dev,
   619					"Timeout %ld ms waiting for boot1 to come up - ret(%d)\n",
   620					BOOT1_STARTUP_TIMEOUT_MS, ret);
   621				goto err_firmware_out;
   622			} else if (is_stdalone) {
   623				uint32_t reg;
   624	
   625				reg = vkread32(vk, BAR_0, BAR_BOOT1_STDALONE_PROGRESS);
   626				if ((reg & BOOT1_STDALONE_PROGRESS_MASK) ==
   627					     BOOT1_STDALONE_SUCCESS) {
   628					dev_info(dev, "Boot1 standalone success\n");
   629					ret = 0;
   630				} else {
   631					dev_err(dev, "Timeout %ld ms - Boot1 standalone failure\n",
   632						BOOT1_STARTUP_TIMEOUT_MS);
   633					ret = -EINVAL;
   634					goto err_firmware_out;
   635				}
   636			}
   637		} else if (load_type == VK_IMAGE_TYPE_BOOT2) {
   638			unsigned long timeout;
   639	
   640			timeout = jiffies + msecs_to_jiffies(LOAD_IMAGE_TIMEOUT_MS);
   641	
   642			/* To send more data to VK than max_buf allowed at a time */
   643			do {
   644				/*
   645				 * Check for ack from card. when Ack is received,
   646				 * it means all the data is received by card.
   647				 * Exit the loop after ack is received.
   648				 */
   649				ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS,
   650						  FW_LOADER_ACK_RCVD_ALL_DATA,
   651						  FW_LOADER_ACK_RCVD_ALL_DATA,
   652						  TXFR_COMPLETE_TIMEOUT_MS);
   653				if (ret == 0) {
   654					dev_info(dev, "Exit boot2 download\n");
   655					break;
   656				} else if (ret == -EFAULT) {
   657					dev_err(dev, "Error detected during ACK waiting");
   658					goto err_firmware_out;
   659				}
   660	
   661				/* exit the loop, if there is no response from card */
   662				if (time_after(jiffies, timeout)) {
   663					dev_err(dev, "Error. No reply from card\n");
   664					ret = -ETIMEDOUT;
   665					goto err_firmware_out;
   666				}
   667	
   668				/* Wait for VK to open BAR space to copy new data */
   669				ret = bcm_vk_wait(vk, BAR_0, offset_codepush,
   670						  codepush, 0,
   671						  TXFR_COMPLETE_TIMEOUT_MS);
   672				if (ret == 0) {
   673					offset += max_buf;
   674					ret = request_partial_firmware_into_buf
   675							(&fw,
   676							 filename,
   677							 dev, bufp,
   678							 max_buf,
   679							 offset);
   680					if (ret) {
   681						dev_err(dev,
   682							"Error %d requesting firmware file: %s offset: 0x%zx\n",
   683							ret, filename, offset);
   684						goto err_firmware_out;
   685					}
   686					dev_dbg(dev, "size=0x%zx\n", fw->size);
   687					dev_dbg(dev, "Signaling 0x%x to 0x%llx\n",
   688						codepush, offset_codepush);
   689					vkwrite32(vk, codepush, BAR_0, offset_codepush);
   690					/* reload timeout after every codepush */
   691					timeout = jiffies +
   692					    msecs_to_jiffies(LOAD_IMAGE_TIMEOUT_MS);
   693				} else if (ret == -EFAULT) {
   694					dev_err(dev, "Error detected waiting for transfer\n");
   695					goto err_firmware_out;
   696				}
   697			} while (1);
   698	
   699			/* wait for fw status bits to indicate app ready */
   700			ret = bcm_vk_wait(vk, BAR_0, VK_BAR_FWSTS,
   701					  VK_FWSTS_READY,
   702					  VK_FWSTS_READY,
   703					  BOOT2_STARTUP_TIMEOUT_MS);

704 if (ret < 0) {
705 dev_err(dev, "Boot2 not ready - ret(%d)\n", ret);
706 goto err_firmware_out;
707 }
708
709 is_stdalone = vkread32(vk, BAR_0, BAR_BOOT_STATUS) &
710 BOOT_STDALONE_RUNNING;
711 if (!is_stdalone) {
712 ret = bcm_vk_intf_ver_chk(vk);
713 if (ret) {
714 dev_err(dev, "failure in intf version check\n");
715 goto err_firmware_out;
716 }
717
718 /*
719 * Next, initialize Message Q if we are loading boot2.
720 * Do a force sync
721 */
722 ret = bcm_vk_sync_msgq(vk, true);
723 if (ret) {
724 dev_err(dev, "Boot2 Error reading comm msg Q info\n");
725 ret = -EIO;
726 goto err_firmware_out;
727 }
728
729 /* sync & channel other info */
730 ret = bcm_vk_sync_card_info(vk);
731 if (ret) {
732 dev_err(dev, "Syncing Card Info failure\n");
733 goto err_firmware_out;
734 }
735 }
736 }
737
738 err_firmware_out:
739 release_firmware(fw);
740
741 err_buf_out:
742 if (bufp)
743 dma_free_coherent(dev, max_buf, bufp, boot_dma_addr);
744
745 return ret;
746 }
747

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/[email protected]

#6 lore raw

Re: [PATCH v3 2/3] misc: bcm-vk: add Broadcom VK driver

In reply to Scott Branden
Quoted 302 lines · Scott Branden

Add Broadcom VK driver offload engine.
This driver interfaces to the VK PCIe offload engine to perform should offload functions as video transcoding on multiple streams in parallel. VK device is booted from files loaded using request_firmware_into_buf mechanism. After booted card status is updated and messages can then be sent to the card.
Such messages contain scatter gather list of addresses
to pull data from the host to perform operations on.

---
 drivers/misc/Kconfig             |    1 +
 drivers/misc/Makefile            |    1 +
 drivers/misc/bcm-vk/Kconfig      |   29 +
 drivers/misc/bcm-vk/Makefile     |   11 +
 drivers/misc/bcm-vk/bcm_vk.h     |  430 +++++++++
 drivers/misc/bcm-vk/bcm_vk_dev.c | 1475 +++++++++++++++++++++++++++++
 drivers/misc/bcm-vk/bcm_vk_msg.c | 1523 ++++++++++++++++++++++++++++++
 drivers/misc/bcm-vk/bcm_vk_msg.h |  211 +++++
 drivers/misc/bcm-vk/bcm_vk_sg.c  |  275 ++++++
 drivers/misc/bcm-vk/bcm_vk_sg.h  |   61 ++
 drivers/misc/bcm-vk/bcm_vk_tty.c |  365 +++++++
 11 files changed, 4382 insertions(+)
 create mode 100644 drivers/misc/bcm-vk/Kconfig
 create mode 100644 drivers/misc/bcm-vk/Makefile
 create mode 100644 drivers/misc/bcm-vk/bcm_vk.h
 create mode 100644 drivers/misc/bcm-vk/bcm_vk_dev.c
 create mode 100644 drivers/misc/bcm-vk/bcm_vk_msg.c
 create mode 100644 drivers/misc/bcm-vk/bcm_vk_msg.h
 create mode 100644 drivers/misc/bcm-vk/bcm_vk_sg.c
 create mode 100644 drivers/misc/bcm-vk/bcm_vk_sg.h
 create mode 100644 drivers/misc/bcm-vk/bcm_vk_tty.c
drivers/misc/Kconfig +1 -0
index ce136d685d14..9d42b5def81b 100644
@@ -469,6 +469,7 @@ source "drivers/misc/genwqe/Kconfig"
 source "drivers/misc/echo/Kconfig"
 source "drivers/misc/cxl/Kconfig"
 source "drivers/misc/ocxl/Kconfig"
+source "drivers/misc/bcm-vk/Kconfig"
 source "drivers/misc/cardreader/Kconfig"
 source "drivers/misc/habanalabs/Kconfig"
 source "drivers/misc/uacce/Kconfig"
drivers/misc/Makefile +1 -0
index c7bd01ac6291..766837e4b961 100644
@@ -52,6 +52,7 @@ obj-$(CONFIG_ECHO)		+= echo/
 obj-$(CONFIG_CXL_BASE)		+= cxl/
 obj-$(CONFIG_PCI_ENDPOINT_TEST)	+= pci_endpoint_test.o
 obj-$(CONFIG_OCXL)		+= ocxl/
+obj-$(CONFIG_BCM_VK)		+= bcm-vk/
 obj-y				+= cardreader/
 obj-$(CONFIG_PVPANIC)   	+= pvpanic.o
 obj-$(CONFIG_HABANA_AI)		+= habanalabs/
drivers/misc/bcm-vk/Kconfig new file +29 -0
new file mode 100644
index 000000000000..a3a020b19e3b
@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Broadcom VK device
+#
+config BCM_VK
+	tristate "Support for Broadcom VK Accelerators"
+	depends on PCI_MSI
+	help
+	  Select this option to enable support for Broadcom
+	  VK Accelerators.  VK is used for performing
+	  specific offload processing.
+	  This driver enables userspace programs to access these
+	  accelerators via /dev/bcm-vk.N devices.
+
+	  If unsure, say N.
+
+if BCM_VK
+
+config BCM_VK_QSTATS
+	bool "VK Queue Statistics"
+	help
+	  Turn on to enable Queue Statistics.
+	  These are useful for debugging purposes.
+	  Some performance loss by enabling this debug config.
+	  For properly operating PCIe hardware no need to enable this.
+
+	  If unsure, say N.
+
+endif
drivers/misc/bcm-vk/Makefile new file +11 -0
new file mode 100644
index 000000000000..05cb213ee826
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for Broadcom VK driver
+#
+
+obj-$(CONFIG_BCM_VK) += bcm_vk.o
+bcm_vk-objs := \
+	bcm_vk_dev.o \
+	bcm_vk_msg.o \
+	bcm_vk_sg.o \
+	bcm_vk_tty.o
drivers/misc/bcm-vk/bcm_vk.h new file +183 -0
new file mode 100644
index 000000000000..33478ccc1be3
@@ -0,0 +1,430 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2018-2020 Broadcom.
+ */
+
+#ifndef BCM_VK_H
+#define BCM_VK_H
+
+#include <linux/atomic.h>
+#include <linux/firmware.h>
+#include <linux/irq.h>
+#include <linux/miscdevice.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+#include <linux/poll.h>
+#include <linux/sched/signal.h>
+#include <linux/tty.h>
+#include <linux/uaccess.h>
+#include <uapi/linux/misc/bcm_vk.h>
+
+#include "bcm_vk_msg.h"
+
+#define DRV_MODULE_NAME		"bcm-vk"
+
+/*
+ * Load Image is completed in two stages:
+ *
+ * 1) When the VK device boot-up, M7 CPU runs and executes the BootROM.
+ * The Secure Boot Loader (SBL) as part of the BootROM will run
+ * to open up ITCM for host to push BOOT1 image.
+ * SBL will authenticate the image before jumping to BOOT1 image.
+ *
+ * 2) Because BOOT1 image is a secured image, we also called it the
+ * Secure Boot Image (SBI). At second stage, SBI will initialize DDR
+ * and wait for host to push BOOT2 image to DDR.
+ * SBI will authenticate the image before jumping to BOOT2 image.
+ *
+ */
+/* Location of registers of interest in BAR0 */
+
+/* Request register for Secure Boot Loader (SBL) download */
+#define BAR_CODEPUSH_SBL		0x400
+/* Start of ITCM */
+#define CODEPUSH_BOOT1_ENTRY		0x00400000
+#define CODEPUSH_MASK		        0xfffff000
+#define CODEPUSH_BOOTSTART		BIT(0)
+
+/* Boot Status register */
+#define BAR_BOOT_STATUS			0x404
+
+#define SRAM_OPEN			BIT(16)
+#define DDR_OPEN			BIT(17)
+
+/* Firmware loader progress status definitions */
+#define FW_LOADER_ACK_SEND_MORE_DATA	BIT(18)
+#define FW_LOADER_ACK_IN_PROGRESS	BIT(19)
+#define FW_LOADER_ACK_RCVD_ALL_DATA	BIT(20)
+
+/* Boot1/2 is running in standalone mode */
+#define BOOT_STDALONE_RUNNING		BIT(21)
+
+/* definitions for boot status register */
+#define BOOT_STATE_MASK			(0xffffffff & \
+					 ~(FW_LOADER_ACK_SEND_MORE_DATA | \
+					   FW_LOADER_ACK_IN_PROGRESS | \
+					   BOOT_STDALONE_RUNNING))
+
+#define BOOT_ERR_SHIFT			4
+#define BOOT_ERR_MASK			(0xf << BOOT_ERR_SHIFT)
+#define BOOT_PROG_MASK			0xf
+
+#define BROM_STATUS_NOT_RUN		0x2
+#define BROM_NOT_RUN			(SRAM_OPEN | BROM_STATUS_NOT_RUN)
+#define BROM_STATUS_COMPLETE		0x6
+#define BROM_RUNNING			(SRAM_OPEN | BROM_STATUS_COMPLETE)
+#define BOOT1_STATUS_COMPLETE		0x6
+#define BOOT1_RUNNING			(DDR_OPEN | BOOT1_STATUS_COMPLETE)
+#define BOOT2_STATUS_COMPLETE		0x6
+#define BOOT2_RUNNING			(FW_LOADER_ACK_RCVD_ALL_DATA | \
+					 BOOT2_STATUS_COMPLETE)
+
+/* Boot request for Secure Boot Image (SBI) */
+#define BAR_CODEPUSH_SBI		0x408
+/* 64M mapped to BAR2 */
+#define CODEPUSH_BOOT2_ENTRY		0x60000000
+
+#define BAR_CARD_STATUS			0x410
+/* CARD_STATUS definitions */
+#define CARD_STATUS_TTYVK0_READY	BIT(0)
+#define CARD_STATUS_TTYVK1_READY	BIT(1)
+
+#define BAR_BOOT1_STDALONE_PROGRESS	0x420
+#define BOOT1_STDALONE_SUCCESS		(BIT(13) | BIT(14))
+#define BOOT1_STDALONE_PROGRESS_MASK	BOOT1_STDALONE_SUCCESS
+
+#define BAR_METADATA_VERSION		0x440
+#define BAR_OS_UPTIME			0x444
+#define BAR_CHIP_ID			0x448
+#define MAJOR_SOC_REV(_chip_id)		(((_chip_id) >> 20) & 0xf)
+
+#define BAR_CARD_TEMPERATURE		0x45c
+/* defines for all temperature sensor */
+#define BCM_VK_TEMP_FIELD_MASK		0xff
+#define BCM_VK_CPU_TEMP_SHIFT		0
+#define BCM_VK_DDR0_TEMP_SHIFT		8
+#define BCM_VK_DDR1_TEMP_SHIFT		16
+
+#define BAR_CARD_VOLTAGE		0x460
+/* defines for voltage rail conversion */
+#define BCM_VK_VOLT_RAIL_MASK		0xffff
+#define BCM_VK_3P3_VOLT_REG_SHIFT	16
+
+#define BAR_CARD_ERR_LOG		0x464
+/* Error log register bit definition - register for error alerts */
+#define ERR_LOG_UECC			BIT(0)
+#define ERR_LOG_SSIM_BUSY		BIT(1)
+#define ERR_LOG_AFBC_BUSY		BIT(2)
+#define ERR_LOG_HIGH_TEMP_ERR		BIT(3)
+#define ERR_LOG_WDOG_TIMEOUT		BIT(4)
+#define ERR_LOG_SYS_FAULT		BIT(5)
+#define ERR_LOG_RAMDUMP			BIT(6)
+#define ERR_LOG_MEM_ALLOC_FAIL		BIT(8)
+#define ERR_LOG_LOW_TEMP_WARN		BIT(9)
+#define ERR_LOG_ECC			BIT(10)
+/* Alert bit definitions detectd on host */
+#define ERR_LOG_HOST_INTF_V_FAIL	BIT(13)
+#define ERR_LOG_HOST_HB_FAIL		BIT(14)
+#define ERR_LOG_HOST_PCIE_DWN		BIT(15)
+
+#define BAR_CARD_ERR_MEM		0x468
+/* defines for mem err, all fields have same width */
+#define BCM_VK_MEM_ERR_FIELD_MASK	0xff
+#define BCM_VK_ECC_MEM_ERR_SHIFT	0
+#define BCM_VK_UECC_MEM_ERR_SHIFT	8
+/* threshold of event occurrence and logs start to come out */
+#define BCM_VK_ECC_THRESHOLD		10
+#define BCM_VK_UECC_THRESHOLD		1
+
+#define BAR_CARD_PWR_AND_THRE		0x46c
+/* defines for power and temp threshold, all fields have same width */
+#define BCM_VK_PWR_AND_THRE_FIELD_MASK	0xff
+#define BCM_VK_LOW_TEMP_THRE_SHIFT	0
+#define BCM_VK_HIGH_TEMP_THRE_SHIFT	8
+#define BCM_VK_PWR_STATE_SHIFT		16
+
+#define BAR_CARD_STATIC_INFO		0x470
+
+#define BAR_INTF_VER			0x47c
+#define BAR_INTF_VER_MAJOR_SHIFT	16
+#define BAR_INTF_VER_MASK		0xffff
+/*
+ * major and minor semantic version numbers supported
+ * Please update as required on interface changes
+ */
+#define SEMANTIC_MAJOR			1
+#define SEMANTIC_MINOR			0
+
+#define BAR_BOOTSRC_SELECT		0xc78
+/* BOOTSRC definitions */
+#define BOOTSRC_SOFT_ENABLE		BIT(14)
+
+/* Card OS Firmware version size */
+#define BAR_FIRMWARE_TAG_SIZE		50
+#define FIRMWARE_STATUS_PRE_INIT_DONE	0x1f
+
+/* VK MSG_ID defines */
+#define VK_MSG_ID_BITMAP_SIZE		4096
+#define VK_MSG_ID_BITMAP_MASK		(VK_MSG_ID_BITMAP_SIZE - 1)
+#define VK_MSG_ID_OVERFLOW		0xffff
+
+/* VK device supports a maximum of 3 bars */
+#define MAX_BAR	3
+
+/* default number of msg blk for inband SGL */
+#define BCM_VK_DEF_IB_SGL_BLK_LEN	 16
+#define BCM_VK_IB_SGL_BLK_MAX		 24
+
+enum pci_barno {
+	BAR_0 = 0,
+	BAR_1,
+	BAR_2
+};
+

drivers/misc/bcm-vk/bcm_vk.h line 193

Quoted 10 lines
drivers/misc/bcm-vk/bcm_vk.h new file +10 -0
+#define BCM_VK_NUM_TTY 2
+
+struct bcm_vk_tty {
+	struct tty_port port;
+	uint32_t to_offset;	/* bar offset to use */
+	uint32_t to_size;	/* to VK buffer size */
+	uint32_t wr;		/* write offset shadow */
+	uint32_t from_offset;	/* bar offset to use */
+	uint32_t from_size;	/* from VK buffer size */
+	uint32_t rd;		/* read offset shadow */

nit, these "unit32_t" stuff really doesn't matter in the kernel, 'u32' is a better choice overall. Same for u8 and others, for this whole driver.

drivers/misc/bcm-vk/bcm_vk.h line 196

Quoted 3 lines
drivers/misc/bcm-vk/bcm_vk.h new file +3 -0
+	pid_t pid;
+	bool irq_enabled;
+	bool is_opened;		/* tracks tty open/close */

Why do you need to track this? Doesn't the tty core handle this for you?

Quoted 71 lines
drivers/misc/bcm-vk/bcm_vk.h new file +71 -0
+};
+
+/* VK device max power state, supports 3, full, reduced and low */
+#define MAX_OPP 3
+#define MAX_CARD_INFO_TAG_SIZE 64
+
+struct bcm_vk_card_info {
+	uint32_t version;
+	char os_tag[MAX_CARD_INFO_TAG_SIZE];
+	char cmpt_tag[MAX_CARD_INFO_TAG_SIZE];
+	uint32_t cpu_freq_mhz;
+	uint32_t cpu_scale[MAX_OPP];
+	uint32_t ddr_freq_mhz;
+	uint32_t ddr_size_MB;
+	uint32_t video_core_freq_mhz;
+};
+
+/* DAUTH related info */
+struct bcm_vk_dauth_key {
+	char store[VK_BAR1_DAUTH_STORE_SIZE];
+	char valid[VK_BAR1_DAUTH_VALID_SIZE];
+};
+
+struct bcm_vk_dauth_info {
+	struct bcm_vk_dauth_key keys[VK_BAR1_DAUTH_MAX];
+};
+
+/*
+ * Control structure of logging messages from the card.  This
+ * buffer is for logmsg that comes from vk
+ */
+struct bcm_vk_peer_log {
+	uint32_t rd_idx;
+	uint32_t wr_idx;
+	uint32_t buf_size;
+	uint32_t mask;
+	char data[0];
+};
+
+/* max size per line of peer log */
+#define BCM_VK_PEER_LOG_LINE_MAX  256
+
+/*
+ * single entry for processing type + utilization
+ */
+#define BCM_VK_PROC_TYPE_TAG_LEN 8
+struct bcm_vk_proc_mon_entry_t {
+	char tag[BCM_VK_PROC_TYPE_TAG_LEN];
+	uint32_t used;
+	uint32_t max; /**< max capacity */
+};
+
+/**
+ * Structure for run time utilization
+ */
+#define BCM_VK_PROC_MON_MAX 8 /* max entries supported */
+struct bcm_vk_proc_mon_info {
+	uint32_t num; /**< no of entries */
+	uint32_t entry_size; /**< per entry size */
+	struct bcm_vk_proc_mon_entry_t entries[BCM_VK_PROC_MON_MAX];
+};
+
+struct bcm_vk_hb_ctrl {
+	struct timer_list timer;
+	uint32_t last_uptime;
+	uint32_t lost_cnt;
+};
+
+struct bcm_vk_alert {
+	uint16_t flags;
+	uint16_t notfs;

drivers/misc/bcm-vk/bcm_vk.h line 277

Quoted 10 lines
drivers/misc/bcm-vk/bcm_vk.h new file +10 -0
+};
+
+/* some alert counters that the driver will keep track */
+struct bcm_vk_alert_cnts {
+	uint16_t ecc;
+	uint16_t uecc;
+};
+
+struct bcm_vk {
+	struct pci_dev *pdev;

So you have a pointer to your parent pci device, and...

Quoted 4 lines
drivers/misc/bcm-vk/bcm_vk.h new file +4 -0
+	void __iomem *bar[MAX_BAR];
+	int num_irqs;
+
+	struct bcm_vk_card_info card_info;

drivers/misc/bcm-vk/bcm_vk.h line 291

Quoted 10 lines
drivers/misc/bcm-vk/bcm_vk.h new file +10 -0
+	struct bcm_vk_proc_mon_info proc_mon_info;
+	struct bcm_vk_dauth_info dauth_info;
+
+#if defined(BCM_VK_LEGACY_API)
+	struct msix_entry msix[32];
+#endif
+	/* mutex to protect the ioctls */
+	struct mutex mutex;
+	struct miscdevice miscdev;
+	int misc_devid; /* dev id allocated */

a misc device, and...

drivers/misc/bcm-vk/bcm_vk.h line 295

Quoted 4 lines
drivers/misc/bcm-vk/bcm_vk.h new file +4 -0
+
+	struct tty_driver *tty_drv;
+	struct timer_list serial_timer;
+	struct bcm_vk_tty tty[BCM_VK_NUM_TTY];

A number of tty ports?

drivers/misc/bcm-vk/bcm_vk.h line 300

Quoted 5 lines
drivers/misc/bcm-vk/bcm_vk.h new file +5 -0
+	struct workqueue_struct *tty_wq_thread;
+	struct work_struct tty_wq_work;
+
+	/* Reference-counting to handle file operations */
+	struct kref kref;

And a kref?

What is controlling the lifetime rules of your structure?

Why a kref?

Why the tty ports?

Why the misc device?

This feels really crazy to me...

Quoted 64 lines
drivers/misc/bcm-vk/bcm_vk.h new file +64 -0
+
+	spinlock_t msg_id_lock; /* Spinlock for msg_id */
+	uint16_t msg_id;
+	DECLARE_BITMAP(bmap, VK_MSG_ID_BITMAP_SIZE);
+	spinlock_t ctx_lock; /* Spinlock for component context */
+	struct bcm_vk_ctx ctx[VK_CMPT_CTX_MAX];
+	struct bcm_vk_ht_entry pid_ht[VK_PID_HT_SZ];
+	pid_t reset_pid; /* process that issue reset */
+
+	atomic_t msgq_inited; /* indicate if info has been synced with vk */
+	struct bcm_vk_msg_chan to_v_msg_chan;
+	struct bcm_vk_msg_chan to_h_msg_chan;
+
+	struct workqueue_struct *wq_thread;
+	struct work_struct wq_work; /* work queue for deferred job */
+	unsigned long wq_offload[1]; /* various flags on wq requested */
+	void *tdma_vaddr; /* test dma segment virtual addr */
+	dma_addr_t tdma_addr; /* test dma segment bus addr */
+
+	struct notifier_block panic_nb;
+	uint32_t ib_sgl_size; /* size allocated for inband sgl insertion */
+
+	/* heart beat mechanism control structure */
+	struct bcm_vk_hb_ctrl hb_ctrl;
+	/* house-keeping variable of error logs */
+	spinlock_t host_alert_lock; /* protection to access host_alert struct */
+	struct bcm_vk_alert host_alert;
+	struct bcm_vk_alert peer_alert; /* bits set by the card */
+	struct bcm_vk_alert_cnts alert_cnts;
+
+	/* offset of the peer log control in BAR2 */
+	uint32_t peerlog_off;
+	struct bcm_vk_peer_log peerlog_info; /* record of peer log info */
+	/* offset of processing monitoring info in BAR2 */
+	uint32_t proc_mon_off;
+};
+
+/* wq offload work items bits definitions */
+enum bcm_vk_wq_offload_flags {
+	BCM_VK_WQ_DWNLD_PEND = 0,
+	BCM_VK_WQ_DWNLD_AUTO = 1,
+	BCM_VK_WQ_NOTF_PEND  = 2,
+};
+
+/* a macro to get an individual field with mask and shift */
+#define BCM_VK_EXTRACT_FIELD(_field, _reg, _mask, _shift) \
+		(_field = (((_reg) >> (_shift)) & (_mask)))
+
+/* structure that is used to faciliate displaying of register content */
+struct bcm_vk_entry {
+	const uint32_t mask;
+	const uint32_t exp_val;
+	const char *str;
+};
+
+/* alerts that could be generated from peer */
+#define BCM_VK_PEER_ERR_NUM 10
+extern struct bcm_vk_entry const bcm_vk_peer_err[BCM_VK_PEER_ERR_NUM];
+/* alerts detected by the host */
+#define BCM_VK_HOST_ERR_NUM 3
+extern struct bcm_vk_entry const bcm_vk_host_err[BCM_VK_HOST_ERR_NUM];
+
+/*
+ * check if PCIe interface is down on read.  Use it when it is

drivers/misc/bcm-vk/bcm_vk.h line 374

Quoted 10 lines
drivers/misc/bcm-vk/bcm_vk.h new file +10 -0
+ * certain that _val should never be all ones.
+ */
+#define BCM_VK_INTF_IS_DOWN(val) ((val) == 0xffffffff)
+
+static inline uint32_t vkread32(struct bcm_vk *vk,
+				enum pci_barno bar,
+				uint64_t offset)
+{
+	return readl(vk->bar[bar] + offset);
+}

"vk"? What is that prefix for?

Quoted 44 lines
drivers/misc/bcm-vk/bcm_vk.h new file +44 -0
+
+static inline void vkwrite32(struct bcm_vk *vk,
+			     uint32_t value,
+			     enum pci_barno bar,
+			     uint64_t offset)
+{
+	writel(value, vk->bar[bar] + offset);
+}
+
+static inline uint8_t vkread8(struct bcm_vk *vk,
+			      enum pci_barno bar,
+			      uint64_t offset)
+{
+	return readb(vk->bar[bar] + offset);
+}
+
+static inline void vkwrite8(struct bcm_vk *vk,
+			    uint8_t value,
+			    enum pci_barno bar,
+			    uint64_t offset)
+{
+	writeb(value, vk->bar[bar] + offset);
+}
+
+int bcm_vk_open(struct inode *inode, struct file *p_file);
+ssize_t bcm_vk_read(struct file *p_file, char __user *buf, size_t count,
+		    loff_t *f_pos);
+ssize_t bcm_vk_write(struct file *p_file, const char __user *buf,
+		     size_t count, loff_t *f_pos);
+__poll_t bcm_vk_poll(struct file *p_file, struct poll_table_struct *wait);
+int bcm_vk_release(struct inode *inode, struct file *p_file);
+void bcm_vk_release_data(struct kref *kref);
+irqreturn_t bcm_vk_msgq_irqhandler(int irq, void *dev_id);
+irqreturn_t bcm_vk_notf_irqhandler(int irq, void *dev_id);
+irqreturn_t bcm_vk_tty_irqhandler(int irq, void *dev_id);
+int bcm_vk_msg_init(struct bcm_vk *vk);
+void bcm_vk_msg_remove(struct bcm_vk *vk);
+int bcm_vk_sync_msgq(struct bcm_vk *vk, bool force_sync);
+bool bcm_vk_msgq_marker_valid(struct bcm_vk *vk);
+void bcm_vk_blk_drv_access(struct bcm_vk *vk);
+int bcm_vk_send_shutdown_msg(struct bcm_vk *vk, uint32_t shut_type,
+			     const pid_t pid, const uint32_t q_num);
+int bcm_vk_trigger_reset(struct bcm_vk *vk);
+void bcm_to_v_q_doorbell(struct bcm_vk *vk, uint32_t q_num, uint32_t db_val);

drivers/misc/bcm-vk/bcm_vk.h line 428

Quoted 10 lines
drivers/misc/bcm-vk/bcm_vk.h new file +10 -0
+void bcm_to_v_reset_doorbell(struct bcm_vk *vk, uint32_t db_val);
+int bcm_vk_auto_load_all_images(struct bcm_vk *vk);
+int bcm_vk_tty_init(struct bcm_vk *vk, char *name);
+void bcm_vk_tty_exit(struct bcm_vk *vk);
+void bcm_vk_tty_terminate_tty_user(struct bcm_vk *vk);
+void bcm_vk_hb_init(struct bcm_vk *vk);
+void bcm_vk_hb_deinit(struct bcm_vk *vk);
+void bcm_vk_handle_notf(struct bcm_vk *vk);
+bool bcm_vk_drv_access_ok(struct bcm_vk *vk);
+void bcm_vk_set_host_alert(struct bcm_vk *vk, uint32_t bit_mask);

Are all of these prototypes really needed as they are global?

At quick glance, it really looks like most of them, if not all, do not need to be here and can be made static to the local .c file.

Quoted 105 lines
drivers/misc/bcm-vk/bcm_vk.h new file +2 -0
+
+#endif
drivers/misc/bcm-vk/bcm_vk_dev.c new file +97 -0
new file mode 100644
index 000000000000..4ce0dc0263ea
@@ -0,0 +1,1475 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2018-2020 Broadcom.
+ */
+
+#include <linux/delay.h>
+#include <linux/firmware.h>
+#include <linux/fs.h>
+#include <linux/idr.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/pci_regs.h>
+#include <linux/poll.h>
+#include <uapi/linux/misc/bcm_vk.h>
+
+#include "bcm_vk.h"
+
+#define PCI_DEVICE_ID_VALKYRIE	0x5e87
+#define PCI_DEVICE_ID_VIPER	0x5e88
+
+static DEFINE_IDA(bcm_vk_ida);
+
+enum soc_idx {
+	VALKYRIE_A0 = 0,
+	VALKYRIE_B0,
+	VIPER,
+	VK_IDX_INVALID
+};
+
+enum img_idx {
+	IMG_PRI = 0,
+	IMG_SEC,
+	IMG_PER_TYPE_MAX
+};
+
+struct load_image_entry {
+	const uint32_t image_type;
+	const char *image_name[IMG_PER_TYPE_MAX];
+};
+
+#define NUM_BOOT_STAGES 2
+/* default firmware images names */
+static const struct load_image_entry image_tab[][NUM_BOOT_STAGES] = {
+	[VALKYRIE_A0] = {
+		{VK_IMAGE_TYPE_BOOT1, {"vk_a0-boot1.bin", "vk-boot1.bin"}},
+		{VK_IMAGE_TYPE_BOOT2, {"vk_a0-boot2.bin", "vk-boot2.bin"}}
+	},
+	[VALKYRIE_B0] = {
+		{VK_IMAGE_TYPE_BOOT1, {"vk_b0-boot1.bin", "vk-boot1.bin"}},
+		{VK_IMAGE_TYPE_BOOT2, {"vk_b0-boot2.bin", "vk-boot2.bin"}}
+	},
+
+	[VIPER] = {
+		{VK_IMAGE_TYPE_BOOT1, {"vp-boot1.bin", ""}},
+		{VK_IMAGE_TYPE_BOOT2, {"vp-boot2.bin", ""}}
+	},
+};
+
+/* Location of memory base addresses of interest in BAR1 */
+/* Load Boot1 to start of ITCM */
+#define BAR1_CODEPUSH_BASE_BOOT1	0x100000
+
+/* Allow minimum 1s for Load Image timeout responses */
+#define LOAD_IMAGE_TIMEOUT_MS		(1 * MSEC_PER_SEC)
+
+/* Image startup timeouts */
+#define BOOT1_STARTUP_TIMEOUT_MS	(5 * MSEC_PER_SEC)
+#define BOOT2_STARTUP_TIMEOUT_MS	(10 * MSEC_PER_SEC)
+
+/* 1ms wait for checking the transfer complete status */
+#define TXFR_COMPLETE_TIMEOUT_MS	1
+
+/* MSIX usages */
+#define VK_MSIX_MSGQ_MAX		3
+#define VK_MSIX_NOTF_MAX		1
+#define VK_MSIX_TTY_MAX			BCM_VK_NUM_TTY
+#define VK_MSIX_IRQ_MAX			(VK_MSIX_MSGQ_MAX + VK_MSIX_NOTF_MAX + \
+					 VK_MSIX_TTY_MAX)
+#define VK_MSIX_IRQ_MIN_REQ             (VK_MSIX_MSGQ_MAX + VK_MSIX_NOTF_MAX)
+
+/* Number of bits set in DMA mask*/
+#define BCM_VK_DMA_BITS			64
+
+/* Ucode boot wait time */
+#define BCM_VK_UCODE_BOOT_US            (100 * USEC_PER_MSEC)
+/* 50% margin */
+#define BCM_VK_UCODE_BOOT_MAX_US        ((BCM_VK_UCODE_BOOT_US * 3) >> 1)
+
+/* deinit time for the card os after receiving doorbell */
+#define BCM_VK_DEINIT_TIME_MS		(2 * MSEC_PER_SEC)
+
+/*
+ * module parameters
+ */
+static bool auto_load = true;
+module_param(auto_load, bool, 0444);

drivers/misc/bcm-vk/bcm_vk_dev.c line 107

Quoted 10 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +10 -0
+MODULE_PARM_DESC(auto_load,
+		 "Load images automatically at PCIe probe time.\n");
+static uint nr_scratch_pages = VK_BAR1_SCRATCH_DEF_NR_PAGES;
+module_param(nr_scratch_pages, uint, 0444);
+MODULE_PARM_DESC(nr_scratch_pages,
+		 "Number of pre allocated DMAable coherent pages.\n");
+static uint nr_ib_sgl_blk = BCM_VK_DEF_IB_SGL_BLK_LEN;
+module_param(nr_ib_sgl_blk, uint, 0444);
+MODULE_PARM_DESC(nr_ib_sgl_blk,
+		 "Number of in-band msg blks for short SGL.\n");

Please do not use module parameters, this isn't the 1990's. We have real ways to configure drivers now that is per-device, which module parameters are not at all.

And why are any of these needed at all? Ideally a driver would "just work" and not need to be tuned at all. Who would ever modify these?

Quoted 15 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +15 -0
+
+/*
+ * alerts that could be generated from peer
+ */
+struct bcm_vk_entry const bcm_vk_peer_err[BCM_VK_PEER_ERR_NUM] = {
+	{ERR_LOG_UECC, ERR_LOG_UECC, "uecc"},
+	{ERR_LOG_SSIM_BUSY, ERR_LOG_SSIM_BUSY, "ssim_busy"},
+	{ERR_LOG_AFBC_BUSY, ERR_LOG_AFBC_BUSY, "afbc_busy"},
+	{ERR_LOG_HIGH_TEMP_ERR, ERR_LOG_HIGH_TEMP_ERR, "high_temp"},
+	{ERR_LOG_WDOG_TIMEOUT, ERR_LOG_WDOG_TIMEOUT, "wdog_timeout"},
+	{ERR_LOG_SYS_FAULT, ERR_LOG_SYS_FAULT, "sys_fault"},
+	{ERR_LOG_RAMDUMP, ERR_LOG_RAMDUMP, "ramdump"},
+	{ERR_LOG_MEM_ALLOC_FAIL, ERR_LOG_MEM_ALLOC_FAIL, "malloc_fail warn"},
+	{ERR_LOG_LOW_TEMP_WARN, ERR_LOG_LOW_TEMP_WARN, "low_temp warn"},
+	{ERR_LOG_ECC, ERR_LOG_ECC, "ecc"},

drivers/misc/bcm-vk/bcm_vk_dev.c line 132

Quoted 10 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +10 -0
+};
+
+/* alerts detected by the host */
+struct bcm_vk_entry const bcm_vk_host_err[BCM_VK_HOST_ERR_NUM] = {
+	{ERR_LOG_HOST_PCIE_DWN, ERR_LOG_HOST_PCIE_DWN, "PCIe_down"},
+	{ERR_LOG_HOST_HB_FAIL, ERR_LOG_HOST_HB_FAIL, "hb_fail"},
+	{ERR_LOG_HOST_INTF_V_FAIL, ERR_LOG_HOST_INTF_V_FAIL, "intf_ver_fail"},
+};
+
+irqreturn_t bcm_vk_notf_irqhandler(int irq, void *dev_id)

global function?

Quoted 18 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +18 -0
+{
+	struct bcm_vk *vk = dev_id;
+
+	if (!bcm_vk_drv_access_ok(vk)) {
+		dev_err(&vk->pdev->dev,
+			"Interrupt %d received when msgq not inited\n", irq);
+		goto skip_schedule_work;
+	}
+
+	/* if notification is not pending, set bit and schedule work */
+	if (test_and_set_bit(BCM_VK_WQ_NOTF_PEND, vk->wq_offload) == 0)
+		queue_work(vk->wq_thread, &vk->wq_work);
+
+skip_schedule_work:
+	return IRQ_HANDLED;
+}
+
+static int bcm_vk_intf_ver_chk(struct bcm_vk *vk)

drivers/misc/bcm-vk/bcm_vk_dev.c line 160

Quoted 10 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +10 -0
+{
+	struct device *dev = &vk->pdev->dev;
+	uint32_t reg;
+	uint16_t major, minor;
+	int ret = 0;
+
+	/* read interface register */
+	reg = vkread32(vk, BAR_0, BAR_INTF_VER);
+	major = (reg >> BAR_INTF_VER_MAJOR_SHIFT) & BAR_INTF_VER_MASK;
+	minor = reg & BAR_INTF_VER_MASK;

You are reading a major/minor device number from the hardware? Are these not the same thing as what we normally call a major/minor device node information in Linux?

If not, what are they?

If so, why does the hardware care?

Quoted 4 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +4 -0
+
+	/*
+	 * if major number is 0, it is pre-release and it would be allowed
+	 * to continue, else, check versions accordingly

drivers/misc/bcm-vk/bcm_vk_dev.c line 174

Quoted 10 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +10 -0
+	 */
+	if (!major) {
+		dev_warn(dev, "Pre-release major.minor=%d.%d - drv %d.%d\n",
+			 major, minor, SEMANTIC_MAJOR, SEMANTIC_MINOR);
+	} else if ((major != SEMANTIC_MAJOR) || (minor > SEMANTIC_MINOR)) {
+		dev_err(dev,
+			"Intf major.minor=%d.%d rejected - drv %d.%d\n",
+			major, minor, SEMANTIC_MAJOR, SEMANTIC_MINOR);
+		bcm_vk_set_host_alert(vk, ERR_LOG_HOST_INTF_V_FAIL);
+		ret = -EIO;

An I/O error did not just happen :(

Quoted 173 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +173 -0
+	} else {
+		dev_dbg(dev,
+			"Intf major.minor=%d.%d passed - drv %d.%d\n",
+			major, minor, SEMANTIC_MAJOR, SEMANTIC_MINOR);
+	}
+	return ret;
+}
+
+static void bcm_vk_log_notf(struct bcm_vk *vk,
+			    struct bcm_vk_alert *alert,
+			    struct bcm_vk_entry const *entry_tab,
+			    const uint32_t table_size)
+{
+	uint32_t i;
+	uint32_t masked_val, latched_val;
+	struct bcm_vk_entry const *entry;
+	uint32_t reg;
+	uint16_t ecc_mem_err, uecc_mem_err;
+	struct device *dev = &vk->pdev->dev;
+
+	for (i = 0; i < table_size; i++) {
+		entry = &entry_tab[i];
+		masked_val = entry->mask & alert->notfs;
+		latched_val = entry->mask & alert->flags;
+
+		if (masked_val == ERR_LOG_UECC) {
+			/*
+			 * if there is difference between stored cnt and it
+			 * is greater than threshold, log it.
+			 */
+			reg = vkread32(vk, BAR_0, BAR_CARD_ERR_MEM);
+			BCM_VK_EXTRACT_FIELD(uecc_mem_err, reg,
+					     BCM_VK_MEM_ERR_FIELD_MASK,
+					     BCM_VK_UECC_MEM_ERR_SHIFT);
+			if ((uecc_mem_err != vk->alert_cnts.uecc) &&
+			    (uecc_mem_err >= BCM_VK_UECC_THRESHOLD))
+				dev_info(dev,
+					 "ALERT! %s.%d uecc RAISED - ErrCnt %d\n",
+					 DRV_MODULE_NAME, vk->misc_devid,
+					 uecc_mem_err);
+			vk->alert_cnts.uecc = uecc_mem_err;
+		} else if (masked_val == ERR_LOG_ECC) {
+			reg = vkread32(vk, BAR_0, BAR_CARD_ERR_MEM);
+			BCM_VK_EXTRACT_FIELD(ecc_mem_err, reg,
+					     BCM_VK_MEM_ERR_FIELD_MASK,
+					     BCM_VK_ECC_MEM_ERR_SHIFT);
+			if ((ecc_mem_err != vk->alert_cnts.ecc) &&
+			    (ecc_mem_err >= BCM_VK_ECC_THRESHOLD))
+				dev_info(dev, "ALERT! %s.%d ecc RAISED - ErrCnt %d\n",
+					 DRV_MODULE_NAME, vk->misc_devid,
+					 ecc_mem_err);
+			vk->alert_cnts.ecc = ecc_mem_err;
+		} else if (masked_val != latched_val) {
+			/* print a log as info */
+			dev_info(dev, "ALERT! %s.%d %s %s\n",
+				 DRV_MODULE_NAME, vk->misc_devid, entry->str,
+				 masked_val ? "RAISED" : "CLEARED");
+		}
+	}
+}
+
+static void bcm_vk_dump_peer_log(struct bcm_vk *vk)
+{
+	struct bcm_vk_peer_log log;
+	struct bcm_vk_peer_log *log_info = &vk->peerlog_info;
+	char loc_buf[BCM_VK_PEER_LOG_LINE_MAX];
+	int cnt;
+	struct device *dev = &vk->pdev->dev;
+	uint data_offset;
+
+	memcpy_fromio(&log, vk->bar[BAR_2] + vk->peerlog_off, sizeof(log));
+
+	dev_dbg(dev, "Peer PANIC: Size 0x%x(0x%x), [Rd Wr] = [%d %d]\n",
+		log.buf_size, log.mask, log.rd_idx, log.wr_idx);
+
+	/* perform range checking for rd/wr idx */
+	if ((log.rd_idx > log_info->mask) ||
+	    (log.wr_idx > log_info->mask) ||
+	    (log.buf_size != log_info->buf_size) ||
+	    (log.mask != log_info->mask)) {
+		dev_err(dev,
+			"Corrupted Ptrs: Size 0x%x(0x%x) Mask 0x%x(0x%x) [Rd Wr] = [%d %d], skip log dump.\n",
+			log_info->buf_size, log.buf_size,
+			log_info->mask, log.mask,
+			log.rd_idx, log.wr_idx);
+		return;
+	}
+
+	cnt = 0;
+	data_offset = vk->peerlog_off + sizeof(struct bcm_vk_peer_log);
+	loc_buf[BCM_VK_PEER_LOG_LINE_MAX - 1] = '\0';
+	while (log.rd_idx != log.wr_idx) {
+		loc_buf[cnt] = vkread8(vk, BAR_2, data_offset + log.rd_idx);
+
+		if ((loc_buf[cnt] == '\0') ||
+		    (cnt == (BCM_VK_PEER_LOG_LINE_MAX - 1))) {
+			dev_err(dev, "%s", loc_buf);
+			cnt = 0;
+		} else {
+			cnt++;
+		}
+		log.rd_idx = (log.rd_idx + 1) & log.mask;
+	}
+	/* update rd idx at the end */
+	vkwrite32(vk, log.rd_idx, BAR_2,
+		  vk->peerlog_off + offsetof(struct bcm_vk_peer_log, rd_idx));
+}
+
+void bcm_vk_handle_notf(struct bcm_vk *vk)
+{
+	uint32_t reg;
+	struct bcm_vk_alert alert;
+	bool intf_down;
+	unsigned long flags;
+
+	/* handle peer alerts and then locally detected ones */
+	reg = vkread32(vk, BAR_0, BAR_CARD_ERR_LOG);
+	intf_down = BCM_VK_INTF_IS_DOWN(reg);
+	if (!intf_down) {
+		vk->peer_alert.notfs = reg;
+		bcm_vk_log_notf(vk, &vk->peer_alert, bcm_vk_peer_err,
+				ARRAY_SIZE(bcm_vk_peer_err));
+		vk->peer_alert.flags = vk->peer_alert.notfs;
+	} else {
+		/* turn off access */
+		bcm_vk_blk_drv_access(vk);
+	}
+
+	/* check and make copy of alert with lock and then free lock */
+	spin_lock_irqsave(&vk->host_alert_lock, flags);
+	if (intf_down)
+		vk->host_alert.notfs |= ERR_LOG_HOST_PCIE_DWN;
+
+	alert = vk->host_alert;
+	vk->host_alert.flags = vk->host_alert.notfs;
+	spin_unlock_irqrestore(&vk->host_alert_lock, flags);
+
+	/* call display with copy */
+	bcm_vk_log_notf(vk, &alert, bcm_vk_host_err,
+			ARRAY_SIZE(bcm_vk_host_err));
+
+	/*
+	 * If it is a sys fault or heartbeat timeout, we would like extract
+	 * log msg from the card so that we would know what is the last fault
+	 */
+	if (!intf_down &&
+	    ((vk->host_alert.flags & ERR_LOG_HOST_HB_FAIL) ||
+	     (vk->peer_alert.flags & ERR_LOG_SYS_FAULT)))
+		bcm_vk_dump_peer_log(vk);
+}
+
+static inline int bcm_vk_wait(struct bcm_vk *vk, enum pci_barno bar,
+			      uint64_t offset, uint32_t mask, uint32_t value,
+			      unsigned long timeout_ms)
+{
+	struct device *dev = &vk->pdev->dev;
+	unsigned long start_time;
+	unsigned long timeout;
+	uint32_t rd_val, boot_status;
+
+	start_time = jiffies;
+	timeout = start_time + msecs_to_jiffies(timeout_ms);
+
+	do {
+		rd_val = vkread32(vk, bar, offset);
+		dev_dbg(dev, "BAR%d Offset=0x%llx: 0x%x\n",
+			bar, offset, rd_val);
+
+		/* check for any boot err condition */
+		boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS);
+		if (boot_status & BOOT_ERR_MASK) {
+			dev_err(dev, "Boot Err 0x%x, progress 0x%x after %d ms\n",
+				(boot_status & BOOT_ERR_MASK) >> BOOT_ERR_SHIFT,

drivers/misc/bcm-vk/bcm_vk_dev.c line 357

Quoted 10 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +10 -0
+				boot_status & BOOT_PROG_MASK,
+				jiffies_to_msecs(jiffies - start_time));
+			return -EFAULT;
+		}
+
+		if (time_after(jiffies, timeout))
+			return -ETIMEDOUT;
+
+		cpu_relax();
+		cond_resched();

Why call both?

And sitting and spinning like this feels odd, but ok...

Quoted 474 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +474 -0
+	} while ((rd_val & mask) != value);
+
+	return 0;
+}
+
+static void bcm_vk_get_card_info(struct bcm_vk *vk)
+{
+	struct device *dev = &vk->pdev->dev;
+	uint32_t offset;
+	int i;
+	uint8_t *dst;
+	struct bcm_vk_card_info *info = &vk->card_info;
+
+	/* first read the offset from spare register */
+	offset = vkread32(vk, BAR_0, BAR_CARD_STATIC_INFO);
+	offset &= (pci_resource_len(vk->pdev, BAR_2 * 2) - 1);
+
+	/* based on the offset, read info to internal card info structure */
+	dst = (uint8_t *)info;
+	for (i = 0; i < sizeof(*info); i++)
+		*dst++ = vkread8(vk, BAR_2, offset++);
+
+#define CARD_INFO_LOG_FMT "version   : %x\n" \
+			  "os_tag    : %s\n" \
+			  "cmpt_tag  : %s\n" \
+			  "cpu_freq  : %d MHz\n" \
+			  "cpu_scale : %d full, %d lowest\n" \
+			  "ddr_freq  : %d MHz\n" \
+			  "ddr_size  : %d MB\n" \
+			  "video_freq: %d MHz\n"
+	dev_dbg(dev, CARD_INFO_LOG_FMT, info->version, info->os_tag,
+		info->cmpt_tag, info->cpu_freq_mhz, info->cpu_scale[0],
+		info->cpu_scale[MAX_OPP - 1], info->ddr_freq_mhz,
+		info->ddr_size_MB, info->video_core_freq_mhz);
+
+	/*
+	 * get the peer log pointer, only need the offset, and get record
+	 * of the log buffer information which would be used for checking
+	 * before dump, in case the BAR2 memory has been corrupted.
+	 */
+	vk->peerlog_off = offset;
+	memcpy_fromio(&vk->peerlog_info, vk->bar[BAR_2] + vk->peerlog_off,
+		      sizeof(vk->peerlog_info));
+	dev_dbg(dev, "Peer log: Size 0x%x(0x%x), [Rd Wr] = [%d %d]\n",
+		vk->peerlog_info.buf_size,
+		vk->peerlog_info.mask,
+		vk->peerlog_info.rd_idx,
+		vk->peerlog_info.wr_idx);
+}
+
+static void bcm_vk_get_proc_mon_info(struct bcm_vk *vk)
+{
+	struct device *dev = &vk->pdev->dev;
+	struct bcm_vk_proc_mon_info *mon = &vk->proc_mon_info;
+	uint32_t num, entry_size, offset, buf_size;
+	uint8_t *dst;
+
+	/* calculate offset which is based on peerlog offset */
+	buf_size = vkread32(vk, BAR_2,
+			    vk->peerlog_off
+			    + offsetof(struct bcm_vk_peer_log, buf_size));
+	offset = vk->peerlog_off + sizeof(struct bcm_vk_peer_log)
+		 + buf_size;
+
+	/* first read the num and entry size */
+	num = vkread32(vk, BAR_2, offset);
+	entry_size = vkread32(vk, BAR_2, offset + sizeof(num));
+
+	/* check for max allowed */
+	if (num > BCM_VK_PROC_MON_MAX) {
+		dev_err(dev, "Processing monitoring entry %d exceeds max %d\n",
+			num, BCM_VK_PROC_MON_MAX);
+		return;
+	}
+	mon->num = num;
+	mon->entry_size = entry_size;
+
+	vk->proc_mon_off = offset;
+
+	/* read it once that will capture those static info */
+	dst = (uint8_t *)&mon->entries[0];
+	offset += sizeof(num) + sizeof(entry_size);
+	memcpy_fromio(dst, vk->bar[BAR_2] + offset, num * entry_size);
+}
+
+static int bcm_vk_sync_card_info(struct bcm_vk *vk)
+{
+	uint32_t rdy_marker = vkread32(vk, BAR_1, VK_BAR1_MSGQ_DEF_RDY);
+
+	/* check for marker, but allow diags mode to skip sync */
+	if (!bcm_vk_msgq_marker_valid(vk))
+		return (rdy_marker == VK_BAR1_DIAG_RDY_MARKER ? 0 : -EINVAL);
+
+	/*
+	 * Write down scratch addr which is used for DMA. For
+	 * signed part, BAR1 is accessible only after boot2 has come
+	 * up
+	 */
+	if (vk->tdma_addr) {
+		vkwrite32(vk, (uint64_t)vk->tdma_addr >> 32, BAR_1,
+			  VK_BAR1_SCRATCH_OFF_HI);
+		vkwrite32(vk, (uint32_t)vk->tdma_addr, BAR_1,
+			  VK_BAR1_SCRATCH_OFF_LO);
+		vkwrite32(vk, nr_scratch_pages * PAGE_SIZE, BAR_1,
+			  VK_BAR1_SCRATCH_SZ_ADDR);
+	}
+
+	/* get static card info, only need to read once */
+	bcm_vk_get_card_info(vk);
+
+	/* get the proc mon info once */
+	bcm_vk_get_proc_mon_info(vk);
+
+	return 0;
+}
+
+void bcm_vk_blk_drv_access(struct bcm_vk *vk)
+{
+	int i;
+
+	/*
+	 * kill all the apps except for the process that is resetting.
+	 * If not called during reset, reset_pid will be 0, and all will be
+	 * killed.
+	 */
+	spin_lock(&vk->ctx_lock);
+
+	/* set msgq_inited to 0 so that all rd/wr will be blocked */
+	atomic_set(&vk->msgq_inited, 0);
+
+	for (i = 0; i < VK_PID_HT_SZ; i++) {
+		struct bcm_vk_ctx *ctx;
+
+		list_for_each_entry(ctx, &vk->pid_ht[i].head, node) {
+			if (ctx->pid != vk->reset_pid) {
+				dev_dbg(&vk->pdev->dev,
+					"Send kill signal to pid %d\n",
+					ctx->pid);
+				kill_pid(find_vpid(ctx->pid), SIGKILL, 1);
+			}
+		}
+	}
+	bcm_vk_tty_terminate_tty_user(vk);
+	spin_unlock(&vk->ctx_lock);
+}
+
+static void bcm_vk_buf_notify(struct bcm_vk *vk, void *bufp,
+			      dma_addr_t host_buf_addr, uint32_t buf_size)
+{
+	/* update the dma address to the card */
+	vkwrite32(vk, (uint64_t)host_buf_addr >> 32, BAR_1,
+		  VK_BAR1_DMA_BUF_OFF_HI);
+	vkwrite32(vk, (uint32_t)host_buf_addr, BAR_1,
+		  VK_BAR1_DMA_BUF_OFF_LO);
+	vkwrite32(vk, buf_size, BAR_1, VK_BAR1_DMA_BUF_SZ);
+}
+
+static int bcm_vk_load_image_by_type(struct bcm_vk *vk, uint32_t load_type,
+				     const char *filename)
+{
+	struct device *dev = &vk->pdev->dev;
+	const struct firmware *fw = NULL;
+	void *bufp = NULL;
+	size_t max_buf, offset;
+	int ret;
+	uint64_t offset_codepush;
+	uint32_t codepush;
+	uint32_t value;
+	dma_addr_t boot_dma_addr;
+	bool is_stdalone;
+
+	if (load_type == VK_IMAGE_TYPE_BOOT1) {
+		/*
+		 * After POR, enable VK soft BOOTSRC so bootrom do not clear
+		 * the pushed image (the TCM memories).
+		 */
+		value = vkread32(vk, BAR_0, BAR_BOOTSRC_SELECT);
+		value |= BOOTSRC_SOFT_ENABLE;
+		vkwrite32(vk, value, BAR_0, BAR_BOOTSRC_SELECT);
+
+		codepush = CODEPUSH_BOOTSTART + CODEPUSH_BOOT1_ENTRY;
+		offset_codepush = BAR_CODEPUSH_SBL;
+
+		/* Write a 1 to request SRAM open bit */
+		vkwrite32(vk, CODEPUSH_BOOTSTART, BAR_0, offset_codepush);
+
+		/* Wait for VK to respond */
+		ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS, SRAM_OPEN,
+				  SRAM_OPEN, LOAD_IMAGE_TIMEOUT_MS);
+		if (ret < 0) {
+			dev_err(dev, "boot1 wait SRAM err - ret(%d)\n", ret);
+			goto err_buf_out;
+		}
+
+		max_buf = SZ_256K;
+		bufp = dma_alloc_coherent(dev,
+					  max_buf,
+					  &boot_dma_addr, GFP_KERNEL);
+		if (!bufp) {
+			dev_err(dev, "Error allocating 0x%zx\n", max_buf);
+			ret = -ENOMEM;
+			goto err_buf_out;
+		}
+	} else if (load_type == VK_IMAGE_TYPE_BOOT2) {
+		codepush = CODEPUSH_BOOT2_ENTRY;
+		offset_codepush = BAR_CODEPUSH_SBI;
+
+		/* Wait for VK to respond */
+		ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS, DDR_OPEN,
+				  DDR_OPEN, LOAD_IMAGE_TIMEOUT_MS);
+		if (ret < 0) {
+			dev_err(dev, "boot2 wait DDR open error - ret(%d)\n",
+				ret);
+			goto err_buf_out;
+		}
+
+		max_buf = SZ_4M;
+		bufp = dma_alloc_coherent(dev,
+					  max_buf,
+					  &boot_dma_addr, GFP_KERNEL);
+		if (!bufp) {
+			dev_err(dev, "Error allocating 0x%zx\n", max_buf);
+			ret = -ENOMEM;
+			goto err_buf_out;
+		}
+
+		bcm_vk_buf_notify(vk, bufp, boot_dma_addr, max_buf);
+	} else {
+		dev_err(dev, "Error invalid image type 0x%x\n", load_type);
+		ret = -EINVAL;
+		goto err_buf_out;
+	}
+
+	offset = 0;
+	ret = request_partial_firmware_into_buf(&fw, filename, dev,
+						bufp, max_buf, offset);
+	if (ret) {
+		dev_err(dev, "Error %d requesting firmware file: %s\n",
+			ret, filename);
+		goto err_firmware_out;
+	}
+	dev_dbg(dev, "size=0x%zx\n", fw->size);
+	if (load_type == VK_IMAGE_TYPE_BOOT1)
+		memcpy_toio(vk->bar[BAR_1] + BAR1_CODEPUSH_BASE_BOOT1,
+			    bufp,
+			    fw->size);
+
+	dev_dbg(dev, "Signaling 0x%x to 0x%llx\n", codepush, offset_codepush);
+	vkwrite32(vk, codepush, BAR_0, offset_codepush);
+
+	if (load_type == VK_IMAGE_TYPE_BOOT1) {
+		/* wait until done */
+		ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS,
+				  BOOT1_RUNNING,
+				  BOOT1_RUNNING,
+				  BOOT1_STARTUP_TIMEOUT_MS);
+
+		is_stdalone = vkread32(vk, BAR_0, BAR_BOOT_STATUS) &
+			      BOOT_STDALONE_RUNNING;
+		if (ret && !is_stdalone) {
+			dev_err(dev,
+				"Timeout %ld ms waiting for boot1 to come up - ret(%d)\n",
+				BOOT1_STARTUP_TIMEOUT_MS, ret);
+			goto err_firmware_out;
+		} else if (is_stdalone) {
+			uint32_t reg;
+
+			reg = vkread32(vk, BAR_0, BAR_BOOT1_STDALONE_PROGRESS);
+			if ((reg & BOOT1_STDALONE_PROGRESS_MASK) ==
+				     BOOT1_STDALONE_SUCCESS) {
+				dev_info(dev, "Boot1 standalone success\n");
+				ret = 0;
+			} else {
+				dev_err(dev, "Timeout %ld ms - Boot1 standalone failure\n",
+					BOOT1_STARTUP_TIMEOUT_MS);
+				ret = -EINVAL;
+				goto err_firmware_out;
+			}
+		}
+	} else if (load_type == VK_IMAGE_TYPE_BOOT2) {
+		unsigned long timeout;
+
+		timeout = jiffies + msecs_to_jiffies(LOAD_IMAGE_TIMEOUT_MS);
+
+		/* To send more data to VK than max_buf allowed at a time */
+		do {
+			/*
+			 * Check for ack from card. when Ack is received,
+			 * it means all the data is received by card.
+			 * Exit the loop after ack is received.
+			 */
+			ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS,
+					  FW_LOADER_ACK_RCVD_ALL_DATA,
+					  FW_LOADER_ACK_RCVD_ALL_DATA,
+					  TXFR_COMPLETE_TIMEOUT_MS);
+			if (ret == 0) {
+				dev_info(dev, "Exit boot2 download\n");
+				break;
+			} else if (ret == -EFAULT) {
+				dev_err(dev, "Error detected during ACK waiting");
+				goto err_firmware_out;
+			}
+
+			/* exit the loop, if there is no response from card */
+			if (time_after(jiffies, timeout)) {
+				dev_err(dev, "Error. No reply from card\n");
+				ret = -ETIMEDOUT;
+				goto err_firmware_out;
+			}
+
+			/* Wait for VK to open BAR space to copy new data */
+			ret = bcm_vk_wait(vk, BAR_0, offset_codepush,
+					  codepush, 0,
+					  TXFR_COMPLETE_TIMEOUT_MS);
+			if (ret == 0) {
+				offset += max_buf;
+				ret = request_partial_firmware_into_buf
+						(&fw,
+						 filename,
+						 dev, bufp,
+						 max_buf,
+						 offset);
+				if (ret) {
+					dev_err(dev,
+						"Error %d requesting firmware file: %s offset: 0x%zx\n",
+						ret, filename, offset);
+					goto err_firmware_out;
+				}
+				dev_dbg(dev, "size=0x%zx\n", fw->size);
+				dev_dbg(dev, "Signaling 0x%x to 0x%llx\n",
+					codepush, offset_codepush);
+				vkwrite32(vk, codepush, BAR_0, offset_codepush);
+				/* reload timeout after every codepush */
+				timeout = jiffies +
+				    msecs_to_jiffies(LOAD_IMAGE_TIMEOUT_MS);
+			} else if (ret == -EFAULT) {
+				dev_err(dev, "Error detected waiting for transfer\n");
+				goto err_firmware_out;
+			}
+		} while (1);
+
+		/* wait for fw status bits to indicate app ready */
+		ret = bcm_vk_wait(vk, BAR_0, VK_BAR_FWSTS,
+				  VK_FWSTS_READY,
+				  VK_FWSTS_READY,
+				  BOOT2_STARTUP_TIMEOUT_MS);
+		if (ret < 0) {
+			dev_err(dev, "Boot2 not ready - ret(%d)\n", ret);
+			goto err_firmware_out;
+		}
+
+		is_stdalone = vkread32(vk, BAR_0, BAR_BOOT_STATUS) &
+			      BOOT_STDALONE_RUNNING;
+		if (!is_stdalone) {
+			ret = bcm_vk_intf_ver_chk(vk);
+			if (ret) {
+				dev_err(dev, "failure in intf version check\n");
+				goto err_firmware_out;
+			}
+
+			/*
+			 * Next, initialize Message Q if we are loading boot2.
+			 * Do a force sync
+			 */
+			ret = bcm_vk_sync_msgq(vk, true);
+			if (ret) {
+				dev_err(dev, "Boot2 Error reading comm msg Q info\n");
+				ret = -EIO;
+				goto err_firmware_out;
+			}
+
+			/* sync & channel other info */
+			ret = bcm_vk_sync_card_info(vk);
+			if (ret) {
+				dev_err(dev, "Syncing Card Info failure\n");
+				goto err_firmware_out;
+			}
+		}
+	}
+
+err_firmware_out:
+	release_firmware(fw);
+
+err_buf_out:
+	if (bufp)
+		dma_free_coherent(dev, max_buf, bufp, boot_dma_addr);
+
+	return ret;
+}
+
+static uint32_t bcm_vk_next_boot_image(struct bcm_vk *vk)
+{
+	uint32_t boot_status;
+	uint32_t fw_status;
+	uint32_t load_type = 0;  /* default for unknown */
+
+	boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS);
+	fw_status = vkread32(vk, BAR_0, VK_BAR_FWSTS);
+
+	if (!BCM_VK_INTF_IS_DOWN(boot_status) && (boot_status & SRAM_OPEN))
+		load_type = VK_IMAGE_TYPE_BOOT1;
+	else if (boot_status == BOOT1_RUNNING)
+		load_type = VK_IMAGE_TYPE_BOOT2;
+
+	/*
+	 * TO_FIX: For now, like to know what value we get everytime
+	 *         for debugging.
+	 */
+	dev_info(&vk->pdev->dev,
+		 "boot-status value for next image: 0x%x : fw-status 0x%x\n",
+		 boot_status, fw_status);
+
+	return load_type;
+}
+
+static enum soc_idx get_soc_idx(struct bcm_vk *vk)
+{
+	struct pci_dev *pdev = vk->pdev;
+	enum soc_idx idx = VK_IDX_INVALID;
+	uint32_t rev;
+	static enum soc_idx const vk_soc_tab[] = { VALKYRIE_A0, VALKYRIE_B0 };
+
+	switch (pdev->device) {
+	case PCI_DEVICE_ID_VALKYRIE:
+		/* get the chip id to decide sub-class */
+		rev = MAJOR_SOC_REV(vkread32(vk, BAR_0, BAR_CHIP_ID));
+		if (rev < ARRAY_SIZE(vk_soc_tab)) {
+			idx = vk_soc_tab[rev];
+		} else {
+			/* Default to A0 firmware for all other chip revs */
+			idx = VALKYRIE_A0;
+			dev_warn(&pdev->dev,
+				 "Rev %d not in image lookup table, default to idx=%d\n",
+				 rev, idx);
+		}
+		break;
+
+	case PCI_DEVICE_ID_VIPER:
+		idx = VIPER;
+		break;
+
+	default:
+		dev_err(&pdev->dev, "no images for 0x%x\n", pdev->device);
+	}
+	return idx;
+}
+
+static const char *get_load_fw_name(struct bcm_vk *vk,
+				    const struct load_image_entry *entry)
+{
+	const struct firmware *fw;
+	struct device *dev = &vk->pdev->dev;
+	int ret;
+	unsigned long dummy;
+	int i;
+
+	for (i = 0; i < IMG_PER_TYPE_MAX; i++) {
+		fw = NULL;
+		ret = request_partial_firmware_into_buf(&fw,
+							entry->image_name[i],
+							dev, &dummy,
+							sizeof(dummy),
+							0);
+		release_firmware(fw);
+		if (!ret)
+			return entry->image_name[i];
+	}
+	return NULL;
+}
+
+int bcm_vk_auto_load_all_images(struct bcm_vk *vk)
+{
+	int i, ret = -1;
+	enum soc_idx idx;

drivers/misc/bcm-vk/bcm_vk_dev.c line 841

Quoted 10 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +10 -0
+	struct device *dev = &vk->pdev->dev;
+	uint32_t curr_type;
+	const char *curr_name;
+
+	idx = get_soc_idx(vk);
+	if (idx == VK_IDX_INVALID)
+		goto auto_load_all_exit;
+
+	/* log a message to know the relative loading order */
+	dev_info(dev, "Load All for device %d\n", vk->misc_devid);

No need to be noise if all is going well, a driver should be quiet only if something goes wrong, then print out a message. No one wants to see this :)

dev_dbg()?

Quoted 5 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +5 -0
+
+	for (i = 0; i < NUM_BOOT_STAGES; i++) {
+		curr_type = image_tab[idx][i].image_type;
+		if (bcm_vk_next_boot_image(vk) == curr_type) {
+			curr_name = get_load_fw_name(vk, &image_tab[idx][i]);

drivers/misc/bcm-vk/bcm_vk_dev.c line 856

Quoted 10 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +10 -0
+			if (!curr_name) {
+				dev_err(dev, "No suitable firmware exists for type %d",
+					curr_type);
+				ret = -ENOENT;
+				goto auto_load_all_exit;
+			}
+			ret = bcm_vk_load_image_by_type(vk, curr_type,
+							curr_name);
+			dev_info(dev, "Auto load %s, ret %d\n",
+				 curr_name, ret);

Same here.

dev_dbg()?

Quoted 73 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +73 -0
+
+			if (ret) {
+				dev_err(dev, "Error loading default %s\n",
+					curr_name);
+				goto auto_load_all_exit;
+			}
+		}
+	}
+
+auto_load_all_exit:
+	return ret;
+}
+
+static int bcm_vk_trigger_autoload(struct bcm_vk *vk)
+{
+	if (test_and_set_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload) != 0)
+		return -EPERM;
+
+	set_bit(BCM_VK_WQ_DWNLD_AUTO, vk->wq_offload);
+	queue_work(vk->wq_thread, &vk->wq_work);
+
+	return 0;
+}
+
+static long bcm_vk_load_image(struct bcm_vk *vk,
+			      const struct vk_image __user *arg)
+{
+	struct device *dev = &vk->pdev->dev;
+	const char *image_name;
+	struct vk_image image;
+	uint32_t next_loadable;
+	enum soc_idx idx;
+	int image_idx;
+	int ret = -EPERM;
+
+	if (copy_from_user(&image, arg, sizeof(image)))
+		return -EACCES;
+
+	if ((image.type != VK_IMAGE_TYPE_BOOT1) &&
+	    (image.type != VK_IMAGE_TYPE_BOOT2)) {
+		dev_err(dev, "invalid image.type %u\n", image.type);
+		return ret;
+	}
+
+	next_loadable = bcm_vk_next_boot_image(vk);
+	if (next_loadable != image.type) {
+		dev_err(dev, "Next expected image %u, Loading %u\n",
+			next_loadable, image.type);
+		return ret;
+	}
+
+	/*
+	 * if something is pending download already.  This could only happen
+	 * for now when the driver is being loaded, or if someone has issued
+	 * another download command in another shell.
+	 */
+	if (test_and_set_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload) != 0) {
+		dev_err(dev, "Download operation already pending.\n");
+		return ret;
+	}
+
+	image_name = image.filename;
+	if (image_name[0] == '\0') {
+		/* Use default image name if NULL */
+		idx = get_soc_idx(vk);
+		if (idx == VK_IDX_INVALID)
+			goto err_idx;
+
+		/* Image idx starts with boot1 */
+		image_idx = image.type - VK_IMAGE_TYPE_BOOT1;
+		image_name = get_load_fw_name(vk, &image_tab[idx][image_idx]);
+		if (!image_name) {
+			dev_err(dev, "No suitable image found for type %d",

drivers/misc/bcm-vk/bcm_vk_dev.c line 939

Quoted 10 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +10 -0
+				image.type);
+			ret = -ENOENT;
+			goto err_idx;
+		}
+	} else {
+		/* Ensure filename is NULL terminated */
+		image.filename[sizeof(image.filename) - 1] = '\0';
+	}
+	ret = bcm_vk_load_image_by_type(vk, image.type, image_name);
+	dev_info(dev, "Load %s, ret %d\n", image_name, ret);

dev_dbg()?

Quoted 50 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +50 -0
+err_idx:
+	clear_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload);
+
+	return ret;
+}
+
+static int bcm_vk_reset_successful(struct bcm_vk *vk)
+{
+	struct device *dev = &vk->pdev->dev;
+	uint32_t fw_status, reset_reason;
+	int ret = -EAGAIN;
+
+	/*
+	 * Reset could be triggered when the card in several state:
+	 *   i)   in bootROM
+	 *   ii)  after boot1
+	 *   iii) boot2 running
+	 *
+	 * i) & ii) - no status bits will be updated.  If vkboot1
+	 * runs automatically after reset, it  will update the reason
+	 * to be unknown reason
+	 * iii) - reboot reason match + deinit done.
+	 */
+	fw_status = vkread32(vk, BAR_0, VK_BAR_FWSTS);
+	/* immediate exit if interface goes down */
+	if (BCM_VK_INTF_IS_DOWN(fw_status)) {
+		dev_err(dev, "PCIe Intf Down!\n");
+		goto reset_exit;
+	}
+
+	reset_reason = (fw_status & VK_FWSTS_RESET_REASON_MASK);
+	if ((reset_reason == VK_FWSTS_RESET_MBOX_DB) ||
+	    (reset_reason == VK_FWSTS_RESET_UNKNOWN))
+		ret = 0;
+
+	/*
+	 * if some of the deinit bits are set, but done
+	 * bit is not, this is a failure if triggered while boot2 is running
+	 */
+	if ((fw_status & VK_FWSTS_DEINIT_TRIGGERED) &&
+	    !(fw_status & VK_FWSTS_RESET_DONE))
+		ret = -EAGAIN;
+
+reset_exit:
+	dev_dbg(dev, "FW status = 0x%x ret %d\n", fw_status, ret);
+
+	return ret;
+}
+
+static long bcm_vk_reset(struct bcm_vk *vk, struct vk_reset __user *arg)

drivers/misc/bcm-vk/bcm_vk_dev.c line 999

Quoted 10 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +10 -0
+{
+	struct device *dev = &vk->pdev->dev;
+	struct vk_reset reset;
+	int ret = 0;
+	uint32_t ramdump_reset;
+	int special_reset;
+
+	if (copy_from_user(&reset, arg, sizeof(struct vk_reset)))
+		return -EFAULT;
+

No validation of user data here? Not nice...

Quoted 137 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +137 -0
+	/* check if any download is in-progress, if so return error */
+	if (test_and_set_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload) != 0) {
+		dev_err(dev, "Download operation pending - skip reset.\n");
+		return -EPERM;
+	}
+
+	ramdump_reset = vk->peer_alert.flags & ERR_LOG_RAMDUMP;
+	dev_info(dev, "Issue Reset %s\n",
+		 ramdump_reset ? "in ramdump mode" : "");
+
+	/*
+	 * The following is the sequence of reset:
+	 * - send card level graceful shut down
+	 * - wait enough time for VK to handle its business, stopping DMA etc
+	 * - kill host apps
+	 * - Trigger interrupt with DB
+	 */
+	bcm_vk_send_shutdown_msg(vk, VK_SHUTDOWN_GRACEFUL, 0, 0);
+
+	spin_lock(&vk->ctx_lock);
+	if (!vk->reset_pid) {
+		vk->reset_pid = task_pid_nr(current);
+	} else {
+		dev_err(dev, "Reset already launched by process pid %d\n",
+			vk->reset_pid);
+		ret = -EACCES;
+	}
+	spin_unlock(&vk->ctx_lock);
+	if (ret)
+		goto err_exit;
+
+	bcm_vk_blk_drv_access(vk);
+	special_reset = bcm_vk_trigger_reset(vk);
+
+	/*
+	 * Wait enough time for card os to deinit
+	 * and populate the reset reason.
+	 */
+	msleep(BCM_VK_DEINIT_TIME_MS);
+
+	if (special_reset) {
+		/* if it is special ramdump reset, return the type to user */
+		reset.arg2 = special_reset;
+		if (copy_to_user(arg, &reset, sizeof(reset)))
+			ret = -EFAULT;
+	} else {
+		ret = bcm_vk_reset_successful(vk);
+	}
+
+err_exit:
+	clear_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload);
+	return ret;
+}
+
+static int bcm_vk_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	struct bcm_vk_ctx *ctx = file->private_data;
+	struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev);
+	unsigned long pg_size;
+
+	/* only BAR2 is mmap possible, which is bar num 4 due to 64bit */
+#define VK_MMAPABLE_BAR 4
+
+	pg_size = ((pci_resource_len(vk->pdev, VK_MMAPABLE_BAR) - 1)
+		    >> PAGE_SHIFT) + 1;
+	if (vma->vm_pgoff + vma_pages(vma) > pg_size)
+		return -EINVAL;
+
+	vma->vm_pgoff += (pci_resource_start(vk->pdev, VK_MMAPABLE_BAR)
+			  >> PAGE_SHIFT);
+	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+
+	return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
+				  vma->vm_end - vma->vm_start,
+				  vma->vm_page_prot);
+}
+
+static long bcm_vk_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	long ret = -EINVAL;
+	struct bcm_vk_ctx *ctx = file->private_data;
+	struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev);
+	void __user *argp = (void __user *)arg;
+
+	dev_dbg(&vk->pdev->dev,
+		"ioctl, cmd=0x%02x, arg=0x%02lx\n",
+		cmd, arg);
+
+	mutex_lock(&vk->mutex);
+
+	switch (cmd) {
+	case VK_IOCTL_LOAD_IMAGE:
+		ret = bcm_vk_load_image(vk, argp);
+		break;
+
+	case VK_IOCTL_RESET:
+		ret = bcm_vk_reset(vk, argp);
+		break;
+
+	default:
+		break;
+	}
+
+	mutex_unlock(&vk->mutex);
+
+	return ret;
+}
+
+static const struct file_operations bcm_vk_fops = {
+	.owner = THIS_MODULE,
+	.open = bcm_vk_open,
+	.read = bcm_vk_read,
+	.write = bcm_vk_write,
+	.poll = bcm_vk_poll,
+	.release = bcm_vk_release,
+	.mmap = bcm_vk_mmap,
+	.unlocked_ioctl = bcm_vk_ioctl,
+};
+
+static int bcm_vk_on_panic(struct notifier_block *nb,
+			   unsigned long e, void *p)
+{
+	struct bcm_vk *vk = container_of(nb, struct bcm_vk, panic_nb);
+
+	bcm_to_v_reset_doorbell(vk, VK_BAR0_RESET_DB_HARD);
+
+	return 0;
+}
+
+static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+	int err;
+	int i;
+	int id;
+	int irq;
+	char name[20];
+	struct bcm_vk *vk;

drivers/misc/bcm-vk/bcm_vk_dev.c line 1146

Quoted 10 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +10 -0
+	struct device *dev = &pdev->dev;
+	struct miscdevice *misc_device;
+	uint32_t boot_status;
+
+	/* allocate vk structure which is tied to kref for freeing */
+	vk = kzalloc(sizeof(*vk), GFP_KERNEL);
+	if (!vk)
+		return -ENOMEM;
+
+	kref_init(&vk->kref);

What does this kref manage? You have a char device, and a pci device, which one is "controlling the lifespan" of this structure?

drivers/misc/bcm-vk/bcm_vk_dev.c line 1153

Quoted 7 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +7 -0
+	if (nr_ib_sgl_blk > BCM_VK_IB_SGL_BLK_MAX) {
+		dev_warn(dev, "Inband SGL blk %d limited to max %d\n",
+			 nr_ib_sgl_blk, BCM_VK_IB_SGL_BLK_MAX);
+		nr_ib_sgl_blk = BCM_VK_IB_SGL_BLK_MAX;
+	}
+	vk->ib_sgl_size = nr_ib_sgl_blk * VK_MSGQ_BLK_SIZE;
+	vk->pdev = pdev;

No reference counting of this? Brave...

Quoted 41 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +41 -0
+	mutex_init(&vk->mutex);
+
+	err = pci_enable_device(pdev);
+	if (err) {
+		dev_err(dev, "Cannot enable PCI device\n");
+		return err;
+	}
+
+	err = pci_request_regions(pdev, DRV_MODULE_NAME);
+	if (err) {
+		dev_err(dev, "Cannot obtain PCI resources\n");
+		goto err_disable_pdev;
+	}
+
+	/* make sure DMA is good */
+	err = dma_set_mask_and_coherent(&pdev->dev,
+					DMA_BIT_MASK(BCM_VK_DMA_BITS));
+	if (err) {
+		dev_err(dev, "failed to set DMA mask\n");
+		goto err_disable_pdev;
+	}
+
+	/* The tdma is a scratch area for some DMA testings. */
+	if (nr_scratch_pages) {
+		vk->tdma_vaddr = dma_alloc_coherent
+					(dev,
+					 nr_scratch_pages * PAGE_SIZE,
+					 &vk->tdma_addr, GFP_KERNEL);
+		if (!vk->tdma_vaddr) {
+			err = -ENOMEM;
+			goto err_disable_pdev;
+		}
+	}
+
+	pci_set_master(pdev);
+	pci_set_drvdata(pdev, vk);
+
+	irq = pci_alloc_irq_vectors(pdev,
+				    1,
+				    VK_MSIX_IRQ_MAX,
+				    PCI_IRQ_MSI | PCI_IRQ_MSIX);

drivers/misc/bcm-vk/bcm_vk_dev.c line 1204

Quoted 10 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +10 -0
+
+	if (irq < VK_MSIX_IRQ_MIN_REQ) {
+		dev_err(dev, "failed to get min %d MSIX interrupts, irq(%d)\n",
+			VK_MSIX_IRQ_MIN_REQ, irq);
+		err = (irq >= 0) ? -EINVAL : irq;
+		goto err_disable_pdev;
+	}
+
+	dev_info(dev, "Number of IRQs %d allocated - requested(%d).\n",
+		 irq, VK_MSIX_IRQ_MAX);

Again, please be quiet. Same for all dev_info() calls...

Quoted 38 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +38 -0
+
+	for (i = 0; i < MAX_BAR; i++) {
+		/* multiple by 2 for 64 bit BAR mapping */
+		vk->bar[i] = pci_ioremap_bar(pdev, i * 2);
+		if (!vk->bar[i]) {
+			dev_err(dev, "failed to remap BAR%d\n", i);
+			goto err_iounmap;
+		}
+	}
+
+	for (vk->num_irqs = 0;
+	     vk->num_irqs < VK_MSIX_MSGQ_MAX;
+	     vk->num_irqs++) {
+		err = devm_request_irq(dev, pci_irq_vector(pdev, vk->num_irqs),
+				       bcm_vk_msgq_irqhandler,
+				       IRQF_SHARED, DRV_MODULE_NAME, vk);
+		if (err) {
+			dev_err(dev, "failed to request msgq IRQ %d for MSIX %d\n",
+				pdev->irq + vk->num_irqs, vk->num_irqs + 1);
+			goto err_irq;
+		}
+	}
+	/* one irq for notification from VK */
+	err = devm_request_irq(dev, pci_irq_vector(pdev, vk->num_irqs),
+			       bcm_vk_notf_irqhandler,
+			       IRQF_SHARED, DRV_MODULE_NAME, vk);
+	if (err) {
+		dev_err(dev, "failed to request notf IRQ %d for MSIX %d\n",
+			pdev->irq + vk->num_irqs, vk->num_irqs + 1);
+		goto err_irq;
+	}
+	vk->num_irqs++;
+
+	for (i = 0;
+	     (i < VK_MSIX_TTY_MAX) && (vk->num_irqs < irq);
+	     i++, vk->num_irqs++) {
+		err = devm_request_irq(dev, pci_irq_vector(pdev, vk->num_irqs),
+				       bcm_vk_tty_irqhandler,

drivers/misc/bcm-vk/bcm_vk_dev.c line 1252

Quoted 10 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +10 -0
+				       IRQF_SHARED, DRV_MODULE_NAME, vk);
+		if (err) {
+			dev_err(dev, "failed request tty IRQ %d for MSIX %d\n",
+				pdev->irq + vk->num_irqs, vk->num_irqs + 1);
+			goto err_irq;
+		}
+		vk->tty[i].irq_enabled = true;
+	}
+
+	id = ida_simple_get(&bcm_vk_ida, 0, 0, GFP_KERNEL);

No locking needed? I can't remember, sorry, this might be ok...

Quoted 45 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +45 -0
+	if (id < 0) {
+		err = id;
+		dev_err(dev, "unable to get id\n");
+		goto err_irq;
+	}
+
+	vk->misc_devid = id;
+	snprintf(name, sizeof(name), DRV_MODULE_NAME ".%d", id);
+	misc_device = &vk->miscdev;
+	misc_device->minor = MISC_DYNAMIC_MINOR;
+	misc_device->name = kstrdup(name, GFP_KERNEL);
+	if (!misc_device->name) {
+		err = -ENOMEM;
+		goto err_ida_remove;
+	}
+	misc_device->fops = &bcm_vk_fops,
+
+	err = misc_register(misc_device);
+	if (err) {
+		dev_err(dev, "failed to register device\n");
+		goto err_kfree_name;
+	}
+
+	err = bcm_vk_msg_init(vk);
+	if (err) {
+		dev_err(dev, "failed to init msg queue info\n");
+		goto err_misc_deregister;
+	}
+
+	/* sync other info */
+	bcm_vk_sync_card_info(vk);
+
+	/* register for panic notifier */
+	vk->panic_nb.notifier_call = bcm_vk_on_panic;
+	atomic_notifier_chain_register(&panic_notifier_list,
+				       &vk->panic_nb);
+
+	snprintf(name, sizeof(name), KBUILD_MODNAME ".%d_ttyVK", id);
+	err = bcm_vk_tty_init(vk, name);
+	if (err)
+		goto err_unregister_panic_notifier;
+
+	/*
+	 * lets trigger an auto download.  We don't want to do it serially here
+	 * because at probing time, it is not supposed to block for a long time.

drivers/misc/bcm-vk/bcm_vk_dev.c line 1307

Quoted 10 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +10 -0
+	 */
+	boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS);
+	if (auto_load) {
+		if ((boot_status & BOOT_STATE_MASK) == BROM_RUNNING) {
+			if (bcm_vk_trigger_autoload(vk))
+				goto err_bcm_vk_tty_exit;
+		} else {
+			dev_info(dev,
+				 "Auto-load skipped - BROM not in proper state (0x%x)\n",
+				 boot_status);

dev_err()? What can someone do about this? If nothing, make it dev_dbg() please.

drivers/misc/bcm-vk/bcm_vk_dev.c line 1314

Quoted 7 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +7 -0
+		}
+	}
+
+	/* enable hb */
+	bcm_vk_hb_init(vk);
+
+	dev_info(dev, "BCM-VK:%u created, 0x%p\n", id, vk);

Again, {shh}, good code runs quiet...

Quoted 37 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +37 -0
+
+	return 0;
+
+err_bcm_vk_tty_exit:
+	bcm_vk_tty_exit(vk);
+
+err_unregister_panic_notifier:
+	atomic_notifier_chain_unregister(&panic_notifier_list,
+					 &vk->panic_nb);
+
+err_misc_deregister:
+	misc_deregister(misc_device);
+
+err_kfree_name:
+	kfree(misc_device->name);
+	misc_device->name = NULL;
+
+err_ida_remove:
+	ida_simple_remove(&bcm_vk_ida, id);
+
+err_irq:
+	for (i = 0; i < vk->num_irqs; i++)
+		devm_free_irq(dev, pci_irq_vector(pdev, i), vk);
+
+	pci_disable_msix(pdev);
+	pci_disable_msi(pdev);
+
+err_iounmap:
+	for (i = 0; i < MAX_BAR; i++) {
+		if (vk->bar[i])
+			pci_iounmap(pdev, vk->bar[i]);
+	}
+	pci_release_regions(pdev);
+
+err_disable_pdev:
+	pci_free_irq_vectors(pdev);
+	pci_disable_device(pdev);

drivers/misc/bcm-vk/bcm_vk_dev.c line 1361

Quoted 10 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +10 -0
+
+	return err;
+}
+
+void bcm_vk_release_data(struct kref *kref)
+{
+	struct bcm_vk *vk = container_of(kref, struct bcm_vk, kref);
+
+	/* use raw print, as dev is gone */
+	pr_info("BCM-VK:%d release data 0x%p\n", vk->misc_devid, vk);

Don't be noisy, this is debugging code, right?

Quoted 235 lines
drivers/misc/bcm-vk/bcm_vk_dev.c new file +114 -0
+	kfree(vk);
+}
+
+static void bcm_vk_remove(struct pci_dev *pdev)
+{
+	int i;
+	struct bcm_vk *vk = pci_get_drvdata(pdev);
+	struct miscdevice *misc_device = &vk->miscdev;
+
+	bcm_vk_hb_deinit(vk);
+
+	/*
+	 * Trigger a reset to card and wait enough time for UCODE to rerun,
+	 * which re-initialize the card into its default state.
+	 * This ensures when driver is re-enumerated it will start from
+	 * a completely clean state.
+	 */
+	bcm_vk_trigger_reset(vk);
+	usleep_range(BCM_VK_UCODE_BOOT_US, BCM_VK_UCODE_BOOT_MAX_US);
+
+	/* unregister panic notifier */
+	atomic_notifier_chain_unregister(&panic_notifier_list,
+					 &vk->panic_nb);
+
+	bcm_vk_msg_remove(vk);
+	bcm_vk_tty_exit(vk);
+
+	if (vk->tdma_vaddr)
+		dma_free_coherent(&pdev->dev, nr_scratch_pages * PAGE_SIZE,
+				  vk->tdma_vaddr, vk->tdma_addr);
+
+	/* remove if name is set which means misc dev registered */
+	if (misc_device->name) {
+		misc_deregister(misc_device);
+		kfree(misc_device->name);
+		ida_simple_remove(&bcm_vk_ida, vk->misc_devid);
+	}
+	for (i = 0; i < vk->num_irqs; i++)
+		devm_free_irq(&pdev->dev, pci_irq_vector(pdev, i), vk);
+
+	pci_disable_msix(pdev);
+	pci_disable_msi(pdev);
+
+	cancel_work_sync(&vk->wq_work);
+	destroy_workqueue(vk->wq_thread);
+	cancel_work_sync(&vk->tty_wq_work);
+	destroy_workqueue(vk->tty_wq_thread);
+
+	for (i = 0; i < MAX_BAR; i++) {
+		if (vk->bar[i])
+			pci_iounmap(pdev, vk->bar[i]);
+	}
+
+	dev_info(&pdev->dev, "BCM-VK:%d released\n", vk->misc_devid);
+
+	pci_release_regions(pdev);
+	pci_free_irq_vectors(pdev);
+	pci_disable_device(pdev);
+
+	kref_put(&vk->kref, bcm_vk_release_data);
+}
+
+static void bcm_vk_shutdown(struct pci_dev *pdev)
+{
+	struct bcm_vk *vk = pci_get_drvdata(pdev);
+	uint32_t reg, boot_stat;
+
+	reg = vkread32(vk, BAR_0, BAR_BOOT_STATUS);
+	boot_stat = reg & BOOT_STATE_MASK;
+
+	if (boot_stat == BOOT1_RUNNING) {
+		/* simply trigger a reset interrupt to park it */
+		bcm_vk_trigger_reset(vk);
+	} else if (boot_stat == BROM_NOT_RUN) {
+		int err;
+		uint16_t lnksta;
+
+		/*
+		 * The boot status only reflects boot condition since last reset
+		 * As ucode will run only once to configure pcie, if multiple
+		 * resets happen, we lost track if ucode has run or not.
+		 * Here, read the current link speed and use that to
+		 * sync up the bootstatus properly so that on reboot-back-up,
+		 * it has the proper state to start with autoload
+		 */
+		err = pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta);
+		if (!err &&
+		    (lnksta & PCI_EXP_LNKSTA_CLS) != PCI_EXP_LNKSTA_CLS_2_5GB) {
+			reg |= BROM_STATUS_COMPLETE;
+			vkwrite32(vk, reg, BAR_0, BAR_BOOT_STATUS);
+		}
+	}
+}
+
+static const struct pci_device_id bcm_vk_ids[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_VALKYRIE), },
+	{ PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_VIPER), },
+	{ }
+};
+MODULE_DEVICE_TABLE(pci, bcm_vk_ids);
+
+static struct pci_driver pci_driver = {
+	.name     = DRV_MODULE_NAME,
+	.id_table = bcm_vk_ids,
+	.probe    = bcm_vk_probe,
+	.remove   = bcm_vk_remove,
+	.shutdown = bcm_vk_shutdown,
+};
+module_pci_driver(pci_driver);
+
+MODULE_DESCRIPTION("Broadcom VK Host Driver");
+MODULE_AUTHOR("Scott Branden <[email protected]>");
+MODULE_LICENSE("GPL v2");
+MODULE_VERSION("1.0");
drivers/misc/bcm-vk/bcm_vk_msg.c new file +115 -0
new file mode 100644
index 000000000000..02ae35d42653
@@ -0,0 +1,1523 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2018-2020 Broadcom.
+ */
+
+#include <linux/delay.h>
+#include <linux/fs.h>
+#include <linux/hash.h>
+#include <linux/interrupt.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/poll.h>
+#include <linux/sizes.h>
+#include <linux/spinlock.h>
+#include <linux/timer.h>
+
+#include "bcm_vk.h"
+#include "bcm_vk_msg.h"
+#include "bcm_vk_sg.h"
+
+/* functions to manipulate the transport id in msg block */
+#define BCM_VK_MSG_Q_SHIFT	 4
+#define BCM_VK_MSG_Q_MASK	 0xF
+#define BCM_VK_MSG_ID_MASK	 0xFFF
+
+#define BCM_VK_DMA_DRAIN_MAX_MS	  2000
+
+/* number x q_size will be the max number of msg processed per loop */
+#define BCM_VK_MSG_PROC_MAX_LOOP 2
+
+/* module parameter */
+static bool hb_mon = true;
+module_param(hb_mon, bool, 0444);
+MODULE_PARM_DESC(hb_mon, "Monitoring heartbeat continuously.\n");
+static int batch_log = 1;
+module_param(batch_log, int, 0444);
+MODULE_PARM_DESC(batch_log, "Max num of logs per batch operation.\n");
+
+static bool hb_mon_is_on(void)
+{
+	return hb_mon;
+}
+
+static uint32_t get_q_num(const struct vk_msg_blk *msg)
+{
+	return (msg->trans_id & BCM_VK_MSG_Q_MASK);
+}
+
+static void set_q_num(struct vk_msg_blk *msg, uint32_t val)
+{
+	msg->trans_id = (msg->trans_id & ~BCM_VK_MSG_Q_MASK) | val;
+}
+
+static uint32_t get_msg_id(const struct vk_msg_blk *msg)
+{
+	return ((msg->trans_id >> BCM_VK_MSG_Q_SHIFT) & BCM_VK_MSG_ID_MASK);
+}
+
+static void set_msg_id(struct vk_msg_blk *msg, uint32_t val)
+{
+	msg->trans_id = (val << BCM_VK_MSG_Q_SHIFT) | get_q_num(msg);
+}
+
+static uint32_t msgq_inc(const struct bcm_vk_sync_qinfo *qinfo,
+			 uint32_t idx,
+			 uint32_t inc)
+{
+	return ((idx + inc) & qinfo->q_mask);
+}
+
+static
+struct vk_msg_blk __iomem *msgq_blk_addr(const struct bcm_vk_sync_qinfo *qinfo,
+					 uint32_t idx)
+{
+	return qinfo->q_start + (VK_MSGQ_BLK_SIZE * idx);
+}
+
+static uint32_t msgq_occupied(const struct bcm_vk_msgq __iomem *msgq,
+			      const struct bcm_vk_sync_qinfo *qinfo)
+{
+	uint32_t wr_idx, rd_idx;
+
+	wr_idx = readl_relaxed(&msgq->wr_idx);
+	rd_idx = readl_relaxed(&msgq->rd_idx);
+
+	return ((wr_idx - rd_idx) & qinfo->q_mask);
+}
+
+static
+uint32_t msgq_avail_space(const struct bcm_vk_msgq __iomem *msgq,
+			  const struct bcm_vk_sync_qinfo *qinfo)
+{
+	return (qinfo->q_size - msgq_occupied(msgq, qinfo) - 1);
+}
+
+#if defined(CONFIG_BCM_VK_QSTATS)
+
+/* Use default value of 20000 rd/wr per update */
+#if !defined(BCM_VK_QSTATS_ACC_CNT)
+#define BCM_VK_QSTATS_ACC_CNT 20000
+#endif
+
+static void bcm_vk_update_qstats(struct bcm_vk *vk,
+				 const char *tag,
+				 struct bcm_vk_qstats *qstats,
+				 uint32_t occupancy)
+{
+	struct bcm_vk_qs_cnts *qcnts = &qstats->qcnts;
+
+	if (occupancy > qcnts->max_occ) {
+		qcnts->max_occ = occupancy;
+		if (occupancy > qcnts->max_abs)
+			qcnts->max_abs = occupancy;
+	}
+

drivers/misc/bcm-vk/bcm_vk_msg.c line 125

Quoted 10 lines
drivers/misc/bcm-vk/bcm_vk_msg.c new file +10 -0
+	qcnts->acc_sum += occupancy;
+	if (++qcnts->cnt >= BCM_VK_QSTATS_ACC_CNT) {
+		/* log average and clear counters */
+		dev_info(&vk->pdev->dev,
+			 "%s[%d]: Max: [%3d/%3d] Acc %d num %d, Aver %d\n",
+			 tag, qstats->q_num,
+			 qcnts->max_occ, qcnts->max_abs,
+			 qcnts->acc_sum,
+			 qcnts->cnt,
+			 qcnts->acc_sum / qcnts->cnt);

again, quiet...

Quoted 5 lines
drivers/misc/bcm-vk/bcm_vk_msg.c new file +5 -0
+
+		qcnts->cnt = 0;
+		qcnts->max_occ = 0;
+		qcnts->acc_sum = 0;
+	}

drivers/misc/bcm-vk/bcm_vk_msg.c line 140

Quoted 10 lines
drivers/misc/bcm-vk/bcm_vk_msg.c new file +10 -0
+}
+#endif
+
+/* number of retries when enqueue message fails before returning EAGAIN */
+#define BCM_VK_H2VK_ENQ_RETRY 10
+#define BCM_VK_H2VK_ENQ_RETRY_DELAY_MS 50
+
+bool bcm_vk_drv_access_ok(struct bcm_vk *vk)
+{
+	return (!!atomic_read(&vk->msgq_inited));

So, what happens _right_ after this call? Does it become "not ok"? Are you sure you can do this?

And why an atomic variable, why do you need that?

Quoted 13 lines
drivers/misc/bcm-vk/bcm_vk_msg.c new file +13 -0
+}
+
+void bcm_vk_set_host_alert(struct bcm_vk *vk, uint32_t bit_mask)
+{
+	struct bcm_vk_alert *alert = &vk->host_alert;
+	unsigned long flags;
+
+	/* use irqsave version as this maybe called inside timer interrupt */
+	spin_lock_irqsave(&vk->host_alert_lock, flags);
+	alert->notfs |= bit_mask;
+	spin_unlock_irqrestore(&vk->host_alert_lock, flags);
+
+	if (test_and_set_bit(BCM_VK_WQ_NOTF_PEND, vk->wq_offload) == 0)

drivers/misc/bcm-vk/bcm_vk_msg.c line 163

Quoted 10 lines
drivers/misc/bcm-vk/bcm_vk_msg.c new file +10 -0
+		queue_work(vk->wq_thread, &vk->wq_work);
+}
+
+#if defined(BCM_VK_LEGACY_API)
+/*
+ * legacy does not support heartbeat mechanism
+ */
+void bcm_vk_hb_init(struct bcm_vk *vk)
+{
+	dev_info(&vk->pdev->dev, "skipped\n");

I'll just let you fix the rest of these...

I've given up now, this is a huge driver and I haven't even found the tty portions, which I have no idea why it is needed.

So can you please split this up into reviewable parts? Like add the pci device handling, then the firmware, then the misc device, and then whatever else is remaining (tty?)

It should be easy to review, as it is, this isn't at all...

thanks,

greg k-h

A message that is not in the archive (sent off-list or not yet received): [email protected]

#8 lore raw

Re: [PATCH v3 2/3] misc: bcm-vk: add Broadcom VK driver

Quoted 11 lines · Scott Branden
Quoted 11 lines

drivers/misc/bcm-vk/bcm_vk.h line 193

Quoted 8 lines
drivers/misc/bcm-vk/bcm_vk.h new file +8 -0
+struct bcm_vk_tty {
+	struct tty_port port;
+	uint32_t to_offset;	/* bar offset to use */
+	uint32_t to_size;	/* to VK buffer size */
+	uint32_t wr;		/* write offset shadow */
+	uint32_t from_offset;	/* bar offset to use */
+	uint32_t from_size;	/* from VK buffer size */
+	uint32_t rd;		/* read offset shadow */

nit, these "unit32_t" stuff really doesn't matter in the kernel, 'u32' is a better choice overall. Same for u8 and others, for this whole driver.

Quoted 3 lines

Other than personal preference, I don't understand how 'u32' is better. uint32_t follows the ANSI stdint.h.  It allows for portable code without the need to define custom u32 types.

The ANSI namespace does not work in the kernel, which is why we have our own types that pre-date those, and work properly everywhere in the kernel.

Quoted 3 lines

stdint types are used in many drivers in the linux kernel already. We would prefer to keep our code as portable as possible and use stdint types in the driver.

You aren't porting this code to other operating systems easily, please use the kernel types :)

And yes, these types are used in other parts, but when you have 25 million lines of code, some crud does slip in at times...

Quoted 7 lines
Quoted 5 lines

drivers/misc/bcm-vk/bcm_vk.h line 196

Quoted 3 lines
drivers/misc/bcm-vk/bcm_vk.h new file +3 -0
+	pid_t pid;
+	bool irq_enabled;
+	bool is_opened;		/* tracks tty open/close */

Why do you need to track this? Doesn't the tty core handle this for you?

I have tried using tty_port_kopened() and it doesn't seem to work. Will need to debug some more unless you have another suggested function to use.

You didn't answer _why_ you need to track this. A tty driver shouldn't care about this type of thing.

Quoted 13 lines
Quoted 13 lines

drivers/misc/bcm-vk/bcm_vk.h line 300

Quoted 5 lines
drivers/misc/bcm-vk/bcm_vk.h new file +5 -0
+	struct workqueue_struct *tty_wq_thread;
+	struct work_struct tty_wq_work;
+
+	/* Reference-counting to handle file operations */
+	struct kref kref;

And a kref?

What is controlling the lifetime rules of your structure?

Why a kref?

Why the tty ports?

Quoted 8 lines
Quoted 3 lines

Why the misc device?

This feels really crazy to me...

Comments mostly from Desmond here:

Yes, we have created a PCIe centric driver that combines with both a misc devices on top (for the read/write/ioctrl), and also ttys. The device sits on PCIe but we are using the misc device for accessing it.
tty is just another on top.  I don't think this is that uncommon to have a hybrid driver.

Ugh, yes, it is uncommon because those are two different things. Why do you need/want a misc driver to control a tty device? Why do you need a tty device? What really is this beast?

We got rid of the old "control path" device nodes for tty devices a long time ago, this feels like a return to that old model, is that why you are doing this?

But again, I really don't understand what this driver is trying to control/manage, so it's hard to review it without that knowledge.

Quoted 1 line

Since we have a hybrid of PCIe + misc + tty, it means that we could simultaneously have opening dev/node to read/write (multiple) + tty o going.

That's almost always a bad idea.

Quoted 2 lines

Since the struct is embedded inside the primary PCIe structure, we need a way to know when all the references are done, and then at that point we could free the primary structure. That is the reason for the kref.  On PCIe device removal, we signal the user space process to stop first, but the data structure can not be freed until the ref goes to 0.

Again, you can not have multiple reference count objects controling a single object. That way is madness and buggy and will never work properly.

You can have different objects with different lifespans, which, if you really really want to do this, is the correct way. Otherwise, stick with one object and one reference count please.

thanks,

greg k-h

author maintainer
#9 lore raw

Re: [PATCH v3 2/3] misc: bcm-vk: add Broadcom VK driver

Hi Greg,

Responses below.  I'll send out the split up patches hopefully today/tomorrow which may make it a bit easier to understand/comment on.

Quoted 19 lines · Greg Kroah-Hartman
Quoted 11 lines · Scott Branden
Quoted 11 lines

drivers/misc/bcm-vk/bcm_vk.h line 193

Quoted 8 lines
drivers/misc/bcm-vk/bcm_vk.h new file +8 -0
+struct bcm_vk_tty {
+	struct tty_port port;
+	uint32_t to_offset;	/* bar offset to use */
+	uint32_t to_size;	/* to VK buffer size */
+	uint32_t wr;		/* write offset shadow */
+	uint32_t from_offset;	/* bar offset to use */
+	uint32_t from_size;	/* from VK buffer size */
+	uint32_t rd;		/* read offset shadow */

nit, these "unit32_t" stuff really doesn't matter in the kernel, 'u32' is a better choice overall. Same for u8 and others, for this whole driver.

Quoted 3 lines

Other than personal preference, I don't understand how 'u32' is better. uint32_t follows the ANSI stdint.h.  It allows for portable code without the need to define custom u32 types.

The ANSI namespace does not work in the kernel, which is why we have our own types that pre-date those, and work properly everywhere in the kernel.

Quoted 8 lines
Quoted 3 lines

stdint types are used in many drivers in the linux kernel already. We would prefer to keep our code as portable as possible and use stdint types in the driver.

You aren't porting this code to other operating systems easily, please use the kernel types :)

And yes, these types are used in other parts, but when you have 25 million lines of code, some crud does slip in at times...

OK, will reformat.
Seems like the stdint typedefs should not have been added to the linux/types.h
back in ancient kernel times.  If they are not to be used in the kernel, then they should be wrapped in a #ifndef __KERNEL__.

Quoted 10 lines
Quoted 8 lines
Quoted 6 lines

drivers/misc/bcm-vk/bcm_vk.h line 196

Quoted 4 lines
drivers/misc/bcm-vk/bcm_vk.h new file +3 -0
+	pid_t pid;
+	bool irq_enabled;
+	bool is_opened;		/* tracks tty open/close */

Why do you need to track this? Doesn't the tty core handle this for you?

I have tried using tty_port_kopened() and it doesn't seem to work. Will need to debug some more unless you have another suggested function to use.

You didn't answer _why_ you need to track this. A tty driver shouldn't care about this type of thing.

We want to leave the data in the shared buffers coming from the card until someone is ready to read them.
So we track whether the particular tty device is open.  If the port is not open, we don't increment
our read pointer and leave the data in the buffer.  If it overflows, so be it, we'll get whatever data is in it when we open the tty device. .

Quoted 17 lines
Quoted 17 lines
Quoted 17 lines

drivers/misc/bcm-vk/bcm_vk.h line 300

Quoted 6 lines
drivers/misc/bcm-vk/bcm_vk.h new file +5 -0
+	struct workqueue_struct *tty_wq_thread;
+	struct work_struct tty_wq_work;
+
+	/* Reference-counting to handle file operations */
+	struct kref kref;

And a kref?

What is controlling the lifetime rules of your structure?

Why a kref?

Why the tty ports?

Why the misc device?

This feels really crazy to me...

Quoted 8 lines
Quoted 5 lines

Comments mostly from Desmond here:

Yes, we have created a PCIe centric driver that combines with both a misc devices on top (for the read/write/ioctrl), and also ttys. The device sits on PCIe but we are using the misc device for accessing it.
tty is just another on top.  I don't think this is that uncommon to have a hybrid driver.

Ugh, yes, it is uncommon because those are two different things. Why do you need/want a misc driver to control a tty device? Why do you need a tty device? What really is this beast?

The beast consists of a PCI card.  The PCI card communicates to the host via shared memory in BAR space and MSIX interrupts. The host communicates through shared memory in BAR space and generates mailbox interrupts.
In addition the PCI card has DMA access to host memory to access data for processing.
The misc driver handles these operations.  Multiple user space processes are accessing the misc device at the same time to perform simultaenous offload operations.  Each process opens the device and sends multiple commands with write operations and receives solicited and unsolicited responses read read operations.  In addition there are sysfs entries to collect statistics and errors. Ioctl operations are present for loading new firmware images to the card and reset operations.

In addition, the card has multiple physical UART connections to access consoles on multi processors on the card. But, in a real system there is no serial cable connected from the card to the server.  And, there could be 16 PCIe cards plugged into a server so that would make it even more unrealistic to access these consoles via a UART.

Fortunately the data sent out to each physical UART exists in a circular buffer.  These circular buffer can be access via the host in BAR space. So we added tty device nodes per UART (2 per PCIe).  These are not the misc device node, these are seperate tty device nodes that are opened and behave like tty devices.

We are not using the misc driver to control tty.
  1) The PCI driver is the foundaton of it which provides the physical interface to the card, 2) misc driver is a presentation to user space to do its regular message - this allows user to treat it as a char-device, so its like fopen/read/write/close etc. and 3) tty is an additional debug channel which could be on or off.

Please suggest how we can access the misc device and tty device nodes other than how we have implemented it in a single driver?

Quoted 4 lines

We got rid of the old "control path" device nodes for tty devices a long time ago, this feels like a return to that old model, is that why you are doing this?

I don't know what old "control path" you are referring to and what the "new" path is?

Quoted 3 lines

But again, I really don't understand what this driver is trying to control/manage, so it's hard to review it without that knowledge.

We have circular buffers in PCI memory that contain serial data.  We need to be able to open/close tty devices in linux for console operations.
And also to be able to perform operations such as lrz/sz.  This needs to have the same ability as if a physical serial cable was connected between the server and the card.  So user is able to either plug a UART cable in or open the "virtual" UART accessed over PCIe.

For misc device, the PCI memory is the physical interface, and on top it is a queue-messaging scheme for information exchange.  This is more for the control-path operations, cmd in/out etc.

Quoted 3 lines
Quoted 2 lines

Since we have a hybrid of PCIe + misc + tty, it means that we could simultaneously have opening dev/node to read/write (multiple) + tty o going.

That's almost always a bad idea.

The mulitple users is a must for us.  We have multiple individual user space processes opening the misc device and communicating to it. We also have 2 tty device nodes per card that can be opened/closed at any time.
And this is all on a PCIe card with shared memory, registers, and MSIX interrupts.
The card can be reset at any time, crash, or have PCIe rescan, etc.  User space processes need to be signalled when events detected so they don't hang. What do you suggest otherwise?

Quoted 6 lines
Quoted 3 lines

Since the struct is embedded inside the primary PCIe structure, we need a way to know when all the references are done, and then at that point we could free the primary structure. That is the reason for the kref.  On PCIe device removal, we signal the user space process to stop first, but the data structure can not be freed until the ref goes to 0.

Again, you can not have multiple reference count objects controling a single object. That way is madness and buggy and will never work properly.

We're using this driver in systems that require a high degree of stability and reliability. We have done extensive testing and don't see the bugs you are referring to. It's working properly.

Quoted 3 lines

You can have different objects with different lifespans, which, if you really really want to do this, is the correct way. Otherwise, stick with one object and one reference count please.

If we draw a hierachy, bcm_vk object encaps the misc_object + pci_object + tty_object.  The kref is for the bcm_vk object or the upper layer one.  It is to guarantee that all the sub-level objects are freed that we free this upper-level one.  I guess this is a result of the hybrid design, and this is mainly to avoid issue when we say "echo 1 > pci->remove".  The global structure under PCI will be removed, but we could not do so unless all its  misc_object/pci_object/tty_object are gone.  We did observe corner case if we don't have this that some of the apps who have opened the misc_device will seg-fault as it access the datastructure after the pcie->remove is done.  Not very common but more a corner case depending on timing.

Quoted 3 lines

thanks,

greg k-h

Regards,
 Scott

#10 lore raw

Re: [PATCH v3 2/3] misc: bcm-vk: add Broadcom VK driver

Quoted 5 lines · Scott Branden
Quoted 3 lines

Ugh, yes, it is uncommon because those are two different things. Why do you need/want a misc driver to control a tty device? Why do you need a tty device? What really is this beast?

The beast consists of a PCI card.  The PCI card communicates to the host via shared memory in BAR space and MSIX interrupts. The host communicates through shared memory in BAR space and generates mailbox interrupts.

That describes any PCI card :)

Quoted 10 lines

In addition the PCI card has DMA access to host memory to access data for processing.
The misc driver handles these operations.  Multiple user space processes are accessing the misc device at the same time to perform simultaenous offload operations.  Each process opens the device and sends multiple commands with write operations and receives solicited and unsolicited responses read read operations.  In addition there are sysfs entries to collect statistics and errors. Ioctl operations are present for loading new firmware images to the card and reset operations.

In addition, the card has multiple physical UART connections to access consoles on multi processors on the card. But, in a real system there is no serial cable connected from the card to the server.  And, there could be 16 PCIe cards plugged into a server so that would make it even more unrealistic to access these consoles via a UART.

Quoted 8 lines

Fortunately the data sent out to each physical UART exists in a circular buffer.  These circular buffer can be access via the host in BAR space. So we added tty device nodes per UART (2 per PCIe).  These are not the misc device node, these are seperate tty device nodes that are opened and behave like tty devices.

We are not using the misc driver to control tty.
  1) The PCI driver is the foundaton of it which provides the physical interface to the card, 2) misc driver is a presentation to user space to do its regular message - this allows user to treat it as a char-device, so its like fopen/read/write/close etc. and 3) tty is an additional debug channel which could be on or off.

Please suggest how we can access the misc device and tty device nodes other than how we have implemented it in a single driver?

You haven't described what the card actually is for. You describe how you have tried to hook it up to userspace, but what is the use-case here? Why even have a kernel driver at all and not do everything in a uio driver?

Creating random misc devices and tty devices implies that the device does not fit into any of the different existing kernel apis, so you need to write your own "special" one, which is always worrying.

Without actually knowing what this device is, it's hard to judge if you really should be using something existing or not.

thanks,

greg k-h

author maintainer
#11 lore raw

Re: [PATCH v3 2/3] misc: bcm-vk: add Broadcom VK driver

Hi Greg,

Quoted 22 lines · Greg Kroah-Hartman
Quoted 5 lines · Scott Branden
Quoted 3 lines

Ugh, yes, it is uncommon because those are two different things. Why do you need/want a misc driver to control a tty device? Why do you need a tty device? What really is this beast?

The beast consists of a PCI card.  The PCI card communicates to the host via shared memory in BAR space and MSIX interrupts. The host communicates through shared memory in BAR space and generates mailbox interrupts.

That describes any PCI card :)

Quoted 14 lines

In addition the PCI card has DMA access to host memory to access data for processing.
The misc driver handles these operations.  Multiple user space processes are accessing the misc device at the same time to perform simultaenous offload operations.  Each process opens the device and sends multiple commands with write operations and receives solicited and unsolicited responses read read operations.  In addition there are sysfs entries to collect statistics and errors. Ioctl operations are present for loading new firmware images to the card and reset operations.

In addition, the card has multiple physical UART connections to access consoles on multi processors on the card. But, in a real system there is no serial cable connected from the card to the server.  And, there could be 16 PCIe cards plugged into a server so that would make it even more unrealistic to access these consoles via a UART.

Fortunately the data sent out to each physical UART exists in a circular buffer.  These circular buffer can be access via the host in BAR space. So we added tty device nodes per UART (2 per PCIe).  These are not the misc device node, these are seperate tty device nodes that are opened and behave like tty devices.

Quoted 7 lines
Quoted 4 lines

We are not using the misc driver to control tty.
  1) The PCI driver is the foundaton of it which provides the physical interface to the card, 2) misc driver is a presentation to user space to do its regular message - this allows user to treat it as a char-device, so its like fopen/read/write/close etc. and 3) tty is an additional debug channel which could be on or off.

Please suggest how we can access the misc device and tty device nodes other than how we have implemented it in a single driver?

You haven't described what the card actually is for. You describe how you have tried to hook it up to userspace, but what is the use-case here?

The driver is used for real-time high performance, high throughput, low latency offload compute engine operations.
These operations are used to offload tasks that take up far too much of a CPU to be run on a host which would only be able to run a few of these tasks. It is used for such tasks as: video and image processing and crypto.
The card is capable of performing many of these task in parallel.  And, many of these cards can be plugged into the same system (say 16 cards). So instead of a host taking all its time running a single one of these tasks, 100's of these tasks can be performed instead. Of course, future cards will be able to perform even more operations, host memory bandwidth becoming a bottleneck.

The main operation of the card is to receive commands from various user space operations to offload processing of data.
This usually involves a sequence of offload operations.  The card will DMA the data from host memory and act upon it, signalling appropriate
events back to the host.  At some point the card will typically write the data back to host memory via DMA.  Various statistics and events can also
be queried via sysfs entries.  Crash and real time logs can also be retrieved.  User space processes need to be killed when the card resets or crashes so those tasks do not hang.  Console access to the card's processors is also required.

Most of this could be done differently with uio and daemons and ipc mechanisms.
The linux kernel already provides all the mechanisms that would need to be reinvented in user space.
User space data would also need to be passed from individual processes to a shared daemon, which is not desirable for security purposes vs. keeping it in the individual user space and passed to the kernel driver. The scatter gather list creation would need to somehow be done.  I don't know if this could be added to a uio driver or if we would need another driver to do such operation.
I guess you would need a custom app to replace every standard app accessing the tty device.
It is just a better model to have kernel operations in the linux kernel and a user space library that opens the device and handles the higher level message through read/write operations.  And then control and debug handled via ioctl and sysfs.

A generic linux model should exist for the offload compute performance and features we are looking for, but doesn't right now.
It would need to have the same reliability and scalability we have and could be tuned as needed to maximize performance, security, and memory usage for the particular use case as needed.

Quoted 9 lines

Why even have a kernel driver at all and not do everything in a
uio driver?

Creating random misc devices and tty devices implies that the device does not fit into any of the different existing kernel apis, so you need to write your own "special" one, which is always worrying.

Without actually knowing what this device is, it's hard to judge if you really should be using something existing or not.

Hopefully this helps explain why we have made the design decision to use a kernel module plus user space library to integrate with customer applications and systems.

Quoted 4 lines

thanks,

greg k-h

#12 lore raw

Re: [PATCH v3 2/3] misc: bcm-vk: add Broadcom VK driver

Quoted 32 lines · Scott Branden

Hi Greg,

Quoted 22 lines · Greg Kroah-Hartman
Quoted 5 lines · Scott Branden
Quoted 3 lines

Ugh, yes, it is uncommon because those are two different things. Why do you need/want a misc driver to control a tty device? Why do you need a tty device? What really is this beast?

The beast consists of a PCI card.  The PCI card communicates to the host via shared memory in BAR space and MSIX interrupts. The host communicates through shared memory in BAR space and generates mailbox interrupts.

That describes any PCI card :)

Quoted 14 lines

In addition the PCI card has DMA access to host memory to access data for processing.
The misc driver handles these operations.  Multiple user space processes are accessing the misc device at the same time to perform simultaenous offload operations.  Each process opens the device and sends multiple commands with write operations and receives solicited and unsolicited responses read read operations.  In addition there are sysfs entries to collect statistics and errors. Ioctl operations are present for loading new firmware images to the card and reset operations.

In addition, the card has multiple physical UART connections to access consoles on multi processors on the card. But, in a real system there is no serial cable connected from the card to the server.  And, there could be 16 PCIe cards plugged into a server so that would make it even more unrealistic to access these consoles via a UART.

Fortunately the data sent out to each physical UART exists in a circular buffer.  These circular buffer can be access via the host in BAR space. So we added tty device nodes per UART (2 per PCIe).  These are not the misc device node, these are seperate tty device nodes that are opened and behave like tty devices.

Quoted 7 lines
Quoted 4 lines

We are not using the misc driver to control tty.
  1) The PCI driver is the foundaton of it which provides the physical interface to the card, 2) misc driver is a presentation to user space to do its regular message - this allows user to treat it as a char-device, so its like fopen/read/write/close etc. and 3) tty is an additional debug channel which could be on or off.

Please suggest how we can access the misc device and tty device nodes other than how we have implemented it in a single driver?

You haven't described what the card actually is for. You describe how you have tried to hook it up to userspace, but what is the use-case here?

Quoted 6 lines

The driver is used for real-time high performance, high throughput, low latency offload compute engine operations.
These operations are used to offload tasks that take up far too much of a CPU to be run on a host which would only be able to run a few of these tasks. It is used for such tasks as: video and image processing and crypto.
The card is capable of performing many of these task in parallel.  And, many of these cards can be plugged into the same system (say 16 cards). So instead of a host taking all its time running a single one of these tasks, 100's of these tasks can be performed instead. Of course, future cards will be able to perform even more operations, host memory bandwidth becoming a bottleneck.

So for off-load engines, as I mentioned in my patch review, we want to also see the userspace library/applications/whatever as well, so that we can properly review this type of thing. Otherwise it's a mess...

Quoted 4 lines

The main operation of the card is to receive commands from various user space operations to offload processing of data.
This usually involves a sequence of offload operations.  The card will DMA the data from host memory and act upon it, signalling appropriate
events back to the host.  At some point the card will typically write the data back to host memory via DMA.  Various statistics and events can also
be queried via sysfs entries.  Crash and real time logs can also be retrieved.  User space processes need to be killed when the card resets or crashes so those tasks do not hang.  Console access to the card's processors is also required.

What do you mean by a "console access"? Is Linux running in there too?

Anyway, if you want a TTY device to talk to the cpu/whatever, then disconnect the tty device logic from your misc device logic, as you can't keep the two bound together for obvious lifetime reasons. I mention that in the patch review as well.

thanks,

greg k-h

author maintainer
#13 lore raw

Re: [PATCH v3 2/3] misc: bcm-vk: add Broadcom VK driver

Quoted 39 lines · Greg Kroah-Hartman
Quoted 32 lines · Scott Branden

Hi Greg,

Quoted 22 lines · Greg Kroah-Hartman
Quoted 5 lines · Scott Branden
Quoted 3 lines

Ugh, yes, it is uncommon because those are two different things. Why do you need/want a misc driver to control a tty device? Why do you need a tty device? What really is this beast?

The beast consists of a PCI card.  The PCI card communicates to the host via shared memory in BAR space and MSIX interrupts. The host communicates through shared memory in BAR space and generates mailbox interrupts.

That describes any PCI card :)

Quoted 14 lines

In addition the PCI card has DMA access to host memory to access data for processing.
The misc driver handles these operations.  Multiple user space processes are accessing the misc device at the same time to perform simultaenous offload operations.  Each process opens the device and sends multiple commands with write operations and receives solicited and unsolicited responses read read operations.  In addition there are sysfs entries to collect statistics and errors. Ioctl operations are present for loading new firmware images to the card and reset operations.

In addition, the card has multiple physical UART connections to access consoles on multi processors on the card. But, in a real system there is no serial cable connected from the card to the server.  And, there could be 16 PCIe cards plugged into a server so that would make it even more unrealistic to access these consoles via a UART.

Fortunately the data sent out to each physical UART exists in a circular buffer.  These circular buffer can be access via the host in BAR space. So we added tty device nodes per UART (2 per PCIe).  These are not the misc device node, these are seperate tty device nodes that are opened and behave like tty devices.

Quoted 7 lines
Quoted 4 lines

We are not using the misc driver to control tty.
  1) The PCI driver is the foundaton of it which provides the physical interface to the card, 2) misc driver is a presentation to user space to do its regular message - this allows user to treat it as a char-device, so its like fopen/read/write/close etc. and 3) tty is an additional debug channel which could be on or off.

Please suggest how we can access the misc device and tty device nodes other than how we have implemented it in a single driver?

You haven't described what the card actually is for. You describe how you have tried to hook it up to userspace, but what is the use-case here?

Quoted 6 lines

The driver is used for real-time high performance, high throughput, low latency offload compute engine operations.
These operations are used to offload tasks that take up far too much of a CPU to be run on a host which would only be able to run a few of these tasks. It is used for such tasks as: video and image processing and crypto.
The card is capable of performing many of these task in parallel.  And, many of these cards can be plugged into the same system (say 16 cards). So instead of a host taking all its time running a single one of these tasks, 100's of these tasks can be performed instead. Of course, future cards will be able to perform even more operations, host memory bandwidth becoming a bottleneck.

Quoted 3 lines

So for off-load engines, as I mentioned in my patch review, we want to also see the userspace library/applications/whatever as well, so that we can properly review this type of thing. Otherwise it's a mess...

Replied in other email and published vkil:
https://github.com/Broadcom/vkil/

Quoted 6 lines
Quoted 5 lines

The main operation of the card is to receive commands from various user space operations to offload processing of data.
This usually involves a sequence of offload operations.  The card will DMA the data from host memory and act upon it, signalling appropriate
events back to the host.  At some point the card will typically write the data back to host memory via DMA.  Various statistics and events can also
be queried via sysfs entries.  Crash and real time logs can also be retrieved.  User space processes need to be killed when the card resets or crashes so those tasks do not hang.  Console access to the card's processors is also required.

What do you mean by a "console access"?

The card's processors have physical UARTs on the VK card as described in the other email. Some of the processors' console programs can be accessed via "UART" emulated over PCIe interface.

Quoted 1 line

Is Linux running in there too?

One of the processors with a console does not run Linux.
Linux is currently running on one of the other processor complexes.  Before Linux boots low level boot code, ATF, etc can output to the same UART circular buffer.

Quoted 5 lines

Anyway, if you want a TTY device to talk to the cpu/whatever, then disconnect the tty device logic from your misc device logic, as you can't keep the two bound together for obvious lifetime reasons. I mention that in the patch review as well.

I described the lifetime dependencies of the tty device on the image loading and internal card resets. It would take some time to figure out how to create the same dependency between multiple drivers. If it helps, I can drop the tty patches from the current patch series and work on the tty afterwards.

Quoted 4 lines

thanks,

greg k-h

author maintainer
#14 lore raw

Re: [PATCH v3 2/3] misc: bcm-vk: add Broadcom VK driver

In reply to Scott Branden
Quoted 6 lines · Scott Branden

Hi Greg,

Responses below.  I'll send out the split up patches hopefully today/tomorrow which may make it a bit easier to understand/comment on.

I have split up the patches and sent out patch v4.
Hopefully this is more understandable split up and you understand the rationale for a misc driver and a tty driver to support this PCIe card.  Also note that the series needs Kee's request_partial_firmware_into_buf patch series to compile
as described in the cover letter.  It would be great if patches 1-3 could be reviewed and integrated as then there is a user of request_partial_firmware_info_buf in the upstream kernel.

Best Regards,
 Scott

#15 lore raw

Re: [PATCH v3 2/3] misc: bcm-vk: add Broadcom VK driver

Quoted 5 lines · Scott Branden
Quoted 2 lines · Scott Branden

Hi Greg,

Quoted 7 lines
Quoted 4 lines

Responses below.  I'll send out the split up patches hopefully today/tomorrow which may make it a bit easier to understand/comment on.

I have split up the patches and sent out patch v4.
Hopefully this is more understandable split up and you understand the rationale for a misc driver and a tty driver to support this PCIe card.

I still don't, but I'll respond to the patch, as it's easier to explain that there :)

thanks,

greg k-h

author maintainer
#16 lore raw

PATCH v3 3/3 MAINTAINERS: bcm-vk: add maintainer for Broadcom VK Driver

In reply to Scott Branden

Add maintainer entry for new Broadcom VK Driver

---
 MAINTAINERS | 7 +++++++
 1 file changed, 7 insertions(+)
MAINTAINERS +7 -0
index 3b186ade3597..5ce9cf6b52d0 100644
@@ -3699,6 +3699,13 @@ L:	[email protected]
 S:	Supported
 F:	drivers/net/ethernet/broadcom/tg3.*
 
+BROADCOM VK DRIVER
+M:	Scott Branden <[email protected]>
+L:	[email protected]
+S:	Supported
+F:	drivers/misc/bcm-vk/
+F:	include/uapi/linux/misc/bcm_vk.h
+
 BROCADE BFA FC SCSI DRIVER
 M:	Anil Gurumurthy <[email protected]>
 M:	Sudarsana Kalluru <[email protected]>
Signature · 1 line
2.17.1