megaraid_sas: NVME passthru command support

Feature - NVME pass-through via MFI interface.
Current MR product support different encapsulation via MFI frame work.

NVME native command should be framed by application and it should be embedded in MFI as payload.
Driver will provide interface to send MFI frame along with payload (In this case, payload is NVME native command) to the Firmware. Driver already had such generic interface for SATA, SMP pass-through. Changes are explained below.

1. Driver will pass MFI command to the FW, if FW support NVME encapsulated
processing (not all SAS3.5 firmware supports this feature).
2. Driver expose sysfs entry ?support_nvme_encapsulation?. This is required for backward compatibility for applications using earlier driver versions
that did not process IOCTL frames and could result in host hang.
This is already fixed as part of
commit 82add4e1b354 ("scsi: megaraid_sas: Incorrect processing of IOCTL frames for SMP/STP commands")

---
 drivers/scsi/megaraid/megaraid_sas.h        | 30 +++++++++++++++++++++++++----
 drivers/scsi/megaraid/megaraid_sas_base.c   | 30 ++++++++++++++++++++++++++++-
 drivers/scsi/megaraid/megaraid_sas_fusion.c |  7 +++++++
 3 files changed, 62 insertions(+), 5 deletions(-)

index 14b82c0b0461..d557ff03eb1d 100644
@@ -197,6 +197,7 @@ enum MFI_CMD_OP {
 	MFI_CMD_ABORT		= 0x6,
 	MFI_CMD_SMP		= 0x7,
 	MFI_CMD_STP		= 0x8,
+	MFI_CMD_NVME		= 0x9,
 	MFI_CMD_OP_COUNT,
 	MFI_CMD_INVALID		= 0xff
 };

@@ -1352,7 +1353,13 @@ struct megasas_ctrl_info {
 
 	struct {
 	#if defined(__BIG_ENDIAN_BITFIELD)
-		u16 reserved:8;
+		u16 reserved:2;
+		u16 support_nvme_passthru:1;
+		u16 support_pl_debug_info:1;
+		u16 support_flash_comp_info:1;
+		u16 support_host_info:1;
+		u16 support_dual_fw_update:1;
+		u16 support_ssc_rev3:1;
 		u16 fw_swaps_bbu_vpd_info:1;
 		u16 support_pd_map_target_id:1;
 		u16 support_ses_ctrl_in_multipathcfg:1;

@@ -1377,7 +1384,19 @@ struct megasas_ctrl_info {
 		 *  provide the data in little endian order
 		 */
 		u16 fw_swaps_bbu_vpd_info:1;
-		u16 reserved:8;
+		u16 support_ssc_rev3:1;
+		/* FW supports CacheCade 3.0, only one SSCD creation allowed */
+		u16 support_dual_fw_update:1;
+		/* FW supports dual firmware update feature */
+		u16 support_host_info:1;
+		/* FW supports MR_DCMD_CTRL_HOST_INFO_SET/GET */
+		u16 support_flash_comp_info:1;
+		/* FW supports MR_DCMD_CTRL_FLASH_COMP_INFO_GET */
+		u16 support_pl_debug_info:1;
+		/* FW supports retrieval of PL debug information through apps */
+		u16 support_nvme_passthru:1;
+		/* FW supports NVMe passthru commands */
+		u16 reserved:2;
 	#endif
 		} adapter_operations4;
 	u8 pad[0x800 - 0x7FE]; /* 0x7FE pad to 2K for expansion */

@@ -1630,7 +1649,8 @@ union megasas_sgl_frame {
 typedef union _MFI_CAPABILITIES {
 	struct {
 #if   defined(__BIG_ENDIAN_BITFIELD)
-	u32     reserved:18;
+	u32     reserved:17;
+	u32	support_nvme_passthru:1;
 	u32     support_64bit_mode:1;
 	u32 support_pd_map_target_id:1;
 	u32     support_qd_throttling:1;

@@ -1660,7 +1680,8 @@ typedef union _MFI_CAPABILITIES {
 	u32     support_qd_throttling:1;
 	u32	support_pd_map_target_id:1;
 	u32     support_64bit_mode:1;
-	u32     reserved:18;
+	u32	support_nvme_passthru:1;
+	u32     reserved:17;
 #endif
 	} mfi_capabilities;
 	__le32		reg;

@@ -2268,6 +2289,7 @@ struct megasas_instance {
 	u32 nvme_page_size;
 	u8 adapter_type;
 	bool consistent_mask_64bit;
+	bool support_nvme_passthru;
 };
 struct MR_LD_VF_MAP {
 	u32 size;

index d92279eec8f8..0f1d88f272be 100644
@@ -181,6 +181,7 @@ static DECLARE_WAIT_QUEUE_HEAD(megasas_poll_wait);
 static u32 support_poll_for_event;
 u32 megasas_dbg_lvl;
 static u32 support_device_change;
+static bool support_nvme_encapsulation;
 
 /* define lock for aen poll */
 spinlock_t poll_aen_lock;

@@ -3334,6 +3335,7 @@ megasas_complete_cmd(struct megasas_instance *instance, struct megasas_cmd *cmd,
 
 	case MFI_CMD_SMP:
 	case MFI_CMD_STP:
+	case MFI_CMD_NVME:
 		megasas_complete_int_cmd(instance, cmd);
 		break;
 

@@ -4721,6 +4723,8 @@ megasas_get_ctrl_info(struct megasas_instance *instance)
 			ci->adapterOperations3.useSeqNumJbodFP;
 		instance->support_morethan256jbod =
 			ci->adapter_operations4.support_pd_map_target_id;
+		instance->support_nvme_passthru =
+			ci->adapter_operations4.support_nvme_passthru;
 
 		/*Check whether controller is iMR or MR */
 		instance->is_imr = (ci->memory_size ? 0 : 1);

@@ -4737,6 +4741,8 @@ megasas_get_ctrl_info(struct megasas_instance *instance)
 			instance->disableOnlineCtrlReset ? "Disabled" : "Enabled");
 		dev_info(&instance->pdev->dev, "Secure JBOD support\t: %s\n",
 			instance->secure_jbod_support ? "Yes" : "No");
+		dev_info(&instance->pdev->dev, "NVMe passthru support\t: %s\n",
+			 instance->support_nvme_passthru ? "Yes" : "No");
 		break;
 
 	case DCMD_TIMEOUT:

@@ -7110,7 +7116,9 @@ megasas_mgmt_fw_ioctl(struct megasas_instance *instance,
 		return -EINVAL;
 	}
 
-	if (ioc->frame.hdr.cmd >= MFI_CMD_OP_COUNT) {
+	if ((ioc->frame.hdr.cmd >= MFI_CMD_OP_COUNT) ||
+	    ((ioc->frame.hdr.cmd == MFI_CMD_NVME) &&
+	    !instance->support_nvme_passthru)) {
 		dev_err(&instance->pdev->dev,
 			"Received invalid ioctl command 0x%x\n",
 			ioc->frame.hdr.cmd);

@@ -7580,6 +7588,14 @@ static ssize_t dbg_lvl_store(struct device_driver *dd, const char *buf,
 }
 static DRIVER_ATTR_RW(dbg_lvl);
 
+static ssize_t
+support_nvme_encapsulation_show(struct device_driver *dd, char *buf)
+{
+	return sprintf(buf, "%u\n", support_nvme_encapsulation);
+}
+
+static DRIVER_ATTR_RO(support_nvme_encapsulation);
+
 static inline void megasas_remove_scsi_device(struct scsi_device *sdev)
 {
 	sdev_printk(KERN_INFO, sdev, "SCSI device is removed\n");

@@ -7768,6 +7784,7 @@ static int __init megasas_init(void)
 
 	support_poll_for_event = 2;
 	support_device_change = 1;
+	support_nvme_encapsulation = true;
 
 	memset(&megasas_mgmt_info, 0, sizeof(megasas_mgmt_info));
 

@@ -7817,8 +7834,17 @@ static int __init megasas_init(void)
 	if (rval)
 		goto err_dcf_support_device_change;
 
+	rval = driver_create_file(&megasas_pci_driver.driver,
+				  &driver_attr_support_nvme_encapsulation);
+	if (rval)
+		goto err_dcf_support_nvme_encapsulation;
+
 	return rval;
 
+err_dcf_support_nvme_encapsulation:
+	driver_remove_file(&megasas_pci_driver.driver,
+			   &driver_attr_support_device_change);
+
 err_dcf_support_device_change:
 	driver_remove_file(&megasas_pci_driver.driver,
 			   &driver_attr_dbg_lvl);

@@ -7851,6 +7877,8 @@ static void __exit megasas_exit(void)
 	driver_remove_file(&megasas_pci_driver.driver,
 			   &driver_attr_release_date);
 	driver_remove_file(&megasas_pci_driver.driver, &driver_attr_version);
+	driver_remove_file(&megasas_pci_driver.driver,
+			   &driver_attr_support_nvme_encapsulation);
 
 	pci_unregister_driver(&megasas_pci_driver);
 	unregister_chrdev(megasas_mgmt_majorno, "megaraid_sas_ioctl");

index 0a85f3c48ef6..f3940b6af290 100644
@@ -1080,6 +1080,7 @@ megasas_ioc_init_fusion(struct megasas_instance *instance)
 
 	drv_ops->mfi_capabilities.support_qd_throttling = 1;
 	drv_ops->mfi_capabilities.support_pd_map_target_id = 1;
+	drv_ops->mfi_capabilities.support_nvme_passthru = 1;
 
 	if (instance->consistent_mask_64bit)
 		drv_ops->mfi_capabilities.support_64bit_mode = 1;

@@ -3994,7 +3995,13 @@ void megasas_refire_mgmt_cmd(struct megasas_instance *instance)
 				result = RETURN_CMD;
 
 			break;
+		case MFI_CMD_NVME:
+			if (!instance->support_nvme_passthru) {
+				cmd_mfi->frame->hdr.cmd_status = MFI_STAT_INVALID_CMD;
+				result = COMPLETE_CMD;
+			}
 
+			break;
 		default:
 			break;
 		}

2.14.1

NAK. Please implement the same ioctl interfaces as the nvme driver instead of inventing your own incomaptible one.

Chris -

Overall NVME support behind MR controller is really a SCSI device. On top of that, for MegaRaid, NVME device can be part of Virtual Disk and those drive will not be exposed to the driver. User application may like to talk to hidden NVME devices (part of VDs). This patch will extend the existing interface for megaraid product in the same way it is currently supported for other protocols like SMP, SATA pass-through.

Example - Current smartmon is using megaraid.h (MFI headers) to send SATA pass-through.

https://github.com/mirror/smartmontools/blob/master/megaraid.h

Any open source application is aware of above interface can extend the similar support for NVME drives. I agree that current nvme-cli type interface is not going to be supported using this method. In current patch, driver processing is very limited since most of the work is handled in application + FW.

NVME behind MR controller is not really NVME device to the operating system at block layer. Considering this, do you agree or still foresee any issues ?

Kashyap

-----Original Message-----
From: Christoph Hellwig [mailto:[email protected]]
Sent: Monday, January 8, 2018 3:36 PM
To: Shivasharan S
Cc: linux-scsi at vger.kernel.org; sumit.saxena at broadcom.com; linux-nvme at lists.infradead.org; kashyap.desai at broadcom.com
Subject: Re: [PATCH 13/14] megaraid_sas: NVME passthru command support

NAK. Please implement the same ioctl interfaces as the nvme driver instead of inventing your own incomaptible one.

And that is exactly the example of why we should have never allowed megaraid any private passthrough ioctls to start with.

Christoph,
Have you tried to do any serious work with <linux/nvme_ioctl.h> and say compared it with FreeBSD and Microsoft's approach? No prize for guessing which one is worst (and least extensible). Looks like the Linux pass-through was at the end of a ToDo list and was "designed" at 5 a.m in the morning.

RAID cards need a pass-through that allows them to address one of many physical disks behind the virtual disk presented to OS. Pass-throughs need to have uncommited room for extra parameters that will be passed through as-is to the RAID LLD.

So until Christoph gives an example of how that can be done with <linux/nvme_ioctl.h> then I would like to see Christoph's objection ignored.

And as a maintainer of smartmontools, I would like to point out that pretty well all supported RAIDs, on all platforms need specialized pass-through code. Start by looking at os_linux.cpp and then at the other OSes. And now smartmontools supports NVMe on most platforms and at the pass-through level, it is just another one, and not a particularly clean one.

IMO Intel had their chance on the pass-through front, and blew it. It is now too late to fix it and that job (impossible ?) should not fall to MegaRaid maintainers.

Douglas Gilbert

What the heck are you talking about? FreeBSD's NVMe passthrough is near identical to Linux, and Linux's existed years prior.

You're not even touching the nvme subsystem, so why are you copying the linux-nvme mailing list to help you with a non-NVMe device? Please take your ignorant and dubious claims elsewhere.

your

Keith -

As we discussed for mpt3sas driver NVME driver support, there was request to add linux-nvme at lists.infradead.org for NVME related discussion.
https://marc.info/?l=linux-kernel&m=149874673729467&w=2

As you mentioned, we are not touching NVME subsystem, we can skip to add NVME mailing list for future submission w.r.t NVME drive behind MR (megaraid_sas) and HBA (mpt3sas).
All the NVME drives behind MegaRaid controller is SCSI device irrespective of transport.

Kashyap

Doug - As you mentioned, I notice the same. This type of issue is common for all RAID controllers vendors.
Whatever Christoph mentioned about NVMe type API to be used is possible, but may need extra hit in firmware side to convert Linux NVME API to FW specific OR deal the same in driver.
It may come with it's own pros/cons. Also may not fulfil the end goal. For other platforms, we still have to depend upon specialized pass-through code. So having said that Firmware of RAID cannot use only one interface for pass-through and they have to choose specialized pass-through code.

NVME-CLI interface is designed for NVME drives attached to block layer. MegaRaid product is design to keep NVME protocol abstracted (much like SATA drives behind SAS controller) and attach those drives/virtual disk to SCSI layer.

If upstream community like to enhance nvme-cli type interface in megaraid_sas driver, we may have to come up with one more layer in megaraid_sas driver to convert NVME-API to specialized pass-through code. It is really not simple to fit into existing design as NVME-CLI/API is considering NVME drive associated with nvme.ko modules (/dev/nvmeX). Also we don't have many sysfs entries nvme-cli is looking for NVME device etc.. We don't have way to talk to Physical disks which is part of VD etc..

Specialized pass-through code is better to extend in application like smartmontools etc.

It seems to me like there is a contradiction in the above paragraph: if some NVMe devices are not exposed to the driver, how can a user space application ever send NVMe commands to it?

Anyway, has it been considered to implement the NVMe support as an NVMe transport driver? The upstream kernel already supports NVMe communication with NVMe PCI devices, NVMe over RDMA and NVMe over FC. If communication to the NVMe devices behind the MegaRaid controller would be implemented as an NVMe transport driver then all functionality of the Linux NVMe driver could be reused, including its sysfs entries.

Bart.

I think that he meant that the NVMe physical devices (e.g. SSDs) are not exposed to the upper layers (e.g. the SCSI mid-layer and above). The SCSI subsystem has a no_uld_attach device flag that lets a LLD attach physical devices but the sd driver and hence the block layer do not "see" them. The idea is that maintenance programs like smartmontools can use them via the bsg or sg drivers. The Megaraid driver code does not seem to use no_uld_attach. Does the NVMe subsystem have similar "generic" (i.e. non-block) devices accessible to the user space?

Broadcom already sell "SAS" HBAs that have "tri-mode" phys. That is a phy that can connect to a SAS device (e.g. a SAS expander), a SATA device or a NVMe device. Now if I was Broadcom designing a 24 Gbps SAS-4 next generation expander I would be thinking of using those tri-mode phys on it. But then there is a problem, SAS currently supports 3 protocols: SSP (for SCSI storage and enclosure management (SES)), STP (for SATA storage ) and SMP (for expander management). The problem is how those NVMe commands, status and data cross the wire between the OS HBA (or MegaRaid type controller) and an expander. Solving that might need some lateral thinking.

On one hand the NVM Express folks seem to have shelved the idea of a SCSI to NVMe Translation Layer (SNTL) and have not updated an old white paper on the subject. Currently there is no SNTL on Linux (there was but it was removed) or FreeBSD but there is one on Windows.

On the other hand I'm informed that recently the same body accepted the SES-3 standard pretty much as-is. That is done with the addition of SES Send and SES Receive commands to NVME-MI. The library under sg_ses has already been modified to use them (by implementing a specialized SNTL).

Doug Gilbert

Bart et al,
Broadcom's Tri-mode HBAs and MegaRAID controllers are capable of connecting with SAS, SATA, NVMe drives, SAS expanders and PCIe switches (with NVMe drives connected behind that) and are capable of creating RAID volumes on top of similar family of drives. In the case of RAID controllers, all of those drives and RAID volumes are exposed to the OS as generic SCSI devices and in the case of HBA only for SAS and SATA the topology is exposed to OS through SAS transport layer and NVMe drives are exposed as generic SCSI devices. The SCSI CDB to specific packet (SATA frames, SSP frames or NVMe) translation occurs in the hardware/firmware. For the OS driver, the interface to interact is common across all the type of devices and it is MPI SCSI IO Request.

The NVMe passthru support added in this patch is only for management purpose and will let Broadcom specific management applications to send some direct NVMe commands to the hardware/firmware solely for management purpose. For normal READ/WRITE I/O the preferred path is to issue SCSI command to our hardware/firmware and let it translate to the NVMe.

We have many architectural constraints to directly expose NVMe drives to NVMe subsystem for normal I/O usage and management usage and hence we prefer not to go down the path.

This patch is just addition of new feature for our management applications (which are common across many OSes) to access a specific type of MPI command to manage NVMe drives connected behind our HBAs (which are non-standard) in a vendor specific way, hence we think this patch is valid to be accepted to megaraid driver. Please let us know if more details are required on the tri-mode controllers.

Thanks
Sathya

-----Original Message-----
From: Linux-nvme [mailto:[email protected]] On Behalf Of Douglas Gilbert
Sent: Wednesday, January 10, 2018 1:06 PM
To: Bart Van Assche; hch at infradead.org; kashyap.desai at broadcom.com; shivasharan.srikanteshwara at broadcom.com
Cc: sumit.saxena at broadcom.com; peter.rivera at broadcom.com; linux-nvme at lists.infradead.org; linux-scsi at vger.kernel.org
Subject: Re: [PATCH 13/14] megaraid_sas: NVME passthru command support

pass-through.

I think that he meant that the NVMe physical devices (e.g. SSDs) are not exposed to the upper layers (e.g. the SCSI mid-layer and above). The SCSI subsystem has a no_uld_attach device flag that lets a LLD attach physical devices but the sd driver and hence the block layer do not "see" them. The idea is that maintenance programs like smartmontools can use them via the bsg or sg drivers. The Megaraid driver code does not seem to use no_uld_attach. Does the NVMe subsystem have similar "generic" (i.e. non-block) devices accessible to the user space?

sysfs entries.

Broadcom already sell "SAS" HBAs that have "tri-mode" phys. That is a phy that can connect to a SAS device (e.g. a SAS expander), a SATA device or a NVMe device. Now if I was Broadcom designing a 24 Gbps SAS-4 next generation expander I would be thinking of using those tri-mode phys on it. But then there is a problem, SAS currently supports 3 protocols: SSP (for SCSI storage and enclosure management (SES)), STP (for SATA storage ) and SMP (for expander management). The problem is how those NVMe commands, status and data cross the wire between the OS HBA (or MegaRaid type controller) and an expander. Solving that might need some lateral thinking.

On one hand the NVM Express folks seem to have shelved the idea of a SCSI to NVMe Translation Layer (SNTL) and have not updated an old white paper on the subject. Currently there is no SNTL on Linux (there was but it was removed) or FreeBSD but there is one on Windows.

On the other hand I'm informed that recently the same body accepted the SES-3 standard pretty much as-is. That is done with the addition of SES Send and SES Receive commands to NVME-MI. The library under sg_ses has already been modified to use them (by implementing a specialized SNTL).

Doug Gilbert

_______________________________________________
Linux-nvme mailing list
Linux-nvme at lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-nvme

So even when used as a RAID member, there will be a device handle at /dev/sdX for each NVMe device the megaraid controller manages?

/dev/sdX for each NVMe device the megaraid controller manages? In megaraid controller, you can expose bare NVMe drives and RAID volumes created out of NVMe drives, when the RAID volume is created underlying member drives will not have /dev/sdX entries associated with them, however for bare NVMe drives there will be associated /dev/sdX entries.

This patch is not yet included because of the ongoing discussion.

Chris H, Martin et all - How are we moving forward with this patch ?

Thanks, Kashyap

Kashyap,

Well, I guess I'll go ahead and queue it up even through I am no fan of these dreadful driver-specific ioctl interfaces. I really wish the SCSI Controller Commands had taken off so we had a standard interface for dealing with RAID controllers and devices behind them.

Martin K. Petersen	Oracle Linux Engineering