ethtool: Add QSFP-DD support
Subsystems (3)
PATCH 1/1 ethtool: Add QSFP-DD support
The Common Management Interface Specification (CMIS) for QSFP-DD shares some similarities with other form factors such as QSFP or SFP, but due to the fact that the module memory map is different, the current ethtool version is not able to provide relevant information about an interface.
This patch adds QSFP-DD support to ethtool. The changes are similar to the ones already existing in qsfp.c, but customized to use the memory addresses and logic as defined in the specifications document.
Page 0x00 (lower and higher memory) are always implemented, so the ethtool expects at least 256 bytes if the identifier matches the one for QSFP-DD. For optical connected cables, additional pages are usually available (the contain module defined thresholds or lane diagnostic information). In this case, ethtool expects to receive 768 bytes in the following format:
+----------+----------+----------+----------+----------+----------+
| Page | Page | Page | Page | Page | Page |
| 0x00 | 0x00 | 0x01 | 0x02 | 0x10 | 0x11 |
| (lower) | (higher) | (higher) | (higher) | (higher) | (higher) |
| 128B | 128B | 128B | 128B | 128B | 128B |
+----------+----------+----------+----------+----------+----------Several functions from qsfp.c could be reused, so an additional parameter was added to each and the functions were moved to sff-common.c.
--- Makefile.am | 3 +- qsfp-dd.c | 560 +++++++++++++++++++++++++++++++++++++++++++++++++++ qsfp-dd.h | 236 ++++++++++++++++++++++ qsfp.c | 60 ++---- qsfp.h | 8 - sff-common.c | 52 +++++ sff-common.h | 26 ++- 7 files changed, 891 insertions(+), 54 deletions(-) create mode 100644 qsfp-dd.c create mode 100644 qsfp-dd.h
Makefile.am +2 -1
@@ -14,7 +14,8 @@ ethtool_SOURCES += \ pcnet32.c realtek.c tg3.c marvell.c vioc.c \ smsc911x.c at76c50x-usb.c sfc.c stmmac.c \ sff-common.c sff-common.h sfpid.c sfpdiag.c \ - ixgbevf.c tse.c vmxnet3.c qsfp.c qsfp.h fjes.c lan78xx.c + ixgbevf.c tse.c vmxnet3.c qsfp.c qsfp.h fjes.c lan78xx.c \ + qsfp-dd.c qsfp-dd.h endif if ENABLE_BASH_COMPLETION
qsfp-dd.c new file +560 -0
560 lines not on this page · raw message
qsfp-dd.h new file +236 -0
236 lines not on this page · raw message
qsfp.c +19 -41
105 lines not on this page · raw message
qsfp.h +0 -8
@@ -31,14 +31,6 @@#define SFF8636_ID_OFFSET 0x00#define SFF8636_REV_COMPLIANCE_OFFSET 0x01 -#define SFF8636_REV_UNSPECIFIED 0x00 -#define SFF8636_REV_8436_48 0x01 -#define SFF8636_REV_8436_8636 0x02 -#define SFF8636_REV_8636_13 0x03 -#define SFF8636_REV_8636_14 0x04 -#define SFF8636_REV_8636_15 0x05 -#define SFF8636_REV_8636_20 0x06 -#define SFF8636_REV_8636_27 0x07#define SFF8636_STATUS_2_OFFSET 0x02/* Flat Memory:0- Paging, 1- Page 0 only */
sff-common.c +52 -0
73 lines not on this page · raw message
sff-common.h +22 -4
@@ -26,7 +26,17 @@#include <stdio.h> #include "internal.h" -#define SFF8024_ID_OFFSET 0x00 +/* Revision compliance */ +#define SFF8636_REV_UNSPECIFIED 0x00 +#define SFF8636_REV_8436_48 0x01 +#define SFF8636_REV_8436_8636 0x02 +#define SFF8636_REV_8636_13 0x03 +#define SFF8636_REV_8636_14 0x04 +#define SFF8636_REV_8636_15 0x05 +#define SFF8636_REV_8636_20 0x06 +#define SFF8636_REV_8636_27 0x07 + +#define SFF8024_ID_OFFSET 0x00#define SFF8024_ID_UNKNOWN 0x00#define SFF8024_ID_GBIC 0x01#define SFF8024_ID_SOLDERED_MODULE 0x02
@@ -51,7 +61,8 @@#define SFF8024_ID_HD8X_FANOUT 0x15#define SFF8024_ID_CDFP_S3 0x16#define SFF8024_ID_MICRO_QSFP 0x17 -#define SFF8024_ID_LAST SFF8024_ID_MICRO_QSFP +#define SFF8024_ID_QSFP_DD 0x18 +#define SFF8024_ID_LAST SFF8024_ID_QSFP_DD#define SFF8024_ID_UNALLOCATED_LAST 0x7F#define SFF8024_ID_VENDOR_START 0x80#define SFF8024_ID_VENDOR_LAST 0xFF
@@ -76,8 +87,14 @@#define SFF8024_CTOR_RJ45 0x22#define SFF8024_CTOR_NO_SEPARABLE 0x23#define SFF8024_CTOR_MXC_2x16 0x24 -#define SFF8024_CTOR_LAST SFF8024_CTOR_MXC_2x16 -#define SFF8024_CTOR_UNALLOCATED_LAST 0x7F +#define SFF8024_CTOR_CS_OPTICAL 0x25 +#define SFF8024_CTOR_CS_OPTICAL_MINI 0x26 +#define SFF8024_CTOR_MPO_2X12 0x27 +#define SFF8024_CTOR_MPO_1X16 0x28 +#define SFF8024_CTOR_LAST SFF8024_CTOR_MPO_1X16 + +#define SFF8024_CTOR_NO_SEP_QSFP_DD 0x6F +#define SFF8024_CTOR_UNALLOCATED_LAST 0x7F#define SFF8024_CTOR_VENDOR_START 0x80#define SFF8024_CTOR_VENDOR_LAST 0xFF
@@ -185,5 +202,6 @@ void sff8024_show_oui(const __u8 *id, int id_offset);void sff8024_show_identifier (const __u8 *id,int id_offset);void sff8024_show_connector (const __u8 *id,int ctor_offset);void sff8024_show_encoding (const __u8 *id,int encoding_offset,int sff_type); +void sff_show_revision_compliance (const __u8 *id,int rev_offset);#endif /* SFF_COMMON_H__ */
Signature · 1 line
2.17.1
Quoted 15 lines · Adrian Pop
The Common Management Interface Specification (CMIS) for QSFP-DD shares some similarities with other form factors such as QSFP or SFP, but due to the fact that the module memory map is different, the current ethtool version is not able to provide relevant information about an interface.
This patch adds QSFP-DD support to ethtool. The changes are similar to the ones already existing in
qsfp.c, but customized to use the memory addresses and logic as defined in the specifications document.Page
0x00(lower and higher memory) are always implemented, so the ethtool expects at least 256 bytes if the identifier matches the one for QSFP-DD. For optical connected cables, additional pages are usually available (the contain module defined thresholds or lane diagnostic information). In this case, ethtool expects to receive 768 bytes in the following format:
Quoted 6 lines
+----------+----------+----------+----------+----------+----------+
| Page | Page | Page | Page | Page | Page |
| 0x00 | 0x00 | 0x01 | 0x02 | 0x10 | 0x11 |
| (lower) | (higher) | (higher) | (higher) | (higher) | (higher) |
| 128B | 128B | 128B | 128B | 128B | 128B |
+----------+----------+----------+----------+----------+----------Hi Adrian
Which in kernel driver is using this format? As far as i can see, only mlx5 makes use of ETH_MODULE_SFF_8436_MAX_LEN, which is 640 bytes. After a very quick look, i could not see any driver returning 768 bytes.
Andrew
A message that is not in the archive (sent off-list or not yet received): CAL_jBfQhVAy24xbz_VbpPM0QtRu-Uzawhyn=AY0b41B9=v3Ytg@mail.gmail.com
Quoted 13 lines · Adrian Pop
Hi Andrew!
Thank you for your email. At the moment, there are no in-kernel drivers that require this support (the 1st version of the Common Management Interface Specification for the QSFP-DD 8X pluggable transceivers came out this year in May [0]). I first came across implementing this extra functionality to ethtool at my company, where we use a custom driver for a NIC that works with the new QSFP-DD transceivers. All the ethtool readings for QSFP-DD were correct and I can provide a sample if needed. Another example of somebody putting QSFP-DD support in their products is Exablaze with their ExaNIC [1] and I'm sure that with time there will be more. Unfortunately at the moment I'm not able to provide help with an update on the mainline driver in the kernel.
Quoted 7 lines
I know that
ETH_MODULE_SFF_8436_MAX_LENis 640 bytes, but to provide for QSFP-DD all the stats as for QSFP (basically to maintain the same behavior/ functionality), we came to the conclusion that we need to read more pages than before, since the memory map is different and the data is more spread around.Please let me know if you have any other feedback and/or suggestions for the patch.
Hi Adrian
You are defining a kernel API here. It is very unusual to define a KAPI without the mainline kernel actually implementing it. So in my opinion, we need an in kernel implementation first.
I guess you are not going to mainline your driver? The ExaNIC is also a long way from being mainline'able.
Could i suggest you extend drivers/net/phy/sfp.c to support QSFP-DD? Or maybe one of Intels or Mellanox drivers? Basically, any hardware with an in kernel driver you can get your hands on and test. Maybe you already have something which you use of interoperability testing.
Andrew
Hi Andrew
You are right about defining a KAPI. Unfortunately due to the nature of my company's business we can't really mainline our driver. I'll try to add the QSFP-DD support myself in the future, although I have nothing planned at the moment. Until then, maybe this thread will be helpful for any other contributors that will decide to work on a driver. As I said, to provide the same stats for QSFP-DD as for QSFP, ethtool needs an extra page. My pull request is well documented and contains information about this.
Adrian
Quoted 38 lines · Andrew Lunn
Quoted 13 lines · Adrian Pop
Hi Andrew!
Thank you for your email. At the moment, there are no in-kernel drivers that require this support (the 1st version of the Common Management Interface Specification for the QSFP-DD 8X pluggable transceivers came out this year in May [0]). I first came across implementing this extra functionality to ethtool at my company, where we use a custom driver for a NIC that works with the new QSFP-DD transceivers. All the ethtool readings for QSFP-DD were correct and I can provide a sample if needed. Another example of somebody putting QSFP-DD support in their products is Exablaze with their ExaNIC [1] and I'm sure that with time there will be more. Unfortunately at the moment I'm not able to provide help with an update on the mainline driver in the kernel.
Quoted 7 lines
I know that
ETH_MODULE_SFF_8436_MAX_LENis 640 bytes, but to provide for QSFP-DD all the stats as for QSFP (basically to maintain the same behavior/ functionality), we came to the conclusion that we need to read more pages than before, since the memory map is different and the data is more spread around.Please let me know if you have any other feedback and/or suggestions for the patch.
Hi Adrian
You are defining a kernel API here. It is very unusual to define a KAPI without the mainline kernel actually implementing it. So in my opinion, we need an in kernel implementation first.
I guess you are not going to mainline your driver? The ExaNIC is also a long way from being mainline'able.
Could i suggest you extend
drivers/net/phy/sfp.cto support QSFP-DD? Or maybe one of Intels or Mellanox drivers? Basically, any hardware with an in kernel driver you can get your hands on and test. Maybe you already have something which you use of interoperability testing.Andrew
A message that is not in the archive (sent off-list or not yet received): [email protected]
Hi Andrew!
The QSFP-DD support I added doesn't replace and/or overrides the current QSFP support and memory mapping. When I started developing this, I tried to reuse as much as the old code as possible, but most of the properties where placed at totally different offsets.
In qsfp.c, I added this:
void sff8636_show_all(const __u8 *id, __u32 eeprom_len)
{
if (id[SFF8636_ID_OFFSET] == SFF8024_ID_QSFP_DD) {
qsfp_dd_show_all(id, eeprom_len);
return;
}
[...]This was needed because although modinfo.type (see ethtool.c, do_getmodule) is still ETH_MODULE_SFF_8636, the byte in the eeprom containing the ID was different (0x18 for QSFP-DD). Since QSFP and QSFP-DD were not fully backwards compatible, I decided to use the ID to differentiate between them (this is done in the same way for QSFP+ for example).
If I run an older version of ethtool on an interface from a network card having a QSFP-DD transceiver plugged in, the tool prints 0x18 and exits. With my changes, if we have a QSFP-DD, the tool prints the correct stats without altering the old flow for normal QSFPs.
Of course, as you stated in the previous messages, it still looks like I'm trying to define an KAPI. If we still want to keep page 3 and just append 0x10 and 0x11 after it, that can be easily done (although there aren't any stats needed for QSFP-DD that are found in page 0x03).
Adrian
Quoted 69 lines · Andrew Lunn
Quoted 15 lines · Adrian Pop
The Common Management Interface Specification (CMIS) for QSFP-DD shares some similarities with other form factors such as QSFP or SFP, but due to the fact that the module memory map is different, the current ethtool version is not able to provide relevant information about an interface.
This patch adds QSFP-DD support to ethtool. The changes are similar to the ones already existing in
qsfp.c, but customized to use the memory addresses and logic as defined in the specifications document.Page
0x00(lower and higher memory) are always implemented, so the ethtool expects at least 256 bytes if the identifier matches the one for QSFP-DD. For optical connected cables, additional pages are usually available (the contain module defined thresholds or lane diagnostic information). In this case, ethtool expects to receive 768 bytes in the following format:Quoted 6 lines
+----------+----------+----------+----------+----------+----------+ | Page | Page | Page | Page | Page | Page | | 0x00 | 0x00 | 0x01 | 0x02 | 0x10 | 0x11 | | (lower) | (higher) | (higher) | (higher) | (higher) | (higher) | | 128B | 128B | 128B | 128B | 128B | 128B | +----------+----------+----------+----------+----------+----------Hi Adrian
From
ethtool/qsfp.c* b) SFF 8636 based 640 bytes memory layout is presented for parser
*
* SFF 8636 based QSFP Memory Map
*
* 2-Wire Serial Address: 1010000x
** Lower Page 00h (128 bytes) * ====================== * | | * |Page Select Byte(127)| * ====================== * | * V * ---------------------------------------- * | | | | * V V V V * ---------- ---------- --------- ------------ * | Upper | | Upper | | Upper | | Upper | * | Page 00h | | Page 01h | | Page 02h | | Page 03h | * | | |(Optional)| |(Optional)| | (Optional) | * | | | | | | | | * | | | | | | | | * | ID | | AST | | User | | For | * | Fields | | Table | | EEPROM | | Cable | * | | | | | Data | | Assemblies | * | | | | | | | | * | | | | | | | | * ----------- ----------- ---------- --------------I don't think dropping page 3 is good for backwards/forwards compatibility. An old ethtool is going to decode what it thinks is page 3 when in fact it is page
0x10. For backwards compatibility, you should not dropping page 3, but append pages0x10and0x11after page 3.Then there is the question, what do we do when page 4 is needed by some other module? Maybe the better solution is to wait until netlink ethtool is available. We can then label each page with an attribute indicate what page it actually is, and so allow sparse pages.
Andrew