net/bnxt: allow configuring vector mode
PATCH 1/1 net/bnxt: allow configuring vector mode
The bnxt driver has vector mode but it has limitations. For example, rte_flow mark won't work in vector mode. For this reason the user should be able to disable vector mode as part of the config.
Make the configuration use the same as other drivers with vector mode: ixge, i40e, ...
This will also make future support of vector mode on other architectures possible.
--- config/common_base | 1 + drivers/net/bnxt/Makefile | 2 +- drivers/net/bnxt/bnxt_ethdev.c | 10 +++------- drivers/net/bnxt/bnxt_ring.c | 4 ++-- drivers/net/bnxt/bnxt_rxr.c | 3 +++ 5 files changed, 10 insertions(+), 10 deletions(-)
config/common_base +1 -0
@@ -219,6 +219,7 @@ CONFIG_RTE_LIBRTE_BNX2X_DEBUG_PERIODIC=n # Compile burst-oriented Broadcom BNXT PMD driver # CONFIG_RTE_LIBRTE_BNXT_PMD=y +CONFIG_RTE_LIBRTE_BNXT_INC_VECTOR=y # # Compile burst-oriented Chelsio Terminator (CXGBE) PMD
drivers/net/bnxt/Makefile +1 -1
@@ -40,7 +40,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_irq.c SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_util.c SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += rte_pmd_bnxt.c ifeq ($(CONFIG_RTE_ARCH_X86), y) -SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_rxtx_vec_sse.c +SRCS-$(CONFIG_RTE_LIBRTE_BNXT_INC_VECTOR) += bnxt_rxtx_vec_sse.c endif #
drivers/net/bnxt/bnxt_ethdev.c +3 -7
@@ -757,8 +757,7 @@ bnxt_receive_function(struct rte_eth_dev *eth_dev) {struct bnxt *bp = eth_dev->data->dev_private; -#ifdef RTE_ARCH_X86 -#ifndef RTE_LIBRTE_IEEE1588 +#ifdef RTE_BNXT_INC_VECTOR /* * Vector mode receive can be enabled only if scatter rx is not * in use and rx offloads are limited to VLAN stripping and
@@ -787,7 +786,6 @@ bnxt_receive_function(struct rte_eth_dev *eth_dev) eth_dev->data->port_id, eth_dev->data->scattered_rx, eth_dev->data->dev_conf.rxmode.offloads); -#endif #endif bp->flags &= ~BNXT_FLAG_RX_VECTOR_PKT_MODE ;return bnxt_recv_pkts;
@@ -796,8 +794,7 @@ bnxt_receive_function(struct rte_eth_dev *eth_dev)static eth_tx_burst_t bnxt_transmit_function(__rte_unused struct rte_eth_dev *eth_dev) { -#ifdef RTE_ARCH_X86 -#ifndef RTE_LIBRTE_IEEE1588 +#ifdef RTE_BNXT_INC_VECTOR /* * Vector mode transmit can be enabled only if not using scatter rx * or tx offloads.
@@ -815,7 +812,6 @@ bnxt_transmit_function(__rte_unused struct rte_eth_dev *eth_dev) eth_dev->data->port_id, eth_dev->data->scattered_rx, eth_dev->data->dev_conf.txmode.offloads); -#endif #endif return bnxt_xmit_pkts; }
@@ -2234,7 +2230,7 @@ int bnxt_mtu_set_op(struct rte_eth_dev *eth_dev, uint16_t new_mtu) new_pkt_size = new_mtu +RTE_ETHER_HDR_LEN +RTE_ETHER_CRC_LEN +VLAN_TAG_SIZE *BNXT_NUM_VLANS ; -#ifdef RTE_ARCH_X86 +#ifdef RTE_BNXT_INC_VECTOR /* * If vector-mode tx/rx is active, disallow any MTU change that would * require scattered receive support.
drivers/net/bnxt/bnxt_ring.c +2 -2
@@ -608,7 +608,7 @@ int bnxt_alloc_hwrm_rx_ring(struct bnxt *bp, int queue_index)bnxt_db_write (&rxr->ag_db, rxr->ag_prod); } rxq->index = queue_index; -#ifdef RTE_ARCH_X86 +#ifdef RTE_BNXT_INC_VECTOR bnxt_rxq_vec_setup (rxq);#endif
@@ -713,7 +713,7 @@ int bnxt_alloc_hwrm_rings(struct bnxt *bp)bnxt_db_write (&rxr->rx_db, rxr->rx_prod);bnxt_db_write (&rxr->ag_db, rxr->ag_prod); rxq->index = i; -#ifdef RTE_ARCH_X86 +#ifdef RTE_BNXT_INC_VECTOR bnxt_rxq_vec_setup (rxq);#endif }
drivers/net/bnxt/bnxt_rxr.c +3 -0
@@ -18,6 +18,9 @@#include "hsi_struct_def_dpdk.h" #ifdef RTE_LIBRTE_IEEE1588 #include "bnxt_hwrm.h" +#ifdef RTE_BNXT_INC_VECTOR +#error "bnxt: IEEE1588 is incompatiable with vector mode" +#endif #endif /*
Signature · 1 line
2.20.1
Hi Stephen,
Quoted 1 line · Stephen Hemminger
<snip>
Quoted 3 lines
Make the configuration use the same as other drivers with vector mode: ixge, i40e, ...
s/ixge/ixgbe/?
Quoted 1 line
<snip>
Quoted 4 lines
This will also make future support of vector mode on other architectures possible.
Fixes bc4a000f2f53 ("net/bnxt: implement SSE vector mode")
<snip>
drivers/net/bnxt/bnxt_rxr.c line 23
Quoted 2 lines
drivers/net/bnxt/bnxt_rxr.c +2
-0+#error "bnxt: IEEE1588 is incompatiable with vector mode" +#endif
s/incompatiable/incompatible/
This was this approach taken in the initial patch set to add bnxt vector mode support,
however based on feedback the current approach was used instead. That discussion can be found here:
http://patches.dpdk.org/patch/53683/
If mark support could be treated as a receive offload, it would be possible to choose
the non-vector receive handler based on whether mark is enabled. Adding a kvargs option to disable vector mode might be another possibility. Otherwise, a build-time
configuration option does seem to be useful.
LGTM... with the above:
Regards,
Lance
Quoted 36 lines · On Thu, 5 Mar 2020 15:10:48 -0500 Lance Richardson…
Hi Stephen,
Quoted 1 line · Stephen Hemminger
<snip>
Quoted 3 lines
Make the configuration use the same as other drivers with vector mode: ixge, i40e, ...
s/ixge/ixgbe/?
Quoted 1 line
<snip>
Quoted 4 lines
This will also make future support of vector mode on other architectures possible.
Fixes bc4a000f2f53 ("net/bnxt: implement SSE vector mode")<snip>
drivers/net/bnxt/bnxt_rxr.c line 23
Quoted 2 lines
drivers/net/bnxt/bnxt_rxr.c +2
-0+#error "bnxt: IEEE1588 is incompatiable with vector mode" +#endif s/incompatiable/incompatible/
This was this approach taken in the initial patch set to add bnxt vector mode support,
however based on feedback the current approach was used instead. That discussion can be found here:http://patches.dpdk.org/patch/53683/If mark support could be treated as a receive offload, it would be possible to choose
the non-vector receive handler based on whether mark is enabled. Adding a kvargs option to disable vector mode might be another possibility. Otherwise, a build-time
configuration option does seem to be useful.
Quoted 3 lines
LGTM... with the above:
Acked-by Lance Richardson <[email protected]>
What ever solution must be consistent across all drivers.
Quoted 39 lines · Stephen Hemminger
Quoted 36 lines · On Thu, 5 Mar 2020 15:10:48 -0500 Lance Richardson…
Hi Stephen,
Quoted 1 line · Stephen Hemminger
<snip>
Quoted 3 lines
Make the configuration use the same as other drivers with vector mode: ixge, i40e, ...
s/ixge/ixgbe/?
Quoted 1 line
<snip>
Quoted 4 lines
This will also make future support of vector mode on other architectures possible.
Fixes bc4a000f2f53 ("net/bnxt: implement SSE vector mode")<snip>
drivers/net/bnxt/bnxt_rxr.c line 23
Quoted 2 lines
drivers/net/bnxt/bnxt_rxr.c +2
-0+#error "bnxt: IEEE1588 is incompatiable with vector mode" +#endif s/incompatiable/incompatible/
This was this approach taken in the initial patch set to add bnxt vector mode support,
however based on feedback the current approach was used instead. That discussion can be found here:http://patches.dpdk.org/patch/53683/If mark support could be treated as a receive offload, it would be possible to choose
the non-vector receive handler based on whether mark is enabled. Adding a kvargs option to disable vector mode might be another possibility. Otherwise, a build-time
configuration option does seem to be useful.
Quoted 6 lines
Quoted 3 lines
LGTM... with the above:
Acked-by Lance Richardson <[email protected]>What ever solution must be consistent across all drivers.
I saw Ajit merged this patch to brcm tree, but I am not sure about it. We have already removed this compile time option from some PMDs, and driver tries to detect to use or not to use vectorization transparently.
This config is also a problem for the meson, which always sets the flag in a hardcoded way.
But also I am not sure about to need to enable/disable vectorization explicitly, this patch seems because of this need. As far as I remember in the past this type of runtime configuration rejected to not make driver configuration more complex.
Quoted 59 lines · Ferruh Yigit
Quoted 21 lines · Stephen Hemminger
Quoted 18 lines · On Thu, 5 Mar 2020 15:10:48 -0500 Lance Richardson…
Hi Stephen,
Quoted 1 line · Stephen Hemminger
<snip>
Quoted 3 lines
Make the configuration use the same as other drivers with vector mode: ixge, i40e, ...
s/ixge/ixgbe/?
Quoted 1 line
<snip>
Quoted 4 lines
This will also make future support of vector mode on other architectures possible.
Fixes bc4a000f2f53 ("net/bnxt: implement SSE vector mode")<snip>
Quoted 8 lines
Quoted 8 lines
drivers/net/bnxt/bnxt_rxr.c line 23
Quoted 2 lines
drivers/net/bnxt/bnxt_rxr.c +2
-0+#error "bnxt: IEEE1588 is incompatiable with vector mode" +#endif s/incompatiable/incompatible/
This was this approach taken in the initial patch set to add bnxt vector mode support,
however based on feedback the current approach was used instead. Thatdiscussion
Quoted 6 lines
Quoted 6 lines
can be found here:
http://patches.dpdk.org/patch/53683/If mark support could be treated as a receive offload, it would be possible to choose
the non-vector receive handler based on whether mark is enabled. Addinga kvargs
Quoted 10 lines
Quoted 7 lines
option to disable vector mode might be another possibility. Otherwise, a build-time
configuration option does seem to be useful.LGTM... with the above:
Acked-by Lance Richardson <[email protected]>What ever solution must be consistent across all drivers.
I saw Ajit merged this patch to brcm tree, but I am not sure about it. We have
already removed this compile time option from some PMDs, and driver tries to
detect to use or not to use vectorization transparently.This config is also a problem for the meson, which always sets the flag in a
hardcoded way.
Quoted 7 lines
But also I am not sure about to need to enable/disable vectorization explicitly,
this patch seems because of this need. As far as I remember in the past this
type of runtime configuration rejected to not make driver configuration more
complex.
Since we need a way to disable or enable vector mode.
May be a dev parameter could be used?
That way it would not interfere with the meson builds and also allow a user/application to set vector mode setting as desired.
31/03/2020 16:31, Ajit Khaparde:
Quoted 69 lines · Ferruh Yigit
Quoted 60 lines · Stephen Hemminger
Quoted 21 lines
Quoted 18 lines · On Thu, 5 Mar 2020 15:10:48 -0500 Lance Richardson…
Hi Stephen,
Quoted 1 line · Stephen Hemminger
<snip>
Quoted 3 lines
Make the configuration use the same as other drivers with vector mode: ixge, i40e, ...
s/ixge/ixgbe/?
Quoted 1 line
<snip>
Quoted 4 lines
This will also make future support of vector mode on other architectures possible.
Fixes bc4a000f2f53 ("net/bnxt: implement SSE vector mode")<snip>
Quoted 8 lines
Quoted 8 lines
drivers/net/bnxt/bnxt_rxr.c line 23
Quoted 2 lines
drivers/net/bnxt/bnxt_rxr.c +2
-0+#error "bnxt: IEEE1588 is incompatiable with vector mode" +#endif s/incompatiable/incompatible/
This was this approach taken in the initial patch set to add bnxt vector mode support,
however based on feedback the current approach was used instead. Thatdiscussion
Quoted 6 lines
Quoted 6 lines
can be found here:
http://patches.dpdk.org/patch/53683/If mark support could be treated as a receive offload, it would be possible to choose
the non-vector receive handler based on whether mark is enabled. Addinga kvargs
Quoted 10 lines
Quoted 7 lines
option to disable vector mode might be another possibility. Otherwise, a build-time
configuration option does seem to be useful.LGTM... with the above:
Acked-by Lance Richardson <[email protected]>What ever solution must be consistent across all drivers.
I saw Ajit merged this patch to brcm tree, but I am not sure about it. We have
already removed this compile time option from some PMDs, and driver tries to
detect to use or not to use vectorization transparently.This config is also a problem for the meson, which always sets the flag in a
hardcoded way.Quoted 7 lines
But also I am not sure about to need to enable/disable vectorization explicitly,
this patch seems because of this need. As far as I remember in the past this
type of runtime configuration rejected to not make driver configuration more
complex.
Quoted 1 line
Since we need a way to disable or enable vector mode.
Why do you need to disable vector optimization?
Is it for debugging?
Quoted 3 lines
May be a dev parameter could be used?
That way it would not interfere with the meson builds and also allow a user/application to set vector mode setting as desired.
Yes diasbling optimization could be done in devargs.
Quoted 70 lines · On Tue, 31 Mar 2020 17:43:55 +0200 Thomas Monjalon <[email protected]> wrote:
31/03/2020 16:31, Ajit Khaparde:
Quoted 69 lines · Ferruh Yigit
Quoted 67 lines · Stephen Hemminger
Quoted 21 lines
Quoted 18 lines · On Thu, 5 Mar 2020 15:10:48 -0500 Lance Richardson…
Hi Stephen,
Quoted 1 line · Stephen Hemminger
<snip>
Quoted 3 lines
Make the configuration use the same as other drivers with vector mode: ixge, i40e, ...
s/ixge/ixgbe/?
Quoted 1 line
<snip>
Quoted 4 lines
This will also make future support of vector mode on other architectures possible.
Fixes bc4a000f2f53 ("net/bnxt: implement SSE vector mode")<snip>
Quoted 8 lines
Quoted 8 lines
drivers/net/bnxt/bnxt_rxr.c line 23
Quoted 2 lines
drivers/net/bnxt/bnxt_rxr.c +2
-0+#error "bnxt: IEEE1588 is incompatiable with vector mode" +#endif s/incompatiable/incompatible/
This was this approach taken in the initial patch set to add bnxt vector mode support,
however based on feedback the current approach was used instead. Thatdiscussion
Quoted 6 lines
Quoted 6 lines
can be found here:
http://patches.dpdk.org/patch/53683/If mark support could be treated as a receive offload, it would be possible to choose
the non-vector receive handler based on whether mark is enabled. Addinga kvargs
Quoted 10 lines
Quoted 7 lines
option to disable vector mode might be another possibility. Otherwise, a build-time
configuration option does seem to be useful.LGTM... with the above:
Acked-by Lance Richardson <[email protected]>What ever solution must be consistent across all drivers.
I saw Ajit merged this patch to brcm tree, but I am not sure about it. We have
already removed this compile time option from some PMDs, and driver tries to
detect to use or not to use vectorization transparently.This config is also a problem for the meson, which always sets the flag in a
hardcoded way.But also I am not sure about to need to enable/disable vectorization explicitly,
this patch seems because of this need. As far as I remember in the past this
type of runtime configuration rejected to not make driver configuration more
complex.
Quoted 4 lines
Quoted 1 line
Since we need a way to disable or enable vector mode.
Why do you need to disable vector optimization?
Is it for debugging?
The rte_flow mark operation does not work with the vector optimization.
The choice to use vector mode is done by the driver earlier in the initialization process, and then when application programs rte_flow it has a problem; the flow create would fail.
31/03/2020 17:58, Stephen Hemminger:
Quoted 24 lines
Quoted 22 lines · On Tue, 31 Mar 2020 17:43:55 +0200 Thomas Monjalon <[email protected]> wrote:
31/03/2020 16:31, Ajit Khaparde:
Quoted 20 lines · Ferruh Yigit
Quoted 10 lines
I saw Ajit merged this patch to brcm tree, but I am not sure about it. We have
already removed this compile time option from some PMDs, and driver tries to
detect to use or not to use vectorization transparently.This config is also a problem for the meson, which always sets the flag in a
hardcoded way.Quoted 7 lines
But also I am not sure about to need to enable/disable vectorization explicitly,
this patch seems because of this need. As far as I remember in the past this
type of runtime configuration rejected to not make driver configuration more
complex.Since we need a way to disable or enable vector mode.
Quoted 8 lines
Quoted 2 lines
Why do you need to disable vector optimization?
Is it for debugging?The
rte_flowmark operation does not work with the vector optimization.The choice to use vector mode is done by the driver earlier in the initialization process, and then when application programs
rte_flowit has a problem; the flow create would fail.
Could we imagine switching the Rx implementation dynamically between 2 bursts?
Quoted 8 lines · Stephen Hemminger
The bnxt driver has vector mode but it has limitations. For example,
rte_flowmark won't work in vector mode. For this reason the user should be able to disable vector mode as part of the config.Make the configuration use the same as other drivers with vector mode: ixge, i40e, ...
Quoted 8 lines
This will also make future support of vector mode on other architectures possible.
Fixes bc4a000f2f53 ("net/bnxt: implement SSE vector mode")Signed-off-by Stephen Hemminger <[email protected]>
Patch applied to dpdk-next-net-brcm. Thanks
Quoted 127 lines
--- config/common_base | 1 + drivers/net/bnxt/Makefile | 2 +- drivers/net/bnxt/bnxt_ethdev.c | 10 +++------- drivers/net/bnxt/bnxt_ring.c | 4 ++-- drivers/net/bnxt/bnxt_rxr.c | 3 +++ 5 files changed, 10 insertions(+), 10 deletions(-)config/common_base +1
-0@@ -219,6 +219,7 @@ CONFIG_RTE_LIBRTE_BNX2X_DEBUG_PERIODIC=n # Compile burst-oriented Broadcom BNXT PMD driver # CONFIG_RTE_LIBRTE_BNXT_PMD=y +CONFIG_RTE_LIBRTE_BNXT_INC_VECTOR=y # # Compile burst-oriented Chelsio Terminator (CXGBE) PMDdrivers/net/bnxt/Makefile +1
-1@@ -40,7 +40,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_irq.c SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_util.c SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += rte_pmd_bnxt.c ifeq ($(CONFIG_RTE_ARCH_X86), y) -SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += bnxt_rxtx_vec_sse.c +SRCS-$(CONFIG_RTE_LIBRTE_BNXT_INC_VECTOR) += bnxt_rxtx_vec_sse.c endif #drivers/net/bnxt/bnxt_ethdev.c +2
-6@@ -757,8 +757,7 @@ bnxt_receive_function(struct rte_eth_dev *eth_dev) {struct bnxt *bp = eth_dev->data->dev_private; -#ifdef RTE_ARCH_X86 -#ifndef RTE_LIBRTE_IEEE1588 +#ifdef RTE_BNXT_INC_VECTOR /* * Vector mode receive can be enabled only if scatter rx is not * in use and rx offloads are limited to VLAN stripping and @@ -787,7 +786,6 @@ bnxt_receive_function(struct rte_eth_dev *eth_dev) eth_dev->data->port_id, eth_dev->data->scattered_rx, eth_dev->data->dev_conf.rxmode.offloads); -#endif #endif bp->flags &= ~BNXT_FLAG_RX_VECTOR_PKT_MODE ;return bnxt_recv_pkts;@@ -796,8 +794,7 @@ bnxt_receive_function(struct rte_eth_dev *eth_dev)static eth_tx_burst_t bnxt_transmit_function(__rte_unused struct rte_eth_dev *eth_dev) { -#ifdef RTE_ARCH_X86 -#ifndef RTE_LIBRTE_IEEE1588 +#ifdef RTE_BNXT_INC_VECTOR /* * Vector mode transmit can be enabled only if not using scatter rx * or tx offloads. @@ -815,7 +812,6 @@ bnxt_transmit_function(__rte_unused struct rte_eth_dev *eth_dev) eth_dev->data->port_id, eth_dev->data->scattered_rx, eth_dev->data->dev_conf.txmode.offloads); -#endif #endif return bnxt_xmit_pkts;}
drivers/net/bnxt/bnxt_ethdev.c +1
-1@@ -2234,7 +2230,7 @@ int bnxt_mtu_set_op(struct rte_eth_dev *eth_dev, uint16_t new_mtu) new_pkt_size = new_mtu +RTE_ETHER_HDR_LEN +RTE_ETHER_CRC_LEN +VLAN_TAG_SIZE *BNXT_NUM_VLANS ; -#ifdef RTE_ARCH_X86 +#ifdef RTE_BNXT_INC_VECTOR /* * If vector-mode tx/rx is active, disallow any MTU change that would
* require scattered receive support.drivers/net/bnxt/bnxt_ring.c +2
-2@@ -608,7 +608,7 @@ int bnxt_alloc_hwrm_rx_ring(struct bnxt *bp, int queue_index) bnxt_db_write(&rxr->ag_db, rxr->ag_prod); } rxq->index = queue_index; -#ifdef RTE_ARCH_X86 +#ifdef RTE_BNXT_INC_VECTOR bnxt_rxq_vec_setup (rxq);#endif @@ -713,7 +713,7 @@ int bnxt_alloc_hwrm_rings(struct bnxt *bp)bnxt_db_write (&rxr->rx_db, rxr->rx_prod);bnxt_db_write (&rxr->ag_db, rxr->ag_prod); rxq->index = i; -#ifdef RTE_ARCH_X86 +#ifdef RTE_BNXT_INC_VECTOR bnxt_rxq_vec_setup (rxq);#endif }drivers/net/bnxt/bnxt_rxr.c +3
-1@@ -18,6 +18,9 @@#include "hsi_struct_def_dpdk.h" #ifdef RTE_LIBRTE_IEEE1588 #include "bnxt_hwrm.h" +#ifdef RTE_BNXT_INC_VECTOR +#error "bnxt: IEEE1588 is incompatiable with vector mode" +#endif #endif /* -- 2.20.1
PATCH 1/1 net/bnxt: disable vector mode if flow mark is used
In reply to Stephen HemmingerSince vector mode can't be used with flow mark actions. The choice of vector or non-vector mode is done dynamically earlier in the initialization process (config) and the user application has no direct control over the selection.
Therefore the best resolution to the conflict is for the driver to disable vector mode if it needs to.
--- drivers/net/bnxt/bnxt_flow.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-)
drivers/net/bnxt/bnxt_flow.c +5 -8
@@ -18,6 +18,7 @@#include "bnxt_hwrm.h" #include "bnxt_ring.h" #include "bnxt_rxq.h" +#include "bnxt_rxr.h" #include "bnxt_vnic.h" #include "hsi_struct_def_dpdk.h"
@@ -1405,14 +1406,10 @@ bnxt_validate_and_parse_flow(struct rte_eth_dev *dev,case RTE_FLOW_ACTION_TYPE_MARK :if (bp->flags &BNXT_FLAG_RX_VECTOR_PKT_MODE ) {PMD_DRV_LOG (DEBUG , -"Disable vector processing for mark\n" ); -rte_flow_error_set (error, -ENOTSUP , -RTE_FLOW_ERROR_TYPE_ACTION , - act, -"Disable vector processing for mark" ); - rc = -rte_errno; - goto ret; +"Disabling vector processing for mark\n" ); + + bp->flags &= ~BNXT_FLAG_RX_VECTOR_PKT_MODE ; + dev->rx_pkt_burst = &bnxt_recv_pkts; }if (bp->mark_table ==NULL ) {
Signature · 1 line
2.20.1