EFA statistics minor updates
Subsystems (2)
cover EFA statistics minor updates
This series contains three small patches to collect a few counters we found helpful when encountering different issues.
Regards,
Gal
Gal Pressman (3):
RDMA/efa: Report create CQ error counter
RDMA/efa: Count mmap failures
RDMA/efa: Count admin commands errors
drivers/infiniband/hw/efa/efa.h | 3 ++- drivers/infiniband/hw/efa/efa_com.c | 5 ++++- drivers/infiniband/hw/efa/efa_com.h | 3 ++- drivers/infiniband/hw/efa/efa_verbs.c | 13 +++++++++++-- 4 files changed, 19 insertions(+), 5 deletions(-)
base-commit: 8f3d9f354286745c751374f5f1fcafee6b3f3136
Signature · 1 line
2.26.1
PATCH 1/3 RDMA/efa: Report create CQ error counter
Create CQ errors are already being counted, report them along all other counters.
--- drivers/infiniband/hw/efa/efa_verbs.c | 2 ++ 1 file changed, 2 insertions(+)
drivers/infiniband/hw/efa/efa_verbs.c +2 -0
@@ -41,6 +41,7 @@ struct efa_user_mmap_entry { op(EFA_KEEP_ALIVE_RCVD, "keep_alive_rcvd") \ op(EFA_ALLOC_PD_ERR, "alloc_pd_err") \ op(EFA_CREATE_QP_ERR, "create_qp_err") \ + op(EFA_CREATE_CQ_ERR, "create_cq_err") \ op(EFA_REG_MR_ERR, "reg_mr_err") \ op(EFA_ALLOC_UCONTEXT_ERR, "alloc_ucontext_err") \op (EFA_CREATE_AH_ERR ,"create_ah_err" )
@@ -1753,6 +1754,7 @@ int efa_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, stats->value[EFA_KEEP_ALIVE_RCVD ] =atomic64_read (&s->keep_alive_rcvd); stats->value[EFA_ALLOC_PD_ERR ] =atomic64_read (&s->sw_stats.alloc_pd_err); stats->value[EFA_CREATE_QP_ERR ] =atomic64_read (&s->sw_stats.create_qp_err); + stats->value[EFA_CREATE_CQ_ERR ] =atomic64_read (&s->sw_stats.create_cq_err); stats->value[EFA_REG_MR_ERR ] =atomic64_read (&s->sw_stats.reg_mr_err); stats->value[EFA_ALLOC_UCONTEXT_ERR ] =atomic64_read (&s->sw_stats.alloc_ucontext_err); stats->value[EFA_CREATE_AH_ERR ] =atomic64_read (&s->sw_stats.create_ah_err);
Signature · 1 line
2.26.1
PATCH 2/3 RDMA/efa: Count mmap failures
In reply to Gal PressmanAdd a new stat that counts mmap failures, which might help when debugging different issues.
--- drivers/infiniband/hw/efa/efa.h | 3 ++- drivers/infiniband/hw/efa/efa_verbs.c | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-)
drivers/infiniband/hw/efa/efa.h +2 -1
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ /* -* Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved. +* Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved. */ #ifndef _EFA_H_
@@ -40,6 +40,7 @@ struct efa_sw_stats {atomic64_t reg_mr_err;atomic64_t alloc_ucontext_err;atomic64_t create_ah_err; +atomic64_t mmap_err; };/* Don't use anything other than atomic64 */
drivers/infiniband/hw/efa/efa_verbs.c +7 -2
@@ -44,7 +44,8 @@ struct efa_user_mmap_entry { op(EFA_CREATE_CQ_ERR, "create_cq_err") \ op(EFA_REG_MR_ERR, "reg_mr_err") \ op(EFA_ALLOC_UCONTEXT_ERR, "alloc_ucontext_err") \ - op(EFA_CREATE_AH_ERR, "create_ah_err") + op(EFA_CREATE_AH_ERR, "create_ah_err") \ + op(EFA_MMAP_ERR, "mmap_err")#define EFA_STATS_ENUM (ename, name) ename,#define EFA_STATS_STR (ename, name) [ename] = name,
@@ -1569,6 +1570,7 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,ibdev_dbg (&dev->ibdev,"pgoff[%#lx] does not have valid entry\n" , vma->vm_pgoff); +atomic64_inc (&dev->stats.sw_stats.mmap_err);return -EINVAL ; } entry =to_emmap (rdma_entry);
@@ -1604,12 +1606,14 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, err = -EINVAL ; } -if (err) +if (err) {ibdev_dbg ( &dev->ibdev,"Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n" , entry->address, rdma_entry->npages *PAGE_SIZE , entry->mmap_flag, err); +atomic64_inc (&dev->stats.sw_stats.mmap_err); + }rdma_user_mmap_entry_put (rdma_entry);return err;
@@ -1758,6 +1762,7 @@ int efa_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, stats->value[EFA_REG_MR_ERR ] =atomic64_read (&s->sw_stats.reg_mr_err); stats->value[EFA_ALLOC_UCONTEXT_ERR ] =atomic64_read (&s->sw_stats.alloc_ucontext_err); stats->value[EFA_CREATE_AH_ERR ] =atomic64_read (&s->sw_stats.create_ah_err); + stats->value[EFA_MMAP_ERR ] =atomic64_read (&s->sw_stats.mmap_err);return ARRAY_SIZE (efa_stats_names); }
Signature · 1 line
2.26.1
PATCH 3/3 RDMA/efa: Count admin commands errors
In reply to Gal PressmanAdd a new stat that counts admin commands failures, which might help when debugging different issues.
--- drivers/infiniband/hw/efa/efa_com.c | 5 ++++- drivers/infiniband/hw/efa/efa_com.h | 3 ++- drivers/infiniband/hw/efa/efa_verbs.c | 2 ++ 3 files changed, 8 insertions(+), 2 deletions(-)
drivers/infiniband/hw/efa/efa_com.c +4 -1
@@ -631,17 +631,20 @@ int efa_com_cmd_exec(struct efa_com_admin_queue *aq, cmd->aq_common_descriptor.opcode, PTR_ERR(comp_ctx));up (&aq->avail_cmds); +atomic64_inc (&aq->stats.cmd_err);return PTR_ERR (comp_ctx); } err =efa_com_wait_and_process_admin_cq (comp_ctx, aq); -if (err) +if (err) {ibdev_err_ratelimited ( aq->efa_dev,"Failed to process command %s (opcode %u) comp_status %d err %d\n" ,efa_com_cmd_str (cmd->aq_common_descriptor.opcode), cmd->aq_common_descriptor.opcode, comp_ctx->comp_status, err); +atomic64_inc (&aq->stats.cmd_err); + }up (&aq->avail_cmds);
drivers/infiniband/hw/efa/efa_com.h +2 -1
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ /* -* Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved. +* Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved. */ #ifndef _EFA_COM_H_
@@ -47,6 +47,7 @@ struct efa_com_admin_sq {struct efa_com_stats_admin {atomic64_t submitted_cmd;atomic64_t completed_cmd; +atomic64_t cmd_err;atomic64_t no_completion; };
drivers/infiniband/hw/efa/efa_verbs.c +2 -0
@@ -37,6 +37,7 @@ struct efa_user_mmap_entry { op(EFA_RX_DROPS, "rx_drops") \ op(EFA_SUBMITTED_CMDS, "submitted_cmds") \ op(EFA_COMPLETED_CMDS, "completed_cmds") \ + op(EFA_CMDS_ERR, "cmds_err") \ op(EFA_NO_COMPLETION_CMDS, "no_completion_cmds") \ op(EFA_KEEP_ALIVE_RCVD, "keep_alive_rcvd") \op (EFA_ALLOC_PD_ERR ,"alloc_pd_err" ) \
@@ -1752,6 +1753,7 @@ int efa_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, as = &dev->edev.aq.stats; stats->value[EFA_SUBMITTED_CMDS ] =atomic64_read (&as->submitted_cmd); stats->value[EFA_COMPLETED_CMDS ] =atomic64_read (&as->completed_cmd); + stats->value[EFA_CMDS_ERR ] =atomic64_read (&as->cmd_err); stats->value[EFA_NO_COMPLETION_CMDS ] =atomic64_read (&as->no_completion); s = &dev->stats;
Signature · 1 line
2.26.1
Re: [PATCH for-next 2/3] RDMA/efa: Count mmap failures
In reply to Gal PressmanQuoted 52 lines · Gal Pressman
Add a new stat that counts mmap failures, which might help when debugging different issues.
Reviewed-by Firas JahJah <[email protected]>Reviewed-by Yossi Leybovich <[email protected]>Signed-off-by Gal Pressman <[email protected]>drivers/infiniband/hw/efa/efa.h | 3 ++- drivers/infiniband/hw/efa/efa_verbs.c | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-)drivers/infiniband/hw/efa/efa.h +3
-1+++ b/drivers/infiniband/hw/efa/efa.h@@ -1,6 +1,6 @@/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ /* -* Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved. +* Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved. */ #ifndef _EFA_H_@@ -40,6 +40,7 @@ struct efa_sw_stats {atomic64_t reg_mr_err;atomic64_t alloc_ucontext_err;atomic64_t create_ah_err; +atomic64_t mmap_err; };/* Don't use anything other than atomic64 */ drivers/infiniband/hw/efa/efa_verbs.c +4
-1+++ b/drivers/infiniband/hw/efa/efa_verbs.c@@ -44,7 +44,8 @@ struct efa_user_mmap_entry { op(EFA_CREATE_CQ_ERR, "create_cq_err") \ op(EFA_REG_MR_ERR, "reg_mr_err") \ op(EFA_ALLOC_UCONTEXT_ERR, "alloc_ucontext_err") \ - op(EFA_CREATE_AH_ERR, "create_ah_err") + op(EFA_CREATE_AH_ERR, "create_ah_err") \ + op(EFA_MMAP_ERR, "mmap_err")#define EFA_STATS_ENUM (ename, name) ename,#define EFA_STATS_STR (ename, name) [ename] = name,@@ -1569,6 +1570,7 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,ibdev_dbg (&dev->ibdev,"pgoff[%#lx] does not have valid entry\n" , vma->vm_pgoff); +atomic64_inc (&dev->stats.sw_stats.mmap_err);return -EINVAL ; } entry =to_emmap (rdma_entry);@@ -1604,12 +1606,14 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, err = -EINVAL ;
drivers/infiniband/hw/efa/efa_verbs.c static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, · line 1615
Quoted 10 lines
drivers/infiniband/hw/efa/efa_verbs.c +2
-1} -if (err) +if (err) {ibdev_dbg ( &dev->ibdev,"Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n" , entry->address, rdma_entry->npages *PAGE_SIZE , entry->mmap_flag, err); +atomic64_inc (&dev->stats.sw_stats.mmap_err);
Really? Isn't this something that is only possible with a buggy rdma-core provider? Why count it?
Jason
Re: [PATCH for-next 2/3] RDMA/efa: Count mmap failures
Quoted 55 lines · Jason Gunthorpe
Quoted 54 lines · Gal Pressman
Add a new stat that counts mmap failures, which might help when debugging different issues.
Reviewed-by Firas JahJah <[email protected]>Reviewed-by Yossi Leybovich <[email protected]>Signed-off-by Gal Pressman <[email protected]>drivers/infiniband/hw/efa/efa.h | 3 ++- drivers/infiniband/hw/efa/efa_verbs.c | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-)drivers/infiniband/hw/efa/efa.h +3
-1+++ b/drivers/infiniband/hw/efa/efa.h@@ -1,6 +1,6 @@/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ /* -* Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved. +* Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved. */ #ifndef _EFA_H_@@ -40,6 +40,7 @@ struct efa_sw_stats {atomic64_t reg_mr_err;atomic64_t alloc_ucontext_err;atomic64_t create_ah_err; +atomic64_t mmap_err; };/* Don't use anything other than atomic64 */ drivers/infiniband/hw/efa/efa_verbs.c +4
-1+++ b/drivers/infiniband/hw/efa/efa_verbs.c@@ -44,7 +44,8 @@ struct efa_user_mmap_entry { op(EFA_CREATE_CQ_ERR, "create_cq_err") \ op(EFA_REG_MR_ERR, "reg_mr_err") \ op(EFA_ALLOC_UCONTEXT_ERR, "alloc_ucontext_err") \ - op(EFA_CREATE_AH_ERR, "create_ah_err") + op(EFA_CREATE_AH_ERR, "create_ah_err") \ + op(EFA_MMAP_ERR, "mmap_err")#define EFA_STATS_ENUM (ename, name) ename,#define EFA_STATS_STR (ename, name) [ename] = name,@@ -1569,6 +1570,7 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,ibdev_dbg (&dev->ibdev,"pgoff[%#lx] does not have valid entry\n" , vma->vm_pgoff); +atomic64_inc (&dev->stats.sw_stats.mmap_err);return -EINVAL ; } entry =to_emmap (rdma_entry);@@ -1604,12 +1606,14 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, err = -EINVAL ; }
Quoted 11 lines
drivers/infiniband/hw/efa/efa_verbs.c static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, · line 1615
Quoted 8 lines
drivers/infiniband/hw/efa/efa_verbs.c +2
-1-if (err) +if (err) {ibdev_dbg ( &dev->ibdev,"Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n" , entry->address, rdma_entry->npages *PAGE_SIZE , entry->mmap_flag, err); +atomic64_inc (&dev->stats.sw_stats.mmap_err);Really? Isn't this something that is only possible with a buggy rdma-core provider? Why count it?
Though unlikely, it could happen, otherwise this error flow wouldn't exist in the first place.
If for some reason a customer app steps on a bug we're not aware of, this counter could serve as a red flag.
Re: [PATCH for-next 2/3] RDMA/efa: Count mmap failures
Quoted 68 lines · Gal Pressman
Quoted 55 lines · Jason Gunthorpe
Quoted 54 lines · Gal Pressman
Add a new stat that counts mmap failures, which might help when debugging different issues.
Reviewed-by Firas JahJah <[email protected]>Reviewed-by Yossi Leybovich <[email protected]>Signed-off-by Gal Pressman <[email protected]>drivers/infiniband/hw/efa/efa.h | 3 ++- drivers/infiniband/hw/efa/efa_verbs.c | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-)drivers/infiniband/hw/efa/efa.h +3
-1+++ b/drivers/infiniband/hw/efa/efa.h@@ -1,6 +1,6 @@/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ /* -* Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved. +* Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved. */ #ifndef _EFA_H_@@ -40,6 +40,7 @@ struct efa_sw_stats {atomic64_t reg_mr_err;atomic64_t alloc_ucontext_err;atomic64_t create_ah_err; +atomic64_t mmap_err; };/* Don't use anything other than atomic64 */ drivers/infiniband/hw/efa/efa_verbs.c +4
-1+++ b/drivers/infiniband/hw/efa/efa_verbs.c@@ -44,7 +44,8 @@ struct efa_user_mmap_entry { op(EFA_CREATE_CQ_ERR, "create_cq_err") \ op(EFA_REG_MR_ERR, "reg_mr_err") \ op(EFA_ALLOC_UCONTEXT_ERR, "alloc_ucontext_err") \ - op(EFA_CREATE_AH_ERR, "create_ah_err") + op(EFA_CREATE_AH_ERR, "create_ah_err") \ + op(EFA_MMAP_ERR, "mmap_err")#define EFA_STATS_ENUM (ename, name) ename,#define EFA_STATS_STR (ename, name) [ename] = name,@@ -1569,6 +1570,7 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,ibdev_dbg (&dev->ibdev,"pgoff[%#lx] does not have valid entry\n" , vma->vm_pgoff); +atomic64_inc (&dev->stats.sw_stats.mmap_err);return -EINVAL ; } entry =to_emmap (rdma_entry);@@ -1604,12 +1606,14 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, err = -EINVAL ; }Quoted 11 lines
drivers/infiniband/hw/efa/efa_verbs.c static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, · line 1615
Quoted 8 lines
drivers/infiniband/hw/efa/efa_verbs.c +2
-1-if (err) +if (err) {ibdev_dbg ( &dev->ibdev,"Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n" , entry->address, rdma_entry->npages *PAGE_SIZE , entry->mmap_flag, err); +atomic64_inc (&dev->stats.sw_stats.mmap_err);Really? Isn't this something that is only possible with a buggy rdma-core provider? Why count it?
Quoted 5 lines
Though unlikely, it could happen, otherwise this error flow wouldn't exist in the first place.
If for some reason a customer app steps on a bug we're not aware of, this counter could serve as a red flag.
But there are lots of cases where a buggy provider can cause error exits, why choose this one to count against all the others?
Jason
Re: [PATCH for-next 2/3] RDMA/efa: Count mmap failures
Quoted 69 lines · Jason Gunthorpe
Quoted 68 lines · Gal Pressman
Quoted 66 lines · Jason Gunthorpe
Quoted 52 lines · Gal Pressman
Add a new stat that counts mmap failures, which might help when debugging different issues.
Reviewed-by Firas JahJah <[email protected]>Reviewed-by Yossi Leybovich <[email protected]>Signed-off-by Gal Pressman <[email protected]>drivers/infiniband/hw/efa/efa.h | 3 ++- drivers/infiniband/hw/efa/efa_verbs.c | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-)drivers/infiniband/hw/efa/efa.h +3
-1+++ b/drivers/infiniband/hw/efa/efa.h@@ -1,6 +1,6 @@/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ /* -* Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved. +* Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved. */ #ifndef _EFA_H_@@ -40,6 +40,7 @@ struct efa_sw_stats {atomic64_t reg_mr_err;atomic64_t alloc_ucontext_err;atomic64_t create_ah_err; +atomic64_t mmap_err; };/* Don't use anything other than atomic64 */ drivers/infiniband/hw/efa/efa_verbs.c +4
-1+++ b/drivers/infiniband/hw/efa/efa_verbs.c@@ -44,7 +44,8 @@ struct efa_user_mmap_entry { op(EFA_CREATE_CQ_ERR, "create_cq_err") \ op(EFA_REG_MR_ERR, "reg_mr_err") \ op(EFA_ALLOC_UCONTEXT_ERR, "alloc_ucontext_err") \ - op(EFA_CREATE_AH_ERR, "create_ah_err") + op(EFA_CREATE_AH_ERR, "create_ah_err") \ + op(EFA_MMAP_ERR, "mmap_err")#define EFA_STATS_ENUM (ename, name) ename,#define EFA_STATS_STR (ename, name) [ename] = name,@@ -1569,6 +1570,7 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,ibdev_dbg (&dev->ibdev,"pgoff[%#lx] does not have valid entry\n" , vma->vm_pgoff); +atomic64_inc (&dev->stats.sw_stats.mmap_err);return -EINVAL ; } entry =to_emmap (rdma_entry);@@ -1604,12 +1606,14 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, err = -EINVAL ;drivers/infiniband/hw/efa/efa_verbs.c static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, · line 1615
Quoted 10 lines
drivers/infiniband/hw/efa/efa_verbs.c +2
-1} -if (err) +if (err) {ibdev_dbg ( &dev->ibdev,"Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n" , entry->address, rdma_entry->npages *PAGE_SIZE , entry->mmap_flag, err); +atomic64_inc (&dev->stats.sw_stats.mmap_err);Really? Isn't this something that is only possible with a buggy rdma-core provider? Why count it?
Quoted 8 lines
Quoted 5 lines
Though unlikely, it could happen, otherwise this error flow wouldn't exist in the first place.
If for some reason a customer app steps on a bug we're not aware of, this counter could serve as a red flag.
But there are lots of cases where a buggy provider can cause error exits, why choose this one to count against all the others?
It's not one against all others, most if not all of our userspace facing API error flows have a similar counter.
And TBH, I think that the mmap flow is quite convoluted with the cookie response from the crate verb, so it deserves a counter IMO.
Re: [PATCH for-next 2/3] RDMA/efa: Count mmap failures
Quoted 73 lines · Gal Pressman
Quoted 72 lines · Jason Gunthorpe
Quoted 71 lines · Gal Pressman
Quoted 55 lines · Jason Gunthorpe
Quoted 54 lines · Gal Pressman
Add a new stat that counts mmap failures, which might help when debugging different issues.
Reviewed-by Firas JahJah <[email protected]>Reviewed-by Yossi Leybovich <[email protected]>Signed-off-by Gal Pressman <[email protected]>drivers/infiniband/hw/efa/efa.h | 3 ++- drivers/infiniband/hw/efa/efa_verbs.c | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-)drivers/infiniband/hw/efa/efa.h +3
-1+++ b/drivers/infiniband/hw/efa/efa.h@@ -1,6 +1,6 @@/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ /* -* Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved. +* Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved. */ #ifndef _EFA_H_@@ -40,6 +40,7 @@ struct efa_sw_stats {atomic64_t reg_mr_err;atomic64_t alloc_ucontext_err;atomic64_t create_ah_err; +atomic64_t mmap_err; };/* Don't use anything other than atomic64 */ drivers/infiniband/hw/efa/efa_verbs.c +4
-1+++ b/drivers/infiniband/hw/efa/efa_verbs.c@@ -44,7 +44,8 @@ struct efa_user_mmap_entry { op(EFA_CREATE_CQ_ERR, "create_cq_err") \ op(EFA_REG_MR_ERR, "reg_mr_err") \ op(EFA_ALLOC_UCONTEXT_ERR, "alloc_ucontext_err") \ - op(EFA_CREATE_AH_ERR, "create_ah_err") + op(EFA_CREATE_AH_ERR, "create_ah_err") \ + op(EFA_MMAP_ERR, "mmap_err")#define EFA_STATS_ENUM (ename, name) ename,#define EFA_STATS_STR (ename, name) [ename] = name,@@ -1569,6 +1570,7 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,ibdev_dbg (&dev->ibdev,"pgoff[%#lx] does not have valid entry\n" , vma->vm_pgoff); +atomic64_inc (&dev->stats.sw_stats.mmap_err);return -EINVAL ; } entry =to_emmap (rdma_entry);@@ -1604,12 +1606,14 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, err = -EINVAL ; }Quoted 11 lines
drivers/infiniband/hw/efa/efa_verbs.c static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, · line 1615
Quoted 8 lines
drivers/infiniband/hw/efa/efa_verbs.c +2
-1-if (err) +if (err) {ibdev_dbg ( &dev->ibdev,"Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n" , entry->address, rdma_entry->npages *PAGE_SIZE , entry->mmap_flag, err); +atomic64_inc (&dev->stats.sw_stats.mmap_err);Really? Isn't this something that is only possible with a buggy rdma-core provider? Why count it?
Though unlikely, it could happen, otherwise this error flow wouldn't exist in the first place.
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
If for some reason a customer app steps on a bug we're not aware of, this counter could serve as a red flag.
But there are lots of cases where a buggy provider can cause error exits, why choose this one to count against all the others?
It's not one against all others, most if not all of our userspace facing API error flows have a similar counter.
Hurm, seems a bit strange, but OK
Quoted 2 lines
And TBH, I think that the mmap flow is quite convoluted with the cookie response from the crate verb, so it deserves a counter IMO.
How so? Userspace takes the u64 from the command and pass it to mmap, what is convoluted?
Jason
Re: [PATCH for-next 2/3] RDMA/efa: Count mmap failures
Quoted 83 lines · Jason Gunthorpe
Quoted 73 lines · Gal Pressman
Quoted 72 lines · Jason Gunthorpe
Quoted 71 lines · Gal Pressman
Quoted 55 lines · Jason Gunthorpe
Quoted 54 lines · Gal Pressman
Add a new stat that counts mmap failures, which might help when debugging different issues.
Reviewed-by Firas JahJah <[email protected]>Reviewed-by Yossi Leybovich <[email protected]>Signed-off-by Gal Pressman <[email protected]>drivers/infiniband/hw/efa/efa.h | 3 ++- drivers/infiniband/hw/efa/efa_verbs.c | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-)drivers/infiniband/hw/efa/efa.h +3
-1+++ b/drivers/infiniband/hw/efa/efa.h@@ -1,6 +1,6 @@/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ /* -* Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved. +* Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved. */ #ifndef _EFA_H_@@ -40,6 +40,7 @@ struct efa_sw_stats {atomic64_t reg_mr_err;atomic64_t alloc_ucontext_err;atomic64_t create_ah_err; +atomic64_t mmap_err; };/* Don't use anything other than atomic64 */ drivers/infiniband/hw/efa/efa_verbs.c +4
-1+++ b/drivers/infiniband/hw/efa/efa_verbs.c@@ -44,7 +44,8 @@ struct efa_user_mmap_entry { op(EFA_CREATE_CQ_ERR, "create_cq_err") \ op(EFA_REG_MR_ERR, "reg_mr_err") \ op(EFA_ALLOC_UCONTEXT_ERR, "alloc_ucontext_err") \ - op(EFA_CREATE_AH_ERR, "create_ah_err") + op(EFA_CREATE_AH_ERR, "create_ah_err") \ + op(EFA_MMAP_ERR, "mmap_err")#define EFA_STATS_ENUM (ename, name) ename,#define EFA_STATS_STR (ename, name) [ename] = name,@@ -1569,6 +1570,7 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,ibdev_dbg (&dev->ibdev,"pgoff[%#lx] does not have valid entry\n" , vma->vm_pgoff); +atomic64_inc (&dev->stats.sw_stats.mmap_err);return -EINVAL ; } entry =to_emmap (rdma_entry);@@ -1604,12 +1606,14 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, err = -EINVAL ; }Quoted 11 lines
drivers/infiniband/hw/efa/efa_verbs.c static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, · line 1615
Quoted 8 lines
drivers/infiniband/hw/efa/efa_verbs.c +2
-1-if (err) +if (err) {ibdev_dbg ( &dev->ibdev,"Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n" , entry->address, rdma_entry->npages *PAGE_SIZE , entry->mmap_flag, err); +atomic64_inc (&dev->stats.sw_stats.mmap_err);Really? Isn't this something that is only possible with a buggy rdma-core provider? Why count it?
Though unlikely, it could happen, otherwise this error flow wouldn't exist in the first place.
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
If for some reason a customer app steps on a bug we're not aware of, this counter could serve as a red flag.
But there are lots of cases where a buggy provider can cause error exits, why choose this one to count against all the others?
It's not one against all others, most if not all of our userspace facing API error flows have a similar counter.
Quoted 7 lines
Hurm, seems a bit strange, but OK
Quoted 2 lines
And TBH, I think that the mmap flow is quite convoluted with the cookie response from the crate verb, so it deserves a counter IMO.
How so? Userspace takes the u64 from the command and pass it to mmap, what is convoluted?
It's the only flow that involves two phases/userspace calls and not a single command-response call, so it's a bit more error prone. Convoluted was a bit harsh :).
Jason Gunthorpe
AppliedQuoted 2 lines · Gal Pressman
This series contains three small patches to collect a few counters we found helpful when encountering different issues.
Applied to for-next, thanks
Jason