Skip to contentSkip to conversation

EFA statistics minor updates

PATCH Applied Gal Pressman 2 participants 11 messages Apr 20, 2020 – May 2, 2020

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover EFA statistics minor updates

This series contains three small patches to collect a few counters we found helpful when encountering different issues.

Regards,
Gal

Gal Pressman (3):
RDMA/efa: Report create CQ error counter
RDMA/efa: Count mmap failures
RDMA/efa: Count admin commands errors

 drivers/infiniband/hw/efa/efa.h       |  3 ++-
 drivers/infiniband/hw/efa/efa_com.c   |  5 ++++-
 drivers/infiniband/hw/efa/efa_com.h   |  3 ++-
 drivers/infiniband/hw/efa/efa_verbs.c | 13 +++++++++++--
 4 files changed, 19 insertions(+), 5 deletions(-)

base-commit: 8f3d9f354286745c751374f5f1fcafee6b3f3136

Signature · 1 line
2.26.1
#1 lore raw

PATCH 1/3 RDMA/efa: Report create CQ error counter

Create CQ errors are already being counted, report them along all other counters.

---
 drivers/infiniband/hw/efa/efa_verbs.c | 2 ++
 1 file changed, 2 insertions(+)
drivers/infiniband/hw/efa/efa_verbs.c +2 -0
index 5c57098a4aee..b555845d6c14 100644
@@ -41,6 +41,7 @@ struct efa_user_mmap_entry {
 	op(EFA_KEEP_ALIVE_RCVD, "keep_alive_rcvd") \
 	op(EFA_ALLOC_PD_ERR, "alloc_pd_err") \
 	op(EFA_CREATE_QP_ERR, "create_qp_err") \
+	op(EFA_CREATE_CQ_ERR, "create_cq_err") \
 	op(EFA_REG_MR_ERR, "reg_mr_err") \
 	op(EFA_ALLOC_UCONTEXT_ERR, "alloc_ucontext_err") \
 	op(EFA_CREATE_AH_ERR, "create_ah_err")
@@ -1753,6 +1754,7 @@ int efa_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
 	stats->value[EFA_KEEP_ALIVE_RCVD] = atomic64_read(&s->keep_alive_rcvd);
 	stats->value[EFA_ALLOC_PD_ERR] = atomic64_read(&s->sw_stats.alloc_pd_err);
 	stats->value[EFA_CREATE_QP_ERR] = atomic64_read(&s->sw_stats.create_qp_err);
+	stats->value[EFA_CREATE_CQ_ERR] = atomic64_read(&s->sw_stats.create_cq_err);
 	stats->value[EFA_REG_MR_ERR] = atomic64_read(&s->sw_stats.reg_mr_err);
 	stats->value[EFA_ALLOC_UCONTEXT_ERR] = atomic64_read(&s->sw_stats.alloc_ucontext_err);
 	stats->value[EFA_CREATE_AH_ERR] = atomic64_read(&s->sw_stats.create_ah_err);
Signature · 1 line
2.26.1
#2 lore raw

PATCH 2/3 RDMA/efa: Count mmap failures

In reply to Gal Pressman

Add a new stat that counts mmap failures, which might help when debugging different issues.

---
 drivers/infiniband/hw/efa/efa.h       | 3 ++-
 drivers/infiniband/hw/efa/efa_verbs.c | 9 +++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)
drivers/infiniband/hw/efa/efa.h +2 -1
index aa7396a1588a..77c9ff798117 100644
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_H_
@@ -40,6 +40,7 @@ struct efa_sw_stats {
 	atomic64_t reg_mr_err;
 	atomic64_t alloc_ucontext_err;
 	atomic64_t create_ah_err;
+	atomic64_t mmap_err;
 };
 
 /* Don't use anything other than atomic64 */
drivers/infiniband/hw/efa/efa_verbs.c +7 -2
index b555845d6c14..75eef1ec2474 100644
@@ -44,7 +44,8 @@ struct efa_user_mmap_entry {
 	op(EFA_CREATE_CQ_ERR, "create_cq_err") \
 	op(EFA_REG_MR_ERR, "reg_mr_err") \
 	op(EFA_ALLOC_UCONTEXT_ERR, "alloc_ucontext_err") \
-	op(EFA_CREATE_AH_ERR, "create_ah_err")
+	op(EFA_CREATE_AH_ERR, "create_ah_err") \
+	op(EFA_MMAP_ERR, "mmap_err")
 
 #define EFA_STATS_ENUM(ename, name) ename,
 #define EFA_STATS_STR(ename, name) [ename] = name,
@@ -1569,6 +1570,7 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,
 		ibdev_dbg(&dev->ibdev,
 			  "pgoff[%#lx] does not have valid entry\n",
 			  vma->vm_pgoff);
+		atomic64_inc(&dev->stats.sw_stats.mmap_err);
 		return -EINVAL;
 	}
 	entry = to_emmap(rdma_entry);
@@ -1604,12 +1606,14 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,
 		err = -EINVAL;
 	}
 
-	if (err)
+	if (err) {
 		ibdev_dbg(
 			&dev->ibdev,
 			"Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
 			entry->address, rdma_entry->npages * PAGE_SIZE,
 			entry->mmap_flag, err);
+		atomic64_inc(&dev->stats.sw_stats.mmap_err);
+	}
 
 	rdma_user_mmap_entry_put(rdma_entry);
 	return err;
@@ -1758,6 +1762,7 @@ int efa_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
 	stats->value[EFA_REG_MR_ERR] = atomic64_read(&s->sw_stats.reg_mr_err);
 	stats->value[EFA_ALLOC_UCONTEXT_ERR] = atomic64_read(&s->sw_stats.alloc_ucontext_err);
 	stats->value[EFA_CREATE_AH_ERR] = atomic64_read(&s->sw_stats.create_ah_err);
+	stats->value[EFA_MMAP_ERR] = atomic64_read(&s->sw_stats.mmap_err);
 
 	return ARRAY_SIZE(efa_stats_names);
 }
Signature · 1 line
2.26.1
#9 lore raw

PATCH 3/3 RDMA/efa: Count admin commands errors

In reply to Gal Pressman

Add a new stat that counts admin commands failures, which might help when debugging different issues.

---
 drivers/infiniband/hw/efa/efa_com.c   | 5 ++++-
 drivers/infiniband/hw/efa/efa_com.h   | 3 ++-
 drivers/infiniband/hw/efa/efa_verbs.c | 2 ++
 3 files changed, 8 insertions(+), 2 deletions(-)
drivers/infiniband/hw/efa/efa_com.c +4 -1
index 7fce69f5568f..336bc2c57bb1 100644
@@ -631,17 +631,20 @@ int efa_com_cmd_exec(struct efa_com_admin_queue *aq,
 			cmd->aq_common_descriptor.opcode, PTR_ERR(comp_ctx));
 
 		up(&aq->avail_cmds);
+		atomic64_inc(&aq->stats.cmd_err);
 		return PTR_ERR(comp_ctx);
 	}
 
 	err = efa_com_wait_and_process_admin_cq(comp_ctx, aq);
-	if (err)
+	if (err) {
 		ibdev_err_ratelimited(
 			aq->efa_dev,
 			"Failed to process command %s (opcode %u) comp_status %d err %d\n",
 			efa_com_cmd_str(cmd->aq_common_descriptor.opcode),
 			cmd->aq_common_descriptor.opcode, comp_ctx->comp_status,
 			err);
+		atomic64_inc(&aq->stats.cmd_err);
+	}
 
 	up(&aq->avail_cmds);
 
drivers/infiniband/hw/efa/efa_com.h +2 -1
index c67dd8109d1c..5e4c88877ddb 100644
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_COM_H_
@@ -47,6 +47,7 @@ struct efa_com_admin_sq {
 struct efa_com_stats_admin {
 	atomic64_t submitted_cmd;
 	atomic64_t completed_cmd;
+	atomic64_t cmd_err;
 	atomic64_t no_completion;
 };
 
drivers/infiniband/hw/efa/efa_verbs.c +2 -0
index 75eef1ec2474..ce7a3ead1ba7 100644
@@ -37,6 +37,7 @@ struct efa_user_mmap_entry {
 	op(EFA_RX_DROPS, "rx_drops") \
 	op(EFA_SUBMITTED_CMDS, "submitted_cmds") \
 	op(EFA_COMPLETED_CMDS, "completed_cmds") \
+	op(EFA_CMDS_ERR, "cmds_err") \
 	op(EFA_NO_COMPLETION_CMDS, "no_completion_cmds") \
 	op(EFA_KEEP_ALIVE_RCVD, "keep_alive_rcvd") \
 	op(EFA_ALLOC_PD_ERR, "alloc_pd_err") \
@@ -1752,6 +1753,7 @@ int efa_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
 	as = &dev->edev.aq.stats;
 	stats->value[EFA_SUBMITTED_CMDS] = atomic64_read(&as->submitted_cmd);
 	stats->value[EFA_COMPLETED_CMDS] = atomic64_read(&as->completed_cmd);
+	stats->value[EFA_CMDS_ERR] = atomic64_read(&as->cmd_err);
 	stats->value[EFA_NO_COMPLETION_CMDS] = atomic64_read(&as->no_completion);
 
 	s = &dev->stats;
Signature · 1 line
2.26.1
#3 lore raw

Re: [PATCH for-next 2/3] RDMA/efa: Count mmap failures

In reply to Gal Pressman
Quoted 52 lines · Gal Pressman

Add a new stat that counts mmap failures, which might help when debugging different issues.

 drivers/infiniband/hw/efa/efa.h       | 3 ++-
 drivers/infiniband/hw/efa/efa_verbs.c | 9 +++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)
drivers/infiniband/hw/efa/efa.h +3 -1
index aa7396a1588a..77c9ff798117 100644
+++ b/drivers/infiniband/hw/efa/efa.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_H_
@@ -40,6 +40,7 @@ struct efa_sw_stats {
 	atomic64_t reg_mr_err;
 	atomic64_t alloc_ucontext_err;
 	atomic64_t create_ah_err;
+	atomic64_t mmap_err;
 };
 
 /* Don't use anything other than atomic64 */
drivers/infiniband/hw/efa/efa_verbs.c +4 -1
index b555845d6c14..75eef1ec2474 100644
+++ b/drivers/infiniband/hw/efa/efa_verbs.c
@@ -44,7 +44,8 @@ struct efa_user_mmap_entry {
 	op(EFA_CREATE_CQ_ERR, "create_cq_err") \
 	op(EFA_REG_MR_ERR, "reg_mr_err") \
 	op(EFA_ALLOC_UCONTEXT_ERR, "alloc_ucontext_err") \
-	op(EFA_CREATE_AH_ERR, "create_ah_err")
+	op(EFA_CREATE_AH_ERR, "create_ah_err") \
+	op(EFA_MMAP_ERR, "mmap_err")
 
 #define EFA_STATS_ENUM(ename, name) ename,
 #define EFA_STATS_STR(ename, name) [ename] = name,
@@ -1569,6 +1570,7 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,
 		ibdev_dbg(&dev->ibdev,
 			  "pgoff[%#lx] does not have valid entry\n",
 			  vma->vm_pgoff);
+		atomic64_inc(&dev->stats.sw_stats.mmap_err);
 		return -EINVAL;
 	}
 	entry = to_emmap(rdma_entry);
@@ -1604,12 +1606,14 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,
 		err = -EINVAL;

drivers/infiniband/hw/efa/efa_verbs.c static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, · line 1615

Quoted 10 lines
drivers/infiniband/hw/efa/efa_verbs.c +2 -1
 	}
 
-	if (err)
+	if (err) {
 		ibdev_dbg(
 			&dev->ibdev,
 			"Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
 			entry->address, rdma_entry->npages * PAGE_SIZE,
 			entry->mmap_flag, err);
+		atomic64_inc(&dev->stats.sw_stats.mmap_err);

Really? Isn't this something that is only possible with a buggy rdma-core provider? Why count it?

Jason

#4 lore raw

Re: [PATCH for-next 2/3] RDMA/efa: Count mmap failures

Quoted 55 lines · Jason Gunthorpe
Quoted 54 lines · Gal Pressman

Add a new stat that counts mmap failures, which might help when debugging different issues.

 drivers/infiniband/hw/efa/efa.h       | 3 ++-
 drivers/infiniband/hw/efa/efa_verbs.c | 9 +++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)
drivers/infiniband/hw/efa/efa.h +3 -1
index aa7396a1588a..77c9ff798117 100644
+++ b/drivers/infiniband/hw/efa/efa.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_H_
@@ -40,6 +40,7 @@ struct efa_sw_stats {
 	atomic64_t reg_mr_err;
 	atomic64_t alloc_ucontext_err;
 	atomic64_t create_ah_err;
+	atomic64_t mmap_err;
 };
 
 /* Don't use anything other than atomic64 */
drivers/infiniband/hw/efa/efa_verbs.c +4 -1
index b555845d6c14..75eef1ec2474 100644
+++ b/drivers/infiniband/hw/efa/efa_verbs.c
@@ -44,7 +44,8 @@ struct efa_user_mmap_entry {
 	op(EFA_CREATE_CQ_ERR, "create_cq_err") \
 	op(EFA_REG_MR_ERR, "reg_mr_err") \
 	op(EFA_ALLOC_UCONTEXT_ERR, "alloc_ucontext_err") \
-	op(EFA_CREATE_AH_ERR, "create_ah_err")
+	op(EFA_CREATE_AH_ERR, "create_ah_err") \
+	op(EFA_MMAP_ERR, "mmap_err")
 
 #define EFA_STATS_ENUM(ename, name) ename,
 #define EFA_STATS_STR(ename, name) [ename] = name,
@@ -1569,6 +1570,7 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,
 		ibdev_dbg(&dev->ibdev,
 			  "pgoff[%#lx] does not have valid entry\n",
 			  vma->vm_pgoff);
+		atomic64_inc(&dev->stats.sw_stats.mmap_err);
 		return -EINVAL;
 	}
 	entry = to_emmap(rdma_entry);
@@ -1604,12 +1606,14 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,
 		err = -EINVAL;
 	}
 
Quoted 11 lines

drivers/infiniband/hw/efa/efa_verbs.c static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, · line 1615

Quoted 8 lines
drivers/infiniband/hw/efa/efa_verbs.c +2 -1
-	if (err)
+	if (err) {
 		ibdev_dbg(
 			&dev->ibdev,
 			"Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
 			entry->address, rdma_entry->npages * PAGE_SIZE,
 			entry->mmap_flag, err);
+		atomic64_inc(&dev->stats.sw_stats.mmap_err);

Really? Isn't this something that is only possible with a buggy rdma-core provider? Why count it?

Though unlikely, it could happen, otherwise this error flow wouldn't exist in the first place.

If for some reason a customer app steps on a bug we're not aware of, this counter could serve as a red flag.

#5 lore raw

Re: [PATCH for-next 2/3] RDMA/efa: Count mmap failures

Quoted 68 lines · Gal Pressman
Quoted 55 lines · Jason Gunthorpe
Quoted 54 lines · Gal Pressman

Add a new stat that counts mmap failures, which might help when debugging different issues.

 drivers/infiniband/hw/efa/efa.h       | 3 ++-
 drivers/infiniband/hw/efa/efa_verbs.c | 9 +++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)
drivers/infiniband/hw/efa/efa.h +3 -1
index aa7396a1588a..77c9ff798117 100644
+++ b/drivers/infiniband/hw/efa/efa.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_H_
@@ -40,6 +40,7 @@ struct efa_sw_stats {
 	atomic64_t reg_mr_err;
 	atomic64_t alloc_ucontext_err;
 	atomic64_t create_ah_err;
+	atomic64_t mmap_err;
 };
 
 /* Don't use anything other than atomic64 */
drivers/infiniband/hw/efa/efa_verbs.c +4 -1
index b555845d6c14..75eef1ec2474 100644
+++ b/drivers/infiniband/hw/efa/efa_verbs.c
@@ -44,7 +44,8 @@ struct efa_user_mmap_entry {
 	op(EFA_CREATE_CQ_ERR, "create_cq_err") \
 	op(EFA_REG_MR_ERR, "reg_mr_err") \
 	op(EFA_ALLOC_UCONTEXT_ERR, "alloc_ucontext_err") \
-	op(EFA_CREATE_AH_ERR, "create_ah_err")
+	op(EFA_CREATE_AH_ERR, "create_ah_err") \
+	op(EFA_MMAP_ERR, "mmap_err")
 
 #define EFA_STATS_ENUM(ename, name) ename,
 #define EFA_STATS_STR(ename, name) [ename] = name,
@@ -1569,6 +1570,7 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,
 		ibdev_dbg(&dev->ibdev,
 			  "pgoff[%#lx] does not have valid entry\n",
 			  vma->vm_pgoff);
+		atomic64_inc(&dev->stats.sw_stats.mmap_err);
 		return -EINVAL;
 	}
 	entry = to_emmap(rdma_entry);
@@ -1604,12 +1606,14 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,
 		err = -EINVAL;
 	}
 
Quoted 11 lines

drivers/infiniband/hw/efa/efa_verbs.c static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, · line 1615

Quoted 8 lines
drivers/infiniband/hw/efa/efa_verbs.c +2 -1
-	if (err)
+	if (err) {
 		ibdev_dbg(
 			&dev->ibdev,
 			"Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
 			entry->address, rdma_entry->npages * PAGE_SIZE,
 			entry->mmap_flag, err);
+		atomic64_inc(&dev->stats.sw_stats.mmap_err);

Really? Isn't this something that is only possible with a buggy rdma-core provider? Why count it?

Quoted 5 lines

Though unlikely, it could happen, otherwise this error flow wouldn't exist in the first place.

If for some reason a customer app steps on a bug we're not aware of, this counter could serve as a red flag.

But there are lots of cases where a buggy provider can cause error exits, why choose this one to count against all the others?

Jason

#6 lore raw

Re: [PATCH for-next 2/3] RDMA/efa: Count mmap failures

Quoted 69 lines · Jason Gunthorpe
Quoted 68 lines · Gal Pressman
Quoted 66 lines · Jason Gunthorpe
Quoted 52 lines · Gal Pressman

Add a new stat that counts mmap failures, which might help when debugging different issues.

 drivers/infiniband/hw/efa/efa.h       | 3 ++-
 drivers/infiniband/hw/efa/efa_verbs.c | 9 +++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)
drivers/infiniband/hw/efa/efa.h +3 -1
index aa7396a1588a..77c9ff798117 100644
+++ b/drivers/infiniband/hw/efa/efa.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_H_
@@ -40,6 +40,7 @@ struct efa_sw_stats {
 	atomic64_t reg_mr_err;
 	atomic64_t alloc_ucontext_err;
 	atomic64_t create_ah_err;
+	atomic64_t mmap_err;
 };
 
 /* Don't use anything other than atomic64 */
drivers/infiniband/hw/efa/efa_verbs.c +4 -1
index b555845d6c14..75eef1ec2474 100644
+++ b/drivers/infiniband/hw/efa/efa_verbs.c
@@ -44,7 +44,8 @@ struct efa_user_mmap_entry {
 	op(EFA_CREATE_CQ_ERR, "create_cq_err") \
 	op(EFA_REG_MR_ERR, "reg_mr_err") \
 	op(EFA_ALLOC_UCONTEXT_ERR, "alloc_ucontext_err") \
-	op(EFA_CREATE_AH_ERR, "create_ah_err")
+	op(EFA_CREATE_AH_ERR, "create_ah_err") \
+	op(EFA_MMAP_ERR, "mmap_err")
 
 #define EFA_STATS_ENUM(ename, name) ename,
 #define EFA_STATS_STR(ename, name) [ename] = name,
@@ -1569,6 +1570,7 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,
 		ibdev_dbg(&dev->ibdev,
 			  "pgoff[%#lx] does not have valid entry\n",
 			  vma->vm_pgoff);
+		atomic64_inc(&dev->stats.sw_stats.mmap_err);
 		return -EINVAL;
 	}
 	entry = to_emmap(rdma_entry);
@@ -1604,12 +1606,14 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,
 		err = -EINVAL;

drivers/infiniband/hw/efa/efa_verbs.c static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, · line 1615

Quoted 10 lines
drivers/infiniband/hw/efa/efa_verbs.c +2 -1
 	}
 
-	if (err)
+	if (err) {
 		ibdev_dbg(
 			&dev->ibdev,
 			"Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
 			entry->address, rdma_entry->npages * PAGE_SIZE,
 			entry->mmap_flag, err);
+		atomic64_inc(&dev->stats.sw_stats.mmap_err);

Really? Isn't this something that is only possible with a buggy rdma-core provider? Why count it?

Quoted 8 lines
Quoted 5 lines

Though unlikely, it could happen, otherwise this error flow wouldn't exist in the first place.

If for some reason a customer app steps on a bug we're not aware of, this counter could serve as a red flag.

But there are lots of cases where a buggy provider can cause error exits, why choose this one to count against all the others?

It's not one against all others, most if not all of our userspace facing API error flows have a similar counter.
And TBH, I think that the mmap flow is quite convoluted with the cookie response from the crate verb, so it deserves a counter IMO.

#7 lore raw

Re: [PATCH for-next 2/3] RDMA/efa: Count mmap failures

Quoted 73 lines · Gal Pressman
Quoted 72 lines · Jason Gunthorpe
Quoted 71 lines · Gal Pressman
Quoted 55 lines · Jason Gunthorpe
Quoted 54 lines · Gal Pressman

Add a new stat that counts mmap failures, which might help when debugging different issues.

 drivers/infiniband/hw/efa/efa.h       | 3 ++-
 drivers/infiniband/hw/efa/efa_verbs.c | 9 +++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)
drivers/infiniband/hw/efa/efa.h +3 -1
index aa7396a1588a..77c9ff798117 100644
+++ b/drivers/infiniband/hw/efa/efa.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_H_
@@ -40,6 +40,7 @@ struct efa_sw_stats {
 	atomic64_t reg_mr_err;
 	atomic64_t alloc_ucontext_err;
 	atomic64_t create_ah_err;
+	atomic64_t mmap_err;
 };
 
 /* Don't use anything other than atomic64 */
drivers/infiniband/hw/efa/efa_verbs.c +4 -1
index b555845d6c14..75eef1ec2474 100644
+++ b/drivers/infiniband/hw/efa/efa_verbs.c
@@ -44,7 +44,8 @@ struct efa_user_mmap_entry {
 	op(EFA_CREATE_CQ_ERR, "create_cq_err") \
 	op(EFA_REG_MR_ERR, "reg_mr_err") \
 	op(EFA_ALLOC_UCONTEXT_ERR, "alloc_ucontext_err") \
-	op(EFA_CREATE_AH_ERR, "create_ah_err")
+	op(EFA_CREATE_AH_ERR, "create_ah_err") \
+	op(EFA_MMAP_ERR, "mmap_err")
 
 #define EFA_STATS_ENUM(ename, name) ename,
 #define EFA_STATS_STR(ename, name) [ename] = name,
@@ -1569,6 +1570,7 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,
 		ibdev_dbg(&dev->ibdev,
 			  "pgoff[%#lx] does not have valid entry\n",
 			  vma->vm_pgoff);
+		atomic64_inc(&dev->stats.sw_stats.mmap_err);
 		return -EINVAL;
 	}
 	entry = to_emmap(rdma_entry);
@@ -1604,12 +1606,14 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,
 		err = -EINVAL;
 	}
 
Quoted 11 lines

drivers/infiniband/hw/efa/efa_verbs.c static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, · line 1615

Quoted 8 lines
drivers/infiniband/hw/efa/efa_verbs.c +2 -1
-	if (err)
+	if (err) {
 		ibdev_dbg(
 			&dev->ibdev,
 			"Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
 			entry->address, rdma_entry->npages * PAGE_SIZE,
 			entry->mmap_flag, err);
+		atomic64_inc(&dev->stats.sw_stats.mmap_err);

Really? Isn't this something that is only possible with a buggy rdma-core provider? Why count it?

Though unlikely, it could happen, otherwise this error flow wouldn't exist in the first place.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

If for some reason a customer app steps on a bug we're not aware of, this counter could serve as a red flag.

But there are lots of cases where a buggy provider can cause error exits, why choose this one to count against all the others?

It's not one against all others, most if not all of our userspace facing API error flows have a similar counter.

Hurm, seems a bit strange, but OK

Quoted 2 lines

And TBH, I think that the mmap flow is quite convoluted with the cookie response from the crate verb, so it deserves a counter IMO.

How so? Userspace takes the u64 from the command and pass it to mmap, what is convoluted?

Jason

#8 lore raw

Re: [PATCH for-next 2/3] RDMA/efa: Count mmap failures

Quoted 83 lines · Jason Gunthorpe
Quoted 73 lines · Gal Pressman
Quoted 72 lines · Jason Gunthorpe
Quoted 71 lines · Gal Pressman
Quoted 55 lines · Jason Gunthorpe
Quoted 54 lines · Gal Pressman

Add a new stat that counts mmap failures, which might help when debugging different issues.

 drivers/infiniband/hw/efa/efa.h       | 3 ++-
 drivers/infiniband/hw/efa/efa_verbs.c | 9 +++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)
drivers/infiniband/hw/efa/efa.h +3 -1
index aa7396a1588a..77c9ff798117 100644
+++ b/drivers/infiniband/hw/efa/efa.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_H_
@@ -40,6 +40,7 @@ struct efa_sw_stats {
 	atomic64_t reg_mr_err;
 	atomic64_t alloc_ucontext_err;
 	atomic64_t create_ah_err;
+	atomic64_t mmap_err;
 };
 
 /* Don't use anything other than atomic64 */
drivers/infiniband/hw/efa/efa_verbs.c +4 -1
index b555845d6c14..75eef1ec2474 100644
+++ b/drivers/infiniband/hw/efa/efa_verbs.c
@@ -44,7 +44,8 @@ struct efa_user_mmap_entry {
 	op(EFA_CREATE_CQ_ERR, "create_cq_err") \
 	op(EFA_REG_MR_ERR, "reg_mr_err") \
 	op(EFA_ALLOC_UCONTEXT_ERR, "alloc_ucontext_err") \
-	op(EFA_CREATE_AH_ERR, "create_ah_err")
+	op(EFA_CREATE_AH_ERR, "create_ah_err") \
+	op(EFA_MMAP_ERR, "mmap_err")
 
 #define EFA_STATS_ENUM(ename, name) ename,
 #define EFA_STATS_STR(ename, name) [ename] = name,
@@ -1569,6 +1570,7 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,
 		ibdev_dbg(&dev->ibdev,
 			  "pgoff[%#lx] does not have valid entry\n",
 			  vma->vm_pgoff);
+		atomic64_inc(&dev->stats.sw_stats.mmap_err);
 		return -EINVAL;
 	}
 	entry = to_emmap(rdma_entry);
@@ -1604,12 +1606,14 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,
 		err = -EINVAL;
 	}
 
Quoted 11 lines

drivers/infiniband/hw/efa/efa_verbs.c static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, · line 1615

Quoted 8 lines
drivers/infiniband/hw/efa/efa_verbs.c +2 -1
-	if (err)
+	if (err) {
 		ibdev_dbg(
 			&dev->ibdev,
 			"Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
 			entry->address, rdma_entry->npages * PAGE_SIZE,
 			entry->mmap_flag, err);
+		atomic64_inc(&dev->stats.sw_stats.mmap_err);

Really? Isn't this something that is only possible with a buggy rdma-core provider? Why count it?

Though unlikely, it could happen, otherwise this error flow wouldn't exist in the first place.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

If for some reason a customer app steps on a bug we're not aware of, this counter could serve as a red flag.

But there are lots of cases where a buggy provider can cause error exits, why choose this one to count against all the others?

It's not one against all others, most if not all of our userspace facing API error flows have a similar counter.

Quoted 7 lines

Hurm, seems a bit strange, but OK

Quoted 2 lines

And TBH, I think that the mmap flow is quite convoluted with the cookie response from the crate verb, so it deserves a counter IMO.

How so? Userspace takes the u64 from the command and pass it to mmap, what is convoluted?

It's the only flow that involves two phases/userspace calls and not a single command-response call, so it's a bit more error prone. Convoluted was a bit harsh :).

#10 lore raw
In reply to Gal Pressman
Quoted 2 lines · Gal Pressman

This series contains three small patches to collect a few counters we found helpful when encountering different issues.

Applied to for-next, thanks

Jason