Skip to contentSkip to conversation

IB/core: Only update PKEY and GID caches on respective events

PATCH Superseded Håkon Bugge 3 participants 6 messages May 4, 2021 – May 6, 2021 InfiniBand

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 IB/core: Only update PKEY and GID caches on respective events

Both the PKEY and GID tables in an HCA can hold in the order of hundreds entries. Reading them are expensive. Partly because the API for retrieving them only returns a single entry at a time. Further, on certain implementations, e.g., CX-3, the VFs are paravirtualized in this respect and have to rely on the PF driver to perform the read. This again demands VF to PF communication.

IB Core's cache is refreshed on all events. Hence, filter the refresh of the PKEY and GID caches based on the event received being IB_EVENT_PKEY_CHANGE and IB_EVENT_GID_CHANGE respectively.

---
 drivers/infiniband/core/cache.c | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)
drivers/infiniband/core/cache.c +16 -9
index 5c9fac7..531ae6b 100644
@@ -1472,10 +1472,14 @@ static int config_non_roce_gid_cache(struct ib_device *device,
 }
 
 static int
-ib_cache_update(struct ib_device *device, u8 port, bool enforce_security)
+ib_cache_update(struct ib_device *device, u8 port, enum ib_event_type event,
+		bool reg_dev, bool enforce_security)
 {
 	struct ib_port_attr       *tprops = NULL;
-	struct ib_pkey_cache      *pkey_cache = NULL, *old_pkey_cache;
+	struct ib_pkey_cache      *pkey_cache = NULL;
+	struct ib_pkey_cache      *old_pkey_cache = NULL;
+	bool			   update_pkey_cache = (reg_dev || event == IB_EVENT_PKEY_CHANGE);
+	bool			   update_gid_cache = (reg_dev || event == IB_EVENT_GID_CHANGE);
 	int                        i;
 	int                        ret;
 
@@ -1492,14 +1496,16 @@ static int config_non_roce_gid_cache(struct ib_device *device,
 		goto err;
 	}
 
-	if (!rdma_protocol_roce(device, port)) {
+	if (!rdma_protocol_roce(device, port) && update_gid_cache) {
 		ret = config_non_roce_gid_cache(device, port,
 						tprops->gid_tbl_len);
 		if (ret)
 			goto err;
 	}
 
-	if (tprops->pkey_tbl_len) {
+	update_pkey_cache &= !!tprops->pkey_tbl_len;
+
+	if (update_pkey_cache) {
 		pkey_cache = kmalloc(struct_size(pkey_cache, table,
 						 tprops->pkey_tbl_len),
 				     GFP_KERNEL);
@@ -1524,9 +1530,10 @@ static int config_non_roce_gid_cache(struct ib_device *device,
 
 	write_lock_irq(&device->cache_lock);
 
-	old_pkey_cache = device->port_data[port].cache.pkey;
-
-	device->port_data[port].cache.pkey = pkey_cache;
+	if (update_pkey_cache) {
+		old_pkey_cache = device->port_data[port].cache.pkey;
+		device->port_data[port].cache.pkey = pkey_cache;
+	}
 	device->port_data[port].cache.lmc = tprops->lmc;
 	device->port_data[port].cache.port_state = tprops->state;
 
@@ -1558,7 +1565,7 @@ static void ib_cache_event_task(struct work_struct *_work)
 	 * the cache.
 	 */
 	ret = ib_cache_update(work->event.device, work->event.element.port_num,
-			      work->enforce_security);
+			      work->event.event, false, work->enforce_security);
 
 	/* GID event is notified already for individual GID entries by
 	 * dispatch_gid_change_event(). Hence, notifiy for rest of the
@@ -1631,7 +1638,7 @@ int ib_cache_setup_one(struct ib_device *device)
 		return err;
 
 	rdma_for_each_port (device, p) {
-		err = ib_cache_update(device, p, true);
+		err = ib_cache_update(device, p, 0, true, true);
 		if (err)
 			return err;
 	}
Signature · 1 line
1.8.3.1
#1 lore raw
Quoted 8 lines · Håkon Bugge

Both the PKEY and GID tables in an HCA can hold in the order of hundreds entries. Reading them are expensive. Partly because the API for retrieving them only returns a single entry at a time. Further, on certain implementations, e.g., CX-3, the VFs are paravirtualized in this respect and have to rely on the PF driver to perform the read. This again demands VF to PF communication.

Quoted 4 lines

IB Core's cache is refreshed on all events. Hence, filter the refresh of the PKEY and GID caches based on the event received being IB_EVENT_PKEY_CHANGE and IB_EVENT_GID_CHANGE respectively.

Missing Fixes?

Zhu Yanjun

Quoted 80 lines
---
 drivers/infiniband/core/cache.c | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)
drivers/infiniband/core/cache.c +16 -10
index 5c9fac7..531ae6b 100644
@@ -1472,10 +1472,14 @@ static int config_non_roce_gid_cache(struct ib_device *device,
 }
 
 static int
-ib_cache_update(struct ib_device *device, u8 port, bool enforce_security)
+ib_cache_update(struct ib_device *device, u8 port, enum ib_event_type event,
+               bool reg_dev, bool enforce_security)
 {
        struct ib_port_attr       *tprops = NULL;
-       struct ib_pkey_cache      *pkey_cache = NULL, *old_pkey_cache;
+       struct ib_pkey_cache      *pkey_cache = NULL;
+       struct ib_pkey_cache      *old_pkey_cache = NULL;
+       bool                       update_pkey_cache = (reg_dev || event == IB_EVENT_PKEY_CHANGE);
+       bool                       update_gid_cache = (reg_dev || event == IB_EVENT_GID_CHANGE);
        int                        i;
        int                        ret;
 
@@ -1492,14 +1496,16 @@ static int config_non_roce_gid_cache(struct ib_device *device,
                goto err;
        }
 
-       if (!rdma_protocol_roce(device, port)) {
+       if (!rdma_protocol_roce(device, port) && update_gid_cache) {
                ret = config_non_roce_gid_cache(device, port,
                                                tprops->gid_tbl_len);
                if (ret)
                        goto err;
        }
 
-       if (tprops->pkey_tbl_len) {
+       update_pkey_cache &= !!tprops->pkey_tbl_len;
+
+       if (update_pkey_cache) {
                pkey_cache = kmalloc(struct_size(pkey_cache, table,
                                                 tprops->pkey_tbl_len),
                                     GFP_KERNEL);
@@ -1524,9 +1530,10 @@ static int config_non_roce_gid_cache(struct ib_device *device,
 
        write_lock_irq(&device->cache_lock);
 
-       old_pkey_cache = device->port_data[port].cache.pkey;
-
-       device->port_data[port].cache.pkey = pkey_cache;
+       if (update_pkey_cache) {
+               old_pkey_cache = device->port_data[port].cache.pkey;
+               device->port_data[port].cache.pkey = pkey_cache;
+       }
        device->port_data[port].cache.lmc = tprops->lmc;
        device->port_data[port].cache.port_state = tprops->state;
 
@@ -1558,7 +1565,7 @@ static void ib_cache_event_task(struct work_struct *_work)
         * the cache.
         */
        ret = ib_cache_update(work->event.device, work->event.element.port_num,
-                             work->enforce_security);
+                             work->event.event, false, work->enforce_security);
 
        /* GID event is notified already for individual GID entries by
         * dispatch_gid_change_event(). Hence, notifiy for rest of the
@@ -1631,7 +1638,7 @@ int ib_cache_setup_one(struct ib_device *device)
                return err;
 
        rdma_for_each_port (device, p) {
-               err = ib_cache_update(device, p, true);
+               err = ib_cache_update(device, p, 0, true, true);
                if (err)
                        return err;
        }
--

1.8.3.1

#2 lore raw
Quoted 13 lines · Zhu Yanjun
Quoted 8 lines · Håkon Bugge

Both the PKEY and GID tables in an HCA can hold in the order of hundreds entries. Reading them are expensive. Partly because the API for retrieving them only returns a single entry at a time. Further, on certain implementations, e.g., CX-3, the VFs are paravirtualized in this respect and have to rely on the PF driver to perform the read. This again demands VF to PF communication.

Quoted 6 lines
Quoted 4 lines

IB Core's cache is refreshed on all events. Hence, filter the refresh of the PKEY and GID caches based on the event received being IB_EVENT_PKEY_CHANGE and IB_EVENT_GID_CHANGE respectively.

Missing Fixes?

I was thinking that this is a performance fix, not a bug fix. Probably not severe enough to go into stable. But by any means, it

Fixes 1da177e4c3f4 ("Linux-2.6.12-rc2")

Thxs, Håkon

Quoted 81 lines

Zhu Yanjun

Quoted 79 lines

---
drivers/infiniband/core/cache.c | 25 ++++++++++++++++---------

1 file changed, 16 insertions(+), 9 deletions(-)
drivers/infiniband/core/cache.c +0 -0
index 5c9fac7..531ae6b 100644
@@ -1472,10 +1472,14 @@ static int config_non_roce_gid_cache(struct ib_device *device,
 }
 

static int

drivers/infiniband/core/cache.c +2 -1
-ib_cache_update(struct ib_device *device, u8 port, bool enforce_security)
+ib_cache_update(struct ib_device *device, u8 port, enum ib_event_type event,
+               bool reg_dev, bool enforce_security)
{
       struct ib_port_attr       *tprops = NULL;
drivers/infiniband/core/cache.c +14 -9
-       struct ib_pkey_cache      *pkey_cache = NULL, *old_pkey_cache;
+       struct ib_pkey_cache      *pkey_cache = NULL;
+       struct ib_pkey_cache      *old_pkey_cache = NULL;
+       bool                       update_pkey_cache = (reg_dev || event == IB_EVENT_PKEY_CHANGE);
+       bool                       update_gid_cache = (reg_dev || event == IB_EVENT_GID_CHANGE);
       int                        i;
       int                        ret;
@@ -1492,14 +1496,16 @@ static int config_non_roce_gid_cache(struct ib_device *device,
               goto err;
       }
 
-       if (!rdma_protocol_roce(device, port)) {
+       if (!rdma_protocol_roce(device, port) && update_gid_cache) {
               ret = config_non_roce_gid_cache(device, port,
                                               tprops->gid_tbl_len);
               if (ret)
                       goto err;
       }
 
-       if (tprops->pkey_tbl_len) {
+       update_pkey_cache &= !!tprops->pkey_tbl_len;
+
+       if (update_pkey_cache) {
               pkey_cache = kmalloc(struct_size(pkey_cache, table,
                                                tprops->pkey_tbl_len),
                                    GFP_KERNEL);
@@ -1524,9 +1530,10 @@ static int config_non_roce_gid_cache(struct ib_device *device,
 
       write_lock_irq(&device->cache_lock);
 
-       old_pkey_cache = device->port_data[port].cache.pkey;
-
-       device->port_data[port].cache.pkey = pkey_cache;
+       if (update_pkey_cache) {
+               old_pkey_cache = device->port_data[port].cache.pkey;
+               device->port_data[port].cache.pkey = pkey_cache;
+       }
       device->port_data[port].cache.lmc = tprops->lmc;
       device->port_data[port].cache.port_state = tprops->state;
 
@@ -1558,7 +1565,7 @@ static void ib_cache_event_task(struct work_struct *_work)
        * the cache.
        */
       ret = ib_cache_update(work->event.device, work->event.element.port_num,
-                             work->enforce_security);
+                             work->event.event, false, work->enforce_security);
 
       /* GID event is notified already for individual GID entries by
        * dispatch_gid_change_event(). Hence, notifiy for rest of the
@@ -1631,7 +1638,7 @@ int ib_cache_setup_one(struct ib_device *device)
               return err;
 
       rdma_for_each_port (device, p) {
-               err = ib_cache_update(device, p, true);
+               err = ib_cache_update(device, p, 0, true, true);
               if (err)
                       return err;
       }
--

1.8.3.1

#3 lore raw
In reply to Håkon Bugge
Quoted 24 lines · Håkon Bugge

Both the PKEY and GID tables in an HCA can hold in the order of hundreds entries. Reading them are expensive. Partly because the API for retrieving them only returns a single entry at a time. Further, on certain implementations, e.g., CX-3, the VFs are paravirtualized in this respect and have to rely on the PF driver to perform the read. This again demands VF to PF communication.

IB Core's cache is refreshed on all events. Hence, filter the refresh of the PKEY and GID caches based on the event received being IB_EVENT_PKEY_CHANGE and IB_EVENT_GID_CHANGE respectively.

---
 drivers/infiniband/core/cache.c | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)
drivers/infiniband/core/cache.c +0 -0
index 5c9fac7..531ae6b 100644
@@ -1472,10 +1472,14 @@ static int config_non_roce_gid_cache(struct ib_device *device,
 }
 
 static int

drivers/infiniband/core/cache.c static int config_non_roce_gid_cache(struct ib_device *device, · line 1482

Quoted 10 lines
drivers/infiniband/core/cache.c +6 -2
-ib_cache_update(struct ib_device *device, u8 port, bool enforce_security)
+ib_cache_update(struct ib_device *device, u8 port, enum ib_event_type event,
+		bool reg_dev, bool enforce_security)
 {
 	struct ib_port_attr       *tprops = NULL;
-	struct ib_pkey_cache      *pkey_cache = NULL, *old_pkey_cache;
+	struct ib_pkey_cache      *pkey_cache = NULL;
+	struct ib_pkey_cache      *old_pkey_cache = NULL;
+	bool			   update_pkey_cache = (reg_dev || event == IB_EVENT_PKEY_CHANGE);
+	bool			   update_gid_cache = (reg_dev || event == IB_EVENT_GID_CHANGE);

I'm not super excited to see "events" in this function and would be more happy to see it is handled in ib_cache_event_task() while the function signature will be:
ib_cache_update(struct ib_device *device, u8 port, bool all_gids, bool all_pkeys, bool enforce_security)

Thanks

#4 lore raw
Quoted 37 lines · Leon Romanovsky
Quoted 32 lines · Håkon Bugge

Both the PKEY and GID tables in an HCA can hold in the order of hundreds entries. Reading them are expensive. Partly because the API for retrieving them only returns a single entry at a time. Further, on certain implementations, e.g., CX-3, the VFs are paravirtualized in this respect and have to rely on the PF driver to perform the read. This again demands VF to PF communication.

IB Core's cache is refreshed on all events. Hence, filter the refresh of the PKEY and GID caches based on the event received being IB_EVENT_PKEY_CHANGE and IB_EVENT_GID_CHANGE respectively.

---
drivers/infiniband/core/cache.c | 25 ++++++++++++++++---------

1 file changed, 16 insertions(+), 9 deletions(-)
drivers/infiniband/core/cache.c +0 -0
index 5c9fac7..531ae6b 100644
@@ -1472,10 +1472,14 @@ static int config_non_roce_gid_cache(struct ib_device *device,
 }
 

static int

drivers/infiniband/core/cache.c +2 -1
-ib_cache_update(struct ib_device *device, u8 port, bool enforce_security)
+ib_cache_update(struct ib_device *device, u8 port, enum ib_event_type event,
+		bool reg_dev, bool enforce_security)
{
	struct ib_port_attr       *tprops = NULL;
drivers/infiniband/core/cache.c +2 -1
-	struct ib_pkey_cache      *pkey_cache = NULL, *old_pkey_cache;
+	struct ib_pkey_cache      *pkey_cache = NULL;
+	struct ib_pkey_cache      *old_pkey_cache = NULL;
Quoted 8 lines

drivers/infiniband/core/cache.c static int config_non_roce_gid_cache(struct ib_device *device, · line 1482

Quoted 2 lines
drivers/infiniband/core/cache.c +2 -0
+	bool			   update_pkey_cache = (reg_dev || event == IB_EVENT_PKEY_CHANGE);
+	bool			   update_gid_cache = (reg_dev || event == IB_EVENT_GID_CHANGE);

I'm not super excited to see "events" in this function and would be more happy to see it is handled in ib_cache_event_task() while the function signature will be:
ib_cache_update(struct ib_device *device, u8 port, bool all_gids, bool all_pkeys, bool enforce_security)

I was thinking the other way around; if a new entity, FOO, gets cached, to be updated on IB_EVENT_FOO_CHANGE, the change to support this would be more contained; you only need to change ib_cache_update().

But by all means, will send a v2 based on you recommendation.

Thxs, Håkon

Quoted 2 lines

Thanks

#5 lore raw
Quoted 46 lines · Haakon Bugge
Quoted 37 lines · Leon Romanovsky
Quoted 32 lines · Håkon Bugge

Both the PKEY and GID tables in an HCA can hold in the order of hundreds entries. Reading them are expensive. Partly because the API for retrieving them only returns a single entry at a time. Further, on certain implementations, e.g., CX-3, the VFs are paravirtualized in this respect and have to rely on the PF driver to perform the read. This again demands VF to PF communication.

IB Core's cache is refreshed on all events. Hence, filter the refresh of the PKEY and GID caches based on the event received being IB_EVENT_PKEY_CHANGE and IB_EVENT_GID_CHANGE respectively.

---
drivers/infiniband/core/cache.c | 25 ++++++++++++++++---------

1 file changed, 16 insertions(+), 9 deletions(-)
drivers/infiniband/core/cache.c +0 -0
index 5c9fac7..531ae6b 100644
@@ -1472,10 +1472,14 @@ static int config_non_roce_gid_cache(struct ib_device *device,
 }
 

static int

drivers/infiniband/core/cache.c +2 -1
-ib_cache_update(struct ib_device *device, u8 port, bool enforce_security)
+ib_cache_update(struct ib_device *device, u8 port, enum ib_event_type event,
+		bool reg_dev, bool enforce_security)
{
	struct ib_port_attr       *tprops = NULL;
drivers/infiniband/core/cache.c +2 -1
-	struct ib_pkey_cache      *pkey_cache = NULL, *old_pkey_cache;
+	struct ib_pkey_cache      *pkey_cache = NULL;
+	struct ib_pkey_cache      *old_pkey_cache = NULL;
Quoted 8 lines

drivers/infiniband/core/cache.c static int config_non_roce_gid_cache(struct ib_device *device, · line 1482

Quoted 2 lines
drivers/infiniband/core/cache.c +2 -0
+	bool			   update_pkey_cache = (reg_dev || event == IB_EVENT_PKEY_CHANGE);
+	bool			   update_gid_cache = (reg_dev || event == IB_EVENT_GID_CHANGE);

I'm not super excited to see "events" in this function and would be more happy to see it is handled in ib_cache_event_task() while the function signature will be:
ib_cache_update(struct ib_device *device, u8 port, bool all_gids, bool all_pkeys, bool enforce_security)

Quoted 1 line

I was thinking the other way around; if a new entity, FOO, gets cached, to be updated on IB_EVENT_FOO_CHANGE, the change to support this would be more contained; you only need to change ib_cache_update().

We are already handling enforce_security outside of ib_cache_update().

Quoted 2 lines

But by all means, will send a v2 based on you recommendation.

Thanks

Quoted 9 lines

Thxs, Håkon

Quoted 2 lines

Thanks