IB/qib: Change lkey table allocation to support more MRs
Subsystems (2)
PATCH 1/1 IB/qib: Change lkey table allocation to support more MRs
The lkey table is allocated with with a get_user_pages() with an order based on a number of index bits from a module parameter.
The underlying kernel code cannot allocate that many contiguous pages.
There is no reason the underlying memory needs to be physically contiguous.
This patch:
- switches the allocation/deallocation to vmalloc/vfree
- caps the number of bits to 23 to insure at least 1 generation bit
o this matches the module parameter description
Cc [email protected] # 3.4
Reviewed-by Vinit Agnihotri <[email protected]>
Signed-off-by Mike Marciniszyn <[email protected]>
Signed-off-by Doug Ledford <[email protected]>
--- drivers/infiniband/hw/qib/qib_keys.c | 4 ++++ drivers/infiniband/hw/qib/qib_verbs.c | 16 +++++++++++----- drivers/infiniband/hw/qib/qib_verbs.h | 2 ++ 3 files changed, 17 insertions(+), 5 deletions(-)
drivers/infiniband/hw/qib/qib_keys.c +4 -0
@@ -69,6 +69,10 @@ int qib_alloc_lkey(struct qib_lkey_table *rkt, struct qib_mregion *mr)* unrestricted LKEY. */ rkt->gen++; +/* +* bits are capped in qib_verbs.c to insure enough bits +* for generation number +*/ mr->lkey = (r << (32 - ib_qib_lkey_table_size)) | ((((1 << (24 - ib_qib_lkey_table_size)) -1 ) & rkt->gen) <<8 );
drivers/infiniband/hw/qib/qib_verbs.c +11 -5
@@ -40,6 +40,7 @@#include <linux/rculist.h> #include <linux/mm.h> #include <linux/random.h> +#include <linux/vmalloc.h> #include "qib.h" #include "qib_common.h"
@@ -2058,10 +2059,16 @@ int qib_register_ib_device(struct qib_devdata *dd)* the LKEY). The remaining bits act as a generation number or tag. */ spin_lock_init (&dev->lk_table.lock); +/* insure generation is at least 4 bits see keys.c */ +if (ib_qib_lkey_table_size >MAX_LKEY_TABLE_BITS ) { +qib_dev_warn (dd,"lkey bits %u too large, reduced to %u\n" , + ib_qib_lkey_table_size,MAX_LKEY_TABLE_BITS ); + ib_qib_lkey_table_size =MAX_LKEY_TABLE_BITS ; + } dev->lk_table.max =1 << ib_qib_lkey_table_size; lk_tab_size = dev->lk_table.max *sizeof (*dev->lk_table.table); - dev->lk_table.table = (struct qib_mregion **) -__get_free_pages (GFP_KERNEL ,get_order (lk_tab_size)); + dev->lk_table.table = (struct qib_mregion __rcu **) +vmalloc (lk_tab_size);if (dev->lk_table.table ==NULL ) { ret = -ENOMEM ; goto err_lk;
@@ -2231,7 +2238,7 @@ err_tx:sizeof (struct qib_pio_header ), dev->pio_hdrs, dev->pio_hdrs_phys); err_hdrs: -free_pages ((unsigned long ) dev->lk_table.table,get_order (lk_tab_size)); +vfree (dev->lk_table.table); err_lk:kfree (dev->qp_table); err_qpt:
@@ -2285,7 +2292,6 @@ void qib_unregister_ib_device(struct qib_devdata *dd)sizeof (struct qib_pio_header ), dev->pio_hdrs, dev->pio_hdrs_phys); lk_tab_size = dev->lk_table.max *sizeof (*dev->lk_table.table); -free_pages ((unsigned long ) dev->lk_table.table, -get_order (lk_tab_size)); +vfree (dev->lk_table.table);kfree (dev->qp_table); }
drivers/infiniband/hw/qib/qib_verbs.h +2 -0
@@ -622,6 +622,8 @@ struct qib_qpn_table {struct qpn_map map[QPNMAP_ENTRIES ]; }; +#define MAX_LKEY_TABLE_BITS 23 +struct qib_lkey_table {spinlock_t lock;/* protect changes in this struct */ u32 next;/* next unused index (speeds search) */