RDMA/rtrs-srv: Incorporate ib_register_client into rtrs server init
Subsystems (2)
PATCH v4 1/1 RDMA/rtrs-srv: Incorporate ib_register_client into rtrs server init
The rnbd_server module's communication manager (cm) initialization depends on the registration of the "network namespace subsystem" of the RDMA CM agent module. As such, when the kernel is configured to load the rnbd_server and the RDMA cma module during initialization; and if the rnbd_server module is initialized before RDMA cma module, a null ptr dereference occurs during the RDMA bind operation.
Call trace below,
[ 1.904782] Call Trace: [ 1.904782] ? xas_load+0xd/0x80 [ 1.904782] xa_load+0x47/0x80 [ 1.904782] cma_ps_find+0x44/0x70 [ 1.904782] rdma_bind_addr+0x782/0x8b0 [ 1.904782] ? get_random_bytes+0x35/0x40 [ 1.904782] rtrs_srv_cm_init+0x50/0x80 [ 1.904782] rtrs_srv_open+0x102/0x180 [ 1.904782] ? rnbd_client_init+0x6e/0x6e [ 1.904782] rnbd_srv_init_module+0x34/0x84 [ 1.904782] ? rnbd_client_init+0x6e/0x6e [ 1.904782] do_one_initcall+0x4a/0x200 [ 1.904782] kernel_init_freeable+0x1f1/0x26e [ 1.904782] ? rest_init+0xb0/0xb0 [ 1.904782] kernel_init+0xe/0x100 [ 1.904782] ret_from_fork+0x22/0x30 [ 1.904782] Modules linked in: [ 1.904782] CR2: 0000000000000015 [ 1.904782] ---[ end trace c42df88d6c7b0a48 ]---
All this happens cause the cm init is in the call chain of the module init, which is not a preferred practice.
So remove the call to rdma_create_id() from the module init call chain. Instead register rtrs-srv as an ib client, which makes sure that the rdma_create_id() is called only when an ib device is added.
---
Change in v4:
Add mutex lock to prevent the add/remove of ib device from racing
Change in v3:
Remove RDMA init error check while rtrs server open Remove -1 assignment for ib_dev_count on RDMA init error
Change in v2:
Use only single variable to track number of IB devices and failure Change according to kernel coding style
drivers/infiniband/ulp/rtrs/rtrs-srv.c | 77 +++++++++++++++++++++++++- drivers/infiniband/ulp/rtrs/rtrs-srv.h | 7 +++ 2 files changed, 81 insertions(+), 3 deletions(-)
drivers/infiniband/ulp/rtrs/rtrs-srv.c +74 -3
@@ -16,6 +16,7 @@#include "rtrs-srv.h" #include "rtrs-log.h" #include <rdma/ib_cm.h> +#include <rdma/ib_verbs.h> MODULE_DESCRIPTION ("RDMA Transport Server" );MODULE_LICENSE ("GPL" );
@@ -31,6 +32,7 @@ MODULE_LICENSE("GPL");static struct rtrs_rdma_dev_pd dev_pd;static mempool_t *chunk_pool;struct class *rtrs_dev_class; +static struct rtrs_srv_ib_ctx ib_ctx;static int __read_mostly max_chunk_size =DEFAULT_MAX_CHUNK_SIZE ;static int __read_mostly sess_queue_depth =DEFAULT_SESS_QUEUE_DEPTH ;
@@ -2033,6 +2035,71 @@ static void free_srv_ctx(struct rtrs_srv_ctx *ctx)kfree (ctx); } +static int rtrs_srv_add_one (struct ib_device *device) +{ +struct rtrs_srv_ctx *ctx; +int ret; + +mutex_lock (&ib_ctx.ib_dev_mutex); + ret =0 ; +if (ib_ctx.ib_dev_count) + goto out; + +/* +* Since our CM IDs are NOT bound to any ib device we will create them +* only once +*/ + ctx = ib_ctx.srv_ctx; + ret =rtrs_srv_rdma_init (ctx, ib_ctx.port); +if (ret) { +/* +* We errored out here. +* According to the ib code, if we encounter an error here then the +* error code is ignored, and no more calls to our ops are made. +*/ +pr_err ("Failed to initialize RDMA connection" ); + goto err_out; + } + +out: +/* +* Keep a track on the number of ib devices added +*/ + ib_ctx.ib_dev_count++; + +err_out: +mutex_unlock (&ib_ctx.ib_dev_mutex); +return ret; +} + +static void rtrs_srv_remove_one (struct ib_device *device,void *client_data) +{ +struct rtrs_srv_ctx *ctx; + +mutex_lock (&ib_ctx.ib_dev_mutex); + ib_ctx.ib_dev_count--; + +if (ib_ctx.ib_dev_count) + goto out; + +/* +* Since our CM IDs are NOT bound to any ib device we will remove them +* only once, when the last device is removed +*/ + ctx = ib_ctx.srv_ctx; +rdma_destroy_id (ctx->cm_id_ip); +rdma_destroy_id (ctx->cm_id_ib); + +out: +mutex_unlock (&ib_ctx.ib_dev_mutex); +} + +static struct ib_client rtrs_srv_client = { + .name ="rtrs_server" , + .add = rtrs_srv_add_one, + .remove = rtrs_srv_remove_one +}; +/** * rtrs_srv_open() - open RTRS server context * @ops: callback functions
@@ -2051,7 +2118,11 @@ struct rtrs_srv_ctx *rtrs_srv_open(struct rtrs_srv_ops *ops, u16 port)if (!ctx)return ERR_PTR (-ENOMEM ); - err =rtrs_srv_rdma_init (ctx, port); +mutex_init (&ib_ctx.ib_dev_mutex); + ib_ctx.srv_ctx = ctx; + ib_ctx.port = port; + + err =ib_register_client (&rtrs_srv_client);if (err) {free_srv_ctx (ctx);return ERR_PTR (err);
@@ -2090,8 +2161,8 @@ static void close_ctx(struct rtrs_srv_ctx *ctx)*/ void rtrs_srv_close (struct rtrs_srv_ctx *ctx) { -rdma_destroy_id (ctx->cm_id_ip); -rdma_destroy_id (ctx->cm_id_ib); +ib_unregister_client (&rtrs_srv_client); +mutex_destroy (&ib_ctx.ib_dev_mutex);close_ctx (ctx);free_srv_ctx (ctx); }
drivers/infiniband/ulp/rtrs/rtrs-srv.h +7 -0
@@ -118,6 +118,13 @@ struct rtrs_srv_ctx {struct list_head srv_list; }; +struct rtrs_srv_ib_ctx { +struct rtrs_srv_ctx *srv_ctx; +u16 port; +struct mutex ib_dev_mutex; +int ib_dev_count; +}; +extern struct class *rtrs_dev_class;void close_sess (struct rtrs_srv_sess *sess);
Signature · 1 line
2.25.1
Quoted 75 lines · Md Haris Iqbal
The
rnbd_servermodule's communication manager (cm) initialization depends on the registration of the "network namespace subsystem" of the RDMA CM agent module. As such, when the kernel is configured to load thernbd_serverand the RDMA cma module during initialization; and if thernbd_servermodule is initialized before RDMA cma module, a null ptr dereference occurs during the RDMA bind operation.Call trace below,
[ 1.904782] Call Trace: [ 1.904782] ? xas_load+0xd/0x80 [ 1.904782] xa_load+0x47/0x80 [ 1.904782] cma_ps_find+0x44/0x70 [ 1.904782] rdma_bind_addr+0x782/0x8b0 [ 1.904782] ? get_random_bytes+0x35/0x40 [ 1.904782] rtrs_srv_cm_init+0x50/0x80 [ 1.904782] rtrs_srv_open+0x102/0x180 [ 1.904782] ? rnbd_client_init+0x6e/0x6e [ 1.904782] rnbd_srv_init_module+0x34/0x84 [ 1.904782] ? rnbd_client_init+0x6e/0x6e [ 1.904782] do_one_initcall+0x4a/0x200 [ 1.904782] kernel_init_freeable+0x1f1/0x26e [ 1.904782] ? rest_init+0xb0/0xb0 [ 1.904782] kernel_init+0xe/0x100 [ 1.904782] ret_from_fork+0x22/0x30 [ 1.904782] Modules linked in: [ 1.904782] CR2: 0000000000000015 [ 1.904782] ---[ end trace c42df88d6c7b0a48 ]---All this happens cause the cm init is in the call chain of the module init, which is not a preferred practice.
So remove the call to
rdma_create_id()from the module init call chain. Instead register rtrs-srv as an ib client, which makes sure that therdma_create_id()is called only when an ib device is added.Fixes 9cb837480424 ("RDMA/rtrs: server: main functionality")Reported-by kernel test robot <[email protected]>Signed-off-by Md Haris Iqbal <[email protected]>Reviewed-by Jack Wang <[email protected]>---
Change in v4:
Add mutex lock to prevent the add/remove of ib device from racing
Change in v3:
Remove RDMA init error check while rtrs server open Remove -1 assignment forib_dev_counton RDMA init error
Change in v2:
Use only single variable to track number of IB devices and failure Change according to kernel coding styledrivers/infiniband/ulp/rtrs/rtrs-srv.c | 77 +++++++++++++++++++++++++- drivers/infiniband/ulp/rtrs/rtrs-srv.h | 7 +++ 2 files changed, 81 insertions(+), 3 deletions(-)drivers/infiniband/ulp/rtrs/rtrs-srv.c +2
-0@@ -16,6 +16,7 @@#include "rtrs-srv.h" #include "rtrs-log.h" #include <rdma/ib_cm.h> +#include <rdma/ib_verbs.h> MODULE_DESCRIPTION ("RDMA Transport Server" );MODULE_LICENSE ("GPL" );@@ -31,6 +32,7 @@ MODULE_LICENSE("GPL");static struct rtrs_rdma_dev_pd dev_pd;static mempool_t *chunk_pool;struct class *rtrs_dev_class; +static struct rtrs_srv_ib_ctx ib_ctx;static int __read_mostly max_chunk_size =DEFAULT_MAX_CHUNK_SIZE ;static int __read_mostly sess_queue_depth =DEFAULT_SESS_QUEUE_DEPTH ;
drivers/infiniband/ulp/rtrs/rtrs-srv.c static void free_srv_ctx(struct rtrs_srv_ctx *ctx) · line 2044
Quoted 10 lines
drivers/infiniband/ulp/rtrs/rtrs-srv.c +7
-0@@ -2033,6 +2035,71 @@ static void free_srv_ctx(struct rtrs_srv_ctx *ctx)kfree (ctx); } +static int rtrs_srv_add_one (struct ib_device *device) +{ +struct rtrs_srv_ctx *ctx; +int ret; + +mutex_lock (&ib_ctx.ib_dev_mutex); + ret =0 ;
int ret = 0;
Other than that,
Quoted 51 lines · Md Haris Iqbal
The
rnbd_servermodule's communication manager (cm) initialization depends on the registration of the "network namespace subsystem" of the RDMA CM agent module. As such, when the kernel is configured to load thernbd_serverand the RDMA cma module during initialization; and if thernbd_servermodule is initialized before RDMA cma module, a null ptr dereference occurs during the RDMA bind operation.Call trace below,
[ 1.904782] Call Trace: [ 1.904782] ? xas_load+0xd/0x80 [ 1.904782] xa_load+0x47/0x80 [ 1.904782] cma_ps_find+0x44/0x70 [ 1.904782] rdma_bind_addr+0x782/0x8b0 [ 1.904782] ? get_random_bytes+0x35/0x40 [ 1.904782] rtrs_srv_cm_init+0x50/0x80 [ 1.904782] rtrs_srv_open+0x102/0x180 [ 1.904782] ? rnbd_client_init+0x6e/0x6e [ 1.904782] rnbd_srv_init_module+0x34/0x84 [ 1.904782] ? rnbd_client_init+0x6e/0x6e [ 1.904782] do_one_initcall+0x4a/0x200 [ 1.904782] kernel_init_freeable+0x1f1/0x26e [ 1.904782] ? rest_init+0xb0/0xb0 [ 1.904782] kernel_init+0xe/0x100 [ 1.904782] ret_from_fork+0x22/0x30 [ 1.904782] Modules linked in: [ 1.904782] CR2: 0000000000000015 [ 1.904782] ---[ end trace c42df88d6c7b0a48 ]---All this happens cause the cm init is in the call chain of the module init, which is not a preferred practice.
So remove the call to
rdma_create_id()from the module init call chain. Instead register rtrs-srv as an ib client, which makes sure that therdma_create_id()is called only when an ib device is added.Fixes 9cb837480424 ("RDMA/rtrs: server: main functionality")Reported-by kernel test robot <[email protected]>Signed-off-by Md Haris Iqbal <[email protected]>Reviewed-by Jack Wang <[email protected]>Reviewed-by Leon Romanovsky <[email protected]>---
Change in v4:
Add mutex lock to prevent the add/remove of ib device from racing
Change in v3:
Remove RDMA init error check while rtrs server open Remove -1 assignment forib_dev_counton RDMA init error
Change in v2:
Use only single variable to track number of IB devices and failure Change according to kernel coding style
Quoted 3 lines
drivers/infiniband/ulp/rtrs/rtrs-srv.c | 77 +++++++++++++++++++++++++- drivers/infiniband/ulp/rtrs/rtrs-srv.h | 7 +++ 2 files changed, 81 insertions(+), 3 deletions(-)
Applied to for-next with Leon's remark
Thanks,
Jason