tcm_vhost hotplug
Subsystems (3)
v5 cover tcm_vhost hotplug
Asias He (3):
tcm_vhost: Introduce tcm_vhost_check_feature()
tcm_vhost: Add helper to check if endpoint is setup
tcm_vhost: Add hotplug/hotunplug support
drivers/vhost/tcm_vhost.c | 236 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 ++ 2 files changed, 242 insertions(+), 4 deletions(-)
Signature · 1 line
1.8.1.4
PATCH v5 1/3 tcm_vhost: Introduce tcm_vhost_check_feature()
This helper is useful to check if a feature is supported.
--- drivers/vhost/tcm_vhost.c | 12 ++++++++++++ 1 file changed, 12 insertions(+)
drivers/vhost/tcm_vhost.c +12 -0
@@ -99,6 +99,18 @@ static int iov_num_pages(struct iovec *iov) ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT; } +static bool tcm_vhost_check_feature (struct vhost_scsi *vs,int feature) +{ +bool ret = false; + +mutex_lock (&vs->dev.mutex); +if (vhost_has_feature (&vs->dev, feature)) + ret = true; +mutex_unlock (&vs->dev.mutex); + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;
Signature · 1 line
1.8.1.4
PATCH v5 2/3 tcm_vhost: Add helper to check if endpoint is setup
In reply to Asias He--- drivers/vhost/tcm_vhost.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)
drivers/vhost/tcm_vhost.c +18 -0
@@ -111,6 +111,24 @@ static bool tcm_vhost_check_feature(struct vhost_scsi *vs, int feature)return ret; } +static bool tcm_vhost_check_endpoint (struct vhost_virtqueue *vq) +{ +bool ret = false; + +/* +* We can handle the vq only after the endpoint is setup by calling the +* VHOST_SCSI_SET_ENDPOINT ioctl. +* +* TODO: Check that we are running from vhost_worker which acts +* as read-side critical section for vhost kind of RCU. +* See the comments in struct vhost_virtqueue in drivers/vhost/vhost.h +*/ +if (rcu_dereference_check (vq->private_data,1 )) + ret = true; + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;
Signature · 1 line
1.8.1.4
PATCH v5 3/3 tcm_vhost: Add hotplug/hotunplug support
In reply to Asias HeIn commit 365a7150094 ([SCSI] virtio-scsi: hotplug support for virtio-scsi), hotplug support is added to virtio-scsi.
This patch adds hotplug and hotunplug support to tcm_vhost.
You can create or delete a LUN in targetcli to hotplug or hotunplug a LUN in guest.
Changes in v5:
- Switch to int from u64 to vs_events_nr
- Set s->vs_events_dropped flag in tcm_vhost_allocate_evt
- Do not nest dev mutex within vq mutex
- Use vs_events_lock to protect vs_events_dropped and vs_events_nr
- Rebase to target/master
Changes in v4:
- Drop tcm_vhost_check_endpoint in tcm_vhost_send_evt
- Add tcm_vhost_check_endpoint in vhost_scsi_evt_handle_kick
Changes in v3:
- Separate the bug fix to another thread
Changes in v2:
- Remove code duplication in tcm_vhost_{hotplug,hotunplug}
- Fix racing of vs_events_nr
- Add flush fix patch to this series
--- drivers/vhost/tcm_vhost.c | 206 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 +++ 2 files changed, 212 insertions(+), 4 deletions(-)
drivers/vhost/tcm_vhost.c +202 -4
@@ -66,11 +66,13 @@ enum {* TODO: debug and remove the workaround. */ enum { -VHOST_SCSI_FEATURES =VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX ) +VHOST_SCSI_FEATURES = (VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX )) | + (1ULL <<VIRTIO_SCSI_F_HOTPLUG ) };#define VHOST_SCSI_MAX_TARGET 256#define VHOST_SCSI_MAX_VQ 128 +#define VHOST_SCSI_MAX_EVENT 128struct vhost_scsi {/* Protected by vhost_scsi->dev.mutex */
@@ -82,6 +84,13 @@ struct vhost_scsi {struct vhost_work vs_completion_work;/* cmd completion work item */ struct llist_head vs_completion_list;/* cmd completion queue */ + +struct vhost_work vs_event_work;/* evt injection work item */ +struct llist_head vs_event_list;/* evt injection queue */ + +struct mutex vs_events_lock;/* protect vs_events_dropped,events_nr */ +bool vs_events_dropped;/* any missed events */ +int vs_events_nr;/* num of pending events */ };/* Local pointer to allocated TCM configfs fabric module */
@@ -129,6 +138,17 @@ static bool tcm_vhost_check_endpoint(struct vhost_virtqueue *vq)return ret; } +static bool tcm_vhost_check_events_dropped (struct vhost_scsi *vs) +{ +bool ret; + +mutex_lock (&vs->vs_events_lock); + ret = vs->vs_events_dropped; +mutex_unlock (&vs->vs_events_lock); + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;
@@ -379,6 +399,37 @@ static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd)return 0 ; } +static void tcm_vhost_free_evt (struct vhost_scsi *vs,struct tcm_vhost_evt *evt) +{ +mutex_lock (&vs->vs_events_lock); + vs->vs_events_nr--; +kfree (evt); +mutex_unlock (&vs->vs_events_lock); +} + +static struct tcm_vhost_evt *tcm_vhost_allocate_evt (struct vhost_scsi *vs, +u32 event,u32 reason) +{ +struct tcm_vhost_evt *evt; + +mutex_lock (&vs->vs_events_lock); +if (vs->vs_events_nr >VHOST_SCSI_MAX_EVENT ) { + vs->vs_events_dropped = true; +mutex_unlock (&vs->vs_events_lock); +return NULL ; + } + + evt =kzalloc (sizeof (*evt),GFP_KERNEL ); +if (evt) { + evt->event.event = event; + evt->event.reason = reason; + vs->vs_events_nr++; + } +mutex_unlock (&vs->vs_events_lock); + +return evt; +} +static void vhost_scsi_free_cmd (struct tcm_vhost_cmd *tv_cmd) {struct se_cmd *se_cmd = &tv_cmd->tvc_se_cmd;
@@ -397,6 +448,74 @@ static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd)kfree (tv_cmd); } +static void tcm_vhost_do_evt_work (struct vhost_scsi *vs, +struct virtio_scsi_event *event) +{ +struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT ]; +struct virtio_scsi_event __user *eventp; +unsigned out, in; +int head, ret; + +if (!tcm_vhost_check_endpoint (vq)) +return ; + +mutex_lock (&vs->vs_events_lock); +mutex_lock (&vq->mutex); +again: +vhost_disable_notify (&vs->dev, vq); + head =vhost_get_vq_desc (&vs->dev, vq, vq->iov, +ARRAY_SIZE (vq->iov), &out, &in, +NULL ,NULL ); +if (head <0 ) { + vs->vs_events_dropped = true; + goto out; + } +if (head == vq->num) { +if (vhost_enable_notify (&vs->dev, vq)) + goto again; + vs->vs_events_dropped = true; + goto out; + } + +if ((vq->iov[out].iov_len !=sizeof (struct virtio_scsi_event ))) { +vq_err (vq,"Expecting virtio_scsi_event, got %zu bytes\n" , + vq->iov[out].iov_len); + vs->vs_events_dropped = true; + goto out; + } + +if (vs->vs_events_dropped) { + event->event |=VIRTIO_SCSI_T_EVENTS_MISSED ; + vs->vs_events_dropped = false; + } + + eventp = vq->iov[out].iov_base; + ret =__copy_to_user (eventp, event,sizeof (*event)); +if (!ret) +vhost_add_used_and_signal (&vs->dev, vq, head,0 ); +else +vq_err (vq,"Faulted on tcm_vhost_send_event\n" ); +out: +mutex_unlock (&vq->mutex); +mutex_unlock (&vs->vs_events_lock); +} + +static void tcm_vhost_evt_work (struct vhost_work *work) +{ + struct vhost_scsi *vs = container_of(work, struct vhost_scsi, + vs_event_work); +struct tcm_vhost_evt *evt; +struct llist_node *llnode; + + llnode =llist_del_all (&vs->vs_event_list); +while (llnode) { + evt = llist_entry(llnode, struct tcm_vhost_evt, list); + llnode =llist_next (llnode); +tcm_vhost_do_evt_work (vs, &evt->event); +tcm_vhost_free_evt (vs, evt); + } +} +/* Fill in status and signal that we are done processing this command * * This is scheduled in the vhost work queue so we are called with the owner
@@ -804,9 +923,42 @@ static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)pr_debug ("%s: The handling func for control queue.\n" , __func__); } +static int tcm_vhost_send_evt (struct vhost_scsi *vs,struct tcm_vhost_tpg *tpg, +struct se_lun *lun,u32 event,u32 reason) +{ +struct tcm_vhost_evt *evt; + + evt =tcm_vhost_allocate_evt (vs, event, reason); +if (!evt) +return -ENOMEM ; + +if (tpg && lun) { +/* TODO: share lun setup code with virtio-scsi.ko */ + evt->event.lun[0 ] =0x01 ; + evt->event.lun[1 ] = tpg->tport_tpgt &0xFF ; +if (lun->unpacked_lun >=256 ) + evt->event.lun[2 ] = lun->unpacked_lun >>8 |0x40 ; + evt->event.lun[3 ] = lun->unpacked_lun &0xFF ; + } + +llist_add (&evt->list, &vs->vs_event_list); +vhost_work_queue (&vs->dev, &vs->vs_event_work); + +return 0 ; +} +static void vhost_scsi_evt_handle_kick (struct vhost_work *work) { -pr_debug ("%s: The handling func for event queue.\n" , __func__); + struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, + poll.work); + struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev); + +if (!tcm_vhost_check_endpoint (vq)) +return ; + +if (tcm_vhost_check_events_dropped (vs)) +tcm_vhost_send_evt (vs,NULL ,NULL ,VIRTIO_SCSI_T_NO_EVENT ,0 ); + }static void vhost_scsi_handle_kick (struct vhost_work *work)
@@ -888,6 +1040,7 @@ static int vhost_scsi_set_endpoint(return -EEXIST ; } tv_tpg->tv_tpg_vhost_count++; + tv_tpg->vhost_scsi = vs; vs_tpg[tv_tpg->tport_tpgt] = tv_tpg;smp_mb__after_atomic_inc (); match = true;
@@ -971,6 +1124,7 @@ static int vhost_scsi_clear_endpoint( goto err_tpg; } tv_tpg->tv_tpg_vhost_count--; + tv_tpg->vhost_scsi =NULL ; vs->vs_tpg[target] =NULL ; match = true;mutex_unlock (&tv_tpg->tv_tpg_mutex);
@@ -1030,6 +1184,11 @@ static int vhost_scsi_open(struct inode *inode, struct file *f)return -ENOMEM ;vhost_work_init (&s->vs_completion_work, vhost_scsi_complete_cmd_work); +vhost_work_init (&s->vs_event_work, tcm_vhost_evt_work); + + s->vs_events_nr =0 ; + s->vs_events_dropped = false; +mutex_init (&s->vs_events_lock); s->vqs[VHOST_SCSI_VQ_CTL ].handle_kick = vhost_scsi_ctl_handle_kick; s->vqs[VHOST_SCSI_VQ_EVT ].handle_kick = vhost_scsi_evt_handle_kick;
@@ -1160,6 +1319,42 @@ static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport)return "Unknown" ; } +static int tcm_vhost_do_plug (struct tcm_vhost_tpg *tpg, +struct se_lun *lun,bool plug) +{ + +struct vhost_scsi *vs = tpg->vhost_scsi; +u32 reason; + +mutex_lock (&tpg->tv_tpg_mutex); + vs = tpg->vhost_scsi; +mutex_unlock (&tpg->tv_tpg_mutex); +if (!vs) +return -EOPNOTSUPP ; + +if (!tcm_vhost_check_feature (vs,VIRTIO_SCSI_F_HOTPLUG )) +return -EOPNOTSUPP ; + +if (plug) + reason =VIRTIO_SCSI_EVT_RESET_RESCAN ; +else + reason =VIRTIO_SCSI_EVT_RESET_REMOVED ; + +return tcm_vhost_send_evt (vs, tpg, lun, +VIRTIO_SCSI_T_TRANSPORT_RESET , + reason); +} + +static int tcm_vhost_hotplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, true); +} + +static int tcm_vhost_hotunplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, false); +} +static int tcm_vhost_port_link (struct se_portal_group *se_tpg,struct se_lun *lun) {
@@ -1170,18 +1365,21 @@ static int tcm_vhost_port_link(struct se_portal_group *se_tpg, tv_tpg->tv_tpg_port_count++;mutex_unlock (&tv_tpg->tv_tpg_mutex); +tcm_vhost_hotplug (tv_tpg, lun); +return 0 ; }static void tcm_vhost_port_unlink (struct se_portal_group *se_tpg, -struct se_lun *se_lun) +struct se_lun *lun) {struct tcm_vhost_tpg *tv_tpg =container_of (se_tpg, struct tcm_vhost_tpg, se_tpg); -mutex_lock (&tv_tpg->tv_tpg_mutex); tv_tpg->tv_tpg_port_count--;mutex_unlock (&tv_tpg->tv_tpg_mutex); + +tcm_vhost_hotunplug (tv_tpg, lun); } static struct se_node_acl *tcm_vhost_make_nodeacl(
drivers/vhost/tcm_vhost.h +10 -0
@@ -53,6 +53,7 @@ struct tcm_vhost_nacl {struct se_node_acl se_node_acl; }; +struct vhost_scsi ;struct tcm_vhost_tpg {/* Vhost port target portal group tag for TCM */ u16 tport_tpgt;
@@ -70,6 +71,8 @@ struct tcm_vhost_tpg {struct tcm_vhost_tport *tport;/* Returned by tcm_vhost_make_tpg() */ struct se_portal_group se_tpg; +/* Pointer back to struct vhost_scsi, protected by tv_tpg_mutex */ +struct vhost_scsi *vhost_scsi; };struct tcm_vhost_tport {
@@ -83,6 +86,13 @@ struct tcm_vhost_tport {struct se_wwn tport_wwn; }; +struct tcm_vhost_evt { +/* virtio_scsi event */ +struct virtio_scsi_event event; +/* virtio_scsi event list, serviced from vhost worker thread */ +struct llist_node list; +}; +/* * As per request from MST, keep TCM_VHOST related ioctl defines out of * linux/vhost.h (user-space) for now..
Signature · 1 line
1.8.1.4
Hi MST,
Ping on this series for use with vhost-scsi-pci code..?
They will need to be heading into target-pending/for-next soon to make the v3.10 window opening in t-minus 9 days. ;)
--nab
Quoted 9 lines · Asias He
Asias He (3):
tcm_vhost: Introducetcm_vhost_check_feature()
tcm_vhost: Add helper to check if endpoint is setup
tcm_vhost: Add hotplug/hotunplug supportdrivers/vhost/tcm_vhost.c | 236 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 ++ 2 files changed, 242 insertions(+), 4 deletions(-)
Hi MST,
Ping on this series for use with vhost-scsi-pci code..?
They will need to be heading into target-pending/for-next soon to make the v3.10 window opening in t-minus 9 days. ;)
--nab
Quoted 9 lines · Asias He
Asias He (3):
tcm_vhost: Introducetcm_vhost_check_feature()
tcm_vhost: Add helper to check if endpoint is setup
tcm_vhost: Add hotplug/hotunplug supportdrivers/vhost/tcm_vhost.c | 236 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 ++ 2 files changed, 242 insertions(+), 4 deletions(-)
v6 cover tcm_vhost hotplug
Rebase + small argument pass change
Asias He (3):
tcm_vhost: Introduce tcm_vhost_check_feature()
tcm_vhost: Add helper to check if endpoint is setup
tcm_vhost: Add hotplug/hotunplug support
drivers/vhost/tcm_vhost.c | 237 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 ++ 2 files changed, 243 insertions(+), 4 deletions(-)
Signature · 1 line
1.8.1.4
PATCH v6 1/3 tcm_vhost: Introduce tcm_vhost_check_feature()
This helper is useful to check if a feature is supported.
--- drivers/vhost/tcm_vhost.c | 12 ++++++++++++ 1 file changed, 12 insertions(+)
drivers/vhost/tcm_vhost.c +12 -0
@@ -99,6 +99,18 @@ static int iov_num_pages(struct iovec *iov) ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT; } +static bool tcm_vhost_check_feature (struct vhost_scsi *vs,int feature) +{ +bool ret = false; + +mutex_lock (&vs->dev.mutex); +if (vhost_has_feature (&vs->dev, feature)) + ret = true; +mutex_unlock (&vs->dev.mutex); + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;
Signature · 1 line
1.8.1.4
PATCH v6 2/3 tcm_vhost: Add helper to check if endpoint is setup
In reply to Asias He--- drivers/vhost/tcm_vhost.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)
drivers/vhost/tcm_vhost.c +18 -0
@@ -111,6 +111,24 @@ static bool tcm_vhost_check_feature(struct vhost_scsi *vs, int feature)return ret; } +static bool tcm_vhost_check_endpoint (struct vhost_virtqueue *vq) +{ +bool ret = false; + +/* +* We can handle the vq only after the endpoint is setup by calling the +* VHOST_SCSI_SET_ENDPOINT ioctl. +* +* TODO: Check that we are running from vhost_worker which acts +* as read-side critical section for vhost kind of RCU. +* See the comments in struct vhost_virtqueue in drivers/vhost/vhost.h +*/ +if (rcu_dereference_check (vq->private_data,1 )) + ret = true; + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;
Signature · 1 line
1.8.1.4
PATCH v6 3/3 tcm_vhost: Add hotplug/hotunplug support
In reply to Asias HeIn commit 365a7150094 ([SCSI] virtio-scsi: hotplug support for virtio-scsi), hotplug support is added to virtio-scsi.
This patch adds hotplug and hotunplug support to tcm_vhost.
You can create or delete a LUN in targetcli to hotplug or hotunplug a LUN in guest.
Changes in v6:
- Pass tcm_vhost_evt to tcm_vhost_do_evt_work
Changes in v5:
- Switch to int from u64 to vs_events_nr
- Set s->vs_events_dropped flag in tcm_vhost_allocate_evt
- Do not nest dev mutex within vq mutex
- Use vs_events_lock to protect vs_events_dropped and vs_events_nr
- Rebase to target/master
Changes in v4:
- Drop tcm_vhost_check_endpoint in tcm_vhost_send_evt
- Add tcm_vhost_check_endpoint in vhost_scsi_evt_handle_kick
Changes in v3:
- Separate the bug fix to another thread
Changes in v2:
- Remove code duplication in tcm_vhost_{hotplug,hotunplug}
- Fix racing of vs_events_nr
- Add flush fix patch to this series
--- drivers/vhost/tcm_vhost.c | 207 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 +++ 2 files changed, 213 insertions(+), 4 deletions(-)
drivers/vhost/tcm_vhost.c +203 -4
@@ -66,11 +66,13 @@ enum {* TODO: debug and remove the workaround. */ enum { -VHOST_SCSI_FEATURES =VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX ) +VHOST_SCSI_FEATURES = (VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX )) | + (1ULL <<VIRTIO_SCSI_F_HOTPLUG ) };#define VHOST_SCSI_MAX_TARGET 256#define VHOST_SCSI_MAX_VQ 128 +#define VHOST_SCSI_MAX_EVENT 128struct vhost_scsi {/* Protected by vhost_scsi->dev.mutex */
@@ -82,6 +84,13 @@ struct vhost_scsi {struct vhost_work vs_completion_work;/* cmd completion work item */ struct llist_head vs_completion_list;/* cmd completion queue */ + +struct vhost_work vs_event_work;/* evt injection work item */ +struct llist_head vs_event_list;/* evt injection queue */ + +struct mutex vs_events_lock;/* protect vs_events_dropped,events_nr */ +bool vs_events_dropped;/* any missed events */ +int vs_events_nr;/* num of pending events */ };/* Local pointer to allocated TCM configfs fabric module */
@@ -129,6 +138,17 @@ static bool tcm_vhost_check_endpoint(struct vhost_virtqueue *vq)return ret; } +static bool tcm_vhost_check_events_dropped (struct vhost_scsi *vs) +{ +bool ret; + +mutex_lock (&vs->vs_events_lock); + ret = vs->vs_events_dropped; +mutex_unlock (&vs->vs_events_lock); + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;
@@ -379,6 +399,37 @@ static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd)return 0 ; } +static void tcm_vhost_free_evt (struct vhost_scsi *vs,struct tcm_vhost_evt *evt) +{ +mutex_lock (&vs->vs_events_lock); + vs->vs_events_nr--; +kfree (evt); +mutex_unlock (&vs->vs_events_lock); +} + +static struct tcm_vhost_evt *tcm_vhost_allocate_evt (struct vhost_scsi *vs, +u32 event,u32 reason) +{ +struct tcm_vhost_evt *evt; + +mutex_lock (&vs->vs_events_lock); +if (vs->vs_events_nr >VHOST_SCSI_MAX_EVENT ) { + vs->vs_events_dropped = true; +mutex_unlock (&vs->vs_events_lock); +return NULL ; + } + + evt =kzalloc (sizeof (*evt),GFP_KERNEL ); +if (evt) { + evt->event.event = event; + evt->event.reason = reason; + vs->vs_events_nr++; + } +mutex_unlock (&vs->vs_events_lock); + +return evt; +} +static void vhost_scsi_free_cmd (struct tcm_vhost_cmd *tv_cmd) {struct se_cmd *se_cmd = &tv_cmd->tvc_se_cmd;
@@ -397,6 +448,75 @@ static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd)kfree (tv_cmd); } +static void tcm_vhost_do_evt_work (struct vhost_scsi *vs, +struct tcm_vhost_evt *evt) +{ +struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT ]; +struct virtio_scsi_event *event = &evt->event; +struct virtio_scsi_event __user *eventp; +unsigned out, in; +int head, ret; + +if (!tcm_vhost_check_endpoint (vq)) +return ; + +mutex_lock (&vs->vs_events_lock); +mutex_lock (&vq->mutex); +again: +vhost_disable_notify (&vs->dev, vq); + head =vhost_get_vq_desc (&vs->dev, vq, vq->iov, +ARRAY_SIZE (vq->iov), &out, &in, +NULL ,NULL ); +if (head <0 ) { + vs->vs_events_dropped = true; + goto out; + } +if (head == vq->num) { +if (vhost_enable_notify (&vs->dev, vq)) + goto again; + vs->vs_events_dropped = true; + goto out; + } + +if ((vq->iov[out].iov_len !=sizeof (struct virtio_scsi_event ))) { +vq_err (vq,"Expecting virtio_scsi_event, got %zu bytes\n" , + vq->iov[out].iov_len); + vs->vs_events_dropped = true; + goto out; + } + +if (vs->vs_events_dropped) { + event->event |=VIRTIO_SCSI_T_EVENTS_MISSED ; + vs->vs_events_dropped = false; + } + + eventp = vq->iov[out].iov_base; + ret =__copy_to_user (eventp, event,sizeof (*event)); +if (!ret) +vhost_add_used_and_signal (&vs->dev, vq, head,0 ); +else +vq_err (vq,"Faulted on tcm_vhost_send_event\n" ); +out: +mutex_unlock (&vq->mutex); +mutex_unlock (&vs->vs_events_lock); +} + +static void tcm_vhost_evt_work (struct vhost_work *work) +{ + struct vhost_scsi *vs = container_of(work, struct vhost_scsi, + vs_event_work); +struct tcm_vhost_evt *evt; +struct llist_node *llnode; + + llnode =llist_del_all (&vs->vs_event_list); +while (llnode) { + evt = llist_entry(llnode, struct tcm_vhost_evt, list); + llnode =llist_next (llnode); +tcm_vhost_do_evt_work (vs, evt); +tcm_vhost_free_evt (vs, evt); + } +} +/* Fill in status and signal that we are done processing this command * * This is scheduled in the vhost work queue so we are called with the owner
@@ -807,9 +927,42 @@ static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)pr_debug ("%s: The handling func for control queue.\n" , __func__); } +static int tcm_vhost_send_evt (struct vhost_scsi *vs,struct tcm_vhost_tpg *tpg, +struct se_lun *lun,u32 event,u32 reason) +{ +struct tcm_vhost_evt *evt; + + evt =tcm_vhost_allocate_evt (vs, event, reason); +if (!evt) +return -ENOMEM ; + +if (tpg && lun) { +/* TODO: share lun setup code with virtio-scsi.ko */ + evt->event.lun[0 ] =0x01 ; + evt->event.lun[1 ] = tpg->tport_tpgt &0xFF ; +if (lun->unpacked_lun >=256 ) + evt->event.lun[2 ] = lun->unpacked_lun >>8 |0x40 ; + evt->event.lun[3 ] = lun->unpacked_lun &0xFF ; + } + +llist_add (&evt->list, &vs->vs_event_list); +vhost_work_queue (&vs->dev, &vs->vs_event_work); + +return 0 ; +} +static void vhost_scsi_evt_handle_kick (struct vhost_work *work) { -pr_debug ("%s: The handling func for event queue.\n" , __func__); + struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, + poll.work); + struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev); + +if (!tcm_vhost_check_endpoint (vq)) +return ; + +if (tcm_vhost_check_events_dropped (vs)) +tcm_vhost_send_evt (vs,NULL ,NULL ,VIRTIO_SCSI_T_NO_EVENT ,0 ); + }static void vhost_scsi_handle_kick (struct vhost_work *work)
@@ -891,6 +1044,7 @@ static int vhost_scsi_set_endpoint(return -EEXIST ; } tv_tpg->tv_tpg_vhost_count++; + tv_tpg->vhost_scsi = vs; vs_tpg[tv_tpg->tport_tpgt] = tv_tpg;smp_mb__after_atomic_inc (); match = true;
@@ -974,6 +1128,7 @@ static int vhost_scsi_clear_endpoint( goto err_tpg; } tv_tpg->tv_tpg_vhost_count--; + tv_tpg->vhost_scsi =NULL ; vs->vs_tpg[target] =NULL ; match = true;mutex_unlock (&tv_tpg->tv_tpg_mutex);
@@ -1033,6 +1188,11 @@ static int vhost_scsi_open(struct inode *inode, struct file *f)return -ENOMEM ;vhost_work_init (&s->vs_completion_work, vhost_scsi_complete_cmd_work); +vhost_work_init (&s->vs_event_work, tcm_vhost_evt_work); + + s->vs_events_nr =0 ; + s->vs_events_dropped = false; +mutex_init (&s->vs_events_lock); s->vqs[VHOST_SCSI_VQ_CTL ].handle_kick = vhost_scsi_ctl_handle_kick; s->vqs[VHOST_SCSI_VQ_EVT ].handle_kick = vhost_scsi_evt_handle_kick;
@@ -1163,6 +1323,42 @@ static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport)return "Unknown" ; } +static int tcm_vhost_do_plug (struct tcm_vhost_tpg *tpg, +struct se_lun *lun,bool plug) +{ + +struct vhost_scsi *vs = tpg->vhost_scsi; +u32 reason; + +mutex_lock (&tpg->tv_tpg_mutex); + vs = tpg->vhost_scsi; +mutex_unlock (&tpg->tv_tpg_mutex); +if (!vs) +return -EOPNOTSUPP ; + +if (!tcm_vhost_check_feature (vs,VIRTIO_SCSI_F_HOTPLUG )) +return -EOPNOTSUPP ; + +if (plug) + reason =VIRTIO_SCSI_EVT_RESET_RESCAN ; +else + reason =VIRTIO_SCSI_EVT_RESET_REMOVED ; + +return tcm_vhost_send_evt (vs, tpg, lun, +VIRTIO_SCSI_T_TRANSPORT_RESET , + reason); +} + +static int tcm_vhost_hotplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, true); +} + +static int tcm_vhost_hotunplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, false); +} +static int tcm_vhost_port_link (struct se_portal_group *se_tpg,struct se_lun *lun) {
@@ -1173,18 +1369,21 @@ static int tcm_vhost_port_link(struct se_portal_group *se_tpg, tv_tpg->tv_tpg_port_count++;mutex_unlock (&tv_tpg->tv_tpg_mutex); +tcm_vhost_hotplug (tv_tpg, lun); +return 0 ; }static void tcm_vhost_port_unlink (struct se_portal_group *se_tpg, -struct se_lun *se_lun) +struct se_lun *lun) {struct tcm_vhost_tpg *tv_tpg =container_of (se_tpg, struct tcm_vhost_tpg, se_tpg); -mutex_lock (&tv_tpg->tv_tpg_mutex); tv_tpg->tv_tpg_port_count--;mutex_unlock (&tv_tpg->tv_tpg_mutex); + +tcm_vhost_hotunplug (tv_tpg, lun); } static struct se_node_acl *tcm_vhost_make_nodeacl(
drivers/vhost/tcm_vhost.h +10 -0
@@ -53,6 +53,7 @@ struct tcm_vhost_nacl {struct se_node_acl se_node_acl; }; +struct vhost_scsi ;struct tcm_vhost_tpg {/* Vhost port target portal group tag for TCM */ u16 tport_tpgt;
@@ -70,6 +71,8 @@ struct tcm_vhost_tpg {struct tcm_vhost_tport *tport;/* Returned by tcm_vhost_make_tpg() */ struct se_portal_group se_tpg; +/* Pointer back to struct vhost_scsi, protected by tv_tpg_mutex */ +struct vhost_scsi *vhost_scsi; };struct tcm_vhost_tport {
@@ -83,6 +86,13 @@ struct tcm_vhost_tport {struct se_wwn tport_wwn; }; +struct tcm_vhost_evt { +/* virtio_scsi event */ +struct virtio_scsi_event event; +/* virtio_scsi event list, serviced from vhost worker thread */ +struct llist_node list; +}; +/* * As per request from MST, keep TCM_VHOST related ioctl defines out of * linux/vhost.h (user-space) for now..
Signature · 1 line
1.8.1.4
PATCH v6 3/3 tcm_vhost: Add hotplug/hotunplug support
In reply to Asias HeIn commit 365a7150094 ([SCSI] virtio-scsi: hotplug support for virtio-scsi), hotplug support is added to virtio-scsi.
This patch adds hotplug and hotunplug support to tcm_vhost.
You can create or delete a LUN in targetcli to hotplug or hotunplug a LUN in guest.
Changes in v6:
- Pass tcm_vhost_evt to tcm_vhost_do_evt_work
Changes in v5:
- Switch to int from u64 to vs_events_nr
- Set s->vs_events_dropped flag in tcm_vhost_allocate_evt
- Do not nest dev mutex within vq mutex
- Use vs_events_lock to protect vs_events_dropped and vs_events_nr
- Rebase to target/master
Changes in v4:
- Drop tcm_vhost_check_endpoint in tcm_vhost_send_evt
- Add tcm_vhost_check_endpoint in vhost_scsi_evt_handle_kick
Changes in v3:
- Separate the bug fix to another thread
Changes in v2:
- Remove code duplication in tcm_vhost_{hotplug,hotunplug}
- Fix racing of vs_events_nr
- Add flush fix patch to this series
--- drivers/vhost/tcm_vhost.c | 207 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 +++ 2 files changed, 213 insertions(+), 4 deletions(-)
drivers/vhost/tcm_vhost.c +203 -4
@@ -66,11 +66,13 @@ enum {* TODO: debug and remove the workaround. */ enum { -VHOST_SCSI_FEATURES =VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX ) +VHOST_SCSI_FEATURES = (VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX )) | + (1ULL <<VIRTIO_SCSI_F_HOTPLUG ) };#define VHOST_SCSI_MAX_TARGET 256#define VHOST_SCSI_MAX_VQ 128 +#define VHOST_SCSI_MAX_EVENT 128struct vhost_scsi {/* Protected by vhost_scsi->dev.mutex */
@@ -82,6 +84,13 @@ struct vhost_scsi {struct vhost_work vs_completion_work;/* cmd completion work item */ struct llist_head vs_completion_list;/* cmd completion queue */ + +struct vhost_work vs_event_work;/* evt injection work item */ +struct llist_head vs_event_list;/* evt injection queue */ + +struct mutex vs_events_lock;/* protect vs_events_dropped,events_nr */ +bool vs_events_dropped;/* any missed events */ +int vs_events_nr;/* num of pending events */ };/* Local pointer to allocated TCM configfs fabric module */
@@ -129,6 +138,17 @@ static bool tcm_vhost_check_endpoint(struct vhost_virtqueue *vq)return ret; } +static bool tcm_vhost_check_events_dropped (struct vhost_scsi *vs) +{ +bool ret; + +mutex_lock (&vs->vs_events_lock); + ret = vs->vs_events_dropped; +mutex_unlock (&vs->vs_events_lock); + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;
@@ -379,6 +399,37 @@ static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd)return 0 ; } +static void tcm_vhost_free_evt (struct vhost_scsi *vs,struct tcm_vhost_evt *evt) +{ +mutex_lock (&vs->vs_events_lock); + vs->vs_events_nr--; +kfree (evt); +mutex_unlock (&vs->vs_events_lock); +} + +static struct tcm_vhost_evt *tcm_vhost_allocate_evt (struct vhost_scsi *vs, +u32 event,u32 reason) +{ +struct tcm_vhost_evt *evt; + +mutex_lock (&vs->vs_events_lock); +if (vs->vs_events_nr >VHOST_SCSI_MAX_EVENT ) { + vs->vs_events_dropped = true; +mutex_unlock (&vs->vs_events_lock); +return NULL ; + } + + evt =kzalloc (sizeof (*evt),GFP_KERNEL ); +if (evt) { + evt->event.event = event; + evt->event.reason = reason; + vs->vs_events_nr++; + } +mutex_unlock (&vs->vs_events_lock); + +return evt; +} +static void vhost_scsi_free_cmd (struct tcm_vhost_cmd *tv_cmd) {struct se_cmd *se_cmd = &tv_cmd->tvc_se_cmd;
@@ -397,6 +448,75 @@ static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd)kfree (tv_cmd); } +static void tcm_vhost_do_evt_work (struct vhost_scsi *vs, +struct tcm_vhost_evt *evt) +{ +struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT ]; +struct virtio_scsi_event *event = &evt->event; +struct virtio_scsi_event __user *eventp; +unsigned out, in; +int head, ret; + +if (!tcm_vhost_check_endpoint (vq)) +return ; + +mutex_lock (&vs->vs_events_lock); +mutex_lock (&vq->mutex); +again: +vhost_disable_notify (&vs->dev, vq); + head =vhost_get_vq_desc (&vs->dev, vq, vq->iov, +ARRAY_SIZE (vq->iov), &out, &in, +NULL ,NULL ); +if (head <0 ) { + vs->vs_events_dropped = true; + goto out; + } +if (head == vq->num) { +if (vhost_enable_notify (&vs->dev, vq)) + goto again; + vs->vs_events_dropped = true; + goto out; + } + +if ((vq->iov[out].iov_len !=sizeof (struct virtio_scsi_event ))) { +vq_err (vq,"Expecting virtio_scsi_event, got %zu bytes\n" , + vq->iov[out].iov_len); + vs->vs_events_dropped = true; + goto out; + } + +if (vs->vs_events_dropped) { + event->event |=VIRTIO_SCSI_T_EVENTS_MISSED ; + vs->vs_events_dropped = false; + } + + eventp = vq->iov[out].iov_base; + ret =__copy_to_user (eventp, event,sizeof (*event)); +if (!ret) +vhost_add_used_and_signal (&vs->dev, vq, head,0 ); +else +vq_err (vq,"Faulted on tcm_vhost_send_event\n" ); +out: +mutex_unlock (&vq->mutex); +mutex_unlock (&vs->vs_events_lock); +} + +static void tcm_vhost_evt_work (struct vhost_work *work) +{ + struct vhost_scsi *vs = container_of(work, struct vhost_scsi, + vs_event_work); +struct tcm_vhost_evt *evt; +struct llist_node *llnode; + + llnode =llist_del_all (&vs->vs_event_list); +while (llnode) { + evt = llist_entry(llnode, struct tcm_vhost_evt, list); + llnode =llist_next (llnode); +tcm_vhost_do_evt_work (vs, evt); +tcm_vhost_free_evt (vs, evt); + } +} +/* Fill in status and signal that we are done processing this command * * This is scheduled in the vhost work queue so we are called with the owner
@@ -807,9 +927,42 @@ static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)pr_debug ("%s: The handling func for control queue.\n" , __func__); } +static int tcm_vhost_send_evt (struct vhost_scsi *vs,struct tcm_vhost_tpg *tpg, +struct se_lun *lun,u32 event,u32 reason) +{ +struct tcm_vhost_evt *evt; + + evt =tcm_vhost_allocate_evt (vs, event, reason); +if (!evt) +return -ENOMEM ; + +if (tpg && lun) { +/* TODO: share lun setup code with virtio-scsi.ko */ + evt->event.lun[0 ] =0x01 ; + evt->event.lun[1 ] = tpg->tport_tpgt &0xFF ; +if (lun->unpacked_lun >=256 ) + evt->event.lun[2 ] = lun->unpacked_lun >>8 |0x40 ; + evt->event.lun[3 ] = lun->unpacked_lun &0xFF ; + } + +llist_add (&evt->list, &vs->vs_event_list); +vhost_work_queue (&vs->dev, &vs->vs_event_work); + +return 0 ; +} +static void vhost_scsi_evt_handle_kick (struct vhost_work *work) { -pr_debug ("%s: The handling func for event queue.\n" , __func__); + struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, + poll.work); + struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev); + +if (!tcm_vhost_check_endpoint (vq)) +return ; + +if (tcm_vhost_check_events_dropped (vs)) +tcm_vhost_send_evt (vs,NULL ,NULL ,VIRTIO_SCSI_T_NO_EVENT ,0 ); + }static void vhost_scsi_handle_kick (struct vhost_work *work)
@@ -891,6 +1044,7 @@ static int vhost_scsi_set_endpoint(return -EEXIST ; } tv_tpg->tv_tpg_vhost_count++; + tv_tpg->vhost_scsi = vs; vs_tpg[tv_tpg->tport_tpgt] = tv_tpg;smp_mb__after_atomic_inc (); match = true;
@@ -974,6 +1128,7 @@ static int vhost_scsi_clear_endpoint( goto err_tpg; } tv_tpg->tv_tpg_vhost_count--; + tv_tpg->vhost_scsi =NULL ; vs->vs_tpg[target] =NULL ; match = true;mutex_unlock (&tv_tpg->tv_tpg_mutex);
@@ -1033,6 +1188,11 @@ static int vhost_scsi_open(struct inode *inode, struct file *f)return -ENOMEM ;vhost_work_init (&s->vs_completion_work, vhost_scsi_complete_cmd_work); +vhost_work_init (&s->vs_event_work, tcm_vhost_evt_work); + + s->vs_events_nr =0 ; + s->vs_events_dropped = false; +mutex_init (&s->vs_events_lock); s->vqs[VHOST_SCSI_VQ_CTL ].handle_kick = vhost_scsi_ctl_handle_kick; s->vqs[VHOST_SCSI_VQ_EVT ].handle_kick = vhost_scsi_evt_handle_kick;
@@ -1163,6 +1323,42 @@ static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport)return "Unknown" ; } +static int tcm_vhost_do_plug (struct tcm_vhost_tpg *tpg, +struct se_lun *lun,bool plug) +{ + +struct vhost_scsi *vs = tpg->vhost_scsi; +u32 reason; + +mutex_lock (&tpg->tv_tpg_mutex); + vs = tpg->vhost_scsi; +mutex_unlock (&tpg->tv_tpg_mutex); +if (!vs) +return -EOPNOTSUPP ; + +if (!tcm_vhost_check_feature (vs,VIRTIO_SCSI_F_HOTPLUG )) +return -EOPNOTSUPP ; + +if (plug) + reason =VIRTIO_SCSI_EVT_RESET_RESCAN ; +else + reason =VIRTIO_SCSI_EVT_RESET_REMOVED ; + +return tcm_vhost_send_evt (vs, tpg, lun, +VIRTIO_SCSI_T_TRANSPORT_RESET , + reason); +} + +static int tcm_vhost_hotplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, true); +} + +static int tcm_vhost_hotunplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, false); +} +static int tcm_vhost_port_link (struct se_portal_group *se_tpg,struct se_lun *lun) {
@@ -1173,18 +1369,21 @@ static int tcm_vhost_port_link(struct se_portal_group *se_tpg, tv_tpg->tv_tpg_port_count++;mutex_unlock (&tv_tpg->tv_tpg_mutex); +tcm_vhost_hotplug (tv_tpg, lun); +return 0 ; }static void tcm_vhost_port_unlink (struct se_portal_group *se_tpg, -struct se_lun *se_lun) +struct se_lun *lun) {struct tcm_vhost_tpg *tv_tpg =container_of (se_tpg, struct tcm_vhost_tpg, se_tpg); -mutex_lock (&tv_tpg->tv_tpg_mutex); tv_tpg->tv_tpg_port_count--;mutex_unlock (&tv_tpg->tv_tpg_mutex); + +tcm_vhost_hotunplug (tv_tpg, lun); } static struct se_node_acl *tcm_vhost_make_nodeacl(
drivers/vhost/tcm_vhost.h +10 -0
@@ -53,6 +53,7 @@ struct tcm_vhost_nacl {struct se_node_acl se_node_acl; }; +struct vhost_scsi ;struct tcm_vhost_tpg {/* Vhost port target portal group tag for TCM */ u16 tport_tpgt;
@@ -70,6 +71,8 @@ struct tcm_vhost_tpg {struct tcm_vhost_tport *tport;/* Returned by tcm_vhost_make_tpg() */ struct se_portal_group se_tpg; +/* Pointer back to struct vhost_scsi, protected by tv_tpg_mutex */ +struct vhost_scsi *vhost_scsi; };struct tcm_vhost_tport {
@@ -83,6 +86,13 @@ struct tcm_vhost_tport {struct se_wwn tport_wwn; }; +struct tcm_vhost_evt { +/* virtio_scsi event */ +struct virtio_scsi_event event; +/* virtio_scsi event list, serviced from vhost worker thread */ +struct llist_node list; +}; +/* * As per request from MST, keep TCM_VHOST related ioctl defines out of * linux/vhost.h (user-space) for now..
Signature · 1 line
1.8.1.4
Michael S. Tsirkin
maintainerQuoted 8 lines · Nicholas A. Bellinger
Hi MST,
Ping on this series for use with vhost-scsi-pci code..?
They will need to be heading into target-pending/for-next soon to make the v3.10 window opening in t-minus 9 days. ;)
--nab
Let me turns this around and ask you what's the target for QEMU userspace?
Quoted 11 lines · Asias He
Quoted 9 lines
Asias He (3):
tcm_vhost: Introducetcm_vhost_check_feature()
tcm_vhost: Add helper to check if endpoint is setup
tcm_vhost: Add hotplug/hotunplug supportdrivers/vhost/tcm_vhost.c | 236 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 ++ 2 files changed, 242 insertions(+), 4 deletions(-)
Quoted 19 lines · Nicholas A. Bellinger
Hi MST,
Ping on this series for use with vhost-scsi-pci code..?
They will need to be heading into target-pending/for-next soon to make the v3.10 window opening in t-minus 9 days. ;)
--nab
Quoted 9 lines · Asias He
Asias He (3):
tcm_vhost: Introducetcm_vhost_check_feature()
tcm_vhost: Add helper to check if endpoint is setup
tcm_vhost: Add hotplug/hotunplug supportdrivers/vhost/tcm_vhost.c | 236 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 ++ 2 files changed, 242 insertions(+), 4 deletions(-)
Okay, Paolo says it's now a sure thing we'll use it in qemu, in fact in 1.5, so let's cross the fingers and start merging new code.
v7 cover tcm_vhost hotplug
Changes in v7:
- Add vhost_work_flush for vs->vs_event_work to this series
Asias He (3):
tcm_vhost: Introduce tcm_vhost_check_feature()
tcm_vhost: Add helper to check if endpoint is setup
tcm_vhost: Add hotplug/hotunplug support
drivers/vhost/tcm_vhost.c | 238 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 ++ 2 files changed, 244 insertions(+), 4 deletions(-)
Signature · 1 line
1.8.1.4
PATCH v7 1/3 tcm_vhost: Introduce tcm_vhost_check_feature()
This helper is useful to check if a feature is supported.
--- drivers/vhost/tcm_vhost.c | 12 ++++++++++++ 1 file changed, 12 insertions(+)
drivers/vhost/tcm_vhost.c +12 -0
@@ -99,6 +99,18 @@ static int iov_num_pages(struct iovec *iov) ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT; } +static bool tcm_vhost_check_feature (struct vhost_scsi *vs,int feature) +{ +bool ret = false; + +mutex_lock (&vs->dev.mutex); +if (vhost_has_feature (&vs->dev, feature)) + ret = true; +mutex_unlock (&vs->dev.mutex); + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;
Signature · 1 line
1.8.1.4
Michael S. Tsirkin
maintainerRe: [PATCH v7 1/3] tcm_vhost: Introduce tcm_vhost_check_feature()
Quoted 4 lines · Asias He
This helper is useful to check if a feature is supported.
Signed-off-by Asias He <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>
Why do we need the locking here by the way?
It seems features don't change while vhost
is running.
Quoted 29 lines
--- drivers/vhost/tcm_vhost.c | 12 ++++++++++++ 1 file changed, 12 insertions(+)drivers/vhost/tcm_vhost.c +12
-0@@ -99,6 +99,18 @@ static int iov_num_pages(struct iovec *iov) ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT; } +static bool tcm_vhost_check_feature (struct vhost_scsi *vs,int feature) +{ +bool ret = false; + +mutex_lock (&vs->dev.mutex); +if (vhost_has_feature (&vs->dev, feature)) + ret = true; +mutex_unlock (&vs->dev.mutex); + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;Signature · 1 line
1.8.1.4
Michael S. Tsirkin
maintainerRe: [PATCH v7 1/3] tcm_vhost: Introduce tcm_vhost_check_feature()
In reply to Asias HeQuoted 4 lines · Asias He
This helper is useful to check if a feature is supported.
Signed-off-by Asias He <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>
Why do we need the locking here by the way?
It seems features don't change while vhost
is running.
Quoted 29 lines
--- drivers/vhost/tcm_vhost.c | 12 ++++++++++++ 1 file changed, 12 insertions(+)drivers/vhost/tcm_vhost.c +12
-0@@ -99,6 +99,18 @@ static int iov_num_pages(struct iovec *iov) ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT; } +static bool tcm_vhost_check_feature (struct vhost_scsi *vs,int feature) +{ +bool ret = false; + +mutex_lock (&vs->dev.mutex); +if (vhost_has_feature (&vs->dev, feature)) + ret = true; +mutex_unlock (&vs->dev.mutex); + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;Signature · 1 line
1.8.1.4
Re: [PATCH v7 1/3] tcm_vhost: Introduce tcm_vhost_check_feature()
Quoted 9 lines · Michael S. Tsirkin
Quoted 4 lines · Asias He
This helper is useful to check if a feature is supported.
Signed-off-by Asias He <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>Why do we need the locking here by the way?
It seems features don't change while vhost
is running.
See:
static inline int vhost_has_feature(struct vhost_dev *dev, int bit)
{
unsigned acked_features;
/* TODO: check that we are running from vhost_worker or dev mutex is
* held? */
acked_features = rcu_dereference_index_check(dev->acked_features, 1);
return acked_features & (1 << bit);
} We are accessing it outside the vhost_worker thread.
Quoted 29 lines
Quoted 29 lines
--- drivers/vhost/tcm_vhost.c | 12 ++++++++++++ 1 file changed, 12 insertions(+)drivers/vhost/tcm_vhost.c +12
-0@@ -99,6 +99,18 @@ static int iov_num_pages(struct iovec *iov) ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT; } +static bool tcm_vhost_check_feature (struct vhost_scsi *vs,int feature) +{ +bool ret = false; + +mutex_lock (&vs->dev.mutex); +if (vhost_has_feature (&vs->dev, feature)) + ret = true; +mutex_unlock (&vs->dev.mutex); + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;Signature · 1 line
1.8.1.4
Signature · 1 line
Asias
Michael S. Tsirkin
maintainerRe: [PATCH v7 1/3] tcm_vhost: Introduce tcm_vhost_check_feature()
Quoted 23 lines · Asias He
Quoted 9 lines · Michael S. Tsirkin
Quoted 4 lines · Asias He
This helper is useful to check if a feature is supported.
Signed-off-by Asias He <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>Why do we need the locking here by the way?
It seems features don't change while vhost
is running.See:
static inline int
vhost_has_feature(struct vhost_dev *dev, int bit){ unsigned acked_features; /* TODO: check that we are running from vhost_worker or dev mutex is * held? */ acked_features = rcu_dereference_index_check(dev->acked_features, 1); return acked_features & (1 << bit); }
Quoted 1 line
We are accessing it outside the
vhost_workerthread.
Hmm I see, we allow changing features on the fly, and the reason is the LOG_ALL feature.
Okay ... do we ever call this from worker thread? Because if we do, this will deadlock. Maybe add a comment about this.
Quoted 32 lines
Quoted 29 lines
Quoted 29 lines
--- drivers/vhost/tcm_vhost.c | 12 ++++++++++++ 1 file changed, 12 insertions(+)drivers/vhost/tcm_vhost.c +12
-0@@ -99,6 +99,18 @@ static int iov_num_pages(struct iovec *iov) ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT; } +static bool tcm_vhost_check_feature (struct vhost_scsi *vs,int feature) +{ +bool ret = false; + +mutex_lock (&vs->dev.mutex); +if (vhost_has_feature (&vs->dev, feature)) + ret = true; +mutex_unlock (&vs->dev.mutex); + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;Signature · 1 line
1.8.1.4Signature · 1 line
Asias
PATCH v7 2/3 tcm_vhost: Add helper to check if endpoint is setup
In reply to Asias He--- drivers/vhost/tcm_vhost.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)
drivers/vhost/tcm_vhost.c +18 -0
@@ -111,6 +111,24 @@ static bool tcm_vhost_check_feature(struct vhost_scsi *vs, int feature)return ret; } +static bool tcm_vhost_check_endpoint (struct vhost_virtqueue *vq) +{ +bool ret = false; + +/* +* We can handle the vq only after the endpoint is setup by calling the +* VHOST_SCSI_SET_ENDPOINT ioctl. +* +* TODO: Check that we are running from vhost_worker which acts +* as read-side critical section for vhost kind of RCU. +* See the comments in struct vhost_virtqueue in drivers/vhost/vhost.h +*/ +if (rcu_dereference_check (vq->private_data,1 )) + ret = true; + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;
Signature · 1 line
1.8.1.4
Michael S. Tsirkin
maintainerRe: [PATCH v7 2/3] tcm_vhost: Add helper to check if endpoint is setup
Quoted 21 lines · Asias He
Signed-off-by Asias He <[email protected]>--- drivers/vhost/tcm_vhost.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)drivers/vhost/tcm_vhost.c +8
-0@@ -111,6 +111,24 @@ static bool tcm_vhost_check_feature(struct vhost_scsi *vs, int feature)return ret; } +static bool tcm_vhost_check_endpoint (struct vhost_virtqueue *vq) +{ +bool ret = false; + +/* +* We can handle the vq only after the endpoint is setup by calling the +* VHOST_SCSI_SET_ENDPOINT ioctl. +*
drivers/vhost/tcm_vhost.c static bool tcm_vhost_check_feature(struct vhost_scsi *vs, int feature) · line 131
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+* TODO: Check that we are running from vhost_worker which acts +* as read-side critical section for vhost kind of RCU. +* See the comments in struct vhost_virtqueue in drivers/vhost/vhost.h +*/ +if (rcu_dereference_check (vq->private_data,1 )) + ret = true; + +return ret; +} +
You can't use RCU in this way. You need to actually derefence. Instead, just move this within vq mutex and do rcu_dereference_protected. document that this function requires vq mutex.
Quoted 5 lines
drivers/vhost/tcm_vhost.c +0
-0static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;Signature · 1 line
1.8.1.4
Re: [PATCH v7 2/3] tcm_vhost: Add helper to check if endpoint is setup
Quoted 29 lines · Michael S. Tsirkin
Quoted 28 lines · Asias He
Signed-off-by Asias He <[email protected]>--- drivers/vhost/tcm_vhost.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)drivers/vhost/tcm_vhost.c +15
-0@@ -111,6 +111,24 @@ static bool tcm_vhost_check_feature(struct vhost_scsi *vs, int feature)return ret; } +static bool tcm_vhost_check_endpoint (struct vhost_virtqueue *vq) +{ +bool ret = false; + +/* +* We can handle the vq only after the endpoint is setup by calling the +* VHOST_SCSI_SET_ENDPOINT ioctl. +* +* TODO: Check that we are running from vhost_worker which acts +* as read-side critical section for vhost kind of RCU. +* See the comments in struct vhost_virtqueue in drivers/vhost/vhost.h +*/ +if (rcu_dereference_check (vq->private_data,1 )) + ret = true; +
Quoted 8 lines
drivers/vhost/tcm_vhost.c static bool tcm_vhost_check_feature(struct vhost_scsi *vs, int feature) · line 131
Quoted 3 lines
drivers/vhost/tcm_vhost.c +3
-0+return ret; +} +You can't use RCU in this way. You need to actually derefence. Instead, just move this within vq mutex and do
rcu_dereference_protected. document that this function requires vq mutex.
Any difference with:
vs_tpg = rcu_dereference_check(vq->private_data, 1); if (vs_tpg)
return ture
else
return false;
Note, tcm_vhost_check_endpoint() is called in multiple places. Having a helper is more convenient.
Quoted 5 lines
Quoted 5 lines
drivers/vhost/tcm_vhost.c +0
-0static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;Signature · 1 line
1.8.1.4
Signature · 1 line
Asias
Michael S. Tsirkin
maintainerRe: [PATCH v7 2/3] tcm_vhost: Add helper to check if endpoint is setup
Quoted 39 lines · Asias He
Quoted 29 lines · Michael S. Tsirkin
Quoted 28 lines · Asias He
Signed-off-by Asias He <[email protected]>--- drivers/vhost/tcm_vhost.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)drivers/vhost/tcm_vhost.c +15
-0@@ -111,6 +111,24 @@ static bool tcm_vhost_check_feature(struct vhost_scsi *vs, int feature)return ret; } +static bool tcm_vhost_check_endpoint (struct vhost_virtqueue *vq) +{ +bool ret = false; + +/* +* We can handle the vq only after the endpoint is setup by calling the +* VHOST_SCSI_SET_ENDPOINT ioctl. +* +* TODO: Check that we are running from vhost_worker which acts +* as read-side critical section for vhost kind of RCU. +* See the comments in struct vhost_virtqueue in drivers/vhost/vhost.h +*/ +if (rcu_dereference_check (vq->private_data,1 )) + ret = true; +Quoted 8 lines
drivers/vhost/tcm_vhost.c static bool tcm_vhost_check_feature(struct vhost_scsi *vs, int feature) · line 131
Quoted 3 lines
drivers/vhost/tcm_vhost.c +3
-0+return ret; +} +You can't use RCU in this way. You need to actually derefence. Instead, just move this within vq mutex and do
rcu_dereference_protected. document that this function requires vq mutex.
Quoted 3 lines
Any difference with:
vs_tpg = rcu_dereference_check(vq->private_data, 1);
Yes, it's not the same. rcu_dereference_protected says "this is generally RCU but here I use it under lock". rcu_dereference_check can only be used if you really dereference later, and you don't.
Quoted 7 lines
if (vs_tpg) return ture else return false;Note,
tcm_vhost_check_endpoint()is called in multiple places. Having a helper is more convenient.
I'm fine with the helper, but fix the implementation and fix all callers to have vq mutex when calling.
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
drivers/vhost/tcm_vhost.c +0
-0static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;Signature · 1 line
1.8.1.4Signature · 1 line
Asias
Re: [PATCH v7 2/3] tcm_vhost: Add helper to check if endpoint is setup
Quoted 40 lines · Michael S. Tsirkin
Quoted 39 lines · Asias He
Quoted 37 lines · Michael S. Tsirkin
Quoted 21 lines · Asias He
Signed-off-by Asias He <[email protected]>--- drivers/vhost/tcm_vhost.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)drivers/vhost/tcm_vhost.c +8
-0@@ -111,6 +111,24 @@ static bool tcm_vhost_check_feature(struct vhost_scsi *vs, int feature)return ret; } +static bool tcm_vhost_check_endpoint (struct vhost_virtqueue *vq) +{ +bool ret = false; + +/* +* We can handle the vq only after the endpoint is setup by calling the +* VHOST_SCSI_SET_ENDPOINT ioctl. +* drivers/vhost/tcm_vhost.c static bool tcm_vhost_check_feature(struct vhost_scsi *vs, int feature) · line 131
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+* TODO: Check that we are running from vhost_worker which acts +* as read-side critical section for vhost kind of RCU. +* See the comments in struct vhost_virtqueue in drivers/vhost/vhost.h +*/ +if (rcu_dereference_check (vq->private_data,1 )) + ret = true; + +return ret; +} +You can't use RCU in this way. You need to actually derefence. Instead, just move this within vq mutex and do
rcu_dereference_protected. document that this function requires vq mutex.
Quoted 8 lines
Quoted 3 lines
Any difference with:
vs_tpg = rcu_dereference_check(vq->private_data, 1);Yes, it's not the same.
rcu_dereference_protectedsays "this is generally RCU but here I use it under lock".rcu_dereference_checkcan only be used if you really dereference later, and you don't.
So you want the helper to be this?
static bool tcm_vhost_check_endpoint()
{
struct tcm_vhost_tpg **vs_tpg;
vs_tpg = rcu_dereference_check(vq->private_data, 1);
if (vs_tpg)
return true;
else
return false
}And what difference code the compiler will generate with this:
static bool tcm_vhost_check_endpoint()
{
if (rcu_dereference_check(vq->private_data, 1))
return true
else
return false
}Quoted 11 lines
Quoted 8 lines
if (vs_tpg) return ture else return false;Note,
tcm_vhost_check_endpoint()is called in multiple places. Having a helper is more convenient.I'm fine with the helper, but fix the implementation and fix all callers to have vq mutex when calling.
All the caller are in the vhost thread. Why need vq mutex to be taken?
Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
drivers/vhost/tcm_vhost.c +0
-0static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;Signature · 1 line
1.8.1.4Signature · 1 line
Asias
Signature · 1 line
Asias
Re: [PATCH v7 2/3] tcm_vhost: Add helper to check if endpoint is setup
In reply to Michael S. TsirkinQuoted 40 lines · Michael S. Tsirkin
Quoted 39 lines · Asias He
Quoted 37 lines · Michael S. Tsirkin
Quoted 21 lines · Asias He
Signed-off-by Asias He <[email protected]>--- drivers/vhost/tcm_vhost.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)drivers/vhost/tcm_vhost.c +8
-0@@ -111,6 +111,24 @@ static bool tcm_vhost_check_feature(struct vhost_scsi *vs, int feature)return ret; } +static bool tcm_vhost_check_endpoint (struct vhost_virtqueue *vq) +{ +bool ret = false; + +/* +* We can handle the vq only after the endpoint is setup by calling the +* VHOST_SCSI_SET_ENDPOINT ioctl. +* drivers/vhost/tcm_vhost.c static bool tcm_vhost_check_feature(struct vhost_scsi *vs, int feature) · line 131
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+* TODO: Check that we are running from vhost_worker which acts +* as read-side critical section for vhost kind of RCU. +* See the comments in struct vhost_virtqueue in drivers/vhost/vhost.h +*/ +if (rcu_dereference_check (vq->private_data,1 )) + ret = true; + +return ret; +} +You can't use RCU in this way. You need to actually derefence. Instead, just move this within vq mutex and do
rcu_dereference_protected. document that this function requires vq mutex.
Quoted 8 lines
Quoted 3 lines
Any difference with:
vs_tpg = rcu_dereference_check(vq->private_data, 1);Yes, it's not the same.
rcu_dereference_protectedsays "this is generally RCU but here I use it under lock".rcu_dereference_checkcan only be used if you really dereference later, and you don't.
So you want the helper to be this?
static bool tcm_vhost_check_endpoint()
{
struct tcm_vhost_tpg **vs_tpg;
vs_tpg = rcu_dereference_check(vq->private_data, 1);
if (vs_tpg)
return true;
else
return false
}And what difference code the compiler will generate with this:
static bool tcm_vhost_check_endpoint()
{
if (rcu_dereference_check(vq->private_data, 1))
return true
else
return false
}Quoted 11 lines
Quoted 8 lines
if (vs_tpg) return ture else return false;Note,
tcm_vhost_check_endpoint()is called in multiple places. Having a helper is more convenient.I'm fine with the helper, but fix the implementation and fix all callers to have vq mutex when calling.
All the caller are in the vhost thread. Why need vq mutex to be taken?
Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
drivers/vhost/tcm_vhost.c +0
-0static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;Signature · 1 line
1.8.1.4Signature · 1 line
Asias
Signature · 1 line
Asias
Michael S. Tsirkin
maintainerRe: [PATCH v7 2/3] tcm_vhost: Add helper to check if endpoint is setup
Quoted 65 lines · Asias He
Quoted 40 lines · Michael S. Tsirkin
Quoted 39 lines · Asias He
Quoted 37 lines · Michael S. Tsirkin
Quoted 21 lines · Asias He
Signed-off-by Asias He <[email protected]>--- drivers/vhost/tcm_vhost.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)drivers/vhost/tcm_vhost.c +8
-0@@ -111,6 +111,24 @@ static bool tcm_vhost_check_feature(struct vhost_scsi *vs, int feature)return ret; } +static bool tcm_vhost_check_endpoint (struct vhost_virtqueue *vq) +{ +bool ret = false; + +/* +* We can handle the vq only after the endpoint is setup by calling the +* VHOST_SCSI_SET_ENDPOINT ioctl. +* drivers/vhost/tcm_vhost.c static bool tcm_vhost_check_feature(struct vhost_scsi *vs, int feature) · line 131
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+* TODO: Check that we are running from vhost_worker which acts +* as read-side critical section for vhost kind of RCU. +* See the comments in struct vhost_virtqueue in drivers/vhost/vhost.h +*/ +if (rcu_dereference_check (vq->private_data,1 )) + ret = true; + +return ret; +} +You can't use RCU in this way. You need to actually derefence. Instead, just move this within vq mutex and do
rcu_dereference_protected. document that this function requires vq mutex.Quoted 8 lines
Quoted 3 lines
Any difference with:
vs_tpg = rcu_dereference_check(vq->private_data, 1);Yes, it's not the same.
rcu_dereference_protectedsays "this is generally RCU but here I use it under lock".rcu_dereference_checkcan only be used if you really dereference later, and you don't.So you want the helper to be this?
static bool
tcm_vhost_check_endpoint(){ struct tcm_vhost_tpg **vs_tpg; vs_tpg = rcu_dereference_check(vq->private_data, 1); if (vs_tpg) return true; else return false }And what difference code the compiler will generate with this:
Quoted 9 lines
static bool
tcm_vhost_check_endpoint()
{if (rcu_dereference_check(vq->private_data, 1)) return true else return false }
No, what I want is that private data is
either dereferenced (not tested against NULL)
or used under vq mutex.
In this case the second option is the easiest
and the cleanest. So move it under mutex
and then test it safely without rcu.
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
if (vs_tpg) return ture else return false;
Quoted 7 lines
Quoted 5 lines
Quoted 2 lines
Note,
tcm_vhost_check_endpoint()is called in multiple places. Having a helper is more convenient.I'm fine with the helper, but fix the implementation and fix all callers to have vq mutex when calling.
All the caller are in the vhost thread. Why need vq mutex to be taken?
vhost thread is an implementation detail, people are experimenting with various threading strategies. In particular, code must
keep working when we move it to use a workqueue.
Quoted 11 lines
Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
drivers/vhost/tcm_vhost.c +0
-0static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;Signature · 1 line
1.8.1.4Signature · 1 line
AsiasSignature · 1 line
Asias
Re: [PATCH v7 2/3] tcm_vhost: Add helper to check if endpoint is setup
Quoted 76 lines · Michael S. Tsirkin
Quoted 65 lines · Asias He
Quoted 40 lines · Michael S. Tsirkin
Quoted 39 lines · Asias He
Quoted 37 lines · Michael S. Tsirkin
Quoted 21 lines · Asias He
Signed-off-by Asias He <[email protected]>--- drivers/vhost/tcm_vhost.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)drivers/vhost/tcm_vhost.c +8
-0@@ -111,6 +111,24 @@ static bool tcm_vhost_check_feature(struct vhost_scsi *vs, int feature)return ret; } +static bool tcm_vhost_check_endpoint (struct vhost_virtqueue *vq) +{ +bool ret = false; + +/* +* We can handle the vq only after the endpoint is setup by calling the +* VHOST_SCSI_SET_ENDPOINT ioctl. +* drivers/vhost/tcm_vhost.c static bool tcm_vhost_check_feature(struct vhost_scsi *vs, int feature) · line 131
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+* TODO: Check that we are running from vhost_worker which acts +* as read-side critical section for vhost kind of RCU. +* See the comments in struct vhost_virtqueue in drivers/vhost/vhost.h +*/ +if (rcu_dereference_check (vq->private_data,1 )) + ret = true; + +return ret; +} +You can't use RCU in this way. You need to actually derefence. Instead, just move this within vq mutex and do
rcu_dereference_protected. document that this function requires vq mutex.Quoted 8 lines
Quoted 3 lines
Any difference with:
vs_tpg = rcu_dereference_check(vq->private_data, 1);Yes, it's not the same.
rcu_dereference_protectedsays "this is generally RCU but here I use it under lock".rcu_dereference_checkcan only be used if you really dereference later, and you don't.So you want the helper to be this?
static bool
tcm_vhost_check_endpoint(){ struct tcm_vhost_tpg **vs_tpg; vs_tpg = rcu_dereference_check(vq->private_data, 1); if (vs_tpg) return true; else return false }And what difference code the compiler will generate with this:
Quoted 9 lines
static bool
tcm_vhost_check_endpoint()
{if (rcu_dereference_check(vq->private_data, 1)) return true else return false }
Quoted 6 lines
No, what I want is that private data is
either dereferenced (not tested against NULL)
or used under vq mutex.
In this case the second option is the easiest
and the cleanest. So move it under mutex
and then test it safely without rcu.
Why you use it in -net? The test against NULL is under in the rcu read critical section. What is wrong with this really?
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 8 lines
if (vs_tpg) return ture else return false;Note,
tcm_vhost_check_endpoint()is called in multiple places. Having a helper is more convenient.
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines
I'm fine with the helper, but fix the implementation and fix all callers to have vq mutex when calling.
All the caller are in the vhost thread. Why need vq mutex to be taken?
vhost thread is an implementation detail, people are experimenting with various threading strategies. In particular, code must
keep working when we move it to use a workqueue.
Well, you should remove your vhost RCU thing first and use real RCU explicitly. Does -net keep working after vhost changes to workqueue? Since when this becomes a requirement?
Quoted 11 lines
Quoted 11 lines
Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
drivers/vhost/tcm_vhost.c +0
-0static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;Signature · 1 line
1.8.1.4Signature · 1 line
AsiasSignature · 1 line
Asias
Signature · 1 line
Asias
Michael S. Tsirkin
maintainerRe: [PATCH v7 2/3] tcm_vhost: Add helper to check if endpoint is setup
Quoted 84 lines · Asias He
Quoted 76 lines · Michael S. Tsirkin
Quoted 65 lines · Asias He
Quoted 40 lines · Michael S. Tsirkin
Quoted 39 lines · Asias He
Quoted 37 lines · Michael S. Tsirkin
Quoted 21 lines · Asias He
Signed-off-by Asias He <[email protected]>--- drivers/vhost/tcm_vhost.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)drivers/vhost/tcm_vhost.c +8
-0@@ -111,6 +111,24 @@ static bool tcm_vhost_check_feature(struct vhost_scsi *vs, int feature)return ret; } +static bool tcm_vhost_check_endpoint (struct vhost_virtqueue *vq) +{ +bool ret = false; + +/* +* We can handle the vq only after the endpoint is setup by calling the +* VHOST_SCSI_SET_ENDPOINT ioctl. +* drivers/vhost/tcm_vhost.c static bool tcm_vhost_check_feature(struct vhost_scsi *vs, int feature) · line 131
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+* TODO: Check that we are running from vhost_worker which acts +* as read-side critical section for vhost kind of RCU. +* See the comments in struct vhost_virtqueue in drivers/vhost/vhost.h +*/ +if (rcu_dereference_check (vq->private_data,1 )) + ret = true; + +return ret; +} +You can't use RCU in this way. You need to actually derefence. Instead, just move this within vq mutex and do
rcu_dereference_protected. document that this function requires vq mutex.Quoted 8 lines
Quoted 3 lines
Any difference with:
vs_tpg = rcu_dereference_check(vq->private_data, 1);Yes, it's not the same.
rcu_dereference_protectedsays "this is generally RCU but here I use it under lock".rcu_dereference_checkcan only be used if you really dereference later, and you don't.So you want the helper to be this?
static bool
tcm_vhost_check_endpoint(){ struct tcm_vhost_tpg **vs_tpg; vs_tpg = rcu_dereference_check(vq->private_data, 1); if (vs_tpg) return true; else return false }And what difference code the compiler will generate with this:
Quoted 9 lines
static bool
tcm_vhost_check_endpoint()
{if (rcu_dereference_check(vq->private_data, 1)) return true else return false }Quoted 6 lines
No, what I want is that private data is
either dereferenced (not tested against NULL)
or used under vq mutex.
In this case the second option is the easiest
and the cleanest. So move it under mutex
and then test it safely without rcu.
Quoted 2 lines
Why you use it in -net? The test against NULL is under in the rcu read critical section. What is wrong with this really?
Look at the code and see that barriers in rcu_dereference assume that pointer really is dereferenced.
You don't dereference it, so rcu_dereference_XXX is wrong.
Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
Quoted 11 lines
Quoted 8 lines
if (vs_tpg) return ture else return false;Note,
tcm_vhost_check_endpoint()is called in multiple places. Having a helper is more convenient.I'm fine with the helper, but fix the implementation and fix all callers to have vq mutex when calling.
Quoted 8 lines
Quoted 5 lines
Quoted 1 line
All the caller are in the vhost thread. Why need vq mutex to be taken?
vhost thread is an implementation detail, people are experimenting with various threading strategies. In particular, code must
keep working when we move it to use a workqueue.Well, you should remove your vhost RCU thing first and use real RCU explicitly.
I don't think it's useful or possible.
Quoted 1 line
Does -net keep working after vhost changes to workqueue?
I think so, yes.
Quoted 1 line
Since when this becomes a requirement?
It's the requirement to use the abstractions that are in place. Devices should use the interfaces and not assume vhost implementation, it happens to be the case but it's not guaranteed.
Quoted 14 lines
Quoted 11 lines
Quoted 11 lines
Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
drivers/vhost/tcm_vhost.c +0
-0static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;Signature · 1 line
1.8.1.4Signature · 1 line
AsiasSignature · 1 line
AsiasSignature · 1 line
Asias
Michael S. Tsirkin
maintainerRe: [PATCH v7 2/3] tcm_vhost: Add helper to check if endpoint is setup
In reply to Asias HeQuoted 84 lines · Asias He
Quoted 76 lines · Michael S. Tsirkin
Quoted 65 lines · Asias He
Quoted 40 lines · Michael S. Tsirkin
Quoted 39 lines · Asias He
Quoted 37 lines · Michael S. Tsirkin
Quoted 21 lines · Asias He
Signed-off-by Asias He <[email protected]>--- drivers/vhost/tcm_vhost.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)drivers/vhost/tcm_vhost.c +8
-0@@ -111,6 +111,24 @@ static bool tcm_vhost_check_feature(struct vhost_scsi *vs, int feature)return ret; } +static bool tcm_vhost_check_endpoint (struct vhost_virtqueue *vq) +{ +bool ret = false; + +/* +* We can handle the vq only after the endpoint is setup by calling the +* VHOST_SCSI_SET_ENDPOINT ioctl. +* drivers/vhost/tcm_vhost.c static bool tcm_vhost_check_feature(struct vhost_scsi *vs, int feature) · line 131
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+* TODO: Check that we are running from vhost_worker which acts +* as read-side critical section for vhost kind of RCU. +* See the comments in struct vhost_virtqueue in drivers/vhost/vhost.h +*/ +if (rcu_dereference_check (vq->private_data,1 )) + ret = true; + +return ret; +} +You can't use RCU in this way. You need to actually derefence. Instead, just move this within vq mutex and do
rcu_dereference_protected. document that this function requires vq mutex.Quoted 8 lines
Quoted 3 lines
Any difference with:
vs_tpg = rcu_dereference_check(vq->private_data, 1);Yes, it's not the same.
rcu_dereference_protectedsays "this is generally RCU but here I use it under lock".rcu_dereference_checkcan only be used if you really dereference later, and you don't.So you want the helper to be this?
static bool
tcm_vhost_check_endpoint(){ struct tcm_vhost_tpg **vs_tpg; vs_tpg = rcu_dereference_check(vq->private_data, 1); if (vs_tpg) return true; else return false }And what difference code the compiler will generate with this:
Quoted 9 lines
static bool
tcm_vhost_check_endpoint()
{if (rcu_dereference_check(vq->private_data, 1)) return true else return false }Quoted 6 lines
No, what I want is that private data is
either dereferenced (not tested against NULL)
or used under vq mutex.
In this case the second option is the easiest
and the cleanest. So move it under mutex
and then test it safely without rcu.
Quoted 2 lines
Why you use it in -net? The test against NULL is under in the rcu read critical section. What is wrong with this really?
Look at the code and see that barriers in rcu_dereference assume that pointer really is dereferenced.
You don't dereference it, so rcu_dereference_XXX is wrong.
Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
Quoted 11 lines
Quoted 8 lines
if (vs_tpg) return ture else return false;Note,
tcm_vhost_check_endpoint()is called in multiple places. Having a helper is more convenient.I'm fine with the helper, but fix the implementation and fix all callers to have vq mutex when calling.
Quoted 8 lines
Quoted 5 lines
Quoted 1 line
All the caller are in the vhost thread. Why need vq mutex to be taken?
vhost thread is an implementation detail, people are experimenting with various threading strategies. In particular, code must
keep working when we move it to use a workqueue.Well, you should remove your vhost RCU thing first and use real RCU explicitly.
I don't think it's useful or possible.
Quoted 1 line
Does -net keep working after vhost changes to workqueue?
I think so, yes.
Quoted 1 line
Since when this becomes a requirement?
It's the requirement to use the abstractions that are in place. Devices should use the interfaces and not assume vhost implementation, it happens to be the case but it's not guaranteed.
Quoted 14 lines
Quoted 11 lines
Quoted 11 lines
Quoted 8 lines
Quoted 8 lines
Quoted 5 lines
Quoted 5 lines
drivers/vhost/tcm_vhost.c +0
-0static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;Signature · 1 line
1.8.1.4Signature · 1 line
AsiasSignature · 1 line
AsiasSignature · 1 line
Asias
PATCH v7 2/3 tcm_vhost: Add helper to check if endpoint is setup
In reply to Asias He--- drivers/vhost/tcm_vhost.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)
drivers/vhost/tcm_vhost.c +18 -0
@@ -111,6 +111,24 @@ static bool tcm_vhost_check_feature(struct vhost_scsi *vs, int feature)return ret; } +static bool tcm_vhost_check_endpoint (struct vhost_virtqueue *vq) +{ +bool ret = false; + +/* +* We can handle the vq only after the endpoint is setup by calling the +* VHOST_SCSI_SET_ENDPOINT ioctl. +* +* TODO: Check that we are running from vhost_worker which acts +* as read-side critical section for vhost kind of RCU. +* See the comments in struct vhost_virtqueue in drivers/vhost/vhost.h +*/ +if (rcu_dereference_check (vq->private_data,1 )) + ret = true; + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;
Signature · 1 line
1.8.1.4
PATCH v7 3/3 tcm_vhost: Add hotplug/hotunplug support
In reply to Asias HeIn commit 365a7150094 ([SCSI] virtio-scsi: hotplug support for virtio-scsi), hotplug support is added to virtio-scsi.
This patch adds hotplug and hotunplug support to tcm_vhost.
You can create or delete a LUN in targetcli to hotplug or hotunplug a LUN in guest.
Changes in v7:
- Add vhost_work_flush for vs->vs_event_work to this series
Changes in v6:
- Pass tcm_vhost_evt to tcm_vhost_do_evt_work
Changes in v5:
- Switch to int from u64 to vs_events_nr
- Set s->vs_events_dropped flag in tcm_vhost_allocate_evt
- Do not nest dev mutex within vq mutex
- Use vs_events_lock to protect vs_events_dropped and vs_events_nr
- Rebase to target/master
Changes in v4:
- Drop tcm_vhost_check_endpoint in tcm_vhost_send_evt
- Add tcm_vhost_check_endpoint in vhost_scsi_evt_handle_kick
Changes in v3:
- Separate the bug fix to another thread
Changes in v2:
- Remove code duplication in tcm_vhost_{hotplug,hotunplug}
- Fix racing of vs_events_nr
- Add flush fix patch to this series
--- drivers/vhost/tcm_vhost.c | 208 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 +++ 2 files changed, 214 insertions(+), 4 deletions(-)
drivers/vhost/tcm_vhost.c +204 -4
@@ -66,11 +66,13 @@ enum {* TODO: debug and remove the workaround. */ enum { -VHOST_SCSI_FEATURES =VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX ) +VHOST_SCSI_FEATURES = (VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX )) | + (1ULL <<VIRTIO_SCSI_F_HOTPLUG ) };#define VHOST_SCSI_MAX_TARGET 256#define VHOST_SCSI_MAX_VQ 128 +#define VHOST_SCSI_MAX_EVENT 128struct vhost_scsi {/* Protected by vhost_scsi->dev.mutex */
@@ -82,6 +84,13 @@ struct vhost_scsi {struct vhost_work vs_completion_work;/* cmd completion work item */ struct llist_head vs_completion_list;/* cmd completion queue */ + +struct vhost_work vs_event_work;/* evt injection work item */ +struct llist_head vs_event_list;/* evt injection queue */ + +struct mutex vs_events_lock;/* protect vs_events_dropped,events_nr */ +bool vs_events_dropped;/* any missed events */ +int vs_events_nr;/* num of pending events */ };/* Local pointer to allocated TCM configfs fabric module */
@@ -129,6 +138,17 @@ static bool tcm_vhost_check_endpoint(struct vhost_virtqueue *vq)return ret; } +static bool tcm_vhost_check_events_dropped (struct vhost_scsi *vs) +{ +bool ret; + +mutex_lock (&vs->vs_events_lock); + ret = vs->vs_events_dropped; +mutex_unlock (&vs->vs_events_lock); + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;
@@ -379,6 +399,37 @@ static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd)return 0 ; } +static void tcm_vhost_free_evt (struct vhost_scsi *vs,struct tcm_vhost_evt *evt) +{ +mutex_lock (&vs->vs_events_lock); + vs->vs_events_nr--; +kfree (evt); +mutex_unlock (&vs->vs_events_lock); +} + +static struct tcm_vhost_evt *tcm_vhost_allocate_evt (struct vhost_scsi *vs, +u32 event,u32 reason) +{ +struct tcm_vhost_evt *evt; + +mutex_lock (&vs->vs_events_lock); +if (vs->vs_events_nr >VHOST_SCSI_MAX_EVENT ) { + vs->vs_events_dropped = true; +mutex_unlock (&vs->vs_events_lock); +return NULL ; + } + + evt =kzalloc (sizeof (*evt),GFP_KERNEL ); +if (evt) { + evt->event.event = event; + evt->event.reason = reason; + vs->vs_events_nr++; + } +mutex_unlock (&vs->vs_events_lock); + +return evt; +} +static void vhost_scsi_free_cmd (struct tcm_vhost_cmd *tv_cmd) {struct se_cmd *se_cmd = &tv_cmd->tvc_se_cmd;
@@ -397,6 +448,75 @@ static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd)kfree (tv_cmd); } +static void tcm_vhost_do_evt_work (struct vhost_scsi *vs, +struct tcm_vhost_evt *evt) +{ +struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT ]; +struct virtio_scsi_event *event = &evt->event; +struct virtio_scsi_event __user *eventp; +unsigned out, in; +int head, ret; + +if (!tcm_vhost_check_endpoint (vq)) +return ; + +mutex_lock (&vs->vs_events_lock); +mutex_lock (&vq->mutex); +again: +vhost_disable_notify (&vs->dev, vq); + head =vhost_get_vq_desc (&vs->dev, vq, vq->iov, +ARRAY_SIZE (vq->iov), &out, &in, +NULL ,NULL ); +if (head <0 ) { + vs->vs_events_dropped = true; + goto out; + } +if (head == vq->num) { +if (vhost_enable_notify (&vs->dev, vq)) + goto again; + vs->vs_events_dropped = true; + goto out; + } + +if ((vq->iov[out].iov_len !=sizeof (struct virtio_scsi_event ))) { +vq_err (vq,"Expecting virtio_scsi_event, got %zu bytes\n" , + vq->iov[out].iov_len); + vs->vs_events_dropped = true; + goto out; + } + +if (vs->vs_events_dropped) { + event->event |=VIRTIO_SCSI_T_EVENTS_MISSED ; + vs->vs_events_dropped = false; + } + + eventp = vq->iov[out].iov_base; + ret =__copy_to_user (eventp, event,sizeof (*event)); +if (!ret) +vhost_add_used_and_signal (&vs->dev, vq, head,0 ); +else +vq_err (vq,"Faulted on tcm_vhost_send_event\n" ); +out: +mutex_unlock (&vq->mutex); +mutex_unlock (&vs->vs_events_lock); +} + +static void tcm_vhost_evt_work (struct vhost_work *work) +{ + struct vhost_scsi *vs = container_of(work, struct vhost_scsi, + vs_event_work); +struct tcm_vhost_evt *evt; +struct llist_node *llnode; + + llnode =llist_del_all (&vs->vs_event_list); +while (llnode) { + evt = llist_entry(llnode, struct tcm_vhost_evt, list); + llnode =llist_next (llnode); +tcm_vhost_do_evt_work (vs, evt); +tcm_vhost_free_evt (vs, evt); + } +} +/* Fill in status and signal that we are done processing this command * * This is scheduled in the vhost work queue so we are called with the owner
@@ -807,9 +927,42 @@ static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)pr_debug ("%s: The handling func for control queue.\n" , __func__); } +static int tcm_vhost_send_evt (struct vhost_scsi *vs,struct tcm_vhost_tpg *tpg, +struct se_lun *lun,u32 event,u32 reason) +{ +struct tcm_vhost_evt *evt; + + evt =tcm_vhost_allocate_evt (vs, event, reason); +if (!evt) +return -ENOMEM ; + +if (tpg && lun) { +/* TODO: share lun setup code with virtio-scsi.ko */ + evt->event.lun[0 ] =0x01 ; + evt->event.lun[1 ] = tpg->tport_tpgt &0xFF ; +if (lun->unpacked_lun >=256 ) + evt->event.lun[2 ] = lun->unpacked_lun >>8 |0x40 ; + evt->event.lun[3 ] = lun->unpacked_lun &0xFF ; + } + +llist_add (&evt->list, &vs->vs_event_list); +vhost_work_queue (&vs->dev, &vs->vs_event_work); + +return 0 ; +} +static void vhost_scsi_evt_handle_kick (struct vhost_work *work) { -pr_debug ("%s: The handling func for event queue.\n" , __func__); + struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, + poll.work); + struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev); + +if (!tcm_vhost_check_endpoint (vq)) +return ; + +if (tcm_vhost_check_events_dropped (vs)) +tcm_vhost_send_evt (vs,NULL ,NULL ,VIRTIO_SCSI_T_NO_EVENT ,0 ); + }static void vhost_scsi_handle_kick (struct vhost_work *work)
@@ -833,6 +986,7 @@ static void vhost_scsi_flush(struct vhost_scsi *vs)for (i =0 ; i <VHOST_SCSI_MAX_VQ ; i++)vhost_scsi_flush_vq (vs, i);vhost_work_flush (&vs->dev, &vs->vs_completion_work); +vhost_work_flush (&vs->dev, &vs->vs_event_work); }/*
@@ -891,6 +1045,7 @@ static int vhost_scsi_set_endpoint(return -EEXIST ; } tv_tpg->tv_tpg_vhost_count++; + tv_tpg->vhost_scsi = vs; vs_tpg[tv_tpg->tport_tpgt] = tv_tpg;smp_mb__after_atomic_inc (); match = true;
@@ -974,6 +1129,7 @@ static int vhost_scsi_clear_endpoint( goto err_tpg; } tv_tpg->tv_tpg_vhost_count--; + tv_tpg->vhost_scsi =NULL ; vs->vs_tpg[target] =NULL ; match = true;mutex_unlock (&tv_tpg->tv_tpg_mutex);
@@ -1033,6 +1189,11 @@ static int vhost_scsi_open(struct inode *inode, struct file *f)return -ENOMEM ;vhost_work_init (&s->vs_completion_work, vhost_scsi_complete_cmd_work); +vhost_work_init (&s->vs_event_work, tcm_vhost_evt_work); + + s->vs_events_nr =0 ; + s->vs_events_dropped = false; +mutex_init (&s->vs_events_lock); s->vqs[VHOST_SCSI_VQ_CTL ].handle_kick = vhost_scsi_ctl_handle_kick; s->vqs[VHOST_SCSI_VQ_EVT ].handle_kick = vhost_scsi_evt_handle_kick;
@@ -1163,6 +1324,42 @@ static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport)return "Unknown" ; } +static int tcm_vhost_do_plug (struct tcm_vhost_tpg *tpg, +struct se_lun *lun,bool plug) +{ + +struct vhost_scsi *vs = tpg->vhost_scsi; +u32 reason; + +mutex_lock (&tpg->tv_tpg_mutex); + vs = tpg->vhost_scsi; +mutex_unlock (&tpg->tv_tpg_mutex); +if (!vs) +return -EOPNOTSUPP ; + +if (!tcm_vhost_check_feature (vs,VIRTIO_SCSI_F_HOTPLUG )) +return -EOPNOTSUPP ; + +if (plug) + reason =VIRTIO_SCSI_EVT_RESET_RESCAN ; +else + reason =VIRTIO_SCSI_EVT_RESET_REMOVED ; + +return tcm_vhost_send_evt (vs, tpg, lun, +VIRTIO_SCSI_T_TRANSPORT_RESET , + reason); +} + +static int tcm_vhost_hotplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, true); +} + +static int tcm_vhost_hotunplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, false); +} +static int tcm_vhost_port_link (struct se_portal_group *se_tpg,struct se_lun *lun) {
@@ -1173,18 +1370,21 @@ static int tcm_vhost_port_link(struct se_portal_group *se_tpg, tv_tpg->tv_tpg_port_count++;mutex_unlock (&tv_tpg->tv_tpg_mutex); +tcm_vhost_hotplug (tv_tpg, lun); +return 0 ; }static void tcm_vhost_port_unlink (struct se_portal_group *se_tpg, -struct se_lun *se_lun) +struct se_lun *lun) {struct tcm_vhost_tpg *tv_tpg =container_of (se_tpg, struct tcm_vhost_tpg, se_tpg); -mutex_lock (&tv_tpg->tv_tpg_mutex); tv_tpg->tv_tpg_port_count--;mutex_unlock (&tv_tpg->tv_tpg_mutex); + +tcm_vhost_hotunplug (tv_tpg, lun); } static struct se_node_acl *tcm_vhost_make_nodeacl(
drivers/vhost/tcm_vhost.h +10 -0
@@ -53,6 +53,7 @@ struct tcm_vhost_nacl {struct se_node_acl se_node_acl; }; +struct vhost_scsi ;struct tcm_vhost_tpg {/* Vhost port target portal group tag for TCM */ u16 tport_tpgt;
@@ -70,6 +71,8 @@ struct tcm_vhost_tpg {struct tcm_vhost_tport *tport;/* Returned by tcm_vhost_make_tpg() */ struct se_portal_group se_tpg; +/* Pointer back to struct vhost_scsi, protected by tv_tpg_mutex */ +struct vhost_scsi *vhost_scsi; };struct tcm_vhost_tport {
@@ -83,6 +86,13 @@ struct tcm_vhost_tport {struct se_wwn tport_wwn; }; +struct tcm_vhost_evt { +/* virtio_scsi event */ +struct virtio_scsi_event event; +/* virtio_scsi event list, serviced from vhost worker thread */ +struct llist_node list; +}; +/* * As per request from MST, keep TCM_VHOST related ioctl defines out of * linux/vhost.h (user-space) for now..
Signature · 1 line
1.8.1.4
PATCH v7 3/3 tcm_vhost: Add hotplug/hotunplug support
In reply to Asias HeIn commit 365a7150094 ([SCSI] virtio-scsi: hotplug support for virtio-scsi), hotplug support is added to virtio-scsi.
This patch adds hotplug and hotunplug support to tcm_vhost.
You can create or delete a LUN in targetcli to hotplug or hotunplug a LUN in guest.
Changes in v7:
- Add vhost_work_flush for vs->vs_event_work to this series
Changes in v6:
- Pass tcm_vhost_evt to tcm_vhost_do_evt_work
Changes in v5:
- Switch to int from u64 to vs_events_nr
- Set s->vs_events_dropped flag in tcm_vhost_allocate_evt
- Do not nest dev mutex within vq mutex
- Use vs_events_lock to protect vs_events_dropped and vs_events_nr
- Rebase to target/master
Changes in v4:
- Drop tcm_vhost_check_endpoint in tcm_vhost_send_evt
- Add tcm_vhost_check_endpoint in vhost_scsi_evt_handle_kick
Changes in v3:
- Separate the bug fix to another thread
Changes in v2:
- Remove code duplication in tcm_vhost_{hotplug,hotunplug}
- Fix racing of vs_events_nr
- Add flush fix patch to this series
--- drivers/vhost/tcm_vhost.c | 208 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 +++ 2 files changed, 214 insertions(+), 4 deletions(-)
drivers/vhost/tcm_vhost.c +204 -4
@@ -66,11 +66,13 @@ enum {* TODO: debug and remove the workaround. */ enum { -VHOST_SCSI_FEATURES =VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX ) +VHOST_SCSI_FEATURES = (VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX )) | + (1ULL <<VIRTIO_SCSI_F_HOTPLUG ) };#define VHOST_SCSI_MAX_TARGET 256#define VHOST_SCSI_MAX_VQ 128 +#define VHOST_SCSI_MAX_EVENT 128struct vhost_scsi {/* Protected by vhost_scsi->dev.mutex */
@@ -82,6 +84,13 @@ struct vhost_scsi {struct vhost_work vs_completion_work;/* cmd completion work item */ struct llist_head vs_completion_list;/* cmd completion queue */ + +struct vhost_work vs_event_work;/* evt injection work item */ +struct llist_head vs_event_list;/* evt injection queue */ + +struct mutex vs_events_lock;/* protect vs_events_dropped,events_nr */ +bool vs_events_dropped;/* any missed events */ +int vs_events_nr;/* num of pending events */ };/* Local pointer to allocated TCM configfs fabric module */
@@ -129,6 +138,17 @@ static bool tcm_vhost_check_endpoint(struct vhost_virtqueue *vq)return ret; } +static bool tcm_vhost_check_events_dropped (struct vhost_scsi *vs) +{ +bool ret; + +mutex_lock (&vs->vs_events_lock); + ret = vs->vs_events_dropped; +mutex_unlock (&vs->vs_events_lock); + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;
@@ -379,6 +399,37 @@ static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd)return 0 ; } +static void tcm_vhost_free_evt (struct vhost_scsi *vs,struct tcm_vhost_evt *evt) +{ +mutex_lock (&vs->vs_events_lock); + vs->vs_events_nr--; +kfree (evt); +mutex_unlock (&vs->vs_events_lock); +} + +static struct tcm_vhost_evt *tcm_vhost_allocate_evt (struct vhost_scsi *vs, +u32 event,u32 reason) +{ +struct tcm_vhost_evt *evt; + +mutex_lock (&vs->vs_events_lock); +if (vs->vs_events_nr >VHOST_SCSI_MAX_EVENT ) { + vs->vs_events_dropped = true; +mutex_unlock (&vs->vs_events_lock); +return NULL ; + } + + evt =kzalloc (sizeof (*evt),GFP_KERNEL ); +if (evt) { + evt->event.event = event; + evt->event.reason = reason; + vs->vs_events_nr++; + } +mutex_unlock (&vs->vs_events_lock); + +return evt; +} +static void vhost_scsi_free_cmd (struct tcm_vhost_cmd *tv_cmd) {struct se_cmd *se_cmd = &tv_cmd->tvc_se_cmd;
@@ -397,6 +448,75 @@ static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd)kfree (tv_cmd); } +static void tcm_vhost_do_evt_work (struct vhost_scsi *vs, +struct tcm_vhost_evt *evt) +{ +struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT ]; +struct virtio_scsi_event *event = &evt->event; +struct virtio_scsi_event __user *eventp; +unsigned out, in; +int head, ret; + +if (!tcm_vhost_check_endpoint (vq)) +return ; + +mutex_lock (&vs->vs_events_lock); +mutex_lock (&vq->mutex); +again: +vhost_disable_notify (&vs->dev, vq); + head =vhost_get_vq_desc (&vs->dev, vq, vq->iov, +ARRAY_SIZE (vq->iov), &out, &in, +NULL ,NULL ); +if (head <0 ) { + vs->vs_events_dropped = true; + goto out; + } +if (head == vq->num) { +if (vhost_enable_notify (&vs->dev, vq)) + goto again; + vs->vs_events_dropped = true; + goto out; + } + +if ((vq->iov[out].iov_len !=sizeof (struct virtio_scsi_event ))) { +vq_err (vq,"Expecting virtio_scsi_event, got %zu bytes\n" , + vq->iov[out].iov_len); + vs->vs_events_dropped = true; + goto out; + } + +if (vs->vs_events_dropped) { + event->event |=VIRTIO_SCSI_T_EVENTS_MISSED ; + vs->vs_events_dropped = false; + } + + eventp = vq->iov[out].iov_base; + ret =__copy_to_user (eventp, event,sizeof (*event)); +if (!ret) +vhost_add_used_and_signal (&vs->dev, vq, head,0 ); +else +vq_err (vq,"Faulted on tcm_vhost_send_event\n" ); +out: +mutex_unlock (&vq->mutex); +mutex_unlock (&vs->vs_events_lock); +} + +static void tcm_vhost_evt_work (struct vhost_work *work) +{ + struct vhost_scsi *vs = container_of(work, struct vhost_scsi, + vs_event_work); +struct tcm_vhost_evt *evt; +struct llist_node *llnode; + + llnode =llist_del_all (&vs->vs_event_list); +while (llnode) { + evt = llist_entry(llnode, struct tcm_vhost_evt, list); + llnode =llist_next (llnode); +tcm_vhost_do_evt_work (vs, evt); +tcm_vhost_free_evt (vs, evt); + } +} +/* Fill in status and signal that we are done processing this command * * This is scheduled in the vhost work queue so we are called with the owner
@@ -807,9 +927,42 @@ static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)pr_debug ("%s: The handling func for control queue.\n" , __func__); } +static int tcm_vhost_send_evt (struct vhost_scsi *vs,struct tcm_vhost_tpg *tpg, +struct se_lun *lun,u32 event,u32 reason) +{ +struct tcm_vhost_evt *evt; + + evt =tcm_vhost_allocate_evt (vs, event, reason); +if (!evt) +return -ENOMEM ; + +if (tpg && lun) { +/* TODO: share lun setup code with virtio-scsi.ko */ + evt->event.lun[0 ] =0x01 ; + evt->event.lun[1 ] = tpg->tport_tpgt &0xFF ; +if (lun->unpacked_lun >=256 ) + evt->event.lun[2 ] = lun->unpacked_lun >>8 |0x40 ; + evt->event.lun[3 ] = lun->unpacked_lun &0xFF ; + } + +llist_add (&evt->list, &vs->vs_event_list); +vhost_work_queue (&vs->dev, &vs->vs_event_work); + +return 0 ; +} +static void vhost_scsi_evt_handle_kick (struct vhost_work *work) { -pr_debug ("%s: The handling func for event queue.\n" , __func__); + struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, + poll.work); + struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev); + +if (!tcm_vhost_check_endpoint (vq)) +return ; + +if (tcm_vhost_check_events_dropped (vs)) +tcm_vhost_send_evt (vs,NULL ,NULL ,VIRTIO_SCSI_T_NO_EVENT ,0 ); + }static void vhost_scsi_handle_kick (struct vhost_work *work)
@@ -833,6 +986,7 @@ static void vhost_scsi_flush(struct vhost_scsi *vs)for (i =0 ; i <VHOST_SCSI_MAX_VQ ; i++)vhost_scsi_flush_vq (vs, i);vhost_work_flush (&vs->dev, &vs->vs_completion_work); +vhost_work_flush (&vs->dev, &vs->vs_event_work); }/*
@@ -891,6 +1045,7 @@ static int vhost_scsi_set_endpoint(return -EEXIST ; } tv_tpg->tv_tpg_vhost_count++; + tv_tpg->vhost_scsi = vs; vs_tpg[tv_tpg->tport_tpgt] = tv_tpg;smp_mb__after_atomic_inc (); match = true;
@@ -974,6 +1129,7 @@ static int vhost_scsi_clear_endpoint( goto err_tpg; } tv_tpg->tv_tpg_vhost_count--; + tv_tpg->vhost_scsi =NULL ; vs->vs_tpg[target] =NULL ; match = true;mutex_unlock (&tv_tpg->tv_tpg_mutex);
@@ -1033,6 +1189,11 @@ static int vhost_scsi_open(struct inode *inode, struct file *f)return -ENOMEM ;vhost_work_init (&s->vs_completion_work, vhost_scsi_complete_cmd_work); +vhost_work_init (&s->vs_event_work, tcm_vhost_evt_work); + + s->vs_events_nr =0 ; + s->vs_events_dropped = false; +mutex_init (&s->vs_events_lock); s->vqs[VHOST_SCSI_VQ_CTL ].handle_kick = vhost_scsi_ctl_handle_kick; s->vqs[VHOST_SCSI_VQ_EVT ].handle_kick = vhost_scsi_evt_handle_kick;
@@ -1163,6 +1324,42 @@ static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport)return "Unknown" ; } +static int tcm_vhost_do_plug (struct tcm_vhost_tpg *tpg, +struct se_lun *lun,bool plug) +{ + +struct vhost_scsi *vs = tpg->vhost_scsi; +u32 reason; + +mutex_lock (&tpg->tv_tpg_mutex); + vs = tpg->vhost_scsi; +mutex_unlock (&tpg->tv_tpg_mutex); +if (!vs) +return -EOPNOTSUPP ; + +if (!tcm_vhost_check_feature (vs,VIRTIO_SCSI_F_HOTPLUG )) +return -EOPNOTSUPP ; + +if (plug) + reason =VIRTIO_SCSI_EVT_RESET_RESCAN ; +else + reason =VIRTIO_SCSI_EVT_RESET_REMOVED ; + +return tcm_vhost_send_evt (vs, tpg, lun, +VIRTIO_SCSI_T_TRANSPORT_RESET , + reason); +} + +static int tcm_vhost_hotplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, true); +} + +static int tcm_vhost_hotunplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, false); +} +static int tcm_vhost_port_link (struct se_portal_group *se_tpg,struct se_lun *lun) {
@@ -1173,18 +1370,21 @@ static int tcm_vhost_port_link(struct se_portal_group *se_tpg, tv_tpg->tv_tpg_port_count++;mutex_unlock (&tv_tpg->tv_tpg_mutex); +tcm_vhost_hotplug (tv_tpg, lun); +return 0 ; }static void tcm_vhost_port_unlink (struct se_portal_group *se_tpg, -struct se_lun *se_lun) +struct se_lun *lun) {struct tcm_vhost_tpg *tv_tpg =container_of (se_tpg, struct tcm_vhost_tpg, se_tpg); -mutex_lock (&tv_tpg->tv_tpg_mutex); tv_tpg->tv_tpg_port_count--;mutex_unlock (&tv_tpg->tv_tpg_mutex); + +tcm_vhost_hotunplug (tv_tpg, lun); } static struct se_node_acl *tcm_vhost_make_nodeacl(
drivers/vhost/tcm_vhost.h +10 -0
@@ -53,6 +53,7 @@ struct tcm_vhost_nacl {struct se_node_acl se_node_acl; }; +struct vhost_scsi ;struct tcm_vhost_tpg {/* Vhost port target portal group tag for TCM */ u16 tport_tpgt;
@@ -70,6 +71,8 @@ struct tcm_vhost_tpg {struct tcm_vhost_tport *tport;/* Returned by tcm_vhost_make_tpg() */ struct se_portal_group se_tpg; +/* Pointer back to struct vhost_scsi, protected by tv_tpg_mutex */ +struct vhost_scsi *vhost_scsi; };struct tcm_vhost_tport {
@@ -83,6 +86,13 @@ struct tcm_vhost_tport {struct se_wwn tport_wwn; }; +struct tcm_vhost_evt { +/* virtio_scsi event */ +struct virtio_scsi_event event; +/* virtio_scsi event list, serviced from vhost worker thread */ +struct llist_node list; +}; +/* * As per request from MST, keep TCM_VHOST related ioctl defines out of * linux/vhost.h (user-space) for now..
Signature · 1 line
1.8.1.4
Michael S. Tsirkin
maintainerRe: [PATCH v7 3/3] tcm_vhost: Add hotplug/hotunplug support
Quoted 60 lines · Asias He
In commit
365a7150094([SCSI] virtio-scsi: hotplug support for virtio-scsi), hotplug support is added to virtio-scsi.This patch adds hotplug and hotunplug support to
tcm_vhost.You can create or delete a LUN in targetcli to hotplug or hotunplug a LUN in guest.
Changes in v7:
- Addvhost_work_flushforvs->vs_event_workto this seriesChanges in v6:
- Passtcm_vhost_evttotcm_vhost_do_evt_workChanges in v5:
- Switch to int from u64 tovs_events_nr
- Sets->vs_events_droppedflag intcm_vhost_allocate_evt
- Do not nest dev mutex within vq mutex
- Usevs_events_lockto protectvs_events_droppedandvs_events_nr
- Rebase to target/masterChanges in v4:
- Droptcm_vhost_check_endpointintcm_vhost_send_evt
- Addtcm_vhost_check_endpointinvhost_scsi_evt_handle_kickChanges in v3:
- Separate the bug fix to another threadChanges in v2:
- Remove code duplication in tcm_vhost_{hotplug,hotunplug}
- Fix racing ofvs_events_nr
- Add flush fix patch to this seriesSigned-off-by Asias He <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- drivers/vhost/tcm_vhost.c | 208 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 +++ 2 files changed, 214 insertions(+), 4 deletions(-)drivers/vhost/tcm_vhost.c +3
-1@@ -66,11 +66,13 @@ enum {* TODO: debug and remove the workaround. */ enum { -VHOST_SCSI_FEATURES =VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX ) +VHOST_SCSI_FEATURES = (VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX )) | + (1ULL <<VIRTIO_SCSI_F_HOTPLUG ) };#define VHOST_SCSI_MAX_TARGET 256#define VHOST_SCSI_MAX_VQ 128 +#define VHOST_SCSI_MAX_EVENT 128struct vhost_scsi {/* Protected by vhost_scsi->dev.mutex */
drivers/vhost/tcm_vhost.c struct vhost_scsi { · line 91
Quoted 8 lines
drivers/vhost/tcm_vhost.c +5
-0@@ -82,6 +84,13 @@ struct vhost_scsi {struct vhost_work vs_completion_work;/* cmd completion work item */ struct llist_head vs_completion_list;/* cmd completion queue */ + +struct vhost_work vs_event_work;/* evt injection work item */ +struct llist_head vs_event_list;/* evt injection queue */ + +struct mutex vs_events_lock;/* protect vs_events_dropped,events_nr */
Looking at this code, there are just so many locks now. This does not make me feel safe :)
At least, please document lock nesting.
Quoted 38 lines
drivers/vhost/tcm_vhost.c +24
-0+bool vs_events_dropped;/* any missed events */ +int vs_events_nr;/* num of pending events */ };/* Local pointer to allocated TCM configfs fabric module */ @@ -129,6 +138,17 @@ static bool tcm_vhost_check_endpoint(struct vhost_virtqueue *vq)return ret; } +static bool tcm_vhost_check_events_dropped (struct vhost_scsi *vs) +{ +bool ret; + +mutex_lock (&vs->vs_events_lock); + ret = vs->vs_events_dropped; +mutex_unlock (&vs->vs_events_lock); + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;@@ -379,6 +399,37 @@ static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd)return 0 ; } +static void tcm_vhost_free_evt (struct vhost_scsi *vs,struct tcm_vhost_evt *evt) +{ +mutex_lock (&vs->vs_events_lock); + vs->vs_events_nr--; +kfree (evt); +mutex_unlock (&vs->vs_events_lock); +} + +static struct tcm_vhost_evt *tcm_vhost_allocate_evt (struct vhost_scsi *vs, +u32 event,u32 reason) +{
drivers/vhost/tcm_vhost.c static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd) · line 422
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+struct tcm_vhost_evt *evt; + +mutex_lock (&vs->vs_events_lock); +if (vs->vs_events_nr >VHOST_SCSI_MAX_EVENT ) { + vs->vs_events_dropped = true; +mutex_unlock (&vs->vs_events_lock); +return NULL ; + } + + evt =kzalloc (sizeof (*evt),GFP_KERNEL );
I think kzalloc not needed here, you init all fields.
Also, this basically means if backend does plug/unplug very quickly, we start consuming kernel memory without a limit. Not good. How about we allocate the event as part of target?
There shouldn't ever be more than one hotplug
event in flight per target, right?
drivers/vhost/tcm_vhost.c static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd) · line 423
Quoted 1 line
drivers/vhost/tcm_vhost.c +1
-0+if (evt) {
Let's do clean error handling here and above:
if (!evt)
goto err;
Quoted 32 lines
drivers/vhost/tcm_vhost.c +25
-0+ evt->event.event = event; + evt->event.reason = reason; + vs->vs_events_nr++; + } +mutex_unlock (&vs->vs_events_lock); + +return evt; +} +static void vhost_scsi_free_cmd (struct tcm_vhost_cmd *tv_cmd) {struct se_cmd *se_cmd = &tv_cmd->tvc_se_cmd;@@ -397,6 +448,75 @@ static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd)kfree (tv_cmd); } +static void tcm_vhost_do_evt_work (struct vhost_scsi *vs, +struct tcm_vhost_evt *evt) +{ +struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT ]; +struct virtio_scsi_event *event = &evt->event; +struct virtio_scsi_event __user *eventp; +unsigned out, in; +int head, ret; + +if (!tcm_vhost_check_endpoint (vq)) +return ; + +mutex_lock (&vs->vs_events_lock); +mutex_lock (&vq->mutex); +again: +vhost_disable_notify (&vs->dev, vq);
drivers/vhost/tcm_vhost.c static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd) · line 476
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+ head =vhost_get_vq_desc (&vs->dev, vq, vq->iov, +ARRAY_SIZE (vq->iov), &out, &in, +NULL ,NULL ); +if (head <0 ) { + vs->vs_events_dropped = true; + goto out; + } +if (head == vq->num) { +if (vhost_enable_notify (&vs->dev, vq)) + goto again;
Please write loops with while() or for().
Not with goto. goto is for error handling.
Quoted 25 lines
drivers/vhost/tcm_vhost.c +25
-0+ vs->vs_events_dropped = true; + goto out; + } + +if ((vq->iov[out].iov_len !=sizeof (struct virtio_scsi_event ))) { +vq_err (vq,"Expecting virtio_scsi_event, got %zu bytes\n" , + vq->iov[out].iov_len); + vs->vs_events_dropped = true; + goto out; + } + +if (vs->vs_events_dropped) { + event->event |=VIRTIO_SCSI_T_EVENTS_MISSED ; + vs->vs_events_dropped = false; + } + + eventp = vq->iov[out].iov_base; + ret =__copy_to_user (eventp, event,sizeof (*event)); +if (!ret) +vhost_add_used_and_signal (&vs->dev, vq, head,0 ); +else +vq_err (vq,"Faulted on tcm_vhost_send_event\n" ); +out: +mutex_unlock (&vq->mutex); +mutex_unlock (&vs->vs_events_lock);
drivers/vhost/tcm_vhost.c static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd) · line 511
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+} + +static void tcm_vhost_evt_work (struct vhost_work *work) +{ + struct vhost_scsi *vs = container_of(work, struct vhost_scsi, + vs_event_work); +struct tcm_vhost_evt *evt; +struct llist_node *llnode; + + llnode =llist_del_all (&vs->vs_event_list);
The assumption is that this is slow path thing, no need to worry about speed, yes? so let's do simple list_add, list_del etc.
Quoted 38 lines
drivers/vhost/tcm_vhost.c +31
-0+while (llnode) { + evt = llist_entry(llnode, struct tcm_vhost_evt, list); + llnode =llist_next (llnode); +tcm_vhost_do_evt_work (vs, evt); +tcm_vhost_free_evt (vs, evt); + } +} +/* Fill in status and signal that we are done processing this command * * This is scheduled in the vhost work queue so we are called with the owner @@ -807,9 +927,42 @@ static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)pr_debug ("%s: The handling func for control queue.\n" , __func__); } +static int tcm_vhost_send_evt (struct vhost_scsi *vs,struct tcm_vhost_tpg *tpg, +struct se_lun *lun,u32 event,u32 reason) +{ +struct tcm_vhost_evt *evt; + + evt =tcm_vhost_allocate_evt (vs, event, reason); +if (!evt) +return -ENOMEM ; + +if (tpg && lun) { +/* TODO: share lun setup code with virtio-scsi.ko */ + evt->event.lun[0 ] =0x01 ; + evt->event.lun[1 ] = tpg->tport_tpgt &0xFF ; +if (lun->unpacked_lun >=256 ) + evt->event.lun[2 ] = lun->unpacked_lun >>8 |0x40 ; + evt->event.lun[3 ] = lun->unpacked_lun &0xFF ; + } + +llist_add (&evt->list, &vs->vs_event_list); +vhost_work_queue (&vs->dev, &vs->vs_event_work); + +return 0 ; +}
drivers/vhost/tcm_vhost.c static void vhost_scsi_ctl_handle_kick(struct vhost_work *work) · line 961
Quoted 10 lines
drivers/vhost/tcm_vhost.c +7
-1+static void vhost_scsi_evt_handle_kick (struct vhost_work *work) { -pr_debug ("%s: The handling func for event queue.\n" , __func__); + struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, + poll.work); + struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev); + +if (!tcm_vhost_check_endpoint (vq)) +return ;
Again just drop this check.
Quoted 49 lines
drivers/vhost/tcm_vhost.c +14
-0+ +if (tcm_vhost_check_events_dropped (vs)) +tcm_vhost_send_evt (vs,NULL ,NULL ,VIRTIO_SCSI_T_NO_EVENT ,0 ); + }static void vhost_scsi_handle_kick (struct vhost_work *work)@@ -833,6 +986,7 @@ static void vhost_scsi_flush(struct vhost_scsi *vs)for (i =0 ; i <VHOST_SCSI_MAX_VQ ; i++)vhost_scsi_flush_vq (vs, i);vhost_work_flush (&vs->dev, &vs->vs_completion_work); +vhost_work_flush (&vs->dev, &vs->vs_event_work); }/* @@ -891,6 +1045,7 @@ static int vhost_scsi_set_endpoint(return -EEXIST ; } tv_tpg->tv_tpg_vhost_count++; + tv_tpg->vhost_scsi = vs; vs_tpg[tv_tpg->tport_tpgt] = tv_tpg;smp_mb__after_atomic_inc (); match = true;@@ -974,6 +1129,7 @@ static int vhost_scsi_clear_endpoint( goto err_tpg; } tv_tpg->tv_tpg_vhost_count--; + tv_tpg->vhost_scsi =NULL ; vs->vs_tpg[target] =NULL ; match = true;mutex_unlock (&tv_tpg->tv_tpg_mutex);@@ -1033,6 +1189,11 @@ static int vhost_scsi_open(struct inode *inode, struct file *f)return -ENOMEM ;vhost_work_init (&s->vs_completion_work, vhost_scsi_complete_cmd_work); +vhost_work_init (&s->vs_event_work, tcm_vhost_evt_work); + + s->vs_events_nr =0 ; + s->vs_events_dropped = false; +mutex_init (&s->vs_events_lock); s->vqs[VHOST_SCSI_VQ_CTL ].handle_kick = vhost_scsi_ctl_handle_kick; s->vqs[VHOST_SCSI_VQ_EVT ].handle_kick = vhost_scsi_evt_handle_kick;@@ -1163,6 +1324,42 @@ static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport)return "Unknown" ; } +static int tcm_vhost_do_plug (struct tcm_vhost_tpg *tpg, +struct se_lun *lun,bool plug)
drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1338
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+{ + +struct vhost_scsi *vs = tpg->vhost_scsi; +u32 reason; + +mutex_lock (&tpg->tv_tpg_mutex); + vs = tpg->vhost_scsi; +mutex_unlock (&tpg->tv_tpg_mutex); +if (!vs) +return -EOPNOTSUPP ;
Why EOPNOTSUPP? When does this happen?
drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1347
Quoted 9 lines
drivers/vhost/tcm_vhost.c +9
-0+ +if (!tcm_vhost_check_feature (vs,VIRTIO_SCSI_F_HOTPLUG )) +return -EOPNOTSUPP ; + +if (plug) + reason =VIRTIO_SCSI_EVT_RESET_RESCAN ; +else + reason =VIRTIO_SCSI_EVT_RESET_REMOVED ; +
You have dropped the tpg lock so tpg->vhost_scsi can become NULL now. Why is this safe?
Quoted 5 lines
drivers/vhost/tcm_vhost.c +5
-0+return tcm_vhost_send_evt (vs, tpg, lun, +VIRTIO_SCSI_T_TRANSPORT_RESET , + reason); +} +
drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1362
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+static int tcm_vhost_hotplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, true); +} + +static int tcm_vhost_hotunplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, false); +} +
What are these wrappers for? Seem useless ...
Quoted 64 lines
drivers/vhost/tcm_vhost.c +5
-2static int tcm_vhost_port_link (struct se_portal_group *se_tpg,struct se_lun *lun) {@@ -1173,18 +1370,21 @@ static int tcm_vhost_port_link(struct se_portal_group *se_tpg, tv_tpg->tv_tpg_port_count++;mutex_unlock (&tv_tpg->tv_tpg_mutex); +tcm_vhost_hotplug (tv_tpg, lun); +return 0 ; }static void tcm_vhost_port_unlink (struct se_portal_group *se_tpg, -struct se_lun *se_lun) +struct se_lun *lun) {struct tcm_vhost_tpg *tv_tpg =container_of (se_tpg, struct tcm_vhost_tpg, se_tpg); -mutex_lock (&tv_tpg->tv_tpg_mutex); tv_tpg->tv_tpg_port_count--;mutex_unlock (&tv_tpg->tv_tpg_mutex); + +tcm_vhost_hotunplug (tv_tpg, lun); } static struct se_node_acl *tcm_vhost_make_nodeacl(drivers/vhost/tcm_vhost.h +10
-0@@ -53,6 +53,7 @@ struct tcm_vhost_nacl {struct se_node_acl se_node_acl; }; +struct vhost_scsi ;struct tcm_vhost_tpg {/* Vhost port target portal group tag for TCM */ u16 tport_tpgt;@@ -70,6 +71,8 @@ struct tcm_vhost_tpg {struct tcm_vhost_tport *tport;/* Returned by tcm_vhost_make_tpg() */ struct se_portal_group se_tpg; +/* Pointer back to struct vhost_scsi, protected by tv_tpg_mutex */ +struct vhost_scsi *vhost_scsi; };struct tcm_vhost_tport {@@ -83,6 +86,13 @@ struct tcm_vhost_tport {struct se_wwn tport_wwn; }; +struct tcm_vhost_evt { +/* virtio_scsi event */ +struct virtio_scsi_event event; +/* virtio_scsi event list, serviced from vhost worker thread */ +struct llist_node list; +}; +/* * As per request from MST, keep TCM_VHOST related ioctl defines out of * linux/vhost.h (user-space) for now.. Signature · 1 line
1.8.1.4
Re: [PATCH v7 3/3] tcm_vhost: Add hotplug/hotunplug support
Quoted 62 lines · Michael S. Tsirkin
Quoted 61 lines · Asias He
In commit
365a7150094([SCSI] virtio-scsi: hotplug support for virtio-scsi), hotplug support is added to virtio-scsi.This patch adds hotplug and hotunplug support to
tcm_vhost.You can create or delete a LUN in targetcli to hotplug or hotunplug a LUN in guest.
Changes in v7:
- Addvhost_work_flushforvs->vs_event_workto this seriesChanges in v6:
- Passtcm_vhost_evttotcm_vhost_do_evt_workChanges in v5:
- Switch to int from u64 tovs_events_nr
- Sets->vs_events_droppedflag intcm_vhost_allocate_evt
- Do not nest dev mutex within vq mutex
- Usevs_events_lockto protectvs_events_droppedandvs_events_nr
- Rebase to target/masterChanges in v4:
- Droptcm_vhost_check_endpointintcm_vhost_send_evt
- Addtcm_vhost_check_endpointinvhost_scsi_evt_handle_kickChanges in v3:
- Separate the bug fix to another threadChanges in v2:
- Remove code duplication in tcm_vhost_{hotplug,hotunplug}
- Fix racing ofvs_events_nr
- Add flush fix patch to this seriesSigned-off-by Asias He <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- drivers/vhost/tcm_vhost.c | 208 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 +++ 2 files changed, 214 insertions(+), 4 deletions(-)drivers/vhost/tcm_vhost.c +3
-1@@ -66,11 +66,13 @@ enum {* TODO: debug and remove the workaround. */ enum { -VHOST_SCSI_FEATURES =VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX ) +VHOST_SCSI_FEATURES = (VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX )) | + (1ULL <<VIRTIO_SCSI_F_HOTPLUG ) };#define VHOST_SCSI_MAX_TARGET 256#define VHOST_SCSI_MAX_VQ 128 +#define VHOST_SCSI_MAX_EVENT 128struct vhost_scsi {/* Protected by vhost_scsi->dev.mutex */ @@ -82,6 +84,13 @@ struct vhost_scsi {
Quoted 12 lines
drivers/vhost/tcm_vhost.c struct vhost_scsi { · line 91
Quoted 7 lines
drivers/vhost/tcm_vhost.c +5
-0struct vhost_work vs_completion_work;/* cmd completion work item */ struct llist_head vs_completion_list;/* cmd completion queue */ + +struct vhost_work vs_event_work;/* evt injection work item */ +struct llist_head vs_event_list;/* evt injection queue */ + +struct mutex vs_events_lock;/* protect vs_events_dropped,events_nr */ Looking at this code, there are just so many locks now. This does not make me feel safe :)
At least, please document lock nesting.
Do you see a real problem?
Quoted 42 lines
Quoted 42 lines
drivers/vhost/tcm_vhost.c +28
-0+bool vs_events_dropped;/* any missed events */ +int vs_events_nr;/* num of pending events */ };/* Local pointer to allocated TCM configfs fabric module */ @@ -129,6 +138,17 @@ static bool tcm_vhost_check_endpoint(struct vhost_virtqueue *vq)return ret; } +static bool tcm_vhost_check_events_dropped (struct vhost_scsi *vs) +{ +bool ret; + +mutex_lock (&vs->vs_events_lock); + ret = vs->vs_events_dropped; +mutex_unlock (&vs->vs_events_lock); + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;@@ -379,6 +399,37 @@ static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd)return 0 ; } +static void tcm_vhost_free_evt (struct vhost_scsi *vs,struct tcm_vhost_evt *evt) +{ +mutex_lock (&vs->vs_events_lock); + vs->vs_events_nr--; +kfree (evt); +mutex_unlock (&vs->vs_events_lock); +} + +static struct tcm_vhost_evt *tcm_vhost_allocate_evt (struct vhost_scsi *vs, +u32 event,u32 reason) +{ +struct tcm_vhost_evt *evt; + +mutex_lock (&vs->vs_events_lock); +if (vs->vs_events_nr >VHOST_SCSI_MAX_EVENT ) {
Quoted 8 lines
drivers/vhost/tcm_vhost.c static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd) · line 422
Quoted 6 lines
drivers/vhost/tcm_vhost.c +6
-0+ vs->vs_events_dropped = true; +mutex_unlock (&vs->vs_events_lock); +return NULL ; + } + + evt =kzalloc (sizeof (*evt),GFP_KERNEL );I think kzalloc not needed here, you init all fields.
Not really! evt->event.lun[4-7] is not initialized. It needs to be 0.
Quoted 5 lines
Also, this basically means if backend does plug/unplug very quickly, we start consuming kernel memory without a limit. Not good. How about we allocate the event as part of target?
There shouldn't ever be more than one hotplug
event in flight per target, right?
It is limited by VHOST_SCSI_MAX_EVENT.
Quoted 6 lines
drivers/vhost/tcm_vhost.c static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd) · line 423
Quoted 2 lines
drivers/vhost/tcm_vhost.c +1
-0+if (evt) {Let's do clean error handling here and above:
if (!evt)goto err;
We can you do in err? You simply unlock and return. Why bother? How cleaner it will be.
Quoted 37 lines
Quoted 37 lines
drivers/vhost/tcm_vhost.c +30
-0+ evt->event.event = event; + evt->event.reason = reason; + vs->vs_events_nr++; + } +mutex_unlock (&vs->vs_events_lock); + +return evt; +} +static void vhost_scsi_free_cmd (struct tcm_vhost_cmd *tv_cmd) {struct se_cmd *se_cmd = &tv_cmd->tvc_se_cmd;@@ -397,6 +448,75 @@ static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd)kfree (tv_cmd); } +static void tcm_vhost_do_evt_work (struct vhost_scsi *vs, +struct tcm_vhost_evt *evt) +{ +struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT ]; +struct virtio_scsi_event *event = &evt->event; +struct virtio_scsi_event __user *eventp; +unsigned out, in; +int head, ret; + +if (!tcm_vhost_check_endpoint (vq)) +return ; + +mutex_lock (&vs->vs_events_lock); +mutex_lock (&vq->mutex); +again: +vhost_disable_notify (&vs->dev, vq); + head =vhost_get_vq_desc (&vs->dev, vq, vq->iov, +ARRAY_SIZE (vq->iov), &out, &in, +NULL ,NULL ); +if (head <0 ) { + vs->vs_events_dropped = true;
Quoted 8 lines
drivers/vhost/tcm_vhost.c static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd) · line 476
Quoted 5 lines
drivers/vhost/tcm_vhost.c +5
-0+ goto out; + } +if (head == vq->num) { +if (vhost_enable_notify (&vs->dev, vq)) + goto again;Please write loops with
while()orfor().
Not with goto. goto is for error handling.
This makes extra indention which is more ugly.
Quoted 30 lines
Quoted 30 lines
drivers/vhost/tcm_vhost.c +30
-0+ vs->vs_events_dropped = true; + goto out; + } + +if ((vq->iov[out].iov_len !=sizeof (struct virtio_scsi_event ))) { +vq_err (vq,"Expecting virtio_scsi_event, got %zu bytes\n" , + vq->iov[out].iov_len); + vs->vs_events_dropped = true; + goto out; + } + +if (vs->vs_events_dropped) { + event->event |=VIRTIO_SCSI_T_EVENTS_MISSED ; + vs->vs_events_dropped = false; + } + + eventp = vq->iov[out].iov_base; + ret =__copy_to_user (eventp, event,sizeof (*event)); +if (!ret) +vhost_add_used_and_signal (&vs->dev, vq, head,0 ); +else +vq_err (vq,"Faulted on tcm_vhost_send_event\n" ); +out: +mutex_unlock (&vq->mutex); +mutex_unlock (&vs->vs_events_lock); +} + +static void tcm_vhost_evt_work (struct vhost_work *work) +{ + struct vhost_scsi *vs = container_of(work, struct vhost_scsi,
Quoted 8 lines
drivers/vhost/tcm_vhost.c static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd) · line 511
Quoted 5 lines
drivers/vhost/tcm_vhost.c +5
-0+ vs_event_work); +struct tcm_vhost_evt *evt; +struct llist_node *llnode; + + llnode =llist_del_all (&vs->vs_event_list);The assumption is that this is slow path thing, no need to worry about speed, yes? so let's do simple
list_add,list_deletc.
Why it is simpler? We are using llist for cmd. Why it is better using one for evt and the different for cmd? Similar code makes people easier to read.
Quoted 42 lines
Quoted 42 lines
drivers/vhost/tcm_vhost.c +32
-1+while (llnode) { + evt = llist_entry(llnode, struct tcm_vhost_evt, list); + llnode =llist_next (llnode); +tcm_vhost_do_evt_work (vs, evt); +tcm_vhost_free_evt (vs, evt); + } +} +/* Fill in status and signal that we are done processing this command * * This is scheduled in the vhost work queue so we are called with the owner @@ -807,9 +927,42 @@ static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)pr_debug ("%s: The handling func for control queue.\n" , __func__); } +static int tcm_vhost_send_evt (struct vhost_scsi *vs,struct tcm_vhost_tpg *tpg, +struct se_lun *lun,u32 event,u32 reason) +{ +struct tcm_vhost_evt *evt; + + evt =tcm_vhost_allocate_evt (vs, event, reason); +if (!evt) +return -ENOMEM ; + +if (tpg && lun) { +/* TODO: share lun setup code with virtio-scsi.ko */ + evt->event.lun[0 ] =0x01 ; + evt->event.lun[1 ] = tpg->tport_tpgt &0xFF ; +if (lun->unpacked_lun >=256 ) + evt->event.lun[2 ] = lun->unpacked_lun >>8 |0x40 ; + evt->event.lun[3 ] = lun->unpacked_lun &0xFF ; + } + +llist_add (&evt->list, &vs->vs_event_list); +vhost_work_queue (&vs->dev, &vs->vs_event_work); + +return 0 ; +} +static void vhost_scsi_evt_handle_kick (struct vhost_work *work) { -pr_debug ("%s: The handling func for event queue.\n" , __func__);
Quoted 8 lines
drivers/vhost/tcm_vhost.c static void vhost_scsi_ctl_handle_kick(struct vhost_work *work) · line 961
Quoted 6 lines
drivers/vhost/tcm_vhost.c +6
-0+ struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, + poll.work); + struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev); + +if (!tcm_vhost_check_endpoint (vq)) +return ;Again just drop this check.
Why?
Quoted 47 lines
Quoted 47 lines
drivers/vhost/tcm_vhost.c +12
-0+ +if (tcm_vhost_check_events_dropped (vs)) +tcm_vhost_send_evt (vs,NULL ,NULL ,VIRTIO_SCSI_T_NO_EVENT ,0 ); + }static void vhost_scsi_handle_kick (struct vhost_work *work)@@ -833,6 +986,7 @@ static void vhost_scsi_flush(struct vhost_scsi *vs)for (i =0 ; i <VHOST_SCSI_MAX_VQ ; i++)vhost_scsi_flush_vq (vs, i);vhost_work_flush (&vs->dev, &vs->vs_completion_work); +vhost_work_flush (&vs->dev, &vs->vs_event_work); }/* @@ -891,6 +1045,7 @@ static int vhost_scsi_set_endpoint(return -EEXIST ; } tv_tpg->tv_tpg_vhost_count++; + tv_tpg->vhost_scsi = vs; vs_tpg[tv_tpg->tport_tpgt] = tv_tpg;smp_mb__after_atomic_inc (); match = true;@@ -974,6 +1129,7 @@ static int vhost_scsi_clear_endpoint( goto err_tpg; } tv_tpg->tv_tpg_vhost_count--; + tv_tpg->vhost_scsi =NULL ; vs->vs_tpg[target] =NULL ; match = true;mutex_unlock (&tv_tpg->tv_tpg_mutex);@@ -1033,6 +1189,11 @@ static int vhost_scsi_open(struct inode *inode, struct file *f)return -ENOMEM ;vhost_work_init (&s->vs_completion_work, vhost_scsi_complete_cmd_work); +vhost_work_init (&s->vs_event_work, tcm_vhost_evt_work); + + s->vs_events_nr =0 ; + s->vs_events_dropped = false; +mutex_init (&s->vs_events_lock); s->vqs[VHOST_SCSI_VQ_CTL ].handle_kick = vhost_scsi_ctl_handle_kick; s->vqs[VHOST_SCSI_VQ_EVT ].handle_kick = vhost_scsi_evt_handle_kick;@@ -1163,6 +1324,42 @@ static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport)return "Unknown" ; }
Quoted 14 lines
drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1338
Quoted 12 lines
drivers/vhost/tcm_vhost.c +12
-0+static int tcm_vhost_do_plug (struct tcm_vhost_tpg *tpg, +struct se_lun *lun,bool plug) +{ + +struct vhost_scsi *vs = tpg->vhost_scsi; +u32 reason; + +mutex_lock (&tpg->tv_tpg_mutex); + vs = tpg->vhost_scsi; +mutex_unlock (&tpg->tv_tpg_mutex); +if (!vs) +return -EOPNOTSUPP ;Why EOPNOTSUPP? When does this happen?
When tpg->vhost_scsi has not been setup. E.g. no one starts a vhost-scsi guest.
Quoted 22 lines
drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1347
Quoted 9 lines
drivers/vhost/tcm_vhost.c +9
-0+ +if (!tcm_vhost_check_feature (vs,VIRTIO_SCSI_F_HOTPLUG )) +return -EOPNOTSUPP ; + +if (plug) + reason =VIRTIO_SCSI_EVT_RESET_RESCAN ; +else + reason =VIRTIO_SCSI_EVT_RESET_REMOVED ; +You have dropped the tpg lock so
tpg->vhost_scsican become NULL now. Why is this safe?Quoted 9 lines
drivers/vhost/tcm_vhost.c +9
-0+return tcm_vhost_send_evt (vs, tpg, lun, +VIRTIO_SCSI_T_TRANSPORT_RESET , + reason); +} + +static int tcm_vhost_hotplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, true); +}
Quoted 8 lines
drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1362
Quoted 6 lines
drivers/vhost/tcm_vhost.c +6
-0+ +static int tcm_vhost_hotunplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, false); +} +What are these wrappers for? Seem useless ...
It make people easier to understand what's the true and false is about in tcm_vhost_do_plug.
Quoted 64 lines
Quoted 64 lines
drivers/vhost/tcm_vhost.c +5
-2static int tcm_vhost_port_link (struct se_portal_group *se_tpg,struct se_lun *lun) {@@ -1173,18 +1370,21 @@ static int tcm_vhost_port_link(struct se_portal_group *se_tpg, tv_tpg->tv_tpg_port_count++;mutex_unlock (&tv_tpg->tv_tpg_mutex); +tcm_vhost_hotplug (tv_tpg, lun); +return 0 ; }static void tcm_vhost_port_unlink (struct se_portal_group *se_tpg, -struct se_lun *se_lun) +struct se_lun *lun) {struct tcm_vhost_tpg *tv_tpg =container_of (se_tpg, struct tcm_vhost_tpg, se_tpg); -mutex_lock (&tv_tpg->tv_tpg_mutex); tv_tpg->tv_tpg_port_count--;mutex_unlock (&tv_tpg->tv_tpg_mutex); + +tcm_vhost_hotunplug (tv_tpg, lun); } static struct se_node_acl *tcm_vhost_make_nodeacl(drivers/vhost/tcm_vhost.h +10
-0@@ -53,6 +53,7 @@ struct tcm_vhost_nacl {struct se_node_acl se_node_acl; }; +struct vhost_scsi ;struct tcm_vhost_tpg {/* Vhost port target portal group tag for TCM */ u16 tport_tpgt;@@ -70,6 +71,8 @@ struct tcm_vhost_tpg {struct tcm_vhost_tport *tport;/* Returned by tcm_vhost_make_tpg() */ struct se_portal_group se_tpg; +/* Pointer back to struct vhost_scsi, protected by tv_tpg_mutex */ +struct vhost_scsi *vhost_scsi; };struct tcm_vhost_tport {@@ -83,6 +86,13 @@ struct tcm_vhost_tport {struct se_wwn tport_wwn; }; +struct tcm_vhost_evt { +/* virtio_scsi event */ +struct virtio_scsi_event event; +/* virtio_scsi event list, serviced from vhost worker thread */ +struct llist_node list; +}; +/* * As per request from MST, keep TCM_VHOST related ioctl defines out of * linux/vhost.h (user-space) for now.. Signature · 1 line
1.8.1.4
Signature · 1 line
Asias
Michael S. Tsirkin
maintainerRe: [PATCH v7 3/3] tcm_vhost: Add hotplug/hotunplug support
Quoted 76 lines · Asias He
Quoted 62 lines · Michael S. Tsirkin
Quoted 61 lines · Asias He
In commit
365a7150094([SCSI] virtio-scsi: hotplug support for virtio-scsi), hotplug support is added to virtio-scsi.This patch adds hotplug and hotunplug support to
tcm_vhost.You can create or delete a LUN in targetcli to hotplug or hotunplug a LUN in guest.
Changes in v7:
- Addvhost_work_flushforvs->vs_event_workto this seriesChanges in v6:
- Passtcm_vhost_evttotcm_vhost_do_evt_workChanges in v5:
- Switch to int from u64 tovs_events_nr
- Sets->vs_events_droppedflag intcm_vhost_allocate_evt
- Do not nest dev mutex within vq mutex
- Usevs_events_lockto protectvs_events_droppedandvs_events_nr
- Rebase to target/masterChanges in v4:
- Droptcm_vhost_check_endpointintcm_vhost_send_evt
- Addtcm_vhost_check_endpointinvhost_scsi_evt_handle_kickChanges in v3:
- Separate the bug fix to another threadChanges in v2:
- Remove code duplication in tcm_vhost_{hotplug,hotunplug}
- Fix racing ofvs_events_nr
- Add flush fix patch to this seriesSigned-off-by Asias He <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- drivers/vhost/tcm_vhost.c | 208 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 +++ 2 files changed, 214 insertions(+), 4 deletions(-)drivers/vhost/tcm_vhost.c +3
-1@@ -66,11 +66,13 @@ enum {* TODO: debug and remove the workaround. */ enum { -VHOST_SCSI_FEATURES =VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX ) +VHOST_SCSI_FEATURES = (VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX )) | + (1ULL <<VIRTIO_SCSI_F_HOTPLUG ) };#define VHOST_SCSI_MAX_TARGET 256#define VHOST_SCSI_MAX_VQ 128 +#define VHOST_SCSI_MAX_EVENT 128struct vhost_scsi {/* Protected by vhost_scsi->dev.mutex */ @@ -82,6 +84,13 @@ struct vhost_scsi {Quoted 12 lines
drivers/vhost/tcm_vhost.c struct vhost_scsi { · line 91
Quoted 7 lines
drivers/vhost/tcm_vhost.c +5
-0struct vhost_work vs_completion_work;/* cmd completion work item */ struct llist_head vs_completion_list;/* cmd completion queue */ + +struct vhost_work vs_event_work;/* evt injection work item */ +struct llist_head vs_event_list;/* evt injection queue */ + +struct mutex vs_events_lock;/* protect vs_events_dropped,events_nr */ Looking at this code, there are just so many locks now. This does not make me feel safe :)
At least, please document lock nesting.
Quoted 1 line
Do you see a real problem?
Complexity is a real problem. My head already spins. No I don't see a bug, but we need to simplify locking.
And I think I see a nice way to do this:
1. move away from a global work to per-event work - so no list
2. remove dynamic allocation of events - so no events_nr
3. set/test overrun flag under the appropriate vq mutex
I think that's ideal. But we can move there in small steps. As a first step - why can't we always take the vq mutex lock and drop vs_events_lock?
Quoted 51 lines
Quoted 42 lines
Quoted 42 lines
drivers/vhost/tcm_vhost.c +28
-0+bool vs_events_dropped;/* any missed events */ +int vs_events_nr;/* num of pending events */ };/* Local pointer to allocated TCM configfs fabric module */ @@ -129,6 +138,17 @@ static bool tcm_vhost_check_endpoint(struct vhost_virtqueue *vq)return ret; } +static bool tcm_vhost_check_events_dropped (struct vhost_scsi *vs) +{ +bool ret; + +mutex_lock (&vs->vs_events_lock); + ret = vs->vs_events_dropped; +mutex_unlock (&vs->vs_events_lock); + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;@@ -379,6 +399,37 @@ static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd)return 0 ; } +static void tcm_vhost_free_evt (struct vhost_scsi *vs,struct tcm_vhost_evt *evt) +{ +mutex_lock (&vs->vs_events_lock); + vs->vs_events_nr--; +kfree (evt); +mutex_unlock (&vs->vs_events_lock); +} + +static struct tcm_vhost_evt *tcm_vhost_allocate_evt (struct vhost_scsi *vs, +u32 event,u32 reason) +{ +struct tcm_vhost_evt *evt; + +mutex_lock (&vs->vs_events_lock); +if (vs->vs_events_nr >VHOST_SCSI_MAX_EVENT ) {Quoted 8 lines
drivers/vhost/tcm_vhost.c static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd) · line 422
Quoted 6 lines
drivers/vhost/tcm_vhost.c +6
-0+ vs->vs_events_dropped = true; +mutex_unlock (&vs->vs_events_lock); +return NULL ; + } + + evt =kzalloc (sizeof (*evt),GFP_KERNEL );I think kzalloc not needed here, you init all fields.
Quoted 1 line
Not really! evt->event.lun[4-7] is not initialized. It needs to be 0.
So that is 4 bytes just init them when you set rest of lun.
Quoted 7 lines
Quoted 5 lines
Also, this basically means if backend does plug/unplug very quickly, we start consuming kernel memory without a limit. Not good. How about we allocate the event as part of target?
There shouldn't ever be more than one hotplug
event in flight per target, right?It is limited by
VHOST_SCSI_MAX_EVENT.
OK I missed that. But where does VHOST_SCSI_MAX_EVENT come from by the way? I don't see it in spec or did I miss it? It seems it's best not to lost events as long as there are descriptors in the event vq.
Quoted 9 lines
Quoted 6 lines
drivers/vhost/tcm_vhost.c static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd) · line 423
Quoted 2 lines
drivers/vhost/tcm_vhost.c +1
-0+if (evt) {Let's do clean error handling here and above:
if (!evt)goto err;We can you do in err? You simply unlock and return. Why bother? How cleaner it will be.
There's another error above and we'll share code.
It's not a big issue, just a bit nicer IMHO.
Quoted 46 lines
Quoted 37 lines
Quoted 37 lines
drivers/vhost/tcm_vhost.c +30
-0+ evt->event.event = event; + evt->event.reason = reason; + vs->vs_events_nr++; + } +mutex_unlock (&vs->vs_events_lock); + +return evt; +} +static void vhost_scsi_free_cmd (struct tcm_vhost_cmd *tv_cmd) {struct se_cmd *se_cmd = &tv_cmd->tvc_se_cmd;@@ -397,6 +448,75 @@ static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd)kfree (tv_cmd); } +static void tcm_vhost_do_evt_work (struct vhost_scsi *vs, +struct tcm_vhost_evt *evt) +{ +struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT ]; +struct virtio_scsi_event *event = &evt->event; +struct virtio_scsi_event __user *eventp; +unsigned out, in; +int head, ret; + +if (!tcm_vhost_check_endpoint (vq)) +return ; + +mutex_lock (&vs->vs_events_lock); +mutex_lock (&vq->mutex); +again: +vhost_disable_notify (&vs->dev, vq); + head =vhost_get_vq_desc (&vs->dev, vq, vq->iov, +ARRAY_SIZE (vq->iov), &out, &in, +NULL ,NULL ); +if (head <0 ) { + vs->vs_events_dropped = true;Quoted 8 lines
drivers/vhost/tcm_vhost.c static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd) · line 476
Quoted 5 lines
drivers/vhost/tcm_vhost.c +5
-0+ goto out; + } +if (head == vq->num) { +if (vhost_enable_notify (&vs->dev, vq)) + goto again;Please write loops with
while()orfor().
Not with goto. goto is for error handling.
Quoted 1 line
This makes extra indention which is more ugly.
I don't care. No loops with goto and that's a hard rule.
Quoted 39 lines
Quoted 30 lines
Quoted 30 lines
drivers/vhost/tcm_vhost.c +30
-0+ vs->vs_events_dropped = true; + goto out; + } + +if ((vq->iov[out].iov_len !=sizeof (struct virtio_scsi_event ))) { +vq_err (vq,"Expecting virtio_scsi_event, got %zu bytes\n" , + vq->iov[out].iov_len); + vs->vs_events_dropped = true; + goto out; + } + +if (vs->vs_events_dropped) { + event->event |=VIRTIO_SCSI_T_EVENTS_MISSED ; + vs->vs_events_dropped = false; + } + + eventp = vq->iov[out].iov_base; + ret =__copy_to_user (eventp, event,sizeof (*event)); +if (!ret) +vhost_add_used_and_signal (&vs->dev, vq, head,0 ); +else +vq_err (vq,"Faulted on tcm_vhost_send_event\n" ); +out: +mutex_unlock (&vq->mutex); +mutex_unlock (&vs->vs_events_lock); +} + +static void tcm_vhost_evt_work (struct vhost_work *work) +{ + struct vhost_scsi *vs = container_of(work, struct vhost_scsi,Quoted 8 lines
drivers/vhost/tcm_vhost.c static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd) · line 511
Quoted 5 lines
drivers/vhost/tcm_vhost.c +5
-0+ vs_event_work); +struct tcm_vhost_evt *evt; +struct llist_node *llnode; + + llnode =llist_del_all (&vs->vs_event_list);The assumption is that this is slow path thing, no need to worry about speed, yes? so let's do simple
list_add,list_deletc.
Quoted 1 line
Why it is simpler?
simple list_ with a lock is easier to use correctly.
Quoted 3 lines
We are using llist for cmd. Why it is better using
one for evt and the different for cmd? Similar code makes people easier to read.
OK fair enough. But my idea is above to use a work structure instead of a list of events. This way we don't need extra list at all, no new locks, nothing.
Quoted 51 lines
Quoted 42 lines
Quoted 42 lines
drivers/vhost/tcm_vhost.c +32
-1+while (llnode) { + evt = llist_entry(llnode, struct tcm_vhost_evt, list); + llnode =llist_next (llnode); +tcm_vhost_do_evt_work (vs, evt); +tcm_vhost_free_evt (vs, evt); + } +} +/* Fill in status and signal that we are done processing this command * * This is scheduled in the vhost work queue so we are called with the owner @@ -807,9 +927,42 @@ static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)pr_debug ("%s: The handling func for control queue.\n" , __func__); } +static int tcm_vhost_send_evt (struct vhost_scsi *vs,struct tcm_vhost_tpg *tpg, +struct se_lun *lun,u32 event,u32 reason) +{ +struct tcm_vhost_evt *evt; + + evt =tcm_vhost_allocate_evt (vs, event, reason); +if (!evt) +return -ENOMEM ; + +if (tpg && lun) { +/* TODO: share lun setup code with virtio-scsi.ko */ + evt->event.lun[0 ] =0x01 ; + evt->event.lun[1 ] = tpg->tport_tpgt &0xFF ; +if (lun->unpacked_lun >=256 ) + evt->event.lun[2 ] = lun->unpacked_lun >>8 |0x40 ; + evt->event.lun[3 ] = lun->unpacked_lun &0xFF ; + } + +llist_add (&evt->list, &vs->vs_event_list); +vhost_work_queue (&vs->dev, &vs->vs_event_work); + +return 0 ; +} +static void vhost_scsi_evt_handle_kick (struct vhost_work *work) { -pr_debug ("%s: The handling func for event queue.\n" , __func__);Quoted 8 lines
drivers/vhost/tcm_vhost.c static void vhost_scsi_ctl_handle_kick(struct vhost_work *work) · line 961
Quoted 6 lines
drivers/vhost/tcm_vhost.c +6
-0+ struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, + poll.work); + struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev); + +if (!tcm_vhost_check_endpoint (vq)) +return ;Again just drop this check.
Quoted 1 line
Why?
Same as above.
Because it's not safe to do this outside the vq mutex.
Quoted 62 lines
Quoted 47 lines
Quoted 47 lines
drivers/vhost/tcm_vhost.c +12
-0+ +if (tcm_vhost_check_events_dropped (vs)) +tcm_vhost_send_evt (vs,NULL ,NULL ,VIRTIO_SCSI_T_NO_EVENT ,0 ); + }static void vhost_scsi_handle_kick (struct vhost_work *work)@@ -833,6 +986,7 @@ static void vhost_scsi_flush(struct vhost_scsi *vs)for (i =0 ; i <VHOST_SCSI_MAX_VQ ; i++)vhost_scsi_flush_vq (vs, i);vhost_work_flush (&vs->dev, &vs->vs_completion_work); +vhost_work_flush (&vs->dev, &vs->vs_event_work); }/* @@ -891,6 +1045,7 @@ static int vhost_scsi_set_endpoint(return -EEXIST ; } tv_tpg->tv_tpg_vhost_count++; + tv_tpg->vhost_scsi = vs; vs_tpg[tv_tpg->tport_tpgt] = tv_tpg;smp_mb__after_atomic_inc (); match = true;@@ -974,6 +1129,7 @@ static int vhost_scsi_clear_endpoint( goto err_tpg; } tv_tpg->tv_tpg_vhost_count--; + tv_tpg->vhost_scsi =NULL ; vs->vs_tpg[target] =NULL ; match = true;mutex_unlock (&tv_tpg->tv_tpg_mutex);@@ -1033,6 +1189,11 @@ static int vhost_scsi_open(struct inode *inode, struct file *f)return -ENOMEM ;vhost_work_init (&s->vs_completion_work, vhost_scsi_complete_cmd_work); +vhost_work_init (&s->vs_event_work, tcm_vhost_evt_work); + + s->vs_events_nr =0 ; + s->vs_events_dropped = false; +mutex_init (&s->vs_events_lock); s->vqs[VHOST_SCSI_VQ_CTL ].handle_kick = vhost_scsi_ctl_handle_kick; s->vqs[VHOST_SCSI_VQ_EVT ].handle_kick = vhost_scsi_evt_handle_kick;@@ -1163,6 +1324,42 @@ static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport)return "Unknown" ; }Quoted 14 lines
drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1338
Quoted 12 lines
drivers/vhost/tcm_vhost.c +12
-0+static int tcm_vhost_do_plug (struct tcm_vhost_tpg *tpg, +struct se_lun *lun,bool plug) +{ + +struct vhost_scsi *vs = tpg->vhost_scsi; +u32 reason; + +mutex_lock (&tpg->tv_tpg_mutex); + vs = tpg->vhost_scsi; +mutex_unlock (&tpg->tv_tpg_mutex); +if (!vs) +return -EOPNOTSUPP ;Why EOPNOTSUPP? When does this happen?
Quoted 2 lines
When
tpg->vhost_scsihas not been setup. E.g. no one starts a vhost-scsi guest.
So ENODEV or something.
Quoted 32 lines
Quoted 22 lines
drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1347
Quoted 9 lines
drivers/vhost/tcm_vhost.c +9
-0+ +if (!tcm_vhost_check_feature (vs,VIRTIO_SCSI_F_HOTPLUG )) +return -EOPNOTSUPP ; + +if (plug) + reason =VIRTIO_SCSI_EVT_RESET_RESCAN ; +else + reason =VIRTIO_SCSI_EVT_RESET_REMOVED ; +You have dropped the tpg lock so
tpg->vhost_scsican become NULL now. Why is this safe?Quoted 9 lines
drivers/vhost/tcm_vhost.c +9
-0+return tcm_vhost_send_evt (vs, tpg, lun, +VIRTIO_SCSI_T_TRANSPORT_RESET , + reason); +} + +static int tcm_vhost_hotplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, true); +}Quoted 8 lines
drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1362
Quoted 6 lines
drivers/vhost/tcm_vhost.c +6
-0+ +static int tcm_vhost_hotunplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, false); +} +What are these wrappers for? Seem useless ...
Quoted 2 lines
It make people easier to understand what's the true and false is about in
tcm_vhost_do_plug.
So just pass in the reason and not a bool.
When readers see VIRTIO_SCSI_EVT_RESET_RESCAN/VIRTIO_SCSI_EVT_RESET_REMOVED they will know what is going on.
Quoted 67 lines
Quoted 64 lines
Quoted 64 lines
drivers/vhost/tcm_vhost.c +5
-2static int tcm_vhost_port_link (struct se_portal_group *se_tpg,struct se_lun *lun) {@@ -1173,18 +1370,21 @@ static int tcm_vhost_port_link(struct se_portal_group *se_tpg, tv_tpg->tv_tpg_port_count++;mutex_unlock (&tv_tpg->tv_tpg_mutex); +tcm_vhost_hotplug (tv_tpg, lun); +return 0 ; }static void tcm_vhost_port_unlink (struct se_portal_group *se_tpg, -struct se_lun *se_lun) +struct se_lun *lun) {struct tcm_vhost_tpg *tv_tpg =container_of (se_tpg, struct tcm_vhost_tpg, se_tpg); -mutex_lock (&tv_tpg->tv_tpg_mutex); tv_tpg->tv_tpg_port_count--;mutex_unlock (&tv_tpg->tv_tpg_mutex); + +tcm_vhost_hotunplug (tv_tpg, lun); } static struct se_node_acl *tcm_vhost_make_nodeacl(drivers/vhost/tcm_vhost.h +10
-0@@ -53,6 +53,7 @@ struct tcm_vhost_nacl {struct se_node_acl se_node_acl; }; +struct vhost_scsi ;struct tcm_vhost_tpg {/* Vhost port target portal group tag for TCM */ u16 tport_tpgt;@@ -70,6 +71,8 @@ struct tcm_vhost_tpg {struct tcm_vhost_tport *tport;/* Returned by tcm_vhost_make_tpg() */ struct se_portal_group se_tpg; +/* Pointer back to struct vhost_scsi, protected by tv_tpg_mutex */ +struct vhost_scsi *vhost_scsi; };struct tcm_vhost_tport {@@ -83,6 +86,13 @@ struct tcm_vhost_tport {struct se_wwn tport_wwn; }; +struct tcm_vhost_evt { +/* virtio_scsi event */ +struct virtio_scsi_event event; +/* virtio_scsi event list, serviced from vhost worker thread */ +struct llist_node list; +}; +/* * As per request from MST, keep TCM_VHOST related ioctl defines out of * linux/vhost.h (user-space) for now.. Signature · 1 line
1.8.1.4Signature · 1 line
Asias
Re: [PATCH v7 3/3] tcm_vhost: Add hotplug/hotunplug support
Quoted 82 lines · Michael S. Tsirkin
Quoted 76 lines · Asias He
Quoted 62 lines · Michael S. Tsirkin
Quoted 61 lines · Asias He
In commit
365a7150094([SCSI] virtio-scsi: hotplug support for virtio-scsi), hotplug support is added to virtio-scsi.This patch adds hotplug and hotunplug support to
tcm_vhost.You can create or delete a LUN in targetcli to hotplug or hotunplug a LUN in guest.
Changes in v7:
- Addvhost_work_flushforvs->vs_event_workto this seriesChanges in v6:
- Passtcm_vhost_evttotcm_vhost_do_evt_workChanges in v5:
- Switch to int from u64 tovs_events_nr
- Sets->vs_events_droppedflag intcm_vhost_allocate_evt
- Do not nest dev mutex within vq mutex
- Usevs_events_lockto protectvs_events_droppedandvs_events_nr
- Rebase to target/masterChanges in v4:
- Droptcm_vhost_check_endpointintcm_vhost_send_evt
- Addtcm_vhost_check_endpointinvhost_scsi_evt_handle_kickChanges in v3:
- Separate the bug fix to another threadChanges in v2:
- Remove code duplication in tcm_vhost_{hotplug,hotunplug}
- Fix racing ofvs_events_nr
- Add flush fix patch to this seriesSigned-off-by Asias He <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- drivers/vhost/tcm_vhost.c | 208 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 +++ 2 files changed, 214 insertions(+), 4 deletions(-)drivers/vhost/tcm_vhost.c +3
-1@@ -66,11 +66,13 @@ enum {* TODO: debug and remove the workaround. */ enum { -VHOST_SCSI_FEATURES =VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX ) +VHOST_SCSI_FEATURES = (VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX )) | + (1ULL <<VIRTIO_SCSI_F_HOTPLUG ) };#define VHOST_SCSI_MAX_TARGET 256#define VHOST_SCSI_MAX_VQ 128 +#define VHOST_SCSI_MAX_EVENT 128struct vhost_scsi {/* Protected by vhost_scsi->dev.mutex */ @@ -82,6 +84,13 @@ struct vhost_scsi {Quoted 12 lines
drivers/vhost/tcm_vhost.c struct vhost_scsi { · line 91
Quoted 7 lines
drivers/vhost/tcm_vhost.c +5
-0struct vhost_work vs_completion_work;/* cmd completion work item */ struct llist_head vs_completion_list;/* cmd completion queue */ + +struct vhost_work vs_event_work;/* evt injection work item */ +struct llist_head vs_event_list;/* evt injection queue */ + +struct mutex vs_events_lock;/* protect vs_events_dropped,events_nr */ Looking at this code, there are just so many locks now. This does not make me feel safe :)
At least, please document lock nesting.Quoted 1 line
Do you see a real problem?
Complexity is a real problem. My head already spins. No I don't see a bug, but we need to simplify locking.
Quoted 8 lines
And I think I see a nice way to do this:
1. move away from a global work to per-event work - so no list
2. remove dynamic allocation of events - so noevents_nr
3. set/test overrun flag under the appropriate vq mutexI think that's ideal. But we can move there in small steps. As a first step - why can't we always take the vq mutex lock and drop
vs_events_lock?
There are really different ways to solve the same problem. You are welcome to implement you ideas.
Quoted 51 lines
Quoted 51 lines
Quoted 50 lines
Quoted 38 lines
drivers/vhost/tcm_vhost.c +24
-0+bool vs_events_dropped;/* any missed events */ +int vs_events_nr;/* num of pending events */ };/* Local pointer to allocated TCM configfs fabric module */ @@ -129,6 +138,17 @@ static bool tcm_vhost_check_endpoint(struct vhost_virtqueue *vq)return ret; } +static bool tcm_vhost_check_events_dropped (struct vhost_scsi *vs) +{ +bool ret; + +mutex_lock (&vs->vs_events_lock); + ret = vs->vs_events_dropped; +mutex_unlock (&vs->vs_events_lock); + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;@@ -379,6 +399,37 @@ static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd)return 0 ; } +static void tcm_vhost_free_evt (struct vhost_scsi *vs,struct tcm_vhost_evt *evt) +{ +mutex_lock (&vs->vs_events_lock); + vs->vs_events_nr--; +kfree (evt); +mutex_unlock (&vs->vs_events_lock); +} + +static struct tcm_vhost_evt *tcm_vhost_allocate_evt (struct vhost_scsi *vs, +u32 event,u32 reason) +{drivers/vhost/tcm_vhost.c static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd) · line 422
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+struct tcm_vhost_evt *evt; + +mutex_lock (&vs->vs_events_lock); +if (vs->vs_events_nr >VHOST_SCSI_MAX_EVENT ) { + vs->vs_events_dropped = true; +mutex_unlock (&vs->vs_events_lock); +return NULL ; + } + + evt =kzalloc (sizeof (*evt),GFP_KERNEL );I think kzalloc not needed here, you init all fields.
Quoted 3 lines
Quoted 1 line
Not really! evt->event.lun[4-7] is not initialized. It needs to be 0.
So that is 4 bytes just init them when you set rest of lun.
It is not in the fast path. You can do it this way but not a must.
Quoted 11 lines
Quoted 7 lines
Quoted 5 lines
Also, this basically means if backend does plug/unplug very quickly, we start consuming kernel memory without a limit. Not good. How about we allocate the event as part of target?
There shouldn't ever be more than one hotplug
event in flight per target, right?It is limited by
VHOST_SCSI_MAX_EVENT.OK I missed that. But where does
VHOST_SCSI_MAX_EVENTcome from by the way? I don't see it in spec or did I miss it? It seems it's best not to lost events as long as there are descriptors in the event vq.
You wanted this.
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
drivers/vhost/tcm_vhost.c static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd) · line 423
Quoted 2 lines
drivers/vhost/tcm_vhost.c +1
-0+if (evt) {Let's do clean error handling here and above:
if (!evt)goto err;
Quoted 5 lines
Quoted 2 lines
We can you do in err? You simply unlock and return. Why bother? How cleaner it will be.
There's another error above and we'll share code.
It's not a big issue, just a bit nicer IMHO.
Can't we focus on real issues other than this?
Quoted 46 lines
Quoted 46 lines
Quoted 45 lines
Quoted 32 lines
drivers/vhost/tcm_vhost.c +25
-0+ evt->event.event = event; + evt->event.reason = reason; + vs->vs_events_nr++; + } +mutex_unlock (&vs->vs_events_lock); + +return evt; +} +static void vhost_scsi_free_cmd (struct tcm_vhost_cmd *tv_cmd) {struct se_cmd *se_cmd = &tv_cmd->tvc_se_cmd;@@ -397,6 +448,75 @@ static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd)kfree (tv_cmd); } +static void tcm_vhost_do_evt_work (struct vhost_scsi *vs, +struct tcm_vhost_evt *evt) +{ +struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT ]; +struct virtio_scsi_event *event = &evt->event; +struct virtio_scsi_event __user *eventp; +unsigned out, in; +int head, ret; + +if (!tcm_vhost_check_endpoint (vq)) +return ; + +mutex_lock (&vs->vs_events_lock); +mutex_lock (&vq->mutex); +again: +vhost_disable_notify (&vs->dev, vq);drivers/vhost/tcm_vhost.c static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd) · line 476
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+ head =vhost_get_vq_desc (&vs->dev, vq, vq->iov, +ARRAY_SIZE (vq->iov), &out, &in, +NULL ,NULL ); +if (head <0 ) { + vs->vs_events_dropped = true; + goto out; + } +if (head == vq->num) { +if (vhost_enable_notify (&vs->dev, vq)) + goto again;Please write loops with
while()orfor().
Not with goto. goto is for error handling.
Quoted 3 lines
Quoted 1 line
This makes extra indention which is more ugly.
I don't care. No loops with goto and that's a hard rule.
It is not a loop.
Quoted 39 lines
Quoted 39 lines
Quoted 38 lines
Quoted 25 lines
drivers/vhost/tcm_vhost.c +25
-0+ vs->vs_events_dropped = true; + goto out; + } + +if ((vq->iov[out].iov_len !=sizeof (struct virtio_scsi_event ))) { +vq_err (vq,"Expecting virtio_scsi_event, got %zu bytes\n" , + vq->iov[out].iov_len); + vs->vs_events_dropped = true; + goto out; + } + +if (vs->vs_events_dropped) { + event->event |=VIRTIO_SCSI_T_EVENTS_MISSED ; + vs->vs_events_dropped = false; + } + + eventp = vq->iov[out].iov_base; + ret =__copy_to_user (eventp, event,sizeof (*event)); +if (!ret) +vhost_add_used_and_signal (&vs->dev, vq, head,0 ); +else +vq_err (vq,"Faulted on tcm_vhost_send_event\n" ); +out: +mutex_unlock (&vq->mutex); +mutex_unlock (&vs->vs_events_lock);drivers/vhost/tcm_vhost.c static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd) · line 511
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+} + +static void tcm_vhost_evt_work (struct vhost_work *work) +{ + struct vhost_scsi *vs = container_of(work, struct vhost_scsi, + vs_event_work); +struct tcm_vhost_evt *evt; +struct llist_node *llnode; + + llnode =llist_del_all (&vs->vs_event_list);The assumption is that this is slow path thing, no need to worry about speed, yes? so let's do simple
list_add,list_deletc.
Quoted 3 lines
Quoted 1 line
Why it is simpler?
simple
list_with a lock is easier to use correctly.
You need to use it correctly anyway for cmd. Why you want a lock here?
Quoted 8 lines
Quoted 3 lines
We are using llist for cmd. Why it is better using
one for evt and the different for cmd? Similar code makes people easier to read.OK fair enough. But my idea is above to use a work structure instead of a list of events. This way we don't need extra list at all, no new locks, nothing.
Again, You are welcome to implement your ideas.
Quoted 51 lines
Quoted 51 lines
Quoted 50 lines
Quoted 38 lines
drivers/vhost/tcm_vhost.c +31
-0+while (llnode) { + evt = llist_entry(llnode, struct tcm_vhost_evt, list); + llnode =llist_next (llnode); +tcm_vhost_do_evt_work (vs, evt); +tcm_vhost_free_evt (vs, evt); + } +} +/* Fill in status and signal that we are done processing this command * * This is scheduled in the vhost work queue so we are called with the owner @@ -807,9 +927,42 @@ static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)pr_debug ("%s: The handling func for control queue.\n" , __func__); } +static int tcm_vhost_send_evt (struct vhost_scsi *vs,struct tcm_vhost_tpg *tpg, +struct se_lun *lun,u32 event,u32 reason) +{ +struct tcm_vhost_evt *evt; + + evt =tcm_vhost_allocate_evt (vs, event, reason); +if (!evt) +return -ENOMEM ; + +if (tpg && lun) { +/* TODO: share lun setup code with virtio-scsi.ko */ + evt->event.lun[0 ] =0x01 ; + evt->event.lun[1 ] = tpg->tport_tpgt &0xFF ; +if (lun->unpacked_lun >=256 ) + evt->event.lun[2 ] = lun->unpacked_lun >>8 |0x40 ; + evt->event.lun[3 ] = lun->unpacked_lun &0xFF ; + } + +llist_add (&evt->list, &vs->vs_event_list); +vhost_work_queue (&vs->dev, &vs->vs_event_work); + +return 0 ; +}drivers/vhost/tcm_vhost.c static void vhost_scsi_ctl_handle_kick(struct vhost_work *work) · line 961
Quoted 10 lines
drivers/vhost/tcm_vhost.c +7
-1+static void vhost_scsi_evt_handle_kick (struct vhost_work *work) { -pr_debug ("%s: The handling func for event queue.\n" , __func__); + struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, + poll.work); + struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev); + +if (!tcm_vhost_check_endpoint (vq)) +return ;Again just drop this check.
Quoted 4 lines
Quoted 1 line
Why?
Same as above.
Because it's not safe to do this outside the vq mutex.
Is it also not safe to the same in -net? Why is it not safe?
Quoted 62 lines
Quoted 62 lines
Quoted 61 lines
Quoted 49 lines
drivers/vhost/tcm_vhost.c +14
-0+ +if (tcm_vhost_check_events_dropped (vs)) +tcm_vhost_send_evt (vs,NULL ,NULL ,VIRTIO_SCSI_T_NO_EVENT ,0 ); + }static void vhost_scsi_handle_kick (struct vhost_work *work)@@ -833,6 +986,7 @@ static void vhost_scsi_flush(struct vhost_scsi *vs)for (i =0 ; i <VHOST_SCSI_MAX_VQ ; i++)vhost_scsi_flush_vq (vs, i);vhost_work_flush (&vs->dev, &vs->vs_completion_work); +vhost_work_flush (&vs->dev, &vs->vs_event_work); }/* @@ -891,6 +1045,7 @@ static int vhost_scsi_set_endpoint(return -EEXIST ; } tv_tpg->tv_tpg_vhost_count++; + tv_tpg->vhost_scsi = vs; vs_tpg[tv_tpg->tport_tpgt] = tv_tpg;smp_mb__after_atomic_inc (); match = true;@@ -974,6 +1129,7 @@ static int vhost_scsi_clear_endpoint( goto err_tpg; } tv_tpg->tv_tpg_vhost_count--; + tv_tpg->vhost_scsi =NULL ; vs->vs_tpg[target] =NULL ; match = true;mutex_unlock (&tv_tpg->tv_tpg_mutex);@@ -1033,6 +1189,11 @@ static int vhost_scsi_open(struct inode *inode, struct file *f)return -ENOMEM ;vhost_work_init (&s->vs_completion_work, vhost_scsi_complete_cmd_work); +vhost_work_init (&s->vs_event_work, tcm_vhost_evt_work); + + s->vs_events_nr =0 ; + s->vs_events_dropped = false; +mutex_init (&s->vs_events_lock); s->vqs[VHOST_SCSI_VQ_CTL ].handle_kick = vhost_scsi_ctl_handle_kick; s->vqs[VHOST_SCSI_VQ_EVT ].handle_kick = vhost_scsi_evt_handle_kick;@@ -1163,6 +1324,42 @@ static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport)return "Unknown" ; } +static int tcm_vhost_do_plug (struct tcm_vhost_tpg *tpg, +struct se_lun *lun,bool plug)drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1338
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+{ + +struct vhost_scsi *vs = tpg->vhost_scsi; +u32 reason; + +mutex_lock (&tpg->tv_tpg_mutex); + vs = tpg->vhost_scsi; +mutex_unlock (&tpg->tv_tpg_mutex); +if (!vs) +return -EOPNOTSUPP ;Why EOPNOTSUPP? When does this happen?
Quoted 4 lines
Quoted 2 lines
When
tpg->vhost_scsihas not been setup. E.g. no one starts a vhost-scsi guest.So ENODEV or something.
okay.
Quoted 32 lines
Quoted 32 lines
Quoted 30 lines
drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1347
Quoted 9 lines
drivers/vhost/tcm_vhost.c +9
-0+ +if (!tcm_vhost_check_feature (vs,VIRTIO_SCSI_F_HOTPLUG )) +return -EOPNOTSUPP ; + +if (plug) + reason =VIRTIO_SCSI_EVT_RESET_RESCAN ; +else + reason =VIRTIO_SCSI_EVT_RESET_REMOVED ; +You have dropped the tpg lock so
tpg->vhost_scsican become NULL now. Why is this safe?Quoted 5 lines
drivers/vhost/tcm_vhost.c +5
-0+return tcm_vhost_send_evt (vs, tpg, lun, +VIRTIO_SCSI_T_TRANSPORT_RESET , + reason); +} +drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1362
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+static int tcm_vhost_hotplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, true); +} + +static int tcm_vhost_hotunplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, false); +} +What are these wrappers for? Seem useless ...
Quoted 7 lines
Quoted 2 lines
It make people easier to understand what's the true and false is about in
tcm_vhost_do_plug.So just pass in the reason and not a bool.
When readers see VIRTIO_SCSI_EVT_RESET_RESCAN/VIRTIO_SCSI_EVT_RESET_REMOVED they will know what is going on.
How can you make the connection that VIRTIO_SCSI_EVT_RESET_RESCAN is hotplug and VIRTIO_SCSI_EVT_RESET_REMOVED is hot-unplug at first glance?
Quoted 67 lines
Quoted 67 lines
Quoted 64 lines
Quoted 64 lines
drivers/vhost/tcm_vhost.c +5
-2static int tcm_vhost_port_link (struct se_portal_group *se_tpg,struct se_lun *lun) {@@ -1173,18 +1370,21 @@ static int tcm_vhost_port_link(struct se_portal_group *se_tpg, tv_tpg->tv_tpg_port_count++;mutex_unlock (&tv_tpg->tv_tpg_mutex); +tcm_vhost_hotplug (tv_tpg, lun); +return 0 ; }static void tcm_vhost_port_unlink (struct se_portal_group *se_tpg, -struct se_lun *se_lun) +struct se_lun *lun) {struct tcm_vhost_tpg *tv_tpg =container_of (se_tpg, struct tcm_vhost_tpg, se_tpg); -mutex_lock (&tv_tpg->tv_tpg_mutex); tv_tpg->tv_tpg_port_count--;mutex_unlock (&tv_tpg->tv_tpg_mutex); + +tcm_vhost_hotunplug (tv_tpg, lun); } static struct se_node_acl *tcm_vhost_make_nodeacl(drivers/vhost/tcm_vhost.h +10
-0@@ -53,6 +53,7 @@ struct tcm_vhost_nacl {struct se_node_acl se_node_acl; }; +struct vhost_scsi ;struct tcm_vhost_tpg {/* Vhost port target portal group tag for TCM */ u16 tport_tpgt;@@ -70,6 +71,8 @@ struct tcm_vhost_tpg {struct tcm_vhost_tport *tport;/* Returned by tcm_vhost_make_tpg() */ struct se_portal_group se_tpg; +/* Pointer back to struct vhost_scsi, protected by tv_tpg_mutex */ +struct vhost_scsi *vhost_scsi; };struct tcm_vhost_tport {@@ -83,6 +86,13 @@ struct tcm_vhost_tport {struct se_wwn tport_wwn; }; +struct tcm_vhost_evt { +/* virtio_scsi event */ +struct virtio_scsi_event event; +/* virtio_scsi event list, serviced from vhost worker thread */ +struct llist_node list; +}; +/* * As per request from MST, keep TCM_VHOST related ioctl defines out of * linux/vhost.h (user-space) for now.. Signature · 1 line
1.8.1.4Signature · 1 line
Asias
Signature · 1 line
Asias
Michael S. Tsirkin
maintainerRe: [PATCH v7 3/3] tcm_vhost: Add hotplug/hotunplug support
Quoted 88 lines · Asias He
Quoted 87 lines · Michael S. Tsirkin
Quoted 76 lines · Asias He
Quoted 62 lines · Michael S. Tsirkin
Quoted 61 lines · Asias He
In commit
365a7150094([SCSI] virtio-scsi: hotplug support for virtio-scsi), hotplug support is added to virtio-scsi.This patch adds hotplug and hotunplug support to
tcm_vhost.You can create or delete a LUN in targetcli to hotplug or hotunplug a LUN in guest.
Changes in v7:
- Addvhost_work_flushforvs->vs_event_workto this seriesChanges in v6:
- Passtcm_vhost_evttotcm_vhost_do_evt_workChanges in v5:
- Switch to int from u64 tovs_events_nr
- Sets->vs_events_droppedflag intcm_vhost_allocate_evt
- Do not nest dev mutex within vq mutex
- Usevs_events_lockto protectvs_events_droppedandvs_events_nr
- Rebase to target/masterChanges in v4:
- Droptcm_vhost_check_endpointintcm_vhost_send_evt
- Addtcm_vhost_check_endpointinvhost_scsi_evt_handle_kickChanges in v3:
- Separate the bug fix to another threadChanges in v2:
- Remove code duplication in tcm_vhost_{hotplug,hotunplug}
- Fix racing ofvs_events_nr
- Add flush fix patch to this seriesSigned-off-by Asias He <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- drivers/vhost/tcm_vhost.c | 208 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 +++ 2 files changed, 214 insertions(+), 4 deletions(-)drivers/vhost/tcm_vhost.c +3
-1@@ -66,11 +66,13 @@ enum {* TODO: debug and remove the workaround. */ enum { -VHOST_SCSI_FEATURES =VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX ) +VHOST_SCSI_FEATURES = (VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX )) | + (1ULL <<VIRTIO_SCSI_F_HOTPLUG ) };#define VHOST_SCSI_MAX_TARGET 256#define VHOST_SCSI_MAX_VQ 128 +#define VHOST_SCSI_MAX_EVENT 128struct vhost_scsi {/* Protected by vhost_scsi->dev.mutex */ @@ -82,6 +84,13 @@ struct vhost_scsi {Quoted 12 lines
drivers/vhost/tcm_vhost.c struct vhost_scsi { · line 91
Quoted 7 lines
drivers/vhost/tcm_vhost.c +5
-0struct vhost_work vs_completion_work;/* cmd completion work item */ struct llist_head vs_completion_list;/* cmd completion queue */ + +struct vhost_work vs_event_work;/* evt injection work item */ +struct llist_head vs_event_list;/* evt injection queue */ + +struct mutex vs_events_lock;/* protect vs_events_dropped,events_nr */ Looking at this code, there are just so many locks now. This does not make me feel safe :)
At least, please document lock nesting.Quoted 1 line
Do you see a real problem?
Complexity is a real problem. My head already spins. No I don't see a bug, but we need to simplify locking.
And I think I see a nice way to do this:
1. move away from a global work to per-event work - so no list
2. remove dynamic allocation of events - so noevents_nr
3. set/test overrun flag under the appropriate vq mutex
Quoted 6 lines
Quoted 3 lines
I think that's ideal. But we can move there in small steps. As a first step - why can't we always take the vq mutex lock and drop
vs_events_lock?There are really different ways to solve the same problem. You are welcome to implement you ideas.
I have a problem with this patch. It just seems too tricky, while it is really trying to do a very simple thing. There's new lock nesting. There are new places which do lock, read data, unlock, use data. There's new RCU. All this feels fragile, and will be hard to maintain even though I don't see bugs.
So I would prefer it if you tried to simplify
this code making it easier to maintain. I sent some ideas on how to do this, but if you like, do it some other way.
I also made some comments about the coding style but these are less important (except goto again I guess) if these will be te only issues, I'd say we can apply.
Quoted 51 lines
Quoted 51 lines
Quoted 51 lines
Quoted 50 lines
Quoted 38 lines
drivers/vhost/tcm_vhost.c +24
-0+bool vs_events_dropped;/* any missed events */ +int vs_events_nr;/* num of pending events */ };/* Local pointer to allocated TCM configfs fabric module */ @@ -129,6 +138,17 @@ static bool tcm_vhost_check_endpoint(struct vhost_virtqueue *vq)return ret; } +static bool tcm_vhost_check_events_dropped (struct vhost_scsi *vs) +{ +bool ret; + +mutex_lock (&vs->vs_events_lock); + ret = vs->vs_events_dropped; +mutex_unlock (&vs->vs_events_lock); + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;@@ -379,6 +399,37 @@ static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd)return 0 ; } +static void tcm_vhost_free_evt (struct vhost_scsi *vs,struct tcm_vhost_evt *evt) +{ +mutex_lock (&vs->vs_events_lock); + vs->vs_events_nr--; +kfree (evt); +mutex_unlock (&vs->vs_events_lock); +} + +static struct tcm_vhost_evt *tcm_vhost_allocate_evt (struct vhost_scsi *vs, +u32 event,u32 reason) +{drivers/vhost/tcm_vhost.c static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd) · line 422
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+struct tcm_vhost_evt *evt; + +mutex_lock (&vs->vs_events_lock); +if (vs->vs_events_nr >VHOST_SCSI_MAX_EVENT ) { + vs->vs_events_dropped = true; +mutex_unlock (&vs->vs_events_lock); +return NULL ; + } + + evt =kzalloc (sizeof (*evt),GFP_KERNEL );I think kzalloc not needed here, you init all fields.
Quoted 5 lines
Quoted 3 lines
Quoted 1 line
Not really! evt->event.lun[4-7] is not initialized. It needs to be 0.
So that is 4 bytes just init them when you set rest of lun.
It is not in the fast path. You can do it this way but not a must.
I think that's a bit cleaner than relying on kzalloc to zero-initialize.
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines
Also, this basically means if backend does plug/unplug very quickly, we start consuming kernel memory without a limit. Not good. How about we allocate the event as part of target?
There shouldn't ever be more than one hotplug
event in flight per target, right?
Quoted 7 lines
Quoted 5 lines
Quoted 1 line
It is limited by
VHOST_SCSI_MAX_EVENT.OK I missed that. But where does
VHOST_SCSI_MAX_EVENTcome from by the way? I don't see it in spec or did I miss it? It seems it's best not to lost events as long as there are descriptors in the event vq.You wanted this.
Yes I wanted some limit on the amount of memory we use up :)
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
drivers/vhost/tcm_vhost.c static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd) · line 423
Quoted 2 lines
drivers/vhost/tcm_vhost.c +1
-0+if (evt) {Let's do clean error handling here and above:
if (!evt)goto err;
Quoted 7 lines
Quoted 5 lines
Quoted 2 lines
We can you do in err? You simply unlock and return. Why bother? How cleaner it will be.
There's another error above and we'll share code.
It's not a big issue, just a bit nicer IMHO.Can't we focus on real issues other than this?
Yes style issues are not the most important here.
Quoted 46 lines
Quoted 46 lines
Quoted 46 lines
Quoted 45 lines
Quoted 32 lines
drivers/vhost/tcm_vhost.c +25
-0+ evt->event.event = event; + evt->event.reason = reason; + vs->vs_events_nr++; + } +mutex_unlock (&vs->vs_events_lock); + +return evt; +} +static void vhost_scsi_free_cmd (struct tcm_vhost_cmd *tv_cmd) {struct se_cmd *se_cmd = &tv_cmd->tvc_se_cmd;@@ -397,6 +448,75 @@ static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd)kfree (tv_cmd); } +static void tcm_vhost_do_evt_work (struct vhost_scsi *vs, +struct tcm_vhost_evt *evt) +{ +struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT ]; +struct virtio_scsi_event *event = &evt->event; +struct virtio_scsi_event __user *eventp; +unsigned out, in; +int head, ret; + +if (!tcm_vhost_check_endpoint (vq)) +return ; + +mutex_lock (&vs->vs_events_lock); +mutex_lock (&vq->mutex); +again: +vhost_disable_notify (&vs->dev, vq);drivers/vhost/tcm_vhost.c static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd) · line 476
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+ head =vhost_get_vq_desc (&vs->dev, vq, vq->iov, +ARRAY_SIZE (vq->iov), &out, &in, +NULL ,NULL ); +if (head <0 ) { + vs->vs_events_dropped = true; + goto out; + } +if (head == vq->num) { +if (vhost_enable_notify (&vs->dev, vq)) + goto again;Please write loops with
while()orfor().
Not with goto. goto is for error handling.
Quoted 5 lines
Quoted 3 lines
Quoted 1 line
This makes extra indention which is more ugly.
I don't care. No loops with goto and that's a hard rule.
It is not a loop.
If same code repeats, it's a loop.
Quoted 49 lines
Quoted 39 lines
Quoted 39 lines
Quoted 38 lines
Quoted 25 lines
drivers/vhost/tcm_vhost.c +25
-0+ vs->vs_events_dropped = true; + goto out; + } + +if ((vq->iov[out].iov_len !=sizeof (struct virtio_scsi_event ))) { +vq_err (vq,"Expecting virtio_scsi_event, got %zu bytes\n" , + vq->iov[out].iov_len); + vs->vs_events_dropped = true; + goto out; + } + +if (vs->vs_events_dropped) { + event->event |=VIRTIO_SCSI_T_EVENTS_MISSED ; + vs->vs_events_dropped = false; + } + + eventp = vq->iov[out].iov_base; + ret =__copy_to_user (eventp, event,sizeof (*event)); +if (!ret) +vhost_add_used_and_signal (&vs->dev, vq, head,0 ); +else +vq_err (vq,"Faulted on tcm_vhost_send_event\n" ); +out: +mutex_unlock (&vq->mutex); +mutex_unlock (&vs->vs_events_lock);drivers/vhost/tcm_vhost.c static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd) · line 511
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+} + +static void tcm_vhost_evt_work (struct vhost_work *work) +{ + struct vhost_scsi *vs = container_of(work, struct vhost_scsi, + vs_event_work); +struct tcm_vhost_evt *evt; +struct llist_node *llnode; + + llnode =llist_del_all (&vs->vs_event_list);The assumption is that this is slow path thing, no need to worry about speed, yes? so let's do simple
list_add,list_deletc.Quoted 3 lines
Quoted 1 line
Why it is simpler?
simple
list_with a lock is easier to use correctly.You need to use it correctly anyway for cmd. Why you want a lock here?
Quoted 4 lines
Quoted 3 lines
We are using llist for cmd. Why it is better using
one for evt and the different for cmd? Similar code makes people easier to read.
Quoted 6 lines
Quoted 4 lines
OK fair enough. But my idea is above to use a work structure instead of a list of events. This way we don't need extra list at all, no new locks, nothing.
Again, You are welcome to implement your ideas.
I might but I'm otherwise occupied at the moment.
Quoted 51 lines
Quoted 51 lines
Quoted 51 lines
Quoted 50 lines
Quoted 38 lines
drivers/vhost/tcm_vhost.c +31
-0+while (llnode) { + evt = llist_entry(llnode, struct tcm_vhost_evt, list); + llnode =llist_next (llnode); +tcm_vhost_do_evt_work (vs, evt); +tcm_vhost_free_evt (vs, evt); + } +} +/* Fill in status and signal that we are done processing this command * * This is scheduled in the vhost work queue so we are called with the owner @@ -807,9 +927,42 @@ static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)pr_debug ("%s: The handling func for control queue.\n" , __func__); } +static int tcm_vhost_send_evt (struct vhost_scsi *vs,struct tcm_vhost_tpg *tpg, +struct se_lun *lun,u32 event,u32 reason) +{ +struct tcm_vhost_evt *evt; + + evt =tcm_vhost_allocate_evt (vs, event, reason); +if (!evt) +return -ENOMEM ; + +if (tpg && lun) { +/* TODO: share lun setup code with virtio-scsi.ko */ + evt->event.lun[0 ] =0x01 ; + evt->event.lun[1 ] = tpg->tport_tpgt &0xFF ; +if (lun->unpacked_lun >=256 ) + evt->event.lun[2 ] = lun->unpacked_lun >>8 |0x40 ; + evt->event.lun[3 ] = lun->unpacked_lun &0xFF ; + } + +llist_add (&evt->list, &vs->vs_event_list); +vhost_work_queue (&vs->dev, &vs->vs_event_work); + +return 0 ; +}drivers/vhost/tcm_vhost.c static void vhost_scsi_ctl_handle_kick(struct vhost_work *work) · line 961
Quoted 10 lines
drivers/vhost/tcm_vhost.c +7
-1+static void vhost_scsi_evt_handle_kick (struct vhost_work *work) { -pr_debug ("%s: The handling func for event queue.\n" , __func__); + struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, + poll.work); + struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev); + +if (!tcm_vhost_check_endpoint (vq)) +return ;Again just drop this check.
Quoted 6 lines
Quoted 4 lines
Quoted 1 line
Why?
Same as above.
Because it's not safe to do this outside the vq mutex.Is it also not safe to the same in -net?
It's safe there since we dereference the pointer.
Quoted 1 line
Why is it not safe?
Because you don't dereference the pointer.
Quoted 182 lines
Quoted 62 lines
Quoted 62 lines
Quoted 61 lines
Quoted 49 lines
drivers/vhost/tcm_vhost.c +14
-0+ +if (tcm_vhost_check_events_dropped (vs)) +tcm_vhost_send_evt (vs,NULL ,NULL ,VIRTIO_SCSI_T_NO_EVENT ,0 ); + }static void vhost_scsi_handle_kick (struct vhost_work *work)@@ -833,6 +986,7 @@ static void vhost_scsi_flush(struct vhost_scsi *vs)for (i =0 ; i <VHOST_SCSI_MAX_VQ ; i++)vhost_scsi_flush_vq (vs, i);vhost_work_flush (&vs->dev, &vs->vs_completion_work); +vhost_work_flush (&vs->dev, &vs->vs_event_work); }/* @@ -891,6 +1045,7 @@ static int vhost_scsi_set_endpoint(return -EEXIST ; } tv_tpg->tv_tpg_vhost_count++; + tv_tpg->vhost_scsi = vs; vs_tpg[tv_tpg->tport_tpgt] = tv_tpg;smp_mb__after_atomic_inc (); match = true;@@ -974,6 +1129,7 @@ static int vhost_scsi_clear_endpoint( goto err_tpg; } tv_tpg->tv_tpg_vhost_count--; + tv_tpg->vhost_scsi =NULL ; vs->vs_tpg[target] =NULL ; match = true;mutex_unlock (&tv_tpg->tv_tpg_mutex);@@ -1033,6 +1189,11 @@ static int vhost_scsi_open(struct inode *inode, struct file *f)return -ENOMEM ;vhost_work_init (&s->vs_completion_work, vhost_scsi_complete_cmd_work); +vhost_work_init (&s->vs_event_work, tcm_vhost_evt_work); + + s->vs_events_nr =0 ; + s->vs_events_dropped = false; +mutex_init (&s->vs_events_lock); s->vqs[VHOST_SCSI_VQ_CTL ].handle_kick = vhost_scsi_ctl_handle_kick; s->vqs[VHOST_SCSI_VQ_EVT ].handle_kick = vhost_scsi_evt_handle_kick;@@ -1163,6 +1324,42 @@ static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport)return "Unknown" ; } +static int tcm_vhost_do_plug (struct tcm_vhost_tpg *tpg, +struct se_lun *lun,bool plug)drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1338
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+{ + +struct vhost_scsi *vs = tpg->vhost_scsi; +u32 reason; + +mutex_lock (&tpg->tv_tpg_mutex); + vs = tpg->vhost_scsi; +mutex_unlock (&tpg->tv_tpg_mutex); +if (!vs) +return -EOPNOTSUPP ;Why EOPNOTSUPP? When does this happen?
Quoted 4 lines
Quoted 2 lines
When
tpg->vhost_scsihas not been setup. E.g. no one starts a vhost-scsi guest.So ENODEV or something.
okay.
Quoted 32 lines
Quoted 32 lines
Quoted 30 lines
drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1347
Quoted 9 lines
drivers/vhost/tcm_vhost.c +9
-0+ +if (!tcm_vhost_check_feature (vs,VIRTIO_SCSI_F_HOTPLUG )) +return -EOPNOTSUPP ; + +if (plug) + reason =VIRTIO_SCSI_EVT_RESET_RESCAN ; +else + reason =VIRTIO_SCSI_EVT_RESET_REMOVED ; +You have dropped the tpg lock so
tpg->vhost_scsican become NULL now. Why is this safe?Quoted 5 lines
drivers/vhost/tcm_vhost.c +5
-0+return tcm_vhost_send_evt (vs, tpg, lun, +VIRTIO_SCSI_T_TRANSPORT_RESET , + reason); +} +drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1362
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+static int tcm_vhost_hotplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, true); +} + +static int tcm_vhost_hotunplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, false); +} +What are these wrappers for? Seem useless ...
Quoted 7 lines
Quoted 2 lines
It make people easier to understand what's the true and false is about in
tcm_vhost_do_plug.So just pass in the reason and not a bool.
When readers see VIRTIO_SCSI_EVT_RESET_RESCAN/VIRTIO_SCSI_EVT_RESET_REMOVED they will know what is going on.How can you make the connection that
VIRTIO_SCSI_EVT_RESET_RESCANis hotplug andVIRTIO_SCSI_EVT_RESET_REMOVEDis hot-unplug at first glance?Quoted 67 lines
Quoted 67 lines
Quoted 64 lines
Quoted 64 lines
drivers/vhost/tcm_vhost.c +5
-2static int tcm_vhost_port_link (struct se_portal_group *se_tpg,struct se_lun *lun) {@@ -1173,18 +1370,21 @@ static int tcm_vhost_port_link(struct se_portal_group *se_tpg, tv_tpg->tv_tpg_port_count++;mutex_unlock (&tv_tpg->tv_tpg_mutex); +tcm_vhost_hotplug (tv_tpg, lun); +return 0 ; }static void tcm_vhost_port_unlink (struct se_portal_group *se_tpg, -struct se_lun *se_lun) +struct se_lun *lun) {struct tcm_vhost_tpg *tv_tpg =container_of (se_tpg, struct tcm_vhost_tpg, se_tpg); -mutex_lock (&tv_tpg->tv_tpg_mutex); tv_tpg->tv_tpg_port_count--;mutex_unlock (&tv_tpg->tv_tpg_mutex); + +tcm_vhost_hotunplug (tv_tpg, lun); } static struct se_node_acl *tcm_vhost_make_nodeacl(drivers/vhost/tcm_vhost.h +10
-0@@ -53,6 +53,7 @@ struct tcm_vhost_nacl {struct se_node_acl se_node_acl; }; +struct vhost_scsi ;struct tcm_vhost_tpg {/* Vhost port target portal group tag for TCM */ u16 tport_tpgt;@@ -70,6 +71,8 @@ struct tcm_vhost_tpg {struct tcm_vhost_tport *tport;/* Returned by tcm_vhost_make_tpg() */ struct se_portal_group se_tpg; +/* Pointer back to struct vhost_scsi, protected by tv_tpg_mutex */ +struct vhost_scsi *vhost_scsi; };struct tcm_vhost_tport {@@ -83,6 +86,13 @@ struct tcm_vhost_tport {struct se_wwn tport_wwn; }; +struct tcm_vhost_evt { +/* virtio_scsi event */ +struct virtio_scsi_event event; +/* virtio_scsi event list, serviced from vhost worker thread */ +struct llist_node list; +}; +/* * As per request from MST, keep TCM_VHOST related ioctl defines out of * linux/vhost.h (user-space) for now.. Signature · 1 line
1.8.1.4Signature · 1 line
AsiasSignature · 1 line
Asias
Re: [PATCH v7 3/3] tcm_vhost: Add hotplug/hotunplug support
Quoted 93 lines · Michael S. Tsirkin
Quoted 92 lines · Asias He
Quoted 90 lines · Michael S. Tsirkin
Quoted 76 lines · Asias He
Quoted 62 lines · Michael S. Tsirkin
Quoted 61 lines · Asias He
In commit
365a7150094([SCSI] virtio-scsi: hotplug support for virtio-scsi), hotplug support is added to virtio-scsi.This patch adds hotplug and hotunplug support to
tcm_vhost.You can create or delete a LUN in targetcli to hotplug or hotunplug a LUN in guest.
Changes in v7:
- Addvhost_work_flushforvs->vs_event_workto this seriesChanges in v6:
- Passtcm_vhost_evttotcm_vhost_do_evt_workChanges in v5:
- Switch to int from u64 tovs_events_nr
- Sets->vs_events_droppedflag intcm_vhost_allocate_evt
- Do not nest dev mutex within vq mutex
- Usevs_events_lockto protectvs_events_droppedandvs_events_nr
- Rebase to target/masterChanges in v4:
- Droptcm_vhost_check_endpointintcm_vhost_send_evt
- Addtcm_vhost_check_endpointinvhost_scsi_evt_handle_kickChanges in v3:
- Separate the bug fix to another threadChanges in v2:
- Remove code duplication in tcm_vhost_{hotplug,hotunplug}
- Fix racing ofvs_events_nr
- Add flush fix patch to this seriesSigned-off-by Asias He <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- drivers/vhost/tcm_vhost.c | 208 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 +++ 2 files changed, 214 insertions(+), 4 deletions(-)drivers/vhost/tcm_vhost.c +3
-1@@ -66,11 +66,13 @@ enum {* TODO: debug and remove the workaround. */ enum { -VHOST_SCSI_FEATURES =VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX ) +VHOST_SCSI_FEATURES = (VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX )) | + (1ULL <<VIRTIO_SCSI_F_HOTPLUG ) };#define VHOST_SCSI_MAX_TARGET 256#define VHOST_SCSI_MAX_VQ 128 +#define VHOST_SCSI_MAX_EVENT 128struct vhost_scsi {/* Protected by vhost_scsi->dev.mutex */ @@ -82,6 +84,13 @@ struct vhost_scsi {Quoted 12 lines
drivers/vhost/tcm_vhost.c struct vhost_scsi { · line 91
Quoted 7 lines
drivers/vhost/tcm_vhost.c +5
-0struct vhost_work vs_completion_work;/* cmd completion work item */ struct llist_head vs_completion_list;/* cmd completion queue */ + +struct vhost_work vs_event_work;/* evt injection work item */ +struct llist_head vs_event_list;/* evt injection queue */ + +struct mutex vs_events_lock;/* protect vs_events_dropped,events_nr */ Looking at this code, there are just so many locks now. This does not make me feel safe :)
At least, please document lock nesting.Quoted 1 line
Do you see a real problem?
Complexity is a real problem. My head already spins. No I don't see a bug, but we need to simplify locking.
And I think I see a nice way to do this:
1. move away from a global work to per-event work - so no list
2. remove dynamic allocation of events - so noevents_nr
3. set/test overrun flag under the appropriate vq mutexI think that's ideal. But we can move there in small steps. As a first step - why can't we always take the vq mutex lock and drop
vs_events_lock?
Quoted 5 lines
Quoted 2 lines
There are really different ways to solve the same problem. You are welcome to implement you ideas.
I have a problem with this patch. It just seems too tricky, while it is really trying to do a very simple thing.
How tricky is it?
Quoted 1 line
There's new lock nesting.
I used dev mutex in the past, but it does not work because it introduces a deadlock in vhost_scsi_set_endpoint. vs_events is a per device stat, so I think it deserves a per device lock instead of a per queue lock.
Quoted 2 lines
There are new places which do lock, read
data, unlock, use data.
With vs_events_lock?
Quoted 1 line
There's new RCU.
It is not a *new* RCU. It uses the existing one vq->private_data.
Quoted 6 lines
All this feels fragile, and
will be hard to maintain even though I don't see bugs.So I would prefer it if you tried to simplify
this code making it easier to maintain. I sent some ideas on how to do this, but if you like, do it some other way.
I think it is pretty simple already.
Quoted 56 lines
I also made some comments about the coding style but these are less important (except goto again I guess) if these will be te only issues, I'd say we can apply.
Quoted 51 lines
Quoted 51 lines
Quoted 51 lines
Quoted 50 lines
Quoted 38 lines
drivers/vhost/tcm_vhost.c +24
-0+bool vs_events_dropped;/* any missed events */ +int vs_events_nr;/* num of pending events */ };/* Local pointer to allocated TCM configfs fabric module */ @@ -129,6 +138,17 @@ static bool tcm_vhost_check_endpoint(struct vhost_virtqueue *vq)return ret; } +static bool tcm_vhost_check_events_dropped (struct vhost_scsi *vs) +{ +bool ret; + +mutex_lock (&vs->vs_events_lock); + ret = vs->vs_events_dropped; +mutex_unlock (&vs->vs_events_lock); + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;@@ -379,6 +399,37 @@ static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd)return 0 ; } +static void tcm_vhost_free_evt (struct vhost_scsi *vs,struct tcm_vhost_evt *evt) +{ +mutex_lock (&vs->vs_events_lock); + vs->vs_events_nr--; +kfree (evt); +mutex_unlock (&vs->vs_events_lock); +} + +static struct tcm_vhost_evt *tcm_vhost_allocate_evt (struct vhost_scsi *vs, +u32 event,u32 reason) +{drivers/vhost/tcm_vhost.c static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd) · line 422
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+struct tcm_vhost_evt *evt; + +mutex_lock (&vs->vs_events_lock); +if (vs->vs_events_nr >VHOST_SCSI_MAX_EVENT ) { + vs->vs_events_dropped = true; +mutex_unlock (&vs->vs_events_lock); +return NULL ; + } + + evt =kzalloc (sizeof (*evt),GFP_KERNEL );I think kzalloc not needed here, you init all fields.
Quoted 7 lines
Quoted 5 lines
Quoted 3 lines
Quoted 1 line
Not really! evt->event.lun[4-7] is not initialized. It needs to be 0.
So that is 4 bytes just init them when you set rest of lun.
It is not in the fast path. You can do it this way but not a must.
I think that's a bit cleaner than relying on kzalloc to zero-initialize.
I think it is a bit shorter, 4 lines saved.
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 7 lines
Quoted 5 lines
Also, this basically means if backend does plug/unplug very quickly, we start consuming kernel memory without a limit. Not good. How about we allocate the event as part of target?
There shouldn't ever be more than one hotplug
event in flight per target, right?It is limited by
VHOST_SCSI_MAX_EVENT.
Quoted 7 lines
Quoted 5 lines
Quoted 3 lines
OK I missed that. But where does
VHOST_SCSI_MAX_EVENTcome from by the way? I don't see it in spec or did I miss it? It seems it's best not to lost events as long as there are descriptors in the event vq.You wanted this.
Yes I wanted some limit on the amount of memory we use up :)
So we limited this already.
Quoted 63 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
drivers/vhost/tcm_vhost.c static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd) · line 423
Quoted 2 lines
drivers/vhost/tcm_vhost.c +1
-0+if (evt) {Let's do clean error handling here and above:
if (!evt)goto err;Quoted 7 lines
Quoted 5 lines
Quoted 2 lines
We can you do in err? You simply unlock and return. Why bother? How cleaner it will be.
There's another error above and we'll share code.
It's not a big issue, just a bit nicer IMHO.Can't we focus on real issues other than this?
Yes style issues are not the most important here.
Quoted 46 lines
Quoted 46 lines
Quoted 46 lines
Quoted 45 lines
Quoted 32 lines
drivers/vhost/tcm_vhost.c +25
-0+ evt->event.event = event; + evt->event.reason = reason; + vs->vs_events_nr++; + } +mutex_unlock (&vs->vs_events_lock); + +return evt; +} +static void vhost_scsi_free_cmd (struct tcm_vhost_cmd *tv_cmd) {struct se_cmd *se_cmd = &tv_cmd->tvc_se_cmd;@@ -397,6 +448,75 @@ static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd)kfree (tv_cmd); } +static void tcm_vhost_do_evt_work (struct vhost_scsi *vs, +struct tcm_vhost_evt *evt) +{ +struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT ]; +struct virtio_scsi_event *event = &evt->event; +struct virtio_scsi_event __user *eventp; +unsigned out, in; +int head, ret; + +if (!tcm_vhost_check_endpoint (vq)) +return ; + +mutex_lock (&vs->vs_events_lock); +mutex_lock (&vq->mutex); +again: +vhost_disable_notify (&vs->dev, vq);drivers/vhost/tcm_vhost.c static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd) · line 476
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+ head =vhost_get_vq_desc (&vs->dev, vq, vq->iov, +ARRAY_SIZE (vq->iov), &out, &in, +NULL ,NULL ); +if (head <0 ) { + vs->vs_events_dropped = true; + goto out; + } +if (head == vq->num) { +if (vhost_enable_notify (&vs->dev, vq)) + goto again;Please write loops with
while()orfor().
Not with goto. goto is for error handling.
Quoted 7 lines
Quoted 5 lines
Quoted 3 lines
Quoted 1 line
This makes extra indention which is more ugly.
I don't care. No loops with goto and that's a hard rule.
It is not a loop.
If same code repeats, it's a loop.
We really need here is to call vhost_get_vq_desc once. It's not a loop like other places to use while() or for() with vhost_get_vq_desc.
Quoted 304 lines
Quoted 49 lines
Quoted 39 lines
Quoted 39 lines
Quoted 38 lines
Quoted 25 lines
drivers/vhost/tcm_vhost.c +25
-0+ vs->vs_events_dropped = true; + goto out; + } + +if ((vq->iov[out].iov_len !=sizeof (struct virtio_scsi_event ))) { +vq_err (vq,"Expecting virtio_scsi_event, got %zu bytes\n" , + vq->iov[out].iov_len); + vs->vs_events_dropped = true; + goto out; + } + +if (vs->vs_events_dropped) { + event->event |=VIRTIO_SCSI_T_EVENTS_MISSED ; + vs->vs_events_dropped = false; + } + + eventp = vq->iov[out].iov_base; + ret =__copy_to_user (eventp, event,sizeof (*event)); +if (!ret) +vhost_add_used_and_signal (&vs->dev, vq, head,0 ); +else +vq_err (vq,"Faulted on tcm_vhost_send_event\n" ); +out: +mutex_unlock (&vq->mutex); +mutex_unlock (&vs->vs_events_lock);drivers/vhost/tcm_vhost.c static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd) · line 511
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+} + +static void tcm_vhost_evt_work (struct vhost_work *work) +{ + struct vhost_scsi *vs = container_of(work, struct vhost_scsi, + vs_event_work); +struct tcm_vhost_evt *evt; +struct llist_node *llnode; + + llnode =llist_del_all (&vs->vs_event_list);The assumption is that this is slow path thing, no need to worry about speed, yes? so let's do simple
list_add,list_deletc.Quoted 3 lines
Quoted 1 line
Why it is simpler?
simple
list_with a lock is easier to use correctly.You need to use it correctly anyway for cmd. Why you want a lock here?
Quoted 4 lines
Quoted 3 lines
We are using llist for cmd. Why it is better using
one for evt and the different for cmd? Similar code makes people easier to read.Quoted 6 lines
Quoted 4 lines
OK fair enough. But my idea is above to use a work structure instead of a list of events. This way we don't need extra list at all, no new locks, nothing.
Again, You are welcome to implement your ideas.
I might but I'm otherwise occupied at the moment.
Quoted 51 lines
Quoted 51 lines
Quoted 51 lines
Quoted 50 lines
Quoted 38 lines
drivers/vhost/tcm_vhost.c +31
-0+while (llnode) { + evt = llist_entry(llnode, struct tcm_vhost_evt, list); + llnode =llist_next (llnode); +tcm_vhost_do_evt_work (vs, evt); +tcm_vhost_free_evt (vs, evt); + } +} +/* Fill in status and signal that we are done processing this command * * This is scheduled in the vhost work queue so we are called with the owner @@ -807,9 +927,42 @@ static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)pr_debug ("%s: The handling func for control queue.\n" , __func__); } +static int tcm_vhost_send_evt (struct vhost_scsi *vs,struct tcm_vhost_tpg *tpg, +struct se_lun *lun,u32 event,u32 reason) +{ +struct tcm_vhost_evt *evt; + + evt =tcm_vhost_allocate_evt (vs, event, reason); +if (!evt) +return -ENOMEM ; + +if (tpg && lun) { +/* TODO: share lun setup code with virtio-scsi.ko */ + evt->event.lun[0 ] =0x01 ; + evt->event.lun[1 ] = tpg->tport_tpgt &0xFF ; +if (lun->unpacked_lun >=256 ) + evt->event.lun[2 ] = lun->unpacked_lun >>8 |0x40 ; + evt->event.lun[3 ] = lun->unpacked_lun &0xFF ; + } + +llist_add (&evt->list, &vs->vs_event_list); +vhost_work_queue (&vs->dev, &vs->vs_event_work); + +return 0 ; +}drivers/vhost/tcm_vhost.c static void vhost_scsi_ctl_handle_kick(struct vhost_work *work) · line 961
Quoted 10 lines
drivers/vhost/tcm_vhost.c +7
-1+static void vhost_scsi_evt_handle_kick (struct vhost_work *work) { -pr_debug ("%s: The handling func for event queue.\n" , __func__); + struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, + poll.work); + struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev); + +if (!tcm_vhost_check_endpoint (vq)) +return ;Again just drop this check.
Quoted 6 lines
Quoted 4 lines
Quoted 1 line
Why?
Same as above.
Because it's not safe to do this outside the vq mutex.Is it also not safe to the same in -net?
It's safe there since we dereference the pointer.
Quoted 1 line
Why is it not safe?
Because you don't dereference the pointer.
Quoted 182 lines
Quoted 62 lines
Quoted 62 lines
Quoted 61 lines
Quoted 49 lines
drivers/vhost/tcm_vhost.c +14
-0+ +if (tcm_vhost_check_events_dropped (vs)) +tcm_vhost_send_evt (vs,NULL ,NULL ,VIRTIO_SCSI_T_NO_EVENT ,0 ); + }static void vhost_scsi_handle_kick (struct vhost_work *work)@@ -833,6 +986,7 @@ static void vhost_scsi_flush(struct vhost_scsi *vs)for (i =0 ; i <VHOST_SCSI_MAX_VQ ; i++)vhost_scsi_flush_vq (vs, i);vhost_work_flush (&vs->dev, &vs->vs_completion_work); +vhost_work_flush (&vs->dev, &vs->vs_event_work); }/* @@ -891,6 +1045,7 @@ static int vhost_scsi_set_endpoint(return -EEXIST ; } tv_tpg->tv_tpg_vhost_count++; + tv_tpg->vhost_scsi = vs; vs_tpg[tv_tpg->tport_tpgt] = tv_tpg;smp_mb__after_atomic_inc (); match = true;@@ -974,6 +1129,7 @@ static int vhost_scsi_clear_endpoint( goto err_tpg; } tv_tpg->tv_tpg_vhost_count--; + tv_tpg->vhost_scsi =NULL ; vs->vs_tpg[target] =NULL ; match = true;mutex_unlock (&tv_tpg->tv_tpg_mutex);@@ -1033,6 +1189,11 @@ static int vhost_scsi_open(struct inode *inode, struct file *f)return -ENOMEM ;vhost_work_init (&s->vs_completion_work, vhost_scsi_complete_cmd_work); +vhost_work_init (&s->vs_event_work, tcm_vhost_evt_work); + + s->vs_events_nr =0 ; + s->vs_events_dropped = false; +mutex_init (&s->vs_events_lock); s->vqs[VHOST_SCSI_VQ_CTL ].handle_kick = vhost_scsi_ctl_handle_kick; s->vqs[VHOST_SCSI_VQ_EVT ].handle_kick = vhost_scsi_evt_handle_kick;@@ -1163,6 +1324,42 @@ static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport)return "Unknown" ; } +static int tcm_vhost_do_plug (struct tcm_vhost_tpg *tpg, +struct se_lun *lun,bool plug)drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1338
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+{ + +struct vhost_scsi *vs = tpg->vhost_scsi; +u32 reason; + +mutex_lock (&tpg->tv_tpg_mutex); + vs = tpg->vhost_scsi; +mutex_unlock (&tpg->tv_tpg_mutex); +if (!vs) +return -EOPNOTSUPP ;Why EOPNOTSUPP? When does this happen?
Quoted 4 lines
Quoted 2 lines
When
tpg->vhost_scsihas not been setup. E.g. no one starts a vhost-scsi guest.So ENODEV or something.
okay.
Quoted 32 lines
Quoted 32 lines
Quoted 30 lines
drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1347
Quoted 9 lines
drivers/vhost/tcm_vhost.c +9
-0+ +if (!tcm_vhost_check_feature (vs,VIRTIO_SCSI_F_HOTPLUG )) +return -EOPNOTSUPP ; + +if (plug) + reason =VIRTIO_SCSI_EVT_RESET_RESCAN ; +else + reason =VIRTIO_SCSI_EVT_RESET_REMOVED ; +You have dropped the tpg lock so
tpg->vhost_scsican become NULL now. Why is this safe?Quoted 5 lines
drivers/vhost/tcm_vhost.c +5
-0+return tcm_vhost_send_evt (vs, tpg, lun, +VIRTIO_SCSI_T_TRANSPORT_RESET , + reason); +} +drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1362
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+static int tcm_vhost_hotplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, true); +} + +static int tcm_vhost_hotunplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, false); +} +What are these wrappers for? Seem useless ...
Quoted 7 lines
Quoted 2 lines
It make people easier to understand what's the true and false is about in
tcm_vhost_do_plug.So just pass in the reason and not a bool.
When readers see VIRTIO_SCSI_EVT_RESET_RESCAN/VIRTIO_SCSI_EVT_RESET_REMOVED they will know what is going on.How can you make the connection that
VIRTIO_SCSI_EVT_RESET_RESCANis hotplug andVIRTIO_SCSI_EVT_RESET_REMOVEDis hot-unplug at first glance?Quoted 67 lines
Quoted 67 lines
Quoted 64 lines
Quoted 64 lines
drivers/vhost/tcm_vhost.c +5
-2static int tcm_vhost_port_link (struct se_portal_group *se_tpg,struct se_lun *lun) {@@ -1173,18 +1370,21 @@ static int tcm_vhost_port_link(struct se_portal_group *se_tpg, tv_tpg->tv_tpg_port_count++;mutex_unlock (&tv_tpg->tv_tpg_mutex); +tcm_vhost_hotplug (tv_tpg, lun); +return 0 ; }static void tcm_vhost_port_unlink (struct se_portal_group *se_tpg, -struct se_lun *se_lun) +struct se_lun *lun) {struct tcm_vhost_tpg *tv_tpg =container_of (se_tpg, struct tcm_vhost_tpg, se_tpg); -mutex_lock (&tv_tpg->tv_tpg_mutex); tv_tpg->tv_tpg_port_count--;mutex_unlock (&tv_tpg->tv_tpg_mutex); + +tcm_vhost_hotunplug (tv_tpg, lun); } static struct se_node_acl *tcm_vhost_make_nodeacl(drivers/vhost/tcm_vhost.h +10
-0@@ -53,6 +53,7 @@ struct tcm_vhost_nacl {struct se_node_acl se_node_acl; }; +struct vhost_scsi ;struct tcm_vhost_tpg {/* Vhost port target portal group tag for TCM */ u16 tport_tpgt;@@ -70,6 +71,8 @@ struct tcm_vhost_tpg {struct tcm_vhost_tport *tport;/* Returned by tcm_vhost_make_tpg() */ struct se_portal_group se_tpg; +/* Pointer back to struct vhost_scsi, protected by tv_tpg_mutex */ +struct vhost_scsi *vhost_scsi; };struct tcm_vhost_tport {@@ -83,6 +86,13 @@ struct tcm_vhost_tport {struct se_wwn tport_wwn; }; +struct tcm_vhost_evt { +/* virtio_scsi event */ +struct virtio_scsi_event event; +/* virtio_scsi event list, serviced from vhost worker thread */ +struct llist_node list; +}; +/* * As per request from MST, keep TCM_VHOST related ioctl defines out of * linux/vhost.h (user-space) for now.. Signature · 1 line
1.8.1.4Signature · 1 line
AsiasSignature · 1 line
Asias
Signature · 1 line
Asias
Michael S. Tsirkin
maintainerRe: [PATCH v7 3/3] tcm_vhost: Add hotplug/hotunplug support
Quoted 94 lines · Asias He
Quoted 93 lines · Michael S. Tsirkin
Quoted 92 lines · Asias He
Quoted 90 lines · Michael S. Tsirkin
Quoted 76 lines · Asias He
Quoted 62 lines · Michael S. Tsirkin
Quoted 61 lines · Asias He
In commit
365a7150094([SCSI] virtio-scsi: hotplug support for virtio-scsi), hotplug support is added to virtio-scsi.This patch adds hotplug and hotunplug support to
tcm_vhost.You can create or delete a LUN in targetcli to hotplug or hotunplug a LUN in guest.
Changes in v7:
- Addvhost_work_flushforvs->vs_event_workto this seriesChanges in v6:
- Passtcm_vhost_evttotcm_vhost_do_evt_workChanges in v5:
- Switch to int from u64 tovs_events_nr
- Sets->vs_events_droppedflag intcm_vhost_allocate_evt
- Do not nest dev mutex within vq mutex
- Usevs_events_lockto protectvs_events_droppedandvs_events_nr
- Rebase to target/masterChanges in v4:
- Droptcm_vhost_check_endpointintcm_vhost_send_evt
- Addtcm_vhost_check_endpointinvhost_scsi_evt_handle_kickChanges in v3:
- Separate the bug fix to another threadChanges in v2:
- Remove code duplication in tcm_vhost_{hotplug,hotunplug}
- Fix racing ofvs_events_nr
- Add flush fix patch to this seriesSigned-off-by Asias He <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- drivers/vhost/tcm_vhost.c | 208 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 +++ 2 files changed, 214 insertions(+), 4 deletions(-)drivers/vhost/tcm_vhost.c +3
-1@@ -66,11 +66,13 @@ enum {* TODO: debug and remove the workaround. */ enum { -VHOST_SCSI_FEATURES =VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX ) +VHOST_SCSI_FEATURES = (VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX )) | + (1ULL <<VIRTIO_SCSI_F_HOTPLUG ) };#define VHOST_SCSI_MAX_TARGET 256#define VHOST_SCSI_MAX_VQ 128 +#define VHOST_SCSI_MAX_EVENT 128struct vhost_scsi {/* Protected by vhost_scsi->dev.mutex */ @@ -82,6 +84,13 @@ struct vhost_scsi {Quoted 12 lines
drivers/vhost/tcm_vhost.c struct vhost_scsi { · line 91
Quoted 7 lines
drivers/vhost/tcm_vhost.c +5
-0struct vhost_work vs_completion_work;/* cmd completion work item */ struct llist_head vs_completion_list;/* cmd completion queue */ + +struct vhost_work vs_event_work;/* evt injection work item */ +struct llist_head vs_event_list;/* evt injection queue */ + +struct mutex vs_events_lock;/* protect vs_events_dropped,events_nr */ Looking at this code, there are just so many locks now. This does not make me feel safe :)
At least, please document lock nesting.Quoted 1 line
Do you see a real problem?
Complexity is a real problem. My head already spins. No I don't see a bug, but we need to simplify locking.
And I think I see a nice way to do this:
1. move away from a global work to per-event work - so no list
2. remove dynamic allocation of events - so noevents_nr
3. set/test overrun flag under the appropriate vq mutexI think that's ideal. But we can move there in small steps. As a first step - why can't we always take the vq mutex lock and drop
vs_events_lock?
Quoted 7 lines
Quoted 5 lines
Quoted 2 lines
There are really different ways to solve the same problem. You are welcome to implement you ideas.
I have a problem with this patch. It just seems too tricky, while it is really trying to do a very simple thing.
How tricky is it?
It has its own locking scheme which seems unwarranted. Find a way to use the existing locking please.
Quoted 5 lines
Quoted 1 line
There's new lock nesting.
I used dev mutex in the past, but it does not work because it introduces a deadlock in
vhost_scsi_set_endpoint.vs_eventsis a per device stat, so I think it deserves a per device lock instead of a per queue lock.
We don't add a lock per field, it just does not make sense.
Quoted 4 lines
Quoted 2 lines
There are new places which do lock, read
data, unlock, use data.With
vs_events_lock?
With this patch.
Quoted 3 lines
Quoted 1 line
There's new RCU.
It is not a *new* RCU. It uses the existing one
vq->private_data.
Incorrectly. It should be used on the workqueue where it's dereferenced.
Quoted 8 lines
Quoted 6 lines
All this feels fragile, and
will be hard to maintain even though I don't see bugs.So I would prefer it if you tried to simplify
this code making it easier to maintain. I sent some ideas on how to do this, but if you like, do it some other way.I think it is pretty simple already.
Here, we disagree. And since I'm one of the people who gets to maintain this, it's important that I'm comfortable with the code, and don't keep worrying "is this racy? is there a deadlock?". This patch makes me uncomfortable. Could you address this please?
Quoted 58 lines
Quoted 58 lines
I also made some comments about the coding style but these are less important (except goto again I guess) if these will be te only issues, I'd say we can apply.
Quoted 53 lines
Quoted 53 lines
Quoted 52 lines
Quoted 42 lines
Quoted 42 lines
drivers/vhost/tcm_vhost.c +28
-0+bool vs_events_dropped;/* any missed events */ +int vs_events_nr;/* num of pending events */ };/* Local pointer to allocated TCM configfs fabric module */ @@ -129,6 +138,17 @@ static bool tcm_vhost_check_endpoint(struct vhost_virtqueue *vq)return ret; } +static bool tcm_vhost_check_events_dropped (struct vhost_scsi *vs) +{ +bool ret; + +mutex_lock (&vs->vs_events_lock); + ret = vs->vs_events_dropped; +mutex_unlock (&vs->vs_events_lock); + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;@@ -379,6 +399,37 @@ static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd)return 0 ; } +static void tcm_vhost_free_evt (struct vhost_scsi *vs,struct tcm_vhost_evt *evt) +{ +mutex_lock (&vs->vs_events_lock); + vs->vs_events_nr--; +kfree (evt); +mutex_unlock (&vs->vs_events_lock); +} + +static struct tcm_vhost_evt *tcm_vhost_allocate_evt (struct vhost_scsi *vs, +u32 event,u32 reason) +{ +struct tcm_vhost_evt *evt; + +mutex_lock (&vs->vs_events_lock); +if (vs->vs_events_nr >VHOST_SCSI_MAX_EVENT ) {Quoted 8 lines
drivers/vhost/tcm_vhost.c static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd) · line 422
Quoted 6 lines
drivers/vhost/tcm_vhost.c +6
-0+ vs->vs_events_dropped = true; +mutex_unlock (&vs->vs_events_lock); +return NULL ; + } + + evt =kzalloc (sizeof (*evt),GFP_KERNEL );I think kzalloc not needed here, you init all fields.
Not really! evt->event.lun[4-7] is not initialized. It needs to be 0.
Quoted 7 lines
Quoted 5 lines
Quoted 3 lines
Quoted 1 line
So that is 4 bytes just init them when you set rest of lun.
It is not in the fast path. You can do it this way but not a must.
I think that's a bit cleaner than relying on kzalloc to zero-initialize.
I think it is a bit shorter, 4 lines saved.
OTOH you don't have to think "what about the rest of the bytes?" then hunt through the code and go "ah, it's kzalloc".
Looks it's a style issue but I prefer explicit initialization.
Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
Quoted 11 lines
Quoted 7 lines
Quoted 5 lines
Also, this basically means if backend does plug/unplug very quickly, we start consuming kernel memory without a limit. Not good. How about we allocate the event as part of target?
There shouldn't ever be more than one hotplug
event in flight per target, right?It is limited by
VHOST_SCSI_MAX_EVENT.OK I missed that. But where does
VHOST_SCSI_MAX_EVENTcome from by the way? I don't see it in spec or did I miss it? It seems it's best not to lost events as long as there are descriptors in the event vq.
Quoted 5 lines
Quoted 3 lines
Quoted 1 line
You wanted this.
Yes I wanted some limit on the amount of memory we use up :)
So we limited this already.
Yes. But my question is this: what happens if an event is lost? Can guest recover and how?
Quoted 65 lines
Quoted 65 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
drivers/vhost/tcm_vhost.c static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd) · line 423
Quoted 2 lines
drivers/vhost/tcm_vhost.c +1
-0+if (evt) {Let's do clean error handling here and above:
if (!evt)goto err;Quoted 7 lines
Quoted 5 lines
Quoted 2 lines
We can you do in err? You simply unlock and return. Why bother? How cleaner it will be.
There's another error above and we'll share code.
It's not a big issue, just a bit nicer IMHO.Can't we focus on real issues other than this?
Yes style issues are not the most important here.
Quoted 48 lines
Quoted 48 lines
Quoted 47 lines
Quoted 37 lines
Quoted 37 lines
drivers/vhost/tcm_vhost.c +30
-0+ evt->event.event = event; + evt->event.reason = reason; + vs->vs_events_nr++; + } +mutex_unlock (&vs->vs_events_lock); + +return evt; +} +static void vhost_scsi_free_cmd (struct tcm_vhost_cmd *tv_cmd) {struct se_cmd *se_cmd = &tv_cmd->tvc_se_cmd;@@ -397,6 +448,75 @@ static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd)kfree (tv_cmd); } +static void tcm_vhost_do_evt_work (struct vhost_scsi *vs, +struct tcm_vhost_evt *evt) +{ +struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT ]; +struct virtio_scsi_event *event = &evt->event; +struct virtio_scsi_event __user *eventp; +unsigned out, in; +int head, ret; + +if (!tcm_vhost_check_endpoint (vq)) +return ; + +mutex_lock (&vs->vs_events_lock); +mutex_lock (&vq->mutex); +again: +vhost_disable_notify (&vs->dev, vq); + head =vhost_get_vq_desc (&vs->dev, vq, vq->iov, +ARRAY_SIZE (vq->iov), &out, &in, +NULL ,NULL ); +if (head <0 ) { + vs->vs_events_dropped = true;Quoted 8 lines
drivers/vhost/tcm_vhost.c static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd) · line 476
Quoted 5 lines
drivers/vhost/tcm_vhost.c +5
-0+ goto out; + } +if (head == vq->num) { +if (vhost_enable_notify (&vs->dev, vq)) + goto again;Please write loops with
while()orfor().
Not with goto. goto is for error handling.This makes extra indention which is more ugly.
Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
Quoted 1 line
I don't care. No loops with goto and that's a hard rule.
It is not a loop.
If same code repeats, it's a loop.
We really need here is to call
vhost_get_vq_desconce. It's not a loop like other places to usewhile()orfor()withvhost_get_vq_desc.
Exactly. Like a mutex for example? We only need to lock it once. See __mutex_lock_common - uses a for loop, doesn't it?
So fix this please, use while or for or even until.
Quoted 307 lines
Quoted 304 lines
Quoted 49 lines
Quoted 39 lines
Quoted 39 lines
Quoted 38 lines
Quoted 25 lines
drivers/vhost/tcm_vhost.c +25
-0+ vs->vs_events_dropped = true; + goto out; + } + +if ((vq->iov[out].iov_len !=sizeof (struct virtio_scsi_event ))) { +vq_err (vq,"Expecting virtio_scsi_event, got %zu bytes\n" , + vq->iov[out].iov_len); + vs->vs_events_dropped = true; + goto out; + } + +if (vs->vs_events_dropped) { + event->event |=VIRTIO_SCSI_T_EVENTS_MISSED ; + vs->vs_events_dropped = false; + } + + eventp = vq->iov[out].iov_base; + ret =__copy_to_user (eventp, event,sizeof (*event)); +if (!ret) +vhost_add_used_and_signal (&vs->dev, vq, head,0 ); +else +vq_err (vq,"Faulted on tcm_vhost_send_event\n" ); +out: +mutex_unlock (&vq->mutex); +mutex_unlock (&vs->vs_events_lock);drivers/vhost/tcm_vhost.c static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd) · line 511
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+} + +static void tcm_vhost_evt_work (struct vhost_work *work) +{ + struct vhost_scsi *vs = container_of(work, struct vhost_scsi, + vs_event_work); +struct tcm_vhost_evt *evt; +struct llist_node *llnode; + + llnode =llist_del_all (&vs->vs_event_list);The assumption is that this is slow path thing, no need to worry about speed, yes? so let's do simple
list_add,list_deletc.Quoted 3 lines
Quoted 1 line
Why it is simpler?
simple
list_with a lock is easier to use correctly.You need to use it correctly anyway for cmd. Why you want a lock here?
Quoted 4 lines
Quoted 3 lines
We are using llist for cmd. Why it is better using
one for evt and the different for cmd? Similar code makes people easier to read.Quoted 6 lines
Quoted 4 lines
OK fair enough. But my idea is above to use a work structure instead of a list of events. This way we don't need extra list at all, no new locks, nothing.
Again, You are welcome to implement your ideas.
I might but I'm otherwise occupied at the moment.
Quoted 51 lines
Quoted 51 lines
Quoted 51 lines
Quoted 50 lines
Quoted 38 lines
drivers/vhost/tcm_vhost.c +31
-0+while (llnode) { + evt = llist_entry(llnode, struct tcm_vhost_evt, list); + llnode =llist_next (llnode); +tcm_vhost_do_evt_work (vs, evt); +tcm_vhost_free_evt (vs, evt); + } +} +/* Fill in status and signal that we are done processing this command * * This is scheduled in the vhost work queue so we are called with the owner @@ -807,9 +927,42 @@ static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)pr_debug ("%s: The handling func for control queue.\n" , __func__); } +static int tcm_vhost_send_evt (struct vhost_scsi *vs,struct tcm_vhost_tpg *tpg, +struct se_lun *lun,u32 event,u32 reason) +{ +struct tcm_vhost_evt *evt; + + evt =tcm_vhost_allocate_evt (vs, event, reason); +if (!evt) +return -ENOMEM ; + +if (tpg && lun) { +/* TODO: share lun setup code with virtio-scsi.ko */ + evt->event.lun[0 ] =0x01 ; + evt->event.lun[1 ] = tpg->tport_tpgt &0xFF ; +if (lun->unpacked_lun >=256 ) + evt->event.lun[2 ] = lun->unpacked_lun >>8 |0x40 ; + evt->event.lun[3 ] = lun->unpacked_lun &0xFF ; + } + +llist_add (&evt->list, &vs->vs_event_list); +vhost_work_queue (&vs->dev, &vs->vs_event_work); + +return 0 ; +}drivers/vhost/tcm_vhost.c static void vhost_scsi_ctl_handle_kick(struct vhost_work *work) · line 961
Quoted 10 lines
drivers/vhost/tcm_vhost.c +7
-1+static void vhost_scsi_evt_handle_kick (struct vhost_work *work) { -pr_debug ("%s: The handling func for event queue.\n" , __func__); + struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, + poll.work); + struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev); + +if (!tcm_vhost_check_endpoint (vq)) +return ;Again just drop this check.
Quoted 6 lines
Quoted 4 lines
Quoted 1 line
Why?
Same as above.
Because it's not safe to do this outside the vq mutex.Is it also not safe to the same in -net?
It's safe there since we dereference the pointer.
Quoted 1 line
Why is it not safe?
Because you don't dereference the pointer.
Quoted 182 lines
Quoted 62 lines
Quoted 62 lines
Quoted 61 lines
Quoted 49 lines
drivers/vhost/tcm_vhost.c +14
-0+ +if (tcm_vhost_check_events_dropped (vs)) +tcm_vhost_send_evt (vs,NULL ,NULL ,VIRTIO_SCSI_T_NO_EVENT ,0 ); + }static void vhost_scsi_handle_kick (struct vhost_work *work)@@ -833,6 +986,7 @@ static void vhost_scsi_flush(struct vhost_scsi *vs)for (i =0 ; i <VHOST_SCSI_MAX_VQ ; i++)vhost_scsi_flush_vq (vs, i);vhost_work_flush (&vs->dev, &vs->vs_completion_work); +vhost_work_flush (&vs->dev, &vs->vs_event_work); }/* @@ -891,6 +1045,7 @@ static int vhost_scsi_set_endpoint(return -EEXIST ; } tv_tpg->tv_tpg_vhost_count++; + tv_tpg->vhost_scsi = vs; vs_tpg[tv_tpg->tport_tpgt] = tv_tpg;smp_mb__after_atomic_inc (); match = true;@@ -974,6 +1129,7 @@ static int vhost_scsi_clear_endpoint( goto err_tpg; } tv_tpg->tv_tpg_vhost_count--; + tv_tpg->vhost_scsi =NULL ; vs->vs_tpg[target] =NULL ; match = true;mutex_unlock (&tv_tpg->tv_tpg_mutex);@@ -1033,6 +1189,11 @@ static int vhost_scsi_open(struct inode *inode, struct file *f)return -ENOMEM ;vhost_work_init (&s->vs_completion_work, vhost_scsi_complete_cmd_work); +vhost_work_init (&s->vs_event_work, tcm_vhost_evt_work); + + s->vs_events_nr =0 ; + s->vs_events_dropped = false; +mutex_init (&s->vs_events_lock); s->vqs[VHOST_SCSI_VQ_CTL ].handle_kick = vhost_scsi_ctl_handle_kick; s->vqs[VHOST_SCSI_VQ_EVT ].handle_kick = vhost_scsi_evt_handle_kick;@@ -1163,6 +1324,42 @@ static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport)return "Unknown" ; } +static int tcm_vhost_do_plug (struct tcm_vhost_tpg *tpg, +struct se_lun *lun,bool plug)drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1338
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+{ + +struct vhost_scsi *vs = tpg->vhost_scsi; +u32 reason; + +mutex_lock (&tpg->tv_tpg_mutex); + vs = tpg->vhost_scsi; +mutex_unlock (&tpg->tv_tpg_mutex); +if (!vs) +return -EOPNOTSUPP ;Why EOPNOTSUPP? When does this happen?
Quoted 4 lines
Quoted 2 lines
When
tpg->vhost_scsihas not been setup. E.g. no one starts a vhost-scsi guest.So ENODEV or something.
okay.
Quoted 32 lines
Quoted 32 lines
Quoted 30 lines
drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1347
Quoted 9 lines
drivers/vhost/tcm_vhost.c +9
-0+ +if (!tcm_vhost_check_feature (vs,VIRTIO_SCSI_F_HOTPLUG )) +return -EOPNOTSUPP ; + +if (plug) + reason =VIRTIO_SCSI_EVT_RESET_RESCAN ; +else + reason =VIRTIO_SCSI_EVT_RESET_REMOVED ; +You have dropped the tpg lock so
tpg->vhost_scsican become NULL now. Why is this safe?Quoted 5 lines
drivers/vhost/tcm_vhost.c +5
-0+return tcm_vhost_send_evt (vs, tpg, lun, +VIRTIO_SCSI_T_TRANSPORT_RESET , + reason); +} +drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1362
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+static int tcm_vhost_hotplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, true); +} + +static int tcm_vhost_hotunplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, false); +} +What are these wrappers for? Seem useless ...
Quoted 7 lines
Quoted 2 lines
It make people easier to understand what's the true and false is about in
tcm_vhost_do_plug.So just pass in the reason and not a bool.
When readers see VIRTIO_SCSI_EVT_RESET_RESCAN/VIRTIO_SCSI_EVT_RESET_REMOVED they will know what is going on.How can you make the connection that
VIRTIO_SCSI_EVT_RESET_RESCANis hotplug andVIRTIO_SCSI_EVT_RESET_REMOVEDis hot-unplug at first glance?Quoted 67 lines
Quoted 67 lines
Quoted 64 lines
Quoted 64 lines
drivers/vhost/tcm_vhost.c +5
-2static int tcm_vhost_port_link (struct se_portal_group *se_tpg,struct se_lun *lun) {@@ -1173,18 +1370,21 @@ static int tcm_vhost_port_link(struct se_portal_group *se_tpg, tv_tpg->tv_tpg_port_count++;mutex_unlock (&tv_tpg->tv_tpg_mutex); +tcm_vhost_hotplug (tv_tpg, lun); +return 0 ; }static void tcm_vhost_port_unlink (struct se_portal_group *se_tpg, -struct se_lun *se_lun) +struct se_lun *lun) {struct tcm_vhost_tpg *tv_tpg =container_of (se_tpg, struct tcm_vhost_tpg, se_tpg); -mutex_lock (&tv_tpg->tv_tpg_mutex); tv_tpg->tv_tpg_port_count--;mutex_unlock (&tv_tpg->tv_tpg_mutex); + +tcm_vhost_hotunplug (tv_tpg, lun); } static struct se_node_acl *tcm_vhost_make_nodeacl(drivers/vhost/tcm_vhost.h +10
-0@@ -53,6 +53,7 @@ struct tcm_vhost_nacl {struct se_node_acl se_node_acl; }; +struct vhost_scsi ;struct tcm_vhost_tpg {/* Vhost port target portal group tag for TCM */ u16 tport_tpgt;@@ -70,6 +71,8 @@ struct tcm_vhost_tpg {struct tcm_vhost_tport *tport;/* Returned by tcm_vhost_make_tpg() */ struct se_portal_group se_tpg; +/* Pointer back to struct vhost_scsi, protected by tv_tpg_mutex */ +struct vhost_scsi *vhost_scsi; };struct tcm_vhost_tport {@@ -83,6 +86,13 @@ struct tcm_vhost_tport {struct se_wwn tport_wwn; }; +struct tcm_vhost_evt { +/* virtio_scsi event */ +struct virtio_scsi_event event; +/* virtio_scsi event list, serviced from vhost worker thread */ +struct llist_node list; +}; +/* * As per request from MST, keep TCM_VHOST related ioctl defines out of * linux/vhost.h (user-space) for now.. Signature · 1 line
1.8.1.4Signature · 1 line
AsiasSignature · 1 line
AsiasSignature · 1 line
Asias
Re: [PATCH v7 3/3] tcm_vhost: Add hotplug/hotunplug support
Quoted 98 lines · Michael S. Tsirkin
Quoted 97 lines · Asias He
Quoted 96 lines · Michael S. Tsirkin
Quoted 94 lines · Asias He
Quoted 82 lines · Michael S. Tsirkin
Quoted 76 lines · Asias He
Quoted 62 lines · Michael S. Tsirkin
Quoted 61 lines · Asias He
In commit
365a7150094([SCSI] virtio-scsi: hotplug support for virtio-scsi), hotplug support is added to virtio-scsi.This patch adds hotplug and hotunplug support to
tcm_vhost.You can create or delete a LUN in targetcli to hotplug or hotunplug a LUN in guest.
Changes in v7:
- Addvhost_work_flushforvs->vs_event_workto this seriesChanges in v6:
- Passtcm_vhost_evttotcm_vhost_do_evt_workChanges in v5:
- Switch to int from u64 tovs_events_nr
- Sets->vs_events_droppedflag intcm_vhost_allocate_evt
- Do not nest dev mutex within vq mutex
- Usevs_events_lockto protectvs_events_droppedandvs_events_nr
- Rebase to target/masterChanges in v4:
- Droptcm_vhost_check_endpointintcm_vhost_send_evt
- Addtcm_vhost_check_endpointinvhost_scsi_evt_handle_kickChanges in v3:
- Separate the bug fix to another threadChanges in v2:
- Remove code duplication in tcm_vhost_{hotplug,hotunplug}
- Fix racing ofvs_events_nr
- Add flush fix patch to this seriesSigned-off-by Asias He <[email protected]>Reviewed-by Stefan Hajnoczi <[email protected]>--- drivers/vhost/tcm_vhost.c | 208 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 +++ 2 files changed, 214 insertions(+), 4 deletions(-)drivers/vhost/tcm_vhost.c +3
-1@@ -66,11 +66,13 @@ enum {* TODO: debug and remove the workaround. */ enum { -VHOST_SCSI_FEATURES =VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX ) +VHOST_SCSI_FEATURES = (VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX )) | + (1ULL <<VIRTIO_SCSI_F_HOTPLUG ) };#define VHOST_SCSI_MAX_TARGET 256#define VHOST_SCSI_MAX_VQ 128 +#define VHOST_SCSI_MAX_EVENT 128struct vhost_scsi {/* Protected by vhost_scsi->dev.mutex */ @@ -82,6 +84,13 @@ struct vhost_scsi {Quoted 12 lines
drivers/vhost/tcm_vhost.c struct vhost_scsi { · line 91
Quoted 7 lines
drivers/vhost/tcm_vhost.c +5
-0struct vhost_work vs_completion_work;/* cmd completion work item */ struct llist_head vs_completion_list;/* cmd completion queue */ + +struct vhost_work vs_event_work;/* evt injection work item */ +struct llist_head vs_event_list;/* evt injection queue */ + +struct mutex vs_events_lock;/* protect vs_events_dropped,events_nr */ Looking at this code, there are just so many locks now. This does not make me feel safe :)
At least, please document lock nesting.Quoted 1 line
Do you see a real problem?
Complexity is a real problem. My head already spins. No I don't see a bug, but we need to simplify locking.
Quoted 8 lines
And I think I see a nice way to do this:
1. move away from a global work to per-event work - so no list
2. remove dynamic allocation of events - so noevents_nr
3. set/test overrun flag under the appropriate vq mutexI think that's ideal. But we can move there in small steps. As a first step - why can't we always take the vq mutex lock and drop
vs_events_lock?There are really different ways to solve the same problem. You are welcome to implement you ideas.
Quoted 7 lines
Quoted 4 lines
Quoted 2 lines
I have a problem with this patch. It just seems too tricky, while it is really trying to do a very simple thing.
How tricky is it?
It has its own locking scheme which seems unwarranted. Find a way to use the existing locking please.
Why is this tricky?
Quoted 6 lines
Quoted 5 lines
Quoted 1 line
There's new lock nesting.
I used dev mutex in the past, but it does not work because it introduces a deadlock in
vhost_scsi_set_endpoint.vs_eventsis a per device stat, so I think it deserves a per device lock instead of a per queue lock.
Quoted 8 lines
We don't add a lock per field, it just does not make sense.
Quoted 4 lines
Quoted 2 lines
There are new places which do lock, read
data, unlock, use data.With
vs_events_lock?With this patch.
Please be explicit.
Quoted 10 lines
Quoted 3 lines
Quoted 1 line
There's new RCU.
It is not a *new* RCU. It uses the existing one
vq->private_data.Incorrectly. It should be used on the workqueue where it's dereferenced.
Quoted 3 lines
Quoted 2 lines
All this feels fragile, and
will be hard to maintain even though I don't see bugs.
Quoted 7 lines
Quoted 5 lines
Quoted 3 lines
So I would prefer it if you tried to simplify
this code making it easier to maintain. I sent some ideas on how to do this, but if you like, do it some other way.I think it is pretty simple already.
Here, we disagree.
Where does 'we' come from?
Quoted 4 lines
And since I'm one of the people who gets to maintain this, it's important that I'm comfortable with the code, and don't keep worrying "is this racy? is there a deadlock?". This patch makes me uncomfortable. Could you address this please?
No, I can not until I really see the real problem. You make me uncomfortable as well.
Quoted 62 lines
Quoted 62 lines
Quoted 62 lines
I also made some comments about the coding style but these are less important (except goto again I guess) if these will be te only issues, I'd say we can apply.
Quoted 56 lines
Quoted 51 lines
Quoted 51 lines
Quoted 50 lines
Quoted 38 lines
drivers/vhost/tcm_vhost.c +24
-0+bool vs_events_dropped;/* any missed events */ +int vs_events_nr;/* num of pending events */ };/* Local pointer to allocated TCM configfs fabric module */ @@ -129,6 +138,17 @@ static bool tcm_vhost_check_endpoint(struct vhost_virtqueue *vq)return ret; } +static bool tcm_vhost_check_events_dropped (struct vhost_scsi *vs) +{ +bool ret; + +mutex_lock (&vs->vs_events_lock); + ret = vs->vs_events_dropped; +mutex_unlock (&vs->vs_events_lock); + +return ret; +} +static int tcm_vhost_check_true (struct se_portal_group *se_tpg) {return 1 ;@@ -379,6 +399,37 @@ static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd)return 0 ; } +static void tcm_vhost_free_evt (struct vhost_scsi *vs,struct tcm_vhost_evt *evt) +{ +mutex_lock (&vs->vs_events_lock); + vs->vs_events_nr--; +kfree (evt); +mutex_unlock (&vs->vs_events_lock); +} + +static struct tcm_vhost_evt *tcm_vhost_allocate_evt (struct vhost_scsi *vs, +u32 event,u32 reason) +{drivers/vhost/tcm_vhost.c static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd) · line 422
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+struct tcm_vhost_evt *evt; + +mutex_lock (&vs->vs_events_lock); +if (vs->vs_events_nr >VHOST_SCSI_MAX_EVENT ) { + vs->vs_events_dropped = true; +mutex_unlock (&vs->vs_events_lock); +return NULL ; + } + + evt =kzalloc (sizeof (*evt),GFP_KERNEL );I think kzalloc not needed here, you init all fields.
Quoted 3 lines
Quoted 1 line
Not really! evt->event.lun[4-7] is not initialized. It needs to be 0.
So that is 4 bytes just init them when you set rest of lun.
It is not in the fast path. You can do it this way but not a must.
Quoted 7 lines
Quoted 3 lines
Quoted 1 line
I think that's a bit cleaner than relying on kzalloc to zero-initialize.
I think it is a bit shorter, 4 lines saved.
OTOH you don't have to think "what about the rest of the bytes?" then hunt through the code and go "ah, it's kzalloc".
Looks it's a style issue but I prefer explicit initialization.
It is just make none sense to argue about this. If you really want you can make patches to remove all the lines where use kzalloc because it is cleaner.
Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
Quoted 12 lines
Quoted 11 lines
Quoted 7 lines
Quoted 5 lines
Also, this basically means if backend does plug/unplug very quickly, we start consuming kernel memory without a limit. Not good. How about we allocate the event as part of target?
There shouldn't ever be more than one hotplug
event in flight per target, right?It is limited by
VHOST_SCSI_MAX_EVENT.OK I missed that. But where does
VHOST_SCSI_MAX_EVENTcome from by the way? I don't see it in spec or did I miss it? It seems it's best not to lost events as long as there are descriptors in the event vq.
Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
Quoted 1 line
You wanted this.
Yes I wanted some limit on the amount of memory we use up :)
So we limited this already.
Yes. But my question is this: what happens if an event is lost? Can guest recover and how?
VIRTIO_SCSI_T_NO_EVENT.
Quoted 69 lines
Quoted 69 lines
Quoted 69 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
drivers/vhost/tcm_vhost.c static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd) · line 423
Quoted 2 lines
drivers/vhost/tcm_vhost.c +1
-0+if (evt) {Let's do clean error handling here and above:
if (!evt)goto err;Quoted 7 lines
Quoted 5 lines
Quoted 2 lines
We can you do in err? You simply unlock and return. Why bother? How cleaner it will be.
There's another error above and we'll share code.
It's not a big issue, just a bit nicer IMHO.Can't we focus on real issues other than this?
Yes style issues are not the most important here.
Quoted 51 lines
Quoted 46 lines
Quoted 46 lines
Quoted 45 lines
Quoted 32 lines
drivers/vhost/tcm_vhost.c +25
-0+ evt->event.event = event; + evt->event.reason = reason; + vs->vs_events_nr++; + } +mutex_unlock (&vs->vs_events_lock); + +return evt; +} +static void vhost_scsi_free_cmd (struct tcm_vhost_cmd *tv_cmd) {struct se_cmd *se_cmd = &tv_cmd->tvc_se_cmd;@@ -397,6 +448,75 @@ static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd)kfree (tv_cmd); } +static void tcm_vhost_do_evt_work (struct vhost_scsi *vs, +struct tcm_vhost_evt *evt) +{ +struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT ]; +struct virtio_scsi_event *event = &evt->event; +struct virtio_scsi_event __user *eventp; +unsigned out, in; +int head, ret; + +if (!tcm_vhost_check_endpoint (vq)) +return ; + +mutex_lock (&vs->vs_events_lock); +mutex_lock (&vq->mutex); +again: +vhost_disable_notify (&vs->dev, vq);drivers/vhost/tcm_vhost.c static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd) · line 476
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+ head =vhost_get_vq_desc (&vs->dev, vq, vq->iov, +ARRAY_SIZE (vq->iov), &out, &in, +NULL ,NULL ); +if (head <0 ) { + vs->vs_events_dropped = true; + goto out; + } +if (head == vq->num) { +if (vhost_enable_notify (&vs->dev, vq)) + goto again;Please write loops with
while()orfor().
Not with goto. goto is for error handling.Quoted 3 lines
Quoted 1 line
This makes extra indention which is more ugly.
I don't care. No loops with goto and that's a hard rule.
It is not a loop.
Quoted 8 lines
Quoted 4 lines
Quoted 1 line
If same code repeats, it's a loop.
We really need here is to call
vhost_get_vq_desconce. It's not a loop like other places to usewhile()orfor()withvhost_get_vq_desc.Exactly. Like a mutex for example? We only need to lock it once. See
__mutex_lock_common- uses a for loop, doesn't it?
So what?
Quoted 1 line
So fix this please, use while or for or even until.
I do not think it is necessary.
Quoted 307 lines
Quoted 307 lines
Quoted 304 lines
Quoted 49 lines
Quoted 39 lines
Quoted 39 lines
Quoted 38 lines
Quoted 25 lines
drivers/vhost/tcm_vhost.c +25
-0+ vs->vs_events_dropped = true; + goto out; + } + +if ((vq->iov[out].iov_len !=sizeof (struct virtio_scsi_event ))) { +vq_err (vq,"Expecting virtio_scsi_event, got %zu bytes\n" , + vq->iov[out].iov_len); + vs->vs_events_dropped = true; + goto out; + } + +if (vs->vs_events_dropped) { + event->event |=VIRTIO_SCSI_T_EVENTS_MISSED ; + vs->vs_events_dropped = false; + } + + eventp = vq->iov[out].iov_base; + ret =__copy_to_user (eventp, event,sizeof (*event)); +if (!ret) +vhost_add_used_and_signal (&vs->dev, vq, head,0 ); +else +vq_err (vq,"Faulted on tcm_vhost_send_event\n" ); +out: +mutex_unlock (&vq->mutex); +mutex_unlock (&vs->vs_events_lock);drivers/vhost/tcm_vhost.c static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd) · line 511
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+} + +static void tcm_vhost_evt_work (struct vhost_work *work) +{ + struct vhost_scsi *vs = container_of(work, struct vhost_scsi, + vs_event_work); +struct tcm_vhost_evt *evt; +struct llist_node *llnode; + + llnode =llist_del_all (&vs->vs_event_list);The assumption is that this is slow path thing, no need to worry about speed, yes? so let's do simple
list_add,list_deletc.Quoted 3 lines
Quoted 1 line
Why it is simpler?
simple
list_with a lock is easier to use correctly.You need to use it correctly anyway for cmd. Why you want a lock here?
Quoted 4 lines
Quoted 3 lines
We are using llist for cmd. Why it is better using
one for evt and the different for cmd? Similar code makes people easier to read.Quoted 6 lines
Quoted 4 lines
OK fair enough. But my idea is above to use a work structure instead of a list of events. This way we don't need extra list at all, no new locks, nothing.
Again, You are welcome to implement your ideas.
I might but I'm otherwise occupied at the moment.
Quoted 51 lines
Quoted 51 lines
Quoted 51 lines
Quoted 50 lines
Quoted 38 lines
drivers/vhost/tcm_vhost.c +31
-0+while (llnode) { + evt = llist_entry(llnode, struct tcm_vhost_evt, list); + llnode =llist_next (llnode); +tcm_vhost_do_evt_work (vs, evt); +tcm_vhost_free_evt (vs, evt); + } +} +/* Fill in status and signal that we are done processing this command * * This is scheduled in the vhost work queue so we are called with the owner @@ -807,9 +927,42 @@ static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)pr_debug ("%s: The handling func for control queue.\n" , __func__); } +static int tcm_vhost_send_evt (struct vhost_scsi *vs,struct tcm_vhost_tpg *tpg, +struct se_lun *lun,u32 event,u32 reason) +{ +struct tcm_vhost_evt *evt; + + evt =tcm_vhost_allocate_evt (vs, event, reason); +if (!evt) +return -ENOMEM ; + +if (tpg && lun) { +/* TODO: share lun setup code with virtio-scsi.ko */ + evt->event.lun[0 ] =0x01 ; + evt->event.lun[1 ] = tpg->tport_tpgt &0xFF ; +if (lun->unpacked_lun >=256 ) + evt->event.lun[2 ] = lun->unpacked_lun >>8 |0x40 ; + evt->event.lun[3 ] = lun->unpacked_lun &0xFF ; + } + +llist_add (&evt->list, &vs->vs_event_list); +vhost_work_queue (&vs->dev, &vs->vs_event_work); + +return 0 ; +}drivers/vhost/tcm_vhost.c static void vhost_scsi_ctl_handle_kick(struct vhost_work *work) · line 961
Quoted 10 lines
drivers/vhost/tcm_vhost.c +7
-1+static void vhost_scsi_evt_handle_kick (struct vhost_work *work) { -pr_debug ("%s: The handling func for event queue.\n" , __func__); + struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, + poll.work); + struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev); + +if (!tcm_vhost_check_endpoint (vq)) +return ;Again just drop this check.
Quoted 6 lines
Quoted 4 lines
Quoted 1 line
Why?
Same as above.
Because it's not safe to do this outside the vq mutex.Is it also not safe to the same in -net?
It's safe there since we dereference the pointer.
Quoted 1 line
Why is it not safe?
Because you don't dereference the pointer.
Quoted 182 lines
Quoted 62 lines
Quoted 62 lines
Quoted 61 lines
Quoted 49 lines
drivers/vhost/tcm_vhost.c +14
-0+ +if (tcm_vhost_check_events_dropped (vs)) +tcm_vhost_send_evt (vs,NULL ,NULL ,VIRTIO_SCSI_T_NO_EVENT ,0 ); + }static void vhost_scsi_handle_kick (struct vhost_work *work)@@ -833,6 +986,7 @@ static void vhost_scsi_flush(struct vhost_scsi *vs)for (i =0 ; i <VHOST_SCSI_MAX_VQ ; i++)vhost_scsi_flush_vq (vs, i);vhost_work_flush (&vs->dev, &vs->vs_completion_work); +vhost_work_flush (&vs->dev, &vs->vs_event_work); }/* @@ -891,6 +1045,7 @@ static int vhost_scsi_set_endpoint(return -EEXIST ; } tv_tpg->tv_tpg_vhost_count++; + tv_tpg->vhost_scsi = vs; vs_tpg[tv_tpg->tport_tpgt] = tv_tpg;smp_mb__after_atomic_inc (); match = true;@@ -974,6 +1129,7 @@ static int vhost_scsi_clear_endpoint( goto err_tpg; } tv_tpg->tv_tpg_vhost_count--; + tv_tpg->vhost_scsi =NULL ; vs->vs_tpg[target] =NULL ; match = true;mutex_unlock (&tv_tpg->tv_tpg_mutex);@@ -1033,6 +1189,11 @@ static int vhost_scsi_open(struct inode *inode, struct file *f)return -ENOMEM ;vhost_work_init (&s->vs_completion_work, vhost_scsi_complete_cmd_work); +vhost_work_init (&s->vs_event_work, tcm_vhost_evt_work); + + s->vs_events_nr =0 ; + s->vs_events_dropped = false; +mutex_init (&s->vs_events_lock); s->vqs[VHOST_SCSI_VQ_CTL ].handle_kick = vhost_scsi_ctl_handle_kick; s->vqs[VHOST_SCSI_VQ_EVT ].handle_kick = vhost_scsi_evt_handle_kick;@@ -1163,6 +1324,42 @@ static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport)return "Unknown" ; } +static int tcm_vhost_do_plug (struct tcm_vhost_tpg *tpg, +struct se_lun *lun,bool plug)drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1338
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+{ + +struct vhost_scsi *vs = tpg->vhost_scsi; +u32 reason; + +mutex_lock (&tpg->tv_tpg_mutex); + vs = tpg->vhost_scsi; +mutex_unlock (&tpg->tv_tpg_mutex); +if (!vs) +return -EOPNOTSUPP ;Why EOPNOTSUPP? When does this happen?
Quoted 4 lines
Quoted 2 lines
When
tpg->vhost_scsihas not been setup. E.g. no one starts a vhost-scsi guest.So ENODEV or something.
okay.
Quoted 32 lines
Quoted 32 lines
Quoted 30 lines
drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1347
Quoted 9 lines
drivers/vhost/tcm_vhost.c +9
-0+ +if (!tcm_vhost_check_feature (vs,VIRTIO_SCSI_F_HOTPLUG )) +return -EOPNOTSUPP ; + +if (plug) + reason =VIRTIO_SCSI_EVT_RESET_RESCAN ; +else + reason =VIRTIO_SCSI_EVT_RESET_REMOVED ; +You have dropped the tpg lock so
tpg->vhost_scsican become NULL now. Why is this safe?Quoted 5 lines
drivers/vhost/tcm_vhost.c +5
-0+return tcm_vhost_send_evt (vs, tpg, lun, +VIRTIO_SCSI_T_TRANSPORT_RESET , + reason); +} +drivers/vhost/tcm_vhost.c static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) · line 1362
Quoted 10 lines
drivers/vhost/tcm_vhost.c +10
-0+static int tcm_vhost_hotplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, true); +} + +static int tcm_vhost_hotunplug (struct tcm_vhost_tpg *tpg,struct se_lun *lun) +{ +return tcm_vhost_do_plug (tpg, lun, false); +} +What are these wrappers for? Seem useless ...
Quoted 7 lines
Quoted 2 lines
It make people easier to understand what's the true and false is about in
tcm_vhost_do_plug.So just pass in the reason and not a bool.
When readers see VIRTIO_SCSI_EVT_RESET_RESCAN/VIRTIO_SCSI_EVT_RESET_REMOVED they will know what is going on.How can you make the connection that
VIRTIO_SCSI_EVT_RESET_RESCANis hotplug andVIRTIO_SCSI_EVT_RESET_REMOVEDis hot-unplug at first glance?Quoted 67 lines
Quoted 67 lines
Quoted 64 lines
Quoted 64 lines
drivers/vhost/tcm_vhost.c +5
-2static int tcm_vhost_port_link (struct se_portal_group *se_tpg,struct se_lun *lun) {@@ -1173,18 +1370,21 @@ static int tcm_vhost_port_link(struct se_portal_group *se_tpg, tv_tpg->tv_tpg_port_count++;mutex_unlock (&tv_tpg->tv_tpg_mutex); +tcm_vhost_hotplug (tv_tpg, lun); +return 0 ; }static void tcm_vhost_port_unlink (struct se_portal_group *se_tpg, -struct se_lun *se_lun) +struct se_lun *lun) {struct tcm_vhost_tpg *tv_tpg =container_of (se_tpg, struct tcm_vhost_tpg, se_tpg); -mutex_lock (&tv_tpg->tv_tpg_mutex); tv_tpg->tv_tpg_port_count--;mutex_unlock (&tv_tpg->tv_tpg_mutex); + +tcm_vhost_hotunplug (tv_tpg, lun); } static struct se_node_acl *tcm_vhost_make_nodeacl(drivers/vhost/tcm_vhost.h +10
-0@@ -53,6 +53,7 @@ struct tcm_vhost_nacl {struct se_node_acl se_node_acl; }; +struct vhost_scsi ;struct tcm_vhost_tpg {/* Vhost port target portal group tag for TCM */ u16 tport_tpgt;@@ -70,6 +71,8 @@ struct tcm_vhost_tpg {struct tcm_vhost_tport *tport;/* Returned by tcm_vhost_make_tpg() */ struct se_portal_group se_tpg; +/* Pointer back to struct vhost_scsi, protected by tv_tpg_mutex */ +struct vhost_scsi *vhost_scsi; };struct tcm_vhost_tport {@@ -83,6 +86,13 @@ struct tcm_vhost_tport {struct se_wwn tport_wwn; }; +struct tcm_vhost_evt { +/* virtio_scsi event */ +struct virtio_scsi_event event; +/* virtio_scsi event list, serviced from vhost worker thread */ +struct llist_node list; +}; +/* * As per request from MST, keep TCM_VHOST related ioctl defines out of * linux/vhost.h (user-space) for now.. Signature · 1 line
1.8.1.4Signature · 1 line
AsiasSignature · 1 line
AsiasSignature · 1 line
Asias
Signature · 1 line
Asias
Re: [PATCH v7 3/3] tcm_vhost: Add hotplug/hotunplug support
Quoted 15 lines · Asias He
Quoted 14 lines · Michael S. Tsirkin
Quoted 13 lines
Quoted 11 lines
Quoted 9 lines
Quoted 7 lines
Quoted 5 lines
Quoted 3 lines
Review
Quoted 1 line
(quoted diff) +1
-0+ evt = kzalloc(sizeof(*evt), GFP_KERNEL);I think kzalloc not needed here, you init all fields.
Not really! evt->event.lun[4-7] is not initialized. It needs to be 0.
So that is 4 bytes just init them when you set rest of lun.
It is not in the fast path. You can do it this way but not a must.
I think that's a bit cleaner than relying on kzalloc to zero-initialize.
I think it is a bit shorter, 4 lines saved.
Quoted 7 lines
Quoted 3 lines
OTOH you don't have to think "what about the rest of the bytes?" then hunt through the code and go "ah, it's kzalloc".
Looks it's a style issue but I prefer explicit initialization.It is just make none sense to argue about this. If you really want you can make patches to remove all the lines where use kzalloc because it is cleaner.
I prefer explicit initialization too, FWIW.
Quoted 45 lines
Quoted 36 lines
Quoted 36 lines
Quoted 36 lines
Quoted 35 lines
Quoted 30 lines
Quoted 30 lines
Quoted 29 lines
Quoted 16 lines
(quoted diff) +16
-0+static void tcm_vhost_do_evt_work(struct vhost_scsi *vs, + struct tcm_vhost_evt *evt) +{ + struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT]; + struct virtio_scsi_event *event = &evt->event; + struct virtio_scsi_event __user *eventp; + unsigned out, in; + int head, ret; + + if (!tcm_vhost_check_endpoint(vq)) + return; + + mutex_lock(&vs->vs_events_lock); + mutex_lock(&vq->mutex); +again: + vhost_disable_notify(&vs->dev, vq);Review
Quoted 10 lines
(quoted diff) +10
-0+ head = vhost_get_vq_desc(&vs->dev, vq, vq->iov, + ARRAY_SIZE(vq->iov), &out, &in, + NULL, NULL); + if (head < 0) { + vs->vs_events_dropped = true; + goto out; + } + if (head == vq->num) { + if (vhost_enable_notify(&vs->dev, vq)) + goto again;Please write loops with
while()orfor().
Not with goto. goto is for error handling.Quoted 3 lines
Quoted 1 line
This makes extra indention which is more ugly.
I don't care. No loops with goto and that's a hard rule.
It is not a loop.
Quoted 8 lines
Quoted 4 lines
Quoted 1 line
If same code repeats, it's a loop.
We really need here is to call
vhost_get_vq_desconce. It's not a loop like other places to usewhile()orfor()withvhost_get_vq_desc.Exactly. Like a mutex for example? We only need to lock it once. See
__mutex_lock_common- uses a for loop, doesn't it?
Quoted 5 lines
So what?
Quoted 1 line
So fix this please, use while or for or even until.
I do not think it is necessary.
I tend to use again: in such corner cases, where I don't expect it to be invoked very often. A do loop looks too normal.
I do exactly the same thing in virtio-net.c, for the same case.
Would you like me to review the entire thing?
Cheers,
Rusty.
Re: [PATCH v7 3/3] tcm_vhost: Add hotplug/hotunplug support
Quoted 20 lines · Rusty Russell
Quoted 19 lines · Asias He
Quoted 17 lines · Michael S. Tsirkin
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines
Quoted 3 lines
Review
Quoted 1 line
(quoted diff) +1
-0+ evt = kzalloc(sizeof(*evt), GFP_KERNEL);I think kzalloc not needed here, you init all fields.
Not really! evt->event.lun[4-7] is not initialized. It needs to be 0.
Quoted 7 lines
Quoted 5 lines
Quoted 3 lines
Quoted 1 line
So that is 4 bytes just init them when you set rest of lun.
It is not in the fast path. You can do it this way but not a must.
I think that's a bit cleaner than relying on kzalloc to zero-initialize.
I think it is a bit shorter, 4 lines saved.
OTOH you don't have to think "what about the rest of the bytes?" then hunt through the code and go "ah, it's kzalloc".
Looks it's a style issue but I prefer explicit initialization.
Quoted 5 lines
Quoted 3 lines
It is just make none sense to argue about this. If you really want you can make patches to remove all the lines where use kzalloc because it is cleaner.
I prefer explicit initialization too, FWIW.
I do not really care. I am OK with either way.
Quoted 49 lines
Quoted 49 lines
Quoted 36 lines
Quoted 36 lines
Quoted 36 lines
Quoted 35 lines
Quoted 30 lines
Quoted 30 lines
Quoted 29 lines
Quoted 16 lines
(quoted diff) +16
-0+static void tcm_vhost_do_evt_work(struct vhost_scsi *vs, + struct tcm_vhost_evt *evt) +{ + struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT]; + struct virtio_scsi_event *event = &evt->event; + struct virtio_scsi_event __user *eventp; + unsigned out, in; + int head, ret; + + if (!tcm_vhost_check_endpoint(vq)) + return; + + mutex_lock(&vs->vs_events_lock); + mutex_lock(&vq->mutex); +again: + vhost_disable_notify(&vs->dev, vq);Review
Quoted 10 lines
(quoted diff) +10
-0+ head = vhost_get_vq_desc(&vs->dev, vq, vq->iov, + ARRAY_SIZE(vq->iov), &out, &in, + NULL, NULL); + if (head < 0) { + vs->vs_events_dropped = true; + goto out; + } + if (head == vq->num) { + if (vhost_enable_notify(&vs->dev, vq)) + goto again;Please write loops with
while()orfor().
Not with goto. goto is for error handling.Quoted 3 lines
Quoted 1 line
This makes extra indention which is more ugly.
I don't care. No loops with goto and that's a hard rule.
It is not a loop.
Quoted 8 lines
Quoted 4 lines
Quoted 1 line
If same code repeats, it's a loop.
We really need here is to call
vhost_get_vq_desconce. It's not a loop like other places to usewhile()orfor()withvhost_get_vq_desc.Exactly. Like a mutex for example? We only need to lock it once. See
__mutex_lock_common- uses a for loop, doesn't it?So what?
Quoted 1 line
So fix this please, use while or for or even until.
Quoted 8 lines
Quoted 1 line
I do not think it is necessary.
I tend to use again: in such corner cases, where I don't expect it to be invoked very often. A do loop looks too normal.
I do exactly the same thing in
virtio-net.c, for the same case.Would you like me to review the entire thing?
Sure. Thanks Rusty!
Quoted 2 lines
Cheers,
Rusty.
Signature · 1 line
Asias
Michael S. Tsirkin
maintainerQuoted 7 lines · Asias He
Changes in v7:
- Addvhost_work_flushforvs->vs_event_workto this seriesAsias He (3):
tcm_vhost: Introducetcm_vhost_check_feature()
tcm_vhost: Add helper to check if endpoint is setup
tcm_vhost: Add hotplug/hotunplug support
Okay it looks like I missed some issues in the previous review round. Sorry about that. Sent more comments now.
They are kind of minor so if already applied,
we can fix them incrementally I think.
Quoted 6 lines
drivers/vhost/tcm_vhost.c | 238 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 ++ 2 files changed, 244 insertions(+), 4 deletions(-)Signature · 1 line
1.8.1.4
v7 cover tcm_vhost hotplug
In reply to Michael S. TsirkinChanges in v7:
- Add vhost_work_flush for vs->vs_event_work to this series
Asias He (3):
tcm_vhost: Introduce tcm_vhost_check_feature()
tcm_vhost: Add helper to check if endpoint is setup
tcm_vhost: Add hotplug/hotunplug support
drivers/vhost/tcm_vhost.c | 238 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 ++ 2 files changed, 244 insertions(+), 4 deletions(-)
Signature · 1 line
1.8.1.4
Quoted 21 lines · Michael S. Tsirkin
Quoted 19 lines · Nicholas A. Bellinger
Hi MST,
Ping on this series for use with vhost-scsi-pci code..?
They will need to be heading into target-pending/for-next soon to make the v3.10 window opening in t-minus 9 days. ;)
--nab
Quoted 9 lines · Asias He
Asias He (3):
tcm_vhost: Introducetcm_vhost_check_feature()
tcm_vhost: Add helper to check if endpoint is setup
tcm_vhost: Add hotplug/hotunplug supportdrivers/vhost/tcm_vhost.c | 236 +++++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 ++ 2 files changed, 242 insertions(+), 4 deletions(-)
Quoted 4 lines
Okay, Paolo says it's now a sure thing we'll use it in qemu, in fact in 1.5, so let's cross the fingers and start merging new code.
Nicholas, please take v7. It contains one hunk belongs to this series.
Signature · 1 line
Asias