dm-netlink: Add netlink events to dm

Index: linux-2.6-patched/drivers/md/dm-netlink.c
===================================================================
@@ -0,0 +1,96 @@
+/*
+ * Device Mapper Netlink Support (dm-netlink)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Copyright IBM Corporation, 2005, 2006
+ * 	Author: Mike Anderson <[email protected]>
+ */
+#include <linux/module.h>
+#include <linux/mempool.h>
+#include <linux/time.h>
+#include <linux/jiffies.h>
+#include <linux/security.h>
+#include <net/sock.h>
+#include <net/netlink.h>
+#include "dm.h"
+
+#define EVENT_SKB_SIZE	NLMSG_GOODSIZE
+
+struct dm_event_cache {
+	struct kmem_cache *cache;
+	unsigned skb_size;
+};
+
+static struct dm_event_cache dme_cache;
+
+static int dme_cache_init(struct dm_event_cache *dc, unsigned skb_size)
+{
+	dc->skb_size = skb_size;
+
+	dc->cache = kmem_cache_create("dm_events",
+			   sizeof(struct dm_event), 0, 0, NULL, NULL);
+
+	if (!dc->cache)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void dme_cache_destroy(struct dm_event_cache *dc)
+{
+	kmem_cache_destroy(dc->cache);
+}
+
+static void dme_cache_event_put(struct dm_event *evt)
+{
+	struct dm_event_cache *dc = evt->cdata;
+	kmem_cache_free(dc->cache, evt);
+}
+
+static struct dm_event* dme_cache_event_get(struct dm_event_cache *dc)
+{
+	struct dm_event *evt;
+
+	evt = kmem_cache_alloc(dc->cache, GFP_ATOMIC);
+	if (evt) {
+		evt->cdata = dc;
+		evt->skb = alloc_skb(dc->skb_size, GFP_ATOMIC);
+		if (!evt->skb)
+			goto cache_out;
+	}
+
+	return evt;
+
+cache_out:
+	dme_cache_event_put(evt);
+	return NULL;
+}
+
+int __init dm_netlink_init(void)
+{
+	int err;
+
+	err = dme_cache_init(&dme_cache, EVENT_SKB_SIZE);
+	if (!err)
+		printk(KERN_DEBUG "dm-netlink version 0.0.6 loaded\n");
+
+	return err;
+}
+
+void dm_netlink_exit(void)
+{
+	dme_cache_destroy(&dme_cache);
+}

Index: linux-2.6-patched/include/linux/netlink.h
===================================================================
@@ -21,7 +21,7 @@
 #define NETLINK_DNRTMSG		14	/* DECnet routing messages */
 #define NETLINK_KOBJECT_UEVENT	15	/* Kernel messages to userspace */
 #define NETLINK_GENERIC		16
-/* leave room for NETLINK_DM (DM Events) */
+#define NETLINK_DM		17	/* Device Mapper */
 #define NETLINK_SCSITRANSPORT	18	/* SCSI Transports */
 
 #define MAX_LINKS 32		

42 lines not on this page · raw message

Index: linux-2.6-patched/drivers/md/dm.c
===================================================================
@@ -178,6 +178,7 @@
 	dm_linear_init,
 	dm_stripe_init,
 	dm_interface_init,
+	dm_netlink_init,
 };
 
 void (*_exits[])(void) = {

@@ -186,6 +187,7 @@
 	dm_linear_exit,
 	dm_stripe_exit,
 	dm_interface_exit,
+	dm_netlink_exit,
 };
 
 static int __init dm_init(void)

Index: linux-2.6-patched/drivers/md/dm.h
===================================================================
@@ -15,6 +15,7 @@
 #include <linux/list.h>
 #include <linux/blkdev.h>
 #include <linux/hdreg.h>
+#include "dm-netlink.h"
 
 #define DM_NAME "device-mapper"
 

Index: linux-2.6-patched/drivers/md/dm-netlink.c
===================================================================
@@ -79,13 +79,213 @@
 	return NULL;
 }
 
+static struct sock *dm_netlink_sock;
+static u32 dm_netlink_daemon_pid;
+static DEFINE_SPINLOCK(pid_lock);
+
+static u64 dm_event_seqnum;
+static DEFINE_SPINLOCK(sequence_lock);
+
+static struct dm_event *dm_netlink_build_path_msg(char* dm_name, int type, int
+					   nr_valid_paths)
+{
+	struct dm_event *evt;
+	struct nlmsghdr	*nlh;
+	struct dm_netlink_msghdr *dm_nlh;
+	u64 seq;
+	struct timeval tv;
+	int err = -ENOBUFS;
+
+	evt = dme_cache_event_get(&dme_cache);
+	if (!evt) {
+		printk(KERN_ERR "%s: dme_cache_event_get %d\n",
+		       __FUNCTION__, err);
+		err = -ENOMEM;
+		goto out;
+	}
+
+	nlh = nlmsg_put(evt->skb, 0, 0, DM_EVENT,
+			NLMSG_ALIGN(sizeof(*dm_nlh)), 0);
+	if (!nlh)
+		goto nla_put_failure;
+
+	dm_nlh = nlmsg_data(nlh);
+	dm_nlh->type = type;
+	dm_nlh->version = DM_E_ATTR_MAX;
+
+	spin_lock(&sequence_lock);
+	seq = ++dm_event_seqnum;
+	spin_unlock(&sequence_lock);
+	do_gettimeofday(&tv);
+
+	NLA_PUT_U64(evt->skb, DM_E_ATTR_SEQNUM, seq);
+	NLA_PUT_U64(evt->skb, DM_E_ATTR_TSSEC, tv.tv_sec);
+	NLA_PUT_U64(evt->skb, DM_E_ATTR_TSUSEC, tv.tv_usec);
+	NLA_PUT_STRING(evt->skb, DM_E_ATTR_DMNAME, dm_name);
+	NLA_PUT_U32(evt->skb, DM_E_ATTR_VLDPTHS, nr_valid_paths);
+
+	return evt;
+
+nla_put_failure:
+	printk(KERN_ERR "%s: nla_put_failure\n", __FUNCTION__);
+	nlmsg_free(evt->skb);
+	dme_cache_event_put(evt);
+out:
+	return ERR_PTR(err);
+
+}
+
+/**
+ * dm_send_event - called to send a completed event message
+ *
+ * @md_name:	string containing md device name
+ * @evt:	pointer of event to send
+ *
+ **/
+void dm_send_event(char *md_name, struct dm_event *evt)
+{
+	struct nlmsghdr *nlh = (struct nlmsghdr *) evt->skb->data;
+	int err;
+
+	NLA_PUT_STRING(evt->skb, DM_E_ATTR_MDNAME, md_name);
+	nlmsg_end(evt->skb, nlh);
+
+	spin_lock(&pid_lock);
+	nlh->nlmsg_pid = dm_netlink_daemon_pid;
+	spin_unlock(&pid_lock);
+
+	if (!nlh->nlmsg_pid)
+		goto no_pid;
+
+	err = nlmsg_unicast(dm_netlink_sock, evt->skb, nlh->nlmsg_pid);
+
+	if (err && (err != -ESRCH))
+		printk(KERN_ERR "%s: nlmsg_unicast failed %d\n",
+		       __FUNCTION__, err);
+	goto out;
+
+nla_put_failure:
+	printk(KERN_ERR "%s: nla_put_failure\n", __FUNCTION__);
+no_pid:
+	nlmsg_free(evt->skb);
+out:
+	dme_cache_event_put(evt);
+	return;
+}
+EXPORT_SYMBOL(dm_send_event);
+
+/**
+ * dm_path_event - called to create a new path event and queue it
+ *
+ * @evt_type:		enum of path event type
+ * @t:			pointer to a dm_table
+ * @dm_name:		dm name string
+ * @nr_valid_paths:	number of valid paths remaining
+ *
+ **/
+void dm_path_event(enum dm_event_type evt_type, struct dm_table *t,
+			char* dm_name, int nr_valid_paths)
+{
+	struct dm_event *evt;
+	evt = dm_netlink_build_path_msg(dm_name, evt_type, nr_valid_paths);
+
+	if (evt) {
+		struct mapped_device *md = dm_table_get_md(t);
+		dm_add_event(md, evt);
+		dm_put(md);
+	}
+
+	return;
+}
+
+EXPORT_SYMBOL(dm_path_event);
+
+static int dm_netlink_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh,
+			       int *errp)
+{
+	int ret = -1;
+
+	if (security_netlink_recv(skb, CAP_SYS_ADMIN)) {
+		*errp = -EPERM;
+		goto out;
+	}
+
+	spin_lock(&pid_lock);
+	if (dm_netlink_daemon_pid) {
+		if (dm_netlink_daemon_pid != nlh->nlmsg_pid) {
+			*errp = -EBUSY;
+			goto out;
+		}
+	} else {
+		dm_netlink_daemon_pid = nlh->nlmsg_pid;
+	}
+	spin_unlock(&pid_lock);
+
+	ret = 0;
+out:
+	return ret;
+
+}
+
+static void dm_netlink_rcv(struct sock *sk, int len)
+{
+	unsigned int qlen;
+
+	do {
+		netlink_run_queue(sk, &qlen, &dm_netlink_rcv_msg);
+	} while (qlen);
+
+}
+
+static int dm_netlink_rcv_event(struct notifier_block *this,
+				unsigned long event, void *ptr)
+{
+	struct netlink_notify *n = ptr;
+
+	spin_lock(&pid_lock);
+	if (event == NETLINK_URELEASE &&
+	    n->protocol == NETLINK_DM && n->pid) {
+		if ( n->pid == dm_netlink_daemon_pid  ) {
+			dm_netlink_daemon_pid = 0;
+		}
+	}
+	spin_unlock(&pid_lock);
+
+	return NOTIFY_DONE;
+}
+
+static struct notifier_block dm_netlink_notifier = {
+	.notifier_call  = dm_netlink_rcv_event,
+};
+
+
 int __init dm_netlink_init(void)
 {
 	int err;
 
+	err = netlink_register_notifier(&dm_netlink_notifier);
+	if (err)
+		return err;
+
+	dm_netlink_sock = netlink_kernel_create(NETLINK_DM, 0,
+					   dm_netlink_rcv, THIS_MODULE);
+	if (!dm_netlink_sock) {
+		err = -ENOBUFS;
+		goto notifier_out;
+	}
 	err = dme_cache_init(&dme_cache, EVENT_SKB_SIZE);
-	if (!err)
-		printk(KERN_DEBUG "dm-netlink version 0.0.6 loaded\n");
+	if (err)
+		goto socket_out;
+
+	printk(KERN_DEBUG "dm-netlink version 0.0.7 loaded\n");
+
+	return err;
+
+socket_out:
+	sock_release(dm_netlink_sock->sk_socket);
+notifier_out:
+	netlink_unregister_notifier(&dm_netlink_notifier);
+	printk(KERN_ERR "%s: failed %d\n", __FUNCTION__, err);
 
 	return err;
 }

Index: linux-2.6-patched/drivers/md/dm-netlink.h
===================================================================
@@ -21,6 +21,8 @@
 #ifndef DM_NETLINK_H
 #define DM_NETLINK_H
 
+#include <linux/dm-netlink.h>
+
 struct dm_event {
 	void *cdata;
 	struct sk_buff *skb;

@@ -28,9 +30,16 @@
 };
 
 #ifdef CONFIG_DM_NETLINK_EVENT
+void dm_send_event(char *, struct dm_event *);
+void dm_path_event(enum dm_event_type, struct dm_table *, char *, int);
 int dm_netlink_init(void);
 void dm_netlink_exit(void);
 #else
+static inline void dm_send_event(char *, struct dm_event *)
+{ }
+static inline void dm_path_event(enum dm_event_type, struct dm_table *, char *,
+				 int)
+{ }
 static inline int __init dm_netlink_init(void)
 {
 	return 0;

50 lines not on this page · raw message

Index: linux-2.6-patched/drivers/md/dm.c
===================================================================
@@ -112,6 +112,8 @@
 	 */
 	atomic_t event_nr;
 	wait_queue_head_t eventq;
+	struct list_head event_list;
+	spinlock_t event_lock;
 
 	/*
 	 * freeze/thaw support require holding onto a super block

@@ -993,6 +995,8 @@
 	atomic_set(&md->holders, 1);
 	atomic_set(&md->open_count, 0);
 	atomic_set(&md->event_nr, 0);
+	INIT_LIST_HEAD(&md->event_list);
+	spin_lock_init(&md->event_lock);
 
 	md->queue = blk_alloc_queue(GFP_KERNEL);
 	if (!md->queue)

@@ -1091,6 +1095,18 @@
 static void event_callback(void *context)
 {
 	struct mapped_device *md = (struct mapped_device *) context;
+	unsigned long flags;
+	struct dm_event *evt, *next;
+	LIST_HEAD(events);
+
+	spin_lock_irqsave(&md->event_lock, flags);
+	list_splice_init(&md->event_list, &events);
+	spin_unlock_irqrestore(&md->event_lock, flags);
+
+	list_for_each_entry_safe(evt, next, &events, elist) {
+		list_del_init(&evt->elist);
+		dm_send_event(md->name, evt);
+	}
 
 	atomic_inc(&md->event_nr);
 	wake_up(&md->eventq);

@@ -1507,6 +1523,16 @@
 			(event_nr != atomic_read(&md->event_nr)));
 }
 
+void dm_add_event(struct mapped_device *md, struct dm_event *evt)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&md->event_lock, flags);
+	list_add(&evt->elist, &md->event_list);
+	spin_unlock_irqrestore(&md->event_lock, flags);
+
+}
+
 /*
  * The gendisk is only valid as long as you have a reference
  * count on 'md'.

Index: linux-2.6-patched/drivers/md/dm.h
===================================================================
@@ -152,4 +152,6 @@
 int dm_open_count(struct mapped_device *md);
 int dm_lock_for_deletion(struct mapped_device *md);
 
+void dm_add_event(struct mapped_device *md, struct dm_event *evt);
+
 #endif

Index: linux-2.6-patched/drivers/md/dm-mpath.c
===================================================================
@@ -834,6 +834,9 @@
 	if (pgpath == m->current_pgpath)
 		m->current_pgpath = NULL;
 
+	dm_path_event(DM_EVENT_FAIL_PATH, m->ti->table,
+			 pgpath->path.dev->name, m->nr_valid_paths);
+
 	queue_work(kmultipathd, &m->trigger_event);
 
 out:

@@ -873,6 +876,9 @@
 	if (!m->nr_valid_paths++ && m->queue_size)
 		queue_work(kmultipathd, &m->process_queued_ios);
 
+	dm_path_event(DM_EVENT_REINSTATE_PATH, m->ti->table,
+			      pgpath->path.dev->name, m->nr_valid_paths);
+
 	queue_work(kmultipathd, &m->trigger_event);
 
 out:

Index: linux-2.6-patched/drivers/md/dm-netlink.c
===================================================================
@@ -17,6 +17,8 @@
  *
  * Copyright IBM Corporation, 2005, 2006
  * 	Author: Mike Anderson <[email protected]>
+ *
+ * skb mempool derived from drivers/scsi/scsi_transport_iscsi.c
  */
 #include <linux/module.h>
 #include <linux/mempool.h>

@@ -27,56 +29,164 @@
 #include <net/netlink.h>
 #include "dm.h"
 
-#define EVENT_SKB_SIZE	NLMSG_GOODSIZE
+#define EVENT_SKB_SIZE		NLMSG_SPACE(128)
+#define MIN_EVENT_SKBS		16
+#define HIWAT_EVENT_SKBS	32
 
 struct dm_event_cache {
 	struct kmem_cache *cache;
 	unsigned skb_size;
+	unsigned hiwat;
+	mempool_t *pool;
+	spinlock_t used_list_lock;
+	struct list_head used_list;
 };
 
 static struct dm_event_cache dme_cache;
 
-static int dme_cache_init(struct dm_event_cache *dc, unsigned skb_size)
+static void* mp_alloc_dm_event(gfp_t gfp_mask, void *pool_data)
+{
+	struct dm_event *evt;
+	struct dm_event_cache *dc = pool_data;
+
+	evt = kmem_cache_alloc(dc->cache, gfp_mask);
+	if (!evt)
+		goto out;
+
+	evt->skb = alloc_skb(dc->skb_size, gfp_mask);
+	if (!evt->skb)
+		goto cache_out;
+	return evt;
+
+cache_out:
+	kmem_cache_free(dc->cache, evt);
+out:
+	return NULL;
+}
+
+static void mp_free_dm_event(void *element, void *pool_data)
+{
+	struct dm_event *evt = element;
+	struct dm_event_cache *dc = pool_data;
+
+	kfree_skb(evt->skb);
+	kmem_cache_free(dc->cache, evt);
+}
+
+static int dme_cache_init(struct dm_event_cache *dc, unsigned skb_size,
+			unsigned min_nr, unsigned hiwat)
 {
 	dc->skb_size = skb_size;
+	dc->hiwat = hiwat;
+	spin_lock_init(&dc->used_list_lock);
+	INIT_LIST_HEAD(&dc->used_list);
 
 	dc->cache = kmem_cache_create("dm_events",
 			   sizeof(struct dm_event), 0, 0, NULL, NULL);
 
 	if (!dc->cache)
-		return -ENOMEM;
+		goto cache_err_out;
+
+	dc->pool = mempool_create(min_nr, mp_alloc_dm_event,
+				   mp_free_dm_event, dc);
+	if (!dc->pool)
+		goto mempool_err_out;
 
 	return 0;
+
+mempool_err_out:
+	kmem_cache_destroy(dc->cache);
+cache_err_out:
+	return -ENOMEM;
 }
 
 static void dme_cache_destroy(struct dm_event_cache *dc)
 {
+	mempool_destroy(dc->pool);
 	kmem_cache_destroy(dc->cache);
 }
 
+static void __dme_cache_event_put(struct dm_event *evt)
+{
+	struct dm_event_cache *dc = evt->cdata;
+	struct sk_buff *skb = evt->skb;
+
+	if (!skb_shared(skb)) {
+		list_del(&evt->used_list);
+		skb_orphan(skb);
+
+		/* Init in case sent to pool */
+		skb->len = 0;
+		skb->tail = skb->head;
+
+		mempool_free(evt, dc->pool);
+	}
+}
+
 static void dme_cache_event_put(struct dm_event *evt)
 {
 	struct dm_event_cache *dc = evt->cdata;
-	kmem_cache_free(dc->cache, evt);
+	unsigned long flags;
+
+	spin_lock_irqsave(&dc->used_list_lock, flags);
+	__dme_cache_event_put(evt);
+	spin_unlock_irqrestore(&dc->used_list_lock, flags);
+}
+
+static void mp_cleanup_evt_pid(int pid, struct dm_event *evt)
+{
+	struct sk_buff *skb = evt->skb;
+	struct nlmsghdr *nlh;
+
+	if (skb->sk) {
+		nlh = (struct nlmsghdr *)skb->data;
+		if (nlh->nlmsg_pid == pid) {
+			if (skb->next && skb->sk)
+				skb_unlink(skb, &skb->sk->sk_receive_queue);
+			atomic_set(&skb->users, 1);
+			__dme_cache_event_put(evt);
+		}
+	}
+}
+
+static void mp_complete(struct dm_event_cache *dc, int release_pid)
+{
+	unsigned long flags;
+	struct dm_event *evt, *n;
+
+	spin_lock_irqsave(&dc->used_list_lock, flags);
+	if (!list_empty(&dc->used_list)) {
+		list_for_each_entry_safe(evt, n, &dc->used_list, used_list) {
+			if (release_pid) {
+				mp_cleanup_evt_pid(release_pid, evt);
+			} else {
+				__dme_cache_event_put(evt);
+			}
+		}
+	}
+	spin_unlock_irqrestore(&dc->used_list_lock, flags);
 }
 
 static struct dm_event* dme_cache_event_get(struct dm_event_cache *dc)
 {
+	unsigned long flags;
 	struct dm_event *evt;
 
-	evt = kmem_cache_alloc(dc->cache, GFP_ATOMIC);
-	if (evt) {
-		evt->cdata = dc;
-		evt->skb = alloc_skb(dc->skb_size, GFP_ATOMIC);
-		if (!evt->skb)
-			goto cache_out;
-	}
+	/* Check for ones we can complete before we alloc */
+	mp_complete(dc, 0);
 
-	return evt;
+	evt = mempool_alloc(dc->pool, GFP_ATOMIC);
+	if (!evt)
+		return NULL;
+
+	evt->cdata = dc;
+	skb_get(evt->skb);
+	INIT_LIST_HEAD(&evt->used_list);
+	spin_lock_irqsave(&dc->used_list_lock, flags);
+	list_add(&evt->used_list, &dc->used_list);
+	spin_unlock_irqrestore(&dc->used_list_lock, flags);
 
-cache_out:
-	dme_cache_event_put(evt);
-	return NULL;
+	return evt;
 }
 
 static struct sock *dm_netlink_sock;

@@ -128,7 +238,8 @@
 
 nla_put_failure:
 	printk(KERN_ERR "%s: nla_put_failure\n", __FUNCTION__);
-	nlmsg_free(evt->skb);
+	/* Set skb users so mp_complete can free */
+	atomic_set(&evt->skb->users, 1);
 	dme_cache_event_put(evt);
 out:
 	return ERR_PTR(err);

@@ -167,9 +278,10 @@
 nla_put_failure:
 	printk(KERN_ERR "%s: nla_put_failure\n", __FUNCTION__);
 no_pid:
-	nlmsg_free(evt->skb);
-out:
+	/* Set skb users so mp_complete can free */
+	atomic_set(&evt->skb->users, 1);
 	dme_cache_event_put(evt);
+out:
 	return;
 }
 EXPORT_SYMBOL(dm_send_event);

@@ -246,6 +358,7 @@
 	if (event == NETLINK_URELEASE &&
 	    n->protocol == NETLINK_DM && n->pid) {
 		if ( n->pid == dm_netlink_daemon_pid  ) {
+			mp_complete(&dme_cache, dm_netlink_daemon_pid);
 			dm_netlink_daemon_pid = 0;
 		}
 	}

@@ -273,7 +386,8 @@
 		err = -ENOBUFS;
 		goto notifier_out;
 	}
-	err = dme_cache_init(&dme_cache, EVENT_SKB_SIZE);
+	err = dme_cache_init(&dme_cache, EVENT_SKB_SIZE, MIN_EVENT_SKBS,
+				HIWAT_EVENT_SKBS);
 	if (err)
 		goto socket_out;
 

Index: linux-2.6-patched/drivers/md/dm-netlink.h
===================================================================
@@ -27,6 +27,7 @@
 	void *cdata;
 	struct sk_buff *skb;
 	struct list_head elist;
+	struct list_head used_list;
 };
 
 #ifdef CONFIG_DM_NETLINK_EVENT