Skip to contentSkip to conversation

Updated cluster log patch (take 2)

PATCH In review Jonathan Brassow 2 participants 2 messages May 7, 2009 – May 10, 2009

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 Updated cluster log patch (take 2)

Evgeniy, I'm copying you on this post because I am making use of your 'connector' interface.

agk, I've replied to your concerns in the previous thread about this patch. This mail contains the updated patch.

brassow

This patch contains the cluster-aware device-mapper log module.

The log types contained herein ("clustered-disk" and "clustered-core") operate in the same way as their single machine counterparts (dm-log.c), but they are cluster coherent. This is done by forwarding most logging requests to userspace, where a daemon processes the request in an ordered fashion with the rest of the nodes in the cluster. This is necessary to prevent log state corruption. (Imagine if two machines are writing to the same region of a mirror. They would both mark the region dirty, but you need a cluster-aware entity that can handle properly marking the region clean when they are done. Otherwise, you might clear the region when the first machine is done, not the second.)

The structure used for communication between kernel and userspace are located in include/linux/dm-log-cluster.h. Due to the frequency, diversity, and 2-way communication nature of the exchanges between kernel and userspace, 'connector' is used as the interface for communication. One existing userspace implementation of the daemon uses openAIS/corosync in order to communicate with guaranteed ordering and delivery. This daemon, which is capable of coherently managing log state from multiple cluster machines, can be found in the LVM2 code repository. Other implementations of the daemon, with no association to LVM or openAIS/corosync, are certainly possible.

Providing a cluster-aware logging type gives device-mapper RAID1 (and potentially other RAIDs) the ability to operate in a cluster-coherent fashion.

drivers/md/Kconfig +10 -0
Index: linux-2.6/drivers/md/Kconfig
===================================================================
@@ -231,6 +231,16 @@ config DM_MIRROR
          Allow volume managers to mirror logical volumes, also
          needed for live data migration tools such as 'pvmove'.
 
+config DM_LOG_CLUSTERED
+	tristate "Mirror cluster logging (EXPERIMENTAL)"
+	depends on DM_MIRROR && EXPERIMENTAL
+	select CONNECTOR
+	---help---
+	  The cluster logging module provides a mechanism for
+	  keeping log state coherent amoung a cluster of machines.
+	  Device-mapper mirroring (RAID1) can leverage this log type
+	  to make mirrors that are cluster-aware.
+
 config DM_ZERO
 	tristate "Zero target"
 	depends on BLK_DEV_DM
drivers/md/Makefile +2 -0
Index: linux-2.6/drivers/md/Makefile
===================================================================
@@ -8,6 +8,7 @@ dm-multipath-y	+= dm-path-selector.o dm-
 dm-snapshot-y	+= dm-snap.o dm-exception.o dm-exception-store.o \
 		   dm-snap-transient.o dm-snap-persistent.o
 dm-mirror-y	+= dm-raid1.o
+dm-log-clustered-y += dm-log-cluster.o dm-log-cluster-transfer.o
 md-mod-y	+= md.o bitmap.o
 raid456-y	+= raid5.o
 raid6_pq-y	+= raid6algos.o raid6recov.o raid6tables.o \
@@ -38,6 +39,7 @@ obj-$(CONFIG_DM_DELAY)		+= dm-delay.o
 obj-$(CONFIG_DM_MULTIPATH)	+= dm-multipath.o dm-round-robin.o
 obj-$(CONFIG_DM_SNAPSHOT)	+= dm-snapshot.o
 obj-$(CONFIG_DM_MIRROR)		+= dm-mirror.o dm-log.o dm-region-hash.o
+obj-$(CONFIG_DM_LOG_CLUSTERED)	+= dm-log-clustered.o
 obj-$(CONFIG_DM_ZERO)		+= dm-zero.o
 
 quiet_cmd_unroll = UNROLL  $@
include/linux/dm-log-cluster.h new file +56 -0

56 lines not on this page · raw message

drivers/md/dm-log-cluster-transfer.c new file +273 -0

273 lines not on this page · raw message

drivers/md/dm-log-cluster-transfer.h new file +18 -0
Index: linux-2.6/drivers/md/dm-log-cluster-transfer.h
===================================================================
@@ -0,0 +1,18 @@
+/*
+ * Copyright (C) 2006-2009 Red Hat, Inc.
+ *
+ * This file is released under the LGPL.
+ */
+
+#ifndef __DM_LOG_CLUSTER_TRANSFER_H__
+#define __DM_LOG_CLUSTER_TRANSFER_H__
+
+#define DM_MSG_PREFIX "dm-log-clustered"
+
+int dm_clog_tfr_init(void);
+void dm_clog_tfr_exit(void);
+int dm_clog_consult_server(const char *uuid, int request_type,
+			   char *data, size_t data_size,
+			   char *rdata, size_t *rdata_size);
+
+#endif /* __DM_LOG_CLUSTER_TRANSFER_H__ */
drivers/md/dm-log-cluster.c new file +797 -0

797 lines not on this page · raw message

Documentation/dm-log.txt new file +63 -0

63 lines not on this page · raw message

include/linux/connector.h +3 -1
Index: linux-2.6/include/linux/connector.h
===================================================================
@@ -41,8 +41,10 @@
 #define CN_IDX_BB			0x5	/* BlackBoard, from the TSP GPL sampling framework */
 #define CN_DST_IDX			0x6
 #define CN_DST_VAL			0x1
+#define CN_IDX_DM			0x7	/* Device Mapper */
+#define CN_VAL_DM_CLUSTER_LOG		0x1
 
-#define CN_NETLINK_USERS		7
+#define CN_NETLINK_USERS		8
 
 /*
  * Maximum connector's message size.
include/linux/Kbuild +1 -0
Index: linux-2.6/include/linux/Kbuild
===================================================================
@@ -57,6 +57,7 @@ header-y += dlmconstants.h
 header-y += dlm_device.h
 header-y += dlm_netlink.h
 header-y += dm-ioctl.h
+header-y += dm-log-cluster.h
 header-y += dn.h
 header-y += dqblk_xfs.h
 header-y += efs_fs_sb.h
#1 lore raw

Hi Jonathan.

Quoted 5 lines · Jonathan Brassow

Evgeniy, I'm copying you on this post because I am making use of your 'connector' interface.

agk, I've replied to your concerns in the previous thread about this patch. This mail contains the updated patch.

I have some trivial issue below, otherwise it looks good, thank you.

drivers/md/dm-log-cluster-transfer.c line 96

Quoted 6 lines
drivers/md/dm-log-cluster-transfer.c new file +6 -0
+static int fill_pkg(struct cn_msg *msg, struct dm_clog_request *tfr)
+{
+	uint32_t rtn_seq = (msg) ? msg->seq : (tfr) ? tfr->seq : 0;
+	struct receiving_pkg *pkg;
+
+	list_for_each_entry(pkg, &receiving_list, list) {

Here we process a list of statically allocated structures, and while for the kernel space it should be ok, but please write a huge comment which will explain that it relies on the kernel stack nature about unified addressing between processes.

Quoted 26 lines
drivers/md/dm-log-cluster-transfer.c new file +26 -0
+		if (rtn_seq != pkg->seq)
+			continue;
+
+		if (msg) {
+			pkg->error = -msg->ack;
+			/*
+			 * If we are trying again, we will need to know our
+			 * storage capacity.  Otherwise, along with the
+			 * error code, we make explicit that we have no data.
+			 */
+			if (pkg->error != -EAGAIN)
+				*(pkg->data_size) = 0;
+		} else if (tfr->data_size > *(pkg->data_size)) {
+			DMERR("Insufficient space to receive package [%u]::",
+			      tfr->request_type);
+			DMERR("  tfr->data_size    = %u", tfr->data_size);
+			DMERR("  *(pkg->data_size) = %lu", *(pkg->data_size));
+
+			*(pkg->data_size) = 0;
+			pkg->error = -ENOSPC;
+		} else {
+			pkg->error = tfr->error;
+			memcpy(pkg->data, tfr->data, tfr->data_size);
+			*(pkg->data_size) = tfr->data_size;
+		}
+		complete(&pkg->complete);

drivers/md/dm-log-cluster-transfer.c line 132

Quoted 10 lines
drivers/md/dm-log-cluster-transfer.c new file +10 -0
+		return 0;
+	}
+
+	return -ENOENT;
+}
+
+/*
+ * cn_clog_callback
+ * @data
+ *

Please also drop this kind of comment prefix, it does not really tells anything about function, but below part does (as well as next function's kerneldoc one).

Quoted 32 lines

+ * This is the connector callback that delivers data
+ * that was sent from userspace.
+ */

drivers/md/dm-log-cluster-transfer.c new file +29 -0
+static void cn_clog_callback(void *data)
+{
+	struct cn_msg *msg = (struct cn_msg *)data;
+	struct dm_clog_request *tfr = (struct dm_clog_request *)(msg + 1);
+
+	spin_lock(&receiving_list_lock);
+	if (msg->len == 0)
+		fill_pkg(msg, NULL);
+	else if (msg->len < sizeof(*tfr))
+		DMERR("Incomplete message received: [%u]", msg->seq);
+	else
+		fill_pkg(NULL, tfr);
+	spin_unlock(&receiving_list_lock);
+}
+
+/*
+ * dm_clog_consult_server
+ * @uuid: log's uuid (must be DM_UUID_LEN in size)
+ * @request_type:
+ * @data: data to tx to the server
+ * @data_size: size of data in bytes
+ * @rdata: place to put return data from server
+ * @rdata_size: value-result (amount of space given/amount of space used)
+ *
+ * Only one process at a time can communicate with the server.
+ * rdata_size is undefined on failure.
+ *
+ * Returns: 0 on success, -EXXX on failure
+ */
Signature · 1 line
	Evgeniy Polyakov