InfiniBand userspace verbs implementation

Here is an initial implementation of InfiniBand userspace verbs. I plan to commit this code to the OpenIB repository shortly, and submit it for inclusion during the 2.6.13 cycle, so I am posting it early for comments.

This code, in conjunction with the libibverbs and libmthca userspace libraries available from the subversion trees at

https://openib.org/svn/gen2/branches/roland-uverbs/src/userspace/libibverbs
https://openib.org/svn/gen2/branches/roland-uverbs/src/userspace/libmthca

enables userspace processes to access InfiniBand HCAs directly.

For those not familiar with the InfiniBand architecture, this so-called "userspace verbs" support allows userspace to post data path commands directly to the HCA. Resource allocation and other control path operations still go through the kernel driver.

Please take a look at this code if you have a chance. I would appreciate high-level criticism of the design and implementation as well as nitpicky complaints about coding style and typos.

In particular, the memory pinning code in in uverbs_mem.c could stand a looking over. In addition, a sanity check of the write()-based scheme for passing commands into the kernel in uverbs_main.c and uverbs_cmd.c is probably worthwhile.

Thanks,
Roland

Add new structs and struct members required by userspace verbs to IB core.

@@ -47,10 +47,11 @@
 {
 	struct ib_pd *pd;
 
-	pd = device->alloc_pd(device);
+	pd = device->alloc_pd(device, NULL, NULL, 0);
 
 	if (!IS_ERR(pd)) {
-		pd->device = device;
+		pd->device  = device;
+		pd->uobject = NULL;
 		atomic_set(&pd->usecnt, 0);
 	}
 

@@ -76,8 +77,9 @@
 	ah = pd->device->create_ah(pd, ah_attr);
 
 	if (!IS_ERR(ah)) {
-		ah->device = pd->device;
-		ah->pd     = pd;
+		ah->device  = pd->device;
+		ah->pd      = pd;
+		ah->uobject = NULL;
 		atomic_inc(&pd->usecnt);
 	}
 

@@ -122,7 +124,7 @@
 {
 	struct ib_qp *qp;
 
-	qp = pd->device->create_qp(pd, qp_init_attr);
+	qp = pd->device->create_qp(pd, qp_init_attr, NULL, 0);
 
 	if (!IS_ERR(qp)) {
 		qp->device     	  = pd->device;

@@ -130,6 +132,7 @@
 		qp->send_cq    	  = qp_init_attr->send_cq;
 		qp->recv_cq    	  = qp_init_attr->recv_cq;
 		qp->srq	       	  = qp_init_attr->srq;
+		qp->uobject       = NULL;
 		qp->event_handler = qp_init_attr->event_handler;
 		qp->qp_context    = qp_init_attr->qp_context;
 		qp->qp_type	  = qp_init_attr->qp_type;

@@ -197,10 +200,11 @@
 {
 	struct ib_cq *cq;
 
-	cq = device->create_cq(device, cqe);
+	cq = device->create_cq(device, cqe, NULL, NULL, 0);
 
 	if (!IS_ERR(cq)) {
 		cq->device        = device;
+		cq->uobject       = NULL;
 		cq->comp_handler  = comp_handler;
 		cq->event_handler = event_handler;
 		cq->cq_context    = cq_context;

@@ -245,8 +249,9 @@
 	mr = pd->device->get_dma_mr(pd, mr_access_flags);
 
 	if (!IS_ERR(mr)) {
-		mr->device = pd->device;
-		mr->pd     = pd;
+		mr->device  = pd->device;
+		mr->pd      = pd;
+		mr->uobject = NULL;
 		atomic_inc(&pd->usecnt);
 		atomic_set(&mr->usecnt, 0);
 	}

@@ -267,8 +272,9 @@
 				     mr_access_flags, iova_start);
 
 	if (!IS_ERR(mr)) {
-		mr->device = pd->device;
-		mr->pd     = pd;
+		mr->device  = pd->device;
+		mr->pd      = pd;
+		mr->uobject = NULL;
 		atomic_inc(&pd->usecnt);
 		atomic_set(&mr->usecnt, 0);
 	}

@@ -344,8 +350,9 @@
 
 	mw = pd->device->alloc_mw(pd);
 	if (!IS_ERR(mw)) {
-		mw->device = pd->device;
-		mw->pd     = pd;
+		mw->device  = pd->device;
+		mw->pd      = pd;
+		mw->uobject = NULL;
 		atomic_inc(&pd->usecnt);
 	}
 

Add device-independent userspace verbs support (ib_uverbs module).

@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * $Id: uverbs.h 2001 2005-03-16 04:15:41Z roland $
+ */
+
+#ifndef UVERBS_H
+#define UVERBS_H
+
+/* Include device.h and fs.h until cdev.h is self-sufficient */
+#include <linux/fs.h>
+#include <linux/device.h>
+#include <linux/cdev.h>
+#include <linux/kref.h>
+#include <linux/idr.h>
+
+#include <ib_verbs.h>
+#include <ib_user_verbs.h>
+
+struct ib_uverbs_device {
+	int                 devnum;
+	struct cdev         dev;
+	struct class_device class_dev;
+	struct ib_device   *ib_dev;
+	int                 num_comp;
+};
+
+struct ib_uverbs_event_file {
+	struct ib_uverbs_file *uverbs_file;
+	spinlock_t             lock;
+	int                    fd;
+	int                    is_async;
+	wait_queue_head_t      poll_wait;
+	struct list_head       event_list;
+};
+
+struct ib_uverbs_file {
+	struct kref                 ref;
+	struct ib_uverbs_device    *device;
+	struct ib_ucontext         *ucontext;
+	struct ib_event_handler     event_handler;
+	struct ib_uverbs_event_file async_file; 
+	struct ib_uverbs_event_file comp_file[1]; 
+};
+
+struct ib_uverbs_async_event {
+	struct ib_uverbs_async_event_desc desc;
+	struct list_head                  list;
+};
+
+struct ib_uverbs_comp_event {
+	struct ib_uverbs_comp_event_desc desc;
+	struct list_head                 list;
+};
+
+struct ib_uobject_mr {
+	struct ib_uobject   uobj;
+	struct page        *page_list;
+	struct scatterlist *sg_list;
+};
+
+extern struct semaphore ib_uverbs_idr_mutex;
+extern struct idr ib_uverbs_pd_idr;
+extern struct idr ib_uverbs_mr_idr;
+extern struct idr ib_uverbs_mw_idr;
+extern struct idr ib_uverbs_ah_idr;
+extern struct idr ib_uverbs_cq_idr;
+extern struct idr ib_uverbs_qp_idr;
+
+void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context);
+void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr);
+void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr);
+
+int ib_umem_get(struct ib_device *dev, struct ib_umem *mem,
+		void *addr, size_t size);
+void ib_umem_release(struct ib_device *dev, struct ib_umem *umem);
+
+#define IB_UVERBS_DECLARE_CMD(name)					\
+	ssize_t ib_uverbs_##name(struct ib_uverbs_file *file,		\
+				 const char __user *buf, int in_len,	\
+				 int out_len)
+
+IB_UVERBS_DECLARE_CMD(query_params);
+IB_UVERBS_DECLARE_CMD(get_context);
+IB_UVERBS_DECLARE_CMD(query_port);
+IB_UVERBS_DECLARE_CMD(alloc_pd);
+IB_UVERBS_DECLARE_CMD(dealloc_pd);
+IB_UVERBS_DECLARE_CMD(reg_mr);
+IB_UVERBS_DECLARE_CMD(dereg_mr);
+IB_UVERBS_DECLARE_CMD(create_cq);
+IB_UVERBS_DECLARE_CMD(destroy_cq);
+IB_UVERBS_DECLARE_CMD(create_qp);
+IB_UVERBS_DECLARE_CMD(modify_qp);
+IB_UVERBS_DECLARE_CMD(destroy_qp);
+
+#endif /* UVERBS_H */

Add Mellanox HCA-specific userspace verbs support to mthca.

@@ -743,6 +743,7 @@
 }
 
 int mthca_init_cq(struct mthca_dev *dev, int nent,
+		  struct mthca_ucontext *ctx, u32 pdn,
 		  struct mthca_cq *cq)
 {
 	int size = nent * MTHCA_CQ_ENTRY_SIZE;

@@ -754,30 +755,33 @@
 
 	might_sleep();
 
-	cq->ibcq.cqe = nent - 1;
+	cq->ibcq.cqe  = nent - 1;
+	cq->is_kernel = !ctx;
 
 	cq->cqn = mthca_alloc(&dev->cq_table.alloc);
 	if (cq->cqn == -1)
 		return -ENOMEM;
 
 	if (mthca_is_memfree(dev)) {
-		cq->arm_sn = 1;
-
 		err = mthca_table_get(dev, dev->cq_table.table, cq->cqn);
 		if (err)
 			goto err_out;
 
-		err = -ENOMEM;
+		if (cq->is_kernel) {
+			cq->arm_sn = 1;
+
+			err = -ENOMEM;
 
-		cq->set_ci_db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_CQ_SET_CI,
-						     cq->cqn, &cq->set_ci_db);
-		if (cq->set_ci_db_index < 0)
-			goto err_out_icm;
-
-		cq->arm_db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_CQ_ARM,
-						  cq->cqn, &cq->arm_db);
-		if (cq->arm_db_index < 0)
-			goto err_out_ci;
+			cq->set_ci_db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_CQ_SET_CI,
+							     cq->cqn, &cq->set_ci_db);
+			if (cq->set_ci_db_index < 0)
+				goto err_out_icm;
+
+			cq->arm_db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_CQ_ARM,
+							  cq->cqn, &cq->arm_db);
+			if (cq->arm_db_index < 0)
+				goto err_out_ci;
+		}
 	}
 
 	mailbox = kmalloc(sizeof (struct mthca_cq_context) + MTHCA_CMD_MAILBOX_EXTRA,

@@ -787,12 +791,14 @@
 
 	cq_context = MAILBOX_ALIGN(mailbox);
 
-	err = mthca_alloc_cq_buf(dev, size, cq);
-	if (err)
-		goto err_out_mailbox;
+	if (cq->is_kernel) {
+		err = mthca_alloc_cq_buf(dev, size, cq);
+		if (err)
+			goto err_out_mailbox;
 
-	for (i = 0; i < nent; ++i)
-		set_cqe_hw(get_cqe(cq, i));
+		for (i = 0; i < nent; ++i)
+			set_cqe_hw(get_cqe(cq, i));
+	}
 
 	spin_lock_init(&cq->lock);
 	atomic_set(&cq->refcount, 1);

@@ -803,11 +809,14 @@
 						  MTHCA_CQ_STATE_DISARMED |
 						  MTHCA_CQ_FLAG_TR);
 	cq_context->start           = cpu_to_be64(0);
-	cq_context->logsize_usrpage = cpu_to_be32((ffs(nent) - 1) << 24 |
-						  dev->driver_uar.index);
+	cq_context->logsize_usrpage = cpu_to_be32((ffs(nent) - 1) << 24);
+	if (ctx)
+		cq_context->logsize_usrpage |= cpu_to_be32(ctx->uar.index);
+	else
+		cq_context->logsize_usrpage |= cpu_to_be32(dev->driver_uar.index);
 	cq_context->error_eqn       = cpu_to_be32(dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn);
 	cq_context->comp_eqn        = cpu_to_be32(dev->eq_table.eq[MTHCA_EQ_COMP].eqn);
-	cq_context->pd              = cpu_to_be32(dev->driver_pd.pd_num);
+	cq_context->pd              = cpu_to_be32(pdn);
 	cq_context->lkey            = cpu_to_be32(cq->mr.ibmr.lkey);
 	cq_context->cqn             = cpu_to_be32(cq->cqn);
 

@@ -845,17 +854,19 @@
 	return 0;
 
 err_out_free_mr:
-	mthca_free_mr(dev, &cq->mr);
-	mthca_free_cq_buf(dev, cq);
+	if (cq->is_kernel) {
+		mthca_free_mr(dev, &cq->mr);
+		mthca_free_cq_buf(dev, cq);
+	}
 
 err_out_mailbox:
 	kfree(mailbox);
 
-	if (mthca_is_memfree(dev))
+	if (cq->is_kernel && mthca_is_memfree(dev))
 		mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM, cq->arm_db_index);
 
 err_out_ci:
-	if (mthca_is_memfree(dev))
+	if (cq->is_kernel)
 		mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index);
 
 err_out_icm:

@@ -895,7 +906,8 @@
 		int j;
 
 		printk(KERN_ERR "context for CQN %x (cons index %x, next sw %d)\n",
-		       cq->cqn, cq->cons_index, !!next_cqe_sw(cq));
+		       cq->cqn, cq->cons_index,
+		       cq->is_kernel ? !!next_cqe_sw(cq) : 0);
 		for (j = 0; j < 16; ++j)
 			printk(KERN_ERR "[%2x] %08x\n", j * 4, be32_to_cpu(ctx[j]));
 	}

@@ -913,15 +925,17 @@
 	atomic_dec(&cq->refcount);
 	wait_event(cq->wait, !atomic_read(&cq->refcount));
 
-	mthca_free_mr(dev, &cq->mr);
-	mthca_free_cq_buf(dev, cq);
-
-	if (mthca_is_memfree(dev)) {
-		mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM,    cq->arm_db_index);
-		mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index);
-		mthca_table_put(dev, dev->cq_table.table, cq->cqn);
+	if (cq->is_kernel) {
+		mthca_free_mr(dev, &cq->mr);
+		mthca_free_cq_buf(dev, cq);
+		if (mthca_is_memfree(dev)) {
+			mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM,    cq->arm_db_index);
+			mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index);
+		}
 	}
 
+	if (mthca_is_memfree(dev))
+		mthca_table_put(dev, dev->cq_table.table, cq->cqn);
 	mthca_free(&dev->cq_table.alloc, cq->cqn);
 	kfree(mailbox);
 }

@@ -49,14 +49,6 @@
 #define DRV_VERSION	"0.06-pre"
 #define DRV_RELDATE	"November 8, 2004"
 
-/* XXX remove once SINAI defines make it into kernel.org */
-#ifndef PCI_DEVICE_ID_MELLANOX_SINAI_OLD
-#define PCI_DEVICE_ID_MELLANOX_SINAI_OLD 0x5e8c
-#endif
-#ifndef PCI_DEVICE_ID_MELLANOX_SINAI
-#define PCI_DEVICE_ID_MELLANOX_SINAI 0x6274
-#endif
-
 enum {
 	MTHCA_FLAG_DDR_HIDDEN = 1 << 1,
 	MTHCA_FLAG_SRQ        = 1 << 2,

@@ -413,6 +405,7 @@
 int mthca_tavor_arm_cq(struct ib_cq *cq, enum ib_cq_notify notify);
 int mthca_arbel_arm_cq(struct ib_cq *cq, enum ib_cq_notify notify);
 int mthca_init_cq(struct mthca_dev *dev, int nent,
+		  struct mthca_ucontext *ctx, u32 pdn,
 		  struct mthca_cq *cq);
 void mthca_free_cq(struct mthca_dev *dev,
 		   struct mthca_cq *cq);

@@ -148,7 +148,7 @@
 	struct semaphore      mutex;
 };
 
-enum {
+enum mthca_db_type {
 	MTHCA_DB_TYPE_INVALID   = 0x0,
 	MTHCA_DB_TYPE_CQ_SET_CI = 0x1,
 	MTHCA_DB_TYPE_CQ_ARM    = 0x2,

@@ -158,6 +158,17 @@
 	MTHCA_DB_TYPE_GROUP_SEP = 0x7
 };
 
+struct mthca_user_db_table;
+struct mthca_uar;
+
+int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar,
+		      struct mthca_user_db_table *db_tab, int index, u64 uaddr);
+void mthca_unmap_user_db(struct mthca_dev *dev, struct mthca_uar *uar,
+			 struct mthca_user_db_table *db_tab, int index);
+struct mthca_user_db_table *mthca_init_user_db_tab(struct mthca_dev *dev);
+void mthca_cleanup_user_db_tab(struct mthca_dev *dev, struct mthca_uar *uar,
+			       struct mthca_user_db_table *db_tab);
+
 int mthca_init_db_tab(struct mthca_dev *dev);
 void mthca_cleanup_db_tab(struct mthca_dev *dev);
 int mthca_alloc_db(struct mthca_dev *dev, int type, u32 qn, u32 **db);

@@ -54,6 +54,14 @@
 	int           index;
 };
 
+struct mthca_user_db_table;
+
+struct mthca_ucontext {
+	struct ib_ucontext          ibucontext;
+	struct mthca_uar            uar;
+	struct mthca_user_db_table *db_tab;
+};
+
 struct mthca_mr {
 	struct ib_mr ibmr;
 	int order;

@@ -167,6 +175,7 @@
 	int                    cqn;
 	u32                    cons_index;
 	int                    is_direct;
+	int                    is_kernel;
 
 	/* Next fields are Arbel only */
 	int                    set_ci_db_index;

@@ -236,6 +245,11 @@
 	dma_addr_t      header_dma;
 };
 
+static inline struct mthca_ucontext *to_mucontext(struct ib_ucontext *ibucontext)
+{
+	return container_of(ibucontext, struct mthca_ucontext, ibucontext);
+}
+
 static inline struct mthca_fmr *to_mfmr(struct ib_fmr *ibmr)
 {
 	return container_of(ibmr, struct mthca_fmr, ibmr);

Hook userspace verbs up to Kconfig and Makefile.

@@ -7,6 +7,14 @@
 	  any protocols you wish to use as well as drivers for your
 	  InfiniBand hardware.
 
+config INFINIBAND_USER_VERBS
+	tristate "InfiniBand userspace verbs support"
+	depends on INFINIBAND
+	---help---
+	  Userspace InfiniBand verbs support.  This is the kernel side
+	  of userspace verbs.  You will also need libibverbs and a
+	  hardware driver library from <http://www.openib.org>.
+
 source "drivers/infiniband/hw/mthca/Kconfig"
 
 source "drivers/infiniband/ulp/ipoib/Kconfig"

@@ -1,7 +1,8 @@
 EXTRA_CFLAGS += -Idrivers/infiniband/include
 
-obj-$(CONFIG_INFINIBAND) +=	ib_core.o ib_mad.o ib_ping.o \
-				ib_cm.o ib_sa.o ib_umad.o ib_ucm.o
+obj-$(CONFIG_INFINIBAND) +=		ib_core.o ib_mad.o ib_ping.o \
+					ib_cm.o ib_sa.o ib_umad.o ib_ucm.o
+obj-$(CONFIG_INFINIBAND_USER_VERBS) +=	ib_uverbs.o
 
 ib_core-y :=			packer.o ud_header.o verbs.o sysfs.o \
 				device.o fmr_pool.o cache.o

@@ -16,4 +17,6 @@
 
 ib_umad-y :=			user_mad.o
 
+ib_uverbs-y :=			uverbs_main.o uverbs_cmd.o uverbs_mem.o
+
 ib_ucm-y :=			ucm.o

Tom> Now, you are really gonna hate me for asking you to put this Tom> in as you probably did not want to include this in the patch Tom> to lkml.

Tom> So, maybe Grant was right ;-)

Oh well, I didn't read the patches over carefully enough. Fortunately it was just my "for review" version.

- R.

...

No, it should actually do something, like so:

include/asm-sparc64/pgtable.h: af9bf175a223cf44310293287d50302e0fd3f9e9

@@ -416,6 +416,11 @@ extern int io_remap_pfn_range(struct vm_
 			       unsigned long pfn,
 			       unsigned long size, pgprot_t prot);
 
+/* Clear virtual and physical cachability, set side-effect bit.  */
+#define pgprot_noncached(prot) \
+	(__pgprot((pgprot_val(prot) & ~(_PAGE_CP | _PAGE_CV)) | \
+	 _PAGE_E))
+
 /*
  * For sparc32&64, the pfn in io_remap_pfn_range() carries <iospace> in
  * its high 4 bits.  These macros/functions put it there or get it from there.

How is memory pinning handled? (I haven't had time to read all the code, so please excuse my ignorance of something obvious).

The old mellanox drivers used to have a hack to call 'sys_mlock', and promiscuously lock memory any old userspace application asked for. What is the API for the new uverbs memory registration, and how will things like memory hotplug and NUMA page migration be able to unpin pages locked by a user program?

I have applications that would benefit from being able to register 15GB of memory on a machine with 16GB. Right now, MPI and other possible users of infiniband in userspace have to play cacheing games and limit what they can register. But locking all that memory without providing the kernel a way to unlock it under memory pressure or for page migration seems like a bad idea.

Troy> How is memory pinning handled? (I haven't had time to read Troy> all the code, so please excuse my ignorance of something Troy> obvious).

The userspace library calls mlock() and then the kernel does get_user_pages().

Troy> The old mellanox drivers used to have a hack to call Troy> 'sys_mlock', and promiscuously lock memory any old userspace Troy> application asked for. What is the API for the new uverbs Troy> memory registration, and how will things like memory hotplug Troy> and NUMA page migration be able to unpin pages locked by a Troy> user program?

The API for uverbs memory registration is ibv_reg_mr(), and right now the memory is pinned and that's it.

- R.

Is there a check in the kernel that the memory is actually mlock()ed?

What if a malicious (or broken) application does ibv_reg_mr() but doesn't lock the memory? Does the IB card get a physical address for a page that might get swapped out?

Troy> Is there a check in the kernel that the memory is actually Troy> mlock()ed?

No.

Troy> What if a malicious (or broken) application does Troy> ibv_reg_mr() but doesn't lock the memory? Does the IB card Troy> get a physical address for a page that might get swapped Troy> out?

No, the kernel does get_user_pages(). So the pages that the HCA gets will not be swapped or used for anything else. The only thing a malicious userspace app can do is screw itself up.

- R.

Do we even need the mlock in userspace then?

Troy> Do we even need the mlock in userspace then?

Yes, because the kernel may go through and unmap pages from userspace while trying to swap. Since we have the page locked in the kernel, the physical page won't go anywhere, but userspace might end up with a different page mapped at the same virtual address.

- R.

That shouldn't happen. If get_user_pages() has elevated the refcount on a page then the following can happen:

- The VM may decide to add the page to swapcache (if it's not mmapped from a file).

- Once the page is backed by either swapcache of a (mmapped) file, the VM may decide the unmap the application's pte's. A later minor fault by the app will cause the same physical page to be remapped.

- The VM may decide to try to write the page to its backing file or swap. If it does, the page is still in core, but is now clean.

- Once all pte's are unmapped and the page is clean, the VM may decide to try to reclaim the page. The VM will then see the elevated refcount and will bale out, leaving the page in core.

- If your code was doing a read-from-disk (modifying memory), then your code should run set_page_dirty() or set_page_dirty_lock() against the page before dropping the refcount which get_user_pages() added. Once the page is dirty, the VM can't reclaim it until it has been been written to swap or mmapped backing file.

IOW: while the page has an elevated refcount from get_user_pages(), that physical page is 100% pinned. Once you've done the set_page_dirty+put_page(), the page is again under control of the VM.

There should be no need to run mlock() from userspace.

Roland> Yes, because the kernel may go through and unmap pages Roland> from userspace while trying to swap. Since we have the Roland> page locked in the kernel, the physical page won't go Roland> anywhere, but userspace might end up with a different page Roland> mapped at the same virtual address.

Andrew> That shouldn't happen. If get_user_pages() has elevated Andrew> the refcount on a page then the following can happen:

...

Andrew> IOW: while the page has an elevated refcount from Andrew> get_user_pages(), that physical page is 100% pinned. Andrew> Once you've done the set_page_dirty+put_page(), the page Andrew> is again under control of the VM.

Hmm... I've never tested it first hand but Libor assures me there is a something like what I said. Libor, did I get the explanation right?

- R.

Quoting r. Roland Dreier <[email protected]>:

Roland, is it possible that what you describe is the behaviour of older kernels?

Digging around in rmap.c, I see the following code in try_to_unmap_one:

/*
* Don't pull an anonymous page out from under get_user_pages.
* GUP carefully breaks COW and raises page count (while holding
* page_table_lock, as we have here) to make sure that the page
* cannot be freed. If we unmap that page here, a user write
* access to the virtual address will bring back the page, but
* its raised count will (ironically) be taken to mean it's not
* an exclusive swap page, do_wp_page will replace it by a copy
* page, and the user never get to see the data GUP was holding
* the original page for.
*
* This test is also useful for when swapoff (unuse_process) has
* to drop page lock: its reference to the page stops existing
* ptes from being unmapped, so swapoff can make progress. */
if (PageSwapCache(page) &&
page_count(page) != page_mapcount(page) + 2) {
ret = SWAP_FAIL;
goto out_unmap;
}

This was added in http://linus.bkbits.net:8080/linux-2.5/[email protected]
on 2004-06-05 , i.e. as far as I can see around 2.6.7, and the comment says:

Seems quite like the situation that you described. Does my analysis make sence?

Since this case seems to be explicitly handled,
it is probably safe to rely on this behaviour or try_to_unmap, avoiding the need for mlock, is it not?

MST - Michael S. Tsirkin

OK, I'm by no means an expert on this, but Libor and I looked at rmap.c a little more, and there is code:

	if ((vma->vm_flags & (VM_LOCKED|VM_RESERVED)) ||
			ptep_clear_flush_young(vma, address, pte)) {
		ret = SWAP_FAIL;
		goto out_unmap;
	}

before the check

	if (PageSwapCache(page) &&
	    page_count(page) != page_mapcount(page) + 2) {
		ret = SWAP_FAIL;
		goto out_unmap;
	}

If userspace allocates some memory but doesn't touch it aside from passing the address in to the kernel, which does get_user_pages(), the PTE will be young in that first test, right? Does that mean that the userspace mapping will be cleared and userspace will get a different physical page if it faults that address back in?

- R.

If get_user_pages() was called with write=1, get_user_pages() will fault in a real page and yes, I guess it'll be pte_young.

If get_user_pages() was called with write=0, get_user_pages() will fault in a mapping of the zero page and we'd never get this far.

We won't try to unmap a page's ptes until that page has file-or-swapcache backing.

If the pte is then cleared, a subsequent minor fault will reestablish the mapping to the same physical page. A major fault cannot happen because the page was pinned by get_user_pages().

With the last few kernels I haven't had a chance to retest the problem that pushed us in the direction of using mlock. I will go back and do so with the latest kernel. Below I've given a quick description of the issue.

The driver did use get_user_pages() to elevated the refcount on all the pages it was going to use for IO, as well as call set_page_dirty() since the pages were going to have data written to them from the device.

The problem we were seeing is that the minor fault by the app resulted in a new physical page getting mapped for the application. The page that had the elevated refcount was still waiting for the data to be written to by the driver at the time that the app accessed the page causing the minor fault. Obviously since the app had a new mapping the data written by the driver was lost.

It looks like code was added to try_to_unmap_one() to address this, so hopefully it's no longer an issue...

-Libor

Thanks Libor, this is much better explanation of the problem than what I posted.

I doubt it. I tried this with an earlier 2.6 kernel, and get_user_pages() was still not enough to really pin the memory down. Maybe it works in 2.6.12, but that doesn't help me any, because our driver needs to support all 2.4 and 2.6 kernels. Currently, mlock() alone seems to be good enough, but I'm going to add calls to get_user_pages() just to be sure.

Timur Tabi
Staff Software Engineer
[email protected]

I wrote a quick test module and program to confirm that the problem
we saw in older kernels with get_user_pages() no longer exists. The module creates a character device with three different ioctl commands:

- Pin the pages of a buffer using get_user_pages()
- Check the pages by calling get_user_pages() a second time and comparing the new and original page list.
- Relase the pages using put_page()

The program opens the charcter device file descriptor, pins the pages
and waits for a signal, before checking the pages, which is sent to the process after running some other program which exercises the VM. On older kernels the check fails, on my 2.6.11 kernel the check succeeds. So mlock is not needed on top of get_user_pages() as it was before.

Thanks for the heads up.

Module and program attached.

-Libor

/*
* Copyright (c) 2005 Topspin Communications. All rights reserved. *
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*

 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:

*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
* $Id: $
*/
#include <linux/init.h>
#include <linux/fs.h>
#include <linux/module.h>
#include <linux/device.h>
#include <linux/err.h>
#include <linux/poll.h>
#include <linux/file.h>
#include <linux/mount.h>
#include <linux/cdev.h>
#include <linux/devfs_fs_kernel.h>

#include <asm/uaccess.h>
#include <asm/highmem.h>

MODULE_AUTHOR("Libor Michalek");
MODULE_DESCRIPTION("Get pages test");
MODULE_LICENSE("GPL");

enum {
	TEST_MAJOR = 232,
	TEST_MINOR = 255
};

#define TEST_DEV MKDEV(TEST_MAJOR, TEST_MINOR)

enum {
TEST_CMD_REGISTER = 1,
TEST_CMD_UNREGISTER = 2,
TEST_CMD_CHECK = 3
};

struct ioctl_arg {
	__u64 addr;
	__u64 size;
};

struct region_root {
	struct semaphore mutex;
	struct list_head regions; /* list of pending events. */
	struct file *filp;
	int nr_region;
};

struct test_region {
	unsigned long user;
	unsigned long addr;
	unsigned long size;
	int  nr_pages;
	struct page **pages;
	struct region_root *root;
	struct list_head region_list; /* member in root region list */
};

static void test_unlock(struct test_region *region)

{
        long i;

	list_del(&region->region_list);

        for (i = 0; i < region->nr_pages; i++)
                put_page(region->pages[i]);

	printk(KERN_ERR "TEST: Unlocked address <%016lx>\n", region->user);

	kfree(region->pages);
	kfree(region);
}

static struct test_region *test_lookup(struct region_root *root,
unsigned long addr)

{
	struct test_region *region;

	list_for_each_entry(region, &root->regions, region_list)
		if (region->user == addr)
			return region;

	return NULL;
}

static int test_lock(struct region_root *root,
unsigned long uaddr,
unsigned long size)

{
	struct test_region *region;
	int nr_pages;
	int result;

	region = kmalloc(sizeof(*region), GFP_KERNEL);
	if (!region)
		return -ENOMEM;

	region->user = uaddr;
	region->addr = uaddr & PAGE_MASK;
	region->size = PAGE_ALIGN(size + (uaddr & ~PAGE_MASK));
	region->root = root;

        nr_pages = (region->size + PAGE_SIZE-1) >> PAGE_SHIFT;

	region->pages = kmalloc(sizeof(struct page *) * nr_pages, GFP_KERNEL);
	if (!region->pages) {

		result = -ENOMEM;
		goto page_err;
	}

        region->nr_pages = get_user_pages(current, current->mm,
					  region->addr,
					  nr_pages,
					  1, 0, 
					  region->pages, NULL);
	if (region->nr_pages != nr_pages) {
		result = -EFAULT;
		goto get_err;
	}

	list_add_tail(&region->region_list, &root->regions);

	printk(KERN_ERR "TEST:   Locked address <%016lx>\n", region->user);

	return 0;
get_err:
	kfree(region->pages);
page_err:
	kfree(region);
	return result;
}

static int test_check(struct test_region *region)

{
	struct page **pages;
	int nr_pages;
	int result = 0;
	int i;

	pages = kmalloc(sizeof(struct page *) * region->nr_pages, GFP_KERNEL);
	if (!pages)
		return -ENOMEM;

        nr_pages = get_user_pages(current, current->mm,
				  region->addr,
				  region->nr_pages,
				  1, 0, 
				  pages, NULL);
	if (region->nr_pages != nr_pages) {
		result = -EFAULT;
		goto get_err;
	}

	for (i = 0; i < nr_pages; i++) {

		if (region->pages[i] != pages[i])
			printk(KERN_ERR "TEST: Check error <%p:%p> "
			       "page <%u> of <%u>\n",
			       pages[i], region->pages[i], i, nr_pages);
		put_page(pages[i]);
	}

get_err:

	kfree(pages);
	return result;
}

static long test_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)

{
	struct region_root *root = filp->private_data;
	struct test_region *region;
	struct ioctl_arg    ureq;
	int result = 0;

	if (!root)
		return -EINVAL;

        if (copy_from_user(&ureq, (void __user *)arg, sizeof(ureq)))
                return -EFAULT;

	down(&root->mutex);

	switch (cmd) {
	case TEST_CMD_REGISTER:

		result = test_lock(root, ureq.addr, ureq.size);
		break;
	case TEST_CMD_UNREGISTER:

		region = test_lookup(root, ureq.addr);
		if (!region)
			result = -ENOENT;
		else
			test_unlock(region);

		break;
	case TEST_CMD_CHECK:

		region = test_lookup(root, ureq.addr);
		if (!region)
			result = -ENOENT;
		else
			result = test_check(region);

		break;
	default:
		result = -ERANGE;
		break;
	}

	up(&root->mutex);
	return result;
}

static int test_open(struct inode *inode, struct file *filp)

{
	struct region_root *root;

	root = kmalloc(sizeof(*root), GFP_KERNEL);
	if (!root)
		return -ENOMEM;

	memset(root, 0, sizeof(*root));

	INIT_LIST_HEAD(&root->regions);
	init_MUTEX(&root->mutex);

	filp->private_data = root;
	root->filp = filp;

	printk(KERN_ERR "TEST: Created root struct\n");

	return 0;
}

static int test_close(struct inode *inode, struct file *filp)

{
	struct region_root *root = filp->private_data;
	struct test_region *region;

	down(&root->mutex);

	while (!list_empty(&root->regions)) {

		region = list_entry(root->regions.next,
				    struct test_region, region_list);
		test_unlock(region);
	}

	up(&root->mutex);

	kfree(root);

	filp->private_data = NULL;

	printk(KERN_ERR "TEST: Deleted root struct\n");
	return 0;
}

static struct file_operations test_fops = {
.owner = THIS_MODULE,
.open = test_open,
.release = test_close,
.compat_ioctl = test_ioctl,
.unlocked_ioctl = test_ioctl,
};

static struct cdev test_cdev;

static int __init test_init(void)

{
	int result;

	result = register_chrdev_region(TEST_DEV, 1, "mltest");
	if (result) {
		printk(KERN_ERR "TEST: Error <%d> registering dev\n", result);
		goto err_chr;
	}

	cdev_init(&test_cdev, &test_fops);

	result = cdev_add(&test_cdev, TEST_DEV, 1);
	if (result) {
		printk(KERN_ERR "TEST: Error <%d> adding cdev\n", result);
		goto err_cdev;
	}

	return 0;
err_cdev:
	unregister_chrdev_region(TEST_DEV, 1);
err_chr:
	return result;
}

static void __exit test_cleanup(void)

{
	cdev_del(&test_cdev);
	unregister_chrdev_region(TEST_DEV, 1);
}

module_init(test_init);
module_exit(test_cleanup);

/*
* Copyright (c) 2005 Topspin Communications. All rights reserved. *
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*

 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:

*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
* $Id: $
*/

#include <stdlib.h>
#include <string.h>
#include <glob.h>
#include <stdio.h>
#include <fcntl.h>
#include <errno.h>
#include <stdint.h>
#include <poll.h>
#include <unistd.h>
#include <signal.h>

#include <linux/types.h>

#define TEST_DEV_PATH "/dev/mltest"
#define TEST_SLEEP_UTIME 50000

struct ioctl_arg {
	__u64 addr;
	__u64 size;
};

enum {
TEST_CMD_REGISTER = 1,
TEST_CMD_UNREGISTER = 2,
TEST_CMD_CHECK = 3
};

static int hold = 1;

void sig_usr(int value)

{
	hold = 0;
}

int main(int argc, char **argv)

{
	struct ioctl_arg req;
	void *data;
	int   param_c = 0;
	int   size;
	int   fd;
	int   result;

	if (2 != argc ||
	    0 > (size = atoi(argv[++param_c]))) { 
		
		fprintf(stderr, "usage: %s <size>\n", argv[0]);
		fprintf(stderr, "  size  - allocated region size in bytes.\n");
		
		exit(1);
	}
	signal(SIGUSR1, sig_usr);

	data = malloc(size);
	if (!data) {
		fprintf(stderr, "Failed to allocated region of size <%d>\n",
			size);
		exit(1);
	}
	
	fd = open(TEST_DEV_PATH, O_RDWR);
	if (fd < 0) {
		
		fprintf(stderr, "Error <%d:%d> opening device <%s>\n",
			fd, errno, TEST_DEV_PATH);
		goto open_err;
	}

	req.addr = (unsigned long)data;
	req.size = size;

	result = ioctl(fd, TEST_CMD_REGISTER, &req);
	if (result) {

		fprintf(stderr, "Error <%d:%d> registering region\n",
			result, errno);
		goto ioctl_err;
	}

	fprintf(stdout, 
		"Address <%016lx> registered, process <%d> waiting...\n",
		data, getpid());

	while (hold) {

		usleep(TEST_SLEEP_UTIME);
	}

	fprintf(stdout, "Process continuing, checking address <%016lx>\n",
		data);

	result = ioctl(fd, TEST_CMD_CHECK, &req);
	if (result) {

		fprintf(stderr, "Error <%d:%d> checking region\n",
			result, errno);
		goto ioctl_err;
	}

	result = ioctl(fd, TEST_CMD_UNREGISTER, &req);
	if (result) {

		fprintf(stderr, "Error <%d:%d> unregistering region\n", 
			result, errno);
		goto ioctl_err;
	}

ioctl_err:

	close(fd);
open_err:
	free(data);

	return 0;
}

Libor,

When you say "older", what exactly do you mean? I have different test that normally fails with just get_user_pages(), but it works with 2.6.9 and above. I haven't been able to get any kernel earlier than 2.6.9 to compile or boot properly, so I'm having a hard time narrowing down the actual point when get_user_pages() started working.

Timur Tabi
Staff Software Engineer
[email protected]

One thing a Southern boy will never say is,
"I don't think duct tape will fix it."
     -- Ed Smylie, NASA engineer for Apollo 13

I haven't gotten a reply to this question, but I've done my own research, and I think I found the answer. Using my own test of get_user_pages(), it appears that the fix was placed in 2.6.7. However, I would like to know specifically what the fix is. Unfortunately, tracking this stuff down is beyond my understanding of the Linux VM.

Assuming that the fix is in try_to_unmap_one(), the only significant change I see between 2.6.6 and 2.6.7 is the addition of this code:

	pgd = pgd_offset(mm, address);
	if (!pgd_present(*pgd))
		goto out_unlock;

	pmd = pmd_offset(pgd, address);
	if (!pmd_present(*pmd))
		goto out_unlock;

	pte = pte_offset_map(pmd, address);
	if (!pte_present(*pte))
		goto out_unmap;

	if (page_to_pfn(page) != pte_pfn(*pte))
		goto out_unmap;

Can anyone tell me if this is the actual fix, or at least a major part of the actual fix?

Timur Tabi
Staff Software Engineer
[email protected]

One thing a Southern boy will never say is,
"I don't think duct tape will fix it."
     -- Ed Smylie, NASA engineer for Apollo 13

I'm also still waiting for a reply to this question. Anyone????

Upon doing some more research, I think the fix might be those code instead:

/*
* Don't pull an anonymous page out from under get_user_pages.
* GUP carefully breaks COW and raises page count (while holding
* page_table_lock, as we have here) to make sure that the page
* cannot be freed. If we unmap that page here, a user write
* access to the virtual address will bring back the page, but
* its raised count will (ironically) be taken to mean it's not
* an exclusive swap page, do_wp_page will replace it by a copy
* page, and the user never get to see the data GUP was holding
* the original page for.
*/
if (PageSwapCache(page) &&
page_count(page) != page->mapcount + 2) {
ret = SWAP_FAIL;
goto out_unmap;
}

Both this change and the other one I mentioned are new to 2.6.7. I suppose I could try applying these patches to the 2.6.6 kernel and see if anything improves, but that won't help me understand what's really going on. The above comment makes sounds almost like it's a fix, but it talks about copy-on-write, which is has nothing to do with the real problem.

Timur Tabi
Staff Software Engineer
[email protected]

One thing a Southern boy will never say is,
"I don't think duct tape will fix it."
     -- Ed Smylie, NASA engineer for Apollo 13

Sorry for not replying earlier (indeed, sorry for not joining in the wider RDMA pinning discussion), concentrating on other stuff at present.

I believe you're right this time - I was rather puzzled by your earlier choice, then unhelpfully forgot to reply and point you a few lines further down to this comment, which does shout "get_user_pages fix" quite loudly.

There's a lot of change in the rmap area between 2.6.6 and 2.6.7, but you're right that this is an isolated fix, which could in principle be applied to earlier releases. Though I don't see it's worth doing now.

Almost? Sorry if my comment doesn't make it obvious it's a fix for a get_user_pages issue - I rewrote Andrea Arcangeli's original commment. The analysis and fix are his.

Oh, well, maybe, but what is the real problem? Are you sure that copy-on-write doesn't come into it?

I haven't reread through the whole thread, but my recollection is that you never quite said what the real problem is: you'd found some time ago that get_user_pages sometimes failed to pin the pages for your complex app, so were forced to mlock too; but couldn't provide any simple test case for the failure (which can indeed be a lot of work to devise), so we were all in the dark as to what went wrong.

But you've now found that 2.6.7 and later kernels allow your app to work correctly without mlock, good. get_user_pages is certainly the right tool to use for such pinning. (On the question of whether mlock guarantees that user virtual addresses map to the same physical addresses, I prefer Arjan's view that it does not; but accept that there might prove to be difficulties in holding that position.)

So, it works now, you've exonerated today's get_user_pages, and you've identified at least one get_user_pages fix which went in at that time:
do we really need to chase this further?

Oh, in writing of copy-on-write, I've just remembered another fix for get_user_pages which I made in 2.6.7 (though I've not heard of anyone seeing the problem fixed): call to do_wp_page in do_swap_page. get_user_pages assumes that the write fault it generates will break copy-on-write i.e. will make a private copy page when necessary, before returning to the caller; but that wasn't happening in the do_swap_page case.

By the way, please don't be worried when soon the try_to_unmap_one comment and code that you identified above disappear. When I'm back in patch submission mode, I'll be sending Andrew a patch which removes it, instead reworking can_share_swap_page to rely on the page_mapcount instead of page_count, which avoids the ironical behaviour my comment refers to, and allows an awkward page migration case to proceed (once unpinned). Andrea and I now both prefer this page_mapcount approach.

Hugh

No, but I do know that my test case doesn't call fork(), so it's reproducible without involving COW. Of course, I'm sure someone's going to tell me now that COW comes into effect even without fork(). If so, please explain.

The short answer: under "extreme" memory pressure, the data inside a page pinned by get_user_pages() is swapped out, moved, or deleted (I'm not sure which). Some other data is placed into that physical location.

By extreme memory pressure, I mean having the process allocate and touch as much memory as possible. Something like this:

num_bytes = get_amount_of_physical_ram();
char *p = malloc(num_bytes);
for (i=0; i<num_bytes; i+=PAGE_SIZE)
p[i] = 0;

The above over-simplified code fails on earlier 2.6 kernels (or earlier versions of glibc that accompany most distros the use the earlier 2.6 kernels). Either malloc() returns NULL, or the p[i]=0 part causes a segfault. I haven't bothered to trace down why. But when it does work, the page pinned by get_user_pages() changes.

My understanding is that mlock() could in theory allow the page to be moved, but that currently nothing in the kernel would actually move it. However, that could change in the future to allow hot-swapping of RAM.

My driver needs to support all 2.4 and 2.6 kernel versions. My makefile scans the kernel source tree with 'grep' to identify various characterists, and I use #ifdefs to conditionally compile code depending on what features are present in the kernel. I can't use the kernel version number, because that's not reliable - distros will incorporate patches from future kernels without changing the version ID.

So I need to take into account distro vendors that use an earlier kernel, like 2.6.5, and back-port the patch from 2.6.7. The distro vendor will keep the 2.6.5 version number, which is why I can't rely on it.

I need to know exactly what the fix is, so that when I scan mm/rmap.c, I know what to look for. Currently, I look for this regex:

try_to_unmap_one.*vm_area_struct

which seems to work. However, now I think it's just a coincidence.

Ugh, that means my regex is probably going to break. Not only that, but I don't understand what you're saying either. Trying to understand the VM is really hard.

I guess in this specific case, it doesn't really matter, because calling mlock() when I should be calling get_user_pages() is not a bad thing.

I'll try. COW comes into effect whenever you're sharing a page and then need to make private changes to it. Fork is one way of sharing (with ancestor and descendant processes). Using the empty zero page is another way of sharing (with all other processes and parts of your own address space with a readonly page full of zeroes). Using a file page from the page cache is another way of sharing.

None of those is actually your case, but our test for whether a page is shared has been inadequate: oversimplifying, if page_count is more than 1 then we have to assume it is shared and do the copy-on-write (if the modifications are to be private). But there are various places where the page_count is temporarily raised (e.g. while paging out), which we cannot distinguish, so occasionally we'll copy on write even when it's not necessary, but we lack the information to tell us so.

In particular, of course, get_user_pages raises page_count to pin the page: so making a page appear shared when it's not shared at all.

Which has to be a bug with get_user_pages, which has no other purpose than to pin the pages. I cannot criticize you for working around it to get your app working on lots of releases, but what _we_ have to do is fix get_user_pages - and it appears that Andrea did so a year ago.

I'm surprised if it's as simple as you describe (you do say over- simplified, maybe the critical points have fallen out), since GUP users would have complained long ago if it wasn't doing the job in normal cases of memory pressure. Andrea's case does involve the process independently trying to touch a page it has pinned for I/O with get_user_pages. Or (and I've only just thought of this, suspect it might be exactly your case) not touch, but apply get_user_pages again to a page already so pinned (while memory pressure has caused try_to_unmap_one temporarily to detach it from the user address space
- the aspect of the problem that Andrea's fix attacks).

That's my understanding too, that nothing currently does so. Aside from hot-swapping RAM, there's also a need to be able to migrate pages around RAM, either to unfragment memory allowing higher-order allocations to succeed more often, or to get around extreme dmamem/normal-mem/highmem imbalances without dedicating huge reserves. Those would more often succeed if uninhibited by mlock.

Perhaps any release based on 2.6.7 or above, or any release which mentions "get_user_pages" in its mm/rmap.c or mm/objrmap.c?

Sorry about that, but suiting your regex is low in our priorities for VM design! I was tempted to offer to keep a comment on get_user_pages in mm/rmap.c after the change, but that's really rather babyish: just assume 2.6.7 and upwards are fixed (or complain if you find not).

Perhaps I'll manage a clearer explanation when I come to write the change description for the patch, we'll have to see.

If you can afford to keep that amount of memory mlocked, and have to capability to do so, yes, it should do no harm. We were just upset to think that mlock was still needed to get around a get_user_pages bug which was fixed a year ago.

Hugh

Hugh,

If I am reading you correctly, you are saying that mlock currently prevents pages from migrating around to unfragment memory, but get_user_pages does not prevent this? If this is the case, this could very easily be the problem Timur was experiencing. He is using get_user_pages to lock pages long term (for the life of the process, beyond the bounds of a single system call).

If it is possible for a page to be migrated in physical memory during extreme virtual memory pressure while the reference count is held with get_user_pages, that would cause the problem where the hardware is no longer mapped to the same page as the application.

BTW: In earlier kernels, I experienced the same issues in our IB drivers when trying to pin pages using only get_user_pages.

Bill Jordan
InfiniCon Systems

No, not what I meant at all. I'm saying that currently (aside from proposed patches) there is no such migration of pages; that we'd prefer to implement migration in such a way that mlock does not inhibit it (though there might prove to be strong arguments defeating that); and that get_user_pages _must_ prevent migration (and if there were already such migration, I'd be saying it _does_ prevent it).

Hugh

Indeed, mlock is a virtual pin and as such it won't be guaranteed to always prevent migration. While get_user_pages is a physical pin on the physical page so it has to prevent migration.

I think for him the physical pin is better since I guess IB would break (at least unless you've some method to call to stop IB, adjust the IB dma tracking, and restart IB, that hotplug can call). For the short term using only get_user_pages sounds simpler IMHO.

This is not the case. Infact get_user_pages is a stronger pin than mlock. But if you call it by hand and you plan to write to the page, you have to use the "write=1" flag, this is fundamental if you want to write to the physical page from userland while it's being tracked by IB dma.

In short you should not use mlock and you should use only get_user_pages(write=1).

If the problem appears again even after the last fix for the COW I did last year, than it means we've another yet another bug to fix.

Using mlock for this is unnecessary. mlock is a "virtual" pin and it provides weaker guarantees than what you need. You need _physical_ pin and get_user_pages(write=1) is the only one that will give it to you.

write=0 is ok too if you're never ever going to write to the page with the cpu from userland.

In the old days there was the concept that get_user_pages wasn't a "pte-pin", but that was infact broken in the way COW was working with threads, but this is fixed now that is really a "pte-pin" again (like in 2.2 which never had the corruption cow bug!) even though the pte may temporarily be set to swapcache or null. In current 2.6 you're guaranteed that despite the pte may be temporarly be set to not-present, the next minor fault will bring into memory the very same physical page that was there before. At least unless you map the thing writeprotect (i.e. write=0) and you write to it from userland.. ;).

All of my memory pinning test cases pass when I use get_user_pages() with kernels 2.6.7 and later.

Timur Tabi
Staff Software Engineer
[email protected]

One thing a Southern boy will never say is,
"I don't think duct tape will fix it."
     -- Ed Smylie, NASA engineer for Apollo 13

Well then your problem was the cow bug, that was corrupting userland with O_DIRECT too...

The older kernel I tried was one of the 2.4.21 RHEL 3 kernels. I hadn't
spent much time investigating the issue since this was a new kernel, so it was a natural one for me to try.

-Libor

That's not what we're seeing. We have hardware that does DMA over the network (much like the Infiniband stuff), and we have a testcase that fails if get_user_pages() is used, but not if mlock() is used. Consider two computers on a network, X and Y. Both have our hardware, which can transfer a page of memory from a given physical address on X to a physical address on Y.

1) Application on X allocates a block of memory, and passes the virtual address to the driver.
2) Driver on X calls get_user_pages() and then obtains a physical address for the memory.
3) Application and driver on Y do the same thing.
4) App X fills memory with some data D.
5) App X then allocates as much memory as it possibly can. It touches every page in this memory, and then frees the memory. This will force other pages to be swapped out, including the supposedly pinned memory.
6) App X then tells Driver X to transfer data D to computer Y.
7) App Y compares data D and finds that it doesn't match with it's supposed to.

Conclusion: during step 5, the data in pinned memory is swapped out or something. I'm not sure where it goes.

We can only demonstrate this problem using our hardware, because you need the ability to transfer memory without using the CPU. We were going to prepare a test case and ship same hardware to a few kernel developers to prove our point, but now that we're able to call mlock() in non-user processes, we decided it wasn't worth our time. Actually, I discovered that I can call cap_raise() and set the ulimit structure, which gives me the ability to call mlock() on any amount of memory from any process in 2.4 and 2.6 kernels, which we need to support. If I had thought of that earlier, I wouldn't have needed all those hacks to call sys_mlock() from the driver.

Timur Tabi
Staff Software Engineer
[email protected]

If you don't share your testcase it's unlikely to be fixed.

As I said, the testcase only works with our hardware, and it's also very large. It's one small test that's part of a huge test suite. It takes a couple hours just to install the damn thing.

We want to produce a simpler test case that demonstrates the problem in an easy-to-understand manner, but we don't have time to do that now.

Timur Tabi
Staff Software Engineer
[email protected]

If your theory is correct then it should be able to demonstrate this problem without any special hardware at all: pin some user memory, then generate memory pressure then check the contents of those pinned pages.

But if, for the DMA transfer, you're using the array of page*'s which were originally obtained from get_user_pages() then it's rather hard to see how the kernel could alter the page's contents.

Then again, if mlock() fixes it then something's up. Very odd.

I tried that, but I couldn't get it to fail. But that was a while ago, and I've learned a few things since then, so I'll try again.

With mlock(), we don't need to use get_user_pages() at all. Arjan tells me the only time an mlocked page can move is with hot (un)plug of memory, but that isn't supported on the systems that we support. We actually prefer mlock() over get_user_pages(), because if the process dies, the locks automatically go away too.

You don't "support" i386 or ia64 or x86-64 or ppc64 systems? What hardware do you support? And what about the fact that you are aiming to get this code into mainline, right? If not, why are you asking here? :)

thanks,

greg k-h

I've never seen or heard of any x86-32 or x86-64 system that supports hot-swap RAM. Our hardware does not support PPC, and our software doesn't support ia-64.

Well, our primary concern is getting our stuff to work. Since get_user_pages() doesn't work, but mlock() does, that's what we use. I don't know how to fix get_user_pages(), and I don't have the time right now to figure it out. I know that technically mlock() is not the right way to do it, and so we're not going to be submitting our code for the mainline until get_user_pages() works and our code uses it instead of mlock().

I know of at least 1 x86-32 box from a three-letter-named company with this feature that has been shipping for a few _years_ now. That box is pretty much everywhere now, and I know that other versions of it are also quite popular (despite the high cost...)

Your hardware is just a pci card, right? Why wouldn't it work on ppc64 and ia64 then?

Wait, what _is_ "your stuff"? The open-ib code? Or some other, private fork? Any pointers to this stuff?

thanks,

greg k-h

Hmm... Well, I think we were already planning on telling our customers that we don't support hot-swap RAM. Is there a CONFIG option for that feature?

It's PCI-X, actually, and I don't think we've ever actually plugged it into a PPC box. Isn't Open Firmware support required for all PPC boxes, anyway? Our PCI card is not OF compatible, AFAIK.

As for IA64, well, we could support it, but it's not a high enough priority. We do have some CPU-specific code in our driver that we would need to port to IA-64.

No, if anything, it's the competition to IB. It's called iWARP (RDMA over TCP/IP), and it's similar to IB except it uses gigabit ethernet instead of whatever hardware IB uses. Because we also support RMDA, we have the same problems as OpenIB, however, we would prefer that the kernel support OpenRDMA instead, since it's more generic.

http://ammasso.com/support.html

The current version of the code calls sys_mlock() directly from the driver. We haven't released yet the version that calls mlock().

The driver to do the ACPI portion of both add and remove is in the kernel today, so it's certainly a feature that's coming relatively soon.

There is a large variety of x86_64, ppc64, ia64 and ia64 hardware that will be doing memory hotplug. I believe that every POWER5 system is capable of supporting it, at least virtually.

I don't think your concerns end with memory hotplug. The same approaches to moving memory around will be used for NUMA memory balancing and for memory defragmentation. Can you say that your cards will never be used on a system which has memory which becomes fragmented?

-- Dave

Timur> With mlock(), we don't need to use get_user_pages() at all. Timur> Arjan tells me the only time an mlocked page can move is Timur> with hot (un)plug of memory, but that isn't supported on Timur> the systems that we support. We actually prefer mlock() Timur> over get_user_pages(), because if the process dies, the Timur> locks automatically go away too.

There actually is another way pages can move, with both get_user_pages() and mlock(): copy-on-write after a fork(). If userspace does a fork(), then all PTEs are marked read-only, and if the original process touches the page after the fork(), a new page will be allocated and mapped at the original virtual address.

This is actually a pretty big pain, because the only good solution seems to be for the kernel to mark these registered regions as VM_DONTCOPY. Right now this means that driver code ends up monkeying with vm_flags for user vmas.

Does it seem reasonable to add a new system call to let userspace mark memory it doesn't want copied into forked processes? Something like

long sys_mark_nocopy(unsigned long addr, size_t len, int mark)

which would set VM_DONTCOPY if mark != 0, and clear it if mark == 0. A better name would be gratefully accepted...

Then to register memory for RDMA, userspace would call sys_mark_nocopy() (with appropriate accounting to handle possibly overlapping regions) and the kernel would call get_user_pages(). The get_user_pages() is of course required because the kernel can't trust userspace to keep the pages locked. mlock() would no longer be necessary. We can trust userspace to call sys_mark_nocopy() as needed, because a process can only hurt itself and its children by misusing the sys_mark_nocopy() call.

If this seems reasonable then I can code a patch.

- R.

add a new MAP_DONTCOPY flag and accept it in mmap and mprotect?

Roland> Does it seem reasonable to add a new system call to let Roland> userspace mark memory it doesn't want copied into forked Roland> processes? Something like

Roland> long sys_mark_nocopy(unsigned long addr, size_t len, int Roland> mark)

Roland> which would set VM_DONTCOPY if mark != 0, and clear it if Roland> mark == 0. A better name would be gratefully accepted...

Christoph> add a new MAP_DONTCOPY flag and accept it in mmap and Christoph> mprotect?

That is much better, thanks. But I think it would need to be PROT_DONTCOPY to work with mprotect(), right?

- R.

Do we care about that? A straightforward scenario under which this can happen is:

a) app starts some read I/O in an asynchronous manner
b) app forks
c) child writes to one of the pages which is still under read I/O
d) the read I/O completes
e) the child is left with the old data plus the child's modification instead of the new data

which is a very silly application which is giving itself unpredictable memory contents anyway.

I assume there's a more sensible scenario?

Andrew> Do we care about that? A straightforward scenario under Andrew> which this can happen is:

Andrew> a) app starts some read I/O in an asynchronous manner
Andrew> b) app forks
Andrew> c) child writes to one of the pages which is still under read I/O Andrew> d) the read I/O completes
Andrew> e) the child is left with the old data plus the child's modification instead
Andrew> of the new data

Andrew> which is a very silly application which is giving itself Andrew> unpredictable memory contents anyway.

Andrew> I assume there's a more sensible scenario?

You're right, that is a silly scenario ;) In fact if we mark vmas with VM_DONTCOPY, then the child just crashes with a seg fault.

The type of thing I'm worried about is something like, for example:

a) app registers memory region with RDMA hardware -- in other words, loads the device's translation table for future I/O
b) app forks
c) app writes to the registered memory region, and the kernel breaks the COW for the (now read-only) page by mapping a new page
d) app starts an I/O that will do a DMA read from the region
e) device reads using the wrong, old mapping

This can be pretty insiduous because for example fork() + immediate exec() or just using system() still leaves the parent with PTEs marked read-only. If an application does overlapping memory registrations so get_user_pages() is called a lot, then as far as I can see can_share_swap_page() will always return 0 and the COW will happen even if the child process has thrown out its original vmas.

Or if the counts are in the correct range, then there's a small window between fork() and exec() where the parent process can screw itself up, so most of the time the app works, until it doesn't.

- R.

Whoa, hang on.

The way we expect get_user_pages() to be used is that the kernel will use get_user_pages() once per application I/O request.

Are you saying that RDMA clients will semi-permanently own pages which were pinned by get_user_pages()? That those pages will be used for multiple separate I/O operations?

If so, then that's a significant design departure and it would be good to hear why it is necessary.

Sure. But such an app could be declared to be buggy...

Yes, absolutely!

The memory buffer is allocated by the process (usually just via malloc) and registed/pinned by the driver. It then stays pinned for the life of the process (typically).

That's just how RMDA works. Once the memory is pinned, if the app wants to send data to another node, it does two things:

1) Puts the data into its buffer
2) Sends a "work request" to the driver with (among other things) the offset and length of the data.

This is a time-critical operation. It must occurs as fast as possible, which means the memory must have already been pinned.

Timur Tabi
Staff Software Engineer
[email protected]

One thing a Southern boy will never say is,
"I don't think duct tape will fix it."
     -- Ed Smylie, NASA engineer for Apollo 13

ug. What stops the memory from leaking if the process exits?

I hope this is a privileged operation?

It would be better to obtain this memory via a mmap() of some special device node, so we can perform appropriate permission checking and clean everything up on unclean application exit.

Andrew> ug. What stops the memory from leaking if the process Andrew> exits?

Andrew> I hope this is a privileged operation?

I don't think it has to be privileged. In my implementation, the driver keeps a per-process list of registered memory regions and unpins/cleans up on process exit.

Andrew> It would be better to obtain this memory via a mmap() of Andrew> some special device node, so we can perform appropriate Andrew> permission checking and clean everything up on unclean Andrew> application exit.

This seems to interact poorly with how applications want to use RDMA, ie typically through a library interface such as MPI. People doing HPC don't want to recode their apps to use a new allocator, they just want to link to a new MPI library and have the app go fast.

- R.

How does the driver detect process exit?

Fair enough.

Andrew> How does the driver detect process exit?

I already answered earlier but just to be clear: registration goes through a character device, and all regions are cleaned up in the ->release() of that device.

I don't currently have any code accounting against RLIMIT_MEMLOCK or testing CAP_FOO, but I have no problem adding whatever is thought appropriate. Userspace also has control over the permissions and owner/group of the /dev node.

- R.

yup.

I guess device node permissions won't be appropriate here, if only because it sounds like everyone will go and set them to 0666.

RLIMIT_MEMLOCK sounds like the appropriate mechanism. We cannot rely upon userspace running mlock(), so perhaps it is appropriate to run sys_mlock() in-kernel because that gives us the appropriate RLIMIT_MEMLOCK checking.

However an hostile app can just go and run munlock() and then allocate some more pinned-by-get_user_pages() memory.

umm, how about we

- force the special pages into a separate vma

- run get_user_pages() against it all

- use RLIMIT_MEMLOCK accounting to check whether the user is allowed to do this thing

- undo the RMLIMIT_MEMLOCK accounting in ->release

This will all interact with user-initiated mlock/munlock in messy ways. Maybe a new kernel-internal vma->vm_flag which works like VM_LOCKED but is unaffected by mlock/munlock activity is needed.

A bit of generalisation in do_mlock() should suit?

I don't see what's wrong with relying on userspace to call mlock(). First all, all RDMA apps call a third-party API, like DAPL or MPI, to register memory. The memory needs to be registered in order for the driver and adapter to know where it is. During this registration, the memory is also pinned. That's when we call mlock().

Isn't mlock() on a per-process basis anyway? How can one process call munlock() on another process' memory?

Isn't this kinda what mlock() does already? Create a new VMA and then VM_LOCK it?

Yes, but do_mlock() needs to prevent pages from being moved during memory hotswap.

All the above refers to well-behaved applications.

Now think about how the syscalls which you provide may be used by applications which are *designed* to cripple or to compromise the machine.

I'm referring to an application which uses your syscalls to obtain pinned memory and uses munlock() so that it may then use your syscalls to obtain evem more pinned memory. With the objective of taking the machine down.

kinda. But applications can undo the mlock which the kernel did.

I haven't even thought about memory hotswap. Surely it'll fail if the pages are pinned by get_user_pages()?

I'm in favor of having drivers call do_mlock() and do_munlock() on behalf of the application. All we need to do is export those functions, and my driver can call them. However, that still doesn't prevent an app from calling munlock().

But I don't understand the distinction between having the driver call do_mlock() vs. the application calling mlock(). Won't we still have the same problems? A malicious app can just call our driver instead of calling mlock() or munlock(). The driver won't know the difference between an authorized app and an unauthorized one.

Besides, isn't the whole point behind RLIMIT_MEMLOCK to limit how much one process can lock?

Any memory registered for RDMA devices obviously can't be swapped out. Technically, the driver could detect that, and reject any attempt to transfer data to those regions until everything is remapped to other RAM. But that's opening a whole new can of worms. I don't know how the memory hotswap mechanism works, so I can't guess what recovery mechanisms can be implemented in the driver.

Precisely. That's why I suggested that we have an alternative vma->vm_flag bit which behaves in a similar manner to VM_LOCKED (say, VM_LOCKED_KERNEL), only userspace cannot alter it.

The driver will set VM_LOCKED_KERNEL, not VM_LOCKED.

Sure. The internal setting of VM_LOCKED_KERNEL should still use RLIMIT_MEMLOCK accounting.

How about calling it VM_PINNED? That way, we can define

Locked - won't be swapped to disk, but can be moved around in memory Pinned - won't be swapped to disk or moved around in memory

Andrew> umm, how about we

Andrew> - force the special pages into a separate vma

Andrew> - run get_user_pages() against it all

Andrew> - use RLIMIT_MEMLOCK accounting to check whether the user Andrew> is allowed to do this thing

Andrew> - undo the RMLIMIT_MEMLOCK accounting in ->release

Andrew> This will all interact with user-initiated mlock/munlock Andrew> in messy ways. Maybe a new kernel-internal vma->vm_flag Andrew> which works like VM_LOCKED but is unaffected by Andrew> mlock/munlock activity is needed.

Andrew> A bit of generalisation in do_mlock() should suit?

Yes, it seems that modifying do_mlock() to something like

int do_mlock(unsigned long start, size_t len,
unsigned int set, unsigned int clear)

and then exporting a function along the lines of

int do_mem_pin(unsigned long start, size_t len, int on)

that sets/clears (VM_LOCKED_KERNEL | VM_DONTCOPY) according to the on flag.

Seem reasonable? If so I can code this up.

- R.

Do you mean that the set/clear parameters to do_mlock() are the
actual flags which are set/cleared by the caller? Also, the issue remains that the flags are not reference counted which is a problem if you are dealing with overlapping memory region, or even if one region ends and another begins on the same page. Since the desire is to be able to pin any memory that a user can malloc this is a likely scenario.

-Libor

Libor> Do you mean that the set/clear parameters to do_mlock()
Libor> are the actual flags which are set/cleared by the caller? Libor> Also, the issue remains that the flags are not reference Libor> counted which is a problem if you are dealing with Libor> overlapping memory region, or even if one region ends and Libor> another begins on the same page. Since the desire is to be Libor> able to pin any memory that a user can malloc this is a Libor> likely scenario.

Good point... we need to figure out how to handle:

    a) app registers 0x0000 through 0x17ff
    b) app registers 0x1800 through 0x2fff
    c) app unregisters 0x0000 through 0x17ff
    d) the page at 0x1000 must stay pinned

hmm...

- R.

The userspace library should be able to track the tree and the overlaps, etc. Things might become interesting when the memory is MAP_SHARED pagecache and multiple independent processes are involved, although I guess that'd work OK.

But afaict the problem wherein part of a page needs VM_DONTCOPY and the other part does not cannot be solved.

    Roland>     a) app registers 0x0000 through 0x17ff
    Roland>     b) app registers 0x1800 through 0x2fff
    Roland>     c) app unregisters 0x0000 through 0x17ff
    Roland>     d) the page at 0x1000 must stay pinned

Andrew> The userspace library should be able to track the tree and Andrew> the overlaps, etc. Things might become interesting when Andrew> the memory is MAP_SHARED pagecache and multiple Andrew> independent processes are involved, although I guess Andrew> that'd work OK.

I used to think I knew how to handle this, but in your scheme where the kernel is doing accounting for pinned memory by marking vmas with VM_KERNEL_LOCKED, at step c), I don't see why the kernel won't unlock vmas covering 0x0000 through 0x1fff and credit 8K back to the process's pinning count.

Sorry to be so dense but can you spell out what you think should happen at steps a), b) and c) above?

Andrew> But afaict the problem wherein part of a page needs Andrew> VM_DONTCOPY and the other part does not cannot be solved.

Yes, I agree. If an app wants to register half a page and pass the other half to a child process, I think the only answer is "don't do that then."

- R.

How can the app know that, though? It would have to allocate I/O buffers with knowledge of page boundaries. Today, the apps just malloc() a bunch of memory and pay no attention to whether the beginning or the end of the buffer shares a page with some other, unrelated object. We may as well tell the app that it needs to page-align all I/O buffers.

My point is that we can't just simply say, "Don't do that". Some entity (the kernel, libraries, whatever) should be able to tell the app that its usage of memory is going to break in some unpredictable way.

Timur Tabi
Staff Software Engineer
[email protected]

One thing a Southern boy will never say is,
"I don't think duct tape will fix it."
     -- Ed Smylie, NASA engineer for Apollo 13

Our point is that contemporary microprocessors cannot electrically do what you want them to do!

Now, conceeeeeeiveably the kernel could keep track of the state of the pages down to the byte level, and could keep track of all COWed pages and could look at faulting addresses at the byte level and could copy sub-page ranges by hand from one process's address space into another process's after I/O completion. I don't think we want to do that.

Methinks your specification is busted.

I agree in principal. However, I expect this issue will come up with more and more new specifications, and if it isn't addressed once in the linux kernel, it will be kludged and broken many times in many drivers.

I believe we need an kernel level interface that will pin user pages, and lock the user vma in a single step. The interface should be used by drivers when the hardware mappings are done. If the process is split into a user operation to lock the memory, and a driver operation to map the hardware, there will always be opportunity for abuse.

Reference counting needs to be done by this interface to allow different hardware to interoperate.

The interface can't overload the VM_LOCKED flag, or rely on any other attributes that the user can tinker with via any other interface.

And as much as I hate to admit it, I think on a fork, we will need to copy parts of pages at the beginning or end of user I/O buffers.

Bill Jordan
InfiniCon Systems

I agree with all but the last part, in my opinion there is no need to deal with fork issues as long as solutions do not result in failures. There is *no* basis for a child process to expect that it will inherit RDMA resources. A child process that uses such resources will get undefined results, nothing further needs to be stated, and no heroic efforts are required to avoid them.

What is definitely needed is kernel counting of locks on user pages. Finer granularity is not expected, it is the RDMA hardware that works at finer granularity. All it needs is to know what bus address a given virtual page maps to -- and it needs to know that said mapping will not change without advance notice.

Further, any revocation of an existing mapping (to deal with hot page swapping or whatever) cannot expect the RDMA hardware to respond any faster than it would to invalidating a memory region.

The RDMA hardware has an inherent need to cache translations. That is why it cannot guarantee that it will cease updating a memory region the nanosecond that a request is made to invalidate an STag. Instead it is allowed to block on such a request and only guarantees to have ceased access when the invalidate request completes.

The same need for a delay exists for any interface that moves memory around, or requests to reclaim memory from the application.

This also applies on process death. The hardware cannot stop on a dime. The best it can do is stop promptly, and given an unambiguous indication to the OS as to when it has stopped.

Is there anything wrong with the following plan?

1) For memory registration, use get_user_pages() in the kernel. Use locked_vm and RLIMIT_MEMLOCK to limit the amount of memory pinned by a given process. One disadvantage of this is that the accounting will overestimate the amount of pinned memory if a process pins the same page twice, but this doesn't seem that bad to me -- it errs on the side of safety.

2) For fork() support:

a) Extend mprotect() with PROT_DONTCOPY so processes can avoid copy-on-write problems.

b) (maybe someday?) Add a VM_ALWAYSCOPY flag and extend mprotect() with PROT_ALWAYSCOPY so processes can mark pages to be pre-copied into child processes, to handle the case where only half a page is registered.

I believe this puts the code that must be trusted into the kernel and gives userspace primitives that let apps handle the rest.

- R.

I think the overestimate will be fine in practice. If a process is
locking a lot of memory it will most likely be in big chunks, so not much page overlap there. If the process is locking lots of tiny buffers with lots of page overlap, the total locked amount will most likely be small. Although it is odd that you could end up with a total locked amount larger then the number of physical pages in the system...

I'm assuming that for libibverbs memory registration you plan on hiding
the mprotect in the library? Without reference counting at the kernel level this could yield unexpected results in a perfectly legitimate app.

For example if the app is managing a buffer it will pass to another
device, but also want's to move data in/out with RDMA hardware, the user marks it themselves with DONTCOPY, registers with libibverbs, performs IO, unregisters with libibverbs. At this point the user expects the buffer to have DONTCOPY set, but it does not because of the unregister... Not that it's likely, but it's a valid thing to do. However, since I don't have a better suggestion, I'm in favour of using mprotect as you outlined.

-Libor

Do you plan to work with David Addison from Quadrics ? For sure, your hardware have very different capabilities. But ioproc_ops is a really nice solution and might help a lot when dealing with deregistration and fork.

For instance, instead of adding PROT_DONT/ALWAYSCOPY, you may use an ioproc hook in the fork path. This hook (a function in your driver) would be called for each registered page. It will decide whether the page should be pre-copied or not and update the registration table (or whatever stores address translations in the NIC). In addition, the driver would probably pre-copy cow pages when registering them.

It's nice to see these two works coming to LKML at the same time. It would be great if we could merge them and get a generic solution that's suitable to both registration based cards (IB/Myri/Ammasso) and MMU-based cards (Quadrics).

Brice

Brice> Do you plan to work with David Addison from Quadrics ? For Brice> sure, your hardware have very different capabilities. But Brice> ioproc_ops is a really nice solution and might help a lot Brice> when dealing with deregistration and fork.

I'm following the discussion with interest. Some hardware (eg Mellanox HCAs) has the ability to use these hooks to avoid pinning pages at all, but in general IB and iWARP need to pin pages so the mapping doesn't change.

Brice> For instance, instead of adding PROT_DONT/ALWAYSCOPY, you Brice> may use an ioproc hook in the fork path. This hook (a Brice> function in your driver) would be called for each Brice> registered page. It will decide whether the page should be Brice> pre-copied or not and update the registration table (or Brice> whatever stores address translations in the NIC). In Brice> addition, the driver would probably pre-copy cow pages when Brice> registering them.

This sort of monkeying around with the VM from driver code seems much more complicated than letting userspace handle it.

- R.

This doesn't scale well as more cards are added to the box. I think I understand why it's good for single cards though.

Aren't the mellanox mem-free cards more or less MMU's as well? I had that impression after attending Dror Goldberg's talk though I don't think he asserted that.
Openib.org developers conf (Feb 2005) slideset is here:
http://www.openib.org/docs/oib_wkshp_022005/memfree-hca-mellanox-dgoldenberg.pdf

Being mostly clueless about Quadrics implementation, I'm probably missing something that makes Quadrics a MMU but not the IB variants. Can someone clue me in please?

thanks,
grant

With the IOPROC patch the device driver hooks are registered on a per process or perhaps better still, a per VMA basis. And for processes/VMAs where there are no registrations the overhead is very low.

With multiple cards in a box, all using different device drivers, I guess there could end up being multiple registrations per process/VMA. But I'm not sure this will be a common case for RDMA use in real life.

Cheers
Addy.

I was originally thinking the registrations are global (for all memory) and not per process. Per process or per VMA seems reasonable to me.

Yes - thanks. I'm still reading the LKML thread you started:

	http://lkml.org/lkml/2005/4/26/198

In particular, the comments from Brice Goglin:

	http://lkml.org/lkml/2005/4/26/222

openib.org folks can find the IOPROC patch for 2.6.12-rc3 archived here:

	http://lkml.org/lkml/diff/2005/4/26/198/1

I agree. Gateways between fabrics is the only case I can think of. This won't be a problem until someone at a large national lab tries to connect two "legacy" fabrics together.

thanks,
grant

Are you suggesting making the partial pages their own VMA, or marking the entire buffer with this flag? I originally thought the entire buffer should be copy on fork (instead of copy on write), and I believe this is the path Mellanox was pursing with the VM_NO_COW flag. However, if applications are registering gigs of ram, it would be very bad to have the entire area copied on fork.

On the other hand, I've always wondered about the choice to leave holes in the child process's address space. I would have chosen to map the zero page instead.

Bill Jordan
InfiniCon Systems

Bill> Are you suggesting making the partial pages their own VMA, Bill> or marking the entire buffer with this flag? I originally Bill> thought the entire buffer should be copy on fork (instead of Bill> copy on write), and I believe this is the path Mellanox was Bill> pursing with the VM_NO_COW flag. However, if applications Bill> are registering gigs of ram, it would be very bad to have Bill> the entire area copied on fork.

It's up to userspace really but I would expect that the partial pages would be in a vma by themselves.

- R.

However, you have a potential problem with registered buffers that
do not begin or end on a page boundary, which is common with malloc. If the buffer resides on a portion of a page, and you mark the vm which contains that entire page VM_DONTCOPY, to ensure that the parent has access to the exact physical page after the fork, the child will not be able to access anything on that entire page. So if the child expects to access data on the same page that happens to contain the registered buffer it will get a segment violation.

The four situations we've discussed are:

1) Physical page does not get used for anything else.
2) Processes virtual to physical mapping remains fixed.
3) Same virtual to physical mapping after forking a child.
4) Forked child has access to all non-registered memory of the parent.

The first two are now taken care of with get_user_pages, (we use to use VM_LOCKED for the second case) third case is handled by setting the vm to VM_DONTCOPY, and on the fourth case we've always punted, but the real answer is to break partial pages into seperate vms and mark them ALWAYS_COPY.

-Libor

Attempting to provide *any* support for applications that fork children after doing RDMA registrations is a ratshole best avoided. The general rule that application developers should follow is to do RDMA *only* in the child processes.

Keep in mind that it is not only the memory regions that must be dealt with, but control data invisible to the user (the QP context, etc.). This data frequently is interlinked between kernel residente and user resident data (such as a QP context has the PD ID somewhere on-chip or in kernel, which the Send Queue ring needs to be in user memory). Having two different user processes that both think they have the user half to this type of split data structure is just asking for trouble, even if you manage to get the copy on write bit timing problems all solved.

All of this can be avoided by a simple rule: don't fork after opening an RDMA device.

I think it's unreasonable to *prohibit* fork-after-registration; for one thing, there's lots of code that forks under the covers. Setuid helpers like getpty just assume that they're going to be able to fork. Even stuff like get*by*(3) can potentially fork. And with site-configured stuff like PAM, you end up with things that work on the developer's system but break in deployment.

I think it's exceedingly reasonable to say "RDMA doesn't work in children". But the child should get a sane memory image: at least zeros in fully-registered pages, and preferably copies of partially-registered pages. Differentiating between fully-registered and partially-registered pages avoids (I think) the pathological case of having to copy a GB of data just to system("/bin/ls > /tmp/tmpfile"). You can still go pathological if you've partially-registered gigabytes of address space (for example a linked list where each node is allocated with malloc and then registered) but that's a case of "Well, don't do that then".

Rather than replacing the fully-registered pages with pages of zeros, you could simply unmap them.

A consistent statement would be

After fork(2), any regions which were registered are UNDEFINED. Region boundaries are byte-accurate; a registration can cover just part of a page, in which case the non-registered part of the page has normal fork COW semantics.

Probably the most sane solution is to simply unmap the fully-registered pages at fork time, and copy any partially-registered pages. But the statement above does not require this.

Obviously, calling *any* RDMA-userland-stuff in the child is completely undefined [1]. One place where I can see a potential problem is in atexit()-type handlers registered by the RDMA library. Since those aren't performance-critical they can and should do sanity checks with getpid() and/or checking with the kernel driver.

[1] You might want to allow the child to start a completely new RDMA context, but I don't see that as necessary.

-andy

That is a reasonable approach.

That is also reasonable. None of the RDMA libraries I have worked on bothered to use an atexit()-type hook because the user was theoretically *required* to close the rnic, and driver code was already reuqired to clean up in case of a total process failure. Adding an intermediate safety-net for applications that exited cleanly but forget to close just didn't seem worthwhile. If the application wants the cleanup performed optimally then it can close the rnic, otherwise it can't complain about forcing the RNIC vendor to clean up in the driver code.

That should be allowed. It is actually more normal to use the parent as a dispatcher and to actually manage the connection in a child process.

I don't like this option. It is nearly free to map all of the pages to the zero-page. You never have to allocate a page if the user never writes to it.

Buf if you unmap the page, there could be issues. The memory region could be on the stack, or malloc'ed. In these cases, the child should be able to return from the function, or free the memory without setting a timebomb.

Bill Jordan
InfiniCon Systems

Unmapping should work fine, as long as the VMA flags are set appropriately. The page fault handler can take care of the rest...

"Debugging is twice as hard as writing the code in the first place.
Therefore, if you write the code as cleverly as possible, you are,
by definition, not smart enough to debug it." - Brian W. Kernighan

I think there may be a difference in terminology here. What I originally proposed (and what I think Bill was reacting to) is the equivalent of sys_munmap() on the range of registered pages. That has the downsides that he mentioned; an address that was valid in the parent will now result in SIGSEGV or SIGBUS in the child, and it's explicitly endorsed by the userland APIs (such as MPI2) that it's valid to register stack addresses (for example).

What I think you're proposing, Rik, is that VMA get destroyed (or split, if only part of it had been registered) and replaced with an anonymous one. That's a very low-overhead way of going about it, I think. Then as you say, the page fault handler will automatically give a zero page to the process when it faults on those addresses.

Did I understand your suggestion correctly? I think I agree with Bill that having the child fault on pages which happened to have been registered by the parent would be a bad thing.

This would, if I understand correctly, be visible in /proc/$$/maps. Which is OK, if a little bit suprising; but the alternatives are worse.

-andy

Well I was vaguely proposing that the userspace library keep track of the byteranges and the underlying page states. So in the above scenario userspace would leave the page at 0x1000 registered until all registrations against that page have been undone.

Andrew> Well I was vaguely proposing that the userspace library Andrew> keep track of the byteranges and the underlying page Andrew> states. So in the above scenario userspace would leave Andrew> the page at 0x1000 registered until all registrations Andrew> against that page have been undone.

OK, I already have code in userspace that keeps reference counts for overlapping regions, etc. However I'm not sure how to tie this in with reliable accounting of pinned memory -- we don't want malicious userspace code to be able fool the accounting, right?

So I'm still trying to puzzle out what to do. I don't want to keep a complicated data structure in the kernel keeping track of what memory has been registered. Right now, I just keep a list of structs, one for each region, and when a process dies, I just go through region by region and do a put_page() to balance off the get_user_pages().

However I don't see how to make it work if I put the reference counting for overlapping regions in userspace but when I want mlock() accounting in the kernel. If a buggy/malicious app does:

    a) register from 0x0000 to 0x2fff
    b) register from 0x1000 to 0x1fff
    c) unregister from 0x0000 to 0x2fff

then it seems the kernel is screwed unless it counts how many times a vma has been pinned. And adding a pin_count member to vm_struct seems like a pretty damn major step.

We definitely have to make sure that userspace is never able to either unpin a page that is still registered with RDMA hardware, because that can lead to DMA to into memory that someone else owns. On the other hand, we don't want userspace to be able to defeat resource accounting by tricking the kernel into keeping page_count elevated after it credits the memory back to a process's limit on locked pages.

The limit on the number of locked pages seems like a natural thing to check against, but perhaps we need a different limit for the number of pages pinned for use by RDMA hardware. Sort of the same way that there's a separate limit on the number of in-flight aios.

- R.

As far as the kernel is concerned, step b) should be a no-op. (The kernel might choose to split the vma, but that's not significant).

The kernel can simply register and unregister ranges for RDMA. So effectively a particular page is in either the registered or unregistered state. Kernel accounting counts the number of registered pages and compares this with rlimits.

On top of all that, your userspace library needs to keep track of when pages should really be registered and unregistered with the kernel. Using overlap logic and per-page refcounting or whatever.

No?

Andrew> The kernel can simply register and unregister ranges for Andrew> RDMA. So effectively a particular page is in either the Andrew> registered or unregistered state. Kernel accounting Andrew> counts the number of registered pages and compares this Andrew> with rlimits.

Andrew> On top of all that, your userspace library needs to keep Andrew> track of when pages should really be registered and Andrew> unregistered with the kernel. Using overlap logic and Andrew> per-page refcounting or whatever.

This is OK as long as userspace is trusted. However I don't see how this works when we don't trust userspace. The problem is that for an RDMA device (IB HCA or iWARP RNIC), a process can create many memory regions, each of which a separate virtual to physical translation map. For example, an app can do:

a) register 0x0000 through 0xffff and get memory handle 1
b) register 0x0000 through 0xffff and get memory handle 2
c) use memory handle 1 for communication with remote app A
d) use memory handle 2 for communication with remote app B

Even though memory handles 1 and 2 both refer to exactly the same memory, they may have different lifetimes, might be attached to different connections, and so on.

Clearly the memory at 0x0000 must stay pinned as long as the RDMA device thinks either memory handle 1 or memory handle 2 is valid. Furthermore, the kernel must be the one keeping track of how many regions refer to a given page because we can't allow userspace to be able to tell a device to go DMA to memory it doesn't own any more.

Creation and destruction of these memory handles will always go through the kernel driver, so this isn't so bad. And get_user_pages() is almost exactly what we need: it stacks perfectly, since it operates on the page_count rather than just setting a bit in vm_flags. The main problem is that it doesn't check against RLIMIT_MEMLOCK.

The most reasonable thing to do would seem to be having the IB kernel memory region code update current->mm->locked_vm and check it against RLIMIT_MEMLOCK. I guess it would be good to figure out an appropriate abstraction to export rather than monkeying with current->mm directly. We could also put this directly in get_user_pages(), but I'd be worried about messing with current users.

I just don't see a way to make VM_KERNEL_LOCKED work.

It would also be nice to have a way for apps to set VM_DONTCOPY appropriately. Christoph's suggestion of extending mmap() and mprotect() with PROT_DONTCOPY seems good to me, especially since it means we don't have to export do_mlock() functionality to modules.

- R.

If "register" and "unregister" is meant in the RDMA sense then the above sequence is totally reasonable. The b) registration could be for a different protection domain that did not require access to all of the larger region.

Unless a full counting lock is available from the kernel, the responsibility of the collective RDMA components would be to a) pin 0x0000 to 0x2fff,
b) nothing c) unpin 0x000 to 0x0fff and 0x2000 to 0x2fff

Which portion of the userspace library? HCA-dependent code, or common code?

The HCA-dependent code would fail to count when the same memory was registered to different HCAs (for example to the internal network device and the external network device).

The vendor-independent code *could* do it, but only by maintaining a complete list of all registrations that had been issued but not cancelled. That data would be redundant with data kept at the verb layer, and by the kernel.

It *would' work, but maintaining highly redundant data at multiple layers is something that I generally try to avoid.

At Mon, 25 Apr 2005 16:58:03 -0700,

Such HPC users cannot use the memory hotremoval feature, and something needs to be implemented so that the NUMA migration can handle such memory properly, but I see your point.

If such memory were allocated by a driver, the memory could be placed in non-hotremovable areas to avoid the above problems.

--
IWAMOTO Toshihiro

How can the driver allocated 3GB of pinned memory on a system with 3.5GB of RAM? Can vmalloc() or get_free_pages() allocate that much memory?

I don't think that we should jump to the conclusion that in the long term HPC users cannot benefit from support of mechanisms such as hotremoval of memory or other forms of page migration in physical memory. In an earlier exchange on the openib-general list Mike Krause sent the message quoted below on very much the same topic. On the other hand I am willing to accept that there is practical value to implementations which are not (yet) sophisticated to enough to support the migration functions.

Steve Langdon

begin:vcard
fn:Steve Langdon
n:Langdon;Stephen
org:Hewlett-Packard;Consulting & Architecture Group adr:MS LKG1-3/B19;;550 King Street;Littleton;MA;01460;USA email;internet:[email protected]
title:Fellow
tel;work:+1 978-506-5771
tel;fax:+1 978-742-1144
tel;home:+1 978-456-8177
tel;cell:+1 978-618-8599
x-mozilla-html:TRUE
version:2.1
end:vcard

Are you saying that the mapping obtained by get_user_pages() is valid only within the context of the IOCtl call? That once the driver returns from the IOCtl, the mapping should no longer be used?

Timur Tabi
Staff Software Engineer
[email protected]

One thing a Southern boy will never say is,
"I don't think duct tape will fix it."
     -- Ed Smylie, NASA engineer for Apollo 13

Yes, we expect that all the pages which get_user_pages() pinned will become unpinned within the context of the syscall which pinned the pages. Or shortly after, in the case of async I/O.

This is because there is no file descriptor or anything else associated with the pages which permits the kernel to clean stuff up on unclean application exit. Also there are the obvious issues with permitting pinning of unbounded amounts of memory.

Then that might explain the "bug" that we're seeing with get_user_pages(). We've been assuming that get_user_pages() mappings are permanent.

Well, I was just about to re-implement get_user_pages() support in our driver to demonstrate the bug. I guess I'll hold off on that.

If you look at the Infiniband code that was recently submitted, I think you'll see it does exactly that: after calling mlock(), the driver calls get_user_pages(), and it stores the page mappings for future use.

Timur Tabi
Staff Software Engineer
[email protected]

One thing a Southern boy will never say is,
"I don't think duct tape will fix it."
     -- Ed Smylie, NASA engineer for Apollo 13

They are permanent until someone runs put_page() against all the pages. What I'm saying is that all current callers of get_user_pages() _do_ run put_page() within the same syscall or upon I/O termination.

Where?

bix:/usr/src/linux-2.6.12-rc3> grep -rl get_user_pages . ./arch/i386/lib/usercopy.c ./arch/sparc64/kernel/ptrace.c ./drivers/video/pvr2fb.c ./drivers/media/video/video-buf.c ./drivers/scsi/sg.c
./drivers/scsi/st.c ./include/asm-ia64/pgtable.h ./include/linux/mm.h ./include/asm-um/archparam-i386.h ./include/asm-i386/fixmap.h
./fs/nfs/direct.c
./fs/aio.c
./fs/binfmt_elf.c
./fs/bio.c
./fs/direct-io.c
./kernel/futex.c
./kernel/ptrace.c
./mm/memory.c
./mm/nommu.c
./mm/rmap.c
./mm/mempolicy.c

Oh, okay then. I guess I'll get back to work!

Actually, with RDMA, "I/O termination" technically doesn't happen until the memory is deregistered. When the memory is registered, all that means is that it's should be pinned and the virtual-to-physical should be stored. No actual I/O occurs at that point.

I was talking about the code that Roland mentioned in the first message of this thread - the user-space verbs support. He said the code calls mlock() and get_user_pages().

FYI, our driver detects the process termination and cleans up everything itself.

Timur Tabi
Staff Software Engineer
[email protected]

One thing a Southern boy will never say is,
"I don't think duct tape will fix it."
     -- Ed Smylie, NASA engineer for Apollo 13

How is this implemented?

Timur> If you look at the Infiniband code that was recently Timur> submitted, I think you'll see it does exactly that: after Timur> calling mlock(), the driver calls get_user_pages(), and it Timur> stores the page mappings for future use.

Andrew> Where?

The code isn't merged yet. I sent a version to lkml for review -- in fact it was this very thread that we're in now. The code in question is in http://lkml.org/lkml/2005/4/4/266

This implements a "userspace verbs" character device that memory registration goes through. This means the kernel has a device node that will be closed when a process dies, and so the memory can be cleaned up.

- R.

Andrew Morton wrote,

There definitely needs to be a mechanism to prevent people from pinning too much memory. We saw issues in the sourceforge stack and some of the vendors stacks where we could lock memory till the system hung. In the sourceforge InfiniBand stack, we put in a
check to make sure that people did not pin too much memory.
It was sort of a crude/bruit force mechanism, but effective. I think that we limited people from locking down more that 1/2 of kernel memory or 70 % of all memory (it was tunable with a module option) and if they exceeded
the limit, their requests to register memory would begin to fail. Arlin can provide details on how we did it or people can look at the IBAL code for an example.

woody

Any limit would have to be very high - definitely more than just half. What if the application needs to pin 2GB? The customer is not going to buy 4+ GB of RAM just because Linux doesn't like pinning more than half. In an x86-32 system, that would required PAE support and slow everything down.

Off the top of my head, I'd say Linux would need to allow all but 512MB to be pinned. So you have 3GB of RAM, Linux should allow you to pin 2.5GB.

Timur Tabi
Staff Software Engineer
[email protected]

One thing a Southern boy will never say is,
"I don't think duct tape will fix it."
     -- Ed Smylie, NASA engineer for Apollo 13

You can pin the whole darn lot *if you have the correct privileges*.

Timur Tabi wrote,

What if the

RAM just

required >PAE

be pinned. >So

That is why we made it tunable, so that people could decide how to allow.

There is probably a better way to do it than some hard limit, but that would take a little more understanding of the VM system than we had, and that is why some of the core kernel folks maybe able to help us come up with a better solution.

woody

When a network protocol is making use of async I/O the amount of time
between posting the read request and getting the completion for that request is unbounded since it depends on the other half of the connection sending some data. In this case the buffer that was pinned during the io_submit() may be pinned, and holding the pages, for a long time. During this time the process might fork, at this point any data received will be placed into the wrong spot.

Correct, the driver must be able to determine that the process has died
and clean up after it, so the pinned region in most implementations is associated with an open file descriptor.

-Libor

Sure.

Well the data is placed in _a_ spot. That's only the "wrong" spot because you've defined it to be wrong!

IOW: what behaviour are you actually looking for here, and why, and does it matter?

How is that association created?

There is not a file descrptor, but there is an rnic handle. Both DAPL and IT-API require that process death will result in the handle and all of its dependent objects being released.

The rnic handle can always be declared to be a "file descriptor" if that makes it follow normal OS conventions more precisiely.

There is also a need for some form of resource manager to approve creation of Memory Regions. Obviously you cannot have multiple applications claiming half of physical memory.

But if you merely require the user to have root privileges in order to create a Memory Region, and then take a first-come first-served attitude, I don't think you end up with something that is truly a general purpose capability.

A general purpose RDMA capability requires the ability to indefinitely pin large portions of user memory. It makes sense to integrate that with OS policy control over resource utilization and to integrate it with memory suspend/resume capabilities so that hotplug memory can be supported. What you can't do is downgrade a Memory Region so that it is no longer a memory region. Doing that means that you are not truly supporting RDMA.

What's an "rnic handle", in Linux terms?

Does that mean that the code has not yet been implemented?

Yes, a Linux fd is appropriate. But we don't have any sane way right now of saying "you need to run put_page() against all these pages in the ->release() handler". That'll need to be coded by yourselves.

The kernel already has considerable resource management capabilities. Please consider using/extending/generalising those before inventing anything new. RLIMIT_MEMLOCK would be a starting point.

We don't want code in the kernel which will permit hostile unprivileged users to trivially cause the box to lock up. RLIMIT_MEMLOCK and, if necessary, CAP_IPC_LOCK sound appropriate here.

For example a network server app has an open connection on which it
uses async IO to submit two buffers for a read operation. Both buffers are pinned using get_user_pages() and the connection waits for data to arrive. The connection received data, it is written into the first buffer, the app is notified using async IO, and it retreives the async IO completion. The app reads the buffer which happens to contain a command to spawn a child, the app forks a child. Now there is still a buffer posted for read and if more data arrives on the connection that data is copied to the pages which were saved when the buffer was pinned. The app is notified, retrieves the async IO completion, but when it goes to read that buffer it will not have the new data.

The kernel module which pinned the memory is responsible for unpinning
it if the file descriptor, which was used to deliver the command that resulted in the pinning, is closed.

-Libor

Andrew> Whoa, hang on.

Andrew> The way we expect get_user_pages() to be used is that the Andrew> kernel will use get_user_pages() once per application I/O Andrew> request.

Andrew> Are you saying that RDMA clients will semi-permanently own Andrew> pages which were pinned by get_user_pages()? That those Andrew> pages will be used for multiple separate I/O operations?

Andrew> If so, then that's a significant design departure and it Andrew> would be good to hear why it is necessary.

The idea is that applications manage the lifetime of pinned memory regions. They can do things like post multiple I/O operations without any page-walking overhead, or pass a buffer descriptor to a remote host who will send data at some indeterminate time in the future. In addition, InfiniBand has the notion of atomic operations, so a cluster application may be using some memory region to implement a global lock.

This might not be the most kernel-friendly design but it is pretty deeply ingrained in the design of RDMA transports like InfiniBand and iWARP (RDMA over IP).

I'm also not opposed to implementing some other mechanism to make this work, but the combiniation of get_user_pages() in the kernel and extending mprotect() to allow setting VM_DONTCOPY seems to work fine.

- R.

Actuallky, no it isn't. All these transports would work just fine with
the mmap a character device to hand out memory from the kernel approach I told you to use multiple times and Andrew mentioned in this thread aswell. What doesn't work with that design are the braindead designed by comittee APIs in the RDMA world - but I don't think we should care about them too much.

RDMA registers and uses the memory the user specifies. That is why byte granularity and multiple redundant registrations are explicitly specified.

The mechanism by which this requirement is implemented is of course OS dependent. But the requirements are that the application specifies what portion of their memory they want registered (or what set of physical pages if they have sufficient privilege) and that request is either honored or refused by a resource manager (one preferably as integrated with general OS resource management as possible).

The other aspect is that remotely enabled memory regions and memory windows most be enabled for hardware access for the duration of the region or window -- indefinitely until process death or explicit termination by the application layer.

Theoretically there is nothing in the wire protocols that requires source buffers to be pinned indefinitely, but that is the only way any RDMA interface has ever worked -- so "brain death" must be pretty widespread.

The fact that this problem must be solved for remotely accessible buffers, and that for cluster applications like MPI there is no distinction between buffers used for inbound messages and outbound messages, might have something to do with this.

User verbs needs to deal with these actual Memory Registration requirements, including the very real application need for Memory Windows. The solution should map to existing OS controls as much as possible.

I think you should. The whole point behind RDMA is that these APIs exist and are being used by real-world applications. You can't just ignore them because they're inconvenient.
If you're not willing to cater to these API's needs, then you may as well tell all the
RDMA developers to forgot about Linux and port everything to Windows instead.

The APIs are here to stay, and the whole point behind this thread is to discuss how Linux can support them.

Timur Tabi
Staff Software Engineer
[email protected]

One thing a Southern boy will never say is,
"I don't think duct tape will fix it."
     -- Ed Smylie, NASA engineer for Apollo 13

Andrew,

Libor Michalek posted a much more reasonable (to my limited understanding) bug description in <[email protected]>.

(And I'd love to provide a URL, but damned if I can figure out how to find that message on gmane. Clue-bat applications gladly accepted.)

Libor Michalek wrote:
# The driver did use get_user_pages() to elevated the refcount on all the

# pages it was going to use for IO, as well as call set_page_dirty() since
# the pages were going to have data written to them from the device.
# 
# The problem we were seeing is that the minor fault by the app resulted
# in a new physical page getting mapped for the application. The page that
# had the elevated refcount was still waiting for the data to be written
# to by the driver at the time that the app accessed the page causing the
# minor fault. Obviously since the app had a new mapping the data written
# by the driver was lost.
# 
# It looks like code was added to try_to_unmap_one() to address this, so   
# hopefully it's no longer an issue...

Which makes me think that Timur's bug is just an
insufficiently-understood version of Libor's.

-andy

Why do you call mlock() and get_user_pages()? In our code, we only call mlock(), and the memory is pinned. We have a test case that fails if only get_user_pages() is called, but it passes if only mlock() is called.

Timur Tabi
Staff Software Engineer
[email protected]

this is a myth; linux is free to move the page about in physical memory even if it's mlock()ed!!

And even then, the user can munlock the memory from another thread etc etc. Not a good idea.

get_user_pages() is used from AIO and other parts of the kernel for similar purposes and in fact is designed for it, so it better work. If it has bugs those should be fixed, not worked around!

Then Linux has a very odd definition of the word "locked".

Well, that's okay, because then the app is doing something stupid, so we don't worry about that.

I've been complaining about get_user_pages() not working for a long time now, but I can only demonstrate the problem with our hardware. See my other post in this thread for details.

Timur Tabi
Staff Software Engineer
[email protected]

you should since that physical page can be reused, say by a root process, and you'd be majorly screwed

I don't understand what you mean by "reused". The whole point behind pinning the memory is that it stays where it is. It doesn't get moved around and it doesn't get swapped out.

Timur Tabi
Staff Software Engineer
[email protected]

you just said that you didn't care that it got munlock'd. So you don't care that it gets freed either. And then reused.

Well, I can live with the app being able to call munlock(), because the apps that our customers use don't call munlock(). What I can't live with is a bug in the kernel that causes pinned pages to be swapped or moved.

Obviously, I would rather call get_user_pages() instead of mlock(), but I can't, because get_user_pages doesn't work. The page doesn't stay pinned at the physical address, but it does if I call mlock() and get_user_pages().

Actually, in our tests, calling mlock() appears to be good enough, but I'll update our code to call get_user_pages() as well.

Timur Tabi
Staff Software Engineer
[email protected]

darn, yes, this is true.
I know people who introduced
#define VM_RESERVED 0x00080000 /* Don't unmap it from swap_out
*/
to vm_flags just because of this. I'll just hold my breath and won't delve further.

Someone (aka Tospin, infinicon, and Amasso) should probably post a patch adding '#define VM_REGISTERD 0x01000000', and some extensions to something like 'madvise' to set pages to be registered.

My preference is said patch will also allow a way for the kernel to reclaim registered memory from an application under memory pressure.

I don't know if VM_REGISTERED is a good idea or not, but it should be absolutely impossible for the kernel to reclaim "registered" (aka pinned) memory, no matter what. For RDMA services (such as Infiniband, iWARP, etc), it's normal for non-root processes to pin hundreds of megabytes of memory, and that memory better be locked to those physical pages until the application deregisters them.

If kernel really thinks it needs to unpin those pages, then at the very least it should kill the process, and the syslog better have a very clear message indicating why. That way, the application doesn't continue thinking that everything's still going to work. If those pages become unpinned, the applications are going to experience serious data corruption.

If you take the hardline position that "the app is the only thing that matters", your code is unlikely to get merged. Linux is a general-purpose OS.

I don't think that Troy was suggesting the kernel should deregister memory without notifying the application. Personally, I envision something like the NetBSD Scheduler Activations (SA) work, where the kernel can notify the app of changes to its state in a very efficient manner. (According to the NetBSD design whitepaper, the kernel does an upcall whenever the multithreaded app gains or loses a CPU!)

In a Linux context, I doubt that fullblown SA is necessary or appropriate. Rather, I'd suggest two new signals, SIGMEMLOW and SIGMEMCRIT. The userland comms library registers handlers for both. When the kernel decides that it needs to reclaim some memory from the app, it sends SIGMEMLOW. The comms library then has the responsibility to un-reserve some memory in an orderly fashion. If a reasonable [1] time has expired since SIGMEMLOW and the kernel is still hungry, the kernel sends SIGMEMCRIT. At this point, the comms lib *must* unregister some memory [2] even if it has to drop state to do so; if it returns from the signal handler without having unregistered the memory, the kernel will SIGKILL.

[1] Part of the interface spec should cover the expectation as to how long the library is allowed to take; I'd guess that 2 timeslices should suffice.
[2] Is there a way for the kernel to pass down to userspace how many pages it wants, maybe in the sigcontext?

You might want to consider what happens with your communication system in a machine running power-saving modes (in the limit, suspend-to-disk). Of course most machines with Infiniband adapters aren't running swsusp, but it's not inconceivable that blade servers might sleep to lower power and cooling costs.

-andy

The problem is that our driver and library implement an API that we don't fully control. The API states that the application allocates the memory and tells the library to register it. The app then goes on its merry way until it's done, at which point it tells the library to deregister the memory. Neither the app nor the API has any provision for the app to be notified that the memory is no longer pinned and therefore can't be trusted. That would be considered a critical failure from the app's perspective, so the kernel would be doing it a favor by killing the process.

Any application that registers memory, will in all likelihood be running at 100% CPU non-stop. The computer is not going to be doing anything else but whatever that app is trying to do. The application could conceiveable register gigabytes of RAM, and if even a single page becomes unpinned, the whole thing is worthless. The application cannot do anything meaningful if it gets a message saying that some of the memory has become unpinned and should not be used.

So the real question is: how important is it to the kernel developers that Linux support these kinds of enterprise-class applications?

Timur Tabi
Staff Software Engineer
[email protected]

One thing a Southern boy will never say is,
"I don't think duct tape will fix it."
     -- Ed Smylie, NASA engineer for Apollo 13

I'm familiar with MPI 1.0 and 2.0, but I haven't been following the development of modern messaging APIs, so I might not make sense here...

Assuming that the app calls into the library on a fairly regular basis, you could implement a fast-path/slow-path scheme where the library normally operates in go-fast mode, but if a "unregister" event has occurred, the library falls back to a less performant mode.

But now having written that I'm thinking that it's not worth the bother
- if you've got a 512P MPP job, it's basically equivalent to job death for one of the nodes to go away in this manner -- even if the process is still running on the node, the fact that you took a giant performance hiccup is unacceptable. Therefore, cluster admins are going to do their darndest to avoid this behavior, so we might as well just kill the job and make it explicit.

While I understand your arguments, this kind of rhetoric is more likely to harden ears than to convince people you're right. I refer you to the "Live Patching Function" thread.

*You* need to come up with a solution that looks good to *the community* if you want it merged. In the long run, this process is likely to result in *your* systems working better than if you had just gone off and done your thing. If you have to do something that "tastes bad" to the average l-k hacker, *justify* it by addressing the alternatives and explaining why your solution is the right one.

I'm leaning towards agreeing that mlock()-alicious code is the right way to solve this problem, and it's not clear to me what the benefit of adding a new VM_REGISTERED flag would be.

Do you guys simply raise RLIMIT_MEMLOCK to allow apps to lock their pages? Or are you doing something more nasty?

(Oh, I see that Libor has contributed to the other branch of this thread... off to read...)

-andy

Not really. The whole point is to have the adapter DMA the data directly from memory to the network. That's why it's called RDMA - remote DMA.

Yes, and if it turns out that the same MPI application dies on Linux but not on Solaris because Linux doesn't really care if the memory stays pinned, then we're going to see a lot of MPI customers transitioning away from Linux.

True, but I'm not going to waste my time adding this support if the consensus I get from the kernel developers that they don't want Linux to behave this way.

A little more nasty. I raise RLIMIT_MEMLOCK in the driver to "unlimited" and also set cap_raise(IPC_LOCK). I do this because I need to support all 2.4 and 2.6 kernel versions with the same driver, but only 2.6.10 and later have any support for non-root mlock().

If and when our driver is submitted to the official kernel, that nastiness will be removed of course.

Timur Tabi
Staff Software Engineer
[email protected]

One thing a Southern boy will never say is,
"I don't think duct tape will fix it."
     -- Ed Smylie, NASA engineer for Apollo 13

FYI, that will not work on all 2.6 kernels. Specifically anything that's not using capabilities.

thanks,
-chris

Linux Security Modules     http://lsm.immunix.org     http://lsm.bkbits.net

It works with every kernel I've tried. I'm sure there are plenty of kernel configuration options that will break our driver. But as long as all the distros our customers use work, as well as reasonably-configured custom kernels, we're happy.

Timur Tabi
Staff Software Engineer
[email protected]

One thing a Southern boy will never say is,
"I don't think duct tape will fix it."
     -- Ed Smylie, NASA engineer for Apollo 13

Hey, if you're happy (and, as you said, you don't intend to merge that bit), I'm happy ;-)

thanks,
-chris

Linux Security Modules     http://lsm.immunix.org     http://lsm.bkbits.net

yeah... drivers giving unprivileged processes more privs belong on bugtraq though, not in the core kernel :)

I think we have been giving you that consensus from the very beginning :)

The very fact that you tried to trot out the "enterprise" card should have raised a huge flag...

thanks,

greg k-h

Can you tell me when Linux actually does this? I know in theory it can happen, but I've never seen it. Does the code to implement moving of data from one physical page to another even exist in any version of Linux?

Also, what would be the point? What reason would there be to move some data from one physical page to another, while keeping the same virtual address?

Timur Tabi
Staff Software Engineer
[email protected]

One thing a Southern boy will never say is,
"I don't think duct tape will fix it."
     -- Ed Smylie, NASA engineer for Apollo 13

hot(un)plug memory.

so that you can hot unplug the dimm in question.

I guess that's a bit of a high end though though... so maybe you don't care about it.

eh? I guess the kernel _is_ free to move the page about, but it doesn't.

We might do at some time in the future for memory hotplug, I guess.

Timur> Why do you call mlock() and get_user_pages()? In our code, Timur> we only call mlock(), and the memory is pinned. We have a Timur> test case that fails if only get_user_pages() is called, Timur> but it passes if only mlock() is called.

What if a buggy/malicious userspace program doesn't call mlock()?

- R.

Our library calls mlock() when the apps requests memory to be "registered". We then call munlock() when the app requests the memory to be unregistered. All apps talk to our library for all services. No apps talk to the driver directly.

Timur Tabi
Staff Software Engineer
[email protected]

Hi!

That does not cover "malicious" part.
Pavel

64 bytes from 195.113.31.123: icmp_seq=28 ttl=51 time=448769.1 ms