Skip to contentSkip to conversation

staging/lustre/llite: cache jobid in lu_env

PATCH In review Peng Tao 3 participants 12 messages Oct 30, 2013 – Feb 4, 2014 Staging

lore
Share & tools
lore

Peng Tao

author
#0 lore raw

PATCH 1/4 staging/lustre/llite: cache jobid in lu_env

We will switch to find jobid from /proc/self/environ and we need an extra memory copy to do it, so let's cache it in lu_env. It is then copied from lu_env to ll_inode_info upon every read/write.

---
 drivers/staging/lustre/lustre/llite/file.c         |    2 ++
 .../staging/lustre/lustre/llite/llite_internal.h   |   22 ++++++++++++++++++++
 drivers/staging/lustre/lustre/llite/llite_mmap.c   |    1 +
 drivers/staging/lustre/lustre/llite/vvp_io.c       |    8 -------
 4 files changed, 25 insertions(+), 8 deletions(-)
drivers/staging/lustre/lustre/llite/file.c +2 -0
index bc534db..2fa0107 100644
@@ -846,6 +846,8 @@ ll_file_io_generic(const struct lu_env *env, struct vvp_io_args *args,
 	struct cl_io	 *io;
 	ssize_t	       result;
 
+	ll_io_set_jobid(env, lli);
+
 restart:
 	io = ccc_env_thread_io(env);
 	ll_io_init(io, file, iot == CIT_WRITE);
drivers/staging/lustre/lustre/llite/llite_internal.h +22 -0
index 47e443d..3cac141 100644
@@ -968,6 +968,7 @@ struct vvp_thread_info {
 	struct ra_io_arg     vti_ria;
 	struct kiocb	 vti_kiocb;
 	struct ll_cl_context vti_io_ctx;
+	char		     vti_jobid[JOBSTATS_JOBID_SIZE];
 };
 
 static inline struct vvp_thread_info *vvp_env_info(const struct lu_env *env)
@@ -980,6 +981,27 @@ static inline struct vvp_thread_info *vvp_env_info(const struct lu_env *env)
 	return info;
 }
 
+static inline char *vvp_env_jobid(const struct lu_env *env)
+{
+	return vvp_env_info(env)->vti_jobid;
+}
+
+#define LL_JOBID_NOT_FOUND 0x1
+static inline void ll_io_set_jobid(const struct lu_env *env, struct ll_inode_info *lli)
+{
+	char *jobid = vvp_env_jobid(env);
+
+	if (jobid[0] == '\0') {
+		lustre_get_jobid(jobid);
+
+		if (jobid[0] == '\0')
+			jobid[0] = LL_JOBID_NOT_FOUND;
+	}
+
+	if (jobid[0] != LL_JOBID_NOT_FOUND)
+		memcpy(lli->lli_jobid, jobid, JOBSTATS_JOBID_SIZE);
+}
+
 static inline struct vvp_io_args *vvp_env_args(const struct lu_env *env,
 					       enum vvp_io_subtype type)
 {
drivers/staging/lustre/lustre/llite/llite_mmap.c +1 -0
index caed642..f64f915 100644
@@ -205,6 +205,7 @@ static int ll_page_mkwrite0(struct vm_area_struct *vma, struct page *vmpage,
 	 * while truncate is on-going. */
 	inode = ccc_object_inode(io->ci_obj);
 	lli = ll_i2info(inode);
+	ll_io_set_jobid(env, lli);
 	down_read(&lli->lli_trunc_sem);
 
 	result = cl_io_loop(env, io);
drivers/staging/lustre/lustre/llite/vvp_io.c +0 -8
index 3ff664c..efcce29 100644
@@ -1117,7 +1117,6 @@ int vvp_io_init(const struct lu_env *env, struct cl_object *obj,
 	result = 0;
 	if (io->ci_type == CIT_READ || io->ci_type == CIT_WRITE) {
 		size_t count;
-		struct ll_inode_info *lli = ll_i2info(inode);
 
 		count = io->u.ci_rw.crw_count;
 		/* "If nbyte is 0, read() will return 0 and have no other
@@ -1128,13 +1127,6 @@ int vvp_io_init(const struct lu_env *env, struct cl_object *obj,
 			cio->cui_tot_count = count;
 			cio->cui_tot_nrsegs = 0;
 		}
-		/* for read/write, we store the jobid in the inode, and
-		 * it'll be fetched by osc when building RPC.
-		 *
-		 * it's not accurate if the file is shared by different
-		 * jobs.
-		 */
-		lustre_get_jobid(lli->lli_jobid);
 	} else if (io->ci_type == CIT_SETATTR) {
 		if (!cl_io_is_trunc(io))
 			io->ci_lockreq = CILR_MANDATORY;
Signature · 1 line
1.7.9.5
author
#1 lore raw

PATCH 2/4 staging/lustre/obdclass: read jobid from proc

so that we can get rid of cfs_get_environ() that needs access_process_vm() that is a core mm function and is not available on some architectures.

---
 drivers/staging/lustre/lustre/obdclass/class_obd.c |  119 +++++++++++++++++++-
 1 file changed, 115 insertions(+), 4 deletions(-)
drivers/staging/lustre/lustre/obdclass/class_obd.c +115 -4
index b1024a6..99ce863 100644
@@ -96,8 +96,117 @@ EXPORT_SYMBOL(obd_dirty_transit_pages);
 char obd_jobid_var[JOBSTATS_JOBID_VAR_MAX_LEN + 1] = JOBSTATS_DISABLE;
 EXPORT_SYMBOL(obd_jobid_var);
 
-/* Get jobid of current process by reading the environment variable
- * stored in between the "env_start" & "env_end" of task struct.
+static char *self_environ_file = "/proc/self/environ";
+static int obd_get_environ(const char *key, char *value, int *val_len)
+{
+	struct mm_struct *mm;
+	struct path path;
+	struct file *filp = NULL;
+	int buf_len = PAGE_CACHE_SIZE;
+	int key_len = strlen(key);
+	char *buffer = NULL;
+	loff_t pos = 0;
+	int rc;
+
+	/*
+	 * test mm->mmap_sem to avoid deadlock if this ever gets called from
+	 * mmap code.
+	 */
+	mm = get_task_mm(current);
+	if (!mm)
+		return -EINVAL;
+	if (down_read_trylock(&mm->mmap_sem) == 0) {
+		mmput(mm);
+		return -EDEADLK;
+	}
+	up_read(&mm->mmap_sem);
+	mmput(mm);
+
+	buffer = kmalloc(buf_len, GFP_NOIO);
+	if (!buffer)
+		return -ENOMEM;
+
+	rc = kern_path(self_environ_file, LOOKUP_FOLLOW, &path);
+	if (rc)
+		goto out;
+
+	filp = dentry_open(&path, O_RDONLY, current_cred());
+	if (IS_ERR(filp)) {
+		rc = PTR_ERR(filp);
+		filp = NULL;
+		goto out;
+	}
+
+	/* loop reading... */
+	while (1) {
+		int scan_len, this_len;
+		char *env_start, *env_end;
+		rc = kernel_read(filp, pos, buffer, buf_len);
+		if (rc <= 0)
+			break;
+
+		pos += rc;
+		/* Parse the buffer to find out the specified key/value pair.
+		 * The "key=value" entries are separated by '\0'. */
+		env_start = buffer;
+		scan_len = this_len = rc;
+		while (scan_len) {
+			char *entry;
+			int entry_len;
+
+			env_end = memscan(env_start, '\0', scan_len);
+			LASSERT(env_end >= env_start &&
+				env_end <= env_start + scan_len);
+
+			/* The last entry of this buffer cross the buffer
+			 * boundary, reread it in next cycle. */
+			if (unlikely(env_end - env_start == scan_len)) {
+				/* This entry is too large to fit in buffer */
+				if (unlikely(scan_len == this_len)) {
+					rc = -EINVAL;
+					CWARN("Environment variable '%s' too long: rc = %d\n",
+					      key, rc);
+					goto out;
+				}
+				pos -= scan_len;
+				break;
+			}
+
+			entry = env_start;
+			entry_len = env_end - env_start;
+
+			/* Key length + length of '=' */
+			if (entry_len > key_len + 1 &&
+			    !memcmp(entry, key, key_len)) {
+				entry += key_len + 1;
+				entry_len -= key_len + 1;
+				/* The 'value' buffer passed in is too small.*/
+				if (entry_len >= *val_len)
+					GOTO(out, rc = -EOVERFLOW);
+
+				memcpy(value, entry, entry_len);
+				*val_len = entry_len;
+				rc = 0;
+				goto out;
+			}
+
+			scan_len -= (env_end - env_start + 1);
+			env_start = env_end + 1;
+		}
+	}
+	if (rc >= 0)
+		rc = -ENOENT;
+out:
+	if (filp)
+		fput(filp);
+	if (buffer)
+		kfree(buffer);
+	return rc;
+}
+
+/*
+ * Get jobid of current process by reading the environment variable
+ * from /proc/self/environ.
  *
  * TODO:
  * It's better to cache the jobid for later use if there is any
@@ -126,14 +235,14 @@ int lustre_get_jobid(char *jobid)
 		return 0;
 	}
 
-	rc = cfs_get_environ(obd_jobid_var, jobid, &jobid_len);
+	rc = obd_get_environ(obd_jobid_var, jobid, &jobid_len);
 	if (rc) {
 		if (rc == -EOVERFLOW) {
 			/* For the PBS_JOBID and LOADL_STEP_ID keys (which are
 			 * variable length strings instead of just numbers), it
 			 * might make sense to keep the unique parts for JobID,
 			 * instead of just returning an error.  That means a
-			 * larger temp buffer for cfs_get_environ(), then
+			 * larger temp buffer for obd_get_environ(), then
 			 * truncating the string at some separator to fit into
 			 * the specified jobid_len.  Fix later if needed. */
 			static bool printed;
@@ -149,6 +258,8 @@ int lustre_get_jobid(char *jobid)
 			       "Get jobid for (%s) failed: rc = %d\n",
 			       obd_jobid_var, rc);
 		}
+	} else {
+		CDEBUG(D_INFO, "Got jobid for (%s) value (%s)\n", obd_jobid_var, jobid);
 	}
 	return rc;
 }
Signature · 1 line
1.7.9.5
#2 lore raw

Re: [PATCH 2/4] staging/lustre/obdclass: read jobid from proc

Quoted 15 lines · Peng Tao

so that we can get rid of cfs_get_environ() that needs access_process_vm() that is a core mm function and is not available on some architectures.

---
 drivers/staging/lustre/lustre/obdclass/class_obd.c |  119 +++++++++++++++++++-
 1 file changed, 115 insertions(+), 4 deletions(-)
drivers/staging/lustre/lustre/obdclass/class_obd.c +0 -0
index b1024a6..99ce863 100644

drivers/staging/lustre/lustre/obdclass/class_obd.c EXPORT_SYMBOL(obd_dirty_transit_pages); · line 99

Quoted 6 lines
drivers/staging/lustre/lustre/obdclass/class_obd.c +1 -2
@@ -96,8 +96,117 @@ EXPORT_SYMBOL(obd_dirty_transit_pages);
 char obd_jobid_var[JOBSTATS_JOBID_VAR_MAX_LEN + 1] = JOBSTATS_DISABLE;
 EXPORT_SYMBOL(obd_jobid_var);
 
-/* Get jobid of current process by reading the environment variable
- * stored in between the "env_start" & "env_end" of task struct.
+static char *self_environ_file = "/proc/self/environ";

Heh, no, that's not ok at all.

This is a _huge_ sign that you are doing something wrong in your driver if you need something that isn't exported, or that you have to dig out of proc.

Sorry, I can't take this, please fix the underlying problems that would even think that you need access to the environment from within a kernel driver.

greg k-h

author
#3 lore raw

Re: [PATCH 2/4] staging/lustre/obdclass: read jobid from proc

Quoted 25 lines · Greg Kroah-Hartman
Quoted 15 lines · Peng Tao

so that we can get rid of cfs_get_environ() that needs access_process_vm() that is a core mm function and is not available on some architectures.

---
 drivers/staging/lustre/lustre/obdclass/class_obd.c |  119 +++++++++++++++++++-
 1 file changed, 115 insertions(+), 4 deletions(-)
drivers/staging/lustre/lustre/obdclass/class_obd.c +0 -0
index b1024a6..99ce863 100644

drivers/staging/lustre/lustre/obdclass/class_obd.c EXPORT_SYMBOL(obd_dirty_transit_pages); · line 99

Quoted 6 lines
drivers/staging/lustre/lustre/obdclass/class_obd.c +1 -2
@@ -96,8 +96,117 @@ EXPORT_SYMBOL(obd_dirty_transit_pages);
 char obd_jobid_var[JOBSTATS_JOBID_VAR_MAX_LEN + 1] = JOBSTATS_DISABLE;
 EXPORT_SYMBOL(obd_jobid_var);
 
-/* Get jobid of current process by reading the environment variable
- * stored in between the "env_start" & "env_end" of task struct.
+static char *self_environ_file = "/proc/self/environ";

Heh, no, that's not ok at all.

Quoted 4 lines

This is a _huge_ sign that you are doing something wrong in your driver if you need something that isn't exported, or that you have to dig out of proc.

Lustre has a functionality that lets applications specify a global jobid by setting the same environment variable across the cluster and thus server can trace IO that is issued by different job. Currently it is implemented by having a private version of cfs_access_process_vm because access_process_vm is not exported. The patch intends to change it to reading from proc. If we cannot dig it out from proc either, how can we keep the functionality then? Please advise.

Thanks,
Tao

Quoted 5 lines

Sorry, I can't take this, please fix the underlying problems that would even think that you need access to the environment from within a kernel driver.

greg k-h

#4 lore raw

Re: [PATCH 2/4] staging/lustre/obdclass: read jobid from proc

Quoted 31 lines · Peng Tao
Quoted 25 lines · Greg Kroah-Hartman
Quoted 15 lines · Peng Tao

so that we can get rid of cfs_get_environ() that needs access_process_vm() that is a core mm function and is not available on some architectures.

---
 drivers/staging/lustre/lustre/obdclass/class_obd.c |  119 +++++++++++++++++++-
 1 file changed, 115 insertions(+), 4 deletions(-)
drivers/staging/lustre/lustre/obdclass/class_obd.c +0 -0
index b1024a6..99ce863 100644

drivers/staging/lustre/lustre/obdclass/class_obd.c EXPORT_SYMBOL(obd_dirty_transit_pages); · line 99

Quoted 6 lines
drivers/staging/lustre/lustre/obdclass/class_obd.c +1 -2
@@ -96,8 +96,117 @@ EXPORT_SYMBOL(obd_dirty_transit_pages);
 char obd_jobid_var[JOBSTATS_JOBID_VAR_MAX_LEN + 1] = JOBSTATS_DISABLE;
 EXPORT_SYMBOL(obd_jobid_var);
 
-/* Get jobid of current process by reading the environment variable
- * stored in between the "env_start" & "env_end" of task struct.
+static char *self_environ_file = "/proc/self/environ";

Heh, no, that's not ok at all.

Quoted 4 lines

This is a _huge_ sign that you are doing something wrong in your driver if you need something that isn't exported, or that you have to dig out of proc.

Quoted 7 lines

Lustre has a functionality that lets applications specify a global jobid by setting the same environment variable across the cluster and thus server can trace IO that is issued by different job. Currently it is implemented by having a private version of cfs_access_process_vm because access_process_vm is not exported. The patch intends to change it to reading from proc. If we cannot dig it out from proc either, how can we keep the functionality then? Please advise.

Kernel code should not care about environment variables. Just because you created a crazy requirement, does not mean it is a valid one, nor that I have to be the one to tell you how to implement it.

I'm never going to take code that parses proc files, you all know better than that.

greg k-h

author
#5 lore raw

Re: [PATCH 2/4] staging/lustre/obdclass: read jobid from proc

Quoted 40 lines · Greg Kroah-Hartman
Quoted 31 lines · Peng Tao
Quoted 25 lines · Greg Kroah-Hartman
Quoted 15 lines · Peng Tao

so that we can get rid of cfs_get_environ() that needs access_process_vm() that is a core mm function and is not available on some architectures.

---
 drivers/staging/lustre/lustre/obdclass/class_obd.c |  119 +++++++++++++++++++-
 1 file changed, 115 insertions(+), 4 deletions(-)
drivers/staging/lustre/lustre/obdclass/class_obd.c +0 -0
index b1024a6..99ce863 100644

drivers/staging/lustre/lustre/obdclass/class_obd.c EXPORT_SYMBOL(obd_dirty_transit_pages); · line 99

Quoted 6 lines
drivers/staging/lustre/lustre/obdclass/class_obd.c +1 -2
@@ -96,8 +96,117 @@ EXPORT_SYMBOL(obd_dirty_transit_pages);
 char obd_jobid_var[JOBSTATS_JOBID_VAR_MAX_LEN + 1] = JOBSTATS_DISABLE;
 EXPORT_SYMBOL(obd_jobid_var);
 
-/* Get jobid of current process by reading the environment variable
- * stored in between the "env_start" & "env_end" of task struct.
+static char *self_environ_file = "/proc/self/environ";

Heh, no, that's not ok at all.

Quoted 4 lines

This is a _huge_ sign that you are doing something wrong in your driver if you need something that isn't exported, or that you have to dig out of proc.

Quoted 7 lines

Lustre has a functionality that lets applications specify a global jobid by setting the same environment variable across the cluster and thus server can trace IO that is issued by different job. Currently it is implemented by having a private version of cfs_access_process_vm because access_process_vm is not exported. The patch intends to change it to reading from proc. If we cannot dig it out from proc either, how can we keep the functionality then? Please advise.

Quoted 6 lines

Kernel code should not care about environment variables. Just because you created a crazy requirement, does not mean it is a valid one, nor that I have to be the one to tell you how to implement it.

I'm never going to take code that parses proc files, you all know better than that.

I see. I'll drop the patchset and keep the cfs_access_process_vm code until better option is found.

Thanks,
Tao

#6 lore raw

Re: [PATCH 2/4] staging/lustre/obdclass: read jobid from proc

In reply to Greg Kroah-Hartman

Hello!

Quoted 5 lines · Greg Kroah-Hartman
Quoted 2 lines

- * stored in between the "env_start" & "env_end" of task struct. +static char *self_environ_file = "/proc/self/environ";

Heh, no, that's not ok at all.

Quoted 7 lines

This is a _huge_ sign that you are doing something wrong in your driver if you need something that isn't exported, or that you have to dig out of proc.

Sorry, I can't take this, please fix the underlying problems that would even think that you need access to the environment from within a kernel driver.

I took a stab at this.
This is not a final patch, I know there's still some number of checkpatch warnings and the proc layout is not finalized yet for example.

But before I spend any more time in polishing this, can you please take a look and advise if this direction would be acceptable for you when driven to completion?

Thanks.

Quoted 1 line

From 6a5b58657cc32163738d4a8c210e8683159b582f Mon Sep 17 00:00:00 2001

From: Oleg Drokin <[email protected]>
Date: Tue, 4 Feb 2014 00:32:12 -0500
Subject: [PATCH] staging/lustre: Obtain jobid invormation via upcall

Replace lustre jobid information fetching directly from process env variable with either node-wide jobid obtained via a proc file, or through an upcall that would provide the jobid if more fine-grained operations are necessary.

---
 .../staging/lustre/include/linux/libcfs/curproc.h  |   1 -
 .../staging/lustre/include/linux/libcfs/lucache.h  |   6 +
 .../staging/lustre/lustre/include/lprocfs_status.h |   1 +
 .../lustre/lustre/libcfs/linux/linux-curproc.c     | 152 -----------------
 drivers/staging/lustre/lustre/obdclass/Makefile    |   2 +-
 drivers/staging/lustre/lustre/obdclass/class_obd.c |  67 ++++----
 .../staging/lustre/lustre/obdclass/jobid_cache.c   | 181 +++++++++++++++++++++
 .../lustre/lustre/obdclass/jobid_internal.h        |  10 ++
 .../lustre/lustre/obdclass/linux/linux-module.c    | 114 +++++++++++++
 9 files changed, 345 insertions(+), 189 deletions(-)
 create mode 100644 drivers/staging/lustre/lustre/obdclass/jobid_cache.c
 create mode 100644 drivers/staging/lustre/lustre/obdclass/jobid_internal.h
drivers/staging/lustre/include/linux/libcfs/curproc.h +0 -1
index 507d16b..cf1f26b 100644
@@ -63,7 +63,6 @@ int    cfs_curproc_groups_nr(void);
 /* check if task is running in compat mode.*/
 #define current_pid()		(current->pid)
 #define current_comm()		(current->comm)
-int cfs_get_environ(const char *key, char *value, int *val_len);
 
 typedef __u32 cfs_cap_t;
 
drivers/staging/lustre/include/linux/libcfs/lucache.h +6 -0
index 9668b39..f8361d7 100644
@@ -82,6 +82,11 @@ struct md_identity {
 	struct md_perm	    *mi_perms;
 };
 
+struct jobid_cache_entry {
+	struct upcall_cache_entry *jce_uc_entry;
+	char			  *jce_jobid;
+};
+
 struct upcall_cache_entry {
 	struct list_head	      ue_hash;
 	__u64		   ue_key;
@@ -92,6 +97,7 @@ struct upcall_cache_entry {
 	cfs_time_t	      ue_expire;
 	union {
 		struct md_identity     identity;
+		struct jobid_cache_entry jobid;
 	} u;
 };
 
drivers/staging/lustre/lustre/include/lprocfs_status.h +1 -0
index 428e3e4..3c99dcf 100644
@@ -369,6 +369,7 @@ static inline void s2dhms(struct dhms *ts, time_t secs)
 #define JOBSTATS_JOBID_VAR_MAX_LEN	20
 #define JOBSTATS_DISABLE		"disable"
 #define JOBSTATS_PROCNAME_UID		"procname_uid"
+#define JOBSTATS_NODELOCAL		"nodelocal"
 
 extern int lprocfs_write_frac_helper(const char *buffer, unsigned long count,
 				     int *val, int mult);
drivers/staging/lustre/lustre/libcfs/linux/linux-curproc.c +0 -152

158 lines not on this page · raw message

drivers/staging/lustre/lustre/obdclass/Makefile +1 -1
index 8a0e08c..7949838 100644
@@ -7,7 +7,7 @@ obdclass-y := linux/linux-module.o linux/linux-obdo.o linux/linux-sysctl.o \
 	      local_storage.o statfs_pack.o obdo.o obd_config.o obd_mount.o\
 	      mea.o lu_object.o dt_object.o capa.o cl_object.o   \
 	      cl_page.o cl_lock.o cl_io.o lu_ref.o acl.o idmap.o	   \
-	      lu_ucred.o
+	      lu_ucred.o jobid_cache.o
 
 
 ccflags-y := -I$(src)/../include
drivers/staging/lustre/lustre/obdclass/class_obd.c +32 -35

98 lines not on this page · raw message

drivers/staging/lustre/lustre/obdclass/jobid_cache.c new file +181 -0

181 lines not on this page · raw message

drivers/staging/lustre/lustre/obdclass/jobid_internal.h new file +10 -0
new file mode 100644
index 0000000..56db606
@@ -0,0 +1,10 @@
+#include <linux/libcfs/lucache.h>
+
+extern struct upcall_cache_ops jobid_cache_upcall_cache_ops;
+extern char obd_jobid_node[];
+extern struct upcall_cache *obd_jobid_upcall;
+
+struct jobid_cache_entry *jobid_cache_get(__u64 pid);
+void jobid_cache_put(struct jobid_cache_entry *jobid);
+void jobid_flush_cache(__u64 pid);
+
drivers/staging/lustre/lustre/obdclass/linux/linux-module.c +114 -0

138 lines not on this page · raw message

Signature · 1 line
1.8.5.3
#7 lore raw

Re: [PATCH 2/4] staging/lustre/obdclass: read jobid from proc

Quoted 16 lines · Oleg Drokin

Hello!

Quoted 5 lines · Greg Kroah-Hartman
Quoted 2 lines

- * stored in between the "env_start" & "env_end" of task struct. +static char *self_environ_file = "/proc/self/environ";

Heh, no, that's not ok at all.

Quoted 7 lines

This is a _huge_ sign that you are doing something wrong in your driver if you need something that isn't exported, or that you have to dig out of proc.

Sorry, I can't take this, please fix the underlying problems that would even think that you need access to the environment from within a kernel driver.

Quoted 7 lines

I took a stab at this.
This is not a final patch, I know there's still some number of checkpatch warnings and the proc layout is not finalized yet for example.

But before I spend any more time in polishing this, can you please take a look and advise if this direction would be acceptable for you when driven to completion?

What exactly are you doing here? Calling out to userspace for what information? And how are you going to handle namespaces and containers by doing that? Are you going to block in the kernel for this information?

What are you trying to solve with this code in the first place?

greg k-h

#8 lore raw

Re: [PATCH 2/4] staging/lustre/obdclass: read jobid from proc

Hello!

Quoted 6 lines · Greg Kroah-Hartman

What exactly are you doing here? Calling out to userspace for what information? And how are you going to handle namespaces and containers by doing that? Are you going to block in the kernel for this information?

What are you trying to solve with this code in the first place?

So, as an overview of a feature:

When you have tens of thousand (and even hundreds) of nodes doing IO, it's no longer practical to tell them apart separately or in some network-related groups based on their address server-side
(for purposes like monitoring and load control).
Since such systems are usually managed by some sort of a job/batch scheduler, it's much more natural to organize them into "jobs" as known to the job scheduler instead. This job id information is harvested and added to all RPCs so that server side can do useful things with this information (like aggregate statistics, identify pathologically bad workcases, QoS and so on).

Most of batch schedulers out there let userspace know their own JOBID as an environment variable. So original implementation was just harvesting this info directly from process environment. I certainly can see why this is not really desirable.

So, the patch does away with the environment parsing, instead it adds two new venues of getting this information:

1. In vast majorities of cases entire node is dedicated to a single job, so we just create a /proc variable where you can input job id from job scheduler prologue (and then clear it from an epilogue). Getting jobid in this case does does not dip in userspace anymore. This also does not block anywhere.

2. In some more rare systems with lots of cores they actually seem to be subdividing individual nodes across jobs. Additionally all systems usually have login/interactive nodes. While these sort of interactive nodes do not have jobs scheduled on them, it still might be useful to distinguish between different user sessions happening there. This is where the upcalls come into play. First time a process does IO an upcall would be called that would provide the kernel with jobid identifier (however it might want to obtain it, we don't really care at this point). This would block (with a timeout) for the reply. The reply is then cached and reused for subsequent io from the same process. I did not really think about containers before, but I think it would work out anyway:
I think namespaces and containers still have non-intersecting pid space so we should be fine in this regard. There is a "master container"/namespace of some sort I think, that is able to see entire pid space, and that's where the upcall would be run (do I need to somehow force that?)

Bye,
Oleg

author
#9 lore raw

PATCH 3/4 staging/lustre: remove cfs_get_environ and cfs_access_process_vm

In reply to Peng Tao
---
 .../staging/lustre/include/linux/libcfs/curproc.h  |    1 -
 .../lustre/lustre/libcfs/linux/linux-curproc.c     |  150 --------------------
 2 files changed, 151 deletions(-)
drivers/staging/lustre/include/linux/libcfs/curproc.h +0 -1
index de8e35b..1c49ef4 100644
@@ -64,7 +64,6 @@ int    cfs_curproc_groups_nr(void);
 int current_is_32bit(void);
 #define current_pid()		(current->pid)
 #define current_comm()		(current->comm)
-int cfs_get_environ(const char *key, char *value, int *val_len);
 
 typedef __u32 cfs_cap_t;
 
drivers/staging/lustre/lustre/libcfs/linux/linux-curproc.c +0 -150

156 lines not on this page · raw message

Signature · 1 line
1.7.9.5
author
#10 lore raw

PATCH 4/4 staging/lustre: enable build on MIPS/XTENSA/SUPERH

In reply to Peng Tao

The missing symbol copy_from_user_page() is not needed now.

---
 drivers/staging/lustre/lustre/Kconfig |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/staging/lustre/lustre/Kconfig +1 -1
index 2156a44..e44965b 100644
@@ -1,6 +1,6 @@
 config LUSTRE_FS
 	tristate "Lustre file system client support"
-	depends on INET && m && !MIPS && !XTENSA && !SUPERH
+	depends on INET && m
 	select LNET
 	select CRYPTO
 	select CRYPTO_CRC32
Signature · 1 line
1.7.9.5
#11 lore raw
In reply to Peng Tao
Quoted 3 lines · Peng Tao

We will switch to find jobid from /proc/self/environ and we need an extra memory copy to do it, so let's cache it in lu_env. It is then copied from lu_env to ll_inode_info upon every read/write.

What does this mean?

And you better not be reading proc values from within the kernel...

greg k-h