rcu/tree: use more permissive parameters when attaching a head

index 6172e6296dd7..24f620a06219 100644
@@ -3148,13 +3148,10 @@ static inline struct rcu_head *attach_rcu_head_to_object(void *obj)
 {
 	unsigned long *ptr;
 
+	/* Try hard to get the memory. */
 	ptr = kmalloc(sizeof(unsigned long *) +
-			sizeof(struct rcu_head), GFP_NOWAIT | __GFP_NOWARN);
-
-	if (!ptr)
-		ptr = kmalloc(sizeof(unsigned long *) +
-				sizeof(struct rcu_head), GFP_ATOMIC | __GFP_NOWARN);
-
+		sizeof(struct rcu_head), GFP_KERNEL |
+			__GFP_ATOMIC | __GFP_HIGH | __GFP_RETRY_MAYFAIL);
 	if (!ptr)
 		return NULL;
 

@@ -3222,9 +3219,20 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
 	if (!success) {
 		/* Is headless object? */
 		if (head == NULL) {
+			/* Drop the lock. */
+			if (krcp->initialized)
+				spin_unlock(&krcp->lock);
+			local_irq_restore(flags);
+
 			head = attach_rcu_head_to_object(ptr);
 			if (head == NULL)
-				goto unlock_return;
+				goto inline_return;
+
+			/* Take it back. */
+			local_irq_save(flags);
+			krcp = this_cpu_ptr(&krc);
+			if (krcp->initialized)
+				spin_lock(&krcp->lock);
 
 			/*
 			 * Tag the headless object. Such objects have a back-pointer

@@ -3263,6 +3271,7 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
 		spin_unlock(&krcp->lock);
 	local_irq_restore(flags);
 
+inline_return:
 	/*
 	 * High memory pressure, so inline kvfree() after
 	 * synchronize_rcu(). We can do it from might_sleep()

index 24f620a06219..5e26145e9ead 100644
@@ -3159,6 +3159,27 @@ static inline struct rcu_head *attach_rcu_head_to_object(void *obj)
 	return ((struct rcu_head *) ++ptr);
 }
 
+static inline struct kfree_rcu_cpu *
+krc_this_cpu_lock(unsigned long *flags)
+{
+	struct kfree_rcu_cpu *krcp;
+
+	local_irq_save(*flags);	// For safely calling this_cpu_ptr().
+	krcp = this_cpu_ptr(&krc);
+	if (likely(krcp->initialized))
+		spin_lock(&krcp->lock);
+
+	return krcp;
+}
+
+static inline void
+krc_this_cpu_unlock(struct kfree_rcu_cpu *krcp, unsigned long flags)
+{
+	if (likely(krcp->initialized))
+		spin_unlock(&krcp->lock);
+	local_irq_restore(flags);
+}
+
 /*
  * Queue a request for lazy invocation of appropriate free routine after a
  * grace period. Please note there are three paths are maintained, two are the

@@ -3195,10 +3216,7 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
 		ptr = (unsigned long *) func;
 	}
 
-	local_irq_save(flags);	// For safely calling this_cpu_ptr().
-	krcp = this_cpu_ptr(&krc);
-	if (krcp->initialized)
-		spin_lock(&krcp->lock);
+	krcp = krc_this_cpu_lock(&flags);
 
 	// Queue the object but don't yet schedule the batch.
 	if (debug_rcu_head_queue(ptr)) {

@@ -3220,19 +3238,14 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
 		/* Is headless object? */
 		if (head == NULL) {
 			/* Drop the lock. */
-			if (krcp->initialized)
-				spin_unlock(&krcp->lock);
-			local_irq_restore(flags);
+			krc_this_cpu_unlock(krcp, flags);
 
 			head = attach_rcu_head_to_object(ptr);
 			if (head == NULL)
 				goto inline_return;
 
 			/* Take it back. */
-			local_irq_save(flags);
-			krcp = this_cpu_ptr(&krc);
-			if (krcp->initialized)
-				spin_lock(&krcp->lock);
+			krcp = krc_this_cpu_lock(&flags);
 
 			/*
 			 * Tag the headless object. Such objects have a back-pointer

@@ -3267,9 +3280,7 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
 	}
 
 unlock_return:
-	if (krcp->initialized)
-		spin_unlock(&krcp->lock);
-	local_irq_restore(flags);
+	krc_this_cpu_unlock(krcp, flags);
 
 inline_return:
 	/*

index 8bbefcaddfe8..1734ba7fc400 100644
@@ -15,6 +15,7 @@
 #include <linux/delay.h>
 #include <linux/rwsem.h>
 #include <linux/mm.h>
+#include <linux/rcupdate.h>
 
 #define __param(type, name, init, msg)		\
 	static type name = init;				\

@@ -43,6 +44,8 @@ __param(int, run_test_mask, INT_MAX,
 		"\t\tid: 32,  name: random_size_align_alloc_test\n"
 		"\t\tid: 64,  name: align_shift_alloc_test\n"
 		"\t\tid: 128, name: pcpu_alloc_test\n"
+		"\t\tid: 256, name: kvfree_rcu_with_head_test\n"
+		"\t\tid: 512, name: kvfree_rcu_head_less_test\n"
 		/* Add a new test case description here. */
 );
 

@@ -328,6 +331,49 @@ pcpu_alloc_test(void)
 	return rv;
 }
 
+struct test_kvfree_rcu {
+	struct rcu_head rcu;
+	unsigned char array[100];
+};
+
+static int
+kvfree_rcu_with_head_test(void)
+{
+	struct test_kvfree_rcu *p;
+	int i;
+
+	for (i = 0; i < test_loop_count; i++) {
+		p = vmalloc(1 * PAGE_SIZE);
+
+		if (!p)
+			return -1;
+
+		p->array[0] = 'a';
+		kvfree_rcu(p, rcu);
+	}
+
+	return 0;
+}
+
+static int
+kvfree_rcu_head_less_test(void)
+{
+	struct test_kvfree_rcu *p;
+	int i;
+
+	for (i = 0; i < test_loop_count; i++) {
+		p = vmalloc(1 * PAGE_SIZE);
+
+		if (!p)
+			return -1;
+
+		p->array[0] = 'a';
+		kvfree_rcu(p);
+	}
+
+	return 0;
+}
+
 struct test_case_desc {
 	const char *test_name;
 	int (*test_func)(void);

@@ -342,6 +388,8 @@ static struct test_case_desc test_case_array[] = {
 	{ "random_size_align_alloc_test", random_size_align_alloc_test },
 	{ "align_shift_alloc_test", align_shift_alloc_test },
 	{ "pcpu_alloc_test", pcpu_alloc_test },
+	{ "kvfree_rcu_with_head_test", kvfree_rcu_with_head_test },
+	{ "kvfree_rcu_head_less_test", kvfree_rcu_head_less_test },
 	/* Add a new test case here. */
 };