Slab Defrag / Slab Targeted Reclaim

Index: slub/include/linux/slab.h
===================================================================
@@ -38,10 +38,13 @@ typedef struct kmem_cache kmem_cache_t _
 void __init kmem_cache_init(void);
 int slab_is_available(void);
 
+struct kmem_cache_ops {
+};
+
 struct kmem_cache *kmem_cache_create(const char *, size_t, size_t,
 			unsigned long,
 			void (*)(void *, struct kmem_cache *, unsigned long),
-			void (*)(void *, struct kmem_cache *, unsigned long));
+			const struct kmem_cache_ops *s);
 void kmem_cache_destroy(struct kmem_cache *);
 int kmem_cache_shrink(struct kmem_cache *);
 void *kmem_cache_alloc(struct kmem_cache *, gfp_t);

@@ -59,9 +62,11 @@ int kmem_ptr_validate(struct kmem_cache 
  * f.e. add ____cacheline_aligned_in_smp to the struct declaration
  * then the objects will be properly aligned in SMP configurations.
  */
-#define KMEM_CACHE(__struct, __flags) kmem_cache_create(#__struct,\
-		sizeof(struct __struct), __alignof__(struct __struct),\
-		(__flags), NULL, NULL)
+#define KMEM_CACHE_OPS(__struct, __flags, __ops) \
+	kmem_cache_create(#__struct, sizeof(struct __struct), \
+	__alignof__(struct __struct), (__flags), NULL, (__ops))
+
+#define KMEM_CACHE(__struct, __flags) KMEM_CACHE_OPS(__struct, __flags, NULL)
 
 #ifdef CONFIG_NUMA
 extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);

115 lines not on this page · raw message

Index: slub/include/linux/slub_def.h
===================================================================
@@ -40,6 +40,7 @@ struct kmem_cache {
 	int objects;		/* Number of objects in slab */
 	int refcount;		/* Refcount for slab cache destroy */
 	void (*ctor)(void *, struct kmem_cache *, unsigned long);
+	const struct kmem_cache_ops *ops;
 	int inuse;		/* Offset to metadata */
 	int align;		/* Alignment */
 	const char *name;	/* Name (only for display!) */

Index: slub/mm/slab.c
===================================================================
@@ -2100,7 +2100,7 @@ static int setup_cpu_cache(struct kmem_c
  * @align: The required alignment for the objects.
  * @flags: SLAB flags
  * @ctor: A constructor for the objects.
- * @dtor: A destructor for the objects (not implemented anymore).
+ * @ops: A kmem_cache_ops structure (ignored).
  *
  * Returns a ptr to the cache on success, NULL on failure.
  * Cannot be called within a int, but can be interrupted.

@@ -2126,7 +2126,7 @@ struct kmem_cache *
 kmem_cache_create (const char *name, size_t size, size_t align,
 	unsigned long flags,
 	void (*ctor)(void*, struct kmem_cache *, unsigned long),
-	void (*dtor)(void*, struct kmem_cache *, unsigned long))
+	const struct kmem_cache_ops *ops)
 {
 	size_t left_over, slab_size, ralign;
 	struct kmem_cache *cachep = NULL, *pc;

@@ -2135,7 +2135,7 @@ kmem_cache_create (const char *name, siz
 	 * Sanity checks... these are all serious usage bugs.
 	 */
 	if (!name || in_interrupt() || (size < BYTES_PER_WORD) ||
-	    size > KMALLOC_MAX_SIZE || dtor) {
+	    size > KMALLOC_MAX_SIZE) {
 		printk(KERN_ERR "%s: Early error in slab %s\n", __FUNCTION__,
 				name);
 		BUG();

Index: slub/mm/slob.c
===================================================================
@@ -285,7 +285,7 @@ struct kmem_cache {
 struct kmem_cache *kmem_cache_create(const char *name, size_t size,
 	size_t align, unsigned long flags,
 	void (*ctor)(void*, struct kmem_cache *, unsigned long),
-	void (*dtor)(void*, struct kmem_cache *, unsigned long))
+	const struct kmem_cache_ops *o)
 {
 	struct kmem_cache *c;
 

Index: slub/include/linux/slab.h
===================================================================
@@ -39,6 +39,39 @@ void __init kmem_cache_init(void);
 int slab_is_available(void);
 
 struct kmem_cache_ops {
+	/*
+	 * Called with slab lock held and interrupts disabled.
+	 * No slab operation may be performed.
+	 *
+	 * Return 0 if reference was successfully obtained
+	 * Return 1 if a concurrent kmem_cache_free is waiting to free object
+	 * Return -errcode if it is not possible to free the object.
+	 * 		No reference was obtained.
+	 */
+	int (*get)(struct kmem_cache *, void *);
+
+	/*
+	 * Use to restore the reference count if we abandon the
+	 * attempt to vacate a slab page due to an unmovable
+	 * object.
+	 */
+	void (*put)(struct kmem_cache *, void *);
+
+	/*
+	 * Called with no locks held and interrupts enabled.
+	 * Any operation may be performed in kick_object.
+	 *
+	 * Return 0 for success
+	 * Return -errcode aborts further kicks to objects in the slab
+	 */
+	int (*kick)(struct kmem_cache *, void *);
+
+	/*
+	 * Callback to make sure that all object freeing is complete.
+	 * If the slab destroys objects by RCU then this needs to be
+	 * set to synchronize_rcu().
+	 */
+	void (*sync)(void);
 };
 
 struct kmem_cache *kmem_cache_create(const char *, size_t, size_t,

@@ -53,6 +86,7 @@ void kmem_cache_free(struct kmem_cache *
 unsigned int kmem_cache_size(struct kmem_cache *);
 const char *kmem_cache_name(struct kmem_cache *);
 int kmem_ptr_validate(struct kmem_cache *cachep, const void *ptr);
+int kmem_cache_vacate(struct page *);
 
 /*
  * Please use this macro to create slab caches. Simply specify the

348 lines not on this page · raw message

Index: slub/mm/slab.c
===================================================================
@@ -2516,6 +2516,15 @@ int kmem_cache_shrink(struct kmem_cache 
 }
 EXPORT_SYMBOL(kmem_cache_shrink);
 
+/*
+ * SLAB does not support slab defragmentation
+ */
+int kmem_cache_vacate(struct page *page)
+{
+	return 0;
+}
+EXPORT_SYMBOL(kmem_cache_vacate);
+
 /**
  * kmem_cache_destroy - delete a cache
  * @cachep: the cache to destroy

Index: slub/mm/slob.c
===================================================================
@@ -394,6 +394,15 @@ int kmem_cache_shrink(struct kmem_cache 
 }
 EXPORT_SYMBOL(kmem_cache_shrink);
 
+/*
+ * SLOB does not support slab defragmentation
+ */
+int kmem_cache_vacate(struct page *page)
+{
+	return 0;
+}
+EXPORT_SYMBOL(kmem_cache_vacate);
+
 int kmem_ptr_validate(struct kmem_cache *a, const void *b)
 {
 	return 0;

Index: slub/fs/dcache.c
===================================================================
@@ -2114,18 +2114,91 @@ static void __init dcache_init_early(voi
 		INIT_HLIST_HEAD(&dentry_hashtable[loop]);
 }
 
+/*
+ * The slab is holding off frees. Thus we can safely examine
+ * the object without the danger of it vanishing from under us.
+ */
+static int get_dentry(struct kmem_cache *s, void *private)
+{
+	struct dentry *dentry = private;
+	int result = 0;
+
+	spin_lock(&dcache_lock);
+	/*
+	 * dentry->d_inode is set to NULL when the dentry
+	 * is freed. Use that as an indicator that we should
+	 * not interfere with the freeing process.
+	 */
+	if (dentry->d_inode) {
+		dget_locked(dentry);
+		if (atomic_read(&dentry->d_count) > 2)
+			/*
+			 * Moving of dentries in use not
+			 * implemented yet.
+			 */
+			result = -EINVAL;
+	} else
+		result = 1;
+	spin_unlock(&dcache_lock);
+	return result;
+}
+
+static void put_dentry(struct kmem_cache *s, void *private)
+{
+	struct dentry *dentry = private;
+
+	dput(dentry);
+}
+
+/*
+ * Slab has dropped all the locks. Get rid of the
+ * refcount we obtained earlier and also rid of the
+ * object.
+ */
+static int kick_dentry(struct kmem_cache *s, void *private)
+{
+	struct dentry *dentry = private;
+
+	spin_lock(&dcache_lock);
+	spin_lock(&dentry->d_lock);
+	if (atomic_read(&dentry->d_count) > 1) {
+		/*
+		 * Reference count was increased.
+		 * We need to abandon the freeing of
+		 * objects.
+		 */
+		spin_unlock(&dentry->d_lock);
+		spin_unlock(&dcache_lock);
+		dput(dentry);
+		return -EBUSY;
+	}
+
+	/* Remove from LRU */
+	if (!list_empty(&dentry->d_lru)) {
+		dentry_stat.nr_unused--;
+		list_del_init(&dentry->d_lru);
+	}
+	/* Drop the entry */
+	prune_one_dentry(dentry, 1);
+	spin_unlock(&dcache_lock);
+	return 0;
+}
+
+static struct kmem_cache_ops dentry_kmem_cache_ops = {
+	.get = get_dentry,
+	.put = put_dentry,
+	.kick = kick_dentry,
+	.sync = synchronize_rcu
+};
+
 static void __init dcache_init(unsigned long mempages)
 {
 	int loop;
 
-	/* 
-	 * A constructor could be added for stable state like the lists,
-	 * but it is probably not worth it because of the cache nature
-	 * of the dcache. 
-	 */
-	dentry_cache = KMEM_CACHE(dentry,
-		SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_MEM_SPREAD);
-	
+	dentry_cache = KMEM_CACHE_OPS(dentry,
+		SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_MEM_SPREAD,
+		&dentry_kmem_cache_ops);
+
 	register_shrinker(&dcache_shrinker);
 
 	/* Hash may have been set up in dcache_init_early */