Directed kmem charging
Subsystems (9)
cover Directed kmem charging
This patchset introduces memcg variant memory allocation functions. The caller can explicitly pass the memcg to charge for kmem allocations. Currently the kernel, for __GFP_ACCOUNT memory allocation requests, extract the memcg of the current task to charge for the kmem allocation. This patch series introduces kmem allocation functions where the caller can pass the pointer to the remote memcg. The remote memcg will be charged for the allocation instead of the memcg of the caller. However the caller must have a reference to the remote memcg.
Shakeel Butt (3):
mm: memcg: plumbing memcg for kmem cache allocations
mm: memcg: plumbing memcg for kmalloc allocations
fs: fsnotify: account fsnotify metadata to kmemcg
fs/notify/dnotify/dnotify.c | 5 +- fs/notify/fanotify/fanotify.c | 12 ++- fs/notify/fanotify/fanotify.h | 3 +- fs/notify/fanotify/fanotify_user.c | 7 +- fs/notify/group.c | 4 + fs/notify/inotify/inotify_fsnotify.c | 2 +- fs/notify/inotify/inotify_user.c | 5 +- fs/notify/mark.c | 6 +- include/linux/fsnotify_backend.h | 12 ++- include/linux/memcontrol.h | 13 ++- include/linux/slab.h | 86 +++++++++++++++- mm/memcontrol.c | 29 ++++-- mm/page_alloc.c | 2 +- mm/slab.c | 107 ++++++++++++++++---- mm/slab.h | 6 +- mm/slab_common.c | 41 +++++++- mm/slub.c | 140 ++++++++++++++++++++++----- 17 files changed, 402 insertions(+), 78 deletions(-)
Signature · 7 lines
2.16.1.291.g4437f3f132-goog -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH 1/3 mm: memcg: plumbing memcg for kmem cache allocations
Introducing the memcg variant for kmem cache allocation functions. Currently the kernel switches the root kmem cache with the memcg specific kmem cache for __GFP_ACCOUNT allocations to charge those allocations to the memcg. However, the memcg to charge is extracted from the current task_struct. This patch introduces the variant of kmem cache allocation functions where the memcg can be provided explicitly by the caller instead of deducing the memcg from the current task.
These functions are useful for use-cases where the allocations should be charged to the memcg different from the memcg of the caller. One such concrete use-case is the allocations for fsnotify event objects where the objects should be charged to the listener instead of the producer.
One requirement to call these functions is that the caller must have the reference to the memcg.
--- include/linux/memcontrol.h | 3 +- include/linux/slab.h | 41 ++++++++++++++++++++ mm/memcontrol.c | 18 +++++++-- mm/slab.c | 78 +++++++++++++++++++++++++++++++++----- mm/slab.h | 6 +-- mm/slub.c | 77 ++++++++++++++++++++++++++++++------- 6 files changed, 192 insertions(+), 31 deletions(-)
include/linux/memcontrol.h +2 -1
@@ -1174,7 +1174,8 @@ static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg) }#endif -struct kmem_cache *memcg_kmem_get_cache (struct kmem_cache *cachep); +struct kmem_cache *memcg_kmem_get_cache (struct kmem_cache *cachep, +struct mem_cgroup *memcg);void memcg_kmem_put_cache (struct kmem_cache *cachep);int memcg_kmem_charge_memcg (struct page *page,gfp_t gfp,int order,struct mem_cgroup *memcg);
include/linux/slab.h +41 -0
86 lines not on this page · raw message
mm/memcontrol.c +15 -3
@@ -701,6 +701,15 @@ static struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)return memcg; } +static struct mem_cgroup *get_mem_cgroup (struct mem_cgroup *memcg) +{ +rcu_read_lock (); +if (!css_tryget_online (&memcg->css)) + memcg =NULL ; +rcu_read_unlock (); +return memcg; +} +/** * mem_cgroup_iter - iterate over memory cgroup hierarchy * @root: hierarchy root
@@ -2246,9 +2255,9 @@ static inline bool memcg_kmem_bypass(void)* done with it, memcg_kmem_put_cache() must be called to release the * reference. */ -struct kmem_cache *memcg_kmem_get_cache (struct kmem_cache *cachep) +struct kmem_cache *memcg_kmem_get_cache (struct kmem_cache *cachep, +struct mem_cgroup *memcg) { -struct mem_cgroup *memcg;struct kmem_cache *memcg_cachep;int kmemcg_id;
@@ -2260,7 +2269,10 @@ struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep)if (current->memcg_kmem_skip_account)return cachep; - memcg =get_mem_cgroup_from_mm (current->mm); +if (memcg) + memcg =get_mem_cgroup (memcg); +if (!memcg) + memcg =get_mem_cgroup_from_mm (current->mm); kmemcg_id =READ_ONCE (memcg->kmemcg_id);if (kmemcg_id <0 ) goto out;
mm/slab.c +68 -10
161 lines not on this page · raw message
mm/slab.h +3 -3
@@ -410,7 +410,7 @@ static inline size_t slab_ksize(const struct kmem_cache *s) }static inline struct kmem_cache *slab_pre_alloc_hook (struct kmem_cache *s, -gfp_t flags) +gfp_t flags,struct mem_cgroup *memcg) { flags &= gfp_allowed_mask;
@@ -423,8 +423,8 @@ static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,return NULL ;if (memcg_kmem_enabled () && - ((flags & __GFP_ACCOUNT) || (s->flags &SLAB_ACCOUNT ))) -return memcg_kmem_get_cache (s); + ((flags & __GFP_ACCOUNT) || (s->flags &SLAB_ACCOUNT ) || memcg)) +return memcg_kmem_get_cache (s, memcg);return s; }
mm/slub.c +63 -14
166 lines not on this page · raw message
Signature · 7 lines
2.16.1.291.g4437f3f132-goog -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH 2/3 mm: memcg: plumbing memcg for kmalloc allocations
In reply to Shakeel ButtIntroducing the memcg variant for kmalloc allocation functions. The kmalloc allocations are underlying served using the kmem caches unless the size of the allocation request is larger than KMALLOC_MAX_CACHE_SIZE, in which case, the kmem caches are bypassed and the request is routed directly to page allocator. So, for __GFP_ACCOUNT kmalloc allocations, the memcg of current task is charged. This patch introduces memcg variant of kmalloc functions to allow callers to provide memcg for charging.
--- include/linux/memcontrol.h | 3 +- include/linux/slab.h | 45 +++++++++++++++++++++++--- mm/memcontrol.c | 9 ++++-- mm/page_alloc.c | 2 +- mm/slab.c | 31 +++++++++++++----- mm/slab_common.c | 41 +++++++++++++++++++++++- mm/slub.c | 65 +++++++++++++++++++++++++++++++------- 7 files changed, 166 insertions(+), 30 deletions(-)
include/linux/memcontrol.h +2 -1
@@ -1179,7 +1179,8 @@ struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep,void memcg_kmem_put_cache (struct kmem_cache *cachep);int memcg_kmem_charge_memcg (struct page *page,gfp_t gfp,int order,struct mem_cgroup *memcg); -int memcg_kmem_charge (struct page *page,gfp_t gfp,int order); +int memcg_kmem_charge (struct page *page,gfp_t gfp,int order, +struct mem_cgroup *memcg);void memcg_kmem_uncharge (struct page *page,int order);#if defined(CONFIG_MEMCG ) && !defined(CONFIG_SLOB )
include/linux/slab.h +40 -5
102 lines not on this page · raw message
mm/memcontrol.c +6 -3
@@ -2348,15 +2348,18 @@ int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,* * Returns 0 on success, an error code on failure. */ -int memcg_kmem_charge (struct page *page,gfp_t gfp,int order) +int memcg_kmem_charge (struct page *page,gfp_t gfp,int order, +struct mem_cgroup *memcg) { -struct mem_cgroup *memcg;int ret =0 ;if (memcg_kmem_bypass ())return 0 ; - memcg =get_mem_cgroup_from_mm (current->mm); +if (memcg) + memcg =get_mem_cgroup (memcg); +if (!memcg) + memcg =get_mem_cgroup_from_mm (current->mm);if (!mem_cgroup_is_root (memcg)) { ret =memcg_kmem_charge_memcg (page, gfp, order, memcg);if (!ret)
mm/page_alloc.c +1 -1
@@ -4261,7 +4261,7 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid, out:if (memcg_kmem_enabled () && (gfp_mask & __GFP_ACCOUNT) && page && -unlikely (memcg_kmem_charge (page, gfp_mask, order) !=0 )) { +unlikely (memcg_kmem_charge (page, gfp_mask, order,NULL ) !=0 )) {__free_pages (page, order); page =NULL ; }
mm/slab.c +23 -8
@@ -3715,7 +3715,8 @@ EXPORT_SYMBOL(kmem_cache_alloc_node_memcg_trace);#endif static __always_inline void * -__do_kmalloc_node (size_t size,gfp_t flags,int node,unsigned long caller) +__do_kmalloc_node (size_t size,gfp_t flags,int node,struct mem_cgroup *memcg, +unsigned long caller) {struct kmem_cache *cachep;void *ret;
@@ -3723,7 +3724,8 @@ __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller) cachep =kmalloc_slab (size, flags);if (unlikely (ZERO_OR_NULL_PTR (cachep)))return cachep; - ret =kmem_cache_alloc_node_trace (cachep, flags, node, size); + ret =kmem_cache_alloc_node_memcg_trace (cachep, flags, node, size, + memcg);kasan_kmalloc (cachep, ret, size, flags);return ret;
@@ -3731,14 +3733,21 @@ __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)void *__kmalloc_node (size_t size,gfp_t flags,int node) { -return __do_kmalloc_node (size, flags, node, _RET_IP_); +return __do_kmalloc_node (size, flags, node,NULL , _RET_IP_); }EXPORT_SYMBOL (__kmalloc_node); +void *__kmalloc_node_memcg (size_t size,gfp_t flags,int node, +struct mem_cgroup *memcg) +{ +return __do_kmalloc_node (size, flags, node, memcg, _RET_IP_); +} +EXPORT_SYMBOL (__kmalloc_node_memcg); +void *__kmalloc_node_track_caller (size_t size,gfp_t flags,int node,unsigned long caller) { -return __do_kmalloc_node (size, flags, node, caller); +return __do_kmalloc_node (size, flags, node,NULL , caller); }EXPORT_SYMBOL (__kmalloc_node_track_caller);#endif /* CONFIG_NUMA */
@@ -3750,7 +3759,7 @@ EXPORT_SYMBOL(__kmalloc_node_track_caller);* @caller: function caller for debug tracking of the caller */ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags, -unsigned long caller) +struct mem_cgroup *memcg,unsigned long caller) {struct kmem_cache *cachep;void *ret;
@@ -3758,7 +3767,7 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags, cachep =kmalloc_slab (size, flags);if (unlikely (ZERO_OR_NULL_PTR (cachep)))return cachep; - ret =slab_alloc (cachep, flags,NULL , caller); + ret =slab_alloc (cachep, flags, memcg, caller);kasan_kmalloc (cachep, ret, size, flags); trace_kmalloc(caller, ret,
@@ -3769,13 +3778,19 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,void *__kmalloc (size_t size,gfp_t flags) { -return __do_kmalloc (size, flags, _RET_IP_); +return __do_kmalloc (size, flags,NULL , _RET_IP_); }EXPORT_SYMBOL (__kmalloc); +void *__kmalloc_memcg (size_t size,gfp_t flags,struct mem_cgroup *memcg) +{ +return __do_kmalloc (size, flags, memcg, _RET_IP_); +} +EXPORT_SYMBOL (__kmalloc_memcg); +void *__kmalloc_track_caller (size_t size,gfp_t flags,unsigned long caller) { -return __do_kmalloc (size, flags, caller); +return __do_kmalloc (size, flags,NULL , caller); }EXPORT_SYMBOL (__kmalloc_track_caller);
mm/slab_common.c +40 -1
66 lines not on this page · raw message
mm/slub.c +54 -11
126 lines not on this page · raw message
Signature · 7 lines
2.16.1.291.g4437f3f132-goog -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
kbuild test robot
build botRe: [PATCH 2/3] mm: memcg: plumbing memcg for kmalloc allocations
Thank you for the patch! Yet something to improve:
Hi Shakeel,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on mmotm/master]
[also build test ERROR on v4.16-rc2 next-20180220]
[cannot apply to linus/master]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]
url: https://github.com/0day-ci/linux/commits/Shakeel-Butt/Directed-kmem-charging/20180221-071026 base: git://git.cmpxchg.org/linux-mmotm.git master config: i386-tinyconfig (attached as .config) compiler: gcc-7 (Debian 7.3.0-1) 7.3.0 reproduce: # save the attached .config to linux build tree make ARCH=i386
All errors (new ones prefixed by >>):
arch/x86/events/core.o: In function `allocate_fake_cpuc':
>> core.c:(.text+0x52b): undefined reference to `__kmalloc_memcg' arch/x86/events/core.o: In function `merge_attr': >> core.c:(.init.text+0x2c): undefined reference to `__kmalloc_memcg' arch/x86/events/intel/core.o: In function `intel_pmu_cpu_prepare': core.c:(.text+0x1674): undefined reference to `__kmalloc_memcg' arch/x86/events/intel/pt.o: In function `pt_init': >> pt.c:(.init.text+0x125): undefined reference to `__kmalloc_memcg'
pt.c:(.init.text+0x13c): undefined reference to __kmalloc_memcg' arch/x86/kernel/e820.o:e820.c:(.init.text+0xa5b): more undefined references to __kmalloc_memcg' follow
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation
kbuild test robot
build botRe: [PATCH 2/3] mm: memcg: plumbing memcg for kmalloc allocations
In reply to Shakeel ButtThank you for the patch! Yet something to improve:
Hi Shakeel,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on mmotm/master]
[also build test ERROR on v4.16-rc2 next-20180220]
[cannot apply to linus/master]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]
url: https://github.com/0day-ci/linux/commits/Shakeel-Butt/Directed-kmem-charging/20180221-071026 base: git://git.cmpxchg.org/linux-mmotm.git master config: i386-randconfig-n0-201807 (attached as .config) compiler: gcc-7 (Debian 7.3.0-1) 7.3.0 reproduce: # save the attached .config to linux build tree make ARCH=i386
All errors (new ones prefixed by >>):
init/initramfs.o: In function `kmalloc_memcg':
>> include/linux/slab.h:588: undefined reference to `__kmalloc_memcg' >> include/linux/slab.h:588: undefined reference to `__kmalloc_memcg' arch/x86/events/core.o: In function `kmalloc_memcg': >> include/linux/slab.h:588: undefined reference to `__kmalloc_memcg' >> include/linux/slab.h:588: undefined reference to `__kmalloc_memcg' arch/x86/kernel/ksysfs.o: In function `kmalloc_memcg': >> include/linux/slab.h:588: undefined reference to `__kmalloc_memcg'
arch/x86/kernel/e820.o:include/linux/slab.h:588: more undefined references to `__kmalloc_memcg' follow
vim +588 include/linux/slab.h
518
519 /**
520 * kmalloc - allocate memory
521 * @size: how many bytes of memory are required.
522 * @flags: the type of memory to allocate.
523 *
524 * kmalloc is the normal method of allocating memory
525 * for objects smaller than page size in the kernel.
526 *
527 * The @flags argument may be one of:
528 *
529 * %GFP_USER - Allocate memory on behalf of user. May sleep.
530 *
531 * %GFP_KERNEL - Allocate normal kernel ram. May sleep.
532 *
533 * %GFP_ATOMIC - Allocation will not sleep. May use emergency pools.
534 * For example, use this inside interrupt handlers.
535 *
536 * %GFP_HIGHUSER - Allocate pages from high memory.
537 *
538 * %GFP_NOIO - Do not do any I/O at all while trying to get memory.
539 *
540 * %GFP_NOFS - Do not make any fs calls while trying to get memory.
541 *
542 * %GFP_NOWAIT - Allocation will not sleep.
543 *
544 * %__GFP_THISNODE - Allocate node-local memory only.
545 *
546 * %GFP_DMA - Allocation suitable for DMA.
547 * Should only be used for kmalloc() caches. Otherwise, use a
548 * slab created with SLAB_DMA.
549 *
550 * Also it is possible to set different flags by OR'ing
551 * in one or more of the following additional @flags:
552 *
553 * %__GFP_HIGH - This allocation has high priority and may use emergency pools.
554 *
555 * %__GFP_NOFAIL - Indicate that this allocation is in no way allowed to fail
556 * (think twice before using).
557 *
558 * %__GFP_NORETRY - If memory is not immediately available,
559 * then give up at once.
560 *
561 * %__GFP_NOWARN - If allocation fails, don't issue any warnings.
562 *
563 * %__GFP_RETRY_MAYFAIL - Try really hard to succeed the allocation but fail
564 * eventually.
565 *
566 * There are other flags available as well, but these are not intended
567 * for general use, and so are not documented here. For a full list of
568 * potential flags, always refer to linux/gfp.h.
569 */
570 static __always_inline void *
571 kmalloc_memcg(size_t size, gfp_t flags, struct mem_cgroup *memcg)
572 {
573 if (__builtin_constant_p(size)) {
574 if (size > KMALLOC_MAX_CACHE_SIZE)
575 return kmalloc_large_memcg(size, flags, memcg);
576 #ifndef CONFIG_SLOB
577 if (!(flags & GFP_DMA)) {
578 int index = kmalloc_index(size);
579
580 if (!index)
581 return ZERO_SIZE_PTR;
582
583 return kmem_cache_alloc_memcg_trace(
584 kmalloc_caches[index], flags, size, memcg);
585 }
586 #endif
587 }
> 588 return __kmalloc_memcg(size, flags, memcg);
589 }
590
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation
PATCH 3/3 fs: fsnotify: account fsnotify metadata to kmemcg
In reply to Shakeel ButtA lot of memory can be consumed by the events generated for the huge or unlimited queues if there is either no or slow listener. This can cause system level memory pressure or OOMs. So, it's better to account the fsnotify kmem caches to the memcg of the listener.
There are seven fsnotify kmem caches and among them allocations from dnotify_struct_cache, dnotify_mark_cache, fanotify_mark_cache and inotify_inode_mark_cachep happens in the context of syscall from the listener. So, SLAB_ACCOUNT is enough for these caches.
The objects from fsnotify_mark_connector_cachep are not accounted as they are small compared to the notification mark or events and it is unclear whom to account connector to since it is shared by all events attached to the inode.
The allocations from the event caches happen in the context of the event producer. For such caches we will need to remote charge the allocations to the listener's memcg. Thus we save the memcg reference in the fsnotify_group structure of the listener.
This patch has also moved the members of fsnotify_group to keep the size same, at least for 64 bit build, even with additional member by filling the holes.
---
Changelog since v1:
- no more charging fsnotify_mark_connector objects
fs/notify/dnotify/dnotify.c | 5 +++-- fs/notify/fanotify/fanotify.c | 12 +++++++----- fs/notify/fanotify/fanotify.h | 3 ++- fs/notify/fanotify/fanotify_user.c | 7 +++++-- fs/notify/group.c | 4 ++++ fs/notify/inotify/inotify_fsnotify.c | 2 +- fs/notify/inotify/inotify_user.c | 5 ++++- fs/notify/mark.c | 6 ++++-- include/linux/fsnotify_backend.h | 12 ++++++++---- include/linux/memcontrol.h | 7 +++++++ mm/memcontrol.c | 2 +- 11 files changed, 46 insertions(+), 19 deletions(-)
fs/notify/dnotify/dnotify.c +3 -2
@@ -384,8 +384,9 @@ int fcntl_dirnotify(int fd, struct file *filp, unsigned long arg)static int __init dnotify_init (void ) { - dnotify_struct_cache =KMEM_CACHE (dnotify_struct,SLAB_PANIC ); - dnotify_mark_cache =KMEM_CACHE (dnotify_mark,SLAB_PANIC ); + dnotify_struct_cache =KMEM_CACHE (dnotify_struct, +SLAB_PANIC |SLAB_ACCOUNT ); + dnotify_mark_cache =KMEM_CACHE (dnotify_mark,SLAB_PANIC |SLAB_ACCOUNT ); dnotify_group =fsnotify_alloc_group (&dnotify_fsnotify_ops);if (IS_ERR (dnotify_group))
fs/notify/fanotify/fanotify.c +7 -5
@@ -140,22 +140,24 @@ static bool fanotify_should_send_event(struct fsnotify_mark *inode_mark, }struct fanotify_event_info *fanotify_alloc_event (struct inode *inode,u32 mask, -const struct path *path) +const struct path *path, +struct mem_cgroup *memcg) {struct fanotify_event_info *event;if (fanotify_is_perm_event (mask)) {struct fanotify_perm_event_info *pevent; - pevent =kmem_cache_alloc (fanotify_perm_event_cachep, -GFP_KERNEL ); + pevent =kmem_cache_alloc_memcg (fanotify_perm_event_cachep, +GFP_KERNEL , memcg);if (!pevent)return NULL ; event = &pevent->fae; pevent->response =0 ; goto init; } - event =kmem_cache_alloc (fanotify_event_cachep,GFP_KERNEL ); + event =kmem_cache_alloc_memcg (fanotify_event_cachep,GFP_KERNEL , + memcg);if (!event)return NULL ; init:__maybe_unused
@@ -210,7 +212,7 @@ static int fanotify_handle_event(struct fsnotify_group *group,return 0 ; } - event =fanotify_alloc_event (inode, mask, data); + event =fanotify_alloc_event (inode, mask, data, group->memcg); ret = -ENOMEM ;if (unlikely (!event)) goto finish;
fs/notify/fanotify/fanotify.h +2 -1
@@ -53,4 +53,5 @@ static inline struct fanotify_event_info *FANOTIFY_E(struct fsnotify_event *fse) }struct fanotify_event_info *fanotify_alloc_event (struct inode *inode,u32 mask, -const struct path *path); +const struct path *path, +struct mem_cgroup *memcg);
fs/notify/fanotify/fanotify_user.c +5 -2
@@ -16,6 +16,7 @@#include <linux/uaccess.h> #include <linux/compat.h> #include <linux/sched/signal.h> +#include <linux/memcontrol.h> #include <asm/ioctls.h>
@@ -756,8 +757,9 @@ SYSCALL_DEFINE2(fanotify_init, unsigned int, flags, unsigned int, event_f_flags) group->fanotify_data.user = user;atomic_inc (&user->fanotify_listeners); + group->memcg =get_mem_cgroup_from_mm (current->mm); - oevent =fanotify_alloc_event (NULL ,FS_Q_OVERFLOW ,NULL ); + oevent =fanotify_alloc_event (NULL ,FS_Q_OVERFLOW ,NULL , group->memcg);if (unlikely (!oevent)) { fd = -ENOMEM ; goto out_destroy_group;
@@ -951,7 +953,8 @@ COMPAT_SYSCALL_DEFINE6(fanotify_mark,*/ static int __init fanotify_user_setup (void ) { - fanotify_mark_cache =KMEM_CACHE (fsnotify_mark,SLAB_PANIC ); + fanotify_mark_cache =KMEM_CACHE (fsnotify_mark, +SLAB_PANIC |SLAB_ACCOUNT ); fanotify_event_cachep =KMEM_CACHE (fanotify_event_info,SLAB_PANIC );if (IS_ENABLED (CONFIG_FANOTIFY_ACCESS_PERMISSIONS )) { fanotify_perm_event_cachep =
fs/notify/group.c +4 -0
@@ -22,6 +22,7 @@#include <linux/srcu.h> #include <linux/rculist.h> #include <linux/wait.h> +#include <linux/memcontrol.h> #include <linux/fsnotify_backend.h> #include "fsnotify.h"
@@ -36,6 +37,9 @@ static void fsnotify_final_destroy_group(struct fsnotify_group *group)if (group->ops->free_group_priv) group->ops->free_group_priv (group); +if (group->memcg) +mem_cgroup_put (group->memcg); +kfree (group); }
fs/notify/inotify/inotify_fsnotify.c +1 -1
@@ -98,7 +98,7 @@ int inotify_handle_event(struct fsnotify_group *group, i_mark = container_of(inode_mark, struct inotify_inode_mark, fsn_mark); - event =kmalloc (alloc_len,GFP_KERNEL ); + event =kmalloc_memcg (alloc_len,GFP_KERNEL , group->memcg);if (unlikely (!event))return -ENOMEM ;
fs/notify/inotify/inotify_user.c +4 -1
@@ -38,6 +38,7 @@#include <linux/uaccess.h> #include <linux/poll.h> #include <linux/wait.h> +#include <linux/memcontrol.h> #include "inotify.h" #include "../fdinfo.h"
@@ -618,6 +619,7 @@ static struct fsnotify_group *inotify_new_group(unsigned int max_events) oevent->name_len =0 ; group->max_events = max_events; + group->memcg =get_mem_cgroup_from_mm (current->mm);spin_lock_init (&group->inotify_data.idr_lock);idr_init (&group->inotify_data.idr);
@@ -785,7 +787,8 @@ static int __init inotify_user_setup(void)BUG_ON (hweight32 (ALL_INOTIFY_BITS ) !=21 ); - inotify_inode_mark_cachep =KMEM_CACHE (inotify_inode_mark,SLAB_PANIC ); + inotify_inode_mark_cachep =KMEM_CACHE (inotify_inode_mark, +SLAB_PANIC |SLAB_ACCOUNT ); inotify_max_queued_events =16384 ; init_user_ns.ucount_max[UCOUNT_INOTIFY_INSTANCES ] =128 ;
fs/notify/mark.c +4 -2
@@ -432,7 +432,8 @@ int fsnotify_compare_groups(struct fsnotify_group *a, struct fsnotify_group *b)static int fsnotify_attach_connector_to_object (struct fsnotify_mark_connector __rcu **connp,struct inode *inode, -struct vfsmount *mnt) +struct vfsmount *mnt, +struct fsnotify_group *group) {struct fsnotify_mark_connector *conn;
@@ -517,7 +518,8 @@ static int fsnotify_add_mark_list(struct fsnotify_mark *mark, conn =fsnotify_grab_connector (connp);if (!conn) {spin_unlock (&mark->lock); - err =fsnotify_attach_connector_to_object (connp, inode, mnt); + err =fsnotify_attach_connector_to_object (connp, inode, mnt, + mark->group);if (err)return err; goto restart;
include/linux/fsnotify_backend.h +8 -4
@@ -84,6 +84,8 @@ struct fsnotify_event_private_data;struct fsnotify_fname ;struct fsnotify_iter_info ; +struct mem_cgroup ; +/* * Each group much define these ops. The fsnotify infrastructure will call * these operations for each relevant group.
@@ -129,6 +131,8 @@ struct fsnotify_event {* everything will be cleaned up. */ struct fsnotify_group { +const struct fsnotify_ops *ops;/* how this group handles things */ +/* * How the refcnt is used is up to each group. When the refcnt hits 0 * fsnotify will clean up all of the resources associated with this group.
@@ -139,8 +143,6 @@ struct fsnotify_group {*/ refcount_t refcnt;/* things with interest in this group */ -const struct fsnotify_ops *ops;/* how this group handles things */ -/* needed to send notification to userspace */ spinlock_t notification_lock;/* protect the notification_list */ struct list_head notification_list;/* list of event_holder this group needs to send to userspace */
@@ -162,6 +164,8 @@ struct fsnotify_group {atomic_t num_marks;/* 1 for each mark and 1 for not being * past the point of no return when freeing * a group */ +atomic_t user_waits;/* Number of tasks waiting for user +* response */ struct list_head marks_list;/* all inode marks for this group */ struct fasync_struct *fsn_fa;/* async notification */
@@ -169,8 +173,8 @@ struct fsnotify_group {struct fsnotify_event *overflow_event;/* Event we queue when the * notification list is too * full */ -atomic_t user_waits;/* Number of tasks waiting for user -* response */ + +struct mem_cgroup *memcg;/* memcg to charge allocations */ /* groups can define private fields here or use the void *private */ union {
include/linux/memcontrol.h +7 -0
@@ -352,6 +352,8 @@ struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){ return css ? container_of(css, struct mem_cgroup, css) : NULL; } +struct mem_cgroup *get_mem_cgroup_from_mm (struct mm_struct *mm); +static inline void mem_cgroup_put (struct mem_cgroup *memcg) {css_put (&memcg->css);
@@ -809,6 +811,11 @@ static inline bool task_in_mem_cgroup(struct task_struct *task,return true; } +static inline struct mem_cgroup *get_mem_cgroup_from_mm (struct mm_struct *mm) +{ +return NULL ; +} +static inline void mem_cgroup_put (struct mem_cgroup *memcg) { }
mm/memcontrol.c +1 -1
@@ -678,7 +678,7 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) }EXPORT_SYMBOL (mem_cgroup_from_task); -static struct mem_cgroup *get_mem_cgroup_from_mm (struct mm_struct *mm) +struct mem_cgroup *get_mem_cgroup_from_mm (struct mm_struct *mm) {struct mem_cgroup *memcg =NULL ;
Signature · 7 lines
2.16.1.291.g4437f3f132-goog -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH 3/3] fs: fsnotify: account fsnotify metadata to kmemcg
Quoted 20 lines · Shakeel Butt
A lot of memory can be consumed by the events generated for the huge or unlimited queues if there is either no or slow listener. This can cause system level memory pressure or OOMs. So, it's better to account the fsnotify kmem caches to the memcg of the listener.
There are seven fsnotify kmem caches and among them allocations from
dnotify_struct_cache,dnotify_mark_cache,fanotify_mark_cacheandinotify_inode_mark_cachephappens in the context of syscall from the listener. So,SLAB_ACCOUNTis enough for these caches.The objects from
fsnotify_mark_connector_cachepare not accounted as they are small compared to the notification mark or events and it is unclear whom to account connector to since it is shared by all events attached to the inode.The allocations from the event caches happen in the context of the event producer. For such caches we will need to remote charge the allocations to the listener's memcg. Thus we save the memcg reference in the
fsnotify_groupstructure of the listener.
Quoted 5 lines
This patch has also moved the members of
fsnotify_groupto keep the size same, at least for 64 bit build, even with additional member by filling the holes.Signed-off-by Shakeel Butt <[email protected]>
Andrew, please don't send this patch to Linus until Jan Kara's changes are merged. I will let you know when that happens.
Quoted 298 lines
---
Changelog since v1:
- no more chargingfsnotify_mark_connectorobjectsfs/notify/dnotify/dnotify.c | 5 +++-- fs/notify/fanotify/fanotify.c | 12 +++++++----- fs/notify/fanotify/fanotify.h | 3 ++- fs/notify/fanotify/fanotify_user.c | 7 +++++-- fs/notify/group.c | 4 ++++ fs/notify/inotify/inotify_fsnotify.c | 2 +- fs/notify/inotify/inotify_user.c | 5 ++++- fs/notify/mark.c | 6 ++++-- include/linux/fsnotify_backend.h | 12 ++++++++---- include/linux/memcontrol.h | 7 +++++++ mm/memcontrol.c | 2 +- 11 files changed, 46 insertions(+), 19 deletions(-)fs/notify/dnotify/dnotify.c +3
-2@@ -384,8 +384,9 @@ int fcntl_dirnotify(int fd, struct file *filp, unsigned long arg)static int __init dnotify_init (void ) { - dnotify_struct_cache =KMEM_CACHE (dnotify_struct,SLAB_PANIC ); - dnotify_mark_cache =KMEM_CACHE (dnotify_mark,SLAB_PANIC ); + dnotify_struct_cache =KMEM_CACHE (dnotify_struct, +SLAB_PANIC |SLAB_ACCOUNT ); + dnotify_mark_cache =KMEM_CACHE (dnotify_mark,SLAB_PANIC |SLAB_ACCOUNT ); dnotify_group =fsnotify_alloc_group (&dnotify_fsnotify_ops);if (IS_ERR (dnotify_group))fs/notify/fanotify/fanotify.c +7
-5@@ -140,22 +140,24 @@ static bool fanotify_should_send_event(struct fsnotify_mark *inode_mark, }struct fanotify_event_info *fanotify_alloc_event (struct inode *inode,u32 mask, -const struct path *path) +const struct path *path, +struct mem_cgroup *memcg) {struct fanotify_event_info *event;if (fanotify_is_perm_event (mask)) {struct fanotify_perm_event_info *pevent; - pevent =kmem_cache_alloc (fanotify_perm_event_cachep, -GFP_KERNEL ); + pevent =kmem_cache_alloc_memcg (fanotify_perm_event_cachep, +GFP_KERNEL , memcg);if (!pevent)return NULL ; event = &pevent->fae; pevent->response =0 ; goto init; } - event =kmem_cache_alloc (fanotify_event_cachep,GFP_KERNEL ); + event =kmem_cache_alloc_memcg (fanotify_event_cachep,GFP_KERNEL , + memcg);if (!event)return NULL ; init:__maybe_unused @@ -210,7 +212,7 @@ static int fanotify_handle_event(struct fsnotify_group *group,return 0 ; } - event =fanotify_alloc_event (inode, mask, data); + event =fanotify_alloc_event (inode, mask, data, group->memcg); ret = -ENOMEM ;if (unlikely (!event)) goto finish;fs/notify/fanotify/fanotify.h +2
-1@@ -53,4 +53,5 @@ static inline struct fanotify_event_info *FANOTIFY_E(struct fsnotify_event *fse) }struct fanotify_event_info *fanotify_alloc_event (struct inode *inode,u32 mask, -const struct path *path); +const struct path *path, +struct mem_cgroup *memcg);fs/notify/fanotify/fanotify_user.c +5
-2@@ -16,6 +16,7 @@#include <linux/uaccess.h> #include <linux/compat.h> #include <linux/sched/signal.h> +#include <linux/memcontrol.h> #include <asm/ioctls.h> @@ -756,8 +757,9 @@ SYSCALL_DEFINE2(fanotify_init, unsigned int, flags, unsigned int, event_f_flags) group->fanotify_data.user = user;atomic_inc (&user->fanotify_listeners); + group->memcg =get_mem_cgroup_from_mm (current->mm); - oevent =fanotify_alloc_event (NULL ,FS_Q_OVERFLOW ,NULL ); + oevent =fanotify_alloc_event (NULL ,FS_Q_OVERFLOW ,NULL , group->memcg);if (unlikely (!oevent)) { fd = -ENOMEM ; goto out_destroy_group;@@ -951,7 +953,8 @@ COMPAT_SYSCALL_DEFINE6(fanotify_mark,*/ static int __init fanotify_user_setup (void ) { - fanotify_mark_cache =KMEM_CACHE (fsnotify_mark,SLAB_PANIC ); + fanotify_mark_cache =KMEM_CACHE (fsnotify_mark, +SLAB_PANIC |SLAB_ACCOUNT ); fanotify_event_cachep =KMEM_CACHE (fanotify_event_info,SLAB_PANIC );if (IS_ENABLED (CONFIG_FANOTIFY_ACCESS_PERMISSIONS )) { fanotify_perm_event_cachep =fs/notify/group.c +4
-0@@ -22,6 +22,7 @@#include <linux/srcu.h> #include <linux/rculist.h> #include <linux/wait.h> +#include <linux/memcontrol.h> #include <linux/fsnotify_backend.h> #include "fsnotify.h" @@ -36,6 +37,9 @@ static void fsnotify_final_destroy_group(struct fsnotify_group *group)if (group->ops->free_group_priv) group->ops->free_group_priv (group); +if (group->memcg) +mem_cgroup_put (group->memcg); +kfree (group); }fs/notify/inotify/inotify_fsnotify.c +1
-1@@ -98,7 +98,7 @@ int inotify_handle_event(struct fsnotify_group *group, i_mark = container_of(inode_mark, struct inotify_inode_mark, fsn_mark); - event =kmalloc (alloc_len,GFP_KERNEL ); + event =kmalloc_memcg (alloc_len,GFP_KERNEL , group->memcg);if (unlikely (!event))return -ENOMEM ;fs/notify/inotify/inotify_user.c +4
-1@@ -38,6 +38,7 @@#include <linux/uaccess.h> #include <linux/poll.h> #include <linux/wait.h> +#include <linux/memcontrol.h> #include "inotify.h" #include "../fdinfo.h" @@ -618,6 +619,7 @@ static struct fsnotify_group *inotify_new_group(unsigned int max_events) oevent->name_len =0 ; group->max_events = max_events; + group->memcg =get_mem_cgroup_from_mm (current->mm);spin_lock_init (&group->inotify_data.idr_lock);idr_init (&group->inotify_data.idr);@@ -785,7 +787,8 @@ static int __init inotify_user_setup(void)BUG_ON (hweight32 (ALL_INOTIFY_BITS ) !=21 ); - inotify_inode_mark_cachep =KMEM_CACHE (inotify_inode_mark,SLAB_PANIC ); + inotify_inode_mark_cachep =KMEM_CACHE (inotify_inode_mark, +SLAB_PANIC |SLAB_ACCOUNT ); inotify_max_queued_events =16384 ; init_user_ns.ucount_max[UCOUNT_INOTIFY_INSTANCES ] =128 ;fs/notify/mark.c +4
-2@@ -432,7 +432,8 @@ int fsnotify_compare_groups(struct fsnotify_group *a, struct fsnotify_group *b)static int fsnotify_attach_connector_to_object (struct fsnotify_mark_connector __rcu **connp,struct inode *inode, -struct vfsmount *mnt) +struct vfsmount *mnt, +struct fsnotify_group *group) {struct fsnotify_mark_connector *conn;@@ -517,7 +518,8 @@ static int fsnotify_add_mark_list(struct fsnotify_mark *mark, conn =fsnotify_grab_connector (connp);if (!conn) {spin_unlock (&mark->lock); - err =fsnotify_attach_connector_to_object (connp, inode, mnt); + err =fsnotify_attach_connector_to_object (connp, inode, mnt, + mark->group);if (err)return err; goto restart;include/linux/fsnotify_backend.h +8
-4@@ -84,6 +84,8 @@ struct fsnotify_event_private_data;struct fsnotify_fname ;struct fsnotify_iter_info ; +struct mem_cgroup ; +/* * Each group much define these ops. The fsnotify infrastructure will call * these operations for each relevant group. @@ -129,6 +131,8 @@ struct fsnotify_event {* everything will be cleaned up. */ struct fsnotify_group { +const struct fsnotify_ops *ops;/* how this group handles things */ +/* * How the refcnt is used is up to each group. When the refcnt hits 0 * fsnotify will clean up all of the resources associated with this group. @@ -139,8 +143,6 @@ struct fsnotify_group {*/ refcount_t refcnt;/* things with interest in this group */ -const struct fsnotify_ops *ops;/* how this group handles things */ -/* needed to send notification to userspace */ spinlock_t notification_lock;/* protect the notification_list */ struct list_head notification_list;/* list of event_holder this group needs to send to userspace */ @@ -162,6 +164,8 @@ struct fsnotify_group {atomic_t num_marks;/* 1 for each mark and 1 for not being * past the point of no return when freeing * a group */ +atomic_t user_waits;/* Number of tasks waiting for user +* response */ struct list_head marks_list;/* all inode marks for this group */ struct fasync_struct *fsn_fa;/* async notification */ @@ -169,8 +173,8 @@ struct fsnotify_group {struct fsnotify_event *overflow_event;/* Event we queue when the * notification list is too * full */ -atomic_t user_waits;/* Number of tasks waiting for user -* response */ + +struct mem_cgroup *memcg;/* memcg to charge allocations */ /* groups can define private fields here or use the void *private */ union {include/linux/memcontrol.h +7
-0@@ -352,6 +352,8 @@ struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){ return css ? container_of(css, struct mem_cgroup, css) : NULL; } +struct mem_cgroup *get_mem_cgroup_from_mm (struct mm_struct *mm); +static inline void mem_cgroup_put (struct mem_cgroup *memcg) {css_put (&memcg->css);@@ -809,6 +811,11 @@ static inline bool task_in_mem_cgroup(struct task_struct *task,return true; } +static inline struct mem_cgroup *get_mem_cgroup_from_mm (struct mm_struct *mm) +{ +return NULL ; +} +static inline void mem_cgroup_put (struct mem_cgroup *memcg) { }mm/memcontrol.c +1
-2@@ -678,7 +678,7 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) }EXPORT_SYMBOL (mem_cgroup_from_task); -static struct mem_cgroup *get_mem_cgroup_from_mm (struct mm_struct *mm) +struct mem_cgroup *get_mem_cgroup_from_mm (struct mm_struct *mm) {struct mem_cgroup *memcg =NULL ; --
Quoted 2 lines
2.16.1.291.g4437f3f132-goog
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
kbuild test robot
build botRe: [PATCH 3/3] fs: fsnotify: account fsnotify metadata to kmemcg
In reply to Shakeel ButtThank you for the patch! Yet something to improve:
Hi Shakeel,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on mmotm/master]
[also build test ERROR on next-20180220]
[cannot apply to linus/master v4.16-rc2]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]
url: https://github.com/0day-ci/linux/commits/Shakeel-Butt/Directed-kmem-charging/20180221-071026 base: git://git.cmpxchg.org/linux-mmotm.git master config: i386-randconfig-n0-201807 (attached as .config) compiler: gcc-7 (Debian 7.3.0-1) 7.3.0 reproduce: # save the attached .config to linux build tree make ARCH=i386
All errors (new ones prefixed by >>):
init/initramfs.o: In function `kmalloc_memcg':
include/linux/slab.h:588: undefined reference to __kmalloc_memcg' include/linux/slab.h:588: undefined reference to __kmalloc_memcg' arch/x86/events/core.o: In function `kmalloc_memcg':
include/linux/slab.h:588: undefined reference to __kmalloc_memcg' include/linux/slab.h:588: undefined reference to __kmalloc_memcg' arch/x86/kernel/ksysfs.o: In function `kmalloc_memcg':
include/linux/slab.h:588: undefined reference to __kmalloc_memcg' arch/x86/kernel/e820.o:include/linux/slab.h:588: more undefined references to __kmalloc_memcg' follow fs/notify/fanotify/fanotify.o: In function `fanotify_alloc_event':
>> fs/notify/fanotify/fanotify.c:159: undefined reference to `kmem_cache_alloc_memcg'
fs/eventpoll.o: In function `kmalloc_memcg':
include/linux/slab.h:588: undefined reference to __kmalloc_memcg' fs/signalfd.o: In function kmalloc_memcg':
include/linux/slab.h:588: undefined reference to `__kmalloc_memcg'
fs/timerfd.o: In function `kmalloc_memcg':
include/linux/slab.h:588: undefined reference to __kmalloc_memcg' include/linux/slab.h:588: undefined reference to __kmalloc_memcg'
fs/eventfd.o: In function `kmalloc_memcg':
include/linux/slab.h:588: undefined reference to __kmalloc_memcg' fs/userfaultfd.o:include/linux/slab.h:588: more undefined references to __kmalloc_memcg' follow
vim +159 fs/notify/fanotify/fanotify.c
141
142 struct fanotify_event_info *fanotify_alloc_event(struct inode *inode, u32 mask,
143 const struct path *path,
144 struct mem_cgroup *memcg)
145 {
146 struct fanotify_event_info *event;
147
148 if (fanotify_is_perm_event(mask)) {
149 struct fanotify_perm_event_info *pevent;
150
151 pevent = kmem_cache_alloc_memcg(fanotify_perm_event_cachep,
152 GFP_KERNEL, memcg);
153 if (!pevent)
154 return NULL;
155 event = &pevent->fae;
156 pevent->response = 0;
157 goto init;
158 }
> 159 event = kmem_cache_alloc_memcg(fanotify_event_cachep, GFP_KERNEL,
160 memcg);
161 if (!event)
162 return NULL;
163 init: __maybe_unused
164 fsnotify_init_event(&event->fse, inode, mask);
165 event->tgid = get_pid(task_tgid(current));
166 if (path) {
167 event->path = *path;
168 path_get(&event->path);
169 } else {
170 event->path.mnt = NULL;
171 event->path.dentry = NULL;
172 }
173 return event;
174 }
175
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation