mm: kmem: move memcg_kmem_bypass() calls to get_mem/obj_cgroup_from_current()

Patch series "mm: kmem: kernel memory accounting in an interrupt context".

This patchset implements memcg-based memory accounting of allocations made
from an interrupt context.

Historically, such allocations were passed unaccounted mostly because
charging the memory cgroup of the current process wasn't an option.  Also
performance reasons were likely a reason too.

The remote charging API allows to temporarily overwrite the currently
active memory cgroup, so that all memory allocations are accounted towards
some specified memory cgroup instead of the memory cgroup of the current
process.

This patchset extends the remote charging API so that it can be used from
an interrupt context.  Then it removes the fence that prevented the
accounting of allocations made from an interrupt context.  It also
contains a couple of optimizations/code refactorings.

This patchset doesn't directly enable accounting for any specific
allocations, but prepares the code base for it.  The bpf memory accounting
will likely be the first user of it: a typical example is a bpf program
parsing an incoming network packet, which allocates an entry in hashmap
map to store some information.

This patch (of 4):

Currently memcg_kmem_bypass() is called before obtaining the current
memory/obj cgroup using get_mem/obj_cgroup_from_current().  Moving
memcg_kmem_bypass() into get_mem/obj_cgroup_from_current() reduces the
number of call sites and allows further code simplifications.

Signed-off-by: Roman Gushchin <guro@fb.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Reviewed-by: Shakeel Butt <shakeelb@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@kernel.org>
Link: http://lkml.kernel.org/r/20200827225843.1270629-1-guro@fb.com
Link: http://lkml.kernel.org/r/20200827225843.1270629-2-guro@fb.com
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 4c741248198b28f03e8db23fbfec1b4282a32a1d..197b9ddb20f3dcd6f36d4d2d2309cd427e857b9a 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -1066,6 +1066,9 @@ EXPORT_SYMBOL(get_mem_cgroup_from_page);
  */
 static __always_inline struct mem_cgroup *get_mem_cgroup_from_current(void)
 {
+       if (memcg_kmem_bypass())
+               return NULL;
+
        if (unlikely(current->active_memcg)) {
                struct mem_cgroup *memcg;
 
@@ -2933,6 +2936,9 @@ __always_inline struct obj_cgroup *get_obj_cgroup_from_current(void)
        struct obj_cgroup *objcg = NULL;
        struct mem_cgroup *memcg;
 
+       if (memcg_kmem_bypass())
+               return NULL;
+
        if (unlikely(!current->mm && !current->active_memcg))
                return NULL;
 
@@ -3059,19 +3065,16 @@ int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
        struct mem_cgroup *memcg;
        int ret = 0;
 
-       if (memcg_kmem_bypass())
-               return 0;
-
        memcg = get_mem_cgroup_from_current();
-       if (!mem_cgroup_is_root(memcg)) {
+       if (memcg && !mem_cgroup_is_root(memcg)) {
                ret = __memcg_kmem_charge(memcg, gfp, 1 << order);
                if (!ret) {
                        page->mem_cgroup = memcg;
                        __SetPageKmemcg(page);
                        return 0;
                }
+               css_put(&memcg->css);
        }
-       css_put(&memcg->css);
        return ret;
 }
 

diff --git a/mm/percpu.c b/mm/percpu.c
index 1ed1a349eab83c0eecd0de37358ac83d4ddc7e06..66a93f096394ab4accb21fa77867f55e702428b2 100644 (file)
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -1584,8 +1584,7 @@ static enum pcpu_chunk_type pcpu_memcg_pre_alloc_hook(size_t size, gfp_t gfp,
 {
        struct obj_cgroup *objcg;
 
-       if (!memcg_kmem_enabled() || !(gfp & __GFP_ACCOUNT) ||
-           memcg_kmem_bypass())
+       if (!memcg_kmem_enabled() || !(gfp & __GFP_ACCOUNT))
                return PCPU_CHUNK_ROOT;
 
        objcg = get_obj_cgroup_from_current();

diff --git a/mm/slab.h b/mm/slab.h
index 06c6587765a342e51fb2c0680444c0e4a205aefc..6d7c6a5056baac906432f3b65daf6f7297aedc99 100644 (file)
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -280,9 +280,6 @@ static inline struct obj_cgroup *memcg_slab_pre_alloc_hook(struct kmem_cache *s,
 {
        struct obj_cgroup *objcg;
 
-       if (memcg_kmem_bypass())
-               return NULL;
-
        objcg = get_obj_cgroup_from_current();
        if (!objcg)
                return NULL;