Merge tag 'omapdrm-4.11-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tomba...

[linux-2.6-block.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index 7ec0a965c6a3a366550b2500f9880f65e5025138..7f4bc7027ed53536efaaf5663f007bd2442de503 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -214,11 +214,13 @@ enum track_item { TRACK_ALLOC, TRACK_FREE };
 static int sysfs_slab_add(struct kmem_cache *);
 static int sysfs_slab_alias(struct kmem_cache *, const char *);
 static void memcg_propagate_slab_attrs(struct kmem_cache *s);
+static void sysfs_slab_remove(struct kmem_cache *s);
 #else
 static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
 static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
                                                        { return 0; }
 static inline void memcg_propagate_slab_attrs(struct kmem_cache *s) { }
+static inline void sysfs_slab_remove(struct kmem_cache *s) { }
 #endif
 
 static inline void stat(const struct kmem_cache *s, enum stat_item si)
@@ -1630,6 +1632,7 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
                flags &= ~GFP_SLAB_BUG_MASK;
                pr_warn("Unexpected gfp: %#x (%pGg). Fixing up to gfp: %#x (%pGg). Fix your code!\n",
                                invalid_mask, &invalid_mask, flags, &flags);
+               dump_stack();
        }
 
        return allocate_slab(s,
@@ -3686,6 +3689,7 @@ int __kmem_cache_shutdown(struct kmem_cache *s)
                if (n->nr_partial || slabs_node(s, node))
                        return 1;
        }
+       sysfs_slab_remove(s);
        return 0;
 }
 
@@ -3952,6 +3956,42 @@ int __kmem_cache_shrink(struct kmem_cache *s)
        return ret;
 }
 
+#ifdef CONFIG_MEMCG
+static void kmemcg_cache_deact_after_rcu(struct kmem_cache *s)
+{
+       /*
+        * Called with all the locks held after a sched RCU grace period.
+        * Even if @s becomes empty after shrinking, we can't know that @s
+        * doesn't have allocations already in-flight and thus can't
+        * destroy @s until the associated memcg is released.
+        *
+        * However, let's remove the sysfs files for empty caches here.
+        * Each cache has a lot of interface files which aren't
+        * particularly useful for empty draining caches; otherwise, we can
+        * easily end up with millions of unnecessary sysfs files on
+        * systems which have a lot of memory and transient cgroups.
+        */
+       if (!__kmem_cache_shrink(s))
+               sysfs_slab_remove(s);
+}
+
+void __kmemcg_cache_deactivate(struct kmem_cache *s)
+{
+       /*
+        * Disable empty slabs caching. Used to avoid pinning offline
+        * memory cgroups by kmem pages that can be freed.
+        */
+       s->cpu_partial = 0;
+       s->min_partial = 0;
+
+       /*
+        * s->cpu_partial is checked locklessly (see put_cpu_partial), so
+        * we have to make sure the change is visible before shrinking.
+        */
+       slab_deactivate_memcg_cache_rcu_sched(s, kmemcg_cache_deact_after_rcu);
+}
+#endif
+
 static int slab_mem_going_offline_callback(void *arg)
 {
        struct kmem_cache *s;
@@ -4108,6 +4148,7 @@ static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
        }
        slab_init_memcg_params(s);
        list_add(&s->list, &slab_caches);
+       memcg_link_cache(s);
        return s;
 }
 
@@ -4667,6 +4708,22 @@ enum slab_stat_type {
 #define SO_OBJECTS     (1 << SL_OBJECTS)
 #define SO_TOTAL       (1 << SL_TOTAL)
 
+#ifdef CONFIG_MEMCG
+static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);
+
+static int __init setup_slub_memcg_sysfs(char *str)
+{
+       int v;
+
+       if (get_option(&str, &v) > 0)
+               memcg_sysfs_enabled = v;
+
+       return 1;
+}
+
+__setup("slub_memcg_sysfs=", setup_slub_memcg_sysfs);
+#endif
+
 static ssize_t show_slab_objects(struct kmem_cache *s,
                            char *buf, unsigned long flags)
 {
@@ -5570,8 +5627,14 @@ static int sysfs_slab_add(struct kmem_cache *s)
 {
        int err;
        const char *name;
+       struct kset *kset = cache_kset(s);
        int unmergeable = slab_unmergeable(s);
 
+       if (!kset) {
+               kobject_init(&s->kobj, &slab_ktype);
+               return 0;
+       }
+
        if (unmergeable) {
                /*
                 * Slabcache can never be merged so we can use the name proper.
@@ -5588,7 +5651,7 @@ static int sysfs_slab_add(struct kmem_cache *s)
                name = create_unique_id(s);
        }
 
-       s->kobj.kset = cache_kset(s);
+       s->kobj.kset = kset;
        err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
        if (err)
                goto out;
@@ -5598,7 +5661,7 @@ static int sysfs_slab_add(struct kmem_cache *s)
                goto out_del_kobj;
 
 #ifdef CONFIG_MEMCG
-       if (is_root_cache(s)) {
+       if (is_root_cache(s) && memcg_sysfs_enabled) {
                s->memcg_kset = kset_create_and_add("cgroup", NULL, &s->kobj);
                if (!s->memcg_kset) {
                        err = -ENOMEM;
@@ -5621,7 +5684,7 @@ out_del_kobj:
        goto out;
 }
 
-void sysfs_slab_remove(struct kmem_cache *s)
+static void sysfs_slab_remove(struct kmem_cache *s)
 {
        if (slab_state < FULL)
                /*
@@ -5630,12 +5693,26 @@ void sysfs_slab_remove(struct kmem_cache *s)
                 */
                return;
 
+       if (!s->kobj.state_in_sysfs)
+               /*
+                * For a memcg cache, this may be called during
+                * deactivation and again on shutdown.  Remove only once.
+                * A cache is never shut down before deactivation is
+                * complete, so no need to worry about synchronization.
+                */
+               return;
+
 #ifdef CONFIG_MEMCG
        kset_unregister(s->memcg_kset);
 #endif
        kobject_uevent(&s->kobj, KOBJ_REMOVE);
        kobject_del(&s->kobj);
-       kobject_put(&s->kobj);
+}
+
+void sysfs_slab_release(struct kmem_cache *s)
+{
+       if (slab_state >= FULL)
+               kobject_put(&s->kobj);
 }
 
 /*