slab: implement bulk free in SLAB allocator

[linux-2.6-block.git] / mm / slab.c

diff --git a/mm/slab.c b/mm/slab.c
index 6ecc697a8bc4670a3910fdcdc155196ee71a539a..2e075c0acfcf842841eae2989a50b3bf2ec072ad 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -2275,7 +2275,7 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
 
        err = setup_cpu_cache(cachep, gfp);
        if (err) {
-               __kmem_cache_shutdown(cachep);
+               __kmem_cache_release(cachep);
                return err;
        }
 
@@ -2413,13 +2413,14 @@ int __kmem_cache_shrink(struct kmem_cache *cachep, bool deactivate)
 }
 
 int __kmem_cache_shutdown(struct kmem_cache *cachep)
+{
+       return __kmem_cache_shrink(cachep, false);
+}
+
+void __kmem_cache_release(struct kmem_cache *cachep)
 {
        int i;
        struct kmem_cache_node *n;
-       int rc = __kmem_cache_shrink(cachep, false);
-
-       if (rc)
-               return rc;
 
        free_percpu(cachep->cpu_cache);
 
@@ -2430,7 +2431,6 @@ int __kmem_cache_shutdown(struct kmem_cache *cachep)
                kfree(n);
                cachep->node[i] = NULL;
        }
-       return 0;
 }
 
 /*
@@ -2926,14 +2926,6 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep,
 #define cache_alloc_debugcheck_after(a,b,objp,d) (objp)
 #endif
 
-static bool slab_should_failslab(struct kmem_cache *cachep, gfp_t flags)
-{
-       if (unlikely(cachep == kmem_cache))
-               return false;
-
-       return should_failslab(cachep->object_size, flags, cachep->flags);
-}
-
 static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
 {
        void *objp;
@@ -3152,14 +3144,10 @@ slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
        int slab_node = numa_mem_id();
 
        flags &= gfp_allowed_mask;
-
-       lockdep_trace_alloc(flags);
-
-       if (slab_should_failslab(cachep, flags))
+       cachep = slab_pre_alloc_hook(cachep, flags);
+       if (unlikely(!cachep))
                return NULL;
 
-       cachep = memcg_kmem_get_cache(cachep, flags);
-
        cache_alloc_debugcheck_before(cachep, flags);
        local_irq_save(save_flags);
 
@@ -3188,16 +3176,11 @@ slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
   out:
        local_irq_restore(save_flags);
        ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller);
-       kmemleak_alloc_recursive(ptr, cachep->object_size, 1, cachep->flags,
-                                flags);
 
-       if (likely(ptr)) {
-               kmemcheck_slab_alloc(cachep, flags, ptr, cachep->object_size);
-               if (unlikely(flags & __GFP_ZERO))
-                       memset(ptr, 0, cachep->object_size);
-       }
+       if (unlikely(flags & __GFP_ZERO) && ptr)
+               memset(ptr, 0, cachep->object_size);
 
-       memcg_kmem_put_cache(cachep);
+       slab_post_alloc_hook(cachep, flags, 1, &ptr);
        return ptr;
 }
 
@@ -3240,30 +3223,21 @@ slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
        void *objp;
 
        flags &= gfp_allowed_mask;
-
-       lockdep_trace_alloc(flags);
-
-       if (slab_should_failslab(cachep, flags))
+       cachep = slab_pre_alloc_hook(cachep, flags);
+       if (unlikely(!cachep))
                return NULL;
 
-       cachep = memcg_kmem_get_cache(cachep, flags);
-
        cache_alloc_debugcheck_before(cachep, flags);
        local_irq_save(save_flags);
        objp = __do_cache_alloc(cachep, flags);
        local_irq_restore(save_flags);
        objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller);
-       kmemleak_alloc_recursive(objp, cachep->object_size, 1, cachep->flags,
-                                flags);
        prefetchw(objp);
 
-       if (likely(objp)) {
-               kmemcheck_slab_alloc(cachep, flags, objp, cachep->object_size);
-               if (unlikely(flags & __GFP_ZERO))
-                       memset(objp, 0, cachep->object_size);
-       }
+       if (unlikely(flags & __GFP_ZERO) && objp)
+               memset(objp, 0, cachep->object_size);
 
-       memcg_kmem_put_cache(cachep);
+       slab_post_alloc_hook(cachep, flags, 1, &objp);
        return objp;
 }
 
@@ -3411,16 +3385,53 @@ void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
 }
 EXPORT_SYMBOL(kmem_cache_alloc);
 
-void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
+static __always_inline void
+cache_alloc_debugcheck_after_bulk(struct kmem_cache *s, gfp_t flags,
+                                 size_t size, void **p, unsigned long caller)
 {
-       __kmem_cache_free_bulk(s, size, p);
+       size_t i;
+
+       for (i = 0; i < size; i++)
+               p[i] = cache_alloc_debugcheck_after(s, flags, p[i], caller);
 }
-EXPORT_SYMBOL(kmem_cache_free_bulk);
 
 int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
-                                                               void **p)
+                         void **p)
 {
-       return __kmem_cache_alloc_bulk(s, flags, size, p);
+       size_t i;
+
+       s = slab_pre_alloc_hook(s, flags);
+       if (!s)
+               return 0;
+
+       cache_alloc_debugcheck_before(s, flags);
+
+       local_irq_disable();
+       for (i = 0; i < size; i++) {
+               void *objp = __do_cache_alloc(s, flags);
+
+               if (unlikely(!objp))
+                       goto error;
+               p[i] = objp;
+       }
+       local_irq_enable();
+
+       cache_alloc_debugcheck_after_bulk(s, flags, size, p, _RET_IP_);
+
+       /* Clear memory outside IRQ disabled section */
+       if (unlikely(flags & __GFP_ZERO))
+               for (i = 0; i < size; i++)
+                       memset(p[i], 0, s->object_size);
+
+       slab_post_alloc_hook(s, flags, size, p);
+       /* FIXME: Trace call missing. Christoph would like a bulk variant */
+       return size;
+error:
+       local_irq_enable();
+       cache_alloc_debugcheck_after_bulk(s, flags, i, p, _RET_IP_);
+       slab_post_alloc_hook(s, flags, i, p);
+       __kmem_cache_free_bulk(s, i, p);
+       return 0;
 }
 EXPORT_SYMBOL(kmem_cache_alloc_bulk);
 
@@ -3567,6 +3578,29 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp)
 }
 EXPORT_SYMBOL(kmem_cache_free);
 
+void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p)
+{
+       struct kmem_cache *s;
+       size_t i;
+
+       local_irq_disable();
+       for (i = 0; i < size; i++) {
+               void *objp = p[i];
+
+               s = cache_from_obj(orig_s, objp);
+
+               debug_check_no_locks_freed(objp, s->object_size);
+               if (!(s->flags & SLAB_DEBUG_OBJECTS))
+                       debug_check_no_obj_freed(objp, s->object_size);
+
+               __cache_free(s, objp, _RET_IP_);
+       }
+       local_irq_enable();
+
+       /* FIXME: add tracing */
+}
+EXPORT_SYMBOL(kmem_cache_free_bulk);
+
 /**
  * kfree - free previously allocated memory
  * @objp: pointer returned by kmalloc.