mm/slab_common: kmalloc_node: pass large requests to page allocator

[linux-2.6-block.git] / include / linux / slab.h

diff --git a/include/linux/slab.h b/include/linux/slab.h
index 0fefdf528e0d2e08287fe349dec0aeeed3e5a78b..fd2e129fc8130c331903956c4a886faed2f2f171 100644 (file)
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -456,38 +456,18 @@ static __always_inline void kfree_bulk(size_t size, void **p)
        kmem_cache_free_bulk(NULL, size, p);
 }
 
-#ifdef CONFIG_NUMA
 void *__kmalloc_node(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment
                                                         __alloc_size(1);
 void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node) __assume_slab_alignment
                                                                         __malloc;
-#else
-static __always_inline __alloc_size(1) void *__kmalloc_node(size_t size, gfp_t flags, int node)
-{
-       return __kmalloc(size, flags);
-}
-
-static __always_inline void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node)
-{
-       return kmem_cache_alloc(s, flags);
-}
-#endif
 
 #ifdef CONFIG_TRACING
 extern void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t flags, size_t size)
                                   __assume_slab_alignment __alloc_size(3);
 
-#ifdef CONFIG_NUMA
 extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s, gfp_t gfpflags,
                                         int node, size_t size) __assume_slab_alignment
                                                                __alloc_size(4);
-#else
-static __always_inline __alloc_size(4) void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
-                                                gfp_t gfpflags, int node, size_t size)
-{
-       return kmem_cache_alloc_trace(s, gfpflags, size);
-}
-#endif /* CONFIG_NUMA */
 
 #else /* CONFIG_TRACING */
 static __always_inline __alloc_size(3) void *kmem_cache_alloc_trace(struct kmem_cache *s,
@@ -509,25 +489,11 @@ static __always_inline void *kmem_cache_alloc_node_trace(struct kmem_cache *s, g
 }
 #endif /* CONFIG_TRACING */
 
-extern void *kmalloc_order(size_t size, gfp_t flags, unsigned int order) __assume_page_alignment
-                                                                        __alloc_size(1);
-
-#ifdef CONFIG_TRACING
-extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)
-                               __assume_page_alignment __alloc_size(1);
-#else
-static __always_inline __alloc_size(1) void *kmalloc_order_trace(size_t size, gfp_t flags,
-                                                                unsigned int order)
-{
-       return kmalloc_order(size, flags, order);
-}
-#endif
+void *kmalloc_large(size_t size, gfp_t flags) __assume_page_alignment
+                                             __alloc_size(1);
 
-static __always_inline __alloc_size(1) void *kmalloc_large(size_t size, gfp_t flags)
-{
-       unsigned int order = get_order(size);
-       return kmalloc_order_trace(size, flags, order);
-}
+void *kmalloc_large_node(size_t size, gfp_t flags, int node) __assume_page_alignment
+                                                            __alloc_size(1);
 
 /**
  * kmalloc - allocate memory
@@ -605,23 +571,35 @@ static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags)
        return __kmalloc(size, flags);
 }
 
+#ifndef CONFIG_SLOB
 static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node)
 {
-#ifndef CONFIG_SLOB
-       if (__builtin_constant_p(size) &&
-               size <= KMALLOC_MAX_CACHE_SIZE) {
-               unsigned int i = kmalloc_index(size);
+       if (__builtin_constant_p(size)) {
+               unsigned int index;
+
+               if (size > KMALLOC_MAX_CACHE_SIZE)
+                       return kmalloc_large_node(size, flags, node);
+
+               index = kmalloc_index(size);
 
-               if (!i)
+               if (!index)
                        return ZERO_SIZE_PTR;
 
                return kmem_cache_alloc_node_trace(
-                               kmalloc_caches[kmalloc_type(flags)][i],
+                               kmalloc_caches[kmalloc_type(flags)][index],
                                                flags, node, size);
        }
-#endif
        return __kmalloc_node(size, flags, node);
 }
+#else
+static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node)
+{
+       if (__builtin_constant_p(size) && size > KMALLOC_MAX_CACHE_SIZE)
+               return kmalloc_large_node(size, flags, node);
+
+       return __kmalloc_node(size, flags, node);
+}
+#endif
 
 /**
  * kmalloc_array - allocate memory for an array.
@@ -671,6 +649,12 @@ static inline __alloc_size(1, 2) void *kcalloc(size_t n, size_t size, gfp_t flag
        return kmalloc_array(n, size, flags | __GFP_ZERO);
 }
 
+void *__kmalloc_node_track_caller(size_t size, gfp_t flags, int node,
+                                 unsigned long caller) __alloc_size(1);
+#define kmalloc_node_track_caller(size, flags, node) \
+       __kmalloc_node_track_caller(size, flags, node, \
+                                   _RET_IP_)
+
 /*
  * kmalloc_track_caller is a special version of kmalloc that records the
  * calling function of the routine calling it for slab leak tracking instead
@@ -679,9 +663,9 @@ static inline __alloc_size(1, 2) void *kcalloc(size_t n, size_t size, gfp_t flag
  * allocator where we care about the real place the memory allocation
  * request comes from.
  */
-extern void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller);
 #define kmalloc_track_caller(size, flags) \
-       __kmalloc_track_caller(size, flags, _RET_IP_)
+       __kmalloc_node_track_caller(size, flags, \
+                                   NUMA_NO_NODE, _RET_IP_)
 
 static inline __alloc_size(1, 2) void *kmalloc_array_node(size_t n, size_t size, gfp_t flags,
                                                          int node)
@@ -700,21 +684,6 @@ static inline __alloc_size(1, 2) void *kcalloc_node(size_t n, size_t size, gfp_t
        return kmalloc_array_node(n, size, flags | __GFP_ZERO, node);
 }
 
-
-#ifdef CONFIG_NUMA
-extern void *__kmalloc_node_track_caller(size_t size, gfp_t flags, int node,
-                                        unsigned long caller) __alloc_size(1);
-#define kmalloc_node_track_caller(size, flags, node) \
-       __kmalloc_node_track_caller(size, flags, node, \
-                       _RET_IP_)
-
-#else /* CONFIG_NUMA */
-
-#define kmalloc_node_track_caller(size, flags, node) \
-       kmalloc_track_caller(size, flags)
-
-#endif /* CONFIG_NUMA */
-
 /*
  * Shortcuts
  */