Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
1366c37e MW |
2 | #include <stdlib.h> |
3 | #include <string.h> | |
4 | #include <malloc.h> | |
bbe9d71f | 5 | #include <pthread.h> |
1366c37e MW |
6 | #include <unistd.h> |
7 | #include <assert.h> | |
8 | ||
12ea6539 | 9 | #include <linux/gfp.h> |
bbe9d71f | 10 | #include <linux/poison.h> |
1366c37e | 11 | #include <linux/slab.h> |
bbe9d71f | 12 | #include <linux/radix-tree.h> |
1366c37e MW |
13 | #include <urcu/uatomic.h> |
14 | ||
15 | int nr_allocated; | |
847d3576 | 16 | int preempt_count; |
73bc029b | 17 | int test_verbose; |
1366c37e | 18 | |
bbe9d71f MW |
19 | struct kmem_cache { |
20 | pthread_mutex_t lock; | |
34eee836 MWO |
21 | unsigned int size; |
22 | unsigned int align; | |
bbe9d71f MW |
23 | int nr_objs; |
24 | void *objs; | |
25 | void (*ctor)(void *); | |
e73cb368 | 26 | unsigned int non_kernel; |
000a4493 LH |
27 | unsigned long nr_allocated; |
28 | unsigned long nr_tallocated; | |
bbe9d71f MW |
29 | }; |
30 | ||
e73cb368 LH |
31 | void kmem_cache_set_non_kernel(struct kmem_cache *cachep, unsigned int val) |
32 | { | |
33 | cachep->non_kernel = val; | |
34 | } | |
35 | ||
000a4493 LH |
36 | unsigned long kmem_cache_get_alloc(struct kmem_cache *cachep) |
37 | { | |
38 | return cachep->size * cachep->nr_allocated; | |
39 | } | |
40 | ||
41 | unsigned long kmem_cache_nr_allocated(struct kmem_cache *cachep) | |
42 | { | |
43 | return cachep->nr_allocated; | |
44 | } | |
45 | ||
46 | unsigned long kmem_cache_nr_tallocated(struct kmem_cache *cachep) | |
47 | { | |
48 | return cachep->nr_tallocated; | |
49 | } | |
50 | ||
51 | void kmem_cache_zero_nr_tallocated(struct kmem_cache *cachep) | |
52 | { | |
53 | cachep->nr_tallocated = 0; | |
54 | } | |
55 | ||
b9663a6f MWO |
56 | void *kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru, |
57 | int gfp) | |
1366c37e | 58 | { |
34eee836 | 59 | void *p; |
31023cd6 | 60 | |
e73cb368 LH |
61 | if (!(gfp & __GFP_DIRECT_RECLAIM)) { |
62 | if (!cachep->non_kernel) | |
63 | return NULL; | |
64 | ||
65 | cachep->non_kernel--; | |
66 | } | |
31023cd6 | 67 | |
bbe9d71f MW |
68 | pthread_mutex_lock(&cachep->lock); |
69 | if (cachep->nr_objs) { | |
34eee836 | 70 | struct radix_tree_node *node = cachep->objs; |
bbe9d71f | 71 | cachep->nr_objs--; |
1293d5c5 | 72 | cachep->objs = node->parent; |
bbe9d71f | 73 | pthread_mutex_unlock(&cachep->lock); |
1293d5c5 | 74 | node->parent = NULL; |
34eee836 | 75 | p = node; |
bbe9d71f MW |
76 | } else { |
77 | pthread_mutex_unlock(&cachep->lock); | |
34eee836 MWO |
78 | if (cachep->align) |
79 | posix_memalign(&p, cachep->align, cachep->size); | |
80 | else | |
81 | p = malloc(cachep->size); | |
bbe9d71f | 82 | if (cachep->ctor) |
34eee836 MWO |
83 | cachep->ctor(p); |
84 | else if (gfp & __GFP_ZERO) | |
85 | memset(p, 0, cachep->size); | |
bbe9d71f MW |
86 | } |
87 | ||
000a4493 | 88 | uatomic_inc(&cachep->nr_allocated); |
1366c37e | 89 | uatomic_inc(&nr_allocated); |
000a4493 | 90 | uatomic_inc(&cachep->nr_tallocated); |
5eeb2d23 | 91 | if (kmalloc_verbose) |
34eee836 MWO |
92 | printf("Allocating %p from slab\n", p); |
93 | return p; | |
1366c37e MW |
94 | } |
95 | ||
96 | void kmem_cache_free(struct kmem_cache *cachep, void *objp) | |
97 | { | |
98 | assert(objp); | |
99 | uatomic_dec(&nr_allocated); | |
000a4493 | 100 | uatomic_dec(&cachep->nr_allocated); |
5eeb2d23 MW |
101 | if (kmalloc_verbose) |
102 | printf("Freeing %p to slab\n", objp); | |
bbe9d71f | 103 | pthread_mutex_lock(&cachep->lock); |
34eee836 | 104 | if (cachep->nr_objs > 10 || cachep->align) { |
bbe9d71f MW |
105 | memset(objp, POISON_FREE, cachep->size); |
106 | free(objp); | |
107 | } else { | |
108 | struct radix_tree_node *node = objp; | |
109 | cachep->nr_objs++; | |
1293d5c5 | 110 | node->parent = cachep->objs; |
bbe9d71f MW |
111 | cachep->objs = node; |
112 | } | |
113 | pthread_mutex_unlock(&cachep->lock); | |
1366c37e MW |
114 | } |
115 | ||
116 | struct kmem_cache * | |
34eee836 MWO |
117 | kmem_cache_create(const char *name, unsigned int size, unsigned int align, |
118 | unsigned int flags, void (*ctor)(void *)) | |
1366c37e MW |
119 | { |
120 | struct kmem_cache *ret = malloc(sizeof(*ret)); | |
121 | ||
bbe9d71f | 122 | pthread_mutex_init(&ret->lock, NULL); |
1366c37e | 123 | ret->size = size; |
34eee836 | 124 | ret->align = align; |
bbe9d71f | 125 | ret->nr_objs = 0; |
000a4493 LH |
126 | ret->nr_allocated = 0; |
127 | ret->nr_tallocated = 0; | |
bbe9d71f | 128 | ret->objs = NULL; |
1366c37e | 129 | ret->ctor = ctor; |
e73cb368 | 130 | ret->non_kernel = 0; |
1366c37e MW |
131 | return ret; |
132 | } |