Commit | Line | Data |
---|---|---|
9b1d6c89 ML |
1 | #include <linux/module.h> |
2 | #include <linux/scatterlist.h> | |
3 | #include <linux/mempool.h> | |
4 | #include <linux/slab.h> | |
5 | ||
6 | #define SG_MEMPOOL_NR ARRAY_SIZE(sg_pools) | |
7 | #define SG_MEMPOOL_SIZE 2 | |
8 | ||
9 | struct sg_pool { | |
10 | size_t size; | |
11 | char *name; | |
12 | struct kmem_cache *slab; | |
13 | mempool_t *pool; | |
14 | }; | |
15 | ||
16 | #define SP(x) { .size = x, "sgpool-" __stringify(x) } | |
17 | #if (SG_CHUNK_SIZE < 32) | |
18 | #error SG_CHUNK_SIZE is too small (must be 32 or greater) | |
19 | #endif | |
20 | static struct sg_pool sg_pools[] = { | |
21 | SP(8), | |
22 | SP(16), | |
23 | #if (SG_CHUNK_SIZE > 32) | |
24 | SP(32), | |
25 | #if (SG_CHUNK_SIZE > 64) | |
26 | SP(64), | |
27 | #if (SG_CHUNK_SIZE > 128) | |
28 | SP(128), | |
29 | #if (SG_CHUNK_SIZE > 256) | |
30 | #error SG_CHUNK_SIZE is too large (256 MAX) | |
31 | #endif | |
32 | #endif | |
33 | #endif | |
34 | #endif | |
35 | SP(SG_CHUNK_SIZE) | |
36 | }; | |
37 | #undef SP | |
38 | ||
39 | static inline unsigned int sg_pool_index(unsigned short nents) | |
40 | { | |
41 | unsigned int index; | |
42 | ||
43 | BUG_ON(nents > SG_CHUNK_SIZE); | |
44 | ||
45 | if (nents <= 8) | |
46 | index = 0; | |
47 | else | |
48 | index = get_count_order(nents) - 3; | |
49 | ||
50 | return index; | |
51 | } | |
52 | ||
53 | static void sg_pool_free(struct scatterlist *sgl, unsigned int nents) | |
54 | { | |
55 | struct sg_pool *sgp; | |
56 | ||
57 | sgp = sg_pools + sg_pool_index(nents); | |
58 | mempool_free(sgl, sgp->pool); | |
59 | } | |
60 | ||
61 | static struct scatterlist *sg_pool_alloc(unsigned int nents, gfp_t gfp_mask) | |
62 | { | |
63 | struct sg_pool *sgp; | |
64 | ||
65 | sgp = sg_pools + sg_pool_index(nents); | |
66 | return mempool_alloc(sgp->pool, gfp_mask); | |
67 | } | |
68 | ||
69 | /** | |
70 | * sg_free_table_chained - Free a previously mapped sg table | |
71 | * @table: The sg table header to use | |
4635873c ML |
72 | * @nents_first_chunk: size of the first_chunk SGL passed to |
73 | * sg_alloc_table_chained | |
9b1d6c89 ML |
74 | * |
75 | * Description: | |
76 | * Free an sg table previously allocated and setup with | |
77 | * sg_alloc_table_chained(). | |
78 | * | |
4635873c ML |
79 | * @nents_first_chunk has to be same with that same parameter passed |
80 | * to sg_alloc_table_chained(). | |
81 | * | |
9b1d6c89 | 82 | **/ |
4635873c ML |
83 | void sg_free_table_chained(struct sg_table *table, |
84 | unsigned nents_first_chunk) | |
9b1d6c89 | 85 | { |
4635873c | 86 | if (table->orig_nents <= nents_first_chunk) |
9b1d6c89 | 87 | return; |
4635873c ML |
88 | |
89 | if (nents_first_chunk == 1) | |
90 | nents_first_chunk = 0; | |
91 | ||
92 | __sg_free_table(table, SG_CHUNK_SIZE, nents_first_chunk, sg_pool_free); | |
9b1d6c89 ML |
93 | } |
94 | EXPORT_SYMBOL_GPL(sg_free_table_chained); | |
95 | ||
96 | /** | |
97 | * sg_alloc_table_chained - Allocate and chain SGLs in an sg table | |
98 | * @table: The sg table header to use | |
99 | * @nents: Number of entries in sg list | |
100 | * @first_chunk: first SGL | |
4635873c | 101 | * @nents_first_chunk: number of the SGL of @first_chunk |
9b1d6c89 ML |
102 | * |
103 | * Description: | |
104 | * Allocate and chain SGLs in an sg table. If @nents@ is larger than | |
4635873c | 105 | * @nents_first_chunk a chained sg table will be setup. |
9b1d6c89 ML |
106 | * |
107 | **/ | |
108 | int sg_alloc_table_chained(struct sg_table *table, int nents, | |
4635873c | 109 | struct scatterlist *first_chunk, unsigned nents_first_chunk) |
9b1d6c89 ML |
110 | { |
111 | int ret; | |
112 | ||
113 | BUG_ON(!nents); | |
114 | ||
4635873c ML |
115 | if (first_chunk && nents_first_chunk) { |
116 | if (nents <= nents_first_chunk) { | |
9b1d6c89 ML |
117 | table->nents = table->orig_nents = nents; |
118 | sg_init_table(table->sgl, nents); | |
119 | return 0; | |
120 | } | |
121 | } | |
122 | ||
4635873c ML |
123 | /* User supposes that the 1st SGL includes real entry */ |
124 | if (nents_first_chunk == 1) { | |
125 | first_chunk = NULL; | |
126 | nents_first_chunk = 0; | |
127 | } | |
128 | ||
9b1d6c89 | 129 | ret = __sg_alloc_table(table, nents, SG_CHUNK_SIZE, |
4635873c ML |
130 | first_chunk, nents_first_chunk, |
131 | GFP_ATOMIC, sg_pool_alloc); | |
9b1d6c89 | 132 | if (unlikely(ret)) |
4635873c | 133 | sg_free_table_chained(table, nents_first_chunk); |
9b1d6c89 ML |
134 | return ret; |
135 | } | |
136 | EXPORT_SYMBOL_GPL(sg_alloc_table_chained); | |
137 | ||
138 | static __init int sg_pool_init(void) | |
139 | { | |
140 | int i; | |
141 | ||
142 | for (i = 0; i < SG_MEMPOOL_NR; i++) { | |
143 | struct sg_pool *sgp = sg_pools + i; | |
144 | int size = sgp->size * sizeof(struct scatterlist); | |
145 | ||
146 | sgp->slab = kmem_cache_create(sgp->name, size, 0, | |
147 | SLAB_HWCACHE_ALIGN, NULL); | |
148 | if (!sgp->slab) { | |
149 | printk(KERN_ERR "SG_POOL: can't init sg slab %s\n", | |
150 | sgp->name); | |
151 | goto cleanup_sdb; | |
152 | } | |
153 | ||
154 | sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE, | |
155 | sgp->slab); | |
156 | if (!sgp->pool) { | |
157 | printk(KERN_ERR "SG_POOL: can't init sg mempool %s\n", | |
158 | sgp->name); | |
159 | goto cleanup_sdb; | |
160 | } | |
161 | } | |
162 | ||
163 | return 0; | |
164 | ||
165 | cleanup_sdb: | |
166 | for (i = 0; i < SG_MEMPOOL_NR; i++) { | |
167 | struct sg_pool *sgp = sg_pools + i; | |
7f476715 | 168 | |
169 | mempool_destroy(sgp->pool); | |
170 | kmem_cache_destroy(sgp->slab); | |
9b1d6c89 ML |
171 | } |
172 | ||
173 | return -ENOMEM; | |
174 | } | |
175 | ||
176 | static __exit void sg_pool_exit(void) | |
177 | { | |
178 | int i; | |
179 | ||
180 | for (i = 0; i < SG_MEMPOOL_NR; i++) { | |
181 | struct sg_pool *sgp = sg_pools + i; | |
182 | mempool_destroy(sgp->pool); | |
183 | kmem_cache_destroy(sgp->slab); | |
184 | } | |
185 | } | |
186 | ||
187 | module_init(sg_pool_init); | |
188 | module_exit(sg_pool_exit); |