mm/zsmalloc: allocate exactly size of struct zs_pool
[linux-2.6-block.git] / drivers / block / zram / zram_drv.c
CommitLineData
306b0c95 1/*
f1e3cfff 2 * Compressed RAM block device
306b0c95 3 *
1130ebba 4 * Copyright (C) 2008, 2009, 2010 Nitin Gupta
7bfb3de8 5 * 2012, 2013 Minchan Kim
306b0c95
NG
6 *
7 * This code is released using a dual license strategy: BSD/GPL
8 * You can choose the licence that better fits your requirements.
9 *
10 * Released under the terms of 3-clause BSD License
11 * Released under the terms of GNU General Public License Version 2.0
12 *
306b0c95
NG
13 */
14
f1e3cfff 15#define KMSG_COMPONENT "zram"
306b0c95
NG
16#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
17
b1f5b81e
RJ
18#ifdef CONFIG_ZRAM_DEBUG
19#define DEBUG
20#endif
21
306b0c95
NG
22#include <linux/module.h>
23#include <linux/kernel.h>
8946a086 24#include <linux/bio.h>
306b0c95
NG
25#include <linux/bitops.h>
26#include <linux/blkdev.h>
27#include <linux/buffer_head.h>
28#include <linux/device.h>
29#include <linux/genhd.h>
30#include <linux/highmem.h>
5a0e3ad6 31#include <linux/slab.h>
306b0c95 32#include <linux/string.h>
306b0c95 33#include <linux/vmalloc.h>
fcfa8d95 34#include <linux/err.h>
306b0c95 35
16a4bfb9 36#include "zram_drv.h"
306b0c95
NG
37
38/* Globals */
f1e3cfff 39static int zram_major;
0f0e3ba3 40static struct zram *zram_devices;
b7ca232e 41static const char *default_compressor = "lzo";
306b0c95 42
306b0c95 43/* Module params (documentation at end) */
ca3d70bd 44static unsigned int num_devices = 1;
33863c21 45
a68eb3b6
SS
46#define ZRAM_ATTR_RO(name) \
47static ssize_t zram_attr_##name##_show(struct device *d, \
48 struct device_attribute *attr, char *b) \
49{ \
50 struct zram *zram = dev_to_zram(d); \
56b4e8cb 51 return scnprintf(b, PAGE_SIZE, "%llu\n", \
a68eb3b6
SS
52 (u64)atomic64_read(&zram->stats.name)); \
53} \
54static struct device_attribute dev_attr_##name = \
55 __ATTR(name, S_IRUGO, zram_attr_##name##_show, NULL);
56
be2d1d56
SS
57static inline int init_done(struct zram *zram)
58{
59 return zram->meta != NULL;
60}
61
9b3bb7ab
SS
62static inline struct zram *dev_to_zram(struct device *dev)
63{
64 return (struct zram *)dev_to_disk(dev)->private_data;
65}
66
67static ssize_t disksize_show(struct device *dev,
68 struct device_attribute *attr, char *buf)
69{
70 struct zram *zram = dev_to_zram(dev);
71
56b4e8cb 72 return scnprintf(buf, PAGE_SIZE, "%llu\n", zram->disksize);
9b3bb7ab
SS
73}
74
75static ssize_t initstate_show(struct device *dev,
76 struct device_attribute *attr, char *buf)
77{
a68eb3b6 78 u32 val;
9b3bb7ab
SS
79 struct zram *zram = dev_to_zram(dev);
80
a68eb3b6
SS
81 down_read(&zram->init_lock);
82 val = init_done(zram);
83 up_read(&zram->init_lock);
9b3bb7ab 84
56b4e8cb 85 return scnprintf(buf, PAGE_SIZE, "%u\n", val);
9b3bb7ab
SS
86}
87
88static ssize_t orig_data_size_show(struct device *dev,
89 struct device_attribute *attr, char *buf)
90{
91 struct zram *zram = dev_to_zram(dev);
92
56b4e8cb 93 return scnprintf(buf, PAGE_SIZE, "%llu\n",
90a7806e 94 (u64)(atomic64_read(&zram->stats.pages_stored)) << PAGE_SHIFT);
9b3bb7ab
SS
95}
96
9b3bb7ab
SS
97static ssize_t mem_used_total_show(struct device *dev,
98 struct device_attribute *attr, char *buf)
99{
100 u64 val = 0;
101 struct zram *zram = dev_to_zram(dev);
9b3bb7ab
SS
102
103 down_read(&zram->init_lock);
5a99e95b
WY
104 if (init_done(zram)) {
105 struct zram_meta *meta = zram->meta;
722cdc17 106 val = zs_get_total_pages(meta->mem_pool);
5a99e95b 107 }
9b3bb7ab
SS
108 up_read(&zram->init_lock);
109
722cdc17 110 return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
9b3bb7ab
SS
111}
112
beca3ec7
SS
113static ssize_t max_comp_streams_show(struct device *dev,
114 struct device_attribute *attr, char *buf)
115{
116 int val;
117 struct zram *zram = dev_to_zram(dev);
118
119 down_read(&zram->init_lock);
120 val = zram->max_comp_streams;
121 up_read(&zram->init_lock);
122
56b4e8cb 123 return scnprintf(buf, PAGE_SIZE, "%d\n", val);
beca3ec7
SS
124}
125
9ada9da9
MK
126static ssize_t mem_limit_show(struct device *dev,
127 struct device_attribute *attr, char *buf)
128{
129 u64 val;
130 struct zram *zram = dev_to_zram(dev);
131
132 down_read(&zram->init_lock);
133 val = zram->limit_pages;
134 up_read(&zram->init_lock);
135
136 return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
137}
138
139static ssize_t mem_limit_store(struct device *dev,
140 struct device_attribute *attr, const char *buf, size_t len)
141{
142 u64 limit;
143 char *tmp;
144 struct zram *zram = dev_to_zram(dev);
145
146 limit = memparse(buf, &tmp);
147 if (buf == tmp) /* no chars parsed, invalid input */
148 return -EINVAL;
149
150 down_write(&zram->init_lock);
151 zram->limit_pages = PAGE_ALIGN(limit) >> PAGE_SHIFT;
152 up_write(&zram->init_lock);
153
154 return len;
155}
156
461a8eee
MK
157static ssize_t mem_used_max_show(struct device *dev,
158 struct device_attribute *attr, char *buf)
159{
160 u64 val = 0;
161 struct zram *zram = dev_to_zram(dev);
162
163 down_read(&zram->init_lock);
164 if (init_done(zram))
165 val = atomic_long_read(&zram->stats.max_used_pages);
166 up_read(&zram->init_lock);
167
168 return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
169}
170
171static ssize_t mem_used_max_store(struct device *dev,
172 struct device_attribute *attr, const char *buf, size_t len)
173{
174 int err;
175 unsigned long val;
176 struct zram *zram = dev_to_zram(dev);
461a8eee
MK
177
178 err = kstrtoul(buf, 10, &val);
179 if (err || val != 0)
180 return -EINVAL;
181
182 down_read(&zram->init_lock);
5a99e95b
WY
183 if (init_done(zram)) {
184 struct zram_meta *meta = zram->meta;
461a8eee
MK
185 atomic_long_set(&zram->stats.max_used_pages,
186 zs_get_total_pages(meta->mem_pool));
5a99e95b 187 }
461a8eee
MK
188 up_read(&zram->init_lock);
189
190 return len;
191}
192
beca3ec7
SS
193static ssize_t max_comp_streams_store(struct device *dev,
194 struct device_attribute *attr, const char *buf, size_t len)
195{
196 int num;
197 struct zram *zram = dev_to_zram(dev);
60a726e3 198 int ret;
beca3ec7 199
60a726e3
MK
200 ret = kstrtoint(buf, 0, &num);
201 if (ret < 0)
202 return ret;
beca3ec7
SS
203 if (num < 1)
204 return -EINVAL;
60a726e3 205
beca3ec7
SS
206 down_write(&zram->init_lock);
207 if (init_done(zram)) {
60a726e3 208 if (!zcomp_set_max_streams(zram->comp, num)) {
fe8eb122 209 pr_info("Cannot change max compression streams\n");
60a726e3
MK
210 ret = -EINVAL;
211 goto out;
212 }
beca3ec7 213 }
60a726e3 214
beca3ec7 215 zram->max_comp_streams = num;
60a726e3
MK
216 ret = len;
217out:
beca3ec7 218 up_write(&zram->init_lock);
60a726e3 219 return ret;
beca3ec7
SS
220}
221
e46b8a03
SS
222static ssize_t comp_algorithm_show(struct device *dev,
223 struct device_attribute *attr, char *buf)
224{
225 size_t sz;
226 struct zram *zram = dev_to_zram(dev);
227
228 down_read(&zram->init_lock);
229 sz = zcomp_available_show(zram->compressor, buf);
230 up_read(&zram->init_lock);
231
232 return sz;
233}
234
235static ssize_t comp_algorithm_store(struct device *dev,
236 struct device_attribute *attr, const char *buf, size_t len)
237{
238 struct zram *zram = dev_to_zram(dev);
239 down_write(&zram->init_lock);
240 if (init_done(zram)) {
241 up_write(&zram->init_lock);
242 pr_info("Can't change algorithm for initialized device\n");
243 return -EBUSY;
244 }
245 strlcpy(zram->compressor, buf, sizeof(zram->compressor));
246 up_write(&zram->init_lock);
247 return len;
248}
249
92967471 250/* flag operations needs meta->tb_lock */
8b3cc3ed 251static int zram_test_flag(struct zram_meta *meta, u32 index,
f1e3cfff 252 enum zram_pageflags flag)
306b0c95 253{
d2d5e762 254 return meta->table[index].value & BIT(flag);
306b0c95
NG
255}
256
8b3cc3ed 257static void zram_set_flag(struct zram_meta *meta, u32 index,
f1e3cfff 258 enum zram_pageflags flag)
306b0c95 259{
d2d5e762 260 meta->table[index].value |= BIT(flag);
306b0c95
NG
261}
262
8b3cc3ed 263static void zram_clear_flag(struct zram_meta *meta, u32 index,
f1e3cfff 264 enum zram_pageflags flag)
306b0c95 265{
d2d5e762
WY
266 meta->table[index].value &= ~BIT(flag);
267}
268
269static size_t zram_get_obj_size(struct zram_meta *meta, u32 index)
270{
271 return meta->table[index].value & (BIT(ZRAM_FLAG_SHIFT) - 1);
272}
273
274static void zram_set_obj_size(struct zram_meta *meta,
275 u32 index, size_t size)
276{
277 unsigned long flags = meta->table[index].value >> ZRAM_FLAG_SHIFT;
278
279 meta->table[index].value = (flags << ZRAM_FLAG_SHIFT) | size;
306b0c95
NG
280}
281
9b3bb7ab
SS
282static inline int is_partial_io(struct bio_vec *bvec)
283{
284 return bvec->bv_len != PAGE_SIZE;
285}
286
287/*
288 * Check if request is within bounds and aligned on zram logical blocks.
289 */
54850e73 290static inline int valid_io_request(struct zram *zram,
291 sector_t start, unsigned int size)
9b3bb7ab 292{
54850e73 293 u64 end, bound;
a539c72a 294
9b3bb7ab 295 /* unaligned request */
54850e73 296 if (unlikely(start & (ZRAM_SECTOR_PER_LOGICAL_BLOCK - 1)))
9b3bb7ab 297 return 0;
54850e73 298 if (unlikely(size & (ZRAM_LOGICAL_BLOCK_SIZE - 1)))
9b3bb7ab
SS
299 return 0;
300
54850e73 301 end = start + (size >> SECTOR_SHIFT);
9b3bb7ab
SS
302 bound = zram->disksize >> SECTOR_SHIFT;
303 /* out of range range */
75c7caf5 304 if (unlikely(start >= bound || end > bound || start > end))
9b3bb7ab
SS
305 return 0;
306
307 /* I/O request is valid */
308 return 1;
309}
310
311static void zram_meta_free(struct zram_meta *meta)
312{
313 zs_destroy_pool(meta->mem_pool);
9b3bb7ab
SS
314 vfree(meta->table);
315 kfree(meta);
316}
317
318static struct zram_meta *zram_meta_alloc(u64 disksize)
319{
320 size_t num_pages;
321 struct zram_meta *meta = kmalloc(sizeof(*meta), GFP_KERNEL);
322 if (!meta)
323 goto out;
324
9b3bb7ab
SS
325 num_pages = disksize >> PAGE_SHIFT;
326 meta->table = vzalloc(num_pages * sizeof(*meta->table));
327 if (!meta->table) {
328 pr_err("Error allocating zram address table\n");
b7ca232e 329 goto free_meta;
9b3bb7ab
SS
330 }
331
332 meta->mem_pool = zs_create_pool(GFP_NOIO | __GFP_HIGHMEM);
333 if (!meta->mem_pool) {
334 pr_err("Error creating memory pool\n");
335 goto free_table;
336 }
337
338 return meta;
339
340free_table:
341 vfree(meta->table);
9b3bb7ab
SS
342free_meta:
343 kfree(meta);
344 meta = NULL;
345out:
346 return meta;
347}
348
349static void update_position(u32 *index, int *offset, struct bio_vec *bvec)
350{
351 if (*offset + bvec->bv_len >= PAGE_SIZE)
352 (*index)++;
353 *offset = (*offset + bvec->bv_len) % PAGE_SIZE;
354}
355
306b0c95
NG
356static int page_zero_filled(void *ptr)
357{
358 unsigned int pos;
359 unsigned long *page;
360
361 page = (unsigned long *)ptr;
362
363 for (pos = 0; pos != PAGE_SIZE / sizeof(*page); pos++) {
364 if (page[pos])
365 return 0;
366 }
367
368 return 1;
369}
370
9b3bb7ab
SS
371static void handle_zero_page(struct bio_vec *bvec)
372{
373 struct page *page = bvec->bv_page;
374 void *user_mem;
375
376 user_mem = kmap_atomic(page);
377 if (is_partial_io(bvec))
378 memset(user_mem + bvec->bv_offset, 0, bvec->bv_len);
379 else
380 clear_page(user_mem);
381 kunmap_atomic(user_mem);
382
383 flush_dcache_page(page);
384}
385
d2d5e762
WY
386
387/*
388 * To protect concurrent access to the same index entry,
389 * caller should hold this table index entry's bit_spinlock to
390 * indicate this index entry is accessing.
391 */
f1e3cfff 392static void zram_free_page(struct zram *zram, size_t index)
306b0c95 393{
8b3cc3ed
MK
394 struct zram_meta *meta = zram->meta;
395 unsigned long handle = meta->table[index].handle;
306b0c95 396
fd1a30de 397 if (unlikely(!handle)) {
2e882281
NG
398 /*
399 * No memory is allocated for zero filled pages.
400 * Simply clear zero page flag.
401 */
8b3cc3ed
MK
402 if (zram_test_flag(meta, index, ZRAM_ZERO)) {
403 zram_clear_flag(meta, index, ZRAM_ZERO);
90a7806e 404 atomic64_dec(&zram->stats.zero_pages);
306b0c95
NG
405 }
406 return;
407 }
408
8b3cc3ed 409 zs_free(meta->mem_pool, handle);
306b0c95 410
d2d5e762
WY
411 atomic64_sub(zram_get_obj_size(meta, index),
412 &zram->stats.compr_data_size);
90a7806e 413 atomic64_dec(&zram->stats.pages_stored);
306b0c95 414
8b3cc3ed 415 meta->table[index].handle = 0;
d2d5e762 416 zram_set_obj_size(meta, index, 0);
306b0c95
NG
417}
418
37b51fdd 419static int zram_decompress_page(struct zram *zram, char *mem, u32 index)
306b0c95 420{
b7ca232e 421 int ret = 0;
37b51fdd 422 unsigned char *cmem;
8b3cc3ed 423 struct zram_meta *meta = zram->meta;
92967471 424 unsigned long handle;
023b409f 425 size_t size;
92967471 426
d2d5e762 427 bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
92967471 428 handle = meta->table[index].handle;
d2d5e762 429 size = zram_get_obj_size(meta, index);
306b0c95 430
8b3cc3ed 431 if (!handle || zram_test_flag(meta, index, ZRAM_ZERO)) {
d2d5e762 432 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
42e99bd9 433 clear_page(mem);
8c921b2b
JM
434 return 0;
435 }
306b0c95 436
8b3cc3ed 437 cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_RO);
92967471 438 if (size == PAGE_SIZE)
42e99bd9 439 copy_page(mem, cmem);
37b51fdd 440 else
b7ca232e 441 ret = zcomp_decompress(zram->comp, cmem, size, mem);
8b3cc3ed 442 zs_unmap_object(meta->mem_pool, handle);
d2d5e762 443 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
a1dd52af 444
8c921b2b 445 /* Should NEVER happen. Return bio error if it does. */
b7ca232e 446 if (unlikely(ret)) {
8c921b2b 447 pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
8c921b2b 448 return ret;
a1dd52af 449 }
306b0c95 450
8c921b2b 451 return 0;
306b0c95
NG
452}
453
37b51fdd 454static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
b627cff3 455 u32 index, int offset)
924bd88d
JM
456{
457 int ret;
37b51fdd
SS
458 struct page *page;
459 unsigned char *user_mem, *uncmem = NULL;
8b3cc3ed 460 struct zram_meta *meta = zram->meta;
37b51fdd
SS
461 page = bvec->bv_page;
462
d2d5e762 463 bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
8b3cc3ed
MK
464 if (unlikely(!meta->table[index].handle) ||
465 zram_test_flag(meta, index, ZRAM_ZERO)) {
d2d5e762 466 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
37b51fdd 467 handle_zero_page(bvec);
924bd88d
JM
468 return 0;
469 }
d2d5e762 470 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
924bd88d 471
37b51fdd
SS
472 if (is_partial_io(bvec))
473 /* Use a temporary buffer to decompress the page */
7e5a5104
MK
474 uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
475
476 user_mem = kmap_atomic(page);
477 if (!is_partial_io(bvec))
37b51fdd
SS
478 uncmem = user_mem;
479
480 if (!uncmem) {
481 pr_info("Unable to allocate temp memory\n");
482 ret = -ENOMEM;
483 goto out_cleanup;
484 }
924bd88d 485
37b51fdd 486 ret = zram_decompress_page(zram, uncmem, index);
924bd88d 487 /* Should NEVER happen. Return bio error if it does. */
b7ca232e 488 if (unlikely(ret))
37b51fdd 489 goto out_cleanup;
924bd88d 490
37b51fdd
SS
491 if (is_partial_io(bvec))
492 memcpy(user_mem + bvec->bv_offset, uncmem + offset,
493 bvec->bv_len);
494
495 flush_dcache_page(page);
496 ret = 0;
497out_cleanup:
498 kunmap_atomic(user_mem);
499 if (is_partial_io(bvec))
500 kfree(uncmem);
501 return ret;
924bd88d
JM
502}
503
461a8eee
MK
504static inline void update_used_max(struct zram *zram,
505 const unsigned long pages)
506{
507 int old_max, cur_max;
508
509 old_max = atomic_long_read(&zram->stats.max_used_pages);
510
511 do {
512 cur_max = old_max;
513 if (pages > cur_max)
514 old_max = atomic_long_cmpxchg(
515 &zram->stats.max_used_pages, cur_max, pages);
516 } while (old_max != cur_max);
517}
518
924bd88d
JM
519static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
520 int offset)
306b0c95 521{
397c6066 522 int ret = 0;
8c921b2b 523 size_t clen;
c2344348 524 unsigned long handle;
130f315a 525 struct page *page;
924bd88d 526 unsigned char *user_mem, *cmem, *src, *uncmem = NULL;
8b3cc3ed 527 struct zram_meta *meta = zram->meta;
b7ca232e 528 struct zcomp_strm *zstrm;
e46e3315 529 bool locked = false;
461a8eee 530 unsigned long alloced_pages;
306b0c95 531
8c921b2b 532 page = bvec->bv_page;
924bd88d
JM
533 if (is_partial_io(bvec)) {
534 /*
535 * This is a partial IO. We need to read the full page
536 * before to write the changes.
537 */
7e5a5104 538 uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
924bd88d 539 if (!uncmem) {
924bd88d
JM
540 ret = -ENOMEM;
541 goto out;
542 }
37b51fdd 543 ret = zram_decompress_page(zram, uncmem, index);
397c6066 544 if (ret)
924bd88d 545 goto out;
924bd88d
JM
546 }
547
b7ca232e 548 zstrm = zcomp_strm_find(zram->comp);
e46e3315 549 locked = true;
ba82fe2e 550 user_mem = kmap_atomic(page);
924bd88d 551
397c6066 552 if (is_partial_io(bvec)) {
924bd88d
JM
553 memcpy(uncmem + offset, user_mem + bvec->bv_offset,
554 bvec->bv_len);
397c6066
NG
555 kunmap_atomic(user_mem);
556 user_mem = NULL;
557 } else {
924bd88d 558 uncmem = user_mem;
397c6066 559 }
924bd88d
JM
560
561 if (page_zero_filled(uncmem)) {
c4065152
WY
562 if (user_mem)
563 kunmap_atomic(user_mem);
f40ac2ae 564 /* Free memory associated with this sector now. */
d2d5e762 565 bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
f40ac2ae 566 zram_free_page(zram, index);
92967471 567 zram_set_flag(meta, index, ZRAM_ZERO);
d2d5e762 568 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
f40ac2ae 569
90a7806e 570 atomic64_inc(&zram->stats.zero_pages);
924bd88d
JM
571 ret = 0;
572 goto out;
8c921b2b 573 }
306b0c95 574
b7ca232e 575 ret = zcomp_compress(zram->comp, zstrm, uncmem, &clen);
397c6066
NG
576 if (!is_partial_io(bvec)) {
577 kunmap_atomic(user_mem);
578 user_mem = NULL;
579 uncmem = NULL;
580 }
306b0c95 581
b7ca232e 582 if (unlikely(ret)) {
8c921b2b 583 pr_err("Compression failed! err=%d\n", ret);
924bd88d 584 goto out;
8c921b2b 585 }
b7ca232e 586 src = zstrm->buffer;
c8f2f0db 587 if (unlikely(clen > max_zpage_size)) {
c8f2f0db 588 clen = PAGE_SIZE;
397c6066
NG
589 if (is_partial_io(bvec))
590 src = uncmem;
c8f2f0db 591 }
a1dd52af 592
8b3cc3ed 593 handle = zs_malloc(meta->mem_pool, clen);
fd1a30de 594 if (!handle) {
596b3dd4
MR
595 pr_info("Error allocating memory for compressed page: %u, size=%zu\n",
596 index, clen);
924bd88d
JM
597 ret = -ENOMEM;
598 goto out;
8c921b2b 599 }
9ada9da9 600
461a8eee
MK
601 alloced_pages = zs_get_total_pages(meta->mem_pool);
602 if (zram->limit_pages && alloced_pages > zram->limit_pages) {
9ada9da9
MK
603 zs_free(meta->mem_pool, handle);
604 ret = -ENOMEM;
605 goto out;
606 }
607
461a8eee
MK
608 update_used_max(zram, alloced_pages);
609
8b3cc3ed 610 cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_WO);
306b0c95 611
42e99bd9 612 if ((clen == PAGE_SIZE) && !is_partial_io(bvec)) {
397c6066 613 src = kmap_atomic(page);
42e99bd9 614 copy_page(cmem, src);
397c6066 615 kunmap_atomic(src);
42e99bd9
JL
616 } else {
617 memcpy(cmem, src, clen);
618 }
306b0c95 619
b7ca232e
SS
620 zcomp_strm_release(zram->comp, zstrm);
621 locked = false;
8b3cc3ed 622 zs_unmap_object(meta->mem_pool, handle);
fd1a30de 623
f40ac2ae
SS
624 /*
625 * Free memory associated with this sector
626 * before overwriting unused sectors.
627 */
d2d5e762 628 bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
f40ac2ae
SS
629 zram_free_page(zram, index);
630
8b3cc3ed 631 meta->table[index].handle = handle;
d2d5e762
WY
632 zram_set_obj_size(meta, index, clen);
633 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
306b0c95 634
8c921b2b 635 /* Update stats */
90a7806e
SS
636 atomic64_add(clen, &zram->stats.compr_data_size);
637 atomic64_inc(&zram->stats.pages_stored);
924bd88d 638out:
e46e3315 639 if (locked)
b7ca232e 640 zcomp_strm_release(zram->comp, zstrm);
397c6066
NG
641 if (is_partial_io(bvec))
642 kfree(uncmem);
924bd88d 643 return ret;
8c921b2b
JM
644}
645
646static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,
b627cff3 647 int offset, int rw)
8c921b2b 648{
c5bde238 649 int ret;
8c921b2b 650
be257c61
SS
651 if (rw == READ) {
652 atomic64_inc(&zram->stats.num_reads);
b627cff3 653 ret = zram_bvec_read(zram, bvec, index, offset);
be257c61
SS
654 } else {
655 atomic64_inc(&zram->stats.num_writes);
c5bde238 656 ret = zram_bvec_write(zram, bvec, index, offset);
be257c61 657 }
c5bde238 658
0cf1e9d6
CY
659 if (unlikely(ret)) {
660 if (rw == READ)
661 atomic64_inc(&zram->stats.failed_reads);
662 else
663 atomic64_inc(&zram->stats.failed_writes);
664 }
665
c5bde238 666 return ret;
924bd88d
JM
667}
668
f4659d8e
JK
669/*
670 * zram_bio_discard - handler on discard request
671 * @index: physical block index in PAGE_SIZE units
672 * @offset: byte offset within physical block
673 */
674static void zram_bio_discard(struct zram *zram, u32 index,
675 int offset, struct bio *bio)
676{
677 size_t n = bio->bi_iter.bi_size;
d2d5e762 678 struct zram_meta *meta = zram->meta;
f4659d8e
JK
679
680 /*
681 * zram manages data in physical block size units. Because logical block
682 * size isn't identical with physical block size on some arch, we
683 * could get a discard request pointing to a specific offset within a
684 * certain physical block. Although we can handle this request by
685 * reading that physiclal block and decompressing and partially zeroing
686 * and re-compressing and then re-storing it, this isn't reasonable
687 * because our intent with a discard request is to save memory. So
688 * skipping this logical block is appropriate here.
689 */
690 if (offset) {
38515c73 691 if (n <= (PAGE_SIZE - offset))
f4659d8e
JK
692 return;
693
38515c73 694 n -= (PAGE_SIZE - offset);
f4659d8e
JK
695 index++;
696 }
697
698 while (n >= PAGE_SIZE) {
d2d5e762 699 bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
f4659d8e 700 zram_free_page(zram, index);
d2d5e762 701 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
015254da 702 atomic64_inc(&zram->stats.notify_free);
f4659d8e
JK
703 index++;
704 n -= PAGE_SIZE;
705 }
706}
707
2b86ab9c 708static void zram_reset_device(struct zram *zram, bool reset_capacity)
924bd88d 709{
9b3bb7ab
SS
710 size_t index;
711 struct zram_meta *meta;
712
644d4787 713 down_write(&zram->init_lock);
9ada9da9
MK
714
715 zram->limit_pages = 0;
716
be2d1d56 717 if (!init_done(zram)) {
644d4787 718 up_write(&zram->init_lock);
9b3bb7ab 719 return;
644d4787 720 }
9b3bb7ab
SS
721
722 meta = zram->meta;
9b3bb7ab
SS
723 /* Free all pages that are still in this zram device */
724 for (index = 0; index < zram->disksize >> PAGE_SHIFT; index++) {
725 unsigned long handle = meta->table[index].handle;
726 if (!handle)
727 continue;
728
729 zs_free(meta->mem_pool, handle);
730 }
731
b7ca232e 732 zcomp_destroy(zram->comp);
beca3ec7
SS
733 zram->max_comp_streams = 1;
734
9b3bb7ab
SS
735 zram_meta_free(zram->meta);
736 zram->meta = NULL;
737 /* Reset stats */
738 memset(&zram->stats, 0, sizeof(zram->stats));
739
740 zram->disksize = 0;
b4c5c609 741 if (reset_capacity)
2b86ab9c 742 set_capacity(zram->disk, 0);
b4c5c609 743
644d4787 744 up_write(&zram->init_lock);
b4c5c609
MK
745
746 /*
747 * Revalidate disk out of the init_lock to avoid lockdep splat.
748 * It's okay because disk's capacity is protected by init_lock
749 * so that revalidate_disk always sees up-to-date capacity.
750 */
751 if (reset_capacity)
752 revalidate_disk(zram->disk);
9b3bb7ab
SS
753}
754
9b3bb7ab
SS
755static ssize_t disksize_store(struct device *dev,
756 struct device_attribute *attr, const char *buf, size_t len)
757{
758 u64 disksize;
d61f98c7 759 struct zcomp *comp;
9b3bb7ab
SS
760 struct zram_meta *meta;
761 struct zram *zram = dev_to_zram(dev);
fcfa8d95 762 int err;
9b3bb7ab
SS
763
764 disksize = memparse(buf, NULL);
765 if (!disksize)
766 return -EINVAL;
767
768 disksize = PAGE_ALIGN(disksize);
769 meta = zram_meta_alloc(disksize);
db5d711e
MK
770 if (!meta)
771 return -ENOMEM;
b67d1ec1 772
d61f98c7 773 comp = zcomp_create(zram->compressor, zram->max_comp_streams);
fcfa8d95 774 if (IS_ERR(comp)) {
d61f98c7
SS
775 pr_info("Cannot initialise %s compressing backend\n",
776 zram->compressor);
fcfa8d95
SS
777 err = PTR_ERR(comp);
778 goto out_free_meta;
d61f98c7
SS
779 }
780
9b3bb7ab 781 down_write(&zram->init_lock);
be2d1d56 782 if (init_done(zram)) {
9b3bb7ab 783 pr_info("Cannot change disksize for initialized device\n");
b7ca232e 784 err = -EBUSY;
fcfa8d95 785 goto out_destroy_comp;
9b3bb7ab
SS
786 }
787
b67d1ec1 788 zram->meta = meta;
d61f98c7 789 zram->comp = comp;
9b3bb7ab
SS
790 zram->disksize = disksize;
791 set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
9b3bb7ab 792 up_write(&zram->init_lock);
b4c5c609
MK
793
794 /*
795 * Revalidate disk out of the init_lock to avoid lockdep splat.
796 * It's okay because disk's capacity is protected by init_lock
797 * so that revalidate_disk always sees up-to-date capacity.
798 */
799 revalidate_disk(zram->disk);
800
9b3bb7ab 801 return len;
b7ca232e 802
fcfa8d95
SS
803out_destroy_comp:
804 up_write(&zram->init_lock);
805 zcomp_destroy(comp);
806out_free_meta:
b7ca232e
SS
807 zram_meta_free(meta);
808 return err;
9b3bb7ab
SS
809}
810
811static ssize_t reset_store(struct device *dev,
812 struct device_attribute *attr, const char *buf, size_t len)
813{
814 int ret;
815 unsigned short do_reset;
816 struct zram *zram;
817 struct block_device *bdev;
818
819 zram = dev_to_zram(dev);
820 bdev = bdget_disk(zram->disk, 0);
821
46a51c80
RK
822 if (!bdev)
823 return -ENOMEM;
824
9b3bb7ab 825 /* Do not reset an active device! */
1b672224
RK
826 if (bdev->bd_holders) {
827 ret = -EBUSY;
828 goto out;
829 }
9b3bb7ab
SS
830
831 ret = kstrtou16(buf, 10, &do_reset);
832 if (ret)
1b672224 833 goto out;
9b3bb7ab 834
1b672224
RK
835 if (!do_reset) {
836 ret = -EINVAL;
837 goto out;
838 }
9b3bb7ab
SS
839
840 /* Make sure all pending I/O is finished */
46a51c80 841 fsync_bdev(bdev);
1b672224 842 bdput(bdev);
9b3bb7ab 843
2b86ab9c 844 zram_reset_device(zram, true);
9b3bb7ab 845 return len;
1b672224
RK
846
847out:
848 bdput(bdev);
849 return ret;
8c921b2b
JM
850}
851
be257c61 852static void __zram_make_request(struct zram *zram, struct bio *bio)
8c921b2b 853{
b627cff3 854 int offset, rw;
8c921b2b 855 u32 index;
7988613b
KO
856 struct bio_vec bvec;
857 struct bvec_iter iter;
8c921b2b 858
4f024f37
KO
859 index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
860 offset = (bio->bi_iter.bi_sector &
861 (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT;
8c921b2b 862
f4659d8e
JK
863 if (unlikely(bio->bi_rw & REQ_DISCARD)) {
864 zram_bio_discard(zram, index, offset, bio);
865 bio_endio(bio, 0);
866 return;
867 }
868
b627cff3 869 rw = bio_data_dir(bio);
7988613b 870 bio_for_each_segment(bvec, bio, iter) {
924bd88d
JM
871 int max_transfer_size = PAGE_SIZE - offset;
872
7988613b 873 if (bvec.bv_len > max_transfer_size) {
924bd88d
JM
874 /*
875 * zram_bvec_rw() can only make operation on a single
876 * zram page. Split the bio vector.
877 */
878 struct bio_vec bv;
879
7988613b 880 bv.bv_page = bvec.bv_page;
924bd88d 881 bv.bv_len = max_transfer_size;
7988613b 882 bv.bv_offset = bvec.bv_offset;
924bd88d 883
b627cff3 884 if (zram_bvec_rw(zram, &bv, index, offset, rw) < 0)
924bd88d
JM
885 goto out;
886
7988613b 887 bv.bv_len = bvec.bv_len - max_transfer_size;
924bd88d 888 bv.bv_offset += max_transfer_size;
b627cff3 889 if (zram_bvec_rw(zram, &bv, index + 1, 0, rw) < 0)
924bd88d
JM
890 goto out;
891 } else
b627cff3 892 if (zram_bvec_rw(zram, &bvec, index, offset, rw) < 0)
924bd88d
JM
893 goto out;
894
7988613b 895 update_position(&index, &offset, &bvec);
a1dd52af 896 }
306b0c95
NG
897
898 set_bit(BIO_UPTODATE, &bio->bi_flags);
899 bio_endio(bio, 0);
7d7854b4 900 return;
306b0c95
NG
901
902out:
306b0c95 903 bio_io_error(bio);
306b0c95
NG
904}
905
306b0c95 906/*
f1e3cfff 907 * Handler function for all zram I/O requests.
306b0c95 908 */
5a7bbad2 909static void zram_make_request(struct request_queue *queue, struct bio *bio)
306b0c95 910{
f1e3cfff 911 struct zram *zram = queue->queuedata;
306b0c95 912
0900beae 913 down_read(&zram->init_lock);
be2d1d56 914 if (unlikely(!init_done(zram)))
3de738cd 915 goto error;
0900beae 916
54850e73 917 if (!valid_io_request(zram, bio->bi_iter.bi_sector,
918 bio->bi_iter.bi_size)) {
da5cc7d3 919 atomic64_inc(&zram->stats.invalid_io);
3de738cd 920 goto error;
6642a67c
JM
921 }
922
be257c61 923 __zram_make_request(zram, bio);
0900beae 924 up_read(&zram->init_lock);
306b0c95 925
b4fdcb02 926 return;
0900beae 927
0900beae 928error:
3de738cd 929 up_read(&zram->init_lock);
0900beae 930 bio_io_error(bio);
306b0c95
NG
931}
932
2ccbec05
NG
933static void zram_slot_free_notify(struct block_device *bdev,
934 unsigned long index)
107c161b 935{
f1e3cfff 936 struct zram *zram;
f614a9f4 937 struct zram_meta *meta;
107c161b 938
f1e3cfff 939 zram = bdev->bd_disk->private_data;
f614a9f4 940 meta = zram->meta;
a0c516cb 941
d2d5e762 942 bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
f614a9f4 943 zram_free_page(zram, index);
d2d5e762 944 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
f614a9f4 945 atomic64_inc(&zram->stats.notify_free);
107c161b
NG
946}
947
8c7f0102 948static int zram_rw_page(struct block_device *bdev, sector_t sector,
949 struct page *page, int rw)
950{
951 int offset, err;
952 u32 index;
953 struct zram *zram;
954 struct bio_vec bv;
955
956 zram = bdev->bd_disk->private_data;
957 if (!valid_io_request(zram, sector, PAGE_SIZE)) {
958 atomic64_inc(&zram->stats.invalid_io);
959 return -EINVAL;
960 }
961
962 down_read(&zram->init_lock);
963 if (unlikely(!init_done(zram))) {
964 err = -EIO;
965 goto out_unlock;
966 }
967
968 index = sector >> SECTORS_PER_PAGE_SHIFT;
969 offset = sector & (SECTORS_PER_PAGE - 1) << SECTOR_SHIFT;
970
971 bv.bv_page = page;
972 bv.bv_len = PAGE_SIZE;
973 bv.bv_offset = 0;
974
975 err = zram_bvec_rw(zram, &bv, index, offset, rw);
976out_unlock:
977 up_read(&zram->init_lock);
978 /*
979 * If I/O fails, just return error(ie, non-zero) without
980 * calling page_endio.
981 * It causes resubmit the I/O with bio request by upper functions
982 * of rw_page(e.g., swap_readpage, __swap_writepage) and
983 * bio->bi_end_io does things to handle the error
984 * (e.g., SetPageError, set_page_dirty and extra works).
985 */
986 if (err == 0)
987 page_endio(page, rw, 0);
988 return err;
989}
990
f1e3cfff 991static const struct block_device_operations zram_devops = {
f1e3cfff 992 .swap_slot_free_notify = zram_slot_free_notify,
8c7f0102 993 .rw_page = zram_rw_page,
107c161b 994 .owner = THIS_MODULE
306b0c95
NG
995};
996
9b3bb7ab
SS
997static DEVICE_ATTR(disksize, S_IRUGO | S_IWUSR,
998 disksize_show, disksize_store);
999static DEVICE_ATTR(initstate, S_IRUGO, initstate_show, NULL);
1000static DEVICE_ATTR(reset, S_IWUSR, NULL, reset_store);
9b3bb7ab 1001static DEVICE_ATTR(orig_data_size, S_IRUGO, orig_data_size_show, NULL);
9b3bb7ab 1002static DEVICE_ATTR(mem_used_total, S_IRUGO, mem_used_total_show, NULL);
9ada9da9
MK
1003static DEVICE_ATTR(mem_limit, S_IRUGO | S_IWUSR, mem_limit_show,
1004 mem_limit_store);
461a8eee
MK
1005static DEVICE_ATTR(mem_used_max, S_IRUGO | S_IWUSR, mem_used_max_show,
1006 mem_used_max_store);
beca3ec7
SS
1007static DEVICE_ATTR(max_comp_streams, S_IRUGO | S_IWUSR,
1008 max_comp_streams_show, max_comp_streams_store);
e46b8a03
SS
1009static DEVICE_ATTR(comp_algorithm, S_IRUGO | S_IWUSR,
1010 comp_algorithm_show, comp_algorithm_store);
9b3bb7ab 1011
a68eb3b6
SS
1012ZRAM_ATTR_RO(num_reads);
1013ZRAM_ATTR_RO(num_writes);
64447249
SS
1014ZRAM_ATTR_RO(failed_reads);
1015ZRAM_ATTR_RO(failed_writes);
a68eb3b6
SS
1016ZRAM_ATTR_RO(invalid_io);
1017ZRAM_ATTR_RO(notify_free);
1018ZRAM_ATTR_RO(zero_pages);
1019ZRAM_ATTR_RO(compr_data_size);
1020
9b3bb7ab
SS
1021static struct attribute *zram_disk_attrs[] = {
1022 &dev_attr_disksize.attr,
1023 &dev_attr_initstate.attr,
1024 &dev_attr_reset.attr,
1025 &dev_attr_num_reads.attr,
1026 &dev_attr_num_writes.attr,
64447249
SS
1027 &dev_attr_failed_reads.attr,
1028 &dev_attr_failed_writes.attr,
9b3bb7ab
SS
1029 &dev_attr_invalid_io.attr,
1030 &dev_attr_notify_free.attr,
1031 &dev_attr_zero_pages.attr,
1032 &dev_attr_orig_data_size.attr,
1033 &dev_attr_compr_data_size.attr,
1034 &dev_attr_mem_used_total.attr,
9ada9da9 1035 &dev_attr_mem_limit.attr,
461a8eee 1036 &dev_attr_mem_used_max.attr,
beca3ec7 1037 &dev_attr_max_comp_streams.attr,
e46b8a03 1038 &dev_attr_comp_algorithm.attr,
9b3bb7ab
SS
1039 NULL,
1040};
1041
1042static struct attribute_group zram_disk_attr_group = {
1043 .attrs = zram_disk_attrs,
1044};
1045
f1e3cfff 1046static int create_device(struct zram *zram, int device_id)
306b0c95 1047{
39a9b8ac 1048 int ret = -ENOMEM;
de1a21a0 1049
0900beae 1050 init_rwsem(&zram->init_lock);
306b0c95 1051
f1e3cfff
NG
1052 zram->queue = blk_alloc_queue(GFP_KERNEL);
1053 if (!zram->queue) {
306b0c95
NG
1054 pr_err("Error allocating disk queue for device %d\n",
1055 device_id);
de1a21a0 1056 goto out;
306b0c95
NG
1057 }
1058
f1e3cfff
NG
1059 blk_queue_make_request(zram->queue, zram_make_request);
1060 zram->queue->queuedata = zram;
306b0c95
NG
1061
1062 /* gendisk structure */
f1e3cfff
NG
1063 zram->disk = alloc_disk(1);
1064 if (!zram->disk) {
94b8435f 1065 pr_warn("Error allocating disk structure for device %d\n",
306b0c95 1066 device_id);
39a9b8ac 1067 goto out_free_queue;
306b0c95
NG
1068 }
1069
f1e3cfff
NG
1070 zram->disk->major = zram_major;
1071 zram->disk->first_minor = device_id;
1072 zram->disk->fops = &zram_devops;
1073 zram->disk->queue = zram->queue;
1074 zram->disk->private_data = zram;
1075 snprintf(zram->disk->disk_name, 16, "zram%d", device_id);
306b0c95 1076
33863c21 1077 /* Actual capacity set using syfs (/sys/block/zram<id>/disksize */
f1e3cfff 1078 set_capacity(zram->disk, 0);
b67d1ec1
SS
1079 /* zram devices sort of resembles non-rotational disks */
1080 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, zram->disk->queue);
b277da0a 1081 queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, zram->disk->queue);
a1dd52af
NG
1082 /*
1083 * To ensure that we always get PAGE_SIZE aligned
1084 * and n*PAGE_SIZED sized I/O requests.
1085 */
f1e3cfff 1086 blk_queue_physical_block_size(zram->disk->queue, PAGE_SIZE);
7b19b8d4
RJ
1087 blk_queue_logical_block_size(zram->disk->queue,
1088 ZRAM_LOGICAL_BLOCK_SIZE);
f1e3cfff
NG
1089 blk_queue_io_min(zram->disk->queue, PAGE_SIZE);
1090 blk_queue_io_opt(zram->disk->queue, PAGE_SIZE);
f4659d8e
JK
1091 zram->disk->queue->limits.discard_granularity = PAGE_SIZE;
1092 zram->disk->queue->limits.max_discard_sectors = UINT_MAX;
1093 /*
1094 * zram_bio_discard() will clear all logical blocks if logical block
1095 * size is identical with physical block size(PAGE_SIZE). But if it is
1096 * different, we will skip discarding some parts of logical blocks in
1097 * the part of the request range which isn't aligned to physical block
1098 * size. So we can't ensure that all discarded logical blocks are
1099 * zeroed.
1100 */
1101 if (ZRAM_LOGICAL_BLOCK_SIZE == PAGE_SIZE)
1102 zram->disk->queue->limits.discard_zeroes_data = 1;
1103 else
1104 zram->disk->queue->limits.discard_zeroes_data = 0;
1105 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, zram->disk->queue);
5d83d5a0 1106
f1e3cfff 1107 add_disk(zram->disk);
306b0c95 1108
33863c21
NG
1109 ret = sysfs_create_group(&disk_to_dev(zram->disk)->kobj,
1110 &zram_disk_attr_group);
1111 if (ret < 0) {
94b8435f 1112 pr_warn("Error creating sysfs group");
39a9b8ac 1113 goto out_free_disk;
33863c21 1114 }
e46b8a03 1115 strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor));
be2d1d56 1116 zram->meta = NULL;
beca3ec7 1117 zram->max_comp_streams = 1;
39a9b8ac 1118 return 0;
de1a21a0 1119
39a9b8ac
JL
1120out_free_disk:
1121 del_gendisk(zram->disk);
1122 put_disk(zram->disk);
1123out_free_queue:
1124 blk_cleanup_queue(zram->queue);
de1a21a0
NG
1125out:
1126 return ret;
306b0c95
NG
1127}
1128
f1e3cfff 1129static void destroy_device(struct zram *zram)
306b0c95 1130{
33863c21
NG
1131 sysfs_remove_group(&disk_to_dev(zram->disk)->kobj,
1132 &zram_disk_attr_group);
33863c21 1133
59d3fe54
RK
1134 del_gendisk(zram->disk);
1135 put_disk(zram->disk);
306b0c95 1136
59d3fe54 1137 blk_cleanup_queue(zram->queue);
306b0c95
NG
1138}
1139
f1e3cfff 1140static int __init zram_init(void)
306b0c95 1141{
de1a21a0 1142 int ret, dev_id;
306b0c95 1143
5fa5a901 1144 if (num_devices > max_num_devices) {
94b8435f 1145 pr_warn("Invalid value for num_devices: %u\n",
5fa5a901 1146 num_devices);
de1a21a0
NG
1147 ret = -EINVAL;
1148 goto out;
306b0c95
NG
1149 }
1150
f1e3cfff
NG
1151 zram_major = register_blkdev(0, "zram");
1152 if (zram_major <= 0) {
94b8435f 1153 pr_warn("Unable to get major number\n");
de1a21a0
NG
1154 ret = -EBUSY;
1155 goto out;
306b0c95
NG
1156 }
1157
306b0c95 1158 /* Allocate the device array and initialize each one */
5fa5a901 1159 zram_devices = kzalloc(num_devices * sizeof(struct zram), GFP_KERNEL);
43801f6e 1160 if (!zram_devices) {
de1a21a0
NG
1161 ret = -ENOMEM;
1162 goto unregister;
1163 }
306b0c95 1164
5fa5a901 1165 for (dev_id = 0; dev_id < num_devices; dev_id++) {
43801f6e 1166 ret = create_device(&zram_devices[dev_id], dev_id);
de1a21a0 1167 if (ret)
3bf040c7 1168 goto free_devices;
de1a21a0
NG
1169 }
1170
ca3d70bd
DB
1171 pr_info("Created %u device(s) ...\n", num_devices);
1172
306b0c95 1173 return 0;
de1a21a0 1174
3bf040c7 1175free_devices:
de1a21a0 1176 while (dev_id)
43801f6e
NW
1177 destroy_device(&zram_devices[--dev_id]);
1178 kfree(zram_devices);
de1a21a0 1179unregister:
f1e3cfff 1180 unregister_blkdev(zram_major, "zram");
de1a21a0 1181out:
306b0c95
NG
1182 return ret;
1183}
1184
f1e3cfff 1185static void __exit zram_exit(void)
306b0c95
NG
1186{
1187 int i;
f1e3cfff 1188 struct zram *zram;
306b0c95 1189
5fa5a901 1190 for (i = 0; i < num_devices; i++) {
43801f6e 1191 zram = &zram_devices[i];
306b0c95 1192
f1e3cfff 1193 destroy_device(zram);
2b86ab9c
MK
1194 /*
1195 * Shouldn't access zram->disk after destroy_device
1196 * because destroy_device already released zram->disk.
1197 */
1198 zram_reset_device(zram, false);
306b0c95
NG
1199 }
1200
f1e3cfff 1201 unregister_blkdev(zram_major, "zram");
306b0c95 1202
43801f6e 1203 kfree(zram_devices);
306b0c95
NG
1204 pr_debug("Cleanup done!\n");
1205}
1206
f1e3cfff
NG
1207module_init(zram_init);
1208module_exit(zram_exit);
306b0c95 1209
9b3bb7ab
SS
1210module_param(num_devices, uint, 0);
1211MODULE_PARM_DESC(num_devices, "Number of zram devices");
1212
306b0c95
NG
1213MODULE_LICENSE("Dual BSD/GPL");
1214MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>");
f1e3cfff 1215MODULE_DESCRIPTION("Compressed RAM Block Device");