mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-26 19:00:25 +00:00
![Minchan Kim](/assets/img/avatar_default.png)
The copy_page is optimized memcpy for page-alinged address. If it is
used with non-page aligned address, it can corrupt memory which means
system corruption. With zram, it can happen with
1. 64K architecture
2. partial IO
3. slub debug
Partial IO need to allocate a page and zram allocates it via kmalloc.
With slub debug, kmalloc(PAGE_SIZE) doesn't return page-size aligned
address. And finally, copy_page(mem, cmem) corrupts memory.
So, this patch changes it to memcpy.
Actuaully, we don't need to change zram_bvec_write part because zsmalloc
returns page-aligned address in case of PAGE_SIZE class but it's not
good to rely on the internal of zsmalloc.
Note:
When this patch is merged to stable, clear_page should be fixed, too.
Unfortunately, recent zram removes it by "same page merge" feature so
it's hard to backport this patch to -stable tree.
I will handle it when I receive the mail from stable tree maintainer to
merge this patch to backport.
Fixes: 42e99bd
("zram: optimize memory operations with clear_page()/copy_page()")
Link: http://lkml.kernel.org/r/1492042622-12074-2-git-send-email-minchan@kernel.org
Signed-off-by: Minchan Kim <minchan@kernel.org>
Cc: Sergey Senozhatsky <sergey.senozhatsky@gmail.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
1410 lines
34 KiB
C
1410 lines
34 KiB
C
/*
|
|
* Compressed RAM block device
|
|
*
|
|
* Copyright (C) 2008, 2009, 2010 Nitin Gupta
|
|
* 2012, 2013 Minchan Kim
|
|
*
|
|
* This code is released using a dual license strategy: BSD/GPL
|
|
* You can choose the licence that better fits your requirements.
|
|
*
|
|
* Released under the terms of 3-clause BSD License
|
|
* Released under the terms of GNU General Public License Version 2.0
|
|
*
|
|
*/
|
|
|
|
#define KMSG_COMPONENT "zram"
|
|
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
|
|
|
|
#include <linux/module.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/bio.h>
|
|
#include <linux/bitops.h>
|
|
#include <linux/blkdev.h>
|
|
#include <linux/buffer_head.h>
|
|
#include <linux/device.h>
|
|
#include <linux/genhd.h>
|
|
#include <linux/highmem.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/backing-dev.h>
|
|
#include <linux/string.h>
|
|
#include <linux/vmalloc.h>
|
|
#include <linux/err.h>
|
|
#include <linux/idr.h>
|
|
#include <linux/sysfs.h>
|
|
#include <linux/cpuhotplug.h>
|
|
|
|
#include "zram_drv.h"
|
|
|
|
static DEFINE_IDR(zram_index_idr);
|
|
/* idr index must be protected */
|
|
static DEFINE_MUTEX(zram_index_mutex);
|
|
|
|
static int zram_major;
|
|
static const char *default_compressor = "lzo";
|
|
|
|
/* Module params (documentation at end) */
|
|
static unsigned int num_devices = 1;
|
|
|
|
static inline bool init_done(struct zram *zram)
|
|
{
|
|
return zram->disksize;
|
|
}
|
|
|
|
static inline struct zram *dev_to_zram(struct device *dev)
|
|
{
|
|
return (struct zram *)dev_to_disk(dev)->private_data;
|
|
}
|
|
|
|
/* flag operations require table entry bit_spin_lock() being held */
|
|
static int zram_test_flag(struct zram_meta *meta, u32 index,
|
|
enum zram_pageflags flag)
|
|
{
|
|
return meta->table[index].value & BIT(flag);
|
|
}
|
|
|
|
static void zram_set_flag(struct zram_meta *meta, u32 index,
|
|
enum zram_pageflags flag)
|
|
{
|
|
meta->table[index].value |= BIT(flag);
|
|
}
|
|
|
|
static void zram_clear_flag(struct zram_meta *meta, u32 index,
|
|
enum zram_pageflags flag)
|
|
{
|
|
meta->table[index].value &= ~BIT(flag);
|
|
}
|
|
|
|
static inline void zram_set_element(struct zram_meta *meta, u32 index,
|
|
unsigned long element)
|
|
{
|
|
meta->table[index].element = element;
|
|
}
|
|
|
|
static inline void zram_clear_element(struct zram_meta *meta, u32 index)
|
|
{
|
|
meta->table[index].element = 0;
|
|
}
|
|
|
|
static size_t zram_get_obj_size(struct zram_meta *meta, u32 index)
|
|
{
|
|
return meta->table[index].value & (BIT(ZRAM_FLAG_SHIFT) - 1);
|
|
}
|
|
|
|
static void zram_set_obj_size(struct zram_meta *meta,
|
|
u32 index, size_t size)
|
|
{
|
|
unsigned long flags = meta->table[index].value >> ZRAM_FLAG_SHIFT;
|
|
|
|
meta->table[index].value = (flags << ZRAM_FLAG_SHIFT) | size;
|
|
}
|
|
|
|
static inline bool is_partial_io(struct bio_vec *bvec)
|
|
{
|
|
return bvec->bv_len != PAGE_SIZE;
|
|
}
|
|
|
|
static void zram_revalidate_disk(struct zram *zram)
|
|
{
|
|
revalidate_disk(zram->disk);
|
|
/* revalidate_disk reset the BDI_CAP_STABLE_WRITES so set again */
|
|
zram->disk->queue->backing_dev_info->capabilities |=
|
|
BDI_CAP_STABLE_WRITES;
|
|
}
|
|
|
|
/*
|
|
* Check if request is within bounds and aligned on zram logical blocks.
|
|
*/
|
|
static inline bool valid_io_request(struct zram *zram,
|
|
sector_t start, unsigned int size)
|
|
{
|
|
u64 end, bound;
|
|
|
|
/* unaligned request */
|
|
if (unlikely(start & (ZRAM_SECTOR_PER_LOGICAL_BLOCK - 1)))
|
|
return false;
|
|
if (unlikely(size & (ZRAM_LOGICAL_BLOCK_SIZE - 1)))
|
|
return false;
|
|
|
|
end = start + (size >> SECTOR_SHIFT);
|
|
bound = zram->disksize >> SECTOR_SHIFT;
|
|
/* out of range range */
|
|
if (unlikely(start >= bound || end > bound || start > end))
|
|
return false;
|
|
|
|
/* I/O request is valid */
|
|
return true;
|
|
}
|
|
|
|
static void update_position(u32 *index, int *offset, struct bio_vec *bvec)
|
|
{
|
|
if (*offset + bvec->bv_len >= PAGE_SIZE)
|
|
(*index)++;
|
|
*offset = (*offset + bvec->bv_len) % PAGE_SIZE;
|
|
}
|
|
|
|
static inline void update_used_max(struct zram *zram,
|
|
const unsigned long pages)
|
|
{
|
|
unsigned long old_max, cur_max;
|
|
|
|
old_max = atomic_long_read(&zram->stats.max_used_pages);
|
|
|
|
do {
|
|
cur_max = old_max;
|
|
if (pages > cur_max)
|
|
old_max = atomic_long_cmpxchg(
|
|
&zram->stats.max_used_pages, cur_max, pages);
|
|
} while (old_max != cur_max);
|
|
}
|
|
|
|
static inline void zram_fill_page(char *ptr, unsigned long len,
|
|
unsigned long value)
|
|
{
|
|
int i;
|
|
unsigned long *page = (unsigned long *)ptr;
|
|
|
|
WARN_ON_ONCE(!IS_ALIGNED(len, sizeof(unsigned long)));
|
|
|
|
if (likely(value == 0)) {
|
|
memset(ptr, 0, len);
|
|
} else {
|
|
for (i = 0; i < len / sizeof(*page); i++)
|
|
page[i] = value;
|
|
}
|
|
}
|
|
|
|
static bool page_same_filled(void *ptr, unsigned long *element)
|
|
{
|
|
unsigned int pos;
|
|
unsigned long *page;
|
|
|
|
page = (unsigned long *)ptr;
|
|
|
|
for (pos = 0; pos < PAGE_SIZE / sizeof(*page) - 1; pos++) {
|
|
if (page[pos] != page[pos + 1])
|
|
return false;
|
|
}
|
|
|
|
*element = page[pos];
|
|
|
|
return true;
|
|
}
|
|
|
|
static void handle_same_page(struct bio_vec *bvec, unsigned long element)
|
|
{
|
|
struct page *page = bvec->bv_page;
|
|
void *user_mem;
|
|
|
|
user_mem = kmap_atomic(page);
|
|
zram_fill_page(user_mem + bvec->bv_offset, bvec->bv_len, element);
|
|
kunmap_atomic(user_mem);
|
|
|
|
flush_dcache_page(page);
|
|
}
|
|
|
|
static ssize_t initstate_show(struct device *dev,
|
|
struct device_attribute *attr, char *buf)
|
|
{
|
|
u32 val;
|
|
struct zram *zram = dev_to_zram(dev);
|
|
|
|
down_read(&zram->init_lock);
|
|
val = init_done(zram);
|
|
up_read(&zram->init_lock);
|
|
|
|
return scnprintf(buf, PAGE_SIZE, "%u\n", val);
|
|
}
|
|
|
|
static ssize_t disksize_show(struct device *dev,
|
|
struct device_attribute *attr, char *buf)
|
|
{
|
|
struct zram *zram = dev_to_zram(dev);
|
|
|
|
return scnprintf(buf, PAGE_SIZE, "%llu\n", zram->disksize);
|
|
}
|
|
|
|
static ssize_t mem_limit_store(struct device *dev,
|
|
struct device_attribute *attr, const char *buf, size_t len)
|
|
{
|
|
u64 limit;
|
|
char *tmp;
|
|
struct zram *zram = dev_to_zram(dev);
|
|
|
|
limit = memparse(buf, &tmp);
|
|
if (buf == tmp) /* no chars parsed, invalid input */
|
|
return -EINVAL;
|
|
|
|
down_write(&zram->init_lock);
|
|
zram->limit_pages = PAGE_ALIGN(limit) >> PAGE_SHIFT;
|
|
up_write(&zram->init_lock);
|
|
|
|
return len;
|
|
}
|
|
|
|
static ssize_t mem_used_max_store(struct device *dev,
|
|
struct device_attribute *attr, const char *buf, size_t len)
|
|
{
|
|
int err;
|
|
unsigned long val;
|
|
struct zram *zram = dev_to_zram(dev);
|
|
|
|
err = kstrtoul(buf, 10, &val);
|
|
if (err || val != 0)
|
|
return -EINVAL;
|
|
|
|
down_read(&zram->init_lock);
|
|
if (init_done(zram)) {
|
|
struct zram_meta *meta = zram->meta;
|
|
atomic_long_set(&zram->stats.max_used_pages,
|
|
zs_get_total_pages(meta->mem_pool));
|
|
}
|
|
up_read(&zram->init_lock);
|
|
|
|
return len;
|
|
}
|
|
|
|
/*
|
|
* We switched to per-cpu streams and this attr is not needed anymore.
|
|
* However, we will keep it around for some time, because:
|
|
* a) we may revert per-cpu streams in the future
|
|
* b) it's visible to user space and we need to follow our 2 years
|
|
* retirement rule; but we already have a number of 'soon to be
|
|
* altered' attrs, so max_comp_streams need to wait for the next
|
|
* layoff cycle.
|
|
*/
|
|
static ssize_t max_comp_streams_show(struct device *dev,
|
|
struct device_attribute *attr, char *buf)
|
|
{
|
|
return scnprintf(buf, PAGE_SIZE, "%d\n", num_online_cpus());
|
|
}
|
|
|
|
static ssize_t max_comp_streams_store(struct device *dev,
|
|
struct device_attribute *attr, const char *buf, size_t len)
|
|
{
|
|
return len;
|
|
}
|
|
|
|
static ssize_t comp_algorithm_show(struct device *dev,
|
|
struct device_attribute *attr, char *buf)
|
|
{
|
|
size_t sz;
|
|
struct zram *zram = dev_to_zram(dev);
|
|
|
|
down_read(&zram->init_lock);
|
|
sz = zcomp_available_show(zram->compressor, buf);
|
|
up_read(&zram->init_lock);
|
|
|
|
return sz;
|
|
}
|
|
|
|
static ssize_t comp_algorithm_store(struct device *dev,
|
|
struct device_attribute *attr, const char *buf, size_t len)
|
|
{
|
|
struct zram *zram = dev_to_zram(dev);
|
|
char compressor[CRYPTO_MAX_ALG_NAME];
|
|
size_t sz;
|
|
|
|
strlcpy(compressor, buf, sizeof(compressor));
|
|
/* ignore trailing newline */
|
|
sz = strlen(compressor);
|
|
if (sz > 0 && compressor[sz - 1] == '\n')
|
|
compressor[sz - 1] = 0x00;
|
|
|
|
if (!zcomp_available_algorithm(compressor))
|
|
return -EINVAL;
|
|
|
|
down_write(&zram->init_lock);
|
|
if (init_done(zram)) {
|
|
up_write(&zram->init_lock);
|
|
pr_info("Can't change algorithm for initialized device\n");
|
|
return -EBUSY;
|
|
}
|
|
|
|
strlcpy(zram->compressor, compressor, sizeof(compressor));
|
|
up_write(&zram->init_lock);
|
|
return len;
|
|
}
|
|
|
|
static ssize_t compact_store(struct device *dev,
|
|
struct device_attribute *attr, const char *buf, size_t len)
|
|
{
|
|
struct zram *zram = dev_to_zram(dev);
|
|
struct zram_meta *meta;
|
|
|
|
down_read(&zram->init_lock);
|
|
if (!init_done(zram)) {
|
|
up_read(&zram->init_lock);
|
|
return -EINVAL;
|
|
}
|
|
|
|
meta = zram->meta;
|
|
zs_compact(meta->mem_pool);
|
|
up_read(&zram->init_lock);
|
|
|
|
return len;
|
|
}
|
|
|
|
static ssize_t io_stat_show(struct device *dev,
|
|
struct device_attribute *attr, char *buf)
|
|
{
|
|
struct zram *zram = dev_to_zram(dev);
|
|
ssize_t ret;
|
|
|
|
down_read(&zram->init_lock);
|
|
ret = scnprintf(buf, PAGE_SIZE,
|
|
"%8llu %8llu %8llu %8llu\n",
|
|
(u64)atomic64_read(&zram->stats.failed_reads),
|
|
(u64)atomic64_read(&zram->stats.failed_writes),
|
|
(u64)atomic64_read(&zram->stats.invalid_io),
|
|
(u64)atomic64_read(&zram->stats.notify_free));
|
|
up_read(&zram->init_lock);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static ssize_t mm_stat_show(struct device *dev,
|
|
struct device_attribute *attr, char *buf)
|
|
{
|
|
struct zram *zram = dev_to_zram(dev);
|
|
struct zs_pool_stats pool_stats;
|
|
u64 orig_size, mem_used = 0;
|
|
long max_used;
|
|
ssize_t ret;
|
|
|
|
memset(&pool_stats, 0x00, sizeof(struct zs_pool_stats));
|
|
|
|
down_read(&zram->init_lock);
|
|
if (init_done(zram)) {
|
|
mem_used = zs_get_total_pages(zram->meta->mem_pool);
|
|
zs_pool_stats(zram->meta->mem_pool, &pool_stats);
|
|
}
|
|
|
|
orig_size = atomic64_read(&zram->stats.pages_stored);
|
|
max_used = atomic_long_read(&zram->stats.max_used_pages);
|
|
|
|
ret = scnprintf(buf, PAGE_SIZE,
|
|
"%8llu %8llu %8llu %8lu %8ld %8llu %8lu\n",
|
|
orig_size << PAGE_SHIFT,
|
|
(u64)atomic64_read(&zram->stats.compr_data_size),
|
|
mem_used << PAGE_SHIFT,
|
|
zram->limit_pages << PAGE_SHIFT,
|
|
max_used << PAGE_SHIFT,
|
|
(u64)atomic64_read(&zram->stats.same_pages),
|
|
pool_stats.pages_compacted);
|
|
up_read(&zram->init_lock);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static ssize_t debug_stat_show(struct device *dev,
|
|
struct device_attribute *attr, char *buf)
|
|
{
|
|
int version = 1;
|
|
struct zram *zram = dev_to_zram(dev);
|
|
ssize_t ret;
|
|
|
|
down_read(&zram->init_lock);
|
|
ret = scnprintf(buf, PAGE_SIZE,
|
|
"version: %d\n%8llu\n",
|
|
version,
|
|
(u64)atomic64_read(&zram->stats.writestall));
|
|
up_read(&zram->init_lock);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static DEVICE_ATTR_RO(io_stat);
|
|
static DEVICE_ATTR_RO(mm_stat);
|
|
static DEVICE_ATTR_RO(debug_stat);
|
|
|
|
static void zram_meta_free(struct zram_meta *meta, u64 disksize)
|
|
{
|
|
size_t num_pages = disksize >> PAGE_SHIFT;
|
|
size_t index;
|
|
|
|
/* Free all pages that are still in this zram device */
|
|
for (index = 0; index < num_pages; index++) {
|
|
unsigned long handle = meta->table[index].handle;
|
|
/*
|
|
* No memory is allocated for same element filled pages.
|
|
* Simply clear same page flag.
|
|
*/
|
|
if (!handle || zram_test_flag(meta, index, ZRAM_SAME))
|
|
continue;
|
|
|
|
zs_free(meta->mem_pool, handle);
|
|
}
|
|
|
|
zs_destroy_pool(meta->mem_pool);
|
|
vfree(meta->table);
|
|
kfree(meta);
|
|
}
|
|
|
|
static struct zram_meta *zram_meta_alloc(char *pool_name, u64 disksize)
|
|
{
|
|
size_t num_pages;
|
|
struct zram_meta *meta = kmalloc(sizeof(*meta), GFP_KERNEL);
|
|
|
|
if (!meta)
|
|
return NULL;
|
|
|
|
num_pages = disksize >> PAGE_SHIFT;
|
|
meta->table = vzalloc(num_pages * sizeof(*meta->table));
|
|
if (!meta->table) {
|
|
pr_err("Error allocating zram address table\n");
|
|
goto out_error;
|
|
}
|
|
|
|
meta->mem_pool = zs_create_pool(pool_name);
|
|
if (!meta->mem_pool) {
|
|
pr_err("Error creating memory pool\n");
|
|
goto out_error;
|
|
}
|
|
|
|
return meta;
|
|
|
|
out_error:
|
|
vfree(meta->table);
|
|
kfree(meta);
|
|
return NULL;
|
|
}
|
|
|
|
/*
|
|
* To protect concurrent access to the same index entry,
|
|
* caller should hold this table index entry's bit_spinlock to
|
|
* indicate this index entry is accessing.
|
|
*/
|
|
static void zram_free_page(struct zram *zram, size_t index)
|
|
{
|
|
struct zram_meta *meta = zram->meta;
|
|
unsigned long handle = meta->table[index].handle;
|
|
|
|
/*
|
|
* No memory is allocated for same element filled pages.
|
|
* Simply clear same page flag.
|
|
*/
|
|
if (zram_test_flag(meta, index, ZRAM_SAME)) {
|
|
zram_clear_flag(meta, index, ZRAM_SAME);
|
|
zram_clear_element(meta, index);
|
|
atomic64_dec(&zram->stats.same_pages);
|
|
return;
|
|
}
|
|
|
|
if (!handle)
|
|
return;
|
|
|
|
zs_free(meta->mem_pool, handle);
|
|
|
|
atomic64_sub(zram_get_obj_size(meta, index),
|
|
&zram->stats.compr_data_size);
|
|
atomic64_dec(&zram->stats.pages_stored);
|
|
|
|
meta->table[index].handle = 0;
|
|
zram_set_obj_size(meta, index, 0);
|
|
}
|
|
|
|
static int zram_decompress_page(struct zram *zram, char *mem, u32 index)
|
|
{
|
|
int ret = 0;
|
|
unsigned char *cmem;
|
|
struct zram_meta *meta = zram->meta;
|
|
unsigned long handle;
|
|
unsigned int size;
|
|
|
|
bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
|
|
handle = meta->table[index].handle;
|
|
size = zram_get_obj_size(meta, index);
|
|
|
|
if (!handle || zram_test_flag(meta, index, ZRAM_SAME)) {
|
|
bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
|
|
zram_fill_page(mem, PAGE_SIZE, meta->table[index].element);
|
|
return 0;
|
|
}
|
|
|
|
cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_RO);
|
|
if (size == PAGE_SIZE) {
|
|
memcpy(mem, cmem, PAGE_SIZE);
|
|
} else {
|
|
struct zcomp_strm *zstrm = zcomp_stream_get(zram->comp);
|
|
|
|
ret = zcomp_decompress(zstrm, cmem, size, mem);
|
|
zcomp_stream_put(zram->comp);
|
|
}
|
|
zs_unmap_object(meta->mem_pool, handle);
|
|
bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
|
|
|
|
/* Should NEVER happen. Return bio error if it does. */
|
|
if (unlikely(ret)) {
|
|
pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
|
|
return ret;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
|
|
u32 index, int offset)
|
|
{
|
|
int ret;
|
|
struct page *page;
|
|
unsigned char *user_mem, *uncmem = NULL;
|
|
struct zram_meta *meta = zram->meta;
|
|
page = bvec->bv_page;
|
|
|
|
bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
|
|
if (unlikely(!meta->table[index].handle) ||
|
|
zram_test_flag(meta, index, ZRAM_SAME)) {
|
|
bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
|
|
handle_same_page(bvec, meta->table[index].element);
|
|
return 0;
|
|
}
|
|
bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
|
|
|
|
if (is_partial_io(bvec))
|
|
/* Use a temporary buffer to decompress the page */
|
|
uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
|
|
|
|
user_mem = kmap_atomic(page);
|
|
if (!is_partial_io(bvec))
|
|
uncmem = user_mem;
|
|
|
|
if (!uncmem) {
|
|
pr_err("Unable to allocate temp memory\n");
|
|
ret = -ENOMEM;
|
|
goto out_cleanup;
|
|
}
|
|
|
|
ret = zram_decompress_page(zram, uncmem, index);
|
|
/* Should NEVER happen. Return bio error if it does. */
|
|
if (unlikely(ret))
|
|
goto out_cleanup;
|
|
|
|
if (is_partial_io(bvec))
|
|
memcpy(user_mem + bvec->bv_offset, uncmem + offset,
|
|
bvec->bv_len);
|
|
|
|
flush_dcache_page(page);
|
|
ret = 0;
|
|
out_cleanup:
|
|
kunmap_atomic(user_mem);
|
|
if (is_partial_io(bvec))
|
|
kfree(uncmem);
|
|
return ret;
|
|
}
|
|
|
|
static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
|
|
int offset)
|
|
{
|
|
int ret = 0;
|
|
unsigned int clen;
|
|
unsigned long handle = 0;
|
|
struct page *page;
|
|
unsigned char *user_mem, *cmem, *src, *uncmem = NULL;
|
|
struct zram_meta *meta = zram->meta;
|
|
struct zcomp_strm *zstrm = NULL;
|
|
unsigned long alloced_pages;
|
|
unsigned long element;
|
|
|
|
page = bvec->bv_page;
|
|
if (is_partial_io(bvec)) {
|
|
/*
|
|
* This is a partial IO. We need to read the full page
|
|
* before to write the changes.
|
|
*/
|
|
uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
|
|
if (!uncmem) {
|
|
ret = -ENOMEM;
|
|
goto out;
|
|
}
|
|
ret = zram_decompress_page(zram, uncmem, index);
|
|
if (ret)
|
|
goto out;
|
|
}
|
|
|
|
compress_again:
|
|
user_mem = kmap_atomic(page);
|
|
if (is_partial_io(bvec)) {
|
|
memcpy(uncmem + offset, user_mem + bvec->bv_offset,
|
|
bvec->bv_len);
|
|
kunmap_atomic(user_mem);
|
|
user_mem = NULL;
|
|
} else {
|
|
uncmem = user_mem;
|
|
}
|
|
|
|
if (page_same_filled(uncmem, &element)) {
|
|
if (user_mem)
|
|
kunmap_atomic(user_mem);
|
|
/* Free memory associated with this sector now. */
|
|
bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
|
|
zram_free_page(zram, index);
|
|
zram_set_flag(meta, index, ZRAM_SAME);
|
|
zram_set_element(meta, index, element);
|
|
bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
|
|
|
|
atomic64_inc(&zram->stats.same_pages);
|
|
ret = 0;
|
|
goto out;
|
|
}
|
|
|
|
zstrm = zcomp_stream_get(zram->comp);
|
|
ret = zcomp_compress(zstrm, uncmem, &clen);
|
|
if (!is_partial_io(bvec)) {
|
|
kunmap_atomic(user_mem);
|
|
user_mem = NULL;
|
|
uncmem = NULL;
|
|
}
|
|
|
|
if (unlikely(ret)) {
|
|
pr_err("Compression failed! err=%d\n", ret);
|
|
goto out;
|
|
}
|
|
|
|
src = zstrm->buffer;
|
|
if (unlikely(clen > max_zpage_size)) {
|
|
clen = PAGE_SIZE;
|
|
if (is_partial_io(bvec))
|
|
src = uncmem;
|
|
}
|
|
|
|
/*
|
|
* handle allocation has 2 paths:
|
|
* a) fast path is executed with preemption disabled (for
|
|
* per-cpu streams) and has __GFP_DIRECT_RECLAIM bit clear,
|
|
* since we can't sleep;
|
|
* b) slow path enables preemption and attempts to allocate
|
|
* the page with __GFP_DIRECT_RECLAIM bit set. we have to
|
|
* put per-cpu compression stream and, thus, to re-do
|
|
* the compression once handle is allocated.
|
|
*
|
|
* if we have a 'non-null' handle here then we are coming
|
|
* from the slow path and handle has already been allocated.
|
|
*/
|
|
if (!handle)
|
|
handle = zs_malloc(meta->mem_pool, clen,
|
|
__GFP_KSWAPD_RECLAIM |
|
|
__GFP_NOWARN |
|
|
__GFP_HIGHMEM |
|
|
__GFP_MOVABLE);
|
|
if (!handle) {
|
|
zcomp_stream_put(zram->comp);
|
|
zstrm = NULL;
|
|
|
|
atomic64_inc(&zram->stats.writestall);
|
|
|
|
handle = zs_malloc(meta->mem_pool, clen,
|
|
GFP_NOIO | __GFP_HIGHMEM |
|
|
__GFP_MOVABLE);
|
|
if (handle)
|
|
goto compress_again;
|
|
|
|
pr_err("Error allocating memory for compressed page: %u, size=%u\n",
|
|
index, clen);
|
|
ret = -ENOMEM;
|
|
goto out;
|
|
}
|
|
|
|
alloced_pages = zs_get_total_pages(meta->mem_pool);
|
|
update_used_max(zram, alloced_pages);
|
|
|
|
if (zram->limit_pages && alloced_pages > zram->limit_pages) {
|
|
zs_free(meta->mem_pool, handle);
|
|
ret = -ENOMEM;
|
|
goto out;
|
|
}
|
|
|
|
cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_WO);
|
|
|
|
if ((clen == PAGE_SIZE) && !is_partial_io(bvec)) {
|
|
src = kmap_atomic(page);
|
|
memcpy(cmem, src, PAGE_SIZE);
|
|
kunmap_atomic(src);
|
|
} else {
|
|
memcpy(cmem, src, clen);
|
|
}
|
|
|
|
zcomp_stream_put(zram->comp);
|
|
zstrm = NULL;
|
|
zs_unmap_object(meta->mem_pool, handle);
|
|
|
|
/*
|
|
* Free memory associated with this sector
|
|
* before overwriting unused sectors.
|
|
*/
|
|
bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
|
|
zram_free_page(zram, index);
|
|
|
|
meta->table[index].handle = handle;
|
|
zram_set_obj_size(meta, index, clen);
|
|
bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
|
|
|
|
/* Update stats */
|
|
atomic64_add(clen, &zram->stats.compr_data_size);
|
|
atomic64_inc(&zram->stats.pages_stored);
|
|
out:
|
|
if (zstrm)
|
|
zcomp_stream_put(zram->comp);
|
|
if (is_partial_io(bvec))
|
|
kfree(uncmem);
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* zram_bio_discard - handler on discard request
|
|
* @index: physical block index in PAGE_SIZE units
|
|
* @offset: byte offset within physical block
|
|
*/
|
|
static void zram_bio_discard(struct zram *zram, u32 index,
|
|
int offset, struct bio *bio)
|
|
{
|
|
size_t n = bio->bi_iter.bi_size;
|
|
struct zram_meta *meta = zram->meta;
|
|
|
|
/*
|
|
* zram manages data in physical block size units. Because logical block
|
|
* size isn't identical with physical block size on some arch, we
|
|
* could get a discard request pointing to a specific offset within a
|
|
* certain physical block. Although we can handle this request by
|
|
* reading that physiclal block and decompressing and partially zeroing
|
|
* and re-compressing and then re-storing it, this isn't reasonable
|
|
* because our intent with a discard request is to save memory. So
|
|
* skipping this logical block is appropriate here.
|
|
*/
|
|
if (offset) {
|
|
if (n <= (PAGE_SIZE - offset))
|
|
return;
|
|
|
|
n -= (PAGE_SIZE - offset);
|
|
index++;
|
|
}
|
|
|
|
while (n >= PAGE_SIZE) {
|
|
bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
|
|
zram_free_page(zram, index);
|
|
bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
|
|
atomic64_inc(&zram->stats.notify_free);
|
|
index++;
|
|
n -= PAGE_SIZE;
|
|
}
|
|
}
|
|
|
|
static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,
|
|
int offset, bool is_write)
|
|
{
|
|
unsigned long start_time = jiffies;
|
|
int rw_acct = is_write ? REQ_OP_WRITE : REQ_OP_READ;
|
|
int ret;
|
|
|
|
generic_start_io_acct(rw_acct, bvec->bv_len >> SECTOR_SHIFT,
|
|
&zram->disk->part0);
|
|
|
|
if (!is_write) {
|
|
atomic64_inc(&zram->stats.num_reads);
|
|
ret = zram_bvec_read(zram, bvec, index, offset);
|
|
} else {
|
|
atomic64_inc(&zram->stats.num_writes);
|
|
ret = zram_bvec_write(zram, bvec, index, offset);
|
|
}
|
|
|
|
generic_end_io_acct(rw_acct, &zram->disk->part0, start_time);
|
|
|
|
if (unlikely(ret)) {
|
|
if (!is_write)
|
|
atomic64_inc(&zram->stats.failed_reads);
|
|
else
|
|
atomic64_inc(&zram->stats.failed_writes);
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
static void __zram_make_request(struct zram *zram, struct bio *bio)
|
|
{
|
|
int offset;
|
|
u32 index;
|
|
struct bio_vec bvec;
|
|
struct bvec_iter iter;
|
|
|
|
index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
|
|
offset = (bio->bi_iter.bi_sector &
|
|
(SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT;
|
|
|
|
if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) {
|
|
zram_bio_discard(zram, index, offset, bio);
|
|
bio_endio(bio);
|
|
return;
|
|
}
|
|
|
|
bio_for_each_segment(bvec, bio, iter) {
|
|
int max_transfer_size = PAGE_SIZE - offset;
|
|
|
|
if (bvec.bv_len > max_transfer_size) {
|
|
/*
|
|
* zram_bvec_rw() can only make operation on a single
|
|
* zram page. Split the bio vector.
|
|
*/
|
|
struct bio_vec bv;
|
|
|
|
bv.bv_page = bvec.bv_page;
|
|
bv.bv_len = max_transfer_size;
|
|
bv.bv_offset = bvec.bv_offset;
|
|
|
|
if (zram_bvec_rw(zram, &bv, index, offset,
|
|
op_is_write(bio_op(bio))) < 0)
|
|
goto out;
|
|
|
|
bv.bv_len = bvec.bv_len - max_transfer_size;
|
|
bv.bv_offset += max_transfer_size;
|
|
if (zram_bvec_rw(zram, &bv, index + 1, 0,
|
|
op_is_write(bio_op(bio))) < 0)
|
|
goto out;
|
|
} else
|
|
if (zram_bvec_rw(zram, &bvec, index, offset,
|
|
op_is_write(bio_op(bio))) < 0)
|
|
goto out;
|
|
|
|
update_position(&index, &offset, &bvec);
|
|
}
|
|
|
|
bio_endio(bio);
|
|
return;
|
|
|
|
out:
|
|
bio_io_error(bio);
|
|
}
|
|
|
|
/*
|
|
* Handler function for all zram I/O requests.
|
|
*/
|
|
static blk_qc_t zram_make_request(struct request_queue *queue, struct bio *bio)
|
|
{
|
|
struct zram *zram = queue->queuedata;
|
|
|
|
blk_queue_split(queue, &bio, queue->bio_split);
|
|
|
|
if (!valid_io_request(zram, bio->bi_iter.bi_sector,
|
|
bio->bi_iter.bi_size)) {
|
|
atomic64_inc(&zram->stats.invalid_io);
|
|
goto error;
|
|
}
|
|
|
|
__zram_make_request(zram, bio);
|
|
return BLK_QC_T_NONE;
|
|
|
|
error:
|
|
bio_io_error(bio);
|
|
return BLK_QC_T_NONE;
|
|
}
|
|
|
|
static void zram_slot_free_notify(struct block_device *bdev,
|
|
unsigned long index)
|
|
{
|
|
struct zram *zram;
|
|
struct zram_meta *meta;
|
|
|
|
zram = bdev->bd_disk->private_data;
|
|
meta = zram->meta;
|
|
|
|
bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
|
|
zram_free_page(zram, index);
|
|
bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
|
|
atomic64_inc(&zram->stats.notify_free);
|
|
}
|
|
|
|
static int zram_rw_page(struct block_device *bdev, sector_t sector,
|
|
struct page *page, bool is_write)
|
|
{
|
|
int offset, err = -EIO;
|
|
u32 index;
|
|
struct zram *zram;
|
|
struct bio_vec bv;
|
|
|
|
zram = bdev->bd_disk->private_data;
|
|
|
|
if (!valid_io_request(zram, sector, PAGE_SIZE)) {
|
|
atomic64_inc(&zram->stats.invalid_io);
|
|
err = -EINVAL;
|
|
goto out;
|
|
}
|
|
|
|
index = sector >> SECTORS_PER_PAGE_SHIFT;
|
|
offset = (sector & (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT;
|
|
|
|
bv.bv_page = page;
|
|
bv.bv_len = PAGE_SIZE;
|
|
bv.bv_offset = 0;
|
|
|
|
err = zram_bvec_rw(zram, &bv, index, offset, is_write);
|
|
out:
|
|
/*
|
|
* If I/O fails, just return error(ie, non-zero) without
|
|
* calling page_endio.
|
|
* It causes resubmit the I/O with bio request by upper functions
|
|
* of rw_page(e.g., swap_readpage, __swap_writepage) and
|
|
* bio->bi_end_io does things to handle the error
|
|
* (e.g., SetPageError, set_page_dirty and extra works).
|
|
*/
|
|
if (err == 0)
|
|
page_endio(page, is_write, 0);
|
|
return err;
|
|
}
|
|
|
|
static void zram_reset_device(struct zram *zram)
|
|
{
|
|
struct zram_meta *meta;
|
|
struct zcomp *comp;
|
|
u64 disksize;
|
|
|
|
down_write(&zram->init_lock);
|
|
|
|
zram->limit_pages = 0;
|
|
|
|
if (!init_done(zram)) {
|
|
up_write(&zram->init_lock);
|
|
return;
|
|
}
|
|
|
|
meta = zram->meta;
|
|
comp = zram->comp;
|
|
disksize = zram->disksize;
|
|
|
|
/* Reset stats */
|
|
memset(&zram->stats, 0, sizeof(zram->stats));
|
|
zram->disksize = 0;
|
|
|
|
set_capacity(zram->disk, 0);
|
|
part_stat_set_all(&zram->disk->part0, 0);
|
|
|
|
up_write(&zram->init_lock);
|
|
/* I/O operation under all of CPU are done so let's free */
|
|
zram_meta_free(meta, disksize);
|
|
zcomp_destroy(comp);
|
|
}
|
|
|
|
static ssize_t disksize_store(struct device *dev,
|
|
struct device_attribute *attr, const char *buf, size_t len)
|
|
{
|
|
u64 disksize;
|
|
struct zcomp *comp;
|
|
struct zram_meta *meta;
|
|
struct zram *zram = dev_to_zram(dev);
|
|
int err;
|
|
|
|
disksize = memparse(buf, NULL);
|
|
if (!disksize)
|
|
return -EINVAL;
|
|
|
|
disksize = PAGE_ALIGN(disksize);
|
|
meta = zram_meta_alloc(zram->disk->disk_name, disksize);
|
|
if (!meta)
|
|
return -ENOMEM;
|
|
|
|
comp = zcomp_create(zram->compressor);
|
|
if (IS_ERR(comp)) {
|
|
pr_err("Cannot initialise %s compressing backend\n",
|
|
zram->compressor);
|
|
err = PTR_ERR(comp);
|
|
goto out_free_meta;
|
|
}
|
|
|
|
down_write(&zram->init_lock);
|
|
if (init_done(zram)) {
|
|
pr_info("Cannot change disksize for initialized device\n");
|
|
err = -EBUSY;
|
|
goto out_destroy_comp;
|
|
}
|
|
|
|
zram->meta = meta;
|
|
zram->comp = comp;
|
|
zram->disksize = disksize;
|
|
set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
|
|
zram_revalidate_disk(zram);
|
|
up_write(&zram->init_lock);
|
|
|
|
return len;
|
|
|
|
out_destroy_comp:
|
|
up_write(&zram->init_lock);
|
|
zcomp_destroy(comp);
|
|
out_free_meta:
|
|
zram_meta_free(meta, disksize);
|
|
return err;
|
|
}
|
|
|
|
static ssize_t reset_store(struct device *dev,
|
|
struct device_attribute *attr, const char *buf, size_t len)
|
|
{
|
|
int ret;
|
|
unsigned short do_reset;
|
|
struct zram *zram;
|
|
struct block_device *bdev;
|
|
|
|
ret = kstrtou16(buf, 10, &do_reset);
|
|
if (ret)
|
|
return ret;
|
|
|
|
if (!do_reset)
|
|
return -EINVAL;
|
|
|
|
zram = dev_to_zram(dev);
|
|
bdev = bdget_disk(zram->disk, 0);
|
|
if (!bdev)
|
|
return -ENOMEM;
|
|
|
|
mutex_lock(&bdev->bd_mutex);
|
|
/* Do not reset an active device or claimed device */
|
|
if (bdev->bd_openers || zram->claim) {
|
|
mutex_unlock(&bdev->bd_mutex);
|
|
bdput(bdev);
|
|
return -EBUSY;
|
|
}
|
|
|
|
/* From now on, anyone can't open /dev/zram[0-9] */
|
|
zram->claim = true;
|
|
mutex_unlock(&bdev->bd_mutex);
|
|
|
|
/* Make sure all the pending I/O are finished */
|
|
fsync_bdev(bdev);
|
|
zram_reset_device(zram);
|
|
zram_revalidate_disk(zram);
|
|
bdput(bdev);
|
|
|
|
mutex_lock(&bdev->bd_mutex);
|
|
zram->claim = false;
|
|
mutex_unlock(&bdev->bd_mutex);
|
|
|
|
return len;
|
|
}
|
|
|
|
static int zram_open(struct block_device *bdev, fmode_t mode)
|
|
{
|
|
int ret = 0;
|
|
struct zram *zram;
|
|
|
|
WARN_ON(!mutex_is_locked(&bdev->bd_mutex));
|
|
|
|
zram = bdev->bd_disk->private_data;
|
|
/* zram was claimed to reset so open request fails */
|
|
if (zram->claim)
|
|
ret = -EBUSY;
|
|
|
|
return ret;
|
|
}
|
|
|
|
static const struct block_device_operations zram_devops = {
|
|
.open = zram_open,
|
|
.swap_slot_free_notify = zram_slot_free_notify,
|
|
.rw_page = zram_rw_page,
|
|
.owner = THIS_MODULE
|
|
};
|
|
|
|
static DEVICE_ATTR_WO(compact);
|
|
static DEVICE_ATTR_RW(disksize);
|
|
static DEVICE_ATTR_RO(initstate);
|
|
static DEVICE_ATTR_WO(reset);
|
|
static DEVICE_ATTR_WO(mem_limit);
|
|
static DEVICE_ATTR_WO(mem_used_max);
|
|
static DEVICE_ATTR_RW(max_comp_streams);
|
|
static DEVICE_ATTR_RW(comp_algorithm);
|
|
|
|
static struct attribute *zram_disk_attrs[] = {
|
|
&dev_attr_disksize.attr,
|
|
&dev_attr_initstate.attr,
|
|
&dev_attr_reset.attr,
|
|
&dev_attr_compact.attr,
|
|
&dev_attr_mem_limit.attr,
|
|
&dev_attr_mem_used_max.attr,
|
|
&dev_attr_max_comp_streams.attr,
|
|
&dev_attr_comp_algorithm.attr,
|
|
&dev_attr_io_stat.attr,
|
|
&dev_attr_mm_stat.attr,
|
|
&dev_attr_debug_stat.attr,
|
|
NULL,
|
|
};
|
|
|
|
static struct attribute_group zram_disk_attr_group = {
|
|
.attrs = zram_disk_attrs,
|
|
};
|
|
|
|
/*
|
|
* Allocate and initialize new zram device. the function returns
|
|
* '>= 0' device_id upon success, and negative value otherwise.
|
|
*/
|
|
static int zram_add(void)
|
|
{
|
|
struct zram *zram;
|
|
struct request_queue *queue;
|
|
int ret, device_id;
|
|
|
|
zram = kzalloc(sizeof(struct zram), GFP_KERNEL);
|
|
if (!zram)
|
|
return -ENOMEM;
|
|
|
|
ret = idr_alloc(&zram_index_idr, zram, 0, 0, GFP_KERNEL);
|
|
if (ret < 0)
|
|
goto out_free_dev;
|
|
device_id = ret;
|
|
|
|
init_rwsem(&zram->init_lock);
|
|
|
|
queue = blk_alloc_queue(GFP_KERNEL);
|
|
if (!queue) {
|
|
pr_err("Error allocating disk queue for device %d\n",
|
|
device_id);
|
|
ret = -ENOMEM;
|
|
goto out_free_idr;
|
|
}
|
|
|
|
blk_queue_make_request(queue, zram_make_request);
|
|
|
|
/* gendisk structure */
|
|
zram->disk = alloc_disk(1);
|
|
if (!zram->disk) {
|
|
pr_err("Error allocating disk structure for device %d\n",
|
|
device_id);
|
|
ret = -ENOMEM;
|
|
goto out_free_queue;
|
|
}
|
|
|
|
zram->disk->major = zram_major;
|
|
zram->disk->first_minor = device_id;
|
|
zram->disk->fops = &zram_devops;
|
|
zram->disk->queue = queue;
|
|
zram->disk->queue->queuedata = zram;
|
|
zram->disk->private_data = zram;
|
|
snprintf(zram->disk->disk_name, 16, "zram%d", device_id);
|
|
|
|
/* Actual capacity set using syfs (/sys/block/zram<id>/disksize */
|
|
set_capacity(zram->disk, 0);
|
|
/* zram devices sort of resembles non-rotational disks */
|
|
queue_flag_set_unlocked(QUEUE_FLAG_NONROT, zram->disk->queue);
|
|
queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, zram->disk->queue);
|
|
/*
|
|
* To ensure that we always get PAGE_SIZE aligned
|
|
* and n*PAGE_SIZED sized I/O requests.
|
|
*/
|
|
blk_queue_physical_block_size(zram->disk->queue, PAGE_SIZE);
|
|
blk_queue_logical_block_size(zram->disk->queue,
|
|
ZRAM_LOGICAL_BLOCK_SIZE);
|
|
blk_queue_io_min(zram->disk->queue, PAGE_SIZE);
|
|
blk_queue_io_opt(zram->disk->queue, PAGE_SIZE);
|
|
zram->disk->queue->limits.discard_granularity = PAGE_SIZE;
|
|
zram->disk->queue->limits.max_sectors = SECTORS_PER_PAGE;
|
|
zram->disk->queue->limits.chunk_sectors = 0;
|
|
blk_queue_max_discard_sectors(zram->disk->queue, UINT_MAX);
|
|
/*
|
|
* zram_bio_discard() will clear all logical blocks if logical block
|
|
* size is identical with physical block size(PAGE_SIZE). But if it is
|
|
* different, we will skip discarding some parts of logical blocks in
|
|
* the part of the request range which isn't aligned to physical block
|
|
* size. So we can't ensure that all discarded logical blocks are
|
|
* zeroed.
|
|
*/
|
|
if (ZRAM_LOGICAL_BLOCK_SIZE == PAGE_SIZE)
|
|
zram->disk->queue->limits.discard_zeroes_data = 1;
|
|
else
|
|
zram->disk->queue->limits.discard_zeroes_data = 0;
|
|
queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, zram->disk->queue);
|
|
|
|
add_disk(zram->disk);
|
|
|
|
ret = sysfs_create_group(&disk_to_dev(zram->disk)->kobj,
|
|
&zram_disk_attr_group);
|
|
if (ret < 0) {
|
|
pr_err("Error creating sysfs group for device %d\n",
|
|
device_id);
|
|
goto out_free_disk;
|
|
}
|
|
strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor));
|
|
zram->meta = NULL;
|
|
|
|
pr_info("Added device: %s\n", zram->disk->disk_name);
|
|
return device_id;
|
|
|
|
out_free_disk:
|
|
del_gendisk(zram->disk);
|
|
put_disk(zram->disk);
|
|
out_free_queue:
|
|
blk_cleanup_queue(queue);
|
|
out_free_idr:
|
|
idr_remove(&zram_index_idr, device_id);
|
|
out_free_dev:
|
|
kfree(zram);
|
|
return ret;
|
|
}
|
|
|
|
static int zram_remove(struct zram *zram)
|
|
{
|
|
struct block_device *bdev;
|
|
|
|
bdev = bdget_disk(zram->disk, 0);
|
|
if (!bdev)
|
|
return -ENOMEM;
|
|
|
|
mutex_lock(&bdev->bd_mutex);
|
|
if (bdev->bd_openers || zram->claim) {
|
|
mutex_unlock(&bdev->bd_mutex);
|
|
bdput(bdev);
|
|
return -EBUSY;
|
|
}
|
|
|
|
zram->claim = true;
|
|
mutex_unlock(&bdev->bd_mutex);
|
|
|
|
/*
|
|
* Remove sysfs first, so no one will perform a disksize
|
|
* store while we destroy the devices. This also helps during
|
|
* hot_remove -- zram_reset_device() is the last holder of
|
|
* ->init_lock, no later/concurrent disksize_store() or any
|
|
* other sysfs handlers are possible.
|
|
*/
|
|
sysfs_remove_group(&disk_to_dev(zram->disk)->kobj,
|
|
&zram_disk_attr_group);
|
|
|
|
/* Make sure all the pending I/O are finished */
|
|
fsync_bdev(bdev);
|
|
zram_reset_device(zram);
|
|
bdput(bdev);
|
|
|
|
pr_info("Removed device: %s\n", zram->disk->disk_name);
|
|
|
|
blk_cleanup_queue(zram->disk->queue);
|
|
del_gendisk(zram->disk);
|
|
put_disk(zram->disk);
|
|
kfree(zram);
|
|
return 0;
|
|
}
|
|
|
|
/* zram-control sysfs attributes */
|
|
static ssize_t hot_add_show(struct class *class,
|
|
struct class_attribute *attr,
|
|
char *buf)
|
|
{
|
|
int ret;
|
|
|
|
mutex_lock(&zram_index_mutex);
|
|
ret = zram_add();
|
|
mutex_unlock(&zram_index_mutex);
|
|
|
|
if (ret < 0)
|
|
return ret;
|
|
return scnprintf(buf, PAGE_SIZE, "%d\n", ret);
|
|
}
|
|
|
|
static ssize_t hot_remove_store(struct class *class,
|
|
struct class_attribute *attr,
|
|
const char *buf,
|
|
size_t count)
|
|
{
|
|
struct zram *zram;
|
|
int ret, dev_id;
|
|
|
|
/* dev_id is gendisk->first_minor, which is `int' */
|
|
ret = kstrtoint(buf, 10, &dev_id);
|
|
if (ret)
|
|
return ret;
|
|
if (dev_id < 0)
|
|
return -EINVAL;
|
|
|
|
mutex_lock(&zram_index_mutex);
|
|
|
|
zram = idr_find(&zram_index_idr, dev_id);
|
|
if (zram) {
|
|
ret = zram_remove(zram);
|
|
if (!ret)
|
|
idr_remove(&zram_index_idr, dev_id);
|
|
} else {
|
|
ret = -ENODEV;
|
|
}
|
|
|
|
mutex_unlock(&zram_index_mutex);
|
|
return ret ? ret : count;
|
|
}
|
|
|
|
/*
|
|
* NOTE: hot_add attribute is not the usual read-only sysfs attribute. In a
|
|
* sense that reading from this file does alter the state of your system -- it
|
|
* creates a new un-initialized zram device and returns back this device's
|
|
* device_id (or an error code if it fails to create a new device).
|
|
*/
|
|
static struct class_attribute zram_control_class_attrs[] = {
|
|
__ATTR(hot_add, 0400, hot_add_show, NULL),
|
|
__ATTR_WO(hot_remove),
|
|
__ATTR_NULL,
|
|
};
|
|
|
|
static struct class zram_control_class = {
|
|
.name = "zram-control",
|
|
.owner = THIS_MODULE,
|
|
.class_attrs = zram_control_class_attrs,
|
|
};
|
|
|
|
static int zram_remove_cb(int id, void *ptr, void *data)
|
|
{
|
|
zram_remove(ptr);
|
|
return 0;
|
|
}
|
|
|
|
static void destroy_devices(void)
|
|
{
|
|
class_unregister(&zram_control_class);
|
|
idr_for_each(&zram_index_idr, &zram_remove_cb, NULL);
|
|
idr_destroy(&zram_index_idr);
|
|
unregister_blkdev(zram_major, "zram");
|
|
cpuhp_remove_multi_state(CPUHP_ZCOMP_PREPARE);
|
|
}
|
|
|
|
static int __init zram_init(void)
|
|
{
|
|
int ret;
|
|
|
|
ret = cpuhp_setup_state_multi(CPUHP_ZCOMP_PREPARE, "block/zram:prepare",
|
|
zcomp_cpu_up_prepare, zcomp_cpu_dead);
|
|
if (ret < 0)
|
|
return ret;
|
|
|
|
ret = class_register(&zram_control_class);
|
|
if (ret) {
|
|
pr_err("Unable to register zram-control class\n");
|
|
cpuhp_remove_multi_state(CPUHP_ZCOMP_PREPARE);
|
|
return ret;
|
|
}
|
|
|
|
zram_major = register_blkdev(0, "zram");
|
|
if (zram_major <= 0) {
|
|
pr_err("Unable to get major number\n");
|
|
class_unregister(&zram_control_class);
|
|
cpuhp_remove_multi_state(CPUHP_ZCOMP_PREPARE);
|
|
return -EBUSY;
|
|
}
|
|
|
|
while (num_devices != 0) {
|
|
mutex_lock(&zram_index_mutex);
|
|
ret = zram_add();
|
|
mutex_unlock(&zram_index_mutex);
|
|
if (ret < 0)
|
|
goto out_error;
|
|
num_devices--;
|
|
}
|
|
|
|
return 0;
|
|
|
|
out_error:
|
|
destroy_devices();
|
|
return ret;
|
|
}
|
|
|
|
static void __exit zram_exit(void)
|
|
{
|
|
destroy_devices();
|
|
}
|
|
|
|
module_init(zram_init);
|
|
module_exit(zram_exit);
|
|
|
|
module_param(num_devices, uint, 0);
|
|
MODULE_PARM_DESC(num_devices, "Number of pre-created zram devices");
|
|
|
|
MODULE_LICENSE("Dual BSD/GPL");
|
|
MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>");
|
|
MODULE_DESCRIPTION("Compressed RAM Block Device");
|