mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-27 19:29:37 +00:00
![Dan Magenheimer](/assets/img/avatar_default.png)
Tmem, as originally specified, assumes that "get" operations performed on persistent pools never flush the page of data out of tmem on a successful get, waiting instead for a flush operation. This is intended to mimic the model of a swap disk, where a disk read is non-destructive. Unlike a disk, however, freeing up the RAM can be valuable. Over the years that frontswap was in the review process, several reviewers (and notably Hugh Dickins in 2010) pointed out that this would result, at least temporarily, in two copies of the data in RAM: one (compressed for zcache) copy in tmem, and one copy in the swap cache. We wondered if this could be done differently, at least optionally. This patch allows tmem backends to instruct the frontswap code that this backend performs exclusive gets. Zcache2 already contains hooks to support this feature. Other backends are completely unaffected unless/until they are updated to support this feature. While it is not clear that exclusive gets are a performance win on all workloads at all times, this small patch allows for experimentation by backends. P.S. Let's not quibble about the naming of "get" vs "read" vs "load" etc. The naming is currently horribly inconsistent between cleancache and frontswap and existing tmem backends, so will need to be straightened out as a separate patch. "Get" is used by the tmem architecture spec, existing backends, and all documentation and presentation material so I am using it in this patch. Signed-off-by: Dan Magenheimer <dan.magenheimer@oracle.com> Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
370 lines
10 KiB
C
370 lines
10 KiB
C
/*
|
|
* Frontswap frontend
|
|
*
|
|
* This code provides the generic "frontend" layer to call a matching
|
|
* "backend" driver implementation of frontswap. See
|
|
* Documentation/vm/frontswap.txt for more information.
|
|
*
|
|
* Copyright (C) 2009-2012 Oracle Corp. All rights reserved.
|
|
* Author: Dan Magenheimer
|
|
*
|
|
* This work is licensed under the terms of the GNU GPL, version 2.
|
|
*/
|
|
|
|
#include <linux/mman.h>
|
|
#include <linux/swap.h>
|
|
#include <linux/swapops.h>
|
|
#include <linux/security.h>
|
|
#include <linux/module.h>
|
|
#include <linux/debugfs.h>
|
|
#include <linux/frontswap.h>
|
|
#include <linux/swapfile.h>
|
|
|
|
/*
|
|
* frontswap_ops is set by frontswap_register_ops to contain the pointers
|
|
* to the frontswap "backend" implementation functions.
|
|
*/
|
|
static struct frontswap_ops frontswap_ops __read_mostly;
|
|
|
|
/*
|
|
* This global enablement flag reduces overhead on systems where frontswap_ops
|
|
* has not been registered, so is preferred to the slower alternative: a
|
|
* function call that checks a non-global.
|
|
*/
|
|
bool frontswap_enabled __read_mostly;
|
|
EXPORT_SYMBOL(frontswap_enabled);
|
|
|
|
/*
|
|
* If enabled, frontswap_store will return failure even on success. As
|
|
* a result, the swap subsystem will always write the page to swap, in
|
|
* effect converting frontswap into a writethrough cache. In this mode,
|
|
* there is no direct reduction in swap writes, but a frontswap backend
|
|
* can unilaterally "reclaim" any pages in use with no data loss, thus
|
|
* providing increases control over maximum memory usage due to frontswap.
|
|
*/
|
|
static bool frontswap_writethrough_enabled __read_mostly;
|
|
|
|
/*
|
|
* If enabled, the underlying tmem implementation is capable of doing
|
|
* exclusive gets, so frontswap_load, on a successful tmem_get must
|
|
* mark the page as no longer in frontswap AND mark it dirty.
|
|
*/
|
|
static bool frontswap_tmem_exclusive_gets_enabled __read_mostly;
|
|
|
|
#ifdef CONFIG_DEBUG_FS
|
|
/*
|
|
* Counters available via /sys/kernel/debug/frontswap (if debugfs is
|
|
* properly configured). These are for information only so are not protected
|
|
* against increment races.
|
|
*/
|
|
static u64 frontswap_loads;
|
|
static u64 frontswap_succ_stores;
|
|
static u64 frontswap_failed_stores;
|
|
static u64 frontswap_invalidates;
|
|
|
|
static inline void inc_frontswap_loads(void) {
|
|
frontswap_loads++;
|
|
}
|
|
static inline void inc_frontswap_succ_stores(void) {
|
|
frontswap_succ_stores++;
|
|
}
|
|
static inline void inc_frontswap_failed_stores(void) {
|
|
frontswap_failed_stores++;
|
|
}
|
|
static inline void inc_frontswap_invalidates(void) {
|
|
frontswap_invalidates++;
|
|
}
|
|
#else
|
|
static inline void inc_frontswap_loads(void) { }
|
|
static inline void inc_frontswap_succ_stores(void) { }
|
|
static inline void inc_frontswap_failed_stores(void) { }
|
|
static inline void inc_frontswap_invalidates(void) { }
|
|
#endif
|
|
/*
|
|
* Register operations for frontswap, returning previous thus allowing
|
|
* detection of multiple backends and possible nesting.
|
|
*/
|
|
struct frontswap_ops frontswap_register_ops(struct frontswap_ops *ops)
|
|
{
|
|
struct frontswap_ops old = frontswap_ops;
|
|
|
|
frontswap_ops = *ops;
|
|
frontswap_enabled = true;
|
|
return old;
|
|
}
|
|
EXPORT_SYMBOL(frontswap_register_ops);
|
|
|
|
/*
|
|
* Enable/disable frontswap writethrough (see above).
|
|
*/
|
|
void frontswap_writethrough(bool enable)
|
|
{
|
|
frontswap_writethrough_enabled = enable;
|
|
}
|
|
EXPORT_SYMBOL(frontswap_writethrough);
|
|
|
|
/*
|
|
* Enable/disable frontswap exclusive gets (see above).
|
|
*/
|
|
void frontswap_tmem_exclusive_gets(bool enable)
|
|
{
|
|
frontswap_tmem_exclusive_gets_enabled = enable;
|
|
}
|
|
EXPORT_SYMBOL(frontswap_tmem_exclusive_gets);
|
|
|
|
/*
|
|
* Called when a swap device is swapon'd.
|
|
*/
|
|
void __frontswap_init(unsigned type)
|
|
{
|
|
struct swap_info_struct *sis = swap_info[type];
|
|
|
|
BUG_ON(sis == NULL);
|
|
if (sis->frontswap_map == NULL)
|
|
return;
|
|
frontswap_ops.init(type);
|
|
}
|
|
EXPORT_SYMBOL(__frontswap_init);
|
|
|
|
static inline void __frontswap_clear(struct swap_info_struct *sis, pgoff_t offset)
|
|
{
|
|
frontswap_clear(sis, offset);
|
|
atomic_dec(&sis->frontswap_pages);
|
|
}
|
|
|
|
/*
|
|
* "Store" data from a page to frontswap and associate it with the page's
|
|
* swaptype and offset. Page must be locked and in the swap cache.
|
|
* If frontswap already contains a page with matching swaptype and
|
|
* offset, the frontswap implementation may either overwrite the data and
|
|
* return success or invalidate the page from frontswap and return failure.
|
|
*/
|
|
int __frontswap_store(struct page *page)
|
|
{
|
|
int ret = -1, dup = 0;
|
|
swp_entry_t entry = { .val = page_private(page), };
|
|
int type = swp_type(entry);
|
|
struct swap_info_struct *sis = swap_info[type];
|
|
pgoff_t offset = swp_offset(entry);
|
|
|
|
BUG_ON(!PageLocked(page));
|
|
BUG_ON(sis == NULL);
|
|
if (frontswap_test(sis, offset))
|
|
dup = 1;
|
|
ret = frontswap_ops.store(type, offset, page);
|
|
if (ret == 0) {
|
|
frontswap_set(sis, offset);
|
|
inc_frontswap_succ_stores();
|
|
if (!dup)
|
|
atomic_inc(&sis->frontswap_pages);
|
|
} else {
|
|
/*
|
|
failed dup always results in automatic invalidate of
|
|
the (older) page from frontswap
|
|
*/
|
|
inc_frontswap_failed_stores();
|
|
if (dup)
|
|
__frontswap_clear(sis, offset);
|
|
}
|
|
if (frontswap_writethrough_enabled)
|
|
/* report failure so swap also writes to swap device */
|
|
ret = -1;
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL(__frontswap_store);
|
|
|
|
/*
|
|
* "Get" data from frontswap associated with swaptype and offset that were
|
|
* specified when the data was put to frontswap and use it to fill the
|
|
* specified page with data. Page must be locked and in the swap cache.
|
|
*/
|
|
int __frontswap_load(struct page *page)
|
|
{
|
|
int ret = -1;
|
|
swp_entry_t entry = { .val = page_private(page), };
|
|
int type = swp_type(entry);
|
|
struct swap_info_struct *sis = swap_info[type];
|
|
pgoff_t offset = swp_offset(entry);
|
|
|
|
BUG_ON(!PageLocked(page));
|
|
BUG_ON(sis == NULL);
|
|
if (frontswap_test(sis, offset))
|
|
ret = frontswap_ops.load(type, offset, page);
|
|
if (ret == 0) {
|
|
inc_frontswap_loads();
|
|
if (frontswap_tmem_exclusive_gets_enabled) {
|
|
SetPageDirty(page);
|
|
frontswap_clear(sis, offset);
|
|
}
|
|
}
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL(__frontswap_load);
|
|
|
|
/*
|
|
* Invalidate any data from frontswap associated with the specified swaptype
|
|
* and offset so that a subsequent "get" will fail.
|
|
*/
|
|
void __frontswap_invalidate_page(unsigned type, pgoff_t offset)
|
|
{
|
|
struct swap_info_struct *sis = swap_info[type];
|
|
|
|
BUG_ON(sis == NULL);
|
|
if (frontswap_test(sis, offset)) {
|
|
frontswap_ops.invalidate_page(type, offset);
|
|
__frontswap_clear(sis, offset);
|
|
inc_frontswap_invalidates();
|
|
}
|
|
}
|
|
EXPORT_SYMBOL(__frontswap_invalidate_page);
|
|
|
|
/*
|
|
* Invalidate all data from frontswap associated with all offsets for the
|
|
* specified swaptype.
|
|
*/
|
|
void __frontswap_invalidate_area(unsigned type)
|
|
{
|
|
struct swap_info_struct *sis = swap_info[type];
|
|
|
|
BUG_ON(sis == NULL);
|
|
if (sis->frontswap_map == NULL)
|
|
return;
|
|
frontswap_ops.invalidate_area(type);
|
|
atomic_set(&sis->frontswap_pages, 0);
|
|
memset(sis->frontswap_map, 0, sis->max / sizeof(long));
|
|
}
|
|
EXPORT_SYMBOL(__frontswap_invalidate_area);
|
|
|
|
static unsigned long __frontswap_curr_pages(void)
|
|
{
|
|
int type;
|
|
unsigned long totalpages = 0;
|
|
struct swap_info_struct *si = NULL;
|
|
|
|
assert_spin_locked(&swap_lock);
|
|
for (type = swap_list.head; type >= 0; type = si->next) {
|
|
si = swap_info[type];
|
|
totalpages += atomic_read(&si->frontswap_pages);
|
|
}
|
|
return totalpages;
|
|
}
|
|
|
|
static int __frontswap_unuse_pages(unsigned long total, unsigned long *unused,
|
|
int *swapid)
|
|
{
|
|
int ret = -EINVAL;
|
|
struct swap_info_struct *si = NULL;
|
|
int si_frontswap_pages;
|
|
unsigned long total_pages_to_unuse = total;
|
|
unsigned long pages = 0, pages_to_unuse = 0;
|
|
int type;
|
|
|
|
assert_spin_locked(&swap_lock);
|
|
for (type = swap_list.head; type >= 0; type = si->next) {
|
|
si = swap_info[type];
|
|
si_frontswap_pages = atomic_read(&si->frontswap_pages);
|
|
if (total_pages_to_unuse < si_frontswap_pages) {
|
|
pages = pages_to_unuse = total_pages_to_unuse;
|
|
} else {
|
|
pages = si_frontswap_pages;
|
|
pages_to_unuse = 0; /* unuse all */
|
|
}
|
|
/* ensure there is enough RAM to fetch pages from frontswap */
|
|
if (security_vm_enough_memory_mm(current->mm, pages)) {
|
|
ret = -ENOMEM;
|
|
continue;
|
|
}
|
|
vm_unacct_memory(pages);
|
|
*unused = pages_to_unuse;
|
|
*swapid = type;
|
|
ret = 0;
|
|
break;
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* Used to check if it's necessory and feasible to unuse pages.
|
|
* Return 1 when nothing to do, 0 when need to shink pages,
|
|
* error code when there is an error.
|
|
*/
|
|
static int __frontswap_shrink(unsigned long target_pages,
|
|
unsigned long *pages_to_unuse,
|
|
int *type)
|
|
{
|
|
unsigned long total_pages = 0, total_pages_to_unuse;
|
|
|
|
assert_spin_locked(&swap_lock);
|
|
|
|
total_pages = __frontswap_curr_pages();
|
|
if (total_pages <= target_pages) {
|
|
/* Nothing to do */
|
|
*pages_to_unuse = 0;
|
|
return 1;
|
|
}
|
|
total_pages_to_unuse = total_pages - target_pages;
|
|
return __frontswap_unuse_pages(total_pages_to_unuse, pages_to_unuse, type);
|
|
}
|
|
|
|
/*
|
|
* Frontswap, like a true swap device, may unnecessarily retain pages
|
|
* under certain circumstances; "shrink" frontswap is essentially a
|
|
* "partial swapoff" and works by calling try_to_unuse to attempt to
|
|
* unuse enough frontswap pages to attempt to -- subject to memory
|
|
* constraints -- reduce the number of pages in frontswap to the
|
|
* number given in the parameter target_pages.
|
|
*/
|
|
void frontswap_shrink(unsigned long target_pages)
|
|
{
|
|
unsigned long pages_to_unuse = 0;
|
|
int uninitialized_var(type), ret;
|
|
|
|
/*
|
|
* we don't want to hold swap_lock while doing a very
|
|
* lengthy try_to_unuse, but swap_list may change
|
|
* so restart scan from swap_list.head each time
|
|
*/
|
|
spin_lock(&swap_lock);
|
|
ret = __frontswap_shrink(target_pages, &pages_to_unuse, &type);
|
|
spin_unlock(&swap_lock);
|
|
if (ret == 0)
|
|
try_to_unuse(type, true, pages_to_unuse);
|
|
return;
|
|
}
|
|
EXPORT_SYMBOL(frontswap_shrink);
|
|
|
|
/*
|
|
* Count and return the number of frontswap pages across all
|
|
* swap devices. This is exported so that backend drivers can
|
|
* determine current usage without reading debugfs.
|
|
*/
|
|
unsigned long frontswap_curr_pages(void)
|
|
{
|
|
unsigned long totalpages = 0;
|
|
|
|
spin_lock(&swap_lock);
|
|
totalpages = __frontswap_curr_pages();
|
|
spin_unlock(&swap_lock);
|
|
|
|
return totalpages;
|
|
}
|
|
EXPORT_SYMBOL(frontswap_curr_pages);
|
|
|
|
static int __init init_frontswap(void)
|
|
{
|
|
#ifdef CONFIG_DEBUG_FS
|
|
struct dentry *root = debugfs_create_dir("frontswap", NULL);
|
|
if (root == NULL)
|
|
return -ENXIO;
|
|
debugfs_create_u64("loads", S_IRUGO, root, &frontswap_loads);
|
|
debugfs_create_u64("succ_stores", S_IRUGO, root, &frontswap_succ_stores);
|
|
debugfs_create_u64("failed_stores", S_IRUGO, root,
|
|
&frontswap_failed_stores);
|
|
debugfs_create_u64("invalidates", S_IRUGO,
|
|
root, &frontswap_invalidates);
|
|
#endif
|
|
return 0;
|
|
}
|
|
|
|
module_init(init_frontswap);
|