mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-27 03:10:12 +00:00
![Rik van Riel](/assets/img/avatar_default.png)
The swap token code no longer fits in with the current VM model. It does not play well with cgroups or the better NUMA placement code in development, since we have only one swap token globally. It also has the potential to mess with scalability of the system, by increasing the number of non-reclaimable pages on the active and inactive anon LRU lists. Last but not least, the swap token code has been broken for a year without complaints, as reported by Konstantin Khlebnikov. This suggests we no longer have much use for it. The days of sub-1G memory systems with heavy use of swap are over. If we ever need thrashing reducing code in the future, we will have to implement something that does scale. Signed-off-by: Rik van Riel <riel@redhat.com> Cc: Konstantin Khlebnikov <khlebnikov@openvz.org> Acked-by: Johannes Weiner <hannes@cmpxchg.org> Cc: Mel Gorman <mel@csn.ul.ie> Cc: Hugh Dickins <hughd@google.com> Acked-by: Bob Picco <bpicco@meloft.net> Acked-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
397 lines
12 KiB
C
397 lines
12 KiB
C
#ifndef _LINUX_MM_TYPES_H
|
|
#define _LINUX_MM_TYPES_H
|
|
|
|
#include <linux/auxvec.h>
|
|
#include <linux/types.h>
|
|
#include <linux/threads.h>
|
|
#include <linux/list.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/prio_tree.h>
|
|
#include <linux/rbtree.h>
|
|
#include <linux/rwsem.h>
|
|
#include <linux/completion.h>
|
|
#include <linux/cpumask.h>
|
|
#include <linux/page-debug-flags.h>
|
|
#include <linux/uprobes.h>
|
|
#include <asm/page.h>
|
|
#include <asm/mmu.h>
|
|
|
|
#ifndef AT_VECTOR_SIZE_ARCH
|
|
#define AT_VECTOR_SIZE_ARCH 0
|
|
#endif
|
|
#define AT_VECTOR_SIZE (2*(AT_VECTOR_SIZE_ARCH + AT_VECTOR_SIZE_BASE + 1))
|
|
|
|
struct address_space;
|
|
|
|
#define USE_SPLIT_PTLOCKS (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS)
|
|
|
|
/*
|
|
* Each physical page in the system has a struct page associated with
|
|
* it to keep track of whatever it is we are using the page for at the
|
|
* moment. Note that we have no way to track which tasks are using
|
|
* a page, though if it is a pagecache page, rmap structures can tell us
|
|
* who is mapping it.
|
|
*
|
|
* The objects in struct page are organized in double word blocks in
|
|
* order to allows us to use atomic double word operations on portions
|
|
* of struct page. That is currently only used by slub but the arrangement
|
|
* allows the use of atomic double word operations on the flags/mapping
|
|
* and lru list pointers also.
|
|
*/
|
|
struct page {
|
|
/* First double word block */
|
|
unsigned long flags; /* Atomic flags, some possibly
|
|
* updated asynchronously */
|
|
struct address_space *mapping; /* If low bit clear, points to
|
|
* inode address_space, or NULL.
|
|
* If page mapped as anonymous
|
|
* memory, low bit is set, and
|
|
* it points to anon_vma object:
|
|
* see PAGE_MAPPING_ANON below.
|
|
*/
|
|
/* Second double word */
|
|
struct {
|
|
union {
|
|
pgoff_t index; /* Our offset within mapping. */
|
|
void *freelist; /* slub first free object */
|
|
};
|
|
|
|
union {
|
|
/* Used for cmpxchg_double in slub */
|
|
unsigned long counters;
|
|
|
|
struct {
|
|
|
|
union {
|
|
/*
|
|
* Count of ptes mapped in
|
|
* mms, to show when page is
|
|
* mapped & limit reverse map
|
|
* searches.
|
|
*
|
|
* Used also for tail pages
|
|
* refcounting instead of
|
|
* _count. Tail pages cannot
|
|
* be mapped and keeping the
|
|
* tail page _count zero at
|
|
* all times guarantees
|
|
* get_page_unless_zero() will
|
|
* never succeed on tail
|
|
* pages.
|
|
*/
|
|
atomic_t _mapcount;
|
|
|
|
struct {
|
|
unsigned inuse:16;
|
|
unsigned objects:15;
|
|
unsigned frozen:1;
|
|
};
|
|
};
|
|
atomic_t _count; /* Usage count, see below. */
|
|
};
|
|
};
|
|
};
|
|
|
|
/* Third double word block */
|
|
union {
|
|
struct list_head lru; /* Pageout list, eg. active_list
|
|
* protected by zone->lru_lock !
|
|
*/
|
|
struct { /* slub per cpu partial pages */
|
|
struct page *next; /* Next partial slab */
|
|
#ifdef CONFIG_64BIT
|
|
int pages; /* Nr of partial slabs left */
|
|
int pobjects; /* Approximate # of objects */
|
|
#else
|
|
short int pages;
|
|
short int pobjects;
|
|
#endif
|
|
};
|
|
};
|
|
|
|
/* Remainder is not double word aligned */
|
|
union {
|
|
unsigned long private; /* Mapping-private opaque data:
|
|
* usually used for buffer_heads
|
|
* if PagePrivate set; used for
|
|
* swp_entry_t if PageSwapCache;
|
|
* indicates order in the buddy
|
|
* system if PG_buddy is set.
|
|
*/
|
|
#if USE_SPLIT_PTLOCKS
|
|
spinlock_t ptl;
|
|
#endif
|
|
struct kmem_cache *slab; /* SLUB: Pointer to slab */
|
|
struct page *first_page; /* Compound tail pages */
|
|
};
|
|
|
|
/*
|
|
* On machines where all RAM is mapped into kernel address space,
|
|
* we can simply calculate the virtual address. On machines with
|
|
* highmem some memory is mapped into kernel virtual memory
|
|
* dynamically, so we need a place to store that address.
|
|
* Note that this field could be 16 bits on x86 ... ;)
|
|
*
|
|
* Architectures with slow multiplication can define
|
|
* WANT_PAGE_VIRTUAL in asm/page.h
|
|
*/
|
|
#if defined(WANT_PAGE_VIRTUAL)
|
|
void *virtual; /* Kernel virtual address (NULL if
|
|
not kmapped, ie. highmem) */
|
|
#endif /* WANT_PAGE_VIRTUAL */
|
|
#ifdef CONFIG_WANT_PAGE_DEBUG_FLAGS
|
|
unsigned long debug_flags; /* Use atomic bitops on this */
|
|
#endif
|
|
|
|
#ifdef CONFIG_KMEMCHECK
|
|
/*
|
|
* kmemcheck wants to track the status of each byte in a page; this
|
|
* is a pointer to such a status block. NULL if not tracked.
|
|
*/
|
|
void *shadow;
|
|
#endif
|
|
}
|
|
/*
|
|
* The struct page can be forced to be double word aligned so that atomic ops
|
|
* on double words work. The SLUB allocator can make use of such a feature.
|
|
*/
|
|
#ifdef CONFIG_HAVE_ALIGNED_STRUCT_PAGE
|
|
__aligned(2 * sizeof(unsigned long))
|
|
#endif
|
|
;
|
|
|
|
struct page_frag {
|
|
struct page *page;
|
|
#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536)
|
|
__u32 offset;
|
|
__u32 size;
|
|
#else
|
|
__u16 offset;
|
|
__u16 size;
|
|
#endif
|
|
};
|
|
|
|
typedef unsigned long __nocast vm_flags_t;
|
|
|
|
/*
|
|
* A region containing a mapping of a non-memory backed file under NOMMU
|
|
* conditions. These are held in a global tree and are pinned by the VMAs that
|
|
* map parts of them.
|
|
*/
|
|
struct vm_region {
|
|
struct rb_node vm_rb; /* link in global region tree */
|
|
vm_flags_t vm_flags; /* VMA vm_flags */
|
|
unsigned long vm_start; /* start address of region */
|
|
unsigned long vm_end; /* region initialised to here */
|
|
unsigned long vm_top; /* region allocated to here */
|
|
unsigned long vm_pgoff; /* the offset in vm_file corresponding to vm_start */
|
|
struct file *vm_file; /* the backing file or NULL */
|
|
|
|
int vm_usage; /* region usage count (access under nommu_region_sem) */
|
|
bool vm_icache_flushed : 1; /* true if the icache has been flushed for
|
|
* this region */
|
|
};
|
|
|
|
/*
|
|
* This struct defines a memory VMM memory area. There is one of these
|
|
* per VM-area/task. A VM area is any part of the process virtual memory
|
|
* space that has a special rule for the page-fault handlers (ie a shared
|
|
* library, the executable area etc).
|
|
*/
|
|
struct vm_area_struct {
|
|
struct mm_struct * vm_mm; /* The address space we belong to. */
|
|
unsigned long vm_start; /* Our start address within vm_mm. */
|
|
unsigned long vm_end; /* The first byte after our end address
|
|
within vm_mm. */
|
|
|
|
/* linked list of VM areas per task, sorted by address */
|
|
struct vm_area_struct *vm_next, *vm_prev;
|
|
|
|
pgprot_t vm_page_prot; /* Access permissions of this VMA. */
|
|
unsigned long vm_flags; /* Flags, see mm.h. */
|
|
|
|
struct rb_node vm_rb;
|
|
|
|
/*
|
|
* For areas with an address space and backing store,
|
|
* linkage into the address_space->i_mmap prio tree, or
|
|
* linkage to the list of like vmas hanging off its node, or
|
|
* linkage of vma in the address_space->i_mmap_nonlinear list.
|
|
*/
|
|
union {
|
|
struct {
|
|
struct list_head list;
|
|
void *parent; /* aligns with prio_tree_node parent */
|
|
struct vm_area_struct *head;
|
|
} vm_set;
|
|
|
|
struct raw_prio_tree_node prio_tree_node;
|
|
} shared;
|
|
|
|
/*
|
|
* A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma
|
|
* list, after a COW of one of the file pages. A MAP_SHARED vma
|
|
* can only be in the i_mmap tree. An anonymous MAP_PRIVATE, stack
|
|
* or brk vma (with NULL file) can only be in an anon_vma list.
|
|
*/
|
|
struct list_head anon_vma_chain; /* Serialized by mmap_sem &
|
|
* page_table_lock */
|
|
struct anon_vma *anon_vma; /* Serialized by page_table_lock */
|
|
|
|
/* Function pointers to deal with this struct. */
|
|
const struct vm_operations_struct *vm_ops;
|
|
|
|
/* Information about our backing store: */
|
|
unsigned long vm_pgoff; /* Offset (within vm_file) in PAGE_SIZE
|
|
units, *not* PAGE_CACHE_SIZE */
|
|
struct file * vm_file; /* File we map to (can be NULL). */
|
|
void * vm_private_data; /* was vm_pte (shared mem) */
|
|
|
|
#ifndef CONFIG_MMU
|
|
struct vm_region *vm_region; /* NOMMU mapping region */
|
|
#endif
|
|
#ifdef CONFIG_NUMA
|
|
struct mempolicy *vm_policy; /* NUMA policy for the VMA */
|
|
#endif
|
|
};
|
|
|
|
struct core_thread {
|
|
struct task_struct *task;
|
|
struct core_thread *next;
|
|
};
|
|
|
|
struct core_state {
|
|
atomic_t nr_threads;
|
|
struct core_thread dumper;
|
|
struct completion startup;
|
|
};
|
|
|
|
enum {
|
|
MM_FILEPAGES,
|
|
MM_ANONPAGES,
|
|
MM_SWAPENTS,
|
|
NR_MM_COUNTERS
|
|
};
|
|
|
|
#if USE_SPLIT_PTLOCKS && defined(CONFIG_MMU)
|
|
#define SPLIT_RSS_COUNTING
|
|
/* per-thread cached information, */
|
|
struct task_rss_stat {
|
|
int events; /* for synchronization threshold */
|
|
int count[NR_MM_COUNTERS];
|
|
};
|
|
#endif /* USE_SPLIT_PTLOCKS */
|
|
|
|
struct mm_rss_stat {
|
|
atomic_long_t count[NR_MM_COUNTERS];
|
|
};
|
|
|
|
struct mm_struct {
|
|
struct vm_area_struct * mmap; /* list of VMAs */
|
|
struct rb_root mm_rb;
|
|
struct vm_area_struct * mmap_cache; /* last find_vma result */
|
|
#ifdef CONFIG_MMU
|
|
unsigned long (*get_unmapped_area) (struct file *filp,
|
|
unsigned long addr, unsigned long len,
|
|
unsigned long pgoff, unsigned long flags);
|
|
void (*unmap_area) (struct mm_struct *mm, unsigned long addr);
|
|
#endif
|
|
unsigned long mmap_base; /* base of mmap area */
|
|
unsigned long task_size; /* size of task vm space */
|
|
unsigned long cached_hole_size; /* if non-zero, the largest hole below free_area_cache */
|
|
unsigned long free_area_cache; /* first hole of size cached_hole_size or larger */
|
|
pgd_t * pgd;
|
|
atomic_t mm_users; /* How many users with user space? */
|
|
atomic_t mm_count; /* How many references to "struct mm_struct" (users count as 1) */
|
|
int map_count; /* number of VMAs */
|
|
|
|
spinlock_t page_table_lock; /* Protects page tables and some counters */
|
|
struct rw_semaphore mmap_sem;
|
|
|
|
struct list_head mmlist; /* List of maybe swapped mm's. These are globally strung
|
|
* together off init_mm.mmlist, and are protected
|
|
* by mmlist_lock
|
|
*/
|
|
|
|
|
|
unsigned long hiwater_rss; /* High-watermark of RSS usage */
|
|
unsigned long hiwater_vm; /* High-water virtual memory usage */
|
|
|
|
unsigned long total_vm; /* Total pages mapped */
|
|
unsigned long locked_vm; /* Pages that have PG_mlocked set */
|
|
unsigned long pinned_vm; /* Refcount permanently increased */
|
|
unsigned long shared_vm; /* Shared pages (files) */
|
|
unsigned long exec_vm; /* VM_EXEC & ~VM_WRITE */
|
|
unsigned long stack_vm; /* VM_GROWSUP/DOWN */
|
|
unsigned long reserved_vm; /* VM_RESERVED|VM_IO pages */
|
|
unsigned long def_flags;
|
|
unsigned long nr_ptes; /* Page table pages */
|
|
unsigned long start_code, end_code, start_data, end_data;
|
|
unsigned long start_brk, brk, start_stack;
|
|
unsigned long arg_start, arg_end, env_start, env_end;
|
|
|
|
unsigned long saved_auxv[AT_VECTOR_SIZE]; /* for /proc/PID/auxv */
|
|
|
|
/*
|
|
* Special counters, in some configurations protected by the
|
|
* page_table_lock, in other configurations by being atomic.
|
|
*/
|
|
struct mm_rss_stat rss_stat;
|
|
|
|
struct linux_binfmt *binfmt;
|
|
|
|
cpumask_var_t cpu_vm_mask_var;
|
|
|
|
/* Architecture-specific MM context */
|
|
mm_context_t context;
|
|
|
|
unsigned long flags; /* Must use atomic bitops to access the bits */
|
|
|
|
struct core_state *core_state; /* coredumping support */
|
|
#ifdef CONFIG_AIO
|
|
spinlock_t ioctx_lock;
|
|
struct hlist_head ioctx_list;
|
|
#endif
|
|
#ifdef CONFIG_MM_OWNER
|
|
/*
|
|
* "owner" points to a task that is regarded as the canonical
|
|
* user/owner of this mm. All of the following must be true in
|
|
* order for it to be changed:
|
|
*
|
|
* current == mm->owner
|
|
* current->mm != mm
|
|
* new_owner->mm == mm
|
|
* new_owner->alloc_lock is held
|
|
*/
|
|
struct task_struct __rcu *owner;
|
|
#endif
|
|
|
|
/* store ref to file /proc/<pid>/exe symlink points to */
|
|
struct file *exe_file;
|
|
unsigned long num_exe_file_vmas;
|
|
#ifdef CONFIG_MMU_NOTIFIER
|
|
struct mmu_notifier_mm *mmu_notifier_mm;
|
|
#endif
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
pgtable_t pmd_huge_pte; /* protected by page_table_lock */
|
|
#endif
|
|
#ifdef CONFIG_CPUMASK_OFFSTACK
|
|
struct cpumask cpumask_allocation;
|
|
#endif
|
|
struct uprobes_state uprobes_state;
|
|
};
|
|
|
|
static inline void mm_init_cpumask(struct mm_struct *mm)
|
|
{
|
|
#ifdef CONFIG_CPUMASK_OFFSTACK
|
|
mm->cpu_vm_mask_var = &mm->cpumask_allocation;
|
|
#endif
|
|
}
|
|
|
|
/* Future-safe accessor for struct mm_struct's cpu_vm_mask. */
|
|
static inline cpumask_t *mm_cpumask(struct mm_struct *mm)
|
|
{
|
|
return mm->cpu_vm_mask_var;
|
|
}
|
|
|
|
#endif /* _LINUX_MM_TYPES_H */
|