mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-23 17:30:02 +00:00
![Andy Lutomirski](/assets/img/avatar_default.png)
try_to_unmap_flush() used to open-code a rather x86-centric flush sequence: local_flush_tlb() + flush_tlb_others(). Rearrange the code so that the arch (only x86 for now) provides arch_tlbbatch_add_mm() and arch_tlbbatch_flush() and the core code calls those functions instead. I'll want this for x86 because, to enable address space ids, I can't support the flush_tlb_others() mode used by exising try_to_unmap_flush() implementation with good performance. I can support the new API fairly easily, though. I imagine that other architectures may be in a similar position. Architectures with strong remote flush primitives (arm64?) may have even worse performance problems with flush_tlb_others() the way that try_to_unmap_flush() uses it. Signed-off-by: Andy Lutomirski <luto@kernel.org> Acked-by: Kees Cook <keescook@chromium.org> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Borislav Petkov <bpetkov@suse.de> Cc: Dave Hansen <dave.hansen@intel.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Mel Gorman <mgorman@suse.de> Cc: Michal Hocko <mhocko@suse.com> Cc: Nadav Amit <nadav.amit@gmail.com> Cc: Nadav Amit <namit@vmware.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Rik van Riel <riel@redhat.com> Cc: Sasha Levin <sasha.levin@oracle.com> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: linux-mm@kvack.org Link: http://lkml.kernel.org/r/19f25a8581f9fb77876b7ff3b001f89835e34ea3.1495492063.git.luto@kernel.org Signed-off-by: Ingo Molnar <mingo@kernel.org>
94 lines
2.4 KiB
C
94 lines
2.4 KiB
C
#ifndef _LINUX_MM_TYPES_TASK_H
|
|
#define _LINUX_MM_TYPES_TASK_H
|
|
|
|
/*
|
|
* Here are the definitions of the MM data types that are embedded in 'struct task_struct'.
|
|
*
|
|
* (These are defined separately to decouple sched.h from mm_types.h as much as possible.)
|
|
*/
|
|
|
|
#include <linux/types.h>
|
|
#include <linux/threads.h>
|
|
#include <linux/atomic.h>
|
|
#include <linux/cpumask.h>
|
|
|
|
#include <asm/page.h>
|
|
|
|
#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
|
|
#include <asm/tlbbatch.h>
|
|
#endif
|
|
|
|
#define USE_SPLIT_PTE_PTLOCKS (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS)
|
|
#define USE_SPLIT_PMD_PTLOCKS (USE_SPLIT_PTE_PTLOCKS && \
|
|
IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK))
|
|
#define ALLOC_SPLIT_PTLOCKS (SPINLOCK_SIZE > BITS_PER_LONG/8)
|
|
|
|
/*
|
|
* The per task VMA cache array:
|
|
*/
|
|
#define VMACACHE_BITS 2
|
|
#define VMACACHE_SIZE (1U << VMACACHE_BITS)
|
|
#define VMACACHE_MASK (VMACACHE_SIZE - 1)
|
|
|
|
struct vmacache {
|
|
u32 seqnum;
|
|
struct vm_area_struct *vmas[VMACACHE_SIZE];
|
|
};
|
|
|
|
enum {
|
|
MM_FILEPAGES, /* Resident file mapping pages */
|
|
MM_ANONPAGES, /* Resident anonymous pages */
|
|
MM_SWAPENTS, /* Anonymous swap entries */
|
|
MM_SHMEMPAGES, /* Resident shared memory pages */
|
|
NR_MM_COUNTERS
|
|
};
|
|
|
|
#if USE_SPLIT_PTE_PTLOCKS && defined(CONFIG_MMU)
|
|
#define SPLIT_RSS_COUNTING
|
|
/* per-thread cached information, */
|
|
struct task_rss_stat {
|
|
int events; /* for synchronization threshold */
|
|
int count[NR_MM_COUNTERS];
|
|
};
|
|
#endif /* USE_SPLIT_PTE_PTLOCKS */
|
|
|
|
struct mm_rss_stat {
|
|
atomic_long_t count[NR_MM_COUNTERS];
|
|
};
|
|
|
|
struct page_frag {
|
|
struct page *page;
|
|
#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536)
|
|
__u32 offset;
|
|
__u32 size;
|
|
#else
|
|
__u16 offset;
|
|
__u16 size;
|
|
#endif
|
|
};
|
|
|
|
/* Track pages that require TLB flushes */
|
|
struct tlbflush_unmap_batch {
|
|
#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
|
|
/*
|
|
* The arch code makes the following promise: generic code can modify a
|
|
* PTE, then call arch_tlbbatch_add_mm() (which internally provides all
|
|
* needed barriers), then call arch_tlbbatch_flush(), and the entries
|
|
* will be flushed on all CPUs by the time that arch_tlbbatch_flush()
|
|
* returns.
|
|
*/
|
|
struct arch_tlbflush_unmap_batch arch;
|
|
|
|
/* True if a flush is needed. */
|
|
bool flush_required;
|
|
|
|
/*
|
|
* If true then the PTE was dirty when unmapped. The entry must be
|
|
* flushed before IO is initiated or a stale TLB entry potentially
|
|
* allows an update without redirtying the page.
|
|
*/
|
|
bool writable;
|
|
#endif
|
|
};
|
|
|
|
#endif /* _LINUX_MM_TYPES_TASK_H */
|