mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-27 19:29:37 +00:00
![Peter Xu](/assets/img/avatar_default.png)
I observed that for each of the shared file-backed page faults, we're very likely to retry one more time for the 1st write fault upon no page. It's because we'll need to release the mmap lock for dirty rate limit purpose with balance_dirty_pages_ratelimited() (in fault_dirty_shared_page()). Then after that throttling we return VM_FAULT_RETRY. We did that probably because VM_FAULT_RETRY is the only way we can return to the fault handler at that time telling it we've released the mmap lock. However that's not ideal because it's very likely the fault does not need to be retried at all since the pgtable was well installed before the throttling, so the next continuous fault (including taking mmap read lock, walk the pgtable, etc.) could be in most cases unnecessary. It's not only slowing down page faults for shared file-backed, but also add more mmap lock contention which is in most cases not needed at all. To observe this, one could try to write to some shmem page and look at "pgfault" value in /proc/vmstat, then we should expect 2 counts for each shmem write simply because we retried, and vm event "pgfault" will capture that. To make it more efficient, add a new VM_FAULT_COMPLETED return code just to show that we've completed the whole fault and released the lock. It's also a hint that we should very possibly not need another fault immediately on this page because we've just completed it. This patch provides a ~12% perf boost on my aarch64 test VM with a simple program sequentially dirtying 400MB shmem file being mmap()ed and these are the time it needs: Before: 650.980 ms (+-1.94%) After: 569.396 ms (+-1.38%) I believe it could help more than that. We need some special care on GUP and the s390 pgfault handler (for gmap code before returning from pgfault), the rest changes in the page fault handlers should be relatively straightforward. Another thing to mention is that mm_account_fault() does take this new fault as a generic fault to be accounted, unlike VM_FAULT_RETRY. I explicitly didn't touch hmm_vma_fault() and break_ksm() because they do not handle VM_FAULT_RETRY even with existing code, so I'm literally keeping them as-is. Link: https://lkml.kernel.org/r/20220530183450.42886-1-peterx@redhat.com Signed-off-by: Peter Xu <peterx@redhat.com> Acked-by: Geert Uytterhoeven <geert@linux-m68k.org> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> Acked-by: Johannes Weiner <hannes@cmpxchg.org> Acked-by: Vineet Gupta <vgupta@kernel.org> Acked-by: Guo Ren <guoren@kernel.org> Acked-by: Max Filippov <jcmvbkbc@gmail.com> Acked-by: Christian Borntraeger <borntraeger@linux.ibm.com> Acked-by: Michael Ellerman <mpe@ellerman.id.au> (powerpc) Acked-by: Catalin Marinas <catalin.marinas@arm.com> Reviewed-by: Alistair Popple <apopple@nvidia.com> Reviewed-by: Ingo Molnar <mingo@kernel.org> Acked-by: Russell King (Oracle) <rmk+kernel@armlinux.org.uk> [arm part] Acked-by: Heiko Carstens <hca@linux.ibm.com> Cc: Vasily Gorbik <gor@linux.ibm.com> Cc: Stafford Horne <shorne@gmail.com> Cc: David S. Miller <davem@davemloft.net> Cc: Johannes Berg <johannes@sipsolutions.net> Cc: Brian Cain <bcain@quicinc.com> Cc: Richard Henderson <rth@twiddle.net> Cc: Richard Weinberger <richard@nod.at> Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Janosch Frank <frankja@linux.ibm.com> Cc: Albert Ou <aou@eecs.berkeley.edu> Cc: Anton Ivanov <anton.ivanov@cambridgegreys.com> Cc: Dave Hansen <dave.hansen@linux.intel.com> Cc: Borislav Petkov <bp@alien8.de> Cc: Sven Schnelle <svens@linux.ibm.com> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: James Bottomley <James.Bottomley@HansenPartnership.com> Cc: Al Viro <viro@zeniv.linux.org.uk> Cc: Alexander Gordeev <agordeev@linux.ibm.com> Cc: Jonas Bonn <jonas@southpole.se> Cc: Will Deacon <will@kernel.org> Cc: Vlastimil Babka <vbabka@suse.cz> Cc: Michal Simek <monstr@monstr.eu> Cc: Matt Turner <mattst88@gmail.com> Cc: Paul Mackerras <paulus@samba.org> Cc: David Hildenbrand <david@redhat.com> Cc: Nicholas Piggin <npiggin@gmail.com> Cc: Palmer Dabbelt <palmer@dabbelt.com> Cc: Stefan Kristiansson <stefan.kristiansson@saunalahti.fi> Cc: Paul Walmsley <paul.walmsley@sifive.com> Cc: Ivan Kokshaysky <ink@jurassic.park.msu.ru> Cc: Chris Zankel <chris@zankel.net> Cc: Hugh Dickins <hughd@google.com> Cc: Dinh Nguyen <dinguyen@kernel.org> Cc: Rich Felker <dalias@libc.org> Cc: H. Peter Anvin <hpa@zytor.com> Cc: Andy Lutomirski <luto@kernel.org> Cc: Thomas Bogendoerfer <tsbogend@alpha.franken.de> Cc: Helge Deller <deller@gmx.de> Cc: Yoshinori Sato <ysato@users.osdn.me> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
197 lines
4.4 KiB
C
197 lines
4.4 KiB
C
// SPDX-License-Identifier: GPL-2.0-only
|
|
/* Page Fault Handling for ARC (TLB Miss / ProtV)
|
|
*
|
|
* Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
|
|
*/
|
|
|
|
#include <linux/signal.h>
|
|
#include <linux/interrupt.h>
|
|
#include <linux/sched/signal.h>
|
|
#include <linux/errno.h>
|
|
#include <linux/ptrace.h>
|
|
#include <linux/uaccess.h>
|
|
#include <linux/kdebug.h>
|
|
#include <linux/perf_event.h>
|
|
#include <linux/mm_types.h>
|
|
#include <asm/mmu.h>
|
|
|
|
/*
|
|
* kernel virtual address is required to implement vmalloc/pkmap/fixmap
|
|
* Refer to asm/processor.h for System Memory Map
|
|
*
|
|
* It simply copies the PMD entry (pointer to 2nd level page table or hugepage)
|
|
* from swapper pgdir to task pgdir. The 2nd level table/page is thus shared
|
|
*/
|
|
noinline static int handle_kernel_vaddr_fault(unsigned long address)
|
|
{
|
|
/*
|
|
* Synchronize this task's top level page-table
|
|
* with the 'reference' page table.
|
|
*/
|
|
pgd_t *pgd, *pgd_k;
|
|
p4d_t *p4d, *p4d_k;
|
|
pud_t *pud, *pud_k;
|
|
pmd_t *pmd, *pmd_k;
|
|
|
|
pgd = pgd_offset(current->active_mm, address);
|
|
pgd_k = pgd_offset_k(address);
|
|
|
|
if (pgd_none (*pgd_k))
|
|
goto bad_area;
|
|
if (!pgd_present(*pgd))
|
|
set_pgd(pgd, *pgd_k);
|
|
|
|
p4d = p4d_offset(pgd, address);
|
|
p4d_k = p4d_offset(pgd_k, address);
|
|
if (p4d_none(*p4d_k))
|
|
goto bad_area;
|
|
if (!p4d_present(*p4d))
|
|
set_p4d(p4d, *p4d_k);
|
|
|
|
pud = pud_offset(p4d, address);
|
|
pud_k = pud_offset(p4d_k, address);
|
|
if (pud_none(*pud_k))
|
|
goto bad_area;
|
|
if (!pud_present(*pud))
|
|
set_pud(pud, *pud_k);
|
|
|
|
pmd = pmd_offset(pud, address);
|
|
pmd_k = pmd_offset(pud_k, address);
|
|
if (pmd_none(*pmd_k))
|
|
goto bad_area;
|
|
if (!pmd_present(*pmd))
|
|
set_pmd(pmd, *pmd_k);
|
|
|
|
/* XXX: create the TLB entry here */
|
|
return 0;
|
|
|
|
bad_area:
|
|
return 1;
|
|
}
|
|
|
|
void do_page_fault(unsigned long address, struct pt_regs *regs)
|
|
{
|
|
struct vm_area_struct *vma = NULL;
|
|
struct task_struct *tsk = current;
|
|
struct mm_struct *mm = tsk->mm;
|
|
int sig, si_code = SEGV_MAPERR;
|
|
unsigned int write = 0, exec = 0, mask;
|
|
vm_fault_t fault = VM_FAULT_SIGSEGV; /* handle_mm_fault() output */
|
|
unsigned int flags; /* handle_mm_fault() input */
|
|
|
|
/*
|
|
* NOTE! We MUST NOT take any locks for this case. We may
|
|
* be in an interrupt or a critical region, and should
|
|
* only copy the information from the master page table,
|
|
* nothing more.
|
|
*/
|
|
if (address >= VMALLOC_START && !user_mode(regs)) {
|
|
if (unlikely(handle_kernel_vaddr_fault(address)))
|
|
goto no_context;
|
|
else
|
|
return;
|
|
}
|
|
|
|
/*
|
|
* If we're in an interrupt or have no user
|
|
* context, we must not take the fault..
|
|
*/
|
|
if (faulthandler_disabled() || !mm)
|
|
goto no_context;
|
|
|
|
if (regs->ecr_cause & ECR_C_PROTV_STORE) /* ST/EX */
|
|
write = 1;
|
|
else if ((regs->ecr_vec == ECR_V_PROTV) &&
|
|
(regs->ecr_cause == ECR_C_PROTV_INST_FETCH))
|
|
exec = 1;
|
|
|
|
flags = FAULT_FLAG_DEFAULT;
|
|
if (user_mode(regs))
|
|
flags |= FAULT_FLAG_USER;
|
|
if (write)
|
|
flags |= FAULT_FLAG_WRITE;
|
|
|
|
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
|
|
retry:
|
|
mmap_read_lock(mm);
|
|
|
|
vma = find_vma(mm, address);
|
|
if (!vma)
|
|
goto bad_area;
|
|
if (unlikely(address < vma->vm_start)) {
|
|
if (!(vma->vm_flags & VM_GROWSDOWN) || expand_stack(vma, address))
|
|
goto bad_area;
|
|
}
|
|
|
|
/*
|
|
* vm_area is good, now check permissions for this memory access
|
|
*/
|
|
mask = VM_READ;
|
|
if (write)
|
|
mask = VM_WRITE;
|
|
if (exec)
|
|
mask = VM_EXEC;
|
|
|
|
if (!(vma->vm_flags & mask)) {
|
|
si_code = SEGV_ACCERR;
|
|
goto bad_area;
|
|
}
|
|
|
|
fault = handle_mm_fault(vma, address, flags, regs);
|
|
|
|
/* Quick path to respond to signals */
|
|
if (fault_signal_pending(fault, regs)) {
|
|
if (!user_mode(regs))
|
|
goto no_context;
|
|
return;
|
|
}
|
|
|
|
/* The fault is fully completed (including releasing mmap lock) */
|
|
if (fault & VM_FAULT_COMPLETED)
|
|
return;
|
|
|
|
/*
|
|
* Fault retry nuances, mmap_lock already relinquished by core mm
|
|
*/
|
|
if (unlikely(fault & VM_FAULT_RETRY)) {
|
|
flags |= FAULT_FLAG_TRIED;
|
|
goto retry;
|
|
}
|
|
|
|
bad_area:
|
|
mmap_read_unlock(mm);
|
|
|
|
/*
|
|
* Major/minor page fault accounting
|
|
* (in case of retry we only land here once)
|
|
*/
|
|
if (likely(!(fault & VM_FAULT_ERROR)))
|
|
/* Normal return path: fault Handled Gracefully */
|
|
return;
|
|
|
|
if (!user_mode(regs))
|
|
goto no_context;
|
|
|
|
if (fault & VM_FAULT_OOM) {
|
|
pagefault_out_of_memory();
|
|
return;
|
|
}
|
|
|
|
if (fault & VM_FAULT_SIGBUS) {
|
|
sig = SIGBUS;
|
|
si_code = BUS_ADRERR;
|
|
}
|
|
else {
|
|
sig = SIGSEGV;
|
|
}
|
|
|
|
tsk->thread.fault_address = address;
|
|
force_sig_fault(sig, si_code, (void __user *)address);
|
|
return;
|
|
|
|
no_context:
|
|
if (fixup_exception(regs))
|
|
return;
|
|
|
|
die("Oops", regs, address);
|
|
}
|