mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-10-08 17:49:45 +00:00
5da8e4a658
The motivations to go rework memcpy_mcsafe() are that the benefit of
doing slow and careful copies is obviated on newer CPUs, and that the
current opt-in list of CPUs to instrument recovery is broken relative to
those CPUs. There is no need to keep an opt-in list up to date on an
ongoing basis if pmem/dax operations are instrumented for recovery by
default. With recovery enabled by default the old "mcsafe_key" opt-in to
careful copying can be made a "fragile" opt-out. Where the "fragile"
list takes steps to not consume poison across cachelines.
The discussion with Linus made clear that the current "_mcsafe" suffix
was imprecise to a fault. The operations that are needed by pmem/dax are
to copy from a source address that might throw #MC to a destination that
may write-fault, if it is a user page.
So copy_to_user_mcsafe() becomes copy_mc_to_user() to indicate
the separate precautions taken on source and destination.
copy_mc_to_kernel() is introduced as a non-SMAP version that does not
expect write-faults on the destination, but is still prepared to abort
with an error code upon taking #MC.
The original copy_mc_fragile() implementation had negative performance
implications since it did not use the fast-string instruction sequence
to perform copies. For this reason copy_mc_to_kernel() fell back to
plain memcpy() to preserve performance on platforms that did not indicate
the capability to recover from machine check exceptions. However, that
capability detection was not architectural and now that some platforms
can recover from fast-string consumption of memory errors the memcpy()
fallback now causes these more capable platforms to fail.
Introduce copy_mc_enhanced_fast_string() as the fast default
implementation of copy_mc_to_kernel() and finalize the transition of
copy_mc_fragile() to be a platform quirk to indicate 'copy-carefully'.
With this in place, copy_mc_to_kernel() is fast and recovery-ready by
default regardless of hardware capability.
Thanks to Vivek for identifying that copy_user_generic() is not suitable
as the copy_mc_to_user() backend since the #MC handler explicitly checks
ex_has_fault_handler(). Thanks to the 0day robot for catching a
performance bug in the x86/copy_mc_to_user implementation.
[ bp: Add the "why" for this change from the 0/2th message, massage. ]
Fixes: 92b0729c34 ("x86/mm, x86/mce: Add memcpy_mcsafe()")
Reported-by: Erwin Tsaur <erwin.tsaur@intel.com>
Reported-by: 0day robot <lkp@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Tony Luck <tony.luck@intel.com>
Tested-by: Erwin Tsaur <erwin.tsaur@intel.com>
Cc: <stable@vger.kernel.org>
Link: https://lkml.kernel.org/r/160195562556.2163339.18063423034951948973.stgit@dwillia2-desk3.amr.corp.intel.com
163 lines
4 KiB
ArmAsm
163 lines
4 KiB
ArmAsm
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */
|
|
|
|
#include <linux/linkage.h>
|
|
#include <asm/copy_mc_test.h>
|
|
#include <asm/export.h>
|
|
#include <asm/asm.h>
|
|
|
|
#ifndef CONFIG_UML
|
|
|
|
#ifdef CONFIG_X86_MCE
|
|
COPY_MC_TEST_CTL
|
|
|
|
/*
|
|
* copy_mc_fragile - copy memory with indication if an exception / fault happened
|
|
*
|
|
* The 'fragile' version is opted into by platform quirks and takes
|
|
* pains to avoid unrecoverable corner cases like 'fast-string'
|
|
* instruction sequences, and consuming poison across a cacheline
|
|
* boundary. The non-fragile version is equivalent to memcpy()
|
|
* regardless of CPU machine-check-recovery capability.
|
|
*/
|
|
SYM_FUNC_START(copy_mc_fragile)
|
|
cmpl $8, %edx
|
|
/* Less than 8 bytes? Go to byte copy loop */
|
|
jb .L_no_whole_words
|
|
|
|
/* Check for bad alignment of source */
|
|
testl $7, %esi
|
|
/* Already aligned */
|
|
jz .L_8byte_aligned
|
|
|
|
/* Copy one byte at a time until source is 8-byte aligned */
|
|
movl %esi, %ecx
|
|
andl $7, %ecx
|
|
subl $8, %ecx
|
|
negl %ecx
|
|
subl %ecx, %edx
|
|
.L_read_leading_bytes:
|
|
movb (%rsi), %al
|
|
COPY_MC_TEST_SRC %rsi 1 .E_leading_bytes
|
|
COPY_MC_TEST_DST %rdi 1 .E_leading_bytes
|
|
.L_write_leading_bytes:
|
|
movb %al, (%rdi)
|
|
incq %rsi
|
|
incq %rdi
|
|
decl %ecx
|
|
jnz .L_read_leading_bytes
|
|
|
|
.L_8byte_aligned:
|
|
movl %edx, %ecx
|
|
andl $7, %edx
|
|
shrl $3, %ecx
|
|
jz .L_no_whole_words
|
|
|
|
.L_read_words:
|
|
movq (%rsi), %r8
|
|
COPY_MC_TEST_SRC %rsi 8 .E_read_words
|
|
COPY_MC_TEST_DST %rdi 8 .E_write_words
|
|
.L_write_words:
|
|
movq %r8, (%rdi)
|
|
addq $8, %rsi
|
|
addq $8, %rdi
|
|
decl %ecx
|
|
jnz .L_read_words
|
|
|
|
/* Any trailing bytes? */
|
|
.L_no_whole_words:
|
|
andl %edx, %edx
|
|
jz .L_done_memcpy_trap
|
|
|
|
/* Copy trailing bytes */
|
|
movl %edx, %ecx
|
|
.L_read_trailing_bytes:
|
|
movb (%rsi), %al
|
|
COPY_MC_TEST_SRC %rsi 1 .E_trailing_bytes
|
|
COPY_MC_TEST_DST %rdi 1 .E_trailing_bytes
|
|
.L_write_trailing_bytes:
|
|
movb %al, (%rdi)
|
|
incq %rsi
|
|
incq %rdi
|
|
decl %ecx
|
|
jnz .L_read_trailing_bytes
|
|
|
|
/* Copy successful. Return zero */
|
|
.L_done_memcpy_trap:
|
|
xorl %eax, %eax
|
|
.L_done:
|
|
ret
|
|
SYM_FUNC_END(copy_mc_fragile)
|
|
EXPORT_SYMBOL_GPL(copy_mc_fragile)
|
|
|
|
.section .fixup, "ax"
|
|
/*
|
|
* Return number of bytes not copied for any failure. Note that
|
|
* there is no "tail" handling since the source buffer is 8-byte
|
|
* aligned and poison is cacheline aligned.
|
|
*/
|
|
.E_read_words:
|
|
shll $3, %ecx
|
|
.E_leading_bytes:
|
|
addl %edx, %ecx
|
|
.E_trailing_bytes:
|
|
mov %ecx, %eax
|
|
jmp .L_done
|
|
|
|
/*
|
|
* For write fault handling, given the destination is unaligned,
|
|
* we handle faults on multi-byte writes with a byte-by-byte
|
|
* copy up to the write-protected page.
|
|
*/
|
|
.E_write_words:
|
|
shll $3, %ecx
|
|
addl %edx, %ecx
|
|
movl %ecx, %edx
|
|
jmp copy_mc_fragile_handle_tail
|
|
|
|
.previous
|
|
|
|
_ASM_EXTABLE_FAULT(.L_read_leading_bytes, .E_leading_bytes)
|
|
_ASM_EXTABLE_FAULT(.L_read_words, .E_read_words)
|
|
_ASM_EXTABLE_FAULT(.L_read_trailing_bytes, .E_trailing_bytes)
|
|
_ASM_EXTABLE(.L_write_leading_bytes, .E_leading_bytes)
|
|
_ASM_EXTABLE(.L_write_words, .E_write_words)
|
|
_ASM_EXTABLE(.L_write_trailing_bytes, .E_trailing_bytes)
|
|
#endif /* CONFIG_X86_MCE */
|
|
|
|
/*
|
|
* copy_mc_enhanced_fast_string - memory copy with exception handling
|
|
*
|
|
* Fast string copy + fault / exception handling. If the CPU does
|
|
* support machine check exception recovery, but does not support
|
|
* recovering from fast-string exceptions then this CPU needs to be
|
|
* added to the copy_mc_fragile_key set of quirks. Otherwise, absent any
|
|
* machine check recovery support this version should be no slower than
|
|
* standard memcpy.
|
|
*/
|
|
SYM_FUNC_START(copy_mc_enhanced_fast_string)
|
|
movq %rdi, %rax
|
|
movq %rdx, %rcx
|
|
.L_copy:
|
|
rep movsb
|
|
/* Copy successful. Return zero */
|
|
xorl %eax, %eax
|
|
ret
|
|
SYM_FUNC_END(copy_mc_enhanced_fast_string)
|
|
|
|
.section .fixup, "ax"
|
|
.E_copy:
|
|
/*
|
|
* On fault %rcx is updated such that the copy instruction could
|
|
* optionally be restarted at the fault position, i.e. it
|
|
* contains 'bytes remaining'. A non-zero return indicates error
|
|
* to copy_mc_generic() users, or indicate short transfers to
|
|
* user-copy routines.
|
|
*/
|
|
movq %rcx, %rax
|
|
ret
|
|
|
|
.previous
|
|
|
|
_ASM_EXTABLE_FAULT(.L_copy, .E_copy)
|
|
#endif /* !CONFIG_UML */
|