mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-10-06 00:39:48 +00:00
tlb: mmu_gather: add tlb_flush_*_range APIs
commit 2631ed00b0
upstream.
tlb_flush_{pte|pmd|pud|p4d}_range() adjust the tlb->start and
tlb->end, then set corresponding cleared_*.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Zhenyu Ye <yezhenyu2@huawei.com>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20200625080314.230-5-yezhenyu2@huawei.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
This commit is contained in:
parent
10e34766d8
commit
e7891b22b2
1 changed files with 40 additions and 15 deletions
|
@ -495,6 +495,38 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
|
|||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* tlb_flush_{pte|pmd|pud|p4d}_range() adjust the tlb->start and tlb->end,
|
||||
* and set corresponding cleared_*.
|
||||
*/
|
||||
static inline void tlb_flush_pte_range(struct mmu_gather *tlb,
|
||||
unsigned long address, unsigned long size)
|
||||
{
|
||||
__tlb_adjust_range(tlb, address, size);
|
||||
tlb->cleared_ptes = 1;
|
||||
}
|
||||
|
||||
static inline void tlb_flush_pmd_range(struct mmu_gather *tlb,
|
||||
unsigned long address, unsigned long size)
|
||||
{
|
||||
__tlb_adjust_range(tlb, address, size);
|
||||
tlb->cleared_pmds = 1;
|
||||
}
|
||||
|
||||
static inline void tlb_flush_pud_range(struct mmu_gather *tlb,
|
||||
unsigned long address, unsigned long size)
|
||||
{
|
||||
__tlb_adjust_range(tlb, address, size);
|
||||
tlb->cleared_puds = 1;
|
||||
}
|
||||
|
||||
static inline void tlb_flush_p4d_range(struct mmu_gather *tlb,
|
||||
unsigned long address, unsigned long size)
|
||||
{
|
||||
__tlb_adjust_range(tlb, address, size);
|
||||
tlb->cleared_p4ds = 1;
|
||||
}
|
||||
|
||||
#ifndef __tlb_remove_tlb_entry
|
||||
#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
|
||||
#endif
|
||||
|
@ -508,19 +540,17 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
|
|||
*/
|
||||
#define tlb_remove_tlb_entry(tlb, ptep, address) \
|
||||
do { \
|
||||
__tlb_adjust_range(tlb, address, PAGE_SIZE); \
|
||||
tlb->cleared_ptes = 1; \
|
||||
tlb_flush_pte_range(tlb, address, PAGE_SIZE); \
|
||||
__tlb_remove_tlb_entry(tlb, ptep, address); \
|
||||
} while (0)
|
||||
|
||||
#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \
|
||||
do { \
|
||||
unsigned long _sz = huge_page_size(h); \
|
||||
__tlb_adjust_range(tlb, address, _sz); \
|
||||
if (_sz == PMD_SIZE) \
|
||||
tlb->cleared_pmds = 1; \
|
||||
tlb_flush_pmd_range(tlb, address, _sz); \
|
||||
else if (_sz == PUD_SIZE) \
|
||||
tlb->cleared_puds = 1; \
|
||||
tlb_flush_pud_range(tlb, address, _sz); \
|
||||
__tlb_remove_tlb_entry(tlb, ptep, address); \
|
||||
} while (0)
|
||||
|
||||
|
@ -534,8 +564,7 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
|
|||
|
||||
#define tlb_remove_pmd_tlb_entry(tlb, pmdp, address) \
|
||||
do { \
|
||||
__tlb_adjust_range(tlb, address, HPAGE_PMD_SIZE); \
|
||||
tlb->cleared_pmds = 1; \
|
||||
tlb_flush_pmd_range(tlb, address, HPAGE_PMD_SIZE); \
|
||||
__tlb_remove_pmd_tlb_entry(tlb, pmdp, address); \
|
||||
} while (0)
|
||||
|
||||
|
@ -549,8 +578,7 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
|
|||
|
||||
#define tlb_remove_pud_tlb_entry(tlb, pudp, address) \
|
||||
do { \
|
||||
__tlb_adjust_range(tlb, address, HPAGE_PUD_SIZE); \
|
||||
tlb->cleared_puds = 1; \
|
||||
tlb_flush_pud_range(tlb, address, HPAGE_PUD_SIZE); \
|
||||
__tlb_remove_pud_tlb_entry(tlb, pudp, address); \
|
||||
} while (0)
|
||||
|
||||
|
@ -575,9 +603,8 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
|
|||
#ifndef pte_free_tlb
|
||||
#define pte_free_tlb(tlb, ptep, address) \
|
||||
do { \
|
||||
__tlb_adjust_range(tlb, address, PAGE_SIZE); \
|
||||
tlb_flush_pmd_range(tlb, address, PAGE_SIZE); \
|
||||
tlb->freed_tables = 1; \
|
||||
tlb->cleared_pmds = 1; \
|
||||
__pte_free_tlb(tlb, ptep, address); \
|
||||
} while (0)
|
||||
#endif
|
||||
|
@ -585,9 +612,8 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
|
|||
#ifndef pmd_free_tlb
|
||||
#define pmd_free_tlb(tlb, pmdp, address) \
|
||||
do { \
|
||||
__tlb_adjust_range(tlb, address, PAGE_SIZE); \
|
||||
tlb_flush_pud_range(tlb, address, PAGE_SIZE); \
|
||||
tlb->freed_tables = 1; \
|
||||
tlb->cleared_puds = 1; \
|
||||
__pmd_free_tlb(tlb, pmdp, address); \
|
||||
} while (0)
|
||||
#endif
|
||||
|
@ -596,9 +622,8 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
|
|||
#ifndef pud_free_tlb
|
||||
#define pud_free_tlb(tlb, pudp, address) \
|
||||
do { \
|
||||
__tlb_adjust_range(tlb, address, PAGE_SIZE); \
|
||||
tlb_flush_p4d_range(tlb, address, PAGE_SIZE); \
|
||||
tlb->freed_tables = 1; \
|
||||
tlb->cleared_p4ds = 1; \
|
||||
__pud_free_tlb(tlb, pudp, address); \
|
||||
} while (0)
|
||||
#endif
|
||||
|
|
Loading…
Reference in a new issue