mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-28 21:33:52 +00:00
hugetlb: introduce pud_huge
Straight forward extensions for huge pages located in the PUD instead of PMDs. Signed-off-by: Andi Kleen <ak@suse.de> Signed-off-by: Nick Piggin <npiggin@suse.de> Cc: Martin Schwidefsky <schwidefsky@de.ibm.com> Cc: Heiko Carstens <heiko.carstens@de.ibm.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
parent
4abd32dbab
commit
ceb8687961
9 changed files with 75 additions and 5 deletions
|
@ -107,6 +107,12 @@ int pmd_huge(pmd_t pmd)
|
||||||
{
|
{
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int pud_huge(pud_t pud)
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
struct page *
|
struct page *
|
||||||
follow_huge_pmd(struct mm_struct *mm, unsigned long address, pmd_t *pmd, int write)
|
follow_huge_pmd(struct mm_struct *mm, unsigned long address, pmd_t *pmd, int write)
|
||||||
{
|
{
|
||||||
|
|
|
@ -369,6 +369,11 @@ int pmd_huge(pmd_t pmd)
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int pud_huge(pud_t pud)
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
struct page *
|
struct page *
|
||||||
follow_huge_pmd(struct mm_struct *mm, unsigned long address,
|
follow_huge_pmd(struct mm_struct *mm, unsigned long address,
|
||||||
pmd_t *pmd, int write)
|
pmd_t *pmd, int write)
|
||||||
|
|
|
@ -120,6 +120,11 @@ int pmd_huge(pmd_t pmd)
|
||||||
return !!(pmd_val(pmd) & _SEGMENT_ENTRY_LARGE);
|
return !!(pmd_val(pmd) & _SEGMENT_ENTRY_LARGE);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int pud_huge(pud_t pud)
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
|
struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
|
||||||
pmd_t *pmdp, int write)
|
pmd_t *pmdp, int write)
|
||||||
{
|
{
|
||||||
|
|
|
@ -79,6 +79,11 @@ int pmd_huge(pmd_t pmd)
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int pud_huge(pud_t pud)
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
|
struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
|
||||||
pmd_t *pmd, int write)
|
pmd_t *pmd, int write)
|
||||||
{
|
{
|
||||||
|
|
|
@ -295,6 +295,11 @@ int pmd_huge(pmd_t pmd)
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int pud_huge(pud_t pud)
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
|
struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
|
||||||
pmd_t *pmd, int write)
|
pmd_t *pmd, int write)
|
||||||
{
|
{
|
||||||
|
|
|
@ -189,6 +189,11 @@ int pmd_huge(pmd_t pmd)
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int pud_huge(pud_t pud)
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
struct page *
|
struct page *
|
||||||
follow_huge_pmd(struct mm_struct *mm, unsigned long address,
|
follow_huge_pmd(struct mm_struct *mm, unsigned long address,
|
||||||
pmd_t *pmd, int write)
|
pmd_t *pmd, int write)
|
||||||
|
@ -209,6 +214,11 @@ int pmd_huge(pmd_t pmd)
|
||||||
return !!(pmd_val(pmd) & _PAGE_PSE);
|
return !!(pmd_val(pmd) & _PAGE_PSE);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int pud_huge(pud_t pud)
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
struct page *
|
struct page *
|
||||||
follow_huge_pmd(struct mm_struct *mm, unsigned long address,
|
follow_huge_pmd(struct mm_struct *mm, unsigned long address,
|
||||||
pmd_t *pmd, int write)
|
pmd_t *pmd, int write)
|
||||||
|
@ -217,9 +227,22 @@ follow_huge_pmd(struct mm_struct *mm, unsigned long address,
|
||||||
|
|
||||||
page = pte_page(*(pte_t *)pmd);
|
page = pte_page(*(pte_t *)pmd);
|
||||||
if (page)
|
if (page)
|
||||||
page += ((address & ~HPAGE_MASK) >> PAGE_SHIFT);
|
page += ((address & ~PMD_MASK) >> PAGE_SHIFT);
|
||||||
return page;
|
return page;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct page *
|
||||||
|
follow_huge_pud(struct mm_struct *mm, unsigned long address,
|
||||||
|
pud_t *pud, int write)
|
||||||
|
{
|
||||||
|
struct page *page;
|
||||||
|
|
||||||
|
page = pte_page(*(pte_t *)pud);
|
||||||
|
if (page)
|
||||||
|
page += ((address & ~PUD_MASK) >> PAGE_SHIFT);
|
||||||
|
return page;
|
||||||
|
}
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
/* x86_64 also uses this file */
|
/* x86_64 also uses this file */
|
||||||
|
|
|
@ -50,7 +50,10 @@ struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address,
|
||||||
int write);
|
int write);
|
||||||
struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
|
struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
|
||||||
pmd_t *pmd, int write);
|
pmd_t *pmd, int write);
|
||||||
|
struct page *follow_huge_pud(struct mm_struct *mm, unsigned long address,
|
||||||
|
pud_t *pud, int write);
|
||||||
int pmd_huge(pmd_t pmd);
|
int pmd_huge(pmd_t pmd);
|
||||||
|
int pud_huge(pud_t pmd);
|
||||||
void hugetlb_change_protection(struct vm_area_struct *vma,
|
void hugetlb_change_protection(struct vm_area_struct *vma,
|
||||||
unsigned long address, unsigned long end, pgprot_t newprot);
|
unsigned long address, unsigned long end, pgprot_t newprot);
|
||||||
|
|
||||||
|
@ -78,8 +81,10 @@ static inline unsigned long hugetlb_total_pages(void)
|
||||||
#define hugetlb_report_meminfo(buf) 0
|
#define hugetlb_report_meminfo(buf) 0
|
||||||
#define hugetlb_report_node_meminfo(n, buf) 0
|
#define hugetlb_report_node_meminfo(n, buf) 0
|
||||||
#define follow_huge_pmd(mm, addr, pmd, write) NULL
|
#define follow_huge_pmd(mm, addr, pmd, write) NULL
|
||||||
|
#define follow_huge_pud(mm, addr, pud, write) NULL
|
||||||
#define prepare_hugepage_range(file, addr, len) (-EINVAL)
|
#define prepare_hugepage_range(file, addr, len) (-EINVAL)
|
||||||
#define pmd_huge(x) 0
|
#define pmd_huge(x) 0
|
||||||
|
#define pud_huge(x) 0
|
||||||
#define is_hugepage_only_range(mm, addr, len) 0
|
#define is_hugepage_only_range(mm, addr, len) 0
|
||||||
#define hugetlb_free_pgd_range(tlb, addr, end, floor, ceiling) ({BUG(); 0; })
|
#define hugetlb_free_pgd_range(tlb, addr, end, floor, ceiling) ({BUG(); 0; })
|
||||||
#define hugetlb_fault(mm, vma, addr, write) ({ BUG(); 0; })
|
#define hugetlb_fault(mm, vma, addr, write) ({ BUG(); 0; })
|
||||||
|
|
|
@ -1996,6 +1996,15 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Can be overriden by architectures */
|
||||||
|
__attribute__((weak)) struct page *
|
||||||
|
follow_huge_pud(struct mm_struct *mm, unsigned long address,
|
||||||
|
pud_t *pud, int write)
|
||||||
|
{
|
||||||
|
BUG();
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
|
int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
|
||||||
struct page **pages, struct vm_area_struct **vmas,
|
struct page **pages, struct vm_area_struct **vmas,
|
||||||
unsigned long *position, int *length, int i,
|
unsigned long *position, int *length, int i,
|
||||||
|
|
15
mm/memory.c
15
mm/memory.c
|
@ -998,19 +998,24 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
|
||||||
goto no_page_table;
|
goto no_page_table;
|
||||||
|
|
||||||
pud = pud_offset(pgd, address);
|
pud = pud_offset(pgd, address);
|
||||||
if (pud_none(*pud) || unlikely(pud_bad(*pud)))
|
if (pud_none(*pud))
|
||||||
goto no_page_table;
|
goto no_page_table;
|
||||||
|
if (pud_huge(*pud)) {
|
||||||
|
BUG_ON(flags & FOLL_GET);
|
||||||
|
page = follow_huge_pud(mm, address, pud, flags & FOLL_WRITE);
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
if (unlikely(pud_bad(*pud)))
|
||||||
|
goto no_page_table;
|
||||||
|
|
||||||
pmd = pmd_offset(pud, address);
|
pmd = pmd_offset(pud, address);
|
||||||
if (pmd_none(*pmd))
|
if (pmd_none(*pmd))
|
||||||
goto no_page_table;
|
goto no_page_table;
|
||||||
|
|
||||||
if (pmd_huge(*pmd)) {
|
if (pmd_huge(*pmd)) {
|
||||||
BUG_ON(flags & FOLL_GET);
|
BUG_ON(flags & FOLL_GET);
|
||||||
page = follow_huge_pmd(mm, address, pmd, flags & FOLL_WRITE);
|
page = follow_huge_pmd(mm, address, pmd, flags & FOLL_WRITE);
|
||||||
goto out;
|
goto out;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (unlikely(pmd_bad(*pmd)))
|
if (unlikely(pmd_bad(*pmd)))
|
||||||
goto no_page_table;
|
goto no_page_table;
|
||||||
|
|
||||||
|
@ -1567,6 +1572,8 @@ static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
|
||||||
unsigned long next;
|
unsigned long next;
|
||||||
int err;
|
int err;
|
||||||
|
|
||||||
|
BUG_ON(pud_huge(*pud));
|
||||||
|
|
||||||
pmd = pmd_alloc(mm, pud, addr);
|
pmd = pmd_alloc(mm, pud, addr);
|
||||||
if (!pmd)
|
if (!pmd)
|
||||||
return -ENOMEM;
|
return -ENOMEM;
|
||||||
|
|
Loading…
Reference in a new issue