2019-06-03 05:44:50 +00:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0-only */
|
2012-03-05 11:49:27 +00:00
|
|
|
/*
|
|
|
|
|
* Copyright (C) 2012 ARM Ltd.
|
|
|
|
|
*/
|
|
|
|
|
#ifndef __ASM_PGTABLE_HWDEF_H
|
|
|
|
|
#define __ASM_PGTABLE_HWDEF_H
|
|
|
|
|
|
2017-12-13 17:07:18 +00:00
|
|
|
#include <asm/memory.h>
|
|
|
|
|
|
2015-10-19 13:19:29 +00:00
|
|
|
/*
|
|
|
|
|
* Number of page-table levels required to address 'va_bits' wide
|
|
|
|
|
* address, without section mapping. We resolve the top (va_bits - PAGE_SHIFT)
|
|
|
|
|
* bits with (PAGE_SHIFT - 3) bits at each page table level. Hence:
|
|
|
|
|
*
|
|
|
|
|
* levels = DIV_ROUND_UP((va_bits - PAGE_SHIFT), (PAGE_SHIFT - 3))
|
|
|
|
|
*
|
|
|
|
|
* where DIV_ROUND_UP(n, d) => (((n) + (d) - 1) / (d))
|
|
|
|
|
*
|
|
|
|
|
* We cannot include linux/kernel.h which defines DIV_ROUND_UP here
|
|
|
|
|
* due to build issues. So we open code DIV_ROUND_UP here:
|
|
|
|
|
*
|
|
|
|
|
* ((((va_bits) - PAGE_SHIFT) + (PAGE_SHIFT - 3) - 1) / (PAGE_SHIFT - 3))
|
|
|
|
|
*
|
|
|
|
|
* which gets simplified as :
|
|
|
|
|
*/
|
|
|
|
|
#define ARM64_HW_PGTABLE_LEVELS(va_bits) (((va_bits) - 4) / (PAGE_SHIFT - 3))
|
|
|
|
|
|
|
|
|
|
/*
|
2024-02-14 12:29:17 +00:00
|
|
|
* Size mapped by an entry at level n ( -1 <= n <= 3)
|
2015-10-19 13:19:29 +00:00
|
|
|
* We map (PAGE_SHIFT - 3) at all translation levels and PAGE_SHIFT bits
|
|
|
|
|
* in the final page. The maximum number of translation levels supported by
|
2024-02-14 12:29:17 +00:00
|
|
|
* the architecture is 5. Hence, starting at level n, we have further
|
2015-10-19 13:19:29 +00:00
|
|
|
* ((4 - n) - 1) levels of translation excluding the offset within the page.
|
|
|
|
|
* So, the total number of bits mapped by an entry at level n is :
|
|
|
|
|
*
|
|
|
|
|
* ((4 - n) - 1) * (PAGE_SHIFT - 3) + PAGE_SHIFT
|
|
|
|
|
*
|
|
|
|
|
* Rearranging it a bit we get :
|
|
|
|
|
* (4 - n) * (PAGE_SHIFT - 3) + 3
|
|
|
|
|
*/
|
|
|
|
|
#define ARM64_HW_PGTABLE_LEVEL_SHIFT(n) ((PAGE_SHIFT - 3) * (4 - (n)) + 3)
|
|
|
|
|
|
2014-07-15 15:35:38 +00:00
|
|
|
#define PTRS_PER_PTE (1 << (PAGE_SHIFT - 3))
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* PMD_SHIFT determines the size a level 2 page table entry can map.
|
|
|
|
|
*/
|
2015-04-14 22:45:39 +00:00
|
|
|
#if CONFIG_PGTABLE_LEVELS > 2
|
2015-10-19 13:19:29 +00:00
|
|
|
#define PMD_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(2)
|
2014-07-15 15:35:38 +00:00
|
|
|
#define PMD_SIZE (_AC(1, UL) << PMD_SHIFT)
|
|
|
|
|
#define PMD_MASK (~(PMD_SIZE-1))
|
2022-04-08 04:10:09 +00:00
|
|
|
#define PTRS_PER_PMD (1 << (PAGE_SHIFT - 3))
|
2014-07-15 15:35:38 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* PUD_SHIFT determines the size a level 1 page table entry can map.
|
|
|
|
|
*/
|
2015-04-14 22:45:39 +00:00
|
|
|
#if CONFIG_PGTABLE_LEVELS > 3
|
2015-10-19 13:19:29 +00:00
|
|
|
#define PUD_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(1)
|
2014-07-15 15:35:38 +00:00
|
|
|
#define PUD_SIZE (_AC(1, UL) << PUD_SHIFT)
|
|
|
|
|
#define PUD_MASK (~(PUD_SIZE-1))
|
2022-04-08 04:10:09 +00:00
|
|
|
#define PTRS_PER_PUD (1 << (PAGE_SHIFT - 3))
|
2012-03-05 11:49:27 +00:00
|
|
|
#endif
|
|
|
|
|
|
2024-02-14 12:29:17 +00:00
|
|
|
#if CONFIG_PGTABLE_LEVELS > 4
|
|
|
|
|
#define P4D_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(0)
|
|
|
|
|
#define P4D_SIZE (_AC(1, UL) << P4D_SHIFT)
|
|
|
|
|
#define P4D_MASK (~(P4D_SIZE-1))
|
|
|
|
|
#define PTRS_PER_P4D (1 << (PAGE_SHIFT - 3))
|
|
|
|
|
#endif
|
|
|
|
|
|
2014-07-15 15:35:38 +00:00
|
|
|
/*
|
|
|
|
|
* PGDIR_SHIFT determines the size a top-level page table entry can map
|
2024-02-14 12:29:17 +00:00
|
|
|
* (depending on the configuration, this level can be -1, 0, 1 or 2).
|
2014-07-15 15:35:38 +00:00
|
|
|
*/
|
2015-10-19 13:19:29 +00:00
|
|
|
#define PGDIR_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(4 - CONFIG_PGTABLE_LEVELS)
|
2014-07-15 15:35:38 +00:00
|
|
|
#define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT)
|
|
|
|
|
#define PGDIR_MASK (~(PGDIR_SIZE-1))
|
2019-11-04 21:56:46 +00:00
|
|
|
#define PTRS_PER_PGD (1 << (VA_BITS - PGDIR_SHIFT))
|
2014-07-15 15:35:38 +00:00
|
|
|
|
2015-10-07 17:00:20 +00:00
|
|
|
/*
|
|
|
|
|
* Contiguous page definitions.
|
|
|
|
|
*/
|
2020-09-10 09:59:35 +00:00
|
|
|
#define CONT_PTE_SHIFT (CONFIG_ARM64_CONT_PTE_SHIFT + PAGE_SHIFT)
|
arm64/mm: Redefine CONT_{PTE, PMD}_SHIFT
Currently, the value of CONT_{PTE, PMD}_SHIFT is off from standard
{PAGE, PMD}_SHIFT. In turn, we have to consider adding {PAGE, PMD}_SHIFT
when using CONT_{PTE, PMD}_SHIFT in the function hugetlbpage_init().
It's a bit confusing.
This redefines CONT_{PTE, PMD}_SHIFT with {PAGE, PMD}_SHIFT included
so that the later values needn't be added when using the former ones
in function hugetlbpage_init(). Note that the values of CONT_{PTES, PMDS}
are unchanged.
Suggested-by: Will Deacon <will@kernel.org>
Signed-off-by: Gavin Shan <gshan@redhat.com>
Reviewed-by: Anshuman Khandual <anshuman.khandual@arm.com>
Link: https://lkml.org/lkml/2020/5/6/190
Link: https://lore.kernel.org/r/20200630062428.194235-1-gshan@redhat.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2020-06-30 06:24:28 +00:00
|
|
|
#define CONT_PTES (1 << (CONT_PTE_SHIFT - PAGE_SHIFT))
|
2015-12-17 19:31:26 +00:00
|
|
|
#define CONT_PTE_SIZE (CONT_PTES * PAGE_SIZE)
|
|
|
|
|
#define CONT_PTE_MASK (~(CONT_PTE_SIZE - 1))
|
2020-09-10 09:59:36 +00:00
|
|
|
|
|
|
|
|
#define CONT_PMD_SHIFT (CONFIG_ARM64_CONT_PMD_SHIFT + PMD_SHIFT)
|
arm64/mm: Redefine CONT_{PTE, PMD}_SHIFT
Currently, the value of CONT_{PTE, PMD}_SHIFT is off from standard
{PAGE, PMD}_SHIFT. In turn, we have to consider adding {PAGE, PMD}_SHIFT
when using CONT_{PTE, PMD}_SHIFT in the function hugetlbpage_init().
It's a bit confusing.
This redefines CONT_{PTE, PMD}_SHIFT with {PAGE, PMD}_SHIFT included
so that the later values needn't be added when using the former ones
in function hugetlbpage_init(). Note that the values of CONT_{PTES, PMDS}
are unchanged.
Suggested-by: Will Deacon <will@kernel.org>
Signed-off-by: Gavin Shan <gshan@redhat.com>
Reviewed-by: Anshuman Khandual <anshuman.khandual@arm.com>
Link: https://lkml.org/lkml/2020/5/6/190
Link: https://lore.kernel.org/r/20200630062428.194235-1-gshan@redhat.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2020-06-30 06:24:28 +00:00
|
|
|
#define CONT_PMDS (1 << (CONT_PMD_SHIFT - PMD_SHIFT))
|
2015-12-17 19:31:26 +00:00
|
|
|
#define CONT_PMD_SIZE (CONT_PMDS * PMD_SIZE)
|
|
|
|
|
#define CONT_PMD_MASK (~(CONT_PMD_SIZE - 1))
|
2015-10-07 17:00:20 +00:00
|
|
|
|
2012-03-05 11:49:27 +00:00
|
|
|
/*
|
|
|
|
|
* Hardware page table definitions.
|
|
|
|
|
*
|
2024-02-14 12:29:17 +00:00
|
|
|
* Level -1 descriptor (PGD).
|
|
|
|
|
*/
|
|
|
|
|
#define PGD_TYPE_TABLE (_AT(pgdval_t, 3) << 0)
|
|
|
|
|
#define PGD_TABLE_BIT (_AT(pgdval_t, 1) << 1)
|
|
|
|
|
#define PGD_TYPE_MASK (_AT(pgdval_t, 3) << 0)
|
|
|
|
|
#define PGD_TABLE_PXN (_AT(pgdval_t, 1) << 59)
|
|
|
|
|
#define PGD_TABLE_UXN (_AT(pgdval_t, 1) << 60)
|
|
|
|
|
|
|
|
|
|
/*
|
2021-03-10 10:49:40 +00:00
|
|
|
* Level 0 descriptor (P4D).
|
|
|
|
|
*/
|
|
|
|
|
#define P4D_TYPE_TABLE (_AT(p4dval_t, 3) << 0)
|
|
|
|
|
#define P4D_TABLE_BIT (_AT(p4dval_t, 1) << 1)
|
|
|
|
|
#define P4D_TYPE_MASK (_AT(p4dval_t, 3) << 0)
|
|
|
|
|
#define P4D_TYPE_SECT (_AT(p4dval_t, 1) << 0)
|
|
|
|
|
#define P4D_SECT_RDONLY (_AT(p4dval_t, 1) << 7) /* AP[2] */
|
2021-03-10 10:49:41 +00:00
|
|
|
#define P4D_TABLE_PXN (_AT(p4dval_t, 1) << 59)
|
|
|
|
|
#define P4D_TABLE_UXN (_AT(p4dval_t, 1) << 60)
|
2021-03-10 10:49:40 +00:00
|
|
|
|
|
|
|
|
/*
|
2013-04-10 12:48:00 +00:00
|
|
|
* Level 1 descriptor (PUD).
|
|
|
|
|
*/
|
2014-05-12 09:40:51 +00:00
|
|
|
#define PUD_TYPE_TABLE (_AT(pudval_t, 3) << 0)
|
2018-01-10 19:07:26 +00:00
|
|
|
#define PUD_TABLE_BIT (_AT(pudval_t, 1) << 1)
|
|
|
|
|
#define PUD_TYPE_MASK (_AT(pudval_t, 3) << 0)
|
|
|
|
|
#define PUD_TYPE_SECT (_AT(pudval_t, 1) << 0)
|
2019-12-04 15:59:22 +00:00
|
|
|
#define PUD_SECT_RDONLY (_AT(pudval_t, 1) << 7) /* AP[2] */
|
2021-03-10 10:49:41 +00:00
|
|
|
#define PUD_TABLE_PXN (_AT(pudval_t, 1) << 59)
|
|
|
|
|
#define PUD_TABLE_UXN (_AT(pudval_t, 1) << 60)
|
2013-04-10 12:48:00 +00:00
|
|
|
|
|
|
|
|
/*
|
2012-03-05 11:49:27 +00:00
|
|
|
* Level 2 descriptor (PMD).
|
|
|
|
|
*/
|
|
|
|
|
#define PMD_TYPE_MASK (_AT(pmdval_t, 3) << 0)
|
|
|
|
|
#define PMD_TYPE_TABLE (_AT(pmdval_t, 3) << 0)
|
|
|
|
|
#define PMD_TYPE_SECT (_AT(pmdval_t, 1) << 0)
|
2013-04-10 12:48:00 +00:00
|
|
|
#define PMD_TABLE_BIT (_AT(pmdval_t, 1) << 1)
|
2012-03-05 11:49:27 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Section
|
|
|
|
|
*/
|
2013-04-19 15:23:57 +00:00
|
|
|
#define PMD_SECT_VALID (_AT(pmdval_t, 1) << 0)
|
|
|
|
|
#define PMD_SECT_USER (_AT(pmdval_t, 1) << 6) /* AP[1] */
|
|
|
|
|
#define PMD_SECT_RDONLY (_AT(pmdval_t, 1) << 7) /* AP[2] */
|
2012-03-05 11:49:27 +00:00
|
|
|
#define PMD_SECT_S (_AT(pmdval_t, 3) << 8)
|
|
|
|
|
#define PMD_SECT_AF (_AT(pmdval_t, 1) << 10)
|
|
|
|
|
#define PMD_SECT_NG (_AT(pmdval_t, 1) << 11)
|
2015-10-07 17:00:20 +00:00
|
|
|
#define PMD_SECT_CONT (_AT(pmdval_t, 1) << 52)
|
2012-11-15 17:21:16 +00:00
|
|
|
#define PMD_SECT_PXN (_AT(pmdval_t, 1) << 53)
|
|
|
|
|
#define PMD_SECT_UXN (_AT(pmdval_t, 1) << 54)
|
2021-03-10 10:49:41 +00:00
|
|
|
#define PMD_TABLE_PXN (_AT(pmdval_t, 1) << 59)
|
|
|
|
|
#define PMD_TABLE_UXN (_AT(pmdval_t, 1) << 60)
|
2012-03-05 11:49:27 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* AttrIndx[2:0] encoding (mapping attributes defined in the MAIR* registers).
|
|
|
|
|
*/
|
|
|
|
|
#define PMD_ATTRINDX(t) (_AT(pmdval_t, (t)) << 2)
|
|
|
|
|
#define PMD_ATTRINDX_MASK (_AT(pmdval_t, 7) << 2)
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Level 3 descriptor (PTE).
|
|
|
|
|
*/
|
2019-05-21 04:06:27 +00:00
|
|
|
#define PTE_VALID (_AT(pteval_t, 1) << 0)
|
2012-03-05 11:49:27 +00:00
|
|
|
#define PTE_TYPE_MASK (_AT(pteval_t, 3) << 0)
|
|
|
|
|
#define PTE_TYPE_PAGE (_AT(pteval_t, 3) << 0)
|
2013-04-10 12:48:00 +00:00
|
|
|
#define PTE_TABLE_BIT (_AT(pteval_t, 1) << 1)
|
2012-03-05 11:49:27 +00:00
|
|
|
#define PTE_USER (_AT(pteval_t, 1) << 6) /* AP[1] */
|
|
|
|
|
#define PTE_RDONLY (_AT(pteval_t, 1) << 7) /* AP[2] */
|
|
|
|
|
#define PTE_SHARED (_AT(pteval_t, 3) << 8) /* SH[1:0], inner shareable */
|
|
|
|
|
#define PTE_AF (_AT(pteval_t, 1) << 10) /* Access Flag */
|
|
|
|
|
#define PTE_NG (_AT(pteval_t, 1) << 11) /* nG */
|
2020-03-16 16:50:45 +00:00
|
|
|
#define PTE_GP (_AT(pteval_t, 1) << 50) /* BTI guarded */
|
2015-07-10 16:24:28 +00:00
|
|
|
#define PTE_DBM (_AT(pteval_t, 1) << 51) /* Dirty Bit Management */
|
2015-10-07 17:00:20 +00:00
|
|
|
#define PTE_CONT (_AT(pteval_t, 1) << 52) /* Contiguous range */
|
2012-11-15 17:21:16 +00:00
|
|
|
#define PTE_PXN (_AT(pteval_t, 1) << 53) /* Privileged XN */
|
|
|
|
|
#define PTE_UXN (_AT(pteval_t, 1) << 54) /* User XN */
|
2012-03-05 11:49:27 +00:00
|
|
|
|
2024-02-14 12:29:16 +00:00
|
|
|
#define PTE_ADDR_LOW (((_AT(pteval_t, 1) << (50 - PAGE_SHIFT)) - 1) << PAGE_SHIFT)
|
2017-12-13 17:07:21 +00:00
|
|
|
#ifdef CONFIG_ARM64_PA_BITS_52
|
2024-02-14 12:29:16 +00:00
|
|
|
#ifdef CONFIG_ARM64_64K_PAGES
|
2017-12-13 17:07:19 +00:00
|
|
|
#define PTE_ADDR_HIGH (_AT(pteval_t, 0xf) << 12)
|
2022-11-07 14:17:53 +00:00
|
|
|
#define PTE_ADDR_HIGH_SHIFT 36
|
2024-02-14 12:29:16 +00:00
|
|
|
#define PHYS_TO_PTE_ADDR_MASK (PTE_ADDR_LOW | PTE_ADDR_HIGH)
|
2017-12-13 17:07:21 +00:00
|
|
|
#else
|
2024-02-14 12:29:16 +00:00
|
|
|
#define PTE_ADDR_HIGH (_AT(pteval_t, 0x3) << 8)
|
|
|
|
|
#define PTE_ADDR_HIGH_SHIFT 42
|
|
|
|
|
#define PHYS_TO_PTE_ADDR_MASK GENMASK_ULL(49, 8)
|
|
|
|
|
#endif
|
2017-12-13 17:07:19 +00:00
|
|
|
#endif
|
|
|
|
|
|
2012-03-05 11:49:27 +00:00
|
|
|
/*
|
|
|
|
|
* AttrIndx[2:0] encoding (mapping attributes defined in the MAIR* registers).
|
|
|
|
|
*/
|
|
|
|
|
#define PTE_ATTRINDX(t) (_AT(pteval_t, (t)) << 2)
|
|
|
|
|
#define PTE_ATTRINDX_MASK (_AT(pteval_t, 7) << 2)
|
|
|
|
|
|
2023-06-06 14:58:54 +00:00
|
|
|
/*
|
|
|
|
|
* PIIndex[3:0] encoding (Permission Indirection Extension)
|
|
|
|
|
*/
|
|
|
|
|
#define PTE_PI_IDX_0 6 /* AP[1], USER */
|
|
|
|
|
#define PTE_PI_IDX_1 51 /* DBM */
|
|
|
|
|
#define PTE_PI_IDX_2 53 /* PXN */
|
|
|
|
|
#define PTE_PI_IDX_3 54 /* UXN */
|
|
|
|
|
|
2012-12-07 18:35:41 +00:00
|
|
|
/*
|
|
|
|
|
* Memory Attribute override for Stage-2 (MemAttr[3:0])
|
|
|
|
|
*/
|
|
|
|
|
#define PTE_S2_MEMATTR(t) (_AT(pteval_t, (t)) << 2)
|
|
|
|
|
|
2012-03-05 11:49:27 +00:00
|
|
|
/*
|
2014-03-07 08:49:25 +00:00
|
|
|
* Highest possible physical address supported.
|
2012-03-05 11:49:27 +00:00
|
|
|
*/
|
2017-12-13 17:07:16 +00:00
|
|
|
#define PHYS_MASK_SHIFT (CONFIG_ARM64_PA_BITS)
|
2012-03-05 11:49:27 +00:00
|
|
|
#define PHYS_MASK ((UL(1) << PHYS_MASK_SHIFT) - 1)
|
|
|
|
|
|
2018-07-31 13:08:56 +00:00
|
|
|
#define TTBR_CNP_BIT (UL(1) << 0)
|
|
|
|
|
|
2012-03-05 11:49:27 +00:00
|
|
|
/*
|
|
|
|
|
* TCR flags.
|
|
|
|
|
*/
|
arm64: mm: increase VA range of identity map
The page size and the number of translation levels, and hence the supported
virtual address range, are build-time configurables on arm64 whose optimal
values are use case dependent. However, in the current implementation, if
the system's RAM is located at a very high offset, the virtual address range
needs to reflect that merely because the identity mapping, which is only used
to enable or disable the MMU, requires the extended virtual range to map the
physical memory at an equal virtual offset.
This patch relaxes that requirement, by increasing the number of translation
levels for the identity mapping only, and only when actually needed, i.e.,
when system RAM's offset is found to be out of reach at runtime.
Tested-by: Laura Abbott <lauraa@codeaurora.org>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Tested-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2015-03-19 16:42:27 +00:00
|
|
|
#define TCR_T0SZ_OFFSET 0
|
|
|
|
|
#define TCR_T1SZ_OFFSET 16
|
|
|
|
|
#define TCR_T0SZ(x) ((UL(64) - (x)) << TCR_T0SZ_OFFSET)
|
|
|
|
|
#define TCR_T1SZ(x) ((UL(64) - (x)) << TCR_T1SZ_OFFSET)
|
|
|
|
|
#define TCR_TxSZ(x) (TCR_T0SZ(x) | TCR_T1SZ(x))
|
|
|
|
|
#define TCR_TxSZ_WIDTH 6
|
2016-09-08 12:55:38 +00:00
|
|
|
#define TCR_T0SZ_MASK (((UL(1) << TCR_TxSZ_WIDTH) - 1) << TCR_T0SZ_OFFSET)
|
2020-05-13 18:52:37 +00:00
|
|
|
#define TCR_T1SZ_MASK (((UL(1) << TCR_TxSZ_WIDTH) - 1) << TCR_T1SZ_OFFSET)
|
2016-04-04 10:43:15 +00:00
|
|
|
|
2018-12-06 17:31:22 +00:00
|
|
|
#define TCR_EPD0_SHIFT 7
|
|
|
|
|
#define TCR_EPD0_MASK (UL(1) << TCR_EPD0_SHIFT)
|
2016-04-04 10:43:15 +00:00
|
|
|
#define TCR_IRGN0_SHIFT 8
|
|
|
|
|
#define TCR_IRGN0_MASK (UL(3) << TCR_IRGN0_SHIFT)
|
|
|
|
|
#define TCR_IRGN0_NC (UL(0) << TCR_IRGN0_SHIFT)
|
|
|
|
|
#define TCR_IRGN0_WBWA (UL(1) << TCR_IRGN0_SHIFT)
|
|
|
|
|
#define TCR_IRGN0_WT (UL(2) << TCR_IRGN0_SHIFT)
|
|
|
|
|
#define TCR_IRGN0_WBnWA (UL(3) << TCR_IRGN0_SHIFT)
|
|
|
|
|
|
2018-12-06 17:31:22 +00:00
|
|
|
#define TCR_EPD1_SHIFT 23
|
|
|
|
|
#define TCR_EPD1_MASK (UL(1) << TCR_EPD1_SHIFT)
|
2016-04-04 10:43:15 +00:00
|
|
|
#define TCR_IRGN1_SHIFT 24
|
|
|
|
|
#define TCR_IRGN1_MASK (UL(3) << TCR_IRGN1_SHIFT)
|
|
|
|
|
#define TCR_IRGN1_NC (UL(0) << TCR_IRGN1_SHIFT)
|
|
|
|
|
#define TCR_IRGN1_WBWA (UL(1) << TCR_IRGN1_SHIFT)
|
|
|
|
|
#define TCR_IRGN1_WT (UL(2) << TCR_IRGN1_SHIFT)
|
|
|
|
|
#define TCR_IRGN1_WBnWA (UL(3) << TCR_IRGN1_SHIFT)
|
|
|
|
|
|
|
|
|
|
#define TCR_IRGN_NC (TCR_IRGN0_NC | TCR_IRGN1_NC)
|
|
|
|
|
#define TCR_IRGN_WBWA (TCR_IRGN0_WBWA | TCR_IRGN1_WBWA)
|
|
|
|
|
#define TCR_IRGN_WT (TCR_IRGN0_WT | TCR_IRGN1_WT)
|
|
|
|
|
#define TCR_IRGN_WBnWA (TCR_IRGN0_WBnWA | TCR_IRGN1_WBnWA)
|
|
|
|
|
#define TCR_IRGN_MASK (TCR_IRGN0_MASK | TCR_IRGN1_MASK)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#define TCR_ORGN0_SHIFT 10
|
|
|
|
|
#define TCR_ORGN0_MASK (UL(3) << TCR_ORGN0_SHIFT)
|
|
|
|
|
#define TCR_ORGN0_NC (UL(0) << TCR_ORGN0_SHIFT)
|
|
|
|
|
#define TCR_ORGN0_WBWA (UL(1) << TCR_ORGN0_SHIFT)
|
|
|
|
|
#define TCR_ORGN0_WT (UL(2) << TCR_ORGN0_SHIFT)
|
|
|
|
|
#define TCR_ORGN0_WBnWA (UL(3) << TCR_ORGN0_SHIFT)
|
|
|
|
|
|
|
|
|
|
#define TCR_ORGN1_SHIFT 26
|
|
|
|
|
#define TCR_ORGN1_MASK (UL(3) << TCR_ORGN1_SHIFT)
|
|
|
|
|
#define TCR_ORGN1_NC (UL(0) << TCR_ORGN1_SHIFT)
|
|
|
|
|
#define TCR_ORGN1_WBWA (UL(1) << TCR_ORGN1_SHIFT)
|
|
|
|
|
#define TCR_ORGN1_WT (UL(2) << TCR_ORGN1_SHIFT)
|
|
|
|
|
#define TCR_ORGN1_WBnWA (UL(3) << TCR_ORGN1_SHIFT)
|
|
|
|
|
|
|
|
|
|
#define TCR_ORGN_NC (TCR_ORGN0_NC | TCR_ORGN1_NC)
|
|
|
|
|
#define TCR_ORGN_WBWA (TCR_ORGN0_WBWA | TCR_ORGN1_WBWA)
|
|
|
|
|
#define TCR_ORGN_WT (TCR_ORGN0_WT | TCR_ORGN1_WT)
|
|
|
|
|
#define TCR_ORGN_WBnWA (TCR_ORGN0_WBnWA | TCR_ORGN1_WBnWA)
|
|
|
|
|
#define TCR_ORGN_MASK (TCR_ORGN0_MASK | TCR_ORGN1_MASK)
|
|
|
|
|
|
|
|
|
|
#define TCR_SH0_SHIFT 12
|
|
|
|
|
#define TCR_SH0_MASK (UL(3) << TCR_SH0_SHIFT)
|
|
|
|
|
#define TCR_SH0_INNER (UL(3) << TCR_SH0_SHIFT)
|
|
|
|
|
|
|
|
|
|
#define TCR_SH1_SHIFT 28
|
|
|
|
|
#define TCR_SH1_MASK (UL(3) << TCR_SH1_SHIFT)
|
|
|
|
|
#define TCR_SH1_INNER (UL(3) << TCR_SH1_SHIFT)
|
|
|
|
|
#define TCR_SHARED (TCR_SH0_INNER | TCR_SH1_INNER)
|
|
|
|
|
|
|
|
|
|
#define TCR_TG0_SHIFT 14
|
|
|
|
|
#define TCR_TG0_MASK (UL(3) << TCR_TG0_SHIFT)
|
|
|
|
|
#define TCR_TG0_4K (UL(0) << TCR_TG0_SHIFT)
|
|
|
|
|
#define TCR_TG0_64K (UL(1) << TCR_TG0_SHIFT)
|
|
|
|
|
#define TCR_TG0_16K (UL(2) << TCR_TG0_SHIFT)
|
|
|
|
|
|
|
|
|
|
#define TCR_TG1_SHIFT 30
|
|
|
|
|
#define TCR_TG1_MASK (UL(3) << TCR_TG1_SHIFT)
|
|
|
|
|
#define TCR_TG1_16K (UL(1) << TCR_TG1_SHIFT)
|
|
|
|
|
#define TCR_TG1_4K (UL(2) << TCR_TG1_SHIFT)
|
|
|
|
|
#define TCR_TG1_64K (UL(3) << TCR_TG1_SHIFT)
|
|
|
|
|
|
2017-12-13 17:07:17 +00:00
|
|
|
#define TCR_IPS_SHIFT 32
|
|
|
|
|
#define TCR_IPS_MASK (UL(7) << TCR_IPS_SHIFT)
|
2017-08-10 12:19:09 +00:00
|
|
|
#define TCR_A1 (UL(1) << 22)
|
2012-03-05 11:49:27 +00:00
|
|
|
#define TCR_ASID16 (UL(1) << 36)
|
2013-06-12 15:28:04 +00:00
|
|
|
#define TCR_TBI0 (UL(1) << 37)
|
2018-12-28 08:30:31 +00:00
|
|
|
#define TCR_TBI1 (UL(1) << 38)
|
2015-07-10 16:24:28 +00:00
|
|
|
#define TCR_HA (UL(1) << 39)
|
|
|
|
|
#define TCR_HD (UL(1) << 40)
|
2020-11-21 09:59:02 +00:00
|
|
|
#define TCR_TBID1 (UL(1) << 52)
|
2019-02-26 18:43:41 +00:00
|
|
|
#define TCR_NFD0 (UL(1) << 53)
|
2018-02-27 14:15:49 +00:00
|
|
|
#define TCR_NFD1 (UL(1) << 54)
|
2019-12-09 18:12:14 +00:00
|
|
|
#define TCR_E0PD0 (UL(1) << 55)
|
|
|
|
|
#define TCR_E0PD1 (UL(1) << 56)
|
2022-01-25 14:38:33 +00:00
|
|
|
#define TCR_TCMA0 (UL(1) << 57)
|
|
|
|
|
#define TCR_TCMA1 (UL(1) << 58)
|
2024-02-14 12:29:15 +00:00
|
|
|
#define TCR_DS (UL(1) << 59)
|
2012-03-05 11:49:27 +00:00
|
|
|
|
2017-12-13 17:07:18 +00:00
|
|
|
/*
|
|
|
|
|
* TTBR.
|
|
|
|
|
*/
|
|
|
|
|
#ifdef CONFIG_ARM64_PA_BITS_52
|
|
|
|
|
/*
|
|
|
|
|
* TTBR_ELx[1] is RES0 in this configuration.
|
|
|
|
|
*/
|
2022-07-08 14:00:56 +00:00
|
|
|
#define TTBR_BADDR_MASK_52 GENMASK_ULL(47, 2)
|
2017-12-13 17:07:18 +00:00
|
|
|
#endif
|
|
|
|
|
|
2019-08-07 15:55:22 +00:00
|
|
|
#ifdef CONFIG_ARM64_VA_BITS_52
|
arm64: mm: Offset TTBR1 to allow 52-bit PTRS_PER_PGD
Enabling 52-bit VAs on arm64 requires that the PGD table expands from 64
entries (for the 48-bit case) to 1024 entries. This quantity,
PTRS_PER_PGD is used as follows to compute which PGD entry corresponds
to a given virtual address, addr:
pgd_index(addr) -> (addr >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)
Userspace addresses are prefixed by 0's, so for a 48-bit userspace
address, uva, the following is true:
(uva >> PGDIR_SHIFT) & (1024 - 1) == (uva >> PGDIR_SHIFT) & (64 - 1)
In other words, a 48-bit userspace address will have the same pgd_index
when using PTRS_PER_PGD = 64 and 1024.
Kernel addresses are prefixed by 1's so, given a 48-bit kernel address,
kva, we have the following inequality:
(kva >> PGDIR_SHIFT) & (1024 - 1) != (kva >> PGDIR_SHIFT) & (64 - 1)
In other words a 48-bit kernel virtual address will have a different
pgd_index when using PTRS_PER_PGD = 64 and 1024.
If, however, we note that:
kva = 0xFFFF << 48 + lower (where lower[63:48] == 0b)
and, PGDIR_SHIFT = 42 (as we are dealing with 64KB PAGE_SIZE)
We can consider:
(kva >> PGDIR_SHIFT) & (1024 - 1) - (kva >> PGDIR_SHIFT) & (64 - 1)
= (0xFFFF << 6) & 0x3FF - (0xFFFF << 6) & 0x3F // "lower" cancels out
= 0x3C0
In other words, one can switch PTRS_PER_PGD to the 52-bit value globally
provided that they increment ttbr1_el1 by 0x3C0 * 8 = 0x1E00 bytes when
running with 48-bit kernel VAs (TCR_EL1.T1SZ = 16).
For kernel configuration where 52-bit userspace VAs are possible, this
patch offsets ttbr1_el1 and sets PTRS_PER_PGD corresponding to the
52-bit value.
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Reviewed-by: Suzuki K Poulose <suzuki.poulose@arm.com>
Suggested-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Steve Capper <steve.capper@arm.com>
[will: added comment to TTBR1_BADDR_4852_OFFSET calculation]
Signed-off-by: Will Deacon <will.deacon@arm.com>
2018-12-06 22:50:39 +00:00
|
|
|
/* Must be at least 64-byte aligned to prevent corruption of the TTBR */
|
|
|
|
|
#define TTBR1_BADDR_4852_OFFSET (((UL(1) << (52 - PGDIR_SHIFT)) - \
|
|
|
|
|
(UL(1) << (48 - PGDIR_SHIFT))) * 8)
|
|
|
|
|
#endif
|
|
|
|
|
|
2012-03-05 11:49:27 +00:00
|
|
|
#endif
|