/*
* Hardware page table definitions.
*
+ * Level 1 descriptor (PUD).
+ */
+
+ #define PUD_TABLE_BIT (_AT(pgdval_t, 1) << 1)
+
+ /*
* Level 2 descriptor (PMD).
*/
#define PMD_TYPE_MASK (_AT(pmdval_t, 3) << 0)
#define PMD_TYPE_FAULT (_AT(pmdval_t, 0) << 0)
#define PMD_TYPE_TABLE (_AT(pmdval_t, 3) << 0)
#define PMD_TYPE_SECT (_AT(pmdval_t, 1) << 0)
+ #define PMD_TABLE_BIT (_AT(pmdval_t, 1) << 1)
/*
* Section
*/
- #define PMD_SECT_USER (_AT(pteval_t, 1) << 6) /* AP[1] */
+ #define PMD_SECT_VALID (_AT(pmdval_t, 1) << 0)
+ #define PMD_SECT_PROT_NONE (_AT(pmdval_t, 1) << 2)
+ #define PMD_SECT_USER (_AT(pmdval_t, 1) << 6) /* AP[1] */
+ #define PMD_SECT_RDONLY (_AT(pmdval_t, 1) << 7) /* AP[2] */
#define PMD_SECT_S (_AT(pmdval_t, 3) << 8)
#define PMD_SECT_AF (_AT(pmdval_t, 1) << 10)
#define PMD_SECT_NG (_AT(pmdval_t, 1) << 11)
#define PTE_TYPE_MASK (_AT(pteval_t, 3) << 0)
#define PTE_TYPE_FAULT (_AT(pteval_t, 0) << 0)
#define PTE_TYPE_PAGE (_AT(pteval_t, 3) << 0)
+ #define PTE_TABLE_BIT (_AT(pteval_t, 1) << 1)
#define PTE_USER (_AT(pteval_t, 1) << 6) /* AP[1] */
#define PTE_RDONLY (_AT(pteval_t, 1) << 7) /* AP[2] */
#define PTE_SHARED (_AT(pteval_t, 3) << 8) /* SH[1:0], inner shareable */
#define PTE_ATTRINDX(t) (_AT(pteval_t, (t)) << 2)
#define PTE_ATTRINDX_MASK (_AT(pteval_t, 7) << 2)
+/*
+ * 2nd stage PTE definitions
+ */
+#define PTE_S2_RDONLY (_AT(pteval_t, 1) << 6) /* HAP[2:1] */
+#define PTE_S2_RDWR (_AT(pteval_t, 3) << 6) /* HAP[2:1] */
+
+/*
+ * Memory Attribute override for Stage-2 (MemAttr[3:0])
+ */
+#define PTE_S2_MEMATTR(t) (_AT(pteval_t, (t)) << 2)
+#define PTE_S2_MEMATTR_MASK (_AT(pteval_t, 0xf) << 2)
+
+/*
+ * EL2/HYP PTE/PMD definitions
+ */
+#define PMD_HYP PMD_SECT_USER
+#define PTE_HYP PTE_USER
+
/*
* 40-bit physical address supported.
*/
* Software defined PTE bits definition.
*/
#define PTE_VALID (_AT(pteval_t, 1) << 0)
- #define PTE_PROT_NONE (_AT(pteval_t, 1) << 1) /* only when !PTE_VALID */
- #define PTE_FILE (_AT(pteval_t, 1) << 2) /* only when !pte_present() */
+ #define PTE_PROT_NONE (_AT(pteval_t, 1) << 2) /* only when !PTE_VALID */
+ #define PTE_FILE (_AT(pteval_t, 1) << 3) /* only when !pte_present() */
#define PTE_DIRTY (_AT(pteval_t, 1) << 55)
#define PTE_SPECIAL (_AT(pteval_t, 1) << 56)
#define _MOD_PROT(p, b) __pgprot_modify(p, 0, b)
- #define PAGE_NONE __pgprot_modify(pgprot_default, PTE_TYPE_MASK, PTE_PROT_NONE)
+ #define PAGE_NONE __pgprot_modify(pgprot_default, PTE_TYPE_MASK, PTE_PROT_NONE | PTE_RDONLY | PTE_PXN | PTE_UXN)
#define PAGE_SHARED _MOD_PROT(pgprot_default, PTE_USER | PTE_NG | PTE_PXN | PTE_UXN)
#define PAGE_SHARED_EXEC _MOD_PROT(pgprot_default, PTE_USER | PTE_NG | PTE_PXN)
#define PAGE_COPY _MOD_PROT(pgprot_default, PTE_USER | PTE_NG | PTE_PXN | PTE_UXN | PTE_RDONLY)
#define PAGE_KERNEL _MOD_PROT(pgprot_default, PTE_PXN | PTE_UXN | PTE_DIRTY)
#define PAGE_KERNEL_EXEC _MOD_PROT(pgprot_default, PTE_UXN | PTE_DIRTY)
- #define __PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_TYPE_MASK) | PTE_PROT_NONE)
+#define PAGE_HYP _MOD_PROT(pgprot_default, PTE_HYP)
+#define PAGE_HYP_DEVICE __pgprot(PROT_DEVICE_nGnRE | PTE_HYP)
+
+#define PAGE_S2 __pgprot_modify(pgprot_default, PTE_S2_MEMATTR_MASK, PTE_S2_MEMATTR(MT_S2_NORMAL) | PTE_S2_RDONLY)
+#define PAGE_S2_DEVICE __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_DEVICE_nGnRE) | PTE_S2_RDWR | PTE_UXN)
+
+ #define __PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_TYPE_MASK) | PTE_PROT_NONE | PTE_RDONLY | PTE_PXN | PTE_UXN)
#define __PAGE_SHARED __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN)
#define __PAGE_SHARED_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN)
#define __PAGE_COPY __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN | PTE_RDONLY)
#define pte_none(pte) (!pte_val(pte))
#define pte_clear(mm,addr,ptep) set_pte(ptep, __pte(0))
#define pte_page(pte) (pfn_to_page(pte_pfn(pte)))
-#define pte_offset_kernel(dir,addr) (pmd_page_vaddr(*(dir)) + __pte_index(addr))
+#define pte_offset_kernel(dir,addr) (pmd_page_vaddr(*(dir)) + pte_index(addr))
#define pte_offset_map(dir,addr) pte_offset_kernel((dir), (addr))
#define pte_offset_map_nested(dir,addr) pte_offset_kernel((dir), (addr))
/*
* Huge pte definitions.
*/
- #define pte_huge(pte) ((pte_val(pte) & PTE_TYPE_MASK) == PTE_TYPE_HUGEPAGE)
- #define pte_mkhuge(pte) (__pte((pte_val(pte) & ~PTE_TYPE_MASK) | PTE_TYPE_HUGEPAGE))
+ #define pte_huge(pte) (!(pte_val(pte) & PTE_TABLE_BIT))
+ #define pte_mkhuge(pte) (__pte(pte_val(pte) & ~PTE_TABLE_BIT))
+
+ /*
+ * Hugetlb definitions.
+ */
+ #define HUGE_MAX_HSTATE 2
+ #define HPAGE_SHIFT PMD_SHIFT
+ #define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT)
+ #define HPAGE_MASK (~(HPAGE_SIZE - 1))
+ #define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT)
#define __HAVE_ARCH_PTE_SPECIAL
+ /*
+ * Software PMD bits for THP
+ */
+
+ #define PMD_SECT_DIRTY (_AT(pmdval_t, 1) << 55)
+ #define PMD_SECT_SPLITTING (_AT(pmdval_t, 1) << 57)
+
+ /*
+ * THP definitions.
+ */
+ #define pmd_young(pmd) (pmd_val(pmd) & PMD_SECT_AF)
+
+ #define __HAVE_ARCH_PMD_WRITE
+ #define pmd_write(pmd) (!(pmd_val(pmd) & PMD_SECT_RDONLY))
+
+ #ifdef CONFIG_TRANSPARENT_HUGEPAGE
+ #define pmd_trans_huge(pmd) (pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT))
+ #define pmd_trans_splitting(pmd) (pmd_val(pmd) & PMD_SECT_SPLITTING)
+ #endif
+
+ #define PMD_BIT_FUNC(fn,op) \
+ static inline pmd_t pmd_##fn(pmd_t pmd) { pmd_val(pmd) op; return pmd; }
+
+ PMD_BIT_FUNC(wrprotect, |= PMD_SECT_RDONLY);
+ PMD_BIT_FUNC(mkold, &= ~PMD_SECT_AF);
+ PMD_BIT_FUNC(mksplitting, |= PMD_SECT_SPLITTING);
+ PMD_BIT_FUNC(mkwrite, &= ~PMD_SECT_RDONLY);
+ PMD_BIT_FUNC(mkdirty, |= PMD_SECT_DIRTY);
+ PMD_BIT_FUNC(mkyoung, |= PMD_SECT_AF);
+ PMD_BIT_FUNC(mknotpresent, &= ~PMD_TYPE_MASK);
+
+ #define pmd_mkhuge(pmd) (__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT))
+
+ #define pmd_pfn(pmd) (((pmd_val(pmd) & PMD_MASK) & PHYS_MASK) >> PAGE_SHIFT)
+ #define pfn_pmd(pfn,prot) (__pmd(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)))
+ #define mk_pmd(page,prot) pfn_pmd(page_to_pfn(page),prot)
+
+ #define pmd_page(pmd) pfn_to_page(__phys_to_pfn(pmd_val(pmd) & PHYS_MASK))
+
+ static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
+ {
+ const pmdval_t mask = PMD_SECT_USER | PMD_SECT_PXN | PMD_SECT_UXN |
+ PMD_SECT_RDONLY | PMD_SECT_PROT_NONE |
+ PMD_SECT_VALID;
+ pmd_val(pmd) = (pmd_val(pmd) & ~mask) | (pgprot_val(newprot) & mask);
+ return pmd;
+ }
+
+ #define set_pmd_at(mm, addr, pmdp, pmd) set_pmd(pmdp, pmd)
+
+ static inline int has_transparent_hugepage(void)
+ {
+ return 1;
+ }
+
/*
* Mark the prot value as uncacheable and unbufferable.
*/
#define pmd_bad(pmd) (!(pmd_val(pmd) & 2))
+#define pmd_table(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \
+ PMD_TYPE_TABLE)
+#define pmd_sect(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \
+ PMD_TYPE_SECT)
+
+
static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
{
*pmdp = pmd;
#endif
/* Find an entry in the third-level page table.. */
-#define __pte_index(addr) (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))
+#define pte_index(addr) (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))
static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
{
/*
* Encode and decode a swap entry:
- * bits 0-1: present (must be zero)
- * bit 2: PTE_FILE
- * bits 3-8: swap type
+ * bits 0, 2: present (must both be zero)
+ * bit 3: PTE_FILE
+ * bits 4-8: swap type
* bits 9-63: swap offset
*/
- #define __SWP_TYPE_SHIFT 3
+ #define __SWP_TYPE_SHIFT 4
#define __SWP_TYPE_BITS 6
#define __SWP_TYPE_MASK ((1 << __SWP_TYPE_BITS) - 1)
#define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT)
/*
* Encode and decode a file entry:
- * bits 0-1: present (must be zero)
- * bit 2: PTE_FILE
- * bits 3-63: file offset / PAGE_SIZE
+ * bits 0, 2: present (must both be zero)
+ * bit 3: PTE_FILE
+ * bits 4-63: file offset / PAGE_SIZE
*/
#define pte_file(pte) (pte_val(pte) & PTE_FILE)
- #define pte_to_pgoff(x) (pte_val(x) >> 3)
- #define pgoff_to_pte(x) __pte(((x) << 3) | PTE_FILE)
+ #define pte_to_pgoff(x) (pte_val(x) >> 4)
+ #define pgoff_to_pte(x) __pte(((x) << 4) | PTE_FILE)
- #define PTE_FILE_MAX_BITS 61
+ #define PTE_FILE_MAX_BITS 60
extern int kern_addr_valid(unsigned long addr);
projects / ~andy / linux / commitdiff