ARM: LPAE: Page table maintenance for the 3-level format

This patch modifies the pgd/pmd/pte manipulation functions to support
the 3-level page table format. Since there is no need for an 'ext'
argument to cpu_set_pte_ext(), this patch conditionally defines a
different prototype for this function when CONFIG_ARM_LPAE.

The patch also introduces the L_PGD_SWAPPER flag to mark pgd entries
pointing to pmd tables pre-allocated in the swapper_pg_dir and avoid
trying to free them at run-time. This flag is 0 with the classic page
table format.

Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm/include/asm/pgalloc.h b/arch/arm/include/asm/pgalloc.h
index 7418894a737fb4374a03325ebff11506a7910319..943504f53f579e58c878165c595d5a110dae00ed 100644 (file)
--- a/arch/arm/include/asm/pgalloc.h
+++ b/arch/arm/include/asm/pgalloc.h
@@ -25,6 +25,26 @@
 #define _PAGE_USER_TABLE       (PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_USER))
 #define _PAGE_KERNEL_TABLE     (PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_KERNEL))
 
+#ifdef CONFIG_ARM_LPAE
+
+static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
+{
+       return (pmd_t *)get_zeroed_page(GFP_KERNEL | __GFP_REPEAT);
+}
+
+static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
+{
+       BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
+       free_page((unsigned long)pmd);
+}
+
+static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
+{
+       set_pud(pud, __pud(__pa(pmd) | PMD_TYPE_TABLE));
+}
+
+#else  /* !CONFIG_ARM_LPAE */
+
 /*
  * Since we have only two-level page tables, these are trivial
  */
@@ -32,6 +52,8 @@
 #define pmd_free(mm, pmd)              do { } while (0)
 #define pud_populate(mm,pmd,pte)       BUG()
 
+#endif /* CONFIG_ARM_LPAE */
+
 extern pgd_t *pgd_alloc(struct mm_struct *mm);
 extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
 
@@ -109,7 +131,9 @@ static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t pte,
 {
        pmdval_t pmdval = (pte + PTE_HWTABLE_OFF) | prot;
        pmdp[0] = __pmd(pmdval);
+#ifndef CONFIG_ARM_LPAE
        pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t));
+#endif
        flush_pmd_entry(pmdp);
 }
 

diff --git a/arch/arm/include/asm/pgtable-3level.h b/arch/arm/include/asm/pgtable-3level.h
index 79bf0acc40dbdb50fa118b3adb57b91ca97b90a6..a6261f51797323dd3fb9ed78bdba7e9b7cc26963 100644 (file)
--- a/arch/arm/include/asm/pgtable-3level.h
+++ b/arch/arm/include/asm/pgtable-3level.h
@@ -99,4 +99,9 @@
 #define L_PTE_MT_DEV_CACHED    (_AT(pteval_t, 3) << 2) /* normal inner write-back */
 #define L_PTE_MT_MASK          (_AT(pteval_t, 7) << 2)
 
+/*
+ * Software PGD flags.
+ */
+#define L_PGD_SWAPPER          (_AT(pgdval_t, 1) << 55)        /* swapper_pg_dir entry */
+
 #endif /* _ASM_PGTABLE_3LEVEL_H */

diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h
index 95fefd964bd0d3e2c75212cc3df3afa6dc164710..1db9ad6decf00f37aa0ec617ccfe32a0cb8d2aa6 100644 (file)
--- a/arch/arm/include/asm/pgtable.h
+++ b/arch/arm/include/asm/pgtable.h
@@ -165,6 +165,31 @@ extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
 /* to find an entry in a kernel page-table-directory */
 #define pgd_offset_k(addr)     pgd_offset(&init_mm, addr)
 
+#ifdef CONFIG_ARM_LPAE
+
+#define pud_none(pud)          (!pud_val(pud))
+#define pud_bad(pud)           (!(pud_val(pud) & 2))
+#define pud_present(pud)       (pud_val(pud))
+
+#define pud_clear(pudp)                        \
+       do {                            \
+               *pudp = __pud(0);       \
+               clean_pmd_entry(pudp);  \
+       } while (0)
+
+#define set_pud(pudp, pud)             \
+       do {                            \
+               *pudp = pud;            \
+               flush_pmd_entry(pudp);  \
+       } while (0)
+
+static inline pmd_t *pud_page_vaddr(pud_t pud)
+{
+       return __va(pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK);
+}
+
+#else  /* !CONFIG_ARM_LPAE */
+
 /*
  * The "pud_xxx()" functions here are trivial when the pmd is folded into
  * the pud: the pud entry is never bad, always exists, and can't be set or
@@ -176,15 +201,43 @@ extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
 #define pud_clear(pudp)                do { } while (0)
 #define set_pud(pud,pudp)      do { } while (0)
 
+#endif /* CONFIG_ARM_LPAE */
 
 /* Find an entry in the second-level page table.. */
+#ifdef CONFIG_ARM_LPAE
+#define pmd_index(addr)                (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
+static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr)
+{
+       return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(addr);
+}
+#else
 static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr)
 {
        return (pmd_t *)pud;
 }
+#endif
 
 #define pmd_none(pmd)          (!pmd_val(pmd))
 #define pmd_present(pmd)       (pmd_val(pmd))
+
+#ifdef CONFIG_ARM_LPAE
+
+#define pmd_bad(pmd)           (!(pmd_val(pmd) & 2))
+
+#define copy_pmd(pmdpd,pmdps)          \
+       do {                            \
+               *pmdpd = *pmdps;        \
+               flush_pmd_entry(pmdpd); \
+       } while (0)
+
+#define pmd_clear(pmdp)                        \
+       do {                            \
+               *pmdp = __pmd(0);       \
+               clean_pmd_entry(pmdp);  \
+       } while (0)
+
+#else  /* !CONFIG_ARM_LPAE */
+
 #define pmd_bad(pmd)           (pmd_val(pmd) & 2)
 
 #define copy_pmd(pmdpd,pmdps)          \
@@ -201,6 +254,8 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr)
                clean_pmd_entry(pmdp);  \
        } while (0)
 
+#endif /* CONFIG_ARM_LPAE */
+
 static inline pte_t *pmd_page_vaddr(pmd_t pmd)
 {
        return __va(pmd_val(pmd) & PHYS_MASK & (s32)PAGE_MASK);
@@ -233,9 +288,14 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd)
 #define pte_page(pte)          pfn_to_page(pte_pfn(pte))
 #define mk_pte(page,prot)      pfn_pte(page_to_pfn(page), prot)
 
-#define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,pte,ext)
 #define pte_clear(mm,addr,ptep)        set_pte_ext(ptep, __pte(0), 0)
 
+#ifdef CONFIG_ARM_LPAE
+#define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,__pte(pte_val(pte)|(ext)))
+#else
+#define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,pte,ext)
+#endif
+
 #if __LINUX_ARM_ARCH__ < 6
 static inline void __sync_icache_dcache(pte_t pteval)
 {

diff --git a/arch/arm/include/asm/proc-fns.h b/arch/arm/include/asm/proc-fns.h
index 8ec535e11fd73c81ebdc12500c1ad24d7a9add35..b5db4f415efd9ba297a295f209efb42c51a17ca8 100644 (file)
--- a/arch/arm/include/asm/proc-fns.h
+++ b/arch/arm/include/asm/proc-fns.h
@@ -65,7 +65,11 @@ extern struct processor {
         * Set a possibly extended PTE.  Non-extended PTEs should
         * ignore 'ext'.
         */
+#ifdef CONFIG_ARM_LPAE
+       void (*set_pte_ext)(pte_t *ptep, pte_t pte);
+#else
        void (*set_pte_ext)(pte_t *ptep, pte_t pte, unsigned int ext);
+#endif
 
        /* Suspend/resume */
        unsigned int suspend_size;
@@ -79,7 +83,11 @@ extern void cpu_proc_fin(void);
 extern int cpu_do_idle(void);
 extern void cpu_dcache_clean_area(void *, int);
 extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm);
+#ifdef CONFIG_ARM_LPAE
+extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte);
+#else
 extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext);
+#endif
 extern void cpu_reset(unsigned long addr) __attribute__((noreturn));
 #else
 #define cpu_proc_init()                        processor._proc_init()
@@ -87,7 +95,11 @@ extern void cpu_reset(unsigned long addr) __attribute__((noreturn));
 #define cpu_reset(addr)                        processor.reset(addr)
 #define cpu_do_idle()                  processor._do_idle()
 #define cpu_dcache_clean_area(addr,sz) processor.dcache_clean_area(addr,sz)
+#ifdef CONFIG_ARM_LPAE
+#define cpu_set_pte_ext(ptep,pte)      processor.set_pte_ext(ptep,pte)
+#else
 #define cpu_set_pte_ext(ptep,pte,ext)  processor.set_pte_ext(ptep,pte,ext)
+#endif
 #define cpu_do_switch_mm(pgd,mm)       processor.switch_mm(pgd,mm)
 #endif
 
@@ -99,6 +111,18 @@ extern void cpu_resume(void);
 
 #define cpu_switch_mm(pgd,mm) cpu_do_switch_mm(virt_to_phys(pgd),mm)
 
+#ifdef CONFIG_ARM_LPAE
+#define cpu_get_pgd()  \
+       ({                                              \
+               unsigned long pg, pg2;                  \
+               __asm__("mrrc   p15, 0, %0, %1, c2"     \
+                       : "=r" (pg), "=r" (pg2)         \
+                       :                               \
+                       : "cc");                        \
+               pg &= ~(PTRS_PER_PGD*sizeof(pgd_t)-1);  \
+               (pgd_t *)phys_to_virt(pg);              \
+       })
+#else
 #define cpu_get_pgd()  \
        ({                                              \
                unsigned long pg;                       \
@@ -107,6 +131,7 @@ extern void cpu_resume(void);
                pg &= ~0x3fff;                          \
                (pgd_t *)phys_to_virt(pg);              \
        })
+#endif
 
 #endif
 

diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c
index 0da0091212407475e4dba8615ad33c78e5c0231f..3a5a9c272c960a73734752b74d971e0a7900a211 100644 (file)
--- a/arch/arm/mm/ioremap.c
+++ b/arch/arm/mm/ioremap.c
@@ -64,7 +64,7 @@ void __check_kvm_seq(struct mm_struct *mm)
        } while (seq != init_mm.context.kvm_seq);
 }
 
-#ifndef CONFIG_SMP
+#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
 /*
  * Section support is unsafe on SMP - If you iounmap and ioremap a region,
  * the other CPUs will not see this change until their next context switch.
@@ -198,11 +198,13 @@ void __iomem * __arm_ioremap_pfn_caller(unsigned long pfn,
        unsigned long addr;
        struct vm_struct * area;
 
+#ifndef CONFIG_ARM_LPAE
        /*
         * High mappings must be supersection aligned
         */
        if (pfn >= 0x100000 && (__pfn_to_phys(pfn) & ~SUPERSECTION_MASK))
                return NULL;
+#endif
 
        /*
         * Don't allow RAM to be mapped - this causes problems with ARMv6+
@@ -224,7 +226,7 @@ void __iomem * __arm_ioremap_pfn_caller(unsigned long pfn,
                return NULL;
        addr = (unsigned long)area->addr;
 
-#ifndef CONFIG_SMP
+#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
        if (DOMAIN_IO == 0 &&
            (((cpu_architecture() >= CPU_ARCH_ARMv6) && (get_cr() & CR_XP)) ||
               cpu_is_xsc3()) && pfn >= 0x100000 &&
@@ -295,7 +297,7 @@ EXPORT_SYMBOL(__arm_ioremap);
 void __iounmap(volatile void __iomem *io_addr)
 {
        void *addr = (void *)(PAGE_MASK & (unsigned long)io_addr);
-#ifndef CONFIG_SMP
+#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
        struct vm_struct **p, *tmp;
 
        /*

diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c
index b2027c154b2a0c946198de99370a89be2f60b6eb..a3e78ccabd655ef871c74b66b6a277eee29290d6 100644 (file)
--- a/arch/arm/mm/pgd.c
+++ b/arch/arm/mm/pgd.c
@@ -10,6 +10,7 @@
 #include <linux/mm.h>
 #include <linux/gfp.h>
 #include <linux/highmem.h>
+#include <linux/slab.h>
 
 #include <asm/pgalloc.h>
 #include <asm/page.h>
@@ -17,6 +18,14 @@
 
 #include "mm.h"
 
+#ifdef CONFIG_ARM_LPAE
+#define __pgd_alloc()  kmalloc(PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL)
+#define __pgd_free(pgd)        kfree(pgd)
+#else
+#define __pgd_alloc()  (pgd_t *)__get_free_pages(GFP_KERNEL, 2)
+#define __pgd_free(pgd)        free_pages((unsigned long)pgd, 2)
+#endif
+
 /*
  * need to get a 16k page for level 1
  */
@@ -27,7 +36,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
        pmd_t *new_pmd, *init_pmd;
        pte_t *new_pte, *init_pte;
 
-       new_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, 2);
+       new_pgd = __pgd_alloc();
        if (!new_pgd)
                goto no_pgd;
 
@@ -42,10 +51,25 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
 
        clean_dcache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t));
 
+#ifdef CONFIG_ARM_LPAE
+       /*
+        * Allocate PMD table for modules and pkmap mappings.
+        */
+       new_pud = pud_alloc(mm, new_pgd + pgd_index(MODULES_VADDR),
+                           MODULES_VADDR);
+       if (!new_pud)
+               goto no_pud;
+
+       new_pmd = pmd_alloc(mm, new_pud, 0);
+       if (!new_pmd)
+               goto no_pmd;
+#endif
+
        if (!vectors_high()) {
                /*
                 * On ARM, first page must always be allocated since it
-                * contains the machine vectors.
+                * contains the machine vectors. The vectors are always high
+                * with LPAE.
                 */
                new_pud = pud_alloc(mm, new_pgd, 0);
                if (!new_pud)
@@ -74,7 +98,7 @@ no_pte:
 no_pmd:
        pud_free(mm, new_pud);
 no_pud:
-       free_pages((unsigned long)new_pgd, 2);
+       __pgd_free(new_pgd);
 no_pgd:
        return NULL;
 }
@@ -111,5 +135,24 @@ no_pud:
        pgd_clear(pgd);
        pud_free(mm, pud);
 no_pgd:
-       free_pages((unsigned long) pgd_base, 2);
+#ifdef CONFIG_ARM_LPAE
+       /*
+        * Free modules/pkmap or identity pmd tables.
+        */
+       for (pgd = pgd_base; pgd < pgd_base + PTRS_PER_PGD; pgd++) {
+               if (pgd_none_or_clear_bad(pgd))
+                       continue;
+               if (pgd_val(*pgd) & L_PGD_SWAPPER)
+                       continue;
+               pud = pud_offset(pgd, 0);
+               if (pud_none_or_clear_bad(pud))
+                       continue;
+               pmd = pmd_offset(pud, 0);
+               pud_clear(pud);
+               pmd_free(mm, pmd);
+               pgd_clear(pgd);
+               pud_free(mm, pud);
+       }
+#endif
+       __pgd_free(pgd_base);
 }