Merge master.kernel.org:/pub/scm/linux/kernel/git/davem/sparc-2.6

[karo-tx-linux.git] / arch / i386 / mm / hugetlbpage.c

diff --git a/arch/i386/mm/hugetlbpage.c b/arch/i386/mm/hugetlbpage.c
index 5aa06001a4bde0537fd22a0eebbdb528bf38ca6b..a7d8915854116264048f62c84d63dc495fc212b7 100644 (file)
--- a/arch/i386/mm/hugetlbpage.c
+++ b/arch/i386/mm/hugetlbpage.c
@@ -22,12 +22,15 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)
 {
        pgd_t *pgd;
        pud_t *pud;
-       pmd_t *pmd = NULL;
+       pte_t *pte = NULL;
 
        pgd = pgd_offset(mm, addr);
        pud = pud_alloc(mm, pgd, addr);
-       pmd = pmd_alloc(mm, pud, addr);
-       return (pte_t *) pmd;
+       if (pud)
+               pte = (pte_t *) pmd_alloc(mm, pud, addr);
+       BUG_ON(pte && !pte_none(*pte) && !pte_huge(*pte));
+
+       return pte;
 }
 
 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
@@ -37,23 +40,14 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
        pmd_t *pmd = NULL;
 
        pgd = pgd_offset(mm, addr);
-       pud = pud_offset(pgd, addr);
-       pmd = pmd_offset(pud, addr);
+       if (pgd_present(*pgd)) {
+               pud = pud_offset(pgd, addr);
+               if (pud_present(*pud))
+                       pmd = pmd_offset(pud, addr);
+       }
        return (pte_t *) pmd;
 }
 
-/*
- * This function checks for proper alignment of input addr and len parameters.
- */
-int is_aligned_hugepage_range(unsigned long addr, unsigned long len)
-{
-       if (len & ~HPAGE_MASK)
-               return -EINVAL;
-       if (addr & ~HPAGE_MASK)
-               return -EINVAL;
-       return 0;
-}
-
 #if 0  /* This is just for testing */
 struct page *
 follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
@@ -118,17 +112,6 @@ follow_huge_pmd(struct mm_struct *mm, unsigned long address,
 }
 #endif
 
-void hugetlb_clean_stale_pgtable(pte_t *pte)
-{
-       pmd_t *pmd = (pmd_t *) pte;
-       struct page *page;
-
-       page = pmd_page(*pmd);
-       pmd_clear(pmd);
-       dec_page_state(nr_page_table_pages);
-       page_cache_release(page);
-}
-
 /* x86_64 also uses this file */
 
 #ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
@@ -140,7 +123,12 @@ static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
        struct vm_area_struct *vma;
        unsigned long start_addr;
 
-       start_addr = mm->free_area_cache;
+       if (len > mm->cached_hole_size) {
+               start_addr = mm->free_area_cache;
+       } else {
+               start_addr = TASK_UNMAPPED_BASE;
+               mm->cached_hole_size = 0;
+       }
 
 full_search:
        addr = ALIGN(start_addr, HPAGE_SIZE);
@@ -154,6 +142,7 @@ full_search:
                         */
                        if (start_addr != TASK_UNMAPPED_BASE) {
                                start_addr = TASK_UNMAPPED_BASE;
+                               mm->cached_hole_size = 0;
                                goto full_search;
                        }
                        return -ENOMEM;
@@ -162,6 +151,8 @@ full_search:
                        mm->free_area_cache = addr + len;
                        return addr;
                }
+               if (addr + mm->cached_hole_size < vma->vm_start)
+                       mm->cached_hole_size = vma->vm_start - addr;
                addr = ALIGN(vma->vm_end, HPAGE_SIZE);
        }
 }
@@ -173,12 +164,17 @@ static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file,
        struct mm_struct *mm = current->mm;
        struct vm_area_struct *vma, *prev_vma;
        unsigned long base = mm->mmap_base, addr = addr0;
+       unsigned long largest_hole = mm->cached_hole_size;
        int first_time = 1;
 
        /* don't allow allocations above current base */
        if (mm->free_area_cache > base)
                mm->free_area_cache = base;
 
+       if (len <= largest_hole) {
+               largest_hole = 0;
+               mm->free_area_cache  = base;
+       }
 try_again:
        /* make sure it can fit in the remaining address space */
        if (mm->free_area_cache < len)
@@ -199,13 +195,21 @@ try_again:
                 * vma->vm_start, use it:
                 */
                if (addr + len <= vma->vm_start &&
-                               (!prev_vma || (addr >= prev_vma->vm_end)))
+                           (!prev_vma || (addr >= prev_vma->vm_end))) {
                        /* remember the address as a hint for next time */
-                       return (mm->free_area_cache = addr);
-               else
+                       mm->cached_hole_size = largest_hole;
+                       return (mm->free_area_cache = addr);
+               } else {
                        /* pull free_area_cache down to the first hole */
-                       if (mm->free_area_cache == vma->vm_end)
+                       if (mm->free_area_cache == vma->vm_end) {
                                mm->free_area_cache = vma->vm_start;
+                               mm->cached_hole_size = largest_hole;
+                       }
+               }
+
+               /* remember the largest hole we saw so far */
+               if (addr + largest_hole < vma->vm_start)
+                       largest_hole = vma->vm_start - addr;
 
                /* try just below the current vma->vm_start */
                addr = (vma->vm_start - len) & HPAGE_MASK;
@@ -218,6 +222,7 @@ fail:
         */
        if (first_time) {
                mm->free_area_cache = base;
+               largest_hole = 0;
                first_time = 0;
                goto try_again;
        }
@@ -228,6 +233,7 @@ fail:
         * allocations.
         */
        mm->free_area_cache = TASK_UNMAPPED_BASE;
+       mm->cached_hole_size = ~0UL;
        addr = hugetlb_get_unmapped_area_bottomup(file, addr0,
                        len, pgoff, flags);
 
@@ -235,6 +241,7 @@ fail:
         * Restore the topdown base:
         */
        mm->free_area_cache = base;
+       mm->cached_hole_size = ~0UL;
 
        return addr;
 }