mm/mprotect.c

   1 /*
   2  *  mm/mprotect.c
   3  *
   4  *  (C) Copyright 1994 Linus Torvalds
   5  *  (C) Copyright 2002 Christoph Hellwig
   6  *
   7  *  Address space accounting code       <alan@lxorguk.ukuu.org.uk>
   8  *  (C) Copyright 2002 Red Hat Inc, All Rights Reserved
   9  */
  10
  11 #include <linux/mm.h>
  12 #include <linux/hugetlb.h>
  13 #include <linux/shm.h>
  14 #include <linux/mman.h>
  15 #include <linux/fs.h>
  16 #include <linux/highmem.h>
  17 #include <linux/security.h>
  18 #include <linux/mempolicy.h>
  19 #include <linux/personality.h>
  20 #include <linux/syscalls.h>
  21 #include <linux/swap.h>
  22 #include <linux/swapops.h>
  23 #include <linux/mmu_notifier.h>
  24 #include <linux/migrate.h>
  25 #include <linux/perf_event.h>
  26 #include <asm/uaccess.h>
  27 #include <asm/pgtable.h>
  28 #include <asm/cacheflush.h>
  29 #include <asm/tlbflush.h>
  30
  31 static void change_pte_range(struct mm_struct *mm, pmd_t *pmd,
  32                 unsigned long addr, unsigned long end, pgprot_t newprot,
  33                 int dirty_accountable)
  34 {
  35         pte_t *pte, oldpte;
  36         spinlock_t *ptl;
  37
  38         pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
  39         arch_enter_lazy_mmu_mode();
  40         do {
  41                 oldpte = *pte;
  42                 if (pte_present(oldpte)) {
  43                         pte_t ptent;
  44
  45                         ptent = ptep_modify_prot_start(mm, addr, pte);
  46                         ptent = pte_modify(ptent, newprot);
  47
  48                         /*
  49                          * Avoid taking write faults for pages we know to be
  50                          * dirty.
  51                          */
  52                         if (dirty_accountable && pte_dirty(ptent))
  53                                 ptent = pte_mkwrite(ptent);
  54
  55                         ptep_modify_prot_commit(mm, addr, pte, ptent);
  56                 } else if (IS_ENABLED(CONFIG_MIGRATION) && !pte_file(oldpte)) {
  57                         swp_entry_t entry = pte_to_swp_entry(oldpte);
  58
  59                         if (is_write_migration_entry(entry)) {
  60                                 /*
  61                                  * A protection check is difficult so
  62                                  * just be safe and disable write
  63                                  */
  64                                 make_migration_entry_read(&entry);
  65                                 set_pte_at(mm, addr, pte,
  66                                         swp_entry_to_pte(entry));
  67                         }
  68                 }
  69         } while (pte++, addr += PAGE_SIZE, addr != end);
  70         arch_leave_lazy_mmu_mode();
  71         pte_unmap_unlock(pte - 1, ptl);
  72 }
  73
  74 static inline void change_pmd_range(struct vm_area_struct *vma, pud_t *pud,
  75                 unsigned long addr, unsigned long end, pgprot_t newprot,
  76                 int dirty_accountable)
  77 {
  78         pmd_t *pmd;
  79         unsigned long next;
  80
  81         pmd = pmd_offset(pud, addr);
  82         do {
  83                 next = pmd_addr_end(addr, end);
  84                 if (pmd_trans_huge(*pmd)) {
  85                         if (next - addr != HPAGE_PMD_SIZE)
  86                                 split_huge_page_pmd(vma->vm_mm, pmd);
  87                         else if (change_huge_pmd(vma, pmd, addr, newprot))
  88                                 continue;
  89                         /* fall through */
  90                 }
  91                 if (pmd_none_or_clear_bad(pmd))
  92                         continue;
  93                 change_pte_range(vma->vm_mm, pmd, addr, next, newprot,
  94                                  dirty_accountable);
  95         } while (pmd++, addr = next, addr != end);
  96 }
  97
  98 static inline void change_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
  99                 unsigned long addr, unsigned long end, pgprot_t newprot,
 100                 int dirty_accountable)
 101 {
 102         pud_t *pud;
 103         unsigned long next;
 104
 105         pud = pud_offset(pgd, addr);
 106         do {
 107                 next = pud_addr_end(addr, end);
 108                 if (pud_none_or_clear_bad(pud))
 109                         continue;
 110                 change_pmd_range(vma, pud, addr, next, newprot,
 111                                  dirty_accountable);
 112         } while (pud++, addr = next, addr != end);
 113 }
 114
 115 static void change_protection_range(struct vm_area_struct *vma,
 116                 unsigned long addr, unsigned long end, pgprot_t newprot,
 117                 int dirty_accountable)
 118 {
 119         struct mm_struct *mm = vma->vm_mm;
 120         pgd_t *pgd;
 121         unsigned long next;
 122         unsigned long start = addr;
 123
 124         BUG_ON(addr >= end);
 125         pgd = pgd_offset(mm, addr);
 126         flush_cache_range(vma, addr, end);
 127         do {
 128                 next = pgd_addr_end(addr, end);
 129                 if (pgd_none_or_clear_bad(pgd))
 130                         continue;
 131                 change_pud_range(vma, pgd, addr, next, newprot,
 132                                  dirty_accountable);
 133         } while (pgd++, addr = next, addr != end);
 134         flush_tlb_range(vma, start, end);
 135 }
 136
 137 void change_protection(struct vm_area_struct *vma, unsigned long start,
 138                        unsigned long end, pgprot_t newprot,
 139                        int dirty_accountable)
 140 {
 141         struct mm_struct *mm = vma->vm_mm;
 142
 143         mmu_notifier_invalidate_range_start(mm, start, end);
 144         if (is_vm_hugetlb_page(vma))
 145                 hugetlb_change_protection(vma, start, end, newprot);
 146         else
 147                 change_protection_range(vma, start, end, newprot, dirty_accountable);
 148         mmu_notifier_invalidate_range_end(mm, start, end);
 149 }
 150
 151 int
 152 mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev,
 153         unsigned long start, unsigned long end, unsigned long newflags)
 154 {
 155         struct mm_struct *mm = vma->vm_mm;
 156         unsigned long oldflags = vma->vm_flags;
 157         long nrpages = (end - start) >> PAGE_SHIFT;
 158         unsigned long charged = 0;
 159         pgoff_t pgoff;
 160         int error;
 161         int dirty_accountable = 0;
 162
 163         if (newflags == oldflags) {
 164                 *pprev = vma;
 165                 return 0;
 166         }
 167
 168         /*
 169          * If we make a private mapping writable we increase our commit;
 170          * but (without finer accounting) cannot reduce our commit if we
 171          * make it unwritable again. hugetlb mapping were accounted for
 172          * even if read-only so there is no need to account for them here
 173          */
 174         if (newflags & VM_WRITE) {
 175                 if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB|
 176                                                 VM_SHARED|VM_NORESERVE))) {
 177                         charged = nrpages;
 178                         if (security_vm_enough_memory_mm(mm, charged))
 179                                 return -ENOMEM;
 180                         newflags |= VM_ACCOUNT;
 181                 }
 182         }
 183
 184         /*
 185          * First try to merge with previous and/or next vma.
 186          */
 187         pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
 188         *pprev = vma_merge(mm, *pprev, start, end, newflags,
 189                         vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma));
 190         if (*pprev) {
 191                 vma = *pprev;
 192                 goto success;
 193         }
 194
 195         *pprev = vma;
 196
 197         if (start != vma->vm_start) {
 198                 error = split_vma(mm, vma, start, 1);
 199                 if (error)
 200                         goto fail;
 201         }
 202
 203         if (end != vma->vm_end) {
 204                 error = split_vma(mm, vma, end, 0);
 205                 if (error)
 206                         goto fail;
 207         }
 208
 209 success:
 210         /*
 211          * vm_flags and vm_page_prot are protected by the mmap_sem
 212          * held in write mode.
 213          */
 214         vma->vm_flags = newflags;
 215         vma->vm_page_prot = pgprot_modify(vma->vm_page_prot,
 216                                           vm_get_page_prot(newflags));
 217
 218         if (vma_wants_writenotify(vma)) {
 219                 vma->vm_page_prot = vm_get_page_prot(newflags & ~VM_SHARED);
 220                 dirty_accountable = 1;
 221         }
 222
 223         change_protection(vma, start, end, vma->vm_page_prot, dirty_accountable);
 224
 225         vm_stat_account(mm, oldflags, vma->vm_file, -nrpages);
 226         vm_stat_account(mm, newflags, vma->vm_file, nrpages);
 227         perf_event_mmap(vma);
 228         return 0;
 229
 230 fail:
 231         vm_unacct_memory(charged);
 232         return error;
 233 }
 234
 235 SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len,
 236                 unsigned long, prot)
 237 {
 238         unsigned long vm_flags, nstart, end, tmp, reqprot;
 239         struct vm_area_struct *vma, *prev;
 240         int error = -EINVAL;
 241         const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP);
 242         prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP);
 243         if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */
 244                 return -EINVAL;
 245
 246         if (start & ~PAGE_MASK)
 247                 return -EINVAL;
 248         if (!len)
 249                 return 0;
 250         len = PAGE_ALIGN(len);
 251         end = start + len;
 252         if (end <= start)
 253                 return -ENOMEM;
 254         if (!arch_validate_prot(prot))
 255                 return -EINVAL;
 256
 257         reqprot = prot;
 258         /*
 259          * Does the application expect PROT_READ to imply PROT_EXEC:
 260          */
 261         if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
 262                 prot |= PROT_EXEC;
 263
 264         vm_flags = calc_vm_prot_bits(prot);
 265
 266         down_write(&current->mm->mmap_sem);
 267
 268         vma = find_vma(current->mm, start);
 269         error = -ENOMEM;
 270         if (!vma)
 271                 goto out;
 272         prev = vma->vm_prev;
 273         if (unlikely(grows & PROT_GROWSDOWN)) {
 274                 if (vma->vm_start >= end)
 275                         goto out;
 276                 start = vma->vm_start;
 277                 error = -EINVAL;
 278                 if (!(vma->vm_flags & VM_GROWSDOWN))
 279                         goto out;
 280         }
 281         else {
 282                 if (vma->vm_start > start)
 283                         goto out;
 284                 if (unlikely(grows & PROT_GROWSUP)) {
 285                         end = vma->vm_end;
 286                         error = -EINVAL;
 287                         if (!(vma->vm_flags & VM_GROWSUP))
 288                                 goto out;
 289                 }
 290         }
 291         if (start > vma->vm_start)
 292                 prev = vma;
 293
 294         for (nstart = start ; ; ) {
 295                 unsigned long newflags;
 296
 297                 /* Here we know that  vma->vm_start <= nstart < vma->vm_end. */
 298
 299                 newflags = vm_flags | (vma->vm_flags & ~(VM_READ | VM_WRITE | VM_EXEC));
 300
 301                 /* newflags >> 4 shift VM_MAY% in place of VM_% */
 302                 if ((newflags & ~(newflags >> 4)) & (VM_READ | VM_WRITE | VM_EXEC)) {
 303                         error = -EACCES;
 304                         goto out;
 305                 }
 306
 307                 error = security_file_mprotect(vma, reqprot, prot);
 308                 if (error)
 309                         goto out;
 310
 311                 tmp = vma->vm_end;
 312                 if (tmp > end)
 313                         tmp = end;
 314                 error = mprotect_fixup(vma, &prev, nstart, tmp, newflags);
 315                 if (error)
 316                         goto out;
 317                 nstart = tmp;
 318
 319                 if (nstart < prev->vm_end)
 320                         nstart = prev->vm_end;
 321                 if (nstart >= end)
 322                         goto out;
 323
 324                 vma = prev->vm_next;
 325                 if (!vma || vma->vm_start != nstart) {
 326                         error = -ENOMEM;
 327                         goto out;
 328                 }
 329         }
 330 out:
 331         up_write(&current->mm->mmap_sem);
 332         return error;
 333 }