Merge tag 'perf-core-for-mingo' of git://git.kernel.org/pub/scm/linux/kernel/git...

[karo-tx-linux.git] / mm / shmem.c

diff --git a/mm/shmem.c b/mm/shmem.c
index 3f696f7d9bace4279c033dff173cdfd9f0c3f03b..bd106361be4bf2d080e67f11252198c710745278 100644 (file)
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -263,6 +263,24 @@ static int shmem_radix_tree_replace(struct address_space *mapping,
        return 0;
 }
 
+/*
+ * Sometimes, before we decide whether to proceed or to fail, we must check
+ * that an entry was not already brought back from swap by a racing thread.
+ *
+ * Checking page is not enough: by the time a SwapCache page is locked, it
+ * might be reused, and again be SwapCache, using the same swap as before.
+ */
+static bool shmem_confirm_swap(struct address_space *mapping,
+                              pgoff_t index, swp_entry_t swap)
+{
+       void *item;
+
+       rcu_read_lock();
+       item = radix_tree_lookup(&mapping->page_tree, index);
+       rcu_read_unlock();
+       return item == swp_to_radix_entry(swap);
+}
+
 /*
  * Like add_to_page_cache_locked, but error if expected item has gone.
  */
@@ -270,40 +288,31 @@ static int shmem_add_to_page_cache(struct page *page,
                                   struct address_space *mapping,
                                   pgoff_t index, gfp_t gfp, void *expected)
 {
-       int error = 0;
+       int error;
 
        VM_BUG_ON(!PageLocked(page));
        VM_BUG_ON(!PageSwapBacked(page));
 
+       page_cache_get(page);
+       page->mapping = mapping;
+       page->index = index;
+
+       spin_lock_irq(&mapping->tree_lock);
        if (!expected)
-               error = radix_tree_preload(gfp & GFP_RECLAIM_MASK);
+               error = radix_tree_insert(&mapping->page_tree, index, page);
+       else
+               error = shmem_radix_tree_replace(mapping, index, expected,
+                                                                page);
        if (!error) {
-               page_cache_get(page);
-               page->mapping = mapping;
-               page->index = index;
-
-               spin_lock_irq(&mapping->tree_lock);
-               if (!expected)
-                       error = radix_tree_insert(&mapping->page_tree,
-                                                       index, page);
-               else
-                       error = shmem_radix_tree_replace(mapping, index,
-                                                       expected, page);
-               if (!error) {
-                       mapping->nrpages++;
-                       __inc_zone_page_state(page, NR_FILE_PAGES);
-                       __inc_zone_page_state(page, NR_SHMEM);
-                       spin_unlock_irq(&mapping->tree_lock);
-               } else {
-                       page->mapping = NULL;
-                       spin_unlock_irq(&mapping->tree_lock);
-                       page_cache_release(page);
-               }
-               if (!expected)
-                       radix_tree_preload_end();
+               mapping->nrpages++;
+               __inc_zone_page_state(page, NR_FILE_PAGES);
+               __inc_zone_page_state(page, NR_SHMEM);
+               spin_unlock_irq(&mapping->tree_lock);
+       } else {
+               page->mapping = NULL;
+               spin_unlock_irq(&mapping->tree_lock);
+               page_cache_release(page);
        }
-       if (error)
-               mem_cgroup_uncharge_cache_page(page);
        return error;
 }
 
@@ -1124,9 +1133,9 @@ repeat:
                /* We have to do this with page locked to prevent races */
                lock_page(page);
                if (!PageSwapCache(page) || page_private(page) != swap.val ||
-                   page->mapping) {
+                   !shmem_confirm_swap(mapping, index, swap)) {
                        error = -EEXIST;        /* try again */
-                       goto failed;
+                       goto unlock;
                }
                if (!PageUptodate(page)) {
                        error = -EIO;
@@ -1142,9 +1151,12 @@ repeat:
 
                error = mem_cgroup_cache_charge(page, current->mm,
                                                gfp & GFP_RECLAIM_MASK);
-               if (!error)
+               if (!error) {
                        error = shmem_add_to_page_cache(page, mapping, index,
                                                gfp, swp_to_radix_entry(swap));
+                       /* We already confirmed swap, and make no allocation */
+                       VM_BUG_ON(error);
+               }
                if (error)
                        goto failed;
 
@@ -1181,11 +1193,18 @@ repeat:
                __set_page_locked(page);
                error = mem_cgroup_cache_charge(page, current->mm,
                                                gfp & GFP_RECLAIM_MASK);
-               if (!error)
-                       error = shmem_add_to_page_cache(page, mapping, index,
-                                               gfp, NULL);
                if (error)
                        goto decused;
+               error = radix_tree_preload(gfp & GFP_RECLAIM_MASK);
+               if (!error) {
+                       error = shmem_add_to_page_cache(page, mapping, index,
+                                                       gfp, NULL);
+                       radix_tree_preload_end();
+               }
+               if (error) {
+                       mem_cgroup_uncharge_cache_page(page);
+                       goto decused;
+               }
                lru_cache_add_anon(page);
 
                spin_lock(&info->lock);
@@ -1245,14 +1264,10 @@ decused:
 unacct:
        shmem_unacct_blocks(info->flags, 1);
 failed:
-       if (swap.val && error != -EINVAL) {
-               struct page *test = find_get_page(mapping, index);
-               if (test && !radix_tree_exceptional_entry(test))
-                       page_cache_release(test);
-               /* Have another try if the entry has changed */
-               if (test != swp_to_radix_entry(swap))
-                       error = -EEXIST;
-       }
+       if (swap.val && error != -EINVAL &&
+           !shmem_confirm_swap(mapping, index, swap))
+               error = -EEXIST;
+unlock:
        if (page) {
                unlock_page(page);
                page_cache_release(page);
@@ -1264,7 +1279,7 @@ failed:
                spin_unlock(&info->lock);
                goto repeat;
        }
-       if (error == -EEXIST)
+       if (error == -EEXIST)   /* from above or from radix_tree_insert */
                goto repeat;
        return error;
 }