thp: transhuge-memcg: commit tail pages at charge

[mv-sheeva.git] / mm / memcontrol.c

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 356d4964fe95502357efefc13b2b006648c0488a..a1bb59d4c9d012042113fc71d1735e4913dc3ab2 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2094,23 +2094,10 @@ struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
  * commit a charge got by __mem_cgroup_try_charge() and makes page_cgroup to be
  * USED state. If already USED, uncharge and return.
  */
-
-static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
-                                      struct page_cgroup *pc,
-                                      enum charge_type ctype,
-                                      int page_size)
+static void ____mem_cgroup_commit_charge(struct mem_cgroup *mem,
+                                        struct page_cgroup *pc,
+                                        enum charge_type ctype)
 {
-       /* try_charge() can return NULL to *memcg, taking care of it. */
-       if (!mem)
-               return;
-
-       lock_page_cgroup(pc);
-       if (unlikely(PageCgroupUsed(pc))) {
-               unlock_page_cgroup(pc);
-               mem_cgroup_cancel_charge(mem, page_size);
-               return;
-       }
-
        pc->mem_cgroup = mem;
        /*
         * We access a page_cgroup asynchronously without lock_page_cgroup().
@@ -2135,6 +2122,33 @@ static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
        }
 
        mem_cgroup_charge_statistics(mem, pc, true);
+}
+
+static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
+                                      struct page_cgroup *pc,
+                                      enum charge_type ctype,
+                                      int page_size)
+{
+       int i;
+       int count = page_size >> PAGE_SHIFT;
+
+       /* try_charge() can return NULL to *memcg, taking care of it. */
+       if (!mem)
+               return;
+
+       lock_page_cgroup(pc);
+       if (unlikely(PageCgroupUsed(pc))) {
+               unlock_page_cgroup(pc);
+               mem_cgroup_cancel_charge(mem, page_size);
+               return;
+       }
+
+       /*
+        * we don't need page_cgroup_lock about tail pages, becase they are not
+        * accessed by any other context at this point.
+        */
+       for (i = 0; i < count; i++)
+               ____mem_cgroup_commit_charge(mem, pc + i, ctype);
 
        unlock_page_cgroup(pc);
        /*
@@ -2532,6 +2546,8 @@ direct_uncharge:
 static struct mem_cgroup *
 __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
 {
+       int i;
+       int count;
        struct page_cgroup *pc;
        struct mem_cgroup *mem = NULL;
        int page_size = PAGE_SIZE;
@@ -2545,6 +2561,7 @@ __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
        if (PageTransHuge(page))
                page_size <<= compound_order(page);
 
+       count = page_size >> PAGE_SHIFT;
        /*
         * Check if our page_cgroup is valid
         */
@@ -2577,7 +2594,8 @@ __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
                break;
        }
 
-       mem_cgroup_charge_statistics(mem, pc, false);
+       for (i = 0; i < count; i++)
+               mem_cgroup_charge_statistics(mem, pc + i, false);
 
        ClearPageCgroupUsed(pc);
        /*