1 /* memcontrol.c - Memory Controller
3 * Copyright IBM Corporation, 2007
4 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
6 * Copyright 2007 OpenVZ SWsoft Inc
7 * Author: Pavel Emelianov <xemul@openvz.org>
10 * Copyright (C) 2009 Nokia Corporation
11 * Author: Kirill A. Shutemov
13 * Kernel Memory Controller
14 * Copyright (C) 2012 Parallels Inc. and Google Inc.
15 * Authors: Glauber Costa and Suleiman Souhlal
18 * Charge lifetime sanitation
19 * Lockless page tracking & accounting
20 * Unified hierarchy configuration model
21 * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
23 * This program is free software; you can redistribute it and/or modify
24 * it under the terms of the GNU General Public License as published by
25 * the Free Software Foundation; either version 2 of the License, or
26 * (at your option) any later version.
28 * This program is distributed in the hope that it will be useful,
29 * but WITHOUT ANY WARRANTY; without even the implied warranty of
30 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
31 * GNU General Public License for more details.
34 #include <linux/page_counter.h>
35 #include <linux/memcontrol.h>
36 #include <linux/cgroup.h>
38 #include <linux/hugetlb.h>
39 #include <linux/pagemap.h>
40 #include <linux/smp.h>
41 #include <linux/page-flags.h>
42 #include <linux/backing-dev.h>
43 #include <linux/bit_spinlock.h>
44 #include <linux/rcupdate.h>
45 #include <linux/limits.h>
46 #include <linux/export.h>
47 #include <linux/mutex.h>
48 #include <linux/rbtree.h>
49 #include <linux/slab.h>
50 #include <linux/swap.h>
51 #include <linux/swapops.h>
52 #include <linux/spinlock.h>
53 #include <linux/eventfd.h>
54 #include <linux/poll.h>
55 #include <linux/sort.h>
57 #include <linux/seq_file.h>
58 #include <linux/vmpressure.h>
59 #include <linux/mm_inline.h>
60 #include <linux/swap_cgroup.h>
61 #include <linux/cpu.h>
62 #include <linux/oom.h>
63 #include <linux/lockdep.h>
64 #include <linux/file.h>
65 #include <linux/tracehook.h>
71 #include <asm/uaccess.h>
73 #include <trace/events/vmscan.h>
75 struct cgroup_subsys memory_cgrp_subsys __read_mostly;
76 EXPORT_SYMBOL(memory_cgrp_subsys);
78 struct mem_cgroup *root_mem_cgroup __read_mostly;
80 #define MEM_CGROUP_RECLAIM_RETRIES 5
82 /* Socket memory accounting disabled? */
83 static bool cgroup_memory_nosocket;
85 /* Kernel memory accounting disabled? */
86 static bool cgroup_memory_nokmem;
88 /* Whether the swap controller is active */
89 #ifdef CONFIG_MEMCG_SWAP
90 int do_swap_account __read_mostly;
92 #define do_swap_account 0
95 /* Whether legacy memory+swap accounting is active */
96 static bool do_memsw_account(void)
98 return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && do_swap_account;
101 static const char * const mem_cgroup_stat_names[] = {
111 static const char * const mem_cgroup_events_names[] = {
118 static const char * const mem_cgroup_lru_names[] = {
126 #define THRESHOLDS_EVENTS_TARGET 128
127 #define SOFTLIMIT_EVENTS_TARGET 1024
128 #define NUMAINFO_EVENTS_TARGET 1024
131 * Cgroups above their limits are maintained in a RB-Tree, independent of
132 * their hierarchy representation
135 struct mem_cgroup_tree_per_node {
136 struct rb_root rb_root;
140 struct mem_cgroup_tree {
141 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
144 static struct mem_cgroup_tree soft_limit_tree __read_mostly;
147 struct mem_cgroup_eventfd_list {
148 struct list_head list;
149 struct eventfd_ctx *eventfd;
153 * cgroup_event represents events which userspace want to receive.
155 struct mem_cgroup_event {
157 * memcg which the event belongs to.
159 struct mem_cgroup *memcg;
161 * eventfd to signal userspace about the event.
163 struct eventfd_ctx *eventfd;
165 * Each of these stored in a list by the cgroup.
167 struct list_head list;
169 * register_event() callback will be used to add new userspace
170 * waiter for changes related to this event. Use eventfd_signal()
171 * on eventfd to send notification to userspace.
173 int (*register_event)(struct mem_cgroup *memcg,
174 struct eventfd_ctx *eventfd, const char *args);
176 * unregister_event() callback will be called when userspace closes
177 * the eventfd or on cgroup removing. This callback must be set,
178 * if you want provide notification functionality.
180 void (*unregister_event)(struct mem_cgroup *memcg,
181 struct eventfd_ctx *eventfd);
183 * All fields below needed to unregister event when
184 * userspace closes eventfd.
187 wait_queue_head_t *wqh;
189 struct work_struct remove;
192 static void mem_cgroup_threshold(struct mem_cgroup *memcg);
193 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
195 /* Stuffs for move charges at task migration. */
197 * Types of charges to be moved.
199 #define MOVE_ANON 0x1U
200 #define MOVE_FILE 0x2U
201 #define MOVE_MASK (MOVE_ANON | MOVE_FILE)
203 /* "mc" and its members are protected by cgroup_mutex */
204 static struct move_charge_struct {
205 spinlock_t lock; /* for from, to */
206 struct mm_struct *mm;
207 struct mem_cgroup *from;
208 struct mem_cgroup *to;
210 unsigned long precharge;
211 unsigned long moved_charge;
212 unsigned long moved_swap;
213 struct task_struct *moving_task; /* a task moving charges */
214 wait_queue_head_t waitq; /* a waitq for other context */
216 .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
217 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
221 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
222 * limit reclaim to prevent infinite loops, if they ever occur.
224 #define MEM_CGROUP_MAX_RECLAIM_LOOPS 100
225 #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
228 MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
229 MEM_CGROUP_CHARGE_TYPE_ANON,
230 MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
231 MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */
235 /* for encoding cft->private value on file */
244 #define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
245 #define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff)
246 #define MEMFILE_ATTR(val) ((val) & 0xffff)
247 /* Used for OOM nofiier */
248 #define OOM_CONTROL (0)
250 /* Some nice accessors for the vmpressure. */
251 struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
254 memcg = root_mem_cgroup;
255 return &memcg->vmpressure;
258 struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
260 return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
263 static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
265 return (memcg == root_mem_cgroup);
270 * This will be the memcg's index in each cache's ->memcg_params.memcg_caches.
271 * The main reason for not using cgroup id for this:
272 * this works better in sparse environments, where we have a lot of memcgs,
273 * but only a few kmem-limited. Or also, if we have, for instance, 200
274 * memcgs, and none but the 200th is kmem-limited, we'd have to have a
275 * 200 entry array for that.
277 * The current size of the caches array is stored in memcg_nr_cache_ids. It
278 * will double each time we have to increase it.
280 static DEFINE_IDA(memcg_cache_ida);
281 int memcg_nr_cache_ids;
283 /* Protects memcg_nr_cache_ids */
284 static DECLARE_RWSEM(memcg_cache_ids_sem);
286 void memcg_get_cache_ids(void)
288 down_read(&memcg_cache_ids_sem);
291 void memcg_put_cache_ids(void)
293 up_read(&memcg_cache_ids_sem);
297 * MIN_SIZE is different than 1, because we would like to avoid going through
298 * the alloc/free process all the time. In a small machine, 4 kmem-limited
299 * cgroups is a reasonable guess. In the future, it could be a parameter or
300 * tunable, but that is strictly not necessary.
302 * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get
303 * this constant directly from cgroup, but it is understandable that this is
304 * better kept as an internal representation in cgroup.c. In any case, the
305 * cgrp_id space is not getting any smaller, and we don't have to necessarily
306 * increase ours as well if it increases.
308 #define MEMCG_CACHES_MIN_SIZE 4
309 #define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
312 * A lot of the calls to the cache allocation functions are expected to be
313 * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
314 * conditional to this static branch, we'll have to allow modules that does
315 * kmem_cache_alloc and the such to see this symbol as well
317 DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key);
318 EXPORT_SYMBOL(memcg_kmem_enabled_key);
320 #endif /* !CONFIG_SLOB */
323 * mem_cgroup_css_from_page - css of the memcg associated with a page
324 * @page: page of interest
326 * If memcg is bound to the default hierarchy, css of the memcg associated
327 * with @page is returned. The returned css remains associated with @page
328 * until it is released.
330 * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
333 struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
335 struct mem_cgroup *memcg;
337 memcg = page->mem_cgroup;
339 if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
340 memcg = root_mem_cgroup;
346 * page_cgroup_ino - return inode number of the memcg a page is charged to
349 * Look up the closest online ancestor of the memory cgroup @page is charged to
350 * and return its inode number or 0 if @page is not charged to any cgroup. It
351 * is safe to call this function without holding a reference to @page.
353 * Note, this function is inherently racy, because there is nothing to prevent
354 * the cgroup inode from getting torn down and potentially reallocated a moment
355 * after page_cgroup_ino() returns, so it only should be used by callers that
356 * do not care (such as procfs interfaces).
358 ino_t page_cgroup_ino(struct page *page)
360 struct mem_cgroup *memcg;
361 unsigned long ino = 0;
364 memcg = READ_ONCE(page->mem_cgroup);
365 while (memcg && !(memcg->css.flags & CSS_ONLINE))
366 memcg = parent_mem_cgroup(memcg);
368 ino = cgroup_ino(memcg->css.cgroup);
373 static struct mem_cgroup_per_node *
374 mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page)
376 int nid = page_to_nid(page);
378 return memcg->nodeinfo[nid];
381 static struct mem_cgroup_tree_per_node *
382 soft_limit_tree_node(int nid)
384 return soft_limit_tree.rb_tree_per_node[nid];
387 static struct mem_cgroup_tree_per_node *
388 soft_limit_tree_from_page(struct page *page)
390 int nid = page_to_nid(page);
392 return soft_limit_tree.rb_tree_per_node[nid];
395 static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
396 struct mem_cgroup_tree_per_node *mctz,
397 unsigned long new_usage_in_excess)
399 struct rb_node **p = &mctz->rb_root.rb_node;
400 struct rb_node *parent = NULL;
401 struct mem_cgroup_per_node *mz_node;
406 mz->usage_in_excess = new_usage_in_excess;
407 if (!mz->usage_in_excess)
411 mz_node = rb_entry(parent, struct mem_cgroup_per_node,
413 if (mz->usage_in_excess < mz_node->usage_in_excess)
416 * We can't avoid mem cgroups that are over their soft
417 * limit by the same amount
419 else if (mz->usage_in_excess >= mz_node->usage_in_excess)
422 rb_link_node(&mz->tree_node, parent, p);
423 rb_insert_color(&mz->tree_node, &mctz->rb_root);
427 static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
428 struct mem_cgroup_tree_per_node *mctz)
432 rb_erase(&mz->tree_node, &mctz->rb_root);
436 static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
437 struct mem_cgroup_tree_per_node *mctz)
441 spin_lock_irqsave(&mctz->lock, flags);
442 __mem_cgroup_remove_exceeded(mz, mctz);
443 spin_unlock_irqrestore(&mctz->lock, flags);
446 static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
448 unsigned long nr_pages = page_counter_read(&memcg->memory);
449 unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
450 unsigned long excess = 0;
452 if (nr_pages > soft_limit)
453 excess = nr_pages - soft_limit;
458 static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
460 unsigned long excess;
461 struct mem_cgroup_per_node *mz;
462 struct mem_cgroup_tree_per_node *mctz;
464 mctz = soft_limit_tree_from_page(page);
466 * Necessary to update all ancestors when hierarchy is used.
467 * because their event counter is not touched.
469 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
470 mz = mem_cgroup_page_nodeinfo(memcg, page);
471 excess = soft_limit_excess(memcg);
473 * We have to update the tree if mz is on RB-tree or
474 * mem is over its softlimit.
476 if (excess || mz->on_tree) {
479 spin_lock_irqsave(&mctz->lock, flags);
480 /* if on-tree, remove it */
482 __mem_cgroup_remove_exceeded(mz, mctz);
484 * Insert again. mz->usage_in_excess will be updated.
485 * If excess is 0, no tree ops.
487 __mem_cgroup_insert_exceeded(mz, mctz, excess);
488 spin_unlock_irqrestore(&mctz->lock, flags);
493 static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
495 struct mem_cgroup_tree_per_node *mctz;
496 struct mem_cgroup_per_node *mz;
500 mz = mem_cgroup_nodeinfo(memcg, nid);
501 mctz = soft_limit_tree_node(nid);
502 mem_cgroup_remove_exceeded(mz, mctz);
506 static struct mem_cgroup_per_node *
507 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
509 struct rb_node *rightmost = NULL;
510 struct mem_cgroup_per_node *mz;
514 rightmost = rb_last(&mctz->rb_root);
516 goto done; /* Nothing to reclaim from */
518 mz = rb_entry(rightmost, struct mem_cgroup_per_node, tree_node);
520 * Remove the node now but someone else can add it back,
521 * we will to add it back at the end of reclaim to its correct
522 * position in the tree.
524 __mem_cgroup_remove_exceeded(mz, mctz);
525 if (!soft_limit_excess(mz->memcg) ||
526 !css_tryget_online(&mz->memcg->css))
532 static struct mem_cgroup_per_node *
533 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
535 struct mem_cgroup_per_node *mz;
537 spin_lock_irq(&mctz->lock);
538 mz = __mem_cgroup_largest_soft_limit_node(mctz);
539 spin_unlock_irq(&mctz->lock);
544 * Return page count for single (non recursive) @memcg.
546 * Implementation Note: reading percpu statistics for memcg.
548 * Both of vmstat[] and percpu_counter has threshold and do periodic
549 * synchronization to implement "quick" read. There are trade-off between
550 * reading cost and precision of value. Then, we may have a chance to implement
551 * a periodic synchronization of counter in memcg's counter.
553 * But this _read() function is used for user interface now. The user accounts
554 * memory usage by memory cgroup and he _always_ requires exact value because
555 * he accounts memory. Even if we provide quick-and-fuzzy read, we always
556 * have to visit all online cpus and make sum. So, for now, unnecessary
557 * synchronization is not implemented. (just implemented for cpu hotplug)
559 * If there are kernel internal actions which can make use of some not-exact
560 * value, and reading all cpu value can be performance bottleneck in some
561 * common workload, threshold and synchronization as vmstat[] should be
565 mem_cgroup_read_stat(struct mem_cgroup *memcg, enum mem_cgroup_stat_index idx)
570 /* Per-cpu values can be negative, use a signed accumulator */
571 for_each_possible_cpu(cpu)
572 val += per_cpu(memcg->stat->count[idx], cpu);
574 * Summing races with updates, so val may be negative. Avoid exposing
575 * transient negative values.
582 static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg,
583 enum mem_cgroup_events_index idx)
585 unsigned long val = 0;
588 for_each_possible_cpu(cpu)
589 val += per_cpu(memcg->stat->events[idx], cpu);
593 static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
595 bool compound, int nr_pages)
598 * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
599 * counted as CACHE even if it's on ANON LRU.
602 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS],
605 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE],
609 VM_BUG_ON_PAGE(!PageTransHuge(page), page);
610 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
614 /* pagein of a big page is an event. So, ignore page size */
616 __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
618 __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
619 nr_pages = -nr_pages; /* for event */
622 __this_cpu_add(memcg->stat->nr_page_events, nr_pages);
625 unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
626 int nid, unsigned int lru_mask)
628 unsigned long nr = 0;
629 struct mem_cgroup_per_node *mz;
632 VM_BUG_ON((unsigned)nid >= nr_node_ids);
635 if (!(BIT(lru) & lru_mask))
637 mz = mem_cgroup_nodeinfo(memcg, nid);
638 nr += mz->lru_size[lru];
643 static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
644 unsigned int lru_mask)
646 unsigned long nr = 0;
649 for_each_node_state(nid, N_MEMORY)
650 nr += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
654 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
655 enum mem_cgroup_events_target target)
657 unsigned long val, next;
659 val = __this_cpu_read(memcg->stat->nr_page_events);
660 next = __this_cpu_read(memcg->stat->targets[target]);
661 /* from time_after() in jiffies.h */
662 if ((long)next - (long)val < 0) {
664 case MEM_CGROUP_TARGET_THRESH:
665 next = val + THRESHOLDS_EVENTS_TARGET;
667 case MEM_CGROUP_TARGET_SOFTLIMIT:
668 next = val + SOFTLIMIT_EVENTS_TARGET;
670 case MEM_CGROUP_TARGET_NUMAINFO:
671 next = val + NUMAINFO_EVENTS_TARGET;
676 __this_cpu_write(memcg->stat->targets[target], next);
683 * Check events in order.
686 static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
688 /* threshold event is triggered in finer grain than soft limit */
689 if (unlikely(mem_cgroup_event_ratelimit(memcg,
690 MEM_CGROUP_TARGET_THRESH))) {
692 bool do_numainfo __maybe_unused;
694 do_softlimit = mem_cgroup_event_ratelimit(memcg,
695 MEM_CGROUP_TARGET_SOFTLIMIT);
697 do_numainfo = mem_cgroup_event_ratelimit(memcg,
698 MEM_CGROUP_TARGET_NUMAINFO);
700 mem_cgroup_threshold(memcg);
701 if (unlikely(do_softlimit))
702 mem_cgroup_update_tree(memcg, page);
704 if (unlikely(do_numainfo))
705 atomic_inc(&memcg->numainfo_events);
710 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
713 * mm_update_next_owner() may clear mm->owner to NULL
714 * if it races with swapoff, page migration, etc.
715 * So this can be called with p == NULL.
720 return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
722 EXPORT_SYMBOL(mem_cgroup_from_task);
724 static struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
726 struct mem_cgroup *memcg = NULL;
731 * Page cache insertions can happen withou an
732 * actual mm context, e.g. during disk probing
733 * on boot, loopback IO, acct() writes etc.
736 memcg = root_mem_cgroup;
738 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
739 if (unlikely(!memcg))
740 memcg = root_mem_cgroup;
742 } while (!css_tryget_online(&memcg->css));
748 * mem_cgroup_iter - iterate over memory cgroup hierarchy
749 * @root: hierarchy root
750 * @prev: previously returned memcg, NULL on first invocation
751 * @reclaim: cookie for shared reclaim walks, NULL for full walks
753 * Returns references to children of the hierarchy below @root, or
754 * @root itself, or %NULL after a full round-trip.
756 * Caller must pass the return value in @prev on subsequent
757 * invocations for reference counting, or use mem_cgroup_iter_break()
758 * to cancel a hierarchy walk before the round-trip is complete.
760 * Reclaimers can specify a zone and a priority level in @reclaim to
761 * divide up the memcgs in the hierarchy among all concurrent
762 * reclaimers operating on the same zone and priority.
764 struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
765 struct mem_cgroup *prev,
766 struct mem_cgroup_reclaim_cookie *reclaim)
768 struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
769 struct cgroup_subsys_state *css = NULL;
770 struct mem_cgroup *memcg = NULL;
771 struct mem_cgroup *pos = NULL;
773 if (mem_cgroup_disabled())
777 root = root_mem_cgroup;
779 if (prev && !reclaim)
782 if (!root->use_hierarchy && root != root_mem_cgroup) {
791 struct mem_cgroup_per_node *mz;
793 mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id);
794 iter = &mz->iter[reclaim->priority];
796 if (prev && reclaim->generation != iter->generation)
800 pos = READ_ONCE(iter->position);
801 if (!pos || css_tryget(&pos->css))
804 * css reference reached zero, so iter->position will
805 * be cleared by ->css_released. However, we should not
806 * rely on this happening soon, because ->css_released
807 * is called from a work queue, and by busy-waiting we
808 * might block it. So we clear iter->position right
811 (void)cmpxchg(&iter->position, pos, NULL);
819 css = css_next_descendant_pre(css, &root->css);
822 * Reclaimers share the hierarchy walk, and a
823 * new one might jump in right at the end of
824 * the hierarchy - make sure they see at least
825 * one group and restart from the beginning.
833 * Verify the css and acquire a reference. The root
834 * is provided by the caller, so we know it's alive
835 * and kicking, and don't take an extra reference.
837 memcg = mem_cgroup_from_css(css);
839 if (css == &root->css)
850 * The position could have already been updated by a competing
851 * thread, so check that the value hasn't changed since we read
852 * it to avoid reclaiming from the same cgroup twice.
854 (void)cmpxchg(&iter->position, pos, memcg);
862 reclaim->generation = iter->generation;
868 if (prev && prev != root)
875 * mem_cgroup_iter_break - abort a hierarchy walk prematurely
876 * @root: hierarchy root
877 * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
879 void mem_cgroup_iter_break(struct mem_cgroup *root,
880 struct mem_cgroup *prev)
883 root = root_mem_cgroup;
884 if (prev && prev != root)
888 static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
890 struct mem_cgroup *memcg = dead_memcg;
891 struct mem_cgroup_reclaim_iter *iter;
892 struct mem_cgroup_per_node *mz;
896 while ((memcg = parent_mem_cgroup(memcg))) {
898 mz = mem_cgroup_nodeinfo(memcg, nid);
899 for (i = 0; i <= DEF_PRIORITY; i++) {
901 cmpxchg(&iter->position,
909 * Iteration constructs for visiting all cgroups (under a tree). If
910 * loops are exited prematurely (break), mem_cgroup_iter_break() must
911 * be used for reference counting.
913 #define for_each_mem_cgroup_tree(iter, root) \
914 for (iter = mem_cgroup_iter(root, NULL, NULL); \
916 iter = mem_cgroup_iter(root, iter, NULL))
918 #define for_each_mem_cgroup(iter) \
919 for (iter = mem_cgroup_iter(NULL, NULL, NULL); \
921 iter = mem_cgroup_iter(NULL, iter, NULL))
924 * mem_cgroup_page_lruvec - return lruvec for isolating/putting an LRU page
926 * @zone: zone of the page
928 * This function is only safe when following the LRU page isolation
929 * and putback protocol: the LRU lock must be held, and the page must
930 * either be PageLRU() or the caller must have isolated/allocated it.
932 struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct pglist_data *pgdat)
934 struct mem_cgroup_per_node *mz;
935 struct mem_cgroup *memcg;
936 struct lruvec *lruvec;
938 if (mem_cgroup_disabled()) {
939 lruvec = &pgdat->lruvec;
943 memcg = page->mem_cgroup;
945 * Swapcache readahead pages are added to the LRU - and
946 * possibly migrated - before they are charged.
949 memcg = root_mem_cgroup;
951 mz = mem_cgroup_page_nodeinfo(memcg, page);
952 lruvec = &mz->lruvec;
955 * Since a node can be onlined after the mem_cgroup was created,
956 * we have to be prepared to initialize lruvec->zone here;
957 * and if offlined then reonlined, we need to reinitialize it.
959 if (unlikely(lruvec->pgdat != pgdat))
960 lruvec->pgdat = pgdat;
965 * mem_cgroup_update_lru_size - account for adding or removing an lru page
966 * @lruvec: mem_cgroup per zone lru vector
967 * @lru: index of lru list the page is sitting on
968 * @nr_pages: positive when adding or negative when removing
970 * This function must be called under lru_lock, just before a page is added
971 * to or just after a page is removed from an lru list (that ordering being
972 * so as to allow it to check that lru_size 0 is consistent with list_empty).
974 void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
977 struct mem_cgroup_per_node *mz;
978 unsigned long *lru_size;
982 if (mem_cgroup_disabled())
985 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
986 lru_size = mz->lru_size + lru;
987 empty = list_empty(lruvec->lists + lru);
990 *lru_size += nr_pages;
993 if (WARN_ONCE(size < 0 || empty != !size,
994 "%s(%p, %d, %d): lru_size %ld but %sempty\n",
995 __func__, lruvec, lru, nr_pages, size, empty ? "" : "not ")) {
1001 *lru_size += nr_pages;
1004 bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg)
1006 struct mem_cgroup *task_memcg;
1007 struct task_struct *p;
1010 p = find_lock_task_mm(task);
1012 task_memcg = get_mem_cgroup_from_mm(p->mm);
1016 * All threads may have already detached their mm's, but the oom
1017 * killer still needs to detect if they have already been oom
1018 * killed to prevent needlessly killing additional tasks.
1021 task_memcg = mem_cgroup_from_task(task);
1022 css_get(&task_memcg->css);
1025 ret = mem_cgroup_is_descendant(task_memcg, memcg);
1026 css_put(&task_memcg->css);
1031 * mem_cgroup_margin - calculate chargeable space of a memory cgroup
1032 * @memcg: the memory cgroup
1034 * Returns the maximum amount of memory @mem can be charged with, in
1037 static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
1039 unsigned long margin = 0;
1040 unsigned long count;
1041 unsigned long limit;
1043 count = page_counter_read(&memcg->memory);
1044 limit = READ_ONCE(memcg->memory.limit);
1046 margin = limit - count;
1048 if (do_memsw_account()) {
1049 count = page_counter_read(&memcg->memsw);
1050 limit = READ_ONCE(memcg->memsw.limit);
1052 margin = min(margin, limit - count);
1061 * A routine for checking "mem" is under move_account() or not.
1063 * Checking a cgroup is mc.from or mc.to or under hierarchy of
1064 * moving cgroups. This is for waiting at high-memory pressure
1067 static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
1069 struct mem_cgroup *from;
1070 struct mem_cgroup *to;
1073 * Unlike task_move routines, we access mc.to, mc.from not under
1074 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1076 spin_lock(&mc.lock);
1082 ret = mem_cgroup_is_descendant(from, memcg) ||
1083 mem_cgroup_is_descendant(to, memcg);
1085 spin_unlock(&mc.lock);
1089 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
1091 if (mc.moving_task && current != mc.moving_task) {
1092 if (mem_cgroup_under_move(memcg)) {
1094 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1095 /* moving charge context might have finished. */
1098 finish_wait(&mc.waitq, &wait);
1105 #define K(x) ((x) << (PAGE_SHIFT-10))
1107 * mem_cgroup_print_oom_info: Print OOM information relevant to memory controller.
1108 * @memcg: The memory cgroup that went over limit
1109 * @p: Task that is going to be killed
1111 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1114 void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
1116 struct mem_cgroup *iter;
1122 pr_info("Task in ");
1123 pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
1124 pr_cont(" killed as a result of limit of ");
1126 pr_info("Memory limit reached of cgroup ");
1129 pr_cont_cgroup_path(memcg->css.cgroup);
1134 pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
1135 K((u64)page_counter_read(&memcg->memory)),
1136 K((u64)memcg->memory.limit), memcg->memory.failcnt);
1137 pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
1138 K((u64)page_counter_read(&memcg->memsw)),
1139 K((u64)memcg->memsw.limit), memcg->memsw.failcnt);
1140 pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
1141 K((u64)page_counter_read(&memcg->kmem)),
1142 K((u64)memcg->kmem.limit), memcg->kmem.failcnt);
1144 for_each_mem_cgroup_tree(iter, memcg) {
1145 pr_info("Memory cgroup stats for ");
1146 pr_cont_cgroup_path(iter->css.cgroup);
1149 for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
1150 if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
1152 pr_cont(" %s:%luKB", mem_cgroup_stat_names[i],
1153 K(mem_cgroup_read_stat(iter, i)));
1156 for (i = 0; i < NR_LRU_LISTS; i++)
1157 pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
1158 K(mem_cgroup_nr_lru_pages(iter, BIT(i))));
1165 * This function returns the number of memcg under hierarchy tree. Returns
1166 * 1(self count) if no children.
1168 static int mem_cgroup_count_children(struct mem_cgroup *memcg)
1171 struct mem_cgroup *iter;
1173 for_each_mem_cgroup_tree(iter, memcg)
1179 * Return the memory (and swap, if configured) limit for a memcg.
1181 static unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg)
1183 unsigned long limit;
1185 limit = memcg->memory.limit;
1186 if (mem_cgroup_swappiness(memcg)) {
1187 unsigned long memsw_limit;
1188 unsigned long swap_limit;
1190 memsw_limit = memcg->memsw.limit;
1191 swap_limit = memcg->swap.limit;
1192 swap_limit = min(swap_limit, (unsigned long)total_swap_pages);
1193 limit = min(limit + swap_limit, memsw_limit);
1198 static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1201 struct oom_control oc = {
1205 .gfp_mask = gfp_mask,
1208 struct mem_cgroup *iter;
1209 unsigned long chosen_points = 0;
1210 unsigned long totalpages;
1211 unsigned int points = 0;
1212 struct task_struct *chosen = NULL;
1214 mutex_lock(&oom_lock);
1217 * If current has a pending SIGKILL or is exiting, then automatically
1218 * select it. The goal is to allow it to allocate so that it may
1219 * quickly exit and free its memory.
1221 if (task_will_free_mem(current)) {
1222 mark_oom_victim(current);
1223 wake_oom_reaper(current);
1227 check_panic_on_oom(&oc, CONSTRAINT_MEMCG);
1228 totalpages = mem_cgroup_get_limit(memcg) ? : 1;
1229 for_each_mem_cgroup_tree(iter, memcg) {
1230 struct css_task_iter it;
1231 struct task_struct *task;
1233 css_task_iter_start(&iter->css, &it);
1234 while ((task = css_task_iter_next(&it))) {
1235 switch (oom_scan_process_thread(&oc, task)) {
1236 case OOM_SCAN_SELECT:
1238 put_task_struct(chosen);
1240 chosen_points = ULONG_MAX;
1241 get_task_struct(chosen);
1243 case OOM_SCAN_CONTINUE:
1245 case OOM_SCAN_ABORT:
1246 css_task_iter_end(&it);
1247 mem_cgroup_iter_break(memcg, iter);
1249 put_task_struct(chosen);
1250 /* Set a dummy value to return "true". */
1251 chosen = (void *) 1;
1256 points = oom_badness(task, memcg, NULL, totalpages);
1257 if (!points || points < chosen_points)
1259 /* Prefer thread group leaders for display purposes */
1260 if (points == chosen_points &&
1261 thread_group_leader(chosen))
1265 put_task_struct(chosen);
1267 chosen_points = points;
1268 get_task_struct(chosen);
1270 css_task_iter_end(&it);
1274 points = chosen_points * 1000 / totalpages;
1275 oom_kill_process(&oc, chosen, points, totalpages,
1276 "Memory cgroup out of memory");
1279 mutex_unlock(&oom_lock);
1283 #if MAX_NUMNODES > 1
1286 * test_mem_cgroup_node_reclaimable
1287 * @memcg: the target memcg
1288 * @nid: the node ID to be checked.
1289 * @noswap : specify true here if the user wants flle only information.
1291 * This function returns whether the specified memcg contains any
1292 * reclaimable pages on a node. Returns true if there are any reclaimable
1293 * pages in the node.
1295 static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
1296 int nid, bool noswap)
1298 if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
1300 if (noswap || !total_swap_pages)
1302 if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
1309 * Always updating the nodemask is not very good - even if we have an empty
1310 * list or the wrong list here, we can start from some node and traverse all
1311 * nodes based on the zonelist. So update the list loosely once per 10 secs.
1314 static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
1318 * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
1319 * pagein/pageout changes since the last update.
1321 if (!atomic_read(&memcg->numainfo_events))
1323 if (atomic_inc_return(&memcg->numainfo_updating) > 1)
1326 /* make a nodemask where this memcg uses memory from */
1327 memcg->scan_nodes = node_states[N_MEMORY];
1329 for_each_node_mask(nid, node_states[N_MEMORY]) {
1331 if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
1332 node_clear(nid, memcg->scan_nodes);
1335 atomic_set(&memcg->numainfo_events, 0);
1336 atomic_set(&memcg->numainfo_updating, 0);
1340 * Selecting a node where we start reclaim from. Because what we need is just
1341 * reducing usage counter, start from anywhere is O,K. Considering
1342 * memory reclaim from current node, there are pros. and cons.
1344 * Freeing memory from current node means freeing memory from a node which
1345 * we'll use or we've used. So, it may make LRU bad. And if several threads
1346 * hit limits, it will see a contention on a node. But freeing from remote
1347 * node means more costs for memory reclaim because of memory latency.
1349 * Now, we use round-robin. Better algorithm is welcomed.
1351 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1355 mem_cgroup_may_update_nodemask(memcg);
1356 node = memcg->last_scanned_node;
1358 node = next_node_in(node, memcg->scan_nodes);
1360 * mem_cgroup_may_update_nodemask might have seen no reclaimmable pages
1361 * last time it really checked all the LRUs due to rate limiting.
1362 * Fallback to the current node in that case for simplicity.
1364 if (unlikely(node == MAX_NUMNODES))
1365 node = numa_node_id();
1367 memcg->last_scanned_node = node;
1371 int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
1377 static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
1380 unsigned long *total_scanned)
1382 struct mem_cgroup *victim = NULL;
1385 unsigned long excess;
1386 unsigned long nr_scanned;
1387 struct mem_cgroup_reclaim_cookie reclaim = {
1392 excess = soft_limit_excess(root_memcg);
1395 victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
1400 * If we have not been able to reclaim
1401 * anything, it might because there are
1402 * no reclaimable pages under this hierarchy
1407 * We want to do more targeted reclaim.
1408 * excess >> 2 is not to excessive so as to
1409 * reclaim too much, nor too less that we keep
1410 * coming back to reclaim from this cgroup
1412 if (total >= (excess >> 2) ||
1413 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
1418 total += mem_cgroup_shrink_node(victim, gfp_mask, false,
1419 pgdat, &nr_scanned);
1420 *total_scanned += nr_scanned;
1421 if (!soft_limit_excess(root_memcg))
1424 mem_cgroup_iter_break(root_memcg, victim);
1428 #ifdef CONFIG_LOCKDEP
1429 static struct lockdep_map memcg_oom_lock_dep_map = {
1430 .name = "memcg_oom_lock",
1434 static DEFINE_SPINLOCK(memcg_oom_lock);
1437 * Check OOM-Killer is already running under our hierarchy.
1438 * If someone is running, return false.
1440 static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
1442 struct mem_cgroup *iter, *failed = NULL;
1444 spin_lock(&memcg_oom_lock);
1446 for_each_mem_cgroup_tree(iter, memcg) {
1447 if (iter->oom_lock) {
1449 * this subtree of our hierarchy is already locked
1450 * so we cannot give a lock.
1453 mem_cgroup_iter_break(memcg, iter);
1456 iter->oom_lock = true;
1461 * OK, we failed to lock the whole subtree so we have
1462 * to clean up what we set up to the failing subtree
1464 for_each_mem_cgroup_tree(iter, memcg) {
1465 if (iter == failed) {
1466 mem_cgroup_iter_break(memcg, iter);
1469 iter->oom_lock = false;
1472 mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
1474 spin_unlock(&memcg_oom_lock);
1479 static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
1481 struct mem_cgroup *iter;
1483 spin_lock(&memcg_oom_lock);
1484 mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_);
1485 for_each_mem_cgroup_tree(iter, memcg)
1486 iter->oom_lock = false;
1487 spin_unlock(&memcg_oom_lock);
1490 static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
1492 struct mem_cgroup *iter;
1494 spin_lock(&memcg_oom_lock);
1495 for_each_mem_cgroup_tree(iter, memcg)
1497 spin_unlock(&memcg_oom_lock);
1500 static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
1502 struct mem_cgroup *iter;
1505 * When a new child is created while the hierarchy is under oom,
1506 * mem_cgroup_oom_lock() may not be called. Watch for underflow.
1508 spin_lock(&memcg_oom_lock);
1509 for_each_mem_cgroup_tree(iter, memcg)
1510 if (iter->under_oom > 0)
1512 spin_unlock(&memcg_oom_lock);
1515 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1517 struct oom_wait_info {
1518 struct mem_cgroup *memcg;
1522 static int memcg_oom_wake_function(wait_queue_t *wait,
1523 unsigned mode, int sync, void *arg)
1525 struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
1526 struct mem_cgroup *oom_wait_memcg;
1527 struct oom_wait_info *oom_wait_info;
1529 oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1530 oom_wait_memcg = oom_wait_info->memcg;
1532 if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
1533 !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
1535 return autoremove_wake_function(wait, mode, sync, arg);
1538 static void memcg_oom_recover(struct mem_cgroup *memcg)
1541 * For the following lockless ->under_oom test, the only required
1542 * guarantee is that it must see the state asserted by an OOM when
1543 * this function is called as a result of userland actions
1544 * triggered by the notification of the OOM. This is trivially
1545 * achieved by invoking mem_cgroup_mark_under_oom() before
1546 * triggering notification.
1548 if (memcg && memcg->under_oom)
1549 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
1552 static void mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
1554 if (!current->memcg_may_oom)
1557 * We are in the middle of the charge context here, so we
1558 * don't want to block when potentially sitting on a callstack
1559 * that holds all kinds of filesystem and mm locks.
1561 * Also, the caller may handle a failed allocation gracefully
1562 * (like optional page cache readahead) and so an OOM killer
1563 * invocation might not even be necessary.
1565 * That's why we don't do anything here except remember the
1566 * OOM context and then deal with it at the end of the page
1567 * fault when the stack is unwound, the locks are released,
1568 * and when we know whether the fault was overall successful.
1570 css_get(&memcg->css);
1571 current->memcg_in_oom = memcg;
1572 current->memcg_oom_gfp_mask = mask;
1573 current->memcg_oom_order = order;
1577 * mem_cgroup_oom_synchronize - complete memcg OOM handling
1578 * @handle: actually kill/wait or just clean up the OOM state
1580 * This has to be called at the end of a page fault if the memcg OOM
1581 * handler was enabled.
1583 * Memcg supports userspace OOM handling where failed allocations must
1584 * sleep on a waitqueue until the userspace task resolves the
1585 * situation. Sleeping directly in the charge context with all kinds
1586 * of locks held is not a good idea, instead we remember an OOM state
1587 * in the task and mem_cgroup_oom_synchronize() has to be called at
1588 * the end of the page fault to complete the OOM handling.
1590 * Returns %true if an ongoing memcg OOM situation was detected and
1591 * completed, %false otherwise.
1593 bool mem_cgroup_oom_synchronize(bool handle)
1595 struct mem_cgroup *memcg = current->memcg_in_oom;
1596 struct oom_wait_info owait;
1599 /* OOM is global, do not handle */
1603 if (!handle || oom_killer_disabled)
1606 owait.memcg = memcg;
1607 owait.wait.flags = 0;
1608 owait.wait.func = memcg_oom_wake_function;
1609 owait.wait.private = current;
1610 INIT_LIST_HEAD(&owait.wait.task_list);
1612 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1613 mem_cgroup_mark_under_oom(memcg);
1615 locked = mem_cgroup_oom_trylock(memcg);
1618 mem_cgroup_oom_notify(memcg);
1620 if (locked && !memcg->oom_kill_disable) {
1621 mem_cgroup_unmark_under_oom(memcg);
1622 finish_wait(&memcg_oom_waitq, &owait.wait);
1623 mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,
1624 current->memcg_oom_order);
1627 mem_cgroup_unmark_under_oom(memcg);
1628 finish_wait(&memcg_oom_waitq, &owait.wait);
1632 mem_cgroup_oom_unlock(memcg);
1634 * There is no guarantee that an OOM-lock contender
1635 * sees the wakeups triggered by the OOM kill
1636 * uncharges. Wake any sleepers explicitely.
1638 memcg_oom_recover(memcg);
1641 current->memcg_in_oom = NULL;
1642 css_put(&memcg->css);
1647 * lock_page_memcg - lock a page->mem_cgroup binding
1650 * This function protects unlocked LRU pages from being moved to
1651 * another cgroup and stabilizes their page->mem_cgroup binding.
1653 void lock_page_memcg(struct page *page)
1655 struct mem_cgroup *memcg;
1656 unsigned long flags;
1659 * The RCU lock is held throughout the transaction. The fast
1660 * path can get away without acquiring the memcg->move_lock
1661 * because page moving starts with an RCU grace period.
1665 if (mem_cgroup_disabled())
1668 memcg = page->mem_cgroup;
1669 if (unlikely(!memcg))
1672 if (atomic_read(&memcg->moving_account) <= 0)
1675 spin_lock_irqsave(&memcg->move_lock, flags);
1676 if (memcg != page->mem_cgroup) {
1677 spin_unlock_irqrestore(&memcg->move_lock, flags);
1682 * When charge migration first begins, we can have locked and
1683 * unlocked page stat updates happening concurrently. Track
1684 * the task who has the lock for unlock_page_memcg().
1686 memcg->move_lock_task = current;
1687 memcg->move_lock_flags = flags;
1691 EXPORT_SYMBOL(lock_page_memcg);
1694 * unlock_page_memcg - unlock a page->mem_cgroup binding
1697 void unlock_page_memcg(struct page *page)
1699 struct mem_cgroup *memcg = page->mem_cgroup;
1701 if (memcg && memcg->move_lock_task == current) {
1702 unsigned long flags = memcg->move_lock_flags;
1704 memcg->move_lock_task = NULL;
1705 memcg->move_lock_flags = 0;
1707 spin_unlock_irqrestore(&memcg->move_lock, flags);
1712 EXPORT_SYMBOL(unlock_page_memcg);
1715 * size of first charge trial. "32" comes from vmscan.c's magic value.
1716 * TODO: maybe necessary to use big numbers in big irons.
1718 #define CHARGE_BATCH 32U
1719 struct memcg_stock_pcp {
1720 struct mem_cgroup *cached; /* this never be root cgroup */
1721 unsigned int nr_pages;
1722 struct work_struct work;
1723 unsigned long flags;
1724 #define FLUSHING_CACHED_CHARGE 0
1726 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
1727 static DEFINE_MUTEX(percpu_charge_mutex);
1730 * consume_stock: Try to consume stocked charge on this cpu.
1731 * @memcg: memcg to consume from.
1732 * @nr_pages: how many pages to charge.
1734 * The charges will only happen if @memcg matches the current cpu's memcg
1735 * stock, and at least @nr_pages are available in that stock. Failure to
1736 * service an allocation will refill the stock.
1738 * returns true if successful, false otherwise.
1740 static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
1742 struct memcg_stock_pcp *stock;
1745 if (nr_pages > CHARGE_BATCH)
1748 stock = &get_cpu_var(memcg_stock);
1749 if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
1750 stock->nr_pages -= nr_pages;
1753 put_cpu_var(memcg_stock);
1758 * Returns stocks cached in percpu and reset cached information.
1760 static void drain_stock(struct memcg_stock_pcp *stock)
1762 struct mem_cgroup *old = stock->cached;
1764 if (stock->nr_pages) {
1765 page_counter_uncharge(&old->memory, stock->nr_pages);
1766 if (do_memsw_account())
1767 page_counter_uncharge(&old->memsw, stock->nr_pages);
1768 css_put_many(&old->css, stock->nr_pages);
1769 stock->nr_pages = 0;
1771 stock->cached = NULL;
1775 * This must be called under preempt disabled or must be called by
1776 * a thread which is pinned to local cpu.
1778 static void drain_local_stock(struct work_struct *dummy)
1780 struct memcg_stock_pcp *stock = this_cpu_ptr(&memcg_stock);
1782 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
1786 * Cache charges(val) to local per_cpu area.
1787 * This will be consumed by consume_stock() function, later.
1789 static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
1791 struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
1793 if (stock->cached != memcg) { /* reset if necessary */
1795 stock->cached = memcg;
1797 stock->nr_pages += nr_pages;
1798 put_cpu_var(memcg_stock);
1802 * Drains all per-CPU charge caches for given root_memcg resp. subtree
1803 * of the hierarchy under it.
1805 static void drain_all_stock(struct mem_cgroup *root_memcg)
1809 /* If someone's already draining, avoid adding running more workers. */
1810 if (!mutex_trylock(&percpu_charge_mutex))
1812 /* Notify other cpus that system-wide "drain" is running */
1815 for_each_online_cpu(cpu) {
1816 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
1817 struct mem_cgroup *memcg;
1819 memcg = stock->cached;
1820 if (!memcg || !stock->nr_pages)
1822 if (!mem_cgroup_is_descendant(memcg, root_memcg))
1824 if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
1826 drain_local_stock(&stock->work);
1828 schedule_work_on(cpu, &stock->work);
1833 mutex_unlock(&percpu_charge_mutex);
1836 static int memcg_cpu_hotplug_callback(struct notifier_block *nb,
1837 unsigned long action,
1840 int cpu = (unsigned long)hcpu;
1841 struct memcg_stock_pcp *stock;
1843 if (action == CPU_ONLINE)
1846 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
1849 stock = &per_cpu(memcg_stock, cpu);
1854 static void reclaim_high(struct mem_cgroup *memcg,
1855 unsigned int nr_pages,
1859 if (page_counter_read(&memcg->memory) <= memcg->high)
1861 mem_cgroup_events(memcg, MEMCG_HIGH, 1);
1862 try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, true);
1863 } while ((memcg = parent_mem_cgroup(memcg)));
1866 static void high_work_func(struct work_struct *work)
1868 struct mem_cgroup *memcg;
1870 memcg = container_of(work, struct mem_cgroup, high_work);
1871 reclaim_high(memcg, CHARGE_BATCH, GFP_KERNEL);
1875 * Scheduled by try_charge() to be executed from the userland return path
1876 * and reclaims memory over the high limit.
1878 void mem_cgroup_handle_over_high(void)
1880 unsigned int nr_pages = current->memcg_nr_pages_over_high;
1881 struct mem_cgroup *memcg;
1883 if (likely(!nr_pages))
1886 memcg = get_mem_cgroup_from_mm(current->mm);
1887 reclaim_high(memcg, nr_pages, GFP_KERNEL);
1888 css_put(&memcg->css);
1889 current->memcg_nr_pages_over_high = 0;
1892 static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
1893 unsigned int nr_pages)
1895 unsigned int batch = max(CHARGE_BATCH, nr_pages);
1896 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
1897 struct mem_cgroup *mem_over_limit;
1898 struct page_counter *counter;
1899 unsigned long nr_reclaimed;
1900 bool may_swap = true;
1901 bool drained = false;
1903 if (mem_cgroup_is_root(memcg))
1906 if (consume_stock(memcg, nr_pages))
1909 if (!do_memsw_account() ||
1910 page_counter_try_charge(&memcg->memsw, batch, &counter)) {
1911 if (page_counter_try_charge(&memcg->memory, batch, &counter))
1913 if (do_memsw_account())
1914 page_counter_uncharge(&memcg->memsw, batch);
1915 mem_over_limit = mem_cgroup_from_counter(counter, memory);
1917 mem_over_limit = mem_cgroup_from_counter(counter, memsw);
1921 if (batch > nr_pages) {
1927 * Unlike in global OOM situations, memcg is not in a physical
1928 * memory shortage. Allow dying and OOM-killed tasks to
1929 * bypass the last charges so that they can exit quickly and
1930 * free their memory.
1932 if (unlikely(test_thread_flag(TIF_MEMDIE) ||
1933 fatal_signal_pending(current) ||
1934 current->flags & PF_EXITING))
1937 if (unlikely(task_in_memcg_oom(current)))
1940 if (!gfpflags_allow_blocking(gfp_mask))
1943 mem_cgroup_events(mem_over_limit, MEMCG_MAX, 1);
1945 nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
1946 gfp_mask, may_swap);
1948 if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
1952 drain_all_stock(mem_over_limit);
1957 if (gfp_mask & __GFP_NORETRY)
1960 * Even though the limit is exceeded at this point, reclaim
1961 * may have been able to free some pages. Retry the charge
1962 * before killing the task.
1964 * Only for regular pages, though: huge pages are rather
1965 * unlikely to succeed so close to the limit, and we fall back
1966 * to regular pages anyway in case of failure.
1968 if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
1971 * At task move, charge accounts can be doubly counted. So, it's
1972 * better to wait until the end of task_move if something is going on.
1974 if (mem_cgroup_wait_acct_move(mem_over_limit))
1980 if (gfp_mask & __GFP_NOFAIL)
1983 if (fatal_signal_pending(current))
1986 mem_cgroup_events(mem_over_limit, MEMCG_OOM, 1);
1988 mem_cgroup_oom(mem_over_limit, gfp_mask,
1989 get_order(nr_pages * PAGE_SIZE));
1991 if (!(gfp_mask & __GFP_NOFAIL))
1995 * The allocation either can't fail or will lead to more memory
1996 * being freed very soon. Allow memory usage go over the limit
1997 * temporarily by force charging it.
1999 page_counter_charge(&memcg->memory, nr_pages);
2000 if (do_memsw_account())
2001 page_counter_charge(&memcg->memsw, nr_pages);
2002 css_get_many(&memcg->css, nr_pages);
2007 css_get_many(&memcg->css, batch);
2008 if (batch > nr_pages)
2009 refill_stock(memcg, batch - nr_pages);
2012 * If the hierarchy is above the normal consumption range, schedule
2013 * reclaim on returning to userland. We can perform reclaim here
2014 * if __GFP_RECLAIM but let's always punt for simplicity and so that
2015 * GFP_KERNEL can consistently be used during reclaim. @memcg is
2016 * not recorded as it most likely matches current's and won't
2017 * change in the meantime. As high limit is checked again before
2018 * reclaim, the cost of mismatch is negligible.
2021 if (page_counter_read(&memcg->memory) > memcg->high) {
2022 /* Don't bother a random interrupted task */
2023 if (in_interrupt()) {
2024 schedule_work(&memcg->high_work);
2027 current->memcg_nr_pages_over_high += batch;
2028 set_notify_resume(current);
2031 } while ((memcg = parent_mem_cgroup(memcg)));
2036 static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
2038 if (mem_cgroup_is_root(memcg))
2041 page_counter_uncharge(&memcg->memory, nr_pages);
2042 if (do_memsw_account())
2043 page_counter_uncharge(&memcg->memsw, nr_pages);
2045 css_put_many(&memcg->css, nr_pages);
2048 static void lock_page_lru(struct page *page, int *isolated)
2050 struct zone *zone = page_zone(page);
2052 spin_lock_irq(zone_lru_lock(zone));
2053 if (PageLRU(page)) {
2054 struct lruvec *lruvec;
2056 lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
2058 del_page_from_lru_list(page, lruvec, page_lru(page));
2064 static void unlock_page_lru(struct page *page, int isolated)
2066 struct zone *zone = page_zone(page);
2069 struct lruvec *lruvec;
2071 lruvec = mem_cgroup_page_lruvec(page, zone->zone_pgdat);
2072 VM_BUG_ON_PAGE(PageLRU(page), page);
2074 add_page_to_lru_list(page, lruvec, page_lru(page));
2076 spin_unlock_irq(zone_lru_lock(zone));
2079 static void commit_charge(struct page *page, struct mem_cgroup *memcg,
2084 VM_BUG_ON_PAGE(page->mem_cgroup, page);
2087 * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
2088 * may already be on some other mem_cgroup's LRU. Take care of it.
2091 lock_page_lru(page, &isolated);
2094 * Nobody should be changing or seriously looking at
2095 * page->mem_cgroup at this point:
2097 * - the page is uncharged
2099 * - the page is off-LRU
2101 * - an anonymous fault has exclusive page access, except for
2102 * a locked page table
2104 * - a page cache insertion, a swapin fault, or a migration
2105 * have the page locked
2107 page->mem_cgroup = memcg;
2110 unlock_page_lru(page, isolated);
2114 static int memcg_alloc_cache_id(void)
2119 id = ida_simple_get(&memcg_cache_ida,
2120 0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
2124 if (id < memcg_nr_cache_ids)
2128 * There's no space for the new id in memcg_caches arrays,
2129 * so we have to grow them.
2131 down_write(&memcg_cache_ids_sem);
2133 size = 2 * (id + 1);
2134 if (size < MEMCG_CACHES_MIN_SIZE)
2135 size = MEMCG_CACHES_MIN_SIZE;
2136 else if (size > MEMCG_CACHES_MAX_SIZE)
2137 size = MEMCG_CACHES_MAX_SIZE;
2139 err = memcg_update_all_caches(size);
2141 err = memcg_update_all_list_lrus(size);
2143 memcg_nr_cache_ids = size;
2145 up_write(&memcg_cache_ids_sem);
2148 ida_simple_remove(&memcg_cache_ida, id);
2154 static void memcg_free_cache_id(int id)
2156 ida_simple_remove(&memcg_cache_ida, id);
2159 struct memcg_kmem_cache_create_work {
2160 struct mem_cgroup *memcg;
2161 struct kmem_cache *cachep;
2162 struct work_struct work;
2165 static void memcg_kmem_cache_create_func(struct work_struct *w)
2167 struct memcg_kmem_cache_create_work *cw =
2168 container_of(w, struct memcg_kmem_cache_create_work, work);
2169 struct mem_cgroup *memcg = cw->memcg;
2170 struct kmem_cache *cachep = cw->cachep;
2172 memcg_create_kmem_cache(memcg, cachep);
2174 css_put(&memcg->css);
2179 * Enqueue the creation of a per-memcg kmem_cache.
2181 static void __memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
2182 struct kmem_cache *cachep)
2184 struct memcg_kmem_cache_create_work *cw;
2186 cw = kmalloc(sizeof(*cw), GFP_NOWAIT);
2190 css_get(&memcg->css);
2193 cw->cachep = cachep;
2194 INIT_WORK(&cw->work, memcg_kmem_cache_create_func);
2196 schedule_work(&cw->work);
2199 static void memcg_schedule_kmem_cache_create(struct mem_cgroup *memcg,
2200 struct kmem_cache *cachep)
2203 * We need to stop accounting when we kmalloc, because if the
2204 * corresponding kmalloc cache is not yet created, the first allocation
2205 * in __memcg_schedule_kmem_cache_create will recurse.
2207 * However, it is better to enclose the whole function. Depending on
2208 * the debugging options enabled, INIT_WORK(), for instance, can
2209 * trigger an allocation. This too, will make us recurse. Because at
2210 * this point we can't allow ourselves back into memcg_kmem_get_cache,
2211 * the safest choice is to do it like this, wrapping the whole function.
2213 current->memcg_kmem_skip_account = 1;
2214 __memcg_schedule_kmem_cache_create(memcg, cachep);
2215 current->memcg_kmem_skip_account = 0;
2218 static inline bool memcg_kmem_bypass(void)
2220 if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD))
2226 * memcg_kmem_get_cache: select the correct per-memcg cache for allocation
2227 * @cachep: the original global kmem cache
2229 * Return the kmem_cache we're supposed to use for a slab allocation.
2230 * We try to use the current memcg's version of the cache.
2232 * If the cache does not exist yet, if we are the first user of it, we
2233 * create it asynchronously in a workqueue and let the current allocation
2234 * go through with the original cache.
2236 * This function takes a reference to the cache it returns to assure it
2237 * won't get destroyed while we are working with it. Once the caller is
2238 * done with it, memcg_kmem_put_cache() must be called to release the
2241 struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep)
2243 struct mem_cgroup *memcg;
2244 struct kmem_cache *memcg_cachep;
2247 VM_BUG_ON(!is_root_cache(cachep));
2249 if (memcg_kmem_bypass())
2252 if (current->memcg_kmem_skip_account)
2255 memcg = get_mem_cgroup_from_mm(current->mm);
2256 kmemcg_id = READ_ONCE(memcg->kmemcg_id);
2260 memcg_cachep = cache_from_memcg_idx(cachep, kmemcg_id);
2261 if (likely(memcg_cachep))
2262 return memcg_cachep;
2265 * If we are in a safe context (can wait, and not in interrupt
2266 * context), we could be be predictable and return right away.
2267 * This would guarantee that the allocation being performed
2268 * already belongs in the new cache.
2270 * However, there are some clashes that can arrive from locking.
2271 * For instance, because we acquire the slab_mutex while doing
2272 * memcg_create_kmem_cache, this means no further allocation
2273 * could happen with the slab_mutex held. So it's better to
2276 memcg_schedule_kmem_cache_create(memcg, cachep);
2278 css_put(&memcg->css);
2283 * memcg_kmem_put_cache: drop reference taken by memcg_kmem_get_cache
2284 * @cachep: the cache returned by memcg_kmem_get_cache
2286 void memcg_kmem_put_cache(struct kmem_cache *cachep)
2288 if (!is_root_cache(cachep))
2289 css_put(&cachep->memcg_params.memcg->css);
2293 * memcg_kmem_charge: charge a kmem page
2294 * @page: page to charge
2295 * @gfp: reclaim mode
2296 * @order: allocation order
2297 * @memcg: memory cgroup to charge
2299 * Returns 0 on success, an error code on failure.
2301 int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
2302 struct mem_cgroup *memcg)
2304 unsigned int nr_pages = 1 << order;
2305 struct page_counter *counter;
2308 ret = try_charge(memcg, gfp, nr_pages);
2312 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
2313 !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
2314 cancel_charge(memcg, nr_pages);
2318 page->mem_cgroup = memcg;
2324 * memcg_kmem_charge: charge a kmem page to the current memory cgroup
2325 * @page: page to charge
2326 * @gfp: reclaim mode
2327 * @order: allocation order
2329 * Returns 0 on success, an error code on failure.
2331 int memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
2333 struct mem_cgroup *memcg;
2336 if (memcg_kmem_bypass())
2339 memcg = get_mem_cgroup_from_mm(current->mm);
2340 if (!mem_cgroup_is_root(memcg))
2341 ret = memcg_kmem_charge_memcg(page, gfp, order, memcg);
2342 css_put(&memcg->css);
2346 * memcg_kmem_uncharge: uncharge a kmem page
2347 * @page: page to uncharge
2348 * @order: allocation order
2350 void memcg_kmem_uncharge(struct page *page, int order)
2352 struct mem_cgroup *memcg = page->mem_cgroup;
2353 unsigned int nr_pages = 1 << order;
2358 VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
2360 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
2361 page_counter_uncharge(&memcg->kmem, nr_pages);
2363 page_counter_uncharge(&memcg->memory, nr_pages);
2364 if (do_memsw_account())
2365 page_counter_uncharge(&memcg->memsw, nr_pages);
2367 page->mem_cgroup = NULL;
2368 css_put_many(&memcg->css, nr_pages);
2370 #endif /* !CONFIG_SLOB */
2372 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
2375 * Because tail pages are not marked as "used", set it. We're under
2376 * zone_lru_lock and migration entries setup in all page mappings.
2378 void mem_cgroup_split_huge_fixup(struct page *head)
2382 if (mem_cgroup_disabled())
2385 for (i = 1; i < HPAGE_PMD_NR; i++)
2386 head[i].mem_cgroup = head->mem_cgroup;
2388 __this_cpu_sub(head->mem_cgroup->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
2391 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
2393 #ifdef CONFIG_MEMCG_SWAP
2394 static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
2397 int val = (charge) ? 1 : -1;
2398 this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAP], val);
2402 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
2403 * @entry: swap entry to be moved
2404 * @from: mem_cgroup which the entry is moved from
2405 * @to: mem_cgroup which the entry is moved to
2407 * It succeeds only when the swap_cgroup's record for this entry is the same
2408 * as the mem_cgroup's id of @from.
2410 * Returns 0 on success, -EINVAL on failure.
2412 * The caller must have charged to @to, IOW, called page_counter_charge() about
2413 * both res and memsw, and called css_get().
2415 static int mem_cgroup_move_swap_account(swp_entry_t entry,
2416 struct mem_cgroup *from, struct mem_cgroup *to)
2418 unsigned short old_id, new_id;
2420 old_id = mem_cgroup_id(from);
2421 new_id = mem_cgroup_id(to);
2423 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
2424 mem_cgroup_swap_statistics(from, false);
2425 mem_cgroup_swap_statistics(to, true);
2431 static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
2432 struct mem_cgroup *from, struct mem_cgroup *to)
2438 static DEFINE_MUTEX(memcg_limit_mutex);
2440 static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
2441 unsigned long limit)
2443 unsigned long curusage;
2444 unsigned long oldusage;
2445 bool enlarge = false;
2450 * For keeping hierarchical_reclaim simple, how long we should retry
2451 * is depends on callers. We set our retry-count to be function
2452 * of # of children which we should visit in this loop.
2454 retry_count = MEM_CGROUP_RECLAIM_RETRIES *
2455 mem_cgroup_count_children(memcg);
2457 oldusage = page_counter_read(&memcg->memory);
2460 if (signal_pending(current)) {
2465 mutex_lock(&memcg_limit_mutex);
2466 if (limit > memcg->memsw.limit) {
2467 mutex_unlock(&memcg_limit_mutex);
2471 if (limit > memcg->memory.limit)
2473 ret = page_counter_limit(&memcg->memory, limit);
2474 mutex_unlock(&memcg_limit_mutex);
2479 try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, true);
2481 curusage = page_counter_read(&memcg->memory);
2482 /* Usage is reduced ? */
2483 if (curusage >= oldusage)
2486 oldusage = curusage;
2487 } while (retry_count);
2489 if (!ret && enlarge)
2490 memcg_oom_recover(memcg);
2495 static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
2496 unsigned long limit)
2498 unsigned long curusage;
2499 unsigned long oldusage;
2500 bool enlarge = false;
2504 /* see mem_cgroup_resize_res_limit */
2505 retry_count = MEM_CGROUP_RECLAIM_RETRIES *
2506 mem_cgroup_count_children(memcg);
2508 oldusage = page_counter_read(&memcg->memsw);
2511 if (signal_pending(current)) {
2516 mutex_lock(&memcg_limit_mutex);
2517 if (limit < memcg->memory.limit) {
2518 mutex_unlock(&memcg_limit_mutex);
2522 if (limit > memcg->memsw.limit)
2524 ret = page_counter_limit(&memcg->memsw, limit);
2525 mutex_unlock(&memcg_limit_mutex);
2530 try_to_free_mem_cgroup_pages(memcg, 1, GFP_KERNEL, false);
2532 curusage = page_counter_read(&memcg->memsw);
2533 /* Usage is reduced ? */
2534 if (curusage >= oldusage)
2537 oldusage = curusage;
2538 } while (retry_count);
2540 if (!ret && enlarge)
2541 memcg_oom_recover(memcg);
2546 unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
2548 unsigned long *total_scanned)
2550 unsigned long nr_reclaimed = 0;
2551 struct mem_cgroup_per_node *mz, *next_mz = NULL;
2552 unsigned long reclaimed;
2554 struct mem_cgroup_tree_per_node *mctz;
2555 unsigned long excess;
2556 unsigned long nr_scanned;
2561 mctz = soft_limit_tree_node(pgdat->node_id);
2563 * This loop can run a while, specially if mem_cgroup's continuously
2564 * keep exceeding their soft limit and putting the system under
2571 mz = mem_cgroup_largest_soft_limit_node(mctz);
2576 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
2577 gfp_mask, &nr_scanned);
2578 nr_reclaimed += reclaimed;
2579 *total_scanned += nr_scanned;
2580 spin_lock_irq(&mctz->lock);
2581 __mem_cgroup_remove_exceeded(mz, mctz);
2584 * If we failed to reclaim anything from this memory cgroup
2585 * it is time to move on to the next cgroup
2589 next_mz = __mem_cgroup_largest_soft_limit_node(mctz);
2591 excess = soft_limit_excess(mz->memcg);
2593 * One school of thought says that we should not add
2594 * back the node to the tree if reclaim returns 0.
2595 * But our reclaim could return 0, simply because due
2596 * to priority we are exposing a smaller subset of
2597 * memory to reclaim from. Consider this as a longer
2600 /* If excess == 0, no tree ops */
2601 __mem_cgroup_insert_exceeded(mz, mctz, excess);
2602 spin_unlock_irq(&mctz->lock);
2603 css_put(&mz->memcg->css);
2606 * Could not reclaim anything and there are no more
2607 * mem cgroups to try or we seem to be looping without
2608 * reclaiming anything.
2610 if (!nr_reclaimed &&
2612 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
2614 } while (!nr_reclaimed);
2616 css_put(&next_mz->memcg->css);
2617 return nr_reclaimed;
2621 * Test whether @memcg has children, dead or alive. Note that this
2622 * function doesn't care whether @memcg has use_hierarchy enabled and
2623 * returns %true if there are child csses according to the cgroup
2624 * hierarchy. Testing use_hierarchy is the caller's responsiblity.
2626 static inline bool memcg_has_children(struct mem_cgroup *memcg)
2631 ret = css_next_child(NULL, &memcg->css);
2637 * Reclaims as many pages from the given memcg as possible.
2639 * Caller is responsible for holding css reference for memcg.
2641 static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
2643 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
2645 /* we call try-to-free pages for make this cgroup empty */
2646 lru_add_drain_all();
2647 /* try to free all pages in this cgroup */
2648 while (nr_retries && page_counter_read(&memcg->memory)) {
2651 if (signal_pending(current))
2654 progress = try_to_free_mem_cgroup_pages(memcg, 1,
2658 /* maybe some writeback is necessary */
2659 congestion_wait(BLK_RW_ASYNC, HZ/10);
2667 static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
2668 char *buf, size_t nbytes,
2671 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
2673 if (mem_cgroup_is_root(memcg))
2675 return mem_cgroup_force_empty(memcg) ?: nbytes;
2678 static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
2681 return mem_cgroup_from_css(css)->use_hierarchy;
2684 static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
2685 struct cftype *cft, u64 val)
2688 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2689 struct mem_cgroup *parent_memcg = mem_cgroup_from_css(memcg->css.parent);
2691 if (memcg->use_hierarchy == val)
2695 * If parent's use_hierarchy is set, we can't make any modifications
2696 * in the child subtrees. If it is unset, then the change can
2697 * occur, provided the current cgroup has no children.
2699 * For the root cgroup, parent_mem is NULL, we allow value to be
2700 * set if there are no children.
2702 if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
2703 (val == 1 || val == 0)) {
2704 if (!memcg_has_children(memcg))
2705 memcg->use_hierarchy = val;
2714 static void tree_stat(struct mem_cgroup *memcg, unsigned long *stat)
2716 struct mem_cgroup *iter;
2719 memset(stat, 0, sizeof(*stat) * MEMCG_NR_STAT);
2721 for_each_mem_cgroup_tree(iter, memcg) {
2722 for (i = 0; i < MEMCG_NR_STAT; i++)
2723 stat[i] += mem_cgroup_read_stat(iter, i);
2727 static void tree_events(struct mem_cgroup *memcg, unsigned long *events)
2729 struct mem_cgroup *iter;
2732 memset(events, 0, sizeof(*events) * MEMCG_NR_EVENTS);
2734 for_each_mem_cgroup_tree(iter, memcg) {
2735 for (i = 0; i < MEMCG_NR_EVENTS; i++)
2736 events[i] += mem_cgroup_read_events(iter, i);
2740 static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
2742 unsigned long val = 0;
2744 if (mem_cgroup_is_root(memcg)) {
2745 struct mem_cgroup *iter;
2747 for_each_mem_cgroup_tree(iter, memcg) {
2748 val += mem_cgroup_read_stat(iter,
2749 MEM_CGROUP_STAT_CACHE);
2750 val += mem_cgroup_read_stat(iter,
2751 MEM_CGROUP_STAT_RSS);
2753 val += mem_cgroup_read_stat(iter,
2754 MEM_CGROUP_STAT_SWAP);
2758 val = page_counter_read(&memcg->memory);
2760 val = page_counter_read(&memcg->memsw);
2773 static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
2776 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
2777 struct page_counter *counter;
2779 switch (MEMFILE_TYPE(cft->private)) {
2781 counter = &memcg->memory;
2784 counter = &memcg->memsw;
2787 counter = &memcg->kmem;
2790 counter = &memcg->tcpmem;
2796 switch (MEMFILE_ATTR(cft->private)) {
2798 if (counter == &memcg->memory)
2799 return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
2800 if (counter == &memcg->memsw)
2801 return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
2802 return (u64)page_counter_read(counter) * PAGE_SIZE;
2804 return (u64)counter->limit * PAGE_SIZE;
2806 return (u64)counter->watermark * PAGE_SIZE;
2808 return counter->failcnt;
2809 case RES_SOFT_LIMIT:
2810 return (u64)memcg->soft_limit * PAGE_SIZE;
2817 static int memcg_online_kmem(struct mem_cgroup *memcg)
2821 if (cgroup_memory_nokmem)
2824 BUG_ON(memcg->kmemcg_id >= 0);
2825 BUG_ON(memcg->kmem_state);
2827 memcg_id = memcg_alloc_cache_id();
2831 static_branch_inc(&memcg_kmem_enabled_key);
2833 * A memory cgroup is considered kmem-online as soon as it gets
2834 * kmemcg_id. Setting the id after enabling static branching will
2835 * guarantee no one starts accounting before all call sites are
2838 memcg->kmemcg_id = memcg_id;
2839 memcg->kmem_state = KMEM_ONLINE;
2844 static void memcg_offline_kmem(struct mem_cgroup *memcg)
2846 struct cgroup_subsys_state *css;
2847 struct mem_cgroup *parent, *child;
2850 if (memcg->kmem_state != KMEM_ONLINE)
2853 * Clear the online state before clearing memcg_caches array
2854 * entries. The slab_mutex in memcg_deactivate_kmem_caches()
2855 * guarantees that no cache will be created for this cgroup
2856 * after we are done (see memcg_create_kmem_cache()).
2858 memcg->kmem_state = KMEM_ALLOCATED;
2860 memcg_deactivate_kmem_caches(memcg);
2862 kmemcg_id = memcg->kmemcg_id;
2863 BUG_ON(kmemcg_id < 0);
2865 parent = parent_mem_cgroup(memcg);
2867 parent = root_mem_cgroup;
2870 * Change kmemcg_id of this cgroup and all its descendants to the
2871 * parent's id, and then move all entries from this cgroup's list_lrus
2872 * to ones of the parent. After we have finished, all list_lrus
2873 * corresponding to this cgroup are guaranteed to remain empty. The
2874 * ordering is imposed by list_lru_node->lock taken by
2875 * memcg_drain_all_list_lrus().
2877 rcu_read_lock(); /* can be called from css_free w/o cgroup_mutex */
2878 css_for_each_descendant_pre(css, &memcg->css) {
2879 child = mem_cgroup_from_css(css);
2880 BUG_ON(child->kmemcg_id != kmemcg_id);
2881 child->kmemcg_id = parent->kmemcg_id;
2882 if (!memcg->use_hierarchy)
2887 memcg_drain_all_list_lrus(kmemcg_id, parent->kmemcg_id);
2889 memcg_free_cache_id(kmemcg_id);
2892 static void memcg_free_kmem(struct mem_cgroup *memcg)
2894 /* css_alloc() failed, offlining didn't happen */
2895 if (unlikely(memcg->kmem_state == KMEM_ONLINE))
2896 memcg_offline_kmem(memcg);
2898 if (memcg->kmem_state == KMEM_ALLOCATED) {
2899 memcg_destroy_kmem_caches(memcg);
2900 static_branch_dec(&memcg_kmem_enabled_key);
2901 WARN_ON(page_counter_read(&memcg->kmem));
2905 static int memcg_online_kmem(struct mem_cgroup *memcg)
2909 static void memcg_offline_kmem(struct mem_cgroup *memcg)
2912 static void memcg_free_kmem(struct mem_cgroup *memcg)
2915 #endif /* !CONFIG_SLOB */
2917 static int memcg_update_kmem_limit(struct mem_cgroup *memcg,
2918 unsigned long limit)
2922 mutex_lock(&memcg_limit_mutex);
2923 ret = page_counter_limit(&memcg->kmem, limit);
2924 mutex_unlock(&memcg_limit_mutex);
2928 static int memcg_update_tcp_limit(struct mem_cgroup *memcg, unsigned long limit)
2932 mutex_lock(&memcg_limit_mutex);
2934 ret = page_counter_limit(&memcg->tcpmem, limit);
2938 if (!memcg->tcpmem_active) {
2940 * The active flag needs to be written after the static_key
2941 * update. This is what guarantees that the socket activation
2942 * function is the last one to run. See sock_update_memcg() for
2943 * details, and note that we don't mark any socket as belonging
2944 * to this memcg until that flag is up.
2946 * We need to do this, because static_keys will span multiple
2947 * sites, but we can't control their order. If we mark a socket
2948 * as accounted, but the accounting functions are not patched in
2949 * yet, we'll lose accounting.
2951 * We never race with the readers in sock_update_memcg(),
2952 * because when this value change, the code to process it is not
2955 static_branch_inc(&memcg_sockets_enabled_key);
2956 memcg->tcpmem_active = true;
2959 mutex_unlock(&memcg_limit_mutex);
2964 * The user of this function is...
2967 static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
2968 char *buf, size_t nbytes, loff_t off)
2970 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
2971 unsigned long nr_pages;
2974 buf = strstrip(buf);
2975 ret = page_counter_memparse(buf, "-1", &nr_pages);
2979 switch (MEMFILE_ATTR(of_cft(of)->private)) {
2981 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
2985 switch (MEMFILE_TYPE(of_cft(of)->private)) {
2987 ret = mem_cgroup_resize_limit(memcg, nr_pages);
2990 ret = mem_cgroup_resize_memsw_limit(memcg, nr_pages);
2993 ret = memcg_update_kmem_limit(memcg, nr_pages);
2996 ret = memcg_update_tcp_limit(memcg, nr_pages);
3000 case RES_SOFT_LIMIT:
3001 memcg->soft_limit = nr_pages;
3005 return ret ?: nbytes;
3008 static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
3009 size_t nbytes, loff_t off)
3011 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
3012 struct page_counter *counter;
3014 switch (MEMFILE_TYPE(of_cft(of)->private)) {
3016 counter = &memcg->memory;
3019 counter = &memcg->memsw;
3022 counter = &memcg->kmem;
3025 counter = &memcg->tcpmem;
3031 switch (MEMFILE_ATTR(of_cft(of)->private)) {
3033 page_counter_reset_watermark(counter);
3036 counter->failcnt = 0;
3045 static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
3048 return mem_cgroup_from_css(css)->move_charge_at_immigrate;
3052 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3053 struct cftype *cft, u64 val)
3055 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3057 if (val & ~MOVE_MASK)
3061 * No kind of locking is needed in here, because ->can_attach() will
3062 * check this value once in the beginning of the process, and then carry
3063 * on with stale data. This means that changes to this value will only
3064 * affect task migrations starting after the change.
3066 memcg->move_charge_at_immigrate = val;
3070 static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
3071 struct cftype *cft, u64 val)
3078 static int memcg_numa_stat_show(struct seq_file *m, void *v)
3082 unsigned int lru_mask;
3085 static const struct numa_stat stats[] = {
3086 { "total", LRU_ALL },
3087 { "file", LRU_ALL_FILE },
3088 { "anon", LRU_ALL_ANON },
3089 { "unevictable", BIT(LRU_UNEVICTABLE) },
3091 const struct numa_stat *stat;
3094 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3096 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3097 nr = mem_cgroup_nr_lru_pages(memcg, stat->lru_mask);
3098 seq_printf(m, "%s=%lu", stat->name, nr);
3099 for_each_node_state(nid, N_MEMORY) {
3100 nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
3102 seq_printf(m, " N%d=%lu", nid, nr);
3107 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
3108 struct mem_cgroup *iter;
3111 for_each_mem_cgroup_tree(iter, memcg)
3112 nr += mem_cgroup_nr_lru_pages(iter, stat->lru_mask);
3113 seq_printf(m, "hierarchical_%s=%lu", stat->name, nr);
3114 for_each_node_state(nid, N_MEMORY) {
3116 for_each_mem_cgroup_tree(iter, memcg)
3117 nr += mem_cgroup_node_nr_lru_pages(
3118 iter, nid, stat->lru_mask);
3119 seq_printf(m, " N%d=%lu", nid, nr);
3126 #endif /* CONFIG_NUMA */
3128 static int memcg_stat_show(struct seq_file *m, void *v)
3130 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
3131 unsigned long memory, memsw;
3132 struct mem_cgroup *mi;
3135 BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_stat_names) !=
3136 MEM_CGROUP_STAT_NSTATS);
3137 BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_events_names) !=
3138 MEM_CGROUP_EVENTS_NSTATS);
3139 BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);
3141 for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
3142 if (i == MEM_CGROUP_STAT_SWAP && !do_memsw_account())
3144 seq_printf(m, "%s %lu\n", mem_cgroup_stat_names[i],
3145 mem_cgroup_read_stat(memcg, i) * PAGE_SIZE);
3148 for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++)
3149 seq_printf(m, "%s %lu\n", mem_cgroup_events_names[i],
3150 mem_cgroup_read_events(memcg, i));
3152 for (i = 0; i < NR_LRU_LISTS; i++)
3153 seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
3154 mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);
3156 /* Hierarchical information */
3157 memory = memsw = PAGE_COUNTER_MAX;
3158 for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
3159 memory = min(memory, mi->memory.limit);
3160 memsw = min(memsw, mi->memsw.limit);
3162 seq_printf(m, "hierarchical_memory_limit %llu\n",
3163 (u64)memory * PAGE_SIZE);
3164 if (do_memsw_account())
3165 seq_printf(m, "hierarchical_memsw_limit %llu\n",
3166 (u64)memsw * PAGE_SIZE);
3168 for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
3169 unsigned long long val = 0;
3171 if (i == MEM_CGROUP_STAT_SWAP && !do_memsw_account())
3173 for_each_mem_cgroup_tree(mi, memcg)
3174 val += mem_cgroup_read_stat(mi, i) * PAGE_SIZE;
3175 seq_printf(m, "total_%s %llu\n", mem_cgroup_stat_names[i], val);
3178 for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
3179 unsigned long long val = 0;
3181 for_each_mem_cgroup_tree(mi, memcg)
3182 val += mem_cgroup_read_events(mi, i);
3183 seq_printf(m, "total_%s %llu\n",
3184 mem_cgroup_events_names[i], val);
3187 for (i = 0; i < NR_LRU_LISTS; i++) {
3188 unsigned long long val = 0;
3190 for_each_mem_cgroup_tree(mi, memcg)
3191 val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE;
3192 seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val);
3195 #ifdef CONFIG_DEBUG_VM
3198 struct mem_cgroup_per_node *mz;
3199 struct zone_reclaim_stat *rstat;
3200 unsigned long recent_rotated[2] = {0, 0};
3201 unsigned long recent_scanned[2] = {0, 0};
3203 for_each_online_pgdat(pgdat) {
3204 mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
3205 rstat = &mz->lruvec.reclaim_stat;
3207 recent_rotated[0] += rstat->recent_rotated[0];
3208 recent_rotated[1] += rstat->recent_rotated[1];
3209 recent_scanned[0] += rstat->recent_scanned[0];
3210 recent_scanned[1] += rstat->recent_scanned[1];
3212 seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
3213 seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
3214 seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
3215 seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
3222 static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
3225 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3227 return mem_cgroup_swappiness(memcg);
3230 static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
3231 struct cftype *cft, u64 val)
3233 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3239 memcg->swappiness = val;
3241 vm_swappiness = val;
3246 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
3248 struct mem_cgroup_threshold_ary *t;
3249 unsigned long usage;
3254 t = rcu_dereference(memcg->thresholds.primary);
3256 t = rcu_dereference(memcg->memsw_thresholds.primary);
3261 usage = mem_cgroup_usage(memcg, swap);
3264 * current_threshold points to threshold just below or equal to usage.
3265 * If it's not true, a threshold was crossed after last
3266 * call of __mem_cgroup_threshold().
3268 i = t->current_threshold;
3271 * Iterate backward over array of thresholds starting from
3272 * current_threshold and check if a threshold is crossed.
3273 * If none of thresholds below usage is crossed, we read
3274 * only one element of the array here.
3276 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
3277 eventfd_signal(t->entries[i].eventfd, 1);
3279 /* i = current_threshold + 1 */
3283 * Iterate forward over array of thresholds starting from
3284 * current_threshold+1 and check if a threshold is crossed.
3285 * If none of thresholds above usage is crossed, we read
3286 * only one element of the array here.
3288 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
3289 eventfd_signal(t->entries[i].eventfd, 1);
3291 /* Update current_threshold */
3292 t->current_threshold = i - 1;
3297 static void mem_cgroup_threshold(struct mem_cgroup *memcg)
3300 __mem_cgroup_threshold(memcg, false);
3301 if (do_memsw_account())
3302 __mem_cgroup_threshold(memcg, true);
3304 memcg = parent_mem_cgroup(memcg);
3308 static int compare_thresholds(const void *a, const void *b)
3310 const struct mem_cgroup_threshold *_a = a;
3311 const struct mem_cgroup_threshold *_b = b;
3313 if (_a->threshold > _b->threshold)
3316 if (_a->threshold < _b->threshold)
3322 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
3324 struct mem_cgroup_eventfd_list *ev;
3326 spin_lock(&memcg_oom_lock);
3328 list_for_each_entry(ev, &memcg->oom_notify, list)
3329 eventfd_signal(ev->eventfd, 1);
3331 spin_unlock(&memcg_oom_lock);
3335 static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
3337 struct mem_cgroup *iter;
3339 for_each_mem_cgroup_tree(iter, memcg)
3340 mem_cgroup_oom_notify_cb(iter);
3343 static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
3344 struct eventfd_ctx *eventfd, const char *args, enum res_type type)
3346 struct mem_cgroup_thresholds *thresholds;
3347 struct mem_cgroup_threshold_ary *new;
3348 unsigned long threshold;
3349 unsigned long usage;
3352 ret = page_counter_memparse(args, "-1", &threshold);
3356 mutex_lock(&memcg->thresholds_lock);
3359 thresholds = &memcg->thresholds;
3360 usage = mem_cgroup_usage(memcg, false);
3361 } else if (type == _MEMSWAP) {
3362 thresholds = &memcg->memsw_thresholds;
3363 usage = mem_cgroup_usage(memcg, true);
3367 /* Check if a threshold crossed before adding a new one */
3368 if (thresholds->primary)
3369 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3371 size = thresholds->primary ? thresholds->primary->size + 1 : 1;
3373 /* Allocate memory for new array of thresholds */
3374 new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
3382 /* Copy thresholds (if any) to new array */
3383 if (thresholds->primary) {
3384 memcpy(new->entries, thresholds->primary->entries, (size - 1) *
3385 sizeof(struct mem_cgroup_threshold));
3388 /* Add new threshold */
3389 new->entries[size - 1].eventfd = eventfd;
3390 new->entries[size - 1].threshold = threshold;
3392 /* Sort thresholds. Registering of new threshold isn't time-critical */
3393 sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
3394 compare_thresholds, NULL);
3396 /* Find current threshold */
3397 new->current_threshold = -1;
3398 for (i = 0; i < size; i++) {
3399 if (new->entries[i].threshold <= usage) {
3401 * new->current_threshold will not be used until
3402 * rcu_assign_pointer(), so it's safe to increment
3405 ++new->current_threshold;
3410 /* Free old spare buffer and save old primary buffer as spare */
3411 kfree(thresholds->spare);
3412 thresholds->spare = thresholds->primary;
3414 rcu_assign_pointer(thresholds->primary, new);
3416 /* To be sure that nobody uses thresholds */
3420 mutex_unlock(&memcg->thresholds_lock);
3425 static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
3426 struct eventfd_ctx *eventfd, const char *args)
3428 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
3431 static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
3432 struct eventfd_ctx *eventfd, const char *args)
3434 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
3437 static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3438 struct eventfd_ctx *eventfd, enum res_type type)
3440 struct mem_cgroup_thresholds *thresholds;
3441 struct mem_cgroup_threshold_ary *new;
3442 unsigned long usage;
3445 mutex_lock(&memcg->thresholds_lock);
3448 thresholds = &memcg->thresholds;
3449 usage = mem_cgroup_usage(memcg, false);
3450 } else if (type == _MEMSWAP) {
3451 thresholds = &memcg->memsw_thresholds;
3452 usage = mem_cgroup_usage(memcg, true);
3456 if (!thresholds->primary)
3459 /* Check if a threshold crossed before removing */
3460 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3462 /* Calculate new number of threshold */
3464 for (i = 0; i < thresholds->primary->size; i++) {
3465 if (thresholds->primary->entries[i].eventfd != eventfd)
3469 new = thresholds->spare;
3471 /* Set thresholds array to NULL if we don't have thresholds */
3480 /* Copy thresholds and find current threshold */
3481 new->current_threshold = -1;
3482 for (i = 0, j = 0; i < thresholds->primary->size; i++) {
3483 if (thresholds->primary->entries[i].eventfd == eventfd)
3486 new->entries[j] = thresholds->primary->entries[i];
3487 if (new->entries[j].threshold <= usage) {
3489 * new->current_threshold will not be used
3490 * until rcu_assign_pointer(), so it's safe to increment
3493 ++new->current_threshold;
3499 /* Swap primary and spare array */
3500 thresholds->spare = thresholds->primary;
3502 rcu_assign_pointer(thresholds->primary, new);
3504 /* To be sure that nobody uses thresholds */
3507 /* If all events are unregistered, free the spare array */
3509 kfree(thresholds->spare);
3510 thresholds->spare = NULL;
3513 mutex_unlock(&memcg->thresholds_lock);
3516 static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3517 struct eventfd_ctx *eventfd)
3519 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
3522 static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
3523 struct eventfd_ctx *eventfd)
3525 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
3528 static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
3529 struct eventfd_ctx *eventfd, const char *args)
3531 struct mem_cgroup_eventfd_list *event;
3533 event = kmalloc(sizeof(*event), GFP_KERNEL);
3537 spin_lock(&memcg_oom_lock);
3539 event->eventfd = eventfd;
3540 list_add(&event->list, &memcg->oom_notify);
3542 /* already in OOM ? */
3543 if (memcg->under_oom)
3544 eventfd_signal(eventfd, 1);
3545 spin_unlock(&memcg_oom_lock);
3550 static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
3551 struct eventfd_ctx *eventfd)
3553 struct mem_cgroup_eventfd_list *ev, *tmp;
3555 spin_lock(&memcg_oom_lock);
3557 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
3558 if (ev->eventfd == eventfd) {
3559 list_del(&ev->list);
3564 spin_unlock(&memcg_oom_lock);
3567 static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
3569 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(sf));
3571 seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
3572 seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
3576 static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
3577 struct cftype *cft, u64 val)
3579 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3581 /* cannot set to root cgroup and only 0 and 1 are allowed */
3582 if (!css->parent || !((val == 0) || (val == 1)))
3585 memcg->oom_kill_disable = val;
3587 memcg_oom_recover(memcg);
3592 #ifdef CONFIG_CGROUP_WRITEBACK
3594 struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg)
3596 return &memcg->cgwb_list;
3599 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
3601 return wb_domain_init(&memcg->cgwb_domain, gfp);
3604 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
3606 wb_domain_exit(&memcg->cgwb_domain);
3609 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
3611 wb_domain_size_changed(&memcg->cgwb_domain);
3614 struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
3616 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
3618 if (!memcg->css.parent)
3621 return &memcg->cgwb_domain;
3625 * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
3626 * @wb: bdi_writeback in question
3627 * @pfilepages: out parameter for number of file pages
3628 * @pheadroom: out parameter for number of allocatable pages according to memcg
3629 * @pdirty: out parameter for number of dirty pages
3630 * @pwriteback: out parameter for number of pages under writeback
3632 * Determine the numbers of file, headroom, dirty, and writeback pages in
3633 * @wb's memcg. File, dirty and writeback are self-explanatory. Headroom
3634 * is a bit more involved.
3636 * A memcg's headroom is "min(max, high) - used". In the hierarchy, the
3637 * headroom is calculated as the lowest headroom of itself and the
3638 * ancestors. Note that this doesn't consider the actual amount of
3639 * available memory in the system. The caller should further cap
3640 * *@pheadroom accordingly.
3642 void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
3643 unsigned long *pheadroom, unsigned long *pdirty,
3644 unsigned long *pwriteback)
3646 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
3647 struct mem_cgroup *parent;
3649 *pdirty = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_DIRTY);
3651 /* this should eventually include NR_UNSTABLE_NFS */
3652 *pwriteback = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_WRITEBACK);
3653 *pfilepages = mem_cgroup_nr_lru_pages(memcg, (1 << LRU_INACTIVE_FILE) |
3654 (1 << LRU_ACTIVE_FILE));
3655 *pheadroom = PAGE_COUNTER_MAX;
3657 while ((parent = parent_mem_cgroup(memcg))) {
3658 unsigned long ceiling = min(memcg->memory.limit, memcg->high);
3659 unsigned long used = page_counter_read(&memcg->memory);
3661 *pheadroom = min(*pheadroom, ceiling - min(ceiling, used));
3666 #else /* CONFIG_CGROUP_WRITEBACK */
3668 static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
3673 static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
3677 static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
3681 #endif /* CONFIG_CGROUP_WRITEBACK */
3684 * DO NOT USE IN NEW FILES.
3686 * "cgroup.event_control" implementation.
3688 * This is way over-engineered. It tries to support fully configurable
3689 * events for each user. Such level of flexibility is completely
3690 * unnecessary especially in the light of the planned unified hierarchy.
3692 * Please deprecate this and replace with something simpler if at all
3697 * Unregister event and free resources.
3699 * Gets called from workqueue.
3701 static void memcg_event_remove(struct work_struct *work)
3703 struct mem_cgroup_event *event =
3704 container_of(work, struct mem_cgroup_event, remove);
3705 struct mem_cgroup *memcg = event->memcg;
3707 remove_wait_queue(event->wqh, &event->wait);
3709 event->unregister_event(memcg, event->eventfd);
3711 /* Notify userspace the event is going away. */
3712 eventfd_signal(event->eventfd, 1);
3714 eventfd_ctx_put(event->eventfd);
3716 css_put(&memcg->css);
3720 * Gets called on POLLHUP on eventfd when user closes it.
3722 * Called with wqh->lock held and interrupts disabled.
3724 static int memcg_event_wake(wait_queue_t *wait, unsigned mode,
3725 int sync, void *key)
3727 struct mem_cgroup_event *event =
3728 container_of(wait, struct mem_cgroup_event, wait);
3729 struct mem_cgroup *memcg = event->memcg;
3730 unsigned long flags = (unsigned long)key;
3732 if (flags & POLLHUP) {
3734 * If the event has been detached at cgroup removal, we
3735 * can simply return knowing the other side will cleanup
3738 * We can't race against event freeing since the other
3739 * side will require wqh->lock via remove_wait_queue(),
3742 spin_lock(&memcg->event_list_lock);
3743 if (!list_empty(&event->list)) {
3744 list_del_init(&event->list);
3746 * We are in atomic context, but cgroup_event_remove()
3747 * may sleep, so we have to call it in workqueue.
3749 schedule_work(&event->remove);
3751 spin_unlock(&memcg->event_list_lock);
3757 static void memcg_event_ptable_queue_proc(struct file *file,
3758 wait_queue_head_t *wqh, poll_table *pt)
3760 struct mem_cgroup_event *event =
3761 container_of(pt, struct mem_cgroup_event, pt);
3764 add_wait_queue(wqh, &event->wait);
3768 * DO NOT USE IN NEW FILES.
3770 * Parse input and register new cgroup event handler.
3772 * Input must be in format '<event_fd> <control_fd> <args>'.
3773 * Interpretation of args is defined by control file implementation.
3775 static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
3776 char *buf, size_t nbytes, loff_t off)
3778 struct cgroup_subsys_state *css = of_css(of);
3779 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
3780 struct mem_cgroup_event *event;
3781 struct cgroup_subsys_state *cfile_css;
3782 unsigned int efd, cfd;
3789 buf = strstrip(buf);
3791 efd = simple_strtoul(buf, &endp, 10);
3796 cfd = simple_strtoul(buf, &endp, 10);
3797 if ((*endp != ' ') && (*endp != '\0'))
3801 event = kzalloc(sizeof(*event), GFP_KERNEL);
3805 event->memcg = memcg;
3806 INIT_LIST_HEAD(&event->list);
3807 init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
3808 init_waitqueue_func_entry(&event->wait, memcg_event_wake);
3809 INIT_WORK(&event->remove, memcg_event_remove);
3817 event->eventfd = eventfd_ctx_fileget(efile.file);
3818 if (IS_ERR(event->eventfd)) {
3819 ret = PTR_ERR(event->eventfd);
3826 goto out_put_eventfd;
3829 /* the process need read permission on control file */
3830 /* AV: shouldn't we check that it's been opened for read instead? */
3831 ret = inode_permission(file_inode(cfile.file), MAY_READ);
3836 * Determine the event callbacks and set them in @event. This used
3837 * to be done via struct cftype but cgroup core no longer knows
3838 * about these events. The following is crude but the whole thing
3839 * is for compatibility anyway.
3841 * DO NOT ADD NEW FILES.
3843 name = cfile.file->f_path.dentry->d_name.name;
3845 if (!strcmp(name, "memory.usage_in_bytes")) {
3846 event->register_event = mem_cgroup_usage_register_event;
3847 event->unregister_event = mem_cgroup_usage_unregister_event;
3848 } else if (!strcmp(name, "memory.oom_control")) {
3849 event->register_event = mem_cgroup_oom_register_event;
3850 event->unregister_event = mem_cgroup_oom_unregister_event;
3851 } else if (!strcmp(name, "memory.pressure_level")) {
3852 event->register_event = vmpressure_register_event;
3853 event->unregister_event = vmpressure_unregister_event;
3854 } else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
3855 event->register_event = memsw_cgroup_usage_register_event;
3856 event->unregister_event = memsw_cgroup_usage_unregister_event;
3863 * Verify @cfile should belong to @css. Also, remaining events are
3864 * automatically removed on cgroup destruction but the removal is
3865 * asynchronous, so take an extra ref on @css.
3867 cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent,
3868 &memory_cgrp_subsys);
3870 if (IS_ERR(cfile_css))
3872 if (cfile_css != css) {
3877 ret = event->register_event(memcg, event->eventfd, buf);
3881 efile.file->f_op->poll(efile.file, &event->pt);
3883 spin_lock(&memcg->event_list_lock);
3884 list_add(&event->list, &memcg->event_list);
3885 spin_unlock(&memcg->event_list_lock);
3897 eventfd_ctx_put(event->eventfd);
3906 static struct cftype mem_cgroup_legacy_files[] = {
3908 .name = "usage_in_bytes",
3909 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
3910 .read_u64 = mem_cgroup_read_u64,
3913 .name = "max_usage_in_bytes",
3914 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
3915 .write = mem_cgroup_reset,
3916 .read_u64 = mem_cgroup_read_u64,
3919 .name = "limit_in_bytes",
3920 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
3921 .write = mem_cgroup_write,
3922 .read_u64 = mem_cgroup_read_u64,
3925 .name = "soft_limit_in_bytes",
3926 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
3927 .write = mem_cgroup_write,
3928 .read_u64 = mem_cgroup_read_u64,
3932 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
3933 .write = mem_cgroup_reset,
3934 .read_u64 = mem_cgroup_read_u64,
3938 .seq_show = memcg_stat_show,
3941 .name = "force_empty",
3942 .write = mem_cgroup_force_empty_write,
3945 .name = "use_hierarchy",
3946 .write_u64 = mem_cgroup_hierarchy_write,
3947 .read_u64 = mem_cgroup_hierarchy_read,
3950 .name = "cgroup.event_control", /* XXX: for compat */
3951 .write = memcg_write_event_control,
3952 .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE,
3955 .name = "swappiness",
3956 .read_u64 = mem_cgroup_swappiness_read,
3957 .write_u64 = mem_cgroup_swappiness_write,
3960 .name = "move_charge_at_immigrate",
3961 .read_u64 = mem_cgroup_move_charge_read,
3962 .write_u64 = mem_cgroup_move_charge_write,
3965 .name = "oom_control",
3966 .seq_show = mem_cgroup_oom_control_read,
3967 .write_u64 = mem_cgroup_oom_control_write,
3968 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
3971 .name = "pressure_level",
3975 .name = "numa_stat",
3976 .seq_show = memcg_numa_stat_show,
3980 .name = "kmem.limit_in_bytes",
3981 .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
3982 .write = mem_cgroup_write,
3983 .read_u64 = mem_cgroup_read_u64,
3986 .name = "kmem.usage_in_bytes",
3987 .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
3988 .read_u64 = mem_cgroup_read_u64,
3991 .name = "kmem.failcnt",
3992 .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
3993 .write = mem_cgroup_reset,
3994 .read_u64 = mem_cgroup_read_u64,
3997 .name = "kmem.max_usage_in_bytes",
3998 .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
3999 .write = mem_cgroup_reset,
4000 .read_u64 = mem_cgroup_read_u64,
4002 #ifdef CONFIG_SLABINFO
4004 .name = "kmem.slabinfo",
4005 .seq_start = slab_start,
4006 .seq_next = slab_next,
4007 .seq_stop = slab_stop,
4008 .seq_show = memcg_slab_show,
4012 .name = "kmem.tcp.limit_in_bytes",
4013 .private = MEMFILE_PRIVATE(_TCP, RES_LIMIT),
4014 .write = mem_cgroup_write,
4015 .read_u64 = mem_cgroup_read_u64,
4018 .name = "kmem.tcp.usage_in_bytes",
4019 .private = MEMFILE_PRIVATE(_TCP, RES_USAGE),
4020 .read_u64 = mem_cgroup_read_u64,
4023 .name = "kmem.tcp.failcnt",
4024 .private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT),
4025 .write = mem_cgroup_reset,
4026 .read_u64 = mem_cgroup_read_u64,
4029 .name = "kmem.tcp.max_usage_in_bytes",
4030 .private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE),
4031 .write = mem_cgroup_reset,
4032 .read_u64 = mem_cgroup_read_u64,
4034 { }, /* terminate */
4038 * Private memory cgroup IDR
4040 * Swap-out records and page cache shadow entries need to store memcg
4041 * references in constrained space, so we maintain an ID space that is
4042 * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of
4043 * memory-controlled cgroups to 64k.
4045 * However, there usually are many references to the oflline CSS after
4046 * the cgroup has been destroyed, such as page cache or reclaimable
4047 * slab objects, that don't need to hang on to the ID. We want to keep
4048 * those dead CSS from occupying IDs, or we might quickly exhaust the
4049 * relatively small ID space and prevent the creation of new cgroups
4050 * even when there are much fewer than 64k cgroups - possibly none.
4052 * Maintain a private 16-bit ID space for memcg, and allow the ID to
4053 * be freed and recycled when it's no longer needed, which is usually
4054 * when the CSS is offlined.
4056 * The only exception to that are records of swapped out tmpfs/shmem
4057 * pages that need to be attributed to live ancestors on swapin. But
4058 * those references are manageable from userspace.
4061 static DEFINE_IDR(mem_cgroup_idr);
4063 static void mem_cgroup_id_get(struct mem_cgroup *memcg)
4065 atomic_inc(&memcg->id.ref);
4068 static void mem_cgroup_id_put(struct mem_cgroup *memcg)
4070 if (atomic_dec_and_test(&memcg->id.ref)) {
4071 idr_remove(&mem_cgroup_idr, memcg->id.id);
4074 /* Memcg ID pins CSS */
4075 css_put(&memcg->css);
4080 * mem_cgroup_from_id - look up a memcg from a memcg id
4081 * @id: the memcg id to look up
4083 * Caller must hold rcu_read_lock().
4085 struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
4087 WARN_ON_ONCE(!rcu_read_lock_held());
4088 return idr_find(&mem_cgroup_idr, id);
4091 static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
4093 struct mem_cgroup_per_node *pn;
4096 * This routine is called against possible nodes.
4097 * But it's BUG to call kmalloc() against offline node.
4099 * TODO: this routine can waste much memory for nodes which will
4100 * never be onlined. It's better to use memory hotplug callback
4103 if (!node_state(node, N_NORMAL_MEMORY))
4105 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
4109 lruvec_init(&pn->lruvec);
4110 pn->usage_in_excess = 0;
4111 pn->on_tree = false;
4114 memcg->nodeinfo[node] = pn;
4118 static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
4120 kfree(memcg->nodeinfo[node]);
4123 static void mem_cgroup_free(struct mem_cgroup *memcg)
4127 memcg_wb_domain_exit(memcg);
4129 free_mem_cgroup_per_node_info(memcg, node);
4130 free_percpu(memcg->stat);
4134 static struct mem_cgroup *mem_cgroup_alloc(void)
4136 struct mem_cgroup *memcg;
4140 size = sizeof(struct mem_cgroup);
4141 size += nr_node_ids * sizeof(struct mem_cgroup_per_node *);
4143 memcg = kzalloc(size, GFP_KERNEL);
4147 memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,
4148 1, MEM_CGROUP_ID_MAX,
4150 if (memcg->id.id < 0)
4153 memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
4158 if (alloc_mem_cgroup_per_node_info(memcg, node))
4161 if (memcg_wb_domain_init(memcg, GFP_KERNEL))
4164 INIT_WORK(&memcg->high_work, high_work_func);
4165 memcg->last_scanned_node = MAX_NUMNODES;
4166 INIT_LIST_HEAD(&memcg->oom_notify);
4167 mutex_init(&memcg->thresholds_lock);
4168 spin_lock_init(&memcg->move_lock);
4169 vmpressure_init(&memcg->vmpressure);
4170 INIT_LIST_HEAD(&memcg->event_list);
4171 spin_lock_init(&memcg->event_list_lock);
4172 memcg->socket_pressure = jiffies;
4174 memcg->kmemcg_id = -1;
4176 #ifdef CONFIG_CGROUP_WRITEBACK
4177 INIT_LIST_HEAD(&memcg->cgwb_list);
4179 idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
4182 if (memcg->id.id > 0)
4183 idr_remove(&mem_cgroup_idr, memcg->id.id);
4184 mem_cgroup_free(memcg);
4188 static struct cgroup_subsys_state * __ref
4189 mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
4191 struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
4192 struct mem_cgroup *memcg;
4193 long error = -ENOMEM;
4195 memcg = mem_cgroup_alloc();
4197 return ERR_PTR(error);
4199 memcg->high = PAGE_COUNTER_MAX;
4200 memcg->soft_limit = PAGE_COUNTER_MAX;
4202 memcg->swappiness = mem_cgroup_swappiness(parent);
4203 memcg->oom_kill_disable = parent->oom_kill_disable;
4205 if (parent && parent->use_hierarchy) {
4206 memcg->use_hierarchy = true;
4207 page_counter_init(&memcg->memory, &parent->memory);
4208 page_counter_init(&memcg->swap, &parent->swap);
4209 page_counter_init(&memcg->memsw, &parent->memsw);
4210 page_counter_init(&memcg->kmem, &parent->kmem);
4211 page_counter_init(&memcg->tcpmem, &parent->tcpmem);
4213 page_counter_init(&memcg->memory, NULL);
4214 page_counter_init(&memcg->swap, NULL);
4215 page_counter_init(&memcg->memsw, NULL);
4216 page_counter_init(&memcg->kmem, NULL);
4217 page_counter_init(&memcg->tcpmem, NULL);
4219 * Deeper hierachy with use_hierarchy == false doesn't make
4220 * much sense so let cgroup subsystem know about this
4221 * unfortunate state in our controller.
4223 if (parent != root_mem_cgroup)
4224 memory_cgrp_subsys.broken_hierarchy = true;
4227 /* The following stuff does not apply to the root */
4229 root_mem_cgroup = memcg;
4233 error = memcg_online_kmem(memcg);
4237 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
4238 static_branch_inc(&memcg_sockets_enabled_key);
4242 mem_cgroup_free(memcg);
4243 return ERR_PTR(-ENOMEM);
4246 static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
4248 /* Online state pins memcg ID, memcg ID pins CSS */
4249 mem_cgroup_id_get(mem_cgroup_from_css(css));
4254 static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
4256 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4257 struct mem_cgroup_event *event, *tmp;
4260 * Unregister events and notify userspace.
4261 * Notify userspace about cgroup removing only after rmdir of cgroup
4262 * directory to avoid race between userspace and kernelspace.
4264 spin_lock(&memcg->event_list_lock);
4265 list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
4266 list_del_init(&event->list);
4267 schedule_work(&event->remove);
4269 spin_unlock(&memcg->event_list_lock);
4271 memcg_offline_kmem(memcg);
4272 wb_memcg_offline(memcg);
4274 mem_cgroup_id_put(memcg);
4277 static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
4279 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4281 invalidate_reclaim_iterators(memcg);
4284 static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
4286 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4288 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
4289 static_branch_dec(&memcg_sockets_enabled_key);
4291 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
4292 static_branch_dec(&memcg_sockets_enabled_key);
4294 vmpressure_cleanup(&memcg->vmpressure);
4295 cancel_work_sync(&memcg->high_work);
4296 mem_cgroup_remove_from_trees(memcg);
4297 memcg_free_kmem(memcg);
4298 mem_cgroup_free(memcg);
4302 * mem_cgroup_css_reset - reset the states of a mem_cgroup
4303 * @css: the target css
4305 * Reset the states of the mem_cgroup associated with @css. This is
4306 * invoked when the userland requests disabling on the default hierarchy
4307 * but the memcg is pinned through dependency. The memcg should stop
4308 * applying policies and should revert to the vanilla state as it may be
4309 * made visible again.
4311 * The current implementation only resets the essential configurations.
4312 * This needs to be expanded to cover all the visible parts.
4314 static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
4316 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
4318 page_counter_limit(&memcg->memory, PAGE_COUNTER_MAX);
4319 page_counter_limit(&memcg->swap, PAGE_COUNTER_MAX);
4320 page_counter_limit(&memcg->memsw, PAGE_COUNTER_MAX);
4321 page_counter_limit(&memcg->kmem, PAGE_COUNTER_MAX);
4322 page_counter_limit(&memcg->tcpmem, PAGE_COUNTER_MAX);
4324 memcg->high = PAGE_COUNTER_MAX;
4325 memcg->soft_limit = PAGE_COUNTER_MAX;
4326 memcg_wb_domain_size_changed(memcg);
4330 /* Handlers for move charge at task migration. */
4331 static int mem_cgroup_do_precharge(unsigned long count)
4335 /* Try a single bulk charge without reclaim first, kswapd may wake */
4336 ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
4338 mc.precharge += count;
4342 /* Try charges one by one with reclaim */
4344 ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_NORETRY, 1);
4358 enum mc_target_type {
4364 static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
4365 unsigned long addr, pte_t ptent)
4367 struct page *page = vm_normal_page(vma, addr, ptent);
4369 if (!page || !page_mapped(page))
4371 if (PageAnon(page)) {
4372 if (!(mc.flags & MOVE_ANON))
4375 if (!(mc.flags & MOVE_FILE))
4378 if (!get_page_unless_zero(page))
4385 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4386 pte_t ptent, swp_entry_t *entry)
4388 struct page *page = NULL;
4389 swp_entry_t ent = pte_to_swp_entry(ptent);
4391 if (!(mc.flags & MOVE_ANON) || non_swap_entry(ent))
4394 * Because lookup_swap_cache() updates some statistics counter,
4395 * we call find_get_page() with swapper_space directly.
4397 page = find_get_page(swap_address_space(ent), ent.val);
4398 if (do_memsw_account())
4399 entry->val = ent.val;
4404 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4405 pte_t ptent, swp_entry_t *entry)
4411 static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
4412 unsigned long addr, pte_t ptent, swp_entry_t *entry)
4414 struct page *page = NULL;
4415 struct address_space *mapping;
4418 if (!vma->vm_file) /* anonymous vma */
4420 if (!(mc.flags & MOVE_FILE))
4423 mapping = vma->vm_file->f_mapping;
4424 pgoff = linear_page_index(vma, addr);
4426 /* page is moved even if it's not RSS of this task(page-faulted). */
4428 /* shmem/tmpfs may report page out on swap: account for that too. */
4429 if (shmem_mapping(mapping)) {
4430 page = find_get_entry(mapping, pgoff);
4431 if (radix_tree_exceptional_entry(page)) {
4432 swp_entry_t swp = radix_to_swp_entry(page);
4433 if (do_memsw_account())
4435 page = find_get_page(swap_address_space(swp), swp.val);
4438 page = find_get_page(mapping, pgoff);
4440 page = find_get_page(mapping, pgoff);
4446 * mem_cgroup_move_account - move account of the page
4448 * @compound: charge the page as compound or small page
4449 * @from: mem_cgroup which the page is moved from.
4450 * @to: mem_cgroup which the page is moved to. @from != @to.
4452 * The caller must make sure the page is not on LRU (isolate_page() is useful.)
4454 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
4457 static int mem_cgroup_move_account(struct page *page,
4459 struct mem_cgroup *from,
4460 struct mem_cgroup *to)
4462 unsigned long flags;
4463 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
4467 VM_BUG_ON(from == to);
4468 VM_BUG_ON_PAGE(PageLRU(page), page);
4469 VM_BUG_ON(compound && !PageTransHuge(page));
4472 * Prevent mem_cgroup_migrate() from looking at
4473 * page->mem_cgroup of its source page while we change it.
4476 if (!trylock_page(page))
4480 if (page->mem_cgroup != from)
4483 anon = PageAnon(page);
4485 spin_lock_irqsave(&from->move_lock, flags);
4487 if (!anon && page_mapped(page)) {
4488 __this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED],
4490 __this_cpu_add(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED],
4495 * move_lock grabbed above and caller set from->moving_account, so
4496 * mem_cgroup_update_page_stat() will serialize updates to PageDirty.
4497 * So mapping should be stable for dirty pages.
4499 if (!anon && PageDirty(page)) {
4500 struct address_space *mapping = page_mapping(page);
4502 if (mapping_cap_account_dirty(mapping)) {
4503 __this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_DIRTY],
4505 __this_cpu_add(to->stat->count[MEM_CGROUP_STAT_DIRTY],
4510 if (PageWriteback(page)) {
4511 __this_cpu_sub(from->stat->count[MEM_CGROUP_STAT_WRITEBACK],
4513 __this_cpu_add(to->stat->count[MEM_CGROUP_STAT_WRITEBACK],
4518 * It is safe to change page->mem_cgroup here because the page
4519 * is referenced, charged, and isolated - we can't race with
4520 * uncharging, charging, migration, or LRU putback.
4523 /* caller should have done css_get */
4524 page->mem_cgroup = to;
4525 spin_unlock_irqrestore(&from->move_lock, flags);
4529 local_irq_disable();
4530 mem_cgroup_charge_statistics(to, page, compound, nr_pages);
4531 memcg_check_events(to, page);
4532 mem_cgroup_charge_statistics(from, page, compound, -nr_pages);
4533 memcg_check_events(from, page);
4542 * get_mctgt_type - get target type of moving charge
4543 * @vma: the vma the pte to be checked belongs
4544 * @addr: the address corresponding to the pte to be checked
4545 * @ptent: the pte to be checked
4546 * @target: the pointer the target page or swap ent will be stored(can be NULL)
4549 * 0(MC_TARGET_NONE): if the pte is not a target for move charge.
4550 * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
4551 * move charge. if @target is not NULL, the page is stored in target->page
4552 * with extra refcnt got(Callers should handle it).
4553 * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
4554 * target for charge migration. if @target is not NULL, the entry is stored
4557 * Called with pte lock held.
4560 static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
4561 unsigned long addr, pte_t ptent, union mc_target *target)
4563 struct page *page = NULL;
4564 enum mc_target_type ret = MC_TARGET_NONE;
4565 swp_entry_t ent = { .val = 0 };
4567 if (pte_present(ptent))
4568 page = mc_handle_present_pte(vma, addr, ptent);
4569 else if (is_swap_pte(ptent))
4570 page = mc_handle_swap_pte(vma, ptent, &ent);
4571 else if (pte_none(ptent))
4572 page = mc_handle_file_pte(vma, addr, ptent, &ent);
4574 if (!page && !ent.val)
4578 * Do only loose check w/o serialization.
4579 * mem_cgroup_move_account() checks the page is valid or
4580 * not under LRU exclusion.
4582 if (page->mem_cgroup == mc.from) {
4583 ret = MC_TARGET_PAGE;
4585 target->page = page;
4587 if (!ret || !target)
4590 /* There is a swap entry and a page doesn't exist or isn't charged */
4591 if (ent.val && !ret &&
4592 mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
4593 ret = MC_TARGET_SWAP;
4600 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4602 * We don't consider swapping or file mapped pages because THP does not
4603 * support them for now.
4604 * Caller should make sure that pmd_trans_huge(pmd) is true.
4606 static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
4607 unsigned long addr, pmd_t pmd, union mc_target *target)
4609 struct page *page = NULL;
4610 enum mc_target_type ret = MC_TARGET_NONE;
4612 page = pmd_page(pmd);
4613 VM_BUG_ON_PAGE(!page || !PageHead(page), page);
4614 if (!(mc.flags & MOVE_ANON))
4616 if (page->mem_cgroup == mc.from) {
4617 ret = MC_TARGET_PAGE;
4620 target->page = page;
4626 static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
4627 unsigned long addr, pmd_t pmd, union mc_target *target)
4629 return MC_TARGET_NONE;
4633 static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
4634 unsigned long addr, unsigned long end,
4635 struct mm_walk *walk)
4637 struct vm_area_struct *vma = walk->vma;
4641 ptl = pmd_trans_huge_lock(pmd, vma);
4643 if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
4644 mc.precharge += HPAGE_PMD_NR;
4649 if (pmd_trans_unstable(pmd))
4651 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4652 for (; addr != end; pte++, addr += PAGE_SIZE)
4653 if (get_mctgt_type(vma, addr, *pte, NULL))
4654 mc.precharge++; /* increment precharge temporarily */
4655 pte_unmap_unlock(pte - 1, ptl);
4661 static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
4663 unsigned long precharge;
4665 struct mm_walk mem_cgroup_count_precharge_walk = {
4666 .pmd_entry = mem_cgroup_count_precharge_pte_range,
4669 down_read(&mm->mmap_sem);
4670 walk_page_range(0, ~0UL, &mem_cgroup_count_precharge_walk);
4671 up_read(&mm->mmap_sem);
4673 precharge = mc.precharge;
4679 static int mem_cgroup_precharge_mc(struct mm_struct *mm)
4681 unsigned long precharge = mem_cgroup_count_precharge(mm);
4683 VM_BUG_ON(mc.moving_task);
4684 mc.moving_task = current;
4685 return mem_cgroup_do_precharge(precharge);
4688 /* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
4689 static void __mem_cgroup_clear_mc(void)
4691 struct mem_cgroup *from = mc.from;
4692 struct mem_cgroup *to = mc.to;
4694 /* we must uncharge all the leftover precharges from mc.to */
4696 cancel_charge(mc.to, mc.precharge);
4700 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
4701 * we must uncharge here.
4703 if (mc.moved_charge) {
4704 cancel_charge(mc.from, mc.moved_charge);
4705 mc.moved_charge = 0;
4707 /* we must fixup refcnts and charges */
4708 if (mc.moved_swap) {
4709 /* uncharge swap account from the old cgroup */
4710 if (!mem_cgroup_is_root(mc.from))
4711 page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
4714 * we charged both to->memory and to->memsw, so we
4715 * should uncharge to->memory.
4717 if (!mem_cgroup_is_root(mc.to))
4718 page_counter_uncharge(&mc.to->memory, mc.moved_swap);
4720 css_put_many(&mc.from->css, mc.moved_swap);
4722 /* we've already done css_get(mc.to) */
4725 memcg_oom_recover(from);
4726 memcg_oom_recover(to);
4727 wake_up_all(&mc.waitq);
4730 static void mem_cgroup_clear_mc(void)
4732 struct mm_struct *mm = mc.mm;
4735 * we must clear moving_task before waking up waiters at the end of
4738 mc.moving_task = NULL;
4739 __mem_cgroup_clear_mc();
4740 spin_lock(&mc.lock);
4744 spin_unlock(&mc.lock);
4749 static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
4751 struct cgroup_subsys_state *css;
4752 struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
4753 struct mem_cgroup *from;
4754 struct task_struct *leader, *p;
4755 struct mm_struct *mm;
4756 unsigned long move_flags;
4759 /* charge immigration isn't supported on the default hierarchy */
4760 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
4764 * Multi-process migrations only happen on the default hierarchy
4765 * where charge immigration is not used. Perform charge
4766 * immigration if @tset contains a leader and whine if there are
4770 cgroup_taskset_for_each_leader(leader, css, tset) {
4773 memcg = mem_cgroup_from_css(css);
4779 * We are now commited to this value whatever it is. Changes in this
4780 * tunable will only affect upcoming migrations, not the current one.
4781 * So we need to save it, and keep it going.
4783 move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
4787 from = mem_cgroup_from_task(p);
4789 VM_BUG_ON(from == memcg);
4791 mm = get_task_mm(p);
4794 /* We move charges only when we move a owner of the mm */
4795 if (mm->owner == p) {
4798 VM_BUG_ON(mc.precharge);
4799 VM_BUG_ON(mc.moved_charge);
4800 VM_BUG_ON(mc.moved_swap);
4802 spin_lock(&mc.lock);
4806 mc.flags = move_flags;
4807 spin_unlock(&mc.lock);
4808 /* We set mc.moving_task later */
4810 ret = mem_cgroup_precharge_mc(mm);
4812 mem_cgroup_clear_mc();
4819 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
4822 mem_cgroup_clear_mc();
4825 static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
4826 unsigned long addr, unsigned long end,
4827 struct mm_walk *walk)
4830 struct vm_area_struct *vma = walk->vma;
4833 enum mc_target_type target_type;
4834 union mc_target target;
4837 ptl = pmd_trans_huge_lock(pmd, vma);
4839 if (mc.precharge < HPAGE_PMD_NR) {
4843 target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
4844 if (target_type == MC_TARGET_PAGE) {
4846 if (!isolate_lru_page(page)) {
4847 if (!mem_cgroup_move_account(page, true,
4849 mc.precharge -= HPAGE_PMD_NR;
4850 mc.moved_charge += HPAGE_PMD_NR;
4852 putback_lru_page(page);
4860 if (pmd_trans_unstable(pmd))
4863 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4864 for (; addr != end; addr += PAGE_SIZE) {
4865 pte_t ptent = *(pte++);
4871 switch (get_mctgt_type(vma, addr, ptent, &target)) {
4872 case MC_TARGET_PAGE:
4875 * We can have a part of the split pmd here. Moving it
4876 * can be done but it would be too convoluted so simply
4877 * ignore such a partial THP and keep it in original
4878 * memcg. There should be somebody mapping the head.
4880 if (PageTransCompound(page))
4882 if (isolate_lru_page(page))
4884 if (!mem_cgroup_move_account(page, false,
4887 /* we uncharge from mc.from later. */
4890 putback_lru_page(page);
4891 put: /* get_mctgt_type() gets the page */
4894 case MC_TARGET_SWAP:
4896 if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
4898 /* we fixup refcnts and charges later. */
4906 pte_unmap_unlock(pte - 1, ptl);
4911 * We have consumed all precharges we got in can_attach().
4912 * We try charge one by one, but don't do any additional
4913 * charges to mc.to if we have failed in charge once in attach()
4916 ret = mem_cgroup_do_precharge(1);
4924 static void mem_cgroup_move_charge(void)
4926 struct mm_walk mem_cgroup_move_charge_walk = {
4927 .pmd_entry = mem_cgroup_move_charge_pte_range,
4931 lru_add_drain_all();
4933 * Signal lock_page_memcg() to take the memcg's move_lock
4934 * while we're moving its pages to another memcg. Then wait
4935 * for already started RCU-only updates to finish.
4937 atomic_inc(&mc.from->moving_account);
4940 if (unlikely(!down_read_trylock(&mc.mm->mmap_sem))) {
4942 * Someone who are holding the mmap_sem might be waiting in
4943 * waitq. So we cancel all extra charges, wake up all waiters,
4944 * and retry. Because we cancel precharges, we might not be able
4945 * to move enough charges, but moving charge is a best-effort
4946 * feature anyway, so it wouldn't be a big problem.
4948 __mem_cgroup_clear_mc();
4953 * When we have consumed all precharges and failed in doing
4954 * additional charge, the page walk just aborts.
4956 walk_page_range(0, ~0UL, &mem_cgroup_move_charge_walk);
4957 up_read(&mc.mm->mmap_sem);
4958 atomic_dec(&mc.from->moving_account);
4961 static void mem_cgroup_move_task(void)
4964 mem_cgroup_move_charge();
4965 mem_cgroup_clear_mc();
4968 #else /* !CONFIG_MMU */
4969 static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
4973 static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
4976 static void mem_cgroup_move_task(void)
4982 * Cgroup retains root cgroups across [un]mount cycles making it necessary
4983 * to verify whether we're attached to the default hierarchy on each mount
4986 static void mem_cgroup_bind(struct cgroup_subsys_state *root_css)
4989 * use_hierarchy is forced on the default hierarchy. cgroup core
4990 * guarantees that @root doesn't have any children, so turning it
4991 * on for the root memcg is enough.
4993 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
4994 root_mem_cgroup->use_hierarchy = true;
4996 root_mem_cgroup->use_hierarchy = false;
4999 static u64 memory_current_read(struct cgroup_subsys_state *css,
5002 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5004 return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
5007 static int memory_low_show(struct seq_file *m, void *v)
5009 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5010 unsigned long low = READ_ONCE(memcg->low);
5012 if (low == PAGE_COUNTER_MAX)
5013 seq_puts(m, "max\n");
5015 seq_printf(m, "%llu\n", (u64)low * PAGE_SIZE);
5020 static ssize_t memory_low_write(struct kernfs_open_file *of,
5021 char *buf, size_t nbytes, loff_t off)
5023 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5027 buf = strstrip(buf);
5028 err = page_counter_memparse(buf, "max", &low);
5037 static int memory_high_show(struct seq_file *m, void *v)
5039 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5040 unsigned long high = READ_ONCE(memcg->high);
5042 if (high == PAGE_COUNTER_MAX)
5043 seq_puts(m, "max\n");
5045 seq_printf(m, "%llu\n", (u64)high * PAGE_SIZE);
5050 static ssize_t memory_high_write(struct kernfs_open_file *of,
5051 char *buf, size_t nbytes, loff_t off)
5053 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5054 unsigned long nr_pages;
5058 buf = strstrip(buf);
5059 err = page_counter_memparse(buf, "max", &high);
5065 nr_pages = page_counter_read(&memcg->memory);
5066 if (nr_pages > high)
5067 try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
5070 memcg_wb_domain_size_changed(memcg);
5074 static int memory_max_show(struct seq_file *m, void *v)
5076 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5077 unsigned long max = READ_ONCE(memcg->memory.limit);
5079 if (max == PAGE_COUNTER_MAX)
5080 seq_puts(m, "max\n");
5082 seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);
5087 static ssize_t memory_max_write(struct kernfs_open_file *of,
5088 char *buf, size_t nbytes, loff_t off)
5090 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5091 unsigned int nr_reclaims = MEM_CGROUP_RECLAIM_RETRIES;
5092 bool drained = false;
5096 buf = strstrip(buf);
5097 err = page_counter_memparse(buf, "max", &max);
5101 xchg(&memcg->memory.limit, max);
5104 unsigned long nr_pages = page_counter_read(&memcg->memory);
5106 if (nr_pages <= max)
5109 if (signal_pending(current)) {
5115 drain_all_stock(memcg);
5121 if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
5127 mem_cgroup_events(memcg, MEMCG_OOM, 1);
5128 if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
5132 memcg_wb_domain_size_changed(memcg);
5136 static int memory_events_show(struct seq_file *m, void *v)
5138 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5140 seq_printf(m, "low %lu\n", mem_cgroup_read_events(memcg, MEMCG_LOW));
5141 seq_printf(m, "high %lu\n", mem_cgroup_read_events(memcg, MEMCG_HIGH));
5142 seq_printf(m, "max %lu\n", mem_cgroup_read_events(memcg, MEMCG_MAX));
5143 seq_printf(m, "oom %lu\n", mem_cgroup_read_events(memcg, MEMCG_OOM));
5148 static int memory_stat_show(struct seq_file *m, void *v)
5150 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5151 unsigned long stat[MEMCG_NR_STAT];
5152 unsigned long events[MEMCG_NR_EVENTS];
5156 * Provide statistics on the state of the memory subsystem as
5157 * well as cumulative event counters that show past behavior.
5159 * This list is ordered following a combination of these gradients:
5160 * 1) generic big picture -> specifics and details
5161 * 2) reflecting userspace activity -> reflecting kernel heuristics
5163 * Current memory state:
5166 tree_stat(memcg, stat);
5167 tree_events(memcg, events);
5169 seq_printf(m, "anon %llu\n",
5170 (u64)stat[MEM_CGROUP_STAT_RSS] * PAGE_SIZE);
5171 seq_printf(m, "file %llu\n",
5172 (u64)stat[MEM_CGROUP_STAT_CACHE] * PAGE_SIZE);
5173 seq_printf(m, "kernel_stack %llu\n",
5174 (u64)stat[MEMCG_KERNEL_STACK] * PAGE_SIZE);
5175 seq_printf(m, "slab %llu\n",
5176 (u64)(stat[MEMCG_SLAB_RECLAIMABLE] +
5177 stat[MEMCG_SLAB_UNRECLAIMABLE]) * PAGE_SIZE);
5178 seq_printf(m, "sock %llu\n",
5179 (u64)stat[MEMCG_SOCK] * PAGE_SIZE);
5181 seq_printf(m, "file_mapped %llu\n",
5182 (u64)stat[MEM_CGROUP_STAT_FILE_MAPPED] * PAGE_SIZE);
5183 seq_printf(m, "file_dirty %llu\n",
5184 (u64)stat[MEM_CGROUP_STAT_DIRTY] * PAGE_SIZE);
5185 seq_printf(m, "file_writeback %llu\n",
5186 (u64)stat[MEM_CGROUP_STAT_WRITEBACK] * PAGE_SIZE);
5188 for (i = 0; i < NR_LRU_LISTS; i++) {
5189 struct mem_cgroup *mi;
5190 unsigned long val = 0;
5192 for_each_mem_cgroup_tree(mi, memcg)
5193 val += mem_cgroup_nr_lru_pages(mi, BIT(i));
5194 seq_printf(m, "%s %llu\n",
5195 mem_cgroup_lru_names[i], (u64)val * PAGE_SIZE);
5198 seq_printf(m, "slab_reclaimable %llu\n",
5199 (u64)stat[MEMCG_SLAB_RECLAIMABLE] * PAGE_SIZE);
5200 seq_printf(m, "slab_unreclaimable %llu\n",
5201 (u64)stat[MEMCG_SLAB_UNRECLAIMABLE] * PAGE_SIZE);
5203 /* Accumulated memory events */
5205 seq_printf(m, "pgfault %lu\n",
5206 events[MEM_CGROUP_EVENTS_PGFAULT]);
5207 seq_printf(m, "pgmajfault %lu\n",
5208 events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
5213 static struct cftype memory_files[] = {
5216 .flags = CFTYPE_NOT_ON_ROOT,
5217 .read_u64 = memory_current_read,
5221 .flags = CFTYPE_NOT_ON_ROOT,
5222 .seq_show = memory_low_show,
5223 .write = memory_low_write,
5227 .flags = CFTYPE_NOT_ON_ROOT,
5228 .seq_show = memory_high_show,
5229 .write = memory_high_write,
5233 .flags = CFTYPE_NOT_ON_ROOT,
5234 .seq_show = memory_max_show,
5235 .write = memory_max_write,
5239 .flags = CFTYPE_NOT_ON_ROOT,
5240 .file_offset = offsetof(struct mem_cgroup, events_file),
5241 .seq_show = memory_events_show,
5245 .flags = CFTYPE_NOT_ON_ROOT,
5246 .seq_show = memory_stat_show,
5251 struct cgroup_subsys memory_cgrp_subsys = {
5252 .css_alloc = mem_cgroup_css_alloc,
5253 .css_online = mem_cgroup_css_online,
5254 .css_offline = mem_cgroup_css_offline,
5255 .css_released = mem_cgroup_css_released,
5256 .css_free = mem_cgroup_css_free,
5257 .css_reset = mem_cgroup_css_reset,
5258 .can_attach = mem_cgroup_can_attach,
5259 .cancel_attach = mem_cgroup_cancel_attach,
5260 .post_attach = mem_cgroup_move_task,
5261 .bind = mem_cgroup_bind,
5262 .dfl_cftypes = memory_files,
5263 .legacy_cftypes = mem_cgroup_legacy_files,
5268 * mem_cgroup_low - check if memory consumption is below the normal range
5269 * @root: the highest ancestor to consider
5270 * @memcg: the memory cgroup to check
5272 * Returns %true if memory consumption of @memcg, and that of all
5273 * configurable ancestors up to @root, is below the normal range.
5275 bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg)
5277 if (mem_cgroup_disabled())
5281 * The toplevel group doesn't have a configurable range, so
5282 * it's never low when looked at directly, and it is not
5283 * considered an ancestor when assessing the hierarchy.
5286 if (memcg == root_mem_cgroup)
5289 if (page_counter_read(&memcg->memory) >= memcg->low)
5292 while (memcg != root) {
5293 memcg = parent_mem_cgroup(memcg);
5295 if (memcg == root_mem_cgroup)
5298 if (page_counter_read(&memcg->memory) >= memcg->low)
5305 * mem_cgroup_try_charge - try charging a page
5306 * @page: page to charge
5307 * @mm: mm context of the victim
5308 * @gfp_mask: reclaim mode
5309 * @memcgp: charged memcg return
5310 * @compound: charge the page as compound or small page
5312 * Try to charge @page to the memcg that @mm belongs to, reclaiming
5313 * pages according to @gfp_mask if necessary.
5315 * Returns 0 on success, with *@memcgp pointing to the charged memcg.
5316 * Otherwise, an error code is returned.
5318 * After page->mapping has been set up, the caller must finalize the
5319 * charge with mem_cgroup_commit_charge(). Or abort the transaction
5320 * with mem_cgroup_cancel_charge() in case page instantiation fails.
5322 int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
5323 gfp_t gfp_mask, struct mem_cgroup **memcgp,
5326 struct mem_cgroup *memcg = NULL;
5327 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5330 if (mem_cgroup_disabled())
5333 if (PageSwapCache(page)) {
5335 * Every swap fault against a single page tries to charge the
5336 * page, bail as early as possible. shmem_unuse() encounters
5337 * already charged pages, too. The USED bit is protected by
5338 * the page lock, which serializes swap cache removal, which
5339 * in turn serializes uncharging.
5341 VM_BUG_ON_PAGE(!PageLocked(page), page);
5342 if (page->mem_cgroup)
5345 if (do_swap_account) {
5346 swp_entry_t ent = { .val = page_private(page), };
5347 unsigned short id = lookup_swap_cgroup_id(ent);
5350 memcg = mem_cgroup_from_id(id);
5351 if (memcg && !css_tryget_online(&memcg->css))
5358 memcg = get_mem_cgroup_from_mm(mm);
5360 ret = try_charge(memcg, gfp_mask, nr_pages);
5362 css_put(&memcg->css);
5369 * mem_cgroup_commit_charge - commit a page charge
5370 * @page: page to charge
5371 * @memcg: memcg to charge the page to
5372 * @lrucare: page might be on LRU already
5373 * @compound: charge the page as compound or small page
5375 * Finalize a charge transaction started by mem_cgroup_try_charge(),
5376 * after page->mapping has been set up. This must happen atomically
5377 * as part of the page instantiation, i.e. under the page table lock
5378 * for anonymous pages, under the page lock for page and swap cache.
5380 * In addition, the page must not be on the LRU during the commit, to
5381 * prevent racing with task migration. If it might be, use @lrucare.
5383 * Use mem_cgroup_cancel_charge() to cancel the transaction instead.
5385 void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
5386 bool lrucare, bool compound)
5388 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5390 VM_BUG_ON_PAGE(!page->mapping, page);
5391 VM_BUG_ON_PAGE(PageLRU(page) && !lrucare, page);
5393 if (mem_cgroup_disabled())
5396 * Swap faults will attempt to charge the same page multiple
5397 * times. But reuse_swap_page() might have removed the page
5398 * from swapcache already, so we can't check PageSwapCache().
5403 commit_charge(page, memcg, lrucare);
5405 local_irq_disable();
5406 mem_cgroup_charge_statistics(memcg, page, compound, nr_pages);
5407 memcg_check_events(memcg, page);
5410 if (do_memsw_account() && PageSwapCache(page)) {
5411 swp_entry_t entry = { .val = page_private(page) };
5413 * The swap entry might not get freed for a long time,
5414 * let's not wait for it. The page already received a
5415 * memory+swap charge, drop the swap entry duplicate.
5417 mem_cgroup_uncharge_swap(entry);
5422 * mem_cgroup_cancel_charge - cancel a page charge
5423 * @page: page to charge
5424 * @memcg: memcg to charge the page to
5425 * @compound: charge the page as compound or small page
5427 * Cancel a charge transaction started by mem_cgroup_try_charge().
5429 void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
5432 unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1;
5434 if (mem_cgroup_disabled())
5437 * Swap faults will attempt to charge the same page multiple
5438 * times. But reuse_swap_page() might have removed the page
5439 * from swapcache already, so we can't check PageSwapCache().
5444 cancel_charge(memcg, nr_pages);
5447 static void uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout,
5448 unsigned long nr_anon, unsigned long nr_file,
5449 unsigned long nr_huge, unsigned long nr_kmem,
5450 struct page *dummy_page)
5452 unsigned long nr_pages = nr_anon + nr_file + nr_kmem;
5453 unsigned long flags;
5455 if (!mem_cgroup_is_root(memcg)) {
5456 page_counter_uncharge(&memcg->memory, nr_pages);
5457 if (do_memsw_account())
5458 page_counter_uncharge(&memcg->memsw, nr_pages);
5459 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && nr_kmem)
5460 page_counter_uncharge(&memcg->kmem, nr_kmem);
5461 memcg_oom_recover(memcg);
5464 local_irq_save(flags);
5465 __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS], nr_anon);
5466 __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_CACHE], nr_file);
5467 __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], nr_huge);
5468 __this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT], pgpgout);
5469 __this_cpu_add(memcg->stat->nr_page_events, nr_pages);
5470 memcg_check_events(memcg, dummy_page);
5471 local_irq_restore(flags);
5473 if (!mem_cgroup_is_root(memcg))
5474 css_put_many(&memcg->css, nr_pages);
5477 static void uncharge_list(struct list_head *page_list)
5479 struct mem_cgroup *memcg = NULL;
5480 unsigned long nr_anon = 0;
5481 unsigned long nr_file = 0;
5482 unsigned long nr_huge = 0;
5483 unsigned long nr_kmem = 0;
5484 unsigned long pgpgout = 0;
5485 struct list_head *next;
5489 * Note that the list can be a single page->lru; hence the
5490 * do-while loop instead of a simple list_for_each_entry().
5492 next = page_list->next;
5494 page = list_entry(next, struct page, lru);
5495 next = page->lru.next;
5497 VM_BUG_ON_PAGE(PageLRU(page), page);
5498 VM_BUG_ON_PAGE(page_count(page), page);
5500 if (!page->mem_cgroup)
5504 * Nobody should be changing or seriously looking at
5505 * page->mem_cgroup at this point, we have fully
5506 * exclusive access to the page.
5509 if (memcg != page->mem_cgroup) {
5511 uncharge_batch(memcg, pgpgout, nr_anon, nr_file,
5512 nr_huge, nr_kmem, page);
5513 pgpgout = nr_anon = nr_file =
5514 nr_huge = nr_kmem = 0;
5516 memcg = page->mem_cgroup;
5519 if (!PageKmemcg(page)) {
5520 unsigned int nr_pages = 1;
5522 if (PageTransHuge(page)) {
5523 nr_pages <<= compound_order(page);
5524 nr_huge += nr_pages;
5527 nr_anon += nr_pages;
5529 nr_file += nr_pages;
5532 nr_kmem += 1 << compound_order(page);
5534 page->mem_cgroup = NULL;
5535 } while (next != page_list);
5538 uncharge_batch(memcg, pgpgout, nr_anon, nr_file,
5539 nr_huge, nr_kmem, page);
5543 * mem_cgroup_uncharge - uncharge a page
5544 * @page: page to uncharge
5546 * Uncharge a page previously charged with mem_cgroup_try_charge() and
5547 * mem_cgroup_commit_charge().
5549 void mem_cgroup_uncharge(struct page *page)
5551 if (mem_cgroup_disabled())
5554 /* Don't touch page->lru of any random page, pre-check: */
5555 if (!page->mem_cgroup)
5558 INIT_LIST_HEAD(&page->lru);
5559 uncharge_list(&page->lru);
5563 * mem_cgroup_uncharge_list - uncharge a list of page
5564 * @page_list: list of pages to uncharge
5566 * Uncharge a list of pages previously charged with
5567 * mem_cgroup_try_charge() and mem_cgroup_commit_charge().
5569 void mem_cgroup_uncharge_list(struct list_head *page_list)
5571 if (mem_cgroup_disabled())
5574 if (!list_empty(page_list))
5575 uncharge_list(page_list);
5579 * mem_cgroup_migrate - charge a page's replacement
5580 * @oldpage: currently circulating page
5581 * @newpage: replacement page
5583 * Charge @newpage as a replacement page for @oldpage. @oldpage will
5584 * be uncharged upon free.
5586 * Both pages must be locked, @newpage->mapping must be set up.
5588 void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
5590 struct mem_cgroup *memcg;
5591 unsigned int nr_pages;
5593 unsigned long flags;
5595 VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
5596 VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
5597 VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage);
5598 VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage),
5601 if (mem_cgroup_disabled())
5604 /* Page cache replacement: new page already charged? */
5605 if (newpage->mem_cgroup)
5608 /* Swapcache readahead pages can get replaced before being charged */
5609 memcg = oldpage->mem_cgroup;
5613 /* Force-charge the new page. The old one will be freed soon */
5614 compound = PageTransHuge(newpage);
5615 nr_pages = compound ? hpage_nr_pages(newpage) : 1;
5617 page_counter_charge(&memcg->memory, nr_pages);
5618 if (do_memsw_account())
5619 page_counter_charge(&memcg->memsw, nr_pages);
5620 css_get_many(&memcg->css, nr_pages);
5622 commit_charge(newpage, memcg, false);
5624 local_irq_save(flags);
5625 mem_cgroup_charge_statistics(memcg, newpage, compound, nr_pages);
5626 memcg_check_events(memcg, newpage);
5627 local_irq_restore(flags);
5630 DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
5631 EXPORT_SYMBOL(memcg_sockets_enabled_key);
5633 void sock_update_memcg(struct sock *sk)
5635 struct mem_cgroup *memcg;
5637 /* Socket cloning can throw us here with sk_cgrp already
5638 * filled. It won't however, necessarily happen from
5639 * process context. So the test for root memcg given
5640 * the current task's memcg won't help us in this case.
5642 * Respecting the original socket's memcg is a better
5643 * decision in this case.
5646 BUG_ON(mem_cgroup_is_root(sk->sk_memcg));
5647 css_get(&sk->sk_memcg->css);
5652 memcg = mem_cgroup_from_task(current);
5653 if (memcg == root_mem_cgroup)
5655 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
5657 if (css_tryget_online(&memcg->css))
5658 sk->sk_memcg = memcg;
5662 EXPORT_SYMBOL(sock_update_memcg);
5664 void sock_release_memcg(struct sock *sk)
5666 WARN_ON(!sk->sk_memcg);
5667 css_put(&sk->sk_memcg->css);
5671 * mem_cgroup_charge_skmem - charge socket memory
5672 * @memcg: memcg to charge
5673 * @nr_pages: number of pages to charge
5675 * Charges @nr_pages to @memcg. Returns %true if the charge fit within
5676 * @memcg's configured limit, %false if the charge had to be forced.
5678 bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
5680 gfp_t gfp_mask = GFP_KERNEL;
5682 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
5683 struct page_counter *fail;
5685 if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
5686 memcg->tcpmem_pressure = 0;
5689 page_counter_charge(&memcg->tcpmem, nr_pages);
5690 memcg->tcpmem_pressure = 1;
5694 /* Don't block in the packet receive path */
5696 gfp_mask = GFP_NOWAIT;
5698 this_cpu_add(memcg->stat->count[MEMCG_SOCK], nr_pages);
5700 if (try_charge(memcg, gfp_mask, nr_pages) == 0)
5703 try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages);
5708 * mem_cgroup_uncharge_skmem - uncharge socket memory
5709 * @memcg - memcg to uncharge
5710 * @nr_pages - number of pages to uncharge
5712 void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
5714 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
5715 page_counter_uncharge(&memcg->tcpmem, nr_pages);
5719 this_cpu_sub(memcg->stat->count[MEMCG_SOCK], nr_pages);
5721 page_counter_uncharge(&memcg->memory, nr_pages);
5722 css_put_many(&memcg->css, nr_pages);
5725 static int __init cgroup_memory(char *s)
5729 while ((token = strsep(&s, ",")) != NULL) {
5732 if (!strcmp(token, "nosocket"))
5733 cgroup_memory_nosocket = true;
5734 if (!strcmp(token, "nokmem"))
5735 cgroup_memory_nokmem = true;
5739 __setup("cgroup.memory=", cgroup_memory);
5742 * subsys_initcall() for memory controller.
5744 * Some parts like hotcpu_notifier() have to be initialized from this context
5745 * because of lock dependencies (cgroup_lock -> cpu hotplug) but basically
5746 * everything that doesn't depend on a specific mem_cgroup structure should
5747 * be initialized from here.
5749 static int __init mem_cgroup_init(void)
5753 hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
5755 for_each_possible_cpu(cpu)
5756 INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
5759 for_each_node(node) {
5760 struct mem_cgroup_tree_per_node *rtpn;
5762 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL,
5763 node_online(node) ? node : NUMA_NO_NODE);
5765 rtpn->rb_root = RB_ROOT;
5766 spin_lock_init(&rtpn->lock);
5767 soft_limit_tree.rb_tree_per_node[node] = rtpn;
5772 subsys_initcall(mem_cgroup_init);
5774 #ifdef CONFIG_MEMCG_SWAP
5776 * mem_cgroup_swapout - transfer a memsw charge to swap
5777 * @page: page whose memsw charge to transfer
5778 * @entry: swap entry to move the charge to
5780 * Transfer the memsw charge of @page to @entry.
5782 void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
5784 struct mem_cgroup *memcg;
5785 unsigned short oldid;
5787 VM_BUG_ON_PAGE(PageLRU(page), page);
5788 VM_BUG_ON_PAGE(page_count(page), page);
5790 if (!do_memsw_account())
5793 memcg = page->mem_cgroup;
5795 /* Readahead page, never charged */
5799 mem_cgroup_id_get(memcg);
5800 oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg));
5801 VM_BUG_ON_PAGE(oldid, page);
5802 mem_cgroup_swap_statistics(memcg, true);
5804 page->mem_cgroup = NULL;
5806 if (!mem_cgroup_is_root(memcg))
5807 page_counter_uncharge(&memcg->memory, 1);
5810 * Interrupts should be disabled here because the caller holds the
5811 * mapping->tree_lock lock which is taken with interrupts-off. It is
5812 * important here to have the interrupts disabled because it is the
5813 * only synchronisation we have for udpating the per-CPU variables.
5815 VM_BUG_ON(!irqs_disabled());
5816 mem_cgroup_charge_statistics(memcg, page, false, -1);
5817 memcg_check_events(memcg, page);
5819 if (!mem_cgroup_is_root(memcg))
5820 css_put(&memcg->css);
5824 * mem_cgroup_try_charge_swap - try charging a swap entry
5825 * @page: page being added to swap
5826 * @entry: swap entry to charge
5828 * Try to charge @entry to the memcg that @page belongs to.
5830 * Returns 0 on success, -ENOMEM on failure.
5832 int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
5834 struct mem_cgroup *memcg;
5835 struct page_counter *counter;
5836 unsigned short oldid;
5838 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) || !do_swap_account)
5841 memcg = page->mem_cgroup;
5843 /* Readahead page, never charged */
5847 if (!mem_cgroup_is_root(memcg) &&
5848 !page_counter_try_charge(&memcg->swap, 1, &counter))
5851 mem_cgroup_id_get(memcg);
5852 oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg));
5853 VM_BUG_ON_PAGE(oldid, page);
5854 mem_cgroup_swap_statistics(memcg, true);
5860 * mem_cgroup_uncharge_swap - uncharge a swap entry
5861 * @entry: swap entry to uncharge
5863 * Drop the swap charge associated with @entry.
5865 void mem_cgroup_uncharge_swap(swp_entry_t entry)
5867 struct mem_cgroup *memcg;
5870 if (!do_swap_account)
5873 id = swap_cgroup_record(entry, 0);
5875 memcg = mem_cgroup_from_id(id);
5877 if (!mem_cgroup_is_root(memcg)) {
5878 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
5879 page_counter_uncharge(&memcg->swap, 1);
5881 page_counter_uncharge(&memcg->memsw, 1);
5883 mem_cgroup_swap_statistics(memcg, false);
5884 mem_cgroup_id_put(memcg);
5889 long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
5891 long nr_swap_pages = get_nr_swap_pages();
5893 if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
5894 return nr_swap_pages;
5895 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
5896 nr_swap_pages = min_t(long, nr_swap_pages,
5897 READ_ONCE(memcg->swap.limit) -
5898 page_counter_read(&memcg->swap));
5899 return nr_swap_pages;
5902 bool mem_cgroup_swap_full(struct page *page)
5904 struct mem_cgroup *memcg;
5906 VM_BUG_ON_PAGE(!PageLocked(page), page);
5910 if (!do_swap_account || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
5913 memcg = page->mem_cgroup;
5917 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
5918 if (page_counter_read(&memcg->swap) * 2 >= memcg->swap.limit)
5924 /* for remember boot option*/
5925 #ifdef CONFIG_MEMCG_SWAP_ENABLED
5926 static int really_do_swap_account __initdata = 1;
5928 static int really_do_swap_account __initdata;
5931 static int __init enable_swap_account(char *s)
5933 if (!strcmp(s, "1"))
5934 really_do_swap_account = 1;
5935 else if (!strcmp(s, "0"))
5936 really_do_swap_account = 0;
5939 __setup("swapaccount=", enable_swap_account);
5941 static u64 swap_current_read(struct cgroup_subsys_state *css,
5944 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5946 return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
5949 static int swap_max_show(struct seq_file *m, void *v)
5951 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m));
5952 unsigned long max = READ_ONCE(memcg->swap.limit);
5954 if (max == PAGE_COUNTER_MAX)
5955 seq_puts(m, "max\n");
5957 seq_printf(m, "%llu\n", (u64)max * PAGE_SIZE);
5962 static ssize_t swap_max_write(struct kernfs_open_file *of,
5963 char *buf, size_t nbytes, loff_t off)
5965 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
5969 buf = strstrip(buf);
5970 err = page_counter_memparse(buf, "max", &max);
5974 mutex_lock(&memcg_limit_mutex);
5975 err = page_counter_limit(&memcg->swap, max);
5976 mutex_unlock(&memcg_limit_mutex);
5983 static struct cftype swap_files[] = {
5985 .name = "swap.current",
5986 .flags = CFTYPE_NOT_ON_ROOT,
5987 .read_u64 = swap_current_read,
5991 .flags = CFTYPE_NOT_ON_ROOT,
5992 .seq_show = swap_max_show,
5993 .write = swap_max_write,
5998 static struct cftype memsw_cgroup_files[] = {
6000 .name = "memsw.usage_in_bytes",
6001 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
6002 .read_u64 = mem_cgroup_read_u64,
6005 .name = "memsw.max_usage_in_bytes",
6006 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
6007 .write = mem_cgroup_reset,
6008 .read_u64 = mem_cgroup_read_u64,
6011 .name = "memsw.limit_in_bytes",
6012 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
6013 .write = mem_cgroup_write,
6014 .read_u64 = mem_cgroup_read_u64,
6017 .name = "memsw.failcnt",
6018 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
6019 .write = mem_cgroup_reset,
6020 .read_u64 = mem_cgroup_read_u64,
6022 { }, /* terminate */
6025 static int __init mem_cgroup_swap_init(void)
6027 if (!mem_cgroup_disabled() && really_do_swap_account) {
6028 do_swap_account = 1;
6029 WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys,
6031 WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys,
6032 memsw_cgroup_files));
6036 subsys_initcall(mem_cgroup_swap_init);
6038 #endif /* CONFIG_MEMCG_SWAP */