4 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
8 * This file contains the default values for the operation of the
9 * Linux VM subsystem. Fine-tuning documentation can be found in
10 * Documentation/sysctl/vm.txt.
12 * Swap aging added 23.2.95, Stephen Tweedie.
13 * Buffermem limits added 12.3.98, Rik van Riel.
17 #include <linux/sched.h>
18 #include <linux/kernel_stat.h>
19 #include <linux/swap.h>
20 #include <linux/mman.h>
21 #include <linux/pagemap.h>
22 #include <linux/pagevec.h>
23 #include <linux/init.h>
24 #include <linux/export.h>
25 #include <linux/mm_inline.h>
26 #include <linux/percpu_counter.h>
27 #include <linux/percpu.h>
28 #include <linux/cpu.h>
29 #include <linux/notifier.h>
30 #include <linux/backing-dev.h>
31 #include <linux/memcontrol.h>
32 #include <linux/gfp.h>
33 #include <linux/uio.h>
34 #include <linux/hugetlb.h>
35 #include <linux/page_idle.h>
39 #define CREATE_TRACE_POINTS
40 #include <trace/events/pagemap.h>
42 /* How many pages do we try to swap or page in/out together? */
45 static DEFINE_PER_CPU(struct pagevec, lru_add_pvec);
46 static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
47 static DEFINE_PER_CPU(struct pagevec, lru_deactivate_file_pvecs);
48 static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs);
51 * This path almost never happens for VM activity - pages are normally
52 * freed via pagevecs. But it gets used by networking.
54 static void __page_cache_release(struct page *page)
57 struct zone *zone = page_zone(page);
58 struct lruvec *lruvec;
61 spin_lock_irqsave(&zone->lru_lock, flags);
62 lruvec = mem_cgroup_page_lruvec(page, zone);
63 VM_BUG_ON_PAGE(!PageLRU(page), page);
65 del_page_from_lru_list(page, lruvec, page_off_lru(page));
66 spin_unlock_irqrestore(&zone->lru_lock, flags);
68 mem_cgroup_uncharge(page);
71 static void __put_single_page(struct page *page)
73 __page_cache_release(page);
74 free_hot_cold_page(page, false);
77 static void __put_compound_page(struct page *page)
79 compound_page_dtor *dtor;
82 * __page_cache_release() is supposed to be called for thp, not for
83 * hugetlb. This is because hugetlb page does never have PageLRU set
84 * (it's never listed to any LRU lists) and no memcg routines should
85 * be called for hugetlb (it has a separate hugetlb_cgroup.)
88 __page_cache_release(page);
89 dtor = get_compound_page_dtor(page);
93 void __put_page(struct page *page)
95 if (unlikely(PageCompound(page)))
96 __put_compound_page(page);
98 __put_single_page(page);
100 EXPORT_SYMBOL(__put_page);
103 * put_pages_list() - release a list of pages
104 * @pages: list of pages threaded on page->lru
106 * Release a list of pages which are strung together on page.lru. Currently
107 * used by read_cache_pages() and related error recovery code.
109 void put_pages_list(struct list_head *pages)
111 while (!list_empty(pages)) {
114 victim = list_entry(pages->prev, struct page, lru);
115 list_del(&victim->lru);
116 page_cache_release(victim);
119 EXPORT_SYMBOL(put_pages_list);
122 * get_kernel_pages() - pin kernel pages in memory
123 * @kiov: An array of struct kvec structures
124 * @nr_segs: number of segments to pin
125 * @write: pinning for read/write, currently ignored
126 * @pages: array that receives pointers to the pages pinned.
127 * Should be at least nr_segs long.
129 * Returns number of pages pinned. This may be fewer than the number
130 * requested. If nr_pages is 0 or negative, returns 0. If no pages
131 * were pinned, returns -errno. Each page returned must be released
132 * with a put_page() call when it is finished with.
134 int get_kernel_pages(const struct kvec *kiov, int nr_segs, int write,
139 for (seg = 0; seg < nr_segs; seg++) {
140 if (WARN_ON(kiov[seg].iov_len != PAGE_SIZE))
143 pages[seg] = kmap_to_page(kiov[seg].iov_base);
144 page_cache_get(pages[seg]);
149 EXPORT_SYMBOL_GPL(get_kernel_pages);
152 * get_kernel_page() - pin a kernel page in memory
153 * @start: starting kernel address
154 * @write: pinning for read/write, currently ignored
155 * @pages: array that receives pointer to the page pinned.
156 * Must be at least nr_segs long.
158 * Returns 1 if page is pinned. If the page was not pinned, returns
159 * -errno. The page returned must be released with a put_page() call
160 * when it is finished with.
162 int get_kernel_page(unsigned long start, int write, struct page **pages)
164 const struct kvec kiov = {
165 .iov_base = (void *)start,
169 return get_kernel_pages(&kiov, 1, write, pages);
171 EXPORT_SYMBOL_GPL(get_kernel_page);
173 static void pagevec_lru_move_fn(struct pagevec *pvec,
174 void (*move_fn)(struct page *page, struct lruvec *lruvec, void *arg),
178 struct zone *zone = NULL;
179 struct lruvec *lruvec;
180 unsigned long flags = 0;
182 for (i = 0; i < pagevec_count(pvec); i++) {
183 struct page *page = pvec->pages[i];
184 struct zone *pagezone = page_zone(page);
186 if (pagezone != zone) {
188 spin_unlock_irqrestore(&zone->lru_lock, flags);
190 spin_lock_irqsave(&zone->lru_lock, flags);
193 lruvec = mem_cgroup_page_lruvec(page, zone);
194 (*move_fn)(page, lruvec, arg);
197 spin_unlock_irqrestore(&zone->lru_lock, flags);
198 release_pages(pvec->pages, pvec->nr, pvec->cold);
199 pagevec_reinit(pvec);
202 static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec,
207 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
208 enum lru_list lru = page_lru_base_type(page);
209 list_move_tail(&page->lru, &lruvec->lists[lru]);
215 * pagevec_move_tail() must be called with IRQ disabled.
216 * Otherwise this may cause nasty races.
218 static void pagevec_move_tail(struct pagevec *pvec)
222 pagevec_lru_move_fn(pvec, pagevec_move_tail_fn, &pgmoved);
223 __count_vm_events(PGROTATED, pgmoved);
227 * Writeback is about to end against a page which has been marked for immediate
228 * reclaim. If it still appears to be reclaimable, move it to the tail of the
231 void rotate_reclaimable_page(struct page *page)
233 if (!PageLocked(page) && !PageDirty(page) && !PageActive(page) &&
234 !PageUnevictable(page) && PageLRU(page)) {
235 struct pagevec *pvec;
238 page_cache_get(page);
239 local_irq_save(flags);
240 pvec = this_cpu_ptr(&lru_rotate_pvecs);
241 if (!pagevec_add(pvec, page))
242 pagevec_move_tail(pvec);
243 local_irq_restore(flags);
247 static void update_page_reclaim_stat(struct lruvec *lruvec,
248 int file, int rotated)
250 struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
252 reclaim_stat->recent_scanned[file]++;
254 reclaim_stat->recent_rotated[file]++;
257 static void __activate_page(struct page *page, struct lruvec *lruvec,
260 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
261 int file = page_is_file_cache(page);
262 int lru = page_lru_base_type(page);
264 del_page_from_lru_list(page, lruvec, lru);
267 add_page_to_lru_list(page, lruvec, lru);
268 trace_mm_lru_activate(page);
270 __count_vm_event(PGACTIVATE);
271 update_page_reclaim_stat(lruvec, file, 1);
276 static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs);
278 static void activate_page_drain(int cpu)
280 struct pagevec *pvec = &per_cpu(activate_page_pvecs, cpu);
282 if (pagevec_count(pvec))
283 pagevec_lru_move_fn(pvec, __activate_page, NULL);
286 static bool need_activate_page_drain(int cpu)
288 return pagevec_count(&per_cpu(activate_page_pvecs, cpu)) != 0;
291 void activate_page(struct page *page)
293 if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
294 struct pagevec *pvec = &get_cpu_var(activate_page_pvecs);
296 page_cache_get(page);
297 if (!pagevec_add(pvec, page))
298 pagevec_lru_move_fn(pvec, __activate_page, NULL);
299 put_cpu_var(activate_page_pvecs);
304 static inline void activate_page_drain(int cpu)
308 static bool need_activate_page_drain(int cpu)
313 void activate_page(struct page *page)
315 struct zone *zone = page_zone(page);
317 spin_lock_irq(&zone->lru_lock);
318 __activate_page(page, mem_cgroup_page_lruvec(page, zone), NULL);
319 spin_unlock_irq(&zone->lru_lock);
323 static void __lru_cache_activate_page(struct page *page)
325 struct pagevec *pvec = &get_cpu_var(lru_add_pvec);
329 * Search backwards on the optimistic assumption that the page being
330 * activated has just been added to this pagevec. Note that only
331 * the local pagevec is examined as a !PageLRU page could be in the
332 * process of being released, reclaimed, migrated or on a remote
333 * pagevec that is currently being drained. Furthermore, marking
334 * a remote pagevec's page PageActive potentially hits a race where
335 * a page is marked PageActive just after it is added to the inactive
336 * list causing accounting errors and BUG_ON checks to trigger.
338 for (i = pagevec_count(pvec) - 1; i >= 0; i--) {
339 struct page *pagevec_page = pvec->pages[i];
341 if (pagevec_page == page) {
347 put_cpu_var(lru_add_pvec);
351 * Mark a page as having seen activity.
353 * inactive,unreferenced -> inactive,referenced
354 * inactive,referenced -> active,unreferenced
355 * active,unreferenced -> active,referenced
357 * When a newly allocated page is not yet visible, so safe for non-atomic ops,
358 * __SetPageReferenced(page) may be substituted for mark_page_accessed(page).
360 void mark_page_accessed(struct page *page)
362 page = compound_head(page);
363 if (!PageActive(page) && !PageUnevictable(page) &&
364 PageReferenced(page)) {
367 * If the page is on the LRU, queue it for activation via
368 * activate_page_pvecs. Otherwise, assume the page is on a
369 * pagevec, mark it active and it'll be moved to the active
370 * LRU on the next drain.
375 __lru_cache_activate_page(page);
376 ClearPageReferenced(page);
377 if (page_is_file_cache(page))
378 workingset_activation(page);
379 } else if (!PageReferenced(page)) {
380 SetPageReferenced(page);
382 if (page_is_idle(page))
383 clear_page_idle(page);
385 EXPORT_SYMBOL(mark_page_accessed);
387 static void __lru_cache_add(struct page *page)
389 struct pagevec *pvec = &get_cpu_var(lru_add_pvec);
391 page_cache_get(page);
392 if (!pagevec_space(pvec))
393 __pagevec_lru_add(pvec);
394 pagevec_add(pvec, page);
395 put_cpu_var(lru_add_pvec);
399 * lru_cache_add: add a page to the page lists
400 * @page: the page to add
402 void lru_cache_add_anon(struct page *page)
404 if (PageActive(page))
405 ClearPageActive(page);
406 __lru_cache_add(page);
409 void lru_cache_add_file(struct page *page)
411 if (PageActive(page))
412 ClearPageActive(page);
413 __lru_cache_add(page);
415 EXPORT_SYMBOL(lru_cache_add_file);
418 * lru_cache_add - add a page to a page list
419 * @page: the page to be added to the LRU.
421 * Queue the page for addition to the LRU via pagevec. The decision on whether
422 * to add the page to the [in]active [file|anon] list is deferred until the
423 * pagevec is drained. This gives a chance for the caller of lru_cache_add()
424 * have the page added to the active list using mark_page_accessed().
426 void lru_cache_add(struct page *page)
428 VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page);
429 VM_BUG_ON_PAGE(PageLRU(page), page);
430 __lru_cache_add(page);
434 * add_page_to_unevictable_list - add a page to the unevictable list
435 * @page: the page to be added to the unevictable list
437 * Add page directly to its zone's unevictable list. To avoid races with
438 * tasks that might be making the page evictable, through eg. munlock,
439 * munmap or exit, while it's not on the lru, we want to add the page
440 * while it's locked or otherwise "invisible" to other tasks. This is
441 * difficult to do when using the pagevec cache, so bypass that.
443 void add_page_to_unevictable_list(struct page *page)
445 struct zone *zone = page_zone(page);
446 struct lruvec *lruvec;
448 spin_lock_irq(&zone->lru_lock);
449 lruvec = mem_cgroup_page_lruvec(page, zone);
450 ClearPageActive(page);
451 SetPageUnevictable(page);
453 add_page_to_lru_list(page, lruvec, LRU_UNEVICTABLE);
454 spin_unlock_irq(&zone->lru_lock);
458 * lru_cache_add_active_or_unevictable
459 * @page: the page to be added to LRU
460 * @vma: vma in which page is mapped for determining reclaimability
462 * Place @page on the active or unevictable LRU list, depending on its
463 * evictability. Note that if the page is not evictable, it goes
464 * directly back onto it's zone's unevictable list, it does NOT use a
467 void lru_cache_add_active_or_unevictable(struct page *page,
468 struct vm_area_struct *vma)
470 VM_BUG_ON_PAGE(PageLRU(page), page);
472 if (likely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) != VM_LOCKED)) {
478 if (!TestSetPageMlocked(page)) {
480 * We use the irq-unsafe __mod_zone_page_stat because this
481 * counter is not modified from interrupt context, and the pte
482 * lock is held(spinlock), which implies preemption disabled.
484 __mod_zone_page_state(page_zone(page), NR_MLOCK,
485 hpage_nr_pages(page));
486 count_vm_event(UNEVICTABLE_PGMLOCKED);
488 add_page_to_unevictable_list(page);
492 * If the page can not be invalidated, it is moved to the
493 * inactive list to speed up its reclaim. It is moved to the
494 * head of the list, rather than the tail, to give the flusher
495 * threads some time to write it out, as this is much more
496 * effective than the single-page writeout from reclaim.
498 * If the page isn't page_mapped and dirty/writeback, the page
499 * could reclaim asap using PG_reclaim.
501 * 1. active, mapped page -> none
502 * 2. active, dirty/writeback page -> inactive, head, PG_reclaim
503 * 3. inactive, mapped page -> none
504 * 4. inactive, dirty/writeback page -> inactive, head, PG_reclaim
505 * 5. inactive, clean -> inactive, tail
508 * In 4, why it moves inactive's head, the VM expects the page would
509 * be write it out by flusher threads as this is much more effective
510 * than the single-page writeout from reclaim.
512 static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec,
521 if (PageUnevictable(page))
524 /* Some processes are using the page */
525 if (page_mapped(page))
528 active = PageActive(page);
529 file = page_is_file_cache(page);
530 lru = page_lru_base_type(page);
532 del_page_from_lru_list(page, lruvec, lru + active);
533 ClearPageActive(page);
534 ClearPageReferenced(page);
535 add_page_to_lru_list(page, lruvec, lru);
537 if (PageWriteback(page) || PageDirty(page)) {
539 * PG_reclaim could be raced with end_page_writeback
540 * It can make readahead confusing. But race window
541 * is _really_ small and it's non-critical problem.
543 SetPageReclaim(page);
546 * The page's writeback ends up during pagevec
547 * We moves tha page into tail of inactive.
549 list_move_tail(&page->lru, &lruvec->lists[lru]);
550 __count_vm_event(PGROTATED);
554 __count_vm_event(PGDEACTIVATE);
555 update_page_reclaim_stat(lruvec, file, 0);
559 static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec,
562 if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) {
563 int file = page_is_file_cache(page);
564 int lru = page_lru_base_type(page);
566 del_page_from_lru_list(page, lruvec, lru + LRU_ACTIVE);
567 ClearPageActive(page);
568 ClearPageReferenced(page);
569 add_page_to_lru_list(page, lruvec, lru);
571 __count_vm_event(PGDEACTIVATE);
572 update_page_reclaim_stat(lruvec, file, 0);
577 * Drain pages out of the cpu's pagevecs.
578 * Either "cpu" is the current CPU, and preemption has already been
579 * disabled; or "cpu" is being hot-unplugged, and is already dead.
581 void lru_add_drain_cpu(int cpu)
583 struct pagevec *pvec = &per_cpu(lru_add_pvec, cpu);
585 if (pagevec_count(pvec))
586 __pagevec_lru_add(pvec);
588 pvec = &per_cpu(lru_rotate_pvecs, cpu);
589 if (pagevec_count(pvec)) {
592 /* No harm done if a racing interrupt already did this */
593 local_irq_save(flags);
594 pagevec_move_tail(pvec);
595 local_irq_restore(flags);
598 pvec = &per_cpu(lru_deactivate_file_pvecs, cpu);
599 if (pagevec_count(pvec))
600 pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL);
602 pvec = &per_cpu(lru_deactivate_pvecs, cpu);
603 if (pagevec_count(pvec))
604 pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
606 activate_page_drain(cpu);
610 * deactivate_file_page - forcefully deactivate a file page
611 * @page: page to deactivate
613 * This function hints the VM that @page is a good reclaim candidate,
614 * for example if its invalidation fails due to the page being dirty
615 * or under writeback.
617 void deactivate_file_page(struct page *page)
620 * In a workload with many unevictable page such as mprotect,
621 * unevictable page deactivation for accelerating reclaim is pointless.
623 if (PageUnevictable(page))
626 if (likely(get_page_unless_zero(page))) {
627 struct pagevec *pvec = &get_cpu_var(lru_deactivate_file_pvecs);
629 if (!pagevec_add(pvec, page))
630 pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL);
631 put_cpu_var(lru_deactivate_file_pvecs);
636 * deactivate_page - deactivate a page
637 * @page: page to deactivate
639 * deactivate_page() moves @page to the inactive list if @page was on the active
640 * list and was not an unevictable page. This is done to accelerate the reclaim
643 void deactivate_page(struct page *page)
645 if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) {
646 struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs);
648 page_cache_get(page);
649 if (!pagevec_add(pvec, page))
650 pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
651 put_cpu_var(lru_deactivate_pvecs);
655 void lru_add_drain(void)
657 lru_add_drain_cpu(get_cpu());
661 static void lru_add_drain_per_cpu(struct work_struct *dummy)
666 static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
668 void lru_add_drain_all(void)
670 static DEFINE_MUTEX(lock);
671 static struct cpumask has_work;
676 cpumask_clear(&has_work);
678 for_each_online_cpu(cpu) {
679 struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
681 if (pagevec_count(&per_cpu(lru_add_pvec, cpu)) ||
682 pagevec_count(&per_cpu(lru_rotate_pvecs, cpu)) ||
683 pagevec_count(&per_cpu(lru_deactivate_file_pvecs, cpu)) ||
684 pagevec_count(&per_cpu(lru_deactivate_pvecs, cpu)) ||
685 need_activate_page_drain(cpu)) {
686 INIT_WORK(work, lru_add_drain_per_cpu);
687 schedule_work_on(cpu, work);
688 cpumask_set_cpu(cpu, &has_work);
692 for_each_cpu(cpu, &has_work)
693 flush_work(&per_cpu(lru_add_drain_work, cpu));
700 * release_pages - batched page_cache_release()
701 * @pages: array of pages to release
702 * @nr: number of pages
703 * @cold: whether the pages are cache cold
705 * Decrement the reference count on all the pages in @pages. If it
706 * fell to zero, remove the page from the LRU and free it.
708 void release_pages(struct page **pages, int nr, bool cold)
711 LIST_HEAD(pages_to_free);
712 struct zone *zone = NULL;
713 struct lruvec *lruvec;
714 unsigned long uninitialized_var(flags);
715 unsigned int uninitialized_var(lock_batch);
717 for (i = 0; i < nr; i++) {
718 struct page *page = pages[i];
721 * Make sure the IRQ-safe lock-holding time does not get
722 * excessive with a continuous string of pages from the
723 * same zone. The lock is held only if zone != NULL.
725 if (zone && ++lock_batch == SWAP_CLUSTER_MAX) {
726 spin_unlock_irqrestore(&zone->lru_lock, flags);
730 page = compound_head(page);
731 if (!put_page_testzero(page))
734 if (PageCompound(page)) {
736 spin_unlock_irqrestore(&zone->lru_lock, flags);
739 __put_compound_page(page);
744 struct zone *pagezone = page_zone(page);
746 if (pagezone != zone) {
748 spin_unlock_irqrestore(&zone->lru_lock,
752 spin_lock_irqsave(&zone->lru_lock, flags);
755 lruvec = mem_cgroup_page_lruvec(page, zone);
756 VM_BUG_ON_PAGE(!PageLRU(page), page);
757 __ClearPageLRU(page);
758 del_page_from_lru_list(page, lruvec, page_off_lru(page));
761 /* Clear Active bit in case of parallel mark_page_accessed */
762 __ClearPageActive(page);
764 list_add(&page->lru, &pages_to_free);
767 spin_unlock_irqrestore(&zone->lru_lock, flags);
769 mem_cgroup_uncharge_list(&pages_to_free);
770 free_hot_cold_page_list(&pages_to_free, cold);
772 EXPORT_SYMBOL(release_pages);
775 * The pages which we're about to release may be in the deferred lru-addition
776 * queues. That would prevent them from really being freed right now. That's
777 * OK from a correctness point of view but is inefficient - those pages may be
778 * cache-warm and we want to give them back to the page allocator ASAP.
780 * So __pagevec_release() will drain those queues here. __pagevec_lru_add()
781 * and __pagevec_lru_add_active() call release_pages() directly to avoid
784 void __pagevec_release(struct pagevec *pvec)
787 release_pages(pvec->pages, pagevec_count(pvec), pvec->cold);
788 pagevec_reinit(pvec);
790 EXPORT_SYMBOL(__pagevec_release);
792 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
793 /* used by __split_huge_page_refcount() */
794 void lru_add_page_tail(struct page *page, struct page *page_tail,
795 struct lruvec *lruvec, struct list_head *list)
799 VM_BUG_ON_PAGE(!PageHead(page), page);
800 VM_BUG_ON_PAGE(PageCompound(page_tail), page);
801 VM_BUG_ON_PAGE(PageLRU(page_tail), page);
802 VM_BUG_ON(NR_CPUS != 1 &&
803 !spin_is_locked(&lruvec_zone(lruvec)->lru_lock));
806 SetPageLRU(page_tail);
808 if (likely(PageLRU(page)))
809 list_add_tail(&page_tail->lru, &page->lru);
811 /* page reclaim is reclaiming a huge page */
813 list_add_tail(&page_tail->lru, list);
815 struct list_head *list_head;
817 * Head page has not yet been counted, as an hpage,
818 * so we must account for each subpage individually.
820 * Use the standard add function to put page_tail on the list,
821 * but then correct its position so they all end up in order.
823 add_page_to_lru_list(page_tail, lruvec, page_lru(page_tail));
824 list_head = page_tail->lru.prev;
825 list_move_tail(&page_tail->lru, list_head);
828 if (!PageUnevictable(page))
829 update_page_reclaim_stat(lruvec, file, PageActive(page_tail));
831 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
833 static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec,
836 int file = page_is_file_cache(page);
837 int active = PageActive(page);
838 enum lru_list lru = page_lru(page);
840 VM_BUG_ON_PAGE(PageLRU(page), page);
843 add_page_to_lru_list(page, lruvec, lru);
844 update_page_reclaim_stat(lruvec, file, active);
845 trace_mm_lru_insertion(page, lru);
849 * Add the passed pages to the LRU, then drop the caller's refcount
850 * on them. Reinitialises the caller's pagevec.
852 void __pagevec_lru_add(struct pagevec *pvec)
854 pagevec_lru_move_fn(pvec, __pagevec_lru_add_fn, NULL);
856 EXPORT_SYMBOL(__pagevec_lru_add);
859 * pagevec_lookup_entries - gang pagecache lookup
860 * @pvec: Where the resulting entries are placed
861 * @mapping: The address_space to search
862 * @start: The starting entry index
863 * @nr_entries: The maximum number of entries
864 * @indices: The cache indices corresponding to the entries in @pvec
866 * pagevec_lookup_entries() will search for and return a group of up
867 * to @nr_entries pages and shadow entries in the mapping. All
868 * entries are placed in @pvec. pagevec_lookup_entries() takes a
869 * reference against actual pages in @pvec.
871 * The search returns a group of mapping-contiguous entries with
872 * ascending indexes. There may be holes in the indices due to
873 * not-present entries.
875 * pagevec_lookup_entries() returns the number of entries which were
878 unsigned pagevec_lookup_entries(struct pagevec *pvec,
879 struct address_space *mapping,
880 pgoff_t start, unsigned nr_pages,
883 pvec->nr = find_get_entries(mapping, start, nr_pages,
884 pvec->pages, indices);
885 return pagevec_count(pvec);
889 * pagevec_remove_exceptionals - pagevec exceptionals pruning
890 * @pvec: The pagevec to prune
892 * pagevec_lookup_entries() fills both pages and exceptional radix
893 * tree entries into the pagevec. This function prunes all
894 * exceptionals from @pvec without leaving holes, so that it can be
895 * passed on to page-only pagevec operations.
897 void pagevec_remove_exceptionals(struct pagevec *pvec)
901 for (i = 0, j = 0; i < pagevec_count(pvec); i++) {
902 struct page *page = pvec->pages[i];
903 if (!radix_tree_exceptional_entry(page))
904 pvec->pages[j++] = page;
910 * pagevec_lookup - gang pagecache lookup
911 * @pvec: Where the resulting pages are placed
912 * @mapping: The address_space to search
913 * @start: The starting page index
914 * @nr_pages: The maximum number of pages
916 * pagevec_lookup() will search for and return a group of up to @nr_pages pages
917 * in the mapping. The pages are placed in @pvec. pagevec_lookup() takes a
918 * reference against the pages in @pvec.
920 * The search returns a group of mapping-contiguous pages with ascending
921 * indexes. There may be holes in the indices due to not-present pages.
923 * pagevec_lookup() returns the number of pages which were found.
925 unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping,
926 pgoff_t start, unsigned nr_pages)
928 pvec->nr = find_get_pages(mapping, start, nr_pages, pvec->pages);
929 return pagevec_count(pvec);
931 EXPORT_SYMBOL(pagevec_lookup);
933 unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping,
934 pgoff_t *index, int tag, unsigned nr_pages)
936 pvec->nr = find_get_pages_tag(mapping, index, tag,
937 nr_pages, pvec->pages);
938 return pagevec_count(pvec);
940 EXPORT_SYMBOL(pagevec_lookup_tag);
943 * Perform any setup for the swap system
945 void __init swap_setup(void)
947 unsigned long megs = totalram_pages >> (20 - PAGE_SHIFT);
951 for (i = 0; i < MAX_SWAPFILES; i++)
952 spin_lock_init(&swapper_spaces[i].tree_lock);
955 /* Use a smaller cluster for small-memory machines */
961 * Right now other parts of the system means that we
962 * _really_ don't want to cluster much more