Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : /*
3 : * linux/mm/swap.c
4 : *
5 : * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
6 : */
7 :
8 : /*
9 : * This file contains the default values for the operation of the
10 : * Linux VM subsystem. Fine-tuning documentation can be found in
11 : * Documentation/admin-guide/sysctl/vm.rst.
12 : * Started 18.12.91
13 : * Swap aging added 23.2.95, Stephen Tweedie.
14 : * Buffermem limits added 12.3.98, Rik van Riel.
15 : */
16 :
17 : #include <linux/mm.h>
18 : #include <linux/sched.h>
19 : #include <linux/kernel_stat.h>
20 : #include <linux/swap.h>
21 : #include <linux/mman.h>
22 : #include <linux/pagemap.h>
23 : #include <linux/pagevec.h>
24 : #include <linux/init.h>
25 : #include <linux/export.h>
26 : #include <linux/mm_inline.h>
27 : #include <linux/percpu_counter.h>
28 : #include <linux/memremap.h>
29 : #include <linux/percpu.h>
30 : #include <linux/cpu.h>
31 : #include <linux/notifier.h>
32 : #include <linux/backing-dev.h>
33 : #include <linux/memcontrol.h>
34 : #include <linux/gfp.h>
35 : #include <linux/uio.h>
36 : #include <linux/hugetlb.h>
37 : #include <linux/page_idle.h>
38 : #include <linux/local_lock.h>
39 : #include <linux/buffer_head.h>
40 :
41 : #include "internal.h"
42 :
43 : #define CREATE_TRACE_POINTS
44 : #include <trace/events/pagemap.h>
45 :
46 : /* How many pages do we try to swap or page in/out together? */
47 : int page_cluster;
48 :
49 : /* Protecting only lru_rotate.pvec which requires disabling interrupts */
50 : struct lru_rotate {
51 : local_lock_t lock;
52 : struct pagevec pvec;
53 : };
54 : static DEFINE_PER_CPU(struct lru_rotate, lru_rotate) = {
55 : .lock = INIT_LOCAL_LOCK(lock),
56 : };
57 :
58 : /*
59 : * The following struct pagevec are grouped together because they are protected
60 : * by disabling preemption (and interrupts remain enabled).
61 : */
62 : struct lru_pvecs {
63 : local_lock_t lock;
64 : struct pagevec lru_add;
65 : struct pagevec lru_deactivate_file;
66 : struct pagevec lru_deactivate;
67 : struct pagevec lru_lazyfree;
68 : #ifdef CONFIG_SMP
69 : struct pagevec activate_page;
70 : #endif
71 : };
72 : static DEFINE_PER_CPU(struct lru_pvecs, lru_pvecs) = {
73 : .lock = INIT_LOCAL_LOCK(lock),
74 : };
75 :
76 : /*
77 : * This path almost never happens for VM activity - pages are normally freed
78 : * via pagevecs. But it gets used by networking - and for compound pages.
79 : */
80 0 : static void __page_cache_release(struct page *page)
81 : {
82 0 : if (PageLRU(page)) {
83 0 : struct folio *folio = page_folio(page);
84 : struct lruvec *lruvec;
85 : unsigned long flags;
86 :
87 0 : lruvec = folio_lruvec_lock_irqsave(folio, &flags);
88 0 : del_page_from_lru_list(page, lruvec);
89 0 : __clear_page_lru_flags(page);
90 0 : unlock_page_lruvec_irqrestore(lruvec, flags);
91 : }
92 : /* See comment on PageMlocked in release_pages() */
93 0 : if (unlikely(PageMlocked(page))) {
94 0 : int nr_pages = thp_nr_pages(page);
95 :
96 0 : __ClearPageMlocked(page);
97 0 : mod_zone_page_state(page_zone(page), NR_MLOCK, -nr_pages);
98 0 : count_vm_events(UNEVICTABLE_PGCLEARED, nr_pages);
99 : }
100 0 : }
101 :
102 : static void __put_single_page(struct page *page)
103 : {
104 0 : __page_cache_release(page);
105 0 : mem_cgroup_uncharge(page_folio(page));
106 0 : free_unref_page(page, 0);
107 : }
108 :
109 : static void __put_compound_page(struct page *page)
110 : {
111 : /*
112 : * __page_cache_release() is supposed to be called for thp, not for
113 : * hugetlb. This is because hugetlb page does never have PageLRU set
114 : * (it's never listed to any LRU lists) and no memcg routines should
115 : * be called for hugetlb (it has a separate hugetlb_cgroup.)
116 : */
117 0 : if (!PageHuge(page))
118 0 : __page_cache_release(page);
119 0 : destroy_compound_page(page);
120 : }
121 :
122 0 : void __put_page(struct page *page)
123 : {
124 0 : if (unlikely(is_zone_device_page(page)))
125 : free_zone_device_page(page);
126 0 : else if (unlikely(PageCompound(page)))
127 : __put_compound_page(page);
128 : else
129 : __put_single_page(page);
130 0 : }
131 : EXPORT_SYMBOL(__put_page);
132 :
133 : /**
134 : * put_pages_list() - release a list of pages
135 : * @pages: list of pages threaded on page->lru
136 : *
137 : * Release a list of pages which are strung together on page.lru.
138 : */
139 0 : void put_pages_list(struct list_head *pages)
140 : {
141 : struct page *page, *next;
142 :
143 0 : list_for_each_entry_safe(page, next, pages, lru) {
144 0 : if (!put_page_testzero(page)) {
145 0 : list_del(&page->lru);
146 0 : continue;
147 : }
148 0 : if (PageHead(page)) {
149 0 : list_del(&page->lru);
150 0 : __put_compound_page(page);
151 0 : continue;
152 : }
153 : /* Cannot be PageLRU because it's passed to us using the lru */
154 : }
155 :
156 0 : free_unref_page_list(pages);
157 0 : INIT_LIST_HEAD(pages);
158 0 : }
159 : EXPORT_SYMBOL(put_pages_list);
160 :
161 : /*
162 : * get_kernel_pages() - pin kernel pages in memory
163 : * @kiov: An array of struct kvec structures
164 : * @nr_segs: number of segments to pin
165 : * @write: pinning for read/write, currently ignored
166 : * @pages: array that receives pointers to the pages pinned.
167 : * Should be at least nr_segs long.
168 : *
169 : * Returns number of pages pinned. This may be fewer than the number
170 : * requested. If nr_pages is 0 or negative, returns 0. If no pages
171 : * were pinned, returns -errno. Each page returned must be released
172 : * with a put_page() call when it is finished with.
173 : */
174 0 : int get_kernel_pages(const struct kvec *kiov, int nr_segs, int write,
175 : struct page **pages)
176 : {
177 : int seg;
178 :
179 0 : for (seg = 0; seg < nr_segs; seg++) {
180 0 : if (WARN_ON(kiov[seg].iov_len != PAGE_SIZE))
181 : return seg;
182 :
183 0 : pages[seg] = kmap_to_page(kiov[seg].iov_base);
184 0 : get_page(pages[seg]);
185 : }
186 :
187 : return seg;
188 : }
189 : EXPORT_SYMBOL_GPL(get_kernel_pages);
190 :
191 0 : static void pagevec_lru_move_fn(struct pagevec *pvec,
192 : void (*move_fn)(struct page *page, struct lruvec *lruvec))
193 : {
194 : int i;
195 0 : struct lruvec *lruvec = NULL;
196 0 : unsigned long flags = 0;
197 :
198 0 : for (i = 0; i < pagevec_count(pvec); i++) {
199 0 : struct page *page = pvec->pages[i];
200 0 : struct folio *folio = page_folio(page);
201 :
202 : /* block memcg migration during page moving between lru */
203 0 : if (!TestClearPageLRU(page))
204 0 : continue;
205 :
206 0 : lruvec = folio_lruvec_relock_irqsave(folio, lruvec, &flags);
207 0 : (*move_fn)(page, lruvec);
208 :
209 : SetPageLRU(page);
210 : }
211 0 : if (lruvec)
212 0 : unlock_page_lruvec_irqrestore(lruvec, flags);
213 0 : release_pages(pvec->pages, pvec->nr);
214 0 : pagevec_reinit(pvec);
215 0 : }
216 :
217 0 : static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec)
218 : {
219 0 : struct folio *folio = page_folio(page);
220 :
221 0 : if (!folio_test_unevictable(folio)) {
222 0 : lruvec_del_folio(lruvec, folio);
223 0 : folio_clear_active(folio);
224 0 : lruvec_add_folio_tail(lruvec, folio);
225 0 : __count_vm_events(PGROTATED, folio_nr_pages(folio));
226 : }
227 0 : }
228 :
229 : /* return true if pagevec needs to drain */
230 : static bool pagevec_add_and_need_flush(struct pagevec *pvec, struct page *page)
231 : {
232 0 : bool ret = false;
233 :
234 0 : if (!pagevec_add(pvec, page) || PageCompound(page) ||
235 : lru_cache_disabled())
236 : ret = true;
237 :
238 : return ret;
239 : }
240 :
241 : /*
242 : * Writeback is about to end against a folio which has been marked for
243 : * immediate reclaim. If it still appears to be reclaimable, move it
244 : * to the tail of the inactive list.
245 : *
246 : * folio_rotate_reclaimable() must disable IRQs, to prevent nasty races.
247 : */
248 0 : void folio_rotate_reclaimable(struct folio *folio)
249 : {
250 0 : if (!folio_test_locked(folio) && !folio_test_dirty(folio) &&
251 0 : !folio_test_unevictable(folio) && folio_test_lru(folio)) {
252 : struct pagevec *pvec;
253 : unsigned long flags;
254 :
255 0 : folio_get(folio);
256 0 : local_lock_irqsave(&lru_rotate.lock, flags);
257 0 : pvec = this_cpu_ptr(&lru_rotate.pvec);
258 0 : if (pagevec_add_and_need_flush(pvec, &folio->page))
259 0 : pagevec_lru_move_fn(pvec, pagevec_move_tail_fn);
260 0 : local_unlock_irqrestore(&lru_rotate.lock, flags);
261 : }
262 0 : }
263 :
264 0 : void lru_note_cost(struct lruvec *lruvec, bool file, unsigned int nr_pages)
265 : {
266 : do {
267 : unsigned long lrusize;
268 :
269 : /*
270 : * Hold lruvec->lru_lock is safe here, since
271 : * 1) The pinned lruvec in reclaim, or
272 : * 2) From a pre-LRU page during refault (which also holds the
273 : * rcu lock, so would be safe even if the page was on the LRU
274 : * and could move simultaneously to a new lruvec).
275 : */
276 0 : spin_lock_irq(&lruvec->lru_lock);
277 : /* Record cost event */
278 0 : if (file)
279 0 : lruvec->file_cost += nr_pages;
280 : else
281 0 : lruvec->anon_cost += nr_pages;
282 :
283 : /*
284 : * Decay previous events
285 : *
286 : * Because workloads change over time (and to avoid
287 : * overflow) we keep these statistics as a floating
288 : * average, which ends up weighing recent refaults
289 : * more than old ones.
290 : */
291 0 : lrusize = lruvec_page_state(lruvec, NR_INACTIVE_ANON) +
292 0 : lruvec_page_state(lruvec, NR_ACTIVE_ANON) +
293 0 : lruvec_page_state(lruvec, NR_INACTIVE_FILE) +
294 0 : lruvec_page_state(lruvec, NR_ACTIVE_FILE);
295 :
296 0 : if (lruvec->file_cost + lruvec->anon_cost > lrusize / 4) {
297 0 : lruvec->file_cost /= 2;
298 0 : lruvec->anon_cost /= 2;
299 : }
300 0 : spin_unlock_irq(&lruvec->lru_lock);
301 0 : } while ((lruvec = parent_lruvec(lruvec)));
302 0 : }
303 :
304 0 : void lru_note_cost_folio(struct folio *folio)
305 : {
306 0 : lru_note_cost(folio_lruvec(folio), folio_is_file_lru(folio),
307 0 : folio_nr_pages(folio));
308 0 : }
309 :
310 0 : static void __folio_activate(struct folio *folio, struct lruvec *lruvec)
311 : {
312 0 : if (!folio_test_active(folio) && !folio_test_unevictable(folio)) {
313 0 : long nr_pages = folio_nr_pages(folio);
314 :
315 0 : lruvec_del_folio(lruvec, folio);
316 0 : folio_set_active(folio);
317 0 : lruvec_add_folio(lruvec, folio);
318 0 : trace_mm_lru_activate(folio);
319 :
320 0 : __count_vm_events(PGACTIVATE, nr_pages);
321 0 : __count_memcg_events(lruvec_memcg(lruvec), PGACTIVATE,
322 : nr_pages);
323 : }
324 0 : }
325 :
326 : #ifdef CONFIG_SMP
327 : static void __activate_page(struct page *page, struct lruvec *lruvec)
328 : {
329 : return __folio_activate(page_folio(page), lruvec);
330 : }
331 :
332 : static void activate_page_drain(int cpu)
333 : {
334 : struct pagevec *pvec = &per_cpu(lru_pvecs.activate_page, cpu);
335 :
336 : if (pagevec_count(pvec))
337 : pagevec_lru_move_fn(pvec, __activate_page);
338 : }
339 :
340 : static bool need_activate_page_drain(int cpu)
341 : {
342 : return pagevec_count(&per_cpu(lru_pvecs.activate_page, cpu)) != 0;
343 : }
344 :
345 : static void folio_activate(struct folio *folio)
346 : {
347 : if (folio_test_lru(folio) && !folio_test_active(folio) &&
348 : !folio_test_unevictable(folio)) {
349 : struct pagevec *pvec;
350 :
351 : folio_get(folio);
352 : local_lock(&lru_pvecs.lock);
353 : pvec = this_cpu_ptr(&lru_pvecs.activate_page);
354 : if (pagevec_add_and_need_flush(pvec, &folio->page))
355 : pagevec_lru_move_fn(pvec, __activate_page);
356 : local_unlock(&lru_pvecs.lock);
357 : }
358 : }
359 :
360 : #else
361 : static inline void activate_page_drain(int cpu)
362 : {
363 : }
364 :
365 0 : static void folio_activate(struct folio *folio)
366 : {
367 : struct lruvec *lruvec;
368 :
369 0 : if (folio_test_clear_lru(folio)) {
370 0 : lruvec = folio_lruvec_lock_irq(folio);
371 0 : __folio_activate(folio, lruvec);
372 0 : unlock_page_lruvec_irq(lruvec);
373 : folio_set_lru(folio);
374 : }
375 0 : }
376 : #endif
377 :
378 : static void __lru_cache_activate_folio(struct folio *folio)
379 : {
380 : struct pagevec *pvec;
381 : int i;
382 :
383 0 : local_lock(&lru_pvecs.lock);
384 0 : pvec = this_cpu_ptr(&lru_pvecs.lru_add);
385 :
386 : /*
387 : * Search backwards on the optimistic assumption that the page being
388 : * activated has just been added to this pagevec. Note that only
389 : * the local pagevec is examined as a !PageLRU page could be in the
390 : * process of being released, reclaimed, migrated or on a remote
391 : * pagevec that is currently being drained. Furthermore, marking
392 : * a remote pagevec's page PageActive potentially hits a race where
393 : * a page is marked PageActive just after it is added to the inactive
394 : * list causing accounting errors and BUG_ON checks to trigger.
395 : */
396 0 : for (i = pagevec_count(pvec) - 1; i >= 0; i--) {
397 0 : struct page *pagevec_page = pvec->pages[i];
398 :
399 0 : if (pagevec_page == &folio->page) {
400 : folio_set_active(folio);
401 : break;
402 : }
403 : }
404 :
405 0 : local_unlock(&lru_pvecs.lock);
406 : }
407 :
408 : /*
409 : * Mark a page as having seen activity.
410 : *
411 : * inactive,unreferenced -> inactive,referenced
412 : * inactive,referenced -> active,unreferenced
413 : * active,unreferenced -> active,referenced
414 : *
415 : * When a newly allocated page is not yet visible, so safe for non-atomic ops,
416 : * __SetPageReferenced(page) may be substituted for mark_page_accessed(page).
417 : */
418 0 : void folio_mark_accessed(struct folio *folio)
419 : {
420 0 : if (!folio_test_referenced(folio)) {
421 : folio_set_referenced(folio);
422 0 : } else if (folio_test_unevictable(folio)) {
423 : /*
424 : * Unevictable pages are on the "LRU_UNEVICTABLE" list. But,
425 : * this list is never rotated or maintained, so marking an
426 : * unevictable page accessed has no effect.
427 : */
428 0 : } else if (!folio_test_active(folio)) {
429 : /*
430 : * If the page is on the LRU, queue it for activation via
431 : * lru_pvecs.activate_page. Otherwise, assume the page is on a
432 : * pagevec, mark it active and it'll be moved to the active
433 : * LRU on the next drain.
434 : */
435 0 : if (folio_test_lru(folio))
436 0 : folio_activate(folio);
437 : else
438 : __lru_cache_activate_folio(folio);
439 0 : folio_clear_referenced(folio);
440 0 : workingset_activation(folio);
441 : }
442 0 : if (folio_test_idle(folio))
443 : folio_clear_idle(folio);
444 0 : }
445 : EXPORT_SYMBOL(folio_mark_accessed);
446 :
447 : /**
448 : * folio_add_lru - Add a folio to an LRU list.
449 : * @folio: The folio to be added to the LRU.
450 : *
451 : * Queue the folio for addition to the LRU. The decision on whether
452 : * to add the page to the [in]active [file|anon] list is deferred until the
453 : * pagevec is drained. This gives a chance for the caller of folio_add_lru()
454 : * have the folio added to the active list using folio_mark_accessed().
455 : */
456 0 : void folio_add_lru(struct folio *folio)
457 : {
458 : struct pagevec *pvec;
459 :
460 : VM_BUG_ON_FOLIO(folio_test_active(folio) && folio_test_unevictable(folio), folio);
461 : VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
462 :
463 0 : folio_get(folio);
464 0 : local_lock(&lru_pvecs.lock);
465 0 : pvec = this_cpu_ptr(&lru_pvecs.lru_add);
466 0 : if (pagevec_add_and_need_flush(pvec, &folio->page))
467 0 : __pagevec_lru_add(pvec);
468 0 : local_unlock(&lru_pvecs.lock);
469 0 : }
470 : EXPORT_SYMBOL(folio_add_lru);
471 :
472 : /**
473 : * lru_cache_add_inactive_or_unevictable
474 : * @page: the page to be added to LRU
475 : * @vma: vma in which page is mapped for determining reclaimability
476 : *
477 : * Place @page on the inactive or unevictable LRU list, depending on its
478 : * evictability.
479 : */
480 0 : void lru_cache_add_inactive_or_unevictable(struct page *page,
481 : struct vm_area_struct *vma)
482 : {
483 : VM_BUG_ON_PAGE(PageLRU(page), page);
484 :
485 0 : if (unlikely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED))
486 0 : mlock_new_page(page);
487 : else
488 0 : lru_cache_add(page);
489 0 : }
490 :
491 : /*
492 : * If the page can not be invalidated, it is moved to the
493 : * inactive list to speed up its reclaim. It is moved to the
494 : * head of the list, rather than the tail, to give the flusher
495 : * threads some time to write it out, as this is much more
496 : * effective than the single-page writeout from reclaim.
497 : *
498 : * If the page isn't page_mapped and dirty/writeback, the page
499 : * could reclaim asap using PG_reclaim.
500 : *
501 : * 1. active, mapped page -> none
502 : * 2. active, dirty/writeback page -> inactive, head, PG_reclaim
503 : * 3. inactive, mapped page -> none
504 : * 4. inactive, dirty/writeback page -> inactive, head, PG_reclaim
505 : * 5. inactive, clean -> inactive, tail
506 : * 6. Others -> none
507 : *
508 : * In 4, why it moves inactive's head, the VM expects the page would
509 : * be write it out by flusher threads as this is much more effective
510 : * than the single-page writeout from reclaim.
511 : */
512 0 : static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec)
513 : {
514 0 : bool active = PageActive(page);
515 0 : int nr_pages = thp_nr_pages(page);
516 :
517 0 : if (PageUnevictable(page))
518 : return;
519 :
520 : /* Some processes are using the page */
521 0 : if (page_mapped(page))
522 : return;
523 :
524 0 : del_page_from_lru_list(page, lruvec);
525 0 : ClearPageActive(page);
526 0 : ClearPageReferenced(page);
527 :
528 0 : if (PageWriteback(page) || PageDirty(page)) {
529 : /*
530 : * PG_reclaim could be raced with end_page_writeback
531 : * It can make readahead confusing. But race window
532 : * is _really_ small and it's non-critical problem.
533 : */
534 0 : add_page_to_lru_list(page, lruvec);
535 : SetPageReclaim(page);
536 : } else {
537 : /*
538 : * The page's writeback ends up during pagevec
539 : * We move that page into tail of inactive.
540 : */
541 0 : add_page_to_lru_list_tail(page, lruvec);
542 0 : __count_vm_events(PGROTATED, nr_pages);
543 : }
544 :
545 0 : if (active) {
546 0 : __count_vm_events(PGDEACTIVATE, nr_pages);
547 0 : __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE,
548 : nr_pages);
549 : }
550 : }
551 :
552 0 : static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec)
553 : {
554 0 : if (PageActive(page) && !PageUnevictable(page)) {
555 0 : int nr_pages = thp_nr_pages(page);
556 :
557 0 : del_page_from_lru_list(page, lruvec);
558 0 : ClearPageActive(page);
559 0 : ClearPageReferenced(page);
560 0 : add_page_to_lru_list(page, lruvec);
561 :
562 0 : __count_vm_events(PGDEACTIVATE, nr_pages);
563 0 : __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE,
564 : nr_pages);
565 : }
566 0 : }
567 :
568 0 : static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec)
569 : {
570 0 : if (PageAnon(page) && PageSwapBacked(page) &&
571 0 : !PageSwapCache(page) && !PageUnevictable(page)) {
572 0 : int nr_pages = thp_nr_pages(page);
573 :
574 0 : del_page_from_lru_list(page, lruvec);
575 0 : ClearPageActive(page);
576 0 : ClearPageReferenced(page);
577 : /*
578 : * Lazyfree pages are clean anonymous pages. They have
579 : * PG_swapbacked flag cleared, to distinguish them from normal
580 : * anonymous pages
581 : */
582 0 : ClearPageSwapBacked(page);
583 0 : add_page_to_lru_list(page, lruvec);
584 :
585 0 : __count_vm_events(PGLAZYFREE, nr_pages);
586 0 : __count_memcg_events(lruvec_memcg(lruvec), PGLAZYFREE,
587 : nr_pages);
588 : }
589 0 : }
590 :
591 : /*
592 : * Drain pages out of the cpu's pagevecs.
593 : * Either "cpu" is the current CPU, and preemption has already been
594 : * disabled; or "cpu" is being hot-unplugged, and is already dead.
595 : */
596 0 : void lru_add_drain_cpu(int cpu)
597 : {
598 0 : struct pagevec *pvec = &per_cpu(lru_pvecs.lru_add, cpu);
599 :
600 0 : if (pagevec_count(pvec))
601 0 : __pagevec_lru_add(pvec);
602 :
603 0 : pvec = &per_cpu(lru_rotate.pvec, cpu);
604 : /* Disabling interrupts below acts as a compiler barrier. */
605 0 : if (data_race(pagevec_count(pvec))) {
606 : unsigned long flags;
607 :
608 : /* No harm done if a racing interrupt already did this */
609 0 : local_lock_irqsave(&lru_rotate.lock, flags);
610 0 : pagevec_lru_move_fn(pvec, pagevec_move_tail_fn);
611 0 : local_unlock_irqrestore(&lru_rotate.lock, flags);
612 : }
613 :
614 0 : pvec = &per_cpu(lru_pvecs.lru_deactivate_file, cpu);
615 0 : if (pagevec_count(pvec))
616 0 : pagevec_lru_move_fn(pvec, lru_deactivate_file_fn);
617 :
618 0 : pvec = &per_cpu(lru_pvecs.lru_deactivate, cpu);
619 0 : if (pagevec_count(pvec))
620 0 : pagevec_lru_move_fn(pvec, lru_deactivate_fn);
621 :
622 0 : pvec = &per_cpu(lru_pvecs.lru_lazyfree, cpu);
623 0 : if (pagevec_count(pvec))
624 0 : pagevec_lru_move_fn(pvec, lru_lazyfree_fn);
625 :
626 0 : activate_page_drain(cpu);
627 0 : }
628 :
629 : /**
630 : * deactivate_file_folio() - Forcefully deactivate a file folio.
631 : * @folio: Folio to deactivate.
632 : *
633 : * This function hints to the VM that @folio is a good reclaim candidate,
634 : * for example if its invalidation fails due to the folio being dirty
635 : * or under writeback.
636 : *
637 : * Context: Caller holds a reference on the page.
638 : */
639 0 : void deactivate_file_folio(struct folio *folio)
640 : {
641 : struct pagevec *pvec;
642 :
643 : /*
644 : * In a workload with many unevictable pages such as mprotect,
645 : * unevictable folio deactivation for accelerating reclaim is pointless.
646 : */
647 0 : if (folio_test_unevictable(folio))
648 : return;
649 :
650 0 : folio_get(folio);
651 0 : local_lock(&lru_pvecs.lock);
652 0 : pvec = this_cpu_ptr(&lru_pvecs.lru_deactivate_file);
653 :
654 0 : if (pagevec_add_and_need_flush(pvec, &folio->page))
655 0 : pagevec_lru_move_fn(pvec, lru_deactivate_file_fn);
656 0 : local_unlock(&lru_pvecs.lock);
657 : }
658 :
659 : /*
660 : * deactivate_page - deactivate a page
661 : * @page: page to deactivate
662 : *
663 : * deactivate_page() moves @page to the inactive list if @page was on the active
664 : * list and was not an unevictable page. This is done to accelerate the reclaim
665 : * of @page.
666 : */
667 0 : void deactivate_page(struct page *page)
668 : {
669 0 : if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) {
670 : struct pagevec *pvec;
671 :
672 0 : local_lock(&lru_pvecs.lock);
673 0 : pvec = this_cpu_ptr(&lru_pvecs.lru_deactivate);
674 0 : get_page(page);
675 0 : if (pagevec_add_and_need_flush(pvec, page))
676 0 : pagevec_lru_move_fn(pvec, lru_deactivate_fn);
677 0 : local_unlock(&lru_pvecs.lock);
678 : }
679 0 : }
680 :
681 : /**
682 : * mark_page_lazyfree - make an anon page lazyfree
683 : * @page: page to deactivate
684 : *
685 : * mark_page_lazyfree() moves @page to the inactive file list.
686 : * This is done to accelerate the reclaim of @page.
687 : */
688 0 : void mark_page_lazyfree(struct page *page)
689 : {
690 0 : if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) &&
691 0 : !PageSwapCache(page) && !PageUnevictable(page)) {
692 : struct pagevec *pvec;
693 :
694 0 : local_lock(&lru_pvecs.lock);
695 0 : pvec = this_cpu_ptr(&lru_pvecs.lru_lazyfree);
696 0 : get_page(page);
697 0 : if (pagevec_add_and_need_flush(pvec, page))
698 0 : pagevec_lru_move_fn(pvec, lru_lazyfree_fn);
699 0 : local_unlock(&lru_pvecs.lock);
700 : }
701 0 : }
702 :
703 0 : void lru_add_drain(void)
704 : {
705 0 : local_lock(&lru_pvecs.lock);
706 0 : lru_add_drain_cpu(smp_processor_id());
707 0 : local_unlock(&lru_pvecs.lock);
708 0 : mlock_page_drain_local();
709 0 : }
710 :
711 : /*
712 : * It's called from per-cpu workqueue context in SMP case so
713 : * lru_add_drain_cpu and invalidate_bh_lrus_cpu should run on
714 : * the same cpu. It shouldn't be a problem in !SMP case since
715 : * the core is only one and the locks will disable preemption.
716 : */
717 : static void lru_add_and_bh_lrus_drain(void)
718 : {
719 0 : local_lock(&lru_pvecs.lock);
720 0 : lru_add_drain_cpu(smp_processor_id());
721 0 : local_unlock(&lru_pvecs.lock);
722 0 : invalidate_bh_lrus_cpu();
723 0 : mlock_page_drain_local();
724 : }
725 :
726 0 : void lru_add_drain_cpu_zone(struct zone *zone)
727 : {
728 0 : local_lock(&lru_pvecs.lock);
729 0 : lru_add_drain_cpu(smp_processor_id());
730 0 : drain_local_pages(zone);
731 0 : local_unlock(&lru_pvecs.lock);
732 0 : mlock_page_drain_local();
733 0 : }
734 :
735 : #ifdef CONFIG_SMP
736 :
737 : static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work);
738 :
739 : static void lru_add_drain_per_cpu(struct work_struct *dummy)
740 : {
741 : lru_add_and_bh_lrus_drain();
742 : }
743 :
744 : /*
745 : * Doesn't need any cpu hotplug locking because we do rely on per-cpu
746 : * kworkers being shut down before our page_alloc_cpu_dead callback is
747 : * executed on the offlined cpu.
748 : * Calling this function with cpu hotplug locks held can actually lead
749 : * to obscure indirect dependencies via WQ context.
750 : */
751 : inline void __lru_add_drain_all(bool force_all_cpus)
752 : {
753 : /*
754 : * lru_drain_gen - Global pages generation number
755 : *
756 : * (A) Definition: global lru_drain_gen = x implies that all generations
757 : * 0 < n <= x are already *scheduled* for draining.
758 : *
759 : * This is an optimization for the highly-contended use case where a
760 : * user space workload keeps constantly generating a flow of pages for
761 : * each CPU.
762 : */
763 : static unsigned int lru_drain_gen;
764 : static struct cpumask has_work;
765 : static DEFINE_MUTEX(lock);
766 : unsigned cpu, this_gen;
767 :
768 : /*
769 : * Make sure nobody triggers this path before mm_percpu_wq is fully
770 : * initialized.
771 : */
772 : if (WARN_ON(!mm_percpu_wq))
773 : return;
774 :
775 : /*
776 : * Guarantee pagevec counter stores visible by this CPU are visible to
777 : * other CPUs before loading the current drain generation.
778 : */
779 : smp_mb();
780 :
781 : /*
782 : * (B) Locally cache global LRU draining generation number
783 : *
784 : * The read barrier ensures that the counter is loaded before the mutex
785 : * is taken. It pairs with smp_mb() inside the mutex critical section
786 : * at (D).
787 : */
788 : this_gen = smp_load_acquire(&lru_drain_gen);
789 :
790 : mutex_lock(&lock);
791 :
792 : /*
793 : * (C) Exit the draining operation if a newer generation, from another
794 : * lru_add_drain_all(), was already scheduled for draining. Check (A).
795 : */
796 : if (unlikely(this_gen != lru_drain_gen && !force_all_cpus))
797 : goto done;
798 :
799 : /*
800 : * (D) Increment global generation number
801 : *
802 : * Pairs with smp_load_acquire() at (B), outside of the critical
803 : * section. Use a full memory barrier to guarantee that the new global
804 : * drain generation number is stored before loading pagevec counters.
805 : *
806 : * This pairing must be done here, before the for_each_online_cpu loop
807 : * below which drains the page vectors.
808 : *
809 : * Let x, y, and z represent some system CPU numbers, where x < y < z.
810 : * Assume CPU #z is in the middle of the for_each_online_cpu loop
811 : * below and has already reached CPU #y's per-cpu data. CPU #x comes
812 : * along, adds some pages to its per-cpu vectors, then calls
813 : * lru_add_drain_all().
814 : *
815 : * If the paired barrier is done at any later step, e.g. after the
816 : * loop, CPU #x will just exit at (C) and miss flushing out all of its
817 : * added pages.
818 : */
819 : WRITE_ONCE(lru_drain_gen, lru_drain_gen + 1);
820 : smp_mb();
821 :
822 : cpumask_clear(&has_work);
823 : for_each_online_cpu(cpu) {
824 : struct work_struct *work = &per_cpu(lru_add_drain_work, cpu);
825 :
826 : if (pagevec_count(&per_cpu(lru_pvecs.lru_add, cpu)) ||
827 : data_race(pagevec_count(&per_cpu(lru_rotate.pvec, cpu))) ||
828 : pagevec_count(&per_cpu(lru_pvecs.lru_deactivate_file, cpu)) ||
829 : pagevec_count(&per_cpu(lru_pvecs.lru_deactivate, cpu)) ||
830 : pagevec_count(&per_cpu(lru_pvecs.lru_lazyfree, cpu)) ||
831 : need_activate_page_drain(cpu) ||
832 : need_mlock_page_drain(cpu) ||
833 : has_bh_in_lru(cpu, NULL)) {
834 : INIT_WORK(work, lru_add_drain_per_cpu);
835 : queue_work_on(cpu, mm_percpu_wq, work);
836 : __cpumask_set_cpu(cpu, &has_work);
837 : }
838 : }
839 :
840 : for_each_cpu(cpu, &has_work)
841 : flush_work(&per_cpu(lru_add_drain_work, cpu));
842 :
843 : done:
844 : mutex_unlock(&lock);
845 : }
846 :
847 : void lru_add_drain_all(void)
848 : {
849 : __lru_add_drain_all(false);
850 : }
851 : #else
852 0 : void lru_add_drain_all(void)
853 : {
854 : lru_add_drain();
855 0 : }
856 : #endif /* CONFIG_SMP */
857 :
858 : atomic_t lru_disable_count = ATOMIC_INIT(0);
859 :
860 : /*
861 : * lru_cache_disable() needs to be called before we start compiling
862 : * a list of pages to be migrated using isolate_lru_page().
863 : * It drains pages on LRU cache and then disable on all cpus until
864 : * lru_cache_enable is called.
865 : *
866 : * Must be paired with a call to lru_cache_enable().
867 : */
868 0 : void lru_cache_disable(void)
869 : {
870 0 : atomic_inc(&lru_disable_count);
871 : /*
872 : * Readers of lru_disable_count are protected by either disabling
873 : * preemption or rcu_read_lock:
874 : *
875 : * preempt_disable, local_irq_disable [bh_lru_lock()]
876 : * rcu_read_lock [rt_spin_lock CONFIG_PREEMPT_RT]
877 : * preempt_disable [local_lock !CONFIG_PREEMPT_RT]
878 : *
879 : * Since v5.1 kernel, synchronize_rcu() is guaranteed to wait on
880 : * preempt_disable() regions of code. So any CPU which sees
881 : * lru_disable_count = 0 will have exited the critical
882 : * section when synchronize_rcu() returns.
883 : */
884 0 : synchronize_rcu();
885 : #ifdef CONFIG_SMP
886 : __lru_add_drain_all(true);
887 : #else
888 : lru_add_and_bh_lrus_drain();
889 : #endif
890 0 : }
891 :
892 : /**
893 : * release_pages - batched put_page()
894 : * @pages: array of pages to release
895 : * @nr: number of pages
896 : *
897 : * Decrement the reference count on all the pages in @pages. If it
898 : * fell to zero, remove the page from the LRU and free it.
899 : */
900 0 : void release_pages(struct page **pages, int nr)
901 : {
902 : int i;
903 0 : LIST_HEAD(pages_to_free);
904 0 : struct lruvec *lruvec = NULL;
905 0 : unsigned long flags = 0;
906 : unsigned int lock_batch;
907 :
908 0 : for (i = 0; i < nr; i++) {
909 0 : struct page *page = pages[i];
910 0 : struct folio *folio = page_folio(page);
911 :
912 : /*
913 : * Make sure the IRQ-safe lock-holding time does not get
914 : * excessive with a continuous string of pages from the
915 : * same lruvec. The lock is held only if lruvec != NULL.
916 : */
917 0 : if (lruvec && ++lock_batch == SWAP_CLUSTER_MAX) {
918 0 : unlock_page_lruvec_irqrestore(lruvec, flags);
919 0 : lruvec = NULL;
920 : }
921 :
922 0 : page = &folio->page;
923 0 : if (is_huge_zero_page(page))
924 : continue;
925 :
926 0 : if (is_zone_device_page(page)) {
927 : if (lruvec) {
928 : unlock_page_lruvec_irqrestore(lruvec, flags);
929 : lruvec = NULL;
930 : }
931 : if (put_devmap_managed_page(page))
932 : continue;
933 : if (put_page_testzero(page))
934 : free_zone_device_page(page);
935 : continue;
936 : }
937 :
938 0 : if (!put_page_testzero(page))
939 0 : continue;
940 :
941 0 : if (PageCompound(page)) {
942 0 : if (lruvec) {
943 0 : unlock_page_lruvec_irqrestore(lruvec, flags);
944 0 : lruvec = NULL;
945 : }
946 0 : __put_compound_page(page);
947 0 : continue;
948 : }
949 :
950 0 : if (PageLRU(page)) {
951 0 : struct lruvec *prev_lruvec = lruvec;
952 :
953 0 : lruvec = folio_lruvec_relock_irqsave(folio, lruvec,
954 : &flags);
955 0 : if (prev_lruvec != lruvec)
956 0 : lock_batch = 0;
957 :
958 0 : del_page_from_lru_list(page, lruvec);
959 : __clear_page_lru_flags(page);
960 : }
961 :
962 : /*
963 : * In rare cases, when truncation or holepunching raced with
964 : * munlock after VM_LOCKED was cleared, Mlocked may still be
965 : * found set here. This does not indicate a problem, unless
966 : * "unevictable_pgs_cleared" appears worryingly large.
967 : */
968 0 : if (unlikely(PageMlocked(page))) {
969 0 : __ClearPageMlocked(page);
970 0 : dec_zone_page_state(page, NR_MLOCK);
971 0 : count_vm_event(UNEVICTABLE_PGCLEARED);
972 : }
973 :
974 0 : list_add(&page->lru, &pages_to_free);
975 : }
976 0 : if (lruvec)
977 0 : unlock_page_lruvec_irqrestore(lruvec, flags);
978 :
979 0 : mem_cgroup_uncharge_list(&pages_to_free);
980 0 : free_unref_page_list(&pages_to_free);
981 0 : }
982 : EXPORT_SYMBOL(release_pages);
983 :
984 : /*
985 : * The pages which we're about to release may be in the deferred lru-addition
986 : * queues. That would prevent them from really being freed right now. That's
987 : * OK from a correctness point of view but is inefficient - those pages may be
988 : * cache-warm and we want to give them back to the page allocator ASAP.
989 : *
990 : * So __pagevec_release() will drain those queues here. __pagevec_lru_add()
991 : * and __pagevec_lru_add_active() call release_pages() directly to avoid
992 : * mutual recursion.
993 : */
994 0 : void __pagevec_release(struct pagevec *pvec)
995 : {
996 0 : if (!pvec->percpu_pvec_drained) {
997 : lru_add_drain();
998 0 : pvec->percpu_pvec_drained = true;
999 : }
1000 0 : release_pages(pvec->pages, pagevec_count(pvec));
1001 0 : pagevec_reinit(pvec);
1002 0 : }
1003 : EXPORT_SYMBOL(__pagevec_release);
1004 :
1005 0 : static void __pagevec_lru_add_fn(struct folio *folio, struct lruvec *lruvec)
1006 : {
1007 0 : int was_unevictable = folio_test_clear_unevictable(folio);
1008 0 : long nr_pages = folio_nr_pages(folio);
1009 :
1010 : VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);
1011 :
1012 0 : folio_set_lru(folio);
1013 : /*
1014 : * Is an smp_mb__after_atomic() still required here, before
1015 : * folio_evictable() tests PageMlocked, to rule out the possibility
1016 : * of stranding an evictable folio on an unevictable LRU? I think
1017 : * not, because __munlock_page() only clears PageMlocked while the LRU
1018 : * lock is held.
1019 : *
1020 : * (That is not true of __page_cache_release(), and not necessarily
1021 : * true of release_pages(): but those only clear PageMlocked after
1022 : * put_page_testzero() has excluded any other users of the page.)
1023 : */
1024 0 : if (folio_evictable(folio)) {
1025 0 : if (was_unevictable)
1026 : __count_vm_events(UNEVICTABLE_PGRESCUED, nr_pages);
1027 : } else {
1028 0 : folio_clear_active(folio);
1029 0 : folio_set_unevictable(folio);
1030 : /*
1031 : * folio->mlock_count = !!folio_test_mlocked(folio)?
1032 : * But that leaves __mlock_page() in doubt whether another
1033 : * actor has already counted the mlock or not. Err on the
1034 : * safe side, underestimate, let page reclaim fix it, rather
1035 : * than leaving a page on the unevictable LRU indefinitely.
1036 : */
1037 0 : folio->mlock_count = 0;
1038 0 : if (!was_unevictable)
1039 : __count_vm_events(UNEVICTABLE_PGCULLED, nr_pages);
1040 : }
1041 :
1042 0 : lruvec_add_folio(lruvec, folio);
1043 0 : trace_mm_lru_insertion(folio);
1044 0 : }
1045 :
1046 : /*
1047 : * Add the passed pages to the LRU, then drop the caller's refcount
1048 : * on them. Reinitialises the caller's pagevec.
1049 : */
1050 0 : void __pagevec_lru_add(struct pagevec *pvec)
1051 : {
1052 : int i;
1053 0 : struct lruvec *lruvec = NULL;
1054 0 : unsigned long flags = 0;
1055 :
1056 0 : for (i = 0; i < pagevec_count(pvec); i++) {
1057 0 : struct folio *folio = page_folio(pvec->pages[i]);
1058 :
1059 0 : lruvec = folio_lruvec_relock_irqsave(folio, lruvec, &flags);
1060 0 : __pagevec_lru_add_fn(folio, lruvec);
1061 : }
1062 0 : if (lruvec)
1063 0 : unlock_page_lruvec_irqrestore(lruvec, flags);
1064 0 : release_pages(pvec->pages, pvec->nr);
1065 0 : pagevec_reinit(pvec);
1066 0 : }
1067 :
1068 : /**
1069 : * folio_batch_remove_exceptionals() - Prune non-folios from a batch.
1070 : * @fbatch: The batch to prune
1071 : *
1072 : * find_get_entries() fills a batch with both folios and shadow/swap/DAX
1073 : * entries. This function prunes all the non-folio entries from @fbatch
1074 : * without leaving holes, so that it can be passed on to folio-only batch
1075 : * operations.
1076 : */
1077 0 : void folio_batch_remove_exceptionals(struct folio_batch *fbatch)
1078 : {
1079 : unsigned int i, j;
1080 :
1081 0 : for (i = 0, j = 0; i < folio_batch_count(fbatch); i++) {
1082 0 : struct folio *folio = fbatch->folios[i];
1083 0 : if (!xa_is_value(folio))
1084 0 : fbatch->folios[j++] = folio;
1085 : }
1086 0 : fbatch->nr = j;
1087 0 : }
1088 :
1089 : /**
1090 : * pagevec_lookup_range - gang pagecache lookup
1091 : * @pvec: Where the resulting pages are placed
1092 : * @mapping: The address_space to search
1093 : * @start: The starting page index
1094 : * @end: The final page index
1095 : *
1096 : * pagevec_lookup_range() will search for & return a group of up to PAGEVEC_SIZE
1097 : * pages in the mapping starting from index @start and upto index @end
1098 : * (inclusive). The pages are placed in @pvec. pagevec_lookup() takes a
1099 : * reference against the pages in @pvec.
1100 : *
1101 : * The search returns a group of mapping-contiguous pages with ascending
1102 : * indexes. There may be holes in the indices due to not-present pages. We
1103 : * also update @start to index the next page for the traversal.
1104 : *
1105 : * pagevec_lookup_range() returns the number of pages which were found. If this
1106 : * number is smaller than PAGEVEC_SIZE, the end of specified range has been
1107 : * reached.
1108 : */
1109 0 : unsigned pagevec_lookup_range(struct pagevec *pvec,
1110 : struct address_space *mapping, pgoff_t *start, pgoff_t end)
1111 : {
1112 0 : pvec->nr = find_get_pages_range(mapping, start, end, PAGEVEC_SIZE,
1113 0 : pvec->pages);
1114 0 : return pagevec_count(pvec);
1115 : }
1116 : EXPORT_SYMBOL(pagevec_lookup_range);
1117 :
1118 0 : unsigned pagevec_lookup_range_tag(struct pagevec *pvec,
1119 : struct address_space *mapping, pgoff_t *index, pgoff_t end,
1120 : xa_mark_t tag)
1121 : {
1122 0 : pvec->nr = find_get_pages_range_tag(mapping, index, end, tag,
1123 0 : PAGEVEC_SIZE, pvec->pages);
1124 0 : return pagevec_count(pvec);
1125 : }
1126 : EXPORT_SYMBOL(pagevec_lookup_range_tag);
1127 :
1128 : /*
1129 : * Perform any setup for the swap system
1130 : */
1131 1 : void __init swap_setup(void)
1132 : {
1133 1 : unsigned long megs = totalram_pages() >> (20 - PAGE_SHIFT);
1134 :
1135 : /* Use a smaller cluster for small-memory machines */
1136 1 : if (megs < 16)
1137 0 : page_cluster = 2;
1138 : else
1139 1 : page_cluster = 3;
1140 : /*
1141 : * Right now other parts of the system means that we
1142 : * _really_ don't want to cluster much more
1143 : */
1144 1 : }
|
