Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
4 : *
5 : * Swap reorganised 29.12.95, Stephen Tweedie.
6 : * kswapd added: 7.1.96 sct
7 : * Removed kswapd_ctl limits, and swap out as many pages as needed
8 : * to bring the system back to freepages.high: 2.4.97, Rik van Riel.
9 : * Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
10 : * Multiqueue VM started 5.8.00, Rik van Riel.
11 : */
12 :
13 : #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14 :
15 : #include <linux/mm.h>
16 : #include <linux/sched/mm.h>
17 : #include <linux/module.h>
18 : #include <linux/gfp.h>
19 : #include <linux/kernel_stat.h>
20 : #include <linux/swap.h>
21 : #include <linux/pagemap.h>
22 : #include <linux/init.h>
23 : #include <linux/highmem.h>
24 : #include <linux/vmpressure.h>
25 : #include <linux/vmstat.h>
26 : #include <linux/file.h>
27 : #include <linux/writeback.h>
28 : #include <linux/blkdev.h>
29 : #include <linux/buffer_head.h> /* for try_to_release_page(),
30 : buffer_heads_over_limit */
31 : #include <linux/mm_inline.h>
32 : #include <linux/backing-dev.h>
33 : #include <linux/rmap.h>
34 : #include <linux/topology.h>
35 : #include <linux/cpu.h>
36 : #include <linux/cpuset.h>
37 : #include <linux/compaction.h>
38 : #include <linux/notifier.h>
39 : #include <linux/rwsem.h>
40 : #include <linux/delay.h>
41 : #include <linux/kthread.h>
42 : #include <linux/freezer.h>
43 : #include <linux/memcontrol.h>
44 : #include <linux/migrate.h>
45 : #include <linux/delayacct.h>
46 : #include <linux/sysctl.h>
47 : #include <linux/oom.h>
48 : #include <linux/pagevec.h>
49 : #include <linux/prefetch.h>
50 : #include <linux/printk.h>
51 : #include <linux/dax.h>
52 : #include <linux/psi.h>
53 :
54 : #include <asm/tlbflush.h>
55 : #include <asm/div64.h>
56 :
57 : #include <linux/swapops.h>
58 : #include <linux/balloon_compaction.h>
59 : #include <linux/sched/sysctl.h>
60 :
61 : #include "internal.h"
62 :
63 : #define CREATE_TRACE_POINTS
64 : #include <trace/events/vmscan.h>
65 :
66 : struct scan_control {
67 : /* How many pages shrink_list() should reclaim */
68 : unsigned long nr_to_reclaim;
69 :
70 : /*
71 : * Nodemask of nodes allowed by the caller. If NULL, all nodes
72 : * are scanned.
73 : */
74 : nodemask_t *nodemask;
75 :
76 : /*
77 : * The memory cgroup that hit its limit and as a result is the
78 : * primary target of this reclaim invocation.
79 : */
80 : struct mem_cgroup *target_mem_cgroup;
81 :
82 : /*
83 : * Scan pressure balancing between anon and file LRUs
84 : */
85 : unsigned long anon_cost;
86 : unsigned long file_cost;
87 :
88 : /* Can active pages be deactivated as part of reclaim? */
89 : #define DEACTIVATE_ANON 1
90 : #define DEACTIVATE_FILE 2
91 : unsigned int may_deactivate:2;
92 : unsigned int force_deactivate:1;
93 : unsigned int skipped_deactivate:1;
94 :
95 : /* Writepage batching in laptop mode; RECLAIM_WRITE */
96 : unsigned int may_writepage:1;
97 :
98 : /* Can mapped pages be reclaimed? */
99 : unsigned int may_unmap:1;
100 :
101 : /* Can pages be swapped as part of reclaim? */
102 : unsigned int may_swap:1;
103 :
104 : /*
105 : * Cgroup memory below memory.low is protected as long as we
106 : * don't threaten to OOM. If any cgroup is reclaimed at
107 : * reduced force or passed over entirely due to its memory.low
108 : * setting (memcg_low_skipped), and nothing is reclaimed as a
109 : * result, then go back for one more cycle that reclaims the protected
110 : * memory (memcg_low_reclaim) to avert OOM.
111 : */
112 : unsigned int memcg_low_reclaim:1;
113 : unsigned int memcg_low_skipped:1;
114 :
115 : unsigned int hibernation_mode:1;
116 :
117 : /* One of the zones is ready for compaction */
118 : unsigned int compaction_ready:1;
119 :
120 : /* There is easily reclaimable cold cache in the current node */
121 : unsigned int cache_trim_mode:1;
122 :
123 : /* The file pages on the current node are dangerously low */
124 : unsigned int file_is_tiny:1;
125 :
126 : /* Always discard instead of demoting to lower tier memory */
127 : unsigned int no_demotion:1;
128 :
129 : /* Allocation order */
130 : s8 order;
131 :
132 : /* Scan (total_size >> priority) pages at once */
133 : s8 priority;
134 :
135 : /* The highest zone to isolate pages for reclaim from */
136 : s8 reclaim_idx;
137 :
138 : /* This context's GFP mask */
139 : gfp_t gfp_mask;
140 :
141 : /* Incremented by the number of inactive pages that were scanned */
142 : unsigned long nr_scanned;
143 :
144 : /* Number of pages freed so far during a call to shrink_zones() */
145 : unsigned long nr_reclaimed;
146 :
147 : struct {
148 : unsigned int dirty;
149 : unsigned int unqueued_dirty;
150 : unsigned int congested;
151 : unsigned int writeback;
152 : unsigned int immediate;
153 : unsigned int file_taken;
154 : unsigned int taken;
155 : } nr;
156 :
157 : /* for recording the reclaimed slab by now */
158 : struct reclaim_state reclaim_state;
159 : };
160 :
161 : #ifdef ARCH_HAS_PREFETCHW
162 : #define prefetchw_prev_lru_page(_page, _base, _field) \
163 : do { \
164 : if ((_page)->lru.prev != _base) { \
165 : struct page *prev; \
166 : \
167 : prev = lru_to_page(&(_page->lru)); \
168 : prefetchw(&prev->_field); \
169 : } \
170 : } while (0)
171 : #else
172 : #define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
173 : #endif
174 :
175 : /*
176 : * From 0 .. 200. Higher means more swappy.
177 : */
178 : int vm_swappiness = 60;
179 :
180 0 : static void set_task_reclaim_state(struct task_struct *task,
181 : struct reclaim_state *rs)
182 : {
183 : /* Check for an overwrite */
184 0 : WARN_ON_ONCE(rs && task->reclaim_state);
185 :
186 : /* Check for the nulling of an already-nulled member */
187 0 : WARN_ON_ONCE(!rs && !task->reclaim_state);
188 :
189 0 : task->reclaim_state = rs;
190 0 : }
191 :
192 : static LIST_HEAD(shrinker_list);
193 : static DECLARE_RWSEM(shrinker_rwsem);
194 :
195 : #ifdef CONFIG_MEMCG
196 : static int shrinker_nr_max;
197 :
198 : /* The shrinker_info is expanded in a batch of BITS_PER_LONG */
199 : static inline int shrinker_map_size(int nr_items)
200 : {
201 : return (DIV_ROUND_UP(nr_items, BITS_PER_LONG) * sizeof(unsigned long));
202 : }
203 :
204 : static inline int shrinker_defer_size(int nr_items)
205 : {
206 : return (round_up(nr_items, BITS_PER_LONG) * sizeof(atomic_long_t));
207 : }
208 :
209 : static struct shrinker_info *shrinker_info_protected(struct mem_cgroup *memcg,
210 : int nid)
211 : {
212 : return rcu_dereference_protected(memcg->nodeinfo[nid]->shrinker_info,
213 : lockdep_is_held(&shrinker_rwsem));
214 : }
215 :
216 : static int expand_one_shrinker_info(struct mem_cgroup *memcg,
217 : int map_size, int defer_size,
218 : int old_map_size, int old_defer_size)
219 : {
220 : struct shrinker_info *new, *old;
221 : struct mem_cgroup_per_node *pn;
222 : int nid;
223 : int size = map_size + defer_size;
224 :
225 : for_each_node(nid) {
226 : pn = memcg->nodeinfo[nid];
227 : old = shrinker_info_protected(memcg, nid);
228 : /* Not yet online memcg */
229 : if (!old)
230 : return 0;
231 :
232 : new = kvmalloc_node(sizeof(*new) + size, GFP_KERNEL, nid);
233 : if (!new)
234 : return -ENOMEM;
235 :
236 : new->nr_deferred = (atomic_long_t *)(new + 1);
237 : new->map = (void *)new->nr_deferred + defer_size;
238 :
239 : /* map: set all old bits, clear all new bits */
240 : memset(new->map, (int)0xff, old_map_size);
241 : memset((void *)new->map + old_map_size, 0, map_size - old_map_size);
242 : /* nr_deferred: copy old values, clear all new values */
243 : memcpy(new->nr_deferred, old->nr_deferred, old_defer_size);
244 : memset((void *)new->nr_deferred + old_defer_size, 0,
245 : defer_size - old_defer_size);
246 :
247 : rcu_assign_pointer(pn->shrinker_info, new);
248 : kvfree_rcu(old, rcu);
249 : }
250 :
251 : return 0;
252 : }
253 :
254 : void free_shrinker_info(struct mem_cgroup *memcg)
255 : {
256 : struct mem_cgroup_per_node *pn;
257 : struct shrinker_info *info;
258 : int nid;
259 :
260 : for_each_node(nid) {
261 : pn = memcg->nodeinfo[nid];
262 : info = rcu_dereference_protected(pn->shrinker_info, true);
263 : kvfree(info);
264 : rcu_assign_pointer(pn->shrinker_info, NULL);
265 : }
266 : }
267 :
268 : int alloc_shrinker_info(struct mem_cgroup *memcg)
269 : {
270 : struct shrinker_info *info;
271 : int nid, size, ret = 0;
272 : int map_size, defer_size = 0;
273 :
274 : down_write(&shrinker_rwsem);
275 : map_size = shrinker_map_size(shrinker_nr_max);
276 : defer_size = shrinker_defer_size(shrinker_nr_max);
277 : size = map_size + defer_size;
278 : for_each_node(nid) {
279 : info = kvzalloc_node(sizeof(*info) + size, GFP_KERNEL, nid);
280 : if (!info) {
281 : free_shrinker_info(memcg);
282 : ret = -ENOMEM;
283 : break;
284 : }
285 : info->nr_deferred = (atomic_long_t *)(info + 1);
286 : info->map = (void *)info->nr_deferred + defer_size;
287 : rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_info, info);
288 : }
289 : up_write(&shrinker_rwsem);
290 :
291 : return ret;
292 : }
293 :
294 : static inline bool need_expand(int nr_max)
295 : {
296 : return round_up(nr_max, BITS_PER_LONG) >
297 : round_up(shrinker_nr_max, BITS_PER_LONG);
298 : }
299 :
300 : static int expand_shrinker_info(int new_id)
301 : {
302 : int ret = 0;
303 : int new_nr_max = new_id + 1;
304 : int map_size, defer_size = 0;
305 : int old_map_size, old_defer_size = 0;
306 : struct mem_cgroup *memcg;
307 :
308 : if (!need_expand(new_nr_max))
309 : goto out;
310 :
311 : if (!root_mem_cgroup)
312 : goto out;
313 :
314 : lockdep_assert_held(&shrinker_rwsem);
315 :
316 : map_size = shrinker_map_size(new_nr_max);
317 : defer_size = shrinker_defer_size(new_nr_max);
318 : old_map_size = shrinker_map_size(shrinker_nr_max);
319 : old_defer_size = shrinker_defer_size(shrinker_nr_max);
320 :
321 : memcg = mem_cgroup_iter(NULL, NULL, NULL);
322 : do {
323 : ret = expand_one_shrinker_info(memcg, map_size, defer_size,
324 : old_map_size, old_defer_size);
325 : if (ret) {
326 : mem_cgroup_iter_break(NULL, memcg);
327 : goto out;
328 : }
329 : } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);
330 : out:
331 : if (!ret)
332 : shrinker_nr_max = new_nr_max;
333 :
334 : return ret;
335 : }
336 :
337 : void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
338 : {
339 : if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {
340 : struct shrinker_info *info;
341 :
342 : rcu_read_lock();
343 : info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);
344 : /* Pairs with smp mb in shrink_slab() */
345 : smp_mb__before_atomic();
346 : set_bit(shrinker_id, info->map);
347 : rcu_read_unlock();
348 : }
349 : }
350 :
351 : static DEFINE_IDR(shrinker_idr);
352 :
353 : static int prealloc_memcg_shrinker(struct shrinker *shrinker)
354 : {
355 : int id, ret = -ENOMEM;
356 :
357 : if (mem_cgroup_disabled())
358 : return -ENOSYS;
359 :
360 : down_write(&shrinker_rwsem);
361 : /* This may call shrinker, so it must use down_read_trylock() */
362 : id = idr_alloc(&shrinker_idr, shrinker, 0, 0, GFP_KERNEL);
363 : if (id < 0)
364 : goto unlock;
365 :
366 : if (id >= shrinker_nr_max) {
367 : if (expand_shrinker_info(id)) {
368 : idr_remove(&shrinker_idr, id);
369 : goto unlock;
370 : }
371 : }
372 : shrinker->id = id;
373 : ret = 0;
374 : unlock:
375 : up_write(&shrinker_rwsem);
376 : return ret;
377 : }
378 :
379 : static void unregister_memcg_shrinker(struct shrinker *shrinker)
380 : {
381 : int id = shrinker->id;
382 :
383 : BUG_ON(id < 0);
384 :
385 : lockdep_assert_held(&shrinker_rwsem);
386 :
387 : idr_remove(&shrinker_idr, id);
388 : }
389 :
390 : static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker,
391 : struct mem_cgroup *memcg)
392 : {
393 : struct shrinker_info *info;
394 :
395 : info = shrinker_info_protected(memcg, nid);
396 : return atomic_long_xchg(&info->nr_deferred[shrinker->id], 0);
397 : }
398 :
399 : static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker,
400 : struct mem_cgroup *memcg)
401 : {
402 : struct shrinker_info *info;
403 :
404 : info = shrinker_info_protected(memcg, nid);
405 : return atomic_long_add_return(nr, &info->nr_deferred[shrinker->id]);
406 : }
407 :
408 : void reparent_shrinker_deferred(struct mem_cgroup *memcg)
409 : {
410 : int i, nid;
411 : long nr;
412 : struct mem_cgroup *parent;
413 : struct shrinker_info *child_info, *parent_info;
414 :
415 : parent = parent_mem_cgroup(memcg);
416 : if (!parent)
417 : parent = root_mem_cgroup;
418 :
419 : /* Prevent from concurrent shrinker_info expand */
420 : down_read(&shrinker_rwsem);
421 : for_each_node(nid) {
422 : child_info = shrinker_info_protected(memcg, nid);
423 : parent_info = shrinker_info_protected(parent, nid);
424 : for (i = 0; i < shrinker_nr_max; i++) {
425 : nr = atomic_long_read(&child_info->nr_deferred[i]);
426 : atomic_long_add(nr, &parent_info->nr_deferred[i]);
427 : }
428 : }
429 : up_read(&shrinker_rwsem);
430 : }
431 :
432 : static bool cgroup_reclaim(struct scan_control *sc)
433 : {
434 : return sc->target_mem_cgroup;
435 : }
436 :
437 : /**
438 : * writeback_throttling_sane - is the usual dirty throttling mechanism available?
439 : * @sc: scan_control in question
440 : *
441 : * The normal page dirty throttling mechanism in balance_dirty_pages() is
442 : * completely broken with the legacy memcg and direct stalling in
443 : * shrink_page_list() is used for throttling instead, which lacks all the
444 : * niceties such as fairness, adaptive pausing, bandwidth proportional
445 : * allocation and configurability.
446 : *
447 : * This function tests whether the vmscan currently in progress can assume
448 : * that the normal dirty throttling mechanism is operational.
449 : */
450 : static bool writeback_throttling_sane(struct scan_control *sc)
451 : {
452 : if (!cgroup_reclaim(sc))
453 : return true;
454 : #ifdef CONFIG_CGROUP_WRITEBACK
455 : if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
456 : return true;
457 : #endif
458 : return false;
459 : }
460 : #else
461 : static int prealloc_memcg_shrinker(struct shrinker *shrinker)
462 : {
463 : return -ENOSYS;
464 : }
465 :
466 : static void unregister_memcg_shrinker(struct shrinker *shrinker)
467 : {
468 : }
469 :
470 : static long xchg_nr_deferred_memcg(int nid, struct shrinker *shrinker,
471 : struct mem_cgroup *memcg)
472 : {
473 : return 0;
474 : }
475 :
476 : static long add_nr_deferred_memcg(long nr, int nid, struct shrinker *shrinker,
477 : struct mem_cgroup *memcg)
478 : {
479 : return 0;
480 : }
481 :
482 : static bool cgroup_reclaim(struct scan_control *sc)
483 : {
484 : return false;
485 : }
486 :
487 : static bool writeback_throttling_sane(struct scan_control *sc)
488 : {
489 : return true;
490 : }
491 : #endif
492 :
493 : static long xchg_nr_deferred(struct shrinker *shrinker,
494 : struct shrink_control *sc)
495 : {
496 0 : int nid = sc->nid;
497 :
498 0 : if (!(shrinker->flags & SHRINKER_NUMA_AWARE))
499 0 : nid = 0;
500 :
501 0 : if (sc->memcg &&
502 0 : (shrinker->flags & SHRINKER_MEMCG_AWARE))
503 : return xchg_nr_deferred_memcg(nid, shrinker,
504 : sc->memcg);
505 :
506 0 : return atomic_long_xchg(&shrinker->nr_deferred[nid], 0);
507 : }
508 :
509 :
510 : static long add_nr_deferred(long nr, struct shrinker *shrinker,
511 : struct shrink_control *sc)
512 : {
513 0 : int nid = sc->nid;
514 :
515 0 : if (!(shrinker->flags & SHRINKER_NUMA_AWARE))
516 0 : nid = 0;
517 :
518 0 : if (sc->memcg &&
519 0 : (shrinker->flags & SHRINKER_MEMCG_AWARE))
520 : return add_nr_deferred_memcg(nr, nid, shrinker,
521 : sc->memcg);
522 :
523 0 : return atomic_long_add_return(nr, &shrinker->nr_deferred[nid]);
524 : }
525 :
526 0 : static bool can_demote(int nid, struct scan_control *sc)
527 : {
528 0 : if (!numa_demotion_enabled)
529 : return false;
530 0 : if (sc) {
531 0 : if (sc->no_demotion)
532 : return false;
533 : /* It is pointless to do demotion in memcg reclaim */
534 : if (cgroup_reclaim(sc))
535 : return false;
536 : }
537 0 : if (next_demotion_node(nid) == NUMA_NO_NODE)
538 : return false;
539 :
540 0 : return true;
541 : }
542 :
543 : static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg,
544 : int nid,
545 : struct scan_control *sc)
546 : {
547 : if (memcg == NULL) {
548 : /*
549 : * For non-memcg reclaim, is there
550 : * space in any swap device?
551 : */
552 0 : if (get_nr_swap_pages() > 0)
553 : return true;
554 : } else {
555 : /* Is the memcg below its swap limit? */
556 : if (mem_cgroup_get_nr_swap_pages(memcg) > 0)
557 : return true;
558 : }
559 :
560 : /*
561 : * The page can not be swapped.
562 : *
563 : * Can it be reclaimed from this node via demotion?
564 : */
565 0 : return can_demote(nid, sc);
566 : }
567 :
568 : /*
569 : * This misses isolated pages which are not accounted for to save counters.
570 : * As the data only determines if reclaim or compaction continues, it is
571 : * not expected that isolated pages will be a dominating factor.
572 : */
573 0 : unsigned long zone_reclaimable_pages(struct zone *zone)
574 : {
575 : unsigned long nr;
576 :
577 0 : nr = zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_FILE) +
578 0 : zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_FILE);
579 0 : if (can_reclaim_anon_pages(NULL, zone_to_nid(zone), NULL))
580 0 : nr += zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_ANON) +
581 0 : zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_ANON);
582 :
583 0 : return nr;
584 : }
585 :
586 : /**
587 : * lruvec_lru_size - Returns the number of pages on the given LRU list.
588 : * @lruvec: lru vector
589 : * @lru: lru to use
590 : * @zone_idx: zones to consider (use MAX_NR_ZONES for the whole LRU list)
591 : */
592 : static unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru,
593 : int zone_idx)
594 : {
595 : unsigned long size = 0;
596 : int zid;
597 :
598 0 : for (zid = 0; zid <= zone_idx && zid < MAX_NR_ZONES; zid++) {
599 0 : struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid];
600 :
601 0 : if (!managed_zone(zone))
602 0 : continue;
603 :
604 : if (!mem_cgroup_disabled())
605 : size += mem_cgroup_get_zone_lru_size(lruvec, lru, zid);
606 : else
607 0 : size += zone_page_state(zone, NR_ZONE_LRU_BASE + lru);
608 : }
609 : return size;
610 : }
611 :
612 : /*
613 : * Add a shrinker callback to be called from the vm.
614 : */
615 11 : int prealloc_shrinker(struct shrinker *shrinker)
616 : {
617 : unsigned int size;
618 : int err;
619 :
620 11 : if (shrinker->flags & SHRINKER_MEMCG_AWARE) {
621 11 : err = prealloc_memcg_shrinker(shrinker);
622 : if (err != -ENOSYS)
623 : return err;
624 :
625 11 : shrinker->flags &= ~SHRINKER_MEMCG_AWARE;
626 : }
627 :
628 11 : size = sizeof(*shrinker->nr_deferred);
629 : if (shrinker->flags & SHRINKER_NUMA_AWARE)
630 : size *= nr_node_ids;
631 :
632 11 : shrinker->nr_deferred = kzalloc(size, GFP_KERNEL);
633 11 : if (!shrinker->nr_deferred)
634 : return -ENOMEM;
635 :
636 11 : return 0;
637 : }
638 :
639 0 : void free_prealloced_shrinker(struct shrinker *shrinker)
640 : {
641 0 : if (shrinker->flags & SHRINKER_MEMCG_AWARE) {
642 0 : down_write(&shrinker_rwsem);
643 0 : unregister_memcg_shrinker(shrinker);
644 0 : up_write(&shrinker_rwsem);
645 0 : return;
646 : }
647 :
648 0 : kfree(shrinker->nr_deferred);
649 0 : shrinker->nr_deferred = NULL;
650 : }
651 :
652 11 : void register_shrinker_prepared(struct shrinker *shrinker)
653 : {
654 11 : down_write(&shrinker_rwsem);
655 22 : list_add_tail(&shrinker->list, &shrinker_list);
656 11 : shrinker->flags |= SHRINKER_REGISTERED;
657 11 : up_write(&shrinker_rwsem);
658 11 : }
659 :
660 0 : int register_shrinker(struct shrinker *shrinker)
661 : {
662 0 : int err = prealloc_shrinker(shrinker);
663 :
664 0 : if (err)
665 : return err;
666 0 : register_shrinker_prepared(shrinker);
667 0 : return 0;
668 : }
669 : EXPORT_SYMBOL(register_shrinker);
670 :
671 : /*
672 : * Remove one
673 : */
674 0 : void unregister_shrinker(struct shrinker *shrinker)
675 : {
676 0 : if (!(shrinker->flags & SHRINKER_REGISTERED))
677 : return;
678 :
679 0 : down_write(&shrinker_rwsem);
680 0 : list_del(&shrinker->list);
681 0 : shrinker->flags &= ~SHRINKER_REGISTERED;
682 : if (shrinker->flags & SHRINKER_MEMCG_AWARE)
683 : unregister_memcg_shrinker(shrinker);
684 0 : up_write(&shrinker_rwsem);
685 :
686 0 : kfree(shrinker->nr_deferred);
687 0 : shrinker->nr_deferred = NULL;
688 : }
689 : EXPORT_SYMBOL(unregister_shrinker);
690 :
691 : /**
692 : * synchronize_shrinkers - Wait for all running shrinkers to complete.
693 : *
694 : * This is equivalent to calling unregister_shrink() and register_shrinker(),
695 : * but atomically and with less overhead. This is useful to guarantee that all
696 : * shrinker invocations have seen an update, before freeing memory, similar to
697 : * rcu.
698 : */
699 0 : void synchronize_shrinkers(void)
700 : {
701 0 : down_write(&shrinker_rwsem);
702 0 : up_write(&shrinker_rwsem);
703 0 : }
704 : EXPORT_SYMBOL(synchronize_shrinkers);
705 :
706 : #define SHRINK_BATCH 128
707 :
708 0 : static unsigned long do_shrink_slab(struct shrink_control *shrinkctl,
709 : struct shrinker *shrinker, int priority)
710 : {
711 0 : unsigned long freed = 0;
712 : unsigned long long delta;
713 : long total_scan;
714 : long freeable;
715 : long nr;
716 : long new_nr;
717 0 : long batch_size = shrinker->batch ? shrinker->batch
718 0 : : SHRINK_BATCH;
719 0 : long scanned = 0, next_deferred;
720 :
721 0 : freeable = shrinker->count_objects(shrinker, shrinkctl);
722 0 : if (freeable == 0 || freeable == SHRINK_EMPTY)
723 : return freeable;
724 :
725 : /*
726 : * copy the current shrinker scan count into a local variable
727 : * and zero it so that other concurrent shrinker invocations
728 : * don't also do this scanning work.
729 : */
730 0 : nr = xchg_nr_deferred(shrinker, shrinkctl);
731 :
732 0 : if (shrinker->seeks) {
733 0 : delta = freeable >> priority;
734 0 : delta *= 4;
735 0 : do_div(delta, shrinker->seeks);
736 : } else {
737 : /*
738 : * These objects don't require any IO to create. Trim
739 : * them aggressively under memory pressure to keep
740 : * them from causing refetches in the IO caches.
741 : */
742 0 : delta = freeable / 2;
743 : }
744 :
745 0 : total_scan = nr >> priority;
746 0 : total_scan += delta;
747 0 : total_scan = min(total_scan, (2 * freeable));
748 :
749 0 : trace_mm_shrink_slab_start(shrinker, shrinkctl, nr,
750 : freeable, delta, total_scan, priority);
751 :
752 : /*
753 : * Normally, we should not scan less than batch_size objects in one
754 : * pass to avoid too frequent shrinker calls, but if the slab has less
755 : * than batch_size objects in total and we are really tight on memory,
756 : * we will try to reclaim all available objects, otherwise we can end
757 : * up failing allocations although there are plenty of reclaimable
758 : * objects spread over several slabs with usage less than the
759 : * batch_size.
760 : *
761 : * We detect the "tight on memory" situations by looking at the total
762 : * number of objects we want to scan (total_scan). If it is greater
763 : * than the total number of objects on slab (freeable), we must be
764 : * scanning at high prio and therefore should try to reclaim as much as
765 : * possible.
766 : */
767 0 : while (total_scan >= batch_size ||
768 0 : total_scan >= freeable) {
769 : unsigned long ret;
770 0 : unsigned long nr_to_scan = min(batch_size, total_scan);
771 :
772 0 : shrinkctl->nr_to_scan = nr_to_scan;
773 0 : shrinkctl->nr_scanned = nr_to_scan;
774 0 : ret = shrinker->scan_objects(shrinker, shrinkctl);
775 0 : if (ret == SHRINK_STOP)
776 : break;
777 0 : freed += ret;
778 :
779 0 : count_vm_events(SLABS_SCANNED, shrinkctl->nr_scanned);
780 0 : total_scan -= shrinkctl->nr_scanned;
781 0 : scanned += shrinkctl->nr_scanned;
782 :
783 0 : cond_resched();
784 : }
785 :
786 : /*
787 : * The deferred work is increased by any new work (delta) that wasn't
788 : * done, decreased by old deferred work that was done now.
789 : *
790 : * And it is capped to two times of the freeable items.
791 : */
792 0 : next_deferred = max_t(long, (nr + delta - scanned), 0);
793 0 : next_deferred = min(next_deferred, (2 * freeable));
794 :
795 : /*
796 : * move the unused scan count back into the shrinker in a
797 : * manner that handles concurrent updates.
798 : */
799 0 : new_nr = add_nr_deferred(next_deferred, shrinker, shrinkctl);
800 :
801 0 : trace_mm_shrink_slab_end(shrinker, shrinkctl->nid, freed, nr, new_nr, total_scan);
802 0 : return freed;
803 : }
804 :
805 : #ifdef CONFIG_MEMCG
806 : static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
807 : struct mem_cgroup *memcg, int priority)
808 : {
809 : struct shrinker_info *info;
810 : unsigned long ret, freed = 0;
811 : int i;
812 :
813 : if (!mem_cgroup_online(memcg))
814 : return 0;
815 :
816 : if (!down_read_trylock(&shrinker_rwsem))
817 : return 0;
818 :
819 : info = shrinker_info_protected(memcg, nid);
820 : if (unlikely(!info))
821 : goto unlock;
822 :
823 : for_each_set_bit(i, info->map, shrinker_nr_max) {
824 : struct shrink_control sc = {
825 : .gfp_mask = gfp_mask,
826 : .nid = nid,
827 : .memcg = memcg,
828 : };
829 : struct shrinker *shrinker;
830 :
831 : shrinker = idr_find(&shrinker_idr, i);
832 : if (unlikely(!shrinker || !(shrinker->flags & SHRINKER_REGISTERED))) {
833 : if (!shrinker)
834 : clear_bit(i, info->map);
835 : continue;
836 : }
837 :
838 : /* Call non-slab shrinkers even though kmem is disabled */
839 : if (!memcg_kmem_enabled() &&
840 : !(shrinker->flags & SHRINKER_NONSLAB))
841 : continue;
842 :
843 : ret = do_shrink_slab(&sc, shrinker, priority);
844 : if (ret == SHRINK_EMPTY) {
845 : clear_bit(i, info->map);
846 : /*
847 : * After the shrinker reported that it had no objects to
848 : * free, but before we cleared the corresponding bit in
849 : * the memcg shrinker map, a new object might have been
850 : * added. To make sure, we have the bit set in this
851 : * case, we invoke the shrinker one more time and reset
852 : * the bit if it reports that it is not empty anymore.
853 : * The memory barrier here pairs with the barrier in
854 : * set_shrinker_bit():
855 : *
856 : * list_lru_add() shrink_slab_memcg()
857 : * list_add_tail() clear_bit()
858 : * <MB> <MB>
859 : * set_bit() do_shrink_slab()
860 : */
861 : smp_mb__after_atomic();
862 : ret = do_shrink_slab(&sc, shrinker, priority);
863 : if (ret == SHRINK_EMPTY)
864 : ret = 0;
865 : else
866 : set_shrinker_bit(memcg, nid, i);
867 : }
868 : freed += ret;
869 :
870 : if (rwsem_is_contended(&shrinker_rwsem)) {
871 : freed = freed ? : 1;
872 : break;
873 : }
874 : }
875 : unlock:
876 : up_read(&shrinker_rwsem);
877 : return freed;
878 : }
879 : #else /* CONFIG_MEMCG */
880 : static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
881 : struct mem_cgroup *memcg, int priority)
882 : {
883 : return 0;
884 : }
885 : #endif /* CONFIG_MEMCG */
886 :
887 : /**
888 : * shrink_slab - shrink slab caches
889 : * @gfp_mask: allocation context
890 : * @nid: node whose slab caches to target
891 : * @memcg: memory cgroup whose slab caches to target
892 : * @priority: the reclaim priority
893 : *
894 : * Call the shrink functions to age shrinkable caches.
895 : *
896 : * @nid is passed along to shrinkers with SHRINKER_NUMA_AWARE set,
897 : * unaware shrinkers will receive a node id of 0 instead.
898 : *
899 : * @memcg specifies the memory cgroup to target. Unaware shrinkers
900 : * are called only if it is the root cgroup.
901 : *
902 : * @priority is sc->priority, we take the number of objects and >> by priority
903 : * in order to get the scan target.
904 : *
905 : * Returns the number of reclaimed slab objects.
906 : */
907 0 : static unsigned long shrink_slab(gfp_t gfp_mask, int nid,
908 : struct mem_cgroup *memcg,
909 : int priority)
910 : {
911 0 : unsigned long ret, freed = 0;
912 : struct shrinker *shrinker;
913 :
914 : /*
915 : * The root memcg might be allocated even though memcg is disabled
916 : * via "cgroup_disable=memory" boot parameter. This could make
917 : * mem_cgroup_is_root() return false, then just run memcg slab
918 : * shrink, but skip global shrink. This may result in premature
919 : * oom.
920 : */
921 : if (!mem_cgroup_disabled() && !mem_cgroup_is_root(memcg))
922 : return shrink_slab_memcg(gfp_mask, nid, memcg, priority);
923 :
924 0 : if (!down_read_trylock(&shrinker_rwsem))
925 : goto out;
926 :
927 0 : list_for_each_entry(shrinker, &shrinker_list, list) {
928 0 : struct shrink_control sc = {
929 : .gfp_mask = gfp_mask,
930 : .nid = nid,
931 : .memcg = memcg,
932 : };
933 :
934 0 : ret = do_shrink_slab(&sc, shrinker, priority);
935 0 : if (ret == SHRINK_EMPTY)
936 0 : ret = 0;
937 0 : freed += ret;
938 : /*
939 : * Bail out if someone want to register a new shrinker to
940 : * prevent the registration from being stalled for long periods
941 : * by parallel ongoing shrinking.
942 : */
943 0 : if (rwsem_is_contended(&shrinker_rwsem)) {
944 0 : freed = freed ? : 1;
945 0 : break;
946 : }
947 : }
948 :
949 0 : up_read(&shrinker_rwsem);
950 : out:
951 0 : cond_resched();
952 : return freed;
953 : }
954 :
955 0 : static void drop_slab_node(int nid)
956 : {
957 : unsigned long freed;
958 0 : int shift = 0;
959 :
960 : do {
961 0 : struct mem_cgroup *memcg = NULL;
962 :
963 0 : if (fatal_signal_pending(current))
964 : return;
965 :
966 0 : freed = 0;
967 0 : memcg = mem_cgroup_iter(NULL, NULL, NULL);
968 : do {
969 0 : freed += shrink_slab(GFP_KERNEL, nid, memcg, 0);
970 0 : } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);
971 0 : } while ((freed >> shift++) > 1);
972 : }
973 :
974 0 : void drop_slab(void)
975 : {
976 : int nid;
977 :
978 0 : for_each_online_node(nid)
979 0 : drop_slab_node(nid);
980 0 : }
981 :
982 : static inline int is_page_cache_freeable(struct folio *folio)
983 : {
984 : /*
985 : * A freeable page cache page is referenced only by the caller
986 : * that isolated the page, the page cache and optional buffer
987 : * heads at page->private.
988 : */
989 0 : return folio_ref_count(folio) - folio_test_private(folio) ==
990 0 : 1 + folio_nr_pages(folio);
991 : }
992 :
993 : /*
994 : * We detected a synchronous write error writing a folio out. Probably
995 : * -ENOSPC. We need to propagate that into the address_space for a subsequent
996 : * fsync(), msync() or close().
997 : *
998 : * The tricky part is that after writepage we cannot touch the mapping: nothing
999 : * prevents it from being freed up. But we have a ref on the folio and once
1000 : * that folio is locked, the mapping is pinned.
1001 : *
1002 : * We're allowed to run sleeping folio_lock() here because we know the caller has
1003 : * __GFP_FS.
1004 : */
1005 0 : static void handle_write_error(struct address_space *mapping,
1006 : struct folio *folio, int error)
1007 : {
1008 0 : folio_lock(folio);
1009 0 : if (folio_mapping(folio) == mapping)
1010 0 : mapping_set_error(mapping, error);
1011 0 : folio_unlock(folio);
1012 0 : }
1013 :
1014 0 : static bool skip_throttle_noprogress(pg_data_t *pgdat)
1015 : {
1016 0 : int reclaimable = 0, write_pending = 0;
1017 : int i;
1018 :
1019 : /*
1020 : * If kswapd is disabled, reschedule if necessary but do not
1021 : * throttle as the system is likely near OOM.
1022 : */
1023 0 : if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
1024 : return true;
1025 :
1026 : /*
1027 : * If there are a lot of dirty/writeback pages then do not
1028 : * throttle as throttling will occur when the pages cycle
1029 : * towards the end of the LRU if still under writeback.
1030 : */
1031 0 : for (i = 0; i < MAX_NR_ZONES; i++) {
1032 0 : struct zone *zone = pgdat->node_zones + i;
1033 :
1034 0 : if (!populated_zone(zone))
1035 0 : continue;
1036 :
1037 0 : reclaimable += zone_reclaimable_pages(zone);
1038 0 : write_pending += zone_page_state_snapshot(zone,
1039 : NR_ZONE_WRITE_PENDING);
1040 : }
1041 0 : if (2 * write_pending <= reclaimable)
1042 : return true;
1043 :
1044 0 : return false;
1045 : }
1046 :
1047 0 : void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason)
1048 : {
1049 0 : wait_queue_head_t *wqh = &pgdat->reclaim_wait[reason];
1050 : long timeout, ret;
1051 0 : DEFINE_WAIT(wait);
1052 :
1053 : /*
1054 : * Do not throttle IO workers, kthreads other than kswapd or
1055 : * workqueues. They may be required for reclaim to make
1056 : * forward progress (e.g. journalling workqueues or kthreads).
1057 : */
1058 0 : if (!current_is_kswapd() &&
1059 0 : current->flags & (PF_IO_WORKER|PF_KTHREAD)) {
1060 0 : cond_resched();
1061 0 : return;
1062 : }
1063 :
1064 : /*
1065 : * These figures are pulled out of thin air.
1066 : * VMSCAN_THROTTLE_ISOLATED is a transient condition based on too many
1067 : * parallel reclaimers which is a short-lived event so the timeout is
1068 : * short. Failing to make progress or waiting on writeback are
1069 : * potentially long-lived events so use a longer timeout. This is shaky
1070 : * logic as a failure to make progress could be due to anything from
1071 : * writeback to a slow device to excessive references pages at the tail
1072 : * of the inactive LRU.
1073 : */
1074 0 : switch(reason) {
1075 : case VMSCAN_THROTTLE_WRITEBACK:
1076 0 : timeout = HZ/10;
1077 :
1078 0 : if (atomic_inc_return(&pgdat->nr_writeback_throttled) == 1) {
1079 0 : WRITE_ONCE(pgdat->nr_reclaim_start,
1080 : node_page_state(pgdat, NR_THROTTLED_WRITTEN));
1081 : }
1082 :
1083 : break;
1084 : case VMSCAN_THROTTLE_CONGESTED:
1085 : fallthrough;
1086 : case VMSCAN_THROTTLE_NOPROGRESS:
1087 0 : if (skip_throttle_noprogress(pgdat)) {
1088 0 : cond_resched();
1089 0 : return;
1090 : }
1091 :
1092 : timeout = 1;
1093 :
1094 : break;
1095 : case VMSCAN_THROTTLE_ISOLATED:
1096 : timeout = HZ/50;
1097 : break;
1098 : default:
1099 0 : WARN_ON_ONCE(1);
1100 : timeout = HZ;
1101 : break;
1102 : }
1103 :
1104 0 : prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
1105 0 : ret = schedule_timeout(timeout);
1106 0 : finish_wait(wqh, &wait);
1107 :
1108 0 : if (reason == VMSCAN_THROTTLE_WRITEBACK)
1109 0 : atomic_dec(&pgdat->nr_writeback_throttled);
1110 :
1111 0 : trace_mm_vmscan_throttled(pgdat->node_id, jiffies_to_usecs(timeout),
1112 0 : jiffies_to_usecs(timeout - ret),
1113 : reason);
1114 : }
1115 :
1116 : /*
1117 : * Account for pages written if tasks are throttled waiting on dirty
1118 : * pages to clean. If enough pages have been cleaned since throttling
1119 : * started then wakeup the throttled tasks.
1120 : */
1121 0 : void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio,
1122 : int nr_throttled)
1123 : {
1124 : unsigned long nr_written;
1125 :
1126 0 : node_stat_add_folio(folio, NR_THROTTLED_WRITTEN);
1127 :
1128 : /*
1129 : * This is an inaccurate read as the per-cpu deltas may not
1130 : * be synchronised. However, given that the system is
1131 : * writeback throttled, it is not worth taking the penalty
1132 : * of getting an accurate count. At worst, the throttle
1133 : * timeout guarantees forward progress.
1134 : */
1135 0 : nr_written = node_page_state(pgdat, NR_THROTTLED_WRITTEN) -
1136 0 : READ_ONCE(pgdat->nr_reclaim_start);
1137 :
1138 0 : if (nr_written > SWAP_CLUSTER_MAX * nr_throttled)
1139 0 : wake_up(&pgdat->reclaim_wait[VMSCAN_THROTTLE_WRITEBACK]);
1140 0 : }
1141 :
1142 : /* possible outcome of pageout() */
1143 : typedef enum {
1144 : /* failed to write page out, page is locked */
1145 : PAGE_KEEP,
1146 : /* move page to the active list, page is locked */
1147 : PAGE_ACTIVATE,
1148 : /* page has been sent to the disk successfully, page is unlocked */
1149 : PAGE_SUCCESS,
1150 : /* page is clean and locked */
1151 : PAGE_CLEAN,
1152 : } pageout_t;
1153 :
1154 : /*
1155 : * pageout is called by shrink_page_list() for each dirty page.
1156 : * Calls ->writepage().
1157 : */
1158 0 : static pageout_t pageout(struct folio *folio, struct address_space *mapping)
1159 : {
1160 : /*
1161 : * If the folio is dirty, only perform writeback if that write
1162 : * will be non-blocking. To prevent this allocation from being
1163 : * stalled by pagecache activity. But note that there may be
1164 : * stalls if we need to run get_block(). We could test
1165 : * PagePrivate for that.
1166 : *
1167 : * If this process is currently in __generic_file_write_iter() against
1168 : * this folio's queue, we can perform writeback even if that
1169 : * will block.
1170 : *
1171 : * If the folio is swapcache, write it back even if that would
1172 : * block, for some throttling. This happens by accident, because
1173 : * swap_backing_dev_info is bust: it doesn't reflect the
1174 : * congestion state of the swapdevs. Easy to fix, if needed.
1175 : */
1176 0 : if (!is_page_cache_freeable(folio))
1177 : return PAGE_KEEP;
1178 0 : if (!mapping) {
1179 : /*
1180 : * Some data journaling orphaned folios can have
1181 : * folio->mapping == NULL while being dirty with clean buffers.
1182 : */
1183 0 : if (folio_test_private(folio)) {
1184 0 : if (try_to_free_buffers(&folio->page)) {
1185 0 : folio_clear_dirty(folio);
1186 0 : pr_info("%s: orphaned folio\n", __func__);
1187 0 : return PAGE_CLEAN;
1188 : }
1189 : }
1190 : return PAGE_KEEP;
1191 : }
1192 0 : if (mapping->a_ops->writepage == NULL)
1193 : return PAGE_ACTIVATE;
1194 :
1195 0 : if (folio_clear_dirty_for_io(folio)) {
1196 : int res;
1197 0 : struct writeback_control wbc = {
1198 : .sync_mode = WB_SYNC_NONE,
1199 : .nr_to_write = SWAP_CLUSTER_MAX,
1200 : .range_start = 0,
1201 : .range_end = LLONG_MAX,
1202 : .for_reclaim = 1,
1203 : };
1204 :
1205 0 : folio_set_reclaim(folio);
1206 0 : res = mapping->a_ops->writepage(&folio->page, &wbc);
1207 0 : if (res < 0)
1208 0 : handle_write_error(mapping, folio, res);
1209 0 : if (res == AOP_WRITEPAGE_ACTIVATE) {
1210 0 : folio_clear_reclaim(folio);
1211 0 : return PAGE_ACTIVATE;
1212 : }
1213 :
1214 0 : if (!folio_test_writeback(folio)) {
1215 : /* synchronous write or broken a_ops? */
1216 : folio_clear_reclaim(folio);
1217 : }
1218 0 : trace_mm_vmscan_write_folio(folio);
1219 0 : node_stat_add_folio(folio, NR_VMSCAN_WRITE);
1220 0 : return PAGE_SUCCESS;
1221 : }
1222 :
1223 : return PAGE_CLEAN;
1224 : }
1225 :
1226 : /*
1227 : * Same as remove_mapping, but if the page is removed from the mapping, it
1228 : * gets returned with a refcount of 0.
1229 : */
1230 0 : static int __remove_mapping(struct address_space *mapping, struct folio *folio,
1231 : bool reclaimed, struct mem_cgroup *target_memcg)
1232 : {
1233 : int refcount;
1234 0 : void *shadow = NULL;
1235 :
1236 0 : BUG_ON(!folio_test_locked(folio));
1237 0 : BUG_ON(mapping != folio_mapping(folio));
1238 :
1239 0 : if (!folio_test_swapcache(folio))
1240 0 : spin_lock(&mapping->host->i_lock);
1241 0 : xa_lock_irq(&mapping->i_pages);
1242 : /*
1243 : * The non racy check for a busy page.
1244 : *
1245 : * Must be careful with the order of the tests. When someone has
1246 : * a ref to the page, it may be possible that they dirty it then
1247 : * drop the reference. So if PageDirty is tested before page_count
1248 : * here, then the following race may occur:
1249 : *
1250 : * get_user_pages(&page);
1251 : * [user mapping goes away]
1252 : * write_to(page);
1253 : * !PageDirty(page) [good]
1254 : * SetPageDirty(page);
1255 : * put_page(page);
1256 : * !page_count(page) [good, discard it]
1257 : *
1258 : * [oops, our write_to data is lost]
1259 : *
1260 : * Reversing the order of the tests ensures such a situation cannot
1261 : * escape unnoticed. The smp_rmb is needed to ensure the page->flags
1262 : * load is not satisfied before that of page->_refcount.
1263 : *
1264 : * Note that if SetPageDirty is always performed via set_page_dirty,
1265 : * and thus under the i_pages lock, then this ordering is not required.
1266 : */
1267 0 : refcount = 1 + folio_nr_pages(folio);
1268 0 : if (!folio_ref_freeze(folio, refcount))
1269 : goto cannot_free;
1270 : /* note: atomic_cmpxchg in page_ref_freeze provides the smp_rmb */
1271 0 : if (unlikely(folio_test_dirty(folio))) {
1272 : folio_ref_unfreeze(folio, refcount);
1273 : goto cannot_free;
1274 : }
1275 :
1276 0 : if (folio_test_swapcache(folio)) {
1277 0 : swp_entry_t swap = folio_swap_entry(folio);
1278 0 : mem_cgroup_swapout(folio, swap);
1279 0 : if (reclaimed && !mapping_exiting(mapping))
1280 0 : shadow = workingset_eviction(folio, target_memcg);
1281 0 : __delete_from_swap_cache(&folio->page, swap, shadow);
1282 0 : xa_unlock_irq(&mapping->i_pages);
1283 0 : put_swap_page(&folio->page, swap);
1284 : } else {
1285 : void (*freepage)(struct page *);
1286 :
1287 0 : freepage = mapping->a_ops->freepage;
1288 : /*
1289 : * Remember a shadow entry for reclaimed file cache in
1290 : * order to detect refaults, thus thrashing, later on.
1291 : *
1292 : * But don't store shadows in an address space that is
1293 : * already exiting. This is not just an optimization,
1294 : * inode reclaim needs to empty out the radix tree or
1295 : * the nodes are lost. Don't plant shadows behind its
1296 : * back.
1297 : *
1298 : * We also don't store shadows for DAX mappings because the
1299 : * only page cache pages found in these are zero pages
1300 : * covering holes, and because we don't want to mix DAX
1301 : * exceptional entries and shadow exceptional entries in the
1302 : * same address_space.
1303 : */
1304 0 : if (reclaimed && folio_is_file_lru(folio) &&
1305 0 : !mapping_exiting(mapping) && !dax_mapping(mapping))
1306 0 : shadow = workingset_eviction(folio, target_memcg);
1307 0 : __filemap_remove_folio(folio, shadow);
1308 0 : xa_unlock_irq(&mapping->i_pages);
1309 0 : if (mapping_shrinkable(mapping))
1310 0 : inode_add_lru(mapping->host);
1311 0 : spin_unlock(&mapping->host->i_lock);
1312 :
1313 0 : if (freepage != NULL)
1314 0 : freepage(&folio->page);
1315 : }
1316 :
1317 : return 1;
1318 :
1319 : cannot_free:
1320 0 : xa_unlock_irq(&mapping->i_pages);
1321 0 : if (!folio_test_swapcache(folio))
1322 0 : spin_unlock(&mapping->host->i_lock);
1323 : return 0;
1324 : }
1325 :
1326 : /**
1327 : * remove_mapping() - Attempt to remove a folio from its mapping.
1328 : * @mapping: The address space.
1329 : * @folio: The folio to remove.
1330 : *
1331 : * If the folio is dirty, under writeback or if someone else has a ref
1332 : * on it, removal will fail.
1333 : * Return: The number of pages removed from the mapping. 0 if the folio
1334 : * could not be removed.
1335 : * Context: The caller should have a single refcount on the folio and
1336 : * hold its lock.
1337 : */
1338 0 : long remove_mapping(struct address_space *mapping, struct folio *folio)
1339 : {
1340 0 : if (__remove_mapping(mapping, folio, false, NULL)) {
1341 : /*
1342 : * Unfreezing the refcount with 1 effectively
1343 : * drops the pagecache ref for us without requiring another
1344 : * atomic operation.
1345 : */
1346 0 : folio_ref_unfreeze(folio, 1);
1347 0 : return folio_nr_pages(folio);
1348 : }
1349 : return 0;
1350 : }
1351 :
1352 : /**
1353 : * folio_putback_lru - Put previously isolated folio onto appropriate LRU list.
1354 : * @folio: Folio to be returned to an LRU list.
1355 : *
1356 : * Add previously isolated @folio to appropriate LRU list.
1357 : * The folio may still be unevictable for other reasons.
1358 : *
1359 : * Context: lru_lock must not be held, interrupts must be enabled.
1360 : */
1361 0 : void folio_putback_lru(struct folio *folio)
1362 : {
1363 0 : folio_add_lru(folio);
1364 0 : folio_put(folio); /* drop ref from isolate */
1365 0 : }
1366 :
1367 : enum page_references {
1368 : PAGEREF_RECLAIM,
1369 : PAGEREF_RECLAIM_CLEAN,
1370 : PAGEREF_KEEP,
1371 : PAGEREF_ACTIVATE,
1372 : };
1373 :
1374 0 : static enum page_references folio_check_references(struct folio *folio,
1375 : struct scan_control *sc)
1376 : {
1377 : int referenced_ptes, referenced_folio;
1378 : unsigned long vm_flags;
1379 :
1380 0 : referenced_ptes = folio_referenced(folio, 1, sc->target_mem_cgroup,
1381 : &vm_flags);
1382 0 : referenced_folio = folio_test_clear_referenced(folio);
1383 :
1384 : /*
1385 : * The supposedly reclaimable folio was found to be in a VM_LOCKED vma.
1386 : * Let the folio, now marked Mlocked, be moved to the unevictable list.
1387 : */
1388 0 : if (vm_flags & VM_LOCKED)
1389 : return PAGEREF_ACTIVATE;
1390 :
1391 0 : if (referenced_ptes) {
1392 : /*
1393 : * All mapped folios start out with page table
1394 : * references from the instantiating fault, so we need
1395 : * to look twice if a mapped file/anon folio is used more
1396 : * than once.
1397 : *
1398 : * Mark it and spare it for another trip around the
1399 : * inactive list. Another page table reference will
1400 : * lead to its activation.
1401 : *
1402 : * Note: the mark is set for activated folios as well
1403 : * so that recently deactivated but used folios are
1404 : * quickly recovered.
1405 : */
1406 0 : folio_set_referenced(folio);
1407 :
1408 0 : if (referenced_folio || referenced_ptes > 1)
1409 : return PAGEREF_ACTIVATE;
1410 :
1411 : /*
1412 : * Activate file-backed executable folios after first usage.
1413 : */
1414 0 : if ((vm_flags & VM_EXEC) && !folio_test_swapbacked(folio))
1415 : return PAGEREF_ACTIVATE;
1416 :
1417 : return PAGEREF_KEEP;
1418 : }
1419 :
1420 : /* Reclaim if clean, defer dirty folios to writeback */
1421 0 : if (referenced_folio && !folio_test_swapbacked(folio))
1422 : return PAGEREF_RECLAIM_CLEAN;
1423 :
1424 : return PAGEREF_RECLAIM;
1425 : }
1426 :
1427 : /* Check if a page is dirty or under writeback */
1428 0 : static void folio_check_dirty_writeback(struct folio *folio,
1429 : bool *dirty, bool *writeback)
1430 : {
1431 : struct address_space *mapping;
1432 :
1433 : /*
1434 : * Anonymous pages are not handled by flushers and must be written
1435 : * from reclaim context. Do not stall reclaim based on them
1436 : */
1437 0 : if (!folio_is_file_lru(folio) ||
1438 0 : (folio_test_anon(folio) && !folio_test_swapbacked(folio))) {
1439 0 : *dirty = false;
1440 0 : *writeback = false;
1441 0 : return;
1442 : }
1443 :
1444 : /* By default assume that the folio flags are accurate */
1445 0 : *dirty = folio_test_dirty(folio);
1446 0 : *writeback = folio_test_writeback(folio);
1447 :
1448 : /* Verify dirty/writeback state if the filesystem supports it */
1449 0 : if (!folio_test_private(folio))
1450 : return;
1451 :
1452 0 : mapping = folio_mapping(folio);
1453 0 : if (mapping && mapping->a_ops->is_dirty_writeback)
1454 0 : mapping->a_ops->is_dirty_writeback(&folio->page, dirty, writeback);
1455 : }
1456 :
1457 0 : static struct page *alloc_demote_page(struct page *page, unsigned long node)
1458 : {
1459 0 : struct migration_target_control mtc = {
1460 : /*
1461 : * Allocate from 'node', or fail quickly and quietly.
1462 : * When this happens, 'page' will likely just be discarded
1463 : * instead of migrated.
1464 : */
1465 : .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) |
1466 : __GFP_THISNODE | __GFP_NOWARN |
1467 : __GFP_NOMEMALLOC | GFP_NOWAIT,
1468 : .nid = node
1469 : };
1470 :
1471 0 : return alloc_migration_target(page, (unsigned long)&mtc);
1472 : }
1473 :
1474 : /*
1475 : * Take pages on @demote_list and attempt to demote them to
1476 : * another node. Pages which are not demoted are left on
1477 : * @demote_pages.
1478 : */
1479 0 : static unsigned int demote_page_list(struct list_head *demote_pages,
1480 : struct pglist_data *pgdat)
1481 : {
1482 0 : int target_nid = next_demotion_node(pgdat->node_id);
1483 : unsigned int nr_succeeded;
1484 :
1485 0 : if (list_empty(demote_pages))
1486 : return 0;
1487 :
1488 0 : if (target_nid == NUMA_NO_NODE)
1489 : return 0;
1490 :
1491 : /* Demotion ignores all cpuset and mempolicy settings */
1492 0 : migrate_pages(demote_pages, alloc_demote_page, NULL,
1493 : target_nid, MIGRATE_ASYNC, MR_DEMOTION,
1494 : &nr_succeeded);
1495 :
1496 0 : if (current_is_kswapd())
1497 0 : __count_vm_events(PGDEMOTE_KSWAPD, nr_succeeded);
1498 : else
1499 0 : __count_vm_events(PGDEMOTE_DIRECT, nr_succeeded);
1500 :
1501 0 : return nr_succeeded;
1502 : }
1503 :
1504 : /*
1505 : * shrink_page_list() returns the number of reclaimed pages
1506 : */
1507 0 : static unsigned int shrink_page_list(struct list_head *page_list,
1508 : struct pglist_data *pgdat,
1509 : struct scan_control *sc,
1510 : struct reclaim_stat *stat,
1511 : bool ignore_references)
1512 : {
1513 0 : LIST_HEAD(ret_pages);
1514 0 : LIST_HEAD(free_pages);
1515 0 : LIST_HEAD(demote_pages);
1516 0 : unsigned int nr_reclaimed = 0;
1517 0 : unsigned int pgactivate = 0;
1518 : bool do_demote_pass;
1519 :
1520 0 : memset(stat, 0, sizeof(*stat));
1521 0 : cond_resched();
1522 0 : do_demote_pass = can_demote(pgdat->node_id, sc);
1523 :
1524 : retry:
1525 0 : while (!list_empty(page_list)) {
1526 : struct address_space *mapping;
1527 : struct page *page;
1528 : struct folio *folio;
1529 0 : enum page_references references = PAGEREF_RECLAIM;
1530 : bool dirty, writeback, may_enter_fs;
1531 : unsigned int nr_pages;
1532 :
1533 0 : cond_resched();
1534 :
1535 0 : folio = lru_to_folio(page_list);
1536 0 : list_del(&folio->lru);
1537 0 : page = &folio->page;
1538 :
1539 0 : if (!trylock_page(page))
1540 : goto keep;
1541 :
1542 : VM_BUG_ON_PAGE(PageActive(page), page);
1543 :
1544 0 : nr_pages = compound_nr(page);
1545 :
1546 : /* Account the number of base pages even though THP */
1547 0 : sc->nr_scanned += nr_pages;
1548 :
1549 0 : if (unlikely(!page_evictable(page)))
1550 : goto activate_locked;
1551 :
1552 0 : if (!sc->may_unmap && page_mapped(page))
1553 : goto keep_locked;
1554 :
1555 0 : may_enter_fs = (sc->gfp_mask & __GFP_FS) ||
1556 0 : (PageSwapCache(page) && (sc->gfp_mask & __GFP_IO));
1557 :
1558 : /*
1559 : * The number of dirty pages determines if a node is marked
1560 : * reclaim_congested. kswapd will stall and start writing
1561 : * pages if the tail of the LRU is all dirty unqueued pages.
1562 : */
1563 0 : folio_check_dirty_writeback(folio, &dirty, &writeback);
1564 0 : if (dirty || writeback)
1565 0 : stat->nr_dirty += nr_pages;
1566 :
1567 0 : if (dirty && !writeback)
1568 0 : stat->nr_unqueued_dirty += nr_pages;
1569 :
1570 : /*
1571 : * Treat this page as congested if the underlying BDI is or if
1572 : * pages are cycling through the LRU so quickly that the
1573 : * pages marked for immediate reclaim are making it to the
1574 : * end of the LRU a second time.
1575 : */
1576 0 : mapping = page_mapping(page);
1577 0 : if (writeback && PageReclaim(page))
1578 0 : stat->nr_congested += nr_pages;
1579 :
1580 : /*
1581 : * If a page at the tail of the LRU is under writeback, there
1582 : * are three cases to consider.
1583 : *
1584 : * 1) If reclaim is encountering an excessive number of pages
1585 : * under writeback and this page is both under writeback and
1586 : * PageReclaim then it indicates that pages are being queued
1587 : * for IO but are being recycled through the LRU before the
1588 : * IO can complete. Waiting on the page itself risks an
1589 : * indefinite stall if it is impossible to writeback the
1590 : * page due to IO error or disconnected storage so instead
1591 : * note that the LRU is being scanned too quickly and the
1592 : * caller can stall after page list has been processed.
1593 : *
1594 : * 2) Global or new memcg reclaim encounters a page that is
1595 : * not marked for immediate reclaim, or the caller does not
1596 : * have __GFP_FS (or __GFP_IO if it's simply going to swap,
1597 : * not to fs). In this case mark the page for immediate
1598 : * reclaim and continue scanning.
1599 : *
1600 : * Require may_enter_fs because we would wait on fs, which
1601 : * may not have submitted IO yet. And the loop driver might
1602 : * enter reclaim, and deadlock if it waits on a page for
1603 : * which it is needed to do the write (loop masks off
1604 : * __GFP_IO|__GFP_FS for this reason); but more thought
1605 : * would probably show more reasons.
1606 : *
1607 : * 3) Legacy memcg encounters a page that is already marked
1608 : * PageReclaim. memcg does not have any dirty pages
1609 : * throttling so we could easily OOM just because too many
1610 : * pages are in writeback and there is nothing else to
1611 : * reclaim. Wait for the writeback to complete.
1612 : *
1613 : * In cases 1) and 2) we activate the pages to get them out of
1614 : * the way while we continue scanning for clean pages on the
1615 : * inactive list and refilling from the active list. The
1616 : * observation here is that waiting for disk writes is more
1617 : * expensive than potentially causing reloads down the line.
1618 : * Since they're marked for immediate reclaim, they won't put
1619 : * memory pressure on the cache working set any longer than it
1620 : * takes to write them to disk.
1621 : */
1622 0 : if (PageWriteback(page)) {
1623 : /* Case 1 above */
1624 0 : if (current_is_kswapd() &&
1625 0 : PageReclaim(page) &&
1626 0 : test_bit(PGDAT_WRITEBACK, &pgdat->flags)) {
1627 0 : stat->nr_immediate += nr_pages;
1628 0 : goto activate_locked;
1629 :
1630 : /* Case 2 above */
1631 0 : } else if (writeback_throttling_sane(sc) ||
1632 : !PageReclaim(page) || !may_enter_fs) {
1633 : /*
1634 : * This is slightly racy - end_page_writeback()
1635 : * might have just cleared PageReclaim, then
1636 : * setting PageReclaim here end up interpreted
1637 : * as PageReadahead - but that does not matter
1638 : * enough to care. What we do want is for this
1639 : * page to have PageReclaim set next time memcg
1640 : * reclaim reaches the tests above, so it will
1641 : * then wait_on_page_writeback() to avoid OOM;
1642 : * and it's also appropriate in global reclaim.
1643 : */
1644 0 : SetPageReclaim(page);
1645 0 : stat->nr_writeback += nr_pages;
1646 0 : goto activate_locked;
1647 :
1648 : /* Case 3 above */
1649 : } else {
1650 : unlock_page(page);
1651 : wait_on_page_writeback(page);
1652 : /* then go back and try same page again */
1653 : list_add_tail(&page->lru, page_list);
1654 0 : continue;
1655 : }
1656 : }
1657 :
1658 0 : if (!ignore_references)
1659 0 : references = folio_check_references(folio, sc);
1660 :
1661 0 : switch (references) {
1662 : case PAGEREF_ACTIVATE:
1663 : goto activate_locked;
1664 : case PAGEREF_KEEP:
1665 0 : stat->nr_ref_keep += nr_pages;
1666 0 : goto keep_locked;
1667 : case PAGEREF_RECLAIM:
1668 : case PAGEREF_RECLAIM_CLEAN:
1669 : ; /* try to reclaim the page below */
1670 : }
1671 :
1672 : /*
1673 : * Before reclaiming the page, try to relocate
1674 : * its contents to another node.
1675 : */
1676 0 : if (do_demote_pass &&
1677 0 : (thp_migration_supported() || !PageTransHuge(page))) {
1678 0 : list_add(&page->lru, &demote_pages);
1679 0 : unlock_page(page);
1680 0 : continue;
1681 : }
1682 :
1683 : /*
1684 : * Anonymous process memory has backing store?
1685 : * Try to allocate it some swap space here.
1686 : * Lazyfree page could be freed directly
1687 : */
1688 0 : if (PageAnon(page) && PageSwapBacked(page)) {
1689 0 : if (!PageSwapCache(page)) {
1690 0 : if (!(sc->gfp_mask & __GFP_IO))
1691 : goto keep_locked;
1692 0 : if (folio_maybe_dma_pinned(folio))
1693 : goto keep_locked;
1694 0 : if (PageTransHuge(page)) {
1695 : /* cannot split THP, skip it */
1696 : if (!can_split_folio(folio, NULL))
1697 : goto activate_locked;
1698 : /*
1699 : * Split pages without a PMD map right
1700 : * away. Chances are some or all of the
1701 : * tail pages can be freed without IO.
1702 : */
1703 : if (!folio_entire_mapcount(folio) &&
1704 : split_folio_to_list(folio,
1705 : page_list))
1706 : goto activate_locked;
1707 : }
1708 0 : if (!add_to_swap(page)) {
1709 0 : if (!PageTransHuge(page))
1710 : goto activate_locked_split;
1711 : /* Fallback to swap normal pages */
1712 : if (split_folio_to_list(folio,
1713 : page_list))
1714 : goto activate_locked;
1715 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1716 : count_vm_event(THP_SWPOUT_FALLBACK);
1717 : #endif
1718 : if (!add_to_swap(page))
1719 : goto activate_locked_split;
1720 : }
1721 :
1722 0 : may_enter_fs = true;
1723 :
1724 : /* Adding to swap updated mapping */
1725 0 : mapping = page_mapping(page);
1726 : }
1727 0 : } else if (PageSwapBacked(page) && PageTransHuge(page)) {
1728 : /* Split shmem THP */
1729 : if (split_folio_to_list(folio, page_list))
1730 : goto keep_locked;
1731 : }
1732 :
1733 : /*
1734 : * THP may get split above, need minus tail pages and update
1735 : * nr_pages to avoid accounting tail pages twice.
1736 : *
1737 : * The tail pages that are added into swap cache successfully
1738 : * reach here.
1739 : */
1740 0 : if ((nr_pages > 1) && !PageTransHuge(page)) {
1741 0 : sc->nr_scanned -= (nr_pages - 1);
1742 0 : nr_pages = 1;
1743 : }
1744 :
1745 : /*
1746 : * The page is mapped into the page tables of one or more
1747 : * processes. Try to unmap it here.
1748 : */
1749 0 : if (page_mapped(page)) {
1750 0 : enum ttu_flags flags = TTU_BATCH_FLUSH;
1751 0 : bool was_swapbacked = PageSwapBacked(page);
1752 :
1753 0 : if (PageTransHuge(page) &&
1754 : thp_order(page) >= HPAGE_PMD_ORDER)
1755 : flags |= TTU_SPLIT_HUGE_PMD;
1756 :
1757 0 : try_to_unmap(folio, flags);
1758 0 : if (page_mapped(page)) {
1759 0 : stat->nr_unmap_fail += nr_pages;
1760 0 : if (!was_swapbacked && PageSwapBacked(page))
1761 0 : stat->nr_lazyfree_fail += nr_pages;
1762 : goto activate_locked;
1763 : }
1764 : }
1765 :
1766 0 : if (PageDirty(page)) {
1767 : /*
1768 : * Only kswapd can writeback filesystem pages
1769 : * to avoid risk of stack overflow. But avoid
1770 : * injecting inefficient single-page IO into
1771 : * flusher writeback as much as possible: only
1772 : * write pages when we've encountered many
1773 : * dirty pages, and when we've already scanned
1774 : * the rest of the LRU for clean pages and see
1775 : * the same dirty pages again (PageReclaim).
1776 : */
1777 0 : if (page_is_file_lru(page) &&
1778 0 : (!current_is_kswapd() || !PageReclaim(page) ||
1779 0 : !test_bit(PGDAT_DIRTY, &pgdat->flags))) {
1780 : /*
1781 : * Immediately reclaim when written back.
1782 : * Similar in principal to deactivate_page()
1783 : * except we already have the page isolated
1784 : * and know it's dirty
1785 : */
1786 0 : inc_node_page_state(page, NR_VMSCAN_IMMEDIATE);
1787 : SetPageReclaim(page);
1788 :
1789 : goto activate_locked;
1790 : }
1791 :
1792 0 : if (references == PAGEREF_RECLAIM_CLEAN)
1793 : goto keep_locked;
1794 0 : if (!may_enter_fs)
1795 : goto keep_locked;
1796 0 : if (!sc->may_writepage)
1797 : goto keep_locked;
1798 :
1799 : /*
1800 : * Page is dirty. Flush the TLB if a writable entry
1801 : * potentially exists to avoid CPU writes after IO
1802 : * starts and then write it out here.
1803 : */
1804 : try_to_unmap_flush_dirty();
1805 0 : switch (pageout(folio, mapping)) {
1806 : case PAGE_KEEP:
1807 : goto keep_locked;
1808 : case PAGE_ACTIVATE:
1809 : goto activate_locked;
1810 : case PAGE_SUCCESS:
1811 0 : stat->nr_pageout += nr_pages;
1812 :
1813 0 : if (PageWriteback(page))
1814 : goto keep;
1815 0 : if (PageDirty(page))
1816 : goto keep;
1817 :
1818 : /*
1819 : * A synchronous write - probably a ramdisk. Go
1820 : * ahead and try to reclaim the page.
1821 : */
1822 0 : if (!trylock_page(page))
1823 : goto keep;
1824 0 : if (PageDirty(page) || PageWriteback(page))
1825 : goto keep_locked;
1826 0 : mapping = page_mapping(page);
1827 : fallthrough;
1828 : case PAGE_CLEAN:
1829 : ; /* try to free the page below */
1830 : }
1831 : }
1832 :
1833 : /*
1834 : * If the page has buffers, try to free the buffer mappings
1835 : * associated with this page. If we succeed we try to free
1836 : * the page as well.
1837 : *
1838 : * We do this even if the page is PageDirty().
1839 : * try_to_release_page() does not perform I/O, but it is
1840 : * possible for a page to have PageDirty set, but it is actually
1841 : * clean (all its buffers are clean). This happens if the
1842 : * buffers were written out directly, with submit_bh(). ext3
1843 : * will do this, as well as the blockdev mapping.
1844 : * try_to_release_page() will discover that cleanness and will
1845 : * drop the buffers and mark the page clean - it can be freed.
1846 : *
1847 : * Rarely, pages can have buffers and no ->mapping. These are
1848 : * the pages which were not successfully invalidated in
1849 : * truncate_cleanup_page(). We try to drop those buffers here
1850 : * and if that worked, and the page is no longer mapped into
1851 : * process address space (page_count == 1) it can be freed.
1852 : * Otherwise, leave the page on the LRU so it is swappable.
1853 : */
1854 0 : if (page_has_private(page)) {
1855 0 : if (!try_to_release_page(page, sc->gfp_mask))
1856 : goto activate_locked;
1857 0 : if (!mapping && page_count(page) == 1) {
1858 0 : unlock_page(page);
1859 0 : if (put_page_testzero(page))
1860 : goto free_it;
1861 : else {
1862 : /*
1863 : * rare race with speculative reference.
1864 : * the speculative reference will free
1865 : * this page shortly, so we may
1866 : * increment nr_reclaimed here (and
1867 : * leave it off the LRU).
1868 : */
1869 0 : nr_reclaimed++;
1870 0 : continue;
1871 : }
1872 : }
1873 : }
1874 :
1875 0 : if (PageAnon(page) && !PageSwapBacked(page)) {
1876 : /* follow __remove_mapping for reference */
1877 0 : if (!page_ref_freeze(page, 1))
1878 : goto keep_locked;
1879 : /*
1880 : * The page has only one reference left, which is
1881 : * from the isolation. After the caller puts the
1882 : * page back on lru and drops the reference, the
1883 : * page will be freed anyway. It doesn't matter
1884 : * which lru it goes. So we don't bother checking
1885 : * PageDirty here.
1886 : */
1887 0 : count_vm_event(PGLAZYFREED);
1888 0 : count_memcg_page_event(page, PGLAZYFREED);
1889 0 : } else if (!mapping || !__remove_mapping(mapping, folio, true,
1890 : sc->target_mem_cgroup))
1891 : goto keep_locked;
1892 :
1893 0 : unlock_page(page);
1894 : free_it:
1895 : /*
1896 : * THP may get swapped out in a whole, need account
1897 : * all base pages.
1898 : */
1899 0 : nr_reclaimed += nr_pages;
1900 :
1901 : /*
1902 : * Is there need to periodically free_page_list? It would
1903 : * appear not as the counts should be low
1904 : */
1905 0 : if (unlikely(PageTransHuge(page)))
1906 : destroy_compound_page(page);
1907 : else
1908 0 : list_add(&page->lru, &free_pages);
1909 0 : continue;
1910 :
1911 : activate_locked_split:
1912 : /*
1913 : * The tail pages that are failed to add into swap cache
1914 : * reach here. Fixup nr_scanned and nr_pages.
1915 : */
1916 0 : if (nr_pages > 1) {
1917 0 : sc->nr_scanned -= (nr_pages - 1);
1918 0 : nr_pages = 1;
1919 : }
1920 : activate_locked:
1921 : /* Not a candidate for swapping, so reclaim swap space. */
1922 0 : if (PageSwapCache(page) && (mem_cgroup_swap_full(page) ||
1923 0 : PageMlocked(page)))
1924 0 : try_to_free_swap(page);
1925 : VM_BUG_ON_PAGE(PageActive(page), page);
1926 0 : if (!PageMlocked(page)) {
1927 0 : int type = page_is_file_lru(page);
1928 0 : SetPageActive(page);
1929 0 : stat->nr_activate[type] += nr_pages;
1930 0 : count_memcg_page_event(page, PGACTIVATE);
1931 : }
1932 : keep_locked:
1933 0 : unlock_page(page);
1934 : keep:
1935 0 : list_add(&page->lru, &ret_pages);
1936 : VM_BUG_ON_PAGE(PageLRU(page) || PageUnevictable(page), page);
1937 : }
1938 : /* 'page_list' is always empty here */
1939 :
1940 : /* Migrate pages selected for demotion */
1941 0 : nr_reclaimed += demote_page_list(&demote_pages, pgdat);
1942 : /* Pages that could not be demoted are still in @demote_pages */
1943 0 : if (!list_empty(&demote_pages)) {
1944 : /* Pages which failed to demoted go back on @page_list for retry: */
1945 : list_splice_init(&demote_pages, page_list);
1946 : do_demote_pass = false;
1947 : goto retry;
1948 : }
1949 :
1950 0 : pgactivate = stat->nr_activate[0] + stat->nr_activate[1];
1951 :
1952 0 : mem_cgroup_uncharge_list(&free_pages);
1953 : try_to_unmap_flush();
1954 0 : free_unref_page_list(&free_pages);
1955 :
1956 0 : list_splice(&ret_pages, page_list);
1957 0 : count_vm_events(PGACTIVATE, pgactivate);
1958 :
1959 0 : return nr_reclaimed;
1960 : }
1961 :
1962 0 : unsigned int reclaim_clean_pages_from_list(struct zone *zone,
1963 : struct list_head *page_list)
1964 : {
1965 0 : struct scan_control sc = {
1966 : .gfp_mask = GFP_KERNEL,
1967 : .may_unmap = 1,
1968 : };
1969 : struct reclaim_stat stat;
1970 : unsigned int nr_reclaimed;
1971 : struct page *page, *next;
1972 0 : LIST_HEAD(clean_pages);
1973 : unsigned int noreclaim_flag;
1974 :
1975 0 : list_for_each_entry_safe(page, next, page_list, lru) {
1976 0 : if (!PageHuge(page) && page_is_file_lru(page) &&
1977 0 : !PageDirty(page) && !__PageMovable(page) &&
1978 0 : !PageUnevictable(page)) {
1979 0 : ClearPageActive(page);
1980 0 : list_move(&page->lru, &clean_pages);
1981 : }
1982 : }
1983 :
1984 : /*
1985 : * We should be safe here since we are only dealing with file pages and
1986 : * we are not kswapd and therefore cannot write dirty file pages. But
1987 : * call memalloc_noreclaim_save() anyway, just in case these conditions
1988 : * change in the future.
1989 : */
1990 0 : noreclaim_flag = memalloc_noreclaim_save();
1991 0 : nr_reclaimed = shrink_page_list(&clean_pages, zone->zone_pgdat, &sc,
1992 : &stat, true);
1993 0 : memalloc_noreclaim_restore(noreclaim_flag);
1994 :
1995 0 : list_splice(&clean_pages, page_list);
1996 0 : mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE,
1997 : -(long)nr_reclaimed);
1998 : /*
1999 : * Since lazyfree pages are isolated from file LRU from the beginning,
2000 : * they will rotate back to anonymous LRU in the end if it failed to
2001 : * discard so isolated count will be mismatched.
2002 : * Compensate the isolated count for both LRU lists.
2003 : */
2004 0 : mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_ANON,
2005 0 : stat.nr_lazyfree_fail);
2006 0 : mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE,
2007 0 : -(long)stat.nr_lazyfree_fail);
2008 0 : return nr_reclaimed;
2009 : }
2010 :
2011 : /*
2012 : * Update LRU sizes after isolating pages. The LRU size updates must
2013 : * be complete before mem_cgroup_update_lru_size due to a sanity check.
2014 : */
2015 : static __always_inline void update_lru_sizes(struct lruvec *lruvec,
2016 : enum lru_list lru, unsigned long *nr_zone_taken)
2017 : {
2018 : int zid;
2019 :
2020 0 : for (zid = 0; zid < MAX_NR_ZONES; zid++) {
2021 0 : if (!nr_zone_taken[zid])
2022 0 : continue;
2023 :
2024 0 : update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
2025 : }
2026 :
2027 : }
2028 :
2029 : /*
2030 : * Isolating page from the lruvec to fill in @dst list by nr_to_scan times.
2031 : *
2032 : * lruvec->lru_lock is heavily contended. Some of the functions that
2033 : * shrink the lists perform better by taking out a batch of pages
2034 : * and working on them outside the LRU lock.
2035 : *
2036 : * For pagecache intensive workloads, this function is the hottest
2037 : * spot in the kernel (apart from copy_*_user functions).
2038 : *
2039 : * Lru_lock must be held before calling this function.
2040 : *
2041 : * @nr_to_scan: The number of eligible pages to look through on the list.
2042 : * @lruvec: The LRU vector to pull pages from.
2043 : * @dst: The temp list to put pages on to.
2044 : * @nr_scanned: The number of pages that were scanned.
2045 : * @sc: The scan_control struct for this reclaim session
2046 : * @lru: LRU list id for isolating
2047 : *
2048 : * returns how many pages were moved onto *@dst.
2049 : */
2050 0 : static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
2051 : struct lruvec *lruvec, struct list_head *dst,
2052 : unsigned long *nr_scanned, struct scan_control *sc,
2053 : enum lru_list lru)
2054 : {
2055 0 : struct list_head *src = &lruvec->lists[lru];
2056 0 : unsigned long nr_taken = 0;
2057 0 : unsigned long nr_zone_taken[MAX_NR_ZONES] = { 0 };
2058 0 : unsigned long nr_skipped[MAX_NR_ZONES] = { 0, };
2059 0 : unsigned long skipped = 0;
2060 : unsigned long scan, total_scan, nr_pages;
2061 0 : LIST_HEAD(pages_skipped);
2062 :
2063 0 : total_scan = 0;
2064 0 : scan = 0;
2065 0 : while (scan < nr_to_scan && !list_empty(src)) {
2066 0 : struct list_head *move_to = src;
2067 : struct page *page;
2068 :
2069 0 : page = lru_to_page(src);
2070 : prefetchw_prev_lru_page(page, src, flags);
2071 :
2072 0 : nr_pages = compound_nr(page);
2073 0 : total_scan += nr_pages;
2074 :
2075 0 : if (page_zonenum(page) > sc->reclaim_idx) {
2076 0 : nr_skipped[page_zonenum(page)] += nr_pages;
2077 0 : move_to = &pages_skipped;
2078 0 : goto move;
2079 : }
2080 :
2081 : /*
2082 : * Do not count skipped pages because that makes the function
2083 : * return with no isolated pages if the LRU mostly contains
2084 : * ineligible pages. This causes the VM to not reclaim any
2085 : * pages, triggering a premature OOM.
2086 : * Account all tail pages of THP.
2087 : */
2088 0 : scan += nr_pages;
2089 :
2090 0 : if (!PageLRU(page))
2091 : goto move;
2092 0 : if (!sc->may_unmap && page_mapped(page))
2093 : goto move;
2094 :
2095 : /*
2096 : * Be careful not to clear PageLRU until after we're
2097 : * sure the page is not being freed elsewhere -- the
2098 : * page release code relies on it.
2099 : */
2100 0 : if (unlikely(!get_page_unless_zero(page)))
2101 : goto move;
2102 :
2103 0 : if (!TestClearPageLRU(page)) {
2104 : /* Another thread is already isolating this page */
2105 0 : put_page(page);
2106 0 : goto move;
2107 : }
2108 :
2109 0 : nr_taken += nr_pages;
2110 0 : nr_zone_taken[page_zonenum(page)] += nr_pages;
2111 0 : move_to = dst;
2112 : move:
2113 0 : list_move(&page->lru, move_to);
2114 : }
2115 :
2116 : /*
2117 : * Splice any skipped pages to the start of the LRU list. Note that
2118 : * this disrupts the LRU order when reclaiming for lower zones but
2119 : * we cannot splice to the tail. If we did then the SWAP_CLUSTER_MAX
2120 : * scanning would soon rescan the same pages to skip and put the
2121 : * system at risk of premature OOM.
2122 : */
2123 0 : if (!list_empty(&pages_skipped)) {
2124 : int zid;
2125 :
2126 : list_splice(&pages_skipped, src);
2127 0 : for (zid = 0; zid < MAX_NR_ZONES; zid++) {
2128 0 : if (!nr_skipped[zid])
2129 0 : continue;
2130 :
2131 0 : __count_zid_vm_events(PGSCAN_SKIP, zid, nr_skipped[zid]);
2132 0 : skipped += nr_skipped[zid];
2133 : }
2134 : }
2135 0 : *nr_scanned = total_scan;
2136 0 : trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan,
2137 : total_scan, skipped, nr_taken,
2138 : sc->may_unmap ? 0 : ISOLATE_UNMAPPED, lru);
2139 0 : update_lru_sizes(lruvec, lru, nr_zone_taken);
2140 0 : return nr_taken;
2141 : }
2142 :
2143 : /**
2144 : * folio_isolate_lru() - Try to isolate a folio from its LRU list.
2145 : * @folio: Folio to isolate from its LRU list.
2146 : *
2147 : * Isolate a @folio from an LRU list and adjust the vmstat statistic
2148 : * corresponding to whatever LRU list the folio was on.
2149 : *
2150 : * The folio will have its LRU flag cleared. If it was found on the
2151 : * active list, it will have the Active flag set. If it was found on the
2152 : * unevictable list, it will have the Unevictable flag set. These flags
2153 : * may need to be cleared by the caller before letting the page go.
2154 : *
2155 : * Context:
2156 : *
2157 : * (1) Must be called with an elevated refcount on the page. This is a
2158 : * fundamental difference from isolate_lru_pages() (which is called
2159 : * without a stable reference).
2160 : * (2) The lru_lock must not be held.
2161 : * (3) Interrupts must be enabled.
2162 : *
2163 : * Return: 0 if the folio was removed from an LRU list.
2164 : * -EBUSY if the folio was not on an LRU list.
2165 : */
2166 0 : int folio_isolate_lru(struct folio *folio)
2167 : {
2168 0 : int ret = -EBUSY;
2169 :
2170 : VM_BUG_ON_FOLIO(!folio_ref_count(folio), folio);
2171 :
2172 0 : if (folio_test_clear_lru(folio)) {
2173 : struct lruvec *lruvec;
2174 :
2175 0 : folio_get(folio);
2176 0 : lruvec = folio_lruvec_lock_irq(folio);
2177 0 : lruvec_del_folio(lruvec, folio);
2178 : unlock_page_lruvec_irq(lruvec);
2179 0 : ret = 0;
2180 : }
2181 :
2182 0 : return ret;
2183 : }
2184 :
2185 : /*
2186 : * A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and
2187 : * then get rescheduled. When there are massive number of tasks doing page
2188 : * allocation, such sleeping direct reclaimers may keep piling up on each CPU,
2189 : * the LRU list will go small and be scanned faster than necessary, leading to
2190 : * unnecessary swapping, thrashing and OOM.
2191 : */
2192 0 : static int too_many_isolated(struct pglist_data *pgdat, int file,
2193 : struct scan_control *sc)
2194 : {
2195 : unsigned long inactive, isolated;
2196 : bool too_many;
2197 :
2198 0 : if (current_is_kswapd())
2199 : return 0;
2200 :
2201 0 : if (!writeback_throttling_sane(sc))
2202 : return 0;
2203 :
2204 0 : if (file) {
2205 0 : inactive = node_page_state(pgdat, NR_INACTIVE_FILE);
2206 0 : isolated = node_page_state(pgdat, NR_ISOLATED_FILE);
2207 : } else {
2208 0 : inactive = node_page_state(pgdat, NR_INACTIVE_ANON);
2209 0 : isolated = node_page_state(pgdat, NR_ISOLATED_ANON);
2210 : }
2211 :
2212 : /*
2213 : * GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they
2214 : * won't get blocked by normal direct-reclaimers, forming a circular
2215 : * deadlock.
2216 : */
2217 0 : if ((sc->gfp_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
2218 0 : inactive >>= 3;
2219 :
2220 0 : too_many = isolated > inactive;
2221 :
2222 : /* Wake up tasks throttled due to too_many_isolated. */
2223 0 : if (!too_many)
2224 : wake_throttle_isolated(pgdat);
2225 :
2226 0 : return too_many;
2227 : }
2228 :
2229 : /*
2230 : * move_pages_to_lru() moves pages from private @list to appropriate LRU list.
2231 : * On return, @list is reused as a list of pages to be freed by the caller.
2232 : *
2233 : * Returns the number of pages moved to the given lruvec.
2234 : */
2235 0 : static unsigned int move_pages_to_lru(struct lruvec *lruvec,
2236 : struct list_head *list)
2237 : {
2238 0 : int nr_pages, nr_moved = 0;
2239 0 : LIST_HEAD(pages_to_free);
2240 : struct page *page;
2241 :
2242 0 : while (!list_empty(list)) {
2243 0 : page = lru_to_page(list);
2244 : VM_BUG_ON_PAGE(PageLRU(page), page);
2245 0 : list_del(&page->lru);
2246 0 : if (unlikely(!page_evictable(page))) {
2247 0 : spin_unlock_irq(&lruvec->lru_lock);
2248 0 : putback_lru_page(page);
2249 0 : spin_lock_irq(&lruvec->lru_lock);
2250 0 : continue;
2251 : }
2252 :
2253 : /*
2254 : * The SetPageLRU needs to be kept here for list integrity.
2255 : * Otherwise:
2256 : * #0 move_pages_to_lru #1 release_pages
2257 : * if !put_page_testzero
2258 : * if (put_page_testzero())
2259 : * !PageLRU //skip lru_lock
2260 : * SetPageLRU()
2261 : * list_add(&page->lru,)
2262 : * list_add(&page->lru,)
2263 : */
2264 0 : SetPageLRU(page);
2265 :
2266 0 : if (unlikely(put_page_testzero(page))) {
2267 0 : __clear_page_lru_flags(page);
2268 :
2269 0 : if (unlikely(PageCompound(page))) {
2270 0 : spin_unlock_irq(&lruvec->lru_lock);
2271 0 : destroy_compound_page(page);
2272 0 : spin_lock_irq(&lruvec->lru_lock);
2273 : } else
2274 0 : list_add(&page->lru, &pages_to_free);
2275 :
2276 0 : continue;
2277 : }
2278 :
2279 : /*
2280 : * All pages were isolated from the same lruvec (and isolation
2281 : * inhibits memcg migration).
2282 : */
2283 : VM_BUG_ON_PAGE(!folio_matches_lruvec(page_folio(page), lruvec), page);
2284 0 : add_page_to_lru_list(page, lruvec);
2285 0 : nr_pages = thp_nr_pages(page);
2286 0 : nr_moved += nr_pages;
2287 0 : if (PageActive(page))
2288 0 : workingset_age_nonresident(lruvec, nr_pages);
2289 : }
2290 :
2291 : /*
2292 : * To save our caller's stack, now use input list for pages to free.
2293 : */
2294 0 : list_splice(&pages_to_free, list);
2295 :
2296 0 : return nr_moved;
2297 : }
2298 :
2299 : /*
2300 : * If a kernel thread (such as nfsd for loop-back mounts) services
2301 : * a backing device by writing to the page cache it sets PF_LOCAL_THROTTLE.
2302 : * In that case we should only throttle if the backing device it is
2303 : * writing to is congested. In other cases it is safe to throttle.
2304 : */
2305 : static int current_may_throttle(void)
2306 : {
2307 0 : return !(current->flags & PF_LOCAL_THROTTLE);
2308 : }
2309 :
2310 : /*
2311 : * shrink_inactive_list() is a helper for shrink_node(). It returns the number
2312 : * of reclaimed pages
2313 : */
2314 : static unsigned long
2315 0 : shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
2316 : struct scan_control *sc, enum lru_list lru)
2317 : {
2318 0 : LIST_HEAD(page_list);
2319 : unsigned long nr_scanned;
2320 0 : unsigned int nr_reclaimed = 0;
2321 : unsigned long nr_taken;
2322 : struct reclaim_stat stat;
2323 0 : bool file = is_file_lru(lru);
2324 : enum vm_event_item item;
2325 0 : struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2326 0 : bool stalled = false;
2327 :
2328 0 : while (unlikely(too_many_isolated(pgdat, file, sc))) {
2329 0 : if (stalled)
2330 : return 0;
2331 :
2332 : /* wait a bit for the reclaimer. */
2333 0 : stalled = true;
2334 0 : reclaim_throttle(pgdat, VMSCAN_THROTTLE_ISOLATED);
2335 :
2336 : /* We are about to die and free our memory. Return now. */
2337 0 : if (fatal_signal_pending(current))
2338 : return SWAP_CLUSTER_MAX;
2339 : }
2340 :
2341 0 : lru_add_drain();
2342 :
2343 0 : spin_lock_irq(&lruvec->lru_lock);
2344 :
2345 0 : nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list,
2346 : &nr_scanned, sc, lru);
2347 :
2348 0 : __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
2349 0 : item = current_is_kswapd() ? PGSCAN_KSWAPD : PGSCAN_DIRECT;
2350 0 : if (!cgroup_reclaim(sc))
2351 0 : __count_vm_events(item, nr_scanned);
2352 0 : __count_memcg_events(lruvec_memcg(lruvec), item, nr_scanned);
2353 0 : __count_vm_events(PGSCAN_ANON + file, nr_scanned);
2354 :
2355 0 : spin_unlock_irq(&lruvec->lru_lock);
2356 :
2357 0 : if (nr_taken == 0)
2358 : return 0;
2359 :
2360 0 : nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, &stat, false);
2361 :
2362 0 : spin_lock_irq(&lruvec->lru_lock);
2363 0 : move_pages_to_lru(lruvec, &page_list);
2364 :
2365 0 : __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
2366 0 : item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT;
2367 0 : if (!cgroup_reclaim(sc))
2368 0 : __count_vm_events(item, nr_reclaimed);
2369 0 : __count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed);
2370 0 : __count_vm_events(PGSTEAL_ANON + file, nr_reclaimed);
2371 0 : spin_unlock_irq(&lruvec->lru_lock);
2372 :
2373 0 : lru_note_cost(lruvec, file, stat.nr_pageout);
2374 0 : mem_cgroup_uncharge_list(&page_list);
2375 0 : free_unref_page_list(&page_list);
2376 :
2377 : /*
2378 : * If dirty pages are scanned that are not queued for IO, it
2379 : * implies that flushers are not doing their job. This can
2380 : * happen when memory pressure pushes dirty pages to the end of
2381 : * the LRU before the dirty limits are breached and the dirty
2382 : * data has expired. It can also happen when the proportion of
2383 : * dirty pages grows not through writes but through memory
2384 : * pressure reclaiming all the clean cache. And in some cases,
2385 : * the flushers simply cannot keep up with the allocation
2386 : * rate. Nudge the flusher threads in case they are asleep.
2387 : */
2388 0 : if (stat.nr_unqueued_dirty == nr_taken)
2389 0 : wakeup_flusher_threads(WB_REASON_VMSCAN);
2390 :
2391 0 : sc->nr.dirty += stat.nr_dirty;
2392 0 : sc->nr.congested += stat.nr_congested;
2393 0 : sc->nr.unqueued_dirty += stat.nr_unqueued_dirty;
2394 0 : sc->nr.writeback += stat.nr_writeback;
2395 0 : sc->nr.immediate += stat.nr_immediate;
2396 0 : sc->nr.taken += nr_taken;
2397 0 : if (file)
2398 0 : sc->nr.file_taken += nr_taken;
2399 :
2400 0 : trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
2401 0 : nr_scanned, nr_reclaimed, &stat, sc->priority, file);
2402 0 : return nr_reclaimed;
2403 : }
2404 :
2405 : /*
2406 : * shrink_active_list() moves pages from the active LRU to the inactive LRU.
2407 : *
2408 : * We move them the other way if the page is referenced by one or more
2409 : * processes.
2410 : *
2411 : * If the pages are mostly unmapped, the processing is fast and it is
2412 : * appropriate to hold lru_lock across the whole operation. But if
2413 : * the pages are mapped, the processing is slow (folio_referenced()), so
2414 : * we should drop lru_lock around each page. It's impossible to balance
2415 : * this, so instead we remove the pages from the LRU while processing them.
2416 : * It is safe to rely on PG_active against the non-LRU pages in here because
2417 : * nobody will play with that bit on a non-LRU page.
2418 : *
2419 : * The downside is that we have to touch page->_refcount against each page.
2420 : * But we had to alter page->flags anyway.
2421 : */
2422 0 : static void shrink_active_list(unsigned long nr_to_scan,
2423 : struct lruvec *lruvec,
2424 : struct scan_control *sc,
2425 : enum lru_list lru)
2426 : {
2427 : unsigned long nr_taken;
2428 : unsigned long nr_scanned;
2429 : unsigned long vm_flags;
2430 0 : LIST_HEAD(l_hold); /* The pages which were snipped off */
2431 0 : LIST_HEAD(l_active);
2432 0 : LIST_HEAD(l_inactive);
2433 : unsigned nr_deactivate, nr_activate;
2434 0 : unsigned nr_rotated = 0;
2435 0 : int file = is_file_lru(lru);
2436 0 : struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2437 :
2438 0 : lru_add_drain();
2439 :
2440 0 : spin_lock_irq(&lruvec->lru_lock);
2441 :
2442 0 : nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold,
2443 : &nr_scanned, sc, lru);
2444 :
2445 0 : __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken);
2446 :
2447 0 : if (!cgroup_reclaim(sc))
2448 0 : __count_vm_events(PGREFILL, nr_scanned);
2449 0 : __count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned);
2450 :
2451 0 : spin_unlock_irq(&lruvec->lru_lock);
2452 :
2453 0 : while (!list_empty(&l_hold)) {
2454 : struct folio *folio;
2455 : struct page *page;
2456 :
2457 0 : cond_resched();
2458 0 : folio = lru_to_folio(&l_hold);
2459 0 : list_del(&folio->lru);
2460 0 : page = &folio->page;
2461 :
2462 0 : if (unlikely(!page_evictable(page))) {
2463 0 : putback_lru_page(page);
2464 0 : continue;
2465 : }
2466 :
2467 0 : if (unlikely(buffer_heads_over_limit)) {
2468 0 : if (page_has_private(page) && trylock_page(page)) {
2469 0 : if (page_has_private(page))
2470 0 : try_to_release_page(page, 0);
2471 0 : unlock_page(page);
2472 : }
2473 : }
2474 :
2475 0 : if (folio_referenced(folio, 0, sc->target_mem_cgroup,
2476 : &vm_flags)) {
2477 : /*
2478 : * Identify referenced, file-backed active pages and
2479 : * give them one more trip around the active list. So
2480 : * that executable code get better chances to stay in
2481 : * memory under moderate memory pressure. Anon pages
2482 : * are not likely to be evicted by use-once streaming
2483 : * IO, plus JVM can create lots of anon VM_EXEC pages,
2484 : * so we ignore them here.
2485 : */
2486 0 : if ((vm_flags & VM_EXEC) && page_is_file_lru(page)) {
2487 0 : nr_rotated += thp_nr_pages(page);
2488 0 : list_add(&page->lru, &l_active);
2489 0 : continue;
2490 : }
2491 : }
2492 :
2493 0 : ClearPageActive(page); /* we are de-activating */
2494 0 : SetPageWorkingset(page);
2495 0 : list_add(&page->lru, &l_inactive);
2496 : }
2497 :
2498 : /*
2499 : * Move pages back to the lru list.
2500 : */
2501 0 : spin_lock_irq(&lruvec->lru_lock);
2502 :
2503 0 : nr_activate = move_pages_to_lru(lruvec, &l_active);
2504 0 : nr_deactivate = move_pages_to_lru(lruvec, &l_inactive);
2505 : /* Keep all free pages in l_active list */
2506 0 : list_splice(&l_inactive, &l_active);
2507 :
2508 0 : __count_vm_events(PGDEACTIVATE, nr_deactivate);
2509 0 : __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate);
2510 :
2511 0 : __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken);
2512 0 : spin_unlock_irq(&lruvec->lru_lock);
2513 :
2514 0 : mem_cgroup_uncharge_list(&l_active);
2515 0 : free_unref_page_list(&l_active);
2516 0 : trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate,
2517 0 : nr_deactivate, nr_rotated, sc->priority, file);
2518 0 : }
2519 :
2520 0 : unsigned long reclaim_pages(struct list_head *page_list)
2521 : {
2522 0 : int nid = NUMA_NO_NODE;
2523 0 : unsigned int nr_reclaimed = 0;
2524 0 : LIST_HEAD(node_page_list);
2525 : struct reclaim_stat dummy_stat;
2526 : struct page *page;
2527 : unsigned int noreclaim_flag;
2528 0 : struct scan_control sc = {
2529 : .gfp_mask = GFP_KERNEL,
2530 : .may_writepage = 1,
2531 : .may_unmap = 1,
2532 : .may_swap = 1,
2533 : .no_demotion = 1,
2534 : };
2535 :
2536 0 : noreclaim_flag = memalloc_noreclaim_save();
2537 :
2538 0 : while (!list_empty(page_list)) {
2539 0 : page = lru_to_page(page_list);
2540 0 : if (nid == NUMA_NO_NODE) {
2541 0 : nid = page_to_nid(page);
2542 : INIT_LIST_HEAD(&node_page_list);
2543 : }
2544 :
2545 0 : if (nid == page_to_nid(page)) {
2546 0 : ClearPageActive(page);
2547 0 : list_move(&page->lru, &node_page_list);
2548 0 : continue;
2549 : }
2550 :
2551 0 : nr_reclaimed += shrink_page_list(&node_page_list,
2552 : NODE_DATA(nid),
2553 : &sc, &dummy_stat, false);
2554 0 : while (!list_empty(&node_page_list)) {
2555 0 : page = lru_to_page(&node_page_list);
2556 0 : list_del(&page->lru);
2557 0 : putback_lru_page(page);
2558 : }
2559 :
2560 : nid = NUMA_NO_NODE;
2561 : }
2562 :
2563 0 : if (!list_empty(&node_page_list)) {
2564 0 : nr_reclaimed += shrink_page_list(&node_page_list,
2565 : NODE_DATA(nid),
2566 : &sc, &dummy_stat, false);
2567 0 : while (!list_empty(&node_page_list)) {
2568 0 : page = lru_to_page(&node_page_list);
2569 0 : list_del(&page->lru);
2570 0 : putback_lru_page(page);
2571 : }
2572 : }
2573 :
2574 0 : memalloc_noreclaim_restore(noreclaim_flag);
2575 :
2576 0 : return nr_reclaimed;
2577 : }
2578 :
2579 0 : static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
2580 : struct lruvec *lruvec, struct scan_control *sc)
2581 : {
2582 0 : if (is_active_lru(lru)) {
2583 0 : if (sc->may_deactivate & (1 << is_file_lru(lru)))
2584 0 : shrink_active_list(nr_to_scan, lruvec, sc, lru);
2585 : else
2586 0 : sc->skipped_deactivate = 1;
2587 : return 0;
2588 : }
2589 :
2590 0 : return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);
2591 : }
2592 :
2593 : /*
2594 : * The inactive anon list should be small enough that the VM never has
2595 : * to do too much work.
2596 : *
2597 : * The inactive file list should be small enough to leave most memory
2598 : * to the established workingset on the scan-resistant active list,
2599 : * but large enough to avoid thrashing the aggregate readahead window.
2600 : *
2601 : * Both inactive lists should also be large enough that each inactive
2602 : * page has a chance to be referenced again before it is reclaimed.
2603 : *
2604 : * If that fails and refaulting is observed, the inactive list grows.
2605 : *
2606 : * The inactive_ratio is the target ratio of ACTIVE to INACTIVE pages
2607 : * on this LRU, maintained by the pageout code. An inactive_ratio
2608 : * of 3 means 3:1 or 25% of the pages are kept on the inactive list.
2609 : *
2610 : * total target max
2611 : * memory ratio inactive
2612 : * -------------------------------------
2613 : * 10MB 1 5MB
2614 : * 100MB 1 50MB
2615 : * 1GB 3 250MB
2616 : * 10GB 10 0.9GB
2617 : * 100GB 31 3GB
2618 : * 1TB 101 10GB
2619 : * 10TB 320 32GB
2620 : */
2621 0 : static bool inactive_is_low(struct lruvec *lruvec, enum lru_list inactive_lru)
2622 : {
2623 0 : enum lru_list active_lru = inactive_lru + LRU_ACTIVE;
2624 : unsigned long inactive, active;
2625 : unsigned long inactive_ratio;
2626 : unsigned long gb;
2627 :
2628 0 : inactive = lruvec_page_state(lruvec, NR_LRU_BASE + inactive_lru);
2629 0 : active = lruvec_page_state(lruvec, NR_LRU_BASE + active_lru);
2630 :
2631 0 : gb = (inactive + active) >> (30 - PAGE_SHIFT);
2632 0 : if (gb)
2633 0 : inactive_ratio = int_sqrt(10 * gb);
2634 : else
2635 : inactive_ratio = 1;
2636 :
2637 0 : return inactive * inactive_ratio < active;
2638 : }
2639 :
2640 : enum scan_balance {
2641 : SCAN_EQUAL,
2642 : SCAN_FRACT,
2643 : SCAN_ANON,
2644 : SCAN_FILE,
2645 : };
2646 :
2647 : /*
2648 : * Determine how aggressively the anon and file LRU lists should be
2649 : * scanned. The relative value of each set of LRU lists is determined
2650 : * by looking at the fraction of the pages scanned we did rotate back
2651 : * onto the active list instead of evict.
2652 : *
2653 : * nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan
2654 : * nr[2] = file inactive pages to scan; nr[3] = file active pages to scan
2655 : */
2656 0 : static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc,
2657 : unsigned long *nr)
2658 : {
2659 0 : struct pglist_data *pgdat = lruvec_pgdat(lruvec);
2660 0 : struct mem_cgroup *memcg = lruvec_memcg(lruvec);
2661 : unsigned long anon_cost, file_cost, total_cost;
2662 0 : int swappiness = mem_cgroup_swappiness(memcg);
2663 : u64 fraction[ANON_AND_FILE];
2664 0 : u64 denominator = 0; /* gcc */
2665 : enum scan_balance scan_balance;
2666 : unsigned long ap, fp;
2667 : enum lru_list lru;
2668 :
2669 : /* If we have no swap space, do not bother scanning anon pages. */
2670 0 : if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) {
2671 : scan_balance = SCAN_FILE;
2672 : goto out;
2673 : }
2674 :
2675 : /*
2676 : * Global reclaim will swap to prevent OOM even with no
2677 : * swappiness, but memcg users want to use this knob to
2678 : * disable swapping for individual groups completely when
2679 : * using the memory controller's swap limit feature would be
2680 : * too expensive.
2681 : */
2682 0 : if (cgroup_reclaim(sc) && !swappiness) {
2683 : scan_balance = SCAN_FILE;
2684 : goto out;
2685 : }
2686 :
2687 : /*
2688 : * Do not apply any pressure balancing cleverness when the
2689 : * system is close to OOM, scan both anon and file equally
2690 : * (unless the swappiness setting disagrees with swapping).
2691 : */
2692 0 : if (!sc->priority && swappiness) {
2693 : scan_balance = SCAN_EQUAL;
2694 : goto out;
2695 : }
2696 :
2697 : /*
2698 : * If the system is almost out of file pages, force-scan anon.
2699 : */
2700 0 : if (sc->file_is_tiny) {
2701 : scan_balance = SCAN_ANON;
2702 : goto out;
2703 : }
2704 :
2705 : /*
2706 : * If there is enough inactive page cache, we do not reclaim
2707 : * anything from the anonymous working right now.
2708 : */
2709 0 : if (sc->cache_trim_mode) {
2710 : scan_balance = SCAN_FILE;
2711 : goto out;
2712 : }
2713 :
2714 0 : scan_balance = SCAN_FRACT;
2715 : /*
2716 : * Calculate the pressure balance between anon and file pages.
2717 : *
2718 : * The amount of pressure we put on each LRU is inversely
2719 : * proportional to the cost of reclaiming each list, as
2720 : * determined by the share of pages that are refaulting, times
2721 : * the relative IO cost of bringing back a swapped out
2722 : * anonymous page vs reloading a filesystem page (swappiness).
2723 : *
2724 : * Although we limit that influence to ensure no list gets
2725 : * left behind completely: at least a third of the pressure is
2726 : * applied, before swappiness.
2727 : *
2728 : * With swappiness at 100, anon and file have equal IO cost.
2729 : */
2730 0 : total_cost = sc->anon_cost + sc->file_cost;
2731 0 : anon_cost = total_cost + sc->anon_cost;
2732 0 : file_cost = total_cost + sc->file_cost;
2733 0 : total_cost = anon_cost + file_cost;
2734 :
2735 0 : ap = swappiness * (total_cost + 1);
2736 0 : ap /= anon_cost + 1;
2737 :
2738 0 : fp = (200 - swappiness) * (total_cost + 1);
2739 0 : fp /= file_cost + 1;
2740 :
2741 0 : fraction[0] = ap;
2742 0 : fraction[1] = fp;
2743 0 : denominator = ap + fp;
2744 : out:
2745 0 : for_each_evictable_lru(lru) {
2746 0 : int file = is_file_lru(lru);
2747 : unsigned long lruvec_size;
2748 : unsigned long low, min;
2749 : unsigned long scan;
2750 :
2751 0 : lruvec_size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx);
2752 0 : mem_cgroup_protection(sc->target_mem_cgroup, memcg,
2753 : &min, &low);
2754 :
2755 : if (min || low) {
2756 : /*
2757 : * Scale a cgroup's reclaim pressure by proportioning
2758 : * its current usage to its memory.low or memory.min
2759 : * setting.
2760 : *
2761 : * This is important, as otherwise scanning aggression
2762 : * becomes extremely binary -- from nothing as we
2763 : * approach the memory protection threshold, to totally
2764 : * nominal as we exceed it. This results in requiring
2765 : * setting extremely liberal protection thresholds. It
2766 : * also means we simply get no protection at all if we
2767 : * set it too low, which is not ideal.
2768 : *
2769 : * If there is any protection in place, we reduce scan
2770 : * pressure by how much of the total memory used is
2771 : * within protection thresholds.
2772 : *
2773 : * There is one special case: in the first reclaim pass,
2774 : * we skip over all groups that are within their low
2775 : * protection. If that fails to reclaim enough pages to
2776 : * satisfy the reclaim goal, we come back and override
2777 : * the best-effort low protection. However, we still
2778 : * ideally want to honor how well-behaved groups are in
2779 : * that case instead of simply punishing them all
2780 : * equally. As such, we reclaim them based on how much
2781 : * memory they are using, reducing the scan pressure
2782 : * again by how much of the total memory used is under
2783 : * hard protection.
2784 : */
2785 : unsigned long cgroup_size = mem_cgroup_size(memcg);
2786 : unsigned long protection;
2787 :
2788 : /* memory.low scaling, make sure we retry before OOM */
2789 : if (!sc->memcg_low_reclaim && low > min) {
2790 : protection = low;
2791 : sc->memcg_low_skipped = 1;
2792 : } else {
2793 : protection = min;
2794 : }
2795 :
2796 : /* Avoid TOCTOU with earlier protection check */
2797 : cgroup_size = max(cgroup_size, protection);
2798 :
2799 : scan = lruvec_size - lruvec_size * protection /
2800 : (cgroup_size + 1);
2801 :
2802 : /*
2803 : * Minimally target SWAP_CLUSTER_MAX pages to keep
2804 : * reclaim moving forwards, avoiding decrementing
2805 : * sc->priority further than desirable.
2806 : */
2807 : scan = max(scan, SWAP_CLUSTER_MAX);
2808 : } else {
2809 0 : scan = lruvec_size;
2810 : }
2811 :
2812 0 : scan >>= sc->priority;
2813 :
2814 : /*
2815 : * If the cgroup's already been deleted, make sure to
2816 : * scrape out the remaining cache.
2817 : */
2818 : if (!scan && !mem_cgroup_online(memcg))
2819 : scan = min(lruvec_size, SWAP_CLUSTER_MAX);
2820 :
2821 0 : switch (scan_balance) {
2822 : case SCAN_EQUAL:
2823 : /* Scan lists relative to size */
2824 : break;
2825 : case SCAN_FRACT:
2826 : /*
2827 : * Scan types proportional to swappiness and
2828 : * their relative recent reclaim efficiency.
2829 : * Make sure we don't miss the last page on
2830 : * the offlined memory cgroups because of a
2831 : * round-off error.
2832 : */
2833 0 : scan = mem_cgroup_online(memcg) ?
2834 0 : div64_u64(scan * fraction[file], denominator) :
2835 : DIV64_U64_ROUND_UP(scan * fraction[file],
2836 : denominator);
2837 0 : break;
2838 : case SCAN_FILE:
2839 : case SCAN_ANON:
2840 : /* Scan one type exclusively */
2841 0 : if ((scan_balance == SCAN_FILE) != file)
2842 0 : scan = 0;
2843 : break;
2844 : default:
2845 : /* Look ma, no brain */
2846 0 : BUG();
2847 : }
2848 :
2849 0 : nr[lru] = scan;
2850 : }
2851 0 : }
2852 :
2853 : /*
2854 : * Anonymous LRU management is a waste if there is
2855 : * ultimately no way to reclaim the memory.
2856 : */
2857 : static bool can_age_anon_pages(struct pglist_data *pgdat,
2858 : struct scan_control *sc)
2859 : {
2860 : /* Aging the anon LRU is valuable if swap is present: */
2861 0 : if (total_swap_pages > 0)
2862 : return true;
2863 :
2864 : /* Also valuable if anon pages can be demoted: */
2865 0 : return can_demote(pgdat->node_id, sc);
2866 : }
2867 :
2868 0 : static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
2869 : {
2870 : unsigned long nr[NR_LRU_LISTS];
2871 : unsigned long targets[NR_LRU_LISTS];
2872 : unsigned long nr_to_scan;
2873 : enum lru_list lru;
2874 0 : unsigned long nr_reclaimed = 0;
2875 0 : unsigned long nr_to_reclaim = sc->nr_to_reclaim;
2876 : struct blk_plug plug;
2877 : bool scan_adjusted;
2878 :
2879 0 : get_scan_count(lruvec, sc, nr);
2880 :
2881 : /* Record the original scan target for proportional adjustments later */
2882 0 : memcpy(targets, nr, sizeof(nr));
2883 :
2884 : /*
2885 : * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal
2886 : * event that can occur when there is little memory pressure e.g.
2887 : * multiple streaming readers/writers. Hence, we do not abort scanning
2888 : * when the requested number of pages are reclaimed when scanning at
2889 : * DEF_PRIORITY on the assumption that the fact we are direct
2890 : * reclaiming implies that kswapd is not keeping up and it is best to
2891 : * do a batch of work at once. For memcg reclaim one check is made to
2892 : * abort proportional reclaim if either the file or anon lru has already
2893 : * dropped to zero at the first pass.
2894 : */
2895 0 : scan_adjusted = (!cgroup_reclaim(sc) && !current_is_kswapd() &&
2896 0 : sc->priority == DEF_PRIORITY);
2897 :
2898 0 : blk_start_plug(&plug);
2899 0 : while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] ||
2900 0 : nr[LRU_INACTIVE_FILE]) {
2901 : unsigned long nr_anon, nr_file, percentage;
2902 : unsigned long nr_scanned;
2903 :
2904 0 : for_each_evictable_lru(lru) {
2905 0 : if (nr[lru]) {
2906 0 : nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX);
2907 0 : nr[lru] -= nr_to_scan;
2908 :
2909 0 : nr_reclaimed += shrink_list(lru, nr_to_scan,
2910 : lruvec, sc);
2911 : }
2912 : }
2913 :
2914 0 : cond_resched();
2915 :
2916 0 : if (nr_reclaimed < nr_to_reclaim || scan_adjusted)
2917 0 : continue;
2918 :
2919 : /*
2920 : * For kswapd and memcg, reclaim at least the number of pages
2921 : * requested. Ensure that the anon and file LRUs are scanned
2922 : * proportionally what was requested by get_scan_count(). We
2923 : * stop reclaiming one LRU and reduce the amount scanning
2924 : * proportional to the original scan target.
2925 : */
2926 0 : nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE];
2927 0 : nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON];
2928 :
2929 : /*
2930 : * It's just vindictive to attack the larger once the smaller
2931 : * has gone to zero. And given the way we stop scanning the
2932 : * smaller below, this makes sure that we only make one nudge
2933 : * towards proportionality once we've got nr_to_reclaim.
2934 : */
2935 0 : if (!nr_file || !nr_anon)
2936 : break;
2937 :
2938 0 : if (nr_file > nr_anon) {
2939 0 : unsigned long scan_target = targets[LRU_INACTIVE_ANON] +
2940 0 : targets[LRU_ACTIVE_ANON] + 1;
2941 0 : lru = LRU_BASE;
2942 0 : percentage = nr_anon * 100 / scan_target;
2943 : } else {
2944 0 : unsigned long scan_target = targets[LRU_INACTIVE_FILE] +
2945 0 : targets[LRU_ACTIVE_FILE] + 1;
2946 0 : lru = LRU_FILE;
2947 0 : percentage = nr_file * 100 / scan_target;
2948 : }
2949 :
2950 : /* Stop scanning the smaller of the LRU */
2951 0 : nr[lru] = 0;
2952 0 : nr[lru + LRU_ACTIVE] = 0;
2953 :
2954 : /*
2955 : * Recalculate the other LRU scan count based on its original
2956 : * scan target and the percentage scanning already complete
2957 : */
2958 0 : lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE;
2959 0 : nr_scanned = targets[lru] - nr[lru];
2960 0 : nr[lru] = targets[lru] * (100 - percentage) / 100;
2961 0 : nr[lru] -= min(nr[lru], nr_scanned);
2962 :
2963 0 : lru += LRU_ACTIVE;
2964 0 : nr_scanned = targets[lru] - nr[lru];
2965 0 : nr[lru] = targets[lru] * (100 - percentage) / 100;
2966 0 : nr[lru] -= min(nr[lru], nr_scanned);
2967 :
2968 0 : scan_adjusted = true;
2969 : }
2970 0 : blk_finish_plug(&plug);
2971 0 : sc->nr_reclaimed += nr_reclaimed;
2972 :
2973 : /*
2974 : * Even if we did not try to evict anon pages at all, we want to
2975 : * rebalance the anon lru active/inactive ratio.
2976 : */
2977 0 : if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) &&
2978 0 : inactive_is_low(lruvec, LRU_INACTIVE_ANON))
2979 0 : shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
2980 : sc, LRU_ACTIVE_ANON);
2981 0 : }
2982 :
2983 : /* Use reclaim/compaction for costly allocs or under memory pressure */
2984 : static bool in_reclaim_compaction(struct scan_control *sc)
2985 : {
2986 0 : if (IS_ENABLED(CONFIG_COMPACTION) && sc->order &&
2987 0 : (sc->order > PAGE_ALLOC_COSTLY_ORDER ||
2988 0 : sc->priority < DEF_PRIORITY - 2))
2989 : return true;
2990 :
2991 : return false;
2992 : }
2993 :
2994 : /*
2995 : * Reclaim/compaction is used for high-order allocation requests. It reclaims
2996 : * order-0 pages before compacting the zone. should_continue_reclaim() returns
2997 : * true if more pages should be reclaimed such that when the page allocator
2998 : * calls try_to_compact_pages() that it will have enough free pages to succeed.
2999 : * It will give up earlier than that if there is difficulty reclaiming pages.
3000 : */
3001 0 : static inline bool should_continue_reclaim(struct pglist_data *pgdat,
3002 : unsigned long nr_reclaimed,
3003 : struct scan_control *sc)
3004 : {
3005 : unsigned long pages_for_compaction;
3006 : unsigned long inactive_lru_pages;
3007 : int z;
3008 :
3009 : /* If not in reclaim/compaction mode, stop */
3010 0 : if (!in_reclaim_compaction(sc))
3011 : return false;
3012 :
3013 : /*
3014 : * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX
3015 : * number of pages that were scanned. This will return to the caller
3016 : * with the risk reclaim/compaction and the resulting allocation attempt
3017 : * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL
3018 : * allocations through requiring that the full LRU list has been scanned
3019 : * first, by assuming that zero delta of sc->nr_scanned means full LRU
3020 : * scan, but that approximation was wrong, and there were corner cases
3021 : * where always a non-zero amount of pages were scanned.
3022 : */
3023 0 : if (!nr_reclaimed)
3024 : return false;
3025 :
3026 : /* If compaction would go ahead or the allocation would succeed, stop */
3027 0 : for (z = 0; z <= sc->reclaim_idx; z++) {
3028 0 : struct zone *zone = &pgdat->node_zones[z];
3029 0 : if (!managed_zone(zone))
3030 0 : continue;
3031 :
3032 0 : switch (compaction_suitable(zone, sc->order, 0, sc->reclaim_idx)) {
3033 : case COMPACT_SUCCESS:
3034 : case COMPACT_CONTINUE:
3035 : return false;
3036 : default:
3037 : /* check next zone */
3038 : ;
3039 : }
3040 : }
3041 :
3042 : /*
3043 : * If we have not reclaimed enough pages for compaction and the
3044 : * inactive lists are large enough, continue reclaiming
3045 : */
3046 0 : pages_for_compaction = compact_gap(sc->order);
3047 0 : inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE);
3048 0 : if (can_reclaim_anon_pages(NULL, pgdat->node_id, sc))
3049 0 : inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON);
3050 :
3051 0 : return inactive_lru_pages > pages_for_compaction;
3052 : }
3053 :
3054 0 : static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc)
3055 : {
3056 0 : struct mem_cgroup *target_memcg = sc->target_mem_cgroup;
3057 : struct mem_cgroup *memcg;
3058 :
3059 0 : memcg = mem_cgroup_iter(target_memcg, NULL, NULL);
3060 : do {
3061 0 : struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
3062 : unsigned long reclaimed;
3063 : unsigned long scanned;
3064 :
3065 : /*
3066 : * This loop can become CPU-bound when target memcgs
3067 : * aren't eligible for reclaim - either because they
3068 : * don't have any reclaimable pages, or because their
3069 : * memory is explicitly protected. Avoid soft lockups.
3070 : */
3071 0 : cond_resched();
3072 :
3073 0 : mem_cgroup_calculate_protection(target_memcg, memcg);
3074 :
3075 0 : if (mem_cgroup_below_min(memcg)) {
3076 : /*
3077 : * Hard protection.
3078 : * If there is no reclaimable memory, OOM.
3079 : */
3080 : continue;
3081 0 : } else if (mem_cgroup_below_low(memcg)) {
3082 : /*
3083 : * Soft protection.
3084 : * Respect the protection only as long as
3085 : * there is an unprotected supply
3086 : * of reclaimable memory from other cgroups.
3087 : */
3088 : if (!sc->memcg_low_reclaim) {
3089 : sc->memcg_low_skipped = 1;
3090 : continue;
3091 : }
3092 : memcg_memory_event(memcg, MEMCG_LOW);
3093 : }
3094 :
3095 0 : reclaimed = sc->nr_reclaimed;
3096 0 : scanned = sc->nr_scanned;
3097 :
3098 0 : shrink_lruvec(lruvec, sc);
3099 :
3100 0 : shrink_slab(sc->gfp_mask, pgdat->node_id, memcg,
3101 0 : sc->priority);
3102 :
3103 : /* Record the group's reclaim efficiency */
3104 0 : vmpressure(sc->gfp_mask, memcg, false,
3105 0 : sc->nr_scanned - scanned,
3106 0 : sc->nr_reclaimed - reclaimed);
3107 :
3108 0 : } while ((memcg = mem_cgroup_iter(target_memcg, memcg, NULL)));
3109 0 : }
3110 :
3111 0 : static void shrink_node(pg_data_t *pgdat, struct scan_control *sc)
3112 : {
3113 0 : struct reclaim_state *reclaim_state = current->reclaim_state;
3114 : unsigned long nr_reclaimed, nr_scanned;
3115 : struct lruvec *target_lruvec;
3116 0 : bool reclaimable = false;
3117 : unsigned long file;
3118 :
3119 0 : target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat);
3120 :
3121 : again:
3122 : /*
3123 : * Flush the memory cgroup stats, so that we read accurate per-memcg
3124 : * lruvec stats for heuristics.
3125 : */
3126 : mem_cgroup_flush_stats();
3127 :
3128 0 : memset(&sc->nr, 0, sizeof(sc->nr));
3129 :
3130 0 : nr_reclaimed = sc->nr_reclaimed;
3131 0 : nr_scanned = sc->nr_scanned;
3132 :
3133 : /*
3134 : * Determine the scan balance between anon and file LRUs.
3135 : */
3136 0 : spin_lock_irq(&target_lruvec->lru_lock);
3137 0 : sc->anon_cost = target_lruvec->anon_cost;
3138 0 : sc->file_cost = target_lruvec->file_cost;
3139 0 : spin_unlock_irq(&target_lruvec->lru_lock);
3140 :
3141 : /*
3142 : * Target desirable inactive:active list ratios for the anon
3143 : * and file LRU lists.
3144 : */
3145 0 : if (!sc->force_deactivate) {
3146 : unsigned long refaults;
3147 :
3148 0 : refaults = lruvec_page_state(target_lruvec,
3149 : WORKINGSET_ACTIVATE_ANON);
3150 0 : if (refaults != target_lruvec->refaults[0] ||
3151 0 : inactive_is_low(target_lruvec, LRU_INACTIVE_ANON))
3152 0 : sc->may_deactivate |= DEACTIVATE_ANON;
3153 : else
3154 0 : sc->may_deactivate &= ~DEACTIVATE_ANON;
3155 :
3156 : /*
3157 : * When refaults are being observed, it means a new
3158 : * workingset is being established. Deactivate to get
3159 : * rid of any stale active pages quickly.
3160 : */
3161 0 : refaults = lruvec_page_state(target_lruvec,
3162 : WORKINGSET_ACTIVATE_FILE);
3163 0 : if (refaults != target_lruvec->refaults[1] ||
3164 0 : inactive_is_low(target_lruvec, LRU_INACTIVE_FILE))
3165 0 : sc->may_deactivate |= DEACTIVATE_FILE;
3166 : else
3167 0 : sc->may_deactivate &= ~DEACTIVATE_FILE;
3168 : } else
3169 0 : sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE;
3170 :
3171 : /*
3172 : * If we have plenty of inactive file pages that aren't
3173 : * thrashing, try to reclaim those first before touching
3174 : * anonymous pages.
3175 : */
3176 0 : file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE);
3177 0 : if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE))
3178 0 : sc->cache_trim_mode = 1;
3179 : else
3180 0 : sc->cache_trim_mode = 0;
3181 :
3182 : /*
3183 : * Prevent the reclaimer from falling into the cache trap: as
3184 : * cache pages start out inactive, every cache fault will tip
3185 : * the scan balance towards the file LRU. And as the file LRU
3186 : * shrinks, so does the window for rotation from references.
3187 : * This means we have a runaway feedback loop where a tiny
3188 : * thrashing file LRU becomes infinitely more attractive than
3189 : * anon pages. Try to detect this based on file LRU size.
3190 : */
3191 0 : if (!cgroup_reclaim(sc)) {
3192 0 : unsigned long total_high_wmark = 0;
3193 : unsigned long free, anon;
3194 : int z;
3195 :
3196 0 : free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES);
3197 0 : file = node_page_state(pgdat, NR_ACTIVE_FILE) +
3198 0 : node_page_state(pgdat, NR_INACTIVE_FILE);
3199 :
3200 0 : for (z = 0; z < MAX_NR_ZONES; z++) {
3201 0 : struct zone *zone = &pgdat->node_zones[z];
3202 0 : if (!managed_zone(zone))
3203 0 : continue;
3204 :
3205 0 : total_high_wmark += high_wmark_pages(zone);
3206 : }
3207 :
3208 : /*
3209 : * Consider anon: if that's low too, this isn't a
3210 : * runaway file reclaim problem, but rather just
3211 : * extreme pressure. Reclaim as per usual then.
3212 : */
3213 0 : anon = node_page_state(pgdat, NR_INACTIVE_ANON);
3214 :
3215 0 : sc->file_is_tiny =
3216 0 : file + free <= total_high_wmark &&
3217 0 : !(sc->may_deactivate & DEACTIVATE_ANON) &&
3218 0 : anon >> sc->priority;
3219 : }
3220 :
3221 0 : shrink_node_memcgs(pgdat, sc);
3222 :
3223 0 : if (reclaim_state) {
3224 0 : sc->nr_reclaimed += reclaim_state->reclaimed_slab;
3225 0 : reclaim_state->reclaimed_slab = 0;
3226 : }
3227 :
3228 : /* Record the subtree's reclaim efficiency */
3229 0 : vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true,
3230 0 : sc->nr_scanned - nr_scanned,
3231 0 : sc->nr_reclaimed - nr_reclaimed);
3232 :
3233 0 : if (sc->nr_reclaimed - nr_reclaimed)
3234 0 : reclaimable = true;
3235 :
3236 0 : if (current_is_kswapd()) {
3237 : /*
3238 : * If reclaim is isolating dirty pages under writeback,
3239 : * it implies that the long-lived page allocation rate
3240 : * is exceeding the page laundering rate. Either the
3241 : * global limits are not being effective at throttling
3242 : * processes due to the page distribution throughout
3243 : * zones or there is heavy usage of a slow backing
3244 : * device. The only option is to throttle from reclaim
3245 : * context which is not ideal as there is no guarantee
3246 : * the dirtying process is throttled in the same way
3247 : * balance_dirty_pages() manages.
3248 : *
3249 : * Once a node is flagged PGDAT_WRITEBACK, kswapd will
3250 : * count the number of pages under pages flagged for
3251 : * immediate reclaim and stall if any are encountered
3252 : * in the nr_immediate check below.
3253 : */
3254 0 : if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken)
3255 0 : set_bit(PGDAT_WRITEBACK, &pgdat->flags);
3256 :
3257 : /* Allow kswapd to start writing pages during reclaim.*/
3258 0 : if (sc->nr.unqueued_dirty == sc->nr.file_taken)
3259 0 : set_bit(PGDAT_DIRTY, &pgdat->flags);
3260 :
3261 : /*
3262 : * If kswapd scans pages marked for immediate
3263 : * reclaim and under writeback (nr_immediate), it
3264 : * implies that pages are cycling through the LRU
3265 : * faster than they are written so forcibly stall
3266 : * until some pages complete writeback.
3267 : */
3268 0 : if (sc->nr.immediate)
3269 0 : reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK);
3270 : }
3271 :
3272 : /*
3273 : * Tag a node/memcg as congested if all the dirty pages were marked
3274 : * for writeback and immediate reclaim (counted in nr.congested).
3275 : *
3276 : * Legacy memcg will stall in page writeback so avoid forcibly
3277 : * stalling in reclaim_throttle().
3278 : */
3279 0 : if ((current_is_kswapd() ||
3280 0 : (cgroup_reclaim(sc) && writeback_throttling_sane(sc))) &&
3281 0 : sc->nr.dirty && sc->nr.dirty == sc->nr.congested)
3282 0 : set_bit(LRUVEC_CONGESTED, &target_lruvec->flags);
3283 :
3284 : /*
3285 : * Stall direct reclaim for IO completions if the lruvec is
3286 : * node is congested. Allow kswapd to continue until it
3287 : * starts encountering unqueued dirty pages or cycling through
3288 : * the LRU too quickly.
3289 : */
3290 0 : if (!current_is_kswapd() && current_may_throttle() &&
3291 0 : !sc->hibernation_mode &&
3292 0 : test_bit(LRUVEC_CONGESTED, &target_lruvec->flags))
3293 0 : reclaim_throttle(pgdat, VMSCAN_THROTTLE_CONGESTED);
3294 :
3295 0 : if (should_continue_reclaim(pgdat, sc->nr_reclaimed - nr_reclaimed,
3296 : sc))
3297 : goto again;
3298 :
3299 : /*
3300 : * Kswapd gives up on balancing particular nodes after too
3301 : * many failures to reclaim anything from them and goes to
3302 : * sleep. On reclaim progress, reset the failure counter. A
3303 : * successful direct reclaim run will revive a dormant kswapd.
3304 : */
3305 0 : if (reclaimable)
3306 0 : pgdat->kswapd_failures = 0;
3307 0 : }
3308 :
3309 : /*
3310 : * Returns true if compaction should go ahead for a costly-order request, or
3311 : * the allocation would already succeed without compaction. Return false if we
3312 : * should reclaim first.
3313 : */
3314 0 : static inline bool compaction_ready(struct zone *zone, struct scan_control *sc)
3315 : {
3316 : unsigned long watermark;
3317 : enum compact_result suitable;
3318 :
3319 0 : suitable = compaction_suitable(zone, sc->order, 0, sc->reclaim_idx);
3320 0 : if (suitable == COMPACT_SUCCESS)
3321 : /* Allocation should succeed already. Don't reclaim. */
3322 : return true;
3323 0 : if (suitable == COMPACT_SKIPPED)
3324 : /* Compaction cannot yet proceed. Do reclaim. */
3325 : return false;
3326 :
3327 : /*
3328 : * Compaction is already possible, but it takes time to run and there
3329 : * are potentially other callers using the pages just freed. So proceed
3330 : * with reclaim to make a buffer of free pages available to give
3331 : * compaction a reasonable chance of completing and allocating the page.
3332 : * Note that we won't actually reclaim the whole buffer in one attempt
3333 : * as the target watermark in should_continue_reclaim() is lower. But if
3334 : * we are already above the high+gap watermark, don't reclaim at all.
3335 : */
3336 0 : watermark = high_wmark_pages(zone) + compact_gap(sc->order);
3337 :
3338 0 : return zone_watermark_ok_safe(zone, 0, watermark, sc->reclaim_idx);
3339 : }
3340 :
3341 0 : static void consider_reclaim_throttle(pg_data_t *pgdat, struct scan_control *sc)
3342 : {
3343 : /*
3344 : * If reclaim is making progress greater than 12% efficiency then
3345 : * wake all the NOPROGRESS throttled tasks.
3346 : */
3347 0 : if (sc->nr_reclaimed > (sc->nr_scanned >> 3)) {
3348 : wait_queue_head_t *wqh;
3349 :
3350 0 : wqh = &pgdat->reclaim_wait[VMSCAN_THROTTLE_NOPROGRESS];
3351 0 : if (waitqueue_active(wqh))
3352 0 : wake_up(wqh);
3353 :
3354 : return;
3355 : }
3356 :
3357 : /*
3358 : * Do not throttle kswapd or cgroup reclaim on NOPROGRESS as it will
3359 : * throttle on VMSCAN_THROTTLE_WRITEBACK if there are too many pages
3360 : * under writeback and marked for immediate reclaim at the tail of the
3361 : * LRU.
3362 : */
3363 0 : if (current_is_kswapd() || cgroup_reclaim(sc))
3364 : return;
3365 :
3366 : /* Throttle if making no progress at high prioities. */
3367 0 : if (sc->priority == 1 && !sc->nr_reclaimed)
3368 0 : reclaim_throttle(pgdat, VMSCAN_THROTTLE_NOPROGRESS);
3369 : }
3370 :
3371 : /*
3372 : * This is the direct reclaim path, for page-allocating processes. We only
3373 : * try to reclaim pages from zones which will satisfy the caller's allocation
3374 : * request.
3375 : *
3376 : * If a zone is deemed to be full of pinned pages then just give it a light
3377 : * scan then give up on it.
3378 : */
3379 0 : static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc)
3380 : {
3381 : struct zoneref *z;
3382 : struct zone *zone;
3383 : unsigned long nr_soft_reclaimed;
3384 : unsigned long nr_soft_scanned;
3385 : gfp_t orig_mask;
3386 0 : pg_data_t *last_pgdat = NULL;
3387 0 : pg_data_t *first_pgdat = NULL;
3388 :
3389 : /*
3390 : * If the number of buffer_heads in the machine exceeds the maximum
3391 : * allowed level, force direct reclaim to scan the highmem zone as
3392 : * highmem pages could be pinning lowmem pages storing buffer_heads
3393 : */
3394 0 : orig_mask = sc->gfp_mask;
3395 0 : if (buffer_heads_over_limit) {
3396 0 : sc->gfp_mask |= __GFP_HIGHMEM;
3397 0 : sc->reclaim_idx = gfp_zone(sc->gfp_mask);
3398 : }
3399 :
3400 0 : for_each_zone_zonelist_nodemask(zone, z, zonelist,
3401 : sc->reclaim_idx, sc->nodemask) {
3402 : /*
3403 : * Take care memory controller reclaiming has small influence
3404 : * to global LRU.
3405 : */
3406 0 : if (!cgroup_reclaim(sc)) {
3407 0 : if (!cpuset_zone_allowed(zone,
3408 : GFP_KERNEL | __GFP_HARDWALL))
3409 : continue;
3410 :
3411 : /*
3412 : * If we already have plenty of memory free for
3413 : * compaction in this zone, don't free any more.
3414 : * Even though compaction is invoked for any
3415 : * non-zero order, only frequent costly order
3416 : * reclamation is disruptive enough to become a
3417 : * noticeable problem, like transparent huge
3418 : * page allocations.
3419 : */
3420 0 : if (IS_ENABLED(CONFIG_COMPACTION) &&
3421 0 : sc->order > PAGE_ALLOC_COSTLY_ORDER &&
3422 0 : compaction_ready(zone, sc)) {
3423 0 : sc->compaction_ready = true;
3424 0 : continue;
3425 : }
3426 :
3427 : /*
3428 : * Shrink each node in the zonelist once. If the
3429 : * zonelist is ordered by zone (not the default) then a
3430 : * node may be shrunk multiple times but in that case
3431 : * the user prefers lower zones being preserved.
3432 : */
3433 0 : if (zone->zone_pgdat == last_pgdat)
3434 0 : continue;
3435 :
3436 : /*
3437 : * This steals pages from memory cgroups over softlimit
3438 : * and returns the number of reclaimed pages and
3439 : * scanned pages. This works for global memory pressure
3440 : * and balancing, not for a memcg's limit.
3441 : */
3442 0 : nr_soft_scanned = 0;
3443 0 : nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(zone->zone_pgdat,
3444 0 : sc->order, sc->gfp_mask,
3445 : &nr_soft_scanned);
3446 : sc->nr_reclaimed += nr_soft_reclaimed;
3447 : sc->nr_scanned += nr_soft_scanned;
3448 : /* need some check for avoid more shrink_zone() */
3449 : }
3450 :
3451 0 : if (!first_pgdat)
3452 0 : first_pgdat = zone->zone_pgdat;
3453 :
3454 : /* See comment about same check for global reclaim above */
3455 : if (zone->zone_pgdat == last_pgdat)
3456 : continue;
3457 0 : last_pgdat = zone->zone_pgdat;
3458 0 : shrink_node(zone->zone_pgdat, sc);
3459 : }
3460 :
3461 0 : if (first_pgdat)
3462 0 : consider_reclaim_throttle(first_pgdat, sc);
3463 :
3464 : /*
3465 : * Restore to original mask to avoid the impact on the caller if we
3466 : * promoted it to __GFP_HIGHMEM.
3467 : */
3468 0 : sc->gfp_mask = orig_mask;
3469 0 : }
3470 :
3471 : static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat)
3472 : {
3473 : struct lruvec *target_lruvec;
3474 : unsigned long refaults;
3475 :
3476 0 : target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
3477 0 : refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON);
3478 0 : target_lruvec->refaults[0] = refaults;
3479 0 : refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_FILE);
3480 0 : target_lruvec->refaults[1] = refaults;
3481 : }
3482 :
3483 : /*
3484 : * This is the main entry point to direct page reclaim.
3485 : *
3486 : * If a full scan of the inactive list fails to free enough memory then we
3487 : * are "out of memory" and something needs to be killed.
3488 : *
3489 : * If the caller is !__GFP_FS then the probability of a failure is reasonably
3490 : * high - the zone may be full of dirty or under-writeback pages, which this
3491 : * caller can't do much about. We kick the writeback threads and take explicit
3492 : * naps in the hope that some of these pages can be written. But if the
3493 : * allocating task holds filesystem locks which prevent writeout this might not
3494 : * work, and the allocation attempt will fail.
3495 : *
3496 : * returns: 0, if no pages reclaimed
3497 : * else, the number of pages reclaimed
3498 : */
3499 0 : static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
3500 : struct scan_control *sc)
3501 : {
3502 0 : int initial_priority = sc->priority;
3503 : pg_data_t *last_pgdat;
3504 : struct zoneref *z;
3505 : struct zone *zone;
3506 : retry:
3507 : delayacct_freepages_start();
3508 :
3509 0 : if (!cgroup_reclaim(sc))
3510 0 : __count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1);
3511 :
3512 : do {
3513 0 : vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup,
3514 0 : sc->priority);
3515 0 : sc->nr_scanned = 0;
3516 0 : shrink_zones(zonelist, sc);
3517 :
3518 0 : if (sc->nr_reclaimed >= sc->nr_to_reclaim)
3519 : break;
3520 :
3521 0 : if (sc->compaction_ready)
3522 : break;
3523 :
3524 : /*
3525 : * If we're getting trouble reclaiming, start doing
3526 : * writepage even in laptop mode.
3527 : */
3528 0 : if (sc->priority < DEF_PRIORITY - 2)
3529 0 : sc->may_writepage = 1;
3530 0 : } while (--sc->priority >= 0);
3531 :
3532 0 : last_pgdat = NULL;
3533 0 : for_each_zone_zonelist_nodemask(zone, z, zonelist, sc->reclaim_idx,
3534 : sc->nodemask) {
3535 0 : if (zone->zone_pgdat == last_pgdat)
3536 0 : continue;
3537 0 : last_pgdat = zone->zone_pgdat;
3538 :
3539 0 : snapshot_refaults(sc->target_mem_cgroup, zone->zone_pgdat);
3540 :
3541 0 : if (cgroup_reclaim(sc)) {
3542 : struct lruvec *lruvec;
3543 :
3544 : lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup,
3545 : zone->zone_pgdat);
3546 : clear_bit(LRUVEC_CONGESTED, &lruvec->flags);
3547 : }
3548 : }
3549 :
3550 : delayacct_freepages_end();
3551 :
3552 0 : if (sc->nr_reclaimed)
3553 : return sc->nr_reclaimed;
3554 :
3555 : /* Aborted reclaim to try compaction? don't OOM, then */
3556 0 : if (sc->compaction_ready)
3557 : return 1;
3558 :
3559 : /*
3560 : * We make inactive:active ratio decisions based on the node's
3561 : * composition of memory, but a restrictive reclaim_idx or a
3562 : * memory.low cgroup setting can exempt large amounts of
3563 : * memory from reclaim. Neither of which are very common, so
3564 : * instead of doing costly eligibility calculations of the
3565 : * entire cgroup subtree up front, we assume the estimates are
3566 : * good, and retry with forcible deactivation if that fails.
3567 : */
3568 0 : if (sc->skipped_deactivate) {
3569 0 : sc->priority = initial_priority;
3570 0 : sc->force_deactivate = 1;
3571 0 : sc->skipped_deactivate = 0;
3572 0 : goto retry;
3573 : }
3574 :
3575 : /* Untapped cgroup reserves? Don't OOM, retry. */
3576 0 : if (sc->memcg_low_skipped) {
3577 0 : sc->priority = initial_priority;
3578 0 : sc->force_deactivate = 0;
3579 0 : sc->memcg_low_reclaim = 1;
3580 0 : sc->memcg_low_skipped = 0;
3581 0 : goto retry;
3582 : }
3583 :
3584 : return 0;
3585 : }
3586 :
3587 0 : static bool allow_direct_reclaim(pg_data_t *pgdat)
3588 : {
3589 : struct zone *zone;
3590 0 : unsigned long pfmemalloc_reserve = 0;
3591 0 : unsigned long free_pages = 0;
3592 : int i;
3593 : bool wmark_ok;
3594 :
3595 0 : if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
3596 : return true;
3597 :
3598 0 : for (i = 0; i <= ZONE_NORMAL; i++) {
3599 0 : zone = &pgdat->node_zones[i];
3600 0 : if (!managed_zone(zone))
3601 0 : continue;
3602 :
3603 0 : if (!zone_reclaimable_pages(zone))
3604 0 : continue;
3605 :
3606 0 : pfmemalloc_reserve += min_wmark_pages(zone);
3607 0 : free_pages += zone_page_state(zone, NR_FREE_PAGES);
3608 : }
3609 :
3610 : /* If there are no reserves (unexpected config) then do not throttle */
3611 0 : if (!pfmemalloc_reserve)
3612 : return true;
3613 :
3614 0 : wmark_ok = free_pages > pfmemalloc_reserve / 2;
3615 :
3616 : /* kswapd must be awake if processes are being throttled */
3617 0 : if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) {
3618 0 : if (READ_ONCE(pgdat->kswapd_highest_zoneidx) > ZONE_NORMAL)
3619 0 : WRITE_ONCE(pgdat->kswapd_highest_zoneidx, ZONE_NORMAL);
3620 :
3621 0 : wake_up_interruptible(&pgdat->kswapd_wait);
3622 : }
3623 :
3624 : return wmark_ok;
3625 : }
3626 :
3627 : /*
3628 : * Throttle direct reclaimers if backing storage is backed by the network
3629 : * and the PFMEMALLOC reserve for the preferred node is getting dangerously
3630 : * depleted. kswapd will continue to make progress and wake the processes
3631 : * when the low watermark is reached.
3632 : *
3633 : * Returns true if a fatal signal was delivered during throttling. If this
3634 : * happens, the page allocator should not consider triggering the OOM killer.
3635 : */
3636 0 : static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist,
3637 : nodemask_t *nodemask)
3638 : {
3639 : struct zoneref *z;
3640 : struct zone *zone;
3641 0 : pg_data_t *pgdat = NULL;
3642 :
3643 : /*
3644 : * Kernel threads should not be throttled as they may be indirectly
3645 : * responsible for cleaning pages necessary for reclaim to make forward
3646 : * progress. kjournald for example may enter direct reclaim while
3647 : * committing a transaction where throttling it could forcing other
3648 : * processes to block on log_wait_commit().
3649 : */
3650 0 : if (current->flags & PF_KTHREAD)
3651 : goto out;
3652 :
3653 : /*
3654 : * If a fatal signal is pending, this process should not throttle.
3655 : * It should return quickly so it can exit and free its memory
3656 : */
3657 0 : if (fatal_signal_pending(current))
3658 : goto out;
3659 :
3660 : /*
3661 : * Check if the pfmemalloc reserves are ok by finding the first node
3662 : * with a usable ZONE_NORMAL or lower zone. The expectation is that
3663 : * GFP_KERNEL will be required for allocating network buffers when
3664 : * swapping over the network so ZONE_HIGHMEM is unusable.
3665 : *
3666 : * Throttling is based on the first usable node and throttled processes
3667 : * wait on a queue until kswapd makes progress and wakes them. There
3668 : * is an affinity then between processes waking up and where reclaim
3669 : * progress has been made assuming the process wakes on the same node.
3670 : * More importantly, processes running on remote nodes will not compete
3671 : * for remote pfmemalloc reserves and processes on different nodes
3672 : * should make reasonable progress.
3673 : */
3674 0 : for_each_zone_zonelist_nodemask(zone, z, zonelist,
3675 : gfp_zone(gfp_mask), nodemask) {
3676 0 : if (zone_idx(zone) > ZONE_NORMAL)
3677 0 : continue;
3678 :
3679 : /* Throttle based on the first usable node */
3680 0 : pgdat = zone->zone_pgdat;
3681 0 : if (allow_direct_reclaim(pgdat))
3682 : goto out;
3683 : break;
3684 : }
3685 :
3686 : /* If no zone was usable by the allocation flags then do not throttle */
3687 0 : if (!pgdat)
3688 : goto out;
3689 :
3690 : /* Account for the throttling */
3691 0 : count_vm_event(PGSCAN_DIRECT_THROTTLE);
3692 :
3693 : /*
3694 : * If the caller cannot enter the filesystem, it's possible that it
3695 : * is due to the caller holding an FS lock or performing a journal
3696 : * transaction in the case of a filesystem like ext[3|4]. In this case,
3697 : * it is not safe to block on pfmemalloc_wait as kswapd could be
3698 : * blocked waiting on the same lock. Instead, throttle for up to a
3699 : * second before continuing.
3700 : */
3701 0 : if (!(gfp_mask & __GFP_FS))
3702 0 : wait_event_interruptible_timeout(pgdat->pfmemalloc_wait,
3703 : allow_direct_reclaim(pgdat), HZ);
3704 : else
3705 : /* Throttle until kswapd wakes the process */
3706 0 : wait_event_killable(zone->zone_pgdat->pfmemalloc_wait,
3707 : allow_direct_reclaim(pgdat));
3708 :
3709 0 : if (fatal_signal_pending(current))
3710 : return true;
3711 :
3712 : out:
3713 : return false;
3714 : }
3715 :
3716 0 : unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
3717 : gfp_t gfp_mask, nodemask_t *nodemask)
3718 : {
3719 : unsigned long nr_reclaimed;
3720 0 : struct scan_control sc = {
3721 : .nr_to_reclaim = SWAP_CLUSTER_MAX,
3722 0 : .gfp_mask = current_gfp_context(gfp_mask),
3723 0 : .reclaim_idx = gfp_zone(gfp_mask),
3724 : .order = order,
3725 : .nodemask = nodemask,
3726 : .priority = DEF_PRIORITY,
3727 0 : .may_writepage = !laptop_mode,
3728 : .may_unmap = 1,
3729 : .may_swap = 1,
3730 : };
3731 :
3732 : /*
3733 : * scan_control uses s8 fields for order, priority, and reclaim_idx.
3734 : * Confirm they are large enough for max values.
3735 : */
3736 : BUILD_BUG_ON(MAX_ORDER > S8_MAX);
3737 : BUILD_BUG_ON(DEF_PRIORITY > S8_MAX);
3738 : BUILD_BUG_ON(MAX_NR_ZONES > S8_MAX);
3739 :
3740 : /*
3741 : * Do not enter reclaim if fatal signal was delivered while throttled.
3742 : * 1 is returned so that the page allocator does not OOM kill at this
3743 : * point.
3744 : */
3745 0 : if (throttle_direct_reclaim(sc.gfp_mask, zonelist, nodemask))
3746 : return 1;
3747 :
3748 0 : set_task_reclaim_state(current, &sc.reclaim_state);
3749 0 : trace_mm_vmscan_direct_reclaim_begin(order, sc.gfp_mask);
3750 :
3751 0 : nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3752 :
3753 0 : trace_mm_vmscan_direct_reclaim_end(nr_reclaimed);
3754 0 : set_task_reclaim_state(current, NULL);
3755 :
3756 0 : return nr_reclaimed;
3757 : }
3758 :
3759 : #ifdef CONFIG_MEMCG
3760 :
3761 : /* Only used by soft limit reclaim. Do not reuse for anything else. */
3762 : unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg,
3763 : gfp_t gfp_mask, bool noswap,
3764 : pg_data_t *pgdat,
3765 : unsigned long *nr_scanned)
3766 : {
3767 : struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
3768 : struct scan_control sc = {
3769 : .nr_to_reclaim = SWAP_CLUSTER_MAX,
3770 : .target_mem_cgroup = memcg,
3771 : .may_writepage = !laptop_mode,
3772 : .may_unmap = 1,
3773 : .reclaim_idx = MAX_NR_ZONES - 1,
3774 : .may_swap = !noswap,
3775 : };
3776 :
3777 : WARN_ON_ONCE(!current->reclaim_state);
3778 :
3779 : sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
3780 : (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
3781 :
3782 : trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.order,
3783 : sc.gfp_mask);
3784 :
3785 : /*
3786 : * NOTE: Although we can get the priority field, using it
3787 : * here is not a good idea, since it limits the pages we can scan.
3788 : * if we don't reclaim here, the shrink_node from balance_pgdat
3789 : * will pick up pages from other mem cgroup's as well. We hack
3790 : * the priority and make it zero.
3791 : */
3792 : shrink_lruvec(lruvec, &sc);
3793 :
3794 : trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);
3795 :
3796 : *nr_scanned = sc.nr_scanned;
3797 :
3798 : return sc.nr_reclaimed;
3799 : }
3800 :
3801 : unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
3802 : unsigned long nr_pages,
3803 : gfp_t gfp_mask,
3804 : bool may_swap)
3805 : {
3806 : unsigned long nr_reclaimed;
3807 : unsigned int noreclaim_flag;
3808 : struct scan_control sc = {
3809 : .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
3810 : .gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) |
3811 : (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK),
3812 : .reclaim_idx = MAX_NR_ZONES - 1,
3813 : .target_mem_cgroup = memcg,
3814 : .priority = DEF_PRIORITY,
3815 : .may_writepage = !laptop_mode,
3816 : .may_unmap = 1,
3817 : .may_swap = may_swap,
3818 : };
3819 : /*
3820 : * Traverse the ZONELIST_FALLBACK zonelist of the current node to put
3821 : * equal pressure on all the nodes. This is based on the assumption that
3822 : * the reclaim does not bail out early.
3823 : */
3824 : struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
3825 :
3826 : set_task_reclaim_state(current, &sc.reclaim_state);
3827 : trace_mm_vmscan_memcg_reclaim_begin(0, sc.gfp_mask);
3828 : noreclaim_flag = memalloc_noreclaim_save();
3829 :
3830 : nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3831 :
3832 : memalloc_noreclaim_restore(noreclaim_flag);
3833 : trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed);
3834 : set_task_reclaim_state(current, NULL);
3835 :
3836 : return nr_reclaimed;
3837 : }
3838 : #endif
3839 :
3840 0 : static void age_active_anon(struct pglist_data *pgdat,
3841 : struct scan_control *sc)
3842 : {
3843 : struct mem_cgroup *memcg;
3844 : struct lruvec *lruvec;
3845 :
3846 0 : if (!can_age_anon_pages(pgdat, sc))
3847 : return;
3848 :
3849 0 : lruvec = mem_cgroup_lruvec(NULL, pgdat);
3850 0 : if (!inactive_is_low(lruvec, LRU_INACTIVE_ANON))
3851 : return;
3852 :
3853 0 : memcg = mem_cgroup_iter(NULL, NULL, NULL);
3854 : do {
3855 0 : lruvec = mem_cgroup_lruvec(memcg, pgdat);
3856 0 : shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
3857 : sc, LRU_ACTIVE_ANON);
3858 0 : memcg = mem_cgroup_iter(NULL, memcg, NULL);
3859 : } while (memcg);
3860 : }
3861 :
3862 : static bool pgdat_watermark_boosted(pg_data_t *pgdat, int highest_zoneidx)
3863 : {
3864 : int i;
3865 : struct zone *zone;
3866 :
3867 : /*
3868 : * Check for watermark boosts top-down as the higher zones
3869 : * are more likely to be boosted. Both watermarks and boosts
3870 : * should not be checked at the same time as reclaim would
3871 : * start prematurely when there is no boosting and a lower
3872 : * zone is balanced.
3873 : */
3874 0 : for (i = highest_zoneidx; i >= 0; i--) {
3875 0 : zone = pgdat->node_zones + i;
3876 0 : if (!managed_zone(zone))
3877 0 : continue;
3878 :
3879 0 : if (zone->watermark_boost)
3880 : return true;
3881 : }
3882 :
3883 : return false;
3884 : }
3885 :
3886 : /*
3887 : * Returns true if there is an eligible zone balanced for the request order
3888 : * and highest_zoneidx
3889 : */
3890 1 : static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx)
3891 : {
3892 : int i;
3893 1 : unsigned long mark = -1;
3894 : struct zone *zone;
3895 :
3896 : /*
3897 : * Check watermarks bottom-up as lower zones are more likely to
3898 : * meet watermarks.
3899 : */
3900 1 : for (i = 0; i <= highest_zoneidx; i++) {
3901 1 : zone = pgdat->node_zones + i;
3902 :
3903 1 : if (!managed_zone(zone))
3904 0 : continue;
3905 :
3906 : if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING)
3907 : mark = wmark_pages(zone, WMARK_PROMO);
3908 : else
3909 1 : mark = high_wmark_pages(zone);
3910 1 : if (zone_watermark_ok_safe(zone, order, mark, highest_zoneidx))
3911 : return true;
3912 : }
3913 :
3914 : /*
3915 : * If a node has no populated zone within highest_zoneidx, it does not
3916 : * need balancing by definition. This can happen if a zone-restricted
3917 : * allocation tries to wake a remote kswapd.
3918 : */
3919 0 : if (mark == -1)
3920 : return true;
3921 :
3922 0 : return false;
3923 : }
3924 :
3925 : /* Clear pgdat state for congested, dirty or under writeback. */
3926 : static void clear_pgdat_congested(pg_data_t *pgdat)
3927 : {
3928 1 : struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat);
3929 :
3930 2 : clear_bit(LRUVEC_CONGESTED, &lruvec->flags);
3931 2 : clear_bit(PGDAT_DIRTY, &pgdat->flags);
3932 2 : clear_bit(PGDAT_WRITEBACK, &pgdat->flags);
3933 : }
3934 :
3935 : /*
3936 : * Prepare kswapd for sleeping. This verifies that there are no processes
3937 : * waiting in throttle_direct_reclaim() and that watermarks have been met.
3938 : *
3939 : * Returns true if kswapd is ready to sleep
3940 : */
3941 1 : static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order,
3942 : int highest_zoneidx)
3943 : {
3944 : /*
3945 : * The throttled processes are normally woken up in balance_pgdat() as
3946 : * soon as allow_direct_reclaim() is true. But there is a potential
3947 : * race between when kswapd checks the watermarks and a process gets
3948 : * throttled. There is also a potential race if processes get
3949 : * throttled, kswapd wakes, a large process exits thereby balancing the
3950 : * zones, which causes kswapd to exit balance_pgdat() before reaching
3951 : * the wake up checks. If kswapd is going to sleep, no process should
3952 : * be sleeping on pfmemalloc_wait, so wake them now if necessary. If
3953 : * the wake up is premature, processes will wake kswapd and get
3954 : * throttled again. The difference from wake ups in balance_pgdat() is
3955 : * that here we are under prepare_to_wait().
3956 : */
3957 2 : if (waitqueue_active(&pgdat->pfmemalloc_wait))
3958 0 : wake_up_all(&pgdat->pfmemalloc_wait);
3959 :
3960 : /* Hopeless node, leave it to direct reclaim */
3961 1 : if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
3962 : return true;
3963 :
3964 1 : if (pgdat_balanced(pgdat, order, highest_zoneidx)) {
3965 1 : clear_pgdat_congested(pgdat);
3966 1 : return true;
3967 : }
3968 :
3969 : return false;
3970 : }
3971 :
3972 : /*
3973 : * kswapd shrinks a node of pages that are at or below the highest usable
3974 : * zone that is currently unbalanced.
3975 : *
3976 : * Returns true if kswapd scanned at least the requested number of pages to
3977 : * reclaim or if the lack of progress was due to pages under writeback.
3978 : * This is used to determine if the scanning priority needs to be raised.
3979 : */
3980 0 : static bool kswapd_shrink_node(pg_data_t *pgdat,
3981 : struct scan_control *sc)
3982 : {
3983 : struct zone *zone;
3984 : int z;
3985 :
3986 : /* Reclaim a number of pages proportional to the number of zones */
3987 0 : sc->nr_to_reclaim = 0;
3988 0 : for (z = 0; z <= sc->reclaim_idx; z++) {
3989 0 : zone = pgdat->node_zones + z;
3990 0 : if (!managed_zone(zone))
3991 0 : continue;
3992 :
3993 0 : sc->nr_to_reclaim += max(high_wmark_pages(zone), SWAP_CLUSTER_MAX);
3994 : }
3995 :
3996 : /*
3997 : * Historically care was taken to put equal pressure on all zones but
3998 : * now pressure is applied based on node LRU order.
3999 : */
4000 0 : shrink_node(pgdat, sc);
4001 :
4002 : /*
4003 : * Fragmentation may mean that the system cannot be rebalanced for
4004 : * high-order allocations. If twice the allocation size has been
4005 : * reclaimed then recheck watermarks only at order-0 to prevent
4006 : * excessive reclaim. Assume that a process requested a high-order
4007 : * can direct reclaim/compact.
4008 : */
4009 0 : if (sc->order && sc->nr_reclaimed >= compact_gap(sc->order))
4010 0 : sc->order = 0;
4011 :
4012 0 : return sc->nr_scanned >= sc->nr_to_reclaim;
4013 : }
4014 :
4015 : /* Page allocator PCP high watermark is lowered if reclaim is active. */
4016 : static inline void
4017 : update_reclaim_active(pg_data_t *pgdat, int highest_zoneidx, bool active)
4018 : {
4019 : int i;
4020 : struct zone *zone;
4021 :
4022 0 : for (i = 0; i <= highest_zoneidx; i++) {
4023 0 : zone = pgdat->node_zones + i;
4024 :
4025 0 : if (!managed_zone(zone))
4026 0 : continue;
4027 :
4028 : if (active)
4029 0 : set_bit(ZONE_RECLAIM_ACTIVE, &zone->flags);
4030 : else
4031 0 : clear_bit(ZONE_RECLAIM_ACTIVE, &zone->flags);
4032 : }
4033 : }
4034 :
4035 : static inline void
4036 : set_reclaim_active(pg_data_t *pgdat, int highest_zoneidx)
4037 : {
4038 0 : update_reclaim_active(pgdat, highest_zoneidx, true);
4039 : }
4040 :
4041 : static inline void
4042 : clear_reclaim_active(pg_data_t *pgdat, int highest_zoneidx)
4043 : {
4044 0 : update_reclaim_active(pgdat, highest_zoneidx, false);
4045 : }
4046 :
4047 : /*
4048 : * For kswapd, balance_pgdat() will reclaim pages across a node from zones
4049 : * that are eligible for use by the caller until at least one zone is
4050 : * balanced.
4051 : *
4052 : * Returns the order kswapd finished reclaiming at.
4053 : *
4054 : * kswapd scans the zones in the highmem->normal->dma direction. It skips
4055 : * zones which have free_pages > high_wmark_pages(zone), but once a zone is
4056 : * found to have free_pages <= high_wmark_pages(zone), any page in that zone
4057 : * or lower is eligible for reclaim until at least one usable zone is
4058 : * balanced.
4059 : */
4060 0 : static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx)
4061 : {
4062 : int i;
4063 : unsigned long nr_soft_reclaimed;
4064 : unsigned long nr_soft_scanned;
4065 : unsigned long pflags;
4066 : unsigned long nr_boost_reclaim;
4067 0 : unsigned long zone_boosts[MAX_NR_ZONES] = { 0, };
4068 : bool boosted;
4069 : struct zone *zone;
4070 0 : struct scan_control sc = {
4071 : .gfp_mask = GFP_KERNEL,
4072 : .order = order,
4073 : .may_unmap = 1,
4074 : };
4075 :
4076 0 : set_task_reclaim_state(current, &sc.reclaim_state);
4077 0 : psi_memstall_enter(&pflags);
4078 0 : __fs_reclaim_acquire(_THIS_IP_);
4079 :
4080 0 : count_vm_event(PAGEOUTRUN);
4081 :
4082 : /*
4083 : * Account for the reclaim boost. Note that the zone boost is left in
4084 : * place so that parallel allocations that are near the watermark will
4085 : * stall or direct reclaim until kswapd is finished.
4086 : */
4087 0 : nr_boost_reclaim = 0;
4088 0 : for (i = 0; i <= highest_zoneidx; i++) {
4089 0 : zone = pgdat->node_zones + i;
4090 0 : if (!managed_zone(zone))
4091 0 : continue;
4092 :
4093 0 : nr_boost_reclaim += zone->watermark_boost;
4094 0 : zone_boosts[i] = zone->watermark_boost;
4095 : }
4096 : boosted = nr_boost_reclaim;
4097 :
4098 : restart:
4099 0 : set_reclaim_active(pgdat, highest_zoneidx);
4100 0 : sc.priority = DEF_PRIORITY;
4101 : do {
4102 0 : unsigned long nr_reclaimed = sc.nr_reclaimed;
4103 0 : bool raise_priority = true;
4104 : bool balanced;
4105 : bool ret;
4106 :
4107 0 : sc.reclaim_idx = highest_zoneidx;
4108 :
4109 : /*
4110 : * If the number of buffer_heads exceeds the maximum allowed
4111 : * then consider reclaiming from all zones. This has a dual
4112 : * purpose -- on 64-bit systems it is expected that
4113 : * buffer_heads are stripped during active rotation. On 32-bit
4114 : * systems, highmem pages can pin lowmem memory and shrinking
4115 : * buffers can relieve lowmem pressure. Reclaim may still not
4116 : * go ahead if all eligible zones for the original allocation
4117 : * request are balanced to avoid excessive reclaim from kswapd.
4118 : */
4119 0 : if (buffer_heads_over_limit) {
4120 0 : for (i = MAX_NR_ZONES - 1; i >= 0; i--) {
4121 0 : zone = pgdat->node_zones + i;
4122 0 : if (!managed_zone(zone))
4123 0 : continue;
4124 :
4125 0 : sc.reclaim_idx = i;
4126 0 : break;
4127 : }
4128 : }
4129 :
4130 : /*
4131 : * If the pgdat is imbalanced then ignore boosting and preserve
4132 : * the watermarks for a later time and restart. Note that the
4133 : * zone watermarks will be still reset at the end of balancing
4134 : * on the grounds that the normal reclaim should be enough to
4135 : * re-evaluate if boosting is required when kswapd next wakes.
4136 : */
4137 0 : balanced = pgdat_balanced(pgdat, sc.order, highest_zoneidx);
4138 0 : if (!balanced && nr_boost_reclaim) {
4139 : nr_boost_reclaim = 0;
4140 : goto restart;
4141 : }
4142 :
4143 : /*
4144 : * If boosting is not active then only reclaim if there are no
4145 : * eligible zones. Note that sc.reclaim_idx is not used as
4146 : * buffer_heads_over_limit may have adjusted it.
4147 : */
4148 0 : if (!nr_boost_reclaim && balanced)
4149 : goto out;
4150 :
4151 : /* Limit the priority of boosting to avoid reclaim writeback */
4152 0 : if (nr_boost_reclaim && sc.priority == DEF_PRIORITY - 2)
4153 0 : raise_priority = false;
4154 :
4155 : /*
4156 : * Do not writeback or swap pages for boosted reclaim. The
4157 : * intent is to relieve pressure not issue sub-optimal IO
4158 : * from reclaim context. If no pages are reclaimed, the
4159 : * reclaim will be aborted.
4160 : */
4161 0 : sc.may_writepage = !laptop_mode && !nr_boost_reclaim;
4162 0 : sc.may_swap = !nr_boost_reclaim;
4163 :
4164 : /*
4165 : * Do some background aging of the anon list, to give
4166 : * pages a chance to be referenced before reclaiming. All
4167 : * pages are rotated regardless of classzone as this is
4168 : * about consistent aging.
4169 : */
4170 0 : age_active_anon(pgdat, &sc);
4171 :
4172 : /*
4173 : * If we're getting trouble reclaiming, start doing writepage
4174 : * even in laptop mode.
4175 : */
4176 0 : if (sc.priority < DEF_PRIORITY - 2)
4177 0 : sc.may_writepage = 1;
4178 :
4179 : /* Call soft limit reclaim before calling shrink_node. */
4180 0 : sc.nr_scanned = 0;
4181 0 : nr_soft_scanned = 0;
4182 0 : nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(pgdat, sc.order,
4183 : sc.gfp_mask, &nr_soft_scanned);
4184 : sc.nr_reclaimed += nr_soft_reclaimed;
4185 :
4186 : /*
4187 : * There should be no need to raise the scanning priority if
4188 : * enough pages are already being scanned that that high
4189 : * watermark would be met at 100% efficiency.
4190 : */
4191 0 : if (kswapd_shrink_node(pgdat, &sc))
4192 0 : raise_priority = false;
4193 :
4194 : /*
4195 : * If the low watermark is met there is no need for processes
4196 : * to be throttled on pfmemalloc_wait as they should not be
4197 : * able to safely make forward progress. Wake them
4198 : */
4199 0 : if (waitqueue_active(&pgdat->pfmemalloc_wait) &&
4200 0 : allow_direct_reclaim(pgdat))
4201 0 : wake_up_all(&pgdat->pfmemalloc_wait);
4202 :
4203 : /* Check if kswapd should be suspending */
4204 0 : __fs_reclaim_release(_THIS_IP_);
4205 0 : ret = try_to_freeze();
4206 0 : __fs_reclaim_acquire(_THIS_IP_);
4207 0 : if (ret || kthread_should_stop())
4208 : break;
4209 :
4210 : /*
4211 : * Raise priority if scanning rate is too low or there was no
4212 : * progress in reclaiming pages
4213 : */
4214 0 : nr_reclaimed = sc.nr_reclaimed - nr_reclaimed;
4215 0 : nr_boost_reclaim -= min(nr_boost_reclaim, nr_reclaimed);
4216 :
4217 : /*
4218 : * If reclaim made no progress for a boost, stop reclaim as
4219 : * IO cannot be queued and it could be an infinite loop in
4220 : * extreme circumstances.
4221 : */
4222 0 : if (nr_boost_reclaim && !nr_reclaimed)
4223 : break;
4224 :
4225 0 : if (raise_priority || !nr_reclaimed)
4226 0 : sc.priority--;
4227 0 : } while (sc.priority >= 1);
4228 :
4229 0 : if (!sc.nr_reclaimed)
4230 0 : pgdat->kswapd_failures++;
4231 :
4232 : out:
4233 0 : clear_reclaim_active(pgdat, highest_zoneidx);
4234 :
4235 : /* If reclaim was boosted, account for the reclaim done in this pass */
4236 0 : if (boosted) {
4237 : unsigned long flags;
4238 :
4239 0 : for (i = 0; i <= highest_zoneidx; i++) {
4240 0 : if (!zone_boosts[i])
4241 0 : continue;
4242 :
4243 : /* Increments are under the zone lock */
4244 0 : zone = pgdat->node_zones + i;
4245 0 : spin_lock_irqsave(&zone->lock, flags);
4246 0 : zone->watermark_boost -= min(zone->watermark_boost, zone_boosts[i]);
4247 0 : spin_unlock_irqrestore(&zone->lock, flags);
4248 : }
4249 :
4250 : /*
4251 : * As there is now likely space, wakeup kcompact to defragment
4252 : * pageblocks.
4253 : */
4254 0 : wakeup_kcompactd(pgdat, pageblock_order, highest_zoneidx);
4255 : }
4256 :
4257 0 : snapshot_refaults(NULL, pgdat);
4258 0 : __fs_reclaim_release(_THIS_IP_);
4259 0 : psi_memstall_leave(&pflags);
4260 0 : set_task_reclaim_state(current, NULL);
4261 :
4262 : /*
4263 : * Return the order kswapd stopped reclaiming at as
4264 : * prepare_kswapd_sleep() takes it into account. If another caller
4265 : * entered the allocator slow path while kswapd was awake, order will
4266 : * remain at the higher level.
4267 : */
4268 0 : return sc.order;
4269 : }
4270 :
4271 : /*
4272 : * The pgdat->kswapd_highest_zoneidx is used to pass the highest zone index to
4273 : * be reclaimed by kswapd from the waker. If the value is MAX_NR_ZONES which is
4274 : * not a valid index then either kswapd runs for first time or kswapd couldn't
4275 : * sleep after previous reclaim attempt (node is still unbalanced). In that
4276 : * case return the zone index of the previous kswapd reclaim cycle.
4277 : */
4278 : static enum zone_type kswapd_highest_zoneidx(pg_data_t *pgdat,
4279 : enum zone_type prev_highest_zoneidx)
4280 : {
4281 1 : enum zone_type curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx);
4282 :
4283 1 : return curr_idx == MAX_NR_ZONES ? prev_highest_zoneidx : curr_idx;
4284 : }
4285 :
4286 1 : static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order,
4287 : unsigned int highest_zoneidx)
4288 : {
4289 1 : long remaining = 0;
4290 2 : DEFINE_WAIT(wait);
4291 :
4292 2 : if (freezing(current) || kthread_should_stop())
4293 0 : return;
4294 :
4295 1 : prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
4296 :
4297 : /*
4298 : * Try to sleep for a short interval. Note that kcompactd will only be
4299 : * woken if it is possible to sleep for a short interval. This is
4300 : * deliberate on the assumption that if reclaim cannot keep an
4301 : * eligible zone balanced that it's also unlikely that compaction will
4302 : * succeed.
4303 : */
4304 1 : if (prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) {
4305 : /*
4306 : * Compaction records what page blocks it recently failed to
4307 : * isolate pages from and skips them in the future scanning.
4308 : * When kswapd is going to sleep, it is reasonable to assume
4309 : * that pages and compaction may succeed so reset the cache.
4310 : */
4311 1 : reset_isolation_suitable(pgdat);
4312 :
4313 : /*
4314 : * We have freed the memory, now we should compact it to make
4315 : * allocation of the requested order possible.
4316 : */
4317 1 : wakeup_kcompactd(pgdat, alloc_order, highest_zoneidx);
4318 :
4319 1 : remaining = schedule_timeout(HZ/10);
4320 :
4321 : /*
4322 : * If woken prematurely then reset kswapd_highest_zoneidx and
4323 : * order. The values will either be from a wakeup request or
4324 : * the previous request that slept prematurely.
4325 : */
4326 0 : if (remaining) {
4327 0 : WRITE_ONCE(pgdat->kswapd_highest_zoneidx,
4328 : kswapd_highest_zoneidx(pgdat,
4329 : highest_zoneidx));
4330 :
4331 0 : if (READ_ONCE(pgdat->kswapd_order) < reclaim_order)
4332 0 : WRITE_ONCE(pgdat->kswapd_order, reclaim_order);
4333 : }
4334 :
4335 0 : finish_wait(&pgdat->kswapd_wait, &wait);
4336 0 : prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
4337 : }
4338 :
4339 : /*
4340 : * After a short sleep, check if it was a premature sleep. If not, then
4341 : * go fully to sleep until explicitly woken up.
4342 : */
4343 0 : if (!remaining &&
4344 0 : prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) {
4345 0 : trace_mm_vmscan_kswapd_sleep(pgdat->node_id);
4346 :
4347 : /*
4348 : * vmstat counters are not perfectly accurate and the estimated
4349 : * value for counters such as NR_FREE_PAGES can deviate from the
4350 : * true value by nr_online_cpus * threshold. To avoid the zone
4351 : * watermarks being breached while under pressure, we reduce the
4352 : * per-cpu vmstat threshold while kswapd is awake and restore
4353 : * them before going back to sleep.
4354 : */
4355 : set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold);
4356 :
4357 0 : if (!kthread_should_stop())
4358 0 : schedule();
4359 :
4360 : set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold);
4361 : } else {
4362 0 : if (remaining)
4363 0 : count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY);
4364 : else
4365 0 : count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY);
4366 : }
4367 0 : finish_wait(&pgdat->kswapd_wait, &wait);
4368 : }
4369 :
4370 : /*
4371 : * The background pageout daemon, started as a kernel thread
4372 : * from the init process.
4373 : *
4374 : * This basically trickles out pages so that we have _some_
4375 : * free memory available even if there is no other activity
4376 : * that frees anything up. This is needed for things like routing
4377 : * etc, where we otherwise might have all activity going on in
4378 : * asynchronous contexts that cannot page things out.
4379 : *
4380 : * If there are applications that are active memory-allocators
4381 : * (most normal use), this basically shouldn't matter.
4382 : */
4383 1 : static int kswapd(void *p)
4384 : {
4385 : unsigned int alloc_order, reclaim_order;
4386 1 : unsigned int highest_zoneidx = MAX_NR_ZONES - 1;
4387 1 : pg_data_t *pgdat = (pg_data_t *)p;
4388 1 : struct task_struct *tsk = current;
4389 1 : const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
4390 :
4391 1 : if (!cpumask_empty(cpumask))
4392 1 : set_cpus_allowed_ptr(tsk, cpumask);
4393 :
4394 : /*
4395 : * Tell the memory management that we're a "memory allocator",
4396 : * and that if we need more memory we should get access to it
4397 : * regardless (see "__alloc_pages()"). "kswapd" should
4398 : * never get caught in the normal page freeing logic.
4399 : *
4400 : * (Kswapd normally doesn't need memory anyway, but sometimes
4401 : * you need a small amount of memory in order to be able to
4402 : * page out something else, and this flag essentially protects
4403 : * us from recursively trying to free more memory as we're
4404 : * trying to free the first piece of memory in the first place).
4405 : */
4406 1 : tsk->flags |= PF_MEMALLOC | PF_KSWAPD;
4407 1 : set_freezable();
4408 :
4409 1 : WRITE_ONCE(pgdat->kswapd_order, 0);
4410 1 : WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES);
4411 1 : atomic_set(&pgdat->nr_writeback_throttled, 0);
4412 : for ( ; ; ) {
4413 : bool ret;
4414 :
4415 1 : alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order);
4416 : highest_zoneidx = kswapd_highest_zoneidx(pgdat,
4417 : highest_zoneidx);
4418 :
4419 : kswapd_try_sleep:
4420 1 : kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order,
4421 : highest_zoneidx);
4422 :
4423 : /* Read the new order and highest_zoneidx */
4424 0 : alloc_order = READ_ONCE(pgdat->kswapd_order);
4425 0 : highest_zoneidx = kswapd_highest_zoneidx(pgdat,
4426 : highest_zoneidx);
4427 0 : WRITE_ONCE(pgdat->kswapd_order, 0);
4428 0 : WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES);
4429 :
4430 0 : ret = try_to_freeze();
4431 0 : if (kthread_should_stop())
4432 : break;
4433 :
4434 : /*
4435 : * We can speed up thawing tasks if we don't call balance_pgdat
4436 : * after returning from the refrigerator
4437 : */
4438 0 : if (ret)
4439 0 : continue;
4440 :
4441 : /*
4442 : * Reclaim begins at the requested order but if a high-order
4443 : * reclaim fails then kswapd falls back to reclaiming for
4444 : * order-0. If that happens, kswapd will consider sleeping
4445 : * for the order it finished reclaiming at (reclaim_order)
4446 : * but kcompactd is woken to compact for the original
4447 : * request (alloc_order).
4448 : */
4449 0 : trace_mm_vmscan_kswapd_wake(pgdat->node_id, highest_zoneidx,
4450 : alloc_order);
4451 0 : reclaim_order = balance_pgdat(pgdat, alloc_order,
4452 : highest_zoneidx);
4453 0 : if (reclaim_order < alloc_order)
4454 : goto kswapd_try_sleep;
4455 : }
4456 :
4457 0 : tsk->flags &= ~(PF_MEMALLOC | PF_KSWAPD);
4458 :
4459 0 : return 0;
4460 : }
4461 :
4462 : /*
4463 : * A zone is low on free memory or too fragmented for high-order memory. If
4464 : * kswapd should reclaim (direct reclaim is deferred), wake it up for the zone's
4465 : * pgdat. It will wake up kcompactd after reclaiming memory. If kswapd reclaim
4466 : * has failed or is not needed, still wake up kcompactd if only compaction is
4467 : * needed.
4468 : */
4469 0 : void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order,
4470 : enum zone_type highest_zoneidx)
4471 : {
4472 : pg_data_t *pgdat;
4473 : enum zone_type curr_idx;
4474 :
4475 0 : if (!managed_zone(zone))
4476 : return;
4477 :
4478 0 : if (!cpuset_zone_allowed(zone, gfp_flags))
4479 : return;
4480 :
4481 0 : pgdat = zone->zone_pgdat;
4482 0 : curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx);
4483 :
4484 0 : if (curr_idx == MAX_NR_ZONES || curr_idx < highest_zoneidx)
4485 0 : WRITE_ONCE(pgdat->kswapd_highest_zoneidx, highest_zoneidx);
4486 :
4487 0 : if (READ_ONCE(pgdat->kswapd_order) < order)
4488 0 : WRITE_ONCE(pgdat->kswapd_order, order);
4489 :
4490 0 : if (!waitqueue_active(&pgdat->kswapd_wait))
4491 : return;
4492 :
4493 : /* Hopeless node, leave it to direct reclaim if possible */
4494 0 : if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ||
4495 0 : (pgdat_balanced(pgdat, order, highest_zoneidx) &&
4496 0 : !pgdat_watermark_boosted(pgdat, highest_zoneidx))) {
4497 : /*
4498 : * There may be plenty of free memory available, but it's too
4499 : * fragmented for high-order allocations. Wake up kcompactd
4500 : * and rely on compaction_suitable() to determine if it's
4501 : * needed. If it fails, it will defer subsequent attempts to
4502 : * ratelimit its work.
4503 : */
4504 0 : if (!(gfp_flags & __GFP_DIRECT_RECLAIM))
4505 0 : wakeup_kcompactd(pgdat, order, highest_zoneidx);
4506 : return;
4507 : }
4508 :
4509 0 : trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, highest_zoneidx, order,
4510 : gfp_flags);
4511 0 : wake_up_interruptible(&pgdat->kswapd_wait);
4512 : }
4513 :
4514 : #ifdef CONFIG_HIBERNATION
4515 : /*
4516 : * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of
4517 : * freed pages.
4518 : *
4519 : * Rather than trying to age LRUs the aim is to preserve the overall
4520 : * LRU order by reclaiming preferentially
4521 : * inactive > active > active referenced > active mapped
4522 : */
4523 : unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
4524 : {
4525 : struct scan_control sc = {
4526 : .nr_to_reclaim = nr_to_reclaim,
4527 : .gfp_mask = GFP_HIGHUSER_MOVABLE,
4528 : .reclaim_idx = MAX_NR_ZONES - 1,
4529 : .priority = DEF_PRIORITY,
4530 : .may_writepage = 1,
4531 : .may_unmap = 1,
4532 : .may_swap = 1,
4533 : .hibernation_mode = 1,
4534 : };
4535 : struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
4536 : unsigned long nr_reclaimed;
4537 : unsigned int noreclaim_flag;
4538 :
4539 : fs_reclaim_acquire(sc.gfp_mask);
4540 : noreclaim_flag = memalloc_noreclaim_save();
4541 : set_task_reclaim_state(current, &sc.reclaim_state);
4542 :
4543 : nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
4544 :
4545 : set_task_reclaim_state(current, NULL);
4546 : memalloc_noreclaim_restore(noreclaim_flag);
4547 : fs_reclaim_release(sc.gfp_mask);
4548 :
4549 : return nr_reclaimed;
4550 : }
4551 : #endif /* CONFIG_HIBERNATION */
4552 :
4553 : /*
4554 : * This kswapd start function will be called by init and node-hot-add.
4555 : * On node-hot-add, kswapd will moved to proper cpus if cpus are hot-added.
4556 : */
4557 1 : void kswapd_run(int nid)
4558 : {
4559 1 : pg_data_t *pgdat = NODE_DATA(nid);
4560 :
4561 1 : if (pgdat->kswapd)
4562 : return;
4563 :
4564 2 : pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);
4565 1 : if (IS_ERR(pgdat->kswapd)) {
4566 : /* failure at boot is fatal */
4567 0 : BUG_ON(system_state < SYSTEM_RUNNING);
4568 0 : pr_err("Failed to start kswapd on node %d\n", nid);
4569 0 : pgdat->kswapd = NULL;
4570 : }
4571 : }
4572 :
4573 : /*
4574 : * Called by memory hotplug when all memory in a node is offlined. Caller must
4575 : * hold mem_hotplug_begin/end().
4576 : */
4577 0 : void kswapd_stop(int nid)
4578 : {
4579 0 : struct task_struct *kswapd = NODE_DATA(nid)->kswapd;
4580 :
4581 0 : if (kswapd) {
4582 0 : kthread_stop(kswapd);
4583 0 : NODE_DATA(nid)->kswapd = NULL;
4584 : }
4585 0 : }
4586 :
4587 1 : static int __init kswapd_init(void)
4588 : {
4589 : int nid;
4590 :
4591 1 : swap_setup();
4592 2 : for_each_node_state(nid, N_MEMORY)
4593 1 : kswapd_run(nid);
4594 1 : return 0;
4595 : }
4596 :
4597 : module_init(kswapd_init)
4598 :
4599 : #ifdef CONFIG_NUMA
4600 : /*
4601 : * Node reclaim mode
4602 : *
4603 : * If non-zero call node_reclaim when the number of free pages falls below
4604 : * the watermarks.
4605 : */
4606 : int node_reclaim_mode __read_mostly;
4607 :
4608 : /*
4609 : * Priority for NODE_RECLAIM. This determines the fraction of pages
4610 : * of a node considered for each zone_reclaim. 4 scans 1/16th of
4611 : * a zone.
4612 : */
4613 : #define NODE_RECLAIM_PRIORITY 4
4614 :
4615 : /*
4616 : * Percentage of pages in a zone that must be unmapped for node_reclaim to
4617 : * occur.
4618 : */
4619 : int sysctl_min_unmapped_ratio = 1;
4620 :
4621 : /*
4622 : * If the number of slab pages in a zone grows beyond this percentage then
4623 : * slab reclaim needs to occur.
4624 : */
4625 : int sysctl_min_slab_ratio = 5;
4626 :
4627 : static inline unsigned long node_unmapped_file_pages(struct pglist_data *pgdat)
4628 : {
4629 : unsigned long file_mapped = node_page_state(pgdat, NR_FILE_MAPPED);
4630 : unsigned long file_lru = node_page_state(pgdat, NR_INACTIVE_FILE) +
4631 : node_page_state(pgdat, NR_ACTIVE_FILE);
4632 :
4633 : /*
4634 : * It's possible for there to be more file mapped pages than
4635 : * accounted for by the pages on the file LRU lists because
4636 : * tmpfs pages accounted for as ANON can also be FILE_MAPPED
4637 : */
4638 : return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0;
4639 : }
4640 :
4641 : /* Work out how many page cache pages we can reclaim in this reclaim_mode */
4642 : static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat)
4643 : {
4644 : unsigned long nr_pagecache_reclaimable;
4645 : unsigned long delta = 0;
4646 :
4647 : /*
4648 : * If RECLAIM_UNMAP is set, then all file pages are considered
4649 : * potentially reclaimable. Otherwise, we have to worry about
4650 : * pages like swapcache and node_unmapped_file_pages() provides
4651 : * a better estimate
4652 : */
4653 : if (node_reclaim_mode & RECLAIM_UNMAP)
4654 : nr_pagecache_reclaimable = node_page_state(pgdat, NR_FILE_PAGES);
4655 : else
4656 : nr_pagecache_reclaimable = node_unmapped_file_pages(pgdat);
4657 :
4658 : /* If we can't clean pages, remove dirty pages from consideration */
4659 : if (!(node_reclaim_mode & RECLAIM_WRITE))
4660 : delta += node_page_state(pgdat, NR_FILE_DIRTY);
4661 :
4662 : /* Watch for any possible underflows due to delta */
4663 : if (unlikely(delta > nr_pagecache_reclaimable))
4664 : delta = nr_pagecache_reclaimable;
4665 :
4666 : return nr_pagecache_reclaimable - delta;
4667 : }
4668 :
4669 : /*
4670 : * Try to free up some pages from this node through reclaim.
4671 : */
4672 : static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
4673 : {
4674 : /* Minimum pages needed in order to stay on node */
4675 : const unsigned long nr_pages = 1 << order;
4676 : struct task_struct *p = current;
4677 : unsigned int noreclaim_flag;
4678 : struct scan_control sc = {
4679 : .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
4680 : .gfp_mask = current_gfp_context(gfp_mask),
4681 : .order = order,
4682 : .priority = NODE_RECLAIM_PRIORITY,
4683 : .may_writepage = !!(node_reclaim_mode & RECLAIM_WRITE),
4684 : .may_unmap = !!(node_reclaim_mode & RECLAIM_UNMAP),
4685 : .may_swap = 1,
4686 : .reclaim_idx = gfp_zone(gfp_mask),
4687 : };
4688 : unsigned long pflags;
4689 :
4690 : trace_mm_vmscan_node_reclaim_begin(pgdat->node_id, order,
4691 : sc.gfp_mask);
4692 :
4693 : cond_resched();
4694 : psi_memstall_enter(&pflags);
4695 : fs_reclaim_acquire(sc.gfp_mask);
4696 : /*
4697 : * We need to be able to allocate from the reserves for RECLAIM_UNMAP
4698 : */
4699 : noreclaim_flag = memalloc_noreclaim_save();
4700 : set_task_reclaim_state(p, &sc.reclaim_state);
4701 :
4702 : if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages) {
4703 : /*
4704 : * Free memory by calling shrink node with increasing
4705 : * priorities until we have enough memory freed.
4706 : */
4707 : do {
4708 : shrink_node(pgdat, &sc);
4709 : } while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0);
4710 : }
4711 :
4712 : set_task_reclaim_state(p, NULL);
4713 : memalloc_noreclaim_restore(noreclaim_flag);
4714 : fs_reclaim_release(sc.gfp_mask);
4715 : psi_memstall_leave(&pflags);
4716 :
4717 : trace_mm_vmscan_node_reclaim_end(sc.nr_reclaimed);
4718 :
4719 : return sc.nr_reclaimed >= nr_pages;
4720 : }
4721 :
4722 : int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order)
4723 : {
4724 : int ret;
4725 :
4726 : /*
4727 : * Node reclaim reclaims unmapped file backed pages and
4728 : * slab pages if we are over the defined limits.
4729 : *
4730 : * A small portion of unmapped file backed pages is needed for
4731 : * file I/O otherwise pages read by file I/O will be immediately
4732 : * thrown out if the node is overallocated. So we do not reclaim
4733 : * if less than a specified percentage of the node is used by
4734 : * unmapped file backed pages.
4735 : */
4736 : if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages &&
4737 : node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) <=
4738 : pgdat->min_slab_pages)
4739 : return NODE_RECLAIM_FULL;
4740 :
4741 : /*
4742 : * Do not scan if the allocation should not be delayed.
4743 : */
4744 : if (!gfpflags_allow_blocking(gfp_mask) || (current->flags & PF_MEMALLOC))
4745 : return NODE_RECLAIM_NOSCAN;
4746 :
4747 : /*
4748 : * Only run node reclaim on the local node or on nodes that do not
4749 : * have associated processors. This will favor the local processor
4750 : * over remote processors and spread off node memory allocations
4751 : * as wide as possible.
4752 : */
4753 : if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id())
4754 : return NODE_RECLAIM_NOSCAN;
4755 :
4756 : if (test_and_set_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags))
4757 : return NODE_RECLAIM_NOSCAN;
4758 :
4759 : ret = __node_reclaim(pgdat, gfp_mask, order);
4760 : clear_bit(PGDAT_RECLAIM_LOCKED, &pgdat->flags);
4761 :
4762 : if (!ret)
4763 : count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED);
4764 :
4765 : return ret;
4766 : }
4767 : #endif
4768 :
4769 : /**
4770 : * check_move_unevictable_pages - check pages for evictability and move to
4771 : * appropriate zone lru list
4772 : * @pvec: pagevec with lru pages to check
4773 : *
4774 : * Checks pages for evictability, if an evictable page is in the unevictable
4775 : * lru list, moves it to the appropriate evictable lru list. This function
4776 : * should be only used for lru pages.
4777 : */
4778 0 : void check_move_unevictable_pages(struct pagevec *pvec)
4779 : {
4780 0 : struct lruvec *lruvec = NULL;
4781 0 : int pgscanned = 0;
4782 0 : int pgrescued = 0;
4783 : int i;
4784 :
4785 0 : for (i = 0; i < pvec->nr; i++) {
4786 0 : struct page *page = pvec->pages[i];
4787 0 : struct folio *folio = page_folio(page);
4788 : int nr_pages;
4789 :
4790 0 : if (PageTransTail(page))
4791 : continue;
4792 :
4793 0 : nr_pages = thp_nr_pages(page);
4794 0 : pgscanned += nr_pages;
4795 :
4796 : /* block memcg migration during page moving between lru */
4797 0 : if (!TestClearPageLRU(page))
4798 0 : continue;
4799 :
4800 0 : lruvec = folio_lruvec_relock_irq(folio, lruvec);
4801 0 : if (page_evictable(page) && PageUnevictable(page)) {
4802 0 : del_page_from_lru_list(page, lruvec);
4803 0 : ClearPageUnevictable(page);
4804 0 : add_page_to_lru_list(page, lruvec);
4805 0 : pgrescued += nr_pages;
4806 : }
4807 : SetPageLRU(page);
4808 : }
4809 :
4810 0 : if (lruvec) {
4811 0 : __count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued);
4812 0 : __count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
4813 0 : unlock_page_lruvec_irq(lruvec);
4814 0 : } else if (pgscanned) {
4815 0 : count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
4816 : }
4817 0 : }
4818 : EXPORT_SYMBOL_GPL(check_move_unevictable_pages);
|
