Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * linux/mm/page_io.c
4 : *
5 : * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
6 : *
7 : * Swap reorganised 29.12.95,
8 : * Asynchronous swapping added 30.12.95. Stephen Tweedie
9 : * Removed race in async swapping. 14.4.1996. Bruno Haible
10 : * Add swap of shared pages through the page cache. 20.2.1998. Stephen Tweedie
11 : * Always use brw_page, life becomes simpler. 12 May 1998 Eric Biederman
12 : */
13 :
14 : #include <linux/mm.h>
15 : #include <linux/kernel_stat.h>
16 : #include <linux/gfp.h>
17 : #include <linux/pagemap.h>
18 : #include <linux/swap.h>
19 : #include <linux/bio.h>
20 : #include <linux/swapops.h>
21 : #include <linux/buffer_head.h>
22 : #include <linux/writeback.h>
23 : #include <linux/frontswap.h>
24 : #include <linux/blkdev.h>
25 : #include <linux/psi.h>
26 : #include <linux/uio.h>
27 : #include <linux/sched/task.h>
28 : #include <linux/delayacct.h>
29 :
30 0 : void end_swap_bio_write(struct bio *bio)
31 : {
32 0 : struct page *page = bio_first_page_all(bio);
33 :
34 0 : if (bio->bi_status) {
35 0 : SetPageError(page);
36 : /*
37 : * We failed to write the page out to swap-space.
38 : * Re-dirty the page in order to avoid it being reclaimed.
39 : * Also print a dire warning that things will go BAD (tm)
40 : * very quickly.
41 : *
42 : * Also clear PG_reclaim to avoid folio_rotate_reclaimable()
43 : */
44 0 : set_page_dirty(page);
45 0 : pr_alert_ratelimited("Write-error on swap-device (%u:%u:%llu)\n",
46 : MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
47 : (unsigned long long)bio->bi_iter.bi_sector);
48 : ClearPageReclaim(page);
49 : }
50 0 : end_page_writeback(page);
51 0 : bio_put(bio);
52 0 : }
53 :
54 0 : static void end_swap_bio_read(struct bio *bio)
55 : {
56 0 : struct page *page = bio_first_page_all(bio);
57 0 : struct task_struct *waiter = bio->bi_private;
58 :
59 0 : if (bio->bi_status) {
60 0 : SetPageError(page);
61 0 : ClearPageUptodate(page);
62 0 : pr_alert_ratelimited("Read-error on swap-device (%u:%u:%llu)\n",
63 : MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
64 : (unsigned long long)bio->bi_iter.bi_sector);
65 : goto out;
66 : }
67 :
68 : SetPageUptodate(page);
69 : out:
70 0 : unlock_page(page);
71 0 : WRITE_ONCE(bio->bi_private, NULL);
72 0 : bio_put(bio);
73 0 : if (waiter) {
74 0 : blk_wake_io_task(waiter);
75 0 : put_task_struct(waiter);
76 : }
77 0 : }
78 :
79 0 : int generic_swapfile_activate(struct swap_info_struct *sis,
80 : struct file *swap_file,
81 : sector_t *span)
82 : {
83 0 : struct address_space *mapping = swap_file->f_mapping;
84 0 : struct inode *inode = mapping->host;
85 : unsigned blocks_per_page;
86 : unsigned long page_no;
87 : unsigned blkbits;
88 : sector_t probe_block;
89 : sector_t last_block;
90 0 : sector_t lowest_block = -1;
91 0 : sector_t highest_block = 0;
92 0 : int nr_extents = 0;
93 : int ret;
94 :
95 0 : blkbits = inode->i_blkbits;
96 0 : blocks_per_page = PAGE_SIZE >> blkbits;
97 :
98 : /*
99 : * Map all the blocks into the extent tree. This code doesn't try
100 : * to be very smart.
101 : */
102 0 : probe_block = 0;
103 0 : page_no = 0;
104 0 : last_block = i_size_read(inode) >> blkbits;
105 0 : while ((probe_block + blocks_per_page) <= last_block &&
106 0 : page_no < sis->max) {
107 : unsigned block_in_page;
108 : sector_t first_block;
109 :
110 0 : cond_resched();
111 :
112 0 : first_block = probe_block;
113 0 : ret = bmap(inode, &first_block);
114 0 : if (ret || !first_block)
115 : goto bad_bmap;
116 :
117 : /*
118 : * It must be PAGE_SIZE aligned on-disk
119 : */
120 0 : if (first_block & (blocks_per_page - 1)) {
121 0 : probe_block++;
122 0 : goto reprobe;
123 : }
124 :
125 0 : for (block_in_page = 1; block_in_page < blocks_per_page;
126 0 : block_in_page++) {
127 : sector_t block;
128 :
129 0 : block = probe_block + block_in_page;
130 0 : ret = bmap(inode, &block);
131 0 : if (ret || !block)
132 : goto bad_bmap;
133 :
134 0 : if (block != first_block + block_in_page) {
135 : /* Discontiguity */
136 0 : probe_block++;
137 0 : goto reprobe;
138 : }
139 : }
140 :
141 0 : first_block >>= (PAGE_SHIFT - blkbits);
142 0 : if (page_no) { /* exclude the header page */
143 0 : if (first_block < lowest_block)
144 0 : lowest_block = first_block;
145 0 : if (first_block > highest_block)
146 0 : highest_block = first_block;
147 : }
148 :
149 : /*
150 : * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks
151 : */
152 0 : ret = add_swap_extent(sis, page_no, 1, first_block);
153 0 : if (ret < 0)
154 : goto out;
155 0 : nr_extents += ret;
156 0 : page_no++;
157 0 : probe_block += blocks_per_page;
158 : reprobe:
159 0 : continue;
160 : }
161 0 : ret = nr_extents;
162 0 : *span = 1 + highest_block - lowest_block;
163 0 : if (page_no == 0)
164 0 : page_no = 1; /* force Empty message */
165 0 : sis->max = page_no;
166 0 : sis->pages = page_no - 1;
167 0 : sis->highest_bit = page_no - 1;
168 : out:
169 0 : return ret;
170 : bad_bmap:
171 0 : pr_err("swapon: swapfile has holes\n");
172 0 : ret = -EINVAL;
173 0 : goto out;
174 : }
175 :
176 : /*
177 : * We may have stale swap cache pages in memory: notice
178 : * them here and get rid of the unnecessary final write.
179 : */
180 0 : int swap_writepage(struct page *page, struct writeback_control *wbc)
181 : {
182 0 : int ret = 0;
183 :
184 0 : if (try_to_free_swap(page)) {
185 0 : unlock_page(page);
186 0 : goto out;
187 : }
188 : /*
189 : * Arch code may have to preserve more data than just the page
190 : * contents, e.g. memory tags.
191 : */
192 0 : ret = arch_prepare_to_swap(page);
193 : if (ret) {
194 : set_page_dirty(page);
195 : unlock_page(page);
196 : goto out;
197 : }
198 0 : if (frontswap_store(page) == 0) {
199 : set_page_writeback(page);
200 : unlock_page(page);
201 : end_page_writeback(page);
202 : goto out;
203 : }
204 0 : ret = __swap_writepage(page, wbc, end_swap_bio_write);
205 : out:
206 0 : return ret;
207 : }
208 :
209 0 : static inline void count_swpout_vm_event(struct page *page)
210 : {
211 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
212 : if (unlikely(PageTransHuge(page)))
213 : count_vm_event(THP_SWPOUT);
214 : #endif
215 0 : count_vm_events(PSWPOUT, thp_nr_pages(page));
216 0 : }
217 :
218 : #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
219 : static void bio_associate_blkg_from_page(struct bio *bio, struct page *page)
220 : {
221 : struct cgroup_subsys_state *css;
222 : struct mem_cgroup *memcg;
223 :
224 : memcg = page_memcg(page);
225 : if (!memcg)
226 : return;
227 :
228 : rcu_read_lock();
229 : css = cgroup_e_css(memcg->css.cgroup, &io_cgrp_subsys);
230 : bio_associate_blkg_from_css(bio, css);
231 : rcu_read_unlock();
232 : }
233 : #else
234 : #define bio_associate_blkg_from_page(bio, page) do { } while (0)
235 : #endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */
236 :
237 0 : int __swap_writepage(struct page *page, struct writeback_control *wbc,
238 : bio_end_io_t end_write_func)
239 : {
240 : struct bio *bio;
241 : int ret;
242 0 : struct swap_info_struct *sis = page_swap_info(page);
243 :
244 : VM_BUG_ON_PAGE(!PageSwapCache(page), page);
245 0 : if (data_race(sis->flags & SWP_FS_OPS)) {
246 : struct kiocb kiocb;
247 0 : struct file *swap_file = sis->swap_file;
248 0 : struct address_space *mapping = swap_file->f_mapping;
249 0 : struct bio_vec bv = {
250 : .bv_page = page,
251 : .bv_len = PAGE_SIZE,
252 : .bv_offset = 0
253 : };
254 : struct iov_iter from;
255 :
256 0 : iov_iter_bvec(&from, WRITE, &bv, 1, PAGE_SIZE);
257 0 : init_sync_kiocb(&kiocb, swap_file);
258 0 : kiocb.ki_pos = page_file_offset(page);
259 :
260 0 : set_page_writeback(page);
261 0 : unlock_page(page);
262 0 : ret = mapping->a_ops->direct_IO(&kiocb, &from);
263 0 : if (ret == PAGE_SIZE) {
264 0 : count_vm_event(PSWPOUT);
265 0 : ret = 0;
266 : } else {
267 : /*
268 : * In the case of swap-over-nfs, this can be a
269 : * temporary failure if the system has limited
270 : * memory for allocating transmit buffers.
271 : * Mark the page dirty and avoid
272 : * folio_rotate_reclaimable but rate-limit the
273 : * messages but do not flag PageError like
274 : * the normal direct-to-bio case as it could
275 : * be temporary.
276 : */
277 0 : set_page_dirty(page);
278 0 : ClearPageReclaim(page);
279 0 : pr_err_ratelimited("Write error on dio swapfile (%llu)\n",
280 : page_file_offset(page));
281 : }
282 0 : end_page_writeback(page);
283 : return ret;
284 : }
285 :
286 0 : ret = bdev_write_page(sis->bdev, swap_page_sector(page), page, wbc);
287 0 : if (!ret) {
288 0 : count_swpout_vm_event(page);
289 0 : return 0;
290 : }
291 :
292 0 : bio = bio_alloc(sis->bdev, 1,
293 0 : REQ_OP_WRITE | REQ_SWAP | wbc_to_write_flags(wbc),
294 : GFP_NOIO);
295 0 : bio->bi_iter.bi_sector = swap_page_sector(page);
296 0 : bio->bi_end_io = end_write_func;
297 0 : bio_add_page(bio, page, thp_size(page), 0);
298 :
299 : bio_associate_blkg_from_page(bio, page);
300 0 : count_swpout_vm_event(page);
301 0 : set_page_writeback(page);
302 0 : unlock_page(page);
303 0 : submit_bio(bio);
304 :
305 0 : return 0;
306 : }
307 :
308 0 : int swap_readpage(struct page *page, bool synchronous)
309 : {
310 : struct bio *bio;
311 0 : int ret = 0;
312 0 : struct swap_info_struct *sis = page_swap_info(page);
313 0 : bool workingset = PageWorkingset(page);
314 : unsigned long pflags;
315 :
316 : VM_BUG_ON_PAGE(!PageSwapCache(page) && !synchronous, page);
317 : VM_BUG_ON_PAGE(!PageLocked(page), page);
318 : VM_BUG_ON_PAGE(PageUptodate(page), page);
319 :
320 : /*
321 : * Count submission time as memory stall. When the device is congested,
322 : * or the submitting cgroup IO-throttled, submission can be a
323 : * significant part of overall IO time.
324 : */
325 : if (workingset)
326 : psi_memstall_enter(&pflags);
327 : delayacct_swapin_start();
328 :
329 0 : if (frontswap_load(page) == 0) {
330 : SetPageUptodate(page);
331 : unlock_page(page);
332 : goto out;
333 : }
334 :
335 0 : if (data_race(sis->flags & SWP_FS_OPS)) {
336 0 : struct file *swap_file = sis->swap_file;
337 0 : struct address_space *mapping = swap_file->f_mapping;
338 :
339 0 : ret = mapping->a_ops->readpage(swap_file, page);
340 0 : if (!ret)
341 0 : count_vm_event(PSWPIN);
342 : goto out;
343 : }
344 :
345 0 : if (sis->flags & SWP_SYNCHRONOUS_IO) {
346 0 : ret = bdev_read_page(sis->bdev, swap_page_sector(page), page);
347 0 : if (!ret) {
348 0 : count_vm_event(PSWPIN);
349 0 : goto out;
350 : }
351 : }
352 :
353 0 : ret = 0;
354 0 : bio = bio_alloc(sis->bdev, 1, REQ_OP_READ, GFP_KERNEL);
355 0 : bio->bi_iter.bi_sector = swap_page_sector(page);
356 0 : bio->bi_end_io = end_swap_bio_read;
357 0 : bio_add_page(bio, page, thp_size(page), 0);
358 : /*
359 : * Keep this task valid during swap readpage because the oom killer may
360 : * attempt to access it in the page fault retry time check.
361 : */
362 0 : if (synchronous) {
363 0 : bio->bi_opf |= REQ_POLLED;
364 0 : get_task_struct(current);
365 0 : bio->bi_private = current;
366 : }
367 0 : count_vm_event(PSWPIN);
368 0 : bio_get(bio);
369 0 : submit_bio(bio);
370 0 : while (synchronous) {
371 0 : set_current_state(TASK_UNINTERRUPTIBLE);
372 0 : if (!READ_ONCE(bio->bi_private))
373 : break;
374 :
375 0 : if (!bio_poll(bio, NULL, 0))
376 0 : blk_io_schedule();
377 : }
378 0 : __set_current_state(TASK_RUNNING);
379 0 : bio_put(bio);
380 :
381 : out:
382 : if (workingset)
383 : psi_memstall_leave(&pflags);
384 : delayacct_swapin_end();
385 0 : return ret;
386 : }
387 :
388 0 : bool swap_dirty_folio(struct address_space *mapping, struct folio *folio)
389 : {
390 0 : struct swap_info_struct *sis = swp_swap_info(folio_swap_entry(folio));
391 :
392 0 : if (data_race(sis->flags & SWP_FS_OPS)) {
393 : const struct address_space_operations *aops;
394 :
395 0 : mapping = sis->swap_file->f_mapping;
396 0 : aops = mapping->a_ops;
397 :
398 : VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio);
399 0 : return aops->dirty_folio(mapping, folio);
400 : } else {
401 0 : return noop_dirty_folio(mapping, folio);
402 : }
403 : }
|
