Line data Source code
1 : /* SPDX-License-Identifier: GPL-2.0 */
2 : /*
3 : * include/linux/backing-dev.h
4 : *
5 : * low-level device information and state which is propagated up through
6 : * to high-level code.
7 : */
8 :
9 : #ifndef _LINUX_BACKING_DEV_H
10 : #define _LINUX_BACKING_DEV_H
11 :
12 : #include <linux/kernel.h>
13 : #include <linux/fs.h>
14 : #include <linux/sched.h>
15 : #include <linux/device.h>
16 : #include <linux/writeback.h>
17 : #include <linux/backing-dev-defs.h>
18 : #include <linux/slab.h>
19 :
20 : struct blkcg;
21 :
22 : static inline struct backing_dev_info *bdi_get(struct backing_dev_info *bdi)
23 : {
24 0 : kref_get(&bdi->refcnt);
25 : return bdi;
26 : }
27 :
28 : struct backing_dev_info *bdi_get_by_id(u64 id);
29 : void bdi_put(struct backing_dev_info *bdi);
30 :
31 : __printf(2, 3)
32 : int bdi_register(struct backing_dev_info *bdi, const char *fmt, ...);
33 : __printf(2, 0)
34 : int bdi_register_va(struct backing_dev_info *bdi, const char *fmt,
35 : va_list args);
36 : void bdi_set_owner(struct backing_dev_info *bdi, struct device *owner);
37 : void bdi_unregister(struct backing_dev_info *bdi);
38 :
39 : struct backing_dev_info *bdi_alloc(int node_id);
40 :
41 : void wb_start_background_writeback(struct bdi_writeback *wb);
42 : void wb_workfn(struct work_struct *work);
43 : void wb_wakeup_delayed(struct bdi_writeback *wb);
44 :
45 : void wb_wait_for_completion(struct wb_completion *done);
46 :
47 : extern spinlock_t bdi_lock;
48 : extern struct list_head bdi_list;
49 :
50 : extern struct workqueue_struct *bdi_wq;
51 : extern struct workqueue_struct *bdi_async_bio_wq;
52 :
53 : static inline bool wb_has_dirty_io(struct bdi_writeback *wb)
54 : {
55 0 : return test_bit(WB_has_dirty_io, &wb->state);
56 : }
57 :
58 : static inline bool bdi_has_dirty_io(struct backing_dev_info *bdi)
59 : {
60 : /*
61 : * @bdi->tot_write_bandwidth is guaranteed to be > 0 if there are
62 : * any dirty wbs. See wb_update_write_bandwidth().
63 : */
64 0 : return atomic_long_read(&bdi->tot_write_bandwidth);
65 : }
66 :
67 : static inline void wb_stat_mod(struct bdi_writeback *wb,
68 : enum wb_stat_item item, s64 amount)
69 : {
70 0 : percpu_counter_add_batch(&wb->stat[item], amount, WB_STAT_BATCH);
71 : }
72 :
73 : static inline void inc_wb_stat(struct bdi_writeback *wb, enum wb_stat_item item)
74 : {
75 : wb_stat_mod(wb, item, 1);
76 : }
77 :
78 : static inline void dec_wb_stat(struct bdi_writeback *wb, enum wb_stat_item item)
79 : {
80 : wb_stat_mod(wb, item, -1);
81 : }
82 :
83 : static inline s64 wb_stat(struct bdi_writeback *wb, enum wb_stat_item item)
84 : {
85 0 : return percpu_counter_read_positive(&wb->stat[item]);
86 : }
87 :
88 : static inline s64 wb_stat_sum(struct bdi_writeback *wb, enum wb_stat_item item)
89 : {
90 0 : return percpu_counter_sum_positive(&wb->stat[item]);
91 : }
92 :
93 : extern void wb_writeout_inc(struct bdi_writeback *wb);
94 :
95 : /*
96 : * maximal error of a stat counter.
97 : */
98 : static inline unsigned long wb_stat_error(void)
99 : {
100 : #ifdef CONFIG_SMP
101 : return nr_cpu_ids * WB_STAT_BATCH;
102 : #else
103 : return 1;
104 : #endif
105 : }
106 :
107 : int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio);
108 : int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned int max_ratio);
109 :
110 : /*
111 : * Flags in backing_dev_info::capability
112 : *
113 : * BDI_CAP_WRITEBACK: Supports dirty page writeback, and dirty pages
114 : * should contribute to accounting
115 : * BDI_CAP_WRITEBACK_ACCT: Automatically account writeback pages
116 : * BDI_CAP_STRICTLIMIT: Keep number of dirty pages below bdi threshold
117 : */
118 : #define BDI_CAP_WRITEBACK (1 << 0)
119 : #define BDI_CAP_WRITEBACK_ACCT (1 << 1)
120 : #define BDI_CAP_STRICTLIMIT (1 << 2)
121 :
122 : extern struct backing_dev_info noop_backing_dev_info;
123 :
124 : /**
125 : * writeback_in_progress - determine whether there is writeback in progress
126 : * @wb: bdi_writeback of interest
127 : *
128 : * Determine whether there is writeback waiting to be handled against a
129 : * bdi_writeback.
130 : */
131 : static inline bool writeback_in_progress(struct bdi_writeback *wb)
132 : {
133 0 : return test_bit(WB_writeback_running, &wb->state);
134 : }
135 :
136 : struct backing_dev_info *inode_to_bdi(struct inode *inode);
137 :
138 : static inline bool mapping_can_writeback(struct address_space *mapping)
139 : {
140 0 : return inode_to_bdi(mapping->host)->capabilities & BDI_CAP_WRITEBACK;
141 : }
142 :
143 : static inline int bdi_sched_wait(void *word)
144 : {
145 : schedule();
146 : return 0;
147 : }
148 :
149 : #ifdef CONFIG_CGROUP_WRITEBACK
150 :
151 : struct bdi_writeback *wb_get_lookup(struct backing_dev_info *bdi,
152 : struct cgroup_subsys_state *memcg_css);
153 : struct bdi_writeback *wb_get_create(struct backing_dev_info *bdi,
154 : struct cgroup_subsys_state *memcg_css,
155 : gfp_t gfp);
156 : void wb_memcg_offline(struct mem_cgroup *memcg);
157 : void wb_blkcg_offline(struct blkcg *blkcg);
158 :
159 : /**
160 : * inode_cgwb_enabled - test whether cgroup writeback is enabled on an inode
161 : * @inode: inode of interest
162 : *
163 : * Cgroup writeback requires support from the filesystem. Also, both memcg and
164 : * iocg have to be on the default hierarchy. Test whether all conditions are
165 : * met.
166 : *
167 : * Note that the test result may change dynamically on the same inode
168 : * depending on how memcg and iocg are configured.
169 : */
170 : static inline bool inode_cgwb_enabled(struct inode *inode)
171 : {
172 : struct backing_dev_info *bdi = inode_to_bdi(inode);
173 :
174 : return cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
175 : cgroup_subsys_on_dfl(io_cgrp_subsys) &&
176 : (bdi->capabilities & BDI_CAP_WRITEBACK) &&
177 : (inode->i_sb->s_iflags & SB_I_CGROUPWB);
178 : }
179 :
180 : /**
181 : * wb_find_current - find wb for %current on a bdi
182 : * @bdi: bdi of interest
183 : *
184 : * Find the wb of @bdi which matches both the memcg and blkcg of %current.
185 : * Must be called under rcu_read_lock() which protects the returend wb.
186 : * NULL if not found.
187 : */
188 : static inline struct bdi_writeback *wb_find_current(struct backing_dev_info *bdi)
189 : {
190 : struct cgroup_subsys_state *memcg_css;
191 : struct bdi_writeback *wb;
192 :
193 : memcg_css = task_css(current, memory_cgrp_id);
194 : if (!memcg_css->parent)
195 : return &bdi->wb;
196 :
197 : wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id);
198 :
199 : /*
200 : * %current's blkcg equals the effective blkcg of its memcg. No
201 : * need to use the relatively expensive cgroup_get_e_css().
202 : */
203 : if (likely(wb && wb->blkcg_css == task_css(current, io_cgrp_id)))
204 : return wb;
205 : return NULL;
206 : }
207 :
208 : /**
209 : * wb_get_create_current - get or create wb for %current on a bdi
210 : * @bdi: bdi of interest
211 : * @gfp: allocation mask
212 : *
213 : * Equivalent to wb_get_create() on %current's memcg. This function is
214 : * called from a relatively hot path and optimizes the common cases using
215 : * wb_find_current().
216 : */
217 : static inline struct bdi_writeback *
218 : wb_get_create_current(struct backing_dev_info *bdi, gfp_t gfp)
219 : {
220 : struct bdi_writeback *wb;
221 :
222 : rcu_read_lock();
223 : wb = wb_find_current(bdi);
224 : if (wb && unlikely(!wb_tryget(wb)))
225 : wb = NULL;
226 : rcu_read_unlock();
227 :
228 : if (unlikely(!wb)) {
229 : struct cgroup_subsys_state *memcg_css;
230 :
231 : memcg_css = task_get_css(current, memory_cgrp_id);
232 : wb = wb_get_create(bdi, memcg_css, gfp);
233 : css_put(memcg_css);
234 : }
235 : return wb;
236 : }
237 :
238 : /**
239 : * inode_to_wb_is_valid - test whether an inode has a wb associated
240 : * @inode: inode of interest
241 : *
242 : * Returns %true if @inode has a wb associated. May be called without any
243 : * locking.
244 : */
245 : static inline bool inode_to_wb_is_valid(struct inode *inode)
246 : {
247 : return inode->i_wb;
248 : }
249 :
250 : /**
251 : * inode_to_wb - determine the wb of an inode
252 : * @inode: inode of interest
253 : *
254 : * Returns the wb @inode is currently associated with. The caller must be
255 : * holding either @inode->i_lock, the i_pages lock, or the
256 : * associated wb's list_lock.
257 : */
258 : static inline struct bdi_writeback *inode_to_wb(const struct inode *inode)
259 : {
260 : #ifdef CONFIG_LOCKDEP
261 : WARN_ON_ONCE(debug_locks &&
262 : (!lockdep_is_held(&inode->i_lock) &&
263 : !lockdep_is_held(&inode->i_mapping->i_pages.xa_lock) &&
264 : !lockdep_is_held(&inode->i_wb->list_lock)));
265 : #endif
266 : return inode->i_wb;
267 : }
268 :
269 : static inline struct bdi_writeback *inode_to_wb_wbc(
270 : struct inode *inode,
271 : struct writeback_control *wbc)
272 : {
273 : /*
274 : * If wbc does not have inode attached, it means cgroup writeback was
275 : * disabled when wbc started. Just use the default wb in that case.
276 : */
277 : return wbc->wb ? wbc->wb : &inode_to_bdi(inode)->wb;
278 : }
279 :
280 : /**
281 : * unlocked_inode_to_wb_begin - begin unlocked inode wb access transaction
282 : * @inode: target inode
283 : * @cookie: output param, to be passed to the end function
284 : *
285 : * The caller wants to access the wb associated with @inode but isn't
286 : * holding inode->i_lock, the i_pages lock or wb->list_lock. This
287 : * function determines the wb associated with @inode and ensures that the
288 : * association doesn't change until the transaction is finished with
289 : * unlocked_inode_to_wb_end().
290 : *
291 : * The caller must call unlocked_inode_to_wb_end() with *@cookie afterwards and
292 : * can't sleep during the transaction. IRQs may or may not be disabled on
293 : * return.
294 : */
295 : static inline struct bdi_writeback *
296 : unlocked_inode_to_wb_begin(struct inode *inode, struct wb_lock_cookie *cookie)
297 : {
298 : rcu_read_lock();
299 :
300 : /*
301 : * Paired with store_release in inode_switch_wbs_work_fn() and
302 : * ensures that we see the new wb if we see cleared I_WB_SWITCH.
303 : */
304 : cookie->locked = smp_load_acquire(&inode->i_state) & I_WB_SWITCH;
305 :
306 : if (unlikely(cookie->locked))
307 : xa_lock_irqsave(&inode->i_mapping->i_pages, cookie->flags);
308 :
309 : /*
310 : * Protected by either !I_WB_SWITCH + rcu_read_lock() or the i_pages
311 : * lock. inode_to_wb() will bark. Deref directly.
312 : */
313 : return inode->i_wb;
314 : }
315 :
316 : /**
317 : * unlocked_inode_to_wb_end - end inode wb access transaction
318 : * @inode: target inode
319 : * @cookie: @cookie from unlocked_inode_to_wb_begin()
320 : */
321 : static inline void unlocked_inode_to_wb_end(struct inode *inode,
322 : struct wb_lock_cookie *cookie)
323 : {
324 : if (unlikely(cookie->locked))
325 : xa_unlock_irqrestore(&inode->i_mapping->i_pages, cookie->flags);
326 :
327 : rcu_read_unlock();
328 : }
329 :
330 : #else /* CONFIG_CGROUP_WRITEBACK */
331 :
332 : static inline bool inode_cgwb_enabled(struct inode *inode)
333 : {
334 : return false;
335 : }
336 :
337 : static inline struct bdi_writeback *wb_find_current(struct backing_dev_info *bdi)
338 : {
339 : return &bdi->wb;
340 : }
341 :
342 : static inline struct bdi_writeback *
343 : wb_get_create_current(struct backing_dev_info *bdi, gfp_t gfp)
344 : {
345 : return &bdi->wb;
346 : }
347 :
348 : static inline bool inode_to_wb_is_valid(struct inode *inode)
349 : {
350 : return true;
351 : }
352 :
353 : static inline struct bdi_writeback *inode_to_wb(struct inode *inode)
354 : {
355 0 : return &inode_to_bdi(inode)->wb;
356 : }
357 :
358 : static inline struct bdi_writeback *inode_to_wb_wbc(
359 : struct inode *inode,
360 : struct writeback_control *wbc)
361 : {
362 0 : return inode_to_wb(inode);
363 : }
364 :
365 :
366 : static inline struct bdi_writeback *
367 : unlocked_inode_to_wb_begin(struct inode *inode, struct wb_lock_cookie *cookie)
368 : {
369 0 : return inode_to_wb(inode);
370 : }
371 :
372 : static inline void unlocked_inode_to_wb_end(struct inode *inode,
373 : struct wb_lock_cookie *cookie)
374 : {
375 : }
376 :
377 : static inline void wb_memcg_offline(struct mem_cgroup *memcg)
378 : {
379 : }
380 :
381 : static inline void wb_blkcg_offline(struct blkcg *blkcg)
382 : {
383 : }
384 :
385 : #endif /* CONFIG_CGROUP_WRITEBACK */
386 :
387 : const char *bdi_dev_name(struct backing_dev_info *bdi);
388 :
389 : #endif /* _LINUX_BACKING_DEV_H */
|
