Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Block device elevator/IO-scheduler.
4 : *
5 : * Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
6 : *
7 : * 30042000 Jens Axboe <axboe@kernel.dk> :
8 : *
9 : * Split the elevator a bit so that it is possible to choose a different
10 : * one or even write a new "plug in". There are three pieces:
11 : * - elevator_fn, inserts a new request in the queue list
12 : * - elevator_merge_fn, decides whether a new buffer can be merged with
13 : * an existing request
14 : * - elevator_dequeue_fn, called when a request is taken off the active list
15 : *
16 : * 20082000 Dave Jones <davej@suse.de> :
17 : * Removed tests for max-bomb-segments, which was breaking elvtune
18 : * when run without -bN
19 : *
20 : * Jens:
21 : * - Rework again to work with bio instead of buffer_heads
22 : * - loose bi_dev comparisons, partition handling is right now
23 : * - completely modularize elevator setup and teardown
24 : *
25 : */
26 : #include <linux/kernel.h>
27 : #include <linux/fs.h>
28 : #include <linux/blkdev.h>
29 : #include <linux/bio.h>
30 : #include <linux/module.h>
31 : #include <linux/slab.h>
32 : #include <linux/init.h>
33 : #include <linux/compiler.h>
34 : #include <linux/blktrace_api.h>
35 : #include <linux/hash.h>
36 : #include <linux/uaccess.h>
37 : #include <linux/pm_runtime.h>
38 :
39 : #include <trace/events/block.h>
40 :
41 : #include "elevator.h"
42 : #include "blk.h"
43 : #include "blk-mq-sched.h"
44 : #include "blk-pm.h"
45 : #include "blk-wbt.h"
46 : #include "blk-cgroup.h"
47 :
48 : static DEFINE_SPINLOCK(elv_list_lock);
49 : static LIST_HEAD(elv_list);
50 :
51 : /*
52 : * Merge hash stuff.
53 : */
54 : #define rq_hash_key(rq) (blk_rq_pos(rq) + blk_rq_sectors(rq))
55 :
56 : /*
57 : * Query io scheduler to see if the current process issuing bio may be
58 : * merged with rq.
59 : */
60 : static int elv_iosched_allow_bio_merge(struct request *rq, struct bio *bio)
61 : {
62 0 : struct request_queue *q = rq->q;
63 0 : struct elevator_queue *e = q->elevator;
64 :
65 0 : if (e->type->ops.allow_merge)
66 0 : return e->type->ops.allow_merge(q, rq, bio);
67 :
68 : return 1;
69 : }
70 :
71 : /*
72 : * can we safely merge with this request?
73 : */
74 0 : bool elv_bio_merge_ok(struct request *rq, struct bio *bio)
75 : {
76 0 : if (!blk_rq_merge_ok(rq, bio))
77 : return false;
78 :
79 0 : if (!elv_iosched_allow_bio_merge(rq, bio))
80 : return false;
81 :
82 0 : return true;
83 : }
84 : EXPORT_SYMBOL(elv_bio_merge_ok);
85 :
86 : static inline bool elv_support_features(unsigned int elv_features,
87 : unsigned int required_features)
88 : {
89 1 : return (required_features & elv_features) == required_features;
90 : }
91 :
92 : /**
93 : * elevator_match - Test an elevator name and features
94 : * @e: Scheduler to test
95 : * @name: Elevator name to test
96 : * @required_features: Features that the elevator must provide
97 : *
98 : * Return true if the elevator @e name matches @name and if @e provides all
99 : * the features specified by @required_features.
100 : */
101 1 : static bool elevator_match(const struct elevator_type *e, const char *name,
102 : unsigned int required_features)
103 : {
104 2 : if (!elv_support_features(e->elevator_features, required_features))
105 : return false;
106 1 : if (!strcmp(e->elevator_name, name))
107 : return true;
108 1 : if (e->elevator_alias && !strcmp(e->elevator_alias, name))
109 : return true;
110 :
111 : return false;
112 : }
113 :
114 : /**
115 : * elevator_find - Find an elevator
116 : * @name: Name of the elevator to find
117 : * @required_features: Features that the elevator must provide
118 : *
119 : * Return the first registered scheduler with name @name and supporting the
120 : * features @required_features and NULL otherwise.
121 : */
122 2 : static struct elevator_type *elevator_find(const char *name,
123 : unsigned int required_features)
124 : {
125 : struct elevator_type *e;
126 :
127 3 : list_for_each_entry(e, &elv_list, list) {
128 1 : if (elevator_match(e, name, required_features))
129 : return e;
130 : }
131 :
132 : return NULL;
133 : }
134 :
135 : static void elevator_put(struct elevator_type *e)
136 : {
137 0 : module_put(e->elevator_owner);
138 : }
139 :
140 0 : static struct elevator_type *elevator_get(struct request_queue *q,
141 : const char *name, bool try_loading)
142 : {
143 : struct elevator_type *e;
144 :
145 0 : spin_lock(&elv_list_lock);
146 :
147 0 : e = elevator_find(name, q->required_elevator_features);
148 0 : if (!e && try_loading) {
149 0 : spin_unlock(&elv_list_lock);
150 0 : request_module("%s-iosched", name);
151 0 : spin_lock(&elv_list_lock);
152 0 : e = elevator_find(name, q->required_elevator_features);
153 : }
154 :
155 : if (e && !try_module_get(e->elevator_owner))
156 : e = NULL;
157 :
158 0 : spin_unlock(&elv_list_lock);
159 0 : return e;
160 : }
161 :
162 : static struct kobj_type elv_ktype;
163 :
164 0 : struct elevator_queue *elevator_alloc(struct request_queue *q,
165 : struct elevator_type *e)
166 : {
167 : struct elevator_queue *eq;
168 :
169 0 : eq = kzalloc_node(sizeof(*eq), GFP_KERNEL, q->node);
170 0 : if (unlikely(!eq))
171 : return NULL;
172 :
173 0 : eq->type = e;
174 0 : kobject_init(&eq->kobj, &elv_ktype);
175 0 : mutex_init(&eq->sysfs_lock);
176 0 : hash_init(eq->hash);
177 :
178 : return eq;
179 : }
180 : EXPORT_SYMBOL(elevator_alloc);
181 :
182 0 : static void elevator_release(struct kobject *kobj)
183 : {
184 : struct elevator_queue *e;
185 :
186 0 : e = container_of(kobj, struct elevator_queue, kobj);
187 0 : elevator_put(e->type);
188 0 : kfree(e);
189 0 : }
190 :
191 0 : void elevator_exit(struct request_queue *q)
192 : {
193 0 : struct elevator_queue *e = q->elevator;
194 :
195 0 : ioc_clear_queue(q);
196 0 : blk_mq_sched_free_rqs(q);
197 :
198 0 : mutex_lock(&e->sysfs_lock);
199 0 : blk_mq_exit_sched(q, e);
200 0 : mutex_unlock(&e->sysfs_lock);
201 :
202 0 : kobject_put(&e->kobj);
203 0 : }
204 :
205 : static inline void __elv_rqhash_del(struct request *rq)
206 : {
207 0 : hash_del(&rq->hash);
208 0 : rq->rq_flags &= ~RQF_HASHED;
209 : }
210 :
211 0 : void elv_rqhash_del(struct request_queue *q, struct request *rq)
212 : {
213 0 : if (ELV_ON_HASH(rq))
214 : __elv_rqhash_del(rq);
215 0 : }
216 : EXPORT_SYMBOL_GPL(elv_rqhash_del);
217 :
218 0 : void elv_rqhash_add(struct request_queue *q, struct request *rq)
219 : {
220 0 : struct elevator_queue *e = q->elevator;
221 :
222 0 : BUG_ON(ELV_ON_HASH(rq));
223 0 : hash_add(e->hash, &rq->hash, rq_hash_key(rq));
224 0 : rq->rq_flags |= RQF_HASHED;
225 0 : }
226 : EXPORT_SYMBOL_GPL(elv_rqhash_add);
227 :
228 0 : void elv_rqhash_reposition(struct request_queue *q, struct request *rq)
229 : {
230 0 : __elv_rqhash_del(rq);
231 0 : elv_rqhash_add(q, rq);
232 0 : }
233 :
234 0 : struct request *elv_rqhash_find(struct request_queue *q, sector_t offset)
235 : {
236 0 : struct elevator_queue *e = q->elevator;
237 : struct hlist_node *next;
238 : struct request *rq;
239 :
240 0 : hash_for_each_possible_safe(e->hash, rq, next, hash, offset) {
241 0 : BUG_ON(!ELV_ON_HASH(rq));
242 :
243 0 : if (unlikely(!rq_mergeable(rq))) {
244 0 : __elv_rqhash_del(rq);
245 0 : continue;
246 : }
247 :
248 0 : if (rq_hash_key(rq) == offset)
249 : return rq;
250 : }
251 :
252 : return NULL;
253 : }
254 :
255 : /*
256 : * RB-tree support functions for inserting/lookup/removal of requests
257 : * in a sorted RB tree.
258 : */
259 0 : void elv_rb_add(struct rb_root *root, struct request *rq)
260 : {
261 0 : struct rb_node **p = &root->rb_node;
262 0 : struct rb_node *parent = NULL;
263 : struct request *__rq;
264 :
265 0 : while (*p) {
266 0 : parent = *p;
267 0 : __rq = rb_entry(parent, struct request, rb_node);
268 :
269 0 : if (blk_rq_pos(rq) < blk_rq_pos(__rq))
270 0 : p = &(*p)->rb_left;
271 0 : else if (blk_rq_pos(rq) >= blk_rq_pos(__rq))
272 0 : p = &(*p)->rb_right;
273 : }
274 :
275 0 : rb_link_node(&rq->rb_node, parent, p);
276 0 : rb_insert_color(&rq->rb_node, root);
277 0 : }
278 : EXPORT_SYMBOL(elv_rb_add);
279 :
280 0 : void elv_rb_del(struct rb_root *root, struct request *rq)
281 : {
282 0 : BUG_ON(RB_EMPTY_NODE(&rq->rb_node));
283 0 : rb_erase(&rq->rb_node, root);
284 0 : RB_CLEAR_NODE(&rq->rb_node);
285 0 : }
286 : EXPORT_SYMBOL(elv_rb_del);
287 :
288 0 : struct request *elv_rb_find(struct rb_root *root, sector_t sector)
289 : {
290 0 : struct rb_node *n = root->rb_node;
291 : struct request *rq;
292 :
293 0 : while (n) {
294 0 : rq = rb_entry(n, struct request, rb_node);
295 :
296 0 : if (sector < blk_rq_pos(rq))
297 0 : n = n->rb_left;
298 0 : else if (sector > blk_rq_pos(rq))
299 0 : n = n->rb_right;
300 : else
301 : return rq;
302 : }
303 :
304 : return NULL;
305 : }
306 : EXPORT_SYMBOL(elv_rb_find);
307 :
308 0 : enum elv_merge elv_merge(struct request_queue *q, struct request **req,
309 : struct bio *bio)
310 : {
311 0 : struct elevator_queue *e = q->elevator;
312 : struct request *__rq;
313 :
314 : /*
315 : * Levels of merges:
316 : * nomerges: No merges at all attempted
317 : * noxmerges: Only simple one-hit cache try
318 : * merges: All merge tries attempted
319 : */
320 0 : if (blk_queue_nomerges(q) || !bio_mergeable(bio))
321 : return ELEVATOR_NO_MERGE;
322 :
323 : /*
324 : * First try one-hit cache.
325 : */
326 0 : if (q->last_merge && elv_bio_merge_ok(q->last_merge, bio)) {
327 0 : enum elv_merge ret = blk_try_merge(q->last_merge, bio);
328 :
329 0 : if (ret != ELEVATOR_NO_MERGE) {
330 0 : *req = q->last_merge;
331 0 : return ret;
332 : }
333 : }
334 :
335 0 : if (blk_queue_noxmerges(q))
336 : return ELEVATOR_NO_MERGE;
337 :
338 : /*
339 : * See if our hash lookup can find a potential backmerge.
340 : */
341 0 : __rq = elv_rqhash_find(q, bio->bi_iter.bi_sector);
342 0 : if (__rq && elv_bio_merge_ok(__rq, bio)) {
343 0 : *req = __rq;
344 :
345 0 : if (blk_discard_mergable(__rq))
346 : return ELEVATOR_DISCARD_MERGE;
347 0 : return ELEVATOR_BACK_MERGE;
348 : }
349 :
350 0 : if (e->type->ops.request_merge)
351 0 : return e->type->ops.request_merge(q, req, bio);
352 :
353 : return ELEVATOR_NO_MERGE;
354 : }
355 :
356 : /*
357 : * Attempt to do an insertion back merge. Only check for the case where
358 : * we can append 'rq' to an existing request, so we can throw 'rq' away
359 : * afterwards.
360 : *
361 : * Returns true if we merged, false otherwise. 'free' will contain all
362 : * requests that need to be freed.
363 : */
364 0 : bool elv_attempt_insert_merge(struct request_queue *q, struct request *rq,
365 : struct list_head *free)
366 : {
367 : struct request *__rq;
368 : bool ret;
369 :
370 0 : if (blk_queue_nomerges(q))
371 : return false;
372 :
373 : /*
374 : * First try one-hit cache.
375 : */
376 0 : if (q->last_merge && blk_attempt_req_merge(q, q->last_merge, rq)) {
377 0 : list_add(&rq->queuelist, free);
378 0 : return true;
379 : }
380 :
381 0 : if (blk_queue_noxmerges(q))
382 : return false;
383 :
384 : ret = false;
385 : /*
386 : * See if our hash lookup can find a potential backmerge.
387 : */
388 : while (1) {
389 0 : __rq = elv_rqhash_find(q, blk_rq_pos(rq));
390 0 : if (!__rq || !blk_attempt_req_merge(q, __rq, rq))
391 : break;
392 :
393 0 : list_add(&rq->queuelist, free);
394 : /* The merged request could be merged with others, try again */
395 0 : ret = true;
396 0 : rq = __rq;
397 : }
398 :
399 : return ret;
400 : }
401 :
402 0 : void elv_merged_request(struct request_queue *q, struct request *rq,
403 : enum elv_merge type)
404 : {
405 0 : struct elevator_queue *e = q->elevator;
406 :
407 0 : if (e->type->ops.request_merged)
408 0 : e->type->ops.request_merged(q, rq, type);
409 :
410 0 : if (type == ELEVATOR_BACK_MERGE)
411 0 : elv_rqhash_reposition(q, rq);
412 :
413 0 : q->last_merge = rq;
414 0 : }
415 :
416 0 : void elv_merge_requests(struct request_queue *q, struct request *rq,
417 : struct request *next)
418 : {
419 0 : struct elevator_queue *e = q->elevator;
420 :
421 0 : if (e->type->ops.requests_merged)
422 0 : e->type->ops.requests_merged(q, rq, next);
423 :
424 0 : elv_rqhash_reposition(q, rq);
425 0 : q->last_merge = rq;
426 0 : }
427 :
428 0 : struct request *elv_latter_request(struct request_queue *q, struct request *rq)
429 : {
430 0 : struct elevator_queue *e = q->elevator;
431 :
432 0 : if (e->type->ops.next_request)
433 0 : return e->type->ops.next_request(q, rq);
434 :
435 : return NULL;
436 : }
437 :
438 0 : struct request *elv_former_request(struct request_queue *q, struct request *rq)
439 : {
440 0 : struct elevator_queue *e = q->elevator;
441 :
442 0 : if (e->type->ops.former_request)
443 0 : return e->type->ops.former_request(q, rq);
444 :
445 : return NULL;
446 : }
447 :
448 : #define to_elv(atr) container_of((atr), struct elv_fs_entry, attr)
449 :
450 : static ssize_t
451 0 : elv_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
452 : {
453 0 : struct elv_fs_entry *entry = to_elv(attr);
454 : struct elevator_queue *e;
455 : ssize_t error;
456 :
457 0 : if (!entry->show)
458 : return -EIO;
459 :
460 0 : e = container_of(kobj, struct elevator_queue, kobj);
461 0 : mutex_lock(&e->sysfs_lock);
462 0 : error = e->type ? entry->show(e, page) : -ENOENT;
463 0 : mutex_unlock(&e->sysfs_lock);
464 0 : return error;
465 : }
466 :
467 : static ssize_t
468 0 : elv_attr_store(struct kobject *kobj, struct attribute *attr,
469 : const char *page, size_t length)
470 : {
471 0 : struct elv_fs_entry *entry = to_elv(attr);
472 : struct elevator_queue *e;
473 : ssize_t error;
474 :
475 0 : if (!entry->store)
476 : return -EIO;
477 :
478 0 : e = container_of(kobj, struct elevator_queue, kobj);
479 0 : mutex_lock(&e->sysfs_lock);
480 0 : error = e->type ? entry->store(e, page, length) : -ENOENT;
481 0 : mutex_unlock(&e->sysfs_lock);
482 0 : return error;
483 : }
484 :
485 : static const struct sysfs_ops elv_sysfs_ops = {
486 : .show = elv_attr_show,
487 : .store = elv_attr_store,
488 : };
489 :
490 : static struct kobj_type elv_ktype = {
491 : .sysfs_ops = &elv_sysfs_ops,
492 : .release = elevator_release,
493 : };
494 :
495 0 : int elv_register_queue(struct request_queue *q, bool uevent)
496 : {
497 0 : struct elevator_queue *e = q->elevator;
498 : int error;
499 :
500 : lockdep_assert_held(&q->sysfs_lock);
501 :
502 0 : error = kobject_add(&e->kobj, &q->kobj, "%s", "iosched");
503 0 : if (!error) {
504 0 : struct elv_fs_entry *attr = e->type->elevator_attrs;
505 0 : if (attr) {
506 0 : while (attr->attr.name) {
507 0 : if (sysfs_create_file(&e->kobj, &attr->attr))
508 : break;
509 0 : attr++;
510 : }
511 : }
512 0 : if (uevent)
513 0 : kobject_uevent(&e->kobj, KOBJ_ADD);
514 :
515 0 : e->registered = 1;
516 : }
517 0 : return error;
518 : }
519 :
520 0 : void elv_unregister_queue(struct request_queue *q)
521 : {
522 0 : struct elevator_queue *e = q->elevator;
523 :
524 : lockdep_assert_held(&q->sysfs_lock);
525 :
526 0 : if (e && e->registered) {
527 0 : struct elevator_queue *e = q->elevator;
528 :
529 0 : kobject_uevent(&e->kobj, KOBJ_REMOVE);
530 0 : kobject_del(&e->kobj);
531 :
532 0 : e->registered = 0;
533 : }
534 0 : }
535 :
536 2 : int elv_register(struct elevator_type *e)
537 : {
538 : /* insert_requests and dispatch_request are mandatory */
539 2 : if (WARN_ON_ONCE(!e->ops.insert_requests || !e->ops.dispatch_request))
540 : return -EINVAL;
541 :
542 : /* create icq_cache if requested */
543 2 : if (e->icq_size) {
544 0 : if (WARN_ON(e->icq_size < sizeof(struct io_cq)) ||
545 0 : WARN_ON(e->icq_align < __alignof__(struct io_cq)))
546 : return -EINVAL;
547 :
548 0 : snprintf(e->icq_cache_name, sizeof(e->icq_cache_name),
549 : "%s_io_cq", e->elevator_name);
550 0 : e->icq_cache = kmem_cache_create(e->icq_cache_name, e->icq_size,
551 0 : e->icq_align, 0, NULL);
552 0 : if (!e->icq_cache)
553 : return -ENOMEM;
554 : }
555 :
556 : /* register, don't allow duplicate names */
557 2 : spin_lock(&elv_list_lock);
558 2 : if (elevator_find(e->elevator_name, 0)) {
559 0 : spin_unlock(&elv_list_lock);
560 0 : kmem_cache_destroy(e->icq_cache);
561 0 : return -EBUSY;
562 : }
563 4 : list_add_tail(&e->list, &elv_list);
564 2 : spin_unlock(&elv_list_lock);
565 :
566 2 : printk(KERN_INFO "io scheduler %s registered\n", e->elevator_name);
567 :
568 2 : return 0;
569 : }
570 : EXPORT_SYMBOL_GPL(elv_register);
571 :
572 0 : void elv_unregister(struct elevator_type *e)
573 : {
574 : /* unregister */
575 0 : spin_lock(&elv_list_lock);
576 0 : list_del_init(&e->list);
577 0 : spin_unlock(&elv_list_lock);
578 :
579 : /*
580 : * Destroy icq_cache if it exists. icq's are RCU managed. Make
581 : * sure all RCU operations are complete before proceeding.
582 : */
583 0 : if (e->icq_cache) {
584 0 : rcu_barrier();
585 0 : kmem_cache_destroy(e->icq_cache);
586 0 : e->icq_cache = NULL;
587 : }
588 0 : }
589 : EXPORT_SYMBOL_GPL(elv_unregister);
590 :
591 0 : int elevator_switch_mq(struct request_queue *q,
592 : struct elevator_type *new_e)
593 : {
594 : int ret;
595 :
596 : lockdep_assert_held(&q->sysfs_lock);
597 :
598 0 : if (q->elevator) {
599 0 : elv_unregister_queue(q);
600 0 : elevator_exit(q);
601 : }
602 :
603 0 : ret = blk_mq_init_sched(q, new_e);
604 0 : if (ret)
605 : goto out;
606 :
607 0 : if (new_e) {
608 0 : ret = elv_register_queue(q, true);
609 0 : if (ret) {
610 0 : elevator_exit(q);
611 0 : goto out;
612 : }
613 : }
614 :
615 : if (new_e)
616 : blk_add_trace_msg(q, "elv switch: %s", new_e->elevator_name);
617 : else
618 : blk_add_trace_msg(q, "elv switch: none");
619 :
620 : out:
621 0 : return ret;
622 : }
623 :
624 : static inline bool elv_support_iosched(struct request_queue *q)
625 : {
626 0 : if (!queue_is_mq(q) ||
627 0 : (q->tag_set && (q->tag_set->flags & BLK_MQ_F_NO_SCHED)))
628 : return false;
629 : return true;
630 : }
631 :
632 : /*
633 : * For single queue devices, default to using mq-deadline. If we have multiple
634 : * queues or mq-deadline is not available, default to "none".
635 : */
636 0 : static struct elevator_type *elevator_get_default(struct request_queue *q)
637 : {
638 0 : if (q->tag_set && q->tag_set->flags & BLK_MQ_F_NO_SCHED_BY_DEFAULT)
639 : return NULL;
640 :
641 0 : if (q->nr_hw_queues != 1 &&
642 0 : !blk_mq_is_shared_tags(q->tag_set->flags))
643 : return NULL;
644 :
645 0 : return elevator_get(q, "mq-deadline", false);
646 : }
647 :
648 : /*
649 : * Get the first elevator providing the features required by the request queue.
650 : * Default to "none" if no matching elevator is found.
651 : */
652 : static struct elevator_type *elevator_get_by_features(struct request_queue *q)
653 : {
654 0 : struct elevator_type *e, *found = NULL;
655 :
656 0 : spin_lock(&elv_list_lock);
657 :
658 0 : list_for_each_entry(e, &elv_list, list) {
659 0 : if (elv_support_features(e->elevator_features,
660 : q->required_elevator_features)) {
661 : found = e;
662 : break;
663 : }
664 : }
665 :
666 : if (found && !try_module_get(found->elevator_owner))
667 : found = NULL;
668 :
669 0 : spin_unlock(&elv_list_lock);
670 : return found;
671 : }
672 :
673 : /*
674 : * For a device queue that has no required features, use the default elevator
675 : * settings. Otherwise, use the first elevator available matching the required
676 : * features. If no suitable elevator is find or if the chosen elevator
677 : * initialization fails, fall back to the "none" elevator (no elevator).
678 : */
679 0 : void elevator_init_mq(struct request_queue *q)
680 : {
681 : struct elevator_type *e;
682 : int err;
683 :
684 0 : if (!elv_support_iosched(q))
685 : return;
686 :
687 0 : WARN_ON_ONCE(blk_queue_registered(q));
688 :
689 0 : if (unlikely(q->elevator))
690 : return;
691 :
692 0 : if (!q->required_elevator_features)
693 0 : e = elevator_get_default(q);
694 : else
695 0 : e = elevator_get_by_features(q);
696 0 : if (!e)
697 : return;
698 :
699 : /*
700 : * We are called before adding disk, when there isn't any FS I/O,
701 : * so freezing queue plus canceling dispatch work is enough to
702 : * drain any dispatch activities originated from passthrough
703 : * requests, then no need to quiesce queue which may add long boot
704 : * latency, especially when lots of disks are involved.
705 : */
706 0 : blk_mq_freeze_queue(q);
707 0 : blk_mq_cancel_work_sync(q);
708 :
709 0 : err = blk_mq_init_sched(q, e);
710 :
711 0 : blk_mq_unfreeze_queue(q);
712 :
713 0 : if (err) {
714 0 : pr_warn("\"%s\" elevator initialization failed, "
715 : "falling back to \"none\"\n", e->elevator_name);
716 0 : elevator_put(e);
717 : }
718 : }
719 :
720 : /*
721 : * switch to new_e io scheduler. be careful not to introduce deadlocks -
722 : * we don't free the old io scheduler, before we have allocated what we
723 : * need for the new one. this way we have a chance of going back to the old
724 : * one, if the new one fails init for some reason.
725 : */
726 0 : static int elevator_switch(struct request_queue *q, struct elevator_type *new_e)
727 : {
728 : int err;
729 :
730 : lockdep_assert_held(&q->sysfs_lock);
731 :
732 0 : blk_mq_freeze_queue(q);
733 0 : blk_mq_quiesce_queue(q);
734 :
735 0 : err = elevator_switch_mq(q, new_e);
736 :
737 0 : blk_mq_unquiesce_queue(q);
738 0 : blk_mq_unfreeze_queue(q);
739 :
740 0 : return err;
741 : }
742 :
743 : /*
744 : * Switch this queue to the given IO scheduler.
745 : */
746 0 : static int __elevator_change(struct request_queue *q, const char *name)
747 : {
748 : char elevator_name[ELV_NAME_MAX];
749 : struct elevator_type *e;
750 :
751 : /* Make sure queue is not in the middle of being removed */
752 0 : if (!blk_queue_registered(q))
753 : return -ENOENT;
754 :
755 : /*
756 : * Special case for mq, turn off scheduling
757 : */
758 0 : if (!strncmp(name, "none", 4)) {
759 0 : if (!q->elevator)
760 : return 0;
761 0 : return elevator_switch(q, NULL);
762 : }
763 :
764 0 : strlcpy(elevator_name, name, sizeof(elevator_name));
765 0 : e = elevator_get(q, strstrip(elevator_name), true);
766 0 : if (!e)
767 : return -EINVAL;
768 :
769 0 : if (q->elevator &&
770 0 : elevator_match(q->elevator->type, elevator_name, 0)) {
771 : elevator_put(e);
772 : return 0;
773 : }
774 :
775 0 : return elevator_switch(q, e);
776 : }
777 :
778 0 : ssize_t elv_iosched_store(struct request_queue *q, const char *name,
779 : size_t count)
780 : {
781 : int ret;
782 :
783 0 : if (!elv_support_iosched(q))
784 0 : return count;
785 :
786 0 : ret = __elevator_change(q, name);
787 0 : if (!ret)
788 0 : return count;
789 :
790 0 : return ret;
791 : }
792 :
793 0 : ssize_t elv_iosched_show(struct request_queue *q, char *name)
794 : {
795 0 : struct elevator_queue *e = q->elevator;
796 0 : struct elevator_type *elv = NULL;
797 : struct elevator_type *__e;
798 0 : int len = 0;
799 :
800 0 : if (!queue_is_mq(q))
801 0 : return sprintf(name, "none\n");
802 :
803 0 : if (!q->elevator)
804 0 : len += sprintf(name+len, "[none] ");
805 : else
806 0 : elv = e->type;
807 :
808 0 : spin_lock(&elv_list_lock);
809 0 : list_for_each_entry(__e, &elv_list, list) {
810 0 : if (elv && elevator_match(elv, __e->elevator_name, 0)) {
811 0 : len += sprintf(name+len, "[%s] ", elv->elevator_name);
812 0 : continue;
813 : }
814 0 : if (elv_support_iosched(q) &&
815 0 : elevator_match(__e, __e->elevator_name,
816 : q->required_elevator_features))
817 0 : len += sprintf(name+len, "%s ", __e->elevator_name);
818 : }
819 0 : spin_unlock(&elv_list_lock);
820 :
821 0 : if (q->elevator)
822 0 : len += sprintf(name+len, "none");
823 :
824 0 : len += sprintf(len+name, "\n");
825 0 : return len;
826 : }
827 :
828 0 : struct request *elv_rb_former_request(struct request_queue *q,
829 : struct request *rq)
830 : {
831 0 : struct rb_node *rbprev = rb_prev(&rq->rb_node);
832 :
833 0 : if (rbprev)
834 0 : return rb_entry_rq(rbprev);
835 :
836 : return NULL;
837 : }
838 : EXPORT_SYMBOL(elv_rb_former_request);
839 :
840 0 : struct request *elv_rb_latter_request(struct request_queue *q,
841 : struct request *rq)
842 : {
843 0 : struct rb_node *rbnext = rb_next(&rq->rb_node);
844 :
845 0 : if (rbnext)
846 0 : return rb_entry_rq(rbnext);
847 :
848 : return NULL;
849 : }
850 : EXPORT_SYMBOL(elv_rb_latter_request);
851 :
852 0 : static int __init elevator_setup(char *str)
853 : {
854 0 : pr_warn("Kernel parameter elevator= does not have any effect anymore.\n"
855 : "Please use sysfs to set IO scheduler for individual devices.\n");
856 0 : return 1;
857 : }
858 :
859 : __setup("elevator=", elevator_setup);
|
