Line data Source code
1 : /* SPDX-License-Identifier: GPL-2.0 OR MIT */
2 : /**************************************************************************
3 : *
4 : * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
5 : * All Rights Reserved.
6 : *
7 : * Permission is hereby granted, free of charge, to any person obtaining a
8 : * copy of this software and associated documentation files (the
9 : * "Software"), to deal in the Software without restriction, including
10 : * without limitation the rights to use, copy, modify, merge, publish,
11 : * distribute, sub license, and/or sell copies of the Software, and to
12 : * permit persons to whom the Software is furnished to do so, subject to
13 : * the following conditions:
14 : *
15 : * The above copyright notice and this permission notice (including the
16 : * next paragraph) shall be included in all copies or substantial portions
17 : * of the Software.
18 : *
19 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 : * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 : * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
22 : * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
23 : * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
24 : * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
25 : * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 : *
27 : **************************************************************************/
28 : /*
29 : * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
30 : */
31 :
32 : #define pr_fmt(fmt) "[TTM] " fmt
33 :
34 : #include <drm/ttm/ttm_bo_driver.h>
35 : #include <drm/ttm/ttm_placement.h>
36 : #include <linux/jiffies.h>
37 : #include <linux/slab.h>
38 : #include <linux/sched.h>
39 : #include <linux/mm.h>
40 : #include <linux/file.h>
41 : #include <linux/module.h>
42 : #include <linux/atomic.h>
43 : #include <linux/dma-resv.h>
44 :
45 : #include "ttm_module.h"
46 :
47 : /* default destructor */
48 0 : static void ttm_bo_default_destroy(struct ttm_buffer_object *bo)
49 : {
50 0 : kfree(bo);
51 0 : }
52 :
53 0 : static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
54 : struct ttm_placement *placement)
55 : {
56 0 : struct drm_printer p = drm_debug_printer(TTM_PFX);
57 : struct ttm_resource_manager *man;
58 : int i, mem_type;
59 :
60 0 : drm_printf(&p, "No space for %p (%lu pages, %zuK, %zuM)\n",
61 0 : bo, bo->resource->num_pages, bo->base.size >> 10,
62 0 : bo->base.size >> 20);
63 0 : for (i = 0; i < placement->num_placement; i++) {
64 0 : mem_type = placement->placement[i].mem_type;
65 0 : drm_printf(&p, " placement[%d]=0x%08X (%d)\n",
66 : i, placement->placement[i].flags, mem_type);
67 0 : man = ttm_manager_type(bo->bdev, mem_type);
68 0 : ttm_resource_manager_debug(man, &p);
69 : }
70 0 : }
71 :
72 : /**
73 : * ttm_bo_move_to_lru_tail
74 : *
75 : * @bo: The buffer object.
76 : *
77 : * Move this BO to the tail of all lru lists used to lookup and reserve an
78 : * object. This function must be called with struct ttm_global::lru_lock
79 : * held, and is used to make a BO less likely to be considered for eviction.
80 : */
81 0 : void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo)
82 : {
83 : dma_resv_assert_held(bo->base.resv);
84 :
85 0 : if (bo->resource)
86 0 : ttm_resource_move_to_lru_tail(bo->resource);
87 0 : }
88 : EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
89 :
90 : /**
91 : * ttm_bo_set_bulk_move - update BOs bulk move object
92 : *
93 : * @bo: The buffer object.
94 : *
95 : * Update the BOs bulk move object, making sure that resources are added/removed
96 : * as well. A bulk move allows to move many resource on the LRU at once,
97 : * resulting in much less overhead of maintaining the LRU.
98 : * The only requirement is that the resources stay together on the LRU and are
99 : * never separated. This is enforces by setting the bulk_move structure on a BO.
100 : * ttm_lru_bulk_move_tail() should be used to move all resources to the tail of
101 : * their LRU list.
102 : */
103 0 : void ttm_bo_set_bulk_move(struct ttm_buffer_object *bo,
104 : struct ttm_lru_bulk_move *bulk)
105 : {
106 : dma_resv_assert_held(bo->base.resv);
107 :
108 0 : if (bo->bulk_move == bulk)
109 : return;
110 :
111 0 : spin_lock(&bo->bdev->lru_lock);
112 0 : if (bo->resource)
113 0 : ttm_resource_del_bulk_move(bo->resource, bo);
114 0 : bo->bulk_move = bulk;
115 0 : if (bo->resource)
116 0 : ttm_resource_add_bulk_move(bo->resource, bo);
117 0 : spin_unlock(&bo->bdev->lru_lock);
118 : }
119 : EXPORT_SYMBOL(ttm_bo_set_bulk_move);
120 :
121 0 : static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
122 : struct ttm_resource *mem, bool evict,
123 : struct ttm_operation_ctx *ctx,
124 : struct ttm_place *hop)
125 : {
126 : struct ttm_resource_manager *old_man, *new_man;
127 0 : struct ttm_device *bdev = bo->bdev;
128 : int ret;
129 :
130 0 : old_man = ttm_manager_type(bdev, bo->resource->mem_type);
131 0 : new_man = ttm_manager_type(bdev, mem->mem_type);
132 :
133 0 : ttm_bo_unmap_virtual(bo);
134 :
135 : /*
136 : * Create and bind a ttm if required.
137 : */
138 :
139 0 : if (new_man->use_tt) {
140 : /* Zero init the new TTM structure if the old location should
141 : * have used one as well.
142 : */
143 0 : ret = ttm_tt_create(bo, old_man->use_tt);
144 0 : if (ret)
145 : goto out_err;
146 :
147 0 : if (mem->mem_type != TTM_PL_SYSTEM) {
148 0 : ret = ttm_tt_populate(bo->bdev, bo->ttm, ctx);
149 0 : if (ret)
150 : goto out_err;
151 : }
152 : }
153 :
154 0 : ret = dma_resv_reserve_fences(bo->base.resv, 1);
155 0 : if (ret)
156 : goto out_err;
157 :
158 0 : ret = bdev->funcs->move(bo, evict, ctx, mem, hop);
159 0 : if (ret) {
160 0 : if (ret == -EMULTIHOP)
161 : return ret;
162 : goto out_err;
163 : }
164 :
165 0 : ctx->bytes_moved += bo->base.size;
166 0 : return 0;
167 :
168 : out_err:
169 0 : new_man = ttm_manager_type(bdev, bo->resource->mem_type);
170 0 : if (!new_man->use_tt)
171 0 : ttm_bo_tt_destroy(bo);
172 :
173 : return ret;
174 : }
175 :
176 : /*
177 : * Call bo::reserved.
178 : * Will release GPU memory type usage on destruction.
179 : * This is the place to put in driver specific hooks to release
180 : * driver private resources.
181 : * Will release the bo::reserved lock.
182 : */
183 :
184 0 : static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
185 : {
186 0 : if (bo->bdev->funcs->delete_mem_notify)
187 0 : bo->bdev->funcs->delete_mem_notify(bo);
188 :
189 0 : ttm_bo_tt_destroy(bo);
190 0 : ttm_resource_free(bo, &bo->resource);
191 0 : }
192 :
193 0 : static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
194 : {
195 : int r;
196 :
197 0 : if (bo->base.resv == &bo->base._resv)
198 : return 0;
199 :
200 0 : BUG_ON(!dma_resv_trylock(&bo->base._resv));
201 :
202 0 : r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv);
203 0 : dma_resv_unlock(&bo->base._resv);
204 0 : if (r)
205 : return r;
206 :
207 0 : if (bo->type != ttm_bo_type_sg) {
208 : /* This works because the BO is about to be destroyed and nobody
209 : * reference it any more. The only tricky case is the trylock on
210 : * the resv object while holding the lru_lock.
211 : */
212 0 : spin_lock(&bo->bdev->lru_lock);
213 0 : bo->base.resv = &bo->base._resv;
214 0 : spin_unlock(&bo->bdev->lru_lock);
215 : }
216 :
217 : return r;
218 : }
219 :
220 0 : static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
221 : {
222 0 : struct dma_resv *resv = &bo->base._resv;
223 : struct dma_resv_iter cursor;
224 : struct dma_fence *fence;
225 :
226 : dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP);
227 0 : dma_resv_for_each_fence_unlocked(&cursor, fence) {
228 0 : if (!fence->ops->signaled)
229 0 : dma_fence_enable_sw_signaling(fence);
230 : }
231 0 : dma_resv_iter_end(&cursor);
232 0 : }
233 :
234 : /**
235 : * ttm_bo_cleanup_refs
236 : * If bo idle, remove from lru lists, and unref.
237 : * If not idle, block if possible.
238 : *
239 : * Must be called with lru_lock and reservation held, this function
240 : * will drop the lru lock and optionally the reservation lock before returning.
241 : *
242 : * @bo: The buffer object to clean-up
243 : * @interruptible: Any sleeps should occur interruptibly.
244 : * @no_wait_gpu: Never wait for gpu. Return -EBUSY instead.
245 : * @unlock_resv: Unlock the reservation lock as well.
246 : */
247 :
248 0 : static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
249 : bool interruptible, bool no_wait_gpu,
250 : bool unlock_resv)
251 : {
252 0 : struct dma_resv *resv = &bo->base._resv;
253 : int ret;
254 :
255 0 : if (dma_resv_test_signaled(resv, DMA_RESV_USAGE_BOOKKEEP))
256 : ret = 0;
257 : else
258 0 : ret = -EBUSY;
259 :
260 0 : if (ret && !no_wait_gpu) {
261 : long lret;
262 :
263 0 : if (unlock_resv)
264 0 : dma_resv_unlock(bo->base.resv);
265 0 : spin_unlock(&bo->bdev->lru_lock);
266 :
267 0 : lret = dma_resv_wait_timeout(resv, DMA_RESV_USAGE_BOOKKEEP,
268 : interruptible,
269 : 30 * HZ);
270 :
271 0 : if (lret < 0)
272 0 : return lret;
273 0 : else if (lret == 0)
274 : return -EBUSY;
275 :
276 0 : spin_lock(&bo->bdev->lru_lock);
277 0 : if (unlock_resv && !dma_resv_trylock(bo->base.resv)) {
278 : /*
279 : * We raced, and lost, someone else holds the reservation now,
280 : * and is probably busy in ttm_bo_cleanup_memtype_use.
281 : *
282 : * Even if it's not the case, because we finished waiting any
283 : * delayed destruction would succeed, so just return success
284 : * here.
285 : */
286 0 : spin_unlock(&bo->bdev->lru_lock);
287 0 : return 0;
288 : }
289 : ret = 0;
290 : }
291 :
292 0 : if (ret || unlikely(list_empty(&bo->ddestroy))) {
293 0 : if (unlock_resv)
294 0 : dma_resv_unlock(bo->base.resv);
295 0 : spin_unlock(&bo->bdev->lru_lock);
296 0 : return ret;
297 : }
298 :
299 0 : list_del_init(&bo->ddestroy);
300 0 : spin_unlock(&bo->bdev->lru_lock);
301 0 : ttm_bo_cleanup_memtype_use(bo);
302 :
303 0 : if (unlock_resv)
304 0 : dma_resv_unlock(bo->base.resv);
305 :
306 0 : ttm_bo_put(bo);
307 :
308 0 : return 0;
309 : }
310 :
311 : /*
312 : * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
313 : * encountered buffers.
314 : */
315 0 : bool ttm_bo_delayed_delete(struct ttm_device *bdev, bool remove_all)
316 : {
317 : struct list_head removed;
318 : bool empty;
319 :
320 0 : INIT_LIST_HEAD(&removed);
321 :
322 0 : spin_lock(&bdev->lru_lock);
323 0 : while (!list_empty(&bdev->ddestroy)) {
324 : struct ttm_buffer_object *bo;
325 :
326 0 : bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object,
327 : ddestroy);
328 0 : list_move_tail(&bo->ddestroy, &removed);
329 0 : if (!ttm_bo_get_unless_zero(bo))
330 0 : continue;
331 :
332 0 : if (remove_all || bo->base.resv != &bo->base._resv) {
333 0 : spin_unlock(&bdev->lru_lock);
334 0 : dma_resv_lock(bo->base.resv, NULL);
335 :
336 0 : spin_lock(&bdev->lru_lock);
337 0 : ttm_bo_cleanup_refs(bo, false, !remove_all, true);
338 :
339 0 : } else if (dma_resv_trylock(bo->base.resv)) {
340 0 : ttm_bo_cleanup_refs(bo, false, !remove_all, true);
341 : } else {
342 0 : spin_unlock(&bdev->lru_lock);
343 : }
344 :
345 0 : ttm_bo_put(bo);
346 0 : spin_lock(&bdev->lru_lock);
347 : }
348 0 : list_splice_tail(&removed, &bdev->ddestroy);
349 0 : empty = list_empty(&bdev->ddestroy);
350 0 : spin_unlock(&bdev->lru_lock);
351 :
352 0 : return empty;
353 : }
354 :
355 0 : static void ttm_bo_release(struct kref *kref)
356 : {
357 0 : struct ttm_buffer_object *bo =
358 0 : container_of(kref, struct ttm_buffer_object, kref);
359 0 : struct ttm_device *bdev = bo->bdev;
360 : int ret;
361 :
362 0 : WARN_ON_ONCE(bo->pin_count);
363 0 : WARN_ON_ONCE(bo->bulk_move);
364 :
365 0 : if (!bo->deleted) {
366 0 : ret = ttm_bo_individualize_resv(bo);
367 0 : if (ret) {
368 : /* Last resort, if we fail to allocate memory for the
369 : * fences block for the BO to become idle
370 : */
371 0 : dma_resv_wait_timeout(bo->base.resv,
372 : DMA_RESV_USAGE_BOOKKEEP, false,
373 : 30 * HZ);
374 : }
375 :
376 0 : if (bo->bdev->funcs->release_notify)
377 0 : bo->bdev->funcs->release_notify(bo);
378 :
379 0 : drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node);
380 0 : ttm_mem_io_free(bdev, bo->resource);
381 : }
382 :
383 0 : if (!dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP) ||
384 0 : !dma_resv_trylock(bo->base.resv)) {
385 : /* The BO is not idle, resurrect it for delayed destroy */
386 0 : ttm_bo_flush_all_fences(bo);
387 0 : bo->deleted = true;
388 :
389 0 : spin_lock(&bo->bdev->lru_lock);
390 :
391 : /*
392 : * Make pinned bos immediately available to
393 : * shrinkers, now that they are queued for
394 : * destruction.
395 : *
396 : * FIXME: QXL is triggering this. Can be removed when the
397 : * driver is fixed.
398 : */
399 0 : if (bo->pin_count) {
400 0 : bo->pin_count = 0;
401 0 : ttm_resource_move_to_lru_tail(bo->resource);
402 : }
403 :
404 0 : kref_init(&bo->kref);
405 0 : list_add_tail(&bo->ddestroy, &bdev->ddestroy);
406 0 : spin_unlock(&bo->bdev->lru_lock);
407 :
408 0 : schedule_delayed_work(&bdev->wq,
409 : ((HZ / 100) < 1) ? 1 : HZ / 100);
410 : return;
411 : }
412 :
413 0 : spin_lock(&bo->bdev->lru_lock);
414 0 : list_del(&bo->ddestroy);
415 0 : spin_unlock(&bo->bdev->lru_lock);
416 :
417 0 : ttm_bo_cleanup_memtype_use(bo);
418 0 : dma_resv_unlock(bo->base.resv);
419 :
420 0 : atomic_dec(&ttm_glob.bo_count);
421 0 : bo->destroy(bo);
422 : }
423 :
424 0 : void ttm_bo_put(struct ttm_buffer_object *bo)
425 : {
426 0 : kref_put(&bo->kref, ttm_bo_release);
427 0 : }
428 : EXPORT_SYMBOL(ttm_bo_put);
429 :
430 0 : int ttm_bo_lock_delayed_workqueue(struct ttm_device *bdev)
431 : {
432 0 : return cancel_delayed_work_sync(&bdev->wq);
433 : }
434 : EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue);
435 :
436 0 : void ttm_bo_unlock_delayed_workqueue(struct ttm_device *bdev, int resched)
437 : {
438 0 : if (resched)
439 0 : schedule_delayed_work(&bdev->wq,
440 : ((HZ / 100) < 1) ? 1 : HZ / 100);
441 0 : }
442 : EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);
443 :
444 0 : static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo,
445 : struct ttm_resource **mem,
446 : struct ttm_operation_ctx *ctx,
447 : struct ttm_place *hop)
448 : {
449 : struct ttm_placement hop_placement;
450 : struct ttm_resource *hop_mem;
451 : int ret;
452 :
453 0 : hop_placement.num_placement = hop_placement.num_busy_placement = 1;
454 0 : hop_placement.placement = hop_placement.busy_placement = hop;
455 :
456 : /* find space in the bounce domain */
457 0 : ret = ttm_bo_mem_space(bo, &hop_placement, &hop_mem, ctx);
458 0 : if (ret)
459 : return ret;
460 : /* move to the bounce domain */
461 0 : ret = ttm_bo_handle_move_mem(bo, hop_mem, false, ctx, NULL);
462 0 : if (ret) {
463 0 : ttm_resource_free(bo, &hop_mem);
464 : return ret;
465 : }
466 : return 0;
467 : }
468 :
469 0 : static int ttm_bo_evict(struct ttm_buffer_object *bo,
470 : struct ttm_operation_ctx *ctx)
471 : {
472 0 : struct ttm_device *bdev = bo->bdev;
473 : struct ttm_resource *evict_mem;
474 : struct ttm_placement placement;
475 : struct ttm_place hop;
476 0 : int ret = 0;
477 :
478 0 : memset(&hop, 0, sizeof(hop));
479 :
480 : dma_resv_assert_held(bo->base.resv);
481 :
482 0 : placement.num_placement = 0;
483 0 : placement.num_busy_placement = 0;
484 0 : bdev->funcs->evict_flags(bo, &placement);
485 :
486 0 : if (!placement.num_placement && !placement.num_busy_placement) {
487 0 : ret = ttm_bo_wait(bo, true, false);
488 0 : if (ret)
489 : return ret;
490 :
491 : /*
492 : * Since we've already synced, this frees backing store
493 : * immediately.
494 : */
495 0 : return ttm_bo_pipeline_gutting(bo);
496 : }
497 :
498 0 : ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx);
499 0 : if (ret) {
500 0 : if (ret != -ERESTARTSYS) {
501 0 : pr_err("Failed to find memory space for buffer 0x%p eviction\n",
502 : bo);
503 0 : ttm_bo_mem_space_debug(bo, &placement);
504 : }
505 : goto out;
506 : }
507 :
508 : bounce:
509 0 : ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop);
510 0 : if (ret == -EMULTIHOP) {
511 0 : ret = ttm_bo_bounce_temp_buffer(bo, &evict_mem, ctx, &hop);
512 0 : if (ret) {
513 0 : pr_err("Buffer eviction failed\n");
514 0 : ttm_resource_free(bo, &evict_mem);
515 0 : goto out;
516 : }
517 : /* try and move to final place now. */
518 : goto bounce;
519 : }
520 : out:
521 : return ret;
522 : }
523 :
524 0 : bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
525 : const struct ttm_place *place)
526 : {
527 : dma_resv_assert_held(bo->base.resv);
528 0 : if (bo->resource->mem_type == TTM_PL_SYSTEM)
529 : return true;
530 :
531 : /* Don't evict this BO if it's outside of the
532 : * requested placement range
533 : */
534 0 : if (place->fpfn >= (bo->resource->start + bo->resource->num_pages) ||
535 0 : (place->lpfn && place->lpfn <= bo->resource->start))
536 : return false;
537 :
538 0 : return true;
539 : }
540 : EXPORT_SYMBOL(ttm_bo_eviction_valuable);
541 :
542 : /*
543 : * Check the target bo is allowable to be evicted or swapout, including cases:
544 : *
545 : * a. if share same reservation object with ctx->resv, have assumption
546 : * reservation objects should already be locked, so not lock again and
547 : * return true directly when either the opreation allow_reserved_eviction
548 : * or the target bo already is in delayed free list;
549 : *
550 : * b. Otherwise, trylock it.
551 : */
552 0 : static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
553 : struct ttm_operation_ctx *ctx,
554 : const struct ttm_place *place,
555 : bool *locked, bool *busy)
556 : {
557 0 : bool ret = false;
558 :
559 0 : if (bo->base.resv == ctx->resv) {
560 : dma_resv_assert_held(bo->base.resv);
561 0 : if (ctx->allow_res_evict)
562 0 : ret = true;
563 0 : *locked = false;
564 0 : if (busy)
565 0 : *busy = false;
566 : } else {
567 0 : ret = dma_resv_trylock(bo->base.resv);
568 0 : *locked = ret;
569 0 : if (busy)
570 0 : *busy = !ret;
571 : }
572 :
573 0 : if (ret && place && (bo->resource->mem_type != place->mem_type ||
574 0 : !bo->bdev->funcs->eviction_valuable(bo, place))) {
575 0 : ret = false;
576 0 : if (*locked) {
577 0 : dma_resv_unlock(bo->base.resv);
578 0 : *locked = false;
579 : }
580 : }
581 :
582 0 : return ret;
583 : }
584 :
585 : /**
586 : * ttm_mem_evict_wait_busy - wait for a busy BO to become available
587 : *
588 : * @busy_bo: BO which couldn't be locked with trylock
589 : * @ctx: operation context
590 : * @ticket: acquire ticket
591 : *
592 : * Try to lock a busy buffer object to avoid failing eviction.
593 : */
594 0 : static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo,
595 : struct ttm_operation_ctx *ctx,
596 : struct ww_acquire_ctx *ticket)
597 : {
598 : int r;
599 :
600 0 : if (!busy_bo || !ticket)
601 : return -EBUSY;
602 :
603 0 : if (ctx->interruptible)
604 0 : r = dma_resv_lock_interruptible(busy_bo->base.resv,
605 : ticket);
606 : else
607 0 : r = dma_resv_lock(busy_bo->base.resv, ticket);
608 :
609 : /*
610 : * TODO: It would be better to keep the BO locked until allocation is at
611 : * least tried one more time, but that would mean a much larger rework
612 : * of TTM.
613 : */
614 0 : if (!r)
615 0 : dma_resv_unlock(busy_bo->base.resv);
616 :
617 0 : return r == -EDEADLK ? -EBUSY : r;
618 : }
619 :
620 0 : int ttm_mem_evict_first(struct ttm_device *bdev,
621 : struct ttm_resource_manager *man,
622 : const struct ttm_place *place,
623 : struct ttm_operation_ctx *ctx,
624 : struct ww_acquire_ctx *ticket)
625 : {
626 0 : struct ttm_buffer_object *bo = NULL, *busy_bo = NULL;
627 : struct ttm_resource_cursor cursor;
628 : struct ttm_resource *res;
629 0 : bool locked = false;
630 : int ret;
631 :
632 0 : spin_lock(&bdev->lru_lock);
633 0 : ttm_resource_manager_for_each_res(man, &cursor, res) {
634 : bool busy;
635 :
636 0 : if (!ttm_bo_evict_swapout_allowable(res->bo, ctx, place,
637 : &locked, &busy)) {
638 0 : if (busy && !busy_bo && ticket !=
639 0 : dma_resv_locking_ctx(res->bo->base.resv))
640 0 : busy_bo = res->bo;
641 0 : continue;
642 : }
643 :
644 0 : if (ttm_bo_get_unless_zero(res->bo)) {
645 0 : bo = res->bo;
646 0 : break;
647 : }
648 0 : if (locked)
649 0 : dma_resv_unlock(res->bo->base.resv);
650 : }
651 :
652 0 : if (!bo) {
653 0 : if (busy_bo && !ttm_bo_get_unless_zero(busy_bo))
654 0 : busy_bo = NULL;
655 0 : spin_unlock(&bdev->lru_lock);
656 0 : ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket);
657 0 : if (busy_bo)
658 : ttm_bo_put(busy_bo);
659 : return ret;
660 : }
661 :
662 0 : if (bo->deleted) {
663 0 : ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
664 0 : ctx->no_wait_gpu, locked);
665 0 : ttm_bo_put(bo);
666 0 : return ret;
667 : }
668 :
669 0 : spin_unlock(&bdev->lru_lock);
670 :
671 0 : ret = ttm_bo_evict(bo, ctx);
672 0 : if (locked)
673 0 : ttm_bo_unreserve(bo);
674 : else
675 0 : ttm_bo_move_to_lru_tail_unlocked(bo);
676 :
677 0 : ttm_bo_put(bo);
678 0 : return ret;
679 : }
680 :
681 : /**
682 : * ttm_bo_pin - Pin the buffer object.
683 : * @bo: The buffer object to pin
684 : *
685 : * Make sure the buffer is not evicted any more during memory pressure.
686 : * @bo must be unpinned again by calling ttm_bo_unpin().
687 : */
688 0 : void ttm_bo_pin(struct ttm_buffer_object *bo)
689 : {
690 : dma_resv_assert_held(bo->base.resv);
691 0 : WARN_ON_ONCE(!kref_read(&bo->kref));
692 0 : spin_lock(&bo->bdev->lru_lock);
693 0 : if (bo->resource)
694 0 : ttm_resource_del_bulk_move(bo->resource, bo);
695 0 : ++bo->pin_count;
696 0 : spin_unlock(&bo->bdev->lru_lock);
697 0 : }
698 : EXPORT_SYMBOL(ttm_bo_pin);
699 :
700 : /**
701 : * ttm_bo_unpin - Unpin the buffer object.
702 : * @bo: The buffer object to unpin
703 : *
704 : * Allows the buffer object to be evicted again during memory pressure.
705 : */
706 0 : void ttm_bo_unpin(struct ttm_buffer_object *bo)
707 : {
708 : dma_resv_assert_held(bo->base.resv);
709 0 : WARN_ON_ONCE(!kref_read(&bo->kref));
710 0 : if (WARN_ON_ONCE(!bo->pin_count))
711 : return;
712 :
713 0 : spin_lock(&bo->bdev->lru_lock);
714 0 : --bo->pin_count;
715 0 : if (bo->resource)
716 0 : ttm_resource_add_bulk_move(bo->resource, bo);
717 0 : spin_unlock(&bo->bdev->lru_lock);
718 : }
719 : EXPORT_SYMBOL(ttm_bo_unpin);
720 :
721 : /*
722 : * Add the last move fence to the BO as kernel dependency and reserve a new
723 : * fence slot.
724 : */
725 0 : static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
726 : struct ttm_resource_manager *man,
727 : struct ttm_resource *mem,
728 : bool no_wait_gpu)
729 : {
730 : struct dma_fence *fence;
731 : int ret;
732 :
733 0 : spin_lock(&man->move_lock);
734 0 : fence = dma_fence_get(man->move);
735 0 : spin_unlock(&man->move_lock);
736 :
737 0 : if (!fence)
738 : return 0;
739 :
740 0 : if (no_wait_gpu) {
741 0 : ret = dma_fence_is_signaled(fence) ? 0 : -EBUSY;
742 0 : dma_fence_put(fence);
743 0 : return ret;
744 : }
745 :
746 0 : dma_resv_add_fence(bo->base.resv, fence, DMA_RESV_USAGE_KERNEL);
747 :
748 0 : ret = dma_resv_reserve_fences(bo->base.resv, 1);
749 0 : dma_fence_put(fence);
750 0 : return ret;
751 : }
752 :
753 : /*
754 : * Repeatedly evict memory from the LRU for @mem_type until we create enough
755 : * space, or we've evicted everything and there isn't enough space.
756 : */
757 0 : static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
758 : const struct ttm_place *place,
759 : struct ttm_resource **mem,
760 : struct ttm_operation_ctx *ctx)
761 : {
762 0 : struct ttm_device *bdev = bo->bdev;
763 : struct ttm_resource_manager *man;
764 : struct ww_acquire_ctx *ticket;
765 : int ret;
766 :
767 0 : man = ttm_manager_type(bdev, place->mem_type);
768 0 : ticket = dma_resv_locking_ctx(bo->base.resv);
769 : do {
770 0 : ret = ttm_resource_alloc(bo, place, mem);
771 0 : if (likely(!ret))
772 : break;
773 0 : if (unlikely(ret != -ENOSPC))
774 : return ret;
775 0 : ret = ttm_mem_evict_first(bdev, man, place, ctx,
776 : ticket);
777 0 : if (unlikely(ret != 0))
778 : return ret;
779 : } while (1);
780 :
781 0 : return ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu);
782 : }
783 :
784 : /*
785 : * Creates space for memory region @mem according to its type.
786 : *
787 : * This function first searches for free space in compatible memory types in
788 : * the priority order defined by the driver. If free space isn't found, then
789 : * ttm_bo_mem_force_space is attempted in priority order to evict and find
790 : * space.
791 : */
792 0 : int ttm_bo_mem_space(struct ttm_buffer_object *bo,
793 : struct ttm_placement *placement,
794 : struct ttm_resource **mem,
795 : struct ttm_operation_ctx *ctx)
796 : {
797 0 : struct ttm_device *bdev = bo->bdev;
798 0 : bool type_found = false;
799 : int i, ret;
800 :
801 0 : ret = dma_resv_reserve_fences(bo->base.resv, 1);
802 0 : if (unlikely(ret))
803 : return ret;
804 :
805 0 : for (i = 0; i < placement->num_placement; ++i) {
806 0 : const struct ttm_place *place = &placement->placement[i];
807 : struct ttm_resource_manager *man;
808 :
809 0 : man = ttm_manager_type(bdev, place->mem_type);
810 0 : if (!man || !ttm_resource_manager_used(man))
811 0 : continue;
812 :
813 0 : type_found = true;
814 0 : ret = ttm_resource_alloc(bo, place, mem);
815 0 : if (ret == -ENOSPC)
816 0 : continue;
817 0 : if (unlikely(ret))
818 : goto error;
819 :
820 0 : ret = ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu);
821 0 : if (unlikely(ret)) {
822 0 : ttm_resource_free(bo, mem);
823 0 : if (ret == -EBUSY)
824 0 : continue;
825 :
826 : goto error;
827 : }
828 : return 0;
829 : }
830 :
831 0 : for (i = 0; i < placement->num_busy_placement; ++i) {
832 0 : const struct ttm_place *place = &placement->busy_placement[i];
833 : struct ttm_resource_manager *man;
834 :
835 0 : man = ttm_manager_type(bdev, place->mem_type);
836 0 : if (!man || !ttm_resource_manager_used(man))
837 0 : continue;
838 :
839 0 : type_found = true;
840 0 : ret = ttm_bo_mem_force_space(bo, place, mem, ctx);
841 0 : if (likely(!ret))
842 : return 0;
843 :
844 0 : if (ret && ret != -EBUSY)
845 : goto error;
846 : }
847 :
848 0 : ret = -ENOMEM;
849 0 : if (!type_found) {
850 0 : pr_err(TTM_PFX "No compatible memory type found\n");
851 0 : ret = -EINVAL;
852 : }
853 :
854 : error:
855 : return ret;
856 : }
857 : EXPORT_SYMBOL(ttm_bo_mem_space);
858 :
859 0 : static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
860 : struct ttm_placement *placement,
861 : struct ttm_operation_ctx *ctx)
862 : {
863 : struct ttm_resource *mem;
864 : struct ttm_place hop;
865 : int ret;
866 :
867 : dma_resv_assert_held(bo->base.resv);
868 :
869 : /*
870 : * Determine where to move the buffer.
871 : *
872 : * If driver determines move is going to need
873 : * an extra step then it will return -EMULTIHOP
874 : * and the buffer will be moved to the temporary
875 : * stop and the driver will be called to make
876 : * the second hop.
877 : */
878 0 : ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
879 0 : if (ret)
880 : return ret;
881 : bounce:
882 0 : ret = ttm_bo_handle_move_mem(bo, mem, false, ctx, &hop);
883 0 : if (ret == -EMULTIHOP) {
884 0 : ret = ttm_bo_bounce_temp_buffer(bo, &mem, ctx, &hop);
885 0 : if (ret)
886 : goto out;
887 : /* try and move to final place now. */
888 : goto bounce;
889 : }
890 : out:
891 0 : if (ret)
892 0 : ttm_resource_free(bo, &mem);
893 : return ret;
894 : }
895 :
896 0 : int ttm_bo_validate(struct ttm_buffer_object *bo,
897 : struct ttm_placement *placement,
898 : struct ttm_operation_ctx *ctx)
899 : {
900 : int ret;
901 :
902 : dma_resv_assert_held(bo->base.resv);
903 :
904 : /*
905 : * Remove the backing store if no placement is given.
906 : */
907 0 : if (!placement->num_placement && !placement->num_busy_placement)
908 0 : return ttm_bo_pipeline_gutting(bo);
909 :
910 : /*
911 : * Check whether we need to move buffer.
912 : */
913 0 : if (!ttm_resource_compat(bo->resource, placement)) {
914 0 : ret = ttm_bo_move_buffer(bo, placement, ctx);
915 0 : if (ret)
916 : return ret;
917 : }
918 : /*
919 : * We might need to add a TTM.
920 : */
921 0 : if (bo->resource->mem_type == TTM_PL_SYSTEM) {
922 0 : ret = ttm_tt_create(bo, true);
923 0 : if (ret)
924 : return ret;
925 : }
926 : return 0;
927 : }
928 : EXPORT_SYMBOL(ttm_bo_validate);
929 :
930 0 : int ttm_bo_init_reserved(struct ttm_device *bdev,
931 : struct ttm_buffer_object *bo,
932 : size_t size,
933 : enum ttm_bo_type type,
934 : struct ttm_placement *placement,
935 : uint32_t page_alignment,
936 : struct ttm_operation_ctx *ctx,
937 : struct sg_table *sg,
938 : struct dma_resv *resv,
939 : void (*destroy) (struct ttm_buffer_object *))
940 : {
941 : static const struct ttm_place sys_mem = { .mem_type = TTM_PL_SYSTEM };
942 : bool locked;
943 : int ret;
944 :
945 0 : bo->destroy = destroy ? destroy : ttm_bo_default_destroy;
946 :
947 0 : kref_init(&bo->kref);
948 0 : INIT_LIST_HEAD(&bo->ddestroy);
949 0 : bo->bdev = bdev;
950 0 : bo->type = type;
951 0 : bo->page_alignment = page_alignment;
952 0 : bo->pin_count = 0;
953 0 : bo->sg = sg;
954 0 : bo->bulk_move = NULL;
955 0 : if (resv) {
956 0 : bo->base.resv = resv;
957 : dma_resv_assert_held(bo->base.resv);
958 : } else {
959 0 : bo->base.resv = &bo->base._resv;
960 : }
961 0 : atomic_inc(&ttm_glob.bo_count);
962 :
963 0 : ret = ttm_resource_alloc(bo, &sys_mem, &bo->resource);
964 0 : if (unlikely(ret)) {
965 0 : ttm_bo_put(bo);
966 0 : return ret;
967 : }
968 :
969 : /*
970 : * For ttm_bo_type_device buffers, allocate
971 : * address space from the device.
972 : */
973 0 : if (bo->type == ttm_bo_type_device ||
974 : bo->type == ttm_bo_type_sg)
975 0 : ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node,
976 0 : bo->resource->num_pages);
977 :
978 : /* passed reservation objects should already be locked,
979 : * since otherwise lockdep will be angered in radeon.
980 : */
981 0 : if (!resv) {
982 0 : locked = dma_resv_trylock(bo->base.resv);
983 0 : WARN_ON(!locked);
984 : }
985 :
986 0 : if (likely(!ret))
987 0 : ret = ttm_bo_validate(bo, placement, ctx);
988 :
989 0 : if (unlikely(ret)) {
990 0 : if (!resv)
991 0 : ttm_bo_unreserve(bo);
992 :
993 0 : ttm_bo_put(bo);
994 0 : return ret;
995 : }
996 :
997 : return ret;
998 : }
999 : EXPORT_SYMBOL(ttm_bo_init_reserved);
1000 :
1001 0 : int ttm_bo_init(struct ttm_device *bdev,
1002 : struct ttm_buffer_object *bo,
1003 : size_t size,
1004 : enum ttm_bo_type type,
1005 : struct ttm_placement *placement,
1006 : uint32_t page_alignment,
1007 : bool interruptible,
1008 : struct sg_table *sg,
1009 : struct dma_resv *resv,
1010 : void (*destroy) (struct ttm_buffer_object *))
1011 : {
1012 0 : struct ttm_operation_ctx ctx = { interruptible, false };
1013 : int ret;
1014 :
1015 0 : ret = ttm_bo_init_reserved(bdev, bo, size, type, placement,
1016 : page_alignment, &ctx, sg, resv, destroy);
1017 0 : if (ret)
1018 : return ret;
1019 :
1020 0 : if (!resv)
1021 0 : ttm_bo_unreserve(bo);
1022 :
1023 : return 0;
1024 : }
1025 : EXPORT_SYMBOL(ttm_bo_init);
1026 :
1027 : /*
1028 : * buffer object vm functions.
1029 : */
1030 :
1031 0 : void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1032 : {
1033 0 : struct ttm_device *bdev = bo->bdev;
1034 :
1035 0 : drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping);
1036 0 : ttm_mem_io_free(bdev, bo->resource);
1037 0 : }
1038 : EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1039 :
1040 0 : int ttm_bo_wait(struct ttm_buffer_object *bo,
1041 : bool interruptible, bool no_wait)
1042 : {
1043 0 : long timeout = 15 * HZ;
1044 :
1045 0 : if (no_wait) {
1046 0 : if (dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP))
1047 : return 0;
1048 : else
1049 0 : return -EBUSY;
1050 : }
1051 :
1052 0 : timeout = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP,
1053 : interruptible, timeout);
1054 0 : if (timeout < 0)
1055 0 : return timeout;
1056 :
1057 0 : if (timeout == 0)
1058 : return -EBUSY;
1059 :
1060 0 : return 0;
1061 : }
1062 : EXPORT_SYMBOL(ttm_bo_wait);
1063 :
1064 0 : int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx,
1065 : gfp_t gfp_flags)
1066 : {
1067 : struct ttm_place place;
1068 : bool locked;
1069 : int ret;
1070 :
1071 : /*
1072 : * While the bo may already reside in SYSTEM placement, set
1073 : * SYSTEM as new placement to cover also the move further below.
1074 : * The driver may use the fact that we're moving from SYSTEM
1075 : * as an indication that we're about to swap out.
1076 : */
1077 0 : memset(&place, 0, sizeof(place));
1078 0 : place.mem_type = bo->resource->mem_type;
1079 0 : if (!ttm_bo_evict_swapout_allowable(bo, ctx, &place, &locked, NULL))
1080 : return -EBUSY;
1081 :
1082 0 : if (!bo->ttm || !ttm_tt_is_populated(bo->ttm) ||
1083 0 : bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL ||
1084 0 : bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED ||
1085 0 : !ttm_bo_get_unless_zero(bo)) {
1086 0 : if (locked)
1087 0 : dma_resv_unlock(bo->base.resv);
1088 : return -EBUSY;
1089 : }
1090 :
1091 0 : if (bo->deleted) {
1092 0 : ret = ttm_bo_cleanup_refs(bo, false, false, locked);
1093 0 : ttm_bo_put(bo);
1094 0 : return ret == -EBUSY ? -ENOSPC : ret;
1095 : }
1096 :
1097 : /* TODO: Cleanup the locking */
1098 0 : spin_unlock(&bo->bdev->lru_lock);
1099 :
1100 : /*
1101 : * Move to system cached
1102 : */
1103 0 : if (bo->resource->mem_type != TTM_PL_SYSTEM) {
1104 0 : struct ttm_operation_ctx ctx = { false, false };
1105 : struct ttm_resource *evict_mem;
1106 : struct ttm_place hop;
1107 :
1108 0 : memset(&hop, 0, sizeof(hop));
1109 0 : place.mem_type = TTM_PL_SYSTEM;
1110 0 : ret = ttm_resource_alloc(bo, &place, &evict_mem);
1111 0 : if (unlikely(ret))
1112 : goto out;
1113 :
1114 0 : ret = ttm_bo_handle_move_mem(bo, evict_mem, true, &ctx, &hop);
1115 0 : if (unlikely(ret != 0)) {
1116 0 : WARN(ret == -EMULTIHOP, "Unexpected multihop in swaput - likely driver bug.\n");
1117 : goto out;
1118 : }
1119 : }
1120 :
1121 : /*
1122 : * Make sure BO is idle.
1123 : */
1124 0 : ret = ttm_bo_wait(bo, false, false);
1125 0 : if (unlikely(ret != 0))
1126 : goto out;
1127 :
1128 0 : ttm_bo_unmap_virtual(bo);
1129 :
1130 : /*
1131 : * Swap out. Buffer will be swapped in again as soon as
1132 : * anyone tries to access a ttm page.
1133 : */
1134 0 : if (bo->bdev->funcs->swap_notify)
1135 0 : bo->bdev->funcs->swap_notify(bo);
1136 :
1137 0 : if (ttm_tt_is_populated(bo->ttm))
1138 0 : ret = ttm_tt_swapout(bo->bdev, bo->ttm, gfp_flags);
1139 : out:
1140 :
1141 : /*
1142 : * Unreserve without putting on LRU to avoid swapping out an
1143 : * already swapped buffer.
1144 : */
1145 0 : if (locked)
1146 0 : dma_resv_unlock(bo->base.resv);
1147 0 : ttm_bo_put(bo);
1148 0 : return ret == -EBUSY ? -ENOSPC : ret;
1149 : }
1150 :
1151 0 : void ttm_bo_tt_destroy(struct ttm_buffer_object *bo)
1152 : {
1153 0 : if (bo->ttm == NULL)
1154 : return;
1155 :
1156 0 : ttm_tt_unpopulate(bo->bdev, bo->ttm);
1157 0 : ttm_tt_destroy(bo->bdev, bo->ttm);
1158 0 : bo->ttm = NULL;
1159 : }
|
