Line data Source code
1 : /*
2 : * Copyright 2020 Advanced Micro Devices, Inc.
3 : *
4 : * Permission is hereby granted, free of charge, to any person obtaining a
5 : * copy of this software and associated documentation files (the "Software"),
6 : * to deal in the Software without restriction, including without limitation
7 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 : * and/or sell copies of the Software, and to permit persons to whom the
9 : * Software is furnished to do so, subject to the following conditions:
10 : *
11 : * The above copyright notice and this permission notice shall be included in
12 : * all copies or substantial portions of the Software.
13 : *
14 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 : * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 : * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 : * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 : * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 : * OTHER DEALINGS IN THE SOFTWARE.
21 : *
22 : * Authors: Christian König
23 : */
24 :
25 : #include <linux/iosys-map.h>
26 : #include <linux/io-mapping.h>
27 : #include <linux/scatterlist.h>
28 :
29 : #include <drm/ttm/ttm_resource.h>
30 : #include <drm/ttm/ttm_bo_driver.h>
31 :
32 : /**
33 : * ttm_lru_bulk_move_init - initialize a bulk move structure
34 : * @bulk: the structure to init
35 : *
36 : * For now just memset the structure to zero.
37 : */
38 0 : void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk)
39 : {
40 0 : memset(bulk, 0, sizeof(*bulk));
41 0 : }
42 : EXPORT_SYMBOL(ttm_lru_bulk_move_init);
43 :
44 : /**
45 : * ttm_lru_bulk_move_tail - bulk move range of resources to the LRU tail.
46 : *
47 : * @bulk: bulk move structure
48 : *
49 : * Bulk move BOs to the LRU tail, only valid to use when driver makes sure that
50 : * resource order never changes. Should be called with &ttm_device.lru_lock held.
51 : */
52 0 : void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk)
53 : {
54 : unsigned i, j;
55 :
56 0 : for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) {
57 0 : for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) {
58 0 : struct ttm_lru_bulk_move_pos *pos = &bulk->pos[i][j];
59 : struct ttm_resource_manager *man;
60 :
61 0 : if (!pos->first)
62 0 : continue;
63 :
64 0 : lockdep_assert_held(&pos->first->bo->bdev->lru_lock);
65 : dma_resv_assert_held(pos->first->bo->base.resv);
66 0 : dma_resv_assert_held(pos->last->bo->base.resv);
67 :
68 0 : man = ttm_manager_type(pos->first->bo->bdev, i);
69 0 : list_bulk_move_tail(&man->lru[j], &pos->first->lru,
70 : &pos->last->lru);
71 : }
72 : }
73 0 : }
74 : EXPORT_SYMBOL(ttm_lru_bulk_move_tail);
75 :
76 : /* Return the bulk move pos object for this resource */
77 : static struct ttm_lru_bulk_move_pos *
78 : ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move *bulk, struct ttm_resource *res)
79 : {
80 0 : return &bulk->pos[res->mem_type][res->bo->priority];
81 : }
82 :
83 : /* Move the resource to the tail of the bulk move range */
84 : static void ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos *pos,
85 : struct ttm_resource *res)
86 : {
87 0 : if (pos->last != res) {
88 0 : list_move(&res->lru, &pos->last->lru);
89 0 : pos->last = res;
90 : }
91 : }
92 :
93 : /* Add the resource to a bulk_move cursor */
94 : static void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk,
95 : struct ttm_resource *res)
96 : {
97 0 : struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
98 :
99 0 : if (!pos->first) {
100 0 : pos->first = res;
101 0 : pos->last = res;
102 : } else {
103 0 : ttm_lru_bulk_move_pos_tail(pos, res);
104 : }
105 : }
106 :
107 : /* Remove the resource from a bulk_move range */
108 0 : static void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk,
109 : struct ttm_resource *res)
110 : {
111 0 : struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
112 :
113 0 : if (unlikely(pos->first == res && pos->last == res)) {
114 0 : pos->first = NULL;
115 0 : pos->last = NULL;
116 0 : } else if (pos->first == res) {
117 0 : pos->first = list_next_entry(res, lru);
118 0 : } else if (pos->last == res) {
119 0 : pos->last = list_prev_entry(res, lru);
120 : } else {
121 0 : list_move(&res->lru, &pos->last->lru);
122 : }
123 0 : }
124 :
125 : /* Add the resource to a bulk move if the BO is configured for it */
126 0 : void ttm_resource_add_bulk_move(struct ttm_resource *res,
127 : struct ttm_buffer_object *bo)
128 : {
129 0 : if (bo->bulk_move && !bo->pin_count)
130 0 : ttm_lru_bulk_move_add(bo->bulk_move, res);
131 0 : }
132 :
133 : /* Remove the resource from a bulk move if the BO is configured for it */
134 0 : void ttm_resource_del_bulk_move(struct ttm_resource *res,
135 : struct ttm_buffer_object *bo)
136 : {
137 0 : if (bo->bulk_move && !bo->pin_count)
138 0 : ttm_lru_bulk_move_del(bo->bulk_move, res);
139 0 : }
140 :
141 : /* Move a resource to the LRU or bulk tail */
142 0 : void ttm_resource_move_to_lru_tail(struct ttm_resource *res)
143 : {
144 0 : struct ttm_buffer_object *bo = res->bo;
145 0 : struct ttm_device *bdev = bo->bdev;
146 :
147 : lockdep_assert_held(&bo->bdev->lru_lock);
148 :
149 0 : if (bo->pin_count) {
150 0 : list_move_tail(&res->lru, &bdev->pinned);
151 :
152 0 : } else if (bo->bulk_move) {
153 0 : struct ttm_lru_bulk_move_pos *pos =
154 : ttm_lru_bulk_move_pos(bo->bulk_move, res);
155 :
156 0 : ttm_lru_bulk_move_pos_tail(pos, res);
157 : } else {
158 : struct ttm_resource_manager *man;
159 :
160 0 : man = ttm_manager_type(bdev, res->mem_type);
161 0 : list_move_tail(&res->lru, &man->lru[bo->priority]);
162 : }
163 0 : }
164 :
165 : /**
166 : * ttm_resource_init - resource object constructure
167 : * @bo: buffer object this resources is allocated for
168 : * @place: placement of the resource
169 : * @res: the resource object to inistilize
170 : *
171 : * Initialize a new resource object. Counterpart of ttm_resource_fini().
172 : */
173 0 : void ttm_resource_init(struct ttm_buffer_object *bo,
174 : const struct ttm_place *place,
175 : struct ttm_resource *res)
176 : {
177 : struct ttm_resource_manager *man;
178 :
179 0 : res->start = 0;
180 0 : res->num_pages = PFN_UP(bo->base.size);
181 0 : res->mem_type = place->mem_type;
182 0 : res->placement = place->flags;
183 0 : res->bus.addr = NULL;
184 0 : res->bus.offset = 0;
185 0 : res->bus.is_iomem = false;
186 0 : res->bus.caching = ttm_cached;
187 0 : res->bo = bo;
188 :
189 0 : man = ttm_manager_type(bo->bdev, place->mem_type);
190 0 : spin_lock(&bo->bdev->lru_lock);
191 0 : if (bo->pin_count)
192 0 : list_add_tail(&res->lru, &bo->bdev->pinned);
193 : else
194 0 : list_add_tail(&res->lru, &man->lru[bo->priority]);
195 0 : man->usage += res->num_pages << PAGE_SHIFT;
196 0 : spin_unlock(&bo->bdev->lru_lock);
197 0 : }
198 : EXPORT_SYMBOL(ttm_resource_init);
199 :
200 : /**
201 : * ttm_resource_fini - resource destructor
202 : * @man: the resource manager this resource belongs to
203 : * @res: the resource to clean up
204 : *
205 : * Should be used by resource manager backends to clean up the TTM resource
206 : * objects before freeing the underlying structure. Makes sure the resource is
207 : * removed from the LRU before destruction.
208 : * Counterpart of ttm_resource_init().
209 : */
210 0 : void ttm_resource_fini(struct ttm_resource_manager *man,
211 : struct ttm_resource *res)
212 : {
213 0 : struct ttm_device *bdev = man->bdev;
214 :
215 0 : spin_lock(&bdev->lru_lock);
216 0 : list_del_init(&res->lru);
217 0 : man->usage -= res->num_pages << PAGE_SHIFT;
218 0 : spin_unlock(&bdev->lru_lock);
219 0 : }
220 : EXPORT_SYMBOL(ttm_resource_fini);
221 :
222 0 : int ttm_resource_alloc(struct ttm_buffer_object *bo,
223 : const struct ttm_place *place,
224 : struct ttm_resource **res_ptr)
225 : {
226 0 : struct ttm_resource_manager *man =
227 0 : ttm_manager_type(bo->bdev, place->mem_type);
228 : int ret;
229 :
230 0 : ret = man->func->alloc(man, bo, place, res_ptr);
231 0 : if (ret)
232 : return ret;
233 :
234 0 : spin_lock(&bo->bdev->lru_lock);
235 0 : ttm_resource_add_bulk_move(*res_ptr, bo);
236 0 : spin_unlock(&bo->bdev->lru_lock);
237 0 : return 0;
238 : }
239 :
240 0 : void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)
241 : {
242 : struct ttm_resource_manager *man;
243 :
244 0 : if (!*res)
245 : return;
246 :
247 0 : spin_lock(&bo->bdev->lru_lock);
248 0 : ttm_resource_del_bulk_move(*res, bo);
249 0 : spin_unlock(&bo->bdev->lru_lock);
250 0 : man = ttm_manager_type(bo->bdev, (*res)->mem_type);
251 0 : man->func->free(man, *res);
252 0 : *res = NULL;
253 : }
254 : EXPORT_SYMBOL(ttm_resource_free);
255 :
256 0 : static bool ttm_resource_places_compat(struct ttm_resource *res,
257 : const struct ttm_place *places,
258 : unsigned num_placement)
259 : {
260 : unsigned i;
261 :
262 0 : if (res->placement & TTM_PL_FLAG_TEMPORARY)
263 : return false;
264 :
265 0 : for (i = 0; i < num_placement; i++) {
266 0 : const struct ttm_place *heap = &places[i];
267 :
268 0 : if (res->start < heap->fpfn || (heap->lpfn &&
269 0 : (res->start + res->num_pages) > heap->lpfn))
270 0 : continue;
271 :
272 0 : if ((res->mem_type == heap->mem_type) &&
273 0 : (!(heap->flags & TTM_PL_FLAG_CONTIGUOUS) ||
274 0 : (res->placement & TTM_PL_FLAG_CONTIGUOUS)))
275 : return true;
276 : }
277 : return false;
278 : }
279 :
280 : /**
281 : * ttm_resource_compat - check if resource is compatible with placement
282 : *
283 : * @res: the resource to check
284 : * @placement: the placement to check against
285 : *
286 : * Returns true if the placement is compatible.
287 : */
288 0 : bool ttm_resource_compat(struct ttm_resource *res,
289 : struct ttm_placement *placement)
290 : {
291 0 : if (ttm_resource_places_compat(res, placement->placement,
292 : placement->num_placement))
293 : return true;
294 :
295 0 : if ((placement->busy_placement != placement->placement ||
296 0 : placement->num_busy_placement > placement->num_placement) &&
297 0 : ttm_resource_places_compat(res, placement->busy_placement,
298 : placement->num_busy_placement))
299 : return true;
300 :
301 : return false;
302 : }
303 : EXPORT_SYMBOL(ttm_resource_compat);
304 :
305 0 : void ttm_resource_set_bo(struct ttm_resource *res,
306 : struct ttm_buffer_object *bo)
307 : {
308 0 : spin_lock(&bo->bdev->lru_lock);
309 0 : res->bo = bo;
310 0 : spin_unlock(&bo->bdev->lru_lock);
311 0 : }
312 :
313 : /**
314 : * ttm_resource_manager_init
315 : *
316 : * @man: memory manager object to init
317 : * @bdev: ttm device this manager belongs to
318 : * @size: size of managed resources in arbitrary units
319 : *
320 : * Initialise core parts of a manager object.
321 : */
322 0 : void ttm_resource_manager_init(struct ttm_resource_manager *man,
323 : struct ttm_device *bdev,
324 : uint64_t size)
325 : {
326 : unsigned i;
327 :
328 0 : spin_lock_init(&man->move_lock);
329 0 : man->bdev = bdev;
330 0 : man->size = size;
331 0 : man->usage = 0;
332 :
333 0 : for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
334 0 : INIT_LIST_HEAD(&man->lru[i]);
335 0 : man->move = NULL;
336 0 : }
337 : EXPORT_SYMBOL(ttm_resource_manager_init);
338 :
339 : /*
340 : * ttm_resource_manager_evict_all
341 : *
342 : * @bdev - device to use
343 : * @man - manager to use
344 : *
345 : * Evict all the objects out of a memory manager until it is empty.
346 : * Part of memory manager cleanup sequence.
347 : */
348 0 : int ttm_resource_manager_evict_all(struct ttm_device *bdev,
349 : struct ttm_resource_manager *man)
350 : {
351 0 : struct ttm_operation_ctx ctx = {
352 : .interruptible = false,
353 : .no_wait_gpu = false,
354 : .force_alloc = true
355 : };
356 : struct dma_fence *fence;
357 : int ret;
358 : unsigned i;
359 :
360 : /*
361 : * Can't use standard list traversal since we're unlocking.
362 : */
363 :
364 0 : spin_lock(&bdev->lru_lock);
365 0 : for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
366 0 : while (!list_empty(&man->lru[i])) {
367 0 : spin_unlock(&bdev->lru_lock);
368 0 : ret = ttm_mem_evict_first(bdev, man, NULL, &ctx,
369 : NULL);
370 0 : if (ret)
371 : return ret;
372 0 : spin_lock(&bdev->lru_lock);
373 : }
374 : }
375 0 : spin_unlock(&bdev->lru_lock);
376 :
377 0 : spin_lock(&man->move_lock);
378 0 : fence = dma_fence_get(man->move);
379 0 : spin_unlock(&man->move_lock);
380 :
381 0 : if (fence) {
382 0 : ret = dma_fence_wait(fence, false);
383 0 : dma_fence_put(fence);
384 0 : if (ret)
385 : return ret;
386 : }
387 :
388 : return 0;
389 : }
390 : EXPORT_SYMBOL(ttm_resource_manager_evict_all);
391 :
392 : /**
393 : * ttm_resource_manager_usage
394 : *
395 : * @man: A memory manager object.
396 : *
397 : * Return how many resources are currently used.
398 : */
399 0 : uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man)
400 : {
401 : uint64_t usage;
402 :
403 0 : spin_lock(&man->bdev->lru_lock);
404 0 : usage = man->usage;
405 0 : spin_unlock(&man->bdev->lru_lock);
406 0 : return usage;
407 : }
408 : EXPORT_SYMBOL(ttm_resource_manager_usage);
409 :
410 : /**
411 : * ttm_resource_manager_debug
412 : *
413 : * @man: manager type to dump.
414 : * @p: printer to use for debug.
415 : */
416 0 : void ttm_resource_manager_debug(struct ttm_resource_manager *man,
417 : struct drm_printer *p)
418 : {
419 0 : drm_printf(p, " use_type: %d\n", man->use_type);
420 0 : drm_printf(p, " use_tt: %d\n", man->use_tt);
421 0 : drm_printf(p, " size: %llu\n", man->size);
422 0 : drm_printf(p, " usage: %llu\n", ttm_resource_manager_usage(man));
423 0 : if (man->func->debug)
424 0 : man->func->debug(man, p);
425 0 : }
426 : EXPORT_SYMBOL(ttm_resource_manager_debug);
427 :
428 : /**
429 : * ttm_resource_manager_first
430 : *
431 : * @man: resource manager to iterate over
432 : * @cursor: cursor to record the position
433 : *
434 : * Returns the first resource from the resource manager.
435 : */
436 : struct ttm_resource *
437 0 : ttm_resource_manager_first(struct ttm_resource_manager *man,
438 : struct ttm_resource_cursor *cursor)
439 : {
440 : struct ttm_resource *res;
441 :
442 : lockdep_assert_held(&man->bdev->lru_lock);
443 :
444 0 : for (cursor->priority = 0; cursor->priority < TTM_MAX_BO_PRIORITY;
445 0 : ++cursor->priority)
446 0 : list_for_each_entry(res, &man->lru[cursor->priority], lru)
447 : return res;
448 :
449 : return NULL;
450 : }
451 :
452 : /**
453 : * ttm_resource_manager_next
454 : *
455 : * @man: resource manager to iterate over
456 : * @cursor: cursor to record the position
457 : * @res: the current resource pointer
458 : *
459 : * Returns the next resource from the resource manager.
460 : */
461 : struct ttm_resource *
462 0 : ttm_resource_manager_next(struct ttm_resource_manager *man,
463 : struct ttm_resource_cursor *cursor,
464 : struct ttm_resource *res)
465 : {
466 : lockdep_assert_held(&man->bdev->lru_lock);
467 :
468 0 : list_for_each_entry_continue(res, &man->lru[cursor->priority], lru)
469 : return res;
470 :
471 0 : for (++cursor->priority; cursor->priority < TTM_MAX_BO_PRIORITY;
472 0 : ++cursor->priority)
473 0 : list_for_each_entry(res, &man->lru[cursor->priority], lru)
474 : return res;
475 :
476 : return NULL;
477 : }
478 :
479 0 : static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
480 : struct iosys_map *dmap,
481 : pgoff_t i)
482 : {
483 0 : struct ttm_kmap_iter_iomap *iter_io =
484 0 : container_of(iter, typeof(*iter_io), base);
485 : void __iomem *addr;
486 :
487 : retry:
488 0 : while (i >= iter_io->cache.end) {
489 0 : iter_io->cache.sg = iter_io->cache.sg ?
490 0 : sg_next(iter_io->cache.sg) : iter_io->st->sgl;
491 0 : iter_io->cache.i = iter_io->cache.end;
492 0 : iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >>
493 : PAGE_SHIFT;
494 0 : iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) -
495 0 : iter_io->start;
496 : }
497 :
498 0 : if (i < iter_io->cache.i) {
499 0 : iter_io->cache.end = 0;
500 0 : iter_io->cache.sg = NULL;
501 0 : goto retry;
502 : }
503 :
504 0 : addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs +
505 0 : (((resource_size_t)i - iter_io->cache.i)
506 0 : << PAGE_SHIFT));
507 0 : iosys_map_set_vaddr_iomem(dmap, addr);
508 0 : }
509 :
510 0 : static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter,
511 : struct iosys_map *map)
512 : {
513 0 : io_mapping_unmap_local(map->vaddr_iomem);
514 0 : }
515 :
516 : static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = {
517 : .map_local = ttm_kmap_iter_iomap_map_local,
518 : .unmap_local = ttm_kmap_iter_iomap_unmap_local,
519 : .maps_tt = false,
520 : };
521 :
522 : /**
523 : * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap
524 : * @iter_io: The struct ttm_kmap_iter_iomap to initialize.
525 : * @iomap: The struct io_mapping representing the underlying linear io_memory.
526 : * @st: sg_table into @iomap, representing the memory of the struct
527 : * ttm_resource.
528 : * @start: Offset that needs to be subtracted from @st to make
529 : * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
530 : *
531 : * Return: Pointer to the embedded struct ttm_kmap_iter.
532 : */
533 : struct ttm_kmap_iter *
534 0 : ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
535 : struct io_mapping *iomap,
536 : struct sg_table *st,
537 : resource_size_t start)
538 : {
539 0 : iter_io->base.ops = &ttm_kmap_iter_io_ops;
540 0 : iter_io->iomap = iomap;
541 0 : iter_io->st = st;
542 0 : iter_io->start = start;
543 0 : memset(&iter_io->cache, 0, sizeof(iter_io->cache));
544 :
545 0 : return &iter_io->base;
546 : }
547 : EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);
548 :
549 : /**
550 : * DOC: Linear io iterator
551 : *
552 : * This code should die in the not too near future. Best would be if we could
553 : * make io-mapping use memremap for all io memory, and have memremap
554 : * implement a kmap_local functionality. We could then strip a huge amount of
555 : * code. These linear io iterators are implemented to mimic old functionality,
556 : * and they don't use kmap_local semantics at all internally. Rather ioremap or
557 : * friends, and at least on 32-bit they add global TLB flushes and points
558 : * of failure.
559 : */
560 :
561 0 : static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter,
562 : struct iosys_map *dmap,
563 : pgoff_t i)
564 : {
565 0 : struct ttm_kmap_iter_linear_io *iter_io =
566 0 : container_of(iter, typeof(*iter_io), base);
567 :
568 0 : *dmap = iter_io->dmap;
569 0 : iosys_map_incr(dmap, i * PAGE_SIZE);
570 0 : }
571 :
572 : static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {
573 : .map_local = ttm_kmap_iter_linear_io_map_local,
574 : .maps_tt = false,
575 : };
576 :
577 : /**
578 : * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory
579 : * @iter_io: The iterator to initialize
580 : * @bdev: The TTM device
581 : * @mem: The ttm resource representing the iomap.
582 : *
583 : * This function is for internal TTM use only. It sets up a memcpy kmap iterator
584 : * pointing at a linear chunk of io memory.
585 : *
586 : * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on
587 : * failure.
588 : */
589 : struct ttm_kmap_iter *
590 0 : ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
591 : struct ttm_device *bdev,
592 : struct ttm_resource *mem)
593 : {
594 : int ret;
595 :
596 0 : ret = ttm_mem_io_reserve(bdev, mem);
597 0 : if (ret)
598 : goto out_err;
599 0 : if (!mem->bus.is_iomem) {
600 : ret = -EINVAL;
601 : goto out_io_free;
602 : }
603 :
604 0 : if (mem->bus.addr) {
605 0 : iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr);
606 0 : iter_io->needs_unmap = false;
607 : } else {
608 0 : size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT;
609 :
610 0 : iter_io->needs_unmap = true;
611 0 : memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
612 0 : if (mem->bus.caching == ttm_write_combined)
613 0 : iosys_map_set_vaddr_iomem(&iter_io->dmap,
614 : ioremap_wc(mem->bus.offset,
615 : bus_size));
616 0 : else if (mem->bus.caching == ttm_cached)
617 0 : iosys_map_set_vaddr(&iter_io->dmap,
618 : memremap(mem->bus.offset, bus_size,
619 : MEMREMAP_WB |
620 : MEMREMAP_WT |
621 : MEMREMAP_WC));
622 :
623 : /* If uncached requested or if mapping cached or wc failed */
624 0 : if (iosys_map_is_null(&iter_io->dmap))
625 0 : iosys_map_set_vaddr_iomem(&iter_io->dmap,
626 : ioremap(mem->bus.offset,
627 : bus_size));
628 :
629 0 : if (iosys_map_is_null(&iter_io->dmap)) {
630 : ret = -ENOMEM;
631 : goto out_io_free;
632 : }
633 : }
634 :
635 0 : iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;
636 0 : return &iter_io->base;
637 :
638 : out_io_free:
639 0 : ttm_mem_io_free(bdev, mem);
640 : out_err:
641 0 : return ERR_PTR(ret);
642 : }
643 :
644 : /**
645 : * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory
646 : * @iter_io: The iterator to initialize
647 : * @bdev: The TTM device
648 : * @mem: The ttm resource representing the iomap.
649 : *
650 : * This function is for internal TTM use only. It cleans up a memcpy kmap
651 : * iterator initialized by ttm_kmap_iter_linear_io_init.
652 : */
653 : void
654 0 : ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
655 : struct ttm_device *bdev,
656 : struct ttm_resource *mem)
657 : {
658 0 : if (iter_io->needs_unmap && iosys_map_is_set(&iter_io->dmap)) {
659 0 : if (iter_io->dmap.is_iomem)
660 0 : iounmap(iter_io->dmap.vaddr_iomem);
661 : else
662 0 : memunmap(iter_io->dmap.vaddr);
663 : }
664 :
665 0 : ttm_mem_io_free(bdev, mem);
666 0 : }
667 :
668 : #if defined(CONFIG_DEBUG_FS)
669 :
670 : static int ttm_resource_manager_show(struct seq_file *m, void *unused)
671 : {
672 : struct ttm_resource_manager *man =
673 : (struct ttm_resource_manager *)m->private;
674 : struct drm_printer p = drm_seq_file_printer(m);
675 : ttm_resource_manager_debug(man, &p);
676 : return 0;
677 : }
678 : DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager);
679 :
680 : #endif
681 :
682 : /**
683 : * ttm_resource_manager_create_debugfs - Create debugfs entry for specified
684 : * resource manager.
685 : * @man: The TTM resource manager for which the debugfs stats file be creates
686 : * @parent: debugfs directory in which the file will reside
687 : * @name: The filename to create.
688 : *
689 : * This function setups up a debugfs file that can be used to look
690 : * at debug statistics of the specified ttm_resource_manager.
691 : */
692 0 : void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
693 : struct dentry * parent,
694 : const char *name)
695 : {
696 : #if defined(CONFIG_DEBUG_FS)
697 : debugfs_create_file(name, 0444, parent, man, &ttm_resource_manager_fops);
698 : #endif
699 0 : }
700 : EXPORT_SYMBOL(ttm_resource_manager_create_debugfs);
|
