Line data Source code
1 : /*
2 : * Copyright 2009 Jerome Glisse.
3 : * All Rights Reserved.
4 : *
5 : * Permission is hereby granted, free of charge, to any person obtaining a
6 : * copy of this software and associated documentation files (the
7 : * "Software"), to deal in the Software without restriction, including
8 : * without limitation the rights to use, copy, modify, merge, publish,
9 : * distribute, sub license, and/or sell copies of the Software, and to
10 : * permit persons to whom the Software is furnished to do so, subject to
11 : * the following conditions:
12 : *
13 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 : * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 : * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 : * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 : * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 : * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 : * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 : *
21 : * The above copyright notice and this permission notice (including the
22 : * next paragraph) shall be included in all copies or substantial portions
23 : * of the Software.
24 : *
25 : */
26 : /*
27 : * Authors:
28 : * Jerome Glisse <glisse@freedesktop.org>
29 : * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
30 : * Dave Airlie
31 : */
32 : #include <linux/list.h>
33 : #include <linux/slab.h>
34 : #include <linux/dma-buf.h>
35 :
36 : #include <drm/drm_drv.h>
37 : #include <drm/amdgpu_drm.h>
38 : #include <drm/drm_cache.h>
39 : #include "amdgpu.h"
40 : #include "amdgpu_trace.h"
41 : #include "amdgpu_amdkfd.h"
42 :
43 : /**
44 : * DOC: amdgpu_object
45 : *
46 : * This defines the interfaces to operate on an &amdgpu_bo buffer object which
47 : * represents memory used by driver (VRAM, system memory, etc.). The driver
48 : * provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces
49 : * to create/destroy/set buffer object which are then managed by the kernel TTM
50 : * memory manager.
51 : * The interfaces are also used internally by kernel clients, including gfx,
52 : * uvd, etc. for kernel managed allocations used by the GPU.
53 : *
54 : */
55 :
56 0 : static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo)
57 : {
58 0 : struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
59 :
60 0 : amdgpu_bo_kunmap(bo);
61 :
62 0 : if (bo->tbo.base.import_attach)
63 0 : drm_prime_gem_destroy(&bo->tbo.base, bo->tbo.sg);
64 0 : drm_gem_object_release(&bo->tbo.base);
65 0 : amdgpu_bo_unref(&bo->parent);
66 0 : kvfree(bo);
67 0 : }
68 :
69 0 : static void amdgpu_bo_user_destroy(struct ttm_buffer_object *tbo)
70 : {
71 0 : struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
72 : struct amdgpu_bo_user *ubo;
73 :
74 0 : ubo = to_amdgpu_bo_user(bo);
75 0 : kfree(ubo->metadata);
76 0 : amdgpu_bo_destroy(tbo);
77 0 : }
78 :
79 0 : static void amdgpu_bo_vm_destroy(struct ttm_buffer_object *tbo)
80 : {
81 0 : struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
82 0 : struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
83 : struct amdgpu_bo_vm *vmbo;
84 :
85 0 : vmbo = to_amdgpu_bo_vm(bo);
86 : /* in case amdgpu_device_recover_vram got NULL of bo->parent */
87 0 : if (!list_empty(&vmbo->shadow_list)) {
88 0 : mutex_lock(&adev->shadow_list_lock);
89 0 : list_del_init(&vmbo->shadow_list);
90 0 : mutex_unlock(&adev->shadow_list_lock);
91 : }
92 :
93 0 : amdgpu_bo_destroy(tbo);
94 0 : }
95 :
96 : /**
97 : * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo
98 : * @bo: buffer object to be checked
99 : *
100 : * Uses destroy function associated with the object to determine if this is
101 : * an &amdgpu_bo.
102 : *
103 : * Returns:
104 : * true if the object belongs to &amdgpu_bo, false if not.
105 : */
106 0 : bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
107 : {
108 0 : if (bo->destroy == &amdgpu_bo_destroy ||
109 0 : bo->destroy == &amdgpu_bo_user_destroy ||
110 : bo->destroy == &amdgpu_bo_vm_destroy)
111 : return true;
112 :
113 0 : return false;
114 : }
115 :
116 : /**
117 : * amdgpu_bo_placement_from_domain - set buffer's placement
118 : * @abo: &amdgpu_bo buffer object whose placement is to be set
119 : * @domain: requested domain
120 : *
121 : * Sets buffer's placement according to requested domain and the buffer's
122 : * flags.
123 : */
124 0 : void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
125 : {
126 0 : struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
127 0 : struct ttm_placement *placement = &abo->placement;
128 0 : struct ttm_place *places = abo->placements;
129 0 : u64 flags = abo->flags;
130 0 : u32 c = 0;
131 :
132 0 : if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
133 0 : unsigned visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
134 :
135 0 : places[c].fpfn = 0;
136 0 : places[c].lpfn = 0;
137 0 : places[c].mem_type = TTM_PL_VRAM;
138 0 : places[c].flags = 0;
139 :
140 0 : if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
141 0 : places[c].lpfn = visible_pfn;
142 : else
143 0 : places[c].flags |= TTM_PL_FLAG_TOPDOWN;
144 :
145 0 : if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
146 0 : places[c].flags |= TTM_PL_FLAG_CONTIGUOUS;
147 : c++;
148 : }
149 :
150 0 : if (domain & AMDGPU_GEM_DOMAIN_GTT) {
151 0 : places[c].fpfn = 0;
152 0 : places[c].lpfn = 0;
153 0 : places[c].mem_type =
154 0 : abo->flags & AMDGPU_GEM_CREATE_PREEMPTIBLE ?
155 0 : AMDGPU_PL_PREEMPT : TTM_PL_TT;
156 0 : places[c].flags = 0;
157 0 : c++;
158 : }
159 :
160 0 : if (domain & AMDGPU_GEM_DOMAIN_CPU) {
161 0 : places[c].fpfn = 0;
162 0 : places[c].lpfn = 0;
163 0 : places[c].mem_type = TTM_PL_SYSTEM;
164 0 : places[c].flags = 0;
165 0 : c++;
166 : }
167 :
168 0 : if (domain & AMDGPU_GEM_DOMAIN_GDS) {
169 0 : places[c].fpfn = 0;
170 0 : places[c].lpfn = 0;
171 0 : places[c].mem_type = AMDGPU_PL_GDS;
172 0 : places[c].flags = 0;
173 0 : c++;
174 : }
175 :
176 0 : if (domain & AMDGPU_GEM_DOMAIN_GWS) {
177 0 : places[c].fpfn = 0;
178 0 : places[c].lpfn = 0;
179 0 : places[c].mem_type = AMDGPU_PL_GWS;
180 0 : places[c].flags = 0;
181 0 : c++;
182 : }
183 :
184 0 : if (domain & AMDGPU_GEM_DOMAIN_OA) {
185 0 : places[c].fpfn = 0;
186 0 : places[c].lpfn = 0;
187 0 : places[c].mem_type = AMDGPU_PL_OA;
188 0 : places[c].flags = 0;
189 0 : c++;
190 : }
191 :
192 0 : if (!c) {
193 0 : places[c].fpfn = 0;
194 0 : places[c].lpfn = 0;
195 0 : places[c].mem_type = TTM_PL_SYSTEM;
196 0 : places[c].flags = 0;
197 0 : c++;
198 : }
199 :
200 0 : BUG_ON(c > AMDGPU_BO_MAX_PLACEMENTS);
201 :
202 0 : placement->num_placement = c;
203 0 : placement->placement = places;
204 :
205 0 : placement->num_busy_placement = c;
206 0 : placement->busy_placement = places;
207 0 : }
208 :
209 : /**
210 : * amdgpu_bo_create_reserved - create reserved BO for kernel use
211 : *
212 : * @adev: amdgpu device object
213 : * @size: size for the new BO
214 : * @align: alignment for the new BO
215 : * @domain: where to place it
216 : * @bo_ptr: used to initialize BOs in structures
217 : * @gpu_addr: GPU addr of the pinned BO
218 : * @cpu_addr: optional CPU address mapping
219 : *
220 : * Allocates and pins a BO for kernel internal use, and returns it still
221 : * reserved.
222 : *
223 : * Note: For bo_ptr new BO is only created if bo_ptr points to NULL.
224 : *
225 : * Returns:
226 : * 0 on success, negative error code otherwise.
227 : */
228 0 : int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
229 : unsigned long size, int align,
230 : u32 domain, struct amdgpu_bo **bo_ptr,
231 : u64 *gpu_addr, void **cpu_addr)
232 : {
233 : struct amdgpu_bo_param bp;
234 0 : bool free = false;
235 : int r;
236 :
237 0 : if (!size) {
238 : amdgpu_bo_unref(bo_ptr);
239 : return 0;
240 : }
241 :
242 0 : memset(&bp, 0, sizeof(bp));
243 0 : bp.size = size;
244 0 : bp.byte_align = align;
245 0 : bp.domain = domain;
246 : bp.flags = cpu_addr ? AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED
247 0 : : AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
248 0 : bp.flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
249 0 : bp.type = ttm_bo_type_kernel;
250 0 : bp.resv = NULL;
251 0 : bp.bo_ptr_size = sizeof(struct amdgpu_bo);
252 :
253 0 : if (!*bo_ptr) {
254 0 : r = amdgpu_bo_create(adev, &bp, bo_ptr);
255 0 : if (r) {
256 0 : dev_err(adev->dev, "(%d) failed to allocate kernel bo\n",
257 : r);
258 0 : return r;
259 : }
260 : free = true;
261 : }
262 :
263 0 : r = amdgpu_bo_reserve(*bo_ptr, false);
264 0 : if (r) {
265 0 : dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r);
266 0 : goto error_free;
267 : }
268 :
269 0 : r = amdgpu_bo_pin(*bo_ptr, domain);
270 0 : if (r) {
271 0 : dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
272 0 : goto error_unreserve;
273 : }
274 :
275 0 : r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo);
276 0 : if (r) {
277 0 : dev_err(adev->dev, "%p bind failed\n", *bo_ptr);
278 0 : goto error_unpin;
279 : }
280 :
281 0 : if (gpu_addr)
282 0 : *gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr);
283 :
284 0 : if (cpu_addr) {
285 0 : r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
286 0 : if (r) {
287 0 : dev_err(adev->dev, "(%d) kernel bo map failed\n", r);
288 0 : goto error_unpin;
289 : }
290 : }
291 :
292 : return 0;
293 :
294 : error_unpin:
295 0 : amdgpu_bo_unpin(*bo_ptr);
296 : error_unreserve:
297 0 : amdgpu_bo_unreserve(*bo_ptr);
298 :
299 : error_free:
300 0 : if (free)
301 : amdgpu_bo_unref(bo_ptr);
302 :
303 : return r;
304 : }
305 :
306 : /**
307 : * amdgpu_bo_create_kernel - create BO for kernel use
308 : *
309 : * @adev: amdgpu device object
310 : * @size: size for the new BO
311 : * @align: alignment for the new BO
312 : * @domain: where to place it
313 : * @bo_ptr: used to initialize BOs in structures
314 : * @gpu_addr: GPU addr of the pinned BO
315 : * @cpu_addr: optional CPU address mapping
316 : *
317 : * Allocates and pins a BO for kernel internal use.
318 : *
319 : * Note: For bo_ptr new BO is only created if bo_ptr points to NULL.
320 : *
321 : * Returns:
322 : * 0 on success, negative error code otherwise.
323 : */
324 0 : int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
325 : unsigned long size, int align,
326 : u32 domain, struct amdgpu_bo **bo_ptr,
327 : u64 *gpu_addr, void **cpu_addr)
328 : {
329 : int r;
330 :
331 0 : r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr,
332 : gpu_addr, cpu_addr);
333 :
334 0 : if (r)
335 : return r;
336 :
337 0 : if (*bo_ptr)
338 0 : amdgpu_bo_unreserve(*bo_ptr);
339 :
340 : return 0;
341 : }
342 :
343 : /**
344 : * amdgpu_bo_create_kernel_at - create BO for kernel use at specific location
345 : *
346 : * @adev: amdgpu device object
347 : * @offset: offset of the BO
348 : * @size: size of the BO
349 : * @domain: where to place it
350 : * @bo_ptr: used to initialize BOs in structures
351 : * @cpu_addr: optional CPU address mapping
352 : *
353 : * Creates a kernel BO at a specific offset in the address space of the domain.
354 : *
355 : * Returns:
356 : * 0 on success, negative error code otherwise.
357 : */
358 0 : int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev,
359 : uint64_t offset, uint64_t size, uint32_t domain,
360 : struct amdgpu_bo **bo_ptr, void **cpu_addr)
361 : {
362 0 : struct ttm_operation_ctx ctx = { false, false };
363 : unsigned int i;
364 : int r;
365 :
366 0 : offset &= PAGE_MASK;
367 0 : size = ALIGN(size, PAGE_SIZE);
368 :
369 0 : r = amdgpu_bo_create_reserved(adev, size, PAGE_SIZE, domain, bo_ptr,
370 : NULL, cpu_addr);
371 0 : if (r)
372 : return r;
373 :
374 0 : if ((*bo_ptr) == NULL)
375 : return 0;
376 :
377 : /*
378 : * Remove the original mem node and create a new one at the request
379 : * position.
380 : */
381 0 : if (cpu_addr)
382 0 : amdgpu_bo_kunmap(*bo_ptr);
383 :
384 0 : ttm_resource_free(&(*bo_ptr)->tbo, &(*bo_ptr)->tbo.resource);
385 :
386 0 : for (i = 0; i < (*bo_ptr)->placement.num_placement; ++i) {
387 0 : (*bo_ptr)->placements[i].fpfn = offset >> PAGE_SHIFT;
388 0 : (*bo_ptr)->placements[i].lpfn = (offset + size) >> PAGE_SHIFT;
389 : }
390 0 : r = ttm_bo_mem_space(&(*bo_ptr)->tbo, &(*bo_ptr)->placement,
391 : &(*bo_ptr)->tbo.resource, &ctx);
392 0 : if (r)
393 : goto error;
394 :
395 0 : if (cpu_addr) {
396 0 : r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
397 0 : if (r)
398 : goto error;
399 : }
400 :
401 0 : amdgpu_bo_unreserve(*bo_ptr);
402 0 : return 0;
403 :
404 : error:
405 0 : amdgpu_bo_unreserve(*bo_ptr);
406 : amdgpu_bo_unref(bo_ptr);
407 : return r;
408 : }
409 :
410 : /**
411 : * amdgpu_bo_free_kernel - free BO for kernel use
412 : *
413 : * @bo: amdgpu BO to free
414 : * @gpu_addr: pointer to where the BO's GPU memory space address was stored
415 : * @cpu_addr: pointer to where the BO's CPU memory space address was stored
416 : *
417 : * unmaps and unpin a BO for kernel internal use.
418 : */
419 0 : void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
420 : void **cpu_addr)
421 : {
422 0 : if (*bo == NULL)
423 : return;
424 :
425 0 : if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {
426 0 : if (cpu_addr)
427 0 : amdgpu_bo_kunmap(*bo);
428 :
429 0 : amdgpu_bo_unpin(*bo);
430 0 : amdgpu_bo_unreserve(*bo);
431 : }
432 0 : amdgpu_bo_unref(bo);
433 :
434 0 : if (gpu_addr)
435 0 : *gpu_addr = 0;
436 :
437 0 : if (cpu_addr)
438 0 : *cpu_addr = NULL;
439 : }
440 :
441 : /* Validate bo size is bit bigger then the request domain */
442 0 : static bool amdgpu_bo_validate_size(struct amdgpu_device *adev,
443 : unsigned long size, u32 domain)
444 : {
445 0 : struct ttm_resource_manager *man = NULL;
446 :
447 : /*
448 : * If GTT is part of requested domains the check must succeed to
449 : * allow fall back to GTT
450 : */
451 0 : if (domain & AMDGPU_GEM_DOMAIN_GTT) {
452 0 : man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
453 :
454 0 : if (size < man->size)
455 : return true;
456 : else
457 : goto fail;
458 : }
459 :
460 0 : if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
461 0 : man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
462 :
463 0 : if (size < man->size)
464 : return true;
465 : else
466 : goto fail;
467 : }
468 :
469 :
470 : /* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU */
471 : return true;
472 :
473 : fail:
474 0 : DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size,
475 : man->size);
476 0 : return false;
477 : }
478 :
479 0 : bool amdgpu_bo_support_uswc(u64 bo_flags)
480 : {
481 :
482 : #ifdef CONFIG_X86_32
483 : /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit
484 : * See https://bugs.freedesktop.org/show_bug.cgi?id=84627
485 : */
486 : return false;
487 : #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT)
488 : /* Don't try to enable write-combining when it can't work, or things
489 : * may be slow
490 : * See https://bugs.freedesktop.org/show_bug.cgi?id=88758
491 : */
492 :
493 : #ifndef CONFIG_COMPILE_TEST
494 : #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \
495 : thanks to write-combining
496 : #endif
497 :
498 : if (bo_flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
499 : DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for "
500 : "better performance thanks to write-combining\n");
501 : return false;
502 : #else
503 : /* For architectures that don't support WC memory,
504 : * mask out the WC flag from the BO
505 : */
506 : if (!drm_arch_can_wc_memory())
507 : return false;
508 :
509 : return true;
510 : #endif
511 : }
512 :
513 : /**
514 : * amdgpu_bo_create - create an &amdgpu_bo buffer object
515 : * @adev: amdgpu device object
516 : * @bp: parameters to be used for the buffer object
517 : * @bo_ptr: pointer to the buffer object pointer
518 : *
519 : * Creates an &amdgpu_bo buffer object.
520 : *
521 : * Returns:
522 : * 0 for success or a negative error code on failure.
523 : */
524 0 : int amdgpu_bo_create(struct amdgpu_device *adev,
525 : struct amdgpu_bo_param *bp,
526 : struct amdgpu_bo **bo_ptr)
527 : {
528 0 : struct ttm_operation_ctx ctx = {
529 0 : .interruptible = (bp->type != ttm_bo_type_kernel),
530 0 : .no_wait_gpu = bp->no_wait_gpu,
531 : /* We opt to avoid OOM on system pages allocations */
532 : .gfp_retry_mayfail = true,
533 : .allow_res_evict = bp->type != ttm_bo_type_kernel,
534 0 : .resv = bp->resv
535 : };
536 : struct amdgpu_bo *bo;
537 0 : unsigned long page_align, size = bp->size;
538 : int r;
539 :
540 : /* Note that GDS/GWS/OA allocates 1 page per byte/resource. */
541 0 : if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
542 : /* GWS and OA don't need any alignment. */
543 0 : page_align = bp->byte_align;
544 0 : size <<= PAGE_SHIFT;
545 0 : } else if (bp->domain & AMDGPU_GEM_DOMAIN_GDS) {
546 : /* Both size and alignment must be a multiple of 4. */
547 0 : page_align = ALIGN(bp->byte_align, 4);
548 0 : size = ALIGN(size, 4) << PAGE_SHIFT;
549 : } else {
550 : /* Memory should be aligned at least to a page size. */
551 0 : page_align = ALIGN(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT;
552 0 : size = ALIGN(size, PAGE_SIZE);
553 : }
554 :
555 0 : if (!amdgpu_bo_validate_size(adev, size, bp->domain))
556 : return -ENOMEM;
557 :
558 0 : BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo));
559 :
560 0 : *bo_ptr = NULL;
561 0 : bo = kvzalloc(bp->bo_ptr_size, GFP_KERNEL);
562 0 : if (bo == NULL)
563 : return -ENOMEM;
564 0 : drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size);
565 0 : bo->vm_bo = NULL;
566 0 : bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain :
567 : bp->domain;
568 0 : bo->allowed_domains = bo->preferred_domains;
569 0 : if (bp->type != ttm_bo_type_kernel &&
570 0 : !(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE) &&
571 : bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
572 0 : bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
573 :
574 0 : bo->flags = bp->flags;
575 :
576 0 : if (!amdgpu_bo_support_uswc(bo->flags))
577 : bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
578 :
579 0 : if (adev->ras_enabled)
580 0 : bo->flags |= AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE;
581 :
582 0 : bo->tbo.bdev = &adev->mman.bdev;
583 0 : if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA |
584 : AMDGPU_GEM_DOMAIN_GDS))
585 0 : amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
586 : else
587 0 : amdgpu_bo_placement_from_domain(bo, bp->domain);
588 0 : if (bp->type == ttm_bo_type_kernel)
589 0 : bo->tbo.priority = 1;
590 :
591 0 : if (!bp->destroy)
592 0 : bp->destroy = &amdgpu_bo_destroy;
593 :
594 0 : r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, bp->type,
595 : &bo->placement, page_align, &ctx, NULL,
596 : bp->resv, bp->destroy);
597 0 : if (unlikely(r != 0))
598 : return r;
599 :
600 0 : if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
601 0 : bo->tbo.resource->mem_type == TTM_PL_VRAM &&
602 0 : bo->tbo.resource->start < adev->gmc.visible_vram_size >> PAGE_SHIFT)
603 0 : amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved,
604 : ctx.bytes_moved);
605 : else
606 0 : amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0);
607 :
608 0 : if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&
609 0 : bo->tbo.resource->mem_type == TTM_PL_VRAM) {
610 : struct dma_fence *fence;
611 :
612 0 : r = amdgpu_fill_buffer(bo, 0, bo->tbo.base.resv, &fence);
613 0 : if (unlikely(r))
614 : goto fail_unreserve;
615 :
616 0 : dma_resv_add_fence(bo->tbo.base.resv, fence,
617 : DMA_RESV_USAGE_KERNEL);
618 0 : dma_fence_put(fence);
619 : }
620 0 : if (!bp->resv)
621 0 : amdgpu_bo_unreserve(bo);
622 0 : *bo_ptr = bo;
623 :
624 0 : trace_amdgpu_bo_create(bo);
625 :
626 : /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */
627 0 : if (bp->type == ttm_bo_type_device)
628 0 : bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
629 :
630 : return 0;
631 :
632 : fail_unreserve:
633 0 : if (!bp->resv)
634 0 : dma_resv_unlock(bo->tbo.base.resv);
635 0 : amdgpu_bo_unref(&bo);
636 0 : return r;
637 : }
638 :
639 : /**
640 : * amdgpu_bo_create_user - create an &amdgpu_bo_user buffer object
641 : * @adev: amdgpu device object
642 : * @bp: parameters to be used for the buffer object
643 : * @ubo_ptr: pointer to the buffer object pointer
644 : *
645 : * Create a BO to be used by user application;
646 : *
647 : * Returns:
648 : * 0 for success or a negative error code on failure.
649 : */
650 :
651 0 : int amdgpu_bo_create_user(struct amdgpu_device *adev,
652 : struct amdgpu_bo_param *bp,
653 : struct amdgpu_bo_user **ubo_ptr)
654 : {
655 : struct amdgpu_bo *bo_ptr;
656 : int r;
657 :
658 0 : bp->bo_ptr_size = sizeof(struct amdgpu_bo_user);
659 0 : bp->destroy = &amdgpu_bo_user_destroy;
660 0 : r = amdgpu_bo_create(adev, bp, &bo_ptr);
661 0 : if (r)
662 : return r;
663 :
664 0 : *ubo_ptr = to_amdgpu_bo_user(bo_ptr);
665 0 : return r;
666 : }
667 :
668 : /**
669 : * amdgpu_bo_create_vm - create an &amdgpu_bo_vm buffer object
670 : * @adev: amdgpu device object
671 : * @bp: parameters to be used for the buffer object
672 : * @vmbo_ptr: pointer to the buffer object pointer
673 : *
674 : * Create a BO to be for GPUVM.
675 : *
676 : * Returns:
677 : * 0 for success or a negative error code on failure.
678 : */
679 :
680 0 : int amdgpu_bo_create_vm(struct amdgpu_device *adev,
681 : struct amdgpu_bo_param *bp,
682 : struct amdgpu_bo_vm **vmbo_ptr)
683 : {
684 : struct amdgpu_bo *bo_ptr;
685 : int r;
686 :
687 : /* bo_ptr_size will be determined by the caller and it depends on
688 : * num of amdgpu_vm_pt entries.
689 : */
690 0 : BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo_vm));
691 0 : bp->destroy = &amdgpu_bo_vm_destroy;
692 0 : r = amdgpu_bo_create(adev, bp, &bo_ptr);
693 0 : if (r)
694 : return r;
695 :
696 0 : *vmbo_ptr = to_amdgpu_bo_vm(bo_ptr);
697 0 : INIT_LIST_HEAD(&(*vmbo_ptr)->shadow_list);
698 0 : return r;
699 : }
700 :
701 : /**
702 : * amdgpu_bo_add_to_shadow_list - add a BO to the shadow list
703 : *
704 : * @vmbo: BO that will be inserted into the shadow list
705 : *
706 : * Insert a BO to the shadow list.
707 : */
708 0 : void amdgpu_bo_add_to_shadow_list(struct amdgpu_bo_vm *vmbo)
709 : {
710 0 : struct amdgpu_device *adev = amdgpu_ttm_adev(vmbo->bo.tbo.bdev);
711 :
712 0 : mutex_lock(&adev->shadow_list_lock);
713 0 : list_add_tail(&vmbo->shadow_list, &adev->shadow_list);
714 0 : mutex_unlock(&adev->shadow_list_lock);
715 0 : }
716 :
717 : /**
718 : * amdgpu_bo_restore_shadow - restore an &amdgpu_bo shadow
719 : *
720 : * @shadow: &amdgpu_bo shadow to be restored
721 : * @fence: dma_fence associated with the operation
722 : *
723 : * Copies a buffer object's shadow content back to the object.
724 : * This is used for recovering a buffer from its shadow in case of a gpu
725 : * reset where vram context may be lost.
726 : *
727 : * Returns:
728 : * 0 for success or a negative error code on failure.
729 : */
730 0 : int amdgpu_bo_restore_shadow(struct amdgpu_bo *shadow, struct dma_fence **fence)
731 :
732 : {
733 0 : struct amdgpu_device *adev = amdgpu_ttm_adev(shadow->tbo.bdev);
734 0 : struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
735 : uint64_t shadow_addr, parent_addr;
736 :
737 0 : shadow_addr = amdgpu_bo_gpu_offset(shadow);
738 0 : parent_addr = amdgpu_bo_gpu_offset(shadow->parent);
739 :
740 0 : return amdgpu_copy_buffer(ring, shadow_addr, parent_addr,
741 0 : amdgpu_bo_size(shadow), NULL, fence,
742 : true, false, false);
743 : }
744 :
745 : /**
746 : * amdgpu_bo_kmap - map an &amdgpu_bo buffer object
747 : * @bo: &amdgpu_bo buffer object to be mapped
748 : * @ptr: kernel virtual address to be returned
749 : *
750 : * Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls
751 : * amdgpu_bo_kptr() to get the kernel virtual address.
752 : *
753 : * Returns:
754 : * 0 for success or a negative error code on failure.
755 : */
756 0 : int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
757 : {
758 : void *kptr;
759 : long r;
760 :
761 0 : if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
762 : return -EPERM;
763 :
764 0 : r = dma_resv_wait_timeout(bo->tbo.base.resv, DMA_RESV_USAGE_KERNEL,
765 : false, MAX_SCHEDULE_TIMEOUT);
766 0 : if (r < 0)
767 0 : return r;
768 :
769 0 : kptr = amdgpu_bo_kptr(bo);
770 0 : if (kptr) {
771 0 : if (ptr)
772 0 : *ptr = kptr;
773 : return 0;
774 : }
775 :
776 0 : r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.resource->num_pages, &bo->kmap);
777 0 : if (r)
778 : return r;
779 :
780 0 : if (ptr)
781 0 : *ptr = amdgpu_bo_kptr(bo);
782 :
783 : return 0;
784 : }
785 :
786 : /**
787 : * amdgpu_bo_kptr - returns a kernel virtual address of the buffer object
788 : * @bo: &amdgpu_bo buffer object
789 : *
790 : * Calls ttm_kmap_obj_virtual() to get the kernel virtual address
791 : *
792 : * Returns:
793 : * the virtual address of a buffer object area.
794 : */
795 0 : void *amdgpu_bo_kptr(struct amdgpu_bo *bo)
796 : {
797 : bool is_iomem;
798 :
799 0 : return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
800 : }
801 :
802 : /**
803 : * amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object
804 : * @bo: &amdgpu_bo buffer object to be unmapped
805 : *
806 : * Unmaps a kernel map set up by amdgpu_bo_kmap().
807 : */
808 0 : void amdgpu_bo_kunmap(struct amdgpu_bo *bo)
809 : {
810 0 : if (bo->kmap.bo)
811 0 : ttm_bo_kunmap(&bo->kmap);
812 0 : }
813 :
814 : /**
815 : * amdgpu_bo_ref - reference an &amdgpu_bo buffer object
816 : * @bo: &amdgpu_bo buffer object
817 : *
818 : * References the contained &ttm_buffer_object.
819 : *
820 : * Returns:
821 : * a refcounted pointer to the &amdgpu_bo buffer object.
822 : */
823 0 : struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
824 : {
825 0 : if (bo == NULL)
826 : return NULL;
827 :
828 0 : ttm_bo_get(&bo->tbo);
829 0 : return bo;
830 : }
831 :
832 : /**
833 : * amdgpu_bo_unref - unreference an &amdgpu_bo buffer object
834 : * @bo: &amdgpu_bo buffer object
835 : *
836 : * Unreferences the contained &ttm_buffer_object and clear the pointer
837 : */
838 0 : void amdgpu_bo_unref(struct amdgpu_bo **bo)
839 : {
840 : struct ttm_buffer_object *tbo;
841 :
842 0 : if ((*bo) == NULL)
843 : return;
844 :
845 0 : tbo = &((*bo)->tbo);
846 0 : ttm_bo_put(tbo);
847 0 : *bo = NULL;
848 : }
849 :
850 : /**
851 : * amdgpu_bo_pin_restricted - pin an &amdgpu_bo buffer object
852 : * @bo: &amdgpu_bo buffer object to be pinned
853 : * @domain: domain to be pinned to
854 : * @min_offset: the start of requested address range
855 : * @max_offset: the end of requested address range
856 : *
857 : * Pins the buffer object according to requested domain and address range. If
858 : * the memory is unbound gart memory, binds the pages into gart table. Adjusts
859 : * pin_count and pin_size accordingly.
860 : *
861 : * Pinning means to lock pages in memory along with keeping them at a fixed
862 : * offset. It is required when a buffer can not be moved, for example, when
863 : * a display buffer is being scanned out.
864 : *
865 : * Compared with amdgpu_bo_pin(), this function gives more flexibility on
866 : * where to pin a buffer if there are specific restrictions on where a buffer
867 : * must be located.
868 : *
869 : * Returns:
870 : * 0 for success or a negative error code on failure.
871 : */
872 0 : int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
873 : u64 min_offset, u64 max_offset)
874 : {
875 0 : struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
876 0 : struct ttm_operation_ctx ctx = { false, false };
877 : int r, i;
878 :
879 0 : if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
880 : return -EPERM;
881 :
882 0 : if (WARN_ON_ONCE(min_offset > max_offset))
883 : return -EINVAL;
884 :
885 : /* Check domain to be pinned to against preferred domains */
886 0 : if (bo->preferred_domains & domain)
887 0 : domain = bo->preferred_domains & domain;
888 :
889 : /* A shared bo cannot be migrated to VRAM */
890 0 : if (bo->tbo.base.import_attach) {
891 0 : if (domain & AMDGPU_GEM_DOMAIN_GTT)
892 : domain = AMDGPU_GEM_DOMAIN_GTT;
893 : else
894 : return -EINVAL;
895 : }
896 :
897 0 : if (bo->tbo.pin_count) {
898 0 : uint32_t mem_type = bo->tbo.resource->mem_type;
899 0 : uint32_t mem_flags = bo->tbo.resource->placement;
900 :
901 0 : if (!(domain & amdgpu_mem_type_to_domain(mem_type)))
902 : return -EINVAL;
903 :
904 0 : if ((mem_type == TTM_PL_VRAM) &&
905 0 : (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) &&
906 0 : !(mem_flags & TTM_PL_FLAG_CONTIGUOUS))
907 : return -EINVAL;
908 :
909 0 : ttm_bo_pin(&bo->tbo);
910 :
911 0 : if (max_offset != 0) {
912 0 : u64 domain_start = amdgpu_ttm_domain_start(adev,
913 : mem_type);
914 0 : WARN_ON_ONCE(max_offset <
915 : (amdgpu_bo_gpu_offset(bo) - domain_start));
916 : }
917 :
918 : return 0;
919 : }
920 :
921 : /* This assumes only APU display buffers are pinned with (VRAM|GTT).
922 : * See function amdgpu_display_supported_domains()
923 : */
924 0 : domain = amdgpu_bo_get_preferred_domain(adev, domain);
925 :
926 0 : if (bo->tbo.base.import_attach)
927 0 : dma_buf_pin(bo->tbo.base.import_attach);
928 :
929 : /* force to pin into visible video ram */
930 0 : if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS))
931 0 : bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
932 0 : amdgpu_bo_placement_from_domain(bo, domain);
933 0 : for (i = 0; i < bo->placement.num_placement; i++) {
934 : unsigned fpfn, lpfn;
935 :
936 0 : fpfn = min_offset >> PAGE_SHIFT;
937 0 : lpfn = max_offset >> PAGE_SHIFT;
938 :
939 0 : if (fpfn > bo->placements[i].fpfn)
940 0 : bo->placements[i].fpfn = fpfn;
941 0 : if (!bo->placements[i].lpfn ||
942 0 : (lpfn && lpfn < bo->placements[i].lpfn))
943 0 : bo->placements[i].lpfn = lpfn;
944 : }
945 :
946 0 : r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
947 0 : if (unlikely(r)) {
948 0 : dev_err(adev->dev, "%p pin failed\n", bo);
949 0 : goto error;
950 : }
951 :
952 0 : ttm_bo_pin(&bo->tbo);
953 :
954 0 : domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type);
955 0 : if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
956 0 : atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size);
957 0 : atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo),
958 : &adev->visible_pin_size);
959 0 : } else if (domain == AMDGPU_GEM_DOMAIN_GTT) {
960 0 : atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size);
961 : }
962 :
963 : error:
964 : return r;
965 : }
966 :
967 : /**
968 : * amdgpu_bo_pin - pin an &amdgpu_bo buffer object
969 : * @bo: &amdgpu_bo buffer object to be pinned
970 : * @domain: domain to be pinned to
971 : *
972 : * A simple wrapper to amdgpu_bo_pin_restricted().
973 : * Provides a simpler API for buffers that do not have any strict restrictions
974 : * on where a buffer must be located.
975 : *
976 : * Returns:
977 : * 0 for success or a negative error code on failure.
978 : */
979 0 : int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain)
980 : {
981 0 : bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
982 0 : return amdgpu_bo_pin_restricted(bo, domain, 0, 0);
983 : }
984 :
985 : /**
986 : * amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object
987 : * @bo: &amdgpu_bo buffer object to be unpinned
988 : *
989 : * Decreases the pin_count, and clears the flags if pin_count reaches 0.
990 : * Changes placement and pin size accordingly.
991 : *
992 : * Returns:
993 : * 0 for success or a negative error code on failure.
994 : */
995 0 : void amdgpu_bo_unpin(struct amdgpu_bo *bo)
996 : {
997 0 : struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
998 :
999 0 : ttm_bo_unpin(&bo->tbo);
1000 0 : if (bo->tbo.pin_count)
1001 : return;
1002 :
1003 0 : if (bo->tbo.base.import_attach)
1004 0 : dma_buf_unpin(bo->tbo.base.import_attach);
1005 :
1006 0 : if (bo->tbo.resource->mem_type == TTM_PL_VRAM) {
1007 0 : atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size);
1008 0 : atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo),
1009 : &adev->visible_pin_size);
1010 0 : } else if (bo->tbo.resource->mem_type == TTM_PL_TT) {
1011 0 : atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size);
1012 : }
1013 : }
1014 :
1015 : static const char *amdgpu_vram_names[] = {
1016 : "UNKNOWN",
1017 : "GDDR1",
1018 : "DDR2",
1019 : "GDDR3",
1020 : "GDDR4",
1021 : "GDDR5",
1022 : "HBM",
1023 : "DDR3",
1024 : "DDR4",
1025 : "GDDR6",
1026 : "DDR5",
1027 : "LPDDR4",
1028 : "LPDDR5"
1029 : };
1030 :
1031 : /**
1032 : * amdgpu_bo_init - initialize memory manager
1033 : * @adev: amdgpu device object
1034 : *
1035 : * Calls amdgpu_ttm_init() to initialize amdgpu memory manager.
1036 : *
1037 : * Returns:
1038 : * 0 for success or a negative error code on failure.
1039 : */
1040 0 : int amdgpu_bo_init(struct amdgpu_device *adev)
1041 : {
1042 : /* On A+A platform, VRAM can be mapped as WB */
1043 0 : if (!adev->gmc.xgmi.connected_to_cpu) {
1044 : /* reserve PAT memory space to WC for VRAM */
1045 0 : int r = arch_io_reserve_memtype_wc(adev->gmc.aper_base,
1046 : adev->gmc.aper_size);
1047 :
1048 : if (r) {
1049 : DRM_ERROR("Unable to set WC memtype for the aperture base\n");
1050 : return r;
1051 : }
1052 :
1053 : /* Add an MTRR for the VRAM */
1054 0 : adev->gmc.vram_mtrr = arch_phys_wc_add(adev->gmc.aper_base,
1055 : adev->gmc.aper_size);
1056 : }
1057 :
1058 0 : DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
1059 : adev->gmc.mc_vram_size >> 20,
1060 : (unsigned long long)adev->gmc.aper_size >> 20);
1061 0 : DRM_INFO("RAM width %dbits %s\n",
1062 : adev->gmc.vram_width, amdgpu_vram_names[adev->gmc.vram_type]);
1063 0 : return amdgpu_ttm_init(adev);
1064 : }
1065 :
1066 : /**
1067 : * amdgpu_bo_fini - tear down memory manager
1068 : * @adev: amdgpu device object
1069 : *
1070 : * Reverses amdgpu_bo_init() to tear down memory manager.
1071 : */
1072 0 : void amdgpu_bo_fini(struct amdgpu_device *adev)
1073 : {
1074 : int idx;
1075 :
1076 0 : amdgpu_ttm_fini(adev);
1077 :
1078 : if (drm_dev_enter(adev_to_drm(adev), &idx)) {
1079 :
1080 : if (!adev->gmc.xgmi.connected_to_cpu) {
1081 : arch_phys_wc_del(adev->gmc.vram_mtrr);
1082 : arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
1083 : }
1084 0 : drm_dev_exit(idx);
1085 : }
1086 0 : }
1087 :
1088 : /**
1089 : * amdgpu_bo_set_tiling_flags - set tiling flags
1090 : * @bo: &amdgpu_bo buffer object
1091 : * @tiling_flags: new flags
1092 : *
1093 : * Sets buffer object's tiling flags with the new one. Used by GEM ioctl or
1094 : * kernel driver to set the tiling flags on a buffer.
1095 : *
1096 : * Returns:
1097 : * 0 for success or a negative error code on failure.
1098 : */
1099 0 : int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags)
1100 : {
1101 0 : struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1102 : struct amdgpu_bo_user *ubo;
1103 :
1104 0 : BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1105 0 : if (adev->family <= AMDGPU_FAMILY_CZ &&
1106 0 : AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6)
1107 : return -EINVAL;
1108 :
1109 0 : ubo = to_amdgpu_bo_user(bo);
1110 0 : ubo->tiling_flags = tiling_flags;
1111 0 : return 0;
1112 : }
1113 :
1114 : /**
1115 : * amdgpu_bo_get_tiling_flags - get tiling flags
1116 : * @bo: &amdgpu_bo buffer object
1117 : * @tiling_flags: returned flags
1118 : *
1119 : * Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to
1120 : * set the tiling flags on a buffer.
1121 : */
1122 0 : void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags)
1123 : {
1124 : struct amdgpu_bo_user *ubo;
1125 :
1126 0 : BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1127 : dma_resv_assert_held(bo->tbo.base.resv);
1128 0 : ubo = to_amdgpu_bo_user(bo);
1129 :
1130 0 : if (tiling_flags)
1131 0 : *tiling_flags = ubo->tiling_flags;
1132 0 : }
1133 :
1134 : /**
1135 : * amdgpu_bo_set_metadata - set metadata
1136 : * @bo: &amdgpu_bo buffer object
1137 : * @metadata: new metadata
1138 : * @metadata_size: size of the new metadata
1139 : * @flags: flags of the new metadata
1140 : *
1141 : * Sets buffer object's metadata, its size and flags.
1142 : * Used via GEM ioctl.
1143 : *
1144 : * Returns:
1145 : * 0 for success or a negative error code on failure.
1146 : */
1147 0 : int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata,
1148 : uint32_t metadata_size, uint64_t flags)
1149 : {
1150 : struct amdgpu_bo_user *ubo;
1151 : void *buffer;
1152 :
1153 0 : BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1154 0 : ubo = to_amdgpu_bo_user(bo);
1155 0 : if (!metadata_size) {
1156 0 : if (ubo->metadata_size) {
1157 0 : kfree(ubo->metadata);
1158 0 : ubo->metadata = NULL;
1159 0 : ubo->metadata_size = 0;
1160 : }
1161 : return 0;
1162 : }
1163 :
1164 0 : if (metadata == NULL)
1165 : return -EINVAL;
1166 :
1167 0 : buffer = kmemdup(metadata, metadata_size, GFP_KERNEL);
1168 0 : if (buffer == NULL)
1169 : return -ENOMEM;
1170 :
1171 0 : kfree(ubo->metadata);
1172 0 : ubo->metadata_flags = flags;
1173 0 : ubo->metadata = buffer;
1174 0 : ubo->metadata_size = metadata_size;
1175 :
1176 0 : return 0;
1177 : }
1178 :
1179 : /**
1180 : * amdgpu_bo_get_metadata - get metadata
1181 : * @bo: &amdgpu_bo buffer object
1182 : * @buffer: returned metadata
1183 : * @buffer_size: size of the buffer
1184 : * @metadata_size: size of the returned metadata
1185 : * @flags: flags of the returned metadata
1186 : *
1187 : * Gets buffer object's metadata, its size and flags. buffer_size shall not be
1188 : * less than metadata_size.
1189 : * Used via GEM ioctl.
1190 : *
1191 : * Returns:
1192 : * 0 for success or a negative error code on failure.
1193 : */
1194 0 : int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
1195 : size_t buffer_size, uint32_t *metadata_size,
1196 : uint64_t *flags)
1197 : {
1198 : struct amdgpu_bo_user *ubo;
1199 :
1200 0 : if (!buffer && !metadata_size)
1201 : return -EINVAL;
1202 :
1203 0 : BUG_ON(bo->tbo.type == ttm_bo_type_kernel);
1204 0 : ubo = to_amdgpu_bo_user(bo);
1205 0 : if (metadata_size)
1206 0 : *metadata_size = ubo->metadata_size;
1207 :
1208 0 : if (buffer) {
1209 0 : if (buffer_size < ubo->metadata_size)
1210 : return -EINVAL;
1211 :
1212 0 : if (ubo->metadata_size)
1213 0 : memcpy(buffer, ubo->metadata, ubo->metadata_size);
1214 : }
1215 :
1216 0 : if (flags)
1217 0 : *flags = ubo->metadata_flags;
1218 :
1219 : return 0;
1220 : }
1221 :
1222 : /**
1223 : * amdgpu_bo_move_notify - notification about a memory move
1224 : * @bo: pointer to a buffer object
1225 : * @evict: if this move is evicting the buffer from the graphics address space
1226 : * @new_mem: new information of the bufer object
1227 : *
1228 : * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs
1229 : * bookkeeping.
1230 : * TTM driver callback which is called when ttm moves a buffer.
1231 : */
1232 0 : void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
1233 : bool evict,
1234 : struct ttm_resource *new_mem)
1235 : {
1236 0 : struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1237 : struct amdgpu_bo *abo;
1238 0 : struct ttm_resource *old_mem = bo->resource;
1239 :
1240 0 : if (!amdgpu_bo_is_amdgpu_bo(bo))
1241 : return;
1242 :
1243 0 : abo = ttm_to_amdgpu_bo(bo);
1244 0 : amdgpu_vm_bo_invalidate(adev, abo, evict);
1245 :
1246 0 : amdgpu_bo_kunmap(abo);
1247 :
1248 0 : if (abo->tbo.base.dma_buf && !abo->tbo.base.import_attach &&
1249 0 : bo->resource->mem_type != TTM_PL_SYSTEM)
1250 0 : dma_buf_move_notify(abo->tbo.base.dma_buf);
1251 :
1252 : /* remember the eviction */
1253 0 : if (evict)
1254 0 : atomic64_inc(&adev->num_evictions);
1255 :
1256 : /* update statistics */
1257 : if (!new_mem)
1258 : return;
1259 :
1260 : /* move_notify is called before move happens */
1261 : trace_amdgpu_bo_move(abo, new_mem->mem_type, old_mem->mem_type);
1262 : }
1263 :
1264 0 : void amdgpu_bo_get_memory(struct amdgpu_bo *bo, uint64_t *vram_mem,
1265 : uint64_t *gtt_mem, uint64_t *cpu_mem)
1266 : {
1267 : unsigned int domain;
1268 :
1269 0 : domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type);
1270 0 : switch (domain) {
1271 : case AMDGPU_GEM_DOMAIN_VRAM:
1272 0 : *vram_mem += amdgpu_bo_size(bo);
1273 0 : break;
1274 : case AMDGPU_GEM_DOMAIN_GTT:
1275 0 : *gtt_mem += amdgpu_bo_size(bo);
1276 0 : break;
1277 : case AMDGPU_GEM_DOMAIN_CPU:
1278 : default:
1279 0 : *cpu_mem += amdgpu_bo_size(bo);
1280 0 : break;
1281 : }
1282 0 : }
1283 :
1284 : /**
1285 : * amdgpu_bo_release_notify - notification about a BO being released
1286 : * @bo: pointer to a buffer object
1287 : *
1288 : * Wipes VRAM buffers whose contents should not be leaked before the
1289 : * memory is released.
1290 : */
1291 0 : void amdgpu_bo_release_notify(struct ttm_buffer_object *bo)
1292 : {
1293 0 : struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1294 0 : struct dma_fence *fence = NULL;
1295 : struct amdgpu_bo *abo;
1296 : int r;
1297 :
1298 0 : if (!amdgpu_bo_is_amdgpu_bo(bo))
1299 0 : return;
1300 :
1301 0 : abo = ttm_to_amdgpu_bo(bo);
1302 :
1303 : if (abo->kfd_bo)
1304 : amdgpu_amdkfd_release_notify(abo);
1305 :
1306 : /* We only remove the fence if the resv has individualized. */
1307 0 : WARN_ON_ONCE(bo->type == ttm_bo_type_kernel
1308 : && bo->base.resv != &bo->base._resv);
1309 0 : if (bo->base.resv == &bo->base._resv)
1310 : amdgpu_amdkfd_remove_fence_on_pt_pd_bos(abo);
1311 :
1312 0 : if (bo->resource->mem_type != TTM_PL_VRAM ||
1313 0 : !(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE) ||
1314 0 : adev->in_suspend || adev->shutdown)
1315 : return;
1316 :
1317 0 : if (WARN_ON_ONCE(!dma_resv_trylock(bo->base.resv)))
1318 : return;
1319 :
1320 0 : r = amdgpu_fill_buffer(abo, AMDGPU_POISON, bo->base.resv, &fence);
1321 0 : if (!WARN_ON(r)) {
1322 0 : amdgpu_bo_fence(abo, fence, false);
1323 0 : dma_fence_put(fence);
1324 : }
1325 :
1326 0 : dma_resv_unlock(bo->base.resv);
1327 : }
1328 :
1329 : /**
1330 : * amdgpu_bo_fault_reserve_notify - notification about a memory fault
1331 : * @bo: pointer to a buffer object
1332 : *
1333 : * Notifies the driver we are taking a fault on this BO and have reserved it,
1334 : * also performs bookkeeping.
1335 : * TTM driver callback for dealing with vm faults.
1336 : *
1337 : * Returns:
1338 : * 0 for success or a negative error code on failure.
1339 : */
1340 0 : vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
1341 : {
1342 0 : struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1343 0 : struct ttm_operation_ctx ctx = { false, false };
1344 0 : struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
1345 : unsigned long offset;
1346 : int r;
1347 :
1348 : /* Remember that this BO was accessed by the CPU */
1349 0 : abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1350 :
1351 0 : if (bo->resource->mem_type != TTM_PL_VRAM)
1352 : return 0;
1353 :
1354 0 : offset = bo->resource->start << PAGE_SHIFT;
1355 0 : if ((offset + bo->base.size) <= adev->gmc.visible_vram_size)
1356 : return 0;
1357 :
1358 : /* Can't move a pinned BO to visible VRAM */
1359 0 : if (abo->tbo.pin_count > 0)
1360 : return VM_FAULT_SIGBUS;
1361 :
1362 : /* hurrah the memory is not visible ! */
1363 0 : atomic64_inc(&adev->num_vram_cpu_page_faults);
1364 0 : amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
1365 : AMDGPU_GEM_DOMAIN_GTT);
1366 :
1367 : /* Avoid costly evictions; only set GTT as a busy placement */
1368 0 : abo->placement.num_busy_placement = 1;
1369 0 : abo->placement.busy_placement = &abo->placements[1];
1370 :
1371 0 : r = ttm_bo_validate(bo, &abo->placement, &ctx);
1372 0 : if (unlikely(r == -EBUSY || r == -ERESTARTSYS))
1373 : return VM_FAULT_NOPAGE;
1374 0 : else if (unlikely(r))
1375 : return VM_FAULT_SIGBUS;
1376 :
1377 0 : offset = bo->resource->start << PAGE_SHIFT;
1378 : /* this should never happen */
1379 0 : if (bo->resource->mem_type == TTM_PL_VRAM &&
1380 0 : (offset + bo->base.size) > adev->gmc.visible_vram_size)
1381 : return VM_FAULT_SIGBUS;
1382 :
1383 0 : ttm_bo_move_to_lru_tail_unlocked(bo);
1384 0 : return 0;
1385 : }
1386 :
1387 : /**
1388 : * amdgpu_bo_fence - add fence to buffer object
1389 : *
1390 : * @bo: buffer object in question
1391 : * @fence: fence to add
1392 : * @shared: true if fence should be added shared
1393 : *
1394 : */
1395 0 : void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,
1396 : bool shared)
1397 : {
1398 0 : struct dma_resv *resv = bo->tbo.base.resv;
1399 : int r;
1400 :
1401 0 : r = dma_resv_reserve_fences(resv, 1);
1402 0 : if (r) {
1403 : /* As last resort on OOM we block for the fence */
1404 : dma_fence_wait(fence, false);
1405 : return;
1406 : }
1407 :
1408 0 : dma_resv_add_fence(resv, fence, shared ? DMA_RESV_USAGE_READ :
1409 : DMA_RESV_USAGE_WRITE);
1410 : }
1411 :
1412 : /**
1413 : * amdgpu_bo_sync_wait_resv - Wait for BO reservation fences
1414 : *
1415 : * @adev: amdgpu device pointer
1416 : * @resv: reservation object to sync to
1417 : * @sync_mode: synchronization mode
1418 : * @owner: fence owner
1419 : * @intr: Whether the wait is interruptible
1420 : *
1421 : * Extract the fences from the reservation object and waits for them to finish.
1422 : *
1423 : * Returns:
1424 : * 0 on success, errno otherwise.
1425 : */
1426 0 : int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv,
1427 : enum amdgpu_sync_mode sync_mode, void *owner,
1428 : bool intr)
1429 : {
1430 : struct amdgpu_sync sync;
1431 : int r;
1432 :
1433 0 : amdgpu_sync_create(&sync);
1434 0 : amdgpu_sync_resv(adev, &sync, resv, sync_mode, owner);
1435 0 : r = amdgpu_sync_wait(&sync, intr);
1436 0 : amdgpu_sync_free(&sync);
1437 0 : return r;
1438 : }
1439 :
1440 : /**
1441 : * amdgpu_bo_sync_wait - Wrapper for amdgpu_bo_sync_wait_resv
1442 : * @bo: buffer object to wait for
1443 : * @owner: fence owner
1444 : * @intr: Whether the wait is interruptible
1445 : *
1446 : * Wrapper to wait for fences in a BO.
1447 : * Returns:
1448 : * 0 on success, errno otherwise.
1449 : */
1450 0 : int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr)
1451 : {
1452 0 : struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1453 :
1454 0 : return amdgpu_bo_sync_wait_resv(adev, bo->tbo.base.resv,
1455 : AMDGPU_SYNC_NE_OWNER, owner, intr);
1456 : }
1457 :
1458 : /**
1459 : * amdgpu_bo_gpu_offset - return GPU offset of bo
1460 : * @bo: amdgpu object for which we query the offset
1461 : *
1462 : * Note: object should either be pinned or reserved when calling this
1463 : * function, it might be useful to add check for this for debugging.
1464 : *
1465 : * Returns:
1466 : * current GPU offset of the object.
1467 : */
1468 0 : u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
1469 : {
1470 0 : WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_SYSTEM);
1471 0 : WARN_ON_ONCE(!dma_resv_is_locked(bo->tbo.base.resv) &&
1472 : !bo->tbo.pin_count && bo->tbo.type != ttm_bo_type_kernel);
1473 0 : WARN_ON_ONCE(bo->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET);
1474 0 : WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_VRAM &&
1475 : !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS));
1476 :
1477 0 : return amdgpu_bo_gpu_offset_no_check(bo);
1478 : }
1479 :
1480 : /**
1481 : * amdgpu_bo_gpu_offset_no_check - return GPU offset of bo
1482 : * @bo: amdgpu object for which we query the offset
1483 : *
1484 : * Returns:
1485 : * current GPU offset of the object without raising warnings.
1486 : */
1487 0 : u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo)
1488 : {
1489 0 : struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1490 : uint64_t offset;
1491 :
1492 0 : offset = (bo->tbo.resource->start << PAGE_SHIFT) +
1493 0 : amdgpu_ttm_domain_start(adev, bo->tbo.resource->mem_type);
1494 :
1495 0 : return amdgpu_gmc_sign_extend(offset);
1496 : }
1497 :
1498 : /**
1499 : * amdgpu_bo_get_preferred_domain - get preferred domain
1500 : * @adev: amdgpu device object
1501 : * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>`
1502 : *
1503 : * Returns:
1504 : * Which of the allowed domains is preferred for allocating the BO.
1505 : */
1506 0 : uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev,
1507 : uint32_t domain)
1508 : {
1509 0 : if (domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) {
1510 0 : domain = AMDGPU_GEM_DOMAIN_VRAM;
1511 0 : if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD)
1512 0 : domain = AMDGPU_GEM_DOMAIN_GTT;
1513 : }
1514 0 : return domain;
1515 : }
1516 :
1517 : #if defined(CONFIG_DEBUG_FS)
1518 : #define amdgpu_bo_print_flag(m, bo, flag) \
1519 : do { \
1520 : if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) { \
1521 : seq_printf((m), " " #flag); \
1522 : } \
1523 : } while (0)
1524 :
1525 : /**
1526 : * amdgpu_bo_print_info - print BO info in debugfs file
1527 : *
1528 : * @id: Index or Id of the BO
1529 : * @bo: Requested BO for printing info
1530 : * @m: debugfs file
1531 : *
1532 : * Print BO information in debugfs file
1533 : *
1534 : * Returns:
1535 : * Size of the BO in bytes.
1536 : */
1537 : u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m)
1538 : {
1539 : struct dma_buf_attachment *attachment;
1540 : struct dma_buf *dma_buf;
1541 : unsigned int domain;
1542 : const char *placement;
1543 : unsigned int pin_count;
1544 : u64 size;
1545 :
1546 : domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type);
1547 : switch (domain) {
1548 : case AMDGPU_GEM_DOMAIN_VRAM:
1549 : placement = "VRAM";
1550 : break;
1551 : case AMDGPU_GEM_DOMAIN_GTT:
1552 : placement = " GTT";
1553 : break;
1554 : case AMDGPU_GEM_DOMAIN_CPU:
1555 : default:
1556 : placement = " CPU";
1557 : break;
1558 : }
1559 :
1560 : size = amdgpu_bo_size(bo);
1561 : seq_printf(m, "\t\t0x%08x: %12lld byte %s",
1562 : id, size, placement);
1563 :
1564 : pin_count = READ_ONCE(bo->tbo.pin_count);
1565 : if (pin_count)
1566 : seq_printf(m, " pin count %d", pin_count);
1567 :
1568 : dma_buf = READ_ONCE(bo->tbo.base.dma_buf);
1569 : attachment = READ_ONCE(bo->tbo.base.import_attach);
1570 :
1571 : if (attachment)
1572 : seq_printf(m, " imported from %p", dma_buf);
1573 : else if (dma_buf)
1574 : seq_printf(m, " exported as %p", dma_buf);
1575 :
1576 : amdgpu_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED);
1577 : amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS);
1578 : amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC);
1579 : amdgpu_bo_print_flag(m, bo, VRAM_CLEARED);
1580 : amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS);
1581 : amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID);
1582 : amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC);
1583 :
1584 : seq_puts(m, "\n");
1585 :
1586 : return size;
1587 : }
1588 : #endif
|
