Line data Source code
1 : /**************************************************************************
2 : *
3 : * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
4 : * All Rights Reserved.
5 : *
6 : * Permission is hereby granted, free of charge, to any person obtaining a
7 : * copy of this software and associated documentation files (the
8 : * "Software"), to deal in the Software without restriction, including
9 : * without limitation the rights to use, copy, modify, merge, publish,
10 : * distribute, sub license, and/or sell copies of the Software, and to
11 : * permit persons to whom the Software is furnished to do so, subject to
12 : * the following conditions:
13 : *
14 : * The above copyright notice and this permission notice (including the
15 : * next paragraph) shall be included in all copies or substantial portions
16 : * of the Software.
17 : *
18 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 : * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 : * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 : * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 : * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 : * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 : * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 : *
26 : **************************************************************************/
27 : /*
28 : * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
29 : */
30 :
31 : #ifndef _TTM_BO_API_H_
32 : #define _TTM_BO_API_H_
33 :
34 : #include <drm/drm_gem.h>
35 : #include <drm/drm_vma_manager.h>
36 : #include <linux/kref.h>
37 : #include <linux/list.h>
38 : #include <linux/wait.h>
39 : #include <linux/mutex.h>
40 : #include <linux/mm.h>
41 : #include <linux/bitmap.h>
42 : #include <linux/dma-resv.h>
43 :
44 : #include "ttm_resource.h"
45 :
46 : struct ttm_global;
47 :
48 : struct ttm_device;
49 :
50 : struct iosys_map;
51 :
52 : struct drm_mm_node;
53 :
54 : struct ttm_placement;
55 :
56 : struct ttm_place;
57 :
58 : /**
59 : * enum ttm_bo_type
60 : *
61 : * @ttm_bo_type_device: These are 'normal' buffers that can
62 : * be mmapped by user space. Each of these bos occupy a slot in the
63 : * device address space, that can be used for normal vm operations.
64 : *
65 : * @ttm_bo_type_kernel: These buffers are like ttm_bo_type_device buffers,
66 : * but they cannot be accessed from user-space. For kernel-only use.
67 : *
68 : * @ttm_bo_type_sg: Buffer made from dmabuf sg table shared with another
69 : * driver.
70 : */
71 :
72 : enum ttm_bo_type {
73 : ttm_bo_type_device,
74 : ttm_bo_type_kernel,
75 : ttm_bo_type_sg
76 : };
77 :
78 : struct ttm_tt;
79 :
80 : /**
81 : * struct ttm_buffer_object
82 : *
83 : * @base: drm_gem_object superclass data.
84 : * @bdev: Pointer to the buffer object device structure.
85 : * @type: The bo type.
86 : * @page_alignment: Page alignment.
87 : * @destroy: Destruction function. If NULL, kfree is used.
88 : * @num_pages: Actual number of pages.
89 : * @kref: Reference count of this buffer object. When this refcount reaches
90 : * zero, the object is destroyed or put on the delayed delete list.
91 : * @mem: structure describing current placement.
92 : * @ttm: TTM structure holding system pages.
93 : * @evicted: Whether the object was evicted without user-space knowing.
94 : * @deleted: True if the object is only a zombie and already deleted.
95 : * @ddestroy: List head for the delayed destroy list.
96 : * @swap: List head for swap LRU list.
97 : * @offset: The current GPU offset, which can have different meanings
98 : * depending on the memory type. For SYSTEM type memory, it should be 0.
99 : * @cur_placement: Hint of current placement.
100 : *
101 : * Base class for TTM buffer object, that deals with data placement and CPU
102 : * mappings. GPU mappings are really up to the driver, but for simpler GPUs
103 : * the driver can usually use the placement offset @offset directly as the
104 : * GPU virtual address. For drivers implementing multiple
105 : * GPU memory manager contexts, the driver should manage the address space
106 : * in these contexts separately and use these objects to get the correct
107 : * placement and caching for these GPU maps. This makes it possible to use
108 : * these objects for even quite elaborate memory management schemes.
109 : * The destroy member, the API visibility of this object makes it possible
110 : * to derive driver specific types.
111 : */
112 :
113 : struct ttm_buffer_object {
114 : struct drm_gem_object base;
115 :
116 : /**
117 : * Members constant at init.
118 : */
119 :
120 : struct ttm_device *bdev;
121 : enum ttm_bo_type type;
122 : uint32_t page_alignment;
123 : void (*destroy) (struct ttm_buffer_object *);
124 :
125 : /**
126 : * Members not needing protection.
127 : */
128 : struct kref kref;
129 :
130 : /**
131 : * Members protected by the bo::resv::reserved lock.
132 : */
133 :
134 : struct ttm_resource *resource;
135 : struct ttm_tt *ttm;
136 : bool deleted;
137 : struct ttm_lru_bulk_move *bulk_move;
138 :
139 : /**
140 : * Members protected by the bdev::lru_lock.
141 : */
142 :
143 : struct list_head ddestroy;
144 :
145 : /**
146 : * Members protected by a bo reservation.
147 : */
148 :
149 : unsigned priority;
150 : unsigned pin_count;
151 :
152 : /**
153 : * Special members that are protected by the reserve lock
154 : * and the bo::lock when written to. Can be read with
155 : * either of these locks held.
156 : */
157 :
158 : struct sg_table *sg;
159 : };
160 :
161 : /**
162 : * struct ttm_bo_kmap_obj
163 : *
164 : * @virtual: The current kernel virtual address.
165 : * @page: The page when kmap'ing a single page.
166 : * @bo_kmap_type: Type of bo_kmap.
167 : *
168 : * Object describing a kernel mapping. Since a TTM bo may be located
169 : * in various memory types with various caching policies, the
170 : * mapping can either be an ioremap, a vmap, a kmap or part of a
171 : * premapped region.
172 : */
173 :
174 : #define TTM_BO_MAP_IOMEM_MASK 0x80
175 : struct ttm_bo_kmap_obj {
176 : void *virtual;
177 : struct page *page;
178 : enum {
179 : ttm_bo_map_iomap = 1 | TTM_BO_MAP_IOMEM_MASK,
180 : ttm_bo_map_vmap = 2,
181 : ttm_bo_map_kmap = 3,
182 : ttm_bo_map_premapped = 4 | TTM_BO_MAP_IOMEM_MASK,
183 : } bo_kmap_type;
184 : struct ttm_buffer_object *bo;
185 : };
186 :
187 : /**
188 : * struct ttm_operation_ctx
189 : *
190 : * @interruptible: Sleep interruptible if sleeping.
191 : * @no_wait_gpu: Return immediately if the GPU is busy.
192 : * @gfp_retry_mayfail: Set the __GFP_RETRY_MAYFAIL when allocation pages.
193 : * @allow_res_evict: Allow eviction of reserved BOs. Can be used when multiple
194 : * BOs share the same reservation object.
195 : * @force_alloc: Don't check the memory account during suspend or CPU page
196 : * faults. Should only be used by TTM internally.
197 : * @resv: Reservation object to allow reserved evictions with.
198 : *
199 : * Context for TTM operations like changing buffer placement or general memory
200 : * allocation.
201 : */
202 : struct ttm_operation_ctx {
203 : bool interruptible;
204 : bool no_wait_gpu;
205 : bool gfp_retry_mayfail;
206 : bool allow_res_evict;
207 : bool force_alloc;
208 : struct dma_resv *resv;
209 : uint64_t bytes_moved;
210 : };
211 :
212 : /**
213 : * ttm_bo_get - reference a struct ttm_buffer_object
214 : *
215 : * @bo: The buffer object.
216 : */
217 : static inline void ttm_bo_get(struct ttm_buffer_object *bo)
218 : {
219 0 : kref_get(&bo->kref);
220 : }
221 :
222 : /**
223 : * ttm_bo_get_unless_zero - reference a struct ttm_buffer_object unless
224 : * its refcount has already reached zero.
225 : * @bo: The buffer object.
226 : *
227 : * Used to reference a TTM buffer object in lookups where the object is removed
228 : * from the lookup structure during the destructor and for RCU lookups.
229 : *
230 : * Returns: @bo if the referencing was successful, NULL otherwise.
231 : */
232 : static inline __must_check struct ttm_buffer_object *
233 : ttm_bo_get_unless_zero(struct ttm_buffer_object *bo)
234 : {
235 0 : if (!kref_get_unless_zero(&bo->kref))
236 : return NULL;
237 : return bo;
238 : }
239 :
240 : /**
241 : * ttm_bo_wait - wait for buffer idle.
242 : *
243 : * @bo: The buffer object.
244 : * @interruptible: Use interruptible wait.
245 : * @no_wait: Return immediately if buffer is busy.
246 : *
247 : * This function must be called with the bo::mutex held, and makes
248 : * sure any previous rendering to the buffer is completed.
249 : * Note: It might be necessary to block validations before the
250 : * wait by reserving the buffer.
251 : * Returns -EBUSY if no_wait is true and the buffer is busy.
252 : * Returns -ERESTARTSYS if interrupted by a signal.
253 : */
254 : int ttm_bo_wait(struct ttm_buffer_object *bo, bool interruptible, bool no_wait);
255 :
256 : static inline int ttm_bo_wait_ctx(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx)
257 : {
258 0 : return ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
259 : }
260 :
261 : /**
262 : * ttm_bo_validate
263 : *
264 : * @bo: The buffer object.
265 : * @placement: Proposed placement for the buffer object.
266 : * @ctx: validation parameters.
267 : *
268 : * Changes placement and caching policy of the buffer object
269 : * according proposed placement.
270 : * Returns
271 : * -EINVAL on invalid proposed placement.
272 : * -ENOMEM on out-of-memory condition.
273 : * -EBUSY if no_wait is true and buffer busy.
274 : * -ERESTARTSYS if interrupted by a signal.
275 : */
276 : int ttm_bo_validate(struct ttm_buffer_object *bo,
277 : struct ttm_placement *placement,
278 : struct ttm_operation_ctx *ctx);
279 :
280 : /**
281 : * ttm_bo_put
282 : *
283 : * @bo: The buffer object.
284 : *
285 : * Unreference a buffer object.
286 : */
287 : void ttm_bo_put(struct ttm_buffer_object *bo);
288 :
289 : void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo);
290 : void ttm_bo_set_bulk_move(struct ttm_buffer_object *bo,
291 : struct ttm_lru_bulk_move *bulk);
292 :
293 : /**
294 : * ttm_bo_lock_delayed_workqueue
295 : *
296 : * Prevent the delayed workqueue from running.
297 : * Returns
298 : * True if the workqueue was queued at the time
299 : */
300 : int ttm_bo_lock_delayed_workqueue(struct ttm_device *bdev);
301 :
302 : /**
303 : * ttm_bo_unlock_delayed_workqueue
304 : *
305 : * Allows the delayed workqueue to run.
306 : */
307 : void ttm_bo_unlock_delayed_workqueue(struct ttm_device *bdev, int resched);
308 :
309 : /**
310 : * ttm_bo_eviction_valuable
311 : *
312 : * @bo: The buffer object to evict
313 : * @place: the placement we need to make room for
314 : *
315 : * Check if it is valuable to evict the BO to make room for the given placement.
316 : */
317 : bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
318 : const struct ttm_place *place);
319 :
320 : /**
321 : * ttm_bo_init_reserved
322 : *
323 : * @bdev: Pointer to a ttm_device struct.
324 : * @bo: Pointer to a ttm_buffer_object to be initialized.
325 : * @size: Requested size of buffer object.
326 : * @type: Requested type of buffer object.
327 : * @placement: Initial placement for buffer object.
328 : * @page_alignment: Data alignment in pages.
329 : * @ctx: TTM operation context for memory allocation.
330 : * @sg: Scatter-gather table.
331 : * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one.
332 : * @destroy: Destroy function. Use NULL for kfree().
333 : *
334 : * This function initializes a pre-allocated struct ttm_buffer_object.
335 : * As this object may be part of a larger structure, this function,
336 : * together with the @destroy function,
337 : * enables driver-specific objects derived from a ttm_buffer_object.
338 : *
339 : * On successful return, the caller owns an object kref to @bo. The kref and
340 : * list_kref are usually set to 1, but note that in some situations, other
341 : * tasks may already be holding references to @bo as well.
342 : * Furthermore, if resv == NULL, the buffer's reservation lock will be held,
343 : * and it is the caller's responsibility to call ttm_bo_unreserve.
344 : *
345 : * If a failure occurs, the function will call the @destroy function, or
346 : * kfree() if @destroy is NULL. Thus, after a failure, dereferencing @bo is
347 : * illegal and will likely cause memory corruption.
348 : *
349 : * Returns
350 : * -ENOMEM: Out of memory.
351 : * -EINVAL: Invalid placement flags.
352 : * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources.
353 : */
354 :
355 : int ttm_bo_init_reserved(struct ttm_device *bdev,
356 : struct ttm_buffer_object *bo,
357 : size_t size, enum ttm_bo_type type,
358 : struct ttm_placement *placement,
359 : uint32_t page_alignment,
360 : struct ttm_operation_ctx *ctx,
361 : struct sg_table *sg, struct dma_resv *resv,
362 : void (*destroy) (struct ttm_buffer_object *));
363 :
364 : /**
365 : * ttm_bo_init
366 : *
367 : * @bdev: Pointer to a ttm_device struct.
368 : * @bo: Pointer to a ttm_buffer_object to be initialized.
369 : * @size: Requested size of buffer object.
370 : * @type: Requested type of buffer object.
371 : * @placement: Initial placement for buffer object.
372 : * @page_alignment: Data alignment in pages.
373 : * @interruptible: If needing to sleep to wait for GPU resources,
374 : * sleep interruptible.
375 : * pinned in physical memory. If this behaviour is not desired, this member
376 : * holds a pointer to a persistent shmem object. Typically, this would
377 : * point to the shmem object backing a GEM object if TTM is used to back a
378 : * GEM user interface.
379 : * @sg: Scatter-gather table.
380 : * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one.
381 : * @destroy: Destroy function. Use NULL for kfree().
382 : *
383 : * This function initializes a pre-allocated struct ttm_buffer_object.
384 : * As this object may be part of a larger structure, this function,
385 : * together with the @destroy function,
386 : * enables driver-specific objects derived from a ttm_buffer_object.
387 : *
388 : * On successful return, the caller owns an object kref to @bo. The kref and
389 : * list_kref are usually set to 1, but note that in some situations, other
390 : * tasks may already be holding references to @bo as well.
391 : *
392 : * If a failure occurs, the function will call the @destroy function, or
393 : * kfree() if @destroy is NULL. Thus, after a failure, dereferencing @bo is
394 : * illegal and will likely cause memory corruption.
395 : *
396 : * Returns
397 : * -ENOMEM: Out of memory.
398 : * -EINVAL: Invalid placement flags.
399 : * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources.
400 : */
401 : int ttm_bo_init(struct ttm_device *bdev, struct ttm_buffer_object *bo,
402 : size_t size, enum ttm_bo_type type,
403 : struct ttm_placement *placement,
404 : uint32_t page_alignment, bool interrubtible,
405 : struct sg_table *sg, struct dma_resv *resv,
406 : void (*destroy) (struct ttm_buffer_object *));
407 :
408 : /**
409 : * ttm_kmap_obj_virtual
410 : *
411 : * @map: A struct ttm_bo_kmap_obj returned from ttm_bo_kmap.
412 : * @is_iomem: Pointer to an integer that on return indicates 1 if the
413 : * virtual map is io memory, 0 if normal memory.
414 : *
415 : * Returns the virtual address of a buffer object area mapped by ttm_bo_kmap.
416 : * If *is_iomem is 1 on return, the virtual address points to an io memory area,
417 : * that should strictly be accessed by the iowriteXX() and similar functions.
418 : */
419 : static inline void *ttm_kmap_obj_virtual(struct ttm_bo_kmap_obj *map,
420 : bool *is_iomem)
421 : {
422 0 : *is_iomem = !!(map->bo_kmap_type & TTM_BO_MAP_IOMEM_MASK);
423 0 : return map->virtual;
424 : }
425 :
426 : /**
427 : * ttm_bo_kmap
428 : *
429 : * @bo: The buffer object.
430 : * @start_page: The first page to map.
431 : * @num_pages: Number of pages to map.
432 : * @map: pointer to a struct ttm_bo_kmap_obj representing the map.
433 : *
434 : * Sets up a kernel virtual mapping, using ioremap, vmap or kmap to the
435 : * data in the buffer object. The ttm_kmap_obj_virtual function can then be
436 : * used to obtain a virtual address to the data.
437 : *
438 : * Returns
439 : * -ENOMEM: Out of memory.
440 : * -EINVAL: Invalid range.
441 : */
442 : int ttm_bo_kmap(struct ttm_buffer_object *bo, unsigned long start_page,
443 : unsigned long num_pages, struct ttm_bo_kmap_obj *map);
444 :
445 : /**
446 : * ttm_bo_kunmap
447 : *
448 : * @map: Object describing the map to unmap.
449 : *
450 : * Unmaps a kernel map set up by ttm_bo_kmap.
451 : */
452 : void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map);
453 :
454 : /**
455 : * ttm_bo_vmap
456 : *
457 : * @bo: The buffer object.
458 : * @map: pointer to a struct iosys_map representing the map.
459 : *
460 : * Sets up a kernel virtual mapping, using ioremap or vmap to the
461 : * data in the buffer object. The parameter @map returns the virtual
462 : * address as struct iosys_map. Unmap the buffer with ttm_bo_vunmap().
463 : *
464 : * Returns
465 : * -ENOMEM: Out of memory.
466 : * -EINVAL: Invalid range.
467 : */
468 : int ttm_bo_vmap(struct ttm_buffer_object *bo, struct iosys_map *map);
469 :
470 : /**
471 : * ttm_bo_vunmap
472 : *
473 : * @bo: The buffer object.
474 : * @map: Object describing the map to unmap.
475 : *
476 : * Unmaps a kernel map set up by ttm_bo_vmap().
477 : */
478 : void ttm_bo_vunmap(struct ttm_buffer_object *bo, struct iosys_map *map);
479 :
480 : /**
481 : * ttm_bo_mmap_obj - mmap memory backed by a ttm buffer object.
482 : *
483 : * @vma: vma as input from the fbdev mmap method.
484 : * @bo: The bo backing the address space.
485 : *
486 : * Maps a buffer object.
487 : */
488 : int ttm_bo_mmap_obj(struct vm_area_struct *vma, struct ttm_buffer_object *bo);
489 :
490 : /**
491 : * ttm_bo_io
492 : *
493 : * @bdev: Pointer to the struct ttm_device.
494 : * @filp: Pointer to the struct file attempting to read / write.
495 : * @wbuf: User-space pointer to address of buffer to write. NULL on read.
496 : * @rbuf: User-space pointer to address of buffer to read into.
497 : * Null on write.
498 : * @count: Number of bytes to read / write.
499 : * @f_pos: Pointer to current file position.
500 : * @write: 1 for read, 0 for write.
501 : *
502 : * This function implements read / write into ttm buffer objects, and is
503 : * intended to
504 : * be called from the fops::read and fops::write method.
505 : * Returns:
506 : * See man (2) write, man(2) read. In particular,
507 : * the function may return -ERESTARTSYS if
508 : * interrupted by a signal.
509 : */
510 : ssize_t ttm_bo_io(struct ttm_device *bdev, struct file *filp,
511 : const char __user *wbuf, char __user *rbuf,
512 : size_t count, loff_t *f_pos, bool write);
513 :
514 : int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx,
515 : gfp_t gfp_flags);
516 :
517 : void ttm_bo_pin(struct ttm_buffer_object *bo);
518 : void ttm_bo_unpin(struct ttm_buffer_object *bo);
519 :
520 : int ttm_mem_evict_first(struct ttm_device *bdev,
521 : struct ttm_resource_manager *man,
522 : const struct ttm_place *place,
523 : struct ttm_operation_ctx *ctx,
524 : struct ww_acquire_ctx *ticket);
525 :
526 : /* Default number of pre-faulted pages in the TTM fault handler */
527 : #define TTM_BO_VM_NUM_PREFAULT 16
528 :
529 : vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo,
530 : struct vm_fault *vmf);
531 :
532 : vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf,
533 : pgprot_t prot,
534 : pgoff_t num_prefault);
535 :
536 : vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf);
537 :
538 : void ttm_bo_vm_open(struct vm_area_struct *vma);
539 :
540 : void ttm_bo_vm_close(struct vm_area_struct *vma);
541 :
542 : int ttm_bo_vm_access(struct vm_area_struct *vma, unsigned long addr,
543 : void *buf, int len, int write);
544 : bool ttm_bo_delayed_delete(struct ttm_device *bdev, bool remove_all);
545 :
546 : vm_fault_t ttm_bo_vm_dummy_page(struct vm_fault *vmf, pgprot_t prot);
547 :
548 : #endif
|
