Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : /*
3 : * Framework for buffer objects that can be shared across devices/subsystems.
4 : *
5 : * Copyright(C) 2011 Linaro Limited. All rights reserved.
6 : * Author: Sumit Semwal <sumit.semwal@ti.com>
7 : *
8 : * Many thanks to linaro-mm-sig list, and specially
9 : * Arnd Bergmann <arnd@arndb.de>, Rob Clark <rob@ti.com> and
10 : * Daniel Vetter <daniel@ffwll.ch> for their support in creation and
11 : * refining of this idea.
12 : */
13 :
14 : #include <linux/fs.h>
15 : #include <linux/slab.h>
16 : #include <linux/dma-buf.h>
17 : #include <linux/dma-fence.h>
18 : #include <linux/anon_inodes.h>
19 : #include <linux/export.h>
20 : #include <linux/debugfs.h>
21 : #include <linux/module.h>
22 : #include <linux/seq_file.h>
23 : #include <linux/poll.h>
24 : #include <linux/dma-resv.h>
25 : #include <linux/mm.h>
26 : #include <linux/mount.h>
27 : #include <linux/pseudo_fs.h>
28 :
29 : #include <uapi/linux/dma-buf.h>
30 : #include <uapi/linux/magic.h>
31 :
32 : #include "dma-buf-sysfs-stats.h"
33 :
34 : static inline int is_dma_buf_file(struct file *);
35 :
36 : struct dma_buf_list {
37 : struct list_head head;
38 : struct mutex lock;
39 : };
40 :
41 : static struct dma_buf_list db_list;
42 :
43 0 : static char *dmabuffs_dname(struct dentry *dentry, char *buffer, int buflen)
44 : {
45 : struct dma_buf *dmabuf;
46 : char name[DMA_BUF_NAME_LEN];
47 0 : size_t ret = 0;
48 :
49 0 : dmabuf = dentry->d_fsdata;
50 0 : spin_lock(&dmabuf->name_lock);
51 0 : if (dmabuf->name)
52 0 : ret = strlcpy(name, dmabuf->name, DMA_BUF_NAME_LEN);
53 0 : spin_unlock(&dmabuf->name_lock);
54 :
55 0 : return dynamic_dname(dentry, buffer, buflen, "/%s:%s",
56 : dentry->d_name.name, ret > 0 ? name : "");
57 : }
58 :
59 0 : static void dma_buf_release(struct dentry *dentry)
60 : {
61 : struct dma_buf *dmabuf;
62 :
63 0 : dmabuf = dentry->d_fsdata;
64 0 : if (unlikely(!dmabuf))
65 : return;
66 :
67 0 : BUG_ON(dmabuf->vmapping_counter);
68 :
69 : /*
70 : * If you hit this BUG() it could mean:
71 : * * There's a file reference imbalance in dma_buf_poll / dma_buf_poll_cb or somewhere else
72 : * * dmabuf->cb_in/out.active are non-0 despite no pending fence callback
73 : */
74 0 : BUG_ON(dmabuf->cb_in.active || dmabuf->cb_out.active);
75 :
76 0 : dma_buf_stats_teardown(dmabuf);
77 0 : dmabuf->ops->release(dmabuf);
78 :
79 0 : if (dmabuf->resv == (struct dma_resv *)&dmabuf[1])
80 0 : dma_resv_fini(dmabuf->resv);
81 :
82 0 : WARN_ON(!list_empty(&dmabuf->attachments));
83 0 : module_put(dmabuf->owner);
84 0 : kfree(dmabuf->name);
85 0 : kfree(dmabuf);
86 : }
87 :
88 0 : static int dma_buf_file_release(struct inode *inode, struct file *file)
89 : {
90 : struct dma_buf *dmabuf;
91 :
92 0 : if (!is_dma_buf_file(file))
93 : return -EINVAL;
94 :
95 0 : dmabuf = file->private_data;
96 :
97 0 : mutex_lock(&db_list.lock);
98 0 : list_del(&dmabuf->list_node);
99 0 : mutex_unlock(&db_list.lock);
100 :
101 0 : return 0;
102 : }
103 :
104 : static const struct dentry_operations dma_buf_dentry_ops = {
105 : .d_dname = dmabuffs_dname,
106 : .d_release = dma_buf_release,
107 : };
108 :
109 : static struct vfsmount *dma_buf_mnt;
110 :
111 1 : static int dma_buf_fs_init_context(struct fs_context *fc)
112 : {
113 : struct pseudo_fs_context *ctx;
114 :
115 1 : ctx = init_pseudo(fc, DMA_BUF_MAGIC);
116 1 : if (!ctx)
117 : return -ENOMEM;
118 1 : ctx->dops = &dma_buf_dentry_ops;
119 1 : return 0;
120 : }
121 :
122 : static struct file_system_type dma_buf_fs_type = {
123 : .name = "dmabuf",
124 : .init_fs_context = dma_buf_fs_init_context,
125 : .kill_sb = kill_anon_super,
126 : };
127 :
128 0 : static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma)
129 : {
130 : struct dma_buf *dmabuf;
131 :
132 0 : if (!is_dma_buf_file(file))
133 : return -EINVAL;
134 :
135 0 : dmabuf = file->private_data;
136 :
137 : /* check if buffer supports mmap */
138 0 : if (!dmabuf->ops->mmap)
139 : return -EINVAL;
140 :
141 : /* check for overflowing the buffer's size */
142 0 : if (vma->vm_pgoff + vma_pages(vma) >
143 0 : dmabuf->size >> PAGE_SHIFT)
144 : return -EINVAL;
145 :
146 0 : return dmabuf->ops->mmap(dmabuf, vma);
147 : }
148 :
149 0 : static loff_t dma_buf_llseek(struct file *file, loff_t offset, int whence)
150 : {
151 : struct dma_buf *dmabuf;
152 : loff_t base;
153 :
154 0 : if (!is_dma_buf_file(file))
155 : return -EBADF;
156 :
157 0 : dmabuf = file->private_data;
158 :
159 : /* only support discovering the end of the buffer,
160 : but also allow SEEK_SET to maintain the idiomatic
161 : SEEK_END(0), SEEK_CUR(0) pattern */
162 0 : if (whence == SEEK_END)
163 0 : base = dmabuf->size;
164 0 : else if (whence == SEEK_SET)
165 : base = 0;
166 : else
167 : return -EINVAL;
168 :
169 0 : if (offset != 0)
170 : return -EINVAL;
171 :
172 0 : return base + offset;
173 : }
174 :
175 : /**
176 : * DOC: implicit fence polling
177 : *
178 : * To support cross-device and cross-driver synchronization of buffer access
179 : * implicit fences (represented internally in the kernel with &struct dma_fence)
180 : * can be attached to a &dma_buf. The glue for that and a few related things are
181 : * provided in the &dma_resv structure.
182 : *
183 : * Userspace can query the state of these implicitly tracked fences using poll()
184 : * and related system calls:
185 : *
186 : * - Checking for EPOLLIN, i.e. read access, can be use to query the state of the
187 : * most recent write or exclusive fence.
188 : *
189 : * - Checking for EPOLLOUT, i.e. write access, can be used to query the state of
190 : * all attached fences, shared and exclusive ones.
191 : *
192 : * Note that this only signals the completion of the respective fences, i.e. the
193 : * DMA transfers are complete. Cache flushing and any other necessary
194 : * preparations before CPU access can begin still need to happen.
195 : */
196 :
197 0 : static void dma_buf_poll_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
198 : {
199 0 : struct dma_buf_poll_cb_t *dcb = (struct dma_buf_poll_cb_t *)cb;
200 0 : struct dma_buf *dmabuf = container_of(dcb->poll, struct dma_buf, poll);
201 : unsigned long flags;
202 :
203 0 : spin_lock_irqsave(&dcb->poll->lock, flags);
204 0 : wake_up_locked_poll(dcb->poll, dcb->active);
205 0 : dcb->active = 0;
206 0 : spin_unlock_irqrestore(&dcb->poll->lock, flags);
207 0 : dma_fence_put(fence);
208 : /* Paired with get_file in dma_buf_poll */
209 0 : fput(dmabuf->file);
210 0 : }
211 :
212 0 : static bool dma_buf_poll_add_cb(struct dma_resv *resv, bool write,
213 : struct dma_buf_poll_cb_t *dcb)
214 : {
215 : struct dma_resv_iter cursor;
216 : struct dma_fence *fence;
217 : int r;
218 :
219 0 : dma_resv_for_each_fence(&cursor, resv, dma_resv_usage_rw(write),
220 : fence) {
221 0 : dma_fence_get(fence);
222 0 : r = dma_fence_add_callback(fence, &dcb->cb, dma_buf_poll_cb);
223 0 : if (!r)
224 : return true;
225 0 : dma_fence_put(fence);
226 : }
227 :
228 : return false;
229 : }
230 :
231 0 : static __poll_t dma_buf_poll(struct file *file, poll_table *poll)
232 : {
233 : struct dma_buf *dmabuf;
234 : struct dma_resv *resv;
235 : __poll_t events;
236 :
237 0 : dmabuf = file->private_data;
238 0 : if (!dmabuf || !dmabuf->resv)
239 : return EPOLLERR;
240 :
241 0 : resv = dmabuf->resv;
242 :
243 0 : poll_wait(file, &dmabuf->poll, poll);
244 :
245 0 : events = poll_requested_events(poll) & (EPOLLIN | EPOLLOUT);
246 0 : if (!events)
247 : return 0;
248 :
249 0 : dma_resv_lock(resv, NULL);
250 :
251 0 : if (events & EPOLLOUT) {
252 0 : struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_out;
253 :
254 : /* Check that callback isn't busy */
255 0 : spin_lock_irq(&dmabuf->poll.lock);
256 0 : if (dcb->active)
257 0 : events &= ~EPOLLOUT;
258 : else
259 0 : dcb->active = EPOLLOUT;
260 0 : spin_unlock_irq(&dmabuf->poll.lock);
261 :
262 0 : if (events & EPOLLOUT) {
263 : /* Paired with fput in dma_buf_poll_cb */
264 0 : get_file(dmabuf->file);
265 :
266 0 : if (!dma_buf_poll_add_cb(resv, true, dcb))
267 : /* No callback queued, wake up any other waiters */
268 0 : dma_buf_poll_cb(NULL, &dcb->cb);
269 : else
270 0 : events &= ~EPOLLOUT;
271 : }
272 : }
273 :
274 0 : if (events & EPOLLIN) {
275 0 : struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_in;
276 :
277 : /* Check that callback isn't busy */
278 0 : spin_lock_irq(&dmabuf->poll.lock);
279 0 : if (dcb->active)
280 0 : events &= ~EPOLLIN;
281 : else
282 0 : dcb->active = EPOLLIN;
283 0 : spin_unlock_irq(&dmabuf->poll.lock);
284 :
285 0 : if (events & EPOLLIN) {
286 : /* Paired with fput in dma_buf_poll_cb */
287 0 : get_file(dmabuf->file);
288 :
289 0 : if (!dma_buf_poll_add_cb(resv, false, dcb))
290 : /* No callback queued, wake up any other waiters */
291 0 : dma_buf_poll_cb(NULL, &dcb->cb);
292 : else
293 0 : events &= ~EPOLLIN;
294 : }
295 : }
296 :
297 0 : dma_resv_unlock(resv);
298 0 : return events;
299 : }
300 :
301 : /**
302 : * dma_buf_set_name - Set a name to a specific dma_buf to track the usage.
303 : * It could support changing the name of the dma-buf if the same
304 : * piece of memory is used for multiple purpose between different devices.
305 : *
306 : * @dmabuf: [in] dmabuf buffer that will be renamed.
307 : * @buf: [in] A piece of userspace memory that contains the name of
308 : * the dma-buf.
309 : *
310 : * Returns 0 on success. If the dma-buf buffer is already attached to
311 : * devices, return -EBUSY.
312 : *
313 : */
314 0 : static long dma_buf_set_name(struct dma_buf *dmabuf, const char __user *buf)
315 : {
316 0 : char *name = strndup_user(buf, DMA_BUF_NAME_LEN);
317 :
318 0 : if (IS_ERR(name))
319 0 : return PTR_ERR(name);
320 :
321 0 : spin_lock(&dmabuf->name_lock);
322 0 : kfree(dmabuf->name);
323 0 : dmabuf->name = name;
324 0 : spin_unlock(&dmabuf->name_lock);
325 :
326 : return 0;
327 : }
328 :
329 0 : static long dma_buf_ioctl(struct file *file,
330 : unsigned int cmd, unsigned long arg)
331 : {
332 : struct dma_buf *dmabuf;
333 : struct dma_buf_sync sync;
334 : enum dma_data_direction direction;
335 : int ret;
336 :
337 0 : dmabuf = file->private_data;
338 :
339 0 : switch (cmd) {
340 : case DMA_BUF_IOCTL_SYNC:
341 0 : if (copy_from_user(&sync, (void __user *) arg, sizeof(sync)))
342 : return -EFAULT;
343 :
344 0 : if (sync.flags & ~DMA_BUF_SYNC_VALID_FLAGS_MASK)
345 : return -EINVAL;
346 :
347 0 : switch (sync.flags & DMA_BUF_SYNC_RW) {
348 : case DMA_BUF_SYNC_READ:
349 : direction = DMA_FROM_DEVICE;
350 : break;
351 : case DMA_BUF_SYNC_WRITE:
352 0 : direction = DMA_TO_DEVICE;
353 0 : break;
354 : case DMA_BUF_SYNC_RW:
355 0 : direction = DMA_BIDIRECTIONAL;
356 0 : break;
357 : default:
358 : return -EINVAL;
359 : }
360 :
361 0 : if (sync.flags & DMA_BUF_SYNC_END)
362 0 : ret = dma_buf_end_cpu_access(dmabuf, direction);
363 : else
364 0 : ret = dma_buf_begin_cpu_access(dmabuf, direction);
365 :
366 0 : return ret;
367 :
368 : case DMA_BUF_SET_NAME_A:
369 : case DMA_BUF_SET_NAME_B:
370 0 : return dma_buf_set_name(dmabuf, (const char __user *)arg);
371 :
372 : default:
373 : return -ENOTTY;
374 : }
375 : }
376 :
377 0 : static void dma_buf_show_fdinfo(struct seq_file *m, struct file *file)
378 : {
379 0 : struct dma_buf *dmabuf = file->private_data;
380 :
381 0 : seq_printf(m, "size:\t%zu\n", dmabuf->size);
382 : /* Don't count the temporary reference taken inside procfs seq_show */
383 0 : seq_printf(m, "count:\t%ld\n", file_count(dmabuf->file) - 1);
384 0 : seq_printf(m, "exp_name:\t%s\n", dmabuf->exp_name);
385 0 : spin_lock(&dmabuf->name_lock);
386 0 : if (dmabuf->name)
387 0 : seq_printf(m, "name:\t%s\n", dmabuf->name);
388 0 : spin_unlock(&dmabuf->name_lock);
389 0 : }
390 :
391 : static const struct file_operations dma_buf_fops = {
392 : .release = dma_buf_file_release,
393 : .mmap = dma_buf_mmap_internal,
394 : .llseek = dma_buf_llseek,
395 : .poll = dma_buf_poll,
396 : .unlocked_ioctl = dma_buf_ioctl,
397 : .compat_ioctl = compat_ptr_ioctl,
398 : .show_fdinfo = dma_buf_show_fdinfo,
399 : };
400 :
401 : /*
402 : * is_dma_buf_file - Check if struct file* is associated with dma_buf
403 : */
404 : static inline int is_dma_buf_file(struct file *file)
405 : {
406 : return file->f_op == &dma_buf_fops;
407 : }
408 :
409 0 : static struct file *dma_buf_getfile(struct dma_buf *dmabuf, int flags)
410 : {
411 : static atomic64_t dmabuf_inode = ATOMIC64_INIT(0);
412 : struct file *file;
413 0 : struct inode *inode = alloc_anon_inode(dma_buf_mnt->mnt_sb);
414 :
415 0 : if (IS_ERR(inode))
416 : return ERR_CAST(inode);
417 :
418 0 : inode->i_size = dmabuf->size;
419 0 : inode_set_bytes(inode, dmabuf->size);
420 :
421 : /*
422 : * The ->i_ino acquired from get_next_ino() is not unique thus
423 : * not suitable for using it as dentry name by dmabuf stats.
424 : * Override ->i_ino with the unique and dmabuffs specific
425 : * value.
426 : */
427 0 : inode->i_ino = atomic64_add_return(1, &dmabuf_inode);
428 0 : file = alloc_file_pseudo(inode, dma_buf_mnt, "dmabuf",
429 : flags, &dma_buf_fops);
430 0 : if (IS_ERR(file))
431 : goto err_alloc_file;
432 0 : file->f_flags = flags & (O_ACCMODE | O_NONBLOCK);
433 0 : file->private_data = dmabuf;
434 0 : file->f_path.dentry->d_fsdata = dmabuf;
435 :
436 0 : return file;
437 :
438 : err_alloc_file:
439 0 : iput(inode);
440 0 : return file;
441 : }
442 :
443 : /**
444 : * DOC: dma buf device access
445 : *
446 : * For device DMA access to a shared DMA buffer the usual sequence of operations
447 : * is fairly simple:
448 : *
449 : * 1. The exporter defines his exporter instance using
450 : * DEFINE_DMA_BUF_EXPORT_INFO() and calls dma_buf_export() to wrap a private
451 : * buffer object into a &dma_buf. It then exports that &dma_buf to userspace
452 : * as a file descriptor by calling dma_buf_fd().
453 : *
454 : * 2. Userspace passes this file-descriptors to all drivers it wants this buffer
455 : * to share with: First the file descriptor is converted to a &dma_buf using
456 : * dma_buf_get(). Then the buffer is attached to the device using
457 : * dma_buf_attach().
458 : *
459 : * Up to this stage the exporter is still free to migrate or reallocate the
460 : * backing storage.
461 : *
462 : * 3. Once the buffer is attached to all devices userspace can initiate DMA
463 : * access to the shared buffer. In the kernel this is done by calling
464 : * dma_buf_map_attachment() and dma_buf_unmap_attachment().
465 : *
466 : * 4. Once a driver is done with a shared buffer it needs to call
467 : * dma_buf_detach() (after cleaning up any mappings) and then release the
468 : * reference acquired with dma_buf_get() by calling dma_buf_put().
469 : *
470 : * For the detailed semantics exporters are expected to implement see
471 : * &dma_buf_ops.
472 : */
473 :
474 : /**
475 : * dma_buf_export - Creates a new dma_buf, and associates an anon file
476 : * with this buffer, so it can be exported.
477 : * Also connect the allocator specific data and ops to the buffer.
478 : * Additionally, provide a name string for exporter; useful in debugging.
479 : *
480 : * @exp_info: [in] holds all the export related information provided
481 : * by the exporter. see &struct dma_buf_export_info
482 : * for further details.
483 : *
484 : * Returns, on success, a newly created struct dma_buf object, which wraps the
485 : * supplied private data and operations for struct dma_buf_ops. On either
486 : * missing ops, or error in allocating struct dma_buf, will return negative
487 : * error.
488 : *
489 : * For most cases the easiest way to create @exp_info is through the
490 : * %DEFINE_DMA_BUF_EXPORT_INFO macro.
491 : */
492 0 : struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info)
493 : {
494 : struct dma_buf *dmabuf;
495 0 : struct dma_resv *resv = exp_info->resv;
496 : struct file *file;
497 0 : size_t alloc_size = sizeof(struct dma_buf);
498 : int ret;
499 :
500 0 : if (!exp_info->resv)
501 : alloc_size += sizeof(struct dma_resv);
502 : else
503 : /* prevent &dma_buf[1] == dma_buf->resv */
504 0 : alloc_size += 1;
505 :
506 0 : if (WARN_ON(!exp_info->priv
507 : || !exp_info->ops
508 : || !exp_info->ops->map_dma_buf
509 : || !exp_info->ops->unmap_dma_buf
510 : || !exp_info->ops->release)) {
511 : return ERR_PTR(-EINVAL);
512 : }
513 :
514 0 : if (WARN_ON(exp_info->ops->cache_sgt_mapping &&
515 : (exp_info->ops->pin || exp_info->ops->unpin)))
516 : return ERR_PTR(-EINVAL);
517 :
518 0 : if (WARN_ON(!exp_info->ops->pin != !exp_info->ops->unpin))
519 : return ERR_PTR(-EINVAL);
520 :
521 0 : if (!try_module_get(exp_info->owner))
522 : return ERR_PTR(-ENOENT);
523 :
524 0 : dmabuf = kzalloc(alloc_size, GFP_KERNEL);
525 0 : if (!dmabuf) {
526 : ret = -ENOMEM;
527 : goto err_module;
528 : }
529 :
530 0 : dmabuf->priv = exp_info->priv;
531 0 : dmabuf->ops = exp_info->ops;
532 0 : dmabuf->size = exp_info->size;
533 0 : dmabuf->exp_name = exp_info->exp_name;
534 0 : dmabuf->owner = exp_info->owner;
535 0 : spin_lock_init(&dmabuf->name_lock);
536 0 : init_waitqueue_head(&dmabuf->poll);
537 0 : dmabuf->cb_in.poll = dmabuf->cb_out.poll = &dmabuf->poll;
538 0 : dmabuf->cb_in.active = dmabuf->cb_out.active = 0;
539 :
540 0 : if (!resv) {
541 0 : resv = (struct dma_resv *)&dmabuf[1];
542 0 : dma_resv_init(resv);
543 : }
544 0 : dmabuf->resv = resv;
545 :
546 0 : file = dma_buf_getfile(dmabuf, exp_info->flags);
547 0 : if (IS_ERR(file)) {
548 0 : ret = PTR_ERR(file);
549 : goto err_dmabuf;
550 : }
551 :
552 0 : file->f_mode |= FMODE_LSEEK;
553 0 : dmabuf->file = file;
554 :
555 0 : mutex_init(&dmabuf->lock);
556 0 : INIT_LIST_HEAD(&dmabuf->attachments);
557 :
558 0 : mutex_lock(&db_list.lock);
559 0 : list_add(&dmabuf->list_node, &db_list.head);
560 0 : mutex_unlock(&db_list.lock);
561 :
562 0 : ret = dma_buf_stats_setup(dmabuf);
563 : if (ret)
564 : goto err_sysfs;
565 :
566 0 : return dmabuf;
567 :
568 : err_sysfs:
569 : /*
570 : * Set file->f_path.dentry->d_fsdata to NULL so that when
571 : * dma_buf_release() gets invoked by dentry_ops, it exits
572 : * early before calling the release() dma_buf op.
573 : */
574 : file->f_path.dentry->d_fsdata = NULL;
575 : fput(file);
576 : err_dmabuf:
577 0 : kfree(dmabuf);
578 : err_module:
579 0 : module_put(exp_info->owner);
580 0 : return ERR_PTR(ret);
581 : }
582 : EXPORT_SYMBOL_NS_GPL(dma_buf_export, DMA_BUF);
583 :
584 : /**
585 : * dma_buf_fd - returns a file descriptor for the given struct dma_buf
586 : * @dmabuf: [in] pointer to dma_buf for which fd is required.
587 : * @flags: [in] flags to give to fd
588 : *
589 : * On success, returns an associated 'fd'. Else, returns error.
590 : */
591 0 : int dma_buf_fd(struct dma_buf *dmabuf, int flags)
592 : {
593 : int fd;
594 :
595 0 : if (!dmabuf || !dmabuf->file)
596 : return -EINVAL;
597 :
598 0 : fd = get_unused_fd_flags(flags);
599 0 : if (fd < 0)
600 : return fd;
601 :
602 0 : fd_install(fd, dmabuf->file);
603 :
604 0 : return fd;
605 : }
606 : EXPORT_SYMBOL_NS_GPL(dma_buf_fd, DMA_BUF);
607 :
608 : /**
609 : * dma_buf_get - returns the struct dma_buf related to an fd
610 : * @fd: [in] fd associated with the struct dma_buf to be returned
611 : *
612 : * On success, returns the struct dma_buf associated with an fd; uses
613 : * file's refcounting done by fget to increase refcount. returns ERR_PTR
614 : * otherwise.
615 : */
616 0 : struct dma_buf *dma_buf_get(int fd)
617 : {
618 : struct file *file;
619 :
620 0 : file = fget(fd);
621 :
622 0 : if (!file)
623 : return ERR_PTR(-EBADF);
624 :
625 0 : if (!is_dma_buf_file(file)) {
626 0 : fput(file);
627 0 : return ERR_PTR(-EINVAL);
628 : }
629 :
630 0 : return file->private_data;
631 : }
632 : EXPORT_SYMBOL_NS_GPL(dma_buf_get, DMA_BUF);
633 :
634 : /**
635 : * dma_buf_put - decreases refcount of the buffer
636 : * @dmabuf: [in] buffer to reduce refcount of
637 : *
638 : * Uses file's refcounting done implicitly by fput().
639 : *
640 : * If, as a result of this call, the refcount becomes 0, the 'release' file
641 : * operation related to this fd is called. It calls &dma_buf_ops.release vfunc
642 : * in turn, and frees the memory allocated for dmabuf when exported.
643 : */
644 0 : void dma_buf_put(struct dma_buf *dmabuf)
645 : {
646 0 : if (WARN_ON(!dmabuf || !dmabuf->file))
647 : return;
648 :
649 0 : fput(dmabuf->file);
650 : }
651 : EXPORT_SYMBOL_NS_GPL(dma_buf_put, DMA_BUF);
652 :
653 : static void mangle_sg_table(struct sg_table *sg_table)
654 : {
655 : #ifdef CONFIG_DMABUF_DEBUG
656 : int i;
657 : struct scatterlist *sg;
658 :
659 : /* To catch abuse of the underlying struct page by importers mix
660 : * up the bits, but take care to preserve the low SG_ bits to
661 : * not corrupt the sgt. The mixing is undone in __unmap_dma_buf
662 : * before passing the sgt back to the exporter. */
663 : for_each_sgtable_sg(sg_table, sg, i)
664 : sg->page_link ^= ~0xffUL;
665 : #endif
666 :
667 : }
668 0 : static struct sg_table * __map_dma_buf(struct dma_buf_attachment *attach,
669 : enum dma_data_direction direction)
670 : {
671 : struct sg_table *sg_table;
672 : signed long ret;
673 :
674 0 : sg_table = attach->dmabuf->ops->map_dma_buf(attach, direction);
675 0 : if (IS_ERR_OR_NULL(sg_table))
676 : return sg_table;
677 :
678 0 : if (!dma_buf_attachment_is_dynamic(attach)) {
679 0 : ret = dma_resv_wait_timeout(attach->dmabuf->resv,
680 : DMA_RESV_USAGE_KERNEL, true,
681 : MAX_SCHEDULE_TIMEOUT);
682 0 : if (ret < 0) {
683 0 : attach->dmabuf->ops->unmap_dma_buf(attach, sg_table,
684 : direction);
685 0 : return ERR_PTR(ret);
686 : }
687 : }
688 :
689 : mangle_sg_table(sg_table);
690 : return sg_table;
691 : }
692 :
693 : /**
694 : * dma_buf_dynamic_attach - Add the device to dma_buf's attachments list
695 : * @dmabuf: [in] buffer to attach device to.
696 : * @dev: [in] device to be attached.
697 : * @importer_ops: [in] importer operations for the attachment
698 : * @importer_priv: [in] importer private pointer for the attachment
699 : *
700 : * Returns struct dma_buf_attachment pointer for this attachment. Attachments
701 : * must be cleaned up by calling dma_buf_detach().
702 : *
703 : * Optionally this calls &dma_buf_ops.attach to allow device-specific attach
704 : * functionality.
705 : *
706 : * Returns:
707 : *
708 : * A pointer to newly created &dma_buf_attachment on success, or a negative
709 : * error code wrapped into a pointer on failure.
710 : *
711 : * Note that this can fail if the backing storage of @dmabuf is in a place not
712 : * accessible to @dev, and cannot be moved to a more suitable place. This is
713 : * indicated with the error code -EBUSY.
714 : */
715 : struct dma_buf_attachment *
716 0 : dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev,
717 : const struct dma_buf_attach_ops *importer_ops,
718 : void *importer_priv)
719 : {
720 : struct dma_buf_attachment *attach;
721 : int ret;
722 :
723 0 : if (WARN_ON(!dmabuf || !dev))
724 : return ERR_PTR(-EINVAL);
725 :
726 0 : if (WARN_ON(importer_ops && !importer_ops->move_notify))
727 : return ERR_PTR(-EINVAL);
728 :
729 0 : attach = kzalloc(sizeof(*attach), GFP_KERNEL);
730 0 : if (!attach)
731 : return ERR_PTR(-ENOMEM);
732 :
733 0 : attach->dev = dev;
734 0 : attach->dmabuf = dmabuf;
735 0 : if (importer_ops)
736 0 : attach->peer2peer = importer_ops->allow_peer2peer;
737 0 : attach->importer_ops = importer_ops;
738 0 : attach->importer_priv = importer_priv;
739 :
740 0 : if (dmabuf->ops->attach) {
741 0 : ret = dmabuf->ops->attach(dmabuf, attach);
742 0 : if (ret)
743 : goto err_attach;
744 : }
745 0 : dma_resv_lock(dmabuf->resv, NULL);
746 0 : list_add(&attach->node, &dmabuf->attachments);
747 0 : dma_resv_unlock(dmabuf->resv);
748 :
749 : /* When either the importer or the exporter can't handle dynamic
750 : * mappings we cache the mapping here to avoid issues with the
751 : * reservation object lock.
752 : */
753 0 : if (dma_buf_attachment_is_dynamic(attach) !=
754 0 : dma_buf_is_dynamic(dmabuf)) {
755 : struct sg_table *sgt;
756 :
757 0 : if (dma_buf_is_dynamic(attach->dmabuf)) {
758 0 : dma_resv_lock(attach->dmabuf->resv, NULL);
759 0 : ret = dmabuf->ops->pin(attach);
760 0 : if (ret)
761 : goto err_unlock;
762 : }
763 :
764 0 : sgt = __map_dma_buf(attach, DMA_BIDIRECTIONAL);
765 0 : if (!sgt)
766 0 : sgt = ERR_PTR(-ENOMEM);
767 0 : if (IS_ERR(sgt)) {
768 0 : ret = PTR_ERR(sgt);
769 : goto err_unpin;
770 : }
771 0 : if (dma_buf_is_dynamic(attach->dmabuf))
772 0 : dma_resv_unlock(attach->dmabuf->resv);
773 0 : attach->sgt = sgt;
774 0 : attach->dir = DMA_BIDIRECTIONAL;
775 : }
776 :
777 : return attach;
778 :
779 : err_attach:
780 0 : kfree(attach);
781 0 : return ERR_PTR(ret);
782 :
783 : err_unpin:
784 0 : if (dma_buf_is_dynamic(attach->dmabuf))
785 0 : dmabuf->ops->unpin(attach);
786 :
787 : err_unlock:
788 0 : if (dma_buf_is_dynamic(attach->dmabuf))
789 0 : dma_resv_unlock(attach->dmabuf->resv);
790 :
791 0 : dma_buf_detach(dmabuf, attach);
792 0 : return ERR_PTR(ret);
793 : }
794 : EXPORT_SYMBOL_NS_GPL(dma_buf_dynamic_attach, DMA_BUF);
795 :
796 : /**
797 : * dma_buf_attach - Wrapper for dma_buf_dynamic_attach
798 : * @dmabuf: [in] buffer to attach device to.
799 : * @dev: [in] device to be attached.
800 : *
801 : * Wrapper to call dma_buf_dynamic_attach() for drivers which still use a static
802 : * mapping.
803 : */
804 0 : struct dma_buf_attachment *dma_buf_attach(struct dma_buf *dmabuf,
805 : struct device *dev)
806 : {
807 0 : return dma_buf_dynamic_attach(dmabuf, dev, NULL, NULL);
808 : }
809 : EXPORT_SYMBOL_NS_GPL(dma_buf_attach, DMA_BUF);
810 :
811 : static void __unmap_dma_buf(struct dma_buf_attachment *attach,
812 : struct sg_table *sg_table,
813 : enum dma_data_direction direction)
814 : {
815 : /* uses XOR, hence this unmangles */
816 0 : mangle_sg_table(sg_table);
817 :
818 0 : attach->dmabuf->ops->unmap_dma_buf(attach, sg_table, direction);
819 : }
820 :
821 : /**
822 : * dma_buf_detach - Remove the given attachment from dmabuf's attachments list
823 : * @dmabuf: [in] buffer to detach from.
824 : * @attach: [in] attachment to be detached; is free'd after this call.
825 : *
826 : * Clean up a device attachment obtained by calling dma_buf_attach().
827 : *
828 : * Optionally this calls &dma_buf_ops.detach for device-specific detach.
829 : */
830 0 : void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach)
831 : {
832 0 : if (WARN_ON(!dmabuf || !attach))
833 : return;
834 :
835 0 : if (attach->sgt) {
836 0 : if (dma_buf_is_dynamic(attach->dmabuf))
837 0 : dma_resv_lock(attach->dmabuf->resv, NULL);
838 :
839 0 : __unmap_dma_buf(attach, attach->sgt, attach->dir);
840 :
841 0 : if (dma_buf_is_dynamic(attach->dmabuf)) {
842 0 : dmabuf->ops->unpin(attach);
843 0 : dma_resv_unlock(attach->dmabuf->resv);
844 : }
845 : }
846 :
847 0 : dma_resv_lock(dmabuf->resv, NULL);
848 0 : list_del(&attach->node);
849 0 : dma_resv_unlock(dmabuf->resv);
850 0 : if (dmabuf->ops->detach)
851 0 : dmabuf->ops->detach(dmabuf, attach);
852 :
853 0 : kfree(attach);
854 : }
855 : EXPORT_SYMBOL_NS_GPL(dma_buf_detach, DMA_BUF);
856 :
857 : /**
858 : * dma_buf_pin - Lock down the DMA-buf
859 : * @attach: [in] attachment which should be pinned
860 : *
861 : * Only dynamic importers (who set up @attach with dma_buf_dynamic_attach()) may
862 : * call this, and only for limited use cases like scanout and not for temporary
863 : * pin operations. It is not permitted to allow userspace to pin arbitrary
864 : * amounts of buffers through this interface.
865 : *
866 : * Buffers must be unpinned by calling dma_buf_unpin().
867 : *
868 : * Returns:
869 : * 0 on success, negative error code on failure.
870 : */
871 0 : int dma_buf_pin(struct dma_buf_attachment *attach)
872 : {
873 0 : struct dma_buf *dmabuf = attach->dmabuf;
874 0 : int ret = 0;
875 :
876 0 : WARN_ON(!dma_buf_attachment_is_dynamic(attach));
877 :
878 : dma_resv_assert_held(dmabuf->resv);
879 :
880 0 : if (dmabuf->ops->pin)
881 0 : ret = dmabuf->ops->pin(attach);
882 :
883 0 : return ret;
884 : }
885 : EXPORT_SYMBOL_NS_GPL(dma_buf_pin, DMA_BUF);
886 :
887 : /**
888 : * dma_buf_unpin - Unpin a DMA-buf
889 : * @attach: [in] attachment which should be unpinned
890 : *
891 : * This unpins a buffer pinned by dma_buf_pin() and allows the exporter to move
892 : * any mapping of @attach again and inform the importer through
893 : * &dma_buf_attach_ops.move_notify.
894 : */
895 0 : void dma_buf_unpin(struct dma_buf_attachment *attach)
896 : {
897 0 : struct dma_buf *dmabuf = attach->dmabuf;
898 :
899 0 : WARN_ON(!dma_buf_attachment_is_dynamic(attach));
900 :
901 : dma_resv_assert_held(dmabuf->resv);
902 :
903 0 : if (dmabuf->ops->unpin)
904 0 : dmabuf->ops->unpin(attach);
905 0 : }
906 : EXPORT_SYMBOL_NS_GPL(dma_buf_unpin, DMA_BUF);
907 :
908 : /**
909 : * dma_buf_map_attachment - Returns the scatterlist table of the attachment;
910 : * mapped into _device_ address space. Is a wrapper for map_dma_buf() of the
911 : * dma_buf_ops.
912 : * @attach: [in] attachment whose scatterlist is to be returned
913 : * @direction: [in] direction of DMA transfer
914 : *
915 : * Returns sg_table containing the scatterlist to be returned; returns ERR_PTR
916 : * on error. May return -EINTR if it is interrupted by a signal.
917 : *
918 : * On success, the DMA addresses and lengths in the returned scatterlist are
919 : * PAGE_SIZE aligned.
920 : *
921 : * A mapping must be unmapped by using dma_buf_unmap_attachment(). Note that
922 : * the underlying backing storage is pinned for as long as a mapping exists,
923 : * therefore users/importers should not hold onto a mapping for undue amounts of
924 : * time.
925 : *
926 : * Important: Dynamic importers must wait for the exclusive fence of the struct
927 : * dma_resv attached to the DMA-BUF first.
928 : */
929 0 : struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *attach,
930 : enum dma_data_direction direction)
931 : {
932 : struct sg_table *sg_table;
933 : int r;
934 :
935 : might_sleep();
936 :
937 0 : if (WARN_ON(!attach || !attach->dmabuf))
938 : return ERR_PTR(-EINVAL);
939 :
940 0 : if (dma_buf_attachment_is_dynamic(attach))
941 : dma_resv_assert_held(attach->dmabuf->resv);
942 :
943 0 : if (attach->sgt) {
944 : /*
945 : * Two mappings with different directions for the same
946 : * attachment are not allowed.
947 : */
948 0 : if (attach->dir != direction &&
949 : attach->dir != DMA_BIDIRECTIONAL)
950 : return ERR_PTR(-EBUSY);
951 :
952 0 : return attach->sgt;
953 : }
954 :
955 0 : if (dma_buf_is_dynamic(attach->dmabuf)) {
956 : dma_resv_assert_held(attach->dmabuf->resv);
957 : if (!IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY)) {
958 0 : r = attach->dmabuf->ops->pin(attach);
959 0 : if (r)
960 0 : return ERR_PTR(r);
961 : }
962 : }
963 :
964 0 : sg_table = __map_dma_buf(attach, direction);
965 0 : if (!sg_table)
966 0 : sg_table = ERR_PTR(-ENOMEM);
967 :
968 0 : if (IS_ERR(sg_table) && dma_buf_is_dynamic(attach->dmabuf) &&
969 : !IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY))
970 0 : attach->dmabuf->ops->unpin(attach);
971 :
972 0 : if (!IS_ERR(sg_table) && attach->dmabuf->ops->cache_sgt_mapping) {
973 0 : attach->sgt = sg_table;
974 0 : attach->dir = direction;
975 : }
976 :
977 : #ifdef CONFIG_DMA_API_DEBUG
978 : if (!IS_ERR(sg_table)) {
979 : struct scatterlist *sg;
980 : u64 addr;
981 : int len;
982 : int i;
983 :
984 : for_each_sgtable_dma_sg(sg_table, sg, i) {
985 : addr = sg_dma_address(sg);
986 : len = sg_dma_len(sg);
987 : if (!PAGE_ALIGNED(addr) || !PAGE_ALIGNED(len)) {
988 : pr_debug("%s: addr %llx or len %x is not page aligned!\n",
989 : __func__, addr, len);
990 : }
991 : }
992 : }
993 : #endif /* CONFIG_DMA_API_DEBUG */
994 : return sg_table;
995 : }
996 : EXPORT_SYMBOL_NS_GPL(dma_buf_map_attachment, DMA_BUF);
997 :
998 : /**
999 : * dma_buf_unmap_attachment - unmaps and decreases usecount of the buffer;might
1000 : * deallocate the scatterlist associated. Is a wrapper for unmap_dma_buf() of
1001 : * dma_buf_ops.
1002 : * @attach: [in] attachment to unmap buffer from
1003 : * @sg_table: [in] scatterlist info of the buffer to unmap
1004 : * @direction: [in] direction of DMA transfer
1005 : *
1006 : * This unmaps a DMA mapping for @attached obtained by dma_buf_map_attachment().
1007 : */
1008 0 : void dma_buf_unmap_attachment(struct dma_buf_attachment *attach,
1009 : struct sg_table *sg_table,
1010 : enum dma_data_direction direction)
1011 : {
1012 : might_sleep();
1013 :
1014 0 : if (WARN_ON(!attach || !attach->dmabuf || !sg_table))
1015 : return;
1016 :
1017 0 : if (dma_buf_attachment_is_dynamic(attach))
1018 : dma_resv_assert_held(attach->dmabuf->resv);
1019 :
1020 0 : if (attach->sgt == sg_table)
1021 : return;
1022 :
1023 0 : if (dma_buf_is_dynamic(attach->dmabuf))
1024 : dma_resv_assert_held(attach->dmabuf->resv);
1025 :
1026 0 : __unmap_dma_buf(attach, sg_table, direction);
1027 :
1028 0 : if (dma_buf_is_dynamic(attach->dmabuf) &&
1029 : !IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY))
1030 0 : dma_buf_unpin(attach);
1031 : }
1032 : EXPORT_SYMBOL_NS_GPL(dma_buf_unmap_attachment, DMA_BUF);
1033 :
1034 : /**
1035 : * dma_buf_move_notify - notify attachments that DMA-buf is moving
1036 : *
1037 : * @dmabuf: [in] buffer which is moving
1038 : *
1039 : * Informs all attachmenst that they need to destroy and recreated all their
1040 : * mappings.
1041 : */
1042 0 : void dma_buf_move_notify(struct dma_buf *dmabuf)
1043 : {
1044 : struct dma_buf_attachment *attach;
1045 :
1046 : dma_resv_assert_held(dmabuf->resv);
1047 :
1048 0 : list_for_each_entry(attach, &dmabuf->attachments, node)
1049 0 : if (attach->importer_ops)
1050 0 : attach->importer_ops->move_notify(attach);
1051 0 : }
1052 : EXPORT_SYMBOL_NS_GPL(dma_buf_move_notify, DMA_BUF);
1053 :
1054 : /**
1055 : * DOC: cpu access
1056 : *
1057 : * There are mutliple reasons for supporting CPU access to a dma buffer object:
1058 : *
1059 : * - Fallback operations in the kernel, for example when a device is connected
1060 : * over USB and the kernel needs to shuffle the data around first before
1061 : * sending it away. Cache coherency is handled by braketing any transactions
1062 : * with calls to dma_buf_begin_cpu_access() and dma_buf_end_cpu_access()
1063 : * access.
1064 : *
1065 : * Since for most kernel internal dma-buf accesses need the entire buffer, a
1066 : * vmap interface is introduced. Note that on very old 32-bit architectures
1067 : * vmalloc space might be limited and result in vmap calls failing.
1068 : *
1069 : * Interfaces::
1070 : *
1071 : * void \*dma_buf_vmap(struct dma_buf \*dmabuf, struct iosys_map \*map)
1072 : * void dma_buf_vunmap(struct dma_buf \*dmabuf, struct iosys_map \*map)
1073 : *
1074 : * The vmap call can fail if there is no vmap support in the exporter, or if
1075 : * it runs out of vmalloc space. Note that the dma-buf layer keeps a reference
1076 : * count for all vmap access and calls down into the exporter's vmap function
1077 : * only when no vmapping exists, and only unmaps it once. Protection against
1078 : * concurrent vmap/vunmap calls is provided by taking the &dma_buf.lock mutex.
1079 : *
1080 : * - For full compatibility on the importer side with existing userspace
1081 : * interfaces, which might already support mmap'ing buffers. This is needed in
1082 : * many processing pipelines (e.g. feeding a software rendered image into a
1083 : * hardware pipeline, thumbnail creation, snapshots, ...). Also, Android's ION
1084 : * framework already supported this and for DMA buffer file descriptors to
1085 : * replace ION buffers mmap support was needed.
1086 : *
1087 : * There is no special interfaces, userspace simply calls mmap on the dma-buf
1088 : * fd. But like for CPU access there's a need to braket the actual access,
1089 : * which is handled by the ioctl (DMA_BUF_IOCTL_SYNC). Note that
1090 : * DMA_BUF_IOCTL_SYNC can fail with -EAGAIN or -EINTR, in which case it must
1091 : * be restarted.
1092 : *
1093 : * Some systems might need some sort of cache coherency management e.g. when
1094 : * CPU and GPU domains are being accessed through dma-buf at the same time.
1095 : * To circumvent this problem there are begin/end coherency markers, that
1096 : * forward directly to existing dma-buf device drivers vfunc hooks. Userspace
1097 : * can make use of those markers through the DMA_BUF_IOCTL_SYNC ioctl. The
1098 : * sequence would be used like following:
1099 : *
1100 : * - mmap dma-buf fd
1101 : * - for each drawing/upload cycle in CPU 1. SYNC_START ioctl, 2. read/write
1102 : * to mmap area 3. SYNC_END ioctl. This can be repeated as often as you
1103 : * want (with the new data being consumed by say the GPU or the scanout
1104 : * device)
1105 : * - munmap once you don't need the buffer any more
1106 : *
1107 : * For correctness and optimal performance, it is always required to use
1108 : * SYNC_START and SYNC_END before and after, respectively, when accessing the
1109 : * mapped address. Userspace cannot rely on coherent access, even when there
1110 : * are systems where it just works without calling these ioctls.
1111 : *
1112 : * - And as a CPU fallback in userspace processing pipelines.
1113 : *
1114 : * Similar to the motivation for kernel cpu access it is again important that
1115 : * the userspace code of a given importing subsystem can use the same
1116 : * interfaces with a imported dma-buf buffer object as with a native buffer
1117 : * object. This is especially important for drm where the userspace part of
1118 : * contemporary OpenGL, X, and other drivers is huge, and reworking them to
1119 : * use a different way to mmap a buffer rather invasive.
1120 : *
1121 : * The assumption in the current dma-buf interfaces is that redirecting the
1122 : * initial mmap is all that's needed. A survey of some of the existing
1123 : * subsystems shows that no driver seems to do any nefarious thing like
1124 : * syncing up with outstanding asynchronous processing on the device or
1125 : * allocating special resources at fault time. So hopefully this is good
1126 : * enough, since adding interfaces to intercept pagefaults and allow pte
1127 : * shootdowns would increase the complexity quite a bit.
1128 : *
1129 : * Interface::
1130 : *
1131 : * int dma_buf_mmap(struct dma_buf \*, struct vm_area_struct \*,
1132 : * unsigned long);
1133 : *
1134 : * If the importing subsystem simply provides a special-purpose mmap call to
1135 : * set up a mapping in userspace, calling do_mmap with &dma_buf.file will
1136 : * equally achieve that for a dma-buf object.
1137 : */
1138 :
1139 : static int __dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
1140 : enum dma_data_direction direction)
1141 : {
1142 0 : bool write = (direction == DMA_BIDIRECTIONAL ||
1143 : direction == DMA_TO_DEVICE);
1144 0 : struct dma_resv *resv = dmabuf->resv;
1145 : long ret;
1146 :
1147 : /* Wait on any implicit rendering fences */
1148 0 : ret = dma_resv_wait_timeout(resv, dma_resv_usage_rw(write),
1149 : true, MAX_SCHEDULE_TIMEOUT);
1150 0 : if (ret < 0)
1151 0 : return ret;
1152 :
1153 : return 0;
1154 : }
1155 :
1156 : /**
1157 : * dma_buf_begin_cpu_access - Must be called before accessing a dma_buf from the
1158 : * cpu in the kernel context. Calls begin_cpu_access to allow exporter-specific
1159 : * preparations. Coherency is only guaranteed in the specified range for the
1160 : * specified access direction.
1161 : * @dmabuf: [in] buffer to prepare cpu access for.
1162 : * @direction: [in] length of range for cpu access.
1163 : *
1164 : * After the cpu access is complete the caller should call
1165 : * dma_buf_end_cpu_access(). Only when cpu access is braketed by both calls is
1166 : * it guaranteed to be coherent with other DMA access.
1167 : *
1168 : * This function will also wait for any DMA transactions tracked through
1169 : * implicit synchronization in &dma_buf.resv. For DMA transactions with explicit
1170 : * synchronization this function will only ensure cache coherency, callers must
1171 : * ensure synchronization with such DMA transactions on their own.
1172 : *
1173 : * Can return negative error values, returns 0 on success.
1174 : */
1175 0 : int dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
1176 : enum dma_data_direction direction)
1177 : {
1178 0 : int ret = 0;
1179 :
1180 0 : if (WARN_ON(!dmabuf))
1181 : return -EINVAL;
1182 :
1183 : might_lock(&dmabuf->resv->lock.base);
1184 :
1185 0 : if (dmabuf->ops->begin_cpu_access)
1186 0 : ret = dmabuf->ops->begin_cpu_access(dmabuf, direction);
1187 :
1188 : /* Ensure that all fences are waited upon - but we first allow
1189 : * the native handler the chance to do so more efficiently if it
1190 : * chooses. A double invocation here will be reasonably cheap no-op.
1191 : */
1192 0 : if (ret == 0)
1193 0 : ret = __dma_buf_begin_cpu_access(dmabuf, direction);
1194 :
1195 : return ret;
1196 : }
1197 : EXPORT_SYMBOL_NS_GPL(dma_buf_begin_cpu_access, DMA_BUF);
1198 :
1199 : /**
1200 : * dma_buf_end_cpu_access - Must be called after accessing a dma_buf from the
1201 : * cpu in the kernel context. Calls end_cpu_access to allow exporter-specific
1202 : * actions. Coherency is only guaranteed in the specified range for the
1203 : * specified access direction.
1204 : * @dmabuf: [in] buffer to complete cpu access for.
1205 : * @direction: [in] length of range for cpu access.
1206 : *
1207 : * This terminates CPU access started with dma_buf_begin_cpu_access().
1208 : *
1209 : * Can return negative error values, returns 0 on success.
1210 : */
1211 0 : int dma_buf_end_cpu_access(struct dma_buf *dmabuf,
1212 : enum dma_data_direction direction)
1213 : {
1214 0 : int ret = 0;
1215 :
1216 0 : WARN_ON(!dmabuf);
1217 :
1218 : might_lock(&dmabuf->resv->lock.base);
1219 :
1220 0 : if (dmabuf->ops->end_cpu_access)
1221 0 : ret = dmabuf->ops->end_cpu_access(dmabuf, direction);
1222 :
1223 0 : return ret;
1224 : }
1225 : EXPORT_SYMBOL_NS_GPL(dma_buf_end_cpu_access, DMA_BUF);
1226 :
1227 :
1228 : /**
1229 : * dma_buf_mmap - Setup up a userspace mmap with the given vma
1230 : * @dmabuf: [in] buffer that should back the vma
1231 : * @vma: [in] vma for the mmap
1232 : * @pgoff: [in] offset in pages where this mmap should start within the
1233 : * dma-buf buffer.
1234 : *
1235 : * This function adjusts the passed in vma so that it points at the file of the
1236 : * dma_buf operation. It also adjusts the starting pgoff and does bounds
1237 : * checking on the size of the vma. Then it calls the exporters mmap function to
1238 : * set up the mapping.
1239 : *
1240 : * Can return negative error values, returns 0 on success.
1241 : */
1242 0 : int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma,
1243 : unsigned long pgoff)
1244 : {
1245 0 : if (WARN_ON(!dmabuf || !vma))
1246 : return -EINVAL;
1247 :
1248 : /* check if buffer supports mmap */
1249 0 : if (!dmabuf->ops->mmap)
1250 : return -EINVAL;
1251 :
1252 : /* check for offset overflow */
1253 0 : if (pgoff + vma_pages(vma) < pgoff)
1254 : return -EOVERFLOW;
1255 :
1256 : /* check for overflowing the buffer's size */
1257 0 : if (pgoff + vma_pages(vma) >
1258 0 : dmabuf->size >> PAGE_SHIFT)
1259 : return -EINVAL;
1260 :
1261 : /* readjust the vma */
1262 0 : vma_set_file(vma, dmabuf->file);
1263 0 : vma->vm_pgoff = pgoff;
1264 :
1265 0 : return dmabuf->ops->mmap(dmabuf, vma);
1266 : }
1267 : EXPORT_SYMBOL_NS_GPL(dma_buf_mmap, DMA_BUF);
1268 :
1269 : /**
1270 : * dma_buf_vmap - Create virtual mapping for the buffer object into kernel
1271 : * address space. Same restrictions as for vmap and friends apply.
1272 : * @dmabuf: [in] buffer to vmap
1273 : * @map: [out] returns the vmap pointer
1274 : *
1275 : * This call may fail due to lack of virtual mapping address space.
1276 : * These calls are optional in drivers. The intended use for them
1277 : * is for mapping objects linear in kernel space for high use objects.
1278 : *
1279 : * To ensure coherency users must call dma_buf_begin_cpu_access() and
1280 : * dma_buf_end_cpu_access() around any cpu access performed through this
1281 : * mapping.
1282 : *
1283 : * Returns 0 on success, or a negative errno code otherwise.
1284 : */
1285 0 : int dma_buf_vmap(struct dma_buf *dmabuf, struct iosys_map *map)
1286 : {
1287 : struct iosys_map ptr;
1288 0 : int ret = 0;
1289 :
1290 0 : iosys_map_clear(map);
1291 :
1292 0 : if (WARN_ON(!dmabuf))
1293 : return -EINVAL;
1294 :
1295 0 : if (!dmabuf->ops->vmap)
1296 : return -EINVAL;
1297 :
1298 0 : mutex_lock(&dmabuf->lock);
1299 0 : if (dmabuf->vmapping_counter) {
1300 0 : dmabuf->vmapping_counter++;
1301 0 : BUG_ON(iosys_map_is_null(&dmabuf->vmap_ptr));
1302 0 : *map = dmabuf->vmap_ptr;
1303 0 : goto out_unlock;
1304 : }
1305 :
1306 0 : BUG_ON(iosys_map_is_set(&dmabuf->vmap_ptr));
1307 :
1308 0 : ret = dmabuf->ops->vmap(dmabuf, &ptr);
1309 0 : if (WARN_ON_ONCE(ret))
1310 : goto out_unlock;
1311 :
1312 0 : dmabuf->vmap_ptr = ptr;
1313 0 : dmabuf->vmapping_counter = 1;
1314 :
1315 0 : *map = dmabuf->vmap_ptr;
1316 :
1317 : out_unlock:
1318 0 : mutex_unlock(&dmabuf->lock);
1319 0 : return ret;
1320 : }
1321 : EXPORT_SYMBOL_NS_GPL(dma_buf_vmap, DMA_BUF);
1322 :
1323 : /**
1324 : * dma_buf_vunmap - Unmap a vmap obtained by dma_buf_vmap.
1325 : * @dmabuf: [in] buffer to vunmap
1326 : * @map: [in] vmap pointer to vunmap
1327 : */
1328 0 : void dma_buf_vunmap(struct dma_buf *dmabuf, struct iosys_map *map)
1329 : {
1330 0 : if (WARN_ON(!dmabuf))
1331 : return;
1332 :
1333 0 : BUG_ON(iosys_map_is_null(&dmabuf->vmap_ptr));
1334 0 : BUG_ON(dmabuf->vmapping_counter == 0);
1335 0 : BUG_ON(!iosys_map_is_equal(&dmabuf->vmap_ptr, map));
1336 :
1337 0 : mutex_lock(&dmabuf->lock);
1338 0 : if (--dmabuf->vmapping_counter == 0) {
1339 0 : if (dmabuf->ops->vunmap)
1340 0 : dmabuf->ops->vunmap(dmabuf, map);
1341 0 : iosys_map_clear(&dmabuf->vmap_ptr);
1342 : }
1343 0 : mutex_unlock(&dmabuf->lock);
1344 : }
1345 : EXPORT_SYMBOL_NS_GPL(dma_buf_vunmap, DMA_BUF);
1346 :
1347 : #ifdef CONFIG_DEBUG_FS
1348 : static int dma_buf_debug_show(struct seq_file *s, void *unused)
1349 : {
1350 : struct dma_buf *buf_obj;
1351 : struct dma_buf_attachment *attach_obj;
1352 : int count = 0, attach_count;
1353 : size_t size = 0;
1354 : int ret;
1355 :
1356 : ret = mutex_lock_interruptible(&db_list.lock);
1357 :
1358 : if (ret)
1359 : return ret;
1360 :
1361 : seq_puts(s, "\nDma-buf Objects:\n");
1362 : seq_printf(s, "%-8s\t%-8s\t%-8s\t%-8s\texp_name\t%-8s\n",
1363 : "size", "flags", "mode", "count", "ino");
1364 :
1365 : list_for_each_entry(buf_obj, &db_list.head, list_node) {
1366 :
1367 : ret = dma_resv_lock_interruptible(buf_obj->resv, NULL);
1368 : if (ret)
1369 : goto error_unlock;
1370 :
1371 :
1372 : spin_lock(&buf_obj->name_lock);
1373 : seq_printf(s, "%08zu\t%08x\t%08x\t%08ld\t%s\t%08lu\t%s\n",
1374 : buf_obj->size,
1375 : buf_obj->file->f_flags, buf_obj->file->f_mode,
1376 : file_count(buf_obj->file),
1377 : buf_obj->exp_name,
1378 : file_inode(buf_obj->file)->i_ino,
1379 : buf_obj->name ?: "");
1380 : spin_unlock(&buf_obj->name_lock);
1381 :
1382 : dma_resv_describe(buf_obj->resv, s);
1383 :
1384 : seq_puts(s, "\tAttached Devices:\n");
1385 : attach_count = 0;
1386 :
1387 : list_for_each_entry(attach_obj, &buf_obj->attachments, node) {
1388 : seq_printf(s, "\t%s\n", dev_name(attach_obj->dev));
1389 : attach_count++;
1390 : }
1391 : dma_resv_unlock(buf_obj->resv);
1392 :
1393 : seq_printf(s, "Total %d devices attached\n\n",
1394 : attach_count);
1395 :
1396 : count++;
1397 : size += buf_obj->size;
1398 : }
1399 :
1400 : seq_printf(s, "\nTotal %d objects, %zu bytes\n", count, size);
1401 :
1402 : mutex_unlock(&db_list.lock);
1403 : return 0;
1404 :
1405 : error_unlock:
1406 : mutex_unlock(&db_list.lock);
1407 : return ret;
1408 : }
1409 :
1410 : DEFINE_SHOW_ATTRIBUTE(dma_buf_debug);
1411 :
1412 : static struct dentry *dma_buf_debugfs_dir;
1413 :
1414 : static int dma_buf_init_debugfs(void)
1415 : {
1416 : struct dentry *d;
1417 : int err = 0;
1418 :
1419 : d = debugfs_create_dir("dma_buf", NULL);
1420 : if (IS_ERR(d))
1421 : return PTR_ERR(d);
1422 :
1423 : dma_buf_debugfs_dir = d;
1424 :
1425 : d = debugfs_create_file("bufinfo", S_IRUGO, dma_buf_debugfs_dir,
1426 : NULL, &dma_buf_debug_fops);
1427 : if (IS_ERR(d)) {
1428 : pr_debug("dma_buf: debugfs: failed to create node bufinfo\n");
1429 : debugfs_remove_recursive(dma_buf_debugfs_dir);
1430 : dma_buf_debugfs_dir = NULL;
1431 : err = PTR_ERR(d);
1432 : }
1433 :
1434 : return err;
1435 : }
1436 :
1437 : static void dma_buf_uninit_debugfs(void)
1438 : {
1439 : debugfs_remove_recursive(dma_buf_debugfs_dir);
1440 : }
1441 : #else
1442 : static inline int dma_buf_init_debugfs(void)
1443 : {
1444 : return 0;
1445 : }
1446 : static inline void dma_buf_uninit_debugfs(void)
1447 : {
1448 : }
1449 : #endif
1450 :
1451 1 : static int __init dma_buf_init(void)
1452 : {
1453 : int ret;
1454 :
1455 1 : ret = dma_buf_init_sysfs_statistics();
1456 : if (ret)
1457 : return ret;
1458 :
1459 1 : dma_buf_mnt = kern_mount(&dma_buf_fs_type);
1460 2 : if (IS_ERR(dma_buf_mnt))
1461 0 : return PTR_ERR(dma_buf_mnt);
1462 :
1463 1 : mutex_init(&db_list.lock);
1464 1 : INIT_LIST_HEAD(&db_list.head);
1465 : dma_buf_init_debugfs();
1466 1 : return 0;
1467 : }
1468 : subsys_initcall(dma_buf_init);
1469 :
1470 0 : static void __exit dma_buf_deinit(void)
1471 : {
1472 : dma_buf_uninit_debugfs();
1473 0 : kern_unmount(dma_buf_mnt);
1474 : dma_buf_uninit_sysfs_statistics();
1475 0 : }
1476 : __exitcall(dma_buf_deinit);
|
