Line data Source code
1 : /*
2 : * Copyright 2008 Advanced Micro Devices, Inc.
3 : * Copyright 2008 Red Hat Inc.
4 : * Copyright 2009 Jerome Glisse.
5 : *
6 : * Permission is hereby granted, free of charge, to any person obtaining a
7 : * copy of this software and associated documentation files (the "Software"),
8 : * to deal in the Software without restriction, including without limitation
9 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 : * and/or sell copies of the Software, and to permit persons to whom the
11 : * Software is furnished to do so, subject to the following conditions:
12 : *
13 : * The above copyright notice and this permission notice shall be included in
14 : * all copies or substantial portions of the Software.
15 : *
16 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 : * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 : * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 : * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 : * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 : * OTHER DEALINGS IN THE SOFTWARE.
23 : *
24 : * Authors: Dave Airlie
25 : * Alex Deucher
26 : * Jerome Glisse
27 : */
28 :
29 : #include <linux/pci.h>
30 : #include <linux/vmalloc.h>
31 :
32 : #include <drm/amdgpu_drm.h>
33 : #ifdef CONFIG_X86
34 : #include <asm/set_memory.h>
35 : #endif
36 : #include "amdgpu.h"
37 : #include <drm/drm_drv.h>
38 :
39 : /*
40 : * GART
41 : * The GART (Graphics Aperture Remapping Table) is an aperture
42 : * in the GPU's address space. System pages can be mapped into
43 : * the aperture and look like contiguous pages from the GPU's
44 : * perspective. A page table maps the pages in the aperture
45 : * to the actual backing pages in system memory.
46 : *
47 : * Radeon GPUs support both an internal GART, as described above,
48 : * and AGP. AGP works similarly, but the GART table is configured
49 : * and maintained by the northbridge rather than the driver.
50 : * Radeon hw has a separate AGP aperture that is programmed to
51 : * point to the AGP aperture provided by the northbridge and the
52 : * requests are passed through to the northbridge aperture.
53 : * Both AGP and internal GART can be used at the same time, however
54 : * that is not currently supported by the driver.
55 : *
56 : * This file handles the common internal GART management.
57 : */
58 :
59 : /*
60 : * Common GART table functions.
61 : */
62 :
63 : /**
64 : * amdgpu_gart_dummy_page_init - init dummy page used by the driver
65 : *
66 : * @adev: amdgpu_device pointer
67 : *
68 : * Allocate the dummy page used by the driver (all asics).
69 : * This dummy page is used by the driver as a filler for gart entries
70 : * when pages are taken out of the GART
71 : * Returns 0 on sucess, -ENOMEM on failure.
72 : */
73 0 : static int amdgpu_gart_dummy_page_init(struct amdgpu_device *adev)
74 : {
75 0 : struct page *dummy_page = ttm_glob.dummy_read_page;
76 :
77 0 : if (adev->dummy_page_addr)
78 : return 0;
79 0 : adev->dummy_page_addr = dma_map_page(&adev->pdev->dev, dummy_page, 0,
80 : PAGE_SIZE, DMA_BIDIRECTIONAL);
81 0 : if (dma_mapping_error(&adev->pdev->dev, adev->dummy_page_addr)) {
82 0 : dev_err(&adev->pdev->dev, "Failed to DMA MAP the dummy page\n");
83 0 : adev->dummy_page_addr = 0;
84 : return -ENOMEM;
85 : }
86 : return 0;
87 : }
88 :
89 : /**
90 : * amdgpu_gart_dummy_page_fini - free dummy page used by the driver
91 : *
92 : * @adev: amdgpu_device pointer
93 : *
94 : * Frees the dummy page used by the driver (all asics).
95 : */
96 0 : void amdgpu_gart_dummy_page_fini(struct amdgpu_device *adev)
97 : {
98 0 : if (!adev->dummy_page_addr)
99 : return;
100 0 : dma_unmap_page(&adev->pdev->dev, adev->dummy_page_addr, PAGE_SIZE,
101 : DMA_BIDIRECTIONAL);
102 0 : adev->dummy_page_addr = 0;
103 : }
104 :
105 : /**
106 : * amdgpu_gart_table_vram_alloc - allocate vram for gart page table
107 : *
108 : * @adev: amdgpu_device pointer
109 : *
110 : * Allocate video memory for GART page table
111 : * (pcie r4xx, r5xx+). These asics require the
112 : * gart table to be in video memory.
113 : * Returns 0 for success, error for failure.
114 : */
115 0 : int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev)
116 : {
117 0 : if (adev->gart.bo != NULL)
118 : return 0;
119 :
120 0 : return amdgpu_bo_create_kernel(adev, adev->gart.table_size, PAGE_SIZE,
121 : AMDGPU_GEM_DOMAIN_VRAM, &adev->gart.bo,
122 : NULL, (void *)&adev->gart.ptr);
123 : }
124 :
125 : /**
126 : * amdgpu_gart_table_vram_free - free gart page table vram
127 : *
128 : * @adev: amdgpu_device pointer
129 : *
130 : * Free the video memory used for the GART page table
131 : * (pcie r4xx, r5xx+). These asics require the gart table to
132 : * be in video memory.
133 : */
134 0 : void amdgpu_gart_table_vram_free(struct amdgpu_device *adev)
135 : {
136 0 : amdgpu_bo_free_kernel(&adev->gart.bo, NULL, (void *)&adev->gart.ptr);
137 0 : }
138 :
139 : /*
140 : * Common gart functions.
141 : */
142 : /**
143 : * amdgpu_gart_unbind - unbind pages from the gart page table
144 : *
145 : * @adev: amdgpu_device pointer
146 : * @offset: offset into the GPU's gart aperture
147 : * @pages: number of pages to unbind
148 : *
149 : * Unbinds the requested pages from the gart page table and
150 : * replaces them with the dummy page (all asics).
151 : * Returns 0 for success, -EINVAL for failure.
152 : */
153 0 : void amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset,
154 : int pages)
155 : {
156 : unsigned t;
157 : unsigned p;
158 : int i, j;
159 : u64 page_base;
160 : /* Starting from VEGA10, system bit must be 0 to mean invalid. */
161 0 : uint64_t flags = 0;
162 : int idx;
163 :
164 0 : if (!adev->gart.ptr)
165 0 : return;
166 :
167 0 : if (!drm_dev_enter(adev_to_drm(adev), &idx))
168 : return;
169 :
170 0 : t = offset / AMDGPU_GPU_PAGE_SIZE;
171 0 : p = t / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
172 0 : for (i = 0; i < pages; i++, p++) {
173 0 : page_base = adev->dummy_page_addr;
174 0 : if (!adev->gart.ptr)
175 0 : continue;
176 :
177 0 : for (j = 0; j < AMDGPU_GPU_PAGES_IN_CPU_PAGE; j++, t++) {
178 0 : amdgpu_gmc_set_pte_pde(adev, adev->gart.ptr,
179 : t, page_base, flags);
180 0 : page_base += AMDGPU_GPU_PAGE_SIZE;
181 : }
182 : }
183 0 : mb();
184 0 : amdgpu_device_flush_hdp(adev, NULL);
185 0 : for (i = 0; i < adev->num_vmhubs; i++)
186 0 : amdgpu_gmc_flush_gpu_tlb(adev, 0, i, 0);
187 :
188 0 : drm_dev_exit(idx);
189 : }
190 :
191 : /**
192 : * amdgpu_gart_map - map dma_addresses into GART entries
193 : *
194 : * @adev: amdgpu_device pointer
195 : * @offset: offset into the GPU's gart aperture
196 : * @pages: number of pages to bind
197 : * @dma_addr: DMA addresses of pages
198 : * @flags: page table entry flags
199 : * @dst: CPU address of the gart table
200 : *
201 : * Map the dma_addresses into GART entries (all asics).
202 : * Returns 0 for success, -EINVAL for failure.
203 : */
204 0 : void amdgpu_gart_map(struct amdgpu_device *adev, uint64_t offset,
205 : int pages, dma_addr_t *dma_addr, uint64_t flags,
206 : void *dst)
207 : {
208 : uint64_t page_base;
209 : unsigned i, j, t;
210 : int idx;
211 :
212 0 : if (!drm_dev_enter(adev_to_drm(adev), &idx))
213 0 : return;
214 :
215 0 : t = offset / AMDGPU_GPU_PAGE_SIZE;
216 :
217 0 : for (i = 0; i < pages; i++) {
218 0 : page_base = dma_addr[i];
219 0 : for (j = 0; j < AMDGPU_GPU_PAGES_IN_CPU_PAGE; j++, t++) {
220 0 : amdgpu_gmc_set_pte_pde(adev, dst, t, page_base, flags);
221 0 : page_base += AMDGPU_GPU_PAGE_SIZE;
222 : }
223 : }
224 0 : drm_dev_exit(idx);
225 : }
226 :
227 : /**
228 : * amdgpu_gart_bind - bind pages into the gart page table
229 : *
230 : * @adev: amdgpu_device pointer
231 : * @offset: offset into the GPU's gart aperture
232 : * @pages: number of pages to bind
233 : * @dma_addr: DMA addresses of pages
234 : * @flags: page table entry flags
235 : *
236 : * Binds the requested pages to the gart page table
237 : * (all asics).
238 : * Returns 0 for success, -EINVAL for failure.
239 : */
240 0 : void amdgpu_gart_bind(struct amdgpu_device *adev, uint64_t offset,
241 : int pages, dma_addr_t *dma_addr,
242 : uint64_t flags)
243 : {
244 0 : if (!adev->gart.ptr)
245 : return;
246 :
247 0 : amdgpu_gart_map(adev, offset, pages, dma_addr, flags, adev->gart.ptr);
248 : }
249 :
250 : /**
251 : * amdgpu_gart_invalidate_tlb - invalidate gart TLB
252 : *
253 : * @adev: amdgpu device driver pointer
254 : *
255 : * Invalidate gart TLB which can be use as a way to flush gart changes
256 : *
257 : */
258 0 : void amdgpu_gart_invalidate_tlb(struct amdgpu_device *adev)
259 : {
260 : int i;
261 :
262 0 : if (!adev->gart.ptr)
263 : return;
264 :
265 0 : mb();
266 0 : amdgpu_device_flush_hdp(adev, NULL);
267 0 : for (i = 0; i < adev->num_vmhubs; i++)
268 0 : amdgpu_gmc_flush_gpu_tlb(adev, 0, i, 0);
269 : }
270 :
271 : /**
272 : * amdgpu_gart_init - init the driver info for managing the gart
273 : *
274 : * @adev: amdgpu_device pointer
275 : *
276 : * Allocate the dummy page and init the gart driver info (all asics).
277 : * Returns 0 for success, error for failure.
278 : */
279 0 : int amdgpu_gart_init(struct amdgpu_device *adev)
280 : {
281 : int r;
282 :
283 0 : if (adev->dummy_page_addr)
284 : return 0;
285 :
286 : /* We need PAGE_SIZE >= AMDGPU_GPU_PAGE_SIZE */
287 : if (PAGE_SIZE < AMDGPU_GPU_PAGE_SIZE) {
288 : DRM_ERROR("Page size is smaller than GPU page size!\n");
289 : return -EINVAL;
290 : }
291 0 : r = amdgpu_gart_dummy_page_init(adev);
292 0 : if (r)
293 : return r;
294 : /* Compute table size */
295 0 : adev->gart.num_cpu_pages = adev->gmc.gart_size / PAGE_SIZE;
296 0 : adev->gart.num_gpu_pages = adev->gmc.gart_size / AMDGPU_GPU_PAGE_SIZE;
297 0 : DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n",
298 : adev->gart.num_cpu_pages, adev->gart.num_gpu_pages);
299 :
300 0 : return 0;
301 : }
|
