Line data Source code
1 : /*
2 : * Copyright 2019 Advanced Micro Devices, Inc.
3 : *
4 : * Permission is hereby granted, free of charge, to any person obtaining a
5 : * copy of this software and associated documentation files (the "Software"),
6 : * to deal in the Software without restriction, including without limitation
7 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 : * and/or sell copies of the Software, and to permit persons to whom the
9 : * Software is furnished to do so, subject to the following conditions:
10 : *
11 : * The above copyright notice and this permission notice shall be included in
12 : * all copies or substantial portions of the Software.
13 : *
14 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 : * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 : * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 : * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 : * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 : * OTHER DEALINGS IN THE SOFTWARE.
21 : *
22 : */
23 :
24 : #include <linux/firmware.h>
25 : #include <drm/drm_drv.h>
26 :
27 : #include "amdgpu.h"
28 : #include "amdgpu_vcn.h"
29 : #include "amdgpu_pm.h"
30 : #include "soc15.h"
31 : #include "soc15d.h"
32 : #include "vcn_v2_0.h"
33 : #include "mmsch_v1_0.h"
34 : #include "vcn_v2_5.h"
35 :
36 : #include "vcn/vcn_2_5_offset.h"
37 : #include "vcn/vcn_2_5_sh_mask.h"
38 : #include "ivsrcid/vcn/irqsrcs_vcn_2_0.h"
39 :
40 : #define VCN_VID_SOC_ADDRESS_2_0 0x1fa00
41 : #define VCN1_VID_SOC_ADDRESS_3_0 0x48200
42 :
43 : #define mmUVD_CONTEXT_ID_INTERNAL_OFFSET 0x27
44 : #define mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET 0x0f
45 : #define mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET 0x10
46 : #define mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET 0x11
47 : #define mmUVD_NO_OP_INTERNAL_OFFSET 0x29
48 : #define mmUVD_GP_SCRATCH8_INTERNAL_OFFSET 0x66
49 : #define mmUVD_SCRATCH9_INTERNAL_OFFSET 0xc01d
50 :
51 : #define mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET 0x431
52 : #define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET 0x3b4
53 : #define mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET 0x3b5
54 : #define mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET 0x25c
55 :
56 : #define VCN25_MAX_HW_INSTANCES_ARCTURUS 2
57 :
58 : static void vcn_v2_5_set_dec_ring_funcs(struct amdgpu_device *adev);
59 : static void vcn_v2_5_set_enc_ring_funcs(struct amdgpu_device *adev);
60 : static void vcn_v2_5_set_irq_funcs(struct amdgpu_device *adev);
61 : static int vcn_v2_5_set_powergating_state(void *handle,
62 : enum amd_powergating_state state);
63 : static int vcn_v2_5_pause_dpg_mode(struct amdgpu_device *adev,
64 : int inst_idx, struct dpg_pause_state *new_state);
65 : static int vcn_v2_5_sriov_start(struct amdgpu_device *adev);
66 : static void vcn_v2_5_set_ras_funcs(struct amdgpu_device *adev);
67 :
68 : static int amdgpu_ih_clientid_vcns[] = {
69 : SOC15_IH_CLIENTID_VCN,
70 : SOC15_IH_CLIENTID_VCN1
71 : };
72 :
73 : /**
74 : * vcn_v2_5_early_init - set function pointers
75 : *
76 : * @handle: amdgpu_device pointer
77 : *
78 : * Set ring and irq function pointers
79 : */
80 0 : static int vcn_v2_5_early_init(void *handle)
81 : {
82 0 : struct amdgpu_device *adev = (struct amdgpu_device *)handle;
83 :
84 0 : if (amdgpu_sriov_vf(adev)) {
85 0 : adev->vcn.num_vcn_inst = 2;
86 0 : adev->vcn.harvest_config = 0;
87 0 : adev->vcn.num_enc_rings = 1;
88 : } else {
89 : u32 harvest;
90 : int i;
91 :
92 0 : for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
93 0 : harvest = RREG32_SOC15(VCN, i, mmCC_UVD_HARVESTING);
94 0 : if (harvest & CC_UVD_HARVESTING__UVD_DISABLE_MASK)
95 0 : adev->vcn.harvest_config |= 1 << i;
96 : }
97 0 : if (adev->vcn.harvest_config == (AMDGPU_VCN_HARVEST_VCN0 |
98 : AMDGPU_VCN_HARVEST_VCN1))
99 : /* both instances are harvested, disable the block */
100 : return -ENOENT;
101 :
102 0 : adev->vcn.num_enc_rings = 2;
103 : }
104 :
105 0 : vcn_v2_5_set_dec_ring_funcs(adev);
106 0 : vcn_v2_5_set_enc_ring_funcs(adev);
107 0 : vcn_v2_5_set_irq_funcs(adev);
108 0 : vcn_v2_5_set_ras_funcs(adev);
109 :
110 0 : return 0;
111 : }
112 :
113 : /**
114 : * vcn_v2_5_sw_init - sw init for VCN block
115 : *
116 : * @handle: amdgpu_device pointer
117 : *
118 : * Load firmware and sw initialization
119 : */
120 0 : static int vcn_v2_5_sw_init(void *handle)
121 : {
122 : struct amdgpu_ring *ring;
123 : int i, j, r;
124 0 : struct amdgpu_device *adev = (struct amdgpu_device *)handle;
125 :
126 0 : for (j = 0; j < adev->vcn.num_vcn_inst; j++) {
127 0 : if (adev->vcn.harvest_config & (1 << j))
128 0 : continue;
129 : /* VCN DEC TRAP */
130 0 : r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[j],
131 : VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT, &adev->vcn.inst[j].irq);
132 0 : if (r)
133 : return r;
134 :
135 : /* VCN ENC TRAP */
136 0 : for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
137 0 : r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[j],
138 0 : i + VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[j].irq);
139 0 : if (r)
140 : return r;
141 : }
142 :
143 : /* VCN POISON TRAP */
144 0 : r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[j],
145 : VCN_2_6__SRCID_UVD_POISON, &adev->vcn.inst[j].irq);
146 0 : if (r)
147 : return r;
148 : }
149 :
150 0 : r = amdgpu_vcn_sw_init(adev);
151 0 : if (r)
152 : return r;
153 :
154 0 : amdgpu_vcn_setup_ucode(adev);
155 :
156 0 : r = amdgpu_vcn_resume(adev);
157 0 : if (r)
158 : return r;
159 :
160 0 : for (j = 0; j < adev->vcn.num_vcn_inst; j++) {
161 : volatile struct amdgpu_fw_shared *fw_shared;
162 :
163 0 : if (adev->vcn.harvest_config & (1 << j))
164 0 : continue;
165 0 : adev->vcn.internal.context_id = mmUVD_CONTEXT_ID_INTERNAL_OFFSET;
166 0 : adev->vcn.internal.ib_vmid = mmUVD_LMI_RBC_IB_VMID_INTERNAL_OFFSET;
167 0 : adev->vcn.internal.ib_bar_low = mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_INTERNAL_OFFSET;
168 0 : adev->vcn.internal.ib_bar_high = mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_INTERNAL_OFFSET;
169 0 : adev->vcn.internal.ib_size = mmUVD_RBC_IB_SIZE_INTERNAL_OFFSET;
170 0 : adev->vcn.internal.gp_scratch8 = mmUVD_GP_SCRATCH8_INTERNAL_OFFSET;
171 :
172 0 : adev->vcn.internal.scratch9 = mmUVD_SCRATCH9_INTERNAL_OFFSET;
173 0 : adev->vcn.inst[j].external.scratch9 = SOC15_REG_OFFSET(VCN, j, mmUVD_SCRATCH9);
174 0 : adev->vcn.internal.data0 = mmUVD_GPCOM_VCPU_DATA0_INTERNAL_OFFSET;
175 0 : adev->vcn.inst[j].external.data0 = SOC15_REG_OFFSET(VCN, j, mmUVD_GPCOM_VCPU_DATA0);
176 0 : adev->vcn.internal.data1 = mmUVD_GPCOM_VCPU_DATA1_INTERNAL_OFFSET;
177 0 : adev->vcn.inst[j].external.data1 = SOC15_REG_OFFSET(VCN, j, mmUVD_GPCOM_VCPU_DATA1);
178 0 : adev->vcn.internal.cmd = mmUVD_GPCOM_VCPU_CMD_INTERNAL_OFFSET;
179 0 : adev->vcn.inst[j].external.cmd = SOC15_REG_OFFSET(VCN, j, mmUVD_GPCOM_VCPU_CMD);
180 0 : adev->vcn.internal.nop = mmUVD_NO_OP_INTERNAL_OFFSET;
181 0 : adev->vcn.inst[j].external.nop = SOC15_REG_OFFSET(VCN, j, mmUVD_NO_OP);
182 :
183 0 : ring = &adev->vcn.inst[j].ring_dec;
184 0 : ring->use_doorbell = true;
185 :
186 0 : ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
187 0 : (amdgpu_sriov_vf(adev) ? 2*j : 8*j);
188 0 : sprintf(ring->name, "vcn_dec_%d", j);
189 0 : r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[j].irq,
190 : 0, AMDGPU_RING_PRIO_DEFAULT, NULL);
191 0 : if (r)
192 : return r;
193 :
194 0 : for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
195 0 : enum amdgpu_ring_priority_level hw_prio = amdgpu_vcn_get_enc_ring_prio(i);
196 :
197 0 : ring = &adev->vcn.inst[j].ring_enc[i];
198 0 : ring->use_doorbell = true;
199 :
200 0 : ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
201 0 : (amdgpu_sriov_vf(adev) ? (1 + i + 2*j) : (2 + i + 8*j));
202 :
203 0 : sprintf(ring->name, "vcn_enc_%d.%d", j, i);
204 0 : r = amdgpu_ring_init(adev, ring, 512,
205 : &adev->vcn.inst[j].irq, 0,
206 : hw_prio, NULL);
207 0 : if (r)
208 : return r;
209 : }
210 :
211 0 : fw_shared = adev->vcn.inst[j].fw_shared.cpu_addr;
212 0 : fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_VCN_MULTI_QUEUE_FLAG);
213 :
214 0 : if (amdgpu_vcnfw_log)
215 0 : amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]);
216 : }
217 :
218 0 : if (amdgpu_sriov_vf(adev)) {
219 0 : r = amdgpu_virt_alloc_mm_table(adev);
220 0 : if (r)
221 : return r;
222 : }
223 :
224 0 : if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
225 0 : adev->vcn.pause_dpg_mode = vcn_v2_5_pause_dpg_mode;
226 :
227 : return 0;
228 : }
229 :
230 : /**
231 : * vcn_v2_5_sw_fini - sw fini for VCN block
232 : *
233 : * @handle: amdgpu_device pointer
234 : *
235 : * VCN suspend and free up sw allocation
236 : */
237 0 : static int vcn_v2_5_sw_fini(void *handle)
238 : {
239 : int i, r, idx;
240 0 : struct amdgpu_device *adev = (struct amdgpu_device *)handle;
241 : volatile struct amdgpu_fw_shared *fw_shared;
242 :
243 0 : if (drm_dev_enter(adev_to_drm(adev), &idx)) {
244 0 : for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
245 0 : if (adev->vcn.harvest_config & (1 << i))
246 0 : continue;
247 0 : fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
248 0 : fw_shared->present_flag_0 = 0;
249 : }
250 0 : drm_dev_exit(idx);
251 : }
252 :
253 :
254 0 : if (amdgpu_sriov_vf(adev))
255 0 : amdgpu_virt_free_mm_table(adev);
256 :
257 0 : r = amdgpu_vcn_suspend(adev);
258 0 : if (r)
259 : return r;
260 :
261 0 : r = amdgpu_vcn_sw_fini(adev);
262 :
263 0 : return r;
264 : }
265 :
266 : /**
267 : * vcn_v2_5_hw_init - start and test VCN block
268 : *
269 : * @handle: amdgpu_device pointer
270 : *
271 : * Initialize the hardware, boot up the VCPU and do some testing
272 : */
273 0 : static int vcn_v2_5_hw_init(void *handle)
274 : {
275 0 : struct amdgpu_device *adev = (struct amdgpu_device *)handle;
276 : struct amdgpu_ring *ring;
277 0 : int i, j, r = 0;
278 :
279 0 : if (amdgpu_sriov_vf(adev))
280 0 : r = vcn_v2_5_sriov_start(adev);
281 :
282 0 : for (j = 0; j < adev->vcn.num_vcn_inst; ++j) {
283 0 : if (adev->vcn.harvest_config & (1 << j))
284 0 : continue;
285 :
286 0 : if (amdgpu_sriov_vf(adev)) {
287 0 : adev->vcn.inst[j].ring_enc[0].sched.ready = true;
288 0 : adev->vcn.inst[j].ring_enc[1].sched.ready = false;
289 0 : adev->vcn.inst[j].ring_enc[2].sched.ready = false;
290 0 : adev->vcn.inst[j].ring_dec.sched.ready = true;
291 : } else {
292 :
293 0 : ring = &adev->vcn.inst[j].ring_dec;
294 :
295 0 : adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
296 0 : ring->doorbell_index, j);
297 :
298 0 : r = amdgpu_ring_test_helper(ring);
299 0 : if (r)
300 : goto done;
301 :
302 0 : for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
303 0 : ring = &adev->vcn.inst[j].ring_enc[i];
304 0 : r = amdgpu_ring_test_helper(ring);
305 0 : if (r)
306 : goto done;
307 : }
308 : }
309 : }
310 :
311 : done:
312 0 : if (!r)
313 0 : DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
314 : (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)?"DPG Mode":"SPG Mode");
315 :
316 0 : return r;
317 : }
318 :
319 : /**
320 : * vcn_v2_5_hw_fini - stop the hardware block
321 : *
322 : * @handle: amdgpu_device pointer
323 : *
324 : * Stop the VCN block, mark ring as not ready any more
325 : */
326 0 : static int vcn_v2_5_hw_fini(void *handle)
327 : {
328 0 : struct amdgpu_device *adev = (struct amdgpu_device *)handle;
329 : int i;
330 :
331 0 : cancel_delayed_work_sync(&adev->vcn.idle_work);
332 :
333 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
334 0 : if (adev->vcn.harvest_config & (1 << i))
335 0 : continue;
336 :
337 0 : if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
338 0 : (adev->vcn.cur_state != AMD_PG_STATE_GATE &&
339 0 : RREG32_SOC15(VCN, i, mmUVD_STATUS)))
340 0 : vcn_v2_5_set_powergating_state(adev, AMD_PG_STATE_GATE);
341 : }
342 :
343 0 : return 0;
344 : }
345 :
346 : /**
347 : * vcn_v2_5_suspend - suspend VCN block
348 : *
349 : * @handle: amdgpu_device pointer
350 : *
351 : * HW fini and suspend VCN block
352 : */
353 0 : static int vcn_v2_5_suspend(void *handle)
354 : {
355 : int r;
356 0 : struct amdgpu_device *adev = (struct amdgpu_device *)handle;
357 :
358 0 : r = vcn_v2_5_hw_fini(adev);
359 0 : if (r)
360 : return r;
361 :
362 0 : r = amdgpu_vcn_suspend(adev);
363 :
364 0 : return r;
365 : }
366 :
367 : /**
368 : * vcn_v2_5_resume - resume VCN block
369 : *
370 : * @handle: amdgpu_device pointer
371 : *
372 : * Resume firmware and hw init VCN block
373 : */
374 0 : static int vcn_v2_5_resume(void *handle)
375 : {
376 : int r;
377 0 : struct amdgpu_device *adev = (struct amdgpu_device *)handle;
378 :
379 0 : r = amdgpu_vcn_resume(adev);
380 0 : if (r)
381 : return r;
382 :
383 0 : r = vcn_v2_5_hw_init(adev);
384 :
385 0 : return r;
386 : }
387 :
388 : /**
389 : * vcn_v2_5_mc_resume - memory controller programming
390 : *
391 : * @adev: amdgpu_device pointer
392 : *
393 : * Let the VCN memory controller know it's offsets
394 : */
395 0 : static void vcn_v2_5_mc_resume(struct amdgpu_device *adev)
396 : {
397 0 : uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
398 : uint32_t offset;
399 : int i;
400 :
401 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
402 0 : if (adev->vcn.harvest_config & (1 << i))
403 0 : continue;
404 : /* cache window 0: fw */
405 0 : if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
406 0 : WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
407 : (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo));
408 0 : WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
409 : (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi));
410 0 : WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET0, 0);
411 : offset = 0;
412 : } else {
413 0 : WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
414 : lower_32_bits(adev->vcn.inst[i].gpu_addr));
415 0 : WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
416 : upper_32_bits(adev->vcn.inst[i].gpu_addr));
417 0 : offset = size;
418 0 : WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET0,
419 : AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
420 : }
421 0 : WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_SIZE0, size);
422 :
423 : /* cache window 1: stack */
424 0 : WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
425 : lower_32_bits(adev->vcn.inst[i].gpu_addr + offset));
426 0 : WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
427 : upper_32_bits(adev->vcn.inst[i].gpu_addr + offset));
428 0 : WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET1, 0);
429 0 : WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
430 :
431 : /* cache window 2: context */
432 0 : WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
433 : lower_32_bits(adev->vcn.inst[i].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
434 0 : WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
435 : upper_32_bits(adev->vcn.inst[i].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
436 0 : WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_OFFSET2, 0);
437 0 : WREG32_SOC15(VCN, i, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
438 :
439 : /* non-cache window */
440 0 : WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
441 : lower_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr));
442 0 : WREG32_SOC15(VCN, i, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
443 : upper_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr));
444 0 : WREG32_SOC15(VCN, i, mmUVD_VCPU_NONCACHE_OFFSET0, 0);
445 0 : WREG32_SOC15(VCN, i, mmUVD_VCPU_NONCACHE_SIZE0,
446 : AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)));
447 : }
448 0 : }
449 :
450 0 : static void vcn_v2_5_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
451 : {
452 0 : uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
453 : uint32_t offset;
454 :
455 : /* cache window 0: fw */
456 0 : if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
457 0 : if (!indirect) {
458 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
459 : VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
460 : (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);
461 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
462 : VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
463 : (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);
464 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
465 : VCN, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
466 : } else {
467 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
468 : VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
469 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
470 : VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
471 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
472 : VCN, 0, mmUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
473 : }
474 : offset = 0;
475 : } else {
476 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
477 : VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
478 : lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
479 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
480 : VCN, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
481 : upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
482 0 : offset = size;
483 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
484 : VCN, 0, mmUVD_VCPU_CACHE_OFFSET0),
485 : AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
486 : }
487 :
488 0 : if (!indirect)
489 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
490 : VCN, 0, mmUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
491 : else
492 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
493 : VCN, 0, mmUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
494 :
495 : /* cache window 1: stack */
496 0 : if (!indirect) {
497 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
498 : VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
499 : lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
500 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
501 : VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
502 : upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
503 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
504 : VCN, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
505 : } else {
506 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
507 : VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
508 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
509 : VCN, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
510 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
511 : VCN, 0, mmUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
512 : }
513 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
514 : VCN, 0, mmUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
515 :
516 : /* cache window 2: context */
517 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
518 : VCN, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
519 : lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
520 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
521 : VCN, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
522 : upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
523 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
524 : VCN, 0, mmUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
525 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
526 : VCN, 0, mmUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
527 :
528 : /* non-cache window */
529 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
530 : VCN, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
531 : lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
532 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
533 : VCN, 0, mmUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
534 : upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
535 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
536 : VCN, 0, mmUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
537 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
538 : VCN, 0, mmUVD_VCPU_NONCACHE_SIZE0),
539 : AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_fw_shared)), 0, indirect);
540 :
541 : /* VCN global tiling registers */
542 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
543 : VCN, 0, mmUVD_GFX8_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
544 0 : }
545 :
546 : /**
547 : * vcn_v2_5_disable_clock_gating - disable VCN clock gating
548 : *
549 : * @adev: amdgpu_device pointer
550 : *
551 : * Disable clock gating for VCN block
552 : */
553 0 : static void vcn_v2_5_disable_clock_gating(struct amdgpu_device *adev)
554 : {
555 : uint32_t data;
556 : int i;
557 :
558 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
559 0 : if (adev->vcn.harvest_config & (1 << i))
560 0 : continue;
561 : /* UVD disable CGC */
562 0 : data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL);
563 0 : if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
564 0 : data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
565 : else
566 0 : data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
567 0 : data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
568 0 : data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
569 0 : WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data);
570 :
571 0 : data = RREG32_SOC15(VCN, i, mmUVD_CGC_GATE);
572 0 : data &= ~(UVD_CGC_GATE__SYS_MASK
573 : | UVD_CGC_GATE__UDEC_MASK
574 : | UVD_CGC_GATE__MPEG2_MASK
575 : | UVD_CGC_GATE__REGS_MASK
576 : | UVD_CGC_GATE__RBC_MASK
577 : | UVD_CGC_GATE__LMI_MC_MASK
578 : | UVD_CGC_GATE__LMI_UMC_MASK
579 : | UVD_CGC_GATE__IDCT_MASK
580 : | UVD_CGC_GATE__MPRD_MASK
581 : | UVD_CGC_GATE__MPC_MASK
582 : | UVD_CGC_GATE__LBSI_MASK
583 : | UVD_CGC_GATE__LRBBM_MASK
584 : | UVD_CGC_GATE__UDEC_RE_MASK
585 : | UVD_CGC_GATE__UDEC_CM_MASK
586 : | UVD_CGC_GATE__UDEC_IT_MASK
587 : | UVD_CGC_GATE__UDEC_DB_MASK
588 : | UVD_CGC_GATE__UDEC_MP_MASK
589 : | UVD_CGC_GATE__WCB_MASK
590 : | UVD_CGC_GATE__VCPU_MASK
591 : | UVD_CGC_GATE__MMSCH_MASK);
592 :
593 0 : WREG32_SOC15(VCN, i, mmUVD_CGC_GATE, data);
594 :
595 0 : SOC15_WAIT_ON_RREG(VCN, i, mmUVD_CGC_GATE, 0, 0xFFFFFFFF);
596 :
597 0 : data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL);
598 0 : data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
599 : | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
600 : | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
601 : | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
602 : | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
603 : | UVD_CGC_CTRL__SYS_MODE_MASK
604 : | UVD_CGC_CTRL__UDEC_MODE_MASK
605 : | UVD_CGC_CTRL__MPEG2_MODE_MASK
606 : | UVD_CGC_CTRL__REGS_MODE_MASK
607 : | UVD_CGC_CTRL__RBC_MODE_MASK
608 : | UVD_CGC_CTRL__LMI_MC_MODE_MASK
609 : | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
610 : | UVD_CGC_CTRL__IDCT_MODE_MASK
611 : | UVD_CGC_CTRL__MPRD_MODE_MASK
612 : | UVD_CGC_CTRL__MPC_MODE_MASK
613 : | UVD_CGC_CTRL__LBSI_MODE_MASK
614 : | UVD_CGC_CTRL__LRBBM_MODE_MASK
615 : | UVD_CGC_CTRL__WCB_MODE_MASK
616 : | UVD_CGC_CTRL__VCPU_MODE_MASK
617 : | UVD_CGC_CTRL__MMSCH_MODE_MASK);
618 0 : WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data);
619 :
620 : /* turn on */
621 0 : data = RREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_GATE);
622 0 : data |= (UVD_SUVD_CGC_GATE__SRE_MASK
623 : | UVD_SUVD_CGC_GATE__SIT_MASK
624 : | UVD_SUVD_CGC_GATE__SMP_MASK
625 : | UVD_SUVD_CGC_GATE__SCM_MASK
626 : | UVD_SUVD_CGC_GATE__SDB_MASK
627 : | UVD_SUVD_CGC_GATE__SRE_H264_MASK
628 : | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
629 : | UVD_SUVD_CGC_GATE__SIT_H264_MASK
630 : | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
631 : | UVD_SUVD_CGC_GATE__SCM_H264_MASK
632 : | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
633 : | UVD_SUVD_CGC_GATE__SDB_H264_MASK
634 : | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
635 : | UVD_SUVD_CGC_GATE__SCLR_MASK
636 : | UVD_SUVD_CGC_GATE__UVD_SC_MASK
637 : | UVD_SUVD_CGC_GATE__ENT_MASK
638 : | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
639 : | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
640 : | UVD_SUVD_CGC_GATE__SITE_MASK
641 : | UVD_SUVD_CGC_GATE__SRE_VP9_MASK
642 : | UVD_SUVD_CGC_GATE__SCM_VP9_MASK
643 : | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
644 : | UVD_SUVD_CGC_GATE__SDB_VP9_MASK
645 : | UVD_SUVD_CGC_GATE__IME_HEVC_MASK);
646 0 : WREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_GATE, data);
647 :
648 0 : data = RREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL);
649 0 : data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
650 : | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
651 : | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
652 : | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
653 : | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
654 : | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
655 : | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
656 : | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
657 : | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
658 : | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
659 0 : WREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL, data);
660 : }
661 0 : }
662 :
663 0 : static void vcn_v2_5_clock_gating_dpg_mode(struct amdgpu_device *adev,
664 : uint8_t sram_sel, int inst_idx, uint8_t indirect)
665 : {
666 0 : uint32_t reg_data = 0;
667 :
668 : /* enable sw clock gating control */
669 0 : if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
670 : reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
671 : else
672 0 : reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
673 0 : reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
674 0 : reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
675 0 : reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
676 : UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
677 : UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
678 : UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
679 : UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
680 : UVD_CGC_CTRL__SYS_MODE_MASK |
681 : UVD_CGC_CTRL__UDEC_MODE_MASK |
682 : UVD_CGC_CTRL__MPEG2_MODE_MASK |
683 : UVD_CGC_CTRL__REGS_MODE_MASK |
684 : UVD_CGC_CTRL__RBC_MODE_MASK |
685 : UVD_CGC_CTRL__LMI_MC_MODE_MASK |
686 : UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
687 : UVD_CGC_CTRL__IDCT_MODE_MASK |
688 : UVD_CGC_CTRL__MPRD_MODE_MASK |
689 : UVD_CGC_CTRL__MPC_MODE_MASK |
690 : UVD_CGC_CTRL__LBSI_MODE_MASK |
691 : UVD_CGC_CTRL__LRBBM_MODE_MASK |
692 : UVD_CGC_CTRL__WCB_MODE_MASK |
693 : UVD_CGC_CTRL__VCPU_MODE_MASK |
694 : UVD_CGC_CTRL__MMSCH_MODE_MASK);
695 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
696 : VCN, 0, mmUVD_CGC_CTRL), reg_data, sram_sel, indirect);
697 :
698 : /* turn off clock gating */
699 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
700 : VCN, 0, mmUVD_CGC_GATE), 0, sram_sel, indirect);
701 :
702 : /* turn on SUVD clock gating */
703 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
704 : VCN, 0, mmUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
705 :
706 : /* turn on sw mode in UVD_SUVD_CGC_CTRL */
707 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
708 : VCN, 0, mmUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
709 0 : }
710 :
711 : /**
712 : * vcn_v2_5_enable_clock_gating - enable VCN clock gating
713 : *
714 : * @adev: amdgpu_device pointer
715 : *
716 : * Enable clock gating for VCN block
717 : */
718 0 : static void vcn_v2_5_enable_clock_gating(struct amdgpu_device *adev)
719 : {
720 0 : uint32_t data = 0;
721 : int i;
722 :
723 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
724 0 : if (adev->vcn.harvest_config & (1 << i))
725 0 : continue;
726 : /* enable UVD CGC */
727 0 : data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL);
728 0 : if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
729 0 : data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
730 : else
731 : data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
732 0 : data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
733 0 : data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
734 0 : WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data);
735 :
736 0 : data = RREG32_SOC15(VCN, i, mmUVD_CGC_CTRL);
737 0 : data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
738 : | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
739 : | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
740 : | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
741 : | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
742 : | UVD_CGC_CTRL__SYS_MODE_MASK
743 : | UVD_CGC_CTRL__UDEC_MODE_MASK
744 : | UVD_CGC_CTRL__MPEG2_MODE_MASK
745 : | UVD_CGC_CTRL__REGS_MODE_MASK
746 : | UVD_CGC_CTRL__RBC_MODE_MASK
747 : | UVD_CGC_CTRL__LMI_MC_MODE_MASK
748 : | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
749 : | UVD_CGC_CTRL__IDCT_MODE_MASK
750 : | UVD_CGC_CTRL__MPRD_MODE_MASK
751 : | UVD_CGC_CTRL__MPC_MODE_MASK
752 : | UVD_CGC_CTRL__LBSI_MODE_MASK
753 : | UVD_CGC_CTRL__LRBBM_MODE_MASK
754 : | UVD_CGC_CTRL__WCB_MODE_MASK
755 : | UVD_CGC_CTRL__VCPU_MODE_MASK);
756 0 : WREG32_SOC15(VCN, i, mmUVD_CGC_CTRL, data);
757 :
758 0 : data = RREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL);
759 0 : data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
760 : | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
761 : | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
762 : | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
763 : | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
764 : | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
765 : | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
766 : | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
767 : | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
768 : | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
769 0 : WREG32_SOC15(VCN, i, mmUVD_SUVD_CGC_CTRL, data);
770 : }
771 0 : }
772 :
773 0 : static int vcn_v2_5_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
774 : {
775 0 : volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
776 : struct amdgpu_ring *ring;
777 : uint32_t rb_bufsz, tmp;
778 :
779 : /* disable register anti-hang mechanism */
780 0 : WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 1,
781 : ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
782 : /* enable dynamic power gating mode */
783 0 : tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS);
784 0 : tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
785 0 : tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
786 0 : WREG32_SOC15(VCN, inst_idx, mmUVD_POWER_STATUS, tmp);
787 :
788 0 : if (indirect)
789 0 : adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
790 :
791 : /* enable clock gating */
792 0 : vcn_v2_5_clock_gating_dpg_mode(adev, 0, inst_idx, indirect);
793 :
794 : /* enable VCPU clock */
795 0 : tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
796 0 : tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
797 0 : tmp |= UVD_VCPU_CNTL__BLK_RST_MASK;
798 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
799 : VCN, 0, mmUVD_VCPU_CNTL), tmp, 0, indirect);
800 :
801 : /* disable master interupt */
802 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
803 : VCN, 0, mmUVD_MASTINT_EN), 0, 0, indirect);
804 :
805 : /* setup mmUVD_LMI_CTRL */
806 0 : tmp = (0x8 | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
807 : UVD_LMI_CTRL__REQ_MODE_MASK |
808 : UVD_LMI_CTRL__CRC_RESET_MASK |
809 : UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
810 : UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
811 : UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
812 : (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
813 : 0x00100000L);
814 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
815 : VCN, 0, mmUVD_LMI_CTRL), tmp, 0, indirect);
816 :
817 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
818 : VCN, 0, mmUVD_MPC_CNTL),
819 : 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
820 :
821 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
822 : VCN, 0, mmUVD_MPC_SET_MUXA0),
823 : ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
824 : (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
825 : (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
826 : (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
827 :
828 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
829 : VCN, 0, mmUVD_MPC_SET_MUXB0),
830 : ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
831 : (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
832 : (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
833 : (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
834 :
835 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
836 : VCN, 0, mmUVD_MPC_SET_MUX),
837 : ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
838 : (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
839 : (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
840 :
841 0 : vcn_v2_5_mc_resume_dpg_mode(adev, inst_idx, indirect);
842 :
843 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
844 : VCN, 0, mmUVD_REG_XX_MASK), 0x10, 0, indirect);
845 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
846 : VCN, 0, mmUVD_RBC_XX_IB_REG_CHECK), 0x3, 0, indirect);
847 :
848 : /* enable LMI MC and UMC channels */
849 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
850 : VCN, 0, mmUVD_LMI_CTRL2), 0, 0, indirect);
851 :
852 : /* unblock VCPU register access */
853 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
854 : VCN, 0, mmUVD_RB_ARB_CTRL), 0, 0, indirect);
855 :
856 0 : tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
857 0 : tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
858 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
859 : VCN, 0, mmUVD_VCPU_CNTL), tmp, 0, indirect);
860 :
861 : /* enable master interrupt */
862 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
863 : VCN, 0, mmUVD_MASTINT_EN),
864 : UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
865 :
866 0 : if (indirect)
867 0 : psp_update_vcn_sram(adev, inst_idx, adev->vcn.inst[inst_idx].dpg_sram_gpu_addr,
868 0 : (uint32_t)((uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_curr_addr -
869 0 : (uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr));
870 :
871 0 : ring = &adev->vcn.inst[inst_idx].ring_dec;
872 : /* force RBC into idle state */
873 0 : rb_bufsz = order_base_2(ring->ring_size);
874 0 : tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
875 0 : tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
876 0 : tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
877 0 : tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
878 0 : tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
879 0 : WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_CNTL, tmp);
880 :
881 : /* Stall DPG before WPTR/RPTR reset */
882 0 : WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
883 : UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
884 : ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
885 0 : fw_shared->multi_queue.decode_queue_mode |= FW_QUEUE_RING_RESET;
886 :
887 : /* set the write pointer delay */
888 0 : WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR_CNTL, 0);
889 :
890 : /* set the wb address */
891 0 : WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR_ADDR,
892 : (upper_32_bits(ring->gpu_addr) >> 2));
893 :
894 : /* program the RB_BASE for ring buffer */
895 0 : WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
896 : lower_32_bits(ring->gpu_addr));
897 0 : WREG32_SOC15(VCN, inst_idx, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
898 : upper_32_bits(ring->gpu_addr));
899 :
900 : /* Initialize the ring buffer's read and write pointers */
901 0 : WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR, 0);
902 :
903 0 : WREG32_SOC15(VCN, inst_idx, mmUVD_SCRATCH2, 0);
904 :
905 0 : ring->wptr = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_RPTR);
906 0 : WREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR,
907 : lower_32_bits(ring->wptr));
908 :
909 0 : fw_shared->multi_queue.decode_queue_mode &= ~FW_QUEUE_RING_RESET;
910 : /* Unstall DPG */
911 0 : WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
912 : 0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
913 :
914 0 : return 0;
915 : }
916 :
917 0 : static int vcn_v2_5_start(struct amdgpu_device *adev)
918 : {
919 : struct amdgpu_ring *ring;
920 : uint32_t rb_bufsz, tmp;
921 : int i, j, k, r;
922 :
923 0 : if (adev->pm.dpm_enabled)
924 0 : amdgpu_dpm_enable_uvd(adev, true);
925 :
926 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
927 0 : if (adev->vcn.harvest_config & (1 << i))
928 0 : continue;
929 0 : if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
930 0 : r = vcn_v2_5_start_dpg_mode(adev, i, adev->vcn.indirect_sram);
931 0 : continue;
932 : }
933 :
934 : /* disable register anti-hang mechanism */
935 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_POWER_STATUS), 0,
936 : ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
937 :
938 : /* set uvd status busy */
939 0 : tmp = RREG32_SOC15(VCN, i, mmUVD_STATUS) | UVD_STATUS__UVD_BUSY;
940 0 : WREG32_SOC15(VCN, i, mmUVD_STATUS, tmp);
941 : }
942 :
943 0 : if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
944 : return 0;
945 :
946 : /*SW clock gating */
947 0 : vcn_v2_5_disable_clock_gating(adev);
948 :
949 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
950 0 : if (adev->vcn.harvest_config & (1 << i))
951 0 : continue;
952 : /* enable VCPU clock */
953 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
954 : UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
955 :
956 : /* disable master interrupt */
957 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN), 0,
958 : ~UVD_MASTINT_EN__VCPU_EN_MASK);
959 :
960 : /* setup mmUVD_LMI_CTRL */
961 0 : tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL);
962 0 : tmp &= ~0xff;
963 0 : WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL, tmp | 0x8|
964 : UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
965 : UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
966 : UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
967 : UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
968 :
969 : /* setup mmUVD_MPC_CNTL */
970 0 : tmp = RREG32_SOC15(VCN, i, mmUVD_MPC_CNTL);
971 0 : tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
972 0 : tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
973 0 : WREG32_SOC15(VCN, i, mmUVD_MPC_CNTL, tmp);
974 :
975 : /* setup UVD_MPC_SET_MUXA0 */
976 0 : WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXA0,
977 : ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
978 : (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
979 : (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
980 : (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
981 :
982 : /* setup UVD_MPC_SET_MUXB0 */
983 0 : WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUXB0,
984 : ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
985 : (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
986 : (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
987 : (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
988 :
989 : /* setup mmUVD_MPC_SET_MUX */
990 0 : WREG32_SOC15(VCN, i, mmUVD_MPC_SET_MUX,
991 : ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
992 : (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
993 : (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
994 : }
995 :
996 0 : vcn_v2_5_mc_resume(adev);
997 :
998 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
999 0 : volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1000 0 : if (adev->vcn.harvest_config & (1 << i))
1001 0 : continue;
1002 : /* VCN global tiling registers */
1003 0 : WREG32_SOC15(VCN, i, mmUVD_GFX8_ADDR_CONFIG,
1004 : adev->gfx.config.gb_addr_config);
1005 0 : WREG32_SOC15(VCN, i, mmUVD_GFX8_ADDR_CONFIG,
1006 : adev->gfx.config.gb_addr_config);
1007 :
1008 : /* enable LMI MC and UMC channels */
1009 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_LMI_CTRL2), 0,
1010 : ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1011 :
1012 : /* unblock VCPU register access */
1013 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL), 0,
1014 : ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1015 :
1016 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1017 : ~UVD_VCPU_CNTL__BLK_RST_MASK);
1018 :
1019 0 : for (k = 0; k < 10; ++k) {
1020 : uint32_t status;
1021 :
1022 0 : for (j = 0; j < 100; ++j) {
1023 0 : status = RREG32_SOC15(VCN, i, mmUVD_STATUS);
1024 0 : if (status & 2)
1025 : break;
1026 0 : if (amdgpu_emu_mode == 1)
1027 0 : msleep(500);
1028 : else
1029 0 : mdelay(10);
1030 : }
1031 0 : r = 0;
1032 0 : if (status & 2)
1033 : break;
1034 :
1035 0 : DRM_ERROR("VCN decode not responding, trying to reset the VCPU!!!\n");
1036 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1037 : UVD_VCPU_CNTL__BLK_RST_MASK,
1038 : ~UVD_VCPU_CNTL__BLK_RST_MASK);
1039 0 : mdelay(10);
1040 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1041 : ~UVD_VCPU_CNTL__BLK_RST_MASK);
1042 :
1043 0 : mdelay(10);
1044 0 : r = -1;
1045 : }
1046 :
1047 0 : if (r) {
1048 0 : DRM_ERROR("VCN decode not responding, giving up!!!\n");
1049 0 : return r;
1050 : }
1051 :
1052 : /* enable master interrupt */
1053 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_MASTINT_EN),
1054 : UVD_MASTINT_EN__VCPU_EN_MASK,
1055 : ~UVD_MASTINT_EN__VCPU_EN_MASK);
1056 :
1057 : /* clear the busy bit of VCN_STATUS */
1058 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_STATUS), 0,
1059 : ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1060 :
1061 0 : WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_VMID, 0);
1062 :
1063 0 : ring = &adev->vcn.inst[i].ring_dec;
1064 : /* force RBC into idle state */
1065 0 : rb_bufsz = order_base_2(ring->ring_size);
1066 0 : tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1067 0 : tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1068 0 : tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1069 0 : tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1070 0 : tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1071 0 : WREG32_SOC15(VCN, i, mmUVD_RBC_RB_CNTL, tmp);
1072 :
1073 0 : fw_shared->multi_queue.decode_queue_mode |= FW_QUEUE_RING_RESET;
1074 : /* program the RB_BASE for ring buffer */
1075 0 : WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1076 : lower_32_bits(ring->gpu_addr));
1077 0 : WREG32_SOC15(VCN, i, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1078 : upper_32_bits(ring->gpu_addr));
1079 :
1080 : /* Initialize the ring buffer's read and write pointers */
1081 0 : WREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR, 0);
1082 :
1083 0 : ring->wptr = RREG32_SOC15(VCN, i, mmUVD_RBC_RB_RPTR);
1084 0 : WREG32_SOC15(VCN, i, mmUVD_RBC_RB_WPTR,
1085 : lower_32_bits(ring->wptr));
1086 0 : fw_shared->multi_queue.decode_queue_mode &= ~FW_QUEUE_RING_RESET;
1087 :
1088 0 : fw_shared->multi_queue.encode_generalpurpose_queue_mode |= FW_QUEUE_RING_RESET;
1089 0 : ring = &adev->vcn.inst[i].ring_enc[0];
1090 0 : WREG32_SOC15(VCN, i, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1091 0 : WREG32_SOC15(VCN, i, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1092 0 : WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO, ring->gpu_addr);
1093 0 : WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1094 0 : WREG32_SOC15(VCN, i, mmUVD_RB_SIZE, ring->ring_size / 4);
1095 0 : fw_shared->multi_queue.encode_generalpurpose_queue_mode &= ~FW_QUEUE_RING_RESET;
1096 :
1097 0 : fw_shared->multi_queue.encode_lowlatency_queue_mode |= FW_QUEUE_RING_RESET;
1098 0 : ring = &adev->vcn.inst[i].ring_enc[1];
1099 0 : WREG32_SOC15(VCN, i, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1100 0 : WREG32_SOC15(VCN, i, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1101 0 : WREG32_SOC15(VCN, i, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1102 0 : WREG32_SOC15(VCN, i, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1103 0 : WREG32_SOC15(VCN, i, mmUVD_RB_SIZE2, ring->ring_size / 4);
1104 0 : fw_shared->multi_queue.encode_lowlatency_queue_mode &= ~FW_QUEUE_RING_RESET;
1105 : }
1106 :
1107 : return 0;
1108 : }
1109 :
1110 0 : static int vcn_v2_5_mmsch_start(struct amdgpu_device *adev,
1111 : struct amdgpu_mm_table *table)
1112 : {
1113 0 : uint32_t data = 0, loop = 0, size = 0;
1114 0 : uint64_t addr = table->gpu_addr;
1115 0 : struct mmsch_v1_1_init_header *header = NULL;
1116 :
1117 0 : header = (struct mmsch_v1_1_init_header *)table->cpu_addr;
1118 0 : size = header->total_size;
1119 :
1120 : /*
1121 : * 1, write to vce_mmsch_vf_ctx_addr_lo/hi register with GPU mc addr of
1122 : * memory descriptor location
1123 : */
1124 0 : WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_LO, lower_32_bits(addr));
1125 0 : WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_ADDR_HI, upper_32_bits(addr));
1126 :
1127 : /* 2, update vmid of descriptor */
1128 0 : data = RREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID);
1129 0 : data &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
1130 : /* use domain0 for MM scheduler */
1131 0 : data |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
1132 0 : WREG32_SOC15(VCN, 0, mmMMSCH_VF_VMID, data);
1133 :
1134 : /* 3, notify mmsch about the size of this descriptor */
1135 0 : WREG32_SOC15(VCN, 0, mmMMSCH_VF_CTX_SIZE, size);
1136 :
1137 : /* 4, set resp to zero */
1138 0 : WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP, 0);
1139 :
1140 : /*
1141 : * 5, kick off the initialization and wait until
1142 : * VCE_MMSCH_VF_MAILBOX_RESP becomes non-zero
1143 : */
1144 0 : WREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_HOST, 0x10000001);
1145 :
1146 0 : data = RREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP);
1147 : loop = 10;
1148 0 : while ((data & 0x10000002) != 0x10000002) {
1149 0 : udelay(100);
1150 0 : data = RREG32_SOC15(VCN, 0, mmMMSCH_VF_MAILBOX_RESP);
1151 0 : loop--;
1152 0 : if (!loop)
1153 : break;
1154 : }
1155 :
1156 0 : if (!loop) {
1157 0 : dev_err(adev->dev,
1158 : "failed to init MMSCH, mmMMSCH_VF_MAILBOX_RESP = %x\n",
1159 : data);
1160 : return -EBUSY;
1161 : }
1162 :
1163 : return 0;
1164 : }
1165 :
1166 0 : static int vcn_v2_5_sriov_start(struct amdgpu_device *adev)
1167 : {
1168 : struct amdgpu_ring *ring;
1169 : uint32_t offset, size, tmp, i, rb_bufsz;
1170 0 : uint32_t table_size = 0;
1171 0 : struct mmsch_v1_0_cmd_direct_write direct_wt = { { 0 } };
1172 0 : struct mmsch_v1_0_cmd_direct_read_modify_write direct_rd_mod_wt = { { 0 } };
1173 0 : struct mmsch_v1_0_cmd_end end = { { 0 } };
1174 0 : uint32_t *init_table = adev->virt.mm_table.cpu_addr;
1175 0 : struct mmsch_v1_1_init_header *header = (struct mmsch_v1_1_init_header *)init_table;
1176 :
1177 : direct_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_WRITE;
1178 0 : direct_rd_mod_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
1179 0 : end.cmd_header.command_type = MMSCH_COMMAND__END;
1180 :
1181 0 : header->version = MMSCH_VERSION;
1182 0 : header->total_size = sizeof(struct mmsch_v1_1_init_header) >> 2;
1183 0 : init_table += header->total_size;
1184 :
1185 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1186 0 : header->eng[i].table_offset = header->total_size;
1187 0 : header->eng[i].init_status = 0;
1188 0 : header->eng[i].table_size = 0;
1189 :
1190 0 : table_size = 0;
1191 :
1192 0 : MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(
1193 : SOC15_REG_OFFSET(VCN, i, mmUVD_STATUS),
1194 : ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);
1195 :
1196 0 : size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
1197 : /* mc resume*/
1198 0 : if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1199 0 : MMSCH_V1_0_INSERT_DIRECT_WT(
1200 : SOC15_REG_OFFSET(VCN, i,
1201 : mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1202 : adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo);
1203 0 : MMSCH_V1_0_INSERT_DIRECT_WT(
1204 : SOC15_REG_OFFSET(VCN, i,
1205 : mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1206 : adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi);
1207 0 : offset = 0;
1208 0 : MMSCH_V1_0_INSERT_DIRECT_WT(
1209 : SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET0), 0);
1210 : } else {
1211 0 : MMSCH_V1_0_INSERT_DIRECT_WT(
1212 : SOC15_REG_OFFSET(VCN, i,
1213 : mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1214 : lower_32_bits(adev->vcn.inst[i].gpu_addr));
1215 0 : MMSCH_V1_0_INSERT_DIRECT_WT(
1216 : SOC15_REG_OFFSET(VCN, i,
1217 : mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1218 : upper_32_bits(adev->vcn.inst[i].gpu_addr));
1219 0 : offset = size;
1220 0 : MMSCH_V1_0_INSERT_DIRECT_WT(
1221 : SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET0),
1222 : AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
1223 : }
1224 :
1225 0 : MMSCH_V1_0_INSERT_DIRECT_WT(
1226 : SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_SIZE0),
1227 : size);
1228 0 : MMSCH_V1_0_INSERT_DIRECT_WT(
1229 : SOC15_REG_OFFSET(VCN, i,
1230 : mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
1231 : lower_32_bits(adev->vcn.inst[i].gpu_addr + offset));
1232 0 : MMSCH_V1_0_INSERT_DIRECT_WT(
1233 : SOC15_REG_OFFSET(VCN, i,
1234 : mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
1235 : upper_32_bits(adev->vcn.inst[i].gpu_addr + offset));
1236 0 : MMSCH_V1_0_INSERT_DIRECT_WT(
1237 : SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET1),
1238 : 0);
1239 0 : MMSCH_V1_0_INSERT_DIRECT_WT(
1240 : SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_SIZE1),
1241 : AMDGPU_VCN_STACK_SIZE);
1242 0 : MMSCH_V1_0_INSERT_DIRECT_WT(
1243 : SOC15_REG_OFFSET(VCN, i,
1244 : mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
1245 : lower_32_bits(adev->vcn.inst[i].gpu_addr + offset +
1246 : AMDGPU_VCN_STACK_SIZE));
1247 0 : MMSCH_V1_0_INSERT_DIRECT_WT(
1248 : SOC15_REG_OFFSET(VCN, i,
1249 : mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
1250 : upper_32_bits(adev->vcn.inst[i].gpu_addr + offset +
1251 : AMDGPU_VCN_STACK_SIZE));
1252 0 : MMSCH_V1_0_INSERT_DIRECT_WT(
1253 : SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_OFFSET2),
1254 : 0);
1255 0 : MMSCH_V1_0_INSERT_DIRECT_WT(
1256 : SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CACHE_SIZE2),
1257 : AMDGPU_VCN_CONTEXT_SIZE);
1258 :
1259 0 : ring = &adev->vcn.inst[i].ring_enc[0];
1260 0 : ring->wptr = 0;
1261 :
1262 0 : MMSCH_V1_0_INSERT_DIRECT_WT(
1263 : SOC15_REG_OFFSET(VCN, i, mmUVD_RB_BASE_LO),
1264 : lower_32_bits(ring->gpu_addr));
1265 0 : MMSCH_V1_0_INSERT_DIRECT_WT(
1266 : SOC15_REG_OFFSET(VCN, i, mmUVD_RB_BASE_HI),
1267 : upper_32_bits(ring->gpu_addr));
1268 0 : MMSCH_V1_0_INSERT_DIRECT_WT(
1269 : SOC15_REG_OFFSET(VCN, i, mmUVD_RB_SIZE),
1270 : ring->ring_size / 4);
1271 :
1272 0 : ring = &adev->vcn.inst[i].ring_dec;
1273 0 : ring->wptr = 0;
1274 0 : MMSCH_V1_0_INSERT_DIRECT_WT(
1275 : SOC15_REG_OFFSET(VCN, i,
1276 : mmUVD_LMI_RBC_RB_64BIT_BAR_LOW),
1277 : lower_32_bits(ring->gpu_addr));
1278 0 : MMSCH_V1_0_INSERT_DIRECT_WT(
1279 : SOC15_REG_OFFSET(VCN, i,
1280 : mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH),
1281 : upper_32_bits(ring->gpu_addr));
1282 :
1283 : /* force RBC into idle state */
1284 0 : rb_bufsz = order_base_2(ring->ring_size);
1285 0 : tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1286 0 : tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1287 0 : tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1288 0 : tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1289 0 : tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1290 0 : MMSCH_V1_0_INSERT_DIRECT_WT(
1291 : SOC15_REG_OFFSET(VCN, i, mmUVD_RBC_RB_CNTL), tmp);
1292 :
1293 : /* add end packet */
1294 0 : memcpy((void *)init_table, &end, sizeof(struct mmsch_v1_0_cmd_end));
1295 0 : table_size += sizeof(struct mmsch_v1_0_cmd_end) / 4;
1296 0 : init_table += sizeof(struct mmsch_v1_0_cmd_end) / 4;
1297 :
1298 : /* refine header */
1299 0 : header->eng[i].table_size = table_size;
1300 0 : header->total_size += table_size;
1301 : }
1302 :
1303 0 : return vcn_v2_5_mmsch_start(adev, &adev->virt.mm_table);
1304 : }
1305 :
1306 0 : static int vcn_v2_5_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)
1307 : {
1308 : uint32_t tmp;
1309 :
1310 : /* Wait for power status to be 1 */
1311 0 : SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1312 : UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1313 :
1314 : /* wait for read ptr to be equal to write ptr */
1315 0 : tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR);
1316 0 : SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1317 :
1318 0 : tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2);
1319 0 : SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RB_RPTR2, tmp, 0xFFFFFFFF);
1320 :
1321 0 : tmp = RREG32_SOC15(VCN, inst_idx, mmUVD_RBC_RB_WPTR) & 0x7FFFFFFF;
1322 0 : SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_RBC_RB_RPTR, tmp, 0xFFFFFFFF);
1323 :
1324 0 : SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 1,
1325 : UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1326 :
1327 : /* disable dynamic power gating mode */
1328 0 : WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS), 0,
1329 : ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1330 :
1331 0 : return 0;
1332 : }
1333 :
1334 0 : static int vcn_v2_5_stop(struct amdgpu_device *adev)
1335 : {
1336 : uint32_t tmp;
1337 0 : int i, r = 0;
1338 :
1339 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1340 0 : if (adev->vcn.harvest_config & (1 << i))
1341 0 : continue;
1342 0 : if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1343 0 : r = vcn_v2_5_stop_dpg_mode(adev, i);
1344 0 : continue;
1345 : }
1346 :
1347 : /* wait for vcn idle */
1348 0 : r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1349 0 : if (r)
1350 : return r;
1351 :
1352 0 : tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1353 : UVD_LMI_STATUS__READ_CLEAN_MASK |
1354 : UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1355 : UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1356 0 : r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);
1357 0 : if (r)
1358 : return r;
1359 :
1360 : /* block LMI UMC channel */
1361 0 : tmp = RREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2);
1362 0 : tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1363 0 : WREG32_SOC15(VCN, i, mmUVD_LMI_CTRL2, tmp);
1364 :
1365 0 : tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK|
1366 : UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1367 0 : r = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_LMI_STATUS, tmp, tmp);
1368 0 : if (r)
1369 : return r;
1370 :
1371 : /* block VCPU register access */
1372 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_RB_ARB_CTRL),
1373 : UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
1374 : ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1375 :
1376 : /* reset VCPU */
1377 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL),
1378 : UVD_VCPU_CNTL__BLK_RST_MASK,
1379 : ~UVD_VCPU_CNTL__BLK_RST_MASK);
1380 :
1381 : /* disable VCPU clock */
1382 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_VCPU_CNTL), 0,
1383 : ~(UVD_VCPU_CNTL__CLK_EN_MASK));
1384 :
1385 : /* clear status */
1386 0 : WREG32_SOC15(VCN, i, mmUVD_STATUS, 0);
1387 :
1388 0 : vcn_v2_5_enable_clock_gating(adev);
1389 :
1390 : /* enable register anti-hang mechanism */
1391 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, mmUVD_POWER_STATUS),
1392 : UVD_POWER_STATUS__UVD_POWER_STATUS_MASK,
1393 : ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1394 : }
1395 :
1396 0 : if (adev->pm.dpm_enabled)
1397 0 : amdgpu_dpm_enable_uvd(adev, false);
1398 :
1399 : return 0;
1400 : }
1401 :
1402 0 : static int vcn_v2_5_pause_dpg_mode(struct amdgpu_device *adev,
1403 : int inst_idx, struct dpg_pause_state *new_state)
1404 : {
1405 : struct amdgpu_ring *ring;
1406 0 : uint32_t reg_data = 0;
1407 0 : int ret_code = 0;
1408 :
1409 : /* pause/unpause if state is changed */
1410 0 : if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1411 0 : DRM_DEBUG("dpg pause state changed %d -> %d",
1412 : adev->vcn.inst[inst_idx].pause_state.fw_based, new_state->fw_based);
1413 0 : reg_data = RREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE) &
1414 : (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1415 :
1416 0 : if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1417 0 : ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 0x1,
1418 : UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1419 :
1420 0 : if (!ret_code) {
1421 0 : volatile struct amdgpu_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
1422 :
1423 : /* pause DPG */
1424 0 : reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1425 0 : WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1426 :
1427 : /* wait for ACK */
1428 0 : SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_DPG_PAUSE,
1429 : UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1430 : UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1431 :
1432 : /* Stall DPG before WPTR/RPTR reset */
1433 0 : WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1434 : UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK,
1435 : ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1436 :
1437 : /* Restore */
1438 0 : fw_shared->multi_queue.encode_generalpurpose_queue_mode |= FW_QUEUE_RING_RESET;
1439 0 : ring = &adev->vcn.inst[inst_idx].ring_enc[0];
1440 0 : ring->wptr = 0;
1441 0 : WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO, ring->gpu_addr);
1442 0 : WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1443 0 : WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE, ring->ring_size / 4);
1444 0 : WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
1445 0 : WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1446 0 : fw_shared->multi_queue.encode_generalpurpose_queue_mode &= ~FW_QUEUE_RING_RESET;
1447 :
1448 0 : fw_shared->multi_queue.encode_lowlatency_queue_mode |= FW_QUEUE_RING_RESET;
1449 0 : ring = &adev->vcn.inst[inst_idx].ring_enc[1];
1450 0 : ring->wptr = 0;
1451 0 : WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_LO2, ring->gpu_addr);
1452 0 : WREG32_SOC15(VCN, inst_idx, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
1453 0 : WREG32_SOC15(VCN, inst_idx, mmUVD_RB_SIZE2, ring->ring_size / 4);
1454 0 : WREG32_SOC15(VCN, inst_idx, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
1455 0 : WREG32_SOC15(VCN, inst_idx, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1456 0 : fw_shared->multi_queue.encode_lowlatency_queue_mode &= ~FW_QUEUE_RING_RESET;
1457 :
1458 : /* Unstall DPG */
1459 0 : WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, mmUVD_POWER_STATUS),
1460 : 0, ~UVD_POWER_STATUS__STALL_DPG_POWER_UP_MASK);
1461 :
1462 0 : SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS,
1463 : UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1464 : }
1465 : } else {
1466 0 : reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1467 0 : WREG32_SOC15(VCN, inst_idx, mmUVD_DPG_PAUSE, reg_data);
1468 0 : SOC15_WAIT_ON_RREG(VCN, inst_idx, mmUVD_POWER_STATUS, 0x1,
1469 : UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1470 : }
1471 0 : adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1472 : }
1473 :
1474 0 : return 0;
1475 : }
1476 :
1477 : /**
1478 : * vcn_v2_5_dec_ring_get_rptr - get read pointer
1479 : *
1480 : * @ring: amdgpu_ring pointer
1481 : *
1482 : * Returns the current hardware read pointer
1483 : */
1484 0 : static uint64_t vcn_v2_5_dec_ring_get_rptr(struct amdgpu_ring *ring)
1485 : {
1486 0 : struct amdgpu_device *adev = ring->adev;
1487 :
1488 0 : return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_RPTR);
1489 : }
1490 :
1491 : /**
1492 : * vcn_v2_5_dec_ring_get_wptr - get write pointer
1493 : *
1494 : * @ring: amdgpu_ring pointer
1495 : *
1496 : * Returns the current hardware write pointer
1497 : */
1498 0 : static uint64_t vcn_v2_5_dec_ring_get_wptr(struct amdgpu_ring *ring)
1499 : {
1500 0 : struct amdgpu_device *adev = ring->adev;
1501 :
1502 0 : if (ring->use_doorbell)
1503 0 : return *ring->wptr_cpu_addr;
1504 : else
1505 0 : return RREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR);
1506 : }
1507 :
1508 : /**
1509 : * vcn_v2_5_dec_ring_set_wptr - set write pointer
1510 : *
1511 : * @ring: amdgpu_ring pointer
1512 : *
1513 : * Commits the write pointer to the hardware
1514 : */
1515 0 : static void vcn_v2_5_dec_ring_set_wptr(struct amdgpu_ring *ring)
1516 : {
1517 0 : struct amdgpu_device *adev = ring->adev;
1518 :
1519 0 : if (ring->use_doorbell) {
1520 0 : *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1521 0 : WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1522 : } else {
1523 0 : WREG32_SOC15(VCN, ring->me, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
1524 : }
1525 0 : }
1526 :
1527 : static const struct amdgpu_ring_funcs vcn_v2_5_dec_ring_vm_funcs = {
1528 : .type = AMDGPU_RING_TYPE_VCN_DEC,
1529 : .align_mask = 0xf,
1530 : .secure_submission_supported = true,
1531 : .vmhub = AMDGPU_MMHUB_1,
1532 : .get_rptr = vcn_v2_5_dec_ring_get_rptr,
1533 : .get_wptr = vcn_v2_5_dec_ring_get_wptr,
1534 : .set_wptr = vcn_v2_5_dec_ring_set_wptr,
1535 : .emit_frame_size =
1536 : SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
1537 : SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
1538 : 8 + /* vcn_v2_0_dec_ring_emit_vm_flush */
1539 : 14 + 14 + /* vcn_v2_0_dec_ring_emit_fence x2 vm fence */
1540 : 6,
1541 : .emit_ib_size = 8, /* vcn_v2_0_dec_ring_emit_ib */
1542 : .emit_ib = vcn_v2_0_dec_ring_emit_ib,
1543 : .emit_fence = vcn_v2_0_dec_ring_emit_fence,
1544 : .emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush,
1545 : .test_ring = vcn_v2_0_dec_ring_test_ring,
1546 : .test_ib = amdgpu_vcn_dec_ring_test_ib,
1547 : .insert_nop = vcn_v2_0_dec_ring_insert_nop,
1548 : .insert_start = vcn_v2_0_dec_ring_insert_start,
1549 : .insert_end = vcn_v2_0_dec_ring_insert_end,
1550 : .pad_ib = amdgpu_ring_generic_pad_ib,
1551 : .begin_use = amdgpu_vcn_ring_begin_use,
1552 : .end_use = amdgpu_vcn_ring_end_use,
1553 : .emit_wreg = vcn_v2_0_dec_ring_emit_wreg,
1554 : .emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait,
1555 : .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1556 : };
1557 :
1558 : static const struct amdgpu_ring_funcs vcn_v2_6_dec_ring_vm_funcs = {
1559 : .type = AMDGPU_RING_TYPE_VCN_DEC,
1560 : .align_mask = 0xf,
1561 : .secure_submission_supported = true,
1562 : .vmhub = AMDGPU_MMHUB_0,
1563 : .get_rptr = vcn_v2_5_dec_ring_get_rptr,
1564 : .get_wptr = vcn_v2_5_dec_ring_get_wptr,
1565 : .set_wptr = vcn_v2_5_dec_ring_set_wptr,
1566 : .emit_frame_size =
1567 : SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
1568 : SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
1569 : 8 + /* vcn_v2_0_dec_ring_emit_vm_flush */
1570 : 14 + 14 + /* vcn_v2_0_dec_ring_emit_fence x2 vm fence */
1571 : 6,
1572 : .emit_ib_size = 8, /* vcn_v2_0_dec_ring_emit_ib */
1573 : .emit_ib = vcn_v2_0_dec_ring_emit_ib,
1574 : .emit_fence = vcn_v2_0_dec_ring_emit_fence,
1575 : .emit_vm_flush = vcn_v2_0_dec_ring_emit_vm_flush,
1576 : .test_ring = vcn_v2_0_dec_ring_test_ring,
1577 : .test_ib = amdgpu_vcn_dec_ring_test_ib,
1578 : .insert_nop = vcn_v2_0_dec_ring_insert_nop,
1579 : .insert_start = vcn_v2_0_dec_ring_insert_start,
1580 : .insert_end = vcn_v2_0_dec_ring_insert_end,
1581 : .pad_ib = amdgpu_ring_generic_pad_ib,
1582 : .begin_use = amdgpu_vcn_ring_begin_use,
1583 : .end_use = amdgpu_vcn_ring_end_use,
1584 : .emit_wreg = vcn_v2_0_dec_ring_emit_wreg,
1585 : .emit_reg_wait = vcn_v2_0_dec_ring_emit_reg_wait,
1586 : .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1587 : };
1588 :
1589 : /**
1590 : * vcn_v2_5_enc_ring_get_rptr - get enc read pointer
1591 : *
1592 : * @ring: amdgpu_ring pointer
1593 : *
1594 : * Returns the current hardware enc read pointer
1595 : */
1596 0 : static uint64_t vcn_v2_5_enc_ring_get_rptr(struct amdgpu_ring *ring)
1597 : {
1598 0 : struct amdgpu_device *adev = ring->adev;
1599 :
1600 0 : if (ring == &adev->vcn.inst[ring->me].ring_enc[0])
1601 0 : return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR);
1602 : else
1603 0 : return RREG32_SOC15(VCN, ring->me, mmUVD_RB_RPTR2);
1604 : }
1605 :
1606 : /**
1607 : * vcn_v2_5_enc_ring_get_wptr - get enc write pointer
1608 : *
1609 : * @ring: amdgpu_ring pointer
1610 : *
1611 : * Returns the current hardware enc write pointer
1612 : */
1613 0 : static uint64_t vcn_v2_5_enc_ring_get_wptr(struct amdgpu_ring *ring)
1614 : {
1615 0 : struct amdgpu_device *adev = ring->adev;
1616 :
1617 0 : if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1618 0 : if (ring->use_doorbell)
1619 0 : return *ring->wptr_cpu_addr;
1620 : else
1621 0 : return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR);
1622 : } else {
1623 0 : if (ring->use_doorbell)
1624 0 : return *ring->wptr_cpu_addr;
1625 : else
1626 0 : return RREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2);
1627 : }
1628 : }
1629 :
1630 : /**
1631 : * vcn_v2_5_enc_ring_set_wptr - set enc write pointer
1632 : *
1633 : * @ring: amdgpu_ring pointer
1634 : *
1635 : * Commits the enc write pointer to the hardware
1636 : */
1637 0 : static void vcn_v2_5_enc_ring_set_wptr(struct amdgpu_ring *ring)
1638 : {
1639 0 : struct amdgpu_device *adev = ring->adev;
1640 :
1641 0 : if (ring == &adev->vcn.inst[ring->me].ring_enc[0]) {
1642 0 : if (ring->use_doorbell) {
1643 0 : *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1644 0 : WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1645 : } else {
1646 0 : WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
1647 : }
1648 : } else {
1649 0 : if (ring->use_doorbell) {
1650 0 : *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1651 0 : WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1652 : } else {
1653 0 : WREG32_SOC15(VCN, ring->me, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
1654 : }
1655 : }
1656 0 : }
1657 :
1658 : static const struct amdgpu_ring_funcs vcn_v2_5_enc_ring_vm_funcs = {
1659 : .type = AMDGPU_RING_TYPE_VCN_ENC,
1660 : .align_mask = 0x3f,
1661 : .nop = VCN_ENC_CMD_NO_OP,
1662 : .vmhub = AMDGPU_MMHUB_1,
1663 : .get_rptr = vcn_v2_5_enc_ring_get_rptr,
1664 : .get_wptr = vcn_v2_5_enc_ring_get_wptr,
1665 : .set_wptr = vcn_v2_5_enc_ring_set_wptr,
1666 : .emit_frame_size =
1667 : SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1668 : SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1669 : 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
1670 : 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
1671 : 1, /* vcn_v2_0_enc_ring_insert_end */
1672 : .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
1673 : .emit_ib = vcn_v2_0_enc_ring_emit_ib,
1674 : .emit_fence = vcn_v2_0_enc_ring_emit_fence,
1675 : .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
1676 : .test_ring = amdgpu_vcn_enc_ring_test_ring,
1677 : .test_ib = amdgpu_vcn_enc_ring_test_ib,
1678 : .insert_nop = amdgpu_ring_insert_nop,
1679 : .insert_end = vcn_v2_0_enc_ring_insert_end,
1680 : .pad_ib = amdgpu_ring_generic_pad_ib,
1681 : .begin_use = amdgpu_vcn_ring_begin_use,
1682 : .end_use = amdgpu_vcn_ring_end_use,
1683 : .emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
1684 : .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
1685 : .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1686 : };
1687 :
1688 : static const struct amdgpu_ring_funcs vcn_v2_6_enc_ring_vm_funcs = {
1689 : .type = AMDGPU_RING_TYPE_VCN_ENC,
1690 : .align_mask = 0x3f,
1691 : .nop = VCN_ENC_CMD_NO_OP,
1692 : .vmhub = AMDGPU_MMHUB_0,
1693 : .get_rptr = vcn_v2_5_enc_ring_get_rptr,
1694 : .get_wptr = vcn_v2_5_enc_ring_get_wptr,
1695 : .set_wptr = vcn_v2_5_enc_ring_set_wptr,
1696 : .emit_frame_size =
1697 : SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1698 : SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1699 : 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
1700 : 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
1701 : 1, /* vcn_v2_0_enc_ring_insert_end */
1702 : .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
1703 : .emit_ib = vcn_v2_0_enc_ring_emit_ib,
1704 : .emit_fence = vcn_v2_0_enc_ring_emit_fence,
1705 : .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
1706 : .test_ring = amdgpu_vcn_enc_ring_test_ring,
1707 : .test_ib = amdgpu_vcn_enc_ring_test_ib,
1708 : .insert_nop = amdgpu_ring_insert_nop,
1709 : .insert_end = vcn_v2_0_enc_ring_insert_end,
1710 : .pad_ib = amdgpu_ring_generic_pad_ib,
1711 : .begin_use = amdgpu_vcn_ring_begin_use,
1712 : .end_use = amdgpu_vcn_ring_end_use,
1713 : .emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
1714 : .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
1715 : .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1716 : };
1717 :
1718 0 : static void vcn_v2_5_set_dec_ring_funcs(struct amdgpu_device *adev)
1719 : {
1720 : int i;
1721 :
1722 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1723 0 : if (adev->vcn.harvest_config & (1 << i))
1724 0 : continue;
1725 0 : if (adev->ip_versions[UVD_HWIP][0] == IP_VERSION(2, 5, 0))
1726 0 : adev->vcn.inst[i].ring_dec.funcs = &vcn_v2_5_dec_ring_vm_funcs;
1727 : else /* CHIP_ALDEBARAN */
1728 0 : adev->vcn.inst[i].ring_dec.funcs = &vcn_v2_6_dec_ring_vm_funcs;
1729 0 : adev->vcn.inst[i].ring_dec.me = i;
1730 0 : DRM_INFO("VCN(%d) decode is enabled in VM mode\n", i);
1731 : }
1732 0 : }
1733 :
1734 0 : static void vcn_v2_5_set_enc_ring_funcs(struct amdgpu_device *adev)
1735 : {
1736 : int i, j;
1737 :
1738 0 : for (j = 0; j < adev->vcn.num_vcn_inst; ++j) {
1739 0 : if (adev->vcn.harvest_config & (1 << j))
1740 0 : continue;
1741 0 : for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
1742 0 : if (adev->ip_versions[UVD_HWIP][0] == IP_VERSION(2, 5, 0))
1743 0 : adev->vcn.inst[j].ring_enc[i].funcs = &vcn_v2_5_enc_ring_vm_funcs;
1744 : else /* CHIP_ALDEBARAN */
1745 0 : adev->vcn.inst[j].ring_enc[i].funcs = &vcn_v2_6_enc_ring_vm_funcs;
1746 0 : adev->vcn.inst[j].ring_enc[i].me = j;
1747 : }
1748 0 : DRM_INFO("VCN(%d) encode is enabled in VM mode\n", j);
1749 : }
1750 0 : }
1751 :
1752 0 : static bool vcn_v2_5_is_idle(void *handle)
1753 : {
1754 0 : struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1755 0 : int i, ret = 1;
1756 :
1757 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1758 0 : if (adev->vcn.harvest_config & (1 << i))
1759 0 : continue;
1760 0 : ret &= (RREG32_SOC15(VCN, i, mmUVD_STATUS) == UVD_STATUS__IDLE);
1761 : }
1762 :
1763 0 : return ret;
1764 : }
1765 :
1766 0 : static int vcn_v2_5_wait_for_idle(void *handle)
1767 : {
1768 0 : struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1769 0 : int i, ret = 0;
1770 :
1771 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1772 0 : if (adev->vcn.harvest_config & (1 << i))
1773 0 : continue;
1774 0 : ret = SOC15_WAIT_ON_RREG(VCN, i, mmUVD_STATUS, UVD_STATUS__IDLE,
1775 : UVD_STATUS__IDLE);
1776 0 : if (ret)
1777 : return ret;
1778 : }
1779 :
1780 : return ret;
1781 : }
1782 :
1783 0 : static int vcn_v2_5_set_clockgating_state(void *handle,
1784 : enum amd_clockgating_state state)
1785 : {
1786 0 : struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1787 0 : bool enable = (state == AMD_CG_STATE_GATE);
1788 :
1789 0 : if (amdgpu_sriov_vf(adev))
1790 : return 0;
1791 :
1792 0 : if (enable) {
1793 0 : if (!vcn_v2_5_is_idle(handle))
1794 : return -EBUSY;
1795 0 : vcn_v2_5_enable_clock_gating(adev);
1796 : } else {
1797 0 : vcn_v2_5_disable_clock_gating(adev);
1798 : }
1799 :
1800 : return 0;
1801 : }
1802 :
1803 0 : static int vcn_v2_5_set_powergating_state(void *handle,
1804 : enum amd_powergating_state state)
1805 : {
1806 0 : struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1807 : int ret;
1808 :
1809 0 : if (amdgpu_sriov_vf(adev))
1810 : return 0;
1811 :
1812 0 : if(state == adev->vcn.cur_state)
1813 : return 0;
1814 :
1815 0 : if (state == AMD_PG_STATE_GATE)
1816 0 : ret = vcn_v2_5_stop(adev);
1817 : else
1818 0 : ret = vcn_v2_5_start(adev);
1819 :
1820 0 : if(!ret)
1821 0 : adev->vcn.cur_state = state;
1822 :
1823 : return ret;
1824 : }
1825 :
1826 0 : static int vcn_v2_5_set_interrupt_state(struct amdgpu_device *adev,
1827 : struct amdgpu_irq_src *source,
1828 : unsigned type,
1829 : enum amdgpu_interrupt_state state)
1830 : {
1831 0 : return 0;
1832 : }
1833 :
1834 0 : static int vcn_v2_5_process_interrupt(struct amdgpu_device *adev,
1835 : struct amdgpu_irq_src *source,
1836 : struct amdgpu_iv_entry *entry)
1837 : {
1838 : uint32_t ip_instance;
1839 :
1840 0 : switch (entry->client_id) {
1841 : case SOC15_IH_CLIENTID_VCN:
1842 : ip_instance = 0;
1843 : break;
1844 : case SOC15_IH_CLIENTID_VCN1:
1845 0 : ip_instance = 1;
1846 0 : break;
1847 : default:
1848 0 : DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
1849 0 : return 0;
1850 : }
1851 :
1852 0 : DRM_DEBUG("IH: VCN TRAP\n");
1853 :
1854 0 : switch (entry->src_id) {
1855 : case VCN_2_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT:
1856 0 : amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_dec);
1857 0 : break;
1858 : case VCN_2_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
1859 0 : amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);
1860 0 : break;
1861 : case VCN_2_0__SRCID__UVD_ENC_LOW_LATENCY:
1862 0 : amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[1]);
1863 0 : break;
1864 : case VCN_2_6__SRCID_UVD_POISON:
1865 0 : amdgpu_vcn_process_poison_irq(adev, source, entry);
1866 0 : break;
1867 : default:
1868 0 : DRM_ERROR("Unhandled interrupt: %d %d\n",
1869 : entry->src_id, entry->src_data[0]);
1870 0 : break;
1871 : }
1872 :
1873 : return 0;
1874 : }
1875 :
1876 : static const struct amdgpu_irq_src_funcs vcn_v2_5_irq_funcs = {
1877 : .set = vcn_v2_5_set_interrupt_state,
1878 : .process = vcn_v2_5_process_interrupt,
1879 : };
1880 :
1881 : static void vcn_v2_5_set_irq_funcs(struct amdgpu_device *adev)
1882 : {
1883 : int i;
1884 :
1885 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1886 0 : if (adev->vcn.harvest_config & (1 << i))
1887 0 : continue;
1888 0 : adev->vcn.inst[i].irq.num_types = adev->vcn.num_enc_rings + 1;
1889 0 : adev->vcn.inst[i].irq.funcs = &vcn_v2_5_irq_funcs;
1890 : }
1891 : }
1892 :
1893 : static const struct amd_ip_funcs vcn_v2_5_ip_funcs = {
1894 : .name = "vcn_v2_5",
1895 : .early_init = vcn_v2_5_early_init,
1896 : .late_init = NULL,
1897 : .sw_init = vcn_v2_5_sw_init,
1898 : .sw_fini = vcn_v2_5_sw_fini,
1899 : .hw_init = vcn_v2_5_hw_init,
1900 : .hw_fini = vcn_v2_5_hw_fini,
1901 : .suspend = vcn_v2_5_suspend,
1902 : .resume = vcn_v2_5_resume,
1903 : .is_idle = vcn_v2_5_is_idle,
1904 : .wait_for_idle = vcn_v2_5_wait_for_idle,
1905 : .check_soft_reset = NULL,
1906 : .pre_soft_reset = NULL,
1907 : .soft_reset = NULL,
1908 : .post_soft_reset = NULL,
1909 : .set_clockgating_state = vcn_v2_5_set_clockgating_state,
1910 : .set_powergating_state = vcn_v2_5_set_powergating_state,
1911 : };
1912 :
1913 : static const struct amd_ip_funcs vcn_v2_6_ip_funcs = {
1914 : .name = "vcn_v2_6",
1915 : .early_init = vcn_v2_5_early_init,
1916 : .late_init = NULL,
1917 : .sw_init = vcn_v2_5_sw_init,
1918 : .sw_fini = vcn_v2_5_sw_fini,
1919 : .hw_init = vcn_v2_5_hw_init,
1920 : .hw_fini = vcn_v2_5_hw_fini,
1921 : .suspend = vcn_v2_5_suspend,
1922 : .resume = vcn_v2_5_resume,
1923 : .is_idle = vcn_v2_5_is_idle,
1924 : .wait_for_idle = vcn_v2_5_wait_for_idle,
1925 : .check_soft_reset = NULL,
1926 : .pre_soft_reset = NULL,
1927 : .soft_reset = NULL,
1928 : .post_soft_reset = NULL,
1929 : .set_clockgating_state = vcn_v2_5_set_clockgating_state,
1930 : .set_powergating_state = vcn_v2_5_set_powergating_state,
1931 : };
1932 :
1933 : const struct amdgpu_ip_block_version vcn_v2_5_ip_block =
1934 : {
1935 : .type = AMD_IP_BLOCK_TYPE_VCN,
1936 : .major = 2,
1937 : .minor = 5,
1938 : .rev = 0,
1939 : .funcs = &vcn_v2_5_ip_funcs,
1940 : };
1941 :
1942 : const struct amdgpu_ip_block_version vcn_v2_6_ip_block =
1943 : {
1944 : .type = AMD_IP_BLOCK_TYPE_VCN,
1945 : .major = 2,
1946 : .minor = 6,
1947 : .rev = 0,
1948 : .funcs = &vcn_v2_6_ip_funcs,
1949 : };
1950 :
1951 0 : static uint32_t vcn_v2_6_query_poison_by_instance(struct amdgpu_device *adev,
1952 : uint32_t instance, uint32_t sub_block)
1953 : {
1954 0 : uint32_t poison_stat = 0, reg_value = 0;
1955 :
1956 0 : switch (sub_block) {
1957 : case AMDGPU_VCN_V2_6_VCPU_VCODEC:
1958 0 : reg_value = RREG32_SOC15(VCN, instance, mmUVD_RAS_VCPU_VCODEC_STATUS);
1959 0 : poison_stat = REG_GET_FIELD(reg_value, UVD_RAS_VCPU_VCODEC_STATUS, POISONED_PF);
1960 0 : break;
1961 : default:
1962 : break;
1963 : }
1964 :
1965 0 : if (poison_stat)
1966 0 : dev_info(adev->dev, "Poison detected in VCN%d, sub_block%d\n",
1967 : instance, sub_block);
1968 :
1969 0 : return poison_stat;
1970 : }
1971 :
1972 0 : static bool vcn_v2_6_query_poison_status(struct amdgpu_device *adev)
1973 : {
1974 : uint32_t inst, sub;
1975 0 : uint32_t poison_stat = 0;
1976 :
1977 0 : for (inst = 0; inst < adev->vcn.num_vcn_inst; inst++)
1978 0 : for (sub = 0; sub < AMDGPU_VCN_V2_6_MAX_SUB_BLOCK; sub++)
1979 0 : poison_stat +=
1980 0 : vcn_v2_6_query_poison_by_instance(adev, inst, sub);
1981 :
1982 0 : return !!poison_stat;
1983 : }
1984 :
1985 : const struct amdgpu_ras_block_hw_ops vcn_v2_6_ras_hw_ops = {
1986 : .query_poison_status = vcn_v2_6_query_poison_status,
1987 : };
1988 :
1989 : static struct amdgpu_vcn_ras vcn_v2_6_ras = {
1990 : .ras_block = {
1991 : .hw_ops = &vcn_v2_6_ras_hw_ops,
1992 : },
1993 : };
1994 :
1995 0 : static void vcn_v2_5_set_ras_funcs(struct amdgpu_device *adev)
1996 : {
1997 0 : switch (adev->ip_versions[VCN_HWIP][0]) {
1998 : case IP_VERSION(2, 6, 0):
1999 0 : adev->vcn.ras = &vcn_v2_6_ras;
2000 0 : break;
2001 : default:
2002 : break;
2003 : }
2004 :
2005 0 : if (adev->vcn.ras) {
2006 0 : amdgpu_ras_register_ras_block(adev, &adev->vcn.ras->ras_block);
2007 :
2008 0 : strcpy(adev->vcn.ras->ras_block.ras_comm.name, "vcn");
2009 0 : adev->vcn.ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__VCN;
2010 0 : adev->vcn.ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__POISON;
2011 0 : adev->vcn.ras_if = &adev->vcn.ras->ras_block.ras_comm;
2012 :
2013 : /* If don't define special ras_late_init function, use default ras_late_init */
2014 0 : if (!adev->vcn.ras->ras_block.ras_late_init)
2015 0 : adev->vcn.ras->ras_block.ras_late_init = amdgpu_ras_block_late_init;
2016 : }
2017 0 : }
|
