Line data Source code
1 : /*
2 : * Copyright 2021 Advanced Micro Devices, Inc.
3 : *
4 : * Permission is hereby granted, free of charge, to any person obtaining a
5 : * copy of this software and associated documentation files (the "Software"),
6 : * to deal in the Software without restriction, including without limitation
7 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 : * and/or sell copies of the Software, and to permit persons to whom the
9 : * Software is furnished to do so, subject to the following conditions:
10 : *
11 : * The above copyright notice and this permission notice shall be included in
12 : * all copies or substantial portions of the Software.
13 : *
14 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 : * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 : * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 : * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 : * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 : * OTHER DEALINGS IN THE SOFTWARE.
21 : *
22 : */
23 :
24 : #include <linux/firmware.h>
25 : #include "amdgpu.h"
26 : #include "amdgpu_vcn.h"
27 : #include "amdgpu_pm.h"
28 : #include "amdgpu_cs.h"
29 : #include "soc15.h"
30 : #include "soc15d.h"
31 : #include "soc15_hw_ip.h"
32 : #include "vcn_v2_0.h"
33 : #include "mmsch_v4_0.h"
34 :
35 : #include "vcn/vcn_4_0_0_offset.h"
36 : #include "vcn/vcn_4_0_0_sh_mask.h"
37 : #include "ivsrcid/vcn/irqsrcs_vcn_4_0.h"
38 :
39 : #include <drm/drm_drv.h>
40 :
41 : #define mmUVD_DPG_LMA_CTL regUVD_DPG_LMA_CTL
42 : #define mmUVD_DPG_LMA_CTL_BASE_IDX regUVD_DPG_LMA_CTL_BASE_IDX
43 : #define mmUVD_DPG_LMA_DATA regUVD_DPG_LMA_DATA
44 : #define mmUVD_DPG_LMA_DATA_BASE_IDX regUVD_DPG_LMA_DATA_BASE_IDX
45 :
46 : #define VCN_VID_SOC_ADDRESS_2_0 0x1fb00
47 : #define VCN1_VID_SOC_ADDRESS_3_0 0x48300
48 :
49 : #define VCN_HARVEST_MMSCH 0
50 :
51 : #define RDECODE_MSG_CREATE 0x00000000
52 : #define RDECODE_MESSAGE_CREATE 0x00000001
53 :
54 : static int amdgpu_ih_clientid_vcns[] = {
55 : SOC15_IH_CLIENTID_VCN,
56 : SOC15_IH_CLIENTID_VCN1
57 : };
58 :
59 : static int vcn_v4_0_start_sriov(struct amdgpu_device *adev);
60 : static void vcn_v4_0_set_unified_ring_funcs(struct amdgpu_device *adev);
61 : static void vcn_v4_0_set_irq_funcs(struct amdgpu_device *adev);
62 : static int vcn_v4_0_set_powergating_state(void *handle,
63 : enum amd_powergating_state state);
64 : static int vcn_v4_0_pause_dpg_mode(struct amdgpu_device *adev,
65 : int inst_idx, struct dpg_pause_state *new_state);
66 : static void vcn_v4_0_unified_ring_set_wptr(struct amdgpu_ring *ring);
67 :
68 : /**
69 : * vcn_v4_0_early_init - set function pointers
70 : *
71 : * @handle: amdgpu_device pointer
72 : *
73 : * Set ring and irq function pointers
74 : */
75 0 : static int vcn_v4_0_early_init(void *handle)
76 : {
77 0 : struct amdgpu_device *adev = (struct amdgpu_device *)handle;
78 :
79 0 : if (amdgpu_sriov_vf(adev))
80 0 : adev->vcn.harvest_config = VCN_HARVEST_MMSCH;
81 :
82 : /* re-use enc ring as unified ring */
83 0 : adev->vcn.num_enc_rings = 1;
84 :
85 0 : vcn_v4_0_set_unified_ring_funcs(adev);
86 0 : vcn_v4_0_set_irq_funcs(adev);
87 :
88 0 : return 0;
89 : }
90 :
91 : /**
92 : * vcn_v4_0_sw_init - sw init for VCN block
93 : *
94 : * @handle: amdgpu_device pointer
95 : *
96 : * Load firmware and sw initialization
97 : */
98 0 : static int vcn_v4_0_sw_init(void *handle)
99 : {
100 : struct amdgpu_ring *ring;
101 0 : struct amdgpu_device *adev = (struct amdgpu_device *)handle;
102 : int i, r;
103 0 : int vcn_doorbell_index = 0;
104 :
105 0 : r = amdgpu_vcn_sw_init(adev);
106 0 : if (r)
107 : return r;
108 :
109 0 : amdgpu_vcn_setup_ucode(adev);
110 :
111 0 : r = amdgpu_vcn_resume(adev);
112 0 : if (r)
113 : return r;
114 :
115 0 : if (amdgpu_sriov_vf(adev)) {
116 0 : vcn_doorbell_index = adev->doorbell_index.vcn.vcn_ring0_1 - MMSCH_DOORBELL_OFFSET;
117 : /* get DWORD offset */
118 0 : vcn_doorbell_index = vcn_doorbell_index << 1;
119 : }
120 :
121 0 : for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
122 : volatile struct amdgpu_vcn4_fw_shared *fw_shared;
123 :
124 0 : if (adev->vcn.harvest_config & (1 << i))
125 0 : continue;
126 :
127 0 : atomic_set(&adev->vcn.inst[i].sched_score, 0);
128 :
129 : /* VCN UNIFIED TRAP */
130 0 : r = amdgpu_irq_add_id(adev, amdgpu_ih_clientid_vcns[i],
131 : VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst[i].irq);
132 0 : if (r)
133 : return r;
134 :
135 0 : ring = &adev->vcn.inst[i].ring_enc[0];
136 0 : ring->use_doorbell = true;
137 0 : if (amdgpu_sriov_vf(adev))
138 0 : ring->doorbell_index = vcn_doorbell_index + i * (adev->vcn.num_enc_rings + 1) + 1;
139 : else
140 0 : ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + 8 * i;
141 :
142 0 : sprintf(ring->name, "vcn_unified_%d", i);
143 :
144 0 : r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[i].irq, 0,
145 : AMDGPU_RING_PRIO_0, &adev->vcn.inst[i].sched_score);
146 0 : if (r)
147 : return r;
148 :
149 0 : fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
150 0 : fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE);
151 0 : fw_shared->sq.is_enabled = 1;
152 :
153 0 : if (amdgpu_sriov_vf(adev))
154 0 : fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG);
155 :
156 0 : if (amdgpu_vcnfw_log)
157 0 : amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]);
158 : }
159 :
160 0 : if (amdgpu_sriov_vf(adev)) {
161 0 : r = amdgpu_virt_alloc_mm_table(adev);
162 0 : if (r)
163 : return r;
164 : }
165 :
166 0 : if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
167 0 : adev->vcn.pause_dpg_mode = vcn_v4_0_pause_dpg_mode;
168 :
169 : return 0;
170 : }
171 :
172 : /**
173 : * vcn_v4_0_sw_fini - sw fini for VCN block
174 : *
175 : * @handle: amdgpu_device pointer
176 : *
177 : * VCN suspend and free up sw allocation
178 : */
179 0 : static int vcn_v4_0_sw_fini(void *handle)
180 : {
181 0 : struct amdgpu_device *adev = (struct amdgpu_device *)handle;
182 : int i, r, idx;
183 :
184 0 : if (drm_dev_enter(adev_to_drm(adev), &idx)) {
185 0 : for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
186 : volatile struct amdgpu_vcn4_fw_shared *fw_shared;
187 :
188 0 : if (adev->vcn.harvest_config & (1 << i))
189 0 : continue;
190 :
191 0 : fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
192 0 : fw_shared->present_flag_0 = 0;
193 0 : fw_shared->sq.is_enabled = 0;
194 : }
195 :
196 0 : drm_dev_exit(idx);
197 : }
198 :
199 0 : if (amdgpu_sriov_vf(adev))
200 0 : amdgpu_virt_free_mm_table(adev);
201 :
202 0 : r = amdgpu_vcn_suspend(adev);
203 0 : if (r)
204 : return r;
205 :
206 0 : r = amdgpu_vcn_sw_fini(adev);
207 :
208 0 : return r;
209 : }
210 :
211 : /**
212 : * vcn_v4_0_hw_init - start and test VCN block
213 : *
214 : * @handle: amdgpu_device pointer
215 : *
216 : * Initialize the hardware, boot up the VCPU and do some testing
217 : */
218 0 : static int vcn_v4_0_hw_init(void *handle)
219 : {
220 0 : struct amdgpu_device *adev = (struct amdgpu_device *)handle;
221 : struct amdgpu_ring *ring;
222 : int i, r;
223 :
224 0 : if (amdgpu_sriov_vf(adev)) {
225 0 : r = vcn_v4_0_start_sriov(adev);
226 0 : if (r)
227 : goto done;
228 :
229 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
230 0 : if (adev->vcn.harvest_config & (1 << i))
231 0 : continue;
232 :
233 0 : ring = &adev->vcn.inst[i].ring_enc[0];
234 0 : if (amdgpu_vcn_is_disabled_vcn(adev, VCN_ENCODE_RING, i)) {
235 0 : ring->sched.ready = false;
236 0 : ring->no_scheduler = true;
237 0 : dev_info(adev->dev, "ring %s is disabled by hypervisor\n", ring->name);
238 : } else {
239 0 : ring->wptr = 0;
240 0 : ring->wptr_old = 0;
241 0 : vcn_v4_0_unified_ring_set_wptr(ring);
242 0 : ring->sched.ready = true;
243 : }
244 : }
245 : } else {
246 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
247 0 : if (adev->vcn.harvest_config & (1 << i))
248 0 : continue;
249 :
250 0 : ring = &adev->vcn.inst[i].ring_enc[0];
251 :
252 0 : adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
253 0 : ((adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i), i);
254 :
255 0 : r = amdgpu_ring_test_helper(ring);
256 0 : if (r)
257 : goto done;
258 :
259 : }
260 : }
261 :
262 : done:
263 0 : if (!r)
264 0 : DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
265 : (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)?"DPG Mode":"SPG Mode");
266 :
267 0 : return r;
268 : }
269 :
270 : /**
271 : * vcn_v4_0_hw_fini - stop the hardware block
272 : *
273 : * @handle: amdgpu_device pointer
274 : *
275 : * Stop the VCN block, mark ring as not ready any more
276 : */
277 0 : static int vcn_v4_0_hw_fini(void *handle)
278 : {
279 0 : struct amdgpu_device *adev = (struct amdgpu_device *)handle;
280 : int i;
281 :
282 0 : cancel_delayed_work_sync(&adev->vcn.idle_work);
283 :
284 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
285 0 : if (adev->vcn.harvest_config & (1 << i))
286 0 : continue;
287 0 : if (!amdgpu_sriov_vf(adev)) {
288 0 : if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
289 0 : (adev->vcn.cur_state != AMD_PG_STATE_GATE &&
290 0 : RREG32_SOC15(VCN, i, regUVD_STATUS))) {
291 0 : vcn_v4_0_set_powergating_state(adev, AMD_PG_STATE_GATE);
292 : }
293 : }
294 :
295 : }
296 :
297 0 : return 0;
298 : }
299 :
300 : /**
301 : * vcn_v4_0_suspend - suspend VCN block
302 : *
303 : * @handle: amdgpu_device pointer
304 : *
305 : * HW fini and suspend VCN block
306 : */
307 0 : static int vcn_v4_0_suspend(void *handle)
308 : {
309 : int r;
310 0 : struct amdgpu_device *adev = (struct amdgpu_device *)handle;
311 :
312 0 : r = vcn_v4_0_hw_fini(adev);
313 0 : if (r)
314 : return r;
315 :
316 0 : r = amdgpu_vcn_suspend(adev);
317 :
318 0 : return r;
319 : }
320 :
321 : /**
322 : * vcn_v4_0_resume - resume VCN block
323 : *
324 : * @handle: amdgpu_device pointer
325 : *
326 : * Resume firmware and hw init VCN block
327 : */
328 0 : static int vcn_v4_0_resume(void *handle)
329 : {
330 : int r;
331 0 : struct amdgpu_device *adev = (struct amdgpu_device *)handle;
332 :
333 0 : r = amdgpu_vcn_resume(adev);
334 0 : if (r)
335 : return r;
336 :
337 0 : r = vcn_v4_0_hw_init(adev);
338 :
339 0 : return r;
340 : }
341 :
342 : /**
343 : * vcn_v4_0_mc_resume - memory controller programming
344 : *
345 : * @adev: amdgpu_device pointer
346 : * @inst: instance number
347 : *
348 : * Let the VCN memory controller know it's offsets
349 : */
350 0 : static void vcn_v4_0_mc_resume(struct amdgpu_device *adev, int inst)
351 : {
352 : uint32_t offset, size;
353 : const struct common_firmware_header *hdr;
354 :
355 0 : hdr = (const struct common_firmware_header *)adev->vcn.fw->data;
356 0 : size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
357 :
358 : /* cache window 0: fw */
359 0 : if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
360 0 : WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
361 : (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_lo));
362 0 : WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
363 : (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst].tmr_mc_addr_hi));
364 0 : WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET0, 0);
365 : offset = 0;
366 : } else {
367 0 : WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
368 : lower_32_bits(adev->vcn.inst[inst].gpu_addr));
369 0 : WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
370 : upper_32_bits(adev->vcn.inst[inst].gpu_addr));
371 0 : offset = size;
372 0 : WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET0, AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
373 : }
374 0 : WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE0, size);
375 :
376 : /* cache window 1: stack */
377 0 : WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
378 : lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
379 0 : WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
380 : upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset));
381 0 : WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET1, 0);
382 0 : WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_STACK_SIZE);
383 :
384 : /* cache window 2: context */
385 0 : WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
386 : lower_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
387 0 : WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
388 : upper_32_bits(adev->vcn.inst[inst].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE));
389 0 : WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_OFFSET2, 0);
390 0 : WREG32_SOC15(VCN, inst, regUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_CONTEXT_SIZE);
391 :
392 : /* non-cache window */
393 0 : WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW,
394 : lower_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
395 0 : WREG32_SOC15(VCN, inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH,
396 : upper_32_bits(adev->vcn.inst[inst].fw_shared.gpu_addr));
397 0 : WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_OFFSET0, 0);
398 0 : WREG32_SOC15(VCN, inst, regUVD_VCPU_NONCACHE_SIZE0,
399 : AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)));
400 0 : }
401 :
402 : /**
403 : * vcn_v4_0_mc_resume_dpg_mode - memory controller programming for dpg mode
404 : *
405 : * @adev: amdgpu_device pointer
406 : * @inst_idx: instance number index
407 : * @indirect: indirectly write sram
408 : *
409 : * Let the VCN memory controller know it's offsets with dpg mode
410 : */
411 0 : static void vcn_v4_0_mc_resume_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
412 : {
413 : uint32_t offset, size;
414 : const struct common_firmware_header *hdr;
415 0 : hdr = (const struct common_firmware_header *)adev->vcn.fw->data;
416 0 : size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8);
417 :
418 : /* cache window 0: fw */
419 0 : if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
420 0 : if (!indirect) {
421 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
422 : VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
423 : (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_lo), 0, indirect);
424 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
425 : VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
426 : (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx].tmr_mc_addr_hi), 0, indirect);
427 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
428 : VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
429 : } else {
430 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
431 : VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect);
432 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
433 : VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect);
434 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
435 : VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect);
436 : }
437 : offset = 0;
438 : } else {
439 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
440 : VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
441 : lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
442 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
443 : VCN, inst_idx, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
444 : upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect);
445 0 : offset = size;
446 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
447 : VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET0),
448 : AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect);
449 : }
450 :
451 0 : if (!indirect)
452 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
453 : VCN, inst_idx, regUVD_VCPU_CACHE_SIZE0), size, 0, indirect);
454 : else
455 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
456 : VCN, inst_idx, regUVD_VCPU_CACHE_SIZE0), 0, 0, indirect);
457 :
458 : /* cache window 1: stack */
459 0 : if (!indirect) {
460 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
461 : VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
462 : lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
463 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
464 : VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
465 : upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect);
466 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
467 : VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
468 : } else {
469 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
470 : VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect);
471 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
472 : VCN, inst_idx, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect);
473 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
474 : VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect);
475 : }
476 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
477 : VCN, inst_idx, regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect);
478 :
479 : /* cache window 2: context */
480 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
481 : VCN, inst_idx, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
482 : lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
483 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
484 : VCN, inst_idx, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
485 : upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + AMDGPU_VCN_STACK_SIZE), 0, indirect);
486 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
487 : VCN, inst_idx, regUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect);
488 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
489 : VCN, inst_idx, regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect);
490 :
491 : /* non-cache window */
492 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
493 : VCN, inst_idx, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
494 : lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
495 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
496 : VCN, inst_idx, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
497 : upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect);
498 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
499 : VCN, inst_idx, regUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect);
500 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
501 : VCN, inst_idx, regUVD_VCPU_NONCACHE_SIZE0),
502 : AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)), 0, indirect);
503 :
504 : /* VCN global tiling registers */
505 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
506 : VCN, 0, regUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect);
507 0 : }
508 :
509 : /**
510 : * vcn_v4_0_disable_static_power_gating - disable VCN static power gating
511 : *
512 : * @adev: amdgpu_device pointer
513 : * @inst: instance number
514 : *
515 : * Disable static power gating for VCN block
516 : */
517 0 : static void vcn_v4_0_disable_static_power_gating(struct amdgpu_device *adev, int inst)
518 : {
519 0 : uint32_t data = 0;
520 :
521 0 : if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
522 0 : data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
523 : | 1 << UVD_PGFSM_CONFIG__UVDS_PWR_CONFIG__SHIFT
524 : | 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
525 : | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
526 : | 2 << UVD_PGFSM_CONFIG__UVDTC_PWR_CONFIG__SHIFT
527 : | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
528 : | 2 << UVD_PGFSM_CONFIG__UVDTA_PWR_CONFIG__SHIFT
529 : | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
530 : | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
531 : | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
532 : | 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
533 : | 2 << UVD_PGFSM_CONFIG__UVDTB_PWR_CONFIG__SHIFT
534 : | 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
535 : | 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
536 :
537 0 : WREG32_SOC15(VCN, inst, regUVD_PGFSM_CONFIG, data);
538 0 : SOC15_WAIT_ON_RREG(VCN, inst, regUVD_PGFSM_STATUS,
539 : UVD_PGFSM_STATUS__UVDM_UVDU_UVDLM_PWR_ON_3_0, 0x3F3FFFFF);
540 : } else {
541 : uint32_t value;
542 :
543 0 : value = (inst) ? 0x2200800 : 0;
544 0 : data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
545 : | 1 << UVD_PGFSM_CONFIG__UVDS_PWR_CONFIG__SHIFT
546 : | 1 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
547 : | 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
548 : | 1 << UVD_PGFSM_CONFIG__UVDTC_PWR_CONFIG__SHIFT
549 : | 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
550 : | 1 << UVD_PGFSM_CONFIG__UVDTA_PWR_CONFIG__SHIFT
551 : | 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
552 : | 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
553 : | 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
554 : | 1 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
555 : | 1 << UVD_PGFSM_CONFIG__UVDTB_PWR_CONFIG__SHIFT
556 : | 1 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
557 : | 1 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
558 :
559 0 : WREG32_SOC15(VCN, inst, regUVD_PGFSM_CONFIG, data);
560 0 : SOC15_WAIT_ON_RREG(VCN, inst, regUVD_PGFSM_STATUS, value, 0x3F3FFFFF);
561 : }
562 :
563 0 : data = RREG32_SOC15(VCN, inst, regUVD_POWER_STATUS);
564 0 : data &= ~0x103;
565 0 : if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
566 0 : data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON |
567 : UVD_POWER_STATUS__UVD_PG_EN_MASK;
568 :
569 0 : WREG32_SOC15(VCN, inst, regUVD_POWER_STATUS, data);
570 :
571 0 : return;
572 : }
573 :
574 : /**
575 : * vcn_v4_0_enable_static_power_gating - enable VCN static power gating
576 : *
577 : * @adev: amdgpu_device pointer
578 : * @inst: instance number
579 : *
580 : * Enable static power gating for VCN block
581 : */
582 0 : static void vcn_v4_0_enable_static_power_gating(struct amdgpu_device *adev, int inst)
583 : {
584 : uint32_t data;
585 :
586 0 : if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
587 : /* Before power off, this indicator has to be turned on */
588 0 : data = RREG32_SOC15(VCN, inst, regUVD_POWER_STATUS);
589 0 : data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
590 0 : data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
591 0 : WREG32_SOC15(VCN, inst, regUVD_POWER_STATUS, data);
592 :
593 0 : data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
594 : | 2 << UVD_PGFSM_CONFIG__UVDS_PWR_CONFIG__SHIFT
595 : | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
596 : | 2 << UVD_PGFSM_CONFIG__UVDTC_PWR_CONFIG__SHIFT
597 : | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
598 : | 2 << UVD_PGFSM_CONFIG__UVDTA_PWR_CONFIG__SHIFT
599 : | 2 << UVD_PGFSM_CONFIG__UVDLM_PWR_CONFIG__SHIFT
600 : | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
601 : | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
602 : | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
603 : | 2 << UVD_PGFSM_CONFIG__UVDAB_PWR_CONFIG__SHIFT
604 : | 2 << UVD_PGFSM_CONFIG__UVDTB_PWR_CONFIG__SHIFT
605 : | 2 << UVD_PGFSM_CONFIG__UVDNA_PWR_CONFIG__SHIFT
606 : | 2 << UVD_PGFSM_CONFIG__UVDNB_PWR_CONFIG__SHIFT);
607 0 : WREG32_SOC15(VCN, inst, regUVD_PGFSM_CONFIG, data);
608 :
609 0 : data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT
610 : | 2 << UVD_PGFSM_STATUS__UVDS_PWR_STATUS__SHIFT
611 : | 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT
612 : | 2 << UVD_PGFSM_STATUS__UVDTC_PWR_STATUS__SHIFT
613 : | 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT
614 : | 2 << UVD_PGFSM_STATUS__UVDTA_PWR_STATUS__SHIFT
615 : | 2 << UVD_PGFSM_STATUS__UVDLM_PWR_STATUS__SHIFT
616 : | 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT
617 : | 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT
618 : | 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT
619 : | 2 << UVD_PGFSM_STATUS__UVDAB_PWR_STATUS__SHIFT
620 : | 2 << UVD_PGFSM_STATUS__UVDTB_PWR_STATUS__SHIFT
621 : | 2 << UVD_PGFSM_STATUS__UVDNA_PWR_STATUS__SHIFT
622 : | 2 << UVD_PGFSM_STATUS__UVDNB_PWR_STATUS__SHIFT);
623 0 : SOC15_WAIT_ON_RREG(VCN, inst, regUVD_PGFSM_STATUS, data, 0x3F3FFFFF);
624 : }
625 :
626 0 : return;
627 : }
628 :
629 : /**
630 : * vcn_v4_0_disable_clock_gating - disable VCN clock gating
631 : *
632 : * @adev: amdgpu_device pointer
633 : * @inst: instance number
634 : *
635 : * Disable clock gating for VCN block
636 : */
637 0 : static void vcn_v4_0_disable_clock_gating(struct amdgpu_device *adev, int inst)
638 : {
639 : uint32_t data;
640 :
641 0 : if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
642 : return;
643 :
644 : /* VCN disable CGC */
645 0 : data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
646 0 : data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
647 0 : data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
648 0 : data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
649 0 : WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
650 :
651 0 : data = RREG32_SOC15(VCN, inst, regUVD_CGC_GATE);
652 0 : data &= ~(UVD_CGC_GATE__SYS_MASK
653 : | UVD_CGC_GATE__UDEC_MASK
654 : | UVD_CGC_GATE__MPEG2_MASK
655 : | UVD_CGC_GATE__REGS_MASK
656 : | UVD_CGC_GATE__RBC_MASK
657 : | UVD_CGC_GATE__LMI_MC_MASK
658 : | UVD_CGC_GATE__LMI_UMC_MASK
659 : | UVD_CGC_GATE__IDCT_MASK
660 : | UVD_CGC_GATE__MPRD_MASK
661 : | UVD_CGC_GATE__MPC_MASK
662 : | UVD_CGC_GATE__LBSI_MASK
663 : | UVD_CGC_GATE__LRBBM_MASK
664 : | UVD_CGC_GATE__UDEC_RE_MASK
665 : | UVD_CGC_GATE__UDEC_CM_MASK
666 : | UVD_CGC_GATE__UDEC_IT_MASK
667 : | UVD_CGC_GATE__UDEC_DB_MASK
668 : | UVD_CGC_GATE__UDEC_MP_MASK
669 : | UVD_CGC_GATE__WCB_MASK
670 : | UVD_CGC_GATE__VCPU_MASK
671 : | UVD_CGC_GATE__MMSCH_MASK);
672 :
673 0 : WREG32_SOC15(VCN, inst, regUVD_CGC_GATE, data);
674 0 : SOC15_WAIT_ON_RREG(VCN, inst, regUVD_CGC_GATE, 0, 0xFFFFFFFF);
675 :
676 0 : data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
677 0 : data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
678 : | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
679 : | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
680 : | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
681 : | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
682 : | UVD_CGC_CTRL__SYS_MODE_MASK
683 : | UVD_CGC_CTRL__UDEC_MODE_MASK
684 : | UVD_CGC_CTRL__MPEG2_MODE_MASK
685 : | UVD_CGC_CTRL__REGS_MODE_MASK
686 : | UVD_CGC_CTRL__RBC_MODE_MASK
687 : | UVD_CGC_CTRL__LMI_MC_MODE_MASK
688 : | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
689 : | UVD_CGC_CTRL__IDCT_MODE_MASK
690 : | UVD_CGC_CTRL__MPRD_MODE_MASK
691 : | UVD_CGC_CTRL__MPC_MODE_MASK
692 : | UVD_CGC_CTRL__LBSI_MODE_MASK
693 : | UVD_CGC_CTRL__LRBBM_MODE_MASK
694 : | UVD_CGC_CTRL__WCB_MODE_MASK
695 : | UVD_CGC_CTRL__VCPU_MODE_MASK
696 : | UVD_CGC_CTRL__MMSCH_MODE_MASK);
697 0 : WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
698 :
699 0 : data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_GATE);
700 0 : data |= (UVD_SUVD_CGC_GATE__SRE_MASK
701 : | UVD_SUVD_CGC_GATE__SIT_MASK
702 : | UVD_SUVD_CGC_GATE__SMP_MASK
703 : | UVD_SUVD_CGC_GATE__SCM_MASK
704 : | UVD_SUVD_CGC_GATE__SDB_MASK
705 : | UVD_SUVD_CGC_GATE__SRE_H264_MASK
706 : | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
707 : | UVD_SUVD_CGC_GATE__SIT_H264_MASK
708 : | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
709 : | UVD_SUVD_CGC_GATE__SCM_H264_MASK
710 : | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
711 : | UVD_SUVD_CGC_GATE__SDB_H264_MASK
712 : | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
713 : | UVD_SUVD_CGC_GATE__SCLR_MASK
714 : | UVD_SUVD_CGC_GATE__UVD_SC_MASK
715 : | UVD_SUVD_CGC_GATE__ENT_MASK
716 : | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
717 : | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
718 : | UVD_SUVD_CGC_GATE__SITE_MASK
719 : | UVD_SUVD_CGC_GATE__SRE_VP9_MASK
720 : | UVD_SUVD_CGC_GATE__SCM_VP9_MASK
721 : | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
722 : | UVD_SUVD_CGC_GATE__SDB_VP9_MASK
723 : | UVD_SUVD_CGC_GATE__IME_HEVC_MASK);
724 0 : WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_GATE, data);
725 :
726 0 : data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL);
727 0 : data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
728 : | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
729 : | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
730 : | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
731 : | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
732 : | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
733 : | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
734 : | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
735 : | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
736 : | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
737 0 : WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL, data);
738 : }
739 :
740 : /**
741 : * vcn_v4_0_disable_clock_gating_dpg_mode - disable VCN clock gating dpg mode
742 : *
743 : * @adev: amdgpu_device pointer
744 : * @sram_sel: sram select
745 : * @inst_idx: instance number index
746 : * @indirect: indirectly write sram
747 : *
748 : * Disable clock gating for VCN block with dpg mode
749 : */
750 0 : static void vcn_v4_0_disable_clock_gating_dpg_mode(struct amdgpu_device *adev, uint8_t sram_sel,
751 : int inst_idx, uint8_t indirect)
752 : {
753 0 : uint32_t reg_data = 0;
754 :
755 0 : if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
756 : return;
757 :
758 : /* enable sw clock gating control */
759 0 : reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
760 0 : reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
761 0 : reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
762 0 : reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
763 : UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
764 : UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
765 : UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
766 : UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
767 : UVD_CGC_CTRL__SYS_MODE_MASK |
768 : UVD_CGC_CTRL__UDEC_MODE_MASK |
769 : UVD_CGC_CTRL__MPEG2_MODE_MASK |
770 : UVD_CGC_CTRL__REGS_MODE_MASK |
771 : UVD_CGC_CTRL__RBC_MODE_MASK |
772 : UVD_CGC_CTRL__LMI_MC_MODE_MASK |
773 : UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
774 : UVD_CGC_CTRL__IDCT_MODE_MASK |
775 : UVD_CGC_CTRL__MPRD_MODE_MASK |
776 : UVD_CGC_CTRL__MPC_MODE_MASK |
777 : UVD_CGC_CTRL__LBSI_MODE_MASK |
778 : UVD_CGC_CTRL__LRBBM_MODE_MASK |
779 : UVD_CGC_CTRL__WCB_MODE_MASK |
780 : UVD_CGC_CTRL__VCPU_MODE_MASK);
781 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
782 : VCN, inst_idx, regUVD_CGC_CTRL), reg_data, sram_sel, indirect);
783 :
784 : /* turn off clock gating */
785 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
786 : VCN, inst_idx, regUVD_CGC_GATE), 0, sram_sel, indirect);
787 :
788 : /* turn on SUVD clock gating */
789 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
790 : VCN, inst_idx, regUVD_SUVD_CGC_GATE), 1, sram_sel, indirect);
791 :
792 : /* turn on sw mode in UVD_SUVD_CGC_CTRL */
793 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
794 : VCN, inst_idx, regUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect);
795 : }
796 :
797 : /**
798 : * vcn_v4_0_enable_clock_gating - enable VCN clock gating
799 : *
800 : * @adev: amdgpu_device pointer
801 : * @inst: instance number
802 : *
803 : * Enable clock gating for VCN block
804 : */
805 0 : static void vcn_v4_0_enable_clock_gating(struct amdgpu_device *adev, int inst)
806 : {
807 : uint32_t data;
808 :
809 0 : if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
810 : return;
811 :
812 : /* enable VCN CGC */
813 0 : data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
814 0 : data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
815 0 : data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
816 0 : data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
817 0 : WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
818 :
819 0 : data = RREG32_SOC15(VCN, inst, regUVD_CGC_CTRL);
820 0 : data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
821 : | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
822 : | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
823 : | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
824 : | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
825 : | UVD_CGC_CTRL__SYS_MODE_MASK
826 : | UVD_CGC_CTRL__UDEC_MODE_MASK
827 : | UVD_CGC_CTRL__MPEG2_MODE_MASK
828 : | UVD_CGC_CTRL__REGS_MODE_MASK
829 : | UVD_CGC_CTRL__RBC_MODE_MASK
830 : | UVD_CGC_CTRL__LMI_MC_MODE_MASK
831 : | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
832 : | UVD_CGC_CTRL__IDCT_MODE_MASK
833 : | UVD_CGC_CTRL__MPRD_MODE_MASK
834 : | UVD_CGC_CTRL__MPC_MODE_MASK
835 : | UVD_CGC_CTRL__LBSI_MODE_MASK
836 : | UVD_CGC_CTRL__LRBBM_MODE_MASK
837 : | UVD_CGC_CTRL__WCB_MODE_MASK
838 : | UVD_CGC_CTRL__VCPU_MODE_MASK
839 : | UVD_CGC_CTRL__MMSCH_MODE_MASK);
840 0 : WREG32_SOC15(VCN, inst, regUVD_CGC_CTRL, data);
841 :
842 0 : data = RREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL);
843 0 : data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
844 : | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
845 : | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
846 : | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
847 : | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
848 : | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
849 : | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
850 : | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
851 : | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
852 : | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
853 0 : WREG32_SOC15(VCN, inst, regUVD_SUVD_CGC_CTRL, data);
854 :
855 : return;
856 : }
857 :
858 : /**
859 : * vcn_v4_0_start_dpg_mode - VCN start with dpg mode
860 : *
861 : * @adev: amdgpu_device pointer
862 : * @inst_idx: instance number index
863 : * @indirect: indirectly write sram
864 : *
865 : * Start VCN block with dpg mode
866 : */
867 0 : static int vcn_v4_0_start_dpg_mode(struct amdgpu_device *adev, int inst_idx, bool indirect)
868 : {
869 0 : volatile struct amdgpu_vcn4_fw_shared *fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr;
870 : struct amdgpu_ring *ring;
871 : uint32_t tmp;
872 :
873 : /* disable register anti-hang mechanism */
874 0 : WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, regUVD_POWER_STATUS), 1,
875 : ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
876 : /* enable dynamic power gating mode */
877 0 : tmp = RREG32_SOC15(VCN, inst_idx, regUVD_POWER_STATUS);
878 0 : tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
879 0 : tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
880 0 : WREG32_SOC15(VCN, inst_idx, regUVD_POWER_STATUS, tmp);
881 :
882 0 : if (indirect)
883 0 : adev->vcn.inst[inst_idx].dpg_sram_curr_addr = (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr;
884 :
885 : /* enable clock gating */
886 0 : vcn_v4_0_disable_clock_gating_dpg_mode(adev, 0, inst_idx, indirect);
887 :
888 : /* enable VCPU clock */
889 0 : tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
890 0 : tmp |= UVD_VCPU_CNTL__CLK_EN_MASK | UVD_VCPU_CNTL__BLK_RST_MASK;
891 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
892 : VCN, inst_idx, regUVD_VCPU_CNTL), tmp, 0, indirect);
893 :
894 : /* disable master interupt */
895 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
896 : VCN, inst_idx, regUVD_MASTINT_EN), 0, 0, indirect);
897 :
898 : /* setup regUVD_LMI_CTRL */
899 0 : tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
900 : UVD_LMI_CTRL__REQ_MODE_MASK |
901 : UVD_LMI_CTRL__CRC_RESET_MASK |
902 : UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
903 : UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
904 : UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
905 : (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
906 : 0x00100000L);
907 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
908 : VCN, inst_idx, regUVD_LMI_CTRL), tmp, 0, indirect);
909 :
910 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
911 : VCN, inst_idx, regUVD_MPC_CNTL),
912 : 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect);
913 :
914 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
915 : VCN, inst_idx, regUVD_MPC_SET_MUXA0),
916 : ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
917 : (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
918 : (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
919 : (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect);
920 :
921 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
922 : VCN, inst_idx, regUVD_MPC_SET_MUXB0),
923 : ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
924 : (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
925 : (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
926 : (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect);
927 :
928 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
929 : VCN, inst_idx, regUVD_MPC_SET_MUX),
930 : ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
931 : (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
932 : (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect);
933 :
934 0 : vcn_v4_0_mc_resume_dpg_mode(adev, inst_idx, indirect);
935 :
936 0 : tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
937 0 : tmp |= UVD_VCPU_CNTL__CLK_EN_MASK;
938 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
939 : VCN, inst_idx, regUVD_VCPU_CNTL), tmp, 0, indirect);
940 :
941 : /* enable LMI MC and UMC channels */
942 0 : tmp = 0x1f << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT;
943 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
944 : VCN, inst_idx, regUVD_LMI_CTRL2), tmp, 0, indirect);
945 :
946 : /* enable master interrupt */
947 0 : WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET(
948 : VCN, inst_idx, regUVD_MASTINT_EN),
949 : UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect);
950 :
951 :
952 0 : if (indirect)
953 0 : psp_update_vcn_sram(adev, inst_idx, adev->vcn.inst[inst_idx].dpg_sram_gpu_addr,
954 0 : (uint32_t)((uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_curr_addr -
955 0 : (uintptr_t)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr));
956 :
957 0 : ring = &adev->vcn.inst[inst_idx].ring_enc[0];
958 :
959 0 : WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_LO, ring->gpu_addr);
960 0 : WREG32_SOC15(VCN, inst_idx, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
961 0 : WREG32_SOC15(VCN, inst_idx, regUVD_RB_SIZE, ring->ring_size / 4);
962 :
963 0 : tmp = RREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE);
964 0 : tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);
965 0 : WREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE, tmp);
966 0 : fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
967 0 : WREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR, 0);
968 0 : WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR, 0);
969 :
970 0 : tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_RPTR);
971 0 : WREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR, tmp);
972 0 : ring->wptr = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR);
973 :
974 0 : tmp = RREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE);
975 0 : tmp |= VCN_RB_ENABLE__RB1_EN_MASK;
976 0 : WREG32_SOC15(VCN, inst_idx, regVCN_RB_ENABLE, tmp);
977 0 : fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
978 :
979 0 : WREG32_SOC15(VCN, inst_idx, regVCN_RB1_DB_CTRL,
980 : ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
981 : VCN_RB1_DB_CTRL__EN_MASK);
982 :
983 0 : return 0;
984 : }
985 :
986 :
987 : /**
988 : * vcn_v4_0_start - VCN start
989 : *
990 : * @adev: amdgpu_device pointer
991 : *
992 : * Start VCN block
993 : */
994 0 : static int vcn_v4_0_start(struct amdgpu_device *adev)
995 : {
996 : volatile struct amdgpu_vcn4_fw_shared *fw_shared;
997 : struct amdgpu_ring *ring;
998 : uint32_t tmp;
999 : int i, j, k, r;
1000 :
1001 0 : if (adev->pm.dpm_enabled)
1002 0 : amdgpu_dpm_enable_uvd(adev, true);
1003 :
1004 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1005 0 : fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1006 :
1007 0 : if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1008 0 : r = vcn_v4_0_start_dpg_mode(adev, i, adev->vcn.indirect_sram);
1009 0 : continue;
1010 : }
1011 :
1012 : /* disable VCN power gating */
1013 0 : vcn_v4_0_disable_static_power_gating(adev, i);
1014 :
1015 : /* set VCN status busy */
1016 0 : tmp = RREG32_SOC15(VCN, i, regUVD_STATUS) | UVD_STATUS__UVD_BUSY;
1017 0 : WREG32_SOC15(VCN, i, regUVD_STATUS, tmp);
1018 :
1019 : /*SW clock gating */
1020 0 : vcn_v4_0_disable_clock_gating(adev, i);
1021 :
1022 : /* enable VCPU clock */
1023 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
1024 : UVD_VCPU_CNTL__CLK_EN_MASK, ~UVD_VCPU_CNTL__CLK_EN_MASK);
1025 :
1026 : /* disable master interrupt */
1027 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_MASTINT_EN), 0,
1028 : ~UVD_MASTINT_EN__VCPU_EN_MASK);
1029 :
1030 : /* enable LMI MC and UMC channels */
1031 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_LMI_CTRL2), 0,
1032 : ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1033 :
1034 0 : tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1035 0 : tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1036 0 : tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1037 0 : WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1038 :
1039 : /* setup regUVD_LMI_CTRL */
1040 0 : tmp = RREG32_SOC15(VCN, i, regUVD_LMI_CTRL);
1041 0 : WREG32_SOC15(VCN, i, regUVD_LMI_CTRL, tmp |
1042 : UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1043 : UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1044 : UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1045 : UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK);
1046 :
1047 : /* setup regUVD_MPC_CNTL */
1048 0 : tmp = RREG32_SOC15(VCN, i, regUVD_MPC_CNTL);
1049 0 : tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK;
1050 0 : tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT;
1051 0 : WREG32_SOC15(VCN, i, regUVD_MPC_CNTL, tmp);
1052 :
1053 : /* setup UVD_MPC_SET_MUXA0 */
1054 0 : WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUXA0,
1055 : ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) |
1056 : (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) |
1057 : (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) |
1058 : (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)));
1059 :
1060 : /* setup UVD_MPC_SET_MUXB0 */
1061 0 : WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUXB0,
1062 : ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) |
1063 : (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) |
1064 : (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) |
1065 : (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)));
1066 :
1067 : /* setup UVD_MPC_SET_MUX */
1068 0 : WREG32_SOC15(VCN, i, regUVD_MPC_SET_MUX,
1069 : ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) |
1070 : (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) |
1071 : (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)));
1072 :
1073 0 : vcn_v4_0_mc_resume(adev, i);
1074 :
1075 : /* VCN global tiling registers */
1076 0 : WREG32_SOC15(VCN, i, regUVD_GFX10_ADDR_CONFIG,
1077 : adev->gfx.config.gb_addr_config);
1078 :
1079 : /* unblock VCPU register access */
1080 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_RB_ARB_CTRL), 0,
1081 : ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1082 :
1083 : /* release VCPU reset to boot */
1084 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
1085 : ~UVD_VCPU_CNTL__BLK_RST_MASK);
1086 :
1087 0 : for (j = 0; j < 10; ++j) {
1088 : uint32_t status;
1089 :
1090 0 : for (k = 0; k < 100; ++k) {
1091 0 : status = RREG32_SOC15(VCN, i, regUVD_STATUS);
1092 0 : if (status & 2)
1093 : break;
1094 0 : mdelay(10);
1095 0 : if (amdgpu_emu_mode==1)
1096 0 : msleep(1);
1097 : }
1098 :
1099 0 : if (amdgpu_emu_mode==1) {
1100 0 : r = -1;
1101 0 : if (status & 2) {
1102 : r = 0;
1103 : break;
1104 : }
1105 : } else {
1106 0 : r = 0;
1107 0 : if (status & 2)
1108 : break;
1109 :
1110 0 : dev_err(adev->dev, "VCN[%d] is not responding, trying to reset the VCPU!!!\n", i);
1111 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
1112 : UVD_VCPU_CNTL__BLK_RST_MASK,
1113 : ~UVD_VCPU_CNTL__BLK_RST_MASK);
1114 0 : mdelay(10);
1115 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
1116 : ~UVD_VCPU_CNTL__BLK_RST_MASK);
1117 :
1118 0 : mdelay(10);
1119 : r = -1;
1120 : }
1121 : }
1122 :
1123 0 : if (r) {
1124 0 : dev_err(adev->dev, "VCN[%d] is not responding, giving up!!!\n", i);
1125 0 : return r;
1126 : }
1127 :
1128 : /* enable master interrupt */
1129 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_MASTINT_EN),
1130 : UVD_MASTINT_EN__VCPU_EN_MASK,
1131 : ~UVD_MASTINT_EN__VCPU_EN_MASK);
1132 :
1133 : /* clear the busy bit of VCN_STATUS */
1134 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_STATUS), 0,
1135 : ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
1136 :
1137 0 : ring = &adev->vcn.inst[i].ring_enc[0];
1138 0 : WREG32_SOC15(VCN, i, regVCN_RB1_DB_CTRL,
1139 : ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT |
1140 : VCN_RB1_DB_CTRL__EN_MASK);
1141 :
1142 0 : WREG32_SOC15(VCN, i, regUVD_RB_BASE_LO, ring->gpu_addr);
1143 0 : WREG32_SOC15(VCN, i, regUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
1144 0 : WREG32_SOC15(VCN, i, regUVD_RB_SIZE, ring->ring_size / 4);
1145 :
1146 0 : tmp = RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);
1147 0 : tmp &= ~(VCN_RB_ENABLE__RB1_EN_MASK);
1148 0 : WREG32_SOC15(VCN, i, regVCN_RB_ENABLE, tmp);
1149 0 : fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET;
1150 0 : WREG32_SOC15(VCN, i, regUVD_RB_RPTR, 0);
1151 0 : WREG32_SOC15(VCN, i, regUVD_RB_WPTR, 0);
1152 :
1153 0 : tmp = RREG32_SOC15(VCN, i, regUVD_RB_RPTR);
1154 0 : WREG32_SOC15(VCN, i, regUVD_RB_WPTR, tmp);
1155 0 : ring->wptr = RREG32_SOC15(VCN, i, regUVD_RB_WPTR);
1156 :
1157 0 : tmp = RREG32_SOC15(VCN, i, regVCN_RB_ENABLE);
1158 0 : tmp |= VCN_RB_ENABLE__RB1_EN_MASK;
1159 0 : WREG32_SOC15(VCN, i, regVCN_RB_ENABLE, tmp);
1160 0 : fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF);
1161 : }
1162 :
1163 : return 0;
1164 : }
1165 :
1166 0 : static int vcn_v4_0_start_sriov(struct amdgpu_device *adev)
1167 : {
1168 : int i;
1169 : struct amdgpu_ring *ring_enc;
1170 : uint64_t cache_addr;
1171 : uint64_t rb_enc_addr;
1172 : uint64_t ctx_addr;
1173 : uint32_t param, resp, expected;
1174 : uint32_t offset, cache_size;
1175 : uint32_t tmp, timeout;
1176 :
1177 0 : struct amdgpu_mm_table *table = &adev->virt.mm_table;
1178 : uint32_t *table_loc;
1179 : uint32_t table_size;
1180 : uint32_t size, size_dw;
1181 : uint32_t init_status;
1182 : uint32_t enabled_vcn;
1183 :
1184 : struct mmsch_v4_0_cmd_direct_write
1185 0 : direct_wt = { {0} };
1186 : struct mmsch_v4_0_cmd_direct_read_modify_write
1187 0 : direct_rd_mod_wt = { {0} };
1188 0 : struct mmsch_v4_0_cmd_end end = { {0} };
1189 : struct mmsch_v4_0_init_header header;
1190 :
1191 : volatile struct amdgpu_vcn4_fw_shared *fw_shared;
1192 : volatile struct amdgpu_fw_shared_rb_setup *rb_setup;
1193 :
1194 : direct_wt.cmd_header.command_type =
1195 : MMSCH_COMMAND__DIRECT_REG_WRITE;
1196 0 : direct_rd_mod_wt.cmd_header.command_type =
1197 : MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
1198 0 : end.cmd_header.command_type =
1199 : MMSCH_COMMAND__END;
1200 :
1201 0 : header.version = MMSCH_VERSION;
1202 0 : header.total_size = sizeof(struct mmsch_v4_0_init_header) >> 2;
1203 0 : for (i = 0; i < AMDGPU_MAX_VCN_INSTANCES; i++) {
1204 0 : header.inst[i].init_status = 0;
1205 0 : header.inst[i].table_offset = 0;
1206 0 : header.inst[i].table_size = 0;
1207 : }
1208 :
1209 0 : table_loc = (uint32_t *)table->cpu_addr;
1210 0 : table_loc += header.total_size;
1211 0 : for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
1212 0 : if (adev->vcn.harvest_config & (1 << i))
1213 0 : continue;
1214 :
1215 0 : table_size = 0;
1216 :
1217 0 : MMSCH_V4_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, i,
1218 : regUVD_STATUS),
1219 : ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);
1220 :
1221 0 : cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
1222 :
1223 0 : if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1224 0 : MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1225 : regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1226 : adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo);
1227 0 : MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1228 : regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1229 : adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi);
1230 0 : offset = 0;
1231 0 : MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1232 : regUVD_VCPU_CACHE_OFFSET0),
1233 : 0);
1234 : } else {
1235 0 : MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1236 : regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
1237 : lower_32_bits(adev->vcn.inst[i].gpu_addr));
1238 0 : MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1239 : regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
1240 : upper_32_bits(adev->vcn.inst[i].gpu_addr));
1241 0 : offset = cache_size;
1242 0 : MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1243 : regUVD_VCPU_CACHE_OFFSET0),
1244 : AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
1245 : }
1246 :
1247 0 : MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1248 : regUVD_VCPU_CACHE_SIZE0),
1249 : cache_size);
1250 :
1251 0 : cache_addr = adev->vcn.inst[i].gpu_addr + offset;
1252 0 : MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1253 : regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
1254 : lower_32_bits(cache_addr));
1255 0 : MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1256 : regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
1257 : upper_32_bits(cache_addr));
1258 0 : MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1259 : regUVD_VCPU_CACHE_OFFSET1),
1260 : 0);
1261 0 : MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1262 : regUVD_VCPU_CACHE_SIZE1),
1263 : AMDGPU_VCN_STACK_SIZE);
1264 :
1265 0 : cache_addr = adev->vcn.inst[i].gpu_addr + offset +
1266 : AMDGPU_VCN_STACK_SIZE;
1267 0 : MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1268 : regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
1269 : lower_32_bits(cache_addr));
1270 0 : MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1271 : regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
1272 : upper_32_bits(cache_addr));
1273 0 : MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1274 : regUVD_VCPU_CACHE_OFFSET2),
1275 : 0);
1276 0 : MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1277 : regUVD_VCPU_CACHE_SIZE2),
1278 : AMDGPU_VCN_CONTEXT_SIZE);
1279 :
1280 0 : fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1281 0 : rb_setup = &fw_shared->rb_setup;
1282 :
1283 0 : ring_enc = &adev->vcn.inst[i].ring_enc[0];
1284 0 : ring_enc->wptr = 0;
1285 0 : rb_enc_addr = ring_enc->gpu_addr;
1286 :
1287 0 : rb_setup->is_rb_enabled_flags |= RB_ENABLED;
1288 0 : rb_setup->rb_addr_lo = lower_32_bits(rb_enc_addr);
1289 0 : rb_setup->rb_addr_hi = upper_32_bits(rb_enc_addr);
1290 0 : rb_setup->rb_size = ring_enc->ring_size / 4;
1291 0 : fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG);
1292 :
1293 0 : MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1294 : regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
1295 : lower_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr));
1296 0 : MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1297 : regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
1298 : upper_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr));
1299 0 : MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
1300 : regUVD_VCPU_NONCACHE_SIZE0),
1301 : AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)));
1302 :
1303 : /* add end packet */
1304 0 : MMSCH_V4_0_INSERT_END();
1305 :
1306 : /* refine header */
1307 0 : header.inst[i].init_status = 0;
1308 0 : header.inst[i].table_offset = header.total_size;
1309 0 : header.inst[i].table_size = table_size;
1310 0 : header.total_size += table_size;
1311 : }
1312 :
1313 : /* Update init table header in memory */
1314 0 : size = sizeof(struct mmsch_v4_0_init_header);
1315 0 : table_loc = (uint32_t *)table->cpu_addr;
1316 0 : memcpy((void *)table_loc, &header, size);
1317 :
1318 : /* message MMSCH (in VCN[0]) to initialize this client
1319 : * 1, write to mmsch_vf_ctx_addr_lo/hi register with GPU mc addr
1320 : * of memory descriptor location
1321 : */
1322 0 : ctx_addr = table->gpu_addr;
1323 0 : WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
1324 0 : WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
1325 :
1326 : /* 2, update vmid of descriptor */
1327 0 : tmp = RREG32_SOC15(VCN, 0, regMMSCH_VF_VMID);
1328 0 : tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
1329 : /* use domain0 for MM scheduler */
1330 0 : tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
1331 0 : WREG32_SOC15(VCN, 0, regMMSCH_VF_VMID, tmp);
1332 :
1333 : /* 3, notify mmsch about the size of this descriptor */
1334 0 : size = header.total_size;
1335 0 : WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_SIZE, size);
1336 :
1337 : /* 4, set resp to zero */
1338 0 : WREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_RESP, 0);
1339 :
1340 : /* 5, kick off the initialization and wait until
1341 : * MMSCH_VF_MAILBOX_RESP becomes non-zero
1342 : */
1343 0 : param = 0x00000001;
1344 0 : WREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_HOST, param);
1345 : tmp = 0;
1346 : timeout = 1000;
1347 : resp = 0;
1348 : expected = MMSCH_VF_MAILBOX_RESP__OK;
1349 0 : while (resp != expected) {
1350 0 : resp = RREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_RESP);
1351 0 : if (resp != 0)
1352 : break;
1353 :
1354 0 : udelay(10);
1355 0 : tmp = tmp + 10;
1356 0 : if (tmp >= timeout) {
1357 0 : DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
1358 : " waiting for regMMSCH_VF_MAILBOX_RESP "\
1359 : "(expected=0x%08x, readback=0x%08x)\n",
1360 : tmp, expected, resp);
1361 0 : return -EBUSY;
1362 : }
1363 : }
1364 0 : enabled_vcn = amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, 0) ? 1 : 0;
1365 0 : init_status = ((struct mmsch_v4_0_init_header *)(table_loc))->inst[enabled_vcn].init_status;
1366 0 : if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE
1367 0 : && init_status != MMSCH_VF_ENGINE_STATUS__PASS)
1368 0 : DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init "\
1369 : "status for VCN%x: 0x%x\n", resp, enabled_vcn, init_status);
1370 :
1371 : return 0;
1372 : }
1373 :
1374 : /**
1375 : * vcn_v4_0_stop_dpg_mode - VCN stop with dpg mode
1376 : *
1377 : * @adev: amdgpu_device pointer
1378 : * @inst_idx: instance number index
1379 : *
1380 : * Stop VCN block with dpg mode
1381 : */
1382 0 : static void vcn_v4_0_stop_dpg_mode(struct amdgpu_device *adev, int inst_idx)
1383 : {
1384 : uint32_t tmp;
1385 :
1386 : /* Wait for power status to be 1 */
1387 0 : SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 1,
1388 : UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1389 :
1390 : /* wait for read ptr to be equal to write ptr */
1391 0 : tmp = RREG32_SOC15(VCN, inst_idx, regUVD_RB_WPTR);
1392 0 : SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_RB_RPTR, tmp, 0xFFFFFFFF);
1393 :
1394 0 : SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 1,
1395 : UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1396 :
1397 : /* disable dynamic power gating mode */
1398 0 : WREG32_P(SOC15_REG_OFFSET(VCN, inst_idx, regUVD_POWER_STATUS), 0,
1399 : ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1400 0 : }
1401 :
1402 : /**
1403 : * vcn_v4_0_stop - VCN stop
1404 : *
1405 : * @adev: amdgpu_device pointer
1406 : *
1407 : * Stop VCN block
1408 : */
1409 0 : static int vcn_v4_0_stop(struct amdgpu_device *adev)
1410 : {
1411 : volatile struct amdgpu_vcn4_fw_shared *fw_shared;
1412 : uint32_t tmp;
1413 0 : int i, r = 0;
1414 :
1415 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1416 0 : fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
1417 0 : fw_shared->sq.queue_mode |= FW_QUEUE_DPG_HOLD_OFF;
1418 :
1419 0 : if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) {
1420 0 : vcn_v4_0_stop_dpg_mode(adev, i);
1421 0 : continue;
1422 : }
1423 :
1424 : /* wait for vcn idle */
1425 0 : r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_STATUS, UVD_STATUS__IDLE, 0x7);
1426 0 : if (r)
1427 : return r;
1428 :
1429 0 : tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK |
1430 : UVD_LMI_STATUS__READ_CLEAN_MASK |
1431 : UVD_LMI_STATUS__WRITE_CLEAN_MASK |
1432 : UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK;
1433 0 : r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_LMI_STATUS, tmp, tmp);
1434 0 : if (r)
1435 : return r;
1436 :
1437 : /* disable LMI UMC channel */
1438 0 : tmp = RREG32_SOC15(VCN, i, regUVD_LMI_CTRL2);
1439 0 : tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK;
1440 0 : WREG32_SOC15(VCN, i, regUVD_LMI_CTRL2, tmp);
1441 0 : tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK |
1442 : UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK;
1443 0 : r = SOC15_WAIT_ON_RREG(VCN, i, regUVD_LMI_STATUS, tmp, tmp);
1444 0 : if (r)
1445 : return r;
1446 :
1447 : /* block VCPU register access */
1448 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_RB_ARB_CTRL),
1449 : UVD_RB_ARB_CTRL__VCPU_DIS_MASK,
1450 : ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK);
1451 :
1452 : /* reset VCPU */
1453 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL),
1454 : UVD_VCPU_CNTL__BLK_RST_MASK,
1455 : ~UVD_VCPU_CNTL__BLK_RST_MASK);
1456 :
1457 : /* disable VCPU clock */
1458 0 : WREG32_P(SOC15_REG_OFFSET(VCN, i, regUVD_VCPU_CNTL), 0,
1459 : ~(UVD_VCPU_CNTL__CLK_EN_MASK));
1460 :
1461 : /* apply soft reset */
1462 0 : tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1463 0 : tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK;
1464 0 : WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1465 0 : tmp = RREG32_SOC15(VCN, i, regUVD_SOFT_RESET);
1466 0 : tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK;
1467 0 : WREG32_SOC15(VCN, i, regUVD_SOFT_RESET, tmp);
1468 :
1469 : /* clear status */
1470 0 : WREG32_SOC15(VCN, i, regUVD_STATUS, 0);
1471 :
1472 : /* apply HW clock gating */
1473 0 : vcn_v4_0_enable_clock_gating(adev, i);
1474 :
1475 : /* enable VCN power gating */
1476 0 : vcn_v4_0_enable_static_power_gating(adev, i);
1477 : }
1478 :
1479 0 : if (adev->pm.dpm_enabled)
1480 0 : amdgpu_dpm_enable_uvd(adev, false);
1481 :
1482 : return 0;
1483 : }
1484 :
1485 : /**
1486 : * vcn_v4_0_pause_dpg_mode - VCN pause with dpg mode
1487 : *
1488 : * @adev: amdgpu_device pointer
1489 : * @inst_idx: instance number index
1490 : * @new_state: pause state
1491 : *
1492 : * Pause dpg mode for VCN block
1493 : */
1494 0 : static int vcn_v4_0_pause_dpg_mode(struct amdgpu_device *adev, int inst_idx,
1495 : struct dpg_pause_state *new_state)
1496 : {
1497 0 : uint32_t reg_data = 0;
1498 : int ret_code;
1499 :
1500 : /* pause/unpause if state is changed */
1501 0 : if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) {
1502 0 : DRM_DEV_DEBUG(adev->dev, "dpg pause state changed %d -> %d",
1503 : adev->vcn.inst[inst_idx].pause_state.fw_based, new_state->fw_based);
1504 0 : reg_data = RREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE) &
1505 : (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1506 :
1507 0 : if (new_state->fw_based == VCN_DPG_STATE__PAUSE) {
1508 0 : ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 0x1,
1509 : UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1510 :
1511 0 : if (!ret_code) {
1512 : /* pause DPG */
1513 0 : reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1514 0 : WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);
1515 :
1516 : /* wait for ACK */
1517 0 : SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_DPG_PAUSE,
1518 : UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK,
1519 : UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK);
1520 :
1521 0 : SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS,
1522 : UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON, UVD_POWER_STATUS__UVD_POWER_STATUS_MASK);
1523 : }
1524 : } else {
1525 : /* unpause dpg, no need to wait */
1526 0 : reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK;
1527 0 : WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data);
1528 : }
1529 0 : adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based;
1530 : }
1531 :
1532 0 : return 0;
1533 : }
1534 :
1535 : /**
1536 : * vcn_v4_0_unified_ring_get_rptr - get unified read pointer
1537 : *
1538 : * @ring: amdgpu_ring pointer
1539 : *
1540 : * Returns the current hardware unified read pointer
1541 : */
1542 0 : static uint64_t vcn_v4_0_unified_ring_get_rptr(struct amdgpu_ring *ring)
1543 : {
1544 0 : struct amdgpu_device *adev = ring->adev;
1545 :
1546 0 : if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1547 0 : DRM_ERROR("wrong ring id is identified in %s", __func__);
1548 :
1549 0 : return RREG32_SOC15(VCN, ring->me, regUVD_RB_RPTR);
1550 : }
1551 :
1552 : /**
1553 : * vcn_v4_0_unified_ring_get_wptr - get unified write pointer
1554 : *
1555 : * @ring: amdgpu_ring pointer
1556 : *
1557 : * Returns the current hardware unified write pointer
1558 : */
1559 0 : static uint64_t vcn_v4_0_unified_ring_get_wptr(struct amdgpu_ring *ring)
1560 : {
1561 0 : struct amdgpu_device *adev = ring->adev;
1562 :
1563 0 : if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1564 0 : DRM_ERROR("wrong ring id is identified in %s", __func__);
1565 :
1566 0 : if (ring->use_doorbell)
1567 0 : return *ring->wptr_cpu_addr;
1568 : else
1569 0 : return RREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR);
1570 : }
1571 :
1572 : /**
1573 : * vcn_v4_0_unified_ring_set_wptr - set enc write pointer
1574 : *
1575 : * @ring: amdgpu_ring pointer
1576 : *
1577 : * Commits the enc write pointer to the hardware
1578 : */
1579 0 : static void vcn_v4_0_unified_ring_set_wptr(struct amdgpu_ring *ring)
1580 : {
1581 0 : struct amdgpu_device *adev = ring->adev;
1582 :
1583 0 : if (ring != &adev->vcn.inst[ring->me].ring_enc[0])
1584 0 : DRM_ERROR("wrong ring id is identified in %s", __func__);
1585 :
1586 0 : if (ring->use_doorbell) {
1587 0 : *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
1588 0 : WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
1589 : } else {
1590 0 : WREG32_SOC15(VCN, ring->me, regUVD_RB_WPTR, lower_32_bits(ring->wptr));
1591 : }
1592 0 : }
1593 :
1594 : static int vcn_v4_0_limit_sched(struct amdgpu_cs_parser *p)
1595 : {
1596 : struct drm_gpu_scheduler **scheds;
1597 :
1598 : /* The create msg must be in the first IB submitted */
1599 0 : if (atomic_read(&p->entity->fence_seq))
1600 : return -EINVAL;
1601 :
1602 0 : scheds = p->adev->gpu_sched[AMDGPU_HW_IP_VCN_ENC]
1603 : [AMDGPU_RING_PRIO_0].sched;
1604 0 : drm_sched_entity_modify_sched(p->entity, scheds, 1);
1605 : return 0;
1606 : }
1607 :
1608 0 : static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, uint64_t addr)
1609 : {
1610 0 : struct ttm_operation_ctx ctx = { false, false };
1611 : struct amdgpu_bo_va_mapping *map;
1612 : uint32_t *msg, num_buffers;
1613 : struct amdgpu_bo *bo;
1614 : uint64_t start, end;
1615 : unsigned int i;
1616 : void *ptr;
1617 : int r;
1618 :
1619 0 : addr &= AMDGPU_GMC_HOLE_MASK;
1620 0 : r = amdgpu_cs_find_mapping(p, addr, &bo, &map);
1621 0 : if (r) {
1622 0 : DRM_ERROR("Can't find BO for addr 0x%08llx\n", addr);
1623 0 : return r;
1624 : }
1625 :
1626 0 : start = map->start * AMDGPU_GPU_PAGE_SIZE;
1627 0 : end = (map->last + 1) * AMDGPU_GPU_PAGE_SIZE;
1628 0 : if (addr & 0x7) {
1629 0 : DRM_ERROR("VCN messages must be 8 byte aligned!\n");
1630 0 : return -EINVAL;
1631 : }
1632 :
1633 0 : bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
1634 0 : amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
1635 0 : r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1636 0 : if (r) {
1637 0 : DRM_ERROR("Failed validating the VCN message BO (%d)!\n", r);
1638 0 : return r;
1639 : }
1640 :
1641 0 : r = amdgpu_bo_kmap(bo, &ptr);
1642 0 : if (r) {
1643 0 : DRM_ERROR("Failed mapping the VCN message (%d)!\n", r);
1644 0 : return r;
1645 : }
1646 :
1647 0 : msg = ptr + addr - start;
1648 :
1649 : /* Check length */
1650 0 : if (msg[1] > end - addr) {
1651 : r = -EINVAL;
1652 : goto out;
1653 : }
1654 :
1655 0 : if (msg[3] != RDECODE_MSG_CREATE)
1656 : goto out;
1657 :
1658 0 : num_buffers = msg[2];
1659 0 : for (i = 0, msg = &msg[6]; i < num_buffers; ++i, msg += 4) {
1660 : uint32_t offset, size, *create;
1661 :
1662 0 : if (msg[0] != RDECODE_MESSAGE_CREATE)
1663 0 : continue;
1664 :
1665 0 : offset = msg[1];
1666 0 : size = msg[2];
1667 :
1668 0 : if (offset + size > end) {
1669 : r = -EINVAL;
1670 : goto out;
1671 : }
1672 :
1673 0 : create = ptr + addr + offset - start;
1674 :
1675 : /* H246, HEVC and VP9 can run on any instance */
1676 0 : if (create[0] == 0x7 || create[0] == 0x10 || create[0] == 0x11)
1677 0 : continue;
1678 :
1679 0 : r = vcn_v4_0_limit_sched(p);
1680 0 : if (r)
1681 : goto out;
1682 : }
1683 :
1684 : out:
1685 0 : amdgpu_bo_kunmap(bo);
1686 0 : return r;
1687 : }
1688 :
1689 : #define RADEON_VCN_ENGINE_TYPE_DECODE (0x00000003)
1690 :
1691 0 : static int vcn_v4_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,
1692 : struct amdgpu_job *job,
1693 : struct amdgpu_ib *ib)
1694 : {
1695 0 : struct amdgpu_ring *ring = to_amdgpu_ring(p->entity->rq->sched);
1696 0 : struct amdgpu_vcn_decode_buffer *decode_buffer = NULL;
1697 : uint32_t val;
1698 0 : int r = 0;
1699 :
1700 : /* The first instance can decode anything */
1701 0 : if (!ring->me)
1702 : return r;
1703 :
1704 : /* unified queue ib header has 8 double words. */
1705 0 : if (ib->length_dw < 8)
1706 : return r;
1707 :
1708 0 : val = amdgpu_ib_get_value(ib, 6); //RADEON_VCN_ENGINE_TYPE
1709 :
1710 0 : if (val == RADEON_VCN_ENGINE_TYPE_DECODE) {
1711 0 : decode_buffer = (struct amdgpu_vcn_decode_buffer *)&ib->ptr[10];
1712 :
1713 0 : if (decode_buffer->valid_buf_flag & 0x1)
1714 0 : r = vcn_v4_0_dec_msg(p, ((u64)decode_buffer->msg_buffer_address_hi) << 32 |
1715 0 : decode_buffer->msg_buffer_address_lo);
1716 : }
1717 : return r;
1718 : }
1719 :
1720 : static const struct amdgpu_ring_funcs vcn_v4_0_unified_ring_vm_funcs = {
1721 : .type = AMDGPU_RING_TYPE_VCN_ENC,
1722 : .align_mask = 0x3f,
1723 : .nop = VCN_ENC_CMD_NO_OP,
1724 : .vmhub = AMDGPU_MMHUB_0,
1725 : .get_rptr = vcn_v4_0_unified_ring_get_rptr,
1726 : .get_wptr = vcn_v4_0_unified_ring_get_wptr,
1727 : .set_wptr = vcn_v4_0_unified_ring_set_wptr,
1728 : .patch_cs_in_place = vcn_v4_0_ring_patch_cs_in_place,
1729 : .emit_frame_size =
1730 : SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1731 : SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1732 : 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */
1733 : 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */
1734 : 1, /* vcn_v2_0_enc_ring_insert_end */
1735 : .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */
1736 : .emit_ib = vcn_v2_0_enc_ring_emit_ib,
1737 : .emit_fence = vcn_v2_0_enc_ring_emit_fence,
1738 : .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush,
1739 : .test_ring = amdgpu_vcn_enc_ring_test_ring,
1740 : .test_ib = amdgpu_vcn_unified_ring_test_ib,
1741 : .insert_nop = amdgpu_ring_insert_nop,
1742 : .insert_end = vcn_v2_0_enc_ring_insert_end,
1743 : .pad_ib = amdgpu_ring_generic_pad_ib,
1744 : .begin_use = amdgpu_vcn_ring_begin_use,
1745 : .end_use = amdgpu_vcn_ring_end_use,
1746 : .emit_wreg = vcn_v2_0_enc_ring_emit_wreg,
1747 : .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait,
1748 : .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1749 : };
1750 :
1751 : /**
1752 : * vcn_v4_0_set_unified_ring_funcs - set unified ring functions
1753 : *
1754 : * @adev: amdgpu_device pointer
1755 : *
1756 : * Set unified ring functions
1757 : */
1758 0 : static void vcn_v4_0_set_unified_ring_funcs(struct amdgpu_device *adev)
1759 : {
1760 : int i;
1761 :
1762 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1763 0 : if (adev->vcn.harvest_config & (1 << i))
1764 0 : continue;
1765 :
1766 0 : adev->vcn.inst[i].ring_enc[0].funcs = &vcn_v4_0_unified_ring_vm_funcs;
1767 0 : adev->vcn.inst[i].ring_enc[0].me = i;
1768 :
1769 0 : DRM_INFO("VCN(%d) encode/decode are enabled in VM mode\n", i);
1770 : }
1771 0 : }
1772 :
1773 : /**
1774 : * vcn_v4_0_is_idle - check VCN block is idle
1775 : *
1776 : * @handle: amdgpu_device pointer
1777 : *
1778 : * Check whether VCN block is idle
1779 : */
1780 0 : static bool vcn_v4_0_is_idle(void *handle)
1781 : {
1782 0 : struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1783 0 : int i, ret = 1;
1784 :
1785 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1786 0 : if (adev->vcn.harvest_config & (1 << i))
1787 0 : continue;
1788 :
1789 0 : ret &= (RREG32_SOC15(VCN, i, regUVD_STATUS) == UVD_STATUS__IDLE);
1790 : }
1791 :
1792 0 : return ret;
1793 : }
1794 :
1795 : /**
1796 : * vcn_v4_0_wait_for_idle - wait for VCN block idle
1797 : *
1798 : * @handle: amdgpu_device pointer
1799 : *
1800 : * Wait for VCN block idle
1801 : */
1802 0 : static int vcn_v4_0_wait_for_idle(void *handle)
1803 : {
1804 0 : struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1805 0 : int i, ret = 0;
1806 :
1807 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1808 0 : if (adev->vcn.harvest_config & (1 << i))
1809 0 : continue;
1810 :
1811 0 : ret = SOC15_WAIT_ON_RREG(VCN, i, regUVD_STATUS, UVD_STATUS__IDLE,
1812 : UVD_STATUS__IDLE);
1813 0 : if (ret)
1814 : return ret;
1815 : }
1816 :
1817 : return ret;
1818 : }
1819 :
1820 : /**
1821 : * vcn_v4_0_set_clockgating_state - set VCN block clockgating state
1822 : *
1823 : * @handle: amdgpu_device pointer
1824 : * @state: clock gating state
1825 : *
1826 : * Set VCN block clockgating state
1827 : */
1828 0 : static int vcn_v4_0_set_clockgating_state(void *handle, enum amd_clockgating_state state)
1829 : {
1830 0 : struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1831 0 : bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
1832 : int i;
1833 :
1834 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1835 0 : if (adev->vcn.harvest_config & (1 << i))
1836 0 : continue;
1837 :
1838 0 : if (enable) {
1839 0 : if (RREG32_SOC15(VCN, i, regUVD_STATUS) != UVD_STATUS__IDLE)
1840 : return -EBUSY;
1841 0 : vcn_v4_0_enable_clock_gating(adev, i);
1842 : } else {
1843 0 : vcn_v4_0_disable_clock_gating(adev, i);
1844 : }
1845 : }
1846 :
1847 : return 0;
1848 : }
1849 :
1850 : /**
1851 : * vcn_v4_0_set_powergating_state - set VCN block powergating state
1852 : *
1853 : * @handle: amdgpu_device pointer
1854 : * @state: power gating state
1855 : *
1856 : * Set VCN block powergating state
1857 : */
1858 0 : static int vcn_v4_0_set_powergating_state(void *handle, enum amd_powergating_state state)
1859 : {
1860 0 : struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1861 : int ret;
1862 :
1863 : /* for SRIOV, guest should not control VCN Power-gating
1864 : * MMSCH FW should control Power-gating and clock-gating
1865 : * guest should avoid touching CGC and PG
1866 : */
1867 0 : if (amdgpu_sriov_vf(adev)) {
1868 0 : adev->vcn.cur_state = AMD_PG_STATE_UNGATE;
1869 0 : return 0;
1870 : }
1871 :
1872 0 : if(state == adev->vcn.cur_state)
1873 : return 0;
1874 :
1875 0 : if (state == AMD_PG_STATE_GATE)
1876 0 : ret = vcn_v4_0_stop(adev);
1877 : else
1878 0 : ret = vcn_v4_0_start(adev);
1879 :
1880 0 : if(!ret)
1881 0 : adev->vcn.cur_state = state;
1882 :
1883 : return ret;
1884 : }
1885 :
1886 : /**
1887 : * vcn_v4_0_set_interrupt_state - set VCN block interrupt state
1888 : *
1889 : * @adev: amdgpu_device pointer
1890 : * @source: interrupt sources
1891 : * @type: interrupt types
1892 : * @state: interrupt states
1893 : *
1894 : * Set VCN block interrupt state
1895 : */
1896 0 : static int vcn_v4_0_set_interrupt_state(struct amdgpu_device *adev, struct amdgpu_irq_src *source,
1897 : unsigned type, enum amdgpu_interrupt_state state)
1898 : {
1899 0 : return 0;
1900 : }
1901 :
1902 : /**
1903 : * vcn_v4_0_process_interrupt - process VCN block interrupt
1904 : *
1905 : * @adev: amdgpu_device pointer
1906 : * @source: interrupt sources
1907 : * @entry: interrupt entry from clients and sources
1908 : *
1909 : * Process VCN block interrupt
1910 : */
1911 0 : static int vcn_v4_0_process_interrupt(struct amdgpu_device *adev, struct amdgpu_irq_src *source,
1912 : struct amdgpu_iv_entry *entry)
1913 : {
1914 : uint32_t ip_instance;
1915 :
1916 0 : switch (entry->client_id) {
1917 : case SOC15_IH_CLIENTID_VCN:
1918 : ip_instance = 0;
1919 : break;
1920 : case SOC15_IH_CLIENTID_VCN1:
1921 0 : ip_instance = 1;
1922 0 : break;
1923 : default:
1924 0 : DRM_ERROR("Unhandled client id: %d\n", entry->client_id);
1925 0 : return 0;
1926 : }
1927 :
1928 0 : DRM_DEBUG("IH: VCN TRAP\n");
1929 :
1930 0 : switch (entry->src_id) {
1931 : case VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE:
1932 0 : amdgpu_fence_process(&adev->vcn.inst[ip_instance].ring_enc[0]);
1933 0 : break;
1934 : default:
1935 0 : DRM_ERROR("Unhandled interrupt: %d %d\n",
1936 : entry->src_id, entry->src_data[0]);
1937 0 : break;
1938 : }
1939 :
1940 : return 0;
1941 : }
1942 :
1943 : static const struct amdgpu_irq_src_funcs vcn_v4_0_irq_funcs = {
1944 : .set = vcn_v4_0_set_interrupt_state,
1945 : .process = vcn_v4_0_process_interrupt,
1946 : };
1947 :
1948 : /**
1949 : * vcn_v4_0_set_irq_funcs - set VCN block interrupt irq functions
1950 : *
1951 : * @adev: amdgpu_device pointer
1952 : *
1953 : * Set VCN block interrupt irq functions
1954 : */
1955 : static void vcn_v4_0_set_irq_funcs(struct amdgpu_device *adev)
1956 : {
1957 : int i;
1958 :
1959 0 : for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
1960 0 : if (adev->vcn.harvest_config & (1 << i))
1961 0 : continue;
1962 :
1963 0 : adev->vcn.inst[i].irq.num_types = adev->vcn.num_enc_rings + 1;
1964 0 : adev->vcn.inst[i].irq.funcs = &vcn_v4_0_irq_funcs;
1965 : }
1966 : }
1967 :
1968 : static const struct amd_ip_funcs vcn_v4_0_ip_funcs = {
1969 : .name = "vcn_v4_0",
1970 : .early_init = vcn_v4_0_early_init,
1971 : .late_init = NULL,
1972 : .sw_init = vcn_v4_0_sw_init,
1973 : .sw_fini = vcn_v4_0_sw_fini,
1974 : .hw_init = vcn_v4_0_hw_init,
1975 : .hw_fini = vcn_v4_0_hw_fini,
1976 : .suspend = vcn_v4_0_suspend,
1977 : .resume = vcn_v4_0_resume,
1978 : .is_idle = vcn_v4_0_is_idle,
1979 : .wait_for_idle = vcn_v4_0_wait_for_idle,
1980 : .check_soft_reset = NULL,
1981 : .pre_soft_reset = NULL,
1982 : .soft_reset = NULL,
1983 : .post_soft_reset = NULL,
1984 : .set_clockgating_state = vcn_v4_0_set_clockgating_state,
1985 : .set_powergating_state = vcn_v4_0_set_powergating_state,
1986 : };
1987 :
1988 : const struct amdgpu_ip_block_version vcn_v4_0_ip_block =
1989 : {
1990 : .type = AMD_IP_BLOCK_TYPE_VCN,
1991 : .major = 4,
1992 : .minor = 0,
1993 : .rev = 0,
1994 : .funcs = &vcn_v4_0_ip_funcs,
1995 : };
|
