2 * Copyright 2012 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
29 #include "radeon_asic.h"
32 #include "cik_blit_shaders.h"
33 #include "radeon_ucode.h"
34 #include "clearstate_ci.h"
36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
42 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
43 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
44 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
45 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
46 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
47 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
48 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
49 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
50 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
51 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
52 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
53 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
54 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
55 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
56 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
57 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
58 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
59 MODULE_FIRMWARE("radeon/KABINI_me.bin");
60 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
61 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
62 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
63 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
65 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
66 extern void r600_ih_ring_fini(struct radeon_device *rdev);
67 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
68 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
69 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
70 extern void sumo_rlc_fini(struct radeon_device *rdev);
71 extern int sumo_rlc_init(struct radeon_device *rdev);
72 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
73 extern void si_rlc_reset(struct radeon_device *rdev);
74 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
75 extern int cik_sdma_resume(struct radeon_device *rdev);
76 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
77 extern void cik_sdma_fini(struct radeon_device *rdev);
78 static void cik_rlc_stop(struct radeon_device *rdev);
79 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
80 static void cik_program_aspm(struct radeon_device *rdev);
81 static void cik_init_pg(struct radeon_device *rdev);
82 static void cik_init_cg(struct radeon_device *rdev);
83 static void cik_fini_pg(struct radeon_device *rdev);
84 static void cik_fini_cg(struct radeon_device *rdev);
85 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
88 /* get temperature in millidegrees */
89 int ci_get_temp(struct radeon_device *rdev)
94 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
100 actual_temp = temp & 0x1ff;
102 actual_temp = actual_temp * 1000;
107 /* get temperature in millidegrees */
108 int kv_get_temp(struct radeon_device *rdev)
113 temp = RREG32_SMC(0xC0300E0C);
116 actual_temp = (temp / 8) - 49;
120 actual_temp = actual_temp * 1000;
126 * Indirect registers accessor
128 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
133 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
134 WREG32(PCIE_INDEX, reg);
135 (void)RREG32(PCIE_INDEX);
136 r = RREG32(PCIE_DATA);
137 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
141 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
145 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
146 WREG32(PCIE_INDEX, reg);
147 (void)RREG32(PCIE_INDEX);
148 WREG32(PCIE_DATA, v);
149 (void)RREG32(PCIE_DATA);
150 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
153 static const u32 spectre_rlc_save_restore_register_list[] =
155 (0x0e00 << 16) | (0xc12c >> 2),
157 (0x0e00 << 16) | (0xc140 >> 2),
159 (0x0e00 << 16) | (0xc150 >> 2),
161 (0x0e00 << 16) | (0xc15c >> 2),
163 (0x0e00 << 16) | (0xc168 >> 2),
165 (0x0e00 << 16) | (0xc170 >> 2),
167 (0x0e00 << 16) | (0xc178 >> 2),
169 (0x0e00 << 16) | (0xc204 >> 2),
171 (0x0e00 << 16) | (0xc2b4 >> 2),
173 (0x0e00 << 16) | (0xc2b8 >> 2),
175 (0x0e00 << 16) | (0xc2bc >> 2),
177 (0x0e00 << 16) | (0xc2c0 >> 2),
179 (0x0e00 << 16) | (0x8228 >> 2),
181 (0x0e00 << 16) | (0x829c >> 2),
183 (0x0e00 << 16) | (0x869c >> 2),
185 (0x0600 << 16) | (0x98f4 >> 2),
187 (0x0e00 << 16) | (0x98f8 >> 2),
189 (0x0e00 << 16) | (0x9900 >> 2),
191 (0x0e00 << 16) | (0xc260 >> 2),
193 (0x0e00 << 16) | (0x90e8 >> 2),
195 (0x0e00 << 16) | (0x3c000 >> 2),
197 (0x0e00 << 16) | (0x3c00c >> 2),
199 (0x0e00 << 16) | (0x8c1c >> 2),
201 (0x0e00 << 16) | (0x9700 >> 2),
203 (0x0e00 << 16) | (0xcd20 >> 2),
205 (0x4e00 << 16) | (0xcd20 >> 2),
207 (0x5e00 << 16) | (0xcd20 >> 2),
209 (0x6e00 << 16) | (0xcd20 >> 2),
211 (0x7e00 << 16) | (0xcd20 >> 2),
213 (0x8e00 << 16) | (0xcd20 >> 2),
215 (0x9e00 << 16) | (0xcd20 >> 2),
217 (0xae00 << 16) | (0xcd20 >> 2),
219 (0xbe00 << 16) | (0xcd20 >> 2),
221 (0x0e00 << 16) | (0x89bc >> 2),
223 (0x0e00 << 16) | (0x8900 >> 2),
226 (0x0e00 << 16) | (0xc130 >> 2),
228 (0x0e00 << 16) | (0xc134 >> 2),
230 (0x0e00 << 16) | (0xc1fc >> 2),
232 (0x0e00 << 16) | (0xc208 >> 2),
234 (0x0e00 << 16) | (0xc264 >> 2),
236 (0x0e00 << 16) | (0xc268 >> 2),
238 (0x0e00 << 16) | (0xc26c >> 2),
240 (0x0e00 << 16) | (0xc270 >> 2),
242 (0x0e00 << 16) | (0xc274 >> 2),
244 (0x0e00 << 16) | (0xc278 >> 2),
246 (0x0e00 << 16) | (0xc27c >> 2),
248 (0x0e00 << 16) | (0xc280 >> 2),
250 (0x0e00 << 16) | (0xc284 >> 2),
252 (0x0e00 << 16) | (0xc288 >> 2),
254 (0x0e00 << 16) | (0xc28c >> 2),
256 (0x0e00 << 16) | (0xc290 >> 2),
258 (0x0e00 << 16) | (0xc294 >> 2),
260 (0x0e00 << 16) | (0xc298 >> 2),
262 (0x0e00 << 16) | (0xc29c >> 2),
264 (0x0e00 << 16) | (0xc2a0 >> 2),
266 (0x0e00 << 16) | (0xc2a4 >> 2),
268 (0x0e00 << 16) | (0xc2a8 >> 2),
270 (0x0e00 << 16) | (0xc2ac >> 2),
272 (0x0e00 << 16) | (0xc2b0 >> 2),
274 (0x0e00 << 16) | (0x301d0 >> 2),
276 (0x0e00 << 16) | (0x30238 >> 2),
278 (0x0e00 << 16) | (0x30250 >> 2),
280 (0x0e00 << 16) | (0x30254 >> 2),
282 (0x0e00 << 16) | (0x30258 >> 2),
284 (0x0e00 << 16) | (0x3025c >> 2),
286 (0x4e00 << 16) | (0xc900 >> 2),
288 (0x5e00 << 16) | (0xc900 >> 2),
290 (0x6e00 << 16) | (0xc900 >> 2),
292 (0x7e00 << 16) | (0xc900 >> 2),
294 (0x8e00 << 16) | (0xc900 >> 2),
296 (0x9e00 << 16) | (0xc900 >> 2),
298 (0xae00 << 16) | (0xc900 >> 2),
300 (0xbe00 << 16) | (0xc900 >> 2),
302 (0x4e00 << 16) | (0xc904 >> 2),
304 (0x5e00 << 16) | (0xc904 >> 2),
306 (0x6e00 << 16) | (0xc904 >> 2),
308 (0x7e00 << 16) | (0xc904 >> 2),
310 (0x8e00 << 16) | (0xc904 >> 2),
312 (0x9e00 << 16) | (0xc904 >> 2),
314 (0xae00 << 16) | (0xc904 >> 2),
316 (0xbe00 << 16) | (0xc904 >> 2),
318 (0x4e00 << 16) | (0xc908 >> 2),
320 (0x5e00 << 16) | (0xc908 >> 2),
322 (0x6e00 << 16) | (0xc908 >> 2),
324 (0x7e00 << 16) | (0xc908 >> 2),
326 (0x8e00 << 16) | (0xc908 >> 2),
328 (0x9e00 << 16) | (0xc908 >> 2),
330 (0xae00 << 16) | (0xc908 >> 2),
332 (0xbe00 << 16) | (0xc908 >> 2),
334 (0x4e00 << 16) | (0xc90c >> 2),
336 (0x5e00 << 16) | (0xc90c >> 2),
338 (0x6e00 << 16) | (0xc90c >> 2),
340 (0x7e00 << 16) | (0xc90c >> 2),
342 (0x8e00 << 16) | (0xc90c >> 2),
344 (0x9e00 << 16) | (0xc90c >> 2),
346 (0xae00 << 16) | (0xc90c >> 2),
348 (0xbe00 << 16) | (0xc90c >> 2),
350 (0x4e00 << 16) | (0xc910 >> 2),
352 (0x5e00 << 16) | (0xc910 >> 2),
354 (0x6e00 << 16) | (0xc910 >> 2),
356 (0x7e00 << 16) | (0xc910 >> 2),
358 (0x8e00 << 16) | (0xc910 >> 2),
360 (0x9e00 << 16) | (0xc910 >> 2),
362 (0xae00 << 16) | (0xc910 >> 2),
364 (0xbe00 << 16) | (0xc910 >> 2),
366 (0x0e00 << 16) | (0xc99c >> 2),
368 (0x0e00 << 16) | (0x9834 >> 2),
370 (0x0000 << 16) | (0x30f00 >> 2),
372 (0x0001 << 16) | (0x30f00 >> 2),
374 (0x0000 << 16) | (0x30f04 >> 2),
376 (0x0001 << 16) | (0x30f04 >> 2),
378 (0x0000 << 16) | (0x30f08 >> 2),
380 (0x0001 << 16) | (0x30f08 >> 2),
382 (0x0000 << 16) | (0x30f0c >> 2),
384 (0x0001 << 16) | (0x30f0c >> 2),
386 (0x0600 << 16) | (0x9b7c >> 2),
388 (0x0e00 << 16) | (0x8a14 >> 2),
390 (0x0e00 << 16) | (0x8a18 >> 2),
392 (0x0600 << 16) | (0x30a00 >> 2),
394 (0x0e00 << 16) | (0x8bf0 >> 2),
396 (0x0e00 << 16) | (0x8bcc >> 2),
398 (0x0e00 << 16) | (0x8b24 >> 2),
400 (0x0e00 << 16) | (0x30a04 >> 2),
402 (0x0600 << 16) | (0x30a10 >> 2),
404 (0x0600 << 16) | (0x30a14 >> 2),
406 (0x0600 << 16) | (0x30a18 >> 2),
408 (0x0600 << 16) | (0x30a2c >> 2),
410 (0x0e00 << 16) | (0xc700 >> 2),
412 (0x0e00 << 16) | (0xc704 >> 2),
414 (0x0e00 << 16) | (0xc708 >> 2),
416 (0x0e00 << 16) | (0xc768 >> 2),
418 (0x0400 << 16) | (0xc770 >> 2),
420 (0x0400 << 16) | (0xc774 >> 2),
422 (0x0400 << 16) | (0xc778 >> 2),
424 (0x0400 << 16) | (0xc77c >> 2),
426 (0x0400 << 16) | (0xc780 >> 2),
428 (0x0400 << 16) | (0xc784 >> 2),
430 (0x0400 << 16) | (0xc788 >> 2),
432 (0x0400 << 16) | (0xc78c >> 2),
434 (0x0400 << 16) | (0xc798 >> 2),
436 (0x0400 << 16) | (0xc79c >> 2),
438 (0x0400 << 16) | (0xc7a0 >> 2),
440 (0x0400 << 16) | (0xc7a4 >> 2),
442 (0x0400 << 16) | (0xc7a8 >> 2),
444 (0x0400 << 16) | (0xc7ac >> 2),
446 (0x0400 << 16) | (0xc7b0 >> 2),
448 (0x0400 << 16) | (0xc7b4 >> 2),
450 (0x0e00 << 16) | (0x9100 >> 2),
452 (0x0e00 << 16) | (0x3c010 >> 2),
454 (0x0e00 << 16) | (0x92a8 >> 2),
456 (0x0e00 << 16) | (0x92ac >> 2),
458 (0x0e00 << 16) | (0x92b4 >> 2),
460 (0x0e00 << 16) | (0x92b8 >> 2),
462 (0x0e00 << 16) | (0x92bc >> 2),
464 (0x0e00 << 16) | (0x92c0 >> 2),
466 (0x0e00 << 16) | (0x92c4 >> 2),
468 (0x0e00 << 16) | (0x92c8 >> 2),
470 (0x0e00 << 16) | (0x92cc >> 2),
472 (0x0e00 << 16) | (0x92d0 >> 2),
474 (0x0e00 << 16) | (0x8c00 >> 2),
476 (0x0e00 << 16) | (0x8c04 >> 2),
478 (0x0e00 << 16) | (0x8c20 >> 2),
480 (0x0e00 << 16) | (0x8c38 >> 2),
482 (0x0e00 << 16) | (0x8c3c >> 2),
484 (0x0e00 << 16) | (0xae00 >> 2),
486 (0x0e00 << 16) | (0x9604 >> 2),
488 (0x0e00 << 16) | (0xac08 >> 2),
490 (0x0e00 << 16) | (0xac0c >> 2),
492 (0x0e00 << 16) | (0xac10 >> 2),
494 (0x0e00 << 16) | (0xac14 >> 2),
496 (0x0e00 << 16) | (0xac58 >> 2),
498 (0x0e00 << 16) | (0xac68 >> 2),
500 (0x0e00 << 16) | (0xac6c >> 2),
502 (0x0e00 << 16) | (0xac70 >> 2),
504 (0x0e00 << 16) | (0xac74 >> 2),
506 (0x0e00 << 16) | (0xac78 >> 2),
508 (0x0e00 << 16) | (0xac7c >> 2),
510 (0x0e00 << 16) | (0xac80 >> 2),
512 (0x0e00 << 16) | (0xac84 >> 2),
514 (0x0e00 << 16) | (0xac88 >> 2),
516 (0x0e00 << 16) | (0xac8c >> 2),
518 (0x0e00 << 16) | (0x970c >> 2),
520 (0x0e00 << 16) | (0x9714 >> 2),
522 (0x0e00 << 16) | (0x9718 >> 2),
524 (0x0e00 << 16) | (0x971c >> 2),
526 (0x0e00 << 16) | (0x31068 >> 2),
528 (0x4e00 << 16) | (0x31068 >> 2),
530 (0x5e00 << 16) | (0x31068 >> 2),
532 (0x6e00 << 16) | (0x31068 >> 2),
534 (0x7e00 << 16) | (0x31068 >> 2),
536 (0x8e00 << 16) | (0x31068 >> 2),
538 (0x9e00 << 16) | (0x31068 >> 2),
540 (0xae00 << 16) | (0x31068 >> 2),
542 (0xbe00 << 16) | (0x31068 >> 2),
544 (0x0e00 << 16) | (0xcd10 >> 2),
546 (0x0e00 << 16) | (0xcd14 >> 2),
548 (0x0e00 << 16) | (0x88b0 >> 2),
550 (0x0e00 << 16) | (0x88b4 >> 2),
552 (0x0e00 << 16) | (0x88b8 >> 2),
554 (0x0e00 << 16) | (0x88bc >> 2),
556 (0x0400 << 16) | (0x89c0 >> 2),
558 (0x0e00 << 16) | (0x88c4 >> 2),
560 (0x0e00 << 16) | (0x88c8 >> 2),
562 (0x0e00 << 16) | (0x88d0 >> 2),
564 (0x0e00 << 16) | (0x88d4 >> 2),
566 (0x0e00 << 16) | (0x88d8 >> 2),
568 (0x0e00 << 16) | (0x8980 >> 2),
570 (0x0e00 << 16) | (0x30938 >> 2),
572 (0x0e00 << 16) | (0x3093c >> 2),
574 (0x0e00 << 16) | (0x30940 >> 2),
576 (0x0e00 << 16) | (0x89a0 >> 2),
578 (0x0e00 << 16) | (0x30900 >> 2),
580 (0x0e00 << 16) | (0x30904 >> 2),
582 (0x0e00 << 16) | (0x89b4 >> 2),
584 (0x0e00 << 16) | (0x3c210 >> 2),
586 (0x0e00 << 16) | (0x3c214 >> 2),
588 (0x0e00 << 16) | (0x3c218 >> 2),
590 (0x0e00 << 16) | (0x8904 >> 2),
593 (0x0e00 << 16) | (0x8c28 >> 2),
594 (0x0e00 << 16) | (0x8c2c >> 2),
595 (0x0e00 << 16) | (0x8c30 >> 2),
596 (0x0e00 << 16) | (0x8c34 >> 2),
597 (0x0e00 << 16) | (0x9600 >> 2),
600 static const u32 kalindi_rlc_save_restore_register_list[] =
602 (0x0e00 << 16) | (0xc12c >> 2),
604 (0x0e00 << 16) | (0xc140 >> 2),
606 (0x0e00 << 16) | (0xc150 >> 2),
608 (0x0e00 << 16) | (0xc15c >> 2),
610 (0x0e00 << 16) | (0xc168 >> 2),
612 (0x0e00 << 16) | (0xc170 >> 2),
614 (0x0e00 << 16) | (0xc204 >> 2),
616 (0x0e00 << 16) | (0xc2b4 >> 2),
618 (0x0e00 << 16) | (0xc2b8 >> 2),
620 (0x0e00 << 16) | (0xc2bc >> 2),
622 (0x0e00 << 16) | (0xc2c0 >> 2),
624 (0x0e00 << 16) | (0x8228 >> 2),
626 (0x0e00 << 16) | (0x829c >> 2),
628 (0x0e00 << 16) | (0x869c >> 2),
630 (0x0600 << 16) | (0x98f4 >> 2),
632 (0x0e00 << 16) | (0x98f8 >> 2),
634 (0x0e00 << 16) | (0x9900 >> 2),
636 (0x0e00 << 16) | (0xc260 >> 2),
638 (0x0e00 << 16) | (0x90e8 >> 2),
640 (0x0e00 << 16) | (0x3c000 >> 2),
642 (0x0e00 << 16) | (0x3c00c >> 2),
644 (0x0e00 << 16) | (0x8c1c >> 2),
646 (0x0e00 << 16) | (0x9700 >> 2),
648 (0x0e00 << 16) | (0xcd20 >> 2),
650 (0x4e00 << 16) | (0xcd20 >> 2),
652 (0x5e00 << 16) | (0xcd20 >> 2),
654 (0x6e00 << 16) | (0xcd20 >> 2),
656 (0x7e00 << 16) | (0xcd20 >> 2),
658 (0x0e00 << 16) | (0x89bc >> 2),
660 (0x0e00 << 16) | (0x8900 >> 2),
663 (0x0e00 << 16) | (0xc130 >> 2),
665 (0x0e00 << 16) | (0xc134 >> 2),
667 (0x0e00 << 16) | (0xc1fc >> 2),
669 (0x0e00 << 16) | (0xc208 >> 2),
671 (0x0e00 << 16) | (0xc264 >> 2),
673 (0x0e00 << 16) | (0xc268 >> 2),
675 (0x0e00 << 16) | (0xc26c >> 2),
677 (0x0e00 << 16) | (0xc270 >> 2),
679 (0x0e00 << 16) | (0xc274 >> 2),
681 (0x0e00 << 16) | (0xc28c >> 2),
683 (0x0e00 << 16) | (0xc290 >> 2),
685 (0x0e00 << 16) | (0xc294 >> 2),
687 (0x0e00 << 16) | (0xc298 >> 2),
689 (0x0e00 << 16) | (0xc2a0 >> 2),
691 (0x0e00 << 16) | (0xc2a4 >> 2),
693 (0x0e00 << 16) | (0xc2a8 >> 2),
695 (0x0e00 << 16) | (0xc2ac >> 2),
697 (0x0e00 << 16) | (0x301d0 >> 2),
699 (0x0e00 << 16) | (0x30238 >> 2),
701 (0x0e00 << 16) | (0x30250 >> 2),
703 (0x0e00 << 16) | (0x30254 >> 2),
705 (0x0e00 << 16) | (0x30258 >> 2),
707 (0x0e00 << 16) | (0x3025c >> 2),
709 (0x4e00 << 16) | (0xc900 >> 2),
711 (0x5e00 << 16) | (0xc900 >> 2),
713 (0x6e00 << 16) | (0xc900 >> 2),
715 (0x7e00 << 16) | (0xc900 >> 2),
717 (0x4e00 << 16) | (0xc904 >> 2),
719 (0x5e00 << 16) | (0xc904 >> 2),
721 (0x6e00 << 16) | (0xc904 >> 2),
723 (0x7e00 << 16) | (0xc904 >> 2),
725 (0x4e00 << 16) | (0xc908 >> 2),
727 (0x5e00 << 16) | (0xc908 >> 2),
729 (0x6e00 << 16) | (0xc908 >> 2),
731 (0x7e00 << 16) | (0xc908 >> 2),
733 (0x4e00 << 16) | (0xc90c >> 2),
735 (0x5e00 << 16) | (0xc90c >> 2),
737 (0x6e00 << 16) | (0xc90c >> 2),
739 (0x7e00 << 16) | (0xc90c >> 2),
741 (0x4e00 << 16) | (0xc910 >> 2),
743 (0x5e00 << 16) | (0xc910 >> 2),
745 (0x6e00 << 16) | (0xc910 >> 2),
747 (0x7e00 << 16) | (0xc910 >> 2),
749 (0x0e00 << 16) | (0xc99c >> 2),
751 (0x0e00 << 16) | (0x9834 >> 2),
753 (0x0000 << 16) | (0x30f00 >> 2),
755 (0x0000 << 16) | (0x30f04 >> 2),
757 (0x0000 << 16) | (0x30f08 >> 2),
759 (0x0000 << 16) | (0x30f0c >> 2),
761 (0x0600 << 16) | (0x9b7c >> 2),
763 (0x0e00 << 16) | (0x8a14 >> 2),
765 (0x0e00 << 16) | (0x8a18 >> 2),
767 (0x0600 << 16) | (0x30a00 >> 2),
769 (0x0e00 << 16) | (0x8bf0 >> 2),
771 (0x0e00 << 16) | (0x8bcc >> 2),
773 (0x0e00 << 16) | (0x8b24 >> 2),
775 (0x0e00 << 16) | (0x30a04 >> 2),
777 (0x0600 << 16) | (0x30a10 >> 2),
779 (0x0600 << 16) | (0x30a14 >> 2),
781 (0x0600 << 16) | (0x30a18 >> 2),
783 (0x0600 << 16) | (0x30a2c >> 2),
785 (0x0e00 << 16) | (0xc700 >> 2),
787 (0x0e00 << 16) | (0xc704 >> 2),
789 (0x0e00 << 16) | (0xc708 >> 2),
791 (0x0e00 << 16) | (0xc768 >> 2),
793 (0x0400 << 16) | (0xc770 >> 2),
795 (0x0400 << 16) | (0xc774 >> 2),
797 (0x0400 << 16) | (0xc798 >> 2),
799 (0x0400 << 16) | (0xc79c >> 2),
801 (0x0e00 << 16) | (0x9100 >> 2),
803 (0x0e00 << 16) | (0x3c010 >> 2),
805 (0x0e00 << 16) | (0x8c00 >> 2),
807 (0x0e00 << 16) | (0x8c04 >> 2),
809 (0x0e00 << 16) | (0x8c20 >> 2),
811 (0x0e00 << 16) | (0x8c38 >> 2),
813 (0x0e00 << 16) | (0x8c3c >> 2),
815 (0x0e00 << 16) | (0xae00 >> 2),
817 (0x0e00 << 16) | (0x9604 >> 2),
819 (0x0e00 << 16) | (0xac08 >> 2),
821 (0x0e00 << 16) | (0xac0c >> 2),
823 (0x0e00 << 16) | (0xac10 >> 2),
825 (0x0e00 << 16) | (0xac14 >> 2),
827 (0x0e00 << 16) | (0xac58 >> 2),
829 (0x0e00 << 16) | (0xac68 >> 2),
831 (0x0e00 << 16) | (0xac6c >> 2),
833 (0x0e00 << 16) | (0xac70 >> 2),
835 (0x0e00 << 16) | (0xac74 >> 2),
837 (0x0e00 << 16) | (0xac78 >> 2),
839 (0x0e00 << 16) | (0xac7c >> 2),
841 (0x0e00 << 16) | (0xac80 >> 2),
843 (0x0e00 << 16) | (0xac84 >> 2),
845 (0x0e00 << 16) | (0xac88 >> 2),
847 (0x0e00 << 16) | (0xac8c >> 2),
849 (0x0e00 << 16) | (0x970c >> 2),
851 (0x0e00 << 16) | (0x9714 >> 2),
853 (0x0e00 << 16) | (0x9718 >> 2),
855 (0x0e00 << 16) | (0x971c >> 2),
857 (0x0e00 << 16) | (0x31068 >> 2),
859 (0x4e00 << 16) | (0x31068 >> 2),
861 (0x5e00 << 16) | (0x31068 >> 2),
863 (0x6e00 << 16) | (0x31068 >> 2),
865 (0x7e00 << 16) | (0x31068 >> 2),
867 (0x0e00 << 16) | (0xcd10 >> 2),
869 (0x0e00 << 16) | (0xcd14 >> 2),
871 (0x0e00 << 16) | (0x88b0 >> 2),
873 (0x0e00 << 16) | (0x88b4 >> 2),
875 (0x0e00 << 16) | (0x88b8 >> 2),
877 (0x0e00 << 16) | (0x88bc >> 2),
879 (0x0400 << 16) | (0x89c0 >> 2),
881 (0x0e00 << 16) | (0x88c4 >> 2),
883 (0x0e00 << 16) | (0x88c8 >> 2),
885 (0x0e00 << 16) | (0x88d0 >> 2),
887 (0x0e00 << 16) | (0x88d4 >> 2),
889 (0x0e00 << 16) | (0x88d8 >> 2),
891 (0x0e00 << 16) | (0x8980 >> 2),
893 (0x0e00 << 16) | (0x30938 >> 2),
895 (0x0e00 << 16) | (0x3093c >> 2),
897 (0x0e00 << 16) | (0x30940 >> 2),
899 (0x0e00 << 16) | (0x89a0 >> 2),
901 (0x0e00 << 16) | (0x30900 >> 2),
903 (0x0e00 << 16) | (0x30904 >> 2),
905 (0x0e00 << 16) | (0x89b4 >> 2),
907 (0x0e00 << 16) | (0x3e1fc >> 2),
909 (0x0e00 << 16) | (0x3c210 >> 2),
911 (0x0e00 << 16) | (0x3c214 >> 2),
913 (0x0e00 << 16) | (0x3c218 >> 2),
915 (0x0e00 << 16) | (0x8904 >> 2),
918 (0x0e00 << 16) | (0x8c28 >> 2),
919 (0x0e00 << 16) | (0x8c2c >> 2),
920 (0x0e00 << 16) | (0x8c30 >> 2),
921 (0x0e00 << 16) | (0x8c34 >> 2),
922 (0x0e00 << 16) | (0x9600 >> 2),
925 static const u32 bonaire_golden_spm_registers[] =
927 0x30800, 0xe0ffffff, 0xe0000000
930 static const u32 bonaire_golden_common_registers[] =
932 0xc770, 0xffffffff, 0x00000800,
933 0xc774, 0xffffffff, 0x00000800,
934 0xc798, 0xffffffff, 0x00007fbf,
935 0xc79c, 0xffffffff, 0x00007faf
938 static const u32 bonaire_golden_registers[] =
940 0x3354, 0x00000333, 0x00000333,
941 0x3350, 0x000c0fc0, 0x00040200,
942 0x9a10, 0x00010000, 0x00058208,
943 0x3c000, 0xffff1fff, 0x00140000,
944 0x3c200, 0xfdfc0fff, 0x00000100,
945 0x3c234, 0x40000000, 0x40000200,
946 0x9830, 0xffffffff, 0x00000000,
947 0x9834, 0xf00fffff, 0x00000400,
948 0x9838, 0x0002021c, 0x00020200,
949 0xc78, 0x00000080, 0x00000000,
950 0x5bb0, 0x000000f0, 0x00000070,
951 0x5bc0, 0xf0311fff, 0x80300000,
952 0x98f8, 0x73773777, 0x12010001,
953 0x350c, 0x00810000, 0x408af000,
954 0x7030, 0x31000111, 0x00000011,
955 0x2f48, 0x73773777, 0x12010001,
956 0x220c, 0x00007fb6, 0x0021a1b1,
957 0x2210, 0x00007fb6, 0x002021b1,
958 0x2180, 0x00007fb6, 0x00002191,
959 0x2218, 0x00007fb6, 0x002121b1,
960 0x221c, 0x00007fb6, 0x002021b1,
961 0x21dc, 0x00007fb6, 0x00002191,
962 0x21e0, 0x00007fb6, 0x00002191,
963 0x3628, 0x0000003f, 0x0000000a,
964 0x362c, 0x0000003f, 0x0000000a,
965 0x2ae4, 0x00073ffe, 0x000022a2,
966 0x240c, 0x000007ff, 0x00000000,
967 0x8a14, 0xf000003f, 0x00000007,
968 0x8bf0, 0x00002001, 0x00000001,
969 0x8b24, 0xffffffff, 0x00ffffff,
970 0x30a04, 0x0000ff0f, 0x00000000,
971 0x28a4c, 0x07ffffff, 0x06000000,
972 0x4d8, 0x00000fff, 0x00000100,
973 0x3e78, 0x00000001, 0x00000002,
974 0x9100, 0x03000000, 0x0362c688,
975 0x8c00, 0x000000ff, 0x00000001,
976 0xe40, 0x00001fff, 0x00001fff,
977 0x9060, 0x0000007f, 0x00000020,
978 0x9508, 0x00010000, 0x00010000,
979 0xac14, 0x000003ff, 0x000000f3,
980 0xac0c, 0xffffffff, 0x00001032
983 static const u32 bonaire_mgcg_cgcg_init[] =
985 0xc420, 0xffffffff, 0xfffffffc,
986 0x30800, 0xffffffff, 0xe0000000,
987 0x3c2a0, 0xffffffff, 0x00000100,
988 0x3c208, 0xffffffff, 0x00000100,
989 0x3c2c0, 0xffffffff, 0xc0000100,
990 0x3c2c8, 0xffffffff, 0xc0000100,
991 0x3c2c4, 0xffffffff, 0xc0000100,
992 0x55e4, 0xffffffff, 0x00600100,
993 0x3c280, 0xffffffff, 0x00000100,
994 0x3c214, 0xffffffff, 0x06000100,
995 0x3c220, 0xffffffff, 0x00000100,
996 0x3c218, 0xffffffff, 0x06000100,
997 0x3c204, 0xffffffff, 0x00000100,
998 0x3c2e0, 0xffffffff, 0x00000100,
999 0x3c224, 0xffffffff, 0x00000100,
1000 0x3c200, 0xffffffff, 0x00000100,
1001 0x3c230, 0xffffffff, 0x00000100,
1002 0x3c234, 0xffffffff, 0x00000100,
1003 0x3c250, 0xffffffff, 0x00000100,
1004 0x3c254, 0xffffffff, 0x00000100,
1005 0x3c258, 0xffffffff, 0x00000100,
1006 0x3c25c, 0xffffffff, 0x00000100,
1007 0x3c260, 0xffffffff, 0x00000100,
1008 0x3c27c, 0xffffffff, 0x00000100,
1009 0x3c278, 0xffffffff, 0x00000100,
1010 0x3c210, 0xffffffff, 0x06000100,
1011 0x3c290, 0xffffffff, 0x00000100,
1012 0x3c274, 0xffffffff, 0x00000100,
1013 0x3c2b4, 0xffffffff, 0x00000100,
1014 0x3c2b0, 0xffffffff, 0x00000100,
1015 0x3c270, 0xffffffff, 0x00000100,
1016 0x30800, 0xffffffff, 0xe0000000,
1017 0x3c020, 0xffffffff, 0x00010000,
1018 0x3c024, 0xffffffff, 0x00030002,
1019 0x3c028, 0xffffffff, 0x00040007,
1020 0x3c02c, 0xffffffff, 0x00060005,
1021 0x3c030, 0xffffffff, 0x00090008,
1022 0x3c034, 0xffffffff, 0x00010000,
1023 0x3c038, 0xffffffff, 0x00030002,
1024 0x3c03c, 0xffffffff, 0x00040007,
1025 0x3c040, 0xffffffff, 0x00060005,
1026 0x3c044, 0xffffffff, 0x00090008,
1027 0x3c048, 0xffffffff, 0x00010000,
1028 0x3c04c, 0xffffffff, 0x00030002,
1029 0x3c050, 0xffffffff, 0x00040007,
1030 0x3c054, 0xffffffff, 0x00060005,
1031 0x3c058, 0xffffffff, 0x00090008,
1032 0x3c05c, 0xffffffff, 0x00010000,
1033 0x3c060, 0xffffffff, 0x00030002,
1034 0x3c064, 0xffffffff, 0x00040007,
1035 0x3c068, 0xffffffff, 0x00060005,
1036 0x3c06c, 0xffffffff, 0x00090008,
1037 0x3c070, 0xffffffff, 0x00010000,
1038 0x3c074, 0xffffffff, 0x00030002,
1039 0x3c078, 0xffffffff, 0x00040007,
1040 0x3c07c, 0xffffffff, 0x00060005,
1041 0x3c080, 0xffffffff, 0x00090008,
1042 0x3c084, 0xffffffff, 0x00010000,
1043 0x3c088, 0xffffffff, 0x00030002,
1044 0x3c08c, 0xffffffff, 0x00040007,
1045 0x3c090, 0xffffffff, 0x00060005,
1046 0x3c094, 0xffffffff, 0x00090008,
1047 0x3c098, 0xffffffff, 0x00010000,
1048 0x3c09c, 0xffffffff, 0x00030002,
1049 0x3c0a0, 0xffffffff, 0x00040007,
1050 0x3c0a4, 0xffffffff, 0x00060005,
1051 0x3c0a8, 0xffffffff, 0x00090008,
1052 0x3c000, 0xffffffff, 0x96e00200,
1053 0x8708, 0xffffffff, 0x00900100,
1054 0xc424, 0xffffffff, 0x0020003f,
1055 0x38, 0xffffffff, 0x0140001c,
1056 0x3c, 0x000f0000, 0x000f0000,
1057 0x220, 0xffffffff, 0xC060000C,
1058 0x224, 0xc0000fff, 0x00000100,
1059 0xf90, 0xffffffff, 0x00000100,
1060 0xf98, 0x00000101, 0x00000000,
1061 0x20a8, 0xffffffff, 0x00000104,
1062 0x55e4, 0xff000fff, 0x00000100,
1063 0x30cc, 0xc0000fff, 0x00000104,
1064 0xc1e4, 0x00000001, 0x00000001,
1065 0xd00c, 0xff000ff0, 0x00000100,
1066 0xd80c, 0xff000ff0, 0x00000100
1069 static const u32 spectre_golden_spm_registers[] =
1071 0x30800, 0xe0ffffff, 0xe0000000
1074 static const u32 spectre_golden_common_registers[] =
1076 0xc770, 0xffffffff, 0x00000800,
1077 0xc774, 0xffffffff, 0x00000800,
1078 0xc798, 0xffffffff, 0x00007fbf,
1079 0xc79c, 0xffffffff, 0x00007faf
1082 static const u32 spectre_golden_registers[] =
1084 0x3c000, 0xffff1fff, 0x96940200,
1085 0x3c00c, 0xffff0001, 0xff000000,
1086 0x3c200, 0xfffc0fff, 0x00000100,
1087 0x6ed8, 0x00010101, 0x00010000,
1088 0x9834, 0xf00fffff, 0x00000400,
1089 0x9838, 0xfffffffc, 0x00020200,
1090 0x5bb0, 0x000000f0, 0x00000070,
1091 0x5bc0, 0xf0311fff, 0x80300000,
1092 0x98f8, 0x73773777, 0x12010001,
1093 0x9b7c, 0x00ff0000, 0x00fc0000,
1094 0x2f48, 0x73773777, 0x12010001,
1095 0x8a14, 0xf000003f, 0x00000007,
1096 0x8b24, 0xffffffff, 0x00ffffff,
1097 0x28350, 0x3f3f3fff, 0x00000082,
1098 0x28355, 0x0000003f, 0x00000000,
1099 0x3e78, 0x00000001, 0x00000002,
1100 0x913c, 0xffff03df, 0x00000004,
1101 0xc768, 0x00000008, 0x00000008,
1102 0x8c00, 0x000008ff, 0x00000800,
1103 0x9508, 0x00010000, 0x00010000,
1104 0xac0c, 0xffffffff, 0x54763210,
1105 0x214f8, 0x01ff01ff, 0x00000002,
1106 0x21498, 0x007ff800, 0x00200000,
1107 0x2015c, 0xffffffff, 0x00000f40,
1108 0x30934, 0xffffffff, 0x00000001
1111 static const u32 spectre_mgcg_cgcg_init[] =
1113 0xc420, 0xffffffff, 0xfffffffc,
1114 0x30800, 0xffffffff, 0xe0000000,
1115 0x3c2a0, 0xffffffff, 0x00000100,
1116 0x3c208, 0xffffffff, 0x00000100,
1117 0x3c2c0, 0xffffffff, 0x00000100,
1118 0x3c2c8, 0xffffffff, 0x00000100,
1119 0x3c2c4, 0xffffffff, 0x00000100,
1120 0x55e4, 0xffffffff, 0x00600100,
1121 0x3c280, 0xffffffff, 0x00000100,
1122 0x3c214, 0xffffffff, 0x06000100,
1123 0x3c220, 0xffffffff, 0x00000100,
1124 0x3c218, 0xffffffff, 0x06000100,
1125 0x3c204, 0xffffffff, 0x00000100,
1126 0x3c2e0, 0xffffffff, 0x00000100,
1127 0x3c224, 0xffffffff, 0x00000100,
1128 0x3c200, 0xffffffff, 0x00000100,
1129 0x3c230, 0xffffffff, 0x00000100,
1130 0x3c234, 0xffffffff, 0x00000100,
1131 0x3c250, 0xffffffff, 0x00000100,
1132 0x3c254, 0xffffffff, 0x00000100,
1133 0x3c258, 0xffffffff, 0x00000100,
1134 0x3c25c, 0xffffffff, 0x00000100,
1135 0x3c260, 0xffffffff, 0x00000100,
1136 0x3c27c, 0xffffffff, 0x00000100,
1137 0x3c278, 0xffffffff, 0x00000100,
1138 0x3c210, 0xffffffff, 0x06000100,
1139 0x3c290, 0xffffffff, 0x00000100,
1140 0x3c274, 0xffffffff, 0x00000100,
1141 0x3c2b4, 0xffffffff, 0x00000100,
1142 0x3c2b0, 0xffffffff, 0x00000100,
1143 0x3c270, 0xffffffff, 0x00000100,
1144 0x30800, 0xffffffff, 0xe0000000,
1145 0x3c020, 0xffffffff, 0x00010000,
1146 0x3c024, 0xffffffff, 0x00030002,
1147 0x3c028, 0xffffffff, 0x00040007,
1148 0x3c02c, 0xffffffff, 0x00060005,
1149 0x3c030, 0xffffffff, 0x00090008,
1150 0x3c034, 0xffffffff, 0x00010000,
1151 0x3c038, 0xffffffff, 0x00030002,
1152 0x3c03c, 0xffffffff, 0x00040007,
1153 0x3c040, 0xffffffff, 0x00060005,
1154 0x3c044, 0xffffffff, 0x00090008,
1155 0x3c048, 0xffffffff, 0x00010000,
1156 0x3c04c, 0xffffffff, 0x00030002,
1157 0x3c050, 0xffffffff, 0x00040007,
1158 0x3c054, 0xffffffff, 0x00060005,
1159 0x3c058, 0xffffffff, 0x00090008,
1160 0x3c05c, 0xffffffff, 0x00010000,
1161 0x3c060, 0xffffffff, 0x00030002,
1162 0x3c064, 0xffffffff, 0x00040007,
1163 0x3c068, 0xffffffff, 0x00060005,
1164 0x3c06c, 0xffffffff, 0x00090008,
1165 0x3c070, 0xffffffff, 0x00010000,
1166 0x3c074, 0xffffffff, 0x00030002,
1167 0x3c078, 0xffffffff, 0x00040007,
1168 0x3c07c, 0xffffffff, 0x00060005,
1169 0x3c080, 0xffffffff, 0x00090008,
1170 0x3c084, 0xffffffff, 0x00010000,
1171 0x3c088, 0xffffffff, 0x00030002,
1172 0x3c08c, 0xffffffff, 0x00040007,
1173 0x3c090, 0xffffffff, 0x00060005,
1174 0x3c094, 0xffffffff, 0x00090008,
1175 0x3c098, 0xffffffff, 0x00010000,
1176 0x3c09c, 0xffffffff, 0x00030002,
1177 0x3c0a0, 0xffffffff, 0x00040007,
1178 0x3c0a4, 0xffffffff, 0x00060005,
1179 0x3c0a8, 0xffffffff, 0x00090008,
1180 0x3c0ac, 0xffffffff, 0x00010000,
1181 0x3c0b0, 0xffffffff, 0x00030002,
1182 0x3c0b4, 0xffffffff, 0x00040007,
1183 0x3c0b8, 0xffffffff, 0x00060005,
1184 0x3c0bc, 0xffffffff, 0x00090008,
1185 0x3c000, 0xffffffff, 0x96e00200,
1186 0x8708, 0xffffffff, 0x00900100,
1187 0xc424, 0xffffffff, 0x0020003f,
1188 0x38, 0xffffffff, 0x0140001c,
1189 0x3c, 0x000f0000, 0x000f0000,
1190 0x220, 0xffffffff, 0xC060000C,
1191 0x224, 0xc0000fff, 0x00000100,
1192 0xf90, 0xffffffff, 0x00000100,
1193 0xf98, 0x00000101, 0x00000000,
1194 0x20a8, 0xffffffff, 0x00000104,
1195 0x55e4, 0xff000fff, 0x00000100,
1196 0x30cc, 0xc0000fff, 0x00000104,
1197 0xc1e4, 0x00000001, 0x00000001,
1198 0xd00c, 0xff000ff0, 0x00000100,
1199 0xd80c, 0xff000ff0, 0x00000100
1202 static const u32 kalindi_golden_spm_registers[] =
1204 0x30800, 0xe0ffffff, 0xe0000000
1207 static const u32 kalindi_golden_common_registers[] =
1209 0xc770, 0xffffffff, 0x00000800,
1210 0xc774, 0xffffffff, 0x00000800,
1211 0xc798, 0xffffffff, 0x00007fbf,
1212 0xc79c, 0xffffffff, 0x00007faf
1215 static const u32 kalindi_golden_registers[] =
1217 0x3c000, 0xffffdfff, 0x6e944040,
1218 0x55e4, 0xff607fff, 0xfc000100,
1219 0x3c220, 0xff000fff, 0x00000100,
1220 0x3c224, 0xff000fff, 0x00000100,
1221 0x3c200, 0xfffc0fff, 0x00000100,
1222 0x6ed8, 0x00010101, 0x00010000,
1223 0x9830, 0xffffffff, 0x00000000,
1224 0x9834, 0xf00fffff, 0x00000400,
1225 0x5bb0, 0x000000f0, 0x00000070,
1226 0x5bc0, 0xf0311fff, 0x80300000,
1227 0x98f8, 0x73773777, 0x12010001,
1228 0x98fc, 0xffffffff, 0x00000010,
1229 0x9b7c, 0x00ff0000, 0x00fc0000,
1230 0x8030, 0x00001f0f, 0x0000100a,
1231 0x2f48, 0x73773777, 0x12010001,
1232 0x2408, 0x000fffff, 0x000c007f,
1233 0x8a14, 0xf000003f, 0x00000007,
1234 0x8b24, 0x3fff3fff, 0x00ffcfff,
1235 0x30a04, 0x0000ff0f, 0x00000000,
1236 0x28a4c, 0x07ffffff, 0x06000000,
1237 0x4d8, 0x00000fff, 0x00000100,
1238 0x3e78, 0x00000001, 0x00000002,
1239 0xc768, 0x00000008, 0x00000008,
1240 0x8c00, 0x000000ff, 0x00000003,
1241 0x214f8, 0x01ff01ff, 0x00000002,
1242 0x21498, 0x007ff800, 0x00200000,
1243 0x2015c, 0xffffffff, 0x00000f40,
1244 0x88c4, 0x001f3ae3, 0x00000082,
1245 0x88d4, 0x0000001f, 0x00000010,
1246 0x30934, 0xffffffff, 0x00000000
1249 static const u32 kalindi_mgcg_cgcg_init[] =
1251 0xc420, 0xffffffff, 0xfffffffc,
1252 0x30800, 0xffffffff, 0xe0000000,
1253 0x3c2a0, 0xffffffff, 0x00000100,
1254 0x3c208, 0xffffffff, 0x00000100,
1255 0x3c2c0, 0xffffffff, 0x00000100,
1256 0x3c2c8, 0xffffffff, 0x00000100,
1257 0x3c2c4, 0xffffffff, 0x00000100,
1258 0x55e4, 0xffffffff, 0x00600100,
1259 0x3c280, 0xffffffff, 0x00000100,
1260 0x3c214, 0xffffffff, 0x06000100,
1261 0x3c220, 0xffffffff, 0x00000100,
1262 0x3c218, 0xffffffff, 0x06000100,
1263 0x3c204, 0xffffffff, 0x00000100,
1264 0x3c2e0, 0xffffffff, 0x00000100,
1265 0x3c224, 0xffffffff, 0x00000100,
1266 0x3c200, 0xffffffff, 0x00000100,
1267 0x3c230, 0xffffffff, 0x00000100,
1268 0x3c234, 0xffffffff, 0x00000100,
1269 0x3c250, 0xffffffff, 0x00000100,
1270 0x3c254, 0xffffffff, 0x00000100,
1271 0x3c258, 0xffffffff, 0x00000100,
1272 0x3c25c, 0xffffffff, 0x00000100,
1273 0x3c260, 0xffffffff, 0x00000100,
1274 0x3c27c, 0xffffffff, 0x00000100,
1275 0x3c278, 0xffffffff, 0x00000100,
1276 0x3c210, 0xffffffff, 0x06000100,
1277 0x3c290, 0xffffffff, 0x00000100,
1278 0x3c274, 0xffffffff, 0x00000100,
1279 0x3c2b4, 0xffffffff, 0x00000100,
1280 0x3c2b0, 0xffffffff, 0x00000100,
1281 0x3c270, 0xffffffff, 0x00000100,
1282 0x30800, 0xffffffff, 0xe0000000,
1283 0x3c020, 0xffffffff, 0x00010000,
1284 0x3c024, 0xffffffff, 0x00030002,
1285 0x3c028, 0xffffffff, 0x00040007,
1286 0x3c02c, 0xffffffff, 0x00060005,
1287 0x3c030, 0xffffffff, 0x00090008,
1288 0x3c034, 0xffffffff, 0x00010000,
1289 0x3c038, 0xffffffff, 0x00030002,
1290 0x3c03c, 0xffffffff, 0x00040007,
1291 0x3c040, 0xffffffff, 0x00060005,
1292 0x3c044, 0xffffffff, 0x00090008,
1293 0x3c000, 0xffffffff, 0x96e00200,
1294 0x8708, 0xffffffff, 0x00900100,
1295 0xc424, 0xffffffff, 0x0020003f,
1296 0x38, 0xffffffff, 0x0140001c,
1297 0x3c, 0x000f0000, 0x000f0000,
1298 0x220, 0xffffffff, 0xC060000C,
1299 0x224, 0xc0000fff, 0x00000100,
1300 0x20a8, 0xffffffff, 0x00000104,
1301 0x55e4, 0xff000fff, 0x00000100,
1302 0x30cc, 0xc0000fff, 0x00000104,
1303 0xc1e4, 0x00000001, 0x00000001,
1304 0xd00c, 0xff000ff0, 0x00000100,
1305 0xd80c, 0xff000ff0, 0x00000100
1308 static const u32 hawaii_golden_spm_registers[] =
1310 0x30800, 0xe0ffffff, 0xe0000000
1313 static const u32 hawaii_golden_common_registers[] =
1315 0x30800, 0xffffffff, 0xe0000000,
1316 0x28350, 0xffffffff, 0x3a00161a,
1317 0x28354, 0xffffffff, 0x0000002e,
1318 0x9a10, 0xffffffff, 0x00018208,
1319 0x98f8, 0xffffffff, 0x12011003
1322 static const u32 hawaii_golden_registers[] =
1324 0x3354, 0x00000333, 0x00000333,
1325 0x9a10, 0x00010000, 0x00058208,
1326 0x9830, 0xffffffff, 0x00000000,
1327 0x9834, 0xf00fffff, 0x00000400,
1328 0x9838, 0x0002021c, 0x00020200,
1329 0xc78, 0x00000080, 0x00000000,
1330 0x5bb0, 0x000000f0, 0x00000070,
1331 0x5bc0, 0xf0311fff, 0x80300000,
1332 0x350c, 0x00810000, 0x408af000,
1333 0x7030, 0x31000111, 0x00000011,
1334 0x2f48, 0x73773777, 0x12010001,
1335 0x2120, 0x0000007f, 0x0000001b,
1336 0x21dc, 0x00007fb6, 0x00002191,
1337 0x3628, 0x0000003f, 0x0000000a,
1338 0x362c, 0x0000003f, 0x0000000a,
1339 0x2ae4, 0x00073ffe, 0x000022a2,
1340 0x240c, 0x000007ff, 0x00000000,
1341 0x8bf0, 0x00002001, 0x00000001,
1342 0x8b24, 0xffffffff, 0x00ffffff,
1343 0x30a04, 0x0000ff0f, 0x00000000,
1344 0x28a4c, 0x07ffffff, 0x06000000,
1345 0x3e78, 0x00000001, 0x00000002,
1346 0xc768, 0x00000008, 0x00000008,
1347 0xc770, 0x00000f00, 0x00000800,
1348 0xc774, 0x00000f00, 0x00000800,
1349 0xc798, 0x00ffffff, 0x00ff7fbf,
1350 0xc79c, 0x00ffffff, 0x00ff7faf,
1351 0x8c00, 0x000000ff, 0x00000800,
1352 0xe40, 0x00001fff, 0x00001fff,
1353 0x9060, 0x0000007f, 0x00000020,
1354 0x9508, 0x00010000, 0x00010000,
1355 0xae00, 0x00100000, 0x000ff07c,
1356 0xac14, 0x000003ff, 0x0000000f,
1357 0xac10, 0xffffffff, 0x7564fdec,
1358 0xac0c, 0xffffffff, 0x3120b9a8,
1359 0xac08, 0x20000000, 0x0f9c0000
1362 static const u32 hawaii_mgcg_cgcg_init[] =
1364 0xc420, 0xffffffff, 0xfffffffd,
1365 0x30800, 0xffffffff, 0xe0000000,
1366 0x3c2a0, 0xffffffff, 0x00000100,
1367 0x3c208, 0xffffffff, 0x00000100,
1368 0x3c2c0, 0xffffffff, 0x00000100,
1369 0x3c2c8, 0xffffffff, 0x00000100,
1370 0x3c2c4, 0xffffffff, 0x00000100,
1371 0x55e4, 0xffffffff, 0x00200100,
1372 0x3c280, 0xffffffff, 0x00000100,
1373 0x3c214, 0xffffffff, 0x06000100,
1374 0x3c220, 0xffffffff, 0x00000100,
1375 0x3c218, 0xffffffff, 0x06000100,
1376 0x3c204, 0xffffffff, 0x00000100,
1377 0x3c2e0, 0xffffffff, 0x00000100,
1378 0x3c224, 0xffffffff, 0x00000100,
1379 0x3c200, 0xffffffff, 0x00000100,
1380 0x3c230, 0xffffffff, 0x00000100,
1381 0x3c234, 0xffffffff, 0x00000100,
1382 0x3c250, 0xffffffff, 0x00000100,
1383 0x3c254, 0xffffffff, 0x00000100,
1384 0x3c258, 0xffffffff, 0x00000100,
1385 0x3c25c, 0xffffffff, 0x00000100,
1386 0x3c260, 0xffffffff, 0x00000100,
1387 0x3c27c, 0xffffffff, 0x00000100,
1388 0x3c278, 0xffffffff, 0x00000100,
1389 0x3c210, 0xffffffff, 0x06000100,
1390 0x3c290, 0xffffffff, 0x00000100,
1391 0x3c274, 0xffffffff, 0x00000100,
1392 0x3c2b4, 0xffffffff, 0x00000100,
1393 0x3c2b0, 0xffffffff, 0x00000100,
1394 0x3c270, 0xffffffff, 0x00000100,
1395 0x30800, 0xffffffff, 0xe0000000,
1396 0x3c020, 0xffffffff, 0x00010000,
1397 0x3c024, 0xffffffff, 0x00030002,
1398 0x3c028, 0xffffffff, 0x00040007,
1399 0x3c02c, 0xffffffff, 0x00060005,
1400 0x3c030, 0xffffffff, 0x00090008,
1401 0x3c034, 0xffffffff, 0x00010000,
1402 0x3c038, 0xffffffff, 0x00030002,
1403 0x3c03c, 0xffffffff, 0x00040007,
1404 0x3c040, 0xffffffff, 0x00060005,
1405 0x3c044, 0xffffffff, 0x00090008,
1406 0x3c048, 0xffffffff, 0x00010000,
1407 0x3c04c, 0xffffffff, 0x00030002,
1408 0x3c050, 0xffffffff, 0x00040007,
1409 0x3c054, 0xffffffff, 0x00060005,
1410 0x3c058, 0xffffffff, 0x00090008,
1411 0x3c05c, 0xffffffff, 0x00010000,
1412 0x3c060, 0xffffffff, 0x00030002,
1413 0x3c064, 0xffffffff, 0x00040007,
1414 0x3c068, 0xffffffff, 0x00060005,
1415 0x3c06c, 0xffffffff, 0x00090008,
1416 0x3c070, 0xffffffff, 0x00010000,
1417 0x3c074, 0xffffffff, 0x00030002,
1418 0x3c078, 0xffffffff, 0x00040007,
1419 0x3c07c, 0xffffffff, 0x00060005,
1420 0x3c080, 0xffffffff, 0x00090008,
1421 0x3c084, 0xffffffff, 0x00010000,
1422 0x3c088, 0xffffffff, 0x00030002,
1423 0x3c08c, 0xffffffff, 0x00040007,
1424 0x3c090, 0xffffffff, 0x00060005,
1425 0x3c094, 0xffffffff, 0x00090008,
1426 0x3c098, 0xffffffff, 0x00010000,
1427 0x3c09c, 0xffffffff, 0x00030002,
1428 0x3c0a0, 0xffffffff, 0x00040007,
1429 0x3c0a4, 0xffffffff, 0x00060005,
1430 0x3c0a8, 0xffffffff, 0x00090008,
1431 0x3c0ac, 0xffffffff, 0x00010000,
1432 0x3c0b0, 0xffffffff, 0x00030002,
1433 0x3c0b4, 0xffffffff, 0x00040007,
1434 0x3c0b8, 0xffffffff, 0x00060005,
1435 0x3c0bc, 0xffffffff, 0x00090008,
1436 0x3c0c0, 0xffffffff, 0x00010000,
1437 0x3c0c4, 0xffffffff, 0x00030002,
1438 0x3c0c8, 0xffffffff, 0x00040007,
1439 0x3c0cc, 0xffffffff, 0x00060005,
1440 0x3c0d0, 0xffffffff, 0x00090008,
1441 0x3c0d4, 0xffffffff, 0x00010000,
1442 0x3c0d8, 0xffffffff, 0x00030002,
1443 0x3c0dc, 0xffffffff, 0x00040007,
1444 0x3c0e0, 0xffffffff, 0x00060005,
1445 0x3c0e4, 0xffffffff, 0x00090008,
1446 0x3c0e8, 0xffffffff, 0x00010000,
1447 0x3c0ec, 0xffffffff, 0x00030002,
1448 0x3c0f0, 0xffffffff, 0x00040007,
1449 0x3c0f4, 0xffffffff, 0x00060005,
1450 0x3c0f8, 0xffffffff, 0x00090008,
1451 0xc318, 0xffffffff, 0x00020200,
1452 0x3350, 0xffffffff, 0x00000200,
1453 0x15c0, 0xffffffff, 0x00000400,
1454 0x55e8, 0xffffffff, 0x00000000,
1455 0x2f50, 0xffffffff, 0x00000902,
1456 0x3c000, 0xffffffff, 0x96940200,
1457 0x8708, 0xffffffff, 0x00900100,
1458 0xc424, 0xffffffff, 0x0020003f,
1459 0x38, 0xffffffff, 0x0140001c,
1460 0x3c, 0x000f0000, 0x000f0000,
1461 0x220, 0xffffffff, 0xc060000c,
1462 0x224, 0xc0000fff, 0x00000100,
1463 0xf90, 0xffffffff, 0x00000100,
1464 0xf98, 0x00000101, 0x00000000,
1465 0x20a8, 0xffffffff, 0x00000104,
1466 0x55e4, 0xff000fff, 0x00000100,
1467 0x30cc, 0xc0000fff, 0x00000104,
1468 0xc1e4, 0x00000001, 0x00000001,
1469 0xd00c, 0xff000ff0, 0x00000100,
1470 0xd80c, 0xff000ff0, 0x00000100
1473 static void cik_init_golden_registers(struct radeon_device *rdev)
1475 switch (rdev->family) {
1477 radeon_program_register_sequence(rdev,
1478 bonaire_mgcg_cgcg_init,
1479 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1480 radeon_program_register_sequence(rdev,
1481 bonaire_golden_registers,
1482 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1483 radeon_program_register_sequence(rdev,
1484 bonaire_golden_common_registers,
1485 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1486 radeon_program_register_sequence(rdev,
1487 bonaire_golden_spm_registers,
1488 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1491 radeon_program_register_sequence(rdev,
1492 kalindi_mgcg_cgcg_init,
1493 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1494 radeon_program_register_sequence(rdev,
1495 kalindi_golden_registers,
1496 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1497 radeon_program_register_sequence(rdev,
1498 kalindi_golden_common_registers,
1499 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1500 radeon_program_register_sequence(rdev,
1501 kalindi_golden_spm_registers,
1502 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1505 radeon_program_register_sequence(rdev,
1506 spectre_mgcg_cgcg_init,
1507 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1508 radeon_program_register_sequence(rdev,
1509 spectre_golden_registers,
1510 (const u32)ARRAY_SIZE(spectre_golden_registers));
1511 radeon_program_register_sequence(rdev,
1512 spectre_golden_common_registers,
1513 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1514 radeon_program_register_sequence(rdev,
1515 spectre_golden_spm_registers,
1516 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1519 radeon_program_register_sequence(rdev,
1520 hawaii_mgcg_cgcg_init,
1521 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1522 radeon_program_register_sequence(rdev,
1523 hawaii_golden_registers,
1524 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1525 radeon_program_register_sequence(rdev,
1526 hawaii_golden_common_registers,
1527 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1528 radeon_program_register_sequence(rdev,
1529 hawaii_golden_spm_registers,
1530 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1538 * cik_get_xclk - get the xclk
1540 * @rdev: radeon_device pointer
1542 * Returns the reference clock used by the gfx engine
1545 u32 cik_get_xclk(struct radeon_device *rdev)
1547 u32 reference_clock = rdev->clock.spll.reference_freq;
1549 if (rdev->flags & RADEON_IS_IGP) {
1550 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1551 return reference_clock / 2;
1553 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1554 return reference_clock / 4;
1556 return reference_clock;
1560 * cik_mm_rdoorbell - read a doorbell dword
1562 * @rdev: radeon_device pointer
1563 * @index: doorbell index
1565 * Returns the value in the doorbell aperture at the
1566 * requested doorbell index (CIK).
1568 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1570 if (index < rdev->doorbell.num_doorbells) {
1571 return readl(rdev->doorbell.ptr + index);
1573 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1579 * cik_mm_wdoorbell - write a doorbell dword
1581 * @rdev: radeon_device pointer
1582 * @index: doorbell index
1583 * @v: value to write
1585 * Writes @v to the doorbell aperture at the
1586 * requested doorbell index (CIK).
1588 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1590 if (index < rdev->doorbell.num_doorbells) {
1591 writel(v, rdev->doorbell.ptr + index);
1593 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1597 #define BONAIRE_IO_MC_REGS_SIZE 36
1599 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1601 {0x00000070, 0x04400000},
1602 {0x00000071, 0x80c01803},
1603 {0x00000072, 0x00004004},
1604 {0x00000073, 0x00000100},
1605 {0x00000074, 0x00ff0000},
1606 {0x00000075, 0x34000000},
1607 {0x00000076, 0x08000014},
1608 {0x00000077, 0x00cc08ec},
1609 {0x00000078, 0x00000400},
1610 {0x00000079, 0x00000000},
1611 {0x0000007a, 0x04090000},
1612 {0x0000007c, 0x00000000},
1613 {0x0000007e, 0x4408a8e8},
1614 {0x0000007f, 0x00000304},
1615 {0x00000080, 0x00000000},
1616 {0x00000082, 0x00000001},
1617 {0x00000083, 0x00000002},
1618 {0x00000084, 0xf3e4f400},
1619 {0x00000085, 0x052024e3},
1620 {0x00000087, 0x00000000},
1621 {0x00000088, 0x01000000},
1622 {0x0000008a, 0x1c0a0000},
1623 {0x0000008b, 0xff010000},
1624 {0x0000008d, 0xffffefff},
1625 {0x0000008e, 0xfff3efff},
1626 {0x0000008f, 0xfff3efbf},
1627 {0x00000092, 0xf7ffffff},
1628 {0x00000093, 0xffffff7f},
1629 {0x00000095, 0x00101101},
1630 {0x00000096, 0x00000fff},
1631 {0x00000097, 0x00116fff},
1632 {0x00000098, 0x60010000},
1633 {0x00000099, 0x10010000},
1634 {0x0000009a, 0x00006000},
1635 {0x0000009b, 0x00001000},
1636 {0x0000009f, 0x00b48000}
1639 #define HAWAII_IO_MC_REGS_SIZE 22
1641 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1643 {0x0000007d, 0x40000000},
1644 {0x0000007e, 0x40180304},
1645 {0x0000007f, 0x0000ff00},
1646 {0x00000081, 0x00000000},
1647 {0x00000083, 0x00000800},
1648 {0x00000086, 0x00000000},
1649 {0x00000087, 0x00000100},
1650 {0x00000088, 0x00020100},
1651 {0x00000089, 0x00000000},
1652 {0x0000008b, 0x00040000},
1653 {0x0000008c, 0x00000100},
1654 {0x0000008e, 0xff010000},
1655 {0x00000090, 0xffffefff},
1656 {0x00000091, 0xfff3efff},
1657 {0x00000092, 0xfff3efbf},
1658 {0x00000093, 0xf7ffffff},
1659 {0x00000094, 0xffffff7f},
1660 {0x00000095, 0x00000fff},
1661 {0x00000096, 0x00116fff},
1662 {0x00000097, 0x60010000},
1663 {0x00000098, 0x10010000},
1664 {0x0000009f, 0x00c79000}
1669 * cik_srbm_select - select specific register instances
1671 * @rdev: radeon_device pointer
1672 * @me: selected ME (micro engine)
1677 * Switches the currently active registers instances. Some
1678 * registers are instanced per VMID, others are instanced per
1679 * me/pipe/queue combination.
1681 static void cik_srbm_select(struct radeon_device *rdev,
1682 u32 me, u32 pipe, u32 queue, u32 vmid)
1684 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1687 QUEUEID(queue & 0x7));
1688 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1693 * ci_mc_load_microcode - load MC ucode into the hw
1695 * @rdev: radeon_device pointer
1697 * Load the GDDR MC ucode into the hw (CIK).
1698 * Returns 0 on success, error on failure.
1700 static int ci_mc_load_microcode(struct radeon_device *rdev)
1702 const __be32 *fw_data;
1703 u32 running, blackout = 0;
1705 int i, ucode_size, regs_size;
1710 switch (rdev->family) {
1712 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1713 ucode_size = CIK_MC_UCODE_SIZE;
1714 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1717 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1718 ucode_size = HAWAII_MC_UCODE_SIZE;
1719 regs_size = HAWAII_IO_MC_REGS_SIZE;
1725 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1729 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1730 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1733 /* reset the engine and set to writable */
1734 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1735 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1737 /* load mc io regs */
1738 for (i = 0; i < regs_size; i++) {
1739 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1740 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1742 /* load the MC ucode */
1743 fw_data = (const __be32 *)rdev->mc_fw->data;
1744 for (i = 0; i < ucode_size; i++)
1745 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1747 /* put the engine back into the active state */
1748 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1749 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1750 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1752 /* wait for training to complete */
1753 for (i = 0; i < rdev->usec_timeout; i++) {
1754 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1758 for (i = 0; i < rdev->usec_timeout; i++) {
1759 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1765 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1772 * cik_init_microcode - load ucode images from disk
1774 * @rdev: radeon_device pointer
1776 * Use the firmware interface to load the ucode images into
1777 * the driver (not loaded into hw).
1778 * Returns 0 on success, error on failure.
1780 static int cik_init_microcode(struct radeon_device *rdev)
1782 const char *chip_name;
1783 size_t pfp_req_size, me_req_size, ce_req_size,
1784 mec_req_size, rlc_req_size, mc_req_size = 0,
1785 sdma_req_size, smc_req_size = 0;
1791 switch (rdev->family) {
1793 chip_name = "BONAIRE";
1794 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1795 me_req_size = CIK_ME_UCODE_SIZE * 4;
1796 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1797 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1798 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1799 mc_req_size = CIK_MC_UCODE_SIZE * 4;
1800 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1801 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1804 chip_name = "HAWAII";
1805 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1806 me_req_size = CIK_ME_UCODE_SIZE * 4;
1807 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1808 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1809 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1810 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1811 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1812 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1815 chip_name = "KAVERI";
1816 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1817 me_req_size = CIK_ME_UCODE_SIZE * 4;
1818 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1819 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1820 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1821 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1824 chip_name = "KABINI";
1825 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1826 me_req_size = CIK_ME_UCODE_SIZE * 4;
1827 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1828 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1829 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1830 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1835 DRM_INFO("Loading %s Microcode\n", chip_name);
1837 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1838 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1841 if (rdev->pfp_fw->size != pfp_req_size) {
1843 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1844 rdev->pfp_fw->size, fw_name);
1849 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1850 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1853 if (rdev->me_fw->size != me_req_size) {
1855 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1856 rdev->me_fw->size, fw_name);
1860 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1861 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1864 if (rdev->ce_fw->size != ce_req_size) {
1866 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1867 rdev->ce_fw->size, fw_name);
1871 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
1872 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
1875 if (rdev->mec_fw->size != mec_req_size) {
1877 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1878 rdev->mec_fw->size, fw_name);
1882 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1883 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1886 if (rdev->rlc_fw->size != rlc_req_size) {
1888 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1889 rdev->rlc_fw->size, fw_name);
1893 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
1894 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
1897 if (rdev->sdma_fw->size != sdma_req_size) {
1899 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1900 rdev->sdma_fw->size, fw_name);
1904 /* No SMC, MC ucode on APUs */
1905 if (!(rdev->flags & RADEON_IS_IGP)) {
1906 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1907 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1910 if (rdev->mc_fw->size != mc_req_size) {
1912 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1913 rdev->mc_fw->size, fw_name);
1917 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1918 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1921 "smc: error loading firmware \"%s\"\n",
1923 release_firmware(rdev->smc_fw);
1924 rdev->smc_fw = NULL;
1926 } else if (rdev->smc_fw->size != smc_req_size) {
1928 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1929 rdev->smc_fw->size, fw_name);
1938 "cik_cp: Failed to load firmware \"%s\"\n",
1940 release_firmware(rdev->pfp_fw);
1941 rdev->pfp_fw = NULL;
1942 release_firmware(rdev->me_fw);
1944 release_firmware(rdev->ce_fw);
1946 release_firmware(rdev->rlc_fw);
1947 rdev->rlc_fw = NULL;
1948 release_firmware(rdev->mc_fw);
1950 release_firmware(rdev->smc_fw);
1951 rdev->smc_fw = NULL;
1960 * cik_tiling_mode_table_init - init the hw tiling table
1962 * @rdev: radeon_device pointer
1964 * Starting with SI, the tiling setup is done globally in a
1965 * set of 32 tiling modes. Rather than selecting each set of
1966 * parameters per surface as on older asics, we just select
1967 * which index in the tiling table we want to use, and the
1968 * surface uses those parameters (CIK).
1970 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1972 const u32 num_tile_mode_states = 32;
1973 const u32 num_secondary_tile_mode_states = 16;
1974 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1975 u32 num_pipe_configs;
1976 u32 num_rbs = rdev->config.cik.max_backends_per_se *
1977 rdev->config.cik.max_shader_engines;
1979 switch (rdev->config.cik.mem_row_size_in_kb) {
1981 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1985 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1988 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1992 num_pipe_configs = rdev->config.cik.max_tile_pipes;
1993 if (num_pipe_configs > 8)
1994 num_pipe_configs = 16;
1996 if (num_pipe_configs == 16) {
1997 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1998 switch (reg_offset) {
2000 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2001 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2002 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2003 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2006 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2007 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2008 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2009 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2012 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2013 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2014 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2015 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2018 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2019 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2020 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2021 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2024 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2025 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2026 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2027 TILE_SPLIT(split_equal_to_row_size));
2030 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2031 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2034 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2035 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2036 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2037 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2040 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2041 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2042 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2043 TILE_SPLIT(split_equal_to_row_size));
2046 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2047 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2050 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2051 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2054 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2055 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2056 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2057 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2060 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2061 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2062 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2063 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2066 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2067 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2068 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2069 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2072 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2073 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2076 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2077 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2078 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2079 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2082 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2083 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2084 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2085 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2088 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2089 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2090 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2091 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2094 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2095 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2098 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2099 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2100 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2101 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2104 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2105 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2106 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2107 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2110 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2111 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2112 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2113 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2119 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2120 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2122 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2123 switch (reg_offset) {
2125 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2126 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2127 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2128 NUM_BANKS(ADDR_SURF_16_BANK));
2131 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2132 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2133 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2134 NUM_BANKS(ADDR_SURF_16_BANK));
2137 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2138 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2139 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2140 NUM_BANKS(ADDR_SURF_16_BANK));
2143 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2144 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2145 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2146 NUM_BANKS(ADDR_SURF_16_BANK));
2149 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2150 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2151 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2152 NUM_BANKS(ADDR_SURF_8_BANK));
2155 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2156 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2157 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2158 NUM_BANKS(ADDR_SURF_4_BANK));
2161 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2162 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2163 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2164 NUM_BANKS(ADDR_SURF_2_BANK));
2167 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2168 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2169 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2170 NUM_BANKS(ADDR_SURF_16_BANK));
2173 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2174 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2175 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2176 NUM_BANKS(ADDR_SURF_16_BANK));
2179 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2180 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2181 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2182 NUM_BANKS(ADDR_SURF_16_BANK));
2185 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2186 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2187 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2188 NUM_BANKS(ADDR_SURF_8_BANK));
2191 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2192 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2193 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2194 NUM_BANKS(ADDR_SURF_4_BANK));
2197 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2198 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2199 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2200 NUM_BANKS(ADDR_SURF_2_BANK));
2203 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2204 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2205 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2206 NUM_BANKS(ADDR_SURF_2_BANK));
2212 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2214 } else if (num_pipe_configs == 8) {
2215 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2216 switch (reg_offset) {
2218 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2219 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2220 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2221 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2224 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2225 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2226 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2227 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2230 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2231 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2232 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2233 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2236 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2237 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2238 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2239 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2242 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2243 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2244 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2245 TILE_SPLIT(split_equal_to_row_size));
2248 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2249 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2252 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2253 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2254 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2255 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2258 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2259 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2260 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2261 TILE_SPLIT(split_equal_to_row_size));
2264 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2265 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2268 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2269 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2272 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2273 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2274 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2275 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2278 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2279 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2280 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2281 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2284 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2285 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2286 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2287 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2290 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2291 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2294 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2295 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2296 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2297 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2300 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2301 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2302 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2303 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2306 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2307 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2308 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2309 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2312 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2313 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2316 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2317 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2318 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2319 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2322 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2323 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2324 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2325 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2328 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2329 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2330 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2331 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2337 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2338 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2340 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2341 switch (reg_offset) {
2343 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2344 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2345 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2346 NUM_BANKS(ADDR_SURF_16_BANK));
2349 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2350 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2351 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2352 NUM_BANKS(ADDR_SURF_16_BANK));
2355 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2356 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2357 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2358 NUM_BANKS(ADDR_SURF_16_BANK));
2361 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2362 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2363 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2364 NUM_BANKS(ADDR_SURF_16_BANK));
2367 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2368 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2369 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2370 NUM_BANKS(ADDR_SURF_8_BANK));
2373 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2374 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2375 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2376 NUM_BANKS(ADDR_SURF_4_BANK));
2379 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2380 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2381 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2382 NUM_BANKS(ADDR_SURF_2_BANK));
2385 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2386 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2387 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2388 NUM_BANKS(ADDR_SURF_16_BANK));
2391 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2392 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2393 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2394 NUM_BANKS(ADDR_SURF_16_BANK));
2397 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2398 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2399 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2400 NUM_BANKS(ADDR_SURF_16_BANK));
2403 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2404 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2405 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2406 NUM_BANKS(ADDR_SURF_16_BANK));
2409 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2410 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2411 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2412 NUM_BANKS(ADDR_SURF_8_BANK));
2415 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2416 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2417 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2418 NUM_BANKS(ADDR_SURF_4_BANK));
2421 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2422 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2423 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2424 NUM_BANKS(ADDR_SURF_2_BANK));
2430 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2431 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2433 } else if (num_pipe_configs == 4) {
2435 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2436 switch (reg_offset) {
2438 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2439 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2440 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2441 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2444 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2445 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2446 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2447 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2450 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2451 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2452 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2453 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2456 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2457 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2458 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2459 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2462 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2463 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2464 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2465 TILE_SPLIT(split_equal_to_row_size));
2468 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2469 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2472 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2473 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2474 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2475 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2478 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2479 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2480 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2481 TILE_SPLIT(split_equal_to_row_size));
2484 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2485 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2488 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2489 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2492 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2493 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2494 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2495 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2498 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2499 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2500 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2501 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2504 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2505 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2506 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2507 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2510 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2511 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2514 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2515 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2516 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2517 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2520 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2521 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2522 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2523 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2526 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2527 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2528 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2529 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2532 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2533 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2536 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2537 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2538 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2539 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2542 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2543 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2544 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2545 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2548 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2549 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2550 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2551 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2557 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2558 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2560 } else if (num_rbs < 4) {
2561 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2562 switch (reg_offset) {
2564 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2565 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2566 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2567 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2570 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2571 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2572 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2573 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2576 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2577 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2578 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2579 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2582 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2583 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2584 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2585 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2588 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2589 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2590 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2591 TILE_SPLIT(split_equal_to_row_size));
2594 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2595 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2598 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2599 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2600 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2601 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2604 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2605 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2606 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2607 TILE_SPLIT(split_equal_to_row_size));
2610 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2611 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2614 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2615 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2618 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2619 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2620 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2621 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2624 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2625 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2626 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2627 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2630 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2631 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2632 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2633 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2636 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2637 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2640 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2641 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2642 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2643 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2646 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2647 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2648 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2649 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2652 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2653 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2654 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2655 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2658 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2659 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2662 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2663 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2664 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2665 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2668 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2669 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2670 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2671 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2674 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2675 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2676 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2677 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2683 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2684 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2687 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2688 switch (reg_offset) {
2690 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2692 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2693 NUM_BANKS(ADDR_SURF_16_BANK));
2696 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2697 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2698 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2699 NUM_BANKS(ADDR_SURF_16_BANK));
2702 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2703 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2704 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2705 NUM_BANKS(ADDR_SURF_16_BANK));
2708 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2709 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2710 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2711 NUM_BANKS(ADDR_SURF_16_BANK));
2714 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2715 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2716 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2717 NUM_BANKS(ADDR_SURF_16_BANK));
2720 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2721 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2722 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2723 NUM_BANKS(ADDR_SURF_8_BANK));
2726 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2728 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2729 NUM_BANKS(ADDR_SURF_4_BANK));
2732 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2733 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2734 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2735 NUM_BANKS(ADDR_SURF_16_BANK));
2738 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2739 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2740 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2741 NUM_BANKS(ADDR_SURF_16_BANK));
2744 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2745 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2746 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2747 NUM_BANKS(ADDR_SURF_16_BANK));
2750 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2751 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2752 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2753 NUM_BANKS(ADDR_SURF_16_BANK));
2756 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2757 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2758 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2759 NUM_BANKS(ADDR_SURF_16_BANK));
2762 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2763 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2764 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2765 NUM_BANKS(ADDR_SURF_8_BANK));
2768 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2769 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2770 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2771 NUM_BANKS(ADDR_SURF_4_BANK));
2777 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2778 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2780 } else if (num_pipe_configs == 2) {
2781 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2782 switch (reg_offset) {
2784 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2785 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2786 PIPE_CONFIG(ADDR_SURF_P2) |
2787 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2790 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2791 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2792 PIPE_CONFIG(ADDR_SURF_P2) |
2793 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2796 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2797 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2798 PIPE_CONFIG(ADDR_SURF_P2) |
2799 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2802 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2803 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2804 PIPE_CONFIG(ADDR_SURF_P2) |
2805 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2808 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2809 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2810 PIPE_CONFIG(ADDR_SURF_P2) |
2811 TILE_SPLIT(split_equal_to_row_size));
2814 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2815 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2818 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2819 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2820 PIPE_CONFIG(ADDR_SURF_P2) |
2821 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2824 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2825 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2826 PIPE_CONFIG(ADDR_SURF_P2) |
2827 TILE_SPLIT(split_equal_to_row_size));
2830 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
2833 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2834 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2837 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2838 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2839 PIPE_CONFIG(ADDR_SURF_P2) |
2840 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2843 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2844 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2845 PIPE_CONFIG(ADDR_SURF_P2) |
2846 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2849 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2850 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2851 PIPE_CONFIG(ADDR_SURF_P2) |
2852 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2855 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2856 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2859 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2860 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2861 PIPE_CONFIG(ADDR_SURF_P2) |
2862 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2865 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2866 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2867 PIPE_CONFIG(ADDR_SURF_P2) |
2868 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2871 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2872 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2873 PIPE_CONFIG(ADDR_SURF_P2) |
2874 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2877 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2878 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2881 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2882 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2883 PIPE_CONFIG(ADDR_SURF_P2) |
2884 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2887 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2888 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2889 PIPE_CONFIG(ADDR_SURF_P2) |
2890 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2893 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2894 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2895 PIPE_CONFIG(ADDR_SURF_P2) |
2896 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2902 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2903 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2905 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2906 switch (reg_offset) {
2908 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2909 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2910 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2911 NUM_BANKS(ADDR_SURF_16_BANK));
2914 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2915 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2916 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2917 NUM_BANKS(ADDR_SURF_16_BANK));
2920 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2921 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2922 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2923 NUM_BANKS(ADDR_SURF_16_BANK));
2926 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2927 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2928 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2929 NUM_BANKS(ADDR_SURF_16_BANK));
2932 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2933 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2934 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2935 NUM_BANKS(ADDR_SURF_16_BANK));
2938 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2939 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2940 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2941 NUM_BANKS(ADDR_SURF_16_BANK));
2944 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2945 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2946 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2947 NUM_BANKS(ADDR_SURF_8_BANK));
2950 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2951 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2952 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2953 NUM_BANKS(ADDR_SURF_16_BANK));
2956 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2957 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2958 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2959 NUM_BANKS(ADDR_SURF_16_BANK));
2962 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2963 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2964 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2965 NUM_BANKS(ADDR_SURF_16_BANK));
2968 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2969 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2970 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2971 NUM_BANKS(ADDR_SURF_16_BANK));
2974 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2975 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2976 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2977 NUM_BANKS(ADDR_SURF_16_BANK));
2980 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2981 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2982 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2983 NUM_BANKS(ADDR_SURF_16_BANK));
2986 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2987 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2988 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2989 NUM_BANKS(ADDR_SURF_8_BANK));
2995 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2996 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2999 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3003 * cik_select_se_sh - select which SE, SH to address
3005 * @rdev: radeon_device pointer
3006 * @se_num: shader engine to address
3007 * @sh_num: sh block to address
3009 * Select which SE, SH combinations to address. Certain
3010 * registers are instanced per SE or SH. 0xffffffff means
3011 * broadcast to all SEs or SHs (CIK).
3013 static void cik_select_se_sh(struct radeon_device *rdev,
3014 u32 se_num, u32 sh_num)
3016 u32 data = INSTANCE_BROADCAST_WRITES;
3018 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3019 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3020 else if (se_num == 0xffffffff)
3021 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3022 else if (sh_num == 0xffffffff)
3023 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3025 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3026 WREG32(GRBM_GFX_INDEX, data);
3030 * cik_create_bitmask - create a bitmask
3032 * @bit_width: length of the mask
3034 * create a variable length bit mask (CIK).
3035 * Returns the bitmask.
3037 static u32 cik_create_bitmask(u32 bit_width)
3041 for (i = 0; i < bit_width; i++) {
3049 * cik_select_se_sh - select which SE, SH to address
3051 * @rdev: radeon_device pointer
3052 * @max_rb_num: max RBs (render backends) for the asic
3053 * @se_num: number of SEs (shader engines) for the asic
3054 * @sh_per_se: number of SH blocks per SE for the asic
3056 * Calculates the bitmask of disabled RBs (CIK).
3057 * Returns the disabled RB bitmask.
3059 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3060 u32 max_rb_num_per_se,
3065 data = RREG32(CC_RB_BACKEND_DISABLE);
3067 data &= BACKEND_DISABLE_MASK;
3070 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3072 data >>= BACKEND_DISABLE_SHIFT;
3074 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3080 * cik_setup_rb - setup the RBs on the asic
3082 * @rdev: radeon_device pointer
3083 * @se_num: number of SEs (shader engines) for the asic
3084 * @sh_per_se: number of SH blocks per SE for the asic
3085 * @max_rb_num: max RBs (render backends) for the asic
3087 * Configures per-SE/SH RB registers (CIK).
3089 static void cik_setup_rb(struct radeon_device *rdev,
3090 u32 se_num, u32 sh_per_se,
3091 u32 max_rb_num_per_se)
3095 u32 disabled_rbs = 0;
3096 u32 enabled_rbs = 0;
3098 for (i = 0; i < se_num; i++) {
3099 for (j = 0; j < sh_per_se; j++) {
3100 cik_select_se_sh(rdev, i, j);
3101 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3102 if (rdev->family == CHIP_HAWAII)
3103 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3105 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3108 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3111 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3112 if (!(disabled_rbs & mask))
3113 enabled_rbs |= mask;
3117 rdev->config.cik.backend_enable_mask = enabled_rbs;
3119 for (i = 0; i < se_num; i++) {
3120 cik_select_se_sh(rdev, i, 0xffffffff);
3122 for (j = 0; j < sh_per_se; j++) {
3123 switch (enabled_rbs & 3) {
3126 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3128 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3131 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3134 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3138 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3143 WREG32(PA_SC_RASTER_CONFIG, data);
3145 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3149 * cik_gpu_init - setup the 3D engine
3151 * @rdev: radeon_device pointer
3153 * Configures the 3D engine and tiling configuration
3154 * registers so that the 3D engine is usable.
3156 static void cik_gpu_init(struct radeon_device *rdev)
3158 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3159 u32 mc_shared_chmap, mc_arb_ramcfg;
3160 u32 hdp_host_path_cntl;
3164 switch (rdev->family) {
3166 rdev->config.cik.max_shader_engines = 2;
3167 rdev->config.cik.max_tile_pipes = 4;
3168 rdev->config.cik.max_cu_per_sh = 7;
3169 rdev->config.cik.max_sh_per_se = 1;
3170 rdev->config.cik.max_backends_per_se = 2;
3171 rdev->config.cik.max_texture_channel_caches = 4;
3172 rdev->config.cik.max_gprs = 256;
3173 rdev->config.cik.max_gs_threads = 32;
3174 rdev->config.cik.max_hw_contexts = 8;
3176 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3177 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3178 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3179 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3180 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3183 rdev->config.cik.max_shader_engines = 4;
3184 rdev->config.cik.max_tile_pipes = 16;
3185 rdev->config.cik.max_cu_per_sh = 11;
3186 rdev->config.cik.max_sh_per_se = 1;
3187 rdev->config.cik.max_backends_per_se = 4;
3188 rdev->config.cik.max_texture_channel_caches = 16;
3189 rdev->config.cik.max_gprs = 256;
3190 rdev->config.cik.max_gs_threads = 32;
3191 rdev->config.cik.max_hw_contexts = 8;
3193 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3194 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3195 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3196 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3197 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3200 rdev->config.cik.max_shader_engines = 1;
3201 rdev->config.cik.max_tile_pipes = 4;
3202 if ((rdev->pdev->device == 0x1304) ||
3203 (rdev->pdev->device == 0x1305) ||
3204 (rdev->pdev->device == 0x130C) ||
3205 (rdev->pdev->device == 0x130F) ||
3206 (rdev->pdev->device == 0x1310) ||
3207 (rdev->pdev->device == 0x1311) ||
3208 (rdev->pdev->device == 0x131C)) {
3209 rdev->config.cik.max_cu_per_sh = 8;
3210 rdev->config.cik.max_backends_per_se = 2;
3211 } else if ((rdev->pdev->device == 0x1309) ||
3212 (rdev->pdev->device == 0x130A) ||
3213 (rdev->pdev->device == 0x130D) ||
3214 (rdev->pdev->device == 0x1313) ||
3215 (rdev->pdev->device == 0x131D)) {
3216 rdev->config.cik.max_cu_per_sh = 6;
3217 rdev->config.cik.max_backends_per_se = 2;
3218 } else if ((rdev->pdev->device == 0x1306) ||
3219 (rdev->pdev->device == 0x1307) ||
3220 (rdev->pdev->device == 0x130B) ||
3221 (rdev->pdev->device == 0x130E) ||
3222 (rdev->pdev->device == 0x1315) ||
3223 (rdev->pdev->device == 0x131B)) {
3224 rdev->config.cik.max_cu_per_sh = 4;
3225 rdev->config.cik.max_backends_per_se = 1;
3227 rdev->config.cik.max_cu_per_sh = 3;
3228 rdev->config.cik.max_backends_per_se = 1;
3230 rdev->config.cik.max_sh_per_se = 1;
3231 rdev->config.cik.max_texture_channel_caches = 4;
3232 rdev->config.cik.max_gprs = 256;
3233 rdev->config.cik.max_gs_threads = 16;
3234 rdev->config.cik.max_hw_contexts = 8;
3236 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3237 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3238 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3239 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3240 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3244 rdev->config.cik.max_shader_engines = 1;
3245 rdev->config.cik.max_tile_pipes = 2;
3246 rdev->config.cik.max_cu_per_sh = 2;
3247 rdev->config.cik.max_sh_per_se = 1;
3248 rdev->config.cik.max_backends_per_se = 1;
3249 rdev->config.cik.max_texture_channel_caches = 2;
3250 rdev->config.cik.max_gprs = 256;
3251 rdev->config.cik.max_gs_threads = 16;
3252 rdev->config.cik.max_hw_contexts = 8;
3254 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3255 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3256 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3257 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3258 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3262 /* Initialize HDP */
3263 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3264 WREG32((0x2c14 + j), 0x00000000);
3265 WREG32((0x2c18 + j), 0x00000000);
3266 WREG32((0x2c1c + j), 0x00000000);
3267 WREG32((0x2c20 + j), 0x00000000);
3268 WREG32((0x2c24 + j), 0x00000000);
3271 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3273 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3275 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3276 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3278 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3279 rdev->config.cik.mem_max_burst_length_bytes = 256;
3280 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3281 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3282 if (rdev->config.cik.mem_row_size_in_kb > 4)
3283 rdev->config.cik.mem_row_size_in_kb = 4;
3284 /* XXX use MC settings? */
3285 rdev->config.cik.shader_engine_tile_size = 32;
3286 rdev->config.cik.num_gpus = 1;
3287 rdev->config.cik.multi_gpu_tile_size = 64;
3289 /* fix up row size */
3290 gb_addr_config &= ~ROW_SIZE_MASK;
3291 switch (rdev->config.cik.mem_row_size_in_kb) {
3294 gb_addr_config |= ROW_SIZE(0);
3297 gb_addr_config |= ROW_SIZE(1);
3300 gb_addr_config |= ROW_SIZE(2);
3304 /* setup tiling info dword. gb_addr_config is not adequate since it does
3305 * not have bank info, so create a custom tiling dword.
3306 * bits 3:0 num_pipes
3307 * bits 7:4 num_banks
3308 * bits 11:8 group_size
3309 * bits 15:12 row_size
3311 rdev->config.cik.tile_config = 0;
3312 switch (rdev->config.cik.num_tile_pipes) {
3314 rdev->config.cik.tile_config |= (0 << 0);
3317 rdev->config.cik.tile_config |= (1 << 0);
3320 rdev->config.cik.tile_config |= (2 << 0);
3324 /* XXX what about 12? */
3325 rdev->config.cik.tile_config |= (3 << 0);
3328 rdev->config.cik.tile_config |=
3329 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3330 rdev->config.cik.tile_config |=
3331 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3332 rdev->config.cik.tile_config |=
3333 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3335 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3336 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3337 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3338 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3339 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3340 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3341 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3342 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3344 cik_tiling_mode_table_init(rdev);
3346 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3347 rdev->config.cik.max_sh_per_se,
3348 rdev->config.cik.max_backends_per_se);
3350 /* set HW defaults for 3D engine */
3351 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3353 WREG32(SX_DEBUG_1, 0x20);
3355 WREG32(TA_CNTL_AUX, 0x00010000);
3357 tmp = RREG32(SPI_CONFIG_CNTL);
3359 WREG32(SPI_CONFIG_CNTL, tmp);
3361 WREG32(SQ_CONFIG, 1);
3363 WREG32(DB_DEBUG, 0);
3365 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3367 WREG32(DB_DEBUG2, tmp);
3369 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3371 WREG32(DB_DEBUG3, tmp);
3373 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3375 WREG32(CB_HW_CONTROL, tmp);
3377 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3379 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3380 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3381 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3382 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3384 WREG32(VGT_NUM_INSTANCES, 1);
3386 WREG32(CP_PERFMON_CNTL, 0);
3388 WREG32(SQ_CONFIG, 0);
3390 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3391 FORCE_EOV_MAX_REZ_CNT(255)));
3393 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3394 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3396 WREG32(VGT_GS_VERTEX_REUSE, 16);
3397 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3399 tmp = RREG32(HDP_MISC_CNTL);
3400 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3401 WREG32(HDP_MISC_CNTL, tmp);
3403 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3404 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3406 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3407 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3413 * GPU scratch registers helpers function.
3416 * cik_scratch_init - setup driver info for CP scratch regs
3418 * @rdev: radeon_device pointer
3420 * Set up the number and offset of the CP scratch registers.
3421 * NOTE: use of CP scratch registers is a legacy inferface and
3422 * is not used by default on newer asics (r6xx+). On newer asics,
3423 * memory buffers are used for fences rather than scratch regs.
3425 static void cik_scratch_init(struct radeon_device *rdev)
3429 rdev->scratch.num_reg = 7;
3430 rdev->scratch.reg_base = SCRATCH_REG0;
3431 for (i = 0; i < rdev->scratch.num_reg; i++) {
3432 rdev->scratch.free[i] = true;
3433 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3438 * cik_ring_test - basic gfx ring test
3440 * @rdev: radeon_device pointer
3441 * @ring: radeon_ring structure holding ring information
3443 * Allocate a scratch register and write to it using the gfx ring (CIK).
3444 * Provides a basic gfx ring test to verify that the ring is working.
3445 * Used by cik_cp_gfx_resume();
3446 * Returns 0 on success, error on failure.
3448 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3455 r = radeon_scratch_get(rdev, &scratch);
3457 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3460 WREG32(scratch, 0xCAFEDEAD);
3461 r = radeon_ring_lock(rdev, ring, 3);
3463 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3464 radeon_scratch_free(rdev, scratch);
3467 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3468 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3469 radeon_ring_write(ring, 0xDEADBEEF);
3470 radeon_ring_unlock_commit(rdev, ring);
3472 for (i = 0; i < rdev->usec_timeout; i++) {
3473 tmp = RREG32(scratch);
3474 if (tmp == 0xDEADBEEF)
3478 if (i < rdev->usec_timeout) {
3479 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3481 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3482 ring->idx, scratch, tmp);
3485 radeon_scratch_free(rdev, scratch);
3490 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3492 * @rdev: radeon_device pointer
3493 * @fence: radeon fence object
3495 * Emits a fence sequnce number on the gfx ring and flushes
3498 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3499 struct radeon_fence *fence)
3501 struct radeon_ring *ring = &rdev->ring[fence->ring];
3502 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3504 /* EVENT_WRITE_EOP - flush caches, send int */
3505 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3506 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3508 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3510 radeon_ring_write(ring, addr & 0xfffffffc);
3511 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3512 radeon_ring_write(ring, fence->seq);
3513 radeon_ring_write(ring, 0);
3515 /* We should be using the new WAIT_REG_MEM special op packet here
3516 * but it causes the CP to hang
3518 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3519 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3520 WRITE_DATA_DST_SEL(0)));
3521 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3522 radeon_ring_write(ring, 0);
3523 radeon_ring_write(ring, 0);
3527 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3529 * @rdev: radeon_device pointer
3530 * @fence: radeon fence object
3532 * Emits a fence sequnce number on the compute ring and flushes
3535 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3536 struct radeon_fence *fence)
3538 struct radeon_ring *ring = &rdev->ring[fence->ring];
3539 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3541 /* RELEASE_MEM - flush caches, send int */
3542 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3543 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3545 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3547 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3548 radeon_ring_write(ring, addr & 0xfffffffc);
3549 radeon_ring_write(ring, upper_32_bits(addr));
3550 radeon_ring_write(ring, fence->seq);
3551 radeon_ring_write(ring, 0);
3553 /* We should be using the new WAIT_REG_MEM special op packet here
3554 * but it causes the CP to hang
3556 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3557 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3558 WRITE_DATA_DST_SEL(0)));
3559 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3560 radeon_ring_write(ring, 0);
3561 radeon_ring_write(ring, 0);
3564 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3565 struct radeon_ring *ring,
3566 struct radeon_semaphore *semaphore,
3569 /* TODO: figure out why semaphore cause lockups */
3571 uint64_t addr = semaphore->gpu_addr;
3572 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3574 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3575 radeon_ring_write(ring, addr & 0xffffffff);
3576 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3585 * cik_copy_cpdma - copy pages using the CP DMA engine
3587 * @rdev: radeon_device pointer
3588 * @src_offset: src GPU address
3589 * @dst_offset: dst GPU address
3590 * @num_gpu_pages: number of GPU pages to xfer
3591 * @fence: radeon fence object
3593 * Copy GPU paging using the CP DMA engine (CIK+).
3594 * Used by the radeon ttm implementation to move pages if
3595 * registered as the asic copy callback.
3597 int cik_copy_cpdma(struct radeon_device *rdev,
3598 uint64_t src_offset, uint64_t dst_offset,
3599 unsigned num_gpu_pages,
3600 struct radeon_fence **fence)
3602 struct radeon_semaphore *sem = NULL;
3603 int ring_index = rdev->asic->copy.blit_ring_index;
3604 struct radeon_ring *ring = &rdev->ring[ring_index];
3605 u32 size_in_bytes, cur_size_in_bytes, control;
3609 r = radeon_semaphore_create(rdev, &sem);
3611 DRM_ERROR("radeon: moving bo (%d).\n", r);
3615 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3616 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3617 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3619 DRM_ERROR("radeon: moving bo (%d).\n", r);
3620 radeon_semaphore_free(rdev, &sem, NULL);
3624 radeon_semaphore_sync_to(sem, *fence);
3625 radeon_semaphore_sync_rings(rdev, sem, ring->idx);
3627 for (i = 0; i < num_loops; i++) {
3628 cur_size_in_bytes = size_in_bytes;
3629 if (cur_size_in_bytes > 0x1fffff)
3630 cur_size_in_bytes = 0x1fffff;
3631 size_in_bytes -= cur_size_in_bytes;
3633 if (size_in_bytes == 0)
3634 control |= PACKET3_DMA_DATA_CP_SYNC;
3635 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3636 radeon_ring_write(ring, control);
3637 radeon_ring_write(ring, lower_32_bits(src_offset));
3638 radeon_ring_write(ring, upper_32_bits(src_offset));
3639 radeon_ring_write(ring, lower_32_bits(dst_offset));
3640 radeon_ring_write(ring, upper_32_bits(dst_offset));
3641 radeon_ring_write(ring, cur_size_in_bytes);
3642 src_offset += cur_size_in_bytes;
3643 dst_offset += cur_size_in_bytes;
3646 r = radeon_fence_emit(rdev, fence, ring->idx);
3648 radeon_ring_unlock_undo(rdev, ring);
3652 radeon_ring_unlock_commit(rdev, ring);
3653 radeon_semaphore_free(rdev, &sem, *fence);
3662 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3664 * @rdev: radeon_device pointer
3665 * @ib: radeon indirect buffer object
3667 * Emits an DE (drawing engine) or CE (constant engine) IB
3668 * on the gfx ring. IBs are usually generated by userspace
3669 * acceleration drivers and submitted to the kernel for
3670 * sheduling on the ring. This function schedules the IB
3671 * on the gfx ring for execution by the GPU.
3673 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3675 struct radeon_ring *ring = &rdev->ring[ib->ring];
3676 u32 header, control = INDIRECT_BUFFER_VALID;
3678 if (ib->is_const_ib) {
3679 /* set switch buffer packet before const IB */
3680 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3681 radeon_ring_write(ring, 0);
3683 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3686 if (ring->rptr_save_reg) {
3687 next_rptr = ring->wptr + 3 + 4;
3688 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3689 radeon_ring_write(ring, ((ring->rptr_save_reg -
3690 PACKET3_SET_UCONFIG_REG_START) >> 2));
3691 radeon_ring_write(ring, next_rptr);
3692 } else if (rdev->wb.enabled) {
3693 next_rptr = ring->wptr + 5 + 4;
3694 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3695 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3696 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3697 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3698 radeon_ring_write(ring, next_rptr);
3701 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3704 control |= ib->length_dw |
3705 (ib->vm ? (ib->vm->id << 24) : 0);
3707 radeon_ring_write(ring, header);
3708 radeon_ring_write(ring,
3712 (ib->gpu_addr & 0xFFFFFFFC));
3713 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3714 radeon_ring_write(ring, control);
3718 * cik_ib_test - basic gfx ring IB test
3720 * @rdev: radeon_device pointer
3721 * @ring: radeon_ring structure holding ring information
3723 * Allocate an IB and execute it on the gfx ring (CIK).
3724 * Provides a basic gfx ring test to verify that IBs are working.
3725 * Returns 0 on success, error on failure.
3727 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3729 struct radeon_ib ib;
3735 r = radeon_scratch_get(rdev, &scratch);
3737 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3740 WREG32(scratch, 0xCAFEDEAD);
3741 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3743 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3744 radeon_scratch_free(rdev, scratch);
3747 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3748 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3749 ib.ptr[2] = 0xDEADBEEF;
3751 r = radeon_ib_schedule(rdev, &ib, NULL);
3753 radeon_scratch_free(rdev, scratch);
3754 radeon_ib_free(rdev, &ib);
3755 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3758 r = radeon_fence_wait(ib.fence, false);
3760 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3761 radeon_scratch_free(rdev, scratch);
3762 radeon_ib_free(rdev, &ib);
3765 for (i = 0; i < rdev->usec_timeout; i++) {
3766 tmp = RREG32(scratch);
3767 if (tmp == 0xDEADBEEF)
3771 if (i < rdev->usec_timeout) {
3772 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3774 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3778 radeon_scratch_free(rdev, scratch);
3779 radeon_ib_free(rdev, &ib);
3785 * On CIK, gfx and compute now have independant command processors.
3788 * Gfx consists of a single ring and can process both gfx jobs and
3789 * compute jobs. The gfx CP consists of three microengines (ME):
3790 * PFP - Pre-Fetch Parser
3792 * CE - Constant Engine
3793 * The PFP and ME make up what is considered the Drawing Engine (DE).
3794 * The CE is an asynchronous engine used for updating buffer desciptors
3795 * used by the DE so that they can be loaded into cache in parallel
3796 * while the DE is processing state update packets.
3799 * The compute CP consists of two microengines (ME):
3800 * MEC1 - Compute MicroEngine 1
3801 * MEC2 - Compute MicroEngine 2
3802 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3803 * The queues are exposed to userspace and are programmed directly
3804 * by the compute runtime.
3807 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3809 * @rdev: radeon_device pointer
3810 * @enable: enable or disable the MEs
3812 * Halts or unhalts the gfx MEs.
3814 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3817 WREG32(CP_ME_CNTL, 0);
3819 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3820 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3826 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3828 * @rdev: radeon_device pointer
3830 * Loads the gfx PFP, ME, and CE ucode.
3831 * Returns 0 for success, -EINVAL if the ucode is not available.
3833 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3835 const __be32 *fw_data;
3838 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3841 cik_cp_gfx_enable(rdev, false);
3844 fw_data = (const __be32 *)rdev->pfp_fw->data;
3845 WREG32(CP_PFP_UCODE_ADDR, 0);
3846 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3847 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3848 WREG32(CP_PFP_UCODE_ADDR, 0);
3851 fw_data = (const __be32 *)rdev->ce_fw->data;
3852 WREG32(CP_CE_UCODE_ADDR, 0);
3853 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3854 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3855 WREG32(CP_CE_UCODE_ADDR, 0);
3858 fw_data = (const __be32 *)rdev->me_fw->data;
3859 WREG32(CP_ME_RAM_WADDR, 0);
3860 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3861 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3862 WREG32(CP_ME_RAM_WADDR, 0);
3864 WREG32(CP_PFP_UCODE_ADDR, 0);
3865 WREG32(CP_CE_UCODE_ADDR, 0);
3866 WREG32(CP_ME_RAM_WADDR, 0);
3867 WREG32(CP_ME_RAM_RADDR, 0);
3872 * cik_cp_gfx_start - start the gfx ring
3874 * @rdev: radeon_device pointer
3876 * Enables the ring and loads the clear state context and other
3877 * packets required to init the ring.
3878 * Returns 0 for success, error for failure.
3880 static int cik_cp_gfx_start(struct radeon_device *rdev)
3882 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3886 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3887 WREG32(CP_ENDIAN_SWAP, 0);
3888 WREG32(CP_DEVICE_ID, 1);
3890 cik_cp_gfx_enable(rdev, true);
3892 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3894 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3898 /* init the CE partitions. CE only used for gfx on CIK */
3899 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3900 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3901 radeon_ring_write(ring, 0xc000);
3902 radeon_ring_write(ring, 0xc000);
3904 /* setup clear context state */
3905 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3906 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3908 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3909 radeon_ring_write(ring, 0x80000000);
3910 radeon_ring_write(ring, 0x80000000);
3912 for (i = 0; i < cik_default_size; i++)
3913 radeon_ring_write(ring, cik_default_state[i]);
3915 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3916 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3918 /* set clear context state */
3919 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3920 radeon_ring_write(ring, 0);
3922 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3923 radeon_ring_write(ring, 0x00000316);
3924 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3925 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3927 radeon_ring_unlock_commit(rdev, ring);
3933 * cik_cp_gfx_fini - stop the gfx ring
3935 * @rdev: radeon_device pointer
3937 * Stop the gfx ring and tear down the driver ring
3940 static void cik_cp_gfx_fini(struct radeon_device *rdev)
3942 cik_cp_gfx_enable(rdev, false);
3943 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3947 * cik_cp_gfx_resume - setup the gfx ring buffer registers
3949 * @rdev: radeon_device pointer
3951 * Program the location and size of the gfx ring buffer
3952 * and test it to make sure it's working.
3953 * Returns 0 for success, error for failure.
3955 static int cik_cp_gfx_resume(struct radeon_device *rdev)
3957 struct radeon_ring *ring;
3963 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3964 if (rdev->family != CHIP_HAWAII)
3965 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3967 /* Set the write pointer delay */
3968 WREG32(CP_RB_WPTR_DELAY, 0);
3970 /* set the RB to use vmid 0 */
3971 WREG32(CP_RB_VMID, 0);
3973 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3975 /* ring 0 - compute and gfx */
3976 /* Set ring buffer size */
3977 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3978 rb_bufsz = order_base_2(ring->ring_size / 8);
3979 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3981 tmp |= BUF_SWAP_32BIT;
3983 WREG32(CP_RB0_CNTL, tmp);
3985 /* Initialize the ring buffer's read and write pointers */
3986 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3988 WREG32(CP_RB0_WPTR, ring->wptr);
3990 /* set the wb address wether it's enabled or not */
3991 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3992 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3994 /* scratch register shadowing is no longer supported */
3995 WREG32(SCRATCH_UMSK, 0);
3997 if (!rdev->wb.enabled)
3998 tmp |= RB_NO_UPDATE;
4001 WREG32(CP_RB0_CNTL, tmp);
4003 rb_addr = ring->gpu_addr >> 8;
4004 WREG32(CP_RB0_BASE, rb_addr);
4005 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4007 ring->rptr = RREG32(CP_RB0_RPTR);
4009 /* start the ring */
4010 cik_cp_gfx_start(rdev);
4011 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4012 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4014 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4020 u32 cik_compute_ring_get_rptr(struct radeon_device *rdev,
4021 struct radeon_ring *ring)
4027 if (rdev->wb.enabled) {
4028 rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
4030 mutex_lock(&rdev->srbm_mutex);
4031 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4032 rptr = RREG32(CP_HQD_PQ_RPTR);
4033 cik_srbm_select(rdev, 0, 0, 0, 0);
4034 mutex_unlock(&rdev->srbm_mutex);
4040 u32 cik_compute_ring_get_wptr(struct radeon_device *rdev,
4041 struct radeon_ring *ring)
4045 if (rdev->wb.enabled) {
4046 wptr = le32_to_cpu(rdev->wb.wb[ring->wptr_offs/4]);
4048 mutex_lock(&rdev->srbm_mutex);
4049 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4050 wptr = RREG32(CP_HQD_PQ_WPTR);
4051 cik_srbm_select(rdev, 0, 0, 0, 0);
4052 mutex_unlock(&rdev->srbm_mutex);
4058 void cik_compute_ring_set_wptr(struct radeon_device *rdev,
4059 struct radeon_ring *ring)
4061 rdev->wb.wb[ring->wptr_offs/4] = cpu_to_le32(ring->wptr);
4062 WDOORBELL32(ring->doorbell_index, ring->wptr);
4066 * cik_cp_compute_enable - enable/disable the compute CP MEs
4068 * @rdev: radeon_device pointer
4069 * @enable: enable or disable the MEs
4071 * Halts or unhalts the compute MEs.
4073 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4076 WREG32(CP_MEC_CNTL, 0);
4078 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4083 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4085 * @rdev: radeon_device pointer
4087 * Loads the compute MEC1&2 ucode.
4088 * Returns 0 for success, -EINVAL if the ucode is not available.
4090 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4092 const __be32 *fw_data;
4098 cik_cp_compute_enable(rdev, false);
4101 fw_data = (const __be32 *)rdev->mec_fw->data;
4102 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4103 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4104 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4105 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4107 if (rdev->family == CHIP_KAVERI) {
4109 fw_data = (const __be32 *)rdev->mec_fw->data;
4110 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4111 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4112 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4113 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4120 * cik_cp_compute_start - start the compute queues
4122 * @rdev: radeon_device pointer
4124 * Enable the compute queues.
4125 * Returns 0 for success, error for failure.
4127 static int cik_cp_compute_start(struct radeon_device *rdev)
4129 cik_cp_compute_enable(rdev, true);
4135 * cik_cp_compute_fini - stop the compute queues
4137 * @rdev: radeon_device pointer
4139 * Stop the compute queues and tear down the driver queue
4142 static void cik_cp_compute_fini(struct radeon_device *rdev)
4146 cik_cp_compute_enable(rdev, false);
4148 for (i = 0; i < 2; i++) {
4150 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4152 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4154 if (rdev->ring[idx].mqd_obj) {
4155 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4156 if (unlikely(r != 0))
4157 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4159 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4160 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4162 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4163 rdev->ring[idx].mqd_obj = NULL;
4168 static void cik_mec_fini(struct radeon_device *rdev)
4172 if (rdev->mec.hpd_eop_obj) {
4173 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4174 if (unlikely(r != 0))
4175 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4176 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4177 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4179 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4180 rdev->mec.hpd_eop_obj = NULL;
4184 #define MEC_HPD_SIZE 2048
4186 static int cik_mec_init(struct radeon_device *rdev)
4192 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4193 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4195 if (rdev->family == CHIP_KAVERI)
4196 rdev->mec.num_mec = 2;
4198 rdev->mec.num_mec = 1;
4199 rdev->mec.num_pipe = 4;
4200 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4202 if (rdev->mec.hpd_eop_obj == NULL) {
4203 r = radeon_bo_create(rdev,
4204 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4206 RADEON_GEM_DOMAIN_GTT, NULL,
4207 &rdev->mec.hpd_eop_obj);
4209 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4214 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4215 if (unlikely(r != 0)) {
4219 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4220 &rdev->mec.hpd_eop_gpu_addr);
4222 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4226 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4228 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4233 /* clear memory. Not sure if this is required or not */
4234 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4236 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4237 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4242 struct hqd_registers
4244 u32 cp_mqd_base_addr;
4245 u32 cp_mqd_base_addr_hi;
4248 u32 cp_hqd_persistent_state;
4249 u32 cp_hqd_pipe_priority;
4250 u32 cp_hqd_queue_priority;
4253 u32 cp_hqd_pq_base_hi;
4255 u32 cp_hqd_pq_rptr_report_addr;
4256 u32 cp_hqd_pq_rptr_report_addr_hi;
4257 u32 cp_hqd_pq_wptr_poll_addr;
4258 u32 cp_hqd_pq_wptr_poll_addr_hi;
4259 u32 cp_hqd_pq_doorbell_control;
4261 u32 cp_hqd_pq_control;
4262 u32 cp_hqd_ib_base_addr;
4263 u32 cp_hqd_ib_base_addr_hi;
4265 u32 cp_hqd_ib_control;
4266 u32 cp_hqd_iq_timer;
4268 u32 cp_hqd_dequeue_request;
4269 u32 cp_hqd_dma_offload;
4270 u32 cp_hqd_sema_cmd;
4271 u32 cp_hqd_msg_type;
4272 u32 cp_hqd_atomic0_preop_lo;
4273 u32 cp_hqd_atomic0_preop_hi;
4274 u32 cp_hqd_atomic1_preop_lo;
4275 u32 cp_hqd_atomic1_preop_hi;
4276 u32 cp_hqd_hq_scheduler0;
4277 u32 cp_hqd_hq_scheduler1;
4284 u32 dispatch_initiator;
4288 u32 pipeline_stat_enable;
4289 u32 perf_counter_enable;
4295 u32 resource_limits;
4296 u32 static_thread_mgmt01[2];
4298 u32 static_thread_mgmt23[2];
4300 u32 thread_trace_enable;
4303 u32 vgtcs_invoke_count[2];
4304 struct hqd_registers queue_state;
4306 u32 interrupt_queue[64];
4310 * cik_cp_compute_resume - setup the compute queue registers
4312 * @rdev: radeon_device pointer
4314 * Program the compute queues and test them to make sure they
4316 * Returns 0 for success, error for failure.
4318 static int cik_cp_compute_resume(struct radeon_device *rdev)
4322 bool use_doorbell = true;
4328 struct bonaire_mqd *mqd;
4330 r = cik_cp_compute_start(rdev);
4334 /* fix up chicken bits */
4335 tmp = RREG32(CP_CPF_DEBUG);
4337 WREG32(CP_CPF_DEBUG, tmp);
4339 /* init the pipes */
4340 mutex_lock(&rdev->srbm_mutex);
4341 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4342 int me = (i < 4) ? 1 : 2;
4343 int pipe = (i < 4) ? i : (i - 4);
4345 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4347 cik_srbm_select(rdev, me, pipe, 0, 0);
4349 /* write the EOP addr */
4350 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4351 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4353 /* set the VMID assigned */
4354 WREG32(CP_HPD_EOP_VMID, 0);
4356 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4357 tmp = RREG32(CP_HPD_EOP_CONTROL);
4358 tmp &= ~EOP_SIZE_MASK;
4359 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4360 WREG32(CP_HPD_EOP_CONTROL, tmp);
4362 cik_srbm_select(rdev, 0, 0, 0, 0);
4363 mutex_unlock(&rdev->srbm_mutex);
4365 /* init the queues. Just two for now. */
4366 for (i = 0; i < 2; i++) {
4368 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4370 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4372 if (rdev->ring[idx].mqd_obj == NULL) {
4373 r = radeon_bo_create(rdev,
4374 sizeof(struct bonaire_mqd),
4376 RADEON_GEM_DOMAIN_GTT, NULL,
4377 &rdev->ring[idx].mqd_obj);
4379 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4384 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4385 if (unlikely(r != 0)) {
4386 cik_cp_compute_fini(rdev);
4389 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4392 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4393 cik_cp_compute_fini(rdev);
4396 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4398 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4399 cik_cp_compute_fini(rdev);
4403 /* init the mqd struct */
4404 memset(buf, 0, sizeof(struct bonaire_mqd));
4406 mqd = (struct bonaire_mqd *)buf;
4407 mqd->header = 0xC0310800;
4408 mqd->static_thread_mgmt01[0] = 0xffffffff;
4409 mqd->static_thread_mgmt01[1] = 0xffffffff;
4410 mqd->static_thread_mgmt23[0] = 0xffffffff;
4411 mqd->static_thread_mgmt23[1] = 0xffffffff;
4413 mutex_lock(&rdev->srbm_mutex);
4414 cik_srbm_select(rdev, rdev->ring[idx].me,
4415 rdev->ring[idx].pipe,
4416 rdev->ring[idx].queue, 0);
4418 /* disable wptr polling */
4419 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4420 tmp &= ~WPTR_POLL_EN;
4421 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4423 /* enable doorbell? */
4424 mqd->queue_state.cp_hqd_pq_doorbell_control =
4425 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4427 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4429 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4430 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4431 mqd->queue_state.cp_hqd_pq_doorbell_control);
4433 /* disable the queue if it's active */
4434 mqd->queue_state.cp_hqd_dequeue_request = 0;
4435 mqd->queue_state.cp_hqd_pq_rptr = 0;
4436 mqd->queue_state.cp_hqd_pq_wptr= 0;
4437 if (RREG32(CP_HQD_ACTIVE) & 1) {
4438 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4439 for (i = 0; i < rdev->usec_timeout; i++) {
4440 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4444 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4445 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4446 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4449 /* set the pointer to the MQD */
4450 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4451 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4452 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4453 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4454 /* set MQD vmid to 0 */
4455 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4456 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4457 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4459 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4460 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4461 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4462 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4463 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4464 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4466 /* set up the HQD, this is similar to CP_RB0_CNTL */
4467 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4468 mqd->queue_state.cp_hqd_pq_control &=
4469 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4471 mqd->queue_state.cp_hqd_pq_control |=
4472 order_base_2(rdev->ring[idx].ring_size / 8);
4473 mqd->queue_state.cp_hqd_pq_control |=
4474 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4476 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4478 mqd->queue_state.cp_hqd_pq_control &=
4479 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4480 mqd->queue_state.cp_hqd_pq_control |=
4481 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4482 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4484 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4486 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4488 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4489 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4490 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4491 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4492 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4493 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4495 /* set the wb address wether it's enabled or not */
4497 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4499 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4500 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4501 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4502 upper_32_bits(wb_gpu_addr) & 0xffff;
4503 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4504 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4505 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4506 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4508 /* enable the doorbell if requested */
4510 mqd->queue_state.cp_hqd_pq_doorbell_control =
4511 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4512 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4513 mqd->queue_state.cp_hqd_pq_doorbell_control |=
4514 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
4515 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4516 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4517 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4520 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4522 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4523 mqd->queue_state.cp_hqd_pq_doorbell_control);
4525 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4526 rdev->ring[idx].wptr = 0;
4527 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4528 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4529 rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
4530 mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
4532 /* set the vmid for the queue */
4533 mqd->queue_state.cp_hqd_vmid = 0;
4534 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4536 /* activate the queue */
4537 mqd->queue_state.cp_hqd_active = 1;
4538 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4540 cik_srbm_select(rdev, 0, 0, 0, 0);
4541 mutex_unlock(&rdev->srbm_mutex);
4543 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4544 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4546 rdev->ring[idx].ready = true;
4547 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4549 rdev->ring[idx].ready = false;
4555 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4557 cik_cp_gfx_enable(rdev, enable);
4558 cik_cp_compute_enable(rdev, enable);
4561 static int cik_cp_load_microcode(struct radeon_device *rdev)
4565 r = cik_cp_gfx_load_microcode(rdev);
4568 r = cik_cp_compute_load_microcode(rdev);
4575 static void cik_cp_fini(struct radeon_device *rdev)
4577 cik_cp_gfx_fini(rdev);
4578 cik_cp_compute_fini(rdev);
4581 static int cik_cp_resume(struct radeon_device *rdev)
4585 cik_enable_gui_idle_interrupt(rdev, false);
4587 r = cik_cp_load_microcode(rdev);
4591 r = cik_cp_gfx_resume(rdev);
4594 r = cik_cp_compute_resume(rdev);
4598 cik_enable_gui_idle_interrupt(rdev, true);
4603 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4605 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4606 RREG32(GRBM_STATUS));
4607 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4608 RREG32(GRBM_STATUS2));
4609 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4610 RREG32(GRBM_STATUS_SE0));
4611 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4612 RREG32(GRBM_STATUS_SE1));
4613 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4614 RREG32(GRBM_STATUS_SE2));
4615 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4616 RREG32(GRBM_STATUS_SE3));
4617 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4618 RREG32(SRBM_STATUS));
4619 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4620 RREG32(SRBM_STATUS2));
4621 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4622 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4623 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4624 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
4625 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4626 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4627 RREG32(CP_STALLED_STAT1));
4628 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4629 RREG32(CP_STALLED_STAT2));
4630 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4631 RREG32(CP_STALLED_STAT3));
4632 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4633 RREG32(CP_CPF_BUSY_STAT));
4634 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4635 RREG32(CP_CPF_STALLED_STAT1));
4636 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4637 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4638 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4639 RREG32(CP_CPC_STALLED_STAT1));
4640 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
4644 * cik_gpu_check_soft_reset - check which blocks are busy
4646 * @rdev: radeon_device pointer
4648 * Check which blocks are busy and return the relevant reset
4649 * mask to be used by cik_gpu_soft_reset().
4650 * Returns a mask of the blocks to be reset.
4652 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
4658 tmp = RREG32(GRBM_STATUS);
4659 if (tmp & (PA_BUSY | SC_BUSY |
4660 BCI_BUSY | SX_BUSY |
4661 TA_BUSY | VGT_BUSY |
4663 GDS_BUSY | SPI_BUSY |
4664 IA_BUSY | IA_BUSY_NO_DMA))
4665 reset_mask |= RADEON_RESET_GFX;
4667 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4668 reset_mask |= RADEON_RESET_CP;
4671 tmp = RREG32(GRBM_STATUS2);
4673 reset_mask |= RADEON_RESET_RLC;
4675 /* SDMA0_STATUS_REG */
4676 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4677 if (!(tmp & SDMA_IDLE))
4678 reset_mask |= RADEON_RESET_DMA;
4680 /* SDMA1_STATUS_REG */
4681 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4682 if (!(tmp & SDMA_IDLE))
4683 reset_mask |= RADEON_RESET_DMA1;
4686 tmp = RREG32(SRBM_STATUS2);
4687 if (tmp & SDMA_BUSY)
4688 reset_mask |= RADEON_RESET_DMA;
4690 if (tmp & SDMA1_BUSY)
4691 reset_mask |= RADEON_RESET_DMA1;
4694 tmp = RREG32(SRBM_STATUS);
4697 reset_mask |= RADEON_RESET_IH;
4700 reset_mask |= RADEON_RESET_SEM;
4702 if (tmp & GRBM_RQ_PENDING)
4703 reset_mask |= RADEON_RESET_GRBM;
4706 reset_mask |= RADEON_RESET_VMC;
4708 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4709 MCC_BUSY | MCD_BUSY))
4710 reset_mask |= RADEON_RESET_MC;
4712 if (evergreen_is_display_hung(rdev))
4713 reset_mask |= RADEON_RESET_DISPLAY;
4715 /* Skip MC reset as it's mostly likely not hung, just busy */
4716 if (reset_mask & RADEON_RESET_MC) {
4717 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4718 reset_mask &= ~RADEON_RESET_MC;
4725 * cik_gpu_soft_reset - soft reset GPU
4727 * @rdev: radeon_device pointer
4728 * @reset_mask: mask of which blocks to reset
4730 * Soft reset the blocks specified in @reset_mask.
4732 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4734 struct evergreen_mc_save save;
4735 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4738 if (reset_mask == 0)
4741 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4743 cik_print_gpu_status_regs(rdev);
4744 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4745 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4746 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4747 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4756 /* Disable GFX parsing/prefetching */
4757 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4759 /* Disable MEC parsing/prefetching */
4760 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4762 if (reset_mask & RADEON_RESET_DMA) {
4764 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4766 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4768 if (reset_mask & RADEON_RESET_DMA1) {
4770 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4772 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4775 evergreen_mc_stop(rdev, &save);
4776 if (evergreen_mc_wait_for_idle(rdev)) {
4777 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4780 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4781 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4783 if (reset_mask & RADEON_RESET_CP) {
4784 grbm_soft_reset |= SOFT_RESET_CP;
4786 srbm_soft_reset |= SOFT_RESET_GRBM;
4789 if (reset_mask & RADEON_RESET_DMA)
4790 srbm_soft_reset |= SOFT_RESET_SDMA;
4792 if (reset_mask & RADEON_RESET_DMA1)
4793 srbm_soft_reset |= SOFT_RESET_SDMA1;
4795 if (reset_mask & RADEON_RESET_DISPLAY)
4796 srbm_soft_reset |= SOFT_RESET_DC;
4798 if (reset_mask & RADEON_RESET_RLC)
4799 grbm_soft_reset |= SOFT_RESET_RLC;
4801 if (reset_mask & RADEON_RESET_SEM)
4802 srbm_soft_reset |= SOFT_RESET_SEM;
4804 if (reset_mask & RADEON_RESET_IH)
4805 srbm_soft_reset |= SOFT_RESET_IH;
4807 if (reset_mask & RADEON_RESET_GRBM)
4808 srbm_soft_reset |= SOFT_RESET_GRBM;
4810 if (reset_mask & RADEON_RESET_VMC)
4811 srbm_soft_reset |= SOFT_RESET_VMC;
4813 if (!(rdev->flags & RADEON_IS_IGP)) {
4814 if (reset_mask & RADEON_RESET_MC)
4815 srbm_soft_reset |= SOFT_RESET_MC;
4818 if (grbm_soft_reset) {
4819 tmp = RREG32(GRBM_SOFT_RESET);
4820 tmp |= grbm_soft_reset;
4821 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4822 WREG32(GRBM_SOFT_RESET, tmp);
4823 tmp = RREG32(GRBM_SOFT_RESET);
4827 tmp &= ~grbm_soft_reset;
4828 WREG32(GRBM_SOFT_RESET, tmp);
4829 tmp = RREG32(GRBM_SOFT_RESET);
4832 if (srbm_soft_reset) {
4833 tmp = RREG32(SRBM_SOFT_RESET);
4834 tmp |= srbm_soft_reset;
4835 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4836 WREG32(SRBM_SOFT_RESET, tmp);
4837 tmp = RREG32(SRBM_SOFT_RESET);
4841 tmp &= ~srbm_soft_reset;
4842 WREG32(SRBM_SOFT_RESET, tmp);
4843 tmp = RREG32(SRBM_SOFT_RESET);
4846 /* Wait a little for things to settle down */
4849 evergreen_mc_resume(rdev, &save);
4852 cik_print_gpu_status_regs(rdev);
4856 * cik_asic_reset - soft reset GPU
4858 * @rdev: radeon_device pointer
4860 * Look up which blocks are hung and attempt
4862 * Returns 0 for success.
4864 int cik_asic_reset(struct radeon_device *rdev)
4868 reset_mask = cik_gpu_check_soft_reset(rdev);
4871 r600_set_bios_scratch_engine_hung(rdev, true);
4873 cik_gpu_soft_reset(rdev, reset_mask);
4875 reset_mask = cik_gpu_check_soft_reset(rdev);
4878 r600_set_bios_scratch_engine_hung(rdev, false);
4884 * cik_gfx_is_lockup - check if the 3D engine is locked up
4886 * @rdev: radeon_device pointer
4887 * @ring: radeon_ring structure holding ring information
4889 * Check if the 3D engine is locked up (CIK).
4890 * Returns true if the engine is locked, false if not.
4892 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4894 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
4896 if (!(reset_mask & (RADEON_RESET_GFX |
4897 RADEON_RESET_COMPUTE |
4898 RADEON_RESET_CP))) {
4899 radeon_ring_lockup_update(ring);
4902 /* force CP activities */
4903 radeon_ring_force_activity(rdev, ring);
4904 return radeon_ring_test_lockup(rdev, ring);
4909 * cik_mc_program - program the GPU memory controller
4911 * @rdev: radeon_device pointer
4913 * Set the location of vram, gart, and AGP in the GPU's
4914 * physical address space (CIK).
4916 static void cik_mc_program(struct radeon_device *rdev)
4918 struct evergreen_mc_save save;
4922 /* Initialize HDP */
4923 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4924 WREG32((0x2c14 + j), 0x00000000);
4925 WREG32((0x2c18 + j), 0x00000000);
4926 WREG32((0x2c1c + j), 0x00000000);
4927 WREG32((0x2c20 + j), 0x00000000);
4928 WREG32((0x2c24 + j), 0x00000000);
4930 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4932 evergreen_mc_stop(rdev, &save);
4933 if (radeon_mc_wait_for_idle(rdev)) {
4934 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4936 /* Lockout access through VGA aperture*/
4937 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4938 /* Update configuration */
4939 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4940 rdev->mc.vram_start >> 12);
4941 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4942 rdev->mc.vram_end >> 12);
4943 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4944 rdev->vram_scratch.gpu_addr >> 12);
4945 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4946 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4947 WREG32(MC_VM_FB_LOCATION, tmp);
4948 /* XXX double check these! */
4949 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4950 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4951 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4952 WREG32(MC_VM_AGP_BASE, 0);
4953 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4954 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4955 if (radeon_mc_wait_for_idle(rdev)) {
4956 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4958 evergreen_mc_resume(rdev, &save);
4959 /* we need to own VRAM, so turn off the VGA renderer here
4960 * to stop it overwriting our objects */
4961 rv515_vga_render_disable(rdev);
4965 * cik_mc_init - initialize the memory controller driver params
4967 * @rdev: radeon_device pointer
4969 * Look up the amount of vram, vram width, and decide how to place
4970 * vram and gart within the GPU's physical address space (CIK).
4971 * Returns 0 for success.
4973 static int cik_mc_init(struct radeon_device *rdev)
4976 int chansize, numchan;
4978 /* Get VRAM informations */
4979 rdev->mc.vram_is_ddr = true;
4980 tmp = RREG32(MC_ARB_RAMCFG);
4981 if (tmp & CHANSIZE_MASK) {
4986 tmp = RREG32(MC_SHARED_CHMAP);
4987 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5017 rdev->mc.vram_width = numchan * chansize;
5018 /* Could aper size report 0 ? */
5019 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5020 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5021 /* size in MB on si */
5022 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5023 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5024 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5025 si_vram_gtt_location(rdev, &rdev->mc);
5026 radeon_update_bandwidth_info(rdev);
5033 * VMID 0 is the physical GPU addresses as used by the kernel.
5034 * VMIDs 1-15 are used for userspace clients and are handled
5035 * by the radeon vm/hsa code.
5038 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5040 * @rdev: radeon_device pointer
5042 * Flush the TLB for the VMID 0 page table (CIK).
5044 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5046 /* flush hdp cache */
5047 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5049 /* bits 0-15 are the VM contexts0-15 */
5050 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5054 * cik_pcie_gart_enable - gart enable
5056 * @rdev: radeon_device pointer
5058 * This sets up the TLBs, programs the page tables for VMID0,
5059 * sets up the hw for VMIDs 1-15 which are allocated on
5060 * demand, and sets up the global locations for the LDS, GDS,
5061 * and GPUVM for FSA64 clients (CIK).
5062 * Returns 0 for success, errors for failure.
5064 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5068 if (rdev->gart.robj == NULL) {
5069 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5072 r = radeon_gart_table_vram_pin(rdev);
5075 radeon_gart_restore(rdev);
5076 /* Setup TLB control */
5077 WREG32(MC_VM_MX_L1_TLB_CNTL,
5080 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5081 ENABLE_ADVANCED_DRIVER_MODEL |
5082 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5083 /* Setup L2 cache */
5084 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5085 ENABLE_L2_FRAGMENT_PROCESSING |
5086 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5087 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5088 EFFECTIVE_L2_QUEUE_SIZE(7) |
5089 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5090 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5091 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5092 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5093 /* setup context0 */
5094 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5095 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5096 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5097 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5098 (u32)(rdev->dummy_page.addr >> 12));
5099 WREG32(VM_CONTEXT0_CNTL2, 0);
5100 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5101 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5107 /* empty context1-15 */
5108 /* FIXME start with 4G, once using 2 level pt switch to full
5111 /* set vm size, must be a multiple of 4 */
5112 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5113 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5114 for (i = 1; i < 16; i++) {
5116 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5117 rdev->gart.table_addr >> 12);
5119 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5120 rdev->gart.table_addr >> 12);
5123 /* enable context1-15 */
5124 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5125 (u32)(rdev->dummy_page.addr >> 12));
5126 WREG32(VM_CONTEXT1_CNTL2, 4);
5127 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5128 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5129 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5130 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5131 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5132 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5133 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5134 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5135 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5136 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5137 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5138 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5139 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5141 /* TC cache setup ??? */
5142 WREG32(TC_CFG_L1_LOAD_POLICY0, 0);
5143 WREG32(TC_CFG_L1_LOAD_POLICY1, 0);
5144 WREG32(TC_CFG_L1_STORE_POLICY, 0);
5146 WREG32(TC_CFG_L2_LOAD_POLICY0, 0);
5147 WREG32(TC_CFG_L2_LOAD_POLICY1, 0);
5148 WREG32(TC_CFG_L2_STORE_POLICY0, 0);
5149 WREG32(TC_CFG_L2_STORE_POLICY1, 0);
5150 WREG32(TC_CFG_L2_ATOMIC_POLICY, 0);
5152 WREG32(TC_CFG_L1_VOLATILE, 0);
5153 WREG32(TC_CFG_L2_VOLATILE, 0);
5155 if (rdev->family == CHIP_KAVERI) {
5156 u32 tmp = RREG32(CHUB_CONTROL);
5158 WREG32(CHUB_CONTROL, tmp);
5161 /* XXX SH_MEM regs */
5162 /* where to put LDS, scratch, GPUVM in FSA64 space */
5163 mutex_lock(&rdev->srbm_mutex);
5164 for (i = 0; i < 16; i++) {
5165 cik_srbm_select(rdev, 0, 0, 0, i);
5166 /* CP and shaders */
5167 WREG32(SH_MEM_CONFIG, 0);
5168 WREG32(SH_MEM_APE1_BASE, 1);
5169 WREG32(SH_MEM_APE1_LIMIT, 0);
5170 WREG32(SH_MEM_BASES, 0);
5172 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5173 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5174 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5175 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5176 /* XXX SDMA RLC - todo */
5178 cik_srbm_select(rdev, 0, 0, 0, 0);
5179 mutex_unlock(&rdev->srbm_mutex);
5181 cik_pcie_gart_tlb_flush(rdev);
5182 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5183 (unsigned)(rdev->mc.gtt_size >> 20),
5184 (unsigned long long)rdev->gart.table_addr);
5185 rdev->gart.ready = true;
5190 * cik_pcie_gart_disable - gart disable
5192 * @rdev: radeon_device pointer
5194 * This disables all VM page table (CIK).
5196 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5198 /* Disable all tables */
5199 WREG32(VM_CONTEXT0_CNTL, 0);
5200 WREG32(VM_CONTEXT1_CNTL, 0);
5201 /* Setup TLB control */
5202 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5203 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5204 /* Setup L2 cache */
5206 ENABLE_L2_FRAGMENT_PROCESSING |
5207 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5208 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5209 EFFECTIVE_L2_QUEUE_SIZE(7) |
5210 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5211 WREG32(VM_L2_CNTL2, 0);
5212 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5213 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5214 radeon_gart_table_vram_unpin(rdev);
5218 * cik_pcie_gart_fini - vm fini callback
5220 * @rdev: radeon_device pointer
5222 * Tears down the driver GART/VM setup (CIK).
5224 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5226 cik_pcie_gart_disable(rdev);
5227 radeon_gart_table_vram_free(rdev);
5228 radeon_gart_fini(rdev);
5233 * cik_ib_parse - vm ib_parse callback
5235 * @rdev: radeon_device pointer
5236 * @ib: indirect buffer pointer
5238 * CIK uses hw IB checking so this is a nop (CIK).
5240 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5247 * VMID 0 is the physical GPU addresses as used by the kernel.
5248 * VMIDs 1-15 are used for userspace clients and are handled
5249 * by the radeon vm/hsa code.
5252 * cik_vm_init - cik vm init callback
5254 * @rdev: radeon_device pointer
5256 * Inits cik specific vm parameters (number of VMs, base of vram for
5257 * VMIDs 1-15) (CIK).
5258 * Returns 0 for success.
5260 int cik_vm_init(struct radeon_device *rdev)
5263 rdev->vm_manager.nvm = 16;
5264 /* base offset of vram pages */
5265 if (rdev->flags & RADEON_IS_IGP) {
5266 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5268 rdev->vm_manager.vram_base_offset = tmp;
5270 rdev->vm_manager.vram_base_offset = 0;
5276 * cik_vm_fini - cik vm fini callback
5278 * @rdev: radeon_device pointer
5280 * Tear down any asic specific VM setup (CIK).
5282 void cik_vm_fini(struct radeon_device *rdev)
5287 * cik_vm_decode_fault - print human readable fault info
5289 * @rdev: radeon_device pointer
5290 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5291 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5293 * Print human readable fault information (CIK).
5295 static void cik_vm_decode_fault(struct radeon_device *rdev,
5296 u32 status, u32 addr, u32 mc_client)
5299 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5300 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5301 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5302 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5304 if (rdev->family == CHIP_HAWAII)
5305 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5307 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5309 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5310 protections, vmid, addr,
5311 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5312 block, mc_client, mc_id);
5316 * cik_vm_flush - cik vm flush using the CP
5318 * @rdev: radeon_device pointer
5320 * Update the page table base and flush the VM TLB
5321 * using the CP (CIK).
5323 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5325 struct radeon_ring *ring = &rdev->ring[ridx];
5330 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5331 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5332 WRITE_DATA_DST_SEL(0)));
5334 radeon_ring_write(ring,
5335 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5337 radeon_ring_write(ring,
5338 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5340 radeon_ring_write(ring, 0);
5341 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5343 /* update SH_MEM_* regs */
5344 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5345 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5346 WRITE_DATA_DST_SEL(0)));
5347 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5348 radeon_ring_write(ring, 0);
5349 radeon_ring_write(ring, VMID(vm->id));
5351 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5352 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5353 WRITE_DATA_DST_SEL(0)));
5354 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5355 radeon_ring_write(ring, 0);
5357 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5358 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5359 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5360 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5362 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5363 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5364 WRITE_DATA_DST_SEL(0)));
5365 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5366 radeon_ring_write(ring, 0);
5367 radeon_ring_write(ring, VMID(0));
5370 /* We should be using the WAIT_REG_MEM packet here like in
5371 * cik_fence_ring_emit(), but it causes the CP to hang in this
5374 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5375 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5376 WRITE_DATA_DST_SEL(0)));
5377 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5378 radeon_ring_write(ring, 0);
5379 radeon_ring_write(ring, 0);
5381 /* bits 0-15 are the VM contexts0-15 */
5382 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5383 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5384 WRITE_DATA_DST_SEL(0)));
5385 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5386 radeon_ring_write(ring, 0);
5387 radeon_ring_write(ring, 1 << vm->id);
5389 /* compute doesn't have PFP */
5390 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
5391 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5392 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5393 radeon_ring_write(ring, 0x0);
5399 * The RLC is a multi-purpose microengine that handles a
5400 * variety of functions, the most important of which is
5401 * the interrupt controller.
5403 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5406 u32 tmp = RREG32(CP_INT_CNTL_RING0);
5409 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5411 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5412 WREG32(CP_INT_CNTL_RING0, tmp);
5415 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5419 tmp = RREG32(RLC_LB_CNTL);
5421 tmp |= LOAD_BALANCE_ENABLE;
5423 tmp &= ~LOAD_BALANCE_ENABLE;
5424 WREG32(RLC_LB_CNTL, tmp);
5427 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5432 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5433 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5434 cik_select_se_sh(rdev, i, j);
5435 for (k = 0; k < rdev->usec_timeout; k++) {
5436 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5442 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5444 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5445 for (k = 0; k < rdev->usec_timeout; k++) {
5446 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5452 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5456 tmp = RREG32(RLC_CNTL);
5458 WREG32(RLC_CNTL, rlc);
5461 static u32 cik_halt_rlc(struct radeon_device *rdev)
5465 orig = data = RREG32(RLC_CNTL);
5467 if (data & RLC_ENABLE) {
5470 data &= ~RLC_ENABLE;
5471 WREG32(RLC_CNTL, data);
5473 for (i = 0; i < rdev->usec_timeout; i++) {
5474 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5479 cik_wait_for_rlc_serdes(rdev);
5485 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5489 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5490 WREG32(RLC_GPR_REG2, tmp);
5492 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5493 for (i = 0; i < rdev->usec_timeout; i++) {
5494 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5499 for (i = 0; i < rdev->usec_timeout; i++) {
5500 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5506 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5510 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5511 WREG32(RLC_GPR_REG2, tmp);
5515 * cik_rlc_stop - stop the RLC ME
5517 * @rdev: radeon_device pointer
5519 * Halt the RLC ME (MicroEngine) (CIK).
5521 static void cik_rlc_stop(struct radeon_device *rdev)
5523 WREG32(RLC_CNTL, 0);
5525 cik_enable_gui_idle_interrupt(rdev, false);
5527 cik_wait_for_rlc_serdes(rdev);
5531 * cik_rlc_start - start the RLC ME
5533 * @rdev: radeon_device pointer
5535 * Unhalt the RLC ME (MicroEngine) (CIK).
5537 static void cik_rlc_start(struct radeon_device *rdev)
5539 WREG32(RLC_CNTL, RLC_ENABLE);
5541 cik_enable_gui_idle_interrupt(rdev, true);
5547 * cik_rlc_resume - setup the RLC hw
5549 * @rdev: radeon_device pointer
5551 * Initialize the RLC registers, load the ucode,
5552 * and start the RLC (CIK).
5553 * Returns 0 for success, -EINVAL if the ucode is not available.
5555 static int cik_rlc_resume(struct radeon_device *rdev)
5558 const __be32 *fw_data;
5563 switch (rdev->family) {
5567 size = BONAIRE_RLC_UCODE_SIZE;
5570 size = KV_RLC_UCODE_SIZE;
5573 size = KB_RLC_UCODE_SIZE;
5580 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5581 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5589 WREG32(RLC_LB_CNTR_INIT, 0);
5590 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5592 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5593 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5594 WREG32(RLC_LB_PARAMS, 0x00600408);
5595 WREG32(RLC_LB_CNTL, 0x80000004);
5597 WREG32(RLC_MC_CNTL, 0);
5598 WREG32(RLC_UCODE_CNTL, 0);
5600 fw_data = (const __be32 *)rdev->rlc_fw->data;
5601 WREG32(RLC_GPM_UCODE_ADDR, 0);
5602 for (i = 0; i < size; i++)
5603 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5604 WREG32(RLC_GPM_UCODE_ADDR, 0);
5606 /* XXX - find out what chips support lbpw */
5607 cik_enable_lbpw(rdev, false);
5609 if (rdev->family == CHIP_BONAIRE)
5610 WREG32(RLC_DRIVER_DMA_STATUS, 0);
5612 cik_rlc_start(rdev);
5617 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5619 u32 data, orig, tmp, tmp2;
5621 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5623 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5624 cik_enable_gui_idle_interrupt(rdev, true);
5626 tmp = cik_halt_rlc(rdev);
5628 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5629 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5630 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5631 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5632 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5634 cik_update_rlc(rdev, tmp);
5636 data |= CGCG_EN | CGLS_EN;
5638 cik_enable_gui_idle_interrupt(rdev, false);
5640 RREG32(CB_CGTT_SCLK_CTRL);
5641 RREG32(CB_CGTT_SCLK_CTRL);
5642 RREG32(CB_CGTT_SCLK_CTRL);
5643 RREG32(CB_CGTT_SCLK_CTRL);
5645 data &= ~(CGCG_EN | CGLS_EN);
5649 WREG32(RLC_CGCG_CGLS_CTRL, data);
5653 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5655 u32 data, orig, tmp = 0;
5657 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5658 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5659 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5660 orig = data = RREG32(CP_MEM_SLP_CNTL);
5661 data |= CP_MEM_LS_EN;
5663 WREG32(CP_MEM_SLP_CNTL, data);
5667 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5670 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5672 tmp = cik_halt_rlc(rdev);
5674 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5675 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5676 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5677 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5678 WREG32(RLC_SERDES_WR_CTRL, data);
5680 cik_update_rlc(rdev, tmp);
5682 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5683 orig = data = RREG32(CGTS_SM_CTRL_REG);
5684 data &= ~SM_MODE_MASK;
5685 data |= SM_MODE(0x2);
5686 data |= SM_MODE_ENABLE;
5687 data &= ~CGTS_OVERRIDE;
5688 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5689 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5690 data &= ~CGTS_LS_OVERRIDE;
5691 data &= ~ON_MONITOR_ADD_MASK;
5692 data |= ON_MONITOR_ADD_EN;
5693 data |= ON_MONITOR_ADD(0x96);
5695 WREG32(CGTS_SM_CTRL_REG, data);
5698 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5701 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5703 data = RREG32(RLC_MEM_SLP_CNTL);
5704 if (data & RLC_MEM_LS_EN) {
5705 data &= ~RLC_MEM_LS_EN;
5706 WREG32(RLC_MEM_SLP_CNTL, data);
5709 data = RREG32(CP_MEM_SLP_CNTL);
5710 if (data & CP_MEM_LS_EN) {
5711 data &= ~CP_MEM_LS_EN;
5712 WREG32(CP_MEM_SLP_CNTL, data);
5715 orig = data = RREG32(CGTS_SM_CTRL_REG);
5716 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5718 WREG32(CGTS_SM_CTRL_REG, data);
5720 tmp = cik_halt_rlc(rdev);
5722 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5723 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5724 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5725 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5726 WREG32(RLC_SERDES_WR_CTRL, data);
5728 cik_update_rlc(rdev, tmp);
5732 static const u32 mc_cg_registers[] =
5745 static void cik_enable_mc_ls(struct radeon_device *rdev,
5751 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5752 orig = data = RREG32(mc_cg_registers[i]);
5753 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5754 data |= MC_LS_ENABLE;
5756 data &= ~MC_LS_ENABLE;
5758 WREG32(mc_cg_registers[i], data);
5762 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5768 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5769 orig = data = RREG32(mc_cg_registers[i]);
5770 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5771 data |= MC_CG_ENABLE;
5773 data &= ~MC_CG_ENABLE;
5775 WREG32(mc_cg_registers[i], data);
5779 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
5784 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5785 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
5786 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
5788 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
5791 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
5793 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
5796 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
5800 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
5805 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
5806 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5809 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5811 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5814 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5816 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5819 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5821 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5824 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5828 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
5833 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5834 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5836 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
5838 orig = data = RREG32(UVD_CGC_CTRL);
5841 WREG32(UVD_CGC_CTRL, data);
5843 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5845 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
5847 orig = data = RREG32(UVD_CGC_CTRL);
5850 WREG32(UVD_CGC_CTRL, data);
5854 static void cik_enable_bif_mgls(struct radeon_device *rdev,
5859 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
5861 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5862 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5863 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5865 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5866 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5869 WREG32_PCIE_PORT(PCIE_CNTL2, data);
5872 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
5877 orig = data = RREG32(HDP_HOST_PATH_CNTL);
5879 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5880 data &= ~CLOCK_GATING_DIS;
5882 data |= CLOCK_GATING_DIS;
5885 WREG32(HDP_HOST_PATH_CNTL, data);
5888 static void cik_enable_hdp_ls(struct radeon_device *rdev,
5893 orig = data = RREG32(HDP_MEM_POWER_LS);
5895 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5896 data |= HDP_LS_ENABLE;
5898 data &= ~HDP_LS_ENABLE;
5901 WREG32(HDP_MEM_POWER_LS, data);
5904 void cik_update_cg(struct radeon_device *rdev,
5905 u32 block, bool enable)
5908 if (block & RADEON_CG_BLOCK_GFX) {
5909 cik_enable_gui_idle_interrupt(rdev, false);
5910 /* order matters! */
5912 cik_enable_mgcg(rdev, true);
5913 cik_enable_cgcg(rdev, true);
5915 cik_enable_cgcg(rdev, false);
5916 cik_enable_mgcg(rdev, false);
5918 cik_enable_gui_idle_interrupt(rdev, true);
5921 if (block & RADEON_CG_BLOCK_MC) {
5922 if (!(rdev->flags & RADEON_IS_IGP)) {
5923 cik_enable_mc_mgcg(rdev, enable);
5924 cik_enable_mc_ls(rdev, enable);
5928 if (block & RADEON_CG_BLOCK_SDMA) {
5929 cik_enable_sdma_mgcg(rdev, enable);
5930 cik_enable_sdma_mgls(rdev, enable);
5933 if (block & RADEON_CG_BLOCK_BIF) {
5934 cik_enable_bif_mgls(rdev, enable);
5937 if (block & RADEON_CG_BLOCK_UVD) {
5939 cik_enable_uvd_mgcg(rdev, enable);
5942 if (block & RADEON_CG_BLOCK_HDP) {
5943 cik_enable_hdp_mgcg(rdev, enable);
5944 cik_enable_hdp_ls(rdev, enable);
5948 static void cik_init_cg(struct radeon_device *rdev)
5951 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
5954 si_init_uvd_internal_cg(rdev);
5956 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5957 RADEON_CG_BLOCK_SDMA |
5958 RADEON_CG_BLOCK_BIF |
5959 RADEON_CG_BLOCK_UVD |
5960 RADEON_CG_BLOCK_HDP), true);
5963 static void cik_fini_cg(struct radeon_device *rdev)
5965 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5966 RADEON_CG_BLOCK_SDMA |
5967 RADEON_CG_BLOCK_BIF |
5968 RADEON_CG_BLOCK_UVD |
5969 RADEON_CG_BLOCK_HDP), false);
5971 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
5974 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
5979 orig = data = RREG32(RLC_PG_CNTL);
5980 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
5981 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5983 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5985 WREG32(RLC_PG_CNTL, data);
5988 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
5993 orig = data = RREG32(RLC_PG_CNTL);
5994 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
5995 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5997 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5999 WREG32(RLC_PG_CNTL, data);
6002 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6006 orig = data = RREG32(RLC_PG_CNTL);
6007 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6008 data &= ~DISABLE_CP_PG;
6010 data |= DISABLE_CP_PG;
6012 WREG32(RLC_PG_CNTL, data);
6015 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6019 orig = data = RREG32(RLC_PG_CNTL);
6020 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6021 data &= ~DISABLE_GDS_PG;
6023 data |= DISABLE_GDS_PG;
6025 WREG32(RLC_PG_CNTL, data);
6028 #define CP_ME_TABLE_SIZE 96
6029 #define CP_ME_TABLE_OFFSET 2048
6030 #define CP_MEC_TABLE_OFFSET 4096
6032 void cik_init_cp_pg_table(struct radeon_device *rdev)
6034 const __be32 *fw_data;
6035 volatile u32 *dst_ptr;
6036 int me, i, max_me = 4;
6040 if (rdev->family == CHIP_KAVERI)
6043 if (rdev->rlc.cp_table_ptr == NULL)
6046 /* write the cp table buffer */
6047 dst_ptr = rdev->rlc.cp_table_ptr;
6048 for (me = 0; me < max_me; me++) {
6050 fw_data = (const __be32 *)rdev->ce_fw->data;
6051 table_offset = CP_ME_TABLE_OFFSET;
6052 } else if (me == 1) {
6053 fw_data = (const __be32 *)rdev->pfp_fw->data;
6054 table_offset = CP_ME_TABLE_OFFSET;
6055 } else if (me == 2) {
6056 fw_data = (const __be32 *)rdev->me_fw->data;
6057 table_offset = CP_ME_TABLE_OFFSET;
6059 fw_data = (const __be32 *)rdev->mec_fw->data;
6060 table_offset = CP_MEC_TABLE_OFFSET;
6063 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
6064 dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6066 bo_offset += CP_ME_TABLE_SIZE;
6070 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6075 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6076 orig = data = RREG32(RLC_PG_CNTL);
6077 data |= GFX_PG_ENABLE;
6079 WREG32(RLC_PG_CNTL, data);
6081 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6084 WREG32(RLC_AUTO_PG_CTRL, data);
6086 orig = data = RREG32(RLC_PG_CNTL);
6087 data &= ~GFX_PG_ENABLE;
6089 WREG32(RLC_PG_CNTL, data);
6091 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6092 data &= ~AUTO_PG_EN;
6094 WREG32(RLC_AUTO_PG_CTRL, data);
6096 data = RREG32(DB_RENDER_CONTROL);
6100 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6102 u32 mask = 0, tmp, tmp1;
6105 cik_select_se_sh(rdev, se, sh);
6106 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6107 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6108 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6115 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6120 return (~tmp) & mask;
6123 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6125 u32 i, j, k, active_cu_number = 0;
6126 u32 mask, counter, cu_bitmap;
6129 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6130 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6134 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6135 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6143 active_cu_number += counter;
6144 tmp |= (cu_bitmap << (i * 16 + j * 8));
6148 WREG32(RLC_PG_AO_CU_MASK, tmp);
6150 tmp = RREG32(RLC_MAX_PG_CU);
6151 tmp &= ~MAX_PU_CU_MASK;
6152 tmp |= MAX_PU_CU(active_cu_number);
6153 WREG32(RLC_MAX_PG_CU, tmp);
6156 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6161 orig = data = RREG32(RLC_PG_CNTL);
6162 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6163 data |= STATIC_PER_CU_PG_ENABLE;
6165 data &= ~STATIC_PER_CU_PG_ENABLE;
6167 WREG32(RLC_PG_CNTL, data);
6170 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6175 orig = data = RREG32(RLC_PG_CNTL);
6176 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6177 data |= DYN_PER_CU_PG_ENABLE;
6179 data &= ~DYN_PER_CU_PG_ENABLE;
6181 WREG32(RLC_PG_CNTL, data);
6184 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6185 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6187 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6192 if (rdev->rlc.cs_data) {
6193 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6194 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6195 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6196 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6198 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6199 for (i = 0; i < 3; i++)
6200 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6202 if (rdev->rlc.reg_list) {
6203 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6204 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6205 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6208 orig = data = RREG32(RLC_PG_CNTL);
6211 WREG32(RLC_PG_CNTL, data);
6213 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6214 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6216 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6217 data &= ~IDLE_POLL_COUNT_MASK;
6218 data |= IDLE_POLL_COUNT(0x60);
6219 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6222 WREG32(RLC_PG_DELAY, data);
6224 data = RREG32(RLC_PG_DELAY_2);
6227 WREG32(RLC_PG_DELAY_2, data);
6229 data = RREG32(RLC_AUTO_PG_CTRL);
6230 data &= ~GRBM_REG_SGIT_MASK;
6231 data |= GRBM_REG_SGIT(0x700);
6232 WREG32(RLC_AUTO_PG_CTRL, data);
6236 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6238 cik_enable_gfx_cgpg(rdev, enable);
6239 cik_enable_gfx_static_mgpg(rdev, enable);
6240 cik_enable_gfx_dynamic_mgpg(rdev, enable);
6243 u32 cik_get_csb_size(struct radeon_device *rdev)
6246 const struct cs_section_def *sect = NULL;
6247 const struct cs_extent_def *ext = NULL;
6249 if (rdev->rlc.cs_data == NULL)
6252 /* begin clear state */
6254 /* context control state */
6257 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6258 for (ext = sect->section; ext->extent != NULL; ++ext) {
6259 if (sect->id == SECT_CONTEXT)
6260 count += 2 + ext->reg_count;
6265 /* pa_sc_raster_config/pa_sc_raster_config1 */
6267 /* end clear state */
6275 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6278 const struct cs_section_def *sect = NULL;
6279 const struct cs_extent_def *ext = NULL;
6281 if (rdev->rlc.cs_data == NULL)
6286 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6287 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6289 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6290 buffer[count++] = cpu_to_le32(0x80000000);
6291 buffer[count++] = cpu_to_le32(0x80000000);
6293 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6294 for (ext = sect->section; ext->extent != NULL; ++ext) {
6295 if (sect->id == SECT_CONTEXT) {
6297 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6298 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6299 for (i = 0; i < ext->reg_count; i++)
6300 buffer[count++] = cpu_to_le32(ext->extent[i]);
6307 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6308 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
6309 switch (rdev->family) {
6311 buffer[count++] = cpu_to_le32(0x16000012);
6312 buffer[count++] = cpu_to_le32(0x00000000);
6315 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6316 buffer[count++] = cpu_to_le32(0x00000000);
6319 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6320 buffer[count++] = cpu_to_le32(0x00000000);
6323 buffer[count++] = 0x3a00161a;
6324 buffer[count++] = 0x0000002e;
6327 buffer[count++] = cpu_to_le32(0x00000000);
6328 buffer[count++] = cpu_to_le32(0x00000000);
6332 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6333 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
6335 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6336 buffer[count++] = cpu_to_le32(0);
6339 static void cik_init_pg(struct radeon_device *rdev)
6341 if (rdev->pg_flags) {
6342 cik_enable_sck_slowdown_on_pu(rdev, true);
6343 cik_enable_sck_slowdown_on_pd(rdev, true);
6344 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6345 cik_init_gfx_cgpg(rdev);
6346 cik_enable_cp_pg(rdev, true);
6347 cik_enable_gds_pg(rdev, true);
6349 cik_init_ao_cu_mask(rdev);
6350 cik_update_gfx_pg(rdev, true);
6354 static void cik_fini_pg(struct radeon_device *rdev)
6356 if (rdev->pg_flags) {
6357 cik_update_gfx_pg(rdev, false);
6358 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6359 cik_enable_cp_pg(rdev, false);
6360 cik_enable_gds_pg(rdev, false);
6367 * Starting with r6xx, interrupts are handled via a ring buffer.
6368 * Ring buffers are areas of GPU accessible memory that the GPU
6369 * writes interrupt vectors into and the host reads vectors out of.
6370 * There is a rptr (read pointer) that determines where the
6371 * host is currently reading, and a wptr (write pointer)
6372 * which determines where the GPU has written. When the
6373 * pointers are equal, the ring is idle. When the GPU
6374 * writes vectors to the ring buffer, it increments the
6375 * wptr. When there is an interrupt, the host then starts
6376 * fetching commands and processing them until the pointers are
6377 * equal again at which point it updates the rptr.
6381 * cik_enable_interrupts - Enable the interrupt ring buffer
6383 * @rdev: radeon_device pointer
6385 * Enable the interrupt ring buffer (CIK).
6387 static void cik_enable_interrupts(struct radeon_device *rdev)
6389 u32 ih_cntl = RREG32(IH_CNTL);
6390 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6392 ih_cntl |= ENABLE_INTR;
6393 ih_rb_cntl |= IH_RB_ENABLE;
6394 WREG32(IH_CNTL, ih_cntl);
6395 WREG32(IH_RB_CNTL, ih_rb_cntl);
6396 rdev->ih.enabled = true;
6400 * cik_disable_interrupts - Disable the interrupt ring buffer
6402 * @rdev: radeon_device pointer
6404 * Disable the interrupt ring buffer (CIK).
6406 static void cik_disable_interrupts(struct radeon_device *rdev)
6408 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6409 u32 ih_cntl = RREG32(IH_CNTL);
6411 ih_rb_cntl &= ~IH_RB_ENABLE;
6412 ih_cntl &= ~ENABLE_INTR;
6413 WREG32(IH_RB_CNTL, ih_rb_cntl);
6414 WREG32(IH_CNTL, ih_cntl);
6415 /* set rptr, wptr to 0 */
6416 WREG32(IH_RB_RPTR, 0);
6417 WREG32(IH_RB_WPTR, 0);
6418 rdev->ih.enabled = false;
6423 * cik_disable_interrupt_state - Disable all interrupt sources
6425 * @rdev: radeon_device pointer
6427 * Clear all interrupt enable bits used by the driver (CIK).
6429 static void cik_disable_interrupt_state(struct radeon_device *rdev)
6434 tmp = RREG32(CP_INT_CNTL_RING0) &
6435 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6436 WREG32(CP_INT_CNTL_RING0, tmp);
6438 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6439 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6440 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6441 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
6442 /* compute queues */
6443 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6444 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6445 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6446 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6447 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6448 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6449 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6450 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6452 WREG32(GRBM_INT_CNTL, 0);
6453 /* vline/vblank, etc. */
6454 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6455 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6456 if (rdev->num_crtc >= 4) {
6457 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6458 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6460 if (rdev->num_crtc >= 6) {
6461 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6462 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6466 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6468 /* digital hotplug */
6469 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6470 WREG32(DC_HPD1_INT_CONTROL, tmp);
6471 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6472 WREG32(DC_HPD2_INT_CONTROL, tmp);
6473 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6474 WREG32(DC_HPD3_INT_CONTROL, tmp);
6475 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6476 WREG32(DC_HPD4_INT_CONTROL, tmp);
6477 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6478 WREG32(DC_HPD5_INT_CONTROL, tmp);
6479 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6480 WREG32(DC_HPD6_INT_CONTROL, tmp);
6485 * cik_irq_init - init and enable the interrupt ring
6487 * @rdev: radeon_device pointer
6489 * Allocate a ring buffer for the interrupt controller,
6490 * enable the RLC, disable interrupts, enable the IH
6491 * ring buffer and enable it (CIK).
6492 * Called at device load and reume.
6493 * Returns 0 for success, errors for failure.
6495 static int cik_irq_init(struct radeon_device *rdev)
6499 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6502 ret = r600_ih_ring_alloc(rdev);
6507 cik_disable_interrupts(rdev);
6510 ret = cik_rlc_resume(rdev);
6512 r600_ih_ring_fini(rdev);
6516 /* setup interrupt control */
6517 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6518 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6519 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6520 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6521 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6523 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6524 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6525 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6526 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6528 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
6529 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
6531 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6532 IH_WPTR_OVERFLOW_CLEAR |
6535 if (rdev->wb.enabled)
6536 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6538 /* set the writeback address whether it's enabled or not */
6539 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6540 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6542 WREG32(IH_RB_CNTL, ih_rb_cntl);
6544 /* set rptr, wptr to 0 */
6545 WREG32(IH_RB_RPTR, 0);
6546 WREG32(IH_RB_WPTR, 0);
6548 /* Default settings for IH_CNTL (disabled at first) */
6549 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6550 /* RPTR_REARM only works if msi's are enabled */
6551 if (rdev->msi_enabled)
6552 ih_cntl |= RPTR_REARM;
6553 WREG32(IH_CNTL, ih_cntl);
6555 /* force the active interrupt state to all disabled */
6556 cik_disable_interrupt_state(rdev);
6558 pci_set_master(rdev->pdev);
6561 cik_enable_interrupts(rdev);
6567 * cik_irq_set - enable/disable interrupt sources
6569 * @rdev: radeon_device pointer
6571 * Enable interrupt sources on the GPU (vblanks, hpd,
6573 * Returns 0 for success, errors for failure.
6575 int cik_irq_set(struct radeon_device *rdev)
6578 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6579 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
6580 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6581 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6582 u32 grbm_int_cntl = 0;
6583 u32 dma_cntl, dma_cntl1;
6586 if (!rdev->irq.installed) {
6587 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6590 /* don't enable anything if the ih is disabled */
6591 if (!rdev->ih.enabled) {
6592 cik_disable_interrupts(rdev);
6593 /* force the active interrupt state to all disabled */
6594 cik_disable_interrupt_state(rdev);
6598 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6599 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6600 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6602 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6603 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6604 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6605 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6606 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6607 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6609 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6610 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6612 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6613 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6614 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6615 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6616 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6617 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6618 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6619 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6621 if (rdev->flags & RADEON_IS_IGP)
6622 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6623 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6625 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6626 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6628 /* enable CP interrupts on all rings */
6629 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6630 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6631 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6633 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6634 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6635 DRM_DEBUG("si_irq_set: sw int cp1\n");
6636 if (ring->me == 1) {
6637 switch (ring->pipe) {
6639 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6642 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6645 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6648 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6651 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6654 } else if (ring->me == 2) {
6655 switch (ring->pipe) {
6657 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6660 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6663 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6666 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6669 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6673 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6676 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6677 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6678 DRM_DEBUG("si_irq_set: sw int cp2\n");
6679 if (ring->me == 1) {
6680 switch (ring->pipe) {
6682 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6685 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6688 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6691 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6694 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6697 } else if (ring->me == 2) {
6698 switch (ring->pipe) {
6700 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6703 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6706 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6709 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6712 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6716 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6720 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6721 DRM_DEBUG("cik_irq_set: sw int dma\n");
6722 dma_cntl |= TRAP_ENABLE;
6725 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6726 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6727 dma_cntl1 |= TRAP_ENABLE;
6730 if (rdev->irq.crtc_vblank_int[0] ||
6731 atomic_read(&rdev->irq.pflip[0])) {
6732 DRM_DEBUG("cik_irq_set: vblank 0\n");
6733 crtc1 |= VBLANK_INTERRUPT_MASK;
6735 if (rdev->irq.crtc_vblank_int[1] ||
6736 atomic_read(&rdev->irq.pflip[1])) {
6737 DRM_DEBUG("cik_irq_set: vblank 1\n");
6738 crtc2 |= VBLANK_INTERRUPT_MASK;
6740 if (rdev->irq.crtc_vblank_int[2] ||
6741 atomic_read(&rdev->irq.pflip[2])) {
6742 DRM_DEBUG("cik_irq_set: vblank 2\n");
6743 crtc3 |= VBLANK_INTERRUPT_MASK;
6745 if (rdev->irq.crtc_vblank_int[3] ||
6746 atomic_read(&rdev->irq.pflip[3])) {
6747 DRM_DEBUG("cik_irq_set: vblank 3\n");
6748 crtc4 |= VBLANK_INTERRUPT_MASK;
6750 if (rdev->irq.crtc_vblank_int[4] ||
6751 atomic_read(&rdev->irq.pflip[4])) {
6752 DRM_DEBUG("cik_irq_set: vblank 4\n");
6753 crtc5 |= VBLANK_INTERRUPT_MASK;
6755 if (rdev->irq.crtc_vblank_int[5] ||
6756 atomic_read(&rdev->irq.pflip[5])) {
6757 DRM_DEBUG("cik_irq_set: vblank 5\n");
6758 crtc6 |= VBLANK_INTERRUPT_MASK;
6760 if (rdev->irq.hpd[0]) {
6761 DRM_DEBUG("cik_irq_set: hpd 1\n");
6762 hpd1 |= DC_HPDx_INT_EN;
6764 if (rdev->irq.hpd[1]) {
6765 DRM_DEBUG("cik_irq_set: hpd 2\n");
6766 hpd2 |= DC_HPDx_INT_EN;
6768 if (rdev->irq.hpd[2]) {
6769 DRM_DEBUG("cik_irq_set: hpd 3\n");
6770 hpd3 |= DC_HPDx_INT_EN;
6772 if (rdev->irq.hpd[3]) {
6773 DRM_DEBUG("cik_irq_set: hpd 4\n");
6774 hpd4 |= DC_HPDx_INT_EN;
6776 if (rdev->irq.hpd[4]) {
6777 DRM_DEBUG("cik_irq_set: hpd 5\n");
6778 hpd5 |= DC_HPDx_INT_EN;
6780 if (rdev->irq.hpd[5]) {
6781 DRM_DEBUG("cik_irq_set: hpd 6\n");
6782 hpd6 |= DC_HPDx_INT_EN;
6785 if (rdev->irq.dpm_thermal) {
6786 DRM_DEBUG("dpm thermal\n");
6787 if (rdev->flags & RADEON_IS_IGP)
6788 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
6790 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
6793 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6795 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
6796 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
6798 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
6799 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
6800 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
6801 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
6802 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
6803 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
6804 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
6805 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
6807 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6809 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6810 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6811 if (rdev->num_crtc >= 4) {
6812 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6813 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6815 if (rdev->num_crtc >= 6) {
6816 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6817 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6820 WREG32(DC_HPD1_INT_CONTROL, hpd1);
6821 WREG32(DC_HPD2_INT_CONTROL, hpd2);
6822 WREG32(DC_HPD3_INT_CONTROL, hpd3);
6823 WREG32(DC_HPD4_INT_CONTROL, hpd4);
6824 WREG32(DC_HPD5_INT_CONTROL, hpd5);
6825 WREG32(DC_HPD6_INT_CONTROL, hpd6);
6827 if (rdev->flags & RADEON_IS_IGP)
6828 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
6830 WREG32_SMC(CG_THERMAL_INT, thermal_int);
6836 * cik_irq_ack - ack interrupt sources
6838 * @rdev: radeon_device pointer
6840 * Ack interrupt sources on the GPU (vblanks, hpd,
6841 * etc.) (CIK). Certain interrupts sources are sw
6842 * generated and do not require an explicit ack.
6844 static inline void cik_irq_ack(struct radeon_device *rdev)
6848 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6849 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6850 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6851 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6852 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6853 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6854 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
6856 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
6857 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6858 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
6859 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6860 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6861 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6862 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6863 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6865 if (rdev->num_crtc >= 4) {
6866 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6867 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6868 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6869 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6870 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6871 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6872 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6873 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6876 if (rdev->num_crtc >= 6) {
6877 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6878 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6879 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6880 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6881 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6882 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6883 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6884 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6887 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
6888 tmp = RREG32(DC_HPD1_INT_CONTROL);
6889 tmp |= DC_HPDx_INT_ACK;
6890 WREG32(DC_HPD1_INT_CONTROL, tmp);
6892 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
6893 tmp = RREG32(DC_HPD2_INT_CONTROL);
6894 tmp |= DC_HPDx_INT_ACK;
6895 WREG32(DC_HPD2_INT_CONTROL, tmp);
6897 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6898 tmp = RREG32(DC_HPD3_INT_CONTROL);
6899 tmp |= DC_HPDx_INT_ACK;
6900 WREG32(DC_HPD3_INT_CONTROL, tmp);
6902 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6903 tmp = RREG32(DC_HPD4_INT_CONTROL);
6904 tmp |= DC_HPDx_INT_ACK;
6905 WREG32(DC_HPD4_INT_CONTROL, tmp);
6907 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6908 tmp = RREG32(DC_HPD5_INT_CONTROL);
6909 tmp |= DC_HPDx_INT_ACK;
6910 WREG32(DC_HPD5_INT_CONTROL, tmp);
6912 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6913 tmp = RREG32(DC_HPD5_INT_CONTROL);
6914 tmp |= DC_HPDx_INT_ACK;
6915 WREG32(DC_HPD6_INT_CONTROL, tmp);
6920 * cik_irq_disable - disable interrupts
6922 * @rdev: radeon_device pointer
6924 * Disable interrupts on the hw (CIK).
6926 static void cik_irq_disable(struct radeon_device *rdev)
6928 cik_disable_interrupts(rdev);
6929 /* Wait and acknowledge irq */
6932 cik_disable_interrupt_state(rdev);
6936 * cik_irq_disable - disable interrupts for suspend
6938 * @rdev: radeon_device pointer
6940 * Disable interrupts and stop the RLC (CIK).
6943 static void cik_irq_suspend(struct radeon_device *rdev)
6945 cik_irq_disable(rdev);
6950 * cik_irq_fini - tear down interrupt support
6952 * @rdev: radeon_device pointer
6954 * Disable interrupts on the hw and free the IH ring
6956 * Used for driver unload.
6958 static void cik_irq_fini(struct radeon_device *rdev)
6960 cik_irq_suspend(rdev);
6961 r600_ih_ring_fini(rdev);
6965 * cik_get_ih_wptr - get the IH ring buffer wptr
6967 * @rdev: radeon_device pointer
6969 * Get the IH ring buffer wptr from either the register
6970 * or the writeback memory buffer (CIK). Also check for
6971 * ring buffer overflow and deal with it.
6972 * Used by cik_irq_process().
6973 * Returns the value of the wptr.
6975 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
6979 if (rdev->wb.enabled)
6980 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6982 wptr = RREG32(IH_RB_WPTR);
6984 if (wptr & RB_OVERFLOW) {
6985 /* When a ring buffer overflow happen start parsing interrupt
6986 * from the last not overwritten vector (wptr + 16). Hopefully
6987 * this should allow us to catchup.
6989 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6990 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6991 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6992 tmp = RREG32(IH_RB_CNTL);
6993 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6994 WREG32(IH_RB_CNTL, tmp);
6996 return (wptr & rdev->ih.ptr_mask);
7000 * Each IV ring entry is 128 bits:
7001 * [7:0] - interrupt source id
7003 * [59:32] - interrupt source data
7004 * [63:60] - reserved
7007 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7008 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7009 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7010 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7011 * PIPE_ID - ME0 0=3D
7012 * - ME1&2 compute dispatcher (4 pipes each)
7014 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7015 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7016 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7019 * [127:96] - reserved
7022 * cik_irq_process - interrupt handler
7024 * @rdev: radeon_device pointer
7026 * Interrupt hander (CIK). Walk the IH ring,
7027 * ack interrupts and schedule work to handle
7029 * Returns irq process return code.
7031 int cik_irq_process(struct radeon_device *rdev)
7033 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7034 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7037 u32 src_id, src_data, ring_id;
7038 u8 me_id, pipe_id, queue_id;
7040 bool queue_hotplug = false;
7041 bool queue_reset = false;
7042 u32 addr, status, mc_client;
7043 bool queue_thermal = false;
7045 if (!rdev->ih.enabled || rdev->shutdown)
7048 wptr = cik_get_ih_wptr(rdev);
7051 /* is somebody else already processing irqs? */
7052 if (atomic_xchg(&rdev->ih.lock, 1))
7055 rptr = rdev->ih.rptr;
7056 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7058 /* Order reading of wptr vs. reading of IH ring data */
7061 /* display interrupts */
7064 while (rptr != wptr) {
7065 /* wptr/rptr are in bytes! */
7066 ring_index = rptr / 4;
7067 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7068 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7069 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7072 case 1: /* D1 vblank/vline */
7074 case 0: /* D1 vblank */
7075 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7076 if (rdev->irq.crtc_vblank_int[0]) {
7077 drm_handle_vblank(rdev->ddev, 0);
7078 rdev->pm.vblank_sync = true;
7079 wake_up(&rdev->irq.vblank_queue);
7081 if (atomic_read(&rdev->irq.pflip[0]))
7082 radeon_crtc_handle_flip(rdev, 0);
7083 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7084 DRM_DEBUG("IH: D1 vblank\n");
7087 case 1: /* D1 vline */
7088 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7089 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7090 DRM_DEBUG("IH: D1 vline\n");
7094 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7098 case 2: /* D2 vblank/vline */
7100 case 0: /* D2 vblank */
7101 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7102 if (rdev->irq.crtc_vblank_int[1]) {
7103 drm_handle_vblank(rdev->ddev, 1);
7104 rdev->pm.vblank_sync = true;
7105 wake_up(&rdev->irq.vblank_queue);
7107 if (atomic_read(&rdev->irq.pflip[1]))
7108 radeon_crtc_handle_flip(rdev, 1);
7109 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7110 DRM_DEBUG("IH: D2 vblank\n");
7113 case 1: /* D2 vline */
7114 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7115 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7116 DRM_DEBUG("IH: D2 vline\n");
7120 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7124 case 3: /* D3 vblank/vline */
7126 case 0: /* D3 vblank */
7127 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7128 if (rdev->irq.crtc_vblank_int[2]) {
7129 drm_handle_vblank(rdev->ddev, 2);
7130 rdev->pm.vblank_sync = true;
7131 wake_up(&rdev->irq.vblank_queue);
7133 if (atomic_read(&rdev->irq.pflip[2]))
7134 radeon_crtc_handle_flip(rdev, 2);
7135 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7136 DRM_DEBUG("IH: D3 vblank\n");
7139 case 1: /* D3 vline */
7140 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7141 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7142 DRM_DEBUG("IH: D3 vline\n");
7146 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7150 case 4: /* D4 vblank/vline */
7152 case 0: /* D4 vblank */
7153 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7154 if (rdev->irq.crtc_vblank_int[3]) {
7155 drm_handle_vblank(rdev->ddev, 3);
7156 rdev->pm.vblank_sync = true;
7157 wake_up(&rdev->irq.vblank_queue);
7159 if (atomic_read(&rdev->irq.pflip[3]))
7160 radeon_crtc_handle_flip(rdev, 3);
7161 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7162 DRM_DEBUG("IH: D4 vblank\n");
7165 case 1: /* D4 vline */
7166 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7167 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7168 DRM_DEBUG("IH: D4 vline\n");
7172 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7176 case 5: /* D5 vblank/vline */
7178 case 0: /* D5 vblank */
7179 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7180 if (rdev->irq.crtc_vblank_int[4]) {
7181 drm_handle_vblank(rdev->ddev, 4);
7182 rdev->pm.vblank_sync = true;
7183 wake_up(&rdev->irq.vblank_queue);
7185 if (atomic_read(&rdev->irq.pflip[4]))
7186 radeon_crtc_handle_flip(rdev, 4);
7187 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7188 DRM_DEBUG("IH: D5 vblank\n");
7191 case 1: /* D5 vline */
7192 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7193 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7194 DRM_DEBUG("IH: D5 vline\n");
7198 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7202 case 6: /* D6 vblank/vline */
7204 case 0: /* D6 vblank */
7205 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7206 if (rdev->irq.crtc_vblank_int[5]) {
7207 drm_handle_vblank(rdev->ddev, 5);
7208 rdev->pm.vblank_sync = true;
7209 wake_up(&rdev->irq.vblank_queue);
7211 if (atomic_read(&rdev->irq.pflip[5]))
7212 radeon_crtc_handle_flip(rdev, 5);
7213 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7214 DRM_DEBUG("IH: D6 vblank\n");
7217 case 1: /* D6 vline */
7218 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7219 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7220 DRM_DEBUG("IH: D6 vline\n");
7224 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7228 case 42: /* HPD hotplug */
7231 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7232 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7233 queue_hotplug = true;
7234 DRM_DEBUG("IH: HPD1\n");
7238 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7239 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7240 queue_hotplug = true;
7241 DRM_DEBUG("IH: HPD2\n");
7245 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7246 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7247 queue_hotplug = true;
7248 DRM_DEBUG("IH: HPD3\n");
7252 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7253 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7254 queue_hotplug = true;
7255 DRM_DEBUG("IH: HPD4\n");
7259 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7260 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7261 queue_hotplug = true;
7262 DRM_DEBUG("IH: HPD5\n");
7266 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7267 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7268 queue_hotplug = true;
7269 DRM_DEBUG("IH: HPD6\n");
7273 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7278 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7279 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7283 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7284 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7285 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
7286 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7287 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
7289 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
7291 cik_vm_decode_fault(rdev, status, addr, mc_client);
7292 /* reset addr and status */
7293 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7295 case 176: /* GFX RB CP_INT */
7296 case 177: /* GFX IB CP_INT */
7297 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7299 case 181: /* CP EOP event */
7300 DRM_DEBUG("IH: CP EOP\n");
7301 /* XXX check the bitfield order! */
7302 me_id = (ring_id & 0x60) >> 5;
7303 pipe_id = (ring_id & 0x18) >> 3;
7304 queue_id = (ring_id & 0x7) >> 0;
7307 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7311 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7312 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7313 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7314 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7318 case 184: /* CP Privileged reg access */
7319 DRM_ERROR("Illegal register access in command stream\n");
7320 /* XXX check the bitfield order! */
7321 me_id = (ring_id & 0x60) >> 5;
7322 pipe_id = (ring_id & 0x18) >> 3;
7323 queue_id = (ring_id & 0x7) >> 0;
7326 /* This results in a full GPU reset, but all we need to do is soft
7327 * reset the CP for gfx
7341 case 185: /* CP Privileged inst */
7342 DRM_ERROR("Illegal instruction in command stream\n");
7343 /* XXX check the bitfield order! */
7344 me_id = (ring_id & 0x60) >> 5;
7345 pipe_id = (ring_id & 0x18) >> 3;
7346 queue_id = (ring_id & 0x7) >> 0;
7349 /* This results in a full GPU reset, but all we need to do is soft
7350 * reset the CP for gfx
7364 case 224: /* SDMA trap event */
7365 /* XXX check the bitfield order! */
7366 me_id = (ring_id & 0x3) >> 0;
7367 queue_id = (ring_id & 0xc) >> 2;
7368 DRM_DEBUG("IH: SDMA trap\n");
7373 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7386 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7398 case 230: /* thermal low to high */
7399 DRM_DEBUG("IH: thermal low to high\n");
7400 rdev->pm.dpm.thermal.high_to_low = false;
7401 queue_thermal = true;
7403 case 231: /* thermal high to low */
7404 DRM_DEBUG("IH: thermal high to low\n");
7405 rdev->pm.dpm.thermal.high_to_low = true;
7406 queue_thermal = true;
7408 case 233: /* GUI IDLE */
7409 DRM_DEBUG("IH: GUI idle\n");
7411 case 241: /* SDMA Privileged inst */
7412 case 247: /* SDMA Privileged inst */
7413 DRM_ERROR("Illegal instruction in SDMA command stream\n");
7414 /* XXX check the bitfield order! */
7415 me_id = (ring_id & 0x3) >> 0;
7416 queue_id = (ring_id & 0xc) >> 2;
7451 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7455 /* wptr/rptr are in bytes! */
7457 rptr &= rdev->ih.ptr_mask;
7460 schedule_work(&rdev->hotplug_work);
7462 schedule_work(&rdev->reset_work);
7464 schedule_work(&rdev->pm.dpm.thermal.work);
7465 rdev->ih.rptr = rptr;
7466 WREG32(IH_RB_RPTR, rdev->ih.rptr);
7467 atomic_set(&rdev->ih.lock, 0);
7469 /* make sure wptr hasn't changed while processing */
7470 wptr = cik_get_ih_wptr(rdev);
7478 * startup/shutdown callbacks
7481 * cik_startup - program the asic to a functional state
7483 * @rdev: radeon_device pointer
7485 * Programs the asic to a functional state (CIK).
7486 * Called by cik_init() and cik_resume().
7487 * Returns 0 for success, error for failure.
7489 static int cik_startup(struct radeon_device *rdev)
7491 struct radeon_ring *ring;
7494 /* enable pcie gen2/3 link */
7495 cik_pcie_gen3_enable(rdev);
7497 cik_program_aspm(rdev);
7499 /* scratch needs to be initialized before MC */
7500 r = r600_vram_scratch_init(rdev);
7504 cik_mc_program(rdev);
7506 if (rdev->flags & RADEON_IS_IGP) {
7507 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7508 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
7509 r = cik_init_microcode(rdev);
7511 DRM_ERROR("Failed to load firmware!\n");
7516 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7517 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
7519 r = cik_init_microcode(rdev);
7521 DRM_ERROR("Failed to load firmware!\n");
7526 r = ci_mc_load_microcode(rdev);
7528 DRM_ERROR("Failed to load MC firmware!\n");
7533 r = cik_pcie_gart_enable(rdev);
7538 /* allocate rlc buffers */
7539 if (rdev->flags & RADEON_IS_IGP) {
7540 if (rdev->family == CHIP_KAVERI) {
7541 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7542 rdev->rlc.reg_list_size =
7543 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7545 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7546 rdev->rlc.reg_list_size =
7547 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7550 rdev->rlc.cs_data = ci_cs_data;
7551 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
7552 r = sumo_rlc_init(rdev);
7554 DRM_ERROR("Failed to init rlc BOs!\n");
7558 /* allocate wb buffer */
7559 r = radeon_wb_init(rdev);
7563 /* allocate mec buffers */
7564 r = cik_mec_init(rdev);
7566 DRM_ERROR("Failed to init MEC BOs!\n");
7570 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7572 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7576 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7578 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7582 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7584 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7588 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7590 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7594 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7596 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7600 r = radeon_uvd_resume(rdev);
7602 r = uvd_v4_2_resume(rdev);
7604 r = radeon_fence_driver_start_ring(rdev,
7605 R600_RING_TYPE_UVD_INDEX);
7607 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7611 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7614 if (!rdev->irq.installed) {
7615 r = radeon_irq_kms_init(rdev);
7620 r = cik_irq_init(rdev);
7622 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7623 radeon_irq_kms_fini(rdev);
7628 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7629 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7630 CP_RB0_RPTR, CP_RB0_WPTR,
7631 PACKET3(PACKET3_NOP, 0x3FFF));
7635 /* set up the compute queues */
7636 /* type-2 packets are deprecated on MEC, use type-3 instead */
7637 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7638 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7639 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
7640 PACKET3(PACKET3_NOP, 0x3FFF));
7643 ring->me = 1; /* first MEC */
7644 ring->pipe = 0; /* first pipe */
7645 ring->queue = 0; /* first queue */
7646 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7648 /* type-2 packets are deprecated on MEC, use type-3 instead */
7649 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7650 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7651 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
7652 PACKET3(PACKET3_NOP, 0x3FFF));
7655 /* dGPU only have 1 MEC */
7656 ring->me = 1; /* first MEC */
7657 ring->pipe = 0; /* first pipe */
7658 ring->queue = 1; /* second queue */
7659 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7661 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7662 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7663 SDMA0_GFX_RB_RPTR + SDMA0_REGISTER_OFFSET,
7664 SDMA0_GFX_RB_WPTR + SDMA0_REGISTER_OFFSET,
7665 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7669 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7670 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7671 SDMA0_GFX_RB_RPTR + SDMA1_REGISTER_OFFSET,
7672 SDMA0_GFX_RB_WPTR + SDMA1_REGISTER_OFFSET,
7673 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7677 r = cik_cp_resume(rdev);
7681 r = cik_sdma_resume(rdev);
7685 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7686 if (ring->ring_size) {
7687 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7688 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
7691 r = uvd_v1_0_init(rdev);
7693 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7696 r = radeon_ib_pool_init(rdev);
7698 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7702 r = radeon_vm_manager_init(rdev);
7704 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7708 r = dce6_audio_init(rdev);
7716 * cik_resume - resume the asic to a functional state
7718 * @rdev: radeon_device pointer
7720 * Programs the asic to a functional state (CIK).
7722 * Returns 0 for success, error for failure.
7724 int cik_resume(struct radeon_device *rdev)
7729 atom_asic_init(rdev->mode_info.atom_context);
7731 /* init golden registers */
7732 cik_init_golden_registers(rdev);
7734 rdev->accel_working = true;
7735 r = cik_startup(rdev);
7737 DRM_ERROR("cik startup failed on resume\n");
7738 rdev->accel_working = false;
7747 * cik_suspend - suspend the asic
7749 * @rdev: radeon_device pointer
7751 * Bring the chip into a state suitable for suspend (CIK).
7752 * Called at suspend.
7753 * Returns 0 for success.
7755 int cik_suspend(struct radeon_device *rdev)
7757 dce6_audio_fini(rdev);
7758 radeon_vm_manager_fini(rdev);
7759 cik_cp_enable(rdev, false);
7760 cik_sdma_enable(rdev, false);
7761 uvd_v1_0_fini(rdev);
7762 radeon_uvd_suspend(rdev);
7765 cik_irq_suspend(rdev);
7766 radeon_wb_disable(rdev);
7767 cik_pcie_gart_disable(rdev);
7771 /* Plan is to move initialization in that function and use
7772 * helper function so that radeon_device_init pretty much
7773 * do nothing more than calling asic specific function. This
7774 * should also allow to remove a bunch of callback function
7778 * cik_init - asic specific driver and hw init
7780 * @rdev: radeon_device pointer
7782 * Setup asic specific driver variables and program the hw
7783 * to a functional state (CIK).
7784 * Called at driver startup.
7785 * Returns 0 for success, errors for failure.
7787 int cik_init(struct radeon_device *rdev)
7789 struct radeon_ring *ring;
7793 if (!radeon_get_bios(rdev)) {
7794 if (ASIC_IS_AVIVO(rdev))
7797 /* Must be an ATOMBIOS */
7798 if (!rdev->is_atom_bios) {
7799 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7802 r = radeon_atombios_init(rdev);
7806 /* Post card if necessary */
7807 if (!radeon_card_posted(rdev)) {
7809 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7812 DRM_INFO("GPU not posted. posting now...\n");
7813 atom_asic_init(rdev->mode_info.atom_context);
7815 /* init golden registers */
7816 cik_init_golden_registers(rdev);
7817 /* Initialize scratch registers */
7818 cik_scratch_init(rdev);
7819 /* Initialize surface registers */
7820 radeon_surface_init(rdev);
7821 /* Initialize clocks */
7822 radeon_get_clock_info(rdev->ddev);
7825 r = radeon_fence_driver_init(rdev);
7829 /* initialize memory controller */
7830 r = cik_mc_init(rdev);
7833 /* Memory manager */
7834 r = radeon_bo_init(rdev);
7838 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7839 ring->ring_obj = NULL;
7840 r600_ring_init(rdev, ring, 1024 * 1024);
7842 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7843 ring->ring_obj = NULL;
7844 r600_ring_init(rdev, ring, 1024 * 1024);
7845 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
7849 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7850 ring->ring_obj = NULL;
7851 r600_ring_init(rdev, ring, 1024 * 1024);
7852 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
7856 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7857 ring->ring_obj = NULL;
7858 r600_ring_init(rdev, ring, 256 * 1024);
7860 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7861 ring->ring_obj = NULL;
7862 r600_ring_init(rdev, ring, 256 * 1024);
7864 r = radeon_uvd_init(rdev);
7866 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7867 ring->ring_obj = NULL;
7868 r600_ring_init(rdev, ring, 4096);
7871 rdev->ih.ring_obj = NULL;
7872 r600_ih_ring_init(rdev, 64 * 1024);
7874 r = r600_pcie_gart_init(rdev);
7878 rdev->accel_working = true;
7879 r = cik_startup(rdev);
7881 dev_err(rdev->dev, "disabling GPU acceleration\n");
7883 cik_sdma_fini(rdev);
7885 sumo_rlc_fini(rdev);
7887 radeon_wb_fini(rdev);
7888 radeon_ib_pool_fini(rdev);
7889 radeon_vm_manager_fini(rdev);
7890 radeon_irq_kms_fini(rdev);
7891 cik_pcie_gart_fini(rdev);
7892 rdev->accel_working = false;
7895 /* Don't start up if the MC ucode is missing.
7896 * The default clocks and voltages before the MC ucode
7897 * is loaded are not suffient for advanced operations.
7899 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
7900 DRM_ERROR("radeon: MC ucode required for NI+.\n");
7908 * cik_fini - asic specific driver and hw fini
7910 * @rdev: radeon_device pointer
7912 * Tear down the asic specific driver variables and program the hw
7913 * to an idle state (CIK).
7914 * Called at driver unload.
7916 void cik_fini(struct radeon_device *rdev)
7919 cik_sdma_fini(rdev);
7923 sumo_rlc_fini(rdev);
7925 radeon_wb_fini(rdev);
7926 radeon_vm_manager_fini(rdev);
7927 radeon_ib_pool_fini(rdev);
7928 radeon_irq_kms_fini(rdev);
7929 uvd_v1_0_fini(rdev);
7930 radeon_uvd_fini(rdev);
7931 cik_pcie_gart_fini(rdev);
7932 r600_vram_scratch_fini(rdev);
7933 radeon_gem_fini(rdev);
7934 radeon_fence_driver_fini(rdev);
7935 radeon_bo_fini(rdev);
7936 radeon_atombios_fini(rdev);
7941 void dce8_program_fmt(struct drm_encoder *encoder)
7943 struct drm_device *dev = encoder->dev;
7944 struct radeon_device *rdev = dev->dev_private;
7945 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
7946 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
7947 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
7950 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
7953 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
7954 bpc = radeon_get_monitor_bpc(connector);
7955 dither = radeon_connector->dither;
7958 /* LVDS/eDP FMT is set up by atom */
7959 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
7962 /* not needed for analog */
7963 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
7964 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
7972 if (dither == RADEON_FMT_DITHER_ENABLE)
7973 /* XXX sort out optimal dither settings */
7974 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7975 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
7977 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
7980 if (dither == RADEON_FMT_DITHER_ENABLE)
7981 /* XXX sort out optimal dither settings */
7982 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7983 FMT_RGB_RANDOM_ENABLE |
7984 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
7986 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
7989 if (dither == RADEON_FMT_DITHER_ENABLE)
7990 /* XXX sort out optimal dither settings */
7991 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
7992 FMT_RGB_RANDOM_ENABLE |
7993 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
7995 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8002 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8005 /* display watermark setup */
8007 * dce8_line_buffer_adjust - Set up the line buffer
8009 * @rdev: radeon_device pointer
8010 * @radeon_crtc: the selected display controller
8011 * @mode: the current display mode on the selected display
8014 * Setup up the line buffer allocation for
8015 * the selected display controller (CIK).
8016 * Returns the line buffer size in pixels.
8018 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8019 struct radeon_crtc *radeon_crtc,
8020 struct drm_display_mode *mode)
8022 u32 tmp, buffer_alloc, i;
8023 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8026 * There are 6 line buffers, one for each display controllers.
8027 * There are 3 partitions per LB. Select the number of partitions
8028 * to enable based on the display width. For display widths larger
8029 * than 4096, you need use to use 2 display controllers and combine
8030 * them using the stereo blender.
8032 if (radeon_crtc->base.enabled && mode) {
8033 if (mode->crtc_hdisplay < 1920) {
8036 } else if (mode->crtc_hdisplay < 2560) {
8039 } else if (mode->crtc_hdisplay < 4096) {
8041 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8043 DRM_DEBUG_KMS("Mode too big for LB!\n");
8045 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8052 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8053 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8055 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8056 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8057 for (i = 0; i < rdev->usec_timeout; i++) {
8058 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8059 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8064 if (radeon_crtc->base.enabled && mode) {
8076 /* controller not enabled, so no lb used */
8081 * cik_get_number_of_dram_channels - get the number of dram channels
8083 * @rdev: radeon_device pointer
8085 * Look up the number of video ram channels (CIK).
8086 * Used for display watermark bandwidth calculations
8087 * Returns the number of dram channels
8089 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8091 u32 tmp = RREG32(MC_SHARED_CHMAP);
8093 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8116 struct dce8_wm_params {
8117 u32 dram_channels; /* number of dram channels */
8118 u32 yclk; /* bandwidth per dram data pin in kHz */
8119 u32 sclk; /* engine clock in kHz */
8120 u32 disp_clk; /* display clock in kHz */
8121 u32 src_width; /* viewport width */
8122 u32 active_time; /* active display time in ns */
8123 u32 blank_time; /* blank time in ns */
8124 bool interlaced; /* mode is interlaced */
8125 fixed20_12 vsc; /* vertical scale ratio */
8126 u32 num_heads; /* number of active crtcs */
8127 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8128 u32 lb_size; /* line buffer allocated to pipe */
8129 u32 vtaps; /* vertical scaler taps */
8133 * dce8_dram_bandwidth - get the dram bandwidth
8135 * @wm: watermark calculation data
8137 * Calculate the raw dram bandwidth (CIK).
8138 * Used for display watermark bandwidth calculations
8139 * Returns the dram bandwidth in MBytes/s
8141 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8143 /* Calculate raw DRAM Bandwidth */
8144 fixed20_12 dram_efficiency; /* 0.7 */
8145 fixed20_12 yclk, dram_channels, bandwidth;
8148 a.full = dfixed_const(1000);
8149 yclk.full = dfixed_const(wm->yclk);
8150 yclk.full = dfixed_div(yclk, a);
8151 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8152 a.full = dfixed_const(10);
8153 dram_efficiency.full = dfixed_const(7);
8154 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8155 bandwidth.full = dfixed_mul(dram_channels, yclk);
8156 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8158 return dfixed_trunc(bandwidth);
8162 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8164 * @wm: watermark calculation data
8166 * Calculate the dram bandwidth used for display (CIK).
8167 * Used for display watermark bandwidth calculations
8168 * Returns the dram bandwidth for display in MBytes/s
8170 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8172 /* Calculate DRAM Bandwidth and the part allocated to display. */
8173 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8174 fixed20_12 yclk, dram_channels, bandwidth;
8177 a.full = dfixed_const(1000);
8178 yclk.full = dfixed_const(wm->yclk);
8179 yclk.full = dfixed_div(yclk, a);
8180 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8181 a.full = dfixed_const(10);
8182 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8183 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8184 bandwidth.full = dfixed_mul(dram_channels, yclk);
8185 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8187 return dfixed_trunc(bandwidth);
8191 * dce8_data_return_bandwidth - get the data return bandwidth
8193 * @wm: watermark calculation data
8195 * Calculate the data return bandwidth used for display (CIK).
8196 * Used for display watermark bandwidth calculations
8197 * Returns the data return bandwidth in MBytes/s
8199 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8201 /* Calculate the display Data return Bandwidth */
8202 fixed20_12 return_efficiency; /* 0.8 */
8203 fixed20_12 sclk, bandwidth;
8206 a.full = dfixed_const(1000);
8207 sclk.full = dfixed_const(wm->sclk);
8208 sclk.full = dfixed_div(sclk, a);
8209 a.full = dfixed_const(10);
8210 return_efficiency.full = dfixed_const(8);
8211 return_efficiency.full = dfixed_div(return_efficiency, a);
8212 a.full = dfixed_const(32);
8213 bandwidth.full = dfixed_mul(a, sclk);
8214 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8216 return dfixed_trunc(bandwidth);
8220 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8222 * @wm: watermark calculation data
8224 * Calculate the dmif bandwidth used for display (CIK).
8225 * Used for display watermark bandwidth calculations
8226 * Returns the dmif bandwidth in MBytes/s
8228 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8230 /* Calculate the DMIF Request Bandwidth */
8231 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8232 fixed20_12 disp_clk, bandwidth;
8235 a.full = dfixed_const(1000);
8236 disp_clk.full = dfixed_const(wm->disp_clk);
8237 disp_clk.full = dfixed_div(disp_clk, a);
8238 a.full = dfixed_const(32);
8239 b.full = dfixed_mul(a, disp_clk);
8241 a.full = dfixed_const(10);
8242 disp_clk_request_efficiency.full = dfixed_const(8);
8243 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8245 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8247 return dfixed_trunc(bandwidth);
8251 * dce8_available_bandwidth - get the min available bandwidth
8253 * @wm: watermark calculation data
8255 * Calculate the min available bandwidth used for display (CIK).
8256 * Used for display watermark bandwidth calculations
8257 * Returns the min available bandwidth in MBytes/s
8259 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8261 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8262 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8263 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8264 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8266 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8270 * dce8_average_bandwidth - get the average available bandwidth
8272 * @wm: watermark calculation data
8274 * Calculate the average available bandwidth used for display (CIK).
8275 * Used for display watermark bandwidth calculations
8276 * Returns the average available bandwidth in MBytes/s
8278 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8280 /* Calculate the display mode Average Bandwidth
8281 * DisplayMode should contain the source and destination dimensions,
8285 fixed20_12 line_time;
8286 fixed20_12 src_width;
8287 fixed20_12 bandwidth;
8290 a.full = dfixed_const(1000);
8291 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8292 line_time.full = dfixed_div(line_time, a);
8293 bpp.full = dfixed_const(wm->bytes_per_pixel);
8294 src_width.full = dfixed_const(wm->src_width);
8295 bandwidth.full = dfixed_mul(src_width, bpp);
8296 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8297 bandwidth.full = dfixed_div(bandwidth, line_time);
8299 return dfixed_trunc(bandwidth);
8303 * dce8_latency_watermark - get the latency watermark
8305 * @wm: watermark calculation data
8307 * Calculate the latency watermark (CIK).
8308 * Used for display watermark bandwidth calculations
8309 * Returns the latency watermark in ns
8311 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8313 /* First calculate the latency in ns */
8314 u32 mc_latency = 2000; /* 2000 ns. */
8315 u32 available_bandwidth = dce8_available_bandwidth(wm);
8316 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8317 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8318 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8319 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8320 (wm->num_heads * cursor_line_pair_return_time);
8321 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8322 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8323 u32 tmp, dmif_size = 12288;
8326 if (wm->num_heads == 0)
8329 a.full = dfixed_const(2);
8330 b.full = dfixed_const(1);
8331 if ((wm->vsc.full > a.full) ||
8332 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8334 ((wm->vsc.full >= a.full) && wm->interlaced))
8335 max_src_lines_per_dst_line = 4;
8337 max_src_lines_per_dst_line = 2;
8339 a.full = dfixed_const(available_bandwidth);
8340 b.full = dfixed_const(wm->num_heads);
8341 a.full = dfixed_div(a, b);
8343 b.full = dfixed_const(mc_latency + 512);
8344 c.full = dfixed_const(wm->disp_clk);
8345 b.full = dfixed_div(b, c);
8347 c.full = dfixed_const(dmif_size);
8348 b.full = dfixed_div(c, b);
8350 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8352 b.full = dfixed_const(1000);
8353 c.full = dfixed_const(wm->disp_clk);
8354 b.full = dfixed_div(c, b);
8355 c.full = dfixed_const(wm->bytes_per_pixel);
8356 b.full = dfixed_mul(b, c);
8358 lb_fill_bw = min(tmp, dfixed_trunc(b));
8360 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8361 b.full = dfixed_const(1000);
8362 c.full = dfixed_const(lb_fill_bw);
8363 b.full = dfixed_div(c, b);
8364 a.full = dfixed_div(a, b);
8365 line_fill_time = dfixed_trunc(a);
8367 if (line_fill_time < wm->active_time)
8370 return latency + (line_fill_time - wm->active_time);
8375 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8376 * average and available dram bandwidth
8378 * @wm: watermark calculation data
8380 * Check if the display average bandwidth fits in the display
8381 * dram bandwidth (CIK).
8382 * Used for display watermark bandwidth calculations
8383 * Returns true if the display fits, false if not.
8385 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8387 if (dce8_average_bandwidth(wm) <=
8388 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8395 * dce8_average_bandwidth_vs_available_bandwidth - check
8396 * average and available bandwidth
8398 * @wm: watermark calculation data
8400 * Check if the display average bandwidth fits in the display
8401 * available bandwidth (CIK).
8402 * Used for display watermark bandwidth calculations
8403 * Returns true if the display fits, false if not.
8405 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8407 if (dce8_average_bandwidth(wm) <=
8408 (dce8_available_bandwidth(wm) / wm->num_heads))
8415 * dce8_check_latency_hiding - check latency hiding
8417 * @wm: watermark calculation data
8419 * Check latency hiding (CIK).
8420 * Used for display watermark bandwidth calculations
8421 * Returns true if the display fits, false if not.
8423 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8425 u32 lb_partitions = wm->lb_size / wm->src_width;
8426 u32 line_time = wm->active_time + wm->blank_time;
8427 u32 latency_tolerant_lines;
8431 a.full = dfixed_const(1);
8432 if (wm->vsc.full > a.full)
8433 latency_tolerant_lines = 1;
8435 if (lb_partitions <= (wm->vtaps + 1))
8436 latency_tolerant_lines = 1;
8438 latency_tolerant_lines = 2;
8441 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8443 if (dce8_latency_watermark(wm) <= latency_hiding)
8450 * dce8_program_watermarks - program display watermarks
8452 * @rdev: radeon_device pointer
8453 * @radeon_crtc: the selected display controller
8454 * @lb_size: line buffer size
8455 * @num_heads: number of display controllers in use
8457 * Calculate and program the display watermarks for the
8458 * selected display controller (CIK).
8460 static void dce8_program_watermarks(struct radeon_device *rdev,
8461 struct radeon_crtc *radeon_crtc,
8462 u32 lb_size, u32 num_heads)
8464 struct drm_display_mode *mode = &radeon_crtc->base.mode;
8465 struct dce8_wm_params wm_low, wm_high;
8468 u32 latency_watermark_a = 0, latency_watermark_b = 0;
8471 if (radeon_crtc->base.enabled && num_heads && mode) {
8472 pixel_period = 1000000 / (u32)mode->clock;
8473 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8475 /* watermark for high clocks */
8476 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8477 rdev->pm.dpm_enabled) {
8479 radeon_dpm_get_mclk(rdev, false) * 10;
8481 radeon_dpm_get_sclk(rdev, false) * 10;
8483 wm_high.yclk = rdev->pm.current_mclk * 10;
8484 wm_high.sclk = rdev->pm.current_sclk * 10;
8487 wm_high.disp_clk = mode->clock;
8488 wm_high.src_width = mode->crtc_hdisplay;
8489 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8490 wm_high.blank_time = line_time - wm_high.active_time;
8491 wm_high.interlaced = false;
8492 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8493 wm_high.interlaced = true;
8494 wm_high.vsc = radeon_crtc->vsc;
8496 if (radeon_crtc->rmx_type != RMX_OFF)
8498 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8499 wm_high.lb_size = lb_size;
8500 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8501 wm_high.num_heads = num_heads;
8503 /* set for high clocks */
8504 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
8506 /* possibly force display priority to high */
8507 /* should really do this at mode validation time... */
8508 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8509 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8510 !dce8_check_latency_hiding(&wm_high) ||
8511 (rdev->disp_priority == 2)) {
8512 DRM_DEBUG_KMS("force priority to high\n");
8515 /* watermark for low clocks */
8516 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8517 rdev->pm.dpm_enabled) {
8519 radeon_dpm_get_mclk(rdev, true) * 10;
8521 radeon_dpm_get_sclk(rdev, true) * 10;
8523 wm_low.yclk = rdev->pm.current_mclk * 10;
8524 wm_low.sclk = rdev->pm.current_sclk * 10;
8527 wm_low.disp_clk = mode->clock;
8528 wm_low.src_width = mode->crtc_hdisplay;
8529 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8530 wm_low.blank_time = line_time - wm_low.active_time;
8531 wm_low.interlaced = false;
8532 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8533 wm_low.interlaced = true;
8534 wm_low.vsc = radeon_crtc->vsc;
8536 if (radeon_crtc->rmx_type != RMX_OFF)
8538 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8539 wm_low.lb_size = lb_size;
8540 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8541 wm_low.num_heads = num_heads;
8543 /* set for low clocks */
8544 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
8546 /* possibly force display priority to high */
8547 /* should really do this at mode validation time... */
8548 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
8549 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
8550 !dce8_check_latency_hiding(&wm_low) ||
8551 (rdev->disp_priority == 2)) {
8552 DRM_DEBUG_KMS("force priority to high\n");
8557 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8559 tmp &= ~LATENCY_WATERMARK_MASK(3);
8560 tmp |= LATENCY_WATERMARK_MASK(1);
8561 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8562 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8563 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
8564 LATENCY_HIGH_WATERMARK(line_time)));
8566 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8567 tmp &= ~LATENCY_WATERMARK_MASK(3);
8568 tmp |= LATENCY_WATERMARK_MASK(2);
8569 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8570 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8571 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
8572 LATENCY_HIGH_WATERMARK(line_time)));
8573 /* restore original selection */
8574 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
8576 /* save values for DPM */
8577 radeon_crtc->line_time = line_time;
8578 radeon_crtc->wm_high = latency_watermark_a;
8579 radeon_crtc->wm_low = latency_watermark_b;
8583 * dce8_bandwidth_update - program display watermarks
8585 * @rdev: radeon_device pointer
8587 * Calculate and program the display watermarks and line
8588 * buffer allocation (CIK).
8590 void dce8_bandwidth_update(struct radeon_device *rdev)
8592 struct drm_display_mode *mode = NULL;
8593 u32 num_heads = 0, lb_size;
8596 radeon_update_display_priority(rdev);
8598 for (i = 0; i < rdev->num_crtc; i++) {
8599 if (rdev->mode_info.crtcs[i]->base.enabled)
8602 for (i = 0; i < rdev->num_crtc; i++) {
8603 mode = &rdev->mode_info.crtcs[i]->base.mode;
8604 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8605 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8610 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8612 * @rdev: radeon_device pointer
8614 * Fetches a GPU clock counter snapshot (SI).
8615 * Returns the 64 bit clock counter snapshot.
8617 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8621 mutex_lock(&rdev->gpu_clock_mutex);
8622 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8623 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8624 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8625 mutex_unlock(&rdev->gpu_clock_mutex);
8629 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8630 u32 cntl_reg, u32 status_reg)
8633 struct atom_clock_dividers dividers;
8636 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8637 clock, false, ÷rs);
8641 tmp = RREG32_SMC(cntl_reg);
8642 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8643 tmp |= dividers.post_divider;
8644 WREG32_SMC(cntl_reg, tmp);
8646 for (i = 0; i < 100; i++) {
8647 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8657 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8661 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8665 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8669 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
8671 struct pci_dev *root = rdev->pdev->bus->self;
8672 int bridge_pos, gpu_pos;
8673 u32 speed_cntl, mask, current_data_rate;
8677 if (radeon_pcie_gen2 == 0)
8680 if (rdev->flags & RADEON_IS_IGP)
8683 if (!(rdev->flags & RADEON_IS_PCIE))
8686 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
8690 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
8693 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8694 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
8695 LC_CURRENT_DATA_RATE_SHIFT;
8696 if (mask & DRM_PCIE_SPEED_80) {
8697 if (current_data_rate == 2) {
8698 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
8701 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
8702 } else if (mask & DRM_PCIE_SPEED_50) {
8703 if (current_data_rate == 1) {
8704 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
8707 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
8710 bridge_pos = pci_pcie_cap(root);
8714 gpu_pos = pci_pcie_cap(rdev->pdev);
8718 if (mask & DRM_PCIE_SPEED_80) {
8719 /* re-try equalization if gen3 is not already enabled */
8720 if (current_data_rate != 2) {
8721 u16 bridge_cfg, gpu_cfg;
8722 u16 bridge_cfg2, gpu_cfg2;
8723 u32 max_lw, current_lw, tmp;
8725 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8726 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8728 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
8729 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8731 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
8732 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8734 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8735 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
8736 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
8738 if (current_lw < max_lw) {
8739 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8740 if (tmp & LC_RENEGOTIATION_SUPPORT) {
8741 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
8742 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
8743 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
8744 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
8748 for (i = 0; i < 10; i++) {
8750 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
8751 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
8754 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8755 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8757 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
8758 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
8760 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8761 tmp |= LC_SET_QUIESCE;
8762 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8764 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8766 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8771 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
8772 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8773 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
8774 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8776 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
8777 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8778 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
8779 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8782 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
8783 tmp16 &= ~((1 << 4) | (7 << 9));
8784 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
8785 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
8787 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8788 tmp16 &= ~((1 << 4) | (7 << 9));
8789 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
8790 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8792 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8793 tmp &= ~LC_SET_QUIESCE;
8794 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8799 /* set the link speed */
8800 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
8801 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
8802 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8804 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8806 if (mask & DRM_PCIE_SPEED_80)
8807 tmp16 |= 3; /* gen3 */
8808 else if (mask & DRM_PCIE_SPEED_50)
8809 tmp16 |= 2; /* gen2 */
8811 tmp16 |= 1; /* gen1 */
8812 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8814 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8815 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
8816 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8818 for (i = 0; i < rdev->usec_timeout; i++) {
8819 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8820 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
8826 static void cik_program_aspm(struct radeon_device *rdev)
8829 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
8830 bool disable_clkreq = false;
8832 if (radeon_aspm == 0)
8835 /* XXX double check IGPs */
8836 if (rdev->flags & RADEON_IS_IGP)
8839 if (!(rdev->flags & RADEON_IS_PCIE))
8842 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8843 data &= ~LC_XMIT_N_FTS_MASK;
8844 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
8846 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
8848 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
8849 data |= LC_GO_TO_RECOVERY;
8851 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
8853 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
8854 data |= P_IGNORE_EDB_ERR;
8856 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
8858 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8859 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
8860 data |= LC_PMI_TO_L1_DIS;
8862 data |= LC_L0S_INACTIVITY(7);
8865 data |= LC_L1_INACTIVITY(7);
8866 data &= ~LC_PMI_TO_L1_DIS;
8868 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8870 if (!disable_plloff_in_l1) {
8871 bool clk_req_support;
8873 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
8874 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8875 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8877 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
8879 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
8880 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8881 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8883 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
8885 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
8886 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8887 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8889 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
8891 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
8892 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8893 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8895 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
8897 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8898 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
8899 data |= LC_DYN_LANES_PWR_STATE(3);
8901 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
8903 if (!disable_clkreq) {
8904 struct pci_dev *root = rdev->pdev->bus->self;
8907 clk_req_support = false;
8908 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
8909 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
8910 clk_req_support = true;
8912 clk_req_support = false;
8915 if (clk_req_support) {
8916 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
8917 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
8919 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
8921 orig = data = RREG32_SMC(THM_CLK_CNTL);
8922 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
8923 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
8925 WREG32_SMC(THM_CLK_CNTL, data);
8927 orig = data = RREG32_SMC(MISC_CLK_CTRL);
8928 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
8929 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
8931 WREG32_SMC(MISC_CLK_CTRL, data);
8933 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
8934 data &= ~BCLK_AS_XCLK;
8936 WREG32_SMC(CG_CLKPIN_CNTL, data);
8938 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
8939 data &= ~FORCE_BIF_REFCLK_EN;
8941 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
8943 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
8944 data &= ~MPLL_CLKOUT_SEL_MASK;
8945 data |= MPLL_CLKOUT_SEL(4);
8947 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
8952 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8955 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
8956 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
8958 WREG32_PCIE_PORT(PCIE_CNTL2, data);
8961 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8962 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
8963 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8964 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
8965 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8966 data &= ~LC_L0S_INACTIVITY_MASK;
8968 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);