2 * Copyright 2012 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
29 #include "radeon_asic.h"
32 #include "cik_blit_shaders.h"
33 #include "radeon_ucode.h"
34 #include "clearstate_ci.h"
36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
42 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
43 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
44 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
45 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
46 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
47 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
48 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
49 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
50 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
51 MODULE_FIRMWARE("radeon/KABINI_me.bin");
52 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
53 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
54 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
55 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
57 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
58 extern void r600_ih_ring_fini(struct radeon_device *rdev);
59 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
60 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
61 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
62 extern void sumo_rlc_fini(struct radeon_device *rdev);
63 extern int sumo_rlc_init(struct radeon_device *rdev);
64 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
65 extern void si_rlc_reset(struct radeon_device *rdev);
66 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
67 extern int cik_sdma_resume(struct radeon_device *rdev);
68 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
69 extern void cik_sdma_fini(struct radeon_device *rdev);
70 extern void cik_sdma_vm_set_page(struct radeon_device *rdev,
73 uint64_t addr, unsigned count,
74 uint32_t incr, uint32_t flags);
75 static void cik_rlc_stop(struct radeon_device *rdev);
76 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
77 static void cik_program_aspm(struct radeon_device *rdev);
78 static void cik_init_pg(struct radeon_device *rdev);
79 static void cik_init_cg(struct radeon_device *rdev);
80 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
83 /* get temperature in millidegrees */
84 int ci_get_temp(struct radeon_device *rdev)
89 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
95 actual_temp = temp & 0x1ff;
97 actual_temp = actual_temp * 1000;
102 /* get temperature in millidegrees */
103 int kv_get_temp(struct radeon_device *rdev)
108 temp = RREG32_SMC(0xC0300E0C);
111 actual_temp = (temp / 8) - 49;
115 actual_temp = actual_temp * 1000;
121 * Indirect registers accessor
123 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
128 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
129 WREG32(PCIE_INDEX, reg);
130 (void)RREG32(PCIE_INDEX);
131 r = RREG32(PCIE_DATA);
132 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
136 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
140 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
141 WREG32(PCIE_INDEX, reg);
142 (void)RREG32(PCIE_INDEX);
143 WREG32(PCIE_DATA, v);
144 (void)RREG32(PCIE_DATA);
145 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
148 static const u32 spectre_rlc_save_restore_register_list[] =
150 (0x0e00 << 16) | (0xc12c >> 2),
152 (0x0e00 << 16) | (0xc140 >> 2),
154 (0x0e00 << 16) | (0xc150 >> 2),
156 (0x0e00 << 16) | (0xc15c >> 2),
158 (0x0e00 << 16) | (0xc168 >> 2),
160 (0x0e00 << 16) | (0xc170 >> 2),
162 (0x0e00 << 16) | (0xc178 >> 2),
164 (0x0e00 << 16) | (0xc204 >> 2),
166 (0x0e00 << 16) | (0xc2b4 >> 2),
168 (0x0e00 << 16) | (0xc2b8 >> 2),
170 (0x0e00 << 16) | (0xc2bc >> 2),
172 (0x0e00 << 16) | (0xc2c0 >> 2),
174 (0x0e00 << 16) | (0x8228 >> 2),
176 (0x0e00 << 16) | (0x829c >> 2),
178 (0x0e00 << 16) | (0x869c >> 2),
180 (0x0600 << 16) | (0x98f4 >> 2),
182 (0x0e00 << 16) | (0x98f8 >> 2),
184 (0x0e00 << 16) | (0x9900 >> 2),
186 (0x0e00 << 16) | (0xc260 >> 2),
188 (0x0e00 << 16) | (0x90e8 >> 2),
190 (0x0e00 << 16) | (0x3c000 >> 2),
192 (0x0e00 << 16) | (0x3c00c >> 2),
194 (0x0e00 << 16) | (0x8c1c >> 2),
196 (0x0e00 << 16) | (0x9700 >> 2),
198 (0x0e00 << 16) | (0xcd20 >> 2),
200 (0x4e00 << 16) | (0xcd20 >> 2),
202 (0x5e00 << 16) | (0xcd20 >> 2),
204 (0x6e00 << 16) | (0xcd20 >> 2),
206 (0x7e00 << 16) | (0xcd20 >> 2),
208 (0x8e00 << 16) | (0xcd20 >> 2),
210 (0x9e00 << 16) | (0xcd20 >> 2),
212 (0xae00 << 16) | (0xcd20 >> 2),
214 (0xbe00 << 16) | (0xcd20 >> 2),
216 (0x0e00 << 16) | (0x89bc >> 2),
218 (0x0e00 << 16) | (0x8900 >> 2),
221 (0x0e00 << 16) | (0xc130 >> 2),
223 (0x0e00 << 16) | (0xc134 >> 2),
225 (0x0e00 << 16) | (0xc1fc >> 2),
227 (0x0e00 << 16) | (0xc208 >> 2),
229 (0x0e00 << 16) | (0xc264 >> 2),
231 (0x0e00 << 16) | (0xc268 >> 2),
233 (0x0e00 << 16) | (0xc26c >> 2),
235 (0x0e00 << 16) | (0xc270 >> 2),
237 (0x0e00 << 16) | (0xc274 >> 2),
239 (0x0e00 << 16) | (0xc278 >> 2),
241 (0x0e00 << 16) | (0xc27c >> 2),
243 (0x0e00 << 16) | (0xc280 >> 2),
245 (0x0e00 << 16) | (0xc284 >> 2),
247 (0x0e00 << 16) | (0xc288 >> 2),
249 (0x0e00 << 16) | (0xc28c >> 2),
251 (0x0e00 << 16) | (0xc290 >> 2),
253 (0x0e00 << 16) | (0xc294 >> 2),
255 (0x0e00 << 16) | (0xc298 >> 2),
257 (0x0e00 << 16) | (0xc29c >> 2),
259 (0x0e00 << 16) | (0xc2a0 >> 2),
261 (0x0e00 << 16) | (0xc2a4 >> 2),
263 (0x0e00 << 16) | (0xc2a8 >> 2),
265 (0x0e00 << 16) | (0xc2ac >> 2),
267 (0x0e00 << 16) | (0xc2b0 >> 2),
269 (0x0e00 << 16) | (0x301d0 >> 2),
271 (0x0e00 << 16) | (0x30238 >> 2),
273 (0x0e00 << 16) | (0x30250 >> 2),
275 (0x0e00 << 16) | (0x30254 >> 2),
277 (0x0e00 << 16) | (0x30258 >> 2),
279 (0x0e00 << 16) | (0x3025c >> 2),
281 (0x4e00 << 16) | (0xc900 >> 2),
283 (0x5e00 << 16) | (0xc900 >> 2),
285 (0x6e00 << 16) | (0xc900 >> 2),
287 (0x7e00 << 16) | (0xc900 >> 2),
289 (0x8e00 << 16) | (0xc900 >> 2),
291 (0x9e00 << 16) | (0xc900 >> 2),
293 (0xae00 << 16) | (0xc900 >> 2),
295 (0xbe00 << 16) | (0xc900 >> 2),
297 (0x4e00 << 16) | (0xc904 >> 2),
299 (0x5e00 << 16) | (0xc904 >> 2),
301 (0x6e00 << 16) | (0xc904 >> 2),
303 (0x7e00 << 16) | (0xc904 >> 2),
305 (0x8e00 << 16) | (0xc904 >> 2),
307 (0x9e00 << 16) | (0xc904 >> 2),
309 (0xae00 << 16) | (0xc904 >> 2),
311 (0xbe00 << 16) | (0xc904 >> 2),
313 (0x4e00 << 16) | (0xc908 >> 2),
315 (0x5e00 << 16) | (0xc908 >> 2),
317 (0x6e00 << 16) | (0xc908 >> 2),
319 (0x7e00 << 16) | (0xc908 >> 2),
321 (0x8e00 << 16) | (0xc908 >> 2),
323 (0x9e00 << 16) | (0xc908 >> 2),
325 (0xae00 << 16) | (0xc908 >> 2),
327 (0xbe00 << 16) | (0xc908 >> 2),
329 (0x4e00 << 16) | (0xc90c >> 2),
331 (0x5e00 << 16) | (0xc90c >> 2),
333 (0x6e00 << 16) | (0xc90c >> 2),
335 (0x7e00 << 16) | (0xc90c >> 2),
337 (0x8e00 << 16) | (0xc90c >> 2),
339 (0x9e00 << 16) | (0xc90c >> 2),
341 (0xae00 << 16) | (0xc90c >> 2),
343 (0xbe00 << 16) | (0xc90c >> 2),
345 (0x4e00 << 16) | (0xc910 >> 2),
347 (0x5e00 << 16) | (0xc910 >> 2),
349 (0x6e00 << 16) | (0xc910 >> 2),
351 (0x7e00 << 16) | (0xc910 >> 2),
353 (0x8e00 << 16) | (0xc910 >> 2),
355 (0x9e00 << 16) | (0xc910 >> 2),
357 (0xae00 << 16) | (0xc910 >> 2),
359 (0xbe00 << 16) | (0xc910 >> 2),
361 (0x0e00 << 16) | (0xc99c >> 2),
363 (0x0e00 << 16) | (0x9834 >> 2),
365 (0x0000 << 16) | (0x30f00 >> 2),
367 (0x0001 << 16) | (0x30f00 >> 2),
369 (0x0000 << 16) | (0x30f04 >> 2),
371 (0x0001 << 16) | (0x30f04 >> 2),
373 (0x0000 << 16) | (0x30f08 >> 2),
375 (0x0001 << 16) | (0x30f08 >> 2),
377 (0x0000 << 16) | (0x30f0c >> 2),
379 (0x0001 << 16) | (0x30f0c >> 2),
381 (0x0600 << 16) | (0x9b7c >> 2),
383 (0x0e00 << 16) | (0x8a14 >> 2),
385 (0x0e00 << 16) | (0x8a18 >> 2),
387 (0x0600 << 16) | (0x30a00 >> 2),
389 (0x0e00 << 16) | (0x8bf0 >> 2),
391 (0x0e00 << 16) | (0x8bcc >> 2),
393 (0x0e00 << 16) | (0x8b24 >> 2),
395 (0x0e00 << 16) | (0x30a04 >> 2),
397 (0x0600 << 16) | (0x30a10 >> 2),
399 (0x0600 << 16) | (0x30a14 >> 2),
401 (0x0600 << 16) | (0x30a18 >> 2),
403 (0x0600 << 16) | (0x30a2c >> 2),
405 (0x0e00 << 16) | (0xc700 >> 2),
407 (0x0e00 << 16) | (0xc704 >> 2),
409 (0x0e00 << 16) | (0xc708 >> 2),
411 (0x0e00 << 16) | (0xc768 >> 2),
413 (0x0400 << 16) | (0xc770 >> 2),
415 (0x0400 << 16) | (0xc774 >> 2),
417 (0x0400 << 16) | (0xc778 >> 2),
419 (0x0400 << 16) | (0xc77c >> 2),
421 (0x0400 << 16) | (0xc780 >> 2),
423 (0x0400 << 16) | (0xc784 >> 2),
425 (0x0400 << 16) | (0xc788 >> 2),
427 (0x0400 << 16) | (0xc78c >> 2),
429 (0x0400 << 16) | (0xc798 >> 2),
431 (0x0400 << 16) | (0xc79c >> 2),
433 (0x0400 << 16) | (0xc7a0 >> 2),
435 (0x0400 << 16) | (0xc7a4 >> 2),
437 (0x0400 << 16) | (0xc7a8 >> 2),
439 (0x0400 << 16) | (0xc7ac >> 2),
441 (0x0400 << 16) | (0xc7b0 >> 2),
443 (0x0400 << 16) | (0xc7b4 >> 2),
445 (0x0e00 << 16) | (0x9100 >> 2),
447 (0x0e00 << 16) | (0x3c010 >> 2),
449 (0x0e00 << 16) | (0x92a8 >> 2),
451 (0x0e00 << 16) | (0x92ac >> 2),
453 (0x0e00 << 16) | (0x92b4 >> 2),
455 (0x0e00 << 16) | (0x92b8 >> 2),
457 (0x0e00 << 16) | (0x92bc >> 2),
459 (0x0e00 << 16) | (0x92c0 >> 2),
461 (0x0e00 << 16) | (0x92c4 >> 2),
463 (0x0e00 << 16) | (0x92c8 >> 2),
465 (0x0e00 << 16) | (0x92cc >> 2),
467 (0x0e00 << 16) | (0x92d0 >> 2),
469 (0x0e00 << 16) | (0x8c00 >> 2),
471 (0x0e00 << 16) | (0x8c04 >> 2),
473 (0x0e00 << 16) | (0x8c20 >> 2),
475 (0x0e00 << 16) | (0x8c38 >> 2),
477 (0x0e00 << 16) | (0x8c3c >> 2),
479 (0x0e00 << 16) | (0xae00 >> 2),
481 (0x0e00 << 16) | (0x9604 >> 2),
483 (0x0e00 << 16) | (0xac08 >> 2),
485 (0x0e00 << 16) | (0xac0c >> 2),
487 (0x0e00 << 16) | (0xac10 >> 2),
489 (0x0e00 << 16) | (0xac14 >> 2),
491 (0x0e00 << 16) | (0xac58 >> 2),
493 (0x0e00 << 16) | (0xac68 >> 2),
495 (0x0e00 << 16) | (0xac6c >> 2),
497 (0x0e00 << 16) | (0xac70 >> 2),
499 (0x0e00 << 16) | (0xac74 >> 2),
501 (0x0e00 << 16) | (0xac78 >> 2),
503 (0x0e00 << 16) | (0xac7c >> 2),
505 (0x0e00 << 16) | (0xac80 >> 2),
507 (0x0e00 << 16) | (0xac84 >> 2),
509 (0x0e00 << 16) | (0xac88 >> 2),
511 (0x0e00 << 16) | (0xac8c >> 2),
513 (0x0e00 << 16) | (0x970c >> 2),
515 (0x0e00 << 16) | (0x9714 >> 2),
517 (0x0e00 << 16) | (0x9718 >> 2),
519 (0x0e00 << 16) | (0x971c >> 2),
521 (0x0e00 << 16) | (0x31068 >> 2),
523 (0x4e00 << 16) | (0x31068 >> 2),
525 (0x5e00 << 16) | (0x31068 >> 2),
527 (0x6e00 << 16) | (0x31068 >> 2),
529 (0x7e00 << 16) | (0x31068 >> 2),
531 (0x8e00 << 16) | (0x31068 >> 2),
533 (0x9e00 << 16) | (0x31068 >> 2),
535 (0xae00 << 16) | (0x31068 >> 2),
537 (0xbe00 << 16) | (0x31068 >> 2),
539 (0x0e00 << 16) | (0xcd10 >> 2),
541 (0x0e00 << 16) | (0xcd14 >> 2),
543 (0x0e00 << 16) | (0x88b0 >> 2),
545 (0x0e00 << 16) | (0x88b4 >> 2),
547 (0x0e00 << 16) | (0x88b8 >> 2),
549 (0x0e00 << 16) | (0x88bc >> 2),
551 (0x0400 << 16) | (0x89c0 >> 2),
553 (0x0e00 << 16) | (0x88c4 >> 2),
555 (0x0e00 << 16) | (0x88c8 >> 2),
557 (0x0e00 << 16) | (0x88d0 >> 2),
559 (0x0e00 << 16) | (0x88d4 >> 2),
561 (0x0e00 << 16) | (0x88d8 >> 2),
563 (0x0e00 << 16) | (0x8980 >> 2),
565 (0x0e00 << 16) | (0x30938 >> 2),
567 (0x0e00 << 16) | (0x3093c >> 2),
569 (0x0e00 << 16) | (0x30940 >> 2),
571 (0x0e00 << 16) | (0x89a0 >> 2),
573 (0x0e00 << 16) | (0x30900 >> 2),
575 (0x0e00 << 16) | (0x30904 >> 2),
577 (0x0e00 << 16) | (0x89b4 >> 2),
579 (0x0e00 << 16) | (0x3c210 >> 2),
581 (0x0e00 << 16) | (0x3c214 >> 2),
583 (0x0e00 << 16) | (0x3c218 >> 2),
585 (0x0e00 << 16) | (0x8904 >> 2),
588 (0x0e00 << 16) | (0x8c28 >> 2),
589 (0x0e00 << 16) | (0x8c2c >> 2),
590 (0x0e00 << 16) | (0x8c30 >> 2),
591 (0x0e00 << 16) | (0x8c34 >> 2),
592 (0x0e00 << 16) | (0x9600 >> 2),
595 static const u32 kalindi_rlc_save_restore_register_list[] =
597 (0x0e00 << 16) | (0xc12c >> 2),
599 (0x0e00 << 16) | (0xc140 >> 2),
601 (0x0e00 << 16) | (0xc150 >> 2),
603 (0x0e00 << 16) | (0xc15c >> 2),
605 (0x0e00 << 16) | (0xc168 >> 2),
607 (0x0e00 << 16) | (0xc170 >> 2),
609 (0x0e00 << 16) | (0xc204 >> 2),
611 (0x0e00 << 16) | (0xc2b4 >> 2),
613 (0x0e00 << 16) | (0xc2b8 >> 2),
615 (0x0e00 << 16) | (0xc2bc >> 2),
617 (0x0e00 << 16) | (0xc2c0 >> 2),
619 (0x0e00 << 16) | (0x8228 >> 2),
621 (0x0e00 << 16) | (0x829c >> 2),
623 (0x0e00 << 16) | (0x869c >> 2),
625 (0x0600 << 16) | (0x98f4 >> 2),
627 (0x0e00 << 16) | (0x98f8 >> 2),
629 (0x0e00 << 16) | (0x9900 >> 2),
631 (0x0e00 << 16) | (0xc260 >> 2),
633 (0x0e00 << 16) | (0x90e8 >> 2),
635 (0x0e00 << 16) | (0x3c000 >> 2),
637 (0x0e00 << 16) | (0x3c00c >> 2),
639 (0x0e00 << 16) | (0x8c1c >> 2),
641 (0x0e00 << 16) | (0x9700 >> 2),
643 (0x0e00 << 16) | (0xcd20 >> 2),
645 (0x4e00 << 16) | (0xcd20 >> 2),
647 (0x5e00 << 16) | (0xcd20 >> 2),
649 (0x6e00 << 16) | (0xcd20 >> 2),
651 (0x7e00 << 16) | (0xcd20 >> 2),
653 (0x0e00 << 16) | (0x89bc >> 2),
655 (0x0e00 << 16) | (0x8900 >> 2),
658 (0x0e00 << 16) | (0xc130 >> 2),
660 (0x0e00 << 16) | (0xc134 >> 2),
662 (0x0e00 << 16) | (0xc1fc >> 2),
664 (0x0e00 << 16) | (0xc208 >> 2),
666 (0x0e00 << 16) | (0xc264 >> 2),
668 (0x0e00 << 16) | (0xc268 >> 2),
670 (0x0e00 << 16) | (0xc26c >> 2),
672 (0x0e00 << 16) | (0xc270 >> 2),
674 (0x0e00 << 16) | (0xc274 >> 2),
676 (0x0e00 << 16) | (0xc28c >> 2),
678 (0x0e00 << 16) | (0xc290 >> 2),
680 (0x0e00 << 16) | (0xc294 >> 2),
682 (0x0e00 << 16) | (0xc298 >> 2),
684 (0x0e00 << 16) | (0xc2a0 >> 2),
686 (0x0e00 << 16) | (0xc2a4 >> 2),
688 (0x0e00 << 16) | (0xc2a8 >> 2),
690 (0x0e00 << 16) | (0xc2ac >> 2),
692 (0x0e00 << 16) | (0x301d0 >> 2),
694 (0x0e00 << 16) | (0x30238 >> 2),
696 (0x0e00 << 16) | (0x30250 >> 2),
698 (0x0e00 << 16) | (0x30254 >> 2),
700 (0x0e00 << 16) | (0x30258 >> 2),
702 (0x0e00 << 16) | (0x3025c >> 2),
704 (0x4e00 << 16) | (0xc900 >> 2),
706 (0x5e00 << 16) | (0xc900 >> 2),
708 (0x6e00 << 16) | (0xc900 >> 2),
710 (0x7e00 << 16) | (0xc900 >> 2),
712 (0x4e00 << 16) | (0xc904 >> 2),
714 (0x5e00 << 16) | (0xc904 >> 2),
716 (0x6e00 << 16) | (0xc904 >> 2),
718 (0x7e00 << 16) | (0xc904 >> 2),
720 (0x4e00 << 16) | (0xc908 >> 2),
722 (0x5e00 << 16) | (0xc908 >> 2),
724 (0x6e00 << 16) | (0xc908 >> 2),
726 (0x7e00 << 16) | (0xc908 >> 2),
728 (0x4e00 << 16) | (0xc90c >> 2),
730 (0x5e00 << 16) | (0xc90c >> 2),
732 (0x6e00 << 16) | (0xc90c >> 2),
734 (0x7e00 << 16) | (0xc90c >> 2),
736 (0x4e00 << 16) | (0xc910 >> 2),
738 (0x5e00 << 16) | (0xc910 >> 2),
740 (0x6e00 << 16) | (0xc910 >> 2),
742 (0x7e00 << 16) | (0xc910 >> 2),
744 (0x0e00 << 16) | (0xc99c >> 2),
746 (0x0e00 << 16) | (0x9834 >> 2),
748 (0x0000 << 16) | (0x30f00 >> 2),
750 (0x0000 << 16) | (0x30f04 >> 2),
752 (0x0000 << 16) | (0x30f08 >> 2),
754 (0x0000 << 16) | (0x30f0c >> 2),
756 (0x0600 << 16) | (0x9b7c >> 2),
758 (0x0e00 << 16) | (0x8a14 >> 2),
760 (0x0e00 << 16) | (0x8a18 >> 2),
762 (0x0600 << 16) | (0x30a00 >> 2),
764 (0x0e00 << 16) | (0x8bf0 >> 2),
766 (0x0e00 << 16) | (0x8bcc >> 2),
768 (0x0e00 << 16) | (0x8b24 >> 2),
770 (0x0e00 << 16) | (0x30a04 >> 2),
772 (0x0600 << 16) | (0x30a10 >> 2),
774 (0x0600 << 16) | (0x30a14 >> 2),
776 (0x0600 << 16) | (0x30a18 >> 2),
778 (0x0600 << 16) | (0x30a2c >> 2),
780 (0x0e00 << 16) | (0xc700 >> 2),
782 (0x0e00 << 16) | (0xc704 >> 2),
784 (0x0e00 << 16) | (0xc708 >> 2),
786 (0x0e00 << 16) | (0xc768 >> 2),
788 (0x0400 << 16) | (0xc770 >> 2),
790 (0x0400 << 16) | (0xc774 >> 2),
792 (0x0400 << 16) | (0xc798 >> 2),
794 (0x0400 << 16) | (0xc79c >> 2),
796 (0x0e00 << 16) | (0x9100 >> 2),
798 (0x0e00 << 16) | (0x3c010 >> 2),
800 (0x0e00 << 16) | (0x8c00 >> 2),
802 (0x0e00 << 16) | (0x8c04 >> 2),
804 (0x0e00 << 16) | (0x8c20 >> 2),
806 (0x0e00 << 16) | (0x8c38 >> 2),
808 (0x0e00 << 16) | (0x8c3c >> 2),
810 (0x0e00 << 16) | (0xae00 >> 2),
812 (0x0e00 << 16) | (0x9604 >> 2),
814 (0x0e00 << 16) | (0xac08 >> 2),
816 (0x0e00 << 16) | (0xac0c >> 2),
818 (0x0e00 << 16) | (0xac10 >> 2),
820 (0x0e00 << 16) | (0xac14 >> 2),
822 (0x0e00 << 16) | (0xac58 >> 2),
824 (0x0e00 << 16) | (0xac68 >> 2),
826 (0x0e00 << 16) | (0xac6c >> 2),
828 (0x0e00 << 16) | (0xac70 >> 2),
830 (0x0e00 << 16) | (0xac74 >> 2),
832 (0x0e00 << 16) | (0xac78 >> 2),
834 (0x0e00 << 16) | (0xac7c >> 2),
836 (0x0e00 << 16) | (0xac80 >> 2),
838 (0x0e00 << 16) | (0xac84 >> 2),
840 (0x0e00 << 16) | (0xac88 >> 2),
842 (0x0e00 << 16) | (0xac8c >> 2),
844 (0x0e00 << 16) | (0x970c >> 2),
846 (0x0e00 << 16) | (0x9714 >> 2),
848 (0x0e00 << 16) | (0x9718 >> 2),
850 (0x0e00 << 16) | (0x971c >> 2),
852 (0x0e00 << 16) | (0x31068 >> 2),
854 (0x4e00 << 16) | (0x31068 >> 2),
856 (0x5e00 << 16) | (0x31068 >> 2),
858 (0x6e00 << 16) | (0x31068 >> 2),
860 (0x7e00 << 16) | (0x31068 >> 2),
862 (0x0e00 << 16) | (0xcd10 >> 2),
864 (0x0e00 << 16) | (0xcd14 >> 2),
866 (0x0e00 << 16) | (0x88b0 >> 2),
868 (0x0e00 << 16) | (0x88b4 >> 2),
870 (0x0e00 << 16) | (0x88b8 >> 2),
872 (0x0e00 << 16) | (0x88bc >> 2),
874 (0x0400 << 16) | (0x89c0 >> 2),
876 (0x0e00 << 16) | (0x88c4 >> 2),
878 (0x0e00 << 16) | (0x88c8 >> 2),
880 (0x0e00 << 16) | (0x88d0 >> 2),
882 (0x0e00 << 16) | (0x88d4 >> 2),
884 (0x0e00 << 16) | (0x88d8 >> 2),
886 (0x0e00 << 16) | (0x8980 >> 2),
888 (0x0e00 << 16) | (0x30938 >> 2),
890 (0x0e00 << 16) | (0x3093c >> 2),
892 (0x0e00 << 16) | (0x30940 >> 2),
894 (0x0e00 << 16) | (0x89a0 >> 2),
896 (0x0e00 << 16) | (0x30900 >> 2),
898 (0x0e00 << 16) | (0x30904 >> 2),
900 (0x0e00 << 16) | (0x89b4 >> 2),
902 (0x0e00 << 16) | (0x3e1fc >> 2),
904 (0x0e00 << 16) | (0x3c210 >> 2),
906 (0x0e00 << 16) | (0x3c214 >> 2),
908 (0x0e00 << 16) | (0x3c218 >> 2),
910 (0x0e00 << 16) | (0x8904 >> 2),
913 (0x0e00 << 16) | (0x8c28 >> 2),
914 (0x0e00 << 16) | (0x8c2c >> 2),
915 (0x0e00 << 16) | (0x8c30 >> 2),
916 (0x0e00 << 16) | (0x8c34 >> 2),
917 (0x0e00 << 16) | (0x9600 >> 2),
920 static const u32 bonaire_golden_spm_registers[] =
922 0x30800, 0xe0ffffff, 0xe0000000
925 static const u32 bonaire_golden_common_registers[] =
927 0xc770, 0xffffffff, 0x00000800,
928 0xc774, 0xffffffff, 0x00000800,
929 0xc798, 0xffffffff, 0x00007fbf,
930 0xc79c, 0xffffffff, 0x00007faf
933 static const u32 bonaire_golden_registers[] =
935 0x3354, 0x00000333, 0x00000333,
936 0x3350, 0x000c0fc0, 0x00040200,
937 0x9a10, 0x00010000, 0x00058208,
938 0x3c000, 0xffff1fff, 0x00140000,
939 0x3c200, 0xfdfc0fff, 0x00000100,
940 0x3c234, 0x40000000, 0x40000200,
941 0x9830, 0xffffffff, 0x00000000,
942 0x9834, 0xf00fffff, 0x00000400,
943 0x9838, 0x0002021c, 0x00020200,
944 0xc78, 0x00000080, 0x00000000,
945 0x5bb0, 0x000000f0, 0x00000070,
946 0x5bc0, 0xf0311fff, 0x80300000,
947 0x98f8, 0x73773777, 0x12010001,
948 0x350c, 0x00810000, 0x408af000,
949 0x7030, 0x31000111, 0x00000011,
950 0x2f48, 0x73773777, 0x12010001,
951 0x220c, 0x00007fb6, 0x0021a1b1,
952 0x2210, 0x00007fb6, 0x002021b1,
953 0x2180, 0x00007fb6, 0x00002191,
954 0x2218, 0x00007fb6, 0x002121b1,
955 0x221c, 0x00007fb6, 0x002021b1,
956 0x21dc, 0x00007fb6, 0x00002191,
957 0x21e0, 0x00007fb6, 0x00002191,
958 0x3628, 0x0000003f, 0x0000000a,
959 0x362c, 0x0000003f, 0x0000000a,
960 0x2ae4, 0x00073ffe, 0x000022a2,
961 0x240c, 0x000007ff, 0x00000000,
962 0x8a14, 0xf000003f, 0x00000007,
963 0x8bf0, 0x00002001, 0x00000001,
964 0x8b24, 0xffffffff, 0x00ffffff,
965 0x30a04, 0x0000ff0f, 0x00000000,
966 0x28a4c, 0x07ffffff, 0x06000000,
967 0x4d8, 0x00000fff, 0x00000100,
968 0x3e78, 0x00000001, 0x00000002,
969 0x9100, 0x03000000, 0x0362c688,
970 0x8c00, 0x000000ff, 0x00000001,
971 0xe40, 0x00001fff, 0x00001fff,
972 0x9060, 0x0000007f, 0x00000020,
973 0x9508, 0x00010000, 0x00010000,
974 0xac14, 0x000003ff, 0x000000f3,
975 0xac0c, 0xffffffff, 0x00001032
978 static const u32 bonaire_mgcg_cgcg_init[] =
980 0xc420, 0xffffffff, 0xfffffffc,
981 0x30800, 0xffffffff, 0xe0000000,
982 0x3c2a0, 0xffffffff, 0x00000100,
983 0x3c208, 0xffffffff, 0x00000100,
984 0x3c2c0, 0xffffffff, 0xc0000100,
985 0x3c2c8, 0xffffffff, 0xc0000100,
986 0x3c2c4, 0xffffffff, 0xc0000100,
987 0x55e4, 0xffffffff, 0x00600100,
988 0x3c280, 0xffffffff, 0x00000100,
989 0x3c214, 0xffffffff, 0x06000100,
990 0x3c220, 0xffffffff, 0x00000100,
991 0x3c218, 0xffffffff, 0x06000100,
992 0x3c204, 0xffffffff, 0x00000100,
993 0x3c2e0, 0xffffffff, 0x00000100,
994 0x3c224, 0xffffffff, 0x00000100,
995 0x3c200, 0xffffffff, 0x00000100,
996 0x3c230, 0xffffffff, 0x00000100,
997 0x3c234, 0xffffffff, 0x00000100,
998 0x3c250, 0xffffffff, 0x00000100,
999 0x3c254, 0xffffffff, 0x00000100,
1000 0x3c258, 0xffffffff, 0x00000100,
1001 0x3c25c, 0xffffffff, 0x00000100,
1002 0x3c260, 0xffffffff, 0x00000100,
1003 0x3c27c, 0xffffffff, 0x00000100,
1004 0x3c278, 0xffffffff, 0x00000100,
1005 0x3c210, 0xffffffff, 0x06000100,
1006 0x3c290, 0xffffffff, 0x00000100,
1007 0x3c274, 0xffffffff, 0x00000100,
1008 0x3c2b4, 0xffffffff, 0x00000100,
1009 0x3c2b0, 0xffffffff, 0x00000100,
1010 0x3c270, 0xffffffff, 0x00000100,
1011 0x30800, 0xffffffff, 0xe0000000,
1012 0x3c020, 0xffffffff, 0x00010000,
1013 0x3c024, 0xffffffff, 0x00030002,
1014 0x3c028, 0xffffffff, 0x00040007,
1015 0x3c02c, 0xffffffff, 0x00060005,
1016 0x3c030, 0xffffffff, 0x00090008,
1017 0x3c034, 0xffffffff, 0x00010000,
1018 0x3c038, 0xffffffff, 0x00030002,
1019 0x3c03c, 0xffffffff, 0x00040007,
1020 0x3c040, 0xffffffff, 0x00060005,
1021 0x3c044, 0xffffffff, 0x00090008,
1022 0x3c048, 0xffffffff, 0x00010000,
1023 0x3c04c, 0xffffffff, 0x00030002,
1024 0x3c050, 0xffffffff, 0x00040007,
1025 0x3c054, 0xffffffff, 0x00060005,
1026 0x3c058, 0xffffffff, 0x00090008,
1027 0x3c05c, 0xffffffff, 0x00010000,
1028 0x3c060, 0xffffffff, 0x00030002,
1029 0x3c064, 0xffffffff, 0x00040007,
1030 0x3c068, 0xffffffff, 0x00060005,
1031 0x3c06c, 0xffffffff, 0x00090008,
1032 0x3c070, 0xffffffff, 0x00010000,
1033 0x3c074, 0xffffffff, 0x00030002,
1034 0x3c078, 0xffffffff, 0x00040007,
1035 0x3c07c, 0xffffffff, 0x00060005,
1036 0x3c080, 0xffffffff, 0x00090008,
1037 0x3c084, 0xffffffff, 0x00010000,
1038 0x3c088, 0xffffffff, 0x00030002,
1039 0x3c08c, 0xffffffff, 0x00040007,
1040 0x3c090, 0xffffffff, 0x00060005,
1041 0x3c094, 0xffffffff, 0x00090008,
1042 0x3c098, 0xffffffff, 0x00010000,
1043 0x3c09c, 0xffffffff, 0x00030002,
1044 0x3c0a0, 0xffffffff, 0x00040007,
1045 0x3c0a4, 0xffffffff, 0x00060005,
1046 0x3c0a8, 0xffffffff, 0x00090008,
1047 0x3c000, 0xffffffff, 0x96e00200,
1048 0x8708, 0xffffffff, 0x00900100,
1049 0xc424, 0xffffffff, 0x0020003f,
1050 0x38, 0xffffffff, 0x0140001c,
1051 0x3c, 0x000f0000, 0x000f0000,
1052 0x220, 0xffffffff, 0xC060000C,
1053 0x224, 0xc0000fff, 0x00000100,
1054 0xf90, 0xffffffff, 0x00000100,
1055 0xf98, 0x00000101, 0x00000000,
1056 0x20a8, 0xffffffff, 0x00000104,
1057 0x55e4, 0xff000fff, 0x00000100,
1058 0x30cc, 0xc0000fff, 0x00000104,
1059 0xc1e4, 0x00000001, 0x00000001,
1060 0xd00c, 0xff000ff0, 0x00000100,
1061 0xd80c, 0xff000ff0, 0x00000100
1064 static const u32 spectre_golden_spm_registers[] =
1066 0x30800, 0xe0ffffff, 0xe0000000
1069 static const u32 spectre_golden_common_registers[] =
1071 0xc770, 0xffffffff, 0x00000800,
1072 0xc774, 0xffffffff, 0x00000800,
1073 0xc798, 0xffffffff, 0x00007fbf,
1074 0xc79c, 0xffffffff, 0x00007faf
1077 static const u32 spectre_golden_registers[] =
1079 0x3c000, 0xffff1fff, 0x96940200,
1080 0x3c00c, 0xffff0001, 0xff000000,
1081 0x3c200, 0xfffc0fff, 0x00000100,
1082 0x6ed8, 0x00010101, 0x00010000,
1083 0x9834, 0xf00fffff, 0x00000400,
1084 0x9838, 0xfffffffc, 0x00020200,
1085 0x5bb0, 0x000000f0, 0x00000070,
1086 0x5bc0, 0xf0311fff, 0x80300000,
1087 0x98f8, 0x73773777, 0x12010001,
1088 0x9b7c, 0x00ff0000, 0x00fc0000,
1089 0x2f48, 0x73773777, 0x12010001,
1090 0x8a14, 0xf000003f, 0x00000007,
1091 0x8b24, 0xffffffff, 0x00ffffff,
1092 0x28350, 0x3f3f3fff, 0x00000082,
1093 0x28355, 0x0000003f, 0x00000000,
1094 0x3e78, 0x00000001, 0x00000002,
1095 0x913c, 0xffff03df, 0x00000004,
1096 0xc768, 0x00000008, 0x00000008,
1097 0x8c00, 0x000008ff, 0x00000800,
1098 0x9508, 0x00010000, 0x00010000,
1099 0xac0c, 0xffffffff, 0x54763210,
1100 0x214f8, 0x01ff01ff, 0x00000002,
1101 0x21498, 0x007ff800, 0x00200000,
1102 0x2015c, 0xffffffff, 0x00000f40,
1103 0x30934, 0xffffffff, 0x00000001
1106 static const u32 spectre_mgcg_cgcg_init[] =
1108 0xc420, 0xffffffff, 0xfffffffc,
1109 0x30800, 0xffffffff, 0xe0000000,
1110 0x3c2a0, 0xffffffff, 0x00000100,
1111 0x3c208, 0xffffffff, 0x00000100,
1112 0x3c2c0, 0xffffffff, 0x00000100,
1113 0x3c2c8, 0xffffffff, 0x00000100,
1114 0x3c2c4, 0xffffffff, 0x00000100,
1115 0x55e4, 0xffffffff, 0x00600100,
1116 0x3c280, 0xffffffff, 0x00000100,
1117 0x3c214, 0xffffffff, 0x06000100,
1118 0x3c220, 0xffffffff, 0x00000100,
1119 0x3c218, 0xffffffff, 0x06000100,
1120 0x3c204, 0xffffffff, 0x00000100,
1121 0x3c2e0, 0xffffffff, 0x00000100,
1122 0x3c224, 0xffffffff, 0x00000100,
1123 0x3c200, 0xffffffff, 0x00000100,
1124 0x3c230, 0xffffffff, 0x00000100,
1125 0x3c234, 0xffffffff, 0x00000100,
1126 0x3c250, 0xffffffff, 0x00000100,
1127 0x3c254, 0xffffffff, 0x00000100,
1128 0x3c258, 0xffffffff, 0x00000100,
1129 0x3c25c, 0xffffffff, 0x00000100,
1130 0x3c260, 0xffffffff, 0x00000100,
1131 0x3c27c, 0xffffffff, 0x00000100,
1132 0x3c278, 0xffffffff, 0x00000100,
1133 0x3c210, 0xffffffff, 0x06000100,
1134 0x3c290, 0xffffffff, 0x00000100,
1135 0x3c274, 0xffffffff, 0x00000100,
1136 0x3c2b4, 0xffffffff, 0x00000100,
1137 0x3c2b0, 0xffffffff, 0x00000100,
1138 0x3c270, 0xffffffff, 0x00000100,
1139 0x30800, 0xffffffff, 0xe0000000,
1140 0x3c020, 0xffffffff, 0x00010000,
1141 0x3c024, 0xffffffff, 0x00030002,
1142 0x3c028, 0xffffffff, 0x00040007,
1143 0x3c02c, 0xffffffff, 0x00060005,
1144 0x3c030, 0xffffffff, 0x00090008,
1145 0x3c034, 0xffffffff, 0x00010000,
1146 0x3c038, 0xffffffff, 0x00030002,
1147 0x3c03c, 0xffffffff, 0x00040007,
1148 0x3c040, 0xffffffff, 0x00060005,
1149 0x3c044, 0xffffffff, 0x00090008,
1150 0x3c048, 0xffffffff, 0x00010000,
1151 0x3c04c, 0xffffffff, 0x00030002,
1152 0x3c050, 0xffffffff, 0x00040007,
1153 0x3c054, 0xffffffff, 0x00060005,
1154 0x3c058, 0xffffffff, 0x00090008,
1155 0x3c05c, 0xffffffff, 0x00010000,
1156 0x3c060, 0xffffffff, 0x00030002,
1157 0x3c064, 0xffffffff, 0x00040007,
1158 0x3c068, 0xffffffff, 0x00060005,
1159 0x3c06c, 0xffffffff, 0x00090008,
1160 0x3c070, 0xffffffff, 0x00010000,
1161 0x3c074, 0xffffffff, 0x00030002,
1162 0x3c078, 0xffffffff, 0x00040007,
1163 0x3c07c, 0xffffffff, 0x00060005,
1164 0x3c080, 0xffffffff, 0x00090008,
1165 0x3c084, 0xffffffff, 0x00010000,
1166 0x3c088, 0xffffffff, 0x00030002,
1167 0x3c08c, 0xffffffff, 0x00040007,
1168 0x3c090, 0xffffffff, 0x00060005,
1169 0x3c094, 0xffffffff, 0x00090008,
1170 0x3c098, 0xffffffff, 0x00010000,
1171 0x3c09c, 0xffffffff, 0x00030002,
1172 0x3c0a0, 0xffffffff, 0x00040007,
1173 0x3c0a4, 0xffffffff, 0x00060005,
1174 0x3c0a8, 0xffffffff, 0x00090008,
1175 0x3c0ac, 0xffffffff, 0x00010000,
1176 0x3c0b0, 0xffffffff, 0x00030002,
1177 0x3c0b4, 0xffffffff, 0x00040007,
1178 0x3c0b8, 0xffffffff, 0x00060005,
1179 0x3c0bc, 0xffffffff, 0x00090008,
1180 0x3c000, 0xffffffff, 0x96e00200,
1181 0x8708, 0xffffffff, 0x00900100,
1182 0xc424, 0xffffffff, 0x0020003f,
1183 0x38, 0xffffffff, 0x0140001c,
1184 0x3c, 0x000f0000, 0x000f0000,
1185 0x220, 0xffffffff, 0xC060000C,
1186 0x224, 0xc0000fff, 0x00000100,
1187 0xf90, 0xffffffff, 0x00000100,
1188 0xf98, 0x00000101, 0x00000000,
1189 0x20a8, 0xffffffff, 0x00000104,
1190 0x55e4, 0xff000fff, 0x00000100,
1191 0x30cc, 0xc0000fff, 0x00000104,
1192 0xc1e4, 0x00000001, 0x00000001,
1193 0xd00c, 0xff000ff0, 0x00000100,
1194 0xd80c, 0xff000ff0, 0x00000100
1197 static const u32 kalindi_golden_spm_registers[] =
1199 0x30800, 0xe0ffffff, 0xe0000000
1202 static const u32 kalindi_golden_common_registers[] =
1204 0xc770, 0xffffffff, 0x00000800,
1205 0xc774, 0xffffffff, 0x00000800,
1206 0xc798, 0xffffffff, 0x00007fbf,
1207 0xc79c, 0xffffffff, 0x00007faf
1210 static const u32 kalindi_golden_registers[] =
1212 0x3c000, 0xffffdfff, 0x6e944040,
1213 0x55e4, 0xff607fff, 0xfc000100,
1214 0x3c220, 0xff000fff, 0x00000100,
1215 0x3c224, 0xff000fff, 0x00000100,
1216 0x3c200, 0xfffc0fff, 0x00000100,
1217 0x6ed8, 0x00010101, 0x00010000,
1218 0x9830, 0xffffffff, 0x00000000,
1219 0x9834, 0xf00fffff, 0x00000400,
1220 0x5bb0, 0x000000f0, 0x00000070,
1221 0x5bc0, 0xf0311fff, 0x80300000,
1222 0x98f8, 0x73773777, 0x12010001,
1223 0x98fc, 0xffffffff, 0x00000010,
1224 0x9b7c, 0x00ff0000, 0x00fc0000,
1225 0x8030, 0x00001f0f, 0x0000100a,
1226 0x2f48, 0x73773777, 0x12010001,
1227 0x2408, 0x000fffff, 0x000c007f,
1228 0x8a14, 0xf000003f, 0x00000007,
1229 0x8b24, 0x3fff3fff, 0x00ffcfff,
1230 0x30a04, 0x0000ff0f, 0x00000000,
1231 0x28a4c, 0x07ffffff, 0x06000000,
1232 0x4d8, 0x00000fff, 0x00000100,
1233 0x3e78, 0x00000001, 0x00000002,
1234 0xc768, 0x00000008, 0x00000008,
1235 0x8c00, 0x000000ff, 0x00000003,
1236 0x214f8, 0x01ff01ff, 0x00000002,
1237 0x21498, 0x007ff800, 0x00200000,
1238 0x2015c, 0xffffffff, 0x00000f40,
1239 0x88c4, 0x001f3ae3, 0x00000082,
1240 0x88d4, 0x0000001f, 0x00000010,
1241 0x30934, 0xffffffff, 0x00000000
1244 static const u32 kalindi_mgcg_cgcg_init[] =
1246 0xc420, 0xffffffff, 0xfffffffc,
1247 0x30800, 0xffffffff, 0xe0000000,
1248 0x3c2a0, 0xffffffff, 0x00000100,
1249 0x3c208, 0xffffffff, 0x00000100,
1250 0x3c2c0, 0xffffffff, 0x00000100,
1251 0x3c2c8, 0xffffffff, 0x00000100,
1252 0x3c2c4, 0xffffffff, 0x00000100,
1253 0x55e4, 0xffffffff, 0x00600100,
1254 0x3c280, 0xffffffff, 0x00000100,
1255 0x3c214, 0xffffffff, 0x06000100,
1256 0x3c220, 0xffffffff, 0x00000100,
1257 0x3c218, 0xffffffff, 0x06000100,
1258 0x3c204, 0xffffffff, 0x00000100,
1259 0x3c2e0, 0xffffffff, 0x00000100,
1260 0x3c224, 0xffffffff, 0x00000100,
1261 0x3c200, 0xffffffff, 0x00000100,
1262 0x3c230, 0xffffffff, 0x00000100,
1263 0x3c234, 0xffffffff, 0x00000100,
1264 0x3c250, 0xffffffff, 0x00000100,
1265 0x3c254, 0xffffffff, 0x00000100,
1266 0x3c258, 0xffffffff, 0x00000100,
1267 0x3c25c, 0xffffffff, 0x00000100,
1268 0x3c260, 0xffffffff, 0x00000100,
1269 0x3c27c, 0xffffffff, 0x00000100,
1270 0x3c278, 0xffffffff, 0x00000100,
1271 0x3c210, 0xffffffff, 0x06000100,
1272 0x3c290, 0xffffffff, 0x00000100,
1273 0x3c274, 0xffffffff, 0x00000100,
1274 0x3c2b4, 0xffffffff, 0x00000100,
1275 0x3c2b0, 0xffffffff, 0x00000100,
1276 0x3c270, 0xffffffff, 0x00000100,
1277 0x30800, 0xffffffff, 0xe0000000,
1278 0x3c020, 0xffffffff, 0x00010000,
1279 0x3c024, 0xffffffff, 0x00030002,
1280 0x3c028, 0xffffffff, 0x00040007,
1281 0x3c02c, 0xffffffff, 0x00060005,
1282 0x3c030, 0xffffffff, 0x00090008,
1283 0x3c034, 0xffffffff, 0x00010000,
1284 0x3c038, 0xffffffff, 0x00030002,
1285 0x3c03c, 0xffffffff, 0x00040007,
1286 0x3c040, 0xffffffff, 0x00060005,
1287 0x3c044, 0xffffffff, 0x00090008,
1288 0x3c000, 0xffffffff, 0x96e00200,
1289 0x8708, 0xffffffff, 0x00900100,
1290 0xc424, 0xffffffff, 0x0020003f,
1291 0x38, 0xffffffff, 0x0140001c,
1292 0x3c, 0x000f0000, 0x000f0000,
1293 0x220, 0xffffffff, 0xC060000C,
1294 0x224, 0xc0000fff, 0x00000100,
1295 0x20a8, 0xffffffff, 0x00000104,
1296 0x55e4, 0xff000fff, 0x00000100,
1297 0x30cc, 0xc0000fff, 0x00000104,
1298 0xc1e4, 0x00000001, 0x00000001,
1299 0xd00c, 0xff000ff0, 0x00000100,
1300 0xd80c, 0xff000ff0, 0x00000100
1303 static void cik_init_golden_registers(struct radeon_device *rdev)
1305 switch (rdev->family) {
1307 radeon_program_register_sequence(rdev,
1308 bonaire_mgcg_cgcg_init,
1309 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1310 radeon_program_register_sequence(rdev,
1311 bonaire_golden_registers,
1312 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1313 radeon_program_register_sequence(rdev,
1314 bonaire_golden_common_registers,
1315 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1316 radeon_program_register_sequence(rdev,
1317 bonaire_golden_spm_registers,
1318 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1321 radeon_program_register_sequence(rdev,
1322 kalindi_mgcg_cgcg_init,
1323 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1324 radeon_program_register_sequence(rdev,
1325 kalindi_golden_registers,
1326 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1327 radeon_program_register_sequence(rdev,
1328 kalindi_golden_common_registers,
1329 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1330 radeon_program_register_sequence(rdev,
1331 kalindi_golden_spm_registers,
1332 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1335 radeon_program_register_sequence(rdev,
1336 spectre_mgcg_cgcg_init,
1337 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1338 radeon_program_register_sequence(rdev,
1339 spectre_golden_registers,
1340 (const u32)ARRAY_SIZE(spectre_golden_registers));
1341 radeon_program_register_sequence(rdev,
1342 spectre_golden_common_registers,
1343 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1344 radeon_program_register_sequence(rdev,
1345 spectre_golden_spm_registers,
1346 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1354 * cik_get_xclk - get the xclk
1356 * @rdev: radeon_device pointer
1358 * Returns the reference clock used by the gfx engine
1361 u32 cik_get_xclk(struct radeon_device *rdev)
1363 u32 reference_clock = rdev->clock.spll.reference_freq;
1365 if (rdev->flags & RADEON_IS_IGP) {
1366 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1367 return reference_clock / 2;
1369 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1370 return reference_clock / 4;
1372 return reference_clock;
1376 * cik_mm_rdoorbell - read a doorbell dword
1378 * @rdev: radeon_device pointer
1379 * @offset: byte offset into the aperture
1381 * Returns the value in the doorbell aperture at the
1382 * requested offset (CIK).
1384 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 offset)
1386 if (offset < rdev->doorbell.size) {
1387 return readl(((void __iomem *)rdev->doorbell.ptr) + offset);
1389 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", offset);
1395 * cik_mm_wdoorbell - write a doorbell dword
1397 * @rdev: radeon_device pointer
1398 * @offset: byte offset into the aperture
1399 * @v: value to write
1401 * Writes @v to the doorbell aperture at the
1402 * requested offset (CIK).
1404 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 offset, u32 v)
1406 if (offset < rdev->doorbell.size) {
1407 writel(v, ((void __iomem *)rdev->doorbell.ptr) + offset);
1409 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", offset);
1413 #define BONAIRE_IO_MC_REGS_SIZE 36
1415 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1417 {0x00000070, 0x04400000},
1418 {0x00000071, 0x80c01803},
1419 {0x00000072, 0x00004004},
1420 {0x00000073, 0x00000100},
1421 {0x00000074, 0x00ff0000},
1422 {0x00000075, 0x34000000},
1423 {0x00000076, 0x08000014},
1424 {0x00000077, 0x00cc08ec},
1425 {0x00000078, 0x00000400},
1426 {0x00000079, 0x00000000},
1427 {0x0000007a, 0x04090000},
1428 {0x0000007c, 0x00000000},
1429 {0x0000007e, 0x4408a8e8},
1430 {0x0000007f, 0x00000304},
1431 {0x00000080, 0x00000000},
1432 {0x00000082, 0x00000001},
1433 {0x00000083, 0x00000002},
1434 {0x00000084, 0xf3e4f400},
1435 {0x00000085, 0x052024e3},
1436 {0x00000087, 0x00000000},
1437 {0x00000088, 0x01000000},
1438 {0x0000008a, 0x1c0a0000},
1439 {0x0000008b, 0xff010000},
1440 {0x0000008d, 0xffffefff},
1441 {0x0000008e, 0xfff3efff},
1442 {0x0000008f, 0xfff3efbf},
1443 {0x00000092, 0xf7ffffff},
1444 {0x00000093, 0xffffff7f},
1445 {0x00000095, 0x00101101},
1446 {0x00000096, 0x00000fff},
1447 {0x00000097, 0x00116fff},
1448 {0x00000098, 0x60010000},
1449 {0x00000099, 0x10010000},
1450 {0x0000009a, 0x00006000},
1451 {0x0000009b, 0x00001000},
1452 {0x0000009f, 0x00b48000}
1456 * cik_srbm_select - select specific register instances
1458 * @rdev: radeon_device pointer
1459 * @me: selected ME (micro engine)
1464 * Switches the currently active registers instances. Some
1465 * registers are instanced per VMID, others are instanced per
1466 * me/pipe/queue combination.
1468 static void cik_srbm_select(struct radeon_device *rdev,
1469 u32 me, u32 pipe, u32 queue, u32 vmid)
1471 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1474 QUEUEID(queue & 0x7));
1475 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1480 * ci_mc_load_microcode - load MC ucode into the hw
1482 * @rdev: radeon_device pointer
1484 * Load the GDDR MC ucode into the hw (CIK).
1485 * Returns 0 on success, error on failure.
1487 static int ci_mc_load_microcode(struct radeon_device *rdev)
1489 const __be32 *fw_data;
1490 u32 running, blackout = 0;
1492 int i, ucode_size, regs_size;
1497 switch (rdev->family) {
1500 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1501 ucode_size = CIK_MC_UCODE_SIZE;
1502 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1506 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1510 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1511 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1514 /* reset the engine and set to writable */
1515 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1516 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1518 /* load mc io regs */
1519 for (i = 0; i < regs_size; i++) {
1520 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1521 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1523 /* load the MC ucode */
1524 fw_data = (const __be32 *)rdev->mc_fw->data;
1525 for (i = 0; i < ucode_size; i++)
1526 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1528 /* put the engine back into the active state */
1529 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1530 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1531 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1533 /* wait for training to complete */
1534 for (i = 0; i < rdev->usec_timeout; i++) {
1535 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1539 for (i = 0; i < rdev->usec_timeout; i++) {
1540 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1546 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1553 * cik_init_microcode - load ucode images from disk
1555 * @rdev: radeon_device pointer
1557 * Use the firmware interface to load the ucode images into
1558 * the driver (not loaded into hw).
1559 * Returns 0 on success, error on failure.
1561 static int cik_init_microcode(struct radeon_device *rdev)
1563 const char *chip_name;
1564 size_t pfp_req_size, me_req_size, ce_req_size,
1565 mec_req_size, rlc_req_size, mc_req_size,
1566 sdma_req_size, smc_req_size;
1572 switch (rdev->family) {
1574 chip_name = "BONAIRE";
1575 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1576 me_req_size = CIK_ME_UCODE_SIZE * 4;
1577 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1578 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1579 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1580 mc_req_size = CIK_MC_UCODE_SIZE * 4;
1581 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1582 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1585 chip_name = "KAVERI";
1586 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1587 me_req_size = CIK_ME_UCODE_SIZE * 4;
1588 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1589 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1590 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1591 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1594 chip_name = "KABINI";
1595 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1596 me_req_size = CIK_ME_UCODE_SIZE * 4;
1597 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1598 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1599 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1600 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1605 DRM_INFO("Loading %s Microcode\n", chip_name);
1607 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1608 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1611 if (rdev->pfp_fw->size != pfp_req_size) {
1613 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1614 rdev->pfp_fw->size, fw_name);
1619 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1620 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1623 if (rdev->me_fw->size != me_req_size) {
1625 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1626 rdev->me_fw->size, fw_name);
1630 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1631 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1634 if (rdev->ce_fw->size != ce_req_size) {
1636 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1637 rdev->ce_fw->size, fw_name);
1641 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
1642 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
1645 if (rdev->mec_fw->size != mec_req_size) {
1647 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1648 rdev->mec_fw->size, fw_name);
1652 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1653 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1656 if (rdev->rlc_fw->size != rlc_req_size) {
1658 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1659 rdev->rlc_fw->size, fw_name);
1663 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
1664 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
1667 if (rdev->sdma_fw->size != sdma_req_size) {
1669 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1670 rdev->sdma_fw->size, fw_name);
1674 /* No SMC, MC ucode on APUs */
1675 if (!(rdev->flags & RADEON_IS_IGP)) {
1676 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1677 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1680 if (rdev->mc_fw->size != mc_req_size) {
1682 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1683 rdev->mc_fw->size, fw_name);
1687 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1688 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1691 "smc: error loading firmware \"%s\"\n",
1693 release_firmware(rdev->smc_fw);
1694 rdev->smc_fw = NULL;
1695 } else if (rdev->smc_fw->size != smc_req_size) {
1697 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1698 rdev->smc_fw->size, fw_name);
1707 "cik_cp: Failed to load firmware \"%s\"\n",
1709 release_firmware(rdev->pfp_fw);
1710 rdev->pfp_fw = NULL;
1711 release_firmware(rdev->me_fw);
1713 release_firmware(rdev->ce_fw);
1715 release_firmware(rdev->rlc_fw);
1716 rdev->rlc_fw = NULL;
1717 release_firmware(rdev->mc_fw);
1719 release_firmware(rdev->smc_fw);
1720 rdev->smc_fw = NULL;
1729 * cik_tiling_mode_table_init - init the hw tiling table
1731 * @rdev: radeon_device pointer
1733 * Starting with SI, the tiling setup is done globally in a
1734 * set of 32 tiling modes. Rather than selecting each set of
1735 * parameters per surface as on older asics, we just select
1736 * which index in the tiling table we want to use, and the
1737 * surface uses those parameters (CIK).
1739 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1741 const u32 num_tile_mode_states = 32;
1742 const u32 num_secondary_tile_mode_states = 16;
1743 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1744 u32 num_pipe_configs;
1745 u32 num_rbs = rdev->config.cik.max_backends_per_se *
1746 rdev->config.cik.max_shader_engines;
1748 switch (rdev->config.cik.mem_row_size_in_kb) {
1750 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1754 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1757 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1761 num_pipe_configs = rdev->config.cik.max_tile_pipes;
1762 if (num_pipe_configs > 8)
1763 num_pipe_configs = 8; /* ??? */
1765 if (num_pipe_configs == 8) {
1766 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1767 switch (reg_offset) {
1769 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1770 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1771 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1772 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
1775 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1776 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1777 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1778 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
1781 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1782 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1783 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1784 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1787 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1788 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1789 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1790 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
1793 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1794 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1795 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1796 TILE_SPLIT(split_equal_to_row_size));
1799 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1800 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1803 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1804 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1805 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1806 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1809 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1810 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1811 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1812 TILE_SPLIT(split_equal_to_row_size));
1815 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1816 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
1819 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1820 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1823 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1824 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1825 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1826 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1829 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1830 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1831 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1832 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1835 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1836 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1837 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1838 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1841 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1842 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1845 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1846 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1847 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1848 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1851 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1852 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1853 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1854 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1857 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1858 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1859 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1860 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1863 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1864 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1867 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1868 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1869 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1870 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1873 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1874 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1875 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1876 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1879 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1880 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1881 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1882 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1888 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
1889 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1891 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
1892 switch (reg_offset) {
1894 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1895 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1896 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1897 NUM_BANKS(ADDR_SURF_16_BANK));
1900 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1901 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1902 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1903 NUM_BANKS(ADDR_SURF_16_BANK));
1906 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1907 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1908 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1909 NUM_BANKS(ADDR_SURF_16_BANK));
1912 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1913 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1914 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1915 NUM_BANKS(ADDR_SURF_16_BANK));
1918 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1919 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1920 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1921 NUM_BANKS(ADDR_SURF_8_BANK));
1924 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1925 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1926 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1927 NUM_BANKS(ADDR_SURF_4_BANK));
1930 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1931 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1932 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1933 NUM_BANKS(ADDR_SURF_2_BANK));
1936 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1937 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
1938 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1939 NUM_BANKS(ADDR_SURF_16_BANK));
1942 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1943 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1944 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1945 NUM_BANKS(ADDR_SURF_16_BANK));
1948 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1949 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1950 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1951 NUM_BANKS(ADDR_SURF_16_BANK));
1954 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1955 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1956 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1957 NUM_BANKS(ADDR_SURF_16_BANK));
1960 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1961 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1962 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1963 NUM_BANKS(ADDR_SURF_8_BANK));
1966 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1967 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1968 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1969 NUM_BANKS(ADDR_SURF_4_BANK));
1972 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1973 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1974 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1975 NUM_BANKS(ADDR_SURF_2_BANK));
1981 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1983 } else if (num_pipe_configs == 4) {
1985 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1986 switch (reg_offset) {
1988 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1989 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1990 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1991 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
1994 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1995 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1996 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1997 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2000 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2001 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2002 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2003 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2006 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2007 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2008 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2009 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2012 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2013 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2014 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2015 TILE_SPLIT(split_equal_to_row_size));
2018 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2019 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2022 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2023 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2024 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2025 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2028 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2029 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2030 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2031 TILE_SPLIT(split_equal_to_row_size));
2034 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2035 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2038 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2039 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2042 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2043 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2044 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2045 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2048 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2049 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2050 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2051 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2054 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2055 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2056 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2057 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2060 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2061 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2064 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2065 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2066 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2067 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2070 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2071 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2072 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2073 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2076 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2077 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2078 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2079 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2082 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2083 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2086 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2087 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2088 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2089 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2092 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2093 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2094 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2095 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2098 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2099 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2100 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2101 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2107 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2108 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2110 } else if (num_rbs < 4) {
2111 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2112 switch (reg_offset) {
2114 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2115 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2116 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2117 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2120 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2121 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2122 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2123 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2126 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2127 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2128 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2129 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2132 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2133 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2134 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2135 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2138 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2139 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2140 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2141 TILE_SPLIT(split_equal_to_row_size));
2144 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2145 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2148 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2149 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2150 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2151 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2154 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2155 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2156 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2157 TILE_SPLIT(split_equal_to_row_size));
2160 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2161 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2164 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2165 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2168 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2169 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2170 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2171 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2174 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2175 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2176 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2177 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2180 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2181 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2182 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2183 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2186 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2187 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2190 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2191 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2192 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2193 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2196 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2197 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2198 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2199 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2202 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2203 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2204 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2205 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2208 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2209 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2212 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2213 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2214 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2215 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2218 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2219 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2220 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2221 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2224 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2225 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2226 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2227 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2233 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2234 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2237 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2238 switch (reg_offset) {
2240 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2241 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2242 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2243 NUM_BANKS(ADDR_SURF_16_BANK));
2246 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2247 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2248 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2249 NUM_BANKS(ADDR_SURF_16_BANK));
2252 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2253 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2254 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2255 NUM_BANKS(ADDR_SURF_16_BANK));
2258 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2259 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2260 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2261 NUM_BANKS(ADDR_SURF_16_BANK));
2264 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2265 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2266 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2267 NUM_BANKS(ADDR_SURF_16_BANK));
2270 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2271 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2272 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2273 NUM_BANKS(ADDR_SURF_8_BANK));
2276 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2277 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2278 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2279 NUM_BANKS(ADDR_SURF_4_BANK));
2282 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2283 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2284 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2285 NUM_BANKS(ADDR_SURF_16_BANK));
2288 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2289 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2290 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2291 NUM_BANKS(ADDR_SURF_16_BANK));
2294 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2295 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2296 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2297 NUM_BANKS(ADDR_SURF_16_BANK));
2300 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2301 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2302 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2303 NUM_BANKS(ADDR_SURF_16_BANK));
2306 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2307 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2308 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2309 NUM_BANKS(ADDR_SURF_16_BANK));
2312 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2313 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2314 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2315 NUM_BANKS(ADDR_SURF_8_BANK));
2318 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2319 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2320 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2321 NUM_BANKS(ADDR_SURF_4_BANK));
2327 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2329 } else if (num_pipe_configs == 2) {
2330 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2331 switch (reg_offset) {
2333 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2334 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2335 PIPE_CONFIG(ADDR_SURF_P2) |
2336 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2339 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2340 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2341 PIPE_CONFIG(ADDR_SURF_P2) |
2342 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2345 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2346 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2347 PIPE_CONFIG(ADDR_SURF_P2) |
2348 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2351 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2352 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2353 PIPE_CONFIG(ADDR_SURF_P2) |
2354 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2357 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2358 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2359 PIPE_CONFIG(ADDR_SURF_P2) |
2360 TILE_SPLIT(split_equal_to_row_size));
2363 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2364 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2367 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2368 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2369 PIPE_CONFIG(ADDR_SURF_P2) |
2370 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2373 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2374 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2375 PIPE_CONFIG(ADDR_SURF_P2) |
2376 TILE_SPLIT(split_equal_to_row_size));
2379 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
2382 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2383 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2386 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2387 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2388 PIPE_CONFIG(ADDR_SURF_P2) |
2389 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2392 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2393 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2394 PIPE_CONFIG(ADDR_SURF_P2) |
2395 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2398 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2399 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2400 PIPE_CONFIG(ADDR_SURF_P2) |
2401 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2404 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2405 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2408 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2409 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2410 PIPE_CONFIG(ADDR_SURF_P2) |
2411 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2414 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2415 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2416 PIPE_CONFIG(ADDR_SURF_P2) |
2417 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2420 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2421 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2422 PIPE_CONFIG(ADDR_SURF_P2) |
2423 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2426 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2427 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2430 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2431 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2432 PIPE_CONFIG(ADDR_SURF_P2) |
2433 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2436 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2437 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2438 PIPE_CONFIG(ADDR_SURF_P2) |
2439 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2442 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2443 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2444 PIPE_CONFIG(ADDR_SURF_P2) |
2445 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2451 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2452 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2454 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2455 switch (reg_offset) {
2457 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2458 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2459 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2460 NUM_BANKS(ADDR_SURF_16_BANK));
2463 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2464 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2465 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2466 NUM_BANKS(ADDR_SURF_16_BANK));
2469 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2470 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2471 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2472 NUM_BANKS(ADDR_SURF_16_BANK));
2475 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2476 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2477 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2478 NUM_BANKS(ADDR_SURF_16_BANK));
2481 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2482 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2483 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2484 NUM_BANKS(ADDR_SURF_16_BANK));
2487 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2488 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2489 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2490 NUM_BANKS(ADDR_SURF_16_BANK));
2493 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2494 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2495 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2496 NUM_BANKS(ADDR_SURF_8_BANK));
2499 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2500 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2501 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2502 NUM_BANKS(ADDR_SURF_16_BANK));
2505 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2506 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2507 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2508 NUM_BANKS(ADDR_SURF_16_BANK));
2511 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2512 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2513 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2514 NUM_BANKS(ADDR_SURF_16_BANK));
2517 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2518 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2519 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2520 NUM_BANKS(ADDR_SURF_16_BANK));
2523 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2524 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2525 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2526 NUM_BANKS(ADDR_SURF_16_BANK));
2529 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2530 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2531 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2532 NUM_BANKS(ADDR_SURF_16_BANK));
2535 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2536 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2537 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2538 NUM_BANKS(ADDR_SURF_8_BANK));
2544 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2547 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
2551 * cik_select_se_sh - select which SE, SH to address
2553 * @rdev: radeon_device pointer
2554 * @se_num: shader engine to address
2555 * @sh_num: sh block to address
2557 * Select which SE, SH combinations to address. Certain
2558 * registers are instanced per SE or SH. 0xffffffff means
2559 * broadcast to all SEs or SHs (CIK).
2561 static void cik_select_se_sh(struct radeon_device *rdev,
2562 u32 se_num, u32 sh_num)
2564 u32 data = INSTANCE_BROADCAST_WRITES;
2566 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2567 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2568 else if (se_num == 0xffffffff)
2569 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2570 else if (sh_num == 0xffffffff)
2571 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2573 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2574 WREG32(GRBM_GFX_INDEX, data);
2578 * cik_create_bitmask - create a bitmask
2580 * @bit_width: length of the mask
2582 * create a variable length bit mask (CIK).
2583 * Returns the bitmask.
2585 static u32 cik_create_bitmask(u32 bit_width)
2589 for (i = 0; i < bit_width; i++) {
2597 * cik_select_se_sh - select which SE, SH to address
2599 * @rdev: radeon_device pointer
2600 * @max_rb_num: max RBs (render backends) for the asic
2601 * @se_num: number of SEs (shader engines) for the asic
2602 * @sh_per_se: number of SH blocks per SE for the asic
2604 * Calculates the bitmask of disabled RBs (CIK).
2605 * Returns the disabled RB bitmask.
2607 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
2608 u32 max_rb_num, u32 se_num,
2613 data = RREG32(CC_RB_BACKEND_DISABLE);
2615 data &= BACKEND_DISABLE_MASK;
2618 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2620 data >>= BACKEND_DISABLE_SHIFT;
2622 mask = cik_create_bitmask(max_rb_num / se_num / sh_per_se);
2628 * cik_setup_rb - setup the RBs on the asic
2630 * @rdev: radeon_device pointer
2631 * @se_num: number of SEs (shader engines) for the asic
2632 * @sh_per_se: number of SH blocks per SE for the asic
2633 * @max_rb_num: max RBs (render backends) for the asic
2635 * Configures per-SE/SH RB registers (CIK).
2637 static void cik_setup_rb(struct radeon_device *rdev,
2638 u32 se_num, u32 sh_per_se,
2643 u32 disabled_rbs = 0;
2644 u32 enabled_rbs = 0;
2646 for (i = 0; i < se_num; i++) {
2647 for (j = 0; j < sh_per_se; j++) {
2648 cik_select_se_sh(rdev, i, j);
2649 data = cik_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2650 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
2653 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2656 for (i = 0; i < max_rb_num; i++) {
2657 if (!(disabled_rbs & mask))
2658 enabled_rbs |= mask;
2662 for (i = 0; i < se_num; i++) {
2663 cik_select_se_sh(rdev, i, 0xffffffff);
2665 for (j = 0; j < sh_per_se; j++) {
2666 switch (enabled_rbs & 3) {
2668 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2671 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2675 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2680 WREG32(PA_SC_RASTER_CONFIG, data);
2682 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2686 * cik_gpu_init - setup the 3D engine
2688 * @rdev: radeon_device pointer
2690 * Configures the 3D engine and tiling configuration
2691 * registers so that the 3D engine is usable.
2693 static void cik_gpu_init(struct radeon_device *rdev)
2695 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
2696 u32 mc_shared_chmap, mc_arb_ramcfg;
2697 u32 hdp_host_path_cntl;
2701 switch (rdev->family) {
2703 rdev->config.cik.max_shader_engines = 2;
2704 rdev->config.cik.max_tile_pipes = 4;
2705 rdev->config.cik.max_cu_per_sh = 7;
2706 rdev->config.cik.max_sh_per_se = 1;
2707 rdev->config.cik.max_backends_per_se = 2;
2708 rdev->config.cik.max_texture_channel_caches = 4;
2709 rdev->config.cik.max_gprs = 256;
2710 rdev->config.cik.max_gs_threads = 32;
2711 rdev->config.cik.max_hw_contexts = 8;
2713 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
2714 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
2715 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
2716 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
2717 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
2720 rdev->config.cik.max_shader_engines = 1;
2721 rdev->config.cik.max_tile_pipes = 4;
2722 if ((rdev->pdev->device == 0x1304) ||
2723 (rdev->pdev->device == 0x1305) ||
2724 (rdev->pdev->device == 0x130C) ||
2725 (rdev->pdev->device == 0x130F) ||
2726 (rdev->pdev->device == 0x1310) ||
2727 (rdev->pdev->device == 0x1311) ||
2728 (rdev->pdev->device == 0x131C)) {
2729 rdev->config.cik.max_cu_per_sh = 8;
2730 rdev->config.cik.max_backends_per_se = 2;
2731 } else if ((rdev->pdev->device == 0x1309) ||
2732 (rdev->pdev->device == 0x130A) ||
2733 (rdev->pdev->device == 0x130D) ||
2734 (rdev->pdev->device == 0x1313) ||
2735 (rdev->pdev->device == 0x131D)) {
2736 rdev->config.cik.max_cu_per_sh = 6;
2737 rdev->config.cik.max_backends_per_se = 2;
2738 } else if ((rdev->pdev->device == 0x1306) ||
2739 (rdev->pdev->device == 0x1307) ||
2740 (rdev->pdev->device == 0x130B) ||
2741 (rdev->pdev->device == 0x130E) ||
2742 (rdev->pdev->device == 0x1315) ||
2743 (rdev->pdev->device == 0x131B)) {
2744 rdev->config.cik.max_cu_per_sh = 4;
2745 rdev->config.cik.max_backends_per_se = 1;
2747 rdev->config.cik.max_cu_per_sh = 3;
2748 rdev->config.cik.max_backends_per_se = 1;
2750 rdev->config.cik.max_sh_per_se = 1;
2751 rdev->config.cik.max_texture_channel_caches = 4;
2752 rdev->config.cik.max_gprs = 256;
2753 rdev->config.cik.max_gs_threads = 16;
2754 rdev->config.cik.max_hw_contexts = 8;
2756 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
2757 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
2758 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
2759 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
2760 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
2764 rdev->config.cik.max_shader_engines = 1;
2765 rdev->config.cik.max_tile_pipes = 2;
2766 rdev->config.cik.max_cu_per_sh = 2;
2767 rdev->config.cik.max_sh_per_se = 1;
2768 rdev->config.cik.max_backends_per_se = 1;
2769 rdev->config.cik.max_texture_channel_caches = 2;
2770 rdev->config.cik.max_gprs = 256;
2771 rdev->config.cik.max_gs_threads = 16;
2772 rdev->config.cik.max_hw_contexts = 8;
2774 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
2775 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
2776 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
2777 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
2778 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
2782 /* Initialize HDP */
2783 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2784 WREG32((0x2c14 + j), 0x00000000);
2785 WREG32((0x2c18 + j), 0x00000000);
2786 WREG32((0x2c1c + j), 0x00000000);
2787 WREG32((0x2c20 + j), 0x00000000);
2788 WREG32((0x2c24 + j), 0x00000000);
2791 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2793 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2795 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2796 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2798 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
2799 rdev->config.cik.mem_max_burst_length_bytes = 256;
2800 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
2801 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
2802 if (rdev->config.cik.mem_row_size_in_kb > 4)
2803 rdev->config.cik.mem_row_size_in_kb = 4;
2804 /* XXX use MC settings? */
2805 rdev->config.cik.shader_engine_tile_size = 32;
2806 rdev->config.cik.num_gpus = 1;
2807 rdev->config.cik.multi_gpu_tile_size = 64;
2809 /* fix up row size */
2810 gb_addr_config &= ~ROW_SIZE_MASK;
2811 switch (rdev->config.cik.mem_row_size_in_kb) {
2814 gb_addr_config |= ROW_SIZE(0);
2817 gb_addr_config |= ROW_SIZE(1);
2820 gb_addr_config |= ROW_SIZE(2);
2824 /* setup tiling info dword. gb_addr_config is not adequate since it does
2825 * not have bank info, so create a custom tiling dword.
2826 * bits 3:0 num_pipes
2827 * bits 7:4 num_banks
2828 * bits 11:8 group_size
2829 * bits 15:12 row_size
2831 rdev->config.cik.tile_config = 0;
2832 switch (rdev->config.cik.num_tile_pipes) {
2834 rdev->config.cik.tile_config |= (0 << 0);
2837 rdev->config.cik.tile_config |= (1 << 0);
2840 rdev->config.cik.tile_config |= (2 << 0);
2844 /* XXX what about 12? */
2845 rdev->config.cik.tile_config |= (3 << 0);
2848 if ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT)
2849 rdev->config.cik.tile_config |= 1 << 4;
2851 rdev->config.cik.tile_config |= 0 << 4;
2852 rdev->config.cik.tile_config |=
2853 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
2854 rdev->config.cik.tile_config |=
2855 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
2857 WREG32(GB_ADDR_CONFIG, gb_addr_config);
2858 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
2859 WREG32(DMIF_ADDR_CALC, gb_addr_config);
2860 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
2861 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
2862 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
2863 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
2864 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
2866 cik_tiling_mode_table_init(rdev);
2868 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
2869 rdev->config.cik.max_sh_per_se,
2870 rdev->config.cik.max_backends_per_se);
2872 /* set HW defaults for 3D engine */
2873 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
2875 WREG32(SX_DEBUG_1, 0x20);
2877 WREG32(TA_CNTL_AUX, 0x00010000);
2879 tmp = RREG32(SPI_CONFIG_CNTL);
2881 WREG32(SPI_CONFIG_CNTL, tmp);
2883 WREG32(SQ_CONFIG, 1);
2885 WREG32(DB_DEBUG, 0);
2887 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
2889 WREG32(DB_DEBUG2, tmp);
2891 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
2893 WREG32(DB_DEBUG3, tmp);
2895 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
2897 WREG32(CB_HW_CONTROL, tmp);
2899 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2901 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
2902 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
2903 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
2904 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
2906 WREG32(VGT_NUM_INSTANCES, 1);
2908 WREG32(CP_PERFMON_CNTL, 0);
2910 WREG32(SQ_CONFIG, 0);
2912 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2913 FORCE_EOV_MAX_REZ_CNT(255)));
2915 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
2916 AUTO_INVLD_EN(ES_AND_GS_AUTO));
2918 WREG32(VGT_GS_VERTEX_REUSE, 16);
2919 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2921 tmp = RREG32(HDP_MISC_CNTL);
2922 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2923 WREG32(HDP_MISC_CNTL, tmp);
2925 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2926 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2928 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2929 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
2935 * GPU scratch registers helpers function.
2938 * cik_scratch_init - setup driver info for CP scratch regs
2940 * @rdev: radeon_device pointer
2942 * Set up the number and offset of the CP scratch registers.
2943 * NOTE: use of CP scratch registers is a legacy inferface and
2944 * is not used by default on newer asics (r6xx+). On newer asics,
2945 * memory buffers are used for fences rather than scratch regs.
2947 static void cik_scratch_init(struct radeon_device *rdev)
2951 rdev->scratch.num_reg = 7;
2952 rdev->scratch.reg_base = SCRATCH_REG0;
2953 for (i = 0; i < rdev->scratch.num_reg; i++) {
2954 rdev->scratch.free[i] = true;
2955 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2960 * cik_ring_test - basic gfx ring test
2962 * @rdev: radeon_device pointer
2963 * @ring: radeon_ring structure holding ring information
2965 * Allocate a scratch register and write to it using the gfx ring (CIK).
2966 * Provides a basic gfx ring test to verify that the ring is working.
2967 * Used by cik_cp_gfx_resume();
2968 * Returns 0 on success, error on failure.
2970 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
2977 r = radeon_scratch_get(rdev, &scratch);
2979 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
2982 WREG32(scratch, 0xCAFEDEAD);
2983 r = radeon_ring_lock(rdev, ring, 3);
2985 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
2986 radeon_scratch_free(rdev, scratch);
2989 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
2990 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
2991 radeon_ring_write(ring, 0xDEADBEEF);
2992 radeon_ring_unlock_commit(rdev, ring);
2994 for (i = 0; i < rdev->usec_timeout; i++) {
2995 tmp = RREG32(scratch);
2996 if (tmp == 0xDEADBEEF)
3000 if (i < rdev->usec_timeout) {
3001 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3003 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3004 ring->idx, scratch, tmp);
3007 radeon_scratch_free(rdev, scratch);
3012 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3014 * @rdev: radeon_device pointer
3015 * @fence: radeon fence object
3017 * Emits a fence sequnce number on the gfx ring and flushes
3020 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3021 struct radeon_fence *fence)
3023 struct radeon_ring *ring = &rdev->ring[fence->ring];
3024 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3026 /* EVENT_WRITE_EOP - flush caches, send int */
3027 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3028 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3030 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3032 radeon_ring_write(ring, addr & 0xfffffffc);
3033 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3034 radeon_ring_write(ring, fence->seq);
3035 radeon_ring_write(ring, 0);
3037 /* We should be using the new WAIT_REG_MEM special op packet here
3038 * but it causes the CP to hang
3040 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3041 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3042 WRITE_DATA_DST_SEL(0)));
3043 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3044 radeon_ring_write(ring, 0);
3045 radeon_ring_write(ring, 0);
3049 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3051 * @rdev: radeon_device pointer
3052 * @fence: radeon fence object
3054 * Emits a fence sequnce number on the compute ring and flushes
3057 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3058 struct radeon_fence *fence)
3060 struct radeon_ring *ring = &rdev->ring[fence->ring];
3061 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3063 /* RELEASE_MEM - flush caches, send int */
3064 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3065 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3067 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3069 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3070 radeon_ring_write(ring, addr & 0xfffffffc);
3071 radeon_ring_write(ring, upper_32_bits(addr));
3072 radeon_ring_write(ring, fence->seq);
3073 radeon_ring_write(ring, 0);
3075 /* We should be using the new WAIT_REG_MEM special op packet here
3076 * but it causes the CP to hang
3078 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3079 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3080 WRITE_DATA_DST_SEL(0)));
3081 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3082 radeon_ring_write(ring, 0);
3083 radeon_ring_write(ring, 0);
3086 void cik_semaphore_ring_emit(struct radeon_device *rdev,
3087 struct radeon_ring *ring,
3088 struct radeon_semaphore *semaphore,
3091 uint64_t addr = semaphore->gpu_addr;
3092 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3094 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3095 radeon_ring_write(ring, addr & 0xffffffff);
3096 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3103 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3105 * @rdev: radeon_device pointer
3106 * @ib: radeon indirect buffer object
3108 * Emits an DE (drawing engine) or CE (constant engine) IB
3109 * on the gfx ring. IBs are usually generated by userspace
3110 * acceleration drivers and submitted to the kernel for
3111 * sheduling on the ring. This function schedules the IB
3112 * on the gfx ring for execution by the GPU.
3114 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3116 struct radeon_ring *ring = &rdev->ring[ib->ring];
3117 u32 header, control = INDIRECT_BUFFER_VALID;
3119 if (ib->is_const_ib) {
3120 /* set switch buffer packet before const IB */
3121 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3122 radeon_ring_write(ring, 0);
3124 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3127 if (ring->rptr_save_reg) {
3128 next_rptr = ring->wptr + 3 + 4;
3129 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3130 radeon_ring_write(ring, ((ring->rptr_save_reg -
3131 PACKET3_SET_UCONFIG_REG_START) >> 2));
3132 radeon_ring_write(ring, next_rptr);
3133 } else if (rdev->wb.enabled) {
3134 next_rptr = ring->wptr + 5 + 4;
3135 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3136 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3137 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3138 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3139 radeon_ring_write(ring, next_rptr);
3142 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3145 control |= ib->length_dw |
3146 (ib->vm ? (ib->vm->id << 24) : 0);
3148 radeon_ring_write(ring, header);
3149 radeon_ring_write(ring,
3153 (ib->gpu_addr & 0xFFFFFFFC));
3154 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3155 radeon_ring_write(ring, control);
3159 * cik_ib_test - basic gfx ring IB test
3161 * @rdev: radeon_device pointer
3162 * @ring: radeon_ring structure holding ring information
3164 * Allocate an IB and execute it on the gfx ring (CIK).
3165 * Provides a basic gfx ring test to verify that IBs are working.
3166 * Returns 0 on success, error on failure.
3168 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3170 struct radeon_ib ib;
3176 r = radeon_scratch_get(rdev, &scratch);
3178 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3181 WREG32(scratch, 0xCAFEDEAD);
3182 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3184 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3187 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3188 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3189 ib.ptr[2] = 0xDEADBEEF;
3191 r = radeon_ib_schedule(rdev, &ib, NULL);
3193 radeon_scratch_free(rdev, scratch);
3194 radeon_ib_free(rdev, &ib);
3195 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3198 r = radeon_fence_wait(ib.fence, false);
3200 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3203 for (i = 0; i < rdev->usec_timeout; i++) {
3204 tmp = RREG32(scratch);
3205 if (tmp == 0xDEADBEEF)
3209 if (i < rdev->usec_timeout) {
3210 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3212 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3216 radeon_scratch_free(rdev, scratch);
3217 radeon_ib_free(rdev, &ib);
3223 * On CIK, gfx and compute now have independant command processors.
3226 * Gfx consists of a single ring and can process both gfx jobs and
3227 * compute jobs. The gfx CP consists of three microengines (ME):
3228 * PFP - Pre-Fetch Parser
3230 * CE - Constant Engine
3231 * The PFP and ME make up what is considered the Drawing Engine (DE).
3232 * The CE is an asynchronous engine used for updating buffer desciptors
3233 * used by the DE so that they can be loaded into cache in parallel
3234 * while the DE is processing state update packets.
3237 * The compute CP consists of two microengines (ME):
3238 * MEC1 - Compute MicroEngine 1
3239 * MEC2 - Compute MicroEngine 2
3240 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3241 * The queues are exposed to userspace and are programmed directly
3242 * by the compute runtime.
3245 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3247 * @rdev: radeon_device pointer
3248 * @enable: enable or disable the MEs
3250 * Halts or unhalts the gfx MEs.
3252 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3255 WREG32(CP_ME_CNTL, 0);
3257 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3258 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3264 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3266 * @rdev: radeon_device pointer
3268 * Loads the gfx PFP, ME, and CE ucode.
3269 * Returns 0 for success, -EINVAL if the ucode is not available.
3271 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3273 const __be32 *fw_data;
3276 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3279 cik_cp_gfx_enable(rdev, false);
3282 fw_data = (const __be32 *)rdev->pfp_fw->data;
3283 WREG32(CP_PFP_UCODE_ADDR, 0);
3284 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3285 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3286 WREG32(CP_PFP_UCODE_ADDR, 0);
3289 fw_data = (const __be32 *)rdev->ce_fw->data;
3290 WREG32(CP_CE_UCODE_ADDR, 0);
3291 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3292 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3293 WREG32(CP_CE_UCODE_ADDR, 0);
3296 fw_data = (const __be32 *)rdev->me_fw->data;
3297 WREG32(CP_ME_RAM_WADDR, 0);
3298 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3299 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3300 WREG32(CP_ME_RAM_WADDR, 0);
3302 WREG32(CP_PFP_UCODE_ADDR, 0);
3303 WREG32(CP_CE_UCODE_ADDR, 0);
3304 WREG32(CP_ME_RAM_WADDR, 0);
3305 WREG32(CP_ME_RAM_RADDR, 0);
3310 * cik_cp_gfx_start - start the gfx ring
3312 * @rdev: radeon_device pointer
3314 * Enables the ring and loads the clear state context and other
3315 * packets required to init the ring.
3316 * Returns 0 for success, error for failure.
3318 static int cik_cp_gfx_start(struct radeon_device *rdev)
3320 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3324 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3325 WREG32(CP_ENDIAN_SWAP, 0);
3326 WREG32(CP_DEVICE_ID, 1);
3328 cik_cp_gfx_enable(rdev, true);
3330 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3332 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3336 /* init the CE partitions. CE only used for gfx on CIK */
3337 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3338 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3339 radeon_ring_write(ring, 0xc000);
3340 radeon_ring_write(ring, 0xc000);
3342 /* setup clear context state */
3343 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3344 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3346 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3347 radeon_ring_write(ring, 0x80000000);
3348 radeon_ring_write(ring, 0x80000000);
3350 for (i = 0; i < cik_default_size; i++)
3351 radeon_ring_write(ring, cik_default_state[i]);
3353 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3354 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3356 /* set clear context state */
3357 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3358 radeon_ring_write(ring, 0);
3360 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3361 radeon_ring_write(ring, 0x00000316);
3362 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3363 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3365 radeon_ring_unlock_commit(rdev, ring);
3371 * cik_cp_gfx_fini - stop the gfx ring
3373 * @rdev: radeon_device pointer
3375 * Stop the gfx ring and tear down the driver ring
3378 static void cik_cp_gfx_fini(struct radeon_device *rdev)
3380 cik_cp_gfx_enable(rdev, false);
3381 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3385 * cik_cp_gfx_resume - setup the gfx ring buffer registers
3387 * @rdev: radeon_device pointer
3389 * Program the location and size of the gfx ring buffer
3390 * and test it to make sure it's working.
3391 * Returns 0 for success, error for failure.
3393 static int cik_cp_gfx_resume(struct radeon_device *rdev)
3395 struct radeon_ring *ring;
3401 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3402 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3404 /* Set the write pointer delay */
3405 WREG32(CP_RB_WPTR_DELAY, 0);
3407 /* set the RB to use vmid 0 */
3408 WREG32(CP_RB_VMID, 0);
3410 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3412 /* ring 0 - compute and gfx */
3413 /* Set ring buffer size */
3414 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3415 rb_bufsz = order_base_2(ring->ring_size / 8);
3416 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3418 tmp |= BUF_SWAP_32BIT;
3420 WREG32(CP_RB0_CNTL, tmp);
3422 /* Initialize the ring buffer's read and write pointers */
3423 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3425 WREG32(CP_RB0_WPTR, ring->wptr);
3427 /* set the wb address wether it's enabled or not */
3428 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3429 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3431 /* scratch register shadowing is no longer supported */
3432 WREG32(SCRATCH_UMSK, 0);
3434 if (!rdev->wb.enabled)
3435 tmp |= RB_NO_UPDATE;
3438 WREG32(CP_RB0_CNTL, tmp);
3440 rb_addr = ring->gpu_addr >> 8;
3441 WREG32(CP_RB0_BASE, rb_addr);
3442 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
3444 ring->rptr = RREG32(CP_RB0_RPTR);
3446 /* start the ring */
3447 cik_cp_gfx_start(rdev);
3448 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3449 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3451 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3457 u32 cik_compute_ring_get_rptr(struct radeon_device *rdev,
3458 struct radeon_ring *ring)
3464 if (rdev->wb.enabled) {
3465 rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
3467 mutex_lock(&rdev->srbm_mutex);
3468 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
3469 rptr = RREG32(CP_HQD_PQ_RPTR);
3470 cik_srbm_select(rdev, 0, 0, 0, 0);
3471 mutex_unlock(&rdev->srbm_mutex);
3477 u32 cik_compute_ring_get_wptr(struct radeon_device *rdev,
3478 struct radeon_ring *ring)
3482 if (rdev->wb.enabled) {
3483 wptr = le32_to_cpu(rdev->wb.wb[ring->wptr_offs/4]);
3485 mutex_lock(&rdev->srbm_mutex);
3486 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
3487 wptr = RREG32(CP_HQD_PQ_WPTR);
3488 cik_srbm_select(rdev, 0, 0, 0, 0);
3489 mutex_unlock(&rdev->srbm_mutex);
3495 void cik_compute_ring_set_wptr(struct radeon_device *rdev,
3496 struct radeon_ring *ring)
3498 rdev->wb.wb[ring->wptr_offs/4] = cpu_to_le32(ring->wptr);
3499 WDOORBELL32(ring->doorbell_offset, ring->wptr);
3503 * cik_cp_compute_enable - enable/disable the compute CP MEs
3505 * @rdev: radeon_device pointer
3506 * @enable: enable or disable the MEs
3508 * Halts or unhalts the compute MEs.
3510 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
3513 WREG32(CP_MEC_CNTL, 0);
3515 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
3520 * cik_cp_compute_load_microcode - load the compute CP ME ucode
3522 * @rdev: radeon_device pointer
3524 * Loads the compute MEC1&2 ucode.
3525 * Returns 0 for success, -EINVAL if the ucode is not available.
3527 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
3529 const __be32 *fw_data;
3535 cik_cp_compute_enable(rdev, false);
3538 fw_data = (const __be32 *)rdev->mec_fw->data;
3539 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
3540 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
3541 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
3542 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
3544 if (rdev->family == CHIP_KAVERI) {
3546 fw_data = (const __be32 *)rdev->mec_fw->data;
3547 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
3548 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
3549 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
3550 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
3557 * cik_cp_compute_start - start the compute queues
3559 * @rdev: radeon_device pointer
3561 * Enable the compute queues.
3562 * Returns 0 for success, error for failure.
3564 static int cik_cp_compute_start(struct radeon_device *rdev)
3566 cik_cp_compute_enable(rdev, true);
3572 * cik_cp_compute_fini - stop the compute queues
3574 * @rdev: radeon_device pointer
3576 * Stop the compute queues and tear down the driver queue
3579 static void cik_cp_compute_fini(struct radeon_device *rdev)
3583 cik_cp_compute_enable(rdev, false);
3585 for (i = 0; i < 2; i++) {
3587 idx = CAYMAN_RING_TYPE_CP1_INDEX;
3589 idx = CAYMAN_RING_TYPE_CP2_INDEX;
3591 if (rdev->ring[idx].mqd_obj) {
3592 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
3593 if (unlikely(r != 0))
3594 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
3596 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
3597 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
3599 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
3600 rdev->ring[idx].mqd_obj = NULL;
3605 static void cik_mec_fini(struct radeon_device *rdev)
3609 if (rdev->mec.hpd_eop_obj) {
3610 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
3611 if (unlikely(r != 0))
3612 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
3613 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
3614 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
3616 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
3617 rdev->mec.hpd_eop_obj = NULL;
3621 #define MEC_HPD_SIZE 2048
3623 static int cik_mec_init(struct radeon_device *rdev)
3629 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
3630 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
3632 if (rdev->family == CHIP_KAVERI)
3633 rdev->mec.num_mec = 2;
3635 rdev->mec.num_mec = 1;
3636 rdev->mec.num_pipe = 4;
3637 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
3639 if (rdev->mec.hpd_eop_obj == NULL) {
3640 r = radeon_bo_create(rdev,
3641 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
3643 RADEON_GEM_DOMAIN_GTT, NULL,
3644 &rdev->mec.hpd_eop_obj);
3646 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
3651 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
3652 if (unlikely(r != 0)) {
3656 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
3657 &rdev->mec.hpd_eop_gpu_addr);
3659 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
3663 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
3665 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
3670 /* clear memory. Not sure if this is required or not */
3671 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
3673 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
3674 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
3679 struct hqd_registers
3681 u32 cp_mqd_base_addr;
3682 u32 cp_mqd_base_addr_hi;
3685 u32 cp_hqd_persistent_state;
3686 u32 cp_hqd_pipe_priority;
3687 u32 cp_hqd_queue_priority;
3690 u32 cp_hqd_pq_base_hi;
3692 u32 cp_hqd_pq_rptr_report_addr;
3693 u32 cp_hqd_pq_rptr_report_addr_hi;
3694 u32 cp_hqd_pq_wptr_poll_addr;
3695 u32 cp_hqd_pq_wptr_poll_addr_hi;
3696 u32 cp_hqd_pq_doorbell_control;
3698 u32 cp_hqd_pq_control;
3699 u32 cp_hqd_ib_base_addr;
3700 u32 cp_hqd_ib_base_addr_hi;
3702 u32 cp_hqd_ib_control;
3703 u32 cp_hqd_iq_timer;
3705 u32 cp_hqd_dequeue_request;
3706 u32 cp_hqd_dma_offload;
3707 u32 cp_hqd_sema_cmd;
3708 u32 cp_hqd_msg_type;
3709 u32 cp_hqd_atomic0_preop_lo;
3710 u32 cp_hqd_atomic0_preop_hi;
3711 u32 cp_hqd_atomic1_preop_lo;
3712 u32 cp_hqd_atomic1_preop_hi;
3713 u32 cp_hqd_hq_scheduler0;
3714 u32 cp_hqd_hq_scheduler1;
3721 u32 dispatch_initiator;
3725 u32 pipeline_stat_enable;
3726 u32 perf_counter_enable;
3732 u32 resource_limits;
3733 u32 static_thread_mgmt01[2];
3735 u32 static_thread_mgmt23[2];
3737 u32 thread_trace_enable;
3740 u32 vgtcs_invoke_count[2];
3741 struct hqd_registers queue_state;
3743 u32 interrupt_queue[64];
3747 * cik_cp_compute_resume - setup the compute queue registers
3749 * @rdev: radeon_device pointer
3751 * Program the compute queues and test them to make sure they
3753 * Returns 0 for success, error for failure.
3755 static int cik_cp_compute_resume(struct radeon_device *rdev)
3759 bool use_doorbell = true;
3765 struct bonaire_mqd *mqd;
3767 r = cik_cp_compute_start(rdev);
3771 /* fix up chicken bits */
3772 tmp = RREG32(CP_CPF_DEBUG);
3774 WREG32(CP_CPF_DEBUG, tmp);
3776 /* init the pipes */
3777 mutex_lock(&rdev->srbm_mutex);
3778 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
3779 int me = (i < 4) ? 1 : 2;
3780 int pipe = (i < 4) ? i : (i - 4);
3782 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
3784 cik_srbm_select(rdev, me, pipe, 0, 0);
3786 /* write the EOP addr */
3787 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
3788 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
3790 /* set the VMID assigned */
3791 WREG32(CP_HPD_EOP_VMID, 0);
3793 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3794 tmp = RREG32(CP_HPD_EOP_CONTROL);
3795 tmp &= ~EOP_SIZE_MASK;
3796 tmp |= order_base_2(MEC_HPD_SIZE / 8);
3797 WREG32(CP_HPD_EOP_CONTROL, tmp);
3799 cik_srbm_select(rdev, 0, 0, 0, 0);
3800 mutex_unlock(&rdev->srbm_mutex);
3802 /* init the queues. Just two for now. */
3803 for (i = 0; i < 2; i++) {
3805 idx = CAYMAN_RING_TYPE_CP1_INDEX;
3807 idx = CAYMAN_RING_TYPE_CP2_INDEX;
3809 if (rdev->ring[idx].mqd_obj == NULL) {
3810 r = radeon_bo_create(rdev,
3811 sizeof(struct bonaire_mqd),
3813 RADEON_GEM_DOMAIN_GTT, NULL,
3814 &rdev->ring[idx].mqd_obj);
3816 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
3821 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
3822 if (unlikely(r != 0)) {
3823 cik_cp_compute_fini(rdev);
3826 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
3829 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
3830 cik_cp_compute_fini(rdev);
3833 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
3835 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
3836 cik_cp_compute_fini(rdev);
3840 /* doorbell offset */
3841 rdev->ring[idx].doorbell_offset =
3842 (rdev->ring[idx].doorbell_page_num * PAGE_SIZE) + 0;
3844 /* init the mqd struct */
3845 memset(buf, 0, sizeof(struct bonaire_mqd));
3847 mqd = (struct bonaire_mqd *)buf;
3848 mqd->header = 0xC0310800;
3849 mqd->static_thread_mgmt01[0] = 0xffffffff;
3850 mqd->static_thread_mgmt01[1] = 0xffffffff;
3851 mqd->static_thread_mgmt23[0] = 0xffffffff;
3852 mqd->static_thread_mgmt23[1] = 0xffffffff;
3854 mutex_lock(&rdev->srbm_mutex);
3855 cik_srbm_select(rdev, rdev->ring[idx].me,
3856 rdev->ring[idx].pipe,
3857 rdev->ring[idx].queue, 0);
3859 /* disable wptr polling */
3860 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
3861 tmp &= ~WPTR_POLL_EN;
3862 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
3864 /* enable doorbell? */
3865 mqd->queue_state.cp_hqd_pq_doorbell_control =
3866 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
3868 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
3870 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
3871 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
3872 mqd->queue_state.cp_hqd_pq_doorbell_control);
3874 /* disable the queue if it's active */
3875 mqd->queue_state.cp_hqd_dequeue_request = 0;
3876 mqd->queue_state.cp_hqd_pq_rptr = 0;
3877 mqd->queue_state.cp_hqd_pq_wptr= 0;
3878 if (RREG32(CP_HQD_ACTIVE) & 1) {
3879 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
3880 for (i = 0; i < rdev->usec_timeout; i++) {
3881 if (!(RREG32(CP_HQD_ACTIVE) & 1))
3885 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
3886 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
3887 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
3890 /* set the pointer to the MQD */
3891 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
3892 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
3893 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
3894 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
3895 /* set MQD vmid to 0 */
3896 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
3897 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
3898 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
3900 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3901 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
3902 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
3903 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3904 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
3905 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
3907 /* set up the HQD, this is similar to CP_RB0_CNTL */
3908 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
3909 mqd->queue_state.cp_hqd_pq_control &=
3910 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
3912 mqd->queue_state.cp_hqd_pq_control |=
3913 order_base_2(rdev->ring[idx].ring_size / 8);
3914 mqd->queue_state.cp_hqd_pq_control |=
3915 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
3917 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
3919 mqd->queue_state.cp_hqd_pq_control &=
3920 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
3921 mqd->queue_state.cp_hqd_pq_control |=
3922 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
3923 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
3925 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
3927 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
3929 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
3930 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
3931 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3932 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
3933 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
3934 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
3936 /* set the wb address wether it's enabled or not */
3938 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
3940 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
3941 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
3942 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
3943 upper_32_bits(wb_gpu_addr) & 0xffff;
3944 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
3945 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
3946 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3947 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
3949 /* enable the doorbell if requested */
3951 mqd->queue_state.cp_hqd_pq_doorbell_control =
3952 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
3953 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
3954 mqd->queue_state.cp_hqd_pq_doorbell_control |=
3955 DOORBELL_OFFSET(rdev->ring[idx].doorbell_offset / 4);
3956 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
3957 mqd->queue_state.cp_hqd_pq_doorbell_control &=
3958 ~(DOORBELL_SOURCE | DOORBELL_HIT);
3961 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
3963 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
3964 mqd->queue_state.cp_hqd_pq_doorbell_control);
3966 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3967 rdev->ring[idx].wptr = 0;
3968 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
3969 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
3970 rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
3971 mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
3973 /* set the vmid for the queue */
3974 mqd->queue_state.cp_hqd_vmid = 0;
3975 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
3977 /* activate the queue */
3978 mqd->queue_state.cp_hqd_active = 1;
3979 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
3981 cik_srbm_select(rdev, 0, 0, 0, 0);
3982 mutex_unlock(&rdev->srbm_mutex);
3984 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
3985 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
3987 rdev->ring[idx].ready = true;
3988 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
3990 rdev->ring[idx].ready = false;
3996 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
3998 cik_cp_gfx_enable(rdev, enable);
3999 cik_cp_compute_enable(rdev, enable);
4002 static int cik_cp_load_microcode(struct radeon_device *rdev)
4006 r = cik_cp_gfx_load_microcode(rdev);
4009 r = cik_cp_compute_load_microcode(rdev);
4016 static void cik_cp_fini(struct radeon_device *rdev)
4018 cik_cp_gfx_fini(rdev);
4019 cik_cp_compute_fini(rdev);
4022 static int cik_cp_resume(struct radeon_device *rdev)
4026 cik_enable_gui_idle_interrupt(rdev, false);
4028 r = cik_cp_load_microcode(rdev);
4032 r = cik_cp_gfx_resume(rdev);
4035 r = cik_cp_compute_resume(rdev);
4039 cik_enable_gui_idle_interrupt(rdev, true);
4044 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4046 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4047 RREG32(GRBM_STATUS));
4048 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4049 RREG32(GRBM_STATUS2));
4050 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4051 RREG32(GRBM_STATUS_SE0));
4052 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4053 RREG32(GRBM_STATUS_SE1));
4054 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4055 RREG32(GRBM_STATUS_SE2));
4056 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4057 RREG32(GRBM_STATUS_SE3));
4058 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4059 RREG32(SRBM_STATUS));
4060 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4061 RREG32(SRBM_STATUS2));
4062 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4063 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4064 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4065 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
4066 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4067 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4068 RREG32(CP_STALLED_STAT1));
4069 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4070 RREG32(CP_STALLED_STAT2));
4071 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4072 RREG32(CP_STALLED_STAT3));
4073 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4074 RREG32(CP_CPF_BUSY_STAT));
4075 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4076 RREG32(CP_CPF_STALLED_STAT1));
4077 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4078 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4079 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4080 RREG32(CP_CPC_STALLED_STAT1));
4081 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
4085 * cik_gpu_check_soft_reset - check which blocks are busy
4087 * @rdev: radeon_device pointer
4089 * Check which blocks are busy and return the relevant reset
4090 * mask to be used by cik_gpu_soft_reset().
4091 * Returns a mask of the blocks to be reset.
4093 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
4099 tmp = RREG32(GRBM_STATUS);
4100 if (tmp & (PA_BUSY | SC_BUSY |
4101 BCI_BUSY | SX_BUSY |
4102 TA_BUSY | VGT_BUSY |
4104 GDS_BUSY | SPI_BUSY |
4105 IA_BUSY | IA_BUSY_NO_DMA))
4106 reset_mask |= RADEON_RESET_GFX;
4108 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4109 reset_mask |= RADEON_RESET_CP;
4112 tmp = RREG32(GRBM_STATUS2);
4114 reset_mask |= RADEON_RESET_RLC;
4116 /* SDMA0_STATUS_REG */
4117 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4118 if (!(tmp & SDMA_IDLE))
4119 reset_mask |= RADEON_RESET_DMA;
4121 /* SDMA1_STATUS_REG */
4122 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4123 if (!(tmp & SDMA_IDLE))
4124 reset_mask |= RADEON_RESET_DMA1;
4127 tmp = RREG32(SRBM_STATUS2);
4128 if (tmp & SDMA_BUSY)
4129 reset_mask |= RADEON_RESET_DMA;
4131 if (tmp & SDMA1_BUSY)
4132 reset_mask |= RADEON_RESET_DMA1;
4135 tmp = RREG32(SRBM_STATUS);
4138 reset_mask |= RADEON_RESET_IH;
4141 reset_mask |= RADEON_RESET_SEM;
4143 if (tmp & GRBM_RQ_PENDING)
4144 reset_mask |= RADEON_RESET_GRBM;
4147 reset_mask |= RADEON_RESET_VMC;
4149 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4150 MCC_BUSY | MCD_BUSY))
4151 reset_mask |= RADEON_RESET_MC;
4153 if (evergreen_is_display_hung(rdev))
4154 reset_mask |= RADEON_RESET_DISPLAY;
4156 /* Skip MC reset as it's mostly likely not hung, just busy */
4157 if (reset_mask & RADEON_RESET_MC) {
4158 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4159 reset_mask &= ~RADEON_RESET_MC;
4166 * cik_gpu_soft_reset - soft reset GPU
4168 * @rdev: radeon_device pointer
4169 * @reset_mask: mask of which blocks to reset
4171 * Soft reset the blocks specified in @reset_mask.
4173 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4175 struct evergreen_mc_save save;
4176 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4179 if (reset_mask == 0)
4182 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4184 cik_print_gpu_status_regs(rdev);
4185 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4186 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4187 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4188 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4193 /* Disable GFX parsing/prefetching */
4194 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4196 /* Disable MEC parsing/prefetching */
4197 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4199 if (reset_mask & RADEON_RESET_DMA) {
4201 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4203 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4205 if (reset_mask & RADEON_RESET_DMA1) {
4207 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4209 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4212 evergreen_mc_stop(rdev, &save);
4213 if (evergreen_mc_wait_for_idle(rdev)) {
4214 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4217 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4218 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4220 if (reset_mask & RADEON_RESET_CP) {
4221 grbm_soft_reset |= SOFT_RESET_CP;
4223 srbm_soft_reset |= SOFT_RESET_GRBM;
4226 if (reset_mask & RADEON_RESET_DMA)
4227 srbm_soft_reset |= SOFT_RESET_SDMA;
4229 if (reset_mask & RADEON_RESET_DMA1)
4230 srbm_soft_reset |= SOFT_RESET_SDMA1;
4232 if (reset_mask & RADEON_RESET_DISPLAY)
4233 srbm_soft_reset |= SOFT_RESET_DC;
4235 if (reset_mask & RADEON_RESET_RLC)
4236 grbm_soft_reset |= SOFT_RESET_RLC;
4238 if (reset_mask & RADEON_RESET_SEM)
4239 srbm_soft_reset |= SOFT_RESET_SEM;
4241 if (reset_mask & RADEON_RESET_IH)
4242 srbm_soft_reset |= SOFT_RESET_IH;
4244 if (reset_mask & RADEON_RESET_GRBM)
4245 srbm_soft_reset |= SOFT_RESET_GRBM;
4247 if (reset_mask & RADEON_RESET_VMC)
4248 srbm_soft_reset |= SOFT_RESET_VMC;
4250 if (!(rdev->flags & RADEON_IS_IGP)) {
4251 if (reset_mask & RADEON_RESET_MC)
4252 srbm_soft_reset |= SOFT_RESET_MC;
4255 if (grbm_soft_reset) {
4256 tmp = RREG32(GRBM_SOFT_RESET);
4257 tmp |= grbm_soft_reset;
4258 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4259 WREG32(GRBM_SOFT_RESET, tmp);
4260 tmp = RREG32(GRBM_SOFT_RESET);
4264 tmp &= ~grbm_soft_reset;
4265 WREG32(GRBM_SOFT_RESET, tmp);
4266 tmp = RREG32(GRBM_SOFT_RESET);
4269 if (srbm_soft_reset) {
4270 tmp = RREG32(SRBM_SOFT_RESET);
4271 tmp |= srbm_soft_reset;
4272 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4273 WREG32(SRBM_SOFT_RESET, tmp);
4274 tmp = RREG32(SRBM_SOFT_RESET);
4278 tmp &= ~srbm_soft_reset;
4279 WREG32(SRBM_SOFT_RESET, tmp);
4280 tmp = RREG32(SRBM_SOFT_RESET);
4283 /* Wait a little for things to settle down */
4286 evergreen_mc_resume(rdev, &save);
4289 cik_print_gpu_status_regs(rdev);
4293 * cik_asic_reset - soft reset GPU
4295 * @rdev: radeon_device pointer
4297 * Look up which blocks are hung and attempt
4299 * Returns 0 for success.
4301 int cik_asic_reset(struct radeon_device *rdev)
4305 reset_mask = cik_gpu_check_soft_reset(rdev);
4308 r600_set_bios_scratch_engine_hung(rdev, true);
4310 cik_gpu_soft_reset(rdev, reset_mask);
4312 reset_mask = cik_gpu_check_soft_reset(rdev);
4315 r600_set_bios_scratch_engine_hung(rdev, false);
4321 * cik_gfx_is_lockup - check if the 3D engine is locked up
4323 * @rdev: radeon_device pointer
4324 * @ring: radeon_ring structure holding ring information
4326 * Check if the 3D engine is locked up (CIK).
4327 * Returns true if the engine is locked, false if not.
4329 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4331 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
4333 if (!(reset_mask & (RADEON_RESET_GFX |
4334 RADEON_RESET_COMPUTE |
4335 RADEON_RESET_CP))) {
4336 radeon_ring_lockup_update(ring);
4339 /* force CP activities */
4340 radeon_ring_force_activity(rdev, ring);
4341 return radeon_ring_test_lockup(rdev, ring);
4346 * cik_mc_program - program the GPU memory controller
4348 * @rdev: radeon_device pointer
4350 * Set the location of vram, gart, and AGP in the GPU's
4351 * physical address space (CIK).
4353 static void cik_mc_program(struct radeon_device *rdev)
4355 struct evergreen_mc_save save;
4359 /* Initialize HDP */
4360 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4361 WREG32((0x2c14 + j), 0x00000000);
4362 WREG32((0x2c18 + j), 0x00000000);
4363 WREG32((0x2c1c + j), 0x00000000);
4364 WREG32((0x2c20 + j), 0x00000000);
4365 WREG32((0x2c24 + j), 0x00000000);
4367 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4369 evergreen_mc_stop(rdev, &save);
4370 if (radeon_mc_wait_for_idle(rdev)) {
4371 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4373 /* Lockout access through VGA aperture*/
4374 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4375 /* Update configuration */
4376 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4377 rdev->mc.vram_start >> 12);
4378 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4379 rdev->mc.vram_end >> 12);
4380 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4381 rdev->vram_scratch.gpu_addr >> 12);
4382 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4383 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4384 WREG32(MC_VM_FB_LOCATION, tmp);
4385 /* XXX double check these! */
4386 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4387 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4388 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4389 WREG32(MC_VM_AGP_BASE, 0);
4390 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4391 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4392 if (radeon_mc_wait_for_idle(rdev)) {
4393 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4395 evergreen_mc_resume(rdev, &save);
4396 /* we need to own VRAM, so turn off the VGA renderer here
4397 * to stop it overwriting our objects */
4398 rv515_vga_render_disable(rdev);
4402 * cik_mc_init - initialize the memory controller driver params
4404 * @rdev: radeon_device pointer
4406 * Look up the amount of vram, vram width, and decide how to place
4407 * vram and gart within the GPU's physical address space (CIK).
4408 * Returns 0 for success.
4410 static int cik_mc_init(struct radeon_device *rdev)
4413 int chansize, numchan;
4415 /* Get VRAM informations */
4416 rdev->mc.vram_is_ddr = true;
4417 tmp = RREG32(MC_ARB_RAMCFG);
4418 if (tmp & CHANSIZE_MASK) {
4423 tmp = RREG32(MC_SHARED_CHMAP);
4424 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4454 rdev->mc.vram_width = numchan * chansize;
4455 /* Could aper size report 0 ? */
4456 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4457 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4458 /* size in MB on si */
4459 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
4460 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
4461 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4462 si_vram_gtt_location(rdev, &rdev->mc);
4463 radeon_update_bandwidth_info(rdev);
4470 * VMID 0 is the physical GPU addresses as used by the kernel.
4471 * VMIDs 1-15 are used for userspace clients and are handled
4472 * by the radeon vm/hsa code.
4475 * cik_pcie_gart_tlb_flush - gart tlb flush callback
4477 * @rdev: radeon_device pointer
4479 * Flush the TLB for the VMID 0 page table (CIK).
4481 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
4483 /* flush hdp cache */
4484 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
4486 /* bits 0-15 are the VM contexts0-15 */
4487 WREG32(VM_INVALIDATE_REQUEST, 0x1);
4491 * cik_pcie_gart_enable - gart enable
4493 * @rdev: radeon_device pointer
4495 * This sets up the TLBs, programs the page tables for VMID0,
4496 * sets up the hw for VMIDs 1-15 which are allocated on
4497 * demand, and sets up the global locations for the LDS, GDS,
4498 * and GPUVM for FSA64 clients (CIK).
4499 * Returns 0 for success, errors for failure.
4501 static int cik_pcie_gart_enable(struct radeon_device *rdev)
4505 if (rdev->gart.robj == NULL) {
4506 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4509 r = radeon_gart_table_vram_pin(rdev);
4512 radeon_gart_restore(rdev);
4513 /* Setup TLB control */
4514 WREG32(MC_VM_MX_L1_TLB_CNTL,
4517 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4518 ENABLE_ADVANCED_DRIVER_MODEL |
4519 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4520 /* Setup L2 cache */
4521 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4522 ENABLE_L2_FRAGMENT_PROCESSING |
4523 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4524 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4525 EFFECTIVE_L2_QUEUE_SIZE(7) |
4526 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4527 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4528 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4529 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
4530 /* setup context0 */
4531 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4532 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4533 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4534 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4535 (u32)(rdev->dummy_page.addr >> 12));
4536 WREG32(VM_CONTEXT0_CNTL2, 0);
4537 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4538 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4544 /* empty context1-15 */
4545 /* FIXME start with 4G, once using 2 level pt switch to full
4548 /* set vm size, must be a multiple of 4 */
4549 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4550 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
4551 for (i = 1; i < 16; i++) {
4553 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4554 rdev->gart.table_addr >> 12);
4556 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4557 rdev->gart.table_addr >> 12);
4560 /* enable context1-15 */
4561 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4562 (u32)(rdev->dummy_page.addr >> 12));
4563 WREG32(VM_CONTEXT1_CNTL2, 4);
4564 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4565 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4566 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4567 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4568 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4569 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4570 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4571 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4572 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4573 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4574 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4575 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4576 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
4578 /* TC cache setup ??? */
4579 WREG32(TC_CFG_L1_LOAD_POLICY0, 0);
4580 WREG32(TC_CFG_L1_LOAD_POLICY1, 0);
4581 WREG32(TC_CFG_L1_STORE_POLICY, 0);
4583 WREG32(TC_CFG_L2_LOAD_POLICY0, 0);
4584 WREG32(TC_CFG_L2_LOAD_POLICY1, 0);
4585 WREG32(TC_CFG_L2_STORE_POLICY0, 0);
4586 WREG32(TC_CFG_L2_STORE_POLICY1, 0);
4587 WREG32(TC_CFG_L2_ATOMIC_POLICY, 0);
4589 WREG32(TC_CFG_L1_VOLATILE, 0);
4590 WREG32(TC_CFG_L2_VOLATILE, 0);
4592 if (rdev->family == CHIP_KAVERI) {
4593 u32 tmp = RREG32(CHUB_CONTROL);
4595 WREG32(CHUB_CONTROL, tmp);
4598 /* XXX SH_MEM regs */
4599 /* where to put LDS, scratch, GPUVM in FSA64 space */
4600 mutex_lock(&rdev->srbm_mutex);
4601 for (i = 0; i < 16; i++) {
4602 cik_srbm_select(rdev, 0, 0, 0, i);
4603 /* CP and shaders */
4604 WREG32(SH_MEM_CONFIG, 0);
4605 WREG32(SH_MEM_APE1_BASE, 1);
4606 WREG32(SH_MEM_APE1_LIMIT, 0);
4607 WREG32(SH_MEM_BASES, 0);
4609 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
4610 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
4611 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
4612 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
4613 /* XXX SDMA RLC - todo */
4615 cik_srbm_select(rdev, 0, 0, 0, 0);
4616 mutex_unlock(&rdev->srbm_mutex);
4618 cik_pcie_gart_tlb_flush(rdev);
4619 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4620 (unsigned)(rdev->mc.gtt_size >> 20),
4621 (unsigned long long)rdev->gart.table_addr);
4622 rdev->gart.ready = true;
4627 * cik_pcie_gart_disable - gart disable
4629 * @rdev: radeon_device pointer
4631 * This disables all VM page table (CIK).
4633 static void cik_pcie_gart_disable(struct radeon_device *rdev)
4635 /* Disable all tables */
4636 WREG32(VM_CONTEXT0_CNTL, 0);
4637 WREG32(VM_CONTEXT1_CNTL, 0);
4638 /* Setup TLB control */
4639 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4640 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4641 /* Setup L2 cache */
4643 ENABLE_L2_FRAGMENT_PROCESSING |
4644 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4645 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4646 EFFECTIVE_L2_QUEUE_SIZE(7) |
4647 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4648 WREG32(VM_L2_CNTL2, 0);
4649 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4650 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
4651 radeon_gart_table_vram_unpin(rdev);
4655 * cik_pcie_gart_fini - vm fini callback
4657 * @rdev: radeon_device pointer
4659 * Tears down the driver GART/VM setup (CIK).
4661 static void cik_pcie_gart_fini(struct radeon_device *rdev)
4663 cik_pcie_gart_disable(rdev);
4664 radeon_gart_table_vram_free(rdev);
4665 radeon_gart_fini(rdev);
4670 * cik_ib_parse - vm ib_parse callback
4672 * @rdev: radeon_device pointer
4673 * @ib: indirect buffer pointer
4675 * CIK uses hw IB checking so this is a nop (CIK).
4677 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4684 * VMID 0 is the physical GPU addresses as used by the kernel.
4685 * VMIDs 1-15 are used for userspace clients and are handled
4686 * by the radeon vm/hsa code.
4689 * cik_vm_init - cik vm init callback
4691 * @rdev: radeon_device pointer
4693 * Inits cik specific vm parameters (number of VMs, base of vram for
4694 * VMIDs 1-15) (CIK).
4695 * Returns 0 for success.
4697 int cik_vm_init(struct radeon_device *rdev)
4700 rdev->vm_manager.nvm = 16;
4701 /* base offset of vram pages */
4702 if (rdev->flags & RADEON_IS_IGP) {
4703 u64 tmp = RREG32(MC_VM_FB_OFFSET);
4705 rdev->vm_manager.vram_base_offset = tmp;
4707 rdev->vm_manager.vram_base_offset = 0;
4713 * cik_vm_fini - cik vm fini callback
4715 * @rdev: radeon_device pointer
4717 * Tear down any asic specific VM setup (CIK).
4719 void cik_vm_fini(struct radeon_device *rdev)
4724 * cik_vm_decode_fault - print human readable fault info
4726 * @rdev: radeon_device pointer
4727 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4728 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4730 * Print human readable fault information (CIK).
4732 static void cik_vm_decode_fault(struct radeon_device *rdev,
4733 u32 status, u32 addr, u32 mc_client)
4735 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4736 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4737 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4738 char *block = (char *)&mc_client;
4740 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4741 protections, vmid, addr,
4742 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4747 * cik_vm_flush - cik vm flush using the CP
4749 * @rdev: radeon_device pointer
4751 * Update the page table base and flush the VM TLB
4752 * using the CP (CIK).
4754 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4756 struct radeon_ring *ring = &rdev->ring[ridx];
4761 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4762 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4763 WRITE_DATA_DST_SEL(0)));
4765 radeon_ring_write(ring,
4766 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4768 radeon_ring_write(ring,
4769 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4771 radeon_ring_write(ring, 0);
4772 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4774 /* update SH_MEM_* regs */
4775 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4776 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4777 WRITE_DATA_DST_SEL(0)));
4778 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
4779 radeon_ring_write(ring, 0);
4780 radeon_ring_write(ring, VMID(vm->id));
4782 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
4783 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4784 WRITE_DATA_DST_SEL(0)));
4785 radeon_ring_write(ring, SH_MEM_BASES >> 2);
4786 radeon_ring_write(ring, 0);
4788 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
4789 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
4790 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
4791 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
4793 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4794 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4795 WRITE_DATA_DST_SEL(0)));
4796 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
4797 radeon_ring_write(ring, 0);
4798 radeon_ring_write(ring, VMID(0));
4801 /* We should be using the WAIT_REG_MEM packet here like in
4802 * cik_fence_ring_emit(), but it causes the CP to hang in this
4805 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4806 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4807 WRITE_DATA_DST_SEL(0)));
4808 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4809 radeon_ring_write(ring, 0);
4810 radeon_ring_write(ring, 0);
4812 /* bits 0-15 are the VM contexts0-15 */
4813 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4814 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4815 WRITE_DATA_DST_SEL(0)));
4816 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4817 radeon_ring_write(ring, 0);
4818 radeon_ring_write(ring, 1 << vm->id);
4820 /* compute doesn't have PFP */
4821 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
4822 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4823 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4824 radeon_ring_write(ring, 0x0);
4829 * cik_vm_set_page - update the page tables using sDMA
4831 * @rdev: radeon_device pointer
4832 * @ib: indirect buffer to fill with commands
4833 * @pe: addr of the page entry
4834 * @addr: dst addr to write into pe
4835 * @count: number of page entries to update
4836 * @incr: increase next addr by incr bytes
4837 * @flags: access flags
4839 * Update the page tables using CP or sDMA (CIK).
4841 void cik_vm_set_page(struct radeon_device *rdev,
4842 struct radeon_ib *ib,
4844 uint64_t addr, unsigned count,
4845 uint32_t incr, uint32_t flags)
4847 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
4851 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4854 ndw = 2 + count * 2;
4858 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4859 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4860 WRITE_DATA_DST_SEL(1));
4861 ib->ptr[ib->length_dw++] = pe;
4862 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
4863 for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4864 if (flags & RADEON_VM_PAGE_SYSTEM) {
4865 value = radeon_vm_map_gart(rdev, addr);
4866 value &= 0xFFFFFFFFFFFFF000ULL;
4867 } else if (flags & RADEON_VM_PAGE_VALID) {
4873 value |= r600_flags;
4874 ib->ptr[ib->length_dw++] = value;
4875 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4880 cik_sdma_vm_set_page(rdev, ib, pe, addr, count, incr, flags);
4886 * The RLC is a multi-purpose microengine that handles a
4887 * variety of functions, the most important of which is
4888 * the interrupt controller.
4890 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
4893 u32 tmp = RREG32(CP_INT_CNTL_RING0);
4896 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4898 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4899 WREG32(CP_INT_CNTL_RING0, tmp);
4902 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
4906 tmp = RREG32(RLC_LB_CNTL);
4908 tmp |= LOAD_BALANCE_ENABLE;
4910 tmp &= ~LOAD_BALANCE_ENABLE;
4911 WREG32(RLC_LB_CNTL, tmp);
4914 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
4919 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
4920 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
4921 cik_select_se_sh(rdev, i, j);
4922 for (k = 0; k < rdev->usec_timeout; k++) {
4923 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
4929 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
4931 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
4932 for (k = 0; k < rdev->usec_timeout; k++) {
4933 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
4939 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
4943 tmp = RREG32(RLC_CNTL);
4945 WREG32(RLC_CNTL, rlc);
4948 static u32 cik_halt_rlc(struct radeon_device *rdev)
4952 orig = data = RREG32(RLC_CNTL);
4954 if (data & RLC_ENABLE) {
4957 data &= ~RLC_ENABLE;
4958 WREG32(RLC_CNTL, data);
4960 for (i = 0; i < rdev->usec_timeout; i++) {
4961 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
4966 cik_wait_for_rlc_serdes(rdev);
4972 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
4976 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
4977 WREG32(RLC_GPR_REG2, tmp);
4979 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
4980 for (i = 0; i < rdev->usec_timeout; i++) {
4981 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
4986 for (i = 0; i < rdev->usec_timeout; i++) {
4987 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
4993 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
4997 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
4998 WREG32(RLC_GPR_REG2, tmp);
5002 * cik_rlc_stop - stop the RLC ME
5004 * @rdev: radeon_device pointer
5006 * Halt the RLC ME (MicroEngine) (CIK).
5008 static void cik_rlc_stop(struct radeon_device *rdev)
5010 WREG32(RLC_CNTL, 0);
5012 cik_enable_gui_idle_interrupt(rdev, false);
5014 cik_wait_for_rlc_serdes(rdev);
5018 * cik_rlc_start - start the RLC ME
5020 * @rdev: radeon_device pointer
5022 * Unhalt the RLC ME (MicroEngine) (CIK).
5024 static void cik_rlc_start(struct radeon_device *rdev)
5026 WREG32(RLC_CNTL, RLC_ENABLE);
5028 cik_enable_gui_idle_interrupt(rdev, true);
5034 * cik_rlc_resume - setup the RLC hw
5036 * @rdev: radeon_device pointer
5038 * Initialize the RLC registers, load the ucode,
5039 * and start the RLC (CIK).
5040 * Returns 0 for success, -EINVAL if the ucode is not available.
5042 static int cik_rlc_resume(struct radeon_device *rdev)
5045 const __be32 *fw_data;
5050 switch (rdev->family) {
5053 size = BONAIRE_RLC_UCODE_SIZE;
5056 size = KV_RLC_UCODE_SIZE;
5059 size = KB_RLC_UCODE_SIZE;
5066 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5067 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5075 WREG32(RLC_LB_CNTR_INIT, 0);
5076 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5078 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5079 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5080 WREG32(RLC_LB_PARAMS, 0x00600408);
5081 WREG32(RLC_LB_CNTL, 0x80000004);
5083 WREG32(RLC_MC_CNTL, 0);
5084 WREG32(RLC_UCODE_CNTL, 0);
5086 fw_data = (const __be32 *)rdev->rlc_fw->data;
5087 WREG32(RLC_GPM_UCODE_ADDR, 0);
5088 for (i = 0; i < size; i++)
5089 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5090 WREG32(RLC_GPM_UCODE_ADDR, 0);
5092 /* XXX - find out what chips support lbpw */
5093 cik_enable_lbpw(rdev, false);
5095 if (rdev->family == CHIP_BONAIRE)
5096 WREG32(RLC_DRIVER_DMA_STATUS, 0);
5098 cik_rlc_start(rdev);
5103 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5105 u32 data, orig, tmp, tmp2;
5107 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5109 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5110 cik_enable_gui_idle_interrupt(rdev, true);
5112 tmp = cik_halt_rlc(rdev);
5114 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5115 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5116 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5117 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5118 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5120 cik_update_rlc(rdev, tmp);
5122 data |= CGCG_EN | CGLS_EN;
5124 cik_enable_gui_idle_interrupt(rdev, false);
5126 RREG32(CB_CGTT_SCLK_CTRL);
5127 RREG32(CB_CGTT_SCLK_CTRL);
5128 RREG32(CB_CGTT_SCLK_CTRL);
5129 RREG32(CB_CGTT_SCLK_CTRL);
5131 data &= ~(CGCG_EN | CGLS_EN);
5135 WREG32(RLC_CGCG_CGLS_CTRL, data);
5139 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5141 u32 data, orig, tmp = 0;
5143 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5144 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5145 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5146 orig = data = RREG32(CP_MEM_SLP_CNTL);
5147 data |= CP_MEM_LS_EN;
5149 WREG32(CP_MEM_SLP_CNTL, data);
5153 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5156 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5158 tmp = cik_halt_rlc(rdev);
5160 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5161 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5162 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5163 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5164 WREG32(RLC_SERDES_WR_CTRL, data);
5166 cik_update_rlc(rdev, tmp);
5168 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5169 orig = data = RREG32(CGTS_SM_CTRL_REG);
5170 data &= ~SM_MODE_MASK;
5171 data |= SM_MODE(0x2);
5172 data |= SM_MODE_ENABLE;
5173 data &= ~CGTS_OVERRIDE;
5174 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5175 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5176 data &= ~CGTS_LS_OVERRIDE;
5177 data &= ~ON_MONITOR_ADD_MASK;
5178 data |= ON_MONITOR_ADD_EN;
5179 data |= ON_MONITOR_ADD(0x96);
5181 WREG32(CGTS_SM_CTRL_REG, data);
5184 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5187 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5189 data = RREG32(RLC_MEM_SLP_CNTL);
5190 if (data & RLC_MEM_LS_EN) {
5191 data &= ~RLC_MEM_LS_EN;
5192 WREG32(RLC_MEM_SLP_CNTL, data);
5195 data = RREG32(CP_MEM_SLP_CNTL);
5196 if (data & CP_MEM_LS_EN) {
5197 data &= ~CP_MEM_LS_EN;
5198 WREG32(CP_MEM_SLP_CNTL, data);
5201 orig = data = RREG32(CGTS_SM_CTRL_REG);
5202 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5204 WREG32(CGTS_SM_CTRL_REG, data);
5206 tmp = cik_halt_rlc(rdev);
5208 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5209 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5210 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5211 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5212 WREG32(RLC_SERDES_WR_CTRL, data);
5214 cik_update_rlc(rdev, tmp);
5218 static const u32 mc_cg_registers[] =
5231 static void cik_enable_mc_ls(struct radeon_device *rdev,
5237 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5238 orig = data = RREG32(mc_cg_registers[i]);
5239 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5240 data |= MC_LS_ENABLE;
5242 data &= ~MC_LS_ENABLE;
5244 WREG32(mc_cg_registers[i], data);
5248 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5254 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5255 orig = data = RREG32(mc_cg_registers[i]);
5256 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5257 data |= MC_CG_ENABLE;
5259 data &= ~MC_CG_ENABLE;
5261 WREG32(mc_cg_registers[i], data);
5265 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
5270 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5271 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
5272 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
5274 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
5277 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
5279 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
5282 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
5286 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
5291 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
5292 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5295 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5297 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5300 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5302 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5305 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5307 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5310 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5314 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
5319 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5320 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5322 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
5324 orig = data = RREG32(UVD_CGC_CTRL);
5327 WREG32(UVD_CGC_CTRL, data);
5329 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5331 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
5333 orig = data = RREG32(UVD_CGC_CTRL);
5336 WREG32(UVD_CGC_CTRL, data);
5340 static void cik_enable_bif_mgls(struct radeon_device *rdev,
5345 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
5347 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5348 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5349 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5351 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5352 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5355 WREG32_PCIE_PORT(PCIE_CNTL2, data);
5358 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
5363 orig = data = RREG32(HDP_HOST_PATH_CNTL);
5365 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5366 data &= ~CLOCK_GATING_DIS;
5368 data |= CLOCK_GATING_DIS;
5371 WREG32(HDP_HOST_PATH_CNTL, data);
5374 static void cik_enable_hdp_ls(struct radeon_device *rdev,
5379 orig = data = RREG32(HDP_MEM_POWER_LS);
5381 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5382 data |= HDP_LS_ENABLE;
5384 data &= ~HDP_LS_ENABLE;
5387 WREG32(HDP_MEM_POWER_LS, data);
5390 void cik_update_cg(struct radeon_device *rdev,
5391 u32 block, bool enable)
5394 if (block & RADEON_CG_BLOCK_GFX) {
5395 cik_enable_gui_idle_interrupt(rdev, false);
5396 /* order matters! */
5398 cik_enable_mgcg(rdev, true);
5399 cik_enable_cgcg(rdev, true);
5401 cik_enable_cgcg(rdev, false);
5402 cik_enable_mgcg(rdev, false);
5404 cik_enable_gui_idle_interrupt(rdev, true);
5407 if (block & RADEON_CG_BLOCK_MC) {
5408 if (!(rdev->flags & RADEON_IS_IGP)) {
5409 cik_enable_mc_mgcg(rdev, enable);
5410 cik_enable_mc_ls(rdev, enable);
5414 if (block & RADEON_CG_BLOCK_SDMA) {
5415 cik_enable_sdma_mgcg(rdev, enable);
5416 cik_enable_sdma_mgls(rdev, enable);
5419 if (block & RADEON_CG_BLOCK_BIF) {
5420 cik_enable_bif_mgls(rdev, enable);
5423 if (block & RADEON_CG_BLOCK_UVD) {
5425 cik_enable_uvd_mgcg(rdev, enable);
5428 if (block & RADEON_CG_BLOCK_HDP) {
5429 cik_enable_hdp_mgcg(rdev, enable);
5430 cik_enable_hdp_ls(rdev, enable);
5434 static void cik_init_cg(struct radeon_device *rdev)
5437 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
5440 si_init_uvd_internal_cg(rdev);
5442 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5443 RADEON_CG_BLOCK_SDMA |
5444 RADEON_CG_BLOCK_BIF |
5445 RADEON_CG_BLOCK_UVD |
5446 RADEON_CG_BLOCK_HDP), true);
5449 static void cik_fini_cg(struct radeon_device *rdev)
5451 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5452 RADEON_CG_BLOCK_SDMA |
5453 RADEON_CG_BLOCK_BIF |
5454 RADEON_CG_BLOCK_UVD |
5455 RADEON_CG_BLOCK_HDP), false);
5457 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
5460 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
5465 orig = data = RREG32(RLC_PG_CNTL);
5466 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
5467 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5469 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5471 WREG32(RLC_PG_CNTL, data);
5474 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
5479 orig = data = RREG32(RLC_PG_CNTL);
5480 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
5481 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5483 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5485 WREG32(RLC_PG_CNTL, data);
5488 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
5492 orig = data = RREG32(RLC_PG_CNTL);
5493 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
5494 data &= ~DISABLE_CP_PG;
5496 data |= DISABLE_CP_PG;
5498 WREG32(RLC_PG_CNTL, data);
5501 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
5505 orig = data = RREG32(RLC_PG_CNTL);
5506 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
5507 data &= ~DISABLE_GDS_PG;
5509 data |= DISABLE_GDS_PG;
5511 WREG32(RLC_PG_CNTL, data);
5514 #define CP_ME_TABLE_SIZE 96
5515 #define CP_ME_TABLE_OFFSET 2048
5516 #define CP_MEC_TABLE_OFFSET 4096
5518 void cik_init_cp_pg_table(struct radeon_device *rdev)
5520 const __be32 *fw_data;
5521 volatile u32 *dst_ptr;
5522 int me, i, max_me = 4;
5526 if (rdev->family == CHIP_KAVERI)
5529 if (rdev->rlc.cp_table_ptr == NULL)
5532 /* write the cp table buffer */
5533 dst_ptr = rdev->rlc.cp_table_ptr;
5534 for (me = 0; me < max_me; me++) {
5536 fw_data = (const __be32 *)rdev->ce_fw->data;
5537 table_offset = CP_ME_TABLE_OFFSET;
5538 } else if (me == 1) {
5539 fw_data = (const __be32 *)rdev->pfp_fw->data;
5540 table_offset = CP_ME_TABLE_OFFSET;
5541 } else if (me == 2) {
5542 fw_data = (const __be32 *)rdev->me_fw->data;
5543 table_offset = CP_ME_TABLE_OFFSET;
5545 fw_data = (const __be32 *)rdev->mec_fw->data;
5546 table_offset = CP_MEC_TABLE_OFFSET;
5549 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
5550 dst_ptr[bo_offset + i] = be32_to_cpu(fw_data[table_offset + i]);
5552 bo_offset += CP_ME_TABLE_SIZE;
5556 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
5561 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
5562 orig = data = RREG32(RLC_PG_CNTL);
5563 data |= GFX_PG_ENABLE;
5565 WREG32(RLC_PG_CNTL, data);
5567 orig = data = RREG32(RLC_AUTO_PG_CTRL);
5570 WREG32(RLC_AUTO_PG_CTRL, data);
5572 orig = data = RREG32(RLC_PG_CNTL);
5573 data &= ~GFX_PG_ENABLE;
5575 WREG32(RLC_PG_CNTL, data);
5577 orig = data = RREG32(RLC_AUTO_PG_CTRL);
5578 data &= ~AUTO_PG_EN;
5580 WREG32(RLC_AUTO_PG_CTRL, data);
5582 data = RREG32(DB_RENDER_CONTROL);
5586 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5588 u32 mask = 0, tmp, tmp1;
5591 cik_select_se_sh(rdev, se, sh);
5592 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5593 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5594 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5601 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
5606 return (~tmp) & mask;
5609 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
5611 u32 i, j, k, active_cu_number = 0;
5612 u32 mask, counter, cu_bitmap;
5615 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5616 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5620 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
5621 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
5629 active_cu_number += counter;
5630 tmp |= (cu_bitmap << (i * 16 + j * 8));
5634 WREG32(RLC_PG_AO_CU_MASK, tmp);
5636 tmp = RREG32(RLC_MAX_PG_CU);
5637 tmp &= ~MAX_PU_CU_MASK;
5638 tmp |= MAX_PU_CU(active_cu_number);
5639 WREG32(RLC_MAX_PG_CU, tmp);
5642 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
5647 orig = data = RREG32(RLC_PG_CNTL);
5648 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
5649 data |= STATIC_PER_CU_PG_ENABLE;
5651 data &= ~STATIC_PER_CU_PG_ENABLE;
5653 WREG32(RLC_PG_CNTL, data);
5656 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
5661 orig = data = RREG32(RLC_PG_CNTL);
5662 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
5663 data |= DYN_PER_CU_PG_ENABLE;
5665 data &= ~DYN_PER_CU_PG_ENABLE;
5667 WREG32(RLC_PG_CNTL, data);
5670 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
5671 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
5673 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
5678 if (rdev->rlc.cs_data) {
5679 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
5680 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
5681 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
5682 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
5684 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
5685 for (i = 0; i < 3; i++)
5686 WREG32(RLC_GPM_SCRATCH_DATA, 0);
5688 if (rdev->rlc.reg_list) {
5689 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
5690 for (i = 0; i < rdev->rlc.reg_list_size; i++)
5691 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
5694 orig = data = RREG32(RLC_PG_CNTL);
5697 WREG32(RLC_PG_CNTL, data);
5699 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5700 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
5702 data = RREG32(CP_RB_WPTR_POLL_CNTL);
5703 data &= ~IDLE_POLL_COUNT_MASK;
5704 data |= IDLE_POLL_COUNT(0x60);
5705 WREG32(CP_RB_WPTR_POLL_CNTL, data);
5708 WREG32(RLC_PG_DELAY, data);
5710 data = RREG32(RLC_PG_DELAY_2);
5713 WREG32(RLC_PG_DELAY_2, data);
5715 data = RREG32(RLC_AUTO_PG_CTRL);
5716 data &= ~GRBM_REG_SGIT_MASK;
5717 data |= GRBM_REG_SGIT(0x700);
5718 WREG32(RLC_AUTO_PG_CTRL, data);
5722 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
5724 cik_enable_gfx_cgpg(rdev, enable);
5725 cik_enable_gfx_static_mgpg(rdev, enable);
5726 cik_enable_gfx_dynamic_mgpg(rdev, enable);
5729 u32 cik_get_csb_size(struct radeon_device *rdev)
5732 const struct cs_section_def *sect = NULL;
5733 const struct cs_extent_def *ext = NULL;
5735 if (rdev->rlc.cs_data == NULL)
5738 /* begin clear state */
5740 /* context control state */
5743 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5744 for (ext = sect->section; ext->extent != NULL; ++ext) {
5745 if (sect->id == SECT_CONTEXT)
5746 count += 2 + ext->reg_count;
5751 /* pa_sc_raster_config/pa_sc_raster_config1 */
5753 /* end clear state */
5761 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5764 const struct cs_section_def *sect = NULL;
5765 const struct cs_extent_def *ext = NULL;
5767 if (rdev->rlc.cs_data == NULL)
5772 buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5773 buffer[count++] = PACKET3_PREAMBLE_BEGIN_CLEAR_STATE;
5775 buffer[count++] = PACKET3(PACKET3_CONTEXT_CONTROL, 1);
5776 buffer[count++] = 0x80000000;
5777 buffer[count++] = 0x80000000;
5779 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5780 for (ext = sect->section; ext->extent != NULL; ++ext) {
5781 if (sect->id == SECT_CONTEXT) {
5782 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count);
5783 buffer[count++] = ext->reg_index - 0xa000;
5784 for (i = 0; i < ext->reg_count; i++)
5785 buffer[count++] = ext->extent[i];
5792 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, 2);
5793 buffer[count++] = PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START;
5794 switch (rdev->family) {
5796 buffer[count++] = 0x16000012;
5797 buffer[count++] = 0x00000000;
5800 buffer[count++] = 0x00000000; /* XXX */
5801 buffer[count++] = 0x00000000;
5804 buffer[count++] = 0x00000000; /* XXX */
5805 buffer[count++] = 0x00000000;
5808 buffer[count++] = 0x00000000;
5809 buffer[count++] = 0x00000000;
5813 buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5814 buffer[count++] = PACKET3_PREAMBLE_END_CLEAR_STATE;
5816 buffer[count++] = PACKET3(PACKET3_CLEAR_STATE, 0);
5817 buffer[count++] = 0;
5820 static void cik_init_pg(struct radeon_device *rdev)
5822 if (rdev->pg_flags) {
5823 cik_enable_sck_slowdown_on_pu(rdev, true);
5824 cik_enable_sck_slowdown_on_pd(rdev, true);
5825 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5826 cik_init_gfx_cgpg(rdev);
5827 cik_enable_cp_pg(rdev, true);
5828 cik_enable_gds_pg(rdev, true);
5830 cik_init_ao_cu_mask(rdev);
5831 cik_update_gfx_pg(rdev, true);
5835 static void cik_fini_pg(struct radeon_device *rdev)
5837 if (rdev->pg_flags) {
5838 cik_update_gfx_pg(rdev, false);
5839 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5840 cik_enable_cp_pg(rdev, false);
5841 cik_enable_gds_pg(rdev, false);
5848 * Starting with r6xx, interrupts are handled via a ring buffer.
5849 * Ring buffers are areas of GPU accessible memory that the GPU
5850 * writes interrupt vectors into and the host reads vectors out of.
5851 * There is a rptr (read pointer) that determines where the
5852 * host is currently reading, and a wptr (write pointer)
5853 * which determines where the GPU has written. When the
5854 * pointers are equal, the ring is idle. When the GPU
5855 * writes vectors to the ring buffer, it increments the
5856 * wptr. When there is an interrupt, the host then starts
5857 * fetching commands and processing them until the pointers are
5858 * equal again at which point it updates the rptr.
5862 * cik_enable_interrupts - Enable the interrupt ring buffer
5864 * @rdev: radeon_device pointer
5866 * Enable the interrupt ring buffer (CIK).
5868 static void cik_enable_interrupts(struct radeon_device *rdev)
5870 u32 ih_cntl = RREG32(IH_CNTL);
5871 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5873 ih_cntl |= ENABLE_INTR;
5874 ih_rb_cntl |= IH_RB_ENABLE;
5875 WREG32(IH_CNTL, ih_cntl);
5876 WREG32(IH_RB_CNTL, ih_rb_cntl);
5877 rdev->ih.enabled = true;
5881 * cik_disable_interrupts - Disable the interrupt ring buffer
5883 * @rdev: radeon_device pointer
5885 * Disable the interrupt ring buffer (CIK).
5887 static void cik_disable_interrupts(struct radeon_device *rdev)
5889 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5890 u32 ih_cntl = RREG32(IH_CNTL);
5892 ih_rb_cntl &= ~IH_RB_ENABLE;
5893 ih_cntl &= ~ENABLE_INTR;
5894 WREG32(IH_RB_CNTL, ih_rb_cntl);
5895 WREG32(IH_CNTL, ih_cntl);
5896 /* set rptr, wptr to 0 */
5897 WREG32(IH_RB_RPTR, 0);
5898 WREG32(IH_RB_WPTR, 0);
5899 rdev->ih.enabled = false;
5904 * cik_disable_interrupt_state - Disable all interrupt sources
5906 * @rdev: radeon_device pointer
5908 * Clear all interrupt enable bits used by the driver (CIK).
5910 static void cik_disable_interrupt_state(struct radeon_device *rdev)
5915 tmp = RREG32(CP_INT_CNTL_RING0) &
5916 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5917 WREG32(CP_INT_CNTL_RING0, tmp);
5919 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5920 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5921 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5922 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5923 /* compute queues */
5924 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
5925 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
5926 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
5927 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
5928 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
5929 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
5930 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
5931 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
5933 WREG32(GRBM_INT_CNTL, 0);
5934 /* vline/vblank, etc. */
5935 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5936 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5937 if (rdev->num_crtc >= 4) {
5938 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5939 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5941 if (rdev->num_crtc >= 6) {
5942 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5943 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5947 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5949 /* digital hotplug */
5950 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5951 WREG32(DC_HPD1_INT_CONTROL, tmp);
5952 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5953 WREG32(DC_HPD2_INT_CONTROL, tmp);
5954 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5955 WREG32(DC_HPD3_INT_CONTROL, tmp);
5956 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5957 WREG32(DC_HPD4_INT_CONTROL, tmp);
5958 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5959 WREG32(DC_HPD5_INT_CONTROL, tmp);
5960 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5961 WREG32(DC_HPD6_INT_CONTROL, tmp);
5966 * cik_irq_init - init and enable the interrupt ring
5968 * @rdev: radeon_device pointer
5970 * Allocate a ring buffer for the interrupt controller,
5971 * enable the RLC, disable interrupts, enable the IH
5972 * ring buffer and enable it (CIK).
5973 * Called at device load and reume.
5974 * Returns 0 for success, errors for failure.
5976 static int cik_irq_init(struct radeon_device *rdev)
5980 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5983 ret = r600_ih_ring_alloc(rdev);
5988 cik_disable_interrupts(rdev);
5991 ret = cik_rlc_resume(rdev);
5993 r600_ih_ring_fini(rdev);
5997 /* setup interrupt control */
5998 /* XXX this should actually be a bus address, not an MC address. same on older asics */
5999 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6000 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6001 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6002 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6004 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6005 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6006 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6007 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6009 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
6010 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
6012 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6013 IH_WPTR_OVERFLOW_CLEAR |
6016 if (rdev->wb.enabled)
6017 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6019 /* set the writeback address whether it's enabled or not */
6020 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6021 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6023 WREG32(IH_RB_CNTL, ih_rb_cntl);
6025 /* set rptr, wptr to 0 */
6026 WREG32(IH_RB_RPTR, 0);
6027 WREG32(IH_RB_WPTR, 0);
6029 /* Default settings for IH_CNTL (disabled at first) */
6030 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6031 /* RPTR_REARM only works if msi's are enabled */
6032 if (rdev->msi_enabled)
6033 ih_cntl |= RPTR_REARM;
6034 WREG32(IH_CNTL, ih_cntl);
6036 /* force the active interrupt state to all disabled */
6037 cik_disable_interrupt_state(rdev);
6039 pci_set_master(rdev->pdev);
6042 cik_enable_interrupts(rdev);
6048 * cik_irq_set - enable/disable interrupt sources
6050 * @rdev: radeon_device pointer
6052 * Enable interrupt sources on the GPU (vblanks, hpd,
6054 * Returns 0 for success, errors for failure.
6056 int cik_irq_set(struct radeon_device *rdev)
6059 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6060 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
6061 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6062 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6063 u32 grbm_int_cntl = 0;
6064 u32 dma_cntl, dma_cntl1;
6067 if (!rdev->irq.installed) {
6068 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6071 /* don't enable anything if the ih is disabled */
6072 if (!rdev->ih.enabled) {
6073 cik_disable_interrupts(rdev);
6074 /* force the active interrupt state to all disabled */
6075 cik_disable_interrupt_state(rdev);
6079 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6080 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6081 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6083 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6084 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6085 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6086 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6087 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6088 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6090 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6091 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6093 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6094 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6095 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6096 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6097 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6098 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6099 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6100 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6102 if (rdev->flags & RADEON_IS_IGP)
6103 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6104 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6106 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6107 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6109 /* enable CP interrupts on all rings */
6110 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6111 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6112 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6114 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6115 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6116 DRM_DEBUG("si_irq_set: sw int cp1\n");
6117 if (ring->me == 1) {
6118 switch (ring->pipe) {
6120 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6123 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6126 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6129 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6132 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6135 } else if (ring->me == 2) {
6136 switch (ring->pipe) {
6138 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6141 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6144 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6147 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6150 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6154 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6157 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6158 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6159 DRM_DEBUG("si_irq_set: sw int cp2\n");
6160 if (ring->me == 1) {
6161 switch (ring->pipe) {
6163 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6166 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6169 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6172 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6175 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6178 } else if (ring->me == 2) {
6179 switch (ring->pipe) {
6181 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6184 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6187 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6190 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6193 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6197 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6201 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6202 DRM_DEBUG("cik_irq_set: sw int dma\n");
6203 dma_cntl |= TRAP_ENABLE;
6206 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6207 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6208 dma_cntl1 |= TRAP_ENABLE;
6211 if (rdev->irq.crtc_vblank_int[0] ||
6212 atomic_read(&rdev->irq.pflip[0])) {
6213 DRM_DEBUG("cik_irq_set: vblank 0\n");
6214 crtc1 |= VBLANK_INTERRUPT_MASK;
6216 if (rdev->irq.crtc_vblank_int[1] ||
6217 atomic_read(&rdev->irq.pflip[1])) {
6218 DRM_DEBUG("cik_irq_set: vblank 1\n");
6219 crtc2 |= VBLANK_INTERRUPT_MASK;
6221 if (rdev->irq.crtc_vblank_int[2] ||
6222 atomic_read(&rdev->irq.pflip[2])) {
6223 DRM_DEBUG("cik_irq_set: vblank 2\n");
6224 crtc3 |= VBLANK_INTERRUPT_MASK;
6226 if (rdev->irq.crtc_vblank_int[3] ||
6227 atomic_read(&rdev->irq.pflip[3])) {
6228 DRM_DEBUG("cik_irq_set: vblank 3\n");
6229 crtc4 |= VBLANK_INTERRUPT_MASK;
6231 if (rdev->irq.crtc_vblank_int[4] ||
6232 atomic_read(&rdev->irq.pflip[4])) {
6233 DRM_DEBUG("cik_irq_set: vblank 4\n");
6234 crtc5 |= VBLANK_INTERRUPT_MASK;
6236 if (rdev->irq.crtc_vblank_int[5] ||
6237 atomic_read(&rdev->irq.pflip[5])) {
6238 DRM_DEBUG("cik_irq_set: vblank 5\n");
6239 crtc6 |= VBLANK_INTERRUPT_MASK;
6241 if (rdev->irq.hpd[0]) {
6242 DRM_DEBUG("cik_irq_set: hpd 1\n");
6243 hpd1 |= DC_HPDx_INT_EN;
6245 if (rdev->irq.hpd[1]) {
6246 DRM_DEBUG("cik_irq_set: hpd 2\n");
6247 hpd2 |= DC_HPDx_INT_EN;
6249 if (rdev->irq.hpd[2]) {
6250 DRM_DEBUG("cik_irq_set: hpd 3\n");
6251 hpd3 |= DC_HPDx_INT_EN;
6253 if (rdev->irq.hpd[3]) {
6254 DRM_DEBUG("cik_irq_set: hpd 4\n");
6255 hpd4 |= DC_HPDx_INT_EN;
6257 if (rdev->irq.hpd[4]) {
6258 DRM_DEBUG("cik_irq_set: hpd 5\n");
6259 hpd5 |= DC_HPDx_INT_EN;
6261 if (rdev->irq.hpd[5]) {
6262 DRM_DEBUG("cik_irq_set: hpd 6\n");
6263 hpd6 |= DC_HPDx_INT_EN;
6266 if (rdev->irq.dpm_thermal) {
6267 DRM_DEBUG("dpm thermal\n");
6268 if (rdev->flags & RADEON_IS_IGP)
6269 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
6271 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
6274 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6276 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
6277 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
6279 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
6280 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
6281 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
6282 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
6283 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
6284 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
6285 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
6286 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
6288 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6290 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6291 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6292 if (rdev->num_crtc >= 4) {
6293 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6294 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6296 if (rdev->num_crtc >= 6) {
6297 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6298 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6301 WREG32(DC_HPD1_INT_CONTROL, hpd1);
6302 WREG32(DC_HPD2_INT_CONTROL, hpd2);
6303 WREG32(DC_HPD3_INT_CONTROL, hpd3);
6304 WREG32(DC_HPD4_INT_CONTROL, hpd4);
6305 WREG32(DC_HPD5_INT_CONTROL, hpd5);
6306 WREG32(DC_HPD6_INT_CONTROL, hpd6);
6308 if (rdev->flags & RADEON_IS_IGP)
6309 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
6311 WREG32_SMC(CG_THERMAL_INT, thermal_int);
6317 * cik_irq_ack - ack interrupt sources
6319 * @rdev: radeon_device pointer
6321 * Ack interrupt sources on the GPU (vblanks, hpd,
6322 * etc.) (CIK). Certain interrupts sources are sw
6323 * generated and do not require an explicit ack.
6325 static inline void cik_irq_ack(struct radeon_device *rdev)
6329 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6330 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6331 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6332 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6333 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6334 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6335 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
6337 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
6338 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6339 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
6340 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6341 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6342 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6343 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6344 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6346 if (rdev->num_crtc >= 4) {
6347 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6348 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6349 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6350 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6351 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6352 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6353 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6354 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6357 if (rdev->num_crtc >= 6) {
6358 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6359 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6360 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6361 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6362 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6363 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6364 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6365 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6368 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
6369 tmp = RREG32(DC_HPD1_INT_CONTROL);
6370 tmp |= DC_HPDx_INT_ACK;
6371 WREG32(DC_HPD1_INT_CONTROL, tmp);
6373 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
6374 tmp = RREG32(DC_HPD2_INT_CONTROL);
6375 tmp |= DC_HPDx_INT_ACK;
6376 WREG32(DC_HPD2_INT_CONTROL, tmp);
6378 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6379 tmp = RREG32(DC_HPD3_INT_CONTROL);
6380 tmp |= DC_HPDx_INT_ACK;
6381 WREG32(DC_HPD3_INT_CONTROL, tmp);
6383 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6384 tmp = RREG32(DC_HPD4_INT_CONTROL);
6385 tmp |= DC_HPDx_INT_ACK;
6386 WREG32(DC_HPD4_INT_CONTROL, tmp);
6388 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6389 tmp = RREG32(DC_HPD5_INT_CONTROL);
6390 tmp |= DC_HPDx_INT_ACK;
6391 WREG32(DC_HPD5_INT_CONTROL, tmp);
6393 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6394 tmp = RREG32(DC_HPD5_INT_CONTROL);
6395 tmp |= DC_HPDx_INT_ACK;
6396 WREG32(DC_HPD6_INT_CONTROL, tmp);
6401 * cik_irq_disable - disable interrupts
6403 * @rdev: radeon_device pointer
6405 * Disable interrupts on the hw (CIK).
6407 static void cik_irq_disable(struct radeon_device *rdev)
6409 cik_disable_interrupts(rdev);
6410 /* Wait and acknowledge irq */
6413 cik_disable_interrupt_state(rdev);
6417 * cik_irq_disable - disable interrupts for suspend
6419 * @rdev: radeon_device pointer
6421 * Disable interrupts and stop the RLC (CIK).
6424 static void cik_irq_suspend(struct radeon_device *rdev)
6426 cik_irq_disable(rdev);
6431 * cik_irq_fini - tear down interrupt support
6433 * @rdev: radeon_device pointer
6435 * Disable interrupts on the hw and free the IH ring
6437 * Used for driver unload.
6439 static void cik_irq_fini(struct radeon_device *rdev)
6441 cik_irq_suspend(rdev);
6442 r600_ih_ring_fini(rdev);
6446 * cik_get_ih_wptr - get the IH ring buffer wptr
6448 * @rdev: radeon_device pointer
6450 * Get the IH ring buffer wptr from either the register
6451 * or the writeback memory buffer (CIK). Also check for
6452 * ring buffer overflow and deal with it.
6453 * Used by cik_irq_process().
6454 * Returns the value of the wptr.
6456 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
6460 if (rdev->wb.enabled)
6461 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6463 wptr = RREG32(IH_RB_WPTR);
6465 if (wptr & RB_OVERFLOW) {
6466 /* When a ring buffer overflow happen start parsing interrupt
6467 * from the last not overwritten vector (wptr + 16). Hopefully
6468 * this should allow us to catchup.
6470 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6471 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6472 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6473 tmp = RREG32(IH_RB_CNTL);
6474 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6475 WREG32(IH_RB_CNTL, tmp);
6477 return (wptr & rdev->ih.ptr_mask);
6481 * Each IV ring entry is 128 bits:
6482 * [7:0] - interrupt source id
6484 * [59:32] - interrupt source data
6485 * [63:60] - reserved
6488 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
6489 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
6490 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
6491 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
6492 * PIPE_ID - ME0 0=3D
6493 * - ME1&2 compute dispatcher (4 pipes each)
6495 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
6496 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
6497 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
6500 * [127:96] - reserved
6503 * cik_irq_process - interrupt handler
6505 * @rdev: radeon_device pointer
6507 * Interrupt hander (CIK). Walk the IH ring,
6508 * ack interrupts and schedule work to handle
6510 * Returns irq process return code.
6512 int cik_irq_process(struct radeon_device *rdev)
6514 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6515 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6518 u32 src_id, src_data, ring_id;
6519 u8 me_id, pipe_id, queue_id;
6521 bool queue_hotplug = false;
6522 bool queue_reset = false;
6523 u32 addr, status, mc_client;
6524 bool queue_thermal = false;
6526 if (!rdev->ih.enabled || rdev->shutdown)
6529 wptr = cik_get_ih_wptr(rdev);
6532 /* is somebody else already processing irqs? */
6533 if (atomic_xchg(&rdev->ih.lock, 1))
6536 rptr = rdev->ih.rptr;
6537 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6539 /* Order reading of wptr vs. reading of IH ring data */
6542 /* display interrupts */
6545 while (rptr != wptr) {
6546 /* wptr/rptr are in bytes! */
6547 ring_index = rptr / 4;
6548 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6549 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6550 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6553 case 1: /* D1 vblank/vline */
6555 case 0: /* D1 vblank */
6556 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
6557 if (rdev->irq.crtc_vblank_int[0]) {
6558 drm_handle_vblank(rdev->ddev, 0);
6559 rdev->pm.vblank_sync = true;
6560 wake_up(&rdev->irq.vblank_queue);
6562 if (atomic_read(&rdev->irq.pflip[0]))
6563 radeon_crtc_handle_flip(rdev, 0);
6564 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6565 DRM_DEBUG("IH: D1 vblank\n");
6568 case 1: /* D1 vline */
6569 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
6570 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6571 DRM_DEBUG("IH: D1 vline\n");
6575 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6579 case 2: /* D2 vblank/vline */
6581 case 0: /* D2 vblank */
6582 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6583 if (rdev->irq.crtc_vblank_int[1]) {
6584 drm_handle_vblank(rdev->ddev, 1);
6585 rdev->pm.vblank_sync = true;
6586 wake_up(&rdev->irq.vblank_queue);
6588 if (atomic_read(&rdev->irq.pflip[1]))
6589 radeon_crtc_handle_flip(rdev, 1);
6590 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6591 DRM_DEBUG("IH: D2 vblank\n");
6594 case 1: /* D2 vline */
6595 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6596 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6597 DRM_DEBUG("IH: D2 vline\n");
6601 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6605 case 3: /* D3 vblank/vline */
6607 case 0: /* D3 vblank */
6608 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6609 if (rdev->irq.crtc_vblank_int[2]) {
6610 drm_handle_vblank(rdev->ddev, 2);
6611 rdev->pm.vblank_sync = true;
6612 wake_up(&rdev->irq.vblank_queue);
6614 if (atomic_read(&rdev->irq.pflip[2]))
6615 radeon_crtc_handle_flip(rdev, 2);
6616 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6617 DRM_DEBUG("IH: D3 vblank\n");
6620 case 1: /* D3 vline */
6621 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6622 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6623 DRM_DEBUG("IH: D3 vline\n");
6627 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6631 case 4: /* D4 vblank/vline */
6633 case 0: /* D4 vblank */
6634 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6635 if (rdev->irq.crtc_vblank_int[3]) {
6636 drm_handle_vblank(rdev->ddev, 3);
6637 rdev->pm.vblank_sync = true;
6638 wake_up(&rdev->irq.vblank_queue);
6640 if (atomic_read(&rdev->irq.pflip[3]))
6641 radeon_crtc_handle_flip(rdev, 3);
6642 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6643 DRM_DEBUG("IH: D4 vblank\n");
6646 case 1: /* D4 vline */
6647 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6648 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6649 DRM_DEBUG("IH: D4 vline\n");
6653 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6657 case 5: /* D5 vblank/vline */
6659 case 0: /* D5 vblank */
6660 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6661 if (rdev->irq.crtc_vblank_int[4]) {
6662 drm_handle_vblank(rdev->ddev, 4);
6663 rdev->pm.vblank_sync = true;
6664 wake_up(&rdev->irq.vblank_queue);
6666 if (atomic_read(&rdev->irq.pflip[4]))
6667 radeon_crtc_handle_flip(rdev, 4);
6668 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6669 DRM_DEBUG("IH: D5 vblank\n");
6672 case 1: /* D5 vline */
6673 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6674 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6675 DRM_DEBUG("IH: D5 vline\n");
6679 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6683 case 6: /* D6 vblank/vline */
6685 case 0: /* D6 vblank */
6686 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6687 if (rdev->irq.crtc_vblank_int[5]) {
6688 drm_handle_vblank(rdev->ddev, 5);
6689 rdev->pm.vblank_sync = true;
6690 wake_up(&rdev->irq.vblank_queue);
6692 if (atomic_read(&rdev->irq.pflip[5]))
6693 radeon_crtc_handle_flip(rdev, 5);
6694 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6695 DRM_DEBUG("IH: D6 vblank\n");
6698 case 1: /* D6 vline */
6699 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6700 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6701 DRM_DEBUG("IH: D6 vline\n");
6705 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6709 case 42: /* HPD hotplug */
6712 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
6713 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
6714 queue_hotplug = true;
6715 DRM_DEBUG("IH: HPD1\n");
6719 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
6720 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6721 queue_hotplug = true;
6722 DRM_DEBUG("IH: HPD2\n");
6726 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6727 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6728 queue_hotplug = true;
6729 DRM_DEBUG("IH: HPD3\n");
6733 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6734 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6735 queue_hotplug = true;
6736 DRM_DEBUG("IH: HPD4\n");
6740 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6741 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6742 queue_hotplug = true;
6743 DRM_DEBUG("IH: HPD5\n");
6747 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6748 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6749 queue_hotplug = true;
6750 DRM_DEBUG("IH: HPD6\n");
6754 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6759 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6760 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6764 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6765 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6766 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
6767 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6768 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
6770 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6772 cik_vm_decode_fault(rdev, status, addr, mc_client);
6773 /* reset addr and status */
6774 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6776 case 176: /* GFX RB CP_INT */
6777 case 177: /* GFX IB CP_INT */
6778 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6780 case 181: /* CP EOP event */
6781 DRM_DEBUG("IH: CP EOP\n");
6782 /* XXX check the bitfield order! */
6783 me_id = (ring_id & 0x60) >> 5;
6784 pipe_id = (ring_id & 0x18) >> 3;
6785 queue_id = (ring_id & 0x7) >> 0;
6788 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6792 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
6793 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6794 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
6795 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6799 case 184: /* CP Privileged reg access */
6800 DRM_ERROR("Illegal register access in command stream\n");
6801 /* XXX check the bitfield order! */
6802 me_id = (ring_id & 0x60) >> 5;
6803 pipe_id = (ring_id & 0x18) >> 3;
6804 queue_id = (ring_id & 0x7) >> 0;
6807 /* This results in a full GPU reset, but all we need to do is soft
6808 * reset the CP for gfx
6822 case 185: /* CP Privileged inst */
6823 DRM_ERROR("Illegal instruction in command stream\n");
6824 /* XXX check the bitfield order! */
6825 me_id = (ring_id & 0x60) >> 5;
6826 pipe_id = (ring_id & 0x18) >> 3;
6827 queue_id = (ring_id & 0x7) >> 0;
6830 /* This results in a full GPU reset, but all we need to do is soft
6831 * reset the CP for gfx
6845 case 224: /* SDMA trap event */
6846 /* XXX check the bitfield order! */
6847 me_id = (ring_id & 0x3) >> 0;
6848 queue_id = (ring_id & 0xc) >> 2;
6849 DRM_DEBUG("IH: SDMA trap\n");
6854 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6867 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6879 case 230: /* thermal low to high */
6880 DRM_DEBUG("IH: thermal low to high\n");
6881 rdev->pm.dpm.thermal.high_to_low = false;
6882 queue_thermal = true;
6884 case 231: /* thermal high to low */
6885 DRM_DEBUG("IH: thermal high to low\n");
6886 rdev->pm.dpm.thermal.high_to_low = true;
6887 queue_thermal = true;
6889 case 233: /* GUI IDLE */
6890 DRM_DEBUG("IH: GUI idle\n");
6892 case 241: /* SDMA Privileged inst */
6893 case 247: /* SDMA Privileged inst */
6894 DRM_ERROR("Illegal instruction in SDMA command stream\n");
6895 /* XXX check the bitfield order! */
6896 me_id = (ring_id & 0x3) >> 0;
6897 queue_id = (ring_id & 0xc) >> 2;
6932 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6936 /* wptr/rptr are in bytes! */
6938 rptr &= rdev->ih.ptr_mask;
6941 schedule_work(&rdev->hotplug_work);
6943 schedule_work(&rdev->reset_work);
6945 schedule_work(&rdev->pm.dpm.thermal.work);
6946 rdev->ih.rptr = rptr;
6947 WREG32(IH_RB_RPTR, rdev->ih.rptr);
6948 atomic_set(&rdev->ih.lock, 0);
6950 /* make sure wptr hasn't changed while processing */
6951 wptr = cik_get_ih_wptr(rdev);
6959 * startup/shutdown callbacks
6962 * cik_startup - program the asic to a functional state
6964 * @rdev: radeon_device pointer
6966 * Programs the asic to a functional state (CIK).
6967 * Called by cik_init() and cik_resume().
6968 * Returns 0 for success, error for failure.
6970 static int cik_startup(struct radeon_device *rdev)
6972 struct radeon_ring *ring;
6975 /* enable pcie gen2/3 link */
6976 cik_pcie_gen3_enable(rdev);
6978 cik_program_aspm(rdev);
6980 /* scratch needs to be initialized before MC */
6981 r = r600_vram_scratch_init(rdev);
6985 cik_mc_program(rdev);
6987 if (rdev->flags & RADEON_IS_IGP) {
6988 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6989 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
6990 r = cik_init_microcode(rdev);
6992 DRM_ERROR("Failed to load firmware!\n");
6997 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6998 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
7000 r = cik_init_microcode(rdev);
7002 DRM_ERROR("Failed to load firmware!\n");
7007 r = ci_mc_load_microcode(rdev);
7009 DRM_ERROR("Failed to load MC firmware!\n");
7014 r = cik_pcie_gart_enable(rdev);
7019 /* allocate rlc buffers */
7020 if (rdev->flags & RADEON_IS_IGP) {
7021 if (rdev->family == CHIP_KAVERI) {
7022 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7023 rdev->rlc.reg_list_size =
7024 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7026 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7027 rdev->rlc.reg_list_size =
7028 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7031 rdev->rlc.cs_data = ci_cs_data;
7032 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
7033 r = sumo_rlc_init(rdev);
7035 DRM_ERROR("Failed to init rlc BOs!\n");
7039 /* allocate wb buffer */
7040 r = radeon_wb_init(rdev);
7044 /* allocate mec buffers */
7045 r = cik_mec_init(rdev);
7047 DRM_ERROR("Failed to init MEC BOs!\n");
7051 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7053 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7057 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7059 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7063 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7065 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7069 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7071 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7075 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7077 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7081 r = radeon_uvd_resume(rdev);
7083 r = uvd_v4_2_resume(rdev);
7085 r = radeon_fence_driver_start_ring(rdev,
7086 R600_RING_TYPE_UVD_INDEX);
7088 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7092 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7095 if (!rdev->irq.installed) {
7096 r = radeon_irq_kms_init(rdev);
7101 r = cik_irq_init(rdev);
7103 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7104 radeon_irq_kms_fini(rdev);
7109 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7110 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7111 CP_RB0_RPTR, CP_RB0_WPTR,
7116 /* set up the compute queues */
7117 /* type-2 packets are deprecated on MEC, use type-3 instead */
7118 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7119 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7120 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
7121 PACKET3(PACKET3_NOP, 0x3FFF));
7124 ring->me = 1; /* first MEC */
7125 ring->pipe = 0; /* first pipe */
7126 ring->queue = 0; /* first queue */
7127 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7129 /* type-2 packets are deprecated on MEC, use type-3 instead */
7130 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7131 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7132 CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
7133 PACKET3(PACKET3_NOP, 0x3FFF));
7136 /* dGPU only have 1 MEC */
7137 ring->me = 1; /* first MEC */
7138 ring->pipe = 0; /* first pipe */
7139 ring->queue = 1; /* second queue */
7140 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7142 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7143 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7144 SDMA0_GFX_RB_RPTR + SDMA0_REGISTER_OFFSET,
7145 SDMA0_GFX_RB_WPTR + SDMA0_REGISTER_OFFSET,
7146 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7150 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7151 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7152 SDMA0_GFX_RB_RPTR + SDMA1_REGISTER_OFFSET,
7153 SDMA0_GFX_RB_WPTR + SDMA1_REGISTER_OFFSET,
7154 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7158 r = cik_cp_resume(rdev);
7162 r = cik_sdma_resume(rdev);
7166 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7167 if (ring->ring_size) {
7168 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7169 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
7172 r = uvd_v1_0_init(rdev);
7174 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7177 r = radeon_ib_pool_init(rdev);
7179 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7183 r = radeon_vm_manager_init(rdev);
7185 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7189 r = dce6_audio_init(rdev);
7197 * cik_resume - resume the asic to a functional state
7199 * @rdev: radeon_device pointer
7201 * Programs the asic to a functional state (CIK).
7203 * Returns 0 for success, error for failure.
7205 int cik_resume(struct radeon_device *rdev)
7210 atom_asic_init(rdev->mode_info.atom_context);
7212 /* init golden registers */
7213 cik_init_golden_registers(rdev);
7215 rdev->accel_working = true;
7216 r = cik_startup(rdev);
7218 DRM_ERROR("cik startup failed on resume\n");
7219 rdev->accel_working = false;
7228 * cik_suspend - suspend the asic
7230 * @rdev: radeon_device pointer
7232 * Bring the chip into a state suitable for suspend (CIK).
7233 * Called at suspend.
7234 * Returns 0 for success.
7236 int cik_suspend(struct radeon_device *rdev)
7238 dce6_audio_fini(rdev);
7239 radeon_vm_manager_fini(rdev);
7240 cik_cp_enable(rdev, false);
7241 cik_sdma_enable(rdev, false);
7242 uvd_v1_0_fini(rdev);
7243 radeon_uvd_suspend(rdev);
7246 cik_irq_suspend(rdev);
7247 radeon_wb_disable(rdev);
7248 cik_pcie_gart_disable(rdev);
7252 /* Plan is to move initialization in that function and use
7253 * helper function so that radeon_device_init pretty much
7254 * do nothing more than calling asic specific function. This
7255 * should also allow to remove a bunch of callback function
7259 * cik_init - asic specific driver and hw init
7261 * @rdev: radeon_device pointer
7263 * Setup asic specific driver variables and program the hw
7264 * to a functional state (CIK).
7265 * Called at driver startup.
7266 * Returns 0 for success, errors for failure.
7268 int cik_init(struct radeon_device *rdev)
7270 struct radeon_ring *ring;
7274 if (!radeon_get_bios(rdev)) {
7275 if (ASIC_IS_AVIVO(rdev))
7278 /* Must be an ATOMBIOS */
7279 if (!rdev->is_atom_bios) {
7280 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7283 r = radeon_atombios_init(rdev);
7287 /* Post card if necessary */
7288 if (!radeon_card_posted(rdev)) {
7290 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7293 DRM_INFO("GPU not posted. posting now...\n");
7294 atom_asic_init(rdev->mode_info.atom_context);
7296 /* init golden registers */
7297 cik_init_golden_registers(rdev);
7298 /* Initialize scratch registers */
7299 cik_scratch_init(rdev);
7300 /* Initialize surface registers */
7301 radeon_surface_init(rdev);
7302 /* Initialize clocks */
7303 radeon_get_clock_info(rdev->ddev);
7306 r = radeon_fence_driver_init(rdev);
7310 /* initialize memory controller */
7311 r = cik_mc_init(rdev);
7314 /* Memory manager */
7315 r = radeon_bo_init(rdev);
7319 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7320 ring->ring_obj = NULL;
7321 r600_ring_init(rdev, ring, 1024 * 1024);
7323 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7324 ring->ring_obj = NULL;
7325 r600_ring_init(rdev, ring, 1024 * 1024);
7326 r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
7330 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7331 ring->ring_obj = NULL;
7332 r600_ring_init(rdev, ring, 1024 * 1024);
7333 r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
7337 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7338 ring->ring_obj = NULL;
7339 r600_ring_init(rdev, ring, 256 * 1024);
7341 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7342 ring->ring_obj = NULL;
7343 r600_ring_init(rdev, ring, 256 * 1024);
7345 r = radeon_uvd_init(rdev);
7347 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7348 ring->ring_obj = NULL;
7349 r600_ring_init(rdev, ring, 4096);
7352 rdev->ih.ring_obj = NULL;
7353 r600_ih_ring_init(rdev, 64 * 1024);
7355 r = r600_pcie_gart_init(rdev);
7359 rdev->accel_working = true;
7360 r = cik_startup(rdev);
7362 dev_err(rdev->dev, "disabling GPU acceleration\n");
7364 cik_sdma_fini(rdev);
7366 sumo_rlc_fini(rdev);
7368 radeon_wb_fini(rdev);
7369 radeon_ib_pool_fini(rdev);
7370 radeon_vm_manager_fini(rdev);
7371 radeon_irq_kms_fini(rdev);
7372 cik_pcie_gart_fini(rdev);
7373 rdev->accel_working = false;
7376 /* Don't start up if the MC ucode is missing.
7377 * The default clocks and voltages before the MC ucode
7378 * is loaded are not suffient for advanced operations.
7380 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
7381 DRM_ERROR("radeon: MC ucode required for NI+.\n");
7389 * cik_fini - asic specific driver and hw fini
7391 * @rdev: radeon_device pointer
7393 * Tear down the asic specific driver variables and program the hw
7394 * to an idle state (CIK).
7395 * Called at driver unload.
7397 void cik_fini(struct radeon_device *rdev)
7400 cik_sdma_fini(rdev);
7404 sumo_rlc_fini(rdev);
7406 radeon_wb_fini(rdev);
7407 radeon_vm_manager_fini(rdev);
7408 radeon_ib_pool_fini(rdev);
7409 radeon_irq_kms_fini(rdev);
7410 uvd_v1_0_fini(rdev);
7411 radeon_uvd_fini(rdev);
7412 cik_pcie_gart_fini(rdev);
7413 r600_vram_scratch_fini(rdev);
7414 radeon_gem_fini(rdev);
7415 radeon_fence_driver_fini(rdev);
7416 radeon_bo_fini(rdev);
7417 radeon_atombios_fini(rdev);
7422 /* display watermark setup */
7424 * dce8_line_buffer_adjust - Set up the line buffer
7426 * @rdev: radeon_device pointer
7427 * @radeon_crtc: the selected display controller
7428 * @mode: the current display mode on the selected display
7431 * Setup up the line buffer allocation for
7432 * the selected display controller (CIK).
7433 * Returns the line buffer size in pixels.
7435 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
7436 struct radeon_crtc *radeon_crtc,
7437 struct drm_display_mode *mode)
7439 u32 tmp, buffer_alloc, i;
7440 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
7443 * There are 6 line buffers, one for each display controllers.
7444 * There are 3 partitions per LB. Select the number of partitions
7445 * to enable based on the display width. For display widths larger
7446 * than 4096, you need use to use 2 display controllers and combine
7447 * them using the stereo blender.
7449 if (radeon_crtc->base.enabled && mode) {
7450 if (mode->crtc_hdisplay < 1920) {
7453 } else if (mode->crtc_hdisplay < 2560) {
7456 } else if (mode->crtc_hdisplay < 4096) {
7458 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
7460 DRM_DEBUG_KMS("Mode too big for LB!\n");
7462 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
7469 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
7470 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
7472 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
7473 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
7474 for (i = 0; i < rdev->usec_timeout; i++) {
7475 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
7476 DMIF_BUFFERS_ALLOCATED_COMPLETED)
7481 if (radeon_crtc->base.enabled && mode) {
7493 /* controller not enabled, so no lb used */
7498 * cik_get_number_of_dram_channels - get the number of dram channels
7500 * @rdev: radeon_device pointer
7502 * Look up the number of video ram channels (CIK).
7503 * Used for display watermark bandwidth calculations
7504 * Returns the number of dram channels
7506 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
7508 u32 tmp = RREG32(MC_SHARED_CHMAP);
7510 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
7533 struct dce8_wm_params {
7534 u32 dram_channels; /* number of dram channels */
7535 u32 yclk; /* bandwidth per dram data pin in kHz */
7536 u32 sclk; /* engine clock in kHz */
7537 u32 disp_clk; /* display clock in kHz */
7538 u32 src_width; /* viewport width */
7539 u32 active_time; /* active display time in ns */
7540 u32 blank_time; /* blank time in ns */
7541 bool interlaced; /* mode is interlaced */
7542 fixed20_12 vsc; /* vertical scale ratio */
7543 u32 num_heads; /* number of active crtcs */
7544 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
7545 u32 lb_size; /* line buffer allocated to pipe */
7546 u32 vtaps; /* vertical scaler taps */
7550 * dce8_dram_bandwidth - get the dram bandwidth
7552 * @wm: watermark calculation data
7554 * Calculate the raw dram bandwidth (CIK).
7555 * Used for display watermark bandwidth calculations
7556 * Returns the dram bandwidth in MBytes/s
7558 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
7560 /* Calculate raw DRAM Bandwidth */
7561 fixed20_12 dram_efficiency; /* 0.7 */
7562 fixed20_12 yclk, dram_channels, bandwidth;
7565 a.full = dfixed_const(1000);
7566 yclk.full = dfixed_const(wm->yclk);
7567 yclk.full = dfixed_div(yclk, a);
7568 dram_channels.full = dfixed_const(wm->dram_channels * 4);
7569 a.full = dfixed_const(10);
7570 dram_efficiency.full = dfixed_const(7);
7571 dram_efficiency.full = dfixed_div(dram_efficiency, a);
7572 bandwidth.full = dfixed_mul(dram_channels, yclk);
7573 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
7575 return dfixed_trunc(bandwidth);
7579 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
7581 * @wm: watermark calculation data
7583 * Calculate the dram bandwidth used for display (CIK).
7584 * Used for display watermark bandwidth calculations
7585 * Returns the dram bandwidth for display in MBytes/s
7587 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
7589 /* Calculate DRAM Bandwidth and the part allocated to display. */
7590 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
7591 fixed20_12 yclk, dram_channels, bandwidth;
7594 a.full = dfixed_const(1000);
7595 yclk.full = dfixed_const(wm->yclk);
7596 yclk.full = dfixed_div(yclk, a);
7597 dram_channels.full = dfixed_const(wm->dram_channels * 4);
7598 a.full = dfixed_const(10);
7599 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
7600 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
7601 bandwidth.full = dfixed_mul(dram_channels, yclk);
7602 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
7604 return dfixed_trunc(bandwidth);
7608 * dce8_data_return_bandwidth - get the data return bandwidth
7610 * @wm: watermark calculation data
7612 * Calculate the data return bandwidth used for display (CIK).
7613 * Used for display watermark bandwidth calculations
7614 * Returns the data return bandwidth in MBytes/s
7616 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
7618 /* Calculate the display Data return Bandwidth */
7619 fixed20_12 return_efficiency; /* 0.8 */
7620 fixed20_12 sclk, bandwidth;
7623 a.full = dfixed_const(1000);
7624 sclk.full = dfixed_const(wm->sclk);
7625 sclk.full = dfixed_div(sclk, a);
7626 a.full = dfixed_const(10);
7627 return_efficiency.full = dfixed_const(8);
7628 return_efficiency.full = dfixed_div(return_efficiency, a);
7629 a.full = dfixed_const(32);
7630 bandwidth.full = dfixed_mul(a, sclk);
7631 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
7633 return dfixed_trunc(bandwidth);
7637 * dce8_dmif_request_bandwidth - get the dmif bandwidth
7639 * @wm: watermark calculation data
7641 * Calculate the dmif bandwidth used for display (CIK).
7642 * Used for display watermark bandwidth calculations
7643 * Returns the dmif bandwidth in MBytes/s
7645 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
7647 /* Calculate the DMIF Request Bandwidth */
7648 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
7649 fixed20_12 disp_clk, bandwidth;
7652 a.full = dfixed_const(1000);
7653 disp_clk.full = dfixed_const(wm->disp_clk);
7654 disp_clk.full = dfixed_div(disp_clk, a);
7655 a.full = dfixed_const(32);
7656 b.full = dfixed_mul(a, disp_clk);
7658 a.full = dfixed_const(10);
7659 disp_clk_request_efficiency.full = dfixed_const(8);
7660 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
7662 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
7664 return dfixed_trunc(bandwidth);
7668 * dce8_available_bandwidth - get the min available bandwidth
7670 * @wm: watermark calculation data
7672 * Calculate the min available bandwidth used for display (CIK).
7673 * Used for display watermark bandwidth calculations
7674 * Returns the min available bandwidth in MBytes/s
7676 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
7678 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
7679 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
7680 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
7681 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
7683 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
7687 * dce8_average_bandwidth - get the average available bandwidth
7689 * @wm: watermark calculation data
7691 * Calculate the average available bandwidth used for display (CIK).
7692 * Used for display watermark bandwidth calculations
7693 * Returns the average available bandwidth in MBytes/s
7695 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
7697 /* Calculate the display mode Average Bandwidth
7698 * DisplayMode should contain the source and destination dimensions,
7702 fixed20_12 line_time;
7703 fixed20_12 src_width;
7704 fixed20_12 bandwidth;
7707 a.full = dfixed_const(1000);
7708 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
7709 line_time.full = dfixed_div(line_time, a);
7710 bpp.full = dfixed_const(wm->bytes_per_pixel);
7711 src_width.full = dfixed_const(wm->src_width);
7712 bandwidth.full = dfixed_mul(src_width, bpp);
7713 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
7714 bandwidth.full = dfixed_div(bandwidth, line_time);
7716 return dfixed_trunc(bandwidth);
7720 * dce8_latency_watermark - get the latency watermark
7722 * @wm: watermark calculation data
7724 * Calculate the latency watermark (CIK).
7725 * Used for display watermark bandwidth calculations
7726 * Returns the latency watermark in ns
7728 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
7730 /* First calculate the latency in ns */
7731 u32 mc_latency = 2000; /* 2000 ns. */
7732 u32 available_bandwidth = dce8_available_bandwidth(wm);
7733 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
7734 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
7735 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
7736 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
7737 (wm->num_heads * cursor_line_pair_return_time);
7738 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
7739 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
7740 u32 tmp, dmif_size = 12288;
7743 if (wm->num_heads == 0)
7746 a.full = dfixed_const(2);
7747 b.full = dfixed_const(1);
7748 if ((wm->vsc.full > a.full) ||
7749 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
7751 ((wm->vsc.full >= a.full) && wm->interlaced))
7752 max_src_lines_per_dst_line = 4;
7754 max_src_lines_per_dst_line = 2;
7756 a.full = dfixed_const(available_bandwidth);
7757 b.full = dfixed_const(wm->num_heads);
7758 a.full = dfixed_div(a, b);
7760 b.full = dfixed_const(mc_latency + 512);
7761 c.full = dfixed_const(wm->disp_clk);
7762 b.full = dfixed_div(b, c);
7764 c.full = dfixed_const(dmif_size);
7765 b.full = dfixed_div(c, b);
7767 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
7769 b.full = dfixed_const(1000);
7770 c.full = dfixed_const(wm->disp_clk);
7771 b.full = dfixed_div(c, b);
7772 c.full = dfixed_const(wm->bytes_per_pixel);
7773 b.full = dfixed_mul(b, c);
7775 lb_fill_bw = min(tmp, dfixed_trunc(b));
7777 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
7778 b.full = dfixed_const(1000);
7779 c.full = dfixed_const(lb_fill_bw);
7780 b.full = dfixed_div(c, b);
7781 a.full = dfixed_div(a, b);
7782 line_fill_time = dfixed_trunc(a);
7784 if (line_fill_time < wm->active_time)
7787 return latency + (line_fill_time - wm->active_time);
7792 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
7793 * average and available dram bandwidth
7795 * @wm: watermark calculation data
7797 * Check if the display average bandwidth fits in the display
7798 * dram bandwidth (CIK).
7799 * Used for display watermark bandwidth calculations
7800 * Returns true if the display fits, false if not.
7802 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
7804 if (dce8_average_bandwidth(wm) <=
7805 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
7812 * dce8_average_bandwidth_vs_available_bandwidth - check
7813 * average and available bandwidth
7815 * @wm: watermark calculation data
7817 * Check if the display average bandwidth fits in the display
7818 * available bandwidth (CIK).
7819 * Used for display watermark bandwidth calculations
7820 * Returns true if the display fits, false if not.
7822 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
7824 if (dce8_average_bandwidth(wm) <=
7825 (dce8_available_bandwidth(wm) / wm->num_heads))
7832 * dce8_check_latency_hiding - check latency hiding
7834 * @wm: watermark calculation data
7836 * Check latency hiding (CIK).
7837 * Used for display watermark bandwidth calculations
7838 * Returns true if the display fits, false if not.
7840 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
7842 u32 lb_partitions = wm->lb_size / wm->src_width;
7843 u32 line_time = wm->active_time + wm->blank_time;
7844 u32 latency_tolerant_lines;
7848 a.full = dfixed_const(1);
7849 if (wm->vsc.full > a.full)
7850 latency_tolerant_lines = 1;
7852 if (lb_partitions <= (wm->vtaps + 1))
7853 latency_tolerant_lines = 1;
7855 latency_tolerant_lines = 2;
7858 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
7860 if (dce8_latency_watermark(wm) <= latency_hiding)
7867 * dce8_program_watermarks - program display watermarks
7869 * @rdev: radeon_device pointer
7870 * @radeon_crtc: the selected display controller
7871 * @lb_size: line buffer size
7872 * @num_heads: number of display controllers in use
7874 * Calculate and program the display watermarks for the
7875 * selected display controller (CIK).
7877 static void dce8_program_watermarks(struct radeon_device *rdev,
7878 struct radeon_crtc *radeon_crtc,
7879 u32 lb_size, u32 num_heads)
7881 struct drm_display_mode *mode = &radeon_crtc->base.mode;
7882 struct dce8_wm_params wm_low, wm_high;
7885 u32 latency_watermark_a = 0, latency_watermark_b = 0;
7888 if (radeon_crtc->base.enabled && num_heads && mode) {
7889 pixel_period = 1000000 / (u32)mode->clock;
7890 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
7892 /* watermark for high clocks */
7893 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
7894 rdev->pm.dpm_enabled) {
7896 radeon_dpm_get_mclk(rdev, false) * 10;
7898 radeon_dpm_get_sclk(rdev, false) * 10;
7900 wm_high.yclk = rdev->pm.current_mclk * 10;
7901 wm_high.sclk = rdev->pm.current_sclk * 10;
7904 wm_high.disp_clk = mode->clock;
7905 wm_high.src_width = mode->crtc_hdisplay;
7906 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
7907 wm_high.blank_time = line_time - wm_high.active_time;
7908 wm_high.interlaced = false;
7909 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
7910 wm_high.interlaced = true;
7911 wm_high.vsc = radeon_crtc->vsc;
7913 if (radeon_crtc->rmx_type != RMX_OFF)
7915 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
7916 wm_high.lb_size = lb_size;
7917 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
7918 wm_high.num_heads = num_heads;
7920 /* set for high clocks */
7921 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
7923 /* possibly force display priority to high */
7924 /* should really do this at mode validation time... */
7925 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
7926 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
7927 !dce8_check_latency_hiding(&wm_high) ||
7928 (rdev->disp_priority == 2)) {
7929 DRM_DEBUG_KMS("force priority to high\n");
7932 /* watermark for low clocks */
7933 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
7934 rdev->pm.dpm_enabled) {
7936 radeon_dpm_get_mclk(rdev, true) * 10;
7938 radeon_dpm_get_sclk(rdev, true) * 10;
7940 wm_low.yclk = rdev->pm.current_mclk * 10;
7941 wm_low.sclk = rdev->pm.current_sclk * 10;
7944 wm_low.disp_clk = mode->clock;
7945 wm_low.src_width = mode->crtc_hdisplay;
7946 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
7947 wm_low.blank_time = line_time - wm_low.active_time;
7948 wm_low.interlaced = false;
7949 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
7950 wm_low.interlaced = true;
7951 wm_low.vsc = radeon_crtc->vsc;
7953 if (radeon_crtc->rmx_type != RMX_OFF)
7955 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
7956 wm_low.lb_size = lb_size;
7957 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
7958 wm_low.num_heads = num_heads;
7960 /* set for low clocks */
7961 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
7963 /* possibly force display priority to high */
7964 /* should really do this at mode validation time... */
7965 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
7966 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
7967 !dce8_check_latency_hiding(&wm_low) ||
7968 (rdev->disp_priority == 2)) {
7969 DRM_DEBUG_KMS("force priority to high\n");
7974 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
7976 tmp &= ~LATENCY_WATERMARK_MASK(3);
7977 tmp |= LATENCY_WATERMARK_MASK(1);
7978 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
7979 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
7980 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
7981 LATENCY_HIGH_WATERMARK(line_time)));
7983 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
7984 tmp &= ~LATENCY_WATERMARK_MASK(3);
7985 tmp |= LATENCY_WATERMARK_MASK(2);
7986 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
7987 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
7988 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
7989 LATENCY_HIGH_WATERMARK(line_time)));
7990 /* restore original selection */
7991 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
7993 /* save values for DPM */
7994 radeon_crtc->line_time = line_time;
7995 radeon_crtc->wm_high = latency_watermark_a;
7996 radeon_crtc->wm_low = latency_watermark_b;
8000 * dce8_bandwidth_update - program display watermarks
8002 * @rdev: radeon_device pointer
8004 * Calculate and program the display watermarks and line
8005 * buffer allocation (CIK).
8007 void dce8_bandwidth_update(struct radeon_device *rdev)
8009 struct drm_display_mode *mode = NULL;
8010 u32 num_heads = 0, lb_size;
8013 radeon_update_display_priority(rdev);
8015 for (i = 0; i < rdev->num_crtc; i++) {
8016 if (rdev->mode_info.crtcs[i]->base.enabled)
8019 for (i = 0; i < rdev->num_crtc; i++) {
8020 mode = &rdev->mode_info.crtcs[i]->base.mode;
8021 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8022 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8027 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8029 * @rdev: radeon_device pointer
8031 * Fetches a GPU clock counter snapshot (SI).
8032 * Returns the 64 bit clock counter snapshot.
8034 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8038 mutex_lock(&rdev->gpu_clock_mutex);
8039 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8040 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8041 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8042 mutex_unlock(&rdev->gpu_clock_mutex);
8046 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8047 u32 cntl_reg, u32 status_reg)
8050 struct atom_clock_dividers dividers;
8053 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8054 clock, false, ÷rs);
8058 tmp = RREG32_SMC(cntl_reg);
8059 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8060 tmp |= dividers.post_divider;
8061 WREG32_SMC(cntl_reg, tmp);
8063 for (i = 0; i < 100; i++) {
8064 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8074 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8078 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8082 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8086 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
8088 struct pci_dev *root = rdev->pdev->bus->self;
8089 int bridge_pos, gpu_pos;
8090 u32 speed_cntl, mask, current_data_rate;
8094 if (radeon_pcie_gen2 == 0)
8097 if (rdev->flags & RADEON_IS_IGP)
8100 if (!(rdev->flags & RADEON_IS_PCIE))
8103 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
8107 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
8110 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8111 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
8112 LC_CURRENT_DATA_RATE_SHIFT;
8113 if (mask & DRM_PCIE_SPEED_80) {
8114 if (current_data_rate == 2) {
8115 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
8118 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
8119 } else if (mask & DRM_PCIE_SPEED_50) {
8120 if (current_data_rate == 1) {
8121 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
8124 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
8127 bridge_pos = pci_pcie_cap(root);
8131 gpu_pos = pci_pcie_cap(rdev->pdev);
8135 if (mask & DRM_PCIE_SPEED_80) {
8136 /* re-try equalization if gen3 is not already enabled */
8137 if (current_data_rate != 2) {
8138 u16 bridge_cfg, gpu_cfg;
8139 u16 bridge_cfg2, gpu_cfg2;
8140 u32 max_lw, current_lw, tmp;
8142 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8143 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8145 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
8146 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8148 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
8149 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8151 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8152 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
8153 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
8155 if (current_lw < max_lw) {
8156 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8157 if (tmp & LC_RENEGOTIATION_SUPPORT) {
8158 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
8159 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
8160 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
8161 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
8165 for (i = 0; i < 10; i++) {
8167 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
8168 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
8171 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8172 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8174 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
8175 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
8177 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8178 tmp |= LC_SET_QUIESCE;
8179 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8181 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8183 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8188 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
8189 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8190 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
8191 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8193 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
8194 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8195 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
8196 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8199 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
8200 tmp16 &= ~((1 << 4) | (7 << 9));
8201 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
8202 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
8204 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8205 tmp16 &= ~((1 << 4) | (7 << 9));
8206 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
8207 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8209 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8210 tmp &= ~LC_SET_QUIESCE;
8211 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8216 /* set the link speed */
8217 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
8218 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
8219 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8221 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8223 if (mask & DRM_PCIE_SPEED_80)
8224 tmp16 |= 3; /* gen3 */
8225 else if (mask & DRM_PCIE_SPEED_50)
8226 tmp16 |= 2; /* gen2 */
8228 tmp16 |= 1; /* gen1 */
8229 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8231 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8232 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
8233 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8235 for (i = 0; i < rdev->usec_timeout; i++) {
8236 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8237 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
8243 static void cik_program_aspm(struct radeon_device *rdev)
8246 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
8247 bool disable_clkreq = false;
8249 if (radeon_aspm == 0)
8252 /* XXX double check IGPs */
8253 if (rdev->flags & RADEON_IS_IGP)
8256 if (!(rdev->flags & RADEON_IS_PCIE))
8259 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8260 data &= ~LC_XMIT_N_FTS_MASK;
8261 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
8263 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
8265 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
8266 data |= LC_GO_TO_RECOVERY;
8268 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
8270 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
8271 data |= P_IGNORE_EDB_ERR;
8273 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
8275 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8276 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
8277 data |= LC_PMI_TO_L1_DIS;
8279 data |= LC_L0S_INACTIVITY(7);
8282 data |= LC_L1_INACTIVITY(7);
8283 data &= ~LC_PMI_TO_L1_DIS;
8285 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8287 if (!disable_plloff_in_l1) {
8288 bool clk_req_support;
8290 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
8291 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8292 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8294 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
8296 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
8297 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8298 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8300 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
8302 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
8303 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8304 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8306 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
8308 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
8309 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8310 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8312 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
8314 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8315 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
8316 data |= LC_DYN_LANES_PWR_STATE(3);
8318 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
8320 if (!disable_clkreq) {
8321 struct pci_dev *root = rdev->pdev->bus->self;
8324 clk_req_support = false;
8325 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
8326 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
8327 clk_req_support = true;
8329 clk_req_support = false;
8332 if (clk_req_support) {
8333 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
8334 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
8336 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
8338 orig = data = RREG32_SMC(THM_CLK_CNTL);
8339 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
8340 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
8342 WREG32_SMC(THM_CLK_CNTL, data);
8344 orig = data = RREG32_SMC(MISC_CLK_CTRL);
8345 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
8346 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
8348 WREG32_SMC(MISC_CLK_CTRL, data);
8350 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
8351 data &= ~BCLK_AS_XCLK;
8353 WREG32_SMC(CG_CLKPIN_CNTL, data);
8355 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
8356 data &= ~FORCE_BIF_REFCLK_EN;
8358 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
8360 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
8361 data &= ~MPLL_CLKOUT_SEL_MASK;
8362 data |= MPLL_CLKOUT_SEL(4);
8364 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
8369 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8372 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
8373 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
8375 WREG32_PCIE_PORT(PCIE_CNTL2, data);
8378 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8379 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
8380 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8381 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
8382 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8383 data &= ~LC_L0S_INACTIVITY_MASK;
8385 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);