2 * Copyright 2011 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
29 #include "radeon_asic.h"
30 #include <drm/radeon_drm.h>
33 #include "si_blit_shaders.h"
34 #include "clearstate_si.h"
35 #include "radeon_ucode.h"
38 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
39 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
40 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
41 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
42 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
43 MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
44 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
45 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
46 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
47 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
48 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
49 MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
50 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
51 MODULE_FIRMWARE("radeon/VERDE_me.bin");
52 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
53 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
54 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
55 MODULE_FIRMWARE("radeon/VERDE_smc.bin");
56 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
57 MODULE_FIRMWARE("radeon/OLAND_me.bin");
58 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
59 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
60 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
61 MODULE_FIRMWARE("radeon/OLAND_smc.bin");
62 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
63 MODULE_FIRMWARE("radeon/HAINAN_me.bin");
64 MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
65 MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
66 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
67 MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
69 static void si_pcie_gen3_enable(struct radeon_device *rdev);
70 static void si_program_aspm(struct radeon_device *rdev);
71 extern void sumo_rlc_fini(struct radeon_device *rdev);
72 extern int sumo_rlc_init(struct radeon_device *rdev);
73 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
74 extern void r600_ih_ring_fini(struct radeon_device *rdev);
75 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
76 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
77 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
78 extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
79 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
80 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
81 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
83 static void si_init_pg(struct radeon_device *rdev);
84 static void si_init_cg(struct radeon_device *rdev);
85 static void si_fini_pg(struct radeon_device *rdev);
86 static void si_fini_cg(struct radeon_device *rdev);
87 static void si_rlc_stop(struct radeon_device *rdev);
89 static const u32 verde_rlc_save_restore_register_list[] =
91 (0x8000 << 16) | (0x98f4 >> 2),
93 (0x8040 << 16) | (0x98f4 >> 2),
95 (0x8000 << 16) | (0xe80 >> 2),
97 (0x8040 << 16) | (0xe80 >> 2),
99 (0x8000 << 16) | (0x89bc >> 2),
101 (0x8040 << 16) | (0x89bc >> 2),
103 (0x8000 << 16) | (0x8c1c >> 2),
105 (0x8040 << 16) | (0x8c1c >> 2),
107 (0x9c00 << 16) | (0x98f0 >> 2),
109 (0x9c00 << 16) | (0xe7c >> 2),
111 (0x8000 << 16) | (0x9148 >> 2),
113 (0x8040 << 16) | (0x9148 >> 2),
115 (0x9c00 << 16) | (0x9150 >> 2),
117 (0x9c00 << 16) | (0x897c >> 2),
119 (0x9c00 << 16) | (0x8d8c >> 2),
121 (0x9c00 << 16) | (0xac54 >> 2),
124 (0x9c00 << 16) | (0x98f8 >> 2),
126 (0x9c00 << 16) | (0x9910 >> 2),
128 (0x9c00 << 16) | (0x9914 >> 2),
130 (0x9c00 << 16) | (0x9918 >> 2),
132 (0x9c00 << 16) | (0x991c >> 2),
134 (0x9c00 << 16) | (0x9920 >> 2),
136 (0x9c00 << 16) | (0x9924 >> 2),
138 (0x9c00 << 16) | (0x9928 >> 2),
140 (0x9c00 << 16) | (0x992c >> 2),
142 (0x9c00 << 16) | (0x9930 >> 2),
144 (0x9c00 << 16) | (0x9934 >> 2),
146 (0x9c00 << 16) | (0x9938 >> 2),
148 (0x9c00 << 16) | (0x993c >> 2),
150 (0x9c00 << 16) | (0x9940 >> 2),
152 (0x9c00 << 16) | (0x9944 >> 2),
154 (0x9c00 << 16) | (0x9948 >> 2),
156 (0x9c00 << 16) | (0x994c >> 2),
158 (0x9c00 << 16) | (0x9950 >> 2),
160 (0x9c00 << 16) | (0x9954 >> 2),
162 (0x9c00 << 16) | (0x9958 >> 2),
164 (0x9c00 << 16) | (0x995c >> 2),
166 (0x9c00 << 16) | (0x9960 >> 2),
168 (0x9c00 << 16) | (0x9964 >> 2),
170 (0x9c00 << 16) | (0x9968 >> 2),
172 (0x9c00 << 16) | (0x996c >> 2),
174 (0x9c00 << 16) | (0x9970 >> 2),
176 (0x9c00 << 16) | (0x9974 >> 2),
178 (0x9c00 << 16) | (0x9978 >> 2),
180 (0x9c00 << 16) | (0x997c >> 2),
182 (0x9c00 << 16) | (0x9980 >> 2),
184 (0x9c00 << 16) | (0x9984 >> 2),
186 (0x9c00 << 16) | (0x9988 >> 2),
188 (0x9c00 << 16) | (0x998c >> 2),
190 (0x9c00 << 16) | (0x8c00 >> 2),
192 (0x9c00 << 16) | (0x8c14 >> 2),
194 (0x9c00 << 16) | (0x8c04 >> 2),
196 (0x9c00 << 16) | (0x8c08 >> 2),
198 (0x8000 << 16) | (0x9b7c >> 2),
200 (0x8040 << 16) | (0x9b7c >> 2),
202 (0x8000 << 16) | (0xe84 >> 2),
204 (0x8040 << 16) | (0xe84 >> 2),
206 (0x8000 << 16) | (0x89c0 >> 2),
208 (0x8040 << 16) | (0x89c0 >> 2),
210 (0x8000 << 16) | (0x914c >> 2),
212 (0x8040 << 16) | (0x914c >> 2),
214 (0x8000 << 16) | (0x8c20 >> 2),
216 (0x8040 << 16) | (0x8c20 >> 2),
218 (0x8000 << 16) | (0x9354 >> 2),
220 (0x8040 << 16) | (0x9354 >> 2),
222 (0x9c00 << 16) | (0x9060 >> 2),
224 (0x9c00 << 16) | (0x9364 >> 2),
226 (0x9c00 << 16) | (0x9100 >> 2),
228 (0x9c00 << 16) | (0x913c >> 2),
230 (0x8000 << 16) | (0x90e0 >> 2),
232 (0x8000 << 16) | (0x90e4 >> 2),
234 (0x8000 << 16) | (0x90e8 >> 2),
236 (0x8040 << 16) | (0x90e0 >> 2),
238 (0x8040 << 16) | (0x90e4 >> 2),
240 (0x8040 << 16) | (0x90e8 >> 2),
242 (0x9c00 << 16) | (0x8bcc >> 2),
244 (0x9c00 << 16) | (0x8b24 >> 2),
246 (0x9c00 << 16) | (0x88c4 >> 2),
248 (0x9c00 << 16) | (0x8e50 >> 2),
250 (0x9c00 << 16) | (0x8c0c >> 2),
252 (0x9c00 << 16) | (0x8e58 >> 2),
254 (0x9c00 << 16) | (0x8e5c >> 2),
256 (0x9c00 << 16) | (0x9508 >> 2),
258 (0x9c00 << 16) | (0x950c >> 2),
260 (0x9c00 << 16) | (0x9494 >> 2),
262 (0x9c00 << 16) | (0xac0c >> 2),
264 (0x9c00 << 16) | (0xac10 >> 2),
266 (0x9c00 << 16) | (0xac14 >> 2),
268 (0x9c00 << 16) | (0xae00 >> 2),
270 (0x9c00 << 16) | (0xac08 >> 2),
272 (0x9c00 << 16) | (0x88d4 >> 2),
274 (0x9c00 << 16) | (0x88c8 >> 2),
276 (0x9c00 << 16) | (0x88cc >> 2),
278 (0x9c00 << 16) | (0x89b0 >> 2),
280 (0x9c00 << 16) | (0x8b10 >> 2),
282 (0x9c00 << 16) | (0x8a14 >> 2),
284 (0x9c00 << 16) | (0x9830 >> 2),
286 (0x9c00 << 16) | (0x9834 >> 2),
288 (0x9c00 << 16) | (0x9838 >> 2),
290 (0x9c00 << 16) | (0x9a10 >> 2),
292 (0x8000 << 16) | (0x9870 >> 2),
294 (0x8000 << 16) | (0x9874 >> 2),
296 (0x8001 << 16) | (0x9870 >> 2),
298 (0x8001 << 16) | (0x9874 >> 2),
300 (0x8040 << 16) | (0x9870 >> 2),
302 (0x8040 << 16) | (0x9874 >> 2),
304 (0x8041 << 16) | (0x9870 >> 2),
306 (0x8041 << 16) | (0x9874 >> 2),
311 static const u32 tahiti_golden_rlc_registers[] =
313 0xc424, 0xffffffff, 0x00601005,
314 0xc47c, 0xffffffff, 0x10104040,
315 0xc488, 0xffffffff, 0x0100000a,
316 0xc314, 0xffffffff, 0x00000800,
317 0xc30c, 0xffffffff, 0x800000f4,
318 0xf4a8, 0xffffffff, 0x00000000
321 static const u32 tahiti_golden_registers[] =
323 0x9a10, 0x00010000, 0x00018208,
324 0x9830, 0xffffffff, 0x00000000,
325 0x9834, 0xf00fffff, 0x00000400,
326 0x9838, 0x0002021c, 0x00020200,
327 0xc78, 0x00000080, 0x00000000,
328 0xd030, 0x000300c0, 0x00800040,
329 0xd830, 0x000300c0, 0x00800040,
330 0x5bb0, 0x000000f0, 0x00000070,
331 0x5bc0, 0x00200000, 0x50100000,
332 0x7030, 0x31000311, 0x00000011,
333 0x277c, 0x00000003, 0x000007ff,
334 0x240c, 0x000007ff, 0x00000000,
335 0x8a14, 0xf000001f, 0x00000007,
336 0x8b24, 0xffffffff, 0x00ffffff,
337 0x8b10, 0x0000ff0f, 0x00000000,
338 0x28a4c, 0x07ffffff, 0x4e000000,
339 0x28350, 0x3f3f3fff, 0x2a00126a,
340 0x30, 0x000000ff, 0x0040,
341 0x34, 0x00000040, 0x00004040,
342 0x9100, 0x07ffffff, 0x03000000,
343 0x8e88, 0x01ff1f3f, 0x00000000,
344 0x8e84, 0x01ff1f3f, 0x00000000,
345 0x9060, 0x0000007f, 0x00000020,
346 0x9508, 0x00010000, 0x00010000,
347 0xac14, 0x00000200, 0x000002fb,
348 0xac10, 0xffffffff, 0x0000543b,
349 0xac0c, 0xffffffff, 0xa9210876,
350 0x88d0, 0xffffffff, 0x000fff40,
351 0x88d4, 0x0000001f, 0x00000010,
352 0x1410, 0x20000000, 0x20fffed8,
353 0x15c0, 0x000c0fc0, 0x000c0400
356 static const u32 tahiti_golden_registers2[] =
358 0xc64, 0x00000001, 0x00000001
361 static const u32 pitcairn_golden_rlc_registers[] =
363 0xc424, 0xffffffff, 0x00601004,
364 0xc47c, 0xffffffff, 0x10102020,
365 0xc488, 0xffffffff, 0x01000020,
366 0xc314, 0xffffffff, 0x00000800,
367 0xc30c, 0xffffffff, 0x800000a4
370 static const u32 pitcairn_golden_registers[] =
372 0x9a10, 0x00010000, 0x00018208,
373 0x9830, 0xffffffff, 0x00000000,
374 0x9834, 0xf00fffff, 0x00000400,
375 0x9838, 0x0002021c, 0x00020200,
376 0xc78, 0x00000080, 0x00000000,
377 0xd030, 0x000300c0, 0x00800040,
378 0xd830, 0x000300c0, 0x00800040,
379 0x5bb0, 0x000000f0, 0x00000070,
380 0x5bc0, 0x00200000, 0x50100000,
381 0x7030, 0x31000311, 0x00000011,
382 0x2ae4, 0x00073ffe, 0x000022a2,
383 0x240c, 0x000007ff, 0x00000000,
384 0x8a14, 0xf000001f, 0x00000007,
385 0x8b24, 0xffffffff, 0x00ffffff,
386 0x8b10, 0x0000ff0f, 0x00000000,
387 0x28a4c, 0x07ffffff, 0x4e000000,
388 0x28350, 0x3f3f3fff, 0x2a00126a,
389 0x30, 0x000000ff, 0x0040,
390 0x34, 0x00000040, 0x00004040,
391 0x9100, 0x07ffffff, 0x03000000,
392 0x9060, 0x0000007f, 0x00000020,
393 0x9508, 0x00010000, 0x00010000,
394 0xac14, 0x000003ff, 0x000000f7,
395 0xac10, 0xffffffff, 0x00000000,
396 0xac0c, 0xffffffff, 0x32761054,
397 0x88d4, 0x0000001f, 0x00000010,
398 0x15c0, 0x000c0fc0, 0x000c0400
401 static const u32 verde_golden_rlc_registers[] =
403 0xc424, 0xffffffff, 0x033f1005,
404 0xc47c, 0xffffffff, 0x10808020,
405 0xc488, 0xffffffff, 0x00800008,
406 0xc314, 0xffffffff, 0x00001000,
407 0xc30c, 0xffffffff, 0x80010014
410 static const u32 verde_golden_registers[] =
412 0x9a10, 0x00010000, 0x00018208,
413 0x9830, 0xffffffff, 0x00000000,
414 0x9834, 0xf00fffff, 0x00000400,
415 0x9838, 0x0002021c, 0x00020200,
416 0xc78, 0x00000080, 0x00000000,
417 0xd030, 0x000300c0, 0x00800040,
418 0xd030, 0x000300c0, 0x00800040,
419 0xd830, 0x000300c0, 0x00800040,
420 0xd830, 0x000300c0, 0x00800040,
421 0x5bb0, 0x000000f0, 0x00000070,
422 0x5bc0, 0x00200000, 0x50100000,
423 0x7030, 0x31000311, 0x00000011,
424 0x2ae4, 0x00073ffe, 0x000022a2,
425 0x2ae4, 0x00073ffe, 0x000022a2,
426 0x2ae4, 0x00073ffe, 0x000022a2,
427 0x240c, 0x000007ff, 0x00000000,
428 0x240c, 0x000007ff, 0x00000000,
429 0x240c, 0x000007ff, 0x00000000,
430 0x8a14, 0xf000001f, 0x00000007,
431 0x8a14, 0xf000001f, 0x00000007,
432 0x8a14, 0xf000001f, 0x00000007,
433 0x8b24, 0xffffffff, 0x00ffffff,
434 0x8b10, 0x0000ff0f, 0x00000000,
435 0x28a4c, 0x07ffffff, 0x4e000000,
436 0x28350, 0x3f3f3fff, 0x0000124a,
437 0x28350, 0x3f3f3fff, 0x0000124a,
438 0x28350, 0x3f3f3fff, 0x0000124a,
439 0x30, 0x000000ff, 0x0040,
440 0x34, 0x00000040, 0x00004040,
441 0x9100, 0x07ffffff, 0x03000000,
442 0x9100, 0x07ffffff, 0x03000000,
443 0x8e88, 0x01ff1f3f, 0x00000000,
444 0x8e88, 0x01ff1f3f, 0x00000000,
445 0x8e88, 0x01ff1f3f, 0x00000000,
446 0x8e84, 0x01ff1f3f, 0x00000000,
447 0x8e84, 0x01ff1f3f, 0x00000000,
448 0x8e84, 0x01ff1f3f, 0x00000000,
449 0x9060, 0x0000007f, 0x00000020,
450 0x9508, 0x00010000, 0x00010000,
451 0xac14, 0x000003ff, 0x00000003,
452 0xac14, 0x000003ff, 0x00000003,
453 0xac14, 0x000003ff, 0x00000003,
454 0xac10, 0xffffffff, 0x00000000,
455 0xac10, 0xffffffff, 0x00000000,
456 0xac10, 0xffffffff, 0x00000000,
457 0xac0c, 0xffffffff, 0x00001032,
458 0xac0c, 0xffffffff, 0x00001032,
459 0xac0c, 0xffffffff, 0x00001032,
460 0x88d4, 0x0000001f, 0x00000010,
461 0x88d4, 0x0000001f, 0x00000010,
462 0x88d4, 0x0000001f, 0x00000010,
463 0x15c0, 0x000c0fc0, 0x000c0400
466 static const u32 oland_golden_rlc_registers[] =
468 0xc424, 0xffffffff, 0x00601005,
469 0xc47c, 0xffffffff, 0x10104040,
470 0xc488, 0xffffffff, 0x0100000a,
471 0xc314, 0xffffffff, 0x00000800,
472 0xc30c, 0xffffffff, 0x800000f4
475 static const u32 oland_golden_registers[] =
477 0x9a10, 0x00010000, 0x00018208,
478 0x9830, 0xffffffff, 0x00000000,
479 0x9834, 0xf00fffff, 0x00000400,
480 0x9838, 0x0002021c, 0x00020200,
481 0xc78, 0x00000080, 0x00000000,
482 0xd030, 0x000300c0, 0x00800040,
483 0xd830, 0x000300c0, 0x00800040,
484 0x5bb0, 0x000000f0, 0x00000070,
485 0x5bc0, 0x00200000, 0x50100000,
486 0x7030, 0x31000311, 0x00000011,
487 0x2ae4, 0x00073ffe, 0x000022a2,
488 0x240c, 0x000007ff, 0x00000000,
489 0x8a14, 0xf000001f, 0x00000007,
490 0x8b24, 0xffffffff, 0x00ffffff,
491 0x8b10, 0x0000ff0f, 0x00000000,
492 0x28a4c, 0x07ffffff, 0x4e000000,
493 0x28350, 0x3f3f3fff, 0x00000082,
494 0x30, 0x000000ff, 0x0040,
495 0x34, 0x00000040, 0x00004040,
496 0x9100, 0x07ffffff, 0x03000000,
497 0x9060, 0x0000007f, 0x00000020,
498 0x9508, 0x00010000, 0x00010000,
499 0xac14, 0x000003ff, 0x000000f3,
500 0xac10, 0xffffffff, 0x00000000,
501 0xac0c, 0xffffffff, 0x00003210,
502 0x88d4, 0x0000001f, 0x00000010,
503 0x15c0, 0x000c0fc0, 0x000c0400
506 static const u32 hainan_golden_registers[] =
508 0x9a10, 0x00010000, 0x00018208,
509 0x9830, 0xffffffff, 0x00000000,
510 0x9834, 0xf00fffff, 0x00000400,
511 0x9838, 0x0002021c, 0x00020200,
512 0xd0c0, 0xff000fff, 0x00000100,
513 0xd030, 0x000300c0, 0x00800040,
514 0xd8c0, 0xff000fff, 0x00000100,
515 0xd830, 0x000300c0, 0x00800040,
516 0x2ae4, 0x00073ffe, 0x000022a2,
517 0x240c, 0x000007ff, 0x00000000,
518 0x8a14, 0xf000001f, 0x00000007,
519 0x8b24, 0xffffffff, 0x00ffffff,
520 0x8b10, 0x0000ff0f, 0x00000000,
521 0x28a4c, 0x07ffffff, 0x4e000000,
522 0x28350, 0x3f3f3fff, 0x00000000,
523 0x30, 0x000000ff, 0x0040,
524 0x34, 0x00000040, 0x00004040,
525 0x9100, 0x03e00000, 0x03600000,
526 0x9060, 0x0000007f, 0x00000020,
527 0x9508, 0x00010000, 0x00010000,
528 0xac14, 0x000003ff, 0x000000f1,
529 0xac10, 0xffffffff, 0x00000000,
530 0xac0c, 0xffffffff, 0x00003210,
531 0x88d4, 0x0000001f, 0x00000010,
532 0x15c0, 0x000c0fc0, 0x000c0400
535 static const u32 hainan_golden_registers2[] =
537 0x98f8, 0xffffffff, 0x02010001
540 static const u32 tahiti_mgcg_cgcg_init[] =
542 0xc400, 0xffffffff, 0xfffffffc,
543 0x802c, 0xffffffff, 0xe0000000,
544 0x9a60, 0xffffffff, 0x00000100,
545 0x92a4, 0xffffffff, 0x00000100,
546 0xc164, 0xffffffff, 0x00000100,
547 0x9774, 0xffffffff, 0x00000100,
548 0x8984, 0xffffffff, 0x06000100,
549 0x8a18, 0xffffffff, 0x00000100,
550 0x92a0, 0xffffffff, 0x00000100,
551 0xc380, 0xffffffff, 0x00000100,
552 0x8b28, 0xffffffff, 0x00000100,
553 0x9144, 0xffffffff, 0x00000100,
554 0x8d88, 0xffffffff, 0x00000100,
555 0x8d8c, 0xffffffff, 0x00000100,
556 0x9030, 0xffffffff, 0x00000100,
557 0x9034, 0xffffffff, 0x00000100,
558 0x9038, 0xffffffff, 0x00000100,
559 0x903c, 0xffffffff, 0x00000100,
560 0xad80, 0xffffffff, 0x00000100,
561 0xac54, 0xffffffff, 0x00000100,
562 0x897c, 0xffffffff, 0x06000100,
563 0x9868, 0xffffffff, 0x00000100,
564 0x9510, 0xffffffff, 0x00000100,
565 0xaf04, 0xffffffff, 0x00000100,
566 0xae04, 0xffffffff, 0x00000100,
567 0x949c, 0xffffffff, 0x00000100,
568 0x802c, 0xffffffff, 0xe0000000,
569 0x9160, 0xffffffff, 0x00010000,
570 0x9164, 0xffffffff, 0x00030002,
571 0x9168, 0xffffffff, 0x00040007,
572 0x916c, 0xffffffff, 0x00060005,
573 0x9170, 0xffffffff, 0x00090008,
574 0x9174, 0xffffffff, 0x00020001,
575 0x9178, 0xffffffff, 0x00040003,
576 0x917c, 0xffffffff, 0x00000007,
577 0x9180, 0xffffffff, 0x00060005,
578 0x9184, 0xffffffff, 0x00090008,
579 0x9188, 0xffffffff, 0x00030002,
580 0x918c, 0xffffffff, 0x00050004,
581 0x9190, 0xffffffff, 0x00000008,
582 0x9194, 0xffffffff, 0x00070006,
583 0x9198, 0xffffffff, 0x000a0009,
584 0x919c, 0xffffffff, 0x00040003,
585 0x91a0, 0xffffffff, 0x00060005,
586 0x91a4, 0xffffffff, 0x00000009,
587 0x91a8, 0xffffffff, 0x00080007,
588 0x91ac, 0xffffffff, 0x000b000a,
589 0x91b0, 0xffffffff, 0x00050004,
590 0x91b4, 0xffffffff, 0x00070006,
591 0x91b8, 0xffffffff, 0x0008000b,
592 0x91bc, 0xffffffff, 0x000a0009,
593 0x91c0, 0xffffffff, 0x000d000c,
594 0x91c4, 0xffffffff, 0x00060005,
595 0x91c8, 0xffffffff, 0x00080007,
596 0x91cc, 0xffffffff, 0x0000000b,
597 0x91d0, 0xffffffff, 0x000a0009,
598 0x91d4, 0xffffffff, 0x000d000c,
599 0x91d8, 0xffffffff, 0x00070006,
600 0x91dc, 0xffffffff, 0x00090008,
601 0x91e0, 0xffffffff, 0x0000000c,
602 0x91e4, 0xffffffff, 0x000b000a,
603 0x91e8, 0xffffffff, 0x000e000d,
604 0x91ec, 0xffffffff, 0x00080007,
605 0x91f0, 0xffffffff, 0x000a0009,
606 0x91f4, 0xffffffff, 0x0000000d,
607 0x91f8, 0xffffffff, 0x000c000b,
608 0x91fc, 0xffffffff, 0x000f000e,
609 0x9200, 0xffffffff, 0x00090008,
610 0x9204, 0xffffffff, 0x000b000a,
611 0x9208, 0xffffffff, 0x000c000f,
612 0x920c, 0xffffffff, 0x000e000d,
613 0x9210, 0xffffffff, 0x00110010,
614 0x9214, 0xffffffff, 0x000a0009,
615 0x9218, 0xffffffff, 0x000c000b,
616 0x921c, 0xffffffff, 0x0000000f,
617 0x9220, 0xffffffff, 0x000e000d,
618 0x9224, 0xffffffff, 0x00110010,
619 0x9228, 0xffffffff, 0x000b000a,
620 0x922c, 0xffffffff, 0x000d000c,
621 0x9230, 0xffffffff, 0x00000010,
622 0x9234, 0xffffffff, 0x000f000e,
623 0x9238, 0xffffffff, 0x00120011,
624 0x923c, 0xffffffff, 0x000c000b,
625 0x9240, 0xffffffff, 0x000e000d,
626 0x9244, 0xffffffff, 0x00000011,
627 0x9248, 0xffffffff, 0x0010000f,
628 0x924c, 0xffffffff, 0x00130012,
629 0x9250, 0xffffffff, 0x000d000c,
630 0x9254, 0xffffffff, 0x000f000e,
631 0x9258, 0xffffffff, 0x00100013,
632 0x925c, 0xffffffff, 0x00120011,
633 0x9260, 0xffffffff, 0x00150014,
634 0x9264, 0xffffffff, 0x000e000d,
635 0x9268, 0xffffffff, 0x0010000f,
636 0x926c, 0xffffffff, 0x00000013,
637 0x9270, 0xffffffff, 0x00120011,
638 0x9274, 0xffffffff, 0x00150014,
639 0x9278, 0xffffffff, 0x000f000e,
640 0x927c, 0xffffffff, 0x00110010,
641 0x9280, 0xffffffff, 0x00000014,
642 0x9284, 0xffffffff, 0x00130012,
643 0x9288, 0xffffffff, 0x00160015,
644 0x928c, 0xffffffff, 0x0010000f,
645 0x9290, 0xffffffff, 0x00120011,
646 0x9294, 0xffffffff, 0x00000015,
647 0x9298, 0xffffffff, 0x00140013,
648 0x929c, 0xffffffff, 0x00170016,
649 0x9150, 0xffffffff, 0x96940200,
650 0x8708, 0xffffffff, 0x00900100,
651 0xc478, 0xffffffff, 0x00000080,
652 0xc404, 0xffffffff, 0x0020003f,
653 0x30, 0xffffffff, 0x0000001c,
654 0x34, 0x000f0000, 0x000f0000,
655 0x160c, 0xffffffff, 0x00000100,
656 0x1024, 0xffffffff, 0x00000100,
657 0x102c, 0x00000101, 0x00000000,
658 0x20a8, 0xffffffff, 0x00000104,
659 0x264c, 0x000c0000, 0x000c0000,
660 0x2648, 0x000c0000, 0x000c0000,
661 0x55e4, 0xff000fff, 0x00000100,
662 0x55e8, 0x00000001, 0x00000001,
663 0x2f50, 0x00000001, 0x00000001,
664 0x30cc, 0xc0000fff, 0x00000104,
665 0xc1e4, 0x00000001, 0x00000001,
666 0xd0c0, 0xfffffff0, 0x00000100,
667 0xd8c0, 0xfffffff0, 0x00000100
670 static const u32 pitcairn_mgcg_cgcg_init[] =
672 0xc400, 0xffffffff, 0xfffffffc,
673 0x802c, 0xffffffff, 0xe0000000,
674 0x9a60, 0xffffffff, 0x00000100,
675 0x92a4, 0xffffffff, 0x00000100,
676 0xc164, 0xffffffff, 0x00000100,
677 0x9774, 0xffffffff, 0x00000100,
678 0x8984, 0xffffffff, 0x06000100,
679 0x8a18, 0xffffffff, 0x00000100,
680 0x92a0, 0xffffffff, 0x00000100,
681 0xc380, 0xffffffff, 0x00000100,
682 0x8b28, 0xffffffff, 0x00000100,
683 0x9144, 0xffffffff, 0x00000100,
684 0x8d88, 0xffffffff, 0x00000100,
685 0x8d8c, 0xffffffff, 0x00000100,
686 0x9030, 0xffffffff, 0x00000100,
687 0x9034, 0xffffffff, 0x00000100,
688 0x9038, 0xffffffff, 0x00000100,
689 0x903c, 0xffffffff, 0x00000100,
690 0xad80, 0xffffffff, 0x00000100,
691 0xac54, 0xffffffff, 0x00000100,
692 0x897c, 0xffffffff, 0x06000100,
693 0x9868, 0xffffffff, 0x00000100,
694 0x9510, 0xffffffff, 0x00000100,
695 0xaf04, 0xffffffff, 0x00000100,
696 0xae04, 0xffffffff, 0x00000100,
697 0x949c, 0xffffffff, 0x00000100,
698 0x802c, 0xffffffff, 0xe0000000,
699 0x9160, 0xffffffff, 0x00010000,
700 0x9164, 0xffffffff, 0x00030002,
701 0x9168, 0xffffffff, 0x00040007,
702 0x916c, 0xffffffff, 0x00060005,
703 0x9170, 0xffffffff, 0x00090008,
704 0x9174, 0xffffffff, 0x00020001,
705 0x9178, 0xffffffff, 0x00040003,
706 0x917c, 0xffffffff, 0x00000007,
707 0x9180, 0xffffffff, 0x00060005,
708 0x9184, 0xffffffff, 0x00090008,
709 0x9188, 0xffffffff, 0x00030002,
710 0x918c, 0xffffffff, 0x00050004,
711 0x9190, 0xffffffff, 0x00000008,
712 0x9194, 0xffffffff, 0x00070006,
713 0x9198, 0xffffffff, 0x000a0009,
714 0x919c, 0xffffffff, 0x00040003,
715 0x91a0, 0xffffffff, 0x00060005,
716 0x91a4, 0xffffffff, 0x00000009,
717 0x91a8, 0xffffffff, 0x00080007,
718 0x91ac, 0xffffffff, 0x000b000a,
719 0x91b0, 0xffffffff, 0x00050004,
720 0x91b4, 0xffffffff, 0x00070006,
721 0x91b8, 0xffffffff, 0x0008000b,
722 0x91bc, 0xffffffff, 0x000a0009,
723 0x91c0, 0xffffffff, 0x000d000c,
724 0x9200, 0xffffffff, 0x00090008,
725 0x9204, 0xffffffff, 0x000b000a,
726 0x9208, 0xffffffff, 0x000c000f,
727 0x920c, 0xffffffff, 0x000e000d,
728 0x9210, 0xffffffff, 0x00110010,
729 0x9214, 0xffffffff, 0x000a0009,
730 0x9218, 0xffffffff, 0x000c000b,
731 0x921c, 0xffffffff, 0x0000000f,
732 0x9220, 0xffffffff, 0x000e000d,
733 0x9224, 0xffffffff, 0x00110010,
734 0x9228, 0xffffffff, 0x000b000a,
735 0x922c, 0xffffffff, 0x000d000c,
736 0x9230, 0xffffffff, 0x00000010,
737 0x9234, 0xffffffff, 0x000f000e,
738 0x9238, 0xffffffff, 0x00120011,
739 0x923c, 0xffffffff, 0x000c000b,
740 0x9240, 0xffffffff, 0x000e000d,
741 0x9244, 0xffffffff, 0x00000011,
742 0x9248, 0xffffffff, 0x0010000f,
743 0x924c, 0xffffffff, 0x00130012,
744 0x9250, 0xffffffff, 0x000d000c,
745 0x9254, 0xffffffff, 0x000f000e,
746 0x9258, 0xffffffff, 0x00100013,
747 0x925c, 0xffffffff, 0x00120011,
748 0x9260, 0xffffffff, 0x00150014,
749 0x9150, 0xffffffff, 0x96940200,
750 0x8708, 0xffffffff, 0x00900100,
751 0xc478, 0xffffffff, 0x00000080,
752 0xc404, 0xffffffff, 0x0020003f,
753 0x30, 0xffffffff, 0x0000001c,
754 0x34, 0x000f0000, 0x000f0000,
755 0x160c, 0xffffffff, 0x00000100,
756 0x1024, 0xffffffff, 0x00000100,
757 0x102c, 0x00000101, 0x00000000,
758 0x20a8, 0xffffffff, 0x00000104,
759 0x55e4, 0xff000fff, 0x00000100,
760 0x55e8, 0x00000001, 0x00000001,
761 0x2f50, 0x00000001, 0x00000001,
762 0x30cc, 0xc0000fff, 0x00000104,
763 0xc1e4, 0x00000001, 0x00000001,
764 0xd0c0, 0xfffffff0, 0x00000100,
765 0xd8c0, 0xfffffff0, 0x00000100
768 static const u32 verde_mgcg_cgcg_init[] =
770 0xc400, 0xffffffff, 0xfffffffc,
771 0x802c, 0xffffffff, 0xe0000000,
772 0x9a60, 0xffffffff, 0x00000100,
773 0x92a4, 0xffffffff, 0x00000100,
774 0xc164, 0xffffffff, 0x00000100,
775 0x9774, 0xffffffff, 0x00000100,
776 0x8984, 0xffffffff, 0x06000100,
777 0x8a18, 0xffffffff, 0x00000100,
778 0x92a0, 0xffffffff, 0x00000100,
779 0xc380, 0xffffffff, 0x00000100,
780 0x8b28, 0xffffffff, 0x00000100,
781 0x9144, 0xffffffff, 0x00000100,
782 0x8d88, 0xffffffff, 0x00000100,
783 0x8d8c, 0xffffffff, 0x00000100,
784 0x9030, 0xffffffff, 0x00000100,
785 0x9034, 0xffffffff, 0x00000100,
786 0x9038, 0xffffffff, 0x00000100,
787 0x903c, 0xffffffff, 0x00000100,
788 0xad80, 0xffffffff, 0x00000100,
789 0xac54, 0xffffffff, 0x00000100,
790 0x897c, 0xffffffff, 0x06000100,
791 0x9868, 0xffffffff, 0x00000100,
792 0x9510, 0xffffffff, 0x00000100,
793 0xaf04, 0xffffffff, 0x00000100,
794 0xae04, 0xffffffff, 0x00000100,
795 0x949c, 0xffffffff, 0x00000100,
796 0x802c, 0xffffffff, 0xe0000000,
797 0x9160, 0xffffffff, 0x00010000,
798 0x9164, 0xffffffff, 0x00030002,
799 0x9168, 0xffffffff, 0x00040007,
800 0x916c, 0xffffffff, 0x00060005,
801 0x9170, 0xffffffff, 0x00090008,
802 0x9174, 0xffffffff, 0x00020001,
803 0x9178, 0xffffffff, 0x00040003,
804 0x917c, 0xffffffff, 0x00000007,
805 0x9180, 0xffffffff, 0x00060005,
806 0x9184, 0xffffffff, 0x00090008,
807 0x9188, 0xffffffff, 0x00030002,
808 0x918c, 0xffffffff, 0x00050004,
809 0x9190, 0xffffffff, 0x00000008,
810 0x9194, 0xffffffff, 0x00070006,
811 0x9198, 0xffffffff, 0x000a0009,
812 0x919c, 0xffffffff, 0x00040003,
813 0x91a0, 0xffffffff, 0x00060005,
814 0x91a4, 0xffffffff, 0x00000009,
815 0x91a8, 0xffffffff, 0x00080007,
816 0x91ac, 0xffffffff, 0x000b000a,
817 0x91b0, 0xffffffff, 0x00050004,
818 0x91b4, 0xffffffff, 0x00070006,
819 0x91b8, 0xffffffff, 0x0008000b,
820 0x91bc, 0xffffffff, 0x000a0009,
821 0x91c0, 0xffffffff, 0x000d000c,
822 0x9200, 0xffffffff, 0x00090008,
823 0x9204, 0xffffffff, 0x000b000a,
824 0x9208, 0xffffffff, 0x000c000f,
825 0x920c, 0xffffffff, 0x000e000d,
826 0x9210, 0xffffffff, 0x00110010,
827 0x9214, 0xffffffff, 0x000a0009,
828 0x9218, 0xffffffff, 0x000c000b,
829 0x921c, 0xffffffff, 0x0000000f,
830 0x9220, 0xffffffff, 0x000e000d,
831 0x9224, 0xffffffff, 0x00110010,
832 0x9228, 0xffffffff, 0x000b000a,
833 0x922c, 0xffffffff, 0x000d000c,
834 0x9230, 0xffffffff, 0x00000010,
835 0x9234, 0xffffffff, 0x000f000e,
836 0x9238, 0xffffffff, 0x00120011,
837 0x923c, 0xffffffff, 0x000c000b,
838 0x9240, 0xffffffff, 0x000e000d,
839 0x9244, 0xffffffff, 0x00000011,
840 0x9248, 0xffffffff, 0x0010000f,
841 0x924c, 0xffffffff, 0x00130012,
842 0x9250, 0xffffffff, 0x000d000c,
843 0x9254, 0xffffffff, 0x000f000e,
844 0x9258, 0xffffffff, 0x00100013,
845 0x925c, 0xffffffff, 0x00120011,
846 0x9260, 0xffffffff, 0x00150014,
847 0x9150, 0xffffffff, 0x96940200,
848 0x8708, 0xffffffff, 0x00900100,
849 0xc478, 0xffffffff, 0x00000080,
850 0xc404, 0xffffffff, 0x0020003f,
851 0x30, 0xffffffff, 0x0000001c,
852 0x34, 0x000f0000, 0x000f0000,
853 0x160c, 0xffffffff, 0x00000100,
854 0x1024, 0xffffffff, 0x00000100,
855 0x102c, 0x00000101, 0x00000000,
856 0x20a8, 0xffffffff, 0x00000104,
857 0x264c, 0x000c0000, 0x000c0000,
858 0x2648, 0x000c0000, 0x000c0000,
859 0x55e4, 0xff000fff, 0x00000100,
860 0x55e8, 0x00000001, 0x00000001,
861 0x2f50, 0x00000001, 0x00000001,
862 0x30cc, 0xc0000fff, 0x00000104,
863 0xc1e4, 0x00000001, 0x00000001,
864 0xd0c0, 0xfffffff0, 0x00000100,
865 0xd8c0, 0xfffffff0, 0x00000100
868 static const u32 oland_mgcg_cgcg_init[] =
870 0xc400, 0xffffffff, 0xfffffffc,
871 0x802c, 0xffffffff, 0xe0000000,
872 0x9a60, 0xffffffff, 0x00000100,
873 0x92a4, 0xffffffff, 0x00000100,
874 0xc164, 0xffffffff, 0x00000100,
875 0x9774, 0xffffffff, 0x00000100,
876 0x8984, 0xffffffff, 0x06000100,
877 0x8a18, 0xffffffff, 0x00000100,
878 0x92a0, 0xffffffff, 0x00000100,
879 0xc380, 0xffffffff, 0x00000100,
880 0x8b28, 0xffffffff, 0x00000100,
881 0x9144, 0xffffffff, 0x00000100,
882 0x8d88, 0xffffffff, 0x00000100,
883 0x8d8c, 0xffffffff, 0x00000100,
884 0x9030, 0xffffffff, 0x00000100,
885 0x9034, 0xffffffff, 0x00000100,
886 0x9038, 0xffffffff, 0x00000100,
887 0x903c, 0xffffffff, 0x00000100,
888 0xad80, 0xffffffff, 0x00000100,
889 0xac54, 0xffffffff, 0x00000100,
890 0x897c, 0xffffffff, 0x06000100,
891 0x9868, 0xffffffff, 0x00000100,
892 0x9510, 0xffffffff, 0x00000100,
893 0xaf04, 0xffffffff, 0x00000100,
894 0xae04, 0xffffffff, 0x00000100,
895 0x949c, 0xffffffff, 0x00000100,
896 0x802c, 0xffffffff, 0xe0000000,
897 0x9160, 0xffffffff, 0x00010000,
898 0x9164, 0xffffffff, 0x00030002,
899 0x9168, 0xffffffff, 0x00040007,
900 0x916c, 0xffffffff, 0x00060005,
901 0x9170, 0xffffffff, 0x00090008,
902 0x9174, 0xffffffff, 0x00020001,
903 0x9178, 0xffffffff, 0x00040003,
904 0x917c, 0xffffffff, 0x00000007,
905 0x9180, 0xffffffff, 0x00060005,
906 0x9184, 0xffffffff, 0x00090008,
907 0x9188, 0xffffffff, 0x00030002,
908 0x918c, 0xffffffff, 0x00050004,
909 0x9190, 0xffffffff, 0x00000008,
910 0x9194, 0xffffffff, 0x00070006,
911 0x9198, 0xffffffff, 0x000a0009,
912 0x919c, 0xffffffff, 0x00040003,
913 0x91a0, 0xffffffff, 0x00060005,
914 0x91a4, 0xffffffff, 0x00000009,
915 0x91a8, 0xffffffff, 0x00080007,
916 0x91ac, 0xffffffff, 0x000b000a,
917 0x91b0, 0xffffffff, 0x00050004,
918 0x91b4, 0xffffffff, 0x00070006,
919 0x91b8, 0xffffffff, 0x0008000b,
920 0x91bc, 0xffffffff, 0x000a0009,
921 0x91c0, 0xffffffff, 0x000d000c,
922 0x91c4, 0xffffffff, 0x00060005,
923 0x91c8, 0xffffffff, 0x00080007,
924 0x91cc, 0xffffffff, 0x0000000b,
925 0x91d0, 0xffffffff, 0x000a0009,
926 0x91d4, 0xffffffff, 0x000d000c,
927 0x9150, 0xffffffff, 0x96940200,
928 0x8708, 0xffffffff, 0x00900100,
929 0xc478, 0xffffffff, 0x00000080,
930 0xc404, 0xffffffff, 0x0020003f,
931 0x30, 0xffffffff, 0x0000001c,
932 0x34, 0x000f0000, 0x000f0000,
933 0x160c, 0xffffffff, 0x00000100,
934 0x1024, 0xffffffff, 0x00000100,
935 0x102c, 0x00000101, 0x00000000,
936 0x20a8, 0xffffffff, 0x00000104,
937 0x264c, 0x000c0000, 0x000c0000,
938 0x2648, 0x000c0000, 0x000c0000,
939 0x55e4, 0xff000fff, 0x00000100,
940 0x55e8, 0x00000001, 0x00000001,
941 0x2f50, 0x00000001, 0x00000001,
942 0x30cc, 0xc0000fff, 0x00000104,
943 0xc1e4, 0x00000001, 0x00000001,
944 0xd0c0, 0xfffffff0, 0x00000100,
945 0xd8c0, 0xfffffff0, 0x00000100
948 static const u32 hainan_mgcg_cgcg_init[] =
950 0xc400, 0xffffffff, 0xfffffffc,
951 0x802c, 0xffffffff, 0xe0000000,
952 0x9a60, 0xffffffff, 0x00000100,
953 0x92a4, 0xffffffff, 0x00000100,
954 0xc164, 0xffffffff, 0x00000100,
955 0x9774, 0xffffffff, 0x00000100,
956 0x8984, 0xffffffff, 0x06000100,
957 0x8a18, 0xffffffff, 0x00000100,
958 0x92a0, 0xffffffff, 0x00000100,
959 0xc380, 0xffffffff, 0x00000100,
960 0x8b28, 0xffffffff, 0x00000100,
961 0x9144, 0xffffffff, 0x00000100,
962 0x8d88, 0xffffffff, 0x00000100,
963 0x8d8c, 0xffffffff, 0x00000100,
964 0x9030, 0xffffffff, 0x00000100,
965 0x9034, 0xffffffff, 0x00000100,
966 0x9038, 0xffffffff, 0x00000100,
967 0x903c, 0xffffffff, 0x00000100,
968 0xad80, 0xffffffff, 0x00000100,
969 0xac54, 0xffffffff, 0x00000100,
970 0x897c, 0xffffffff, 0x06000100,
971 0x9868, 0xffffffff, 0x00000100,
972 0x9510, 0xffffffff, 0x00000100,
973 0xaf04, 0xffffffff, 0x00000100,
974 0xae04, 0xffffffff, 0x00000100,
975 0x949c, 0xffffffff, 0x00000100,
976 0x802c, 0xffffffff, 0xe0000000,
977 0x9160, 0xffffffff, 0x00010000,
978 0x9164, 0xffffffff, 0x00030002,
979 0x9168, 0xffffffff, 0x00040007,
980 0x916c, 0xffffffff, 0x00060005,
981 0x9170, 0xffffffff, 0x00090008,
982 0x9174, 0xffffffff, 0x00020001,
983 0x9178, 0xffffffff, 0x00040003,
984 0x917c, 0xffffffff, 0x00000007,
985 0x9180, 0xffffffff, 0x00060005,
986 0x9184, 0xffffffff, 0x00090008,
987 0x9188, 0xffffffff, 0x00030002,
988 0x918c, 0xffffffff, 0x00050004,
989 0x9190, 0xffffffff, 0x00000008,
990 0x9194, 0xffffffff, 0x00070006,
991 0x9198, 0xffffffff, 0x000a0009,
992 0x919c, 0xffffffff, 0x00040003,
993 0x91a0, 0xffffffff, 0x00060005,
994 0x91a4, 0xffffffff, 0x00000009,
995 0x91a8, 0xffffffff, 0x00080007,
996 0x91ac, 0xffffffff, 0x000b000a,
997 0x91b0, 0xffffffff, 0x00050004,
998 0x91b4, 0xffffffff, 0x00070006,
999 0x91b8, 0xffffffff, 0x0008000b,
1000 0x91bc, 0xffffffff, 0x000a0009,
1001 0x91c0, 0xffffffff, 0x000d000c,
1002 0x91c4, 0xffffffff, 0x00060005,
1003 0x91c8, 0xffffffff, 0x00080007,
1004 0x91cc, 0xffffffff, 0x0000000b,
1005 0x91d0, 0xffffffff, 0x000a0009,
1006 0x91d4, 0xffffffff, 0x000d000c,
1007 0x9150, 0xffffffff, 0x96940200,
1008 0x8708, 0xffffffff, 0x00900100,
1009 0xc478, 0xffffffff, 0x00000080,
1010 0xc404, 0xffffffff, 0x0020003f,
1011 0x30, 0xffffffff, 0x0000001c,
1012 0x34, 0x000f0000, 0x000f0000,
1013 0x160c, 0xffffffff, 0x00000100,
1014 0x1024, 0xffffffff, 0x00000100,
1015 0x20a8, 0xffffffff, 0x00000104,
1016 0x264c, 0x000c0000, 0x000c0000,
1017 0x2648, 0x000c0000, 0x000c0000,
1018 0x2f50, 0x00000001, 0x00000001,
1019 0x30cc, 0xc0000fff, 0x00000104,
1020 0xc1e4, 0x00000001, 0x00000001,
1021 0xd0c0, 0xfffffff0, 0x00000100,
1022 0xd8c0, 0xfffffff0, 0x00000100
1025 static u32 verde_pg_init[] =
1027 0x353c, 0xffffffff, 0x40000,
1028 0x3538, 0xffffffff, 0x200010ff,
1029 0x353c, 0xffffffff, 0x0,
1030 0x353c, 0xffffffff, 0x0,
1031 0x353c, 0xffffffff, 0x0,
1032 0x353c, 0xffffffff, 0x0,
1033 0x353c, 0xffffffff, 0x0,
1034 0x353c, 0xffffffff, 0x7007,
1035 0x3538, 0xffffffff, 0x300010ff,
1036 0x353c, 0xffffffff, 0x0,
1037 0x353c, 0xffffffff, 0x0,
1038 0x353c, 0xffffffff, 0x0,
1039 0x353c, 0xffffffff, 0x0,
1040 0x353c, 0xffffffff, 0x0,
1041 0x353c, 0xffffffff, 0x400000,
1042 0x3538, 0xffffffff, 0x100010ff,
1043 0x353c, 0xffffffff, 0x0,
1044 0x353c, 0xffffffff, 0x0,
1045 0x353c, 0xffffffff, 0x0,
1046 0x353c, 0xffffffff, 0x0,
1047 0x353c, 0xffffffff, 0x0,
1048 0x353c, 0xffffffff, 0x120200,
1049 0x3538, 0xffffffff, 0x500010ff,
1050 0x353c, 0xffffffff, 0x0,
1051 0x353c, 0xffffffff, 0x0,
1052 0x353c, 0xffffffff, 0x0,
1053 0x353c, 0xffffffff, 0x0,
1054 0x353c, 0xffffffff, 0x0,
1055 0x353c, 0xffffffff, 0x1e1e16,
1056 0x3538, 0xffffffff, 0x600010ff,
1057 0x353c, 0xffffffff, 0x0,
1058 0x353c, 0xffffffff, 0x0,
1059 0x353c, 0xffffffff, 0x0,
1060 0x353c, 0xffffffff, 0x0,
1061 0x353c, 0xffffffff, 0x0,
1062 0x353c, 0xffffffff, 0x171f1e,
1063 0x3538, 0xffffffff, 0x700010ff,
1064 0x353c, 0xffffffff, 0x0,
1065 0x353c, 0xffffffff, 0x0,
1066 0x353c, 0xffffffff, 0x0,
1067 0x353c, 0xffffffff, 0x0,
1068 0x353c, 0xffffffff, 0x0,
1069 0x353c, 0xffffffff, 0x0,
1070 0x3538, 0xffffffff, 0x9ff,
1071 0x3500, 0xffffffff, 0x0,
1072 0x3504, 0xffffffff, 0x10000800,
1073 0x3504, 0xffffffff, 0xf,
1074 0x3504, 0xffffffff, 0xf,
1075 0x3500, 0xffffffff, 0x4,
1076 0x3504, 0xffffffff, 0x1000051e,
1077 0x3504, 0xffffffff, 0xffff,
1078 0x3504, 0xffffffff, 0xffff,
1079 0x3500, 0xffffffff, 0x8,
1080 0x3504, 0xffffffff, 0x80500,
1081 0x3500, 0xffffffff, 0x12,
1082 0x3504, 0xffffffff, 0x9050c,
1083 0x3500, 0xffffffff, 0x1d,
1084 0x3504, 0xffffffff, 0xb052c,
1085 0x3500, 0xffffffff, 0x2a,
1086 0x3504, 0xffffffff, 0x1053e,
1087 0x3500, 0xffffffff, 0x2d,
1088 0x3504, 0xffffffff, 0x10546,
1089 0x3500, 0xffffffff, 0x30,
1090 0x3504, 0xffffffff, 0xa054e,
1091 0x3500, 0xffffffff, 0x3c,
1092 0x3504, 0xffffffff, 0x1055f,
1093 0x3500, 0xffffffff, 0x3f,
1094 0x3504, 0xffffffff, 0x10567,
1095 0x3500, 0xffffffff, 0x42,
1096 0x3504, 0xffffffff, 0x1056f,
1097 0x3500, 0xffffffff, 0x45,
1098 0x3504, 0xffffffff, 0x10572,
1099 0x3500, 0xffffffff, 0x48,
1100 0x3504, 0xffffffff, 0x20575,
1101 0x3500, 0xffffffff, 0x4c,
1102 0x3504, 0xffffffff, 0x190801,
1103 0x3500, 0xffffffff, 0x67,
1104 0x3504, 0xffffffff, 0x1082a,
1105 0x3500, 0xffffffff, 0x6a,
1106 0x3504, 0xffffffff, 0x1b082d,
1107 0x3500, 0xffffffff, 0x87,
1108 0x3504, 0xffffffff, 0x310851,
1109 0x3500, 0xffffffff, 0xba,
1110 0x3504, 0xffffffff, 0x891,
1111 0x3500, 0xffffffff, 0xbc,
1112 0x3504, 0xffffffff, 0x893,
1113 0x3500, 0xffffffff, 0xbe,
1114 0x3504, 0xffffffff, 0x20895,
1115 0x3500, 0xffffffff, 0xc2,
1116 0x3504, 0xffffffff, 0x20899,
1117 0x3500, 0xffffffff, 0xc6,
1118 0x3504, 0xffffffff, 0x2089d,
1119 0x3500, 0xffffffff, 0xca,
1120 0x3504, 0xffffffff, 0x8a1,
1121 0x3500, 0xffffffff, 0xcc,
1122 0x3504, 0xffffffff, 0x8a3,
1123 0x3500, 0xffffffff, 0xce,
1124 0x3504, 0xffffffff, 0x308a5,
1125 0x3500, 0xffffffff, 0xd3,
1126 0x3504, 0xffffffff, 0x6d08cd,
1127 0x3500, 0xffffffff, 0x142,
1128 0x3504, 0xffffffff, 0x2000095a,
1129 0x3504, 0xffffffff, 0x1,
1130 0x3500, 0xffffffff, 0x144,
1131 0x3504, 0xffffffff, 0x301f095b,
1132 0x3500, 0xffffffff, 0x165,
1133 0x3504, 0xffffffff, 0xc094d,
1134 0x3500, 0xffffffff, 0x173,
1135 0x3504, 0xffffffff, 0xf096d,
1136 0x3500, 0xffffffff, 0x184,
1137 0x3504, 0xffffffff, 0x15097f,
1138 0x3500, 0xffffffff, 0x19b,
1139 0x3504, 0xffffffff, 0xc0998,
1140 0x3500, 0xffffffff, 0x1a9,
1141 0x3504, 0xffffffff, 0x409a7,
1142 0x3500, 0xffffffff, 0x1af,
1143 0x3504, 0xffffffff, 0xcdc,
1144 0x3500, 0xffffffff, 0x1b1,
1145 0x3504, 0xffffffff, 0x800,
1146 0x3508, 0xffffffff, 0x6c9b2000,
1147 0x3510, 0xfc00, 0x2000,
1148 0x3544, 0xffffffff, 0xfc0,
1149 0x28d4, 0x00000100, 0x100
1152 static void si_init_golden_registers(struct radeon_device *rdev)
1154 switch (rdev->family) {
1156 radeon_program_register_sequence(rdev,
1157 tahiti_golden_registers,
1158 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1159 radeon_program_register_sequence(rdev,
1160 tahiti_golden_rlc_registers,
1161 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1162 radeon_program_register_sequence(rdev,
1163 tahiti_mgcg_cgcg_init,
1164 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1165 radeon_program_register_sequence(rdev,
1166 tahiti_golden_registers2,
1167 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1170 radeon_program_register_sequence(rdev,
1171 pitcairn_golden_registers,
1172 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1173 radeon_program_register_sequence(rdev,
1174 pitcairn_golden_rlc_registers,
1175 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1176 radeon_program_register_sequence(rdev,
1177 pitcairn_mgcg_cgcg_init,
1178 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1181 radeon_program_register_sequence(rdev,
1182 verde_golden_registers,
1183 (const u32)ARRAY_SIZE(verde_golden_registers));
1184 radeon_program_register_sequence(rdev,
1185 verde_golden_rlc_registers,
1186 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1187 radeon_program_register_sequence(rdev,
1188 verde_mgcg_cgcg_init,
1189 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1190 radeon_program_register_sequence(rdev,
1192 (const u32)ARRAY_SIZE(verde_pg_init));
1195 radeon_program_register_sequence(rdev,
1196 oland_golden_registers,
1197 (const u32)ARRAY_SIZE(oland_golden_registers));
1198 radeon_program_register_sequence(rdev,
1199 oland_golden_rlc_registers,
1200 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1201 radeon_program_register_sequence(rdev,
1202 oland_mgcg_cgcg_init,
1203 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1206 radeon_program_register_sequence(rdev,
1207 hainan_golden_registers,
1208 (const u32)ARRAY_SIZE(hainan_golden_registers));
1209 radeon_program_register_sequence(rdev,
1210 hainan_golden_registers2,
1211 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1212 radeon_program_register_sequence(rdev,
1213 hainan_mgcg_cgcg_init,
1214 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1221 #define PCIE_BUS_CLK 10000
1222 #define TCLK (PCIE_BUS_CLK / 10)
1225 * si_get_xclk - get the xclk
1227 * @rdev: radeon_device pointer
1229 * Returns the reference clock used by the gfx engine
1232 u32 si_get_xclk(struct radeon_device *rdev)
1234 u32 reference_clock = rdev->clock.spll.reference_freq;
1237 tmp = RREG32(CG_CLKPIN_CNTL_2);
1238 if (tmp & MUX_TCLK_TO_XCLK)
1241 tmp = RREG32(CG_CLKPIN_CNTL);
1242 if (tmp & XTALIN_DIVIDE)
1243 return reference_clock / 4;
1245 return reference_clock;
1248 /* get temperature in millidegrees */
1249 int si_get_temp(struct radeon_device *rdev)
1252 int actual_temp = 0;
1254 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1260 actual_temp = temp & 0x1ff;
1262 actual_temp = (actual_temp * 1000);
1267 #define TAHITI_IO_MC_REGS_SIZE 36
1269 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1270 {0x0000006f, 0x03044000},
1271 {0x00000070, 0x0480c018},
1272 {0x00000071, 0x00000040},
1273 {0x00000072, 0x01000000},
1274 {0x00000074, 0x000000ff},
1275 {0x00000075, 0x00143400},
1276 {0x00000076, 0x08ec0800},
1277 {0x00000077, 0x040000cc},
1278 {0x00000079, 0x00000000},
1279 {0x0000007a, 0x21000409},
1280 {0x0000007c, 0x00000000},
1281 {0x0000007d, 0xe8000000},
1282 {0x0000007e, 0x044408a8},
1283 {0x0000007f, 0x00000003},
1284 {0x00000080, 0x00000000},
1285 {0x00000081, 0x01000000},
1286 {0x00000082, 0x02000000},
1287 {0x00000083, 0x00000000},
1288 {0x00000084, 0xe3f3e4f4},
1289 {0x00000085, 0x00052024},
1290 {0x00000087, 0x00000000},
1291 {0x00000088, 0x66036603},
1292 {0x00000089, 0x01000000},
1293 {0x0000008b, 0x1c0a0000},
1294 {0x0000008c, 0xff010000},
1295 {0x0000008e, 0xffffefff},
1296 {0x0000008f, 0xfff3efff},
1297 {0x00000090, 0xfff3efbf},
1298 {0x00000094, 0x00101101},
1299 {0x00000095, 0x00000fff},
1300 {0x00000096, 0x00116fff},
1301 {0x00000097, 0x60010000},
1302 {0x00000098, 0x10010000},
1303 {0x00000099, 0x00006000},
1304 {0x0000009a, 0x00001000},
1305 {0x0000009f, 0x00a77400}
1308 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1309 {0x0000006f, 0x03044000},
1310 {0x00000070, 0x0480c018},
1311 {0x00000071, 0x00000040},
1312 {0x00000072, 0x01000000},
1313 {0x00000074, 0x000000ff},
1314 {0x00000075, 0x00143400},
1315 {0x00000076, 0x08ec0800},
1316 {0x00000077, 0x040000cc},
1317 {0x00000079, 0x00000000},
1318 {0x0000007a, 0x21000409},
1319 {0x0000007c, 0x00000000},
1320 {0x0000007d, 0xe8000000},
1321 {0x0000007e, 0x044408a8},
1322 {0x0000007f, 0x00000003},
1323 {0x00000080, 0x00000000},
1324 {0x00000081, 0x01000000},
1325 {0x00000082, 0x02000000},
1326 {0x00000083, 0x00000000},
1327 {0x00000084, 0xe3f3e4f4},
1328 {0x00000085, 0x00052024},
1329 {0x00000087, 0x00000000},
1330 {0x00000088, 0x66036603},
1331 {0x00000089, 0x01000000},
1332 {0x0000008b, 0x1c0a0000},
1333 {0x0000008c, 0xff010000},
1334 {0x0000008e, 0xffffefff},
1335 {0x0000008f, 0xfff3efff},
1336 {0x00000090, 0xfff3efbf},
1337 {0x00000094, 0x00101101},
1338 {0x00000095, 0x00000fff},
1339 {0x00000096, 0x00116fff},
1340 {0x00000097, 0x60010000},
1341 {0x00000098, 0x10010000},
1342 {0x00000099, 0x00006000},
1343 {0x0000009a, 0x00001000},
1344 {0x0000009f, 0x00a47400}
1347 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1348 {0x0000006f, 0x03044000},
1349 {0x00000070, 0x0480c018},
1350 {0x00000071, 0x00000040},
1351 {0x00000072, 0x01000000},
1352 {0x00000074, 0x000000ff},
1353 {0x00000075, 0x00143400},
1354 {0x00000076, 0x08ec0800},
1355 {0x00000077, 0x040000cc},
1356 {0x00000079, 0x00000000},
1357 {0x0000007a, 0x21000409},
1358 {0x0000007c, 0x00000000},
1359 {0x0000007d, 0xe8000000},
1360 {0x0000007e, 0x044408a8},
1361 {0x0000007f, 0x00000003},
1362 {0x00000080, 0x00000000},
1363 {0x00000081, 0x01000000},
1364 {0x00000082, 0x02000000},
1365 {0x00000083, 0x00000000},
1366 {0x00000084, 0xe3f3e4f4},
1367 {0x00000085, 0x00052024},
1368 {0x00000087, 0x00000000},
1369 {0x00000088, 0x66036603},
1370 {0x00000089, 0x01000000},
1371 {0x0000008b, 0x1c0a0000},
1372 {0x0000008c, 0xff010000},
1373 {0x0000008e, 0xffffefff},
1374 {0x0000008f, 0xfff3efff},
1375 {0x00000090, 0xfff3efbf},
1376 {0x00000094, 0x00101101},
1377 {0x00000095, 0x00000fff},
1378 {0x00000096, 0x00116fff},
1379 {0x00000097, 0x60010000},
1380 {0x00000098, 0x10010000},
1381 {0x00000099, 0x00006000},
1382 {0x0000009a, 0x00001000},
1383 {0x0000009f, 0x00a37400}
1386 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1387 {0x0000006f, 0x03044000},
1388 {0x00000070, 0x0480c018},
1389 {0x00000071, 0x00000040},
1390 {0x00000072, 0x01000000},
1391 {0x00000074, 0x000000ff},
1392 {0x00000075, 0x00143400},
1393 {0x00000076, 0x08ec0800},
1394 {0x00000077, 0x040000cc},
1395 {0x00000079, 0x00000000},
1396 {0x0000007a, 0x21000409},
1397 {0x0000007c, 0x00000000},
1398 {0x0000007d, 0xe8000000},
1399 {0x0000007e, 0x044408a8},
1400 {0x0000007f, 0x00000003},
1401 {0x00000080, 0x00000000},
1402 {0x00000081, 0x01000000},
1403 {0x00000082, 0x02000000},
1404 {0x00000083, 0x00000000},
1405 {0x00000084, 0xe3f3e4f4},
1406 {0x00000085, 0x00052024},
1407 {0x00000087, 0x00000000},
1408 {0x00000088, 0x66036603},
1409 {0x00000089, 0x01000000},
1410 {0x0000008b, 0x1c0a0000},
1411 {0x0000008c, 0xff010000},
1412 {0x0000008e, 0xffffefff},
1413 {0x0000008f, 0xfff3efff},
1414 {0x00000090, 0xfff3efbf},
1415 {0x00000094, 0x00101101},
1416 {0x00000095, 0x00000fff},
1417 {0x00000096, 0x00116fff},
1418 {0x00000097, 0x60010000},
1419 {0x00000098, 0x10010000},
1420 {0x00000099, 0x00006000},
1421 {0x0000009a, 0x00001000},
1422 {0x0000009f, 0x00a17730}
1425 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1426 {0x0000006f, 0x03044000},
1427 {0x00000070, 0x0480c018},
1428 {0x00000071, 0x00000040},
1429 {0x00000072, 0x01000000},
1430 {0x00000074, 0x000000ff},
1431 {0x00000075, 0x00143400},
1432 {0x00000076, 0x08ec0800},
1433 {0x00000077, 0x040000cc},
1434 {0x00000079, 0x00000000},
1435 {0x0000007a, 0x21000409},
1436 {0x0000007c, 0x00000000},
1437 {0x0000007d, 0xe8000000},
1438 {0x0000007e, 0x044408a8},
1439 {0x0000007f, 0x00000003},
1440 {0x00000080, 0x00000000},
1441 {0x00000081, 0x01000000},
1442 {0x00000082, 0x02000000},
1443 {0x00000083, 0x00000000},
1444 {0x00000084, 0xe3f3e4f4},
1445 {0x00000085, 0x00052024},
1446 {0x00000087, 0x00000000},
1447 {0x00000088, 0x66036603},
1448 {0x00000089, 0x01000000},
1449 {0x0000008b, 0x1c0a0000},
1450 {0x0000008c, 0xff010000},
1451 {0x0000008e, 0xffffefff},
1452 {0x0000008f, 0xfff3efff},
1453 {0x00000090, 0xfff3efbf},
1454 {0x00000094, 0x00101101},
1455 {0x00000095, 0x00000fff},
1456 {0x00000096, 0x00116fff},
1457 {0x00000097, 0x60010000},
1458 {0x00000098, 0x10010000},
1459 {0x00000099, 0x00006000},
1460 {0x0000009a, 0x00001000},
1461 {0x0000009f, 0x00a07730}
1465 int si_mc_load_microcode(struct radeon_device *rdev)
1467 const __be32 *fw_data;
1468 u32 running, blackout = 0;
1470 int i, ucode_size, regs_size;
1475 switch (rdev->family) {
1477 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1478 ucode_size = SI_MC_UCODE_SIZE;
1479 regs_size = TAHITI_IO_MC_REGS_SIZE;
1482 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1483 ucode_size = SI_MC_UCODE_SIZE;
1484 regs_size = TAHITI_IO_MC_REGS_SIZE;
1488 io_mc_regs = (u32 *)&verde_io_mc_regs;
1489 ucode_size = SI_MC_UCODE_SIZE;
1490 regs_size = TAHITI_IO_MC_REGS_SIZE;
1493 io_mc_regs = (u32 *)&oland_io_mc_regs;
1494 ucode_size = OLAND_MC_UCODE_SIZE;
1495 regs_size = TAHITI_IO_MC_REGS_SIZE;
1498 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1499 ucode_size = OLAND_MC_UCODE_SIZE;
1500 regs_size = TAHITI_IO_MC_REGS_SIZE;
1504 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1508 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1509 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1512 /* reset the engine and set to writable */
1513 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1514 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1516 /* load mc io regs */
1517 for (i = 0; i < regs_size; i++) {
1518 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1519 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1521 /* load the MC ucode */
1522 fw_data = (const __be32 *)rdev->mc_fw->data;
1523 for (i = 0; i < ucode_size; i++)
1524 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1526 /* put the engine back into the active state */
1527 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1528 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1529 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1531 /* wait for training to complete */
1532 for (i = 0; i < rdev->usec_timeout; i++) {
1533 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1537 for (i = 0; i < rdev->usec_timeout; i++) {
1538 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1544 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1550 static int si_init_microcode(struct radeon_device *rdev)
1552 const char *chip_name;
1553 const char *rlc_chip_name;
1554 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1555 size_t smc_req_size;
1561 switch (rdev->family) {
1563 chip_name = "TAHITI";
1564 rlc_chip_name = "TAHITI";
1565 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1566 me_req_size = SI_PM4_UCODE_SIZE * 4;
1567 ce_req_size = SI_CE_UCODE_SIZE * 4;
1568 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1569 mc_req_size = SI_MC_UCODE_SIZE * 4;
1570 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1573 chip_name = "PITCAIRN";
1574 rlc_chip_name = "PITCAIRN";
1575 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1576 me_req_size = SI_PM4_UCODE_SIZE * 4;
1577 ce_req_size = SI_CE_UCODE_SIZE * 4;
1578 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1579 mc_req_size = SI_MC_UCODE_SIZE * 4;
1580 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1583 chip_name = "VERDE";
1584 rlc_chip_name = "VERDE";
1585 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1586 me_req_size = SI_PM4_UCODE_SIZE * 4;
1587 ce_req_size = SI_CE_UCODE_SIZE * 4;
1588 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1589 mc_req_size = SI_MC_UCODE_SIZE * 4;
1590 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1593 chip_name = "OLAND";
1594 rlc_chip_name = "OLAND";
1595 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1596 me_req_size = SI_PM4_UCODE_SIZE * 4;
1597 ce_req_size = SI_CE_UCODE_SIZE * 4;
1598 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1599 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1600 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1603 chip_name = "HAINAN";
1604 rlc_chip_name = "HAINAN";
1605 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1606 me_req_size = SI_PM4_UCODE_SIZE * 4;
1607 ce_req_size = SI_CE_UCODE_SIZE * 4;
1608 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1609 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1610 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1615 DRM_INFO("Loading %s Microcode\n", chip_name);
1617 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1618 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1621 if (rdev->pfp_fw->size != pfp_req_size) {
1623 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1624 rdev->pfp_fw->size, fw_name);
1629 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1630 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1633 if (rdev->me_fw->size != me_req_size) {
1635 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1636 rdev->me_fw->size, fw_name);
1640 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1641 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1644 if (rdev->ce_fw->size != ce_req_size) {
1646 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1647 rdev->ce_fw->size, fw_name);
1651 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1652 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1655 if (rdev->rlc_fw->size != rlc_req_size) {
1657 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1658 rdev->rlc_fw->size, fw_name);
1662 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1663 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1666 if (rdev->mc_fw->size != mc_req_size) {
1668 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1669 rdev->mc_fw->size, fw_name);
1673 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1674 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1677 "smc: error loading firmware \"%s\"\n",
1679 release_firmware(rdev->smc_fw);
1680 rdev->smc_fw = NULL;
1682 } else if (rdev->smc_fw->size != smc_req_size) {
1684 "si_smc: Bogus length %zu in firmware \"%s\"\n",
1685 rdev->smc_fw->size, fw_name);
1693 "si_cp: Failed to load firmware \"%s\"\n",
1695 release_firmware(rdev->pfp_fw);
1696 rdev->pfp_fw = NULL;
1697 release_firmware(rdev->me_fw);
1699 release_firmware(rdev->ce_fw);
1701 release_firmware(rdev->rlc_fw);
1702 rdev->rlc_fw = NULL;
1703 release_firmware(rdev->mc_fw);
1705 release_firmware(rdev->smc_fw);
1706 rdev->smc_fw = NULL;
1711 /* watermark setup */
1712 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1713 struct radeon_crtc *radeon_crtc,
1714 struct drm_display_mode *mode,
1715 struct drm_display_mode *other_mode)
1717 u32 tmp, buffer_alloc, i;
1718 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1721 * There are 3 line buffers, each one shared by 2 display controllers.
1722 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1723 * the display controllers. The paritioning is done via one of four
1724 * preset allocations specified in bits 21:20:
1726 * 2 - whole lb, other crtc must be disabled
1728 /* this can get tricky if we have two large displays on a paired group
1729 * of crtcs. Ideally for multiple large displays we'd assign them to
1730 * non-linked crtcs for maximum line buffer allocation.
1732 if (radeon_crtc->base.enabled && mode) {
1737 tmp = 2; /* whole */
1745 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1746 DC_LB_MEMORY_CONFIG(tmp));
1748 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1749 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1750 for (i = 0; i < rdev->usec_timeout; i++) {
1751 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1752 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1757 if (radeon_crtc->base.enabled && mode) {
1767 /* controller not enabled, so no lb used */
1771 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1773 u32 tmp = RREG32(MC_SHARED_CHMAP);
1775 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1798 struct dce6_wm_params {
1799 u32 dram_channels; /* number of dram channels */
1800 u32 yclk; /* bandwidth per dram data pin in kHz */
1801 u32 sclk; /* engine clock in kHz */
1802 u32 disp_clk; /* display clock in kHz */
1803 u32 src_width; /* viewport width */
1804 u32 active_time; /* active display time in ns */
1805 u32 blank_time; /* blank time in ns */
1806 bool interlaced; /* mode is interlaced */
1807 fixed20_12 vsc; /* vertical scale ratio */
1808 u32 num_heads; /* number of active crtcs */
1809 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1810 u32 lb_size; /* line buffer allocated to pipe */
1811 u32 vtaps; /* vertical scaler taps */
1814 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1816 /* Calculate raw DRAM Bandwidth */
1817 fixed20_12 dram_efficiency; /* 0.7 */
1818 fixed20_12 yclk, dram_channels, bandwidth;
1821 a.full = dfixed_const(1000);
1822 yclk.full = dfixed_const(wm->yclk);
1823 yclk.full = dfixed_div(yclk, a);
1824 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1825 a.full = dfixed_const(10);
1826 dram_efficiency.full = dfixed_const(7);
1827 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1828 bandwidth.full = dfixed_mul(dram_channels, yclk);
1829 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1831 return dfixed_trunc(bandwidth);
1834 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1836 /* Calculate DRAM Bandwidth and the part allocated to display. */
1837 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1838 fixed20_12 yclk, dram_channels, bandwidth;
1841 a.full = dfixed_const(1000);
1842 yclk.full = dfixed_const(wm->yclk);
1843 yclk.full = dfixed_div(yclk, a);
1844 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1845 a.full = dfixed_const(10);
1846 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1847 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1848 bandwidth.full = dfixed_mul(dram_channels, yclk);
1849 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1851 return dfixed_trunc(bandwidth);
1854 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1856 /* Calculate the display Data return Bandwidth */
1857 fixed20_12 return_efficiency; /* 0.8 */
1858 fixed20_12 sclk, bandwidth;
1861 a.full = dfixed_const(1000);
1862 sclk.full = dfixed_const(wm->sclk);
1863 sclk.full = dfixed_div(sclk, a);
1864 a.full = dfixed_const(10);
1865 return_efficiency.full = dfixed_const(8);
1866 return_efficiency.full = dfixed_div(return_efficiency, a);
1867 a.full = dfixed_const(32);
1868 bandwidth.full = dfixed_mul(a, sclk);
1869 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1871 return dfixed_trunc(bandwidth);
1874 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1879 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1881 /* Calculate the DMIF Request Bandwidth */
1882 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1883 fixed20_12 disp_clk, sclk, bandwidth;
1884 fixed20_12 a, b1, b2;
1887 a.full = dfixed_const(1000);
1888 disp_clk.full = dfixed_const(wm->disp_clk);
1889 disp_clk.full = dfixed_div(disp_clk, a);
1890 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1891 b1.full = dfixed_mul(a, disp_clk);
1893 a.full = dfixed_const(1000);
1894 sclk.full = dfixed_const(wm->sclk);
1895 sclk.full = dfixed_div(sclk, a);
1896 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1897 b2.full = dfixed_mul(a, sclk);
1899 a.full = dfixed_const(10);
1900 disp_clk_request_efficiency.full = dfixed_const(8);
1901 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1903 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1905 a.full = dfixed_const(min_bandwidth);
1906 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1908 return dfixed_trunc(bandwidth);
1911 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1913 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1914 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1915 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1916 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1918 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1921 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1923 /* Calculate the display mode Average Bandwidth
1924 * DisplayMode should contain the source and destination dimensions,
1928 fixed20_12 line_time;
1929 fixed20_12 src_width;
1930 fixed20_12 bandwidth;
1933 a.full = dfixed_const(1000);
1934 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1935 line_time.full = dfixed_div(line_time, a);
1936 bpp.full = dfixed_const(wm->bytes_per_pixel);
1937 src_width.full = dfixed_const(wm->src_width);
1938 bandwidth.full = dfixed_mul(src_width, bpp);
1939 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1940 bandwidth.full = dfixed_div(bandwidth, line_time);
1942 return dfixed_trunc(bandwidth);
1945 static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1947 /* First calcualte the latency in ns */
1948 u32 mc_latency = 2000; /* 2000 ns. */
1949 u32 available_bandwidth = dce6_available_bandwidth(wm);
1950 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1951 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1952 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1953 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1954 (wm->num_heads * cursor_line_pair_return_time);
1955 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1956 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1957 u32 tmp, dmif_size = 12288;
1960 if (wm->num_heads == 0)
1963 a.full = dfixed_const(2);
1964 b.full = dfixed_const(1);
1965 if ((wm->vsc.full > a.full) ||
1966 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1968 ((wm->vsc.full >= a.full) && wm->interlaced))
1969 max_src_lines_per_dst_line = 4;
1971 max_src_lines_per_dst_line = 2;
1973 a.full = dfixed_const(available_bandwidth);
1974 b.full = dfixed_const(wm->num_heads);
1975 a.full = dfixed_div(a, b);
1977 b.full = dfixed_const(mc_latency + 512);
1978 c.full = dfixed_const(wm->disp_clk);
1979 b.full = dfixed_div(b, c);
1981 c.full = dfixed_const(dmif_size);
1982 b.full = dfixed_div(c, b);
1984 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1986 b.full = dfixed_const(1000);
1987 c.full = dfixed_const(wm->disp_clk);
1988 b.full = dfixed_div(c, b);
1989 c.full = dfixed_const(wm->bytes_per_pixel);
1990 b.full = dfixed_mul(b, c);
1992 lb_fill_bw = min(tmp, dfixed_trunc(b));
1994 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1995 b.full = dfixed_const(1000);
1996 c.full = dfixed_const(lb_fill_bw);
1997 b.full = dfixed_div(c, b);
1998 a.full = dfixed_div(a, b);
1999 line_fill_time = dfixed_trunc(a);
2001 if (line_fill_time < wm->active_time)
2004 return latency + (line_fill_time - wm->active_time);
2008 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2010 if (dce6_average_bandwidth(wm) <=
2011 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2017 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2019 if (dce6_average_bandwidth(wm) <=
2020 (dce6_available_bandwidth(wm) / wm->num_heads))
2026 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2028 u32 lb_partitions = wm->lb_size / wm->src_width;
2029 u32 line_time = wm->active_time + wm->blank_time;
2030 u32 latency_tolerant_lines;
2034 a.full = dfixed_const(1);
2035 if (wm->vsc.full > a.full)
2036 latency_tolerant_lines = 1;
2038 if (lb_partitions <= (wm->vtaps + 1))
2039 latency_tolerant_lines = 1;
2041 latency_tolerant_lines = 2;
2044 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2046 if (dce6_latency_watermark(wm) <= latency_hiding)
2052 static void dce6_program_watermarks(struct radeon_device *rdev,
2053 struct radeon_crtc *radeon_crtc,
2054 u32 lb_size, u32 num_heads)
2056 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2057 struct dce6_wm_params wm_low, wm_high;
2061 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2062 u32 priority_a_mark = 0, priority_b_mark = 0;
2063 u32 priority_a_cnt = PRIORITY_OFF;
2064 u32 priority_b_cnt = PRIORITY_OFF;
2065 u32 tmp, arb_control3;
2068 if (radeon_crtc->base.enabled && num_heads && mode) {
2069 pixel_period = 1000000 / (u32)mode->clock;
2070 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2074 if (rdev->family == CHIP_ARUBA)
2075 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2077 dram_channels = si_get_number_of_dram_channels(rdev);
2079 /* watermark for high clocks */
2080 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2082 radeon_dpm_get_mclk(rdev, false) * 10;
2084 radeon_dpm_get_sclk(rdev, false) * 10;
2086 wm_high.yclk = rdev->pm.current_mclk * 10;
2087 wm_high.sclk = rdev->pm.current_sclk * 10;
2090 wm_high.disp_clk = mode->clock;
2091 wm_high.src_width = mode->crtc_hdisplay;
2092 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2093 wm_high.blank_time = line_time - wm_high.active_time;
2094 wm_high.interlaced = false;
2095 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2096 wm_high.interlaced = true;
2097 wm_high.vsc = radeon_crtc->vsc;
2099 if (radeon_crtc->rmx_type != RMX_OFF)
2101 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2102 wm_high.lb_size = lb_size;
2103 wm_high.dram_channels = dram_channels;
2104 wm_high.num_heads = num_heads;
2106 /* watermark for low clocks */
2107 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2109 radeon_dpm_get_mclk(rdev, true) * 10;
2111 radeon_dpm_get_sclk(rdev, true) * 10;
2113 wm_low.yclk = rdev->pm.current_mclk * 10;
2114 wm_low.sclk = rdev->pm.current_sclk * 10;
2117 wm_low.disp_clk = mode->clock;
2118 wm_low.src_width = mode->crtc_hdisplay;
2119 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2120 wm_low.blank_time = line_time - wm_low.active_time;
2121 wm_low.interlaced = false;
2122 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2123 wm_low.interlaced = true;
2124 wm_low.vsc = radeon_crtc->vsc;
2126 if (radeon_crtc->rmx_type != RMX_OFF)
2128 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2129 wm_low.lb_size = lb_size;
2130 wm_low.dram_channels = dram_channels;
2131 wm_low.num_heads = num_heads;
2133 /* set for high clocks */
2134 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2135 /* set for low clocks */
2136 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2138 /* possibly force display priority to high */
2139 /* should really do this at mode validation time... */
2140 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2141 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2142 !dce6_check_latency_hiding(&wm_high) ||
2143 (rdev->disp_priority == 2)) {
2144 DRM_DEBUG_KMS("force priority to high\n");
2145 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2146 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2148 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2149 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2150 !dce6_check_latency_hiding(&wm_low) ||
2151 (rdev->disp_priority == 2)) {
2152 DRM_DEBUG_KMS("force priority to high\n");
2153 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2154 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2157 a.full = dfixed_const(1000);
2158 b.full = dfixed_const(mode->clock);
2159 b.full = dfixed_div(b, a);
2160 c.full = dfixed_const(latency_watermark_a);
2161 c.full = dfixed_mul(c, b);
2162 c.full = dfixed_mul(c, radeon_crtc->hsc);
2163 c.full = dfixed_div(c, a);
2164 a.full = dfixed_const(16);
2165 c.full = dfixed_div(c, a);
2166 priority_a_mark = dfixed_trunc(c);
2167 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2169 a.full = dfixed_const(1000);
2170 b.full = dfixed_const(mode->clock);
2171 b.full = dfixed_div(b, a);
2172 c.full = dfixed_const(latency_watermark_b);
2173 c.full = dfixed_mul(c, b);
2174 c.full = dfixed_mul(c, radeon_crtc->hsc);
2175 c.full = dfixed_div(c, a);
2176 a.full = dfixed_const(16);
2177 c.full = dfixed_div(c, a);
2178 priority_b_mark = dfixed_trunc(c);
2179 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2183 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2185 tmp &= ~LATENCY_WATERMARK_MASK(3);
2186 tmp |= LATENCY_WATERMARK_MASK(1);
2187 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2188 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2189 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2190 LATENCY_HIGH_WATERMARK(line_time)));
2192 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2193 tmp &= ~LATENCY_WATERMARK_MASK(3);
2194 tmp |= LATENCY_WATERMARK_MASK(2);
2195 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2196 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2197 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2198 LATENCY_HIGH_WATERMARK(line_time)));
2199 /* restore original selection */
2200 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2202 /* write the priority marks */
2203 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2204 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2206 /* save values for DPM */
2207 radeon_crtc->line_time = line_time;
2208 radeon_crtc->wm_high = latency_watermark_a;
2209 radeon_crtc->wm_low = latency_watermark_b;
2212 void dce6_bandwidth_update(struct radeon_device *rdev)
2214 struct drm_display_mode *mode0 = NULL;
2215 struct drm_display_mode *mode1 = NULL;
2216 u32 num_heads = 0, lb_size;
2219 radeon_update_display_priority(rdev);
2221 for (i = 0; i < rdev->num_crtc; i++) {
2222 if (rdev->mode_info.crtcs[i]->base.enabled)
2225 for (i = 0; i < rdev->num_crtc; i += 2) {
2226 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2227 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2228 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2229 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2230 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2231 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2238 static void si_tiling_mode_table_init(struct radeon_device *rdev)
2240 const u32 num_tile_mode_states = 32;
2241 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2243 switch (rdev->config.si.mem_row_size_in_kb) {
2245 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2249 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2252 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2256 if ((rdev->family == CHIP_TAHITI) ||
2257 (rdev->family == CHIP_PITCAIRN)) {
2258 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2259 switch (reg_offset) {
2260 case 0: /* non-AA compressed depth or any compressed stencil */
2261 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2262 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2263 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2264 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2265 NUM_BANKS(ADDR_SURF_16_BANK) |
2266 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2267 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2268 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2270 case 1: /* 2xAA/4xAA compressed depth only */
2271 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2272 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2273 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2274 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2275 NUM_BANKS(ADDR_SURF_16_BANK) |
2276 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2277 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2278 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2280 case 2: /* 8xAA compressed depth only */
2281 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2282 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2283 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2284 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2285 NUM_BANKS(ADDR_SURF_16_BANK) |
2286 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2287 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2288 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2290 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2291 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2292 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2293 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2294 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2295 NUM_BANKS(ADDR_SURF_16_BANK) |
2296 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2297 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2298 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2300 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2301 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2302 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2303 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2304 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2305 NUM_BANKS(ADDR_SURF_16_BANK) |
2306 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2307 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2308 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2310 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2311 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2312 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2313 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2314 TILE_SPLIT(split_equal_to_row_size) |
2315 NUM_BANKS(ADDR_SURF_16_BANK) |
2316 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2317 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2318 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2320 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2321 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2322 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2323 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2324 TILE_SPLIT(split_equal_to_row_size) |
2325 NUM_BANKS(ADDR_SURF_16_BANK) |
2326 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2327 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2328 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2330 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2331 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2332 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2333 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2334 TILE_SPLIT(split_equal_to_row_size) |
2335 NUM_BANKS(ADDR_SURF_16_BANK) |
2336 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2337 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2338 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2340 case 8: /* 1D and 1D Array Surfaces */
2341 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2342 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2343 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2344 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2345 NUM_BANKS(ADDR_SURF_16_BANK) |
2346 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2347 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2348 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2350 case 9: /* Displayable maps. */
2351 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2352 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2353 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2354 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2355 NUM_BANKS(ADDR_SURF_16_BANK) |
2356 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2357 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2358 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2360 case 10: /* Display 8bpp. */
2361 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2362 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2363 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2364 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2365 NUM_BANKS(ADDR_SURF_16_BANK) |
2366 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2367 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2368 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2370 case 11: /* Display 16bpp. */
2371 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2372 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2373 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2374 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2375 NUM_BANKS(ADDR_SURF_16_BANK) |
2376 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2377 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2378 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2380 case 12: /* Display 32bpp. */
2381 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2382 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2383 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2384 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2385 NUM_BANKS(ADDR_SURF_16_BANK) |
2386 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2387 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2388 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2390 case 13: /* Thin. */
2391 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2392 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2393 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2394 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2395 NUM_BANKS(ADDR_SURF_16_BANK) |
2396 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2397 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2398 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2400 case 14: /* Thin 8 bpp. */
2401 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2402 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2403 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2404 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2405 NUM_BANKS(ADDR_SURF_16_BANK) |
2406 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2407 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2408 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2410 case 15: /* Thin 16 bpp. */
2411 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2412 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2413 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2414 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2415 NUM_BANKS(ADDR_SURF_16_BANK) |
2416 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2417 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2418 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2420 case 16: /* Thin 32 bpp. */
2421 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2422 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2423 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2424 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2425 NUM_BANKS(ADDR_SURF_16_BANK) |
2426 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2427 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2428 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2430 case 17: /* Thin 64 bpp. */
2431 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2432 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2433 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2434 TILE_SPLIT(split_equal_to_row_size) |
2435 NUM_BANKS(ADDR_SURF_16_BANK) |
2436 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2437 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2438 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2440 case 21: /* 8 bpp PRT. */
2441 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2442 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2443 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2444 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2445 NUM_BANKS(ADDR_SURF_16_BANK) |
2446 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2447 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2448 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2450 case 22: /* 16 bpp PRT */
2451 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2452 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2453 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2454 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2455 NUM_BANKS(ADDR_SURF_16_BANK) |
2456 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2457 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2458 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2460 case 23: /* 32 bpp PRT */
2461 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2462 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2463 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2464 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2465 NUM_BANKS(ADDR_SURF_16_BANK) |
2466 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2467 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2468 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2470 case 24: /* 64 bpp PRT */
2471 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2472 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2473 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2474 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2475 NUM_BANKS(ADDR_SURF_16_BANK) |
2476 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2477 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2478 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2480 case 25: /* 128 bpp PRT */
2481 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2482 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2483 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2484 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2485 NUM_BANKS(ADDR_SURF_8_BANK) |
2486 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2487 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2488 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2494 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2495 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2497 } else if ((rdev->family == CHIP_VERDE) ||
2498 (rdev->family == CHIP_OLAND) ||
2499 (rdev->family == CHIP_HAINAN)) {
2500 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2501 switch (reg_offset) {
2502 case 0: /* non-AA compressed depth or any compressed stencil */
2503 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2504 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2505 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2506 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2507 NUM_BANKS(ADDR_SURF_16_BANK) |
2508 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2509 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2510 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2512 case 1: /* 2xAA/4xAA compressed depth only */
2513 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2514 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2515 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2516 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2517 NUM_BANKS(ADDR_SURF_16_BANK) |
2518 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2519 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2520 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2522 case 2: /* 8xAA compressed depth only */
2523 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2524 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2525 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2526 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2527 NUM_BANKS(ADDR_SURF_16_BANK) |
2528 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2529 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2530 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2532 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2533 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2534 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2535 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2536 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2537 NUM_BANKS(ADDR_SURF_16_BANK) |
2538 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2539 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2540 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2542 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2543 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2544 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2545 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2546 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2547 NUM_BANKS(ADDR_SURF_16_BANK) |
2548 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2549 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2550 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2552 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2553 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2554 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2555 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2556 TILE_SPLIT(split_equal_to_row_size) |
2557 NUM_BANKS(ADDR_SURF_16_BANK) |
2558 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2559 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2560 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2562 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2563 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2564 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2565 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2566 TILE_SPLIT(split_equal_to_row_size) |
2567 NUM_BANKS(ADDR_SURF_16_BANK) |
2568 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2569 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2570 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2572 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2573 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2574 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2575 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2576 TILE_SPLIT(split_equal_to_row_size) |
2577 NUM_BANKS(ADDR_SURF_16_BANK) |
2578 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2579 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2580 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2582 case 8: /* 1D and 1D Array Surfaces */
2583 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2584 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2585 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2586 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2587 NUM_BANKS(ADDR_SURF_16_BANK) |
2588 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2589 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2590 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2592 case 9: /* Displayable maps. */
2593 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2594 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2595 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2596 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2597 NUM_BANKS(ADDR_SURF_16_BANK) |
2598 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2599 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2600 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2602 case 10: /* Display 8bpp. */
2603 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2604 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2605 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2606 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2607 NUM_BANKS(ADDR_SURF_16_BANK) |
2608 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2609 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2610 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2612 case 11: /* Display 16bpp. */
2613 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2614 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2615 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2616 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2617 NUM_BANKS(ADDR_SURF_16_BANK) |
2618 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2619 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2620 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2622 case 12: /* Display 32bpp. */
2623 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2624 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2625 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2626 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2627 NUM_BANKS(ADDR_SURF_16_BANK) |
2628 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2629 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2630 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2632 case 13: /* Thin. */
2633 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2634 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2635 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2636 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2637 NUM_BANKS(ADDR_SURF_16_BANK) |
2638 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2639 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2640 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2642 case 14: /* Thin 8 bpp. */
2643 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2644 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2645 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2646 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2647 NUM_BANKS(ADDR_SURF_16_BANK) |
2648 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2649 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2650 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2652 case 15: /* Thin 16 bpp. */
2653 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2654 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2655 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2656 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2657 NUM_BANKS(ADDR_SURF_16_BANK) |
2658 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2659 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2660 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2662 case 16: /* Thin 32 bpp. */
2663 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2664 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2665 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2666 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2667 NUM_BANKS(ADDR_SURF_16_BANK) |
2668 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2669 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2670 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2672 case 17: /* Thin 64 bpp. */
2673 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2674 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2675 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2676 TILE_SPLIT(split_equal_to_row_size) |
2677 NUM_BANKS(ADDR_SURF_16_BANK) |
2678 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2679 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2680 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2682 case 21: /* 8 bpp PRT. */
2683 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2684 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2685 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2686 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2687 NUM_BANKS(ADDR_SURF_16_BANK) |
2688 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2689 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2690 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2692 case 22: /* 16 bpp PRT */
2693 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2694 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2695 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2696 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2697 NUM_BANKS(ADDR_SURF_16_BANK) |
2698 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2699 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2700 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2702 case 23: /* 32 bpp PRT */
2703 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2704 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2705 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2706 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2707 NUM_BANKS(ADDR_SURF_16_BANK) |
2708 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2709 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2710 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2712 case 24: /* 64 bpp PRT */
2713 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2714 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2715 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2716 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2717 NUM_BANKS(ADDR_SURF_16_BANK) |
2718 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2719 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2720 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2722 case 25: /* 128 bpp PRT */
2723 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2724 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2725 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2726 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2727 NUM_BANKS(ADDR_SURF_8_BANK) |
2728 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2729 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2730 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2736 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2737 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2740 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2743 static void si_select_se_sh(struct radeon_device *rdev,
2744 u32 se_num, u32 sh_num)
2746 u32 data = INSTANCE_BROADCAST_WRITES;
2748 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2749 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2750 else if (se_num == 0xffffffff)
2751 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2752 else if (sh_num == 0xffffffff)
2753 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2755 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2756 WREG32(GRBM_GFX_INDEX, data);
2759 static u32 si_create_bitmask(u32 bit_width)
2763 for (i = 0; i < bit_width; i++) {
2770 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2774 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2776 data &= INACTIVE_CUS_MASK;
2779 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2781 data >>= INACTIVE_CUS_SHIFT;
2783 mask = si_create_bitmask(cu_per_sh);
2785 return ~data & mask;
2788 static void si_setup_spi(struct radeon_device *rdev,
2789 u32 se_num, u32 sh_per_se,
2793 u32 data, mask, active_cu;
2795 for (i = 0; i < se_num; i++) {
2796 for (j = 0; j < sh_per_se; j++) {
2797 si_select_se_sh(rdev, i, j);
2798 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2799 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2802 for (k = 0; k < 16; k++) {
2804 if (active_cu & mask) {
2806 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2812 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2815 static u32 si_get_rb_disabled(struct radeon_device *rdev,
2816 u32 max_rb_num_per_se,
2821 data = RREG32(CC_RB_BACKEND_DISABLE);
2823 data &= BACKEND_DISABLE_MASK;
2826 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2828 data >>= BACKEND_DISABLE_SHIFT;
2830 mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
2835 static void si_setup_rb(struct radeon_device *rdev,
2836 u32 se_num, u32 sh_per_se,
2837 u32 max_rb_num_per_se)
2841 u32 disabled_rbs = 0;
2842 u32 enabled_rbs = 0;
2844 for (i = 0; i < se_num; i++) {
2845 for (j = 0; j < sh_per_se; j++) {
2846 si_select_se_sh(rdev, i, j);
2847 data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
2848 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2851 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2854 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
2855 if (!(disabled_rbs & mask))
2856 enabled_rbs |= mask;
2860 rdev->config.si.backend_enable_mask = enabled_rbs;
2862 for (i = 0; i < se_num; i++) {
2863 si_select_se_sh(rdev, i, 0xffffffff);
2865 for (j = 0; j < sh_per_se; j++) {
2866 switch (enabled_rbs & 3) {
2868 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2871 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2875 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2880 WREG32(PA_SC_RASTER_CONFIG, data);
2882 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2885 static void si_gpu_init(struct radeon_device *rdev)
2887 u32 gb_addr_config = 0;
2888 u32 mc_shared_chmap, mc_arb_ramcfg;
2890 u32 hdp_host_path_cntl;
2894 switch (rdev->family) {
2896 rdev->config.si.max_shader_engines = 2;
2897 rdev->config.si.max_tile_pipes = 12;
2898 rdev->config.si.max_cu_per_sh = 8;
2899 rdev->config.si.max_sh_per_se = 2;
2900 rdev->config.si.max_backends_per_se = 4;
2901 rdev->config.si.max_texture_channel_caches = 12;
2902 rdev->config.si.max_gprs = 256;
2903 rdev->config.si.max_gs_threads = 32;
2904 rdev->config.si.max_hw_contexts = 8;
2906 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2907 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2908 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2909 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2910 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2913 rdev->config.si.max_shader_engines = 2;
2914 rdev->config.si.max_tile_pipes = 8;
2915 rdev->config.si.max_cu_per_sh = 5;
2916 rdev->config.si.max_sh_per_se = 2;
2917 rdev->config.si.max_backends_per_se = 4;
2918 rdev->config.si.max_texture_channel_caches = 8;
2919 rdev->config.si.max_gprs = 256;
2920 rdev->config.si.max_gs_threads = 32;
2921 rdev->config.si.max_hw_contexts = 8;
2923 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2924 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2925 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2926 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2927 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2931 rdev->config.si.max_shader_engines = 1;
2932 rdev->config.si.max_tile_pipes = 4;
2933 rdev->config.si.max_cu_per_sh = 5;
2934 rdev->config.si.max_sh_per_se = 2;
2935 rdev->config.si.max_backends_per_se = 4;
2936 rdev->config.si.max_texture_channel_caches = 4;
2937 rdev->config.si.max_gprs = 256;
2938 rdev->config.si.max_gs_threads = 32;
2939 rdev->config.si.max_hw_contexts = 8;
2941 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2942 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2943 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2944 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2945 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2948 rdev->config.si.max_shader_engines = 1;
2949 rdev->config.si.max_tile_pipes = 4;
2950 rdev->config.si.max_cu_per_sh = 6;
2951 rdev->config.si.max_sh_per_se = 1;
2952 rdev->config.si.max_backends_per_se = 2;
2953 rdev->config.si.max_texture_channel_caches = 4;
2954 rdev->config.si.max_gprs = 256;
2955 rdev->config.si.max_gs_threads = 16;
2956 rdev->config.si.max_hw_contexts = 8;
2958 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2959 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2960 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2961 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2962 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2965 rdev->config.si.max_shader_engines = 1;
2966 rdev->config.si.max_tile_pipes = 4;
2967 rdev->config.si.max_cu_per_sh = 5;
2968 rdev->config.si.max_sh_per_se = 1;
2969 rdev->config.si.max_backends_per_se = 1;
2970 rdev->config.si.max_texture_channel_caches = 2;
2971 rdev->config.si.max_gprs = 256;
2972 rdev->config.si.max_gs_threads = 16;
2973 rdev->config.si.max_hw_contexts = 8;
2975 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2976 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2977 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2978 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2979 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2983 /* Initialize HDP */
2984 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2985 WREG32((0x2c14 + j), 0x00000000);
2986 WREG32((0x2c18 + j), 0x00000000);
2987 WREG32((0x2c1c + j), 0x00000000);
2988 WREG32((0x2c20 + j), 0x00000000);
2989 WREG32((0x2c24 + j), 0x00000000);
2992 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2994 evergreen_fix_pci_max_read_req_size(rdev);
2996 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2998 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2999 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3001 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3002 rdev->config.si.mem_max_burst_length_bytes = 256;
3003 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3004 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3005 if (rdev->config.si.mem_row_size_in_kb > 4)
3006 rdev->config.si.mem_row_size_in_kb = 4;
3007 /* XXX use MC settings? */
3008 rdev->config.si.shader_engine_tile_size = 32;
3009 rdev->config.si.num_gpus = 1;
3010 rdev->config.si.multi_gpu_tile_size = 64;
3012 /* fix up row size */
3013 gb_addr_config &= ~ROW_SIZE_MASK;
3014 switch (rdev->config.si.mem_row_size_in_kb) {
3017 gb_addr_config |= ROW_SIZE(0);
3020 gb_addr_config |= ROW_SIZE(1);
3023 gb_addr_config |= ROW_SIZE(2);
3027 /* setup tiling info dword. gb_addr_config is not adequate since it does
3028 * not have bank info, so create a custom tiling dword.
3029 * bits 3:0 num_pipes
3030 * bits 7:4 num_banks
3031 * bits 11:8 group_size
3032 * bits 15:12 row_size
3034 rdev->config.si.tile_config = 0;
3035 switch (rdev->config.si.num_tile_pipes) {
3037 rdev->config.si.tile_config |= (0 << 0);
3040 rdev->config.si.tile_config |= (1 << 0);
3043 rdev->config.si.tile_config |= (2 << 0);
3047 /* XXX what about 12? */
3048 rdev->config.si.tile_config |= (3 << 0);
3051 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3052 case 0: /* four banks */
3053 rdev->config.si.tile_config |= 0 << 4;
3055 case 1: /* eight banks */
3056 rdev->config.si.tile_config |= 1 << 4;
3058 case 2: /* sixteen banks */
3060 rdev->config.si.tile_config |= 2 << 4;
3063 rdev->config.si.tile_config |=
3064 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3065 rdev->config.si.tile_config |=
3066 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3068 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3069 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3070 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3071 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3072 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3073 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3074 if (rdev->has_uvd) {
3075 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3076 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3077 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3080 si_tiling_mode_table_init(rdev);
3082 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3083 rdev->config.si.max_sh_per_se,
3084 rdev->config.si.max_backends_per_se);
3086 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3087 rdev->config.si.max_sh_per_se,
3088 rdev->config.si.max_cu_per_sh);
3091 /* set HW defaults for 3D engine */
3092 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3093 ROQ_IB2_START(0x2b)));
3094 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3096 sx_debug_1 = RREG32(SX_DEBUG_1);
3097 WREG32(SX_DEBUG_1, sx_debug_1);
3099 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3101 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3102 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3103 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3104 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3106 WREG32(VGT_NUM_INSTANCES, 1);
3108 WREG32(CP_PERFMON_CNTL, 0);
3110 WREG32(SQ_CONFIG, 0);
3112 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3113 FORCE_EOV_MAX_REZ_CNT(255)));
3115 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3116 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3118 WREG32(VGT_GS_VERTEX_REUSE, 16);
3119 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3121 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3122 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3123 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3124 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3125 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3126 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3127 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3128 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3130 tmp = RREG32(HDP_MISC_CNTL);
3131 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3132 WREG32(HDP_MISC_CNTL, tmp);
3134 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3135 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3137 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3143 * GPU scratch registers helpers function.
3145 static void si_scratch_init(struct radeon_device *rdev)
3149 rdev->scratch.num_reg = 7;
3150 rdev->scratch.reg_base = SCRATCH_REG0;
3151 for (i = 0; i < rdev->scratch.num_reg; i++) {
3152 rdev->scratch.free[i] = true;
3153 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3157 void si_fence_ring_emit(struct radeon_device *rdev,
3158 struct radeon_fence *fence)
3160 struct radeon_ring *ring = &rdev->ring[fence->ring];
3161 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3163 /* flush read cache over gart */
3164 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3165 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3166 radeon_ring_write(ring, 0);
3167 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3168 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3169 PACKET3_TC_ACTION_ENA |
3170 PACKET3_SH_KCACHE_ACTION_ENA |
3171 PACKET3_SH_ICACHE_ACTION_ENA);
3172 radeon_ring_write(ring, 0xFFFFFFFF);
3173 radeon_ring_write(ring, 0);
3174 radeon_ring_write(ring, 10); /* poll interval */
3175 /* EVENT_WRITE_EOP - flush caches, send int */
3176 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3177 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3178 radeon_ring_write(ring, addr & 0xffffffff);
3179 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3180 radeon_ring_write(ring, fence->seq);
3181 radeon_ring_write(ring, 0);
3187 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3189 struct radeon_ring *ring = &rdev->ring[ib->ring];
3192 if (ib->is_const_ib) {
3193 /* set switch buffer packet before const IB */
3194 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3195 radeon_ring_write(ring, 0);
3197 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3200 if (ring->rptr_save_reg) {
3201 next_rptr = ring->wptr + 3 + 4 + 8;
3202 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3203 radeon_ring_write(ring, ((ring->rptr_save_reg -
3204 PACKET3_SET_CONFIG_REG_START) >> 2));
3205 radeon_ring_write(ring, next_rptr);
3206 } else if (rdev->wb.enabled) {
3207 next_rptr = ring->wptr + 5 + 4 + 8;
3208 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3209 radeon_ring_write(ring, (1 << 8));
3210 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3211 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3212 radeon_ring_write(ring, next_rptr);
3215 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3218 radeon_ring_write(ring, header);
3219 radeon_ring_write(ring,
3223 (ib->gpu_addr & 0xFFFFFFFC));
3224 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3225 radeon_ring_write(ring, ib->length_dw |
3226 (ib->vm ? (ib->vm->id << 24) : 0));
3228 if (!ib->is_const_ib) {
3229 /* flush read cache over gart for this vmid */
3230 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3231 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3232 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
3233 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3234 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3235 PACKET3_TC_ACTION_ENA |
3236 PACKET3_SH_KCACHE_ACTION_ENA |
3237 PACKET3_SH_ICACHE_ACTION_ENA);
3238 radeon_ring_write(ring, 0xFFFFFFFF);
3239 radeon_ring_write(ring, 0);
3240 radeon_ring_write(ring, 10); /* poll interval */
3247 static void si_cp_enable(struct radeon_device *rdev, bool enable)
3250 WREG32(CP_ME_CNTL, 0);
3252 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3253 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3254 WREG32(SCRATCH_UMSK, 0);
3255 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3256 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3257 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3262 static int si_cp_load_microcode(struct radeon_device *rdev)
3264 const __be32 *fw_data;
3267 if (!rdev->me_fw || !rdev->pfp_fw)
3270 si_cp_enable(rdev, false);
3273 fw_data = (const __be32 *)rdev->pfp_fw->data;
3274 WREG32(CP_PFP_UCODE_ADDR, 0);
3275 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3276 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3277 WREG32(CP_PFP_UCODE_ADDR, 0);
3280 fw_data = (const __be32 *)rdev->ce_fw->data;
3281 WREG32(CP_CE_UCODE_ADDR, 0);
3282 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3283 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3284 WREG32(CP_CE_UCODE_ADDR, 0);
3287 fw_data = (const __be32 *)rdev->me_fw->data;
3288 WREG32(CP_ME_RAM_WADDR, 0);
3289 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3290 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3291 WREG32(CP_ME_RAM_WADDR, 0);
3293 WREG32(CP_PFP_UCODE_ADDR, 0);
3294 WREG32(CP_CE_UCODE_ADDR, 0);
3295 WREG32(CP_ME_RAM_WADDR, 0);
3296 WREG32(CP_ME_RAM_RADDR, 0);
3300 static int si_cp_start(struct radeon_device *rdev)
3302 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3305 r = radeon_ring_lock(rdev, ring, 7 + 4);
3307 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3311 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3312 radeon_ring_write(ring, 0x1);
3313 radeon_ring_write(ring, 0x0);
3314 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3315 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3316 radeon_ring_write(ring, 0);
3317 radeon_ring_write(ring, 0);
3319 /* init the CE partitions */
3320 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3321 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3322 radeon_ring_write(ring, 0xc000);
3323 radeon_ring_write(ring, 0xe000);
3324 radeon_ring_unlock_commit(rdev, ring);
3326 si_cp_enable(rdev, true);
3328 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3330 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3334 /* setup clear context state */
3335 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3336 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3338 for (i = 0; i < si_default_size; i++)
3339 radeon_ring_write(ring, si_default_state[i]);
3341 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3342 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3344 /* set clear context state */
3345 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3346 radeon_ring_write(ring, 0);
3348 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3349 radeon_ring_write(ring, 0x00000316);
3350 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3351 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3353 radeon_ring_unlock_commit(rdev, ring);
3355 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3356 ring = &rdev->ring[i];
3357 r = radeon_ring_lock(rdev, ring, 2);
3359 /* clear the compute context state */
3360 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3361 radeon_ring_write(ring, 0);
3363 radeon_ring_unlock_commit(rdev, ring);
3369 static void si_cp_fini(struct radeon_device *rdev)
3371 struct radeon_ring *ring;
3372 si_cp_enable(rdev, false);
3374 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3375 radeon_ring_fini(rdev, ring);
3376 radeon_scratch_free(rdev, ring->rptr_save_reg);
3378 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3379 radeon_ring_fini(rdev, ring);
3380 radeon_scratch_free(rdev, ring->rptr_save_reg);
3382 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3383 radeon_ring_fini(rdev, ring);
3384 radeon_scratch_free(rdev, ring->rptr_save_reg);
3387 static int si_cp_resume(struct radeon_device *rdev)
3389 struct radeon_ring *ring;
3394 si_enable_gui_idle_interrupt(rdev, false);
3396 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3397 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3399 /* Set the write pointer delay */
3400 WREG32(CP_RB_WPTR_DELAY, 0);
3402 WREG32(CP_DEBUG, 0);
3403 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3405 /* ring 0 - compute and gfx */
3406 /* Set ring buffer size */
3407 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3408 rb_bufsz = order_base_2(ring->ring_size / 8);
3409 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3411 tmp |= BUF_SWAP_32BIT;
3413 WREG32(CP_RB0_CNTL, tmp);
3415 /* Initialize the ring buffer's read and write pointers */
3416 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3418 WREG32(CP_RB0_WPTR, ring->wptr);
3420 /* set the wb address whether it's enabled or not */
3421 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3422 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3424 if (rdev->wb.enabled)
3425 WREG32(SCRATCH_UMSK, 0xff);
3427 tmp |= RB_NO_UPDATE;
3428 WREG32(SCRATCH_UMSK, 0);
3432 WREG32(CP_RB0_CNTL, tmp);
3434 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3436 ring->rptr = RREG32(CP_RB0_RPTR);
3438 /* ring1 - compute only */
3439 /* Set ring buffer size */
3440 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3441 rb_bufsz = order_base_2(ring->ring_size / 8);
3442 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3444 tmp |= BUF_SWAP_32BIT;
3446 WREG32(CP_RB1_CNTL, tmp);
3448 /* Initialize the ring buffer's read and write pointers */
3449 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3451 WREG32(CP_RB1_WPTR, ring->wptr);
3453 /* set the wb address whether it's enabled or not */
3454 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3455 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3458 WREG32(CP_RB1_CNTL, tmp);
3460 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3462 ring->rptr = RREG32(CP_RB1_RPTR);
3464 /* ring2 - compute only */
3465 /* Set ring buffer size */
3466 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3467 rb_bufsz = order_base_2(ring->ring_size / 8);
3468 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3470 tmp |= BUF_SWAP_32BIT;
3472 WREG32(CP_RB2_CNTL, tmp);
3474 /* Initialize the ring buffer's read and write pointers */
3475 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3477 WREG32(CP_RB2_WPTR, ring->wptr);
3479 /* set the wb address whether it's enabled or not */
3480 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3481 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3484 WREG32(CP_RB2_CNTL, tmp);
3486 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3488 ring->rptr = RREG32(CP_RB2_RPTR);
3490 /* start the rings */
3492 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3493 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3494 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3495 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3497 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3498 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3499 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3502 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3504 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3506 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3508 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3511 si_enable_gui_idle_interrupt(rdev, true);
3516 u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3522 tmp = RREG32(GRBM_STATUS);
3523 if (tmp & (PA_BUSY | SC_BUSY |
3524 BCI_BUSY | SX_BUSY |
3525 TA_BUSY | VGT_BUSY |
3527 GDS_BUSY | SPI_BUSY |
3528 IA_BUSY | IA_BUSY_NO_DMA))
3529 reset_mask |= RADEON_RESET_GFX;
3531 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3532 CP_BUSY | CP_COHERENCY_BUSY))
3533 reset_mask |= RADEON_RESET_CP;
3535 if (tmp & GRBM_EE_BUSY)
3536 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3539 tmp = RREG32(GRBM_STATUS2);
3540 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3541 reset_mask |= RADEON_RESET_RLC;
3543 /* DMA_STATUS_REG 0 */
3544 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3545 if (!(tmp & DMA_IDLE))
3546 reset_mask |= RADEON_RESET_DMA;
3548 /* DMA_STATUS_REG 1 */
3549 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3550 if (!(tmp & DMA_IDLE))
3551 reset_mask |= RADEON_RESET_DMA1;
3554 tmp = RREG32(SRBM_STATUS2);
3556 reset_mask |= RADEON_RESET_DMA;
3558 if (tmp & DMA1_BUSY)
3559 reset_mask |= RADEON_RESET_DMA1;
3562 tmp = RREG32(SRBM_STATUS);
3565 reset_mask |= RADEON_RESET_IH;
3568 reset_mask |= RADEON_RESET_SEM;
3570 if (tmp & GRBM_RQ_PENDING)
3571 reset_mask |= RADEON_RESET_GRBM;
3574 reset_mask |= RADEON_RESET_VMC;
3576 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3577 MCC_BUSY | MCD_BUSY))
3578 reset_mask |= RADEON_RESET_MC;
3580 if (evergreen_is_display_hung(rdev))
3581 reset_mask |= RADEON_RESET_DISPLAY;
3584 tmp = RREG32(VM_L2_STATUS);
3586 reset_mask |= RADEON_RESET_VMC;
3588 /* Skip MC reset as it's mostly likely not hung, just busy */
3589 if (reset_mask & RADEON_RESET_MC) {
3590 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3591 reset_mask &= ~RADEON_RESET_MC;
3597 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3599 struct evergreen_mc_save save;
3600 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3603 if (reset_mask == 0)
3606 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3608 evergreen_print_gpu_status_regs(rdev);
3609 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3610 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3611 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3612 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3621 /* Disable CP parsing/prefetching */
3622 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3624 if (reset_mask & RADEON_RESET_DMA) {
3626 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3627 tmp &= ~DMA_RB_ENABLE;
3628 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3630 if (reset_mask & RADEON_RESET_DMA1) {
3632 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3633 tmp &= ~DMA_RB_ENABLE;
3634 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3639 evergreen_mc_stop(rdev, &save);
3640 if (evergreen_mc_wait_for_idle(rdev)) {
3641 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3644 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3645 grbm_soft_reset = SOFT_RESET_CB |
3659 if (reset_mask & RADEON_RESET_CP) {
3660 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3662 srbm_soft_reset |= SOFT_RESET_GRBM;
3665 if (reset_mask & RADEON_RESET_DMA)
3666 srbm_soft_reset |= SOFT_RESET_DMA;
3668 if (reset_mask & RADEON_RESET_DMA1)
3669 srbm_soft_reset |= SOFT_RESET_DMA1;
3671 if (reset_mask & RADEON_RESET_DISPLAY)
3672 srbm_soft_reset |= SOFT_RESET_DC;
3674 if (reset_mask & RADEON_RESET_RLC)
3675 grbm_soft_reset |= SOFT_RESET_RLC;
3677 if (reset_mask & RADEON_RESET_SEM)
3678 srbm_soft_reset |= SOFT_RESET_SEM;
3680 if (reset_mask & RADEON_RESET_IH)
3681 srbm_soft_reset |= SOFT_RESET_IH;
3683 if (reset_mask & RADEON_RESET_GRBM)
3684 srbm_soft_reset |= SOFT_RESET_GRBM;
3686 if (reset_mask & RADEON_RESET_VMC)
3687 srbm_soft_reset |= SOFT_RESET_VMC;
3689 if (reset_mask & RADEON_RESET_MC)
3690 srbm_soft_reset |= SOFT_RESET_MC;
3692 if (grbm_soft_reset) {
3693 tmp = RREG32(GRBM_SOFT_RESET);
3694 tmp |= grbm_soft_reset;
3695 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3696 WREG32(GRBM_SOFT_RESET, tmp);
3697 tmp = RREG32(GRBM_SOFT_RESET);
3701 tmp &= ~grbm_soft_reset;
3702 WREG32(GRBM_SOFT_RESET, tmp);
3703 tmp = RREG32(GRBM_SOFT_RESET);
3706 if (srbm_soft_reset) {
3707 tmp = RREG32(SRBM_SOFT_RESET);
3708 tmp |= srbm_soft_reset;
3709 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3710 WREG32(SRBM_SOFT_RESET, tmp);
3711 tmp = RREG32(SRBM_SOFT_RESET);
3715 tmp &= ~srbm_soft_reset;
3716 WREG32(SRBM_SOFT_RESET, tmp);
3717 tmp = RREG32(SRBM_SOFT_RESET);
3720 /* Wait a little for things to settle down */
3723 evergreen_mc_resume(rdev, &save);
3726 evergreen_print_gpu_status_regs(rdev);
3729 static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3733 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3734 tmp |= SPLL_BYPASS_EN;
3735 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3737 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3738 tmp |= SPLL_CTLREQ_CHG;
3739 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3741 for (i = 0; i < rdev->usec_timeout; i++) {
3742 if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
3747 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3748 tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
3749 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3751 tmp = RREG32(MPLL_CNTL_MODE);
3752 tmp &= ~MPLL_MCLK_SEL;
3753 WREG32(MPLL_CNTL_MODE, tmp);
3756 static void si_spll_powerdown(struct radeon_device *rdev)
3760 tmp = RREG32(SPLL_CNTL_MODE);
3761 tmp |= SPLL_SW_DIR_CONTROL;
3762 WREG32(SPLL_CNTL_MODE, tmp);
3764 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3766 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3768 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3770 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3772 tmp = RREG32(SPLL_CNTL_MODE);
3773 tmp &= ~SPLL_SW_DIR_CONTROL;
3774 WREG32(SPLL_CNTL_MODE, tmp);
3777 static void si_gpu_pci_config_reset(struct radeon_device *rdev)
3779 struct evergreen_mc_save save;
3782 dev_info(rdev->dev, "GPU pci config reset\n");
3790 /* Disable CP parsing/prefetching */
3791 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3793 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3794 tmp &= ~DMA_RB_ENABLE;
3795 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3797 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3798 tmp &= ~DMA_RB_ENABLE;
3799 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3800 /* XXX other engines? */
3802 /* halt the rlc, disable cp internal ints */
3807 /* disable mem access */
3808 evergreen_mc_stop(rdev, &save);
3809 if (evergreen_mc_wait_for_idle(rdev)) {
3810 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3813 /* set mclk/sclk to bypass */
3814 si_set_clk_bypass_mode(rdev);
3815 /* powerdown spll */
3816 si_spll_powerdown(rdev);
3818 pci_clear_master(rdev->pdev);
3820 radeon_pci_config_reset(rdev);
3821 /* wait for asic to come out of reset */
3822 for (i = 0; i < rdev->usec_timeout; i++) {
3823 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
3829 int si_asic_reset(struct radeon_device *rdev)
3833 reset_mask = si_gpu_check_soft_reset(rdev);
3836 r600_set_bios_scratch_engine_hung(rdev, true);
3838 /* try soft reset */
3839 si_gpu_soft_reset(rdev, reset_mask);
3841 reset_mask = si_gpu_check_soft_reset(rdev);
3843 /* try pci config reset */
3844 if (reset_mask && radeon_hard_reset)
3845 si_gpu_pci_config_reset(rdev);
3847 reset_mask = si_gpu_check_soft_reset(rdev);
3850 r600_set_bios_scratch_engine_hung(rdev, false);
3856 * si_gfx_is_lockup - Check if the GFX engine is locked up
3858 * @rdev: radeon_device pointer
3859 * @ring: radeon_ring structure holding ring information
3861 * Check if the GFX engine is locked up.
3862 * Returns true if the engine appears to be locked up, false if not.
3864 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3866 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3868 if (!(reset_mask & (RADEON_RESET_GFX |
3869 RADEON_RESET_COMPUTE |
3870 RADEON_RESET_CP))) {
3871 radeon_ring_lockup_update(ring);
3874 /* force CP activities */
3875 radeon_ring_force_activity(rdev, ring);
3876 return radeon_ring_test_lockup(rdev, ring);
3880 static void si_mc_program(struct radeon_device *rdev)
3882 struct evergreen_mc_save save;
3886 /* Initialize HDP */
3887 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3888 WREG32((0x2c14 + j), 0x00000000);
3889 WREG32((0x2c18 + j), 0x00000000);
3890 WREG32((0x2c1c + j), 0x00000000);
3891 WREG32((0x2c20 + j), 0x00000000);
3892 WREG32((0x2c24 + j), 0x00000000);
3894 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3896 evergreen_mc_stop(rdev, &save);
3897 if (radeon_mc_wait_for_idle(rdev)) {
3898 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3900 if (!ASIC_IS_NODCE(rdev))
3901 /* Lockout access through VGA aperture*/
3902 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
3903 /* Update configuration */
3904 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3905 rdev->mc.vram_start >> 12);
3906 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3907 rdev->mc.vram_end >> 12);
3908 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3909 rdev->vram_scratch.gpu_addr >> 12);
3910 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3911 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3912 WREG32(MC_VM_FB_LOCATION, tmp);
3913 /* XXX double check these! */
3914 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3915 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3916 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3917 WREG32(MC_VM_AGP_BASE, 0);
3918 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3919 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3920 if (radeon_mc_wait_for_idle(rdev)) {
3921 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3923 evergreen_mc_resume(rdev, &save);
3924 if (!ASIC_IS_NODCE(rdev)) {
3925 /* we need to own VRAM, so turn off the VGA renderer here
3926 * to stop it overwriting our objects */
3927 rv515_vga_render_disable(rdev);
3931 void si_vram_gtt_location(struct radeon_device *rdev,
3932 struct radeon_mc *mc)
3934 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3935 /* leave room for at least 1024M GTT */
3936 dev_warn(rdev->dev, "limiting VRAM\n");
3937 mc->real_vram_size = 0xFFC0000000ULL;
3938 mc->mc_vram_size = 0xFFC0000000ULL;
3940 radeon_vram_location(rdev, &rdev->mc, 0);
3941 rdev->mc.gtt_base_align = 0;
3942 radeon_gtt_location(rdev, mc);
3945 static int si_mc_init(struct radeon_device *rdev)
3948 int chansize, numchan;
3950 /* Get VRAM informations */
3951 rdev->mc.vram_is_ddr = true;
3952 tmp = RREG32(MC_ARB_RAMCFG);
3953 if (tmp & CHANSIZE_OVERRIDE) {
3955 } else if (tmp & CHANSIZE_MASK) {
3960 tmp = RREG32(MC_SHARED_CHMAP);
3961 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3991 rdev->mc.vram_width = numchan * chansize;
3992 /* Could aper size report 0 ? */
3993 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3994 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3995 /* size in MB on si */
3996 tmp = RREG32(CONFIG_MEMSIZE);
3997 /* some boards may have garbage in the upper 16 bits */
3998 if (tmp & 0xffff0000) {
3999 DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
4003 rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4004 rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
4005 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4006 si_vram_gtt_location(rdev, &rdev->mc);
4007 radeon_update_bandwidth_info(rdev);
4015 void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4017 /* flush hdp cache */
4018 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4020 /* bits 0-15 are the VM contexts0-15 */
4021 WREG32(VM_INVALIDATE_REQUEST, 1);
4024 static int si_pcie_gart_enable(struct radeon_device *rdev)
4028 if (rdev->gart.robj == NULL) {
4029 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4032 r = radeon_gart_table_vram_pin(rdev);
4035 radeon_gart_restore(rdev);
4036 /* Setup TLB control */
4037 WREG32(MC_VM_MX_L1_TLB_CNTL,
4040 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4041 ENABLE_ADVANCED_DRIVER_MODEL |
4042 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4043 /* Setup L2 cache */
4044 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4045 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4046 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4047 EFFECTIVE_L2_QUEUE_SIZE(7) |
4048 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4049 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4050 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4051 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4052 /* setup context0 */
4053 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4054 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4055 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4056 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4057 (u32)(rdev->dummy_page.addr >> 12));
4058 WREG32(VM_CONTEXT0_CNTL2, 0);
4059 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4060 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4066 /* empty context1-15 */
4067 /* set vm size, must be a multiple of 4 */
4068 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4069 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
4070 /* Assign the pt base to something valid for now; the pts used for
4071 * the VMs are determined by the application and setup and assigned
4072 * on the fly in the vm part of radeon_gart.c
4074 for (i = 1; i < 16; i++) {
4076 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4077 rdev->gart.table_addr >> 12);
4079 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4080 rdev->gart.table_addr >> 12);
4083 /* enable context1-15 */
4084 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4085 (u32)(rdev->dummy_page.addr >> 12));
4086 WREG32(VM_CONTEXT1_CNTL2, 4);
4087 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4088 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4089 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4090 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4091 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4092 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4093 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4094 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4095 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4096 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4097 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4098 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4099 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
4101 si_pcie_gart_tlb_flush(rdev);
4102 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4103 (unsigned)(rdev->mc.gtt_size >> 20),
4104 (unsigned long long)rdev->gart.table_addr);
4105 rdev->gart.ready = true;
4109 static void si_pcie_gart_disable(struct radeon_device *rdev)
4111 /* Disable all tables */
4112 WREG32(VM_CONTEXT0_CNTL, 0);
4113 WREG32(VM_CONTEXT1_CNTL, 0);
4114 /* Setup TLB control */
4115 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4116 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4117 /* Setup L2 cache */
4118 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4119 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4120 EFFECTIVE_L2_QUEUE_SIZE(7) |
4121 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4122 WREG32(VM_L2_CNTL2, 0);
4123 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4124 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4125 radeon_gart_table_vram_unpin(rdev);
4128 static void si_pcie_gart_fini(struct radeon_device *rdev)
4130 si_pcie_gart_disable(rdev);
4131 radeon_gart_table_vram_free(rdev);
4132 radeon_gart_fini(rdev);
4136 static bool si_vm_reg_valid(u32 reg)
4138 /* context regs are fine */
4142 /* check config regs */
4144 case GRBM_GFX_INDEX:
4145 case CP_STRMOUT_CNTL:
4146 case VGT_VTX_VECT_EJECT_REG:
4147 case VGT_CACHE_INVALIDATION:
4148 case VGT_ESGS_RING_SIZE:
4149 case VGT_GSVS_RING_SIZE:
4150 case VGT_GS_VERTEX_REUSE:
4151 case VGT_PRIMITIVE_TYPE:
4152 case VGT_INDEX_TYPE:
4153 case VGT_NUM_INDICES:
4154 case VGT_NUM_INSTANCES:
4155 case VGT_TF_RING_SIZE:
4156 case VGT_HS_OFFCHIP_PARAM:
4157 case VGT_TF_MEMORY_BASE:
4159 case PA_SU_LINE_STIPPLE_VALUE:
4160 case PA_SC_LINE_STIPPLE_STATE:
4163 case SPI_STATIC_THREAD_MGMT_1:
4164 case SPI_STATIC_THREAD_MGMT_2:
4165 case SPI_STATIC_THREAD_MGMT_3:
4166 case SPI_PS_MAX_WAVE_ID:
4167 case SPI_CONFIG_CNTL:
4168 case SPI_CONFIG_CNTL_1:
4172 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4177 static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4178 u32 *ib, struct radeon_cs_packet *pkt)
4180 switch (pkt->opcode) {
4182 case PACKET3_SET_BASE:
4183 case PACKET3_SET_CE_DE_COUNTERS:
4184 case PACKET3_LOAD_CONST_RAM:
4185 case PACKET3_WRITE_CONST_RAM:
4186 case PACKET3_WRITE_CONST_RAM_OFFSET:
4187 case PACKET3_DUMP_CONST_RAM:
4188 case PACKET3_INCREMENT_CE_COUNTER:
4189 case PACKET3_WAIT_ON_DE_COUNTER:
4190 case PACKET3_CE_WRITE:
4193 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4199 static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4201 u32 start_reg, reg, i;
4202 u32 command = ib[idx + 4];
4203 u32 info = ib[idx + 1];
4204 u32 idx_value = ib[idx];
4205 if (command & PACKET3_CP_DMA_CMD_SAS) {
4206 /* src address space is register */
4207 if (((info & 0x60000000) >> 29) == 0) {
4208 start_reg = idx_value << 2;
4209 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4211 if (!si_vm_reg_valid(reg)) {
4212 DRM_ERROR("CP DMA Bad SRC register\n");
4216 for (i = 0; i < (command & 0x1fffff); i++) {
4217 reg = start_reg + (4 * i);
4218 if (!si_vm_reg_valid(reg)) {
4219 DRM_ERROR("CP DMA Bad SRC register\n");
4226 if (command & PACKET3_CP_DMA_CMD_DAS) {
4227 /* dst address space is register */
4228 if (((info & 0x00300000) >> 20) == 0) {
4229 start_reg = ib[idx + 2];
4230 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4232 if (!si_vm_reg_valid(reg)) {
4233 DRM_ERROR("CP DMA Bad DST register\n");
4237 for (i = 0; i < (command & 0x1fffff); i++) {
4238 reg = start_reg + (4 * i);
4239 if (!si_vm_reg_valid(reg)) {
4240 DRM_ERROR("CP DMA Bad DST register\n");
4250 static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4251 u32 *ib, struct radeon_cs_packet *pkt)
4254 u32 idx = pkt->idx + 1;
4255 u32 idx_value = ib[idx];
4256 u32 start_reg, end_reg, reg, i;
4258 switch (pkt->opcode) {
4260 case PACKET3_SET_BASE:
4261 case PACKET3_CLEAR_STATE:
4262 case PACKET3_INDEX_BUFFER_SIZE:
4263 case PACKET3_DISPATCH_DIRECT:
4264 case PACKET3_DISPATCH_INDIRECT:
4265 case PACKET3_ALLOC_GDS:
4266 case PACKET3_WRITE_GDS_RAM:
4267 case PACKET3_ATOMIC_GDS:
4268 case PACKET3_ATOMIC:
4269 case PACKET3_OCCLUSION_QUERY:
4270 case PACKET3_SET_PREDICATION:
4271 case PACKET3_COND_EXEC:
4272 case PACKET3_PRED_EXEC:
4273 case PACKET3_DRAW_INDIRECT:
4274 case PACKET3_DRAW_INDEX_INDIRECT:
4275 case PACKET3_INDEX_BASE:
4276 case PACKET3_DRAW_INDEX_2:
4277 case PACKET3_CONTEXT_CONTROL:
4278 case PACKET3_INDEX_TYPE:
4279 case PACKET3_DRAW_INDIRECT_MULTI:
4280 case PACKET3_DRAW_INDEX_AUTO:
4281 case PACKET3_DRAW_INDEX_IMMD:
4282 case PACKET3_NUM_INSTANCES:
4283 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4284 case PACKET3_STRMOUT_BUFFER_UPDATE:
4285 case PACKET3_DRAW_INDEX_OFFSET_2:
4286 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4287 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4288 case PACKET3_MPEG_INDEX:
4289 case PACKET3_WAIT_REG_MEM:
4290 case PACKET3_MEM_WRITE:
4291 case PACKET3_PFP_SYNC_ME:
4292 case PACKET3_SURFACE_SYNC:
4293 case PACKET3_EVENT_WRITE:
4294 case PACKET3_EVENT_WRITE_EOP:
4295 case PACKET3_EVENT_WRITE_EOS:
4296 case PACKET3_SET_CONTEXT_REG:
4297 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4298 case PACKET3_SET_SH_REG:
4299 case PACKET3_SET_SH_REG_OFFSET:
4300 case PACKET3_INCREMENT_DE_COUNTER:
4301 case PACKET3_WAIT_ON_CE_COUNTER:
4302 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4303 case PACKET3_ME_WRITE:
4305 case PACKET3_COPY_DATA:
4306 if ((idx_value & 0xf00) == 0) {
4307 reg = ib[idx + 3] * 4;
4308 if (!si_vm_reg_valid(reg))
4312 case PACKET3_WRITE_DATA:
4313 if ((idx_value & 0xf00) == 0) {
4314 start_reg = ib[idx + 1] * 4;
4315 if (idx_value & 0x10000) {
4316 if (!si_vm_reg_valid(start_reg))
4319 for (i = 0; i < (pkt->count - 2); i++) {
4320 reg = start_reg + (4 * i);
4321 if (!si_vm_reg_valid(reg))
4327 case PACKET3_COND_WRITE:
4328 if (idx_value & 0x100) {
4329 reg = ib[idx + 5] * 4;
4330 if (!si_vm_reg_valid(reg))
4334 case PACKET3_COPY_DW:
4335 if (idx_value & 0x2) {
4336 reg = ib[idx + 3] * 4;
4337 if (!si_vm_reg_valid(reg))
4341 case PACKET3_SET_CONFIG_REG:
4342 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4343 end_reg = 4 * pkt->count + start_reg - 4;
4344 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4345 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4346 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4347 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4350 for (i = 0; i < pkt->count; i++) {
4351 reg = start_reg + (4 * i);
4352 if (!si_vm_reg_valid(reg))
4356 case PACKET3_CP_DMA:
4357 r = si_vm_packet3_cp_dma_check(ib, idx);
4362 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4368 static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4369 u32 *ib, struct radeon_cs_packet *pkt)
4372 u32 idx = pkt->idx + 1;
4373 u32 idx_value = ib[idx];
4374 u32 start_reg, reg, i;
4376 switch (pkt->opcode) {
4378 case PACKET3_SET_BASE:
4379 case PACKET3_CLEAR_STATE:
4380 case PACKET3_DISPATCH_DIRECT:
4381 case PACKET3_DISPATCH_INDIRECT:
4382 case PACKET3_ALLOC_GDS:
4383 case PACKET3_WRITE_GDS_RAM:
4384 case PACKET3_ATOMIC_GDS:
4385 case PACKET3_ATOMIC:
4386 case PACKET3_OCCLUSION_QUERY:
4387 case PACKET3_SET_PREDICATION:
4388 case PACKET3_COND_EXEC:
4389 case PACKET3_PRED_EXEC:
4390 case PACKET3_CONTEXT_CONTROL:
4391 case PACKET3_STRMOUT_BUFFER_UPDATE:
4392 case PACKET3_WAIT_REG_MEM:
4393 case PACKET3_MEM_WRITE:
4394 case PACKET3_PFP_SYNC_ME:
4395 case PACKET3_SURFACE_SYNC:
4396 case PACKET3_EVENT_WRITE:
4397 case PACKET3_EVENT_WRITE_EOP:
4398 case PACKET3_EVENT_WRITE_EOS:
4399 case PACKET3_SET_CONTEXT_REG:
4400 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4401 case PACKET3_SET_SH_REG:
4402 case PACKET3_SET_SH_REG_OFFSET:
4403 case PACKET3_INCREMENT_DE_COUNTER:
4404 case PACKET3_WAIT_ON_CE_COUNTER:
4405 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4406 case PACKET3_ME_WRITE:
4408 case PACKET3_COPY_DATA:
4409 if ((idx_value & 0xf00) == 0) {
4410 reg = ib[idx + 3] * 4;
4411 if (!si_vm_reg_valid(reg))
4415 case PACKET3_WRITE_DATA:
4416 if ((idx_value & 0xf00) == 0) {
4417 start_reg = ib[idx + 1] * 4;
4418 if (idx_value & 0x10000) {
4419 if (!si_vm_reg_valid(start_reg))
4422 for (i = 0; i < (pkt->count - 2); i++) {
4423 reg = start_reg + (4 * i);
4424 if (!si_vm_reg_valid(reg))
4430 case PACKET3_COND_WRITE:
4431 if (idx_value & 0x100) {
4432 reg = ib[idx + 5] * 4;
4433 if (!si_vm_reg_valid(reg))
4437 case PACKET3_COPY_DW:
4438 if (idx_value & 0x2) {
4439 reg = ib[idx + 3] * 4;
4440 if (!si_vm_reg_valid(reg))
4444 case PACKET3_CP_DMA:
4445 r = si_vm_packet3_cp_dma_check(ib, idx);
4450 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4456 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4460 struct radeon_cs_packet pkt;
4464 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4465 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4468 case RADEON_PACKET_TYPE0:
4469 dev_err(rdev->dev, "Packet0 not allowed!\n");
4472 case RADEON_PACKET_TYPE2:
4475 case RADEON_PACKET_TYPE3:
4476 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4477 if (ib->is_const_ib)
4478 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4481 case RADEON_RING_TYPE_GFX_INDEX:
4482 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4484 case CAYMAN_RING_TYPE_CP1_INDEX:
4485 case CAYMAN_RING_TYPE_CP2_INDEX:
4486 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4489 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4494 idx += pkt.count + 2;
4497 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4503 } while (idx < ib->length_dw);
4511 int si_vm_init(struct radeon_device *rdev)
4514 rdev->vm_manager.nvm = 16;
4515 /* base offset of vram pages */
4516 rdev->vm_manager.vram_base_offset = 0;
4521 void si_vm_fini(struct radeon_device *rdev)
4526 * si_vm_decode_fault - print human readable fault info
4528 * @rdev: radeon_device pointer
4529 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4530 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4532 * Print human readable fault information (SI).
4534 static void si_vm_decode_fault(struct radeon_device *rdev,
4535 u32 status, u32 addr)
4537 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4538 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4539 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4542 if (rdev->family == CHIP_TAHITI) {
4783 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4784 protections, vmid, addr,
4785 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4789 void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4791 struct radeon_ring *ring = &rdev->ring[ridx];
4796 /* write new base address */
4797 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4798 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4799 WRITE_DATA_DST_SEL(0)));
4802 radeon_ring_write(ring,
4803 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4805 radeon_ring_write(ring,
4806 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4808 radeon_ring_write(ring, 0);
4809 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4811 /* flush hdp cache */
4812 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4813 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4814 WRITE_DATA_DST_SEL(0)));
4815 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4816 radeon_ring_write(ring, 0);
4817 radeon_ring_write(ring, 0x1);
4819 /* bits 0-15 are the VM contexts0-15 */
4820 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4821 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4822 WRITE_DATA_DST_SEL(0)));
4823 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4824 radeon_ring_write(ring, 0);
4825 radeon_ring_write(ring, 1 << vm->id);
4827 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4828 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4829 radeon_ring_write(ring, 0x0);
4833 * Power and clock gating
4835 static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
4839 for (i = 0; i < rdev->usec_timeout; i++) {
4840 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
4845 for (i = 0; i < rdev->usec_timeout; i++) {
4846 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
4852 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
4855 u32 tmp = RREG32(CP_INT_CNTL_RING0);
4860 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4862 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4863 WREG32(CP_INT_CNTL_RING0, tmp);
4866 /* read a gfx register */
4867 tmp = RREG32(DB_DEPTH_INFO);
4869 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
4870 for (i = 0; i < rdev->usec_timeout; i++) {
4871 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
4878 static void si_set_uvd_dcm(struct radeon_device *rdev,
4883 tmp = RREG32(UVD_CGC_CTRL);
4884 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
4885 tmp |= DCM | CG_DT(1) | CLK_OD(4);
4889 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
4895 WREG32(UVD_CGC_CTRL, tmp);
4896 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
4899 void si_init_uvd_internal_cg(struct radeon_device *rdev)
4901 bool hw_mode = true;
4904 si_set_uvd_dcm(rdev, false);
4906 u32 tmp = RREG32(UVD_CGC_CTRL);
4908 WREG32(UVD_CGC_CTRL, tmp);
4912 static u32 si_halt_rlc(struct radeon_device *rdev)
4916 orig = data = RREG32(RLC_CNTL);
4918 if (data & RLC_ENABLE) {
4919 data &= ~RLC_ENABLE;
4920 WREG32(RLC_CNTL, data);
4922 si_wait_for_rlc_serdes(rdev);
4928 static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
4932 tmp = RREG32(RLC_CNTL);
4934 WREG32(RLC_CNTL, rlc);
4937 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
4941 orig = data = RREG32(DMA_PG);
4942 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
4943 data |= PG_CNTL_ENABLE;
4945 data &= ~PG_CNTL_ENABLE;
4947 WREG32(DMA_PG, data);
4950 static void si_init_dma_pg(struct radeon_device *rdev)
4954 WREG32(DMA_PGFSM_WRITE, 0x00002000);
4955 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
4957 for (tmp = 0; tmp < 5; tmp++)
4958 WREG32(DMA_PGFSM_WRITE, 0);
4961 static void si_enable_gfx_cgpg(struct radeon_device *rdev,
4966 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
4967 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
4968 WREG32(RLC_TTOP_D, tmp);
4970 tmp = RREG32(RLC_PG_CNTL);
4971 tmp |= GFX_PG_ENABLE;
4972 WREG32(RLC_PG_CNTL, tmp);
4974 tmp = RREG32(RLC_AUTO_PG_CTRL);
4976 WREG32(RLC_AUTO_PG_CTRL, tmp);
4978 tmp = RREG32(RLC_AUTO_PG_CTRL);
4980 WREG32(RLC_AUTO_PG_CTRL, tmp);
4982 tmp = RREG32(DB_RENDER_CONTROL);
4986 static void si_init_gfx_cgpg(struct radeon_device *rdev)
4990 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4992 tmp = RREG32(RLC_PG_CNTL);
4994 WREG32(RLC_PG_CNTL, tmp);
4996 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4998 tmp = RREG32(RLC_AUTO_PG_CTRL);
5000 tmp &= ~GRBM_REG_SGIT_MASK;
5001 tmp |= GRBM_REG_SGIT(0x700);
5002 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5003 WREG32(RLC_AUTO_PG_CTRL, tmp);
5006 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5008 u32 mask = 0, tmp, tmp1;
5011 si_select_se_sh(rdev, se, sh);
5012 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5013 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5014 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5021 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5026 return (~tmp) & mask;
5029 static void si_init_ao_cu_mask(struct radeon_device *rdev)
5031 u32 i, j, k, active_cu_number = 0;
5032 u32 mask, counter, cu_bitmap;
5035 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5036 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5040 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
5041 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
5049 active_cu_number += counter;
5050 tmp |= (cu_bitmap << (i * 16 + j * 8));
5054 WREG32(RLC_PG_AO_CU_MASK, tmp);
5056 tmp = RREG32(RLC_MAX_PG_CU);
5057 tmp &= ~MAX_PU_CU_MASK;
5058 tmp |= MAX_PU_CU(active_cu_number);
5059 WREG32(RLC_MAX_PG_CU, tmp);
5062 static void si_enable_cgcg(struct radeon_device *rdev,
5065 u32 data, orig, tmp;
5067 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5069 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5070 si_enable_gui_idle_interrupt(rdev, true);
5072 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5074 tmp = si_halt_rlc(rdev);
5076 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5077 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5078 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5080 si_wait_for_rlc_serdes(rdev);
5082 si_update_rlc(rdev, tmp);
5084 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5086 data |= CGCG_EN | CGLS_EN;
5088 si_enable_gui_idle_interrupt(rdev, false);
5090 RREG32(CB_CGTT_SCLK_CTRL);
5091 RREG32(CB_CGTT_SCLK_CTRL);
5092 RREG32(CB_CGTT_SCLK_CTRL);
5093 RREG32(CB_CGTT_SCLK_CTRL);
5095 data &= ~(CGCG_EN | CGLS_EN);
5099 WREG32(RLC_CGCG_CGLS_CTRL, data);
5102 static void si_enable_mgcg(struct radeon_device *rdev,
5105 u32 data, orig, tmp = 0;
5107 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5108 orig = data = RREG32(CGTS_SM_CTRL_REG);
5111 WREG32(CGTS_SM_CTRL_REG, data);
5113 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5114 orig = data = RREG32(CP_MEM_SLP_CNTL);
5115 data |= CP_MEM_LS_EN;
5117 WREG32(CP_MEM_SLP_CNTL, data);
5120 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5123 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5125 tmp = si_halt_rlc(rdev);
5127 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5128 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5129 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5131 si_update_rlc(rdev, tmp);
5133 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5136 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5138 data = RREG32(CP_MEM_SLP_CNTL);
5139 if (data & CP_MEM_LS_EN) {
5140 data &= ~CP_MEM_LS_EN;
5141 WREG32(CP_MEM_SLP_CNTL, data);
5143 orig = data = RREG32(CGTS_SM_CTRL_REG);
5144 data |= LS_OVERRIDE | OVERRIDE;
5146 WREG32(CGTS_SM_CTRL_REG, data);
5148 tmp = si_halt_rlc(rdev);
5150 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5151 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5152 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5154 si_update_rlc(rdev, tmp);
5158 static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5161 u32 orig, data, tmp;
5163 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5164 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5166 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5168 orig = data = RREG32(UVD_CGC_CTRL);
5171 WREG32(UVD_CGC_CTRL, data);
5173 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5174 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5176 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5178 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5180 orig = data = RREG32(UVD_CGC_CTRL);
5183 WREG32(UVD_CGC_CTRL, data);
5185 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5186 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5190 static const u32 mc_cg_registers[] =
5203 static void si_enable_mc_ls(struct radeon_device *rdev,
5209 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5210 orig = data = RREG32(mc_cg_registers[i]);
5211 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5212 data |= MC_LS_ENABLE;
5214 data &= ~MC_LS_ENABLE;
5216 WREG32(mc_cg_registers[i], data);
5220 static void si_enable_mc_mgcg(struct radeon_device *rdev,
5226 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5227 orig = data = RREG32(mc_cg_registers[i]);
5228 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5229 data |= MC_CG_ENABLE;
5231 data &= ~MC_CG_ENABLE;
5233 WREG32(mc_cg_registers[i], data);
5237 static void si_enable_dma_mgcg(struct radeon_device *rdev,
5240 u32 orig, data, offset;
5243 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5244 for (i = 0; i < 2; i++) {
5246 offset = DMA0_REGISTER_OFFSET;
5248 offset = DMA1_REGISTER_OFFSET;
5249 orig = data = RREG32(DMA_POWER_CNTL + offset);
5250 data &= ~MEM_POWER_OVERRIDE;
5252 WREG32(DMA_POWER_CNTL + offset, data);
5253 WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5256 for (i = 0; i < 2; i++) {
5258 offset = DMA0_REGISTER_OFFSET;
5260 offset = DMA1_REGISTER_OFFSET;
5261 orig = data = RREG32(DMA_POWER_CNTL + offset);
5262 data |= MEM_POWER_OVERRIDE;
5264 WREG32(DMA_POWER_CNTL + offset, data);
5266 orig = data = RREG32(DMA_CLK_CTRL + offset);
5269 WREG32(DMA_CLK_CTRL + offset, data);
5274 static void si_enable_bif_mgls(struct radeon_device *rdev,
5279 orig = data = RREG32_PCIE(PCIE_CNTL2);
5281 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5282 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5283 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5285 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5286 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5289 WREG32_PCIE(PCIE_CNTL2, data);
5292 static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5297 orig = data = RREG32(HDP_HOST_PATH_CNTL);
5299 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5300 data &= ~CLOCK_GATING_DIS;
5302 data |= CLOCK_GATING_DIS;
5305 WREG32(HDP_HOST_PATH_CNTL, data);
5308 static void si_enable_hdp_ls(struct radeon_device *rdev,
5313 orig = data = RREG32(HDP_MEM_POWER_LS);
5315 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5316 data |= HDP_LS_ENABLE;
5318 data &= ~HDP_LS_ENABLE;
5321 WREG32(HDP_MEM_POWER_LS, data);
5324 static void si_update_cg(struct radeon_device *rdev,
5325 u32 block, bool enable)
5327 if (block & RADEON_CG_BLOCK_GFX) {
5328 si_enable_gui_idle_interrupt(rdev, false);
5329 /* order matters! */
5331 si_enable_mgcg(rdev, true);
5332 si_enable_cgcg(rdev, true);
5334 si_enable_cgcg(rdev, false);
5335 si_enable_mgcg(rdev, false);
5337 si_enable_gui_idle_interrupt(rdev, true);
5340 if (block & RADEON_CG_BLOCK_MC) {
5341 si_enable_mc_mgcg(rdev, enable);
5342 si_enable_mc_ls(rdev, enable);
5345 if (block & RADEON_CG_BLOCK_SDMA) {
5346 si_enable_dma_mgcg(rdev, enable);
5349 if (block & RADEON_CG_BLOCK_BIF) {
5350 si_enable_bif_mgls(rdev, enable);
5353 if (block & RADEON_CG_BLOCK_UVD) {
5354 if (rdev->has_uvd) {
5355 si_enable_uvd_mgcg(rdev, enable);
5359 if (block & RADEON_CG_BLOCK_HDP) {
5360 si_enable_hdp_mgcg(rdev, enable);
5361 si_enable_hdp_ls(rdev, enable);
5365 static void si_init_cg(struct radeon_device *rdev)
5367 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5368 RADEON_CG_BLOCK_MC |
5369 RADEON_CG_BLOCK_SDMA |
5370 RADEON_CG_BLOCK_BIF |
5371 RADEON_CG_BLOCK_HDP), true);
5372 if (rdev->has_uvd) {
5373 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5374 si_init_uvd_internal_cg(rdev);
5378 static void si_fini_cg(struct radeon_device *rdev)
5380 if (rdev->has_uvd) {
5381 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5383 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5384 RADEON_CG_BLOCK_MC |
5385 RADEON_CG_BLOCK_SDMA |
5386 RADEON_CG_BLOCK_BIF |
5387 RADEON_CG_BLOCK_HDP), false);
5390 u32 si_get_csb_size(struct radeon_device *rdev)
5393 const struct cs_section_def *sect = NULL;
5394 const struct cs_extent_def *ext = NULL;
5396 if (rdev->rlc.cs_data == NULL)
5399 /* begin clear state */
5401 /* context control state */
5404 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5405 for (ext = sect->section; ext->extent != NULL; ++ext) {
5406 if (sect->id == SECT_CONTEXT)
5407 count += 2 + ext->reg_count;
5412 /* pa_sc_raster_config */
5414 /* end clear state */
5422 void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5425 const struct cs_section_def *sect = NULL;
5426 const struct cs_extent_def *ext = NULL;
5428 if (rdev->rlc.cs_data == NULL)
5433 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5434 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
5436 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5437 buffer[count++] = cpu_to_le32(0x80000000);
5438 buffer[count++] = cpu_to_le32(0x80000000);
5440 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5441 for (ext = sect->section; ext->extent != NULL; ++ext) {
5442 if (sect->id == SECT_CONTEXT) {
5444 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5445 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
5446 for (i = 0; i < ext->reg_count; i++)
5447 buffer[count++] = cpu_to_le32(ext->extent[i]);
5454 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5455 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
5456 switch (rdev->family) {
5459 buffer[count++] = cpu_to_le32(0x2a00126a);
5462 buffer[count++] = cpu_to_le32(0x0000124a);
5465 buffer[count++] = cpu_to_le32(0x00000082);
5468 buffer[count++] = cpu_to_le32(0x00000000);
5471 buffer[count++] = cpu_to_le32(0x00000000);
5475 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5476 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
5478 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5479 buffer[count++] = cpu_to_le32(0);
5482 static void si_init_pg(struct radeon_device *rdev)
5484 if (rdev->pg_flags) {
5485 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5486 si_init_dma_pg(rdev);
5488 si_init_ao_cu_mask(rdev);
5489 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5490 si_init_gfx_cgpg(rdev);
5492 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5493 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5495 si_enable_dma_pg(rdev, true);
5496 si_enable_gfx_cgpg(rdev, true);
5498 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5499 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5503 static void si_fini_pg(struct radeon_device *rdev)
5505 if (rdev->pg_flags) {
5506 si_enable_dma_pg(rdev, false);
5507 si_enable_gfx_cgpg(rdev, false);
5514 void si_rlc_reset(struct radeon_device *rdev)
5516 u32 tmp = RREG32(GRBM_SOFT_RESET);
5518 tmp |= SOFT_RESET_RLC;
5519 WREG32(GRBM_SOFT_RESET, tmp);
5521 tmp &= ~SOFT_RESET_RLC;
5522 WREG32(GRBM_SOFT_RESET, tmp);
5526 static void si_rlc_stop(struct radeon_device *rdev)
5528 WREG32(RLC_CNTL, 0);
5530 si_enable_gui_idle_interrupt(rdev, false);
5532 si_wait_for_rlc_serdes(rdev);
5535 static void si_rlc_start(struct radeon_device *rdev)
5537 WREG32(RLC_CNTL, RLC_ENABLE);
5539 si_enable_gui_idle_interrupt(rdev, true);
5544 static bool si_lbpw_supported(struct radeon_device *rdev)
5548 /* Enable LBPW only for DDR3 */
5549 tmp = RREG32(MC_SEQ_MISC0);
5550 if ((tmp & 0xF0000000) == 0xB0000000)
5555 static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5559 tmp = RREG32(RLC_LB_CNTL);
5561 tmp |= LOAD_BALANCE_ENABLE;
5563 tmp &= ~LOAD_BALANCE_ENABLE;
5564 WREG32(RLC_LB_CNTL, tmp);
5567 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5568 WREG32(SPI_LB_CU_MASK, 0x00ff);
5572 static int si_rlc_resume(struct radeon_device *rdev)
5575 const __be32 *fw_data;
5588 WREG32(RLC_RL_BASE, 0);
5589 WREG32(RLC_RL_SIZE, 0);
5590 WREG32(RLC_LB_CNTL, 0);
5591 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5592 WREG32(RLC_LB_CNTR_INIT, 0);
5593 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5595 WREG32(RLC_MC_CNTL, 0);
5596 WREG32(RLC_UCODE_CNTL, 0);
5598 fw_data = (const __be32 *)rdev->rlc_fw->data;
5599 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5600 WREG32(RLC_UCODE_ADDR, i);
5601 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5603 WREG32(RLC_UCODE_ADDR, 0);
5605 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5612 static void si_enable_interrupts(struct radeon_device *rdev)
5614 u32 ih_cntl = RREG32(IH_CNTL);
5615 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5617 ih_cntl |= ENABLE_INTR;
5618 ih_rb_cntl |= IH_RB_ENABLE;
5619 WREG32(IH_CNTL, ih_cntl);
5620 WREG32(IH_RB_CNTL, ih_rb_cntl);
5621 rdev->ih.enabled = true;
5624 static void si_disable_interrupts(struct radeon_device *rdev)
5626 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5627 u32 ih_cntl = RREG32(IH_CNTL);
5629 ih_rb_cntl &= ~IH_RB_ENABLE;
5630 ih_cntl &= ~ENABLE_INTR;
5631 WREG32(IH_RB_CNTL, ih_rb_cntl);
5632 WREG32(IH_CNTL, ih_cntl);
5633 /* set rptr, wptr to 0 */
5634 WREG32(IH_RB_RPTR, 0);
5635 WREG32(IH_RB_WPTR, 0);
5636 rdev->ih.enabled = false;
5640 static void si_disable_interrupt_state(struct radeon_device *rdev)
5644 tmp = RREG32(CP_INT_CNTL_RING0) &
5645 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5646 WREG32(CP_INT_CNTL_RING0, tmp);
5647 WREG32(CP_INT_CNTL_RING1, 0);
5648 WREG32(CP_INT_CNTL_RING2, 0);
5649 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5650 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5651 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5652 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5653 WREG32(GRBM_INT_CNTL, 0);
5654 if (rdev->num_crtc >= 2) {
5655 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5656 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5658 if (rdev->num_crtc >= 4) {
5659 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5660 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5662 if (rdev->num_crtc >= 6) {
5663 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5664 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5667 if (rdev->num_crtc >= 2) {
5668 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5669 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5671 if (rdev->num_crtc >= 4) {
5672 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5673 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5675 if (rdev->num_crtc >= 6) {
5676 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5677 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5680 if (!ASIC_IS_NODCE(rdev)) {
5681 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
5683 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5684 WREG32(DC_HPD1_INT_CONTROL, tmp);
5685 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5686 WREG32(DC_HPD2_INT_CONTROL, tmp);
5687 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5688 WREG32(DC_HPD3_INT_CONTROL, tmp);
5689 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5690 WREG32(DC_HPD4_INT_CONTROL, tmp);
5691 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5692 WREG32(DC_HPD5_INT_CONTROL, tmp);
5693 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5694 WREG32(DC_HPD6_INT_CONTROL, tmp);
5698 static int si_irq_init(struct radeon_device *rdev)
5702 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5705 ret = r600_ih_ring_alloc(rdev);
5710 si_disable_interrupts(rdev);
5713 ret = si_rlc_resume(rdev);
5715 r600_ih_ring_fini(rdev);
5719 /* setup interrupt control */
5720 /* set dummy read address to ring address */
5721 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5722 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5723 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5724 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5726 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5727 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5728 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5729 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5731 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5732 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
5734 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5735 IH_WPTR_OVERFLOW_CLEAR |
5738 if (rdev->wb.enabled)
5739 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5741 /* set the writeback address whether it's enabled or not */
5742 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5743 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5745 WREG32(IH_RB_CNTL, ih_rb_cntl);
5747 /* set rptr, wptr to 0 */
5748 WREG32(IH_RB_RPTR, 0);
5749 WREG32(IH_RB_WPTR, 0);
5751 /* Default settings for IH_CNTL (disabled at first) */
5752 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5753 /* RPTR_REARM only works if msi's are enabled */
5754 if (rdev->msi_enabled)
5755 ih_cntl |= RPTR_REARM;
5756 WREG32(IH_CNTL, ih_cntl);
5758 /* force the active interrupt state to all disabled */
5759 si_disable_interrupt_state(rdev);
5761 pci_set_master(rdev->pdev);
5764 si_enable_interrupts(rdev);
5769 int si_irq_set(struct radeon_device *rdev)
5772 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5773 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5774 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
5775 u32 grbm_int_cntl = 0;
5776 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
5777 u32 dma_cntl, dma_cntl1;
5778 u32 thermal_int = 0;
5780 if (!rdev->irq.installed) {
5781 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5784 /* don't enable anything if the ih is disabled */
5785 if (!rdev->ih.enabled) {
5786 si_disable_interrupts(rdev);
5787 /* force the active interrupt state to all disabled */
5788 si_disable_interrupt_state(rdev);
5792 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
5793 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5795 if (!ASIC_IS_NODCE(rdev)) {
5796 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5797 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5798 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5799 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5800 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5801 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5804 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5805 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5807 thermal_int = RREG32(CG_THERMAL_INT) &
5808 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
5810 /* enable CP interrupts on all rings */
5811 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
5812 DRM_DEBUG("si_irq_set: sw int gfx\n");
5813 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5815 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
5816 DRM_DEBUG("si_irq_set: sw int cp1\n");
5817 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
5819 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
5820 DRM_DEBUG("si_irq_set: sw int cp2\n");
5821 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
5823 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5824 DRM_DEBUG("si_irq_set: sw int dma\n");
5825 dma_cntl |= TRAP_ENABLE;
5828 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5829 DRM_DEBUG("si_irq_set: sw int dma1\n");
5830 dma_cntl1 |= TRAP_ENABLE;
5832 if (rdev->irq.crtc_vblank_int[0] ||
5833 atomic_read(&rdev->irq.pflip[0])) {
5834 DRM_DEBUG("si_irq_set: vblank 0\n");
5835 crtc1 |= VBLANK_INT_MASK;
5837 if (rdev->irq.crtc_vblank_int[1] ||
5838 atomic_read(&rdev->irq.pflip[1])) {
5839 DRM_DEBUG("si_irq_set: vblank 1\n");
5840 crtc2 |= VBLANK_INT_MASK;
5842 if (rdev->irq.crtc_vblank_int[2] ||
5843 atomic_read(&rdev->irq.pflip[2])) {
5844 DRM_DEBUG("si_irq_set: vblank 2\n");
5845 crtc3 |= VBLANK_INT_MASK;
5847 if (rdev->irq.crtc_vblank_int[3] ||
5848 atomic_read(&rdev->irq.pflip[3])) {
5849 DRM_DEBUG("si_irq_set: vblank 3\n");
5850 crtc4 |= VBLANK_INT_MASK;
5852 if (rdev->irq.crtc_vblank_int[4] ||
5853 atomic_read(&rdev->irq.pflip[4])) {
5854 DRM_DEBUG("si_irq_set: vblank 4\n");
5855 crtc5 |= VBLANK_INT_MASK;
5857 if (rdev->irq.crtc_vblank_int[5] ||
5858 atomic_read(&rdev->irq.pflip[5])) {
5859 DRM_DEBUG("si_irq_set: vblank 5\n");
5860 crtc6 |= VBLANK_INT_MASK;
5862 if (rdev->irq.hpd[0]) {
5863 DRM_DEBUG("si_irq_set: hpd 1\n");
5864 hpd1 |= DC_HPDx_INT_EN;
5866 if (rdev->irq.hpd[1]) {
5867 DRM_DEBUG("si_irq_set: hpd 2\n");
5868 hpd2 |= DC_HPDx_INT_EN;
5870 if (rdev->irq.hpd[2]) {
5871 DRM_DEBUG("si_irq_set: hpd 3\n");
5872 hpd3 |= DC_HPDx_INT_EN;
5874 if (rdev->irq.hpd[3]) {
5875 DRM_DEBUG("si_irq_set: hpd 4\n");
5876 hpd4 |= DC_HPDx_INT_EN;
5878 if (rdev->irq.hpd[4]) {
5879 DRM_DEBUG("si_irq_set: hpd 5\n");
5880 hpd5 |= DC_HPDx_INT_EN;
5882 if (rdev->irq.hpd[5]) {
5883 DRM_DEBUG("si_irq_set: hpd 6\n");
5884 hpd6 |= DC_HPDx_INT_EN;
5887 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5888 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
5889 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
5891 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
5892 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
5894 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5896 if (rdev->irq.dpm_thermal) {
5897 DRM_DEBUG("dpm thermal\n");
5898 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
5901 if (rdev->num_crtc >= 2) {
5902 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5903 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5905 if (rdev->num_crtc >= 4) {
5906 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5907 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5909 if (rdev->num_crtc >= 6) {
5910 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5911 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5914 if (rdev->num_crtc >= 2) {
5915 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
5916 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
5918 if (rdev->num_crtc >= 4) {
5919 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
5920 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
5922 if (rdev->num_crtc >= 6) {
5923 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
5924 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
5927 if (!ASIC_IS_NODCE(rdev)) {
5928 WREG32(DC_HPD1_INT_CONTROL, hpd1);
5929 WREG32(DC_HPD2_INT_CONTROL, hpd2);
5930 WREG32(DC_HPD3_INT_CONTROL, hpd3);
5931 WREG32(DC_HPD4_INT_CONTROL, hpd4);
5932 WREG32(DC_HPD5_INT_CONTROL, hpd5);
5933 WREG32(DC_HPD6_INT_CONTROL, hpd6);
5936 WREG32(CG_THERMAL_INT, thermal_int);
5941 static inline void si_irq_ack(struct radeon_device *rdev)
5945 if (ASIC_IS_NODCE(rdev))
5948 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
5949 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
5950 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
5951 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
5952 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
5953 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
5954 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
5955 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
5956 if (rdev->num_crtc >= 4) {
5957 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
5958 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
5960 if (rdev->num_crtc >= 6) {
5961 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
5962 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
5965 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
5966 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5967 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
5968 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5969 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
5970 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
5971 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
5972 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
5973 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
5974 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
5975 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
5976 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
5978 if (rdev->num_crtc >= 4) {
5979 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
5980 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5981 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
5982 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5983 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
5984 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
5985 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
5986 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
5987 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
5988 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
5989 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
5990 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
5993 if (rdev->num_crtc >= 6) {
5994 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
5995 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5996 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
5997 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5998 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
5999 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6000 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6001 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6002 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6003 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6004 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6005 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6008 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6009 tmp = RREG32(DC_HPD1_INT_CONTROL);
6010 tmp |= DC_HPDx_INT_ACK;
6011 WREG32(DC_HPD1_INT_CONTROL, tmp);
6013 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6014 tmp = RREG32(DC_HPD2_INT_CONTROL);
6015 tmp |= DC_HPDx_INT_ACK;
6016 WREG32(DC_HPD2_INT_CONTROL, tmp);
6018 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6019 tmp = RREG32(DC_HPD3_INT_CONTROL);
6020 tmp |= DC_HPDx_INT_ACK;
6021 WREG32(DC_HPD3_INT_CONTROL, tmp);
6023 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6024 tmp = RREG32(DC_HPD4_INT_CONTROL);
6025 tmp |= DC_HPDx_INT_ACK;
6026 WREG32(DC_HPD4_INT_CONTROL, tmp);
6028 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6029 tmp = RREG32(DC_HPD5_INT_CONTROL);
6030 tmp |= DC_HPDx_INT_ACK;
6031 WREG32(DC_HPD5_INT_CONTROL, tmp);
6033 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6034 tmp = RREG32(DC_HPD5_INT_CONTROL);
6035 tmp |= DC_HPDx_INT_ACK;
6036 WREG32(DC_HPD6_INT_CONTROL, tmp);
6040 static void si_irq_disable(struct radeon_device *rdev)
6042 si_disable_interrupts(rdev);
6043 /* Wait and acknowledge irq */
6046 si_disable_interrupt_state(rdev);
6049 static void si_irq_suspend(struct radeon_device *rdev)
6051 si_irq_disable(rdev);
6055 static void si_irq_fini(struct radeon_device *rdev)
6057 si_irq_suspend(rdev);
6058 r600_ih_ring_fini(rdev);
6061 static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6065 if (rdev->wb.enabled)
6066 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6068 wptr = RREG32(IH_RB_WPTR);
6070 if (wptr & RB_OVERFLOW) {
6071 /* When a ring buffer overflow happen start parsing interrupt
6072 * from the last not overwritten vector (wptr + 16). Hopefully
6073 * this should allow us to catchup.
6075 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6076 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6077 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6078 tmp = RREG32(IH_RB_CNTL);
6079 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6080 WREG32(IH_RB_CNTL, tmp);
6082 return (wptr & rdev->ih.ptr_mask);
6086 * Each IV ring entry is 128 bits:
6087 * [7:0] - interrupt source id
6089 * [59:32] - interrupt source data
6090 * [63:60] - reserved
6093 * [127:80] - reserved
6095 int si_irq_process(struct radeon_device *rdev)
6099 u32 src_id, src_data, ring_id;
6101 bool queue_hotplug = false;
6102 bool queue_thermal = false;
6105 if (!rdev->ih.enabled || rdev->shutdown)
6108 wptr = si_get_ih_wptr(rdev);
6111 /* is somebody else already processing irqs? */
6112 if (atomic_xchg(&rdev->ih.lock, 1))
6115 rptr = rdev->ih.rptr;
6116 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6118 /* Order reading of wptr vs. reading of IH ring data */
6121 /* display interrupts */
6124 while (rptr != wptr) {
6125 /* wptr/rptr are in bytes! */
6126 ring_index = rptr / 4;
6127 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6128 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6129 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6132 case 1: /* D1 vblank/vline */
6134 case 0: /* D1 vblank */
6135 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6136 if (rdev->irq.crtc_vblank_int[0]) {
6137 drm_handle_vblank(rdev->ddev, 0);
6138 rdev->pm.vblank_sync = true;
6139 wake_up(&rdev->irq.vblank_queue);
6141 if (atomic_read(&rdev->irq.pflip[0]))
6142 radeon_crtc_handle_flip(rdev, 0);
6143 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6144 DRM_DEBUG("IH: D1 vblank\n");
6147 case 1: /* D1 vline */
6148 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
6149 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6150 DRM_DEBUG("IH: D1 vline\n");
6154 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6158 case 2: /* D2 vblank/vline */
6160 case 0: /* D2 vblank */
6161 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6162 if (rdev->irq.crtc_vblank_int[1]) {
6163 drm_handle_vblank(rdev->ddev, 1);
6164 rdev->pm.vblank_sync = true;
6165 wake_up(&rdev->irq.vblank_queue);
6167 if (atomic_read(&rdev->irq.pflip[1]))
6168 radeon_crtc_handle_flip(rdev, 1);
6169 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6170 DRM_DEBUG("IH: D2 vblank\n");
6173 case 1: /* D2 vline */
6174 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6175 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6176 DRM_DEBUG("IH: D2 vline\n");
6180 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6184 case 3: /* D3 vblank/vline */
6186 case 0: /* D3 vblank */
6187 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6188 if (rdev->irq.crtc_vblank_int[2]) {
6189 drm_handle_vblank(rdev->ddev, 2);
6190 rdev->pm.vblank_sync = true;
6191 wake_up(&rdev->irq.vblank_queue);
6193 if (atomic_read(&rdev->irq.pflip[2]))
6194 radeon_crtc_handle_flip(rdev, 2);
6195 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6196 DRM_DEBUG("IH: D3 vblank\n");
6199 case 1: /* D3 vline */
6200 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6201 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6202 DRM_DEBUG("IH: D3 vline\n");
6206 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6210 case 4: /* D4 vblank/vline */
6212 case 0: /* D4 vblank */
6213 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6214 if (rdev->irq.crtc_vblank_int[3]) {
6215 drm_handle_vblank(rdev->ddev, 3);
6216 rdev->pm.vblank_sync = true;
6217 wake_up(&rdev->irq.vblank_queue);
6219 if (atomic_read(&rdev->irq.pflip[3]))
6220 radeon_crtc_handle_flip(rdev, 3);
6221 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6222 DRM_DEBUG("IH: D4 vblank\n");
6225 case 1: /* D4 vline */
6226 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6227 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6228 DRM_DEBUG("IH: D4 vline\n");
6232 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6236 case 5: /* D5 vblank/vline */
6238 case 0: /* D5 vblank */
6239 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6240 if (rdev->irq.crtc_vblank_int[4]) {
6241 drm_handle_vblank(rdev->ddev, 4);
6242 rdev->pm.vblank_sync = true;
6243 wake_up(&rdev->irq.vblank_queue);
6245 if (atomic_read(&rdev->irq.pflip[4]))
6246 radeon_crtc_handle_flip(rdev, 4);
6247 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6248 DRM_DEBUG("IH: D5 vblank\n");
6251 case 1: /* D5 vline */
6252 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6253 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6254 DRM_DEBUG("IH: D5 vline\n");
6258 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6262 case 6: /* D6 vblank/vline */
6264 case 0: /* D6 vblank */
6265 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6266 if (rdev->irq.crtc_vblank_int[5]) {
6267 drm_handle_vblank(rdev->ddev, 5);
6268 rdev->pm.vblank_sync = true;
6269 wake_up(&rdev->irq.vblank_queue);
6271 if (atomic_read(&rdev->irq.pflip[5]))
6272 radeon_crtc_handle_flip(rdev, 5);
6273 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6274 DRM_DEBUG("IH: D6 vblank\n");
6277 case 1: /* D6 vline */
6278 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6279 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6280 DRM_DEBUG("IH: D6 vline\n");
6284 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6288 case 42: /* HPD hotplug */
6291 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6292 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6293 queue_hotplug = true;
6294 DRM_DEBUG("IH: HPD1\n");
6298 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6299 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6300 queue_hotplug = true;
6301 DRM_DEBUG("IH: HPD2\n");
6305 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6306 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6307 queue_hotplug = true;
6308 DRM_DEBUG("IH: HPD3\n");
6312 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6313 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6314 queue_hotplug = true;
6315 DRM_DEBUG("IH: HPD4\n");
6319 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6320 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6321 queue_hotplug = true;
6322 DRM_DEBUG("IH: HPD5\n");
6326 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6327 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6328 queue_hotplug = true;
6329 DRM_DEBUG("IH: HPD6\n");
6333 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6339 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6340 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6341 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6342 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
6344 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6346 si_vm_decode_fault(rdev, status, addr);
6347 /* reset addr and status */
6348 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6350 case 176: /* RINGID0 CP_INT */
6351 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6353 case 177: /* RINGID1 CP_INT */
6354 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6356 case 178: /* RINGID2 CP_INT */
6357 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6359 case 181: /* CP EOP event */
6360 DRM_DEBUG("IH: CP EOP\n");
6363 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6366 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6369 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6373 case 224: /* DMA trap event */
6374 DRM_DEBUG("IH: DMA trap\n");
6375 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6377 case 230: /* thermal low to high */
6378 DRM_DEBUG("IH: thermal low to high\n");
6379 rdev->pm.dpm.thermal.high_to_low = false;
6380 queue_thermal = true;
6382 case 231: /* thermal high to low */
6383 DRM_DEBUG("IH: thermal high to low\n");
6384 rdev->pm.dpm.thermal.high_to_low = true;
6385 queue_thermal = true;
6387 case 233: /* GUI IDLE */
6388 DRM_DEBUG("IH: GUI idle\n");
6390 case 244: /* DMA trap event */
6391 DRM_DEBUG("IH: DMA1 trap\n");
6392 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6395 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6399 /* wptr/rptr are in bytes! */
6401 rptr &= rdev->ih.ptr_mask;
6404 schedule_work(&rdev->hotplug_work);
6405 if (queue_thermal && rdev->pm.dpm_enabled)
6406 schedule_work(&rdev->pm.dpm.thermal.work);
6407 rdev->ih.rptr = rptr;
6408 WREG32(IH_RB_RPTR, rdev->ih.rptr);
6409 atomic_set(&rdev->ih.lock, 0);
6411 /* make sure wptr hasn't changed while processing */
6412 wptr = si_get_ih_wptr(rdev);
6420 * startup/shutdown callbacks
6422 static int si_startup(struct radeon_device *rdev)
6424 struct radeon_ring *ring;
6427 /* enable pcie gen2/3 link */
6428 si_pcie_gen3_enable(rdev);
6430 si_program_aspm(rdev);
6432 /* scratch needs to be initialized before MC */
6433 r = r600_vram_scratch_init(rdev);
6437 si_mc_program(rdev);
6439 if (!rdev->pm.dpm_enabled) {
6440 r = si_mc_load_microcode(rdev);
6442 DRM_ERROR("Failed to load MC firmware!\n");
6447 r = si_pcie_gart_enable(rdev);
6452 /* allocate rlc buffers */
6453 if (rdev->family == CHIP_VERDE) {
6454 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6455 rdev->rlc.reg_list_size =
6456 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6458 rdev->rlc.cs_data = si_cs_data;
6459 r = sumo_rlc_init(rdev);
6461 DRM_ERROR("Failed to init rlc BOs!\n");
6465 /* allocate wb buffer */
6466 r = radeon_wb_init(rdev);
6470 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6472 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6476 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6478 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6482 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6484 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6488 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6490 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6494 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6496 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6500 if (rdev->has_uvd) {
6501 r = uvd_v2_2_resume(rdev);
6503 r = radeon_fence_driver_start_ring(rdev,
6504 R600_RING_TYPE_UVD_INDEX);
6506 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6509 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6513 if (!rdev->irq.installed) {
6514 r = radeon_irq_kms_init(rdev);
6519 r = si_irq_init(rdev);
6521 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6522 radeon_irq_kms_fini(rdev);
6527 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6528 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6533 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6534 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6539 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6540 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6545 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6546 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6547 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6551 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6552 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6553 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6557 r = si_cp_load_microcode(rdev);
6560 r = si_cp_resume(rdev);
6564 r = cayman_dma_resume(rdev);
6568 if (rdev->has_uvd) {
6569 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6570 if (ring->ring_size) {
6571 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
6574 r = uvd_v1_0_init(rdev);
6576 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6580 r = radeon_ib_pool_init(rdev);
6582 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
6586 r = radeon_vm_manager_init(rdev);
6588 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
6592 r = dce6_audio_init(rdev);
6599 int si_resume(struct radeon_device *rdev)
6603 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6604 * posting will perform necessary task to bring back GPU into good
6608 atom_asic_init(rdev->mode_info.atom_context);
6610 /* init golden registers */
6611 si_init_golden_registers(rdev);
6613 radeon_pm_resume(rdev);
6615 rdev->accel_working = true;
6616 r = si_startup(rdev);
6618 DRM_ERROR("si startup failed on resume\n");
6619 rdev->accel_working = false;
6627 int si_suspend(struct radeon_device *rdev)
6629 radeon_pm_suspend(rdev);
6630 dce6_audio_fini(rdev);
6631 radeon_vm_manager_fini(rdev);
6632 si_cp_enable(rdev, false);
6633 cayman_dma_stop(rdev);
6634 if (rdev->has_uvd) {
6635 uvd_v1_0_fini(rdev);
6636 radeon_uvd_suspend(rdev);
6640 si_irq_suspend(rdev);
6641 radeon_wb_disable(rdev);
6642 si_pcie_gart_disable(rdev);
6646 /* Plan is to move initialization in that function and use
6647 * helper function so that radeon_device_init pretty much
6648 * do nothing more than calling asic specific function. This
6649 * should also allow to remove a bunch of callback function
6652 int si_init(struct radeon_device *rdev)
6654 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6658 if (!radeon_get_bios(rdev)) {
6659 if (ASIC_IS_AVIVO(rdev))
6662 /* Must be an ATOMBIOS */
6663 if (!rdev->is_atom_bios) {
6664 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6667 r = radeon_atombios_init(rdev);
6671 /* Post card if necessary */
6672 if (!radeon_card_posted(rdev)) {
6674 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6677 DRM_INFO("GPU not posted. posting now...\n");
6678 atom_asic_init(rdev->mode_info.atom_context);
6680 /* init golden registers */
6681 si_init_golden_registers(rdev);
6682 /* Initialize scratch registers */
6683 si_scratch_init(rdev);
6684 /* Initialize surface registers */
6685 radeon_surface_init(rdev);
6686 /* Initialize clocks */
6687 radeon_get_clock_info(rdev->ddev);
6690 r = radeon_fence_driver_init(rdev);
6694 /* initialize memory controller */
6695 r = si_mc_init(rdev);
6698 /* Memory manager */
6699 r = radeon_bo_init(rdev);
6703 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6704 !rdev->rlc_fw || !rdev->mc_fw) {
6705 r = si_init_microcode(rdev);
6707 DRM_ERROR("Failed to load firmware!\n");
6712 /* Initialize power management */
6713 radeon_pm_init(rdev);
6715 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6716 ring->ring_obj = NULL;
6717 r600_ring_init(rdev, ring, 1024 * 1024);
6719 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6720 ring->ring_obj = NULL;
6721 r600_ring_init(rdev, ring, 1024 * 1024);
6723 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6724 ring->ring_obj = NULL;
6725 r600_ring_init(rdev, ring, 1024 * 1024);
6727 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6728 ring->ring_obj = NULL;
6729 r600_ring_init(rdev, ring, 64 * 1024);
6731 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6732 ring->ring_obj = NULL;
6733 r600_ring_init(rdev, ring, 64 * 1024);
6735 if (rdev->has_uvd) {
6736 r = radeon_uvd_init(rdev);
6738 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6739 ring->ring_obj = NULL;
6740 r600_ring_init(rdev, ring, 4096);
6744 rdev->ih.ring_obj = NULL;
6745 r600_ih_ring_init(rdev, 64 * 1024);
6747 r = r600_pcie_gart_init(rdev);
6751 rdev->accel_working = true;
6752 r = si_startup(rdev);
6754 dev_err(rdev->dev, "disabling GPU acceleration\n");
6756 cayman_dma_fini(rdev);
6758 sumo_rlc_fini(rdev);
6759 radeon_wb_fini(rdev);
6760 radeon_ib_pool_fini(rdev);
6761 radeon_vm_manager_fini(rdev);
6762 radeon_irq_kms_fini(rdev);
6763 si_pcie_gart_fini(rdev);
6764 rdev->accel_working = false;
6767 /* Don't start up if the MC ucode is missing.
6768 * The default clocks and voltages before the MC ucode
6769 * is loaded are not suffient for advanced operations.
6772 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6779 void si_fini(struct radeon_device *rdev)
6781 radeon_pm_fini(rdev);
6783 cayman_dma_fini(rdev);
6787 sumo_rlc_fini(rdev);
6788 radeon_wb_fini(rdev);
6789 radeon_vm_manager_fini(rdev);
6790 radeon_ib_pool_fini(rdev);
6791 radeon_irq_kms_fini(rdev);
6792 if (rdev->has_uvd) {
6793 uvd_v1_0_fini(rdev);
6794 radeon_uvd_fini(rdev);
6796 si_pcie_gart_fini(rdev);
6797 r600_vram_scratch_fini(rdev);
6798 radeon_gem_fini(rdev);
6799 radeon_fence_driver_fini(rdev);
6800 radeon_bo_fini(rdev);
6801 radeon_atombios_fini(rdev);
6807 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6809 * @rdev: radeon_device pointer
6811 * Fetches a GPU clock counter snapshot (SI).
6812 * Returns the 64 bit clock counter snapshot.
6814 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6818 mutex_lock(&rdev->gpu_clock_mutex);
6819 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6820 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6821 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6822 mutex_unlock(&rdev->gpu_clock_mutex);
6826 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6828 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
6831 /* bypass vclk and dclk with bclk */
6832 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6833 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6834 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6836 /* put PLL in bypass mode */
6837 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6839 if (!vclk || !dclk) {
6840 /* keep the Bypass mode, put PLL to sleep */
6841 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6845 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6846 16384, 0x03FFFFFF, 0, 128, 5,
6847 &fb_div, &vclk_div, &dclk_div);
6851 /* set RESET_ANTI_MUX to 0 */
6852 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
6854 /* set VCO_MODE to 1 */
6855 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
6857 /* toggle UPLL_SLEEP to 1 then back to 0 */
6858 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6859 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
6861 /* deassert UPLL_RESET */
6862 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6866 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
6870 /* assert UPLL_RESET again */
6871 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
6873 /* disable spread spectrum. */
6874 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
6876 /* set feedback divider */
6877 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
6879 /* set ref divider to 0 */
6880 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
6882 if (fb_div < 307200)
6883 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
6885 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
6887 /* set PDIV_A and PDIV_B */
6888 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6889 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
6890 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
6892 /* give the PLL some time to settle */
6895 /* deassert PLL_RESET */
6896 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6900 /* switch from bypass mode to normal mode */
6901 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
6903 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
6907 /* switch VCLK and DCLK selection */
6908 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6909 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6910 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6917 static void si_pcie_gen3_enable(struct radeon_device *rdev)
6919 struct pci_dev *root = rdev->pdev->bus->self;
6920 int bridge_pos, gpu_pos;
6921 u32 speed_cntl, mask, current_data_rate;
6925 if (radeon_pcie_gen2 == 0)
6928 if (rdev->flags & RADEON_IS_IGP)
6931 if (!(rdev->flags & RADEON_IS_PCIE))
6934 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
6938 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
6941 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6942 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
6943 LC_CURRENT_DATA_RATE_SHIFT;
6944 if (mask & DRM_PCIE_SPEED_80) {
6945 if (current_data_rate == 2) {
6946 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
6949 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
6950 } else if (mask & DRM_PCIE_SPEED_50) {
6951 if (current_data_rate == 1) {
6952 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
6955 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
6958 bridge_pos = pci_pcie_cap(root);
6962 gpu_pos = pci_pcie_cap(rdev->pdev);
6966 if (mask & DRM_PCIE_SPEED_80) {
6967 /* re-try equalization if gen3 is not already enabled */
6968 if (current_data_rate != 2) {
6969 u16 bridge_cfg, gpu_cfg;
6970 u16 bridge_cfg2, gpu_cfg2;
6971 u32 max_lw, current_lw, tmp;
6973 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6974 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6976 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
6977 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6979 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
6980 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6982 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
6983 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
6984 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
6986 if (current_lw < max_lw) {
6987 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6988 if (tmp & LC_RENEGOTIATION_SUPPORT) {
6989 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
6990 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
6991 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
6992 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
6996 for (i = 0; i < 10; i++) {
6998 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
6999 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7002 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7003 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7005 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
7006 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
7008 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7009 tmp |= LC_SET_QUIESCE;
7010 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7012 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7014 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7019 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
7020 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7021 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
7022 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7024 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
7025 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7026 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
7027 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7030 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
7031 tmp16 &= ~((1 << 4) | (7 << 9));
7032 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
7033 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
7035 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7036 tmp16 &= ~((1 << 4) | (7 << 9));
7037 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
7038 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7040 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7041 tmp &= ~LC_SET_QUIESCE;
7042 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7047 /* set the link speed */
7048 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7049 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7050 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7052 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7054 if (mask & DRM_PCIE_SPEED_80)
7055 tmp16 |= 3; /* gen3 */
7056 else if (mask & DRM_PCIE_SPEED_50)
7057 tmp16 |= 2; /* gen2 */
7059 tmp16 |= 1; /* gen1 */
7060 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7062 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7063 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7064 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7066 for (i = 0; i < rdev->usec_timeout; i++) {
7067 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7068 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7074 static void si_program_aspm(struct radeon_device *rdev)
7077 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7078 bool disable_clkreq = false;
7080 if (radeon_aspm == 0)
7083 if (!(rdev->flags & RADEON_IS_PCIE))
7086 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7087 data &= ~LC_XMIT_N_FTS_MASK;
7088 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7090 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7092 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7093 data |= LC_GO_TO_RECOVERY;
7095 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7097 orig = data = RREG32_PCIE(PCIE_P_CNTL);
7098 data |= P_IGNORE_EDB_ERR;
7100 WREG32_PCIE(PCIE_P_CNTL, data);
7102 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7103 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7104 data |= LC_PMI_TO_L1_DIS;
7106 data |= LC_L0S_INACTIVITY(7);
7109 data |= LC_L1_INACTIVITY(7);
7110 data &= ~LC_PMI_TO_L1_DIS;
7112 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7114 if (!disable_plloff_in_l1) {
7115 bool clk_req_support;
7117 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7118 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7119 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7121 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7123 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7124 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7125 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7127 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7129 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7130 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7131 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7133 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7135 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7136 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7137 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7139 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7141 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7142 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7143 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7145 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7147 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7148 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7150 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7152 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7153 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7155 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7157 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7158 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7160 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7162 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7163 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7165 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7167 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7168 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7170 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7172 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7173 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7175 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7177 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7178 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7180 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7182 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7183 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7184 data |= LC_DYN_LANES_PWR_STATE(3);
7186 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7188 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7189 data &= ~LS2_EXIT_TIME_MASK;
7190 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7191 data |= LS2_EXIT_TIME(5);
7193 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7195 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7196 data &= ~LS2_EXIT_TIME_MASK;
7197 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7198 data |= LS2_EXIT_TIME(5);
7200 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7202 if (!disable_clkreq) {
7203 struct pci_dev *root = rdev->pdev->bus->self;
7206 clk_req_support = false;
7207 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7208 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7209 clk_req_support = true;
7211 clk_req_support = false;
7214 if (clk_req_support) {
7215 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7216 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7218 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7220 orig = data = RREG32(THM_CLK_CNTL);
7221 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7222 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7224 WREG32(THM_CLK_CNTL, data);
7226 orig = data = RREG32(MISC_CLK_CNTL);
7227 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7228 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7230 WREG32(MISC_CLK_CNTL, data);
7232 orig = data = RREG32(CG_CLKPIN_CNTL);
7233 data &= ~BCLK_AS_XCLK;
7235 WREG32(CG_CLKPIN_CNTL, data);
7237 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7238 data &= ~FORCE_BIF_REFCLK_EN;
7240 WREG32(CG_CLKPIN_CNTL_2, data);
7242 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7243 data &= ~MPLL_CLKOUT_SEL_MASK;
7244 data |= MPLL_CLKOUT_SEL(4);
7246 WREG32(MPLL_BYPASSCLK_SEL, data);
7248 orig = data = RREG32(SPLL_CNTL_MODE);
7249 data &= ~SPLL_REFCLK_SEL_MASK;
7251 WREG32(SPLL_CNTL_MODE, data);
7256 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7259 orig = data = RREG32_PCIE(PCIE_CNTL2);
7260 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7262 WREG32_PCIE(PCIE_CNTL2, data);
7265 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7266 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7267 data = RREG32_PCIE(PCIE_LC_STATUS1);
7268 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7269 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7270 data &= ~LC_L0S_INACTIVITY_MASK;
7272 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
projects / karo-tx-linux.git / blob