2 * Copyright 2011 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
29 #include "radeon_asic.h"
30 #include <drm/radeon_drm.h>
33 #include "si_blit_shaders.h"
34 #include "clearstate_si.h"
35 #include "radeon_ucode.h"
38 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
39 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
40 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
41 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
42 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
43 MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
44 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
45 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
46 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
47 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
48 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
49 MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
50 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
51 MODULE_FIRMWARE("radeon/VERDE_me.bin");
52 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
53 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
54 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
55 MODULE_FIRMWARE("radeon/VERDE_smc.bin");
56 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
57 MODULE_FIRMWARE("radeon/OLAND_me.bin");
58 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
59 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
60 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
61 MODULE_FIRMWARE("radeon/OLAND_smc.bin");
62 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
63 MODULE_FIRMWARE("radeon/HAINAN_me.bin");
64 MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
65 MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
66 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
67 MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
69 static void si_pcie_gen3_enable(struct radeon_device *rdev);
70 static void si_program_aspm(struct radeon_device *rdev);
71 extern void sumo_rlc_fini(struct radeon_device *rdev);
72 extern int sumo_rlc_init(struct radeon_device *rdev);
73 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
74 extern void r600_ih_ring_fini(struct radeon_device *rdev);
75 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
76 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
77 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
78 extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
79 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
80 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
81 extern void si_dma_vm_set_page(struct radeon_device *rdev,
84 uint64_t addr, unsigned count,
85 uint32_t incr, uint32_t flags);
86 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
88 static void si_fini_pg(struct radeon_device *rdev);
89 static void si_fini_cg(struct radeon_device *rdev);
90 static void si_rlc_stop(struct radeon_device *rdev);
92 static const u32 verde_rlc_save_restore_register_list[] =
94 (0x8000 << 16) | (0x98f4 >> 2),
96 (0x8040 << 16) | (0x98f4 >> 2),
98 (0x8000 << 16) | (0xe80 >> 2),
100 (0x8040 << 16) | (0xe80 >> 2),
102 (0x8000 << 16) | (0x89bc >> 2),
104 (0x8040 << 16) | (0x89bc >> 2),
106 (0x8000 << 16) | (0x8c1c >> 2),
108 (0x8040 << 16) | (0x8c1c >> 2),
110 (0x9c00 << 16) | (0x98f0 >> 2),
112 (0x9c00 << 16) | (0xe7c >> 2),
114 (0x8000 << 16) | (0x9148 >> 2),
116 (0x8040 << 16) | (0x9148 >> 2),
118 (0x9c00 << 16) | (0x9150 >> 2),
120 (0x9c00 << 16) | (0x897c >> 2),
122 (0x9c00 << 16) | (0x8d8c >> 2),
124 (0x9c00 << 16) | (0xac54 >> 2),
127 (0x9c00 << 16) | (0x98f8 >> 2),
129 (0x9c00 << 16) | (0x9910 >> 2),
131 (0x9c00 << 16) | (0x9914 >> 2),
133 (0x9c00 << 16) | (0x9918 >> 2),
135 (0x9c00 << 16) | (0x991c >> 2),
137 (0x9c00 << 16) | (0x9920 >> 2),
139 (0x9c00 << 16) | (0x9924 >> 2),
141 (0x9c00 << 16) | (0x9928 >> 2),
143 (0x9c00 << 16) | (0x992c >> 2),
145 (0x9c00 << 16) | (0x9930 >> 2),
147 (0x9c00 << 16) | (0x9934 >> 2),
149 (0x9c00 << 16) | (0x9938 >> 2),
151 (0x9c00 << 16) | (0x993c >> 2),
153 (0x9c00 << 16) | (0x9940 >> 2),
155 (0x9c00 << 16) | (0x9944 >> 2),
157 (0x9c00 << 16) | (0x9948 >> 2),
159 (0x9c00 << 16) | (0x994c >> 2),
161 (0x9c00 << 16) | (0x9950 >> 2),
163 (0x9c00 << 16) | (0x9954 >> 2),
165 (0x9c00 << 16) | (0x9958 >> 2),
167 (0x9c00 << 16) | (0x995c >> 2),
169 (0x9c00 << 16) | (0x9960 >> 2),
171 (0x9c00 << 16) | (0x9964 >> 2),
173 (0x9c00 << 16) | (0x9968 >> 2),
175 (0x9c00 << 16) | (0x996c >> 2),
177 (0x9c00 << 16) | (0x9970 >> 2),
179 (0x9c00 << 16) | (0x9974 >> 2),
181 (0x9c00 << 16) | (0x9978 >> 2),
183 (0x9c00 << 16) | (0x997c >> 2),
185 (0x9c00 << 16) | (0x9980 >> 2),
187 (0x9c00 << 16) | (0x9984 >> 2),
189 (0x9c00 << 16) | (0x9988 >> 2),
191 (0x9c00 << 16) | (0x998c >> 2),
193 (0x9c00 << 16) | (0x8c00 >> 2),
195 (0x9c00 << 16) | (0x8c14 >> 2),
197 (0x9c00 << 16) | (0x8c04 >> 2),
199 (0x9c00 << 16) | (0x8c08 >> 2),
201 (0x8000 << 16) | (0x9b7c >> 2),
203 (0x8040 << 16) | (0x9b7c >> 2),
205 (0x8000 << 16) | (0xe84 >> 2),
207 (0x8040 << 16) | (0xe84 >> 2),
209 (0x8000 << 16) | (0x89c0 >> 2),
211 (0x8040 << 16) | (0x89c0 >> 2),
213 (0x8000 << 16) | (0x914c >> 2),
215 (0x8040 << 16) | (0x914c >> 2),
217 (0x8000 << 16) | (0x8c20 >> 2),
219 (0x8040 << 16) | (0x8c20 >> 2),
221 (0x8000 << 16) | (0x9354 >> 2),
223 (0x8040 << 16) | (0x9354 >> 2),
225 (0x9c00 << 16) | (0x9060 >> 2),
227 (0x9c00 << 16) | (0x9364 >> 2),
229 (0x9c00 << 16) | (0x9100 >> 2),
231 (0x9c00 << 16) | (0x913c >> 2),
233 (0x8000 << 16) | (0x90e0 >> 2),
235 (0x8000 << 16) | (0x90e4 >> 2),
237 (0x8000 << 16) | (0x90e8 >> 2),
239 (0x8040 << 16) | (0x90e0 >> 2),
241 (0x8040 << 16) | (0x90e4 >> 2),
243 (0x8040 << 16) | (0x90e8 >> 2),
245 (0x9c00 << 16) | (0x8bcc >> 2),
247 (0x9c00 << 16) | (0x8b24 >> 2),
249 (0x9c00 << 16) | (0x88c4 >> 2),
251 (0x9c00 << 16) | (0x8e50 >> 2),
253 (0x9c00 << 16) | (0x8c0c >> 2),
255 (0x9c00 << 16) | (0x8e58 >> 2),
257 (0x9c00 << 16) | (0x8e5c >> 2),
259 (0x9c00 << 16) | (0x9508 >> 2),
261 (0x9c00 << 16) | (0x950c >> 2),
263 (0x9c00 << 16) | (0x9494 >> 2),
265 (0x9c00 << 16) | (0xac0c >> 2),
267 (0x9c00 << 16) | (0xac10 >> 2),
269 (0x9c00 << 16) | (0xac14 >> 2),
271 (0x9c00 << 16) | (0xae00 >> 2),
273 (0x9c00 << 16) | (0xac08 >> 2),
275 (0x9c00 << 16) | (0x88d4 >> 2),
277 (0x9c00 << 16) | (0x88c8 >> 2),
279 (0x9c00 << 16) | (0x88cc >> 2),
281 (0x9c00 << 16) | (0x89b0 >> 2),
283 (0x9c00 << 16) | (0x8b10 >> 2),
285 (0x9c00 << 16) | (0x8a14 >> 2),
287 (0x9c00 << 16) | (0x9830 >> 2),
289 (0x9c00 << 16) | (0x9834 >> 2),
291 (0x9c00 << 16) | (0x9838 >> 2),
293 (0x9c00 << 16) | (0x9a10 >> 2),
295 (0x8000 << 16) | (0x9870 >> 2),
297 (0x8000 << 16) | (0x9874 >> 2),
299 (0x8001 << 16) | (0x9870 >> 2),
301 (0x8001 << 16) | (0x9874 >> 2),
303 (0x8040 << 16) | (0x9870 >> 2),
305 (0x8040 << 16) | (0x9874 >> 2),
307 (0x8041 << 16) | (0x9870 >> 2),
309 (0x8041 << 16) | (0x9874 >> 2),
314 static const u32 tahiti_golden_rlc_registers[] =
316 0xc424, 0xffffffff, 0x00601005,
317 0xc47c, 0xffffffff, 0x10104040,
318 0xc488, 0xffffffff, 0x0100000a,
319 0xc314, 0xffffffff, 0x00000800,
320 0xc30c, 0xffffffff, 0x800000f4,
321 0xf4a8, 0xffffffff, 0x00000000
324 static const u32 tahiti_golden_registers[] =
326 0x9a10, 0x00010000, 0x00018208,
327 0x9830, 0xffffffff, 0x00000000,
328 0x9834, 0xf00fffff, 0x00000400,
329 0x9838, 0x0002021c, 0x00020200,
330 0xc78, 0x00000080, 0x00000000,
331 0xd030, 0x000300c0, 0x00800040,
332 0xd830, 0x000300c0, 0x00800040,
333 0x5bb0, 0x000000f0, 0x00000070,
334 0x5bc0, 0x00200000, 0x50100000,
335 0x7030, 0x31000311, 0x00000011,
336 0x277c, 0x00000003, 0x000007ff,
337 0x240c, 0x000007ff, 0x00000000,
338 0x8a14, 0xf000001f, 0x00000007,
339 0x8b24, 0xffffffff, 0x00ffffff,
340 0x8b10, 0x0000ff0f, 0x00000000,
341 0x28a4c, 0x07ffffff, 0x4e000000,
342 0x28350, 0x3f3f3fff, 0x2a00126a,
343 0x30, 0x000000ff, 0x0040,
344 0x34, 0x00000040, 0x00004040,
345 0x9100, 0x07ffffff, 0x03000000,
346 0x8e88, 0x01ff1f3f, 0x00000000,
347 0x8e84, 0x01ff1f3f, 0x00000000,
348 0x9060, 0x0000007f, 0x00000020,
349 0x9508, 0x00010000, 0x00010000,
350 0xac14, 0x00000200, 0x000002fb,
351 0xac10, 0xffffffff, 0x0000543b,
352 0xac0c, 0xffffffff, 0xa9210876,
353 0x88d0, 0xffffffff, 0x000fff40,
354 0x88d4, 0x0000001f, 0x00000010,
355 0x1410, 0x20000000, 0x20fffed8,
356 0x15c0, 0x000c0fc0, 0x000c0400
359 static const u32 tahiti_golden_registers2[] =
361 0xc64, 0x00000001, 0x00000001
364 static const u32 pitcairn_golden_rlc_registers[] =
366 0xc424, 0xffffffff, 0x00601004,
367 0xc47c, 0xffffffff, 0x10102020,
368 0xc488, 0xffffffff, 0x01000020,
369 0xc314, 0xffffffff, 0x00000800,
370 0xc30c, 0xffffffff, 0x800000a4
373 static const u32 pitcairn_golden_registers[] =
375 0x9a10, 0x00010000, 0x00018208,
376 0x9830, 0xffffffff, 0x00000000,
377 0x9834, 0xf00fffff, 0x00000400,
378 0x9838, 0x0002021c, 0x00020200,
379 0xc78, 0x00000080, 0x00000000,
380 0xd030, 0x000300c0, 0x00800040,
381 0xd830, 0x000300c0, 0x00800040,
382 0x5bb0, 0x000000f0, 0x00000070,
383 0x5bc0, 0x00200000, 0x50100000,
384 0x7030, 0x31000311, 0x00000011,
385 0x2ae4, 0x00073ffe, 0x000022a2,
386 0x240c, 0x000007ff, 0x00000000,
387 0x8a14, 0xf000001f, 0x00000007,
388 0x8b24, 0xffffffff, 0x00ffffff,
389 0x8b10, 0x0000ff0f, 0x00000000,
390 0x28a4c, 0x07ffffff, 0x4e000000,
391 0x28350, 0x3f3f3fff, 0x2a00126a,
392 0x30, 0x000000ff, 0x0040,
393 0x34, 0x00000040, 0x00004040,
394 0x9100, 0x07ffffff, 0x03000000,
395 0x9060, 0x0000007f, 0x00000020,
396 0x9508, 0x00010000, 0x00010000,
397 0xac14, 0x000003ff, 0x000000f7,
398 0xac10, 0xffffffff, 0x00000000,
399 0xac0c, 0xffffffff, 0x32761054,
400 0x88d4, 0x0000001f, 0x00000010,
401 0x15c0, 0x000c0fc0, 0x000c0400
404 static const u32 verde_golden_rlc_registers[] =
406 0xc424, 0xffffffff, 0x033f1005,
407 0xc47c, 0xffffffff, 0x10808020,
408 0xc488, 0xffffffff, 0x00800008,
409 0xc314, 0xffffffff, 0x00001000,
410 0xc30c, 0xffffffff, 0x80010014
413 static const u32 verde_golden_registers[] =
415 0x9a10, 0x00010000, 0x00018208,
416 0x9830, 0xffffffff, 0x00000000,
417 0x9834, 0xf00fffff, 0x00000400,
418 0x9838, 0x0002021c, 0x00020200,
419 0xc78, 0x00000080, 0x00000000,
420 0xd030, 0x000300c0, 0x00800040,
421 0xd030, 0x000300c0, 0x00800040,
422 0xd830, 0x000300c0, 0x00800040,
423 0xd830, 0x000300c0, 0x00800040,
424 0x5bb0, 0x000000f0, 0x00000070,
425 0x5bc0, 0x00200000, 0x50100000,
426 0x7030, 0x31000311, 0x00000011,
427 0x2ae4, 0x00073ffe, 0x000022a2,
428 0x2ae4, 0x00073ffe, 0x000022a2,
429 0x2ae4, 0x00073ffe, 0x000022a2,
430 0x240c, 0x000007ff, 0x00000000,
431 0x240c, 0x000007ff, 0x00000000,
432 0x240c, 0x000007ff, 0x00000000,
433 0x8a14, 0xf000001f, 0x00000007,
434 0x8a14, 0xf000001f, 0x00000007,
435 0x8a14, 0xf000001f, 0x00000007,
436 0x8b24, 0xffffffff, 0x00ffffff,
437 0x8b10, 0x0000ff0f, 0x00000000,
438 0x28a4c, 0x07ffffff, 0x4e000000,
439 0x28350, 0x3f3f3fff, 0x0000124a,
440 0x28350, 0x3f3f3fff, 0x0000124a,
441 0x28350, 0x3f3f3fff, 0x0000124a,
442 0x30, 0x000000ff, 0x0040,
443 0x34, 0x00000040, 0x00004040,
444 0x9100, 0x07ffffff, 0x03000000,
445 0x9100, 0x07ffffff, 0x03000000,
446 0x8e88, 0x01ff1f3f, 0x00000000,
447 0x8e88, 0x01ff1f3f, 0x00000000,
448 0x8e88, 0x01ff1f3f, 0x00000000,
449 0x8e84, 0x01ff1f3f, 0x00000000,
450 0x8e84, 0x01ff1f3f, 0x00000000,
451 0x8e84, 0x01ff1f3f, 0x00000000,
452 0x9060, 0x0000007f, 0x00000020,
453 0x9508, 0x00010000, 0x00010000,
454 0xac14, 0x000003ff, 0x00000003,
455 0xac14, 0x000003ff, 0x00000003,
456 0xac14, 0x000003ff, 0x00000003,
457 0xac10, 0xffffffff, 0x00000000,
458 0xac10, 0xffffffff, 0x00000000,
459 0xac10, 0xffffffff, 0x00000000,
460 0xac0c, 0xffffffff, 0x00001032,
461 0xac0c, 0xffffffff, 0x00001032,
462 0xac0c, 0xffffffff, 0x00001032,
463 0x88d4, 0x0000001f, 0x00000010,
464 0x88d4, 0x0000001f, 0x00000010,
465 0x88d4, 0x0000001f, 0x00000010,
466 0x15c0, 0x000c0fc0, 0x000c0400
469 static const u32 oland_golden_rlc_registers[] =
471 0xc424, 0xffffffff, 0x00601005,
472 0xc47c, 0xffffffff, 0x10104040,
473 0xc488, 0xffffffff, 0x0100000a,
474 0xc314, 0xffffffff, 0x00000800,
475 0xc30c, 0xffffffff, 0x800000f4
478 static const u32 oland_golden_registers[] =
480 0x9a10, 0x00010000, 0x00018208,
481 0x9830, 0xffffffff, 0x00000000,
482 0x9834, 0xf00fffff, 0x00000400,
483 0x9838, 0x0002021c, 0x00020200,
484 0xc78, 0x00000080, 0x00000000,
485 0xd030, 0x000300c0, 0x00800040,
486 0xd830, 0x000300c0, 0x00800040,
487 0x5bb0, 0x000000f0, 0x00000070,
488 0x5bc0, 0x00200000, 0x50100000,
489 0x7030, 0x31000311, 0x00000011,
490 0x2ae4, 0x00073ffe, 0x000022a2,
491 0x240c, 0x000007ff, 0x00000000,
492 0x8a14, 0xf000001f, 0x00000007,
493 0x8b24, 0xffffffff, 0x00ffffff,
494 0x8b10, 0x0000ff0f, 0x00000000,
495 0x28a4c, 0x07ffffff, 0x4e000000,
496 0x28350, 0x3f3f3fff, 0x00000082,
497 0x30, 0x000000ff, 0x0040,
498 0x34, 0x00000040, 0x00004040,
499 0x9100, 0x07ffffff, 0x03000000,
500 0x9060, 0x0000007f, 0x00000020,
501 0x9508, 0x00010000, 0x00010000,
502 0xac14, 0x000003ff, 0x000000f3,
503 0xac10, 0xffffffff, 0x00000000,
504 0xac0c, 0xffffffff, 0x00003210,
505 0x88d4, 0x0000001f, 0x00000010,
506 0x15c0, 0x000c0fc0, 0x000c0400
509 static const u32 hainan_golden_registers[] =
511 0x9a10, 0x00010000, 0x00018208,
512 0x9830, 0xffffffff, 0x00000000,
513 0x9834, 0xf00fffff, 0x00000400,
514 0x9838, 0x0002021c, 0x00020200,
515 0xd0c0, 0xff000fff, 0x00000100,
516 0xd030, 0x000300c0, 0x00800040,
517 0xd8c0, 0xff000fff, 0x00000100,
518 0xd830, 0x000300c0, 0x00800040,
519 0x2ae4, 0x00073ffe, 0x000022a2,
520 0x240c, 0x000007ff, 0x00000000,
521 0x8a14, 0xf000001f, 0x00000007,
522 0x8b24, 0xffffffff, 0x00ffffff,
523 0x8b10, 0x0000ff0f, 0x00000000,
524 0x28a4c, 0x07ffffff, 0x4e000000,
525 0x28350, 0x3f3f3fff, 0x00000000,
526 0x30, 0x000000ff, 0x0040,
527 0x34, 0x00000040, 0x00004040,
528 0x9100, 0x03e00000, 0x03600000,
529 0x9060, 0x0000007f, 0x00000020,
530 0x9508, 0x00010000, 0x00010000,
531 0xac14, 0x000003ff, 0x000000f1,
532 0xac10, 0xffffffff, 0x00000000,
533 0xac0c, 0xffffffff, 0x00003210,
534 0x88d4, 0x0000001f, 0x00000010,
535 0x15c0, 0x000c0fc0, 0x000c0400
538 static const u32 hainan_golden_registers2[] =
540 0x98f8, 0xffffffff, 0x02010001
543 static const u32 tahiti_mgcg_cgcg_init[] =
545 0xc400, 0xffffffff, 0xfffffffc,
546 0x802c, 0xffffffff, 0xe0000000,
547 0x9a60, 0xffffffff, 0x00000100,
548 0x92a4, 0xffffffff, 0x00000100,
549 0xc164, 0xffffffff, 0x00000100,
550 0x9774, 0xffffffff, 0x00000100,
551 0x8984, 0xffffffff, 0x06000100,
552 0x8a18, 0xffffffff, 0x00000100,
553 0x92a0, 0xffffffff, 0x00000100,
554 0xc380, 0xffffffff, 0x00000100,
555 0x8b28, 0xffffffff, 0x00000100,
556 0x9144, 0xffffffff, 0x00000100,
557 0x8d88, 0xffffffff, 0x00000100,
558 0x8d8c, 0xffffffff, 0x00000100,
559 0x9030, 0xffffffff, 0x00000100,
560 0x9034, 0xffffffff, 0x00000100,
561 0x9038, 0xffffffff, 0x00000100,
562 0x903c, 0xffffffff, 0x00000100,
563 0xad80, 0xffffffff, 0x00000100,
564 0xac54, 0xffffffff, 0x00000100,
565 0x897c, 0xffffffff, 0x06000100,
566 0x9868, 0xffffffff, 0x00000100,
567 0x9510, 0xffffffff, 0x00000100,
568 0xaf04, 0xffffffff, 0x00000100,
569 0xae04, 0xffffffff, 0x00000100,
570 0x949c, 0xffffffff, 0x00000100,
571 0x802c, 0xffffffff, 0xe0000000,
572 0x9160, 0xffffffff, 0x00010000,
573 0x9164, 0xffffffff, 0x00030002,
574 0x9168, 0xffffffff, 0x00040007,
575 0x916c, 0xffffffff, 0x00060005,
576 0x9170, 0xffffffff, 0x00090008,
577 0x9174, 0xffffffff, 0x00020001,
578 0x9178, 0xffffffff, 0x00040003,
579 0x917c, 0xffffffff, 0x00000007,
580 0x9180, 0xffffffff, 0x00060005,
581 0x9184, 0xffffffff, 0x00090008,
582 0x9188, 0xffffffff, 0x00030002,
583 0x918c, 0xffffffff, 0x00050004,
584 0x9190, 0xffffffff, 0x00000008,
585 0x9194, 0xffffffff, 0x00070006,
586 0x9198, 0xffffffff, 0x000a0009,
587 0x919c, 0xffffffff, 0x00040003,
588 0x91a0, 0xffffffff, 0x00060005,
589 0x91a4, 0xffffffff, 0x00000009,
590 0x91a8, 0xffffffff, 0x00080007,
591 0x91ac, 0xffffffff, 0x000b000a,
592 0x91b0, 0xffffffff, 0x00050004,
593 0x91b4, 0xffffffff, 0x00070006,
594 0x91b8, 0xffffffff, 0x0008000b,
595 0x91bc, 0xffffffff, 0x000a0009,
596 0x91c0, 0xffffffff, 0x000d000c,
597 0x91c4, 0xffffffff, 0x00060005,
598 0x91c8, 0xffffffff, 0x00080007,
599 0x91cc, 0xffffffff, 0x0000000b,
600 0x91d0, 0xffffffff, 0x000a0009,
601 0x91d4, 0xffffffff, 0x000d000c,
602 0x91d8, 0xffffffff, 0x00070006,
603 0x91dc, 0xffffffff, 0x00090008,
604 0x91e0, 0xffffffff, 0x0000000c,
605 0x91e4, 0xffffffff, 0x000b000a,
606 0x91e8, 0xffffffff, 0x000e000d,
607 0x91ec, 0xffffffff, 0x00080007,
608 0x91f0, 0xffffffff, 0x000a0009,
609 0x91f4, 0xffffffff, 0x0000000d,
610 0x91f8, 0xffffffff, 0x000c000b,
611 0x91fc, 0xffffffff, 0x000f000e,
612 0x9200, 0xffffffff, 0x00090008,
613 0x9204, 0xffffffff, 0x000b000a,
614 0x9208, 0xffffffff, 0x000c000f,
615 0x920c, 0xffffffff, 0x000e000d,
616 0x9210, 0xffffffff, 0x00110010,
617 0x9214, 0xffffffff, 0x000a0009,
618 0x9218, 0xffffffff, 0x000c000b,
619 0x921c, 0xffffffff, 0x0000000f,
620 0x9220, 0xffffffff, 0x000e000d,
621 0x9224, 0xffffffff, 0x00110010,
622 0x9228, 0xffffffff, 0x000b000a,
623 0x922c, 0xffffffff, 0x000d000c,
624 0x9230, 0xffffffff, 0x00000010,
625 0x9234, 0xffffffff, 0x000f000e,
626 0x9238, 0xffffffff, 0x00120011,
627 0x923c, 0xffffffff, 0x000c000b,
628 0x9240, 0xffffffff, 0x000e000d,
629 0x9244, 0xffffffff, 0x00000011,
630 0x9248, 0xffffffff, 0x0010000f,
631 0x924c, 0xffffffff, 0x00130012,
632 0x9250, 0xffffffff, 0x000d000c,
633 0x9254, 0xffffffff, 0x000f000e,
634 0x9258, 0xffffffff, 0x00100013,
635 0x925c, 0xffffffff, 0x00120011,
636 0x9260, 0xffffffff, 0x00150014,
637 0x9264, 0xffffffff, 0x000e000d,
638 0x9268, 0xffffffff, 0x0010000f,
639 0x926c, 0xffffffff, 0x00000013,
640 0x9270, 0xffffffff, 0x00120011,
641 0x9274, 0xffffffff, 0x00150014,
642 0x9278, 0xffffffff, 0x000f000e,
643 0x927c, 0xffffffff, 0x00110010,
644 0x9280, 0xffffffff, 0x00000014,
645 0x9284, 0xffffffff, 0x00130012,
646 0x9288, 0xffffffff, 0x00160015,
647 0x928c, 0xffffffff, 0x0010000f,
648 0x9290, 0xffffffff, 0x00120011,
649 0x9294, 0xffffffff, 0x00000015,
650 0x9298, 0xffffffff, 0x00140013,
651 0x929c, 0xffffffff, 0x00170016,
652 0x9150, 0xffffffff, 0x96940200,
653 0x8708, 0xffffffff, 0x00900100,
654 0xc478, 0xffffffff, 0x00000080,
655 0xc404, 0xffffffff, 0x0020003f,
656 0x30, 0xffffffff, 0x0000001c,
657 0x34, 0x000f0000, 0x000f0000,
658 0x160c, 0xffffffff, 0x00000100,
659 0x1024, 0xffffffff, 0x00000100,
660 0x102c, 0x00000101, 0x00000000,
661 0x20a8, 0xffffffff, 0x00000104,
662 0x264c, 0x000c0000, 0x000c0000,
663 0x2648, 0x000c0000, 0x000c0000,
664 0x55e4, 0xff000fff, 0x00000100,
665 0x55e8, 0x00000001, 0x00000001,
666 0x2f50, 0x00000001, 0x00000001,
667 0x30cc, 0xc0000fff, 0x00000104,
668 0xc1e4, 0x00000001, 0x00000001,
669 0xd0c0, 0xfffffff0, 0x00000100,
670 0xd8c0, 0xfffffff0, 0x00000100
673 static const u32 pitcairn_mgcg_cgcg_init[] =
675 0xc400, 0xffffffff, 0xfffffffc,
676 0x802c, 0xffffffff, 0xe0000000,
677 0x9a60, 0xffffffff, 0x00000100,
678 0x92a4, 0xffffffff, 0x00000100,
679 0xc164, 0xffffffff, 0x00000100,
680 0x9774, 0xffffffff, 0x00000100,
681 0x8984, 0xffffffff, 0x06000100,
682 0x8a18, 0xffffffff, 0x00000100,
683 0x92a0, 0xffffffff, 0x00000100,
684 0xc380, 0xffffffff, 0x00000100,
685 0x8b28, 0xffffffff, 0x00000100,
686 0x9144, 0xffffffff, 0x00000100,
687 0x8d88, 0xffffffff, 0x00000100,
688 0x8d8c, 0xffffffff, 0x00000100,
689 0x9030, 0xffffffff, 0x00000100,
690 0x9034, 0xffffffff, 0x00000100,
691 0x9038, 0xffffffff, 0x00000100,
692 0x903c, 0xffffffff, 0x00000100,
693 0xad80, 0xffffffff, 0x00000100,
694 0xac54, 0xffffffff, 0x00000100,
695 0x897c, 0xffffffff, 0x06000100,
696 0x9868, 0xffffffff, 0x00000100,
697 0x9510, 0xffffffff, 0x00000100,
698 0xaf04, 0xffffffff, 0x00000100,
699 0xae04, 0xffffffff, 0x00000100,
700 0x949c, 0xffffffff, 0x00000100,
701 0x802c, 0xffffffff, 0xe0000000,
702 0x9160, 0xffffffff, 0x00010000,
703 0x9164, 0xffffffff, 0x00030002,
704 0x9168, 0xffffffff, 0x00040007,
705 0x916c, 0xffffffff, 0x00060005,
706 0x9170, 0xffffffff, 0x00090008,
707 0x9174, 0xffffffff, 0x00020001,
708 0x9178, 0xffffffff, 0x00040003,
709 0x917c, 0xffffffff, 0x00000007,
710 0x9180, 0xffffffff, 0x00060005,
711 0x9184, 0xffffffff, 0x00090008,
712 0x9188, 0xffffffff, 0x00030002,
713 0x918c, 0xffffffff, 0x00050004,
714 0x9190, 0xffffffff, 0x00000008,
715 0x9194, 0xffffffff, 0x00070006,
716 0x9198, 0xffffffff, 0x000a0009,
717 0x919c, 0xffffffff, 0x00040003,
718 0x91a0, 0xffffffff, 0x00060005,
719 0x91a4, 0xffffffff, 0x00000009,
720 0x91a8, 0xffffffff, 0x00080007,
721 0x91ac, 0xffffffff, 0x000b000a,
722 0x91b0, 0xffffffff, 0x00050004,
723 0x91b4, 0xffffffff, 0x00070006,
724 0x91b8, 0xffffffff, 0x0008000b,
725 0x91bc, 0xffffffff, 0x000a0009,
726 0x91c0, 0xffffffff, 0x000d000c,
727 0x9200, 0xffffffff, 0x00090008,
728 0x9204, 0xffffffff, 0x000b000a,
729 0x9208, 0xffffffff, 0x000c000f,
730 0x920c, 0xffffffff, 0x000e000d,
731 0x9210, 0xffffffff, 0x00110010,
732 0x9214, 0xffffffff, 0x000a0009,
733 0x9218, 0xffffffff, 0x000c000b,
734 0x921c, 0xffffffff, 0x0000000f,
735 0x9220, 0xffffffff, 0x000e000d,
736 0x9224, 0xffffffff, 0x00110010,
737 0x9228, 0xffffffff, 0x000b000a,
738 0x922c, 0xffffffff, 0x000d000c,
739 0x9230, 0xffffffff, 0x00000010,
740 0x9234, 0xffffffff, 0x000f000e,
741 0x9238, 0xffffffff, 0x00120011,
742 0x923c, 0xffffffff, 0x000c000b,
743 0x9240, 0xffffffff, 0x000e000d,
744 0x9244, 0xffffffff, 0x00000011,
745 0x9248, 0xffffffff, 0x0010000f,
746 0x924c, 0xffffffff, 0x00130012,
747 0x9250, 0xffffffff, 0x000d000c,
748 0x9254, 0xffffffff, 0x000f000e,
749 0x9258, 0xffffffff, 0x00100013,
750 0x925c, 0xffffffff, 0x00120011,
751 0x9260, 0xffffffff, 0x00150014,
752 0x9150, 0xffffffff, 0x96940200,
753 0x8708, 0xffffffff, 0x00900100,
754 0xc478, 0xffffffff, 0x00000080,
755 0xc404, 0xffffffff, 0x0020003f,
756 0x30, 0xffffffff, 0x0000001c,
757 0x34, 0x000f0000, 0x000f0000,
758 0x160c, 0xffffffff, 0x00000100,
759 0x1024, 0xffffffff, 0x00000100,
760 0x102c, 0x00000101, 0x00000000,
761 0x20a8, 0xffffffff, 0x00000104,
762 0x55e4, 0xff000fff, 0x00000100,
763 0x55e8, 0x00000001, 0x00000001,
764 0x2f50, 0x00000001, 0x00000001,
765 0x30cc, 0xc0000fff, 0x00000104,
766 0xc1e4, 0x00000001, 0x00000001,
767 0xd0c0, 0xfffffff0, 0x00000100,
768 0xd8c0, 0xfffffff0, 0x00000100
771 static const u32 verde_mgcg_cgcg_init[] =
773 0xc400, 0xffffffff, 0xfffffffc,
774 0x802c, 0xffffffff, 0xe0000000,
775 0x9a60, 0xffffffff, 0x00000100,
776 0x92a4, 0xffffffff, 0x00000100,
777 0xc164, 0xffffffff, 0x00000100,
778 0x9774, 0xffffffff, 0x00000100,
779 0x8984, 0xffffffff, 0x06000100,
780 0x8a18, 0xffffffff, 0x00000100,
781 0x92a0, 0xffffffff, 0x00000100,
782 0xc380, 0xffffffff, 0x00000100,
783 0x8b28, 0xffffffff, 0x00000100,
784 0x9144, 0xffffffff, 0x00000100,
785 0x8d88, 0xffffffff, 0x00000100,
786 0x8d8c, 0xffffffff, 0x00000100,
787 0x9030, 0xffffffff, 0x00000100,
788 0x9034, 0xffffffff, 0x00000100,
789 0x9038, 0xffffffff, 0x00000100,
790 0x903c, 0xffffffff, 0x00000100,
791 0xad80, 0xffffffff, 0x00000100,
792 0xac54, 0xffffffff, 0x00000100,
793 0x897c, 0xffffffff, 0x06000100,
794 0x9868, 0xffffffff, 0x00000100,
795 0x9510, 0xffffffff, 0x00000100,
796 0xaf04, 0xffffffff, 0x00000100,
797 0xae04, 0xffffffff, 0x00000100,
798 0x949c, 0xffffffff, 0x00000100,
799 0x802c, 0xffffffff, 0xe0000000,
800 0x9160, 0xffffffff, 0x00010000,
801 0x9164, 0xffffffff, 0x00030002,
802 0x9168, 0xffffffff, 0x00040007,
803 0x916c, 0xffffffff, 0x00060005,
804 0x9170, 0xffffffff, 0x00090008,
805 0x9174, 0xffffffff, 0x00020001,
806 0x9178, 0xffffffff, 0x00040003,
807 0x917c, 0xffffffff, 0x00000007,
808 0x9180, 0xffffffff, 0x00060005,
809 0x9184, 0xffffffff, 0x00090008,
810 0x9188, 0xffffffff, 0x00030002,
811 0x918c, 0xffffffff, 0x00050004,
812 0x9190, 0xffffffff, 0x00000008,
813 0x9194, 0xffffffff, 0x00070006,
814 0x9198, 0xffffffff, 0x000a0009,
815 0x919c, 0xffffffff, 0x00040003,
816 0x91a0, 0xffffffff, 0x00060005,
817 0x91a4, 0xffffffff, 0x00000009,
818 0x91a8, 0xffffffff, 0x00080007,
819 0x91ac, 0xffffffff, 0x000b000a,
820 0x91b0, 0xffffffff, 0x00050004,
821 0x91b4, 0xffffffff, 0x00070006,
822 0x91b8, 0xffffffff, 0x0008000b,
823 0x91bc, 0xffffffff, 0x000a0009,
824 0x91c0, 0xffffffff, 0x000d000c,
825 0x9200, 0xffffffff, 0x00090008,
826 0x9204, 0xffffffff, 0x000b000a,
827 0x9208, 0xffffffff, 0x000c000f,
828 0x920c, 0xffffffff, 0x000e000d,
829 0x9210, 0xffffffff, 0x00110010,
830 0x9214, 0xffffffff, 0x000a0009,
831 0x9218, 0xffffffff, 0x000c000b,
832 0x921c, 0xffffffff, 0x0000000f,
833 0x9220, 0xffffffff, 0x000e000d,
834 0x9224, 0xffffffff, 0x00110010,
835 0x9228, 0xffffffff, 0x000b000a,
836 0x922c, 0xffffffff, 0x000d000c,
837 0x9230, 0xffffffff, 0x00000010,
838 0x9234, 0xffffffff, 0x000f000e,
839 0x9238, 0xffffffff, 0x00120011,
840 0x923c, 0xffffffff, 0x000c000b,
841 0x9240, 0xffffffff, 0x000e000d,
842 0x9244, 0xffffffff, 0x00000011,
843 0x9248, 0xffffffff, 0x0010000f,
844 0x924c, 0xffffffff, 0x00130012,
845 0x9250, 0xffffffff, 0x000d000c,
846 0x9254, 0xffffffff, 0x000f000e,
847 0x9258, 0xffffffff, 0x00100013,
848 0x925c, 0xffffffff, 0x00120011,
849 0x9260, 0xffffffff, 0x00150014,
850 0x9150, 0xffffffff, 0x96940200,
851 0x8708, 0xffffffff, 0x00900100,
852 0xc478, 0xffffffff, 0x00000080,
853 0xc404, 0xffffffff, 0x0020003f,
854 0x30, 0xffffffff, 0x0000001c,
855 0x34, 0x000f0000, 0x000f0000,
856 0x160c, 0xffffffff, 0x00000100,
857 0x1024, 0xffffffff, 0x00000100,
858 0x102c, 0x00000101, 0x00000000,
859 0x20a8, 0xffffffff, 0x00000104,
860 0x264c, 0x000c0000, 0x000c0000,
861 0x2648, 0x000c0000, 0x000c0000,
862 0x55e4, 0xff000fff, 0x00000100,
863 0x55e8, 0x00000001, 0x00000001,
864 0x2f50, 0x00000001, 0x00000001,
865 0x30cc, 0xc0000fff, 0x00000104,
866 0xc1e4, 0x00000001, 0x00000001,
867 0xd0c0, 0xfffffff0, 0x00000100,
868 0xd8c0, 0xfffffff0, 0x00000100
871 static const u32 oland_mgcg_cgcg_init[] =
873 0xc400, 0xffffffff, 0xfffffffc,
874 0x802c, 0xffffffff, 0xe0000000,
875 0x9a60, 0xffffffff, 0x00000100,
876 0x92a4, 0xffffffff, 0x00000100,
877 0xc164, 0xffffffff, 0x00000100,
878 0x9774, 0xffffffff, 0x00000100,
879 0x8984, 0xffffffff, 0x06000100,
880 0x8a18, 0xffffffff, 0x00000100,
881 0x92a0, 0xffffffff, 0x00000100,
882 0xc380, 0xffffffff, 0x00000100,
883 0x8b28, 0xffffffff, 0x00000100,
884 0x9144, 0xffffffff, 0x00000100,
885 0x8d88, 0xffffffff, 0x00000100,
886 0x8d8c, 0xffffffff, 0x00000100,
887 0x9030, 0xffffffff, 0x00000100,
888 0x9034, 0xffffffff, 0x00000100,
889 0x9038, 0xffffffff, 0x00000100,
890 0x903c, 0xffffffff, 0x00000100,
891 0xad80, 0xffffffff, 0x00000100,
892 0xac54, 0xffffffff, 0x00000100,
893 0x897c, 0xffffffff, 0x06000100,
894 0x9868, 0xffffffff, 0x00000100,
895 0x9510, 0xffffffff, 0x00000100,
896 0xaf04, 0xffffffff, 0x00000100,
897 0xae04, 0xffffffff, 0x00000100,
898 0x949c, 0xffffffff, 0x00000100,
899 0x802c, 0xffffffff, 0xe0000000,
900 0x9160, 0xffffffff, 0x00010000,
901 0x9164, 0xffffffff, 0x00030002,
902 0x9168, 0xffffffff, 0x00040007,
903 0x916c, 0xffffffff, 0x00060005,
904 0x9170, 0xffffffff, 0x00090008,
905 0x9174, 0xffffffff, 0x00020001,
906 0x9178, 0xffffffff, 0x00040003,
907 0x917c, 0xffffffff, 0x00000007,
908 0x9180, 0xffffffff, 0x00060005,
909 0x9184, 0xffffffff, 0x00090008,
910 0x9188, 0xffffffff, 0x00030002,
911 0x918c, 0xffffffff, 0x00050004,
912 0x9190, 0xffffffff, 0x00000008,
913 0x9194, 0xffffffff, 0x00070006,
914 0x9198, 0xffffffff, 0x000a0009,
915 0x919c, 0xffffffff, 0x00040003,
916 0x91a0, 0xffffffff, 0x00060005,
917 0x91a4, 0xffffffff, 0x00000009,
918 0x91a8, 0xffffffff, 0x00080007,
919 0x91ac, 0xffffffff, 0x000b000a,
920 0x91b0, 0xffffffff, 0x00050004,
921 0x91b4, 0xffffffff, 0x00070006,
922 0x91b8, 0xffffffff, 0x0008000b,
923 0x91bc, 0xffffffff, 0x000a0009,
924 0x91c0, 0xffffffff, 0x000d000c,
925 0x91c4, 0xffffffff, 0x00060005,
926 0x91c8, 0xffffffff, 0x00080007,
927 0x91cc, 0xffffffff, 0x0000000b,
928 0x91d0, 0xffffffff, 0x000a0009,
929 0x91d4, 0xffffffff, 0x000d000c,
930 0x9150, 0xffffffff, 0x96940200,
931 0x8708, 0xffffffff, 0x00900100,
932 0xc478, 0xffffffff, 0x00000080,
933 0xc404, 0xffffffff, 0x0020003f,
934 0x30, 0xffffffff, 0x0000001c,
935 0x34, 0x000f0000, 0x000f0000,
936 0x160c, 0xffffffff, 0x00000100,
937 0x1024, 0xffffffff, 0x00000100,
938 0x102c, 0x00000101, 0x00000000,
939 0x20a8, 0xffffffff, 0x00000104,
940 0x264c, 0x000c0000, 0x000c0000,
941 0x2648, 0x000c0000, 0x000c0000,
942 0x55e4, 0xff000fff, 0x00000100,
943 0x55e8, 0x00000001, 0x00000001,
944 0x2f50, 0x00000001, 0x00000001,
945 0x30cc, 0xc0000fff, 0x00000104,
946 0xc1e4, 0x00000001, 0x00000001,
947 0xd0c0, 0xfffffff0, 0x00000100,
948 0xd8c0, 0xfffffff0, 0x00000100
951 static const u32 hainan_mgcg_cgcg_init[] =
953 0xc400, 0xffffffff, 0xfffffffc,
954 0x802c, 0xffffffff, 0xe0000000,
955 0x9a60, 0xffffffff, 0x00000100,
956 0x92a4, 0xffffffff, 0x00000100,
957 0xc164, 0xffffffff, 0x00000100,
958 0x9774, 0xffffffff, 0x00000100,
959 0x8984, 0xffffffff, 0x06000100,
960 0x8a18, 0xffffffff, 0x00000100,
961 0x92a0, 0xffffffff, 0x00000100,
962 0xc380, 0xffffffff, 0x00000100,
963 0x8b28, 0xffffffff, 0x00000100,
964 0x9144, 0xffffffff, 0x00000100,
965 0x8d88, 0xffffffff, 0x00000100,
966 0x8d8c, 0xffffffff, 0x00000100,
967 0x9030, 0xffffffff, 0x00000100,
968 0x9034, 0xffffffff, 0x00000100,
969 0x9038, 0xffffffff, 0x00000100,
970 0x903c, 0xffffffff, 0x00000100,
971 0xad80, 0xffffffff, 0x00000100,
972 0xac54, 0xffffffff, 0x00000100,
973 0x897c, 0xffffffff, 0x06000100,
974 0x9868, 0xffffffff, 0x00000100,
975 0x9510, 0xffffffff, 0x00000100,
976 0xaf04, 0xffffffff, 0x00000100,
977 0xae04, 0xffffffff, 0x00000100,
978 0x949c, 0xffffffff, 0x00000100,
979 0x802c, 0xffffffff, 0xe0000000,
980 0x9160, 0xffffffff, 0x00010000,
981 0x9164, 0xffffffff, 0x00030002,
982 0x9168, 0xffffffff, 0x00040007,
983 0x916c, 0xffffffff, 0x00060005,
984 0x9170, 0xffffffff, 0x00090008,
985 0x9174, 0xffffffff, 0x00020001,
986 0x9178, 0xffffffff, 0x00040003,
987 0x917c, 0xffffffff, 0x00000007,
988 0x9180, 0xffffffff, 0x00060005,
989 0x9184, 0xffffffff, 0x00090008,
990 0x9188, 0xffffffff, 0x00030002,
991 0x918c, 0xffffffff, 0x00050004,
992 0x9190, 0xffffffff, 0x00000008,
993 0x9194, 0xffffffff, 0x00070006,
994 0x9198, 0xffffffff, 0x000a0009,
995 0x919c, 0xffffffff, 0x00040003,
996 0x91a0, 0xffffffff, 0x00060005,
997 0x91a4, 0xffffffff, 0x00000009,
998 0x91a8, 0xffffffff, 0x00080007,
999 0x91ac, 0xffffffff, 0x000b000a,
1000 0x91b0, 0xffffffff, 0x00050004,
1001 0x91b4, 0xffffffff, 0x00070006,
1002 0x91b8, 0xffffffff, 0x0008000b,
1003 0x91bc, 0xffffffff, 0x000a0009,
1004 0x91c0, 0xffffffff, 0x000d000c,
1005 0x91c4, 0xffffffff, 0x00060005,
1006 0x91c8, 0xffffffff, 0x00080007,
1007 0x91cc, 0xffffffff, 0x0000000b,
1008 0x91d0, 0xffffffff, 0x000a0009,
1009 0x91d4, 0xffffffff, 0x000d000c,
1010 0x9150, 0xffffffff, 0x96940200,
1011 0x8708, 0xffffffff, 0x00900100,
1012 0xc478, 0xffffffff, 0x00000080,
1013 0xc404, 0xffffffff, 0x0020003f,
1014 0x30, 0xffffffff, 0x0000001c,
1015 0x34, 0x000f0000, 0x000f0000,
1016 0x160c, 0xffffffff, 0x00000100,
1017 0x1024, 0xffffffff, 0x00000100,
1018 0x20a8, 0xffffffff, 0x00000104,
1019 0x264c, 0x000c0000, 0x000c0000,
1020 0x2648, 0x000c0000, 0x000c0000,
1021 0x2f50, 0x00000001, 0x00000001,
1022 0x30cc, 0xc0000fff, 0x00000104,
1023 0xc1e4, 0x00000001, 0x00000001,
1024 0xd0c0, 0xfffffff0, 0x00000100,
1025 0xd8c0, 0xfffffff0, 0x00000100
1028 static u32 verde_pg_init[] =
1030 0x353c, 0xffffffff, 0x40000,
1031 0x3538, 0xffffffff, 0x200010ff,
1032 0x353c, 0xffffffff, 0x0,
1033 0x353c, 0xffffffff, 0x0,
1034 0x353c, 0xffffffff, 0x0,
1035 0x353c, 0xffffffff, 0x0,
1036 0x353c, 0xffffffff, 0x0,
1037 0x353c, 0xffffffff, 0x7007,
1038 0x3538, 0xffffffff, 0x300010ff,
1039 0x353c, 0xffffffff, 0x0,
1040 0x353c, 0xffffffff, 0x0,
1041 0x353c, 0xffffffff, 0x0,
1042 0x353c, 0xffffffff, 0x0,
1043 0x353c, 0xffffffff, 0x0,
1044 0x353c, 0xffffffff, 0x400000,
1045 0x3538, 0xffffffff, 0x100010ff,
1046 0x353c, 0xffffffff, 0x0,
1047 0x353c, 0xffffffff, 0x0,
1048 0x353c, 0xffffffff, 0x0,
1049 0x353c, 0xffffffff, 0x0,
1050 0x353c, 0xffffffff, 0x0,
1051 0x353c, 0xffffffff, 0x120200,
1052 0x3538, 0xffffffff, 0x500010ff,
1053 0x353c, 0xffffffff, 0x0,
1054 0x353c, 0xffffffff, 0x0,
1055 0x353c, 0xffffffff, 0x0,
1056 0x353c, 0xffffffff, 0x0,
1057 0x353c, 0xffffffff, 0x0,
1058 0x353c, 0xffffffff, 0x1e1e16,
1059 0x3538, 0xffffffff, 0x600010ff,
1060 0x353c, 0xffffffff, 0x0,
1061 0x353c, 0xffffffff, 0x0,
1062 0x353c, 0xffffffff, 0x0,
1063 0x353c, 0xffffffff, 0x0,
1064 0x353c, 0xffffffff, 0x0,
1065 0x353c, 0xffffffff, 0x171f1e,
1066 0x3538, 0xffffffff, 0x700010ff,
1067 0x353c, 0xffffffff, 0x0,
1068 0x353c, 0xffffffff, 0x0,
1069 0x353c, 0xffffffff, 0x0,
1070 0x353c, 0xffffffff, 0x0,
1071 0x353c, 0xffffffff, 0x0,
1072 0x353c, 0xffffffff, 0x0,
1073 0x3538, 0xffffffff, 0x9ff,
1074 0x3500, 0xffffffff, 0x0,
1075 0x3504, 0xffffffff, 0x10000800,
1076 0x3504, 0xffffffff, 0xf,
1077 0x3504, 0xffffffff, 0xf,
1078 0x3500, 0xffffffff, 0x4,
1079 0x3504, 0xffffffff, 0x1000051e,
1080 0x3504, 0xffffffff, 0xffff,
1081 0x3504, 0xffffffff, 0xffff,
1082 0x3500, 0xffffffff, 0x8,
1083 0x3504, 0xffffffff, 0x80500,
1084 0x3500, 0xffffffff, 0x12,
1085 0x3504, 0xffffffff, 0x9050c,
1086 0x3500, 0xffffffff, 0x1d,
1087 0x3504, 0xffffffff, 0xb052c,
1088 0x3500, 0xffffffff, 0x2a,
1089 0x3504, 0xffffffff, 0x1053e,
1090 0x3500, 0xffffffff, 0x2d,
1091 0x3504, 0xffffffff, 0x10546,
1092 0x3500, 0xffffffff, 0x30,
1093 0x3504, 0xffffffff, 0xa054e,
1094 0x3500, 0xffffffff, 0x3c,
1095 0x3504, 0xffffffff, 0x1055f,
1096 0x3500, 0xffffffff, 0x3f,
1097 0x3504, 0xffffffff, 0x10567,
1098 0x3500, 0xffffffff, 0x42,
1099 0x3504, 0xffffffff, 0x1056f,
1100 0x3500, 0xffffffff, 0x45,
1101 0x3504, 0xffffffff, 0x10572,
1102 0x3500, 0xffffffff, 0x48,
1103 0x3504, 0xffffffff, 0x20575,
1104 0x3500, 0xffffffff, 0x4c,
1105 0x3504, 0xffffffff, 0x190801,
1106 0x3500, 0xffffffff, 0x67,
1107 0x3504, 0xffffffff, 0x1082a,
1108 0x3500, 0xffffffff, 0x6a,
1109 0x3504, 0xffffffff, 0x1b082d,
1110 0x3500, 0xffffffff, 0x87,
1111 0x3504, 0xffffffff, 0x310851,
1112 0x3500, 0xffffffff, 0xba,
1113 0x3504, 0xffffffff, 0x891,
1114 0x3500, 0xffffffff, 0xbc,
1115 0x3504, 0xffffffff, 0x893,
1116 0x3500, 0xffffffff, 0xbe,
1117 0x3504, 0xffffffff, 0x20895,
1118 0x3500, 0xffffffff, 0xc2,
1119 0x3504, 0xffffffff, 0x20899,
1120 0x3500, 0xffffffff, 0xc6,
1121 0x3504, 0xffffffff, 0x2089d,
1122 0x3500, 0xffffffff, 0xca,
1123 0x3504, 0xffffffff, 0x8a1,
1124 0x3500, 0xffffffff, 0xcc,
1125 0x3504, 0xffffffff, 0x8a3,
1126 0x3500, 0xffffffff, 0xce,
1127 0x3504, 0xffffffff, 0x308a5,
1128 0x3500, 0xffffffff, 0xd3,
1129 0x3504, 0xffffffff, 0x6d08cd,
1130 0x3500, 0xffffffff, 0x142,
1131 0x3504, 0xffffffff, 0x2000095a,
1132 0x3504, 0xffffffff, 0x1,
1133 0x3500, 0xffffffff, 0x144,
1134 0x3504, 0xffffffff, 0x301f095b,
1135 0x3500, 0xffffffff, 0x165,
1136 0x3504, 0xffffffff, 0xc094d,
1137 0x3500, 0xffffffff, 0x173,
1138 0x3504, 0xffffffff, 0xf096d,
1139 0x3500, 0xffffffff, 0x184,
1140 0x3504, 0xffffffff, 0x15097f,
1141 0x3500, 0xffffffff, 0x19b,
1142 0x3504, 0xffffffff, 0xc0998,
1143 0x3500, 0xffffffff, 0x1a9,
1144 0x3504, 0xffffffff, 0x409a7,
1145 0x3500, 0xffffffff, 0x1af,
1146 0x3504, 0xffffffff, 0xcdc,
1147 0x3500, 0xffffffff, 0x1b1,
1148 0x3504, 0xffffffff, 0x800,
1149 0x3508, 0xffffffff, 0x6c9b2000,
1150 0x3510, 0xfc00, 0x2000,
1151 0x3544, 0xffffffff, 0xfc0,
1152 0x28d4, 0x00000100, 0x100
1155 static void si_init_golden_registers(struct radeon_device *rdev)
1157 switch (rdev->family) {
1159 radeon_program_register_sequence(rdev,
1160 tahiti_golden_registers,
1161 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1162 radeon_program_register_sequence(rdev,
1163 tahiti_golden_rlc_registers,
1164 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1165 radeon_program_register_sequence(rdev,
1166 tahiti_mgcg_cgcg_init,
1167 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1168 radeon_program_register_sequence(rdev,
1169 tahiti_golden_registers2,
1170 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1173 radeon_program_register_sequence(rdev,
1174 pitcairn_golden_registers,
1175 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1176 radeon_program_register_sequence(rdev,
1177 pitcairn_golden_rlc_registers,
1178 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1179 radeon_program_register_sequence(rdev,
1180 pitcairn_mgcg_cgcg_init,
1181 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1184 radeon_program_register_sequence(rdev,
1185 verde_golden_registers,
1186 (const u32)ARRAY_SIZE(verde_golden_registers));
1187 radeon_program_register_sequence(rdev,
1188 verde_golden_rlc_registers,
1189 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1190 radeon_program_register_sequence(rdev,
1191 verde_mgcg_cgcg_init,
1192 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1193 radeon_program_register_sequence(rdev,
1195 (const u32)ARRAY_SIZE(verde_pg_init));
1198 radeon_program_register_sequence(rdev,
1199 oland_golden_registers,
1200 (const u32)ARRAY_SIZE(oland_golden_registers));
1201 radeon_program_register_sequence(rdev,
1202 oland_golden_rlc_registers,
1203 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1204 radeon_program_register_sequence(rdev,
1205 oland_mgcg_cgcg_init,
1206 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1209 radeon_program_register_sequence(rdev,
1210 hainan_golden_registers,
1211 (const u32)ARRAY_SIZE(hainan_golden_registers));
1212 radeon_program_register_sequence(rdev,
1213 hainan_golden_registers2,
1214 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1215 radeon_program_register_sequence(rdev,
1216 hainan_mgcg_cgcg_init,
1217 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1224 #define PCIE_BUS_CLK 10000
1225 #define TCLK (PCIE_BUS_CLK / 10)
1228 * si_get_xclk - get the xclk
1230 * @rdev: radeon_device pointer
1232 * Returns the reference clock used by the gfx engine
1235 u32 si_get_xclk(struct radeon_device *rdev)
1237 u32 reference_clock = rdev->clock.spll.reference_freq;
1240 tmp = RREG32(CG_CLKPIN_CNTL_2);
1241 if (tmp & MUX_TCLK_TO_XCLK)
1244 tmp = RREG32(CG_CLKPIN_CNTL);
1245 if (tmp & XTALIN_DIVIDE)
1246 return reference_clock / 4;
1248 return reference_clock;
1251 /* get temperature in millidegrees */
1252 int si_get_temp(struct radeon_device *rdev)
1255 int actual_temp = 0;
1257 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1263 actual_temp = temp & 0x1ff;
1265 actual_temp = (actual_temp * 1000);
1270 #define TAHITI_IO_MC_REGS_SIZE 36
1272 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1273 {0x0000006f, 0x03044000},
1274 {0x00000070, 0x0480c018},
1275 {0x00000071, 0x00000040},
1276 {0x00000072, 0x01000000},
1277 {0x00000074, 0x000000ff},
1278 {0x00000075, 0x00143400},
1279 {0x00000076, 0x08ec0800},
1280 {0x00000077, 0x040000cc},
1281 {0x00000079, 0x00000000},
1282 {0x0000007a, 0x21000409},
1283 {0x0000007c, 0x00000000},
1284 {0x0000007d, 0xe8000000},
1285 {0x0000007e, 0x044408a8},
1286 {0x0000007f, 0x00000003},
1287 {0x00000080, 0x00000000},
1288 {0x00000081, 0x01000000},
1289 {0x00000082, 0x02000000},
1290 {0x00000083, 0x00000000},
1291 {0x00000084, 0xe3f3e4f4},
1292 {0x00000085, 0x00052024},
1293 {0x00000087, 0x00000000},
1294 {0x00000088, 0x66036603},
1295 {0x00000089, 0x01000000},
1296 {0x0000008b, 0x1c0a0000},
1297 {0x0000008c, 0xff010000},
1298 {0x0000008e, 0xffffefff},
1299 {0x0000008f, 0xfff3efff},
1300 {0x00000090, 0xfff3efbf},
1301 {0x00000094, 0x00101101},
1302 {0x00000095, 0x00000fff},
1303 {0x00000096, 0x00116fff},
1304 {0x00000097, 0x60010000},
1305 {0x00000098, 0x10010000},
1306 {0x00000099, 0x00006000},
1307 {0x0000009a, 0x00001000},
1308 {0x0000009f, 0x00a77400}
1311 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1312 {0x0000006f, 0x03044000},
1313 {0x00000070, 0x0480c018},
1314 {0x00000071, 0x00000040},
1315 {0x00000072, 0x01000000},
1316 {0x00000074, 0x000000ff},
1317 {0x00000075, 0x00143400},
1318 {0x00000076, 0x08ec0800},
1319 {0x00000077, 0x040000cc},
1320 {0x00000079, 0x00000000},
1321 {0x0000007a, 0x21000409},
1322 {0x0000007c, 0x00000000},
1323 {0x0000007d, 0xe8000000},
1324 {0x0000007e, 0x044408a8},
1325 {0x0000007f, 0x00000003},
1326 {0x00000080, 0x00000000},
1327 {0x00000081, 0x01000000},
1328 {0x00000082, 0x02000000},
1329 {0x00000083, 0x00000000},
1330 {0x00000084, 0xe3f3e4f4},
1331 {0x00000085, 0x00052024},
1332 {0x00000087, 0x00000000},
1333 {0x00000088, 0x66036603},
1334 {0x00000089, 0x01000000},
1335 {0x0000008b, 0x1c0a0000},
1336 {0x0000008c, 0xff010000},
1337 {0x0000008e, 0xffffefff},
1338 {0x0000008f, 0xfff3efff},
1339 {0x00000090, 0xfff3efbf},
1340 {0x00000094, 0x00101101},
1341 {0x00000095, 0x00000fff},
1342 {0x00000096, 0x00116fff},
1343 {0x00000097, 0x60010000},
1344 {0x00000098, 0x10010000},
1345 {0x00000099, 0x00006000},
1346 {0x0000009a, 0x00001000},
1347 {0x0000009f, 0x00a47400}
1350 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1351 {0x0000006f, 0x03044000},
1352 {0x00000070, 0x0480c018},
1353 {0x00000071, 0x00000040},
1354 {0x00000072, 0x01000000},
1355 {0x00000074, 0x000000ff},
1356 {0x00000075, 0x00143400},
1357 {0x00000076, 0x08ec0800},
1358 {0x00000077, 0x040000cc},
1359 {0x00000079, 0x00000000},
1360 {0x0000007a, 0x21000409},
1361 {0x0000007c, 0x00000000},
1362 {0x0000007d, 0xe8000000},
1363 {0x0000007e, 0x044408a8},
1364 {0x0000007f, 0x00000003},
1365 {0x00000080, 0x00000000},
1366 {0x00000081, 0x01000000},
1367 {0x00000082, 0x02000000},
1368 {0x00000083, 0x00000000},
1369 {0x00000084, 0xe3f3e4f4},
1370 {0x00000085, 0x00052024},
1371 {0x00000087, 0x00000000},
1372 {0x00000088, 0x66036603},
1373 {0x00000089, 0x01000000},
1374 {0x0000008b, 0x1c0a0000},
1375 {0x0000008c, 0xff010000},
1376 {0x0000008e, 0xffffefff},
1377 {0x0000008f, 0xfff3efff},
1378 {0x00000090, 0xfff3efbf},
1379 {0x00000094, 0x00101101},
1380 {0x00000095, 0x00000fff},
1381 {0x00000096, 0x00116fff},
1382 {0x00000097, 0x60010000},
1383 {0x00000098, 0x10010000},
1384 {0x00000099, 0x00006000},
1385 {0x0000009a, 0x00001000},
1386 {0x0000009f, 0x00a37400}
1389 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1390 {0x0000006f, 0x03044000},
1391 {0x00000070, 0x0480c018},
1392 {0x00000071, 0x00000040},
1393 {0x00000072, 0x01000000},
1394 {0x00000074, 0x000000ff},
1395 {0x00000075, 0x00143400},
1396 {0x00000076, 0x08ec0800},
1397 {0x00000077, 0x040000cc},
1398 {0x00000079, 0x00000000},
1399 {0x0000007a, 0x21000409},
1400 {0x0000007c, 0x00000000},
1401 {0x0000007d, 0xe8000000},
1402 {0x0000007e, 0x044408a8},
1403 {0x0000007f, 0x00000003},
1404 {0x00000080, 0x00000000},
1405 {0x00000081, 0x01000000},
1406 {0x00000082, 0x02000000},
1407 {0x00000083, 0x00000000},
1408 {0x00000084, 0xe3f3e4f4},
1409 {0x00000085, 0x00052024},
1410 {0x00000087, 0x00000000},
1411 {0x00000088, 0x66036603},
1412 {0x00000089, 0x01000000},
1413 {0x0000008b, 0x1c0a0000},
1414 {0x0000008c, 0xff010000},
1415 {0x0000008e, 0xffffefff},
1416 {0x0000008f, 0xfff3efff},
1417 {0x00000090, 0xfff3efbf},
1418 {0x00000094, 0x00101101},
1419 {0x00000095, 0x00000fff},
1420 {0x00000096, 0x00116fff},
1421 {0x00000097, 0x60010000},
1422 {0x00000098, 0x10010000},
1423 {0x00000099, 0x00006000},
1424 {0x0000009a, 0x00001000},
1425 {0x0000009f, 0x00a17730}
1428 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1429 {0x0000006f, 0x03044000},
1430 {0x00000070, 0x0480c018},
1431 {0x00000071, 0x00000040},
1432 {0x00000072, 0x01000000},
1433 {0x00000074, 0x000000ff},
1434 {0x00000075, 0x00143400},
1435 {0x00000076, 0x08ec0800},
1436 {0x00000077, 0x040000cc},
1437 {0x00000079, 0x00000000},
1438 {0x0000007a, 0x21000409},
1439 {0x0000007c, 0x00000000},
1440 {0x0000007d, 0xe8000000},
1441 {0x0000007e, 0x044408a8},
1442 {0x0000007f, 0x00000003},
1443 {0x00000080, 0x00000000},
1444 {0x00000081, 0x01000000},
1445 {0x00000082, 0x02000000},
1446 {0x00000083, 0x00000000},
1447 {0x00000084, 0xe3f3e4f4},
1448 {0x00000085, 0x00052024},
1449 {0x00000087, 0x00000000},
1450 {0x00000088, 0x66036603},
1451 {0x00000089, 0x01000000},
1452 {0x0000008b, 0x1c0a0000},
1453 {0x0000008c, 0xff010000},
1454 {0x0000008e, 0xffffefff},
1455 {0x0000008f, 0xfff3efff},
1456 {0x00000090, 0xfff3efbf},
1457 {0x00000094, 0x00101101},
1458 {0x00000095, 0x00000fff},
1459 {0x00000096, 0x00116fff},
1460 {0x00000097, 0x60010000},
1461 {0x00000098, 0x10010000},
1462 {0x00000099, 0x00006000},
1463 {0x0000009a, 0x00001000},
1464 {0x0000009f, 0x00a07730}
1468 static int si_mc_load_microcode(struct radeon_device *rdev)
1470 const __be32 *fw_data;
1471 u32 running, blackout = 0;
1473 int i, ucode_size, regs_size;
1478 switch (rdev->family) {
1480 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1481 ucode_size = SI_MC_UCODE_SIZE;
1482 regs_size = TAHITI_IO_MC_REGS_SIZE;
1485 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1486 ucode_size = SI_MC_UCODE_SIZE;
1487 regs_size = TAHITI_IO_MC_REGS_SIZE;
1491 io_mc_regs = (u32 *)&verde_io_mc_regs;
1492 ucode_size = SI_MC_UCODE_SIZE;
1493 regs_size = TAHITI_IO_MC_REGS_SIZE;
1496 io_mc_regs = (u32 *)&oland_io_mc_regs;
1497 ucode_size = OLAND_MC_UCODE_SIZE;
1498 regs_size = TAHITI_IO_MC_REGS_SIZE;
1501 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1502 ucode_size = OLAND_MC_UCODE_SIZE;
1503 regs_size = TAHITI_IO_MC_REGS_SIZE;
1507 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1511 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1512 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1515 /* reset the engine and set to writable */
1516 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1517 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1519 /* load mc io regs */
1520 for (i = 0; i < regs_size; i++) {
1521 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1522 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1524 /* load the MC ucode */
1525 fw_data = (const __be32 *)rdev->mc_fw->data;
1526 for (i = 0; i < ucode_size; i++)
1527 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1529 /* put the engine back into the active state */
1530 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1531 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1532 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1534 /* wait for training to complete */
1535 for (i = 0; i < rdev->usec_timeout; i++) {
1536 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1540 for (i = 0; i < rdev->usec_timeout; i++) {
1541 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1547 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1553 static int si_init_microcode(struct radeon_device *rdev)
1555 const char *chip_name;
1556 const char *rlc_chip_name;
1557 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1558 size_t smc_req_size;
1564 switch (rdev->family) {
1566 chip_name = "TAHITI";
1567 rlc_chip_name = "TAHITI";
1568 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1569 me_req_size = SI_PM4_UCODE_SIZE * 4;
1570 ce_req_size = SI_CE_UCODE_SIZE * 4;
1571 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1572 mc_req_size = SI_MC_UCODE_SIZE * 4;
1573 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1576 chip_name = "PITCAIRN";
1577 rlc_chip_name = "PITCAIRN";
1578 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1579 me_req_size = SI_PM4_UCODE_SIZE * 4;
1580 ce_req_size = SI_CE_UCODE_SIZE * 4;
1581 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1582 mc_req_size = SI_MC_UCODE_SIZE * 4;
1583 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1586 chip_name = "VERDE";
1587 rlc_chip_name = "VERDE";
1588 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1589 me_req_size = SI_PM4_UCODE_SIZE * 4;
1590 ce_req_size = SI_CE_UCODE_SIZE * 4;
1591 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1592 mc_req_size = SI_MC_UCODE_SIZE * 4;
1593 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1596 chip_name = "OLAND";
1597 rlc_chip_name = "OLAND";
1598 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1599 me_req_size = SI_PM4_UCODE_SIZE * 4;
1600 ce_req_size = SI_CE_UCODE_SIZE * 4;
1601 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1602 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1603 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1606 chip_name = "HAINAN";
1607 rlc_chip_name = "HAINAN";
1608 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1609 me_req_size = SI_PM4_UCODE_SIZE * 4;
1610 ce_req_size = SI_CE_UCODE_SIZE * 4;
1611 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1612 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1613 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1618 DRM_INFO("Loading %s Microcode\n", chip_name);
1620 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1621 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1624 if (rdev->pfp_fw->size != pfp_req_size) {
1626 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1627 rdev->pfp_fw->size, fw_name);
1632 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1633 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1636 if (rdev->me_fw->size != me_req_size) {
1638 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1639 rdev->me_fw->size, fw_name);
1643 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1644 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1647 if (rdev->ce_fw->size != ce_req_size) {
1649 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1650 rdev->ce_fw->size, fw_name);
1654 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1655 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1658 if (rdev->rlc_fw->size != rlc_req_size) {
1660 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1661 rdev->rlc_fw->size, fw_name);
1665 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1666 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1669 if (rdev->mc_fw->size != mc_req_size) {
1671 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1672 rdev->mc_fw->size, fw_name);
1676 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1677 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1680 "smc: error loading firmware \"%s\"\n",
1682 release_firmware(rdev->smc_fw);
1683 rdev->smc_fw = NULL;
1685 } else if (rdev->smc_fw->size != smc_req_size) {
1687 "si_smc: Bogus length %zu in firmware \"%s\"\n",
1688 rdev->smc_fw->size, fw_name);
1696 "si_cp: Failed to load firmware \"%s\"\n",
1698 release_firmware(rdev->pfp_fw);
1699 rdev->pfp_fw = NULL;
1700 release_firmware(rdev->me_fw);
1702 release_firmware(rdev->ce_fw);
1704 release_firmware(rdev->rlc_fw);
1705 rdev->rlc_fw = NULL;
1706 release_firmware(rdev->mc_fw);
1708 release_firmware(rdev->smc_fw);
1709 rdev->smc_fw = NULL;
1714 /* watermark setup */
1715 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1716 struct radeon_crtc *radeon_crtc,
1717 struct drm_display_mode *mode,
1718 struct drm_display_mode *other_mode)
1720 u32 tmp, buffer_alloc, i;
1721 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1724 * There are 3 line buffers, each one shared by 2 display controllers.
1725 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1726 * the display controllers. The paritioning is done via one of four
1727 * preset allocations specified in bits 21:20:
1729 * 2 - whole lb, other crtc must be disabled
1731 /* this can get tricky if we have two large displays on a paired group
1732 * of crtcs. Ideally for multiple large displays we'd assign them to
1733 * non-linked crtcs for maximum line buffer allocation.
1735 if (radeon_crtc->base.enabled && mode) {
1740 tmp = 2; /* whole */
1748 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1749 DC_LB_MEMORY_CONFIG(tmp));
1751 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1752 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1753 for (i = 0; i < rdev->usec_timeout; i++) {
1754 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1755 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1760 if (radeon_crtc->base.enabled && mode) {
1770 /* controller not enabled, so no lb used */
1774 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1776 u32 tmp = RREG32(MC_SHARED_CHMAP);
1778 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1801 struct dce6_wm_params {
1802 u32 dram_channels; /* number of dram channels */
1803 u32 yclk; /* bandwidth per dram data pin in kHz */
1804 u32 sclk; /* engine clock in kHz */
1805 u32 disp_clk; /* display clock in kHz */
1806 u32 src_width; /* viewport width */
1807 u32 active_time; /* active display time in ns */
1808 u32 blank_time; /* blank time in ns */
1809 bool interlaced; /* mode is interlaced */
1810 fixed20_12 vsc; /* vertical scale ratio */
1811 u32 num_heads; /* number of active crtcs */
1812 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1813 u32 lb_size; /* line buffer allocated to pipe */
1814 u32 vtaps; /* vertical scaler taps */
1817 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1819 /* Calculate raw DRAM Bandwidth */
1820 fixed20_12 dram_efficiency; /* 0.7 */
1821 fixed20_12 yclk, dram_channels, bandwidth;
1824 a.full = dfixed_const(1000);
1825 yclk.full = dfixed_const(wm->yclk);
1826 yclk.full = dfixed_div(yclk, a);
1827 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1828 a.full = dfixed_const(10);
1829 dram_efficiency.full = dfixed_const(7);
1830 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1831 bandwidth.full = dfixed_mul(dram_channels, yclk);
1832 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1834 return dfixed_trunc(bandwidth);
1837 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1839 /* Calculate DRAM Bandwidth and the part allocated to display. */
1840 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1841 fixed20_12 yclk, dram_channels, bandwidth;
1844 a.full = dfixed_const(1000);
1845 yclk.full = dfixed_const(wm->yclk);
1846 yclk.full = dfixed_div(yclk, a);
1847 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1848 a.full = dfixed_const(10);
1849 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1850 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1851 bandwidth.full = dfixed_mul(dram_channels, yclk);
1852 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1854 return dfixed_trunc(bandwidth);
1857 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1859 /* Calculate the display Data return Bandwidth */
1860 fixed20_12 return_efficiency; /* 0.8 */
1861 fixed20_12 sclk, bandwidth;
1864 a.full = dfixed_const(1000);
1865 sclk.full = dfixed_const(wm->sclk);
1866 sclk.full = dfixed_div(sclk, a);
1867 a.full = dfixed_const(10);
1868 return_efficiency.full = dfixed_const(8);
1869 return_efficiency.full = dfixed_div(return_efficiency, a);
1870 a.full = dfixed_const(32);
1871 bandwidth.full = dfixed_mul(a, sclk);
1872 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1874 return dfixed_trunc(bandwidth);
1877 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1882 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1884 /* Calculate the DMIF Request Bandwidth */
1885 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1886 fixed20_12 disp_clk, sclk, bandwidth;
1887 fixed20_12 a, b1, b2;
1890 a.full = dfixed_const(1000);
1891 disp_clk.full = dfixed_const(wm->disp_clk);
1892 disp_clk.full = dfixed_div(disp_clk, a);
1893 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1894 b1.full = dfixed_mul(a, disp_clk);
1896 a.full = dfixed_const(1000);
1897 sclk.full = dfixed_const(wm->sclk);
1898 sclk.full = dfixed_div(sclk, a);
1899 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1900 b2.full = dfixed_mul(a, sclk);
1902 a.full = dfixed_const(10);
1903 disp_clk_request_efficiency.full = dfixed_const(8);
1904 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1906 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1908 a.full = dfixed_const(min_bandwidth);
1909 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1911 return dfixed_trunc(bandwidth);
1914 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1916 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1917 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1918 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1919 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1921 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1924 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1926 /* Calculate the display mode Average Bandwidth
1927 * DisplayMode should contain the source and destination dimensions,
1931 fixed20_12 line_time;
1932 fixed20_12 src_width;
1933 fixed20_12 bandwidth;
1936 a.full = dfixed_const(1000);
1937 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1938 line_time.full = dfixed_div(line_time, a);
1939 bpp.full = dfixed_const(wm->bytes_per_pixel);
1940 src_width.full = dfixed_const(wm->src_width);
1941 bandwidth.full = dfixed_mul(src_width, bpp);
1942 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1943 bandwidth.full = dfixed_div(bandwidth, line_time);
1945 return dfixed_trunc(bandwidth);
1948 static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1950 /* First calcualte the latency in ns */
1951 u32 mc_latency = 2000; /* 2000 ns. */
1952 u32 available_bandwidth = dce6_available_bandwidth(wm);
1953 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1954 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1955 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1956 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1957 (wm->num_heads * cursor_line_pair_return_time);
1958 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1959 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1960 u32 tmp, dmif_size = 12288;
1963 if (wm->num_heads == 0)
1966 a.full = dfixed_const(2);
1967 b.full = dfixed_const(1);
1968 if ((wm->vsc.full > a.full) ||
1969 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1971 ((wm->vsc.full >= a.full) && wm->interlaced))
1972 max_src_lines_per_dst_line = 4;
1974 max_src_lines_per_dst_line = 2;
1976 a.full = dfixed_const(available_bandwidth);
1977 b.full = dfixed_const(wm->num_heads);
1978 a.full = dfixed_div(a, b);
1980 b.full = dfixed_const(mc_latency + 512);
1981 c.full = dfixed_const(wm->disp_clk);
1982 b.full = dfixed_div(b, c);
1984 c.full = dfixed_const(dmif_size);
1985 b.full = dfixed_div(c, b);
1987 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1989 b.full = dfixed_const(1000);
1990 c.full = dfixed_const(wm->disp_clk);
1991 b.full = dfixed_div(c, b);
1992 c.full = dfixed_const(wm->bytes_per_pixel);
1993 b.full = dfixed_mul(b, c);
1995 lb_fill_bw = min(tmp, dfixed_trunc(b));
1997 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1998 b.full = dfixed_const(1000);
1999 c.full = dfixed_const(lb_fill_bw);
2000 b.full = dfixed_div(c, b);
2001 a.full = dfixed_div(a, b);
2002 line_fill_time = dfixed_trunc(a);
2004 if (line_fill_time < wm->active_time)
2007 return latency + (line_fill_time - wm->active_time);
2011 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2013 if (dce6_average_bandwidth(wm) <=
2014 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2020 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2022 if (dce6_average_bandwidth(wm) <=
2023 (dce6_available_bandwidth(wm) / wm->num_heads))
2029 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2031 u32 lb_partitions = wm->lb_size / wm->src_width;
2032 u32 line_time = wm->active_time + wm->blank_time;
2033 u32 latency_tolerant_lines;
2037 a.full = dfixed_const(1);
2038 if (wm->vsc.full > a.full)
2039 latency_tolerant_lines = 1;
2041 if (lb_partitions <= (wm->vtaps + 1))
2042 latency_tolerant_lines = 1;
2044 latency_tolerant_lines = 2;
2047 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2049 if (dce6_latency_watermark(wm) <= latency_hiding)
2055 static void dce6_program_watermarks(struct radeon_device *rdev,
2056 struct radeon_crtc *radeon_crtc,
2057 u32 lb_size, u32 num_heads)
2059 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2060 struct dce6_wm_params wm_low, wm_high;
2064 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2065 u32 priority_a_mark = 0, priority_b_mark = 0;
2066 u32 priority_a_cnt = PRIORITY_OFF;
2067 u32 priority_b_cnt = PRIORITY_OFF;
2068 u32 tmp, arb_control3;
2071 if (radeon_crtc->base.enabled && num_heads && mode) {
2072 pixel_period = 1000000 / (u32)mode->clock;
2073 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2077 if (rdev->family == CHIP_ARUBA)
2078 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2080 dram_channels = si_get_number_of_dram_channels(rdev);
2082 /* watermark for high clocks */
2083 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2085 radeon_dpm_get_mclk(rdev, false) * 10;
2087 radeon_dpm_get_sclk(rdev, false) * 10;
2089 wm_high.yclk = rdev->pm.current_mclk * 10;
2090 wm_high.sclk = rdev->pm.current_sclk * 10;
2093 wm_high.disp_clk = mode->clock;
2094 wm_high.src_width = mode->crtc_hdisplay;
2095 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2096 wm_high.blank_time = line_time - wm_high.active_time;
2097 wm_high.interlaced = false;
2098 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2099 wm_high.interlaced = true;
2100 wm_high.vsc = radeon_crtc->vsc;
2102 if (radeon_crtc->rmx_type != RMX_OFF)
2104 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2105 wm_high.lb_size = lb_size;
2106 wm_high.dram_channels = dram_channels;
2107 wm_high.num_heads = num_heads;
2109 /* watermark for low clocks */
2110 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2112 radeon_dpm_get_mclk(rdev, true) * 10;
2114 radeon_dpm_get_sclk(rdev, true) * 10;
2116 wm_low.yclk = rdev->pm.current_mclk * 10;
2117 wm_low.sclk = rdev->pm.current_sclk * 10;
2120 wm_low.disp_clk = mode->clock;
2121 wm_low.src_width = mode->crtc_hdisplay;
2122 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2123 wm_low.blank_time = line_time - wm_low.active_time;
2124 wm_low.interlaced = false;
2125 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2126 wm_low.interlaced = true;
2127 wm_low.vsc = radeon_crtc->vsc;
2129 if (radeon_crtc->rmx_type != RMX_OFF)
2131 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2132 wm_low.lb_size = lb_size;
2133 wm_low.dram_channels = dram_channels;
2134 wm_low.num_heads = num_heads;
2136 /* set for high clocks */
2137 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2138 /* set for low clocks */
2139 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2141 /* possibly force display priority to high */
2142 /* should really do this at mode validation time... */
2143 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2144 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2145 !dce6_check_latency_hiding(&wm_high) ||
2146 (rdev->disp_priority == 2)) {
2147 DRM_DEBUG_KMS("force priority to high\n");
2148 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2149 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2151 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2152 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2153 !dce6_check_latency_hiding(&wm_low) ||
2154 (rdev->disp_priority == 2)) {
2155 DRM_DEBUG_KMS("force priority to high\n");
2156 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2157 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2160 a.full = dfixed_const(1000);
2161 b.full = dfixed_const(mode->clock);
2162 b.full = dfixed_div(b, a);
2163 c.full = dfixed_const(latency_watermark_a);
2164 c.full = dfixed_mul(c, b);
2165 c.full = dfixed_mul(c, radeon_crtc->hsc);
2166 c.full = dfixed_div(c, a);
2167 a.full = dfixed_const(16);
2168 c.full = dfixed_div(c, a);
2169 priority_a_mark = dfixed_trunc(c);
2170 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2172 a.full = dfixed_const(1000);
2173 b.full = dfixed_const(mode->clock);
2174 b.full = dfixed_div(b, a);
2175 c.full = dfixed_const(latency_watermark_b);
2176 c.full = dfixed_mul(c, b);
2177 c.full = dfixed_mul(c, radeon_crtc->hsc);
2178 c.full = dfixed_div(c, a);
2179 a.full = dfixed_const(16);
2180 c.full = dfixed_div(c, a);
2181 priority_b_mark = dfixed_trunc(c);
2182 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2186 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2188 tmp &= ~LATENCY_WATERMARK_MASK(3);
2189 tmp |= LATENCY_WATERMARK_MASK(1);
2190 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2191 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2192 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2193 LATENCY_HIGH_WATERMARK(line_time)));
2195 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2196 tmp &= ~LATENCY_WATERMARK_MASK(3);
2197 tmp |= LATENCY_WATERMARK_MASK(2);
2198 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2199 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2200 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2201 LATENCY_HIGH_WATERMARK(line_time)));
2202 /* restore original selection */
2203 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2205 /* write the priority marks */
2206 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2207 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2209 /* save values for DPM */
2210 radeon_crtc->line_time = line_time;
2211 radeon_crtc->wm_high = latency_watermark_a;
2212 radeon_crtc->wm_low = latency_watermark_b;
2215 void dce6_bandwidth_update(struct radeon_device *rdev)
2217 struct drm_display_mode *mode0 = NULL;
2218 struct drm_display_mode *mode1 = NULL;
2219 u32 num_heads = 0, lb_size;
2222 radeon_update_display_priority(rdev);
2224 for (i = 0; i < rdev->num_crtc; i++) {
2225 if (rdev->mode_info.crtcs[i]->base.enabled)
2228 for (i = 0; i < rdev->num_crtc; i += 2) {
2229 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2230 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2231 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2232 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2233 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2234 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2241 static void si_tiling_mode_table_init(struct radeon_device *rdev)
2243 const u32 num_tile_mode_states = 32;
2244 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2246 switch (rdev->config.si.mem_row_size_in_kb) {
2248 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2252 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2255 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2259 if ((rdev->family == CHIP_TAHITI) ||
2260 (rdev->family == CHIP_PITCAIRN)) {
2261 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2262 switch (reg_offset) {
2263 case 0: /* non-AA compressed depth or any compressed stencil */
2264 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2265 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2266 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2267 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2268 NUM_BANKS(ADDR_SURF_16_BANK) |
2269 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2270 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2271 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2273 case 1: /* 2xAA/4xAA compressed depth only */
2274 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2275 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2276 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2277 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2278 NUM_BANKS(ADDR_SURF_16_BANK) |
2279 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2280 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2281 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2283 case 2: /* 8xAA compressed depth only */
2284 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2285 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2286 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2287 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2288 NUM_BANKS(ADDR_SURF_16_BANK) |
2289 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2290 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2291 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2293 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2294 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2295 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2296 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2297 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2298 NUM_BANKS(ADDR_SURF_16_BANK) |
2299 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2300 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2301 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2303 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2304 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2305 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2306 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2307 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2308 NUM_BANKS(ADDR_SURF_16_BANK) |
2309 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2310 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2311 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2313 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2314 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2315 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2316 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2317 TILE_SPLIT(split_equal_to_row_size) |
2318 NUM_BANKS(ADDR_SURF_16_BANK) |
2319 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2320 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2321 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2323 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2324 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2325 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2326 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2327 TILE_SPLIT(split_equal_to_row_size) |
2328 NUM_BANKS(ADDR_SURF_16_BANK) |
2329 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2330 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2331 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2333 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2334 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2335 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2336 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2337 TILE_SPLIT(split_equal_to_row_size) |
2338 NUM_BANKS(ADDR_SURF_16_BANK) |
2339 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2340 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2341 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2343 case 8: /* 1D and 1D Array Surfaces */
2344 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2345 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2346 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2347 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2348 NUM_BANKS(ADDR_SURF_16_BANK) |
2349 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2350 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2351 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2353 case 9: /* Displayable maps. */
2354 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2355 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2356 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2357 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2358 NUM_BANKS(ADDR_SURF_16_BANK) |
2359 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2360 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2361 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2363 case 10: /* Display 8bpp. */
2364 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2365 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2366 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2367 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2368 NUM_BANKS(ADDR_SURF_16_BANK) |
2369 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2370 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2371 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2373 case 11: /* Display 16bpp. */
2374 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2375 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2376 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2377 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2378 NUM_BANKS(ADDR_SURF_16_BANK) |
2379 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2380 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2381 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2383 case 12: /* Display 32bpp. */
2384 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2385 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2386 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2387 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2388 NUM_BANKS(ADDR_SURF_16_BANK) |
2389 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2390 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2391 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2393 case 13: /* Thin. */
2394 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2395 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2396 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2397 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2398 NUM_BANKS(ADDR_SURF_16_BANK) |
2399 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2400 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2401 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2403 case 14: /* Thin 8 bpp. */
2404 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2405 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2406 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2407 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2408 NUM_BANKS(ADDR_SURF_16_BANK) |
2409 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2410 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2411 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2413 case 15: /* Thin 16 bpp. */
2414 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2415 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2416 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2417 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2418 NUM_BANKS(ADDR_SURF_16_BANK) |
2419 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2420 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2421 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2423 case 16: /* Thin 32 bpp. */
2424 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2425 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2426 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2427 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2428 NUM_BANKS(ADDR_SURF_16_BANK) |
2429 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2430 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2431 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2433 case 17: /* Thin 64 bpp. */
2434 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2435 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2436 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2437 TILE_SPLIT(split_equal_to_row_size) |
2438 NUM_BANKS(ADDR_SURF_16_BANK) |
2439 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2440 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2441 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2443 case 21: /* 8 bpp PRT. */
2444 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2445 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2446 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2447 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2448 NUM_BANKS(ADDR_SURF_16_BANK) |
2449 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2450 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2451 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2453 case 22: /* 16 bpp PRT */
2454 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2455 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2456 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2457 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2458 NUM_BANKS(ADDR_SURF_16_BANK) |
2459 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2460 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2461 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2463 case 23: /* 32 bpp PRT */
2464 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2465 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2466 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2467 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2468 NUM_BANKS(ADDR_SURF_16_BANK) |
2469 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2470 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2471 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2473 case 24: /* 64 bpp PRT */
2474 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2475 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2476 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2477 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2478 NUM_BANKS(ADDR_SURF_16_BANK) |
2479 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2480 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2481 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2483 case 25: /* 128 bpp PRT */
2484 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2485 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2486 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2487 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2488 NUM_BANKS(ADDR_SURF_8_BANK) |
2489 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2490 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2491 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2497 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2498 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2500 } else if ((rdev->family == CHIP_VERDE) ||
2501 (rdev->family == CHIP_OLAND) ||
2502 (rdev->family == CHIP_HAINAN)) {
2503 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2504 switch (reg_offset) {
2505 case 0: /* non-AA compressed depth or any compressed stencil */
2506 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2507 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2508 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2509 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2510 NUM_BANKS(ADDR_SURF_16_BANK) |
2511 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2512 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2513 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2515 case 1: /* 2xAA/4xAA compressed depth only */
2516 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2517 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2518 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2519 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2520 NUM_BANKS(ADDR_SURF_16_BANK) |
2521 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2522 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2523 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2525 case 2: /* 8xAA compressed depth only */
2526 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2527 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2528 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2529 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2530 NUM_BANKS(ADDR_SURF_16_BANK) |
2531 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2532 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2533 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2535 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2536 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2537 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2538 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2539 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2540 NUM_BANKS(ADDR_SURF_16_BANK) |
2541 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2542 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2543 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2545 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2546 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2547 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2548 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2549 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2550 NUM_BANKS(ADDR_SURF_16_BANK) |
2551 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2552 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2553 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2555 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2556 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2557 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2558 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2559 TILE_SPLIT(split_equal_to_row_size) |
2560 NUM_BANKS(ADDR_SURF_16_BANK) |
2561 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2562 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2563 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2565 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2566 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2567 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2568 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2569 TILE_SPLIT(split_equal_to_row_size) |
2570 NUM_BANKS(ADDR_SURF_16_BANK) |
2571 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2572 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2573 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2575 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2576 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2577 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2578 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2579 TILE_SPLIT(split_equal_to_row_size) |
2580 NUM_BANKS(ADDR_SURF_16_BANK) |
2581 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2582 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2583 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2585 case 8: /* 1D and 1D Array Surfaces */
2586 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2587 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2588 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2589 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2590 NUM_BANKS(ADDR_SURF_16_BANK) |
2591 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2592 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2593 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2595 case 9: /* Displayable maps. */
2596 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2597 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2598 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2599 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2600 NUM_BANKS(ADDR_SURF_16_BANK) |
2601 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2602 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2603 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2605 case 10: /* Display 8bpp. */
2606 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2607 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2608 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2609 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2610 NUM_BANKS(ADDR_SURF_16_BANK) |
2611 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2612 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2613 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2615 case 11: /* Display 16bpp. */
2616 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2617 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2618 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2619 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2620 NUM_BANKS(ADDR_SURF_16_BANK) |
2621 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2622 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2623 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2625 case 12: /* Display 32bpp. */
2626 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2627 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2628 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2629 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2630 NUM_BANKS(ADDR_SURF_16_BANK) |
2631 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2632 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2633 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2635 case 13: /* Thin. */
2636 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2637 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2638 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2639 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2640 NUM_BANKS(ADDR_SURF_16_BANK) |
2641 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2642 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2643 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2645 case 14: /* Thin 8 bpp. */
2646 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2647 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2648 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2649 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2650 NUM_BANKS(ADDR_SURF_16_BANK) |
2651 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2652 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2653 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2655 case 15: /* Thin 16 bpp. */
2656 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2657 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2658 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2659 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2660 NUM_BANKS(ADDR_SURF_16_BANK) |
2661 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2662 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2663 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2665 case 16: /* Thin 32 bpp. */
2666 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2667 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2668 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2669 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2670 NUM_BANKS(ADDR_SURF_16_BANK) |
2671 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2672 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2673 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2675 case 17: /* Thin 64 bpp. */
2676 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2677 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2678 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2679 TILE_SPLIT(split_equal_to_row_size) |
2680 NUM_BANKS(ADDR_SURF_16_BANK) |
2681 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2682 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2683 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2685 case 21: /* 8 bpp PRT. */
2686 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2687 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2688 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2689 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2690 NUM_BANKS(ADDR_SURF_16_BANK) |
2691 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2692 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2693 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2695 case 22: /* 16 bpp PRT */
2696 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2697 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2698 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2699 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2700 NUM_BANKS(ADDR_SURF_16_BANK) |
2701 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2702 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2703 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2705 case 23: /* 32 bpp PRT */
2706 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2707 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2708 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2709 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2710 NUM_BANKS(ADDR_SURF_16_BANK) |
2711 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2712 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2713 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2715 case 24: /* 64 bpp PRT */
2716 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2717 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2718 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2719 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2720 NUM_BANKS(ADDR_SURF_16_BANK) |
2721 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2722 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2723 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2725 case 25: /* 128 bpp PRT */
2726 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2727 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2728 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2729 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2730 NUM_BANKS(ADDR_SURF_8_BANK) |
2731 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2732 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2733 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2739 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2740 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2743 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2746 static void si_select_se_sh(struct radeon_device *rdev,
2747 u32 se_num, u32 sh_num)
2749 u32 data = INSTANCE_BROADCAST_WRITES;
2751 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2752 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2753 else if (se_num == 0xffffffff)
2754 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2755 else if (sh_num == 0xffffffff)
2756 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2758 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2759 WREG32(GRBM_GFX_INDEX, data);
2762 static u32 si_create_bitmask(u32 bit_width)
2766 for (i = 0; i < bit_width; i++) {
2773 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2777 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2779 data &= INACTIVE_CUS_MASK;
2782 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2784 data >>= INACTIVE_CUS_SHIFT;
2786 mask = si_create_bitmask(cu_per_sh);
2788 return ~data & mask;
2791 static void si_setup_spi(struct radeon_device *rdev,
2792 u32 se_num, u32 sh_per_se,
2796 u32 data, mask, active_cu;
2798 for (i = 0; i < se_num; i++) {
2799 for (j = 0; j < sh_per_se; j++) {
2800 si_select_se_sh(rdev, i, j);
2801 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2802 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2805 for (k = 0; k < 16; k++) {
2807 if (active_cu & mask) {
2809 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2815 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2818 static u32 si_get_rb_disabled(struct radeon_device *rdev,
2819 u32 max_rb_num, u32 se_num,
2824 data = RREG32(CC_RB_BACKEND_DISABLE);
2826 data &= BACKEND_DISABLE_MASK;
2829 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2831 data >>= BACKEND_DISABLE_SHIFT;
2833 mask = si_create_bitmask(max_rb_num / se_num / sh_per_se);
2838 static void si_setup_rb(struct radeon_device *rdev,
2839 u32 se_num, u32 sh_per_se,
2844 u32 disabled_rbs = 0;
2845 u32 enabled_rbs = 0;
2847 for (i = 0; i < se_num; i++) {
2848 for (j = 0; j < sh_per_se; j++) {
2849 si_select_se_sh(rdev, i, j);
2850 data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2851 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2854 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2857 for (i = 0; i < max_rb_num; i++) {
2858 if (!(disabled_rbs & mask))
2859 enabled_rbs |= mask;
2863 for (i = 0; i < se_num; i++) {
2864 si_select_se_sh(rdev, i, 0xffffffff);
2866 for (j = 0; j < sh_per_se; j++) {
2867 switch (enabled_rbs & 3) {
2869 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2872 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2876 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2881 WREG32(PA_SC_RASTER_CONFIG, data);
2883 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2886 static void si_gpu_init(struct radeon_device *rdev)
2888 u32 gb_addr_config = 0;
2889 u32 mc_shared_chmap, mc_arb_ramcfg;
2891 u32 hdp_host_path_cntl;
2895 switch (rdev->family) {
2897 rdev->config.si.max_shader_engines = 2;
2898 rdev->config.si.max_tile_pipes = 12;
2899 rdev->config.si.max_cu_per_sh = 8;
2900 rdev->config.si.max_sh_per_se = 2;
2901 rdev->config.si.max_backends_per_se = 4;
2902 rdev->config.si.max_texture_channel_caches = 12;
2903 rdev->config.si.max_gprs = 256;
2904 rdev->config.si.max_gs_threads = 32;
2905 rdev->config.si.max_hw_contexts = 8;
2907 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2908 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2909 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2910 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2911 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2914 rdev->config.si.max_shader_engines = 2;
2915 rdev->config.si.max_tile_pipes = 8;
2916 rdev->config.si.max_cu_per_sh = 5;
2917 rdev->config.si.max_sh_per_se = 2;
2918 rdev->config.si.max_backends_per_se = 4;
2919 rdev->config.si.max_texture_channel_caches = 8;
2920 rdev->config.si.max_gprs = 256;
2921 rdev->config.si.max_gs_threads = 32;
2922 rdev->config.si.max_hw_contexts = 8;
2924 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2925 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2926 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2927 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2928 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2932 rdev->config.si.max_shader_engines = 1;
2933 rdev->config.si.max_tile_pipes = 4;
2934 rdev->config.si.max_cu_per_sh = 5;
2935 rdev->config.si.max_sh_per_se = 2;
2936 rdev->config.si.max_backends_per_se = 4;
2937 rdev->config.si.max_texture_channel_caches = 4;
2938 rdev->config.si.max_gprs = 256;
2939 rdev->config.si.max_gs_threads = 32;
2940 rdev->config.si.max_hw_contexts = 8;
2942 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2943 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2944 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2945 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2946 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2949 rdev->config.si.max_shader_engines = 1;
2950 rdev->config.si.max_tile_pipes = 4;
2951 rdev->config.si.max_cu_per_sh = 6;
2952 rdev->config.si.max_sh_per_se = 1;
2953 rdev->config.si.max_backends_per_se = 2;
2954 rdev->config.si.max_texture_channel_caches = 4;
2955 rdev->config.si.max_gprs = 256;
2956 rdev->config.si.max_gs_threads = 16;
2957 rdev->config.si.max_hw_contexts = 8;
2959 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2960 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2961 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2962 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2963 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2966 rdev->config.si.max_shader_engines = 1;
2967 rdev->config.si.max_tile_pipes = 4;
2968 rdev->config.si.max_cu_per_sh = 5;
2969 rdev->config.si.max_sh_per_se = 1;
2970 rdev->config.si.max_backends_per_se = 1;
2971 rdev->config.si.max_texture_channel_caches = 2;
2972 rdev->config.si.max_gprs = 256;
2973 rdev->config.si.max_gs_threads = 16;
2974 rdev->config.si.max_hw_contexts = 8;
2976 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2977 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2978 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2979 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2980 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2984 /* Initialize HDP */
2985 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2986 WREG32((0x2c14 + j), 0x00000000);
2987 WREG32((0x2c18 + j), 0x00000000);
2988 WREG32((0x2c1c + j), 0x00000000);
2989 WREG32((0x2c20 + j), 0x00000000);
2990 WREG32((0x2c24 + j), 0x00000000);
2993 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2995 evergreen_fix_pci_max_read_req_size(rdev);
2997 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2999 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3000 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3002 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3003 rdev->config.si.mem_max_burst_length_bytes = 256;
3004 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3005 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3006 if (rdev->config.si.mem_row_size_in_kb > 4)
3007 rdev->config.si.mem_row_size_in_kb = 4;
3008 /* XXX use MC settings? */
3009 rdev->config.si.shader_engine_tile_size = 32;
3010 rdev->config.si.num_gpus = 1;
3011 rdev->config.si.multi_gpu_tile_size = 64;
3013 /* fix up row size */
3014 gb_addr_config &= ~ROW_SIZE_MASK;
3015 switch (rdev->config.si.mem_row_size_in_kb) {
3018 gb_addr_config |= ROW_SIZE(0);
3021 gb_addr_config |= ROW_SIZE(1);
3024 gb_addr_config |= ROW_SIZE(2);
3028 /* setup tiling info dword. gb_addr_config is not adequate since it does
3029 * not have bank info, so create a custom tiling dword.
3030 * bits 3:0 num_pipes
3031 * bits 7:4 num_banks
3032 * bits 11:8 group_size
3033 * bits 15:12 row_size
3035 rdev->config.si.tile_config = 0;
3036 switch (rdev->config.si.num_tile_pipes) {
3038 rdev->config.si.tile_config |= (0 << 0);
3041 rdev->config.si.tile_config |= (1 << 0);
3044 rdev->config.si.tile_config |= (2 << 0);
3048 /* XXX what about 12? */
3049 rdev->config.si.tile_config |= (3 << 0);
3052 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3053 case 0: /* four banks */
3054 rdev->config.si.tile_config |= 0 << 4;
3056 case 1: /* eight banks */
3057 rdev->config.si.tile_config |= 1 << 4;
3059 case 2: /* sixteen banks */
3061 rdev->config.si.tile_config |= 2 << 4;
3064 rdev->config.si.tile_config |=
3065 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3066 rdev->config.si.tile_config |=
3067 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3069 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3070 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3071 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3072 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3073 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3074 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3075 if (rdev->has_uvd) {
3076 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3077 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3078 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3081 si_tiling_mode_table_init(rdev);
3083 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3084 rdev->config.si.max_sh_per_se,
3085 rdev->config.si.max_backends_per_se);
3087 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3088 rdev->config.si.max_sh_per_se,
3089 rdev->config.si.max_cu_per_sh);
3092 /* set HW defaults for 3D engine */
3093 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3094 ROQ_IB2_START(0x2b)));
3095 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3097 sx_debug_1 = RREG32(SX_DEBUG_1);
3098 WREG32(SX_DEBUG_1, sx_debug_1);
3100 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3102 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3103 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3104 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3105 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3107 WREG32(VGT_NUM_INSTANCES, 1);
3109 WREG32(CP_PERFMON_CNTL, 0);
3111 WREG32(SQ_CONFIG, 0);
3113 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3114 FORCE_EOV_MAX_REZ_CNT(255)));
3116 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3117 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3119 WREG32(VGT_GS_VERTEX_REUSE, 16);
3120 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3122 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3123 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3124 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3125 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3126 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3127 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3128 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3129 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3131 tmp = RREG32(HDP_MISC_CNTL);
3132 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3133 WREG32(HDP_MISC_CNTL, tmp);
3135 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3136 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3138 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3144 * GPU scratch registers helpers function.
3146 static void si_scratch_init(struct radeon_device *rdev)
3150 rdev->scratch.num_reg = 7;
3151 rdev->scratch.reg_base = SCRATCH_REG0;
3152 for (i = 0; i < rdev->scratch.num_reg; i++) {
3153 rdev->scratch.free[i] = true;
3154 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3158 void si_fence_ring_emit(struct radeon_device *rdev,
3159 struct radeon_fence *fence)
3161 struct radeon_ring *ring = &rdev->ring[fence->ring];
3162 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3164 /* flush read cache over gart */
3165 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3166 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3167 radeon_ring_write(ring, 0);
3168 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3169 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3170 PACKET3_TC_ACTION_ENA |
3171 PACKET3_SH_KCACHE_ACTION_ENA |
3172 PACKET3_SH_ICACHE_ACTION_ENA);
3173 radeon_ring_write(ring, 0xFFFFFFFF);
3174 radeon_ring_write(ring, 0);
3175 radeon_ring_write(ring, 10); /* poll interval */
3176 /* EVENT_WRITE_EOP - flush caches, send int */
3177 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3178 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3179 radeon_ring_write(ring, addr & 0xffffffff);
3180 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3181 radeon_ring_write(ring, fence->seq);
3182 radeon_ring_write(ring, 0);
3188 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3190 struct radeon_ring *ring = &rdev->ring[ib->ring];
3193 if (ib->is_const_ib) {
3194 /* set switch buffer packet before const IB */
3195 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3196 radeon_ring_write(ring, 0);
3198 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3201 if (ring->rptr_save_reg) {
3202 next_rptr = ring->wptr + 3 + 4 + 8;
3203 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3204 radeon_ring_write(ring, ((ring->rptr_save_reg -
3205 PACKET3_SET_CONFIG_REG_START) >> 2));
3206 radeon_ring_write(ring, next_rptr);
3207 } else if (rdev->wb.enabled) {
3208 next_rptr = ring->wptr + 5 + 4 + 8;
3209 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3210 radeon_ring_write(ring, (1 << 8));
3211 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3212 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3213 radeon_ring_write(ring, next_rptr);
3216 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3219 radeon_ring_write(ring, header);
3220 radeon_ring_write(ring,
3224 (ib->gpu_addr & 0xFFFFFFFC));
3225 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3226 radeon_ring_write(ring, ib->length_dw |
3227 (ib->vm ? (ib->vm->id << 24) : 0));
3229 if (!ib->is_const_ib) {
3230 /* flush read cache over gart for this vmid */
3231 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3232 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3233 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
3234 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3235 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3236 PACKET3_TC_ACTION_ENA |
3237 PACKET3_SH_KCACHE_ACTION_ENA |
3238 PACKET3_SH_ICACHE_ACTION_ENA);
3239 radeon_ring_write(ring, 0xFFFFFFFF);
3240 radeon_ring_write(ring, 0);
3241 radeon_ring_write(ring, 10); /* poll interval */
3248 static void si_cp_enable(struct radeon_device *rdev, bool enable)
3251 WREG32(CP_ME_CNTL, 0);
3253 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3254 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3255 WREG32(SCRATCH_UMSK, 0);
3256 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3257 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3258 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3263 static int si_cp_load_microcode(struct radeon_device *rdev)
3265 const __be32 *fw_data;
3268 if (!rdev->me_fw || !rdev->pfp_fw)
3271 si_cp_enable(rdev, false);
3274 fw_data = (const __be32 *)rdev->pfp_fw->data;
3275 WREG32(CP_PFP_UCODE_ADDR, 0);
3276 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3277 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3278 WREG32(CP_PFP_UCODE_ADDR, 0);
3281 fw_data = (const __be32 *)rdev->ce_fw->data;
3282 WREG32(CP_CE_UCODE_ADDR, 0);
3283 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3284 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3285 WREG32(CP_CE_UCODE_ADDR, 0);
3288 fw_data = (const __be32 *)rdev->me_fw->data;
3289 WREG32(CP_ME_RAM_WADDR, 0);
3290 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3291 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3292 WREG32(CP_ME_RAM_WADDR, 0);
3294 WREG32(CP_PFP_UCODE_ADDR, 0);
3295 WREG32(CP_CE_UCODE_ADDR, 0);
3296 WREG32(CP_ME_RAM_WADDR, 0);
3297 WREG32(CP_ME_RAM_RADDR, 0);
3301 static int si_cp_start(struct radeon_device *rdev)
3303 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3306 r = radeon_ring_lock(rdev, ring, 7 + 4);
3308 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3312 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3313 radeon_ring_write(ring, 0x1);
3314 radeon_ring_write(ring, 0x0);
3315 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3316 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3317 radeon_ring_write(ring, 0);
3318 radeon_ring_write(ring, 0);
3320 /* init the CE partitions */
3321 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3322 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3323 radeon_ring_write(ring, 0xc000);
3324 radeon_ring_write(ring, 0xe000);
3325 radeon_ring_unlock_commit(rdev, ring);
3327 si_cp_enable(rdev, true);
3329 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3331 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3335 /* setup clear context state */
3336 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3337 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3339 for (i = 0; i < si_default_size; i++)
3340 radeon_ring_write(ring, si_default_state[i]);
3342 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3343 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3345 /* set clear context state */
3346 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3347 radeon_ring_write(ring, 0);
3349 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3350 radeon_ring_write(ring, 0x00000316);
3351 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3352 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3354 radeon_ring_unlock_commit(rdev, ring);
3356 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3357 ring = &rdev->ring[i];
3358 r = radeon_ring_lock(rdev, ring, 2);
3360 /* clear the compute context state */
3361 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3362 radeon_ring_write(ring, 0);
3364 radeon_ring_unlock_commit(rdev, ring);
3370 static void si_cp_fini(struct radeon_device *rdev)
3372 struct radeon_ring *ring;
3373 si_cp_enable(rdev, false);
3375 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3376 radeon_ring_fini(rdev, ring);
3377 radeon_scratch_free(rdev, ring->rptr_save_reg);
3379 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3380 radeon_ring_fini(rdev, ring);
3381 radeon_scratch_free(rdev, ring->rptr_save_reg);
3383 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3384 radeon_ring_fini(rdev, ring);
3385 radeon_scratch_free(rdev, ring->rptr_save_reg);
3388 static int si_cp_resume(struct radeon_device *rdev)
3390 struct radeon_ring *ring;
3395 si_enable_gui_idle_interrupt(rdev, false);
3397 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3398 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3400 /* Set the write pointer delay */
3401 WREG32(CP_RB_WPTR_DELAY, 0);
3403 WREG32(CP_DEBUG, 0);
3404 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3406 /* ring 0 - compute and gfx */
3407 /* Set ring buffer size */
3408 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3409 rb_bufsz = order_base_2(ring->ring_size / 8);
3410 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3412 tmp |= BUF_SWAP_32BIT;
3414 WREG32(CP_RB0_CNTL, tmp);
3416 /* Initialize the ring buffer's read and write pointers */
3417 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3419 WREG32(CP_RB0_WPTR, ring->wptr);
3421 /* set the wb address whether it's enabled or not */
3422 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3423 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3425 if (rdev->wb.enabled)
3426 WREG32(SCRATCH_UMSK, 0xff);
3428 tmp |= RB_NO_UPDATE;
3429 WREG32(SCRATCH_UMSK, 0);
3433 WREG32(CP_RB0_CNTL, tmp);
3435 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3437 ring->rptr = RREG32(CP_RB0_RPTR);
3439 /* ring1 - compute only */
3440 /* Set ring buffer size */
3441 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3442 rb_bufsz = order_base_2(ring->ring_size / 8);
3443 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3445 tmp |= BUF_SWAP_32BIT;
3447 WREG32(CP_RB1_CNTL, tmp);
3449 /* Initialize the ring buffer's read and write pointers */
3450 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3452 WREG32(CP_RB1_WPTR, ring->wptr);
3454 /* set the wb address whether it's enabled or not */
3455 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3456 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3459 WREG32(CP_RB1_CNTL, tmp);
3461 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3463 ring->rptr = RREG32(CP_RB1_RPTR);
3465 /* ring2 - compute only */
3466 /* Set ring buffer size */
3467 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3468 rb_bufsz = order_base_2(ring->ring_size / 8);
3469 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3471 tmp |= BUF_SWAP_32BIT;
3473 WREG32(CP_RB2_CNTL, tmp);
3475 /* Initialize the ring buffer's read and write pointers */
3476 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3478 WREG32(CP_RB2_WPTR, ring->wptr);
3480 /* set the wb address whether it's enabled or not */
3481 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3482 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3485 WREG32(CP_RB2_CNTL, tmp);
3487 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3489 ring->rptr = RREG32(CP_RB2_RPTR);
3491 /* start the rings */
3493 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3494 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3495 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3496 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3498 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3499 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3500 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3503 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3505 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3507 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3509 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3512 si_enable_gui_idle_interrupt(rdev, true);
3517 u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3523 tmp = RREG32(GRBM_STATUS);
3524 if (tmp & (PA_BUSY | SC_BUSY |
3525 BCI_BUSY | SX_BUSY |
3526 TA_BUSY | VGT_BUSY |
3528 GDS_BUSY | SPI_BUSY |
3529 IA_BUSY | IA_BUSY_NO_DMA))
3530 reset_mask |= RADEON_RESET_GFX;
3532 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3533 CP_BUSY | CP_COHERENCY_BUSY))
3534 reset_mask |= RADEON_RESET_CP;
3536 if (tmp & GRBM_EE_BUSY)
3537 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3540 tmp = RREG32(GRBM_STATUS2);
3541 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3542 reset_mask |= RADEON_RESET_RLC;
3544 /* DMA_STATUS_REG 0 */
3545 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3546 if (!(tmp & DMA_IDLE))
3547 reset_mask |= RADEON_RESET_DMA;
3549 /* DMA_STATUS_REG 1 */
3550 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3551 if (!(tmp & DMA_IDLE))
3552 reset_mask |= RADEON_RESET_DMA1;
3555 tmp = RREG32(SRBM_STATUS2);
3557 reset_mask |= RADEON_RESET_DMA;
3559 if (tmp & DMA1_BUSY)
3560 reset_mask |= RADEON_RESET_DMA1;
3563 tmp = RREG32(SRBM_STATUS);
3566 reset_mask |= RADEON_RESET_IH;
3569 reset_mask |= RADEON_RESET_SEM;
3571 if (tmp & GRBM_RQ_PENDING)
3572 reset_mask |= RADEON_RESET_GRBM;
3575 reset_mask |= RADEON_RESET_VMC;
3577 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3578 MCC_BUSY | MCD_BUSY))
3579 reset_mask |= RADEON_RESET_MC;
3581 if (evergreen_is_display_hung(rdev))
3582 reset_mask |= RADEON_RESET_DISPLAY;
3585 tmp = RREG32(VM_L2_STATUS);
3587 reset_mask |= RADEON_RESET_VMC;
3589 /* Skip MC reset as it's mostly likely not hung, just busy */
3590 if (reset_mask & RADEON_RESET_MC) {
3591 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3592 reset_mask &= ~RADEON_RESET_MC;
3598 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3600 struct evergreen_mc_save save;
3601 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3604 if (reset_mask == 0)
3607 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3609 evergreen_print_gpu_status_regs(rdev);
3610 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3611 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3612 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3613 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3622 /* Disable CP parsing/prefetching */
3623 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3625 if (reset_mask & RADEON_RESET_DMA) {
3627 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3628 tmp &= ~DMA_RB_ENABLE;
3629 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3631 if (reset_mask & RADEON_RESET_DMA1) {
3633 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3634 tmp &= ~DMA_RB_ENABLE;
3635 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3640 evergreen_mc_stop(rdev, &save);
3641 if (evergreen_mc_wait_for_idle(rdev)) {
3642 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3645 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3646 grbm_soft_reset = SOFT_RESET_CB |
3660 if (reset_mask & RADEON_RESET_CP) {
3661 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3663 srbm_soft_reset |= SOFT_RESET_GRBM;
3666 if (reset_mask & RADEON_RESET_DMA)
3667 srbm_soft_reset |= SOFT_RESET_DMA;
3669 if (reset_mask & RADEON_RESET_DMA1)
3670 srbm_soft_reset |= SOFT_RESET_DMA1;
3672 if (reset_mask & RADEON_RESET_DISPLAY)
3673 srbm_soft_reset |= SOFT_RESET_DC;
3675 if (reset_mask & RADEON_RESET_RLC)
3676 grbm_soft_reset |= SOFT_RESET_RLC;
3678 if (reset_mask & RADEON_RESET_SEM)
3679 srbm_soft_reset |= SOFT_RESET_SEM;
3681 if (reset_mask & RADEON_RESET_IH)
3682 srbm_soft_reset |= SOFT_RESET_IH;
3684 if (reset_mask & RADEON_RESET_GRBM)
3685 srbm_soft_reset |= SOFT_RESET_GRBM;
3687 if (reset_mask & RADEON_RESET_VMC)
3688 srbm_soft_reset |= SOFT_RESET_VMC;
3690 if (reset_mask & RADEON_RESET_MC)
3691 srbm_soft_reset |= SOFT_RESET_MC;
3693 if (grbm_soft_reset) {
3694 tmp = RREG32(GRBM_SOFT_RESET);
3695 tmp |= grbm_soft_reset;
3696 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3697 WREG32(GRBM_SOFT_RESET, tmp);
3698 tmp = RREG32(GRBM_SOFT_RESET);
3702 tmp &= ~grbm_soft_reset;
3703 WREG32(GRBM_SOFT_RESET, tmp);
3704 tmp = RREG32(GRBM_SOFT_RESET);
3707 if (srbm_soft_reset) {
3708 tmp = RREG32(SRBM_SOFT_RESET);
3709 tmp |= srbm_soft_reset;
3710 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3711 WREG32(SRBM_SOFT_RESET, tmp);
3712 tmp = RREG32(SRBM_SOFT_RESET);
3716 tmp &= ~srbm_soft_reset;
3717 WREG32(SRBM_SOFT_RESET, tmp);
3718 tmp = RREG32(SRBM_SOFT_RESET);
3721 /* Wait a little for things to settle down */
3724 evergreen_mc_resume(rdev, &save);
3727 evergreen_print_gpu_status_regs(rdev);
3730 int si_asic_reset(struct radeon_device *rdev)
3734 reset_mask = si_gpu_check_soft_reset(rdev);
3737 r600_set_bios_scratch_engine_hung(rdev, true);
3739 si_gpu_soft_reset(rdev, reset_mask);
3741 reset_mask = si_gpu_check_soft_reset(rdev);
3744 r600_set_bios_scratch_engine_hung(rdev, false);
3750 * si_gfx_is_lockup - Check if the GFX engine is locked up
3752 * @rdev: radeon_device pointer
3753 * @ring: radeon_ring structure holding ring information
3755 * Check if the GFX engine is locked up.
3756 * Returns true if the engine appears to be locked up, false if not.
3758 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3760 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3762 if (!(reset_mask & (RADEON_RESET_GFX |
3763 RADEON_RESET_COMPUTE |
3764 RADEON_RESET_CP))) {
3765 radeon_ring_lockup_update(ring);
3768 /* force CP activities */
3769 radeon_ring_force_activity(rdev, ring);
3770 return radeon_ring_test_lockup(rdev, ring);
3774 static void si_mc_program(struct radeon_device *rdev)
3776 struct evergreen_mc_save save;
3780 /* Initialize HDP */
3781 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3782 WREG32((0x2c14 + j), 0x00000000);
3783 WREG32((0x2c18 + j), 0x00000000);
3784 WREG32((0x2c1c + j), 0x00000000);
3785 WREG32((0x2c20 + j), 0x00000000);
3786 WREG32((0x2c24 + j), 0x00000000);
3788 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3790 evergreen_mc_stop(rdev, &save);
3791 if (radeon_mc_wait_for_idle(rdev)) {
3792 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3794 if (!ASIC_IS_NODCE(rdev))
3795 /* Lockout access through VGA aperture*/
3796 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
3797 /* Update configuration */
3798 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3799 rdev->mc.vram_start >> 12);
3800 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3801 rdev->mc.vram_end >> 12);
3802 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3803 rdev->vram_scratch.gpu_addr >> 12);
3804 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3805 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3806 WREG32(MC_VM_FB_LOCATION, tmp);
3807 /* XXX double check these! */
3808 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3809 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3810 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3811 WREG32(MC_VM_AGP_BASE, 0);
3812 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3813 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3814 if (radeon_mc_wait_for_idle(rdev)) {
3815 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3817 evergreen_mc_resume(rdev, &save);
3818 if (!ASIC_IS_NODCE(rdev)) {
3819 /* we need to own VRAM, so turn off the VGA renderer here
3820 * to stop it overwriting our objects */
3821 rv515_vga_render_disable(rdev);
3825 void si_vram_gtt_location(struct radeon_device *rdev,
3826 struct radeon_mc *mc)
3828 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3829 /* leave room for at least 1024M GTT */
3830 dev_warn(rdev->dev, "limiting VRAM\n");
3831 mc->real_vram_size = 0xFFC0000000ULL;
3832 mc->mc_vram_size = 0xFFC0000000ULL;
3834 radeon_vram_location(rdev, &rdev->mc, 0);
3835 rdev->mc.gtt_base_align = 0;
3836 radeon_gtt_location(rdev, mc);
3839 static int si_mc_init(struct radeon_device *rdev)
3842 int chansize, numchan;
3844 /* Get VRAM informations */
3845 rdev->mc.vram_is_ddr = true;
3846 tmp = RREG32(MC_ARB_RAMCFG);
3847 if (tmp & CHANSIZE_OVERRIDE) {
3849 } else if (tmp & CHANSIZE_MASK) {
3854 tmp = RREG32(MC_SHARED_CHMAP);
3855 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3885 rdev->mc.vram_width = numchan * chansize;
3886 /* Could aper size report 0 ? */
3887 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3888 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3889 /* size in MB on si */
3890 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3891 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3892 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3893 si_vram_gtt_location(rdev, &rdev->mc);
3894 radeon_update_bandwidth_info(rdev);
3902 void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
3904 /* flush hdp cache */
3905 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3907 /* bits 0-15 are the VM contexts0-15 */
3908 WREG32(VM_INVALIDATE_REQUEST, 1);
3911 static int si_pcie_gart_enable(struct radeon_device *rdev)
3915 if (rdev->gart.robj == NULL) {
3916 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
3919 r = radeon_gart_table_vram_pin(rdev);
3922 radeon_gart_restore(rdev);
3923 /* Setup TLB control */
3924 WREG32(MC_VM_MX_L1_TLB_CNTL,
3927 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3928 ENABLE_ADVANCED_DRIVER_MODEL |
3929 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3930 /* Setup L2 cache */
3931 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
3932 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3933 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3934 EFFECTIVE_L2_QUEUE_SIZE(7) |
3935 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3936 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
3937 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3938 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3939 /* setup context0 */
3940 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
3941 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3942 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
3943 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
3944 (u32)(rdev->dummy_page.addr >> 12));
3945 WREG32(VM_CONTEXT0_CNTL2, 0);
3946 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
3947 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
3953 /* empty context1-15 */
3954 /* set vm size, must be a multiple of 4 */
3955 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
3956 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
3957 /* Assign the pt base to something valid for now; the pts used for
3958 * the VMs are determined by the application and setup and assigned
3959 * on the fly in the vm part of radeon_gart.c
3961 for (i = 1; i < 16; i++) {
3963 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
3964 rdev->gart.table_addr >> 12);
3966 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
3967 rdev->gart.table_addr >> 12);
3970 /* enable context1-15 */
3971 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
3972 (u32)(rdev->dummy_page.addr >> 12));
3973 WREG32(VM_CONTEXT1_CNTL2, 4);
3974 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
3975 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3976 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3977 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3978 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3979 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
3980 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
3981 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
3982 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
3983 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
3984 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
3985 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3986 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
3988 si_pcie_gart_tlb_flush(rdev);
3989 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3990 (unsigned)(rdev->mc.gtt_size >> 20),
3991 (unsigned long long)rdev->gart.table_addr);
3992 rdev->gart.ready = true;
3996 static void si_pcie_gart_disable(struct radeon_device *rdev)
3998 /* Disable all tables */
3999 WREG32(VM_CONTEXT0_CNTL, 0);
4000 WREG32(VM_CONTEXT1_CNTL, 0);
4001 /* Setup TLB control */
4002 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4003 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4004 /* Setup L2 cache */
4005 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4006 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4007 EFFECTIVE_L2_QUEUE_SIZE(7) |
4008 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4009 WREG32(VM_L2_CNTL2, 0);
4010 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4011 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4012 radeon_gart_table_vram_unpin(rdev);
4015 static void si_pcie_gart_fini(struct radeon_device *rdev)
4017 si_pcie_gart_disable(rdev);
4018 radeon_gart_table_vram_free(rdev);
4019 radeon_gart_fini(rdev);
4023 static bool si_vm_reg_valid(u32 reg)
4025 /* context regs are fine */
4029 /* check config regs */
4031 case GRBM_GFX_INDEX:
4032 case CP_STRMOUT_CNTL:
4033 case VGT_VTX_VECT_EJECT_REG:
4034 case VGT_CACHE_INVALIDATION:
4035 case VGT_ESGS_RING_SIZE:
4036 case VGT_GSVS_RING_SIZE:
4037 case VGT_GS_VERTEX_REUSE:
4038 case VGT_PRIMITIVE_TYPE:
4039 case VGT_INDEX_TYPE:
4040 case VGT_NUM_INDICES:
4041 case VGT_NUM_INSTANCES:
4042 case VGT_TF_RING_SIZE:
4043 case VGT_HS_OFFCHIP_PARAM:
4044 case VGT_TF_MEMORY_BASE:
4046 case PA_SU_LINE_STIPPLE_VALUE:
4047 case PA_SC_LINE_STIPPLE_STATE:
4050 case SPI_STATIC_THREAD_MGMT_1:
4051 case SPI_STATIC_THREAD_MGMT_2:
4052 case SPI_STATIC_THREAD_MGMT_3:
4053 case SPI_PS_MAX_WAVE_ID:
4054 case SPI_CONFIG_CNTL:
4055 case SPI_CONFIG_CNTL_1:
4059 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4064 static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4065 u32 *ib, struct radeon_cs_packet *pkt)
4067 switch (pkt->opcode) {
4069 case PACKET3_SET_BASE:
4070 case PACKET3_SET_CE_DE_COUNTERS:
4071 case PACKET3_LOAD_CONST_RAM:
4072 case PACKET3_WRITE_CONST_RAM:
4073 case PACKET3_WRITE_CONST_RAM_OFFSET:
4074 case PACKET3_DUMP_CONST_RAM:
4075 case PACKET3_INCREMENT_CE_COUNTER:
4076 case PACKET3_WAIT_ON_DE_COUNTER:
4077 case PACKET3_CE_WRITE:
4080 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4086 static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4088 u32 start_reg, reg, i;
4089 u32 command = ib[idx + 4];
4090 u32 info = ib[idx + 1];
4091 u32 idx_value = ib[idx];
4092 if (command & PACKET3_CP_DMA_CMD_SAS) {
4093 /* src address space is register */
4094 if (((info & 0x60000000) >> 29) == 0) {
4095 start_reg = idx_value << 2;
4096 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4098 if (!si_vm_reg_valid(reg)) {
4099 DRM_ERROR("CP DMA Bad SRC register\n");
4103 for (i = 0; i < (command & 0x1fffff); i++) {
4104 reg = start_reg + (4 * i);
4105 if (!si_vm_reg_valid(reg)) {
4106 DRM_ERROR("CP DMA Bad SRC register\n");
4113 if (command & PACKET3_CP_DMA_CMD_DAS) {
4114 /* dst address space is register */
4115 if (((info & 0x00300000) >> 20) == 0) {
4116 start_reg = ib[idx + 2];
4117 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4119 if (!si_vm_reg_valid(reg)) {
4120 DRM_ERROR("CP DMA Bad DST register\n");
4124 for (i = 0; i < (command & 0x1fffff); i++) {
4125 reg = start_reg + (4 * i);
4126 if (!si_vm_reg_valid(reg)) {
4127 DRM_ERROR("CP DMA Bad DST register\n");
4137 static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4138 u32 *ib, struct radeon_cs_packet *pkt)
4141 u32 idx = pkt->idx + 1;
4142 u32 idx_value = ib[idx];
4143 u32 start_reg, end_reg, reg, i;
4145 switch (pkt->opcode) {
4147 case PACKET3_SET_BASE:
4148 case PACKET3_CLEAR_STATE:
4149 case PACKET3_INDEX_BUFFER_SIZE:
4150 case PACKET3_DISPATCH_DIRECT:
4151 case PACKET3_DISPATCH_INDIRECT:
4152 case PACKET3_ALLOC_GDS:
4153 case PACKET3_WRITE_GDS_RAM:
4154 case PACKET3_ATOMIC_GDS:
4155 case PACKET3_ATOMIC:
4156 case PACKET3_OCCLUSION_QUERY:
4157 case PACKET3_SET_PREDICATION:
4158 case PACKET3_COND_EXEC:
4159 case PACKET3_PRED_EXEC:
4160 case PACKET3_DRAW_INDIRECT:
4161 case PACKET3_DRAW_INDEX_INDIRECT:
4162 case PACKET3_INDEX_BASE:
4163 case PACKET3_DRAW_INDEX_2:
4164 case PACKET3_CONTEXT_CONTROL:
4165 case PACKET3_INDEX_TYPE:
4166 case PACKET3_DRAW_INDIRECT_MULTI:
4167 case PACKET3_DRAW_INDEX_AUTO:
4168 case PACKET3_DRAW_INDEX_IMMD:
4169 case PACKET3_NUM_INSTANCES:
4170 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4171 case PACKET3_STRMOUT_BUFFER_UPDATE:
4172 case PACKET3_DRAW_INDEX_OFFSET_2:
4173 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4174 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4175 case PACKET3_MPEG_INDEX:
4176 case PACKET3_WAIT_REG_MEM:
4177 case PACKET3_MEM_WRITE:
4178 case PACKET3_PFP_SYNC_ME:
4179 case PACKET3_SURFACE_SYNC:
4180 case PACKET3_EVENT_WRITE:
4181 case PACKET3_EVENT_WRITE_EOP:
4182 case PACKET3_EVENT_WRITE_EOS:
4183 case PACKET3_SET_CONTEXT_REG:
4184 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4185 case PACKET3_SET_SH_REG:
4186 case PACKET3_SET_SH_REG_OFFSET:
4187 case PACKET3_INCREMENT_DE_COUNTER:
4188 case PACKET3_WAIT_ON_CE_COUNTER:
4189 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4190 case PACKET3_ME_WRITE:
4192 case PACKET3_COPY_DATA:
4193 if ((idx_value & 0xf00) == 0) {
4194 reg = ib[idx + 3] * 4;
4195 if (!si_vm_reg_valid(reg))
4199 case PACKET3_WRITE_DATA:
4200 if ((idx_value & 0xf00) == 0) {
4201 start_reg = ib[idx + 1] * 4;
4202 if (idx_value & 0x10000) {
4203 if (!si_vm_reg_valid(start_reg))
4206 for (i = 0; i < (pkt->count - 2); i++) {
4207 reg = start_reg + (4 * i);
4208 if (!si_vm_reg_valid(reg))
4214 case PACKET3_COND_WRITE:
4215 if (idx_value & 0x100) {
4216 reg = ib[idx + 5] * 4;
4217 if (!si_vm_reg_valid(reg))
4221 case PACKET3_COPY_DW:
4222 if (idx_value & 0x2) {
4223 reg = ib[idx + 3] * 4;
4224 if (!si_vm_reg_valid(reg))
4228 case PACKET3_SET_CONFIG_REG:
4229 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4230 end_reg = 4 * pkt->count + start_reg - 4;
4231 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4232 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4233 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4234 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4237 for (i = 0; i < pkt->count; i++) {
4238 reg = start_reg + (4 * i);
4239 if (!si_vm_reg_valid(reg))
4243 case PACKET3_CP_DMA:
4244 r = si_vm_packet3_cp_dma_check(ib, idx);
4249 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4255 static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4256 u32 *ib, struct radeon_cs_packet *pkt)
4259 u32 idx = pkt->idx + 1;
4260 u32 idx_value = ib[idx];
4261 u32 start_reg, reg, i;
4263 switch (pkt->opcode) {
4265 case PACKET3_SET_BASE:
4266 case PACKET3_CLEAR_STATE:
4267 case PACKET3_DISPATCH_DIRECT:
4268 case PACKET3_DISPATCH_INDIRECT:
4269 case PACKET3_ALLOC_GDS:
4270 case PACKET3_WRITE_GDS_RAM:
4271 case PACKET3_ATOMIC_GDS:
4272 case PACKET3_ATOMIC:
4273 case PACKET3_OCCLUSION_QUERY:
4274 case PACKET3_SET_PREDICATION:
4275 case PACKET3_COND_EXEC:
4276 case PACKET3_PRED_EXEC:
4277 case PACKET3_CONTEXT_CONTROL:
4278 case PACKET3_STRMOUT_BUFFER_UPDATE:
4279 case PACKET3_WAIT_REG_MEM:
4280 case PACKET3_MEM_WRITE:
4281 case PACKET3_PFP_SYNC_ME:
4282 case PACKET3_SURFACE_SYNC:
4283 case PACKET3_EVENT_WRITE:
4284 case PACKET3_EVENT_WRITE_EOP:
4285 case PACKET3_EVENT_WRITE_EOS:
4286 case PACKET3_SET_CONTEXT_REG:
4287 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4288 case PACKET3_SET_SH_REG:
4289 case PACKET3_SET_SH_REG_OFFSET:
4290 case PACKET3_INCREMENT_DE_COUNTER:
4291 case PACKET3_WAIT_ON_CE_COUNTER:
4292 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4293 case PACKET3_ME_WRITE:
4295 case PACKET3_COPY_DATA:
4296 if ((idx_value & 0xf00) == 0) {
4297 reg = ib[idx + 3] * 4;
4298 if (!si_vm_reg_valid(reg))
4302 case PACKET3_WRITE_DATA:
4303 if ((idx_value & 0xf00) == 0) {
4304 start_reg = ib[idx + 1] * 4;
4305 if (idx_value & 0x10000) {
4306 if (!si_vm_reg_valid(start_reg))
4309 for (i = 0; i < (pkt->count - 2); i++) {
4310 reg = start_reg + (4 * i);
4311 if (!si_vm_reg_valid(reg))
4317 case PACKET3_COND_WRITE:
4318 if (idx_value & 0x100) {
4319 reg = ib[idx + 5] * 4;
4320 if (!si_vm_reg_valid(reg))
4324 case PACKET3_COPY_DW:
4325 if (idx_value & 0x2) {
4326 reg = ib[idx + 3] * 4;
4327 if (!si_vm_reg_valid(reg))
4331 case PACKET3_CP_DMA:
4332 r = si_vm_packet3_cp_dma_check(ib, idx);
4337 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4343 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4347 struct radeon_cs_packet pkt;
4351 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4352 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4355 case RADEON_PACKET_TYPE0:
4356 dev_err(rdev->dev, "Packet0 not allowed!\n");
4359 case RADEON_PACKET_TYPE2:
4362 case RADEON_PACKET_TYPE3:
4363 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4364 if (ib->is_const_ib)
4365 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4368 case RADEON_RING_TYPE_GFX_INDEX:
4369 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4371 case CAYMAN_RING_TYPE_CP1_INDEX:
4372 case CAYMAN_RING_TYPE_CP2_INDEX:
4373 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4376 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4381 idx += pkt.count + 2;
4384 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4390 } while (idx < ib->length_dw);
4398 int si_vm_init(struct radeon_device *rdev)
4401 rdev->vm_manager.nvm = 16;
4402 /* base offset of vram pages */
4403 rdev->vm_manager.vram_base_offset = 0;
4408 void si_vm_fini(struct radeon_device *rdev)
4413 * si_vm_decode_fault - print human readable fault info
4415 * @rdev: radeon_device pointer
4416 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4417 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4419 * Print human readable fault information (SI).
4421 static void si_vm_decode_fault(struct radeon_device *rdev,
4422 u32 status, u32 addr)
4424 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4425 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4426 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4429 if (rdev->family == CHIP_TAHITI) {
4670 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4671 protections, vmid, addr,
4672 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4677 * si_vm_set_page - update the page tables using the CP
4679 * @rdev: radeon_device pointer
4680 * @ib: indirect buffer to fill with commands
4681 * @pe: addr of the page entry
4682 * @addr: dst addr to write into pe
4683 * @count: number of page entries to update
4684 * @incr: increase next addr by incr bytes
4685 * @flags: access flags
4687 * Update the page tables using the CP (SI).
4689 void si_vm_set_page(struct radeon_device *rdev,
4690 struct radeon_ib *ib,
4692 uint64_t addr, unsigned count,
4693 uint32_t incr, uint32_t flags)
4695 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
4699 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4701 ndw = 2 + count * 2;
4705 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4706 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4707 WRITE_DATA_DST_SEL(1));
4708 ib->ptr[ib->length_dw++] = pe;
4709 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
4710 for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4711 if (flags & RADEON_VM_PAGE_SYSTEM) {
4712 value = radeon_vm_map_gart(rdev, addr);
4713 value &= 0xFFFFFFFFFFFFF000ULL;
4714 } else if (flags & RADEON_VM_PAGE_VALID) {
4720 value |= r600_flags;
4721 ib->ptr[ib->length_dw++] = value;
4722 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4727 si_dma_vm_set_page(rdev, ib, pe, addr, count, incr, flags);
4731 void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4733 struct radeon_ring *ring = &rdev->ring[ridx];
4738 /* write new base address */
4739 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4740 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4741 WRITE_DATA_DST_SEL(0)));
4744 radeon_ring_write(ring,
4745 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4747 radeon_ring_write(ring,
4748 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4750 radeon_ring_write(ring, 0);
4751 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4753 /* flush hdp cache */
4754 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4755 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4756 WRITE_DATA_DST_SEL(0)));
4757 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4758 radeon_ring_write(ring, 0);
4759 radeon_ring_write(ring, 0x1);
4761 /* bits 0-15 are the VM contexts0-15 */
4762 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4763 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4764 WRITE_DATA_DST_SEL(0)));
4765 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4766 radeon_ring_write(ring, 0);
4767 radeon_ring_write(ring, 1 << vm->id);
4769 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4770 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4771 radeon_ring_write(ring, 0x0);
4775 * Power and clock gating
4777 static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
4781 for (i = 0; i < rdev->usec_timeout; i++) {
4782 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
4787 for (i = 0; i < rdev->usec_timeout; i++) {
4788 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
4794 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
4797 u32 tmp = RREG32(CP_INT_CNTL_RING0);
4802 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4804 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4805 WREG32(CP_INT_CNTL_RING0, tmp);
4808 /* read a gfx register */
4809 tmp = RREG32(DB_DEPTH_INFO);
4811 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
4812 for (i = 0; i < rdev->usec_timeout; i++) {
4813 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
4820 static void si_set_uvd_dcm(struct radeon_device *rdev,
4825 tmp = RREG32(UVD_CGC_CTRL);
4826 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
4827 tmp |= DCM | CG_DT(1) | CLK_OD(4);
4831 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
4837 WREG32(UVD_CGC_CTRL, tmp);
4838 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
4841 void si_init_uvd_internal_cg(struct radeon_device *rdev)
4843 bool hw_mode = true;
4846 si_set_uvd_dcm(rdev, false);
4848 u32 tmp = RREG32(UVD_CGC_CTRL);
4850 WREG32(UVD_CGC_CTRL, tmp);
4854 static u32 si_halt_rlc(struct radeon_device *rdev)
4858 orig = data = RREG32(RLC_CNTL);
4860 if (data & RLC_ENABLE) {
4861 data &= ~RLC_ENABLE;
4862 WREG32(RLC_CNTL, data);
4864 si_wait_for_rlc_serdes(rdev);
4870 static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
4874 tmp = RREG32(RLC_CNTL);
4876 WREG32(RLC_CNTL, rlc);
4879 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
4883 orig = data = RREG32(DMA_PG);
4884 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
4885 data |= PG_CNTL_ENABLE;
4887 data &= ~PG_CNTL_ENABLE;
4889 WREG32(DMA_PG, data);
4892 static void si_init_dma_pg(struct radeon_device *rdev)
4896 WREG32(DMA_PGFSM_WRITE, 0x00002000);
4897 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
4899 for (tmp = 0; tmp < 5; tmp++)
4900 WREG32(DMA_PGFSM_WRITE, 0);
4903 static void si_enable_gfx_cgpg(struct radeon_device *rdev,
4908 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
4909 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
4910 WREG32(RLC_TTOP_D, tmp);
4912 tmp = RREG32(RLC_PG_CNTL);
4913 tmp |= GFX_PG_ENABLE;
4914 WREG32(RLC_PG_CNTL, tmp);
4916 tmp = RREG32(RLC_AUTO_PG_CTRL);
4918 WREG32(RLC_AUTO_PG_CTRL, tmp);
4920 tmp = RREG32(RLC_AUTO_PG_CTRL);
4922 WREG32(RLC_AUTO_PG_CTRL, tmp);
4924 tmp = RREG32(DB_RENDER_CONTROL);
4928 static void si_init_gfx_cgpg(struct radeon_device *rdev)
4932 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4934 tmp = RREG32(RLC_PG_CNTL);
4936 WREG32(RLC_PG_CNTL, tmp);
4938 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4940 tmp = RREG32(RLC_AUTO_PG_CTRL);
4942 tmp &= ~GRBM_REG_SGIT_MASK;
4943 tmp |= GRBM_REG_SGIT(0x700);
4944 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
4945 WREG32(RLC_AUTO_PG_CTRL, tmp);
4948 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
4950 u32 mask = 0, tmp, tmp1;
4953 si_select_se_sh(rdev, se, sh);
4954 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
4955 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
4956 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
4963 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
4968 return (~tmp) & mask;
4971 static void si_init_ao_cu_mask(struct radeon_device *rdev)
4973 u32 i, j, k, active_cu_number = 0;
4974 u32 mask, counter, cu_bitmap;
4977 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
4978 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
4982 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
4983 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
4991 active_cu_number += counter;
4992 tmp |= (cu_bitmap << (i * 16 + j * 8));
4996 WREG32(RLC_PG_AO_CU_MASK, tmp);
4998 tmp = RREG32(RLC_MAX_PG_CU);
4999 tmp &= ~MAX_PU_CU_MASK;
5000 tmp |= MAX_PU_CU(active_cu_number);
5001 WREG32(RLC_MAX_PG_CU, tmp);
5004 static void si_enable_cgcg(struct radeon_device *rdev,
5007 u32 data, orig, tmp;
5009 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5011 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5012 si_enable_gui_idle_interrupt(rdev, true);
5014 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5016 tmp = si_halt_rlc(rdev);
5018 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5019 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5020 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5022 si_wait_for_rlc_serdes(rdev);
5024 si_update_rlc(rdev, tmp);
5026 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5028 data |= CGCG_EN | CGLS_EN;
5030 si_enable_gui_idle_interrupt(rdev, false);
5032 RREG32(CB_CGTT_SCLK_CTRL);
5033 RREG32(CB_CGTT_SCLK_CTRL);
5034 RREG32(CB_CGTT_SCLK_CTRL);
5035 RREG32(CB_CGTT_SCLK_CTRL);
5037 data &= ~(CGCG_EN | CGLS_EN);
5041 WREG32(RLC_CGCG_CGLS_CTRL, data);
5044 static void si_enable_mgcg(struct radeon_device *rdev,
5047 u32 data, orig, tmp = 0;
5049 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5050 orig = data = RREG32(CGTS_SM_CTRL_REG);
5053 WREG32(CGTS_SM_CTRL_REG, data);
5055 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5056 orig = data = RREG32(CP_MEM_SLP_CNTL);
5057 data |= CP_MEM_LS_EN;
5059 WREG32(CP_MEM_SLP_CNTL, data);
5062 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5065 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5067 tmp = si_halt_rlc(rdev);
5069 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5070 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5071 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5073 si_update_rlc(rdev, tmp);
5075 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5078 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5080 data = RREG32(CP_MEM_SLP_CNTL);
5081 if (data & CP_MEM_LS_EN) {
5082 data &= ~CP_MEM_LS_EN;
5083 WREG32(CP_MEM_SLP_CNTL, data);
5085 orig = data = RREG32(CGTS_SM_CTRL_REG);
5086 data |= LS_OVERRIDE | OVERRIDE;
5088 WREG32(CGTS_SM_CTRL_REG, data);
5090 tmp = si_halt_rlc(rdev);
5092 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5093 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5094 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5096 si_update_rlc(rdev, tmp);
5100 static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5103 u32 orig, data, tmp;
5105 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5106 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5108 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5110 orig = data = RREG32(UVD_CGC_CTRL);
5113 WREG32(UVD_CGC_CTRL, data);
5115 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5116 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5118 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5120 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5122 orig = data = RREG32(UVD_CGC_CTRL);
5125 WREG32(UVD_CGC_CTRL, data);
5127 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5128 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5132 static const u32 mc_cg_registers[] =
5145 static void si_enable_mc_ls(struct radeon_device *rdev,
5151 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5152 orig = data = RREG32(mc_cg_registers[i]);
5153 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5154 data |= MC_LS_ENABLE;
5156 data &= ~MC_LS_ENABLE;
5158 WREG32(mc_cg_registers[i], data);
5162 static void si_enable_mc_mgcg(struct radeon_device *rdev,
5168 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5169 orig = data = RREG32(mc_cg_registers[i]);
5170 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5171 data |= MC_CG_ENABLE;
5173 data &= ~MC_CG_ENABLE;
5175 WREG32(mc_cg_registers[i], data);
5179 static void si_enable_dma_mgcg(struct radeon_device *rdev,
5182 u32 orig, data, offset;
5185 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5186 for (i = 0; i < 2; i++) {
5188 offset = DMA0_REGISTER_OFFSET;
5190 offset = DMA1_REGISTER_OFFSET;
5191 orig = data = RREG32(DMA_POWER_CNTL + offset);
5192 data &= ~MEM_POWER_OVERRIDE;
5194 WREG32(DMA_POWER_CNTL + offset, data);
5195 WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5198 for (i = 0; i < 2; i++) {
5200 offset = DMA0_REGISTER_OFFSET;
5202 offset = DMA1_REGISTER_OFFSET;
5203 orig = data = RREG32(DMA_POWER_CNTL + offset);
5204 data |= MEM_POWER_OVERRIDE;
5206 WREG32(DMA_POWER_CNTL + offset, data);
5208 orig = data = RREG32(DMA_CLK_CTRL + offset);
5211 WREG32(DMA_CLK_CTRL + offset, data);
5216 static void si_enable_bif_mgls(struct radeon_device *rdev,
5221 orig = data = RREG32_PCIE(PCIE_CNTL2);
5223 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5224 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5225 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5227 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5228 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5231 WREG32_PCIE(PCIE_CNTL2, data);
5234 static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5239 orig = data = RREG32(HDP_HOST_PATH_CNTL);
5241 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5242 data &= ~CLOCK_GATING_DIS;
5244 data |= CLOCK_GATING_DIS;
5247 WREG32(HDP_HOST_PATH_CNTL, data);
5250 static void si_enable_hdp_ls(struct radeon_device *rdev,
5255 orig = data = RREG32(HDP_MEM_POWER_LS);
5257 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5258 data |= HDP_LS_ENABLE;
5260 data &= ~HDP_LS_ENABLE;
5263 WREG32(HDP_MEM_POWER_LS, data);
5266 void si_update_cg(struct radeon_device *rdev,
5267 u32 block, bool enable)
5269 if (block & RADEON_CG_BLOCK_GFX) {
5270 si_enable_gui_idle_interrupt(rdev, false);
5271 /* order matters! */
5273 si_enable_mgcg(rdev, true);
5274 si_enable_cgcg(rdev, true);
5276 si_enable_cgcg(rdev, false);
5277 si_enable_mgcg(rdev, false);
5279 si_enable_gui_idle_interrupt(rdev, true);
5282 if (block & RADEON_CG_BLOCK_MC) {
5283 si_enable_mc_mgcg(rdev, enable);
5284 si_enable_mc_ls(rdev, enable);
5287 if (block & RADEON_CG_BLOCK_SDMA) {
5288 si_enable_dma_mgcg(rdev, enable);
5291 if (block & RADEON_CG_BLOCK_BIF) {
5292 si_enable_bif_mgls(rdev, enable);
5295 if (block & RADEON_CG_BLOCK_UVD) {
5296 if (rdev->has_uvd) {
5297 si_enable_uvd_mgcg(rdev, enable);
5301 if (block & RADEON_CG_BLOCK_HDP) {
5302 si_enable_hdp_mgcg(rdev, enable);
5303 si_enable_hdp_ls(rdev, enable);
5307 static void si_init_cg(struct radeon_device *rdev)
5309 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5310 RADEON_CG_BLOCK_MC |
5311 RADEON_CG_BLOCK_SDMA |
5312 RADEON_CG_BLOCK_BIF |
5313 RADEON_CG_BLOCK_HDP), true);
5314 if (rdev->has_uvd) {
5315 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5316 si_init_uvd_internal_cg(rdev);
5320 static void si_fini_cg(struct radeon_device *rdev)
5322 if (rdev->has_uvd) {
5323 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5325 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5326 RADEON_CG_BLOCK_MC |
5327 RADEON_CG_BLOCK_SDMA |
5328 RADEON_CG_BLOCK_BIF |
5329 RADEON_CG_BLOCK_HDP), false);
5332 u32 si_get_csb_size(struct radeon_device *rdev)
5335 const struct cs_section_def *sect = NULL;
5336 const struct cs_extent_def *ext = NULL;
5338 if (rdev->rlc.cs_data == NULL)
5341 /* begin clear state */
5343 /* context control state */
5346 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5347 for (ext = sect->section; ext->extent != NULL; ++ext) {
5348 if (sect->id == SECT_CONTEXT)
5349 count += 2 + ext->reg_count;
5354 /* pa_sc_raster_config */
5356 /* end clear state */
5364 void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5367 const struct cs_section_def *sect = NULL;
5368 const struct cs_extent_def *ext = NULL;
5370 if (rdev->rlc.cs_data == NULL)
5375 buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5376 buffer[count++] = PACKET3_PREAMBLE_BEGIN_CLEAR_STATE;
5378 buffer[count++] = PACKET3(PACKET3_CONTEXT_CONTROL, 1);
5379 buffer[count++] = 0x80000000;
5380 buffer[count++] = 0x80000000;
5382 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5383 for (ext = sect->section; ext->extent != NULL; ++ext) {
5384 if (sect->id == SECT_CONTEXT) {
5385 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count);
5386 buffer[count++] = ext->reg_index - 0xa000;
5387 for (i = 0; i < ext->reg_count; i++)
5388 buffer[count++] = ext->extent[i];
5395 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, 1);
5396 buffer[count++] = PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START;
5397 switch (rdev->family) {
5400 buffer[count++] = 0x2a00126a;
5403 buffer[count++] = 0x0000124a;
5406 buffer[count++] = 0x00000082;
5409 buffer[count++] = 0x00000000;
5412 buffer[count++] = 0x00000000;
5416 buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5417 buffer[count++] = PACKET3_PREAMBLE_END_CLEAR_STATE;
5419 buffer[count++] = PACKET3(PACKET3_CLEAR_STATE, 0);
5420 buffer[count++] = 0;
5423 static void si_init_pg(struct radeon_device *rdev)
5425 if (rdev->pg_flags) {
5426 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5427 si_init_dma_pg(rdev);
5429 si_init_ao_cu_mask(rdev);
5430 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5431 si_init_gfx_cgpg(rdev);
5433 si_enable_dma_pg(rdev, true);
5434 si_enable_gfx_cgpg(rdev, true);
5436 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5437 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5441 static void si_fini_pg(struct radeon_device *rdev)
5443 if (rdev->pg_flags) {
5444 si_enable_dma_pg(rdev, false);
5445 si_enable_gfx_cgpg(rdev, false);
5452 void si_rlc_reset(struct radeon_device *rdev)
5454 u32 tmp = RREG32(GRBM_SOFT_RESET);
5456 tmp |= SOFT_RESET_RLC;
5457 WREG32(GRBM_SOFT_RESET, tmp);
5459 tmp &= ~SOFT_RESET_RLC;
5460 WREG32(GRBM_SOFT_RESET, tmp);
5464 static void si_rlc_stop(struct radeon_device *rdev)
5466 WREG32(RLC_CNTL, 0);
5468 si_enable_gui_idle_interrupt(rdev, false);
5470 si_wait_for_rlc_serdes(rdev);
5473 static void si_rlc_start(struct radeon_device *rdev)
5475 WREG32(RLC_CNTL, RLC_ENABLE);
5477 si_enable_gui_idle_interrupt(rdev, true);
5482 static bool si_lbpw_supported(struct radeon_device *rdev)
5486 /* Enable LBPW only for DDR3 */
5487 tmp = RREG32(MC_SEQ_MISC0);
5488 if ((tmp & 0xF0000000) == 0xB0000000)
5493 static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5497 tmp = RREG32(RLC_LB_CNTL);
5499 tmp |= LOAD_BALANCE_ENABLE;
5501 tmp &= ~LOAD_BALANCE_ENABLE;
5502 WREG32(RLC_LB_CNTL, tmp);
5505 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5506 WREG32(SPI_LB_CU_MASK, 0x00ff);
5510 static int si_rlc_resume(struct radeon_device *rdev)
5513 const __be32 *fw_data;
5526 WREG32(RLC_RL_BASE, 0);
5527 WREG32(RLC_RL_SIZE, 0);
5528 WREG32(RLC_LB_CNTL, 0);
5529 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5530 WREG32(RLC_LB_CNTR_INIT, 0);
5531 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5533 WREG32(RLC_MC_CNTL, 0);
5534 WREG32(RLC_UCODE_CNTL, 0);
5536 fw_data = (const __be32 *)rdev->rlc_fw->data;
5537 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5538 WREG32(RLC_UCODE_ADDR, i);
5539 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5541 WREG32(RLC_UCODE_ADDR, 0);
5543 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5550 static void si_enable_interrupts(struct radeon_device *rdev)
5552 u32 ih_cntl = RREG32(IH_CNTL);
5553 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5555 ih_cntl |= ENABLE_INTR;
5556 ih_rb_cntl |= IH_RB_ENABLE;
5557 WREG32(IH_CNTL, ih_cntl);
5558 WREG32(IH_RB_CNTL, ih_rb_cntl);
5559 rdev->ih.enabled = true;
5562 static void si_disable_interrupts(struct radeon_device *rdev)
5564 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5565 u32 ih_cntl = RREG32(IH_CNTL);
5567 ih_rb_cntl &= ~IH_RB_ENABLE;
5568 ih_cntl &= ~ENABLE_INTR;
5569 WREG32(IH_RB_CNTL, ih_rb_cntl);
5570 WREG32(IH_CNTL, ih_cntl);
5571 /* set rptr, wptr to 0 */
5572 WREG32(IH_RB_RPTR, 0);
5573 WREG32(IH_RB_WPTR, 0);
5574 rdev->ih.enabled = false;
5578 static void si_disable_interrupt_state(struct radeon_device *rdev)
5582 tmp = RREG32(CP_INT_CNTL_RING0) &
5583 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5584 WREG32(CP_INT_CNTL_RING0, tmp);
5585 WREG32(CP_INT_CNTL_RING1, 0);
5586 WREG32(CP_INT_CNTL_RING2, 0);
5587 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5588 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5589 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5590 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5591 WREG32(GRBM_INT_CNTL, 0);
5592 if (rdev->num_crtc >= 2) {
5593 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5594 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5596 if (rdev->num_crtc >= 4) {
5597 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5598 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5600 if (rdev->num_crtc >= 6) {
5601 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5602 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5605 if (rdev->num_crtc >= 2) {
5606 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5607 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5609 if (rdev->num_crtc >= 4) {
5610 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5611 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5613 if (rdev->num_crtc >= 6) {
5614 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5615 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5618 if (!ASIC_IS_NODCE(rdev)) {
5619 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
5621 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5622 WREG32(DC_HPD1_INT_CONTROL, tmp);
5623 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5624 WREG32(DC_HPD2_INT_CONTROL, tmp);
5625 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5626 WREG32(DC_HPD3_INT_CONTROL, tmp);
5627 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5628 WREG32(DC_HPD4_INT_CONTROL, tmp);
5629 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5630 WREG32(DC_HPD5_INT_CONTROL, tmp);
5631 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5632 WREG32(DC_HPD6_INT_CONTROL, tmp);
5636 static int si_irq_init(struct radeon_device *rdev)
5640 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5643 ret = r600_ih_ring_alloc(rdev);
5648 si_disable_interrupts(rdev);
5651 ret = si_rlc_resume(rdev);
5653 r600_ih_ring_fini(rdev);
5657 /* setup interrupt control */
5658 /* set dummy read address to ring address */
5659 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5660 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5661 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5662 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5664 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5665 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5666 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5667 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5669 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5670 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
5672 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5673 IH_WPTR_OVERFLOW_CLEAR |
5676 if (rdev->wb.enabled)
5677 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5679 /* set the writeback address whether it's enabled or not */
5680 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5681 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5683 WREG32(IH_RB_CNTL, ih_rb_cntl);
5685 /* set rptr, wptr to 0 */
5686 WREG32(IH_RB_RPTR, 0);
5687 WREG32(IH_RB_WPTR, 0);
5689 /* Default settings for IH_CNTL (disabled at first) */
5690 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5691 /* RPTR_REARM only works if msi's are enabled */
5692 if (rdev->msi_enabled)
5693 ih_cntl |= RPTR_REARM;
5694 WREG32(IH_CNTL, ih_cntl);
5696 /* force the active interrupt state to all disabled */
5697 si_disable_interrupt_state(rdev);
5699 pci_set_master(rdev->pdev);
5702 si_enable_interrupts(rdev);
5707 int si_irq_set(struct radeon_device *rdev)
5710 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5711 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5712 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
5713 u32 grbm_int_cntl = 0;
5714 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
5715 u32 dma_cntl, dma_cntl1;
5716 u32 thermal_int = 0;
5718 if (!rdev->irq.installed) {
5719 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5722 /* don't enable anything if the ih is disabled */
5723 if (!rdev->ih.enabled) {
5724 si_disable_interrupts(rdev);
5725 /* force the active interrupt state to all disabled */
5726 si_disable_interrupt_state(rdev);
5730 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
5731 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5733 if (!ASIC_IS_NODCE(rdev)) {
5734 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5735 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5736 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5737 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5738 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5739 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5742 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5743 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5745 thermal_int = RREG32(CG_THERMAL_INT) &
5746 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
5748 /* enable CP interrupts on all rings */
5749 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
5750 DRM_DEBUG("si_irq_set: sw int gfx\n");
5751 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5753 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
5754 DRM_DEBUG("si_irq_set: sw int cp1\n");
5755 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
5757 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
5758 DRM_DEBUG("si_irq_set: sw int cp2\n");
5759 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
5761 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5762 DRM_DEBUG("si_irq_set: sw int dma\n");
5763 dma_cntl |= TRAP_ENABLE;
5766 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5767 DRM_DEBUG("si_irq_set: sw int dma1\n");
5768 dma_cntl1 |= TRAP_ENABLE;
5770 if (rdev->irq.crtc_vblank_int[0] ||
5771 atomic_read(&rdev->irq.pflip[0])) {
5772 DRM_DEBUG("si_irq_set: vblank 0\n");
5773 crtc1 |= VBLANK_INT_MASK;
5775 if (rdev->irq.crtc_vblank_int[1] ||
5776 atomic_read(&rdev->irq.pflip[1])) {
5777 DRM_DEBUG("si_irq_set: vblank 1\n");
5778 crtc2 |= VBLANK_INT_MASK;
5780 if (rdev->irq.crtc_vblank_int[2] ||
5781 atomic_read(&rdev->irq.pflip[2])) {
5782 DRM_DEBUG("si_irq_set: vblank 2\n");
5783 crtc3 |= VBLANK_INT_MASK;
5785 if (rdev->irq.crtc_vblank_int[3] ||
5786 atomic_read(&rdev->irq.pflip[3])) {
5787 DRM_DEBUG("si_irq_set: vblank 3\n");
5788 crtc4 |= VBLANK_INT_MASK;
5790 if (rdev->irq.crtc_vblank_int[4] ||
5791 atomic_read(&rdev->irq.pflip[4])) {
5792 DRM_DEBUG("si_irq_set: vblank 4\n");
5793 crtc5 |= VBLANK_INT_MASK;
5795 if (rdev->irq.crtc_vblank_int[5] ||
5796 atomic_read(&rdev->irq.pflip[5])) {
5797 DRM_DEBUG("si_irq_set: vblank 5\n");
5798 crtc6 |= VBLANK_INT_MASK;
5800 if (rdev->irq.hpd[0]) {
5801 DRM_DEBUG("si_irq_set: hpd 1\n");
5802 hpd1 |= DC_HPDx_INT_EN;
5804 if (rdev->irq.hpd[1]) {
5805 DRM_DEBUG("si_irq_set: hpd 2\n");
5806 hpd2 |= DC_HPDx_INT_EN;
5808 if (rdev->irq.hpd[2]) {
5809 DRM_DEBUG("si_irq_set: hpd 3\n");
5810 hpd3 |= DC_HPDx_INT_EN;
5812 if (rdev->irq.hpd[3]) {
5813 DRM_DEBUG("si_irq_set: hpd 4\n");
5814 hpd4 |= DC_HPDx_INT_EN;
5816 if (rdev->irq.hpd[4]) {
5817 DRM_DEBUG("si_irq_set: hpd 5\n");
5818 hpd5 |= DC_HPDx_INT_EN;
5820 if (rdev->irq.hpd[5]) {
5821 DRM_DEBUG("si_irq_set: hpd 6\n");
5822 hpd6 |= DC_HPDx_INT_EN;
5825 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5826 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
5827 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
5829 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
5830 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
5832 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5834 if (rdev->irq.dpm_thermal) {
5835 DRM_DEBUG("dpm thermal\n");
5836 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
5839 if (rdev->num_crtc >= 2) {
5840 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5841 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5843 if (rdev->num_crtc >= 4) {
5844 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5845 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5847 if (rdev->num_crtc >= 6) {
5848 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5849 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5852 if (rdev->num_crtc >= 2) {
5853 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
5854 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
5856 if (rdev->num_crtc >= 4) {
5857 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
5858 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
5860 if (rdev->num_crtc >= 6) {
5861 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
5862 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
5865 if (!ASIC_IS_NODCE(rdev)) {
5866 WREG32(DC_HPD1_INT_CONTROL, hpd1);
5867 WREG32(DC_HPD2_INT_CONTROL, hpd2);
5868 WREG32(DC_HPD3_INT_CONTROL, hpd3);
5869 WREG32(DC_HPD4_INT_CONTROL, hpd4);
5870 WREG32(DC_HPD5_INT_CONTROL, hpd5);
5871 WREG32(DC_HPD6_INT_CONTROL, hpd6);
5874 WREG32(CG_THERMAL_INT, thermal_int);
5879 static inline void si_irq_ack(struct radeon_device *rdev)
5883 if (ASIC_IS_NODCE(rdev))
5886 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
5887 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
5888 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
5889 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
5890 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
5891 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
5892 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
5893 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
5894 if (rdev->num_crtc >= 4) {
5895 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
5896 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
5898 if (rdev->num_crtc >= 6) {
5899 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
5900 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
5903 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
5904 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5905 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
5906 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5907 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
5908 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
5909 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
5910 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
5911 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
5912 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
5913 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
5914 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
5916 if (rdev->num_crtc >= 4) {
5917 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
5918 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5919 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
5920 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5921 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
5922 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
5923 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
5924 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
5925 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
5926 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
5927 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
5928 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
5931 if (rdev->num_crtc >= 6) {
5932 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
5933 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5934 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
5935 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5936 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
5937 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
5938 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
5939 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
5940 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
5941 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
5942 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
5943 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
5946 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5947 tmp = RREG32(DC_HPD1_INT_CONTROL);
5948 tmp |= DC_HPDx_INT_ACK;
5949 WREG32(DC_HPD1_INT_CONTROL, tmp);
5951 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5952 tmp = RREG32(DC_HPD2_INT_CONTROL);
5953 tmp |= DC_HPDx_INT_ACK;
5954 WREG32(DC_HPD2_INT_CONTROL, tmp);
5956 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5957 tmp = RREG32(DC_HPD3_INT_CONTROL);
5958 tmp |= DC_HPDx_INT_ACK;
5959 WREG32(DC_HPD3_INT_CONTROL, tmp);
5961 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5962 tmp = RREG32(DC_HPD4_INT_CONTROL);
5963 tmp |= DC_HPDx_INT_ACK;
5964 WREG32(DC_HPD4_INT_CONTROL, tmp);
5966 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5967 tmp = RREG32(DC_HPD5_INT_CONTROL);
5968 tmp |= DC_HPDx_INT_ACK;
5969 WREG32(DC_HPD5_INT_CONTROL, tmp);
5971 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5972 tmp = RREG32(DC_HPD5_INT_CONTROL);
5973 tmp |= DC_HPDx_INT_ACK;
5974 WREG32(DC_HPD6_INT_CONTROL, tmp);
5978 static void si_irq_disable(struct radeon_device *rdev)
5980 si_disable_interrupts(rdev);
5981 /* Wait and acknowledge irq */
5984 si_disable_interrupt_state(rdev);
5987 static void si_irq_suspend(struct radeon_device *rdev)
5989 si_irq_disable(rdev);
5993 static void si_irq_fini(struct radeon_device *rdev)
5995 si_irq_suspend(rdev);
5996 r600_ih_ring_fini(rdev);
5999 static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6003 if (rdev->wb.enabled)
6004 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6006 wptr = RREG32(IH_RB_WPTR);
6008 if (wptr & RB_OVERFLOW) {
6009 /* When a ring buffer overflow happen start parsing interrupt
6010 * from the last not overwritten vector (wptr + 16). Hopefully
6011 * this should allow us to catchup.
6013 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6014 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6015 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6016 tmp = RREG32(IH_RB_CNTL);
6017 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6018 WREG32(IH_RB_CNTL, tmp);
6020 return (wptr & rdev->ih.ptr_mask);
6024 * Each IV ring entry is 128 bits:
6025 * [7:0] - interrupt source id
6027 * [59:32] - interrupt source data
6028 * [63:60] - reserved
6031 * [127:80] - reserved
6033 int si_irq_process(struct radeon_device *rdev)
6037 u32 src_id, src_data, ring_id;
6039 bool queue_hotplug = false;
6040 bool queue_thermal = false;
6043 if (!rdev->ih.enabled || rdev->shutdown)
6046 wptr = si_get_ih_wptr(rdev);
6049 /* is somebody else already processing irqs? */
6050 if (atomic_xchg(&rdev->ih.lock, 1))
6053 rptr = rdev->ih.rptr;
6054 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6056 /* Order reading of wptr vs. reading of IH ring data */
6059 /* display interrupts */
6062 while (rptr != wptr) {
6063 /* wptr/rptr are in bytes! */
6064 ring_index = rptr / 4;
6065 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6066 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6067 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6070 case 1: /* D1 vblank/vline */
6072 case 0: /* D1 vblank */
6073 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6074 if (rdev->irq.crtc_vblank_int[0]) {
6075 drm_handle_vblank(rdev->ddev, 0);
6076 rdev->pm.vblank_sync = true;
6077 wake_up(&rdev->irq.vblank_queue);
6079 if (atomic_read(&rdev->irq.pflip[0]))
6080 radeon_crtc_handle_flip(rdev, 0);
6081 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6082 DRM_DEBUG("IH: D1 vblank\n");
6085 case 1: /* D1 vline */
6086 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
6087 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6088 DRM_DEBUG("IH: D1 vline\n");
6092 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6096 case 2: /* D2 vblank/vline */
6098 case 0: /* D2 vblank */
6099 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6100 if (rdev->irq.crtc_vblank_int[1]) {
6101 drm_handle_vblank(rdev->ddev, 1);
6102 rdev->pm.vblank_sync = true;
6103 wake_up(&rdev->irq.vblank_queue);
6105 if (atomic_read(&rdev->irq.pflip[1]))
6106 radeon_crtc_handle_flip(rdev, 1);
6107 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6108 DRM_DEBUG("IH: D2 vblank\n");
6111 case 1: /* D2 vline */
6112 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6113 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6114 DRM_DEBUG("IH: D2 vline\n");
6118 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6122 case 3: /* D3 vblank/vline */
6124 case 0: /* D3 vblank */
6125 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6126 if (rdev->irq.crtc_vblank_int[2]) {
6127 drm_handle_vblank(rdev->ddev, 2);
6128 rdev->pm.vblank_sync = true;
6129 wake_up(&rdev->irq.vblank_queue);
6131 if (atomic_read(&rdev->irq.pflip[2]))
6132 radeon_crtc_handle_flip(rdev, 2);
6133 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6134 DRM_DEBUG("IH: D3 vblank\n");
6137 case 1: /* D3 vline */
6138 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6139 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6140 DRM_DEBUG("IH: D3 vline\n");
6144 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6148 case 4: /* D4 vblank/vline */
6150 case 0: /* D4 vblank */
6151 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6152 if (rdev->irq.crtc_vblank_int[3]) {
6153 drm_handle_vblank(rdev->ddev, 3);
6154 rdev->pm.vblank_sync = true;
6155 wake_up(&rdev->irq.vblank_queue);
6157 if (atomic_read(&rdev->irq.pflip[3]))
6158 radeon_crtc_handle_flip(rdev, 3);
6159 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6160 DRM_DEBUG("IH: D4 vblank\n");
6163 case 1: /* D4 vline */
6164 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6165 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6166 DRM_DEBUG("IH: D4 vline\n");
6170 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6174 case 5: /* D5 vblank/vline */
6176 case 0: /* D5 vblank */
6177 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6178 if (rdev->irq.crtc_vblank_int[4]) {
6179 drm_handle_vblank(rdev->ddev, 4);
6180 rdev->pm.vblank_sync = true;
6181 wake_up(&rdev->irq.vblank_queue);
6183 if (atomic_read(&rdev->irq.pflip[4]))
6184 radeon_crtc_handle_flip(rdev, 4);
6185 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6186 DRM_DEBUG("IH: D5 vblank\n");
6189 case 1: /* D5 vline */
6190 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6191 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6192 DRM_DEBUG("IH: D5 vline\n");
6196 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6200 case 6: /* D6 vblank/vline */
6202 case 0: /* D6 vblank */
6203 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6204 if (rdev->irq.crtc_vblank_int[5]) {
6205 drm_handle_vblank(rdev->ddev, 5);
6206 rdev->pm.vblank_sync = true;
6207 wake_up(&rdev->irq.vblank_queue);
6209 if (atomic_read(&rdev->irq.pflip[5]))
6210 radeon_crtc_handle_flip(rdev, 5);
6211 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6212 DRM_DEBUG("IH: D6 vblank\n");
6215 case 1: /* D6 vline */
6216 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6217 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6218 DRM_DEBUG("IH: D6 vline\n");
6222 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6226 case 42: /* HPD hotplug */
6229 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6230 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6231 queue_hotplug = true;
6232 DRM_DEBUG("IH: HPD1\n");
6236 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6237 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6238 queue_hotplug = true;
6239 DRM_DEBUG("IH: HPD2\n");
6243 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6244 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6245 queue_hotplug = true;
6246 DRM_DEBUG("IH: HPD3\n");
6250 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6251 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6252 queue_hotplug = true;
6253 DRM_DEBUG("IH: HPD4\n");
6257 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6258 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6259 queue_hotplug = true;
6260 DRM_DEBUG("IH: HPD5\n");
6264 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6265 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6266 queue_hotplug = true;
6267 DRM_DEBUG("IH: HPD6\n");
6271 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6277 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6278 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6279 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6280 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
6282 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6284 si_vm_decode_fault(rdev, status, addr);
6285 /* reset addr and status */
6286 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6288 case 176: /* RINGID0 CP_INT */
6289 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6291 case 177: /* RINGID1 CP_INT */
6292 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6294 case 178: /* RINGID2 CP_INT */
6295 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6297 case 181: /* CP EOP event */
6298 DRM_DEBUG("IH: CP EOP\n");
6301 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6304 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6307 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6311 case 224: /* DMA trap event */
6312 DRM_DEBUG("IH: DMA trap\n");
6313 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6315 case 230: /* thermal low to high */
6316 DRM_DEBUG("IH: thermal low to high\n");
6317 rdev->pm.dpm.thermal.high_to_low = false;
6318 queue_thermal = true;
6320 case 231: /* thermal high to low */
6321 DRM_DEBUG("IH: thermal high to low\n");
6322 rdev->pm.dpm.thermal.high_to_low = true;
6323 queue_thermal = true;
6325 case 233: /* GUI IDLE */
6326 DRM_DEBUG("IH: GUI idle\n");
6328 case 244: /* DMA trap event */
6329 DRM_DEBUG("IH: DMA1 trap\n");
6330 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6333 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6337 /* wptr/rptr are in bytes! */
6339 rptr &= rdev->ih.ptr_mask;
6342 schedule_work(&rdev->hotplug_work);
6343 if (queue_thermal && rdev->pm.dpm_enabled)
6344 schedule_work(&rdev->pm.dpm.thermal.work);
6345 rdev->ih.rptr = rptr;
6346 WREG32(IH_RB_RPTR, rdev->ih.rptr);
6347 atomic_set(&rdev->ih.lock, 0);
6349 /* make sure wptr hasn't changed while processing */
6350 wptr = si_get_ih_wptr(rdev);
6358 * startup/shutdown callbacks
6360 static int si_startup(struct radeon_device *rdev)
6362 struct radeon_ring *ring;
6365 /* enable pcie gen2/3 link */
6366 si_pcie_gen3_enable(rdev);
6368 si_program_aspm(rdev);
6370 /* scratch needs to be initialized before MC */
6371 r = r600_vram_scratch_init(rdev);
6375 si_mc_program(rdev);
6377 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6378 !rdev->rlc_fw || !rdev->mc_fw) {
6379 r = si_init_microcode(rdev);
6381 DRM_ERROR("Failed to load firmware!\n");
6386 r = si_mc_load_microcode(rdev);
6388 DRM_ERROR("Failed to load MC firmware!\n");
6392 r = si_pcie_gart_enable(rdev);
6397 /* allocate rlc buffers */
6398 if (rdev->family == CHIP_VERDE) {
6399 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6400 rdev->rlc.reg_list_size =
6401 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6403 rdev->rlc.cs_data = si_cs_data;
6404 r = sumo_rlc_init(rdev);
6406 DRM_ERROR("Failed to init rlc BOs!\n");
6410 /* allocate wb buffer */
6411 r = radeon_wb_init(rdev);
6415 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6417 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6421 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6423 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6427 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6429 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6433 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6435 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6439 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6441 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6445 if (rdev->has_uvd) {
6446 r = uvd_v2_2_resume(rdev);
6448 r = radeon_fence_driver_start_ring(rdev,
6449 R600_RING_TYPE_UVD_INDEX);
6451 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6454 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6458 if (!rdev->irq.installed) {
6459 r = radeon_irq_kms_init(rdev);
6464 r = si_irq_init(rdev);
6466 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6467 radeon_irq_kms_fini(rdev);
6472 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6473 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6474 CP_RB0_RPTR, CP_RB0_WPTR,
6479 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6480 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6481 CP_RB1_RPTR, CP_RB1_WPTR,
6486 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6487 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6488 CP_RB2_RPTR, CP_RB2_WPTR,
6493 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6494 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6495 DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
6496 DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
6497 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6501 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6502 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6503 DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
6504 DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
6505 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6509 r = si_cp_load_microcode(rdev);
6512 r = si_cp_resume(rdev);
6516 r = cayman_dma_resume(rdev);
6520 if (rdev->has_uvd) {
6521 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6522 if (ring->ring_size) {
6523 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
6524 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
6527 r = uvd_v1_0_init(rdev);
6529 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6533 r = radeon_ib_pool_init(rdev);
6535 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
6539 r = radeon_vm_manager_init(rdev);
6541 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
6545 r = dce6_audio_init(rdev);
6552 int si_resume(struct radeon_device *rdev)
6556 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6557 * posting will perform necessary task to bring back GPU into good
6561 atom_asic_init(rdev->mode_info.atom_context);
6563 /* init golden registers */
6564 si_init_golden_registers(rdev);
6566 rdev->accel_working = true;
6567 r = si_startup(rdev);
6569 DRM_ERROR("si startup failed on resume\n");
6570 rdev->accel_working = false;
6578 int si_suspend(struct radeon_device *rdev)
6580 dce6_audio_fini(rdev);
6581 radeon_vm_manager_fini(rdev);
6582 si_cp_enable(rdev, false);
6583 cayman_dma_stop(rdev);
6584 if (rdev->has_uvd) {
6585 uvd_v1_0_fini(rdev);
6586 radeon_uvd_suspend(rdev);
6590 si_irq_suspend(rdev);
6591 radeon_wb_disable(rdev);
6592 si_pcie_gart_disable(rdev);
6596 /* Plan is to move initialization in that function and use
6597 * helper function so that radeon_device_init pretty much
6598 * do nothing more than calling asic specific function. This
6599 * should also allow to remove a bunch of callback function
6602 int si_init(struct radeon_device *rdev)
6604 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6608 if (!radeon_get_bios(rdev)) {
6609 if (ASIC_IS_AVIVO(rdev))
6612 /* Must be an ATOMBIOS */
6613 if (!rdev->is_atom_bios) {
6614 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6617 r = radeon_atombios_init(rdev);
6621 /* Post card if necessary */
6622 if (!radeon_card_posted(rdev)) {
6624 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6627 DRM_INFO("GPU not posted. posting now...\n");
6628 atom_asic_init(rdev->mode_info.atom_context);
6630 /* init golden registers */
6631 si_init_golden_registers(rdev);
6632 /* Initialize scratch registers */
6633 si_scratch_init(rdev);
6634 /* Initialize surface registers */
6635 radeon_surface_init(rdev);
6636 /* Initialize clocks */
6637 radeon_get_clock_info(rdev->ddev);
6640 r = radeon_fence_driver_init(rdev);
6644 /* initialize memory controller */
6645 r = si_mc_init(rdev);
6648 /* Memory manager */
6649 r = radeon_bo_init(rdev);
6653 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6654 ring->ring_obj = NULL;
6655 r600_ring_init(rdev, ring, 1024 * 1024);
6657 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6658 ring->ring_obj = NULL;
6659 r600_ring_init(rdev, ring, 1024 * 1024);
6661 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6662 ring->ring_obj = NULL;
6663 r600_ring_init(rdev, ring, 1024 * 1024);
6665 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6666 ring->ring_obj = NULL;
6667 r600_ring_init(rdev, ring, 64 * 1024);
6669 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6670 ring->ring_obj = NULL;
6671 r600_ring_init(rdev, ring, 64 * 1024);
6673 if (rdev->has_uvd) {
6674 r = radeon_uvd_init(rdev);
6676 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6677 ring->ring_obj = NULL;
6678 r600_ring_init(rdev, ring, 4096);
6682 rdev->ih.ring_obj = NULL;
6683 r600_ih_ring_init(rdev, 64 * 1024);
6685 r = r600_pcie_gart_init(rdev);
6689 rdev->accel_working = true;
6690 r = si_startup(rdev);
6692 dev_err(rdev->dev, "disabling GPU acceleration\n");
6694 cayman_dma_fini(rdev);
6696 sumo_rlc_fini(rdev);
6697 radeon_wb_fini(rdev);
6698 radeon_ib_pool_fini(rdev);
6699 radeon_vm_manager_fini(rdev);
6700 radeon_irq_kms_fini(rdev);
6701 si_pcie_gart_fini(rdev);
6702 rdev->accel_working = false;
6705 /* Don't start up if the MC ucode is missing.
6706 * The default clocks and voltages before the MC ucode
6707 * is loaded are not suffient for advanced operations.
6710 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6717 void si_fini(struct radeon_device *rdev)
6720 cayman_dma_fini(rdev);
6724 sumo_rlc_fini(rdev);
6725 radeon_wb_fini(rdev);
6726 radeon_vm_manager_fini(rdev);
6727 radeon_ib_pool_fini(rdev);
6728 radeon_irq_kms_fini(rdev);
6729 if (rdev->has_uvd) {
6730 uvd_v1_0_fini(rdev);
6731 radeon_uvd_fini(rdev);
6733 si_pcie_gart_fini(rdev);
6734 r600_vram_scratch_fini(rdev);
6735 radeon_gem_fini(rdev);
6736 radeon_fence_driver_fini(rdev);
6737 radeon_bo_fini(rdev);
6738 radeon_atombios_fini(rdev);
6744 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6746 * @rdev: radeon_device pointer
6748 * Fetches a GPU clock counter snapshot (SI).
6749 * Returns the 64 bit clock counter snapshot.
6751 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6755 mutex_lock(&rdev->gpu_clock_mutex);
6756 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6757 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6758 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6759 mutex_unlock(&rdev->gpu_clock_mutex);
6763 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6765 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
6768 /* bypass vclk and dclk with bclk */
6769 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6770 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6771 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6773 /* put PLL in bypass mode */
6774 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6776 if (!vclk || !dclk) {
6777 /* keep the Bypass mode, put PLL to sleep */
6778 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6782 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6783 16384, 0x03FFFFFF, 0, 128, 5,
6784 &fb_div, &vclk_div, &dclk_div);
6788 /* set RESET_ANTI_MUX to 0 */
6789 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
6791 /* set VCO_MODE to 1 */
6792 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
6794 /* toggle UPLL_SLEEP to 1 then back to 0 */
6795 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6796 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
6798 /* deassert UPLL_RESET */
6799 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6803 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
6807 /* assert UPLL_RESET again */
6808 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
6810 /* disable spread spectrum. */
6811 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
6813 /* set feedback divider */
6814 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
6816 /* set ref divider to 0 */
6817 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
6819 if (fb_div < 307200)
6820 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
6822 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
6824 /* set PDIV_A and PDIV_B */
6825 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6826 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
6827 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
6829 /* give the PLL some time to settle */
6832 /* deassert PLL_RESET */
6833 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6837 /* switch from bypass mode to normal mode */
6838 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
6840 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
6844 /* switch VCLK and DCLK selection */
6845 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6846 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6847 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6854 static void si_pcie_gen3_enable(struct radeon_device *rdev)
6856 struct pci_dev *root = rdev->pdev->bus->self;
6857 int bridge_pos, gpu_pos;
6858 u32 speed_cntl, mask, current_data_rate;
6862 if (radeon_pcie_gen2 == 0)
6865 if (rdev->flags & RADEON_IS_IGP)
6868 if (!(rdev->flags & RADEON_IS_PCIE))
6871 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
6875 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
6878 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6879 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
6880 LC_CURRENT_DATA_RATE_SHIFT;
6881 if (mask & DRM_PCIE_SPEED_80) {
6882 if (current_data_rate == 2) {
6883 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
6886 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
6887 } else if (mask & DRM_PCIE_SPEED_50) {
6888 if (current_data_rate == 1) {
6889 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
6892 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
6895 bridge_pos = pci_pcie_cap(root);
6899 gpu_pos = pci_pcie_cap(rdev->pdev);
6903 if (mask & DRM_PCIE_SPEED_80) {
6904 /* re-try equalization if gen3 is not already enabled */
6905 if (current_data_rate != 2) {
6906 u16 bridge_cfg, gpu_cfg;
6907 u16 bridge_cfg2, gpu_cfg2;
6908 u32 max_lw, current_lw, tmp;
6910 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6911 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6913 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
6914 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6916 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
6917 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6919 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
6920 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
6921 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
6923 if (current_lw < max_lw) {
6924 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6925 if (tmp & LC_RENEGOTIATION_SUPPORT) {
6926 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
6927 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
6928 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
6929 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
6933 for (i = 0; i < 10; i++) {
6935 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
6936 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
6939 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6940 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6942 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
6943 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
6945 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6946 tmp |= LC_SET_QUIESCE;
6947 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6949 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6951 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6956 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
6957 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6958 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
6959 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6961 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
6962 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6963 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
6964 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6967 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
6968 tmp16 &= ~((1 << 4) | (7 << 9));
6969 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
6970 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
6972 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6973 tmp16 &= ~((1 << 4) | (7 << 9));
6974 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
6975 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6977 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6978 tmp &= ~LC_SET_QUIESCE;
6979 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6984 /* set the link speed */
6985 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
6986 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
6987 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6989 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6991 if (mask & DRM_PCIE_SPEED_80)
6992 tmp16 |= 3; /* gen3 */
6993 else if (mask & DRM_PCIE_SPEED_50)
6994 tmp16 |= 2; /* gen2 */
6996 tmp16 |= 1; /* gen1 */
6997 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6999 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7000 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7001 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7003 for (i = 0; i < rdev->usec_timeout; i++) {
7004 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7005 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7011 static void si_program_aspm(struct radeon_device *rdev)
7014 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7015 bool disable_clkreq = false;
7017 if (radeon_aspm == 0)
7020 if (!(rdev->flags & RADEON_IS_PCIE))
7023 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7024 data &= ~LC_XMIT_N_FTS_MASK;
7025 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7027 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7029 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7030 data |= LC_GO_TO_RECOVERY;
7032 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7034 orig = data = RREG32_PCIE(PCIE_P_CNTL);
7035 data |= P_IGNORE_EDB_ERR;
7037 WREG32_PCIE(PCIE_P_CNTL, data);
7039 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7040 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7041 data |= LC_PMI_TO_L1_DIS;
7043 data |= LC_L0S_INACTIVITY(7);
7046 data |= LC_L1_INACTIVITY(7);
7047 data &= ~LC_PMI_TO_L1_DIS;
7049 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7051 if (!disable_plloff_in_l1) {
7052 bool clk_req_support;
7054 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7055 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7056 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7058 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7060 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7061 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7062 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7064 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7066 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7067 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7068 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7070 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7072 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7073 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7074 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7076 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7078 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7079 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7080 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7082 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7084 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7085 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7087 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7089 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7090 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7092 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7094 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7095 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7097 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7099 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7100 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7102 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7104 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7105 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7107 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7109 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7110 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7112 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7114 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7115 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7117 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7119 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7120 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7121 data |= LC_DYN_LANES_PWR_STATE(3);
7123 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7125 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7126 data &= ~LS2_EXIT_TIME_MASK;
7127 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7128 data |= LS2_EXIT_TIME(5);
7130 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7132 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7133 data &= ~LS2_EXIT_TIME_MASK;
7134 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7135 data |= LS2_EXIT_TIME(5);
7137 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7139 if (!disable_clkreq) {
7140 struct pci_dev *root = rdev->pdev->bus->self;
7143 clk_req_support = false;
7144 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7145 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7146 clk_req_support = true;
7148 clk_req_support = false;
7151 if (clk_req_support) {
7152 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7153 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7155 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7157 orig = data = RREG32(THM_CLK_CNTL);
7158 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7159 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7161 WREG32(THM_CLK_CNTL, data);
7163 orig = data = RREG32(MISC_CLK_CNTL);
7164 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7165 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7167 WREG32(MISC_CLK_CNTL, data);
7169 orig = data = RREG32(CG_CLKPIN_CNTL);
7170 data &= ~BCLK_AS_XCLK;
7172 WREG32(CG_CLKPIN_CNTL, data);
7174 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7175 data &= ~FORCE_BIF_REFCLK_EN;
7177 WREG32(CG_CLKPIN_CNTL_2, data);
7179 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7180 data &= ~MPLL_CLKOUT_SEL_MASK;
7181 data |= MPLL_CLKOUT_SEL(4);
7183 WREG32(MPLL_BYPASSCLK_SEL, data);
7185 orig = data = RREG32(SPLL_CNTL_MODE);
7186 data &= ~SPLL_REFCLK_SEL_MASK;
7188 WREG32(SPLL_CNTL_MODE, data);
7193 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7196 orig = data = RREG32_PCIE(PCIE_CNTL2);
7197 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7199 WREG32_PCIE(PCIE_CNTL2, data);
7202 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7203 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7204 data = RREG32_PCIE(PCIE_LC_STATUS1);
7205 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7206 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7207 data &= ~LC_L0S_INACTIVITY_MASK;
7209 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);