2 * Copyright 2012 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
25 #include "amdgpu_ih.h"
28 #include "bif/bif_4_1_d.h"
29 #include "bif/bif_4_1_sh_mask.h"
31 #include "oss/oss_2_0_d.h"
32 #include "oss/oss_2_0_sh_mask.h"
36 * Starting with r6xx, interrupts are handled via a ring buffer.
37 * Ring buffers are areas of GPU accessible memory that the GPU
38 * writes interrupt vectors into and the host reads vectors out of.
39 * There is a rptr (read pointer) that determines where the
40 * host is currently reading, and a wptr (write pointer)
41 * which determines where the GPU has written. When the
42 * pointers are equal, the ring is idle. When the GPU
43 * writes vectors to the ring buffer, it increments the
44 * wptr. When there is an interrupt, the host then starts
45 * fetching commands and processing them until the pointers are
46 * equal again at which point it updates the rptr.
49 static void cik_ih_set_interrupt_funcs(struct amdgpu_device *adev);
52 * cik_ih_enable_interrupts - Enable the interrupt ring buffer
54 * @adev: amdgpu_device pointer
56 * Enable the interrupt ring buffer (CIK).
58 static void cik_ih_enable_interrupts(struct amdgpu_device *adev)
60 u32 ih_cntl = RREG32(mmIH_CNTL);
61 u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL);
63 ih_cntl |= IH_CNTL__ENABLE_INTR_MASK;
64 ih_rb_cntl |= IH_RB_CNTL__RB_ENABLE_MASK;
65 WREG32(mmIH_CNTL, ih_cntl);
66 WREG32(mmIH_RB_CNTL, ih_rb_cntl);
67 adev->irq.ih.enabled = true;
71 * cik_ih_disable_interrupts - Disable the interrupt ring buffer
73 * @adev: amdgpu_device pointer
75 * Disable the interrupt ring buffer (CIK).
77 static void cik_ih_disable_interrupts(struct amdgpu_device *adev)
79 u32 ih_rb_cntl = RREG32(mmIH_RB_CNTL);
80 u32 ih_cntl = RREG32(mmIH_CNTL);
82 ih_rb_cntl &= ~IH_RB_CNTL__RB_ENABLE_MASK;
83 ih_cntl &= ~IH_CNTL__ENABLE_INTR_MASK;
84 WREG32(mmIH_RB_CNTL, ih_rb_cntl);
85 WREG32(mmIH_CNTL, ih_cntl);
86 /* set rptr, wptr to 0 */
87 WREG32(mmIH_RB_RPTR, 0);
88 WREG32(mmIH_RB_WPTR, 0);
89 adev->irq.ih.enabled = false;
90 adev->irq.ih.rptr = 0;
94 * cik_ih_irq_init - init and enable the interrupt ring
96 * @adev: amdgpu_device pointer
98 * Allocate a ring buffer for the interrupt controller,
99 * enable the RLC, disable interrupts, enable the IH
100 * ring buffer and enable it (CIK).
101 * Called at device load and reume.
102 * Returns 0 for success, errors for failure.
104 static int cik_ih_irq_init(struct amdgpu_device *adev)
108 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
112 cik_ih_disable_interrupts(adev);
114 /* setup interrupt control */
115 WREG32(mmINTERRUPT_CNTL2, adev->dummy_page.addr >> 8);
116 interrupt_cntl = RREG32(mmINTERRUPT_CNTL);
117 /* INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi
118 * INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN
120 interrupt_cntl &= ~INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK;
121 /* INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */
122 interrupt_cntl &= ~INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK;
123 WREG32(mmINTERRUPT_CNTL, interrupt_cntl);
125 WREG32(mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8);
126 rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);
128 ih_rb_cntl = (IH_RB_CNTL__WPTR_OVERFLOW_ENABLE_MASK |
129 IH_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK |
132 ih_rb_cntl |= IH_RB_CNTL__WPTR_WRITEBACK_ENABLE_MASK;
134 /* set the writeback address whether it's enabled or not */
135 wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4);
136 WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(wptr_off));
137 WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(wptr_off) & 0xFF);
139 WREG32(mmIH_RB_CNTL, ih_rb_cntl);
141 /* set rptr, wptr to 0 */
142 WREG32(mmIH_RB_RPTR, 0);
143 WREG32(mmIH_RB_WPTR, 0);
145 /* Default settings for IH_CNTL (disabled at first) */
146 ih_cntl = (0x10 << IH_CNTL__MC_WRREQ_CREDIT__SHIFT) |
147 (0x10 << IH_CNTL__MC_WR_CLEAN_CNT__SHIFT) |
148 (0 << IH_CNTL__MC_VMID__SHIFT);
149 /* IH_CNTL__RPTR_REARM_MASK only works if msi's are enabled */
150 if (adev->irq.msi_enabled)
151 ih_cntl |= IH_CNTL__RPTR_REARM_MASK;
152 WREG32(mmIH_CNTL, ih_cntl);
154 pci_set_master(adev->pdev);
157 cik_ih_enable_interrupts(adev);
163 * cik_ih_irq_disable - disable interrupts
165 * @adev: amdgpu_device pointer
167 * Disable interrupts on the hw (CIK).
169 static void cik_ih_irq_disable(struct amdgpu_device *adev)
171 cik_ih_disable_interrupts(adev);
172 /* Wait and acknowledge irq */
177 * cik_ih_get_wptr - get the IH ring buffer wptr
179 * @adev: amdgpu_device pointer
181 * Get the IH ring buffer wptr from either the register
182 * or the writeback memory buffer (CIK). Also check for
183 * ring buffer overflow and deal with it.
184 * Used by cik_irq_process().
185 * Returns the value of the wptr.
187 static u32 cik_ih_get_wptr(struct amdgpu_device *adev)
191 wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]);
193 if (wptr & IH_RB_WPTR__RB_OVERFLOW_MASK) {
194 wptr &= ~IH_RB_WPTR__RB_OVERFLOW_MASK;
195 /* When a ring buffer overflow happen start parsing interrupt
196 * from the last not overwritten vector (wptr + 16). Hopefully
197 * this should allow us to catchup.
199 dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
200 wptr, adev->irq.ih.rptr, (wptr + 16) & adev->irq.ih.ptr_mask);
201 adev->irq.ih.rptr = (wptr + 16) & adev->irq.ih.ptr_mask;
202 tmp = RREG32(mmIH_RB_CNTL);
203 tmp |= IH_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK;
204 WREG32(mmIH_RB_CNTL, tmp);
206 return (wptr & adev->irq.ih.ptr_mask);
210 * Each IV ring entry is 128 bits:
211 * [7:0] - interrupt source id
213 * [59:32] - interrupt source data
217 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
218 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
219 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
220 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
222 * - ME1&2 compute dispatcher (4 pipes each)
224 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
225 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
226 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
229 * [127:96] - reserved
233 * cik_ih_decode_iv - decode an interrupt vector
235 * @adev: amdgpu_device pointer
237 * Decodes the interrupt vector at the current rptr
238 * position and also advance the position.
240 static void cik_ih_decode_iv(struct amdgpu_device *adev,
241 struct amdgpu_iv_entry *entry)
243 /* wptr/rptr are in bytes! */
244 u32 ring_index = adev->irq.ih.rptr >> 2;
247 dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
248 dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]);
249 dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]);
250 dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
252 entry->src_id = dw[0] & 0xff;
253 entry->src_data = dw[1] & 0xfffffff;
254 entry->ring_id = dw[2] & 0xff;
255 entry->vm_id = (dw[2] >> 8) & 0xff;
256 entry->pas_id = (dw[2] >> 16) & 0xffff;
258 /* wptr/rptr are in bytes! */
259 adev->irq.ih.rptr += 16;
263 * cik_ih_set_rptr - set the IH ring buffer rptr
265 * @adev: amdgpu_device pointer
267 * Set the IH ring buffer rptr.
269 static void cik_ih_set_rptr(struct amdgpu_device *adev)
271 WREG32(mmIH_RB_RPTR, adev->irq.ih.rptr);
274 static int cik_ih_early_init(void *handle)
276 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
279 ret = amdgpu_irq_add_domain(adev);
283 cik_ih_set_interrupt_funcs(adev);
288 static int cik_ih_sw_init(void *handle)
291 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
293 r = amdgpu_ih_ring_init(adev, 64 * 1024, false);
297 r = amdgpu_irq_init(adev);
302 static int cik_ih_sw_fini(void *handle)
304 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
306 amdgpu_irq_fini(adev);
307 amdgpu_ih_ring_fini(adev);
308 amdgpu_irq_remove_domain(adev);
313 static int cik_ih_hw_init(void *handle)
316 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
318 r = cik_ih_irq_init(adev);
325 static int cik_ih_hw_fini(void *handle)
327 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
329 cik_ih_irq_disable(adev);
334 static int cik_ih_suspend(void *handle)
336 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
338 return cik_ih_hw_fini(adev);
341 static int cik_ih_resume(void *handle)
343 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
345 return cik_ih_hw_init(adev);
348 static bool cik_ih_is_idle(void *handle)
350 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
351 u32 tmp = RREG32(mmSRBM_STATUS);
353 if (tmp & SRBM_STATUS__IH_BUSY_MASK)
359 static int cik_ih_wait_for_idle(void *handle)
363 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
365 for (i = 0; i < adev->usec_timeout; i++) {
367 tmp = RREG32(mmSRBM_STATUS) & SRBM_STATUS__IH_BUSY_MASK;
375 static void cik_ih_print_status(void *handle)
377 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
379 dev_info(adev->dev, "CIK IH registers\n");
380 dev_info(adev->dev, " SRBM_STATUS=0x%08X\n",
381 RREG32(mmSRBM_STATUS));
382 dev_info(adev->dev, " SRBM_STATUS2=0x%08X\n",
383 RREG32(mmSRBM_STATUS2));
384 dev_info(adev->dev, " INTERRUPT_CNTL=0x%08X\n",
385 RREG32(mmINTERRUPT_CNTL));
386 dev_info(adev->dev, " INTERRUPT_CNTL2=0x%08X\n",
387 RREG32(mmINTERRUPT_CNTL2));
388 dev_info(adev->dev, " IH_CNTL=0x%08X\n",
390 dev_info(adev->dev, " IH_RB_CNTL=0x%08X\n",
391 RREG32(mmIH_RB_CNTL));
392 dev_info(adev->dev, " IH_RB_BASE=0x%08X\n",
393 RREG32(mmIH_RB_BASE));
394 dev_info(adev->dev, " IH_RB_WPTR_ADDR_LO=0x%08X\n",
395 RREG32(mmIH_RB_WPTR_ADDR_LO));
396 dev_info(adev->dev, " IH_RB_WPTR_ADDR_HI=0x%08X\n",
397 RREG32(mmIH_RB_WPTR_ADDR_HI));
398 dev_info(adev->dev, " IH_RB_RPTR=0x%08X\n",
399 RREG32(mmIH_RB_RPTR));
400 dev_info(adev->dev, " IH_RB_WPTR=0x%08X\n",
401 RREG32(mmIH_RB_WPTR));
404 static int cik_ih_soft_reset(void *handle)
406 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
408 u32 srbm_soft_reset = 0;
409 u32 tmp = RREG32(mmSRBM_STATUS);
411 if (tmp & SRBM_STATUS__IH_BUSY_MASK)
412 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_IH_MASK;
414 if (srbm_soft_reset) {
415 cik_ih_print_status((void *)adev);
417 tmp = RREG32(mmSRBM_SOFT_RESET);
418 tmp |= srbm_soft_reset;
419 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
420 WREG32(mmSRBM_SOFT_RESET, tmp);
421 tmp = RREG32(mmSRBM_SOFT_RESET);
425 tmp &= ~srbm_soft_reset;
426 WREG32(mmSRBM_SOFT_RESET, tmp);
427 tmp = RREG32(mmSRBM_SOFT_RESET);
429 /* Wait a little for things to settle down */
432 cik_ih_print_status((void *)adev);
438 static int cik_ih_set_clockgating_state(void *handle,
439 enum amd_clockgating_state state)
444 static int cik_ih_set_powergating_state(void *handle,
445 enum amd_powergating_state state)
450 const struct amd_ip_funcs cik_ih_ip_funcs = {
451 .early_init = cik_ih_early_init,
453 .sw_init = cik_ih_sw_init,
454 .sw_fini = cik_ih_sw_fini,
455 .hw_init = cik_ih_hw_init,
456 .hw_fini = cik_ih_hw_fini,
457 .suspend = cik_ih_suspend,
458 .resume = cik_ih_resume,
459 .is_idle = cik_ih_is_idle,
460 .wait_for_idle = cik_ih_wait_for_idle,
461 .soft_reset = cik_ih_soft_reset,
462 .print_status = cik_ih_print_status,
463 .set_clockgating_state = cik_ih_set_clockgating_state,
464 .set_powergating_state = cik_ih_set_powergating_state,
467 static const struct amdgpu_ih_funcs cik_ih_funcs = {
468 .get_wptr = cik_ih_get_wptr,
469 .decode_iv = cik_ih_decode_iv,
470 .set_rptr = cik_ih_set_rptr
473 static void cik_ih_set_interrupt_funcs(struct amdgpu_device *adev)
475 if (adev->irq.ih_funcs == NULL)
476 adev->irq.ih_funcs = &cik_ih_funcs;