2 * Copyright 2009 Jerome Glisse.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
28 * Jerome Glisse <glisse@freedesktop.org>
31 #include <linux/seq_file.h>
32 #include <linux/atomic.h>
33 #include <linux/wait.h>
34 #include <linux/list.h>
35 #include <linux/kref.h>
36 #include <linux/slab.h>
39 #include "radeon_reg.h"
41 #include "radeon_trace.h"
43 static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring)
45 if (rdev->wb.enabled) {
46 *rdev->fence_drv[ring].cpu_addr = cpu_to_le32(seq);
48 WREG32(rdev->fence_drv[ring].scratch_reg, seq);
52 static u32 radeon_fence_read(struct radeon_device *rdev, int ring)
56 if (rdev->wb.enabled) {
57 seq = le32_to_cpu(*rdev->fence_drv[ring].cpu_addr);
59 seq = RREG32(rdev->fence_drv[ring].scratch_reg);
64 int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence)
66 unsigned long irq_flags;
68 write_lock_irqsave(&rdev->fence_lock, irq_flags);
70 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
73 fence->seq = atomic_add_return(1, &rdev->fence_drv[fence->ring].seq);
74 if (!rdev->ring[fence->ring].ready)
75 /* FIXME: cp is not running assume everythings is done right
78 radeon_fence_write(rdev, fence->seq, fence->ring);
80 radeon_fence_ring_emit(rdev, fence->ring, fence);
82 trace_radeon_fence_emit(rdev->ddev, fence->seq);
83 fence->emitted = true;
84 list_move_tail(&fence->list, &rdev->fence_drv[fence->ring].emitted);
85 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
89 static bool radeon_fence_poll_locked(struct radeon_device *rdev, int ring)
91 struct radeon_fence *fence;
92 struct list_head *i, *n;
95 unsigned long cjiffies;
97 seq = radeon_fence_read(rdev, ring);
98 if (seq != rdev->fence_drv[ring].last_seq) {
99 rdev->fence_drv[ring].last_seq = seq;
100 rdev->fence_drv[ring].last_jiffies = jiffies;
101 rdev->fence_drv[ring].last_timeout = RADEON_FENCE_JIFFIES_TIMEOUT;
104 if (time_after(cjiffies, rdev->fence_drv[ring].last_jiffies)) {
105 cjiffies -= rdev->fence_drv[ring].last_jiffies;
106 if (time_after(rdev->fence_drv[ring].last_timeout, cjiffies)) {
107 /* update the timeout */
108 rdev->fence_drv[ring].last_timeout -= cjiffies;
110 /* the 500ms timeout is elapsed we should test
113 rdev->fence_drv[ring].last_timeout = 1;
116 /* wrap around update last jiffies, we will just wait
119 rdev->fence_drv[ring].last_jiffies = cjiffies;
124 list_for_each(i, &rdev->fence_drv[ring].emitted) {
125 fence = list_entry(i, struct radeon_fence, list);
126 if (fence->seq == seq) {
131 /* all fence previous to this one are considered as signaled */
136 list_move_tail(i, &rdev->fence_drv[ring].signaled);
137 fence = list_entry(i, struct radeon_fence, list);
138 fence->signaled = true;
140 } while (i != &rdev->fence_drv[ring].emitted);
146 static void radeon_fence_destroy(struct kref *kref)
148 unsigned long irq_flags;
149 struct radeon_fence *fence;
151 fence = container_of(kref, struct radeon_fence, kref);
152 write_lock_irqsave(&fence->rdev->fence_lock, irq_flags);
153 list_del(&fence->list);
154 fence->emitted = false;
155 write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags);
156 if (fence->semaphore)
157 radeon_semaphore_free(fence->rdev, fence->semaphore);
161 int radeon_fence_create(struct radeon_device *rdev,
162 struct radeon_fence **fence,
165 unsigned long irq_flags;
167 *fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL);
168 if ((*fence) == NULL) {
171 kref_init(&((*fence)->kref));
172 (*fence)->rdev = rdev;
173 (*fence)->emitted = false;
174 (*fence)->signaled = false;
176 (*fence)->ring = ring;
177 (*fence)->semaphore = NULL;
178 INIT_LIST_HEAD(&(*fence)->list);
180 write_lock_irqsave(&rdev->fence_lock, irq_flags);
181 list_add_tail(&(*fence)->list, &rdev->fence_drv[ring].created);
182 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
186 bool radeon_fence_signaled(struct radeon_fence *fence)
188 unsigned long irq_flags;
189 bool signaled = false;
194 if (fence->rdev->gpu_lockup)
197 write_lock_irqsave(&fence->rdev->fence_lock, irq_flags);
198 signaled = fence->signaled;
199 /* if we are shuting down report all fence as signaled */
200 if (fence->rdev->shutdown) {
203 if (!fence->emitted) {
204 WARN(1, "Querying an unemitted fence : %p !\n", fence);
208 radeon_fence_poll_locked(fence->rdev, fence->ring);
209 signaled = fence->signaled;
211 write_unlock_irqrestore(&fence->rdev->fence_lock, irq_flags);
215 int radeon_fence_wait(struct radeon_fence *fence, bool intr)
217 struct radeon_device *rdev;
218 unsigned long irq_flags, timeout;
223 WARN(1, "Querying an invalid fence : %p !\n", fence);
227 if (radeon_fence_signaled(fence)) {
230 timeout = rdev->fence_drv[fence->ring].last_timeout;
232 /* save current sequence used to check for GPU lockup */
233 seq = rdev->fence_drv[fence->ring].last_seq;
234 trace_radeon_fence_wait_begin(rdev->ddev, seq);
236 radeon_irq_kms_sw_irq_get(rdev, fence->ring);
237 r = wait_event_interruptible_timeout(rdev->fence_drv[fence->ring].queue,
238 radeon_fence_signaled(fence), timeout);
239 radeon_irq_kms_sw_irq_put(rdev, fence->ring);
240 if (unlikely(r < 0)) {
244 radeon_irq_kms_sw_irq_get(rdev, fence->ring);
245 r = wait_event_timeout(rdev->fence_drv[fence->ring].queue,
246 radeon_fence_signaled(fence), timeout);
247 radeon_irq_kms_sw_irq_put(rdev, fence->ring);
249 trace_radeon_fence_wait_end(rdev->ddev, seq);
250 if (unlikely(!radeon_fence_signaled(fence))) {
251 /* we were interrupted for some reason and fence isn't
252 * isn't signaled yet, resume wait
258 /* don't protect read access to rdev->fence_drv[t].last_seq
259 * if we experiencing a lockup the value doesn't change
261 if (seq == rdev->fence_drv[fence->ring].last_seq &&
262 radeon_gpu_is_lockup(rdev, &rdev->ring[fence->ring])) {
263 /* good news we believe it's a lockup */
264 printk(KERN_WARNING "GPU lockup (waiting for 0x%08X last fence id 0x%08X)\n",
266 /* FIXME: what should we do ? marking everyone
267 * as signaled for now
269 rdev->gpu_lockup = true;
270 r = radeon_gpu_reset(rdev);
273 radeon_fence_write(rdev, fence->seq, fence->ring);
274 rdev->gpu_lockup = false;
276 timeout = RADEON_FENCE_JIFFIES_TIMEOUT;
277 write_lock_irqsave(&rdev->fence_lock, irq_flags);
278 rdev->fence_drv[fence->ring].last_timeout = RADEON_FENCE_JIFFIES_TIMEOUT;
279 rdev->fence_drv[fence->ring].last_jiffies = jiffies;
280 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
286 int radeon_fence_wait_next(struct radeon_device *rdev, int ring)
288 unsigned long irq_flags;
289 struct radeon_fence *fence;
292 if (rdev->gpu_lockup) {
295 write_lock_irqsave(&rdev->fence_lock, irq_flags);
296 if (list_empty(&rdev->fence_drv[ring].emitted)) {
297 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
300 fence = list_entry(rdev->fence_drv[ring].emitted.next,
301 struct radeon_fence, list);
302 radeon_fence_ref(fence);
303 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
304 r = radeon_fence_wait(fence, false);
305 radeon_fence_unref(&fence);
309 int radeon_fence_wait_last(struct radeon_device *rdev, int ring)
311 unsigned long irq_flags;
312 struct radeon_fence *fence;
315 if (rdev->gpu_lockup) {
318 write_lock_irqsave(&rdev->fence_lock, irq_flags);
319 if (list_empty(&rdev->fence_drv[ring].emitted)) {
320 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
323 fence = list_entry(rdev->fence_drv[ring].emitted.prev,
324 struct radeon_fence, list);
325 radeon_fence_ref(fence);
326 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
327 r = radeon_fence_wait(fence, false);
328 radeon_fence_unref(&fence);
332 struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence)
334 kref_get(&fence->kref);
338 void radeon_fence_unref(struct radeon_fence **fence)
340 struct radeon_fence *tmp = *fence;
344 kref_put(&tmp->kref, radeon_fence_destroy);
348 void radeon_fence_process(struct radeon_device *rdev, int ring)
350 unsigned long irq_flags;
353 write_lock_irqsave(&rdev->fence_lock, irq_flags);
354 wake = radeon_fence_poll_locked(rdev, ring);
355 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
357 wake_up_all(&rdev->fence_drv[ring].queue);
361 int radeon_fence_count_emitted(struct radeon_device *rdev, int ring)
363 unsigned long irq_flags;
364 int not_processed = 0;
366 read_lock_irqsave(&rdev->fence_lock, irq_flags);
367 if (!rdev->fence_drv[ring].initialized) {
368 read_unlock_irqrestore(&rdev->fence_lock, irq_flags);
372 if (!list_empty(&rdev->fence_drv[ring].emitted)) {
373 struct list_head *ptr;
374 list_for_each(ptr, &rdev->fence_drv[ring].emitted) {
375 /* count up to 3, that's enought info */
376 if (++not_processed >= 3)
380 read_unlock_irqrestore(&rdev->fence_lock, irq_flags);
381 return not_processed;
384 int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring)
386 unsigned long irq_flags;
390 write_lock_irqsave(&rdev->fence_lock, irq_flags);
391 radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
392 if (rdev->wb.use_event) {
393 rdev->fence_drv[ring].scratch_reg = 0;
394 index = R600_WB_EVENT_OFFSET + ring * 4;
396 r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg);
398 dev_err(rdev->dev, "fence failed to get scratch register\n");
399 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
402 index = RADEON_WB_SCRATCH_OFFSET +
403 rdev->fence_drv[ring].scratch_reg -
404 rdev->scratch.reg_base;
406 rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4];
407 rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index;
408 radeon_fence_write(rdev, atomic_read(&rdev->fence_drv[ring].seq), ring);
409 rdev->fence_drv[ring].initialized = true;
410 DRM_INFO("fence driver on ring %d use gpu addr 0x%08Lx and cpu addr 0x%p\n",
411 ring, rdev->fence_drv[ring].gpu_addr, rdev->fence_drv[ring].cpu_addr);
412 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
416 static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring)
418 rdev->fence_drv[ring].scratch_reg = -1;
419 rdev->fence_drv[ring].cpu_addr = NULL;
420 rdev->fence_drv[ring].gpu_addr = 0;
421 atomic_set(&rdev->fence_drv[ring].seq, 0);
422 INIT_LIST_HEAD(&rdev->fence_drv[ring].created);
423 INIT_LIST_HEAD(&rdev->fence_drv[ring].emitted);
424 INIT_LIST_HEAD(&rdev->fence_drv[ring].signaled);
425 init_waitqueue_head(&rdev->fence_drv[ring].queue);
426 rdev->fence_drv[ring].initialized = false;
429 int radeon_fence_driver_init(struct radeon_device *rdev)
431 unsigned long irq_flags;
434 write_lock_irqsave(&rdev->fence_lock, irq_flags);
435 for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
436 radeon_fence_driver_init_ring(rdev, ring);
438 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
439 if (radeon_debugfs_fence_init(rdev)) {
440 dev_err(rdev->dev, "fence debugfs file creation failed\n");
445 void radeon_fence_driver_fini(struct radeon_device *rdev)
447 unsigned long irq_flags;
450 for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
451 if (!rdev->fence_drv[ring].initialized)
453 radeon_fence_wait_last(rdev, ring);
454 wake_up_all(&rdev->fence_drv[ring].queue);
455 write_lock_irqsave(&rdev->fence_lock, irq_flags);
456 radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
457 write_unlock_irqrestore(&rdev->fence_lock, irq_flags);
458 rdev->fence_drv[ring].initialized = false;
466 #if defined(CONFIG_DEBUG_FS)
467 static int radeon_debugfs_fence_info(struct seq_file *m, void *data)
469 struct drm_info_node *node = (struct drm_info_node *)m->private;
470 struct drm_device *dev = node->minor->dev;
471 struct radeon_device *rdev = dev->dev_private;
472 struct radeon_fence *fence;
475 for (i = 0; i < RADEON_NUM_RINGS; ++i) {
476 if (!rdev->fence_drv[i].initialized)
479 seq_printf(m, "--- ring %d ---\n", i);
480 seq_printf(m, "Last signaled fence 0x%08X\n",
481 radeon_fence_read(rdev, i));
482 if (!list_empty(&rdev->fence_drv[i].emitted)) {
483 fence = list_entry(rdev->fence_drv[i].emitted.prev,
484 struct radeon_fence, list);
485 seq_printf(m, "Last emitted fence %p with 0x%08X\n",
492 static struct drm_info_list radeon_debugfs_fence_list[] = {
493 {"radeon_fence_info", &radeon_debugfs_fence_info, 0, NULL},
497 int radeon_debugfs_fence_init(struct radeon_device *rdev)
499 #if defined(CONFIG_DEBUG_FS)
500 return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 1);