2 * Copyright (C) 2007 Ben Skeggs.
5 * Permission is hereby granted, free of charge, to any person obtaining
6 * a copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sublicense, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
13 * The above copyright notice and this permission notice (including the
14 * next paragraph) shall be included in all copies or substantial
15 * portions of the Software.
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
30 #include "nouveau_drv.h"
31 #include "nouveau_ramht.h"
32 #include "nouveau_dma.h"
34 #define USE_REFCNT(dev) (nouveau_private(dev)->chipset >= 0x10)
35 #define USE_SEMA(dev) (nouveau_private(dev)->chipset >= 0x17)
37 struct nouveau_fence {
38 struct nouveau_channel *channel;
40 struct list_head entry;
45 void (*work)(void *priv, bool signalled);
49 struct nouveau_semaphore {
51 struct drm_device *dev;
52 struct drm_mm_node *mem;
55 static inline struct nouveau_fence *
56 nouveau_fence(void *sync_obj)
58 return (struct nouveau_fence *)sync_obj;
62 nouveau_fence_del(struct kref *ref)
64 struct nouveau_fence *fence =
65 container_of(ref, struct nouveau_fence, refcount);
71 nouveau_fence_update(struct nouveau_channel *chan)
73 struct drm_device *dev = chan->dev;
74 struct nouveau_fence *tmp, *fence;
77 spin_lock(&chan->fence.lock);
80 sequence = nvchan_rd32(chan, 0x48);
82 sequence = atomic_read(&chan->fence.last_sequence_irq);
84 if (chan->fence.sequence_ack == sequence)
86 chan->fence.sequence_ack = sequence;
88 list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
89 sequence = fence->sequence;
90 fence->signalled = true;
91 list_del(&fence->entry);
93 if (unlikely(fence->work))
94 fence->work(fence->priv, true);
96 kref_put(&fence->refcount, nouveau_fence_del);
98 if (sequence == chan->fence.sequence_ack)
102 spin_unlock(&chan->fence.lock);
106 nouveau_fence_new(struct nouveau_channel *chan, struct nouveau_fence **pfence,
109 struct nouveau_fence *fence;
112 fence = kzalloc(sizeof(*fence), GFP_KERNEL);
115 kref_init(&fence->refcount);
116 fence->channel = chan;
119 ret = nouveau_fence_emit(fence);
122 nouveau_fence_unref((void *)&fence);
127 struct nouveau_channel *
128 nouveau_fence_channel(struct nouveau_fence *fence)
130 return fence ? fence->channel : NULL;
134 nouveau_fence_emit(struct nouveau_fence *fence)
136 struct nouveau_channel *chan = fence->channel;
137 struct drm_device *dev = chan->dev;
140 ret = RING_SPACE(chan, 2);
144 if (unlikely(chan->fence.sequence == chan->fence.sequence_ack - 1)) {
145 nouveau_fence_update(chan);
147 BUG_ON(chan->fence.sequence ==
148 chan->fence.sequence_ack - 1);
151 fence->sequence = ++chan->fence.sequence;
153 kref_get(&fence->refcount);
154 spin_lock(&chan->fence.lock);
155 list_add_tail(&fence->entry, &chan->fence.pending);
156 spin_unlock(&chan->fence.lock);
158 BEGIN_RING(chan, NvSubSw, USE_REFCNT(dev) ? 0x0050 : 0x0150, 1);
159 OUT_RING(chan, fence->sequence);
166 nouveau_fence_work(struct nouveau_fence *fence,
167 void (*work)(void *priv, bool signalled),
172 spin_lock(&fence->channel->fence.lock);
174 if (fence->signalled) {
181 spin_unlock(&fence->channel->fence.lock);
185 nouveau_fence_unref(void **sync_obj)
187 struct nouveau_fence *fence = nouveau_fence(*sync_obj);
190 kref_put(&fence->refcount, nouveau_fence_del);
195 nouveau_fence_ref(void *sync_obj)
197 struct nouveau_fence *fence = nouveau_fence(sync_obj);
199 kref_get(&fence->refcount);
204 nouveau_fence_signalled(void *sync_obj, void *sync_arg)
206 struct nouveau_fence *fence = nouveau_fence(sync_obj);
207 struct nouveau_channel *chan = fence->channel;
209 if (fence->signalled)
212 nouveau_fence_update(chan);
213 return fence->signalled;
217 nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr)
219 unsigned long timeout = jiffies + (3 * DRM_HZ);
223 if (nouveau_fence_signalled(sync_obj, sync_arg))
226 if (time_after_eq(jiffies, timeout)) {
231 __set_current_state(intr ? TASK_INTERRUPTIBLE
232 : TASK_UNINTERRUPTIBLE);
236 if (intr && signal_pending(current)) {
242 __set_current_state(TASK_RUNNING);
247 static struct nouveau_semaphore *
248 alloc_semaphore(struct drm_device *dev)
250 struct drm_nouveau_private *dev_priv = dev->dev_private;
251 struct nouveau_semaphore *sema;
257 sema = kmalloc(sizeof(*sema), GFP_KERNEL);
261 ret = drm_mm_pre_get(&dev_priv->fence.heap);
265 spin_lock(&dev_priv->fence.lock);
266 sema->mem = drm_mm_search_free(&dev_priv->fence.heap, 4, 0, 0);
268 sema->mem = drm_mm_get_block_atomic(sema->mem, 4, 0);
269 spin_unlock(&dev_priv->fence.lock);
274 kref_init(&sema->ref);
276 nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 0);
285 free_semaphore(struct kref *ref)
287 struct nouveau_semaphore *sema =
288 container_of(ref, struct nouveau_semaphore, ref);
289 struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
291 spin_lock(&dev_priv->fence.lock);
292 drm_mm_put_block(sema->mem);
293 spin_unlock(&dev_priv->fence.lock);
299 semaphore_work(void *priv, bool signalled)
301 struct nouveau_semaphore *sema = priv;
302 struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
304 if (unlikely(!signalled))
305 nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 1);
307 kref_put(&sema->ref, free_semaphore);
311 emit_semaphore(struct nouveau_channel *chan, int method,
312 struct nouveau_semaphore *sema)
314 struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
315 struct nouveau_fence *fence;
316 bool smart = (dev_priv->card_type >= NV_50);
319 ret = RING_SPACE(chan, smart ? 8 : 4);
324 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
325 OUT_RING(chan, NvSema);
327 BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_OFFSET, 1);
328 OUT_RING(chan, sema->mem->start);
330 if (smart && method == NV_SW_SEMAPHORE_ACQUIRE) {
332 * NV50 tries to be too smart and context-switch
333 * between semaphores instead of doing a "first come,
334 * first served" strategy like previous cards
337 * That's bad because the ACQUIRE latency can get as
338 * large as the PFIFO context time slice in the
339 * typical DRI2 case where you have several
340 * outstanding semaphores at the same moment.
342 * If we're going to ACQUIRE, force the card to
343 * context switch before, just in case the matching
344 * RELEASE is already scheduled to be executed in
347 BEGIN_RING(chan, NvSubSw, NV_SW_YIELD, 1);
351 BEGIN_RING(chan, NvSubSw, method, 1);
354 if (smart && method == NV_SW_SEMAPHORE_RELEASE) {
356 * Force the card to context switch, there may be
357 * another channel waiting for the semaphore we just
360 BEGIN_RING(chan, NvSubSw, NV_SW_YIELD, 1);
364 /* Delay semaphore destruction until its work is done */
365 ret = nouveau_fence_new(chan, &fence, true);
369 kref_get(&sema->ref);
370 nouveau_fence_work(fence, semaphore_work, sema);
371 nouveau_fence_unref((void *)&fence);
377 nouveau_fence_sync(struct nouveau_fence *fence,
378 struct nouveau_channel *wchan)
380 struct nouveau_channel *chan = nouveau_fence_channel(fence);
381 struct drm_device *dev = wchan->dev;
382 struct nouveau_semaphore *sema;
385 if (likely(!fence || chan == wchan ||
386 nouveau_fence_signalled(fence, NULL)))
389 sema = alloc_semaphore(dev);
391 /* Early card or broken userspace, fall back to
393 return nouveau_fence_wait(fence, NULL, false, false);
396 /* try to take chan's mutex, if we can't take it right away
397 * we have to fallback to software sync to prevent locking
400 if (!mutex_trylock(&chan->mutex)) {
401 free_semaphore(&sema->ref);
402 return nouveau_fence_wait(fence, NULL, false, false);
405 /* Make wchan wait until it gets signalled */
406 ret = emit_semaphore(wchan, NV_SW_SEMAPHORE_ACQUIRE, sema);
410 /* Signal the semaphore from chan */
411 ret = emit_semaphore(chan, NV_SW_SEMAPHORE_RELEASE, sema);
412 mutex_unlock(&chan->mutex);
414 kref_put(&sema->ref, free_semaphore);
419 nouveau_fence_flush(void *sync_obj, void *sync_arg)
425 nouveau_fence_channel_init(struct nouveau_channel *chan)
427 struct drm_device *dev = chan->dev;
428 struct drm_nouveau_private *dev_priv = dev->dev_private;
429 struct nouveau_gpuobj *obj = NULL;
432 /* Create an NV_SW object for various sync purposes */
433 ret = nouveau_gpuobj_sw_new(chan, NV_SW, &obj);
437 ret = nouveau_ramht_insert(chan, NvSw, obj);
438 nouveau_gpuobj_ref(NULL, &obj);
442 ret = RING_SPACE(chan, 2);
445 BEGIN_RING(chan, NvSubSw, 0, 1);
446 OUT_RING(chan, NvSw);
448 /* Create a DMA object for the shared cross-channel sync area. */
450 struct drm_mm_node *mem = dev_priv->fence.bo->bo.mem.mm_node;
452 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
453 mem->start << PAGE_SHIFT,
454 mem->size << PAGE_SHIFT,
456 NV_DMA_TARGET_VIDMEM, &obj);
460 ret = nouveau_ramht_insert(chan, NvSema, obj);
461 nouveau_gpuobj_ref(NULL, &obj);
465 ret = RING_SPACE(chan, 2);
468 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
469 OUT_RING(chan, NvSema);
474 INIT_LIST_HEAD(&chan->fence.pending);
475 spin_lock_init(&chan->fence.lock);
476 atomic_set(&chan->fence.last_sequence_irq, 0);
482 nouveau_fence_channel_fini(struct nouveau_channel *chan)
484 struct nouveau_fence *tmp, *fence;
486 list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
487 fence->signalled = true;
488 list_del(&fence->entry);
490 if (unlikely(fence->work))
491 fence->work(fence->priv, false);
493 kref_put(&fence->refcount, nouveau_fence_del);
498 nouveau_fence_init(struct drm_device *dev)
500 struct drm_nouveau_private *dev_priv = dev->dev_private;
503 /* Create a shared VRAM heap for cross-channel sync. */
505 ret = nouveau_bo_new(dev, NULL, 4096, 0, TTM_PL_FLAG_VRAM,
506 0, 0, false, true, &dev_priv->fence.bo);
510 ret = nouveau_bo_pin(dev_priv->fence.bo, TTM_PL_FLAG_VRAM);
514 ret = nouveau_bo_map(dev_priv->fence.bo);
518 ret = drm_mm_init(&dev_priv->fence.heap, 0,
519 dev_priv->fence.bo->bo.mem.size);
523 spin_lock_init(&dev_priv->fence.lock);
528 nouveau_bo_unmap(dev_priv->fence.bo);
529 nouveau_bo_ref(NULL, &dev_priv->fence.bo);
534 nouveau_fence_fini(struct drm_device *dev)
536 struct drm_nouveau_private *dev_priv = dev->dev_private;
539 drm_mm_takedown(&dev_priv->fence.heap);
540 nouveau_bo_unmap(dev_priv->fence.bo);
541 nouveau_bo_unpin(dev_priv->fence.bo);
542 nouveau_bo_ref(NULL, &dev_priv->fence.bo);