2 * Copyright © 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 #include <linux/kthread.h>
26 #include <uapi/linux/sched/types.h>
30 static void intel_breadcrumbs_hangcheck(unsigned long data)
32 struct intel_engine_cs *engine = (struct intel_engine_cs *)data;
33 struct intel_breadcrumbs *b = &engine->breadcrumbs;
38 if (time_before(jiffies, b->timeout)) {
39 mod_timer(&b->hangcheck, b->timeout);
43 DRM_DEBUG("Hangcheck timer elapsed... %s idle\n", engine->name);
44 set_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings);
45 mod_timer(&engine->breadcrumbs.fake_irq, jiffies + 1);
47 /* Ensure that even if the GPU hangs, we get woken up.
49 * However, note that if no one is waiting, we never notice
50 * a gpu hang. Eventually, we will have to wait for a resource
51 * held by the GPU and so trigger a hangcheck. In the most
52 * pathological case, this will be upon memory starvation! To
53 * prevent this, we also queue the hangcheck from the retire
56 i915_queue_hangcheck(engine->i915);
59 static unsigned long wait_timeout(void)
61 return round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES);
64 static void intel_breadcrumbs_fake_irq(unsigned long data)
66 struct intel_engine_cs *engine = (struct intel_engine_cs *)data;
69 * The timer persists in case we cannot enable interrupts,
70 * or if we have previously seen seqno/interrupt incoherency
71 * ("missed interrupt" syndrome). Here the worker will wake up
72 * every jiffie in order to kick the oldest waiter to do the
73 * coherent seqno check.
75 if (intel_engine_wakeup(engine))
76 mod_timer(&engine->breadcrumbs.fake_irq, jiffies + 1);
79 static void irq_enable(struct intel_engine_cs *engine)
81 /* Enabling the IRQ may miss the generation of the interrupt, but
82 * we still need to force the barrier before reading the seqno,
85 engine->breadcrumbs.irq_posted = true;
87 /* Caller disables interrupts */
88 spin_lock(&engine->i915->irq_lock);
89 engine->irq_enable(engine);
90 spin_unlock(&engine->i915->irq_lock);
93 static void irq_disable(struct intel_engine_cs *engine)
95 /* Caller disables interrupts */
96 spin_lock(&engine->i915->irq_lock);
97 engine->irq_disable(engine);
98 spin_unlock(&engine->i915->irq_lock);
100 engine->breadcrumbs.irq_posted = false;
103 static void __intel_breadcrumbs_enable_irq(struct intel_breadcrumbs *b)
105 struct intel_engine_cs *engine =
106 container_of(b, struct intel_engine_cs, breadcrumbs);
107 struct drm_i915_private *i915 = engine->i915;
109 assert_spin_locked(&b->lock);
113 /* Since we are waiting on a request, the GPU should be busy
114 * and should have its own rpm reference. For completeness,
115 * record an rpm reference for ourselves to cover the
116 * interrupt we unmask.
118 intel_runtime_pm_get_noresume(i915);
119 b->rpm_wakelock = true;
121 /* No interrupts? Kick the waiter every jiffie! */
122 if (intel_irqs_enabled(i915)) {
123 if (!test_bit(engine->id, &i915->gpu_error.test_irq_rings))
125 b->irq_enabled = true;
128 if (!b->irq_enabled ||
129 test_bit(engine->id, &i915->gpu_error.missed_irq_rings)) {
130 mod_timer(&b->fake_irq, jiffies + 1);
132 /* Ensure we never sleep indefinitely */
133 GEM_BUG_ON(!time_after(b->timeout, jiffies));
134 mod_timer(&b->hangcheck, b->timeout);
138 static void __intel_breadcrumbs_disable_irq(struct intel_breadcrumbs *b)
140 struct intel_engine_cs *engine =
141 container_of(b, struct intel_engine_cs, breadcrumbs);
143 assert_spin_locked(&b->lock);
144 if (!b->rpm_wakelock)
147 if (b->irq_enabled) {
149 b->irq_enabled = false;
152 intel_runtime_pm_put(engine->i915);
153 b->rpm_wakelock = false;
156 static inline struct intel_wait *to_wait(struct rb_node *node)
158 return rb_entry(node, struct intel_wait, node);
161 static inline void __intel_breadcrumbs_finish(struct intel_breadcrumbs *b,
162 struct intel_wait *wait)
164 assert_spin_locked(&b->lock);
166 /* This request is completed, so remove it from the tree, mark it as
167 * complete, and *then* wake up the associated task.
169 rb_erase(&wait->node, &b->waiters);
170 RB_CLEAR_NODE(&wait->node);
172 wake_up_process(wait->tsk); /* implicit smp_wmb() */
175 static bool __intel_engine_add_wait(struct intel_engine_cs *engine,
176 struct intel_wait *wait)
178 struct intel_breadcrumbs *b = &engine->breadcrumbs;
179 struct rb_node **p, *parent, *completed;
183 /* Insert the request into the retirement ordered list
184 * of waiters by walking the rbtree. If we are the oldest
185 * seqno in the tree (the first to be retired), then
186 * set ourselves as the bottom-half.
188 * As we descend the tree, prune completed branches since we hold the
189 * spinlock we know that the first_waiter must be delayed and can
190 * reduce some of the sequential wake up latency if we take action
191 * ourselves and wake up the completed tasks in parallel. Also, by
192 * removing stale elements in the tree, we may be able to reduce the
193 * ping-pong between the old bottom-half and ourselves as first-waiter.
198 seqno = intel_engine_get_seqno(engine);
200 /* If the request completed before we managed to grab the spinlock,
201 * return now before adding ourselves to the rbtree. We let the
202 * current bottom-half handle any pending wakeups and instead
203 * try and get out of the way quickly.
205 if (i915_seqno_passed(seqno, wait->seqno)) {
206 RB_CLEAR_NODE(&wait->node);
210 p = &b->waiters.rb_node;
213 if (wait->seqno == to_wait(parent)->seqno) {
214 /* We have multiple waiters on the same seqno, select
215 * the highest priority task (that with the smallest
216 * task->prio) to serve as the bottom-half for this
219 if (wait->tsk->prio > to_wait(parent)->tsk->prio) {
220 p = &parent->rb_right;
223 p = &parent->rb_left;
225 } else if (i915_seqno_passed(wait->seqno,
226 to_wait(parent)->seqno)) {
227 p = &parent->rb_right;
228 if (i915_seqno_passed(seqno, to_wait(parent)->seqno))
233 p = &parent->rb_left;
236 rb_link_node(&wait->node, parent, p);
237 rb_insert_color(&wait->node, &b->waiters);
238 GEM_BUG_ON(!first && !rcu_access_pointer(b->irq_seqno_bh));
241 struct rb_node *next = rb_next(completed);
243 GEM_BUG_ON(!next && !first);
244 if (next && next != &wait->node) {
246 b->timeout = wait_timeout();
247 b->first_wait = to_wait(next);
248 rcu_assign_pointer(b->irq_seqno_bh, b->first_wait->tsk);
249 /* As there is a delay between reading the current
250 * seqno, processing the completed tasks and selecting
251 * the next waiter, we may have missed the interrupt
252 * and so need for the next bottom-half to wakeup.
254 * Also as we enable the IRQ, we may miss the
255 * interrupt for that seqno, so we have to wake up
256 * the next bottom-half in order to do a coherent check
257 * in case the seqno passed.
259 __intel_breadcrumbs_enable_irq(b);
260 if (READ_ONCE(b->irq_posted))
261 wake_up_process(to_wait(next)->tsk);
265 struct intel_wait *crumb = to_wait(completed);
266 completed = rb_prev(completed);
267 __intel_breadcrumbs_finish(b, crumb);
272 GEM_BUG_ON(rb_first(&b->waiters) != &wait->node);
273 b->timeout = wait_timeout();
274 b->first_wait = wait;
275 rcu_assign_pointer(b->irq_seqno_bh, wait->tsk);
276 /* After assigning ourselves as the new bottom-half, we must
277 * perform a cursory check to prevent a missed interrupt.
278 * Either we miss the interrupt whilst programming the hardware,
279 * or if there was a previous waiter (for a later seqno) they
280 * may be woken instead of us (due to the inherent race
281 * in the unlocked read of b->irq_seqno_bh in the irq handler)
282 * and so we miss the wake up.
284 __intel_breadcrumbs_enable_irq(b);
286 GEM_BUG_ON(!rcu_access_pointer(b->irq_seqno_bh));
287 GEM_BUG_ON(!b->first_wait);
288 GEM_BUG_ON(rb_first(&b->waiters) != &b->first_wait->node);
293 bool intel_engine_add_wait(struct intel_engine_cs *engine,
294 struct intel_wait *wait)
296 struct intel_breadcrumbs *b = &engine->breadcrumbs;
299 spin_lock_irq(&b->lock);
300 first = __intel_engine_add_wait(engine, wait);
301 spin_unlock_irq(&b->lock);
306 static inline bool chain_wakeup(struct rb_node *rb, int priority)
308 return rb && to_wait(rb)->tsk->prio <= priority;
311 static inline int wakeup_priority(struct intel_breadcrumbs *b,
312 struct task_struct *tsk)
314 if (tsk == b->signaler)
320 void intel_engine_remove_wait(struct intel_engine_cs *engine,
321 struct intel_wait *wait)
323 struct intel_breadcrumbs *b = &engine->breadcrumbs;
325 /* Quick check to see if this waiter was already decoupled from
326 * the tree by the bottom-half to avoid contention on the spinlock
329 if (RB_EMPTY_NODE(&wait->node))
332 spin_lock_irq(&b->lock);
334 if (RB_EMPTY_NODE(&wait->node))
337 if (b->first_wait == wait) {
338 const int priority = wakeup_priority(b, wait->tsk);
339 struct rb_node *next;
341 GEM_BUG_ON(rcu_access_pointer(b->irq_seqno_bh) != wait->tsk);
343 /* We are the current bottom-half. Find the next candidate,
344 * the first waiter in the queue on the remaining oldest
345 * request. As multiple seqnos may complete in the time it
346 * takes us to wake up and find the next waiter, we have to
347 * wake up that waiter for it to perform its own coherent
350 next = rb_next(&wait->node);
351 if (chain_wakeup(next, priority)) {
352 /* If the next waiter is already complete,
353 * wake it up and continue onto the next waiter. So
354 * if have a small herd, they will wake up in parallel
355 * rather than sequentially, which should reduce
356 * the overall latency in waking all the completed
359 * However, waking up a chain adds extra latency to
360 * the first_waiter. This is undesirable if that
361 * waiter is a high priority task.
363 u32 seqno = intel_engine_get_seqno(engine);
365 while (i915_seqno_passed(seqno, to_wait(next)->seqno)) {
366 struct rb_node *n = rb_next(next);
368 __intel_breadcrumbs_finish(b, to_wait(next));
370 if (!chain_wakeup(next, priority))
376 /* In our haste, we may have completed the first waiter
377 * before we enabled the interrupt. Do so now as we
378 * have a second waiter for a future seqno. Afterwards,
379 * we have to wake up that waiter in case we missed
380 * the interrupt, or if we have to handle an
381 * exception rather than a seqno completion.
383 b->timeout = wait_timeout();
384 b->first_wait = to_wait(next);
385 rcu_assign_pointer(b->irq_seqno_bh, b->first_wait->tsk);
386 if (b->first_wait->seqno != wait->seqno)
387 __intel_breadcrumbs_enable_irq(b);
388 wake_up_process(b->first_wait->tsk);
390 b->first_wait = NULL;
391 rcu_assign_pointer(b->irq_seqno_bh, NULL);
392 __intel_breadcrumbs_disable_irq(b);
395 GEM_BUG_ON(rb_first(&b->waiters) == &wait->node);
398 GEM_BUG_ON(RB_EMPTY_NODE(&wait->node));
399 rb_erase(&wait->node, &b->waiters);
402 GEM_BUG_ON(b->first_wait == wait);
403 GEM_BUG_ON(rb_first(&b->waiters) !=
404 (b->first_wait ? &b->first_wait->node : NULL));
405 GEM_BUG_ON(!rcu_access_pointer(b->irq_seqno_bh) ^ RB_EMPTY_ROOT(&b->waiters));
406 spin_unlock_irq(&b->lock);
409 static bool signal_complete(struct drm_i915_gem_request *request)
414 /* If another process served as the bottom-half it may have already
415 * signalled that this wait is already completed.
417 if (intel_wait_complete(&request->signaling.wait))
420 /* Carefully check if the request is complete, giving time for the
421 * seqno to be visible or if the GPU hung.
423 if (__i915_request_irq_complete(request))
429 static struct drm_i915_gem_request *to_signaler(struct rb_node *rb)
431 return rb_entry(rb, struct drm_i915_gem_request, signaling.node);
434 static void signaler_set_rtpriority(void)
436 struct sched_param param = { .sched_priority = 1 };
438 sched_setscheduler_nocheck(current, SCHED_FIFO, ¶m);
441 static int intel_breadcrumbs_signaler(void *arg)
443 struct intel_engine_cs *engine = arg;
444 struct intel_breadcrumbs *b = &engine->breadcrumbs;
445 struct drm_i915_gem_request *request;
447 /* Install ourselves with high priority to reduce signalling latency */
448 signaler_set_rtpriority();
451 set_current_state(TASK_INTERRUPTIBLE);
453 /* We are either woken up by the interrupt bottom-half,
454 * or by a client adding a new signaller. In both cases,
455 * the GPU seqno may have advanced beyond our oldest signal.
456 * If it has, propagate the signal, remove the waiter and
457 * check again with the next oldest signal. Otherwise we
458 * need to wait for a new interrupt from the GPU or for
461 request = READ_ONCE(b->first_signal);
462 if (signal_complete(request)) {
463 /* Wake up all other completed waiters and select the
464 * next bottom-half for the next user interrupt.
466 intel_engine_remove_wait(engine,
467 &request->signaling.wait);
470 dma_fence_signal(&request->fence);
471 local_bh_enable(); /* kick start the tasklets */
473 /* Find the next oldest signal. Note that as we have
474 * not been holding the lock, another client may
475 * have installed an even older signal than the one
476 * we just completed - so double check we are still
477 * the oldest before picking the next one.
479 spin_lock_irq(&b->lock);
480 if (request == b->first_signal) {
482 rb_next(&request->signaling.node);
483 b->first_signal = rb ? to_signaler(rb) : NULL;
485 rb_erase(&request->signaling.node, &b->signals);
486 spin_unlock_irq(&b->lock);
488 i915_gem_request_put(request);
490 if (kthread_should_stop())
496 __set_current_state(TASK_RUNNING);
501 void intel_engine_enable_signaling(struct drm_i915_gem_request *request)
503 struct intel_engine_cs *engine = request->engine;
504 struct intel_breadcrumbs *b = &engine->breadcrumbs;
505 struct rb_node *parent, **p;
508 /* Note that we may be called from an interrupt handler on another
509 * device (e.g. nouveau signaling a fence completion causing us
510 * to submit a request, and so enable signaling). As such,
511 * we need to make sure that all other users of b->lock protect
512 * against interrupts, i.e. use spin_lock_irqsave.
515 /* locked by dma_fence_enable_sw_signaling() (irqsafe fence->lock) */
516 assert_spin_locked(&request->lock);
517 if (!request->global_seqno)
520 request->signaling.wait.tsk = b->signaler;
521 request->signaling.wait.seqno = request->global_seqno;
522 i915_gem_request_get(request);
526 /* First add ourselves into the list of waiters, but register our
527 * bottom-half as the signaller thread. As per usual, only the oldest
528 * waiter (not just signaller) is tasked as the bottom-half waking
529 * up all completed waiters after the user interrupt.
531 * If we are the oldest waiter, enable the irq (after which we
532 * must double check that the seqno did not complete).
534 wakeup = __intel_engine_add_wait(engine, &request->signaling.wait);
536 /* Now insert ourselves into the retirement ordered list of signals
537 * on this engine. We track the oldest seqno as that will be the
538 * first signal to complete.
542 p = &b->signals.rb_node;
545 if (i915_seqno_passed(request->global_seqno,
546 to_signaler(parent)->global_seqno)) {
547 p = &parent->rb_right;
550 p = &parent->rb_left;
553 rb_link_node(&request->signaling.node, parent, p);
554 rb_insert_color(&request->signaling.node, &b->signals);
556 smp_store_mb(b->first_signal, request);
558 spin_unlock(&b->lock);
561 wake_up_process(b->signaler);
564 int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine)
566 struct intel_breadcrumbs *b = &engine->breadcrumbs;
567 struct task_struct *tsk;
569 spin_lock_init(&b->lock);
570 setup_timer(&b->fake_irq,
571 intel_breadcrumbs_fake_irq,
572 (unsigned long)engine);
573 setup_timer(&b->hangcheck,
574 intel_breadcrumbs_hangcheck,
575 (unsigned long)engine);
577 /* Spawn a thread to provide a common bottom-half for all signals.
578 * As this is an asynchronous interface we cannot steal the current
579 * task for handling the bottom-half to the user interrupt, therefore
580 * we create a thread to do the coherent seqno dance after the
581 * interrupt and then signal the waitqueue (via the dma-buf/fence).
583 tsk = kthread_run(intel_breadcrumbs_signaler, engine,
584 "i915/signal:%d", engine->id);
593 static void cancel_fake_irq(struct intel_engine_cs *engine)
595 struct intel_breadcrumbs *b = &engine->breadcrumbs;
597 del_timer_sync(&b->hangcheck);
598 del_timer_sync(&b->fake_irq);
599 clear_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings);
602 void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine)
604 struct intel_breadcrumbs *b = &engine->breadcrumbs;
606 cancel_fake_irq(engine);
607 spin_lock_irq(&b->lock);
609 __intel_breadcrumbs_disable_irq(b);
610 if (intel_engine_has_waiter(engine)) {
611 b->timeout = wait_timeout();
612 __intel_breadcrumbs_enable_irq(b);
613 if (READ_ONCE(b->irq_posted))
614 wake_up_process(b->first_wait->tsk);
616 /* sanitize the IMR and unmask any auxiliary interrupts */
620 spin_unlock_irq(&b->lock);
623 void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
625 struct intel_breadcrumbs *b = &engine->breadcrumbs;
627 /* The engines should be idle and all requests accounted for! */
628 WARN_ON(READ_ONCE(b->first_wait));
629 WARN_ON(!RB_EMPTY_ROOT(&b->waiters));
630 WARN_ON(READ_ONCE(b->first_signal));
631 WARN_ON(!RB_EMPTY_ROOT(&b->signals));
633 if (!IS_ERR_OR_NULL(b->signaler))
634 kthread_stop(b->signaler);
636 cancel_fake_irq(engine);
639 unsigned int intel_breadcrumbs_busy(struct drm_i915_private *i915)
641 struct intel_engine_cs *engine;
642 enum intel_engine_id id;
643 unsigned int mask = 0;
645 for_each_engine(engine, i915, id) {
646 struct intel_breadcrumbs *b = &engine->breadcrumbs;
648 spin_lock_irq(&b->lock);
651 wake_up_process(b->first_wait->tsk);
652 mask |= intel_engine_flag(engine);
655 if (b->first_signal) {
656 wake_up_process(b->signaler);
657 mask |= intel_engine_flag(engine);
660 spin_unlock_irq(&b->lock);