From fca6af6a5976dfa00182232f666b4f789c98bd0c Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 2 Jun 2017 18:13:37 +0200 Subject: [PATCH] iommu/amd: Add per-domain timer to flush per-cpu queues Add a timer to each dma_ops domain so that we flush unused IOTLB entries regularily, even if the queues don't get full all the time. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 84 +++++++++++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 17 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 795208bd39bd..00c1796e07bf 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -182,6 +182,13 @@ struct dma_ops_domain { * flushed out of the IOTLB of the domain. */ atomic64_t flush_finish_cnt; + + /* + * Timer to make sure we don't keep IOVAs around unflushed + * for too long + */ + struct timer_list flush_timer; + atomic_t flush_timer_on; }; static struct iova_domain reserved_iova_ranges; @@ -1834,6 +1841,14 @@ static int dma_ops_domain_alloc_flush_queue(struct dma_ops_domain *dom) return 0; } +static void dma_ops_domain_flush_tlb(struct dma_ops_domain *dom) +{ + atomic64_inc(&dom->flush_start_cnt); + domain_flush_tlb(&dom->domain); + domain_flush_complete(&dom->domain); + atomic64_inc(&dom->flush_finish_cnt); +} + static inline bool queue_ring_full(struct flush_queue *queue) { assert_spin_locked(&queue->lock); @@ -1875,22 +1890,12 @@ static inline void queue_ring_remove_head(struct flush_queue *queue) queue->head = (queue->head + 1) % FLUSH_QUEUE_SIZE; } -static void queue_add(struct dma_ops_domain *dom, - unsigned long address, unsigned long pages) +static void queue_ring_free_flushed(struct dma_ops_domain *dom, + struct flush_queue *queue) { - struct flush_queue *queue; - unsigned long flags; - u64 counter; + u64 counter = atomic64_read(&dom->flush_finish_cnt); int idx; - pages = __roundup_pow_of_two(pages); - address >>= PAGE_SHIFT; - - counter = atomic64_read(&dom->flush_finish_cnt); - - queue = get_cpu_ptr(dom->flush_queue); - spin_lock_irqsave(&queue->lock, flags); - queue_ring_for_each(idx, queue) { /* * This assumes that counter values in the ring-buffer are @@ -1905,12 +1910,25 @@ static void queue_add(struct dma_ops_domain *dom, queue_ring_remove_head(queue); } +} + +static void queue_add(struct dma_ops_domain *dom, + unsigned long address, unsigned long pages) +{ + struct flush_queue *queue; + unsigned long flags; + int idx; + + pages = __roundup_pow_of_two(pages); + address >>= PAGE_SHIFT; + + queue = get_cpu_ptr(dom->flush_queue); + spin_lock_irqsave(&queue->lock, flags); + + queue_ring_free_flushed(dom, queue); if (queue_ring_full(queue)) { - atomic64_inc(&dom->flush_start_cnt); - domain_flush_tlb(&dom->domain); - domain_flush_complete(&dom->domain); - atomic64_inc(&dom->flush_finish_cnt); + dma_ops_domain_flush_tlb(dom); queue_release(dom, queue); } @@ -1921,9 +1939,33 @@ static void queue_add(struct dma_ops_domain *dom, queue->entries[idx].counter = atomic64_read(&dom->flush_start_cnt); spin_unlock_irqrestore(&queue->lock, flags); + + if (atomic_cmpxchg(&dom->flush_timer_on, 0, 1) == 0) + mod_timer(&dom->flush_timer, jiffies + msecs_to_jiffies(10)); + put_cpu_ptr(dom->flush_queue); } +static void queue_flush_timeout(unsigned long data) +{ + struct dma_ops_domain *dom = (struct dma_ops_domain *)data; + int cpu; + + atomic_set(&dom->flush_timer_on, 0); + + dma_ops_domain_flush_tlb(dom); + + for_each_possible_cpu(cpu) { + struct flush_queue *queue; + unsigned long flags; + + queue = per_cpu_ptr(dom->flush_queue, cpu); + spin_lock_irqsave(&queue->lock, flags); + queue_ring_free_flushed(dom, queue); + spin_unlock_irqrestore(&queue->lock, flags); + } +} + /* * Free a domain, only used if something went wrong in the * allocation path and we need to free an already allocated page table @@ -1935,6 +1977,9 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom) del_domain_from_list(&dom->domain); + if (timer_pending(&dom->flush_timer)) + del_timer(&dom->flush_timer); + dma_ops_domain_free_flush_queue(dom); put_iova_domain(&dom->iovad); @@ -1978,6 +2023,11 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void) if (dma_ops_domain_alloc_flush_queue(dma_dom)) goto free_dma_dom; + setup_timer(&dma_dom->flush_timer, queue_flush_timeout, + (unsigned long)dma_dom); + + atomic_set(&dma_dom->flush_timer_on, 0); + add_domain_to_list(&dma_dom->domain); return dma_dom; -- 2.39.5