2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
23 #include <linux/mutex.h>
24 #include <linux/log2.h>
25 #include <linux/sched.h>
26 #include <linux/slab.h>
27 #include <linux/amd-iommu.h>
28 #include <linux/notifier.h>
34 * Initial size for the array of queues.
35 * The allocated size is doubled each time
36 * it is exceeded up to MAX_PROCESS_QUEUES.
38 #define INITIAL_QUEUE_ARRAY_SIZE 16
41 * List of struct kfd_process (field kfd_process).
42 * Unique/indexed by mm_struct*
44 #define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
45 static DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
46 static DEFINE_MUTEX(kfd_processes_mutex);
48 DEFINE_STATIC_SRCU(kfd_processes_srcu);
50 static struct workqueue_struct *kfd_process_wq;
52 struct kfd_process_release_work {
53 struct work_struct kfd_work;
54 struct kfd_process *p;
57 static struct kfd_process *find_process(const struct task_struct *thread);
58 static struct kfd_process *create_process(const struct task_struct *thread);
60 void kfd_process_create_wq(void)
63 kfd_process_wq = create_workqueue("kfd_process_wq");
66 void kfd_process_destroy_wq(void)
69 flush_workqueue(kfd_process_wq);
70 destroy_workqueue(kfd_process_wq);
71 kfd_process_wq = NULL;
75 struct kfd_process *kfd_create_process(const struct task_struct *thread)
77 struct kfd_process *process;
79 BUG_ON(!kfd_process_wq);
81 if (thread->mm == NULL)
82 return ERR_PTR(-EINVAL);
84 /* Only the pthreads threading model is supported. */
85 if (thread->group_leader->mm != thread->mm)
86 return ERR_PTR(-EINVAL);
88 /* Take mmap_sem because we call __mmu_notifier_register inside */
89 down_write(&thread->mm->mmap_sem);
92 * take kfd processes mutex before starting of process creation
93 * so there won't be a case where two threads of the same process
94 * create two kfd_process structures
96 mutex_lock(&kfd_processes_mutex);
98 /* A prior open of /dev/kfd could have already created the process. */
99 process = find_process(thread);
101 pr_debug("kfd: process already found\n");
104 process = create_process(thread);
106 mutex_unlock(&kfd_processes_mutex);
108 up_write(&thread->mm->mmap_sem);
113 struct kfd_process *kfd_get_process(const struct task_struct *thread)
115 struct kfd_process *process;
117 if (thread->mm == NULL)
118 return ERR_PTR(-EINVAL);
120 /* Only the pthreads threading model is supported. */
121 if (thread->group_leader->mm != thread->mm)
122 return ERR_PTR(-EINVAL);
124 process = find_process(thread);
129 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
131 struct kfd_process *process;
133 hash_for_each_possible_rcu(kfd_processes_table, process,
134 kfd_processes, (uintptr_t)mm)
135 if (process->mm == mm)
141 static struct kfd_process *find_process(const struct task_struct *thread)
143 struct kfd_process *p;
146 idx = srcu_read_lock(&kfd_processes_srcu);
147 p = find_process_by_mm(thread->mm);
148 srcu_read_unlock(&kfd_processes_srcu, idx);
153 static void kfd_process_wq_release(struct work_struct *work)
155 struct kfd_process_release_work *my_work;
156 struct kfd_process_device *pdd, *temp;
157 struct kfd_process *p;
159 my_work = (struct kfd_process_release_work *) work;
163 mutex_lock(&p->mutex);
165 list_for_each_entry_safe(pdd, temp, &p->per_device_data,
167 amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
168 list_del(&pdd->per_device_list);
173 kfd_pasid_free(p->pasid);
175 mutex_unlock(&p->mutex);
177 mutex_destroy(&p->mutex);
186 static void kfd_process_destroy_delayed(struct rcu_head *rcu)
188 struct kfd_process_release_work *work;
189 struct kfd_process *p;
191 BUG_ON(!kfd_process_wq);
193 p = container_of(rcu, struct kfd_process, rcu);
194 BUG_ON(atomic_read(&p->mm->mm_count) <= 0);
198 work = (struct kfd_process_release_work *)
199 kmalloc(sizeof(struct kfd_process_release_work), GFP_ATOMIC);
202 INIT_WORK((struct work_struct *) work, kfd_process_wq_release);
204 queue_work(kfd_process_wq, (struct work_struct *) work);
208 static void kfd_process_notifier_release(struct mmu_notifier *mn,
209 struct mm_struct *mm)
211 struct kfd_process *p;
214 * The kfd_process structure can not be free because the
215 * mmu_notifier srcu is read locked
217 p = container_of(mn, struct kfd_process, mmu_notifier);
220 mutex_lock(&kfd_processes_mutex);
221 hash_del_rcu(&p->kfd_processes);
222 mutex_unlock(&kfd_processes_mutex);
223 synchronize_srcu(&kfd_processes_srcu);
225 mutex_lock(&p->mutex);
227 /* In case our notifier is called before IOMMU notifier */
230 mutex_unlock(&p->mutex);
233 * Because we drop mm_count inside kfd_process_destroy_delayed
234 * and because the mmu_notifier_unregister function also drop
235 * mm_count we need to take an extra count here.
237 atomic_inc(&p->mm->mm_count);
238 mmu_notifier_unregister_no_release(&p->mmu_notifier, p->mm);
239 mmu_notifier_call_srcu(&p->rcu, &kfd_process_destroy_delayed);
242 static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
243 .release = kfd_process_notifier_release,
246 static struct kfd_process *create_process(const struct task_struct *thread)
248 struct kfd_process *process;
251 process = kzalloc(sizeof(*process), GFP_KERNEL);
254 goto err_alloc_process;
256 process->queues = kmalloc_array(INITIAL_QUEUE_ARRAY_SIZE,
257 sizeof(process->queues[0]), GFP_KERNEL);
258 if (!process->queues)
259 goto err_alloc_queues;
261 process->pasid = kfd_pasid_alloc();
262 if (process->pasid == 0)
263 goto err_alloc_pasid;
265 mutex_init(&process->mutex);
267 process->mm = thread->mm;
269 /* register notifier */
270 process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
271 err = __mmu_notifier_register(&process->mmu_notifier, process->mm);
273 goto err_mmu_notifier;
275 hash_add_rcu(kfd_processes_table, &process->kfd_processes,
276 (uintptr_t)process->mm);
278 process->lead_thread = thread->group_leader;
280 process->queue_array_size = INITIAL_QUEUE_ARRAY_SIZE;
282 INIT_LIST_HEAD(&process->per_device_data);
284 err = pqm_init(&process->pqm, process);
286 goto err_process_pqm_init;
290 err_process_pqm_init:
291 hash_del_rcu(&process->kfd_processes);
293 mmu_notifier_unregister_no_release(&process->mmu_notifier, process->mm);
295 kfd_pasid_free(process->pasid);
297 kfree(process->queues);
304 struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
305 struct kfd_process *p,
308 struct kfd_process_device *pdd = NULL;
310 list_for_each_entry(pdd, &p->per_device_data, per_device_list)
315 pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
318 INIT_LIST_HEAD(&pdd->qpd.queues_list);
319 INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
320 pdd->qpd.dqm = dev->dqm;
321 list_add(&pdd->per_device_list, &p->per_device_data);
329 * Direct the IOMMU to bind the process (specifically the pasid->mm)
331 * Unbinding occurs when the process dies or the device is removed.
333 * Assumes that the process lock is held.
335 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
336 struct kfd_process *p)
338 struct kfd_process_device *pdd = kfd_get_process_device_data(dev, p, 1);
342 return ERR_PTR(-ENOMEM);
347 err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
356 void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid)
358 struct kfd_process *p;
359 struct kfd_process_device *pdd;
364 idx = srcu_read_lock(&kfd_processes_srcu);
366 hash_for_each_rcu(kfd_processes_table, i, p, kfd_processes)
367 if (p->pasid == pasid)
370 srcu_read_unlock(&kfd_processes_srcu, idx);
372 BUG_ON(p->pasid != pasid);
374 mutex_lock(&p->mutex);
378 pdd = kfd_get_process_device_data(dev, p, 0);
381 * Just mark pdd as unbound, because we still need it to call
382 * amd_iommu_unbind_pasid() in when the process exits.
383 * We don't call amd_iommu_unbind_pasid() here
384 * because the IOMMU called us.
389 mutex_unlock(&p->mutex);
392 struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p)
394 return list_first_entry(&p->per_device_data,
395 struct kfd_process_device,
399 struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p,
400 struct kfd_process_device *pdd)
402 if (list_is_last(&pdd->per_device_list, &p->per_device_data))
404 return list_next_entry(pdd, per_device_list);
407 bool kfd_has_process_device_data(struct kfd_process *p)
409 return !(list_empty(&p->per_device_data));