iommu_v2.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (C) 2010-2012 Advanced Micro Devices, Inc.
  4. * Author: Joerg Roedel <jroedel@suse.de>
  5. */
  6. #define pr_fmt(fmt) "AMD-Vi: " fmt
  7. #include <linux/mmu_notifier.h>
  8. #include <linux/amd-iommu.h>
  9. #include <linux/mm_types.h>
  10. #include <linux/profile.h>
  11. #include <linux/module.h>
  12. #include <linux/sched.h>
  13. #include <linux/sched/mm.h>
  14. #include <linux/wait.h>
  15. #include <linux/pci.h>
  16. #include <linux/gfp.h>
  17. #include "amd_iommu.h"
  18. MODULE_LICENSE("GPL v2");
  19. MODULE_AUTHOR("Joerg Roedel <jroedel@suse.de>");
  20. #define MAX_DEVICES 0x10000
  21. #define PRI_QUEUE_SIZE 512
  22. struct pri_queue {
  23. atomic_t inflight;
  24. bool finish;
  25. int status;
  26. };
  27. struct pasid_state {
  28. struct list_head list; /* For global state-list */
  29. atomic_t count; /* Reference count */
  30. unsigned mmu_notifier_count; /* Counting nested mmu_notifier
  31. calls */
  32. struct mm_struct *mm; /* mm_struct for the faults */
  33. struct mmu_notifier mn; /* mmu_notifier handle */
  34. struct pri_queue pri[PRI_QUEUE_SIZE]; /* PRI tag states */
  35. struct device_state *device_state; /* Link to our device_state */
  36. u32 pasid; /* PASID index */
  37. bool invalid; /* Used during setup and
  38. teardown of the pasid */
  39. spinlock_t lock; /* Protect pri_queues and
  40. mmu_notifer_count */
  41. wait_queue_head_t wq; /* To wait for count == 0 */
  42. };
  43. struct device_state {
  44. struct list_head list;
  45. u16 devid;
  46. atomic_t count;
  47. struct pci_dev *pdev;
  48. struct pasid_state **states;
  49. struct iommu_domain *domain;
  50. int pasid_levels;
  51. int max_pasids;
  52. amd_iommu_invalid_ppr_cb inv_ppr_cb;
  53. amd_iommu_invalidate_ctx inv_ctx_cb;
  54. spinlock_t lock;
  55. wait_queue_head_t wq;
  56. };
  57. struct fault {
  58. struct work_struct work;
  59. struct device_state *dev_state;
  60. struct pasid_state *state;
  61. struct mm_struct *mm;
  62. u64 address;
  63. u16 devid;
  64. u32 pasid;
  65. u16 tag;
  66. u16 finish;
  67. u16 flags;
  68. };
  69. static LIST_HEAD(state_list);
  70. static spinlock_t state_lock;
  71. static struct workqueue_struct *iommu_wq;
  72. static void free_pasid_states(struct device_state *dev_state);
  73. static u16 device_id(struct pci_dev *pdev)
  74. {
  75. u16 devid;
  76. devid = pdev->bus->number;
  77. devid = (devid << 8) | pdev->devfn;
  78. return devid;
  79. }
  80. static struct device_state *__get_device_state(u16 devid)
  81. {
  82. struct device_state *dev_state;
  83. list_for_each_entry(dev_state, &state_list, list) {
  84. if (dev_state->devid == devid)
  85. return dev_state;
  86. }
  87. return NULL;
  88. }
  89. static struct device_state *get_device_state(u16 devid)
  90. {
  91. struct device_state *dev_state;
  92. unsigned long flags;
  93. spin_lock_irqsave(&state_lock, flags);
  94. dev_state = __get_device_state(devid);
  95. if (dev_state != NULL)
  96. atomic_inc(&dev_state->count);
  97. spin_unlock_irqrestore(&state_lock, flags);
  98. return dev_state;
  99. }
  100. static void free_device_state(struct device_state *dev_state)
  101. {
  102. struct iommu_group *group;
  103. /*
  104. * First detach device from domain - No more PRI requests will arrive
  105. * from that device after it is unbound from the IOMMUv2 domain.
  106. */
  107. group = iommu_group_get(&dev_state->pdev->dev);
  108. if (WARN_ON(!group))
  109. return;
  110. iommu_detach_group(dev_state->domain, group);
  111. iommu_group_put(group);
  112. /* Everything is down now, free the IOMMUv2 domain */
  113. iommu_domain_free(dev_state->domain);
  114. /* Finally get rid of the device-state */
  115. kfree(dev_state);
  116. }
  117. static void put_device_state(struct device_state *dev_state)
  118. {
  119. if (atomic_dec_and_test(&dev_state->count))
  120. wake_up(&dev_state->wq);
  121. }
  122. /* Must be called under dev_state->lock */
  123. static struct pasid_state **__get_pasid_state_ptr(struct device_state *dev_state,
  124. u32 pasid, bool alloc)
  125. {
  126. struct pasid_state **root, **ptr;
  127. int level, index;
  128. level = dev_state->pasid_levels;
  129. root = dev_state->states;
  130. while (true) {
  131. index = (pasid >> (9 * level)) & 0x1ff;
  132. ptr = &root[index];
  133. if (level == 0)
  134. break;
  135. if (*ptr == NULL) {
  136. if (!alloc)
  137. return NULL;
  138. *ptr = (void *)get_zeroed_page(GFP_ATOMIC);
  139. if (*ptr == NULL)
  140. return NULL;
  141. }
  142. root = (struct pasid_state **)*ptr;
  143. level -= 1;
  144. }
  145. return ptr;
  146. }
  147. static int set_pasid_state(struct device_state *dev_state,
  148. struct pasid_state *pasid_state,
  149. u32 pasid)
  150. {
  151. struct pasid_state **ptr;
  152. unsigned long flags;
  153. int ret;
  154. spin_lock_irqsave(&dev_state->lock, flags);
  155. ptr = __get_pasid_state_ptr(dev_state, pasid, true);
  156. ret = -ENOMEM;
  157. if (ptr == NULL)
  158. goto out_unlock;
  159. ret = -ENOMEM;
  160. if (*ptr != NULL)
  161. goto out_unlock;
  162. *ptr = pasid_state;
  163. ret = 0;
  164. out_unlock:
  165. spin_unlock_irqrestore(&dev_state->lock, flags);
  166. return ret;
  167. }
  168. static void clear_pasid_state(struct device_state *dev_state, u32 pasid)
  169. {
  170. struct pasid_state **ptr;
  171. unsigned long flags;
  172. spin_lock_irqsave(&dev_state->lock, flags);
  173. ptr = __get_pasid_state_ptr(dev_state, pasid, true);
  174. if (ptr == NULL)
  175. goto out_unlock;
  176. *ptr = NULL;
  177. out_unlock:
  178. spin_unlock_irqrestore(&dev_state->lock, flags);
  179. }
  180. static struct pasid_state *get_pasid_state(struct device_state *dev_state,
  181. u32 pasid)
  182. {
  183. struct pasid_state **ptr, *ret = NULL;
  184. unsigned long flags;
  185. spin_lock_irqsave(&dev_state->lock, flags);
  186. ptr = __get_pasid_state_ptr(dev_state, pasid, false);
  187. if (ptr == NULL)
  188. goto out_unlock;
  189. ret = *ptr;
  190. if (ret)
  191. atomic_inc(&ret->count);
  192. out_unlock:
  193. spin_unlock_irqrestore(&dev_state->lock, flags);
  194. return ret;
  195. }
  196. static void free_pasid_state(struct pasid_state *pasid_state)
  197. {
  198. kfree(pasid_state);
  199. }
  200. static void put_pasid_state(struct pasid_state *pasid_state)
  201. {
  202. if (atomic_dec_and_test(&pasid_state->count))
  203. wake_up(&pasid_state->wq);
  204. }
  205. static void put_pasid_state_wait(struct pasid_state *pasid_state)
  206. {
  207. atomic_dec(&pasid_state->count);
  208. wait_event(pasid_state->wq, !atomic_read(&pasid_state->count));
  209. free_pasid_state(pasid_state);
  210. }
  211. static void unbind_pasid(struct pasid_state *pasid_state)
  212. {
  213. struct iommu_domain *domain;
  214. domain = pasid_state->device_state->domain;
  215. /*
  216. * Mark pasid_state as invalid, no more faults will we added to the
  217. * work queue after this is visible everywhere.
  218. */
  219. pasid_state->invalid = true;
  220. /* Make sure this is visible */
  221. smp_wmb();
  222. /* After this the device/pasid can't access the mm anymore */
  223. amd_iommu_domain_clear_gcr3(domain, pasid_state->pasid);
  224. /* Make sure no more pending faults are in the queue */
  225. flush_workqueue(iommu_wq);
  226. }
  227. static void free_pasid_states_level1(struct pasid_state **tbl)
  228. {
  229. int i;
  230. for (i = 0; i < 512; ++i) {
  231. if (tbl[i] == NULL)
  232. continue;
  233. free_page((unsigned long)tbl[i]);
  234. }
  235. }
  236. static void free_pasid_states_level2(struct pasid_state **tbl)
  237. {
  238. struct pasid_state **ptr;
  239. int i;
  240. for (i = 0; i < 512; ++i) {
  241. if (tbl[i] == NULL)
  242. continue;
  243. ptr = (struct pasid_state **)tbl[i];
  244. free_pasid_states_level1(ptr);
  245. }
  246. }
  247. static void free_pasid_states(struct device_state *dev_state)
  248. {
  249. struct pasid_state *pasid_state;
  250. int i;
  251. for (i = 0; i < dev_state->max_pasids; ++i) {
  252. pasid_state = get_pasid_state(dev_state, i);
  253. if (pasid_state == NULL)
  254. continue;
  255. put_pasid_state(pasid_state);
  256. /*
  257. * This will call the mn_release function and
  258. * unbind the PASID
  259. */
  260. mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);
  261. put_pasid_state_wait(pasid_state); /* Reference taken in
  262. amd_iommu_bind_pasid */
  263. /* Drop reference taken in amd_iommu_bind_pasid */
  264. put_device_state(dev_state);
  265. }
  266. if (dev_state->pasid_levels == 2)
  267. free_pasid_states_level2(dev_state->states);
  268. else if (dev_state->pasid_levels == 1)
  269. free_pasid_states_level1(dev_state->states);
  270. else
  271. BUG_ON(dev_state->pasid_levels != 0);
  272. free_page((unsigned long)dev_state->states);
  273. }
  274. static struct pasid_state *mn_to_state(struct mmu_notifier *mn)
  275. {
  276. return container_of(mn, struct pasid_state, mn);
  277. }
  278. static void mn_invalidate_range(struct mmu_notifier *mn,
  279. struct mm_struct *mm,
  280. unsigned long start, unsigned long end)
  281. {
  282. struct pasid_state *pasid_state;
  283. struct device_state *dev_state;
  284. pasid_state = mn_to_state(mn);
  285. dev_state = pasid_state->device_state;
  286. if ((start ^ (end - 1)) < PAGE_SIZE)
  287. amd_iommu_flush_page(dev_state->domain, pasid_state->pasid,
  288. start);
  289. else
  290. amd_iommu_flush_tlb(dev_state->domain, pasid_state->pasid);
  291. }
  292. static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm)
  293. {
  294. struct pasid_state *pasid_state;
  295. struct device_state *dev_state;
  296. bool run_inv_ctx_cb;
  297. might_sleep();
  298. pasid_state = mn_to_state(mn);
  299. dev_state = pasid_state->device_state;
  300. run_inv_ctx_cb = !pasid_state->invalid;
  301. if (run_inv_ctx_cb && dev_state->inv_ctx_cb)
  302. dev_state->inv_ctx_cb(dev_state->pdev, pasid_state->pasid);
  303. unbind_pasid(pasid_state);
  304. }
  305. static const struct mmu_notifier_ops iommu_mn = {
  306. .release = mn_release,
  307. .invalidate_range = mn_invalidate_range,
  308. };
  309. static void set_pri_tag_status(struct pasid_state *pasid_state,
  310. u16 tag, int status)
  311. {
  312. unsigned long flags;
  313. spin_lock_irqsave(&pasid_state->lock, flags);
  314. pasid_state->pri[tag].status = status;
  315. spin_unlock_irqrestore(&pasid_state->lock, flags);
  316. }
  317. static void finish_pri_tag(struct device_state *dev_state,
  318. struct pasid_state *pasid_state,
  319. u16 tag)
  320. {
  321. unsigned long flags;
  322. spin_lock_irqsave(&pasid_state->lock, flags);
  323. if (atomic_dec_and_test(&pasid_state->pri[tag].inflight) &&
  324. pasid_state->pri[tag].finish) {
  325. amd_iommu_complete_ppr(dev_state->pdev, pasid_state->pasid,
  326. pasid_state->pri[tag].status, tag);
  327. pasid_state->pri[tag].finish = false;
  328. pasid_state->pri[tag].status = PPR_SUCCESS;
  329. }
  330. spin_unlock_irqrestore(&pasid_state->lock, flags);
  331. }
  332. static void handle_fault_error(struct fault *fault)
  333. {
  334. int status;
  335. if (!fault->dev_state->inv_ppr_cb) {
  336. set_pri_tag_status(fault->state, fault->tag, PPR_INVALID);
  337. return;
  338. }
  339. status = fault->dev_state->inv_ppr_cb(fault->dev_state->pdev,
  340. fault->pasid,
  341. fault->address,
  342. fault->flags);
  343. switch (status) {
  344. case AMD_IOMMU_INV_PRI_RSP_SUCCESS:
  345. set_pri_tag_status(fault->state, fault->tag, PPR_SUCCESS);
  346. break;
  347. case AMD_IOMMU_INV_PRI_RSP_INVALID:
  348. set_pri_tag_status(fault->state, fault->tag, PPR_INVALID);
  349. break;
  350. case AMD_IOMMU_INV_PRI_RSP_FAIL:
  351. set_pri_tag_status(fault->state, fault->tag, PPR_FAILURE);
  352. break;
  353. default:
  354. BUG();
  355. }
  356. }
  357. static bool access_error(struct vm_area_struct *vma, struct fault *fault)
  358. {
  359. unsigned long requested = 0;
  360. if (fault->flags & PPR_FAULT_EXEC)
  361. requested |= VM_EXEC;
  362. if (fault->flags & PPR_FAULT_READ)
  363. requested |= VM_READ;
  364. if (fault->flags & PPR_FAULT_WRITE)
  365. requested |= VM_WRITE;
  366. return (requested & ~vma->vm_flags) != 0;
  367. }
  368. static void do_fault(struct work_struct *work)
  369. {
  370. struct fault *fault = container_of(work, struct fault, work);
  371. struct vm_area_struct *vma;
  372. vm_fault_t ret = VM_FAULT_ERROR;
  373. unsigned int flags = 0;
  374. struct mm_struct *mm;
  375. u64 address;
  376. mm = fault->state->mm;
  377. address = fault->address;
  378. if (fault->flags & PPR_FAULT_USER)
  379. flags |= FAULT_FLAG_USER;
  380. if (fault->flags & PPR_FAULT_WRITE)
  381. flags |= FAULT_FLAG_WRITE;
  382. flags |= FAULT_FLAG_REMOTE;
  383. mmap_read_lock(mm);
  384. vma = find_extend_vma(mm, address);
  385. if (!vma || address < vma->vm_start)
  386. /* failed to get a vma in the right range */
  387. goto out;
  388. /* Check if we have the right permissions on the vma */
  389. if (access_error(vma, fault))
  390. goto out;
  391. ret = handle_mm_fault(vma, address, flags, NULL);
  392. out:
  393. mmap_read_unlock(mm);
  394. if (ret & VM_FAULT_ERROR)
  395. /* failed to service fault */
  396. handle_fault_error(fault);
  397. finish_pri_tag(fault->dev_state, fault->state, fault->tag);
  398. put_pasid_state(fault->state);
  399. kfree(fault);
  400. }
  401. static int ppr_notifier(struct notifier_block *nb, unsigned long e, void *data)
  402. {
  403. struct amd_iommu_fault *iommu_fault;
  404. struct pasid_state *pasid_state;
  405. struct device_state *dev_state;
  406. struct pci_dev *pdev = NULL;
  407. unsigned long flags;
  408. struct fault *fault;
  409. bool finish;
  410. u16 tag, devid;
  411. int ret;
  412. iommu_fault = data;
  413. tag = iommu_fault->tag & 0x1ff;
  414. finish = (iommu_fault->tag >> 9) & 1;
  415. devid = iommu_fault->device_id;
  416. pdev = pci_get_domain_bus_and_slot(0, PCI_BUS_NUM(devid),
  417. devid & 0xff);
  418. if (!pdev)
  419. return -ENODEV;
  420. ret = NOTIFY_DONE;
  421. /* In kdump kernel pci dev is not initialized yet -> send INVALID */
  422. if (amd_iommu_is_attach_deferred(NULL, &pdev->dev)) {
  423. amd_iommu_complete_ppr(pdev, iommu_fault->pasid,
  424. PPR_INVALID, tag);
  425. goto out;
  426. }
  427. dev_state = get_device_state(iommu_fault->device_id);
  428. if (dev_state == NULL)
  429. goto out;
  430. pasid_state = get_pasid_state(dev_state, iommu_fault->pasid);
  431. if (pasid_state == NULL || pasid_state->invalid) {
  432. /* We know the device but not the PASID -> send INVALID */
  433. amd_iommu_complete_ppr(dev_state->pdev, iommu_fault->pasid,
  434. PPR_INVALID, tag);
  435. goto out_drop_state;
  436. }
  437. spin_lock_irqsave(&pasid_state->lock, flags);
  438. atomic_inc(&pasid_state->pri[tag].inflight);
  439. if (finish)
  440. pasid_state->pri[tag].finish = true;
  441. spin_unlock_irqrestore(&pasid_state->lock, flags);
  442. fault = kzalloc(sizeof(*fault), GFP_ATOMIC);
  443. if (fault == NULL) {
  444. /* We are OOM - send success and let the device re-fault */
  445. finish_pri_tag(dev_state, pasid_state, tag);
  446. goto out_drop_state;
  447. }
  448. fault->dev_state = dev_state;
  449. fault->address = iommu_fault->address;
  450. fault->state = pasid_state;
  451. fault->tag = tag;
  452. fault->finish = finish;
  453. fault->pasid = iommu_fault->pasid;
  454. fault->flags = iommu_fault->flags;
  455. INIT_WORK(&fault->work, do_fault);
  456. queue_work(iommu_wq, &fault->work);
  457. ret = NOTIFY_OK;
  458. out_drop_state:
  459. if (ret != NOTIFY_OK && pasid_state)
  460. put_pasid_state(pasid_state);
  461. put_device_state(dev_state);
  462. out:
  463. return ret;
  464. }
  465. static struct notifier_block ppr_nb = {
  466. .notifier_call = ppr_notifier,
  467. };
  468. int amd_iommu_bind_pasid(struct pci_dev *pdev, u32 pasid,
  469. struct task_struct *task)
  470. {
  471. struct pasid_state *pasid_state;
  472. struct device_state *dev_state;
  473. struct mm_struct *mm;
  474. u16 devid;
  475. int ret;
  476. might_sleep();
  477. if (!amd_iommu_v2_supported())
  478. return -ENODEV;
  479. devid = device_id(pdev);
  480. dev_state = get_device_state(devid);
  481. if (dev_state == NULL)
  482. return -EINVAL;
  483. ret = -EINVAL;
  484. if (pasid >= dev_state->max_pasids)
  485. goto out;
  486. ret = -ENOMEM;
  487. pasid_state = kzalloc(sizeof(*pasid_state), GFP_KERNEL);
  488. if (pasid_state == NULL)
  489. goto out;
  490. atomic_set(&pasid_state->count, 1);
  491. init_waitqueue_head(&pasid_state->wq);
  492. spin_lock_init(&pasid_state->lock);
  493. mm = get_task_mm(task);
  494. pasid_state->mm = mm;
  495. pasid_state->device_state = dev_state;
  496. pasid_state->pasid = pasid;
  497. pasid_state->invalid = true; /* Mark as valid only if we are
  498. done with setting up the pasid */
  499. pasid_state->mn.ops = &iommu_mn;
  500. if (pasid_state->mm == NULL)
  501. goto out_free;
  502. mmu_notifier_register(&pasid_state->mn, mm);
  503. ret = set_pasid_state(dev_state, pasid_state, pasid);
  504. if (ret)
  505. goto out_unregister;
  506. ret = amd_iommu_domain_set_gcr3(dev_state->domain, pasid,
  507. __pa(pasid_state->mm->pgd));
  508. if (ret)
  509. goto out_clear_state;
  510. /* Now we are ready to handle faults */
  511. pasid_state->invalid = false;
  512. /*
  513. * Drop the reference to the mm_struct here. We rely on the
  514. * mmu_notifier release call-back to inform us when the mm
  515. * is going away.
  516. */
  517. mmput(mm);
  518. return 0;
  519. out_clear_state:
  520. clear_pasid_state(dev_state, pasid);
  521. out_unregister:
  522. mmu_notifier_unregister(&pasid_state->mn, mm);
  523. mmput(mm);
  524. out_free:
  525. free_pasid_state(pasid_state);
  526. out:
  527. put_device_state(dev_state);
  528. return ret;
  529. }
  530. EXPORT_SYMBOL(amd_iommu_bind_pasid);
  531. void amd_iommu_unbind_pasid(struct pci_dev *pdev, u32 pasid)
  532. {
  533. struct pasid_state *pasid_state;
  534. struct device_state *dev_state;
  535. u16 devid;
  536. might_sleep();
  537. if (!amd_iommu_v2_supported())
  538. return;
  539. devid = device_id(pdev);
  540. dev_state = get_device_state(devid);
  541. if (dev_state == NULL)
  542. return;
  543. if (pasid >= dev_state->max_pasids)
  544. goto out;
  545. pasid_state = get_pasid_state(dev_state, pasid);
  546. if (pasid_state == NULL)
  547. goto out;
  548. /*
  549. * Drop reference taken here. We are safe because we still hold
  550. * the reference taken in the amd_iommu_bind_pasid function.
  551. */
  552. put_pasid_state(pasid_state);
  553. /* Clear the pasid state so that the pasid can be re-used */
  554. clear_pasid_state(dev_state, pasid_state->pasid);
  555. /*
  556. * Call mmu_notifier_unregister to drop our reference
  557. * to pasid_state->mm
  558. */
  559. mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);
  560. put_pasid_state_wait(pasid_state); /* Reference taken in
  561. amd_iommu_bind_pasid */
  562. out:
  563. /* Drop reference taken in this function */
  564. put_device_state(dev_state);
  565. /* Drop reference taken in amd_iommu_bind_pasid */
  566. put_device_state(dev_state);
  567. }
  568. EXPORT_SYMBOL(amd_iommu_unbind_pasid);
  569. int amd_iommu_init_device(struct pci_dev *pdev, int pasids)
  570. {
  571. struct device_state *dev_state;
  572. struct iommu_group *group;
  573. unsigned long flags;
  574. int ret, tmp;
  575. u16 devid;
  576. might_sleep();
  577. /*
  578. * When memory encryption is active the device is likely not in a
  579. * direct-mapped domain. Forbid using IOMMUv2 functionality for now.
  580. */
  581. if (mem_encrypt_active())
  582. return -ENODEV;
  583. if (!amd_iommu_v2_supported())
  584. return -ENODEV;
  585. if (pasids <= 0 || pasids > (PASID_MASK + 1))
  586. return -EINVAL;
  587. devid = device_id(pdev);
  588. dev_state = kzalloc(sizeof(*dev_state), GFP_KERNEL);
  589. if (dev_state == NULL)
  590. return -ENOMEM;
  591. spin_lock_init(&dev_state->lock);
  592. init_waitqueue_head(&dev_state->wq);
  593. dev_state->pdev = pdev;
  594. dev_state->devid = devid;
  595. tmp = pasids;
  596. for (dev_state->pasid_levels = 0; (tmp - 1) & ~0x1ff; tmp >>= 9)
  597. dev_state->pasid_levels += 1;
  598. atomic_set(&dev_state->count, 1);
  599. dev_state->max_pasids = pasids;
  600. ret = -ENOMEM;
  601. dev_state->states = (void *)get_zeroed_page(GFP_KERNEL);
  602. if (dev_state->states == NULL)
  603. goto out_free_dev_state;
  604. dev_state->domain = iommu_domain_alloc(&pci_bus_type);
  605. if (dev_state->domain == NULL)
  606. goto out_free_states;
  607. amd_iommu_domain_direct_map(dev_state->domain);
  608. ret = amd_iommu_domain_enable_v2(dev_state->domain, pasids);
  609. if (ret)
  610. goto out_free_domain;
  611. group = iommu_group_get(&pdev->dev);
  612. if (!group) {
  613. ret = -EINVAL;
  614. goto out_free_domain;
  615. }
  616. ret = iommu_attach_group(dev_state->domain, group);
  617. if (ret != 0)
  618. goto out_drop_group;
  619. iommu_group_put(group);
  620. spin_lock_irqsave(&state_lock, flags);
  621. if (__get_device_state(devid) != NULL) {
  622. spin_unlock_irqrestore(&state_lock, flags);
  623. ret = -EBUSY;
  624. goto out_free_domain;
  625. }
  626. list_add_tail(&dev_state->list, &state_list);
  627. spin_unlock_irqrestore(&state_lock, flags);
  628. return 0;
  629. out_drop_group:
  630. iommu_group_put(group);
  631. out_free_domain:
  632. iommu_domain_free(dev_state->domain);
  633. out_free_states:
  634. free_page((unsigned long)dev_state->states);
  635. out_free_dev_state:
  636. kfree(dev_state);
  637. return ret;
  638. }
  639. EXPORT_SYMBOL(amd_iommu_init_device);
  640. void amd_iommu_free_device(struct pci_dev *pdev)
  641. {
  642. struct device_state *dev_state;
  643. unsigned long flags;
  644. u16 devid;
  645. if (!amd_iommu_v2_supported())
  646. return;
  647. devid = device_id(pdev);
  648. spin_lock_irqsave(&state_lock, flags);
  649. dev_state = __get_device_state(devid);
  650. if (dev_state == NULL) {
  651. spin_unlock_irqrestore(&state_lock, flags);
  652. return;
  653. }
  654. list_del(&dev_state->list);
  655. spin_unlock_irqrestore(&state_lock, flags);
  656. /* Get rid of any remaining pasid states */
  657. free_pasid_states(dev_state);
  658. put_device_state(dev_state);
  659. /*
  660. * Wait until the last reference is dropped before freeing
  661. * the device state.
  662. */
  663. wait_event(dev_state->wq, !atomic_read(&dev_state->count));
  664. free_device_state(dev_state);
  665. }
  666. EXPORT_SYMBOL(amd_iommu_free_device);
  667. int amd_iommu_set_invalid_ppr_cb(struct pci_dev *pdev,
  668. amd_iommu_invalid_ppr_cb cb)
  669. {
  670. struct device_state *dev_state;
  671. unsigned long flags;
  672. u16 devid;
  673. int ret;
  674. if (!amd_iommu_v2_supported())
  675. return -ENODEV;
  676. devid = device_id(pdev);
  677. spin_lock_irqsave(&state_lock, flags);
  678. ret = -EINVAL;
  679. dev_state = __get_device_state(devid);
  680. if (dev_state == NULL)
  681. goto out_unlock;
  682. dev_state->inv_ppr_cb = cb;
  683. ret = 0;
  684. out_unlock:
  685. spin_unlock_irqrestore(&state_lock, flags);
  686. return ret;
  687. }
  688. EXPORT_SYMBOL(amd_iommu_set_invalid_ppr_cb);
  689. int amd_iommu_set_invalidate_ctx_cb(struct pci_dev *pdev,
  690. amd_iommu_invalidate_ctx cb)
  691. {
  692. struct device_state *dev_state;
  693. unsigned long flags;
  694. u16 devid;
  695. int ret;
  696. if (!amd_iommu_v2_supported())
  697. return -ENODEV;
  698. devid = device_id(pdev);
  699. spin_lock_irqsave(&state_lock, flags);
  700. ret = -EINVAL;
  701. dev_state = __get_device_state(devid);
  702. if (dev_state == NULL)
  703. goto out_unlock;
  704. dev_state->inv_ctx_cb = cb;
  705. ret = 0;
  706. out_unlock:
  707. spin_unlock_irqrestore(&state_lock, flags);
  708. return ret;
  709. }
  710. EXPORT_SYMBOL(amd_iommu_set_invalidate_ctx_cb);
  711. static int __init amd_iommu_v2_init(void)
  712. {
  713. int ret;
  714. if (!amd_iommu_v2_supported()) {
  715. pr_info("AMD IOMMUv2 functionality not available on this system - This is not a bug.\n");
  716. /*
  717. * Load anyway to provide the symbols to other modules
  718. * which may use AMD IOMMUv2 optionally.
  719. */
  720. return 0;
  721. }
  722. spin_lock_init(&state_lock);
  723. ret = -ENOMEM;
  724. iommu_wq = alloc_workqueue("amd_iommu_v2", WQ_MEM_RECLAIM, 0);
  725. if (iommu_wq == NULL)
  726. goto out;
  727. amd_iommu_register_ppr_notifier(&ppr_nb);
  728. pr_info("AMD IOMMUv2 loaded and initialized\n");
  729. return 0;
  730. out:
  731. return ret;
  732. }
  733. static void __exit amd_iommu_v2_exit(void)
  734. {
  735. struct device_state *dev_state;
  736. int i;
  737. if (!amd_iommu_v2_supported())
  738. return;
  739. amd_iommu_unregister_ppr_notifier(&ppr_nb);
  740. flush_workqueue(iommu_wq);
  741. /*
  742. * The loop below might call flush_workqueue(), so call
  743. * destroy_workqueue() after it
  744. */
  745. for (i = 0; i < MAX_DEVICES; ++i) {
  746. dev_state = get_device_state(i);
  747. if (dev_state == NULL)
  748. continue;
  749. WARN_ON_ONCE(1);
  750. put_device_state(dev_state);
  751. amd_iommu_free_device(dev_state->pdev);
  752. }
  753. destroy_workqueue(iommu_wq);
  754. }
  755. module_init(amd_iommu_v2_init);
  756. module_exit(amd_iommu_v2_exit);