intel-iommu.h 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818
  1. /* SPDX-License-Identifier: GPL-2.0-only */
  2. /*
  3. * Copyright © 2006-2015, Intel Corporation.
  4. *
  5. * Authors: Ashok Raj <ashok.raj@intel.com>
  6. * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  7. * David Woodhouse <David.Woodhouse@intel.com>
  8. */
  9. #ifndef _INTEL_IOMMU_H_
  10. #define _INTEL_IOMMU_H_
  11. #include <linux/types.h>
  12. #include <linux/iova.h>
  13. #include <linux/io.h>
  14. #include <linux/idr.h>
  15. #include <linux/mmu_notifier.h>
  16. #include <linux/list.h>
  17. #include <linux/iommu.h>
  18. #include <linux/io-64-nonatomic-lo-hi.h>
  19. #include <linux/dmar.h>
  20. #include <linux/ioasid.h>
  21. #include <asm/cacheflush.h>
  22. #include <asm/iommu.h>
  23. /*
  24. * VT-d hardware uses 4KiB page size regardless of host page size.
  25. */
  26. #define VTD_PAGE_SHIFT (12)
  27. #define VTD_PAGE_SIZE (1UL << VTD_PAGE_SHIFT)
  28. #define VTD_PAGE_MASK (((u64)-1) << VTD_PAGE_SHIFT)
  29. #define VTD_PAGE_ALIGN(addr) (((addr) + VTD_PAGE_SIZE - 1) & VTD_PAGE_MASK)
  30. #define VTD_STRIDE_SHIFT (9)
  31. #define VTD_STRIDE_MASK (((u64)-1) << VTD_STRIDE_SHIFT)
  32. #define DMA_PTE_READ BIT_ULL(0)
  33. #define DMA_PTE_WRITE BIT_ULL(1)
  34. #define DMA_PTE_LARGE_PAGE BIT_ULL(7)
  35. #define DMA_PTE_SNP BIT_ULL(11)
  36. #define DMA_FL_PTE_PRESENT BIT_ULL(0)
  37. #define DMA_FL_PTE_US BIT_ULL(2)
  38. #define DMA_FL_PTE_ACCESS BIT_ULL(5)
  39. #define DMA_FL_PTE_DIRTY BIT_ULL(6)
  40. #define DMA_FL_PTE_XD BIT_ULL(63)
  41. #define ADDR_WIDTH_5LEVEL (57)
  42. #define ADDR_WIDTH_4LEVEL (48)
  43. #define CONTEXT_TT_MULTI_LEVEL 0
  44. #define CONTEXT_TT_DEV_IOTLB 1
  45. #define CONTEXT_TT_PASS_THROUGH 2
  46. #define CONTEXT_PASIDE BIT_ULL(3)
  47. /*
  48. * Intel IOMMU register specification per version 1.0 public spec.
  49. */
  50. #define DMAR_VER_REG 0x0 /* Arch version supported by this IOMMU */
  51. #define DMAR_CAP_REG 0x8 /* Hardware supported capabilities */
  52. #define DMAR_ECAP_REG 0x10 /* Extended capabilities supported */
  53. #define DMAR_GCMD_REG 0x18 /* Global command register */
  54. #define DMAR_GSTS_REG 0x1c /* Global status register */
  55. #define DMAR_RTADDR_REG 0x20 /* Root entry table */
  56. #define DMAR_CCMD_REG 0x28 /* Context command reg */
  57. #define DMAR_FSTS_REG 0x34 /* Fault Status register */
  58. #define DMAR_FECTL_REG 0x38 /* Fault control register */
  59. #define DMAR_FEDATA_REG 0x3c /* Fault event interrupt data register */
  60. #define DMAR_FEADDR_REG 0x40 /* Fault event interrupt addr register */
  61. #define DMAR_FEUADDR_REG 0x44 /* Upper address register */
  62. #define DMAR_AFLOG_REG 0x58 /* Advanced Fault control */
  63. #define DMAR_PMEN_REG 0x64 /* Enable Protected Memory Region */
  64. #define DMAR_PLMBASE_REG 0x68 /* PMRR Low addr */
  65. #define DMAR_PLMLIMIT_REG 0x6c /* PMRR low limit */
  66. #define DMAR_PHMBASE_REG 0x70 /* pmrr high base addr */
  67. #define DMAR_PHMLIMIT_REG 0x78 /* pmrr high limit */
  68. #define DMAR_IQH_REG 0x80 /* Invalidation queue head register */
  69. #define DMAR_IQT_REG 0x88 /* Invalidation queue tail register */
  70. #define DMAR_IQ_SHIFT 4 /* Invalidation queue head/tail shift */
  71. #define DMAR_IQA_REG 0x90 /* Invalidation queue addr register */
  72. #define DMAR_ICS_REG 0x9c /* Invalidation complete status register */
  73. #define DMAR_IRTA_REG 0xb8 /* Interrupt remapping table addr register */
  74. #define DMAR_PQH_REG 0xc0 /* Page request queue head register */
  75. #define DMAR_PQT_REG 0xc8 /* Page request queue tail register */
  76. #define DMAR_PQA_REG 0xd0 /* Page request queue address register */
  77. #define DMAR_PRS_REG 0xdc /* Page request status register */
  78. #define DMAR_PECTL_REG 0xe0 /* Page request event control register */
  79. #define DMAR_PEDATA_REG 0xe4 /* Page request event interrupt data register */
  80. #define DMAR_PEADDR_REG 0xe8 /* Page request event interrupt addr register */
  81. #define DMAR_PEUADDR_REG 0xec /* Page request event Upper address register */
  82. #define DMAR_MTRRCAP_REG 0x100 /* MTRR capability register */
  83. #define DMAR_MTRRDEF_REG 0x108 /* MTRR default type register */
  84. #define DMAR_MTRR_FIX64K_00000_REG 0x120 /* MTRR Fixed range registers */
  85. #define DMAR_MTRR_FIX16K_80000_REG 0x128
  86. #define DMAR_MTRR_FIX16K_A0000_REG 0x130
  87. #define DMAR_MTRR_FIX4K_C0000_REG 0x138
  88. #define DMAR_MTRR_FIX4K_C8000_REG 0x140
  89. #define DMAR_MTRR_FIX4K_D0000_REG 0x148
  90. #define DMAR_MTRR_FIX4K_D8000_REG 0x150
  91. #define DMAR_MTRR_FIX4K_E0000_REG 0x158
  92. #define DMAR_MTRR_FIX4K_E8000_REG 0x160
  93. #define DMAR_MTRR_FIX4K_F0000_REG 0x168
  94. #define DMAR_MTRR_FIX4K_F8000_REG 0x170
  95. #define DMAR_MTRR_PHYSBASE0_REG 0x180 /* MTRR Variable range registers */
  96. #define DMAR_MTRR_PHYSMASK0_REG 0x188
  97. #define DMAR_MTRR_PHYSBASE1_REG 0x190
  98. #define DMAR_MTRR_PHYSMASK1_REG 0x198
  99. #define DMAR_MTRR_PHYSBASE2_REG 0x1a0
  100. #define DMAR_MTRR_PHYSMASK2_REG 0x1a8
  101. #define DMAR_MTRR_PHYSBASE3_REG 0x1b0
  102. #define DMAR_MTRR_PHYSMASK3_REG 0x1b8
  103. #define DMAR_MTRR_PHYSBASE4_REG 0x1c0
  104. #define DMAR_MTRR_PHYSMASK4_REG 0x1c8
  105. #define DMAR_MTRR_PHYSBASE5_REG 0x1d0
  106. #define DMAR_MTRR_PHYSMASK5_REG 0x1d8
  107. #define DMAR_MTRR_PHYSBASE6_REG 0x1e0
  108. #define DMAR_MTRR_PHYSMASK6_REG 0x1e8
  109. #define DMAR_MTRR_PHYSBASE7_REG 0x1f0
  110. #define DMAR_MTRR_PHYSMASK7_REG 0x1f8
  111. #define DMAR_MTRR_PHYSBASE8_REG 0x200
  112. #define DMAR_MTRR_PHYSMASK8_REG 0x208
  113. #define DMAR_MTRR_PHYSBASE9_REG 0x210
  114. #define DMAR_MTRR_PHYSMASK9_REG 0x218
  115. #define DMAR_VCCAP_REG 0xe30 /* Virtual command capability register */
  116. #define DMAR_VCMD_REG 0xe00 /* Virtual command register */
  117. #define DMAR_VCRSP_REG 0xe10 /* Virtual command response register */
  118. #define OFFSET_STRIDE (9)
  119. #define dmar_readq(a) readq(a)
  120. #define dmar_writeq(a,v) writeq(v,a)
  121. #define dmar_readl(a) readl(a)
  122. #define dmar_writel(a, v) writel(v, a)
  123. #define DMAR_VER_MAJOR(v) (((v) & 0xf0) >> 4)
  124. #define DMAR_VER_MINOR(v) ((v) & 0x0f)
  125. /*
  126. * Decoding Capability Register
  127. */
  128. #define cap_5lp_support(c) (((c) >> 60) & 1)
  129. #define cap_pi_support(c) (((c) >> 59) & 1)
  130. #define cap_fl1gp_support(c) (((c) >> 56) & 1)
  131. #define cap_read_drain(c) (((c) >> 55) & 1)
  132. #define cap_write_drain(c) (((c) >> 54) & 1)
  133. #define cap_max_amask_val(c) (((c) >> 48) & 0x3f)
  134. #define cap_num_fault_regs(c) ((((c) >> 40) & 0xff) + 1)
  135. #define cap_pgsel_inv(c) (((c) >> 39) & 1)
  136. #define cap_super_page_val(c) (((c) >> 34) & 0xf)
  137. #define cap_super_offset(c) (((find_first_bit(&cap_super_page_val(c), 4)) \
  138. * OFFSET_STRIDE) + 21)
  139. #define cap_fault_reg_offset(c) ((((c) >> 24) & 0x3ff) * 16)
  140. #define cap_max_fault_reg_offset(c) \
  141. (cap_fault_reg_offset(c) + cap_num_fault_regs(c) * 16)
  142. #define cap_zlr(c) (((c) >> 22) & 1)
  143. #define cap_isoch(c) (((c) >> 23) & 1)
  144. #define cap_mgaw(c) ((((c) >> 16) & 0x3f) + 1)
  145. #define cap_sagaw(c) (((c) >> 8) & 0x1f)
  146. #define cap_caching_mode(c) (((c) >> 7) & 1)
  147. #define cap_phmr(c) (((c) >> 6) & 1)
  148. #define cap_plmr(c) (((c) >> 5) & 1)
  149. #define cap_rwbf(c) (((c) >> 4) & 1)
  150. #define cap_afl(c) (((c) >> 3) & 1)
  151. #define cap_ndoms(c) (((unsigned long)1) << (4 + 2 * ((c) & 0x7)))
  152. /*
  153. * Extended Capability Register
  154. */
  155. #define ecap_smpwc(e) (((e) >> 48) & 0x1)
  156. #define ecap_flts(e) (((e) >> 47) & 0x1)
  157. #define ecap_slts(e) (((e) >> 46) & 0x1)
  158. #define ecap_vcs(e) (((e) >> 44) & 0x1)
  159. #define ecap_smts(e) (((e) >> 43) & 0x1)
  160. #define ecap_dit(e) ((e >> 41) & 0x1)
  161. #define ecap_pasid(e) ((e >> 40) & 0x1)
  162. #define ecap_pss(e) ((e >> 35) & 0x1f)
  163. #define ecap_eafs(e) ((e >> 34) & 0x1)
  164. #define ecap_nwfs(e) ((e >> 33) & 0x1)
  165. #define ecap_srs(e) ((e >> 31) & 0x1)
  166. #define ecap_ers(e) ((e >> 30) & 0x1)
  167. #define ecap_prs(e) ((e >> 29) & 0x1)
  168. #define ecap_broken_pasid(e) ((e >> 28) & 0x1)
  169. #define ecap_dis(e) ((e >> 27) & 0x1)
  170. #define ecap_nest(e) ((e >> 26) & 0x1)
  171. #define ecap_mts(e) ((e >> 25) & 0x1)
  172. #define ecap_ecs(e) ((e >> 24) & 0x1)
  173. #define ecap_iotlb_offset(e) ((((e) >> 8) & 0x3ff) * 16)
  174. #define ecap_max_iotlb_offset(e) (ecap_iotlb_offset(e) + 16)
  175. #define ecap_coherent(e) ((e) & 0x1)
  176. #define ecap_qis(e) ((e) & 0x2)
  177. #define ecap_pass_through(e) ((e >> 6) & 0x1)
  178. #define ecap_eim_support(e) ((e >> 4) & 0x1)
  179. #define ecap_ir_support(e) ((e >> 3) & 0x1)
  180. #define ecap_dev_iotlb_support(e) (((e) >> 2) & 0x1)
  181. #define ecap_max_handle_mask(e) ((e >> 20) & 0xf)
  182. #define ecap_sc_support(e) ((e >> 7) & 0x1) /* Snooping Control */
  183. /* Virtual command interface capability */
  184. #define vccap_pasid(v) (((v) & DMA_VCS_PAS)) /* PASID allocation */
  185. /* IOTLB_REG */
  186. #define DMA_TLB_FLUSH_GRANU_OFFSET 60
  187. #define DMA_TLB_GLOBAL_FLUSH (((u64)1) << 60)
  188. #define DMA_TLB_DSI_FLUSH (((u64)2) << 60)
  189. #define DMA_TLB_PSI_FLUSH (((u64)3) << 60)
  190. #define DMA_TLB_IIRG(type) ((type >> 60) & 3)
  191. #define DMA_TLB_IAIG(val) (((val) >> 57) & 3)
  192. #define DMA_TLB_READ_DRAIN (((u64)1) << 49)
  193. #define DMA_TLB_WRITE_DRAIN (((u64)1) << 48)
  194. #define DMA_TLB_DID(id) (((u64)((id) & 0xffff)) << 32)
  195. #define DMA_TLB_IVT (((u64)1) << 63)
  196. #define DMA_TLB_IH_NONLEAF (((u64)1) << 6)
  197. #define DMA_TLB_MAX_SIZE (0x3f)
  198. /* INVALID_DESC */
  199. #define DMA_CCMD_INVL_GRANU_OFFSET 61
  200. #define DMA_ID_TLB_GLOBAL_FLUSH (((u64)1) << 4)
  201. #define DMA_ID_TLB_DSI_FLUSH (((u64)2) << 4)
  202. #define DMA_ID_TLB_PSI_FLUSH (((u64)3) << 4)
  203. #define DMA_ID_TLB_READ_DRAIN (((u64)1) << 7)
  204. #define DMA_ID_TLB_WRITE_DRAIN (((u64)1) << 6)
  205. #define DMA_ID_TLB_DID(id) (((u64)((id & 0xffff) << 16)))
  206. #define DMA_ID_TLB_IH_NONLEAF (((u64)1) << 6)
  207. #define DMA_ID_TLB_ADDR(addr) (addr)
  208. #define DMA_ID_TLB_ADDR_MASK(mask) (mask)
  209. /* PMEN_REG */
  210. #define DMA_PMEN_EPM (((u32)1)<<31)
  211. #define DMA_PMEN_PRS (((u32)1)<<0)
  212. /* GCMD_REG */
  213. #define DMA_GCMD_TE (((u32)1) << 31)
  214. #define DMA_GCMD_SRTP (((u32)1) << 30)
  215. #define DMA_GCMD_SFL (((u32)1) << 29)
  216. #define DMA_GCMD_EAFL (((u32)1) << 28)
  217. #define DMA_GCMD_WBF (((u32)1) << 27)
  218. #define DMA_GCMD_QIE (((u32)1) << 26)
  219. #define DMA_GCMD_SIRTP (((u32)1) << 24)
  220. #define DMA_GCMD_IRE (((u32) 1) << 25)
  221. #define DMA_GCMD_CFI (((u32) 1) << 23)
  222. /* GSTS_REG */
  223. #define DMA_GSTS_TES (((u32)1) << 31)
  224. #define DMA_GSTS_RTPS (((u32)1) << 30)
  225. #define DMA_GSTS_FLS (((u32)1) << 29)
  226. #define DMA_GSTS_AFLS (((u32)1) << 28)
  227. #define DMA_GSTS_WBFS (((u32)1) << 27)
  228. #define DMA_GSTS_QIES (((u32)1) << 26)
  229. #define DMA_GSTS_IRTPS (((u32)1) << 24)
  230. #define DMA_GSTS_IRES (((u32)1) << 25)
  231. #define DMA_GSTS_CFIS (((u32)1) << 23)
  232. /* DMA_RTADDR_REG */
  233. #define DMA_RTADDR_RTT (((u64)1) << 11)
  234. #define DMA_RTADDR_SMT (((u64)1) << 10)
  235. /* CCMD_REG */
  236. #define DMA_CCMD_ICC (((u64)1) << 63)
  237. #define DMA_CCMD_GLOBAL_INVL (((u64)1) << 61)
  238. #define DMA_CCMD_DOMAIN_INVL (((u64)2) << 61)
  239. #define DMA_CCMD_DEVICE_INVL (((u64)3) << 61)
  240. #define DMA_CCMD_FM(m) (((u64)((m) & 0x3)) << 32)
  241. #define DMA_CCMD_MASK_NOBIT 0
  242. #define DMA_CCMD_MASK_1BIT 1
  243. #define DMA_CCMD_MASK_2BIT 2
  244. #define DMA_CCMD_MASK_3BIT 3
  245. #define DMA_CCMD_SID(s) (((u64)((s) & 0xffff)) << 16)
  246. #define DMA_CCMD_DID(d) ((u64)((d) & 0xffff))
  247. /* FECTL_REG */
  248. #define DMA_FECTL_IM (((u32)1) << 31)
  249. /* FSTS_REG */
  250. #define DMA_FSTS_PFO (1 << 0) /* Primary Fault Overflow */
  251. #define DMA_FSTS_PPF (1 << 1) /* Primary Pending Fault */
  252. #define DMA_FSTS_IQE (1 << 4) /* Invalidation Queue Error */
  253. #define DMA_FSTS_ICE (1 << 5) /* Invalidation Completion Error */
  254. #define DMA_FSTS_ITE (1 << 6) /* Invalidation Time-out Error */
  255. #define DMA_FSTS_PRO (1 << 7) /* Page Request Overflow */
  256. #define dma_fsts_fault_record_index(s) (((s) >> 8) & 0xff)
  257. /* FRCD_REG, 32 bits access */
  258. #define DMA_FRCD_F (((u32)1) << 31)
  259. #define dma_frcd_type(d) ((d >> 30) & 1)
  260. #define dma_frcd_fault_reason(c) (c & 0xff)
  261. #define dma_frcd_source_id(c) (c & 0xffff)
  262. #define dma_frcd_pasid_value(c) (((c) >> 8) & 0xfffff)
  263. #define dma_frcd_pasid_present(c) (((c) >> 31) & 1)
  264. /* low 64 bit */
  265. #define dma_frcd_page_addr(d) (d & (((u64)-1) << PAGE_SHIFT))
  266. /* PRS_REG */
  267. #define DMA_PRS_PPR ((u32)1)
  268. #define DMA_PRS_PRO ((u32)2)
  269. #define DMA_VCS_PAS ((u64)1)
  270. #define IOMMU_WAIT_OP(iommu, offset, op, cond, sts) \
  271. do { \
  272. cycles_t start_time = get_cycles(); \
  273. while (1) { \
  274. sts = op(iommu->reg + offset); \
  275. if (cond) \
  276. break; \
  277. if (DMAR_OPERATION_TIMEOUT < (get_cycles() - start_time))\
  278. panic("DMAR hardware is malfunctioning\n"); \
  279. cpu_relax(); \
  280. } \
  281. } while (0)
  282. #define QI_LENGTH 256 /* queue length */
  283. enum {
  284. QI_FREE,
  285. QI_IN_USE,
  286. QI_DONE,
  287. QI_ABORT
  288. };
  289. #define QI_CC_TYPE 0x1
  290. #define QI_IOTLB_TYPE 0x2
  291. #define QI_DIOTLB_TYPE 0x3
  292. #define QI_IEC_TYPE 0x4
  293. #define QI_IWD_TYPE 0x5
  294. #define QI_EIOTLB_TYPE 0x6
  295. #define QI_PC_TYPE 0x7
  296. #define QI_DEIOTLB_TYPE 0x8
  297. #define QI_PGRP_RESP_TYPE 0x9
  298. #define QI_PSTRM_RESP_TYPE 0xa
  299. #define QI_IEC_SELECTIVE (((u64)1) << 4)
  300. #define QI_IEC_IIDEX(idx) (((u64)(idx & 0xffff) << 32))
  301. #define QI_IEC_IM(m) (((u64)(m & 0x1f) << 27))
  302. #define QI_IWD_STATUS_DATA(d) (((u64)d) << 32)
  303. #define QI_IWD_STATUS_WRITE (((u64)1) << 5)
  304. #define QI_IWD_FENCE (((u64)1) << 6)
  305. #define QI_IWD_PRQ_DRAIN (((u64)1) << 7)
  306. #define QI_IOTLB_DID(did) (((u64)did) << 16)
  307. #define QI_IOTLB_DR(dr) (((u64)dr) << 7)
  308. #define QI_IOTLB_DW(dw) (((u64)dw) << 6)
  309. #define QI_IOTLB_GRAN(gran) (((u64)gran) >> (DMA_TLB_FLUSH_GRANU_OFFSET-4))
  310. #define QI_IOTLB_ADDR(addr) (((u64)addr) & VTD_PAGE_MASK)
  311. #define QI_IOTLB_IH(ih) (((u64)ih) << 6)
  312. #define QI_IOTLB_AM(am) (((u8)am) & 0x3f)
  313. #define QI_CC_FM(fm) (((u64)fm) << 48)
  314. #define QI_CC_SID(sid) (((u64)sid) << 32)
  315. #define QI_CC_DID(did) (((u64)did) << 16)
  316. #define QI_CC_GRAN(gran) (((u64)gran) >> (DMA_CCMD_INVL_GRANU_OFFSET-4))
  317. #define QI_DEV_IOTLB_SID(sid) ((u64)((sid) & 0xffff) << 32)
  318. #define QI_DEV_IOTLB_QDEP(qdep) (((qdep) & 0x1f) << 16)
  319. #define QI_DEV_IOTLB_ADDR(addr) ((u64)(addr) & VTD_PAGE_MASK)
  320. #define QI_DEV_IOTLB_PFSID(pfsid) (((u64)(pfsid & 0xf) << 12) | \
  321. ((u64)((pfsid >> 4) & 0xfff) << 52))
  322. #define QI_DEV_IOTLB_SIZE 1
  323. #define QI_DEV_IOTLB_MAX_INVS 32
  324. #define QI_PC_PASID(pasid) (((u64)pasid) << 32)
  325. #define QI_PC_DID(did) (((u64)did) << 16)
  326. #define QI_PC_GRAN(gran) (((u64)gran) << 4)
  327. /* PASID cache invalidation granu */
  328. #define QI_PC_ALL_PASIDS 0
  329. #define QI_PC_PASID_SEL 1
  330. #define QI_PC_GLOBAL 3
  331. #define QI_EIOTLB_ADDR(addr) ((u64)(addr) & VTD_PAGE_MASK)
  332. #define QI_EIOTLB_IH(ih) (((u64)ih) << 6)
  333. #define QI_EIOTLB_AM(am) (((u64)am) & 0x3f)
  334. #define QI_EIOTLB_PASID(pasid) (((u64)pasid) << 32)
  335. #define QI_EIOTLB_DID(did) (((u64)did) << 16)
  336. #define QI_EIOTLB_GRAN(gran) (((u64)gran) << 4)
  337. /* QI Dev-IOTLB inv granu */
  338. #define QI_DEV_IOTLB_GRAN_ALL 1
  339. #define QI_DEV_IOTLB_GRAN_PASID_SEL 0
  340. #define QI_DEV_EIOTLB_ADDR(a) ((u64)(a) & VTD_PAGE_MASK)
  341. #define QI_DEV_EIOTLB_SIZE (((u64)1) << 11)
  342. #define QI_DEV_EIOTLB_PASID(p) ((u64)((p) & 0xfffff) << 32)
  343. #define QI_DEV_EIOTLB_SID(sid) ((u64)((sid) & 0xffff) << 16)
  344. #define QI_DEV_EIOTLB_QDEP(qd) ((u64)((qd) & 0x1f) << 4)
  345. #define QI_DEV_EIOTLB_PFSID(pfsid) (((u64)(pfsid & 0xf) << 12) | \
  346. ((u64)((pfsid >> 4) & 0xfff) << 52))
  347. #define QI_DEV_EIOTLB_MAX_INVS 32
  348. /* Page group response descriptor QW0 */
  349. #define QI_PGRP_PASID_P(p) (((u64)(p)) << 4)
  350. #define QI_PGRP_PDP(p) (((u64)(p)) << 5)
  351. #define QI_PGRP_RESP_CODE(res) (((u64)(res)) << 12)
  352. #define QI_PGRP_DID(rid) (((u64)(rid)) << 16)
  353. #define QI_PGRP_PASID(pasid) (((u64)(pasid)) << 32)
  354. /* Page group response descriptor QW1 */
  355. #define QI_PGRP_LPIG(x) (((u64)(x)) << 2)
  356. #define QI_PGRP_IDX(idx) (((u64)(idx)) << 3)
  357. #define QI_RESP_SUCCESS 0x0
  358. #define QI_RESP_INVALID 0x1
  359. #define QI_RESP_FAILURE 0xf
  360. #define QI_GRAN_NONG_PASID 2
  361. #define QI_GRAN_PSI_PASID 3
  362. #define qi_shift(iommu) (DMAR_IQ_SHIFT + !!ecap_smts((iommu)->ecap))
  363. struct qi_desc {
  364. u64 qw0;
  365. u64 qw1;
  366. u64 qw2;
  367. u64 qw3;
  368. };
  369. struct q_inval {
  370. raw_spinlock_t q_lock;
  371. void *desc; /* invalidation queue */
  372. int *desc_status; /* desc status */
  373. int free_head; /* first free entry */
  374. int free_tail; /* last free entry */
  375. int free_cnt;
  376. };
  377. struct dmar_pci_notify_info;
  378. #ifdef CONFIG_IRQ_REMAP
  379. /* 1MB - maximum possible interrupt remapping table size */
  380. #define INTR_REMAP_PAGE_ORDER 8
  381. #define INTR_REMAP_TABLE_REG_SIZE 0xf
  382. #define INTR_REMAP_TABLE_REG_SIZE_MASK 0xf
  383. #define INTR_REMAP_TABLE_ENTRIES 65536
  384. struct irq_domain;
  385. struct ir_table {
  386. struct irte *base;
  387. unsigned long *bitmap;
  388. };
  389. void intel_irq_remap_add_device(struct dmar_pci_notify_info *info);
  390. #else
  391. static inline void
  392. intel_irq_remap_add_device(struct dmar_pci_notify_info *info) { }
  393. #endif
  394. struct iommu_flush {
  395. void (*flush_context)(struct intel_iommu *iommu, u16 did, u16 sid,
  396. u8 fm, u64 type);
  397. void (*flush_iotlb)(struct intel_iommu *iommu, u16 did, u64 addr,
  398. unsigned int size_order, u64 type);
  399. };
  400. enum {
  401. SR_DMAR_FECTL_REG,
  402. SR_DMAR_FEDATA_REG,
  403. SR_DMAR_FEADDR_REG,
  404. SR_DMAR_FEUADDR_REG,
  405. MAX_SR_DMAR_REGS
  406. };
  407. #define VTD_FLAG_TRANS_PRE_ENABLED (1 << 0)
  408. #define VTD_FLAG_IRQ_REMAP_PRE_ENABLED (1 << 1)
  409. #define VTD_FLAG_SVM_CAPABLE (1 << 2)
  410. extern int intel_iommu_sm;
  411. extern spinlock_t device_domain_lock;
  412. #define sm_supported(iommu) (intel_iommu_sm && ecap_smts((iommu)->ecap))
  413. #define pasid_supported(iommu) (sm_supported(iommu) && \
  414. ecap_pasid((iommu)->ecap))
  415. struct pasid_entry;
  416. struct pasid_state_entry;
  417. struct page_req_dsc;
  418. /*
  419. * 0: Present
  420. * 1-11: Reserved
  421. * 12-63: Context Ptr (12 - (haw-1))
  422. * 64-127: Reserved
  423. */
  424. struct root_entry {
  425. u64 lo;
  426. u64 hi;
  427. };
  428. /*
  429. * low 64 bits:
  430. * 0: present
  431. * 1: fault processing disable
  432. * 2-3: translation type
  433. * 12-63: address space root
  434. * high 64 bits:
  435. * 0-2: address width
  436. * 3-6: aval
  437. * 8-23: domain id
  438. */
  439. struct context_entry {
  440. u64 lo;
  441. u64 hi;
  442. };
  443. /* si_domain contains mulitple devices */
  444. #define DOMAIN_FLAG_STATIC_IDENTITY BIT(0)
  445. /*
  446. * When VT-d works in the scalable mode, it allows DMA translation to
  447. * happen through either first level or second level page table. This
  448. * bit marks that the DMA translation for the domain goes through the
  449. * first level page table, otherwise, it goes through the second level.
  450. */
  451. #define DOMAIN_FLAG_USE_FIRST_LEVEL BIT(1)
  452. /*
  453. * Domain represents a virtual machine which demands iommu nested
  454. * translation mode support.
  455. */
  456. #define DOMAIN_FLAG_NESTING_MODE BIT(2)
  457. struct dmar_domain {
  458. int nid; /* node id */
  459. unsigned iommu_refcnt[DMAR_UNITS_SUPPORTED];
  460. /* Refcount of devices per iommu */
  461. u16 iommu_did[DMAR_UNITS_SUPPORTED];
  462. /* Domain ids per IOMMU. Use u16 since
  463. * domain ids are 16 bit wide according
  464. * to VT-d spec, section 9.3 */
  465. unsigned int auxd_refcnt; /* Refcount of auxiliary attaching */
  466. bool has_iotlb_device;
  467. struct list_head devices; /* all devices' list */
  468. struct list_head auxd; /* link to device's auxiliary list */
  469. struct iova_domain iovad; /* iova's that belong to this domain */
  470. struct dma_pte *pgd; /* virtual address */
  471. int gaw; /* max guest address width */
  472. /* adjusted guest address width, 0 is level 2 30-bit */
  473. int agaw;
  474. int flags; /* flags to find out type of domain */
  475. int iommu_coherency;/* indicate coherency of iommu access */
  476. int iommu_snooping; /* indicate snooping control feature*/
  477. int iommu_count; /* reference count of iommu */
  478. int iommu_superpage;/* Level of superpages supported:
  479. 0 == 4KiB (no superpages), 1 == 2MiB,
  480. 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
  481. u64 max_addr; /* maximum mapped address */
  482. u32 default_pasid; /*
  483. * The default pasid used for non-SVM
  484. * traffic on mediated devices.
  485. */
  486. struct iommu_domain domain; /* generic domain data structure for
  487. iommu core */
  488. };
  489. struct intel_iommu {
  490. void __iomem *reg; /* Pointer to hardware regs, virtual addr */
  491. u64 reg_phys; /* physical address of hw register set */
  492. u64 reg_size; /* size of hw register set */
  493. u64 cap;
  494. u64 ecap;
  495. u64 vccap;
  496. u32 gcmd; /* Holds TE, EAFL. Don't need SRTP, SFL, WBF */
  497. raw_spinlock_t register_lock; /* protect register handling */
  498. int seq_id; /* sequence id of the iommu */
  499. int agaw; /* agaw of this iommu */
  500. int msagaw; /* max sagaw of this iommu */
  501. unsigned int irq, pr_irq;
  502. u16 segment; /* PCI segment# */
  503. unsigned char name[13]; /* Device Name */
  504. #ifdef CONFIG_INTEL_IOMMU
  505. unsigned long *domain_ids; /* bitmap of domains */
  506. struct dmar_domain ***domains; /* ptr to domains */
  507. spinlock_t lock; /* protect context, domain ids */
  508. struct root_entry *root_entry; /* virtual address */
  509. struct iommu_flush flush;
  510. #endif
  511. #ifdef CONFIG_INTEL_IOMMU_SVM
  512. struct page_req_dsc *prq;
  513. unsigned char prq_name[16]; /* Name for PRQ interrupt */
  514. struct completion prq_complete;
  515. struct ioasid_allocator_ops pasid_allocator; /* Custom allocator for PASIDs */
  516. #endif
  517. struct q_inval *qi; /* Queued invalidation info */
  518. u32 *iommu_state; /* Store iommu states between suspend and resume.*/
  519. #ifdef CONFIG_IRQ_REMAP
  520. struct ir_table *ir_table; /* Interrupt remapping info */
  521. struct irq_domain *ir_domain;
  522. struct irq_domain *ir_msi_domain;
  523. #endif
  524. struct iommu_device iommu; /* IOMMU core code handle */
  525. int node;
  526. u32 flags; /* Software defined flags */
  527. struct dmar_drhd_unit *drhd;
  528. };
  529. /* PCI domain-device relationship */
  530. struct device_domain_info {
  531. struct list_head link; /* link to domain siblings */
  532. struct list_head global; /* link to global list */
  533. struct list_head table; /* link to pasid table */
  534. struct list_head auxiliary_domains; /* auxiliary domains
  535. * attached to this device
  536. */
  537. u32 segment; /* PCI segment number */
  538. u8 bus; /* PCI bus number */
  539. u8 devfn; /* PCI devfn number */
  540. u16 pfsid; /* SRIOV physical function source ID */
  541. u8 pasid_supported:3;
  542. u8 pasid_enabled:1;
  543. u8 pri_supported:1;
  544. u8 pri_enabled:1;
  545. u8 ats_supported:1;
  546. u8 ats_enabled:1;
  547. u8 auxd_enabled:1; /* Multiple domains per device */
  548. u8 ats_qdep;
  549. struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
  550. struct intel_iommu *iommu; /* IOMMU used by this device */
  551. struct dmar_domain *domain; /* pointer to domain */
  552. struct pasid_table *pasid_table; /* pasid table */
  553. };
  554. static inline void __iommu_flush_cache(
  555. struct intel_iommu *iommu, void *addr, int size)
  556. {
  557. if (!ecap_coherent(iommu->ecap))
  558. clflush_cache_range(addr, size);
  559. }
  560. /* Convert generic struct iommu_domain to private struct dmar_domain */
  561. static inline struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
  562. {
  563. return container_of(dom, struct dmar_domain, domain);
  564. }
  565. /*
  566. * 0: readable
  567. * 1: writable
  568. * 2-6: reserved
  569. * 7: super page
  570. * 8-10: available
  571. * 11: snoop behavior
  572. * 12-63: Host physcial address
  573. */
  574. struct dma_pte {
  575. u64 val;
  576. };
  577. static inline void dma_clear_pte(struct dma_pte *pte)
  578. {
  579. pte->val = 0;
  580. }
  581. static inline u64 dma_pte_addr(struct dma_pte *pte)
  582. {
  583. #ifdef CONFIG_64BIT
  584. return pte->val & VTD_PAGE_MASK & (~DMA_FL_PTE_XD);
  585. #else
  586. /* Must have a full atomic 64-bit read */
  587. return __cmpxchg64(&pte->val, 0ULL, 0ULL) &
  588. VTD_PAGE_MASK & (~DMA_FL_PTE_XD);
  589. #endif
  590. }
  591. static inline bool dma_pte_present(struct dma_pte *pte)
  592. {
  593. return (pte->val & 3) != 0;
  594. }
  595. static inline bool dma_pte_superpage(struct dma_pte *pte)
  596. {
  597. return (pte->val & DMA_PTE_LARGE_PAGE);
  598. }
  599. static inline int first_pte_in_page(struct dma_pte *pte)
  600. {
  601. return !((unsigned long)pte & ~VTD_PAGE_MASK);
  602. }
  603. extern struct dmar_drhd_unit * dmar_find_matched_drhd_unit(struct pci_dev *dev);
  604. extern int dmar_find_matched_atsr_unit(struct pci_dev *dev);
  605. extern int dmar_enable_qi(struct intel_iommu *iommu);
  606. extern void dmar_disable_qi(struct intel_iommu *iommu);
  607. extern int dmar_reenable_qi(struct intel_iommu *iommu);
  608. extern void qi_global_iec(struct intel_iommu *iommu);
  609. extern void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid,
  610. u8 fm, u64 type);
  611. extern void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
  612. unsigned int size_order, u64 type);
  613. extern void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
  614. u16 qdep, u64 addr, unsigned mask);
  615. void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
  616. unsigned long npages, bool ih);
  617. void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid,
  618. u32 pasid, u16 qdep, u64 addr,
  619. unsigned int size_order);
  620. void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did, u64 granu,
  621. u32 pasid);
  622. int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc,
  623. unsigned int count, unsigned long options);
  624. /*
  625. * Options used in qi_submit_sync:
  626. * QI_OPT_WAIT_DRAIN - Wait for PRQ drain completion, spec 6.5.2.8.
  627. */
  628. #define QI_OPT_WAIT_DRAIN BIT(0)
  629. extern int dmar_ir_support(void);
  630. void *alloc_pgtable_page(int node);
  631. void free_pgtable_page(void *vaddr);
  632. struct intel_iommu *domain_get_iommu(struct dmar_domain *domain);
  633. int for_each_device_domain(int (*fn)(struct device_domain_info *info,
  634. void *data), void *data);
  635. void iommu_flush_write_buffer(struct intel_iommu *iommu);
  636. int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev);
  637. struct dmar_domain *find_domain(struct device *dev);
  638. struct device_domain_info *get_domain_info(struct device *dev);
  639. struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn);
  640. #ifdef CONFIG_INTEL_IOMMU_SVM
  641. extern void intel_svm_check(struct intel_iommu *iommu);
  642. extern int intel_svm_enable_prq(struct intel_iommu *iommu);
  643. extern int intel_svm_finish_prq(struct intel_iommu *iommu);
  644. int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev,
  645. struct iommu_gpasid_bind_data *data);
  646. int intel_svm_unbind_gpasid(struct device *dev, u32 pasid);
  647. struct iommu_sva *intel_svm_bind(struct device *dev, struct mm_struct *mm,
  648. void *drvdata);
  649. void intel_svm_unbind(struct iommu_sva *handle);
  650. u32 intel_svm_get_pasid(struct iommu_sva *handle);
  651. int intel_svm_page_response(struct device *dev, struct iommu_fault_event *evt,
  652. struct iommu_page_response *msg);
  653. struct svm_dev_ops;
  654. struct intel_svm_dev {
  655. struct list_head list;
  656. struct rcu_head rcu;
  657. struct device *dev;
  658. struct intel_iommu *iommu;
  659. struct svm_dev_ops *ops;
  660. struct iommu_sva sva;
  661. u32 pasid;
  662. int users;
  663. u16 did;
  664. u16 dev_iotlb:1;
  665. u16 sid, qdep;
  666. };
  667. struct intel_svm {
  668. struct mmu_notifier notifier;
  669. struct mm_struct *mm;
  670. unsigned int flags;
  671. u32 pasid;
  672. int gpasid; /* In case that guest PASID is different from host PASID */
  673. struct list_head devs;
  674. struct list_head list;
  675. };
  676. #else
  677. static inline void intel_svm_check(struct intel_iommu *iommu) {}
  678. #endif
  679. #ifdef CONFIG_INTEL_IOMMU_DEBUGFS
  680. void intel_iommu_debugfs_init(void);
  681. #else
  682. static inline void intel_iommu_debugfs_init(void) {}
  683. #endif /* CONFIG_INTEL_IOMMU_DEBUGFS */
  684. extern const struct attribute_group *intel_iommu_groups[];
  685. bool context_present(struct context_entry *context);
  686. struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
  687. u8 devfn, int alloc);
  688. #ifdef CONFIG_INTEL_IOMMU
  689. extern int iommu_calculate_agaw(struct intel_iommu *iommu);
  690. extern int iommu_calculate_max_sagaw(struct intel_iommu *iommu);
  691. extern int dmar_disabled;
  692. extern int intel_iommu_enabled;
  693. extern int intel_iommu_gfx_mapped;
  694. #else
  695. static inline int iommu_calculate_agaw(struct intel_iommu *iommu)
  696. {
  697. return 0;
  698. }
  699. static inline int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
  700. {
  701. return 0;
  702. }
  703. #define dmar_disabled (1)
  704. #define intel_iommu_enabled (0)
  705. #endif
  706. #endif