pci.c 59 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Copyright 2014 IBM Corp.
  4. */
  5. #include <linux/pci_regs.h>
  6. #include <linux/pci_ids.h>
  7. #include <linux/device.h>
  8. #include <linux/module.h>
  9. #include <linux/kernel.h>
  10. #include <linux/slab.h>
  11. #include <linux/sort.h>
  12. #include <linux/pci.h>
  13. #include <linux/of.h>
  14. #include <linux/delay.h>
  15. #include <asm/opal.h>
  16. #include <asm/msi_bitmap.h>
  17. #include <asm/pnv-pci.h>
  18. #include <asm/io.h>
  19. #include <asm/reg.h>
  20. #include "cxl.h"
  21. #include <misc/cxl.h>
  22. #define CXL_PCI_VSEC_ID 0x1280
  23. #define CXL_VSEC_MIN_SIZE 0x80
  24. #define CXL_READ_VSEC_LENGTH(dev, vsec, dest) \
  25. { \
  26. pci_read_config_word(dev, vsec + 0x6, dest); \
  27. *dest >>= 4; \
  28. }
  29. #define CXL_READ_VSEC_NAFUS(dev, vsec, dest) \
  30. pci_read_config_byte(dev, vsec + 0x8, dest)
  31. #define CXL_READ_VSEC_STATUS(dev, vsec, dest) \
  32. pci_read_config_byte(dev, vsec + 0x9, dest)
  33. #define CXL_STATUS_SECOND_PORT 0x80
  34. #define CXL_STATUS_MSI_X_FULL 0x40
  35. #define CXL_STATUS_MSI_X_SINGLE 0x20
  36. #define CXL_STATUS_FLASH_RW 0x08
  37. #define CXL_STATUS_FLASH_RO 0x04
  38. #define CXL_STATUS_LOADABLE_AFU 0x02
  39. #define CXL_STATUS_LOADABLE_PSL 0x01
  40. /* If we see these features we won't try to use the card */
  41. #define CXL_UNSUPPORTED_FEATURES \
  42. (CXL_STATUS_MSI_X_FULL | CXL_STATUS_MSI_X_SINGLE)
  43. #define CXL_READ_VSEC_MODE_CONTROL(dev, vsec, dest) \
  44. pci_read_config_byte(dev, vsec + 0xa, dest)
  45. #define CXL_WRITE_VSEC_MODE_CONTROL(dev, vsec, val) \
  46. pci_write_config_byte(dev, vsec + 0xa, val)
  47. #define CXL_VSEC_PROTOCOL_MASK 0xe0
  48. #define CXL_VSEC_PROTOCOL_1024TB 0x80
  49. #define CXL_VSEC_PROTOCOL_512TB 0x40
  50. #define CXL_VSEC_PROTOCOL_256TB 0x20 /* Power 8/9 uses this */
  51. #define CXL_VSEC_PROTOCOL_ENABLE 0x01
  52. #define CXL_READ_VSEC_PSL_REVISION(dev, vsec, dest) \
  53. pci_read_config_word(dev, vsec + 0xc, dest)
  54. #define CXL_READ_VSEC_CAIA_MINOR(dev, vsec, dest) \
  55. pci_read_config_byte(dev, vsec + 0xe, dest)
  56. #define CXL_READ_VSEC_CAIA_MAJOR(dev, vsec, dest) \
  57. pci_read_config_byte(dev, vsec + 0xf, dest)
  58. #define CXL_READ_VSEC_BASE_IMAGE(dev, vsec, dest) \
  59. pci_read_config_word(dev, vsec + 0x10, dest)
  60. #define CXL_READ_VSEC_IMAGE_STATE(dev, vsec, dest) \
  61. pci_read_config_byte(dev, vsec + 0x13, dest)
  62. #define CXL_WRITE_VSEC_IMAGE_STATE(dev, vsec, val) \
  63. pci_write_config_byte(dev, vsec + 0x13, val)
  64. #define CXL_VSEC_USER_IMAGE_LOADED 0x80 /* RO */
  65. #define CXL_VSEC_PERST_LOADS_IMAGE 0x20 /* RW */
  66. #define CXL_VSEC_PERST_SELECT_USER 0x10 /* RW */
  67. #define CXL_READ_VSEC_AFU_DESC_OFF(dev, vsec, dest) \
  68. pci_read_config_dword(dev, vsec + 0x20, dest)
  69. #define CXL_READ_VSEC_AFU_DESC_SIZE(dev, vsec, dest) \
  70. pci_read_config_dword(dev, vsec + 0x24, dest)
  71. #define CXL_READ_VSEC_PS_OFF(dev, vsec, dest) \
  72. pci_read_config_dword(dev, vsec + 0x28, dest)
  73. #define CXL_READ_VSEC_PS_SIZE(dev, vsec, dest) \
  74. pci_read_config_dword(dev, vsec + 0x2c, dest)
  75. /* This works a little different than the p1/p2 register accesses to make it
  76. * easier to pull out individual fields */
  77. #define AFUD_READ(afu, off) in_be64(afu->native->afu_desc_mmio + off)
  78. #define AFUD_READ_LE(afu, off) in_le64(afu->native->afu_desc_mmio + off)
  79. #define EXTRACT_PPC_BIT(val, bit) (!!(val & PPC_BIT(bit)))
  80. #define EXTRACT_PPC_BITS(val, bs, be) ((val & PPC_BITMASK(bs, be)) >> PPC_BITLSHIFT(be))
  81. #define AFUD_READ_INFO(afu) AFUD_READ(afu, 0x0)
  82. #define AFUD_NUM_INTS_PER_PROC(val) EXTRACT_PPC_BITS(val, 0, 15)
  83. #define AFUD_NUM_PROCS(val) EXTRACT_PPC_BITS(val, 16, 31)
  84. #define AFUD_NUM_CRS(val) EXTRACT_PPC_BITS(val, 32, 47)
  85. #define AFUD_MULTIMODE(val) EXTRACT_PPC_BIT(val, 48)
  86. #define AFUD_PUSH_BLOCK_TRANSFER(val) EXTRACT_PPC_BIT(val, 55)
  87. #define AFUD_DEDICATED_PROCESS(val) EXTRACT_PPC_BIT(val, 59)
  88. #define AFUD_AFU_DIRECTED(val) EXTRACT_PPC_BIT(val, 61)
  89. #define AFUD_TIME_SLICED(val) EXTRACT_PPC_BIT(val, 63)
  90. #define AFUD_READ_CR(afu) AFUD_READ(afu, 0x20)
  91. #define AFUD_CR_LEN(val) EXTRACT_PPC_BITS(val, 8, 63)
  92. #define AFUD_READ_CR_OFF(afu) AFUD_READ(afu, 0x28)
  93. #define AFUD_READ_PPPSA(afu) AFUD_READ(afu, 0x30)
  94. #define AFUD_PPPSA_PP(val) EXTRACT_PPC_BIT(val, 6)
  95. #define AFUD_PPPSA_PSA(val) EXTRACT_PPC_BIT(val, 7)
  96. #define AFUD_PPPSA_LEN(val) EXTRACT_PPC_BITS(val, 8, 63)
  97. #define AFUD_READ_PPPSA_OFF(afu) AFUD_READ(afu, 0x38)
  98. #define AFUD_READ_EB(afu) AFUD_READ(afu, 0x40)
  99. #define AFUD_EB_LEN(val) EXTRACT_PPC_BITS(val, 8, 63)
  100. #define AFUD_READ_EB_OFF(afu) AFUD_READ(afu, 0x48)
  101. static const struct pci_device_id cxl_pci_tbl[] = {
  102. { PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x0477), },
  103. { PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x044b), },
  104. { PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x04cf), },
  105. { PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x0601), },
  106. { PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x0623), },
  107. { PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x0628), },
  108. { }
  109. };
  110. MODULE_DEVICE_TABLE(pci, cxl_pci_tbl);
  111. /*
  112. * Mostly using these wrappers to avoid confusion:
  113. * priv 1 is BAR2, while priv 2 is BAR0
  114. */
  115. static inline resource_size_t p1_base(struct pci_dev *dev)
  116. {
  117. return pci_resource_start(dev, 2);
  118. }
  119. static inline resource_size_t p1_size(struct pci_dev *dev)
  120. {
  121. return pci_resource_len(dev, 2);
  122. }
  123. static inline resource_size_t p2_base(struct pci_dev *dev)
  124. {
  125. return pci_resource_start(dev, 0);
  126. }
  127. static inline resource_size_t p2_size(struct pci_dev *dev)
  128. {
  129. return pci_resource_len(dev, 0);
  130. }
  131. static int find_cxl_vsec(struct pci_dev *dev)
  132. {
  133. int vsec = 0;
  134. u16 val;
  135. while ((vsec = pci_find_next_ext_capability(dev, vsec, PCI_EXT_CAP_ID_VNDR))) {
  136. pci_read_config_word(dev, vsec + 0x4, &val);
  137. if (val == CXL_PCI_VSEC_ID)
  138. return vsec;
  139. }
  140. return 0;
  141. }
  142. static void dump_cxl_config_space(struct pci_dev *dev)
  143. {
  144. int vsec;
  145. u32 val;
  146. dev_info(&dev->dev, "dump_cxl_config_space\n");
  147. pci_read_config_dword(dev, PCI_BASE_ADDRESS_0, &val);
  148. dev_info(&dev->dev, "BAR0: %#.8x\n", val);
  149. pci_read_config_dword(dev, PCI_BASE_ADDRESS_1, &val);
  150. dev_info(&dev->dev, "BAR1: %#.8x\n", val);
  151. pci_read_config_dword(dev, PCI_BASE_ADDRESS_2, &val);
  152. dev_info(&dev->dev, "BAR2: %#.8x\n", val);
  153. pci_read_config_dword(dev, PCI_BASE_ADDRESS_3, &val);
  154. dev_info(&dev->dev, "BAR3: %#.8x\n", val);
  155. pci_read_config_dword(dev, PCI_BASE_ADDRESS_4, &val);
  156. dev_info(&dev->dev, "BAR4: %#.8x\n", val);
  157. pci_read_config_dword(dev, PCI_BASE_ADDRESS_5, &val);
  158. dev_info(&dev->dev, "BAR5: %#.8x\n", val);
  159. dev_info(&dev->dev, "p1 regs: %#llx, len: %#llx\n",
  160. p1_base(dev), p1_size(dev));
  161. dev_info(&dev->dev, "p2 regs: %#llx, len: %#llx\n",
  162. p2_base(dev), p2_size(dev));
  163. dev_info(&dev->dev, "BAR 4/5: %#llx, len: %#llx\n",
  164. pci_resource_start(dev, 4), pci_resource_len(dev, 4));
  165. if (!(vsec = find_cxl_vsec(dev)))
  166. return;
  167. #define show_reg(name, what) \
  168. dev_info(&dev->dev, "cxl vsec: %30s: %#x\n", name, what)
  169. pci_read_config_dword(dev, vsec + 0x0, &val);
  170. show_reg("Cap ID", (val >> 0) & 0xffff);
  171. show_reg("Cap Ver", (val >> 16) & 0xf);
  172. show_reg("Next Cap Ptr", (val >> 20) & 0xfff);
  173. pci_read_config_dword(dev, vsec + 0x4, &val);
  174. show_reg("VSEC ID", (val >> 0) & 0xffff);
  175. show_reg("VSEC Rev", (val >> 16) & 0xf);
  176. show_reg("VSEC Length", (val >> 20) & 0xfff);
  177. pci_read_config_dword(dev, vsec + 0x8, &val);
  178. show_reg("Num AFUs", (val >> 0) & 0xff);
  179. show_reg("Status", (val >> 8) & 0xff);
  180. show_reg("Mode Control", (val >> 16) & 0xff);
  181. show_reg("Reserved", (val >> 24) & 0xff);
  182. pci_read_config_dword(dev, vsec + 0xc, &val);
  183. show_reg("PSL Rev", (val >> 0) & 0xffff);
  184. show_reg("CAIA Ver", (val >> 16) & 0xffff);
  185. pci_read_config_dword(dev, vsec + 0x10, &val);
  186. show_reg("Base Image Rev", (val >> 0) & 0xffff);
  187. show_reg("Reserved", (val >> 16) & 0x0fff);
  188. show_reg("Image Control", (val >> 28) & 0x3);
  189. show_reg("Reserved", (val >> 30) & 0x1);
  190. show_reg("Image Loaded", (val >> 31) & 0x1);
  191. pci_read_config_dword(dev, vsec + 0x14, &val);
  192. show_reg("Reserved", val);
  193. pci_read_config_dword(dev, vsec + 0x18, &val);
  194. show_reg("Reserved", val);
  195. pci_read_config_dword(dev, vsec + 0x1c, &val);
  196. show_reg("Reserved", val);
  197. pci_read_config_dword(dev, vsec + 0x20, &val);
  198. show_reg("AFU Descriptor Offset", val);
  199. pci_read_config_dword(dev, vsec + 0x24, &val);
  200. show_reg("AFU Descriptor Size", val);
  201. pci_read_config_dword(dev, vsec + 0x28, &val);
  202. show_reg("Problem State Offset", val);
  203. pci_read_config_dword(dev, vsec + 0x2c, &val);
  204. show_reg("Problem State Size", val);
  205. pci_read_config_dword(dev, vsec + 0x30, &val);
  206. show_reg("Reserved", val);
  207. pci_read_config_dword(dev, vsec + 0x34, &val);
  208. show_reg("Reserved", val);
  209. pci_read_config_dword(dev, vsec + 0x38, &val);
  210. show_reg("Reserved", val);
  211. pci_read_config_dword(dev, vsec + 0x3c, &val);
  212. show_reg("Reserved", val);
  213. pci_read_config_dword(dev, vsec + 0x40, &val);
  214. show_reg("PSL Programming Port", val);
  215. pci_read_config_dword(dev, vsec + 0x44, &val);
  216. show_reg("PSL Programming Control", val);
  217. pci_read_config_dword(dev, vsec + 0x48, &val);
  218. show_reg("Reserved", val);
  219. pci_read_config_dword(dev, vsec + 0x4c, &val);
  220. show_reg("Reserved", val);
  221. pci_read_config_dword(dev, vsec + 0x50, &val);
  222. show_reg("Flash Address Register", val);
  223. pci_read_config_dword(dev, vsec + 0x54, &val);
  224. show_reg("Flash Size Register", val);
  225. pci_read_config_dword(dev, vsec + 0x58, &val);
  226. show_reg("Flash Status/Control Register", val);
  227. pci_read_config_dword(dev, vsec + 0x58, &val);
  228. show_reg("Flash Data Port", val);
  229. #undef show_reg
  230. }
  231. static void dump_afu_descriptor(struct cxl_afu *afu)
  232. {
  233. u64 val, afu_cr_num, afu_cr_off, afu_cr_len;
  234. int i;
  235. #define show_reg(name, what) \
  236. dev_info(&afu->dev, "afu desc: %30s: %#llx\n", name, what)
  237. val = AFUD_READ_INFO(afu);
  238. show_reg("num_ints_per_process", AFUD_NUM_INTS_PER_PROC(val));
  239. show_reg("num_of_processes", AFUD_NUM_PROCS(val));
  240. show_reg("num_of_afu_CRs", AFUD_NUM_CRS(val));
  241. show_reg("req_prog_mode", val & 0xffffULL);
  242. afu_cr_num = AFUD_NUM_CRS(val);
  243. val = AFUD_READ(afu, 0x8);
  244. show_reg("Reserved", val);
  245. val = AFUD_READ(afu, 0x10);
  246. show_reg("Reserved", val);
  247. val = AFUD_READ(afu, 0x18);
  248. show_reg("Reserved", val);
  249. val = AFUD_READ_CR(afu);
  250. show_reg("Reserved", (val >> (63-7)) & 0xff);
  251. show_reg("AFU_CR_len", AFUD_CR_LEN(val));
  252. afu_cr_len = AFUD_CR_LEN(val) * 256;
  253. val = AFUD_READ_CR_OFF(afu);
  254. afu_cr_off = val;
  255. show_reg("AFU_CR_offset", val);
  256. val = AFUD_READ_PPPSA(afu);
  257. show_reg("PerProcessPSA_control", (val >> (63-7)) & 0xff);
  258. show_reg("PerProcessPSA Length", AFUD_PPPSA_LEN(val));
  259. val = AFUD_READ_PPPSA_OFF(afu);
  260. show_reg("PerProcessPSA_offset", val);
  261. val = AFUD_READ_EB(afu);
  262. show_reg("Reserved", (val >> (63-7)) & 0xff);
  263. show_reg("AFU_EB_len", AFUD_EB_LEN(val));
  264. val = AFUD_READ_EB_OFF(afu);
  265. show_reg("AFU_EB_offset", val);
  266. for (i = 0; i < afu_cr_num; i++) {
  267. val = AFUD_READ_LE(afu, afu_cr_off + i * afu_cr_len);
  268. show_reg("CR Vendor", val & 0xffff);
  269. show_reg("CR Device", (val >> 16) & 0xffff);
  270. }
  271. #undef show_reg
  272. }
  273. #define P8_CAPP_UNIT0_ID 0xBA
  274. #define P8_CAPP_UNIT1_ID 0XBE
  275. #define P9_CAPP_UNIT0_ID 0xC0
  276. #define P9_CAPP_UNIT1_ID 0xE0
  277. static int get_phb_index(struct device_node *np, u32 *phb_index)
  278. {
  279. if (of_property_read_u32(np, "ibm,phb-index", phb_index))
  280. return -ENODEV;
  281. return 0;
  282. }
  283. static u64 get_capp_unit_id(struct device_node *np, u32 phb_index)
  284. {
  285. /*
  286. * POWER 8:
  287. * - For chips other than POWER8NVL, we only have CAPP 0,
  288. * irrespective of which PHB is used.
  289. * - For POWER8NVL, assume CAPP 0 is attached to PHB0 and
  290. * CAPP 1 is attached to PHB1.
  291. */
  292. if (cxl_is_power8()) {
  293. if (!pvr_version_is(PVR_POWER8NVL))
  294. return P8_CAPP_UNIT0_ID;
  295. if (phb_index == 0)
  296. return P8_CAPP_UNIT0_ID;
  297. if (phb_index == 1)
  298. return P8_CAPP_UNIT1_ID;
  299. }
  300. /*
  301. * POWER 9:
  302. * PEC0 (PHB0). Capp ID = CAPP0 (0b1100_0000)
  303. * PEC1 (PHB1 - PHB2). No capi mode
  304. * PEC2 (PHB3 - PHB4 - PHB5): Capi mode on PHB3 only. Capp ID = CAPP1 (0b1110_0000)
  305. */
  306. if (cxl_is_power9()) {
  307. if (phb_index == 0)
  308. return P9_CAPP_UNIT0_ID;
  309. if (phb_index == 3)
  310. return P9_CAPP_UNIT1_ID;
  311. }
  312. return 0;
  313. }
  314. int cxl_calc_capp_routing(struct pci_dev *dev, u64 *chipid,
  315. u32 *phb_index, u64 *capp_unit_id)
  316. {
  317. int rc;
  318. struct device_node *np;
  319. const __be32 *prop;
  320. if (!(np = pnv_pci_get_phb_node(dev)))
  321. return -ENODEV;
  322. while (np && !(prop = of_get_property(np, "ibm,chip-id", NULL)))
  323. np = of_get_next_parent(np);
  324. if (!np)
  325. return -ENODEV;
  326. *chipid = be32_to_cpup(prop);
  327. rc = get_phb_index(np, phb_index);
  328. if (rc) {
  329. pr_err("cxl: invalid phb index\n");
  330. return rc;
  331. }
  332. *capp_unit_id = get_capp_unit_id(np, *phb_index);
  333. of_node_put(np);
  334. if (!*capp_unit_id) {
  335. pr_err("cxl: No capp unit found for PHB[%lld,%d]. Make sure the adapter is on a capi-compatible slot\n",
  336. *chipid, *phb_index);
  337. return -ENODEV;
  338. }
  339. return 0;
  340. }
  341. static DEFINE_MUTEX(indications_mutex);
  342. static int get_phb_indications(struct pci_dev *dev, u64 *capiind, u64 *asnind,
  343. u64 *nbwind)
  344. {
  345. static u64 nbw, asn, capi = 0;
  346. struct device_node *np;
  347. const __be32 *prop;
  348. mutex_lock(&indications_mutex);
  349. if (!capi) {
  350. if (!(np = pnv_pci_get_phb_node(dev))) {
  351. mutex_unlock(&indications_mutex);
  352. return -ENODEV;
  353. }
  354. prop = of_get_property(np, "ibm,phb-indications", NULL);
  355. if (!prop) {
  356. nbw = 0x0300UL; /* legacy values */
  357. asn = 0x0400UL;
  358. capi = 0x0200UL;
  359. } else {
  360. nbw = (u64)be32_to_cpu(prop[2]);
  361. asn = (u64)be32_to_cpu(prop[1]);
  362. capi = (u64)be32_to_cpu(prop[0]);
  363. }
  364. of_node_put(np);
  365. }
  366. *capiind = capi;
  367. *asnind = asn;
  368. *nbwind = nbw;
  369. mutex_unlock(&indications_mutex);
  370. return 0;
  371. }
  372. int cxl_get_xsl9_dsnctl(struct pci_dev *dev, u64 capp_unit_id, u64 *reg)
  373. {
  374. u64 xsl_dsnctl;
  375. u64 capiind, asnind, nbwind;
  376. /*
  377. * CAPI Identifier bits [0:7]
  378. * bit 61:60 MSI bits --> 0
  379. * bit 59 TVT selector --> 0
  380. */
  381. if (get_phb_indications(dev, &capiind, &asnind, &nbwind))
  382. return -ENODEV;
  383. /*
  384. * Tell XSL where to route data to.
  385. * The field chipid should match the PHB CAPI_CMPM register
  386. */
  387. xsl_dsnctl = (capiind << (63-15)); /* Bit 57 */
  388. xsl_dsnctl |= (capp_unit_id << (63-15));
  389. /* nMMU_ID Defaults to: b’000001001’*/
  390. xsl_dsnctl |= ((u64)0x09 << (63-28));
  391. /*
  392. * Used to identify CAPI packets which should be sorted into
  393. * the Non-Blocking queues by the PHB. This field should match
  394. * the PHB PBL_NBW_CMPM register
  395. * nbwind=0x03, bits [57:58], must include capi indicator.
  396. * Not supported on P9 DD1.
  397. */
  398. xsl_dsnctl |= (nbwind << (63-55));
  399. /*
  400. * Upper 16b address bits of ASB_Notify messages sent to the
  401. * system. Need to match the PHB’s ASN Compare/Mask Register.
  402. * Not supported on P9 DD1.
  403. */
  404. xsl_dsnctl |= asnind;
  405. *reg = xsl_dsnctl;
  406. return 0;
  407. }
  408. static int init_implementation_adapter_regs_psl9(struct cxl *adapter,
  409. struct pci_dev *dev)
  410. {
  411. u64 xsl_dsnctl, psl_fircntl;
  412. u64 chipid;
  413. u32 phb_index;
  414. u64 capp_unit_id;
  415. u64 psl_debug;
  416. int rc;
  417. rc = cxl_calc_capp_routing(dev, &chipid, &phb_index, &capp_unit_id);
  418. if (rc)
  419. return rc;
  420. rc = cxl_get_xsl9_dsnctl(dev, capp_unit_id, &xsl_dsnctl);
  421. if (rc)
  422. return rc;
  423. cxl_p1_write(adapter, CXL_XSL9_DSNCTL, xsl_dsnctl);
  424. /* Set fir_cntl to recommended value for production env */
  425. psl_fircntl = (0x2ULL << (63-3)); /* ce_report */
  426. psl_fircntl |= (0x1ULL << (63-6)); /* FIR_report */
  427. psl_fircntl |= 0x1ULL; /* ce_thresh */
  428. cxl_p1_write(adapter, CXL_PSL9_FIR_CNTL, psl_fircntl);
  429. /* Setup the PSL to transmit packets on the PCIe before the
  430. * CAPP is enabled. Make sure that CAPP virtual machines are disabled
  431. */
  432. cxl_p1_write(adapter, CXL_PSL9_DSNDCTL, 0x0001001000012A10ULL);
  433. /*
  434. * A response to an ASB_Notify request is returned by the
  435. * system as an MMIO write to the address defined in
  436. * the PSL_TNR_ADDR register.
  437. * keep the Reset Value: 0x00020000E0000000
  438. */
  439. /* Enable XSL rty limit */
  440. cxl_p1_write(adapter, CXL_XSL9_DEF, 0x51F8000000000005ULL);
  441. /* Change XSL_INV dummy read threshold */
  442. cxl_p1_write(adapter, CXL_XSL9_INV, 0x0000040007FFC200ULL);
  443. if (phb_index == 3) {
  444. /* disable machines 31-47 and 20-27 for DMA */
  445. cxl_p1_write(adapter, CXL_PSL9_APCDEDTYPE, 0x40000FF3FFFF0000ULL);
  446. }
  447. /* Snoop machines */
  448. cxl_p1_write(adapter, CXL_PSL9_APCDEDALLOC, 0x800F000200000000ULL);
  449. /* Enable NORST and DD2 features */
  450. cxl_p1_write(adapter, CXL_PSL9_DEBUG, 0xC000000000000000ULL);
  451. /*
  452. * Check if PSL has data-cache. We need to flush adapter datacache
  453. * when as its about to be removed.
  454. */
  455. psl_debug = cxl_p1_read(adapter, CXL_PSL9_DEBUG);
  456. if (psl_debug & CXL_PSL_DEBUG_CDC) {
  457. dev_dbg(&dev->dev, "No data-cache present\n");
  458. adapter->native->no_data_cache = true;
  459. }
  460. return 0;
  461. }
  462. static int init_implementation_adapter_regs_psl8(struct cxl *adapter, struct pci_dev *dev)
  463. {
  464. u64 psl_dsnctl, psl_fircntl;
  465. u64 chipid;
  466. u32 phb_index;
  467. u64 capp_unit_id;
  468. int rc;
  469. rc = cxl_calc_capp_routing(dev, &chipid, &phb_index, &capp_unit_id);
  470. if (rc)
  471. return rc;
  472. psl_dsnctl = 0x0000900000000000ULL; /* pteupd ttype, scdone */
  473. psl_dsnctl |= (0x2ULL << (63-38)); /* MMIO hang pulse: 256 us */
  474. /* Tell PSL where to route data to */
  475. psl_dsnctl |= (chipid << (63-5));
  476. psl_dsnctl |= (capp_unit_id << (63-13));
  477. cxl_p1_write(adapter, CXL_PSL_DSNDCTL, psl_dsnctl);
  478. cxl_p1_write(adapter, CXL_PSL_RESLCKTO, 0x20000000200ULL);
  479. /* snoop write mask */
  480. cxl_p1_write(adapter, CXL_PSL_SNWRALLOC, 0x00000000FFFFFFFFULL);
  481. /* set fir_cntl to recommended value for production env */
  482. psl_fircntl = (0x2ULL << (63-3)); /* ce_report */
  483. psl_fircntl |= (0x1ULL << (63-6)); /* FIR_report */
  484. psl_fircntl |= 0x1ULL; /* ce_thresh */
  485. cxl_p1_write(adapter, CXL_PSL_FIR_CNTL, psl_fircntl);
  486. /* for debugging with trace arrays */
  487. cxl_p1_write(adapter, CXL_PSL_TRACE, 0x0000FF7C00000000ULL);
  488. return 0;
  489. }
  490. /* PSL */
  491. #define TBSYNC_CAL(n) (((u64)n & 0x7) << (63-3))
  492. #define TBSYNC_CNT(n) (((u64)n & 0x7) << (63-6))
  493. /* For the PSL this is a multiple for 0 < n <= 7: */
  494. #define PSL_2048_250MHZ_CYCLES 1
  495. static void write_timebase_ctrl_psl8(struct cxl *adapter)
  496. {
  497. cxl_p1_write(adapter, CXL_PSL_TB_CTLSTAT,
  498. TBSYNC_CNT(2 * PSL_2048_250MHZ_CYCLES));
  499. }
  500. static u64 timebase_read_psl9(struct cxl *adapter)
  501. {
  502. return cxl_p1_read(adapter, CXL_PSL9_Timebase);
  503. }
  504. static u64 timebase_read_psl8(struct cxl *adapter)
  505. {
  506. return cxl_p1_read(adapter, CXL_PSL_Timebase);
  507. }
  508. static void cxl_setup_psl_timebase(struct cxl *adapter, struct pci_dev *dev)
  509. {
  510. struct device_node *np;
  511. adapter->psl_timebase_synced = false;
  512. if (!(np = pnv_pci_get_phb_node(dev)))
  513. return;
  514. /* Do not fail when CAPP timebase sync is not supported by OPAL */
  515. of_node_get(np);
  516. if (! of_get_property(np, "ibm,capp-timebase-sync", NULL)) {
  517. of_node_put(np);
  518. dev_info(&dev->dev, "PSL timebase inactive: OPAL support missing\n");
  519. return;
  520. }
  521. of_node_put(np);
  522. /*
  523. * Setup PSL Timebase Control and Status register
  524. * with the recommended Timebase Sync Count value
  525. */
  526. if (adapter->native->sl_ops->write_timebase_ctrl)
  527. adapter->native->sl_ops->write_timebase_ctrl(adapter);
  528. /* Enable PSL Timebase */
  529. cxl_p1_write(adapter, CXL_PSL_Control, 0x0000000000000000);
  530. cxl_p1_write(adapter, CXL_PSL_Control, CXL_PSL_Control_tb);
  531. return;
  532. }
  533. static int init_implementation_afu_regs_psl9(struct cxl_afu *afu)
  534. {
  535. return 0;
  536. }
  537. static int init_implementation_afu_regs_psl8(struct cxl_afu *afu)
  538. {
  539. /* read/write masks for this slice */
  540. cxl_p1n_write(afu, CXL_PSL_APCALLOC_A, 0xFFFFFFFEFEFEFEFEULL);
  541. /* APC read/write masks for this slice */
  542. cxl_p1n_write(afu, CXL_PSL_COALLOC_A, 0xFF000000FEFEFEFEULL);
  543. /* for debugging with trace arrays */
  544. cxl_p1n_write(afu, CXL_PSL_SLICE_TRACE, 0x0000FFFF00000000ULL);
  545. cxl_p1n_write(afu, CXL_PSL_RXCTL_A, CXL_PSL_RXCTL_AFUHP_4S);
  546. return 0;
  547. }
  548. int cxl_pci_setup_irq(struct cxl *adapter, unsigned int hwirq,
  549. unsigned int virq)
  550. {
  551. struct pci_dev *dev = to_pci_dev(adapter->dev.parent);
  552. return pnv_cxl_ioda_msi_setup(dev, hwirq, virq);
  553. }
  554. int cxl_update_image_control(struct cxl *adapter)
  555. {
  556. struct pci_dev *dev = to_pci_dev(adapter->dev.parent);
  557. int rc;
  558. int vsec;
  559. u8 image_state;
  560. if (!(vsec = find_cxl_vsec(dev))) {
  561. dev_err(&dev->dev, "ABORTING: CXL VSEC not found!\n");
  562. return -ENODEV;
  563. }
  564. if ((rc = CXL_READ_VSEC_IMAGE_STATE(dev, vsec, &image_state))) {
  565. dev_err(&dev->dev, "failed to read image state: %i\n", rc);
  566. return rc;
  567. }
  568. if (adapter->perst_loads_image)
  569. image_state |= CXL_VSEC_PERST_LOADS_IMAGE;
  570. else
  571. image_state &= ~CXL_VSEC_PERST_LOADS_IMAGE;
  572. if (adapter->perst_select_user)
  573. image_state |= CXL_VSEC_PERST_SELECT_USER;
  574. else
  575. image_state &= ~CXL_VSEC_PERST_SELECT_USER;
  576. if ((rc = CXL_WRITE_VSEC_IMAGE_STATE(dev, vsec, image_state))) {
  577. dev_err(&dev->dev, "failed to update image control: %i\n", rc);
  578. return rc;
  579. }
  580. return 0;
  581. }
  582. int cxl_pci_alloc_one_irq(struct cxl *adapter)
  583. {
  584. struct pci_dev *dev = to_pci_dev(adapter->dev.parent);
  585. return pnv_cxl_alloc_hwirqs(dev, 1);
  586. }
  587. void cxl_pci_release_one_irq(struct cxl *adapter, int hwirq)
  588. {
  589. struct pci_dev *dev = to_pci_dev(adapter->dev.parent);
  590. return pnv_cxl_release_hwirqs(dev, hwirq, 1);
  591. }
  592. int cxl_pci_alloc_irq_ranges(struct cxl_irq_ranges *irqs,
  593. struct cxl *adapter, unsigned int num)
  594. {
  595. struct pci_dev *dev = to_pci_dev(adapter->dev.parent);
  596. return pnv_cxl_alloc_hwirq_ranges(irqs, dev, num);
  597. }
  598. void cxl_pci_release_irq_ranges(struct cxl_irq_ranges *irqs,
  599. struct cxl *adapter)
  600. {
  601. struct pci_dev *dev = to_pci_dev(adapter->dev.parent);
  602. pnv_cxl_release_hwirq_ranges(irqs, dev);
  603. }
  604. static int setup_cxl_bars(struct pci_dev *dev)
  605. {
  606. /* Safety check in case we get backported to < 3.17 without M64 */
  607. if ((p1_base(dev) < 0x100000000ULL) ||
  608. (p2_base(dev) < 0x100000000ULL)) {
  609. dev_err(&dev->dev, "ABORTING: M32 BAR assignment incompatible with CXL\n");
  610. return -ENODEV;
  611. }
  612. /*
  613. * BAR 4/5 has a special meaning for CXL and must be programmed with a
  614. * special value corresponding to the CXL protocol address range.
  615. * For POWER 8/9 that means bits 48:49 must be set to 10
  616. */
  617. pci_write_config_dword(dev, PCI_BASE_ADDRESS_4, 0x00000000);
  618. pci_write_config_dword(dev, PCI_BASE_ADDRESS_5, 0x00020000);
  619. return 0;
  620. }
  621. /* pciex node: ibm,opal-m64-window = <0x3d058 0x0 0x3d058 0x0 0x8 0x0>; */
  622. static int switch_card_to_cxl(struct pci_dev *dev)
  623. {
  624. int vsec;
  625. u8 val;
  626. int rc;
  627. dev_info(&dev->dev, "switch card to CXL\n");
  628. if (!(vsec = find_cxl_vsec(dev))) {
  629. dev_err(&dev->dev, "ABORTING: CXL VSEC not found!\n");
  630. return -ENODEV;
  631. }
  632. if ((rc = CXL_READ_VSEC_MODE_CONTROL(dev, vsec, &val))) {
  633. dev_err(&dev->dev, "failed to read current mode control: %i", rc);
  634. return rc;
  635. }
  636. val &= ~CXL_VSEC_PROTOCOL_MASK;
  637. val |= CXL_VSEC_PROTOCOL_256TB | CXL_VSEC_PROTOCOL_ENABLE;
  638. if ((rc = CXL_WRITE_VSEC_MODE_CONTROL(dev, vsec, val))) {
  639. dev_err(&dev->dev, "failed to enable CXL protocol: %i", rc);
  640. return rc;
  641. }
  642. /*
  643. * The CAIA spec (v0.12 11.6 Bi-modal Device Support) states
  644. * we must wait 100ms after this mode switch before touching
  645. * PCIe config space.
  646. */
  647. msleep(100);
  648. return 0;
  649. }
  650. static int pci_map_slice_regs(struct cxl_afu *afu, struct cxl *adapter, struct pci_dev *dev)
  651. {
  652. u64 p1n_base, p2n_base, afu_desc;
  653. const u64 p1n_size = 0x100;
  654. const u64 p2n_size = 0x1000;
  655. p1n_base = p1_base(dev) + 0x10000 + (afu->slice * p1n_size);
  656. p2n_base = p2_base(dev) + (afu->slice * p2n_size);
  657. afu->psn_phys = p2_base(dev) + (adapter->native->ps_off + (afu->slice * adapter->ps_size));
  658. afu_desc = p2_base(dev) + adapter->native->afu_desc_off + (afu->slice * adapter->native->afu_desc_size);
  659. if (!(afu->native->p1n_mmio = ioremap(p1n_base, p1n_size)))
  660. goto err;
  661. if (!(afu->p2n_mmio = ioremap(p2n_base, p2n_size)))
  662. goto err1;
  663. if (afu_desc) {
  664. if (!(afu->native->afu_desc_mmio = ioremap(afu_desc, adapter->native->afu_desc_size)))
  665. goto err2;
  666. }
  667. return 0;
  668. err2:
  669. iounmap(afu->p2n_mmio);
  670. err1:
  671. iounmap(afu->native->p1n_mmio);
  672. err:
  673. dev_err(&afu->dev, "Error mapping AFU MMIO regions\n");
  674. return -ENOMEM;
  675. }
  676. static void pci_unmap_slice_regs(struct cxl_afu *afu)
  677. {
  678. if (afu->p2n_mmio) {
  679. iounmap(afu->p2n_mmio);
  680. afu->p2n_mmio = NULL;
  681. }
  682. if (afu->native->p1n_mmio) {
  683. iounmap(afu->native->p1n_mmio);
  684. afu->native->p1n_mmio = NULL;
  685. }
  686. if (afu->native->afu_desc_mmio) {
  687. iounmap(afu->native->afu_desc_mmio);
  688. afu->native->afu_desc_mmio = NULL;
  689. }
  690. }
  691. void cxl_pci_release_afu(struct device *dev)
  692. {
  693. struct cxl_afu *afu = to_cxl_afu(dev);
  694. pr_devel("%s\n", __func__);
  695. idr_destroy(&afu->contexts_idr);
  696. cxl_release_spa(afu);
  697. kfree(afu->native);
  698. kfree(afu);
  699. }
  700. /* Expects AFU struct to have recently been zeroed out */
  701. static int cxl_read_afu_descriptor(struct cxl_afu *afu)
  702. {
  703. u64 val;
  704. val = AFUD_READ_INFO(afu);
  705. afu->pp_irqs = AFUD_NUM_INTS_PER_PROC(val);
  706. afu->max_procs_virtualised = AFUD_NUM_PROCS(val);
  707. afu->crs_num = AFUD_NUM_CRS(val);
  708. if (AFUD_AFU_DIRECTED(val))
  709. afu->modes_supported |= CXL_MODE_DIRECTED;
  710. if (AFUD_DEDICATED_PROCESS(val))
  711. afu->modes_supported |= CXL_MODE_DEDICATED;
  712. if (AFUD_TIME_SLICED(val))
  713. afu->modes_supported |= CXL_MODE_TIME_SLICED;
  714. val = AFUD_READ_PPPSA(afu);
  715. afu->pp_size = AFUD_PPPSA_LEN(val) * 4096;
  716. afu->psa = AFUD_PPPSA_PSA(val);
  717. if ((afu->pp_psa = AFUD_PPPSA_PP(val)))
  718. afu->native->pp_offset = AFUD_READ_PPPSA_OFF(afu);
  719. val = AFUD_READ_CR(afu);
  720. afu->crs_len = AFUD_CR_LEN(val) * 256;
  721. afu->crs_offset = AFUD_READ_CR_OFF(afu);
  722. /* eb_len is in multiple of 4K */
  723. afu->eb_len = AFUD_EB_LEN(AFUD_READ_EB(afu)) * 4096;
  724. afu->eb_offset = AFUD_READ_EB_OFF(afu);
  725. /* eb_off is 4K aligned so lower 12 bits are always zero */
  726. if (EXTRACT_PPC_BITS(afu->eb_offset, 0, 11) != 0) {
  727. dev_warn(&afu->dev,
  728. "Invalid AFU error buffer offset %Lx\n",
  729. afu->eb_offset);
  730. dev_info(&afu->dev,
  731. "Ignoring AFU error buffer in the descriptor\n");
  732. /* indicate that no afu buffer exists */
  733. afu->eb_len = 0;
  734. }
  735. return 0;
  736. }
  737. static int cxl_afu_descriptor_looks_ok(struct cxl_afu *afu)
  738. {
  739. int i, rc;
  740. u32 val;
  741. if (afu->psa && afu->adapter->ps_size <
  742. (afu->native->pp_offset + afu->pp_size*afu->max_procs_virtualised)) {
  743. dev_err(&afu->dev, "per-process PSA can't fit inside the PSA!\n");
  744. return -ENODEV;
  745. }
  746. if (afu->pp_psa && (afu->pp_size < PAGE_SIZE))
  747. dev_warn(&afu->dev, "AFU uses pp_size(%#016llx) < PAGE_SIZE per-process PSA!\n", afu->pp_size);
  748. for (i = 0; i < afu->crs_num; i++) {
  749. rc = cxl_ops->afu_cr_read32(afu, i, 0, &val);
  750. if (rc || val == 0) {
  751. dev_err(&afu->dev, "ABORTING: AFU configuration record %i is invalid\n", i);
  752. return -EINVAL;
  753. }
  754. }
  755. if ((afu->modes_supported & ~CXL_MODE_DEDICATED) && afu->max_procs_virtualised == 0) {
  756. /*
  757. * We could also check this for the dedicated process model
  758. * since the architecture indicates it should be set to 1, but
  759. * in that case we ignore the value and I'd rather not risk
  760. * breaking any existing dedicated process AFUs that left it as
  761. * 0 (not that I'm aware of any). It is clearly an error for an
  762. * AFU directed AFU to set this to 0, and would have previously
  763. * triggered a bug resulting in the maximum not being enforced
  764. * at all since idr_alloc treats 0 as no maximum.
  765. */
  766. dev_err(&afu->dev, "AFU does not support any processes\n");
  767. return -EINVAL;
  768. }
  769. return 0;
  770. }
  771. static int sanitise_afu_regs_psl9(struct cxl_afu *afu)
  772. {
  773. u64 reg;
  774. /*
  775. * Clear out any regs that contain either an IVTE or address or may be
  776. * waiting on an acknowledgment to try to be a bit safer as we bring
  777. * it online
  778. */
  779. reg = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
  780. if ((reg & CXL_AFU_Cntl_An_ES_MASK) != CXL_AFU_Cntl_An_ES_Disabled) {
  781. dev_warn(&afu->dev, "WARNING: AFU was not disabled: %#016llx\n", reg);
  782. if (cxl_ops->afu_reset(afu))
  783. return -EIO;
  784. if (cxl_afu_disable(afu))
  785. return -EIO;
  786. if (cxl_psl_purge(afu))
  787. return -EIO;
  788. }
  789. cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0x0000000000000000);
  790. cxl_p1n_write(afu, CXL_PSL_AMBAR_An, 0x0000000000000000);
  791. reg = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
  792. if (reg) {
  793. dev_warn(&afu->dev, "AFU had pending DSISR: %#016llx\n", reg);
  794. if (reg & CXL_PSL9_DSISR_An_TF)
  795. cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE);
  796. else
  797. cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A);
  798. }
  799. if (afu->adapter->native->sl_ops->register_serr_irq) {
  800. reg = cxl_p1n_read(afu, CXL_PSL_SERR_An);
  801. if (reg) {
  802. if (reg & ~0x000000007fffffff)
  803. dev_warn(&afu->dev, "AFU had pending SERR: %#016llx\n", reg);
  804. cxl_p1n_write(afu, CXL_PSL_SERR_An, reg & ~0xffff);
  805. }
  806. }
  807. reg = cxl_p2n_read(afu, CXL_PSL_ErrStat_An);
  808. if (reg) {
  809. dev_warn(&afu->dev, "AFU had pending error status: %#016llx\n", reg);
  810. cxl_p2n_write(afu, CXL_PSL_ErrStat_An, reg);
  811. }
  812. return 0;
  813. }
  814. static int sanitise_afu_regs_psl8(struct cxl_afu *afu)
  815. {
  816. u64 reg;
  817. /*
  818. * Clear out any regs that contain either an IVTE or address or may be
  819. * waiting on an acknowledgement to try to be a bit safer as we bring
  820. * it online
  821. */
  822. reg = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
  823. if ((reg & CXL_AFU_Cntl_An_ES_MASK) != CXL_AFU_Cntl_An_ES_Disabled) {
  824. dev_warn(&afu->dev, "WARNING: AFU was not disabled: %#016llx\n", reg);
  825. if (cxl_ops->afu_reset(afu))
  826. return -EIO;
  827. if (cxl_afu_disable(afu))
  828. return -EIO;
  829. if (cxl_psl_purge(afu))
  830. return -EIO;
  831. }
  832. cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0x0000000000000000);
  833. cxl_p1n_write(afu, CXL_PSL_IVTE_Limit_An, 0x0000000000000000);
  834. cxl_p1n_write(afu, CXL_PSL_IVTE_Offset_An, 0x0000000000000000);
  835. cxl_p1n_write(afu, CXL_PSL_AMBAR_An, 0x0000000000000000);
  836. cxl_p1n_write(afu, CXL_PSL_SPOffset_An, 0x0000000000000000);
  837. cxl_p1n_write(afu, CXL_HAURP_An, 0x0000000000000000);
  838. cxl_p2n_write(afu, CXL_CSRP_An, 0x0000000000000000);
  839. cxl_p2n_write(afu, CXL_AURP1_An, 0x0000000000000000);
  840. cxl_p2n_write(afu, CXL_AURP0_An, 0x0000000000000000);
  841. cxl_p2n_write(afu, CXL_SSTP1_An, 0x0000000000000000);
  842. cxl_p2n_write(afu, CXL_SSTP0_An, 0x0000000000000000);
  843. reg = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
  844. if (reg) {
  845. dev_warn(&afu->dev, "AFU had pending DSISR: %#016llx\n", reg);
  846. if (reg & CXL_PSL_DSISR_TRANS)
  847. cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE);
  848. else
  849. cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A);
  850. }
  851. if (afu->adapter->native->sl_ops->register_serr_irq) {
  852. reg = cxl_p1n_read(afu, CXL_PSL_SERR_An);
  853. if (reg) {
  854. if (reg & ~0xffff)
  855. dev_warn(&afu->dev, "AFU had pending SERR: %#016llx\n", reg);
  856. cxl_p1n_write(afu, CXL_PSL_SERR_An, reg & ~0xffff);
  857. }
  858. }
  859. reg = cxl_p2n_read(afu, CXL_PSL_ErrStat_An);
  860. if (reg) {
  861. dev_warn(&afu->dev, "AFU had pending error status: %#016llx\n", reg);
  862. cxl_p2n_write(afu, CXL_PSL_ErrStat_An, reg);
  863. }
  864. return 0;
  865. }
  866. #define ERR_BUFF_MAX_COPY_SIZE PAGE_SIZE
  867. /*
  868. * afu_eb_read:
  869. * Called from sysfs and reads the afu error info buffer. The h/w only supports
  870. * 4/8 bytes aligned access. So in case the requested offset/count arent 8 byte
  871. * aligned the function uses a bounce buffer which can be max PAGE_SIZE.
  872. */
  873. ssize_t cxl_pci_afu_read_err_buffer(struct cxl_afu *afu, char *buf,
  874. loff_t off, size_t count)
  875. {
  876. loff_t aligned_start, aligned_end;
  877. size_t aligned_length;
  878. void *tbuf;
  879. const void __iomem *ebuf = afu->native->afu_desc_mmio + afu->eb_offset;
  880. if (count == 0 || off < 0 || (size_t)off >= afu->eb_len)
  881. return 0;
  882. /* calculate aligned read window */
  883. count = min((size_t)(afu->eb_len - off), count);
  884. aligned_start = round_down(off, 8);
  885. aligned_end = round_up(off + count, 8);
  886. aligned_length = aligned_end - aligned_start;
  887. /* max we can copy in one read is PAGE_SIZE */
  888. if (aligned_length > ERR_BUFF_MAX_COPY_SIZE) {
  889. aligned_length = ERR_BUFF_MAX_COPY_SIZE;
  890. count = ERR_BUFF_MAX_COPY_SIZE - (off & 0x7);
  891. }
  892. /* use bounce buffer for copy */
  893. tbuf = (void *)__get_free_page(GFP_KERNEL);
  894. if (!tbuf)
  895. return -ENOMEM;
  896. /* perform aligned read from the mmio region */
  897. memcpy_fromio(tbuf, ebuf + aligned_start, aligned_length);
  898. memcpy(buf, tbuf + (off & 0x7), count);
  899. free_page((unsigned long)tbuf);
  900. return count;
  901. }
  902. static int pci_configure_afu(struct cxl_afu *afu, struct cxl *adapter, struct pci_dev *dev)
  903. {
  904. int rc;
  905. if ((rc = pci_map_slice_regs(afu, adapter, dev)))
  906. return rc;
  907. if (adapter->native->sl_ops->sanitise_afu_regs) {
  908. rc = adapter->native->sl_ops->sanitise_afu_regs(afu);
  909. if (rc)
  910. goto err1;
  911. }
  912. /* We need to reset the AFU before we can read the AFU descriptor */
  913. if ((rc = cxl_ops->afu_reset(afu)))
  914. goto err1;
  915. if (cxl_verbose)
  916. dump_afu_descriptor(afu);
  917. if ((rc = cxl_read_afu_descriptor(afu)))
  918. goto err1;
  919. if ((rc = cxl_afu_descriptor_looks_ok(afu)))
  920. goto err1;
  921. if (adapter->native->sl_ops->afu_regs_init)
  922. if ((rc = adapter->native->sl_ops->afu_regs_init(afu)))
  923. goto err1;
  924. if (adapter->native->sl_ops->register_serr_irq)
  925. if ((rc = adapter->native->sl_ops->register_serr_irq(afu)))
  926. goto err1;
  927. if ((rc = cxl_native_register_psl_irq(afu)))
  928. goto err2;
  929. atomic_set(&afu->configured_state, 0);
  930. return 0;
  931. err2:
  932. if (adapter->native->sl_ops->release_serr_irq)
  933. adapter->native->sl_ops->release_serr_irq(afu);
  934. err1:
  935. pci_unmap_slice_regs(afu);
  936. return rc;
  937. }
  938. static void pci_deconfigure_afu(struct cxl_afu *afu)
  939. {
  940. /*
  941. * It's okay to deconfigure when AFU is already locked, otherwise wait
  942. * until there are no readers
  943. */
  944. if (atomic_read(&afu->configured_state) != -1) {
  945. while (atomic_cmpxchg(&afu->configured_state, 0, -1) != -1)
  946. schedule();
  947. }
  948. cxl_native_release_psl_irq(afu);
  949. if (afu->adapter->native->sl_ops->release_serr_irq)
  950. afu->adapter->native->sl_ops->release_serr_irq(afu);
  951. pci_unmap_slice_regs(afu);
  952. }
  953. static int pci_init_afu(struct cxl *adapter, int slice, struct pci_dev *dev)
  954. {
  955. struct cxl_afu *afu;
  956. int rc = -ENOMEM;
  957. afu = cxl_alloc_afu(adapter, slice);
  958. if (!afu)
  959. return -ENOMEM;
  960. afu->native = kzalloc(sizeof(struct cxl_afu_native), GFP_KERNEL);
  961. if (!afu->native)
  962. goto err_free_afu;
  963. mutex_init(&afu->native->spa_mutex);
  964. rc = dev_set_name(&afu->dev, "afu%i.%i", adapter->adapter_num, slice);
  965. if (rc)
  966. goto err_free_native;
  967. rc = pci_configure_afu(afu, adapter, dev);
  968. if (rc)
  969. goto err_free_native;
  970. /* Don't care if this fails */
  971. cxl_debugfs_afu_add(afu);
  972. /*
  973. * After we call this function we must not free the afu directly, even
  974. * if it returns an error!
  975. */
  976. if ((rc = cxl_register_afu(afu)))
  977. goto err_put1;
  978. if ((rc = cxl_sysfs_afu_add(afu)))
  979. goto err_put1;
  980. adapter->afu[afu->slice] = afu;
  981. if ((rc = cxl_pci_vphb_add(afu)))
  982. dev_info(&afu->dev, "Can't register vPHB\n");
  983. return 0;
  984. err_put1:
  985. pci_deconfigure_afu(afu);
  986. cxl_debugfs_afu_remove(afu);
  987. device_unregister(&afu->dev);
  988. return rc;
  989. err_free_native:
  990. kfree(afu->native);
  991. err_free_afu:
  992. kfree(afu);
  993. return rc;
  994. }
  995. static void cxl_pci_remove_afu(struct cxl_afu *afu)
  996. {
  997. pr_devel("%s\n", __func__);
  998. if (!afu)
  999. return;
  1000. cxl_pci_vphb_remove(afu);
  1001. cxl_sysfs_afu_remove(afu);
  1002. cxl_debugfs_afu_remove(afu);
  1003. spin_lock(&afu->adapter->afu_list_lock);
  1004. afu->adapter->afu[afu->slice] = NULL;
  1005. spin_unlock(&afu->adapter->afu_list_lock);
  1006. cxl_context_detach_all(afu);
  1007. cxl_ops->afu_deactivate_mode(afu, afu->current_mode);
  1008. pci_deconfigure_afu(afu);
  1009. device_unregister(&afu->dev);
  1010. }
  1011. int cxl_pci_reset(struct cxl *adapter)
  1012. {
  1013. struct pci_dev *dev = to_pci_dev(adapter->dev.parent);
  1014. int rc;
  1015. if (adapter->perst_same_image) {
  1016. dev_warn(&dev->dev,
  1017. "cxl: refusing to reset/reflash when perst_reloads_same_image is set.\n");
  1018. return -EINVAL;
  1019. }
  1020. dev_info(&dev->dev, "CXL reset\n");
  1021. /*
  1022. * The adapter is about to be reset, so ignore errors.
  1023. */
  1024. cxl_data_cache_flush(adapter);
  1025. /* pcie_warm_reset requests a fundamental pci reset which includes a
  1026. * PERST assert/deassert. PERST triggers a loading of the image
  1027. * if "user" or "factory" is selected in sysfs */
  1028. if ((rc = pci_set_pcie_reset_state(dev, pcie_warm_reset))) {
  1029. dev_err(&dev->dev, "cxl: pcie_warm_reset failed\n");
  1030. return rc;
  1031. }
  1032. return rc;
  1033. }
  1034. static int cxl_map_adapter_regs(struct cxl *adapter, struct pci_dev *dev)
  1035. {
  1036. if (pci_request_region(dev, 2, "priv 2 regs"))
  1037. goto err1;
  1038. if (pci_request_region(dev, 0, "priv 1 regs"))
  1039. goto err2;
  1040. pr_devel("cxl_map_adapter_regs: p1: %#016llx %#llx, p2: %#016llx %#llx",
  1041. p1_base(dev), p1_size(dev), p2_base(dev), p2_size(dev));
  1042. if (!(adapter->native->p1_mmio = ioremap(p1_base(dev), p1_size(dev))))
  1043. goto err3;
  1044. if (!(adapter->native->p2_mmio = ioremap(p2_base(dev), p2_size(dev))))
  1045. goto err4;
  1046. return 0;
  1047. err4:
  1048. iounmap(adapter->native->p1_mmio);
  1049. adapter->native->p1_mmio = NULL;
  1050. err3:
  1051. pci_release_region(dev, 0);
  1052. err2:
  1053. pci_release_region(dev, 2);
  1054. err1:
  1055. return -ENOMEM;
  1056. }
  1057. static void cxl_unmap_adapter_regs(struct cxl *adapter)
  1058. {
  1059. if (adapter->native->p1_mmio) {
  1060. iounmap(adapter->native->p1_mmio);
  1061. adapter->native->p1_mmio = NULL;
  1062. pci_release_region(to_pci_dev(adapter->dev.parent), 2);
  1063. }
  1064. if (adapter->native->p2_mmio) {
  1065. iounmap(adapter->native->p2_mmio);
  1066. adapter->native->p2_mmio = NULL;
  1067. pci_release_region(to_pci_dev(adapter->dev.parent), 0);
  1068. }
  1069. }
  1070. static int cxl_read_vsec(struct cxl *adapter, struct pci_dev *dev)
  1071. {
  1072. int vsec;
  1073. u32 afu_desc_off, afu_desc_size;
  1074. u32 ps_off, ps_size;
  1075. u16 vseclen;
  1076. u8 image_state;
  1077. if (!(vsec = find_cxl_vsec(dev))) {
  1078. dev_err(&dev->dev, "ABORTING: CXL VSEC not found!\n");
  1079. return -ENODEV;
  1080. }
  1081. CXL_READ_VSEC_LENGTH(dev, vsec, &vseclen);
  1082. if (vseclen < CXL_VSEC_MIN_SIZE) {
  1083. dev_err(&dev->dev, "ABORTING: CXL VSEC too short\n");
  1084. return -EINVAL;
  1085. }
  1086. CXL_READ_VSEC_STATUS(dev, vsec, &adapter->vsec_status);
  1087. CXL_READ_VSEC_PSL_REVISION(dev, vsec, &adapter->psl_rev);
  1088. CXL_READ_VSEC_CAIA_MAJOR(dev, vsec, &adapter->caia_major);
  1089. CXL_READ_VSEC_CAIA_MINOR(dev, vsec, &adapter->caia_minor);
  1090. CXL_READ_VSEC_BASE_IMAGE(dev, vsec, &adapter->base_image);
  1091. CXL_READ_VSEC_IMAGE_STATE(dev, vsec, &image_state);
  1092. adapter->user_image_loaded = !!(image_state & CXL_VSEC_USER_IMAGE_LOADED);
  1093. adapter->perst_select_user = !!(image_state & CXL_VSEC_USER_IMAGE_LOADED);
  1094. adapter->perst_loads_image = !!(image_state & CXL_VSEC_PERST_LOADS_IMAGE);
  1095. CXL_READ_VSEC_NAFUS(dev, vsec, &adapter->slices);
  1096. CXL_READ_VSEC_AFU_DESC_OFF(dev, vsec, &afu_desc_off);
  1097. CXL_READ_VSEC_AFU_DESC_SIZE(dev, vsec, &afu_desc_size);
  1098. CXL_READ_VSEC_PS_OFF(dev, vsec, &ps_off);
  1099. CXL_READ_VSEC_PS_SIZE(dev, vsec, &ps_size);
  1100. /* Convert everything to bytes, because there is NO WAY I'd look at the
  1101. * code a month later and forget what units these are in ;-) */
  1102. adapter->native->ps_off = ps_off * 64 * 1024;
  1103. adapter->ps_size = ps_size * 64 * 1024;
  1104. adapter->native->afu_desc_off = afu_desc_off * 64 * 1024;
  1105. adapter->native->afu_desc_size = afu_desc_size * 64 * 1024;
  1106. /* Total IRQs - 1 PSL ERROR - #AFU*(1 slice error + 1 DSI) */
  1107. adapter->user_irqs = pnv_cxl_get_irq_count(dev) - 1 - 2*adapter->slices;
  1108. return 0;
  1109. }
  1110. /*
  1111. * Workaround a PCIe Host Bridge defect on some cards, that can cause
  1112. * malformed Transaction Layer Packet (TLP) errors to be erroneously
  1113. * reported. Mask this error in the Uncorrectable Error Mask Register.
  1114. *
  1115. * The upper nibble of the PSL revision is used to distinguish between
  1116. * different cards. The affected ones have it set to 0.
  1117. */
  1118. static void cxl_fixup_malformed_tlp(struct cxl *adapter, struct pci_dev *dev)
  1119. {
  1120. int aer;
  1121. u32 data;
  1122. if (adapter->psl_rev & 0xf000)
  1123. return;
  1124. if (!(aer = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR)))
  1125. return;
  1126. pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, &data);
  1127. if (data & PCI_ERR_UNC_MALF_TLP)
  1128. if (data & PCI_ERR_UNC_INTN)
  1129. return;
  1130. data |= PCI_ERR_UNC_MALF_TLP;
  1131. data |= PCI_ERR_UNC_INTN;
  1132. pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, data);
  1133. }
  1134. static bool cxl_compatible_caia_version(struct cxl *adapter)
  1135. {
  1136. if (cxl_is_power8() && (adapter->caia_major == 1))
  1137. return true;
  1138. if (cxl_is_power9() && (adapter->caia_major == 2))
  1139. return true;
  1140. return false;
  1141. }
  1142. static int cxl_vsec_looks_ok(struct cxl *adapter, struct pci_dev *dev)
  1143. {
  1144. if (adapter->vsec_status & CXL_STATUS_SECOND_PORT)
  1145. return -EBUSY;
  1146. if (adapter->vsec_status & CXL_UNSUPPORTED_FEATURES) {
  1147. dev_err(&dev->dev, "ABORTING: CXL requires unsupported features\n");
  1148. return -EINVAL;
  1149. }
  1150. if (!cxl_compatible_caia_version(adapter)) {
  1151. dev_info(&dev->dev, "Ignoring card. PSL type is not supported (caia version: %d)\n",
  1152. adapter->caia_major);
  1153. return -ENODEV;
  1154. }
  1155. if (!adapter->slices) {
  1156. /* Once we support dynamic reprogramming we can use the card if
  1157. * it supports loadable AFUs */
  1158. dev_err(&dev->dev, "ABORTING: Device has no AFUs\n");
  1159. return -EINVAL;
  1160. }
  1161. if (!adapter->native->afu_desc_off || !adapter->native->afu_desc_size) {
  1162. dev_err(&dev->dev, "ABORTING: VSEC shows no AFU descriptors\n");
  1163. return -EINVAL;
  1164. }
  1165. if (adapter->ps_size > p2_size(dev) - adapter->native->ps_off) {
  1166. dev_err(&dev->dev, "ABORTING: Problem state size larger than "
  1167. "available in BAR2: 0x%llx > 0x%llx\n",
  1168. adapter->ps_size, p2_size(dev) - adapter->native->ps_off);
  1169. return -EINVAL;
  1170. }
  1171. return 0;
  1172. }
  1173. ssize_t cxl_pci_read_adapter_vpd(struct cxl *adapter, void *buf, size_t len)
  1174. {
  1175. return pci_read_vpd(to_pci_dev(adapter->dev.parent), 0, len, buf);
  1176. }
  1177. static void cxl_release_adapter(struct device *dev)
  1178. {
  1179. struct cxl *adapter = to_cxl_adapter(dev);
  1180. pr_devel("cxl_release_adapter\n");
  1181. cxl_remove_adapter_nr(adapter);
  1182. kfree(adapter->native);
  1183. kfree(adapter);
  1184. }
  1185. #define CXL_PSL_ErrIVTE_tberror (0x1ull << (63-31))
  1186. static int sanitise_adapter_regs(struct cxl *adapter)
  1187. {
  1188. int rc = 0;
  1189. /* Clear PSL tberror bit by writing 1 to it */
  1190. cxl_p1_write(adapter, CXL_PSL_ErrIVTE, CXL_PSL_ErrIVTE_tberror);
  1191. if (adapter->native->sl_ops->invalidate_all) {
  1192. /* do not invalidate ERAT entries when not reloading on PERST */
  1193. if (cxl_is_power9() && (adapter->perst_loads_image))
  1194. return 0;
  1195. rc = adapter->native->sl_ops->invalidate_all(adapter);
  1196. }
  1197. return rc;
  1198. }
  1199. /* This should contain *only* operations that can safely be done in
  1200. * both creation and recovery.
  1201. */
  1202. static int cxl_configure_adapter(struct cxl *adapter, struct pci_dev *dev)
  1203. {
  1204. int rc;
  1205. adapter->dev.parent = &dev->dev;
  1206. adapter->dev.release = cxl_release_adapter;
  1207. pci_set_drvdata(dev, adapter);
  1208. rc = pci_enable_device(dev);
  1209. if (rc) {
  1210. dev_err(&dev->dev, "pci_enable_device failed: %i\n", rc);
  1211. return rc;
  1212. }
  1213. if ((rc = cxl_read_vsec(adapter, dev)))
  1214. return rc;
  1215. if ((rc = cxl_vsec_looks_ok(adapter, dev)))
  1216. return rc;
  1217. cxl_fixup_malformed_tlp(adapter, dev);
  1218. if ((rc = setup_cxl_bars(dev)))
  1219. return rc;
  1220. if ((rc = switch_card_to_cxl(dev)))
  1221. return rc;
  1222. if ((rc = cxl_update_image_control(adapter)))
  1223. return rc;
  1224. if ((rc = cxl_map_adapter_regs(adapter, dev)))
  1225. return rc;
  1226. if ((rc = sanitise_adapter_regs(adapter)))
  1227. goto err;
  1228. if ((rc = adapter->native->sl_ops->adapter_regs_init(adapter, dev)))
  1229. goto err;
  1230. /* Required for devices using CAPP DMA mode, harmless for others */
  1231. pci_set_master(dev);
  1232. adapter->tunneled_ops_supported = false;
  1233. if (cxl_is_power9()) {
  1234. if (pnv_pci_set_tunnel_bar(dev, 0x00020000E0000000ull, 1))
  1235. dev_info(&dev->dev, "Tunneled operations unsupported\n");
  1236. else
  1237. adapter->tunneled_ops_supported = true;
  1238. }
  1239. if ((rc = pnv_phb_to_cxl_mode(dev, adapter->native->sl_ops->capi_mode)))
  1240. goto err;
  1241. /* If recovery happened, the last step is to turn on snooping.
  1242. * In the non-recovery case this has no effect */
  1243. if ((rc = pnv_phb_to_cxl_mode(dev, OPAL_PHB_CAPI_MODE_SNOOP_ON)))
  1244. goto err;
  1245. /* Ignore error, adapter init is not dependant on timebase sync */
  1246. cxl_setup_psl_timebase(adapter, dev);
  1247. if ((rc = cxl_native_register_psl_err_irq(adapter)))
  1248. goto err;
  1249. return 0;
  1250. err:
  1251. cxl_unmap_adapter_regs(adapter);
  1252. return rc;
  1253. }
  1254. static void cxl_deconfigure_adapter(struct cxl *adapter)
  1255. {
  1256. struct pci_dev *pdev = to_pci_dev(adapter->dev.parent);
  1257. if (cxl_is_power9())
  1258. pnv_pci_set_tunnel_bar(pdev, 0x00020000E0000000ull, 0);
  1259. cxl_native_release_psl_err_irq(adapter);
  1260. cxl_unmap_adapter_regs(adapter);
  1261. pci_disable_device(pdev);
  1262. }
  1263. static void cxl_stop_trace_psl9(struct cxl *adapter)
  1264. {
  1265. int traceid;
  1266. u64 trace_state, trace_mask;
  1267. struct pci_dev *dev = to_pci_dev(adapter->dev.parent);
  1268. /* read each tracearray state and issue mmio to stop them is needed */
  1269. for (traceid = 0; traceid <= CXL_PSL9_TRACEID_MAX; ++traceid) {
  1270. trace_state = cxl_p1_read(adapter, CXL_PSL9_CTCCFG);
  1271. trace_mask = (0x3ULL << (62 - traceid * 2));
  1272. trace_state = (trace_state & trace_mask) >> (62 - traceid * 2);
  1273. dev_dbg(&dev->dev, "cxl: Traceid-%d trace_state=0x%0llX\n",
  1274. traceid, trace_state);
  1275. /* issue mmio if the trace array isn't in FIN state */
  1276. if (trace_state != CXL_PSL9_TRACESTATE_FIN)
  1277. cxl_p1_write(adapter, CXL_PSL9_TRACECFG,
  1278. 0x8400000000000000ULL | traceid);
  1279. }
  1280. }
  1281. static void cxl_stop_trace_psl8(struct cxl *adapter)
  1282. {
  1283. int slice;
  1284. /* Stop the trace */
  1285. cxl_p1_write(adapter, CXL_PSL_TRACE, 0x8000000000000017LL);
  1286. /* Stop the slice traces */
  1287. spin_lock(&adapter->afu_list_lock);
  1288. for (slice = 0; slice < adapter->slices; slice++) {
  1289. if (adapter->afu[slice])
  1290. cxl_p1n_write(adapter->afu[slice], CXL_PSL_SLICE_TRACE,
  1291. 0x8000000000000000LL);
  1292. }
  1293. spin_unlock(&adapter->afu_list_lock);
  1294. }
  1295. static const struct cxl_service_layer_ops psl9_ops = {
  1296. .adapter_regs_init = init_implementation_adapter_regs_psl9,
  1297. .invalidate_all = cxl_invalidate_all_psl9,
  1298. .afu_regs_init = init_implementation_afu_regs_psl9,
  1299. .sanitise_afu_regs = sanitise_afu_regs_psl9,
  1300. .register_serr_irq = cxl_native_register_serr_irq,
  1301. .release_serr_irq = cxl_native_release_serr_irq,
  1302. .handle_interrupt = cxl_irq_psl9,
  1303. .fail_irq = cxl_fail_irq_psl,
  1304. .activate_dedicated_process = cxl_activate_dedicated_process_psl9,
  1305. .attach_afu_directed = cxl_attach_afu_directed_psl9,
  1306. .attach_dedicated_process = cxl_attach_dedicated_process_psl9,
  1307. .update_dedicated_ivtes = cxl_update_dedicated_ivtes_psl9,
  1308. .debugfs_add_adapter_regs = cxl_debugfs_add_adapter_regs_psl9,
  1309. .debugfs_add_afu_regs = cxl_debugfs_add_afu_regs_psl9,
  1310. .psl_irq_dump_registers = cxl_native_irq_dump_regs_psl9,
  1311. .err_irq_dump_registers = cxl_native_err_irq_dump_regs_psl9,
  1312. .debugfs_stop_trace = cxl_stop_trace_psl9,
  1313. .timebase_read = timebase_read_psl9,
  1314. .capi_mode = OPAL_PHB_CAPI_MODE_CAPI,
  1315. .needs_reset_before_disable = true,
  1316. };
  1317. static const struct cxl_service_layer_ops psl8_ops = {
  1318. .adapter_regs_init = init_implementation_adapter_regs_psl8,
  1319. .invalidate_all = cxl_invalidate_all_psl8,
  1320. .afu_regs_init = init_implementation_afu_regs_psl8,
  1321. .sanitise_afu_regs = sanitise_afu_regs_psl8,
  1322. .register_serr_irq = cxl_native_register_serr_irq,
  1323. .release_serr_irq = cxl_native_release_serr_irq,
  1324. .handle_interrupt = cxl_irq_psl8,
  1325. .fail_irq = cxl_fail_irq_psl,
  1326. .activate_dedicated_process = cxl_activate_dedicated_process_psl8,
  1327. .attach_afu_directed = cxl_attach_afu_directed_psl8,
  1328. .attach_dedicated_process = cxl_attach_dedicated_process_psl8,
  1329. .update_dedicated_ivtes = cxl_update_dedicated_ivtes_psl8,
  1330. .debugfs_add_adapter_regs = cxl_debugfs_add_adapter_regs_psl8,
  1331. .debugfs_add_afu_regs = cxl_debugfs_add_afu_regs_psl8,
  1332. .psl_irq_dump_registers = cxl_native_irq_dump_regs_psl8,
  1333. .err_irq_dump_registers = cxl_native_err_irq_dump_regs_psl8,
  1334. .debugfs_stop_trace = cxl_stop_trace_psl8,
  1335. .write_timebase_ctrl = write_timebase_ctrl_psl8,
  1336. .timebase_read = timebase_read_psl8,
  1337. .capi_mode = OPAL_PHB_CAPI_MODE_CAPI,
  1338. .needs_reset_before_disable = true,
  1339. };
  1340. static void set_sl_ops(struct cxl *adapter, struct pci_dev *dev)
  1341. {
  1342. if (cxl_is_power8()) {
  1343. dev_info(&dev->dev, "Device uses a PSL8\n");
  1344. adapter->native->sl_ops = &psl8_ops;
  1345. } else {
  1346. dev_info(&dev->dev, "Device uses a PSL9\n");
  1347. adapter->native->sl_ops = &psl9_ops;
  1348. }
  1349. }
  1350. static struct cxl *cxl_pci_init_adapter(struct pci_dev *dev)
  1351. {
  1352. struct cxl *adapter;
  1353. int rc;
  1354. adapter = cxl_alloc_adapter();
  1355. if (!adapter)
  1356. return ERR_PTR(-ENOMEM);
  1357. adapter->native = kzalloc(sizeof(struct cxl_native), GFP_KERNEL);
  1358. if (!adapter->native) {
  1359. rc = -ENOMEM;
  1360. goto err_release;
  1361. }
  1362. set_sl_ops(adapter, dev);
  1363. /* Set defaults for parameters which need to persist over
  1364. * configure/reconfigure
  1365. */
  1366. adapter->perst_loads_image = true;
  1367. adapter->perst_same_image = false;
  1368. rc = cxl_configure_adapter(adapter, dev);
  1369. if (rc) {
  1370. pci_disable_device(dev);
  1371. goto err_release;
  1372. }
  1373. /* Don't care if this one fails: */
  1374. cxl_debugfs_adapter_add(adapter);
  1375. /*
  1376. * After we call this function we must not free the adapter directly,
  1377. * even if it returns an error!
  1378. */
  1379. if ((rc = cxl_register_adapter(adapter)))
  1380. goto err_put1;
  1381. if ((rc = cxl_sysfs_adapter_add(adapter)))
  1382. goto err_put1;
  1383. /* Release the context lock as adapter is configured */
  1384. cxl_adapter_context_unlock(adapter);
  1385. return adapter;
  1386. err_put1:
  1387. /* This should mirror cxl_remove_adapter, except without the
  1388. * sysfs parts
  1389. */
  1390. cxl_debugfs_adapter_remove(adapter);
  1391. cxl_deconfigure_adapter(adapter);
  1392. device_unregister(&adapter->dev);
  1393. return ERR_PTR(rc);
  1394. err_release:
  1395. cxl_release_adapter(&adapter->dev);
  1396. return ERR_PTR(rc);
  1397. }
  1398. static void cxl_pci_remove_adapter(struct cxl *adapter)
  1399. {
  1400. pr_devel("cxl_remove_adapter\n");
  1401. cxl_sysfs_adapter_remove(adapter);
  1402. cxl_debugfs_adapter_remove(adapter);
  1403. /*
  1404. * Flush adapter datacache as its about to be removed.
  1405. */
  1406. cxl_data_cache_flush(adapter);
  1407. cxl_deconfigure_adapter(adapter);
  1408. device_unregister(&adapter->dev);
  1409. }
  1410. #define CXL_MAX_PCIEX_PARENT 2
  1411. int cxl_slot_is_switched(struct pci_dev *dev)
  1412. {
  1413. struct device_node *np;
  1414. int depth = 0;
  1415. if (!(np = pci_device_to_OF_node(dev))) {
  1416. pr_err("cxl: np = NULL\n");
  1417. return -ENODEV;
  1418. }
  1419. of_node_get(np);
  1420. while (np) {
  1421. np = of_get_next_parent(np);
  1422. if (!of_node_is_type(np, "pciex"))
  1423. break;
  1424. depth++;
  1425. }
  1426. of_node_put(np);
  1427. return (depth > CXL_MAX_PCIEX_PARENT);
  1428. }
  1429. static int cxl_probe(struct pci_dev *dev, const struct pci_device_id *id)
  1430. {
  1431. struct cxl *adapter;
  1432. int slice;
  1433. int rc;
  1434. if (cxl_pci_is_vphb_device(dev)) {
  1435. dev_dbg(&dev->dev, "cxl_init_adapter: Ignoring cxl vphb device\n");
  1436. return -ENODEV;
  1437. }
  1438. if (cxl_slot_is_switched(dev)) {
  1439. dev_info(&dev->dev, "Ignoring card on incompatible PCI slot\n");
  1440. return -ENODEV;
  1441. }
  1442. if (cxl_is_power9() && !radix_enabled()) {
  1443. dev_info(&dev->dev, "Only Radix mode supported\n");
  1444. return -ENODEV;
  1445. }
  1446. if (cxl_verbose)
  1447. dump_cxl_config_space(dev);
  1448. adapter = cxl_pci_init_adapter(dev);
  1449. if (IS_ERR(adapter)) {
  1450. dev_err(&dev->dev, "cxl_init_adapter failed: %li\n", PTR_ERR(adapter));
  1451. return PTR_ERR(adapter);
  1452. }
  1453. for (slice = 0; slice < adapter->slices; slice++) {
  1454. if ((rc = pci_init_afu(adapter, slice, dev))) {
  1455. dev_err(&dev->dev, "AFU %i failed to initialise: %i\n", slice, rc);
  1456. continue;
  1457. }
  1458. rc = cxl_afu_select_best_mode(adapter->afu[slice]);
  1459. if (rc)
  1460. dev_err(&dev->dev, "AFU %i failed to start: %i\n", slice, rc);
  1461. }
  1462. return 0;
  1463. }
  1464. static void cxl_remove(struct pci_dev *dev)
  1465. {
  1466. struct cxl *adapter = pci_get_drvdata(dev);
  1467. struct cxl_afu *afu;
  1468. int i;
  1469. /*
  1470. * Lock to prevent someone grabbing a ref through the adapter list as
  1471. * we are removing it
  1472. */
  1473. for (i = 0; i < adapter->slices; i++) {
  1474. afu = adapter->afu[i];
  1475. cxl_pci_remove_afu(afu);
  1476. }
  1477. cxl_pci_remove_adapter(adapter);
  1478. }
  1479. static pci_ers_result_t cxl_vphb_error_detected(struct cxl_afu *afu,
  1480. pci_channel_state_t state)
  1481. {
  1482. struct pci_dev *afu_dev;
  1483. pci_ers_result_t result = PCI_ERS_RESULT_NEED_RESET;
  1484. pci_ers_result_t afu_result = PCI_ERS_RESULT_NEED_RESET;
  1485. /* There should only be one entry, but go through the list
  1486. * anyway
  1487. */
  1488. if (afu == NULL || afu->phb == NULL)
  1489. return result;
  1490. list_for_each_entry(afu_dev, &afu->phb->bus->devices, bus_list) {
  1491. if (!afu_dev->driver)
  1492. continue;
  1493. afu_dev->error_state = state;
  1494. if (afu_dev->driver->err_handler)
  1495. afu_result = afu_dev->driver->err_handler->error_detected(afu_dev,
  1496. state);
  1497. /* Disconnect trumps all, NONE trumps NEED_RESET */
  1498. if (afu_result == PCI_ERS_RESULT_DISCONNECT)
  1499. result = PCI_ERS_RESULT_DISCONNECT;
  1500. else if ((afu_result == PCI_ERS_RESULT_NONE) &&
  1501. (result == PCI_ERS_RESULT_NEED_RESET))
  1502. result = PCI_ERS_RESULT_NONE;
  1503. }
  1504. return result;
  1505. }
  1506. static pci_ers_result_t cxl_pci_error_detected(struct pci_dev *pdev,
  1507. pci_channel_state_t state)
  1508. {
  1509. struct cxl *adapter = pci_get_drvdata(pdev);
  1510. struct cxl_afu *afu;
  1511. pci_ers_result_t result = PCI_ERS_RESULT_NEED_RESET;
  1512. pci_ers_result_t afu_result = PCI_ERS_RESULT_NEED_RESET;
  1513. int i;
  1514. /* At this point, we could still have an interrupt pending.
  1515. * Let's try to get them out of the way before they do
  1516. * anything we don't like.
  1517. */
  1518. schedule();
  1519. /* If we're permanently dead, give up. */
  1520. if (state == pci_channel_io_perm_failure) {
  1521. spin_lock(&adapter->afu_list_lock);
  1522. for (i = 0; i < adapter->slices; i++) {
  1523. afu = adapter->afu[i];
  1524. /*
  1525. * Tell the AFU drivers; but we don't care what they
  1526. * say, we're going away.
  1527. */
  1528. cxl_vphb_error_detected(afu, state);
  1529. }
  1530. spin_unlock(&adapter->afu_list_lock);
  1531. return PCI_ERS_RESULT_DISCONNECT;
  1532. }
  1533. /* Are we reflashing?
  1534. *
  1535. * If we reflash, we could come back as something entirely
  1536. * different, including a non-CAPI card. As such, by default
  1537. * we don't participate in the process. We'll be unbound and
  1538. * the slot re-probed. (TODO: check EEH doesn't blindly rebind
  1539. * us!)
  1540. *
  1541. * However, this isn't the entire story: for reliablity
  1542. * reasons, we usually want to reflash the FPGA on PERST in
  1543. * order to get back to a more reliable known-good state.
  1544. *
  1545. * This causes us a bit of a problem: if we reflash we can't
  1546. * trust that we'll come back the same - we could have a new
  1547. * image and been PERSTed in order to load that
  1548. * image. However, most of the time we actually *will* come
  1549. * back the same - for example a regular EEH event.
  1550. *
  1551. * Therefore, we allow the user to assert that the image is
  1552. * indeed the same and that we should continue on into EEH
  1553. * anyway.
  1554. */
  1555. if (adapter->perst_loads_image && !adapter->perst_same_image) {
  1556. /* TODO take the PHB out of CXL mode */
  1557. dev_info(&pdev->dev, "reflashing, so opting out of EEH!\n");
  1558. return PCI_ERS_RESULT_NONE;
  1559. }
  1560. /*
  1561. * At this point, we want to try to recover. We'll always
  1562. * need a complete slot reset: we don't trust any other reset.
  1563. *
  1564. * Now, we go through each AFU:
  1565. * - We send the driver, if bound, an error_detected callback.
  1566. * We expect it to clean up, but it can also tell us to give
  1567. * up and permanently detach the card. To simplify things, if
  1568. * any bound AFU driver doesn't support EEH, we give up on EEH.
  1569. *
  1570. * - We detach all contexts associated with the AFU. This
  1571. * does not free them, but puts them into a CLOSED state
  1572. * which causes any the associated files to return useful
  1573. * errors to userland. It also unmaps, but does not free,
  1574. * any IRQs.
  1575. *
  1576. * - We clean up our side: releasing and unmapping resources we hold
  1577. * so we can wire them up again when the hardware comes back up.
  1578. *
  1579. * Driver authors should note:
  1580. *
  1581. * - Any contexts you create in your kernel driver (except
  1582. * those associated with anonymous file descriptors) are
  1583. * your responsibility to free and recreate. Likewise with
  1584. * any attached resources.
  1585. *
  1586. * - We will take responsibility for re-initialising the
  1587. * device context (the one set up for you in
  1588. * cxl_pci_enable_device_hook and accessed through
  1589. * cxl_get_context). If you've attached IRQs or other
  1590. * resources to it, they remains yours to free.
  1591. *
  1592. * You can call the same functions to release resources as you
  1593. * normally would: we make sure that these functions continue
  1594. * to work when the hardware is down.
  1595. *
  1596. * Two examples:
  1597. *
  1598. * 1) If you normally free all your resources at the end of
  1599. * each request, or if you use anonymous FDs, your
  1600. * error_detected callback can simply set a flag to tell
  1601. * your driver not to start any new calls. You can then
  1602. * clear the flag in the resume callback.
  1603. *
  1604. * 2) If you normally allocate your resources on startup:
  1605. * * Set a flag in error_detected as above.
  1606. * * Let CXL detach your contexts.
  1607. * * In slot_reset, free the old resources and allocate new ones.
  1608. * * In resume, clear the flag to allow things to start.
  1609. */
  1610. /* Make sure no one else changes the afu list */
  1611. spin_lock(&adapter->afu_list_lock);
  1612. for (i = 0; i < adapter->slices; i++) {
  1613. afu = adapter->afu[i];
  1614. if (afu == NULL)
  1615. continue;
  1616. afu_result = cxl_vphb_error_detected(afu, state);
  1617. cxl_context_detach_all(afu);
  1618. cxl_ops->afu_deactivate_mode(afu, afu->current_mode);
  1619. pci_deconfigure_afu(afu);
  1620. /* Disconnect trumps all, NONE trumps NEED_RESET */
  1621. if (afu_result == PCI_ERS_RESULT_DISCONNECT)
  1622. result = PCI_ERS_RESULT_DISCONNECT;
  1623. else if ((afu_result == PCI_ERS_RESULT_NONE) &&
  1624. (result == PCI_ERS_RESULT_NEED_RESET))
  1625. result = PCI_ERS_RESULT_NONE;
  1626. }
  1627. spin_unlock(&adapter->afu_list_lock);
  1628. /* should take the context lock here */
  1629. if (cxl_adapter_context_lock(adapter) != 0)
  1630. dev_warn(&adapter->dev,
  1631. "Couldn't take context lock with %d active-contexts\n",
  1632. atomic_read(&adapter->contexts_num));
  1633. cxl_deconfigure_adapter(adapter);
  1634. return result;
  1635. }
  1636. static pci_ers_result_t cxl_pci_slot_reset(struct pci_dev *pdev)
  1637. {
  1638. struct cxl *adapter = pci_get_drvdata(pdev);
  1639. struct cxl_afu *afu;
  1640. struct cxl_context *ctx;
  1641. struct pci_dev *afu_dev;
  1642. pci_ers_result_t afu_result = PCI_ERS_RESULT_RECOVERED;
  1643. pci_ers_result_t result = PCI_ERS_RESULT_RECOVERED;
  1644. int i;
  1645. if (cxl_configure_adapter(adapter, pdev))
  1646. goto err;
  1647. /*
  1648. * Unlock context activation for the adapter. Ideally this should be
  1649. * done in cxl_pci_resume but cxlflash module tries to activate the
  1650. * master context as part of slot_reset callback.
  1651. */
  1652. cxl_adapter_context_unlock(adapter);
  1653. spin_lock(&adapter->afu_list_lock);
  1654. for (i = 0; i < adapter->slices; i++) {
  1655. afu = adapter->afu[i];
  1656. if (afu == NULL)
  1657. continue;
  1658. if (pci_configure_afu(afu, adapter, pdev))
  1659. goto err_unlock;
  1660. if (cxl_afu_select_best_mode(afu))
  1661. goto err_unlock;
  1662. if (afu->phb == NULL)
  1663. continue;
  1664. list_for_each_entry(afu_dev, &afu->phb->bus->devices, bus_list) {
  1665. /* Reset the device context.
  1666. * TODO: make this less disruptive
  1667. */
  1668. ctx = cxl_get_context(afu_dev);
  1669. if (ctx && cxl_release_context(ctx))
  1670. goto err_unlock;
  1671. ctx = cxl_dev_context_init(afu_dev);
  1672. if (IS_ERR(ctx))
  1673. goto err_unlock;
  1674. afu_dev->dev.archdata.cxl_ctx = ctx;
  1675. if (cxl_ops->afu_check_and_enable(afu))
  1676. goto err_unlock;
  1677. afu_dev->error_state = pci_channel_io_normal;
  1678. /* If there's a driver attached, allow it to
  1679. * chime in on recovery. Drivers should check
  1680. * if everything has come back OK, but
  1681. * shouldn't start new work until we call
  1682. * their resume function.
  1683. */
  1684. if (!afu_dev->driver)
  1685. continue;
  1686. if (afu_dev->driver->err_handler &&
  1687. afu_dev->driver->err_handler->slot_reset)
  1688. afu_result = afu_dev->driver->err_handler->slot_reset(afu_dev);
  1689. if (afu_result == PCI_ERS_RESULT_DISCONNECT)
  1690. result = PCI_ERS_RESULT_DISCONNECT;
  1691. }
  1692. }
  1693. spin_unlock(&adapter->afu_list_lock);
  1694. return result;
  1695. err_unlock:
  1696. spin_unlock(&adapter->afu_list_lock);
  1697. err:
  1698. /* All the bits that happen in both error_detected and cxl_remove
  1699. * should be idempotent, so we don't need to worry about leaving a mix
  1700. * of unconfigured and reconfigured resources.
  1701. */
  1702. dev_err(&pdev->dev, "EEH recovery failed. Asking to be disconnected.\n");
  1703. return PCI_ERS_RESULT_DISCONNECT;
  1704. }
  1705. static void cxl_pci_resume(struct pci_dev *pdev)
  1706. {
  1707. struct cxl *adapter = pci_get_drvdata(pdev);
  1708. struct cxl_afu *afu;
  1709. struct pci_dev *afu_dev;
  1710. int i;
  1711. /* Everything is back now. Drivers should restart work now.
  1712. * This is not the place to be checking if everything came back up
  1713. * properly, because there's no return value: do that in slot_reset.
  1714. */
  1715. spin_lock(&adapter->afu_list_lock);
  1716. for (i = 0; i < adapter->slices; i++) {
  1717. afu = adapter->afu[i];
  1718. if (afu == NULL || afu->phb == NULL)
  1719. continue;
  1720. list_for_each_entry(afu_dev, &afu->phb->bus->devices, bus_list) {
  1721. if (afu_dev->driver && afu_dev->driver->err_handler &&
  1722. afu_dev->driver->err_handler->resume)
  1723. afu_dev->driver->err_handler->resume(afu_dev);
  1724. }
  1725. }
  1726. spin_unlock(&adapter->afu_list_lock);
  1727. }
  1728. static const struct pci_error_handlers cxl_err_handler = {
  1729. .error_detected = cxl_pci_error_detected,
  1730. .slot_reset = cxl_pci_slot_reset,
  1731. .resume = cxl_pci_resume,
  1732. };
  1733. struct pci_driver cxl_pci_driver = {
  1734. .name = "cxl-pci",
  1735. .id_table = cxl_pci_tbl,
  1736. .probe = cxl_probe,
  1737. .remove = cxl_remove,
  1738. .shutdown = cxl_remove,
  1739. .err_handler = &cxl_err_handler,
  1740. };