xen-pcifront.c 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Xen PCI Frontend
  4. *
  5. * Author: Ryan Wilson <hap9@epoch.ncsc.mil>
  6. */
  7. #include <linux/module.h>
  8. #include <linux/init.h>
  9. #include <linux/mm.h>
  10. #include <xen/xenbus.h>
  11. #include <xen/events.h>
  12. #include <xen/grant_table.h>
  13. #include <xen/page.h>
  14. #include <linux/spinlock.h>
  15. #include <linux/pci.h>
  16. #include <linux/msi.h>
  17. #include <xen/interface/io/pciif.h>
  18. #include <asm/xen/pci.h>
  19. #include <linux/interrupt.h>
  20. #include <linux/atomic.h>
  21. #include <linux/workqueue.h>
  22. #include <linux/bitops.h>
  23. #include <linux/time.h>
  24. #include <linux/ktime.h>
  25. #include <linux/swiotlb.h>
  26. #include <xen/platform_pci.h>
  27. #include <asm/xen/swiotlb-xen.h>
  28. #define INVALID_GRANT_REF (0)
  29. #define INVALID_EVTCHN (-1)
  30. struct pci_bus_entry {
  31. struct list_head list;
  32. struct pci_bus *bus;
  33. };
  34. #define _PDEVB_op_active (0)
  35. #define PDEVB_op_active (1 << (_PDEVB_op_active))
  36. struct pcifront_device {
  37. struct xenbus_device *xdev;
  38. struct list_head root_buses;
  39. int evtchn;
  40. int gnt_ref;
  41. int irq;
  42. /* Lock this when doing any operations in sh_info */
  43. spinlock_t sh_info_lock;
  44. struct xen_pci_sharedinfo *sh_info;
  45. struct work_struct op_work;
  46. unsigned long flags;
  47. };
  48. struct pcifront_sd {
  49. struct pci_sysdata sd;
  50. struct pcifront_device *pdev;
  51. };
  52. static inline struct pcifront_device *
  53. pcifront_get_pdev(struct pcifront_sd *sd)
  54. {
  55. return sd->pdev;
  56. }
  57. static inline void pcifront_init_sd(struct pcifront_sd *sd,
  58. unsigned int domain, unsigned int bus,
  59. struct pcifront_device *pdev)
  60. {
  61. /* Because we do not expose that information via XenBus. */
  62. sd->sd.node = first_online_node;
  63. sd->sd.domain = domain;
  64. sd->pdev = pdev;
  65. }
  66. static DEFINE_SPINLOCK(pcifront_dev_lock);
  67. static struct pcifront_device *pcifront_dev;
  68. static int errno_to_pcibios_err(int errno)
  69. {
  70. switch (errno) {
  71. case XEN_PCI_ERR_success:
  72. return PCIBIOS_SUCCESSFUL;
  73. case XEN_PCI_ERR_dev_not_found:
  74. return PCIBIOS_DEVICE_NOT_FOUND;
  75. case XEN_PCI_ERR_invalid_offset:
  76. case XEN_PCI_ERR_op_failed:
  77. return PCIBIOS_BAD_REGISTER_NUMBER;
  78. case XEN_PCI_ERR_not_implemented:
  79. return PCIBIOS_FUNC_NOT_SUPPORTED;
  80. case XEN_PCI_ERR_access_denied:
  81. return PCIBIOS_SET_FAILED;
  82. }
  83. return errno;
  84. }
  85. static inline void schedule_pcifront_aer_op(struct pcifront_device *pdev)
  86. {
  87. if (test_bit(_XEN_PCIB_active, (unsigned long *)&pdev->sh_info->flags)
  88. && !test_and_set_bit(_PDEVB_op_active, &pdev->flags)) {
  89. dev_dbg(&pdev->xdev->dev, "schedule aer frontend job\n");
  90. schedule_work(&pdev->op_work);
  91. }
  92. }
  93. static int do_pci_op(struct pcifront_device *pdev, struct xen_pci_op *op)
  94. {
  95. int err = 0;
  96. struct xen_pci_op *active_op = &pdev->sh_info->op;
  97. unsigned long irq_flags;
  98. evtchn_port_t port = pdev->evtchn;
  99. unsigned irq = pdev->irq;
  100. s64 ns, ns_timeout;
  101. spin_lock_irqsave(&pdev->sh_info_lock, irq_flags);
  102. memcpy(active_op, op, sizeof(struct xen_pci_op));
  103. /* Go */
  104. wmb();
  105. set_bit(_XEN_PCIF_active, (unsigned long *)&pdev->sh_info->flags);
  106. notify_remote_via_evtchn(port);
  107. /*
  108. * We set a poll timeout of 3 seconds but give up on return after
  109. * 2 seconds. It is better to time out too late rather than too early
  110. * (in the latter case we end up continually re-executing poll() with a
  111. * timeout in the past). 1s difference gives plenty of slack for error.
  112. */
  113. ns_timeout = ktime_get_ns() + 2 * (s64)NSEC_PER_SEC;
  114. xen_clear_irq_pending(irq);
  115. while (test_bit(_XEN_PCIF_active,
  116. (unsigned long *)&pdev->sh_info->flags)) {
  117. xen_poll_irq_timeout(irq, jiffies + 3*HZ);
  118. xen_clear_irq_pending(irq);
  119. ns = ktime_get_ns();
  120. if (ns > ns_timeout) {
  121. dev_err(&pdev->xdev->dev,
  122. "pciback not responding!!!\n");
  123. clear_bit(_XEN_PCIF_active,
  124. (unsigned long *)&pdev->sh_info->flags);
  125. err = XEN_PCI_ERR_dev_not_found;
  126. goto out;
  127. }
  128. }
  129. /*
  130. * We might lose backend service request since we
  131. * reuse same evtchn with pci_conf backend response. So re-schedule
  132. * aer pcifront service.
  133. */
  134. if (test_bit(_XEN_PCIB_active,
  135. (unsigned long *)&pdev->sh_info->flags)) {
  136. dev_err(&pdev->xdev->dev,
  137. "schedule aer pcifront service\n");
  138. schedule_pcifront_aer_op(pdev);
  139. }
  140. memcpy(op, active_op, sizeof(struct xen_pci_op));
  141. err = op->err;
  142. out:
  143. spin_unlock_irqrestore(&pdev->sh_info_lock, irq_flags);
  144. return err;
  145. }
  146. /* Access to this function is spinlocked in drivers/pci/access.c */
  147. static int pcifront_bus_read(struct pci_bus *bus, unsigned int devfn,
  148. int where, int size, u32 *val)
  149. {
  150. int err = 0;
  151. struct xen_pci_op op = {
  152. .cmd = XEN_PCI_OP_conf_read,
  153. .domain = pci_domain_nr(bus),
  154. .bus = bus->number,
  155. .devfn = devfn,
  156. .offset = where,
  157. .size = size,
  158. };
  159. struct pcifront_sd *sd = bus->sysdata;
  160. struct pcifront_device *pdev = pcifront_get_pdev(sd);
  161. dev_dbg(&pdev->xdev->dev,
  162. "read dev=%04x:%02x:%02x.%d - offset %x size %d\n",
  163. pci_domain_nr(bus), bus->number, PCI_SLOT(devfn),
  164. PCI_FUNC(devfn), where, size);
  165. err = do_pci_op(pdev, &op);
  166. if (likely(!err)) {
  167. dev_dbg(&pdev->xdev->dev, "read got back value %x\n",
  168. op.value);
  169. *val = op.value;
  170. } else if (err == -ENODEV) {
  171. /* No device here, pretend that it just returned 0 */
  172. err = 0;
  173. *val = 0;
  174. }
  175. return errno_to_pcibios_err(err);
  176. }
  177. /* Access to this function is spinlocked in drivers/pci/access.c */
  178. static int pcifront_bus_write(struct pci_bus *bus, unsigned int devfn,
  179. int where, int size, u32 val)
  180. {
  181. struct xen_pci_op op = {
  182. .cmd = XEN_PCI_OP_conf_write,
  183. .domain = pci_domain_nr(bus),
  184. .bus = bus->number,
  185. .devfn = devfn,
  186. .offset = where,
  187. .size = size,
  188. .value = val,
  189. };
  190. struct pcifront_sd *sd = bus->sysdata;
  191. struct pcifront_device *pdev = pcifront_get_pdev(sd);
  192. dev_dbg(&pdev->xdev->dev,
  193. "write dev=%04x:%02x:%02x.%d - offset %x size %d val %x\n",
  194. pci_domain_nr(bus), bus->number,
  195. PCI_SLOT(devfn), PCI_FUNC(devfn), where, size, val);
  196. return errno_to_pcibios_err(do_pci_op(pdev, &op));
  197. }
  198. static struct pci_ops pcifront_bus_ops = {
  199. .read = pcifront_bus_read,
  200. .write = pcifront_bus_write,
  201. };
  202. #ifdef CONFIG_PCI_MSI
  203. static int pci_frontend_enable_msix(struct pci_dev *dev,
  204. int vector[], int nvec)
  205. {
  206. int err;
  207. int i;
  208. struct xen_pci_op op = {
  209. .cmd = XEN_PCI_OP_enable_msix,
  210. .domain = pci_domain_nr(dev->bus),
  211. .bus = dev->bus->number,
  212. .devfn = dev->devfn,
  213. .value = nvec,
  214. };
  215. struct pcifront_sd *sd = dev->bus->sysdata;
  216. struct pcifront_device *pdev = pcifront_get_pdev(sd);
  217. struct msi_desc *entry;
  218. if (nvec > SH_INFO_MAX_VEC) {
  219. pci_err(dev, "too many vectors (0x%x) for PCI frontend:"
  220. " Increase SH_INFO_MAX_VEC\n", nvec);
  221. return -EINVAL;
  222. }
  223. i = 0;
  224. for_each_pci_msi_entry(entry, dev) {
  225. op.msix_entries[i].entry = entry->msi_attrib.entry_nr;
  226. /* Vector is useless at this point. */
  227. op.msix_entries[i].vector = -1;
  228. i++;
  229. }
  230. err = do_pci_op(pdev, &op);
  231. if (likely(!err)) {
  232. if (likely(!op.value)) {
  233. /* we get the result */
  234. for (i = 0; i < nvec; i++) {
  235. if (op.msix_entries[i].vector <= 0) {
  236. pci_warn(dev, "MSI-X entry %d is invalid: %d!\n",
  237. i, op.msix_entries[i].vector);
  238. err = -EINVAL;
  239. vector[i] = -1;
  240. continue;
  241. }
  242. vector[i] = op.msix_entries[i].vector;
  243. }
  244. } else {
  245. pr_info("enable msix get value %x\n", op.value);
  246. err = op.value;
  247. }
  248. } else {
  249. pci_err(dev, "enable msix get err %x\n", err);
  250. }
  251. return err;
  252. }
  253. static void pci_frontend_disable_msix(struct pci_dev *dev)
  254. {
  255. int err;
  256. struct xen_pci_op op = {
  257. .cmd = XEN_PCI_OP_disable_msix,
  258. .domain = pci_domain_nr(dev->bus),
  259. .bus = dev->bus->number,
  260. .devfn = dev->devfn,
  261. };
  262. struct pcifront_sd *sd = dev->bus->sysdata;
  263. struct pcifront_device *pdev = pcifront_get_pdev(sd);
  264. err = do_pci_op(pdev, &op);
  265. /* What should do for error ? */
  266. if (err)
  267. pci_err(dev, "pci_disable_msix get err %x\n", err);
  268. }
  269. static int pci_frontend_enable_msi(struct pci_dev *dev, int vector[])
  270. {
  271. int err;
  272. struct xen_pci_op op = {
  273. .cmd = XEN_PCI_OP_enable_msi,
  274. .domain = pci_domain_nr(dev->bus),
  275. .bus = dev->bus->number,
  276. .devfn = dev->devfn,
  277. };
  278. struct pcifront_sd *sd = dev->bus->sysdata;
  279. struct pcifront_device *pdev = pcifront_get_pdev(sd);
  280. err = do_pci_op(pdev, &op);
  281. if (likely(!err)) {
  282. vector[0] = op.value;
  283. if (op.value <= 0) {
  284. pci_warn(dev, "MSI entry is invalid: %d!\n",
  285. op.value);
  286. err = -EINVAL;
  287. vector[0] = -1;
  288. }
  289. } else {
  290. pci_err(dev, "pci frontend enable msi failed for dev "
  291. "%x:%x\n", op.bus, op.devfn);
  292. err = -EINVAL;
  293. }
  294. return err;
  295. }
  296. static void pci_frontend_disable_msi(struct pci_dev *dev)
  297. {
  298. int err;
  299. struct xen_pci_op op = {
  300. .cmd = XEN_PCI_OP_disable_msi,
  301. .domain = pci_domain_nr(dev->bus),
  302. .bus = dev->bus->number,
  303. .devfn = dev->devfn,
  304. };
  305. struct pcifront_sd *sd = dev->bus->sysdata;
  306. struct pcifront_device *pdev = pcifront_get_pdev(sd);
  307. err = do_pci_op(pdev, &op);
  308. if (err == XEN_PCI_ERR_dev_not_found) {
  309. /* XXX No response from backend, what shall we do? */
  310. pr_info("get no response from backend for disable MSI\n");
  311. return;
  312. }
  313. if (err)
  314. /* how can pciback notify us fail? */
  315. pr_info("get fake response from backend\n");
  316. }
  317. static struct xen_pci_frontend_ops pci_frontend_ops = {
  318. .enable_msi = pci_frontend_enable_msi,
  319. .disable_msi = pci_frontend_disable_msi,
  320. .enable_msix = pci_frontend_enable_msix,
  321. .disable_msix = pci_frontend_disable_msix,
  322. };
  323. static void pci_frontend_registrar(int enable)
  324. {
  325. if (enable)
  326. xen_pci_frontend = &pci_frontend_ops;
  327. else
  328. xen_pci_frontend = NULL;
  329. };
  330. #else
  331. static inline void pci_frontend_registrar(int enable) { };
  332. #endif /* CONFIG_PCI_MSI */
  333. /* Claim resources for the PCI frontend as-is, backend won't allow changes */
  334. static int pcifront_claim_resource(struct pci_dev *dev, void *data)
  335. {
  336. struct pcifront_device *pdev = data;
  337. int i;
  338. struct resource *r;
  339. for (i = 0; i < PCI_NUM_RESOURCES; i++) {
  340. r = &dev->resource[i];
  341. if (!r->parent && r->start && r->flags) {
  342. dev_info(&pdev->xdev->dev, "claiming resource %s/%d\n",
  343. pci_name(dev), i);
  344. if (pci_claim_resource(dev, i)) {
  345. dev_err(&pdev->xdev->dev, "Could not claim resource %s/%d! "
  346. "Device offline. Try using e820_host=1 in the guest config.\n",
  347. pci_name(dev), i);
  348. }
  349. }
  350. }
  351. return 0;
  352. }
  353. static int pcifront_scan_bus(struct pcifront_device *pdev,
  354. unsigned int domain, unsigned int bus,
  355. struct pci_bus *b)
  356. {
  357. struct pci_dev *d;
  358. unsigned int devfn;
  359. /* Scan the bus for functions and add.
  360. * We omit handling of PCI bridge attachment because pciback prevents
  361. * bridges from being exported.
  362. */
  363. for (devfn = 0; devfn < 0x100; devfn++) {
  364. d = pci_get_slot(b, devfn);
  365. if (d) {
  366. /* Device is already known. */
  367. pci_dev_put(d);
  368. continue;
  369. }
  370. d = pci_scan_single_device(b, devfn);
  371. if (d)
  372. dev_info(&pdev->xdev->dev, "New device on "
  373. "%04x:%02x:%02x.%d found.\n", domain, bus,
  374. PCI_SLOT(devfn), PCI_FUNC(devfn));
  375. }
  376. return 0;
  377. }
  378. static int pcifront_scan_root(struct pcifront_device *pdev,
  379. unsigned int domain, unsigned int bus)
  380. {
  381. struct pci_bus *b;
  382. LIST_HEAD(resources);
  383. struct pcifront_sd *sd = NULL;
  384. struct pci_bus_entry *bus_entry = NULL;
  385. int err = 0;
  386. static struct resource busn_res = {
  387. .start = 0,
  388. .end = 255,
  389. .flags = IORESOURCE_BUS,
  390. };
  391. #ifndef CONFIG_PCI_DOMAINS
  392. if (domain != 0) {
  393. dev_err(&pdev->xdev->dev,
  394. "PCI Root in non-zero PCI Domain! domain=%d\n", domain);
  395. dev_err(&pdev->xdev->dev,
  396. "Please compile with CONFIG_PCI_DOMAINS\n");
  397. err = -EINVAL;
  398. goto err_out;
  399. }
  400. #endif
  401. dev_info(&pdev->xdev->dev, "Creating PCI Frontend Bus %04x:%02x\n",
  402. domain, bus);
  403. bus_entry = kzalloc(sizeof(*bus_entry), GFP_KERNEL);
  404. sd = kzalloc(sizeof(*sd), GFP_KERNEL);
  405. if (!bus_entry || !sd) {
  406. err = -ENOMEM;
  407. goto err_out;
  408. }
  409. pci_add_resource(&resources, &ioport_resource);
  410. pci_add_resource(&resources, &iomem_resource);
  411. pci_add_resource(&resources, &busn_res);
  412. pcifront_init_sd(sd, domain, bus, pdev);
  413. pci_lock_rescan_remove();
  414. b = pci_scan_root_bus(&pdev->xdev->dev, bus,
  415. &pcifront_bus_ops, sd, &resources);
  416. if (!b) {
  417. dev_err(&pdev->xdev->dev,
  418. "Error creating PCI Frontend Bus!\n");
  419. err = -ENOMEM;
  420. pci_unlock_rescan_remove();
  421. pci_free_resource_list(&resources);
  422. goto err_out;
  423. }
  424. bus_entry->bus = b;
  425. list_add(&bus_entry->list, &pdev->root_buses);
  426. /* pci_scan_root_bus skips devices which do not have a
  427. * devfn==0. The pcifront_scan_bus enumerates all devfn. */
  428. err = pcifront_scan_bus(pdev, domain, bus, b);
  429. /* Claim resources before going "live" with our devices */
  430. pci_walk_bus(b, pcifront_claim_resource, pdev);
  431. /* Create SysFS and notify udev of the devices. Aka: "going live" */
  432. pci_bus_add_devices(b);
  433. pci_unlock_rescan_remove();
  434. return err;
  435. err_out:
  436. kfree(bus_entry);
  437. kfree(sd);
  438. return err;
  439. }
  440. static int pcifront_rescan_root(struct pcifront_device *pdev,
  441. unsigned int domain, unsigned int bus)
  442. {
  443. int err;
  444. struct pci_bus *b;
  445. #ifndef CONFIG_PCI_DOMAINS
  446. if (domain != 0) {
  447. dev_err(&pdev->xdev->dev,
  448. "PCI Root in non-zero PCI Domain! domain=%d\n", domain);
  449. dev_err(&pdev->xdev->dev,
  450. "Please compile with CONFIG_PCI_DOMAINS\n");
  451. return -EINVAL;
  452. }
  453. #endif
  454. dev_info(&pdev->xdev->dev, "Rescanning PCI Frontend Bus %04x:%02x\n",
  455. domain, bus);
  456. b = pci_find_bus(domain, bus);
  457. if (!b)
  458. /* If the bus is unknown, create it. */
  459. return pcifront_scan_root(pdev, domain, bus);
  460. err = pcifront_scan_bus(pdev, domain, bus, b);
  461. /* Claim resources before going "live" with our devices */
  462. pci_walk_bus(b, pcifront_claim_resource, pdev);
  463. /* Create SysFS and notify udev of the devices. Aka: "going live" */
  464. pci_bus_add_devices(b);
  465. return err;
  466. }
  467. static void free_root_bus_devs(struct pci_bus *bus)
  468. {
  469. struct pci_dev *dev;
  470. while (!list_empty(&bus->devices)) {
  471. dev = container_of(bus->devices.next, struct pci_dev,
  472. bus_list);
  473. pci_dbg(dev, "removing device\n");
  474. pci_stop_and_remove_bus_device(dev);
  475. }
  476. }
  477. static void pcifront_free_roots(struct pcifront_device *pdev)
  478. {
  479. struct pci_bus_entry *bus_entry, *t;
  480. dev_dbg(&pdev->xdev->dev, "cleaning up root buses\n");
  481. pci_lock_rescan_remove();
  482. list_for_each_entry_safe(bus_entry, t, &pdev->root_buses, list) {
  483. list_del(&bus_entry->list);
  484. free_root_bus_devs(bus_entry->bus);
  485. kfree(bus_entry->bus->sysdata);
  486. device_unregister(bus_entry->bus->bridge);
  487. pci_remove_bus(bus_entry->bus);
  488. kfree(bus_entry);
  489. }
  490. pci_unlock_rescan_remove();
  491. }
  492. static pci_ers_result_t pcifront_common_process(int cmd,
  493. struct pcifront_device *pdev,
  494. pci_channel_state_t state)
  495. {
  496. pci_ers_result_t result;
  497. struct pci_driver *pdrv;
  498. int bus = pdev->sh_info->aer_op.bus;
  499. int devfn = pdev->sh_info->aer_op.devfn;
  500. int domain = pdev->sh_info->aer_op.domain;
  501. struct pci_dev *pcidev;
  502. int flag = 0;
  503. dev_dbg(&pdev->xdev->dev,
  504. "pcifront AER process: cmd %x (bus:%x, devfn%x)",
  505. cmd, bus, devfn);
  506. result = PCI_ERS_RESULT_NONE;
  507. pcidev = pci_get_domain_bus_and_slot(domain, bus, devfn);
  508. if (!pcidev || !pcidev->driver) {
  509. dev_err(&pdev->xdev->dev, "device or AER driver is NULL\n");
  510. pci_dev_put(pcidev);
  511. return result;
  512. }
  513. pdrv = pcidev->driver;
  514. if (pdrv) {
  515. if (pdrv->err_handler && pdrv->err_handler->error_detected) {
  516. pci_dbg(pcidev, "trying to call AER service\n");
  517. if (pcidev) {
  518. flag = 1;
  519. switch (cmd) {
  520. case XEN_PCI_OP_aer_detected:
  521. result = pdrv->err_handler->
  522. error_detected(pcidev, state);
  523. break;
  524. case XEN_PCI_OP_aer_mmio:
  525. result = pdrv->err_handler->
  526. mmio_enabled(pcidev);
  527. break;
  528. case XEN_PCI_OP_aer_slotreset:
  529. result = pdrv->err_handler->
  530. slot_reset(pcidev);
  531. break;
  532. case XEN_PCI_OP_aer_resume:
  533. pdrv->err_handler->resume(pcidev);
  534. break;
  535. default:
  536. dev_err(&pdev->xdev->dev,
  537. "bad request in aer recovery "
  538. "operation!\n");
  539. }
  540. }
  541. }
  542. }
  543. if (!flag)
  544. result = PCI_ERS_RESULT_NONE;
  545. return result;
  546. }
  547. static void pcifront_do_aer(struct work_struct *data)
  548. {
  549. struct pcifront_device *pdev =
  550. container_of(data, struct pcifront_device, op_work);
  551. int cmd = pdev->sh_info->aer_op.cmd;
  552. pci_channel_state_t state =
  553. (pci_channel_state_t)pdev->sh_info->aer_op.err;
  554. /*If a pci_conf op is in progress,
  555. we have to wait until it is done before service aer op*/
  556. dev_dbg(&pdev->xdev->dev,
  557. "pcifront service aer bus %x devfn %x\n",
  558. pdev->sh_info->aer_op.bus, pdev->sh_info->aer_op.devfn);
  559. pdev->sh_info->aer_op.err = pcifront_common_process(cmd, pdev, state);
  560. /* Post the operation to the guest. */
  561. wmb();
  562. clear_bit(_XEN_PCIB_active, (unsigned long *)&pdev->sh_info->flags);
  563. notify_remote_via_evtchn(pdev->evtchn);
  564. /*in case of we lost an aer request in four lines time_window*/
  565. smp_mb__before_atomic();
  566. clear_bit(_PDEVB_op_active, &pdev->flags);
  567. smp_mb__after_atomic();
  568. schedule_pcifront_aer_op(pdev);
  569. }
  570. static irqreturn_t pcifront_handler_aer(int irq, void *dev)
  571. {
  572. struct pcifront_device *pdev = dev;
  573. schedule_pcifront_aer_op(pdev);
  574. return IRQ_HANDLED;
  575. }
  576. static int pcifront_connect_and_init_dma(struct pcifront_device *pdev)
  577. {
  578. int err = 0;
  579. spin_lock(&pcifront_dev_lock);
  580. if (!pcifront_dev) {
  581. dev_info(&pdev->xdev->dev, "Installing PCI frontend\n");
  582. pcifront_dev = pdev;
  583. } else
  584. err = -EEXIST;
  585. spin_unlock(&pcifront_dev_lock);
  586. if (!err && !swiotlb_nr_tbl()) {
  587. err = pci_xen_swiotlb_init_late();
  588. if (err)
  589. dev_err(&pdev->xdev->dev, "Could not setup SWIOTLB!\n");
  590. }
  591. return err;
  592. }
  593. static void pcifront_disconnect(struct pcifront_device *pdev)
  594. {
  595. spin_lock(&pcifront_dev_lock);
  596. if (pdev == pcifront_dev) {
  597. dev_info(&pdev->xdev->dev,
  598. "Disconnecting PCI Frontend Buses\n");
  599. pcifront_dev = NULL;
  600. }
  601. spin_unlock(&pcifront_dev_lock);
  602. }
  603. static struct pcifront_device *alloc_pdev(struct xenbus_device *xdev)
  604. {
  605. struct pcifront_device *pdev;
  606. pdev = kzalloc(sizeof(struct pcifront_device), GFP_KERNEL);
  607. if (pdev == NULL)
  608. goto out;
  609. pdev->sh_info =
  610. (struct xen_pci_sharedinfo *)__get_free_page(GFP_KERNEL);
  611. if (pdev->sh_info == NULL) {
  612. kfree(pdev);
  613. pdev = NULL;
  614. goto out;
  615. }
  616. pdev->sh_info->flags = 0;
  617. /*Flag for registering PV AER handler*/
  618. set_bit(_XEN_PCIB_AERHANDLER, (void *)&pdev->sh_info->flags);
  619. dev_set_drvdata(&xdev->dev, pdev);
  620. pdev->xdev = xdev;
  621. INIT_LIST_HEAD(&pdev->root_buses);
  622. spin_lock_init(&pdev->sh_info_lock);
  623. pdev->evtchn = INVALID_EVTCHN;
  624. pdev->gnt_ref = INVALID_GRANT_REF;
  625. pdev->irq = -1;
  626. INIT_WORK(&pdev->op_work, pcifront_do_aer);
  627. dev_dbg(&xdev->dev, "Allocated pdev @ 0x%p pdev->sh_info @ 0x%p\n",
  628. pdev, pdev->sh_info);
  629. out:
  630. return pdev;
  631. }
  632. static void free_pdev(struct pcifront_device *pdev)
  633. {
  634. dev_dbg(&pdev->xdev->dev, "freeing pdev @ 0x%p\n", pdev);
  635. pcifront_free_roots(pdev);
  636. cancel_work_sync(&pdev->op_work);
  637. if (pdev->irq >= 0)
  638. unbind_from_irqhandler(pdev->irq, pdev);
  639. if (pdev->evtchn != INVALID_EVTCHN)
  640. xenbus_free_evtchn(pdev->xdev, pdev->evtchn);
  641. if (pdev->gnt_ref != INVALID_GRANT_REF)
  642. gnttab_end_foreign_access(pdev->gnt_ref, 0 /* r/w page */,
  643. (unsigned long)pdev->sh_info);
  644. else
  645. free_page((unsigned long)pdev->sh_info);
  646. dev_set_drvdata(&pdev->xdev->dev, NULL);
  647. kfree(pdev);
  648. }
  649. static int pcifront_publish_info(struct pcifront_device *pdev)
  650. {
  651. int err = 0;
  652. struct xenbus_transaction trans;
  653. grant_ref_t gref;
  654. err = xenbus_grant_ring(pdev->xdev, pdev->sh_info, 1, &gref);
  655. if (err < 0)
  656. goto out;
  657. pdev->gnt_ref = gref;
  658. err = xenbus_alloc_evtchn(pdev->xdev, &pdev->evtchn);
  659. if (err)
  660. goto out;
  661. err = bind_evtchn_to_irqhandler(pdev->evtchn, pcifront_handler_aer,
  662. 0, "pcifront", pdev);
  663. if (err < 0)
  664. return err;
  665. pdev->irq = err;
  666. do_publish:
  667. err = xenbus_transaction_start(&trans);
  668. if (err) {
  669. xenbus_dev_fatal(pdev->xdev, err,
  670. "Error writing configuration for backend "
  671. "(start transaction)");
  672. goto out;
  673. }
  674. err = xenbus_printf(trans, pdev->xdev->nodename,
  675. "pci-op-ref", "%u", pdev->gnt_ref);
  676. if (!err)
  677. err = xenbus_printf(trans, pdev->xdev->nodename,
  678. "event-channel", "%u", pdev->evtchn);
  679. if (!err)
  680. err = xenbus_printf(trans, pdev->xdev->nodename,
  681. "magic", XEN_PCI_MAGIC);
  682. if (err) {
  683. xenbus_transaction_end(trans, 1);
  684. xenbus_dev_fatal(pdev->xdev, err,
  685. "Error writing configuration for backend");
  686. goto out;
  687. } else {
  688. err = xenbus_transaction_end(trans, 0);
  689. if (err == -EAGAIN)
  690. goto do_publish;
  691. else if (err) {
  692. xenbus_dev_fatal(pdev->xdev, err,
  693. "Error completing transaction "
  694. "for backend");
  695. goto out;
  696. }
  697. }
  698. xenbus_switch_state(pdev->xdev, XenbusStateInitialised);
  699. dev_dbg(&pdev->xdev->dev, "publishing successful!\n");
  700. out:
  701. return err;
  702. }
  703. static int pcifront_try_connect(struct pcifront_device *pdev)
  704. {
  705. int err = -EFAULT;
  706. int i, num_roots, len;
  707. char str[64];
  708. unsigned int domain, bus;
  709. /* Only connect once */
  710. if (xenbus_read_driver_state(pdev->xdev->nodename) !=
  711. XenbusStateInitialised)
  712. goto out;
  713. err = pcifront_connect_and_init_dma(pdev);
  714. if (err && err != -EEXIST) {
  715. xenbus_dev_fatal(pdev->xdev, err,
  716. "Error setting up PCI Frontend");
  717. goto out;
  718. }
  719. err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend,
  720. "root_num", "%d", &num_roots);
  721. if (err == -ENOENT) {
  722. xenbus_dev_error(pdev->xdev, err,
  723. "No PCI Roots found, trying 0000:00");
  724. err = pcifront_scan_root(pdev, 0, 0);
  725. if (err) {
  726. xenbus_dev_fatal(pdev->xdev, err,
  727. "Error scanning PCI root 0000:00");
  728. goto out;
  729. }
  730. num_roots = 0;
  731. } else if (err != 1) {
  732. if (err == 0)
  733. err = -EINVAL;
  734. xenbus_dev_fatal(pdev->xdev, err,
  735. "Error reading number of PCI roots");
  736. goto out;
  737. }
  738. for (i = 0; i < num_roots; i++) {
  739. len = snprintf(str, sizeof(str), "root-%d", i);
  740. if (unlikely(len >= (sizeof(str) - 1))) {
  741. err = -ENOMEM;
  742. goto out;
  743. }
  744. err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend, str,
  745. "%x:%x", &domain, &bus);
  746. if (err != 2) {
  747. if (err >= 0)
  748. err = -EINVAL;
  749. xenbus_dev_fatal(pdev->xdev, err,
  750. "Error reading PCI root %d", i);
  751. goto out;
  752. }
  753. err = pcifront_scan_root(pdev, domain, bus);
  754. if (err) {
  755. xenbus_dev_fatal(pdev->xdev, err,
  756. "Error scanning PCI root %04x:%02x",
  757. domain, bus);
  758. goto out;
  759. }
  760. }
  761. err = xenbus_switch_state(pdev->xdev, XenbusStateConnected);
  762. out:
  763. return err;
  764. }
  765. static int pcifront_try_disconnect(struct pcifront_device *pdev)
  766. {
  767. int err = 0;
  768. enum xenbus_state prev_state;
  769. prev_state = xenbus_read_driver_state(pdev->xdev->nodename);
  770. if (prev_state >= XenbusStateClosing)
  771. goto out;
  772. if (prev_state == XenbusStateConnected) {
  773. pcifront_free_roots(pdev);
  774. pcifront_disconnect(pdev);
  775. }
  776. err = xenbus_switch_state(pdev->xdev, XenbusStateClosed);
  777. out:
  778. return err;
  779. }
  780. static int pcifront_attach_devices(struct pcifront_device *pdev)
  781. {
  782. int err = -EFAULT;
  783. int i, num_roots, len;
  784. unsigned int domain, bus;
  785. char str[64];
  786. if (xenbus_read_driver_state(pdev->xdev->nodename) !=
  787. XenbusStateReconfiguring)
  788. goto out;
  789. err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend,
  790. "root_num", "%d", &num_roots);
  791. if (err == -ENOENT) {
  792. xenbus_dev_error(pdev->xdev, err,
  793. "No PCI Roots found, trying 0000:00");
  794. err = pcifront_rescan_root(pdev, 0, 0);
  795. if (err) {
  796. xenbus_dev_fatal(pdev->xdev, err,
  797. "Error scanning PCI root 0000:00");
  798. goto out;
  799. }
  800. num_roots = 0;
  801. } else if (err != 1) {
  802. if (err == 0)
  803. err = -EINVAL;
  804. xenbus_dev_fatal(pdev->xdev, err,
  805. "Error reading number of PCI roots");
  806. goto out;
  807. }
  808. for (i = 0; i < num_roots; i++) {
  809. len = snprintf(str, sizeof(str), "root-%d", i);
  810. if (unlikely(len >= (sizeof(str) - 1))) {
  811. err = -ENOMEM;
  812. goto out;
  813. }
  814. err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend, str,
  815. "%x:%x", &domain, &bus);
  816. if (err != 2) {
  817. if (err >= 0)
  818. err = -EINVAL;
  819. xenbus_dev_fatal(pdev->xdev, err,
  820. "Error reading PCI root %d", i);
  821. goto out;
  822. }
  823. err = pcifront_rescan_root(pdev, domain, bus);
  824. if (err) {
  825. xenbus_dev_fatal(pdev->xdev, err,
  826. "Error scanning PCI root %04x:%02x",
  827. domain, bus);
  828. goto out;
  829. }
  830. }
  831. xenbus_switch_state(pdev->xdev, XenbusStateConnected);
  832. out:
  833. return err;
  834. }
  835. static int pcifront_detach_devices(struct pcifront_device *pdev)
  836. {
  837. int err = 0;
  838. int i, num_devs;
  839. unsigned int domain, bus, slot, func;
  840. struct pci_dev *pci_dev;
  841. char str[64];
  842. if (xenbus_read_driver_state(pdev->xdev->nodename) !=
  843. XenbusStateConnected)
  844. goto out;
  845. err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend, "num_devs", "%d",
  846. &num_devs);
  847. if (err != 1) {
  848. if (err >= 0)
  849. err = -EINVAL;
  850. xenbus_dev_fatal(pdev->xdev, err,
  851. "Error reading number of PCI devices");
  852. goto out;
  853. }
  854. /* Find devices being detached and remove them. */
  855. for (i = 0; i < num_devs; i++) {
  856. int l, state;
  857. l = snprintf(str, sizeof(str), "state-%d", i);
  858. if (unlikely(l >= (sizeof(str) - 1))) {
  859. err = -ENOMEM;
  860. goto out;
  861. }
  862. state = xenbus_read_unsigned(pdev->xdev->otherend, str,
  863. XenbusStateUnknown);
  864. if (state != XenbusStateClosing)
  865. continue;
  866. /* Remove device. */
  867. l = snprintf(str, sizeof(str), "vdev-%d", i);
  868. if (unlikely(l >= (sizeof(str) - 1))) {
  869. err = -ENOMEM;
  870. goto out;
  871. }
  872. err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend, str,
  873. "%x:%x:%x.%x", &domain, &bus, &slot, &func);
  874. if (err != 4) {
  875. if (err >= 0)
  876. err = -EINVAL;
  877. xenbus_dev_fatal(pdev->xdev, err,
  878. "Error reading PCI device %d", i);
  879. goto out;
  880. }
  881. pci_dev = pci_get_domain_bus_and_slot(domain, bus,
  882. PCI_DEVFN(slot, func));
  883. if (!pci_dev) {
  884. dev_dbg(&pdev->xdev->dev,
  885. "Cannot get PCI device %04x:%02x:%02x.%d\n",
  886. domain, bus, slot, func);
  887. continue;
  888. }
  889. pci_lock_rescan_remove();
  890. pci_stop_and_remove_bus_device(pci_dev);
  891. pci_dev_put(pci_dev);
  892. pci_unlock_rescan_remove();
  893. dev_dbg(&pdev->xdev->dev,
  894. "PCI device %04x:%02x:%02x.%d removed.\n",
  895. domain, bus, slot, func);
  896. }
  897. err = xenbus_switch_state(pdev->xdev, XenbusStateReconfiguring);
  898. out:
  899. return err;
  900. }
  901. static void __ref pcifront_backend_changed(struct xenbus_device *xdev,
  902. enum xenbus_state be_state)
  903. {
  904. struct pcifront_device *pdev = dev_get_drvdata(&xdev->dev);
  905. switch (be_state) {
  906. case XenbusStateUnknown:
  907. case XenbusStateInitialising:
  908. case XenbusStateInitWait:
  909. case XenbusStateInitialised:
  910. break;
  911. case XenbusStateConnected:
  912. pcifront_try_connect(pdev);
  913. break;
  914. case XenbusStateClosed:
  915. if (xdev->state == XenbusStateClosed)
  916. break;
  917. fallthrough; /* Missed the backend's CLOSING state */
  918. case XenbusStateClosing:
  919. dev_warn(&xdev->dev, "backend going away!\n");
  920. pcifront_try_disconnect(pdev);
  921. break;
  922. case XenbusStateReconfiguring:
  923. pcifront_detach_devices(pdev);
  924. break;
  925. case XenbusStateReconfigured:
  926. pcifront_attach_devices(pdev);
  927. break;
  928. }
  929. }
  930. static int pcifront_xenbus_probe(struct xenbus_device *xdev,
  931. const struct xenbus_device_id *id)
  932. {
  933. int err = 0;
  934. struct pcifront_device *pdev = alloc_pdev(xdev);
  935. if (pdev == NULL) {
  936. err = -ENOMEM;
  937. xenbus_dev_fatal(xdev, err,
  938. "Error allocating pcifront_device struct");
  939. goto out;
  940. }
  941. err = pcifront_publish_info(pdev);
  942. if (err)
  943. free_pdev(pdev);
  944. out:
  945. return err;
  946. }
  947. static int pcifront_xenbus_remove(struct xenbus_device *xdev)
  948. {
  949. struct pcifront_device *pdev = dev_get_drvdata(&xdev->dev);
  950. if (pdev)
  951. free_pdev(pdev);
  952. return 0;
  953. }
  954. static const struct xenbus_device_id xenpci_ids[] = {
  955. {"pci"},
  956. {""},
  957. };
  958. static struct xenbus_driver xenpci_driver = {
  959. .name = "pcifront",
  960. .ids = xenpci_ids,
  961. .probe = pcifront_xenbus_probe,
  962. .remove = pcifront_xenbus_remove,
  963. .otherend_changed = pcifront_backend_changed,
  964. };
  965. static int __init pcifront_init(void)
  966. {
  967. if (!xen_pv_domain() || xen_initial_domain())
  968. return -ENODEV;
  969. if (!xen_has_pv_devices())
  970. return -ENODEV;
  971. pci_frontend_registrar(1 /* enable */);
  972. return xenbus_register_frontend(&xenpci_driver);
  973. }
  974. static void __exit pcifront_cleanup(void)
  975. {
  976. xenbus_unregister_driver(&xenpci_driver);
  977. pci_frontend_registrar(0 /* disable */);
  978. }
  979. module_init(pcifront_init);
  980. module_exit(pcifront_cleanup);
  981. MODULE_DESCRIPTION("Xen PCI passthrough frontend.");
  982. MODULE_LICENSE("GPL");
  983. MODULE_ALIAS("xen:pci");