bus.c 31 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
  4. */
  5. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  6. #include <linux/libnvdimm.h>
  7. #include <linux/sched/mm.h>
  8. #include <linux/vmalloc.h>
  9. #include <linux/uaccess.h>
  10. #include <linux/module.h>
  11. #include <linux/blkdev.h>
  12. #include <linux/fcntl.h>
  13. #include <linux/async.h>
  14. #include <linux/genhd.h>
  15. #include <linux/ndctl.h>
  16. #include <linux/sched.h>
  17. #include <linux/slab.h>
  18. #include <linux/cpu.h>
  19. #include <linux/fs.h>
  20. #include <linux/io.h>
  21. #include <linux/mm.h>
  22. #include <linux/nd.h>
  23. #include "nd-core.h"
  24. #include "nd.h"
  25. #include "pfn.h"
  26. int nvdimm_major;
  27. static int nvdimm_bus_major;
  28. struct class *nd_class;
  29. static DEFINE_IDA(nd_ida);
  30. static int to_nd_device_type(struct device *dev)
  31. {
  32. if (is_nvdimm(dev))
  33. return ND_DEVICE_DIMM;
  34. else if (is_memory(dev))
  35. return ND_DEVICE_REGION_PMEM;
  36. else if (is_nd_blk(dev))
  37. return ND_DEVICE_REGION_BLK;
  38. else if (is_nd_dax(dev))
  39. return ND_DEVICE_DAX_PMEM;
  40. else if (is_nd_region(dev->parent))
  41. return nd_region_to_nstype(to_nd_region(dev->parent));
  42. return 0;
  43. }
  44. static int nvdimm_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
  45. {
  46. return add_uevent_var(env, "MODALIAS=" ND_DEVICE_MODALIAS_FMT,
  47. to_nd_device_type(dev));
  48. }
  49. static struct module *to_bus_provider(struct device *dev)
  50. {
  51. /* pin bus providers while regions are enabled */
  52. if (is_nd_region(dev)) {
  53. struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
  54. return nvdimm_bus->nd_desc->module;
  55. }
  56. return NULL;
  57. }
  58. static void nvdimm_bus_probe_start(struct nvdimm_bus *nvdimm_bus)
  59. {
  60. nvdimm_bus_lock(&nvdimm_bus->dev);
  61. nvdimm_bus->probe_active++;
  62. nvdimm_bus_unlock(&nvdimm_bus->dev);
  63. }
  64. static void nvdimm_bus_probe_end(struct nvdimm_bus *nvdimm_bus)
  65. {
  66. nvdimm_bus_lock(&nvdimm_bus->dev);
  67. if (--nvdimm_bus->probe_active == 0)
  68. wake_up(&nvdimm_bus->wait);
  69. nvdimm_bus_unlock(&nvdimm_bus->dev);
  70. }
  71. static int nvdimm_bus_probe(struct device *dev)
  72. {
  73. struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver);
  74. struct module *provider = to_bus_provider(dev);
  75. struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
  76. int rc;
  77. if (!try_module_get(provider))
  78. return -ENXIO;
  79. dev_dbg(&nvdimm_bus->dev, "START: %s.probe(%s)\n",
  80. dev->driver->name, dev_name(dev));
  81. nvdimm_bus_probe_start(nvdimm_bus);
  82. debug_nvdimm_lock(dev);
  83. rc = nd_drv->probe(dev);
  84. debug_nvdimm_unlock(dev);
  85. if ((rc == 0 || rc == -EOPNOTSUPP) &&
  86. dev->parent && is_nd_region(dev->parent))
  87. nd_region_advance_seeds(to_nd_region(dev->parent), dev);
  88. nvdimm_bus_probe_end(nvdimm_bus);
  89. dev_dbg(&nvdimm_bus->dev, "END: %s.probe(%s) = %d\n", dev->driver->name,
  90. dev_name(dev), rc);
  91. if (rc != 0)
  92. module_put(provider);
  93. return rc;
  94. }
  95. static int nvdimm_bus_remove(struct device *dev)
  96. {
  97. struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver);
  98. struct module *provider = to_bus_provider(dev);
  99. struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
  100. int rc = 0;
  101. if (nd_drv->remove) {
  102. debug_nvdimm_lock(dev);
  103. rc = nd_drv->remove(dev);
  104. debug_nvdimm_unlock(dev);
  105. }
  106. dev_dbg(&nvdimm_bus->dev, "%s.remove(%s) = %d\n", dev->driver->name,
  107. dev_name(dev), rc);
  108. module_put(provider);
  109. return rc;
  110. }
  111. static void nvdimm_bus_shutdown(struct device *dev)
  112. {
  113. struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
  114. struct nd_device_driver *nd_drv = NULL;
  115. if (dev->driver)
  116. nd_drv = to_nd_device_driver(dev->driver);
  117. if (nd_drv && nd_drv->shutdown) {
  118. nd_drv->shutdown(dev);
  119. dev_dbg(&nvdimm_bus->dev, "%s.shutdown(%s)\n",
  120. dev->driver->name, dev_name(dev));
  121. }
  122. }
  123. void nd_device_notify(struct device *dev, enum nvdimm_event event)
  124. {
  125. nd_device_lock(dev);
  126. if (dev->driver) {
  127. struct nd_device_driver *nd_drv;
  128. nd_drv = to_nd_device_driver(dev->driver);
  129. if (nd_drv->notify)
  130. nd_drv->notify(dev, event);
  131. }
  132. nd_device_unlock(dev);
  133. }
  134. EXPORT_SYMBOL(nd_device_notify);
  135. void nvdimm_region_notify(struct nd_region *nd_region, enum nvdimm_event event)
  136. {
  137. struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&nd_region->dev);
  138. if (!nvdimm_bus)
  139. return;
  140. /* caller is responsible for holding a reference on the device */
  141. nd_device_notify(&nd_region->dev, event);
  142. }
  143. EXPORT_SYMBOL_GPL(nvdimm_region_notify);
  144. struct clear_badblocks_context {
  145. resource_size_t phys, cleared;
  146. };
  147. static int nvdimm_clear_badblocks_region(struct device *dev, void *data)
  148. {
  149. struct clear_badblocks_context *ctx = data;
  150. struct nd_region *nd_region;
  151. resource_size_t ndr_end;
  152. sector_t sector;
  153. /* make sure device is a region */
  154. if (!is_memory(dev))
  155. return 0;
  156. nd_region = to_nd_region(dev);
  157. ndr_end = nd_region->ndr_start + nd_region->ndr_size - 1;
  158. /* make sure we are in the region */
  159. if (ctx->phys < nd_region->ndr_start
  160. || (ctx->phys + ctx->cleared) > ndr_end)
  161. return 0;
  162. sector = (ctx->phys - nd_region->ndr_start) / 512;
  163. badblocks_clear(&nd_region->bb, sector, ctx->cleared / 512);
  164. if (nd_region->bb_state)
  165. sysfs_notify_dirent(nd_region->bb_state);
  166. return 0;
  167. }
  168. static void nvdimm_clear_badblocks_regions(struct nvdimm_bus *nvdimm_bus,
  169. phys_addr_t phys, u64 cleared)
  170. {
  171. struct clear_badblocks_context ctx = {
  172. .phys = phys,
  173. .cleared = cleared,
  174. };
  175. device_for_each_child(&nvdimm_bus->dev, &ctx,
  176. nvdimm_clear_badblocks_region);
  177. }
  178. static void nvdimm_account_cleared_poison(struct nvdimm_bus *nvdimm_bus,
  179. phys_addr_t phys, u64 cleared)
  180. {
  181. if (cleared > 0)
  182. badrange_forget(&nvdimm_bus->badrange, phys, cleared);
  183. if (cleared > 0 && cleared / 512)
  184. nvdimm_clear_badblocks_regions(nvdimm_bus, phys, cleared);
  185. }
  186. long nvdimm_clear_poison(struct device *dev, phys_addr_t phys,
  187. unsigned int len)
  188. {
  189. struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
  190. struct nvdimm_bus_descriptor *nd_desc;
  191. struct nd_cmd_clear_error clear_err;
  192. struct nd_cmd_ars_cap ars_cap;
  193. u32 clear_err_unit, mask;
  194. unsigned int noio_flag;
  195. int cmd_rc, rc;
  196. if (!nvdimm_bus)
  197. return -ENXIO;
  198. nd_desc = nvdimm_bus->nd_desc;
  199. /*
  200. * if ndctl does not exist, it's PMEM_LEGACY and
  201. * we want to just pretend everything is handled.
  202. */
  203. if (!nd_desc->ndctl)
  204. return len;
  205. memset(&ars_cap, 0, sizeof(ars_cap));
  206. ars_cap.address = phys;
  207. ars_cap.length = len;
  208. noio_flag = memalloc_noio_save();
  209. rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, &ars_cap,
  210. sizeof(ars_cap), &cmd_rc);
  211. memalloc_noio_restore(noio_flag);
  212. if (rc < 0)
  213. return rc;
  214. if (cmd_rc < 0)
  215. return cmd_rc;
  216. clear_err_unit = ars_cap.clear_err_unit;
  217. if (!clear_err_unit || !is_power_of_2(clear_err_unit))
  218. return -ENXIO;
  219. mask = clear_err_unit - 1;
  220. if ((phys | len) & mask)
  221. return -ENXIO;
  222. memset(&clear_err, 0, sizeof(clear_err));
  223. clear_err.address = phys;
  224. clear_err.length = len;
  225. noio_flag = memalloc_noio_save();
  226. rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_CLEAR_ERROR, &clear_err,
  227. sizeof(clear_err), &cmd_rc);
  228. memalloc_noio_restore(noio_flag);
  229. if (rc < 0)
  230. return rc;
  231. if (cmd_rc < 0)
  232. return cmd_rc;
  233. nvdimm_account_cleared_poison(nvdimm_bus, phys, clear_err.cleared);
  234. return clear_err.cleared;
  235. }
  236. EXPORT_SYMBOL_GPL(nvdimm_clear_poison);
  237. static int nvdimm_bus_match(struct device *dev, struct device_driver *drv);
  238. static struct bus_type nvdimm_bus_type = {
  239. .name = "nd",
  240. .uevent = nvdimm_bus_uevent,
  241. .match = nvdimm_bus_match,
  242. .probe = nvdimm_bus_probe,
  243. .remove = nvdimm_bus_remove,
  244. .shutdown = nvdimm_bus_shutdown,
  245. };
  246. static void nvdimm_bus_release(struct device *dev)
  247. {
  248. struct nvdimm_bus *nvdimm_bus;
  249. nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
  250. ida_simple_remove(&nd_ida, nvdimm_bus->id);
  251. kfree(nvdimm_bus);
  252. }
  253. static const struct device_type nvdimm_bus_dev_type = {
  254. .release = nvdimm_bus_release,
  255. .groups = nvdimm_bus_attribute_groups,
  256. };
  257. bool is_nvdimm_bus(struct device *dev)
  258. {
  259. return dev->type == &nvdimm_bus_dev_type;
  260. }
  261. struct nvdimm_bus *walk_to_nvdimm_bus(struct device *nd_dev)
  262. {
  263. struct device *dev;
  264. for (dev = nd_dev; dev; dev = dev->parent)
  265. if (is_nvdimm_bus(dev))
  266. break;
  267. dev_WARN_ONCE(nd_dev, !dev, "invalid dev, not on nd bus\n");
  268. if (dev)
  269. return to_nvdimm_bus(dev);
  270. return NULL;
  271. }
  272. struct nvdimm_bus *to_nvdimm_bus(struct device *dev)
  273. {
  274. struct nvdimm_bus *nvdimm_bus;
  275. nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
  276. WARN_ON(!is_nvdimm_bus(dev));
  277. return nvdimm_bus;
  278. }
  279. EXPORT_SYMBOL_GPL(to_nvdimm_bus);
  280. struct nvdimm_bus *nvdimm_to_bus(struct nvdimm *nvdimm)
  281. {
  282. return to_nvdimm_bus(nvdimm->dev.parent);
  283. }
  284. EXPORT_SYMBOL_GPL(nvdimm_to_bus);
  285. struct nvdimm_bus *nvdimm_bus_register(struct device *parent,
  286. struct nvdimm_bus_descriptor *nd_desc)
  287. {
  288. struct nvdimm_bus *nvdimm_bus;
  289. int rc;
  290. nvdimm_bus = kzalloc(sizeof(*nvdimm_bus), GFP_KERNEL);
  291. if (!nvdimm_bus)
  292. return NULL;
  293. INIT_LIST_HEAD(&nvdimm_bus->list);
  294. INIT_LIST_HEAD(&nvdimm_bus->mapping_list);
  295. init_waitqueue_head(&nvdimm_bus->wait);
  296. nvdimm_bus->id = ida_simple_get(&nd_ida, 0, 0, GFP_KERNEL);
  297. if (nvdimm_bus->id < 0) {
  298. kfree(nvdimm_bus);
  299. return NULL;
  300. }
  301. mutex_init(&nvdimm_bus->reconfig_mutex);
  302. badrange_init(&nvdimm_bus->badrange);
  303. nvdimm_bus->nd_desc = nd_desc;
  304. nvdimm_bus->dev.parent = parent;
  305. nvdimm_bus->dev.type = &nvdimm_bus_dev_type;
  306. nvdimm_bus->dev.groups = nd_desc->attr_groups;
  307. nvdimm_bus->dev.bus = &nvdimm_bus_type;
  308. nvdimm_bus->dev.of_node = nd_desc->of_node;
  309. dev_set_name(&nvdimm_bus->dev, "ndbus%d", nvdimm_bus->id);
  310. rc = device_register(&nvdimm_bus->dev);
  311. if (rc) {
  312. dev_dbg(&nvdimm_bus->dev, "registration failed: %d\n", rc);
  313. goto err;
  314. }
  315. return nvdimm_bus;
  316. err:
  317. put_device(&nvdimm_bus->dev);
  318. return NULL;
  319. }
  320. EXPORT_SYMBOL_GPL(nvdimm_bus_register);
  321. void nvdimm_bus_unregister(struct nvdimm_bus *nvdimm_bus)
  322. {
  323. if (!nvdimm_bus)
  324. return;
  325. device_unregister(&nvdimm_bus->dev);
  326. }
  327. EXPORT_SYMBOL_GPL(nvdimm_bus_unregister);
  328. static int child_unregister(struct device *dev, void *data)
  329. {
  330. /*
  331. * the singular ndctl class device per bus needs to be
  332. * "device_destroy"ed, so skip it here
  333. *
  334. * i.e. remove classless children
  335. */
  336. if (dev->class)
  337. return 0;
  338. if (is_nvdimm(dev)) {
  339. struct nvdimm *nvdimm = to_nvdimm(dev);
  340. bool dev_put = false;
  341. /* We are shutting down. Make state frozen artificially. */
  342. nvdimm_bus_lock(dev);
  343. set_bit(NVDIMM_SECURITY_FROZEN, &nvdimm->sec.flags);
  344. if (test_and_clear_bit(NDD_WORK_PENDING, &nvdimm->flags))
  345. dev_put = true;
  346. nvdimm_bus_unlock(dev);
  347. cancel_delayed_work_sync(&nvdimm->dwork);
  348. if (dev_put)
  349. put_device(dev);
  350. }
  351. nd_device_unregister(dev, ND_SYNC);
  352. return 0;
  353. }
  354. static void free_badrange_list(struct list_head *badrange_list)
  355. {
  356. struct badrange_entry *bre, *next;
  357. list_for_each_entry_safe(bre, next, badrange_list, list) {
  358. list_del(&bre->list);
  359. kfree(bre);
  360. }
  361. list_del_init(badrange_list);
  362. }
  363. static int nd_bus_remove(struct device *dev)
  364. {
  365. struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
  366. mutex_lock(&nvdimm_bus_list_mutex);
  367. list_del_init(&nvdimm_bus->list);
  368. mutex_unlock(&nvdimm_bus_list_mutex);
  369. wait_event(nvdimm_bus->wait,
  370. atomic_read(&nvdimm_bus->ioctl_active) == 0);
  371. nd_synchronize();
  372. device_for_each_child(&nvdimm_bus->dev, NULL, child_unregister);
  373. spin_lock(&nvdimm_bus->badrange.lock);
  374. free_badrange_list(&nvdimm_bus->badrange.list);
  375. spin_unlock(&nvdimm_bus->badrange.lock);
  376. nvdimm_bus_destroy_ndctl(nvdimm_bus);
  377. return 0;
  378. }
  379. static int nd_bus_probe(struct device *dev)
  380. {
  381. struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
  382. int rc;
  383. rc = nvdimm_bus_create_ndctl(nvdimm_bus);
  384. if (rc)
  385. return rc;
  386. mutex_lock(&nvdimm_bus_list_mutex);
  387. list_add_tail(&nvdimm_bus->list, &nvdimm_bus_list);
  388. mutex_unlock(&nvdimm_bus_list_mutex);
  389. /* enable bus provider attributes to look up their local context */
  390. dev_set_drvdata(dev, nvdimm_bus->nd_desc);
  391. return 0;
  392. }
  393. static struct nd_device_driver nd_bus_driver = {
  394. .probe = nd_bus_probe,
  395. .remove = nd_bus_remove,
  396. .drv = {
  397. .name = "nd_bus",
  398. .suppress_bind_attrs = true,
  399. .bus = &nvdimm_bus_type,
  400. .owner = THIS_MODULE,
  401. .mod_name = KBUILD_MODNAME,
  402. },
  403. };
  404. static int nvdimm_bus_match(struct device *dev, struct device_driver *drv)
  405. {
  406. struct nd_device_driver *nd_drv = to_nd_device_driver(drv);
  407. if (is_nvdimm_bus(dev) && nd_drv == &nd_bus_driver)
  408. return true;
  409. return !!test_bit(to_nd_device_type(dev), &nd_drv->type);
  410. }
  411. static ASYNC_DOMAIN_EXCLUSIVE(nd_async_domain);
  412. void nd_synchronize(void)
  413. {
  414. async_synchronize_full_domain(&nd_async_domain);
  415. }
  416. EXPORT_SYMBOL_GPL(nd_synchronize);
  417. static void nd_async_device_register(void *d, async_cookie_t cookie)
  418. {
  419. struct device *dev = d;
  420. if (device_add(dev) != 0) {
  421. dev_err(dev, "%s: failed\n", __func__);
  422. put_device(dev);
  423. }
  424. put_device(dev);
  425. if (dev->parent)
  426. put_device(dev->parent);
  427. }
  428. static void nd_async_device_unregister(void *d, async_cookie_t cookie)
  429. {
  430. struct device *dev = d;
  431. /* flush bus operations before delete */
  432. nvdimm_bus_lock(dev);
  433. nvdimm_bus_unlock(dev);
  434. device_unregister(dev);
  435. put_device(dev);
  436. }
  437. void __nd_device_register(struct device *dev)
  438. {
  439. if (!dev)
  440. return;
  441. /*
  442. * Ensure that region devices always have their NUMA node set as
  443. * early as possible. This way we are able to make certain that
  444. * any memory associated with the creation and the creation
  445. * itself of the region is associated with the correct node.
  446. */
  447. if (is_nd_region(dev))
  448. set_dev_node(dev, to_nd_region(dev)->numa_node);
  449. dev->bus = &nvdimm_bus_type;
  450. if (dev->parent) {
  451. get_device(dev->parent);
  452. if (dev_to_node(dev) == NUMA_NO_NODE)
  453. set_dev_node(dev, dev_to_node(dev->parent));
  454. }
  455. get_device(dev);
  456. async_schedule_dev_domain(nd_async_device_register, dev,
  457. &nd_async_domain);
  458. }
  459. void nd_device_register(struct device *dev)
  460. {
  461. device_initialize(dev);
  462. __nd_device_register(dev);
  463. }
  464. EXPORT_SYMBOL(nd_device_register);
  465. void nd_device_unregister(struct device *dev, enum nd_async_mode mode)
  466. {
  467. bool killed;
  468. switch (mode) {
  469. case ND_ASYNC:
  470. /*
  471. * In the async case this is being triggered with the
  472. * device lock held and the unregistration work needs to
  473. * be moved out of line iff this is thread has won the
  474. * race to schedule the deletion.
  475. */
  476. if (!kill_device(dev))
  477. return;
  478. get_device(dev);
  479. async_schedule_domain(nd_async_device_unregister, dev,
  480. &nd_async_domain);
  481. break;
  482. case ND_SYNC:
  483. /*
  484. * In the sync case the device is being unregistered due
  485. * to a state change of the parent. Claim the kill state
  486. * to synchronize against other unregistration requests,
  487. * or otherwise let the async path handle it if the
  488. * unregistration was already queued.
  489. */
  490. nd_device_lock(dev);
  491. killed = kill_device(dev);
  492. nd_device_unlock(dev);
  493. if (!killed)
  494. return;
  495. nd_synchronize();
  496. device_unregister(dev);
  497. break;
  498. }
  499. }
  500. EXPORT_SYMBOL(nd_device_unregister);
  501. /**
  502. * __nd_driver_register() - register a region or a namespace driver
  503. * @nd_drv: driver to register
  504. * @owner: automatically set by nd_driver_register() macro
  505. * @mod_name: automatically set by nd_driver_register() macro
  506. */
  507. int __nd_driver_register(struct nd_device_driver *nd_drv, struct module *owner,
  508. const char *mod_name)
  509. {
  510. struct device_driver *drv = &nd_drv->drv;
  511. if (!nd_drv->type) {
  512. pr_debug("driver type bitmask not set (%ps)\n",
  513. __builtin_return_address(0));
  514. return -EINVAL;
  515. }
  516. if (!nd_drv->probe) {
  517. pr_debug("%s ->probe() must be specified\n", mod_name);
  518. return -EINVAL;
  519. }
  520. drv->bus = &nvdimm_bus_type;
  521. drv->owner = owner;
  522. drv->mod_name = mod_name;
  523. return driver_register(drv);
  524. }
  525. EXPORT_SYMBOL(__nd_driver_register);
  526. void nvdimm_check_and_set_ro(struct gendisk *disk)
  527. {
  528. struct device *dev = disk_to_dev(disk)->parent;
  529. struct nd_region *nd_region = to_nd_region(dev->parent);
  530. int disk_ro = get_disk_ro(disk);
  531. /*
  532. * Upgrade to read-only if the region is read-only preserve as
  533. * read-only if the disk is already read-only.
  534. */
  535. if (disk_ro || nd_region->ro == disk_ro)
  536. return;
  537. dev_info(dev, "%s read-only, marking %s read-only\n",
  538. dev_name(&nd_region->dev), disk->disk_name);
  539. set_disk_ro(disk, 1);
  540. }
  541. EXPORT_SYMBOL(nvdimm_check_and_set_ro);
  542. static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
  543. char *buf)
  544. {
  545. return sprintf(buf, ND_DEVICE_MODALIAS_FMT "\n",
  546. to_nd_device_type(dev));
  547. }
  548. static DEVICE_ATTR_RO(modalias);
  549. static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
  550. char *buf)
  551. {
  552. return sprintf(buf, "%s\n", dev->type->name);
  553. }
  554. static DEVICE_ATTR_RO(devtype);
  555. static struct attribute *nd_device_attributes[] = {
  556. &dev_attr_modalias.attr,
  557. &dev_attr_devtype.attr,
  558. NULL,
  559. };
  560. /*
  561. * nd_device_attribute_group - generic attributes for all devices on an nd bus
  562. */
  563. const struct attribute_group nd_device_attribute_group = {
  564. .attrs = nd_device_attributes,
  565. };
  566. static ssize_t numa_node_show(struct device *dev,
  567. struct device_attribute *attr, char *buf)
  568. {
  569. return sprintf(buf, "%d\n", dev_to_node(dev));
  570. }
  571. static DEVICE_ATTR_RO(numa_node);
  572. static int nvdimm_dev_to_target_node(struct device *dev)
  573. {
  574. struct device *parent = dev->parent;
  575. struct nd_region *nd_region = NULL;
  576. if (is_nd_region(dev))
  577. nd_region = to_nd_region(dev);
  578. else if (parent && is_nd_region(parent))
  579. nd_region = to_nd_region(parent);
  580. if (!nd_region)
  581. return NUMA_NO_NODE;
  582. return nd_region->target_node;
  583. }
  584. static ssize_t target_node_show(struct device *dev,
  585. struct device_attribute *attr, char *buf)
  586. {
  587. return sprintf(buf, "%d\n", nvdimm_dev_to_target_node(dev));
  588. }
  589. static DEVICE_ATTR_RO(target_node);
  590. static struct attribute *nd_numa_attributes[] = {
  591. &dev_attr_numa_node.attr,
  592. &dev_attr_target_node.attr,
  593. NULL,
  594. };
  595. static umode_t nd_numa_attr_visible(struct kobject *kobj, struct attribute *a,
  596. int n)
  597. {
  598. struct device *dev = container_of(kobj, typeof(*dev), kobj);
  599. if (!IS_ENABLED(CONFIG_NUMA))
  600. return 0;
  601. if (a == &dev_attr_target_node.attr &&
  602. nvdimm_dev_to_target_node(dev) == NUMA_NO_NODE)
  603. return 0;
  604. return a->mode;
  605. }
  606. /*
  607. * nd_numa_attribute_group - NUMA attributes for all devices on an nd bus
  608. */
  609. const struct attribute_group nd_numa_attribute_group = {
  610. .attrs = nd_numa_attributes,
  611. .is_visible = nd_numa_attr_visible,
  612. };
  613. int nvdimm_bus_create_ndctl(struct nvdimm_bus *nvdimm_bus)
  614. {
  615. dev_t devt = MKDEV(nvdimm_bus_major, nvdimm_bus->id);
  616. struct device *dev;
  617. dev = device_create(nd_class, &nvdimm_bus->dev, devt, nvdimm_bus,
  618. "ndctl%d", nvdimm_bus->id);
  619. if (IS_ERR(dev))
  620. dev_dbg(&nvdimm_bus->dev, "failed to register ndctl%d: %ld\n",
  621. nvdimm_bus->id, PTR_ERR(dev));
  622. return PTR_ERR_OR_ZERO(dev);
  623. }
  624. void nvdimm_bus_destroy_ndctl(struct nvdimm_bus *nvdimm_bus)
  625. {
  626. device_destroy(nd_class, MKDEV(nvdimm_bus_major, nvdimm_bus->id));
  627. }
  628. static const struct nd_cmd_desc __nd_cmd_dimm_descs[] = {
  629. [ND_CMD_IMPLEMENTED] = { },
  630. [ND_CMD_SMART] = {
  631. .out_num = 2,
  632. .out_sizes = { 4, 128, },
  633. },
  634. [ND_CMD_SMART_THRESHOLD] = {
  635. .out_num = 2,
  636. .out_sizes = { 4, 8, },
  637. },
  638. [ND_CMD_DIMM_FLAGS] = {
  639. .out_num = 2,
  640. .out_sizes = { 4, 4 },
  641. },
  642. [ND_CMD_GET_CONFIG_SIZE] = {
  643. .out_num = 3,
  644. .out_sizes = { 4, 4, 4, },
  645. },
  646. [ND_CMD_GET_CONFIG_DATA] = {
  647. .in_num = 2,
  648. .in_sizes = { 4, 4, },
  649. .out_num = 2,
  650. .out_sizes = { 4, UINT_MAX, },
  651. },
  652. [ND_CMD_SET_CONFIG_DATA] = {
  653. .in_num = 3,
  654. .in_sizes = { 4, 4, UINT_MAX, },
  655. .out_num = 1,
  656. .out_sizes = { 4, },
  657. },
  658. [ND_CMD_VENDOR] = {
  659. .in_num = 3,
  660. .in_sizes = { 4, 4, UINT_MAX, },
  661. .out_num = 3,
  662. .out_sizes = { 4, 4, UINT_MAX, },
  663. },
  664. [ND_CMD_CALL] = {
  665. .in_num = 2,
  666. .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
  667. .out_num = 1,
  668. .out_sizes = { UINT_MAX, },
  669. },
  670. };
  671. const struct nd_cmd_desc *nd_cmd_dimm_desc(int cmd)
  672. {
  673. if (cmd < ARRAY_SIZE(__nd_cmd_dimm_descs))
  674. return &__nd_cmd_dimm_descs[cmd];
  675. return NULL;
  676. }
  677. EXPORT_SYMBOL_GPL(nd_cmd_dimm_desc);
  678. static const struct nd_cmd_desc __nd_cmd_bus_descs[] = {
  679. [ND_CMD_IMPLEMENTED] = { },
  680. [ND_CMD_ARS_CAP] = {
  681. .in_num = 2,
  682. .in_sizes = { 8, 8, },
  683. .out_num = 4,
  684. .out_sizes = { 4, 4, 4, 4, },
  685. },
  686. [ND_CMD_ARS_START] = {
  687. .in_num = 5,
  688. .in_sizes = { 8, 8, 2, 1, 5, },
  689. .out_num = 2,
  690. .out_sizes = { 4, 4, },
  691. },
  692. [ND_CMD_ARS_STATUS] = {
  693. .out_num = 3,
  694. .out_sizes = { 4, 4, UINT_MAX, },
  695. },
  696. [ND_CMD_CLEAR_ERROR] = {
  697. .in_num = 2,
  698. .in_sizes = { 8, 8, },
  699. .out_num = 3,
  700. .out_sizes = { 4, 4, 8, },
  701. },
  702. [ND_CMD_CALL] = {
  703. .in_num = 2,
  704. .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
  705. .out_num = 1,
  706. .out_sizes = { UINT_MAX, },
  707. },
  708. };
  709. const struct nd_cmd_desc *nd_cmd_bus_desc(int cmd)
  710. {
  711. if (cmd < ARRAY_SIZE(__nd_cmd_bus_descs))
  712. return &__nd_cmd_bus_descs[cmd];
  713. return NULL;
  714. }
  715. EXPORT_SYMBOL_GPL(nd_cmd_bus_desc);
  716. u32 nd_cmd_in_size(struct nvdimm *nvdimm, int cmd,
  717. const struct nd_cmd_desc *desc, int idx, void *buf)
  718. {
  719. if (idx >= desc->in_num)
  720. return UINT_MAX;
  721. if (desc->in_sizes[idx] < UINT_MAX)
  722. return desc->in_sizes[idx];
  723. if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA && idx == 2) {
  724. struct nd_cmd_set_config_hdr *hdr = buf;
  725. return hdr->in_length;
  726. } else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2) {
  727. struct nd_cmd_vendor_hdr *hdr = buf;
  728. return hdr->in_length;
  729. } else if (cmd == ND_CMD_CALL) {
  730. struct nd_cmd_pkg *pkg = buf;
  731. return pkg->nd_size_in;
  732. }
  733. return UINT_MAX;
  734. }
  735. EXPORT_SYMBOL_GPL(nd_cmd_in_size);
  736. u32 nd_cmd_out_size(struct nvdimm *nvdimm, int cmd,
  737. const struct nd_cmd_desc *desc, int idx, const u32 *in_field,
  738. const u32 *out_field, unsigned long remainder)
  739. {
  740. if (idx >= desc->out_num)
  741. return UINT_MAX;
  742. if (desc->out_sizes[idx] < UINT_MAX)
  743. return desc->out_sizes[idx];
  744. if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA && idx == 1)
  745. return in_field[1];
  746. else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2)
  747. return out_field[1];
  748. else if (!nvdimm && cmd == ND_CMD_ARS_STATUS && idx == 2) {
  749. /*
  750. * Per table 9-276 ARS Data in ACPI 6.1, out_field[1] is
  751. * "Size of Output Buffer in bytes, including this
  752. * field."
  753. */
  754. if (out_field[1] < 4)
  755. return 0;
  756. /*
  757. * ACPI 6.1 is ambiguous if 'status' is included in the
  758. * output size. If we encounter an output size that
  759. * overshoots the remainder by 4 bytes, assume it was
  760. * including 'status'.
  761. */
  762. if (out_field[1] - 4 == remainder)
  763. return remainder;
  764. return out_field[1] - 8;
  765. } else if (cmd == ND_CMD_CALL) {
  766. struct nd_cmd_pkg *pkg = (struct nd_cmd_pkg *) in_field;
  767. return pkg->nd_size_out;
  768. }
  769. return UINT_MAX;
  770. }
  771. EXPORT_SYMBOL_GPL(nd_cmd_out_size);
  772. void wait_nvdimm_bus_probe_idle(struct device *dev)
  773. {
  774. struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
  775. do {
  776. if (nvdimm_bus->probe_active == 0)
  777. break;
  778. nvdimm_bus_unlock(dev);
  779. nd_device_unlock(dev);
  780. wait_event(nvdimm_bus->wait,
  781. nvdimm_bus->probe_active == 0);
  782. nd_device_lock(dev);
  783. nvdimm_bus_lock(dev);
  784. } while (true);
  785. }
  786. static int nd_pmem_forget_poison_check(struct device *dev, void *data)
  787. {
  788. struct nd_cmd_clear_error *clear_err =
  789. (struct nd_cmd_clear_error *)data;
  790. struct nd_btt *nd_btt = is_nd_btt(dev) ? to_nd_btt(dev) : NULL;
  791. struct nd_pfn *nd_pfn = is_nd_pfn(dev) ? to_nd_pfn(dev) : NULL;
  792. struct nd_dax *nd_dax = is_nd_dax(dev) ? to_nd_dax(dev) : NULL;
  793. struct nd_namespace_common *ndns = NULL;
  794. struct nd_namespace_io *nsio;
  795. resource_size_t offset = 0, end_trunc = 0, start, end, pstart, pend;
  796. if (nd_dax || !dev->driver)
  797. return 0;
  798. start = clear_err->address;
  799. end = clear_err->address + clear_err->cleared - 1;
  800. if (nd_btt || nd_pfn || nd_dax) {
  801. if (nd_btt)
  802. ndns = nd_btt->ndns;
  803. else if (nd_pfn)
  804. ndns = nd_pfn->ndns;
  805. else if (nd_dax)
  806. ndns = nd_dax->nd_pfn.ndns;
  807. if (!ndns)
  808. return 0;
  809. } else
  810. ndns = to_ndns(dev);
  811. nsio = to_nd_namespace_io(&ndns->dev);
  812. pstart = nsio->res.start + offset;
  813. pend = nsio->res.end - end_trunc;
  814. if ((pstart >= start) && (pend <= end))
  815. return -EBUSY;
  816. return 0;
  817. }
  818. static int nd_ns_forget_poison_check(struct device *dev, void *data)
  819. {
  820. return device_for_each_child(dev, data, nd_pmem_forget_poison_check);
  821. }
  822. /* set_config requires an idle interleave set */
  823. static int nd_cmd_clear_to_send(struct nvdimm_bus *nvdimm_bus,
  824. struct nvdimm *nvdimm, unsigned int cmd, void *data)
  825. {
  826. struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
  827. /* ask the bus provider if it would like to block this request */
  828. if (nd_desc->clear_to_send) {
  829. int rc = nd_desc->clear_to_send(nd_desc, nvdimm, cmd, data);
  830. if (rc)
  831. return rc;
  832. }
  833. /* require clear error to go through the pmem driver */
  834. if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR)
  835. return device_for_each_child(&nvdimm_bus->dev, data,
  836. nd_ns_forget_poison_check);
  837. if (!nvdimm || cmd != ND_CMD_SET_CONFIG_DATA)
  838. return 0;
  839. /* prevent label manipulation while the kernel owns label updates */
  840. wait_nvdimm_bus_probe_idle(&nvdimm_bus->dev);
  841. if (atomic_read(&nvdimm->busy))
  842. return -EBUSY;
  843. return 0;
  844. }
  845. static int __nd_ioctl(struct nvdimm_bus *nvdimm_bus, struct nvdimm *nvdimm,
  846. int read_only, unsigned int ioctl_cmd, unsigned long arg)
  847. {
  848. struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
  849. const struct nd_cmd_desc *desc = NULL;
  850. unsigned int cmd = _IOC_NR(ioctl_cmd);
  851. struct device *dev = &nvdimm_bus->dev;
  852. void __user *p = (void __user *) arg;
  853. char *out_env = NULL, *in_env = NULL;
  854. const char *cmd_name, *dimm_name;
  855. u32 in_len = 0, out_len = 0;
  856. unsigned int func = cmd;
  857. unsigned long cmd_mask;
  858. struct nd_cmd_pkg pkg;
  859. int rc, i, cmd_rc;
  860. void *buf = NULL;
  861. u64 buf_len = 0;
  862. if (nvdimm) {
  863. desc = nd_cmd_dimm_desc(cmd);
  864. cmd_name = nvdimm_cmd_name(cmd);
  865. cmd_mask = nvdimm->cmd_mask;
  866. dimm_name = dev_name(&nvdimm->dev);
  867. } else {
  868. desc = nd_cmd_bus_desc(cmd);
  869. cmd_name = nvdimm_bus_cmd_name(cmd);
  870. cmd_mask = nd_desc->cmd_mask;
  871. dimm_name = "bus";
  872. }
  873. /* Validate command family support against bus declared support */
  874. if (cmd == ND_CMD_CALL) {
  875. unsigned long *mask;
  876. if (copy_from_user(&pkg, p, sizeof(pkg)))
  877. return -EFAULT;
  878. if (nvdimm) {
  879. if (pkg.nd_family > NVDIMM_FAMILY_MAX)
  880. return -EINVAL;
  881. mask = &nd_desc->dimm_family_mask;
  882. } else {
  883. if (pkg.nd_family > NVDIMM_BUS_FAMILY_MAX)
  884. return -EINVAL;
  885. mask = &nd_desc->bus_family_mask;
  886. }
  887. if (!test_bit(pkg.nd_family, mask))
  888. return -EINVAL;
  889. }
  890. if (!desc ||
  891. (desc->out_num + desc->in_num == 0) ||
  892. cmd > ND_CMD_CALL ||
  893. !test_bit(cmd, &cmd_mask))
  894. return -ENOTTY;
  895. /* fail write commands (when read-only) */
  896. if (read_only)
  897. switch (cmd) {
  898. case ND_CMD_VENDOR:
  899. case ND_CMD_SET_CONFIG_DATA:
  900. case ND_CMD_ARS_START:
  901. case ND_CMD_CLEAR_ERROR:
  902. case ND_CMD_CALL:
  903. dev_dbg(dev, "'%s' command while read-only.\n",
  904. nvdimm ? nvdimm_cmd_name(cmd)
  905. : nvdimm_bus_cmd_name(cmd));
  906. return -EPERM;
  907. default:
  908. break;
  909. }
  910. /* process an input envelope */
  911. in_env = kzalloc(ND_CMD_MAX_ENVELOPE, GFP_KERNEL);
  912. if (!in_env)
  913. return -ENOMEM;
  914. for (i = 0; i < desc->in_num; i++) {
  915. u32 in_size, copy;
  916. in_size = nd_cmd_in_size(nvdimm, cmd, desc, i, in_env);
  917. if (in_size == UINT_MAX) {
  918. dev_err(dev, "%s:%s unknown input size cmd: %s field: %d\n",
  919. __func__, dimm_name, cmd_name, i);
  920. rc = -ENXIO;
  921. goto out;
  922. }
  923. if (in_len < ND_CMD_MAX_ENVELOPE)
  924. copy = min_t(u32, ND_CMD_MAX_ENVELOPE - in_len, in_size);
  925. else
  926. copy = 0;
  927. if (copy && copy_from_user(&in_env[in_len], p + in_len, copy)) {
  928. rc = -EFAULT;
  929. goto out;
  930. }
  931. in_len += in_size;
  932. }
  933. if (cmd == ND_CMD_CALL) {
  934. func = pkg.nd_command;
  935. dev_dbg(dev, "%s, idx: %llu, in: %u, out: %u, len %llu\n",
  936. dimm_name, pkg.nd_command,
  937. in_len, out_len, buf_len);
  938. }
  939. /* process an output envelope */
  940. out_env = kzalloc(ND_CMD_MAX_ENVELOPE, GFP_KERNEL);
  941. if (!out_env) {
  942. rc = -ENOMEM;
  943. goto out;
  944. }
  945. for (i = 0; i < desc->out_num; i++) {
  946. u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i,
  947. (u32 *) in_env, (u32 *) out_env, 0);
  948. u32 copy;
  949. if (out_size == UINT_MAX) {
  950. dev_dbg(dev, "%s unknown output size cmd: %s field: %d\n",
  951. dimm_name, cmd_name, i);
  952. rc = -EFAULT;
  953. goto out;
  954. }
  955. if (out_len < ND_CMD_MAX_ENVELOPE)
  956. copy = min_t(u32, ND_CMD_MAX_ENVELOPE - out_len, out_size);
  957. else
  958. copy = 0;
  959. if (copy && copy_from_user(&out_env[out_len],
  960. p + in_len + out_len, copy)) {
  961. rc = -EFAULT;
  962. goto out;
  963. }
  964. out_len += out_size;
  965. }
  966. buf_len = (u64) out_len + (u64) in_len;
  967. if (buf_len > ND_IOCTL_MAX_BUFLEN) {
  968. dev_dbg(dev, "%s cmd: %s buf_len: %llu > %d\n", dimm_name,
  969. cmd_name, buf_len, ND_IOCTL_MAX_BUFLEN);
  970. rc = -EINVAL;
  971. goto out;
  972. }
  973. buf = vmalloc(buf_len);
  974. if (!buf) {
  975. rc = -ENOMEM;
  976. goto out;
  977. }
  978. if (copy_from_user(buf, p, buf_len)) {
  979. rc = -EFAULT;
  980. goto out;
  981. }
  982. nd_device_lock(dev);
  983. nvdimm_bus_lock(dev);
  984. rc = nd_cmd_clear_to_send(nvdimm_bus, nvdimm, func, buf);
  985. if (rc)
  986. goto out_unlock;
  987. rc = nd_desc->ndctl(nd_desc, nvdimm, cmd, buf, buf_len, &cmd_rc);
  988. if (rc < 0)
  989. goto out_unlock;
  990. if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR && cmd_rc >= 0) {
  991. struct nd_cmd_clear_error *clear_err = buf;
  992. nvdimm_account_cleared_poison(nvdimm_bus, clear_err->address,
  993. clear_err->cleared);
  994. }
  995. if (copy_to_user(p, buf, buf_len))
  996. rc = -EFAULT;
  997. out_unlock:
  998. nvdimm_bus_unlock(dev);
  999. nd_device_unlock(dev);
  1000. out:
  1001. kfree(in_env);
  1002. kfree(out_env);
  1003. vfree(buf);
  1004. return rc;
  1005. }
  1006. enum nd_ioctl_mode {
  1007. BUS_IOCTL,
  1008. DIMM_IOCTL,
  1009. };
  1010. static int match_dimm(struct device *dev, void *data)
  1011. {
  1012. long id = (long) data;
  1013. if (is_nvdimm(dev)) {
  1014. struct nvdimm *nvdimm = to_nvdimm(dev);
  1015. return nvdimm->id == id;
  1016. }
  1017. return 0;
  1018. }
  1019. static long nd_ioctl(struct file *file, unsigned int cmd, unsigned long arg,
  1020. enum nd_ioctl_mode mode)
  1021. {
  1022. struct nvdimm_bus *nvdimm_bus, *found = NULL;
  1023. long id = (long) file->private_data;
  1024. struct nvdimm *nvdimm = NULL;
  1025. int rc, ro;
  1026. ro = ((file->f_flags & O_ACCMODE) == O_RDONLY);
  1027. mutex_lock(&nvdimm_bus_list_mutex);
  1028. list_for_each_entry(nvdimm_bus, &nvdimm_bus_list, list) {
  1029. if (mode == DIMM_IOCTL) {
  1030. struct device *dev;
  1031. dev = device_find_child(&nvdimm_bus->dev,
  1032. file->private_data, match_dimm);
  1033. if (!dev)
  1034. continue;
  1035. nvdimm = to_nvdimm(dev);
  1036. found = nvdimm_bus;
  1037. } else if (nvdimm_bus->id == id) {
  1038. found = nvdimm_bus;
  1039. }
  1040. if (found) {
  1041. atomic_inc(&nvdimm_bus->ioctl_active);
  1042. break;
  1043. }
  1044. }
  1045. mutex_unlock(&nvdimm_bus_list_mutex);
  1046. if (!found)
  1047. return -ENXIO;
  1048. nvdimm_bus = found;
  1049. rc = __nd_ioctl(nvdimm_bus, nvdimm, ro, cmd, arg);
  1050. if (nvdimm)
  1051. put_device(&nvdimm->dev);
  1052. if (atomic_dec_and_test(&nvdimm_bus->ioctl_active))
  1053. wake_up(&nvdimm_bus->wait);
  1054. return rc;
  1055. }
  1056. static long bus_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
  1057. {
  1058. return nd_ioctl(file, cmd, arg, BUS_IOCTL);
  1059. }
  1060. static long dimm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
  1061. {
  1062. return nd_ioctl(file, cmd, arg, DIMM_IOCTL);
  1063. }
  1064. static int nd_open(struct inode *inode, struct file *file)
  1065. {
  1066. long minor = iminor(inode);
  1067. file->private_data = (void *) minor;
  1068. return 0;
  1069. }
  1070. static const struct file_operations nvdimm_bus_fops = {
  1071. .owner = THIS_MODULE,
  1072. .open = nd_open,
  1073. .unlocked_ioctl = bus_ioctl,
  1074. .compat_ioctl = compat_ptr_ioctl,
  1075. .llseek = noop_llseek,
  1076. };
  1077. static const struct file_operations nvdimm_fops = {
  1078. .owner = THIS_MODULE,
  1079. .open = nd_open,
  1080. .unlocked_ioctl = dimm_ioctl,
  1081. .compat_ioctl = compat_ptr_ioctl,
  1082. .llseek = noop_llseek,
  1083. };
  1084. int __init nvdimm_bus_init(void)
  1085. {
  1086. int rc;
  1087. rc = bus_register(&nvdimm_bus_type);
  1088. if (rc)
  1089. return rc;
  1090. rc = register_chrdev(0, "ndctl", &nvdimm_bus_fops);
  1091. if (rc < 0)
  1092. goto err_bus_chrdev;
  1093. nvdimm_bus_major = rc;
  1094. rc = register_chrdev(0, "dimmctl", &nvdimm_fops);
  1095. if (rc < 0)
  1096. goto err_dimm_chrdev;
  1097. nvdimm_major = rc;
  1098. nd_class = class_create(THIS_MODULE, "nd");
  1099. if (IS_ERR(nd_class)) {
  1100. rc = PTR_ERR(nd_class);
  1101. goto err_class;
  1102. }
  1103. rc = driver_register(&nd_bus_driver.drv);
  1104. if (rc)
  1105. goto err_nd_bus;
  1106. return 0;
  1107. err_nd_bus:
  1108. class_destroy(nd_class);
  1109. err_class:
  1110. unregister_chrdev(nvdimm_major, "dimmctl");
  1111. err_dimm_chrdev:
  1112. unregister_chrdev(nvdimm_bus_major, "ndctl");
  1113. err_bus_chrdev:
  1114. bus_unregister(&nvdimm_bus_type);
  1115. return rc;
  1116. }
  1117. void nvdimm_bus_exit(void)
  1118. {
  1119. driver_unregister(&nd_bus_driver.drv);
  1120. class_destroy(nd_class);
  1121. unregister_chrdev(nvdimm_bus_major, "ndctl");
  1122. unregister_chrdev(nvdimm_major, "dimmctl");
  1123. bus_unregister(&nvdimm_bus_type);
  1124. ida_destroy(&nd_ida);
  1125. }