virtio_mem.c 52 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Virtio-mem device driver.
  4. *
  5. * Copyright Red Hat, Inc. 2020
  6. *
  7. * Author(s): David Hildenbrand <david@redhat.com>
  8. */
  9. #include <linux/virtio.h>
  10. #include <linux/virtio_mem.h>
  11. #include <linux/workqueue.h>
  12. #include <linux/slab.h>
  13. #include <linux/module.h>
  14. #include <linux/mm.h>
  15. #include <linux/memory_hotplug.h>
  16. #include <linux/memory.h>
  17. #include <linux/hrtimer.h>
  18. #include <linux/crash_dump.h>
  19. #include <linux/mutex.h>
  20. #include <linux/bitmap.h>
  21. #include <linux/lockdep.h>
  22. #include <acpi/acpi_numa.h>
  23. static bool unplug_online = true;
  24. module_param(unplug_online, bool, 0644);
  25. MODULE_PARM_DESC(unplug_online, "Try to unplug online memory");
  26. enum virtio_mem_mb_state {
  27. /* Unplugged, not added to Linux. Can be reused later. */
  28. VIRTIO_MEM_MB_STATE_UNUSED = 0,
  29. /* (Partially) plugged, not added to Linux. Error on add_memory(). */
  30. VIRTIO_MEM_MB_STATE_PLUGGED,
  31. /* Fully plugged, fully added to Linux, offline. */
  32. VIRTIO_MEM_MB_STATE_OFFLINE,
  33. /* Partially plugged, fully added to Linux, offline. */
  34. VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL,
  35. /* Fully plugged, fully added to Linux, online. */
  36. VIRTIO_MEM_MB_STATE_ONLINE,
  37. /* Partially plugged, fully added to Linux, online. */
  38. VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL,
  39. VIRTIO_MEM_MB_STATE_COUNT
  40. };
  41. struct virtio_mem {
  42. struct virtio_device *vdev;
  43. /* We might first have to unplug all memory when starting up. */
  44. bool unplug_all_required;
  45. /* Workqueue that processes the plug/unplug requests. */
  46. struct work_struct wq;
  47. atomic_t config_changed;
  48. /* Virtqueue for guest->host requests. */
  49. struct virtqueue *vq;
  50. /* Wait for a host response to a guest request. */
  51. wait_queue_head_t host_resp;
  52. /* Space for one guest request and the host response. */
  53. struct virtio_mem_req req;
  54. struct virtio_mem_resp resp;
  55. /* The current size of the device. */
  56. uint64_t plugged_size;
  57. /* The requested size of the device. */
  58. uint64_t requested_size;
  59. /* The device block size (for communicating with the device). */
  60. uint64_t device_block_size;
  61. /* The translated node id. NUMA_NO_NODE in case not specified. */
  62. int nid;
  63. /* Physical start address of the memory region. */
  64. uint64_t addr;
  65. /* Maximum region size in bytes. */
  66. uint64_t region_size;
  67. /* The subblock size. */
  68. uint64_t subblock_size;
  69. /* The number of subblocks per memory block. */
  70. uint32_t nb_sb_per_mb;
  71. /* Id of the first memory block of this device. */
  72. unsigned long first_mb_id;
  73. /* Id of the last memory block of this device. */
  74. unsigned long last_mb_id;
  75. /* Id of the last usable memory block of this device. */
  76. unsigned long last_usable_mb_id;
  77. /* Id of the next memory bock to prepare when needed. */
  78. unsigned long next_mb_id;
  79. /* The parent resource for all memory added via this device. */
  80. struct resource *parent_resource;
  81. /*
  82. * Copy of "System RAM (virtio_mem)" to be used for
  83. * add_memory_driver_managed().
  84. */
  85. const char *resource_name;
  86. /* Summary of all memory block states. */
  87. unsigned long nb_mb_state[VIRTIO_MEM_MB_STATE_COUNT];
  88. #define VIRTIO_MEM_NB_OFFLINE_THRESHOLD 10
  89. /*
  90. * One byte state per memory block.
  91. *
  92. * Allocated via vmalloc(). When preparing new blocks, resized
  93. * (alloc+copy+free) when needed (crossing pages with the next mb).
  94. * (when crossing pages).
  95. *
  96. * With 128MB memory blocks, we have states for 512GB of memory in one
  97. * page.
  98. */
  99. uint8_t *mb_state;
  100. /*
  101. * $nb_sb_per_mb bit per memory block. Handled similar to mb_state.
  102. *
  103. * With 4MB subblocks, we manage 128GB of memory in one page.
  104. */
  105. unsigned long *sb_bitmap;
  106. /*
  107. * Mutex that protects the nb_mb_state, mb_state, and sb_bitmap.
  108. *
  109. * When this lock is held the pointers can't change, ONLINE and
  110. * OFFLINE blocks can't change the state and no subblocks will get
  111. * plugged/unplugged.
  112. */
  113. struct mutex hotplug_mutex;
  114. bool hotplug_active;
  115. /* An error occurred we cannot handle - stop processing requests. */
  116. bool broken;
  117. /* The driver is being removed. */
  118. spinlock_t removal_lock;
  119. bool removing;
  120. /* Timer for retrying to plug/unplug memory. */
  121. struct hrtimer retry_timer;
  122. unsigned int retry_timer_ms;
  123. #define VIRTIO_MEM_RETRY_TIMER_MIN_MS 50000
  124. #define VIRTIO_MEM_RETRY_TIMER_MAX_MS 300000
  125. /* Memory notifier (online/offline events). */
  126. struct notifier_block memory_notifier;
  127. /* Next device in the list of virtio-mem devices. */
  128. struct list_head next;
  129. };
  130. /*
  131. * We have to share a single online_page callback among all virtio-mem
  132. * devices. We use RCU to iterate the list in the callback.
  133. */
  134. static DEFINE_MUTEX(virtio_mem_mutex);
  135. static LIST_HEAD(virtio_mem_devices);
  136. static void virtio_mem_online_page_cb(struct page *page, unsigned int order);
  137. /*
  138. * Register a virtio-mem device so it will be considered for the online_page
  139. * callback.
  140. */
  141. static int register_virtio_mem_device(struct virtio_mem *vm)
  142. {
  143. int rc = 0;
  144. /* First device registers the callback. */
  145. mutex_lock(&virtio_mem_mutex);
  146. if (list_empty(&virtio_mem_devices))
  147. rc = set_online_page_callback(&virtio_mem_online_page_cb);
  148. if (!rc)
  149. list_add_rcu(&vm->next, &virtio_mem_devices);
  150. mutex_unlock(&virtio_mem_mutex);
  151. return rc;
  152. }
  153. /*
  154. * Unregister a virtio-mem device so it will no longer be considered for the
  155. * online_page callback.
  156. */
  157. static void unregister_virtio_mem_device(struct virtio_mem *vm)
  158. {
  159. /* Last device unregisters the callback. */
  160. mutex_lock(&virtio_mem_mutex);
  161. list_del_rcu(&vm->next);
  162. if (list_empty(&virtio_mem_devices))
  163. restore_online_page_callback(&virtio_mem_online_page_cb);
  164. mutex_unlock(&virtio_mem_mutex);
  165. synchronize_rcu();
  166. }
  167. /*
  168. * Calculate the memory block id of a given address.
  169. */
  170. static unsigned long virtio_mem_phys_to_mb_id(unsigned long addr)
  171. {
  172. return addr / memory_block_size_bytes();
  173. }
  174. /*
  175. * Calculate the physical start address of a given memory block id.
  176. */
  177. static unsigned long virtio_mem_mb_id_to_phys(unsigned long mb_id)
  178. {
  179. return mb_id * memory_block_size_bytes();
  180. }
  181. /*
  182. * Calculate the subblock id of a given address.
  183. */
  184. static unsigned long virtio_mem_phys_to_sb_id(struct virtio_mem *vm,
  185. unsigned long addr)
  186. {
  187. const unsigned long mb_id = virtio_mem_phys_to_mb_id(addr);
  188. const unsigned long mb_addr = virtio_mem_mb_id_to_phys(mb_id);
  189. return (addr - mb_addr) / vm->subblock_size;
  190. }
  191. /*
  192. * Set the state of a memory block, taking care of the state counter.
  193. */
  194. static void virtio_mem_mb_set_state(struct virtio_mem *vm, unsigned long mb_id,
  195. enum virtio_mem_mb_state state)
  196. {
  197. const unsigned long idx = mb_id - vm->first_mb_id;
  198. enum virtio_mem_mb_state old_state;
  199. old_state = vm->mb_state[idx];
  200. vm->mb_state[idx] = state;
  201. BUG_ON(vm->nb_mb_state[old_state] == 0);
  202. vm->nb_mb_state[old_state]--;
  203. vm->nb_mb_state[state]++;
  204. }
  205. /*
  206. * Get the state of a memory block.
  207. */
  208. static enum virtio_mem_mb_state virtio_mem_mb_get_state(struct virtio_mem *vm,
  209. unsigned long mb_id)
  210. {
  211. const unsigned long idx = mb_id - vm->first_mb_id;
  212. return vm->mb_state[idx];
  213. }
  214. /*
  215. * Prepare the state array for the next memory block.
  216. */
  217. static int virtio_mem_mb_state_prepare_next_mb(struct virtio_mem *vm)
  218. {
  219. unsigned long old_bytes = vm->next_mb_id - vm->first_mb_id + 1;
  220. unsigned long new_bytes = vm->next_mb_id - vm->first_mb_id + 2;
  221. int old_pages = PFN_UP(old_bytes);
  222. int new_pages = PFN_UP(new_bytes);
  223. uint8_t *new_mb_state;
  224. if (vm->mb_state && old_pages == new_pages)
  225. return 0;
  226. new_mb_state = vzalloc(new_pages * PAGE_SIZE);
  227. if (!new_mb_state)
  228. return -ENOMEM;
  229. mutex_lock(&vm->hotplug_mutex);
  230. if (vm->mb_state)
  231. memcpy(new_mb_state, vm->mb_state, old_pages * PAGE_SIZE);
  232. vfree(vm->mb_state);
  233. vm->mb_state = new_mb_state;
  234. mutex_unlock(&vm->hotplug_mutex);
  235. return 0;
  236. }
  237. #define virtio_mem_for_each_mb_state(_vm, _mb_id, _state) \
  238. for (_mb_id = _vm->first_mb_id; \
  239. _mb_id < _vm->next_mb_id && _vm->nb_mb_state[_state]; \
  240. _mb_id++) \
  241. if (virtio_mem_mb_get_state(_vm, _mb_id) == _state)
  242. #define virtio_mem_for_each_mb_state_rev(_vm, _mb_id, _state) \
  243. for (_mb_id = _vm->next_mb_id - 1; \
  244. _mb_id >= _vm->first_mb_id && _vm->nb_mb_state[_state]; \
  245. _mb_id--) \
  246. if (virtio_mem_mb_get_state(_vm, _mb_id) == _state)
  247. /*
  248. * Mark all selected subblocks plugged.
  249. *
  250. * Will not modify the state of the memory block.
  251. */
  252. static void virtio_mem_mb_set_sb_plugged(struct virtio_mem *vm,
  253. unsigned long mb_id, int sb_id,
  254. int count)
  255. {
  256. const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id;
  257. __bitmap_set(vm->sb_bitmap, bit, count);
  258. }
  259. /*
  260. * Mark all selected subblocks unplugged.
  261. *
  262. * Will not modify the state of the memory block.
  263. */
  264. static void virtio_mem_mb_set_sb_unplugged(struct virtio_mem *vm,
  265. unsigned long mb_id, int sb_id,
  266. int count)
  267. {
  268. const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id;
  269. __bitmap_clear(vm->sb_bitmap, bit, count);
  270. }
  271. /*
  272. * Test if all selected subblocks are plugged.
  273. */
  274. static bool virtio_mem_mb_test_sb_plugged(struct virtio_mem *vm,
  275. unsigned long mb_id, int sb_id,
  276. int count)
  277. {
  278. const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id;
  279. if (count == 1)
  280. return test_bit(bit, vm->sb_bitmap);
  281. /* TODO: Helper similar to bitmap_set() */
  282. return find_next_zero_bit(vm->sb_bitmap, bit + count, bit) >=
  283. bit + count;
  284. }
  285. /*
  286. * Test if all selected subblocks are unplugged.
  287. */
  288. static bool virtio_mem_mb_test_sb_unplugged(struct virtio_mem *vm,
  289. unsigned long mb_id, int sb_id,
  290. int count)
  291. {
  292. const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb + sb_id;
  293. /* TODO: Helper similar to bitmap_set() */
  294. return find_next_bit(vm->sb_bitmap, bit + count, bit) >= bit + count;
  295. }
  296. /*
  297. * Find the first unplugged subblock. Returns vm->nb_sb_per_mb in case there is
  298. * none.
  299. */
  300. static int virtio_mem_mb_first_unplugged_sb(struct virtio_mem *vm,
  301. unsigned long mb_id)
  302. {
  303. const int bit = (mb_id - vm->first_mb_id) * vm->nb_sb_per_mb;
  304. return find_next_zero_bit(vm->sb_bitmap, bit + vm->nb_sb_per_mb, bit) -
  305. bit;
  306. }
  307. /*
  308. * Prepare the subblock bitmap for the next memory block.
  309. */
  310. static int virtio_mem_sb_bitmap_prepare_next_mb(struct virtio_mem *vm)
  311. {
  312. const unsigned long old_nb_mb = vm->next_mb_id - vm->first_mb_id;
  313. const unsigned long old_nb_bits = old_nb_mb * vm->nb_sb_per_mb;
  314. const unsigned long new_nb_bits = (old_nb_mb + 1) * vm->nb_sb_per_mb;
  315. int old_pages = PFN_UP(BITS_TO_LONGS(old_nb_bits) * sizeof(long));
  316. int new_pages = PFN_UP(BITS_TO_LONGS(new_nb_bits) * sizeof(long));
  317. unsigned long *new_sb_bitmap, *old_sb_bitmap;
  318. if (vm->sb_bitmap && old_pages == new_pages)
  319. return 0;
  320. new_sb_bitmap = vzalloc(new_pages * PAGE_SIZE);
  321. if (!new_sb_bitmap)
  322. return -ENOMEM;
  323. mutex_lock(&vm->hotplug_mutex);
  324. if (new_sb_bitmap)
  325. memcpy(new_sb_bitmap, vm->sb_bitmap, old_pages * PAGE_SIZE);
  326. old_sb_bitmap = vm->sb_bitmap;
  327. vm->sb_bitmap = new_sb_bitmap;
  328. mutex_unlock(&vm->hotplug_mutex);
  329. vfree(old_sb_bitmap);
  330. return 0;
  331. }
  332. /*
  333. * Try to add a memory block to Linux. This will usually only fail
  334. * if out of memory.
  335. *
  336. * Must not be called with the vm->hotplug_mutex held (possible deadlock with
  337. * onlining code).
  338. *
  339. * Will not modify the state of the memory block.
  340. */
  341. static int virtio_mem_mb_add(struct virtio_mem *vm, unsigned long mb_id)
  342. {
  343. const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
  344. int nid = vm->nid;
  345. if (nid == NUMA_NO_NODE)
  346. nid = memory_add_physaddr_to_nid(addr);
  347. /*
  348. * When force-unloading the driver and we still have memory added to
  349. * Linux, the resource name has to stay.
  350. */
  351. if (!vm->resource_name) {
  352. vm->resource_name = kstrdup_const("System RAM (virtio_mem)",
  353. GFP_KERNEL);
  354. if (!vm->resource_name)
  355. return -ENOMEM;
  356. }
  357. dev_dbg(&vm->vdev->dev, "adding memory block: %lu\n", mb_id);
  358. return add_memory_driver_managed(nid, addr, memory_block_size_bytes(),
  359. vm->resource_name,
  360. MEMHP_MERGE_RESOURCE);
  361. }
  362. /*
  363. * Try to remove a memory block from Linux. Will only fail if the memory block
  364. * is not offline.
  365. *
  366. * Must not be called with the vm->hotplug_mutex held (possible deadlock with
  367. * onlining code).
  368. *
  369. * Will not modify the state of the memory block.
  370. */
  371. static int virtio_mem_mb_remove(struct virtio_mem *vm, unsigned long mb_id)
  372. {
  373. const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
  374. int nid = vm->nid;
  375. if (nid == NUMA_NO_NODE)
  376. nid = memory_add_physaddr_to_nid(addr);
  377. dev_dbg(&vm->vdev->dev, "removing memory block: %lu\n", mb_id);
  378. return remove_memory(nid, addr, memory_block_size_bytes());
  379. }
  380. /*
  381. * Try to offline and remove a memory block from Linux.
  382. *
  383. * Must not be called with the vm->hotplug_mutex held (possible deadlock with
  384. * onlining code).
  385. *
  386. * Will not modify the state of the memory block.
  387. */
  388. static int virtio_mem_mb_offline_and_remove(struct virtio_mem *vm,
  389. unsigned long mb_id)
  390. {
  391. const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id);
  392. int nid = vm->nid;
  393. if (nid == NUMA_NO_NODE)
  394. nid = memory_add_physaddr_to_nid(addr);
  395. dev_dbg(&vm->vdev->dev, "offlining and removing memory block: %lu\n",
  396. mb_id);
  397. return offline_and_remove_memory(nid, addr, memory_block_size_bytes());
  398. }
  399. /*
  400. * Trigger the workqueue so the device can perform its magic.
  401. */
  402. static void virtio_mem_retry(struct virtio_mem *vm)
  403. {
  404. unsigned long flags;
  405. spin_lock_irqsave(&vm->removal_lock, flags);
  406. if (!vm->removing)
  407. queue_work(system_freezable_wq, &vm->wq);
  408. spin_unlock_irqrestore(&vm->removal_lock, flags);
  409. }
  410. static int virtio_mem_translate_node_id(struct virtio_mem *vm, uint16_t node_id)
  411. {
  412. int node = NUMA_NO_NODE;
  413. #if defined(CONFIG_ACPI_NUMA)
  414. if (virtio_has_feature(vm->vdev, VIRTIO_MEM_F_ACPI_PXM))
  415. node = pxm_to_node(node_id);
  416. #endif
  417. return node;
  418. }
  419. /*
  420. * Test if a virtio-mem device overlaps with the given range. Can be called
  421. * from (notifier) callbacks lockless.
  422. */
  423. static bool virtio_mem_overlaps_range(struct virtio_mem *vm,
  424. unsigned long start, unsigned long size)
  425. {
  426. unsigned long dev_start = virtio_mem_mb_id_to_phys(vm->first_mb_id);
  427. unsigned long dev_end = virtio_mem_mb_id_to_phys(vm->last_mb_id) +
  428. memory_block_size_bytes();
  429. return start < dev_end && dev_start < start + size;
  430. }
  431. /*
  432. * Test if a virtio-mem device owns a memory block. Can be called from
  433. * (notifier) callbacks lockless.
  434. */
  435. static bool virtio_mem_owned_mb(struct virtio_mem *vm, unsigned long mb_id)
  436. {
  437. return mb_id >= vm->first_mb_id && mb_id <= vm->last_mb_id;
  438. }
  439. static int virtio_mem_notify_going_online(struct virtio_mem *vm,
  440. unsigned long mb_id)
  441. {
  442. switch (virtio_mem_mb_get_state(vm, mb_id)) {
  443. case VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL:
  444. case VIRTIO_MEM_MB_STATE_OFFLINE:
  445. return NOTIFY_OK;
  446. default:
  447. break;
  448. }
  449. dev_warn_ratelimited(&vm->vdev->dev,
  450. "memory block onlining denied\n");
  451. return NOTIFY_BAD;
  452. }
  453. static void virtio_mem_notify_offline(struct virtio_mem *vm,
  454. unsigned long mb_id)
  455. {
  456. switch (virtio_mem_mb_get_state(vm, mb_id)) {
  457. case VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL:
  458. virtio_mem_mb_set_state(vm, mb_id,
  459. VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL);
  460. break;
  461. case VIRTIO_MEM_MB_STATE_ONLINE:
  462. virtio_mem_mb_set_state(vm, mb_id,
  463. VIRTIO_MEM_MB_STATE_OFFLINE);
  464. break;
  465. default:
  466. BUG();
  467. break;
  468. }
  469. /*
  470. * Trigger the workqueue, maybe we can now unplug memory. Also,
  471. * when we offline and remove a memory block, this will re-trigger
  472. * us immediately - which is often nice because the removal of
  473. * the memory block (e.g., memmap) might have freed up memory
  474. * on other memory blocks we manage.
  475. */
  476. virtio_mem_retry(vm);
  477. }
  478. static void virtio_mem_notify_online(struct virtio_mem *vm, unsigned long mb_id)
  479. {
  480. unsigned long nb_offline;
  481. switch (virtio_mem_mb_get_state(vm, mb_id)) {
  482. case VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL:
  483. virtio_mem_mb_set_state(vm, mb_id,
  484. VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL);
  485. break;
  486. case VIRTIO_MEM_MB_STATE_OFFLINE:
  487. virtio_mem_mb_set_state(vm, mb_id, VIRTIO_MEM_MB_STATE_ONLINE);
  488. break;
  489. default:
  490. BUG();
  491. break;
  492. }
  493. nb_offline = vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE] +
  494. vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL];
  495. /* see if we can add new blocks now that we onlined one block */
  496. if (nb_offline == VIRTIO_MEM_NB_OFFLINE_THRESHOLD - 1)
  497. virtio_mem_retry(vm);
  498. }
  499. static void virtio_mem_notify_going_offline(struct virtio_mem *vm,
  500. unsigned long mb_id)
  501. {
  502. const unsigned long nr_pages = PFN_DOWN(vm->subblock_size);
  503. struct page *page;
  504. unsigned long pfn;
  505. int sb_id, i;
  506. for (sb_id = 0; sb_id < vm->nb_sb_per_mb; sb_id++) {
  507. if (virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))
  508. continue;
  509. /*
  510. * Drop our reference to the pages so the memory can get
  511. * offlined and add the unplugged pages to the managed
  512. * page counters (so offlining code can correctly subtract
  513. * them again).
  514. */
  515. pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
  516. sb_id * vm->subblock_size);
  517. adjust_managed_page_count(pfn_to_page(pfn), nr_pages);
  518. for (i = 0; i < nr_pages; i++) {
  519. page = pfn_to_page(pfn + i);
  520. if (WARN_ON(!page_ref_dec_and_test(page)))
  521. dump_page(page, "unplugged page referenced");
  522. }
  523. }
  524. }
  525. static void virtio_mem_notify_cancel_offline(struct virtio_mem *vm,
  526. unsigned long mb_id)
  527. {
  528. const unsigned long nr_pages = PFN_DOWN(vm->subblock_size);
  529. unsigned long pfn;
  530. int sb_id, i;
  531. for (sb_id = 0; sb_id < vm->nb_sb_per_mb; sb_id++) {
  532. if (virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))
  533. continue;
  534. /*
  535. * Get the reference we dropped when going offline and
  536. * subtract the unplugged pages from the managed page
  537. * counters.
  538. */
  539. pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
  540. sb_id * vm->subblock_size);
  541. adjust_managed_page_count(pfn_to_page(pfn), -nr_pages);
  542. for (i = 0; i < nr_pages; i++)
  543. page_ref_inc(pfn_to_page(pfn + i));
  544. }
  545. }
  546. /*
  547. * This callback will either be called synchronously from add_memory() or
  548. * asynchronously (e.g., triggered via user space). We have to be careful
  549. * with locking when calling add_memory().
  550. */
  551. static int virtio_mem_memory_notifier_cb(struct notifier_block *nb,
  552. unsigned long action, void *arg)
  553. {
  554. struct virtio_mem *vm = container_of(nb, struct virtio_mem,
  555. memory_notifier);
  556. struct memory_notify *mhp = arg;
  557. const unsigned long start = PFN_PHYS(mhp->start_pfn);
  558. const unsigned long size = PFN_PHYS(mhp->nr_pages);
  559. const unsigned long mb_id = virtio_mem_phys_to_mb_id(start);
  560. int rc = NOTIFY_OK;
  561. if (!virtio_mem_overlaps_range(vm, start, size))
  562. return NOTIFY_DONE;
  563. /*
  564. * Memory is onlined/offlined in memory block granularity. We cannot
  565. * cross virtio-mem device boundaries and memory block boundaries. Bail
  566. * out if this ever changes.
  567. */
  568. if (WARN_ON_ONCE(size != memory_block_size_bytes() ||
  569. !IS_ALIGNED(start, memory_block_size_bytes())))
  570. return NOTIFY_BAD;
  571. /*
  572. * Avoid circular locking lockdep warnings. We lock the mutex
  573. * e.g., in MEM_GOING_ONLINE and unlock it in MEM_ONLINE. The
  574. * blocking_notifier_call_chain() has it's own lock, which gets unlocked
  575. * between both notifier calls and will bail out. False positive.
  576. */
  577. lockdep_off();
  578. switch (action) {
  579. case MEM_GOING_OFFLINE:
  580. mutex_lock(&vm->hotplug_mutex);
  581. if (vm->removing) {
  582. rc = notifier_from_errno(-EBUSY);
  583. mutex_unlock(&vm->hotplug_mutex);
  584. break;
  585. }
  586. vm->hotplug_active = true;
  587. virtio_mem_notify_going_offline(vm, mb_id);
  588. break;
  589. case MEM_GOING_ONLINE:
  590. mutex_lock(&vm->hotplug_mutex);
  591. if (vm->removing) {
  592. rc = notifier_from_errno(-EBUSY);
  593. mutex_unlock(&vm->hotplug_mutex);
  594. break;
  595. }
  596. vm->hotplug_active = true;
  597. rc = virtio_mem_notify_going_online(vm, mb_id);
  598. break;
  599. case MEM_OFFLINE:
  600. virtio_mem_notify_offline(vm, mb_id);
  601. vm->hotplug_active = false;
  602. mutex_unlock(&vm->hotplug_mutex);
  603. break;
  604. case MEM_ONLINE:
  605. virtio_mem_notify_online(vm, mb_id);
  606. vm->hotplug_active = false;
  607. mutex_unlock(&vm->hotplug_mutex);
  608. break;
  609. case MEM_CANCEL_OFFLINE:
  610. if (!vm->hotplug_active)
  611. break;
  612. virtio_mem_notify_cancel_offline(vm, mb_id);
  613. vm->hotplug_active = false;
  614. mutex_unlock(&vm->hotplug_mutex);
  615. break;
  616. case MEM_CANCEL_ONLINE:
  617. if (!vm->hotplug_active)
  618. break;
  619. vm->hotplug_active = false;
  620. mutex_unlock(&vm->hotplug_mutex);
  621. break;
  622. default:
  623. break;
  624. }
  625. lockdep_on();
  626. return rc;
  627. }
  628. /*
  629. * Set a range of pages PG_offline. Remember pages that were never onlined
  630. * (via generic_online_page()) using PageDirty().
  631. */
  632. static void virtio_mem_set_fake_offline(unsigned long pfn,
  633. unsigned int nr_pages, bool onlined)
  634. {
  635. for (; nr_pages--; pfn++) {
  636. struct page *page = pfn_to_page(pfn);
  637. __SetPageOffline(page);
  638. if (!onlined) {
  639. SetPageDirty(page);
  640. /* FIXME: remove after cleanups */
  641. ClearPageReserved(page);
  642. }
  643. }
  644. }
  645. /*
  646. * Clear PG_offline from a range of pages. If the pages were never onlined,
  647. * (via generic_online_page()), clear PageDirty().
  648. */
  649. static void virtio_mem_clear_fake_offline(unsigned long pfn,
  650. unsigned int nr_pages, bool onlined)
  651. {
  652. for (; nr_pages--; pfn++) {
  653. struct page *page = pfn_to_page(pfn);
  654. __ClearPageOffline(page);
  655. if (!onlined)
  656. ClearPageDirty(page);
  657. }
  658. }
  659. /*
  660. * Release a range of fake-offline pages to the buddy, effectively
  661. * fake-onlining them.
  662. */
  663. static void virtio_mem_fake_online(unsigned long pfn, unsigned int nr_pages)
  664. {
  665. const int order = MAX_ORDER - 1;
  666. int i;
  667. /*
  668. * We are always called with subblock granularity, which is at least
  669. * aligned to MAX_ORDER - 1.
  670. */
  671. for (i = 0; i < nr_pages; i += 1 << order) {
  672. struct page *page = pfn_to_page(pfn + i);
  673. /*
  674. * If the page is PageDirty(), it was kept fake-offline when
  675. * onlining the memory block. Otherwise, it was allocated
  676. * using alloc_contig_range(). All pages in a subblock are
  677. * alike.
  678. */
  679. if (PageDirty(page)) {
  680. virtio_mem_clear_fake_offline(pfn + i, 1 << order,
  681. false);
  682. generic_online_page(page, order);
  683. } else {
  684. virtio_mem_clear_fake_offline(pfn + i, 1 << order,
  685. true);
  686. free_contig_range(pfn + i, 1 << order);
  687. adjust_managed_page_count(page, 1 << order);
  688. }
  689. }
  690. }
  691. static void virtio_mem_online_page_cb(struct page *page, unsigned int order)
  692. {
  693. const unsigned long addr = page_to_phys(page);
  694. const unsigned long mb_id = virtio_mem_phys_to_mb_id(addr);
  695. struct virtio_mem *vm;
  696. int sb_id;
  697. /*
  698. * We exploit here that subblocks have at least MAX_ORDER - 1
  699. * size/alignment and that this callback is is called with such a
  700. * size/alignment. So we cannot cross subblocks and therefore
  701. * also not memory blocks.
  702. */
  703. rcu_read_lock();
  704. list_for_each_entry_rcu(vm, &virtio_mem_devices, next) {
  705. if (!virtio_mem_owned_mb(vm, mb_id))
  706. continue;
  707. sb_id = virtio_mem_phys_to_sb_id(vm, addr);
  708. /*
  709. * If plugged, online the pages, otherwise, set them fake
  710. * offline (PageOffline).
  711. */
  712. if (virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))
  713. generic_online_page(page, order);
  714. else
  715. virtio_mem_set_fake_offline(PFN_DOWN(addr), 1 << order,
  716. false);
  717. rcu_read_unlock();
  718. return;
  719. }
  720. rcu_read_unlock();
  721. /* not virtio-mem memory, but e.g., a DIMM. online it */
  722. generic_online_page(page, order);
  723. }
  724. static uint64_t virtio_mem_send_request(struct virtio_mem *vm,
  725. const struct virtio_mem_req *req)
  726. {
  727. struct scatterlist *sgs[2], sg_req, sg_resp;
  728. unsigned int len;
  729. int rc;
  730. /* don't use the request residing on the stack (vaddr) */
  731. vm->req = *req;
  732. /* out: buffer for request */
  733. sg_init_one(&sg_req, &vm->req, sizeof(vm->req));
  734. sgs[0] = &sg_req;
  735. /* in: buffer for response */
  736. sg_init_one(&sg_resp, &vm->resp, sizeof(vm->resp));
  737. sgs[1] = &sg_resp;
  738. rc = virtqueue_add_sgs(vm->vq, sgs, 1, 1, vm, GFP_KERNEL);
  739. if (rc < 0)
  740. return rc;
  741. virtqueue_kick(vm->vq);
  742. /* wait for a response */
  743. wait_event(vm->host_resp, virtqueue_get_buf(vm->vq, &len));
  744. return virtio16_to_cpu(vm->vdev, vm->resp.type);
  745. }
  746. static int virtio_mem_send_plug_request(struct virtio_mem *vm, uint64_t addr,
  747. uint64_t size)
  748. {
  749. const uint64_t nb_vm_blocks = size / vm->device_block_size;
  750. const struct virtio_mem_req req = {
  751. .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_PLUG),
  752. .u.plug.addr = cpu_to_virtio64(vm->vdev, addr),
  753. .u.plug.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks),
  754. };
  755. if (atomic_read(&vm->config_changed))
  756. return -EAGAIN;
  757. switch (virtio_mem_send_request(vm, &req)) {
  758. case VIRTIO_MEM_RESP_ACK:
  759. vm->plugged_size += size;
  760. return 0;
  761. case VIRTIO_MEM_RESP_NACK:
  762. return -EAGAIN;
  763. case VIRTIO_MEM_RESP_BUSY:
  764. return -ETXTBSY;
  765. case VIRTIO_MEM_RESP_ERROR:
  766. return -EINVAL;
  767. default:
  768. return -ENOMEM;
  769. }
  770. }
  771. static int virtio_mem_send_unplug_request(struct virtio_mem *vm, uint64_t addr,
  772. uint64_t size)
  773. {
  774. const uint64_t nb_vm_blocks = size / vm->device_block_size;
  775. const struct virtio_mem_req req = {
  776. .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_UNPLUG),
  777. .u.unplug.addr = cpu_to_virtio64(vm->vdev, addr),
  778. .u.unplug.nb_blocks = cpu_to_virtio16(vm->vdev, nb_vm_blocks),
  779. };
  780. if (atomic_read(&vm->config_changed))
  781. return -EAGAIN;
  782. switch (virtio_mem_send_request(vm, &req)) {
  783. case VIRTIO_MEM_RESP_ACK:
  784. vm->plugged_size -= size;
  785. return 0;
  786. case VIRTIO_MEM_RESP_BUSY:
  787. return -ETXTBSY;
  788. case VIRTIO_MEM_RESP_ERROR:
  789. return -EINVAL;
  790. default:
  791. return -ENOMEM;
  792. }
  793. }
  794. static int virtio_mem_send_unplug_all_request(struct virtio_mem *vm)
  795. {
  796. const struct virtio_mem_req req = {
  797. .type = cpu_to_virtio16(vm->vdev, VIRTIO_MEM_REQ_UNPLUG_ALL),
  798. };
  799. switch (virtio_mem_send_request(vm, &req)) {
  800. case VIRTIO_MEM_RESP_ACK:
  801. vm->unplug_all_required = false;
  802. vm->plugged_size = 0;
  803. /* usable region might have shrunk */
  804. atomic_set(&vm->config_changed, 1);
  805. return 0;
  806. case VIRTIO_MEM_RESP_BUSY:
  807. return -ETXTBSY;
  808. default:
  809. return -ENOMEM;
  810. }
  811. }
  812. /*
  813. * Plug selected subblocks. Updates the plugged state, but not the state
  814. * of the memory block.
  815. */
  816. static int virtio_mem_mb_plug_sb(struct virtio_mem *vm, unsigned long mb_id,
  817. int sb_id, int count)
  818. {
  819. const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id) +
  820. sb_id * vm->subblock_size;
  821. const uint64_t size = count * vm->subblock_size;
  822. int rc;
  823. dev_dbg(&vm->vdev->dev, "plugging memory block: %lu : %i - %i\n", mb_id,
  824. sb_id, sb_id + count - 1);
  825. rc = virtio_mem_send_plug_request(vm, addr, size);
  826. if (!rc)
  827. virtio_mem_mb_set_sb_plugged(vm, mb_id, sb_id, count);
  828. return rc;
  829. }
  830. /*
  831. * Unplug selected subblocks. Updates the plugged state, but not the state
  832. * of the memory block.
  833. */
  834. static int virtio_mem_mb_unplug_sb(struct virtio_mem *vm, unsigned long mb_id,
  835. int sb_id, int count)
  836. {
  837. const uint64_t addr = virtio_mem_mb_id_to_phys(mb_id) +
  838. sb_id * vm->subblock_size;
  839. const uint64_t size = count * vm->subblock_size;
  840. int rc;
  841. dev_dbg(&vm->vdev->dev, "unplugging memory block: %lu : %i - %i\n",
  842. mb_id, sb_id, sb_id + count - 1);
  843. rc = virtio_mem_send_unplug_request(vm, addr, size);
  844. if (!rc)
  845. virtio_mem_mb_set_sb_unplugged(vm, mb_id, sb_id, count);
  846. return rc;
  847. }
  848. /*
  849. * Unplug the desired number of plugged subblocks of a offline or not-added
  850. * memory block. Will fail if any subblock cannot get unplugged (instead of
  851. * skipping it).
  852. *
  853. * Will not modify the state of the memory block.
  854. *
  855. * Note: can fail after some subblocks were unplugged.
  856. */
  857. static int virtio_mem_mb_unplug_any_sb(struct virtio_mem *vm,
  858. unsigned long mb_id, uint64_t *nb_sb)
  859. {
  860. int sb_id, count;
  861. int rc;
  862. sb_id = vm->nb_sb_per_mb - 1;
  863. while (*nb_sb) {
  864. /* Find the next candidate subblock */
  865. while (sb_id >= 0 &&
  866. virtio_mem_mb_test_sb_unplugged(vm, mb_id, sb_id, 1))
  867. sb_id--;
  868. if (sb_id < 0)
  869. break;
  870. /* Try to unplug multiple subblocks at a time */
  871. count = 1;
  872. while (count < *nb_sb && sb_id > 0 &&
  873. virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id - 1, 1)) {
  874. count++;
  875. sb_id--;
  876. }
  877. rc = virtio_mem_mb_unplug_sb(vm, mb_id, sb_id, count);
  878. if (rc)
  879. return rc;
  880. *nb_sb -= count;
  881. sb_id--;
  882. }
  883. return 0;
  884. }
  885. /*
  886. * Unplug all plugged subblocks of an offline or not-added memory block.
  887. *
  888. * Will not modify the state of the memory block.
  889. *
  890. * Note: can fail after some subblocks were unplugged.
  891. */
  892. static int virtio_mem_mb_unplug(struct virtio_mem *vm, unsigned long mb_id)
  893. {
  894. uint64_t nb_sb = vm->nb_sb_per_mb;
  895. return virtio_mem_mb_unplug_any_sb(vm, mb_id, &nb_sb);
  896. }
  897. /*
  898. * Prepare tracking data for the next memory block.
  899. */
  900. static int virtio_mem_prepare_next_mb(struct virtio_mem *vm,
  901. unsigned long *mb_id)
  902. {
  903. int rc;
  904. if (vm->next_mb_id > vm->last_usable_mb_id)
  905. return -ENOSPC;
  906. /* Resize the state array if required. */
  907. rc = virtio_mem_mb_state_prepare_next_mb(vm);
  908. if (rc)
  909. return rc;
  910. /* Resize the subblock bitmap if required. */
  911. rc = virtio_mem_sb_bitmap_prepare_next_mb(vm);
  912. if (rc)
  913. return rc;
  914. vm->nb_mb_state[VIRTIO_MEM_MB_STATE_UNUSED]++;
  915. *mb_id = vm->next_mb_id++;
  916. return 0;
  917. }
  918. /*
  919. * Don't add too many blocks that are not onlined yet to avoid running OOM.
  920. */
  921. static bool virtio_mem_too_many_mb_offline(struct virtio_mem *vm)
  922. {
  923. unsigned long nb_offline;
  924. nb_offline = vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE] +
  925. vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL];
  926. return nb_offline >= VIRTIO_MEM_NB_OFFLINE_THRESHOLD;
  927. }
  928. /*
  929. * Try to plug the desired number of subblocks and add the memory block
  930. * to Linux.
  931. *
  932. * Will modify the state of the memory block.
  933. */
  934. static int virtio_mem_mb_plug_and_add(struct virtio_mem *vm,
  935. unsigned long mb_id,
  936. uint64_t *nb_sb)
  937. {
  938. const int count = min_t(int, *nb_sb, vm->nb_sb_per_mb);
  939. int rc, rc2;
  940. if (WARN_ON_ONCE(!count))
  941. return -EINVAL;
  942. /*
  943. * Plug the requested number of subblocks before adding it to linux,
  944. * so that onlining will directly online all plugged subblocks.
  945. */
  946. rc = virtio_mem_mb_plug_sb(vm, mb_id, 0, count);
  947. if (rc)
  948. return rc;
  949. /*
  950. * Mark the block properly offline before adding it to Linux,
  951. * so the memory notifiers will find the block in the right state.
  952. */
  953. if (count == vm->nb_sb_per_mb)
  954. virtio_mem_mb_set_state(vm, mb_id,
  955. VIRTIO_MEM_MB_STATE_OFFLINE);
  956. else
  957. virtio_mem_mb_set_state(vm, mb_id,
  958. VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL);
  959. /* Add the memory block to linux - if that fails, try to unplug. */
  960. rc = virtio_mem_mb_add(vm, mb_id);
  961. if (rc) {
  962. enum virtio_mem_mb_state new_state = VIRTIO_MEM_MB_STATE_UNUSED;
  963. dev_err(&vm->vdev->dev,
  964. "adding memory block %lu failed with %d\n", mb_id, rc);
  965. rc2 = virtio_mem_mb_unplug_sb(vm, mb_id, 0, count);
  966. /*
  967. * TODO: Linux MM does not properly clean up yet in all cases
  968. * where adding of memory failed - especially on -ENOMEM.
  969. */
  970. if (rc2)
  971. new_state = VIRTIO_MEM_MB_STATE_PLUGGED;
  972. virtio_mem_mb_set_state(vm, mb_id, new_state);
  973. return rc;
  974. }
  975. *nb_sb -= count;
  976. return 0;
  977. }
  978. /*
  979. * Try to plug the desired number of subblocks of a memory block that
  980. * is already added to Linux.
  981. *
  982. * Will modify the state of the memory block.
  983. *
  984. * Note: Can fail after some subblocks were successfully plugged.
  985. */
  986. static int virtio_mem_mb_plug_any_sb(struct virtio_mem *vm, unsigned long mb_id,
  987. uint64_t *nb_sb, bool online)
  988. {
  989. unsigned long pfn, nr_pages;
  990. int sb_id, count;
  991. int rc;
  992. if (WARN_ON_ONCE(!*nb_sb))
  993. return -EINVAL;
  994. while (*nb_sb) {
  995. sb_id = virtio_mem_mb_first_unplugged_sb(vm, mb_id);
  996. if (sb_id >= vm->nb_sb_per_mb)
  997. break;
  998. count = 1;
  999. while (count < *nb_sb &&
  1000. sb_id + count < vm->nb_sb_per_mb &&
  1001. !virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id + count,
  1002. 1))
  1003. count++;
  1004. rc = virtio_mem_mb_plug_sb(vm, mb_id, sb_id, count);
  1005. if (rc)
  1006. return rc;
  1007. *nb_sb -= count;
  1008. if (!online)
  1009. continue;
  1010. /* fake-online the pages if the memory block is online */
  1011. pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
  1012. sb_id * vm->subblock_size);
  1013. nr_pages = PFN_DOWN(count * vm->subblock_size);
  1014. virtio_mem_fake_online(pfn, nr_pages);
  1015. }
  1016. if (virtio_mem_mb_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
  1017. if (online)
  1018. virtio_mem_mb_set_state(vm, mb_id,
  1019. VIRTIO_MEM_MB_STATE_ONLINE);
  1020. else
  1021. virtio_mem_mb_set_state(vm, mb_id,
  1022. VIRTIO_MEM_MB_STATE_OFFLINE);
  1023. }
  1024. return 0;
  1025. }
  1026. /*
  1027. * Try to plug the requested amount of memory.
  1028. */
  1029. static int virtio_mem_plug_request(struct virtio_mem *vm, uint64_t diff)
  1030. {
  1031. uint64_t nb_sb = diff / vm->subblock_size;
  1032. unsigned long mb_id;
  1033. int rc;
  1034. if (!nb_sb)
  1035. return 0;
  1036. /* Don't race with onlining/offlining */
  1037. mutex_lock(&vm->hotplug_mutex);
  1038. /* Try to plug subblocks of partially plugged online blocks. */
  1039. virtio_mem_for_each_mb_state(vm, mb_id,
  1040. VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL) {
  1041. rc = virtio_mem_mb_plug_any_sb(vm, mb_id, &nb_sb, true);
  1042. if (rc || !nb_sb)
  1043. goto out_unlock;
  1044. cond_resched();
  1045. }
  1046. /* Try to plug subblocks of partially plugged offline blocks. */
  1047. virtio_mem_for_each_mb_state(vm, mb_id,
  1048. VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL) {
  1049. rc = virtio_mem_mb_plug_any_sb(vm, mb_id, &nb_sb, false);
  1050. if (rc || !nb_sb)
  1051. goto out_unlock;
  1052. cond_resched();
  1053. }
  1054. /*
  1055. * We won't be working on online/offline memory blocks from this point,
  1056. * so we can't race with memory onlining/offlining. Drop the mutex.
  1057. */
  1058. mutex_unlock(&vm->hotplug_mutex);
  1059. /* Try to plug and add unused blocks */
  1060. virtio_mem_for_each_mb_state(vm, mb_id, VIRTIO_MEM_MB_STATE_UNUSED) {
  1061. if (virtio_mem_too_many_mb_offline(vm))
  1062. return -ENOSPC;
  1063. rc = virtio_mem_mb_plug_and_add(vm, mb_id, &nb_sb);
  1064. if (rc || !nb_sb)
  1065. return rc;
  1066. cond_resched();
  1067. }
  1068. /* Try to prepare, plug and add new blocks */
  1069. while (nb_sb) {
  1070. if (virtio_mem_too_many_mb_offline(vm))
  1071. return -ENOSPC;
  1072. rc = virtio_mem_prepare_next_mb(vm, &mb_id);
  1073. if (rc)
  1074. return rc;
  1075. rc = virtio_mem_mb_plug_and_add(vm, mb_id, &nb_sb);
  1076. if (rc)
  1077. return rc;
  1078. cond_resched();
  1079. }
  1080. return 0;
  1081. out_unlock:
  1082. mutex_unlock(&vm->hotplug_mutex);
  1083. return rc;
  1084. }
  1085. /*
  1086. * Unplug the desired number of plugged subblocks of an offline memory block.
  1087. * Will fail if any subblock cannot get unplugged (instead of skipping it).
  1088. *
  1089. * Will modify the state of the memory block. Might temporarily drop the
  1090. * hotplug_mutex.
  1091. *
  1092. * Note: Can fail after some subblocks were successfully unplugged.
  1093. */
  1094. static int virtio_mem_mb_unplug_any_sb_offline(struct virtio_mem *vm,
  1095. unsigned long mb_id,
  1096. uint64_t *nb_sb)
  1097. {
  1098. int rc;
  1099. rc = virtio_mem_mb_unplug_any_sb(vm, mb_id, nb_sb);
  1100. /* some subblocks might have been unplugged even on failure */
  1101. if (!virtio_mem_mb_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb))
  1102. virtio_mem_mb_set_state(vm, mb_id,
  1103. VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL);
  1104. if (rc)
  1105. return rc;
  1106. if (virtio_mem_mb_test_sb_unplugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
  1107. /*
  1108. * Remove the block from Linux - this should never fail.
  1109. * Hinder the block from getting onlined by marking it
  1110. * unplugged. Temporarily drop the mutex, so
  1111. * any pending GOING_ONLINE requests can be serviced/rejected.
  1112. */
  1113. virtio_mem_mb_set_state(vm, mb_id,
  1114. VIRTIO_MEM_MB_STATE_UNUSED);
  1115. mutex_unlock(&vm->hotplug_mutex);
  1116. rc = virtio_mem_mb_remove(vm, mb_id);
  1117. BUG_ON(rc);
  1118. mutex_lock(&vm->hotplug_mutex);
  1119. }
  1120. return 0;
  1121. }
  1122. /*
  1123. * Unplug the given plugged subblocks of an online memory block.
  1124. *
  1125. * Will modify the state of the memory block.
  1126. */
  1127. static int virtio_mem_mb_unplug_sb_online(struct virtio_mem *vm,
  1128. unsigned long mb_id, int sb_id,
  1129. int count)
  1130. {
  1131. const unsigned long nr_pages = PFN_DOWN(vm->subblock_size) * count;
  1132. unsigned long start_pfn;
  1133. int rc;
  1134. struct acr_info dummy;
  1135. start_pfn = PFN_DOWN(virtio_mem_mb_id_to_phys(mb_id) +
  1136. sb_id * vm->subblock_size);
  1137. rc = alloc_contig_range(start_pfn, start_pfn + nr_pages,
  1138. MIGRATE_MOVABLE, GFP_KERNEL, &dummy);
  1139. if (rc == -ENOMEM)
  1140. /* whoops, out of memory */
  1141. return rc;
  1142. if (rc)
  1143. return -EBUSY;
  1144. /* Mark it as fake-offline before unplugging it */
  1145. virtio_mem_set_fake_offline(start_pfn, nr_pages, true);
  1146. adjust_managed_page_count(pfn_to_page(start_pfn), -nr_pages);
  1147. /* Try to unplug the allocated memory */
  1148. rc = virtio_mem_mb_unplug_sb(vm, mb_id, sb_id, count);
  1149. if (rc) {
  1150. /* Return the memory to the buddy. */
  1151. virtio_mem_fake_online(start_pfn, nr_pages);
  1152. return rc;
  1153. }
  1154. virtio_mem_mb_set_state(vm, mb_id,
  1155. VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL);
  1156. return 0;
  1157. }
  1158. /*
  1159. * Unplug the desired number of plugged subblocks of an online memory block.
  1160. * Will skip subblock that are busy.
  1161. *
  1162. * Will modify the state of the memory block. Might temporarily drop the
  1163. * hotplug_mutex.
  1164. *
  1165. * Note: Can fail after some subblocks were successfully unplugged. Can
  1166. * return 0 even if subblocks were busy and could not get unplugged.
  1167. */
  1168. static int virtio_mem_mb_unplug_any_sb_online(struct virtio_mem *vm,
  1169. unsigned long mb_id,
  1170. uint64_t *nb_sb)
  1171. {
  1172. int rc, sb_id;
  1173. /* If possible, try to unplug the complete block in one shot. */
  1174. if (*nb_sb >= vm->nb_sb_per_mb &&
  1175. virtio_mem_mb_test_sb_plugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
  1176. rc = virtio_mem_mb_unplug_sb_online(vm, mb_id, 0,
  1177. vm->nb_sb_per_mb);
  1178. if (!rc) {
  1179. *nb_sb -= vm->nb_sb_per_mb;
  1180. goto unplugged;
  1181. } else if (rc != -EBUSY)
  1182. return rc;
  1183. }
  1184. /* Fallback to single subblocks. */
  1185. for (sb_id = vm->nb_sb_per_mb - 1; sb_id >= 0 && *nb_sb; sb_id--) {
  1186. /* Find the next candidate subblock */
  1187. while (sb_id >= 0 &&
  1188. !virtio_mem_mb_test_sb_plugged(vm, mb_id, sb_id, 1))
  1189. sb_id--;
  1190. if (sb_id < 0)
  1191. break;
  1192. rc = virtio_mem_mb_unplug_sb_online(vm, mb_id, sb_id, 1);
  1193. if (rc == -EBUSY)
  1194. continue;
  1195. else if (rc)
  1196. return rc;
  1197. *nb_sb -= 1;
  1198. }
  1199. unplugged:
  1200. /*
  1201. * Once all subblocks of a memory block were unplugged, offline and
  1202. * remove it. This will usually not fail, as no memory is in use
  1203. * anymore - however some other notifiers might NACK the request.
  1204. */
  1205. if (virtio_mem_mb_test_sb_unplugged(vm, mb_id, 0, vm->nb_sb_per_mb)) {
  1206. mutex_unlock(&vm->hotplug_mutex);
  1207. rc = virtio_mem_mb_offline_and_remove(vm, mb_id);
  1208. mutex_lock(&vm->hotplug_mutex);
  1209. if (!rc)
  1210. virtio_mem_mb_set_state(vm, mb_id,
  1211. VIRTIO_MEM_MB_STATE_UNUSED);
  1212. }
  1213. return 0;
  1214. }
  1215. /*
  1216. * Try to unplug the requested amount of memory.
  1217. */
  1218. static int virtio_mem_unplug_request(struct virtio_mem *vm, uint64_t diff)
  1219. {
  1220. uint64_t nb_sb = diff / vm->subblock_size;
  1221. unsigned long mb_id;
  1222. int rc;
  1223. if (!nb_sb)
  1224. return 0;
  1225. /*
  1226. * We'll drop the mutex a couple of times when it is safe to do so.
  1227. * This might result in some blocks switching the state (online/offline)
  1228. * and we could miss them in this run - we will retry again later.
  1229. */
  1230. mutex_lock(&vm->hotplug_mutex);
  1231. /* Try to unplug subblocks of partially plugged offline blocks. */
  1232. virtio_mem_for_each_mb_state_rev(vm, mb_id,
  1233. VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL) {
  1234. rc = virtio_mem_mb_unplug_any_sb_offline(vm, mb_id,
  1235. &nb_sb);
  1236. if (rc || !nb_sb)
  1237. goto out_unlock;
  1238. cond_resched();
  1239. }
  1240. /* Try to unplug subblocks of plugged offline blocks. */
  1241. virtio_mem_for_each_mb_state_rev(vm, mb_id,
  1242. VIRTIO_MEM_MB_STATE_OFFLINE) {
  1243. rc = virtio_mem_mb_unplug_any_sb_offline(vm, mb_id,
  1244. &nb_sb);
  1245. if (rc || !nb_sb)
  1246. goto out_unlock;
  1247. cond_resched();
  1248. }
  1249. if (!unplug_online) {
  1250. mutex_unlock(&vm->hotplug_mutex);
  1251. return 0;
  1252. }
  1253. /* Try to unplug subblocks of partially plugged online blocks. */
  1254. virtio_mem_for_each_mb_state_rev(vm, mb_id,
  1255. VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL) {
  1256. rc = virtio_mem_mb_unplug_any_sb_online(vm, mb_id,
  1257. &nb_sb);
  1258. if (rc || !nb_sb)
  1259. goto out_unlock;
  1260. mutex_unlock(&vm->hotplug_mutex);
  1261. cond_resched();
  1262. mutex_lock(&vm->hotplug_mutex);
  1263. }
  1264. /* Try to unplug subblocks of plugged online blocks. */
  1265. virtio_mem_for_each_mb_state_rev(vm, mb_id,
  1266. VIRTIO_MEM_MB_STATE_ONLINE) {
  1267. rc = virtio_mem_mb_unplug_any_sb_online(vm, mb_id,
  1268. &nb_sb);
  1269. if (rc || !nb_sb)
  1270. goto out_unlock;
  1271. mutex_unlock(&vm->hotplug_mutex);
  1272. cond_resched();
  1273. mutex_lock(&vm->hotplug_mutex);
  1274. }
  1275. mutex_unlock(&vm->hotplug_mutex);
  1276. return nb_sb ? -EBUSY : 0;
  1277. out_unlock:
  1278. mutex_unlock(&vm->hotplug_mutex);
  1279. return rc;
  1280. }
  1281. /*
  1282. * Try to unplug all blocks that couldn't be unplugged before, for example,
  1283. * because the hypervisor was busy.
  1284. */
  1285. static int virtio_mem_unplug_pending_mb(struct virtio_mem *vm)
  1286. {
  1287. unsigned long mb_id;
  1288. int rc;
  1289. virtio_mem_for_each_mb_state(vm, mb_id, VIRTIO_MEM_MB_STATE_PLUGGED) {
  1290. rc = virtio_mem_mb_unplug(vm, mb_id);
  1291. if (rc)
  1292. return rc;
  1293. virtio_mem_mb_set_state(vm, mb_id, VIRTIO_MEM_MB_STATE_UNUSED);
  1294. }
  1295. return 0;
  1296. }
  1297. /*
  1298. * Update all parts of the config that could have changed.
  1299. */
  1300. static void virtio_mem_refresh_config(struct virtio_mem *vm)
  1301. {
  1302. const uint64_t phys_limit = 1UL << MAX_PHYSMEM_BITS;
  1303. uint64_t new_plugged_size, usable_region_size, end_addr;
  1304. /* the plugged_size is just a reflection of what _we_ did previously */
  1305. virtio_cread_le(vm->vdev, struct virtio_mem_config, plugged_size,
  1306. &new_plugged_size);
  1307. if (WARN_ON_ONCE(new_plugged_size != vm->plugged_size))
  1308. vm->plugged_size = new_plugged_size;
  1309. /* calculate the last usable memory block id */
  1310. virtio_cread_le(vm->vdev, struct virtio_mem_config,
  1311. usable_region_size, &usable_region_size);
  1312. end_addr = vm->addr + usable_region_size;
  1313. end_addr = min(end_addr, phys_limit);
  1314. vm->last_usable_mb_id = virtio_mem_phys_to_mb_id(end_addr) - 1;
  1315. /* see if there is a request to change the size */
  1316. virtio_cread_le(vm->vdev, struct virtio_mem_config, requested_size,
  1317. &vm->requested_size);
  1318. dev_info(&vm->vdev->dev, "plugged size: 0x%llx", vm->plugged_size);
  1319. dev_info(&vm->vdev->dev, "requested size: 0x%llx", vm->requested_size);
  1320. }
  1321. /*
  1322. * Workqueue function for handling plug/unplug requests and config updates.
  1323. */
  1324. static void virtio_mem_run_wq(struct work_struct *work)
  1325. {
  1326. struct virtio_mem *vm = container_of(work, struct virtio_mem, wq);
  1327. uint64_t diff;
  1328. int rc;
  1329. hrtimer_cancel(&vm->retry_timer);
  1330. if (vm->broken)
  1331. return;
  1332. retry:
  1333. rc = 0;
  1334. /* Make sure we start with a clean state if there are leftovers. */
  1335. if (unlikely(vm->unplug_all_required))
  1336. rc = virtio_mem_send_unplug_all_request(vm);
  1337. if (atomic_read(&vm->config_changed)) {
  1338. atomic_set(&vm->config_changed, 0);
  1339. virtio_mem_refresh_config(vm);
  1340. }
  1341. /* Unplug any leftovers from previous runs */
  1342. if (!rc)
  1343. rc = virtio_mem_unplug_pending_mb(vm);
  1344. if (!rc && vm->requested_size != vm->plugged_size) {
  1345. if (vm->requested_size > vm->plugged_size) {
  1346. diff = vm->requested_size - vm->plugged_size;
  1347. rc = virtio_mem_plug_request(vm, diff);
  1348. } else {
  1349. diff = vm->plugged_size - vm->requested_size;
  1350. rc = virtio_mem_unplug_request(vm, diff);
  1351. }
  1352. }
  1353. switch (rc) {
  1354. case 0:
  1355. vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS;
  1356. break;
  1357. case -ENOSPC:
  1358. /*
  1359. * We cannot add any more memory (alignment, physical limit)
  1360. * or we have too many offline memory blocks.
  1361. */
  1362. break;
  1363. case -ETXTBSY:
  1364. /*
  1365. * The hypervisor cannot process our request right now
  1366. * (e.g., out of memory, migrating);
  1367. */
  1368. case -EBUSY:
  1369. /*
  1370. * We cannot free up any memory to unplug it (all plugged memory
  1371. * is busy).
  1372. */
  1373. case -ENOMEM:
  1374. /* Out of memory, try again later. */
  1375. hrtimer_start(&vm->retry_timer, ms_to_ktime(vm->retry_timer_ms),
  1376. HRTIMER_MODE_REL);
  1377. break;
  1378. case -EAGAIN:
  1379. /* Retry immediately (e.g., the config changed). */
  1380. goto retry;
  1381. default:
  1382. /* Unknown error, mark as broken */
  1383. dev_err(&vm->vdev->dev,
  1384. "unknown error, marking device broken: %d\n", rc);
  1385. vm->broken = true;
  1386. }
  1387. }
  1388. static enum hrtimer_restart virtio_mem_timer_expired(struct hrtimer *timer)
  1389. {
  1390. struct virtio_mem *vm = container_of(timer, struct virtio_mem,
  1391. retry_timer);
  1392. virtio_mem_retry(vm);
  1393. vm->retry_timer_ms = min_t(unsigned int, vm->retry_timer_ms * 2,
  1394. VIRTIO_MEM_RETRY_TIMER_MAX_MS);
  1395. return HRTIMER_NORESTART;
  1396. }
  1397. static void virtio_mem_handle_response(struct virtqueue *vq)
  1398. {
  1399. struct virtio_mem *vm = vq->vdev->priv;
  1400. wake_up(&vm->host_resp);
  1401. }
  1402. static int virtio_mem_init_vq(struct virtio_mem *vm)
  1403. {
  1404. struct virtqueue *vq;
  1405. vq = virtio_find_single_vq(vm->vdev, virtio_mem_handle_response,
  1406. "guest-request");
  1407. if (IS_ERR(vq))
  1408. return PTR_ERR(vq);
  1409. vm->vq = vq;
  1410. return 0;
  1411. }
  1412. static int virtio_mem_init(struct virtio_mem *vm)
  1413. {
  1414. const uint64_t phys_limit = 1UL << MAX_PHYSMEM_BITS;
  1415. uint16_t node_id;
  1416. if (!vm->vdev->config->get) {
  1417. dev_err(&vm->vdev->dev, "config access disabled\n");
  1418. return -EINVAL;
  1419. }
  1420. /*
  1421. * We don't want to (un)plug or reuse any memory when in kdump. The
  1422. * memory is still accessible (but not mapped).
  1423. */
  1424. if (is_kdump_kernel()) {
  1425. dev_warn(&vm->vdev->dev, "disabled in kdump kernel\n");
  1426. return -EBUSY;
  1427. }
  1428. /* Fetch all properties that can't change. */
  1429. virtio_cread_le(vm->vdev, struct virtio_mem_config, plugged_size,
  1430. &vm->plugged_size);
  1431. virtio_cread_le(vm->vdev, struct virtio_mem_config, block_size,
  1432. &vm->device_block_size);
  1433. virtio_cread_le(vm->vdev, struct virtio_mem_config, node_id,
  1434. &node_id);
  1435. vm->nid = virtio_mem_translate_node_id(vm, node_id);
  1436. virtio_cread_le(vm->vdev, struct virtio_mem_config, addr, &vm->addr);
  1437. virtio_cread_le(vm->vdev, struct virtio_mem_config, region_size,
  1438. &vm->region_size);
  1439. /*
  1440. * We always hotplug memory in memory block granularity. This way,
  1441. * we have to wait for exactly one memory block to online.
  1442. */
  1443. if (vm->device_block_size > memory_block_size_bytes()) {
  1444. dev_err(&vm->vdev->dev,
  1445. "The block size is not supported (too big).\n");
  1446. return -EINVAL;
  1447. }
  1448. /* bad device setup - warn only */
  1449. if (!IS_ALIGNED(vm->addr, memory_block_size_bytes()))
  1450. dev_warn(&vm->vdev->dev,
  1451. "The alignment of the physical start address can make some memory unusable.\n");
  1452. if (!IS_ALIGNED(vm->addr + vm->region_size, memory_block_size_bytes()))
  1453. dev_warn(&vm->vdev->dev,
  1454. "The alignment of the physical end address can make some memory unusable.\n");
  1455. if (vm->addr + vm->region_size > phys_limit)
  1456. dev_warn(&vm->vdev->dev,
  1457. "Some memory is not addressable. This can make some memory unusable.\n");
  1458. /*
  1459. * Calculate the subblock size:
  1460. * - At least MAX_ORDER - 1 / pageblock_order.
  1461. * - At least the device block size.
  1462. * In the worst case, a single subblock per memory block.
  1463. */
  1464. vm->subblock_size = PAGE_SIZE * 1ul << max_t(uint32_t, MAX_ORDER - 1,
  1465. pageblock_order);
  1466. vm->subblock_size = max_t(uint64_t, vm->device_block_size,
  1467. vm->subblock_size);
  1468. vm->nb_sb_per_mb = memory_block_size_bytes() / vm->subblock_size;
  1469. /* Round up to the next full memory block */
  1470. vm->first_mb_id = virtio_mem_phys_to_mb_id(vm->addr - 1 +
  1471. memory_block_size_bytes());
  1472. vm->next_mb_id = vm->first_mb_id;
  1473. vm->last_mb_id = virtio_mem_phys_to_mb_id(vm->addr +
  1474. vm->region_size) - 1;
  1475. dev_info(&vm->vdev->dev, "start address: 0x%llx", vm->addr);
  1476. dev_info(&vm->vdev->dev, "region size: 0x%llx", vm->region_size);
  1477. dev_info(&vm->vdev->dev, "device block size: 0x%llx",
  1478. (unsigned long long)vm->device_block_size);
  1479. dev_info(&vm->vdev->dev, "memory block size: 0x%lx",
  1480. memory_block_size_bytes());
  1481. dev_info(&vm->vdev->dev, "subblock size: 0x%llx",
  1482. (unsigned long long)vm->subblock_size);
  1483. if (vm->nid != NUMA_NO_NODE)
  1484. dev_info(&vm->vdev->dev, "nid: %d", vm->nid);
  1485. return 0;
  1486. }
  1487. static int virtio_mem_create_resource(struct virtio_mem *vm)
  1488. {
  1489. /*
  1490. * When force-unloading the driver and removing the device, we
  1491. * could have a garbage pointer. Duplicate the string.
  1492. */
  1493. const char *name = kstrdup(dev_name(&vm->vdev->dev), GFP_KERNEL);
  1494. if (!name)
  1495. return -ENOMEM;
  1496. vm->parent_resource = __request_mem_region(vm->addr, vm->region_size,
  1497. name, IORESOURCE_SYSTEM_RAM);
  1498. if (!vm->parent_resource) {
  1499. kfree(name);
  1500. dev_warn(&vm->vdev->dev, "could not reserve device region\n");
  1501. dev_info(&vm->vdev->dev,
  1502. "reloading the driver is not supported\n");
  1503. return -EBUSY;
  1504. }
  1505. /* The memory is not actually busy - make add_memory() work. */
  1506. vm->parent_resource->flags &= ~IORESOURCE_BUSY;
  1507. return 0;
  1508. }
  1509. static void virtio_mem_delete_resource(struct virtio_mem *vm)
  1510. {
  1511. const char *name;
  1512. if (!vm->parent_resource)
  1513. return;
  1514. name = vm->parent_resource->name;
  1515. release_resource(vm->parent_resource);
  1516. kfree(vm->parent_resource);
  1517. kfree(name);
  1518. vm->parent_resource = NULL;
  1519. }
  1520. static int virtio_mem_probe(struct virtio_device *vdev)
  1521. {
  1522. struct virtio_mem *vm;
  1523. int rc;
  1524. BUILD_BUG_ON(sizeof(struct virtio_mem_req) != 24);
  1525. BUILD_BUG_ON(sizeof(struct virtio_mem_resp) != 10);
  1526. vdev->priv = vm = kzalloc(sizeof(*vm), GFP_KERNEL);
  1527. if (!vm)
  1528. return -ENOMEM;
  1529. init_waitqueue_head(&vm->host_resp);
  1530. vm->vdev = vdev;
  1531. INIT_WORK(&vm->wq, virtio_mem_run_wq);
  1532. mutex_init(&vm->hotplug_mutex);
  1533. INIT_LIST_HEAD(&vm->next);
  1534. spin_lock_init(&vm->removal_lock);
  1535. hrtimer_init(&vm->retry_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  1536. vm->retry_timer.function = virtio_mem_timer_expired;
  1537. vm->retry_timer_ms = VIRTIO_MEM_RETRY_TIMER_MIN_MS;
  1538. /* register the virtqueue */
  1539. rc = virtio_mem_init_vq(vm);
  1540. if (rc)
  1541. goto out_free_vm;
  1542. /* initialize the device by querying the config */
  1543. rc = virtio_mem_init(vm);
  1544. if (rc)
  1545. goto out_del_vq;
  1546. /* create the parent resource for all memory */
  1547. rc = virtio_mem_create_resource(vm);
  1548. if (rc)
  1549. goto out_del_vq;
  1550. /*
  1551. * If we still have memory plugged, we have to unplug all memory first.
  1552. * Registering our parent resource makes sure that this memory isn't
  1553. * actually in use (e.g., trying to reload the driver).
  1554. */
  1555. if (vm->plugged_size) {
  1556. vm->unplug_all_required = 1;
  1557. dev_info(&vm->vdev->dev, "unplugging all memory is required\n");
  1558. }
  1559. /* register callbacks */
  1560. vm->memory_notifier.notifier_call = virtio_mem_memory_notifier_cb;
  1561. rc = register_memory_notifier(&vm->memory_notifier);
  1562. if (rc)
  1563. goto out_del_resource;
  1564. rc = register_virtio_mem_device(vm);
  1565. if (rc)
  1566. goto out_unreg_mem;
  1567. virtio_device_ready(vdev);
  1568. /* trigger a config update to start processing the requested_size */
  1569. atomic_set(&vm->config_changed, 1);
  1570. queue_work(system_freezable_wq, &vm->wq);
  1571. return 0;
  1572. out_unreg_mem:
  1573. unregister_memory_notifier(&vm->memory_notifier);
  1574. out_del_resource:
  1575. virtio_mem_delete_resource(vm);
  1576. out_del_vq:
  1577. vdev->config->del_vqs(vdev);
  1578. out_free_vm:
  1579. kfree(vm);
  1580. vdev->priv = NULL;
  1581. return rc;
  1582. }
  1583. static void virtio_mem_remove(struct virtio_device *vdev)
  1584. {
  1585. struct virtio_mem *vm = vdev->priv;
  1586. unsigned long mb_id;
  1587. int rc;
  1588. /*
  1589. * Make sure the workqueue won't be triggered anymore and no memory
  1590. * blocks can be onlined/offlined until we're finished here.
  1591. */
  1592. mutex_lock(&vm->hotplug_mutex);
  1593. spin_lock_irq(&vm->removal_lock);
  1594. vm->removing = true;
  1595. spin_unlock_irq(&vm->removal_lock);
  1596. mutex_unlock(&vm->hotplug_mutex);
  1597. /* wait until the workqueue stopped */
  1598. cancel_work_sync(&vm->wq);
  1599. hrtimer_cancel(&vm->retry_timer);
  1600. /*
  1601. * After we unregistered our callbacks, user space can online partially
  1602. * plugged offline blocks. Make sure to remove them.
  1603. */
  1604. virtio_mem_for_each_mb_state(vm, mb_id,
  1605. VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL) {
  1606. rc = virtio_mem_mb_remove(vm, mb_id);
  1607. BUG_ON(rc);
  1608. virtio_mem_mb_set_state(vm, mb_id, VIRTIO_MEM_MB_STATE_UNUSED);
  1609. }
  1610. /*
  1611. * After we unregistered our callbacks, user space can no longer
  1612. * offline partially plugged online memory blocks. No need to worry
  1613. * about them.
  1614. */
  1615. /* unregister callbacks */
  1616. unregister_virtio_mem_device(vm);
  1617. unregister_memory_notifier(&vm->memory_notifier);
  1618. /*
  1619. * There is no way we could reliably remove all memory we have added to
  1620. * the system. And there is no way to stop the driver/device from going
  1621. * away. Warn at least.
  1622. */
  1623. if (vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE] ||
  1624. vm->nb_mb_state[VIRTIO_MEM_MB_STATE_OFFLINE_PARTIAL] ||
  1625. vm->nb_mb_state[VIRTIO_MEM_MB_STATE_ONLINE] ||
  1626. vm->nb_mb_state[VIRTIO_MEM_MB_STATE_ONLINE_PARTIAL]) {
  1627. dev_warn(&vdev->dev, "device still has system memory added\n");
  1628. } else {
  1629. virtio_mem_delete_resource(vm);
  1630. kfree_const(vm->resource_name);
  1631. }
  1632. /* remove all tracking data - no locking needed */
  1633. vfree(vm->mb_state);
  1634. vfree(vm->sb_bitmap);
  1635. /* reset the device and cleanup the queues */
  1636. vdev->config->reset(vdev);
  1637. vdev->config->del_vqs(vdev);
  1638. kfree(vm);
  1639. vdev->priv = NULL;
  1640. }
  1641. static void virtio_mem_config_changed(struct virtio_device *vdev)
  1642. {
  1643. struct virtio_mem *vm = vdev->priv;
  1644. atomic_set(&vm->config_changed, 1);
  1645. virtio_mem_retry(vm);
  1646. }
  1647. #ifdef CONFIG_PM_SLEEP
  1648. static int virtio_mem_freeze(struct virtio_device *vdev)
  1649. {
  1650. /*
  1651. * When restarting the VM, all memory is usually unplugged. Don't
  1652. * allow to suspend/hibernate.
  1653. */
  1654. dev_err(&vdev->dev, "save/restore not supported.\n");
  1655. return -EPERM;
  1656. }
  1657. static int virtio_mem_restore(struct virtio_device *vdev)
  1658. {
  1659. return -EPERM;
  1660. }
  1661. #endif
  1662. static unsigned int virtio_mem_features[] = {
  1663. #if defined(CONFIG_NUMA) && defined(CONFIG_ACPI_NUMA)
  1664. VIRTIO_MEM_F_ACPI_PXM,
  1665. #endif
  1666. };
  1667. static const struct virtio_device_id virtio_mem_id_table[] = {
  1668. { VIRTIO_ID_MEM, VIRTIO_DEV_ANY_ID },
  1669. { 0 },
  1670. };
  1671. static struct virtio_driver virtio_mem_driver = {
  1672. .feature_table = virtio_mem_features,
  1673. .feature_table_size = ARRAY_SIZE(virtio_mem_features),
  1674. .driver.name = KBUILD_MODNAME,
  1675. .driver.owner = THIS_MODULE,
  1676. .id_table = virtio_mem_id_table,
  1677. .probe = virtio_mem_probe,
  1678. .remove = virtio_mem_remove,
  1679. .config_changed = virtio_mem_config_changed,
  1680. #ifdef CONFIG_PM_SLEEP
  1681. .freeze = virtio_mem_freeze,
  1682. .restore = virtio_mem_restore,
  1683. #endif
  1684. };
  1685. module_virtio_driver(virtio_mem_driver);
  1686. MODULE_DEVICE_TABLE(virtio, virtio_mem_id_table);
  1687. MODULE_AUTHOR("David Hildenbrand <david@redhat.com>");
  1688. MODULE_DESCRIPTION("Virtio-mem driver");
  1689. MODULE_LICENSE("GPL");