hv_balloon.c 45 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2012, Microsoft Corporation.
  4. *
  5. * Author:
  6. * K. Y. Srinivasan <kys@microsoft.com>
  7. */
  8. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  9. #include <linux/kernel.h>
  10. #include <linux/jiffies.h>
  11. #include <linux/mman.h>
  12. #include <linux/delay.h>
  13. #include <linux/init.h>
  14. #include <linux/module.h>
  15. #include <linux/slab.h>
  16. #include <linux/kthread.h>
  17. #include <linux/completion.h>
  18. #include <linux/memory_hotplug.h>
  19. #include <linux/memory.h>
  20. #include <linux/notifier.h>
  21. #include <linux/percpu_counter.h>
  22. #include <linux/hyperv.h>
  23. #include <asm/hyperv-tlfs.h>
  24. #include <asm/mshyperv.h>
  25. #define CREATE_TRACE_POINTS
  26. #include "hv_trace_balloon.h"
  27. /*
  28. * We begin with definitions supporting the Dynamic Memory protocol
  29. * with the host.
  30. *
  31. * Begin protocol definitions.
  32. */
  33. /*
  34. * Protocol versions. The low word is the minor version, the high word the major
  35. * version.
  36. *
  37. * History:
  38. * Initial version 1.0
  39. * Changed to 0.1 on 2009/03/25
  40. * Changes to 0.2 on 2009/05/14
  41. * Changes to 0.3 on 2009/12/03
  42. * Changed to 1.0 on 2011/04/05
  43. */
  44. #define DYNMEM_MAKE_VERSION(Major, Minor) ((__u32)(((Major) << 16) | (Minor)))
  45. #define DYNMEM_MAJOR_VERSION(Version) ((__u32)(Version) >> 16)
  46. #define DYNMEM_MINOR_VERSION(Version) ((__u32)(Version) & 0xff)
  47. enum {
  48. DYNMEM_PROTOCOL_VERSION_1 = DYNMEM_MAKE_VERSION(0, 3),
  49. DYNMEM_PROTOCOL_VERSION_2 = DYNMEM_MAKE_VERSION(1, 0),
  50. DYNMEM_PROTOCOL_VERSION_3 = DYNMEM_MAKE_VERSION(2, 0),
  51. DYNMEM_PROTOCOL_VERSION_WIN7 = DYNMEM_PROTOCOL_VERSION_1,
  52. DYNMEM_PROTOCOL_VERSION_WIN8 = DYNMEM_PROTOCOL_VERSION_2,
  53. DYNMEM_PROTOCOL_VERSION_WIN10 = DYNMEM_PROTOCOL_VERSION_3,
  54. DYNMEM_PROTOCOL_VERSION_CURRENT = DYNMEM_PROTOCOL_VERSION_WIN10
  55. };
  56. /*
  57. * Message Types
  58. */
  59. enum dm_message_type {
  60. /*
  61. * Version 0.3
  62. */
  63. DM_ERROR = 0,
  64. DM_VERSION_REQUEST = 1,
  65. DM_VERSION_RESPONSE = 2,
  66. DM_CAPABILITIES_REPORT = 3,
  67. DM_CAPABILITIES_RESPONSE = 4,
  68. DM_STATUS_REPORT = 5,
  69. DM_BALLOON_REQUEST = 6,
  70. DM_BALLOON_RESPONSE = 7,
  71. DM_UNBALLOON_REQUEST = 8,
  72. DM_UNBALLOON_RESPONSE = 9,
  73. DM_MEM_HOT_ADD_REQUEST = 10,
  74. DM_MEM_HOT_ADD_RESPONSE = 11,
  75. DM_VERSION_03_MAX = 11,
  76. /*
  77. * Version 1.0.
  78. */
  79. DM_INFO_MESSAGE = 12,
  80. DM_VERSION_1_MAX = 12
  81. };
  82. /*
  83. * Structures defining the dynamic memory management
  84. * protocol.
  85. */
  86. union dm_version {
  87. struct {
  88. __u16 minor_version;
  89. __u16 major_version;
  90. };
  91. __u32 version;
  92. } __packed;
  93. union dm_caps {
  94. struct {
  95. __u64 balloon:1;
  96. __u64 hot_add:1;
  97. /*
  98. * To support guests that may have alignment
  99. * limitations on hot-add, the guest can specify
  100. * its alignment requirements; a value of n
  101. * represents an alignment of 2^n in mega bytes.
  102. */
  103. __u64 hot_add_alignment:4;
  104. __u64 reservedz:58;
  105. } cap_bits;
  106. __u64 caps;
  107. } __packed;
  108. union dm_mem_page_range {
  109. struct {
  110. /*
  111. * The PFN number of the first page in the range.
  112. * 40 bits is the architectural limit of a PFN
  113. * number for AMD64.
  114. */
  115. __u64 start_page:40;
  116. /*
  117. * The number of pages in the range.
  118. */
  119. __u64 page_cnt:24;
  120. } finfo;
  121. __u64 page_range;
  122. } __packed;
  123. /*
  124. * The header for all dynamic memory messages:
  125. *
  126. * type: Type of the message.
  127. * size: Size of the message in bytes; including the header.
  128. * trans_id: The guest is responsible for manufacturing this ID.
  129. */
  130. struct dm_header {
  131. __u16 type;
  132. __u16 size;
  133. __u32 trans_id;
  134. } __packed;
  135. /*
  136. * A generic message format for dynamic memory.
  137. * Specific message formats are defined later in the file.
  138. */
  139. struct dm_message {
  140. struct dm_header hdr;
  141. __u8 data[]; /* enclosed message */
  142. } __packed;
  143. /*
  144. * Specific message types supporting the dynamic memory protocol.
  145. */
  146. /*
  147. * Version negotiation message. Sent from the guest to the host.
  148. * The guest is free to try different versions until the host
  149. * accepts the version.
  150. *
  151. * dm_version: The protocol version requested.
  152. * is_last_attempt: If TRUE, this is the last version guest will request.
  153. * reservedz: Reserved field, set to zero.
  154. */
  155. struct dm_version_request {
  156. struct dm_header hdr;
  157. union dm_version version;
  158. __u32 is_last_attempt:1;
  159. __u32 reservedz:31;
  160. } __packed;
  161. /*
  162. * Version response message; Host to Guest and indicates
  163. * if the host has accepted the version sent by the guest.
  164. *
  165. * is_accepted: If TRUE, host has accepted the version and the guest
  166. * should proceed to the next stage of the protocol. FALSE indicates that
  167. * guest should re-try with a different version.
  168. *
  169. * reservedz: Reserved field, set to zero.
  170. */
  171. struct dm_version_response {
  172. struct dm_header hdr;
  173. __u64 is_accepted:1;
  174. __u64 reservedz:63;
  175. } __packed;
  176. /*
  177. * Message reporting capabilities. This is sent from the guest to the
  178. * host.
  179. */
  180. struct dm_capabilities {
  181. struct dm_header hdr;
  182. union dm_caps caps;
  183. __u64 min_page_cnt;
  184. __u64 max_page_number;
  185. } __packed;
  186. /*
  187. * Response to the capabilities message. This is sent from the host to the
  188. * guest. This message notifies if the host has accepted the guest's
  189. * capabilities. If the host has not accepted, the guest must shutdown
  190. * the service.
  191. *
  192. * is_accepted: Indicates if the host has accepted guest's capabilities.
  193. * reservedz: Must be 0.
  194. */
  195. struct dm_capabilities_resp_msg {
  196. struct dm_header hdr;
  197. __u64 is_accepted:1;
  198. __u64 reservedz:63;
  199. } __packed;
  200. /*
  201. * This message is used to report memory pressure from the guest.
  202. * This message is not part of any transaction and there is no
  203. * response to this message.
  204. *
  205. * num_avail: Available memory in pages.
  206. * num_committed: Committed memory in pages.
  207. * page_file_size: The accumulated size of all page files
  208. * in the system in pages.
  209. * zero_free: The nunber of zero and free pages.
  210. * page_file_writes: The writes to the page file in pages.
  211. * io_diff: An indicator of file cache efficiency or page file activity,
  212. * calculated as File Cache Page Fault Count - Page Read Count.
  213. * This value is in pages.
  214. *
  215. * Some of these metrics are Windows specific and fortunately
  216. * the algorithm on the host side that computes the guest memory
  217. * pressure only uses num_committed value.
  218. */
  219. struct dm_status {
  220. struct dm_header hdr;
  221. __u64 num_avail;
  222. __u64 num_committed;
  223. __u64 page_file_size;
  224. __u64 zero_free;
  225. __u32 page_file_writes;
  226. __u32 io_diff;
  227. } __packed;
  228. /*
  229. * Message to ask the guest to allocate memory - balloon up message.
  230. * This message is sent from the host to the guest. The guest may not be
  231. * able to allocate as much memory as requested.
  232. *
  233. * num_pages: number of pages to allocate.
  234. */
  235. struct dm_balloon {
  236. struct dm_header hdr;
  237. __u32 num_pages;
  238. __u32 reservedz;
  239. } __packed;
  240. /*
  241. * Balloon response message; this message is sent from the guest
  242. * to the host in response to the balloon message.
  243. *
  244. * reservedz: Reserved; must be set to zero.
  245. * more_pages: If FALSE, this is the last message of the transaction.
  246. * if TRUE there will atleast one more message from the guest.
  247. *
  248. * range_count: The number of ranges in the range array.
  249. *
  250. * range_array: An array of page ranges returned to the host.
  251. *
  252. */
  253. struct dm_balloon_response {
  254. struct dm_header hdr;
  255. __u32 reservedz;
  256. __u32 more_pages:1;
  257. __u32 range_count:31;
  258. union dm_mem_page_range range_array[];
  259. } __packed;
  260. /*
  261. * Un-balloon message; this message is sent from the host
  262. * to the guest to give guest more memory.
  263. *
  264. * more_pages: If FALSE, this is the last message of the transaction.
  265. * if TRUE there will atleast one more message from the guest.
  266. *
  267. * reservedz: Reserved; must be set to zero.
  268. *
  269. * range_count: The number of ranges in the range array.
  270. *
  271. * range_array: An array of page ranges returned to the host.
  272. *
  273. */
  274. struct dm_unballoon_request {
  275. struct dm_header hdr;
  276. __u32 more_pages:1;
  277. __u32 reservedz:31;
  278. __u32 range_count;
  279. union dm_mem_page_range range_array[];
  280. } __packed;
  281. /*
  282. * Un-balloon response message; this message is sent from the guest
  283. * to the host in response to an unballoon request.
  284. *
  285. */
  286. struct dm_unballoon_response {
  287. struct dm_header hdr;
  288. } __packed;
  289. /*
  290. * Hot add request message. Message sent from the host to the guest.
  291. *
  292. * mem_range: Memory range to hot add.
  293. *
  294. */
  295. struct dm_hot_add {
  296. struct dm_header hdr;
  297. union dm_mem_page_range range;
  298. } __packed;
  299. /*
  300. * Hot add response message.
  301. * This message is sent by the guest to report the status of a hot add request.
  302. * If page_count is less than the requested page count, then the host should
  303. * assume all further hot add requests will fail, since this indicates that
  304. * the guest has hit an upper physical memory barrier.
  305. *
  306. * Hot adds may also fail due to low resources; in this case, the guest must
  307. * not complete this message until the hot add can succeed, and the host must
  308. * not send a new hot add request until the response is sent.
  309. * If VSC fails to hot add memory DYNMEM_NUMBER_OF_UNSUCCESSFUL_HOTADD_ATTEMPTS
  310. * times it fails the request.
  311. *
  312. *
  313. * page_count: number of pages that were successfully hot added.
  314. *
  315. * result: result of the operation 1: success, 0: failure.
  316. *
  317. */
  318. struct dm_hot_add_response {
  319. struct dm_header hdr;
  320. __u32 page_count;
  321. __u32 result;
  322. } __packed;
  323. /*
  324. * Types of information sent from host to the guest.
  325. */
  326. enum dm_info_type {
  327. INFO_TYPE_MAX_PAGE_CNT = 0,
  328. MAX_INFO_TYPE
  329. };
  330. /*
  331. * Header for the information message.
  332. */
  333. struct dm_info_header {
  334. enum dm_info_type type;
  335. __u32 data_size;
  336. } __packed;
  337. /*
  338. * This message is sent from the host to the guest to pass
  339. * some relevant information (win8 addition).
  340. *
  341. * reserved: no used.
  342. * info_size: size of the information blob.
  343. * info: information blob.
  344. */
  345. struct dm_info_msg {
  346. struct dm_header hdr;
  347. __u32 reserved;
  348. __u32 info_size;
  349. __u8 info[];
  350. };
  351. /*
  352. * End protocol definitions.
  353. */
  354. /*
  355. * State to manage hot adding memory into the guest.
  356. * The range start_pfn : end_pfn specifies the range
  357. * that the host has asked us to hot add. The range
  358. * start_pfn : ha_end_pfn specifies the range that we have
  359. * currently hot added. We hot add in multiples of 128M
  360. * chunks; it is possible that we may not be able to bring
  361. * online all the pages in the region. The range
  362. * covered_start_pfn:covered_end_pfn defines the pages that can
  363. * be brough online.
  364. */
  365. struct hv_hotadd_state {
  366. struct list_head list;
  367. unsigned long start_pfn;
  368. unsigned long covered_start_pfn;
  369. unsigned long covered_end_pfn;
  370. unsigned long ha_end_pfn;
  371. unsigned long end_pfn;
  372. /*
  373. * A list of gaps.
  374. */
  375. struct list_head gap_list;
  376. };
  377. struct hv_hotadd_gap {
  378. struct list_head list;
  379. unsigned long start_pfn;
  380. unsigned long end_pfn;
  381. };
  382. struct balloon_state {
  383. __u32 num_pages;
  384. struct work_struct wrk;
  385. };
  386. struct hot_add_wrk {
  387. union dm_mem_page_range ha_page_range;
  388. union dm_mem_page_range ha_region_range;
  389. struct work_struct wrk;
  390. };
  391. static bool allow_hibernation;
  392. static bool hot_add = true;
  393. static bool do_hot_add;
  394. /*
  395. * Delay reporting memory pressure by
  396. * the specified number of seconds.
  397. */
  398. static uint pressure_report_delay = 45;
  399. /*
  400. * The last time we posted a pressure report to host.
  401. */
  402. static unsigned long last_post_time;
  403. module_param(hot_add, bool, (S_IRUGO | S_IWUSR));
  404. MODULE_PARM_DESC(hot_add, "If set attempt memory hot_add");
  405. module_param(pressure_report_delay, uint, (S_IRUGO | S_IWUSR));
  406. MODULE_PARM_DESC(pressure_report_delay, "Delay in secs in reporting pressure");
  407. static atomic_t trans_id = ATOMIC_INIT(0);
  408. static int dm_ring_size = 20 * 1024;
  409. /*
  410. * Driver specific state.
  411. */
  412. enum hv_dm_state {
  413. DM_INITIALIZING = 0,
  414. DM_INITIALIZED,
  415. DM_BALLOON_UP,
  416. DM_BALLOON_DOWN,
  417. DM_HOT_ADD,
  418. DM_INIT_ERROR
  419. };
  420. static __u8 recv_buffer[HV_HYP_PAGE_SIZE];
  421. static __u8 balloon_up_send_buffer[HV_HYP_PAGE_SIZE];
  422. #define PAGES_IN_2M (2 * 1024 * 1024 / PAGE_SIZE)
  423. #define HA_CHUNK (128 * 1024 * 1024 / PAGE_SIZE)
  424. struct hv_dynmem_device {
  425. struct hv_device *dev;
  426. enum hv_dm_state state;
  427. struct completion host_event;
  428. struct completion config_event;
  429. /*
  430. * Number of pages we have currently ballooned out.
  431. */
  432. unsigned int num_pages_ballooned;
  433. unsigned int num_pages_onlined;
  434. unsigned int num_pages_added;
  435. /*
  436. * State to manage the ballooning (up) operation.
  437. */
  438. struct balloon_state balloon_wrk;
  439. /*
  440. * State to execute the "hot-add" operation.
  441. */
  442. struct hot_add_wrk ha_wrk;
  443. /*
  444. * This state tracks if the host has specified a hot-add
  445. * region.
  446. */
  447. bool host_specified_ha_region;
  448. /*
  449. * State to synchronize hot-add.
  450. */
  451. struct completion ol_waitevent;
  452. /*
  453. * This thread handles hot-add
  454. * requests from the host as well as notifying
  455. * the host with regards to memory pressure in
  456. * the guest.
  457. */
  458. struct task_struct *thread;
  459. /*
  460. * Protects ha_region_list, num_pages_onlined counter and individual
  461. * regions from ha_region_list.
  462. */
  463. spinlock_t ha_lock;
  464. /*
  465. * A list of hot-add regions.
  466. */
  467. struct list_head ha_region_list;
  468. /*
  469. * We start with the highest version we can support
  470. * and downgrade based on the host; we save here the
  471. * next version to try.
  472. */
  473. __u32 next_version;
  474. /*
  475. * The negotiated version agreed by host.
  476. */
  477. __u32 version;
  478. };
  479. static struct hv_dynmem_device dm_device;
  480. static void post_status(struct hv_dynmem_device *dm);
  481. #ifdef CONFIG_MEMORY_HOTPLUG
  482. static inline bool has_pfn_is_backed(struct hv_hotadd_state *has,
  483. unsigned long pfn)
  484. {
  485. struct hv_hotadd_gap *gap;
  486. /* The page is not backed. */
  487. if ((pfn < has->covered_start_pfn) || (pfn >= has->covered_end_pfn))
  488. return false;
  489. /* Check for gaps. */
  490. list_for_each_entry(gap, &has->gap_list, list) {
  491. if ((pfn >= gap->start_pfn) && (pfn < gap->end_pfn))
  492. return false;
  493. }
  494. return true;
  495. }
  496. static unsigned long hv_page_offline_check(unsigned long start_pfn,
  497. unsigned long nr_pages)
  498. {
  499. unsigned long pfn = start_pfn, count = 0;
  500. struct hv_hotadd_state *has;
  501. bool found;
  502. while (pfn < start_pfn + nr_pages) {
  503. /*
  504. * Search for HAS which covers the pfn and when we find one
  505. * count how many consequitive PFNs are covered.
  506. */
  507. found = false;
  508. list_for_each_entry(has, &dm_device.ha_region_list, list) {
  509. while ((pfn >= has->start_pfn) &&
  510. (pfn < has->end_pfn) &&
  511. (pfn < start_pfn + nr_pages)) {
  512. found = true;
  513. if (has_pfn_is_backed(has, pfn))
  514. count++;
  515. pfn++;
  516. }
  517. }
  518. /*
  519. * This PFN is not in any HAS (e.g. we're offlining a region
  520. * which was present at boot), no need to account for it. Go
  521. * to the next one.
  522. */
  523. if (!found)
  524. pfn++;
  525. }
  526. return count;
  527. }
  528. static int hv_memory_notifier(struct notifier_block *nb, unsigned long val,
  529. void *v)
  530. {
  531. struct memory_notify *mem = (struct memory_notify *)v;
  532. unsigned long flags, pfn_count;
  533. switch (val) {
  534. case MEM_ONLINE:
  535. case MEM_CANCEL_ONLINE:
  536. complete(&dm_device.ol_waitevent);
  537. break;
  538. case MEM_OFFLINE:
  539. spin_lock_irqsave(&dm_device.ha_lock, flags);
  540. pfn_count = hv_page_offline_check(mem->start_pfn,
  541. mem->nr_pages);
  542. if (pfn_count <= dm_device.num_pages_onlined) {
  543. dm_device.num_pages_onlined -= pfn_count;
  544. } else {
  545. /*
  546. * We're offlining more pages than we managed to online.
  547. * This is unexpected. In any case don't let
  548. * num_pages_onlined wrap around zero.
  549. */
  550. WARN_ON_ONCE(1);
  551. dm_device.num_pages_onlined = 0;
  552. }
  553. spin_unlock_irqrestore(&dm_device.ha_lock, flags);
  554. break;
  555. case MEM_GOING_ONLINE:
  556. case MEM_GOING_OFFLINE:
  557. case MEM_CANCEL_OFFLINE:
  558. break;
  559. }
  560. return NOTIFY_OK;
  561. }
  562. static struct notifier_block hv_memory_nb = {
  563. .notifier_call = hv_memory_notifier,
  564. .priority = 0
  565. };
  566. /* Check if the particular page is backed and can be onlined and online it. */
  567. static void hv_page_online_one(struct hv_hotadd_state *has, struct page *pg)
  568. {
  569. if (!has_pfn_is_backed(has, page_to_pfn(pg))) {
  570. if (!PageOffline(pg))
  571. __SetPageOffline(pg);
  572. return;
  573. }
  574. if (PageOffline(pg))
  575. __ClearPageOffline(pg);
  576. /* This frame is currently backed; online the page. */
  577. generic_online_page(pg, 0);
  578. lockdep_assert_held(&dm_device.ha_lock);
  579. dm_device.num_pages_onlined++;
  580. }
  581. static void hv_bring_pgs_online(struct hv_hotadd_state *has,
  582. unsigned long start_pfn, unsigned long size)
  583. {
  584. int i;
  585. pr_debug("Online %lu pages starting at pfn 0x%lx\n", size, start_pfn);
  586. for (i = 0; i < size; i++)
  587. hv_page_online_one(has, pfn_to_page(start_pfn + i));
  588. }
  589. static void hv_mem_hot_add(unsigned long start, unsigned long size,
  590. unsigned long pfn_count,
  591. struct hv_hotadd_state *has)
  592. {
  593. int ret = 0;
  594. int i, nid;
  595. unsigned long start_pfn;
  596. unsigned long processed_pfn;
  597. unsigned long total_pfn = pfn_count;
  598. unsigned long flags;
  599. for (i = 0; i < (size/HA_CHUNK); i++) {
  600. start_pfn = start + (i * HA_CHUNK);
  601. spin_lock_irqsave(&dm_device.ha_lock, flags);
  602. has->ha_end_pfn += HA_CHUNK;
  603. if (total_pfn > HA_CHUNK) {
  604. processed_pfn = HA_CHUNK;
  605. total_pfn -= HA_CHUNK;
  606. } else {
  607. processed_pfn = total_pfn;
  608. total_pfn = 0;
  609. }
  610. has->covered_end_pfn += processed_pfn;
  611. spin_unlock_irqrestore(&dm_device.ha_lock, flags);
  612. reinit_completion(&dm_device.ol_waitevent);
  613. nid = memory_add_physaddr_to_nid(PFN_PHYS(start_pfn));
  614. ret = add_memory(nid, PFN_PHYS((start_pfn)),
  615. (HA_CHUNK << PAGE_SHIFT), MEMHP_MERGE_RESOURCE);
  616. if (ret) {
  617. pr_err("hot_add memory failed error is %d\n", ret);
  618. if (ret == -EEXIST) {
  619. /*
  620. * This error indicates that the error
  621. * is not a transient failure. This is the
  622. * case where the guest's physical address map
  623. * precludes hot adding memory. Stop all further
  624. * memory hot-add.
  625. */
  626. do_hot_add = false;
  627. }
  628. spin_lock_irqsave(&dm_device.ha_lock, flags);
  629. has->ha_end_pfn -= HA_CHUNK;
  630. has->covered_end_pfn -= processed_pfn;
  631. spin_unlock_irqrestore(&dm_device.ha_lock, flags);
  632. break;
  633. }
  634. /*
  635. * Wait for memory to get onlined. If the kernel onlined the
  636. * memory when adding it, this will return directly. Otherwise,
  637. * it will wait for user space to online the memory. This helps
  638. * to avoid adding memory faster than it is getting onlined. As
  639. * adding succeeded, it is ok to proceed even if the memory was
  640. * not onlined in time.
  641. */
  642. wait_for_completion_timeout(&dm_device.ol_waitevent, 5 * HZ);
  643. post_status(&dm_device);
  644. }
  645. }
  646. static void hv_online_page(struct page *pg, unsigned int order)
  647. {
  648. struct hv_hotadd_state *has;
  649. unsigned long flags;
  650. unsigned long pfn = page_to_pfn(pg);
  651. spin_lock_irqsave(&dm_device.ha_lock, flags);
  652. list_for_each_entry(has, &dm_device.ha_region_list, list) {
  653. /* The page belongs to a different HAS. */
  654. if ((pfn < has->start_pfn) ||
  655. (pfn + (1UL << order) > has->end_pfn))
  656. continue;
  657. hv_bring_pgs_online(has, pfn, 1UL << order);
  658. break;
  659. }
  660. spin_unlock_irqrestore(&dm_device.ha_lock, flags);
  661. }
  662. static int pfn_covered(unsigned long start_pfn, unsigned long pfn_cnt)
  663. {
  664. struct hv_hotadd_state *has;
  665. struct hv_hotadd_gap *gap;
  666. unsigned long residual, new_inc;
  667. int ret = 0;
  668. unsigned long flags;
  669. spin_lock_irqsave(&dm_device.ha_lock, flags);
  670. list_for_each_entry(has, &dm_device.ha_region_list, list) {
  671. /*
  672. * If the pfn range we are dealing with is not in the current
  673. * "hot add block", move on.
  674. */
  675. if (start_pfn < has->start_pfn || start_pfn >= has->end_pfn)
  676. continue;
  677. /*
  678. * If the current start pfn is not where the covered_end
  679. * is, create a gap and update covered_end_pfn.
  680. */
  681. if (has->covered_end_pfn != start_pfn) {
  682. gap = kzalloc(sizeof(struct hv_hotadd_gap), GFP_ATOMIC);
  683. if (!gap) {
  684. ret = -ENOMEM;
  685. break;
  686. }
  687. INIT_LIST_HEAD(&gap->list);
  688. gap->start_pfn = has->covered_end_pfn;
  689. gap->end_pfn = start_pfn;
  690. list_add_tail(&gap->list, &has->gap_list);
  691. has->covered_end_pfn = start_pfn;
  692. }
  693. /*
  694. * If the current hot add-request extends beyond
  695. * our current limit; extend it.
  696. */
  697. if ((start_pfn + pfn_cnt) > has->end_pfn) {
  698. residual = (start_pfn + pfn_cnt - has->end_pfn);
  699. /*
  700. * Extend the region by multiples of HA_CHUNK.
  701. */
  702. new_inc = (residual / HA_CHUNK) * HA_CHUNK;
  703. if (residual % HA_CHUNK)
  704. new_inc += HA_CHUNK;
  705. has->end_pfn += new_inc;
  706. }
  707. ret = 1;
  708. break;
  709. }
  710. spin_unlock_irqrestore(&dm_device.ha_lock, flags);
  711. return ret;
  712. }
  713. static unsigned long handle_pg_range(unsigned long pg_start,
  714. unsigned long pg_count)
  715. {
  716. unsigned long start_pfn = pg_start;
  717. unsigned long pfn_cnt = pg_count;
  718. unsigned long size;
  719. struct hv_hotadd_state *has;
  720. unsigned long pgs_ol = 0;
  721. unsigned long old_covered_state;
  722. unsigned long res = 0, flags;
  723. pr_debug("Hot adding %lu pages starting at pfn 0x%lx.\n", pg_count,
  724. pg_start);
  725. spin_lock_irqsave(&dm_device.ha_lock, flags);
  726. list_for_each_entry(has, &dm_device.ha_region_list, list) {
  727. /*
  728. * If the pfn range we are dealing with is not in the current
  729. * "hot add block", move on.
  730. */
  731. if (start_pfn < has->start_pfn || start_pfn >= has->end_pfn)
  732. continue;
  733. old_covered_state = has->covered_end_pfn;
  734. if (start_pfn < has->ha_end_pfn) {
  735. /*
  736. * This is the case where we are backing pages
  737. * in an already hot added region. Bring
  738. * these pages online first.
  739. */
  740. pgs_ol = has->ha_end_pfn - start_pfn;
  741. if (pgs_ol > pfn_cnt)
  742. pgs_ol = pfn_cnt;
  743. has->covered_end_pfn += pgs_ol;
  744. pfn_cnt -= pgs_ol;
  745. /*
  746. * Check if the corresponding memory block is already
  747. * online. It is possible to observe struct pages still
  748. * being uninitialized here so check section instead.
  749. * In case the section is online we need to bring the
  750. * rest of pfns (which were not backed previously)
  751. * online too.
  752. */
  753. if (start_pfn > has->start_pfn &&
  754. online_section_nr(pfn_to_section_nr(start_pfn)))
  755. hv_bring_pgs_online(has, start_pfn, pgs_ol);
  756. }
  757. if ((has->ha_end_pfn < has->end_pfn) && (pfn_cnt > 0)) {
  758. /*
  759. * We have some residual hot add range
  760. * that needs to be hot added; hot add
  761. * it now. Hot add a multiple of
  762. * of HA_CHUNK that fully covers the pages
  763. * we have.
  764. */
  765. size = (has->end_pfn - has->ha_end_pfn);
  766. if (pfn_cnt <= size) {
  767. size = ((pfn_cnt / HA_CHUNK) * HA_CHUNK);
  768. if (pfn_cnt % HA_CHUNK)
  769. size += HA_CHUNK;
  770. } else {
  771. pfn_cnt = size;
  772. }
  773. spin_unlock_irqrestore(&dm_device.ha_lock, flags);
  774. hv_mem_hot_add(has->ha_end_pfn, size, pfn_cnt, has);
  775. spin_lock_irqsave(&dm_device.ha_lock, flags);
  776. }
  777. /*
  778. * If we managed to online any pages that were given to us,
  779. * we declare success.
  780. */
  781. res = has->covered_end_pfn - old_covered_state;
  782. break;
  783. }
  784. spin_unlock_irqrestore(&dm_device.ha_lock, flags);
  785. return res;
  786. }
  787. static unsigned long process_hot_add(unsigned long pg_start,
  788. unsigned long pfn_cnt,
  789. unsigned long rg_start,
  790. unsigned long rg_size)
  791. {
  792. struct hv_hotadd_state *ha_region = NULL;
  793. int covered;
  794. unsigned long flags;
  795. if (pfn_cnt == 0)
  796. return 0;
  797. if (!dm_device.host_specified_ha_region) {
  798. covered = pfn_covered(pg_start, pfn_cnt);
  799. if (covered < 0)
  800. return 0;
  801. if (covered)
  802. goto do_pg_range;
  803. }
  804. /*
  805. * If the host has specified a hot-add range; deal with it first.
  806. */
  807. if (rg_size != 0) {
  808. ha_region = kzalloc(sizeof(struct hv_hotadd_state), GFP_KERNEL);
  809. if (!ha_region)
  810. return 0;
  811. INIT_LIST_HEAD(&ha_region->list);
  812. INIT_LIST_HEAD(&ha_region->gap_list);
  813. ha_region->start_pfn = rg_start;
  814. ha_region->ha_end_pfn = rg_start;
  815. ha_region->covered_start_pfn = pg_start;
  816. ha_region->covered_end_pfn = pg_start;
  817. ha_region->end_pfn = rg_start + rg_size;
  818. spin_lock_irqsave(&dm_device.ha_lock, flags);
  819. list_add_tail(&ha_region->list, &dm_device.ha_region_list);
  820. spin_unlock_irqrestore(&dm_device.ha_lock, flags);
  821. }
  822. do_pg_range:
  823. /*
  824. * Process the page range specified; bringing them
  825. * online if possible.
  826. */
  827. return handle_pg_range(pg_start, pfn_cnt);
  828. }
  829. #endif
  830. static void hot_add_req(struct work_struct *dummy)
  831. {
  832. struct dm_hot_add_response resp;
  833. #ifdef CONFIG_MEMORY_HOTPLUG
  834. unsigned long pg_start, pfn_cnt;
  835. unsigned long rg_start, rg_sz;
  836. #endif
  837. struct hv_dynmem_device *dm = &dm_device;
  838. memset(&resp, 0, sizeof(struct dm_hot_add_response));
  839. resp.hdr.type = DM_MEM_HOT_ADD_RESPONSE;
  840. resp.hdr.size = sizeof(struct dm_hot_add_response);
  841. #ifdef CONFIG_MEMORY_HOTPLUG
  842. pg_start = dm->ha_wrk.ha_page_range.finfo.start_page;
  843. pfn_cnt = dm->ha_wrk.ha_page_range.finfo.page_cnt;
  844. rg_start = dm->ha_wrk.ha_region_range.finfo.start_page;
  845. rg_sz = dm->ha_wrk.ha_region_range.finfo.page_cnt;
  846. if ((rg_start == 0) && (!dm->host_specified_ha_region)) {
  847. unsigned long region_size;
  848. unsigned long region_start;
  849. /*
  850. * The host has not specified the hot-add region.
  851. * Based on the hot-add page range being specified,
  852. * compute a hot-add region that can cover the pages
  853. * that need to be hot-added while ensuring the alignment
  854. * and size requirements of Linux as it relates to hot-add.
  855. */
  856. region_start = pg_start;
  857. region_size = (pfn_cnt / HA_CHUNK) * HA_CHUNK;
  858. if (pfn_cnt % HA_CHUNK)
  859. region_size += HA_CHUNK;
  860. region_start = (pg_start / HA_CHUNK) * HA_CHUNK;
  861. rg_start = region_start;
  862. rg_sz = region_size;
  863. }
  864. if (do_hot_add)
  865. resp.page_count = process_hot_add(pg_start, pfn_cnt,
  866. rg_start, rg_sz);
  867. dm->num_pages_added += resp.page_count;
  868. #endif
  869. /*
  870. * The result field of the response structure has the
  871. * following semantics:
  872. *
  873. * 1. If all or some pages hot-added: Guest should return success.
  874. *
  875. * 2. If no pages could be hot-added:
  876. *
  877. * If the guest returns success, then the host
  878. * will not attempt any further hot-add operations. This
  879. * signifies a permanent failure.
  880. *
  881. * If the guest returns failure, then this failure will be
  882. * treated as a transient failure and the host may retry the
  883. * hot-add operation after some delay.
  884. */
  885. if (resp.page_count > 0)
  886. resp.result = 1;
  887. else if (!do_hot_add)
  888. resp.result = 1;
  889. else
  890. resp.result = 0;
  891. if (!do_hot_add || resp.page_count == 0) {
  892. if (!allow_hibernation)
  893. pr_err("Memory hot add failed\n");
  894. else
  895. pr_info("Ignore hot-add request!\n");
  896. }
  897. dm->state = DM_INITIALIZED;
  898. resp.hdr.trans_id = atomic_inc_return(&trans_id);
  899. vmbus_sendpacket(dm->dev->channel, &resp,
  900. sizeof(struct dm_hot_add_response),
  901. (unsigned long)NULL,
  902. VM_PKT_DATA_INBAND, 0);
  903. }
  904. static void process_info(struct hv_dynmem_device *dm, struct dm_info_msg *msg)
  905. {
  906. struct dm_info_header *info_hdr;
  907. info_hdr = (struct dm_info_header *)msg->info;
  908. switch (info_hdr->type) {
  909. case INFO_TYPE_MAX_PAGE_CNT:
  910. if (info_hdr->data_size == sizeof(__u64)) {
  911. __u64 *max_page_count = (__u64 *)&info_hdr[1];
  912. pr_info("Max. dynamic memory size: %llu MB\n",
  913. (*max_page_count) >> (20 - HV_HYP_PAGE_SHIFT));
  914. }
  915. break;
  916. default:
  917. pr_warn("Received Unknown type: %d\n", info_hdr->type);
  918. }
  919. }
  920. static unsigned long compute_balloon_floor(void)
  921. {
  922. unsigned long min_pages;
  923. unsigned long nr_pages = totalram_pages();
  924. #define MB2PAGES(mb) ((mb) << (20 - PAGE_SHIFT))
  925. /* Simple continuous piecewiese linear function:
  926. * max MiB -> min MiB gradient
  927. * 0 0
  928. * 16 16
  929. * 32 24
  930. * 128 72 (1/2)
  931. * 512 168 (1/4)
  932. * 2048 360 (1/8)
  933. * 8192 744 (1/16)
  934. * 32768 1512 (1/32)
  935. */
  936. if (nr_pages < MB2PAGES(128))
  937. min_pages = MB2PAGES(8) + (nr_pages >> 1);
  938. else if (nr_pages < MB2PAGES(512))
  939. min_pages = MB2PAGES(40) + (nr_pages >> 2);
  940. else if (nr_pages < MB2PAGES(2048))
  941. min_pages = MB2PAGES(104) + (nr_pages >> 3);
  942. else if (nr_pages < MB2PAGES(8192))
  943. min_pages = MB2PAGES(232) + (nr_pages >> 4);
  944. else
  945. min_pages = MB2PAGES(488) + (nr_pages >> 5);
  946. #undef MB2PAGES
  947. return min_pages;
  948. }
  949. /*
  950. * Post our status as it relates memory pressure to the
  951. * host. Host expects the guests to post this status
  952. * periodically at 1 second intervals.
  953. *
  954. * The metrics specified in this protocol are very Windows
  955. * specific and so we cook up numbers here to convey our memory
  956. * pressure.
  957. */
  958. static void post_status(struct hv_dynmem_device *dm)
  959. {
  960. struct dm_status status;
  961. unsigned long now = jiffies;
  962. unsigned long last_post = last_post_time;
  963. if (pressure_report_delay > 0) {
  964. --pressure_report_delay;
  965. return;
  966. }
  967. if (!time_after(now, (last_post_time + HZ)))
  968. return;
  969. memset(&status, 0, sizeof(struct dm_status));
  970. status.hdr.type = DM_STATUS_REPORT;
  971. status.hdr.size = sizeof(struct dm_status);
  972. status.hdr.trans_id = atomic_inc_return(&trans_id);
  973. /*
  974. * The host expects the guest to report free and committed memory.
  975. * Furthermore, the host expects the pressure information to include
  976. * the ballooned out pages. For a given amount of memory that we are
  977. * managing we need to compute a floor below which we should not
  978. * balloon. Compute this and add it to the pressure report.
  979. * We also need to report all offline pages (num_pages_added -
  980. * num_pages_onlined) as committed to the host, otherwise it can try
  981. * asking us to balloon them out.
  982. */
  983. status.num_avail = si_mem_available();
  984. status.num_committed = vm_memory_committed() +
  985. dm->num_pages_ballooned +
  986. (dm->num_pages_added > dm->num_pages_onlined ?
  987. dm->num_pages_added - dm->num_pages_onlined : 0) +
  988. compute_balloon_floor();
  989. trace_balloon_status(status.num_avail, status.num_committed,
  990. vm_memory_committed(), dm->num_pages_ballooned,
  991. dm->num_pages_added, dm->num_pages_onlined);
  992. /*
  993. * If our transaction ID is no longer current, just don't
  994. * send the status. This can happen if we were interrupted
  995. * after we picked our transaction ID.
  996. */
  997. if (status.hdr.trans_id != atomic_read(&trans_id))
  998. return;
  999. /*
  1000. * If the last post time that we sampled has changed,
  1001. * we have raced, don't post the status.
  1002. */
  1003. if (last_post != last_post_time)
  1004. return;
  1005. last_post_time = jiffies;
  1006. vmbus_sendpacket(dm->dev->channel, &status,
  1007. sizeof(struct dm_status),
  1008. (unsigned long)NULL,
  1009. VM_PKT_DATA_INBAND, 0);
  1010. }
  1011. static void free_balloon_pages(struct hv_dynmem_device *dm,
  1012. union dm_mem_page_range *range_array)
  1013. {
  1014. int num_pages = range_array->finfo.page_cnt;
  1015. __u64 start_frame = range_array->finfo.start_page;
  1016. struct page *pg;
  1017. int i;
  1018. for (i = 0; i < num_pages; i++) {
  1019. pg = pfn_to_page(i + start_frame);
  1020. __ClearPageOffline(pg);
  1021. __free_page(pg);
  1022. dm->num_pages_ballooned--;
  1023. }
  1024. }
  1025. static unsigned int alloc_balloon_pages(struct hv_dynmem_device *dm,
  1026. unsigned int num_pages,
  1027. struct dm_balloon_response *bl_resp,
  1028. int alloc_unit)
  1029. {
  1030. unsigned int i, j;
  1031. struct page *pg;
  1032. for (i = 0; i < num_pages / alloc_unit; i++) {
  1033. if (bl_resp->hdr.size + sizeof(union dm_mem_page_range) >
  1034. HV_HYP_PAGE_SIZE)
  1035. return i * alloc_unit;
  1036. /*
  1037. * We execute this code in a thread context. Furthermore,
  1038. * we don't want the kernel to try too hard.
  1039. */
  1040. pg = alloc_pages(GFP_HIGHUSER | __GFP_NORETRY |
  1041. __GFP_NOMEMALLOC | __GFP_NOWARN,
  1042. get_order(alloc_unit << PAGE_SHIFT));
  1043. if (!pg)
  1044. return i * alloc_unit;
  1045. dm->num_pages_ballooned += alloc_unit;
  1046. /*
  1047. * If we allocatted 2M pages; split them so we
  1048. * can free them in any order we get.
  1049. */
  1050. if (alloc_unit != 1)
  1051. split_page(pg, get_order(alloc_unit << PAGE_SHIFT));
  1052. /* mark all pages offline */
  1053. for (j = 0; j < (1 << get_order(alloc_unit << PAGE_SHIFT)); j++)
  1054. __SetPageOffline(pg + j);
  1055. bl_resp->range_count++;
  1056. bl_resp->range_array[i].finfo.start_page =
  1057. page_to_pfn(pg);
  1058. bl_resp->range_array[i].finfo.page_cnt = alloc_unit;
  1059. bl_resp->hdr.size += sizeof(union dm_mem_page_range);
  1060. }
  1061. return i * alloc_unit;
  1062. }
  1063. static void balloon_up(struct work_struct *dummy)
  1064. {
  1065. unsigned int num_pages = dm_device.balloon_wrk.num_pages;
  1066. unsigned int num_ballooned = 0;
  1067. struct dm_balloon_response *bl_resp;
  1068. int alloc_unit;
  1069. int ret;
  1070. bool done = false;
  1071. int i;
  1072. long avail_pages;
  1073. unsigned long floor;
  1074. /*
  1075. * We will attempt 2M allocations. However, if we fail to
  1076. * allocate 2M chunks, we will go back to PAGE_SIZE allocations.
  1077. */
  1078. alloc_unit = PAGES_IN_2M;
  1079. avail_pages = si_mem_available();
  1080. floor = compute_balloon_floor();
  1081. /* Refuse to balloon below the floor. */
  1082. if (avail_pages < num_pages || avail_pages - num_pages < floor) {
  1083. pr_info("Balloon request will be partially fulfilled. %s\n",
  1084. avail_pages < num_pages ? "Not enough memory." :
  1085. "Balloon floor reached.");
  1086. num_pages = avail_pages > floor ? (avail_pages - floor) : 0;
  1087. }
  1088. while (!done) {
  1089. memset(balloon_up_send_buffer, 0, HV_HYP_PAGE_SIZE);
  1090. bl_resp = (struct dm_balloon_response *)balloon_up_send_buffer;
  1091. bl_resp->hdr.type = DM_BALLOON_RESPONSE;
  1092. bl_resp->hdr.size = sizeof(struct dm_balloon_response);
  1093. bl_resp->more_pages = 1;
  1094. num_pages -= num_ballooned;
  1095. num_ballooned = alloc_balloon_pages(&dm_device, num_pages,
  1096. bl_resp, alloc_unit);
  1097. if (alloc_unit != 1 && num_ballooned == 0) {
  1098. alloc_unit = 1;
  1099. continue;
  1100. }
  1101. if (num_ballooned == 0 || num_ballooned == num_pages) {
  1102. pr_debug("Ballooned %u out of %u requested pages.\n",
  1103. num_pages, dm_device.balloon_wrk.num_pages);
  1104. bl_resp->more_pages = 0;
  1105. done = true;
  1106. dm_device.state = DM_INITIALIZED;
  1107. }
  1108. /*
  1109. * We are pushing a lot of data through the channel;
  1110. * deal with transient failures caused because of the
  1111. * lack of space in the ring buffer.
  1112. */
  1113. do {
  1114. bl_resp->hdr.trans_id = atomic_inc_return(&trans_id);
  1115. ret = vmbus_sendpacket(dm_device.dev->channel,
  1116. bl_resp,
  1117. bl_resp->hdr.size,
  1118. (unsigned long)NULL,
  1119. VM_PKT_DATA_INBAND, 0);
  1120. if (ret == -EAGAIN)
  1121. msleep(20);
  1122. post_status(&dm_device);
  1123. } while (ret == -EAGAIN);
  1124. if (ret) {
  1125. /*
  1126. * Free up the memory we allocatted.
  1127. */
  1128. pr_err("Balloon response failed\n");
  1129. for (i = 0; i < bl_resp->range_count; i++)
  1130. free_balloon_pages(&dm_device,
  1131. &bl_resp->range_array[i]);
  1132. done = true;
  1133. }
  1134. }
  1135. }
  1136. static void balloon_down(struct hv_dynmem_device *dm,
  1137. struct dm_unballoon_request *req)
  1138. {
  1139. union dm_mem_page_range *range_array = req->range_array;
  1140. int range_count = req->range_count;
  1141. struct dm_unballoon_response resp;
  1142. int i;
  1143. unsigned int prev_pages_ballooned = dm->num_pages_ballooned;
  1144. for (i = 0; i < range_count; i++) {
  1145. free_balloon_pages(dm, &range_array[i]);
  1146. complete(&dm_device.config_event);
  1147. }
  1148. pr_debug("Freed %u ballooned pages.\n",
  1149. prev_pages_ballooned - dm->num_pages_ballooned);
  1150. if (req->more_pages == 1)
  1151. return;
  1152. memset(&resp, 0, sizeof(struct dm_unballoon_response));
  1153. resp.hdr.type = DM_UNBALLOON_RESPONSE;
  1154. resp.hdr.trans_id = atomic_inc_return(&trans_id);
  1155. resp.hdr.size = sizeof(struct dm_unballoon_response);
  1156. vmbus_sendpacket(dm_device.dev->channel, &resp,
  1157. sizeof(struct dm_unballoon_response),
  1158. (unsigned long)NULL,
  1159. VM_PKT_DATA_INBAND, 0);
  1160. dm->state = DM_INITIALIZED;
  1161. }
  1162. static void balloon_onchannelcallback(void *context);
  1163. static int dm_thread_func(void *dm_dev)
  1164. {
  1165. struct hv_dynmem_device *dm = dm_dev;
  1166. while (!kthread_should_stop()) {
  1167. wait_for_completion_interruptible_timeout(
  1168. &dm_device.config_event, 1*HZ);
  1169. /*
  1170. * The host expects us to post information on the memory
  1171. * pressure every second.
  1172. */
  1173. reinit_completion(&dm_device.config_event);
  1174. post_status(dm);
  1175. }
  1176. return 0;
  1177. }
  1178. static void version_resp(struct hv_dynmem_device *dm,
  1179. struct dm_version_response *vresp)
  1180. {
  1181. struct dm_version_request version_req;
  1182. int ret;
  1183. if (vresp->is_accepted) {
  1184. /*
  1185. * We are done; wakeup the
  1186. * context waiting for version
  1187. * negotiation.
  1188. */
  1189. complete(&dm->host_event);
  1190. return;
  1191. }
  1192. /*
  1193. * If there are more versions to try, continue
  1194. * with negotiations; if not
  1195. * shutdown the service since we are not able
  1196. * to negotiate a suitable version number
  1197. * with the host.
  1198. */
  1199. if (dm->next_version == 0)
  1200. goto version_error;
  1201. memset(&version_req, 0, sizeof(struct dm_version_request));
  1202. version_req.hdr.type = DM_VERSION_REQUEST;
  1203. version_req.hdr.size = sizeof(struct dm_version_request);
  1204. version_req.hdr.trans_id = atomic_inc_return(&trans_id);
  1205. version_req.version.version = dm->next_version;
  1206. dm->version = version_req.version.version;
  1207. /*
  1208. * Set the next version to try in case current version fails.
  1209. * Win7 protocol ought to be the last one to try.
  1210. */
  1211. switch (version_req.version.version) {
  1212. case DYNMEM_PROTOCOL_VERSION_WIN8:
  1213. dm->next_version = DYNMEM_PROTOCOL_VERSION_WIN7;
  1214. version_req.is_last_attempt = 0;
  1215. break;
  1216. default:
  1217. dm->next_version = 0;
  1218. version_req.is_last_attempt = 1;
  1219. }
  1220. ret = vmbus_sendpacket(dm->dev->channel, &version_req,
  1221. sizeof(struct dm_version_request),
  1222. (unsigned long)NULL,
  1223. VM_PKT_DATA_INBAND, 0);
  1224. if (ret)
  1225. goto version_error;
  1226. return;
  1227. version_error:
  1228. dm->state = DM_INIT_ERROR;
  1229. complete(&dm->host_event);
  1230. }
  1231. static void cap_resp(struct hv_dynmem_device *dm,
  1232. struct dm_capabilities_resp_msg *cap_resp)
  1233. {
  1234. if (!cap_resp->is_accepted) {
  1235. pr_err("Capabilities not accepted by host\n");
  1236. dm->state = DM_INIT_ERROR;
  1237. }
  1238. complete(&dm->host_event);
  1239. }
  1240. static void balloon_onchannelcallback(void *context)
  1241. {
  1242. struct hv_device *dev = context;
  1243. u32 recvlen;
  1244. u64 requestid;
  1245. struct dm_message *dm_msg;
  1246. struct dm_header *dm_hdr;
  1247. struct hv_dynmem_device *dm = hv_get_drvdata(dev);
  1248. struct dm_balloon *bal_msg;
  1249. struct dm_hot_add *ha_msg;
  1250. union dm_mem_page_range *ha_pg_range;
  1251. union dm_mem_page_range *ha_region;
  1252. memset(recv_buffer, 0, sizeof(recv_buffer));
  1253. vmbus_recvpacket(dev->channel, recv_buffer,
  1254. HV_HYP_PAGE_SIZE, &recvlen, &requestid);
  1255. if (recvlen > 0) {
  1256. dm_msg = (struct dm_message *)recv_buffer;
  1257. dm_hdr = &dm_msg->hdr;
  1258. switch (dm_hdr->type) {
  1259. case DM_VERSION_RESPONSE:
  1260. version_resp(dm,
  1261. (struct dm_version_response *)dm_msg);
  1262. break;
  1263. case DM_CAPABILITIES_RESPONSE:
  1264. cap_resp(dm,
  1265. (struct dm_capabilities_resp_msg *)dm_msg);
  1266. break;
  1267. case DM_BALLOON_REQUEST:
  1268. if (allow_hibernation) {
  1269. pr_info("Ignore balloon-up request!\n");
  1270. break;
  1271. }
  1272. if (dm->state == DM_BALLOON_UP)
  1273. pr_warn("Currently ballooning\n");
  1274. bal_msg = (struct dm_balloon *)recv_buffer;
  1275. dm->state = DM_BALLOON_UP;
  1276. dm_device.balloon_wrk.num_pages = bal_msg->num_pages;
  1277. schedule_work(&dm_device.balloon_wrk.wrk);
  1278. break;
  1279. case DM_UNBALLOON_REQUEST:
  1280. if (allow_hibernation) {
  1281. pr_info("Ignore balloon-down request!\n");
  1282. break;
  1283. }
  1284. dm->state = DM_BALLOON_DOWN;
  1285. balloon_down(dm,
  1286. (struct dm_unballoon_request *)recv_buffer);
  1287. break;
  1288. case DM_MEM_HOT_ADD_REQUEST:
  1289. if (dm->state == DM_HOT_ADD)
  1290. pr_warn("Currently hot-adding\n");
  1291. dm->state = DM_HOT_ADD;
  1292. ha_msg = (struct dm_hot_add *)recv_buffer;
  1293. if (ha_msg->hdr.size == sizeof(struct dm_hot_add)) {
  1294. /*
  1295. * This is a normal hot-add request specifying
  1296. * hot-add memory.
  1297. */
  1298. dm->host_specified_ha_region = false;
  1299. ha_pg_range = &ha_msg->range;
  1300. dm->ha_wrk.ha_page_range = *ha_pg_range;
  1301. dm->ha_wrk.ha_region_range.page_range = 0;
  1302. } else {
  1303. /*
  1304. * Host is specifying that we first hot-add
  1305. * a region and then partially populate this
  1306. * region.
  1307. */
  1308. dm->host_specified_ha_region = true;
  1309. ha_pg_range = &ha_msg->range;
  1310. ha_region = &ha_pg_range[1];
  1311. dm->ha_wrk.ha_page_range = *ha_pg_range;
  1312. dm->ha_wrk.ha_region_range = *ha_region;
  1313. }
  1314. schedule_work(&dm_device.ha_wrk.wrk);
  1315. break;
  1316. case DM_INFO_MESSAGE:
  1317. process_info(dm, (struct dm_info_msg *)dm_msg);
  1318. break;
  1319. default:
  1320. pr_warn_ratelimited("Unhandled message: type: %d\n", dm_hdr->type);
  1321. }
  1322. }
  1323. }
  1324. static int balloon_connect_vsp(struct hv_device *dev)
  1325. {
  1326. struct dm_version_request version_req;
  1327. struct dm_capabilities cap_msg;
  1328. unsigned long t;
  1329. int ret;
  1330. ret = vmbus_open(dev->channel, dm_ring_size, dm_ring_size, NULL, 0,
  1331. balloon_onchannelcallback, dev);
  1332. if (ret)
  1333. return ret;
  1334. /*
  1335. * Initiate the hand shake with the host and negotiate
  1336. * a version that the host can support. We start with the
  1337. * highest version number and go down if the host cannot
  1338. * support it.
  1339. */
  1340. memset(&version_req, 0, sizeof(struct dm_version_request));
  1341. version_req.hdr.type = DM_VERSION_REQUEST;
  1342. version_req.hdr.size = sizeof(struct dm_version_request);
  1343. version_req.hdr.trans_id = atomic_inc_return(&trans_id);
  1344. version_req.version.version = DYNMEM_PROTOCOL_VERSION_WIN10;
  1345. version_req.is_last_attempt = 0;
  1346. dm_device.version = version_req.version.version;
  1347. ret = vmbus_sendpacket(dev->channel, &version_req,
  1348. sizeof(struct dm_version_request),
  1349. (unsigned long)NULL, VM_PKT_DATA_INBAND, 0);
  1350. if (ret)
  1351. goto out;
  1352. t = wait_for_completion_timeout(&dm_device.host_event, 5*HZ);
  1353. if (t == 0) {
  1354. ret = -ETIMEDOUT;
  1355. goto out;
  1356. }
  1357. /*
  1358. * If we could not negotiate a compatible version with the host
  1359. * fail the probe function.
  1360. */
  1361. if (dm_device.state == DM_INIT_ERROR) {
  1362. ret = -EPROTO;
  1363. goto out;
  1364. }
  1365. pr_info("Using Dynamic Memory protocol version %u.%u\n",
  1366. DYNMEM_MAJOR_VERSION(dm_device.version),
  1367. DYNMEM_MINOR_VERSION(dm_device.version));
  1368. /*
  1369. * Now submit our capabilities to the host.
  1370. */
  1371. memset(&cap_msg, 0, sizeof(struct dm_capabilities));
  1372. cap_msg.hdr.type = DM_CAPABILITIES_REPORT;
  1373. cap_msg.hdr.size = sizeof(struct dm_capabilities);
  1374. cap_msg.hdr.trans_id = atomic_inc_return(&trans_id);
  1375. /*
  1376. * When hibernation (i.e. virtual ACPI S4 state) is enabled, the host
  1377. * currently still requires the bits to be set, so we have to add code
  1378. * to fail the host's hot-add and balloon up/down requests, if any.
  1379. */
  1380. cap_msg.caps.cap_bits.balloon = 1;
  1381. cap_msg.caps.cap_bits.hot_add = 1;
  1382. /*
  1383. * Specify our alignment requirements as it relates
  1384. * memory hot-add. Specify 128MB alignment.
  1385. */
  1386. cap_msg.caps.cap_bits.hot_add_alignment = 7;
  1387. /*
  1388. * Currently the host does not use these
  1389. * values and we set them to what is done in the
  1390. * Windows driver.
  1391. */
  1392. cap_msg.min_page_cnt = 0;
  1393. cap_msg.max_page_number = -1;
  1394. ret = vmbus_sendpacket(dev->channel, &cap_msg,
  1395. sizeof(struct dm_capabilities),
  1396. (unsigned long)NULL, VM_PKT_DATA_INBAND, 0);
  1397. if (ret)
  1398. goto out;
  1399. t = wait_for_completion_timeout(&dm_device.host_event, 5*HZ);
  1400. if (t == 0) {
  1401. ret = -ETIMEDOUT;
  1402. goto out;
  1403. }
  1404. /*
  1405. * If the host does not like our capabilities,
  1406. * fail the probe function.
  1407. */
  1408. if (dm_device.state == DM_INIT_ERROR) {
  1409. ret = -EPROTO;
  1410. goto out;
  1411. }
  1412. return 0;
  1413. out:
  1414. vmbus_close(dev->channel);
  1415. return ret;
  1416. }
  1417. static int balloon_probe(struct hv_device *dev,
  1418. const struct hv_vmbus_device_id *dev_id)
  1419. {
  1420. int ret;
  1421. allow_hibernation = hv_is_hibernation_supported();
  1422. if (allow_hibernation)
  1423. hot_add = false;
  1424. #ifdef CONFIG_MEMORY_HOTPLUG
  1425. do_hot_add = hot_add;
  1426. #else
  1427. do_hot_add = false;
  1428. #endif
  1429. dm_device.dev = dev;
  1430. dm_device.state = DM_INITIALIZING;
  1431. dm_device.next_version = DYNMEM_PROTOCOL_VERSION_WIN8;
  1432. init_completion(&dm_device.host_event);
  1433. init_completion(&dm_device.config_event);
  1434. INIT_LIST_HEAD(&dm_device.ha_region_list);
  1435. spin_lock_init(&dm_device.ha_lock);
  1436. INIT_WORK(&dm_device.balloon_wrk.wrk, balloon_up);
  1437. INIT_WORK(&dm_device.ha_wrk.wrk, hot_add_req);
  1438. dm_device.host_specified_ha_region = false;
  1439. #ifdef CONFIG_MEMORY_HOTPLUG
  1440. set_online_page_callback(&hv_online_page);
  1441. init_completion(&dm_device.ol_waitevent);
  1442. register_memory_notifier(&hv_memory_nb);
  1443. #endif
  1444. hv_set_drvdata(dev, &dm_device);
  1445. ret = balloon_connect_vsp(dev);
  1446. if (ret != 0)
  1447. return ret;
  1448. dm_device.state = DM_INITIALIZED;
  1449. dm_device.thread =
  1450. kthread_run(dm_thread_func, &dm_device, "hv_balloon");
  1451. if (IS_ERR(dm_device.thread)) {
  1452. ret = PTR_ERR(dm_device.thread);
  1453. goto probe_error;
  1454. }
  1455. return 0;
  1456. probe_error:
  1457. dm_device.state = DM_INIT_ERROR;
  1458. dm_device.thread = NULL;
  1459. vmbus_close(dev->channel);
  1460. #ifdef CONFIG_MEMORY_HOTPLUG
  1461. unregister_memory_notifier(&hv_memory_nb);
  1462. restore_online_page_callback(&hv_online_page);
  1463. #endif
  1464. return ret;
  1465. }
  1466. static int balloon_remove(struct hv_device *dev)
  1467. {
  1468. struct hv_dynmem_device *dm = hv_get_drvdata(dev);
  1469. struct hv_hotadd_state *has, *tmp;
  1470. struct hv_hotadd_gap *gap, *tmp_gap;
  1471. unsigned long flags;
  1472. if (dm->num_pages_ballooned != 0)
  1473. pr_warn("Ballooned pages: %d\n", dm->num_pages_ballooned);
  1474. cancel_work_sync(&dm->balloon_wrk.wrk);
  1475. cancel_work_sync(&dm->ha_wrk.wrk);
  1476. kthread_stop(dm->thread);
  1477. vmbus_close(dev->channel);
  1478. #ifdef CONFIG_MEMORY_HOTPLUG
  1479. unregister_memory_notifier(&hv_memory_nb);
  1480. restore_online_page_callback(&hv_online_page);
  1481. #endif
  1482. spin_lock_irqsave(&dm_device.ha_lock, flags);
  1483. list_for_each_entry_safe(has, tmp, &dm->ha_region_list, list) {
  1484. list_for_each_entry_safe(gap, tmp_gap, &has->gap_list, list) {
  1485. list_del(&gap->list);
  1486. kfree(gap);
  1487. }
  1488. list_del(&has->list);
  1489. kfree(has);
  1490. }
  1491. spin_unlock_irqrestore(&dm_device.ha_lock, flags);
  1492. return 0;
  1493. }
  1494. static int balloon_suspend(struct hv_device *hv_dev)
  1495. {
  1496. struct hv_dynmem_device *dm = hv_get_drvdata(hv_dev);
  1497. tasklet_disable(&hv_dev->channel->callback_event);
  1498. cancel_work_sync(&dm->balloon_wrk.wrk);
  1499. cancel_work_sync(&dm->ha_wrk.wrk);
  1500. if (dm->thread) {
  1501. kthread_stop(dm->thread);
  1502. dm->thread = NULL;
  1503. vmbus_close(hv_dev->channel);
  1504. }
  1505. tasklet_enable(&hv_dev->channel->callback_event);
  1506. return 0;
  1507. }
  1508. static int balloon_resume(struct hv_device *dev)
  1509. {
  1510. int ret;
  1511. dm_device.state = DM_INITIALIZING;
  1512. ret = balloon_connect_vsp(dev);
  1513. if (ret != 0)
  1514. goto out;
  1515. dm_device.thread =
  1516. kthread_run(dm_thread_func, &dm_device, "hv_balloon");
  1517. if (IS_ERR(dm_device.thread)) {
  1518. ret = PTR_ERR(dm_device.thread);
  1519. dm_device.thread = NULL;
  1520. goto close_channel;
  1521. }
  1522. dm_device.state = DM_INITIALIZED;
  1523. return 0;
  1524. close_channel:
  1525. vmbus_close(dev->channel);
  1526. out:
  1527. dm_device.state = DM_INIT_ERROR;
  1528. #ifdef CONFIG_MEMORY_HOTPLUG
  1529. unregister_memory_notifier(&hv_memory_nb);
  1530. restore_online_page_callback(&hv_online_page);
  1531. #endif
  1532. return ret;
  1533. }
  1534. static const struct hv_vmbus_device_id id_table[] = {
  1535. /* Dynamic Memory Class ID */
  1536. /* 525074DC-8985-46e2-8057-A307DC18A502 */
  1537. { HV_DM_GUID, },
  1538. { },
  1539. };
  1540. MODULE_DEVICE_TABLE(vmbus, id_table);
  1541. static struct hv_driver balloon_drv = {
  1542. .name = "hv_balloon",
  1543. .id_table = id_table,
  1544. .probe = balloon_probe,
  1545. .remove = balloon_remove,
  1546. .suspend = balloon_suspend,
  1547. .resume = balloon_resume,
  1548. .driver = {
  1549. .probe_type = PROBE_PREFER_ASYNCHRONOUS,
  1550. },
  1551. };
  1552. static int __init init_balloon_drv(void)
  1553. {
  1554. return vmbus_driver_register(&balloon_drv);
  1555. }
  1556. module_init(init_balloon_drv);
  1557. MODULE_DESCRIPTION("Hyper-V Balloon");
  1558. MODULE_LICENSE("GPL");