xpc_uv.c 47 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817
  1. /*
  2. * This file is subject to the terms and conditions of the GNU General Public
  3. * License. See the file "COPYING" in the main directory of this archive
  4. * for more details.
  5. *
  6. * Copyright (c) 2008-2009 Silicon Graphics, Inc. All Rights Reserved.
  7. */
  8. /*
  9. * Cross Partition Communication (XPC) uv-based functions.
  10. *
  11. * Architecture specific implementation of common functions.
  12. *
  13. */
  14. #include <linux/kernel.h>
  15. #include <linux/mm.h>
  16. #include <linux/interrupt.h>
  17. #include <linux/delay.h>
  18. #include <linux/device.h>
  19. #include <linux/cpu.h>
  20. #include <linux/module.h>
  21. #include <linux/err.h>
  22. #include <linux/slab.h>
  23. #include <linux/numa.h>
  24. #include <asm/uv/uv_hub.h>
  25. #if defined CONFIG_X86_64
  26. #include <asm/uv/bios.h>
  27. #include <asm/uv/uv_irq.h>
  28. #elif defined CONFIG_IA64_SGI_UV
  29. #include <asm/sn/intr.h>
  30. #include <asm/sn/sn_sal.h>
  31. #endif
  32. #include "../sgi-gru/gru.h"
  33. #include "../sgi-gru/grukservices.h"
  34. #include "xpc.h"
  35. #if defined CONFIG_IA64_SGI_UV
  36. struct uv_IO_APIC_route_entry {
  37. __u64 vector : 8,
  38. delivery_mode : 3,
  39. dest_mode : 1,
  40. delivery_status : 1,
  41. polarity : 1,
  42. __reserved_1 : 1,
  43. trigger : 1,
  44. mask : 1,
  45. __reserved_2 : 15,
  46. dest : 32;
  47. };
  48. #define sn_partition_id 0
  49. #endif
  50. static struct xpc_heartbeat_uv *xpc_heartbeat_uv;
  51. #define XPC_ACTIVATE_MSG_SIZE_UV (1 * GRU_CACHE_LINE_BYTES)
  52. #define XPC_ACTIVATE_MQ_SIZE_UV (4 * XP_MAX_NPARTITIONS_UV * \
  53. XPC_ACTIVATE_MSG_SIZE_UV)
  54. #define XPC_ACTIVATE_IRQ_NAME "xpc_activate"
  55. #define XPC_NOTIFY_MSG_SIZE_UV (2 * GRU_CACHE_LINE_BYTES)
  56. #define XPC_NOTIFY_MQ_SIZE_UV (4 * XP_MAX_NPARTITIONS_UV * \
  57. XPC_NOTIFY_MSG_SIZE_UV)
  58. #define XPC_NOTIFY_IRQ_NAME "xpc_notify"
  59. static int xpc_mq_node = NUMA_NO_NODE;
  60. static struct xpc_gru_mq_uv *xpc_activate_mq_uv;
  61. static struct xpc_gru_mq_uv *xpc_notify_mq_uv;
  62. static int
  63. xpc_setup_partitions_uv(void)
  64. {
  65. short partid;
  66. struct xpc_partition_uv *part_uv;
  67. for (partid = 0; partid < XP_MAX_NPARTITIONS_UV; partid++) {
  68. part_uv = &xpc_partitions[partid].sn.uv;
  69. mutex_init(&part_uv->cached_activate_gru_mq_desc_mutex);
  70. spin_lock_init(&part_uv->flags_lock);
  71. part_uv->remote_act_state = XPC_P_AS_INACTIVE;
  72. }
  73. return 0;
  74. }
  75. static void
  76. xpc_teardown_partitions_uv(void)
  77. {
  78. short partid;
  79. struct xpc_partition_uv *part_uv;
  80. unsigned long irq_flags;
  81. for (partid = 0; partid < XP_MAX_NPARTITIONS_UV; partid++) {
  82. part_uv = &xpc_partitions[partid].sn.uv;
  83. if (part_uv->cached_activate_gru_mq_desc != NULL) {
  84. mutex_lock(&part_uv->cached_activate_gru_mq_desc_mutex);
  85. spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
  86. part_uv->flags &= ~XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV;
  87. spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags);
  88. kfree(part_uv->cached_activate_gru_mq_desc);
  89. part_uv->cached_activate_gru_mq_desc = NULL;
  90. mutex_unlock(&part_uv->
  91. cached_activate_gru_mq_desc_mutex);
  92. }
  93. }
  94. }
  95. static int
  96. xpc_get_gru_mq_irq_uv(struct xpc_gru_mq_uv *mq, int cpu, char *irq_name)
  97. {
  98. int mmr_pnode = uv_blade_to_pnode(mq->mmr_blade);
  99. #if defined CONFIG_X86_64
  100. mq->irq = uv_setup_irq(irq_name, cpu, mq->mmr_blade, mq->mmr_offset,
  101. UV_AFFINITY_CPU);
  102. if (mq->irq < 0)
  103. return mq->irq;
  104. mq->mmr_value = uv_read_global_mmr64(mmr_pnode, mq->mmr_offset);
  105. #elif defined CONFIG_IA64_SGI_UV
  106. if (strcmp(irq_name, XPC_ACTIVATE_IRQ_NAME) == 0)
  107. mq->irq = SGI_XPC_ACTIVATE;
  108. else if (strcmp(irq_name, XPC_NOTIFY_IRQ_NAME) == 0)
  109. mq->irq = SGI_XPC_NOTIFY;
  110. else
  111. return -EINVAL;
  112. mq->mmr_value = (unsigned long)cpu_physical_id(cpu) << 32 | mq->irq;
  113. uv_write_global_mmr64(mmr_pnode, mq->mmr_offset, mq->mmr_value);
  114. #else
  115. #error not a supported configuration
  116. #endif
  117. return 0;
  118. }
  119. static void
  120. xpc_release_gru_mq_irq_uv(struct xpc_gru_mq_uv *mq)
  121. {
  122. #if defined CONFIG_X86_64
  123. uv_teardown_irq(mq->irq);
  124. #elif defined CONFIG_IA64_SGI_UV
  125. int mmr_pnode;
  126. unsigned long mmr_value;
  127. mmr_pnode = uv_blade_to_pnode(mq->mmr_blade);
  128. mmr_value = 1UL << 16;
  129. uv_write_global_mmr64(mmr_pnode, mq->mmr_offset, mmr_value);
  130. #else
  131. #error not a supported configuration
  132. #endif
  133. }
  134. static int
  135. xpc_gru_mq_watchlist_alloc_uv(struct xpc_gru_mq_uv *mq)
  136. {
  137. int ret;
  138. #if defined CONFIG_IA64_SGI_UV
  139. int mmr_pnode = uv_blade_to_pnode(mq->mmr_blade);
  140. ret = sn_mq_watchlist_alloc(mmr_pnode, (void *)uv_gpa(mq->address),
  141. mq->order, &mq->mmr_offset);
  142. if (ret < 0) {
  143. dev_err(xpc_part, "sn_mq_watchlist_alloc() failed, ret=%d\n",
  144. ret);
  145. return -EBUSY;
  146. }
  147. #elif defined CONFIG_X86_64
  148. ret = uv_bios_mq_watchlist_alloc(uv_gpa(mq->address),
  149. mq->order, &mq->mmr_offset);
  150. if (ret < 0) {
  151. dev_err(xpc_part, "uv_bios_mq_watchlist_alloc() failed, "
  152. "ret=%d\n", ret);
  153. return ret;
  154. }
  155. #else
  156. #error not a supported configuration
  157. #endif
  158. mq->watchlist_num = ret;
  159. return 0;
  160. }
  161. static void
  162. xpc_gru_mq_watchlist_free_uv(struct xpc_gru_mq_uv *mq)
  163. {
  164. int ret;
  165. int mmr_pnode = uv_blade_to_pnode(mq->mmr_blade);
  166. #if defined CONFIG_X86_64
  167. ret = uv_bios_mq_watchlist_free(mmr_pnode, mq->watchlist_num);
  168. BUG_ON(ret != BIOS_STATUS_SUCCESS);
  169. #elif defined CONFIG_IA64_SGI_UV
  170. ret = sn_mq_watchlist_free(mmr_pnode, mq->watchlist_num);
  171. BUG_ON(ret != SALRET_OK);
  172. #else
  173. #error not a supported configuration
  174. #endif
  175. }
  176. static struct xpc_gru_mq_uv *
  177. xpc_create_gru_mq_uv(unsigned int mq_size, int cpu, char *irq_name,
  178. irq_handler_t irq_handler)
  179. {
  180. enum xp_retval xp_ret;
  181. int ret;
  182. int nid;
  183. int nasid;
  184. int pg_order;
  185. struct page *page;
  186. struct xpc_gru_mq_uv *mq;
  187. struct uv_IO_APIC_route_entry *mmr_value;
  188. mq = kmalloc(sizeof(struct xpc_gru_mq_uv), GFP_KERNEL);
  189. if (mq == NULL) {
  190. dev_err(xpc_part, "xpc_create_gru_mq_uv() failed to kmalloc() "
  191. "a xpc_gru_mq_uv structure\n");
  192. ret = -ENOMEM;
  193. goto out_0;
  194. }
  195. mq->gru_mq_desc = kzalloc(sizeof(struct gru_message_queue_desc),
  196. GFP_KERNEL);
  197. if (mq->gru_mq_desc == NULL) {
  198. dev_err(xpc_part, "xpc_create_gru_mq_uv() failed to kmalloc() "
  199. "a gru_message_queue_desc structure\n");
  200. ret = -ENOMEM;
  201. goto out_1;
  202. }
  203. pg_order = get_order(mq_size);
  204. mq->order = pg_order + PAGE_SHIFT;
  205. mq_size = 1UL << mq->order;
  206. mq->mmr_blade = uv_cpu_to_blade_id(cpu);
  207. nid = cpu_to_node(cpu);
  208. page = __alloc_pages_node(nid,
  209. GFP_KERNEL | __GFP_ZERO | __GFP_THISNODE,
  210. pg_order);
  211. if (page == NULL) {
  212. dev_err(xpc_part, "xpc_create_gru_mq_uv() failed to alloc %d "
  213. "bytes of memory on nid=%d for GRU mq\n", mq_size, nid);
  214. ret = -ENOMEM;
  215. goto out_2;
  216. }
  217. mq->address = page_address(page);
  218. /* enable generation of irq when GRU mq operation occurs to this mq */
  219. ret = xpc_gru_mq_watchlist_alloc_uv(mq);
  220. if (ret != 0)
  221. goto out_3;
  222. ret = xpc_get_gru_mq_irq_uv(mq, cpu, irq_name);
  223. if (ret != 0)
  224. goto out_4;
  225. ret = request_irq(mq->irq, irq_handler, 0, irq_name, NULL);
  226. if (ret != 0) {
  227. dev_err(xpc_part, "request_irq(irq=%d) returned error=%d\n",
  228. mq->irq, -ret);
  229. goto out_5;
  230. }
  231. nasid = UV_PNODE_TO_NASID(uv_cpu_to_pnode(cpu));
  232. mmr_value = (struct uv_IO_APIC_route_entry *)&mq->mmr_value;
  233. ret = gru_create_message_queue(mq->gru_mq_desc, mq->address, mq_size,
  234. nasid, mmr_value->vector, mmr_value->dest);
  235. if (ret != 0) {
  236. dev_err(xpc_part, "gru_create_message_queue() returned "
  237. "error=%d\n", ret);
  238. ret = -EINVAL;
  239. goto out_6;
  240. }
  241. /* allow other partitions to access this GRU mq */
  242. xp_ret = xp_expand_memprotect(xp_pa(mq->address), mq_size);
  243. if (xp_ret != xpSuccess) {
  244. ret = -EACCES;
  245. goto out_6;
  246. }
  247. return mq;
  248. /* something went wrong */
  249. out_6:
  250. free_irq(mq->irq, NULL);
  251. out_5:
  252. xpc_release_gru_mq_irq_uv(mq);
  253. out_4:
  254. xpc_gru_mq_watchlist_free_uv(mq);
  255. out_3:
  256. free_pages((unsigned long)mq->address, pg_order);
  257. out_2:
  258. kfree(mq->gru_mq_desc);
  259. out_1:
  260. kfree(mq);
  261. out_0:
  262. return ERR_PTR(ret);
  263. }
  264. static void
  265. xpc_destroy_gru_mq_uv(struct xpc_gru_mq_uv *mq)
  266. {
  267. unsigned int mq_size;
  268. int pg_order;
  269. int ret;
  270. /* disallow other partitions to access GRU mq */
  271. mq_size = 1UL << mq->order;
  272. ret = xp_restrict_memprotect(xp_pa(mq->address), mq_size);
  273. BUG_ON(ret != xpSuccess);
  274. /* unregister irq handler and release mq irq/vector mapping */
  275. free_irq(mq->irq, NULL);
  276. xpc_release_gru_mq_irq_uv(mq);
  277. /* disable generation of irq when GRU mq op occurs to this mq */
  278. xpc_gru_mq_watchlist_free_uv(mq);
  279. pg_order = mq->order - PAGE_SHIFT;
  280. free_pages((unsigned long)mq->address, pg_order);
  281. kfree(mq);
  282. }
  283. static enum xp_retval
  284. xpc_send_gru_msg(struct gru_message_queue_desc *gru_mq_desc, void *msg,
  285. size_t msg_size)
  286. {
  287. enum xp_retval xp_ret;
  288. int ret;
  289. while (1) {
  290. ret = gru_send_message_gpa(gru_mq_desc, msg, msg_size);
  291. if (ret == MQE_OK) {
  292. xp_ret = xpSuccess;
  293. break;
  294. }
  295. if (ret == MQE_QUEUE_FULL) {
  296. dev_dbg(xpc_chan, "gru_send_message_gpa() returned "
  297. "error=MQE_QUEUE_FULL\n");
  298. /* !!! handle QLimit reached; delay & try again */
  299. /* ??? Do we add a limit to the number of retries? */
  300. (void)msleep_interruptible(10);
  301. } else if (ret == MQE_CONGESTION) {
  302. dev_dbg(xpc_chan, "gru_send_message_gpa() returned "
  303. "error=MQE_CONGESTION\n");
  304. /* !!! handle LB Overflow; simply try again */
  305. /* ??? Do we add a limit to the number of retries? */
  306. } else {
  307. /* !!! Currently this is MQE_UNEXPECTED_CB_ERR */
  308. dev_err(xpc_chan, "gru_send_message_gpa() returned "
  309. "error=%d\n", ret);
  310. xp_ret = xpGruSendMqError;
  311. break;
  312. }
  313. }
  314. return xp_ret;
  315. }
  316. static void
  317. xpc_process_activate_IRQ_rcvd_uv(void)
  318. {
  319. unsigned long irq_flags;
  320. short partid;
  321. struct xpc_partition *part;
  322. u8 act_state_req;
  323. DBUG_ON(xpc_activate_IRQ_rcvd == 0);
  324. spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
  325. for (partid = 0; partid < XP_MAX_NPARTITIONS_UV; partid++) {
  326. part = &xpc_partitions[partid];
  327. if (part->sn.uv.act_state_req == 0)
  328. continue;
  329. xpc_activate_IRQ_rcvd--;
  330. BUG_ON(xpc_activate_IRQ_rcvd < 0);
  331. act_state_req = part->sn.uv.act_state_req;
  332. part->sn.uv.act_state_req = 0;
  333. spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
  334. if (act_state_req == XPC_P_ASR_ACTIVATE_UV) {
  335. if (part->act_state == XPC_P_AS_INACTIVE)
  336. xpc_activate_partition(part);
  337. else if (part->act_state == XPC_P_AS_DEACTIVATING)
  338. XPC_DEACTIVATE_PARTITION(part, xpReactivating);
  339. } else if (act_state_req == XPC_P_ASR_REACTIVATE_UV) {
  340. if (part->act_state == XPC_P_AS_INACTIVE)
  341. xpc_activate_partition(part);
  342. else
  343. XPC_DEACTIVATE_PARTITION(part, xpReactivating);
  344. } else if (act_state_req == XPC_P_ASR_DEACTIVATE_UV) {
  345. XPC_DEACTIVATE_PARTITION(part, part->sn.uv.reason);
  346. } else {
  347. BUG();
  348. }
  349. spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
  350. if (xpc_activate_IRQ_rcvd == 0)
  351. break;
  352. }
  353. spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
  354. }
  355. static void
  356. xpc_handle_activate_mq_msg_uv(struct xpc_partition *part,
  357. struct xpc_activate_mq_msghdr_uv *msg_hdr,
  358. int part_setup,
  359. int *wakeup_hb_checker)
  360. {
  361. unsigned long irq_flags;
  362. struct xpc_partition_uv *part_uv = &part->sn.uv;
  363. struct xpc_openclose_args *args;
  364. part_uv->remote_act_state = msg_hdr->act_state;
  365. switch (msg_hdr->type) {
  366. case XPC_ACTIVATE_MQ_MSG_SYNC_ACT_STATE_UV:
  367. /* syncing of remote_act_state was just done above */
  368. break;
  369. case XPC_ACTIVATE_MQ_MSG_ACTIVATE_REQ_UV: {
  370. struct xpc_activate_mq_msg_activate_req_uv *msg;
  371. /*
  372. * ??? Do we deal here with ts_jiffies being different
  373. * ??? if act_state != XPC_P_AS_INACTIVE instead of
  374. * ??? below?
  375. */
  376. msg = container_of(msg_hdr, struct
  377. xpc_activate_mq_msg_activate_req_uv, hdr);
  378. spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
  379. if (part_uv->act_state_req == 0)
  380. xpc_activate_IRQ_rcvd++;
  381. part_uv->act_state_req = XPC_P_ASR_ACTIVATE_UV;
  382. part->remote_rp_pa = msg->rp_gpa; /* !!! _pa is _gpa */
  383. part->remote_rp_ts_jiffies = msg_hdr->rp_ts_jiffies;
  384. part_uv->heartbeat_gpa = msg->heartbeat_gpa;
  385. if (msg->activate_gru_mq_desc_gpa !=
  386. part_uv->activate_gru_mq_desc_gpa) {
  387. spin_lock(&part_uv->flags_lock);
  388. part_uv->flags &= ~XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV;
  389. spin_unlock(&part_uv->flags_lock);
  390. part_uv->activate_gru_mq_desc_gpa =
  391. msg->activate_gru_mq_desc_gpa;
  392. }
  393. spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
  394. (*wakeup_hb_checker)++;
  395. break;
  396. }
  397. case XPC_ACTIVATE_MQ_MSG_DEACTIVATE_REQ_UV: {
  398. struct xpc_activate_mq_msg_deactivate_req_uv *msg;
  399. msg = container_of(msg_hdr, struct
  400. xpc_activate_mq_msg_deactivate_req_uv, hdr);
  401. spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
  402. if (part_uv->act_state_req == 0)
  403. xpc_activate_IRQ_rcvd++;
  404. part_uv->act_state_req = XPC_P_ASR_DEACTIVATE_UV;
  405. part_uv->reason = msg->reason;
  406. spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
  407. (*wakeup_hb_checker)++;
  408. return;
  409. }
  410. case XPC_ACTIVATE_MQ_MSG_CHCTL_CLOSEREQUEST_UV: {
  411. struct xpc_activate_mq_msg_chctl_closerequest_uv *msg;
  412. if (!part_setup)
  413. break;
  414. msg = container_of(msg_hdr, struct
  415. xpc_activate_mq_msg_chctl_closerequest_uv,
  416. hdr);
  417. args = &part->remote_openclose_args[msg->ch_number];
  418. args->reason = msg->reason;
  419. spin_lock_irqsave(&part->chctl_lock, irq_flags);
  420. part->chctl.flags[msg->ch_number] |= XPC_CHCTL_CLOSEREQUEST;
  421. spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
  422. xpc_wakeup_channel_mgr(part);
  423. break;
  424. }
  425. case XPC_ACTIVATE_MQ_MSG_CHCTL_CLOSEREPLY_UV: {
  426. struct xpc_activate_mq_msg_chctl_closereply_uv *msg;
  427. if (!part_setup)
  428. break;
  429. msg = container_of(msg_hdr, struct
  430. xpc_activate_mq_msg_chctl_closereply_uv,
  431. hdr);
  432. spin_lock_irqsave(&part->chctl_lock, irq_flags);
  433. part->chctl.flags[msg->ch_number] |= XPC_CHCTL_CLOSEREPLY;
  434. spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
  435. xpc_wakeup_channel_mgr(part);
  436. break;
  437. }
  438. case XPC_ACTIVATE_MQ_MSG_CHCTL_OPENREQUEST_UV: {
  439. struct xpc_activate_mq_msg_chctl_openrequest_uv *msg;
  440. if (!part_setup)
  441. break;
  442. msg = container_of(msg_hdr, struct
  443. xpc_activate_mq_msg_chctl_openrequest_uv,
  444. hdr);
  445. args = &part->remote_openclose_args[msg->ch_number];
  446. args->entry_size = msg->entry_size;
  447. args->local_nentries = msg->local_nentries;
  448. spin_lock_irqsave(&part->chctl_lock, irq_flags);
  449. part->chctl.flags[msg->ch_number] |= XPC_CHCTL_OPENREQUEST;
  450. spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
  451. xpc_wakeup_channel_mgr(part);
  452. break;
  453. }
  454. case XPC_ACTIVATE_MQ_MSG_CHCTL_OPENREPLY_UV: {
  455. struct xpc_activate_mq_msg_chctl_openreply_uv *msg;
  456. if (!part_setup)
  457. break;
  458. msg = container_of(msg_hdr, struct
  459. xpc_activate_mq_msg_chctl_openreply_uv, hdr);
  460. args = &part->remote_openclose_args[msg->ch_number];
  461. args->remote_nentries = msg->remote_nentries;
  462. args->local_nentries = msg->local_nentries;
  463. args->local_msgqueue_pa = msg->notify_gru_mq_desc_gpa;
  464. spin_lock_irqsave(&part->chctl_lock, irq_flags);
  465. part->chctl.flags[msg->ch_number] |= XPC_CHCTL_OPENREPLY;
  466. spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
  467. xpc_wakeup_channel_mgr(part);
  468. break;
  469. }
  470. case XPC_ACTIVATE_MQ_MSG_CHCTL_OPENCOMPLETE_UV: {
  471. struct xpc_activate_mq_msg_chctl_opencomplete_uv *msg;
  472. if (!part_setup)
  473. break;
  474. msg = container_of(msg_hdr, struct
  475. xpc_activate_mq_msg_chctl_opencomplete_uv, hdr);
  476. spin_lock_irqsave(&part->chctl_lock, irq_flags);
  477. part->chctl.flags[msg->ch_number] |= XPC_CHCTL_OPENCOMPLETE;
  478. spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
  479. xpc_wakeup_channel_mgr(part);
  480. }
  481. fallthrough;
  482. case XPC_ACTIVATE_MQ_MSG_MARK_ENGAGED_UV:
  483. spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
  484. part_uv->flags |= XPC_P_ENGAGED_UV;
  485. spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags);
  486. break;
  487. case XPC_ACTIVATE_MQ_MSG_MARK_DISENGAGED_UV:
  488. spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
  489. part_uv->flags &= ~XPC_P_ENGAGED_UV;
  490. spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags);
  491. break;
  492. default:
  493. dev_err(xpc_part, "received unknown activate_mq msg type=%d "
  494. "from partition=%d\n", msg_hdr->type, XPC_PARTID(part));
  495. /* get hb checker to deactivate from the remote partition */
  496. spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
  497. if (part_uv->act_state_req == 0)
  498. xpc_activate_IRQ_rcvd++;
  499. part_uv->act_state_req = XPC_P_ASR_DEACTIVATE_UV;
  500. part_uv->reason = xpBadMsgType;
  501. spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
  502. (*wakeup_hb_checker)++;
  503. return;
  504. }
  505. if (msg_hdr->rp_ts_jiffies != part->remote_rp_ts_jiffies &&
  506. part->remote_rp_ts_jiffies != 0) {
  507. /*
  508. * ??? Does what we do here need to be sensitive to
  509. * ??? act_state or remote_act_state?
  510. */
  511. spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
  512. if (part_uv->act_state_req == 0)
  513. xpc_activate_IRQ_rcvd++;
  514. part_uv->act_state_req = XPC_P_ASR_REACTIVATE_UV;
  515. spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
  516. (*wakeup_hb_checker)++;
  517. }
  518. }
  519. static irqreturn_t
  520. xpc_handle_activate_IRQ_uv(int irq, void *dev_id)
  521. {
  522. struct xpc_activate_mq_msghdr_uv *msg_hdr;
  523. short partid;
  524. struct xpc_partition *part;
  525. int wakeup_hb_checker = 0;
  526. int part_referenced;
  527. while (1) {
  528. msg_hdr = gru_get_next_message(xpc_activate_mq_uv->gru_mq_desc);
  529. if (msg_hdr == NULL)
  530. break;
  531. partid = msg_hdr->partid;
  532. if (partid < 0 || partid >= XP_MAX_NPARTITIONS_UV) {
  533. dev_err(xpc_part, "xpc_handle_activate_IRQ_uv() "
  534. "received invalid partid=0x%x in message\n",
  535. partid);
  536. } else {
  537. part = &xpc_partitions[partid];
  538. part_referenced = xpc_part_ref(part);
  539. xpc_handle_activate_mq_msg_uv(part, msg_hdr,
  540. part_referenced,
  541. &wakeup_hb_checker);
  542. if (part_referenced)
  543. xpc_part_deref(part);
  544. }
  545. gru_free_message(xpc_activate_mq_uv->gru_mq_desc, msg_hdr);
  546. }
  547. if (wakeup_hb_checker)
  548. wake_up_interruptible(&xpc_activate_IRQ_wq);
  549. return IRQ_HANDLED;
  550. }
  551. static enum xp_retval
  552. xpc_cache_remote_gru_mq_desc_uv(struct gru_message_queue_desc *gru_mq_desc,
  553. unsigned long gru_mq_desc_gpa)
  554. {
  555. enum xp_retval ret;
  556. ret = xp_remote_memcpy(uv_gpa(gru_mq_desc), gru_mq_desc_gpa,
  557. sizeof(struct gru_message_queue_desc));
  558. if (ret == xpSuccess)
  559. gru_mq_desc->mq = NULL;
  560. return ret;
  561. }
  562. static enum xp_retval
  563. xpc_send_activate_IRQ_uv(struct xpc_partition *part, void *msg, size_t msg_size,
  564. int msg_type)
  565. {
  566. struct xpc_activate_mq_msghdr_uv *msg_hdr = msg;
  567. struct xpc_partition_uv *part_uv = &part->sn.uv;
  568. struct gru_message_queue_desc *gru_mq_desc;
  569. unsigned long irq_flags;
  570. enum xp_retval ret;
  571. DBUG_ON(msg_size > XPC_ACTIVATE_MSG_SIZE_UV);
  572. msg_hdr->type = msg_type;
  573. msg_hdr->partid = xp_partition_id;
  574. msg_hdr->act_state = part->act_state;
  575. msg_hdr->rp_ts_jiffies = xpc_rsvd_page->ts_jiffies;
  576. mutex_lock(&part_uv->cached_activate_gru_mq_desc_mutex);
  577. again:
  578. if (!(part_uv->flags & XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV)) {
  579. gru_mq_desc = part_uv->cached_activate_gru_mq_desc;
  580. if (gru_mq_desc == NULL) {
  581. gru_mq_desc = kmalloc(sizeof(struct
  582. gru_message_queue_desc),
  583. GFP_ATOMIC);
  584. if (gru_mq_desc == NULL) {
  585. ret = xpNoMemory;
  586. goto done;
  587. }
  588. part_uv->cached_activate_gru_mq_desc = gru_mq_desc;
  589. }
  590. ret = xpc_cache_remote_gru_mq_desc_uv(gru_mq_desc,
  591. part_uv->
  592. activate_gru_mq_desc_gpa);
  593. if (ret != xpSuccess)
  594. goto done;
  595. spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
  596. part_uv->flags |= XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV;
  597. spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags);
  598. }
  599. /* ??? Is holding a spin_lock (ch->lock) during this call a bad idea? */
  600. ret = xpc_send_gru_msg(part_uv->cached_activate_gru_mq_desc, msg,
  601. msg_size);
  602. if (ret != xpSuccess) {
  603. smp_rmb(); /* ensure a fresh copy of part_uv->flags */
  604. if (!(part_uv->flags & XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV))
  605. goto again;
  606. }
  607. done:
  608. mutex_unlock(&part_uv->cached_activate_gru_mq_desc_mutex);
  609. return ret;
  610. }
  611. static void
  612. xpc_send_activate_IRQ_part_uv(struct xpc_partition *part, void *msg,
  613. size_t msg_size, int msg_type)
  614. {
  615. enum xp_retval ret;
  616. ret = xpc_send_activate_IRQ_uv(part, msg, msg_size, msg_type);
  617. if (unlikely(ret != xpSuccess))
  618. XPC_DEACTIVATE_PARTITION(part, ret);
  619. }
  620. static void
  621. xpc_send_activate_IRQ_ch_uv(struct xpc_channel *ch, unsigned long *irq_flags,
  622. void *msg, size_t msg_size, int msg_type)
  623. {
  624. struct xpc_partition *part = &xpc_partitions[ch->partid];
  625. enum xp_retval ret;
  626. ret = xpc_send_activate_IRQ_uv(part, msg, msg_size, msg_type);
  627. if (unlikely(ret != xpSuccess)) {
  628. if (irq_flags != NULL)
  629. spin_unlock_irqrestore(&ch->lock, *irq_flags);
  630. XPC_DEACTIVATE_PARTITION(part, ret);
  631. if (irq_flags != NULL)
  632. spin_lock_irqsave(&ch->lock, *irq_flags);
  633. }
  634. }
  635. static void
  636. xpc_send_local_activate_IRQ_uv(struct xpc_partition *part, int act_state_req)
  637. {
  638. unsigned long irq_flags;
  639. struct xpc_partition_uv *part_uv = &part->sn.uv;
  640. /*
  641. * !!! Make our side think that the remote partition sent an activate
  642. * !!! mq message our way by doing what the activate IRQ handler would
  643. * !!! do had one really been sent.
  644. */
  645. spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
  646. if (part_uv->act_state_req == 0)
  647. xpc_activate_IRQ_rcvd++;
  648. part_uv->act_state_req = act_state_req;
  649. spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
  650. wake_up_interruptible(&xpc_activate_IRQ_wq);
  651. }
  652. static enum xp_retval
  653. xpc_get_partition_rsvd_page_pa_uv(void *buf, u64 *cookie, unsigned long *rp_pa,
  654. size_t *len)
  655. {
  656. s64 status;
  657. enum xp_retval ret;
  658. #if defined CONFIG_X86_64
  659. status = uv_bios_reserved_page_pa((u64)buf, cookie, (u64 *)rp_pa,
  660. (u64 *)len);
  661. if (status == BIOS_STATUS_SUCCESS)
  662. ret = xpSuccess;
  663. else if (status == BIOS_STATUS_MORE_PASSES)
  664. ret = xpNeedMoreInfo;
  665. else
  666. ret = xpBiosError;
  667. #elif defined CONFIG_IA64_SGI_UV
  668. status = sn_partition_reserved_page_pa((u64)buf, cookie, rp_pa, len);
  669. if (status == SALRET_OK)
  670. ret = xpSuccess;
  671. else if (status == SALRET_MORE_PASSES)
  672. ret = xpNeedMoreInfo;
  673. else
  674. ret = xpSalError;
  675. #else
  676. #error not a supported configuration
  677. #endif
  678. return ret;
  679. }
  680. static int
  681. xpc_setup_rsvd_page_uv(struct xpc_rsvd_page *rp)
  682. {
  683. xpc_heartbeat_uv =
  684. &xpc_partitions[sn_partition_id].sn.uv.cached_heartbeat;
  685. rp->sn.uv.heartbeat_gpa = uv_gpa(xpc_heartbeat_uv);
  686. rp->sn.uv.activate_gru_mq_desc_gpa =
  687. uv_gpa(xpc_activate_mq_uv->gru_mq_desc);
  688. return 0;
  689. }
  690. static void
  691. xpc_allow_hb_uv(short partid)
  692. {
  693. }
  694. static void
  695. xpc_disallow_hb_uv(short partid)
  696. {
  697. }
  698. static void
  699. xpc_disallow_all_hbs_uv(void)
  700. {
  701. }
  702. static void
  703. xpc_increment_heartbeat_uv(void)
  704. {
  705. xpc_heartbeat_uv->value++;
  706. }
  707. static void
  708. xpc_offline_heartbeat_uv(void)
  709. {
  710. xpc_increment_heartbeat_uv();
  711. xpc_heartbeat_uv->offline = 1;
  712. }
  713. static void
  714. xpc_online_heartbeat_uv(void)
  715. {
  716. xpc_increment_heartbeat_uv();
  717. xpc_heartbeat_uv->offline = 0;
  718. }
  719. static void
  720. xpc_heartbeat_init_uv(void)
  721. {
  722. xpc_heartbeat_uv->value = 1;
  723. xpc_heartbeat_uv->offline = 0;
  724. }
  725. static void
  726. xpc_heartbeat_exit_uv(void)
  727. {
  728. xpc_offline_heartbeat_uv();
  729. }
  730. static enum xp_retval
  731. xpc_get_remote_heartbeat_uv(struct xpc_partition *part)
  732. {
  733. struct xpc_partition_uv *part_uv = &part->sn.uv;
  734. enum xp_retval ret;
  735. ret = xp_remote_memcpy(uv_gpa(&part_uv->cached_heartbeat),
  736. part_uv->heartbeat_gpa,
  737. sizeof(struct xpc_heartbeat_uv));
  738. if (ret != xpSuccess)
  739. return ret;
  740. if (part_uv->cached_heartbeat.value == part->last_heartbeat &&
  741. !part_uv->cached_heartbeat.offline) {
  742. ret = xpNoHeartbeat;
  743. } else {
  744. part->last_heartbeat = part_uv->cached_heartbeat.value;
  745. }
  746. return ret;
  747. }
  748. static void
  749. xpc_request_partition_activation_uv(struct xpc_rsvd_page *remote_rp,
  750. unsigned long remote_rp_gpa, int nasid)
  751. {
  752. short partid = remote_rp->SAL_partid;
  753. struct xpc_partition *part = &xpc_partitions[partid];
  754. struct xpc_activate_mq_msg_activate_req_uv msg;
  755. part->remote_rp_pa = remote_rp_gpa; /* !!! _pa here is really _gpa */
  756. part->remote_rp_ts_jiffies = remote_rp->ts_jiffies;
  757. part->sn.uv.heartbeat_gpa = remote_rp->sn.uv.heartbeat_gpa;
  758. part->sn.uv.activate_gru_mq_desc_gpa =
  759. remote_rp->sn.uv.activate_gru_mq_desc_gpa;
  760. /*
  761. * ??? Is it a good idea to make this conditional on what is
  762. * ??? potentially stale state information?
  763. */
  764. if (part->sn.uv.remote_act_state == XPC_P_AS_INACTIVE) {
  765. msg.rp_gpa = uv_gpa(xpc_rsvd_page);
  766. msg.heartbeat_gpa = xpc_rsvd_page->sn.uv.heartbeat_gpa;
  767. msg.activate_gru_mq_desc_gpa =
  768. xpc_rsvd_page->sn.uv.activate_gru_mq_desc_gpa;
  769. xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg),
  770. XPC_ACTIVATE_MQ_MSG_ACTIVATE_REQ_UV);
  771. }
  772. if (part->act_state == XPC_P_AS_INACTIVE)
  773. xpc_send_local_activate_IRQ_uv(part, XPC_P_ASR_ACTIVATE_UV);
  774. }
  775. static void
  776. xpc_request_partition_reactivation_uv(struct xpc_partition *part)
  777. {
  778. xpc_send_local_activate_IRQ_uv(part, XPC_P_ASR_ACTIVATE_UV);
  779. }
  780. static void
  781. xpc_request_partition_deactivation_uv(struct xpc_partition *part)
  782. {
  783. struct xpc_activate_mq_msg_deactivate_req_uv msg;
  784. /*
  785. * ??? Is it a good idea to make this conditional on what is
  786. * ??? potentially stale state information?
  787. */
  788. if (part->sn.uv.remote_act_state != XPC_P_AS_DEACTIVATING &&
  789. part->sn.uv.remote_act_state != XPC_P_AS_INACTIVE) {
  790. msg.reason = part->reason;
  791. xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg),
  792. XPC_ACTIVATE_MQ_MSG_DEACTIVATE_REQ_UV);
  793. }
  794. }
  795. static void
  796. xpc_cancel_partition_deactivation_request_uv(struct xpc_partition *part)
  797. {
  798. /* nothing needs to be done */
  799. return;
  800. }
  801. static void
  802. xpc_init_fifo_uv(struct xpc_fifo_head_uv *head)
  803. {
  804. head->first = NULL;
  805. head->last = NULL;
  806. spin_lock_init(&head->lock);
  807. head->n_entries = 0;
  808. }
  809. static void *
  810. xpc_get_fifo_entry_uv(struct xpc_fifo_head_uv *head)
  811. {
  812. unsigned long irq_flags;
  813. struct xpc_fifo_entry_uv *first;
  814. spin_lock_irqsave(&head->lock, irq_flags);
  815. first = head->first;
  816. if (head->first != NULL) {
  817. head->first = first->next;
  818. if (head->first == NULL)
  819. head->last = NULL;
  820. head->n_entries--;
  821. BUG_ON(head->n_entries < 0);
  822. first->next = NULL;
  823. }
  824. spin_unlock_irqrestore(&head->lock, irq_flags);
  825. return first;
  826. }
  827. static void
  828. xpc_put_fifo_entry_uv(struct xpc_fifo_head_uv *head,
  829. struct xpc_fifo_entry_uv *last)
  830. {
  831. unsigned long irq_flags;
  832. last->next = NULL;
  833. spin_lock_irqsave(&head->lock, irq_flags);
  834. if (head->last != NULL)
  835. head->last->next = last;
  836. else
  837. head->first = last;
  838. head->last = last;
  839. head->n_entries++;
  840. spin_unlock_irqrestore(&head->lock, irq_flags);
  841. }
  842. static int
  843. xpc_n_of_fifo_entries_uv(struct xpc_fifo_head_uv *head)
  844. {
  845. return head->n_entries;
  846. }
  847. /*
  848. * Setup the channel structures that are uv specific.
  849. */
  850. static enum xp_retval
  851. xpc_setup_ch_structures_uv(struct xpc_partition *part)
  852. {
  853. struct xpc_channel_uv *ch_uv;
  854. int ch_number;
  855. for (ch_number = 0; ch_number < part->nchannels; ch_number++) {
  856. ch_uv = &part->channels[ch_number].sn.uv;
  857. xpc_init_fifo_uv(&ch_uv->msg_slot_free_list);
  858. xpc_init_fifo_uv(&ch_uv->recv_msg_list);
  859. }
  860. return xpSuccess;
  861. }
  862. /*
  863. * Teardown the channel structures that are uv specific.
  864. */
  865. static void
  866. xpc_teardown_ch_structures_uv(struct xpc_partition *part)
  867. {
  868. /* nothing needs to be done */
  869. return;
  870. }
  871. static enum xp_retval
  872. xpc_make_first_contact_uv(struct xpc_partition *part)
  873. {
  874. struct xpc_activate_mq_msg_uv msg;
  875. /*
  876. * We send a sync msg to get the remote partition's remote_act_state
  877. * updated to our current act_state which at this point should
  878. * be XPC_P_AS_ACTIVATING.
  879. */
  880. xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg),
  881. XPC_ACTIVATE_MQ_MSG_SYNC_ACT_STATE_UV);
  882. while (!((part->sn.uv.remote_act_state == XPC_P_AS_ACTIVATING) ||
  883. (part->sn.uv.remote_act_state == XPC_P_AS_ACTIVE))) {
  884. dev_dbg(xpc_part, "waiting to make first contact with "
  885. "partition %d\n", XPC_PARTID(part));
  886. /* wait a 1/4 of a second or so */
  887. (void)msleep_interruptible(250);
  888. if (part->act_state == XPC_P_AS_DEACTIVATING)
  889. return part->reason;
  890. }
  891. return xpSuccess;
  892. }
  893. static u64
  894. xpc_get_chctl_all_flags_uv(struct xpc_partition *part)
  895. {
  896. unsigned long irq_flags;
  897. union xpc_channel_ctl_flags chctl;
  898. spin_lock_irqsave(&part->chctl_lock, irq_flags);
  899. chctl = part->chctl;
  900. if (chctl.all_flags != 0)
  901. part->chctl.all_flags = 0;
  902. spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
  903. return chctl.all_flags;
  904. }
  905. static enum xp_retval
  906. xpc_allocate_send_msg_slot_uv(struct xpc_channel *ch)
  907. {
  908. struct xpc_channel_uv *ch_uv = &ch->sn.uv;
  909. struct xpc_send_msg_slot_uv *msg_slot;
  910. unsigned long irq_flags;
  911. int nentries;
  912. int entry;
  913. size_t nbytes;
  914. for (nentries = ch->local_nentries; nentries > 0; nentries--) {
  915. nbytes = nentries * sizeof(struct xpc_send_msg_slot_uv);
  916. ch_uv->send_msg_slots = kzalloc(nbytes, GFP_KERNEL);
  917. if (ch_uv->send_msg_slots == NULL)
  918. continue;
  919. for (entry = 0; entry < nentries; entry++) {
  920. msg_slot = &ch_uv->send_msg_slots[entry];
  921. msg_slot->msg_slot_number = entry;
  922. xpc_put_fifo_entry_uv(&ch_uv->msg_slot_free_list,
  923. &msg_slot->next);
  924. }
  925. spin_lock_irqsave(&ch->lock, irq_flags);
  926. if (nentries < ch->local_nentries)
  927. ch->local_nentries = nentries;
  928. spin_unlock_irqrestore(&ch->lock, irq_flags);
  929. return xpSuccess;
  930. }
  931. return xpNoMemory;
  932. }
  933. static enum xp_retval
  934. xpc_allocate_recv_msg_slot_uv(struct xpc_channel *ch)
  935. {
  936. struct xpc_channel_uv *ch_uv = &ch->sn.uv;
  937. struct xpc_notify_mq_msg_uv *msg_slot;
  938. unsigned long irq_flags;
  939. int nentries;
  940. int entry;
  941. size_t nbytes;
  942. for (nentries = ch->remote_nentries; nentries > 0; nentries--) {
  943. nbytes = nentries * ch->entry_size;
  944. ch_uv->recv_msg_slots = kzalloc(nbytes, GFP_KERNEL);
  945. if (ch_uv->recv_msg_slots == NULL)
  946. continue;
  947. for (entry = 0; entry < nentries; entry++) {
  948. msg_slot = ch_uv->recv_msg_slots +
  949. entry * ch->entry_size;
  950. msg_slot->hdr.msg_slot_number = entry;
  951. }
  952. spin_lock_irqsave(&ch->lock, irq_flags);
  953. if (nentries < ch->remote_nentries)
  954. ch->remote_nentries = nentries;
  955. spin_unlock_irqrestore(&ch->lock, irq_flags);
  956. return xpSuccess;
  957. }
  958. return xpNoMemory;
  959. }
  960. /*
  961. * Allocate msg_slots associated with the channel.
  962. */
  963. static enum xp_retval
  964. xpc_setup_msg_structures_uv(struct xpc_channel *ch)
  965. {
  966. static enum xp_retval ret;
  967. struct xpc_channel_uv *ch_uv = &ch->sn.uv;
  968. DBUG_ON(ch->flags & XPC_C_SETUP);
  969. ch_uv->cached_notify_gru_mq_desc = kmalloc(sizeof(struct
  970. gru_message_queue_desc),
  971. GFP_KERNEL);
  972. if (ch_uv->cached_notify_gru_mq_desc == NULL)
  973. return xpNoMemory;
  974. ret = xpc_allocate_send_msg_slot_uv(ch);
  975. if (ret == xpSuccess) {
  976. ret = xpc_allocate_recv_msg_slot_uv(ch);
  977. if (ret != xpSuccess) {
  978. kfree(ch_uv->send_msg_slots);
  979. xpc_init_fifo_uv(&ch_uv->msg_slot_free_list);
  980. }
  981. }
  982. return ret;
  983. }
  984. /*
  985. * Free up msg_slots and clear other stuff that were setup for the specified
  986. * channel.
  987. */
  988. static void
  989. xpc_teardown_msg_structures_uv(struct xpc_channel *ch)
  990. {
  991. struct xpc_channel_uv *ch_uv = &ch->sn.uv;
  992. lockdep_assert_held(&ch->lock);
  993. kfree(ch_uv->cached_notify_gru_mq_desc);
  994. ch_uv->cached_notify_gru_mq_desc = NULL;
  995. if (ch->flags & XPC_C_SETUP) {
  996. xpc_init_fifo_uv(&ch_uv->msg_slot_free_list);
  997. kfree(ch_uv->send_msg_slots);
  998. xpc_init_fifo_uv(&ch_uv->recv_msg_list);
  999. kfree(ch_uv->recv_msg_slots);
  1000. }
  1001. }
  1002. static void
  1003. xpc_send_chctl_closerequest_uv(struct xpc_channel *ch, unsigned long *irq_flags)
  1004. {
  1005. struct xpc_activate_mq_msg_chctl_closerequest_uv msg;
  1006. msg.ch_number = ch->number;
  1007. msg.reason = ch->reason;
  1008. xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg),
  1009. XPC_ACTIVATE_MQ_MSG_CHCTL_CLOSEREQUEST_UV);
  1010. }
  1011. static void
  1012. xpc_send_chctl_closereply_uv(struct xpc_channel *ch, unsigned long *irq_flags)
  1013. {
  1014. struct xpc_activate_mq_msg_chctl_closereply_uv msg;
  1015. msg.ch_number = ch->number;
  1016. xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg),
  1017. XPC_ACTIVATE_MQ_MSG_CHCTL_CLOSEREPLY_UV);
  1018. }
  1019. static void
  1020. xpc_send_chctl_openrequest_uv(struct xpc_channel *ch, unsigned long *irq_flags)
  1021. {
  1022. struct xpc_activate_mq_msg_chctl_openrequest_uv msg;
  1023. msg.ch_number = ch->number;
  1024. msg.entry_size = ch->entry_size;
  1025. msg.local_nentries = ch->local_nentries;
  1026. xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg),
  1027. XPC_ACTIVATE_MQ_MSG_CHCTL_OPENREQUEST_UV);
  1028. }
  1029. static void
  1030. xpc_send_chctl_openreply_uv(struct xpc_channel *ch, unsigned long *irq_flags)
  1031. {
  1032. struct xpc_activate_mq_msg_chctl_openreply_uv msg;
  1033. msg.ch_number = ch->number;
  1034. msg.local_nentries = ch->local_nentries;
  1035. msg.remote_nentries = ch->remote_nentries;
  1036. msg.notify_gru_mq_desc_gpa = uv_gpa(xpc_notify_mq_uv->gru_mq_desc);
  1037. xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg),
  1038. XPC_ACTIVATE_MQ_MSG_CHCTL_OPENREPLY_UV);
  1039. }
  1040. static void
  1041. xpc_send_chctl_opencomplete_uv(struct xpc_channel *ch, unsigned long *irq_flags)
  1042. {
  1043. struct xpc_activate_mq_msg_chctl_opencomplete_uv msg;
  1044. msg.ch_number = ch->number;
  1045. xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg),
  1046. XPC_ACTIVATE_MQ_MSG_CHCTL_OPENCOMPLETE_UV);
  1047. }
  1048. static void
  1049. xpc_send_chctl_local_msgrequest_uv(struct xpc_partition *part, int ch_number)
  1050. {
  1051. unsigned long irq_flags;
  1052. spin_lock_irqsave(&part->chctl_lock, irq_flags);
  1053. part->chctl.flags[ch_number] |= XPC_CHCTL_MSGREQUEST;
  1054. spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
  1055. xpc_wakeup_channel_mgr(part);
  1056. }
  1057. static enum xp_retval
  1058. xpc_save_remote_msgqueue_pa_uv(struct xpc_channel *ch,
  1059. unsigned long gru_mq_desc_gpa)
  1060. {
  1061. struct xpc_channel_uv *ch_uv = &ch->sn.uv;
  1062. DBUG_ON(ch_uv->cached_notify_gru_mq_desc == NULL);
  1063. return xpc_cache_remote_gru_mq_desc_uv(ch_uv->cached_notify_gru_mq_desc,
  1064. gru_mq_desc_gpa);
  1065. }
  1066. static void
  1067. xpc_indicate_partition_engaged_uv(struct xpc_partition *part)
  1068. {
  1069. struct xpc_activate_mq_msg_uv msg;
  1070. xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg),
  1071. XPC_ACTIVATE_MQ_MSG_MARK_ENGAGED_UV);
  1072. }
  1073. static void
  1074. xpc_indicate_partition_disengaged_uv(struct xpc_partition *part)
  1075. {
  1076. struct xpc_activate_mq_msg_uv msg;
  1077. xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg),
  1078. XPC_ACTIVATE_MQ_MSG_MARK_DISENGAGED_UV);
  1079. }
  1080. static void
  1081. xpc_assume_partition_disengaged_uv(short partid)
  1082. {
  1083. struct xpc_partition_uv *part_uv = &xpc_partitions[partid].sn.uv;
  1084. unsigned long irq_flags;
  1085. spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
  1086. part_uv->flags &= ~XPC_P_ENGAGED_UV;
  1087. spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags);
  1088. }
  1089. static int
  1090. xpc_partition_engaged_uv(short partid)
  1091. {
  1092. return (xpc_partitions[partid].sn.uv.flags & XPC_P_ENGAGED_UV) != 0;
  1093. }
  1094. static int
  1095. xpc_any_partition_engaged_uv(void)
  1096. {
  1097. struct xpc_partition_uv *part_uv;
  1098. short partid;
  1099. for (partid = 0; partid < XP_MAX_NPARTITIONS_UV; partid++) {
  1100. part_uv = &xpc_partitions[partid].sn.uv;
  1101. if ((part_uv->flags & XPC_P_ENGAGED_UV) != 0)
  1102. return 1;
  1103. }
  1104. return 0;
  1105. }
  1106. static enum xp_retval
  1107. xpc_allocate_msg_slot_uv(struct xpc_channel *ch, u32 flags,
  1108. struct xpc_send_msg_slot_uv **address_of_msg_slot)
  1109. {
  1110. enum xp_retval ret;
  1111. struct xpc_send_msg_slot_uv *msg_slot;
  1112. struct xpc_fifo_entry_uv *entry;
  1113. while (1) {
  1114. entry = xpc_get_fifo_entry_uv(&ch->sn.uv.msg_slot_free_list);
  1115. if (entry != NULL)
  1116. break;
  1117. if (flags & XPC_NOWAIT)
  1118. return xpNoWait;
  1119. ret = xpc_allocate_msg_wait(ch);
  1120. if (ret != xpInterrupted && ret != xpTimeout)
  1121. return ret;
  1122. }
  1123. msg_slot = container_of(entry, struct xpc_send_msg_slot_uv, next);
  1124. *address_of_msg_slot = msg_slot;
  1125. return xpSuccess;
  1126. }
  1127. static void
  1128. xpc_free_msg_slot_uv(struct xpc_channel *ch,
  1129. struct xpc_send_msg_slot_uv *msg_slot)
  1130. {
  1131. xpc_put_fifo_entry_uv(&ch->sn.uv.msg_slot_free_list, &msg_slot->next);
  1132. /* wakeup anyone waiting for a free msg slot */
  1133. if (atomic_read(&ch->n_on_msg_allocate_wq) > 0)
  1134. wake_up(&ch->msg_allocate_wq);
  1135. }
  1136. static void
  1137. xpc_notify_sender_uv(struct xpc_channel *ch,
  1138. struct xpc_send_msg_slot_uv *msg_slot,
  1139. enum xp_retval reason)
  1140. {
  1141. xpc_notify_func func = msg_slot->func;
  1142. if (func != NULL && cmpxchg(&msg_slot->func, func, NULL) == func) {
  1143. atomic_dec(&ch->n_to_notify);
  1144. dev_dbg(xpc_chan, "msg_slot->func() called, msg_slot=0x%p "
  1145. "msg_slot_number=%d partid=%d channel=%d\n", msg_slot,
  1146. msg_slot->msg_slot_number, ch->partid, ch->number);
  1147. func(reason, ch->partid, ch->number, msg_slot->key);
  1148. dev_dbg(xpc_chan, "msg_slot->func() returned, msg_slot=0x%p "
  1149. "msg_slot_number=%d partid=%d channel=%d\n", msg_slot,
  1150. msg_slot->msg_slot_number, ch->partid, ch->number);
  1151. }
  1152. }
  1153. static void
  1154. xpc_handle_notify_mq_ack_uv(struct xpc_channel *ch,
  1155. struct xpc_notify_mq_msg_uv *msg)
  1156. {
  1157. struct xpc_send_msg_slot_uv *msg_slot;
  1158. int entry = msg->hdr.msg_slot_number % ch->local_nentries;
  1159. msg_slot = &ch->sn.uv.send_msg_slots[entry];
  1160. BUG_ON(msg_slot->msg_slot_number != msg->hdr.msg_slot_number);
  1161. msg_slot->msg_slot_number += ch->local_nentries;
  1162. if (msg_slot->func != NULL)
  1163. xpc_notify_sender_uv(ch, msg_slot, xpMsgDelivered);
  1164. xpc_free_msg_slot_uv(ch, msg_slot);
  1165. }
  1166. static void
  1167. xpc_handle_notify_mq_msg_uv(struct xpc_partition *part,
  1168. struct xpc_notify_mq_msg_uv *msg)
  1169. {
  1170. struct xpc_partition_uv *part_uv = &part->sn.uv;
  1171. struct xpc_channel *ch;
  1172. struct xpc_channel_uv *ch_uv;
  1173. struct xpc_notify_mq_msg_uv *msg_slot;
  1174. unsigned long irq_flags;
  1175. int ch_number = msg->hdr.ch_number;
  1176. if (unlikely(ch_number >= part->nchannels)) {
  1177. dev_err(xpc_part, "xpc_handle_notify_IRQ_uv() received invalid "
  1178. "channel number=0x%x in message from partid=%d\n",
  1179. ch_number, XPC_PARTID(part));
  1180. /* get hb checker to deactivate from the remote partition */
  1181. spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
  1182. if (part_uv->act_state_req == 0)
  1183. xpc_activate_IRQ_rcvd++;
  1184. part_uv->act_state_req = XPC_P_ASR_DEACTIVATE_UV;
  1185. part_uv->reason = xpBadChannelNumber;
  1186. spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
  1187. wake_up_interruptible(&xpc_activate_IRQ_wq);
  1188. return;
  1189. }
  1190. ch = &part->channels[ch_number];
  1191. xpc_msgqueue_ref(ch);
  1192. if (!(ch->flags & XPC_C_CONNECTED)) {
  1193. xpc_msgqueue_deref(ch);
  1194. return;
  1195. }
  1196. /* see if we're really dealing with an ACK for a previously sent msg */
  1197. if (msg->hdr.size == 0) {
  1198. xpc_handle_notify_mq_ack_uv(ch, msg);
  1199. xpc_msgqueue_deref(ch);
  1200. return;
  1201. }
  1202. /* we're dealing with a normal message sent via the notify_mq */
  1203. ch_uv = &ch->sn.uv;
  1204. msg_slot = ch_uv->recv_msg_slots +
  1205. (msg->hdr.msg_slot_number % ch->remote_nentries) * ch->entry_size;
  1206. BUG_ON(msg_slot->hdr.size != 0);
  1207. memcpy(msg_slot, msg, msg->hdr.size);
  1208. xpc_put_fifo_entry_uv(&ch_uv->recv_msg_list, &msg_slot->hdr.u.next);
  1209. if (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE) {
  1210. /*
  1211. * If there is an existing idle kthread get it to deliver
  1212. * the payload, otherwise we'll have to get the channel mgr
  1213. * for this partition to create a kthread to do the delivery.
  1214. */
  1215. if (atomic_read(&ch->kthreads_idle) > 0)
  1216. wake_up_nr(&ch->idle_wq, 1);
  1217. else
  1218. xpc_send_chctl_local_msgrequest_uv(part, ch->number);
  1219. }
  1220. xpc_msgqueue_deref(ch);
  1221. }
  1222. static irqreturn_t
  1223. xpc_handle_notify_IRQ_uv(int irq, void *dev_id)
  1224. {
  1225. struct xpc_notify_mq_msg_uv *msg;
  1226. short partid;
  1227. struct xpc_partition *part;
  1228. while ((msg = gru_get_next_message(xpc_notify_mq_uv->gru_mq_desc)) !=
  1229. NULL) {
  1230. partid = msg->hdr.partid;
  1231. if (partid < 0 || partid >= XP_MAX_NPARTITIONS_UV) {
  1232. dev_err(xpc_part, "xpc_handle_notify_IRQ_uv() received "
  1233. "invalid partid=0x%x in message\n", partid);
  1234. } else {
  1235. part = &xpc_partitions[partid];
  1236. if (xpc_part_ref(part)) {
  1237. xpc_handle_notify_mq_msg_uv(part, msg);
  1238. xpc_part_deref(part);
  1239. }
  1240. }
  1241. gru_free_message(xpc_notify_mq_uv->gru_mq_desc, msg);
  1242. }
  1243. return IRQ_HANDLED;
  1244. }
  1245. static int
  1246. xpc_n_of_deliverable_payloads_uv(struct xpc_channel *ch)
  1247. {
  1248. return xpc_n_of_fifo_entries_uv(&ch->sn.uv.recv_msg_list);
  1249. }
  1250. static void
  1251. xpc_process_msg_chctl_flags_uv(struct xpc_partition *part, int ch_number)
  1252. {
  1253. struct xpc_channel *ch = &part->channels[ch_number];
  1254. int ndeliverable_payloads;
  1255. xpc_msgqueue_ref(ch);
  1256. ndeliverable_payloads = xpc_n_of_deliverable_payloads_uv(ch);
  1257. if (ndeliverable_payloads > 0 &&
  1258. (ch->flags & XPC_C_CONNECTED) &&
  1259. (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE)) {
  1260. xpc_activate_kthreads(ch, ndeliverable_payloads);
  1261. }
  1262. xpc_msgqueue_deref(ch);
  1263. }
  1264. static enum xp_retval
  1265. xpc_send_payload_uv(struct xpc_channel *ch, u32 flags, void *payload,
  1266. u16 payload_size, u8 notify_type, xpc_notify_func func,
  1267. void *key)
  1268. {
  1269. enum xp_retval ret = xpSuccess;
  1270. struct xpc_send_msg_slot_uv *msg_slot = NULL;
  1271. struct xpc_notify_mq_msg_uv *msg;
  1272. u8 msg_buffer[XPC_NOTIFY_MSG_SIZE_UV];
  1273. size_t msg_size;
  1274. DBUG_ON(notify_type != XPC_N_CALL);
  1275. msg_size = sizeof(struct xpc_notify_mq_msghdr_uv) + payload_size;
  1276. if (msg_size > ch->entry_size)
  1277. return xpPayloadTooBig;
  1278. xpc_msgqueue_ref(ch);
  1279. if (ch->flags & XPC_C_DISCONNECTING) {
  1280. ret = ch->reason;
  1281. goto out_1;
  1282. }
  1283. if (!(ch->flags & XPC_C_CONNECTED)) {
  1284. ret = xpNotConnected;
  1285. goto out_1;
  1286. }
  1287. ret = xpc_allocate_msg_slot_uv(ch, flags, &msg_slot);
  1288. if (ret != xpSuccess)
  1289. goto out_1;
  1290. if (func != NULL) {
  1291. atomic_inc(&ch->n_to_notify);
  1292. msg_slot->key = key;
  1293. smp_wmb(); /* a non-NULL func must hit memory after the key */
  1294. msg_slot->func = func;
  1295. if (ch->flags & XPC_C_DISCONNECTING) {
  1296. ret = ch->reason;
  1297. goto out_2;
  1298. }
  1299. }
  1300. msg = (struct xpc_notify_mq_msg_uv *)&msg_buffer;
  1301. msg->hdr.partid = xp_partition_id;
  1302. msg->hdr.ch_number = ch->number;
  1303. msg->hdr.size = msg_size;
  1304. msg->hdr.msg_slot_number = msg_slot->msg_slot_number;
  1305. memcpy(&msg->payload, payload, payload_size);
  1306. ret = xpc_send_gru_msg(ch->sn.uv.cached_notify_gru_mq_desc, msg,
  1307. msg_size);
  1308. if (ret == xpSuccess)
  1309. goto out_1;
  1310. XPC_DEACTIVATE_PARTITION(&xpc_partitions[ch->partid], ret);
  1311. out_2:
  1312. if (func != NULL) {
  1313. /*
  1314. * Try to NULL the msg_slot's func field. If we fail, then
  1315. * xpc_notify_senders_of_disconnect_uv() beat us to it, in which
  1316. * case we need to pretend we succeeded to send the message
  1317. * since the user will get a callout for the disconnect error
  1318. * by xpc_notify_senders_of_disconnect_uv(), and to also get an
  1319. * error returned here will confuse them. Additionally, since
  1320. * in this case the channel is being disconnected we don't need
  1321. * to put the the msg_slot back on the free list.
  1322. */
  1323. if (cmpxchg(&msg_slot->func, func, NULL) != func) {
  1324. ret = xpSuccess;
  1325. goto out_1;
  1326. }
  1327. msg_slot->key = NULL;
  1328. atomic_dec(&ch->n_to_notify);
  1329. }
  1330. xpc_free_msg_slot_uv(ch, msg_slot);
  1331. out_1:
  1332. xpc_msgqueue_deref(ch);
  1333. return ret;
  1334. }
  1335. /*
  1336. * Tell the callers of xpc_send_notify() that the status of their payloads
  1337. * is unknown because the channel is now disconnecting.
  1338. *
  1339. * We don't worry about putting these msg_slots on the free list since the
  1340. * msg_slots themselves are about to be kfree'd.
  1341. */
  1342. static void
  1343. xpc_notify_senders_of_disconnect_uv(struct xpc_channel *ch)
  1344. {
  1345. struct xpc_send_msg_slot_uv *msg_slot;
  1346. int entry;
  1347. DBUG_ON(!(ch->flags & XPC_C_DISCONNECTING));
  1348. for (entry = 0; entry < ch->local_nentries; entry++) {
  1349. if (atomic_read(&ch->n_to_notify) == 0)
  1350. break;
  1351. msg_slot = &ch->sn.uv.send_msg_slots[entry];
  1352. if (msg_slot->func != NULL)
  1353. xpc_notify_sender_uv(ch, msg_slot, ch->reason);
  1354. }
  1355. }
  1356. /*
  1357. * Get the next deliverable message's payload.
  1358. */
  1359. static void *
  1360. xpc_get_deliverable_payload_uv(struct xpc_channel *ch)
  1361. {
  1362. struct xpc_fifo_entry_uv *entry;
  1363. struct xpc_notify_mq_msg_uv *msg;
  1364. void *payload = NULL;
  1365. if (!(ch->flags & XPC_C_DISCONNECTING)) {
  1366. entry = xpc_get_fifo_entry_uv(&ch->sn.uv.recv_msg_list);
  1367. if (entry != NULL) {
  1368. msg = container_of(entry, struct xpc_notify_mq_msg_uv,
  1369. hdr.u.next);
  1370. payload = &msg->payload;
  1371. }
  1372. }
  1373. return payload;
  1374. }
  1375. static void
  1376. xpc_received_payload_uv(struct xpc_channel *ch, void *payload)
  1377. {
  1378. struct xpc_notify_mq_msg_uv *msg;
  1379. enum xp_retval ret;
  1380. msg = container_of(payload, struct xpc_notify_mq_msg_uv, payload);
  1381. /* return an ACK to the sender of this message */
  1382. msg->hdr.partid = xp_partition_id;
  1383. msg->hdr.size = 0; /* size of zero indicates this is an ACK */
  1384. ret = xpc_send_gru_msg(ch->sn.uv.cached_notify_gru_mq_desc, msg,
  1385. sizeof(struct xpc_notify_mq_msghdr_uv));
  1386. if (ret != xpSuccess)
  1387. XPC_DEACTIVATE_PARTITION(&xpc_partitions[ch->partid], ret);
  1388. }
  1389. static const struct xpc_arch_operations xpc_arch_ops_uv = {
  1390. .setup_partitions = xpc_setup_partitions_uv,
  1391. .teardown_partitions = xpc_teardown_partitions_uv,
  1392. .process_activate_IRQ_rcvd = xpc_process_activate_IRQ_rcvd_uv,
  1393. .get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_uv,
  1394. .setup_rsvd_page = xpc_setup_rsvd_page_uv,
  1395. .allow_hb = xpc_allow_hb_uv,
  1396. .disallow_hb = xpc_disallow_hb_uv,
  1397. .disallow_all_hbs = xpc_disallow_all_hbs_uv,
  1398. .increment_heartbeat = xpc_increment_heartbeat_uv,
  1399. .offline_heartbeat = xpc_offline_heartbeat_uv,
  1400. .online_heartbeat = xpc_online_heartbeat_uv,
  1401. .heartbeat_init = xpc_heartbeat_init_uv,
  1402. .heartbeat_exit = xpc_heartbeat_exit_uv,
  1403. .get_remote_heartbeat = xpc_get_remote_heartbeat_uv,
  1404. .request_partition_activation =
  1405. xpc_request_partition_activation_uv,
  1406. .request_partition_reactivation =
  1407. xpc_request_partition_reactivation_uv,
  1408. .request_partition_deactivation =
  1409. xpc_request_partition_deactivation_uv,
  1410. .cancel_partition_deactivation_request =
  1411. xpc_cancel_partition_deactivation_request_uv,
  1412. .setup_ch_structures = xpc_setup_ch_structures_uv,
  1413. .teardown_ch_structures = xpc_teardown_ch_structures_uv,
  1414. .make_first_contact = xpc_make_first_contact_uv,
  1415. .get_chctl_all_flags = xpc_get_chctl_all_flags_uv,
  1416. .send_chctl_closerequest = xpc_send_chctl_closerequest_uv,
  1417. .send_chctl_closereply = xpc_send_chctl_closereply_uv,
  1418. .send_chctl_openrequest = xpc_send_chctl_openrequest_uv,
  1419. .send_chctl_openreply = xpc_send_chctl_openreply_uv,
  1420. .send_chctl_opencomplete = xpc_send_chctl_opencomplete_uv,
  1421. .process_msg_chctl_flags = xpc_process_msg_chctl_flags_uv,
  1422. .save_remote_msgqueue_pa = xpc_save_remote_msgqueue_pa_uv,
  1423. .setup_msg_structures = xpc_setup_msg_structures_uv,
  1424. .teardown_msg_structures = xpc_teardown_msg_structures_uv,
  1425. .indicate_partition_engaged = xpc_indicate_partition_engaged_uv,
  1426. .indicate_partition_disengaged = xpc_indicate_partition_disengaged_uv,
  1427. .assume_partition_disengaged = xpc_assume_partition_disengaged_uv,
  1428. .partition_engaged = xpc_partition_engaged_uv,
  1429. .any_partition_engaged = xpc_any_partition_engaged_uv,
  1430. .n_of_deliverable_payloads = xpc_n_of_deliverable_payloads_uv,
  1431. .send_payload = xpc_send_payload_uv,
  1432. .get_deliverable_payload = xpc_get_deliverable_payload_uv,
  1433. .received_payload = xpc_received_payload_uv,
  1434. .notify_senders_of_disconnect = xpc_notify_senders_of_disconnect_uv,
  1435. };
  1436. static int
  1437. xpc_init_mq_node(int nid)
  1438. {
  1439. int cpu;
  1440. get_online_cpus();
  1441. for_each_cpu(cpu, cpumask_of_node(nid)) {
  1442. xpc_activate_mq_uv =
  1443. xpc_create_gru_mq_uv(XPC_ACTIVATE_MQ_SIZE_UV, nid,
  1444. XPC_ACTIVATE_IRQ_NAME,
  1445. xpc_handle_activate_IRQ_uv);
  1446. if (!IS_ERR(xpc_activate_mq_uv))
  1447. break;
  1448. }
  1449. if (IS_ERR(xpc_activate_mq_uv)) {
  1450. put_online_cpus();
  1451. return PTR_ERR(xpc_activate_mq_uv);
  1452. }
  1453. for_each_cpu(cpu, cpumask_of_node(nid)) {
  1454. xpc_notify_mq_uv =
  1455. xpc_create_gru_mq_uv(XPC_NOTIFY_MQ_SIZE_UV, nid,
  1456. XPC_NOTIFY_IRQ_NAME,
  1457. xpc_handle_notify_IRQ_uv);
  1458. if (!IS_ERR(xpc_notify_mq_uv))
  1459. break;
  1460. }
  1461. if (IS_ERR(xpc_notify_mq_uv)) {
  1462. xpc_destroy_gru_mq_uv(xpc_activate_mq_uv);
  1463. put_online_cpus();
  1464. return PTR_ERR(xpc_notify_mq_uv);
  1465. }
  1466. put_online_cpus();
  1467. return 0;
  1468. }
  1469. int
  1470. xpc_init_uv(void)
  1471. {
  1472. int nid;
  1473. int ret = 0;
  1474. xpc_arch_ops = xpc_arch_ops_uv;
  1475. if (sizeof(struct xpc_notify_mq_msghdr_uv) > XPC_MSG_HDR_MAX_SIZE) {
  1476. dev_err(xpc_part, "xpc_notify_mq_msghdr_uv is larger than %d\n",
  1477. XPC_MSG_HDR_MAX_SIZE);
  1478. return -E2BIG;
  1479. }
  1480. if (xpc_mq_node < 0)
  1481. for_each_online_node(nid) {
  1482. ret = xpc_init_mq_node(nid);
  1483. if (!ret)
  1484. break;
  1485. }
  1486. else
  1487. ret = xpc_init_mq_node(xpc_mq_node);
  1488. if (ret < 0)
  1489. dev_err(xpc_part, "xpc_init_mq_node() returned error=%d\n",
  1490. -ret);
  1491. return ret;
  1492. }
  1493. void
  1494. xpc_exit_uv(void)
  1495. {
  1496. xpc_destroy_gru_mq_uv(xpc_notify_mq_uv);
  1497. xpc_destroy_gru_mq_uv(xpc_activate_mq_uv);
  1498. }
  1499. module_param(xpc_mq_node, int, 0);
  1500. MODULE_PARM_DESC(xpc_mq_node, "Node number on which to allocate message queues.");