book3s_pr_papr.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (C) 2011. Freescale Inc. All rights reserved.
  4. *
  5. * Authors:
  6. * Alexander Graf <agraf@suse.de>
  7. * Paul Mackerras <paulus@samba.org>
  8. *
  9. * Description:
  10. *
  11. * Hypercall handling for running PAPR guests in PR KVM on Book 3S
  12. * processors.
  13. */
  14. #include <linux/anon_inodes.h>
  15. #include <linux/uaccess.h>
  16. #include <asm/kvm_ppc.h>
  17. #include <asm/kvm_book3s.h>
  18. #define HPTE_SIZE 16 /* bytes per HPT entry */
  19. static unsigned long get_pteg_addr(struct kvm_vcpu *vcpu, long pte_index)
  20. {
  21. struct kvmppc_vcpu_book3s *vcpu_book3s = to_book3s(vcpu);
  22. unsigned long pteg_addr;
  23. pte_index <<= 4;
  24. pte_index &= ((1 << ((vcpu_book3s->sdr1 & 0x1f) + 11)) - 1) << 7 | 0x70;
  25. pteg_addr = vcpu_book3s->sdr1 & 0xfffffffffffc0000ULL;
  26. pteg_addr |= pte_index;
  27. return pteg_addr;
  28. }
  29. static int kvmppc_h_pr_enter(struct kvm_vcpu *vcpu)
  30. {
  31. long flags = kvmppc_get_gpr(vcpu, 4);
  32. long pte_index = kvmppc_get_gpr(vcpu, 5);
  33. __be64 pteg[2 * 8];
  34. __be64 *hpte;
  35. unsigned long pteg_addr, i;
  36. long int ret;
  37. i = pte_index & 7;
  38. pte_index &= ~7UL;
  39. pteg_addr = get_pteg_addr(vcpu, pte_index);
  40. mutex_lock(&vcpu->kvm->arch.hpt_mutex);
  41. ret = H_FUNCTION;
  42. if (copy_from_user(pteg, (void __user *)pteg_addr, sizeof(pteg)))
  43. goto done;
  44. hpte = pteg;
  45. ret = H_PTEG_FULL;
  46. if (likely((flags & H_EXACT) == 0)) {
  47. for (i = 0; ; ++i) {
  48. if (i == 8)
  49. goto done;
  50. if ((be64_to_cpu(*hpte) & HPTE_V_VALID) == 0)
  51. break;
  52. hpte += 2;
  53. }
  54. } else {
  55. hpte += i * 2;
  56. if (*hpte & HPTE_V_VALID)
  57. goto done;
  58. }
  59. hpte[0] = cpu_to_be64(kvmppc_get_gpr(vcpu, 6));
  60. hpte[1] = cpu_to_be64(kvmppc_get_gpr(vcpu, 7));
  61. pteg_addr += i * HPTE_SIZE;
  62. ret = H_FUNCTION;
  63. if (copy_to_user((void __user *)pteg_addr, hpte, HPTE_SIZE))
  64. goto done;
  65. kvmppc_set_gpr(vcpu, 4, pte_index | i);
  66. ret = H_SUCCESS;
  67. done:
  68. mutex_unlock(&vcpu->kvm->arch.hpt_mutex);
  69. kvmppc_set_gpr(vcpu, 3, ret);
  70. return EMULATE_DONE;
  71. }
  72. static int kvmppc_h_pr_remove(struct kvm_vcpu *vcpu)
  73. {
  74. unsigned long flags= kvmppc_get_gpr(vcpu, 4);
  75. unsigned long pte_index = kvmppc_get_gpr(vcpu, 5);
  76. unsigned long avpn = kvmppc_get_gpr(vcpu, 6);
  77. unsigned long v = 0, pteg, rb;
  78. unsigned long pte[2];
  79. long int ret;
  80. pteg = get_pteg_addr(vcpu, pte_index);
  81. mutex_lock(&vcpu->kvm->arch.hpt_mutex);
  82. ret = H_FUNCTION;
  83. if (copy_from_user(pte, (void __user *)pteg, sizeof(pte)))
  84. goto done;
  85. pte[0] = be64_to_cpu((__force __be64)pte[0]);
  86. pte[1] = be64_to_cpu((__force __be64)pte[1]);
  87. ret = H_NOT_FOUND;
  88. if ((pte[0] & HPTE_V_VALID) == 0 ||
  89. ((flags & H_AVPN) && (pte[0] & ~0x7fUL) != avpn) ||
  90. ((flags & H_ANDCOND) && (pte[0] & avpn) != 0))
  91. goto done;
  92. ret = H_FUNCTION;
  93. if (copy_to_user((void __user *)pteg, &v, sizeof(v)))
  94. goto done;
  95. rb = compute_tlbie_rb(pte[0], pte[1], pte_index);
  96. vcpu->arch.mmu.tlbie(vcpu, rb, rb & 1 ? true : false);
  97. ret = H_SUCCESS;
  98. kvmppc_set_gpr(vcpu, 4, pte[0]);
  99. kvmppc_set_gpr(vcpu, 5, pte[1]);
  100. done:
  101. mutex_unlock(&vcpu->kvm->arch.hpt_mutex);
  102. kvmppc_set_gpr(vcpu, 3, ret);
  103. return EMULATE_DONE;
  104. }
  105. /* Request defs for kvmppc_h_pr_bulk_remove() */
  106. #define H_BULK_REMOVE_TYPE 0xc000000000000000ULL
  107. #define H_BULK_REMOVE_REQUEST 0x4000000000000000ULL
  108. #define H_BULK_REMOVE_RESPONSE 0x8000000000000000ULL
  109. #define H_BULK_REMOVE_END 0xc000000000000000ULL
  110. #define H_BULK_REMOVE_CODE 0x3000000000000000ULL
  111. #define H_BULK_REMOVE_SUCCESS 0x0000000000000000ULL
  112. #define H_BULK_REMOVE_NOT_FOUND 0x1000000000000000ULL
  113. #define H_BULK_REMOVE_PARM 0x2000000000000000ULL
  114. #define H_BULK_REMOVE_HW 0x3000000000000000ULL
  115. #define H_BULK_REMOVE_RC 0x0c00000000000000ULL
  116. #define H_BULK_REMOVE_FLAGS 0x0300000000000000ULL
  117. #define H_BULK_REMOVE_ABSOLUTE 0x0000000000000000ULL
  118. #define H_BULK_REMOVE_ANDCOND 0x0100000000000000ULL
  119. #define H_BULK_REMOVE_AVPN 0x0200000000000000ULL
  120. #define H_BULK_REMOVE_PTEX 0x00ffffffffffffffULL
  121. #define H_BULK_REMOVE_MAX_BATCH 4
  122. static int kvmppc_h_pr_bulk_remove(struct kvm_vcpu *vcpu)
  123. {
  124. int i;
  125. int paramnr = 4;
  126. int ret = H_SUCCESS;
  127. mutex_lock(&vcpu->kvm->arch.hpt_mutex);
  128. for (i = 0; i < H_BULK_REMOVE_MAX_BATCH; i++) {
  129. unsigned long tsh = kvmppc_get_gpr(vcpu, paramnr+(2*i));
  130. unsigned long tsl = kvmppc_get_gpr(vcpu, paramnr+(2*i)+1);
  131. unsigned long pteg, rb, flags;
  132. unsigned long pte[2];
  133. unsigned long v = 0;
  134. if ((tsh & H_BULK_REMOVE_TYPE) == H_BULK_REMOVE_END) {
  135. break; /* Exit success */
  136. } else if ((tsh & H_BULK_REMOVE_TYPE) !=
  137. H_BULK_REMOVE_REQUEST) {
  138. ret = H_PARAMETER;
  139. break; /* Exit fail */
  140. }
  141. tsh &= H_BULK_REMOVE_PTEX | H_BULK_REMOVE_FLAGS;
  142. tsh |= H_BULK_REMOVE_RESPONSE;
  143. if ((tsh & H_BULK_REMOVE_ANDCOND) &&
  144. (tsh & H_BULK_REMOVE_AVPN)) {
  145. tsh |= H_BULK_REMOVE_PARM;
  146. kvmppc_set_gpr(vcpu, paramnr+(2*i), tsh);
  147. ret = H_PARAMETER;
  148. break; /* Exit fail */
  149. }
  150. pteg = get_pteg_addr(vcpu, tsh & H_BULK_REMOVE_PTEX);
  151. if (copy_from_user(pte, (void __user *)pteg, sizeof(pte))) {
  152. ret = H_FUNCTION;
  153. break;
  154. }
  155. pte[0] = be64_to_cpu((__force __be64)pte[0]);
  156. pte[1] = be64_to_cpu((__force __be64)pte[1]);
  157. /* tsl = AVPN */
  158. flags = (tsh & H_BULK_REMOVE_FLAGS) >> 26;
  159. if ((pte[0] & HPTE_V_VALID) == 0 ||
  160. ((flags & H_AVPN) && (pte[0] & ~0x7fUL) != tsl) ||
  161. ((flags & H_ANDCOND) && (pte[0] & tsl) != 0)) {
  162. tsh |= H_BULK_REMOVE_NOT_FOUND;
  163. } else {
  164. /* Splat the pteg in (userland) hpt */
  165. if (copy_to_user((void __user *)pteg, &v, sizeof(v))) {
  166. ret = H_FUNCTION;
  167. break;
  168. }
  169. rb = compute_tlbie_rb(pte[0], pte[1],
  170. tsh & H_BULK_REMOVE_PTEX);
  171. vcpu->arch.mmu.tlbie(vcpu, rb, rb & 1 ? true : false);
  172. tsh |= H_BULK_REMOVE_SUCCESS;
  173. tsh |= (pte[1] & (HPTE_R_C | HPTE_R_R)) << 43;
  174. }
  175. kvmppc_set_gpr(vcpu, paramnr+(2*i), tsh);
  176. }
  177. mutex_unlock(&vcpu->kvm->arch.hpt_mutex);
  178. kvmppc_set_gpr(vcpu, 3, ret);
  179. return EMULATE_DONE;
  180. }
  181. static int kvmppc_h_pr_protect(struct kvm_vcpu *vcpu)
  182. {
  183. unsigned long flags = kvmppc_get_gpr(vcpu, 4);
  184. unsigned long pte_index = kvmppc_get_gpr(vcpu, 5);
  185. unsigned long avpn = kvmppc_get_gpr(vcpu, 6);
  186. unsigned long rb, pteg, r, v;
  187. unsigned long pte[2];
  188. long int ret;
  189. pteg = get_pteg_addr(vcpu, pte_index);
  190. mutex_lock(&vcpu->kvm->arch.hpt_mutex);
  191. ret = H_FUNCTION;
  192. if (copy_from_user(pte, (void __user *)pteg, sizeof(pte)))
  193. goto done;
  194. pte[0] = be64_to_cpu((__force __be64)pte[0]);
  195. pte[1] = be64_to_cpu((__force __be64)pte[1]);
  196. ret = H_NOT_FOUND;
  197. if ((pte[0] & HPTE_V_VALID) == 0 ||
  198. ((flags & H_AVPN) && (pte[0] & ~0x7fUL) != avpn))
  199. goto done;
  200. v = pte[0];
  201. r = pte[1];
  202. r &= ~(HPTE_R_PP0 | HPTE_R_PP | HPTE_R_N | HPTE_R_KEY_HI |
  203. HPTE_R_KEY_LO);
  204. r |= (flags << 55) & HPTE_R_PP0;
  205. r |= (flags << 48) & HPTE_R_KEY_HI;
  206. r |= flags & (HPTE_R_PP | HPTE_R_N | HPTE_R_KEY_LO);
  207. pte[1] = r;
  208. rb = compute_tlbie_rb(v, r, pte_index);
  209. vcpu->arch.mmu.tlbie(vcpu, rb, rb & 1 ? true : false);
  210. pte[0] = (__force u64)cpu_to_be64(pte[0]);
  211. pte[1] = (__force u64)cpu_to_be64(pte[1]);
  212. ret = H_FUNCTION;
  213. if (copy_to_user((void __user *)pteg, pte, sizeof(pte)))
  214. goto done;
  215. ret = H_SUCCESS;
  216. done:
  217. mutex_unlock(&vcpu->kvm->arch.hpt_mutex);
  218. kvmppc_set_gpr(vcpu, 3, ret);
  219. return EMULATE_DONE;
  220. }
  221. static int kvmppc_h_pr_logical_ci_load(struct kvm_vcpu *vcpu)
  222. {
  223. long rc;
  224. rc = kvmppc_h_logical_ci_load(vcpu);
  225. if (rc == H_TOO_HARD)
  226. return EMULATE_FAIL;
  227. kvmppc_set_gpr(vcpu, 3, rc);
  228. return EMULATE_DONE;
  229. }
  230. static int kvmppc_h_pr_logical_ci_store(struct kvm_vcpu *vcpu)
  231. {
  232. long rc;
  233. rc = kvmppc_h_logical_ci_store(vcpu);
  234. if (rc == H_TOO_HARD)
  235. return EMULATE_FAIL;
  236. kvmppc_set_gpr(vcpu, 3, rc);
  237. return EMULATE_DONE;
  238. }
  239. #ifdef CONFIG_SPAPR_TCE_IOMMU
  240. static int kvmppc_h_pr_put_tce(struct kvm_vcpu *vcpu)
  241. {
  242. unsigned long liobn = kvmppc_get_gpr(vcpu, 4);
  243. unsigned long ioba = kvmppc_get_gpr(vcpu, 5);
  244. unsigned long tce = kvmppc_get_gpr(vcpu, 6);
  245. long rc;
  246. rc = kvmppc_h_put_tce(vcpu, liobn, ioba, tce);
  247. if (rc == H_TOO_HARD)
  248. return EMULATE_FAIL;
  249. kvmppc_set_gpr(vcpu, 3, rc);
  250. return EMULATE_DONE;
  251. }
  252. static int kvmppc_h_pr_put_tce_indirect(struct kvm_vcpu *vcpu)
  253. {
  254. unsigned long liobn = kvmppc_get_gpr(vcpu, 4);
  255. unsigned long ioba = kvmppc_get_gpr(vcpu, 5);
  256. unsigned long tce = kvmppc_get_gpr(vcpu, 6);
  257. unsigned long npages = kvmppc_get_gpr(vcpu, 7);
  258. long rc;
  259. rc = kvmppc_h_put_tce_indirect(vcpu, liobn, ioba,
  260. tce, npages);
  261. if (rc == H_TOO_HARD)
  262. return EMULATE_FAIL;
  263. kvmppc_set_gpr(vcpu, 3, rc);
  264. return EMULATE_DONE;
  265. }
  266. static int kvmppc_h_pr_stuff_tce(struct kvm_vcpu *vcpu)
  267. {
  268. unsigned long liobn = kvmppc_get_gpr(vcpu, 4);
  269. unsigned long ioba = kvmppc_get_gpr(vcpu, 5);
  270. unsigned long tce_value = kvmppc_get_gpr(vcpu, 6);
  271. unsigned long npages = kvmppc_get_gpr(vcpu, 7);
  272. long rc;
  273. rc = kvmppc_h_stuff_tce(vcpu, liobn, ioba, tce_value, npages);
  274. if (rc == H_TOO_HARD)
  275. return EMULATE_FAIL;
  276. kvmppc_set_gpr(vcpu, 3, rc);
  277. return EMULATE_DONE;
  278. }
  279. #else /* CONFIG_SPAPR_TCE_IOMMU */
  280. static int kvmppc_h_pr_put_tce(struct kvm_vcpu *vcpu)
  281. {
  282. return EMULATE_FAIL;
  283. }
  284. static int kvmppc_h_pr_put_tce_indirect(struct kvm_vcpu *vcpu)
  285. {
  286. return EMULATE_FAIL;
  287. }
  288. static int kvmppc_h_pr_stuff_tce(struct kvm_vcpu *vcpu)
  289. {
  290. return EMULATE_FAIL;
  291. }
  292. #endif /* CONFIG_SPAPR_TCE_IOMMU */
  293. static int kvmppc_h_pr_xics_hcall(struct kvm_vcpu *vcpu, u32 cmd)
  294. {
  295. long rc = kvmppc_xics_hcall(vcpu, cmd);
  296. kvmppc_set_gpr(vcpu, 3, rc);
  297. return EMULATE_DONE;
  298. }
  299. int kvmppc_h_pr(struct kvm_vcpu *vcpu, unsigned long cmd)
  300. {
  301. int rc, idx;
  302. if (cmd <= MAX_HCALL_OPCODE &&
  303. !test_bit(cmd/4, vcpu->kvm->arch.enabled_hcalls))
  304. return EMULATE_FAIL;
  305. switch (cmd) {
  306. case H_ENTER:
  307. return kvmppc_h_pr_enter(vcpu);
  308. case H_REMOVE:
  309. return kvmppc_h_pr_remove(vcpu);
  310. case H_PROTECT:
  311. return kvmppc_h_pr_protect(vcpu);
  312. case H_BULK_REMOVE:
  313. return kvmppc_h_pr_bulk_remove(vcpu);
  314. case H_PUT_TCE:
  315. return kvmppc_h_pr_put_tce(vcpu);
  316. case H_PUT_TCE_INDIRECT:
  317. return kvmppc_h_pr_put_tce_indirect(vcpu);
  318. case H_STUFF_TCE:
  319. return kvmppc_h_pr_stuff_tce(vcpu);
  320. case H_CEDE:
  321. kvmppc_set_msr_fast(vcpu, kvmppc_get_msr(vcpu) | MSR_EE);
  322. kvm_vcpu_block(vcpu);
  323. kvm_clear_request(KVM_REQ_UNHALT, vcpu);
  324. vcpu->stat.halt_wakeup++;
  325. return EMULATE_DONE;
  326. case H_LOGICAL_CI_LOAD:
  327. return kvmppc_h_pr_logical_ci_load(vcpu);
  328. case H_LOGICAL_CI_STORE:
  329. return kvmppc_h_pr_logical_ci_store(vcpu);
  330. case H_XIRR:
  331. case H_CPPR:
  332. case H_EOI:
  333. case H_IPI:
  334. case H_IPOLL:
  335. case H_XIRR_X:
  336. if (kvmppc_xics_enabled(vcpu))
  337. return kvmppc_h_pr_xics_hcall(vcpu, cmd);
  338. break;
  339. case H_RTAS:
  340. if (list_empty(&vcpu->kvm->arch.rtas_tokens))
  341. break;
  342. idx = srcu_read_lock(&vcpu->kvm->srcu);
  343. rc = kvmppc_rtas_hcall(vcpu);
  344. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  345. if (rc)
  346. break;
  347. kvmppc_set_gpr(vcpu, 3, 0);
  348. return EMULATE_DONE;
  349. }
  350. return EMULATE_FAIL;
  351. }
  352. int kvmppc_hcall_impl_pr(unsigned long cmd)
  353. {
  354. switch (cmd) {
  355. case H_ENTER:
  356. case H_REMOVE:
  357. case H_PROTECT:
  358. case H_BULK_REMOVE:
  359. case H_PUT_TCE:
  360. case H_PUT_TCE_INDIRECT:
  361. case H_STUFF_TCE:
  362. case H_CEDE:
  363. case H_LOGICAL_CI_LOAD:
  364. case H_LOGICAL_CI_STORE:
  365. #ifdef CONFIG_KVM_XICS
  366. case H_XIRR:
  367. case H_CPPR:
  368. case H_EOI:
  369. case H_IPI:
  370. case H_IPOLL:
  371. case H_XIRR_X:
  372. #endif
  373. return 1;
  374. }
  375. return 0;
  376. }
  377. /*
  378. * List of hcall numbers to enable by default.
  379. * For compatibility with old userspace, we enable by default
  380. * all hcalls that were implemented before the hcall-enabling
  381. * facility was added. Note this list should not include H_RTAS.
  382. */
  383. static unsigned int default_hcall_list[] = {
  384. H_ENTER,
  385. H_REMOVE,
  386. H_PROTECT,
  387. H_BULK_REMOVE,
  388. H_PUT_TCE,
  389. H_CEDE,
  390. #ifdef CONFIG_KVM_XICS
  391. H_XIRR,
  392. H_CPPR,
  393. H_EOI,
  394. H_IPI,
  395. H_IPOLL,
  396. H_XIRR_X,
  397. #endif
  398. 0
  399. };
  400. void kvmppc_pr_init_default_hcalls(struct kvm *kvm)
  401. {
  402. int i;
  403. unsigned int hcall;
  404. for (i = 0; default_hcall_list[i]; ++i) {
  405. hcall = default_hcall_list[i];
  406. WARN_ON(!kvmppc_hcall_impl_pr(hcall));
  407. __set_bit(hcall / 4, kvm->arch.enabled_hcalls);
  408. }
  409. }