enlighten.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. #include <xen/xen.h>
  3. #include <xen/events.h>
  4. #include <xen/grant_table.h>
  5. #include <xen/hvm.h>
  6. #include <xen/interface/vcpu.h>
  7. #include <xen/interface/xen.h>
  8. #include <xen/interface/memory.h>
  9. #include <xen/interface/hvm/params.h>
  10. #include <xen/features.h>
  11. #include <xen/platform_pci.h>
  12. #include <xen/xenbus.h>
  13. #include <xen/page.h>
  14. #include <xen/interface/sched.h>
  15. #include <xen/xen-ops.h>
  16. #include <asm/xen/hypervisor.h>
  17. #include <asm/xen/hypercall.h>
  18. #include <asm/system_misc.h>
  19. #include <asm/efi.h>
  20. #include <linux/interrupt.h>
  21. #include <linux/irqreturn.h>
  22. #include <linux/module.h>
  23. #include <linux/of.h>
  24. #include <linux/of_fdt.h>
  25. #include <linux/of_irq.h>
  26. #include <linux/of_address.h>
  27. #include <linux/cpuidle.h>
  28. #include <linux/cpufreq.h>
  29. #include <linux/cpu.h>
  30. #include <linux/console.h>
  31. #include <linux/pvclock_gtod.h>
  32. #include <linux/reboot.h>
  33. #include <linux/time64.h>
  34. #include <linux/timekeeping.h>
  35. #include <linux/timekeeper_internal.h>
  36. #include <linux/acpi.h>
  37. #include <linux/mm.h>
  38. static struct start_info _xen_start_info;
  39. struct start_info *xen_start_info = &_xen_start_info;
  40. EXPORT_SYMBOL(xen_start_info);
  41. enum xen_domain_type xen_domain_type = XEN_NATIVE;
  42. EXPORT_SYMBOL(xen_domain_type);
  43. struct shared_info xen_dummy_shared_info;
  44. struct shared_info *HYPERVISOR_shared_info = (void *)&xen_dummy_shared_info;
  45. DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu);
  46. static struct vcpu_info __percpu *xen_vcpu_info;
  47. /* Linux <-> Xen vCPU id mapping */
  48. DEFINE_PER_CPU(uint32_t, xen_vcpu_id);
  49. EXPORT_PER_CPU_SYMBOL(xen_vcpu_id);
  50. /* These are unused until we support booting "pre-ballooned" */
  51. unsigned long xen_released_pages;
  52. struct xen_memory_region xen_extra_mem[XEN_EXTRA_MEM_MAX_REGIONS] __initdata;
  53. static __read_mostly unsigned int xen_events_irq;
  54. uint32_t xen_start_flags;
  55. EXPORT_SYMBOL(xen_start_flags);
  56. int xen_unmap_domain_gfn_range(struct vm_area_struct *vma,
  57. int nr, struct page **pages)
  58. {
  59. return xen_xlate_unmap_gfn_range(vma, nr, pages);
  60. }
  61. EXPORT_SYMBOL_GPL(xen_unmap_domain_gfn_range);
  62. static void xen_read_wallclock(struct timespec64 *ts)
  63. {
  64. u32 version;
  65. struct timespec64 now, ts_monotonic;
  66. struct shared_info *s = HYPERVISOR_shared_info;
  67. struct pvclock_wall_clock *wall_clock = &(s->wc);
  68. /* get wallclock at system boot */
  69. do {
  70. version = wall_clock->version;
  71. rmb(); /* fetch version before time */
  72. now.tv_sec = ((uint64_t)wall_clock->sec_hi << 32) | wall_clock->sec;
  73. now.tv_nsec = wall_clock->nsec;
  74. rmb(); /* fetch time before checking version */
  75. } while ((wall_clock->version & 1) || (version != wall_clock->version));
  76. /* time since system boot */
  77. ktime_get_ts64(&ts_monotonic);
  78. *ts = timespec64_add(now, ts_monotonic);
  79. }
  80. static int xen_pvclock_gtod_notify(struct notifier_block *nb,
  81. unsigned long was_set, void *priv)
  82. {
  83. /* Protected by the calling core code serialization */
  84. static struct timespec64 next_sync;
  85. struct xen_platform_op op;
  86. struct timespec64 now, system_time;
  87. struct timekeeper *tk = priv;
  88. now.tv_sec = tk->xtime_sec;
  89. now.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
  90. system_time = timespec64_add(now, tk->wall_to_monotonic);
  91. /*
  92. * We only take the expensive HV call when the clock was set
  93. * or when the 11 minutes RTC synchronization time elapsed.
  94. */
  95. if (!was_set && timespec64_compare(&now, &next_sync) < 0)
  96. return NOTIFY_OK;
  97. op.cmd = XENPF_settime64;
  98. op.u.settime64.mbz = 0;
  99. op.u.settime64.secs = now.tv_sec;
  100. op.u.settime64.nsecs = now.tv_nsec;
  101. op.u.settime64.system_time = timespec64_to_ns(&system_time);
  102. (void)HYPERVISOR_platform_op(&op);
  103. /*
  104. * Move the next drift compensation time 11 minutes
  105. * ahead. That's emulating the sync_cmos_clock() update for
  106. * the hardware RTC.
  107. */
  108. next_sync = now;
  109. next_sync.tv_sec += 11 * 60;
  110. return NOTIFY_OK;
  111. }
  112. static struct notifier_block xen_pvclock_gtod_notifier = {
  113. .notifier_call = xen_pvclock_gtod_notify,
  114. };
  115. static int xen_starting_cpu(unsigned int cpu)
  116. {
  117. struct vcpu_register_vcpu_info info;
  118. struct vcpu_info *vcpup;
  119. int err;
  120. /*
  121. * VCPUOP_register_vcpu_info cannot be called twice for the same
  122. * vcpu, so if vcpu_info is already registered, just get out. This
  123. * can happen with cpu-hotplug.
  124. */
  125. if (per_cpu(xen_vcpu, cpu) != NULL)
  126. goto after_register_vcpu_info;
  127. pr_info("Xen: initializing cpu%d\n", cpu);
  128. vcpup = per_cpu_ptr(xen_vcpu_info, cpu);
  129. info.mfn = percpu_to_gfn(vcpup);
  130. info.offset = xen_offset_in_page(vcpup);
  131. err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, xen_vcpu_nr(cpu),
  132. &info);
  133. BUG_ON(err);
  134. per_cpu(xen_vcpu, cpu) = vcpup;
  135. if (!xen_kernel_unmapped_at_usr())
  136. xen_setup_runstate_info(cpu);
  137. after_register_vcpu_info:
  138. enable_percpu_irq(xen_events_irq, 0);
  139. return 0;
  140. }
  141. static int xen_dying_cpu(unsigned int cpu)
  142. {
  143. disable_percpu_irq(xen_events_irq);
  144. return 0;
  145. }
  146. void xen_reboot(int reason)
  147. {
  148. struct sched_shutdown r = { .reason = reason };
  149. int rc;
  150. rc = HYPERVISOR_sched_op(SCHEDOP_shutdown, &r);
  151. BUG_ON(rc);
  152. }
  153. static int xen_restart(struct notifier_block *nb, unsigned long action,
  154. void *data)
  155. {
  156. xen_reboot(SHUTDOWN_reboot);
  157. return NOTIFY_DONE;
  158. }
  159. static struct notifier_block xen_restart_nb = {
  160. .notifier_call = xen_restart,
  161. .priority = 192,
  162. };
  163. static void xen_power_off(void)
  164. {
  165. xen_reboot(SHUTDOWN_poweroff);
  166. }
  167. static irqreturn_t xen_arm_callback(int irq, void *arg)
  168. {
  169. xen_hvm_evtchn_do_upcall();
  170. return IRQ_HANDLED;
  171. }
  172. static __initdata struct {
  173. const char *compat;
  174. const char *prefix;
  175. const char *version;
  176. bool found;
  177. } hyper_node = {"xen,xen", "xen,xen-", NULL, false};
  178. static int __init fdt_find_hyper_node(unsigned long node, const char *uname,
  179. int depth, void *data)
  180. {
  181. const void *s = NULL;
  182. int len;
  183. if (depth != 1 || strcmp(uname, "hypervisor") != 0)
  184. return 0;
  185. if (of_flat_dt_is_compatible(node, hyper_node.compat))
  186. hyper_node.found = true;
  187. s = of_get_flat_dt_prop(node, "compatible", &len);
  188. if (strlen(hyper_node.prefix) + 3 < len &&
  189. !strncmp(hyper_node.prefix, s, strlen(hyper_node.prefix)))
  190. hyper_node.version = s + strlen(hyper_node.prefix);
  191. /*
  192. * Check if Xen supports EFI by checking whether there is the
  193. * "/hypervisor/uefi" node in DT. If so, runtime services are available
  194. * through proxy functions (e.g. in case of Xen dom0 EFI implementation
  195. * they call special hypercall which executes relevant EFI functions)
  196. * and that is why they are always enabled.
  197. */
  198. if (IS_ENABLED(CONFIG_XEN_EFI)) {
  199. if ((of_get_flat_dt_subnode_by_name(node, "uefi") > 0) &&
  200. !efi_runtime_disabled())
  201. set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
  202. }
  203. return 0;
  204. }
  205. /*
  206. * see Documentation/devicetree/bindings/arm/xen.txt for the
  207. * documentation of the Xen Device Tree format.
  208. */
  209. void __init xen_early_init(void)
  210. {
  211. of_scan_flat_dt(fdt_find_hyper_node, NULL);
  212. if (!hyper_node.found) {
  213. pr_debug("No Xen support\n");
  214. return;
  215. }
  216. if (hyper_node.version == NULL) {
  217. pr_debug("Xen version not found\n");
  218. return;
  219. }
  220. pr_info("Xen %s support found\n", hyper_node.version);
  221. xen_domain_type = XEN_HVM_DOMAIN;
  222. xen_setup_features();
  223. if (xen_feature(XENFEAT_dom0))
  224. xen_start_flags |= SIF_INITDOMAIN|SIF_PRIVILEGED;
  225. if (!console_set_on_cmdline && !xen_initial_domain())
  226. add_preferred_console("hvc", 0, NULL);
  227. }
  228. static void __init xen_acpi_guest_init(void)
  229. {
  230. #ifdef CONFIG_ACPI
  231. struct xen_hvm_param a;
  232. int interrupt, trigger, polarity;
  233. a.domid = DOMID_SELF;
  234. a.index = HVM_PARAM_CALLBACK_IRQ;
  235. if (HYPERVISOR_hvm_op(HVMOP_get_param, &a)
  236. || (a.value >> 56) != HVM_PARAM_CALLBACK_TYPE_PPI) {
  237. xen_events_irq = 0;
  238. return;
  239. }
  240. interrupt = a.value & 0xff;
  241. trigger = ((a.value >> 8) & 0x1) ? ACPI_EDGE_SENSITIVE
  242. : ACPI_LEVEL_SENSITIVE;
  243. polarity = ((a.value >> 8) & 0x2) ? ACPI_ACTIVE_LOW
  244. : ACPI_ACTIVE_HIGH;
  245. xen_events_irq = acpi_register_gsi(NULL, interrupt, trigger, polarity);
  246. #endif
  247. }
  248. static void __init xen_dt_guest_init(void)
  249. {
  250. struct device_node *xen_node;
  251. xen_node = of_find_compatible_node(NULL, NULL, "xen,xen");
  252. if (!xen_node) {
  253. pr_err("Xen support was detected before, but it has disappeared\n");
  254. return;
  255. }
  256. xen_events_irq = irq_of_parse_and_map(xen_node, 0);
  257. }
  258. static int __init xen_guest_init(void)
  259. {
  260. struct xen_add_to_physmap xatp;
  261. struct shared_info *shared_info_page = NULL;
  262. int cpu;
  263. if (!xen_domain())
  264. return 0;
  265. if (!acpi_disabled)
  266. xen_acpi_guest_init();
  267. else
  268. xen_dt_guest_init();
  269. if (!xen_events_irq) {
  270. pr_err("Xen event channel interrupt not found\n");
  271. return -ENODEV;
  272. }
  273. /*
  274. * The fdt parsing codes have set EFI_RUNTIME_SERVICES if Xen EFI
  275. * parameters are found. Force enable runtime services.
  276. */
  277. if (efi_enabled(EFI_RUNTIME_SERVICES))
  278. xen_efi_runtime_setup();
  279. shared_info_page = (struct shared_info *)get_zeroed_page(GFP_KERNEL);
  280. if (!shared_info_page) {
  281. pr_err("not enough memory\n");
  282. return -ENOMEM;
  283. }
  284. xatp.domid = DOMID_SELF;
  285. xatp.idx = 0;
  286. xatp.space = XENMAPSPACE_shared_info;
  287. xatp.gpfn = virt_to_gfn(shared_info_page);
  288. if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp))
  289. BUG();
  290. HYPERVISOR_shared_info = (struct shared_info *)shared_info_page;
  291. /* xen_vcpu is a pointer to the vcpu_info struct in the shared_info
  292. * page, we use it in the event channel upcall and in some pvclock
  293. * related functions.
  294. * The shared info contains exactly 1 CPU (the boot CPU). The guest
  295. * is required to use VCPUOP_register_vcpu_info to place vcpu info
  296. * for secondary CPUs as they are brought up.
  297. * For uniformity we use VCPUOP_register_vcpu_info even on cpu0.
  298. */
  299. xen_vcpu_info = alloc_percpu(struct vcpu_info);
  300. if (xen_vcpu_info == NULL)
  301. return -ENOMEM;
  302. /* Direct vCPU id mapping for ARM guests. */
  303. for_each_possible_cpu(cpu)
  304. per_cpu(xen_vcpu_id, cpu) = cpu;
  305. xen_auto_xlat_grant_frames.count = gnttab_max_grant_frames();
  306. if (xen_xlate_map_ballooned_pages(&xen_auto_xlat_grant_frames.pfn,
  307. &xen_auto_xlat_grant_frames.vaddr,
  308. xen_auto_xlat_grant_frames.count)) {
  309. free_percpu(xen_vcpu_info);
  310. return -ENOMEM;
  311. }
  312. gnttab_init();
  313. /*
  314. * Making sure board specific code will not set up ops for
  315. * cpu idle and cpu freq.
  316. */
  317. disable_cpuidle();
  318. disable_cpufreq();
  319. xen_init_IRQ();
  320. if (request_percpu_irq(xen_events_irq, xen_arm_callback,
  321. "events", &xen_vcpu)) {
  322. pr_err("Error request IRQ %d\n", xen_events_irq);
  323. return -EINVAL;
  324. }
  325. if (!xen_kernel_unmapped_at_usr())
  326. xen_time_setup_guest();
  327. if (xen_initial_domain())
  328. pvclock_gtod_register_notifier(&xen_pvclock_gtod_notifier);
  329. return cpuhp_setup_state(CPUHP_AP_ARM_XEN_STARTING,
  330. "arm/xen:starting", xen_starting_cpu,
  331. xen_dying_cpu);
  332. }
  333. early_initcall(xen_guest_init);
  334. static int __init xen_pm_init(void)
  335. {
  336. if (!xen_domain())
  337. return -ENODEV;
  338. pm_power_off = xen_power_off;
  339. register_restart_handler(&xen_restart_nb);
  340. if (!xen_initial_domain()) {
  341. struct timespec64 ts;
  342. xen_read_wallclock(&ts);
  343. do_settimeofday64(&ts);
  344. }
  345. return 0;
  346. }
  347. late_initcall(xen_pm_init);
  348. /* empty stubs */
  349. void xen_arch_pre_suspend(void) { }
  350. void xen_arch_post_suspend(int suspend_cancelled) { }
  351. void xen_timer_resume(void) { }
  352. void xen_arch_resume(void) { }
  353. void xen_arch_suspend(void) { }
  354. /* In the hypercall.S file. */
  355. EXPORT_SYMBOL_GPL(HYPERVISOR_event_channel_op);
  356. EXPORT_SYMBOL_GPL(HYPERVISOR_grant_table_op);
  357. EXPORT_SYMBOL_GPL(HYPERVISOR_xen_version);
  358. EXPORT_SYMBOL_GPL(HYPERVISOR_console_io);
  359. EXPORT_SYMBOL_GPL(HYPERVISOR_sched_op);
  360. EXPORT_SYMBOL_GPL(HYPERVISOR_hvm_op);
  361. EXPORT_SYMBOL_GPL(HYPERVISOR_memory_op);
  362. EXPORT_SYMBOL_GPL(HYPERVISOR_physdev_op);
  363. EXPORT_SYMBOL_GPL(HYPERVISOR_vcpu_op);
  364. EXPORT_SYMBOL_GPL(HYPERVISOR_tmem_op);
  365. EXPORT_SYMBOL_GPL(HYPERVISOR_platform_op_raw);
  366. EXPORT_SYMBOL_GPL(HYPERVISOR_multicall);
  367. EXPORT_SYMBOL_GPL(HYPERVISOR_vm_assist);
  368. EXPORT_SYMBOL_GPL(HYPERVISOR_dm_op);
  369. EXPORT_SYMBOL_GPL(privcmd_call);