process.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Based on arch/arm/kernel/process.c
  4. *
  5. * Original Copyright (C) 1995 Linus Torvalds
  6. * Copyright (C) 1996-2000 Russell King - Converted to ARM.
  7. * Copyright (C) 2012 ARM Ltd.
  8. */
  9. #include <stdarg.h>
  10. #include <linux/compat.h>
  11. #include <linux/efi.h>
  12. #include <linux/elf.h>
  13. #include <linux/export.h>
  14. #include <linux/sched.h>
  15. #include <linux/sched/debug.h>
  16. #include <linux/sched/task.h>
  17. #include <linux/sched/task_stack.h>
  18. #include <linux/kernel.h>
  19. #include <linux/lockdep.h>
  20. #include <linux/mman.h>
  21. #include <linux/mm.h>
  22. #include <linux/nospec.h>
  23. #include <linux/stddef.h>
  24. #include <linux/sysctl.h>
  25. #include <linux/unistd.h>
  26. #include <linux/user.h>
  27. #include <linux/delay.h>
  28. #include <linux/reboot.h>
  29. #include <linux/interrupt.h>
  30. #include <linux/init.h>
  31. #include <linux/cpu.h>
  32. #include <linux/elfcore.h>
  33. #include <linux/pm.h>
  34. #include <linux/tick.h>
  35. #include <linux/utsname.h>
  36. #include <linux/uaccess.h>
  37. #include <linux/random.h>
  38. #include <linux/hw_breakpoint.h>
  39. #include <linux/personality.h>
  40. #include <linux/notifier.h>
  41. #include <trace/events/power.h>
  42. #include <linux/percpu.h>
  43. #include <linux/thread_info.h>
  44. #include <linux/prctl.h>
  45. #include <trace/hooks/fpsimd.h>
  46. #include <asm/alternative.h>
  47. #include <asm/arch_gicv3.h>
  48. #include <asm/compat.h>
  49. #include <asm/cpufeature.h>
  50. #include <asm/cacheflush.h>
  51. #include <asm/exec.h>
  52. #include <asm/fpsimd.h>
  53. #include <asm/mmu_context.h>
  54. #include <asm/mte.h>
  55. #include <asm/processor.h>
  56. #include <asm/pointer_auth.h>
  57. #include <asm/stacktrace.h>
  58. #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK)
  59. #include <linux/stackprotector.h>
  60. unsigned long __stack_chk_guard __ro_after_init;
  61. EXPORT_SYMBOL(__stack_chk_guard);
  62. #endif
  63. /*
  64. * Function pointers to optional machine specific functions
  65. */
  66. void (*pm_power_off)(void);
  67. EXPORT_SYMBOL_GPL(pm_power_off);
  68. static void noinstr __cpu_do_idle(void)
  69. {
  70. dsb(sy);
  71. wfi();
  72. }
  73. static void noinstr __cpu_do_idle_irqprio(void)
  74. {
  75. unsigned long pmr;
  76. unsigned long daif_bits;
  77. daif_bits = read_sysreg(daif);
  78. write_sysreg(daif_bits | PSR_I_BIT, daif);
  79. /*
  80. * Unmask PMR before going idle to make sure interrupts can
  81. * be raised.
  82. */
  83. pmr = gic_read_pmr();
  84. gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
  85. __cpu_do_idle();
  86. gic_write_pmr(pmr);
  87. write_sysreg(daif_bits, daif);
  88. }
  89. /*
  90. * cpu_do_idle()
  91. *
  92. * Idle the processor (wait for interrupt).
  93. *
  94. * If the CPU supports priority masking we must do additional work to
  95. * ensure that interrupts are not masked at the PMR (because the core will
  96. * not wake up if we block the wake up signal in the interrupt controller).
  97. */
  98. void noinstr cpu_do_idle(void)
  99. {
  100. if (system_uses_irq_prio_masking())
  101. __cpu_do_idle_irqprio();
  102. else
  103. __cpu_do_idle();
  104. }
  105. /*
  106. * This is our default idle handler.
  107. */
  108. void noinstr arch_cpu_idle(void)
  109. {
  110. /*
  111. * This should do all the clock switching and wait for interrupt
  112. * tricks
  113. */
  114. cpu_do_idle();
  115. raw_local_irq_enable();
  116. }
  117. #ifdef CONFIG_HOTPLUG_CPU
  118. void arch_cpu_idle_dead(void)
  119. {
  120. cpu_die();
  121. }
  122. #endif
  123. /*
  124. * Called by kexec, immediately prior to machine_kexec().
  125. *
  126. * This must completely disable all secondary CPUs; simply causing those CPUs
  127. * to execute e.g. a RAM-based pin loop is not sufficient. This allows the
  128. * kexec'd kernel to use any and all RAM as it sees fit, without having to
  129. * avoid any code or data used by any SW CPU pin loop. The CPU hotplug
  130. * functionality embodied in smpt_shutdown_nonboot_cpus() to achieve this.
  131. */
  132. void machine_shutdown(void)
  133. {
  134. smp_shutdown_nonboot_cpus(reboot_cpu);
  135. }
  136. /*
  137. * Halting simply requires that the secondary CPUs stop performing any
  138. * activity (executing tasks, handling interrupts). smp_send_stop()
  139. * achieves this.
  140. */
  141. void machine_halt(void)
  142. {
  143. local_irq_disable();
  144. smp_send_stop();
  145. while (1);
  146. }
  147. /*
  148. * Power-off simply requires that the secondary CPUs stop performing any
  149. * activity (executing tasks, handling interrupts). smp_send_stop()
  150. * achieves this. When the system power is turned off, it will take all CPUs
  151. * with it.
  152. */
  153. void machine_power_off(void)
  154. {
  155. local_irq_disable();
  156. smp_send_stop();
  157. if (pm_power_off)
  158. pm_power_off();
  159. }
  160. /*
  161. * Restart requires that the secondary CPUs stop performing any activity
  162. * while the primary CPU resets the system. Systems with multiple CPUs must
  163. * provide a HW restart implementation, to ensure that all CPUs reset at once.
  164. * This is required so that any code running after reset on the primary CPU
  165. * doesn't have to co-ordinate with other CPUs to ensure they aren't still
  166. * executing pre-reset code, and using RAM that the primary CPU's code wishes
  167. * to use. Implementing such co-ordination would be essentially impossible.
  168. */
  169. void machine_restart(char *cmd)
  170. {
  171. /* Disable interrupts first */
  172. local_irq_disable();
  173. smp_send_stop();
  174. /*
  175. * UpdateCapsule() depends on the system being reset via
  176. * ResetSystem().
  177. */
  178. if (efi_enabled(EFI_RUNTIME_SERVICES))
  179. efi_reboot(reboot_mode, NULL);
  180. /* Now call the architecture specific reboot code. */
  181. do_kernel_restart(cmd);
  182. /*
  183. * Whoops - the architecture was unable to reboot.
  184. */
  185. printk("Reboot failed -- System halted\n");
  186. while (1);
  187. }
  188. #define bstr(suffix, str) [PSR_BTYPE_ ## suffix >> PSR_BTYPE_SHIFT] = str
  189. static const char *const btypes[] = {
  190. bstr(NONE, "--"),
  191. bstr( JC, "jc"),
  192. bstr( C, "-c"),
  193. bstr( J , "j-")
  194. };
  195. #undef bstr
  196. static void print_pstate(struct pt_regs *regs)
  197. {
  198. u64 pstate = regs->pstate;
  199. if (compat_user_mode(regs)) {
  200. printk("pstate: %08llx (%c%c%c%c %c %s %s %c%c%c)\n",
  201. pstate,
  202. pstate & PSR_AA32_N_BIT ? 'N' : 'n',
  203. pstate & PSR_AA32_Z_BIT ? 'Z' : 'z',
  204. pstate & PSR_AA32_C_BIT ? 'C' : 'c',
  205. pstate & PSR_AA32_V_BIT ? 'V' : 'v',
  206. pstate & PSR_AA32_Q_BIT ? 'Q' : 'q',
  207. pstate & PSR_AA32_T_BIT ? "T32" : "A32",
  208. pstate & PSR_AA32_E_BIT ? "BE" : "LE",
  209. pstate & PSR_AA32_A_BIT ? 'A' : 'a',
  210. pstate & PSR_AA32_I_BIT ? 'I' : 'i',
  211. pstate & PSR_AA32_F_BIT ? 'F' : 'f');
  212. } else {
  213. const char *btype_str = btypes[(pstate & PSR_BTYPE_MASK) >>
  214. PSR_BTYPE_SHIFT];
  215. printk("pstate: %08llx (%c%c%c%c %c%c%c%c %cPAN %cUAO %cTCO BTYPE=%s)\n",
  216. pstate,
  217. pstate & PSR_N_BIT ? 'N' : 'n',
  218. pstate & PSR_Z_BIT ? 'Z' : 'z',
  219. pstate & PSR_C_BIT ? 'C' : 'c',
  220. pstate & PSR_V_BIT ? 'V' : 'v',
  221. pstate & PSR_D_BIT ? 'D' : 'd',
  222. pstate & PSR_A_BIT ? 'A' : 'a',
  223. pstate & PSR_I_BIT ? 'I' : 'i',
  224. pstate & PSR_F_BIT ? 'F' : 'f',
  225. pstate & PSR_PAN_BIT ? '+' : '-',
  226. pstate & PSR_UAO_BIT ? '+' : '-',
  227. pstate & PSR_TCO_BIT ? '+' : '-',
  228. btype_str);
  229. }
  230. }
  231. void __show_regs(struct pt_regs *regs)
  232. {
  233. int i, top_reg;
  234. u64 lr, sp;
  235. if (compat_user_mode(regs)) {
  236. lr = regs->compat_lr;
  237. sp = regs->compat_sp;
  238. top_reg = 12;
  239. } else {
  240. lr = regs->regs[30];
  241. sp = regs->sp;
  242. top_reg = 29;
  243. }
  244. show_regs_print_info(KERN_DEFAULT);
  245. print_pstate(regs);
  246. if (!user_mode(regs)) {
  247. printk("pc : %pS\n", (void *)regs->pc);
  248. printk("lr : %pS\n", (void *)ptrauth_strip_insn_pac(lr));
  249. } else {
  250. printk("pc : %016llx\n", regs->pc);
  251. printk("lr : %016llx\n", lr);
  252. }
  253. printk("sp : %016llx\n", sp);
  254. if (system_uses_irq_prio_masking())
  255. printk("pmr_save: %08llx\n", regs->pmr_save);
  256. i = top_reg;
  257. while (i >= 0) {
  258. printk("x%-2d: %016llx ", i, regs->regs[i]);
  259. i--;
  260. if (i % 2 == 0) {
  261. pr_cont("x%-2d: %016llx ", i, regs->regs[i]);
  262. i--;
  263. }
  264. pr_cont("\n");
  265. }
  266. }
  267. void show_regs(struct pt_regs * regs)
  268. {
  269. __show_regs(regs);
  270. dump_backtrace(regs, NULL, KERN_DEFAULT);
  271. }
  272. EXPORT_SYMBOL_GPL(show_regs);
  273. static void tls_thread_flush(void)
  274. {
  275. write_sysreg(0, tpidr_el0);
  276. if (is_compat_task()) {
  277. current->thread.uw.tp_value = 0;
  278. /*
  279. * We need to ensure ordering between the shadow state and the
  280. * hardware state, so that we don't corrupt the hardware state
  281. * with a stale shadow state during context switch.
  282. */
  283. barrier();
  284. write_sysreg(0, tpidrro_el0);
  285. }
  286. }
  287. static void flush_tagged_addr_state(void)
  288. {
  289. if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI))
  290. clear_thread_flag(TIF_TAGGED_ADDR);
  291. }
  292. void flush_thread(void)
  293. {
  294. fpsimd_flush_thread();
  295. tls_thread_flush();
  296. flush_ptrace_hw_breakpoint(current);
  297. flush_tagged_addr_state();
  298. }
  299. void release_thread(struct task_struct *dead_task)
  300. {
  301. }
  302. void arch_release_task_struct(struct task_struct *tsk)
  303. {
  304. fpsimd_release_task(tsk);
  305. }
  306. int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
  307. {
  308. if (current->mm)
  309. fpsimd_preserve_current_state();
  310. *dst = *src;
  311. /* We rely on the above assignment to initialize dst's thread_flags: */
  312. BUILD_BUG_ON(!IS_ENABLED(CONFIG_THREAD_INFO_IN_TASK));
  313. /*
  314. * Detach src's sve_state (if any) from dst so that it does not
  315. * get erroneously used or freed prematurely. dst's sve_state
  316. * will be allocated on demand later on if dst uses SVE.
  317. * For consistency, also clear TIF_SVE here: this could be done
  318. * later in copy_process(), but to avoid tripping up future
  319. * maintainers it is best not to leave TIF_SVE and sve_state in
  320. * an inconsistent state, even temporarily.
  321. */
  322. dst->thread.sve_state = NULL;
  323. clear_tsk_thread_flag(dst, TIF_SVE);
  324. /* clear any pending asynchronous tag fault raised by the parent */
  325. clear_tsk_thread_flag(dst, TIF_MTE_ASYNC_FAULT);
  326. return 0;
  327. }
  328. asmlinkage void ret_from_fork(void) asm("ret_from_fork");
  329. int copy_thread(unsigned long clone_flags, unsigned long stack_start,
  330. unsigned long stk_sz, struct task_struct *p, unsigned long tls)
  331. {
  332. struct pt_regs *childregs = task_pt_regs(p);
  333. memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context));
  334. /*
  335. * In case p was allocated the same task_struct pointer as some
  336. * other recently-exited task, make sure p is disassociated from
  337. * any cpu that may have run that now-exited task recently.
  338. * Otherwise we could erroneously skip reloading the FPSIMD
  339. * registers for p.
  340. */
  341. fpsimd_flush_task_state(p);
  342. ptrauth_thread_init_kernel(p);
  343. if (likely(!(p->flags & PF_KTHREAD))) {
  344. *childregs = *current_pt_regs();
  345. childregs->regs[0] = 0;
  346. /*
  347. * Read the current TLS pointer from tpidr_el0 as it may be
  348. * out-of-sync with the saved value.
  349. */
  350. *task_user_tls(p) = read_sysreg(tpidr_el0);
  351. if (stack_start) {
  352. if (is_compat_thread(task_thread_info(p)))
  353. childregs->compat_sp = stack_start;
  354. else
  355. childregs->sp = stack_start;
  356. }
  357. /*
  358. * If a TLS pointer was passed to clone, use it for the new
  359. * thread.
  360. */
  361. if (clone_flags & CLONE_SETTLS)
  362. p->thread.uw.tp_value = tls;
  363. } else {
  364. /*
  365. * A kthread has no context to ERET to, so ensure any buggy
  366. * ERET is treated as an illegal exception return.
  367. *
  368. * When a user task is created from a kthread, childregs will
  369. * be initialized by start_thread() or start_compat_thread().
  370. */
  371. memset(childregs, 0, sizeof(struct pt_regs));
  372. childregs->pstate = PSR_MODE_EL1h | PSR_IL_BIT;
  373. p->thread.cpu_context.x19 = stack_start;
  374. p->thread.cpu_context.x20 = stk_sz;
  375. }
  376. p->thread.cpu_context.pc = (unsigned long)ret_from_fork;
  377. p->thread.cpu_context.sp = (unsigned long)childregs;
  378. ptrace_hw_copy_thread(p);
  379. return 0;
  380. }
  381. void tls_preserve_current_state(void)
  382. {
  383. *task_user_tls(current) = read_sysreg(tpidr_el0);
  384. }
  385. static void tls_thread_switch(struct task_struct *next)
  386. {
  387. tls_preserve_current_state();
  388. if (is_compat_thread(task_thread_info(next)))
  389. write_sysreg(next->thread.uw.tp_value, tpidrro_el0);
  390. else if (!arm64_kernel_unmapped_at_el0())
  391. write_sysreg(0, tpidrro_el0);
  392. write_sysreg(*task_user_tls(next), tpidr_el0);
  393. }
  394. /* Restore the UAO state depending on next's addr_limit */
  395. void uao_thread_switch(struct task_struct *next)
  396. {
  397. if (IS_ENABLED(CONFIG_ARM64_UAO)) {
  398. if (task_thread_info(next)->addr_limit == KERNEL_DS)
  399. asm(ALTERNATIVE("nop", SET_PSTATE_UAO(1), ARM64_HAS_UAO));
  400. else
  401. asm(ALTERNATIVE("nop", SET_PSTATE_UAO(0), ARM64_HAS_UAO));
  402. }
  403. }
  404. /*
  405. * Force SSBS state on context-switch, since it may be lost after migrating
  406. * from a CPU which treats the bit as RES0 in a heterogeneous system.
  407. */
  408. static void ssbs_thread_switch(struct task_struct *next)
  409. {
  410. /*
  411. * Nothing to do for kernel threads, but 'regs' may be junk
  412. * (e.g. idle task) so check the flags and bail early.
  413. */
  414. if (unlikely(next->flags & PF_KTHREAD))
  415. return;
  416. /*
  417. * If all CPUs implement the SSBS extension, then we just need to
  418. * context-switch the PSTATE field.
  419. */
  420. if (cpus_have_const_cap(ARM64_SSBS))
  421. return;
  422. spectre_v4_enable_task_mitigation(next);
  423. }
  424. /*
  425. * We store our current task in sp_el0, which is clobbered by userspace. Keep a
  426. * shadow copy so that we can restore this upon entry from userspace.
  427. *
  428. * This is *only* for exception entry from EL0, and is not valid until we
  429. * __switch_to() a user task.
  430. */
  431. DEFINE_PER_CPU(struct task_struct *, __entry_task);
  432. static void entry_task_switch(struct task_struct *next)
  433. {
  434. __this_cpu_write(__entry_task, next);
  435. }
  436. /*
  437. * ARM erratum 1418040 handling, affecting the 32bit view of CNTVCT.
  438. * Ensure access is disabled when switching to a 32bit task, ensure
  439. * access is enabled when switching to a 64bit task.
  440. */
  441. static void erratum_1418040_thread_switch(struct task_struct *next)
  442. {
  443. if (!IS_ENABLED(CONFIG_ARM64_ERRATUM_1418040) ||
  444. !this_cpu_has_cap(ARM64_WORKAROUND_1418040))
  445. return;
  446. if (is_compat_thread(task_thread_info(next)))
  447. sysreg_clear_set(cntkctl_el1, ARCH_TIMER_USR_VCT_ACCESS_EN, 0);
  448. else
  449. sysreg_clear_set(cntkctl_el1, 0, ARCH_TIMER_USR_VCT_ACCESS_EN);
  450. }
  451. static void erratum_1418040_new_exec(void)
  452. {
  453. preempt_disable();
  454. erratum_1418040_thread_switch(current);
  455. preempt_enable();
  456. }
  457. /*
  458. * __switch_to() checks current->thread.sctlr_user as an optimisation. Therefore
  459. * this function must be called with preemption disabled and the update to
  460. * sctlr_user must be made in the same preemption disabled block so that
  461. * __switch_to() does not see the variable update before the SCTLR_EL1 one.
  462. */
  463. void update_sctlr_el1(u64 sctlr)
  464. {
  465. /*
  466. * EnIA must not be cleared while in the kernel as this is necessary for
  467. * in-kernel PAC. It will be cleared on kernel exit if needed.
  468. */
  469. sysreg_clear_set(sctlr_el1, SCTLR_USER_MASK & ~SCTLR_ELx_ENIA, sctlr);
  470. /* ISB required for the kernel uaccess routines when setting TCF0. */
  471. isb();
  472. }
  473. /*
  474. * Thread switching.
  475. */
  476. __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,
  477. struct task_struct *next)
  478. {
  479. struct task_struct *last;
  480. fpsimd_thread_switch(next);
  481. tls_thread_switch(next);
  482. hw_breakpoint_thread_switch(next);
  483. contextidr_thread_switch(next);
  484. entry_task_switch(next);
  485. uao_thread_switch(next);
  486. ssbs_thread_switch(next);
  487. erratum_1418040_thread_switch(next);
  488. ptrauth_thread_switch_user(next);
  489. /*
  490. * Complete any pending TLB or cache maintenance on this CPU in case
  491. * the thread migrates to a different CPU.
  492. * This full barrier is also required by the membarrier system
  493. * call.
  494. */
  495. dsb(ish);
  496. /*
  497. * MTE thread switching must happen after the DSB above to ensure that
  498. * any asynchronous tag check faults have been logged in the TFSR*_EL1
  499. * registers.
  500. */
  501. mte_thread_switch(next);
  502. /* avoid expensive SCTLR_EL1 accesses if no change */
  503. if (prev->thread.sctlr_user != next->thread.sctlr_user)
  504. update_sctlr_el1(next->thread.sctlr_user);
  505. trace_android_vh_is_fpsimd_save(prev, next);
  506. /* the actual thread switch */
  507. last = cpu_switch_to(prev, next);
  508. return last;
  509. }
  510. unsigned long get_wchan(struct task_struct *p)
  511. {
  512. struct stackframe frame;
  513. unsigned long stack_page, ret = 0;
  514. int count = 0;
  515. if (!p || p == current || p->state == TASK_RUNNING)
  516. return 0;
  517. stack_page = (unsigned long)try_get_task_stack(p);
  518. if (!stack_page)
  519. return 0;
  520. start_backtrace(&frame, thread_saved_fp(p), thread_saved_pc(p));
  521. do {
  522. if (unwind_frame(p, &frame))
  523. goto out;
  524. if (!in_sched_functions(frame.pc)) {
  525. ret = frame.pc;
  526. goto out;
  527. }
  528. } while (count ++ < 16);
  529. out:
  530. put_task_stack(p);
  531. return ret;
  532. }
  533. EXPORT_SYMBOL_GPL(get_wchan);
  534. unsigned long arch_align_stack(unsigned long sp)
  535. {
  536. if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
  537. sp -= get_random_int() & ~PAGE_MASK;
  538. return sp & ~0xf;
  539. }
  540. /*
  541. * Called from setup_new_exec() after (COMPAT_)SET_PERSONALITY.
  542. */
  543. void arch_setup_new_exec(void)
  544. {
  545. unsigned long mmflags = 0;
  546. if (is_compat_task()) {
  547. mmflags = MMCF_AARCH32;
  548. /*
  549. * Restrict the CPU affinity mask for a 32-bit task so that
  550. * it contains only 32-bit-capable CPUs.
  551. *
  552. * From the perspective of the task, this looks similar to
  553. * what would happen if the 64-bit-only CPUs were hot-unplugged
  554. * at the point of execve(), although we try a bit harder to
  555. * honour the cpuset hierarchy.
  556. */
  557. if (static_branch_unlikely(&arm64_mismatched_32bit_el0))
  558. force_compatible_cpus_allowed_ptr(current);
  559. }
  560. current->mm->context.flags = mmflags;
  561. ptrauth_thread_init_user();
  562. mte_thread_init_user();
  563. erratum_1418040_new_exec();
  564. if (task_spec_ssb_noexec(current)) {
  565. arch_prctl_spec_ctrl_set(current, PR_SPEC_STORE_BYPASS,
  566. PR_SPEC_ENABLE);
  567. }
  568. }
  569. #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI
  570. /*
  571. * Control the relaxed ABI allowing tagged user addresses into the kernel.
  572. */
  573. static unsigned int tagged_addr_disabled;
  574. long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg)
  575. {
  576. unsigned long valid_mask = PR_TAGGED_ADDR_ENABLE;
  577. struct thread_info *ti = task_thread_info(task);
  578. if (is_compat_thread(ti))
  579. return -EINVAL;
  580. if (system_supports_mte())
  581. valid_mask |= PR_MTE_TCF_MASK | PR_MTE_TAG_MASK;
  582. if (arg & ~valid_mask)
  583. return -EINVAL;
  584. /*
  585. * Do not allow the enabling of the tagged address ABI if globally
  586. * disabled via sysctl abi.tagged_addr_disabled.
  587. */
  588. if (arg & PR_TAGGED_ADDR_ENABLE && tagged_addr_disabled)
  589. return -EINVAL;
  590. if (set_mte_ctrl(task, arg) != 0)
  591. return -EINVAL;
  592. update_ti_thread_flag(ti, TIF_TAGGED_ADDR, arg & PR_TAGGED_ADDR_ENABLE);
  593. return 0;
  594. }
  595. long get_tagged_addr_ctrl(struct task_struct *task)
  596. {
  597. long ret = 0;
  598. struct thread_info *ti = task_thread_info(task);
  599. if (is_compat_thread(ti))
  600. return -EINVAL;
  601. if (test_ti_thread_flag(ti, TIF_TAGGED_ADDR))
  602. ret = PR_TAGGED_ADDR_ENABLE;
  603. ret |= get_mte_ctrl(task);
  604. return ret;
  605. }
  606. /*
  607. * Global sysctl to disable the tagged user addresses support. This control
  608. * only prevents the tagged address ABI enabling via prctl() and does not
  609. * disable it for tasks that already opted in to the relaxed ABI.
  610. */
  611. static struct ctl_table tagged_addr_sysctl_table[] = {
  612. {
  613. .procname = "tagged_addr_disabled",
  614. .mode = 0644,
  615. .data = &tagged_addr_disabled,
  616. .maxlen = sizeof(int),
  617. .proc_handler = proc_dointvec_minmax,
  618. .extra1 = SYSCTL_ZERO,
  619. .extra2 = SYSCTL_ONE,
  620. },
  621. { }
  622. };
  623. static int __init tagged_addr_init(void)
  624. {
  625. if (!register_sysctl("abi", tagged_addr_sysctl_table))
  626. return -EINVAL;
  627. return 0;
  628. }
  629. core_initcall(tagged_addr_init);
  630. #endif /* CONFIG_ARM64_TAGGED_ADDR_ABI */
  631. asmlinkage void __sched arm64_preempt_schedule_irq(void)
  632. {
  633. lockdep_assert_irqs_disabled();
  634. /*
  635. * Preempting a task from an IRQ means we leave copies of PSTATE
  636. * on the stack. cpufeature's enable calls may modify PSTATE, but
  637. * resuming one of these preempted tasks would undo those changes.
  638. *
  639. * Only allow a task to be preempted once cpufeatures have been
  640. * enabled.
  641. */
  642. if (system_capabilities_finalized())
  643. preempt_schedule_irq();
  644. }
  645. #ifdef CONFIG_BINFMT_ELF
  646. int arch_elf_adjust_prot(int prot, const struct arch_elf_state *state,
  647. bool has_interp, bool is_interp)
  648. {
  649. /*
  650. * For dynamically linked executables the interpreter is
  651. * responsible for setting PROT_BTI on everything except
  652. * itself.
  653. */
  654. if (is_interp != has_interp)
  655. return prot;
  656. if (!(state->flags & ARM64_ELF_BTI))
  657. return prot;
  658. if (prot & PROT_EXEC)
  659. prot |= PROT_BTI;
  660. return prot;
  661. }
  662. #endif