unaligned_64.c 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * unaligned.c: Unaligned load/store trap handling with special
  4. * cases for the kernel to do them more quickly.
  5. *
  6. * Copyright (C) 1996,2008 David S. Miller (davem@davemloft.net)
  7. * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
  8. */
  9. #include <linux/jiffies.h>
  10. #include <linux/kernel.h>
  11. #include <linux/sched.h>
  12. #include <linux/mm.h>
  13. #include <linux/extable.h>
  14. #include <asm/asi.h>
  15. #include <asm/ptrace.h>
  16. #include <asm/pstate.h>
  17. #include <asm/processor.h>
  18. #include <linux/uaccess.h>
  19. #include <linux/smp.h>
  20. #include <linux/bitops.h>
  21. #include <linux/perf_event.h>
  22. #include <linux/ratelimit.h>
  23. #include <linux/context_tracking.h>
  24. #include <asm/fpumacro.h>
  25. #include <asm/cacheflush.h>
  26. #include <asm/setup.h>
  27. #include "entry.h"
  28. #include "kernel.h"
  29. enum direction {
  30. load, /* ld, ldd, ldh, ldsh */
  31. store, /* st, std, sth, stsh */
  32. both, /* Swap, ldstub, cas, ... */
  33. fpld,
  34. fpst,
  35. invalid,
  36. };
  37. static inline enum direction decode_direction(unsigned int insn)
  38. {
  39. unsigned long tmp = (insn >> 21) & 1;
  40. if (!tmp)
  41. return load;
  42. else {
  43. switch ((insn>>19)&0xf) {
  44. case 15: /* swap* */
  45. return both;
  46. default:
  47. return store;
  48. }
  49. }
  50. }
  51. /* 16 = double-word, 8 = extra-word, 4 = word, 2 = half-word */
  52. static inline int decode_access_size(struct pt_regs *regs, unsigned int insn)
  53. {
  54. unsigned int tmp;
  55. tmp = ((insn >> 19) & 0xf);
  56. if (tmp == 11 || tmp == 14) /* ldx/stx */
  57. return 8;
  58. tmp &= 3;
  59. if (!tmp)
  60. return 4;
  61. else if (tmp == 3)
  62. return 16; /* ldd/std - Although it is actually 8 */
  63. else if (tmp == 2)
  64. return 2;
  65. else {
  66. printk("Impossible unaligned trap. insn=%08x\n", insn);
  67. die_if_kernel("Byte sized unaligned access?!?!", regs);
  68. /* GCC should never warn that control reaches the end
  69. * of this function without returning a value because
  70. * die_if_kernel() is marked with attribute 'noreturn'.
  71. * Alas, some versions do...
  72. */
  73. return 0;
  74. }
  75. }
  76. static inline int decode_asi(unsigned int insn, struct pt_regs *regs)
  77. {
  78. if (insn & 0x800000) {
  79. if (insn & 0x2000)
  80. return (unsigned char)(regs->tstate >> 24); /* %asi */
  81. else
  82. return (unsigned char)(insn >> 5); /* imm_asi */
  83. } else
  84. return ASI_P;
  85. }
  86. /* 0x400000 = signed, 0 = unsigned */
  87. static inline int decode_signedness(unsigned int insn)
  88. {
  89. return (insn & 0x400000);
  90. }
  91. static inline void maybe_flush_windows(unsigned int rs1, unsigned int rs2,
  92. unsigned int rd, int from_kernel)
  93. {
  94. if (rs2 >= 16 || rs1 >= 16 || rd >= 16) {
  95. if (from_kernel != 0)
  96. __asm__ __volatile__("flushw");
  97. else
  98. flushw_user();
  99. }
  100. }
  101. static inline long sign_extend_imm13(long imm)
  102. {
  103. return imm << 51 >> 51;
  104. }
  105. static unsigned long fetch_reg(unsigned int reg, struct pt_regs *regs)
  106. {
  107. unsigned long value, fp;
  108. if (reg < 16)
  109. return (!reg ? 0 : regs->u_regs[reg]);
  110. fp = regs->u_regs[UREG_FP];
  111. if (regs->tstate & TSTATE_PRIV) {
  112. struct reg_window *win;
  113. win = (struct reg_window *)(fp + STACK_BIAS);
  114. value = win->locals[reg - 16];
  115. } else if (!test_thread_64bit_stack(fp)) {
  116. struct reg_window32 __user *win32;
  117. win32 = (struct reg_window32 __user *)((unsigned long)((u32)fp));
  118. get_user(value, &win32->locals[reg - 16]);
  119. } else {
  120. struct reg_window __user *win;
  121. win = (struct reg_window __user *)(fp + STACK_BIAS);
  122. get_user(value, &win->locals[reg - 16]);
  123. }
  124. return value;
  125. }
  126. static unsigned long *fetch_reg_addr(unsigned int reg, struct pt_regs *regs)
  127. {
  128. unsigned long fp;
  129. if (reg < 16)
  130. return &regs->u_regs[reg];
  131. fp = regs->u_regs[UREG_FP];
  132. if (regs->tstate & TSTATE_PRIV) {
  133. struct reg_window *win;
  134. win = (struct reg_window *)(fp + STACK_BIAS);
  135. return &win->locals[reg - 16];
  136. } else if (!test_thread_64bit_stack(fp)) {
  137. struct reg_window32 *win32;
  138. win32 = (struct reg_window32 *)((unsigned long)((u32)fp));
  139. return (unsigned long *)&win32->locals[reg - 16];
  140. } else {
  141. struct reg_window *win;
  142. win = (struct reg_window *)(fp + STACK_BIAS);
  143. return &win->locals[reg - 16];
  144. }
  145. }
  146. unsigned long compute_effective_address(struct pt_regs *regs,
  147. unsigned int insn, unsigned int rd)
  148. {
  149. int from_kernel = (regs->tstate & TSTATE_PRIV) != 0;
  150. unsigned int rs1 = (insn >> 14) & 0x1f;
  151. unsigned int rs2 = insn & 0x1f;
  152. unsigned long addr;
  153. if (insn & 0x2000) {
  154. maybe_flush_windows(rs1, 0, rd, from_kernel);
  155. addr = (fetch_reg(rs1, regs) + sign_extend_imm13(insn));
  156. } else {
  157. maybe_flush_windows(rs1, rs2, rd, from_kernel);
  158. addr = (fetch_reg(rs1, regs) + fetch_reg(rs2, regs));
  159. }
  160. if (!from_kernel && test_thread_flag(TIF_32BIT))
  161. addr &= 0xffffffff;
  162. return addr;
  163. }
  164. /* This is just to make gcc think die_if_kernel does return... */
  165. static void __used unaligned_panic(char *str, struct pt_regs *regs)
  166. {
  167. die_if_kernel(str, regs);
  168. }
  169. extern int do_int_load(unsigned long *dest_reg, int size,
  170. unsigned long *saddr, int is_signed, int asi);
  171. extern int __do_int_store(unsigned long *dst_addr, int size,
  172. unsigned long src_val, int asi);
  173. static inline int do_int_store(int reg_num, int size, unsigned long *dst_addr,
  174. struct pt_regs *regs, int asi, int orig_asi)
  175. {
  176. unsigned long zero = 0;
  177. unsigned long *src_val_p = &zero;
  178. unsigned long src_val;
  179. if (size == 16) {
  180. size = 8;
  181. zero = (((long)(reg_num ?
  182. (unsigned int)fetch_reg(reg_num, regs) : 0)) << 32) |
  183. (unsigned int)fetch_reg(reg_num + 1, regs);
  184. } else if (reg_num) {
  185. src_val_p = fetch_reg_addr(reg_num, regs);
  186. }
  187. src_val = *src_val_p;
  188. if (unlikely(asi != orig_asi)) {
  189. switch (size) {
  190. case 2:
  191. src_val = swab16(src_val);
  192. break;
  193. case 4:
  194. src_val = swab32(src_val);
  195. break;
  196. case 8:
  197. src_val = swab64(src_val);
  198. break;
  199. case 16:
  200. default:
  201. BUG();
  202. break;
  203. }
  204. }
  205. return __do_int_store(dst_addr, size, src_val, asi);
  206. }
  207. static inline void advance(struct pt_regs *regs)
  208. {
  209. regs->tpc = regs->tnpc;
  210. regs->tnpc += 4;
  211. if (test_thread_flag(TIF_32BIT)) {
  212. regs->tpc &= 0xffffffff;
  213. regs->tnpc &= 0xffffffff;
  214. }
  215. }
  216. static inline int floating_point_load_or_store_p(unsigned int insn)
  217. {
  218. return (insn >> 24) & 1;
  219. }
  220. static inline int ok_for_kernel(unsigned int insn)
  221. {
  222. return !floating_point_load_or_store_p(insn);
  223. }
  224. static void kernel_mna_trap_fault(int fixup_tstate_asi)
  225. {
  226. struct pt_regs *regs = current_thread_info()->kern_una_regs;
  227. unsigned int insn = current_thread_info()->kern_una_insn;
  228. const struct exception_table_entry *entry;
  229. entry = search_exception_tables(regs->tpc);
  230. if (!entry) {
  231. unsigned long address;
  232. address = compute_effective_address(regs, insn,
  233. ((insn >> 25) & 0x1f));
  234. if (address < PAGE_SIZE) {
  235. printk(KERN_ALERT "Unable to handle kernel NULL "
  236. "pointer dereference in mna handler");
  237. } else
  238. printk(KERN_ALERT "Unable to handle kernel paging "
  239. "request in mna handler");
  240. printk(KERN_ALERT " at virtual address %016lx\n",address);
  241. printk(KERN_ALERT "current->{active_,}mm->context = %016lx\n",
  242. (current->mm ? CTX_HWBITS(current->mm->context) :
  243. CTX_HWBITS(current->active_mm->context)));
  244. printk(KERN_ALERT "current->{active_,}mm->pgd = %016lx\n",
  245. (current->mm ? (unsigned long) current->mm->pgd :
  246. (unsigned long) current->active_mm->pgd));
  247. die_if_kernel("Oops", regs);
  248. /* Not reached */
  249. }
  250. regs->tpc = entry->fixup;
  251. regs->tnpc = regs->tpc + 4;
  252. if (fixup_tstate_asi) {
  253. regs->tstate &= ~TSTATE_ASI;
  254. regs->tstate |= (ASI_AIUS << 24UL);
  255. }
  256. }
  257. static void log_unaligned(struct pt_regs *regs)
  258. {
  259. static DEFINE_RATELIMIT_STATE(ratelimit, 5 * HZ, 5);
  260. if (__ratelimit(&ratelimit)) {
  261. printk("Kernel unaligned access at TPC[%lx] %pS\n",
  262. regs->tpc, (void *) regs->tpc);
  263. }
  264. }
  265. asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn)
  266. {
  267. enum direction dir = decode_direction(insn);
  268. int size = decode_access_size(regs, insn);
  269. int orig_asi, asi;
  270. current_thread_info()->kern_una_regs = regs;
  271. current_thread_info()->kern_una_insn = insn;
  272. orig_asi = asi = decode_asi(insn, regs);
  273. /* If this is a {get,put}_user() on an unaligned userspace pointer,
  274. * just signal a fault and do not log the event.
  275. */
  276. if (asi == ASI_AIUS) {
  277. kernel_mna_trap_fault(0);
  278. return;
  279. }
  280. log_unaligned(regs);
  281. if (!ok_for_kernel(insn) || dir == both) {
  282. printk("Unsupported unaligned load/store trap for kernel "
  283. "at <%016lx>.\n", regs->tpc);
  284. unaligned_panic("Kernel does fpu/atomic "
  285. "unaligned load/store.", regs);
  286. kernel_mna_trap_fault(0);
  287. } else {
  288. unsigned long addr, *reg_addr;
  289. int err;
  290. addr = compute_effective_address(regs, insn,
  291. ((insn >> 25) & 0x1f));
  292. perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr);
  293. switch (asi) {
  294. case ASI_NL:
  295. case ASI_AIUPL:
  296. case ASI_AIUSL:
  297. case ASI_PL:
  298. case ASI_SL:
  299. case ASI_PNFL:
  300. case ASI_SNFL:
  301. asi &= ~0x08;
  302. break;
  303. }
  304. switch (dir) {
  305. case load:
  306. reg_addr = fetch_reg_addr(((insn>>25)&0x1f), regs);
  307. err = do_int_load(reg_addr, size,
  308. (unsigned long *) addr,
  309. decode_signedness(insn), asi);
  310. if (likely(!err) && unlikely(asi != orig_asi)) {
  311. unsigned long val_in = *reg_addr;
  312. switch (size) {
  313. case 2:
  314. val_in = swab16(val_in);
  315. break;
  316. case 4:
  317. val_in = swab32(val_in);
  318. break;
  319. case 8:
  320. val_in = swab64(val_in);
  321. break;
  322. case 16:
  323. default:
  324. BUG();
  325. break;
  326. }
  327. *reg_addr = val_in;
  328. }
  329. break;
  330. case store:
  331. err = do_int_store(((insn>>25)&0x1f), size,
  332. (unsigned long *) addr, regs,
  333. asi, orig_asi);
  334. break;
  335. default:
  336. panic("Impossible kernel unaligned trap.");
  337. /* Not reached... */
  338. }
  339. if (unlikely(err))
  340. kernel_mna_trap_fault(1);
  341. else
  342. advance(regs);
  343. }
  344. }
  345. int handle_popc(u32 insn, struct pt_regs *regs)
  346. {
  347. int from_kernel = (regs->tstate & TSTATE_PRIV) != 0;
  348. int ret, rd = ((insn >> 25) & 0x1f);
  349. u64 value;
  350. perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0);
  351. if (insn & 0x2000) {
  352. maybe_flush_windows(0, 0, rd, from_kernel);
  353. value = sign_extend_imm13(insn);
  354. } else {
  355. maybe_flush_windows(0, insn & 0x1f, rd, from_kernel);
  356. value = fetch_reg(insn & 0x1f, regs);
  357. }
  358. ret = hweight64(value);
  359. if (rd < 16) {
  360. if (rd)
  361. regs->u_regs[rd] = ret;
  362. } else {
  363. unsigned long fp = regs->u_regs[UREG_FP];
  364. if (!test_thread_64bit_stack(fp)) {
  365. struct reg_window32 __user *win32;
  366. win32 = (struct reg_window32 __user *)((unsigned long)((u32)fp));
  367. put_user(ret, &win32->locals[rd - 16]);
  368. } else {
  369. struct reg_window __user *win;
  370. win = (struct reg_window __user *)(fp + STACK_BIAS);
  371. put_user(ret, &win->locals[rd - 16]);
  372. }
  373. }
  374. advance(regs);
  375. return 1;
  376. }
  377. extern void do_fpother(struct pt_regs *regs);
  378. extern void do_privact(struct pt_regs *regs);
  379. extern void sun4v_data_access_exception(struct pt_regs *regs,
  380. unsigned long addr,
  381. unsigned long type_ctx);
  382. int handle_ldf_stq(u32 insn, struct pt_regs *regs)
  383. {
  384. unsigned long addr = compute_effective_address(regs, insn, 0);
  385. int freg;
  386. struct fpustate *f = FPUSTATE;
  387. int asi = decode_asi(insn, regs);
  388. int flag;
  389. perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0);
  390. save_and_clear_fpu();
  391. current_thread_info()->xfsr[0] &= ~0x1c000;
  392. if (insn & 0x200000) {
  393. /* STQ */
  394. u64 first = 0, second = 0;
  395. freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20);
  396. flag = (freg < 32) ? FPRS_DL : FPRS_DU;
  397. if (freg & 3) {
  398. current_thread_info()->xfsr[0] |= (6 << 14) /* invalid_fp_register */;
  399. do_fpother(regs);
  400. return 0;
  401. }
  402. if (current_thread_info()->fpsaved[0] & flag) {
  403. first = *(u64 *)&f->regs[freg];
  404. second = *(u64 *)&f->regs[freg+2];
  405. }
  406. if (asi < 0x80) {
  407. do_privact(regs);
  408. return 1;
  409. }
  410. switch (asi) {
  411. case ASI_P:
  412. case ASI_S: break;
  413. case ASI_PL:
  414. case ASI_SL:
  415. {
  416. /* Need to convert endians */
  417. u64 tmp = __swab64p(&first);
  418. first = __swab64p(&second);
  419. second = tmp;
  420. break;
  421. }
  422. default:
  423. if (tlb_type == hypervisor)
  424. sun4v_data_access_exception(regs, addr, 0);
  425. else
  426. spitfire_data_access_exception(regs, 0, addr);
  427. return 1;
  428. }
  429. if (put_user (first >> 32, (u32 __user *)addr) ||
  430. __put_user ((u32)first, (u32 __user *)(addr + 4)) ||
  431. __put_user (second >> 32, (u32 __user *)(addr + 8)) ||
  432. __put_user ((u32)second, (u32 __user *)(addr + 12))) {
  433. if (tlb_type == hypervisor)
  434. sun4v_data_access_exception(regs, addr, 0);
  435. else
  436. spitfire_data_access_exception(regs, 0, addr);
  437. return 1;
  438. }
  439. } else {
  440. /* LDF, LDDF, LDQF */
  441. u32 data[4] __attribute__ ((aligned(8)));
  442. int size, i;
  443. int err;
  444. if (asi < 0x80) {
  445. do_privact(regs);
  446. return 1;
  447. } else if (asi > ASI_SNFL) {
  448. if (tlb_type == hypervisor)
  449. sun4v_data_access_exception(regs, addr, 0);
  450. else
  451. spitfire_data_access_exception(regs, 0, addr);
  452. return 1;
  453. }
  454. switch (insn & 0x180000) {
  455. case 0x000000: size = 1; break;
  456. case 0x100000: size = 4; break;
  457. default: size = 2; break;
  458. }
  459. if (size == 1)
  460. freg = (insn >> 25) & 0x1f;
  461. else
  462. freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20);
  463. flag = (freg < 32) ? FPRS_DL : FPRS_DU;
  464. for (i = 0; i < size; i++)
  465. data[i] = 0;
  466. err = get_user (data[0], (u32 __user *) addr);
  467. if (!err) {
  468. for (i = 1; i < size; i++)
  469. err |= __get_user (data[i], (u32 __user *)(addr + 4*i));
  470. }
  471. if (err && !(asi & 0x2 /* NF */)) {
  472. if (tlb_type == hypervisor)
  473. sun4v_data_access_exception(regs, addr, 0);
  474. else
  475. spitfire_data_access_exception(regs, 0, addr);
  476. return 1;
  477. }
  478. if (asi & 0x8) /* Little */ {
  479. u64 tmp;
  480. switch (size) {
  481. case 1: data[0] = le32_to_cpup(data + 0); break;
  482. default:*(u64 *)(data + 0) = le64_to_cpup((u64 *)(data + 0));
  483. break;
  484. case 4: tmp = le64_to_cpup((u64 *)(data + 0));
  485. *(u64 *)(data + 0) = le64_to_cpup((u64 *)(data + 2));
  486. *(u64 *)(data + 2) = tmp;
  487. break;
  488. }
  489. }
  490. if (!(current_thread_info()->fpsaved[0] & FPRS_FEF)) {
  491. current_thread_info()->fpsaved[0] = FPRS_FEF;
  492. current_thread_info()->gsr[0] = 0;
  493. }
  494. if (!(current_thread_info()->fpsaved[0] & flag)) {
  495. if (freg < 32)
  496. memset(f->regs, 0, 32*sizeof(u32));
  497. else
  498. memset(f->regs+32, 0, 32*sizeof(u32));
  499. }
  500. memcpy(f->regs + freg, data, size * 4);
  501. current_thread_info()->fpsaved[0] |= flag;
  502. }
  503. advance(regs);
  504. return 1;
  505. }
  506. void handle_ld_nf(u32 insn, struct pt_regs *regs)
  507. {
  508. int rd = ((insn >> 25) & 0x1f);
  509. int from_kernel = (regs->tstate & TSTATE_PRIV) != 0;
  510. unsigned long *reg;
  511. perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0);
  512. maybe_flush_windows(0, 0, rd, from_kernel);
  513. reg = fetch_reg_addr(rd, regs);
  514. if (from_kernel || rd < 16) {
  515. reg[0] = 0;
  516. if ((insn & 0x780000) == 0x180000)
  517. reg[1] = 0;
  518. } else if (!test_thread_64bit_stack(regs->u_regs[UREG_FP])) {
  519. put_user(0, (int __user *) reg);
  520. if ((insn & 0x780000) == 0x180000)
  521. put_user(0, ((int __user *) reg) + 1);
  522. } else {
  523. put_user(0, (unsigned long __user *) reg);
  524. if ((insn & 0x780000) == 0x180000)
  525. put_user(0, (unsigned long __user *) reg + 1);
  526. }
  527. advance(regs);
  528. }
  529. void handle_lddfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr)
  530. {
  531. enum ctx_state prev_state = exception_enter();
  532. unsigned long pc = regs->tpc;
  533. unsigned long tstate = regs->tstate;
  534. u32 insn;
  535. u64 value;
  536. u8 freg;
  537. int flag;
  538. struct fpustate *f = FPUSTATE;
  539. if (tstate & TSTATE_PRIV)
  540. die_if_kernel("lddfmna from kernel", regs);
  541. perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, sfar);
  542. if (test_thread_flag(TIF_32BIT))
  543. pc = (u32)pc;
  544. if (get_user(insn, (u32 __user *) pc) != -EFAULT) {
  545. int asi = decode_asi(insn, regs);
  546. u32 first, second;
  547. int err;
  548. if ((asi > ASI_SNFL) ||
  549. (asi < ASI_P))
  550. goto daex;
  551. first = second = 0;
  552. err = get_user(first, (u32 __user *)sfar);
  553. if (!err)
  554. err = get_user(second, (u32 __user *)(sfar + 4));
  555. if (err) {
  556. if (!(asi & 0x2))
  557. goto daex;
  558. first = second = 0;
  559. }
  560. save_and_clear_fpu();
  561. freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20);
  562. value = (((u64)first) << 32) | second;
  563. if (asi & 0x8) /* Little */
  564. value = __swab64p(&value);
  565. flag = (freg < 32) ? FPRS_DL : FPRS_DU;
  566. if (!(current_thread_info()->fpsaved[0] & FPRS_FEF)) {
  567. current_thread_info()->fpsaved[0] = FPRS_FEF;
  568. current_thread_info()->gsr[0] = 0;
  569. }
  570. if (!(current_thread_info()->fpsaved[0] & flag)) {
  571. if (freg < 32)
  572. memset(f->regs, 0, 32*sizeof(u32));
  573. else
  574. memset(f->regs+32, 0, 32*sizeof(u32));
  575. }
  576. *(u64 *)(f->regs + freg) = value;
  577. current_thread_info()->fpsaved[0] |= flag;
  578. } else {
  579. daex:
  580. if (tlb_type == hypervisor)
  581. sun4v_data_access_exception(regs, sfar, sfsr);
  582. else
  583. spitfire_data_access_exception(regs, sfsr, sfar);
  584. goto out;
  585. }
  586. advance(regs);
  587. out:
  588. exception_exit(prev_state);
  589. }
  590. void handle_stdfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr)
  591. {
  592. enum ctx_state prev_state = exception_enter();
  593. unsigned long pc = regs->tpc;
  594. unsigned long tstate = regs->tstate;
  595. u32 insn;
  596. u64 value;
  597. u8 freg;
  598. int flag;
  599. struct fpustate *f = FPUSTATE;
  600. if (tstate & TSTATE_PRIV)
  601. die_if_kernel("stdfmna from kernel", regs);
  602. perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, sfar);
  603. if (test_thread_flag(TIF_32BIT))
  604. pc = (u32)pc;
  605. if (get_user(insn, (u32 __user *) pc) != -EFAULT) {
  606. int asi = decode_asi(insn, regs);
  607. freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20);
  608. value = 0;
  609. flag = (freg < 32) ? FPRS_DL : FPRS_DU;
  610. if ((asi > ASI_SNFL) ||
  611. (asi < ASI_P))
  612. goto daex;
  613. save_and_clear_fpu();
  614. if (current_thread_info()->fpsaved[0] & flag)
  615. value = *(u64 *)&f->regs[freg];
  616. switch (asi) {
  617. case ASI_P:
  618. case ASI_S: break;
  619. case ASI_PL:
  620. case ASI_SL:
  621. value = __swab64p(&value); break;
  622. default: goto daex;
  623. }
  624. if (put_user (value >> 32, (u32 __user *) sfar) ||
  625. __put_user ((u32)value, (u32 __user *)(sfar + 4)))
  626. goto daex;
  627. } else {
  628. daex:
  629. if (tlb_type == hypervisor)
  630. sun4v_data_access_exception(regs, sfar, sfsr);
  631. else
  632. spitfire_data_access_exception(regs, sfsr, sfar);
  633. goto out;
  634. }
  635. advance(regs);
  636. out:
  637. exception_exit(prev_state);
  638. }