vfpdouble.c 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197
  1. /*
  2. * linux/arch/arm/vfp/vfpdouble.c
  3. *
  4. * This code is derived in part from John R. Housers softfloat library, which
  5. * carries the following notice:
  6. *
  7. * ===========================================================================
  8. * This C source file is part of the SoftFloat IEC/IEEE Floating-point
  9. * Arithmetic Package, Release 2.
  10. *
  11. * Written by John R. Hauser. This work was made possible in part by the
  12. * International Computer Science Institute, located at Suite 600, 1947 Center
  13. * Street, Berkeley, California 94704. Funding was partially provided by the
  14. * National Science Foundation under grant MIP-9311980. The original version
  15. * of this code was written as part of a project to build a fixed-point vector
  16. * processor in collaboration with the University of California at Berkeley,
  17. * overseen by Profs. Nelson Morgan and John Wawrzynek. More information
  18. * is available through the web page `http://HTTP.CS.Berkeley.EDU/~jhauser/
  19. * arithmetic/softfloat.html'.
  20. *
  21. * THIS SOFTWARE IS DISTRIBUTED AS IS, FOR FREE. Although reasonable effort
  22. * has been made to avoid it, THIS SOFTWARE MAY CONTAIN FAULTS THAT WILL AT
  23. * TIMES RESULT IN INCORRECT BEHAVIOR. USE OF THIS SOFTWARE IS RESTRICTED TO
  24. * PERSONS AND ORGANIZATIONS WHO CAN AND WILL TAKE FULL RESPONSIBILITY FOR ANY
  25. * AND ALL LOSSES, COSTS, OR OTHER PROBLEMS ARISING FROM ITS USE.
  26. *
  27. * Derivative works are acceptable, even for commercial purposes, so long as
  28. * (1) they include prominent notice that the work is derivative, and (2) they
  29. * include prominent notice akin to these three paragraphs for those parts of
  30. * this code that are retained.
  31. * ===========================================================================
  32. */
  33. #include <linux/kernel.h>
  34. #include <linux/bitops.h>
  35. #include <asm/div64.h>
  36. #include <asm/ptrace.h>
  37. #include <asm/vfp.h>
  38. #include "vfpinstr.h"
  39. #include "vfp.h"
  40. static struct vfp_double vfp_double_default_qnan = {
  41. .exponent = 2047,
  42. .sign = 0,
  43. .significand = VFP_DOUBLE_SIGNIFICAND_QNAN,
  44. };
  45. static void vfp_double_dump(const char *str, struct vfp_double *d)
  46. {
  47. pr_debug("VFP: %s: sign=%d exponent=%d significand=%016llx\n",
  48. str, d->sign != 0, d->exponent, d->significand);
  49. }
  50. static void vfp_double_normalise_denormal(struct vfp_double *vd)
  51. {
  52. int bits = 31 - fls(vd->significand >> 32);
  53. if (bits == 31)
  54. bits = 63 - fls(vd->significand);
  55. vfp_double_dump("normalise_denormal: in", vd);
  56. if (bits) {
  57. vd->exponent -= bits - 1;
  58. vd->significand <<= bits;
  59. }
  60. vfp_double_dump("normalise_denormal: out", vd);
  61. }
  62. u32 vfp_double_normaliseround(int dd, struct vfp_double *vd, u32 fpscr, u32 exceptions, const char *func)
  63. {
  64. u64 significand, incr;
  65. int exponent, shift, underflow;
  66. u32 rmode;
  67. vfp_double_dump("pack: in", vd);
  68. /*
  69. * Infinities and NaNs are a special case.
  70. */
  71. if (vd->exponent == 2047 && (vd->significand == 0 || exceptions))
  72. goto pack;
  73. /*
  74. * Special-case zero.
  75. */
  76. if (vd->significand == 0) {
  77. vd->exponent = 0;
  78. goto pack;
  79. }
  80. exponent = vd->exponent;
  81. significand = vd->significand;
  82. shift = 32 - fls(significand >> 32);
  83. if (shift == 32)
  84. shift = 64 - fls(significand);
  85. if (shift) {
  86. exponent -= shift;
  87. significand <<= shift;
  88. }
  89. #ifdef DEBUG
  90. vd->exponent = exponent;
  91. vd->significand = significand;
  92. vfp_double_dump("pack: normalised", vd);
  93. #endif
  94. /*
  95. * Tiny number?
  96. */
  97. underflow = exponent < 0;
  98. if (underflow) {
  99. significand = vfp_shiftright64jamming(significand, -exponent);
  100. exponent = 0;
  101. #ifdef DEBUG
  102. vd->exponent = exponent;
  103. vd->significand = significand;
  104. vfp_double_dump("pack: tiny number", vd);
  105. #endif
  106. if (!(significand & ((1ULL << (VFP_DOUBLE_LOW_BITS + 1)) - 1)))
  107. underflow = 0;
  108. }
  109. /*
  110. * Select rounding increment.
  111. */
  112. incr = 0;
  113. rmode = fpscr & FPSCR_RMODE_MASK;
  114. if (rmode == FPSCR_ROUND_NEAREST) {
  115. incr = 1ULL << VFP_DOUBLE_LOW_BITS;
  116. if ((significand & (1ULL << (VFP_DOUBLE_LOW_BITS + 1))) == 0)
  117. incr -= 1;
  118. } else if (rmode == FPSCR_ROUND_TOZERO) {
  119. incr = 0;
  120. } else if ((rmode == FPSCR_ROUND_PLUSINF) ^ (vd->sign != 0))
  121. incr = (1ULL << (VFP_DOUBLE_LOW_BITS + 1)) - 1;
  122. pr_debug("VFP: rounding increment = 0x%08llx\n", incr);
  123. /*
  124. * Is our rounding going to overflow?
  125. */
  126. if ((significand + incr) < significand) {
  127. exponent += 1;
  128. significand = (significand >> 1) | (significand & 1);
  129. incr >>= 1;
  130. #ifdef DEBUG
  131. vd->exponent = exponent;
  132. vd->significand = significand;
  133. vfp_double_dump("pack: overflow", vd);
  134. #endif
  135. }
  136. /*
  137. * If any of the low bits (which will be shifted out of the
  138. * number) are non-zero, the result is inexact.
  139. */
  140. if (significand & ((1 << (VFP_DOUBLE_LOW_BITS + 1)) - 1))
  141. exceptions |= FPSCR_IXC;
  142. /*
  143. * Do our rounding.
  144. */
  145. significand += incr;
  146. /*
  147. * Infinity?
  148. */
  149. if (exponent >= 2046) {
  150. exceptions |= FPSCR_OFC | FPSCR_IXC;
  151. if (incr == 0) {
  152. vd->exponent = 2045;
  153. vd->significand = 0x7fffffffffffffffULL;
  154. } else {
  155. vd->exponent = 2047; /* infinity */
  156. vd->significand = 0;
  157. }
  158. } else {
  159. if (significand >> (VFP_DOUBLE_LOW_BITS + 1) == 0)
  160. exponent = 0;
  161. if (exponent || significand > 0x8000000000000000ULL)
  162. underflow = 0;
  163. if (underflow)
  164. exceptions |= FPSCR_UFC;
  165. vd->exponent = exponent;
  166. vd->significand = significand >> 1;
  167. }
  168. pack:
  169. vfp_double_dump("pack: final", vd);
  170. {
  171. s64 d = vfp_double_pack(vd);
  172. pr_debug("VFP: %s: d(d%d)=%016llx exceptions=%08x\n", func,
  173. dd, d, exceptions);
  174. vfp_put_double(d, dd);
  175. }
  176. return exceptions;
  177. }
  178. /*
  179. * Propagate the NaN, setting exceptions if it is signalling.
  180. * 'n' is always a NaN. 'm' may be a number, NaN or infinity.
  181. */
  182. static u32
  183. vfp_propagate_nan(struct vfp_double *vdd, struct vfp_double *vdn,
  184. struct vfp_double *vdm, u32 fpscr)
  185. {
  186. struct vfp_double *nan;
  187. int tn, tm = 0;
  188. tn = vfp_double_type(vdn);
  189. if (vdm)
  190. tm = vfp_double_type(vdm);
  191. if (fpscr & FPSCR_DEFAULT_NAN)
  192. /*
  193. * Default NaN mode - always returns a quiet NaN
  194. */
  195. nan = &vfp_double_default_qnan;
  196. else {
  197. /*
  198. * Contemporary mode - select the first signalling
  199. * NAN, or if neither are signalling, the first
  200. * quiet NAN.
  201. */
  202. if (tn == VFP_SNAN || (tm != VFP_SNAN && tn == VFP_QNAN))
  203. nan = vdn;
  204. else
  205. nan = vdm;
  206. /*
  207. * Make the NaN quiet.
  208. */
  209. nan->significand |= VFP_DOUBLE_SIGNIFICAND_QNAN;
  210. }
  211. *vdd = *nan;
  212. /*
  213. * If one was a signalling NAN, raise invalid operation.
  214. */
  215. return tn == VFP_SNAN || tm == VFP_SNAN ? FPSCR_IOC : VFP_NAN_FLAG;
  216. }
  217. /*
  218. * Extended operations
  219. */
  220. static u32 vfp_double_fabs(int dd, int unused, int dm, u32 fpscr)
  221. {
  222. vfp_put_double(vfp_double_packed_abs(vfp_get_double(dm)), dd);
  223. return 0;
  224. }
  225. static u32 vfp_double_fcpy(int dd, int unused, int dm, u32 fpscr)
  226. {
  227. vfp_put_double(vfp_get_double(dm), dd);
  228. return 0;
  229. }
  230. static u32 vfp_double_fneg(int dd, int unused, int dm, u32 fpscr)
  231. {
  232. vfp_put_double(vfp_double_packed_negate(vfp_get_double(dm)), dd);
  233. return 0;
  234. }
  235. static u32 vfp_double_fsqrt(int dd, int unused, int dm, u32 fpscr)
  236. {
  237. struct vfp_double vdm, vdd;
  238. int ret, tm;
  239. vfp_double_unpack(&vdm, vfp_get_double(dm));
  240. tm = vfp_double_type(&vdm);
  241. if (tm & (VFP_NAN|VFP_INFINITY)) {
  242. struct vfp_double *vdp = &vdd;
  243. if (tm & VFP_NAN)
  244. ret = vfp_propagate_nan(vdp, &vdm, NULL, fpscr);
  245. else if (vdm.sign == 0) {
  246. sqrt_copy:
  247. vdp = &vdm;
  248. ret = 0;
  249. } else {
  250. sqrt_invalid:
  251. vdp = &vfp_double_default_qnan;
  252. ret = FPSCR_IOC;
  253. }
  254. vfp_put_double(vfp_double_pack(vdp), dd);
  255. return ret;
  256. }
  257. /*
  258. * sqrt(+/- 0) == +/- 0
  259. */
  260. if (tm & VFP_ZERO)
  261. goto sqrt_copy;
  262. /*
  263. * Normalise a denormalised number
  264. */
  265. if (tm & VFP_DENORMAL)
  266. vfp_double_normalise_denormal(&vdm);
  267. /*
  268. * sqrt(<0) = invalid
  269. */
  270. if (vdm.sign)
  271. goto sqrt_invalid;
  272. vfp_double_dump("sqrt", &vdm);
  273. /*
  274. * Estimate the square root.
  275. */
  276. vdd.sign = 0;
  277. vdd.exponent = ((vdm.exponent - 1023) >> 1) + 1023;
  278. vdd.significand = (u64)vfp_estimate_sqrt_significand(vdm.exponent, vdm.significand >> 32) << 31;
  279. vfp_double_dump("sqrt estimate1", &vdd);
  280. vdm.significand >>= 1 + (vdm.exponent & 1);
  281. vdd.significand += 2 + vfp_estimate_div128to64(vdm.significand, 0, vdd.significand);
  282. vfp_double_dump("sqrt estimate2", &vdd);
  283. /*
  284. * And now adjust.
  285. */
  286. if ((vdd.significand & VFP_DOUBLE_LOW_BITS_MASK) <= 5) {
  287. if (vdd.significand < 2) {
  288. vdd.significand = ~0ULL;
  289. } else {
  290. u64 termh, terml, remh, reml;
  291. vdm.significand <<= 2;
  292. mul64to128(&termh, &terml, vdd.significand, vdd.significand);
  293. sub128(&remh, &reml, vdm.significand, 0, termh, terml);
  294. while ((s64)remh < 0) {
  295. vdd.significand -= 1;
  296. shift64left(&termh, &terml, vdd.significand);
  297. terml |= 1;
  298. add128(&remh, &reml, remh, reml, termh, terml);
  299. }
  300. vdd.significand |= (remh | reml) != 0;
  301. }
  302. }
  303. vdd.significand = vfp_shiftright64jamming(vdd.significand, 1);
  304. return vfp_double_normaliseround(dd, &vdd, fpscr, 0, "fsqrt");
  305. }
  306. /*
  307. * Equal := ZC
  308. * Less than := N
  309. * Greater than := C
  310. * Unordered := CV
  311. */
  312. static u32 vfp_compare(int dd, int signal_on_qnan, int dm, u32 fpscr)
  313. {
  314. s64 d, m;
  315. u32 ret = 0;
  316. m = vfp_get_double(dm);
  317. if (vfp_double_packed_exponent(m) == 2047 && vfp_double_packed_mantissa(m)) {
  318. ret |= FPSCR_C | FPSCR_V;
  319. if (signal_on_qnan || !(vfp_double_packed_mantissa(m) & (1ULL << (VFP_DOUBLE_MANTISSA_BITS - 1))))
  320. /*
  321. * Signalling NaN, or signalling on quiet NaN
  322. */
  323. ret |= FPSCR_IOC;
  324. }
  325. d = vfp_get_double(dd);
  326. if (vfp_double_packed_exponent(d) == 2047 && vfp_double_packed_mantissa(d)) {
  327. ret |= FPSCR_C | FPSCR_V;
  328. if (signal_on_qnan || !(vfp_double_packed_mantissa(d) & (1ULL << (VFP_DOUBLE_MANTISSA_BITS - 1))))
  329. /*
  330. * Signalling NaN, or signalling on quiet NaN
  331. */
  332. ret |= FPSCR_IOC;
  333. }
  334. if (ret == 0) {
  335. if (d == m || vfp_double_packed_abs(d | m) == 0) {
  336. /*
  337. * equal
  338. */
  339. ret |= FPSCR_Z | FPSCR_C;
  340. } else if (vfp_double_packed_sign(d ^ m)) {
  341. /*
  342. * different signs
  343. */
  344. if (vfp_double_packed_sign(d))
  345. /*
  346. * d is negative, so d < m
  347. */
  348. ret |= FPSCR_N;
  349. else
  350. /*
  351. * d is positive, so d > m
  352. */
  353. ret |= FPSCR_C;
  354. } else if ((vfp_double_packed_sign(d) != 0) ^ (d < m)) {
  355. /*
  356. * d < m
  357. */
  358. ret |= FPSCR_N;
  359. } else if ((vfp_double_packed_sign(d) != 0) ^ (d > m)) {
  360. /*
  361. * d > m
  362. */
  363. ret |= FPSCR_C;
  364. }
  365. }
  366. return ret;
  367. }
  368. static u32 vfp_double_fcmp(int dd, int unused, int dm, u32 fpscr)
  369. {
  370. return vfp_compare(dd, 0, dm, fpscr);
  371. }
  372. static u32 vfp_double_fcmpe(int dd, int unused, int dm, u32 fpscr)
  373. {
  374. return vfp_compare(dd, 1, dm, fpscr);
  375. }
  376. static u32 vfp_double_fcmpz(int dd, int unused, int dm, u32 fpscr)
  377. {
  378. return vfp_compare(dd, 0, VFP_REG_ZERO, fpscr);
  379. }
  380. static u32 vfp_double_fcmpez(int dd, int unused, int dm, u32 fpscr)
  381. {
  382. return vfp_compare(dd, 1, VFP_REG_ZERO, fpscr);
  383. }
  384. static u32 vfp_double_fcvts(int sd, int unused, int dm, u32 fpscr)
  385. {
  386. struct vfp_double vdm;
  387. struct vfp_single vsd;
  388. int tm;
  389. u32 exceptions = 0;
  390. vfp_double_unpack(&vdm, vfp_get_double(dm));
  391. tm = vfp_double_type(&vdm);
  392. /*
  393. * If we have a signalling NaN, signal invalid operation.
  394. */
  395. if (tm == VFP_SNAN)
  396. exceptions = FPSCR_IOC;
  397. if (tm & VFP_DENORMAL)
  398. vfp_double_normalise_denormal(&vdm);
  399. vsd.sign = vdm.sign;
  400. vsd.significand = vfp_hi64to32jamming(vdm.significand);
  401. /*
  402. * If we have an infinity or a NaN, the exponent must be 255
  403. */
  404. if (tm & (VFP_INFINITY|VFP_NAN)) {
  405. vsd.exponent = 255;
  406. if (tm == VFP_QNAN)
  407. vsd.significand |= VFP_SINGLE_SIGNIFICAND_QNAN;
  408. goto pack_nan;
  409. } else if (tm & VFP_ZERO)
  410. vsd.exponent = 0;
  411. else
  412. vsd.exponent = vdm.exponent - (1023 - 127);
  413. return vfp_single_normaliseround(sd, &vsd, fpscr, exceptions, "fcvts");
  414. pack_nan:
  415. vfp_put_float(vfp_single_pack(&vsd), sd);
  416. return exceptions;
  417. }
  418. static u32 vfp_double_fuito(int dd, int unused, int dm, u32 fpscr)
  419. {
  420. struct vfp_double vdm;
  421. u32 m = vfp_get_float(dm);
  422. vdm.sign = 0;
  423. vdm.exponent = 1023 + 63 - 1;
  424. vdm.significand = (u64)m;
  425. return vfp_double_normaliseround(dd, &vdm, fpscr, 0, "fuito");
  426. }
  427. static u32 vfp_double_fsito(int dd, int unused, int dm, u32 fpscr)
  428. {
  429. struct vfp_double vdm;
  430. u32 m = vfp_get_float(dm);
  431. vdm.sign = (m & 0x80000000) >> 16;
  432. vdm.exponent = 1023 + 63 - 1;
  433. vdm.significand = vdm.sign ? -m : m;
  434. return vfp_double_normaliseround(dd, &vdm, fpscr, 0, "fsito");
  435. }
  436. static u32 vfp_double_ftoui(int sd, int unused, int dm, u32 fpscr)
  437. {
  438. struct vfp_double vdm;
  439. u32 d, exceptions = 0;
  440. int rmode = fpscr & FPSCR_RMODE_MASK;
  441. int tm;
  442. vfp_double_unpack(&vdm, vfp_get_double(dm));
  443. /*
  444. * Do we have a denormalised number?
  445. */
  446. tm = vfp_double_type(&vdm);
  447. if (tm & VFP_DENORMAL)
  448. exceptions |= FPSCR_IDC;
  449. if (tm & VFP_NAN)
  450. vdm.sign = 0;
  451. if (vdm.exponent >= 1023 + 32) {
  452. d = vdm.sign ? 0 : 0xffffffff;
  453. exceptions = FPSCR_IOC;
  454. } else if (vdm.exponent >= 1023 - 1) {
  455. int shift = 1023 + 63 - vdm.exponent;
  456. u64 rem, incr = 0;
  457. /*
  458. * 2^0 <= m < 2^32-2^8
  459. */
  460. d = (vdm.significand << 1) >> shift;
  461. rem = vdm.significand << (65 - shift);
  462. if (rmode == FPSCR_ROUND_NEAREST) {
  463. incr = 0x8000000000000000ULL;
  464. if ((d & 1) == 0)
  465. incr -= 1;
  466. } else if (rmode == FPSCR_ROUND_TOZERO) {
  467. incr = 0;
  468. } else if ((rmode == FPSCR_ROUND_PLUSINF) ^ (vdm.sign != 0)) {
  469. incr = ~0ULL;
  470. }
  471. if ((rem + incr) < rem) {
  472. if (d < 0xffffffff)
  473. d += 1;
  474. else
  475. exceptions |= FPSCR_IOC;
  476. }
  477. if (d && vdm.sign) {
  478. d = 0;
  479. exceptions |= FPSCR_IOC;
  480. } else if (rem)
  481. exceptions |= FPSCR_IXC;
  482. } else {
  483. d = 0;
  484. if (vdm.exponent | vdm.significand) {
  485. exceptions |= FPSCR_IXC;
  486. if (rmode == FPSCR_ROUND_PLUSINF && vdm.sign == 0)
  487. d = 1;
  488. else if (rmode == FPSCR_ROUND_MINUSINF && vdm.sign) {
  489. d = 0;
  490. exceptions |= FPSCR_IOC;
  491. }
  492. }
  493. }
  494. pr_debug("VFP: ftoui: d(s%d)=%08x exceptions=%08x\n", sd, d, exceptions);
  495. vfp_put_float(d, sd);
  496. return exceptions;
  497. }
  498. static u32 vfp_double_ftouiz(int sd, int unused, int dm, u32 fpscr)
  499. {
  500. return vfp_double_ftoui(sd, unused, dm, FPSCR_ROUND_TOZERO);
  501. }
  502. static u32 vfp_double_ftosi(int sd, int unused, int dm, u32 fpscr)
  503. {
  504. struct vfp_double vdm;
  505. u32 d, exceptions = 0;
  506. int rmode = fpscr & FPSCR_RMODE_MASK;
  507. int tm;
  508. vfp_double_unpack(&vdm, vfp_get_double(dm));
  509. vfp_double_dump("VDM", &vdm);
  510. /*
  511. * Do we have denormalised number?
  512. */
  513. tm = vfp_double_type(&vdm);
  514. if (tm & VFP_DENORMAL)
  515. exceptions |= FPSCR_IDC;
  516. if (tm & VFP_NAN) {
  517. d = 0;
  518. exceptions |= FPSCR_IOC;
  519. } else if (vdm.exponent >= 1023 + 32) {
  520. d = 0x7fffffff;
  521. if (vdm.sign)
  522. d = ~d;
  523. exceptions |= FPSCR_IOC;
  524. } else if (vdm.exponent >= 1023 - 1) {
  525. int shift = 1023 + 63 - vdm.exponent; /* 58 */
  526. u64 rem, incr = 0;
  527. d = (vdm.significand << 1) >> shift;
  528. rem = vdm.significand << (65 - shift);
  529. if (rmode == FPSCR_ROUND_NEAREST) {
  530. incr = 0x8000000000000000ULL;
  531. if ((d & 1) == 0)
  532. incr -= 1;
  533. } else if (rmode == FPSCR_ROUND_TOZERO) {
  534. incr = 0;
  535. } else if ((rmode == FPSCR_ROUND_PLUSINF) ^ (vdm.sign != 0)) {
  536. incr = ~0ULL;
  537. }
  538. if ((rem + incr) < rem && d < 0xffffffff)
  539. d += 1;
  540. if (d > 0x7fffffff + (vdm.sign != 0)) {
  541. d = 0x7fffffff + (vdm.sign != 0);
  542. exceptions |= FPSCR_IOC;
  543. } else if (rem)
  544. exceptions |= FPSCR_IXC;
  545. if (vdm.sign)
  546. d = -d;
  547. } else {
  548. d = 0;
  549. if (vdm.exponent | vdm.significand) {
  550. exceptions |= FPSCR_IXC;
  551. if (rmode == FPSCR_ROUND_PLUSINF && vdm.sign == 0)
  552. d = 1;
  553. else if (rmode == FPSCR_ROUND_MINUSINF && vdm.sign)
  554. d = -1;
  555. }
  556. }
  557. pr_debug("VFP: ftosi: d(s%d)=%08x exceptions=%08x\n", sd, d, exceptions);
  558. vfp_put_float((s32)d, sd);
  559. return exceptions;
  560. }
  561. static u32 vfp_double_ftosiz(int dd, int unused, int dm, u32 fpscr)
  562. {
  563. return vfp_double_ftosi(dd, unused, dm, FPSCR_ROUND_TOZERO);
  564. }
  565. static struct op fops_ext[32] = {
  566. [FEXT_TO_IDX(FEXT_FCPY)] = { vfp_double_fcpy, 0 },
  567. [FEXT_TO_IDX(FEXT_FABS)] = { vfp_double_fabs, 0 },
  568. [FEXT_TO_IDX(FEXT_FNEG)] = { vfp_double_fneg, 0 },
  569. [FEXT_TO_IDX(FEXT_FSQRT)] = { vfp_double_fsqrt, 0 },
  570. [FEXT_TO_IDX(FEXT_FCMP)] = { vfp_double_fcmp, OP_SCALAR },
  571. [FEXT_TO_IDX(FEXT_FCMPE)] = { vfp_double_fcmpe, OP_SCALAR },
  572. [FEXT_TO_IDX(FEXT_FCMPZ)] = { vfp_double_fcmpz, OP_SCALAR },
  573. [FEXT_TO_IDX(FEXT_FCMPEZ)] = { vfp_double_fcmpez, OP_SCALAR },
  574. [FEXT_TO_IDX(FEXT_FCVT)] = { vfp_double_fcvts, OP_SCALAR|OP_SD },
  575. [FEXT_TO_IDX(FEXT_FUITO)] = { vfp_double_fuito, OP_SCALAR },
  576. [FEXT_TO_IDX(FEXT_FSITO)] = { vfp_double_fsito, OP_SCALAR },
  577. [FEXT_TO_IDX(FEXT_FTOUI)] = { vfp_double_ftoui, OP_SCALAR|OP_SD },
  578. [FEXT_TO_IDX(FEXT_FTOUIZ)] = { vfp_double_ftouiz, OP_SCALAR|OP_SD },
  579. [FEXT_TO_IDX(FEXT_FTOSI)] = { vfp_double_ftosi, OP_SCALAR|OP_SD },
  580. [FEXT_TO_IDX(FEXT_FTOSIZ)] = { vfp_double_ftosiz, OP_SCALAR|OP_SD },
  581. };
  582. static u32
  583. vfp_double_fadd_nonnumber(struct vfp_double *vdd, struct vfp_double *vdn,
  584. struct vfp_double *vdm, u32 fpscr)
  585. {
  586. struct vfp_double *vdp;
  587. u32 exceptions = 0;
  588. int tn, tm;
  589. tn = vfp_double_type(vdn);
  590. tm = vfp_double_type(vdm);
  591. if (tn & tm & VFP_INFINITY) {
  592. /*
  593. * Two infinities. Are they different signs?
  594. */
  595. if (vdn->sign ^ vdm->sign) {
  596. /*
  597. * different signs -> invalid
  598. */
  599. exceptions = FPSCR_IOC;
  600. vdp = &vfp_double_default_qnan;
  601. } else {
  602. /*
  603. * same signs -> valid
  604. */
  605. vdp = vdn;
  606. }
  607. } else if (tn & VFP_INFINITY && tm & VFP_NUMBER) {
  608. /*
  609. * One infinity and one number -> infinity
  610. */
  611. vdp = vdn;
  612. } else {
  613. /*
  614. * 'n' is a NaN of some type
  615. */
  616. return vfp_propagate_nan(vdd, vdn, vdm, fpscr);
  617. }
  618. *vdd = *vdp;
  619. return exceptions;
  620. }
  621. static u32
  622. vfp_double_add(struct vfp_double *vdd, struct vfp_double *vdn,
  623. struct vfp_double *vdm, u32 fpscr)
  624. {
  625. u32 exp_diff;
  626. u64 m_sig;
  627. if (vdn->significand & (1ULL << 63) ||
  628. vdm->significand & (1ULL << 63)) {
  629. pr_info("VFP: bad FP values in %s\n", __func__);
  630. vfp_double_dump("VDN", vdn);
  631. vfp_double_dump("VDM", vdm);
  632. }
  633. /*
  634. * Ensure that 'n' is the largest magnitude number. Note that
  635. * if 'n' and 'm' have equal exponents, we do not swap them.
  636. * This ensures that NaN propagation works correctly.
  637. */
  638. if (vdn->exponent < vdm->exponent) {
  639. struct vfp_double *t = vdn;
  640. vdn = vdm;
  641. vdm = t;
  642. }
  643. /*
  644. * Is 'n' an infinity or a NaN? Note that 'm' may be a number,
  645. * infinity or a NaN here.
  646. */
  647. if (vdn->exponent == 2047)
  648. return vfp_double_fadd_nonnumber(vdd, vdn, vdm, fpscr);
  649. /*
  650. * We have two proper numbers, where 'vdn' is the larger magnitude.
  651. *
  652. * Copy 'n' to 'd' before doing the arithmetic.
  653. */
  654. *vdd = *vdn;
  655. /*
  656. * Align 'm' with the result.
  657. */
  658. exp_diff = vdn->exponent - vdm->exponent;
  659. m_sig = vfp_shiftright64jamming(vdm->significand, exp_diff);
  660. /*
  661. * If the signs are different, we are really subtracting.
  662. */
  663. if (vdn->sign ^ vdm->sign) {
  664. m_sig = vdn->significand - m_sig;
  665. if ((s64)m_sig < 0) {
  666. vdd->sign = vfp_sign_negate(vdd->sign);
  667. m_sig = -m_sig;
  668. } else if (m_sig == 0) {
  669. vdd->sign = (fpscr & FPSCR_RMODE_MASK) ==
  670. FPSCR_ROUND_MINUSINF ? 0x8000 : 0;
  671. }
  672. } else {
  673. m_sig += vdn->significand;
  674. }
  675. vdd->significand = m_sig;
  676. return 0;
  677. }
  678. static u32
  679. vfp_double_multiply(struct vfp_double *vdd, struct vfp_double *vdn,
  680. struct vfp_double *vdm, u32 fpscr)
  681. {
  682. vfp_double_dump("VDN", vdn);
  683. vfp_double_dump("VDM", vdm);
  684. /*
  685. * Ensure that 'n' is the largest magnitude number. Note that
  686. * if 'n' and 'm' have equal exponents, we do not swap them.
  687. * This ensures that NaN propagation works correctly.
  688. */
  689. if (vdn->exponent < vdm->exponent) {
  690. struct vfp_double *t = vdn;
  691. vdn = vdm;
  692. vdm = t;
  693. pr_debug("VFP: swapping M <-> N\n");
  694. }
  695. vdd->sign = vdn->sign ^ vdm->sign;
  696. /*
  697. * If 'n' is an infinity or NaN, handle it. 'm' may be anything.
  698. */
  699. if (vdn->exponent == 2047) {
  700. if (vdn->significand || (vdm->exponent == 2047 && vdm->significand))
  701. return vfp_propagate_nan(vdd, vdn, vdm, fpscr);
  702. if ((vdm->exponent | vdm->significand) == 0) {
  703. *vdd = vfp_double_default_qnan;
  704. return FPSCR_IOC;
  705. }
  706. vdd->exponent = vdn->exponent;
  707. vdd->significand = 0;
  708. return 0;
  709. }
  710. /*
  711. * If 'm' is zero, the result is always zero. In this case,
  712. * 'n' may be zero or a number, but it doesn't matter which.
  713. */
  714. if ((vdm->exponent | vdm->significand) == 0) {
  715. vdd->exponent = 0;
  716. vdd->significand = 0;
  717. return 0;
  718. }
  719. /*
  720. * We add 2 to the destination exponent for the same reason
  721. * as the addition case - though this time we have +1 from
  722. * each input operand.
  723. */
  724. vdd->exponent = vdn->exponent + vdm->exponent - 1023 + 2;
  725. vdd->significand = vfp_hi64multiply64(vdn->significand, vdm->significand);
  726. vfp_double_dump("VDD", vdd);
  727. return 0;
  728. }
  729. #define NEG_MULTIPLY (1 << 0)
  730. #define NEG_SUBTRACT (1 << 1)
  731. static u32
  732. vfp_double_multiply_accumulate(int dd, int dn, int dm, u32 fpscr, u32 negate, char *func)
  733. {
  734. struct vfp_double vdd, vdp, vdn, vdm;
  735. u32 exceptions;
  736. vfp_double_unpack(&vdn, vfp_get_double(dn));
  737. if (vdn.exponent == 0 && vdn.significand)
  738. vfp_double_normalise_denormal(&vdn);
  739. vfp_double_unpack(&vdm, vfp_get_double(dm));
  740. if (vdm.exponent == 0 && vdm.significand)
  741. vfp_double_normalise_denormal(&vdm);
  742. exceptions = vfp_double_multiply(&vdp, &vdn, &vdm, fpscr);
  743. if (negate & NEG_MULTIPLY)
  744. vdp.sign = vfp_sign_negate(vdp.sign);
  745. vfp_double_unpack(&vdn, vfp_get_double(dd));
  746. if (negate & NEG_SUBTRACT)
  747. vdn.sign = vfp_sign_negate(vdn.sign);
  748. exceptions |= vfp_double_add(&vdd, &vdn, &vdp, fpscr);
  749. return vfp_double_normaliseround(dd, &vdd, fpscr, exceptions, func);
  750. }
  751. /*
  752. * Standard operations
  753. */
  754. /*
  755. * sd = sd + (sn * sm)
  756. */
  757. static u32 vfp_double_fmac(int dd, int dn, int dm, u32 fpscr)
  758. {
  759. return vfp_double_multiply_accumulate(dd, dn, dm, fpscr, 0, "fmac");
  760. }
  761. /*
  762. * sd = sd - (sn * sm)
  763. */
  764. static u32 vfp_double_fnmac(int dd, int dn, int dm, u32 fpscr)
  765. {
  766. return vfp_double_multiply_accumulate(dd, dn, dm, fpscr, NEG_MULTIPLY, "fnmac");
  767. }
  768. /*
  769. * sd = -sd + (sn * sm)
  770. */
  771. static u32 vfp_double_fmsc(int dd, int dn, int dm, u32 fpscr)
  772. {
  773. return vfp_double_multiply_accumulate(dd, dn, dm, fpscr, NEG_SUBTRACT, "fmsc");
  774. }
  775. /*
  776. * sd = -sd - (sn * sm)
  777. */
  778. static u32 vfp_double_fnmsc(int dd, int dn, int dm, u32 fpscr)
  779. {
  780. return vfp_double_multiply_accumulate(dd, dn, dm, fpscr, NEG_SUBTRACT | NEG_MULTIPLY, "fnmsc");
  781. }
  782. /*
  783. * sd = sn * sm
  784. */
  785. static u32 vfp_double_fmul(int dd, int dn, int dm, u32 fpscr)
  786. {
  787. struct vfp_double vdd, vdn, vdm;
  788. u32 exceptions;
  789. vfp_double_unpack(&vdn, vfp_get_double(dn));
  790. if (vdn.exponent == 0 && vdn.significand)
  791. vfp_double_normalise_denormal(&vdn);
  792. vfp_double_unpack(&vdm, vfp_get_double(dm));
  793. if (vdm.exponent == 0 && vdm.significand)
  794. vfp_double_normalise_denormal(&vdm);
  795. exceptions = vfp_double_multiply(&vdd, &vdn, &vdm, fpscr);
  796. return vfp_double_normaliseround(dd, &vdd, fpscr, exceptions, "fmul");
  797. }
  798. /*
  799. * sd = -(sn * sm)
  800. */
  801. static u32 vfp_double_fnmul(int dd, int dn, int dm, u32 fpscr)
  802. {
  803. struct vfp_double vdd, vdn, vdm;
  804. u32 exceptions;
  805. vfp_double_unpack(&vdn, vfp_get_double(dn));
  806. if (vdn.exponent == 0 && vdn.significand)
  807. vfp_double_normalise_denormal(&vdn);
  808. vfp_double_unpack(&vdm, vfp_get_double(dm));
  809. if (vdm.exponent == 0 && vdm.significand)
  810. vfp_double_normalise_denormal(&vdm);
  811. exceptions = vfp_double_multiply(&vdd, &vdn, &vdm, fpscr);
  812. vdd.sign = vfp_sign_negate(vdd.sign);
  813. return vfp_double_normaliseround(dd, &vdd, fpscr, exceptions, "fnmul");
  814. }
  815. /*
  816. * sd = sn + sm
  817. */
  818. static u32 vfp_double_fadd(int dd, int dn, int dm, u32 fpscr)
  819. {
  820. struct vfp_double vdd, vdn, vdm;
  821. u32 exceptions;
  822. vfp_double_unpack(&vdn, vfp_get_double(dn));
  823. if (vdn.exponent == 0 && vdn.significand)
  824. vfp_double_normalise_denormal(&vdn);
  825. vfp_double_unpack(&vdm, vfp_get_double(dm));
  826. if (vdm.exponent == 0 && vdm.significand)
  827. vfp_double_normalise_denormal(&vdm);
  828. exceptions = vfp_double_add(&vdd, &vdn, &vdm, fpscr);
  829. return vfp_double_normaliseround(dd, &vdd, fpscr, exceptions, "fadd");
  830. }
  831. /*
  832. * sd = sn - sm
  833. */
  834. static u32 vfp_double_fsub(int dd, int dn, int dm, u32 fpscr)
  835. {
  836. struct vfp_double vdd, vdn, vdm;
  837. u32 exceptions;
  838. vfp_double_unpack(&vdn, vfp_get_double(dn));
  839. if (vdn.exponent == 0 && vdn.significand)
  840. vfp_double_normalise_denormal(&vdn);
  841. vfp_double_unpack(&vdm, vfp_get_double(dm));
  842. if (vdm.exponent == 0 && vdm.significand)
  843. vfp_double_normalise_denormal(&vdm);
  844. /*
  845. * Subtraction is like addition, but with a negated operand.
  846. */
  847. vdm.sign = vfp_sign_negate(vdm.sign);
  848. exceptions = vfp_double_add(&vdd, &vdn, &vdm, fpscr);
  849. return vfp_double_normaliseround(dd, &vdd, fpscr, exceptions, "fsub");
  850. }
  851. /*
  852. * sd = sn / sm
  853. */
  854. static u32 vfp_double_fdiv(int dd, int dn, int dm, u32 fpscr)
  855. {
  856. struct vfp_double vdd, vdn, vdm;
  857. u32 exceptions = 0;
  858. int tm, tn;
  859. vfp_double_unpack(&vdn, vfp_get_double(dn));
  860. vfp_double_unpack(&vdm, vfp_get_double(dm));
  861. vdd.sign = vdn.sign ^ vdm.sign;
  862. tn = vfp_double_type(&vdn);
  863. tm = vfp_double_type(&vdm);
  864. /*
  865. * Is n a NAN?
  866. */
  867. if (tn & VFP_NAN)
  868. goto vdn_nan;
  869. /*
  870. * Is m a NAN?
  871. */
  872. if (tm & VFP_NAN)
  873. goto vdm_nan;
  874. /*
  875. * If n and m are infinity, the result is invalid
  876. * If n and m are zero, the result is invalid
  877. */
  878. if (tm & tn & (VFP_INFINITY|VFP_ZERO))
  879. goto invalid;
  880. /*
  881. * If n is infinity, the result is infinity
  882. */
  883. if (tn & VFP_INFINITY)
  884. goto infinity;
  885. /*
  886. * If m is zero, raise div0 exceptions
  887. */
  888. if (tm & VFP_ZERO)
  889. goto divzero;
  890. /*
  891. * If m is infinity, or n is zero, the result is zero
  892. */
  893. if (tm & VFP_INFINITY || tn & VFP_ZERO)
  894. goto zero;
  895. if (tn & VFP_DENORMAL)
  896. vfp_double_normalise_denormal(&vdn);
  897. if (tm & VFP_DENORMAL)
  898. vfp_double_normalise_denormal(&vdm);
  899. /*
  900. * Ok, we have two numbers, we can perform division.
  901. */
  902. vdd.exponent = vdn.exponent - vdm.exponent + 1023 - 1;
  903. vdm.significand <<= 1;
  904. if (vdm.significand <= (2 * vdn.significand)) {
  905. vdn.significand >>= 1;
  906. vdd.exponent++;
  907. }
  908. vdd.significand = vfp_estimate_div128to64(vdn.significand, 0, vdm.significand);
  909. if ((vdd.significand & 0x1ff) <= 2) {
  910. u64 termh, terml, remh, reml;
  911. mul64to128(&termh, &terml, vdm.significand, vdd.significand);
  912. sub128(&remh, &reml, vdn.significand, 0, termh, terml);
  913. while ((s64)remh < 0) {
  914. vdd.significand -= 1;
  915. add128(&remh, &reml, remh, reml, 0, vdm.significand);
  916. }
  917. vdd.significand |= (reml != 0);
  918. }
  919. return vfp_double_normaliseround(dd, &vdd, fpscr, 0, "fdiv");
  920. vdn_nan:
  921. exceptions = vfp_propagate_nan(&vdd, &vdn, &vdm, fpscr);
  922. pack:
  923. vfp_put_double(vfp_double_pack(&vdd), dd);
  924. return exceptions;
  925. vdm_nan:
  926. exceptions = vfp_propagate_nan(&vdd, &vdm, &vdn, fpscr);
  927. goto pack;
  928. zero:
  929. vdd.exponent = 0;
  930. vdd.significand = 0;
  931. goto pack;
  932. divzero:
  933. exceptions = FPSCR_DZC;
  934. infinity:
  935. vdd.exponent = 2047;
  936. vdd.significand = 0;
  937. goto pack;
  938. invalid:
  939. vfp_put_double(vfp_double_pack(&vfp_double_default_qnan), dd);
  940. return FPSCR_IOC;
  941. }
  942. static struct op fops[16] = {
  943. [FOP_TO_IDX(FOP_FMAC)] = { vfp_double_fmac, 0 },
  944. [FOP_TO_IDX(FOP_FNMAC)] = { vfp_double_fnmac, 0 },
  945. [FOP_TO_IDX(FOP_FMSC)] = { vfp_double_fmsc, 0 },
  946. [FOP_TO_IDX(FOP_FNMSC)] = { vfp_double_fnmsc, 0 },
  947. [FOP_TO_IDX(FOP_FMUL)] = { vfp_double_fmul, 0 },
  948. [FOP_TO_IDX(FOP_FNMUL)] = { vfp_double_fnmul, 0 },
  949. [FOP_TO_IDX(FOP_FADD)] = { vfp_double_fadd, 0 },
  950. [FOP_TO_IDX(FOP_FSUB)] = { vfp_double_fsub, 0 },
  951. [FOP_TO_IDX(FOP_FDIV)] = { vfp_double_fdiv, 0 },
  952. };
  953. #define FREG_BANK(x) ((x) & 0x0c)
  954. #define FREG_IDX(x) ((x) & 3)
  955. u32 vfp_double_cpdo(u32 inst, u32 fpscr)
  956. {
  957. u32 op = inst & FOP_MASK;
  958. u32 exceptions = 0;
  959. unsigned int dest;
  960. unsigned int dn = vfp_get_dn(inst);
  961. unsigned int dm = vfp_get_dm(inst);
  962. unsigned int vecitr, veclen, vecstride;
  963. struct op *fop;
  964. vecstride = (1 + ((fpscr & FPSCR_STRIDE_MASK) == FPSCR_STRIDE_MASK)) * 2;
  965. fop = (op == FOP_EXT) ? &fops_ext[FEXT_TO_IDX(inst)] : &fops[FOP_TO_IDX(op)];
  966. /*
  967. * fcvtds takes an sN register number as destination, not dN.
  968. * It also always operates on scalars.
  969. */
  970. if (fop->flags & OP_SD)
  971. dest = vfp_get_sd(inst);
  972. else
  973. dest = vfp_get_dd(inst);
  974. /*
  975. * If destination bank is zero, vector length is always '1'.
  976. * ARM DDI0100F C5.1.3, C5.3.2.
  977. */
  978. if ((fop->flags & OP_SCALAR) || (FREG_BANK(dest) == 0))
  979. veclen = 0;
  980. else
  981. veclen = fpscr & FPSCR_LENGTH_MASK;
  982. pr_debug("VFP: vecstride=%u veclen=%u\n", vecstride,
  983. (veclen >> FPSCR_LENGTH_BIT) + 1);
  984. if (!fop->fn)
  985. goto invalid;
  986. for (vecitr = 0; vecitr <= veclen; vecitr += 1 << FPSCR_LENGTH_BIT) {
  987. u32 except;
  988. char type;
  989. type = fop->flags & OP_SD ? 's' : 'd';
  990. if (op == FOP_EXT)
  991. pr_debug("VFP: itr%d (%c%u) = op[%u] (d%u)\n",
  992. vecitr >> FPSCR_LENGTH_BIT,
  993. type, dest, dn, dm);
  994. else
  995. pr_debug("VFP: itr%d (%c%u) = (d%u) op[%u] (d%u)\n",
  996. vecitr >> FPSCR_LENGTH_BIT,
  997. type, dest, dn, FOP_TO_IDX(op), dm);
  998. except = fop->fn(dest, dn, dm, fpscr);
  999. pr_debug("VFP: itr%d: exceptions=%08x\n",
  1000. vecitr >> FPSCR_LENGTH_BIT, except);
  1001. exceptions |= except;
  1002. /*
  1003. * CHECK: It appears to be undefined whether we stop when
  1004. * we encounter an exception. We continue.
  1005. */
  1006. dest = FREG_BANK(dest) + ((FREG_IDX(dest) + vecstride) & 6);
  1007. dn = FREG_BANK(dn) + ((FREG_IDX(dn) + vecstride) & 6);
  1008. if (FREG_BANK(dm) != 0)
  1009. dm = FREG_BANK(dm) + ((FREG_IDX(dm) + vecstride) & 6);
  1010. }
  1011. return exceptions;
  1012. invalid:
  1013. return ~0;
  1014. }