vfpdouble.c 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206
  1. /*
  2. * linux/arch/arm/vfp/vfpdouble.c
  3. *
  4. * This code is derived in part from John R. Housers softfloat library, which
  5. * carries the following notice:
  6. *
  7. * ===========================================================================
  8. * This C source file is part of the SoftFloat IEC/IEEE Floating-point
  9. * Arithmetic Package, Release 2.
  10. *
  11. * Written by John R. Hauser. This work was made possible in part by the
  12. * International Computer Science Institute, located at Suite 600, 1947 Center
  13. * Street, Berkeley, California 94704. Funding was partially provided by the
  14. * National Science Foundation under grant MIP-9311980. The original version
  15. * of this code was written as part of a project to build a fixed-point vector
  16. * processor in collaboration with the University of California at Berkeley,
  17. * overseen by Profs. Nelson Morgan and John Wawrzynek. More information
  18. * is available through the web page `http://HTTP.CS.Berkeley.EDU/~jhauser/
  19. * arithmetic/softfloat.html'.
  20. *
  21. * THIS SOFTWARE IS DISTRIBUTED AS IS, FOR FREE. Although reasonable effort
  22. * has been made to avoid it, THIS SOFTWARE MAY CONTAIN FAULTS THAT WILL AT
  23. * TIMES RESULT IN INCORRECT BEHAVIOR. USE OF THIS SOFTWARE IS RESTRICTED TO
  24. * PERSONS AND ORGANIZATIONS WHO CAN AND WILL TAKE FULL RESPONSIBILITY FOR ANY
  25. * AND ALL LOSSES, COSTS, OR OTHER PROBLEMS ARISING FROM ITS USE.
  26. *
  27. * Derivative works are acceptable, even for commercial purposes, so long as
  28. * (1) they include prominent notice that the work is derivative, and (2) they
  29. * include prominent notice akin to these three paragraphs for those parts of
  30. * this code that are retained.
  31. * ===========================================================================
  32. */
  33. #include <linux/kernel.h>
  34. #include <linux/bitops.h>
  35. #include <asm/div64.h>
  36. #include <asm/vfp.h>
  37. #include "vfpinstr.h"
  38. #include "vfp.h"
  39. static struct vfp_double vfp_double_default_qnan = {
  40. .exponent = 2047,
  41. .sign = 0,
  42. .significand = VFP_DOUBLE_SIGNIFICAND_QNAN,
  43. };
  44. static void vfp_double_dump(const char *str, struct vfp_double *d)
  45. {
  46. pr_debug("VFP: %s: sign=%d exponent=%d significand=%016llx\n",
  47. str, d->sign != 0, d->exponent, d->significand);
  48. }
  49. static void vfp_double_normalise_denormal(struct vfp_double *vd)
  50. {
  51. int bits = 31 - fls(vd->significand >> 32);
  52. if (bits == 31)
  53. bits = 63 - fls(vd->significand);
  54. vfp_double_dump("normalise_denormal: in", vd);
  55. if (bits) {
  56. vd->exponent -= bits - 1;
  57. vd->significand <<= bits;
  58. }
  59. vfp_double_dump("normalise_denormal: out", vd);
  60. }
  61. u32 vfp_double_normaliseround(int dd, struct vfp_double *vd, u32 fpscr, u32 exceptions, const char *func)
  62. {
  63. u64 significand, incr;
  64. int exponent, shift, underflow;
  65. u32 rmode;
  66. vfp_double_dump("pack: in", vd);
  67. /*
  68. * Infinities and NaNs are a special case.
  69. */
  70. if (vd->exponent == 2047 && (vd->significand == 0 || exceptions))
  71. goto pack;
  72. /*
  73. * Special-case zero.
  74. */
  75. if (vd->significand == 0) {
  76. vd->exponent = 0;
  77. goto pack;
  78. }
  79. exponent = vd->exponent;
  80. significand = vd->significand;
  81. shift = 32 - fls(significand >> 32);
  82. if (shift == 32)
  83. shift = 64 - fls(significand);
  84. if (shift) {
  85. exponent -= shift;
  86. significand <<= shift;
  87. }
  88. #ifdef DEBUG
  89. vd->exponent = exponent;
  90. vd->significand = significand;
  91. vfp_double_dump("pack: normalised", vd);
  92. #endif
  93. /*
  94. * Tiny number?
  95. */
  96. underflow = exponent < 0;
  97. if (underflow) {
  98. significand = vfp_shiftright64jamming(significand, -exponent);
  99. exponent = 0;
  100. #ifdef DEBUG
  101. vd->exponent = exponent;
  102. vd->significand = significand;
  103. vfp_double_dump("pack: tiny number", vd);
  104. #endif
  105. if (!(significand & ((1ULL << (VFP_DOUBLE_LOW_BITS + 1)) - 1)))
  106. underflow = 0;
  107. }
  108. /*
  109. * Select rounding increment.
  110. */
  111. incr = 0;
  112. rmode = fpscr & FPSCR_RMODE_MASK;
  113. if (rmode == FPSCR_ROUND_NEAREST) {
  114. incr = 1ULL << VFP_DOUBLE_LOW_BITS;
  115. if ((significand & (1ULL << (VFP_DOUBLE_LOW_BITS + 1))) == 0)
  116. incr -= 1;
  117. } else if (rmode == FPSCR_ROUND_TOZERO) {
  118. incr = 0;
  119. } else if ((rmode == FPSCR_ROUND_PLUSINF) ^ (vd->sign != 0))
  120. incr = (1ULL << (VFP_DOUBLE_LOW_BITS + 1)) - 1;
  121. pr_debug("VFP: rounding increment = 0x%08llx\n", incr);
  122. /*
  123. * Is our rounding going to overflow?
  124. */
  125. if ((significand + incr) < significand) {
  126. exponent += 1;
  127. significand = (significand >> 1) | (significand & 1);
  128. incr >>= 1;
  129. #ifdef DEBUG
  130. vd->exponent = exponent;
  131. vd->significand = significand;
  132. vfp_double_dump("pack: overflow", vd);
  133. #endif
  134. }
  135. /*
  136. * If any of the low bits (which will be shifted out of the
  137. * number) are non-zero, the result is inexact.
  138. */
  139. if (significand & ((1 << (VFP_DOUBLE_LOW_BITS + 1)) - 1))
  140. exceptions |= FPSCR_IXC;
  141. /*
  142. * Do our rounding.
  143. */
  144. significand += incr;
  145. /*
  146. * Infinity?
  147. */
  148. if (exponent >= 2046) {
  149. exceptions |= FPSCR_OFC | FPSCR_IXC;
  150. if (incr == 0) {
  151. vd->exponent = 2045;
  152. vd->significand = 0x7fffffffffffffffULL;
  153. } else {
  154. vd->exponent = 2047; /* infinity */
  155. vd->significand = 0;
  156. }
  157. } else {
  158. if (significand >> (VFP_DOUBLE_LOW_BITS + 1) == 0)
  159. exponent = 0;
  160. if (exponent || significand > 0x8000000000000000ULL)
  161. underflow = 0;
  162. if (underflow)
  163. exceptions |= FPSCR_UFC;
  164. vd->exponent = exponent;
  165. vd->significand = significand >> 1;
  166. }
  167. pack:
  168. vfp_double_dump("pack: final", vd);
  169. {
  170. s64 d = vfp_double_pack(vd);
  171. pr_debug("VFP: %s: d(d%d)=%016llx exceptions=%08x\n", func,
  172. dd, d, exceptions);
  173. vfp_put_double(d, dd);
  174. }
  175. return exceptions;
  176. }
  177. /*
  178. * Propagate the NaN, setting exceptions if it is signalling.
  179. * 'n' is always a NaN. 'm' may be a number, NaN or infinity.
  180. */
  181. static u32
  182. vfp_propagate_nan(struct vfp_double *vdd, struct vfp_double *vdn,
  183. struct vfp_double *vdm, u32 fpscr)
  184. {
  185. struct vfp_double *nan;
  186. int tn, tm = 0;
  187. tn = vfp_double_type(vdn);
  188. if (vdm)
  189. tm = vfp_double_type(vdm);
  190. if (fpscr & FPSCR_DEFAULT_NAN)
  191. /*
  192. * Default NaN mode - always returns a quiet NaN
  193. */
  194. nan = &vfp_double_default_qnan;
  195. else {
  196. /*
  197. * Contemporary mode - select the first signalling
  198. * NAN, or if neither are signalling, the first
  199. * quiet NAN.
  200. */
  201. if (tn == VFP_SNAN || (tm != VFP_SNAN && tn == VFP_QNAN))
  202. nan = vdn;
  203. else
  204. nan = vdm;
  205. /*
  206. * Make the NaN quiet.
  207. */
  208. nan->significand |= VFP_DOUBLE_SIGNIFICAND_QNAN;
  209. }
  210. *vdd = *nan;
  211. /*
  212. * If one was a signalling NAN, raise invalid operation.
  213. */
  214. return tn == VFP_SNAN || tm == VFP_SNAN ? FPSCR_IOC : VFP_NAN_FLAG;
  215. }
  216. /*
  217. * Extended operations
  218. */
  219. static u32 vfp_double_fabs(int dd, int unused, int dm, u32 fpscr)
  220. {
  221. vfp_put_double(vfp_double_packed_abs(vfp_get_double(dm)), dd);
  222. return 0;
  223. }
  224. static u32 vfp_double_fcpy(int dd, int unused, int dm, u32 fpscr)
  225. {
  226. vfp_put_double(vfp_get_double(dm), dd);
  227. return 0;
  228. }
  229. static u32 vfp_double_fneg(int dd, int unused, int dm, u32 fpscr)
  230. {
  231. vfp_put_double(vfp_double_packed_negate(vfp_get_double(dm)), dd);
  232. return 0;
  233. }
  234. static u32 vfp_double_fsqrt(int dd, int unused, int dm, u32 fpscr)
  235. {
  236. struct vfp_double vdm, vdd;
  237. int ret, tm;
  238. vfp_double_unpack(&vdm, vfp_get_double(dm));
  239. tm = vfp_double_type(&vdm);
  240. if (tm & (VFP_NAN|VFP_INFINITY)) {
  241. struct vfp_double *vdp = &vdd;
  242. if (tm & VFP_NAN)
  243. ret = vfp_propagate_nan(vdp, &vdm, NULL, fpscr);
  244. else if (vdm.sign == 0) {
  245. sqrt_copy:
  246. vdp = &vdm;
  247. ret = 0;
  248. } else {
  249. sqrt_invalid:
  250. vdp = &vfp_double_default_qnan;
  251. ret = FPSCR_IOC;
  252. }
  253. vfp_put_double(vfp_double_pack(vdp), dd);
  254. return ret;
  255. }
  256. /*
  257. * sqrt(+/- 0) == +/- 0
  258. */
  259. if (tm & VFP_ZERO)
  260. goto sqrt_copy;
  261. /*
  262. * Normalise a denormalised number
  263. */
  264. if (tm & VFP_DENORMAL)
  265. vfp_double_normalise_denormal(&vdm);
  266. /*
  267. * sqrt(<0) = invalid
  268. */
  269. if (vdm.sign)
  270. goto sqrt_invalid;
  271. vfp_double_dump("sqrt", &vdm);
  272. /*
  273. * Estimate the square root.
  274. */
  275. vdd.sign = 0;
  276. vdd.exponent = ((vdm.exponent - 1023) >> 1) + 1023;
  277. vdd.significand = (u64)vfp_estimate_sqrt_significand(vdm.exponent, vdm.significand >> 32) << 31;
  278. vfp_double_dump("sqrt estimate1", &vdd);
  279. vdm.significand >>= 1 + (vdm.exponent & 1);
  280. vdd.significand += 2 + vfp_estimate_div128to64(vdm.significand, 0, vdd.significand);
  281. vfp_double_dump("sqrt estimate2", &vdd);
  282. /*
  283. * And now adjust.
  284. */
  285. if ((vdd.significand & VFP_DOUBLE_LOW_BITS_MASK) <= 5) {
  286. if (vdd.significand < 2) {
  287. vdd.significand = ~0ULL;
  288. } else {
  289. u64 termh, terml, remh, reml;
  290. vdm.significand <<= 2;
  291. mul64to128(&termh, &terml, vdd.significand, vdd.significand);
  292. sub128(&remh, &reml, vdm.significand, 0, termh, terml);
  293. while ((s64)remh < 0) {
  294. vdd.significand -= 1;
  295. shift64left(&termh, &terml, vdd.significand);
  296. terml |= 1;
  297. add128(&remh, &reml, remh, reml, termh, terml);
  298. }
  299. vdd.significand |= (remh | reml) != 0;
  300. }
  301. }
  302. vdd.significand = vfp_shiftright64jamming(vdd.significand, 1);
  303. return vfp_double_normaliseround(dd, &vdd, fpscr, 0, "fsqrt");
  304. }
  305. /*
  306. * Equal := ZC
  307. * Less than := N
  308. * Greater than := C
  309. * Unordered := CV
  310. */
  311. static u32 vfp_compare(int dd, int signal_on_qnan, int dm, u32 fpscr)
  312. {
  313. s64 d, m;
  314. u32 ret = 0;
  315. m = vfp_get_double(dm);
  316. if (vfp_double_packed_exponent(m) == 2047 && vfp_double_packed_mantissa(m)) {
  317. ret |= FPSCR_C | FPSCR_V;
  318. if (signal_on_qnan || !(vfp_double_packed_mantissa(m) & (1ULL << (VFP_DOUBLE_MANTISSA_BITS - 1))))
  319. /*
  320. * Signalling NaN, or signalling on quiet NaN
  321. */
  322. ret |= FPSCR_IOC;
  323. }
  324. d = vfp_get_double(dd);
  325. if (vfp_double_packed_exponent(d) == 2047 && vfp_double_packed_mantissa(d)) {
  326. ret |= FPSCR_C | FPSCR_V;
  327. if (signal_on_qnan || !(vfp_double_packed_mantissa(d) & (1ULL << (VFP_DOUBLE_MANTISSA_BITS - 1))))
  328. /*
  329. * Signalling NaN, or signalling on quiet NaN
  330. */
  331. ret |= FPSCR_IOC;
  332. }
  333. if (ret == 0) {
  334. if (d == m || vfp_double_packed_abs(d | m) == 0) {
  335. /*
  336. * equal
  337. */
  338. ret |= FPSCR_Z | FPSCR_C;
  339. } else if (vfp_double_packed_sign(d ^ m)) {
  340. /*
  341. * different signs
  342. */
  343. if (vfp_double_packed_sign(d))
  344. /*
  345. * d is negative, so d < m
  346. */
  347. ret |= FPSCR_N;
  348. else
  349. /*
  350. * d is positive, so d > m
  351. */
  352. ret |= FPSCR_C;
  353. } else if ((vfp_double_packed_sign(d) != 0) ^ (d < m)) {
  354. /*
  355. * d < m
  356. */
  357. ret |= FPSCR_N;
  358. } else if ((vfp_double_packed_sign(d) != 0) ^ (d > m)) {
  359. /*
  360. * d > m
  361. */
  362. ret |= FPSCR_C;
  363. }
  364. }
  365. return ret;
  366. }
  367. static u32 vfp_double_fcmp(int dd, int unused, int dm, u32 fpscr)
  368. {
  369. return vfp_compare(dd, 0, dm, fpscr);
  370. }
  371. static u32 vfp_double_fcmpe(int dd, int unused, int dm, u32 fpscr)
  372. {
  373. return vfp_compare(dd, 1, dm, fpscr);
  374. }
  375. static u32 vfp_double_fcmpz(int dd, int unused, int dm, u32 fpscr)
  376. {
  377. return vfp_compare(dd, 0, VFP_REG_ZERO, fpscr);
  378. }
  379. static u32 vfp_double_fcmpez(int dd, int unused, int dm, u32 fpscr)
  380. {
  381. return vfp_compare(dd, 1, VFP_REG_ZERO, fpscr);
  382. }
  383. static u32 vfp_double_fcvts(int sd, int unused, int dm, u32 fpscr)
  384. {
  385. struct vfp_double vdm;
  386. struct vfp_single vsd;
  387. int tm;
  388. u32 exceptions = 0;
  389. vfp_double_unpack(&vdm, vfp_get_double(dm));
  390. tm = vfp_double_type(&vdm);
  391. /*
  392. * If we have a signalling NaN, signal invalid operation.
  393. */
  394. if (tm == VFP_SNAN)
  395. exceptions = FPSCR_IOC;
  396. if (tm & VFP_DENORMAL)
  397. vfp_double_normalise_denormal(&vdm);
  398. vsd.sign = vdm.sign;
  399. vsd.significand = vfp_hi64to32jamming(vdm.significand);
  400. /*
  401. * If we have an infinity or a NaN, the exponent must be 255
  402. */
  403. if (tm & (VFP_INFINITY|VFP_NAN)) {
  404. vsd.exponent = 255;
  405. if (tm == VFP_QNAN)
  406. vsd.significand |= VFP_SINGLE_SIGNIFICAND_QNAN;
  407. goto pack_nan;
  408. } else if (tm & VFP_ZERO)
  409. vsd.exponent = 0;
  410. else
  411. vsd.exponent = vdm.exponent - (1023 - 127);
  412. return vfp_single_normaliseround(sd, &vsd, fpscr, exceptions, "fcvts");
  413. pack_nan:
  414. vfp_put_float(vfp_single_pack(&vsd), sd);
  415. return exceptions;
  416. }
  417. static u32 vfp_double_fuito(int dd, int unused, int dm, u32 fpscr)
  418. {
  419. struct vfp_double vdm;
  420. u32 m = vfp_get_float(dm);
  421. vdm.sign = 0;
  422. vdm.exponent = 1023 + 63 - 1;
  423. vdm.significand = (u64)m;
  424. return vfp_double_normaliseround(dd, &vdm, fpscr, 0, "fuito");
  425. }
  426. static u32 vfp_double_fsito(int dd, int unused, int dm, u32 fpscr)
  427. {
  428. struct vfp_double vdm;
  429. u32 m = vfp_get_float(dm);
  430. vdm.sign = (m & 0x80000000) >> 16;
  431. vdm.exponent = 1023 + 63 - 1;
  432. vdm.significand = vdm.sign ? -m : m;
  433. return vfp_double_normaliseround(dd, &vdm, fpscr, 0, "fsito");
  434. }
  435. static u32 vfp_double_ftoui(int sd, int unused, int dm, u32 fpscr)
  436. {
  437. struct vfp_double vdm;
  438. u32 d, exceptions = 0;
  439. int rmode = fpscr & FPSCR_RMODE_MASK;
  440. int tm;
  441. vfp_double_unpack(&vdm, vfp_get_double(dm));
  442. /*
  443. * Do we have a denormalised number?
  444. */
  445. tm = vfp_double_type(&vdm);
  446. if (tm & VFP_DENORMAL)
  447. exceptions |= FPSCR_IDC;
  448. if (tm & VFP_NAN)
  449. vdm.sign = 0;
  450. if (vdm.exponent >= 1023 + 32) {
  451. d = vdm.sign ? 0 : 0xffffffff;
  452. exceptions = FPSCR_IOC;
  453. } else if (vdm.exponent >= 1023 - 1) {
  454. int shift = 1023 + 63 - vdm.exponent;
  455. u64 rem, incr = 0;
  456. /*
  457. * 2^0 <= m < 2^32-2^8
  458. */
  459. d = (vdm.significand << 1) >> shift;
  460. rem = vdm.significand << (65 - shift);
  461. if (rmode == FPSCR_ROUND_NEAREST) {
  462. incr = 0x8000000000000000ULL;
  463. if ((d & 1) == 0)
  464. incr -= 1;
  465. } else if (rmode == FPSCR_ROUND_TOZERO) {
  466. incr = 0;
  467. } else if ((rmode == FPSCR_ROUND_PLUSINF) ^ (vdm.sign != 0)) {
  468. incr = ~0ULL;
  469. }
  470. if ((rem + incr) < rem) {
  471. if (d < 0xffffffff)
  472. d += 1;
  473. else
  474. exceptions |= FPSCR_IOC;
  475. }
  476. if (d && vdm.sign) {
  477. d = 0;
  478. exceptions |= FPSCR_IOC;
  479. } else if (rem)
  480. exceptions |= FPSCR_IXC;
  481. } else {
  482. d = 0;
  483. if (vdm.exponent | vdm.significand) {
  484. exceptions |= FPSCR_IXC;
  485. if (rmode == FPSCR_ROUND_PLUSINF && vdm.sign == 0)
  486. d = 1;
  487. else if (rmode == FPSCR_ROUND_MINUSINF && vdm.sign) {
  488. d = 0;
  489. exceptions |= FPSCR_IOC;
  490. }
  491. }
  492. }
  493. pr_debug("VFP: ftoui: d(s%d)=%08x exceptions=%08x\n", sd, d, exceptions);
  494. vfp_put_float(d, sd);
  495. return exceptions;
  496. }
  497. static u32 vfp_double_ftouiz(int sd, int unused, int dm, u32 fpscr)
  498. {
  499. return vfp_double_ftoui(sd, unused, dm, FPSCR_ROUND_TOZERO);
  500. }
  501. static u32 vfp_double_ftosi(int sd, int unused, int dm, u32 fpscr)
  502. {
  503. struct vfp_double vdm;
  504. u32 d, exceptions = 0;
  505. int rmode = fpscr & FPSCR_RMODE_MASK;
  506. int tm;
  507. vfp_double_unpack(&vdm, vfp_get_double(dm));
  508. vfp_double_dump("VDM", &vdm);
  509. /*
  510. * Do we have denormalised number?
  511. */
  512. tm = vfp_double_type(&vdm);
  513. if (tm & VFP_DENORMAL)
  514. exceptions |= FPSCR_IDC;
  515. if (tm & VFP_NAN) {
  516. d = 0;
  517. exceptions |= FPSCR_IOC;
  518. } else if (vdm.exponent >= 1023 + 32) {
  519. d = 0x7fffffff;
  520. if (vdm.sign)
  521. d = ~d;
  522. exceptions |= FPSCR_IOC;
  523. } else if (vdm.exponent >= 1023 - 1) {
  524. int shift = 1023 + 63 - vdm.exponent; /* 58 */
  525. u64 rem, incr = 0;
  526. d = (vdm.significand << 1) >> shift;
  527. rem = vdm.significand << (65 - shift);
  528. if (rmode == FPSCR_ROUND_NEAREST) {
  529. incr = 0x8000000000000000ULL;
  530. if ((d & 1) == 0)
  531. incr -= 1;
  532. } else if (rmode == FPSCR_ROUND_TOZERO) {
  533. incr = 0;
  534. } else if ((rmode == FPSCR_ROUND_PLUSINF) ^ (vdm.sign != 0)) {
  535. incr = ~0ULL;
  536. }
  537. if ((rem + incr) < rem && d < 0xffffffff)
  538. d += 1;
  539. if (d > 0x7fffffff + (vdm.sign != 0)) {
  540. d = 0x7fffffff + (vdm.sign != 0);
  541. exceptions |= FPSCR_IOC;
  542. } else if (rem)
  543. exceptions |= FPSCR_IXC;
  544. if (vdm.sign)
  545. d = -d;
  546. } else {
  547. d = 0;
  548. if (vdm.exponent | vdm.significand) {
  549. exceptions |= FPSCR_IXC;
  550. if (rmode == FPSCR_ROUND_PLUSINF && vdm.sign == 0)
  551. d = 1;
  552. else if (rmode == FPSCR_ROUND_MINUSINF && vdm.sign)
  553. d = -1;
  554. }
  555. }
  556. pr_debug("VFP: ftosi: d(s%d)=%08x exceptions=%08x\n", sd, d, exceptions);
  557. vfp_put_float((s32)d, sd);
  558. return exceptions;
  559. }
  560. static u32 vfp_double_ftosiz(int dd, int unused, int dm, u32 fpscr)
  561. {
  562. return vfp_double_ftosi(dd, unused, dm, FPSCR_ROUND_TOZERO);
  563. }
  564. static struct op fops_ext[32] = {
  565. [FEXT_TO_IDX(FEXT_FCPY)] = { vfp_double_fcpy, 0 },
  566. [FEXT_TO_IDX(FEXT_FABS)] = { vfp_double_fabs, 0 },
  567. [FEXT_TO_IDX(FEXT_FNEG)] = { vfp_double_fneg, 0 },
  568. [FEXT_TO_IDX(FEXT_FSQRT)] = { vfp_double_fsqrt, 0 },
  569. [FEXT_TO_IDX(FEXT_FCMP)] = { vfp_double_fcmp, OP_SCALAR },
  570. [FEXT_TO_IDX(FEXT_FCMPE)] = { vfp_double_fcmpe, OP_SCALAR },
  571. [FEXT_TO_IDX(FEXT_FCMPZ)] = { vfp_double_fcmpz, OP_SCALAR },
  572. [FEXT_TO_IDX(FEXT_FCMPEZ)] = { vfp_double_fcmpez, OP_SCALAR },
  573. [FEXT_TO_IDX(FEXT_FCVT)] = { vfp_double_fcvts, OP_SCALAR|OP_SD },
  574. [FEXT_TO_IDX(FEXT_FUITO)] = { vfp_double_fuito, OP_SCALAR|OP_SM },
  575. [FEXT_TO_IDX(FEXT_FSITO)] = { vfp_double_fsito, OP_SCALAR|OP_SM },
  576. [FEXT_TO_IDX(FEXT_FTOUI)] = { vfp_double_ftoui, OP_SCALAR|OP_SD },
  577. [FEXT_TO_IDX(FEXT_FTOUIZ)] = { vfp_double_ftouiz, OP_SCALAR|OP_SD },
  578. [FEXT_TO_IDX(FEXT_FTOSI)] = { vfp_double_ftosi, OP_SCALAR|OP_SD },
  579. [FEXT_TO_IDX(FEXT_FTOSIZ)] = { vfp_double_ftosiz, OP_SCALAR|OP_SD },
  580. };
  581. static u32
  582. vfp_double_fadd_nonnumber(struct vfp_double *vdd, struct vfp_double *vdn,
  583. struct vfp_double *vdm, u32 fpscr)
  584. {
  585. struct vfp_double *vdp;
  586. u32 exceptions = 0;
  587. int tn, tm;
  588. tn = vfp_double_type(vdn);
  589. tm = vfp_double_type(vdm);
  590. if (tn & tm & VFP_INFINITY) {
  591. /*
  592. * Two infinities. Are they different signs?
  593. */
  594. if (vdn->sign ^ vdm->sign) {
  595. /*
  596. * different signs -> invalid
  597. */
  598. exceptions = FPSCR_IOC;
  599. vdp = &vfp_double_default_qnan;
  600. } else {
  601. /*
  602. * same signs -> valid
  603. */
  604. vdp = vdn;
  605. }
  606. } else if (tn & VFP_INFINITY && tm & VFP_NUMBER) {
  607. /*
  608. * One infinity and one number -> infinity
  609. */
  610. vdp = vdn;
  611. } else {
  612. /*
  613. * 'n' is a NaN of some type
  614. */
  615. return vfp_propagate_nan(vdd, vdn, vdm, fpscr);
  616. }
  617. *vdd = *vdp;
  618. return exceptions;
  619. }
  620. static u32
  621. vfp_double_add(struct vfp_double *vdd, struct vfp_double *vdn,
  622. struct vfp_double *vdm, u32 fpscr)
  623. {
  624. u32 exp_diff;
  625. u64 m_sig;
  626. if (vdn->significand & (1ULL << 63) ||
  627. vdm->significand & (1ULL << 63)) {
  628. pr_info("VFP: bad FP values in %s\n", __func__);
  629. vfp_double_dump("VDN", vdn);
  630. vfp_double_dump("VDM", vdm);
  631. }
  632. /*
  633. * Ensure that 'n' is the largest magnitude number. Note that
  634. * if 'n' and 'm' have equal exponents, we do not swap them.
  635. * This ensures that NaN propagation works correctly.
  636. */
  637. if (vdn->exponent < vdm->exponent) {
  638. struct vfp_double *t = vdn;
  639. vdn = vdm;
  640. vdm = t;
  641. }
  642. /*
  643. * Is 'n' an infinity or a NaN? Note that 'm' may be a number,
  644. * infinity or a NaN here.
  645. */
  646. if (vdn->exponent == 2047)
  647. return vfp_double_fadd_nonnumber(vdd, vdn, vdm, fpscr);
  648. /*
  649. * We have two proper numbers, where 'vdn' is the larger magnitude.
  650. *
  651. * Copy 'n' to 'd' before doing the arithmetic.
  652. */
  653. *vdd = *vdn;
  654. /*
  655. * Align 'm' with the result.
  656. */
  657. exp_diff = vdn->exponent - vdm->exponent;
  658. m_sig = vfp_shiftright64jamming(vdm->significand, exp_diff);
  659. /*
  660. * If the signs are different, we are really subtracting.
  661. */
  662. if (vdn->sign ^ vdm->sign) {
  663. m_sig = vdn->significand - m_sig;
  664. if ((s64)m_sig < 0) {
  665. vdd->sign = vfp_sign_negate(vdd->sign);
  666. m_sig = -m_sig;
  667. } else if (m_sig == 0) {
  668. vdd->sign = (fpscr & FPSCR_RMODE_MASK) ==
  669. FPSCR_ROUND_MINUSINF ? 0x8000 : 0;
  670. }
  671. } else {
  672. m_sig += vdn->significand;
  673. }
  674. vdd->significand = m_sig;
  675. return 0;
  676. }
  677. static u32
  678. vfp_double_multiply(struct vfp_double *vdd, struct vfp_double *vdn,
  679. struct vfp_double *vdm, u32 fpscr)
  680. {
  681. vfp_double_dump("VDN", vdn);
  682. vfp_double_dump("VDM", vdm);
  683. /*
  684. * Ensure that 'n' is the largest magnitude number. Note that
  685. * if 'n' and 'm' have equal exponents, we do not swap them.
  686. * This ensures that NaN propagation works correctly.
  687. */
  688. if (vdn->exponent < vdm->exponent) {
  689. struct vfp_double *t = vdn;
  690. vdn = vdm;
  691. vdm = t;
  692. pr_debug("VFP: swapping M <-> N\n");
  693. }
  694. vdd->sign = vdn->sign ^ vdm->sign;
  695. /*
  696. * If 'n' is an infinity or NaN, handle it. 'm' may be anything.
  697. */
  698. if (vdn->exponent == 2047) {
  699. if (vdn->significand || (vdm->exponent == 2047 && vdm->significand))
  700. return vfp_propagate_nan(vdd, vdn, vdm, fpscr);
  701. if ((vdm->exponent | vdm->significand) == 0) {
  702. *vdd = vfp_double_default_qnan;
  703. return FPSCR_IOC;
  704. }
  705. vdd->exponent = vdn->exponent;
  706. vdd->significand = 0;
  707. return 0;
  708. }
  709. /*
  710. * If 'm' is zero, the result is always zero. In this case,
  711. * 'n' may be zero or a number, but it doesn't matter which.
  712. */
  713. if ((vdm->exponent | vdm->significand) == 0) {
  714. vdd->exponent = 0;
  715. vdd->significand = 0;
  716. return 0;
  717. }
  718. /*
  719. * We add 2 to the destination exponent for the same reason
  720. * as the addition case - though this time we have +1 from
  721. * each input operand.
  722. */
  723. vdd->exponent = vdn->exponent + vdm->exponent - 1023 + 2;
  724. vdd->significand = vfp_hi64multiply64(vdn->significand, vdm->significand);
  725. vfp_double_dump("VDD", vdd);
  726. return 0;
  727. }
  728. #define NEG_MULTIPLY (1 << 0)
  729. #define NEG_SUBTRACT (1 << 1)
  730. static u32
  731. vfp_double_multiply_accumulate(int dd, int dn, int dm, u32 fpscr, u32 negate, char *func)
  732. {
  733. struct vfp_double vdd, vdp, vdn, vdm;
  734. u32 exceptions;
  735. vfp_double_unpack(&vdn, vfp_get_double(dn));
  736. if (vdn.exponent == 0 && vdn.significand)
  737. vfp_double_normalise_denormal(&vdn);
  738. vfp_double_unpack(&vdm, vfp_get_double(dm));
  739. if (vdm.exponent == 0 && vdm.significand)
  740. vfp_double_normalise_denormal(&vdm);
  741. exceptions = vfp_double_multiply(&vdp, &vdn, &vdm, fpscr);
  742. if (negate & NEG_MULTIPLY)
  743. vdp.sign = vfp_sign_negate(vdp.sign);
  744. vfp_double_unpack(&vdn, vfp_get_double(dd));
  745. if (vdn.exponent == 0 && vdn.significand)
  746. vfp_double_normalise_denormal(&vdn);
  747. if (negate & NEG_SUBTRACT)
  748. vdn.sign = vfp_sign_negate(vdn.sign);
  749. exceptions |= vfp_double_add(&vdd, &vdn, &vdp, fpscr);
  750. return vfp_double_normaliseround(dd, &vdd, fpscr, exceptions, func);
  751. }
  752. /*
  753. * Standard operations
  754. */
  755. /*
  756. * sd = sd + (sn * sm)
  757. */
  758. static u32 vfp_double_fmac(int dd, int dn, int dm, u32 fpscr)
  759. {
  760. return vfp_double_multiply_accumulate(dd, dn, dm, fpscr, 0, "fmac");
  761. }
  762. /*
  763. * sd = sd - (sn * sm)
  764. */
  765. static u32 vfp_double_fnmac(int dd, int dn, int dm, u32 fpscr)
  766. {
  767. return vfp_double_multiply_accumulate(dd, dn, dm, fpscr, NEG_MULTIPLY, "fnmac");
  768. }
  769. /*
  770. * sd = -sd + (sn * sm)
  771. */
  772. static u32 vfp_double_fmsc(int dd, int dn, int dm, u32 fpscr)
  773. {
  774. return vfp_double_multiply_accumulate(dd, dn, dm, fpscr, NEG_SUBTRACT, "fmsc");
  775. }
  776. /*
  777. * sd = -sd - (sn * sm)
  778. */
  779. static u32 vfp_double_fnmsc(int dd, int dn, int dm, u32 fpscr)
  780. {
  781. return vfp_double_multiply_accumulate(dd, dn, dm, fpscr, NEG_SUBTRACT | NEG_MULTIPLY, "fnmsc");
  782. }
  783. /*
  784. * sd = sn * sm
  785. */
  786. static u32 vfp_double_fmul(int dd, int dn, int dm, u32 fpscr)
  787. {
  788. struct vfp_double vdd, vdn, vdm;
  789. u32 exceptions;
  790. vfp_double_unpack(&vdn, vfp_get_double(dn));
  791. if (vdn.exponent == 0 && vdn.significand)
  792. vfp_double_normalise_denormal(&vdn);
  793. vfp_double_unpack(&vdm, vfp_get_double(dm));
  794. if (vdm.exponent == 0 && vdm.significand)
  795. vfp_double_normalise_denormal(&vdm);
  796. exceptions = vfp_double_multiply(&vdd, &vdn, &vdm, fpscr);
  797. return vfp_double_normaliseround(dd, &vdd, fpscr, exceptions, "fmul");
  798. }
  799. /*
  800. * sd = -(sn * sm)
  801. */
  802. static u32 vfp_double_fnmul(int dd, int dn, int dm, u32 fpscr)
  803. {
  804. struct vfp_double vdd, vdn, vdm;
  805. u32 exceptions;
  806. vfp_double_unpack(&vdn, vfp_get_double(dn));
  807. if (vdn.exponent == 0 && vdn.significand)
  808. vfp_double_normalise_denormal(&vdn);
  809. vfp_double_unpack(&vdm, vfp_get_double(dm));
  810. if (vdm.exponent == 0 && vdm.significand)
  811. vfp_double_normalise_denormal(&vdm);
  812. exceptions = vfp_double_multiply(&vdd, &vdn, &vdm, fpscr);
  813. vdd.sign = vfp_sign_negate(vdd.sign);
  814. return vfp_double_normaliseround(dd, &vdd, fpscr, exceptions, "fnmul");
  815. }
  816. /*
  817. * sd = sn + sm
  818. */
  819. static u32 vfp_double_fadd(int dd, int dn, int dm, u32 fpscr)
  820. {
  821. struct vfp_double vdd, vdn, vdm;
  822. u32 exceptions;
  823. vfp_double_unpack(&vdn, vfp_get_double(dn));
  824. if (vdn.exponent == 0 && vdn.significand)
  825. vfp_double_normalise_denormal(&vdn);
  826. vfp_double_unpack(&vdm, vfp_get_double(dm));
  827. if (vdm.exponent == 0 && vdm.significand)
  828. vfp_double_normalise_denormal(&vdm);
  829. exceptions = vfp_double_add(&vdd, &vdn, &vdm, fpscr);
  830. return vfp_double_normaliseround(dd, &vdd, fpscr, exceptions, "fadd");
  831. }
  832. /*
  833. * sd = sn - sm
  834. */
  835. static u32 vfp_double_fsub(int dd, int dn, int dm, u32 fpscr)
  836. {
  837. struct vfp_double vdd, vdn, vdm;
  838. u32 exceptions;
  839. vfp_double_unpack(&vdn, vfp_get_double(dn));
  840. if (vdn.exponent == 0 && vdn.significand)
  841. vfp_double_normalise_denormal(&vdn);
  842. vfp_double_unpack(&vdm, vfp_get_double(dm));
  843. if (vdm.exponent == 0 && vdm.significand)
  844. vfp_double_normalise_denormal(&vdm);
  845. /*
  846. * Subtraction is like addition, but with a negated operand.
  847. */
  848. vdm.sign = vfp_sign_negate(vdm.sign);
  849. exceptions = vfp_double_add(&vdd, &vdn, &vdm, fpscr);
  850. return vfp_double_normaliseround(dd, &vdd, fpscr, exceptions, "fsub");
  851. }
  852. /*
  853. * sd = sn / sm
  854. */
  855. static u32 vfp_double_fdiv(int dd, int dn, int dm, u32 fpscr)
  856. {
  857. struct vfp_double vdd, vdn, vdm;
  858. u32 exceptions = 0;
  859. int tm, tn;
  860. vfp_double_unpack(&vdn, vfp_get_double(dn));
  861. vfp_double_unpack(&vdm, vfp_get_double(dm));
  862. vdd.sign = vdn.sign ^ vdm.sign;
  863. tn = vfp_double_type(&vdn);
  864. tm = vfp_double_type(&vdm);
  865. /*
  866. * Is n a NAN?
  867. */
  868. if (tn & VFP_NAN)
  869. goto vdn_nan;
  870. /*
  871. * Is m a NAN?
  872. */
  873. if (tm & VFP_NAN)
  874. goto vdm_nan;
  875. /*
  876. * If n and m are infinity, the result is invalid
  877. * If n and m are zero, the result is invalid
  878. */
  879. if (tm & tn & (VFP_INFINITY|VFP_ZERO))
  880. goto invalid;
  881. /*
  882. * If n is infinity, the result is infinity
  883. */
  884. if (tn & VFP_INFINITY)
  885. goto infinity;
  886. /*
  887. * If m is zero, raise div0 exceptions
  888. */
  889. if (tm & VFP_ZERO)
  890. goto divzero;
  891. /*
  892. * If m is infinity, or n is zero, the result is zero
  893. */
  894. if (tm & VFP_INFINITY || tn & VFP_ZERO)
  895. goto zero;
  896. if (tn & VFP_DENORMAL)
  897. vfp_double_normalise_denormal(&vdn);
  898. if (tm & VFP_DENORMAL)
  899. vfp_double_normalise_denormal(&vdm);
  900. /*
  901. * Ok, we have two numbers, we can perform division.
  902. */
  903. vdd.exponent = vdn.exponent - vdm.exponent + 1023 - 1;
  904. vdm.significand <<= 1;
  905. if (vdm.significand <= (2 * vdn.significand)) {
  906. vdn.significand >>= 1;
  907. vdd.exponent++;
  908. }
  909. vdd.significand = vfp_estimate_div128to64(vdn.significand, 0, vdm.significand);
  910. if ((vdd.significand & 0x1ff) <= 2) {
  911. u64 termh, terml, remh, reml;
  912. mul64to128(&termh, &terml, vdm.significand, vdd.significand);
  913. sub128(&remh, &reml, vdn.significand, 0, termh, terml);
  914. while ((s64)remh < 0) {
  915. vdd.significand -= 1;
  916. add128(&remh, &reml, remh, reml, 0, vdm.significand);
  917. }
  918. vdd.significand |= (reml != 0);
  919. }
  920. return vfp_double_normaliseround(dd, &vdd, fpscr, 0, "fdiv");
  921. vdn_nan:
  922. exceptions = vfp_propagate_nan(&vdd, &vdn, &vdm, fpscr);
  923. pack:
  924. vfp_put_double(vfp_double_pack(&vdd), dd);
  925. return exceptions;
  926. vdm_nan:
  927. exceptions = vfp_propagate_nan(&vdd, &vdm, &vdn, fpscr);
  928. goto pack;
  929. zero:
  930. vdd.exponent = 0;
  931. vdd.significand = 0;
  932. goto pack;
  933. divzero:
  934. exceptions = FPSCR_DZC;
  935. infinity:
  936. vdd.exponent = 2047;
  937. vdd.significand = 0;
  938. goto pack;
  939. invalid:
  940. vfp_put_double(vfp_double_pack(&vfp_double_default_qnan), dd);
  941. return FPSCR_IOC;
  942. }
  943. static struct op fops[16] = {
  944. [FOP_TO_IDX(FOP_FMAC)] = { vfp_double_fmac, 0 },
  945. [FOP_TO_IDX(FOP_FNMAC)] = { vfp_double_fnmac, 0 },
  946. [FOP_TO_IDX(FOP_FMSC)] = { vfp_double_fmsc, 0 },
  947. [FOP_TO_IDX(FOP_FNMSC)] = { vfp_double_fnmsc, 0 },
  948. [FOP_TO_IDX(FOP_FMUL)] = { vfp_double_fmul, 0 },
  949. [FOP_TO_IDX(FOP_FNMUL)] = { vfp_double_fnmul, 0 },
  950. [FOP_TO_IDX(FOP_FADD)] = { vfp_double_fadd, 0 },
  951. [FOP_TO_IDX(FOP_FSUB)] = { vfp_double_fsub, 0 },
  952. [FOP_TO_IDX(FOP_FDIV)] = { vfp_double_fdiv, 0 },
  953. };
  954. #define FREG_BANK(x) ((x) & 0x0c)
  955. #define FREG_IDX(x) ((x) & 3)
  956. u32 vfp_double_cpdo(u32 inst, u32 fpscr)
  957. {
  958. u32 op = inst & FOP_MASK;
  959. u32 exceptions = 0;
  960. unsigned int dest;
  961. unsigned int dn = vfp_get_dn(inst);
  962. unsigned int dm;
  963. unsigned int vecitr, veclen, vecstride;
  964. struct op *fop;
  965. vecstride = (1 + ((fpscr & FPSCR_STRIDE_MASK) == FPSCR_STRIDE_MASK));
  966. fop = (op == FOP_EXT) ? &fops_ext[FEXT_TO_IDX(inst)] : &fops[FOP_TO_IDX(op)];
  967. /*
  968. * fcvtds takes an sN register number as destination, not dN.
  969. * It also always operates on scalars.
  970. */
  971. if (fop->flags & OP_SD)
  972. dest = vfp_get_sd(inst);
  973. else
  974. dest = vfp_get_dd(inst);
  975. /*
  976. * f[us]ito takes a sN operand, not a dN operand.
  977. */
  978. if (fop->flags & OP_SM)
  979. dm = vfp_get_sm(inst);
  980. else
  981. dm = vfp_get_dm(inst);
  982. /*
  983. * If destination bank is zero, vector length is always '1'.
  984. * ARM DDI0100F C5.1.3, C5.3.2.
  985. */
  986. if ((fop->flags & OP_SCALAR) || (FREG_BANK(dest) == 0))
  987. veclen = 0;
  988. else
  989. veclen = fpscr & FPSCR_LENGTH_MASK;
  990. pr_debug("VFP: vecstride=%u veclen=%u\n", vecstride,
  991. (veclen >> FPSCR_LENGTH_BIT) + 1);
  992. if (!fop->fn)
  993. goto invalid;
  994. for (vecitr = 0; vecitr <= veclen; vecitr += 1 << FPSCR_LENGTH_BIT) {
  995. u32 except;
  996. char type;
  997. type = fop->flags & OP_SD ? 's' : 'd';
  998. if (op == FOP_EXT)
  999. pr_debug("VFP: itr%d (%c%u) = op[%u] (d%u)\n",
  1000. vecitr >> FPSCR_LENGTH_BIT,
  1001. type, dest, dn, dm);
  1002. else
  1003. pr_debug("VFP: itr%d (%c%u) = (d%u) op[%u] (d%u)\n",
  1004. vecitr >> FPSCR_LENGTH_BIT,
  1005. type, dest, dn, FOP_TO_IDX(op), dm);
  1006. except = fop->fn(dest, dn, dm, fpscr);
  1007. pr_debug("VFP: itr%d: exceptions=%08x\n",
  1008. vecitr >> FPSCR_LENGTH_BIT, except);
  1009. exceptions |= except;
  1010. /*
  1011. * CHECK: It appears to be undefined whether we stop when
  1012. * we encounter an exception. We continue.
  1013. */
  1014. dest = FREG_BANK(dest) + ((FREG_IDX(dest) + vecstride) & 3);
  1015. dn = FREG_BANK(dn) + ((FREG_IDX(dn) + vecstride) & 3);
  1016. if (FREG_BANK(dm) != 0)
  1017. dm = FREG_BANK(dm) + ((FREG_IDX(dm) + vecstride) & 3);
  1018. }
  1019. return exceptions;
  1020. invalid:
  1021. return ~0;
  1022. }