Tables of fres outputs for sequential inputs: 4010000000000001 = 3FDFFE8000000000 4010000000000002 = 3FDFFE8000000000 4010000000000004 = 3FDFFE8000000000 4010000000000008 = 3FDFFE8000000000 4010000000000010 = 3FDFFE8000000000 4010000000000020 = 3FDFFE8000000000 4010000000000040 = 3FDFFE8000000000 4010000000000080 = 3FDFFE8000000000 4010000000000100 = 3FDFFE8000000000 4010000000000200 = 3FDFFE8000000000 4010000000000400 = 3FDFFE8000000000 4010000000000800 = 3FDFFE8000000000 4010000000001000 = 3FDFFE8000000000 4010000000002000 = 3FDFFE8000000000 4010000000004000 = 3FDFFE8000000000 4010000000008000 = 3FDFFE8000000000 4010000000010000 = 3FDFFE8000000000 4010000000020000 = 3FDFFE8000000000 4010000000040000 = 3FDFFE8000000000 4010000000080000 = 3FDFFE8000000000 4010000000100000 = 3FDFFE8000000000 4010000000200000 = 3FDFFE8000000000 4010000000400000 = 3FDFFE8000000000 4010000000800000 = 3FDFFE8000000000 4010000001000000 = 3FDFFE8000000000 4010000002000000 = 3FDFFE8000000000 4010000004000000 = 3FDFFE8000000000 4010000008000000 = 3FDFFE8000000000 4010000010000000 = 3FDFFE8000000000 4010000020000000 = 3FDFFE8000000000 4010000040000000 = 3FDFFE8000000000 4010000080000000 = 3FDFFE8000000000 4010000100000000 = 3FDFFE8000000000 4010000200000000 = 3FDFFE8000000000 4010000400000000 = 3FDFFE8000000000 4010000800000000 = 3FDFFE8000000000 4010001000000000 = 3FDFFE8000000000 4010002000000000 = 3FDFFE6170000000 // The 750CL docs lie! "frD ... [is] representable in single-precision format" -- NOT! 4010004000000000 = 3FDFFE42E0000000 4010008000000000 = 3FDFFE05C0000000 4010010000000000 = 3FDFFD8B80000000 4010020000000000 = 3FDFFC9700000000 4010040000000000 = 3FDFFAAE00000000 4010080000000000 = 3FDFF6DC00000000 4010100000000000 = 3FDFEF3800000000 4010200000000000 = 3FDFDFF000000000 4010400000000000 = 3FDFC16000000000 4010800000000000 = 3FDF844000000000 [4010000000000000] 3FDFFE8000000000 3FDFC16000000000 3FDF844000000000 3FDF472000000000 3FDF0A4000000000 3FDED24000000000 3FDE9A4000000000 3FDE624000000000 3FDE2A8000000000 3FDDF70000000000 3FDDC38000000000 3FDD900000000000 3FDD5C8000000000 3FDD2CF000000000 3FDCFD6000000000 3FDCCDD000000000 [4014000000000000] 3FDC9E4000000000 3FDC722000000000 3FDC460000000000 3FDC19E000000000 3FDBEDC000000000 3FDBC4A000000000 3FDB9B8000000000 3FDC726000000000 3FDB498000000000 3FDB232000000000 3FDAFCC000000000 3FDAD66000000000 3FDAB00000000000 3FDA8C1000000000 3FDA682000000000 3FDA443000000000 [4018000000000000] 3FDA204000000000 3FD9FE7000000000 3FD9DCA000000000 3FD9BAD000000000 3FD9994000000000 3FD9797000000000 3FD959A000000000 3FD939D000000000 3FD91A0000000000 3FD8FBF000000000 3FD8DDE000000000 3FD8BFD000000000 3FD8A1C000000000 3FD8855000000000 3FD868E000000000 3FD84C7000000000 [401C000000000000] 3FD8304000000000 3FD8155000000000 3FD7FA6000000000 3FD7DF7000000000 3FD7C48000000000 3FD7AAF000000000 3FD7916000000000 3FD777D000000000 3FD75E4000000000 3FD745F000000000 3FD72DA000000000 3FD7155000000000 3FD6FD0000000000 3FD6E5D000000000 3FD6CEA000000000 3FD6B77000000000 [4020000000000000] 3FD69FA000000000 3FD6746000000000 3FD6492000000000 3FD61DE000000000 3FD5F2E000000000 3FD5CB4800000000 3FD5A3B000000000 3FD571C800000000 3FD554A000000000 3FD5303800000000 3FD50BD000000000 3FD4E76800000000 3FD4C30000000000 3FD4A15800000000 3FD47FB000000000 3FD45E0800000000 [4024000000000000] 3FD43C8000000000 3FD41D4800000000 3FD3FE1000000000 3FD3DED800000000 3FD3BFC000000000 3FD3A2B000000000 3FD385A000000000 3FD3689000000000 3FD34B8000000000 3FD3305800000000 3FD3153000000000 3FD2FA0800000000 3FD2DF0000000000 3FD2C59000000000 3FD2AC2000000000 3FD292B000000000 [4028000000000000] 3FD2794000000000 3FD2616000000000 3FD2498000000000 3FD231A000000000 3FD219C000000000 3FD2034000000000 3FD1ECC000000000 3FD1D64000000000 3FD1BFC000000000 3FD1AA8000000000 3FD1954000000000 3FD1800000000000 3FD16AE000000000 3FD156C800000000 3FD142B000000000 3FD12E9800000000 [402C000000000000] 3FD11A8000000000 3FD1077800000000 3FD0F47000000000 3FD0E16800000000 3FD0CE6000000000 3FD0BC4800000000 3FD0AA3000000000 3FD0981800000000 3FD0862000000000 3FD074F000000000 3FD063C000000000 3FD0529000000000 3FD0416000000000 3FD0310800000000 3FD020B000000000 3FD0105800000000 [400FFC0000000000] 3FE0010580000000 -- all values repeated 2 times 3FE000FD54000000 3FE000F528000000 3FE000ECFC000000 3FE000E4D0000000 3FE000DCA4000000 3FE000D478000000 3FE000CC4C000000 [400FFD0000000000] 3FE000C420000000 3FE000BBF4000000 3FE000B3C8000000 3FE000AB9C000000 3FE000A370000000 3FE0009B44000000 3FE0009318000000 3FE0008AEC000000 [400FFE0000000000] 3FE00082C0000000 3FE0007A94000000 3FE0007268000000 3FE0006A3C000000 3FE0006210000000 3FE00059E4000000 3FE00051B8000000 3FE000498C000000 [400FFF0000000000] 3FE0004160000000 3FE0003934000000 3FE0003108000000 3FE00028DC000000 3FE00020B0000000 3FE0001884000000 3FE0001058000000 3FE000082C000000 [4010000000000000] 3FDFFE8000000000 3FDFFE6170000000 3FDFFE42E0000000 3FDFFE2450000000 3FDFFE05C0000000 3FDFFDE730000000 3FDFFDC8A0000000 3FDFFDAA10000000 [4010010000000000] 3FDFFD8B80000000 3FDFFD6CF0000000 3FDFFD4E60000000 3FDFFD2FD0000000 3FDFFD1140000000 3FDFFCF2B0000000 3FDFFCD420000000 3FDFFCB590000000 [4010020000000000] 3FDFFC9700000000 3FDFFC7870000000 3FDFFC59E0000000 3FDFFC3B50000000 3FDFFC1CC0000000 3FDFFBFE30000000 3FDFFBDFA0000000 3FDFFBC110000000 [4010030000000000] 3FDFFBA280000000 3FDFFB83F0000000 3FDFFB6560000000 3FDFFB46D0000000 3FDFFB2840000000 3FDFFB09B0000000 3FDFFAEB20000000 3FDFFACC90000000 [0000000000000000] 7FF0000000000000 617FFE8000000000 61769FA000000000 6172794000000000 616FFE8000000000 616C9E4000000000 616A204000000000 6168304000000000 61669FA000000000 616554A000000000 61643C8000000000 61634B8000000000 6162794000000000 6161BFC000000000 61611A8000000000 6160862000000000 [0000000000000010] 615FFE8000000000 615F0A4000000000 615E2A8000000000 615D5C8000000000 615C9E4000000000 615BEDC000000000 616B498000000000 615AB00000000000 615A204000000000 6159994000000000 61591A0000000000 6158A1C000000000 6158304000000000 6157C48000000000 61575E4000000000 6156FD0000000000 [0000000000000020] 61569FA000000000 6156492000000000 6155F2E000000000 6155A3B000000000 615554A000000000 61550BD000000000 6154C30000000000 61547FD000000000 61543C8000000000 6153FE1000000000 6153BFC000000000 615385A000000000 61534B8000000000 6153153000000000 6152DF0000000000 6152AC3000000000 [0000000000000030] 6152794000000000 6152498000000000 615219C000000000 6151ECC000000000 6151BFC000000000 6151954000000000 61516AE000000000 615142B000000000 61511A8000000000 6150F47000000000 6150CE6000000000 6150AA3000000000 6150862000000000 615063C000000000 6150416000000000 615020B000000000 [0000000000000040] 614FFE8000000000 614FC16000000000 614F844000000000 614F472000000000 614F0A4000000000 614ED24000000000 614E9A4000000000 614E624000000000 614E2A8000000000 614DF70000000000 614DC38000000000 614D900000000000 614D5C8000000000 614D2CF000000000 614CFD6000000000 614CCDD000000000 [0000000000000050] 614C9E4000000000 614C722000000000 614C460000000000 614C19E000000000 614BEDC000000000 614BC4A000000000 614B9B8000000000 614B726000000000 614B498000000000 614B232000000000 614AFCC000000000 614AD66000000000 614AB00000000000 614A8C1000000000 614A682000000000 614A443000000000 [0000000000000060] 614A204000000000 6149FE7000000000 6149DCA000000000 6149BAD000000000 6149994000000000 6149797000000000 614959A000000000 614939D000000000 61491A0000000000 6148FBF000000000 6148DDE000000000 6148BFD000000000 6148A1C000000000 6148855000000000 614868E000000000 61484C7000000000 [0000000000000070] 6148304000000000 6148155000000000 6147FA6000000000 6147DF7000000000 6147C48000000000 6147AAF000000000 6147916000000000 614777D000000000 61475E4000000000 614745F000000000 61472DA000000000 6147155000000000 6146FD0000000000 6146E5D000000000 6146CEA000000000 6146B77000000000 frsqrte(sign, exp, mant) { // The first 16 bases and deltas are all even; they're probably stored // internally in 1 bit less and shifted left. base[] = {0x7FF4,0x7852,0x7154,0x6AE4,0x64F2,0x5F6E,0x5A4C,0x5580, 0x5102,0x4CCA,0x48D0,0x450E,0x4182,0x3E24,0x3AF2,0x37E8, 0x34FD,0x2F97,0x2AA5,0x2618,0x21E4,0x1DFE,0x1A5C,0x16F8, 0x13CA,0x10CE,0x0DFE,0x0B57,0x08D4,0x0673,0x0431,0x020B} delta[] = {0x7A4,0x700,0x670,0x5F2,0x584,0x524,0x4CC,0x47E, 0x43A,0x3FA,0x3C2,0x38E,0x35E,0x332,0x30A,0x2E6, 0x568,0x4F3,0x48D,0x435,0x3E7,0x3A2,0x365,0x32E, 0x2FC,0x2D0,0x2A8,0x283,0x261,0x243,0x226,0x20B} if exp == 0 and mant == 0 return sign, 255, 0 // infinity if exp == 255 and mant != 0 return sign, 255, mant | 1<<22 // quieted NaN if sign return QNaN // VXSQRT if exp == 255 and mant == 0 return 0, 0, 0 // zero exp, mant = normalize(exp, mant) index = mant>>48 | (exp & 1 ? 0 : 16) exp = (3068 - exp) >> 1 mant = ( (base[index]<<11) - (delta[index] * (mant>>37 & 0x7FF)) ) << 26 return 0, exp, mant }