Commit 16c97cd7 authored by abolz's avatar abolz

Fix implicit signed/unsigned conversion and a small glitch in the error computation

parent 2ea43433
...@@ -162,10 +162,10 @@ inline bool StrtodDiyFp(const char* decimals, int dLen, int dExp, double* result ...@@ -162,10 +162,10 @@ inline bool StrtodDiyFp(const char* decimals, int dLen, int dExp, double* result
DiyFp(RAPIDJSON_UINT64_C2(0xf4240000, 00000000), -44), // 10^6 DiyFp(RAPIDJSON_UINT64_C2(0xf4240000, 00000000), -44), // 10^6
DiyFp(RAPIDJSON_UINT64_C2(0x98968000, 00000000), -40) // 10^7 DiyFp(RAPIDJSON_UINT64_C2(0x98968000, 00000000), -40) // 10^7
}; };
int adjustment = dExp - actualExp - 1; int adjustment = dExp - actualExp;
RAPIDJSON_ASSERT(adjustment >= 0 && adjustment < 7); RAPIDJSON_ASSERT(adjustment >= 1 && adjustment < 8);
v = v * kPow10[adjustment]; v = v * kPow10[adjustment - 1];
if (dLen + static_cast<unsigned>(adjustment)> 19u) // has more digits than decimal digits in 64-bit if (dLen + adjustment > 19u) // has more digits than decimal digits in 64-bit
error += kUlp / 2; error += kUlp / 2;
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment