Commit 16c97cd7 authored by abolz's avatar abolz

Fix implicit signed/unsigned conversion and a small glitch in the error computation

parent 2ea43433
......@@ -162,10 +162,10 @@ inline bool StrtodDiyFp(const char* decimals, int dLen, int dExp, double* result
DiyFp(RAPIDJSON_UINT64_C2(0xf4240000, 00000000), -44), // 10^6
DiyFp(RAPIDJSON_UINT64_C2(0x98968000, 00000000), -40) // 10^7
};
int adjustment = dExp - actualExp - 1;
RAPIDJSON_ASSERT(adjustment >= 0 && adjustment < 7);
v = v * kPow10[adjustment];
if (dLen + static_cast<unsigned>(adjustment)> 19u) // has more digits than decimal digits in 64-bit
int adjustment = dExp - actualExp;
RAPIDJSON_ASSERT(adjustment >= 1 && adjustment < 8);
v = v * kPow10[adjustment - 1];
if (dLen + adjustment > 19u) // has more digits than decimal digits in 64-bit
error += kUlp / 2;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment