Skip to content

C
capnproto
Commit df52bf86 authored by Kenton Varda's avatar Kenton Varda
Browse files
Options

Improve KJ encoding lib error handling: 

- Rename UtfResult -> EncodingResult
- Make it usable like a Maybe, so that we don't need separate "try" functions.
- Check errors in hex decoding and URI decoding.
parent 03800dfa
Hide whitespace changes
Inline Side-by-side
Showing with 256 additions and 243 deletions
c++/src/kj/common.h
View file @ df52bf86
......@@ -915,7 +915,6 @@ public:
return value;
}
private: // internal interface used by friends only
inline NullableValue() noexcept: isSet(false) {}
inline NullableValue(T&& t) noexcept(noexcept(T(instance<T&&>())))
: isSet(true) {
......
c++/src/kj/encoding-test.c++
View file @ df52bf86
......@@ -26,6 +26,7 @@
namespace kj {
namespace {
CappedArray<char, sizeof(char ) * 2 + 1> hex(byte i) { return kj::hex((uint8_t )i); }
CappedArray<char, sizeof(char ) * 2 + 1> hex(char i) { return kj::hex((uint8_t )i); }
CappedArray<char, sizeof(char16_t) * 2 + 1> hex(char16_t i) { return kj::hex((uint16_t)i); }
CappedArray<char, sizeof(char32_t) * 2 + 1> hex(char32_t i) { return kj::hex((uint32_t)i); }
......@@ -34,7 +35,7 @@ CappedArray<char, sizeof(char32_t) * 2 + 1> hex(char32_t i) { return kj::hex((ui
// TODO(cleanup): Should this go into string.h with the other definitions of hex()?
template <typename T>
void expectUtf(UtfResult<T> result,
void expectRes(EncodingResult<T> result,
ArrayPtr<const Decay<decltype(result[0])>> expected,
bool errors = false) {
if (errors) {
......@@ -50,147 +51,153 @@ void expectUtf(UtfResult<T> result,
}
template <typename T, size_t s>
void expectUtf(UtfResult<T> result,
void expectRes(EncodingResult<T> result,
const Decay<decltype(result[0])> (&expected)[s],
bool errors = false) {
expectUtf(kj::mv(result), arrayPtr(expected, s - 1), errors);
expectRes(kj::mv(result), arrayPtr(expected, s - 1), errors);
}
template <typename T, size_t s>
void expectRes(EncodingResult<T> result,
byte (&expected)[s],
bool errors = false) {
expectRes(kj::mv(result), arrayPtr(expected, s), errors);
}
KJ_TEST("encode UTF-8 to UTF-16") {
expectUtf(encodeUtf16(u8"foo"), u"foo");
expectUtf(encodeUtf16(u8"Здравствуйте"), u"Здравствуйте");
expectUtf(encodeUtf16(u8"中国网络"), u"中国网络");
expectUtf(encodeUtf16(u8"😺☁☄🐵"), u"😺☁☄🐵");
expectRes(encodeUtf16(u8"foo"), u"foo");
expectRes(encodeUtf16(u8"Здравствуйте"), u"Здравствуйте");
expectRes(encodeUtf16(u8"中国网络"), u"中国网络");
expectRes(encodeUtf16(u8"😺☁☄🐵"), u"😺☁☄🐵");
}
KJ_TEST("invalid UTF-8 to UTF-16") {
// Disembodied continuation byte.
expectUtf(encodeUtf16("\x80"), u"\ufffd", true);
expectUtf(encodeUtf16("f\xbfo"), u"f\ufffdo", true);
expectUtf(encodeUtf16("f\xbf\x80\xb0o"), u"f\ufffdo", true);
expectRes(encodeUtf16("\x80"), u"\ufffd", true);
expectRes(encodeUtf16("f\xbfo"), u"f\ufffdo", true);
expectRes(encodeUtf16("f\xbf\x80\xb0o"), u"f\ufffdo", true);
// Missing continuation bytes.
expectUtf(encodeUtf16("\xc2x"), u"\ufffdx", true);
expectUtf(encodeUtf16("\xe0x"), u"\ufffdx", true);
expectUtf(encodeUtf16("\xe0\xa0x"), u"\ufffdx", true);
expectUtf(encodeUtf16("\xf0x"), u"\ufffdx", true);
expectUtf(encodeUtf16("\xf0\x90x"), u"\ufffdx", true);
expectUtf(encodeUtf16("\xf0\x90\x80x"), u"\ufffdx", true);
expectRes(encodeUtf16("\xc2x"), u"\ufffdx", true);
expectRes(encodeUtf16("\xe0x"), u"\ufffdx", true);
expectRes(encodeUtf16("\xe0\xa0x"), u"\ufffdx", true);
expectRes(encodeUtf16("\xf0x"), u"\ufffdx", true);
expectRes(encodeUtf16("\xf0\x90x"), u"\ufffdx", true);
expectRes(encodeUtf16("\xf0\x90\x80x"), u"\ufffdx", true);
// Overlong sequences.
expectUtf(encodeUtf16("\xc0\x80"), u"\ufffd", true);
expectUtf(encodeUtf16("\xc1\xbf"), u"\ufffd", true);
expectUtf(encodeUtf16("\xc2\x80"), u"\u0080", false);
expectUtf(encodeUtf16("\xdf\xbf"), u"\u07ff", false);
expectRes(encodeUtf16("\xc0\x80"), u"\ufffd", true);
expectRes(encodeUtf16("\xc1\xbf"), u"\ufffd", true);
expectRes(encodeUtf16("\xc2\x80"), u"\u0080", false);
expectRes(encodeUtf16("\xdf\xbf"), u"\u07ff", false);
expectUtf(encodeUtf16("\xe0\x80\x80"), u"\ufffd", true);
expectUtf(encodeUtf16("\xe0\x9f\xbf"), u"\ufffd", true);
expectUtf(encodeUtf16("\xe0\xa0\x80"), u"\u0800", false);
expectUtf(encodeUtf16("\xef\xbf\xbf"), u"\uffff", false);
expectRes(encodeUtf16("\xe0\x80\x80"), u"\ufffd", true);
expectRes(encodeUtf16("\xe0\x9f\xbf"), u"\ufffd", true);
expectRes(encodeUtf16("\xe0\xa0\x80"), u"\u0800", false);
expectRes(encodeUtf16("\xef\xbf\xbf"), u"\uffff", false);
expectUtf(encodeUtf16("\xf0\x80\x80\x80"), u"\ufffd", true);
expectUtf(encodeUtf16("\xf0\x8f\xbf\xbf"), u"\ufffd", true);
expectUtf(encodeUtf16("\xf0\x90\x80\x80"), u"\U00010000", false);
expectUtf(encodeUtf16("\xf4\x8f\xbf\xbf"), u"\U0010ffff", false);
expectRes(encodeUtf16("\xf0\x80\x80\x80"), u"\ufffd", true);
expectRes(encodeUtf16("\xf0\x8f\xbf\xbf"), u"\ufffd", true);
expectRes(encodeUtf16("\xf0\x90\x80\x80"), u"\U00010000", false);
expectRes(encodeUtf16("\xf4\x8f\xbf\xbf"), u"\U0010ffff", false);
// Out of Unicode range.
expectUtf(encodeUtf16("\xf5\x80\x80\x80"), u"\ufffd", true);
expectUtf(encodeUtf16("\xf8\xbf\x80\x80\x80"), u"\ufffd", true);
expectUtf(encodeUtf16("\xfc\xbf\x80\x80\x80\x80"), u"\ufffd", true);
expectUtf(encodeUtf16("\xfe\xbf\x80\x80\x80\x80\x80"), u"\ufffd", true);
expectUtf(encodeUtf16("\xff\xbf\x80\x80\x80\x80\x80\x80"), u"\ufffd", true);
expectRes(encodeUtf16("\xf5\x80\x80\x80"), u"\ufffd", true);
expectRes(encodeUtf16("\xf8\xbf\x80\x80\x80"), u"\ufffd", true);
expectRes(encodeUtf16("\xfc\xbf\x80\x80\x80\x80"), u"\ufffd", true);
expectRes(encodeUtf16("\xfe\xbf\x80\x80\x80\x80\x80"), u"\ufffd", true);
expectRes(encodeUtf16("\xff\xbf\x80\x80\x80\x80\x80\x80"), u"\ufffd", true);
}
KJ_TEST("encode UTF-8 to UTF-32") {
expectUtf(encodeUtf32(u8"foo"), U"foo");
expectUtf(encodeUtf32(u8"Здравствуйте"), U"Здравствуйте");
expectUtf(encodeUtf32(u8"中国网络"), U"中国网络");
expectUtf(encodeUtf32(u8"😺☁☄🐵"), U"😺☁☄🐵");
expectRes(encodeUtf32(u8"foo"), U"foo");
expectRes(encodeUtf32(u8"Здравствуйте"), U"Здравствуйте");
expectRes(encodeUtf32(u8"中国网络"), U"中国网络");
expectRes(encodeUtf32(u8"😺☁☄🐵"), U"😺☁☄🐵");
}
KJ_TEST("invalid UTF-8 to UTF-32") {
// Disembodied continuation byte.
expectUtf(encodeUtf32("\x80"), U"\ufffd", true);
expectUtf(encodeUtf32("f\xbfo"), U"f\ufffdo", true);
expectUtf(encodeUtf32("f\xbf\x80\xb0o"), U"f\ufffdo", true);
expectRes(encodeUtf32("\x80"), U"\ufffd", true);
expectRes(encodeUtf32("f\xbfo"), U"f\ufffdo", true);
expectRes(encodeUtf32("f\xbf\x80\xb0o"), U"f\ufffdo", true);
// Missing continuation bytes.
expectUtf(encodeUtf32("\xc2x"), U"\ufffdx", true);
expectUtf(encodeUtf32("\xe0x"), U"\ufffdx", true);
expectUtf(encodeUtf32("\xe0\xa0x"), U"\ufffdx", true);
expectUtf(encodeUtf32("\xf0x"), U"\ufffdx", true);
expectUtf(encodeUtf32("\xf0\x90x"), U"\ufffdx", true);
expectUtf(encodeUtf32("\xf0\x90\x80x"), U"\ufffdx", true);
expectRes(encodeUtf32("\xc2x"), U"\ufffdx", true);
expectRes(encodeUtf32("\xe0x"), U"\ufffdx", true);
expectRes(encodeUtf32("\xe0\xa0x"), U"\ufffdx", true);
expectRes(encodeUtf32("\xf0x"), U"\ufffdx", true);
expectRes(encodeUtf32("\xf0\x90x"), U"\ufffdx", true);
expectRes(encodeUtf32("\xf0\x90\x80x"), U"\ufffdx", true);
// Overlong sequences.
expectUtf(encodeUtf32("\xc0\x80"), U"\ufffd", true);
expectUtf(encodeUtf32("\xc1\xbf"), U"\ufffd", true);
expectUtf(encodeUtf32("\xc2\x80"), U"\u0080", false);
expectUtf(encodeUtf32("\xdf\xbf"), U"\u07ff", false);
expectRes(encodeUtf32("\xc0\x80"), U"\ufffd", true);
expectRes(encodeUtf32("\xc1\xbf"), U"\ufffd", true);
expectRes(encodeUtf32("\xc2\x80"), U"\u0080", false);
expectRes(encodeUtf32("\xdf\xbf"), U"\u07ff", false);
expectUtf(encodeUtf32("\xe0\x80\x80"), U"\ufffd", true);
expectUtf(encodeUtf32("\xe0\x9f\xbf"), U"\ufffd", true);
expectUtf(encodeUtf32("\xe0\xa0\x80"), U"\u0800", false);
expectUtf(encodeUtf32("\xef\xbf\xbf"), U"\uffff", false);
expectRes(encodeUtf32("\xe0\x80\x80"), U"\ufffd", true);
expectRes(encodeUtf32("\xe0\x9f\xbf"), U"\ufffd", true);
expectRes(encodeUtf32("\xe0\xa0\x80"), U"\u0800", false);
expectRes(encodeUtf32("\xef\xbf\xbf"), U"\uffff", false);
expectUtf(encodeUtf32("\xf0\x80\x80\x80"), U"\ufffd", true);
expectUtf(encodeUtf32("\xf0\x8f\xbf\xbf"), U"\ufffd", true);
expectUtf(encodeUtf32("\xf0\x90\x80\x80"), U"\U00010000", false);
expectUtf(encodeUtf32("\xf4\x8f\xbf\xbf"), U"\U0010ffff", false);
expectRes(encodeUtf32("\xf0\x80\x80\x80"), U"\ufffd", true);
expectRes(encodeUtf32("\xf0\x8f\xbf\xbf"), U"\ufffd", true);
expectRes(encodeUtf32("\xf0\x90\x80\x80"), U"\U00010000", false);
expectRes(encodeUtf32("\xf4\x8f\xbf\xbf"), U"\U0010ffff", false);
// Out of Unicode range.
expectUtf(encodeUtf32("\xf5\x80\x80\x80"), U"\ufffd", true);
expectUtf(encodeUtf32("\xf8\xbf\x80\x80\x80"), U"\ufffd", true);
expectUtf(encodeUtf32("\xfc\xbf\x80\x80\x80\x80"), U"\ufffd", true);
expectUtf(encodeUtf32("\xfe\xbf\x80\x80\x80\x80\x80"), U"\ufffd", true);
expectUtf(encodeUtf32("\xff\xbf\x80\x80\x80\x80\x80\x80"), U"\ufffd", true);
expectRes(encodeUtf32("\xf5\x80\x80\x80"), U"\ufffd", true);
expectRes(encodeUtf32("\xf8\xbf\x80\x80\x80"), U"\ufffd", true);
expectRes(encodeUtf32("\xfc\xbf\x80\x80\x80\x80"), U"\ufffd", true);
expectRes(encodeUtf32("\xfe\xbf\x80\x80\x80\x80\x80"), U"\ufffd", true);
expectRes(encodeUtf32("\xff\xbf\x80\x80\x80\x80\x80\x80"), U"\ufffd", true);
}
KJ_TEST("decode UTF-16 to UTF-8") {
expectUtf(decodeUtf16(u"foo"), u8"foo");
expectUtf(decodeUtf16(u"Здравствуйте"), u8"Здравствуйте");
expectUtf(decodeUtf16(u"中国网络"), u8"中国网络");
expectUtf(decodeUtf16(u"😺☁☄🐵"), u8"😺☁☄🐵");
expectRes(decodeUtf16(u"foo"), u8"foo");
expectRes(decodeUtf16(u"Здравствуйте"), u8"Здравствуйте");
expectRes(decodeUtf16(u"中国网络"), u8"中国网络");
expectRes(decodeUtf16(u"😺☁☄🐵"), u8"😺☁☄🐵");
}
KJ_TEST("invalid UTF-16 to UTF-8") {
// Surrogates in wrong order.
expectUtf(decodeUtf16(u"\xd7ff\xdc00\xdfff\xe000"), u8"\ud7ff\ufffd\ufffd\ue000", true);
expectRes(decodeUtf16(u"\xd7ff\xdc00\xdfff\xe000"), u8"\ud7ff\ufffd\ufffd\ue000", true);
// Missing second surrogate.
expectUtf(decodeUtf16(u"f\xd800"), u8"f\ufffd", true);
expectUtf(decodeUtf16(u"f\xd800x"), u8"f\ufffdx", true);
expectUtf(decodeUtf16(u"f\xd800\xd800x"), u8"f\ufffd\ufffdx", true);
expectRes(decodeUtf16(u"f\xd800"), u8"f\ufffd", true);
expectRes(decodeUtf16(u"f\xd800x"), u8"f\ufffdx", true);
expectRes(decodeUtf16(u"f\xd800\xd800x"), u8"f\ufffd\ufffdx", true);
}
KJ_TEST("decode UTF-32 to UTF-8") {
expectUtf(decodeUtf32(U"foo"), u8"foo");
expectUtf(decodeUtf32(U"Здравствуйте"), u8"Здравствуйте");
expectUtf(decodeUtf32(U"中国网络"), u8"中国网络");
expectUtf(decodeUtf32(U"😺☁☄🐵"), u8"😺☁☄🐵");
expectRes(decodeUtf32(U"foo"), u8"foo");
expectRes(decodeUtf32(U"Здравствуйте"), u8"Здравствуйте");
expectRes(decodeUtf32(U"中国网络"), u8"中国网络");
expectRes(decodeUtf32(U"😺☁☄🐵"), u8"😺☁☄🐵");
}
KJ_TEST("invalid UTF-32 to UTF-8") {
// Surrogates rejected.
expectUtf(decodeUtf32(U"\xd7ff\xdc00\xdfff\xe000"), u8"\ud7ff\ufffd\ufffd\ue000", true);
expectRes(decodeUtf32(U"\xd7ff\xdc00\xdfff\xe000"), u8"\ud7ff\ufffd\ufffd\ue000", true);
// Even if it would be a valid surrogate pair in UTF-16.
expectUtf(decodeUtf32(U"\xd7ff\xd800\xdfff\xe000"), u8"\ud7ff\ufffd\ufffd\ue000", true);
expectRes(decodeUtf32(U"\xd7ff\xd800\xdfff\xe000"), u8"\ud7ff\ufffd\ufffd\ue000", true);
}
KJ_TEST("tryEncode / tryDecode") {
KJ_EXPECT(tryEncodeUtf16("\x80") == nullptr);
KJ_EXPECT(ArrayPtr<const char16_t>(KJ_ASSERT_NONNULL(tryEncodeUtf16("foo")))
== arrayPtr(u"foo", 3));
KJ_TEST("EncodingResult as a Maybe") {
KJ_IF_MAYBE(result, encodeUtf16("\x80")) {
KJ_FAIL_EXPECT("expected failure");
}
KJ_EXPECT(tryEncodeUtf32("\x80") == nullptr);
KJ_EXPECT(ArrayPtr<const char32_t>(KJ_ASSERT_NONNULL(tryEncodeUtf32("foo")))
== arrayPtr(U"foo", 3));
KJ_IF_MAYBE(result, encodeUtf16("foo")) {
// good
} else {
KJ_FAIL_EXPECT("expected success");
}
KJ_EXPECT(tryDecodeUtf16(u"\xd800") == nullptr);
KJ_EXPECT(KJ_ASSERT_NONNULL(tryDecodeUtf16(u"foo")) == "foo");
KJ_EXPECT(tryDecodeUtf32(U"\xd800") == nullptr);
KJ_EXPECT(KJ_ASSERT_NONNULL(tryDecodeUtf32(U"foo")) == "foo");
KJ_EXPECT(KJ_ASSERT_NONNULL(decodeUtf16(u"foo")) == "foo");
}
// =======================================================================================
......@@ -199,7 +206,16 @@ KJ_TEST("hex encoding/decoding") {
byte bytes[] = {0x12, 0x34, 0xab, 0xf2};
KJ_EXPECT(encodeHex(bytes) == "1234abf2");
KJ_EXPECT(decodeHex("1234abf2").asPtr() == bytes);
expectRes(decodeHex("1234abf2"), bytes);
expectRes(decodeHex("1234abf21"), bytes, true);
bytes[2] = 0xa0;
expectRes(decodeHex("1234axf2"), bytes, true);
bytes[2] = 0x0b;
expectRes(decodeHex("1234xbf2"), bytes, true);
}
KJ_TEST("URI encoding/decoding") {
......@@ -208,8 +224,13 @@ KJ_TEST("URI encoding/decoding") {
KJ_EXPECT(encodeUriComponent("\xab\xba") == "%ab%ba");
KJ_EXPECT(encodeUriComponent(StringPtr("foo\0bar", 7)) == "foo%00bar");
KJ_EXPECT(decodeUriComponent("foo%20bar") == "foo bar");
KJ_EXPECT(decodeUriComponent("%ab%BA") == "\xab\xba");
expectRes(decodeUriComponent("foo%20bar"), "foo bar");
expectRes(decodeUriComponent("%ab%BA"), "\xab\xba");
expectRes(decodeUriComponent("foo%1xxx"), "foo\1xxx", true);
expectRes(decodeUriComponent("foo%1"), "foo\1", true);
expectRes(decodeUriComponent("foo%xxx"), "fooxxx", true);
expectRes(decodeUriComponent("foo%"), "foo", true);
byte bytes[] = {12, 34, 56};
KJ_EXPECT(decodeBinaryUriComponent(encodeUriComponent(bytes)).asPtr() == bytes);
......@@ -221,23 +242,23 @@ KJ_TEST("C escape encoding/decoding") {
KJ_EXPECT(encodeCEscape("foo\x01\x7fxxx") ==
"foo\\001\\177xxx");
expectUtf(decodeCEscape("fooo\\a\\b\\f\\n\\r\\t\\v\\\'\\\"\\\\bar"),
expectRes(decodeCEscape("fooo\\a\\b\\f\\n\\r\\t\\v\\\'\\\"\\\\bar"),
"fooo\a\b\f\n\r\t\v\'\"\\bar");
expectUtf(decodeCEscape("foo\\x01\\x7fxxx"), "foo\x01\x7fxxx");
expectUtf(decodeCEscape("foo\\001\\177234"), "foo\001\177234");
expectUtf(decodeCEscape("foo\\x1"), "foo\x1");
expectUtf(decodeCEscape("foo\\1"), "foo\1");
expectRes(decodeCEscape("foo\\x01\\x7fxxx"), "foo\x01\x7fxxx");
expectRes(decodeCEscape("foo\\001\\177234"), "foo\001\177234");
expectRes(decodeCEscape("foo\\x1"), "foo\x1");
expectRes(decodeCEscape("foo\\1"), "foo\1");
expectUtf(decodeCEscape("foo\\u1234bar"), u8"foo\u1234bar");
expectUtf(decodeCEscape("foo\\U00045678bar"), u8"foo\U00045678bar");
expectRes(decodeCEscape("foo\\u1234bar"), u8"foo\u1234bar");
expectRes(decodeCEscape("foo\\U00045678bar"), u8"foo\U00045678bar");
// Error cases.
expectUtf(decodeCEscape("foo\\"), "foo", true);
expectUtf(decodeCEscape("foo\\x123x"), u8"foo\x23x", true);
expectUtf(decodeCEscape("foo\\u12"), u8"foo\u0012", true);
expectUtf(decodeCEscape("foo\\u12xxx"), u8"foo\u0012xxx", true);
expectUtf(decodeCEscape("foo\\U12"), u8"foo\u0012", true);
expectUtf(decodeCEscape("foo\\U12xxxxxxxx"), u8"foo\u0012xxxxxxxx", true);
expectRes(decodeCEscape("foo\\"), "foo", true);
expectRes(decodeCEscape("foo\\x123x"), u8"foo\x23x", true);
expectRes(decodeCEscape("foo\\u12"), u8"foo\u0012", true);
expectRes(decodeCEscape("foo\\u12xxx"), u8"foo\u0012xxx", true);
expectRes(decodeCEscape("foo\\U12"), u8"foo\u0012", true);
expectRes(decodeCEscape("foo\\U12xxxxxxxx"), u8"foo\u0012xxxxxxxx", true);
}
KJ_TEST("base64 encoding/decoding") {
......
c++/src/kj/encoding.c++
View file @ df52bf86
......@@ -41,7 +41,7 @@ inline void addChar32(Vector<char32_t>& vec, char32_t u) {
}
template <typename T>
UtfResult<Array<T>> encodeUtf(ArrayPtr<const char> text, bool nulTerminate) {
EncodingResult<Array<T>> encodeUtf(ArrayPtr<const char> text, bool nulTerminate) {
Vector<T> result(text.size() + nulTerminate);
bool hadErrors = false;
......@@ -125,33 +125,15 @@ UtfResult<Array<T>> encodeUtf(ArrayPtr<const char> text, bool nulTerminate) {
} // namespace
UtfResult<Array<char16_t>> encodeUtf16(ArrayPtr<const char> text, bool nulTerminate) {
EncodingResult<Array<char16_t>> encodeUtf16(ArrayPtr<const char> text, bool nulTerminate) {
return encodeUtf<char16_t>(text, nulTerminate);
}
UtfResult<Array<char32_t>> encodeUtf32(ArrayPtr<const char> text, bool nulTerminate) {
EncodingResult<Array<char32_t>> encodeUtf32(ArrayPtr<const char> text, bool nulTerminate) {
return encodeUtf<char32_t>(text, nulTerminate);
}
Maybe<Array<char16_t>> tryEncodeUtf16(ArrayPtr<const char> text, bool nulTerminate) {
auto result = encodeUtf16(text, nulTerminate);
if (result.hadErrors) {
return nullptr;
} else {
return kj::mv(result);
}
}
Maybe<Array<char32_t>> tryEncodeUtf32(ArrayPtr<const char> text, bool nulTerminate) {
auto result = encodeUtf32(text, nulTerminate);
if (result.hadErrors) {
return nullptr;
} else {
return kj::mv(result);
}
}
UtfResult<String> decodeUtf16(ArrayPtr<const char16_t> utf16) {
EncodingResult<String> decodeUtf16(ArrayPtr<const char16_t> utf16) {
Vector<char> result(utf16.size() + 1);
bool hadErrors = false;
......@@ -202,7 +184,7 @@ UtfResult<String> decodeUtf16(ArrayPtr<const char16_t> utf16) {
return { String(result.releaseAsArray()), hadErrors };
}
UtfResult<String> decodeUtf32(ArrayPtr<const char32_t> utf16) {
EncodingResult<String> decodeUtf32(ArrayPtr<const char32_t> utf16) {
Vector<char> result(utf16.size() + 1);
bool hadErrors = false;
......@@ -247,28 +229,33 @@ UtfResult<String> decodeUtf32(ArrayPtr<const char32_t> utf16) {
return { String(result.releaseAsArray()), hadErrors };
}
Maybe<String> tryDecodeUtf16(ArrayPtr<const char16_t> utf16) {
auto result = decodeUtf16(utf16);
if (result.hadErrors) {
return nullptr;
// =======================================================================================
namespace {
const char HEX_DIGITS[] = "0123456789abcdef";
static Maybe<uint> tryFromHexDigit(char c) {
if ('0' <= c && c <= '9') {
return c - '0';
} else if ('a' <= c && c <= 'f') {
return c - ('a' - 10);
} else if ('A' <= c && c <= 'F') {
return c - ('A' - 10);
} else {
return kj::mv(result);
return nullptr;
}
}
Maybe<String> tryDecodeUtf32(ArrayPtr<const char32_t> utf32) {
auto result = decodeUtf32(utf32);
if (result.hadErrors) {
return nullptr;
static Maybe<uint> tryFromOctDigit(char c) {
if ('0' <= c && c <= '7') {
return c - '0';
} else {
return kj::mv(result);
return nullptr;
}
}
// =======================================================================================
namespace {
const char HEX_DIGITS[] = "0123456789abcdef";
}
} // namespace
String encodeHex(ArrayPtr<const byte> input) {
return strArray(KJ_MAP(b, input) {
......@@ -276,27 +263,26 @@ String encodeHex(ArrayPtr<const byte> input) {
}, "");
}
static uint fromDigit(char c) {
if ('0' <= c && c <= '9') {
return c - '0';
} else if ('a' <= c && c <= 'z') {
return c - ('a' - 10);
} else if ('A' <= c && c <= 'Z') {
return c - ('A' - 10);
} else {
return 0;
}
}
Array<byte> decodeHex(ArrayPtr<const char> text) {
EncodingResult<Array<byte>> decodeHex(ArrayPtr<const char> text) {
auto result = heapArray<byte>(text.size() / 2);
bool hadErrors = text.size() % 2;
for (auto i: kj::indices(result)) {
result[i] = (fromDigit(text[i*2]) << 4)
| (fromDigit(text[i*2+1]));
byte b = 0;
KJ_IF_MAYBE(d1, tryFromHexDigit(text[i*2])) {
b = *d1 << 4;
} else {
hadErrors = true;
}
KJ_IF_MAYBE(d2, tryFromHexDigit(text[i*2+1])) {
b |= *d2;
} else {
hadErrors = true;
}
result[i] = b;
}
return result;
return { kj::mv(result), hadErrors };
}
String encodeUriComponent(ArrayPtr<const byte> bytes) {
......@@ -316,26 +302,41 @@ String encodeUriComponent(ArrayPtr<const byte> bytes) {
return String(result.releaseAsArray());
}
Array<byte> decodeBinaryUriComponent(ArrayPtr<const char> text, bool nulTerminate) {
EncodingResult<Array<byte>> decodeBinaryUriComponent(
ArrayPtr<const char> text, bool nulTerminate) {
Vector<byte> result(text.size() + nulTerminate);
bool hadErrors = false;
const char* ptr = text.begin();
const char* end = text.end();
while (ptr < end) {
if (*ptr == '%') {
++ptr;
if (ptr == end) break;
byte b = fromDigit(*ptr++) << 4;
if (ptr == end) break;
b |= fromDigit(*ptr++);
result.add(b);
if (ptr == end) {
hadErrors = true;
} else KJ_IF_MAYBE(d1, tryFromHexDigit(*ptr)) {
byte b = *d1;
++ptr;
if (ptr == end) {
hadErrors = true;
} else KJ_IF_MAYBE(d2, tryFromHexDigit(*ptr)) {
b = (b << 4) | *d2;
++ptr;
} else {
hadErrors = true;
}
result.add(b);
} else {
hadErrors = true;
}
} else {
result.add(*ptr++);
}
}
if (nulTerminate) result.add(0);
return result.releaseAsArray();
return { result.releaseAsArray(), hadErrors };
}
// =======================================================================================
......@@ -374,31 +375,7 @@ String encodeCEscape(ArrayPtr<const byte> bytes) {
return String(escaped.releaseAsArray());
}
namespace {
static Maybe<uint> tryFromHexDigit(char c) {
if ('0' <= c && c <= '9') {
return c - '0';
} else if ('a' <= c && c <= 'f') {
return c - ('a' - 10);
} else if ('A' <= c && c <= 'F') {
return c - ('A' - 10);
} else {
return nullptr;
}
}
static Maybe<uint> tryFromOctDigit(char c) {
if ('0' <= c && c <= '7') {
return c - '0';
} else {
return nullptr;
}
}
} // namespace
UtfResult<Array<byte>> decodeBinaryCEscape(ArrayPtr<const char> text, bool nulTerminate) {
EncodingResult<Array<byte>> decodeBinaryCEscape(ArrayPtr<const char> text, bool nulTerminate) {
Vector<byte> result(text.size() + nulTerminate);
bool hadErrors = false;
......
c++/src/kj/encoding.h
View file @ df52bf86
......@@ -36,24 +36,24 @@
namespace kj {
template <typename ResultType>
struct UtfResult: public ResultType {
struct EncodingResult: public ResultType {
// Equivalent to ResultType (a String or wide-char array) for all intents and purposes, except
// that the bool `hadErrors` can be inspected to see if any errors were encountered in the input,
// resulting in instances of the replacement character (U+FFFD) in the output.
inline UtfResult(ResultType&& result, bool hadErrors)
// that the bool `hadErrors` can be inspected to see if any errors were encountered in the input.
// Each encoding/decoding function that returns this type will "work around" errors in some way,
// so an application doesn't strictly have to check for errors. E.g. the Unicode functions
// replace errors with U+FFFD in the output.
//
// Through magic, KJ_IF_MAYBE() and KJ_{REQUIRE,ASSERT}_NONNULL() work on EncodingResult<T>
// exactly if it were a Maybe<T> that is null in case of errors.
inline EncodingResult(ResultType&& result, bool hadErrors)
: ResultType(kj::mv(result)), hadErrors(hadErrors) {}
const bool hadErrors;
// If true, then invalid sequences were detected in the input and were replaced with the Unicode
// replacement character (U+FFFD) in the output. Many applications will chose to ignore this
// boolean and continue on with the damaged data.
};
UtfResult<Array<char16_t>> encodeUtf16(ArrayPtr<const char> text, bool nulTerminate = false);
UtfResult<Array<char32_t>> encodeUtf32(ArrayPtr<const char> text, bool nulTerminate = false);
Maybe<Array<char16_t>> tryEncodeUtf16(ArrayPtr<const char> text, bool nulTerminate = false);
Maybe<Array<char32_t>> tryEncodeUtf32(ArrayPtr<const char> text, bool nulTerminate = false);
EncodingResult<Array<char16_t>> encodeUtf16(ArrayPtr<const char> text, bool nulTerminate = false);
EncodingResult<Array<char32_t>> encodeUtf32(ArrayPtr<const char> text, bool nulTerminate = false);
// Convert UTF-8 text (which KJ strings use) to UTF-16 or UTF-32.
//
// If `nulTerminate` is true, an extra NUL character will be added to the end of the output.
......@@ -64,10 +64,8 @@ Maybe<Array<char32_t>> tryEncodeUtf32(ArrayPtr<const char> text, bool nulTermina
// The returned arrays are in platform-native endianness (otherwise they wouldn't really be
// char16_t / char32_t).
UtfResult<String> decodeUtf16(ArrayPtr<const char16_t> utf16);
UtfResult<String> decodeUtf32(ArrayPtr<const char32_t> utf32);
Maybe<String> tryDecodeUtf16(ArrayPtr<const char16_t> utf16);
Maybe<String> tryDecodeUtf32(ArrayPtr<const char32_t> utf32);
EncodingResult<String> decodeUtf16(ArrayPtr<const char16_t> utf16);
EncodingResult<String> decodeUtf32(ArrayPtr<const char32_t> utf32);
// Convert UTF-16 or UTF-32 to UTF-8 (which KJ strings use).
//
// The input should NOT include a NUL terminator; any NUL characters in the input array will be
......@@ -79,41 +77,76 @@ Maybe<String> tryDecodeUtf32(ArrayPtr<const char32_t> utf32);
// The input must be in platform-native endianness. BOMs are NOT recognized by these functions.
String encodeHex(ArrayPtr<const byte> bytes);
Array<byte> decodeHex(ArrayPtr<const char> text);
EncodingResult<Array<byte>> decodeHex(ArrayPtr<const char> text);
// Encode/decode bytes as hex strings.
String encodeUriComponent(ArrayPtr<const byte> bytes);
String encodeUriComponent(ArrayPtr<const char> bytes);
Array<byte> decodeBinaryUriComponent(ArrayPtr<const char> text, bool nulTerminate = false);
String decodeUriComponent(ArrayPtr<const char> text);
EncodingResult<Array<byte>> decodeBinaryUriComponent(
ArrayPtr<const char> text, bool nulTerminate = false);
EncodingResult<String> decodeUriComponent(ArrayPtr<const char> text);
// Encode/decode URI components using % escapes. See Javascript's encodeURIComponent().
String encodeCEscape(ArrayPtr<const byte> bytes);
String encodeCEscape(ArrayPtr<const char> bytes);
UtfResult<Array<byte>> decodeBinaryCEscape(ArrayPtr<const char> text, bool nulTerminate = false);
UtfResult<String> decodeCEscape(ArrayPtr<const char> text);
EncodingResult<Array<byte>> decodeBinaryCEscape(
ArrayPtr<const char> text, bool nulTerminate = false);
EncodingResult<String> decodeCEscape(ArrayPtr<const char> text);
String encodeBase64(ArrayPtr<const byte> bytes, bool breakLines = false);
// Encode the given bytes as base64 text. If `breakLines` is true, line breaks will be inserted
// into the output every 72 characters (e.g. for encoding e-mail bodies).
Array<byte> decodeBase64(ArrayPtr<const char> text);
// Decode base64 text. Non-base64 characters are ignored.
// Decode base64 text. Non-base64 characters are ignored and padding characters are not requried;
// as such, this function never fails.
// =======================================================================================
// inline implementation details
namespace _ { // private
template <typename T>
NullableValue<T> readMaybe(EncodingResult<T>&& value) {
if (value.hadErrors) {
return nullptr;
} else {
return kj::mv(value);
}
}
template <typename T>
T* readMaybe(EncodingResult<T>& value) {
if (value.hadErrors) {
return nullptr;
} else {
return &value;
}
}
template <typename T>
const T* readMaybe(const EncodingResult<T>& value) {
if (value.hadErrors) {
return nullptr;
} else {
return &value;
}
}
} // namespace _ (private)
inline String encodeUriComponent(ArrayPtr<const char> text) {
return encodeUriComponent(text.asBytes());
}
inline String decodeUriComponent(ArrayPtr<const char> text) {
return String(decodeBinaryUriComponent(text, true).releaseAsChars());
inline EncodingResult<String> decodeUriComponent(ArrayPtr<const char> text) {
auto result = decodeBinaryUriComponent(text, true);
return { String(result.releaseAsChars()), result.hadErrors };
}
inline String encodeCEscape(ArrayPtr<const char> text) {
return encodeCEscape(text.asBytes());
}
inline UtfResult<String> decodeCEscape(ArrayPtr<const char> text) {
inline EncodingResult<String> decodeCEscape(ArrayPtr<const char> text) {
auto result = decodeBinaryCEscape(text, true);
return { String(result.releaseAsChars()), result.hadErrors };
}
......@@ -123,39 +156,23 @@ inline UtfResult<String> decodeCEscape(ArrayPtr<const char> text) {
// only even matters for encoding-test.c++.
template <size_t s>
inline UtfResult<Array<char16_t>> encodeUtf16(const char (&text)[s], bool nulTerminate = false) {
inline EncodingResult<Array<char16_t>> encodeUtf16(const char (&text)[s], bool nulTerminate=false) {
return encodeUtf16(arrayPtr(text, s - 1), nulTerminate);
}
template <size_t s>
inline UtfResult<Array<char32_t>> encodeUtf32(const char (&text)[s], bool nulTerminate = false) {
inline EncodingResult<Array<char32_t>> encodeUtf32(const char (&text)[s], bool nulTerminate=false) {
return encodeUtf32(arrayPtr(text, s - 1), nulTerminate);
}
template <size_t s>
inline Maybe<Array<char16_t>> tryEncodeUtf16(const char (&text)[s], bool nulTerminate = false) {
return tryEncodeUtf16(arrayPtr(text, s - 1), nulTerminate);
}
template <size_t s>
inline Maybe<Array<char32_t>> tryEncodeUtf32(const char (&text)[s], bool nulTerminate = false) {
return tryEncodeUtf32(arrayPtr(text, s - 1), nulTerminate);
}
template <size_t s>
inline UtfResult<String> decodeUtf16(const char16_t (&utf16)[s]) {
inline EncodingResult<String> decodeUtf16(const char16_t (&utf16)[s]) {
return decodeUtf16(arrayPtr(utf16, s - 1));
}
template <size_t s>
inline UtfResult<String> decodeUtf32(const char32_t (&utf32)[s]) {
inline EncodingResult<String> decodeUtf32(const char32_t (&utf32)[s]) {
return decodeUtf32(arrayPtr(utf32, s - 1));
}
template <size_t s>
inline Maybe<String> tryDecodeUtf16(const char16_t (&utf16)[s]) {
return tryDecodeUtf16(arrayPtr(utf16, s - 1));
}
template <size_t s>
inline Maybe<String> tryDecodeUtf32(const char32_t (&utf32)[s]) {
return tryDecodeUtf32(arrayPtr(utf32, s - 1));
}
template <size_t s>
inline Array<byte> decodeHex(const char (&text)[s]) {
inline EncodingResult<Array<byte>> decodeHex(const char (&text)[s]) {
return decodeHex(arrayPtr(text, s - 1));
}
template <size_t s>
......@@ -167,21 +184,20 @@ inline Array<byte> decodeBinaryUriComponent(const char (&text)[s]) {
return decodeBinaryUriComponent(arrayPtr(text, s - 1));
}
template <size_t s>
inline String decodeUriComponent(const char (&text)[s]) {
return String(decodeBinaryUriComponent(arrayPtr(text, s - 1), true).releaseAsChars());
inline EncodingResult<String> decodeUriComponent(const char (&text)[s]) {
return decodeUriComponent(arrayPtr(text, s-1));
}
template <size_t s>
inline String encodeCEscape(const char (&text)[s]) {
return encodeCEscape(arrayPtr(text, s - 1));
}
template <size_t s>
inline UtfResult<Array<byte>> decodeBinaryCEscape(const char (&text)[s]) {
inline EncodingResult<Array<byte>> decodeBinaryCEscape(const char (&text)[s]) {
return decodeBinaryCEscape(arrayPtr(text, s - 1));
}
template <size_t s>
inline UtfResult<String> decodeCEscape(const char (&text)[s]) {
auto result = decodeBinaryCEscape(arrayPtr(text, s - 1), true);
return { String(result.releaseAsChars()), result.hadErrors };
inline EncodingResult<String> decodeCEscape(const char (&text)[s]) {
return decodeCEscape(arrayPtr(text, s-1));
}
template <size_t s>
Array<byte> decodeBase64(const char (&text)[s]) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment