Skip to content

C
capnproto
Unverified Commit ac10f36b authored by Kenton Varda's avatar Kenton Varda Committed by GitHub
Browse files
Options

Merge pull request #596 from capnproto/filesystem-win32 

Implement KJ Filesystem API on Win32
parents 5822b19c 38541d69
Show whitespace changes
Inline Side-by-side
Showing with 2787 additions and 329 deletions
c++/Makefile.am
View file @ ac10f36b
......@@ -142,10 +142,12 @@ includekj_HEADERS = \
src/kj/mutex.h \
src/kj/thread.h \
src/kj/threadlocal.h \
src/kj/filesystem.h \
src/kj/async-prelude.h \
src/kj/async.h \
src/kj/async-inl.h \
src/kj/time.h \
src/kj/timer.h \
src/kj/async-unix.h \
src/kj/async-win32.h \
src/kj/async-io.h \
......@@ -230,6 +232,10 @@ libkj_la_SOURCES= \
src/kj/io.c++ \
src/kj/mutex.c++ \
src/kj/thread.c++ \
src/kj/time.c++ \
src/kj/filesystem.c++ \
src/kj/filesystem-disk-unix.c++ \
src/kj/filesystem-disk-win32.c++ \
src/kj/test-helpers.c++ \
src/kj/main.c++ \
src/kj/parse/char.c++
......@@ -248,7 +254,7 @@ libkj_async_la_SOURCES= \
src/kj/async-io.c++ \
src/kj/async-io-unix.c++ \
src/kj/async-io-win32.c++ \
src/kj/time.c++
src/kj/timer.c++
libkj_http_la_LIBADD = libkj-async.la libkj.la $(ASYNC_LIBS) $(PTHREAD_LIBS)
libkj_http_la_LDFLAGS = -release $(SO_VERSION) -no-undefined
......@@ -449,7 +455,9 @@ capnp_test_LDADD = \
libkj-http.la \
libkj-async.la \
libkj-test.la \
libkj.la
libkj.la \
$(ASYNC_LIBS) \
$(PTHREAD_LIBS)
endif !LITE_MODE
......@@ -473,6 +481,8 @@ capnp_test_SOURCES = \
src/kj/mutex-test.c++ \
src/kj/threadlocal-test.c++ \
src/kj/threadlocal-pthread-test.c++ \
src/kj/filesystem-test.c++ \
src/kj/filesystem-disk-test.c++ \
src/kj/test-test.c++ \
src/capnp/common-test.c++ \
src/capnp/blob-test.c++ \
......
c++/src/kj/CMakeLists.txt
View file @ ac10f36b
......@@ -20,6 +20,10 @@ set(kj_sources_heavy
refcount.c++
string-tree.c++
encoding.c++
time.c++
filesystem.c++
filesystem-disk-unix.c++
filesystem-disk-win32.c++
parse/char.c++
)
if(NOT CAPNP_LITE)
......@@ -48,6 +52,8 @@ set(kj_headers
mutex.h
thread.h
threadlocal.h
filesystem.h
time.h
main.h
windows-sanity.h
)
......@@ -105,7 +111,7 @@ set(kj-async_sources
async-io-win32.c++
async-io.c++
async-io-unix.c++
time.c++
timer.c++
)
set(kj-async_headers
async-prelude.h
......@@ -114,7 +120,7 @@ set(kj-async_headers
async-unix.h
async-win32.h
async-io.h
time.h
timer.h
)
if(NOT CAPNP_LITE)
add_library(kj-async ${kj-async_sources})
......@@ -188,6 +194,8 @@ if(BUILD_TESTING)
one-of-test.c++
function-test.c++
threadlocal-pthread-test.c++
filesystem-test.c++
filesystem-disk-test.c++
parse/common-test.c++
parse/char-test.c++
compat/url-test.c++
......
c++/src/kj/async-io.h
View file @ ac10f36b
......@@ -29,7 +29,7 @@
#include "async.h"
#include "function.h"
#include "thread.h"
#include "time.h"
#include "timer.h"
struct sockaddr;
......
c++/src/kj/async-unix.h
View file @ ac10f36b
......@@ -31,7 +31,7 @@
#endif
#include "async.h"
#include "time.h"
#include "timer.h"
#include "vector.h"
#include "io.h"
#include <signal.h>
......
c++/src/kj/async-win32.h
View file @ ac10f36b
......@@ -27,7 +27,7 @@
#endif
#include "async.h"
#include "time.h"
#include "timer.h"
#include "io.h"
#include <atomic>
#include <inttypes.h>
......
c++/src/kj/compat/readiness-io-test.c++
View file @ ac10f36b
......@@ -169,12 +169,6 @@ KJ_TEST("readiness IO: read many even") {
auto io = setupAsyncIo();
auto pipe = io.provider->newOneWayPipe();
// Abort on hang.
// TODO(now): Remove this.
io.provider->getTimer().afterDelay(1 * kj::SECONDS).then([]() {
abort();
}).detach([](kj::Exception&&) {});
char dummy[8192];
for (auto i: kj::indices(dummy)) {
dummy[i] = "ba"[i%2];
......
c++/src/kj/debug.c++
View file @ ac10f36b
......@@ -34,6 +34,7 @@
#define NOIME 1
#include <windows.h>
#include "windows-sanity.h"
#include "encoding.h"
#endif
namespace kj {
......@@ -354,29 +355,33 @@ void Debug::Fault::init(
#if _WIN32
void Debug::Fault::init(
const char* file, int line, Win32Error osErrorNumber,
const char* file, int line, Win32Result osErrorNumber,
const char* condition, const char* macroArgs, ArrayPtr<String> argValues) {
LPVOID ptr;
// TODO(soon): Use FormatMessageW() instead.
// TODO(soon): Why doesn't this work for winsock errors?
DWORD result = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER |
DWORD result = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER |
FORMAT_MESSAGE_FROM_SYSTEM |
FORMAT_MESSAGE_IGNORE_INSERTS,
NULL, osErrorNumber.number,
MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
(LPTSTR) &ptr, 0, NULL);
(LPWSTR) &ptr, 0, NULL);
String message;
if (result > 0) {
KJ_DEFER(LocalFree(ptr));
exception = new Exception(typeOfWin32Error(osErrorNumber.number), file, line,
makeDescriptionImpl(SYSCALL, condition, 0, reinterpret_cast<char*>(ptr),
macroArgs, argValues));
const wchar_t* desc = reinterpret_cast<wchar_t*>(ptr);
size_t len = wcslen(desc);
if (len > 0 && desc[len-1] == '\n') --len;
if (len > 0 && desc[len-1] == '\r') --len;
message = kj::str('#', osErrorNumber.number, ' ',
decodeWideString(arrayPtr(desc, len)));
} else {
auto message = kj::str("win32 error code: ", osErrorNumber.number);
message = kj::str("win32 error code: ", osErrorNumber.number);
}
exception = new Exception(typeOfWin32Error(osErrorNumber.number), file, line,
makeDescriptionImpl(SYSCALL, condition, 0, message.cStr(),
macroArgs, argValues));
}
}
#endif
......@@ -395,8 +400,8 @@ int Debug::getOsErrorNumber(bool nonblocking) {
}
#if _WIN32
Debug::Win32Error Debug::getWin32Error() {
return Win32Error(::GetLastError());
uint Debug::getWin32ErrorCode() {
return ::GetLastError();
}
#endif
......
c++/src/kj/debug.h
View file @ ac10f36b
......@@ -168,18 +168,18 @@ namespace kj {
#if _WIN32
#define KJ_WIN32(call, ...) \
if (::kj::_::Debug::isWin32Success(call)) {} else \
if (auto _kjWin32Result = ::kj::_::Debug::win32Call(call)) {} else \
for (::kj::_::Debug::Fault f(__FILE__, __LINE__, \
::kj::_::Debug::getWin32Error(), #call, "" #__VA_ARGS__, __VA_ARGS__);; f.fatal())
_kjWin32Result, #call, "" #__VA_ARGS__, __VA_ARGS__);; f.fatal())
#define KJ_WINSOCK(call, ...) \
if ((call) != SOCKET_ERROR) {} else \
if (auto _kjWin32Result = ::kj::_::Debug::winsockCall(call)) {} else \
for (::kj::_::Debug::Fault f(__FILE__, __LINE__, \
::kj::_::Debug::getWin32Error(), #call, "" #__VA_ARGS__, __VA_ARGS__);; f.fatal())
_kjWin32Result, #call, "" #__VA_ARGS__, __VA_ARGS__);; f.fatal())
#define KJ_FAIL_WIN32(code, errorNumber, ...) \
for (::kj::_::Debug::Fault f(__FILE__, __LINE__, \
::kj::_::Debug::Win32Error(errorNumber), code, "" #__VA_ARGS__, __VA_ARGS__);; f.fatal())
::kj::_::Debug::Win32Result(errorNumber), code, "" #__VA_ARGS__, __VA_ARGS__);; f.fatal())
#endif
......@@ -247,18 +247,23 @@ namespace kj {
#if _WIN32
#define KJ_WIN32(call, ...) \
if (::kj::_::Debug::isWin32Success(call)) {} else \
if (auto _kjWin32Result = ::kj::_::Debug::win32Call(call)) {} else \
for (::kj::_::Debug::Fault f(__FILE__, __LINE__, \
::kj::_::Debug::getWin32Error(), #call, #__VA_ARGS__, ##__VA_ARGS__);; f.fatal())
_kjWin32Result, #call, #__VA_ARGS__, ##__VA_ARGS__);; f.fatal())
// Invoke a Win32 syscall that returns either BOOL or HANDLE, and throw an exception if it fails.
#define KJ_WINSOCK(call, ...) \
if ((call) != SOCKET_ERROR) {} else \
if (auto _kjWin32Result = ::kj::_::Debug::winsockCall(call)) {} else \
for (::kj::_::Debug::Fault f(__FILE__, __LINE__, \
::kj::_::Debug::getWin32Error(), #call, #__VA_ARGS__, ##__VA_ARGS__);; f.fatal())
_kjWin32Result, #call, #__VA_ARGS__, ##__VA_ARGS__);; f.fatal())
// Like KJ_WIN32 but for winsock calls which return `int` with SOCKET_ERROR indicating failure.
//
// Unfortunately, it's impossible to distinguish these from BOOL-returning Win32 calls by type,
// since BOOL is in fact an alias for `int`. :(
#define KJ_FAIL_WIN32(code, errorNumber, ...) \
for (::kj::_::Debug::Fault f(__FILE__, __LINE__, \
::kj::_::Debug::Win32Error(errorNumber), code, #__VA_ARGS__, ##__VA_ARGS__);; f.fatal())
::kj::_::Debug::Win32Result(errorNumber), code, #__VA_ARGS__, ##__VA_ARGS__);; f.fatal())
#endif
......@@ -292,7 +297,7 @@ namespace kj {
#define KJ_SYSCALL_HANDLE_ERRORS(call) \
if (int _kjSyscallError = ::kj::_::Debug::syscallError([&](){return (call);}, false)) \
switch (int error = _kjSyscallError)
switch (int error KJ_UNUSED = _kjSyscallError)
// Like KJ_SYSCALL, but doesn't throw. Instead, the block after the macro is a switch block on the
// error. Additionally, the int value `error` is defined within the block. So you can do:
//
......@@ -309,6 +314,29 @@ namespace kj {
// handleSuccessCase();
// }
#if _WIN32
#define KJ_WIN32_HANDLE_ERRORS(call) \
if (uint _kjWin32Error = ::kj::_::Debug::win32Call(call).number) \
switch (uint error KJ_UNUSED = _kjWin32Error)
// Like KJ_WIN32, but doesn't throw. Instead, the block after the macro is a switch block on the
// error. Additionally, the int value `error` is defined within the block. So you can do:
//
// KJ_SYSCALL_HANDLE_ERRORS(foo()) {
// case ERROR_FILE_NOT_FOUND:
// handleNoSuchFile();
// break;
// case ERROR_FILE_EXISTS:
// handleExists();
// break;
// default:
// KJ_FAIL_WIN32("foo()", error);
// } else {
// handleSuccessCase();
// }
#endif
#define KJ_ASSERT KJ_REQUIRE
#define KJ_FAIL_ASSERT KJ_FAIL_REQUIRE
#define KJ_ASSERT_NONNULL KJ_REQUIRE_NONNULL
......@@ -334,10 +362,10 @@ public:
typedef LogSeverity Severity; // backwards-compatibility
#if _WIN32
struct Win32Error {
// Hack for overloading purposes.
struct Win32Result {
uint number;
inline explicit Win32Error(uint number): number(number) {}
inline explicit Win32Result(uint number): number(number) {}
operator bool() const { return number == 0; }
};
#endif
......@@ -363,7 +391,7 @@ public:
Fault(const char* file, int line, int osErrorNumber,
const char* condition, const char* macroArgs);
#if _WIN32
Fault(const char* file, int line, Win32Error osErrorNumber,
Fault(const char* file, int line, Win32Result osErrorNumber,
const char* condition, const char* macroArgs);
#endif
~Fault() noexcept(false);
......@@ -377,7 +405,7 @@ public:
void init(const char* file, int line, int osErrorNumber,
const char* condition, const char* macroArgs, ArrayPtr<String> argValues);
#if _WIN32
void init(const char* file, int line, Win32Error osErrorNumber,
void init(const char* file, int line, Win32Result osErrorNumber,
const char* condition, const char* macroArgs, ArrayPtr<String> argValues);
#endif
......@@ -400,9 +428,10 @@ public:
static int syscallError(Call&& call, bool nonblocking);
#if _WIN32
static bool isWin32Success(int boolean);
static bool isWin32Success(void* handle);
static Win32Error getWin32Error();
static Win32Result win32Call(int boolean);
static Win32Result win32Call(void* handle);
static Win32Result winsockCall(int result);
static uint getWin32ErrorCode();
#endif
class Context: public ExceptionCallback {
......@@ -495,18 +524,22 @@ inline Debug::Fault::Fault(const char* file, int line, kj::Exception::Type type,
}
#if _WIN32
inline Debug::Fault::Fault(const char* file, int line, Win32Error osErrorNumber,
inline Debug::Fault::Fault(const char* file, int line, Win32Result osErrorNumber,
const char* condition, const char* macroArgs)
: exception(nullptr) {
init(file, line, osErrorNumber, condition, macroArgs, nullptr);
}
inline bool Debug::isWin32Success(int boolean) {
return boolean;
inline Debug::Win32Result Debug::win32Call(int boolean) {
return boolean ? Win32Result(0) : Win32Result(getWin32ErrorCode());
}
inline bool Debug::isWin32Success(void* handle) {
inline Debug::Win32Result Debug::win32Call(void* handle) {
// Assume null and INVALID_HANDLE_VALUE mean failure.
return handle != nullptr && handle != (void*)-1;
return win32Call(handle != nullptr && handle != (void*)-1);
}
inline Debug::Win32Result Debug::winsockCall(int result) {
// Expect a return value of SOCKET_ERROR means failure.
return win32Call(result != -1);
}
#endif
......
c++/src/kj/encoding-test.c++
View file @ ac10f36b
......@@ -30,6 +30,7 @@ CappedArray<char, sizeof(char ) * 2 + 1> hex(byte i) { return kj::hex((ui
CappedArray<char, sizeof(char ) * 2 + 1> hex(char i) { return kj::hex((uint8_t )i); }
CappedArray<char, sizeof(char16_t) * 2 + 1> hex(char16_t i) { return kj::hex((uint16_t)i); }
CappedArray<char, sizeof(char32_t) * 2 + 1> hex(char32_t i) { return kj::hex((uint32_t)i); }
CappedArray<char, sizeof(uint32_t) * 2 + 1> hex(wchar_t i) { return kj::hex((uint32_t)i); }
// Hexify chars correctly.
//
// TODO(cleanup): Should this go into string.h with the other definitions of hex()?
......@@ -64,6 +65,13 @@ void expectRes(EncodingResult<T> result,
expectResImpl(kj::mv(result), arrayPtr<const byte>(expected, s), errors);
}
// Handy reference for surrogate pair edge cases:
//
// \ud800 -> \xed\xa0\x80
// \udc00 -> \xed\xb0\x80
// \udbff -> \xed\xaf\xbf
// \udfff -> \xed\xbf\xbf
KJ_TEST("encode UTF-8 to UTF-16") {
expectRes(encodeUtf16(u8"foo"), u"foo");
expectRes(encodeUtf16(u8"Здравствуйте"), u"Здравствуйте");
......@@ -113,6 +121,21 @@ KJ_TEST("invalid UTF-8 to UTF-16") {
expectRes(encodeUtf16("\xfc\xbf\x80\x80\x80\x80"), u"\ufffd", true);
expectRes(encodeUtf16("\xfe\xbf\x80\x80\x80\x80\x80"), u"\ufffd", true);
expectRes(encodeUtf16("\xff\xbf\x80\x80\x80\x80\x80\x80"), u"\ufffd", true);
// Surrogates encoded as separate UTF-8 code points are flagged as errors but allowed to decode
// to UTF-16 surrogate values.
expectRes(encodeUtf16("\xed\xb0\x80\xed\xaf\xbf"), u"\xdc00\xdbff", true);
expectRes(encodeUtf16("\xed\xbf\xbf\xed\xa0\x80"), u"\xdfff\xd800", true);
expectRes(encodeUtf16("\xed\xb0\x80\xed\xbf\xbf"), u"\xdc00\xdfff", true);
expectRes(encodeUtf16("f\xed\xa0\x80"), u"f\xd800", true);
expectRes(encodeUtf16("f\xed\xa0\x80x"), u"f\xd800x", true);
expectRes(encodeUtf16("f\xed\xa0\x80\xed\xa0\x80x"), u"f\xd800\xd800x", true);
// However, if successive UTF-8 codepoints decode to a proper surrogate pair, the second
// surrogate is replaced with the Unicode replacement character to avoid creating valid UTF-16.
expectRes(encodeUtf16("\xed\xa0\x80\xed\xbf\xbf"), u"\xd800\xfffd", true);
expectRes(encodeUtf16("\xed\xaf\xbf\xed\xb0\x80"), u"\xdbff\xfffd", true);
}
KJ_TEST("encode UTF-8 to UTF-32") {
......@@ -169,12 +192,15 @@ KJ_TEST("decode UTF-16 to UTF-8") {
KJ_TEST("invalid UTF-16 to UTF-8") {
// Surrogates in wrong order.
expectRes(decodeUtf16(u"\xd7ff\xdc00\xdfff\xe000"), u8"\ud7ff\ufffd\ufffd\ue000", true);
expectRes(decodeUtf16(u"\xdc00\xdbff"),
"\xed\xb0\x80\xed\xaf\xbf", true);
expectRes(decodeUtf16(u"\xdfff\xd800"),
"\xed\xbf\xbf\xed\xa0\x80", true);
// Missing second surrogate.
expectRes(decodeUtf16(u"f\xd800"), u8"f\ufffd", true);
expectRes(decodeUtf16(u"f\xd800x"), u8"f\ufffdx", true);
expectRes(decodeUtf16(u"f\xd800\xd800x"), u8"f\ufffd\ufffdx", true);
expectRes(decodeUtf16(u"f\xd800"), "f\xed\xa0\x80", true);
expectRes(decodeUtf16(u"f\xd800x"), "f\xed\xa0\x80x", true);
expectRes(decodeUtf16(u"f\xd800\xd800x"), "f\xed\xa0\x80\xed\xa0\x80x", true);
}
KJ_TEST("decode UTF-32 to UTF-8") {
......@@ -186,10 +212,19 @@ KJ_TEST("decode UTF-32 to UTF-8") {
KJ_TEST("invalid UTF-32 to UTF-8") {
// Surrogates rejected.
expectRes(decodeUtf32(U"\xd7ff\xdc00\xdfff\xe000"), u8"\ud7ff\ufffd\ufffd\ue000", true);
expectRes(decodeUtf32(U"\xdfff\xd800"),
"\xed\xbf\xbf\xed\xa0\x80", true);
// Even if it would be a valid surrogate pair in UTF-16.
expectRes(decodeUtf32(U"\xd7ff\xd800\xdfff\xe000"), u8"\ud7ff\ufffd\ufffd\ue000", true);
expectRes(decodeUtf32(U"\xd800\xdfff"),
"\xed\xa0\x80\xed\xbf\xbf", true);
}
KJ_TEST("round-trip invalid UTF-16") {
const char16_t INVALID[] = u"\xdfff foo \xd800\xdc00 bar \xdc00\xd800 baz \xdbff qux \xd800";
expectRes(encodeUtf16(decodeUtf16(INVALID)), INVALID, true);
expectRes(encodeUtf16(decodeUtf32(encodeUtf32(decodeUtf16(INVALID)))), INVALID, true);
}
KJ_TEST("EncodingResult as a Maybe") {
......@@ -206,6 +241,20 @@ KJ_TEST("EncodingResult as a Maybe") {
KJ_EXPECT(KJ_ASSERT_NONNULL(decodeUtf16(u"foo")) == "foo");
}
KJ_TEST("encode to wchar_t") {
expectRes(encodeWideString(u8"foo"), L"foo");
expectRes(encodeWideString(u8"Здравствуйте"), L"Здравствуйте");
expectRes(encodeWideString(u8"中国网络"), L"中国网络");
expectRes(encodeWideString(u8"😺☁☄🐵"), L"😺☁☄🐵");
}
KJ_TEST("decode from wchar_t") {
expectRes(decodeWideString(L"foo"), u8"foo");
expectRes(decodeWideString(L"Здравствуйте"), u8"Здравствуйте");
expectRes(decodeWideString(L"中国网络"), u8"中国网络");
expectRes(decodeWideString(L"😺☁☄🐵"), u8"😺☁☄🐵");
}
// =======================================================================================
KJ_TEST("hex encoding/decoding") {
......
c++/src/kj/encoding.c++
View file @ ac10f36b
......@@ -79,8 +79,23 @@ EncodingResult<Array<T>> encodeUtf(ArrayPtr<const char> text, bool nulTerminate)
// Disallow overlong sequence.
GOTO_ERROR_IF(u < 0x0800);
// Disallow surrogate pair code points.
GOTO_ERROR_IF((u & 0xf800) == 0xd800);
// Flag surrogate pair code points as errors, but allow them through.
if (KJ_UNLIKELY((u & 0xf800) == 0xd800)) {
if (result.size() > 0 &&
(u & 0xfc00) == 0xdc00 &&
(result.back() & 0xfc00) == 0xd800) {
// Whoops, the *previous* character was also an invalid surrogate, and if we add this
// one too, they'll form a valid surrogate pair. If we allowed this, then it would mean
// invalid UTF-8 round-tripped to UTF-16 and back could actually change meaning entirely.
// OTOH, the reason we allow dangling surrogates is to allow invalid UTF-16 to round-trip
// to UTF-8 without loss, but if the original UTF-16 had a valid surrogate pair, it would
// have been encoded as a valid single UTF-8 codepoint, not as separate UTF-8 codepoints
// for each surrogate.
goto error;
}
hadErrors = true;
}
result.add(u);
continue;
......@@ -153,9 +168,12 @@ EncodingResult<String> decodeUtf16(ArrayPtr<const char16_t> utf16) {
} else if ((u & 0xf800) == 0xd800) {
// surrogate pair
char16_t u2;
GOTO_ERROR_IF(i == utf16.size() // missing second half
if (KJ_UNLIKELY(i == utf16.size() // missing second half
|| (u & 0x0400) != 0 // first half in wrong range
|| ((u2 = utf16[i]) & 0xfc00) != 0xdc00); // second half in wrong range
|| ((u2 = utf16[i]) & 0xfc00) != 0xdc00)) { // second half in wrong range
hadErrors = true;
goto threeByte;
}
++i;
char32_t u32 = (((u & 0x03ff) << 10) | (u2 & 0x03ff)) + 0x10000;
......@@ -167,6 +185,7 @@ EncodingResult<String> decodeUtf16(ArrayPtr<const char16_t> utf16) {
});
continue;
} else {
threeByte:
result.addAll<std::initializer_list<char>>({
static_cast<char>(((u >> 12) ) | 0xe0),
static_cast<char>(((u >> 6) & 0x3f) | 0x80),
......@@ -174,10 +193,6 @@ EncodingResult<String> decodeUtf16(ArrayPtr<const char16_t> utf16) {
});
continue;
}
error:
result.addAll(StringPtr(u8"\ufffd"));
hadErrors = true;
}
result.add(0);
......@@ -202,7 +217,10 @@ EncodingResult<String> decodeUtf32(ArrayPtr<const char32_t> utf16) {
});
continue;
} else if (u < 0x10000) {
GOTO_ERROR_IF((u & 0xfffff800) == 0xd800); // no surrogates allowed in utf-32
if (KJ_UNLIKELY((u & 0xfffff800) == 0xd800)) {
// no surrogates allowed in utf-32
hadErrors = true;
}
result.addAll<std::initializer_list<char>>({
static_cast<char>(((u >> 12) ) | 0xe0),
static_cast<char>(((u >> 6) & 0x3f) | 0x80),
......@@ -229,6 +247,85 @@ EncodingResult<String> decodeUtf32(ArrayPtr<const char32_t> utf16) {
return { String(result.releaseAsArray()), hadErrors };
}
namespace {
template <typename To, typename From>
Array<To> coerceTo(Array<From>&& array) {
static_assert(sizeof(To) == sizeof(From), "incompatible coercion");
Array<wchar_t> result;
memcpy(&result, &array, sizeof(array));
memset(&array, 0, sizeof(array));
return result;
}
template <typename To, typename From>
ArrayPtr<To> coerceTo(ArrayPtr<From> array) {
static_assert(sizeof(To) == sizeof(From), "incompatible coercion");
return arrayPtr(reinterpret_cast<To*>(array.begin()), array.size());
}
template <typename To, typename From>
EncodingResult<Array<To>> coerceTo(EncodingResult<Array<From>>&& result) {
return { coerceTo<To>(Array<From>(kj::mv(result))), result.hadErrors };
}
template <size_t s>
struct WideConverter;
template <>
struct WideConverter<sizeof(char)> {
typedef char Type;
static EncodingResult<Array<char>> encode(ArrayPtr<const char> text, bool nulTerminate) {
auto result = heapArray<char>(text.size() + nulTerminate);
memcpy(result.begin(), text.begin(), text.size());
if (nulTerminate) result.back() = 0;
return { kj::mv(result), false };
}
static EncodingResult<kj::String> decode(ArrayPtr<const char> text) {
return { kj::heapString(text), false };
}
};
template <>
struct WideConverter<sizeof(char16_t)> {
typedef char16_t Type;
static inline EncodingResult<Array<char16_t>> encode(
ArrayPtr<const char> text, bool nulTerminate) {
return encodeUtf16(text, nulTerminate);
}
static inline EncodingResult<kj::String> decode(ArrayPtr<const char16_t> text) {
return decodeUtf16(text);
}
};
template <>
struct WideConverter<sizeof(char32_t)> {
typedef char32_t Type;
static inline EncodingResult<Array<char32_t>> encode(
ArrayPtr<const char> text, bool nulTerminate) {
return encodeUtf32(text, nulTerminate);
}
static inline EncodingResult<kj::String> decode(ArrayPtr<const char32_t> text) {
return decodeUtf32(text);
}
};
} // namespace
EncodingResult<Array<wchar_t>> encodeWideString(ArrayPtr<const char> text, bool nulTerminate) {
return coerceTo<wchar_t>(WideConverter<sizeof(wchar_t)>::encode(text, nulTerminate));
}
EncodingResult<String> decodeWideString(ArrayPtr<const wchar_t> wide) {
using Converter = WideConverter<sizeof(wchar_t)>;
return Converter::decode(coerceTo<const Converter::Type>(wide));
}
// =======================================================================================
namespace {
......
c++/src/kj/encoding.h
View file @ ac10f36b
......@@ -52,17 +52,24 @@ struct EncodingResult: public ResultType {
const bool hadErrors;
};
template <typename T>
inline auto KJ_STRINGIFY(const EncodingResult<T>& value)
-> decltype(toCharSequence(implicitCast<const T&>(value))) {
return toCharSequence(implicitCast<const T&>(value));
}
EncodingResult<Array<char16_t>> encodeUtf16(ArrayPtr<const char> text, bool nulTerminate = false);
EncodingResult<Array<char32_t>> encodeUtf32(ArrayPtr<const char> text, bool nulTerminate = false);
// Convert UTF-8 text (which KJ strings use) to UTF-16 or UTF-32.
//
// If `nulTerminate` is true, an extra NUL character will be added to the end of the output.
//
// The `try` versions return null if the input is invalid; the non-`try` versions return data
// containing the Unicode replacement character (U+FFFD).
//
// The returned arrays are in platform-native endianness (otherwise they wouldn't really be
// char16_t / char32_t).
//
// Note that the KJ Unicode encoding and decoding functions actually implement
// [WTF-8 encoding](http://simonsapin.github.io/wtf-8/), which affects how invalid input is
// handled. See comments on decodeUtf16() for more info.
EncodingResult<String> decodeUtf16(ArrayPtr<const char16_t> utf16);
EncodingResult<String> decodeUtf32(ArrayPtr<const char32_t> utf32);
......@@ -71,10 +78,46 @@ EncodingResult<String> decodeUtf32(ArrayPtr<const char32_t> utf32);
// The input should NOT include a NUL terminator; any NUL characters in the input array will be
// preserved in the output.
//
// The `try` versions return null if the input is invalid; the non-`try` versions return data
// containing the Unicode replacement character (U+FFFD).
//
// The input must be in platform-native endianness. BOMs are NOT recognized by these functions.
//
// Note that the KJ Unicode encoding and decoding functions actually implement
// [WTF-8 encoding](http://simonsapin.github.io/wtf-8/). This means that if you start with an array
// of char16_t and you pass it through any number of conversions to other Unicode encodings,
// eventually returning it to UTF-16, all the while ignoring `hadErrors`, you will end up with
// exactly the same char16_t array you started with, *even if* the array is not valid UTF-16. This
// is useful because many real-world systems that were designed for UCS-2 (plain 16-bit Unicode)
// and later "upgraded" to UTF-16 do not enforce that their UTF-16 is well-formed. For example,
// file names on Windows NT are encoded using 16-bit characters, without enforcing that the
// character sequence is valid UTF-16. It is important that programs on Windows be able to handle
// such filenames, even if they choose to convert the name to UTF-8 for internal processing.
//
// Specifically, KJ's Unicode handling allows unpaired surrogate code points to round-trip through
// UTF-8 and UTF-32. Unpaired surrogates will be flagged as an error (setting `hadErrors` in the
// result), but will NOT be replaced with the Unicode replacement character as other erroneous
// sequences would be, but rather encoded as an invalid surrogate codepoint in the target encoding.
//
// KJ makes the following guarantees about invalid input:
// - A round trip from UTF-16 to other encodings and back will produce exactly the original input,
// with every leg of the trip raising the `hadErrors` flag if the original input was not valid.
// - A round trip from UTF-8 or UTF-32 to other encodings and back will either produce exactly
// the original input, or will have replaced some invalid sequences with the Unicode replacement
// character, U+FFFD. No code units will ever be removed unless they are replaced with U+FFFD,
// and no code units will ever be added except to encode U+FFFD. If the original input was not
// valid, the `hadErrors` flag will be raised on the first leg of the trip, and will also be
// raised on subsequent legs unless all invalid sequences were replaced with U+FFFD (which, after
// all, is a valid code point).
EncodingResult<Array<wchar_t>> encodeWideString(
ArrayPtr<const char> text, bool nulTerminate = false);
EncodingResult<String> decodeWideString(ArrayPtr<const wchar_t> wide);
// Encode / decode strings of wchar_t, aka "wide strings". Unfortunately, different platforms have
// different definitions for wchar_t. For example, on Windows they are 16-bit and encode UTF-16,
// but on Linux they are 32-bit and encode UTF-32. Some platforms even define wchar_t as 8-bit,
// encoding UTF-8 (e.g. BeOS did this).
//
// KJ assumes that wide strings use the UTF encoding that corresponds to the size of wchar_t on
// the target platform. So, these functions are simple aliases for encodeUtf*/decodeUtf*, above
// (or simply make a copy if wchar_t is 8 bits).
String encodeHex(ArrayPtr<const byte> bytes);
EncodingResult<Array<byte>> decodeHex(ArrayPtr<const char> text);
......@@ -164,6 +207,11 @@ inline EncodingResult<Array<char32_t>> encodeUtf32(const char (&text)[s], bool n
return encodeUtf32(arrayPtr(text, s - 1), nulTerminate);
}
template <size_t s>
inline EncodingResult<Array<wchar_t>> encodeWideString(
const char (&text)[s], bool nulTerminate=false) {
return encodeWideString(arrayPtr(text, s - 1), nulTerminate);
}
template <size_t s>
inline EncodingResult<String> decodeUtf16(const char16_t (&utf16)[s]) {
return decodeUtf16(arrayPtr(utf16, s - 1));
}
......@@ -172,6 +220,10 @@ inline EncodingResult<String> decodeUtf32(const char32_t (&utf32)[s]) {
return decodeUtf32(arrayPtr(utf32, s - 1));
}
template <size_t s>
inline EncodingResult<String> decodeWideString(const wchar_t (&utf32)[s]) {
return decodeWideString(arrayPtr(utf32, s - 1));
}
template <size_t s>
inline EncodingResult<Array<byte>> decodeHex(const char (&text)[s]) {
return decodeHex(arrayPtr(text, s - 1));
}
......
c++/src/kj/exception.c++
View file @ ac10f36b
......@@ -152,7 +152,9 @@ ArrayPtr<void* const> getStackTrace(ArrayPtr<void*> space, uint ignoreCount,
break;
}
space[count] = reinterpret_cast<void*>(frame.AddrPC.Offset);
// Subtract 1 from each address so that we identify the calling instructions, rather than the
// return addresses (which are typically the instruction after the call).
space[count] = reinterpret_cast<void*>(frame.AddrPC.Offset - 1);
}
return space.slice(kj::min(ignoreCount, count), count);
......@@ -346,7 +348,7 @@ BOOL WINAPI breakHandler(DWORD type) {
context.ContextFlags = CONTEXT_FULL;
if (GetThreadContext(thread, &context)) {
void* traceSpace[32];
auto trace = getStackTrace(traceSpace, 2, thread, context);
auto trace = getStackTrace(traceSpace, 0, thread, context);
ResumeThread(thread);
auto message = kj::str("*** Received CTRL+C. stack: ",
stringifyStackTraceAddresses(trace),
......@@ -367,11 +369,51 @@ BOOL WINAPI breakHandler(DWORD type) {
return FALSE; // still crash
}
kj::StringPtr exceptionDescription(DWORD code) {
switch (code) {
case EXCEPTION_ACCESS_VIOLATION: return "access violation";
case EXCEPTION_ARRAY_BOUNDS_EXCEEDED: return "array bounds exceeded";
case EXCEPTION_BREAKPOINT: return "breakpoint";
case EXCEPTION_DATATYPE_MISALIGNMENT: return "datatype misalignment";
case EXCEPTION_FLT_DENORMAL_OPERAND: return "denormal floating point operand";
case EXCEPTION_FLT_DIVIDE_BY_ZERO: return "floating point division by zero";
case EXCEPTION_FLT_INEXACT_RESULT: return "inexact floating point result";
case EXCEPTION_FLT_INVALID_OPERATION: return "invalid floating point operation";
case EXCEPTION_FLT_OVERFLOW: return "floating point overflow";
case EXCEPTION_FLT_STACK_CHECK: return "floating point stack overflow";
case EXCEPTION_FLT_UNDERFLOW: return "floating point underflow";
case EXCEPTION_ILLEGAL_INSTRUCTION: return "illegal instruction";
case EXCEPTION_IN_PAGE_ERROR: return "page error";
case EXCEPTION_INT_DIVIDE_BY_ZERO: return "integer divided by zero";
case EXCEPTION_INT_OVERFLOW: return "integer overflow";
case EXCEPTION_INVALID_DISPOSITION: return "invalid disposition";
case EXCEPTION_NONCONTINUABLE_EXCEPTION: return "noncontinuable exception";
case EXCEPTION_PRIV_INSTRUCTION: return "privileged instruction";
case EXCEPTION_SINGLE_STEP: return "single step";
case EXCEPTION_STACK_OVERFLOW: return "stack overflow";
default: return "(unknown exception code)";
}
}
LONG WINAPI sehHandler(EXCEPTION_POINTERS* info) {
void* traceSpace[32];
auto trace = getStackTrace(traceSpace, 0, GetCurrentThread(), *info->ContextRecord);
auto message = kj::str("*** Received structured exception #0x",
hex(info->ExceptionRecord->ExceptionCode), ": ",
exceptionDescription(info->ExceptionRecord->ExceptionCode),
"; stack: ",
stringifyStackTraceAddresses(trace),
stringifyStackTrace(trace), '\n');
FdOutputStream(STDERR_FILENO).write(message.begin(), message.size());
return EXCEPTION_EXECUTE_HANDLER; // still crash
}
} // namespace
void printStackTraceOnCrash() {
mainThreadId = GetCurrentThreadId();
KJ_WIN32(SetConsoleCtrlHandler(breakHandler, TRUE));
SetUnhandledExceptionFilter(&sehHandler);
}
#elif KJ_HAS_BACKTRACE
......
c++/src/kj/filesystem-disk-generic-test.c++
View file @ ac10f36b
......@@ -19,8 +19,18 @@
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
// This test compiles filesystem-disk.c++ with various features #undefed, causing it to take
#if __linux__
// This test compiles filesystem-disk-unix.c++ with various features #undefed, causing it to take
// different code paths, then runs filesystem-disk-test.c++ against that.
//
// This test is only intended to run on Linux, but is intended to make the code behave like it
// would on a generic flavor of Unix.
//
// At present this test only runs under Ekam builds. Integrating it into other builds would be
// awkward since it #includes filesystem-disk-unix.c++, so it cannot link against that file, but
// needs to link against the rest of KJ. Ekam "just figures it out", but other build systems would
// require a lot of work here.
#include "filesystem.h"
#include "debug.h"
......@@ -53,5 +63,7 @@
#define HOLES_NOT_SUPPORTED
#include "filesystem-disk.c++"
#include "filesystem-disk-unix.c++"
#include "filesystem-disk-test.c++"
#endif // __linux__
c++/src/kj/filesystem-disk-old-kernel-test.c++
View file @ ac10f36b
......@@ -29,6 +29,11 @@
//
// This test must be compiled as a separate program, since it alters the calling process by
// enabling seccomp to disable the kernel features.
//
// At present this test only runs under Ekam builds. It *could* reasonably easily be added to the
// autotools or cmake builds, but would require compiling a separate test binary, which is a bit
// weird, and may lead to spurious error reports on systems that don't support seccomp for whatever
// reason.
#include <syscall.h>
#include <unistd.h>
......
c++/src/kj/filesystem-disk-test.c++
View file @ ac10f36b
......@@ -21,17 +21,185 @@
#include "filesystem.h"
#include "test.h"
#include "encoding.h"
#include <stdlib.h>
#if _WIN32
#include <windows.h>
#include "windows-sanity.h"
#else
#include <unistd.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <dirent.h>
#endif
namespace kj {
namespace {
bool isWine() KJ_UNUSED;
#if _WIN32
bool detectWine() {
HMODULE hntdll = GetModuleHandle("ntdll.dll");
if(hntdll == NULL) return false;
return GetProcAddress(hntdll, "wine_get_version") != nullptr;
}
bool isWine() {
static bool result = detectWine();
return result;
}
template <typename Func>
static auto newTemp(Func&& create)
-> Decay<decltype(*kj::_::readMaybe(create(Array<wchar_t>())))> {
wchar_t wtmpdir[MAX_PATH + 1];
DWORD len = GetTempPathW(kj::size(wtmpdir), wtmpdir);
KJ_ASSERT(len < kj::size(wtmpdir));
auto tmpdir = decodeWideString(arrayPtr(wtmpdir, len));
static uint counter = 0;
for (;;) {
auto path = kj::str(tmpdir, "kj-filesystem-test.", GetCurrentProcessId(), ".", counter++);
KJ_IF_MAYBE(result, create(encodeWideString(path, true))) {
return kj::mv(*result);
}
}
}
static Own<File> newTempFile() {
return newTemp([](Array<wchar_t> candidatePath) -> Maybe<Own<File>> {
HANDLE handle;
KJ_WIN32_HANDLE_ERRORS(handle = CreateFileW(
candidatePath.begin(),
FILE_GENERIC_READ | FILE_GENERIC_WRITE,
0,
NULL,
CREATE_NEW,
FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE,
NULL)) {
case ERROR_ALREADY_EXISTS:
case ERROR_FILE_EXISTS:
return nullptr;
default:
KJ_FAIL_WIN32("CreateFileW", error);
}
return newDiskFile(AutoCloseHandle(handle));
});
}
static Array<wchar_t> join16(ArrayPtr<const wchar_t> path, const wchar_t* file) {
// Assumes `path` ends with a NUL terminator (and `file` is of course NUL terminated as well).
size_t len = wcslen(file) + 1;
auto result = kj::heapArray<wchar_t>(path.size() + len);
memcpy(result.begin(), path.begin(), path.asBytes().size() - sizeof(wchar_t));
result[path.size() - 1] = '\\';
memcpy(result.begin() + path.size(), file, len * sizeof(wchar_t));
return result;
}
class TempDir {
public:
TempDir(): filename(newTemp([](Array<wchar_t> candidatePath) -> Maybe<Array<wchar_t>> {
KJ_WIN32_HANDLE_ERRORS(CreateDirectoryW(candidatePath.begin(), NULL)) {
case ERROR_ALREADY_EXISTS:
case ERROR_FILE_EXISTS:
return nullptr;
default:
KJ_FAIL_WIN32("CreateDirectoryW", error);
}
return kj::mv(candidatePath);
})) {}
Own<Directory> get() {
HANDLE handle;
KJ_WIN32(handle = CreateFileW(
filename.begin(),
GENERIC_READ,
FILE_SHARE_READ | FILE_SHARE_WRITE,
NULL,
OPEN_EXISTING,
FILE_FLAG_BACKUP_SEMANTICS, // apparently, this flag is required for directories
NULL));
return newDiskDirectory(AutoCloseHandle(handle));
}
~TempDir() noexcept(false) {
recursiveDelete(filename);
}
private:
Array<wchar_t> filename;
static void recursiveDelete(ArrayPtr<const wchar_t> path) {
// Recursively delete the temp dir, verifying that no .kj-tmp. files were left over.
//
// Mostly copied from rmrfChildren() in filesystem-win32.c++.
auto glob = join16(path, L"\\*");
WIN32_FIND_DATAW data;
HANDLE handle = FindFirstFileW(glob.begin(), &data);
if (handle == INVALID_HANDLE_VALUE) {
auto error = GetLastError();
if (error == ERROR_FILE_NOT_FOUND) return;
KJ_FAIL_WIN32("FindFirstFile", error, path) { return; }
}
KJ_DEFER(KJ_WIN32(FindClose(handle)) { break; });
do {
// Ignore "." and "..", ugh.
if (data.cFileName[0] == L'.') {
if (data.cFileName[1] == L'\0' ||
(data.cFileName[1] == L'.' && data.cFileName[2] == L'\0')) {
continue;
}
}
String utf8Name = decodeWideString(arrayPtr(data.cFileName, wcslen(data.cFileName)));
KJ_EXPECT(!utf8Name.startsWith(".kj-tmp."), "temp file not cleaned up", utf8Name);
auto child = join16(path, data.cFileName);
if ((data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) &&
!(data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) {
recursiveDelete(child);
} else {
KJ_WIN32(DeleteFileW(child.begin()));
}
} while (FindNextFileW(handle, &data));
auto error = GetLastError();
if (error != ERROR_NO_MORE_FILES) {
KJ_FAIL_WIN32("FindNextFile", error, path) { return; }
}
uint retryCount = 0;
retry:
KJ_WIN32_HANDLE_ERRORS(RemoveDirectoryW(path.begin())) {
case ERROR_DIR_NOT_EMPTY:
if (retryCount++ < 10) {
Sleep(10);
goto retry;
}
// fallthrough
default:
KJ_FAIL_WIN32("RemoveDirectory", error) { break; }
}
}
};
#else
bool isWine() { return false; }
#if __APPLE__
#define HOLES_NOT_SUPPORTED 1
#endif
static Own<File> newTempFile() {
char filename[] = "/var/tmp/kj-filesystem-test.XXXXXX";
int fd;
......@@ -95,6 +263,8 @@ private:
}
};
#endif // _WIN32, else
KJ_TEST("DiskFile") {
auto file = newTempFile();
......@@ -154,6 +324,7 @@ KJ_TEST("DiskFile") {
file->write(12, StringPtr("corge").asBytes());
KJ_EXPECT(kj::str(mapping.slice(12, 17).asChars()) == "corge");
#if !_WIN32 // Windows doesn't allow the file size to change while mapped.
// Can shrink.
file->truncate(6);
KJ_EXPECT(kj::str(mapping.slice(12, 17).asChars()) == kj::StringPtr("\0\0\0\0\0", 5));
......@@ -164,6 +335,7 @@ KJ_TEST("DiskFile") {
// Can even regrow past previous capacity.
file->truncate(100);
#endif
}
file->truncate(6);
......@@ -332,6 +504,11 @@ KJ_TEST("DiskDirectory") {
KJ_EXPECT(dir->exists(Path("bar")));
KJ_EXPECT(!dir->tryRemove(Path({"bar", "baz"})));
#if _WIN32
// On Windows, we can't delete a directory while we still have it open.
subdir = nullptr;
#endif
KJ_EXPECT(dir->exists(Path("corge")));
KJ_EXPECT(dir->exists(Path({"corge", "grault"})));
dir->remove(Path("corge"));
......@@ -340,6 +517,7 @@ KJ_TEST("DiskDirectory") {
KJ_EXPECT(!dir->tryRemove(Path("corge")));
}
#if !_WIN32 // Creating symlinks on Win32 requires admin privileges prior to Windows 10.
KJ_TEST("DiskDirectory symlinks") {
TempDir tempDir;
auto dir = tempDir.get();
......@@ -394,6 +572,7 @@ KJ_TEST("DiskDirectory symlinks") {
KJ_EXPECT(!dir->exists(Path("foo")));
KJ_EXPECT(dir->tryOpenFile(Path("foo")) == nullptr);
}
#endif
KJ_TEST("DiskDirectory link") {
TempDir tempDirSrc;
......@@ -447,6 +626,43 @@ KJ_TEST("DiskDirectory copy") {
KJ_EXPECT(dst->openFile(Path({"link", "bar"}))->readAllText() == "foobar");
}
KJ_TEST("DiskDirectory copy-replace") {
TempDir tempDirSrc;
TempDir tempDirDst;
auto src = tempDirSrc.get();
auto dst = tempDirDst.get();
src->openFile(Path({"foo", "bar"}), WriteMode::CREATE | WriteMode::CREATE_PARENT)
->writeAll("foobar");
src->openFile(Path({"foo", "baz", "qux"}), WriteMode::CREATE | WriteMode::CREATE_PARENT)
->writeAll("bazqux");
dst->openFile(Path({"link", "corge"}), WriteMode::CREATE | WriteMode::CREATE_PARENT)
->writeAll("abcd");
// CREATE fails.
KJ_EXPECT(!dst->tryTransfer(Path("link"), WriteMode::CREATE,
*src, Path("foo"), TransferMode::COPY));
// Verify nothing changed.
KJ_EXPECT(dst->openFile(Path({"link", "corge"}))->readAllText() == "abcd");
KJ_EXPECT(!dst->exists(Path({"foo", "bar"})));
// Now try MODIFY.
dst->transfer(Path("link"), WriteMode::MODIFY, *src, Path("foo"), TransferMode::COPY);
KJ_EXPECT(src->openFile(Path({"foo", "bar"}))->readAllText() == "foobar");
KJ_EXPECT(src->openFile(Path({"foo", "baz", "qux"}))->readAllText() == "bazqux");
KJ_EXPECT(dst->openFile(Path({"link", "bar"}))->readAllText() == "foobar");
KJ_EXPECT(dst->openFile(Path({"link", "baz", "qux"}))->readAllText() == "bazqux");
KJ_EXPECT(!dst->exists(Path({"link", "corge"})));
KJ_EXPECT(dst->exists(Path({"link", "bar"})));
src->remove(Path({"foo", "bar"}));
KJ_EXPECT(dst->openFile(Path({"link", "bar"}))->readAllText() == "foobar");
}
KJ_TEST("DiskDirectory move") {
TempDir tempDirSrc;
TempDir tempDirDst;
......@@ -466,6 +682,38 @@ KJ_TEST("DiskDirectory move") {
KJ_EXPECT(dst->openFile(Path({"link", "baz", "qux"}))->readAllText() == "bazqux");
}
KJ_TEST("DiskDirectory move-replace") {
TempDir tempDirSrc;
TempDir tempDirDst;
auto src = tempDirSrc.get();
auto dst = tempDirDst.get();
src->openFile(Path({"foo", "bar"}), WriteMode::CREATE | WriteMode::CREATE_PARENT)
->writeAll("foobar");
src->openFile(Path({"foo", "baz", "qux"}), WriteMode::CREATE | WriteMode::CREATE_PARENT)
->writeAll("bazqux");
dst->openFile(Path({"link", "corge"}), WriteMode::CREATE | WriteMode::CREATE_PARENT)
->writeAll("abcd");
// CREATE fails.
KJ_EXPECT(!dst->tryTransfer(Path("link"), WriteMode::CREATE,
*src, Path("foo"), TransferMode::MOVE));
// Verify nothing changed.
KJ_EXPECT(dst->openFile(Path({"link", "corge"}))->readAllText() == "abcd");
KJ_EXPECT(!dst->exists(Path({"foo", "bar"})));
KJ_EXPECT(src->exists(Path({"foo"})));
// Now try MODIFY.
dst->transfer(Path("link"), WriteMode::MODIFY, *src, Path("foo"), TransferMode::MOVE);
KJ_EXPECT(!src->exists(Path({"foo"})));
KJ_EXPECT(dst->openFile(Path({"link", "bar"}))->readAllText() == "foobar");
KJ_EXPECT(dst->openFile(Path({"link", "baz", "qux"}))->readAllText() == "bazqux");
}
KJ_TEST("DiskDirectory createTemporary") {
TempDir tempDir;
auto dir = tempDir.get();
......@@ -475,6 +723,46 @@ KJ_TEST("DiskDirectory createTemporary") {
KJ_EXPECT(dir->listNames() == nullptr);
}
KJ_TEST("DiskDirectory replaceSubdir()") {
TempDir tempDir;
auto dir = tempDir.get();
{
auto replacer = dir->replaceSubdir(Path("foo"), WriteMode::CREATE);
replacer->get().openFile(Path("bar"), WriteMode::CREATE)->writeAll("original");
KJ_EXPECT(replacer->get().openFile(Path("bar"))->readAllText() == "original");
KJ_EXPECT(!dir->exists(Path({"foo", "bar"})));
replacer->commit();
KJ_EXPECT(replacer->get().openFile(Path("bar"))->readAllText() == "original");
KJ_EXPECT(dir->openFile(Path({"foo", "bar"}))->readAllText() == "original");
}
{
// CREATE fails -- already exists.
auto replacer = dir->replaceSubdir(Path("foo"), WriteMode::CREATE);
replacer->get().openFile(Path("corge"), WriteMode::CREATE)->writeAll("bazqux");
KJ_EXPECT(dir->listNames().size() == 1 && dir->listNames()[0] == "foo");
KJ_EXPECT(!replacer->tryCommit());
}
// Unchanged.
KJ_EXPECT(dir->openFile(Path({"foo", "bar"}))->readAllText() == "original");
KJ_EXPECT(!dir->exists(Path({"foo", "corge"})));
{
// MODIFY succeeds.
auto replacer = dir->replaceSubdir(Path("foo"), WriteMode::MODIFY);
replacer->get().openFile(Path("corge"), WriteMode::CREATE)->writeAll("bazqux");
KJ_EXPECT(dir->listNames().size() == 1 && dir->listNames()[0] == "foo");
replacer->commit();
}
// Replaced with new contents.
KJ_EXPECT(!dir->exists(Path({"foo", "bar"})));
KJ_EXPECT(dir->openFile(Path({"foo", "corge"}))->readAllText() == "bazqux");
}
KJ_TEST("DiskDirectory replace directory with file") {
TempDir tempDir;
auto dir = tempDir.get();
......@@ -535,19 +823,41 @@ KJ_TEST("DiskDirectory replace file with directory") {
#ifndef HOLES_NOT_SUPPORTED
KJ_TEST("DiskFile holes") {
if (isWine()) {
// WINE doesn't support sparse files.
return;
}
TempDir tempDir;
auto dir = tempDir.get();
auto file = dir->openFile(Path("holes"), WriteMode::CREATE);
#if _WIN32
FILE_SET_SPARSE_BUFFER sparseInfo;
memset(&sparseInfo, 0, sizeof(sparseInfo));
sparseInfo.SetSparse = TRUE;
DWORD dummy;
KJ_WIN32(DeviceIoControl(
KJ_ASSERT_NONNULL(file->getWin32Handle()),
FSCTL_SET_SPARSE, &sparseInfo, sizeof(sparseInfo),
NULL, 0, &dummy, NULL));
#endif
file->writeAll("foobar");
file->write(1 << 20, StringPtr("foobar").asBytes());
// Some filesystems, like BTRFS, report zero `spaceUsed` until synced.
file->datasync();
// Allow for block sizes as low as 512 bytes and as high as 64k.
auto meta = file->stat();
KJ_EXPECT(meta.spaceUsed >= 2 * 512);
KJ_EXPECT(meta.spaceUsed >= 2 * 512, meta.spaceUsed);
KJ_EXPECT(meta.spaceUsed <= 2 * 65536);
byte buf[7];
#if !_WIN32 // Win32 CopyFile() does NOT preserve sparseness.
{
// Copy doesn't fill in holes.
dir->transfer(Path("copy"), WriteMode::CREATE, Path("holes"), TransferMode::COPY);
......@@ -562,12 +872,15 @@ KJ_TEST("DiskFile holes") {
KJ_EXPECT(copy->read(1 << 19, buf) == 7);
KJ_EXPECT(StringPtr(reinterpret_cast<char*>(buf), 6) == StringPtr("\0\0\0\0\0\0", 6));
}
#endif
file->truncate(1 << 21);
file->datasync();
KJ_EXPECT(file->stat().spaceUsed == meta.spaceUsed);
KJ_EXPECT(file->read(1 << 20, buf) == 7);
KJ_EXPECT(StringPtr(reinterpret_cast<char*>(buf), 6) == "foobar");
#if !_WIN32 // Win32 CopyFile() does NOT preserve sparseness.
{
dir->transfer(Path("copy"), WriteMode::MODIFY, Path("holes"), TransferMode::COPY);
auto copy = dir->openFile(Path("copy"));
......@@ -581,6 +894,18 @@ KJ_TEST("DiskFile holes") {
KJ_EXPECT(copy->read(1 << 19, buf) == 7);
KJ_EXPECT(StringPtr(reinterpret_cast<char*>(buf), 6) == StringPtr("\0\0\0\0\0\0", 6));
}
#endif
// Try punching a hole with zero().
file->zero(1 << 20, 4096);
file->datasync();
#if !_WIN32
// TODO(someday): This doesn't work on Windows. I don't know why. We're definitely using the
// proper ioctl. Oh well.
KJ_EXPECT(file->stat().spaceUsed < meta.spaceUsed);
#endif
KJ_EXPECT(file->read(1 << 20, buf) == 7);
KJ_EXPECT(StringPtr(reinterpret_cast<char*>(buf), 6) == StringPtr("\0\0\0\0\0\0", 6));
}
#endif
......
c++/src/kj/filesystem-disk.c++ → c++/src/kj/filesystem-disk-unix.c++
View file @ ac10f36b
......@@ -19,6 +19,8 @@
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
#if !_WIN32
#include "filesystem.h"
#include "debug.h"
#include <sys/types.h>
......@@ -30,13 +32,13 @@
#include <sys/mman.h>
#include <errno.h>
#include <dirent.h>
#include <syscall.h>
#include <stdlib.h>
#include "vector.h"
#include "miniposix.h"
#include <algorithm>
#if __linux__
#include <syscall.h>
#include <linux/fs.h>
#include <sys/sendfile.h>
#endif
......@@ -61,6 +63,11 @@ namespace {
#define MAYBE_O_DIRECTORY 0
#endif
#if __APPLE__
// Mac OSX defines SEEK_HOLE, but it doesn't work. ("Inappropriate ioctl for device", it says.)
#undef SEEK_HOLE
#endif
static void setCloexec(int fd) KJ_UNUSED;
static void setCloexec(int fd) {
// Set the O_CLOEXEC flag on the given fd.
......@@ -117,7 +124,11 @@ static FsNode::Metadata statToMetadata(struct stat& stats) {
modeToType(stats.st_mode),
implicitCast<uint64_t>(stats.st_size),
implicitCast<uint64_t>(stats.st_blocks * 512u),
#if __APPLE__
toKjDate(stats.st_mtimespec),
#else
toKjDate(stats.st_mtim),
#endif
implicitCast<uint>(stats.st_nlink)
};
}
......@@ -223,12 +234,7 @@ protected:
size_t capacity, void (*destroyElement)(void*)) const {
auto range = getMmapRange(reinterpret_cast<uintptr_t>(firstElement),
elementSize * elementCount);
#if _WIN32
KJ_ASSERT(UnmapViewOfFile(reinterpret_cast<byte*>(range.offset))) { break; }
#else
KJ_SYSCALL(munmap(reinterpret_cast<byte*>(range.offset), range.size)) { break; }
#endif
}
};
......@@ -283,8 +289,26 @@ public:
return statToMetadata(stats);
}
void sync() { KJ_SYSCALL(fsync(fd)); }
void datasync() { KJ_SYSCALL(fdatasync(fd)); }
void sync() {
#if __APPLE__
// For whatever reason, fsync() on OSX only flushes kernel buffers. It does not flush hardware
// disk buffers. This makes it not very useful. But OSX documents fcntl F_FULLFSYNC which does
// the right thing. Why they don't just make fsync() do the right thing, I do not know.
KJ_SYSCALL(fcntl(fd, F_FULLFSYNC));
#else
KJ_SYSCALL(fsync(fd));
#endif
}
void datasync() {
// The presence of the _POSIX_SYNCHRONIZED_IO define is supposed to tell us that fdatasync()
// exists. But Apple defines this yet doesn't offer fdatasync(). Thanks, Apple.
#if _POSIX_SYNCHRONIZED_IO && !__APPLE__
KJ_SYSCALL(fdatasync(fd));
#else
this->sync();
#endif
}
// ReadableFile --------------------------------------------------------------
......@@ -353,26 +377,34 @@ public:
}
#endif
// Use a 4k buffer of zeros amplified by iov to write zeros with as few syscalls as possible.
byte buffer[4096];
memset(buffer, 0, sizeof(buffer));
static const byte ZEROS[4096] = { 0 };
size_t count = (size + sizeof(buffer) - 1) / sizeof(buffer);
#if __APPLE__
// Mac doesn't have pwritev().
while (size > sizeof(ZEROS)) {
write(offset, ZEROS);
size -= sizeof(ZEROS);
offset += sizeof(ZEROS);
}
write(offset, kj::arrayPtr(ZEROS, size));
#else
// Use a 4k buffer of zeros amplified by iov to write zeros with as few syscalls as possible.
size_t count = (size + sizeof(ZEROS) - 1) / sizeof(ZEROS);
const size_t iovmax = miniposix::iovMax(count);
KJ_STACK_ARRAY(struct iovec, iov, kj::min(iovmax, count), 16, 256);
for (auto& item: iov) {
item.iov_base = buffer;
item.iov_len = sizeof(buffer);
item.iov_base = const_cast<byte*>(ZEROS);
item.iov_len = sizeof(ZEROS);
}
while (size > 0) {
size_t iovCount;
if (size >= iov.size() * sizeof(buffer)) {
if (size >= iov.size() * sizeof(ZEROS)) {
iovCount = iov.size();
} else {
iovCount = size / sizeof(buffer);
size_t rem = size % sizeof(buffer);
iovCount = size / sizeof(ZEROS);
size_t rem = size % sizeof(ZEROS);
if (rem > 0) {
iov[iovCount++].iov_len = rem;
}
......@@ -385,13 +417,14 @@ public:
offset += n;
size -= n;
}
#endif
}
void truncate(uint64_t size) {
KJ_SYSCALL(ftruncate(fd, size));
}
class WritableFileMappingImpl: public WritableFileMapping {
class WritableFileMappingImpl final: public WritableFileMapping {
public:
WritableFileMappingImpl(Array<byte> bytes): bytes(kj::mv(bytes)) {}
......@@ -1016,7 +1049,7 @@ public:
}
}
#ifdef RENAME_EXCHANGE
#if __linux__ && defined(RENAME_EXCHANGE)
// Try to use Linux's renameat2() to atomically check preconditions and apply.
if (has(mode, WriteMode::MODIFY)) {
......@@ -1096,7 +1129,15 @@ public:
if (S_ISDIR(stats.st_mode)) {
return mkdirat(fd, candidatePath.cStr(), 0700);
} else {
#if __APPLE__
// No mknodat() on OSX, gotta open() a file, ugh.
int newFd = openat(fd, candidatePath.cStr(),
O_RDWR | O_CREAT | O_EXCL | MAYBE_O_CLOEXEC, 0700);
if (newFd >= 0) close(newFd);
return newFd;
#else
return mknodat(fd, candidatePath.cStr(), S_IFREG | 0600, dev_t());
#endif
}
})) {
away = kj::mv(*awayPath);
......@@ -1172,7 +1213,7 @@ public:
}
template <typename T>
class ReplacerImpl: public Directory::Replacer<T> {
class ReplacerImpl final: public Directory::Replacer<T> {
public:
ReplacerImpl(Own<T>&& object, DiskHandle& handle,
String&& tempPath, String&& path, WriteMode mode)
......@@ -1205,7 +1246,7 @@ public:
};
template <typename T>
class BrokenReplacer: public Directory::Replacer<T> {
class BrokenReplacer final: public Directory::Replacer<T> {
// For recovery path when exceptions are disabled.
public:
......@@ -1254,7 +1295,7 @@ public:
Own<File> createTemporary() {
int newFd_;
#ifdef O_TMPFILE
#if __linux__ && defined(O_TMPFILE)
// Use syscall() to work around glibc bug with O_TMPFILE:
// https://sourceware.org/bugzilla/show_bug.cgi?id=17523
KJ_SYSCALL_HANDLE_ERRORS(newFd_ = syscall(
......@@ -1655,3 +1696,5 @@ Own<Filesystem> newDiskFilesystem() {
}
} // namespace kj
#endif // !_WIN32
c++/src/kj/filesystem-disk-win32.c++ 0 → 100644
View file @ ac10f36b
// Copyright (c) 2015 Sandstorm Development Group, Inc. and contributors
// Licensed under the MIT License:
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
#if _WIN32
// For Unix implementation, see filesystem-disk-unix.c++.
#include "filesystem.h"
#include "debug.h"
#include "encoding.h"
#include "vector.h"
#include <algorithm>
#include <wchar.h>
// Request Vista-level APIs.
#define WINVER 0x0600
#define _WIN32_WINNT 0x0600
#define WIN32_LEAN_AND_MEAN // ::eyeroll::
#include <windows.h>
#include <winioctl.h>
#include "windows-sanity.h"
namespace kj {
static Own<ReadableDirectory> newDiskReadableDirectory(AutoCloseHandle fd, Path&& path);
static Own<Directory> newDiskDirectory(AutoCloseHandle fd, Path&& path);
static AutoCloseHandle* getHandlePointerHack(File& file) { return nullptr; }
static AutoCloseHandle* getHandlePointerHack(Directory& dir);
static Path* getPathPointerHack(File& file) { return nullptr; }
static Path* getPathPointerHack(Directory& dir);
namespace {
struct REPARSE_DATA_BUFFER {
// From ntifs.h, which is part of the driver development kit so not necessarily available I
// guess.
ULONG ReparseTag;
USHORT ReparseDataLength;
USHORT Reserved;
union {
struct {
USHORT SubstituteNameOffset;
USHORT SubstituteNameLength;
USHORT PrintNameOffset;
USHORT PrintNameLength;
ULONG Flags;
WCHAR PathBuffer[1];
} SymbolicLinkReparseBuffer;
struct {
USHORT SubstituteNameOffset;
USHORT SubstituteNameLength;
USHORT PrintNameOffset;
USHORT PrintNameLength;
WCHAR PathBuffer[1];
} MountPointReparseBuffer;
struct {
UCHAR DataBuffer[1];
} GenericReparseBuffer;
};
};
#define HIDDEN_PREFIX ".kj-tmp."
// Prefix for temp files which should be hidden when listing a directory.
//
// If you change this, make sure to update the unit test.
static constexpr int64_t WIN32_EPOCH_OFFSET = 116444736000000000ull;
// Number of 100ns intervals from Jan 1, 1601 to Jan 1, 1970.
static Date toKjDate(FILETIME t) {
int64_t value = (static_cast<uint64_t>(t.dwHighDateTime) << 32) | t.dwLowDateTime;
return (value - WIN32_EPOCH_OFFSET) * (100 * kj::NANOSECONDS) + UNIX_EPOCH;
}
static FsNode::Type modeToType(DWORD attrs, DWORD reparseTag) {
if ((attrs & FILE_ATTRIBUTE_REPARSE_POINT) &&
reparseTag == IO_REPARSE_TAG_SYMLINK) {
return FsNode::Type::SYMLINK;
}
if (attrs & FILE_ATTRIBUTE_DIRECTORY) return FsNode::Type::DIRECTORY;
return FsNode::Type::FILE;
}
static FsNode::Metadata statToMetadata(const BY_HANDLE_FILE_INFORMATION& stats) {
uint64_t size = (implicitCast<uint64_t>(stats.nFileSizeHigh) << 32) | stats.nFileSizeLow;
return FsNode::Metadata {
modeToType(stats.dwFileAttributes, 0),
size,
// In theory, spaceUsed should be based on GetCompressedFileSize(), but requiring an extra
// syscall for something rarely used would be sad.
size,
toKjDate(stats.ftLastWriteTime),
stats.nNumberOfLinks
};
}
static FsNode::Metadata statToMetadata(const WIN32_FIND_DATAW& stats) {
uint64_t size = (implicitCast<uint64_t>(stats.nFileSizeHigh) << 32) | stats.nFileSizeLow;
return FsNode::Metadata {
modeToType(stats.dwFileAttributes, stats.dwReserved0),
size,
// In theory, spaceUsed should be based on GetCompressedFileSize(), but requiring an extra
// syscall for something rarely used would be sad.
size,
toKjDate(stats.ftLastWriteTime),
// We can't get the number of links without opening the file, apparently. Meh.
1
};
}
static Array<wchar_t> join16(ArrayPtr<const wchar_t> path, const wchar_t* file) {
// Assumes `path` ends with a NUL terminator (and `file` is of course NUL terminated as well).
size_t len = wcslen(file) + 1;
auto result = kj::heapArray<wchar_t>(path.size() + len);
memcpy(result.begin(), path.begin(), path.asBytes().size() - sizeof(wchar_t));
result[path.size() - 1] = '\\';
memcpy(result.begin() + path.size(), file, len * sizeof(wchar_t));
return result;
}
static String dbgStr(ArrayPtr<const wchar_t> wstr) {
if (wstr.size() > 0 && wstr[wstr.size() - 1] == L'\0') {
wstr = wstr.slice(0, wstr.size() - 1);
}
return decodeWideString(wstr);
}
static void rmrfChildren(ArrayPtr<const wchar_t> path) {
auto glob = join16(path, L"*");
WIN32_FIND_DATAW data;
HANDLE handle = FindFirstFileW(glob.begin(), &data);
if (handle == INVALID_HANDLE_VALUE) {
auto error = GetLastError();
if (error == ERROR_FILE_NOT_FOUND) return;
KJ_FAIL_WIN32("FindFirstFile", error, dbgStr(glob)) { return; }
}
KJ_DEFER(KJ_WIN32(FindClose(handle)) { break; });
do {
// Ignore "." and "..", ugh.
if (data.cFileName[0] == L'.') {
if (data.cFileName[1] == L'\0' ||
(data.cFileName[1] == L'.' && data.cFileName[2] == L'\0')) {
continue;
}
}
auto child = join16(path, data.cFileName);
// For rmrf purposes, we assume any "reparse points" are symlink-like, even if they aren't
// actually the "symbolic link" reparse type, because we don't want to recursively delete any
// shared content.
if ((data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) &&
!(data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) {
rmrfChildren(child);
uint retryCount = 0;
retry:
KJ_WIN32_HANDLE_ERRORS(RemoveDirectoryW(child.begin())) {
case ERROR_DIR_NOT_EMPTY:
// On Windows, deleting a file actually only schedules it for deletion. Under heavy
// load it may take a bit for the deletion to go through. Or, if another process has
// the file open, it may not be deleted until that process closes it.
//
// We'll repeatedly retry for up to 100ms, then give up. This is awful but there's no
// way to tell for sure if the system is just being slow or if someone has the file
// open.
if (retryCount++ < 10) {
Sleep(10);
goto retry;
}
// fallthrough
default:
KJ_FAIL_WIN32("RemoveDirectory", error, dbgStr(child)) { break; }
}
} else {
KJ_WIN32(DeleteFileW(child.begin()));
}
} while (FindNextFileW(handle, &data));
auto error = GetLastError();
if (error != ERROR_NO_MORE_FILES) {
KJ_FAIL_WIN32("FindNextFile", error, dbgStr(path)) { return; }
}
}
static bool rmrf(ArrayPtr<const wchar_t> path) {
// Figure out whether this is a file or a directory.
//
// We use FindFirstFileW() because in the case of symlinks it will return info about the
// symlink rather than info about the target.
WIN32_FIND_DATAW data;
HANDLE handle = FindFirstFileW(path.begin(), &data);
if (handle == INVALID_HANDLE_VALUE) {
auto error = GetLastError();
if (error == ERROR_FILE_NOT_FOUND) return false;
KJ_FAIL_WIN32("FindFirstFile", error, dbgStr(path));
}
KJ_WIN32(FindClose(handle));
// For remove purposes, we assume any "reparse points" are symlink-like, even if they aren't
// actually the "symbolic link" reparse type, because we don't want to recursively delete any
// shared content.
if ((data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) &&
!(data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) {
// directory
rmrfChildren(path);
KJ_WIN32(RemoveDirectoryW(path.begin()), dbgStr(path));
} else {
KJ_WIN32(DeleteFileW(path.begin()), dbgStr(path));
}
return true;
}
static Path getPathFromHandle(HANDLE handle) {
DWORD tryLen = MAX_PATH;
for (;;) {
auto temp = kj::heapArray<wchar_t>(tryLen + 1);
DWORD len = GetFinalPathNameByHandleW(handle, temp.begin(), tryLen, 0);
if (len == 0) {
KJ_FAIL_WIN32("GetFinalPathNameByHandleW", GetLastError());
}
if (len < temp.size()) {
return Path::parseWin32Api(temp.slice(0, len));
}
// Try again with new length.
tryLen = len;
}
}
struct MmapRange {
uint64_t offset;
uint64_t size;
};
static size_t getAllocationGranularity() {
SYSTEM_INFO info;
GetSystemInfo(&info);
return info.dwAllocationGranularity;
};
static MmapRange getMmapRange(uint64_t offset, uint64_t size) {
// Rounds the given offset down to the nearest page boundary, and adjusts the size up to match.
// (This is somewhat different from Unix: we do NOT round the size up to an even multiple of
// pages.)
static const uint64_t pageSize = getAllocationGranularity();
uint64_t pageMask = pageSize - 1;
uint64_t realOffset = offset & ~pageMask;
uint64_t end = offset + size;
return { realOffset, end - realOffset };
}
class MmapDisposer: public ArrayDisposer {
protected:
void disposeImpl(void* firstElement, size_t elementSize, size_t elementCount,
size_t capacity, void (*destroyElement)(void*)) const {
auto range = getMmapRange(reinterpret_cast<uintptr_t>(firstElement),
elementSize * elementCount);
void* mapping = reinterpret_cast<void*>(range.offset);
if (mapping != nullptr) {
KJ_ASSERT(UnmapViewOfFile(mapping)) { break; }
}
}
};
#if _MSC_VER < 1910
// TODO(msvc): MSVC 2015 can't initialize a constexpr's vtable correctly.
const MmapDisposer mmapDisposer = MmapDisposer();
#else
constexpr MmapDisposer mmapDisposer = MmapDisposer();
#endif
void* win32Mmap(HANDLE handle, MmapRange range, DWORD pageProtect, DWORD access) {
HANDLE mappingHandle;
mappingHandle = CreateFileMappingW(handle, NULL, pageProtect, 0, 0, NULL);
if (mappingHandle == INVALID_HANDLE_VALUE) {
auto error = GetLastError();
if (error == ERROR_FILE_INVALID && range.size == 0) {
// The documentation says that CreateFileMapping will fail with ERROR_FILE_INVALID if the
// file size is zero. Ugh.
return nullptr;
}
KJ_FAIL_WIN32("CreateFileMapping", error);
}
KJ_DEFER(KJ_WIN32(CloseHandle(mappingHandle)) { break; });
void* mapping = MapViewOfFile(mappingHandle, access,
static_cast<DWORD>(range.offset >> 32), static_cast<DWORD>(range.offset), range.size);
if (mapping == nullptr) {
KJ_FAIL_WIN32("MapViewOfFile", GetLastError());
}
// It's unclear from the documentation whether mappings will always start at a multiple of the
// allocation granularity, but we depend on that later, so check it...
KJ_ASSERT(getMmapRange(reinterpret_cast<uintptr_t>(mapping), 0).size == 0);
return mapping;
}
class DiskHandle {
// We need to implement each of ReadableFile, AppendableFile, File, ReadableDirectory, and
// Directory for disk handles. There is a lot of implementation overlap between these, especially
// stat(), sync(), etc. We can't have everything inherit from a common DiskFsNode that implements
// these because then we get diamond inheritance which means we need to make all our inheritance
// virtual which means downcasting requires RTTI which violates our goal of supporting compiling
// with no RTTI. So instead we have the DiskHandle class which implements all the methods without
// inheriting anything, and then we have DiskFile, DiskDirectory, etc. hold this and delegate to
// it. Ugly, but works.
public:
DiskHandle(AutoCloseHandle&& handle, Maybe<Path> dirPath)
: handle(kj::mv(handle)), dirPath(kj::mv(dirPath)) {}
AutoCloseHandle handle;
kj::Maybe<Path> dirPath; // needed for directories, empty for files
Array<wchar_t> nativePath(PathPtr path) {
return KJ_ASSERT_NONNULL(dirPath).append(path).forWin32Api(true);
}
// OsHandle ------------------------------------------------------------------
AutoCloseHandle clone() {
HANDLE newHandle;
KJ_WIN32(DuplicateHandle(GetCurrentProcess(), handle, GetCurrentProcess(), &newHandle,
0, FALSE, DUPLICATE_SAME_ACCESS));
return AutoCloseHandle(newHandle);
}
HANDLE getWin32Handle() {
return handle.get();
}
// FsNode --------------------------------------------------------------------
FsNode::Metadata stat() {
BY_HANDLE_FILE_INFORMATION stats;
KJ_WIN32(GetFileInformationByHandle(handle, &stats));
auto metadata = statToMetadata(stats);
// Get space usage, e.g. for sparse files. Apparently the correct way to do this is to query
// "compression".
FILE_COMPRESSION_INFO compInfo;
KJ_WIN32_HANDLE_ERRORS(GetFileInformationByHandleEx(
handle, FileCompressionInfo, &compInfo, sizeof(compInfo))) {
case ERROR_CALL_NOT_IMPLEMENTED:
// Probably WINE.
break;
default:
KJ_FAIL_WIN32("GetFileInformationByHandleEx(FileCompressionInfo)", error) { break; }
break;
} else {
metadata.spaceUsed = compInfo.CompressedFileSize.QuadPart;
}
return metadata;
}
void sync() { KJ_WIN32(FlushFileBuffers(handle)); }
void datasync() { KJ_WIN32(FlushFileBuffers(handle)); }
// ReadableFile --------------------------------------------------------------
size_t read(uint64_t offset, ArrayPtr<byte> buffer) {
// ReadFile() probably never returns short reads unless it hits EOF. Unfortunately, though,
// this is not documented, and it's unclear whether we can rely on it.
size_t total = 0;
while (buffer.size() > 0) {
// Apparently, the way to fake pread() on Windows is to provide an OVERLAPPED structure even
// though we're not doing overlapped I/O.
OVERLAPPED overlapped;
memset(&overlapped, 0, sizeof(overlapped));
overlapped.Offset = static_cast<DWORD>(offset);
overlapped.OffsetHigh = static_cast<DWORD>(offset >> 32);
DWORD n;
KJ_WIN32_HANDLE_ERRORS(ReadFile(handle, buffer.begin(), buffer.size(), &n, &overlapped)) {
case ERROR_HANDLE_EOF:
// The documentation claims this shouldn't happen for synchronous reads, but it seems
// to happen for me, at least under WINE.
n = 0;
break;
default:
KJ_FAIL_WIN32("ReadFile", offset, buffer.size()) { return total; }
}
if (n == 0) break;
total += n;
offset += n;
buffer = buffer.slice(n, buffer.size());
}
return total;
}
Array<const byte> mmap(uint64_t offset, uint64_t size) {
auto range = getMmapRange(offset, size);
const void* mapping = win32Mmap(handle, range, PAGE_READONLY, FILE_MAP_READ);
return Array<const byte>(reinterpret_cast<const byte*>(mapping) + (offset - range.offset),
size, mmapDisposer);
}
Array<byte> mmapPrivate(uint64_t offset, uint64_t size) {
auto range = getMmapRange(offset, size);
void* mapping = win32Mmap(handle, range, PAGE_READONLY, FILE_MAP_COPY);
return Array<byte>(reinterpret_cast<byte*>(mapping) + (offset - range.offset),
size, mmapDisposer);
}
// File ----------------------------------------------------------------------
void write(uint64_t offset, ArrayPtr<const byte> data) {
// WriteFile() probably never returns short writes unless there's no space left on disk.
// Unfortunately, though, this is not documented, and it's unclear whether we can rely on it.
while (data.size() > 0) {
// Apparently, the way to fake pwrite() on Windows is to provide an OVERLAPPED structure even
// though we're not doing overlapped I/O.
OVERLAPPED overlapped;
memset(&overlapped, 0, sizeof(overlapped));
overlapped.Offset = static_cast<DWORD>(offset);
overlapped.OffsetHigh = static_cast<DWORD>(offset >> 32);
DWORD n;
KJ_WIN32(WriteFile(handle, data.begin(), data.size(), &n, &overlapped));
KJ_ASSERT(n > 0, "WriteFile() returned zero?");
offset += n;
data = data.slice(n, data.size());
}
}
void zero(uint64_t offset, uint64_t size) {
FILE_ZERO_DATA_INFORMATION info;
memset(&info, 0, sizeof(info));
info.FileOffset.QuadPart = offset;
info.BeyondFinalZero.QuadPart = offset + size;
DWORD dummy;
KJ_WIN32_HANDLE_ERRORS(DeviceIoControl(handle, FSCTL_SET_ZERO_DATA, &info,
sizeof(info), NULL, 0, &dummy, NULL)) {
case ERROR_NOT_SUPPORTED: {
// Dang. Let's do it the hard way.
static const byte ZEROS[4096] = { 0 };
while (size > sizeof(ZEROS)) {
write(offset, ZEROS);
size -= sizeof(ZEROS);
offset += sizeof(ZEROS);
}
write(offset, kj::arrayPtr(ZEROS, size));
break;
}
default:
KJ_FAIL_WIN32("DeviceIoControl(FSCTL_SET_ZERO_DATA)", error);
break;
}
}
void truncate(uint64_t size) {
// SetEndOfFile() would require seeking the file. It looks like SetFileInformationByHandle()
// lets us avoid this!
FILE_END_OF_FILE_INFO info;
memset(&info, 0, sizeof(info));
info.EndOfFile.QuadPart = size;
KJ_WIN32_HANDLE_ERRORS(
SetFileInformationByHandle(handle, FileEndOfFileInfo, &info, sizeof(info))) {
case ERROR_CALL_NOT_IMPLEMENTED: {
// Wine doesn't implement this. :(
LONG currentHigh = 0;
LONG currentLow = SetFilePointer(handle, 0, &currentHigh, FILE_CURRENT);
if (currentLow == INVALID_SET_FILE_POINTER) {
KJ_FAIL_WIN32("SetFilePointer", GetLastError());
}
uint64_t current = (uint64_t(currentHigh) << 32) | uint64_t((ULONG)currentLow);
LONG endLow = size & 0x00000000ffffffffull;
LONG endHigh = size >> 32;
if (SetFilePointer(handle, endLow, &endHigh, FILE_BEGIN) == INVALID_SET_FILE_POINTER) {
KJ_FAIL_WIN32("SetFilePointer", GetLastError());
}
KJ_WIN32(SetEndOfFile(handle));
if (current < size) {
if (SetFilePointer(handle, currentLow, &currentHigh, FILE_BEGIN) ==
INVALID_SET_FILE_POINTER) {
KJ_FAIL_WIN32("SetFilePointer", GetLastError());
}
}
break;
}
default:
KJ_FAIL_WIN32("SetFileInformationByHandle", error);
}
}
class WritableFileMappingImpl final: public WritableFileMapping {
public:
WritableFileMappingImpl(Array<byte> bytes): bytes(kj::mv(bytes)) {}
ArrayPtr<byte> get() override {
return bytes;
}
void changed(ArrayPtr<byte> slice) override {
KJ_REQUIRE(slice.begin() >= bytes.begin() && slice.end() <= bytes.end(),
"byte range is not part of this mapping");
// Nothing needed here -- NT tracks dirty pages.
}
void sync(ArrayPtr<byte> slice) override {
KJ_REQUIRE(slice.begin() >= bytes.begin() && slice.end() <= bytes.end(),
"byte range is not part of this mapping");
// Zero is treated specially by FlushViewOfFile(), so check for it.
if (slice.size() > 0) {
KJ_WIN32(FlushViewOfFile(slice.begin(), slice.size()));
}
}
private:
Array<byte> bytes;
};
Own<WritableFileMapping> mmapWritable(uint64_t offset, uint64_t size) {
auto range = getMmapRange(offset, size);
void* mapping = win32Mmap(handle, range, PAGE_READWRITE, FILE_MAP_ALL_ACCESS);
auto array = Array<byte>(reinterpret_cast<byte*>(mapping) + (offset - range.offset),
size, mmapDisposer);
return heap<WritableFileMappingImpl>(kj::mv(array));
}
// copy() is not optimized on Windows.
// ReadableDirectory ---------------------------------------------------------
template <typename Func>
auto list(bool needTypes, Func&& func)
-> Array<Decay<decltype(func(instance<StringPtr>(), instance<FsNode::Type>()))>> {
PathPtr path = KJ_ASSERT_NONNULL(dirPath);
auto glob = join16(path.forWin32Api(true), L"*");
// TODO(perf): Use FindFileEx() with FindExInfoBasic? Not apparently supported on Vista.
// TODO(someday): Use NtQueryDirectoryObject() instead? It's "internal", but so much cleaner.
WIN32_FIND_DATAW data;
HANDLE handle = FindFirstFileW(glob.begin(), &data);
if (handle == INVALID_HANDLE_VALUE) {
auto error = GetLastError();
if (error == ERROR_FILE_NOT_FOUND) return nullptr;
KJ_FAIL_WIN32("FindFirstFile", error, dbgStr(glob));
}
KJ_DEFER(KJ_WIN32(FindClose(handle)) { break; });
typedef Decay<decltype(func(instance<StringPtr>(), instance<FsNode::Type>()))> Entry;
kj::Vector<Entry> entries;
do {
auto name = decodeUtf16(
arrayPtr(reinterpret_cast<char16_t*>(data.cFileName), wcslen(data.cFileName)));
if (name != "." && name != ".." && !name.startsWith(HIDDEN_PREFIX)) {
entries.add(func(name, modeToType(data.dwFileAttributes, data.dwReserved0)));
}
} while (FindNextFileW(handle, &data));
auto error = GetLastError();
if (error != ERROR_NO_MORE_FILES) {
KJ_FAIL_WIN32("FindNextFile", error, path);
}
auto result = entries.releaseAsArray();
std::sort(result.begin(), result.end());
return result;
}
Array<String> listNames() {
return list(false, [](StringPtr name, FsNode::Type type) { return heapString(name); });
}
Array<ReadableDirectory::Entry> listEntries() {
return list(true, [](StringPtr name, FsNode::Type type) {
return ReadableDirectory::Entry { type, heapString(name), };
});
}
bool exists(PathPtr path) {
DWORD result = GetFileAttributesW(nativePath(path).begin());
if (result == INVALID_FILE_ATTRIBUTES) {
auto error = GetLastError();
switch (error) {
case ERROR_FILE_NOT_FOUND:
case ERROR_PATH_NOT_FOUND:
return false;
default:
KJ_FAIL_WIN32("GetFileAttributesEx(path)", error, path) { return false; }
}
} else {
return true;
}
}
Maybe<FsNode::Metadata> tryLstat(PathPtr path) {
// We use FindFirstFileW() because in the case of symlinks it will return info about the
// symlink rather than info about the target.
WIN32_FIND_DATAW data;
HANDLE handle = FindFirstFileW(nativePath(path).begin(), &data);
if (handle == INVALID_HANDLE_VALUE) {
auto error = GetLastError();
if (error == ERROR_FILE_NOT_FOUND) return nullptr;
KJ_FAIL_WIN32("FindFirstFile", error, path);
} else {
KJ_WIN32(FindClose(handle));
return statToMetadata(data);
}
}
Maybe<Own<ReadableFile>> tryOpenFile(PathPtr path) {
HANDLE newHandle;
KJ_WIN32_HANDLE_ERRORS(newHandle = CreateFileW(
nativePath(path).begin(),
GENERIC_READ,
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
NULL,
OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL,
NULL)) {
case ERROR_FILE_NOT_FOUND:
case ERROR_PATH_NOT_FOUND:
return nullptr;
default:
KJ_FAIL_WIN32("CreateFile(path, OPEN_EXISTING)", error, path) { return nullptr; }
}
return newDiskReadableFile(kj::AutoCloseHandle(newHandle));
}
Maybe<AutoCloseHandle> tryOpenSubdirInternal(PathPtr path) {
HANDLE newHandle;
KJ_WIN32_HANDLE_ERRORS(newHandle = CreateFileW(
nativePath(path).begin(),
GENERIC_READ,
// When opening directories, we do NOT use FILE_SHARE_DELETE, because we need the directory
// path to remain vaild.
//
// TODO(someday): Use NtCreateFile() and related "internal" APIs that allow for
// openat()-like behavior?
FILE_SHARE_READ | FILE_SHARE_WRITE,
NULL,
OPEN_EXISTING,
FILE_FLAG_BACKUP_SEMANTICS, // apparently, this flag is required for directories
NULL)) {
case ERROR_FILE_NOT_FOUND:
case ERROR_PATH_NOT_FOUND:
return nullptr;
default:
KJ_FAIL_WIN32("CreateFile(directoryPath, OPEN_EXISTING)", error, path) { return nullptr; }
}
kj::AutoCloseHandle ownHandle(newHandle);
BY_HANDLE_FILE_INFORMATION info;
KJ_WIN32(GetFileInformationByHandle(ownHandle, &info));
KJ_REQUIRE(info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY, "not a directory", path);
return kj::mv(ownHandle);
}
Maybe<Own<ReadableDirectory>> tryOpenSubdir(PathPtr path) {
return tryOpenSubdirInternal(path).map([&](AutoCloseHandle&& handle) {
return newDiskReadableDirectory(kj::mv(handle), KJ_ASSERT_NONNULL(dirPath).append(path));
});
}
Maybe<String> tryReadlink(PathPtr path) {
// Windows symlinks work differently from Unix. Generally they are set up by the system
// administrator and apps are expected to treat them transparently. Hence, on Windows, we act
// as if nothing is a symlink by always returning null here.
// TODO(someday): If we want to treat Windows symlinks more like Unix ones, start by reverting
// the comment that added this comment.
return nullptr;
}
// Directory -----------------------------------------------------------------
static LPSECURITY_ATTRIBUTES makeSecAttr(WriteMode mode) {
if (has(mode, WriteMode::PRIVATE)) {
KJ_UNIMPLEMENTED("WriteMode::PRIVATE on Win32 is not implemented");
}
return nullptr;
}
bool tryMkdir(PathPtr path, WriteMode mode, bool noThrow) {
// Internal function to make a directory.
auto filename = nativePath(path);
KJ_WIN32_HANDLE_ERRORS(CreateDirectoryW(filename.begin(), makeSecAttr(mode))) {
case ERROR_ALREADY_EXISTS:
case ERROR_FILE_EXISTS: {
// Apparently this path exists.
if (!has(mode, WriteMode::MODIFY)) {
// Require exclusive create.
return false;
}
// MODIFY is allowed, so we just need to check whether the existing entry is a directory.
DWORD attr = GetFileAttributesW(filename.begin());
if (attr == INVALID_FILE_ATTRIBUTES) {
// CreateDirectory() says it already exists but we can't get attributes. Maybe it's a
// dangling link, or maybe we can't access it for some reason. Assume failure.
//
// TODO(someday): Maybe we should be creating the directory at the target of the
// link?
goto failed;
}
return attr & FILE_ATTRIBUTE_DIRECTORY;
}
case ERROR_PATH_NOT_FOUND:
if (has(mode, WriteMode::CREATE_PARENT) && path.size() > 0 &&
tryMkdir(path.parent(), WriteMode::CREATE | WriteMode::MODIFY |
WriteMode::CREATE_PARENT, true)) {
// Retry, but make sure we don't try to create the parent again.
return tryMkdir(path, mode - WriteMode::CREATE_PARENT, noThrow);
} else {
goto failed;
}
default:
failed:
if (noThrow) {
// Caller requested no throwing.
return false;
} else {
KJ_FAIL_WIN32("CreateDirectory", error, path);
}
}
return true;
}
kj::Maybe<Array<wchar_t>> createNamedTemporary(
PathPtr finalName, WriteMode mode, Path& kjTempPath,
Function<BOOL(const wchar_t*)> tryCreate) {
// Create a temporary file which will eventually replace `finalName`.
//
// Calls `tryCreate` to actually create the temporary, passing in the desired path. tryCreate()
// is expected to behave like a win32 call, returning a BOOL and setting `GetLastError()` on
// error. tryCreate() MUST fail with ERROR_{FILE,ALREADY}_EXISTS if the path exists -- this is
// not checked in advance, since it needs to be checked atomically. In the case of
// ERROR_*_EXISTS, tryCreate() will be called again with a new path.
//
// Returns the temporary path that succeeded. Only returns nullptr if there was an exception
// but we're compiled with -fno-exceptions.
//
// The optional parameter `kjTempPath` is filled in with the KJ Path of the temporary.
if (finalName.size() == 0) {
KJ_FAIL_REQUIRE("can't replace self") { break; }
return nullptr;
}
static uint counter = 0;
static const DWORD pid = GetCurrentProcessId();
auto tempName = kj::str(HIDDEN_PREFIX, pid, '.', counter++, '.',
finalName.basename()[0], ".partial");
kjTempPath = finalName.parent().append(tempName);
auto path = nativePath(kjTempPath);
KJ_WIN32_HANDLE_ERRORS(tryCreate(path.begin())) {
case ERROR_ALREADY_EXISTS:
case ERROR_FILE_EXISTS:
// Try again with a new counter value.
return createNamedTemporary(finalName, mode, kj::mv(tryCreate));
case ERROR_PATH_NOT_FOUND:
if (has(mode, WriteMode::CREATE_PARENT) && finalName.size() > 1 &&
tryMkdir(finalName.parent(), WriteMode::CREATE | WriteMode::MODIFY |
WriteMode::CREATE_PARENT, true)) {
// Retry, but make sure we don't try to create the parent again.
mode = mode - WriteMode::CREATE_PARENT;
return createNamedTemporary(finalName, mode, kj::mv(tryCreate));
}
// fallthrough
default:
KJ_FAIL_WIN32("create(path)", error, path) { break; }
return nullptr;
}
return kj::mv(path);
}
kj::Maybe<Array<wchar_t>> createNamedTemporary(
PathPtr finalName, WriteMode mode, Function<BOOL(const wchar_t*)> tryCreate) {
Path dummy = nullptr;
return createNamedTemporary(finalName, mode, dummy, kj::mv(tryCreate));
}
bool tryReplaceNode(PathPtr path, WriteMode mode, Function<BOOL(const wchar_t*)> tryCreate) {
// Replaces the given path with an object created by calling tryCreate().
//
// tryCreate() must behave like a win32 call which creates the node at the path passed to it,
// returning FALSE error. If the path passed to tryCreate already exists, it MUST fail with
// ERROR_{FILE,ALREADY}_EXISTS.
//
// When `mode` includes MODIFY, replaceNode() reacts to ERROR_*_EXISTS by creating the
// node in a temporary location and then rename()ing it into place.
if (path.size() == 0) {
KJ_FAIL_REQUIRE("can't replace self") { return false; }
}
auto filename = nativePath(path);
if (has(mode, WriteMode::CREATE)) {
// First try just cerating the node in-place.
KJ_WIN32_HANDLE_ERRORS(tryCreate(filename.begin())) {
case ERROR_ALREADY_EXISTS:
case ERROR_FILE_EXISTS:
// Target exists.
if (has(mode, WriteMode::MODIFY)) {
// Fall back to MODIFY path, below.
break;
} else {
return false;
}
case ERROR_PATH_NOT_FOUND:
if (has(mode, WriteMode::CREATE_PARENT) && path.size() > 0 &&
tryMkdir(path.parent(), WriteMode::CREATE | WriteMode::MODIFY |
WriteMode::CREATE_PARENT, true)) {
// Retry, but make sure we don't try to create the parent again.
return tryReplaceNode(path, mode - WriteMode::CREATE_PARENT, kj::mv(tryCreate));
}
default:
KJ_FAIL_WIN32("create(path)", error, path) { return false; }
} else {
// Success.
return true;
}
}
// Either we don't have CREATE mode or the target already exists. We need to perform a
// replacement instead.
KJ_IF_MAYBE(tempPath, createNamedTemporary(path, mode, kj::mv(tryCreate))) {
if (tryCommitReplacement(path, *tempPath, mode)) {
return true;
} else {
KJ_WIN32_HANDLE_ERRORS(DeleteFileW(tempPath->begin())) {
case ERROR_FILE_NOT_FOUND:
// meh
break;
default:
KJ_FAIL_WIN32("DeleteFile(tempPath)", error, dbgStr(*tempPath));
}
return false;
}
} else {
// threw, but exceptions are disabled
return false;
}
}
Maybe<AutoCloseHandle> tryOpenFileInternal(PathPtr path, WriteMode mode, bool append) {
DWORD disposition;
if (has(mode, WriteMode::MODIFY)) {
if (has(mode, WriteMode::CREATE)) {
disposition = OPEN_ALWAYS;
} else {
disposition = OPEN_EXISTING;
}
} else {
if (has(mode, WriteMode::CREATE)) {
disposition = CREATE_NEW;
} else {
// Neither CREATE nor MODIFY -- impossible to satisfy preconditions.
return nullptr;
}
}
DWORD access = GENERIC_READ | GENERIC_WRITE;
if (append) {
// FILE_GENERIC_WRITE includes both FILE_APPEND_DATA and FILE_WRITE_DATA, but we only want
// the former. There are also a zillion other bits that we need, annoyingly.
access = (FILE_READ_ATTRIBUTES | FILE_GENERIC_WRITE) & ~FILE_WRITE_DATA;
}
auto filename = path.toString();
HANDLE newHandle;
KJ_WIN32_HANDLE_ERRORS(newHandle = CreateFileW(
nativePath(path).begin(),
access,
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
makeSecAttr(mode),
disposition,
FILE_ATTRIBUTE_NORMAL,
NULL)) {
case ERROR_PATH_NOT_FOUND:
if (has(mode, WriteMode::CREATE)) {
// A parent directory didn't exist. Maybe cerate it.
if (has(mode, WriteMode::CREATE_PARENT) && path.size() > 0 &&
tryMkdir(path.parent(), WriteMode::CREATE | WriteMode::MODIFY |
WriteMode::CREATE_PARENT, true)) {
// Retry, but make sure we don't try to create the parent again.
return tryOpenFileInternal(path, mode - WriteMode::CREATE_PARENT, append);
}
KJ_FAIL_REQUIRE("parent is not a directory", path) { return nullptr; }
} else {
// MODIFY-only mode. ERROR_PATH_NOT_FOUND = parent path doesn't exist = return null.
return nullptr;
}
case ERROR_FILE_NOT_FOUND:
if (!has(mode, WriteMode::CREATE)) {
// MODIFY-only mode. ERROR_FILE_NOT_FOUND = doesn't exist = return null.
return nullptr;
}
goto failed;
case ERROR_ALREADY_EXISTS:
case ERROR_FILE_EXISTS:
if (!has(mode, WriteMode::MODIFY)) {
// CREATE-only mode. ERROR_ALREADY_EXISTS = already exists = return null.
return nullptr;
}
goto failed;
default:
failed:
KJ_FAIL_WIN32("CreateFile", error, path) { return nullptr; }
}
return kj::AutoCloseHandle(newHandle);
}
bool tryCommitReplacement(
PathPtr toPath, ArrayPtr<const wchar_t> fromPath,
WriteMode mode, kj::Maybe<kj::PathPtr> pathForCreatingParents = nullptr) {
// Try to use MoveFileEx() to replace `toPath` with `fromPath`.
auto wToPath = nativePath(toPath);
DWORD flags = has(mode, WriteMode::MODIFY) ? MOVEFILE_REPLACE_EXISTING : 0;
if (!has(mode, WriteMode::CREATE)) {
// Non-atomically verify that target exists. There's no way to make this atomic.
DWORD result = GetFileAttributesW(wToPath.begin());
if (result == INVALID_FILE_ATTRIBUTES) {
auto error = GetLastError();
switch (error) {
case ERROR_FILE_NOT_FOUND:
case ERROR_PATH_NOT_FOUND:
return false;
default:
KJ_FAIL_WIN32("GetFileAttributesEx(toPath)", error, toPath) { return false; }
}
}
}
KJ_WIN32_HANDLE_ERRORS(MoveFileExW(fromPath.begin(), wToPath.begin(), flags)) {
case ERROR_ALREADY_EXISTS:
case ERROR_FILE_EXISTS:
// We must not be in MODIFY mode.
return false;
case ERROR_PATH_NOT_FOUND:
KJ_IF_MAYBE(p, pathForCreatingParents) {
if (has(mode, WriteMode::CREATE_PARENT) &&
p->size() > 0 && tryMkdir(p->parent(),
WriteMode::CREATE | WriteMode::MODIFY | WriteMode::CREATE_PARENT, true)) {
// Retry, but make sure we don't try to create the parent again.
return tryCommitReplacement(toPath, fromPath, mode - WriteMode::CREATE_PARENT);
}
}
goto default_;
case ERROR_ACCESS_DENIED: {
// This often means that the target already exists and cannot be replaced, e.g. because
// it is a directory. Move it out of the way first, then move our replacement in, then
// delete the old thing.
if (has(mode, WriteMode::MODIFY)) {
KJ_IF_MAYBE(tempName,
createNamedTemporary(toPath, WriteMode::CREATE, [&](const wchar_t* tempName2) {
return MoveFileW(wToPath.begin(), tempName2);
})) {
KJ_WIN32_HANDLE_ERRORS(MoveFileW(fromPath.begin(), wToPath.begin())) {
default:
// Try to move back.
MoveFileW(tempName->begin(), wToPath.begin());
KJ_FAIL_WIN32("MoveFile", error, dbgStr(fromPath), dbgStr(wToPath)) {
return false;
}
}
// Succeded, delete temporary.
rmrf(*tempName);
return true;
} else {
// createNamedTemporary() threw exception but exceptions are disabled.
return false;
}
} else {
// Not MODIFY, so no overwrite allowed. If the file really does exist, we need to return
// false.
if (GetFileAttributesW(wToPath.begin()) != INVALID_FILE_ATTRIBUTES) {
return false;
}
}
goto default_;
}
default:
default_:
KJ_FAIL_WIN32("MoveFileEx", error, dbgStr(wToPath), dbgStr(fromPath)) { return false; }
}
return true;
}
template <typename T>
class ReplacerImpl final: public Directory::Replacer<T> {
public:
ReplacerImpl(Own<T>&& object, DiskHandle& parentDirectory,
Array<wchar_t>&& tempPath, Path&& path, WriteMode mode)
: Directory::Replacer<T>(mode),
object(kj::mv(object)), parentDirectory(parentDirectory),
tempPath(kj::mv(tempPath)), path(kj::mv(path)) {}
~ReplacerImpl() noexcept(false) {
if (!committed) {
object = Own<T>(); // Force close of handle before trying to delete.
if (kj::isSameType<T, File>()) {
KJ_WIN32(DeleteFileW(tempPath.begin())) { break; }
} else {
rmrfChildren(tempPath);
KJ_WIN32(RemoveDirectoryW(tempPath.begin())) { break; }
}
}
}
T& get() override {
return *object;
}
bool tryCommit() override {
KJ_ASSERT(!committed, "already committed") { return false; }
// For directories, we intentionally don't use FILE_SHARE_DELETE on our handle because if the
// directory name changes our paths would be wrong. But, this means we can't rename the
// directory here to commit it. So, we need to close the handle and then re-open it
// afterwards. Ick.
AutoCloseHandle* objectHandle = getHandlePointerHack(*object);
if (kj::isSameType<T, Directory>()) {
*objectHandle = nullptr;
}
KJ_DEFER({
if (kj::isSameType<T, Directory>()) {
HANDLE newHandle;
KJ_WIN32(newHandle = CreateFileW(
committed ? parentDirectory.nativePath(path).begin() : tempPath.begin(),
GENERIC_READ,
FILE_SHARE_READ | FILE_SHARE_WRITE,
NULL,
OPEN_EXISTING,
FILE_FLAG_BACKUP_SEMANTICS, // apparently, this flag is required for directories
NULL)) { return; }
*objectHandle = AutoCloseHandle(newHandle);
*getPathPointerHack(*object) = KJ_ASSERT_NONNULL(parentDirectory.dirPath).append(path);
}
});
return committed = parentDirectory.tryCommitReplacement(
path, tempPath, Directory::Replacer<T>::mode);
}
private:
Own<T> object;
DiskHandle& parentDirectory;
Array<wchar_t> tempPath;
Path path;
bool committed = false; // true if *successfully* committed (in which case tempPath is gone)
};
template <typename T>
class BrokenReplacer final: public Directory::Replacer<T> {
// For recovery path when exceptions are disabled.
public:
BrokenReplacer(Own<T> inner)
: Directory::Replacer<T>(WriteMode::CREATE | WriteMode::MODIFY),
inner(kj::mv(inner)) {}
T& get() override { return *inner; }
bool tryCommit() override { return false; }
private:
Own<T> inner;
};
Maybe<Own<File>> tryOpenFile(PathPtr path, WriteMode mode) {
return tryOpenFileInternal(path, mode, false).map(newDiskFile);
}
Own<Directory::Replacer<File>> replaceFile(PathPtr path, WriteMode mode) {
HANDLE newHandle_;
KJ_IF_MAYBE(temp, createNamedTemporary(path, mode,
[&](const wchar_t* candidatePath) {
newHandle_ = CreateFileW(
candidatePath,
GENERIC_READ | GENERIC_WRITE,
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
makeSecAttr(mode),
CREATE_NEW,
FILE_ATTRIBUTE_NORMAL,
NULL);
return newHandle_ != INVALID_HANDLE_VALUE;
})) {
AutoCloseHandle newHandle(newHandle_);
return heap<ReplacerImpl<File>>(newDiskFile(kj::mv(newHandle)), *this, kj::mv(*temp),
path.clone(), mode);
} else {
// threw, but exceptions are disabled
return heap<BrokenReplacer<File>>(newInMemoryFile(nullClock()));
}
}
Own<File> createTemporary() {
HANDLE newHandle_;
KJ_IF_MAYBE(temp, createNamedTemporary(Path("unnamed"), WriteMode::CREATE,
[&](const wchar_t* candidatePath) {
newHandle_ = CreateFileW(
candidatePath,
GENERIC_READ | GENERIC_WRITE,
0,
NULL, // TODO(soon): makeSecAttr(WriteMode::PRIVATE), when it's implemented
CREATE_NEW,
FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE,
NULL);
return newHandle_ != INVALID_HANDLE_VALUE;
})) {
AutoCloseHandle newHandle(newHandle_);
return newDiskFile(kj::mv(newHandle));
} else {
// threw, but exceptions are disabled
return newInMemoryFile(nullClock());
}
}
Maybe<Own<AppendableFile>> tryAppendFile(PathPtr path, WriteMode mode) {
return tryOpenFileInternal(path, mode, true).map(newDiskAppendableFile);
}
Maybe<Own<Directory>> tryOpenSubdir(PathPtr path, WriteMode mode) {
// Must create before open.
if (has(mode, WriteMode::CREATE)) {
if (!tryMkdir(path, mode, false)) return nullptr;
}
return tryOpenSubdirInternal(path).map([&](AutoCloseHandle&& handle) {
return newDiskDirectory(kj::mv(handle), KJ_ASSERT_NONNULL(dirPath).append(path));
});
}
Own<Directory::Replacer<Directory>> replaceSubdir(PathPtr path, WriteMode mode) {
Path kjTempPath = nullptr;
KJ_IF_MAYBE(temp, createNamedTemporary(path, mode, kjTempPath,
[&](const wchar_t* candidatePath) {
return CreateDirectoryW(candidatePath, makeSecAttr(mode));
})) {
HANDLE subdirHandle_;
KJ_WIN32_HANDLE_ERRORS(subdirHandle_ = CreateFileW(
temp->begin(),
GENERIC_READ,
FILE_SHARE_READ | FILE_SHARE_WRITE,
NULL,
OPEN_EXISTING,
FILE_FLAG_BACKUP_SEMANTICS, // apparently, this flag is required for directories
NULL)) {
default:
KJ_FAIL_WIN32("CreateFile(just-created-temporary, OPEN_EXISTING)", error, path) {
goto fail;
}
}
AutoCloseHandle subdirHandle(subdirHandle_);
return heap<ReplacerImpl<Directory>>(
newDiskDirectory(kj::mv(subdirHandle),
KJ_ASSERT_NONNULL(dirPath).append(kj::mv(kjTempPath))),
*this, kj::mv(*temp), path.clone(), mode);
} else {
// threw, but exceptions are disabled
fail:
return heap<BrokenReplacer<Directory>>(newInMemoryDirectory(nullClock()));
}
}
bool trySymlink(PathPtr linkpath, StringPtr content, WriteMode mode) {
// We can't really create symlinks on Windows. Reasons:
// - We'd need to know whether the target is a file or a directory to pass the correct flags.
// That means we'd need to evaluate the link content and track down the target. What if the
// taget doesn't exist? It's unclear if this is even allowed on Windows.
// - Apparently, creating symlinks is a privileged operation on Windows prior to Windows 10.
// The flag SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE is very new.
KJ_UNIMPLEMENTED(
"Creating symbolic links is not supported on Windows due to semantic differences.");
}
bool tryTransfer(PathPtr toPath, WriteMode toMode,
Directory& fromDirectory, PathPtr fromPath,
TransferMode mode, Directory& self) {
KJ_REQUIRE(toPath.size() > 0, "can't replace self") { return false; }
// Try to get the "from" path.
Array<wchar_t> rawFromPath;
#if !KJ_NO_RTTI
// Oops, dynamicDowncastIfAvailable() doesn't work since this isn't a downcast, it's a
// side-cast...
if (auto dh = dynamic_cast<DiskHandle*>(&fromDirectory)) {
rawFromPath = dh->nativePath(fromPath);
} else
#endif
KJ_IF_MAYBE(h, fromDirectory.getWin32Handle()) {
// Can't downcast to DiskHandle, but getWin32Handle() returns a handle... maybe RTTI is
// disabled? Or maybe this is some kind of wrapper?
rawFromPath = getPathFromHandle(*h).append(fromPath).forWin32Api(true);
} else {
// Not a disk directory, so fall back to default implementation.
return self.Directory::tryTransfer(toPath, toMode, fromDirectory, fromPath, mode);
}
if (mode == TransferMode::LINK) {
return tryReplaceNode(toPath, toMode, [&](const wchar_t* candidatePath) {
return CreateHardLinkW(candidatePath, rawFromPath.begin(), NULL);
});
} else if (mode == TransferMode::MOVE) {
return tryCommitReplacement(toPath, rawFromPath, toMode, toPath);
} else if (mode == TransferMode::COPY) {
// We can accellerate copies on Windows.
if (!has(toMode, WriteMode::CREATE)) {
// Non-atomically verify that target exists. There's no way to make this atomic.
if (!exists(toPath)) return false;
}
bool failIfExists = !has(toMode, WriteMode::MODIFY);
KJ_WIN32_HANDLE_ERRORS(
CopyFileW(rawFromPath.begin(), nativePath(toPath).begin(), failIfExists)) {
case ERROR_ALREADY_EXISTS:
case ERROR_FILE_EXISTS:
case ERROR_FILE_NOT_FOUND:
case ERROR_PATH_NOT_FOUND:
return false;
case ERROR_ACCESS_DENIED:
// This usually means that fromPath was a directory or toPath was a direcotry. Fall back
// to default implementation.
break;
default:
KJ_FAIL_WIN32("CopyFile", error, fromPath, toPath) { return false; }
} else {
// Copy succeded.
return true;
}
}
// OK, we can't do anything efficient using the OS. Fall back to default implementation.
return self.Directory::tryTransfer(toPath, toMode, fromDirectory, fromPath, mode);
}
bool tryRemove(PathPtr path) {
return rmrf(nativePath(path));
}
};
#define FSNODE_METHODS \
Maybe<void*> getWin32Handle() override { return DiskHandle::getWin32Handle(); } \
\
Metadata stat() override { return DiskHandle::stat(); } \
void sync() override { DiskHandle::sync(); } \
void datasync() override { DiskHandle::datasync(); }
class DiskReadableFile final: public ReadableFile, public DiskHandle {
public:
DiskReadableFile(AutoCloseHandle&& handle): DiskHandle(kj::mv(handle), nullptr) {}
Own<FsNode> cloneFsNode() override {
return heap<DiskReadableFile>(DiskHandle::clone());
}
FSNODE_METHODS
size_t read(uint64_t offset, ArrayPtr<byte> buffer) override {
return DiskHandle::read(offset, buffer);
}
Array<const byte> mmap(uint64_t offset, uint64_t size) override {
return DiskHandle::mmap(offset, size);
}
Array<byte> mmapPrivate(uint64_t offset, uint64_t size) override {
return DiskHandle::mmapPrivate(offset, size);
}
};
class DiskAppendableFile final: public AppendableFile, public DiskHandle {
public:
DiskAppendableFile(AutoCloseHandle&& handle)
: DiskHandle(kj::mv(handle), nullptr),
stream(DiskHandle::handle.get()) {}
Own<FsNode> cloneFsNode() override {
return heap<DiskAppendableFile>(DiskHandle::clone());
}
FSNODE_METHODS
void write(const void* buffer, size_t size) override { stream.write(buffer, size); }
void write(ArrayPtr<const ArrayPtr<const byte>> pieces) override {
implicitCast<OutputStream&>(stream).write(pieces);
}
private:
HandleOutputStream stream;
};
class DiskFile final: public File, public DiskHandle {
public:
DiskFile(AutoCloseHandle&& handle): DiskHandle(kj::mv(handle), nullptr) {}
Own<FsNode> cloneFsNode() override {
return heap<DiskFile>(DiskHandle::clone());
}
FSNODE_METHODS
size_t read(uint64_t offset, ArrayPtr<byte> buffer) override {
return DiskHandle::read(offset, buffer);
}
Array<const byte> mmap(uint64_t offset, uint64_t size) override {
return DiskHandle::mmap(offset, size);
}
Array<byte> mmapPrivate(uint64_t offset, uint64_t size) override {
return DiskHandle::mmapPrivate(offset, size);
}
void write(uint64_t offset, ArrayPtr<const byte> data) override {
DiskHandle::write(offset, data);
}
void zero(uint64_t offset, uint64_t size) override {
DiskHandle::zero(offset, size);
}
void truncate(uint64_t size) override {
DiskHandle::truncate(size);
}
Own<WritableFileMapping> mmapWritable(uint64_t offset, uint64_t size) override {
return DiskHandle::mmapWritable(offset, size);
}
// copy() is not optimized on Windows.
};
class DiskReadableDirectory final: public ReadableDirectory, public DiskHandle {
public:
DiskReadableDirectory(AutoCloseHandle&& handle, Path&& path)
: DiskHandle(kj::mv(handle), kj::mv(path)) {}
Own<FsNode> cloneFsNode() override {
return heap<DiskReadableDirectory>(DiskHandle::clone(), KJ_ASSERT_NONNULL(dirPath).clone());
}
FSNODE_METHODS
Array<String> listNames() override { return DiskHandle::listNames(); }
Array<Entry> listEntries() override { return DiskHandle::listEntries(); }
bool exists(PathPtr path) override { return DiskHandle::exists(path); }
Maybe<FsNode::Metadata> tryLstat(PathPtr path) override { return DiskHandle::tryLstat(path); }
Maybe<Own<ReadableFile>> tryOpenFile(PathPtr path) override {
return DiskHandle::tryOpenFile(path);
}
Maybe<Own<ReadableDirectory>> tryOpenSubdir(PathPtr path) override {
return DiskHandle::tryOpenSubdir(path);
}
Maybe<String> tryReadlink(PathPtr path) override { return DiskHandle::tryReadlink(path); }
};
class DiskDirectoryBase: public Directory, public DiskHandle {
public:
DiskDirectoryBase(AutoCloseHandle&& handle, Path&& path)
: DiskHandle(kj::mv(handle), kj::mv(path)) {}
bool exists(PathPtr path) override { return DiskHandle::exists(path); }
Maybe<FsNode::Metadata> tryLstat(PathPtr path) override { return DiskHandle::tryLstat(path); }
Maybe<Own<ReadableFile>> tryOpenFile(PathPtr path) override {
return DiskHandle::tryOpenFile(path);
}
Maybe<Own<ReadableDirectory>> tryOpenSubdir(PathPtr path) override {
return DiskHandle::tryOpenSubdir(path);
}
Maybe<String> tryReadlink(PathPtr path) override { return DiskHandle::tryReadlink(path); }
Maybe<Own<File>> tryOpenFile(PathPtr path, WriteMode mode) override {
return DiskHandle::tryOpenFile(path, mode);
}
Own<Replacer<File>> replaceFile(PathPtr path, WriteMode mode) override {
return DiskHandle::replaceFile(path, mode);
}
Maybe<Own<AppendableFile>> tryAppendFile(PathPtr path, WriteMode mode) override {
return DiskHandle::tryAppendFile(path, mode);
}
Maybe<Own<Directory>> tryOpenSubdir(PathPtr path, WriteMode mode) override {
return DiskHandle::tryOpenSubdir(path, mode);
}
Own<Replacer<Directory>> replaceSubdir(PathPtr path, WriteMode mode) override {
return DiskHandle::replaceSubdir(path, mode);
}
bool trySymlink(PathPtr linkpath, StringPtr content, WriteMode mode) override {
return DiskHandle::trySymlink(linkpath, content, mode);
}
bool tryTransfer(PathPtr toPath, WriteMode toMode,
Directory& fromDirectory, PathPtr fromPath,
TransferMode mode) override {
return DiskHandle::tryTransfer(toPath, toMode, fromDirectory, fromPath, mode, *this);
}
// tryTransferTo() not implemented because we have nothing special we can do.
bool tryRemove(PathPtr path) override {
return DiskHandle::tryRemove(path);
}
};
class DiskDirectory final: public DiskDirectoryBase {
public:
DiskDirectory(AutoCloseHandle&& handle, Path&& path)
: DiskDirectoryBase(kj::mv(handle), kj::mv(path)) {}
Own<FsNode> cloneFsNode() override {
return heap<DiskDirectory>(DiskHandle::clone(), KJ_ASSERT_NONNULL(dirPath).clone());
}
FSNODE_METHODS
Array<String> listNames() override { return DiskHandle::listNames(); }
Array<Entry> listEntries() override { return DiskHandle::listEntries(); }
Own<File> createTemporary() override {
return DiskHandle::createTemporary();
}
};
class RootDiskDirectory final: public DiskDirectoryBase {
// On Windows, the root directory is special.
//
// HACK: We only override a few functions of DiskDirectory, and we rely on the fact that
// Path::forWin32Api(true) throws an exception complaining about missing drive letter if the
// path is totally empty.
public:
RootDiskDirectory(): DiskDirectoryBase(nullptr, Path(nullptr)) {}
Own<FsNode> cloneFsNode() override {
return heap<RootDiskDirectory>();
}
Metadata stat() override {
return { Type::DIRECTORY, 0, 0, UNIX_EPOCH, 1 };
}
void sync() override {}
void datasync() override {}
Array<String> listNames() override {
return KJ_MAP(e, listEntries()) { return kj::mv(e.name); };
}
Array<Entry> listEntries() override {
DWORD drives = GetLogicalDrives();
if (drives == 0) {
KJ_FAIL_WIN32("GetLogicalDrives()", GetLastError()) { return nullptr; }
}
Vector<Entry> results;
for (uint i = 0; i < 26; i++) {
if (drives & (1 << i)) {
char name[2] = { 'A' + i, ':' };
results.add(Entry { FsNode::Type::DIRECTORY, kj::heapString(name, 2) });
}
}
return results.releaseAsArray();
}
Own<File> createTemporary() override {
KJ_FAIL_REQUIRE("can't create temporaries in Windows pseudo-root directory (the drive list)");
}
};
class DiskFilesystem final: public Filesystem {
public:
DiskFilesystem()
: DiskFilesystem(computeCurrentPath()) {}
DiskFilesystem(Path currentPath)
: current(KJ_ASSERT_NONNULL(root.tryOpenSubdirInternal(currentPath),
"path returned by GetCurrentDirectory() doesn't exist?"),
kj::mv(currentPath)) {}
Directory& getRoot() override {
return root;
}
Directory& getCurrent() override {
return current;
}
PathPtr getCurrentPath() override {
return KJ_ASSERT_NONNULL(current.dirPath);
}
private:
RootDiskDirectory root;
DiskDirectory current;
static Path computeCurrentPath() {
DWORD tryLen = MAX_PATH;
for (;;) {
auto temp = kj::heapArray<wchar_t>(tryLen + 1);
DWORD len = GetCurrentDirectoryW(temp.size(), temp.begin());
if (len == 0) {
KJ_FAIL_WIN32("GetCurrentDirectory", GetLastError()) { break; }
return Path(".");
}
if (len < temp.size()) {
return Path::parseWin32Api(temp.slice(0, len));
}
// Try again with new length.
tryLen = len;
}
}
};
} // namespace
Own<ReadableFile> newDiskReadableFile(AutoCloseHandle fd) {
return heap<DiskReadableFile>(kj::mv(fd));
}
Own<AppendableFile> newDiskAppendableFile(AutoCloseHandle fd) {
return heap<DiskAppendableFile>(kj::mv(fd));
}
Own<File> newDiskFile(AutoCloseHandle fd) {
return heap<DiskFile>(kj::mv(fd));
}
Own<ReadableDirectory> newDiskReadableDirectory(AutoCloseHandle fd) {
return heap<DiskReadableDirectory>(kj::mv(fd), getPathFromHandle(fd));
}
static Own<ReadableDirectory> newDiskReadableDirectory(AutoCloseHandle fd, Path&& path) {
return heap<DiskReadableDirectory>(kj::mv(fd), kj::mv(path));
}
Own<Directory> newDiskDirectory(AutoCloseHandle fd) {
return heap<DiskDirectory>(kj::mv(fd), getPathFromHandle(fd));
}
static Own<Directory> newDiskDirectory(AutoCloseHandle fd, Path&& path) {
return heap<DiskDirectory>(kj::mv(fd), kj::mv(path));
}
Own<Filesystem> newDiskFilesystem() {
return heap<DiskFilesystem>();
}
static AutoCloseHandle* getHandlePointerHack(Directory& dir) {
return &static_cast<DiskDirectoryBase&>(dir).handle;
}
static Path* getPathPointerHack(Directory& dir) {
return &KJ_ASSERT_NONNULL(static_cast<DiskDirectoryBase&>(dir).dirPath);
}
} // namespace kj
#endif // _WIN32
c++/src/kj/filesystem-test.c++
View file @ ac10f36b
......@@ -21,6 +21,7 @@
#include "filesystem.h"
#include "test.h"
#include <wchar.h>
namespace kj {
namespace {
......@@ -108,6 +109,14 @@ KJ_TEST("Path exceptions") {
KJ_EXPECT_THROW_MESSAGE("root path has no parent", Path(nullptr).parent());
}
static inline bool operator==(const Array<wchar_t>& arr, const wchar_t* expected) {
return wcscmp(arr.begin(), expected) == 0;
}
constexpr kj::ArrayPtr<const wchar_t> operator "" _a(const wchar_t* str, size_t n) {
return { str, n };
}
KJ_TEST("Win32 Path") {
KJ_EXPECT(Path({"foo", "bar"}).toWin32String() == "foo\\bar");
KJ_EXPECT(Path({"foo", "bar"}).toWin32String(true) == "\\\\foo\\bar");
......@@ -147,6 +156,16 @@ KJ_TEST("Win32 Path") {
.toWin32String(true) == "d:\\qux");
KJ_EXPECT(Path({"foo", "bar", "baz"}).evalWin32("\\qux")
.toWin32String(true) == "\\\\foo\\bar\\qux");
KJ_EXPECT(Path({"foo", "bar"}).forWin32Api(false) == L"foo\\bar");
KJ_EXPECT(Path({"foo", "bar"}).forWin32Api(true) == L"\\\\?\\UNC\\foo\\bar");
KJ_EXPECT(Path({"c:", "foo", "bar"}).forWin32Api(true) == L"\\\\?\\c:\\foo\\bar");
KJ_EXPECT(Path({"A:", "foo", "bar"}).forWin32Api(true) == L"\\\\?\\A:\\foo\\bar");
KJ_EXPECT(Path::parseWin32Api(L"\\\\?\\c:\\foo\\bar"_a).toString() == "c:/foo/bar");
KJ_EXPECT(Path::parseWin32Api(L"\\\\?\\UNC\\foo\\bar"_a).toString() == "foo/bar");
KJ_EXPECT(Path::parseWin32Api(L"c:\\foo\\bar"_a).toString() == "c:/foo/bar");
KJ_EXPECT(Path::parseWin32Api(L"\\\\foo\\bar"_a).toString() == "foo/bar");
}
KJ_TEST("Win32 Path exceptions") {
......
c++/src/kj/filesystem.c++
View file @ ac10f36b
......@@ -23,6 +23,8 @@
#include "vector.h"
#include "debug.h"
#include "one-of.h"
#include "encoding.h"
#include "refcount.h"
#include <map>
namespace kj {
......@@ -54,13 +56,18 @@ Path Path::parse(StringPtr path) {
return evalImpl(Vector<String>(countParts(path)), path);
}
Path PathPtr::append(Path suffix) const {
Path Path::parseWin32Api(ArrayPtr<const wchar_t> text) {
auto utf8 = decodeWideString(text);
return evalWin32Impl(Vector<String>(countPartsWin32(utf8)), utf8, true);
}
Path PathPtr::append(Path&& suffix) const {
auto newParts = kj::heapArrayBuilder<String>(parts.size() + suffix.parts.size());
for (auto& p: parts) newParts.add(heapString(p));
for (auto& p: suffix.parts) newParts.add(kj::mv(p));
return Path(newParts.finish(), Path::ALREADY_CHECKED);
}
Path Path::append(Path suffix) && {
Path Path::append(Path&& suffix) && {
auto newParts = kj::heapArrayBuilder<String>(parts.size() + suffix.parts.size());
for (auto& p: parts) newParts.add(kj::mv(p));
for (auto& p: suffix.parts) newParts.add(kj::mv(p));
......@@ -159,7 +166,7 @@ Path Path::evalWin32(StringPtr pathText) && {
return evalWin32Impl(kj::mv(newParts), pathText);
}
String PathPtr::toWin32String(bool absolute) const {
String PathPtr::toWin32StringImpl(bool absolute, bool forApi) const {
if (parts.size() == 0) {
// Special-case empty path.
KJ_REQUIRE(!absolute, "absolute path is missing disk designator") {
......@@ -178,19 +185,37 @@ String PathPtr::toWin32String(bool absolute) const {
KJ_FAIL_REQUIRE("absolute win32 path must start with drive letter or netbios host name",
parts[0]);
}
} else {
// Currently we do nothing differently in the forApi case for relative paths.
forApi = false;
}
size_t size = (isUncPath ? 2 : 0) + (parts.size() - 1);
size_t size = forApi
? (isUncPath ? 8 : 4) + (parts.size() - 1)
: (isUncPath ? 2 : 0) + (parts.size() - 1);
for (auto& p: parts) size += p.size();
String result = heapString(size);
char* ptr = result.begin();
if (forApi) {
*ptr++ = '\\';
*ptr++ = '\\';
*ptr++ = '?';
*ptr++ = '\\';
if (isUncPath) {
*ptr++ = 'U';
*ptr++ = 'N';
*ptr++ = 'C';
*ptr++ = '\\';
}
} else {
if (isUncPath) {
*ptr++ = '\\';
*ptr++ = '\\';
}
}
bool leadingSlash = false;
for (auto& p: parts) {
......@@ -217,7 +242,7 @@ String PathPtr::toWin32String(bool absolute) const {
// appearing to start with a drive letter.
for (size_t i: kj::indices(result)) {
if (result[i] == ':') {
if (absolute && i == 1) {
if (absolute && i == (forApi ? 5 : 1)) {
// False alarm: this is the drive letter.
} else {
KJ_FAIL_REQUIRE(
......@@ -233,6 +258,10 @@ String PathPtr::toWin32String(bool absolute) const {
return result;
}
Array<wchar_t> PathPtr::forWin32Api(bool absolute) const {
return encodeWideString(toWin32StringImpl(absolute, true), true);
}
// -----------------------------------------------------------------------------
String Path::stripNul(String input) {
......@@ -290,10 +319,10 @@ Path Path::evalImpl(Vector<String>&& parts, StringPtr path) {
return Path(parts.releaseAsArray(), Path::ALREADY_CHECKED);
}
Path Path::evalWin32Impl(Vector<String>&& parts, StringPtr path) {
Path Path::evalWin32Impl(Vector<String>&& parts, StringPtr path, bool fromApi) {
// Convert all forward slashes to backslashes.
String ownPath;
if (path.findFirst('/') != nullptr) {
if (!fromApi && path.findFirst('/') != nullptr) {
ownPath = heapString(path);
for (char& c: ownPath) {
if (c == '/') c = '\\';
......@@ -302,13 +331,23 @@ Path Path::evalWin32Impl(Vector<String>&& parts, StringPtr path) {
}
// Interpret various forms of absolute paths.
if (path.startsWith("\\\\")) {
if (fromApi && path.startsWith("\\\\?\\")) {
path = path.slice(4);
if (path.startsWith("UNC\\")) {
path = path.slice(4);
}
// The path is absolute.
parts.clear();
} else if (path.startsWith("\\\\")) {
// UNC path.
path = path.slice(2);
// This path is absolute. The first component is a server name.
parts.clear();
} else if (path.startsWith("\\")) {
KJ_REQUIRE(!fromApi, "parseWin32Api() requires absolute path");
// Path is relative to the current drive / network share.
if (parts.size() >= 1 && isWin32Drive(parts[0])) {
// Leading \ interpreted as root of current drive.
......@@ -329,6 +368,8 @@ Path Path::evalWin32Impl(Vector<String>&& parts, StringPtr path) {
isWin32Drive(path.slice(0, 2))) {
// Starts with a drive letter.
parts.clear();
} else {
KJ_REQUIRE(!fromApi, "parseWin32Api() requires absolute path");
}
size_t partStart = 0;
......@@ -671,6 +712,8 @@ bool Directory::tryTransfer(PathPtr toPath, WriteMode toMode,
case TransferMode::LINK:
KJ_FAIL_REQUIRE("can't link across different Directory implementations") { return false; }
}
KJ_UNREACHABLE;
}
Maybe<bool> Directory::tryTransferTo(Directory& toDirectory, PathPtr toPath, WriteMode toMode,
......@@ -1295,7 +1338,7 @@ private:
Date lastModified;
template <typename T>
class ReplacerImpl: public Replacer<T> {
class ReplacerImpl final: public Replacer<T> {
public:
ReplacerImpl(InMemoryDirectory& directory, kj::StringPtr name, Own<T> inner, WriteMode mode)
: Replacer<T>(mode), directory(addRef(directory)), name(heapString(name)),
......@@ -1323,7 +1366,7 @@ private:
};
template <typename T>
class BrokenReplacer: public Replacer<T> {
class BrokenReplacer final: public Replacer<T> {
// For recovery path when exceptions are disabled.
public:
......
c++/src/kj/filesystem.h
View file @ ac10f36b
......@@ -25,7 +25,7 @@
#include "memory.h"
#include "io.h"
#include <inttypes.h>
#include "time.h" // TODO(now): problematic
#include "time.h"
#include "function.h"
namespace kj {
......@@ -97,14 +97,14 @@ public:
// is NOT accepted -- if that is a problem, you probably want `eval()`. Trailing '/'s are
// ignored.
Path append(Path suffix) const&;
Path append(Path suffix) &&;
Path append(Path&& suffix) const&;
Path append(Path&& suffix) &&;
Path append(PathPtr suffix) const&;
Path append(PathPtr suffix) &&;
Path append(StringPtr suffix) const&;
Path append(StringPtr suffix) &&;
Path append(String suffix) const&;
Path append(String suffix) &&;
Path append(String&& suffix) const&;
Path append(String&& suffix) &&;
// Create a new path by appending the given path to this path.
//
// `suffix` cannot contain '/' characters. Instead, you can append an array:
......@@ -151,7 +151,8 @@ public:
Path evalWin32(StringPtr pathText) const&;
Path evalWin32(StringPtr pathText) &&;
// Evaluates a Win32-style path. Differences from `eval()` include:
// Evaluates a Win32-style path, as might be written by a user. Differences from `eval()`
// include:
//
// - Backslashes can be used as path separators.
// - Absolute paths begin with a drive letter followed by a colon. The drive letter, including
......@@ -159,9 +160,9 @@ public:
// - A network path like "\\host\share\path" is parsed as {"host", "share", "path"}.
String toWin32String(bool absolute = false) const;
// Converts the path to a Win32 path string.
// Converts the path to a Win32 path string, as you might display to a user.
//
// (In most cases you'll want to further convert the returned string from UTF-8 to UTF-16.)
// This is meant for display. For making Win32 system calls, consider `toWin32Api()` instead.
//
// If `absolute` is true, the path is expected to be an absolute path, meaning the first
// component is a drive letter, namespace, or network host name. These are converted to their
......@@ -171,6 +172,24 @@ public:
// Windows, such as if it contains backslashes (within a path component), colons, or special
// names like "con".
Array<wchar_t> forWin32Api(bool absolute) const;
// Like toWin32String, but additionally:
// - Converts the path to UTF-16, with a NUL terminator included.
// - For absolute paths, adds the "\\?\" prefix which opts into permitting paths longer than
// MAX_PATH, and turns off relative path processing (which KJ paths already handle in userspace
// anyway).
//
// This method is good to use when making a Win32 API call, e.g.:
//
// DeleteFileW(path.forWin32Api(true).begin());
static Path parseWin32Api(ArrayPtr<const wchar_t> text);
// Parses an absolute path as returned by a Win32 API call like GetFinalPathNameByHandle() or
// GetCurrentDirectory(). A "\\?\" prefix is optional but understood if present.
//
// Since such Win32 API calls generally return a length, this function inputs an array slice.
// The slice should not include any NUL terminator.
private:
Array<String> parts;
......@@ -186,7 +205,7 @@ private:
static void validatePart(StringPtr part);
static void evalPart(Vector<String>& parts, ArrayPtr<const char> part);
static Path evalImpl(Vector<String>&& parts, StringPtr path);
static Path evalWin32Impl(Vector<String>&& parts, StringPtr path);
static Path evalWin32Impl(Vector<String>&& parts, StringPtr path, bool fromApi = false);
static size_t countParts(StringPtr path);
static size_t countPartsWin32(StringPtr path);
static bool isWin32Drive(ArrayPtr<const char> part);
......@@ -204,10 +223,10 @@ public:
PathPtr(const Path& path);
Path clone();
Path append(Path suffix) const;
Path append(Path&& suffix) const;
Path append(PathPtr suffix) const;
Path append(StringPtr suffix) const;
Path append(String suffix) const;
Path append(String&& suffix) const;
Path eval(StringPtr pathText) const;
PathPtr basename() const;
PathPtr parent() const;
......@@ -219,6 +238,7 @@ public:
PathPtr slice(size_t start, size_t end) const;
Path evalWin32(StringPtr pathText) const;
String toWin32String(bool absolute = false) const;
Array<wchar_t> forWin32Api(bool absolute) const;
// Equivalent to the corresponding methods of `Path`.
private:
......@@ -226,6 +246,8 @@ private:
explicit PathPtr(ArrayPtr<const String> parts);
String toWin32StringImpl(bool absolute, bool forApi) const;
friend class Path;
};
......@@ -251,10 +273,14 @@ public:
//
// Under the hood, this will call dup(), so the FD number will not be the same.
virtual Maybe<int> getFd() = 0;
// Get the underlying file descriptor, if any. Returns nullptr if this object actually isn't
virtual Maybe<int> getFd() { return nullptr; }
// Get the underlying Unix file descriptor, if any. Returns nullptr if this object actually isn't
// wrapping a file descriptor.
virtual Maybe<void*> getWin32Handle() { return nullptr; }
// Get the underlying Win32 HANDLE, if any. Returns nullptr if this object actually isn't
// wrapping a handle.
enum class Type {
FILE,
DIRECTORY,
......@@ -380,6 +406,12 @@ public:
// returning.
//
// `slice` must be a slice of `bytes()`.
//
// On Windows, this calls FlushViewOfFile(). The documentation for this function implies that in
// some circumstances, to fully sync to physical disk, you may need to call FlushFileBuffers() on
// the file HANDLE as well. The documentation is not very clear on when and why this is needed.
// If you believe your program needs this, you can accomplish it by calling `.sync()` on the File
// object after calling `.sync()` on the WritableFileMapping.
};
class File: public ReadableFile {
......@@ -473,6 +505,13 @@ public:
virtual Maybe<String> tryReadlink(PathPtr path) = 0;
// If `path` is a symlink, reads and returns the link contents.
//
// Note that tryReadlink() differs subtly from tryOpen*(). For example, tryOpenFile() throws if
// the path is not a file (e.g. if it's a directory); it only returns null if the path doesn't
// exist at all. tryReadlink() returns null if either the path doesn't exist, or if it does exist
// but isn't a symlink. This is because if it were to throw instead, then almost every real-world
// use case of tryReadlink() would be forced to perform an lstat() first for the sole purpose of
// checking if it is a link, wasting a syscall and a path traversal.
//
// See Directory::symlink() for warnings about symlinks.
};
......@@ -501,8 +540,6 @@ enum class WriteMode {
CREATE = 1,
// Create a new empty file.
//
// This can be OR'd with MODIFY, but not with REPLACE.
//
// When not combined with MODIFY, if the file already exists (including as a broken symlink),
// tryOpenFile() returns null (and openFile() throws).
//
......@@ -513,8 +550,6 @@ enum class WriteMode {
MODIFY = 2,
// Modify an existing file.
//
// This can be OR'd with CREATE, but not with REPLACE.
//
// When not combined with CREATE, if the file doesn't exist (including if it is a broken symlink),
// tryOpenFile() returns null (and openFile() throws).
//
......@@ -590,11 +625,19 @@ class Directory: public ReadableDirectory {
// A `Directory` object *only* provides access to children of the directory, not parents. That
// is, you cannot open the file "..", nor jump to the root directory with "/".
//
// On OSs that support in, a `Directory` is backed by an open handle to the directory node. This
// On OSs that support it, a `Directory` is backed by an open handle to the directory node. This
// means:
// - If the directory is renamed on-disk, the `Directory` object still points at it.
// - Opening files in the directory only requires the OS to traverse the path from the directory
// to the file; it doesn't have to re-traverse all the way from the filesystem root.
//
// On Windows, a `Directory` object holds a lock on the underlying directory such that it cannot
// be renamed nor deleted while the object exists. This is necessary because Windows does not
// fully support traversing paths relative to file handles (it does for some operations but not
// all), so the KJ filesystem implementation is forced to remember the full path and needs to
// ensure that the path is not invalidated. If, in the future, Windows fully supports
// handle-relative paths, KJ may stop locking directories in this way, so do not rely on this
// behavior.
public:
Own<Directory> clone();
......@@ -666,6 +709,11 @@ public:
// disappearing after the replacement (actually, a swap) has taken place. This differs from
// files, where a process that has opened a file before it is replaced will continue see the
// file's old content unchanged after the replacement.
// - On Windows, there are multiple ways to replace one file with another in a single system
// call, but none are documented as being atomic. KJ always uses `MoveFileEx()` with
// MOVEFILE_REPLACE_EXISTING. While the alternative `ReplaceFile()` is attractive for many
// reasons, it has the critical problem that it cannot be used when the source file has open
// file handles, which is generally the case when using Replacer.
protected:
const WriteMode mode;
......@@ -829,12 +877,18 @@ Own<AppendableFile> newFileAppender(Own<File> inner);
// are happening simultaneously, as is achieved with the O_APPEND flag to open(2), but that
// behavior is not possible to emulate on top of `File`.
Own<ReadableFile> newDiskReadableFile(kj::AutoCloseFd fd);
Own<AppendableFile> newDiskAppendableFile(kj::AutoCloseFd fd);
Own<File> newDiskFile(kj::AutoCloseFd fd);
Own<ReadableDirectory> newDiskReadableDirectory(kj::AutoCloseFd fd);
Own<Directory> newDiskDirectory(kj::AutoCloseFd fd);
// Wrap a file descriptor as various filesystem types.
#if _WIN32
typedef AutoCloseHandle OsFileHandle;
#else
typedef AutoCloseFd OsFileHandle;
#endif
Own<ReadableFile> newDiskReadableFile(OsFileHandle fd);
Own<AppendableFile> newDiskAppendableFile(OsFileHandle fd);
Own<File> newDiskFile(OsFileHandle fd);
Own<ReadableDirectory> newDiskReadableDirectory(OsFileHandle fd);
Own<Directory> newDiskDirectory(OsFileHandle fd);
// Wrap a file descriptor (or Windows HANDLE) as various filesystem types.
Own<Filesystem> newDiskFilesystem();
// Get at implementation of `Filesystem` representing the real filesystem.
......@@ -856,12 +910,12 @@ inline Path::Path(std::initializer_list<StringPtr> parts)
inline Path::Path(Array<String> parts, decltype(ALREADY_CHECKED))
: parts(kj::mv(parts)) {}
inline Path Path::clone() const { return PathPtr(*this).clone(); }
inline Path Path::append(Path suffix) const& { return PathPtr(*this).append(kj::mv(suffix)); }
inline Path Path::append(Path&& suffix) const& { return PathPtr(*this).append(kj::mv(suffix)); }
inline Path Path::append(PathPtr suffix) const& { return PathPtr(*this).append(suffix); }
inline Path Path::append(StringPtr suffix) const& { return append(Path(suffix)); }
inline Path Path::append(StringPtr suffix) && { return kj::mv(*this).append(Path(suffix)); }
inline Path Path::append(String suffix) const& { return append(Path(kj::mv(suffix))); }
inline Path Path::append(String suffix) && { return kj::mv(*this).append(Path(kj::mv(suffix))); }
inline Path Path::append(String&& suffix) const& { return append(Path(kj::mv(suffix))); }
inline Path Path::append(String&& suffix) && { return kj::mv(*this).append(Path(kj::mv(suffix))); }
inline Path Path::eval(StringPtr pathText) const& { return PathPtr(*this).eval(pathText); }
inline PathPtr Path::basename() const& { return PathPtr(*this).basename(); }
inline PathPtr Path::parent() const& { return PathPtr(*this).parent(); }
......@@ -880,12 +934,15 @@ inline Path Path::evalWin32(StringPtr pathText) const& {
inline String Path::toWin32String(bool absolute) const {
return PathPtr(*this).toWin32String(absolute);
}
inline Array<wchar_t> Path::forWin32Api(bool absolute) const {
return PathPtr(*this).forWin32Api(absolute);
}
inline PathPtr::PathPtr(decltype(nullptr)): parts(nullptr) {}
inline PathPtr::PathPtr(const Path& path): parts(path.parts) {}
inline PathPtr::PathPtr(ArrayPtr<const String> parts): parts(parts) {}
inline Path PathPtr::append(StringPtr suffix) const { return append(Path(suffix)); }
inline Path PathPtr::append(String suffix) const { return append(Path(kj::mv(suffix))); }
inline Path PathPtr::append(String&& suffix) const { return append(Path(kj::mv(suffix))); }
inline const String& PathPtr::operator[](size_t i) const { return parts[i]; }
inline size_t PathPtr::size() const { return parts.size(); }
inline const String* PathPtr::begin() const { return parts.begin(); }
......@@ -893,6 +950,9 @@ inline const String* PathPtr::end() const { return parts.end(); }
inline PathPtr PathPtr::slice(size_t start, size_t end) const {
return PathPtr(parts.slice(start, end));
}
inline String PathPtr::toWin32String(bool absolute) const {
return toWin32StringImpl(absolute, false);
}
inline Own<FsNode> FsNode::clone() { return cloneFsNode().downcast<FsNode>(); }
inline Own<ReadableFile> ReadableFile::clone() { return cloneFsNode().downcast<ReadableFile>(); }
......
c++/src/kj/time.c++
View file @ ac10f36b
......@@ -26,10 +26,6 @@
namespace kj {
kj::Exception Timer::makeTimeoutException() {
return KJ_EXCEPTION(OVERLOADED, "operation timed out");
}
Clock& nullClock() {
class NullClock final: public Clock {
public:
......@@ -39,96 +35,4 @@ Clock& nullClock() {
return NULL_CLOCK;
}
struct TimerImpl::Impl {
struct TimerBefore {
bool operator()(TimerPromiseAdapter* lhs, TimerPromiseAdapter* rhs);
};
using Timers = std::multiset<TimerPromiseAdapter*, TimerBefore>;
Timers timers;
};
class TimerImpl::TimerPromiseAdapter {
public:
TimerPromiseAdapter(PromiseFulfiller<void>& fulfiller, TimerImpl::Impl& impl, TimePoint time)
: time(time), fulfiller(fulfiller), impl(impl) {
pos = impl.timers.insert(this);
}
~TimerPromiseAdapter() {
if (pos != impl.timers.end()) {
impl.timers.erase(pos);
}
}
void fulfill() {
fulfiller.fulfill();
impl.timers.erase(pos);
pos = impl.timers.end();
}
const TimePoint time;
private:
PromiseFulfiller<void>& fulfiller;
TimerImpl::Impl& impl;
Impl::Timers::const_iterator pos;
};
inline bool TimerImpl::Impl::TimerBefore::operator()(
TimerPromiseAdapter* lhs, TimerPromiseAdapter* rhs) {
return lhs->time < rhs->time;
}
Promise<void> TimerImpl::atTime(TimePoint time) {
return newAdaptedPromise<void, TimerPromiseAdapter>(*impl, time);
}
Promise<void> TimerImpl::afterDelay(Duration delay) {
return newAdaptedPromise<void, TimerPromiseAdapter>(*impl, time + delay);
}
TimerImpl::TimerImpl(TimePoint startTime)
: time(startTime), impl(heap<Impl>()) {}
TimerImpl::~TimerImpl() noexcept(false) {}
Maybe<TimePoint> TimerImpl::nextEvent() {
auto iter = impl->timers.begin();
if (iter == impl->timers.end()) {
return nullptr;
} else {
return (*iter)->time;
}
}
Maybe<uint64_t> TimerImpl::timeoutToNextEvent(TimePoint start, Duration unit, uint64_t max) {
return nextEvent().map([&](TimePoint nextTime) -> uint64_t {
if (nextTime <= start) return 0;
Duration timeout = nextTime - start;
uint64_t result = timeout / unit;
bool roundUp = timeout % unit > 0 * SECONDS;
if (result >= max) {
return max;
} else {
return result + roundUp;
}
});
}
void TimerImpl::advanceTo(TimePoint newTime) {
KJ_REQUIRE(newTime >= time, "can't advance backwards in time") { return; }
time = newTime;
for (;;) {
auto front = impl->timers.begin();
if (front == impl->timers.end() || (*front)->time > time) {
break;
}
(*front)->fulfill();
}
}
} // namespace kj
c++/src/kj/time.h
View file @ ac10f36b
......@@ -27,7 +27,6 @@
#pragma GCC system_header
#endif
#include "async.h"
#include "units.h"
#include <inttypes.h>
......@@ -71,104 +70,6 @@ Clock& nullClock();
// A clock which always returns UNIX_EPOCH as the current time. Useful when you don't care about
// time.
class Timer {
// Interface to time and timer functionality.
//
// Each `Timer` may have a different origin, and some `Timer`s may in fact tick at a different
// rate than real time (e.g. a `Timer` could represent CPU time consumed by a thread). However,
// all `Timer`s are monotonic: time will never appear to move backwards, even if the calendar
// date as tracked by the system is manually modified.
public:
virtual TimePoint now() = 0;
// Returns the current value of a clock that moves steadily forward, independent of any
// changes in the wall clock. The value is updated every time the event loop waits,
// and is constant in-between waits.
virtual Promise<void> atTime(TimePoint time) = 0;
// Returns a promise that returns as soon as now() >= time.
virtual Promise<void> afterDelay(Duration delay) = 0;
// Equivalent to atTime(now() + delay).
template <typename T>
Promise<T> timeoutAt(TimePoint time, Promise<T>&& promise) KJ_WARN_UNUSED_RESULT;
// Return a promise equivalent to `promise` but which throws an exception (and cancels the
// original promise) if it hasn't completed by `time`. The thrown exception is of type
// "OVERLOADED".
template <typename T>
Promise<T> timeoutAfter(Duration delay, Promise<T>&& promise) KJ_WARN_UNUSED_RESULT;
// Return a promise equivalent to `promise` but which throws an exception (and cancels the
// original promise) if it hasn't completed after `delay` from now. The thrown exception is of
// type "OVERLOADED".
private:
static kj::Exception makeTimeoutException();
};
class TimerImpl final: public Timer {
// Implementation of Timer that expects an external caller -- usually, the EventPort
// implementation -- to tell it when time has advanced.
public:
TimerImpl(TimePoint startTime);
~TimerImpl() noexcept(false);
Maybe<TimePoint> nextEvent();
// Returns the time at which the next scheduled timer event will occur, or null if no timer
// events are scheduled.
Maybe<uint64_t> timeoutToNextEvent(TimePoint start, Duration unit, uint64_t max);
// Convenience method which computes a timeout value to pass to an event-waiting system call to
// cause it to time out when the next timer event occurs.
//
// `start` is the time at which the timeout starts counting. This is typically not the same as
// now() since some time may have passed since the last time advanceTo() was called.
//
// `unit` is the time unit in which the timeout is measured. This is often MILLISECONDS. Note
// that this method will fractional values *up*, to guarantee that the returned timeout waits
// until just *after* the time the event is scheduled.
//
// The timeout will be clamped to `max`. Use this to avoid an overflow if e.g. the OS wants a
// 32-bit value or a signed value.
//
// Returns nullptr if there are no future events.
void advanceTo(TimePoint newTime);
// Set the time to `time` and fire any at() events that have been passed.
// implements Timer ----------------------------------------------------------
TimePoint now() override;
Promise<void> atTime(TimePoint time) override;
Promise<void> afterDelay(Duration delay) override;
private:
struct Impl;
class TimerPromiseAdapter;
TimePoint time;
Own<Impl> impl;
};
// =======================================================================================
// inline implementation details
template <typename T>
Promise<T> Timer::timeoutAt(TimePoint time, Promise<T>&& promise) {
return promise.exclusiveJoin(atTime(time).then([]() -> kj::Promise<T> {
return makeTimeoutException();
}));
}
template <typename T>
Promise<T> Timer::timeoutAfter(Duration delay, Promise<T>&& promise) {
return promise.exclusiveJoin(afterDelay(delay).then([]() -> kj::Promise<T> {
return makeTimeoutException();
}));
}
inline TimePoint TimerImpl::now() { return time; }
} // namespace kj
#endif // KJ_TIME_H_
c++/src/kj/timer.c++ 0 → 100644
View file @ ac10f36b
// Copyright (c) 2014 Google Inc. (contributed by Remy Blank <rblank@google.com>)
// Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors
// Licensed under the MIT License:
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
#include "timer.h"
#include "debug.h"
#include <set>
namespace kj {
kj::Exception Timer::makeTimeoutException() {
return KJ_EXCEPTION(OVERLOADED, "operation timed out");
}
struct TimerImpl::Impl {
struct TimerBefore {
bool operator()(TimerPromiseAdapter* lhs, TimerPromiseAdapter* rhs);
};
using Timers = std::multiset<TimerPromiseAdapter*, TimerBefore>;
Timers timers;
};
class TimerImpl::TimerPromiseAdapter {
public:
TimerPromiseAdapter(PromiseFulfiller<void>& fulfiller, TimerImpl::Impl& impl, TimePoint time)
: time(time), fulfiller(fulfiller), impl(impl) {
pos = impl.timers.insert(this);
}
~TimerPromiseAdapter() {
if (pos != impl.timers.end()) {
impl.timers.erase(pos);
}
}
void fulfill() {
fulfiller.fulfill();
impl.timers.erase(pos);
pos = impl.timers.end();
}
const TimePoint time;
private:
PromiseFulfiller<void>& fulfiller;
TimerImpl::Impl& impl;
Impl::Timers::const_iterator pos;
};
inline bool TimerImpl::Impl::TimerBefore::operator()(
TimerPromiseAdapter* lhs, TimerPromiseAdapter* rhs) {
return lhs->time < rhs->time;
}
Promise<void> TimerImpl::atTime(TimePoint time) {
return newAdaptedPromise<void, TimerPromiseAdapter>(*impl, time);
}
Promise<void> TimerImpl::afterDelay(Duration delay) {
return newAdaptedPromise<void, TimerPromiseAdapter>(*impl, time + delay);
}
TimerImpl::TimerImpl(TimePoint startTime)
: time(startTime), impl(heap<Impl>()) {}
TimerImpl::~TimerImpl() noexcept(false) {}
Maybe<TimePoint> TimerImpl::nextEvent() {
auto iter = impl->timers.begin();
if (iter == impl->timers.end()) {
return nullptr;
} else {
return (*iter)->time;
}
}
Maybe<uint64_t> TimerImpl::timeoutToNextEvent(TimePoint start, Duration unit, uint64_t max) {
return nextEvent().map([&](TimePoint nextTime) -> uint64_t {
if (nextTime <= start) return 0;
Duration timeout = nextTime - start;
uint64_t result = timeout / unit;
bool roundUp = timeout % unit > 0 * SECONDS;
if (result >= max) {
return max;
} else {
return result + roundUp;
}
});
}
void TimerImpl::advanceTo(TimePoint newTime) {
KJ_REQUIRE(newTime >= time, "can't advance backwards in time") { return; }
time = newTime;
for (;;) {
auto front = impl->timers.begin();
if (front == impl->timers.end() || (*front)->time > time) {
break;
}
(*front)->fulfill();
}
}
} // namespace kj
c++/src/kj/timer.h 0 → 100644
View file @ ac10f36b
// Copyright (c) 2014 Google Inc. (contributed by Remy Blank <rblank@google.com>)
// Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors
// Licensed under the MIT License:
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
#ifndef KJ_TIMER_H_
#define KJ_TIMER_H_
#if defined(__GNUC__) && !KJ_HEADER_WARNINGS
#pragma GCC system_header
#endif
#include "time.h"
#include "async.h"
namespace kj {
class Timer {
// Interface to time and timer functionality.
//
// Each `Timer` may have a different origin, and some `Timer`s may in fact tick at a different
// rate than real time (e.g. a `Timer` could represent CPU time consumed by a thread). However,
// all `Timer`s are monotonic: time will never appear to move backwards, even if the calendar
// date as tracked by the system is manually modified.
public:
virtual TimePoint now() = 0;
// Returns the current value of a clock that moves steadily forward, independent of any
// changes in the wall clock. The value is updated every time the event loop waits,
// and is constant in-between waits.
virtual Promise<void> atTime(TimePoint time) = 0;
// Returns a promise that returns as soon as now() >= time.
virtual Promise<void> afterDelay(Duration delay) = 0;
// Equivalent to atTime(now() + delay).
template <typename T>
Promise<T> timeoutAt(TimePoint time, Promise<T>&& promise) KJ_WARN_UNUSED_RESULT;
// Return a promise equivalent to `promise` but which throws an exception (and cancels the
// original promise) if it hasn't completed by `time`. The thrown exception is of type
// "OVERLOADED".
template <typename T>
Promise<T> timeoutAfter(Duration delay, Promise<T>&& promise) KJ_WARN_UNUSED_RESULT;
// Return a promise equivalent to `promise` but which throws an exception (and cancels the
// original promise) if it hasn't completed after `delay` from now. The thrown exception is of
// type "OVERLOADED".
private:
static kj::Exception makeTimeoutException();
};
class TimerImpl final: public Timer {
// Implementation of Timer that expects an external caller -- usually, the EventPort
// implementation -- to tell it when time has advanced.
public:
TimerImpl(TimePoint startTime);
~TimerImpl() noexcept(false);
Maybe<TimePoint> nextEvent();
// Returns the time at which the next scheduled timer event will occur, or null if no timer
// events are scheduled.
Maybe<uint64_t> timeoutToNextEvent(TimePoint start, Duration unit, uint64_t max);
// Convenience method which computes a timeout value to pass to an event-waiting system call to
// cause it to time out when the next timer event occurs.
//
// `start` is the time at which the timeout starts counting. This is typically not the same as
// now() since some time may have passed since the last time advanceTo() was called.
//
// `unit` is the time unit in which the timeout is measured. This is often MILLISECONDS. Note
// that this method will fractional values *up*, to guarantee that the returned timeout waits
// until just *after* the time the event is scheduled.
//
// The timeout will be clamped to `max`. Use this to avoid an overflow if e.g. the OS wants a
// 32-bit value or a signed value.
//
// Returns nullptr if there are no future events.
void advanceTo(TimePoint newTime);
// Set the time to `time` and fire any at() events that have been passed.
// implements Timer ----------------------------------------------------------
TimePoint now() override;
Promise<void> atTime(TimePoint time) override;
Promise<void> afterDelay(Duration delay) override;
private:
struct Impl;
class TimerPromiseAdapter;
TimePoint time;
Own<Impl> impl;
};
// =======================================================================================
// inline implementation details
template <typename T>
Promise<T> Timer::timeoutAt(TimePoint time, Promise<T>&& promise) {
return promise.exclusiveJoin(atTime(time).then([]() -> kj::Promise<T> {
return makeTimeoutException();
}));
}
template <typename T>
Promise<T> Timer::timeoutAfter(Duration delay, Promise<T>&& promise) {
return promise.exclusiveJoin(afterDelay(delay).then([]() -> kj::Promise<T> {
return makeTimeoutException();
}));
}
inline TimePoint TimerImpl::now() { return time; }
} // namespace kj
#endif // KJ_TIMER_H_
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment