Skip to content

C
capnproto
Commit b1701b88 authored by Kenton Varda's avatar Kenton Varda Committed by GitHub
Browse files
Options

Merge pull request #384 from sandstorm-io/filesystem 

KJ Filesystem API
parents ab5e35e4 6d4c05da
Hide whitespace changes
Inline Side-by-side
Showing with 5616 additions and 0 deletions
c++/src/kj/filesystem-disk-generic-test.c++ 0 → 100644
View file @ b1701b88
// Copyright (c) 2016 Sandstorm Development Group, Inc. and contributors
// Licensed under the MIT License:
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
// This test compiles filesystem-disk.c++ with various features #undefed, causing it to take
// different code paths, then runs filesystem-disk-test.c++ against that.
#include "filesystem.h"
#include "debug.h"
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdio.h>
#include <sys/mman.h>
#include <errno.h>
#include <dirent.h>
#include <syscall.h>
#include "vector.h"
#include "miniposix.h"
#undef __linux__
#undef O_CLOEXEC
#undef O_DIRECTORY
#undef O_TMPFILE
#undef FIOCLEX
#undef DT_UNKNOWN
#undef F_DUPFD_CLOEXEC
#undef FALLOC_FL_PUNCH_HOLE
#undef FICLONE
#undef FICLONERANGE
#undef SEEK_HOLE
#undef SEEK_DATA
#undef RENAME_EXCHANGE
#define HOLES_NOT_SUPPORTED
#include "filesystem-disk.c++"
#include "filesystem-disk-test.c++"
c++/src/kj/filesystem-disk-old-kernel-test.c++ 0 → 100644
View file @ b1701b88
// Copyright (c) 2016 Sandstorm Development Group, Inc. and contributors
// Licensed under the MIT License:
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
#if __linux__ && __x86_64__ && defined(__has_include)
#if __has_include(<linux/seccomp.h>) && \
__has_include(<linux/filter.h>) && \
__has_include(<linux/audit.h>) && \
__has_include(<linux/signal.h>) && \
__has_include(<sys/ptrace.h>)
// This test re-runs filesystem-disk-test.c++ with newfangled Linux kernel features disabled.
//
// This test must be complied as a separate program, since it alters the calling process by
// enabling seccomp to disable the kernel features.
#include <syscall.h>
#include <unistd.h>
#include <sys/prctl.h>
#include <sys/ptrace.h>
#include <linux/seccomp.h>
#include <linux/filter.h>
#include <kj/debug.h>
#ifdef SECCOMP_SET_MODE_FILTER
namespace {
#if 0
// Source code of the seccomp filter:
ld [0] /* offsetof(struct seccomp_data, nr) */
jeq #8, lseek /* __NR_lseek */
jeq #16, inval /* __NR_ioctl */
jeq #40, nosys /* __NR_sendfile */
jeq #257, openat /* __NR_openat */
jeq #285, notsup /* __NR_fallocate */
jeq #316, nosys /* __NR_renameat2 */
jmp good
openat:
ld [32] /* offsetof(struct seccomp_data, args[2]), aka flags */
and #4259840 /* O_TMPFILE */
jeq #4259840, notsup
jmp good
lseek:
ld [32] /* offsetof(struct seccomp_data, args[2]), aka whence */
jeq #3, inval /* SEEK_DATA */
jeq #4, inval /* SEEK_HOLE */
jmp good
inval: ret #0x00050016 /* SECCOMP_RET_ERRNO | EINVAL */
nosys: ret #0x00050026 /* SECCOMP_RET_ERRNO | ENOSYS */
notsup: ret #0x0005005f /* SECCOMP_RET_ERRNO | EOPNOTSUPP */
good: ret #0x7fff0000 /* SECCOMP_RET_ALLOW */
#endif
struct SetupSeccompForFilesystemTest {
SetupSeccompForFilesystemTest() {
struct sock_filter filter[] {
{ 0x20, 0, 0, 0000000000 },
{ 0x15, 10, 0, 0x00000008 },
{ 0x15, 13, 0, 0x00000010 },
{ 0x15, 13, 0, 0x00000028 },
{ 0x15, 3, 0, 0x00000101 },
{ 0x15, 12, 0, 0x0000011d },
{ 0x15, 10, 0, 0x0000013c },
{ 0x05, 0, 0, 0x0000000b },
{ 0x20, 0, 0, 0x00000020 },
{ 0x54, 0, 0, 0x00410000 },
{ 0x15, 7, 0, 0x00410000 },
{ 0x05, 0, 0, 0x00000007 },
{ 0x20, 0, 0, 0x00000020 },
{ 0x15, 2, 0, 0x00000003 },
{ 0x15, 1, 0, 0x00000004 },
{ 0x05, 0, 0, 0x00000003 },
{ 0x06, 0, 0, 0x00050016 },
{ 0x06, 0, 0, 0x00050026 },
{ 0x06, 0, 0, 0x0005005f },
{ 0x06, 0, 0, 0x7fff0000 },
};
struct sock_fprog prog { sizeof(filter) / sizeof(filter[0]), filter };
KJ_SYSCALL(prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0));
KJ_SYSCALL(syscall(__NR_seccomp, SECCOMP_SET_MODE_FILTER, 0, &prog));
}
};
SetupSeccompForFilesystemTest setupSeccompForFilesystemTest;
} // namespace
#define HOLES_NOT_SUPPORTED
// OK, now run all the regular filesystem tests!
#include "filesystem-disk-test.c++"
#endif
#endif
#endif
c++/src/kj/filesystem-disk-test.c++ 0 → 100644
View file @ b1701b88
// Copyright (c) 2016 Sandstorm Development Group, Inc. and contributors
// Licensed under the MIT License:
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
#include "filesystem.h"
#include "test.h"
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <dirent.h>
namespace kj {
namespace {
static Own<File> newTempFile() {
char filename[] = "/var/tmp/kj-filesystem-test.XXXXXX";
int fd;
KJ_SYSCALL(fd = mkstemp(filename));
KJ_DEFER(KJ_SYSCALL(unlink(filename)));
return newDiskFile(AutoCloseFd(fd));
}
class TempDir {
public:
TempDir(): filename(heapString("/var/tmp/kj-filesystem-test.XXXXXX")) {
if (mkdtemp(filename.begin()) == nullptr) {
KJ_FAIL_SYSCALL("mkdtemp", errno, filename);
}
}
Own<Directory> get() {
int fd;
KJ_SYSCALL(fd = open(filename.cStr(), O_RDONLY));
return newDiskDirectory(AutoCloseFd(fd));
}
~TempDir() noexcept(false) {
recursiveDelete(filename);
}
private:
String filename;
static void recursiveDelete(StringPtr path) {
// Recursively delete the temp dir, verifying that no .kj-tmp. files were left over.
{
DIR* dir = opendir(path.cStr());
KJ_ASSERT(dir != nullptr);
KJ_DEFER(closedir(dir));
for (;;) {
auto entry = readdir(dir);
if (entry == nullptr) break;
StringPtr name = entry->d_name;
if (name == "." || name == "..") continue;
auto subPath = kj::str(path, '/', entry->d_name);
KJ_EXPECT(!name.startsWith(".kj-tmp."), "temp file not cleaned up", subPath);
struct stat stats;
KJ_SYSCALL(lstat(subPath.cStr(), &stats));
if (S_ISDIR(stats.st_mode)) {
recursiveDelete(subPath);
} else {
KJ_SYSCALL(unlink(subPath.cStr()));
}
}
}
KJ_SYSCALL(rmdir(path.cStr()));
}
};
KJ_TEST("DiskFile") {
auto file = newTempFile();
KJ_EXPECT(file->readAllText() == "");
file->writeAll("foo");
KJ_EXPECT(file->readAllText() == "foo");
file->write(3, StringPtr("bar").asBytes());
KJ_EXPECT(file->readAllText() == "foobar");
file->write(3, StringPtr("baz").asBytes());
KJ_EXPECT(file->readAllText() == "foobaz");
file->write(9, StringPtr("qux").asBytes());
KJ_EXPECT(file->readAllText() == kj::StringPtr("foobaz\0\0\0qux", 12));
file->truncate(6);
KJ_EXPECT(file->readAllText() == "foobaz");
file->truncate(18);
KJ_EXPECT(file->readAllText() == kj::StringPtr("foobaz\0\0\0\0\0\0\0\0\0\0\0\0", 18));
{
auto mapping = file->mmap(0, 18);
auto privateMapping = file->mmapPrivate(0, 18);
auto writableMapping = file->mmapWritable(0, 18);
KJ_EXPECT(mapping.size() == 18);
KJ_EXPECT(privateMapping.size() == 18);
KJ_EXPECT(writableMapping->get().size() == 18);
KJ_EXPECT(writableMapping->get().begin() != mapping.begin());
KJ_EXPECT(privateMapping.begin() != mapping.begin());
KJ_EXPECT(writableMapping->get().begin() != privateMapping.begin());
KJ_EXPECT(kj::str(mapping.slice(0, 6).asChars()) == "foobaz");
KJ_EXPECT(kj::str(writableMapping->get().slice(0, 6).asChars()) == "foobaz");
KJ_EXPECT(kj::str(privateMapping.slice(0, 6).asChars()) == "foobaz");
privateMapping[0] = 'F';
KJ_EXPECT(kj::str(mapping.slice(0, 6).asChars()) == "foobaz");
KJ_EXPECT(kj::str(writableMapping->get().slice(0, 6).asChars()) == "foobaz");
KJ_EXPECT(kj::str(privateMapping.slice(0, 6).asChars()) == "Foobaz");
writableMapping->get()[0] = 'D';
writableMapping->changed(writableMapping->get().slice(0, 1));
KJ_EXPECT(kj::str(mapping.slice(0, 6).asChars()) == "Doobaz");
KJ_EXPECT(kj::str(writableMapping->get().slice(0, 6).asChars()) == "Doobaz");
KJ_EXPECT(kj::str(privateMapping.slice(0, 6).asChars()) == "Foobaz");
file->write(0, StringPtr("qux").asBytes());
KJ_EXPECT(kj::str(mapping.slice(0, 6).asChars()) == "quxbaz");
KJ_EXPECT(kj::str(writableMapping->get().slice(0, 6).asChars()) == "quxbaz");
KJ_EXPECT(kj::str(privateMapping.slice(0, 6).asChars()) == "Foobaz");
file->write(12, StringPtr("corge").asBytes());
KJ_EXPECT(kj::str(mapping.slice(12, 17).asChars()) == "corge");
// Can shrink.
file->truncate(6);
KJ_EXPECT(kj::str(mapping.slice(12, 17).asChars()) == kj::StringPtr("\0\0\0\0\0", 5));
// Can regrow.
file->truncate(18);
KJ_EXPECT(kj::str(mapping.slice(12, 17).asChars()) == kj::StringPtr("\0\0\0\0\0", 5));
// Can even regrow past previous capacity.
file->truncate(100);
}
file->truncate(6);
KJ_EXPECT(file->readAllText() == "quxbaz");
file->zero(3, 3);
KJ_EXPECT(file->readAllText() == StringPtr("qux\0\0\0", 6));
}
KJ_TEST("DiskFile::copy()") {
auto source = newTempFile();
source->writeAll("foobarbaz");
auto dest = newTempFile();
dest->writeAll("quxcorge");
KJ_EXPECT(dest->copy(3, *source, 6, kj::maxValue) == 3);
KJ_EXPECT(dest->readAllText() == "quxbazge");
KJ_EXPECT(dest->copy(0, *source, 3, 4) == 4);
KJ_EXPECT(dest->readAllText() == "barbazge");
KJ_EXPECT(dest->copy(0, *source, 128, kj::maxValue) == 0);
KJ_EXPECT(dest->copy(4, *source, 3, 0) == 0);
String bigString = strArray(repeat("foobar", 10000), "");
source->truncate(bigString.size() + 1000);
source->write(123, bigString.asBytes());
dest->copy(321, *source, 123, bigString.size());
KJ_EXPECT(dest->readAllText().slice(321) == bigString);
}
KJ_TEST("DiskDirectory") {
TempDir tempDir;
auto dir = tempDir.get();
KJ_EXPECT(dir->listNames() == nullptr);
KJ_EXPECT(dir->listEntries() == nullptr);
KJ_EXPECT(!dir->exists(Path("foo")));
KJ_EXPECT(dir->tryOpenFile(Path("foo")) == nullptr);
KJ_EXPECT(dir->tryOpenFile(Path("foo"), WriteMode::MODIFY) == nullptr);
{
auto file = dir->openFile(Path("foo"), WriteMode::CREATE);
file->writeAll("foobar");
}
KJ_EXPECT(dir->exists(Path("foo")));
{
auto stats = dir->lstat(Path("foo"));
KJ_EXPECT(stats.type == FsNode::Type::FILE);
KJ_EXPECT(stats.size == 6);
}
{
auto list = dir->listNames();
KJ_ASSERT(list.size() == 1);
KJ_EXPECT(list[0] == "foo");
}
{
auto list = dir->listEntries();
KJ_ASSERT(list.size() == 1);
KJ_EXPECT(list[0].name == "foo");
KJ_EXPECT(list[0].type == FsNode::Type::FILE);
}
KJ_EXPECT(dir->openFile(Path("foo"))->readAllText() == "foobar");
KJ_EXPECT(dir->tryOpenFile(Path({"foo", "bar"}), WriteMode::MODIFY) == nullptr);
KJ_EXPECT(dir->tryOpenFile(Path({"bar", "baz"}), WriteMode::MODIFY) == nullptr);
KJ_EXPECT_THROW_MESSAGE("parent is not a directory",
dir->tryOpenFile(Path({"bar", "baz"}), WriteMode::CREATE));
{
auto file = dir->openFile(Path({"bar", "baz"}), WriteMode::CREATE | WriteMode::CREATE_PARENT);
file->writeAll("bazqux");
}
KJ_EXPECT(dir->openFile(Path({"bar", "baz"}))->readAllText() == "bazqux");
{
auto stats = dir->lstat(Path("bar"));
KJ_EXPECT(stats.type == FsNode::Type::DIRECTORY);
}
{
auto list = dir->listNames();
KJ_ASSERT(list.size() == 2);
KJ_EXPECT(list[0] == "bar");
KJ_EXPECT(list[1] == "foo");
}
{
auto list = dir->listEntries();
KJ_ASSERT(list.size() == 2);
KJ_EXPECT(list[0].name == "bar");
KJ_EXPECT(list[0].type == FsNode::Type::DIRECTORY);
KJ_EXPECT(list[1].name == "foo");
KJ_EXPECT(list[1].type == FsNode::Type::FILE);
}
{
auto subdir = dir->openSubdir(Path("bar"));
KJ_EXPECT(subdir->openFile(Path("baz"))->readAllText() == "bazqux");
}
auto subdir = dir->openSubdir(Path("corge"), WriteMode::CREATE);
subdir->openFile(Path("grault"), WriteMode::CREATE)->writeAll("garply");
KJ_EXPECT(dir->openFile(Path({"corge", "grault"}))->readAllText() == "garply");
dir->openFile(Path({"corge", "grault"}), WriteMode::CREATE | WriteMode::MODIFY)
->write(0, StringPtr("rag").asBytes());
KJ_EXPECT(dir->openFile(Path({"corge", "grault"}))->readAllText() == "ragply");
KJ_EXPECT(dir->openSubdir(Path("corge"))->listNames().size() == 1);
{
auto replacer =
dir->replaceFile(Path({"corge", "grault"}), WriteMode::CREATE | WriteMode::MODIFY);
replacer->get().writeAll("rag");
// temp file not in list
KJ_EXPECT(dir->openSubdir(Path("corge"))->listNames().size() == 1);
// Don't commit.
}
KJ_EXPECT(dir->openFile(Path({"corge", "grault"}))->readAllText() == "ragply");
{
auto replacer =
dir->replaceFile(Path({"corge", "grault"}), WriteMode::CREATE | WriteMode::MODIFY);
replacer->get().writeAll("rag");
// temp file not in list
KJ_EXPECT(dir->openSubdir(Path("corge"))->listNames().size() == 1);
replacer->commit();
KJ_EXPECT(dir->openFile(Path({"corge", "grault"}))->readAllText() == "rag");
}
KJ_EXPECT(dir->openFile(Path({"corge", "grault"}))->readAllText() == "rag");
{
auto appender = dir->appendFile(Path({"corge", "grault"}), WriteMode::MODIFY);
appender->write("waldo", 5);
appender->write("fred", 4);
}
KJ_EXPECT(dir->openFile(Path({"corge", "grault"}))->readAllText() == "ragwaldofred");
KJ_EXPECT(dir->exists(Path("foo")));
dir->remove(Path("foo"));
KJ_EXPECT(!dir->exists(Path("foo")));
KJ_EXPECT(!dir->tryRemove(Path("foo")));
KJ_EXPECT(dir->exists(Path({"bar", "baz"})));
dir->remove(Path({"bar", "baz"}));
KJ_EXPECT(!dir->exists(Path({"bar", "baz"})));
KJ_EXPECT(dir->exists(Path("bar")));
KJ_EXPECT(!dir->tryRemove(Path({"bar", "baz"})));
KJ_EXPECT(dir->exists(Path("corge")));
KJ_EXPECT(dir->exists(Path({"corge", "grault"})));
dir->remove(Path("corge"));
KJ_EXPECT(!dir->exists(Path("corge")));
KJ_EXPECT(!dir->exists(Path({"corge", "grault"})));
KJ_EXPECT(!dir->tryRemove(Path("corge")));
}
KJ_TEST("DiskDirectory symlinks") {
TempDir tempDir;
auto dir = tempDir.get();
dir->symlink(Path("foo"), "bar/qux/../baz", WriteMode::CREATE);
KJ_EXPECT(!dir->trySymlink(Path("foo"), "bar/qux/../baz", WriteMode::CREATE));
{
auto stats = dir->lstat(Path("foo"));
KJ_EXPECT(stats.type == FsNode::Type::SYMLINK);
}
KJ_EXPECT(dir->readlink(Path("foo")) == "bar/qux/../baz");
// Broken link into non-existing directory cannot be opened in any mode.
KJ_EXPECT(dir->tryOpenFile(Path("foo")) == nullptr);
KJ_EXPECT(dir->tryOpenFile(Path("foo"), WriteMode::CREATE) == nullptr);
KJ_EXPECT(dir->tryOpenFile(Path("foo"), WriteMode::MODIFY) == nullptr);
KJ_EXPECT_THROW_MESSAGE("parent is not a directory",
dir->tryOpenFile(Path("foo"), WriteMode::CREATE | WriteMode::MODIFY));
KJ_EXPECT_THROW_MESSAGE("parent is not a directory",
dir->tryOpenFile(Path("foo"),
WriteMode::CREATE | WriteMode::MODIFY | WriteMode::CREATE_PARENT));
// Create the directory.
auto subdir = dir->openSubdir(Path("bar"), WriteMode::CREATE);
subdir->openSubdir(Path("qux"), WriteMode::CREATE);
// Link still points to non-existing file so cannot be open in most modes.
KJ_EXPECT(dir->tryOpenFile(Path("foo")) == nullptr);
KJ_EXPECT(dir->tryOpenFile(Path("foo"), WriteMode::CREATE) == nullptr);
KJ_EXPECT(dir->tryOpenFile(Path("foo"), WriteMode::MODIFY) == nullptr);
// But... CREATE | MODIFY works.
dir->openFile(Path("foo"), WriteMode::CREATE | WriteMode::MODIFY)
->writeAll("foobar");
KJ_EXPECT(dir->openFile(Path({"bar", "baz"}))->readAllText() == "foobar");
KJ_EXPECT(dir->openFile(Path("foo"))->readAllText() == "foobar");
KJ_EXPECT(dir->openFile(Path("foo"), WriteMode::MODIFY)->readAllText() == "foobar");
// operations that modify the symlink
dir->symlink(Path("foo"), "corge", WriteMode::MODIFY);
KJ_EXPECT(dir->openFile(Path({"bar", "baz"}))->readAllText() == "foobar");
KJ_EXPECT(dir->readlink(Path("foo")) == "corge");
KJ_EXPECT(!dir->exists(Path("foo")));
KJ_EXPECT(dir->lstat(Path("foo")).type == FsNode::Type::SYMLINK);
KJ_EXPECT(dir->tryOpenFile(Path("foo")) == nullptr);
dir->remove(Path("foo"));
KJ_EXPECT(!dir->exists(Path("foo")));
KJ_EXPECT(dir->tryOpenFile(Path("foo")) == nullptr);
}
KJ_TEST("DiskDirectory link") {
TempDir tempDirSrc;
TempDir tempDirDst;
auto src = tempDirSrc.get();
auto dst = tempDirDst.get();
src->openFile(Path("foo"), WriteMode::CREATE | WriteMode::CREATE_PARENT)
->writeAll("foobar");
dst->transfer(Path("link"), WriteMode::CREATE, *src, Path("foo"), TransferMode::LINK);
KJ_EXPECT(dst->openFile(Path("link"))->readAllText() == "foobar");
// Writing the old location modifies the new.
src->openFile(Path("foo"), WriteMode::MODIFY)->writeAll("bazqux");
KJ_EXPECT(dst->openFile(Path("link"))->readAllText() == "bazqux");
// Replacing the old location doesn't modify the new.
{
auto replacer = src->replaceFile(Path("foo"), WriteMode::MODIFY);
replacer->get().writeAll("corge");
replacer->commit();
}
KJ_EXPECT(src->openFile(Path("foo"))->readAllText() == "corge");
KJ_EXPECT(dst->openFile(Path("link"))->readAllText() == "bazqux");
}
KJ_TEST("DiskDirectory copy") {
TempDir tempDirSrc;
TempDir tempDirDst;
auto src = tempDirSrc.get();
auto dst = tempDirDst.get();
src->openFile(Path({"foo", "bar"}), WriteMode::CREATE | WriteMode::CREATE_PARENT)
->writeAll("foobar");
src->openFile(Path({"foo", "baz", "qux"}), WriteMode::CREATE | WriteMode::CREATE_PARENT)
->writeAll("bazqux");
dst->transfer(Path("link"), WriteMode::CREATE, *src, Path("foo"), TransferMode::COPY);
KJ_EXPECT(src->openFile(Path({"foo", "bar"}))->readAllText() == "foobar");
KJ_EXPECT(src->openFile(Path({"foo", "baz", "qux"}))->readAllText() == "bazqux");
KJ_EXPECT(dst->openFile(Path({"link", "bar"}))->readAllText() == "foobar");
KJ_EXPECT(dst->openFile(Path({"link", "baz", "qux"}))->readAllText() == "bazqux");
KJ_EXPECT(dst->exists(Path({"link", "bar"})));
src->remove(Path({"foo", "bar"}));
KJ_EXPECT(dst->openFile(Path({"link", "bar"}))->readAllText() == "foobar");
}
KJ_TEST("DiskDirectory move") {
TempDir tempDirSrc;
TempDir tempDirDst;
auto src = tempDirSrc.get();
auto dst = tempDirDst.get();
src->openFile(Path({"foo", "bar"}), WriteMode::CREATE | WriteMode::CREATE_PARENT)
->writeAll("foobar");
src->openFile(Path({"foo", "baz", "qux"}), WriteMode::CREATE | WriteMode::CREATE_PARENT)
->writeAll("bazqux");
dst->transfer(Path("link"), WriteMode::CREATE, *src, Path("foo"), TransferMode::MOVE);
KJ_EXPECT(!src->exists(Path({"foo"})));
KJ_EXPECT(dst->openFile(Path({"link", "bar"}))->readAllText() == "foobar");
KJ_EXPECT(dst->openFile(Path({"link", "baz", "qux"}))->readAllText() == "bazqux");
}
KJ_TEST("DiskDirectory createTemporary") {
TempDir tempDir;
auto dir = tempDir.get();
auto file = dir->createTemporary();
file->writeAll("foobar");
KJ_EXPECT(file->readAllText() == "foobar");
KJ_EXPECT(dir->listNames() == nullptr);
}
KJ_TEST("DiskDirectory replace directory with file") {
TempDir tempDir;
auto dir = tempDir.get();
dir->openFile(Path({"foo", "bar"}), WriteMode::CREATE | WriteMode::CREATE_PARENT)
->writeAll("foobar");
{
// CREATE fails -- already exists.
auto replacer = dir->replaceFile(Path("foo"), WriteMode::CREATE);
replacer->get().writeAll("bazqux");
KJ_EXPECT(!replacer->tryCommit());
}
// Still a directory.
KJ_EXPECT(dir->lstat(Path("foo")).type == FsNode::Type::DIRECTORY);
{
// MODIFY succeeds.
auto replacer = dir->replaceFile(Path("foo"), WriteMode::MODIFY);
replacer->get().writeAll("bazqux");
replacer->commit();
}
// Replaced with file.
KJ_EXPECT(dir->openFile(Path("foo"))->readAllText() == "bazqux");
}
KJ_TEST("DiskDirectory replace file with directory") {
TempDir tempDir;
auto dir = tempDir.get();
dir->openFile(Path("foo"), WriteMode::CREATE)
->writeAll("foobar");
{
// CREATE fails -- already exists.
auto replacer = dir->replaceSubdir(Path("foo"), WriteMode::CREATE);
replacer->get().openFile(Path("bar"), WriteMode::CREATE)->writeAll("bazqux");
KJ_EXPECT(dir->listNames().size() == 1 && dir->listNames()[0] == "foo");
KJ_EXPECT(!replacer->tryCommit());
}
// Still a file.
KJ_EXPECT(dir->openFile(Path("foo"))->readAllText() == "foobar");
{
// MODIFY succeeds.
auto replacer = dir->replaceSubdir(Path("foo"), WriteMode::MODIFY);
replacer->get().openFile(Path("bar"), WriteMode::CREATE)->writeAll("bazqux");
KJ_EXPECT(dir->listNames().size() == 1 && dir->listNames()[0] == "foo");
replacer->commit();
}
// Replaced with directory.
KJ_EXPECT(dir->openFile(Path({"foo", "bar"}))->readAllText() == "bazqux");
}
#ifndef HOLES_NOT_SUPPORTED
KJ_TEST("DiskFile holes") {
TempDir tempDir;
auto dir = tempDir.get();
auto file = dir->openFile(Path("holes"), WriteMode::CREATE);
file->writeAll("foobar");
file->write(1 << 20, StringPtr("foobar").asBytes());
// Allow for block sizes as low as 512 bytes and as high as 64k.
auto meta = file->stat();
KJ_EXPECT(meta.spaceUsed >= 2 * 512);
KJ_EXPECT(meta.spaceUsed <= 2 * 65536);
byte buf[7];
{
// Copy doesn't fill in holes.
dir->transfer(Path("copy"), WriteMode::CREATE, Path("holes"), TransferMode::COPY);
auto copy = dir->openFile(Path("copy"));
KJ_EXPECT(copy->stat().spaceUsed == meta.spaceUsed);
KJ_EXPECT(copy->read(0, buf) == 7);
KJ_EXPECT(StringPtr(reinterpret_cast<char*>(buf), 6) == "foobar");
KJ_EXPECT(copy->read(1 << 20, buf) == 6);
KJ_EXPECT(StringPtr(reinterpret_cast<char*>(buf), 6) == "foobar");
KJ_EXPECT(copy->read(1 << 19, buf) == 7);
KJ_EXPECT(StringPtr(reinterpret_cast<char*>(buf), 6) == StringPtr("\0\0\0\0\0\0", 6));
}
file->truncate(1 << 21);
KJ_EXPECT(file->stat().spaceUsed == meta.spaceUsed);
KJ_EXPECT(file->read(1 << 20, buf) == 7);
KJ_EXPECT(StringPtr(reinterpret_cast<char*>(buf), 6) == "foobar");
{
dir->transfer(Path("copy"), WriteMode::MODIFY, Path("holes"), TransferMode::COPY);
auto copy = dir->openFile(Path("copy"));
KJ_EXPECT(copy->stat().spaceUsed == meta.spaceUsed);
KJ_EXPECT(copy->read(0, buf) == 7);
KJ_EXPECT(StringPtr(reinterpret_cast<char*>(buf), 6) == "foobar");
KJ_EXPECT(copy->read(1 << 20, buf) == 7);
KJ_EXPECT(StringPtr(reinterpret_cast<char*>(buf), 6) == "foobar");
KJ_EXPECT(copy->read(1 << 19, buf) == 7);
KJ_EXPECT(StringPtr(reinterpret_cast<char*>(buf), 6) == StringPtr("\0\0\0\0\0\0", 6));
}
}
#endif
} // namespace
} // namespace kj
c++/src/kj/filesystem-disk.c++ 0 → 100644
View file @ b1701b88
// Copyright (c) 2015 Sandstorm Development Group, Inc. and contributors
// Licensed under the MIT License:
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
#include "filesystem.h"
#include "debug.h"
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/ioctl.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdio.h>
#include <sys/mman.h>
#include <errno.h>
#include <dirent.h>
#include <syscall.h>
#include <stdlib.h>
#include "vector.h"
#include "miniposix.h"
#if __linux__
#include <linux/fs.h>
#include <sys/sendfile.h>
#endif
namespace kj {
namespace {
#define HIDDEN_PREFIX ".kj-tmp."
// Prefix for temp files which should be hidden when listing a directory.
//
// If you change this, make sure to update the unit test.
#ifdef O_CLOEXEC
#define MAYBE_O_CLOEXEC O_CLOEXEC
#else
#define MAYBE_O_CLOEXEC 0
#endif
#ifdef O_DIRECTORY
#define MAYBE_O_DIRECTORY O_DIRECTORY
#else
#define MAYBE_O_DIRECTORY 0
#endif
static void setCloexec(int fd) KJ_UNUSED;
static void setCloexec(int fd) {
// Set the O_CLOEXEC flag on the given fd.
//
// We try to avoid the need to call this by taking advantage of syscall flags that set it
// atomically on new file descriptors. Unfortunately some platforms do not support such syscalls.
#ifdef FIOCLEX
// Yay, we can set the flag in one call.
KJ_SYSCALL_HANDLE_ERRORS(ioctl(fd, FIOCLEX)) {
case EINVAL:
case EOPNOTSUPP:
break;
default:
KJ_FAIL_SYSCALL("ioctl(fd, FIOCLEX)", error) { break; }
break;
} else {
// success
return;
}
#endif
// Sadness, we must resort to read/modify/write.
//
// (On many platforms, FD_CLOEXEC is the only flag modifiable via F_SETFD and therefore we could
// skip the read... but it seems dangerous to assume that's true of all platforms, and anyway
// most platforms support FIOCLEX.)
int flags;
KJ_SYSCALL(flags = fcntl(fd, F_GETFD));
if (!(flags & FD_CLOEXEC)) {
KJ_SYSCALL(fcntl(fd, F_SETFD, flags | FD_CLOEXEC));
}
}
static Date toKjDate(struct timespec tv) {
return tv.tv_sec * SECONDS + tv.tv_nsec * NANOSECONDS + UNIX_EPOCH;
}
static FsNode::Type modeToType(mode_t mode) {
switch (mode & S_IFMT) {
case S_IFREG : return FsNode::Type::FILE;
case S_IFDIR : return FsNode::Type::DIRECTORY;
case S_IFLNK : return FsNode::Type::SYMLINK;
case S_IFBLK : return FsNode::Type::BLOCK_DEVICE;
case S_IFCHR : return FsNode::Type::CHARACTER_DEVICE;
case S_IFIFO : return FsNode::Type::NAMED_PIPE;
case S_IFSOCK: return FsNode::Type::SOCKET;
default: return FsNode::Type::OTHER;
}
}
static FsNode::Metadata statToMetadata(struct stat& stats) {
return FsNode::Metadata {
modeToType(stats.st_mode),
implicitCast<uint64_t>(stats.st_size),
implicitCast<uint64_t>(stats.st_blocks * 512u),
toKjDate(stats.st_mtim),
implicitCast<uint>(stats.st_nlink)
};
}
static bool rmrf(int fd, StringPtr path);
static void rmrfChildrenAndClose(int fd) {
// Assumes fd is seeked to beginning.
DIR* dir = fdopendir(fd);
if (dir == nullptr) {
close(fd);
KJ_FAIL_SYSCALL("fdopendir", errno);
};
KJ_DEFER(closedir(dir));
for (;;) {
errno = 0;
struct dirent* entry = readdir(dir);
if (entry == nullptr) {
int error = errno;
if (error == 0) {
break;
} else {
KJ_FAIL_SYSCALL("readdir", error);
}
}
if (entry->d_name[0] == '.' &&
(entry->d_name[1] == '\0' ||
(entry->d_name[1] == '.' &&
entry->d_name[2] == '\0'))) {
// ignore . and ..
} else {
#ifdef DT_UNKNOWN // d_type is not available on all platforms.
if (entry->d_type == DT_DIR) {
int subdirFd;
KJ_SYSCALL(subdirFd = openat(
fd, entry->d_name, O_RDONLY | MAYBE_O_DIRECTORY | MAYBE_O_CLOEXEC));
rmrfChildrenAndClose(subdirFd);
KJ_SYSCALL(unlinkat(fd, entry->d_name, AT_REMOVEDIR));
} else if (entry->d_type != DT_UNKNOWN) {
KJ_SYSCALL(unlinkat(fd, entry->d_name, 0));
} else {
#endif
KJ_ASSERT(rmrf(fd, entry->d_name));
#ifdef DT_UNKNOWN
}
#endif
}
}
}
static bool rmrf(int fd, StringPtr path) {
struct stat stats;
KJ_SYSCALL_HANDLE_ERRORS(fstatat(fd, path.cStr(), &stats, AT_SYMLINK_NOFOLLOW)) {
case ENOENT:
case ENOTDIR:
// Doesn't exist.
return false;
default:
KJ_FAIL_SYSCALL("lstat(path)", error, path) { return false; }
}
if (S_ISDIR(stats.st_mode)) {
int subdirFd;
KJ_SYSCALL(subdirFd = openat(
fd, path.cStr(), O_RDONLY | MAYBE_O_DIRECTORY | MAYBE_O_CLOEXEC)) { return false; }
rmrfChildrenAndClose(subdirFd);
KJ_SYSCALL(unlinkat(fd, path.cStr(), AT_REMOVEDIR)) { return false; }
} else {
KJ_SYSCALL(unlinkat(fd, path.cStr(), 0)) { return false; }
}
return true;
}
struct MmapRange {
uint64_t offset;
uint64_t size;
};
static MmapRange getMmapRange(uint64_t offset, uint64_t size) {
// Rounds the given byte range up to page boundaries.
#ifndef _SC_PAGESIZE
#define _SC_PAGESIZE _SC_PAGE_SIZE
#endif
static const uint64_t pageSize = sysconf(_SC_PAGESIZE);
uint64_t pageMask = pageSize - 1;
uint64_t realOffset = offset & ~pageMask;
uint64_t end = offset + size;
uint64_t realEnd = (end + pageMask) & ~pageMask;
return { realOffset, realEnd - realOffset };
}
class MmapDisposer: public ArrayDisposer {
protected:
void disposeImpl(void* firstElement, size_t elementSize, size_t elementCount,
size_t capacity, void (*destroyElement)(void*)) const {
auto range = getMmapRange(reinterpret_cast<uintptr_t>(firstElement),
elementSize * elementCount);
#if _WIN32
KJ_ASSERT(UnmapViewOfFile(reinterpret_cast<byte*>(range.offset))) { break; }
#else
KJ_SYSCALL(munmap(reinterpret_cast<byte*>(range.offset), range.size)) { break; }
#endif
}
};
constexpr MmapDisposer mmapDisposer = MmapDisposer();
class DiskHandle {
// We need to implement each of ReadableFile, AppendableFile, File, ReadableDirectory, and
// Directory for disk handles. There is a lot of implementation overlap between these, especially
// stat(), sync(), etc. We can't have everything inherit from a common DiskFsNode that implements
// these because then we get diamond inheritance which means we need to make all our inheritance
// virtual which means downcasting requires RTTI which violates our goal of supporting compiling
// with no RTTI. So instead we have the DiskHandle class which implements all the methods without
// inheriting anything, and then we have DiskFile, DiskDirectory, etc. hold this and delegate to
// it. Ugly, but works.
public:
DiskHandle(AutoCloseFd&& fd): fd(kj::mv(fd)) {}
// OsHandle ------------------------------------------------------------------
AutoCloseFd clone() {
int fd2;
#ifdef F_DUPFD_CLOEXEC
KJ_SYSCALL_HANDLE_ERRORS(fd2 = fcntl(fd, F_DUPFD_CLOEXEC, 3)) {
case EINVAL:
case EOPNOTSUPP:
// fall back
break;
default:
KJ_FAIL_SYSCALL("fnctl(fd, F_DUPFD_CLOEXEC, 3)", error) { break; }
break;
} else {
return AutoCloseFd(fd2);
}
#endif
KJ_SYSCALL(fd2 = ::dup(fd));
AutoCloseFd result(fd2);
setCloexec(result);
return result;
}
int getFd() {
return fd.get();
}
// FsNode --------------------------------------------------------------------
FsNode::Metadata stat() {
struct stat stats;
KJ_SYSCALL(::fstat(fd, &stats));
return statToMetadata(stats);
}
void sync() { KJ_SYSCALL(fsync(fd)); }
void datasync() { KJ_SYSCALL(fdatasync(fd)); }
// ReadableFile --------------------------------------------------------------
size_t read(uint64_t offset, ArrayPtr<byte> buffer) {
// pread() probably never returns short reads unless it hits EOF. Unfortunately, though, per
// spec we are not allowed to assume this.
size_t total = 0;
while (buffer.size() > 0) {
ssize_t n;
KJ_SYSCALL(n = pread(fd, buffer.begin(), buffer.size(), offset));
if (n == 0) break;
total += n;
offset += n;
buffer = buffer.slice(n, buffer.size());
}
return total;
}
Array<const byte> mmap(uint64_t offset, uint64_t size) {
auto range = getMmapRange(offset, size);
const void* mapping = ::mmap(NULL, range.size, PROT_READ, MAP_SHARED, fd, range.offset);
if (mapping == MAP_FAILED) {
KJ_FAIL_SYSCALL("mmap", errno);
}
return Array<const byte>(reinterpret_cast<const byte*>(mapping) + (offset - range.offset),
size, mmapDisposer);
}
Array<byte> mmapPrivate(uint64_t offset, uint64_t size) {
auto range = getMmapRange(offset, size);
void* mapping = ::mmap(NULL, range.size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, range.offset);
if (mapping == MAP_FAILED) {
KJ_FAIL_SYSCALL("mmap", errno);
}
return Array<byte>(reinterpret_cast<byte*>(mapping) + (offset - range.offset),
size, mmapDisposer);
}
// File ----------------------------------------------------------------------
void write(uint64_t offset, ArrayPtr<const byte> data) {
// pwrite() probably never returns short writes unless there's no space left on disk.
// Unfortunately, though, per spec we are not allowed to assume this.
while (data.size() > 0) {
ssize_t n;
KJ_SYSCALL(n = pwrite(fd, data.begin(), data.size(), offset));
KJ_ASSERT(n > 0, "pwrite() returned zero?");
offset += n;
data = data.slice(n, data.size());
}
}
void zero(uint64_t offset, uint64_t size) {
#ifdef FALLOC_FL_PUNCH_HOLE
KJ_SYSCALL_HANDLE_ERRORS(
fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, offset, size)) {
case EOPNOTSUPP:
// fall back to below
break;
default:
KJ_FAIL_SYSCALL("fallocate(FALLOC_FL_PUNCH_HOLE)", error) { return; }
} else {
return;
}
#endif
// Use a 4k buffer of zeros amplified by iov to write zeros with as few syscalls as possible.
byte buffer[4096];
memset(buffer, 0, sizeof(buffer));
size_t count = (size + sizeof(buffer) - 1) / sizeof(buffer);
const size_t iovmax = miniposix::iovMax(count);
KJ_STACK_ARRAY(struct iovec, iov, kj::min(iovmax, count), 16, 256);
for (auto& item: iov) {
item.iov_base = buffer;
item.iov_len = sizeof(buffer);
}
while (size > 0) {
size_t iovCount;
if (size >= iov.size() * sizeof(buffer)) {
iovCount = iov.size();
} else {
iovCount = size / sizeof(buffer);
size_t rem = size % sizeof(buffer);
if (rem > 0) {
iov[iovCount++].iov_len = rem;
}
}
ssize_t n;
KJ_SYSCALL(n = pwritev(fd, iov.begin(), count, offset));
KJ_ASSERT(n > 0, "pwrite() returned zero?");
offset += n;
size -= n;
}
}
void truncate(uint64_t size) {
KJ_SYSCALL(ftruncate(fd, size));
}
class WritableFileMappingImpl: public WritableFileMapping {
public:
WritableFileMappingImpl(Array<byte> bytes): bytes(kj::mv(bytes)) {}
ArrayPtr<byte> get() override {
return bytes;
}
void changed(ArrayPtr<byte> slice) override {
KJ_REQUIRE(slice.begin() >= bytes.begin() && slice.end() <= bytes.end(),
"byte range is not part of this mapping");
KJ_SYSCALL(msync(slice.begin(), slice.size(), MS_ASYNC));
}
void sync(ArrayPtr<byte> slice) override {
KJ_REQUIRE(slice.begin() >= bytes.begin() && slice.end() <= bytes.end(),
"byte range is not part of this mapping");
KJ_SYSCALL(msync(slice.begin(), slice.size(), MS_SYNC));
}
private:
Array<byte> bytes;
};
Own<WritableFileMapping> mmapWritable(uint64_t offset, uint64_t size) {
auto range = getMmapRange(offset, size);
void* mapping = ::mmap(NULL, range.size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, range.offset);
if (mapping == MAP_FAILED) {
KJ_FAIL_SYSCALL("mmap", errno);
}
auto array = Array<byte>(reinterpret_cast<byte*>(mapping) + (offset - range.offset),
size, mmapDisposer);
return heap<WritableFileMappingImpl>(kj::mv(array));
}
size_t copyChunk(uint64_t offset, int fromFd, uint64_t fromOffset, uint64_t size) {
// Copies a range of bytes from `fromFd` to this file in the most efficient way possible for
// the OS. Only returns less than `size` if EOF. Does not account for holes.
#if __linux__
{
KJ_SYSCALL(lseek(fd, offset, SEEK_SET));
off_t fromPos = fromOffset;
off_t end = fromOffset + size;
while (fromPos < end) {
ssize_t n;
KJ_SYSCALL_HANDLE_ERRORS(n = sendfile(fd, fromFd, &fromPos, end - fromPos)) {
case EINVAL:
case ENOSYS:
goto sendfileNotAvailable;
default:
KJ_FAIL_SYSCALL("sendfile", error) { return fromPos - fromOffset; }
}
}
return fromPos - fromOffset;
}
sendfileNotAvailable:
#endif
uint64_t total = 0;
while (size > 0) {
byte buffer[4096];
ssize_t n;
KJ_SYSCALL(n = pread(fromFd, buffer, kj::min(sizeof(buffer), size), fromOffset));
if (n == 0) break;
write(offset, arrayPtr(buffer, n));
fromOffset += n;
offset += n;
total += n;
size -= n;
}
return total;
}
kj::Maybe<size_t> copy(uint64_t offset, ReadableFile& from, uint64_t fromOffset, uint64_t size) {
KJ_IF_MAYBE(otherFd, from.getFd()) {
#ifdef FICLONE
if (offset == 0 && fromOffset == 0 && size == kj::maxValue && stat().size == 0) {
if (ioctl(fd, FICLONE, *otherFd) >= 0) {
return stat().size;
}
} else if (size > 0) { // src_length = 0 has special meaning for the syscall, so avoid.
struct file_clone_range range;
memset(&range, 0, sizeof(range));
range.src_fd = *otherFd;
range.dest_offset = offset;
range.src_offset = fromOffset;
range.src_length = size == kj::maxValue ? 0 : size;
if (ioctl(fd, FICLONERANGE, &range) >= 0) {
// TODO(someday): What does FICLONERANGE actually do if the range goes past EOF? The docs
// don't say. Maybe it only copies the parts that exist. Maybe it punches holes for the
// rest. Where does the destination file's EOF marker end up? Who knows?
return kj::min(from.stat().size - fromOffset, size);
}
} else {
// size == 0
return size_t(0);
}
// ioctl failed. Almost all failures documented for these are of the form "the operation is
// not supported for the filesystem(s) specified", so fall back to other approaches.
#endif
off_t toPos = offset;
off_t fromPos = fromOffset;
off_t end = size == kj::maxValue ? off_t(kj::maxValue) : off_t(fromOffset + size);
for (;;) {
// Handle data.
{
// Find out how much data there is before the next hole.
off_t nextHole;
#ifdef SEEK_HOLE
KJ_SYSCALL_HANDLE_ERRORS(nextHole = lseek(*otherFd, fromPos, SEEK_HOLE)) {
case EINVAL:
// SEEK_HOLE probably not supported. Assume no holes.
nextHole = end;
break;
case ENXIO:
// Past EOF. Stop here.
return fromPos - fromOffset;
default:
KJ_FAIL_SYSCALL("lseek(fd, pos, SEEK_HOLE)", error) { return fromPos - fromOffset; }
}
#else
// SEEK_HOLE not supported. Assume no holes.
nextHole = end;
#endif
// Copy the next chunk of data.
off_t copyTo = kj::min(end, nextHole);
size_t amount = copyTo - fromPos;
if (amount > 0) {
size_t n = copyChunk(toPos, *otherFd, fromPos, amount);
fromPos += n;
toPos += n;
if (n < amount) {
return fromPos - fromOffset;
}
}
if (fromPos == end) {
return fromPos - fromOffset;
}
}
#ifdef SEEK_HOLE
// Handle hole.
{
// Find out how much hole there is before the next data.
off_t nextData;
KJ_SYSCALL_HANDLE_ERRORS(nextData = lseek(*otherFd, fromPos, SEEK_DATA)) {
case EINVAL:
// SEEK_DATA probably not supported. But we should only have gotten here if we
// were expecting a hole.
KJ_FAIL_ASSERT("can't determine hole size; SEEK_DATA not supported");
break;
case ENXIO:
// No more data. Set to EOF.
KJ_SYSCALL(nextData = lseek(*otherFd, 0, SEEK_END));
if (nextData > end) {
end = nextData;
}
break;
default:
KJ_FAIL_SYSCALL("lseek(fd, pos, SEEK_HOLE)", error) { return fromPos - fromOffset; }
}
// Write zeros.
off_t zeroTo = kj::min(end, nextData);
off_t amount = zeroTo - fromPos;
if (amount > 0) {
zero(toPos, amount);
toPos += amount;
fromPos = zeroTo;
}
if (fromPos == end) {
return fromPos - fromOffset;
}
}
#endif
}
}
// Indicates caller should call File::copy() default implementation.
return nullptr;
}
// ReadableDirectory ---------------------------------------------------------
template <typename Func>
auto list(bool needTypes, Func&& func)
-> Array<Decay<decltype(func(instance<StringPtr>(), instance<FsNode::Type>()))>> {
// Seek to start of directory.
KJ_SYSCALL(lseek(fd, 0, SEEK_SET));
// Unfortunately, fdopendir() takes ownership of the file descriptor. Therefore we need to
// make a duplicate.
int duped;
KJ_SYSCALL(duped = dup(fd));
DIR* dir = fdopendir(duped);
if (dir == nullptr) {
close(duped);
KJ_FAIL_SYSCALL("fdopendir", errno);
}
KJ_DEFER(closedir(dir));
kj::Vector<Decay<decltype(func(instance<StringPtr>(), instance<FsNode::Type>()))>> entries;
for (;;) {
errno = 0;
struct dirent* entry = readdir(dir);
if (entry == nullptr) {
int error = errno;
if (error == 0) {
break;
} else {
KJ_FAIL_SYSCALL("readdir", error);
}
}
kj::StringPtr name = entry->d_name;
if (name != "." && name != ".." && !name.startsWith(HIDDEN_PREFIX)) {
#ifdef DT_UNKNOWN // d_type is not available on all platforms.
if (entry->d_type != DT_UNKNOWN) {
entries.add(func(name, modeToType(DTTOIF(entry->d_type))));
} else {
#endif
if (needTypes) {
// Unknown type. Fall back to stat.
struct stat stats;
KJ_SYSCALL(fstatat(fd, name.cStr(), &stats, AT_SYMLINK_NOFOLLOW));
entries.add(func(name, modeToType(stats.st_mode)));
} else {
entries.add(func(name, FsNode::Type::OTHER));
}
#ifdef DT_UNKNOWN
}
#endif
}
}
return entries.releaseAsArray();
}
Array<String> listNames() {
return list(false, [](StringPtr name, FsNode::Type type) { return heapString(name); });
}
Array<ReadableDirectory::Entry> listEntries() {
return list(true, [](StringPtr name, FsNode::Type type) {
return ReadableDirectory::Entry { type, heapString(name), };
});
}
bool exists(PathPtr path) {
KJ_SYSCALL_HANDLE_ERRORS(faccessat(fd, path.toString().cStr(), F_OK, 0)) {
case ENOENT:
case ENOTDIR:
return false;
default:
KJ_FAIL_SYSCALL("faccessat(fd, path)", error, path) { return false; }
}
return true;
}
Maybe<FsNode::Metadata> tryLstat(PathPtr path) {
struct stat stats;
KJ_SYSCALL_HANDLE_ERRORS(fstatat(fd, path.toString().cStr(), &stats, AT_SYMLINK_NOFOLLOW)) {
case ENOENT:
case ENOTDIR:
return nullptr;
default:
KJ_FAIL_SYSCALL("faccessat(fd, path)", error, path) { return nullptr; }
}
return statToMetadata(stats);
}
Maybe<Own<ReadableFile>> tryOpenFile(PathPtr path) {
int newFd;
KJ_SYSCALL_HANDLE_ERRORS(newFd = openat(
fd, path.toString().cStr(), O_RDONLY | MAYBE_O_CLOEXEC)) {
case ENOENT:
case ENOTDIR:
return nullptr;
default:
KJ_FAIL_SYSCALL("openat(fd, path, O_RDONLY)", error, path) { return nullptr; }
}
kj::AutoCloseFd result(newFd);
#ifndef O_CLOEXEC
setCloexec(result);
#endif
return newDiskReadableFile(kj::mv(result));
}
Maybe<AutoCloseFd> tryOpenSubdirInternal(PathPtr path) {
int newFd;
KJ_SYSCALL_HANDLE_ERRORS(newFd = openat(
fd, path.toString().cStr(), O_RDONLY | MAYBE_O_CLOEXEC | MAYBE_O_DIRECTORY)) {
case ENOENT:
return nullptr;
case ENOTDIR:
// Could mean that a parent is not a directory, which we treat as "doesn't exist".
// Could also mean that the specified file is not a directory, which should throw.
// Check using exists().
if (!exists(path)) {
return nullptr;
}
// fallthrough
default:
KJ_FAIL_SYSCALL("openat(fd, path, O_DIRECTORY)", error, path) { return nullptr; }
}
kj::AutoCloseFd result(newFd);
#ifndef O_CLOEXEC
setCloexec(result);
#endif
return kj::mv(result);
}
Maybe<Own<ReadableDirectory>> tryOpenSubdir(PathPtr path) {
return tryOpenSubdirInternal(path).map(newDiskReadableDirectory);
}
Maybe<String> tryReadlink(PathPtr path) {
size_t trySize = 256;
for (;;) {
KJ_STACK_ARRAY(char, buf, trySize, 256, 4096);
ssize_t n = readlinkat(fd, path.toString().cStr(), buf.begin(), buf.size());
if (n < 0) {
int error = errno;
switch (error) {
case EINTR:
continue;
case ENOENT:
case ENOTDIR:
case EINVAL: // not a link
return nullptr;
default:
KJ_FAIL_SYSCALL("readlinkat(fd, path)", error, path) { return nullptr; }
}
}
if (n >= buf.size()) {
// Didn't give it enough space. Better retry with a bigger buffer.
trySize *= 2;
continue;
}
return heapString(buf.begin(), n);
}
}
// Directory -----------------------------------------------------------------
bool tryMkdir(PathPtr path, WriteMode mode, bool noThrow) {
// Internal function to make a directory.
auto filename = path.toString();
mode_t acl = has(mode, WriteMode::PRIVATE) ? 0700 : 0777;
KJ_SYSCALL_HANDLE_ERRORS(mkdirat(fd, filename.cStr(), acl)) {
case EEXIST: {
// Apparently this path exists.
if (!has(mode, WriteMode::MODIFY)) {
// Require exclusive create.
return false;
}
// MODIFY is allowed, so we just need to check whether the existing entry is a directory.
struct stat stats;
KJ_SYSCALL_HANDLE_ERRORS(fstatat(fd, filename.cStr(), &stats, 0)) {
default:
// mkdir() says EEXIST but we can't stat it. Maybe it's a dangling link, or maybe
// we can't access it for some reason. Assume failure.
//
// TODO(someday): Maybe we should be creating the directory at the target of the
// link?
goto failed;
}
return (stats.st_mode & S_IFMT) == S_IFDIR;
}
case ENOENT:
if (has(mode, WriteMode::CREATE_PARENT) && path.size() > 0 &&
tryMkdir(path.parent(), WriteMode::CREATE | WriteMode::MODIFY |
WriteMode::CREATE_PARENT, true)) {
// Retry, but make sure we don't try to create the parent again.
return tryMkdir(path, mode - WriteMode::CREATE_PARENT, noThrow);
} else {
goto failed;
}
default:
failed:
if (noThrow) {
// Caller requested no throwing.
return false;
} else {
KJ_FAIL_SYSCALL("mkdirat(fd, path)", error, path);
}
}
return true;
}
kj::Maybe<String> createNamedTemporary(
PathPtr finalName, WriteMode mode, Function<int(StringPtr)> tryCreate) {
// Create a temporary file which will eventually replace `finalName`.
//
// Calls `tryCreate` to actually create the temporary, passing in the desired path. tryCreate()
// is expected to behave like a syscall, returning a negative value and setting `errno` on
// error. tryCreate() MUST fail with EEXIST if the path exists -- this is not checked in
// advance, since it needs to be checked atomically. In the case of EEXIST, tryCreate() will
// be called again with a new path.
//
// Returns the temporary path that succeeded. Only returns nullptr if there was an exception
// but we're complied with -fno-exceptions.
if (finalName.size() == 0) {
KJ_FAIL_REQUIRE("can't replace self") { break; }
return nullptr;
}
static uint counter = 0;
static const pid_t pid = getpid();
String pathPrefix;
if (finalName.size() > 1) {
pathPrefix = kj::str(finalName.parent(), '/');
}
auto path = kj::str(pathPrefix, HIDDEN_PREFIX, pid, '.', counter++, '.',
finalName.basename()[0], ".partial");
KJ_SYSCALL_HANDLE_ERRORS(tryCreate(path)) {
case EEXIST:
return createNamedTemporary(finalName, mode, kj::mv(tryCreate));
case ENOENT:
if (has(mode, WriteMode::CREATE_PARENT) && finalName.size() > 1 &&
tryMkdir(finalName.parent(), WriteMode::CREATE | WriteMode::MODIFY |
WriteMode::CREATE_PARENT, true)) {
// Retry, but make sure we don't try to create the parent again.
mode = mode - WriteMode::CREATE_PARENT;
return createNamedTemporary(finalName, mode, kj::mv(tryCreate));
}
// fallthrough
default:
KJ_FAIL_SYSCALL("create(path)", error, path) { break; }
return nullptr;
}
return kj::mv(path);
}
bool tryReplaceNode(PathPtr path, WriteMode mode, Function<int(StringPtr)> tryCreate) {
// Replaces the given path with an object created by calling tryCreate().
//
// tryCreate() must behave like a syscall which creates the node at the path passed to it,
// returning a negative value on error. If the path passed to tryCreate already exists, it
// MUST fail with EEXIST.
//
// When `mode` includes MODIFY, replaceNode() reacts to EEXIST by creating the node in a
// temporary location and then rename()ing it into place.
if (path.size() == 0) {
KJ_FAIL_REQUIRE("can't replace self") { return false; }
}
auto filename = path.toString();
if (has(mode, WriteMode::CREATE)) {
// First try just cerating the node in-place.
KJ_SYSCALL_HANDLE_ERRORS(tryCreate(filename)) {
case EEXIST:
// Target exists.
if (has(mode, WriteMode::MODIFY)) {
// Fall back to MODIFY path, below.
break;
} else {
return false;
}
case ENOENT:
if (has(mode, WriteMode::CREATE_PARENT) && path.size() > 0 &&
tryMkdir(path.parent(), WriteMode::CREATE | WriteMode::MODIFY |
WriteMode::CREATE_PARENT, true)) {
// Retry, but make sure we don't try to create the parent again.
return tryReplaceNode(path, mode - WriteMode::CREATE_PARENT, kj::mv(tryCreate));
}
default:
KJ_FAIL_SYSCALL("create(path)", error, path) { return false; }
} else {
// Success.
return true;
}
}
// Either we don't have CREATE mode or the target already exists. We need to perform a
// replacement instead.
KJ_IF_MAYBE(tempPath, createNamedTemporary(path, mode, kj::mv(tryCreate))) {
if (tryCommitReplacement(filename, fd, *tempPath, mode)) {
return true;
} else {
KJ_SYSCALL_HANDLE_ERRORS(unlinkat(fd, tempPath->cStr(), 0)) {
case ENOENT:
// meh
break;
default:
KJ_FAIL_SYSCALL("unlinkat(fd, tempPath, 0)", error, *tempPath);
}
return false;
}
} else {
// threw, but exceptions are disabled
return false;
}
}
Maybe<AutoCloseFd> tryOpenFileInternal(PathPtr path, WriteMode mode, bool append) {
uint flags = O_RDWR | MAYBE_O_CLOEXEC;
mode_t acl = 0666;
if (has(mode, WriteMode::CREATE)) {
flags |= O_CREAT;
}
if (!has(mode, WriteMode::MODIFY)) {
if (!has(mode, WriteMode::CREATE)) {
// Neither CREATE nor MODIFY -- impossible to satisfy preconditions.
return nullptr;
}
flags |= O_EXCL;
}
if (append) {
flags |= O_APPEND;
}
if (has(mode, WriteMode::EXECUTABLE)) {
acl = 0777;
}
if (has(mode, WriteMode::PRIVATE)) {
acl &= 0700;
}
auto filename = path.toString();
int newFd;
KJ_SYSCALL_HANDLE_ERRORS(newFd = openat(fd, filename.cStr(), flags, acl)) {
case ENOENT:
if (has(mode, WriteMode::CREATE)) {
// Either:
// - The file is a broken symlink.
// - A parent directory didn't exist.
if (has(mode, WriteMode::CREATE_PARENT) && path.size() > 0 &&
tryMkdir(path.parent(), WriteMode::CREATE | WriteMode::MODIFY |
WriteMode::CREATE_PARENT, true)) {
// Retry, but make sure we don't try to create the parent again.
return tryOpenFileInternal(path, mode - WriteMode::CREATE_PARENT, append);
}
// Check for broken link.
if (!has(mode, WriteMode::MODIFY) &&
faccessat(fd, filename.cStr(), F_OK, AT_SYMLINK_NOFOLLOW) >= 0) {
// Yep. We treat this as already-exists, which means in CREATE-only mode this is a
// simple failure.
return nullptr;
}
KJ_FAIL_REQUIRE("parent is not a directory", path) { return nullptr; }
} else {
// MODIFY-only mode. ENOENT = doesn't exist = return null.
return nullptr;
}
case ENOTDIR:
if (!has(mode, WriteMode::CREATE)) {
// MODIFY-only mode. ENOTDIR = parent not a directory = doesn't exist = return null.
return nullptr;
}
goto failed;
case EEXIST:
if (!has(mode, WriteMode::MODIFY)) {
// CREATE-only mode. EEXIST = already exists = return null.
return nullptr;
}
goto failed;
default:
failed:
KJ_FAIL_SYSCALL("openat(fd, path, O_RDWR | ...)", error, path) { return nullptr; }
}
kj::AutoCloseFd result(newFd);
#ifndef O_CLOEXEC
setCloexec(result);
#endif
return kj::mv(result);
}
bool tryCommitReplacement(StringPtr toPath, int fromDirFd, StringPtr fromPath, WriteMode mode,
int* errorReason = nullptr) {
if (has(mode, WriteMode::CREATE) && has(mode, WriteMode::MODIFY)) {
// Always clobber. Try it.
KJ_SYSCALL_HANDLE_ERRORS(renameat(fromDirFd, fromPath.cStr(), fd.get(), toPath.cStr())) {
case EISDIR:
case ENOTDIR:
case ENOTEMPTY:
case EEXIST:
// Failed because target exists and due to the various weird quirks of rename(), it
// can't remove it for us. On Linux we can try an exchange instead. On others we have
// to move the target out of the way.
break;
default:
if (errorReason == nullptr) {
KJ_FAIL_SYSCALL("rename(fromPath, toPath)", error, fromPath, toPath) { return false; }
} else {
*errorReason = error;
return false;
}
} else {
return true;
}
}
#ifdef RENAME_EXCHANGE
// Try to use Linux's renameat2() to atomically check preconditions and apply.
if (has(mode, WriteMode::MODIFY)) {
// Use an exchange to implement modification.
//
// We reach this branch when performing a MODIFY-only, or when performing a CREATE | MODIFY
// in which we determined above that there's a node of a different type blocking the
// exchange.
KJ_SYSCALL_HANDLE_ERRORS(syscall(SYS_renameat2,
fromDirFd, fromPath.cStr(), fd.get(), toPath.cStr(), RENAME_EXCHANGE)) {
case ENOSYS:
break; // fall back to traditional means
case ENOENT:
// Presumably because the target path doesn't exist.
if (has(mode, WriteMode::CREATE)) {
KJ_FAIL_ASSERT("rename(tmp, path) claimed path exists but "
"renameat2(fromPath, toPath, EXCAHNGE) said it doest; concurrent modification?",
fromPath, toPath) { return false; }
} else {
// Assume target doesn't exist.
return false;
}
default:
if (errorReason == nullptr) {
KJ_FAIL_SYSCALL("renameat2(fromPath, toPath, EXCHANGE)", error, fromPath, toPath) {
return false;
}
} else {
*errorReason = error;
return false;
}
} else {
// Successful swap! Delete swapped-out content.
rmrf(fromDirFd, fromPath);
return true;
}
} else if (has(mode, WriteMode::CREATE)) {
KJ_SYSCALL_HANDLE_ERRORS(syscall(SYS_renameat2,
fromDirFd, fromPath.cStr(), fd.get(), toPath.cStr(), RENAME_NOREPLACE)) {
case ENOSYS:
break; // fall back to traditional means
case EEXIST:
return false;
default:
if (errorReason == nullptr) {
KJ_FAIL_SYSCALL("renameat2(fromPath, toPath, NOREPLACE)", error, fromPath, toPath) {
return false;
}
} else {
*errorReason = error;
return false;
}
} else {
return true;
}
}
#endif
// We're unable to do what we wanted atomically. :(
if (has(mode, WriteMode::CREATE) && has(mode, WriteMode::MODIFY)) {
// We failed to atomically delete the target previously. So now we need to do two calls in
// rapid succession to move the old file away then move the new one into place.
// Find out what kind of file exists at the target path.
struct stat stats;
KJ_SYSCALL(fstatat(fd, toPath.cStr(), &stats, AT_SYMLINK_NOFOLLOW)) { return false; }
// Create a temporary location to move the existing object to. Note that rename() allows a
// non-directory to replace a non-directory, and allows a directory to replace an empty
// directory. So we have to create the right type.
Path toPathParsed = Path::parse(toPath);
String away;
KJ_IF_MAYBE(awayPath, createNamedTemporary(toPathParsed, WriteMode::CREATE,
[&](StringPtr candidatePath) {
if (S_ISDIR(stats.st_mode)) {
return mkdirat(fd, candidatePath.cStr(), 0700);
} else {
return mknodat(fd, candidatePath.cStr(), S_IFREG | 0600, dev_t());
}
})) {
away = kj::mv(*awayPath);
} else {
// Already threw.
return false;
}
// OK, now move the target object to replace the thing we just created.
KJ_SYSCALL(renameat(fd, toPath.cStr(), fd, away.cStr())) {
// Something went wrong. Remove the thing we just created.
unlinkat(fd, away.cStr(), S_ISDIR(stats.st_mode) ? AT_REMOVEDIR : 0);
return false;
}
// Now move the source object to the target location.
KJ_SYSCALL_HANDLE_ERRORS(renameat(fromDirFd, fromPath.cStr(), fd, toPath.cStr())) {
default:
// Try to put things back where they were. If this fails, though, then we have little
// choice but to leave things broken.
KJ_SYSCALL_HANDLE_ERRORS(renameat(fd, away.cStr(), fd, toPath.cStr())) {
default: break;
}
if (errorReason == nullptr) {
KJ_FAIL_SYSCALL("rename(fromPath, toPath)", error, fromPath, toPath) {
return false;
}
} else {
*errorReason = error;
return false;
}
}
// OK, success. Delete the old content.
rmrf(fd, away);
return true;
} else {
// Only one of CREATE or MODIFY is specified, so we need to verify non-atomically that the
// corresponding precondition (must-not-exist or must-exist, respectively) is held.
if (has(mode, WriteMode::CREATE)) {
struct stat stats;
KJ_SYSCALL_HANDLE_ERRORS(fstatat(fd.get(), toPath.cStr(), &stats, AT_SYMLINK_NOFOLLOW)) {
case ENOENT:
case ENOTDIR:
break; // doesn't exist; continue
default:
KJ_FAIL_SYSCALL("fstatat(fd, toPath)", error, toPath) { return false; }
} else {
return false; // already exists; fail
}
} else if (has(mode, WriteMode::MODIFY)) {
struct stat stats;
KJ_SYSCALL_HANDLE_ERRORS(fstatat(fd.get(), toPath.cStr(), &stats, AT_SYMLINK_NOFOLLOW)) {
case ENOENT:
case ENOTDIR:
return false; // doesn't exist; fail
default:
KJ_FAIL_SYSCALL("fstatat(fd, toPath)", error, toPath) { return false; }
} else {
// already exists; continue
}
} else {
// Neither CREATE nor MODIFY.
return false;
}
// Start over in create-and-modify mode.
return tryCommitReplacement(toPath, fromDirFd, fromPath,
WriteMode::CREATE | WriteMode::MODIFY,
errorReason);
}
}
template <typename T>
class ReplacerImpl: public Directory::Replacer<T> {
public:
ReplacerImpl(Own<T>&& object, DiskHandle& handle,
String&& tempPath, String&& path, WriteMode mode)
: Directory::Replacer<T>(mode),
object(kj::mv(object)), handle(handle),
tempPath(kj::mv(tempPath)), path(kj::mv(path)) {}
~ReplacerImpl() noexcept(false) {
if (!committed) {
rmrf(handle.fd, tempPath);
}
}
T& get() override {
return *object;
}
bool tryCommit() override {
KJ_ASSERT(!committed, "already committed") { return false; }
return committed = handle.tryCommitReplacement(path, handle.fd, tempPath,
Directory::Replacer<T>::mode);
}
private:
Own<T> object;
DiskHandle& handle;
String tempPath;
String path;
bool committed = false; // true if *successfully* committed (in which case tempPath is gone)
};
template <typename T>
class BrokenReplacer: public Directory::Replacer<T> {
// For recovery path when exceptions are disabled.
public:
BrokenReplacer(Own<T> inner)
: Directory::Replacer<T>(WriteMode::CREATE | WriteMode::MODIFY),
inner(kj::mv(inner)) {}
T& get() override { return *inner; }
bool tryCommit() override { return false; }
private:
Own<T> inner;
};
Maybe<Own<File>> tryOpenFile(PathPtr path, WriteMode mode) {
return tryOpenFileInternal(path, mode, false).map(newDiskFile);
}
Own<Directory::Replacer<File>> replaceFile(PathPtr path, WriteMode mode) {
mode_t acl = 0666;
if (has(mode, WriteMode::EXECUTABLE)) {
acl = 0777;
}
if (has(mode, WriteMode::PRIVATE)) {
acl &= 0700;
}
int newFd_;
KJ_IF_MAYBE(temp, createNamedTemporary(path, mode,
[&](StringPtr candidatePath) {
return newFd_ = openat(fd, candidatePath.cStr(),
O_RDWR | O_CREAT | O_EXCL | MAYBE_O_CLOEXEC, acl);
})) {
AutoCloseFd newFd(newFd_);
#ifndef O_CLOEXEC
setCloexec(newFd);
#endif
return heap<ReplacerImpl<File>>(newDiskFile(kj::mv(newFd)), *this, kj::mv(*temp),
path.toString(), mode);
} else {
// threw, but exceptions are disabled
return heap<BrokenReplacer<File>>(newInMemoryFile(nullClock()));
}
}
Own<File> createTemporary() {
int newFd_;
#ifdef O_TMPFILE
// Use syscall() to work around glibc bug with O_TMPFILE:
// https://sourceware.org/bugzilla/show_bug.cgi?id=17523
KJ_SYSCALL_HANDLE_ERRORS(newFd_ = syscall(
SYS_openat, fd.get(), ".", O_RDWR | O_TMPFILE, 0700)) {
case EOPNOTSUPP:
case EINVAL:
// Maybe not supported by this kernel / filesystem. Fall back to below.
break;
default:
KJ_FAIL_SYSCALL("open(O_TMPFILE)", error) { break; }
break;
} else {
AutoCloseFd newFd(newFd_);
#ifndef O_CLOEXEC
setCloexec(newFd);
#endif
return newDiskFile(kj::mv(newFd));
}
#endif
KJ_IF_MAYBE(temp, createNamedTemporary(Path("unnamed"), WriteMode::CREATE,
[&](StringPtr path) {
return newFd_ = openat(fd, path.cStr(), O_RDWR | O_CREAT | O_EXCL | MAYBE_O_CLOEXEC, 0600);
})) {
AutoCloseFd newFd(newFd_);
#ifndef O_CLOEXEC
setCloexec(newFd);
#endif
auto result = newDiskFile(kj::mv(newFd));
KJ_SYSCALL(unlinkat(fd, temp->cStr(), 0)) { break; }
return kj::mv(result);
} else {
// threw, but exceptions are disabled
return newInMemoryFile(nullClock());
}
}
Maybe<Own<AppendableFile>> tryAppendFile(PathPtr path, WriteMode mode) {
return tryOpenFileInternal(path, mode, true).map(newDiskAppendableFile);
}
Maybe<Own<Directory>> tryOpenSubdir(PathPtr path, WriteMode mode) {
// Must create before open.
if (has(mode, WriteMode::CREATE)) {
if (!tryMkdir(path, mode, false)) return nullptr;
}
return tryOpenSubdirInternal(path).map(newDiskDirectory);
}
Own<Directory::Replacer<Directory>> replaceSubdir(PathPtr path, WriteMode mode) {
mode_t acl = has(mode, WriteMode::PRIVATE) ? 0700 : 0777;
KJ_IF_MAYBE(temp, createNamedTemporary(path, mode,
[&](StringPtr candidatePath) {
return mkdirat(fd, candidatePath.cStr(), acl);
})) {
int subdirFd_;
KJ_SYSCALL_HANDLE_ERRORS(subdirFd_ = openat(
fd, temp->cStr(), O_RDONLY | MAYBE_O_CLOEXEC | MAYBE_O_DIRECTORY)) {
default:
KJ_FAIL_SYSCALL("open(just-created-temporary)", error);
return heap<BrokenReplacer<Directory>>(newInMemoryDirectory(nullClock()));
}
AutoCloseFd subdirFd(subdirFd_);
#ifndef O_CLOEXEC
setCloexec(subdirFd);
#endif
return heap<ReplacerImpl<Directory>>(
newDiskDirectory(kj::mv(subdirFd)), *this, kj::mv(*temp), path.toString(), mode);
} else {
// threw, but exceptions are disabled
return heap<BrokenReplacer<Directory>>(newInMemoryDirectory(nullClock()));
}
}
bool trySymlink(PathPtr linkpath, StringPtr content, WriteMode mode) {
return tryReplaceNode(linkpath, mode, [&](StringPtr candidatePath) {
return symlinkat(content.cStr(), fd, candidatePath.cStr());
});
}
bool tryTransfer(PathPtr toPath, WriteMode toMode,
Directory& fromDirectory, PathPtr fromPath,
TransferMode mode, Directory& self) {
KJ_REQUIRE(toPath.size() > 0, "can't replace self") { return false; }
if (mode == TransferMode::LINK) {
KJ_IF_MAYBE(fromFd, fromDirectory.getFd()) {
// Other is a disk directory, so we can hopefully do an efficient move/link.
return tryReplaceNode(toPath, toMode, [&](StringPtr candidatePath) {
return linkat(*fromFd, fromPath.toString().cStr(), fd, candidatePath.cStr(), 0);
});
};
} else if (mode == TransferMode::MOVE) {
KJ_IF_MAYBE(fromFd, fromDirectory.getFd()) {
KJ_ASSERT(mode == TransferMode::MOVE);
int error = 0;
if (tryCommitReplacement(toPath.toString(), *fromFd, fromPath.toString(), toMode,
&error)) {
return true;
} else switch (error) {
case 0:
// Plain old WriteMode precondition failure.
return false;
case EXDEV:
// Can't move between devices. Fall back to default implementation, which does
// copy/delete.
break;
case ENOENT:
// Either the destination directory doesn't exist or the source path doesn't exist.
// Unfortunately we don't really know. If CREATE_PARENT was provided, try creating
// the parent directory. Otherwise, we don't actually need to distinguish between
// these two errors; just return false.
if (has(toMode, WriteMode::CREATE) && has(toMode, WriteMode::CREATE_PARENT) &&
toPath.size() > 0 && tryMkdir(toPath.parent(),
WriteMode::CREATE | WriteMode::MODIFY | WriteMode::CREATE_PARENT, true)) {
// Retry, but make sure we don't try to create the parent again.
return tryTransfer(toPath, toMode - WriteMode::CREATE_PARENT,
fromDirectory, fromPath, mode, self);
}
return false;
default:
KJ_FAIL_SYSCALL("rename(fromPath, toPath)", error, fromPath, toPath) {
return false;
}
}
}
}
// OK, we can't do anything efficient using the OS. Fall back to default implementation.
return self.Directory::tryTransfer(toPath, toMode, fromDirectory, fromPath, mode);
}
bool tryRemove(PathPtr path) {
return rmrf(fd, path.toString());
}
protected:
AutoCloseFd fd;
};
#define FSNODE_METHODS(classname) \
Maybe<int> getFd() override { return DiskHandle::getFd(); } \
\
Own<FsNode> cloneFsNode() override { \
return heap<classname>(DiskHandle::clone()); \
} \
\
Metadata stat() override { return DiskHandle::stat(); } \
void sync() override { DiskHandle::sync(); } \
void datasync() override { DiskHandle::datasync(); }
class DiskReadableFile final: public ReadableFile, public DiskHandle {
public:
DiskReadableFile(AutoCloseFd&& fd): DiskHandle(kj::mv(fd)) {}
FSNODE_METHODS(DiskReadableFile);
size_t read(uint64_t offset, ArrayPtr<byte> buffer) override {
return DiskHandle::read(offset, buffer);
}
Array<const byte> mmap(uint64_t offset, uint64_t size) override {
return DiskHandle::mmap(offset, size);
}
Array<byte> mmapPrivate(uint64_t offset, uint64_t size) override {
return DiskHandle::mmapPrivate(offset, size);
}
};
class DiskAppendableFile final: public AppendableFile, public DiskHandle, public FdOutputStream {
public:
DiskAppendableFile(AutoCloseFd&& fd)
: DiskHandle(kj::mv(fd)),
FdOutputStream(DiskHandle::fd.get()) {}
FSNODE_METHODS(DiskAppendableFile);
void write(const void* buffer, size_t size) override { FdOutputStream::write(buffer, size); }
void write(ArrayPtr<const ArrayPtr<const byte>> pieces) override {
FdOutputStream::write(pieces);
}
};
class DiskFile final: public File, public DiskHandle {
public:
DiskFile(AutoCloseFd&& fd): DiskHandle(kj::mv(fd)) {}
FSNODE_METHODS(DiskFile);
size_t read(uint64_t offset, ArrayPtr<byte> buffer) override {
return DiskHandle::read(offset, buffer);
}
Array<const byte> mmap(uint64_t offset, uint64_t size) override {
return DiskHandle::mmap(offset, size);
}
Array<byte> mmapPrivate(uint64_t offset, uint64_t size) override {
return DiskHandle::mmapPrivate(offset, size);
}
void write(uint64_t offset, ArrayPtr<const byte> data) override {
DiskHandle::write(offset, data);
}
void zero(uint64_t offset, uint64_t size) override {
DiskHandle::zero(offset, size);
}
void truncate(uint64_t size) override {
DiskHandle::truncate(size);
}
Own<WritableFileMapping> mmapWritable(uint64_t offset, uint64_t size) override {
return DiskHandle::mmapWritable(offset, size);
}
size_t copy(uint64_t offset, ReadableFile& from, uint64_t fromOffset, uint64_t size) override {
KJ_IF_MAYBE(result, DiskHandle::copy(offset, from, fromOffset, size)) {
return *result;
} else {
return File::copy(offset, from, fromOffset, size);
}
}
};
class DiskReadableDirectory final: public ReadableDirectory, public DiskHandle {
public:
DiskReadableDirectory(AutoCloseFd&& fd): DiskHandle(kj::mv(fd)) {}
FSNODE_METHODS(DiskReadableDirectory);
Array<String> listNames() override { return DiskHandle::listNames(); }
Array<Entry> listEntries() override { return DiskHandle::listEntries(); }
bool exists(PathPtr path) override { return DiskHandle::exists(path); }
Maybe<FsNode::Metadata> tryLstat(PathPtr path) override { return DiskHandle::tryLstat(path); }
Maybe<Own<ReadableFile>> tryOpenFile(PathPtr path) override {
return DiskHandle::tryOpenFile(path);
}
Maybe<Own<ReadableDirectory>> tryOpenSubdir(PathPtr path) override {
return DiskHandle::tryOpenSubdir(path);
}
Maybe<String> tryReadlink(PathPtr path) override { return DiskHandle::tryReadlink(path); }
};
class DiskDirectory final: public Directory, public DiskHandle {
public:
DiskDirectory(AutoCloseFd&& fd): DiskHandle(kj::mv(fd)) {}
FSNODE_METHODS(DiskDirectory);
Array<String> listNames() override { return DiskHandle::listNames(); }
Array<Entry> listEntries() override { return DiskHandle::listEntries(); }
bool exists(PathPtr path) override { return DiskHandle::exists(path); }
Maybe<FsNode::Metadata> tryLstat(PathPtr path) override { return DiskHandle::tryLstat(path); }
Maybe<Own<ReadableFile>> tryOpenFile(PathPtr path) override {
return DiskHandle::tryOpenFile(path);
}
Maybe<Own<ReadableDirectory>> tryOpenSubdir(PathPtr path) override {
return DiskHandle::tryOpenSubdir(path);
}
Maybe<String> tryReadlink(PathPtr path) override { return DiskHandle::tryReadlink(path); }
Maybe<Own<File>> tryOpenFile(PathPtr path, WriteMode mode) override {
return DiskHandle::tryOpenFile(path, mode);
}
Own<Replacer<File>> replaceFile(PathPtr path, WriteMode mode) override {
return DiskHandle::replaceFile(path, mode);
}
Own<File> createTemporary() override {
return DiskHandle::createTemporary();
}
Maybe<Own<AppendableFile>> tryAppendFile(PathPtr path, WriteMode mode) override {
return DiskHandle::tryAppendFile(path, mode);
}
Maybe<Own<Directory>> tryOpenSubdir(PathPtr path, WriteMode mode) override {
return DiskHandle::tryOpenSubdir(path, mode);
}
Own<Replacer<Directory>> replaceSubdir(PathPtr path, WriteMode mode) override {
return DiskHandle::replaceSubdir(path, mode);
}
bool trySymlink(PathPtr linkpath, StringPtr content, WriteMode mode) override {
return DiskHandle::trySymlink(linkpath, content, mode);
}
bool tryTransfer(PathPtr toPath, WriteMode toMode,
Directory& fromDirectory, PathPtr fromPath,
TransferMode mode) override {
return DiskHandle::tryTransfer(toPath, toMode, fromDirectory, fromPath, mode, *this);
}
// tryTransferTo() not implemented because we have nothing special we can do.
bool tryRemove(PathPtr path) override {
return DiskHandle::tryRemove(path);
}
};
class DiskFilesystem final: public Filesystem {
public:
DiskFilesystem()
: root(openDir("/")),
current(openDir(".")),
currentPath(computeCurrentPath()) {}
Directory& getRoot() override {
return root;
}
Directory& getCurrent() override {
return current;
}
PathPtr getCurrentPath() override {
return currentPath;
}
private:
DiskDirectory root;
DiskDirectory current;
Path currentPath;
static AutoCloseFd openDir(const char* dir) {
int newFd;
KJ_SYSCALL(newFd = open(dir, O_RDONLY | MAYBE_O_CLOEXEC | MAYBE_O_DIRECTORY));
AutoCloseFd result(newFd);
#ifndef O_CLOEXEC
setCloexec(result);
#endif
return result;
}
static Path computeCurrentPath() {
// If env var PWD is set and points to the current directory, use it. This captures the current
// path according to the user's shell, which may differ from the kernel's idea in the presence
// of symlinks.
const char* pwd = getenv("PWD");
if (pwd != nullptr) {
Path result = nullptr;
struct stat pwdStat, dotStat;
KJ_IF_MAYBE(e, kj::runCatchingExceptions([&]() {
KJ_ASSERT(pwd[0] == '/') { return; }
result = Path::parse(pwd + 1);
KJ_SYSCALL(lstat(result.toString(true).cStr(), &pwdStat), result) { return; }
KJ_SYSCALL(lstat(".", &dotStat)) { return; }
})) {
// failed, give up on PWD
KJ_LOG(WARNING, "PWD environment variable seems invalid", pwd, *e);
} else {
if (pwdStat.st_ino == dotStat.st_ino &&
pwdStat.st_dev == dotStat.st_dev) {
return kj::mv(result);
} else {
KJ_LOG(WARNING, "PWD environment variable doesn't match current directory", pwd);
}
}
}
size_t size = 256;
retry:
KJ_STACK_ARRAY(char, buf, size, 256, 4096);
if (getcwd(buf.begin(), size) == nullptr) {
int error = errno;
if (error == ENAMETOOLONG) {
size *= 2;
goto retry;
} else {
KJ_FAIL_SYSCALL("getcwd()", error);
}
}
StringPtr path = buf.begin();
// On Linux, the path will start with "(unreachable)" if the working directory is not a subdir
// of the root directory, which is possible via chroot() or mount namespaces.
KJ_ASSERT(!path.startsWith("(unreachable)"),
"working directory is not reachable from root", path);
KJ_ASSERT(path.startsWith("/"), "current directory is not absolute", path);
return Path::parse(path.slice(1));
}
};
} // namespace
Own<ReadableFile> newDiskReadableFile(kj::AutoCloseFd fd) {
return heap<DiskReadableFile>(kj::mv(fd));
}
Own<AppendableFile> newDiskAppendableFile(kj::AutoCloseFd fd) {
return heap<DiskAppendableFile>(kj::mv(fd));
}
Own<File> newDiskFile(kj::AutoCloseFd fd) {
return heap<DiskFile>(kj::mv(fd));
}
Own<ReadableDirectory> newDiskReadableDirectory(kj::AutoCloseFd fd) {
return heap<DiskReadableDirectory>(kj::mv(fd));
}
Own<Directory> newDiskDirectory(kj::AutoCloseFd fd) {
return heap<DiskDirectory>(kj::mv(fd));
}
Own<Filesystem> newDiskFilesystem() {
return heap<DiskFilesystem>();
}
} // namespace kj
c++/src/kj/filesystem-test.c++ 0 → 100644
View file @ b1701b88
// Copyright (c) 2016 Sandstorm Development Group, Inc. and contributors
// Licensed under the MIT License:
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
#include "filesystem.h"
#include "test.h"
namespace kj {
namespace {
KJ_TEST("Path") {
KJ_EXPECT(Path(nullptr).toString() == ".");
KJ_EXPECT(Path(nullptr).toString(true) == "/");
KJ_EXPECT(Path("foo").toString() == "foo");
KJ_EXPECT(Path("foo").toString(true) == "/foo");
KJ_EXPECT(Path({"foo", "bar"}).toString() == "foo/bar");
KJ_EXPECT(Path({"foo", "bar"}).toString(true) == "/foo/bar");
KJ_EXPECT(Path::parse("foo/bar").toString() == "foo/bar");
KJ_EXPECT(Path::parse("foo//bar").toString() == "foo/bar");
KJ_EXPECT(Path::parse("foo/./bar").toString() == "foo/bar");
KJ_EXPECT(Path::parse("foo/../bar").toString() == "bar");
KJ_EXPECT(Path::parse("foo/bar/..").toString() == "foo");
KJ_EXPECT(Path::parse("foo/bar/../..").toString() == ".");
KJ_EXPECT(Path({"foo", "bar"}).eval("baz").toString() == "foo/bar/baz");
KJ_EXPECT(Path({"foo", "bar"}).eval("./baz").toString() == "foo/bar/baz");
KJ_EXPECT(Path({"foo", "bar"}).eval("baz/qux").toString() == "foo/bar/baz/qux");
KJ_EXPECT(Path({"foo", "bar"}).eval("baz//qux").toString() == "foo/bar/baz/qux");
KJ_EXPECT(Path({"foo", "bar"}).eval("baz/./qux").toString() == "foo/bar/baz/qux");
KJ_EXPECT(Path({"foo", "bar"}).eval("baz/../qux").toString() == "foo/bar/qux");
KJ_EXPECT(Path({"foo", "bar"}).eval("baz/qux/..").toString() == "foo/bar/baz");
KJ_EXPECT(Path({"foo", "bar"}).eval("../baz").toString() == "foo/baz");
KJ_EXPECT(Path({"foo", "bar"}).eval("baz/../../qux/").toString() == "foo/qux");
KJ_EXPECT(Path({"foo", "bar"}).eval("/baz/qux").toString() == "baz/qux");
KJ_EXPECT(Path({"foo", "bar"}).eval("//baz/qux").toString() == "baz/qux");
KJ_EXPECT(Path({"foo", "bar"}).eval("/baz/../qux").toString() == "qux");
KJ_EXPECT(Path({"foo", "bar"}).basename()[0] == "bar");
KJ_EXPECT(Path({"foo", "bar", "baz"}).parent().toString() == "foo/bar");
KJ_EXPECT(Path({"foo", "bar"}).append("baz").toString() == "foo/bar/baz");
KJ_EXPECT(Path({"foo", "bar"}).append(Path({"baz", "qux"})).toString() == "foo/bar/baz/qux");
{
// Test methods which are overloaded for && on a non-rvalue path.
Path path({"foo", "bar"});
KJ_EXPECT(path.eval("baz").toString() == "foo/bar/baz");
KJ_EXPECT(path.eval("./baz").toString() == "foo/bar/baz");
KJ_EXPECT(path.eval("baz/qux").toString() == "foo/bar/baz/qux");
KJ_EXPECT(path.eval("baz//qux").toString() == "foo/bar/baz/qux");
KJ_EXPECT(path.eval("baz/./qux").toString() == "foo/bar/baz/qux");
KJ_EXPECT(path.eval("baz/../qux").toString() == "foo/bar/qux");
KJ_EXPECT(path.eval("baz/qux/..").toString() == "foo/bar/baz");
KJ_EXPECT(path.eval("../baz").toString() == "foo/baz");
KJ_EXPECT(path.eval("baz/../../qux/").toString() == "foo/qux");
KJ_EXPECT(path.eval("/baz/qux").toString() == "baz/qux");
KJ_EXPECT(path.eval("/baz/../qux").toString() == "qux");
KJ_EXPECT(path.basename()[0] == "bar");
KJ_EXPECT(path.parent().toString() == "foo");
KJ_EXPECT(path.append("baz").toString() == "foo/bar/baz");
KJ_EXPECT(path.append(Path({"baz", "qux"})).toString() == "foo/bar/baz/qux");
}
KJ_EXPECT(kj::str(Path({"foo", "bar"})) == "foo/bar");
}
KJ_TEST("Path exceptions") {
KJ_EXPECT_THROW_MESSAGE("invalid path component", Path(""));
KJ_EXPECT_THROW_MESSAGE("invalid path component", Path("."));
KJ_EXPECT_THROW_MESSAGE("invalid path component", Path(".."));
KJ_EXPECT_THROW_MESSAGE("NUL character", Path(StringPtr("foo\0bar", 7)));
KJ_EXPECT_THROW_MESSAGE("break out of starting", Path::parse(".."));
KJ_EXPECT_THROW_MESSAGE("break out of starting", Path::parse("../foo"));
KJ_EXPECT_THROW_MESSAGE("break out of starting", Path::parse("foo/../.."));
KJ_EXPECT_THROW_MESSAGE("expected a relative path", Path::parse("/foo"));
KJ_EXPECT_THROW_MESSAGE("NUL character", Path::parse(kj::StringPtr("foo\0bar", 7)));
KJ_EXPECT_THROW_MESSAGE("break out of starting", Path({"foo", "bar"}).eval("../../.."));
KJ_EXPECT_THROW_MESSAGE("break out of starting", Path({"foo", "bar"}).eval("../baz/../../.."));
KJ_EXPECT_THROW_MESSAGE("break out of starting", Path({"foo", "bar"}).eval("baz/../../../.."));
KJ_EXPECT_THROW_MESSAGE("break out of starting", Path({"foo", "bar"}).eval("/.."));
KJ_EXPECT_THROW_MESSAGE("break out of starting", Path({"foo", "bar"}).eval("/baz/../.."));
KJ_EXPECT_THROW_MESSAGE("root path has no basename", Path(nullptr).basename());
KJ_EXPECT_THROW_MESSAGE("root path has no parent", Path(nullptr).parent());
}
KJ_TEST("Win32 Path") {
KJ_EXPECT(Path({"foo", "bar"}).toWin32String() == "foo\\bar");
KJ_EXPECT(Path({"foo", "bar"}).toWin32String(true) == "\\\\foo\\bar");
KJ_EXPECT(Path({"c:", "foo", "bar"}).toWin32String(true) == "c:\\foo\\bar");
KJ_EXPECT(Path({"A:", "foo", "bar"}).toWin32String(true) == "A:\\foo\\bar");
KJ_EXPECT(Path({"foo", "bar"}).evalWin32("baz").toWin32String() == "foo\\bar\\baz");
KJ_EXPECT(Path({"foo", "bar"}).evalWin32("./baz").toWin32String() == "foo\\bar\\baz");
KJ_EXPECT(Path({"foo", "bar"}).evalWin32("baz/qux").toWin32String() == "foo\\bar\\baz\\qux");
KJ_EXPECT(Path({"foo", "bar"}).evalWin32("baz//qux").toWin32String() == "foo\\bar\\baz\\qux");
KJ_EXPECT(Path({"foo", "bar"}).evalWin32("baz/./qux").toWin32String() == "foo\\bar\\baz\\qux");
KJ_EXPECT(Path({"foo", "bar"}).evalWin32("baz/../qux").toWin32String() == "foo\\bar\\qux");
KJ_EXPECT(Path({"foo", "bar"}).evalWin32("baz/qux/..").toWin32String() == "foo\\bar\\baz");
KJ_EXPECT(Path({"foo", "bar"}).evalWin32("../baz").toWin32String() == "foo\\baz");
KJ_EXPECT(Path({"foo", "bar"}).evalWin32("baz/../../qux/").toWin32String() == "foo\\qux");
KJ_EXPECT(Path({"foo", "bar"}).evalWin32(".\\baz").toWin32String() == "foo\\bar\\baz");
KJ_EXPECT(Path({"foo", "bar"}).evalWin32("baz\\qux").toWin32String() == "foo\\bar\\baz\\qux");
KJ_EXPECT(Path({"foo", "bar"}).evalWin32("baz\\\\qux").toWin32String() == "foo\\bar\\baz\\qux");
KJ_EXPECT(Path({"foo", "bar"}).evalWin32("baz\\.\\qux").toWin32String() == "foo\\bar\\baz\\qux");
KJ_EXPECT(Path({"foo", "bar"}).evalWin32("baz\\..\\qux").toWin32String() == "foo\\bar\\qux");
KJ_EXPECT(Path({"foo", "bar"}).evalWin32("baz\\qux\\..").toWin32String() == "foo\\bar\\baz");
KJ_EXPECT(Path({"foo", "bar"}).evalWin32("..\\baz").toWin32String() == "foo\\baz");
KJ_EXPECT(Path({"foo", "bar"}).evalWin32("baz\\..\\..\\qux\\").toWin32String() == "foo\\qux");
KJ_EXPECT(Path({"foo", "bar"}).evalWin32("baz\\../..\\qux/").toWin32String() == "foo\\qux");
KJ_EXPECT(Path({"c:", "foo", "bar"}).evalWin32("/baz/qux")
.toWin32String(true) == "c:\\baz\\qux");
KJ_EXPECT(Path({"c:", "foo", "bar"}).evalWin32("\\baz\\qux")
.toWin32String(true) == "c:\\baz\\qux");
KJ_EXPECT(Path({"c:", "foo", "bar"}).evalWin32("d:\\baz\\qux")
.toWin32String(true) == "d:\\baz\\qux");
KJ_EXPECT(Path({"c:", "foo", "bar"}).evalWin32("d:\\baz\\..\\qux")
.toWin32String(true) == "d:\\qux");
KJ_EXPECT(Path({"c:", "foo", "bar"}).evalWin32("\\\\baz\\qux")
.toWin32String(true) == "\\\\baz\\qux");
KJ_EXPECT(Path({"foo", "bar"}).evalWin32("d:\\baz\\..\\qux")
.toWin32String(true) == "d:\\qux");
KJ_EXPECT(Path({"foo", "bar", "baz"}).evalWin32("\\qux")
.toWin32String(true) == "\\\\foo\\bar\\qux");
}
KJ_TEST("Win32 Path exceptions") {
KJ_EXPECT_THROW_MESSAGE("colons are prohibited", Path({"c:", "foo", "bar"}).toWin32String());
KJ_EXPECT_THROW_MESSAGE("colons are prohibited", Path({"c:", "foo:bar"}).toWin32String(true));
KJ_EXPECT_THROW_MESSAGE("DOS reserved name", Path({"con"}).toWin32String());
KJ_EXPECT_THROW_MESSAGE("DOS reserved name", Path({"CON", "bar"}).toWin32String());
KJ_EXPECT_THROW_MESSAGE("DOS reserved name", Path({"foo", "cOn"}).toWin32String());
KJ_EXPECT_THROW_MESSAGE("DOS reserved name", Path({"prn"}).toWin32String());
KJ_EXPECT_THROW_MESSAGE("DOS reserved name", Path({"aux"}).toWin32String());
KJ_EXPECT_THROW_MESSAGE("DOS reserved name", Path({"NUL"}).toWin32String());
KJ_EXPECT_THROW_MESSAGE("DOS reserved name", Path({"nul.txt"}).toWin32String());
KJ_EXPECT_THROW_MESSAGE("DOS reserved name", Path({"com3"}).toWin32String());
KJ_EXPECT_THROW_MESSAGE("DOS reserved name", Path({"lpt9"}).toWin32String());
KJ_EXPECT_THROW_MESSAGE("DOS reserved name", Path({"com1.hello"}).toWin32String());
KJ_EXPECT_THROW_MESSAGE("drive letter or netbios", Path({"?", "foo"}).toWin32String(true));
KJ_EXPECT_THROW_MESSAGE("break out of starting", Path({"foo", "bar"}).evalWin32("../../.."));
KJ_EXPECT_THROW_MESSAGE("break out of starting", Path({"foo", "bar"}).evalWin32("../baz/../../.."));
KJ_EXPECT_THROW_MESSAGE("break out of starting", Path({"foo", "bar"}).evalWin32("baz/../../../.."));
KJ_EXPECT_THROW_MESSAGE("break out of starting", Path({"foo", "bar"}).evalWin32("c:\\..\\.."));
KJ_EXPECT_THROW_MESSAGE("break out of starting",
Path({"c:", "foo", "bar"}).evalWin32("/baz/../../.."));
KJ_EXPECT_THROW_MESSAGE("must specify drive letter", Path({"foo"}).evalWin32("\\baz\\qux"));
}
KJ_TEST("WriteMode operators") {
WriteMode createOrModify = WriteMode::CREATE | WriteMode::MODIFY;
KJ_EXPECT(has(createOrModify, WriteMode::MODIFY));
KJ_EXPECT(has(createOrModify, WriteMode::CREATE));
KJ_EXPECT(!has(createOrModify, WriteMode::CREATE_PARENT));
KJ_EXPECT(has(createOrModify, createOrModify));
KJ_EXPECT(!has(createOrModify, createOrModify | WriteMode::CREATE_PARENT));
KJ_EXPECT(!has(createOrModify, WriteMode::CREATE | WriteMode::CREATE_PARENT));
KJ_EXPECT(!has(WriteMode::CREATE, createOrModify));
KJ_EXPECT(createOrModify != WriteMode::MODIFY);
KJ_EXPECT(createOrModify != WriteMode::CREATE);
KJ_EXPECT(createOrModify - WriteMode::CREATE == WriteMode::MODIFY);
KJ_EXPECT(WriteMode::CREATE + WriteMode::MODIFY == createOrModify);
// Adding existing bit / subtracting non-existing bit are no-ops.
KJ_EXPECT(createOrModify + WriteMode::MODIFY == createOrModify);
KJ_EXPECT(createOrModify - WriteMode::CREATE_PARENT == createOrModify);
}
// ======================================================================================
class TestClock final: public Clock {
public:
void tick() {
time += 1 * SECONDS;
}
Date now() override { return time; }
void expectChanged(FsNode& file) {
KJ_EXPECT(file.stat().lastModified == time);
time += 1 * SECONDS;
}
void expectUnchanged(FsNode& file) {
KJ_EXPECT(file.stat().lastModified != time);
}
private:
Date time = UNIX_EPOCH + 1 * SECONDS;
};
KJ_TEST("InMemoryFile") {
TestClock clock;
auto file = newInMemoryFile(clock);
clock.expectChanged(*file);
KJ_EXPECT(file->readAllText() == "");
clock.expectUnchanged(*file);
file->writeAll("foo");
clock.expectChanged(*file);
KJ_EXPECT(file->readAllText() == "foo");
file->write(3, StringPtr("bar").asBytes());
clock.expectChanged(*file);
KJ_EXPECT(file->readAllText() == "foobar");
file->write(3, StringPtr("baz").asBytes());
clock.expectChanged(*file);
KJ_EXPECT(file->readAllText() == "foobaz");
file->write(9, StringPtr("qux").asBytes());
clock.expectChanged(*file);
KJ_EXPECT(file->readAllText() == kj::StringPtr("foobaz\0\0\0qux", 12));
file->truncate(6);
clock.expectChanged(*file);
KJ_EXPECT(file->readAllText() == "foobaz");
file->truncate(18);
clock.expectChanged(*file);
KJ_EXPECT(file->readAllText() == kj::StringPtr("foobaz\0\0\0\0\0\0\0\0\0\0\0\0", 18));
{
auto mapping = file->mmap(0, 18);
auto privateMapping = file->mmapPrivate(0, 18);
auto writableMapping = file->mmapWritable(0, 18);
clock.expectUnchanged(*file);
KJ_EXPECT(mapping.size() == 18);
KJ_EXPECT(privateMapping.size() == 18);
KJ_EXPECT(writableMapping->get().size() == 18);
clock.expectUnchanged(*file);
KJ_EXPECT(writableMapping->get().begin() == mapping.begin());
KJ_EXPECT(privateMapping.begin() != mapping.begin());
KJ_EXPECT(kj::str(mapping.slice(0, 6).asChars()) == "foobaz");
KJ_EXPECT(kj::str(privateMapping.slice(0, 6).asChars()) == "foobaz");
clock.expectUnchanged(*file);
file->write(0, StringPtr("qux").asBytes());
clock.expectChanged(*file);
KJ_EXPECT(kj::str(mapping.slice(0, 6).asChars()) == "quxbaz");
KJ_EXPECT(kj::str(privateMapping.slice(0, 6).asChars()) == "foobaz");
file->write(12, StringPtr("corge").asBytes());
KJ_EXPECT(kj::str(mapping.slice(12, 17).asChars()) == "corge");
// Can shrink.
file->truncate(6);
KJ_EXPECT(kj::str(mapping.slice(12, 17).asChars()) == kj::StringPtr("\0\0\0\0\0", 5));
// Can regrow.
file->truncate(18);
KJ_EXPECT(kj::str(mapping.slice(12, 17).asChars()) == kj::StringPtr("\0\0\0\0\0", 5));
// Can't grow past previoous capacity.
KJ_EXPECT_THROW_MESSAGE("cannot resize the file backing store", file->truncate(100));
clock.expectChanged(*file);
writableMapping->changed(writableMapping->get().slice(0, 3));
clock.expectChanged(*file);
writableMapping->sync(writableMapping->get().slice(0, 3));
clock.expectChanged(*file);
}
// But now we can since the mapping is gone.
file->truncate(100);
file->truncate(6);
clock.expectChanged(*file);
KJ_EXPECT(file->readAllText() == "quxbaz");
file->zero(3, 3);
clock.expectChanged(*file);
KJ_EXPECT(file->readAllText() == StringPtr("qux\0\0\0", 6));
}
KJ_TEST("InMemoryFile::copy()") {
TestClock clock;
auto source = newInMemoryFile(clock);
source->writeAll("foobarbaz");
auto dest = newInMemoryFile(clock);
dest->writeAll("quxcorge");
clock.expectChanged(*dest);
KJ_EXPECT(dest->copy(3, *source, 6, kj::maxValue) == 3);
clock.expectChanged(*dest);
KJ_EXPECT(dest->readAllText() == "quxbazge");
KJ_EXPECT(dest->copy(0, *source, 3, 4) == 4);
clock.expectChanged(*dest);
KJ_EXPECT(dest->readAllText() == "barbazge");
KJ_EXPECT(dest->copy(0, *source, 128, kj::maxValue) == 0);
clock.expectUnchanged(*dest);
KJ_EXPECT(dest->copy(4, *source, 3, 0) == 0);
clock.expectUnchanged(*dest);
String bigString = strArray(repeat("foobar", 10000), "");
source->truncate(bigString.size() + 1000);
source->write(123, bigString.asBytes());
dest->copy(321, *source, 123, bigString.size());
KJ_EXPECT(dest->readAllText().slice(321) == bigString);
}
KJ_TEST("File::copy()") {
TestClock clock;
auto source = newInMemoryFile(clock);
source->writeAll("foobarbaz");
auto dest = newInMemoryFile(clock);
dest->writeAll("quxcorge");
clock.expectChanged(*dest);
KJ_EXPECT(dest->File::copy(3, *source, 6, kj::maxValue) == 3);
clock.expectChanged(*dest);
KJ_EXPECT(dest->readAllText() == "quxbazge");
KJ_EXPECT(dest->File::copy(0, *source, 3, 4) == 4);
clock.expectChanged(*dest);
KJ_EXPECT(dest->readAllText() == "barbazge");
KJ_EXPECT(dest->File::copy(0, *source, 128, kj::maxValue) == 0);
clock.expectUnchanged(*dest);
KJ_EXPECT(dest->File::copy(4, *source, 3, 0) == 0);
clock.expectUnchanged(*dest);
String bigString = strArray(repeat("foobar", 10000), "");
source->truncate(bigString.size() + 1000);
source->write(123, bigString.asBytes());
dest->File::copy(321, *source, 123, bigString.size());
KJ_EXPECT(dest->readAllText().slice(321) == bigString);
}
KJ_TEST("InMemoryDirectory") {
TestClock clock;
auto dir = newInMemoryDirectory(clock);
clock.expectChanged(*dir);
KJ_EXPECT(dir->listNames() == nullptr);
KJ_EXPECT(dir->listEntries() == nullptr);
KJ_EXPECT(!dir->exists(Path("foo")));
KJ_EXPECT(dir->tryOpenFile(Path("foo")) == nullptr);
KJ_EXPECT(dir->tryOpenFile(Path("foo"), WriteMode::MODIFY) == nullptr);
clock.expectUnchanged(*dir);
{
auto file = dir->openFile(Path("foo"), WriteMode::CREATE);
clock.expectChanged(*dir);
file->writeAll("foobar");
clock.expectUnchanged(*dir);
}
clock.expectUnchanged(*dir);
KJ_EXPECT(dir->exists(Path("foo")));
clock.expectUnchanged(*dir);
{
auto stats = dir->lstat(Path("foo"));
clock.expectUnchanged(*dir);
KJ_EXPECT(stats.type == FsNode::Type::FILE);
KJ_EXPECT(stats.size == 6);
}
{
auto list = dir->listNames();
clock.expectUnchanged(*dir);
KJ_ASSERT(list.size() == 1);
KJ_EXPECT(list[0] == "foo");
}
{
auto list = dir->listEntries();
clock.expectUnchanged(*dir);
KJ_ASSERT(list.size() == 1);
KJ_EXPECT(list[0].name == "foo");
KJ_EXPECT(list[0].type == FsNode::Type::FILE);
}
KJ_EXPECT(dir->openFile(Path("foo"))->readAllText() == "foobar");
clock.expectUnchanged(*dir);
KJ_EXPECT(dir->tryOpenFile(Path({"foo", "bar"}), WriteMode::MODIFY) == nullptr);
KJ_EXPECT(dir->tryOpenFile(Path({"bar", "baz"}), WriteMode::MODIFY) == nullptr);
KJ_EXPECT_THROW_MESSAGE("parent is not a directory",
dir->tryOpenFile(Path({"bar", "baz"}), WriteMode::CREATE));
clock.expectUnchanged(*dir);
{
auto file = dir->openFile(Path({"bar", "baz"}), WriteMode::CREATE | WriteMode::CREATE_PARENT);
clock.expectChanged(*dir);
file->writeAll("bazqux");
clock.expectUnchanged(*dir);
}
clock.expectUnchanged(*dir);
KJ_EXPECT(dir->openFile(Path({"bar", "baz"}))->readAllText() == "bazqux");
clock.expectUnchanged(*dir);
{
auto stats = dir->lstat(Path("bar"));
clock.expectUnchanged(*dir);
KJ_EXPECT(stats.type == FsNode::Type::DIRECTORY);
}
{
auto list = dir->listNames();
clock.expectUnchanged(*dir);
KJ_ASSERT(list.size() == 2);
KJ_EXPECT(list[0] == "bar");
KJ_EXPECT(list[1] == "foo");
}
{
auto list = dir->listEntries();
clock.expectUnchanged(*dir);
KJ_ASSERT(list.size() == 2);
KJ_EXPECT(list[0].name == "bar");
KJ_EXPECT(list[0].type == FsNode::Type::DIRECTORY);
KJ_EXPECT(list[1].name == "foo");
KJ_EXPECT(list[1].type == FsNode::Type::FILE);
}
{
auto subdir = dir->openSubdir(Path("bar"));
clock.expectUnchanged(*dir);
clock.expectUnchanged(*subdir);
KJ_EXPECT(subdir->openFile(Path("baz"))->readAllText() == "bazqux");
clock.expectUnchanged(*subdir);
}
auto subdir = dir->openSubdir(Path("corge"), WriteMode::CREATE);
clock.expectChanged(*dir);
subdir->openFile(Path("grault"), WriteMode::CREATE)->writeAll("garply");
clock.expectUnchanged(*dir);
clock.expectChanged(*subdir);
KJ_EXPECT(dir->openFile(Path({"corge", "grault"}))->readAllText() == "garply");
dir->openFile(Path({"corge", "grault"}), WriteMode::CREATE | WriteMode::MODIFY)
->write(0, StringPtr("rag").asBytes());
KJ_EXPECT(dir->openFile(Path({"corge", "grault"}))->readAllText() == "ragply");
clock.expectUnchanged(*dir);
{
auto replacer =
dir->replaceFile(Path({"corge", "grault"}), WriteMode::CREATE | WriteMode::MODIFY);
clock.expectUnchanged(*subdir);
replacer->get().writeAll("rag");
clock.expectUnchanged(*subdir);
// Don't commit.
}
clock.expectUnchanged(*subdir);
KJ_EXPECT(dir->openFile(Path({"corge", "grault"}))->readAllText() == "ragply");
{
auto replacer =
dir->replaceFile(Path({"corge", "grault"}), WriteMode::CREATE | WriteMode::MODIFY);
clock.expectUnchanged(*subdir);
replacer->get().writeAll("rag");
clock.expectUnchanged(*subdir);
replacer->commit();
clock.expectChanged(*subdir);
KJ_EXPECT(dir->openFile(Path({"corge", "grault"}))->readAllText() == "rag");
}
KJ_EXPECT(dir->openFile(Path({"corge", "grault"}))->readAllText() == "rag");
{
auto appender = dir->appendFile(Path({"corge", "grault"}), WriteMode::MODIFY);
appender->write("waldo", 5);
appender->write("fred", 4);
}
KJ_EXPECT(dir->openFile(Path({"corge", "grault"}))->readAllText() == "ragwaldofred");
KJ_EXPECT(dir->exists(Path("foo")));
clock.expectUnchanged(*dir);
dir->remove(Path("foo"));
clock.expectChanged(*dir);
KJ_EXPECT(!dir->exists(Path("foo")));
KJ_EXPECT(!dir->tryRemove(Path("foo")));
clock.expectUnchanged(*dir);
KJ_EXPECT(dir->exists(Path({"bar", "baz"})));
clock.expectUnchanged(*dir);
dir->remove(Path({"bar", "baz"}));
clock.expectUnchanged(*dir);
KJ_EXPECT(!dir->exists(Path({"bar", "baz"})));
KJ_EXPECT(dir->exists(Path("bar")));
KJ_EXPECT(!dir->tryRemove(Path({"bar", "baz"})));
clock.expectUnchanged(*dir);
KJ_EXPECT(dir->exists(Path("corge")));
KJ_EXPECT(dir->exists(Path({"corge", "grault"})));
clock.expectUnchanged(*dir);
dir->remove(Path("corge"));
clock.expectChanged(*dir);
KJ_EXPECT(!dir->exists(Path("corge")));
KJ_EXPECT(!dir->exists(Path({"corge", "grault"})));
KJ_EXPECT(!dir->tryRemove(Path("corge")));
clock.expectUnchanged(*dir);
}
KJ_TEST("InMemoryDirectory symlinks") {
TestClock clock;
auto dir = newInMemoryDirectory(clock);
clock.expectChanged(*dir);
dir->symlink(Path("foo"), "bar/qux/../baz", WriteMode::CREATE);
clock.expectChanged(*dir);
KJ_EXPECT(!dir->trySymlink(Path("foo"), "bar/qux/../baz", WriteMode::CREATE));
clock.expectUnchanged(*dir);
{
auto stats = dir->lstat(Path("foo"));
clock.expectUnchanged(*dir);
KJ_EXPECT(stats.type == FsNode::Type::SYMLINK);
}
KJ_EXPECT(dir->readlink(Path("foo")) == "bar/qux/../baz");
// Broken link into non-existing directory cannot be opened in any mode.
KJ_EXPECT(dir->tryOpenFile(Path("foo")) == nullptr);
KJ_EXPECT(dir->tryOpenFile(Path("foo"), WriteMode::CREATE) == nullptr);
KJ_EXPECT(dir->tryOpenFile(Path("foo"), WriteMode::MODIFY) == nullptr);
KJ_EXPECT_THROW_MESSAGE("parent is not a directory",
dir->tryOpenFile(Path("foo"), WriteMode::CREATE | WriteMode::MODIFY));
KJ_EXPECT_THROW_MESSAGE("parent is not a directory",
dir->tryOpenFile(Path("foo"),
WriteMode::CREATE | WriteMode::MODIFY | WriteMode::CREATE_PARENT));
// Create the directory.
auto subdir = dir->openSubdir(Path("bar"), WriteMode::CREATE);
clock.expectChanged(*dir);
// Link still points to non-existing file so cannot be open in most modes.
KJ_EXPECT(dir->tryOpenFile(Path("foo")) == nullptr);
KJ_EXPECT(dir->tryOpenFile(Path("foo"), WriteMode::CREATE) == nullptr);
KJ_EXPECT(dir->tryOpenFile(Path("foo"), WriteMode::MODIFY) == nullptr);
clock.expectUnchanged(*dir);
// But... CREATE | MODIFY works.
dir->openFile(Path("foo"), WriteMode::CREATE | WriteMode::MODIFY)
->writeAll("foobar");
clock.expectUnchanged(*dir); // Change is only to subdir!
KJ_EXPECT(dir->openFile(Path({"bar", "baz"}))->readAllText() == "foobar");
KJ_EXPECT(dir->openFile(Path("foo"))->readAllText() == "foobar");
KJ_EXPECT(dir->openFile(Path("foo"), WriteMode::MODIFY)->readAllText() == "foobar");
// operations that modify the symlink
dir->symlink(Path("foo"), "corge", WriteMode::MODIFY);
KJ_EXPECT(dir->openFile(Path({"bar", "baz"}))->readAllText() == "foobar");
KJ_EXPECT(dir->readlink(Path("foo")) == "corge");
KJ_EXPECT(!dir->exists(Path("foo")));
KJ_EXPECT(dir->lstat(Path("foo")).type == FsNode::Type::SYMLINK);
KJ_EXPECT(dir->tryOpenFile(Path("foo")) == nullptr);
dir->remove(Path("foo"));
KJ_EXPECT(!dir->exists(Path("foo")));
KJ_EXPECT(dir->tryOpenFile(Path("foo")) == nullptr);
}
KJ_TEST("InMemoryDirectory link") {
TestClock clock;
auto src = newInMemoryDirectory(clock);
auto dst = newInMemoryDirectory(clock);
src->openFile(Path({"foo", "bar"}), WriteMode::CREATE | WriteMode::CREATE_PARENT)
->writeAll("foobar");
src->openFile(Path({"foo", "baz", "qux"}), WriteMode::CREATE | WriteMode::CREATE_PARENT)
->writeAll("bazqux");
clock.expectChanged(*src);
clock.expectUnchanged(*dst);
dst->transfer(Path("link"), WriteMode::CREATE, *src, Path("foo"), TransferMode::LINK);
clock.expectUnchanged(*src);
clock.expectChanged(*dst);
KJ_EXPECT(dst->openFile(Path({"link", "bar"}))->readAllText() == "foobar");
KJ_EXPECT(dst->openFile(Path({"link", "baz", "qux"}))->readAllText() == "bazqux");
KJ_EXPECT(dst->exists(Path({"link", "bar"})));
src->remove(Path({"foo", "bar"}));
KJ_EXPECT(!dst->exists(Path({"link", "bar"})));
}
KJ_TEST("InMemoryDirectory copy") {
TestClock clock;
auto src = newInMemoryDirectory(clock);
auto dst = newInMemoryDirectory(clock);
src->openFile(Path({"foo", "bar"}), WriteMode::CREATE | WriteMode::CREATE_PARENT)
->writeAll("foobar");
src->openFile(Path({"foo", "baz", "qux"}), WriteMode::CREATE | WriteMode::CREATE_PARENT)
->writeAll("bazqux");
clock.expectChanged(*src);
clock.expectUnchanged(*dst);
dst->transfer(Path("link"), WriteMode::CREATE, *src, Path("foo"), TransferMode::COPY);
clock.expectUnchanged(*src);
clock.expectChanged(*dst);
KJ_EXPECT(src->openFile(Path({"foo", "bar"}))->readAllText() == "foobar");
KJ_EXPECT(src->openFile(Path({"foo", "baz", "qux"}))->readAllText() == "bazqux");
KJ_EXPECT(dst->openFile(Path({"link", "bar"}))->readAllText() == "foobar");
KJ_EXPECT(dst->openFile(Path({"link", "baz", "qux"}))->readAllText() == "bazqux");
KJ_EXPECT(dst->exists(Path({"link", "bar"})));
src->remove(Path({"foo", "bar"}));
KJ_EXPECT(dst->openFile(Path({"link", "bar"}))->readAllText() == "foobar");
}
KJ_TEST("InMemoryDirectory move") {
TestClock clock;
auto src = newInMemoryDirectory(clock);
auto dst = newInMemoryDirectory(clock);
src->openFile(Path({"foo", "bar"}), WriteMode::CREATE | WriteMode::CREATE_PARENT)
->writeAll("foobar");
src->openFile(Path({"foo", "baz", "qux"}), WriteMode::CREATE | WriteMode::CREATE_PARENT)
->writeAll("bazqux");
clock.expectChanged(*src);
clock.expectUnchanged(*dst);
dst->transfer(Path("link"), WriteMode::CREATE, *src, Path("foo"), TransferMode::MOVE);
clock.expectChanged(*src);
KJ_EXPECT(!src->exists(Path({"foo"})));
KJ_EXPECT(dst->openFile(Path({"link", "bar"}))->readAllText() == "foobar");
KJ_EXPECT(dst->openFile(Path({"link", "baz", "qux"}))->readAllText() == "bazqux");
}
KJ_TEST("InMemoryDirectory createTemporary") {
TestClock clock;
auto dir = newInMemoryDirectory(clock);
auto file = dir->createTemporary();
file->writeAll("foobar");
KJ_EXPECT(file->readAllText() == "foobar");
KJ_EXPECT(dir->listNames() == nullptr);
}
} // namespace
} // namespace kj
c++/src/kj/filesystem.c++ 0 → 100644
View file @ b1701b88
// Copyright (c) 2015 Sandstorm Development Group, Inc. and contributors
// Licensed under the MIT License:
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
#include "filesystem.h"
#include "vector.h"
#include "debug.h"
#include "one-of.h"
#include <map>
namespace kj {
Path::Path(StringPtr name): Path(heapString(name)) {}
Path::Path(String&& name): parts(heapArray<String>(1)) {
parts[0] = kj::mv(name);
validatePart(parts[0]);
}
Path::Path(ArrayPtr<const StringPtr> parts)
: Path(KJ_MAP(p, parts) { return heapString(p); }) {}
Path::Path(Array<String> partsParam)
: Path(kj::mv(partsParam), ALREADY_CHECKED) {
for (auto& p: parts) {
validatePart(p);
}
}
Path PathPtr::clone() {
return Path(KJ_MAP(p, parts) { return heapString(p); }, Path::ALREADY_CHECKED);
}
Path Path::parse(StringPtr path) {
KJ_REQUIRE(!path.startsWith("/"), "expected a relative path, got absolute", path) {
// When exceptions are disabled, go on -- the leading '/' will end up ignored.
break;
}
return evalImpl(Vector<String>(countParts(path)), path);
}
Path PathPtr::append(Path suffix) const {
auto newParts = kj::heapArrayBuilder<String>(parts.size() + suffix.parts.size());
for (auto& p: parts) newParts.add(heapString(p));
for (auto& p: suffix.parts) newParts.add(kj::mv(p));
return Path(newParts.finish(), Path::ALREADY_CHECKED);
}
Path Path::append(Path suffix) && {
auto newParts = kj::heapArrayBuilder<String>(parts.size() + suffix.parts.size());
for (auto& p: parts) newParts.add(kj::mv(p));
for (auto& p: suffix.parts) newParts.add(kj::mv(p));
return Path(newParts.finish(), ALREADY_CHECKED);
}
Path PathPtr::append(PathPtr suffix) const {
auto newParts = kj::heapArrayBuilder<String>(parts.size() + suffix.parts.size());
for (auto& p: parts) newParts.add(heapString(p));
for (auto& p: suffix.parts) newParts.add(heapString(p));
return Path(newParts.finish(), Path::ALREADY_CHECKED);
}
Path Path::append(PathPtr suffix) && {
auto newParts = kj::heapArrayBuilder<String>(parts.size() + suffix.parts.size());
for (auto& p: parts) newParts.add(kj::mv(p));
for (auto& p: suffix.parts) newParts.add(heapString(p));
return Path(newParts.finish(), ALREADY_CHECKED);
}
Path PathPtr::eval(StringPtr pathText) const {
if (pathText.startsWith("/")) {
// Optimization: avoid copying parts that will just be dropped.
return Path::evalImpl(Vector<String>(Path::countParts(pathText)), pathText);
} else {
Vector<String> newParts(parts.size() + Path::countParts(pathText));
for (auto& p: parts) newParts.add(heapString(p));
return Path::evalImpl(kj::mv(newParts), pathText);
}
}
Path Path::eval(StringPtr pathText) && {
if (pathText.startsWith("/")) {
// Optimization: avoid copying parts that will just be dropped.
return evalImpl(Vector<String>(countParts(pathText)), pathText);
} else {
Vector<String> newParts(parts.size() + countParts(pathText));
for (auto& p: parts) newParts.add(kj::mv(p));
return evalImpl(kj::mv(newParts), pathText);
}
}
PathPtr PathPtr::basename() const {
KJ_REQUIRE(parts.size() > 0, "root path has no basename");
return PathPtr(parts.slice(parts.size() - 1, parts.size()));
}
Path Path::basename() && {
KJ_REQUIRE(parts.size() > 0, "root path has no basename");
auto newParts = kj::heapArrayBuilder<String>(1);
newParts.add(kj::mv(parts[parts.size() - 1]));
return Path(newParts.finish(), ALREADY_CHECKED);
}
PathPtr PathPtr::parent() const {
KJ_REQUIRE(parts.size() > 0, "root path has no parent");
return PathPtr(parts.slice(0, parts.size() - 1));
}
Path Path::parent() && {
KJ_REQUIRE(parts.size() > 0, "root path has no parent");
return Path(KJ_MAP(p, parts.slice(0, parts.size() - 1)) { return kj::mv(p); }, ALREADY_CHECKED);
}
String PathPtr::toString(bool absolute) const {
if (parts.size() == 0) {
// Special-case empty path.
return absolute ? kj::str("/") : kj::str(".");
}
size_t size = absolute + (parts.size() - 1);
for (auto& p: parts) size += p.size();
String result = kj::heapString(size);
char* ptr = result.begin();
bool leadingSlash = absolute;
for (auto& p: parts) {
if (leadingSlash) *ptr++ = '/';
leadingSlash = true;
memcpy(ptr, p.begin(), p.size());
ptr += p.size();
}
KJ_ASSERT(ptr == result.end());
return result;
}
Path Path::slice(size_t start, size_t end) && {
return Path(KJ_MAP(p, parts.slice(start, end)) { return kj::mv(p); });
}
Path PathPtr::evalWin32(StringPtr pathText) const {
Vector<String> newParts(parts.size() + Path::countPartsWin32(pathText));
for (auto& p: parts) newParts.add(heapString(p));
return Path::evalWin32Impl(kj::mv(newParts), pathText);
}
Path Path::evalWin32(StringPtr pathText) && {
Vector<String> newParts(parts.size() + countPartsWin32(pathText));
for (auto& p: parts) newParts.add(kj::mv(p));
return evalWin32Impl(kj::mv(newParts), pathText);
}
String PathPtr::toWin32String(bool absolute) const {
if (parts.size() == 0) {
// Special-case empty path.
KJ_REQUIRE(!absolute, "absolute path is missing disk designator") {
break;
}
return absolute ? kj::str("\\\\") : kj::str(".");
}
bool isUncPath = false;
if (absolute) {
if (Path::isWin32Drive(parts[0])) {
// It's a win32 drive
} else if (Path::isNetbiosName(parts[0])) {
isUncPath = true;
} else {
KJ_FAIL_REQUIRE("absolute win32 path must start with drive letter or netbios host name",
parts[0]);
}
}
size_t size = (isUncPath ? 2 : 0) + (parts.size() - 1);
for (auto& p: parts) size += p.size();
String result = heapString(size);
char* ptr = result.begin();
if (isUncPath) {
*ptr++ = '\\';
*ptr++ = '\\';
}
bool leadingSlash = false;
for (auto& p: parts) {
if (leadingSlash) *ptr++ = '\\';
leadingSlash = true;
KJ_REQUIRE(!Path::isWin32Special(p), "path cannot contain DOS reserved name", p) {
// Recover by blotting out the name with invalid characters which Win32 syscalls will reject.
for (size_t i = 0; i < p.size(); i++) {
*ptr++ = '|';
}
continue;
}
memcpy(ptr, p.begin(), p.size());
ptr += p.size();
}
KJ_ASSERT(ptr == result.end());
// Check for colons (other than in drive letter), which on NTFS would be interpreted as an
// "alternate data stream", which can lead to surprising results. If we want to support ADS, we
// should do so using an explicit API. Note that this check also prevents a relative path from
// appearing to start with a drive letter.
for (size_t i: kj::indices(result)) {
if (result[i] == ':') {
if (absolute && i == 1) {
// False alarm: this is the drive letter.
} else {
KJ_FAIL_REQUIRE(
"colons are prohibited in win32 paths to avoid triggering alterante data streams",
result) {
// Recover by using a different character which we know Win32 syscalls will reject.
result[i] = '|';
}
}
}
}
return result;
}
// -----------------------------------------------------------------------------
String Path::stripNul(String input) {
kj::Vector<char> output(input.size());
for (char c: input) {
if (c != '\0') output.add(c);
}
output.add('\0');
return String(output.releaseAsArray());
}
void Path::validatePart(StringPtr part) {
KJ_REQUIRE(part != "" && part != "." && part != "..", "invalid path component", part);
KJ_REQUIRE(strlen(part.begin()) == part.size(), "NUL character in path component", part);
KJ_REQUIRE(part.findFirst('/') == nullptr,
"'/' character in path component; did you mean to use Path::parse()?", part);
}
void Path::evalPart(Vector<String>& parts, ArrayPtr<const char> part) {
if (part.size() == 0) {
// Ignore consecutive or trailing '/'s.
} else if (part.size() == 1 && part[0] == '.') {
// Refers to current directory; ignore.
} else if (part.size() == 2 && part[0] == '.' && part [1] == '.') {
KJ_REQUIRE(parts.size() > 0, "can't use \"..\" to break out of starting directory") {
// When exceptions are disabled, ignore.
return;
}
parts.removeLast();
} else {
auto str = heapString(part);
KJ_REQUIRE(strlen(str.begin()) == str.size(), "NUL character in path component", str) {
// When exceptions are disabled, strip out '\0' chars.
str = stripNul(kj::mv(str));
break;
}
parts.add(kj::mv(str));
}
}
Path Path::evalImpl(Vector<String>&& parts, StringPtr path) {
if (path.startsWith("/")) {
parts.clear();
}
size_t partStart = 0;
for (auto i: kj::indices(path)) {
if (path[i] == '/') {
evalPart(parts, path.slice(partStart, i));
partStart = i + 1;
}
}
evalPart(parts, path.slice(partStart));
return Path(parts.releaseAsArray(), Path::ALREADY_CHECKED);
}
Path Path::evalWin32Impl(Vector<String>&& parts, StringPtr path) {
// Convert all forward slashes to backslashes.
String ownPath;
if (path.findFirst('/') != nullptr) {
ownPath = heapString(path);
for (char& c: ownPath) {
if (c == '/') c = '\\';
}
path = ownPath;
}
// Interpret various forms of absolute paths.
if (path.startsWith("\\\\")) {
// UNC path.
path = path.slice(2);
// This path is absolute. The first component is a server name.
parts.clear();
} else if (path.startsWith("\\")) {
// Path is relative to the current drive / network share.
if (parts.size() >= 1 && isWin32Drive(parts[0])) {
// Leading \ interpreted as root of current drive.
parts.truncate(1);
} else if (parts.size() >= 2) {
// Leading \ interpreted as root of current network share (which is indicated by the first
// *two* components of the path).
parts.truncate(2);
} else {
KJ_FAIL_REQUIRE("must specify drive letter", path) {
// Recover by assuming C drive.
parts.clear();
parts.add(kj::str("c:"));
break;
}
}
} else if ((path.size() == 2 || (path.size() > 2 && path[2] == '\\')) &&
isWin32Drive(path.slice(0, 2))) {
// Starts with a drive letter.
parts.clear();
}
size_t partStart = 0;
for (auto i: kj::indices(path)) {
if (path[i] == '\\') {
evalPart(parts, path.slice(partStart, i));
partStart = i + 1;
}
}
evalPart(parts, path.slice(partStart));
return Path(parts.releaseAsArray(), Path::ALREADY_CHECKED);
}
size_t Path::countParts(StringPtr path) {
size_t result = 1;
for (char c: path) {
result += (c == '/');
}
return result;
}
size_t Path::countPartsWin32(StringPtr path) {
size_t result = 1;
for (char c: path) {
result += (c == '/' || c == '\\');
}
return result;
}
bool Path::isWin32Drive(ArrayPtr<const char> part) {
return part.size() == 2 && part[1] == ':' &&
(('a' <= part[0] && part[0] <= 'z') || ('A' <= part[0] && part[0] <= 'Z'));
}
bool Path::isNetbiosName(ArrayPtr<const char> part) {
// Characters must be alphanumeric or '.' or '-'.
for (char c: part) {
if (c != '.' && c != '-' &&
(c < 'a' || 'z' < c) &&
(c < 'A' || 'Z' < c) &&
(c < '0' || '9' < c)) {
return false;
}
}
// Can't be empty nor start or end with a '.' or a '-'.
return part.size() > 0 &&
part[0] != '.' && part[0] != '-' &&
part[part.size() - 1] != '.' && part[part.size() - 1] != '-';
}
bool Path::isWin32Special(StringPtr part) {
bool isNumbered;
if (part.size() == 3 || (part.size() > 3 && part[3] == '.')) {
// Filename is three characters or three characters followed by an extension.
isNumbered = false;
} else if ((part.size() == 4 || (part.size() > 4 && part[4] == '.')) &&
'1' <= part[3] && part[3] <= '9') {
// Filename is four characters or four characters followed by an extension, and the fourth
// character is a nonzero digit.
isNumbered = true;
} else {
return false;
}
// OK, this could be a Win32 special filename. We need to match the first three letters against
// the list of specials, case-insensitively.
char tmp[4];
memcpy(tmp, part.begin(), 3);
tmp[3] = '\0';
for (char& c: tmp) {
if ('A' <= c && c <= 'Z') {
c += 'a' - 'A';
}
}
StringPtr str(tmp, 3);
if (isNumbered) {
// Specials that are followed by a digit.
return str == "com" || str == "lpt";
} else {
// Specials that are not followed by a digit.
return str == "con" || str == "prn" || str == "aux" || str == "nul";
}
}
// =======================================================================================
String ReadableFile::readAllText() {
String result = heapString(stat().size);
size_t n = read(0, result.asBytes());
if (n < result.size()) {
// Apparently file was truncated concurrently. Reduce to new size to match.
result = heapString(result.slice(0, n));
}
return result;
}
Array<byte> ReadableFile::readAllBytes() {
Array<byte> result = heapArray<byte>(stat().size);
size_t n = read(0, result.asBytes());
if (n < result.size()) {
// Apparently file was truncated concurrently. Reduce to new size to match.
result = heapArray(result.slice(0, n));
}
return result;
}
void File::writeAll(ArrayPtr<const byte> bytes) {
truncate(0);
write(0, bytes);
}
void File::writeAll(StringPtr text) {
writeAll(text.asBytes());
}
size_t File::copy(uint64_t offset, ReadableFile& from, uint64_t fromOffset, uint64_t size) {
byte buffer[8192];
size_t result = 0;
while (size > 0) {
size_t n = from.read(fromOffset, kj::arrayPtr(buffer, kj::min(sizeof(buffer), size)));
write(offset, arrayPtr(buffer, n));
result += n;
if (n < sizeof(buffer)) {
// Either we copied the amount requested or we hit EOF.
break;
}
fromOffset += n;
offset += n;
size -= n;
}
return result;
}
FsNode::Metadata ReadableDirectory::lstat(PathPtr path) {
KJ_IF_MAYBE(meta, tryLstat(path)) {
return *meta;
} else {
KJ_FAIL_REQUIRE("no such file", path) { break; }
return FsNode::Metadata();
}
}
Own<ReadableFile> ReadableDirectory::openFile(PathPtr path) {
KJ_IF_MAYBE(file, tryOpenFile(path)) {
return kj::mv(*file);
} else {
KJ_FAIL_REQUIRE("no such directory", path) { break; }
return newInMemoryFile(nullClock());
}
}
Own<ReadableDirectory> ReadableDirectory::openSubdir(PathPtr path) {
KJ_IF_MAYBE(dir, tryOpenSubdir(path)) {
return kj::mv(*dir);
} else {
KJ_FAIL_REQUIRE("no such file or directory", path) { break; }
return newInMemoryDirectory(nullClock());
}
}
String ReadableDirectory::readlink(PathPtr path) {
KJ_IF_MAYBE(p, tryReadlink(path)) {
return kj::mv(*p);
} else {
KJ_FAIL_REQUIRE("not a symlink", path) { break; }
return kj::str(".");
}
}
Own<File> Directory::openFile(PathPtr path, WriteMode mode) {
KJ_IF_MAYBE(f, tryOpenFile(path, mode)) {
return kj::mv(*f);
} else if (has(mode, WriteMode::CREATE) && !has(mode, WriteMode::MODIFY)) {
KJ_FAIL_REQUIRE("file already exists", path) { break; }
} else if (has(mode, WriteMode::MODIFY) && !has(mode, WriteMode::CREATE)) {
KJ_FAIL_REQUIRE("file does not exist", path) { break; }
} else if (!has(mode, WriteMode::MODIFY) && !has(mode, WriteMode::CREATE)) {
KJ_FAIL_ASSERT("neither WriteMode::CREATE nor WriteMode::MODIFY was given", path) { break; }
} else {
// Shouldn't happen.
KJ_FAIL_ASSERT("tryOpenFile() returned null despite no preconditions", path) { break; }
}
return newInMemoryFile(nullClock());
}
Own<AppendableFile> Directory::appendFile(PathPtr path, WriteMode mode) {
KJ_IF_MAYBE(f, tryAppendFile(path, mode)) {
return kj::mv(*f);
} else if (has(mode, WriteMode::CREATE) && !has(mode, WriteMode::MODIFY)) {
KJ_FAIL_REQUIRE("file already exists", path) { break; }
} else if (has(mode, WriteMode::MODIFY) && !has(mode, WriteMode::CREATE)) {
KJ_FAIL_REQUIRE("file does not exist", path) { break; }
} else if (!has(mode, WriteMode::MODIFY) && !has(mode, WriteMode::CREATE)) {
KJ_FAIL_ASSERT("neither WriteMode::CREATE nor WriteMode::MODIFY was given", path) { break; }
} else {
// Shouldn't happen.
KJ_FAIL_ASSERT("tryAppendFile() returned null despite no preconditions", path) { break; }
}
return newFileAppender(newInMemoryFile(nullClock()));
}
Own<Directory> Directory::openSubdir(PathPtr path, WriteMode mode) {
KJ_IF_MAYBE(f, tryOpenSubdir(path, mode)) {
return kj::mv(*f);
} else if (has(mode, WriteMode::CREATE) && !has(mode, WriteMode::MODIFY)) {
KJ_FAIL_REQUIRE("directory already exists", path) { break; }
} else if (has(mode, WriteMode::MODIFY) && !has(mode, WriteMode::CREATE)) {
KJ_FAIL_REQUIRE("directory does not exist", path) { break; }
} else if (!has(mode, WriteMode::MODIFY) && !has(mode, WriteMode::CREATE)) {
KJ_FAIL_ASSERT("neither WriteMode::CREATE nor WriteMode::MODIFY was given", path) { break; }
} else {
// Shouldn't happen.
KJ_FAIL_ASSERT("tryOpenSubdir() returned null despite no preconditions", path) { break; }
}
return newInMemoryDirectory(nullClock());
}
void Directory::symlink(PathPtr linkpath, StringPtr content, WriteMode mode) {
if (!trySymlink(linkpath, content, mode)) {
if (has(mode, WriteMode::CREATE)) {
KJ_FAIL_REQUIRE("path already exsits", linkpath) { break; }
} else {
// Shouldn't happen.
KJ_FAIL_ASSERT("symlink() returned null despite no preconditions", linkpath) { break; }
}
}
}
void Directory::transfer(PathPtr toPath, WriteMode toMode,
Directory& fromDirectory, PathPtr fromPath,
TransferMode mode) {
if (!tryTransfer(toPath, toMode, fromDirectory, fromPath, mode)) {
if (has(toMode, WriteMode::CREATE)) {
KJ_FAIL_REQUIRE("toPath already exists or fromPath doesn't exist", toPath, fromPath) {
break;
}
} else {
KJ_FAIL_ASSERT("fromPath doesn't exist", fromPath) { break; }
}
}
}
static void copyContents(Directory& to, ReadableDirectory& from);
static bool tryCopyDirectoryEntry(Directory& to, PathPtr toPath, WriteMode toMode,
ReadableDirectory& from, PathPtr fromPath,
FsNode::Type type, bool atomic) {
// TODO(cleanup): Make this reusable?
switch (type) {
case FsNode::Type::FILE: {
KJ_IF_MAYBE(fromFile, from.tryOpenFile(fromPath)) {
if (atomic) {
auto replacer = to.replaceFile(toPath, toMode);
replacer->get().copy(0, **fromFile, 0, kj::maxValue);
return replacer->tryCommit();
} else KJ_IF_MAYBE(toFile, to.tryOpenFile(toPath, toMode)) {
toFile->get()->copy(0, **fromFile, 0, kj::maxValue);
return true;
} else {
return false;
}
} else {
// Apparently disappeared. Treat as source-doesn't-exist.
return false;
}
}
case FsNode::Type::DIRECTORY:
KJ_IF_MAYBE(fromSubdir, from.tryOpenSubdir(fromPath)) {
if (atomic) {
auto replacer = to.replaceSubdir(toPath, toMode);
copyContents(replacer->get(), **fromSubdir);
return replacer->tryCommit();
} else KJ_IF_MAYBE(toSubdir, to.tryOpenSubdir(toPath, toMode)) {
copyContents(**toSubdir, **fromSubdir);
return true;
} else {
return false;
}
} else {
// Apparently disappeared. Treat as source-doesn't-exist.
return false;
}
case FsNode::Type::SYMLINK:
KJ_IF_MAYBE(content, from.tryReadlink(fromPath)) {
return to.trySymlink(toPath, *content, toMode);
} else {
// Apparently disappeared. Treat as source-doesn't-exist.
return false;
}
break;
default:
// Note: Unclear whether it's better to throw an error here or just ignore it / log a
// warning. Can reconsider when we see an actual use case.
KJ_FAIL_REQUIRE("can only copy files, directories, and symlinks", fromPath) {
return false;
}
}
}
static void copyContents(Directory& to, ReadableDirectory& from) {
for (auto& entry: from.listEntries()) {
Path subPath(kj::mv(entry.name));
tryCopyDirectoryEntry(to, subPath, WriteMode::CREATE, from, subPath, entry.type, false);
}
}
bool Directory::tryTransfer(PathPtr toPath, WriteMode toMode,
Directory& fromDirectory, PathPtr fromPath,
TransferMode mode) {
KJ_REQUIRE(toPath.size() > 0, "can't replace self") { return false; }
// First try reversing.
KJ_IF_MAYBE(result, fromDirectory.tryTransferTo(*this, toPath, toMode, fromPath, mode)) {
return *result;
}
switch (mode) {
case TransferMode::COPY:
KJ_IF_MAYBE(meta, fromDirectory.tryLstat(fromPath)) {
return tryCopyDirectoryEntry(*this, toPath, toMode, fromDirectory,
fromPath, meta->type, true);
} else {
// Source doesn't exist.
return false;
}
case TransferMode::MOVE:
// Implement move as copy-then-delete.
if (!tryTransfer(toPath, toMode, fromDirectory, fromPath, TransferMode::COPY)) {
return false;
}
fromDirectory.remove(fromPath);
return true;
case TransferMode::LINK:
KJ_FAIL_REQUIRE("can't link across different Directory implementations") { return false; }
}
}
Maybe<bool> Directory::tryTransferTo(Directory& toDirectory, PathPtr toPath, WriteMode toMode,
PathPtr fromPath, TransferMode mode) {
return nullptr;
}
void Directory::remove(PathPtr path) {
if (!tryRemove(path)) {
KJ_FAIL_REQUIRE("path to remove doesn't exist", path) { break; }
}
}
void Directory::commitFailed(WriteMode mode) {
if (has(mode, WriteMode::CREATE) && !has(mode, WriteMode::MODIFY)) {
KJ_FAIL_REQUIRE("replace target already exists") { break; }
} else if (has(mode, WriteMode::MODIFY) && !has(mode, WriteMode::CREATE)) {
KJ_FAIL_REQUIRE("replace target does not exist") { break; }
} else if (!has(mode, WriteMode::MODIFY) && !has(mode, WriteMode::CREATE)) {
KJ_FAIL_ASSERT("neither WriteMode::CREATE nor WriteMode::MODIFY was given") { break; }
} else {
KJ_FAIL_ASSERT("tryCommit() returned null despite no preconditions") { break; }
}
}
// =======================================================================================
namespace {
class InMemoryFile final: public File, public Refcounted {
public:
InMemoryFile(Clock& clock): clock(clock), lastModified(clock.now()) {}
Own<FsNode> cloneFsNode() override {
return addRef(*this);
}
Maybe<int> getFd() override {
return nullptr;
}
Metadata stat() override {
return Metadata { Type::FILE, size, size, lastModified, 1 };
}
void sync() override {}
void datasync() override {}
// no-ops
size_t read(uint64_t offset, ArrayPtr<byte> buffer) override {
if (offset >= size) {
// Entirely out-of-range.
return 0;
}
size_t readSize = kj::min(buffer.size(), size - offset);
memcpy(buffer.begin(), bytes.begin() + offset, readSize);
return readSize;
}
Array<const byte> mmap(uint64_t offset, uint64_t size) override {
KJ_REQUIRE(offset + size >= offset, "mmap() request overflows uint64");
ensureCapacity(offset + size);
ArrayDisposer* disposer = new MmapDisposer(addRef(*this));
return Array<const byte>(bytes.begin() + offset, size, *disposer);
}
Array<byte> mmapPrivate(uint64_t offset, uint64_t size) override {
// Return a copy.
// Allocate exactly the size requested.
auto result = heapArray<byte>(size);
// Use read() to fill it.
size_t actual = read(offset, result);
// Ignore the rest.
if (actual < size) {
memset(result.begin() + actual, 0, size - actual);
}
return result;
}
void write(uint64_t offset, ArrayPtr<const byte> data) override {
if (data.size() == 0) return;
modified();
uint64_t end = offset + data.size();
KJ_REQUIRE(end >= offset, "write() request overflows uint64");
ensureCapacity(end);
size = kj::max(size, end);
memcpy(bytes.begin() + offset, data.begin(), data.size());
}
void zero(uint64_t offset, uint64_t zeroSize) override {
if (zeroSize == 0) return;
modified();
uint64_t end = offset + zeroSize;
KJ_REQUIRE(end >= offset, "zero() request overflows uint64");
ensureCapacity(end);
size = kj::max(size, end);
memset(bytes.begin() + offset, 0, zeroSize);
}
void truncate(uint64_t newSize) override {
if (newSize < size) {
modified();
memset(bytes.begin() + newSize, 0, size - newSize);
size = newSize;
} else if (newSize > size) {
modified();
ensureCapacity(newSize);
size = newSize;
}
}
Own<WritableFileMapping> mmapWritable(uint64_t offset, uint64_t size) override {
uint64_t end = offset + size;
KJ_REQUIRE(end >= offset, "mmapWritable() request overflows uint64");
ensureCapacity(end);
return heap<WritableFileMappingImpl>(addRef(*this), bytes.slice(offset, end));
}
size_t copy(uint64_t offset, ReadableFile& from,
uint64_t fromOffset, uint64_t copySize) override {
size_t fromFileSize = from.stat().size;
if (fromFileSize <= fromOffset) return 0;
// Clamp size to EOF.
copySize = kj::min(copySize, fromFileSize - fromOffset);
if (copySize == 0) return 0;
// Allocate space for the copy.
uint64_t end = offset + copySize;
ensureCapacity(end);
// Read directly into our backing store.
size_t n = from.read(fromOffset, bytes.slice(offset, end));
size = kj::max(size, offset + n);
modified();
return n;
}
private:
Clock& clock;
Array<byte> bytes;
size_t size = 0; // bytes may be larger than this to accommodate mmaps
Date lastModified;
uint mmapCount = 0; // number of mappings outstanding
void ensureCapacity(size_t capacity) {
if (bytes.size() < capacity) {
KJ_ASSERT(mmapCount == 0,
"InMemoryFile cannot resize the file backing store while memory mappings exist.");
auto newBytes = heapArray<byte>(kj::max(capacity, bytes.size() * 2));
memcpy(newBytes.begin(), bytes.begin(), size);
memset(newBytes.begin() + size, 0, newBytes.size() - size);
bytes = kj::mv(newBytes);
}
}
void modified() {
lastModified = clock.now();
}
class MmapDisposer final: public ArrayDisposer {
public:
MmapDisposer(Own<InMemoryFile>&& refParam): ref(kj::mv(refParam)) {
++ref->mmapCount;
}
~MmapDisposer() noexcept(false) {
--ref->mmapCount;
}
void disposeImpl(void* firstElement, size_t elementSize, size_t elementCount,
size_t capacity, void (*destroyElement)(void*)) const override {
delete this;
}
private:
Own<InMemoryFile> ref;
};
class WritableFileMappingImpl final: public WritableFileMapping {
public:
WritableFileMappingImpl(Own<InMemoryFile>&& refParam, ArrayPtr<byte> range)
: ref(kj::mv(refParam)), range(range) {
++ref->mmapCount;
}
~WritableFileMappingImpl() noexcept(false) {
--ref->mmapCount;
}
ArrayPtr<byte> get() override {
return range;
}
void changed(ArrayPtr<byte> slice) override {
ref->modified();
}
void sync(ArrayPtr<byte> slice) override {
ref->modified();
}
private:
Own<InMemoryFile> ref;
ArrayPtr<byte> range;
};
};
// -----------------------------------------------------------------------------
class InMemoryDirectory final: public Directory, public Refcounted {
public:
InMemoryDirectory(Clock& clock)
: clock(clock), lastModified(clock.now()) {}
Own<FsNode> cloneFsNode() override {
return addRef(*this);
}
Maybe<int> getFd() override {
return nullptr;
}
Metadata stat() override {
return Metadata { Type::DIRECTORY, 0, 0, lastModified, 1 };
}
void sync() override {}
void datasync() override {}
// no-ops
Array<String> listNames() override {
return KJ_MAP(e, entries) { return heapString(e.first); };
}
Array<Entry> listEntries() override {
return KJ_MAP(e, entries) {
FsNode::Type type;
if (e.second.node.is<SymlinkNode>()) {
type = FsNode::Type::SYMLINK;
} else if (e.second.node.is<FileNode>()) {
type = FsNode::Type::FILE;
} else {
KJ_ASSERT(e.second.node.is<DirectoryNode>());
type = FsNode::Type::DIRECTORY;
}
return Entry { type, heapString(e.first) };
};
}
bool exists(PathPtr path) override {
if (path.size() == 0) {
return true;
} else if (path.size() == 1) {
KJ_IF_MAYBE(entry, tryGetEntry(path[0])) {
return exists(*entry);
} else {
return false;
}
} else {
KJ_IF_MAYBE(subdir, tryGetParent(path[0])) {
return subdir->get()->exists(path.slice(1, path.size()));
} else {
return false;
}
}
}
Maybe<FsNode::Metadata> tryLstat(PathPtr path) override {
if (path.size() == 0) {
return stat();
} else if (path.size() == 1) {
KJ_IF_MAYBE(entry, tryGetEntry(path[0])) {
if (entry->node.is<FileNode>()) {
return entry->node.get<FileNode>().file->stat();
} else if (entry->node.is<DirectoryNode>()) {
return entry->node.get<DirectoryNode>().directory->stat();
} else if (entry->node.is<SymlinkNode>()) {
auto& link = entry->node.get<SymlinkNode>();
return FsNode::Metadata { FsNode::Type::SYMLINK, 0, 0, link.lastModified, 1 };
} else {
KJ_FAIL_ASSERT("unknown node type") { return nullptr; }
}
} else {
return nullptr;
}
} else {
KJ_IF_MAYBE(subdir, tryGetParent(path[0])) {
return subdir->get()->tryLstat(path.slice(1, path.size()));
} else {
return nullptr;
}
}
}
Maybe<Own<ReadableFile>> tryOpenFile(PathPtr path) override {
if (path.size() == 0) {
KJ_FAIL_REQUIRE("not a file") { return nullptr; }
} else if (path.size() == 1) {
KJ_IF_MAYBE(entry, tryGetEntry(path[0])) {
return asFile(*entry);
} else {
return nullptr;
}
} else {
KJ_IF_MAYBE(subdir, tryGetParent(path[0])) {
return subdir->get()->tryOpenFile(path.slice(1, path.size()));
} else {
return nullptr;
}
}
}
Maybe<Own<ReadableDirectory>> tryOpenSubdir(PathPtr path) override {
if (path.size() == 0) {
return clone();
} else if (path.size() == 1) {
KJ_IF_MAYBE(entry, tryGetEntry(path[0])) {
return asDirectory(*entry);
} else {
return nullptr;
}
} else {
KJ_IF_MAYBE(subdir, tryGetParent(path[0])) {
return subdir->get()->tryOpenSubdir(path.slice(1, path.size()));
} else {
return nullptr;
}
}
}
Maybe<String> tryReadlink(PathPtr path) override {
if (path.size() == 0) {
KJ_FAIL_REQUIRE("not a symlink") { return nullptr; }
} else if (path.size() == 1) {
KJ_IF_MAYBE(entry, tryGetEntry(path[0])) {
return asSymlink(*entry);
} else {
return nullptr;
}
} else {
KJ_IF_MAYBE(subdir, tryGetParent(path[0])) {
return subdir->get()->tryReadlink(path.slice(1, path.size()));
} else {
return nullptr;
}
}
}
Maybe<Own<File>> tryOpenFile(PathPtr path, WriteMode mode) override {
if (path.size() == 0) {
if (has(mode, WriteMode::MODIFY)) {
KJ_FAIL_REQUIRE("not a file") { return nullptr; }
} else if (has(mode, WriteMode::CREATE)) {
return nullptr; // already exists (as a directory)
} else {
KJ_FAIL_REQUIRE("can't replace self") { return nullptr; }
}
} else if (path.size() == 1) {
KJ_IF_MAYBE(entry, openEntry(path[0], mode)) {
return asFile(*entry, mode);
} else {
return nullptr;
}
} else {
KJ_IF_MAYBE(child, tryGetParent(path[0], mode)) {
return child->get()->tryOpenFile(path.slice(1, path.size()), mode);
} else {
return nullptr;
}
}
}
Own<Replacer<File>> replaceFile(PathPtr path, WriteMode mode) override {
if (path.size() == 0) {
KJ_FAIL_REQUIRE("can't replace self") { break; }
} else if (path.size() == 1) {
return heap<ReplacerImpl<File>>(*this, path[0], newInMemoryFile(clock), mode);
} else {
KJ_IF_MAYBE(child, tryGetParent(path[0], mode)) {
return child->get()->replaceFile(path.slice(1, path.size()), mode);
}
}
return heap<BrokenReplacer<File>>(newInMemoryFile(clock));
}
Maybe<Own<Directory>> tryOpenSubdir(PathPtr path, WriteMode mode) override {
if (path.size() == 0) {
if (has(mode, WriteMode::MODIFY)) {
return addRef(*this);
} else if (has(mode, WriteMode::CREATE)) {
return nullptr; // already exists
} else {
KJ_FAIL_REQUIRE("can't replace self") { return nullptr; }
}
} else if (path.size() == 1) {
KJ_IF_MAYBE(entry, openEntry(path[0], mode)) {
return asDirectory(*entry, mode);
} else {
return nullptr;
}
} else {
KJ_IF_MAYBE(child, tryGetParent(path[0], mode)) {
return child->get()->tryOpenSubdir(path.slice(1, path.size()), mode);
} else {
return nullptr;
}
}
}
Own<Replacer<Directory>> replaceSubdir(PathPtr path, WriteMode mode) override {
if (path.size() == 0) {
KJ_FAIL_REQUIRE("can't replace self") { break; }
} else if (path.size() == 1) {
return heap<ReplacerImpl<Directory>>(*this, path[0], newInMemoryDirectory(clock), mode);
} else {
KJ_IF_MAYBE(child, tryGetParent(path[0], mode)) {
return child->get()->replaceSubdir(path.slice(1, path.size()), mode);
}
}
return heap<BrokenReplacer<Directory>>(newInMemoryDirectory(clock));
}
Maybe<Own<AppendableFile>> tryAppendFile(PathPtr path, WriteMode mode) override {
if (path.size() == 0) {
if (has(mode, WriteMode::MODIFY)) {
KJ_FAIL_REQUIRE("not a file") { return nullptr; }
} else if (has(mode, WriteMode::CREATE)) {
return nullptr; // already exists (as a directory)
} else {
KJ_FAIL_REQUIRE("can't replace self") { return nullptr; }
}
} else if (path.size() == 1) {
KJ_IF_MAYBE(entry, openEntry(path[0], mode)) {
return asFile(*entry, mode).map(newFileAppender);
} else {
return nullptr;
}
} else {
KJ_IF_MAYBE(child, tryGetParent(path[0], mode)) {
return child->get()->tryAppendFile(path.slice(1, path.size()), mode);
} else {
return nullptr;
}
}
}
bool trySymlink(PathPtr path, StringPtr content, WriteMode mode) override {
if (path.size() == 0) {
if (has(mode, WriteMode::CREATE)) {
return false;
} else {
KJ_FAIL_REQUIRE("can't replace self") { return false; }
}
} else if (path.size() == 1) {
KJ_IF_MAYBE(entry, openEntry(path[0], mode)) {
entry->init(SymlinkNode { clock.now(), heapString(content) });
modified();
return true;
} else {
return false;
}
} else {
KJ_IF_MAYBE(child, tryGetParent(path[0], mode)) {
return child->get()->trySymlink(path.slice(1, path.size()), content, mode);
} else {
KJ_FAIL_REQUIRE("couldn't create parent directory") { return false; }
}
}
}
Own<File> createTemporary() override {
return newInMemoryFile(clock);
}
bool tryTransfer(PathPtr toPath, WriteMode toMode,
Directory& fromDirectory, PathPtr fromPath, TransferMode mode) override {
if (toPath.size() == 0) {
if (has(toMode, WriteMode::CREATE)) {
return false;
} else {
KJ_FAIL_REQUIRE("can't replace self") { return false; }
}
} else if (toPath.size() == 1) {
// tryTransferChild() needs to at least know the node type, so do an lstat.
KJ_IF_MAYBE(meta, fromDirectory.tryLstat(fromPath)) {
KJ_IF_MAYBE(entry, openEntry(toPath[0], toMode)) {
// Make sure if we just cerated a new entry, and we don't successfully transfer to it, we
// remove the entry before returning.
bool needRollback = entry->node == nullptr;
KJ_DEFER(if (needRollback) { entries.erase(toPath[0]); });
if (tryTransferChild(*entry, meta->type, meta->lastModified, meta->size,
fromDirectory, fromPath, mode)) {
modified();
needRollback = false;
return true;
} else {
KJ_FAIL_REQUIRE("InMemoryDirectory can't link an inode of this type", fromPath) {
return false;
}
}
} else {
return false;
}
} else {
return false;
}
} else {
// TODO(someday): Ideally we wouldn't create parent directories if fromPath doesn't exist.
// This requires a different approach to the code here, though.
KJ_IF_MAYBE(child, tryGetParent(toPath[0], toMode)) {
return child->get()->tryTransfer(
toPath.slice(1, toPath.size()), toMode, fromDirectory, fromPath, mode);
} else {
return false;
}
}
}
Maybe<bool> tryTransferTo(Directory& toDirectory, PathPtr toPath, WriteMode toMode,
PathPtr fromPath, TransferMode mode) override {
if (fromPath.size() <= 1) {
// If `fromPath` is in this directory (or *is* this directory) then we don't have any
// optimizations.
return nullptr;
}
// `fromPath` is in a subdirectory. It could turn out that that subdirectory is not an
// InMemoryDirectory and is instead something `toDirectory` is friendly with. So let's follow
// the path.
KJ_IF_MAYBE(child, tryGetParent(fromPath[0], WriteMode::MODIFY)) {
// OK, switch back to tryTransfer() but use the subdirectory.
return toDirectory.tryTransfer(toPath, toMode,
**child, fromPath.slice(1, fromPath.size()), mode);
} else {
// Hmm, doesn't exist. Fall back to standard path.
return nullptr;
}
}
bool tryRemove(PathPtr path) override {
if (path.size() == 0) {
KJ_FAIL_REQUIRE("can't remove self from self") { return false; }
} else if (path.size() == 1) {
auto iter = entries.find(path[0]);
if (iter == entries.end()) {
return false;
} else {
entries.erase(iter);
modified();
return true;
}
} else {
KJ_IF_MAYBE(child, tryGetParent(path[0], WriteMode::MODIFY)) {
return child->get()->tryRemove(path.slice(1, path.size()));
} else {
return false;
}
}
}
private:
struct FileNode {
Own<File> file;
};
struct DirectoryNode {
Own<Directory> directory;
};
struct SymlinkNode {
Date lastModified;
String content;
Path parse() {
KJ_CONTEXT("parsing symlink", content);
return Path::parse(content);
}
};
struct EntryImpl {
String name;
OneOf<FileNode, DirectoryNode, SymlinkNode> node;
EntryImpl(String&& name): name(kj::mv(name)) {}
Own<File> init(FileNode&& value) {
return node.init<FileNode>(kj::mv(value)).file->clone();
}
Own<Directory> init(DirectoryNode&& value) {
return node.init<DirectoryNode>(kj::mv(value)).directory->clone();
}
void init(SymlinkNode&& value) {
node.init<SymlinkNode>(kj::mv(value));
}
bool init(OneOf<FileNode, DirectoryNode, SymlinkNode>&& value) {
node = kj::mv(value);
return node != nullptr;
}
void set(Own<File>&& value) {
node.init<FileNode>(FileNode { kj::mv(value) });
}
void set(Own<Directory>&& value) {
node.init<DirectoryNode>(DirectoryNode { kj::mv(value) });
}
};
Clock& clock;
std::map<StringPtr, EntryImpl> entries;
Date lastModified;
template <typename T>
class ReplacerImpl: public Replacer<T> {
public:
ReplacerImpl(InMemoryDirectory& directory, kj::StringPtr name, Own<T> inner, WriteMode mode)
: Replacer<T>(mode), directory(addRef(directory)), name(heapString(name)),
inner(kj::mv(inner)) {}
T& get() override { return *inner; }
bool tryCommit() override {
KJ_REQUIRE(!committed, "commit() already called") { return true; }
KJ_IF_MAYBE(entry, directory->openEntry(name, Replacer<T>::mode)) {
entry->set(inner->clone());
directory->modified();
return true;
} else {
return false;
}
}
private:
bool committed = false;
Own<InMemoryDirectory> directory;
kj::String name;
Own<T> inner;
};
template <typename T>
class BrokenReplacer: public Replacer<T> {
// For recovery path when exceptions are disabled.
public:
BrokenReplacer(Own<T> inner)
: Replacer<T>(WriteMode::CREATE | WriteMode::MODIFY),
inner(kj::mv(inner)) {}
T& get() override { return *inner; }
bool tryCommit() override { return false; }
private:
Own<T> inner;
};
Maybe<EntryImpl&> openEntry(kj::StringPtr name, WriteMode mode) {
// TODO(perf): We could avoid a copy if the entry exists, at the expense of a double-lookup if
// it doesn't. Maybe a better map implementation will solve everything?
return openEntry(heapString(name), mode);
}
Maybe<EntryImpl&> openEntry(String&& name, WriteMode mode) {
if (has(mode, WriteMode::CREATE)) {
EntryImpl entry(kj::mv(name));
StringPtr nameRef = entry.name;
auto insertResult = entries.insert(std::make_pair(nameRef, kj::mv(entry)));
if (!insertResult.second && !has(mode, WriteMode::MODIFY)) {
// Entry already existed and MODIFY not specified.
return nullptr;
}
return insertResult.first->second;
} else if (has(mode, WriteMode::MODIFY)) {
return tryGetEntry(name);
} else {
// Neither CREATE nor MODIFY specified: precondition always fails.
return nullptr;
}
}
kj::Maybe<EntryImpl&> tryGetEntry(kj::StringPtr name) {
auto iter = entries.find(name);
if (iter == entries.end()) {
return nullptr;
} else {
return iter->second;
}
}
kj::Maybe<Own<ReadableDirectory>> tryGetParent(kj::StringPtr name) {
KJ_IF_MAYBE(entry, tryGetEntry(name)) {
return asDirectory(*entry);
} else {
return nullptr;
}
}
kj::Maybe<Own<Directory>> tryGetParent(kj::StringPtr name, WriteMode mode) {
// Get a directory which is a parent of the eventual target. If `mode` includes
// WriteMode::CREATE_PARENTS, possibly create the parent directory.
WriteMode parentMode = has(mode, WriteMode::CREATE) && has(mode, WriteMode::CREATE_PARENT)
? WriteMode::CREATE | WriteMode::MODIFY // create parent
: WriteMode::MODIFY; // don't create parent
// Possibly create parent.
KJ_IF_MAYBE(entry, openEntry(name, parentMode)) {
if (entry->node.is<DirectoryNode>()) {
return entry->node.get<DirectoryNode>().directory->clone();
} else if (entry->node == nullptr) {
modified();
return entry->init(DirectoryNode { newInMemoryDirectory(clock) });
}
// Continue on.
}
if (has(mode, WriteMode::CREATE)) {
// CREATE is documented as returning null when the file already exists. In this case, the
// file does NOT exist because the parent directory does not exist or is not a directory.
KJ_FAIL_REQUIRE("parent is not a directory") { return nullptr; }
} else {
return nullptr;
}
}
bool exists(EntryImpl& entry) {
if (entry.node.is<SymlinkNode>()) {
return exists(entry.node.get<SymlinkNode>().parse());
} else {
return true;
}
}
Maybe<Own<ReadableFile>> asFile(EntryImpl& entry) {
if (entry.node.is<FileNode>()) {
return entry.node.get<FileNode>().file->clone();
} else if (entry.node.is<SymlinkNode>()) {
return tryOpenFile(entry.node.get<SymlinkNode>().parse());
} else {
KJ_FAIL_REQUIRE("not a file") { return nullptr; }
}
}
Maybe<Own<ReadableDirectory>> asDirectory(EntryImpl& entry) {
if (entry.node.is<DirectoryNode>()) {
return entry.node.get<DirectoryNode>().directory->clone();
} else if (entry.node.is<SymlinkNode>()) {
return tryOpenSubdir(entry.node.get<SymlinkNode>().parse());
} else {
KJ_FAIL_REQUIRE("not a directory") { return nullptr; }
}
}
Maybe<String> asSymlink(EntryImpl& entry) {
if (entry.node.is<SymlinkNode>()) {
return heapString(entry.node.get<SymlinkNode>().content);
} else {
KJ_FAIL_REQUIRE("not a symlink") { return nullptr; }
}
}
Maybe<Own<File>> asFile(EntryImpl& entry, WriteMode mode) {
if (entry.node.is<FileNode>()) {
return entry.node.get<FileNode>().file->clone();
} else if (entry.node.is<SymlinkNode>()) {
// CREATE_PARENT doesn't apply to creating the parents of a symlink target. However, the
// target itself can still be created.
return tryOpenFile(entry.node.get<SymlinkNode>().parse(), mode - WriteMode::CREATE_PARENT);
} else if (entry.node == nullptr) {
KJ_ASSERT(has(mode, WriteMode::CREATE));
modified();
return entry.init(FileNode { newInMemoryFile(clock) });
} else {
KJ_FAIL_REQUIRE("not a file") { return nullptr; }
}
}
Maybe<Own<Directory>> asDirectory(EntryImpl& entry, WriteMode mode) {
if (entry.node.is<DirectoryNode>()) {
return entry.node.get<DirectoryNode>().directory->clone();
} else if (entry.node.is<SymlinkNode>()) {
// CREATE_PARENT doesn't apply to creating the parents of a symlink target. However, the
// target itself can still be created.
return tryOpenSubdir(entry.node.get<SymlinkNode>().parse(), mode - WriteMode::CREATE_PARENT);
} else if (entry.node == nullptr) {
KJ_ASSERT(has(mode, WriteMode::CREATE));
modified();
return entry.init(DirectoryNode { newInMemoryDirectory(clock) });
} else {
KJ_FAIL_REQUIRE("not a directory") { return nullptr; }
}
}
void modified() {
lastModified = clock.now();
}
bool tryTransferChild(EntryImpl& entry, const FsNode::Type type, kj::Maybe<Date> lastModified,
kj::Maybe<uint64_t> size, Directory& fromDirectory, PathPtr fromPath,
TransferMode mode) {
switch (type) {
case FsNode::Type::FILE:
KJ_IF_MAYBE(file, fromDirectory.tryOpenFile(fromPath, WriteMode::MODIFY)) {
if (mode == TransferMode::COPY) {
auto copy = newInMemoryFile(clock);
copy->copy(0, **file, 0, size.orDefault(kj::maxValue));
entry.set(kj::mv(copy));
} else {
if (mode == TransferMode::MOVE) {
KJ_ASSERT(fromDirectory.tryRemove(fromPath), "could't move node", fromPath) {
return false;
}
}
entry.set(kj::mv(*file));
}
return true;
} else {
KJ_FAIL_ASSERT("source node deleted concurrently during transfer", fromPath) {
return false;
}
}
case FsNode::Type::DIRECTORY:
KJ_IF_MAYBE(subdir, fromDirectory.tryOpenSubdir(fromPath, WriteMode::MODIFY)) {
if (mode == TransferMode::COPY) {
auto copy = refcounted<InMemoryDirectory>(clock);
for (auto& subEntry: subdir->get()->listEntries()) {
EntryImpl newEntry(kj::mv(subEntry.name));
Path filename(newEntry.name);
if (!copy->tryTransferChild(newEntry, subEntry.type, nullptr, nullptr, **subdir,
filename, TransferMode::COPY)) {
KJ_LOG(ERROR, "couldn't copy node of type not supported by InMemoryDirectory",
filename);
} else {
StringPtr nameRef = newEntry.name;
copy->entries.insert(std::make_pair(nameRef, kj::mv(newEntry)));
}
}
entry.set(kj::mv(copy));
} else {
if (mode == TransferMode::MOVE) {
KJ_ASSERT(fromDirectory.tryRemove(fromPath), "could't move node", fromPath) {
return false;
}
}
entry.set(kj::mv(*subdir));
}
return true;
} else {
KJ_FAIL_ASSERT("source node deleted concurrently during transfer", fromPath) {
return false;
}
}
case FsNode::Type::SYMLINK:
KJ_IF_MAYBE(content, fromDirectory.tryReadlink(fromPath)) {
// Since symlinks are immutable, we can implement LINK the same as COPY.
entry.init(SymlinkNode { lastModified.orDefault(clock.now()), kj::mv(*content) });
if (mode == TransferMode::MOVE) {
KJ_ASSERT(fromDirectory.tryRemove(fromPath), "could't move node", fromPath) {
return false;
}
}
return true;
} else {
KJ_FAIL_ASSERT("source node deleted concurrently during transfer", fromPath) {
return false;
}
}
default:
return false;
}
}
};
// -----------------------------------------------------------------------------
class AppendableFileImpl final: public AppendableFile {
public:
AppendableFileImpl(Own<File>&& fileParam): file(kj::mv(fileParam)) {}
Own<FsNode> cloneFsNode() override {
return heap<AppendableFileImpl>(file->clone());
}
Maybe<int> getFd() override {
return nullptr;
}
Metadata stat() override {
return file->stat();
}
void sync() override { file->sync(); }
void datasync() override { file->datasync(); }
void write(const void* buffer, size_t size) override {
file->write(file->stat().size, arrayPtr(reinterpret_cast<const byte*>(buffer), size));
}
private:
Own<File> file;
};
} // namespace
// -----------------------------------------------------------------------------
Own<File> newInMemoryFile(Clock& clock) {
return refcounted<InMemoryFile>(clock);
}
Own<Directory> newInMemoryDirectory(Clock& clock) {
return refcounted<InMemoryDirectory>(clock);
}
Own<AppendableFile> newFileAppender(Own<File> inner) {
return heap<AppendableFileImpl>(kj::mv(inner));
}
} // namespace kj
c++/src/kj/filesystem.h 0 → 100644
View file @ b1701b88
// Copyright (c) 2015 Sandstorm Development Group, Inc. and contributors
// Licensed under the MIT License:
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
#ifndef KJ_FILESYSTEM_H_
#define KJ_FILESYSTEM_H_
#include "memory.h"
#include "io.h"
#include <inttypes.h>
#include "time.h" // TODO(now): problematic
#include "function.h"
namespace kj {
template <typename T>
class Vector;
class PathPtr;
class Path {
// A Path identifies a file in a directory tree.
//
// In KJ, we avoid representing paths as plain strings because this can lead to path injection
// bugs as well as numerous kinds of bugs relating to path parsing edge cases. The Path class's
// interface is designed to "make it hard to screw up".
//
// A "Path" is in fact a list of strings, each string being one component of the path (as would
// normally be separated by '/'s). Path components are not allowed to contain '/' nor '\0', nor
// are they allowed to be the special names "", ".", nor "..".
//
// If you explicitly want to parse a path that contains '/'s, ".", and "..", you must use
// parse() and/or eval(). However, users of this interface are encouraged to avoid parsing
// paths at all, and instead express paths as string arrays.
//
// Note that when using the Path class, ".." is always canonicalized in path space without
// consulting the actual filesystem. This means that "foo/some-symlink/../bar" is exactly
// equivalent to "foo/bar". This differs from the kernel's behavior when resolving paths passed
// to system calls: the kernel would have resolved "some-symlink" to its target physical path,
// and then would have interpreted ".." relative to that. In practice, the kernel's behavior is
// rarely what the user or programmer intended, hence canonicalizing in path space produces a
// better result.
//
// Path objects are "immutable": functions that "modify" the path return a new path. However,
// if the path being operated on is an rvalue, copying can be avoided. Hence it makes sense to
// write code like:
//
// Path p = ...;
// p = kj::mv(p).append("bar"); // in-place update, avoids string copying
public:
Path(decltype(nullptr)); // empty path
explicit Path(StringPtr name);
explicit Path(String&& name);
// Create a Path containing only one component. `name` is a single filename; it cannot contain
// '/' nor '\0' nor can it be exactly "" nor "." nor "..".
//
// If you want to allow '/'s and such, you must call Path::parse(). We force you to do this to
// prevent path injection bugs where you didn't consider what would happen if the path contained
// a '/'.
explicit Path(std::initializer_list<StringPtr> parts);
explicit Path(ArrayPtr<const StringPtr> parts);
explicit Path(Array<String> parts);
// Construct a path from an array. Note that this means you can do:
//
// Path{"foo", "bar", "baz"} // equivalent to Path::parse("foo/bar/baz")
KJ_DISALLOW_COPY(Path);
Path(Path&&) = default;
Path& operator=(Path&&) = default;
Path clone() const;
static Path parse(StringPtr path);
// Parses a path in traditional format. Components are separated by '/'. Any use of "." or
// ".." will be canonicalized (if they can't be canonicalized, e.g. because the path starts with
// "..", an exception is thrown). Multiple consecutive '/'s will be collapsed. A leading '/'
// is NOT accepted -- if that is a problem, you probably want `eval()`. Trailing '/'s are
// ignored.
Path append(Path suffix) const&;
Path append(Path suffix) &&;
Path append(PathPtr suffix) const&;
Path append(PathPtr suffix) &&;
Path append(StringPtr suffix) const&;
Path append(StringPtr suffix) &&;
Path append(String suffix) const&;
Path append(String suffix) &&;
// Create a new path by appending the given path to this path.
//
// `suffix` cannot contain '/' characters. Instead, you can append an array:
//
// path.append({"foo", "bar"})
//
// Or, use Path::parse():
//
// path.append(Path::parse("foo//baz/../bar"))
Path eval(StringPtr pathText) const&;
Path eval(StringPtr pathText) &&;
// Evaluates a traditional path relative to this one. `pathText` is parsed like `parse()` would,
// except that:
// - It can contain leading ".." components that traverse up the tree.
// - It can have a leading '/' which completely replaces the current path.
//
// THE NAME OF THIS METHOD WAS CHOSEN TO INSPIRE FEAR.
//
// Instead of using `path.eval(str)`, always consider whether you really want
// `path.append(Path::parse(str))`. The former is much riskier than the latter in terms of path
// injection vulnerabilities.
PathPtr basename() const&;
Path basename() &&;
// Get the last component of the path. (Use `basename()[0]` to get just the string.)
PathPtr parent() const&;
Path parent() &&;
// Get the parent path.
String toString(bool absolute = false) const;
// Converts the path to a traditional path string, appropriate to pass to a unix system call.
// Never throws.
const String& operator[](size_t i) const&;
String operator[](size_t i) &&;
size_t size() const;
const String* begin() const;
const String* end() const;
PathPtr slice(size_t start, size_t end) const&;
Path slice(size_t start, size_t end) &&;
// A Path can be accessed as an array of strings.
Path evalWin32(StringPtr pathText) const&;
Path evalWin32(StringPtr pathText) &&;
// Evaluates a Win32-style path. Differences from `eval()` include:
//
// - Backslashes can be used as path separators.
// - Absolute paths begin with a drive letter followed by a colon. The drive letter, including
// the colon, will become the first component of the path, e.g. "c:\foo" becomes {"c:", "foo"}.
// - A network path like "\\host\share\path" is parsed as {"host", "share", "path"}.
String toWin32String(bool absolute = false) const;
// Converts the path to a Win32 path string.
//
// (In most cases you'll want to further convert the returned string from UTF-8 to UTF-16.)
//
// If `absolute` is true, the path is expected to be an absolute path, meaning the first
// component is a drive letter, namespace, or network host name. These are converted to their
// regular Win32 format -- i.e. this method does the reverse of `evalWin32()`.
//
// This throws if the path would have unexpected special meaning or is otherwise invalid on
// Windows, such as if it contains backslashes (within a path component), colons, or special
// names like "con".
private:
Array<String> parts;
// TODO(perf): Consider unrolling one element from `parts`, so that a one-element path doesn't
// require allocation of an array.
enum { ALREADY_CHECKED };
Path(Array<String> parts, decltype(ALREADY_CHECKED));
friend class PathPtr;
static String stripNul(String input);
static void validatePart(StringPtr part);
static void evalPart(Vector<String>& parts, ArrayPtr<const char> part);
static Path evalImpl(Vector<String>&& parts, StringPtr path);
static Path evalWin32Impl(Vector<String>&& parts, StringPtr path);
static size_t countParts(StringPtr path);
static size_t countPartsWin32(StringPtr path);
static bool isWin32Drive(ArrayPtr<const char> part);
static bool isNetbiosName(ArrayPtr<const char> part);
static bool isWin32Special(StringPtr part);
};
class PathPtr {
// Points to a Path or a slice of a Path, but doesn't own it.
//
// PathPtr is to Path as ArrayPtr is to Array and StringPtr is to String.
public:
PathPtr(decltype(nullptr));
PathPtr(const Path& path);
Path clone();
Path append(Path suffix) const;
Path append(PathPtr suffix) const;
Path append(StringPtr suffix) const;
Path append(String suffix) const;
Path eval(StringPtr pathText) const;
PathPtr basename() const;
PathPtr parent() const;
String toString(bool absolute = false) const;
const String& operator[](size_t i) const;
size_t size() const;
const String* begin() const;
const String* end() const;
PathPtr slice(size_t start, size_t end) const;
Path evalWin32(StringPtr pathText) const;
String toWin32String(bool absolute = false) const;
// Equivalent to the corresponding methods of `Path`.
private:
ArrayPtr<const String> parts;
explicit PathPtr(ArrayPtr<const String> parts);
friend class Path;
};
// =======================================================================================
// The filesystem API
//
// This API is strictly synchronous because, unfortunately, there's no such thing as asynchronous
// filesystem access in practice. The filesystem drivers on Linux are written to assume they can
// block. The AIO API is only actually asynchronous for reading/writing the raw file blocks, but if
// the filesystem needs to be involved (to allocate blocks, update metadata, etc.) that will block.
// It's best to imagine that the filesystem is just another tier of memory that happens to be
// slower than RAM (which is slower than L3 cache, which is slower than L2, which is slower than
// L1). You can't do asynchronous RAM access so why asynchronous filesystem? The only way to
// parallelize these is using threads.
class FsNode {
// Base class for filesystem node types.
public:
Own<FsNode> clone();
// Creates a new object of exactly the same type as this one, pointing at exactly the same
// external object.
//
// Under the hood, this will call dup(), so the FD number will not be the same.
virtual Maybe<int> getFd() = 0;
// Get the underlying file descriptor, if any. Returns nullptr if this object actually isn't
// wrapping a file descriptor.
enum class Type {
FILE,
DIRECTORY,
SYMLINK,
BLOCK_DEVICE,
CHARACTER_DEVICE,
NAMED_PIPE,
SOCKET,
OTHER,
};
struct Metadata {
Type type = Type::FILE;
uint64_t size = 0;
// Logical size of the file.
uint64_t spaceUsed = 0;
// Physical size of the file on disk. May be smaller for sparse files, or larger for
// pre-allocated files.
Date lastModified = UNIX_EPOCH;
// Last modification time of the file.
uint linkCount = 1;
// Number of hard links pointing to this node.
// Not currently included:
// - Device / inode number: Rarely useful, and not safe to use in practice anyway.
// - Access control info: Differs wildly across platforms, and KJ prefers capabilities anyway.
// - Other timestamps: Differs across platforms.
// - Device number: If you care, you're probably doing platform-specific stuff anyway.
};
virtual Metadata stat() = 0;
virtual void sync() = 0;
virtual void datasync() = 0;
// Maps to fsync() and fdatasync() system calls.
//
// Also, when creating or overwriting a file, the first call to sync() atomically links the file
// into the filesystem (*after* syncing the data), so than incomplete data is never visible to
// other processes. (In practice this works by writing into a temporary file and then rename()ing
// it.)
protected:
virtual Own<FsNode> cloneFsNode() = 0;
// Implements clone(). Required to return an object with exactly the same type as this one.
// Hence, every subclass must implement this.
};
class ReadableFile: public FsNode {
public:
Own<ReadableFile> clone();
String readAllText();
// Read all text in the file and return as a big string.
Array<byte> readAllBytes();
// Read all bytes in the file and return as a big byte array.
//
// This differs from mmap() in that the read is performed all at once. Future changes to the file
// do not affect the returned copy. Consider using mmap() instead, particularly for large files.
virtual size_t read(uint64_t offset, ArrayPtr<byte> buffer) = 0;
// Fills `buffer` with data starting at `offset`. Returns the number of bytes actually read --
// the only time this is less than `buffer.size()` is when EOF occurs mid-buffer.
virtual Array<const byte> mmap(uint64_t offset, uint64_t size) = 0;
// Maps the file to memory read-only. The returned array always has exactly the requested size.
// Depending on the capabilities of the OS and filesystem, the mapping may or may not reflect
// changes that happen to the file after mmap() returns.
//
// Multiple calls to mmap() on the same file may or may not return the same mapping (it is
// immutable, so there's no possibility of interference).
//
// If the file cannot be mmap()ed, an implementation may choose to allocate a buffer on the heap,
// read into it, and return that. This should only happen if a real mmap() is impossible.
//
// The returned array is always exactly the size requested. However, accessing bytes beyond the
// current end of the file may raise SIGBUS, or may simply return zero.
virtual Array<byte> mmapPrivate(uint64_t offset, uint64_t size) = 0;
// Like mmap() but returns a view that the caller can modify. Modifications will not be written
// to the underlying file. Every call to this method returns a unique mapping. Changes made to
// the underlying file by other clients may or may not be reflected in the mapping -- in fact,
// some changes may be reflected while others aren't, even within the same mapping.
//
// In practice this is often implemented using copy-on-write pages. When you first write to a
// page, a copy is made. Hence, changes to the underlying file within that page stop being
// reflected in the mapping.
};
class AppendableFile: public FsNode, public OutputStream {
public:
Own<AppendableFile> clone();
// All methods are inherited.
};
class WritableFileMapping {
public:
virtual ArrayPtr<byte> get() = 0;
// Gets the mapped bytes. The returned array can be modified, and those changes may be written to
// the underlying file, but there is no guarantee that they are written unless you subsequently
// call changed().
virtual void changed(ArrayPtr<byte> slice) = 0;
// Notifies the implementation that the given bytes have changed. For some implementations this
// may be a no-op while for others it may be necessary in order for the changes to be written
// back at all.
//
// `slice` must be a slice of `bytes()`.
virtual void sync(ArrayPtr<byte> slice) = 0;
// Implies `changed()`, and then waits until the range has actually been written to disk before
// returning.
//
// `slice` must be a slice of `bytes()`.
};
class File: public ReadableFile {
public:
Own<File> clone();
void writeAll(ArrayPtr<const byte> bytes);
void writeAll(StringPtr text);
// Completely replace the file with the given bytes or text.
virtual void write(uint64_t offset, ArrayPtr<const byte> data) = 0;
// Write the given data starting at the given offset in the file.
virtual void zero(uint64_t offset, uint64_t size) = 0;
// Write zeros to the file, starting at `offset` and continuing for `size` bytes. If the platform
// supports it, this will "punch a hole" in the file, such that blocks that are entirely zeros
// do not take space on disk.
virtual void truncate(uint64_t size) = 0;
// Set the file end pointer to `size`. If `size` is less than the current size, data past the end
// is truncated. If `size` is larger than the current size, zeros are added to the end of the
// file. If the platform supports it, blocks containing all-zeros will not be stored to disk.
virtual Own<WritableFileMapping> mmapWritable(uint64_t offset, uint64_t size) = 0;
// Like ReadableFile::mmap() but returns a mapping for which any changes will be immediately
// visible in other mappings of the file on the same system and will eventually be written back
// to the file.
virtual size_t copy(uint64_t offset, ReadableFile& from, uint64_t fromOffset, uint64_t size);
// Copies bytes from one file to another.
//
// Copies `size` bytes or to EOF, whichever comes first. Returns the number of bytes actually
// copied. Hint: Pass kj::maxValue for `size` to always copy to EOF.
//
// The copy is not atomic. Concurrent writes may lead to garbage results.
//
// The default implementation performs a series of reads and writes. Subclasses can often provide
// superior implementations that offload the work to the OS or even implement copy-on-write.
};
class ReadableDirectory: public FsNode {
// Read-only subset of `Directory`.
public:
Own<ReadableDirectory> clone();
virtual Array<String> listNames() = 0;
// List the contents of this directory. Does NOT include "." nor "..".
struct Entry {
FsNode::Type type;
String name;
};
virtual Array<Entry> listEntries() = 0;
// List the contents of the directory including the type of each file. On some platforms and
// filesystems, this is just as fast as listNames(), but on others it may require stat()ing each
// file.
virtual bool exists(PathPtr path) = 0;
// Does the specified path exist?
//
// If the path is a symlink, the symlink is followed and the return value indicates if the target
// exists. If you want to know if the symlink exists, use lstat(). (This implies that listNames()
// may return names for which exists() reports false.)
FsNode::Metadata lstat(PathPtr path);
virtual Maybe<FsNode::Metadata> tryLstat(PathPtr path) = 0;
// Gets metadata about the path. If the path is a symlink, it is not followed -- the metadata
// describes the symlink itself. `tryLstat()` returns null if the path doesn't exist.
Own<ReadableFile> openFile(PathPtr path);
virtual Maybe<Own<ReadableFile>> tryOpenFile(PathPtr path) = 0;
// Open a file for reading.
//
// `tryOpenFile()` returns null if the path doesn't exist. Other errors still throw exceptions.
Own<ReadableDirectory> openSubdir(PathPtr path);
virtual Maybe<Own<ReadableDirectory>> tryOpenSubdir(PathPtr path) = 0;
// Opens a subdirectory.
//
// `tryOpenSubdir()` returns null if the path doesn't exist. Other errors still throw exceptions.
String readlink(PathPtr path);
virtual Maybe<String> tryReadlink(PathPtr path) = 0;
// If `path` is a symlink, reads and returns the link contents.
//
// See Directory::symlink() for warnings about symlinks.
};
enum class WriteMode {
// Mode for opening a file (or directory) for write.
//
// (To open a file or directory read-only, do not specify a mode.)
//
// WriteMode is a bitfield. Hence, it overloads the bitwise logic operators. To check if a
// particular bit is set in a bitfield, use kj::has(), like:
//
// if (kj::has(mode, WriteMode::MUST_EXIST)) {
// requireExists(path);
// }
//
// (`if (mode & WriteMode::MUST_EXIST)` doesn't work because WriteMode is an enum class, which
// cannot be converted to bool. Alas, C++ does not allow you to define a conversion operator
// on an enum type, so we can't define a conversion to bool.)
// -----------------------------------------
// Core flags
//
// At least one of CREATE or MODIFY must be specified. Optionally, the two flags can be combined
// with a bitwise-OR.
CREATE = 1,
// Create a new empty file.
//
// This can be OR'd with MODIFY, but not with REPLACE.
//
// When not combined with MODIFY, if the file already exists (including as a broken symlink),
// tryOpenFile() returns null (and openFile() throws).
//
// When combined with MODIFY, if the path already exists, it will be opened as if CREATE hadn't
// been specified at all. If the path refers to a broken symlink, the file at the target of the
// link will be created (if its parent directory exists).
MODIFY = 2,
// Modify an existing file.
//
// This can be OR'd with CREATE, but not with REPLACE.
//
// When not combined with CREATE, if the file doesn't exist (including if it is a broken symlink),
// tryOpenFile() returns null (and openFile() throws).
//
// When combined with CREATE, if the path doesn't exist, it will be created as if MODIFY hadn't
// been specified at all. If the path refers to a broken symlink, the file at the target of the
// link will be created (if its parent directory exists).
// -----------------------------------------
// Additional flags
//
// Any number of these may be OR'd with the core flags.
CREATE_PARENT = 4,
// Indicates that if the target node's parent directory doesn't exist, it should be created
// automatically, along with its parent, and so on. This creation is NOT atomic.
//
// This bit only makes sense with CREATE or REPLACE.
EXECUTABLE = 8,
// Mark this file executable, if this is a meaningful designation on the host platform.
PRIVATE = 16,
// Indicates that this file is sensitive and should have permissions masked so that it is only
// accessible by the current user.
//
// When this is not used, the platform's default access control settings are used. On Unix,
// that usually means the umask is applied. On Windows, it means permissions are inherited from
// the parent.
};
inline constexpr WriteMode operator|(WriteMode a, WriteMode b) {
return static_cast<WriteMode>(static_cast<uint>(a) | static_cast<uint>(b));
}
inline constexpr WriteMode operator&(WriteMode a, WriteMode b) {
return static_cast<WriteMode>(static_cast<uint>(a) & static_cast<uint>(b));
}
inline constexpr WriteMode operator+(WriteMode a, WriteMode b) {
return static_cast<WriteMode>(static_cast<uint>(a) | static_cast<uint>(b));
}
inline constexpr WriteMode operator-(WriteMode a, WriteMode b) {
return static_cast<WriteMode>(static_cast<uint>(a) & ~static_cast<uint>(b));
}
template <typename T, typename = EnableIf<__is_enum(T)>>
bool has(T haystack, T needle) {
return (static_cast<__underlying_type(T)>(haystack) &
static_cast<__underlying_type(T)>(needle)) ==
static_cast<__underlying_type(T)>(needle);
}
enum class TransferMode {
// Specifies desired behavior for Directory::transfer().
MOVE,
// The node is moved to the new location, i.e. the old location is deleted. If possible, this
// move is performed without copying, otherwise it is performed as a copy followed by a delete.
LINK,
// The new location becomes a synonym for the old location (a "hard link"). Filesystems have
// varying support for this -- typically, it is not supported on directories.
COPY
// The new location becomes a copy of the old.
//
// Some filesystems may implement this in terms of copy-on-write.
//
// If the filesystem supports sparse files, COPY takes sparseness into account -- it will punch
// holes in the target file where holes exist in the source file.
};
class Directory: public ReadableDirectory {
// Refers to a specific directory on disk.
//
// A `Directory` object *only* provides access to children of the directory, not parents. That
// is, you cannot open the file "..", nor jump to the root directory with "/".
//
// On OSs that support in, a `Directory` is backed by an open handle to the directory node. This
// means:
// - If the directory is renamed on-disk, the `Directory` object still points at it.
// - Opening files in the directory only requires the OS to traverse the path from the directory
// to the file; it doesn't have to re-traverse all the way from the filesystem root.
public:
Own<Directory> clone();
template <typename T>
class Replacer {
// Implements an atomic replacement of a file or directory, allowing changes to be made to
// storage in a way that avoids losing data in a power outage and prevents other processes
// from observing content in an inconsistent state.
//
// `T` may be `File` or `Directory`. For readability, the text below describes replacing a
// file, but the logic is the same for directories.
//
// When you call `Directory::replaceFile()`, a temporary file is created, but the specified
// path is not yet touched. You may call `get()` to obtain the temporary file object, through
// which you may initialize its content, knowing that no other process can see it yet. The file
// is atomically moved to its final path when you call `commit()`. If you destroy the Replacer
// without calling commit(), the temporary file is deleted.
//
// Note that most operating systems sadly do not support creating a truly unnamed temporary file
// and then linking it in later. Moreover, the file cannot necessarily be created in the system
// temporary directory because it might not be on the same filesystem as the target. Therefore,
// the replacement file may initially be created in the same directory as its eventual target.
// The implementation of Directory will choose a name that is unique and "hidden" according to
// the conventions of the filesystem. Additionally, the implementation of Directory will avoid
// returning these temporary files from its list*() methods, in order to avoid observable
// inconsistencies across platforms.
public:
explicit Replacer(WriteMode mode);
virtual T& get() = 0;
// Gets the File or Directory representing the replacement data. Fill in this object before
// calling commit().
void commit();
virtual bool tryCommit() = 0;
// Commit the replacement.
//
// `tryCommit()` may return false based on the CREATE/MODIFY bits passed as the WriteMode when
// the replacement was initiated. (If CREATE but not MODIFY was used, tryCommit() returns
// false to indicate that the target file already existed. If MODIFY but not CREATE was used,
// tryCommit() returns false to indicate that the file didn't exist.)
//
// `commit()` is atomic, meaning that there is no point in time at which other processes
// observing the file will see it in an intermediate state -- they will either see the old
// content or the complete new content. This includes in the case of a power outage or machine
// failure: on recovery, the file will either be in the old state or the new state, but not in
// some intermediate state.
//
// It's important to note that a power failure *after commit() returns* can still revert the
// file to its previous state. That is, `commit()` does NOT guarantee that, upon return, the
// new content is durable. In order to guarantee this, you must call `sync()` on the immediate
// parent directory of the replaced file.
//
// Note that, sadly, not all filesystems / platforms are capable of supporting all of the
// guarantees documented above. In such cases, commit() will make a best-effort attempt to do
// what it claims. Some examples of possible problems include:
// - Any guarantees about durability through a power outage probably require a journaling
// filesystem.
// - Many platforms do not support atomically replacing a non-empty directory. Linux does as
// of kernel 3.15 (via the renameat2() syscall using RENAME_EXCHANGE). Where not supported,
// the old directory will be moved away just before the replacement is moved into place.
// - Many platforms do not support atomically requiring the existence or non-existence of a
// file before replacing it. In these cases, commit() may have to perform the check as a
// separate step, with a small window for a race condition.
// - Many platforms do not support "unlinking" a non-empty directory, meaning that a replaced
// directory will need to be deconstructed by deleting all contents. If another process has
// the directory open when it is replaced, that process will observe the contents
// disappearing after the replacement (actually, a swap) has taken place. This differs from
// files, where a process that has opened a file before it is replaced will continue see the
// file's old content unchanged after the replacement.
protected:
const WriteMode mode;
};
using ReadableDirectory::openFile;
using ReadableDirectory::openSubdir;
using ReadableDirectory::tryOpenFile;
using ReadableDirectory::tryOpenSubdir;
Own<File> openFile(PathPtr path, WriteMode mode);
virtual Maybe<Own<File>> tryOpenFile(PathPtr path, WriteMode mode) = 0;
// Open a file for writing.
//
// `tryOpenFile()` returns null if the path is required to exist but doesn't (MODIFY or REPLACE)
// or if the path is required not to exist but does (CREATE or RACE).
virtual Own<Replacer<File>> replaceFile(PathPtr path, WriteMode mode) = 0;
// Construct a file which, when ready, will be atomically moved to `path`, replacing whatever
// is there already. See `Replacer<T>` for detalis.
//
// The `CREATE` and `MODIFY` bits of `mode` are not enforced until commit time, hence
// `replaceFile()` has no "try" variant.
virtual Own<File> createTemporary() = 0;
// Create a temporary file backed by this directory's filesystem, but which isn't linked into
// the directory tree. The file is deleted from disk when all references to it have been dropped.
Own<AppendableFile> appendFile(PathPtr path, WriteMode mode);
virtual Maybe<Own<AppendableFile>> tryAppendFile(PathPtr path, WriteMode mode) = 0;
// Opens the file for appending only. Useful for log files.
//
// If the underlying filesystem supports it, writes to the file will always be appended even if
// other writers are writing to the same file at the same time -- however, some implementations
// may instead assume that no other process is changing the file size between writes.
Own<Directory> openSubdir(PathPtr path, WriteMode mode);
virtual Maybe<Own<Directory>> tryOpenSubdir(PathPtr path, WriteMode mode) = 0;
// Opens a subdirectory for writing.
virtual Own<Replacer<Directory>> replaceSubdir(PathPtr path, WriteMode mode) = 0;
// Construct a directory which, when ready, will be atomically moved to `path`, replacing
// whatever is there already. See `Replacer<T>` for detalis.
//
// The `CREATE` and `MODIFY` bits of `mode` are not enforced until commit time, hence
// `replaceSubdir()` has no "try" variant.
void symlink(PathPtr linkpath, StringPtr content, WriteMode mode);
virtual bool trySymlink(PathPtr linkpath, StringPtr content, WriteMode mode) = 0;
// Create a symlink. `content` is the raw text which will be written into the symlink node.
// How this text is interpreted is entirely dependent on the filesystem. Note in particular that:
// - Windows will require a path that uses backslashes as the separator.
// - InMemoryDirectory does not support symlinks containing "..".
//
// Unfortunately under many implementations symlink() can be used to break out of the directory
// by writing an absolute path or utilizing "..". Do not call this method with a value for
// `target` that you don't trust.
//
// `mode` must be CREATE or REPLACE, not MODIFY. CREATE_PARENT is honored but EXECUTABLE and
// PRIVATE have no effect. `trySymlink()` returns false in CREATE mode when the target already
// exists.
void transfer(PathPtr toPath, WriteMode toMode,
PathPtr fromPath, TransferMode mode);
void transfer(PathPtr toPath, WriteMode toMode,
Directory& fromDirectory, PathPtr fromPath,
TransferMode mode);
virtual bool tryTransfer(PathPtr toPath, WriteMode toMode,
Directory& fromDirectory, PathPtr fromPath,
TransferMode mode);
virtual Maybe<bool> tryTransferTo(Directory& toDirectory, PathPtr toPath, WriteMode toMode,
PathPtr fromPath, TransferMode mode);
// Move, link, or copy a file/directory tree from one location to another.
//
// Filesystems vary in what kinds of transfers are allowed, especially for TransferMode::LINK,
// and whether TransferMode::MOVE is implemented as an actual move vs. copy+delete.
//
// tryTransfer() returns false if the source location didn't exist, or when `toMode` is CREATE
// and the target already exists. The default implementation implements only TransferMode::COPY.
//
// tryTransferTo() exists to implement double-dispatch. It should be called as a fallback by
// implementations of tryTransfer() in cases where the target directory would otherwise fail or
// perform a pessimal transfer. The default implementation returns nullptr, which the caller
// should interpret as: "I don't have any special optimizations; do the obvious thing."
//
// `toMode` controls how the target path is created. CREATE_PARENT is honored but EXECUTABLE and
// PRIVATE have no effect.
void remove(PathPtr path);
virtual bool tryRemove(PathPtr path) = 0;
// Deletes/unlinks the given path. If the path names a directory, it is recursively deleted.
//
// tryRemove() returns false if the path doesn't exist; remove() throws in this case.
// TODO(someday):
// - Support sockets? There's no openat()-like interface for sockets, so it's hard to support
// them currently. Also you'd probably want to use them with the async library.
// - Support named pipes? Unclear if there's a use case that isn't better-served by sockets.
// Then again, they can be openat()ed.
// - Support watching for changes (inotify). Probably also requires the async library. Also
// lacks openat()-like semantics.
// - xattrs -- linux-specific
// - chown/chmod/etc. -- unix-specific, ACLs, eww
// - set timestamps -- only needed by archiving programs/
// - advisory locks
// - sendfile?
// - fadvise and such
private:
static void commitFailed(WriteMode mode);
};
class Filesystem {
public:
virtual Directory& getRoot() = 0;
// Get the filesystem's root directory, as of the time the Filesystem object was created.
virtual Directory& getCurrent() = 0;
// Get the filesystem's current directory, as of the time the Filesystem object was created.
virtual PathPtr getCurrentPath() = 0;
// Get the path from the root to the current directory, as of the time the Filesystem object was
// created. Note that because a `Directory` does not provide access to its parent, if you want to
// follow `..` from the current directory, you must use `getCurrentPath().eval("..")` or
// `getCurrentPath().parent()`.
//
// This function attempts to determine the path as it appeared in the user's shell before this
// program was started. That means, if the user had `cd`ed into a symlink, the path through that
// symlink is returned, *not* the canonical path.
//
// Because of this, there is an important difference between how the operating system interprets
// "../foo" and what you get when you write `getCurrentPath().eval("../foo")`: The former
// will interpret ".." relative to the directory's canonical path, whereas the latter will
// interpret it relative to the path shown in the user's shell. In practice, the latter is
// almost always what the user wants! But the former behavior is what almost all commands do
// in practice, and it leads to confusion. KJ commands should implement the behavior the user
// expects.
};
// =======================================================================================
Own<File> newInMemoryFile(Clock& clock);
Own<Directory> newInMemoryDirectory(Clock& clock);
// Construct file and directory objects which reside in-memory.
//
// InMemoryFile has the following special properties:
// - The backing store is not sparse and never gets smaller even if you truncate the file.
// - While a non-private memory mapping exists, the backing store cannot get larger. Any operation
// which would expand it will throw.
//
// InMemoryDirectory has the following special properties:
// - Symlinks are processed using Path::parse(). This implies tha a symlink cannot point to a
// parent directory -- InMemoryDirectory does not know its parent.
// - link() can link directory nodes in addition to files.
// - link() and rename() accept any kind of Directory as `fromDirectory` -- it doesn't need to be
// another InMemoryDirectory. However, for rename(), the from path must be a directory.
Own<AppendableFile> newFileAppender(Own<File> inner);
// Creates an AppendableFile by wrapping a File. Note that this implementation assumes it is the
// only writer. A correct implementation should always append to the file even if other writes
// are happening simultaneously, as is achieved with the O_APPEND flag to open(2), but that
// behavior is not possible to emulate on top of `File`.
Own<ReadableFile> newDiskReadableFile(kj::AutoCloseFd fd);
Own<AppendableFile> newDiskAppendableFile(kj::AutoCloseFd fd);
Own<File> newDiskFile(kj::AutoCloseFd fd);
Own<ReadableDirectory> newDiskReadableDirectory(kj::AutoCloseFd fd);
Own<Directory> newDiskDirectory(kj::AutoCloseFd fd);
// Wrap a file descriptor as various filesystem types.
Own<Filesystem> newDiskFilesystem();
// Get at implementation of `Filesystem` representing the real filesystem.
//
// DO NOT CALL THIS except at the top level of your program, e.g. in main(). Anywhere else, you
// should instead have your caller pass in a Filesystem object, or a specific Directory object,
// or whatever it is that your code needs. This ensures that your code supports dependency
// injection, which makes it more reusable and testable.
//
// newDiskFilesystem() reads the current working directory at the time it is called. The returned
// object is not affected by subsequent calls to chdir().
// =======================================================================================
// inline implementation details
inline Path::Path(decltype(nullptr)): parts(nullptr) {}
inline Path::Path(std::initializer_list<StringPtr> parts)
: Path(arrayPtr(parts.begin(), parts.end())) {}
inline Path::Path(Array<String> parts, decltype(ALREADY_CHECKED))
: parts(kj::mv(parts)) {}
inline Path Path::clone() const { return PathPtr(*this).clone(); }
inline Path Path::append(Path suffix) const& { return PathPtr(*this).append(kj::mv(suffix)); }
inline Path Path::append(PathPtr suffix) const& { return PathPtr(*this).append(suffix); }
inline Path Path::append(StringPtr suffix) const& { return append(Path(suffix)); }
inline Path Path::append(StringPtr suffix) && { return kj::mv(*this).append(Path(suffix)); }
inline Path Path::append(String suffix) const& { return append(Path(kj::mv(suffix))); }
inline Path Path::append(String suffix) && { return kj::mv(*this).append(Path(kj::mv(suffix))); }
inline Path Path::eval(StringPtr pathText) const& { return PathPtr(*this).eval(pathText); }
inline PathPtr Path::basename() const& { return PathPtr(*this).basename(); }
inline PathPtr Path::parent() const& { return PathPtr(*this).parent(); }
inline const String& Path::operator[](size_t i) const& { return parts[i]; }
inline String Path::operator[](size_t i) && { return kj::mv(parts[i]); }
inline size_t Path::size() const { return parts.size(); }
inline const String* Path::begin() const { return parts.begin(); }
inline const String* Path::end() const { return parts.end(); }
inline PathPtr Path::slice(size_t start, size_t end) const& {
return PathPtr(*this).slice(start, end);
}
inline String Path::toString(bool absolute) const { return PathPtr(*this).toString(absolute); }
inline Path Path::evalWin32(StringPtr pathText) const& {
return PathPtr(*this).evalWin32(pathText);
}
inline String Path::toWin32String(bool absolute) const {
return PathPtr(*this).toWin32String(absolute);
}
inline PathPtr::PathPtr(decltype(nullptr)): parts(nullptr) {}
inline PathPtr::PathPtr(const Path& path): parts(path.parts) {}
inline PathPtr::PathPtr(ArrayPtr<const String> parts): parts(parts) {}
inline Path PathPtr::append(StringPtr suffix) const { return append(Path(suffix)); }
inline Path PathPtr::append(String suffix) const { return append(Path(kj::mv(suffix))); }
inline const String& PathPtr::operator[](size_t i) const { return parts[i]; }
inline size_t PathPtr::size() const { return parts.size(); }
inline const String* PathPtr::begin() const { return parts.begin(); }
inline const String* PathPtr::end() const { return parts.end(); }
inline PathPtr PathPtr::slice(size_t start, size_t end) const {
return PathPtr(parts.slice(start, end));
}
inline Own<FsNode> FsNode::clone() { return cloneFsNode().downcast<FsNode>(); }
inline Own<ReadableFile> ReadableFile::clone() { return cloneFsNode().downcast<ReadableFile>(); }
inline Own<AppendableFile> AppendableFile::clone() {
return cloneFsNode().downcast<AppendableFile>();
}
inline Own<File> File::clone() { return cloneFsNode().downcast<File>(); }
inline Own<ReadableDirectory> ReadableDirectory::clone() {
return cloneFsNode().downcast<ReadableDirectory>();
}
inline Own<Directory> Directory::clone() { return cloneFsNode().downcast<Directory>(); }
inline void Directory::transfer(
PathPtr toPath, WriteMode toMode, PathPtr fromPath, TransferMode mode) {
return transfer(toPath, toMode, *this, fromPath, mode);
}
template <typename T>
inline Directory::Replacer<T>::Replacer(WriteMode mode): mode(mode) {}
template <typename T>
void Directory::Replacer<T>::commit() {
if (!tryCommit()) commitFailed(mode);
}
} // namespace kj
#endif // KJ_FILESYSTEM_H_
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment