Skip to content

C
capnproto
Unverified Commit ff4cd3d7 authored by Kenton Varda's avatar Kenton Varda Committed by GitHub
Browse files
Options

Merge pull request #605 from capnproto/compiler-use-filesystem-api 

Change capnp compiler to use KJ filesystem API.
parents ca469daa 12b479a4
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 258 additions and 142 deletions
c++/src/capnp/compiler/capnp-test.sh
View file @ ff4cd3d7
...@@ -40,6 +40,13 @@ fi ...@@ -40,6 +40,13 @@ fi
SCHEMA=`dirname "$0"`/../test.capnp SCHEMA=`dirname "$0"`/../test.capnp
TESTDATA=`dirname "$0"`/../testdata TESTDATA=`dirname "$0"`/../testdata
SUFFIX=${TESTDATA#*/src/}
PREFIX=${TESTDATA%${SUFFIX}}
if [ "$PREFIX" = "" ]; then
PREFIX=.
fi
# ======================================================================================== # ========================================================================================
# convert # convert
...@@ -102,5 +109,5 @@ test_eval 'TestListDefaults.lists.int32ListList[2][0]' 12341234 ...@@ -102,5 +109,5 @@ test_eval 'TestListDefaults.lists.int32ListList[2][0]' 12341234
test "x`$CAPNP eval $SCHEMA -ojson globalPrintableStruct | tr -d '\r'`" = "x{\"someText\": \"foo\"}" || fail eval json "globalPrintableStruct == {someText = \"foo\"}" test "x`$CAPNP eval $SCHEMA -ojson globalPrintableStruct | tr -d '\r'`" = "x{\"someText\": \"foo\"}" || fail eval json "globalPrintableStruct == {someText = \"foo\"}"
$CAPNP compile -ofoo $TESTDATA/errors.capnp.nobuild 2>&1 | sed -e "s,^.*/errors[.]capnp[.]nobuild,file,g" | tr -d '\r' | $CAPNP compile --src-prefix="$PREFIX" -ofoo $TESTDATA/errors.capnp.nobuild 2>&1 | sed -e "s,^.*errors[.]capnp[.]nobuild:,file:,g" | tr -d '\r' |
cmp $TESTDATA/errors.txt - || fail error output cmp $TESTDATA/errors.txt - || fail error output
c++/src/capnp/compiler/capnp.c++
View file @ ff4cd3d7
This diff is collapsed.
c++/src/capnp/compiler/capnpc-c++.c++
View file @ ff4cd3d7
...@@ -28,19 +28,25 @@ ...@@ -28,19 +28,25 @@
#include <kj/string-tree.h> #include <kj/string-tree.h>
#include <kj/tuple.h> #include <kj/tuple.h>
#include <kj/vector.h> #include <kj/vector.h>
#include <kj/filesystem.h>
#include "../schema-loader.h" #include "../schema-loader.h"
#include "../dynamic.h" #include "../dynamic.h"
#include <kj/miniposix.h>
#include <unordered_map> #include <unordered_map>
#include <unordered_set> #include <unordered_set>
#include <map> #include <map>
#include <set> #include <set>
#include <kj/main.h> #include <kj/main.h>
#include <algorithm> #include <algorithm>
#include <sys/types.h>
#include <sys/stat.h> #if _WIN32
#include <fcntl.h> #define WIN32_LEAN_AND_MEAN // ::eyeroll::
#include <errno.h> #include <windows.h>
#include <kj/windows-sanity.h>
#undef VOID
#undef CONST
#else
#include <sys/time.h>
#endif
#if HAVE_CONFIG_H #if HAVE_CONFIG_H
#include "config.h" #include "config.h"
...@@ -2995,42 +3001,67 @@ private: ...@@ -2995,42 +3001,67 @@ private:
// ----------------------------------------------------------------- // -----------------------------------------------------------------
void makeDirectory(kj::StringPtr path) { kj::Own<kj::Filesystem> fs = kj::newDiskFilesystem();
KJ_IF_MAYBE(slashpos, path.findLast('/')) {
// Make the parent dir.
makeDirectory(kj::str(path.slice(0, *slashpos)));
}
if (kj::miniposix::mkdir(path.cStr(), 0777) < 0) {
int error = errno;
if (error != EEXIST) {
KJ_FAIL_SYSCALL("mkdir(path)", error, path);
}
}
}
void writeFile(kj::StringPtr filename, const kj::StringTree& text) { void writeFile(kj::StringPtr filename, const kj::StringTree& text) {
if (!filename.startsWith("/")) { // We don't use replaceFile() here because atomic replacements are actually detrimental for
KJ_IF_MAYBE(slashpos, filename.findLast('/')) { // build tools:
// Make the parent dir. // - It's the responsibility of the build pipeline to ensure that no one else is concurrently
makeDirectory(kj::str(filename.slice(0, *slashpos))); // reading the file when we write it, so atomicity brings no benefit.
// - Atomic replacements force disk syncs which could slow us down for no benefit at all.
// - Opening the existing file and overwriting it may allow the filesystem to reuse
// already-allocated blocks, or maybe even notice that no actual changes occurred.
// - In a power outage scenario, the user would obviously restart the build from scratch
// anyway.
//
// We do, however, use mmap(), allowing us to write directly to page cache, avoiding a copy,
// and even allowing us to avoid dirtying the file pages if they're not modified, which will
// commonly be the case with incremental builds.
//
// Yes, I overengineered this a bit... but it was fun.
auto path = kj::Path::parse(filename);
auto file = fs->getCurrent().openFile(path,
kj::WriteMode::CREATE | kj::WriteMode::MODIFY | kj::WriteMode::CREATE_PARENT);
file->truncate(text.size());
auto mapping = file->mmapWritable(0, text.size());
auto bytes = mapping->get();
byte* target = bytes.begin();
byte* firstModified = nullptr;
text.visit([&](kj::ArrayPtr<const char> text) {
if (firstModified == nullptr) {
if (memcmp(target, text.begin(), text.size()) == 0) {
target += text.size();
return;
}
firstModified = target;
} }
memcpy(target, text.begin(), text.size());
target += text.size();
});
KJ_ASSERT(target == bytes.end());
if (firstModified == nullptr) {
// The file is completely unchanged. But we should probably update the modification time
// anyway so that build systems don't get confused.
//
// TODO(cleanup): Add touch() to kj::FsNode.
#if _WIN32
FILETIME time;
GetSystemTimeAsFileTime(&time);
KJ_WIN32(SetFileTime(KJ_ASSERT_NONNULL(file->getWin32Handle()), NULL, &time, &time));
#else
KJ_SYSCALL(futimes(KJ_ASSERT_NONNULL(file->getFd()), nullptr));
#endif
} else {
mapping->changed(kj::arrayPtr(firstModified, bytes.end()));
} }
int fd;
KJ_SYSCALL(fd = open(filename.cStr(), O_CREAT | O_WRONLY | O_TRUNC, 0666), filename);
kj::FdOutputStream out((kj::AutoCloseFd(fd)));
text.visit(
[&](kj::ArrayPtr<const char> text) {
out.write(text.begin(), text.size());
});
} }
kj::MainBuilder::Validity run() { kj::MainBuilder::Validity run() {
ReaderOptions options; ReaderOptions options;
options.traversalLimitInWords = 1 << 30; // Don't limit. options.traversalLimitInWords = 1 << 30; // Don't limit.
StreamFdMessageReader reader(STDIN_FILENO, options); StreamFdMessageReader reader(0, options);
auto request = reader.getRoot<schema::CodeGeneratorRequest>(); auto request = reader.getRoot<schema::CodeGeneratorRequest>();
auto capnpVersion = request.getCapnpVersion(); auto capnpVersion = request.getCapnpVersion();
...@@ -3058,9 +3089,6 @@ private: ...@@ -3058,9 +3089,6 @@ private:
schemaLoader.load(node); schemaLoader.load(node);
} }
kj::FdOutputStream rawOut(STDOUT_FILENO);
kj::BufferedOutputStreamWrapper out(rawOut);
for (auto requestedFile: request.getRequestedFiles()) { for (auto requestedFile: request.getRequestedFiles()) {
auto schema = schemaLoader.get(requestedFile.getId()); auto schema = schemaLoader.get(requestedFile.getId());
auto fileText = makeFileText(schema, requestedFile); auto fileText = makeFileText(schema, requestedFile);
......
c++/src/capnp/compiler/error-reporter.h
View file @ ff4cd3d7
...@@ -30,6 +30,7 @@ ...@@ -30,6 +30,7 @@
#include <kj/string.h> #include <kj/string.h>
#include <kj/exception.h> #include <kj/exception.h>
#include <kj/vector.h> #include <kj/vector.h>
#include <kj/filesystem.h>
namespace capnp { namespace capnp {
namespace compiler { namespace compiler {
...@@ -68,7 +69,8 @@ public: ...@@ -68,7 +69,8 @@ public:
uint column; uint column;
}; };
virtual void addError(kj::StringPtr file, SourcePos start, SourcePos end, virtual void addError(const kj::ReadableDirectory& directory, kj::PathPtr path,
SourcePos start, SourcePos end,
kj::StringPtr message) = 0; kj::StringPtr message) = 0;
// Report an error at the given location in the given file. // Report an error at the given location in the given file.
......
c++/src/capnp/compiler/module-loader.c++
View file @ ff4cd3d7
This diff is collapsed.
c++/src/capnp/compiler/module-loader.h
View file @ ff4cd3d7
...@@ -31,6 +31,7 @@ ...@@ -31,6 +31,7 @@
#include <kj/memory.h> #include <kj/memory.h>
#include <kj/array.h> #include <kj/array.h>
#include <kj/string.h> #include <kj/string.h>
#include <kj/filesystem.h>
namespace capnp { namespace capnp {
namespace compiler { namespace compiler {
...@@ -44,13 +45,12 @@ public: ...@@ -44,13 +45,12 @@ public:
~ModuleLoader() noexcept(false); ~ModuleLoader() noexcept(false);
void addImportPath(kj::String path); void addImportPath(const kj::ReadableDirectory& dir);
// Add a directory to the list of paths that is searched for imports that start with a '/'. // Add a directory to the list of paths that is searched for imports that start with a '/'.
kj::Maybe<Module&> loadModule(kj::StringPtr localName, kj::StringPtr sourceName); kj::Maybe<Module&> loadModule(const kj::ReadableDirectory& dir, kj::PathPtr path);
// Tries to load the module with the given filename. `localName` is the path to the file on // Tries to load a module with the given path inside the given directory. Returns nullptr if the
// disk (as you'd pass to open(2)), and `sourceName` is the canonical name it should be given // file doesn't exist.
// in the schema (this is used e.g. to decide output file locations). Often, these are the same.
private: private:
class Impl; class Impl;
......
c++/src/capnp/schema-parser-test.c++
View file @ ff4cd3d7
...@@ -30,26 +30,28 @@ ...@@ -30,26 +30,28 @@
namespace capnp { namespace capnp {
namespace { namespace {
class FakeFileReader final: public SchemaFile::FileReader { #if _WIN32
#define ABS(x) "C:\\" x
#else
#define ABS(x) "/" x
#endif
class FakeFileReader final: public kj::Filesystem {
public: public:
void add(kj::StringPtr name, kj::StringPtr content) { void add(kj::StringPtr name, kj::StringPtr content) {
files[name] = content; root->openFile(cwd.evalNative(name), kj::WriteMode::CREATE | kj::WriteMode::CREATE_PARENT)
->writeAll(content);
} }
bool exists(kj::StringPtr path) const override { const kj::Directory& getRoot() const override { return *root; }
return files.count(path) > 0; const kj::Directory& getCurrent() const override { return *current; }
} kj::PathPtr getCurrentPath() const override { return cwd; }
kj::Array<const char> read(kj::StringPtr path) const override {
auto iter = files.find(path);
KJ_ASSERT(iter != files.end(), "FakeFileReader has no such file.", path);
auto result = kj::heapArray<char>(iter->second.size());
memcpy(result.begin(), iter->second.begin(), iter->second.size());
return kj::mv(result);
}
private: private:
std::map<kj::StringPtr, kj::StringPtr> files; kj::Own<const kj::Directory> root = kj::newInMemoryDirectory(kj::nullClock());
kj::Path cwd = kj::Path({}).evalNative(ABS("path/to/current/dir"));
kj::Own<const kj::Directory> current = root->openSubdir(cwd,
kj::WriteMode::CREATE | kj::WriteMode::CREATE_PARENT);
}; };
static uint64_t getFieldTypeFileId(StructSchema::Field field) { static uint64_t getFieldTypeFileId(StructSchema::Field field) {
...@@ -59,8 +61,9 @@ static uint64_t getFieldTypeFileId(StructSchema::Field field) { ...@@ -59,8 +61,9 @@ static uint64_t getFieldTypeFileId(StructSchema::Field field) {
} }
TEST(SchemaParser, Basic) { TEST(SchemaParser, Basic) {
SchemaParser parser;
FakeFileReader reader; FakeFileReader reader;
SchemaParser parser;
parser.setDiskFilesystem(reader);
reader.add("src/foo/bar.capnp", reader.add("src/foo/bar.capnp",
"@0x8123456789abcdef;\n" "@0x8123456789abcdef;\n"
...@@ -76,22 +79,22 @@ TEST(SchemaParser, Basic) { ...@@ -76,22 +79,22 @@ TEST(SchemaParser, Basic) {
reader.add("src/qux/corge.capnp", reader.add("src/qux/corge.capnp",
"@0x83456789abcdef12;\n" "@0x83456789abcdef12;\n"
"struct Corge {}\n"); "struct Corge {}\n");
reader.add("/usr/include/grault.capnp", reader.add(ABS("usr/include/grault.capnp"),
"@0x8456789abcdef123;\n" "@0x8456789abcdef123;\n"
"struct Grault {}\n"); "struct Grault {}\n");
reader.add("/opt/include/grault.capnp", reader.add(ABS("opt/include/grault.capnp"),
"@0x8000000000000001;\n" "@0x8000000000000001;\n"
"struct WrongGrault {}\n"); "struct WrongGrault {}\n");
reader.add("/usr/local/include/garply.capnp", reader.add(ABS("usr/local/include/garply.capnp"),
"@0x856789abcdef1234;\n" "@0x856789abcdef1234;\n"
"struct Garply {}\n"); "struct Garply {}\n");
kj::StringPtr importPath[] = { kj::StringPtr importPath[] = {
"/usr/include", "/usr/local/include", "/opt/include" ABS("usr/include"), ABS("usr/local/include"), ABS("opt/include")
}; };
ParsedSchema barSchema = parser.parseFile(SchemaFile::newDiskFile( ParsedSchema barSchema = parser.parseDiskFile(
"foo2/bar2.capnp", "src/foo/bar.capnp", importPath, reader)); "foo2/bar2.capnp", "src/foo/bar.capnp", importPath);
auto barProto = barSchema.getProto(); auto barProto = barSchema.getProto();
EXPECT_EQ(0x8123456789abcdefull, barProto.getId()); EXPECT_EQ(0x8123456789abcdefull, barProto.getId());
...@@ -109,25 +112,25 @@ TEST(SchemaParser, Basic) { ...@@ -109,25 +112,25 @@ TEST(SchemaParser, Basic) {
EXPECT_EQ("garply", barFields[3].getProto().getName()); EXPECT_EQ("garply", barFields[3].getProto().getName());
EXPECT_EQ(0x856789abcdef1234ull, getFieldTypeFileId(barFields[3])); EXPECT_EQ(0x856789abcdef1234ull, getFieldTypeFileId(barFields[3]));
auto bazSchema = parser.parseFile(SchemaFile::newDiskFile( auto bazSchema = parser.parseDiskFile(
"not/used/because/already/loaded", "not/used/because/already/loaded",
"src/foo/baz.capnp", importPath, reader)); "src/foo/baz.capnp", importPath);
EXPECT_EQ(0x823456789abcdef1ull, bazSchema.getProto().getId()); EXPECT_EQ(0x823456789abcdef1ull, bazSchema.getProto().getId());
EXPECT_EQ("foo2/baz.capnp", bazSchema.getProto().getDisplayName()); EXPECT_EQ("foo2/baz.capnp", bazSchema.getProto().getDisplayName());
auto bazStruct = bazSchema.getNested("Baz").asStruct(); auto bazStruct = bazSchema.getNested("Baz").asStruct();
EXPECT_EQ(bazStruct, barStruct.getDependency(bazStruct.getProto().getId())); EXPECT_EQ(bazStruct, barStruct.getDependency(bazStruct.getProto().getId()));
auto corgeSchema = parser.parseFile(SchemaFile::newDiskFile( auto corgeSchema = parser.parseDiskFile(
"not/used/because/already/loaded", "not/used/because/already/loaded",
"src/qux/corge.capnp", importPath, reader)); "src/qux/corge.capnp", importPath);
EXPECT_EQ(0x83456789abcdef12ull, corgeSchema.getProto().getId()); EXPECT_EQ(0x83456789abcdef12ull, corgeSchema.getProto().getId());
EXPECT_EQ("qux/corge.capnp", corgeSchema.getProto().getDisplayName()); EXPECT_EQ("qux/corge.capnp", corgeSchema.getProto().getDisplayName());
auto corgeStruct = corgeSchema.getNested("Corge").asStruct(); auto corgeStruct = corgeSchema.getNested("Corge").asStruct();
EXPECT_EQ(corgeStruct, barStruct.getDependency(corgeStruct.getProto().getId())); EXPECT_EQ(corgeStruct, barStruct.getDependency(corgeStruct.getProto().getId()));
auto graultSchema = parser.parseFile(SchemaFile::newDiskFile( auto graultSchema = parser.parseDiskFile(
"not/used/because/already/loaded", "not/used/because/already/loaded",
"/usr/include/grault.capnp", importPath, reader)); ABS("usr/include/grault.capnp"), importPath);
EXPECT_EQ(0x8456789abcdef123ull, graultSchema.getProto().getId()); EXPECT_EQ(0x8456789abcdef123ull, graultSchema.getProto().getId());
EXPECT_EQ("grault.capnp", graultSchema.getProto().getDisplayName()); EXPECT_EQ("grault.capnp", graultSchema.getProto().getDisplayName());
auto graultStruct = graultSchema.getNested("Grault").asStruct(); auto graultStruct = graultSchema.getNested("Grault").asStruct();
...@@ -135,9 +138,9 @@ TEST(SchemaParser, Basic) { ...@@ -135,9 +138,9 @@ TEST(SchemaParser, Basic) {
// Try importing the other grault.capnp directly. It'll get the display name we specify since // Try importing the other grault.capnp directly. It'll get the display name we specify since
// it wasn't imported before. // it wasn't imported before.
auto wrongGraultSchema = parser.parseFile(SchemaFile::newDiskFile( auto wrongGraultSchema = parser.parseDiskFile(
"weird/display/name.capnp", "weird/display/name.capnp",
"/opt/include/grault.capnp", importPath, reader)); ABS("opt/include/grault.capnp"), importPath);
EXPECT_EQ(0x8000000000000001ull, wrongGraultSchema.getProto().getId()); EXPECT_EQ(0x8000000000000001ull, wrongGraultSchema.getProto().getId());
EXPECT_EQ("weird/display/name.capnp", wrongGraultSchema.getProto().getDisplayName()); EXPECT_EQ("weird/display/name.capnp", wrongGraultSchema.getProto().getDisplayName());
} }
...@@ -147,8 +150,9 @@ TEST(SchemaParser, Constants) { ...@@ -147,8 +150,9 @@ TEST(SchemaParser, Constants) {
// constants are not actually accessible from the generated code API, so the only way to ever // constants are not actually accessible from the generated code API, so the only way to ever
// get a ConstSchema is by parsing it. // get a ConstSchema is by parsing it.
SchemaParser parser;
FakeFileReader reader; FakeFileReader reader;
SchemaParser parser;
parser.setDiskFilesystem(reader);
reader.add("const.capnp", reader.add("const.capnp",
"@0x8123456789abcdef;\n" "@0x8123456789abcdef;\n"
...@@ -164,8 +168,8 @@ TEST(SchemaParser, Constants) { ...@@ -164,8 +168,8 @@ TEST(SchemaParser, Constants) {
" value @0 :T;\n" " value @0 :T;\n"
"}\n"); "}\n");
ParsedSchema fileSchema = parser.parseFile(SchemaFile::newDiskFile( ParsedSchema fileSchema = parser.parseDiskFile(
"const.capnp", "const.capnp", nullptr, reader)); "const.capnp", "const.capnp", nullptr);
EXPECT_EQ(1234, fileSchema.getNested("uint32Const").asConst().as<uint32_t>()); EXPECT_EQ(1234, fileSchema.getNested("uint32Const").asConst().as<uint32_t>());
...@@ -198,8 +202,9 @@ void expectSourceInfo(schema::Node::SourceInfo::Reader sourceInfo, ...@@ -198,8 +202,9 @@ void expectSourceInfo(schema::Node::SourceInfo::Reader sourceInfo,
} }
TEST(SchemaParser, SourceInfo) { TEST(SchemaParser, SourceInfo) {
SchemaParser parser;
FakeFileReader reader; FakeFileReader reader;
SchemaParser parser;
parser.setDiskFilesystem(reader);
reader.add("foo.capnp", reader.add("foo.capnp",
"@0x84a2c6051e1061ed;\n" "@0x84a2c6051e1061ed;\n"
...@@ -234,8 +239,8 @@ TEST(SchemaParser, SourceInfo) { ...@@ -234,8 +239,8 @@ TEST(SchemaParser, SourceInfo) {
"struct Thud @0xcca9972702b730b4 {}\n" "struct Thud @0xcca9972702b730b4 {}\n"
"# post-comment\n"); "# post-comment\n");
ParsedSchema file = parser.parseFile(SchemaFile::newDiskFile( ParsedSchema file = parser.parseDiskFile(
"foo.capnp", "foo.capnp", nullptr, reader)); "foo.capnp", "foo.capnp", nullptr);
ParsedSchema foo = file.getNested("Foo"); ParsedSchema foo = file.getNested("Foo");
expectSourceInfo(file.getSourceInfo(), 0x84a2c6051e1061edull, "file doc comment\n", {}); expectSourceInfo(file.getSourceInfo(), 0x84a2c6051e1061edull, "file doc comment\n", {});
......
c++/src/capnp/schema-parser.c++
View file @ ff4cd3d7
This diff is collapsed.
c++/src/capnp/schema-parser.h
View file @ ff4cd3d7
...@@ -28,6 +28,7 @@ ...@@ -28,6 +28,7 @@
#include "schema-loader.h" #include "schema-loader.h"
#include <kj/string.h> #include <kj/string.h>
#include <kj/filesystem.h>
namespace capnp { namespace capnp {
...@@ -43,31 +44,86 @@ public: ...@@ -43,31 +44,86 @@ public:
SchemaParser(); SchemaParser();
~SchemaParser() noexcept(false); ~SchemaParser() noexcept(false);
ParsedSchema parseDiskFile(kj::StringPtr displayName, kj::StringPtr diskPath, ParsedSchema parseFromDirectory(
kj::ArrayPtr<const kj::StringPtr> importPath) const; const kj::ReadableDirectory& baseDir, kj::Path path,
// Parse a file located on disk. Throws an exception if the file dosen't exist. kj::ArrayPtr<const kj::ReadableDirectory* const> importPath) const;
// // Parse a file from the KJ filesystem API. Throws an exception if the file dosen't exist.
// Parameters: //
// * `displayName`: The name that will appear in the file's schema node. (If the file has // `baseDir` and `path` are used together to resolve relative imports. `path` is the source
// already been parsed, this will be ignored and the display name from the first time it was // file's path within `baseDir`. Relative imports will be interpreted relative to `path` and
// parsed will be kept.) // will be opened using `baseDir`. Note that the KJ filesystem API prohibits "breaking out" of
// * `diskPath`: The path to the file on disk. // a directory using "..", so relative imports will be restricted to children of `baseDir`.
// * `importPath`: Directories to search when resolving absolute imports within this file //
// (imports that start with a `/`). Must remain valid until the SchemaParser is destroyed. // `importPath` is used for absolute imports (imports that start with a '/'). Each directory in
// (If the file has already been parsed, this will be ignored and the import path from the // the array will be searched in order until a file is found.
// first time it was parsed will be kept.) //
// All `ReadableDirectory` objects must remain valid until the `SchemaParser` is destroyed. Also,
// the `importPath` array must remain valid. `path` will be copied; it need not remain valid.
// //
// This method is a shortcut, equivalent to: // This method is a shortcut, equivalent to:
// parser.parseFile(SchemaFile::newDiskFile(displayName, diskPath, importPath))`; // parser.parseFromDirectory(SchemaFile::newDiskFile(baseDir, path, importPath))`;
// //
// This method throws an exception if any errors are encountered in the file or in anything the // This method throws an exception if any errors are encountered in the file or in anything the
// file depends on. Note that merely importing another file does not count as a dependency on // file depends on. Note that merely importing another file does not count as a dependency on
// anything in the imported file -- only the imported types which are actually used are // anything in the imported file -- only the imported types which are actually used are
// "dependencies". // "dependencies".
//
// Hint: Use kj::newDiskFilesystem() to initialize the KJ filesystem API. Usually you should do
// this at a high level in your program, e.g. the main() function, and then pass down the
// appropriate File/Directory objects to the components that need them. Example:
//
// auto fs = kj::newDiskFilesystem();
// SchemaParser parser;
// auto schema = parser->parseFromDirectory(fs->getCurrent(),
// kj::Path::parse("foo/bar.capnp"), nullptr);
//
// Hint: To use in-memory data rather than real disk, you can use kj::newInMemoryDirectory(),
// write the files you want, then pass it to SchemaParser. Example:
//
// auto dir = kj::newInMemoryDirectory(kj::nullClock());
// auto path = kj::Path::parse("foo/bar.capnp");
// dir->openFile(path, kj::WriteMode::CREATE | kj::WriteMode::CREATE_PARENT)
// ->writeAll("struct Foo {}");
// auto schema = parser->parseFromDirectory(*dir, path, nullptr);
//
// Hint: You can create an in-memory directory but then populate it with real files from disk,
// in order to control what is visible while also avoiding reading files yourself or making
// extra copies. Example:
//
// auto fs = kj::newDiskFilesystem();
// auto dir = kj::newInMemoryDirectory(kj::nullClock());
// auto fakePath = kj::Path::parse("foo/bar.capnp");
// auto realPath = kj::Path::parse("path/to/some/file.capnp");
// dir->transfer(fakePath, kj::WriteMode::CREATE | kj::WriteMode::CREATE_PARENT,
// fs->getCurrent(), realPath, kj::TransferMode::LINK);
// auto schema = parser->parseFromDirectory(*dir, fakePath, nullptr);
//
// In this example, note that any imports in the file will fail, since the in-memory directory
// you created contains no files except the specific one you linked in.
ParsedSchema parseDiskFile(kj::StringPtr displayName, kj::StringPtr diskPath,
kj::ArrayPtr<const kj::StringPtr> importPath) const
CAPNP_DEPRECATED("Use parseFromDirectory() instead.");
// Creates a private kj::Filesystem and uses it to parse files from the real disk.
//
// DO NOT USE in new code. Use parseFromDirectory() instead.
//
// This API has a serious problem: the file can import and embed files located anywhere on disk
// using relative paths. Even if you specify no `importPath`, relative imports still work. By
// using `parseFromDirectory()`, you can arrange so that imports are only allowed within a
// particular directory, or even set up a dummy filesystem where other files are not visible.
void setDiskFilesystem(kj::Filesystem& fs)
CAPNP_DEPRECATED("Use parseFromDirectory() instead.");
// Call before calling parseDiskFile() to choose an alternative disk filesystem implementation.
// This exists mostly for testing purposes; new code should use parseFromDirectory() instead.
//
// If parseDiskFile() is called without having called setDiskFilesystem(), then
// kj::newDiskFilesystem() will be used instead.
ParsedSchema parseFile(kj::Own<SchemaFile>&& file) const; ParsedSchema parseFile(kj::Own<SchemaFile>&& file) const;
// Advanced interface for parsing a file that may or may not be located in any global namespace. // Advanced interface for parsing a file that may or may not be located in any global namespace.
// Most users will prefer `parseDiskFile()`. // Most users will prefer `parseFromDirectory()`.
// //
// If the file has already been parsed (that is, a SchemaFile that compares equal to this one // If the file has already been parsed (that is, a SchemaFile that compares equal to this one
// was parsed previously), the existing schema will be returned again. // was parsed previously), the existing schema will be returned again.
...@@ -90,6 +146,7 @@ public: ...@@ -90,6 +146,7 @@ public:
private: private:
struct Impl; struct Impl;
struct DiskFileCompat;
class ModuleImpl; class ModuleImpl;
kj::Own<Impl> impl; kj::Own<Impl> impl;
mutable bool hadErrors = false; mutable bool hadErrors = false;
...@@ -135,44 +192,20 @@ class SchemaFile { ...@@ -135,44 +192,20 @@ class SchemaFile {
// `SchemaFile::newDiskFile()`. // `SchemaFile::newDiskFile()`.
public: public:
class FileReader { // Note: Cap'n Proto 0.6.x and below had classes FileReader and DiskFileReader and a method
public: // newDiskFile() defined here. These were removed when SchemaParser was transitioned to use the
virtual bool exists(kj::StringPtr path) const = 0; // KJ filesystem API. You should be able to get the same effect by subclassing
virtual kj::Array<const char> read(kj::StringPtr path) const = 0; // kj::ReadableDirectory, or using kj::newInMemoryDirectory().
};
static kj::Own<SchemaFile> newFromDirectory(
class DiskFileReader final: public FileReader { const kj::ReadableDirectory& baseDir, kj::Path path,
// Implementation of FileReader that uses the local disk. Files are read using mmap() if kj::ArrayPtr<const kj::ReadableDirectory* const> importPath,
// possible. kj::Maybe<kj::String> displayNameOverride = nullptr);
// Construct a SchemaFile representing a file in a kj::ReadableDirectory. This is used to
public: // implement SchemaParser::parseFromDirectory(); see there for details.
static const DiskFileReader instance; //
// The SchemaFile compares equal to any other SchemaFile that has exactly the same `baseDir`
bool exists(kj::StringPtr path) const override; // object (by identity) and `path` (by value).
kj::Array<const char> read(kj::StringPtr path) const override;
};
static kj::Own<SchemaFile> newDiskFile(
kj::StringPtr displayName, kj::StringPtr diskPath,
kj::ArrayPtr<const kj::StringPtr> importPath,
const FileReader& fileReader = DiskFileReader::instance);
// Construct a SchemaFile representing a file on disk (or located in the filesystem-like
// namespace represented by `fileReader`).
//
// Parameters:
// * `displayName`: The name that will appear in the file's schema node.
// * `diskPath`: The path to the file on disk.
// * `importPath`: Directories to search when resolving absolute imports within this file
// (imports that start with a `/`). The array content must remain valid as long as the
// SchemaFile exists (which is at least as long as the SchemaParser that parses it exists).
// * `fileReader`: Allows you to use a filesystem other than the actual local disk. Although,
// if you find yourself using this, it may make more sense for you to implement SchemaFile
// yourself.
//
// The SchemaFile compares equal to any other SchemaFile that has exactly the same disk path,
// after canonicalization.
//
// The SchemaFile will throw an exception if any errors are reported.
// ----------------------------------------------------------------- // -----------------------------------------------------------------
// For more control, you can implement this interface. // For more control, you can implement this interface.
......
c++/src/kj/filesystem-disk-test.c++
View file @ ff4cd3d7
...@@ -310,10 +310,10 @@ KJ_TEST("DiskFile") { ...@@ -310,10 +310,10 @@ KJ_TEST("DiskFile") {
KJ_EXPECT(kj::str(writableMapping->get().slice(0, 6).asChars()) == "foobaz"); KJ_EXPECT(kj::str(writableMapping->get().slice(0, 6).asChars()) == "foobaz");
KJ_EXPECT(kj::str(privateMapping.slice(0, 6).asChars()) == "Foobaz"); KJ_EXPECT(kj::str(privateMapping.slice(0, 6).asChars()) == "Foobaz");
writableMapping->get()[0] = 'D'; writableMapping->get()[1] = 'D';
writableMapping->changed(writableMapping->get().slice(0, 1)); writableMapping->changed(writableMapping->get().slice(1, 2));
KJ_EXPECT(kj::str(mapping.slice(0, 6).asChars()) == "Doobaz"); KJ_EXPECT(kj::str(mapping.slice(0, 6).asChars()) == "fDobaz");
KJ_EXPECT(kj::str(writableMapping->get().slice(0, 6).asChars()) == "Doobaz"); KJ_EXPECT(kj::str(writableMapping->get().slice(0, 6).asChars()) == "fDobaz");
KJ_EXPECT(kj::str(privateMapping.slice(0, 6).asChars()) == "Foobaz"); KJ_EXPECT(kj::str(privateMapping.slice(0, 6).asChars()) == "Foobaz");
file->write(0, StringPtr("qux").asBytes()); file->write(0, StringPtr("qux").asBytes());
......
c++/src/kj/filesystem-disk-unix.c++
View file @ ff4cd3d7
This diff is collapsed.
c++/src/kj/filesystem-disk-win32.c++
View file @ ff4cd3d7
This diff is collapsed.
c++/src/kj/filesystem-test.c++
View file @ ff4cd3d7
...@@ -86,6 +86,47 @@ KJ_TEST("Path") { ...@@ -86,6 +86,47 @@ KJ_TEST("Path") {
KJ_EXPECT(kj::str(Path({"foo", "bar"})) == "foo/bar"); KJ_EXPECT(kj::str(Path({"foo", "bar"})) == "foo/bar");
} }
KJ_TEST("Path comparisons") {
KJ_EXPECT(Path({"foo", "bar"}) == Path({"foo", "bar"}));
KJ_EXPECT(!(Path({"foo", "bar"}) != Path({"foo", "bar"})));
KJ_EXPECT(Path({"foo", "bar"}) != Path({"foo", "baz"}));
KJ_EXPECT(!(Path({"foo", "bar"}) == Path({"foo", "baz"})));
KJ_EXPECT(Path({"foo", "bar"}) != Path({"fob", "bar"}));
KJ_EXPECT(Path({"foo", "bar"}) != Path({"foo", "bar", "baz"}));
KJ_EXPECT(Path({"foo", "bar", "baz"}) != Path({"foo", "bar"}));
KJ_EXPECT(Path({"foo", "bar"}) <= Path({"foo", "bar"}));
KJ_EXPECT(Path({"foo", "bar"}) >= Path({"foo", "bar"}));
KJ_EXPECT(!(Path({"foo", "bar"}) < Path({"foo", "bar"})));
KJ_EXPECT(!(Path({"foo", "bar"}) > Path({"foo", "bar"})));
KJ_EXPECT(Path({"foo", "bar"}) < Path({"foo", "bar", "baz"}));
KJ_EXPECT(!(Path({"foo", "bar"}) > Path({"foo", "bar", "baz"})));
KJ_EXPECT(Path({"foo", "bar", "baz"}) > Path({"foo", "bar"}));
KJ_EXPECT(!(Path({"foo", "bar", "baz"}) < Path({"foo", "bar"})));
KJ_EXPECT(Path({"foo", "bar"}) < Path({"foo", "baz"}));
KJ_EXPECT(Path({"foo", "bar"}) > Path({"foo", "baa"}));
KJ_EXPECT(Path({"foo", "bar"}) > Path({"foo"}));
KJ_EXPECT(Path({"foo", "bar"}).startsWith(Path({})));
KJ_EXPECT(Path({"foo", "bar"}).startsWith(Path({"foo"})));
KJ_EXPECT(Path({"foo", "bar"}).startsWith(Path({"foo", "bar"})));
KJ_EXPECT(!Path({"foo", "bar"}).startsWith(Path({"foo", "bar", "baz"})));
KJ_EXPECT(!Path({"foo", "bar"}).startsWith(Path({"foo", "baz"})));
KJ_EXPECT(!Path({"foo", "bar"}).startsWith(Path({"baz", "foo", "bar"})));
KJ_EXPECT(!Path({"foo", "bar"}).startsWith(Path({"baz"})));
KJ_EXPECT(Path({"foo", "bar"}).endsWith(Path({})));
KJ_EXPECT(Path({"foo", "bar"}).endsWith(Path({"bar"})));
KJ_EXPECT(Path({"foo", "bar"}).endsWith(Path({"foo", "bar"})));
KJ_EXPECT(!Path({"foo", "bar"}).endsWith(Path({"baz", "foo", "bar"})));
KJ_EXPECT(!Path({"foo", "bar"}).endsWith(Path({"fob", "bar"})));
KJ_EXPECT(!Path({"foo", "bar"}).endsWith(Path({"foo", "bar", "baz"})));
KJ_EXPECT(!Path({"foo", "bar"}).endsWith(Path({"baz"})));
}
KJ_TEST("Path exceptions") { KJ_TEST("Path exceptions") {
KJ_EXPECT_THROW_MESSAGE("invalid path component", Path("")); KJ_EXPECT_THROW_MESSAGE("invalid path component", Path(""));
KJ_EXPECT_THROW_MESSAGE("invalid path component", Path(".")); KJ_EXPECT_THROW_MESSAGE("invalid path component", Path("."));
...@@ -223,13 +264,13 @@ public: ...@@ -223,13 +264,13 @@ public:
time += 1 * SECONDS; time += 1 * SECONDS;
} }
Date now() override { return time; } Date now() const override { return time; }
void expectChanged(FsNode& file) { void expectChanged(const FsNode& file) {
KJ_EXPECT(file.stat().lastModified == time); KJ_EXPECT(file.stat().lastModified == time);
time += 1 * SECONDS; time += 1 * SECONDS;
} }
void expectUnchanged(FsNode& file) { void expectUnchanged(const FsNode& file) {
KJ_EXPECT(file.stat().lastModified != time); KJ_EXPECT(file.stat().lastModified != time);
} }
......
c++/src/kj/filesystem.c++
View file @ ff4cd3d7
This diff is collapsed.
c++/src/kj/filesystem.h
View file @ ff4cd3d7
This diff is collapsed.
c++/src/kj/refcount.c++
View file @ ff4cd3d7
This diff is collapsed.
c++/src/kj/refcount.h
View file @ ff4cd3d7
This diff is collapsed.
c++/src/kj/time.c++
View file @ ff4cd3d7
...@@ -26,12 +26,12 @@ ...@@ -26,12 +26,12 @@
namespace kj { namespace kj {
Clock& nullClock() { const Clock& nullClock() {
class NullClock final: public Clock { class NullClock final: public Clock {
public: public:
Date now() override { return UNIX_EPOCH; } Date now() const override { return UNIX_EPOCH; }
}; };
static NullClock NULL_CLOCK; static KJ_CONSTEXPR(const) NullClock NULL_CLOCK;
return NULL_CLOCK; return NULL_CLOCK;
} }
......
c++/src/kj/time.h
View file @ ff4cd3d7
...@@ -63,10 +63,10 @@ constexpr Date UNIX_EPOCH = origin<Date>(); ...@@ -63,10 +63,10 @@ constexpr Date UNIX_EPOCH = origin<Date>();
class Clock { class Clock {
// Interface to read the current date and time. // Interface to read the current date and time.
public: public:
virtual Date now() = 0; virtual Date now() const = 0;
}; };
Clock& nullClock(); const Clock& nullClock();
// A clock which always returns UNIX_EPOCH as the current time. Useful when you don't care about // A clock which always returns UNIX_EPOCH as the current time. Useful when you don't care about
// time. // time.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment