Commit e422fc1d authored by Kenton Varda's avatar Kenton Varda

Simplify mutex usage in compiler code.

parent e1fdba61
......@@ -298,7 +298,7 @@ public:
}
private:
kj::Maybe<const Module&> loadModule(kj::StringPtr file) {
kj::Maybe<Module&> loadModule(kj::StringPtr file) {
size_t longestPrefix = 0;
for (auto& prefix: sourcePrefixes) {
......@@ -1218,12 +1218,12 @@ private:
kj::ArrayPtr<const char> content)
: globalReporter(globalReporter), lineBreaks(content) {}
void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) const override {
void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) override {
globalReporter.addError("<stdin>", lineBreaks.toSourcePos(startByte),
lineBreaks.toSourcePos(endByte), message);
}
bool hadErrors() const override {
bool hadErrors() override {
return globalReporter.hadErrors();
}
......@@ -1234,7 +1234,7 @@ private:
class ValueResolverGlue final: public ValueTranslator::Resolver {
public:
ValueResolverGlue(const SchemaLoader& loader, const ErrorReporter& errorReporter)
ValueResolverGlue(const SchemaLoader& loader, ErrorReporter& errorReporter)
: loader(loader), errorReporter(errorReporter) {}
kj::Maybe<Schema> resolveType(uint64_t id) {
......@@ -1266,14 +1266,14 @@ private:
private:
const SchemaLoader& loader;
const ErrorReporter& errorReporter;
ErrorReporter& errorReporter;
};
public:
// =====================================================================================
void addError(kj::StringPtr file, SourcePos start, SourcePos end,
kj::StringPtr message) const override {
kj::StringPtr message) override {
kj::String wholeMessage;
if (end.line == start.line) {
if (end.column == start.column) {
......@@ -1289,11 +1289,11 @@ public:
}
context.error(wholeMessage);
__atomic_store_n(&hadErrors_, true, __ATOMIC_RELAXED);
hadErrors_ = true;
}
bool hadErrors() const override {
return __atomic_load_n(&hadErrors_, __ATOMIC_RELAXED);
bool hadErrors() override {
return hadErrors_;
}
private:
......@@ -1326,7 +1326,7 @@ private:
struct SourceFile {
uint64_t id;
kj::StringPtr name;
const Module* module;
Module* module;
};
kj::Vector<SourceFile> sourceFiles;
......@@ -1337,7 +1337,7 @@ private:
};
kj::Vector<OutputDirective> outputs;
mutable bool hadErrors_ = false;
bool hadErrors_ = false;
};
} // namespace compiler
......
......@@ -39,15 +39,15 @@ namespace compiler {
class Compiler::Alias {
public:
Alias(const Node& parent, const DeclName::Reader& targetName)
Alias(Node& parent, const DeclName::Reader& targetName)
: parent(parent), targetName(targetName) {}
kj::Maybe<const Node&> getTarget() const;
kj::Maybe<Node&> getTarget();
private:
const Node& parent;
Node& parent;
DeclName::Reader targetName;
kj::Lazy<kj::Maybe<const Node&>> target;
kj::Lazy<kj::Maybe<Node&>> target;
};
class Compiler::Node: public NodeTranslator::Resolver {
......@@ -64,44 +64,46 @@ public:
explicit Node(CompiledModule& module);
// Create a root node representing the given file. May
Node(const Node& parent, const Declaration::Reader& declaration);
Node(Node& parent, const Declaration::Reader& declaration);
// Create a child node.
Node(kj::StringPtr name, Declaration::Which kind);
// Create a dummy node representing a built-in declaration, like "Int32" or "true".
uint64_t getId() const { return id; }
Declaration::Which getKind() const { return kind; }
uint64_t getId() { return id; }
Declaration::Which getKind() { return kind; }
kj::Maybe<const Node&> lookupMember(kj::StringPtr name) const;
kj::Maybe<Node&> lookupMember(kj::StringPtr name);
// Find a direct member of this node with the given name.
kj::Maybe<const Node&> lookupLexical(kj::StringPtr name) const;
kj::Maybe<Node&> lookupLexical(kj::StringPtr name);
// Look up the given name first as a member of this Node, then in its parent, and so on, until
// it is found or there are no more parents to search.
kj::Maybe<const Node&> lookup(const DeclName::Reader& name) const;
kj::Maybe<Node&> lookup(const DeclName::Reader& name);
// Resolve an arbitrary DeclName to a Node.
kj::Maybe<Schema> getBootstrapSchema() const;
kj::Maybe<schema::Node::Reader> getFinalSchema() const;
kj::Maybe<Schema> getBootstrapSchema();
kj::Maybe<schema::Node::Reader> getFinalSchema();
void loadFinalSchema(const SchemaLoader& loader);
void traverse(uint eagerness, std::unordered_map<const Node*, uint>& seen) const;
void traverse(uint eagerness, std::unordered_map<Node*, uint>& seen,
const SchemaLoader& finalLoader);
// Get the final schema for this node, and also possibly traverse the node's children and
// dependencies to ensure that they are loaded, depending on the mode.
void addError(kj::StringPtr error) const;
void addError(kj::StringPtr error);
// Report an error on this Node.
// implements NodeTranslator::Resolver -----------------------------
kj::Maybe<ResolvedName> resolve(const DeclName::Reader& name) const override;
kj::Maybe<Schema> resolveBootstrapSchema(uint64_t id) const override;
kj::Maybe<schema::Node::Reader> resolveFinalSchema(uint64_t id) const override;
kj::Maybe<uint64_t> resolveImport(kj::StringPtr name) const override;
kj::Maybe<ResolvedName> resolve(const DeclName::Reader& name) override;
kj::Maybe<Schema> resolveBootstrapSchema(uint64_t id) override;
kj::Maybe<schema::Node::Reader> resolveFinalSchema(uint64_t id) override;
kj::Maybe<uint64_t> resolveImport(kj::StringPtr name) override;
private:
const CompiledModule* module; // null iff isBuiltin is true
kj::Maybe<const Node&> parent;
CompiledModule* module; // null iff isBuiltin is true
kj::Maybe<Node&> parent;
Declaration::Reader declaration;
// AST of the declaration parsed from the schema file. May become invalid once the content
......@@ -135,13 +137,13 @@ private:
FINISHED
};
State state;
// Indicates which fields below are valid. Must update with atomic-release semantics.
// Indicates which fields below are valid.
inline bool stateHasReached(State minimumState) const {
return __atomic_load_n(&state, __ATOMIC_ACQUIRE) >= minimumState;
inline bool stateHasReached(State minimumState) {
return state >= minimumState;
}
inline void advanceState(State newState) {
__atomic_store_n(&state, newState, __ATOMIC_RELEASE);
state = newState;
}
// EXPANDED ------------------------------------
......@@ -166,15 +168,15 @@ private:
// FINISHED ------------------------------------
kj::Maybe<Schema> finalSchema;
// The complete schema as loaded by the compiler's main SchemaLoader. Null if the final
// loader threw an exception.
kj::Maybe<schema::Node::Reader> finalSchema;
// The completed schema, ready to load into the real schema loader.
kj::Array<Schema> auxSchemas;
kj::Array<schema::Node::Reader> auxSchemas;
// Schemas for all auxiliary nodes built by the NodeTranslator.
};
kj::MutexGuarded<Content> content;
Content guardedContent; // Read using getContent() only!
bool inGetContent = false; // True while getContent() is running; detects cycles.
// ---------------------------------------------
......@@ -183,46 +185,51 @@ private:
// Extract the ID from the declaration, or if it has none, generate one based on the name and
// parent ID.
static kj::StringPtr joinDisplayName(const kj::Arena& arena, const Node& parent,
static kj::StringPtr joinDisplayName(const kj::Arena& arena, Node& parent,
kj::StringPtr declName);
// Join the parent's display name with the child's unqualified name to construct the child's
// display name.
const Content& getContent(Content::State minimumState) const;
// Advances the content to at least the given state and returns it. Does not lock if the content
// is already at or past the given state.
kj::Maybe<Content&> getContent(Content::State minimumState);
// Advances the content to at least the given state and returns it. Returns null if getContent()
// is being called recursively and the given state has not yet been reached, as this indicates
// that the declaration recursively depends on itself.
void traverseNodeDependencies(const schema::Node::Reader& schemaNode, uint eagerness,
std::unordered_map<const Node*, uint>& seen) const;
std::unordered_map<Node*, uint>& seen,
const SchemaLoader& finalLoader);
void traverseType(const schema::Type::Reader& type, uint eagerness,
std::unordered_map<const Node*, uint>& seen) const;
std::unordered_map<Node*, uint>& seen,
const SchemaLoader& finalLoader);
void traverseAnnotations(const List<schema::Annotation>::Reader& annotations, uint eagerness,
std::unordered_map<const Node*, uint>& seen) const;
std::unordered_map<Node*, uint>& seen,
const SchemaLoader& finalLoader);
void traverseDependency(uint64_t depId, uint eagerness,
std::unordered_map<const Node*, uint>& seen,
bool ignoreIfNotFound = false) const;
std::unordered_map<Node*, uint>& seen,
const SchemaLoader& finalLoader,
bool ignoreIfNotFound = false);
// Helpers for traverse().
};
class Compiler::CompiledModule {
public:
CompiledModule(const Compiler::Impl& compiler, const Module& parserModule);
CompiledModule(Compiler::Impl& compiler, Module& parserModule);
const Compiler::Impl& getCompiler() const { return compiler; }
Compiler::Impl& getCompiler() { return compiler; }
const ErrorReporter& getErrorReporter() const { return parserModule; }
ParsedFile::Reader getParsedFile() const { return content.getReader(); }
const Node& getRootNode() const { return rootNode; }
kj::StringPtr getSourceName() const { return parserModule.getSourceName(); }
ErrorReporter& getErrorReporter() { return parserModule; }
ParsedFile::Reader getParsedFile() { return content.getReader(); }
Node& getRootNode() { return rootNode; }
kj::StringPtr getSourceName() { return parserModule.getSourceName(); }
kj::Maybe<const CompiledModule&> importRelative(kj::StringPtr importPath) const;
kj::Maybe<CompiledModule&> importRelative(kj::StringPtr importPath);
Orphan<List<schema::CodeGeneratorRequest::RequestedFile::Import>>
getFileImportTable(Orphanage orphanage) const;
getFileImportTable(Orphanage orphanage);
private:
const Compiler::Impl& compiler;
const Module& parserModule;
Compiler::Impl& compiler;
Module& parserModule;
MallocMessageBuilder contentArena;
Orphan<ParsedFile> content;
Node rootNode;
......@@ -233,12 +240,12 @@ public:
explicit Impl(AnnotationFlag annotationFlag);
virtual ~Impl() noexcept(false);
uint64_t add(const Module& module) const;
kj::Maybe<uint64_t> lookup(uint64_t parent, kj::StringPtr childName) const;
uint64_t add(Module& module);
kj::Maybe<uint64_t> lookup(uint64_t parent, kj::StringPtr childName);
Orphan<List<schema::CodeGeneratorRequest::RequestedFile::Import>>
getFileImportTable(const Module& module, Orphanage orphanage) const;
void eagerlyCompile(uint64_t id, uint eagerness) const;
const CompiledModule& addInternal(const Module& parsedModule) const;
getFileImportTable(Module& module, Orphanage orphanage);
void eagerlyCompile(uint64_t id, uint eagerness, const SchemaLoader& loader);
CompiledModule& addInternal(Module& parsedModule);
struct Workspace {
// Scratch space where stuff can be allocated while working. The Workspace is available
......@@ -267,33 +274,33 @@ public:
bootstrapLoader(loaderCallback) {}
};
const kj::Arena& getNodeArena() const { return nodeArena; }
const kj::Arena& getNodeArena() { return nodeArena; }
// Arena where nodes and other permanent objects should be allocated.
const SchemaLoader& getFinalLoader() const { return finalLoader; }
// Schema loader containing final versions of schemas.
const Workspace& getWorkspace() { return workspace; }
// Temporary workspace that can be used to construct bootstrap objects.
const Workspace& getWorkspace() const { return workspace.getAlreadyLockedShared(); }
// Temporary workspace that can be used to construct bootstrap objects. We assume that the
// caller already holds the workspace lock somewhere up the stack.
inline bool shouldCompileAnnotations() const {
inline bool shouldCompileAnnotations() {
return annotationFlag == AnnotationFlag::COMPILE_ANNOTATIONS;
}
void clearWorkspace();
// Reset the temporary workspace.
uint64_t addNode(uint64_t desiredId, Node& node) const;
uint64_t addNode(uint64_t desiredId, Node& node);
// Add the given node to the by-ID map under the given ID. If another node with the same ID
// already exists, choose a new one arbitrarily and use that instead. Return the ID that was
// finally used.
kj::Maybe<const Node&> findNode(uint64_t id) const;
kj::Maybe<Node&> findNode(uint64_t id);
kj::Maybe<const Node&> lookupBuiltin(kj::StringPtr name) const;
kj::Maybe<Node&> lookupBuiltin(kj::StringPtr name);
void load(const SchemaLoader& loader, uint64_t id) const override;
// SchemaLoader callback for the bootstrap loader.
void loadFinal(const SchemaLoader& loader, uint64_t id);
// Called from the SchemaLoader callback for the final loader.
private:
AnnotationFlag annotationFlag;
......@@ -301,33 +308,26 @@ private:
kj::Arena nodeArena;
// Arena used to allocate nodes and other permanent objects.
SchemaLoader finalLoader;
// The loader where we put final output of the compiler.
kj::MutexGuarded<Workspace> workspace;
Workspace workspace;
// The temporary workspace.
typedef std::unordered_map<const Module*, kj::Own<CompiledModule>> ModuleMap;
kj::MutexGuarded<ModuleMap> modules;
std::unordered_map<Module*, kj::Own<CompiledModule>> modules;
// Map of parser modules to compiler modules.
typedef std::unordered_map<uint64_t, const Node*> NodeMap;
kj::MutexGuarded<NodeMap> nodesById;
std::unordered_map<uint64_t, Node*> nodesById;
// Map of nodes by ID.
std::map<kj::StringPtr, kj::Own<Node>> builtinDecls;
// Map of built-in declarations, like "Int32" and "List", which make up the global scope.
mutable uint32_t nextBogusId = 1000;
// Counter for assigning bogus IDs to nodes whose real ID is a duplicate. 32-bit so that we
// can atomically increment it on 32-bit machines. It will never overflow since that would
// require compiling at least 2^32 nodes in one process.
uint64_t nextBogusId = 1000;
// Counter for assigning bogus IDs to nodes whose real ID is a duplicate.
};
// =======================================================================================
kj::Maybe<const Compiler::Node&> Compiler::Alias::getTarget() const {
return target.get([this](kj::SpaceFor<kj::Maybe<const Node&>>& space) {
kj::Maybe<Compiler::Node&> Compiler::Alias::getTarget() {
return target.get([this](kj::SpaceFor<kj::Maybe<Node&>>& space) {
return space.construct(parent.lookup(targetName));
});
}
......@@ -354,7 +354,7 @@ Compiler::Node::Node(CompiledModule& module)
id = module.getCompiler().addNode(id, *this);
}
Compiler::Node::Node(const Node& parent, const Declaration::Reader& declaration)
Compiler::Node::Node(Node& parent, const Declaration::Reader& declaration)
: module(parent.module),
parent(parent),
declaration(declaration),
......@@ -395,7 +395,7 @@ uint64_t Compiler::Node::generateId(uint64_t parentId, kj::StringPtr declName,
}
kj::StringPtr Compiler::Node::joinDisplayName(
const kj::Arena& arena, const Node& parent, kj::StringPtr declName) {
const kj::Arena& arena, Node& parent, kj::StringPtr declName) {
kj::ArrayPtr<char> result = arena.allocateArray<char>(
parent.displayName.size() + declName.size() + 2);
......@@ -407,16 +407,24 @@ kj::StringPtr Compiler::Node::joinDisplayName(
return kj::StringPtr(result.begin(), result.size() - 1);
}
const Compiler::Node::Content& Compiler::Node::getContent(Content::State minimumState) const {
kj::Maybe<Compiler::Node::Content&> Compiler::Node::getContent(Content::State minimumState) {
KJ_REQUIRE(!isBuiltin, "illegal method call for built-in declaration");
if (content.getWithoutLock().stateHasReached(minimumState)) {
return content.getWithoutLock();
auto& content = guardedContent;
if (content.stateHasReached(minimumState)) {
return content;
}
if (inGetContent) {
addError("Declaration recursively depends on itself.");
return nullptr;
}
auto locked = content.lockExclusive();
inGetContent = true;
KJ_DEFER(inGetContent = false);
switch (locked->state) {
switch (content.state) {
case Content::STUB: {
if (minimumState <= Content::STUB) break;
......@@ -433,8 +441,8 @@ const Compiler::Node::Content& Compiler::Node::getContent(Content::State minimum
case Declaration::INTERFACE: {
kj::Own<Node> subNode = arena.allocateOwn<Node>(*this, nestedDecl);
kj::StringPtr name = nestedDecl.getName().getValue();
locked->orderedNestedNodes.add(subNode);
locked->nestedNodes.insert(std::make_pair(name, kj::mv(subNode)));
content.orderedNestedNodes.add(subNode);
content.nestedNodes.insert(std::make_pair(name, kj::mv(subNode)));
break;
}
......@@ -442,7 +450,7 @@ const Compiler::Node::Content& Compiler::Node::getContent(Content::State minimum
kj::Own<Alias> alias = arena.allocateOwn<Alias>(
*this, nestedDecl.getUsing().getTarget());
kj::StringPtr name = nestedDecl.getName().getValue();
locked->aliases.insert(std::make_pair(name, kj::mv(alias)));
content.aliases.insert(std::make_pair(name, kj::mv(alias)));
break;
}
case Declaration::ENUMERANT:
......@@ -460,7 +468,7 @@ const Compiler::Node::Content& Compiler::Node::getContent(Content::State minimum
}
}
locked->advanceState(Content::EXPANDED);
content.advanceState(Content::EXPANDED);
// no break
}
......@@ -478,25 +486,25 @@ const Compiler::Node::Content& Compiler::Node::getContent(Content::State minimum
builder.setScopeId(p->id);
}
auto nestedNodes = builder.initNestedNodes(locked->orderedNestedNodes.size());
auto nestedNodes = builder.initNestedNodes(content.orderedNestedNodes.size());
auto nestedIter = nestedNodes.begin();
for (auto node: locked->orderedNestedNodes) {
for (auto node: content.orderedNestedNodes) {
nestedIter->setName(node->declaration.getName().getValue());
nestedIter->setId(node->id);
++nestedIter;
}
locked->translator = &workspace.arena.allocate<NodeTranslator>(
content.translator = &workspace.arena.allocate<NodeTranslator>(
*this, module->getErrorReporter(), declaration, kj::mv(schemaNode),
module->getCompiler().shouldCompileAnnotations());
KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&](){
auto nodeSet = locked->translator->getBootstrapNode();
auto nodeSet = content.translator->getBootstrapNode();
for (auto& auxNode: nodeSet.auxNodes) {
workspace.bootstrapLoader.loadOnce(auxNode);
}
locked->bootstrapSchema = workspace.bootstrapLoader.loadOnce(nodeSet.node);
content.bootstrapSchema = workspace.bootstrapLoader.loadOnce(nodeSet.node);
})) {
locked->bootstrapSchema = nullptr;
content.bootstrapSchema = nullptr;
// Only bother to report validation failures if we think we haven't seen any errors.
// Otherwise we assume that the errors caused the validation failure.
if (!module->getErrorReporter().hadErrors()) {
......@@ -508,15 +516,14 @@ const Compiler::Node::Content& Compiler::Node::getContent(Content::State minimum
// If the Workspace is destroyed while this Node is still in the BOOTSTRAP state,
// revert it to the EXPANDED state, because the NodeTranslator is no longer valid in this
// case.
Content* contentPtr = locked.get();
workspace.arena.copy(kj::defer([contentPtr]() {
contentPtr->bootstrapSchema = nullptr;
if (contentPtr->state == Content::BOOTSTRAP) {
contentPtr->state = Content::EXPANDED;
workspace.arena.copy(kj::defer([&content]() {
content.bootstrapSchema = nullptr;
if (content.state == Content::BOOTSTRAP) {
content.state = Content::EXPANDED;
}
}));
locked->advanceState(Content::BOOTSTRAP);
content.advanceState(Content::BOOTSTRAP);
// no break
}
......@@ -524,24 +531,11 @@ const Compiler::Node::Content& Compiler::Node::getContent(Content::State minimum
if (minimumState <= Content::BOOTSTRAP) break;
// Create the final schema.
auto nodeSet = locked->translator->finish();
KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&](){
locked->auxSchemas = KJ_MAP(auxNode, nodeSet.auxNodes) {
return module->getCompiler().getFinalLoader().loadOnce(auxNode);
};
locked->finalSchema = module->getCompiler().getFinalLoader().loadOnce(nodeSet.node);
})) {
locked->finalSchema = nullptr;
auto nodeSet = content.translator->finish();
content.finalSchema = nodeSet.node;
content.auxSchemas = kj::mv(nodeSet.auxNodes);
// Only bother to report validation failures if we think we haven't seen any errors.
// Otherwise we assume that the errors caused the validation failure.
if (!module->getErrorReporter().hadErrors()) {
addError(kj::str("Internal compiler bug: Schema failed validation:\n",
*exception));
}
}
locked->advanceState(Content::FINISHED);
content.advanceState(Content::FINISHED);
// no break
}
......@@ -549,29 +543,30 @@ const Compiler::Node::Content& Compiler::Node::getContent(Content::State minimum
break;
}
return *locked;
return content;
}
kj::Maybe<const Compiler::Node&> Compiler::Node::lookupMember(kj::StringPtr name) const {
kj::Maybe<Compiler::Node&> Compiler::Node::lookupMember(kj::StringPtr name) {
if (isBuiltin) return nullptr;
auto& content = getContent(Content::EXPANDED);
KJ_IF_MAYBE(content, getContent(Content::EXPANDED)) {
{
auto iter = content.nestedNodes.find(name);
if (iter != content.nestedNodes.end()) {
auto iter = content->nestedNodes.find(name);
if (iter != content->nestedNodes.end()) {
return *iter->second;
}
}
{
auto iter = content.aliases.find(name);
if (iter != content.aliases.end()) {
auto iter = content->aliases.find(name);
if (iter != content->aliases.end()) {
return iter->second->getTarget();
}
}
}
return nullptr;
}
kj::Maybe<const Compiler::Node&> Compiler::Node::lookupLexical(kj::StringPtr name) const {
kj::Maybe<Compiler::Node&> Compiler::Node::lookupLexical(kj::StringPtr name) {
KJ_REQUIRE(!isBuiltin, "illegal method call for built-in declaration");
auto result = lookupMember(name);
......@@ -585,10 +580,10 @@ kj::Maybe<const Compiler::Node&> Compiler::Node::lookupLexical(kj::StringPtr nam
return result;
}
kj::Maybe<const Compiler::Node&> Compiler::Node::lookup(const DeclName::Reader& name) const {
kj::Maybe<Compiler::Node&> Compiler::Node::lookup(const DeclName::Reader& name) {
KJ_REQUIRE(!isBuiltin, "illegal method call for built-in declaration");
const Node* node = nullptr;
Node* node = nullptr;
auto base = name.getBase();
switch (base.which()) {
......@@ -642,29 +637,55 @@ kj::Maybe<const Compiler::Node&> Compiler::Node::lookup(const DeclName::Reader&
return *node;
}
kj::Maybe<Schema> Compiler::Node::getBootstrapSchema() const {
auto& content = getContent(Content::BOOTSTRAP);
if (__atomic_load_n(&content.state, __ATOMIC_ACQUIRE) == Content::FINISHED &&
content.bootstrapSchema == nullptr) {
kj::Maybe<Schema> Compiler::Node::getBootstrapSchema() {
KJ_IF_MAYBE(content, getContent(Content::BOOTSTRAP)) {
if (content->state == Content::FINISHED && content->bootstrapSchema == nullptr) {
// The bootstrap schema was discarded. Copy it from the final schema.
// (We can't just return the final schema because using it could trigger schema loader
// callbacks that would deadlock.)
KJ_IF_MAYBE(finalSchema, content.finalSchema) {
return module->getCompiler().getWorkspace().bootstrapLoader.loadOnce(finalSchema->getProto());
KJ_IF_MAYBE(finalSchema, content->finalSchema) {
return module->getCompiler().getWorkspace().bootstrapLoader.loadOnce(*finalSchema);
} else {
return nullptr;
}
} else {
return content.bootstrapSchema;
return content->bootstrapSchema;
}
} else {
return nullptr;
}
}
kj::Maybe<schema::Node::Reader> Compiler::Node::getFinalSchema() {
KJ_IF_MAYBE(content, getContent(Content::FINISHED)) {
return content->finalSchema;
} else {
return nullptr;
}
}
kj::Maybe<schema::Node::Reader> Compiler::Node::getFinalSchema() const {
return getContent(Content::FINISHED).finalSchema.map(
[](const Schema& schema) { return schema.getProto(); });
void Compiler::Node::loadFinalSchema(const SchemaLoader& loader) {
KJ_IF_MAYBE(content, getContent(Content::FINISHED)) {
KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&](){
KJ_IF_MAYBE(finalSchema, content->finalSchema) {
KJ_MAP(auxSchema, content->auxSchemas) {
return loader.loadOnce(auxSchema);
};
loader.loadOnce(*finalSchema);
}
})) {
// Don't try loading this again.
content->finalSchema = nullptr;
// Only bother to report validation failures if we think we haven't seen any errors.
// Otherwise we assume that the errors caused the validation failure.
if (!module->getErrorReporter().hadErrors()) {
addError(kj::str("Internal compiler bug: Schema failed validation:\n", *exception));
}
}
}
}
void Compiler::Node::traverse(uint eagerness, std::unordered_map<const Node*, uint>& seen) const {
void Compiler::Node::traverse(uint eagerness, std::unordered_map<Node*, uint>& seen,
const SchemaLoader& finalLoader) {
uint& slot = seen[this];
if ((slot & eagerness) == eagerness) {
// We've already covered this node.
......@@ -672,54 +693,61 @@ void Compiler::Node::traverse(uint eagerness, std::unordered_map<const Node*, ui
}
slot |= eagerness;
KJ_IF_MAYBE(content, getContent(Content::FINISHED)) {
loadFinalSchema(finalLoader);
KJ_IF_MAYBE(schema, getFinalSchema()) {
if (eagerness / DEPENDENCIES != 0) {
// For traversing dependencies, discard the bits lower than DEPENDENCIES and replace
// them with the bits above DEPENDENCIES shifted over.
uint newEagerness = (eagerness & ~(DEPENDENCIES - 1)) | (eagerness / DEPENDENCIES);
traverseNodeDependencies(*schema, newEagerness, seen);
for (auto& aux: getContent(Content::FINISHED).auxSchemas) {
traverseNodeDependencies(aux.getProto(), newEagerness, seen);
traverseNodeDependencies(*schema, newEagerness, seen, finalLoader);
for (auto& aux: content->auxSchemas) {
traverseNodeDependencies(aux, newEagerness, seen, finalLoader);
}
}
}
}
if (eagerness & PARENTS) {
KJ_IF_MAYBE(p, parent) {
p->traverse(eagerness, seen);
p->traverse(eagerness, seen, finalLoader);
}
}
if (eagerness & CHILDREN) {
for (auto& child: getContent(Content::EXPANDED).orderedNestedNodes) {
child->traverse(eagerness, seen);
KJ_IF_MAYBE(content, getContent(Content::EXPANDED)) {
for (auto& child: content->orderedNestedNodes) {
child->traverse(eagerness, seen, finalLoader);
}
}
}
}
void Compiler::Node::traverseNodeDependencies(
const schema::Node::Reader& schemaNode, uint eagerness,
std::unordered_map<const Node*, uint>& seen) const {
std::unordered_map<Node*, uint>& seen,
const SchemaLoader& finalLoader) {
switch (schemaNode.which()) {
case schema::Node::STRUCT:
for (auto field: schemaNode.getStruct().getFields()) {
switch (field.which()) {
case schema::Field::SLOT:
traverseType(field.getSlot().getType(), eagerness, seen);
traverseType(field.getSlot().getType(), eagerness, seen, finalLoader);
break;
case schema::Field::GROUP:
// Aux node will be scanned later.
break;
}
traverseAnnotations(field.getAnnotations(), eagerness, seen);
traverseAnnotations(field.getAnnotations(), eagerness, seen, finalLoader);
}
break;
case schema::Node::ENUM:
for (auto enumerant: schemaNode.getEnum().getEnumerants()) {
traverseAnnotations(enumerant.getAnnotations(), eagerness, seen);
traverseAnnotations(enumerant.getAnnotations(), eagerness, seen, finalLoader);
}
break;
......@@ -727,13 +755,13 @@ void Compiler::Node::traverseNodeDependencies(
auto interface = schemaNode.getInterface();
for (auto extend: interface.getExtends()) {
if (extend != 0) { // if zero, we reported an error earlier
traverseDependency(extend, eagerness, seen);
traverseDependency(extend, eagerness, seen, finalLoader);
}
}
for (auto method: interface.getMethods()) {
traverseDependency(method.getParamStructType(), eagerness, seen, true);
traverseDependency(method.getResultStructType(), eagerness, seen, true);
traverseAnnotations(method.getAnnotations(), eagerness, seen);
traverseDependency(method.getParamStructType(), eagerness, seen, finalLoader, true);
traverseDependency(method.getResultStructType(), eagerness, seen, finalLoader, true);
traverseAnnotations(method.getAnnotations(), eagerness, seen, finalLoader);
}
break;
}
......@@ -742,11 +770,12 @@ void Compiler::Node::traverseNodeDependencies(
break;
}
traverseAnnotations(schemaNode.getAnnotations(), eagerness, seen);
traverseAnnotations(schemaNode.getAnnotations(), eagerness, seen, finalLoader);
}
void Compiler::Node::traverseType(const schema::Type::Reader& type, uint eagerness,
std::unordered_map<const Node*, uint>& seen) const {
std::unordered_map<Node*, uint>& seen,
const SchemaLoader& finalLoader) {
uint64_t id = 0;
switch (type.which()) {
case schema::Type::STRUCT:
......@@ -759,20 +788,21 @@ void Compiler::Node::traverseType(const schema::Type::Reader& type, uint eagerne
id = type.getInterface().getTypeId();
break;
case schema::Type::LIST:
traverseType(type.getList().getElementType(), eagerness, seen);
traverseType(type.getList().getElementType(), eagerness, seen, finalLoader);
return;
default:
return;
}
traverseDependency(id, eagerness, seen);
traverseDependency(id, eagerness, seen, finalLoader);
}
void Compiler::Node::traverseDependency(uint64_t depId, uint eagerness,
std::unordered_map<const Node*, uint>& seen,
bool ignoreIfNotFound) const {
std::unordered_map<Node*, uint>& seen,
const SchemaLoader& finalLoader,
bool ignoreIfNotFound) {
KJ_IF_MAYBE(node, module->getCompiler().findNode(depId)) {
node->traverse(eagerness, seen);
node->traverse(eagerness, seen, finalLoader);
} else if (!ignoreIfNotFound) {
KJ_FAIL_ASSERT("Dependency ID not present in compiler?", depId);
}
......@@ -780,27 +810,28 @@ void Compiler::Node::traverseDependency(uint64_t depId, uint eagerness,
void Compiler::Node::traverseAnnotations(const List<schema::Annotation>::Reader& annotations,
uint eagerness,
std::unordered_map<const Node*, uint>& seen) const {
std::unordered_map<Node*, uint>& seen,
const SchemaLoader& finalLoader) {
for (auto annotation: annotations) {
KJ_IF_MAYBE(node, module->getCompiler().findNode(annotation.getId())) {
node->traverse(eagerness, seen);
node->traverse(eagerness, seen, finalLoader);
}
}
}
void Compiler::Node::addError(kj::StringPtr error) const {
void Compiler::Node::addError(kj::StringPtr error) {
module->getErrorReporter().addError(startByte, endByte, error);
}
kj::Maybe<NodeTranslator::Resolver::ResolvedName> Compiler::Node::resolve(
const DeclName::Reader& name) const {
return lookup(name).map([](const Node& node) {
const DeclName::Reader& name) {
return lookup(name).map([](Node& node) {
return ResolvedName { node.id, node.kind };
});
}
kj::Maybe<Schema> Compiler::Node::resolveBootstrapSchema(uint64_t id) const {
kj::Maybe<Schema> Compiler::Node::resolveBootstrapSchema(uint64_t id) {
KJ_IF_MAYBE(node, module->getCompiler().findNode(id)) {
return node->getBootstrapSchema();
} else {
......@@ -808,7 +839,7 @@ kj::Maybe<Schema> Compiler::Node::resolveBootstrapSchema(uint64_t id) const {
}
}
kj::Maybe<schema::Node::Reader> Compiler::Node::resolveFinalSchema(uint64_t id) const {
kj::Maybe<schema::Node::Reader> Compiler::Node::resolveFinalSchema(uint64_t id) {
KJ_IF_MAYBE(node, module->getCompiler().findNode(id)) {
return node->getFinalSchema();
} else {
......@@ -816,7 +847,7 @@ kj::Maybe<schema::Node::Reader> Compiler::Node::resolveFinalSchema(uint64_t id)
}
}
kj::Maybe<uint64_t> Compiler::Node::resolveImport(kj::StringPtr name) const {
kj::Maybe<uint64_t> Compiler::Node::resolveImport(kj::StringPtr name) {
KJ_IF_MAYBE(m, module->importRelative(name)) {
return m->getRootNode().getId();
} else {
......@@ -826,16 +857,15 @@ kj::Maybe<uint64_t> Compiler::Node::resolveImport(kj::StringPtr name) const {
// =======================================================================================
Compiler::CompiledModule::CompiledModule(
const Compiler::Impl& compiler, const Module& parserModule)
Compiler::CompiledModule::CompiledModule(Compiler::Impl& compiler, Module& parserModule)
: compiler(compiler), parserModule(parserModule),
content(parserModule.loadContent(contentArena.getOrphanage())),
rootNode(*this) {}
kj::Maybe<const Compiler::CompiledModule&> Compiler::CompiledModule::importRelative(
kj::StringPtr importPath) const {
kj::Maybe<Compiler::CompiledModule&> Compiler::CompiledModule::importRelative(
kj::StringPtr importPath) {
return parserModule.importRelative(importPath).map(
[this](const Module& module) -> const Compiler::CompiledModule& {
[this](Module& module) -> Compiler::CompiledModule& {
return compiler.addInternal(module);
});
}
......@@ -913,7 +943,7 @@ static void findImports(Declaration::Reader decl, std::set<kj::StringPtr>& outpu
}
Orphan<List<schema::CodeGeneratorRequest::RequestedFile::Import>>
Compiler::CompiledModule::getFileImportTable(Orphanage orphanage) const {
Compiler::CompiledModule::getFileImportTable(Orphanage orphanage) {
std::set<kj::StringPtr> importNames;
findImports(content.getReader().getRoot(), importNames);
......@@ -935,7 +965,7 @@ Orphan<List<schema::CodeGeneratorRequest::RequestedFile::Import>>
// =======================================================================================
Compiler::Impl::Impl(AnnotationFlag annotationFlag)
: annotationFlag(annotationFlag), finalLoader(*this), workspace(*this) {
: annotationFlag(annotationFlag), workspace(*this) {
// Reflectively interpret the members of Declaration.body. Any member prefixed by "builtin"
// defines a builtin declaration visible in the global scope.
......@@ -956,17 +986,13 @@ Compiler::Impl::Impl(AnnotationFlag annotationFlag)
Compiler::Impl::~Impl() noexcept(false) {}
void Compiler::Impl::clearWorkspace() {
auto lock = workspace.lockExclusive();
// Make sure we reconstruct the workspace even if destroying it throws an exception.
KJ_DEFER(kj::ctor(*lock, *this));
kj::dtor(*lock);
KJ_DEFER(kj::ctor(workspace, *this));
kj::dtor(workspace);
}
const Compiler::CompiledModule& Compiler::Impl::addInternal(const Module& parsedModule) const {
auto locked = modules.lockExclusive();
kj::Own<CompiledModule>& slot = (*locked)[&parsedModule];
Compiler::CompiledModule& Compiler::Impl::addInternal(Module& parsedModule) {
kj::Own<CompiledModule>& slot = modules[&parsedModule];
if (slot.get() == nullptr) {
slot = kj::heap<CompiledModule>(*this, parsedModule);
}
......@@ -974,10 +1000,9 @@ const Compiler::CompiledModule& Compiler::Impl::addInternal(const Module& parsed
return *slot;
}
uint64_t Compiler::Impl::addNode(uint64_t desiredId, Node& node) const {
auto lock = nodesById.lockExclusive();
uint64_t Compiler::Impl::addNode(uint64_t desiredId, Node& node) {
for (;;) {
auto insertResult = lock->insert(std::make_pair(desiredId, &node));
auto insertResult = nodesById.insert(std::make_pair(desiredId, &node));
if (insertResult.second) {
return desiredId;
}
......@@ -992,21 +1017,20 @@ uint64_t Compiler::Impl::addNode(uint64_t desiredId, Node& node) const {
}
// Assign a new bogus ID.
desiredId = __atomic_fetch_add(&nextBogusId, 1, __ATOMIC_RELAXED);
desiredId = nextBogusId++;
}
}
kj::Maybe<const Compiler::Node&> Compiler::Impl::findNode(uint64_t id) const {
auto lock = nodesById.lockShared();
auto iter = lock->find(id);
if (iter == lock->end()) {
kj::Maybe<Compiler::Node&> Compiler::Impl::findNode(uint64_t id) {
auto iter = nodesById.find(id);
if (iter == nodesById.end()) {
return nullptr;
} else {
return *iter->second;
}
}
kj::Maybe<const Compiler::Node&> Compiler::Impl::lookupBuiltin(kj::StringPtr name) const {
kj::Maybe<Compiler::Node&> Compiler::Impl::lookupBuiltin(kj::StringPtr name) {
auto iter = builtinDecls.find(name);
if (iter == builtinDecls.end()) {
return nullptr;
......@@ -1015,11 +1039,11 @@ kj::Maybe<const Compiler::Node&> Compiler::Impl::lookupBuiltin(kj::StringPtr nam
}
}
uint64_t Compiler::Impl::add(const Module& module) const {
uint64_t Compiler::Impl::add(Module& module) {
return addInternal(module).getRootNode().getId();
}
kj::Maybe<uint64_t> Compiler::Impl::lookup(uint64_t parent, kj::StringPtr childName) const {
kj::Maybe<uint64_t> Compiler::Impl::lookup(uint64_t parent, kj::StringPtr childName) {
// Looking up members does not use the workspace, so we don't need to lock it.
KJ_IF_MAYBE(parentNode, findNode(parent)) {
KJ_IF_MAYBE(child, parentNode->lookupMember(childName)) {
......@@ -1033,61 +1057,75 @@ kj::Maybe<uint64_t> Compiler::Impl::lookup(uint64_t parent, kj::StringPtr childN
}
Orphan<List<schema::CodeGeneratorRequest::RequestedFile::Import>>
Compiler::Impl::getFileImportTable(const Module& module, Orphanage orphanage) const {
Compiler::Impl::getFileImportTable(Module& module, Orphanage orphanage) {
return addInternal(module).getFileImportTable(orphanage);
}
void Compiler::Impl::eagerlyCompile(uint64_t id, uint eagerness) const {
void Compiler::Impl::eagerlyCompile(uint64_t id, uint eagerness,
const SchemaLoader& finalLoader) {
KJ_IF_MAYBE(node, findNode(id)) {
auto lock = this->workspace.lockShared();
std::unordered_map<const Node*, uint> seen;
node->traverse(eagerness, seen);
std::unordered_map<Node*, uint> seen;
node->traverse(eagerness, seen, finalLoader);
} else {
KJ_FAIL_REQUIRE("id did not come from this Compiler.", id);
}
}
void Compiler::Impl::load(const SchemaLoader& loader, uint64_t id) const {
KJ_IF_MAYBE(node, findNode(id)) {
if (&loader == &finalLoader) {
auto lock = this->workspace.lockShared();
node->getFinalSchema();
} else {
// Must be the bootstrap loader. Workspace should already be locked.
this->workspace.getAlreadyLockedShared();
// We know that this load() is only called from the bootstrap loader which is already protected
// by our mutex, so we can drop thread-safety.
auto& self = const_cast<Compiler::Impl&>(*this);
KJ_IF_MAYBE(node, self.findNode(id)) {
node->getBootstrapSchema();
}
}
void Compiler::Impl::loadFinal(const SchemaLoader& loader, uint64_t id) {
KJ_IF_MAYBE(node, findNode(id)) {
KJ_IF_MAYBE(schema, node->getFinalSchema()) {
loader.loadOnce(*schema);
}
// Schema loads can happen lazily while using the dynamic API. The caller doesn't necessarily
// know that the compiler was invoked and so won't know to clear the workspace. Probably, if
// we don't clear the workspace, it will waste memory for the lifetime of the process, whereas
// if we do clear it, we're only imposing a little more work at startup time / the first time
// each type is used.
clearWorkspace();
}
}
// =======================================================================================
Compiler::Compiler(AnnotationFlag annotationFlag): impl(kj::heap<Impl>(annotationFlag)) {}
Compiler::Compiler(AnnotationFlag annotationFlag)
: impl(kj::heap<Impl>(annotationFlag)),
loader(*this) {}
Compiler::~Compiler() noexcept(false) {}
uint64_t Compiler::add(const Module& module) const {
return impl->add(module);
uint64_t Compiler::add(Module& module) const {
return impl.lockExclusive()->get()->add(module);
}
kj::Maybe<uint64_t> Compiler::lookup(uint64_t parent, kj::StringPtr childName) const {
return impl->lookup(parent, childName);
return impl.lockExclusive()->get()->lookup(parent, childName);
}
Orphan<List<schema::CodeGeneratorRequest::RequestedFile::Import>>
Compiler::getFileImportTable(const Module& module, Orphanage orphanage) const {
return impl->getFileImportTable(module, orphanage);
Compiler::getFileImportTable(Module& module, Orphanage orphanage) const {
return impl.lockExclusive()->get()->getFileImportTable(module, orphanage);
}
void Compiler::eagerlyCompile(uint64_t id, uint eagerness) const {
return impl->eagerlyCompile(id, eagerness);
impl.lockExclusive()->get()->eagerlyCompile(id, eagerness, loader);
}
const SchemaLoader& Compiler::getLoader() const {
return impl->getFinalLoader();
void Compiler::clearWorkspace() const {
impl.lockExclusive()->get()->clearWorkspace();
}
void Compiler::clearWorkspace() {
impl->clearWorkspace();
void Compiler::load(const SchemaLoader& loader, uint64_t id) const {
impl.lockExclusive()->get()->loadFinal(loader, id);
}
} // namespace compiler
......
......@@ -34,21 +34,23 @@ namespace compiler {
class Module: public ErrorReporter {
public:
virtual kj::StringPtr getSourceName() const = 0;
virtual kj::StringPtr getSourceName() = 0;
// The name of the module file relative to the source tree. Used to decide where to output
// generated code and to form the `displayName` in the schema.
virtual Orphan<ParsedFile> loadContent(Orphanage orphanage) const = 0;
virtual Orphan<ParsedFile> loadContent(Orphanage orphanage) = 0;
// Loads the module content, using the given orphanage to allocate objects if necessary.
virtual kj::Maybe<const Module&> importRelative(kj::StringPtr importPath) const = 0;
virtual kj::Maybe<Module&> importRelative(kj::StringPtr importPath) = 0;
// Find another module, relative to this one. Importing the same logical module twice should
// produce the exact same object, comparable by identity. These objects are owned by some
// outside pool that outlives the Compiler instance.
};
class Compiler {
class Compiler: private SchemaLoader::LazyLoadCallback {
// Cross-links separate modules (schema files) and translates them into schema nodes.
//
// This class is thread-safe, hence all its methods are const.
public:
enum AnnotationFlag {
......@@ -71,7 +73,7 @@ public:
~Compiler() noexcept(false);
KJ_DISALLOW_COPY(Compiler);
uint64_t add(const Module& module) const;
uint64_t add(Module& module) const;
// Add a module to the Compiler, returning the module's file ID. The ID can then be looked up in
// the `SchemaLoader` returned by `getLoader()`. However, the SchemaLoader may behave as if the
// schema node doesn't exist if any compilation errors occur (reported via the module's
......@@ -85,7 +87,7 @@ public:
// given name. Neither the parent nor the child schema node is actually compiled.
Orphan<List<schema::CodeGeneratorRequest::RequestedFile::Import>>
getFileImportTable(const Module& module, Orphanage orphanage) const;
getFileImportTable(Module& module, Orphanage orphanage) const;
// Build the import table for the CodeGeneratorRequest for the given module.
enum Eagerness: uint32_t {
......@@ -159,11 +161,11 @@ public:
// If this returns and no errors have been reported, then it is guaranteed that the compiled
// nodes can be found in the SchemaLoader returned by `getLoader()`.
const SchemaLoader& getLoader() const;
const SchemaLoader& getLoader() const { return loader; }
// Get a SchemaLoader backed by this compiler. Schema nodes will be lazily constructed as you
// traverse them using this loader.
void clearWorkspace();
void clearWorkspace() const;
// The compiler builds a lot of temporary tables and data structures while it works. It's
// useful to keep these around if more work is expected (especially if you are using lazy
// compilation and plan to look up Schema nodes that haven't already been seen), but once
......@@ -174,11 +176,14 @@ public:
private:
class Impl;
kj::Own<Impl> impl;
kj::MutexGuarded<kj::Own<Impl>> impl;
SchemaLoader loader;
class CompiledModule;
class Node;
class Alias;
void load(const SchemaLoader& loader, uint64_t id) const override;
};
} // namespace compiler
......
......@@ -36,20 +36,20 @@ class ErrorReporter {
// Callback for reporting errors within a particular file.
public:
virtual void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) const = 0;
virtual void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) = 0;
// Report an error at the given location in the input text. `startByte` and `endByte` indicate
// the span of text that is erroneous. They may be equal, in which case the parser was only
// able to identify where the error begins, not where it ends.
template <typename T>
inline void addErrorOn(T&& decl, kj::StringPtr message) const {
inline void addErrorOn(T&& decl, kj::StringPtr message) {
// Works for any `T` that defines `getStartByte()` and `getEndByte()` methods, which many
// of the Cap'n Proto types defined in `grammar.capnp` do.
addError(decl.getStartByte(), decl.getEndByte(), message);
}
virtual bool hadErrors() const = 0;
virtual bool hadErrors() = 0;
// Return true if any errors have been reported, globally. The main use case for this callback
// is to inhibit the reporting of errors which may have been caused by previous errors, or to
// allow the compiler to bail out entirely if it gets confused and thinks this could be because
......@@ -67,10 +67,10 @@ public:
};
virtual void addError(kj::StringPtr file, SourcePos start, SourcePos end,
kj::StringPtr message) const = 0;
kj::StringPtr message) = 0;
// Report an error at the given location in the given file.
virtual bool hadErrors() const = 0;
virtual bool hadErrors() = 0;
// Return true if any errors have been reported, globally. The main use case for this callback
// is to inhibit the reporting of errors which may have been caused by previous errors, or to
// allow the compiler to bail out entirely if it gets confused and thinks this could be because
......
......@@ -632,18 +632,18 @@ class ModuleImpl final: public Module {
public:
explicit ModuleImpl(ParsedFile::Reader content): content(content) {}
kj::StringPtr getSourceName() const override { return "evolving-schema.capnp"; }
Orphan<ParsedFile> loadContent(Orphanage orphanage) const override {
kj::StringPtr getSourceName() override { return "evolving-schema.capnp"; }
Orphan<ParsedFile> loadContent(Orphanage orphanage) override {
return orphanage.newOrphanCopy(content);
}
kj::Maybe<const Module&> importRelative(kj::StringPtr importPath) const override {
kj::Maybe<Module&> importRelative(kj::StringPtr importPath) override {
return nullptr;
}
void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) const override {
void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) override {
KJ_FAIL_ASSERT("Unexpected parse error.", startByte, endByte, message);
}
bool hadErrors() const override {
bool hadErrors() override {
return false;
}
......
......@@ -31,11 +31,11 @@ namespace {
class TestFailingErrorReporter: public ErrorReporter {
public:
void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) const override {
void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) override {
ADD_FAILURE() << "Parse failed: (" << startByte << "-" << endByte << ") " << message.cStr();
}
bool hadErrors() const override {
bool hadErrors() override {
// Not used by lexer.
return false;
}
......
......@@ -31,7 +31,7 @@ namespace compiler {
namespace p = kj::parse;
bool lex(kj::ArrayPtr<const char> input, LexedStatements::Builder result,
const ErrorReporter& errorReporter) {
ErrorReporter& errorReporter) {
Lexer lexer(Orphanage::getForMessageContaining(result), errorReporter);
auto parser = p::sequence(lexer.getParsers().statementSequence, p::endOfInput);
......@@ -53,7 +53,7 @@ bool lex(kj::ArrayPtr<const char> input, LexedStatements::Builder result,
}
bool lex(kj::ArrayPtr<const char> input, LexedTokens::Builder result,
const ErrorReporter& errorReporter) {
ErrorReporter& errorReporter) {
Lexer lexer(Orphanage::getForMessageContaining(result), errorReporter);
auto parser = p::sequence(lexer.getParsers().tokenSequence, p::endOfInput);
......@@ -138,7 +138,7 @@ constexpr auto docComment = p::optional(p::sequence(
} // namespace
Lexer::Lexer(Orphanage orphanageParam, const ErrorReporter& errorReporterParam)
Lexer::Lexer(Orphanage orphanageParam, ErrorReporter& errorReporterParam)
: orphanage(orphanageParam) {
// Note that because passing an lvalue to a parser constructor uses it by-referencee, it's safe
......
......@@ -33,9 +33,8 @@ namespace capnp {
namespace compiler {
bool lex(kj::ArrayPtr<const char> input, LexedStatements::Builder result,
const ErrorReporter& errorReporter);
bool lex(kj::ArrayPtr<const char> input, LexedTokens::Builder result,
const ErrorReporter& errorReporter);
ErrorReporter& errorReporter);
bool lex(kj::ArrayPtr<const char> input, LexedTokens::Builder result, ErrorReporter& errorReporter);
// Lex the given source code, placing the results in `result`. Returns true if there
// were no errors, false if there were. Even when errors are present, the file may have partial
// content which can be fed into later stages of parsing in order to find more errors.
......@@ -49,7 +48,7 @@ class Lexer {
// into your own parsers.
public:
Lexer(Orphanage orphanage, const ErrorReporter& errorReporter);
Lexer(Orphanage orphanage, ErrorReporter& errorReporter);
// `orphanage` is used to allocate Cap'n Proto message objects in the result. `inputStart` is
// a pointer to the beginning of the input, used to compute byte offsets.
......
......@@ -198,41 +198,40 @@ kj::String catPath(kj::StringPtr base, kj::StringPtr add) {
class ModuleLoader::Impl {
public:
Impl(const GlobalErrorReporter& errorReporter): errorReporter(errorReporter) {}
Impl(GlobalErrorReporter& errorReporter): errorReporter(errorReporter) {}
void addImportPath(kj::String path) {
searchPath.add(kj::heapString(kj::mv(path)));
}
kj::Maybe<const Module&> loadModule(kj::StringPtr localName, kj::StringPtr sourceName) const;
kj::Maybe<const Module&> loadModuleFromSearchPath(kj::StringPtr sourceName) const;
const GlobalErrorReporter& getErrorReporter() const { return errorReporter; }
kj::Maybe<Module&> loadModule(kj::StringPtr localName, kj::StringPtr sourceName);
kj::Maybe<Module&> loadModuleFromSearchPath(kj::StringPtr sourceName);
GlobalErrorReporter& getErrorReporter() { return errorReporter; }
private:
const GlobalErrorReporter& errorReporter;
GlobalErrorReporter& errorReporter;
kj::Vector<kj::String> searchPath;
kj::MutexGuarded<std::map<kj::StringPtr, kj::Own<Module>>> modules;
};
class ModuleLoader::ModuleImpl final: public Module {
public:
ModuleImpl(const ModuleLoader::Impl& loader, kj::String localName, kj::String sourceName)
ModuleImpl(ModuleLoader::Impl& loader, kj::String localName, kj::String sourceName)
: loader(loader), localName(kj::mv(localName)), sourceName(kj::mv(sourceName)) {}
kj::StringPtr getLocalName() const {
kj::StringPtr getLocalName() {
return localName;
}
kj::StringPtr getSourceName() const override {
kj::StringPtr getSourceName() override {
return sourceName;
}
Orphan<ParsedFile> loadContent(Orphanage orphanage) const override {
Orphan<ParsedFile> loadContent(Orphanage orphanage) override {
kj::Array<const char> content = mmapForRead(localName);
lineBreaks.get([&](kj::SpaceFor<LineBreakTable>& space) {
return space.construct(content);
});
lineBreaks = nullptr; // In case loadContent() is called multiple times.
lineBreaks = lineBreaksSpace.construct(content);
MallocMessageBuilder lexedBuilder;
auto statements = lexedBuilder.initRoot<LexedStatements>();
......@@ -243,7 +242,7 @@ public:
return parsed;
}
kj::Maybe<const Module&> importRelative(kj::StringPtr importPath) const override {
kj::Maybe<Module&> importRelative(kj::StringPtr importPath) override {
if (importPath.size() > 0 && importPath[0] == '/') {
return loader.loadModuleFromSearchPath(importPath.slice(1));
} else {
......@@ -251,32 +250,31 @@ public:
}
}
void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) const override {
auto& lines = lineBreaks.get(
[](kj::SpaceFor<LineBreakTable>& space) -> kj::Own<LineBreakTable> {
KJ_FAIL_REQUIRE("Can't report errors until loadContent() is called.");
});
void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) override {
auto& lines = *KJ_REQUIRE_NONNULL(lineBreaks,
"Can't report errors until loadContent() is called.");
loader.getErrorReporter().addError(
localName, lines.toSourcePos(startByte), lines.toSourcePos(endByte), message);
}
bool hadErrors() const override {
bool hadErrors() override {
return loader.getErrorReporter().hadErrors();
}
private:
const ModuleLoader::Impl& loader;
ModuleLoader::Impl& loader;
kj::String localName;
kj::String sourceName;
kj::Lazy<LineBreakTable> lineBreaks;
kj::SpaceFor<LineBreakTable> lineBreaksSpace;
kj::Maybe<kj::Own<LineBreakTable>> lineBreaks;
};
// =======================================================================================
kj::Maybe<const Module&> ModuleLoader::Impl::loadModule(
kj::StringPtr localName, kj::StringPtr sourceName) const {
kj::Maybe<Module&> ModuleLoader::Impl::loadModule(
kj::StringPtr localName, kj::StringPtr sourceName) {
kj::String canonicalLocalName = canonicalizePath(localName);
kj::String canonicalSourceName = canonicalizePath(sourceName);
......@@ -300,8 +298,7 @@ kj::Maybe<const Module&> ModuleLoader::Impl::loadModule(
return result;
}
kj::Maybe<const Module&> ModuleLoader::Impl::loadModuleFromSearchPath(
kj::StringPtr sourceName) const {
kj::Maybe<Module&> ModuleLoader::Impl::loadModuleFromSearchPath(kj::StringPtr sourceName) {
for (auto& search: searchPath) {
kj::String candidate = kj::str(search, "/", sourceName);
char* end = canonicalizePath(candidate.begin() + (candidate[0] == '/'));
......@@ -316,14 +313,13 @@ kj::Maybe<const Module&> ModuleLoader::Impl::loadModuleFromSearchPath(
// =======================================================================================
ModuleLoader::ModuleLoader(const GlobalErrorReporter& errorReporter)
ModuleLoader::ModuleLoader(GlobalErrorReporter& errorReporter)
: impl(kj::heap<Impl>(errorReporter)) {}
ModuleLoader::~ModuleLoader() noexcept(false) {}
void ModuleLoader::addImportPath(kj::String path) { impl->addImportPath(kj::mv(path)); }
kj::Maybe<const Module&> ModuleLoader::loadModule(
kj::StringPtr localName, kj::StringPtr sourceName) const {
kj::Maybe<Module&> ModuleLoader::loadModule(kj::StringPtr localName, kj::StringPtr sourceName) {
return impl->loadModule(localName, sourceName);
}
......
......@@ -35,7 +35,7 @@ namespace compiler {
class ModuleLoader {
public:
explicit ModuleLoader(const GlobalErrorReporter& errorReporter);
explicit ModuleLoader(GlobalErrorReporter& errorReporter);
// Create a ModuleLoader that reports error messages to the given reporter.
KJ_DISALLOW_COPY(ModuleLoader);
......@@ -45,7 +45,7 @@ public:
void addImportPath(kj::String path);
// Add a directory to the list of paths that is searched for imports that start with a '/'.
kj::Maybe<const Module&> loadModule(kj::StringPtr localName, kj::StringPtr sourceName) const;
kj::Maybe<Module&> loadModule(kj::StringPtr localName, kj::StringPtr sourceName);
// Tries to load the module with the given filename. `localName` is the path to the file on
// disk (as you'd pass to open(2)), and `sourceName` is the canonical name it should be given
// in the schema (this is used e.g. to decide output file locations). Often, these are the same.
......
......@@ -524,7 +524,7 @@ private:
// =======================================================================================
NodeTranslator::NodeTranslator(
const Resolver& resolver, const ErrorReporter& errorReporter,
Resolver& resolver, ErrorReporter& errorReporter,
const Declaration::Reader& decl, Orphan<schema::Node> wipNodeParam,
bool compileAnnotations)
: resolver(resolver), errorReporter(errorReporter),
......@@ -561,12 +561,12 @@ NodeTranslator::NodeSet NodeTranslator::finish() {
class NodeTranslator::DuplicateNameDetector {
public:
inline explicit DuplicateNameDetector(const ErrorReporter& errorReporter)
inline explicit DuplicateNameDetector(ErrorReporter& errorReporter)
: errorReporter(errorReporter) {}
void check(List<Declaration>::Reader nestedDecls, Declaration::Which parentKind);
private:
const ErrorReporter& errorReporter;
ErrorReporter& errorReporter;
std::map<kj::StringPtr, LocatedText::Reader> names;
};
......@@ -757,7 +757,7 @@ void NodeTranslator::compileAnnotation(Declaration::Annotation::Reader decl,
class NodeTranslator::DuplicateOrdinalDetector {
public:
DuplicateOrdinalDetector(const ErrorReporter& errorReporter): errorReporter(errorReporter) {}
DuplicateOrdinalDetector(ErrorReporter& errorReporter): errorReporter(errorReporter) {}
void check(LocatedInteger::Reader ordinal) {
if (ordinal.getValue() < expectedOrdinal) {
......@@ -780,7 +780,7 @@ public:
}
private:
const ErrorReporter& errorReporter;
ErrorReporter& errorReporter;
uint expectedOrdinal = 0;
kj::Maybe<LocatedInteger::Reader> lastOrdinalLocation;
};
......@@ -842,7 +842,7 @@ public:
private:
NodeTranslator& translator;
const ErrorReporter& errorReporter;
ErrorReporter& errorReporter;
StructLayout layout;
kj::Arena arena;
......
......@@ -49,11 +49,11 @@ public:
Declaration::Which kind;
};
virtual kj::Maybe<ResolvedName> resolve(const DeclName::Reader& name) const = 0;
virtual kj::Maybe<ResolvedName> resolve(const DeclName::Reader& name) = 0;
// Look up the given name, relative to this node, and return basic information about the
// target.
virtual kj::Maybe<Schema> resolveBootstrapSchema(uint64_t id) const = 0;
virtual kj::Maybe<Schema> resolveBootstrapSchema(uint64_t id) = 0;
// Get the schema for the given ID. If a schema is returned, it must be safe to traverse its
// dependencies using Schema::getDependency(). A schema that is only at the bootstrap stage
// is acceptable.
......@@ -62,7 +62,7 @@ public:
// traversing other schemas. Returns null if the ID is recognized, but the corresponding
// schema node failed to be built for reasons that were already reported.
virtual kj::Maybe<schema::Node::Reader> resolveFinalSchema(uint64_t id) const = 0;
virtual kj::Maybe<schema::Node::Reader> resolveFinalSchema(uint64_t id) = 0;
// Get the final schema for the given ID. A bootstrap schema is not acceptable. A raw
// node reader is returned rather than a Schema object because using a Schema object built
// by the final schema loader could trigger lazy initialization of dependencies which could
......@@ -72,11 +72,11 @@ public:
// traversing other schemas. Returns null if the ID is recognized, but the corresponding
// schema node failed to be built for reasons that were already reported.
virtual kj::Maybe<uint64_t> resolveImport(kj::StringPtr name) const = 0;
virtual kj::Maybe<uint64_t> resolveImport(kj::StringPtr name) = 0;
// Get the ID of an imported file given the import path.
};
NodeTranslator(const Resolver& resolver, const ErrorReporter& errorReporter,
NodeTranslator(Resolver& resolver, ErrorReporter& errorReporter,
const Declaration::Reader& decl, Orphan<schema::Node> wipNode,
bool compileAnnotations);
// Construct a NodeTranslator to translate the given declaration. The wipNode starts out with
......@@ -107,8 +107,8 @@ public:
// bootstrap node) and return it.
private:
const Resolver& resolver;
const ErrorReporter& errorReporter;
Resolver& resolver;
ErrorReporter& errorReporter;
Orphanage orphanage;
bool compileAnnotations;
......@@ -195,7 +195,7 @@ public:
virtual kj::Maybe<DynamicValue::Reader> resolveConstant(DeclName::Reader name) = 0;
};
ValueTranslator(Resolver& resolver, const ErrorReporter& errorReporter, Orphanage orphanage)
ValueTranslator(Resolver& resolver, ErrorReporter& errorReporter, Orphanage orphanage)
: resolver(resolver), errorReporter(errorReporter), orphanage(orphanage) {}
kj::Maybe<Orphan<DynamicValue>> compileValue(
......@@ -203,7 +203,7 @@ public:
private:
Resolver& resolver;
const ErrorReporter& errorReporter;
ErrorReporter& errorReporter;
Orphanage orphanage;
Orphan<DynamicValue> compileValueInner(ValueExpression::Reader src, schema::Type::Reader type);
......
......@@ -122,7 +122,7 @@ uint64_t generateMethodParamsId(uint64_t parentId, uint16_t methodOrdinal, bool
}
void parseFile(List<Statement>::Reader statements, ParsedFile::Builder result,
const ErrorReporter& errorReporter) {
ErrorReporter& errorReporter) {
CapnpParser parser(Orphanage::getForMessageContaining(result), errorReporter);
kj::Vector<Orphan<Declaration>> decls(statements.size());
......@@ -272,7 +272,7 @@ class ParseListItems {
// Transformer that parses all items in the input token sequence list using the given parser.
public:
constexpr ParseListItems(ItemParser&& itemParser, const ErrorReporter& errorReporter)
constexpr ParseListItems(ItemParser&& itemParser, ErrorReporter& errorReporter)
: itemParser(p::sequence(kj::fwd<ItemParser>(itemParser), p::endOfInput)),
errorReporter(errorReporter) {}
......@@ -310,12 +310,11 @@ public:
private:
decltype(p::sequence(kj::instance<ItemParser>(), p::endOfInput)) itemParser;
const ErrorReporter& errorReporter;
ErrorReporter& errorReporter;
};
template <typename ItemParser>
constexpr auto parenthesizedList(ItemParser&& itemParser,
const ErrorReporter& errorReporter) -> decltype(
constexpr auto parenthesizedList(ItemParser&& itemParser, ErrorReporter& errorReporter) -> decltype(
transform(rawParenthesizedList, ParseListItems<ItemParser>(
kj::fwd<ItemParser>(itemParser), errorReporter))) {
return transform(rawParenthesizedList, ParseListItems<ItemParser>(
......@@ -323,8 +322,7 @@ constexpr auto parenthesizedList(ItemParser&& itemParser,
}
template <typename ItemParser>
constexpr auto bracketedList(ItemParser&& itemParser,
const ErrorReporter& errorReporter) -> decltype(
constexpr auto bracketedList(ItemParser&& itemParser, ErrorReporter& errorReporter) -> decltype(
transform(rawBracketedList, ParseListItems<ItemParser>(
kj::fwd<ItemParser>(itemParser), errorReporter))) {
return transform(rawBracketedList, ParseListItems<ItemParser>(
......@@ -384,7 +382,7 @@ void initLocation(kj::parse::Span<List<Token>::Reader::Iterator> location,
// =======================================================================================
CapnpParser::CapnpParser(Orphanage orphanageParam, const ErrorReporter& errorReporterParam)
CapnpParser::CapnpParser(Orphanage orphanageParam, ErrorReporter& errorReporterParam)
: orphanage(orphanageParam), errorReporter(errorReporterParam) {
parsers.declName = arena.copy(p::transformWithLocation(
p::sequence(
......
......@@ -34,7 +34,7 @@ namespace capnp {
namespace compiler {
void parseFile(List<Statement>::Reader statements, ParsedFile::Builder result,
const ErrorReporter& errorReporter);
ErrorReporter& errorReporter);
// Parse a list of statements to build a ParsedFile.
//
// If any errors are reported, then the output is not usable. However, it may be passed on through
......@@ -59,7 +59,7 @@ class CapnpParser {
// them into your own parsers.
public:
CapnpParser(Orphanage orphanage, const ErrorReporter& errorReporter);
CapnpParser(Orphanage orphanage, ErrorReporter& errorReporter);
// `orphanage` is used to allocate Cap'n Proto message objects in the result. `inputStart` is
// a pointer to the beginning of the input, used to compute byte offsets.
......@@ -141,7 +141,7 @@ public:
private:
Orphanage orphanage;
const ErrorReporter& errorReporter;
ErrorReporter& errorReporter;
kj::Arena arena;
Parsers parsers;
};
......
......@@ -72,11 +72,11 @@ public:
ModuleImpl(const SchemaParser& parser, kj::Own<const SchemaFile>&& file)
: parser(parser), file(kj::mv(file)) {}
kj::StringPtr getSourceName() const override {
kj::StringPtr getSourceName() override {
return file->getDisplayName();
}
Orphan<compiler::ParsedFile> loadContent(Orphanage orphanage) const override {
Orphan<compiler::ParsedFile> loadContent(Orphanage orphanage) override {
kj::Array<const char> content = file->readContent();
lineBreaks.get([&](kj::SpaceFor<kj::Vector<uint>>& space) {
......@@ -99,7 +99,7 @@ public:
return parsed;
}
kj::Maybe<const Module&> importRelative(kj::StringPtr importPath) const override {
kj::Maybe<Module&> importRelative(kj::StringPtr importPath) override {
KJ_IF_MAYBE(importedFile, file->import(importPath)) {
return parser.getModuleImpl(kj::mv(*importedFile));
} else {
......@@ -107,7 +107,7 @@ public:
}
}
void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) const override {
void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) override {
auto& lines = lineBreaks.get(
[](kj::SpaceFor<kj::Vector<uint>>& space) {
KJ_FAIL_REQUIRE("Can't report errors until loadContent() is called.");
......@@ -129,7 +129,7 @@ public:
__atomic_store_n(&parser.hadErrors, true, __ATOMIC_RELAXED);
}
bool hadErrors() const override {
bool hadErrors() override {
return __atomic_load_n(&parser.hadErrors, __ATOMIC_RELAXED);
}
......@@ -162,7 +162,7 @@ struct SchemaFileEq {
struct SchemaParser::Impl {
typedef std::unordered_map<
const SchemaFile*, kj::Own<const ModuleImpl>, SchemaFileHash, SchemaFileEq> FileMap;
const SchemaFile*, kj::Own<ModuleImpl>, SchemaFileHash, SchemaFileEq> FileMap;
kj::MutexGuarded<FileMap> fileMap;
compiler::Compiler compiler;
};
......@@ -172,11 +172,11 @@ SchemaParser::~SchemaParser() noexcept(false) {}
ParsedSchema SchemaParser::parseDiskFile(
kj::StringPtr displayName, kj::StringPtr diskPath,
kj::ArrayPtr<const kj::StringPtr> importPath) {
kj::ArrayPtr<const kj::StringPtr> importPath) const {
return parseFile(SchemaFile::newDiskFile(displayName, diskPath, importPath));
}
ParsedSchema SchemaParser::parseFile(kj::Own<SchemaFile>&& file) {
ParsedSchema SchemaParser::parseFile(kj::Own<SchemaFile>&& file) const {
KJ_DEFER(impl->compiler.clearWorkspace());
uint64_t id = impl->compiler.add(getModuleImpl(kj::mv(file)));
impl->compiler.eagerlyCompile(id,
......@@ -185,7 +185,7 @@ ParsedSchema SchemaParser::parseFile(kj::Own<SchemaFile>&& file) {
return ParsedSchema(impl->compiler.getLoader().get(id), *this);
}
const SchemaParser::ModuleImpl& SchemaParser::getModuleImpl(kj::Own<SchemaFile>&& file) const {
SchemaParser::ModuleImpl& SchemaParser::getModuleImpl(kj::Own<SchemaFile>&& file) const {
auto lock = impl->fileMap.lockExclusive();
auto insertResult = lock->insert(std::make_pair(file.get(), kj::Own<ModuleImpl>()));
......@@ -196,14 +196,14 @@ const SchemaParser::ModuleImpl& SchemaParser::getModuleImpl(kj::Own<SchemaFile>&
return *insertResult.first->second;
}
kj::Maybe<ParsedSchema> ParsedSchema::findNested(kj::StringPtr name) {
kj::Maybe<ParsedSchema> ParsedSchema::findNested(kj::StringPtr name) const {
return parser->impl->compiler.lookup(getProto().getId(), name).map(
[this](uint64_t childId) {
return ParsedSchema(parser->impl->compiler.getLoader().get(childId), *parser);
});
}
ParsedSchema ParsedSchema::getNested(kj::StringPtr nestedName) {
ParsedSchema ParsedSchema::getNested(kj::StringPtr nestedName) const {
KJ_IF_MAYBE(nested, findNested(nestedName)) {
return *nested;
} else {
......
......@@ -34,13 +34,15 @@ class SchemaFile;
class SchemaParser {
// Parses `.capnp` files to produce `Schema` objects.
//
// This class is thread-safe, hence all its methods are const.
public:
SchemaParser();
~SchemaParser() noexcept(false);
ParsedSchema parseDiskFile(kj::StringPtr displayName, kj::StringPtr diskPath,
kj::ArrayPtr<const kj::StringPtr> importPath);
kj::ArrayPtr<const kj::StringPtr> importPath) const;
// Parse a file located on disk. Throws an exception if the file dosen't exist.
//
// Parameters:
......@@ -61,7 +63,7 @@ public:
// anything in the imported file -- only the imported types which are actually used are
// "dependencies".
ParsedSchema parseFile(kj::Own<SchemaFile>&& file);
ParsedSchema parseFile(kj::Own<SchemaFile>&& file) const;
// Advanced interface for parsing a file that may or may not be located in any global namespace.
// Most users will prefer `parseDiskFile()`.
//
......@@ -79,7 +81,7 @@ private:
kj::Own<Impl> impl;
mutable bool hadErrors = false;
const ModuleImpl& getModuleImpl(kj::Own<SchemaFile>&& file) const;
ModuleImpl& getModuleImpl(kj::Own<SchemaFile>&& file) const;
friend class ParsedSchema;
};
......@@ -91,11 +93,11 @@ class ParsedSchema: public Schema {
public:
inline ParsedSchema(): parser(nullptr) {}
kj::Maybe<ParsedSchema> findNested(kj::StringPtr name);
kj::Maybe<ParsedSchema> findNested(kj::StringPtr name) const;
// Gets the nested node with the given name, or returns null if there is no such nested
// declaration.
ParsedSchema getNested(kj::StringPtr name);
ParsedSchema getNested(kj::StringPtr name) const;
// Gets the nested node with the given name, or throws an exception if there is no such nested
// declaration.
......
......@@ -137,3 +137,5 @@ enum DupEnumerants {
dupNumber1 @2;
dupNumber2 @2;
}
const recursive: UInt32 = .recursive;
......@@ -48,3 +48,4 @@ file:136:3-10: error: 'dupName' is already defined in this scope.
file:135:3-10: error: 'dupName' previously defined here.
file:138:15-16: error: Duplicate ordinal number.
file:137:15-16: error: Ordinal @2 originally used here.
file:141:7-16: error: Declaration recursively depends on itself.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment