Skip to content

C
capnproto
Commit 0757e6ef authored by Kenton Varda's avatar Kenton Varda
Browse files
Options

Implement adoptWithCaveats() for struct lists, and some parser tweaks.

parent ce4162e8
Show whitespace changes
Inline Side-by-side
Showing with 336 additions and 157 deletions
c++/src/capnp/compiler/lexer-test.c++
View file @ 0757e6ef
......@@ -59,54 +59,54 @@ kj::String doLex(kj::StringPtr constText) {
TEST(Lexer, Tokens) {
EXPECT_STREQ(
"(tokens = ["
"(token = (body = identifier('foo'), endByte = 3)), "
"(token = (body = identifier('bar'), startByte = 4, endByte = 7))"
"(body = identifier('foo'), endByte = 3), "
"(body = identifier('bar'), startByte = 4, endByte = 7)"
"])",
doLex<LexedTokens>("foo bar").cStr());
EXPECT_STREQ(
"(tokens = ["
"(token = (body = identifier('foo'), endByte = 3)), "
"(token = (body = identifier('bar'), startByte = 15, endByte = 18))"
"(body = identifier('foo'), endByte = 3), "
"(body = identifier('bar'), startByte = 15, endByte = 18)"
"])",
doLex<LexedTokens>("foo # comment\n bar").cStr());
EXPECT_STREQ(
"(tokens = ["
"(token = (body = stringLiteral('foo '), startByte = 2, endByte = 11)), "
"(token = (body = integerLiteral(123), startByte = 12, endByte = 15)), "
"(token = (body = floatLiteral(2.75), startByte = 16, endByte = 20)), "
"(token = (body = floatLiteral(60000), startByte = 21, endByte = 24)), "
"(token = (body = operator('+'), startByte = 25, endByte = 26)), "
"(token = (body = operator('-='), startByte = 27, endByte = 29))"
"(body = stringLiteral('foo '), startByte = 2, endByte = 11), "
"(body = integerLiteral(123), startByte = 12, endByte = 15), "
"(body = floatLiteral(2.75), startByte = 16, endByte = 20), "
"(body = floatLiteral(60000), startByte = 21, endByte = 24), "
"(body = operator('+'), startByte = 25, endByte = 26), "
"(body = operator('-='), startByte = 27, endByte = 29)"
"])",
doLex<LexedTokens>(" 'foo\\x20' 123 2.75 6e4 + -= ").cStr());
EXPECT_STREQ(
"(tokens = ["
"(token = (body = parenthesizedList(["
"(body = parenthesizedList(["
"["
"(token = (body = identifier('foo'), startByte = 1, endByte = 4)), "
"(token = (body = identifier('bar'), startByte = 5, endByte = 8))"
"(body = identifier('foo'), startByte = 1, endByte = 4), "
"(body = identifier('bar'), startByte = 5, endByte = 8)"
"], ["
"(token = (body = identifier('baz'), startByte = 10, endByte = 13)), "
"(token = (body = identifier('qux'), startByte = 14, endByte = 17))"
"(body = identifier('baz'), startByte = 10, endByte = 13), "
"(body = identifier('qux'), startByte = 14, endByte = 17)"
"], ["
"(token = (body = identifier('corge'), startByte = 19, endByte = 24)), "
"(token = (body = identifier('grault'), startByte = 25, endByte = 31))"
"(body = identifier('corge'), startByte = 19, endByte = 24), "
"(body = identifier('grault'), startByte = 25, endByte = 31)"
"]"
"]), endByte = 32))"
"]), endByte = 32)"
"])",
doLex<LexedTokens>("(foo bar, baz qux, corge grault)").cStr());
EXPECT_STREQ(
"(tokens = ["
"(token = (body = parenthesizedList(["
"(body = parenthesizedList(["
"["
"(token = (body = identifier('foo'), startByte = 1, endByte = 4)), "
"(token = (body = identifier('bar'), startByte = 5, endByte = 8))"
"(body = identifier('foo'), startByte = 1, endByte = 4), "
"(body = identifier('bar'), startByte = 5, endByte = 8)"
"]"
"]), endByte = 9))"
"]), endByte = 9)"
"])",
doLex<LexedTokens>("(foo bar)").cStr());
......@@ -114,50 +114,50 @@ TEST(Lexer, Tokens) {
// list.
EXPECT_STREQ(
"(tokens = ["
"(token = (body = parenthesizedList([]), endByte = 4))"
"(body = parenthesizedList([]), endByte = 4)"
"])",
doLex<LexedTokens>("( )").cStr());
EXPECT_STREQ(
"(tokens = ["
"(token = (body = bracketedList(["
"(body = bracketedList(["
"["
"(token = (body = identifier('foo'), startByte = 1, endByte = 4)), "
"(token = (body = identifier('bar'), startByte = 5, endByte = 8))"
"(body = identifier('foo'), startByte = 1, endByte = 4), "
"(body = identifier('bar'), startByte = 5, endByte = 8)"
"], ["
"(token = (body = identifier('baz'), startByte = 10, endByte = 13)), "
"(token = (body = identifier('qux'), startByte = 14, endByte = 17))"
"(body = identifier('baz'), startByte = 10, endByte = 13), "
"(body = identifier('qux'), startByte = 14, endByte = 17)"
"], ["
"(token = (body = identifier('corge'), startByte = 19, endByte = 24)), "
"(token = (body = identifier('grault'), startByte = 25, endByte = 31))"
"(body = identifier('corge'), startByte = 19, endByte = 24), "
"(body = identifier('grault'), startByte = 25, endByte = 31)"
"]"
"]), endByte = 32))"
"]), endByte = 32)"
"])",
doLex<LexedTokens>("[foo bar, baz qux, corge grault]").cStr());
EXPECT_STREQ(
"(tokens = ["
"(token = (body = bracketedList(["
"(body = bracketedList(["
"["
"(token = (body = identifier('foo'), startByte = 1, endByte = 4))"
"(body = identifier('foo'), startByte = 1, endByte = 4)"
"], ["
"(token = (body = parenthesizedList(["
"(body = parenthesizedList(["
"["
"(token = (body = identifier('bar'), startByte = 7, endByte = 10))"
"(body = identifier('bar'), startByte = 7, endByte = 10)"
"], ["
"(token = (body = identifier('baz'), startByte = 12, endByte = 15))"
"(body = identifier('baz'), startByte = 12, endByte = 15)"
"]"
"]), startByte = 6, endByte = 16))"
"]), startByte = 6, endByte = 16)"
"]"
"]), endByte = 17)), "
"(token = (body = identifier('qux'), startByte = 18, endByte = 21))"
"]), endByte = 17), "
"(body = identifier('qux'), startByte = 18, endByte = 21)"
"])",
doLex<LexedTokens>("[foo, (bar, baz)] qux").cStr());
EXPECT_STREQ(
"(tokens = ["
"(token = (body = identifier('foo'), endByte = 3)), "
"(token = (body = identifier('bar'), startByte = 7, endByte = 10))"
"(body = identifier('foo'), endByte = 3), "
"(body = identifier('bar'), startByte = 7, endByte = 10)"
"])",
doLex<LexedTokens>("foo\n\r\t\vbar").cStr());
}
......@@ -165,45 +165,46 @@ TEST(Lexer, Tokens) {
TEST(Lexer, Statements) {
EXPECT_STREQ(
"(statements = ["
"(statement = (tokens = ["
"(token = (body = identifier('foo'), endByte = 3)), "
"(token = (body = identifier('bar'), startByte = 4, endByte = 7))"
"]))"
"(tokens = ["
"(body = identifier('foo'), endByte = 3), "
"(body = identifier('bar'), startByte = 4, endByte = 7)"
"], endByte = 8)"
"])",
doLex<LexedStatements>("foo bar;").cStr());
EXPECT_STREQ(
"(statements = ["
"(statement = (tokens = ["
"(token = (body = identifier('foo'), endByte = 3))"
"])), "
"(statement = (tokens = ["
"(token = (body = identifier('bar'), startByte = 5, endByte = 8))"
"])), "
"(statement = (tokens = ["
"(token = (body = identifier('baz'), startByte = 10, endByte = 13))"
"]))"
"(tokens = ["
"(body = identifier('foo'), endByte = 3)"
"], endByte = 4), "
"(tokens = ["
"(body = identifier('bar'), startByte = 5, endByte = 8)"
"], startByte = 5, endByte = 9), "
"(tokens = ["
"(body = identifier('baz'), startByte = 10, endByte = 13)"
"], startByte = 10, endByte = 14)"
"])",
doLex<LexedStatements>("foo; bar; baz; ").cStr());
EXPECT_STREQ(
"(statements = ["
"(statement = ("
"("
"tokens = ["
"(token = (body = identifier('foo'), endByte = 3))"
"(body = identifier('foo'), endByte = 3)"
"], "
"block = statements(["
"(statement = (tokens = ["
"(token = (body = identifier('bar'), startByte = 5, endByte = 8))"
"])), "
"(statement = (tokens = ["
"(token = (body = identifier('baz'), startByte = 10, endByte = 13))"
"]))"
"]))"
"(tokens = ["
"(body = identifier('bar'), startByte = 5, endByte = 8)"
"], startByte = 5, endByte = 9), "
"(tokens = ["
"(body = identifier('baz'), startByte = 10, endByte = 13)"
"], startByte = 10, endByte = 14)"
"]), "
"endByte = 15"
"), "
"(statement = (tokens = ["
"(token = (body = identifier('qux'), startByte = 16, endByte = 19))"
"]))"
"(tokens = ["
"(body = identifier('qux'), startByte = 16, endByte = 19)"
"], startByte = 16, endByte = 20)"
"])",
doLex<LexedStatements>("foo {bar; baz;} qux;").cStr());
}
......@@ -211,90 +212,120 @@ TEST(Lexer, Statements) {
TEST(Lexer, DocComments) {
EXPECT_STREQ(
"(statements = ["
"(statement = ("
"("
"tokens = ["
"(token = (body = identifier('foo'), endByte = 3))"
"(body = identifier('foo'), endByte = 3)"
"], "
"docComment = 'blah blah\\n'"
"))"
"docComment = 'blah blah\\n', "
"endByte = 16"
")"
"])",
doLex<LexedStatements>("foo; # blah blah").cStr());
EXPECT_STREQ(
"(statements = ["
"(statement = ("
"("
"tokens = ["
"(token = (body = identifier('foo'), endByte = 3))"
"(body = identifier('foo'), endByte = 3)"
"], "
"docComment = 'blah blah\\n'"
"))"
"docComment = 'blah blah\\n', "
"endByte = 15"
")"
"])",
doLex<LexedStatements>("foo; #blah blah").cStr());
EXPECT_STREQ(
"(statements = ["
"(statement = ("
"("
"tokens = ["
"(token = (body = identifier('foo'), endByte = 3))"
"(body = identifier('foo'), endByte = 3)"
"], "
"docComment = ' blah blah\\n'"
"))"
"docComment = ' blah blah\\n', "
"endByte = 17"
")"
"])",
doLex<LexedStatements>("foo; # blah blah").cStr());
EXPECT_STREQ(
"(statements = ["
"(statement = ("
"("
"tokens = ["
"(token = (body = identifier('foo'), endByte = 3))"
"(body = identifier('foo'), endByte = 3)"
"], "
"docComment = 'blah blah\\n'"
"))"
"docComment = 'blah blah\\n', "
"endByte = 16"
")"
"])",
doLex<LexedStatements>("foo;\n# blah blah").cStr());
EXPECT_STREQ(
"(statements = ["
"(statement = ("
"("
"tokens = ["
"(token = (body = identifier('foo'), endByte = 3))"
"]"
"))"
"(body = identifier('foo'), endByte = 3)"
"], "
"endByte = 4"
")"
"])",
doLex<LexedStatements>("foo;\n\n# blah blah").cStr());
EXPECT_STREQ(
"(statements = ["
"(statement = ("
"("
"tokens = ["
"(token = (body = identifier('foo'), endByte = 3))"
"(body = identifier('foo'), endByte = 3)"
"], "
"docComment = 'bar baz\\nqux corge\\n'"
"))"
"docComment = 'bar baz\\nqux corge\\n', "
"endByte = 30"
")"
"])",
doLex<LexedStatements>("foo;\n # bar baz\n # qux corge\n\n# grault\n# garply").cStr());
EXPECT_STREQ(
"(statements = ["
"(statement = ("
"("
"tokens = ["
"(token = (body = identifier('foo'), endByte = 3))"
"(body = identifier('foo'), endByte = 3)"
"], "
"block = statements(["
"(statement = (tokens = ["
"(token = (body = identifier('bar'), startByte = 17, endByte = 20))"
"], docComment = 'hi\\n')), "
"(statement = (tokens = ["
"(token = (body = identifier('baz'), startByte = 28, endByte = 31))"
"]))"
"(tokens = ["
"(body = identifier('bar'), startByte = 17, endByte = 20)"
"], docComment = 'hi\\n', startByte = 17, endByte = 27), "
"(tokens = ["
"(body = identifier('baz'), startByte = 28, endByte = 31)"
"], startByte = 28, endByte = 32)"
"]), "
"docComment = 'blah blah\\n'"
")), "
"(statement = (tokens = ["
"(token = (body = identifier('qux'), startByte = 44, endByte = 47))"
"]))"
"docComment = 'blah blah\\n', "
"endByte = 44"
"), "
"(tokens = ["
"(body = identifier('qux'), startByte = 44, endByte = 47)"
"], startByte = 44, endByte = 48)"
"])",
doLex<LexedStatements>("foo {# blah blah\nbar; # hi\n baz;} # ignored\nqux;").cStr());
EXPECT_STREQ(
"(statements = ["
"("
"tokens = ["
"(body = identifier('foo'), endByte = 3)"
"], "
"block = statements(["
"(tokens = ["
"(body = identifier('bar'), startByte = 5, endByte = 8)"
"], startByte = 5, endByte = 9), "
"(tokens = ["
"(body = identifier('baz'), startByte = 10, endByte = 13)"
"], startByte = 10, endByte = 14)"
"]), "
"docComment = 'late comment\\n', "
"endByte = 31"
"), "
"(tokens = ["
"(body = identifier('qux'), startByte = 31, endByte = 34)"
"], startByte = 31, endByte = 35)"
"])",
doLex<LexedStatements>("foo {bar; baz;}\n# late comment\nqux;").cStr());
}
} // namespace
......
c++/src/capnp/compiler/lexer.c++
View file @ 0757e6ef
......@@ -42,7 +42,7 @@ bool lex(kj::ArrayPtr<const char> input, LexedStatements::Builder result) {
KJ_IF_MAYBE(output, parseOutput) {
auto l = result.initStatements(output->size());
for (uint i = 0; i < output->size(); i++) {
l[i].adoptStatement(kj::mv((*output)[i]));
l.adoptWithCaveats(i, kj::mv((*output)[i]));
}
return true;
} else {
......@@ -64,7 +64,7 @@ bool lex(kj::ArrayPtr<const char> input, LexedTokens::Builder result) {
KJ_IF_MAYBE(output, parseOutput) {
auto l = result.initTokens(output->size());
for (uint i = 0; i < output->size(); i++) {
l[i].adoptToken(kj::mv((*output)[i]));
l.adoptWithCaveats(i, kj::mv((*output)[i]));
}
return true;
} else {
......@@ -85,13 +85,13 @@ Token::Body::Builder initTok(Orphan<Token>& t, const Location& loc) {
return tb.getBody();
}
void buildTokenSequenceList(List<List<TokenPointer>>::Builder builder,
void buildTokenSequenceList(List<List<Token>>::Builder builder,
kj::Array<kj::Array<Orphan<Token>>>&& items) {
for (uint i = 0; i < items.size(); i++) {
auto& item = items[i];
auto itemBuilder = builder.init(i, item.size());
for (uint j = 0; j < item.size(); j++) {
itemBuilder[j].adoptToken(kj::mv(item[j]));
itemBuilder.adoptWithCaveats(j, kj::mv(item[j]));
}
}
}
......@@ -101,7 +101,6 @@ void attachDocComment(Statement::Builder statement, kj::Array<kj::String>&& comm
for (auto& line: comment) {
size += line.size() + 1; // include newline
}
if (size > 0) {
Text::Builder builder = statement.initDocComment(size);
char* pos = builder.begin();
for (auto& line: comment) {
......@@ -110,7 +109,6 @@ void attachDocComment(Statement::Builder statement, kj::Array<kj::String>&& comm
*pos++ = '\n';
}
KJ_ASSERT(pos == builder.end());
}
}
constexpr auto discardComment =
......@@ -131,16 +129,16 @@ constexpr auto newline = p::oneOf(
p::exactChar<'\n'>(),
sequence(p::exactChar<'\r'>(), p::discard(p::optional(p::exactChar<'\n'>()))));
constexpr auto docComment = sequence(
constexpr auto docComment = p::optional(p::sequence(
discardLineWhitespace,
p::discard(p::optional(newline)),
p::many(p::sequence(discardLineWhitespace, saveComment)));
p::oneOrMore(p::sequence(discardLineWhitespace, saveComment))));
// Parses a set of comment lines preceded by at most one newline and with no intervening blank
// lines.
} // namespace
Lexer::Lexer(Orphanage orphanage): orphanage(orphanage) {
Lexer::Lexer(Orphanage orphanageParam): orphanage(orphanageParam) {
// Note that because passing an lvalue to a parser constructor uses it by-referencee, it's safe
// for us to use parsers.tokenSequence even though we haven't yet constructed it.
......@@ -148,7 +146,7 @@ Lexer::Lexer(Orphanage orphanage): orphanage(orphanage) {
auto& commaDelimitedList = arena.copy(p::transform(
p::sequence(tokenSequence, p::many(p::sequence(p::exactChar<','>(), tokenSequence))),
[&](kj::Array<Orphan<Token>>&& first, kj::Array<kj::Array<Orphan<Token>>>&& rest)
[this](kj::Array<Orphan<Token>>&& first, kj::Array<kj::Array<Orphan<Token>>>&& rest)
-> kj::Array<kj::Array<Orphan<Token>>> {
if (first == nullptr && rest == nullptr) {
// Completely empty list.
......@@ -165,39 +163,39 @@ Lexer::Lexer(Orphanage orphanage): orphanage(orphanage) {
auto& token = arena.copy(p::oneOf(
p::transformWithLocation(p::identifier,
[&](Location loc, kj::String name) -> Orphan<Token> {
[this](Location loc, kj::String name) -> Orphan<Token> {
auto t = orphanage.newOrphan<Token>();
initTok(t, loc).setIdentifier(name);
return t;
}),
p::transformWithLocation(p::doubleQuotedString,
[&](Location loc, kj::String text) -> Orphan<Token> {
[this](Location loc, kj::String text) -> Orphan<Token> {
auto t = orphanage.newOrphan<Token>();
initTok(t, loc).setStringLiteral(text);
return t;
}),
p::transformWithLocation(p::integer,
[&](Location loc, uint64_t i) -> Orphan<Token> {
[this](Location loc, uint64_t i) -> Orphan<Token> {
auto t = orphanage.newOrphan<Token>();
initTok(t, loc).setIntegerLiteral(i);
return t;
}),
p::transformWithLocation(p::number,
[&](Location loc, double x) -> Orphan<Token> {
[this](Location loc, double x) -> Orphan<Token> {
auto t = orphanage.newOrphan<Token>();
initTok(t, loc).setFloatLiteral(x);
return t;
}),
p::transformWithLocation(
p::charsToString(p::oneOrMore(p::anyOfChars("!$%&*+-./:<=>?@^|~"))),
[&](Location loc, kj::String text) -> Orphan<Token> {
[this](Location loc, kj::String text) -> Orphan<Token> {
auto t = orphanage.newOrphan<Token>();
initTok(t, loc).setOperator(text);
return t;
}),
p::transformWithLocation(
sequence(p::exactChar<'('>(), commaDelimitedList, p::exactChar<')'>()),
[&](Location loc, kj::Array<kj::Array<Orphan<Token>>>&& items) -> Orphan<Token> {
[this](Location loc, kj::Array<kj::Array<Orphan<Token>>>&& items) -> Orphan<Token> {
auto t = orphanage.newOrphan<Token>();
buildTokenSequenceList(
initTok(t, loc).initParenthesizedList(items.size()), kj::mv(items));
......@@ -205,7 +203,7 @@ Lexer::Lexer(Orphanage orphanage): orphanage(orphanage) {
}),
p::transformWithLocation(
sequence(p::exactChar<'['>(), commaDelimitedList, p::exactChar<']'>()),
[&](Location loc, kj::Array<kj::Array<Orphan<Token>>>&& items) -> Orphan<Token> {
[this](Location loc, kj::Array<kj::Array<Orphan<Token>>>&& items) -> Orphan<Token> {
auto t = orphanage.newOrphan<Token>();
buildTokenSequenceList(
initTok(t, loc).initBracketedList(items.size()), kj::mv(items));
......@@ -219,34 +217,46 @@ Lexer::Lexer(Orphanage orphanage): orphanage(orphanage) {
auto& statementEnd = arena.copy(p::oneOf(
transform(p::sequence(p::exactChar<';'>(), docComment),
[&](kj::Array<kj::String>&& comment) -> Orphan<Statement> {
[this](kj::Maybe<kj::Array<kj::String>>&& comment) -> Orphan<Statement> {
auto result = orphanage.newOrphan<Statement>();
auto builder = result.get();
attachDocComment(builder, kj::mv(comment));
KJ_IF_MAYBE(c, comment) {
attachDocComment(builder, kj::mv(*c));
}
builder.getBlock().setNone();
return result;
}),
transform(
p::sequence(p::exactChar<'{'>(), docComment, statementSequence, p::exactChar<'}'>()),
[&](kj::Array<kj::String>&& comment, kj::Array<Orphan<Statement>>&& statements)
p::sequence(p::exactChar<'{'>(), docComment, statementSequence, p::exactChar<'}'>(),
docComment),
[this](kj::Maybe<kj::Array<kj::String>>&& comment,
kj::Array<Orphan<Statement>>&& statements,
kj::Maybe<kj::Array<kj::String>>&& lateComment)
-> Orphan<Statement> {
auto result = orphanage.newOrphan<Statement>();
auto builder = result.get();
attachDocComment(builder, kj::mv(comment));
KJ_IF_MAYBE(c, comment) {
attachDocComment(builder, kj::mv(*c));
} else KJ_IF_MAYBE(c, lateComment) {
attachDocComment(builder, kj::mv(*c));
}
auto list = builder.getBlock().initStatements(statements.size());
for (uint i = 0; i < statements.size(); i++) {
list[i].adoptStatement(kj::mv(statements[i]));
list.adoptWithCaveats(i, kj::mv(statements[i]));
}
return result;
})
));
auto& statement = arena.copy(p::transform(p::sequence(tokenSequence, statementEnd),
[&](kj::Array<Orphan<Token>>&& tokens, Orphan<Statement>&& statement) {
auto tokensBuilder = statement.get().initTokens(tokens.size());
auto& statement = arena.copy(p::transformWithLocation(p::sequence(tokenSequence, statementEnd),
[this](Location loc, kj::Array<Orphan<Token>>&& tokens, Orphan<Statement>&& statement) {
auto builder = statement.get();
auto tokensBuilder = builder.initTokens(tokens.size());
for (uint i = 0; i < tokens.size(); i++) {
tokensBuilder[i].adoptToken(kj::mv(tokens[i]));
tokensBuilder.adoptWithCaveats(i, kj::mv(tokens[i]));
}
builder.setStartByte(loc.begin());
builder.setEndByte(loc.end());
return kj::mv(statement);
}));
......
c++/src/capnp/compiler/lexer.capnp
View file @ 0757e6ef
# Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@0xa73956d2621fc3ee;
using Cxx = import "/capnp/c++.capnp";
......@@ -11,48 +34,35 @@ struct Token {
integerLiteral @3 :UInt64;
floatLiteral @4 :Float64;
operator @5 :Text;
parenthesizedList @6 :List(List(TokenPointer));
bracketedList @7 :List(List(TokenPointer));
parenthesizedList @6 :List(List(Token));
bracketedList @7 :List(List(Token));
}
startByte @8 :UInt32;
endByte @9 :UInt32;
}
struct TokenPointer {
# Hack to deal with the fact that struct lists cannot adopt elements.
#
# TODO(cleanup): Find a better approach.
token @0 :Token;
}
struct Statement {
tokens @0 :List(TokenPointer);
tokens @0 :List(Token);
block @1 union {
none @2 :Void;
statements @3 :List(StatementPointer);
statements @3 :List(Statement);
}
docComment @4 :Text;
}
struct StatementPointer {
# Hack to deal with the fact that struct lists cannot adopt elements.
#
# TODO(cleanup): Find a better approach.
statement @0 :Statement;
startByte @5 :UInt32;
endByte @6 :UInt32;
}
struct LexedTokens {
# Lexer output when asked to parse tokens that don't form statements.
tokens @0 :List(TokenPointer);
tokens @0 :List(Token);
}
struct LexedStatements {
# Lexer output when asked to parse statements.
statements @0 :List(StatementPointer);
statements @0 :List(Statement);
}
c++/src/capnp/layout.c++
View file @ 0757e6ef
......@@ -698,6 +698,10 @@ struct WireHelpers {
// mistakenly thinks the source location still owns the object. transferPointer() doesn't do
// this zeroing itself because many callers transfer several pointers in a loop then zero out
// the whole section.
KJ_DASSERT(dst->isNull());
// We expect the caller to ensure the target is already null so won't leak.
if (src->isNull()) {
memset(dst, 0, sizeof(WirePointer));
} else if (src->kind() == WirePointer::FAR) {
......@@ -2034,6 +2038,45 @@ OrphanBuilder StructBuilder::disown(WirePointerCount ptrIndex) {
return WireHelpers::disown(segment, pointers + ptrIndex);
}
void StructBuilder::transferContentFrom(StructBuilder other) {
// Determine the amount of data the builders have in common.
BitCount sharedDataSize = kj::min(dataSize, other.dataSize);
if (dataSize > sharedDataSize) {
// Since the target is larger than the source, make sure to zero out the extra bits that the
// source doesn't have.
if (dataSize == 1 * BITS) {
setDataField<bool>(0 * ELEMENTS, false);
} else {
byte* unshared = reinterpret_cast<byte*>(data) + sharedDataSize / BITS_PER_BYTE / BYTES;
memset(unshared, 0, (dataSize - sharedDataSize) / BITS_PER_BYTE / BYTES);
}
}
// Copy over the shared part.
if (sharedDataSize == 1 * BITS) {
setDataField<bool>(0 * ELEMENTS, other.getDataField<bool>(0 * ELEMENTS));
} else {
memcpy(data, other.data, sharedDataSize / BITS_PER_BYTE / BYTES);
}
// Zero out all pointers in the target.
for (uint i = 0; i < pointerCount / POINTERS; i++) {
WireHelpers::zeroObject(segment, pointers + i);
}
// Transfer the pointers.
WirePointerCount sharedPointerCount = kj::min(pointerCount, other.pointerCount);
for (uint i = 0; i < sharedPointerCount / POINTERS; i++) {
WireHelpers::transferPointer(segment, pointers + i, other.segment, other.pointers + i);
}
// Zero out the pointers that were transferred in the source because it no longer has ownership.
// If the source had any extra pointers that the destination didn't have space for, we
// intentionally leave them be, so that they'll be cleaned up later.
memset(other.pointers, 0, sharedPointerCount * BYTES_PER_POINTER / BYTES);
}
bool StructBuilder::isPointerFieldNull(WirePointerCount ptrIndex) {
return (pointers + ptrIndex)->isNull();
}
......
c++/src/capnp/layout.h
View file @ 0757e6ef
......@@ -364,6 +364,11 @@ public:
// Detach the given pointer field from this object. The pointer becomes null, and the child
// object is returned as an orphan.
void transferContentFrom(StructBuilder other);
// Adopt all pointers from `other`, and also copy all data. If `other`'s sections are larger
// than this, the extra data is not transferred, meaning there is a risk of data loss when
// transferring from messages built with future versions of the protocol.
bool isPointerFieldNull(WirePointerCount ptrIndex);
StructReader asReader() const;
......
c++/src/capnp/list.h
View file @ 0757e6ef
......@@ -249,6 +249,24 @@ struct List<T, Kind::STRUCT> {
return typename T::Builder(builder.getStructElement(index * ELEMENTS));
}
inline void adoptWithCaveats(uint index, Orphan<T>&& orphan) {
// Mostly behaves like you'd expect `adopt` to behave, but with two caveats originating from
// the fact that structs in a struct list are allocated inline rather than by pointer:
// * This actually performs a shallow copy, effectively adopting each of the orphan's
// children rather than adopting the orphan itself. The orphan ends up being discarded,
// possibly wasting space in the message object.
// * If the orphan is larger than the target struct -- say, because the orphan was built
// using a newer version of the schema that has additional fields -- it will be truncated,
// losing data.
// We pass a zero-valued StructSize to asStruct() because we do not want the struct to be
// expanded under any circumstances. We're just going to throw it away anyway, and
// transferContentFrom() already carefully compares the struct sizes before transferring.
builder.getStructElement(index * ELEMENTS).transferContentFrom(
orphan.builder.asStruct(_::StructSize(
0 * WORDS, 0 * POINTERS, _::FieldSize::VOID)));
}
// There are no init(), set(), adopt(), or disown() methods for lists of structs because the
// elements of the list are inlined and are initialized when the list is initialized. This
// means that init() would be redundant, and set() would risk data loss if the input struct
......
c++/src/kj/parse/common-test.c++
View file @ 0757e6ef
......@@ -344,6 +344,37 @@ TEST(CommonParsers, TransformParser) {
}
}
TEST(CommonParsers, TransformOrRejectParser) {
auto parser = transformOrReject(many(any),
[](Array<char> chars) -> Maybe<int> {
if (heapString(chars) == "foo") {
return 123;
} else {
return nullptr;
}
});
{
StringPtr text = "foo";
Input input(text.begin(), text.end());
Maybe<int> result = parser(input);
KJ_IF_MAYBE(i, result) {
EXPECT_EQ(123, *i);
} else {
ADD_FAILURE() << "Expected 123, got null.";
}
EXPECT_TRUE(input.atEnd());
}
{
StringPtr text = "bar";
Input input(text.begin(), text.end());
Maybe<int> result = parser(input);
EXPECT_TRUE(result == nullptr);
EXPECT_TRUE(input.atEnd());
}
}
TEST(CommonParsers, References) {
struct TransformFunc {
int value;
......
c++/src/kj/parse/common.h
View file @ 0757e6ef
......@@ -71,11 +71,11 @@ public:
}
bool atEnd() { return pos == end; }
const Element& current() {
auto current() -> decltype(*instance<Iterator>()) {
KJ_IREQUIRE(!atEnd());
return *pos;
}
const Element& consume() {
auto consume() -> decltype(*instance<Iterator>()) {
KJ_IREQUIRE(!atEnd());
return *pos++;
}
......@@ -274,7 +274,6 @@ private:
template <typename SubParser, typename Result>
constexpr ConstResult_<SubParser, Result> constResult(SubParser&& subParser, Result&& result) {
// Constructs a parser which returns exactly `result` if `subParser` is successful.
return ConstResult_<SubParser, Result>(kj::fwd<SubParser>(subParser), kj::fwd<Result>(result));
}
......@@ -571,6 +570,27 @@ private:
TransformFunc transform;
};
template <typename SubParser, typename TransformFunc>
class TransformOrReject_ {
public:
explicit constexpr TransformOrReject_(SubParser&& subParser, TransformFunc&& transform)
: subParser(kj::fwd<SubParser>(subParser)), transform(kj::fwd<TransformFunc>(transform)) {}
template <typename Input>
decltype(kj::apply(instance<TransformFunc&>(), instance<OutputType<SubParser, Input>&&>()))
operator()(Input& input) const {
KJ_IF_MAYBE(subResult, subParser(input)) {
return kj::apply(transform, kj::mv(*subResult));
} else {
return nullptr;
}
}
private:
SubParser subParser;
TransformFunc transform;
};
template <typename SubParser, typename TransformFunc>
class TransformWithLocation_ {
public:
......@@ -606,12 +626,21 @@ constexpr Transform_<SubParser, TransformFunc> transform(
kj::fwd<SubParser>(subParser), kj::fwd<TransformFunc>(functor));
}
template <typename SubParser, typename TransformFunc>
constexpr TransformOrReject_<SubParser, TransformFunc> transformOrReject(
SubParser&& subParser, TransformFunc&& functor) {
// Like `transform()` except that `functor` returns a `Maybe`. If it returns null, parsing fails,
// otherwise the parser's result is the content of the `Maybe`.
return TransformOrReject_<SubParser, TransformFunc>(
kj::fwd<SubParser>(subParser), kj::fwd<TransformFunc>(functor));
}
template <typename SubParser, typename TransformFunc>
constexpr TransformWithLocation_<SubParser, TransformFunc> transformWithLocation(
SubParser&& subParser, TransformFunc&& functor) {
// Constructs a parser which executes some other parser and then transforms the result by invoking
// `functor` on it. Typically `functor` is a lambda. It is invoked using `kj::apply`,
// meaning tuples will be unpacked as arguments.
// Like `transform` except that `functor` also takes a `Span` as its first parameter specifying
// the location of the parsed content. The span's position type is whatever the parser input's
// getPosition() returns.
return TransformWithLocation_<SubParser, TransformFunc>(
kj::fwd<SubParser>(subParser), kj::fwd<TransformFunc>(functor));
}
......@@ -650,6 +679,8 @@ constexpr AcceptIf_<SubParser, Condition> acceptIf(SubParser&& subParser, Condit
// `condition` on the result to check if it is valid. Typically, `condition` is a lambda
// returning true or false. Like with `transform()`, `condition` is invoked using `kj::apply`
// to unpack tuples.
//
// TODO(soon): Remove in favor of transformOrReject()?
return AcceptIf_<SubParser, Condition>(
kj::fwd<SubParser>(subParser), kj::fwd<Condition>(condition));
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment