Skip to content

C
capnproto
Commit 567fd38e authored by Kenton Varda's avatar Kenton Varda
Browse files
Options

Add char parsers for various token types.

parent 0bd5f9ce
Hide whitespace changes
Inline Side-by-side
Showing with 580 additions and 106 deletions
c++/src/kj/parse/char-test.c++
View file @ 567fd38e
......@@ -177,53 +177,221 @@ TEST(CharParsers, CharGroupCombo) {
}
}
TEST(CharParsers, DiscardCharRange) {
constexpr auto parser = many(discardCharRange('a', 'z'));
TEST(CharParsers, ExactChar) {
constexpr auto parser = exactChar<'a'>();
{
StringPtr text = "a";
Input input(text.begin(), text.end());
EXPECT_TRUE(parser(input) != nullptr);
EXPECT_TRUE(input.atEnd());
}
{
StringPtr text = "foo-bar";
StringPtr text = "b";
Input input(text.begin(), text.end());
Maybe<int> result = parser(input);
EXPECT_TRUE(parser(input) == nullptr);
EXPECT_FALSE(input.atEnd());
}
}
TEST(CharParsers, Identifier) {
constexpr auto parser = identifier;
{
StringPtr text = "helloWorld123 ";
Input input(text.begin(), text.end());
Maybe<String> result = parser(input);
KJ_IF_MAYBE(value, result) {
EXPECT_EQ(3, *value);
EXPECT_EQ("helloWorld123", *value);
} else {
ADD_FAILURE() << "Expected 3, got null.";
ADD_FAILURE() << "Expected string, got null.";
}
EXPECT_FALSE(input.atEnd());
}
}
TEST(CharParsers, DiscardAnyOfChars) {
constexpr auto parser = many(discardAnyOfChars("abcd"));
TEST(CharParsers, Integer) {
constexpr auto parser = integer;
{
StringPtr text = "cadbfoo";
StringPtr text = "12349";
Input input(text.begin(), text.end());
Maybe<int> result = parser(input);
Maybe<uint64_t> result = parser(input);
KJ_IF_MAYBE(value, result) {
EXPECT_EQ(4, *value);
EXPECT_EQ(12349, *value);
} else {
ADD_FAILURE() << "Expected 4, got null.";
ADD_FAILURE() << "Expected integer, got null.";
}
EXPECT_FALSE(input.atEnd());
EXPECT_TRUE(input.atEnd());
}
{
StringPtr text = "0x1aF0";
Input input(text.begin(), text.end());
Maybe<uint64_t> result = parser(input);
KJ_IF_MAYBE(value, result) {
EXPECT_EQ(0x1aF0, *value);
} else {
ADD_FAILURE() << "Expected integer, got null.";
}
EXPECT_TRUE(input.atEnd());
}
{
StringPtr text = "064270";
Input input(text.begin(), text.end());
Maybe<uint64_t> result = parser(input);
KJ_IF_MAYBE(value, result) {
EXPECT_EQ(064270, *value);
} else {
ADD_FAILURE() << "Expected integer, got null.";
}
EXPECT_TRUE(input.atEnd());
}
}
TEST(CommonParsers, ExactChar) {
constexpr auto parser = exactChar<'a'>();
TEST(CharParsers, Number) {
constexpr auto parser = number;
{
StringPtr text = "a";
StringPtr text = "12345";
Input input(text.begin(), text.end());
EXPECT_TRUE(parser(input) != nullptr);
Maybe<double> result = parser(input);
KJ_IF_MAYBE(value, result) {
EXPECT_EQ(12345, *value);
} else {
ADD_FAILURE() << "Expected number, got null.";
}
EXPECT_TRUE(input.atEnd());
}
{
StringPtr text = "b";
StringPtr text = "123.25";
Input input(text.begin(), text.end());
EXPECT_TRUE(parser(input) == nullptr);
EXPECT_FALSE(input.atEnd());
Maybe<double> result = parser(input);
KJ_IF_MAYBE(value, result) {
EXPECT_EQ(123.25, *value);
} else {
ADD_FAILURE() << "Expected number, got null.";
}
EXPECT_TRUE(input.atEnd());
}
{
StringPtr text = "123e10";
Input input(text.begin(), text.end());
Maybe<double> result = parser(input);
KJ_IF_MAYBE(value, result) {
EXPECT_EQ(123e10, *value);
} else {
ADD_FAILURE() << "Expected number, got null.";
}
EXPECT_TRUE(input.atEnd());
}
{
StringPtr text = "123.25E+10";
Input input(text.begin(), text.end());
Maybe<double> result = parser(input);
KJ_IF_MAYBE(value, result) {
EXPECT_EQ(123.25E+10, *value);
} else {
ADD_FAILURE() << "Expected number, got null.";
}
EXPECT_TRUE(input.atEnd());
}
{
StringPtr text = "25e-2";
Input input(text.begin(), text.end());
Maybe<double> result = parser(input);
KJ_IF_MAYBE(value, result) {
EXPECT_EQ(25e-2, *value);
} else {
ADD_FAILURE() << "Expected number, got null.";
}
EXPECT_TRUE(input.atEnd());
}
}
TEST(CharParsers, DoubleQuotedString) {
constexpr auto parser = doubleQuotedString;
{
StringPtr text = "\"hello\"";
Input input(text.begin(), text.end());
Maybe<String> result = parser(input);
KJ_IF_MAYBE(value, result) {
EXPECT_EQ("hello", *value);
} else {
ADD_FAILURE() << "Expected \"hello\", got null.";
}
EXPECT_TRUE(input.atEnd());
}
{
StringPtr text = "\"test\\a\\b\\f\\n\\r\\t\\v\\\'\\\"\\\?\x01\2\34\156\"";
Input input(text.begin(), text.end());
Maybe<String> result = parser(input);
KJ_IF_MAYBE(value, result) {
EXPECT_EQ("test\a\b\f\n\r\t\v\'\"\?\x01\2\34\156", *value);
} else {
ADD_FAILURE() << "Expected string, got null.";
}
EXPECT_TRUE(input.atEnd());
}
{
StringPtr text = "\"foo'bar\"";
Input input(text.begin(), text.end());
Maybe<String> result = parser(input);
KJ_IF_MAYBE(value, result) {
EXPECT_EQ("foo'bar", *value);
} else {
ADD_FAILURE() << "Expected string, got null.";
}
EXPECT_TRUE(input.atEnd());
}
}
TEST(CharParsers, SingleQuotedString) {
constexpr auto parser = singleQuotedString;
{
StringPtr text = "\'hello\'";
Input input(text.begin(), text.end());
Maybe<String> result = parser(input);
KJ_IF_MAYBE(value, result) {
EXPECT_EQ("hello", *value);
} else {
ADD_FAILURE() << "Expected \"hello\", got null.";
}
EXPECT_TRUE(input.atEnd());
}
{
StringPtr text = "\'test\\a\\b\\f\\n\\r\\t\\v\\\'\\\"\\\?\x01\2\34\156\'";
Input input(text.begin(), text.end());
Maybe<String> result = parser(input);
KJ_IF_MAYBE(value, result) {
EXPECT_EQ("test\a\b\f\n\r\t\v\'\"\?\x01\2\34\156", *value);
} else {
ADD_FAILURE() << "Expected string, got null.";
}
EXPECT_TRUE(input.atEnd());
}
{
StringPtr text = "\'foo\"bar\'";
Input input(text.begin(), text.end());
Maybe<String> result = parser(input);
KJ_IF_MAYBE(value, result) {
EXPECT_EQ("foo\"bar", *value);
} else {
ADD_FAILURE() << "Expected string, got null.";
}
EXPECT_TRUE(input.atEnd());
}
}
......
c++/src/kj/parse/char.c++
View file @ 567fd38e
......@@ -22,11 +22,51 @@
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "char.h"
#include "../string.h"
#include <gtest/gtest.h>
#include "../debug.h"
#include <stdlib.h>
namespace kj {
namespace parse {
namespace _ { // private
double ParseFloat::operator()(const Array<char>& digits,
const Maybe<Array<char>>& fraction,
const Maybe<Tuple<Maybe<char>, Array<char>>>& exponent) const {
size_t bufSize = digits.size();
KJ_IF_MAYBE(f, fraction) {
bufSize += 1 + f->size();
}
KJ_IF_MAYBE(e, exponent) {
bufSize += 1 + (get<0>(*e) != nullptr) + get<1>(*e).size();
}
KJ_STACK_ARRAY(char, buf, bufSize + 1, 128, 128);
char* pos = buf.begin();
memcpy(pos, digits.begin(), digits.size());
pos += digits.size();
KJ_IF_MAYBE(f, fraction) {
*pos++ = '.';
memcpy(pos, f->begin(), f->size());
pos += f->size();
}
KJ_IF_MAYBE(e, exponent) {
*pos++ = 'e';
KJ_IF_MAYBE(sign, get<0>(*e)) {
*pos++ = *sign;
}
memcpy(pos, get<1>(*e).begin(), get<1>(*e).size());
pos += get<1>(*e).size();
}
*pos++ = '\0';
KJ_DASSERT(pos == buf.end());
return strtod(buf.begin(), nullptr);
}
} // namespace _ (private)
} // namespace parse
} // namespace kj
c++/src/kj/parse/char.h
View file @ 567fd38e
......@@ -25,13 +25,14 @@
#define KJ_PARSE_CHAR_H_
#include "common.h"
#include "../string.h"
#include <inttypes.h>
namespace kj {
namespace parse {
// =======================================================================================
template <typename ReturnType>
class CharGroup_ {
public:
constexpr CharGroup_(): bits{0, 0, 0, 0} {}
......@@ -54,12 +55,28 @@ public:
bits[3] | bit(c - 256));
}
constexpr CharGroup_ orGroup(CharGroup_ other) const {
return CharGroup_(bits[0] | other.bits[0],
bits[1] | other.bits[1],
bits[2] | other.bits[2],
bits[3] | other.bits[3]);
}
constexpr CharGroup_ invert() const {
return CharGroup_(~bits[0], ~bits[1], ~bits[2], ~bits[3]);
}
template <typename Input>
Maybe<ReturnType> operator()(Input& input) const;
Maybe<char> operator()(Input& input) const {
if (input.atEnd()) return nullptr;
unsigned char c = input.current();
if ((bits[c / 64] & (1ll << (c % 64))) != 0) {
input.next();
return c;
} else {
return nullptr;
}
}
private:
typedef unsigned long long Bits64;
......@@ -75,31 +92,7 @@ private:
}
};
template <>
template <typename Input>
Maybe<char> CharGroup_<char>::operator()(Input& input) const {
unsigned char c = input.current();
if ((bits[c / 64] & (1ll << (c % 64))) != 0) {
input.next();
return c;
} else {
return nullptr;
}
}
template <>
template <typename Input>
Maybe<Tuple<>> CharGroup_<Tuple<>>::operator()(Input& input) const {
unsigned char c = input.current();
if ((bits[c / 64] & (1ll << (c % 64))) != 0) {
input.next();
return tuple();
} else {
return nullptr;
}
}
constexpr CharGroup_<char> charRange(char first, char last) {
constexpr CharGroup_ charRange(char first, char last) {
// Create a parser which accepts any character in the range from `first` to `last`, inclusive.
// For example: `charRange('a', 'z')` matches all lower-case letters. The parser's result is the
// character matched.
......@@ -111,27 +104,15 @@ constexpr CharGroup_<char> charRange(char first, char last) {
//
// You can also use `.invert()` to match the opposite set of characters.
return CharGroup_<char>().orRange(first, last);
return CharGroup_().orRange(first, last);
}
constexpr CharGroup_<char> anyOfChars(const char* chars) {
constexpr CharGroup_ anyOfChars(const char* chars) {
// Returns a parser that accepts any of the characters in the given string (which should usually
// be a literal). The returned parser is of the same type as returned by `charRange()` -- see
// that function for more info.
return CharGroup_<char>().orAny(chars);
}
constexpr CharGroup_<Tuple<>> discardCharRange(char first, char last) {
// Like `charRange()` except that the parser returns an empty tuple.
return CharGroup_<Tuple<>>().orRange(first, last);
}
constexpr CharGroup_<Tuple<>> discardAnyOfChars(const char* chars) {
// Like `anyChar()` except that the parser returns an empty tuple.
return CharGroup_<Tuple<>>().orAny(chars);
return CharGroup_().orAny(chars);
}
template <char c>
......@@ -141,6 +122,177 @@ constexpr ExactlyConst_<char, c> exactChar() {
return ExactlyConst_<char, c>();
}
// =======================================================================================
namespace _ { // private
struct ArrayToString {
inline String operator()(const Array<char>& arr) const {
return heapString(arr);
}
};
} // namespace _ (private)
template <typename SubParser>
constexpr auto charsToString(SubParser&& subParser)
-> decltype(transform(kj::fwd<SubParser>(subParser), _::ArrayToString())) {
// Wraps a parser that returns Array<char> such that it returns String instead.
return parse::transform(kj::fwd<SubParser>(subParser), _::ArrayToString());
}
// =======================================================================================
// Basic character classes.
constexpr auto alpha = charRange('a', 'z').orRange('A', 'Z');
constexpr auto digit = charRange('0', '9');
constexpr auto alphaNumeric = alpha.orGroup(digit);
constexpr auto nameStart = alpha.orChar('_');
constexpr auto nameChar = alphaNumeric.orChar('_');
constexpr auto hexDigit = charRange('0', '9').orRange('a', 'f').orRange('A', 'F');
constexpr auto octDigit = charRange('0', '7');
constexpr auto whitespace = many(anyOfChars(" \f\n\r\t\v"));
constexpr auto discardWhitespace = discard(many(discard(anyOfChars(" \f\n\r\t\v"))));
// Like discard(whitespace) but avoids some memory allocation.
// =======================================================================================
// Identifiers
namespace _ { // private
struct IdentifierToString {
inline String operator()(char first, const Array<char>& rest) const {
String result = heapString(rest.size() + 1);
result[0] = first;
memcpy(result.begin() + 1, rest.begin(), rest.size());
return result;
}
};
} // namespace _ (private)
constexpr auto identifier = transform(sequence(nameStart, many(nameChar)), _::IdentifierToString());
// Parses an identifier (e.g. a C variable name).
// =======================================================================================
// Integers
namespace _ { // private
inline char parseDigit(char c) {
if (c < 'A') return c - '0';
if (c < 'a') return c - 'A' + 10;
return c - 'a' + 10;
}
template <uint base>
struct ParseInteger {
inline uint64_t operator()(const Array<char>& digits) const {
return operator()('0', digits);
}
uint64_t operator()(char first, const Array<char>& digits) const {
uint64_t result = parseDigit(first);
for (char digit: digits) {
result = result * base + parseDigit(digit);
}
return result;
}
};
} // namespace _ (private)
constexpr auto integer = sequence(
oneOf(
transform(sequence(exactChar<'0'>(), exactChar<'x'>(), many(hexDigit)), _::ParseInteger<16>()),
transform(sequence(exactChar<'0'>(), many(octDigit)), _::ParseInteger<8>()),
transform(sequence(charRange('1', '9'), many(digit)), _::ParseInteger<10>())),
notLookingAt(alpha.orAny("_.")));
// =======================================================================================
// Numbers (i.e. floats)
namespace _ { // private
struct ParseFloat {
double operator()(const Array<char>& digits,
const Maybe<Array<char>>& fraction,
const Maybe<Tuple<Maybe<char>, Array<char>>>& exponent) const;
};
} // namespace _ (private)
constexpr auto number = transform(
sequence(
many(digit),
optional(sequence(exactChar<'.'>(), many(digit))),
optional(sequence(discard(anyOfChars("eE")), optional(anyOfChars("+-")), many(digit))),
notLookingAt(alpha.orAny("_."))),
_::ParseFloat());
// =======================================================================================
// Quoted strings
namespace _ { // private
struct InterpretEscape {
char operator()(char c) const {
switch (c) {
case 'a': return '\a';
case 'b': return '\b';
case 'f': return '\f';
case 'n': return '\n';
case 'r': return '\r';
case 't': return '\t';
case 'v': return '\v';
default: return c;
}
}
};
struct ParseHexEscape {
inline char operator()(char first, char second) const {
return (parseDigit(first) << 4) | second;
}
};
struct ParseOctEscape {
inline char operator()(char first, Maybe<char> second, Maybe<char> third) const {
char result = first - '0';
KJ_IF_MAYBE(digit1, second) {
result = (result << 3) | (*digit1 - '0');
KJ_IF_MAYBE(digit2, third) {
result = (result << 3) | (*digit2 - '0');
}
}
return result;
}
};
} // namespace _ (private)
constexpr auto escapeSequence =
sequence(exactChar<'\\'>(), oneOf(
transform(anyOfChars("abfnrtv'\"\\\?"), _::InterpretEscape()),
transform(sequence(exactChar<'x'>(), hexDigit, hexDigit), _::ParseHexEscape()),
transform(sequence(octDigit, optional(octDigit), optional(octDigit)),
_::ParseOctEscape())));
// A parser that parses a C-string-style escape sequence (starting with a backslash). Returns
// a char.
constexpr auto doubleQuotedString = charsToString(sequence(
exactChar<'\"'>(),
many(oneOf(anyOfChars("\\\n\"").invert(), escapeSequence)),
exactChar<'\"'>()));
// Parses a C-style double-quoted string.
constexpr auto singleQuotedString = charsToString(sequence(
exactChar<'\''>(),
many(oneOf(anyOfChars("\\\n\'").invert(), escapeSequence)),
exactChar<'\''>()));
// Parses a C-style single-quoted string.
} // namespace parse
} // namespace kj
......
c++/src/kj/parse/common-test.c++
View file @ 567fd38e
......@@ -30,13 +30,13 @@ namespace parse {
namespace {
typedef IteratorInput<char, const char*> Input;
typedef Span<const char*> TestLocation;
TEST(CommonParsers, AnyParser) {
StringPtr text = "foo";
Input input(text.begin(), text.end());
constexpr auto parser = any;
Maybe<char> result = any()(input);
Maybe<char> result = parser(input);
KJ_IF_MAYBE(c, result) {
EXPECT_EQ('f', *c);
} else {
......@@ -44,7 +44,7 @@ TEST(CommonParsers, AnyParser) {
}
EXPECT_FALSE(input.atEnd());
result = any()(input);
result = parser(input);
KJ_IF_MAYBE(c, result) {
EXPECT_EQ('o', *c);
} else {
......@@ -52,7 +52,7 @@ TEST(CommonParsers, AnyParser) {
}
EXPECT_FALSE(input.atEnd());
result = any()(input);
result = parser(input);
KJ_IF_MAYBE(c, result) {
EXPECT_EQ('o', *c);
} else {
......@@ -60,7 +60,7 @@ TEST(CommonParsers, AnyParser) {
}
EXPECT_TRUE(input.atEnd());
result = any()(input);
result = parser(input);
EXPECT_TRUE(result == nullptr);
EXPECT_TRUE(input.atEnd());
}
......@@ -125,6 +125,16 @@ TEST(CommonParsers, ConstResultParser) {
EXPECT_TRUE(input.atEnd());
}
TEST(CommonParsers, DiscardParser) {
auto parser = discard(any);
StringPtr text = "o";
Input input(text.begin(), text.end());
Maybe<Tuple<>> result = parser(input);
EXPECT_TRUE(result != nullptr);
EXPECT_TRUE(input.atEnd());
}
TEST(CommonParsers, SequenceParser) {
StringPtr text = "foo";
......@@ -167,7 +177,7 @@ TEST(CommonParsers, SequenceParser) {
{
Input input(text.begin(), text.end());
Maybe<int> result = sequence(transform(exactly('f'), [](TestLocation){return 123;}),
Maybe<int> result = sequence(transform(exactly('f'), [](){return 123;}),
exactly('o'), exactly('o'))(input);
KJ_IF_MAYBE(i, result) {
EXPECT_EQ(123, *i);
......@@ -220,7 +230,7 @@ TEST(CommonParsers, ManyParserCountOnly) {
TEST(CommonParsers, ManyParserSubResult) {
StringPtr text = "foooob";
auto parser = many(any());
auto parser = many(any);
{
Input input(text.begin(), text.end());
......@@ -236,9 +246,9 @@ TEST(CommonParsers, ManyParserSubResult) {
TEST(CommonParsers, OptionalParser) {
auto parser = sequence(
transform(exactly('b'), [](TestLocation) -> uint { return 123; }),
optional(transform(exactly('a'), [](TestLocation) -> uint { return 456; })),
transform(exactly('r'), [](TestLocation) -> uint { return 789; }));
transform(exactly('b'), []() -> uint { return 123; }),
optional(transform(exactly('a'), []() -> uint { return 456; })),
transform(exactly('r'), []() -> uint { return 789; }));
{
StringPtr text = "bar";
......@@ -283,9 +293,9 @@ TEST(CommonParsers, OptionalParser) {
TEST(CommonParsers, OneOfParser) {
auto parser = oneOf(
transform(sequence(exactly('f'), exactly('o'), exactly('o')),
[](TestLocation) -> StringPtr { return "foo"; }),
[]() -> StringPtr { return "foo"; }),
transform(sequence(exactly('b'), exactly('a'), exactly('r')),
[](TestLocation) -> StringPtr { return "bar"; }));
[]() -> StringPtr { return "bar"; }));
{
StringPtr text = "foo";
......@@ -315,9 +325,9 @@ TEST(CommonParsers, OneOfParser) {
TEST(CommonParsers, TransformParser) {
StringPtr text = "foo";
auto parser = transform(
auto parser = transformWithLocation(
sequence(exactly('f'), exactly('o'), exactly('o')),
[](TestLocation location) -> int {
[](Span<const char*> location) -> int {
EXPECT_EQ("foo", StringPtr(location.begin(), location.end()));
return 123;
});
......@@ -340,7 +350,7 @@ TEST(CommonParsers, References) {
TransformFunc(int value): value(value) {}
int operator()(TestLocation) const { return value; }
int operator()() const { return value; }
};
// Don't use auto for the parsers here in order to verify that the templates are properly choosing
......@@ -360,7 +370,7 @@ TEST(CommonParsers, References) {
StringPtr text = "foob";
auto parser = transform(
sequence(parser1, parser2, exactly('o'), parser3),
[](TestLocation, int i, int j, int k) { return i + j + k; });
[](int i, int j, int k) { return i + j + k; });
{
Input input(text.begin(), text.end());
......@@ -376,9 +386,9 @@ TEST(CommonParsers, References) {
TEST(CommonParsers, AcceptIfParser) {
auto parser = acceptIf(
oneOf(transform(exactly('a'), [](TestLocation) -> uint { return 123; }),
transform(exactly('b'), [](TestLocation) -> uint { return 456; }),
transform(exactly('c'), [](TestLocation) -> uint { return 789; })),
oneOf(transform(exactly('a'), []() -> uint { return 123; }),
transform(exactly('b'), []() -> uint { return 456; }),
transform(exactly('c'), []() -> uint { return 789; })),
[](uint i) {return i > 200;});
{
......@@ -413,6 +423,37 @@ TEST(CommonParsers, AcceptIfParser) {
}
}
TEST(CommonParsers, NotLookingAt) {
auto parser = notLookingAt(exactly('a'));
{
StringPtr text = "a";
Input input(text.begin(), text.end());
EXPECT_TRUE(parser(input) == nullptr);
EXPECT_FALSE(input.atEnd());
}
{
StringPtr text = "b";
Input input(text.begin(), text.end());
EXPECT_TRUE(parser(input) != nullptr);
EXPECT_FALSE(input.atEnd());
}
}
TEST(CommonParsers, EndOfInput) {
auto parser = endOfInput;
{
StringPtr text = "a";
Input input(text.begin(), text.end());
EXPECT_TRUE(parser(input) == nullptr);
EXPECT_TRUE(parser(input) == nullptr);
input.next();
EXPECT_FALSE(parser(input) == nullptr);
}
}
} // namespace
} // namespace parse
} // namespace kj
c++/src/kj/parse/common.h
View file @ 567fd38e
......@@ -66,6 +66,9 @@ public:
void advanceParent() {
parent->pos = pos;
}
void forgetParent() {
parent = nullptr;
}
bool atEnd() { return pos == end; }
const Element& current() {
......@@ -151,27 +154,23 @@ constexpr ParserRef<Input, OutputType<ParserImpl, Input>> ref(ParserImpl& impl)
}
// -------------------------------------------------------------------
// any()
// any
// Output = one token
class Any_ {
public:
template <typename Input>
Maybe<Decay<decltype(instance<Input>().current())>> operator()(Input& input) const {
Maybe<Decay<decltype(instance<Input>().consume())>> operator()(Input& input) const {
if (input.atEnd()) {
return nullptr;
} else {
auto result = input.current();
input.next();
return result;
return input.consume();
}
}
};
constexpr Any_ any() {
// Constructs a parser which matches any token and simply returns it.
return Any_();
}
constexpr Any_ any = Any_();
// A parser which matches any token and simply returns it.
// -------------------------------------------------------------------
// exactly()
......@@ -263,6 +262,12 @@ constexpr ConstResult_<SubParser, Result> constResult(SubParser&& subParser, Res
return ConstResult_<SubParser, Result>(kj::fwd<SubParser>(subParser), kj::fwd<Result>(result));
}
template <typename SubParser>
constexpr ConstResult_<SubParser, Tuple<>> discard(SubParser&& subParser) {
// Constructs a parser which wraps `subParser` but discards the result.
return constResult(kj::fwd<SubParser>(subParser), Tuple<>());
}
// -------------------------------------------------------------------
// sequence()
// Output = Flattened Tuple of outputs of sub-parsers.
......@@ -534,6 +539,28 @@ public:
explicit constexpr Transform_(SubParser&& subParser, TransformFunc&& transform)
: subParser(kj::fwd<SubParser>(subParser)), transform(kj::fwd<TransformFunc>(transform)) {}
template <typename Input>
Maybe<decltype(kj::apply(instance<TransformFunc&>(),
instance<OutputType<SubParser, Input>&&>()))>
operator()(Input& input) const {
KJ_IF_MAYBE(subResult, subParser(input)) {
return kj::apply(transform, kj::mv(*subResult));
} else {
return nullptr;
}
}
private:
SubParser subParser;
TransformFunc transform;
};
template <typename SubParser, typename TransformFunc>
class TransformWithLocation_ {
public:
explicit constexpr TransformWithLocation_(SubParser&& subParser, TransformFunc&& transform)
: subParser(kj::fwd<SubParser>(subParser)), transform(kj::fwd<TransformFunc>(transform)) {}
template <typename Input>
Maybe<decltype(kj::apply(instance<TransformFunc&>(),
instance<Span<Decay<decltype(instance<Input&>().getPosition())>>>(),
......@@ -563,6 +590,16 @@ constexpr Transform_<SubParser, TransformFunc> transform(
kj::fwd<SubParser>(subParser), kj::fwd<TransformFunc>(functor));
}
template <typename SubParser, typename TransformFunc>
constexpr TransformWithLocation_<SubParser, TransformFunc> transformWithLocation(
SubParser&& subParser, TransformFunc&& functor) {
// Constructs a parser which executes some other parser and then transforms the result by invoking
// `functor` on it. Typically `functor` is a lambda. It is invoked using `kj::apply`,
// meaning tuples will be unpacked as arguments.
return TransformWithLocation_<SubParser, TransformFunc>(
kj::fwd<SubParser>(subParser), kj::fwd<TransformFunc>(functor));
}
// -------------------------------------------------------------------
// acceptIf()
// Output = Same as SubParser
......@@ -601,6 +638,37 @@ constexpr AcceptIf_<SubParser, Condition> acceptIf(SubParser&& subParser, Condit
kj::fwd<SubParser>(subParser), kj::fwd<Condition>(condition));
}
// -------------------------------------------------------------------
// notLookingAt()
// Fails if the given parser succeeds at the current location.
template <typename SubParser>
class NotLookingAt_ {
public:
explicit constexpr NotLookingAt_(SubParser&& subParser): subParser(kj::mv(subParser)) {}
template <typename Input>
Maybe<Tuple<>> operator()(Input& input) const {
Input subInput(input);
subInput.forgetParent();
if (subParser(subInput) == nullptr) {
return Tuple<>();
} else {
return nullptr;
}
}
private:
SubParser subParser;
};
template <typename SubParser>
constexpr NotLookingAt_<SubParser> notLookingAt(SubParser&& subParser) {
// Constructs a parser which fails at any position where the given parser succeeds. Otherwise,
// it succeeds without consuming any input and returns an empty tuple.
return NotLookingAt_<SubParser>(kj::fwd<SubParser>(subParser));
}
// -------------------------------------------------------------------
// endOfInput()
// Output = Tuple<>, only succeeds if at end-of-input
......@@ -617,10 +685,8 @@ public:
}
};
constexpr EndOfInput_ endOfInput() {
// Constructs a parser that succeeds only if it is called with no input.
return EndOfInput_();
}
constexpr EndOfInput_ endOfInput = EndOfInput_();
// A parser that succeeds only if it is called with no input.
} // namespace parse
} // namespace kj
......
c++/src/kj/string.c++
View file @ 567fd38e
......@@ -39,7 +39,8 @@ String heapString(size_t size) {
String heapString(const char* value, size_t size) {
char* buffer = _::HeapArrayDisposer::allocate<char>(size + 1);
memcpy(buffer, value, size + 1);
memcpy(buffer, value, size);
buffer[size] = '\0';
return String(buffer, size, _::HeapArrayDisposer::instance);
}
......
c++/src/kj/tuple.h
View file @ 567fd38e
......@@ -93,8 +93,8 @@ struct TupleElement {
T value;
TupleElement() = default;
inline TupleElement(const T& value): value(value) {}
inline TupleElement(T&& value): value(kj::mv(value)) {}
constexpr inline TupleElement(const T& value): value(value) {}
constexpr inline TupleElement(T&& value): value(kj::mv(value)) {}
};
template <uint index, typename T>
......@@ -134,13 +134,13 @@ struct TupleImpl<Indexes<indexes...>, Types...>
}
template <typename... U>
inline TupleImpl(Tuple<U...>&& other)
constexpr inline TupleImpl(Tuple<U...>&& other)
: TupleElement<indexes, Types>(kj::mv(getImpl<indexes>(other)))... {}
template <typename... U>
inline TupleImpl(Tuple<U...>& other)
constexpr inline TupleImpl(Tuple<U...>& other)
: TupleElement<indexes, Types>(getImpl<indexes>(other))... {}
template <typename... U>
inline TupleImpl(const Tuple<U...>& other)
constexpr inline TupleImpl(const Tuple<U...>& other)
: TupleElement<indexes, Types>(getImpl<indexes>(other))... {}
};
......@@ -153,15 +153,15 @@ class Tuple {
public:
Tuple() = default;
template <typename... U>
Tuple(Tuple<U...>&& other): impl(kj::mv(other)) {}
constexpr inline Tuple(Tuple<U...>&& other): impl(kj::mv(other)) {}
template <typename... U>
Tuple(Tuple<U...>& other): impl(other) {}
constexpr inline Tuple(Tuple<U...>& other): impl(other) {}
template <typename... U>
Tuple(const Tuple<U...>& other): impl(other) {}
constexpr inline Tuple(const Tuple<U...>& other): impl(other) {}
private:
template <typename... Params>
Tuple(Params&&... params): impl(kj::fwd<Params>(params)...) {}
constexpr Tuple(Params&&... params): impl(kj::fwd<Params>(params)...) {}
TupleImpl<MakeIndexes<sizeof...(T)>, T...> impl;
......@@ -174,6 +174,12 @@ private:
friend struct MakeTupleFunc;
};
template <>
class Tuple<> {
// Simplified zero-member version of Tuple. In particular this is important to make sure that
// Tuple<>() is constexpr.
};
template <size_t index, typename... T>
inline TypeByIndex<index, T...>& getImpl(Tuple<T...>& tuple) {
// Get member of a Tuple by index, e.g. `get<2>(myTuple)`.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment