Discarding version of char parsers. Also, many() should return a count rather…

Discarding version of char parsers.  Also, many() should return a count rather than an array if sub-parsers have empty results.

c++/src/kj/parse/char-test.c++

@@ -96,8 +96,8 @@ TEST(CharParsers, CharRange) {
   }
 }
 
-TEST(CharParsers, AnyChar) {
-  constexpr auto parser = anyChar("axn2B");
+TEST(CharParsers, AnyOfChars) {
+  constexpr auto parser = anyOfChars("axn2B");
 
   {
     StringPtr text = "a";
@@ -177,6 +177,38 @@ TEST(CharParsers, CharGroupCombo) {
   }
 }
 
+TEST(CharParsers, DiscardCharRange) {
+  constexpr auto parser = many(discardCharRange('a', 'z'));
+
+  {
+    StringPtr text = "foo-bar";
+    Input input(text.begin(), text.end());
+    Maybe<int> result = parser(input);
+    KJ_IF_MAYBE(value, result) {
+      EXPECT_EQ(3, *value);
+    } else {
+      ADD_FAILURE() << "Expected 3, got null.";
+    }
+    EXPECT_FALSE(input.atEnd());
+  }
+}
+
+TEST(CharParsers, DiscardAnyOfChars) {
+  constexpr auto parser = many(discardAnyOfChars("abcd"));
+
+  {
+    StringPtr text = "cadbfoo";
+    Input input(text.begin(), text.end());
+    Maybe<int> result = parser(input);
+    KJ_IF_MAYBE(value, result) {
+      EXPECT_EQ(4, *value);
+    } else {
+      ADD_FAILURE() << "Expected 4, got null.";
+    }
+    EXPECT_FALSE(input.atEnd());
+  }
+}
+
 TEST(CommonParsers, ExactChar) {
   constexpr auto parser = exactChar<'a'>();
 

c++/src/kj/parse/char.c++

+// Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "char.h"
+#include "../string.h"
+#include <gtest/gtest.h>
+
+namespace kj {
+namespace parse {
+
+}  // namespace parse
+}  // namespace kj

c++/src/kj/parse/char.h

@@ -31,6 +31,7 @@ namespace parse {
 
 // =======================================================================================
 
+template <typename ReturnType>
 class CharGroup_ {
 public:
   constexpr CharGroup_(): bits{0, 0, 0, 0} {}
@@ -58,15 +59,7 @@ public:
   }
 
   template <typename Input>
-  Maybe<char> operator()(Input& input) const {
-    unsigned char c = input.current();
-    if ((bits[c / 64] & (1ll << (c % 64))) != 0) {
-      input.next();
-      return c;
-    } else {
-      return nullptr;
-    }
-  }
+  Maybe<ReturnType> operator()(Input& input) const;
 
 private:
   typedef unsigned long long Bits64;
@@ -82,7 +75,31 @@ private:
   }
 };
 
-constexpr CharGroup_ charRange(char first, char last) {
+template <>
+template <typename Input>
+Maybe<char> CharGroup_<char>::operator()(Input& input) const {
+  unsigned char c = input.current();
+  if ((bits[c / 64] & (1ll << (c % 64))) != 0) {
+    input.next();
+    return c;
+  } else {
+    return nullptr;
+  }
+}
+
+template <>
+template <typename Input>
+Maybe<Tuple<>> CharGroup_<Tuple<>>::operator()(Input& input) const {
+  unsigned char c = input.current();
+  if ((bits[c / 64] & (1ll << (c % 64))) != 0) {
+    input.next();
+    return tuple();
+  } else {
+    return nullptr;
+  }
+}
+
+constexpr CharGroup_<char> charRange(char first, char last) {
   // Create a parser which accepts any character in the range from `first` to `last`, inclusive.
   // For example: `charRange('a', 'z')` matches all lower-case letters.  The parser's result is the
   // character matched.
@@ -94,15 +111,27 @@ constexpr CharGroup_ charRange(char first, char last) {
   //
   // You can also use `.invert()` to match the opposite set of characters.
 
-  return CharGroup_().orRange(first, last);
+  return CharGroup_<char>().orRange(first, last);
 }
 
-constexpr CharGroup_ anyChar(const char* chars) {
+constexpr CharGroup_<char> anyOfChars(const char* chars) {
   // Returns a parser that accepts any of the characters in the given string (which should usually
   // be a literal).  The returned parser is of the same type as returned by `charRange()` -- see
   // that function for more info.
 
-  return CharGroup_().orAny(chars);
+  return CharGroup_<char>().orAny(chars);
+}
+
+constexpr CharGroup_<Tuple<>> discardCharRange(char first, char last) {
+  // Like `charRange()` except that the parser returns an empty tuple.
+
+  return CharGroup_<Tuple<>>().orRange(first, last);
+}
+
+constexpr CharGroup_<Tuple<>> discardAnyOfChars(const char* chars) {
+  // Like `anyChar()` except that the parser returns an empty tuple.
+
+  return CharGroup_<Tuple<>>().orAny(chars);
 }
 
 template <char c>

c++/src/kj/parse/common-test.c++

@@ -32,6 +32,39 @@ namespace {
 typedef IteratorInput<char, const char*> Input;
 typedef Span<const char*> TestLocation;
 
+TEST(CommonParsers, AnyParser) {
+  StringPtr text = "foo";
+  Input input(text.begin(), text.end());
+
+  Maybe<char> result = any()(input);
+  KJ_IF_MAYBE(c, result) {
+    EXPECT_EQ('f', *c);
+  } else {
+    ADD_FAILURE() << "Expected 'c', got null.";
+  }
+  EXPECT_FALSE(input.atEnd());
+
+  result = any()(input);
+  KJ_IF_MAYBE(c, result) {
+    EXPECT_EQ('o', *c);
+  } else {
+    ADD_FAILURE() << "Expected 'o', got null.";
+  }
+  EXPECT_FALSE(input.atEnd());
+
+  result = any()(input);
+  KJ_IF_MAYBE(c, result) {
+    EXPECT_EQ('o', *c);
+  } else {
+    ADD_FAILURE() << "Expected 'o', got null.";
+  }
+  EXPECT_TRUE(input.atEnd());
+
+  result = any()(input);
+  EXPECT_TRUE(result == nullptr);
+  EXPECT_TRUE(input.atEnd());
+}
+
 TEST(CommonParsers, ExactElementParser) {
   StringPtr text = "foo";
   Input input(text.begin(), text.end());
@@ -145,12 +178,10 @@ TEST(CommonParsers, SequenceParser) {
   }
 }
 
-TEST(CommonParsers, ManyParser) {
+TEST(CommonParsers, ManyParserCountOnly) {
   StringPtr text = "foooob";
 
-  auto parser = transform(
-      sequence(exactly('f'), many(exactly('o'))),
-      [](TestLocation, ArrayPtr<Tuple<>> values) -> int { return values.size(); });
+  auto parser = sequence(exactly('f'), many(exactly('o')));
 
   {
     Input input(text.begin(), text.begin() + 3);
@@ -186,6 +217,23 @@ TEST(CommonParsers, ManyParser) {
   }
 }
 
+TEST(CommonParsers, ManyParserSubResult) {
+  StringPtr text = "foooob";
+
+  auto parser = many(any());
+
+  {
+    Input input(text.begin(), text.end());
+    Maybe<Array<char>> result = parser(input);
+    KJ_IF_MAYBE(chars, result) {
+      EXPECT_EQ(text, heapString(*chars));
+    } else {
+      ADD_FAILURE() << "Expected char array, got null.";
+    }
+    EXPECT_TRUE(input.atEnd());
+  }
+}
+
 TEST(CommonParsers, OptionalParser) {
   auto parser = sequence(
       transform(exactly('b'), [](TestLocation) -> uint { return 123; }),

c++/src/kj/parse/common.h

@@ -150,6 +150,29 @@ constexpr ParserRef<Input, OutputType<ParserImpl, Input>> ref(ParserImpl& impl)
   return ParserRef<Input, OutputType<ParserImpl, Input>>(impl);
 }
 
+// -------------------------------------------------------------------
+// any()
+// Output = one token
+
+class Any_ {
+public:
+  template <typename Input>
+  Maybe<Decay<decltype(instance<Input>().current())>> operator()(Input& input) const {
+    if (input.atEnd()) {
+      return nullptr;
+    } else {
+      auto result = input.current();
+      input.next();
+      return result;
+    }
+  }
+};
+
+constexpr Any_ any() {
+  // Constructs a parser which matches any token and simply returns it.
+  return Any_();
+}
+
 // -------------------------------------------------------------------
 // exactly()
 // Output = Tuple<>
@@ -305,16 +328,28 @@ constexpr Sequence_<SubParsers...> sequence(SubParsers&&... subParsers) {
 
 // -------------------------------------------------------------------
 // many()
-// Output = Array of output of sub-parser.
+// Output = Array of output of sub-parser, or just a uint count if the sub-parser returns Tuple<>.
 
 template <typename SubParser, bool atLeastOne>
 class Many_ {
+  template <typename Input, typename Output = OutputType<SubParser, Input>>
+  struct Impl;
 public:
   explicit constexpr Many_(SubParser&& subParser)
       : subParser(kj::mv(subParser)) {}
 
   template <typename Input>
-  Maybe<Array<OutputType<SubParser, Input>>> operator()(Input& input) const {
+  auto operator()(Input& input) const
+      -> decltype(Impl<Input>::apply(instance<const SubParser&>(), input));
+
+private:
+  SubParser subParser;
+};
+
+template <typename SubParser, bool atLeastOne>
+template <typename Input, typename Output>
+struct Many_<SubParser, atLeastOne>::Impl {
+  static Maybe<Array<Output>> apply(const SubParser& subParser, Input& input) {
     typedef Vector<OutputType<SubParser, Input>> Results;
     Results results;
 
@@ -335,15 +370,44 @@ public:
 
     return results.releaseAsArray();
   }
+};
 
-private:
-  SubParser subParser;
+template <typename SubParser, bool atLeastOne>
+template <typename Input>
+struct Many_<SubParser, atLeastOne>::Impl<Input, Tuple<>> {
+  static Maybe<uint> apply(const SubParser& subParser, Input& input) {
+    uint count = 0;
+
+    while (!input.atEnd()) {
+      Input subInput(input);
+
+      KJ_IF_MAYBE(subResult, subParser(subInput)) {
+        subInput.advanceParent();
+        ++count;
+      } else {
+        break;
+      }
+    }
+
+    if (atLeastOne && count == 0) {
+      return nullptr;
+    }
+
+    return count;
+  }
 };
 
+template <typename SubParser, bool atLeastOne>
+template <typename Input>
+auto Many_<SubParser, atLeastOne>::operator()(Input& input) const
+    -> decltype(Impl<Input>::apply(instance<const SubParser&>(), input)) {
+  return Impl<Input, OutputType<SubParser, Input>>::apply(subParser, input);
+}
+
 template <typename SubParser>
 constexpr Many_<SubParser, false> many(SubParser&& subParser) {
   // Constructs a parser that repeatedly executes the given parser until it fails, returning an
-  // Array of the results.
+  // Array of the results (or a uint count if `subParser` returns an empty tuple).
   return Many_<SubParser, false>(kj::fwd<SubParser>(subParser));
 }