conformance_test.h 11.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc.  All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


// This file defines a protocol for running the conformance test suite
// in-process.  In other words, the suite itself will run in the same process as
// the code under test.
//
// For pros and cons of this approach, please see conformance.proto.

#ifndef CONFORMANCE_CONFORMANCE_TEST_H
#define CONFORMANCE_CONFORMANCE_TEST_H

41
#include <functional>
42
#include <string>
43 44
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/util/type_resolver.h>
45 46
#include <google/protobuf/wire_format_lite.h>

47
#include "third_party/jsoncpp/json.h"
48

49 50 51 52 53
namespace conformance {
class ConformanceRequest;
class ConformanceResponse;
}  // namespace conformance

54 55
namespace protobuf_test_messages {
namespace proto3 {
56
class TestAllTypesProto3;
57 58 59
}  // namespace proto3
}  // namespace protobuf_test_messages

60 61 62 63 64
namespace google {
namespace protobuf {

class ConformanceTestRunner {
 public:
65 66
  virtual ~ConformanceTestRunner() {}

67 68 69 70 71 72 73
  // Call to run a single conformance test.
  //
  // "input" is a serialized conformance.ConformanceRequest.
  // "output" should be set to a serialized conformance.ConformanceResponse.
  //
  // If there is any error in running the test itself, set "runtime_error" in
  // the response.
74 75 76
  virtual void RunTest(const std::string& test_name,
                       const std::string& input,
                       std::string* output) = 0;
77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
};

// Class representing the test suite itself.  To run it, implement your own
// class derived from ConformanceTestRunner and then write code like:
//
//    class MyConformanceTestRunner : public ConformanceTestRunner {
//     public:
//      virtual void RunTest(...) {
//        // INSERT YOUR FRAMEWORK-SPECIFIC CODE HERE.
//      }
//    };
//
//    int main() {
//      MyConformanceTestRunner runner;
//      google::protobuf::ConformanceTestSuite suite;
//
//      std::string output;
//      suite.RunSuite(&runner, &output);
//    }
//
class ConformanceTestSuite {
 public:
Bo Yang's avatar
Bo Yang committed
99
  ConformanceTestSuite() : verbose_(false), enforce_recommended_(false) {}
100

101 102
  void SetVerbose(bool verbose) { verbose_ = verbose; }

103 104 105
  // Sets the list of tests that are expected to fail when RunSuite() is called.
  // RunSuite() will fail unless the set of failing tests is exactly the same
  // as this list.
106 107 108 109 110
  //
  // The filename here is *only* used to create/format useful error messages for
  // how to update the failure list.  We do NOT read this file at all.
  void SetFailureList(const std::string& filename,
                      const std::vector<std::string>& failure_list);
111

Bo Yang's avatar
Bo Yang committed
112 113 114 115 116 117 118 119 120 121 122 123
  // Whether to require the testee to pass RECOMMENDED tests. By default failing
  // a RECOMMENDED test case will not fail the entire suite but will only
  // generated a warning. If this flag is set to true, RECOMMENDED tests will
  // be treated the same way as REQUIRED tests and failing a RECOMMENDED test
  // case will cause the entire test suite to fail as well. An implementation
  // can enable this if it wants to be strictly conforming to protobuf spec.
  // See the comments about ConformanceLevel below to learn more about the
  // difference between REQUIRED and RECOMMENDED test cases.
  void SetEnforceRecommended(bool value) {
    enforce_recommended_ = value;
  }

124 125
  // Run all the conformance tests against the given test runner.
  // Test output will be stored in "output".
126 127 128 129 130
  //
  // Returns true if the set of failing tests was exactly the same as the
  // failure list.  If SetFailureList() was not called, returns true if all
  // tests passed.
  bool RunSuite(ConformanceTestRunner* runner, std::string* output);
131 132

 private:
Bo Yang's avatar
Bo Yang committed
133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150
  // Test cases are classified into a few categories:
  //   REQUIRED: the test case must be passed for an implementation to be
  //             interoperable with other implementations. For example, a
  //             parser implementaiton must accept both packed and unpacked
  //             form of repeated primitive fields.
  //   RECOMMENDED: the test case is not required for the implementation to
  //                be interoperable with other implementations, but is
  //                recommended for best performance and compatibility. For
  //                example, a proto3 serializer should serialize repeated
  //                primitive fields in packed form, but an implementation
  //                failing to do so will still be able to communicate with
  //                other implementations.
  enum ConformanceLevel {
    REQUIRED = 0,
    RECOMMENDED = 1,
  };
  string ConformanceLevelToString(ConformanceLevel level);

151
  void ReportSuccess(const std::string& test_name);
152
  void ReportFailure(const string& test_name,
Bo Yang's avatar
Bo Yang committed
153
                     ConformanceLevel level,
154 155 156 157 158 159
                     const conformance::ConformanceRequest& request,
                     const conformance::ConformanceResponse& response,
                     const char* fmt, ...);
  void ReportSkip(const string& test_name,
                  const conformance::ConformanceRequest& request,
                  const conformance::ConformanceResponse& response);
160 161
  void RunTest(const std::string& test_name,
               const conformance::ConformanceRequest& request,
162
               conformance::ConformanceResponse* response);
Bo Yang's avatar
Bo Yang committed
163 164 165
  void RunValidInputTest(const string& test_name,
                         ConformanceLevel level,
                         const string& input,
166 167
                         conformance::WireFormat input_format,
                         const string& equivalent_text_format,
Yilun Chong's avatar
Yilun Chong committed
168 169
                         conformance::WireFormat requested_output,
                         bool isProto3);
170 171 172 173 174 175 176
  void RunValidBinaryInputTest(const string& test_name,
                               ConformanceLevel level,
                               const string& input,
                               conformance::WireFormat input_format,
                               const string& equivalent_wire_format,
                               conformance::WireFormat requested_output,
                               bool isProto3);
Bo Yang's avatar
Bo Yang committed
177 178 179
  void RunValidJsonTest(const string& test_name,
                        ConformanceLevel level,
                        const string& input_json,
180
                        const string& equivalent_text_format);
181 182 183
  void RunValidJsonTestWithProtobufInput(
      const string& test_name,
      ConformanceLevel level,
184
      const protobuf_test_messages::proto3::TestAllTypesProto3& input,
185
      const string& equivalent_text_format);
186 187
  void RunValidProtobufTest(const string& test_name, ConformanceLevel level,
                            const string& input_protobuf,
Yilun Chong's avatar
Yilun Chong committed
188 189
                            const string& equivalent_text_format,
                            bool isProto3);
190 191 192 193
  void RunValidBinaryProtobufTest(const string& test_name,
                                  ConformanceLevel level,
                                  const string& input_protobuf,
                                  bool isProto3);
194 195
  void RunValidProtobufTestWithMessage(
      const string& test_name, ConformanceLevel level,
196
      const Message *input,
Yilun Chong's avatar
Yilun Chong committed
197 198
      const string& equivalent_text_format,
      bool isProto3);
199 200 201

  typedef std::function<bool(const Json::Value&)> Validator;
  void RunValidJsonTestWithValidator(const string& test_name,
Bo Yang's avatar
Bo Yang committed
202
                                     ConformanceLevel level,
203 204 205
                                     const string& input_json,
                                     const Validator& validator);
  void ExpectParseFailureForJson(const string& test_name,
Bo Yang's avatar
Bo Yang committed
206
                                 ConformanceLevel level,
207 208
                                 const string& input_json);
  void ExpectSerializeFailureForJson(const string& test_name,
Bo Yang's avatar
Bo Yang committed
209
                                     ConformanceLevel level,
210
                                     const string& text_format);
211 212 213 214
  void ExpectParseFailureForProtoWithProtoVersion (const string& proto,
                                                   const string& test_name,
                                                   ConformanceLevel level,
                                                   bool isProto3);
215
  void ExpectParseFailureForProto(const std::string& proto,
Bo Yang's avatar
Bo Yang committed
216 217
                                  const std::string& test_name,
                                  ConformanceLevel level);
218
  void ExpectHardParseFailureForProto(const std::string& proto,
Bo Yang's avatar
Bo Yang committed
219 220
                                      const std::string& test_name,
                                      ConformanceLevel level);
221
  void TestPrematureEOFForType(google::protobuf::FieldDescriptor::Type type);
222
  void TestIllegalTags();
223 224 225
  template <class MessageType>
  void TestOneofMessage (MessageType &message,
                         bool isProto3);
226 227 228
  template <class MessageType>
  void TestUnknownMessage (MessageType &message,
                           bool isProto3);
229 230 231
  void TestValidDataForType(
      google::protobuf::FieldDescriptor::Type,
      std::vector<std::pair<std::string, std::string>> values);
232
  bool CheckSetEmpty(const std::set<string>& set_to_check,
233
                     const std::string& write_to_file, const std::string& msg);
234 235
  ConformanceTestRunner* runner_;
  int successes_;
236
  int expected_failures_;
237
  bool verbose_;
Bo Yang's avatar
Bo Yang committed
238
  bool enforce_recommended_;
239
  std::string output_;
240
  std::string failure_list_filename_;
241 242 243 244 245 246 247 248 249 250 251 252 253 254

  // The set of test names that are expected to fail in this run, but haven't
  // failed yet.
  std::set<std::string> expected_to_fail_;

  // The set of test names that have been run.  Used to ensure that there are no
  // duplicate names in the suite.
  std::set<std::string> test_names_;

  // The set of tests that failed, but weren't expected to.
  std::set<std::string> unexpected_failing_tests_;

  // The set of tests that succeeded, but weren't expected to.
  std::set<std::string> unexpected_succeeding_tests_;
255 256 257 258 259 260 261

  // The set of tests that the testee opted out of;
  std::set<std::string> skipped_;

  google::protobuf::internal::scoped_ptr<google::protobuf::util::TypeResolver>
      type_resolver_;
  std::string type_url_;
262 263 264 265 266 267
};

}  // namespace protobuf
}  // namespace google

#endif  // CONFORMANCE_CONFORMANCE_TEST_H