Allow conformance test runner to tolerate crashes, and re-enable conformance tests.

conformance/Makefile.am

 ## Process this file with automake to produce Makefile.in
 
 protoc_inputs =                                                \
-  conformance.proto
+  conformance.proto                                            \
+  $(top_srcdir)/src/google/protobuf/any.proto                  \
+  $(top_srcdir)/src/google/protobuf/duration.proto             \
+  $(top_srcdir)/src/google/protobuf/field_mask.proto           \
+  $(top_srcdir)/src/google/protobuf/struct.proto               \
+  $(top_srcdir)/src/google/protobuf/timestamp.proto            \
+  $(top_srcdir)/src/google/protobuf/wrappers.proto
+
 
 protoc_outputs =                                               \
   conformance.pb.cc                                            \
@@ -75,7 +82,7 @@ endif
 if USE_EXTERNAL_PROTOC
 
 protoc_middleman: $(protoc_inputs)
-	$(PROTOC) -I$(srcdir) --cpp_out=. --java_out=. --ruby_out=. --objc_out=. --python_out=.$^
+	$(PROTOC) -I$(srcdir) -I$(top_srcdir) --cpp_out=. --java_out=. --ruby_out=. --objc_out=. --python_out=.$^
 	touch protoc_middleman
 
 else
@@ -135,10 +142,10 @@ test_ruby: protoc_middleman conformance-test-runner $(other_language_protoc_outp
 # These depend on library paths being properly set up.  The easiest way to
 # run them is to just use "tox" from the python dir.
 test_python: protoc_middleman conformance-test-runner
-	./conformance-test-runner --failure_list failure_list_python.txt ./conformance_python.py
+	./conformance-test-runner --failure_list failure_list_python.txt $(CONFORMANCE_PYTHON_EXTRA_FAILURES) ./conformance_python.py
 
 test_python_cpp: protoc_middleman conformance-test-runner
-	./conformance-test-runner --failure_list failure_list_python_cpp.txt ./conformance_python.py
+	./conformance-test-runner --failure_list failure_list_python_cpp.txt $(CONFORMANCE_PYTHON_EXTRA_FAILURES) ./conformance_python.py
 
 if OBJC_CONFORMANCE_TEST
 

conformance/conformance_test.cc

@@ -299,7 +299,13 @@ void ConformanceTestSuite::RunValidInputTest(
         return;
       }
 
-      GOOGLE_CHECK(test_message.ParseFromString(binary_protobuf));
+      if (!test_message.ParseFromString(binary_protobuf)) {
+        ReportFailure(test_name, request, response,
+                      "INTERNAL ERROR: internal JSON->protobuf transcode "
+                      "yielded unparseable proto.");
+        return;
+      }
+
       break;
     }
 

conformance/conformance_test_runner.cc

@@ -55,16 +55,21 @@
 
 #include <algorithm>
 #include <errno.h>
-#include <unistd.h>
 #include <fstream>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
 #include <vector>
 
+#include <google/protobuf/stubs/stringprintf.h>
+
 #include "conformance.pb.h"
 #include "conformance_test.h"
 
 using conformance::ConformanceRequest;
 using conformance::ConformanceResponse;
 using google::protobuf::internal::scoped_array;
+using google::protobuf::StringAppendF;
 using std::string;
 using std::vector;
 
@@ -81,14 +86,14 @@ using std::vector;
 class ForkPipeRunner : public google::protobuf::ConformanceTestRunner {
  public:
   ForkPipeRunner(const std::string &executable)
-      : running_(false), executable_(executable) {}
+      : child_pid_(-1), executable_(executable) {}
 
   virtual ~ForkPipeRunner() {}
 
   void RunTest(const std::string& test_name,
                const std::string& request,
                std::string* response) {
-    if (!running_) {
+    if (child_pid_ < 0) {
       SpawnTestProgram();
     }
 
@@ -97,7 +102,31 @@ class ForkPipeRunner : public google::protobuf::ConformanceTestRunner {
     uint32_t len = request.size();
     CheckedWrite(write_fd_, &len, sizeof(uint32_t));
     CheckedWrite(write_fd_, request.c_str(), request.size());
-    CheckedRead(read_fd_, &len, sizeof(uint32_t));
+
+    if (!TryRead(read_fd_, &len, sizeof(uint32_t))) {
+      // We failed to read from the child, assume a crash and try to reap.
+      GOOGLE_LOG(INFO) << "Trying to reap child, pid=" << child_pid_;
+
+      int status;
+      waitpid(child_pid_, &status, WEXITED);
+
+      string error_msg;
+      if (WIFEXITED(status)) {
+        StringAppendF(&error_msg,
+                      "child exited, status=%d", WEXITSTATUS(status));
+      } else if (WIFSIGNALED(status)) {
+        StringAppendF(&error_msg,
+                      "child killed by signal %d", WTERMSIG(status));
+      }
+      GOOGLE_LOG(INFO) << error_msg;
+      child_pid_ = -1;
+
+      conformance::ConformanceResponse response_obj;
+      response_obj.set_runtime_error(error_msg);
+      response_obj.SerializeToString(response);
+      return;
+    }
+
     response->resize(len);
     CheckedRead(read_fd_, (void*)response->c_str(), len);
   }
@@ -141,7 +170,7 @@ class ForkPipeRunner : public google::protobuf::ConformanceTestRunner {
       CHECK_SYSCALL(close(fromproc_pipe_fd[1]));
       write_fd_ = toproc_pipe_fd[1];
       read_fd_ = fromproc_pipe_fd[0];
-      running_ = true;
+      child_pid_ = pid;
     } else {
       // Child.
       CHECK_SYSCALL(close(STDIN_FILENO));
@@ -171,28 +200,40 @@ class ForkPipeRunner : public google::protobuf::ConformanceTestRunner {
     }
   }
 
-  void CheckedRead(int fd, void *buf, size_t len) {
+  bool TryRead(int fd, void *buf, size_t len) {
     size_t ofs = 0;
     while (len > 0) {
       ssize_t bytes_read = read(fd, (char*)buf + ofs, len);
 
       if (bytes_read == 0) {
-        GOOGLE_LOG(FATAL) << current_test_name_
+        GOOGLE_LOG(ERROR) << current_test_name_
                           << ": unexpected EOF from test program";
+        return false;
       } else if (bytes_read < 0) {
-        GOOGLE_LOG(FATAL) << current_test_name_
+        GOOGLE_LOG(ERROR) << current_test_name_
                           << ": error reading from test program: "
                           << strerror(errno);
+        return false;
       }
 
       len -= bytes_read;
       ofs += bytes_read;
     }
+
+    return true;
+  }
+
+  void CheckedRead(int fd, void *buf, size_t len) {
+    if (!TryRead(fd, buf, len)) {
+      GOOGLE_LOG(FATAL) << current_test_name_
+                        << ": error reading from test program: "
+                        << strerror(errno);
+    }
   }
 
   int write_fd_;
   int read_fd_;
-  bool running_;
+  pid_t child_pid_;
   std::string executable_;
   std::string current_test_name_;
 };
@@ -239,12 +280,12 @@ int main(int argc, char *argv[]) {
   char *program;
   google::protobuf::ConformanceTestSuite suite;
 
+  vector<string> failure_list;
+
   for (int arg = 1; arg < argc; ++arg) {
     if (strcmp(argv[arg], "--failure_list") == 0) {
       if (++arg == argc) UsageError();
-      vector<string> failure_list;
       ParseFailureList(argv[arg], &failure_list);
-      suite.SetFailureList(failure_list);
     } else if (strcmp(argv[arg], "--verbose") == 0) {
       suite.SetVerbose(true);
     } else if (argv[arg][0] == '-') {
@@ -259,6 +300,7 @@ int main(int argc, char *argv[]) {
     }
   }
 
+  suite.SetFailureList(failure_list);
   ForkPipeRunner runner(program);
 
   std::string output;

conformance/failure_list_cpp.txt

@@ -36,9 +36,6 @@ JsonInput.FieldNameInLowerCamelCase.Validator
 JsonInput.FieldNameInSnakeCase.JsonOutput
 JsonInput.FieldNameInSnakeCase.ProtobufOutput
 JsonInput.FieldNameNotQuoted
-JsonInput.FieldNameWithMixedCases.JsonOutput
-JsonInput.FieldNameWithMixedCases.ProtobufOutput
-JsonInput.FieldNameWithMixedCases.Validator
 JsonInput.FloatFieldTooLarge
 JsonInput.FloatFieldTooSmall
 JsonInput.Int32FieldLeadingSpace

conformance/failure_list_java.txt

@@ -17,48 +17,27 @@ JsonInput.BoolFieldCamelCaseTrue
 JsonInput.BoolFieldDoubleQuotedFalse
 JsonInput.BoolFieldDoubleQuotedTrue
 JsonInput.BoolMapFieldKeyNotQuoted
-JsonInput.BytesFieldNoPadding
 JsonInput.DoubleFieldInfinityNotQuoted
 JsonInput.DoubleFieldNanNotQuoted
 JsonInput.DoubleFieldNegativeInfinityNotQuoted
 JsonInput.EnumFieldNotQuoted
-JsonInput.EnumFieldNumericValueNonZero.JsonOutput
-JsonInput.EnumFieldNumericValueNonZero.ProtobufOutput
-JsonInput.EnumFieldNumericValueZero.JsonOutput
-JsonInput.EnumFieldNumericValueZero.ProtobufOutput
-JsonInput.EnumFieldUnknownValue.Validator
 JsonInput.FieldMask.ProtobufOutput
 JsonInput.FieldMaskInvalidCharacter
 JsonInput.FieldNameDuplicate
-JsonInput.FieldNameDuplicateDifferentCasing1
-JsonInput.FieldNameDuplicateDifferentCasing2
 JsonInput.FieldNameInSnakeCase.JsonOutput
 JsonInput.FieldNameNotQuoted
-JsonInput.FieldNameWithMixedCases.JsonOutput
 JsonInput.FloatFieldInfinityNotQuoted
 JsonInput.FloatFieldNanNotQuoted
 JsonInput.FloatFieldNegativeInfinityNotQuoted
-JsonInput.Int32FieldExponentialFormat.JsonOutput
-JsonInput.Int32FieldExponentialFormat.ProtobufOutput
-JsonInput.Int32FieldFloatTrailingZero.JsonOutput
-JsonInput.Int32FieldFloatTrailingZero.ProtobufOutput
 JsonInput.Int32FieldLeadingZero
-JsonInput.Int32FieldMaxFloatValue.JsonOutput
-JsonInput.Int32FieldMaxFloatValue.ProtobufOutput
 JsonInput.Int32FieldMinFloatValue.JsonOutput
-JsonInput.Int32FieldMinFloatValue.ProtobufOutput
 JsonInput.Int32FieldMinValue.JsonOutput
 JsonInput.Int32FieldNegativeWithLeadingZero
 JsonInput.Int32FieldPlusSign
 JsonInput.Int32MapFieldKeyNotQuoted
 JsonInput.Int64MapFieldKeyNotQuoted
 JsonInput.JsonWithComments
-JsonInput.MapFieldValueIsNull
-JsonInput.OneofFieldDuplicate
 JsonInput.OriginalProtoFieldName.JsonOutput
-JsonInput.RepeatedFieldMessageElementIsNull
-JsonInput.RepeatedFieldPrimitiveElementIsNull
-JsonInput.RepeatedFieldTrailingComma
 JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool
 JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt
 JsonInput.StringFieldNotAString
@@ -66,9 +45,5 @@ JsonInput.StringFieldSurrogateInWrongOrder
 JsonInput.StringFieldUnpairedHighSurrogate
 JsonInput.StringFieldUnpairedLowSurrogate
 JsonInput.StringFieldUppercaseEscapeLetter
-JsonInput.Uint32FieldMaxFloatValue.JsonOutput
-JsonInput.Uint32FieldMaxFloatValue.ProtobufOutput
 JsonInput.Uint32MapFieldKeyNotQuoted
 JsonInput.Uint64MapFieldKeyNotQuoted
-JsonInput.ValueAcceptNull.JsonOutput
-JsonInput.ValueAcceptNull.ProtobufOutput

conformance/failure_list_python.txt

+DurationProtoInputTooLarge.JsonOutput
+DurationProtoInputTooSmall.JsonOutput
+FieldMaskNumbersDontRoundTrip.JsonOutput
+FieldMaskPathsDontRoundTrip.JsonOutput
+FieldMaskTooManyUnderscore.JsonOutput
+JsonInput.AllFieldAcceptNull.JsonOutput
+JsonInput.Any.JsonOutput
+JsonInput.AnyNested.JsonOutput
+JsonInput.AnyNested.ProtobufOutput
+JsonInput.Any.ProtobufOutput
+JsonInput.AnyUnorderedTypeTag.JsonOutput
+JsonInput.AnyUnorderedTypeTag.ProtobufOutput
+JsonInput.AnyWithDuration.JsonOutput
+JsonInput.AnyWithDuration.ProtobufOutput
+JsonInput.AnyWithFieldMask.JsonOutput
+JsonInput.AnyWithFieldMask.ProtobufOutput
+JsonInput.AnyWithInt32ValueWrapper.JsonOutput
+JsonInput.AnyWithInt32ValueWrapper.ProtobufOutput
+JsonInput.AnyWithStruct.JsonOutput
+JsonInput.AnyWithStruct.ProtobufOutput
+JsonInput.AnyWithTimestamp.JsonOutput
+JsonInput.AnyWithTimestamp.ProtobufOutput
+JsonInput.AnyWithValueForInteger.JsonOutput
+JsonInput.AnyWithValueForInteger.ProtobufOutput
+JsonInput.AnyWithValueForJsonObject.JsonOutput
+JsonInput.AnyWithValueForJsonObject.ProtobufOutput
+JsonInput.BoolFieldFalse.JsonOutput
+JsonInput.BoolFieldTrue.JsonOutput
+JsonInput.BoolMapEscapedKey.JsonOutput
+JsonInput.BoolMapField.JsonOutput
+JsonInput.BytesFieldInvalidBase64Characters
+JsonInput.BytesField.JsonOutput
+JsonInput.BytesRepeatedField.JsonOutput
+JsonInput.DoubleFieldInfinity.JsonOutput
+JsonInput.DoubleFieldInfinityNotQuoted
+JsonInput.DoubleFieldMaxNegativeValue.JsonOutput
+JsonInput.DoubleFieldMaxPositiveValue.JsonOutput
+JsonInput.DoubleFieldMinNegativeValue.JsonOutput
+JsonInput.DoubleFieldMinPositiveValue.JsonOutput
+JsonInput.DoubleFieldNan.JsonOutput
+JsonInput.DoubleFieldNanNotQuoted
+JsonInput.DoubleFieldNegativeInfinity.JsonOutput
+JsonInput.DoubleFieldNegativeInfinityNotQuoted
+JsonInput.DoubleFieldQuotedValue.JsonOutput
+JsonInput.DoubleFieldTooSmall
+JsonInput.DurationJsonInputTooLarge
+JsonInput.DurationJsonInputTooSmall
+JsonInput.DurationMaxValue.JsonOutput
+JsonInput.DurationMinValue.JsonOutput
+JsonInput.DurationMissingS
+JsonInput.DurationRepeatedValue.JsonOutput
+JsonInput.EnumField.JsonOutput
+JsonInput.EnumFieldNumericValueNonZero.JsonOutput
+JsonInput.EnumFieldNumericValueNonZero.ProtobufOutput
+JsonInput.EnumFieldNumericValueZero.JsonOutput
+JsonInput.EnumFieldNumericValueZero.ProtobufOutput
+JsonInput.EnumFieldUnknownValue.Validator
+JsonInput.EnumRepeatedField.JsonOutput
+JsonInput.FieldMaskInvalidCharacter
+JsonInput.FieldMask.JsonOutput
+JsonInput.FieldMask.ProtobufOutput
+JsonInput.FieldNameEscaped.JsonOutput
+JsonInput.FieldNameInLowerCamelCase.Validator
+JsonInput.FieldNameInSnakeCase.JsonOutput
+JsonInput.FieldNameInSnakeCase.ProtobufOutput
+JsonInput.FieldNameWithMixedCases.JsonOutput
+JsonInput.FieldNameWithNumbers.JsonOutput
+JsonInput.FloatFieldInfinity.JsonOutput
+JsonInput.FloatFieldInfinityNotQuoted
+JsonInput.FloatFieldMaxNegativeValue.JsonOutput
+JsonInput.FloatFieldMaxPositiveValue.JsonOutput
+JsonInput.FloatFieldMinNegativeValue.JsonOutput
+JsonInput.FloatFieldMinPositiveValue.JsonOutput
+JsonInput.FloatFieldNan.JsonOutput
+JsonInput.FloatFieldNanNotQuoted
+JsonInput.FloatFieldNegativeInfinity.JsonOutput
+JsonInput.FloatFieldNegativeInfinityNotQuoted
+JsonInput.FloatFieldQuotedValue.JsonOutput
+JsonInput.FloatFieldTooLarge
+JsonInput.FloatFieldTooSmall
+JsonInput.HelloWorld.JsonOutput
+JsonInput.Int32FieldExponentialFormat.JsonOutput
+JsonInput.Int32FieldExponentialFormat.ProtobufOutput
+JsonInput.Int32FieldFloatTrailingZero.JsonOutput
+JsonInput.Int32FieldFloatTrailingZero.ProtobufOutput
+JsonInput.Int32FieldMaxFloatValue.JsonOutput
+JsonInput.Int32FieldMaxFloatValue.ProtobufOutput
+JsonInput.Int32FieldMaxValue.JsonOutput
+JsonInput.Int32FieldMinFloatValue.JsonOutput
+JsonInput.Int32FieldMinFloatValue.ProtobufOutput
+JsonInput.Int32FieldMinValue.JsonOutput
+JsonInput.Int32FieldStringValueEscaped.JsonOutput
+JsonInput.Int32FieldStringValue.JsonOutput
+JsonInput.Int32MapEscapedKey.JsonOutput
+JsonInput.Int32MapField.JsonOutput
+JsonInput.Int64FieldMaxValue.JsonOutput
+JsonInput.Int64FieldMaxValueNotQuoted.JsonOutput
+JsonInput.Int64FieldMinValue.JsonOutput
+JsonInput.Int64FieldMinValueNotQuoted.JsonOutput
+JsonInput.Int64MapEscapedKey.JsonOutput
+JsonInput.Int64MapField.JsonOutput
+JsonInput.MessageField.JsonOutput
+JsonInput.MessageMapField.JsonOutput
+JsonInput.MessageRepeatedField.JsonOutput
+JsonInput.OptionalBoolWrapper.JsonOutput
+JsonInput.OptionalBytesWrapper.JsonOutput
+JsonInput.OptionalDoubleWrapper.JsonOutput
+JsonInput.OptionalFloatWrapper.JsonOutput
+JsonInput.OptionalInt32Wrapper.JsonOutput
+JsonInput.OptionalInt64Wrapper.JsonOutput
+JsonInput.OptionalStringWrapper.JsonOutput
+JsonInput.OptionalUint32Wrapper.JsonOutput
+JsonInput.OptionalUint64Wrapper.JsonOutput
+JsonInput.OptionalWrapperTypesWithNonDefaultValue.JsonOutput
+JsonInput.OriginalProtoFieldName.JsonOutput
+JsonInput.OriginalProtoFieldName.ProtobufOutput
+JsonInput.PrimitiveRepeatedField.JsonOutput
+JsonInput.RepeatedBoolWrapper.JsonOutput
+JsonInput.RepeatedBytesWrapper.JsonOutput
+JsonInput.RepeatedDoubleWrapper.JsonOutput
+JsonInput.RepeatedFieldMessageElementIsNull
+JsonInput.RepeatedFieldPrimitiveElementIsNull
+JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool
+JsonInput.RepeatedFloatWrapper.JsonOutput
+JsonInput.RepeatedInt32Wrapper.JsonOutput
+JsonInput.RepeatedInt64Wrapper.JsonOutput
+JsonInput.RepeatedStringWrapper.JsonOutput
+JsonInput.RepeatedUint32Wrapper.JsonOutput
+JsonInput.RepeatedUint64Wrapper.JsonOutput
+JsonInput.StringFieldEscape.JsonOutput
+JsonInput.StringField.JsonOutput
+JsonInput.StringFieldSurrogatePair.JsonOutput
+JsonInput.StringFieldUnicodeEscape.JsonOutput
+JsonInput.StringFieldUnicodeEscapeWithLowercaseHexLetters.JsonOutput
+JsonInput.StringFieldUnicode.JsonOutput
+JsonInput.StringFieldUnpairedLowSurrogate
+JsonInput.StringRepeatedField.JsonOutput
+JsonInput.Struct.JsonOutput
+JsonInput.Struct.ProtobufOutput
+JsonInput.TimestampJsonInputLowercaseT
+JsonInput.TimestampMaxValue.JsonOutput
+JsonInput.TimestampMinValue.JsonOutput
+JsonInput.TimestampRepeatedValue.JsonOutput
+JsonInput.TimestampWithNegativeOffset.JsonOutput
+JsonInput.TimestampWithPositiveOffset.JsonOutput
+JsonInput.Uint32FieldMaxFloatValue.JsonOutput
+JsonInput.Uint32FieldMaxFloatValue.ProtobufOutput
+JsonInput.Uint32FieldMaxValue.JsonOutput
+JsonInput.Uint32MapField.JsonOutput
+JsonInput.Uint64FieldMaxValue.JsonOutput
+JsonInput.Uint64FieldMaxValueNotQuoted.JsonOutput
+JsonInput.Uint64MapField.JsonOutput
+JsonInput.ValueAcceptBool.JsonOutput
+JsonInput.ValueAcceptBool.ProtobufOutput
+JsonInput.ValueAcceptFloat.JsonOutput
+JsonInput.ValueAcceptFloat.ProtobufOutput
+JsonInput.ValueAcceptInteger.JsonOutput
+JsonInput.ValueAcceptInteger.ProtobufOutput
+JsonInput.ValueAcceptList.JsonOutput
+JsonInput.ValueAcceptList.ProtobufOutput
+JsonInput.ValueAcceptNull.JsonOutput
+JsonInput.ValueAcceptNull.ProtobufOutput
+JsonInput.ValueAcceptObject.JsonOutput
+JsonInput.ValueAcceptObject.ProtobufOutput
+JsonInput.ValueAcceptString.JsonOutput
+JsonInput.ValueAcceptString.ProtobufOutput
+JsonInput.WrapperTypesWithNullValue.JsonOutput
+ProtobufInput.DoubleFieldNormalizeQuietNan.JsonOutput
+ProtobufInput.DoubleFieldNormalizeSignalingNan.JsonOutput
+ProtobufInput.FloatFieldNormalizeQuietNan.JsonOutput
+ProtobufInput.FloatFieldNormalizeSignalingNan.JsonOutput
+TimestampProtoInputTooLarge.JsonOutput
+TimestampProtoInputTooSmall.JsonOutput

conformance/failure_list_python_cpp.txt

@@ -7,6 +7,177 @@
 # TODO(haberman): insert links to corresponding bugs tracking the issue.
 # Should we use GitHub issues or the Google-internal bug tracker?
 
+DurationProtoInputTooLarge.JsonOutput
+DurationProtoInputTooSmall.JsonOutput
+FieldMaskNumbersDontRoundTrip.JsonOutput
+FieldMaskPathsDontRoundTrip.JsonOutput
+FieldMaskTooManyUnderscore.JsonOutput
+JsonInput.AllFieldAcceptNull.JsonOutput
+JsonInput.Any.JsonOutput
+JsonInput.AnyNested.JsonOutput
+JsonInput.AnyNested.ProtobufOutput
+JsonInput.Any.ProtobufOutput
+JsonInput.AnyUnorderedTypeTag.JsonOutput
+JsonInput.AnyUnorderedTypeTag.ProtobufOutput
+JsonInput.AnyWithDuration.JsonOutput
+JsonInput.AnyWithDuration.ProtobufOutput
+JsonInput.AnyWithFieldMask.JsonOutput
+JsonInput.AnyWithFieldMask.ProtobufOutput
+JsonInput.AnyWithInt32ValueWrapper.JsonOutput
+JsonInput.AnyWithInt32ValueWrapper.ProtobufOutput
+JsonInput.AnyWithStruct.JsonOutput
+JsonInput.AnyWithStruct.ProtobufOutput
+JsonInput.AnyWithTimestamp.JsonOutput
+JsonInput.AnyWithTimestamp.ProtobufOutput
+JsonInput.AnyWithValueForInteger.JsonOutput
+JsonInput.AnyWithValueForInteger.ProtobufOutput
+JsonInput.AnyWithValueForJsonObject.JsonOutput
+JsonInput.AnyWithValueForJsonObject.ProtobufOutput
+JsonInput.BoolFieldFalse.JsonOutput
+JsonInput.BoolFieldTrue.JsonOutput
+JsonInput.BoolMapEscapedKey.JsonOutput
+JsonInput.BoolMapField.JsonOutput
+JsonInput.BytesFieldInvalidBase64Characters
+JsonInput.BytesField.JsonOutput
+JsonInput.BytesRepeatedField.JsonOutput
+JsonInput.DoubleFieldInfinity.JsonOutput
+JsonInput.DoubleFieldInfinityNotQuoted
+JsonInput.DoubleFieldMaxNegativeValue.JsonOutput
+JsonInput.DoubleFieldMaxPositiveValue.JsonOutput
+JsonInput.DoubleFieldMinNegativeValue.JsonOutput
+JsonInput.DoubleFieldMinPositiveValue.JsonOutput
+JsonInput.DoubleFieldNan.JsonOutput
+JsonInput.DoubleFieldNanNotQuoted
+JsonInput.DoubleFieldNegativeInfinity.JsonOutput
+JsonInput.DoubleFieldNegativeInfinityNotQuoted
+JsonInput.DoubleFieldQuotedValue.JsonOutput
+JsonInput.DoubleFieldTooSmall
+JsonInput.DurationJsonInputTooLarge
+JsonInput.DurationJsonInputTooSmall
+JsonInput.DurationMaxValue.JsonOutput
+JsonInput.DurationMinValue.JsonOutput
+JsonInput.DurationMissingS
+JsonInput.DurationRepeatedValue.JsonOutput
+JsonInput.EnumField.JsonOutput
+JsonInput.EnumFieldNumericValueNonZero.JsonOutput
+JsonInput.EnumFieldNumericValueNonZero.ProtobufOutput
+JsonInput.EnumFieldNumericValueZero.JsonOutput
+JsonInput.EnumFieldNumericValueZero.ProtobufOutput
+JsonInput.EnumFieldUnknownValue.Validator
+JsonInput.EnumRepeatedField.JsonOutput
+JsonInput.FieldMaskInvalidCharacter
+JsonInput.FieldMask.JsonOutput
+JsonInput.FieldMask.ProtobufOutput
+JsonInput.FieldNameEscaped.JsonOutput
+JsonInput.FieldNameInLowerCamelCase.Validator
+JsonInput.FieldNameInSnakeCase.JsonOutput
+JsonInput.FieldNameInSnakeCase.ProtobufOutput
+JsonInput.FieldNameWithMixedCases.JsonOutput
+JsonInput.FieldNameWithNumbers.JsonOutput
+JsonInput.FloatFieldInfinity.JsonOutput
+JsonInput.FloatFieldInfinityNotQuoted
+JsonInput.FloatFieldMaxNegativeValue.JsonOutput
+JsonInput.FloatFieldMaxPositiveValue.JsonOutput
+JsonInput.FloatFieldMinNegativeValue.JsonOutput
+JsonInput.FloatFieldMinPositiveValue.JsonOutput
+JsonInput.FloatFieldNan.JsonOutput
+JsonInput.FloatFieldNanNotQuoted
+JsonInput.FloatFieldNegativeInfinity.JsonOutput
+JsonInput.FloatFieldNegativeInfinityNotQuoted
+JsonInput.FloatFieldQuotedValue.JsonOutput
+JsonInput.FloatFieldTooLarge
+JsonInput.FloatFieldTooSmall
+JsonInput.HelloWorld.JsonOutput
+JsonInput.Int32FieldExponentialFormat.JsonOutput
+JsonInput.Int32FieldExponentialFormat.ProtobufOutput
+JsonInput.Int32FieldFloatTrailingZero.JsonOutput
+JsonInput.Int32FieldFloatTrailingZero.ProtobufOutput
+JsonInput.Int32FieldMaxFloatValue.JsonOutput
+JsonInput.Int32FieldMaxFloatValue.ProtobufOutput
+JsonInput.Int32FieldMaxValue.JsonOutput
+JsonInput.Int32FieldMinFloatValue.JsonOutput
+JsonInput.Int32FieldMinFloatValue.ProtobufOutput
+JsonInput.Int32FieldMinValue.JsonOutput
+JsonInput.Int32FieldStringValueEscaped.JsonOutput
+JsonInput.Int32FieldStringValue.JsonOutput
+JsonInput.Int32MapEscapedKey.JsonOutput
+JsonInput.Int32MapField.JsonOutput
+JsonInput.Int64FieldMaxValue.JsonOutput
+JsonInput.Int64FieldMaxValueNotQuoted.JsonOutput
+JsonInput.Int64FieldMinValue.JsonOutput
+JsonInput.Int64FieldMinValueNotQuoted.JsonOutput
+JsonInput.Int64MapEscapedKey.JsonOutput
+JsonInput.Int64MapField.JsonOutput
+JsonInput.MessageField.JsonOutput
+JsonInput.MessageMapField.JsonOutput
+JsonInput.MessageRepeatedField.JsonOutput
+JsonInput.OptionalBoolWrapper.JsonOutput
+JsonInput.OptionalBytesWrapper.JsonOutput
+JsonInput.OptionalDoubleWrapper.JsonOutput
+JsonInput.OptionalFloatWrapper.JsonOutput
+JsonInput.OptionalInt32Wrapper.JsonOutput
+JsonInput.OptionalInt64Wrapper.JsonOutput
+JsonInput.OptionalStringWrapper.JsonOutput
+JsonInput.OptionalUint32Wrapper.JsonOutput
+JsonInput.OptionalUint64Wrapper.JsonOutput
+JsonInput.OptionalWrapperTypesWithNonDefaultValue.JsonOutput
+JsonInput.OriginalProtoFieldName.JsonOutput
+JsonInput.OriginalProtoFieldName.ProtobufOutput
+JsonInput.PrimitiveRepeatedField.JsonOutput
+JsonInput.RepeatedBoolWrapper.JsonOutput
+JsonInput.RepeatedBytesWrapper.JsonOutput
+JsonInput.RepeatedDoubleWrapper.JsonOutput
+JsonInput.RepeatedFieldMessageElementIsNull
+JsonInput.RepeatedFieldPrimitiveElementIsNull
+JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool
+JsonInput.RepeatedFloatWrapper.JsonOutput
+JsonInput.RepeatedInt32Wrapper.JsonOutput
+JsonInput.RepeatedInt64Wrapper.JsonOutput
+JsonInput.RepeatedStringWrapper.JsonOutput
+JsonInput.RepeatedUint32Wrapper.JsonOutput
+JsonInput.RepeatedUint64Wrapper.JsonOutput
+JsonInput.StringFieldEscape.JsonOutput
+JsonInput.StringField.JsonOutput
+JsonInput.StringFieldSurrogatePair.JsonOutput
+JsonInput.StringFieldUnicodeEscape.JsonOutput
+JsonInput.StringFieldUnicodeEscapeWithLowercaseHexLetters.JsonOutput
+JsonInput.StringFieldUnicode.JsonOutput
+JsonInput.StringFieldUnpairedLowSurrogate
+JsonInput.StringRepeatedField.JsonOutput
+JsonInput.Struct.JsonOutput
+JsonInput.Struct.ProtobufOutput
+JsonInput.TimestampJsonInputLowercaseT
+JsonInput.TimestampMaxValue.JsonOutput
+JsonInput.TimestampMinValue.JsonOutput
+JsonInput.TimestampRepeatedValue.JsonOutput
+JsonInput.TimestampWithNegativeOffset.JsonOutput
+JsonInput.TimestampWithPositiveOffset.JsonOutput
+JsonInput.Uint32FieldMaxFloatValue.JsonOutput
+JsonInput.Uint32FieldMaxFloatValue.ProtobufOutput
+JsonInput.Uint32FieldMaxValue.JsonOutput
+JsonInput.Uint32MapField.JsonOutput
+JsonInput.Uint64FieldMaxValue.JsonOutput
+JsonInput.Uint64FieldMaxValueNotQuoted.JsonOutput
+JsonInput.Uint64MapField.JsonOutput
+JsonInput.ValueAcceptBool.JsonOutput
+JsonInput.ValueAcceptBool.ProtobufOutput
+JsonInput.ValueAcceptFloat.JsonOutput
+JsonInput.ValueAcceptFloat.ProtobufOutput
+JsonInput.ValueAcceptInteger.JsonOutput
+JsonInput.ValueAcceptInteger.ProtobufOutput
+JsonInput.ValueAcceptList.JsonOutput
+JsonInput.ValueAcceptList.ProtobufOutput
+JsonInput.ValueAcceptNull.JsonOutput
+JsonInput.ValueAcceptNull.ProtobufOutput
+JsonInput.ValueAcceptObject.JsonOutput
+JsonInput.ValueAcceptObject.ProtobufOutput
+JsonInput.ValueAcceptString.JsonOutput
+JsonInput.ValueAcceptString.ProtobufOutput
+JsonInput.WrapperTypesWithNullValue.JsonOutput
+ProtobufInput.DoubleFieldNormalizeQuietNan.JsonOutput
+ProtobufInput.DoubleFieldNormalizeSignalingNan.JsonOutput
+ProtobufInput.FloatFieldNormalizeQuietNan.JsonOutput
+ProtobufInput.FloatFieldNormalizeSignalingNan.JsonOutput
 ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE
 ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE
 ProtobufInput.PrematureEofInPackedField.BOOL
@@ -23,3 +194,5 @@ ProtobufInput.PrematureEofInPackedField.SINT32
 ProtobufInput.PrematureEofInPackedField.SINT64
 ProtobufInput.PrematureEofInPackedField.UINT32
 ProtobufInput.PrematureEofInPackedField.UINT64
+TimestampProtoInputTooLarge.JsonOutput
+TimestampProtoInputTooSmall.JsonOutput

python/setup.py

@@ -149,6 +149,10 @@ class build_py(_build_py):
 class test_conformance(_build_py):
   target = 'test_python'
   def run(self):
+    if sys.version_info >= (2, 7):
+      # Python 2.6 dodges these extra failures.
+      os.environ["CONFORMANCE_PYTHON_EXTRA_FAILURES"] = (
+          "--failure_list failure_list_python-post26.txt")
     cmd = 'cd ../conformance && make %s' % (test_conformance.target)
     status = subprocess.check_call(cmd, shell=True)
 

python/tox.ini

@@ -15,10 +15,8 @@ commands =
     cpp: python setup.py -q build --cpp_implementation --warnings_as_errors
     python: python setup.py -q test -q
     cpp: python setup.py -q test -q --cpp_implementation
-# TODO(xiaofeng): Disabled because the Python testee program crashes on
-# some conformance test inputs.
-# python: python setup.py -q test_conformance
-# cpp: python setup.py -q test_conformance --cpp_implementation
+    python: python setup.py -q test_conformance
+    cpp: python setup.py -q test_conformance --cpp_implementation
 deps =
     # Keep this list of dependencies in sync with setup.py.
     six>=1.9

ruby/travis-test.sh

@@ -18,10 +18,7 @@ test_version() {
        which ruby && \
        gem install bundler && bundle && \
        rake test"
-    # TODO(xiaofeng): Conformance tests are disabled because the ruby testee
-    # program crashes on some inputs.
-    #  cd ../conformance && \
-    #  make test_ruby
+    cd ../conformance && make test_ruby
   fi
 }
 

travis.sh

@@ -53,9 +53,7 @@ build_csharp() {
 
   (cd csharp/src; mono ../../nuget.exe restore)
   csharp/buildall.sh
-  # TODO(xiaofeng): The conformance tests are disable because the testee program
-  # crashes on some inputs.
-  # cd conformance && make test_csharp && cd ..
+  cd conformance && make test_csharp && cd ..
 }
 
 build_golang() {