Merge pull request #1022 from haberman/python-conformance2

Conformance test implementation for Python.

conformance/Makefile.am

@@ -53,7 +53,7 @@ endif
 if USE_EXTERNAL_PROTOC
 
 protoc_middleman: $(protoc_inputs)
-	$(PROTOC) -I$(srcdir) --cpp_out=. --java_out=. --ruby_out=. --objc_out=. $^
+	$(PROTOC) -I$(srcdir) --cpp_out=. --java_out=. --ruby_out=. --objc_out=. --python_out=.$^
 	touch protoc_middleman
 
 else
@@ -62,7 +62,7 @@ else
 # relative to srcdir, which may not be the same as the current directory when
 # building out-of-tree.
 protoc_middleman: $(top_srcdir)/src/protoc$(EXEEXT) $(protoc_inputs)
-	oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. --cpp_out=$$oldpwd --java_out=$$oldpwd --ruby_out=$$oldpwd --objc_out=$$oldpwd $(protoc_inputs) )
+	oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. --cpp_out=$$oldpwd --java_out=$$oldpwd --ruby_out=$$oldpwd --objc_out=$$oldpwd --python_out=$$oldpwd $(protoc_inputs) )
 	touch protoc_middleman
 
 endif
@@ -110,6 +110,14 @@ test_csharp: protoc_middleman conformance-test-runner conformance-csharp
 test_ruby: protoc_middleman conformance-test-runner $(other_language_protoc_outputs)
 	RUBYLIB=../ruby/lib:. ./conformance-test-runner --failure_list failure_list_ruby.txt ./conformance_ruby.rb
 
+# These depend on library paths being properly set up.  The easiest way to
+# run them is to just use "tox" from the python dir.
+test_python: protoc_middleman conformance-test-runner
+	./conformance-test-runner --failure_list failure_list_python.txt ./conformance_python.py
+
+test_python_cpp: protoc_middleman conformance-test-runner
+	./conformance-test-runner --failure_list failure_list_python_cpp.txt ./conformance_python.py
+
 if OBJC_CONFORMANCE_TEST
 
 test_objc: protoc_middleman conformance-test-runner conformance-objc

conformance/conformance_python.py

+#!/usr/bin/env python
+#
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# https://developers.google.com/protocol-buffers/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""A conformance test implementation for the Python protobuf library.
+
+See conformance.proto for more information.
+"""
+
+import struct
+import sys
+import os
+from google.protobuf import message
+from google.protobuf import json_format
+import conformance_pb2
+
+sys.stdout = os.fdopen(sys.stdout.fileno(), 'wb', 0)
+sys.stdin = os.fdopen(sys.stdin.fileno(), 'rb', 0)
+
+test_count = 0
+verbose = False
+
+class ProtocolError(Exception):
+  pass
+
+def do_test(request):
+  test_message = conformance_pb2.TestAllTypes()
+  response = conformance_pb2.ConformanceResponse()
+  test_message = conformance_pb2.TestAllTypes()
+
+  try:
+    if request.WhichOneof('payload') == 'protobuf_payload':
+      try:
+        test_message.ParseFromString(request.protobuf_payload)
+      except message.DecodeError as e:
+        response.parse_error = str(e)
+        return response
+
+    elif request.WhichOneof('payload') == 'json_payload':
+      try:
+        json_format.Parse(request.json_payload, test_message)
+      except json_format.ParseError as e:
+        response.parse_error = str(e)
+        return response
+
+    else:
+      raise ProtocolError("Request didn't have payload.")
+
+    if request.requested_output_format == conformance_pb2.UNSPECIFIED:
+      raise ProtocolError("Unspecified output format")
+
+    elif request.requested_output_format == conformance_pb2.PROTOBUF:
+      response.protobuf_payload = test_message.SerializeToString()
+
+    elif request.requested_output_format == conformance_pb2.JSON:
+      response.json_payload = json_format.MessageToJson(test_message)
+
+  except Exception as e:
+    response.runtime_error = str(e)
+
+  return response
+
+def do_test_io():
+  length_bytes = sys.stdin.read(4)
+  if len(length_bytes) == 0:
+    return False   # EOF
+  elif len(length_bytes) != 4:
+    raise IOError("I/O error")
+
+  # "I" is "unsigned int", so this depends on running on a platform with
+  # 32-bit "unsigned int" type.  The Python struct module unfortunately
+  # has no format specifier for uint32_t.
+  length = struct.unpack("<I", length_bytes)[0]
+  serialized_request = sys.stdin.read(length)
+  if len(serialized_request) != length:
+    raise IOError("I/O error")
+
+  request = conformance_pb2.ConformanceRequest()
+  request.ParseFromString(serialized_request)
+
+  response = do_test(request)
+
+  serialized_response = response.SerializeToString()
+  sys.stdout.write(struct.pack("<I", len(serialized_response)))
+  sys.stdout.write(serialized_response)
+  sys.stdout.flush()
+
+  if verbose:
+    sys.stderr.write("conformance_python: request=%s, response=%s\n" % (
+                       request.ShortDebugString().c_str(),
+                       response.ShortDebugString().c_str()))
+
+  global test_count
+  test_count += 1
+
+  return True
+
+while True:
+  if not do_test_io():
+    sys.stderr.write("conformance_python: received EOF from test runner " +
+                     "after %s tests, exiting\n" % (test_count))
+    sys.exit(0)

conformance/conformance_test.cc

@@ -514,9 +514,11 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
                     "These tests succeeded, even though they were listed in "
                     "the failure list.  Remove them from the failure list");
 
-  CheckSetEmpty(skipped_,
-                "These tests were skipped (probably because support for some "
-                "features is not implemented)");
+  if (verbose_) {
+    CheckSetEmpty(skipped_,
+                  "These tests were skipped (probably because support for some "
+                  "features is not implemented)");
+  }
 
   StringAppendF(&output_,
                 "CONFORMANCE SUITE %s: %d successes, %d skipped, "

conformance/conformance_test_runner.cc

@@ -202,6 +202,12 @@ void UsageError() {
 
 void ParseFailureList(const char *filename, vector<string>* failure_list) {
   std::ifstream infile(filename);
+
+  if (!infile.is_open()) {
+    fprintf(stderr, "Couldn't open failure list file: %s\n", filename);
+    exit(1);
+  }
+
   for (string line; getline(infile, line);) {
     // Remove whitespace.
     line.erase(std::remove_if(line.begin(), line.end(), ::isspace),

conformance/failure_list_python_cpp.txt

+# This is the list of conformance tests that are known to fail for the
+# Python/C++ implementation right now.  These should be fixed.
+#
+# By listing them here we can keep tabs on which ones are failing and be sure
+# that we don't introduce regressions in other tests.
+#
+# TODO(haberman): insert links to corresponding bugs tracking the issue.
+# Should we use GitHub issues or the Google-internal bug tracker?
+
+ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE
+ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE
+ProtobufInput.PrematureEofInPackedField.BOOL
+ProtobufInput.PrematureEofInPackedField.DOUBLE
+ProtobufInput.PrematureEofInPackedField.ENUM
+ProtobufInput.PrematureEofInPackedField.FIXED32
+ProtobufInput.PrematureEofInPackedField.FIXED64
+ProtobufInput.PrematureEofInPackedField.FLOAT
+ProtobufInput.PrematureEofInPackedField.INT32
+ProtobufInput.PrematureEofInPackedField.INT64
+ProtobufInput.PrematureEofInPackedField.SFIXED32
+ProtobufInput.PrematureEofInPackedField.SFIXED64
+ProtobufInput.PrematureEofInPackedField.SINT32
+ProtobufInput.PrematureEofInPackedField.SINT64
+ProtobufInput.PrematureEofInPackedField.UINT32
+ProtobufInput.PrematureEofInPackedField.UINT64

python/setup.py

@@ -144,6 +144,12 @@ class build_py(_build_py):
     # _build_py is an old-style class, so super() doesn't work.
     _build_py.run(self)
 
+class test_conformance(_build_py):
+  target = 'test_python'
+  def run(self):
+    cmd = 'cd ../conformance && make %s' % (test_conformance.target)
+    status = subprocess.check_call(cmd, shell=True)
+
 
 if __name__ == '__main__':
   ext_module_list = []
@@ -152,6 +158,7 @@ if __name__ == '__main__':
   if cpp_impl in sys.argv:
     sys.argv.remove(cpp_impl)
     extra_compile_args = ['-Wno-write-strings', '-Wno-invalid-offsetof']
+    test_conformance.target = 'test_python_cpp'
 
     if "clang" in os.popen('$CC --version').read():
       extra_compile_args.append('-Wno-shorten-64-to-32')
@@ -207,6 +214,7 @@ if __name__ == '__main__':
       cmdclass={
           'clean': clean,
           'build_py': build_py,
+          'test_conformance': test_conformance,
       },
       install_requires=install_requires,
       ext_modules=ext_module_list,

python/tox.ini

@@ -15,6 +15,8 @@ commands =
     cpp: python setup.py -q build --cpp_implementation --warnings_as_errors
     python: python setup.py -q test -q
     cpp: python setup.py -q test -q --cpp_implementation
+    python: python setup.py -q test_conformance
+    cpp: python setup.py -q test_conformance --cpp_implementation
 deps =
     # Keep this list of dependencies in sync with setup.py.
     six