Merge pull request #4158 from BSBandme/FixBenchmarks

Fix java benchmark to use parser, fix cpp benchmark new arena to use …

benchmarks/Makefile.am

@@ -45,7 +45,7 @@ protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_pr
 protoc_middleman2:  make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs_proto2) $(well_known_type_protoc_inputs)
 	oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java --python_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2) )
 	touch protoc_middleman2
-	
+
 endif
 
 all_data = `find . -type f -name "dataset.*.pb"`
@@ -75,7 +75,7 @@ benchmarks_protoc_outputs_proto2_header =                                  \
 	datasets/google_message4/benchmark_message4.pb.h                         \
 	datasets/google_message4/benchmark_message4_1.pb.h                       \
 	datasets/google_message4/benchmark_message4_2.pb.h                       \
-	datasets/google_message4/benchmark_message4_3.pb.h	
+	datasets/google_message4/benchmark_message4_3.pb.h
 
 benchmarks_protoc_outputs_proto2 =                                         \
 	datasets/google_message1/benchmark_message1_proto2.pb.cc                 \
@@ -144,7 +144,7 @@ javac_middleman: $(java_benchmark_testing_files) protoc_middleman protoc_middlem
 	cd ../..
 	@touch javac_middleman
 
-java-benchmark: javac_middleman 
+java-benchmark: javac_middleman
 	@echo "Writing shortcut script java-benchmark..."
 	@echo '#! /bin/sh' > java-benchmark
 	@echo 'java -cp '"tmp/java/target/*.jar"' com.google.protobuf.ProtoBench $$@' >> java-benchmark
@@ -173,7 +173,7 @@ python_cpp_pkg_flags = `pkg-config --cflags --libs python`
 
 lib_LTLIBRARIES = libbenchmark_messages.la
 libbenchmark_messages_la_SOURCES = python_benchmark_messages.cc
-libbenchmark_messages_la_LIBADD = $(top_srcdir)/src/.libs/libprotobuf.la 
+libbenchmark_messages_la_LIBADD = $(top_srcdir)/src/.libs/libprotobuf.la
 libbenchmark_messages_la_LDFLAGS = -version-info 1:0:0 -export-dynamic
 libbenchmark_messages_la_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir) $(python_cpp_pkg_flags)
 libbenchmark_messages_la-libbenchmark_messages_la.$(OBJEXT): $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2)

benchmarks/README.md

@@ -7,45 +7,46 @@ protobuf language runtime.
 
 ## Prerequisite
 
-First, you need to follow the instruction in the root directory's README to 
+First, you need to follow the instruction in the root directory's README to
 build your language's protobuf, then:
 
 ### CPP
 You need to install [cmake](https://cmake.org/) before building the benchmark.
 
-We are using [google/benchmark](https://github.com/google/benchmark) as the 
-benchmark tool for testing cpp. This will be automaticly made during build the 
+We are using [google/benchmark](https://github.com/google/benchmark) as the
+benchmark tool for testing cpp. This will be automaticly made during build the
 cpp benchmark.
 
 ### Java
-We're using maven to build the java benchmarks, which is the same as to build 
-the Java protobuf. There're no other tools need to install. We're using 
-[google/caliper](https://github.com/google/caliper) as benchmark tool, which 
+We're using maven to build the java benchmarks, which is the same as to build
+the Java protobuf. There're no other tools need to install. We're using
+[google/caliper](https://github.com/google/caliper) as benchmark tool, which
 can be automaticly included by maven.
 
 ### Python
-We're using python C++ API for testing the generated 
+We're using python C++ API for testing the generated
 CPP proto version of python protobuf, which is also a prerequisite for Python
-protobuf cpp implementation. You need to install the correct version of Python 
-C++ extension package before run generated CPP proto version of Python 
-protobuf's benchmark. e.g. under Ubuntu, you need to 
+protobuf cpp implementation. You need to install the correct version of Python
+C++ extension package before run generated CPP proto version of Python
+protobuf's benchmark. e.g. under Ubuntu, you need to
 
 ```
-$ sudo apt-get install python-dev 
+$ sudo apt-get install python-dev
 $ sudo apt-get install python3-dev
 ```
-And you also need to make sure `pkg-config` is installed.  
+And you also need to make sure `pkg-config` is installed.
 
 ### Big data
 
-There's some optional big testing data which is not included in the directory initially, you need to 
-run the following command to download the testing data:
+There's some optional big testing data which is not included in the directory
+initially, you need to run the following command to download the testing data:
 
 ```
-$ ./download_data.sh 
+$ ./download_data.sh
 ```
 
-After doing this the big data file will automaticly generated in the benchmark directory.  
+After doing this the big data file will automaticly generated in the
+benchmark directory.
 
 ## Run instructions
 
@@ -65,8 +66,8 @@ $ make cpp
 
 ### Python:
 
-We have three versions of python protobuf implementation: pure python, cpp reflection and 
-cpp generated code. To run these version benchmark, you need to:
+We have three versions of python protobuf implementation: pure python, cpp
+reflection and cpp generated code. To run these version benchmark, you need to:
 
 #### Pure Python:
 
@@ -84,8 +85,8 @@ $ make python-cpp-reflection
 
 ```
 $ make python-cpp-generated-code
-```        
-  
+```
+
 To run a specific dataset:
 
 ### Java:

benchmarks/cpp_benchmark.cc

@@ -121,9 +121,10 @@ class ParseNewArenaFixture : public Fixture {
   virtual void BenchmarkCase(benchmark::State& state) {
     WrappingCounter i(payloads_.size());
     size_t total = 0;
+    Arena arena;
 
     while (state.KeepRunning()) {
-      Arena arena;
+      arena.Reset();
       Message* m = Arena::CreateMessage<T>(&arena);
       const std::string& payload = payloads_[i.Next()];
       total += payload.size();

benchmarks/java/pom.xml

 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" 
-  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-  
+
   <modelVersion>4.0.0</modelVersion>
   <artifactId>protobuf-java-benchmark</artifactId>
   <groupId>com.google.protobuf</groupId>
   <version>1.0.0</version>
   <name>Protocol Buffers [Benchmark]</name>
   <description>The benchmark tools for Protobuf Java.</description>
-  
+
   <dependencies>
     <dependency>
       <groupId>com.google.protobuf</groupId>
@@ -22,7 +22,7 @@
       <version>1.0-beta-2</version>
     </dependency>
   </dependencies>
-  
+
   <build>
     <pluginManagement>
       <plugins>

benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java

@@ -153,10 +153,8 @@ public class ProtoBenchCaliper {
   @Benchmark
   void deserializeFromByteString(int reps) throws IOException {
     for (int i = 0; i < reps; i++) {
-      defaultMessage
-        .newBuilderForType()
-        .mergeFrom(inputStringList.get((int) (counter % inputStringList.size())), extensions)
-        .build();
+      benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom(
+          inputStringList.get((int) (counter % inputStringList.size())), extensions);
       counter++;
     }
   }
@@ -164,10 +162,8 @@ public class ProtoBenchCaliper {
   @Benchmark
   void deserializeFromByteArray(int reps) throws IOException {
     for (int i = 0; i < reps; i++) {
-      defaultMessage
-        .newBuilderForType()
-        .mergeFrom(inputDataList.get((int) (counter % inputDataList.size())), extensions)
-        .build();
+      benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom(
+          inputDataList.get((int) (counter % inputDataList.size())), extensions);
       counter++;
     }
   }
@@ -175,10 +171,8 @@ public class ProtoBenchCaliper {
   @Benchmark
   void deserializeFromMemoryStream(int reps) throws IOException {
     for (int i = 0; i < reps; i++) {
-      defaultMessage
-        .newBuilderForType()
-        .mergeFrom(inputStreamList.get((int) (counter % inputStreamList.size())), extensions)
-        .build();
+      benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom(
+          inputStreamList.get((int) (counter % inputStreamList.size())), extensions);
       inputStreamList.get((int) (counter % inputStreamList.size())).reset();
       counter++;
     }

benchmarks/py_benchmark.py

@@ -4,6 +4,7 @@ import timeit
 import math
 import fnmatch
 
+# BEGIN CPP GENERATED MESSAGE
 # CPP generated code must be linked before importing the generated Python code
 # for the descriptor can be found in the pool
 if len(sys.argv) < 2:
@@ -14,13 +15,14 @@ if sys.argv[1] == "true":
   sys.path.append( os.path.dirname( os.path.dirname( os.path.abspath(__file__) ) ) + "/tmp" )
 elif sys.argv[1] != "false":
   raise IOError("Need string argument \"true\" or \"false\" for whether to use cpp generated code")
+# END CPP GENERATED MESSAGE
 
 import datasets.google_message1.benchmark_message1_proto2_pb2 as benchmark_message1_proto2_pb2
 import datasets.google_message1.benchmark_message1_proto3_pb2 as benchmark_message1_proto3_pb2
 import datasets.google_message2.benchmark_message2_pb2 as benchmark_message2_pb2
 import datasets.google_message3.benchmark_message3_pb2 as benchmark_message3_pb2
 import datasets.google_message4.benchmark_message4_pb2 as benchmark_message4_pb2
-import benchmarks_pb2
+import benchmarks_pb2 as benchmarks_pb2
 
 
 def run_one_test(filename):
@@ -37,15 +39,15 @@ def run_one_test(filename):
   benchmark_util.set_test_method("serialize_to_benchmark")
   print benchmark_util.run_benchmark(setup_method_args='"%s"' % (filename))
   print ""
-  
-def init(filename): 
+
+def init(filename):
   global benchmark_dataset, message_class, message_list, counter
   message_list=[]
   counter = 0
   data = open(os.path.dirname(sys.argv[0]) + "/../" + filename).read()
   benchmark_dataset = benchmarks_pb2.BenchmarkDataset()
   benchmark_dataset.ParseFromString(data)
-     
+
   if benchmark_dataset.message_name == "benchmarks.proto3.GoogleMessage1":
     message_class = benchmark_message1_proto3_pb2.GoogleMessage1
   elif benchmark_dataset.message_name == "benchmarks.proto2.GoogleMessage1":
@@ -53,63 +55,63 @@ def init(filename):
   elif benchmark_dataset.message_name == "benchmarks.proto2.GoogleMessage2":
     message_class = benchmark_message2_pb2.GoogleMessage2
   elif benchmark_dataset.message_name == "benchmarks.google_message3.GoogleMessage3":
-    message_class = benchmark_message3_pb2.GoogleMessage3  
+    message_class = benchmark_message3_pb2.GoogleMessage3
   elif benchmark_dataset.message_name == "benchmarks.google_message4.GoogleMessage4":
     message_class = benchmark_message4_pb2.GoogleMessage4
-  else: 
+  else:
     raise IOError("Message %s not found!" % (benchmark_dataset.message_name))
-   
+
   for one_payload in benchmark_dataset.payload:
     temp = message_class()
     temp.ParseFromString(one_payload)
     message_list.append(temp)
-  
+
 def parse_from_benchmark():
   global counter, message_class, benchmark_dataset
   m = message_class().ParseFromString(benchmark_dataset.payload[counter % len(benchmark_dataset.payload)])
   counter = counter + 1
-  
+
 def serialize_to_benchmark():
   global counter, message_list, message_class
   s = message_list[counter % len(benchmark_dataset.payload)].SerializeToString()
   counter = counter + 1
- 
+
 
 class Benchmark:
-  def __init__(self, module=None, test_method=None, 
+  def __init__(self, module=None, test_method=None,
                setup_method=None, full_iteration = 1):
     self.full_iteration = full_iteration
     self.module = module
     self.test_method = test_method
     self.setup_method = setup_method
-    
+
   def set_test_method(self, test_method):
     self.test_method = test_method
-  
+
   def full_setup_code(self, setup_method_args=''):
     setup_code = ""
     setup_code += "from %s import %s\n" % (self.module, self.test_method)
     setup_code += "from %s import %s\n" % (self.module, self.setup_method)
     setup_code += "%s(%s)\n" % (self.setup_method, setup_method_args)
-    return setup_code 
-  
+    return setup_code
+
   def dry_run(self, test_method_args='', setup_method_args=''):
     return timeit.timeit(stmt="%s(%s)" % (self.test_method, test_method_args),
                          setup=self.full_setup_code(setup_method_args),
                          number=self.full_iteration);
-  
+
   def run_benchmark(self, test_method_args='', setup_method_args=''):
-    reps = self.full_iteration; 
+    reps = self.full_iteration;
     t = self.dry_run(test_method_args, setup_method_args);
     if t < 3 :
-      reps = int(math.ceil(3 / t)) * self.full_iteration 
+      reps = int(math.ceil(3 / t)) * self.full_iteration
     t = timeit.timeit(stmt="%s(%s)" % (self.test_method, test_method_args),
                       setup=self.full_setup_code(setup_method_args),
                       number=reps);
     return "Average time for %s: %.2f ns" % \
       (self.test_method, 1.0 * t / reps * (10 ** 9))
-  
-  
+
+
 if __name__ == "__main__":
   for i in range(2, len(sys.argv)):
     run_one_test(sys.argv[i])