Fix java benchmark bug, fix python library_path

benchmarks/Makefile.am

@@ -126,8 +126,10 @@ java_benchmark_testing_files =                                      \
 	java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java
 
 javac_middleman: $(java_benchmark_testing_files) protoc_middleman protoc_middleman2
-	cp -r $(srcdir)/java tmp && cd tmp/java &&  mvn clean compile assembly:single
-	cd ../..
+	cp -r $(srcdir)/java tmp 
+	mkdir -p tmp/java/lib
+	cp $(top_srcdir)/java/core/target/*.jar tmp/java/lib/protobuf-java.jar
+	cd tmp/java && mvn clean compile assembly:single -Dprotobuf.version=$(PACKAGE_VERSION) && cd ../..
 	@touch javac_middleman
 
 java-benchmark: javac_middleman
@@ -137,10 +139,10 @@ java-benchmark: javac_middleman
 	@echo 'conf=()' >> java-benchmark
 	@echo 'data_files=""' >> java-benchmark
 	@echo 'for arg in $$@; do if [[ $${arg:0:1} == "-" ]]; then conf+=($$arg); else data_files+="$$arg,"; fi; done' >> java-benchmark
-	@echo 'java -cp '"tmp/java/target/*.jar"' com.google.caliper.runner.CaliperMain com.google.protobuf.ProtoCaliperBenchmark -i runtime '"\\"  >> java-benchmark
-	@echo '-b serializeToByteString,serializeToByteArray,serializeToMemoryStream,'"\\"  >> java-benchmark
-	@echo 'deserializeFromByteString,deserializeFromByteArray,deserializeFromMemoryStream '"\\" >> java-benchmark
-	@echo '-DdataFile=$${data_files:0:-1} $${conf[*]}' >> java-benchmark
+	@echo 'java -cp '\"tmp/java/target/*:$(top_srcdir)/java/core/target/*:$(top_srcdir)/java/util/target/*\"" \\" >>java-benchmark
+	@echo '   com.google.caliper.runner.CaliperMain com.google.protobuf.ProtoCaliperBenchmark -i runtime '"\\"  >> java-benchmark
+	@echo '   -b serializeToByteArray,serializeToMemoryStream,deserializeFromByteArray,deserializeFromMemoryStream '"\\" >> java-benchmark
+	@echo '   -DdataFile=$${data_files:0:-1} $${conf[*]}' >> java-benchmark
 	@chmod +x java-benchmark
 
 java: protoc_middleman protoc_middleman2 java-benchmark

benchmarks/java/pom.xml

@@ -14,7 +14,10 @@
     <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
-      <version>3.5.0</version>
+      <version>${protobuf.version}</version>
+      <type>jar</type>
+      <scope>system</scope>
+      <systemPath>${project.basedir}/lib/protobuf-java.jar</systemPath>
     </dependency>
     <dependency>
       <groupId>com.google.caliper</groupId>

benchmarks/java/src/main/java/com/google/protobuf/ProtoCaliperBenchmark.java

@@ -5,6 +5,7 @@ import com.google.caliper.BeforeExperiment;
 import com.google.caliper.AfterExperiment;
 import com.google.caliper.Benchmark;
 import com.google.caliper.Param;
+import com.google.caliper.api.VmOptions;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.CodedOutputStream;
 import com.google.protobuf.ExtensionRegistry;
@@ -22,6 +23,12 @@ import java.io.RandomAccessFile;
 import java.util.ArrayList;
 import java.util.List;
 
+// Caliper set CICompilerCount to 1 for making sure compilation doesn't run in parallel with itself,
+// This makes TieredCompilation not working. We just disable TieredCompilation by default. In master 
+// branch this has been disabled by default in caliper: 
+// https://github.com/google/caliper/blob/master/caliper-runner/src/main/java/com/google/caliper/runner/target/Jvm.java#L38:14
+// But this haven't been added into most recent release.
+@VmOptions("-XX:-TieredCompilation")
 public class ProtoCaliperBenchmark {
   public enum BenchmarkMessageType {
     GOOGLE_MESSAGE1_PROTO3 {
@@ -151,18 +158,6 @@ public class ProtoCaliperBenchmark {
   }
   
   
-  @Benchmark
-  void serializeToByteString(int reps) throws IOException {
-    if (sampleMessageList.size() == 0) {
-      return;
-    }
-    for (int i = 0; i < reps; i++) {
-      for (int j = 0; j < sampleMessageList.size(); j++) {
-        sampleMessageList.get(j).toByteString();        
-      }
-    }
-  }
-  
   @Benchmark
   void serializeToByteArray(int reps) throws IOException {
     if (sampleMessageList.size() == 0) {
@@ -188,19 +183,6 @@ public class ProtoCaliperBenchmark {
     }
   }
   
-  @Benchmark
-  void deserializeFromByteString(int reps) throws IOException {
-    if (inputStringList.size() == 0) {
-      return;
-    }
-    for (int i = 0; i < reps; i++) {
-      for (int j = 0; j < inputStringList.size(); j++) {
-        benchmarkMessageType.getDefaultInstance().getParserForType().parseFrom(
-          inputStringList.get(j), extensions);
-      }
-    }
-  }
-  
   @Benchmark
   void deserializeFromByteArray(int reps) throws IOException {
     if (inputDataList.size() == 0) {

kokoro/linux/benchmark/build.sh

@@ -41,10 +41,10 @@ echo "benchmarking pure python..."
 ./python-pure-python-benchmark --json --behavior_prefix="pure-python-benchmark" $datasets  >> tmp/python_result.json
 echo "," >> "tmp/python_result.json"
 echo "benchmarking python cpp reflection..."
-env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" ./python-cpp-reflection-benchmark --json --behavior_prefix="cpp-reflection-benchmark" $datasets  >> tmp/python_result.json
+env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-reflection-benchmark --json --behavior_prefix="cpp-reflection-benchmark" $datasets  >> tmp/python_result.json
 echo "," >> "tmp/python_result.json"
 echo "benchmarking python cpp generated code..."
-env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" ./python-cpp-generated-code-benchmark --json --behavior_prefix="cpp-generated-code-benchmark" $datasets >> tmp/python_result.json
+env LD_PRELOAD="$oldpwd/gperftools/.libs/libtcmalloc.so" LD_LIBRARY_PATH="$oldpwd/src/.libs" ./python-cpp-generated-code-benchmark --json --behavior_prefix="cpp-generated-code-benchmark" $datasets >> tmp/python_result.json
 echo "]" >> "tmp/python_result.json"
 cd $oldpwd
 
@@ -52,6 +52,11 @@ cd $oldpwd
 ./configure
 make clean && make -j8
 
+# build Java protobuf
+cd java
+mvn package
+cd ..
+
 # build CPP benchmark
 cd benchmarks
 mv tmp/python_result.json . && make clean && make -j8 cpp-benchmark && mv python_result.json tmp
@@ -78,11 +83,10 @@ echo "benchmarking go..."
 make java-benchmark
 echo "benchmarking java..."
 ./java-benchmark -Cresults.file.options.file="tmp/java_result.json" $datasets
-cat $(find /tmp -name "trail-1.log")
 
 # upload result to bq
 make python_add_init
-python util/run_and_upload.py -cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" \
+env LD_LIBRARY_PATH="$oldpwd/src/.libs" python util/run_and_upload.py -cpp="../tmp/cpp_result.json" -java="../tmp/java_result.json" \
     -python="../tmp/python_result.json" -go="../tmp/go_result.txt"
 
 cd $oldpwd