Merge pull request #4014 from BSBandme/JavaCaliper

Add caliper supported to java benchmark

benchmarks/Makefile.am

@@ -20,7 +20,7 @@ benchmarks_protoc_inputs_proto2 =                                          \
 	datasets/google_message4/benchmark_message4_3.proto
 
 make_tmp_dir:
-	mkdir -p 'tmp'
+	mkdir -p 'tmp/java/src/main/java'
 	touch make_tmp_dir
 
 if USE_EXTERNAL_PROTOC
@@ -39,11 +39,11 @@ else
 # relative to srcdir, which may not be the same as the current directory when
 # building out-of-tree.
 protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs)
-	oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) )
+	oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java $(benchmarks_protoc_inputs) )
 	touch protoc_middleman
 
 protoc_middleman2:  make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs_proto2) $(well_known_type_protoc_inputs)
-	oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2) )
+	oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java $(benchmarks_protoc_inputs_proto2) )
 	touch protoc_middleman2
 	
 endif
@@ -100,6 +100,17 @@ $(benchmarks_protoc_outputs_header): protoc_middleman
 $(benchmarks_protoc_outputs_proto2): protoc_middleman2
 $(benchmarks_protoc_outputs_proto2_header): protoc_middleman2
 
+initialize_submodule:
+	oldpwd=`pwd`
+	cd $(top_srcdir)/third_party
+	git submodule update --init -r
+	cd $(top_srcdir)/third_party/benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make
+	cd $$oldpwd
+	touch initialize_submodule
+	
+$(top_srcdir)/src/libprotobuf.la: initialize_submodule
+$(top_srcdir)/third_party/benchmark/src/libbenchmark.a: initialize_submodule
+
 AM_CXXFLAGS = $(NO_OPT_CXXFLAGS) $(PROTOBUF_OPT_FLAG) -Wall -Wwrite-strings -Woverloaded-virtual -Wno-sign-compare
 
 bin_PROGRAMS = cpp-benchmark
@@ -110,32 +121,33 @@ cpp_benchmark_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir) -I$(top_srcdir)/third_p
 # Explicit deps because BUILT_SOURCES are only done before a "make all/check"
 # so a direct "make test_cpp" could fail if parallel enough.
 # See: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html#Recording-Dependencies-manually
-cpp_benchmark-cpp_benchmark.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header)
+cpp_benchmark-cpp_benchmark.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a
 nodist_cpp_benchmark_SOURCES =                                             \
 	$(benchmarks_protoc_outputs)                                             \
 	$(benchmarks_protoc_outputs_proto2)                                      \
 	$(benchmarks_protoc_outputs_proto2_header)                               \
 	$(benchmarks_protoc_outputs_header)
 
-cpp: protoc_middleman protoc_middleman2 cpp-benchmark
+cpp: protoc_middleman protoc_middleman2 cpp-benchmark initialize_submodule
 	./cpp-benchmark $(all_data)
 
 ############ CPP RULES END ############
 
 ############# JAVA RULES ##############
 
-javac_middleman: ProtoBench.java protoc_middleman protoc_middleman2
-	jar=`ls $(top_srcdir)/java/util/target/*.jar` &&                         \
-	jar1=`ls $(top_srcdir)/java/core/target/*.jar` &&                        \
-	javac -d tmp -classpath ../java/target/classes:$$jar:$$jar1 ProtoBench.java $(benchmark_java_protoc_outputs_proto2) $(benchmarks_java_protoc_outputs) `find tmp -type f -name "*.java"`
+java_benchmark_testing_files =                                      \
+	java/src/main/java/com/google/protobuf/ProtoBench.java            \
+	java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java
+
+javac_middleman: $(java_benchmark_testing_files) protoc_middleman protoc_middleman2
+	cp -r java tmp && cd tmp/java &&  mvn clean compile assembly:single
+	cd ../..
 	@touch javac_middleman
 
 java-benchmark: javac_middleman 
 	@echo "Writing shortcut script java-benchmark..."
 	@echo '#! /bin/sh' > java-benchmark
-	@jar=`ls $(top_srcdir)/java/util/target/*.jar` &&                        \
-	jar1=`ls $(top_srcdir)/java/core/target/*.jar` &&                        \
-	echo java -classpath tmp:../java/target/classes:$$jar:$$jar1 com.google.protocolbuffers.ProtoBench '$$@' >> java-benchmark
+	@echo 'java -cp '"tmp/java/target/*.jar"' com.google.protobuf.ProtoBench $$@' >> java-benchmark
 	@chmod +x java-benchmark
 
 java: protoc_middleman protoc_middleman2 java-benchmark
@@ -151,6 +163,7 @@ CLEANFILES =                                                               \
 	$(benchmarks_protoc_outputs_header)                                      \
 	$(benchmarks_protoc_outputs_proto2)                                      \
 	$(benchmarks_protoc_outputs_proto2_header)                               \
+	initialize_submodule                                                     \
 	make_tmp_dir                                                             \
 	protoc_middleman                                                         \
 	protoc_middleman2                                                        \

benchmarks/README.md

@@ -5,7 +5,36 @@ This directory contains benchmarking schemas and data sets that you
 can use to test a variety of performance scenarios against your
 protobuf language runtime.
 
-The schema for the datasets is described in `benchmarks.proto`.
+## Prerequisite
+
+First, you need to follow the instruction in the root directory's README to 
+build your language's protobuf, then:
+
+### CPP
+You need to install [cmake](https://cmake.org/) before building the benchmark.
+
+We are using [google/benchmark](https://github.com/google/benchmark) as the 
+benchmark tool for testing cpp. This will be automaticly made during build the 
+cpp benchmark.
+
+### JAVA
+We're using maven to build the java benchmarks, which is the same as to build 
+the Java protobuf. There're no other tools need to install. We're using 
+[google/caliper](https://github.com/google/caliper) as benchmark tool, which 
+can be automaticly included by maven.
+
+### Big data
+
+There's some optional big testing data which is not included in the directory initially, you need to 
+run the following command to download the testing data:
+
+```
+$ ./download_data.sh 
+```
+
+After doing this the big data file will automaticly generated in the benchmark directory.  
+
+## Run instructions
 
 To run all the benchmark dataset:
 
@@ -26,22 +55,27 @@ To run a specific dataset:
 For java:
 
 ```
-$ make java
-$ ./java-benchmark $(specific generated dataset file name)
+$ make java-benchmark
+$ ./java-benchmark $(specific generated dataset file name) [-- $(caliper option)]
 ```
 
 For cpp:
 
 ```
-$ make cpp
+$ make cpp-benchmark
 $ ./cpp-benchmark $(specific generated dataset file name)
 ```
 
+## Benchmark datasets
+
 Each data set is in the format of benchmarks.proto:
+
 1. name is the benchmark dataset's name.
 2. message_name is the benchmark's message type full name (including package and message name)
 3. payload is the list of raw data.
 
+The schema for the datasets is described in `benchmarks.proto`.
+
 Benchmark likely want to run several benchmarks against each data set (parse,
 serialize, possibly JSON, possibly using different APIs, etc).
 

benchmarks/java/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" 
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  
+  <modelVersion>4.0.0</modelVersion>
+  <artifactId>protobuf-java-benchmark</artifactId>
+  <groupId>com.google.protobuf</groupId>
+  <version>1.0.0</version>
+  <name>Protocol Buffers [Benchmark]</name>
+  <description>The benchmark tools for Protobuf Java.</description>
+  
+  <dependencies>
+    <dependency>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java</artifactId>
+      <version>3.5.0</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.caliper</groupId>
+      <artifactId>caliper</artifactId>
+      <version>1.0-beta-2</version>
+    </dependency>
+  </dependencies>
+  
+  <build>
+    <pluginManagement>
+      <plugins>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-assembly-plugin</artifactId>
+          <version>2.4.1</version>
+          <configuration>
+            <!-- get all project dependencies -->
+            <descriptorRefs>
+              <descriptorRef>jar-with-dependencies</descriptorRef>
+            </descriptorRefs>
+            <!-- MainClass in mainfest make a executable jar -->
+            <archive>
+              <manifest>
+                <mainClass>com.mkyong.core.utils.App</mainClass>
+              </manifest>
+            </archive>
+          </configuration>
+          <executions>
+            <execution>
+              <id>make-assembly</id>
+              <!-- bind to the packaging phase -->
+              <phase>package</phase>
+              <goals>
+                <goal>single</goal>
+              </goals>
+            </execution>
+          </executions>
+        </plugin>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-compiler-plugin</artifactId>
+          <version>3.5.1</version>
+          <configuration>
+            <source>1.8</source>
+            <target>1.8</target>
+          </configuration>
+        </plugin>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-jar-plugin</artifactId>
+          <version>2.5</version>
+          <configuration>
+            <archive>
+              <manifest>
+                <addClasspath>true</addClasspath>
+                <mainClass>com.google.protocolbuffers.ProtoBench</mainClass>
+              </manifest>
+            </archive>
+          </configuration>
+        </plugin>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-source-plugin</artifactId>
+          <version>2.4</version>
+          <executions>
+            <execution>
+              <id>attach-sources</id>
+              <goals>
+                <goal>jar-no-fork</goal>
+              </goals>
+            </execution>
+          </executions>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+  </build>
+</project>
+

benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java

+
+package com.google.protobuf;
+
+import com.google.caliper.BeforeExperiment;
+import com.google.caliper.Benchmark;
+import com.google.caliper.Param;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.ExtensionRegistry;
+import com.google.protobuf.Message;
+import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class ProtoBenchCaliper {
+  public enum BenchmarkMessageType {
+    GOOGLE_MESSAGE1_PROTO3 {
+      @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
+      @Override
+      Message getDefaultInstance() {
+        return com.google.protobuf.benchmarks.BenchmarkMessage1Proto3.GoogleMessage1
+            .getDefaultInstance();
+      }
+    },
+    GOOGLE_MESSAGE1_PROTO2 {
+      @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
+      @Override
+      Message getDefaultInstance() {
+        return com.google.protobuf.benchmarks.BenchmarkMessage1Proto2.GoogleMessage1
+            .getDefaultInstance();
+      }
+    },
+    GOOGLE_MESSAGE2 {
+      @Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
+      @Override
+      Message getDefaultInstance() {
+        return com.google.protobuf.benchmarks.BenchmarkMessage2.GoogleMessage2.getDefaultInstance();
+      }
+    },
+    GOOGLE_MESSAGE3 {
+      @Override
+      ExtensionRegistry getExtensionRegistry() {
+        ExtensionRegistry extensions = ExtensionRegistry.newInstance();
+        benchmarks.google_message3.BenchmarkMessage38.registerAllExtensions(extensions);
+        benchmarks.google_message3.BenchmarkMessage37.registerAllExtensions(extensions);
+        benchmarks.google_message3.BenchmarkMessage36.registerAllExtensions(extensions);
+        benchmarks.google_message3.BenchmarkMessage35.registerAllExtensions(extensions);
+        benchmarks.google_message3.BenchmarkMessage34.registerAllExtensions(extensions);
+        benchmarks.google_message3.BenchmarkMessage33.registerAllExtensions(extensions);
+        benchmarks.google_message3.BenchmarkMessage32.registerAllExtensions(extensions);
+        benchmarks.google_message3.BenchmarkMessage31.registerAllExtensions(extensions);
+        benchmarks.google_message3.BenchmarkMessage3.registerAllExtensions(extensions);
+        return extensions;
+      }
+      @Override
+      Message getDefaultInstance() {
+        return benchmarks.google_message3.BenchmarkMessage3.GoogleMessage3.getDefaultInstance();
+      }
+    },
+    GOOGLE_MESSAGE4 {
+      @Override
+      ExtensionRegistry getExtensionRegistry() {
+        ExtensionRegistry extensions = ExtensionRegistry.newInstance();
+        benchmarks.google_message4.BenchmarkMessage43.registerAllExtensions(extensions);
+        benchmarks.google_message4.BenchmarkMessage42.registerAllExtensions(extensions);
+        benchmarks.google_message4.BenchmarkMessage41.registerAllExtensions(extensions);
+        benchmarks.google_message4.BenchmarkMessage4.registerAllExtensions(extensions);
+        return extensions;
+      }
+      @Override
+      Message getDefaultInstance() {
+        return benchmarks.google_message4.BenchmarkMessage4.GoogleMessage4.getDefaultInstance();
+      }
+    };
+    
+    abstract ExtensionRegistry getExtensionRegistry();
+    abstract Message getDefaultInstance();
+  }
+  
+  @Param 
+  private BenchmarkMessageType benchmarkMessageType;
+  @Param
+  private String dataFile;
+  
+  private byte[] inputData;
+  private BenchmarkDataset benchmarkDataset;
+  private Message defaultMessage;
+  private ExtensionRegistry extensions;
+  private List<byte[]> inputDataList;
+  private List<ByteArrayInputStream> inputStreamList;
+  private List<ByteString> inputStringList;
+  private List<Message> sampleMessageList;
+  private int counter;
+  
+  @BeforeExperiment
+  void setUp() throws IOException {
+    inputData = ProtoBench.readAllBytes(dataFile);
+    benchmarkDataset = BenchmarkDataset.parseFrom(inputData);
+    defaultMessage = benchmarkMessageType.getDefaultInstance();
+    extensions = benchmarkMessageType.getExtensionRegistry();
+    inputDataList = new ArrayList<byte[]>();
+    inputStreamList = new ArrayList<ByteArrayInputStream>();
+    inputStringList = new ArrayList<ByteString>();
+    sampleMessageList = new ArrayList<Message>();
+    
+    for (int i = 0; i < benchmarkDataset.getPayloadCount(); i++) {
+      byte[] singleInputData = benchmarkDataset.getPayload(i).toByteArray();
+      inputDataList.add(benchmarkDataset.getPayload(i).toByteArray());
+      inputStreamList.add(new ByteArrayInputStream(
+          benchmarkDataset.getPayload(i).toByteArray()));
+      inputStringList.add(benchmarkDataset.getPayload(i));
+      sampleMessageList.add(
+          defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build());
+    }
+    
+    counter = 0;
+  }
+  
+  
+  @Benchmark
+  void serializeToByteString(int reps) throws IOException {
+    for (int i = 0; i < reps; i++) {
+      sampleMessageList.get(counter % sampleMessageList.size()).toByteString();
+      counter++;
+    }
+  }
+  
+  @Benchmark
+  void serializeToByteArray(int reps) throws IOException {
+    for (int i = 0; i < reps; i++) {
+      sampleMessageList.get(counter % sampleMessageList.size()).toByteArray();
+      counter++;
+    }
+  }
+  
+  @Benchmark
+  void serializeToMemoryStream(int reps) throws IOException {
+    for (int i = 0; i < reps; i++) {
+      ByteArrayOutputStream output = new ByteArrayOutputStream();
+      sampleMessageList.get(counter % sampleMessageList.size()).writeTo(output);
+      counter++;
+    }
+  }
+  
+  @Benchmark
+  void deserializeFromByteString(int reps) throws IOException {
+    for (int i = 0; i < reps; i++) {
+      defaultMessage
+        .newBuilderForType()
+        .mergeFrom(inputStringList.get(counter % inputStringList.size()), extensions)
+        .build();
+      counter++;
+    }
+  }
+  
+  @Benchmark
+  void deserializeFromByteArray(int reps) throws IOException {
+    for (int i = 0; i < reps; i++) {
+      defaultMessage
+        .newBuilderForType()
+        .mergeFrom(inputDataList.get(counter % inputDataList.size()), extensions)
+        .build();
+      counter++;
+    }
+  }
+  
+  @Benchmark
+  void deserializeFromMemoryStream(int reps) throws IOException {
+    for (int i = 0; i < reps; i++) {
+      defaultMessage
+        .newBuilderForType()
+        .mergeFrom(inputStreamList.get(counter % inputStreamList.size()), extensions)
+        .build();
+      inputStreamList.get(counter % inputStreamList.size()).reset();
+      counter++;
+    }
+  }
+}

tests.sh

@@ -44,9 +44,6 @@ build_cpp() {
   # appears to be missing it: https://github.com/travis-ci/travis-ci/issues/6996
   if [[ $(type cmake 2>/dev/null) ]]; then
     # Verify benchmarking code can build successfully.
-    git submodule init
-    git submodule update
-    cd third_party/benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ../..
     cd benchmarks && make cpp-benchmark && cd ..
   else
     echo ""