Skip to content

P
protobuf
Unverified Commit 43caa38d authored by Yilun Chong's avatar Yilun Chong Committed by GitHub
Browse files
Options

Merge pull request #4014 from BSBandme/JavaCaliper 

Add caliper supported to java benchmark
parents 156161df aca6c155
Show whitespace changes
Inline Side-by-side
Showing with 386 additions and 235 deletions
benchmarks/Makefile.am 100644 → 100755
View file @ 43caa38d
...@@ -20,7 +20,7 @@ benchmarks_protoc_inputs_proto2 = \ ...@@ -20,7 +20,7 @@ benchmarks_protoc_inputs_proto2 = \
datasets/google_message4/benchmark_message4_3.proto datasets/google_message4/benchmark_message4_3.proto
make_tmp_dir: make_tmp_dir:
mkdir -p 'tmp' mkdir -p 'tmp/java/src/main/java'
touch make_tmp_dir touch make_tmp_dir
if USE_EXTERNAL_PROTOC if USE_EXTERNAL_PROTOC
...@@ -39,11 +39,11 @@ else ...@@ -39,11 +39,11 @@ else
# relative to srcdir, which may not be the same as the current directory when # relative to srcdir, which may not be the same as the current directory when
# building out-of-tree. # building out-of-tree.
protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs) protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs)
oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) ) oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java $(benchmarks_protoc_inputs) )
touch protoc_middleman touch protoc_middleman
protoc_middleman2: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs_proto2) $(well_known_type_protoc_inputs) protoc_middleman2: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs_proto2) $(well_known_type_protoc_inputs)
oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2) ) oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java $(benchmarks_protoc_inputs_proto2) )
touch protoc_middleman2 touch protoc_middleman2
endif endif
...@@ -100,6 +100,17 @@ $(benchmarks_protoc_outputs_header): protoc_middleman ...@@ -100,6 +100,17 @@ $(benchmarks_protoc_outputs_header): protoc_middleman
$(benchmarks_protoc_outputs_proto2): protoc_middleman2 $(benchmarks_protoc_outputs_proto2): protoc_middleman2
$(benchmarks_protoc_outputs_proto2_header): protoc_middleman2 $(benchmarks_protoc_outputs_proto2_header): protoc_middleman2
initialize_submodule:
oldpwd=`pwd`
cd $(top_srcdir)/third_party
git submodule update --init -r
cd $(top_srcdir)/third_party/benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make
cd $$oldpwd
touch initialize_submodule
$(top_srcdir)/src/libprotobuf.la: initialize_submodule
$(top_srcdir)/third_party/benchmark/src/libbenchmark.a: initialize_submodule
AM_CXXFLAGS = $(NO_OPT_CXXFLAGS) $(PROTOBUF_OPT_FLAG) -Wall -Wwrite-strings -Woverloaded-virtual -Wno-sign-compare AM_CXXFLAGS = $(NO_OPT_CXXFLAGS) $(PROTOBUF_OPT_FLAG) -Wall -Wwrite-strings -Woverloaded-virtual -Wno-sign-compare
bin_PROGRAMS = cpp-benchmark bin_PROGRAMS = cpp-benchmark
...@@ -110,32 +121,33 @@ cpp_benchmark_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir) -I$(top_srcdir)/third_p ...@@ -110,32 +121,33 @@ cpp_benchmark_CPPFLAGS = -I$(top_srcdir)/src -I$(srcdir) -I$(top_srcdir)/third_p
# Explicit deps because BUILT_SOURCES are only done before a "make all/check" # Explicit deps because BUILT_SOURCES are only done before a "make all/check"
# so a direct "make test_cpp" could fail if parallel enough. # so a direct "make test_cpp" could fail if parallel enough.
# See: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html#Recording-Dependencies-manually # See: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html#Recording-Dependencies-manually
cpp_benchmark-cpp_benchmark.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) cpp_benchmark-cpp_benchmark.$(OBJEXT): $(benchmarks_protoc_outputs) $(benchmarks_protoc_outputs_proto2) $(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_proto2_header) $(top_srcdir)/src/libprotobuf.la $(top_srcdir)/third_party/benchmark/src/libbenchmark.a
nodist_cpp_benchmark_SOURCES = \ nodist_cpp_benchmark_SOURCES = \
$(benchmarks_protoc_outputs) \ $(benchmarks_protoc_outputs) \
$(benchmarks_protoc_outputs_proto2) \ $(benchmarks_protoc_outputs_proto2) \
$(benchmarks_protoc_outputs_proto2_header) \ $(benchmarks_protoc_outputs_proto2_header) \
$(benchmarks_protoc_outputs_header) $(benchmarks_protoc_outputs_header)
cpp: protoc_middleman protoc_middleman2 cpp-benchmark cpp: protoc_middleman protoc_middleman2 cpp-benchmark initialize_submodule
./cpp-benchmark $(all_data) ./cpp-benchmark $(all_data)
############ CPP RULES END ############ ############ CPP RULES END ############
############# JAVA RULES ############## ############# JAVA RULES ##############
javac_middleman: ProtoBench.java protoc_middleman protoc_middleman2 java_benchmark_testing_files = \
jar=`ls $(top_srcdir)/java/util/target/*.jar` && \ java/src/main/java/com/google/protobuf/ProtoBench.java \
jar1=`ls $(top_srcdir)/java/core/target/*.jar` && \ java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java
javac -d tmp -classpath ../java/target/classes:$$jar:$$jar1 ProtoBench.java $(benchmark_java_protoc_outputs_proto2) $(benchmarks_java_protoc_outputs) `find tmp -type f -name "*.java"`
javac_middleman: $(java_benchmark_testing_files) protoc_middleman protoc_middleman2
cp -r java tmp && cd tmp/java && mvn clean compile assembly:single
cd ../..
@touch javac_middleman @touch javac_middleman
java-benchmark: javac_middleman java-benchmark: javac_middleman
@echo "Writing shortcut script java-benchmark..." @echo "Writing shortcut script java-benchmark..."
@echo '#! /bin/sh' > java-benchmark @echo '#! /bin/sh' > java-benchmark
@jar=`ls $(top_srcdir)/java/util/target/*.jar` && \ @echo 'java -cp '"tmp/java/target/*.jar"' com.google.protobuf.ProtoBench $$@' >> java-benchmark
jar1=`ls $(top_srcdir)/java/core/target/*.jar` && \
echo java -classpath tmp:../java/target/classes:$$jar:$$jar1 com.google.protocolbuffers.ProtoBench '$$@' >> java-benchmark
@chmod +x java-benchmark @chmod +x java-benchmark
java: protoc_middleman protoc_middleman2 java-benchmark java: protoc_middleman protoc_middleman2 java-benchmark
...@@ -151,6 +163,7 @@ CLEANFILES = \ ...@@ -151,6 +163,7 @@ CLEANFILES = \
$(benchmarks_protoc_outputs_header) \ $(benchmarks_protoc_outputs_header) \
$(benchmarks_protoc_outputs_proto2) \ $(benchmarks_protoc_outputs_proto2) \
$(benchmarks_protoc_outputs_proto2_header) \ $(benchmarks_protoc_outputs_proto2_header) \
initialize_submodule \
make_tmp_dir \ make_tmp_dir \
protoc_middleman \ protoc_middleman \
protoc_middleman2 \ protoc_middleman2 \
......
benchmarks/README.md
View file @ 43caa38d
...@@ -5,7 +5,36 @@ This directory contains benchmarking schemas and data sets that you ...@@ -5,7 +5,36 @@ This directory contains benchmarking schemas and data sets that you
can use to test a variety of performance scenarios against your can use to test a variety of performance scenarios against your
protobuf language runtime. protobuf language runtime.
The schema for the datasets is described in `benchmarks.proto`. ## Prerequisite
First, you need to follow the instruction in the root directory's README to
build your language's protobuf, then:
### CPP
You need to install [cmake](https://cmake.org/) before building the benchmark.
We are using [google/benchmark](https://github.com/google/benchmark) as the
benchmark tool for testing cpp. This will be automaticly made during build the
cpp benchmark.
### JAVA
We're using maven to build the java benchmarks, which is the same as to build
the Java protobuf. There're no other tools need to install. We're using
[google/caliper](https://github.com/google/caliper) as benchmark tool, which
can be automaticly included by maven.
### Big data
There's some optional big testing data which is not included in the directory initially, you need to
run the following command to download the testing data:
```
$ ./download_data.sh
```
After doing this the big data file will automaticly generated in the benchmark directory.
## Run instructions
To run all the benchmark dataset: To run all the benchmark dataset:
...@@ -26,22 +55,27 @@ To run a specific dataset: ...@@ -26,22 +55,27 @@ To run a specific dataset:
For java: For java:
``` ```
$ make java $ make java-benchmark
$ ./java-benchmark $(specific generated dataset file name) $ ./java-benchmark $(specific generated dataset file name) [-- $(caliper option)]
``` ```
For cpp: For cpp:
``` ```
$ make cpp $ make cpp-benchmark
$ ./cpp-benchmark $(specific generated dataset file name) $ ./cpp-benchmark $(specific generated dataset file name)
``` ```
## Benchmark datasets
Each data set is in the format of benchmarks.proto: Each data set is in the format of benchmarks.proto:
1. name is the benchmark dataset's name. 1. name is the benchmark dataset's name.
2. message_name is the benchmark's message type full name (including package and message name) 2. message_name is the benchmark's message type full name (including package and message name)
3. payload is the list of raw data. 3. payload is the list of raw data.
The schema for the datasets is described in `benchmarks.proto`.
Benchmark likely want to run several benchmarks against each data set (parse, Benchmark likely want to run several benchmarks against each data set (parse,
serialize, possibly JSON, possibly using different APIs, etc). serialize, possibly JSON, possibly using different APIs, etc).
......
benchmarks/java/pom.xml 0 → 100755
View file @ 43caa38d
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>protobuf-java-benchmark</artifactId>
<groupId>com.google.protobuf</groupId>
<version>1.0.0</version>
<name>Protocol Buffers [Benchmark]</name>
<description>The benchmark tools for Protobuf Java.</description>
<dependencies>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>3.5.0</version>
</dependency>
<dependency>
<groupId>com.google.caliper</groupId>
<artifactId>caliper</artifactId>
<version>1.0-beta-2</version>
</dependency>
</dependencies>
<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4.1</version>
<configuration>
<!-- get all project dependencies -->
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<!-- MainClass in mainfest make a executable jar -->
<archive>
<manifest>
<mainClass>com.mkyong.core.utils.App</mainClass>
</manifest>
</archive>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<!-- bind to the packaging phase -->
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.5.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.5</version>
<configuration>
<archive>
<manifest>
<addClasspath>true</addClasspath>
<mainClass>com.google.protocolbuffers.ProtoBench</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>2.4</version>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar-no-fork</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</pluginManagement>
</build>
</project>
benchmarks/ProtoBench.java → benchmarks/java/src/main/java/com/google/protobuf/ProtoBench.java 100644 → 100755
View file @ 43caa38d
...@@ -28,8 +28,13 @@ ...@@ -28,8 +28,13 @@
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package com.google.protocolbuffers;
package com.google.protobuf;
import com.google.caliper.BeforeExperiment;
import com.google.caliper.Benchmark;
import com.google.caliper.Param;
import com.google.caliper.runner.CaliperMain;
import com.google.protobuf.ByteString; import com.google.protobuf.ByteString;
import com.google.protobuf.CodedInputStream; import com.google.protobuf.CodedInputStream;
import com.google.protobuf.CodedOutputStream; import com.google.protobuf.CodedOutputStream;
...@@ -42,14 +47,13 @@ import java.io.File; ...@@ -42,14 +47,13 @@ import java.io.File;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.PrintWriter;
import java.io.RandomAccessFile; import java.io.RandomAccessFile;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
public class ProtoBench {
private static final long MIN_SAMPLE_TIME_MS = 1 * 1000; public class ProtoBench {
private static final long TARGET_TIME_MS = 5 * 1000;
private ProtoBench() { private ProtoBench() {
// Prevent instantiation // Prevent instantiation
...@@ -68,63 +72,13 @@ public class ProtoBench { ...@@ -68,63 +72,13 @@ public class ProtoBench {
System.exit(success ? 0 : 1); System.exit(success ? 0 : 1);
} }
public static ExtensionRegistry getExtensionsRegistry(BenchmarkDataset benchmarkDataset) {
ExtensionRegistry extensions = ExtensionRegistry.newInstance();
if (benchmarkDataset.getMessageName().equals("benchmarks.google_message3.GoogleMessage3")) {
benchmarks.google_message3.BenchmarkMessage38.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage37.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage36.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage35.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage34.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage33.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage32.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage31.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage3.registerAllExtensions(extensions);
} else if (benchmarkDataset.getMessageName().equals(
"benchmarks.google_message4.GoogleMessage4")) {
benchmarks.google_message4.BenchmarkMessage43.registerAllExtensions(extensions);
benchmarks.google_message4.BenchmarkMessage42.registerAllExtensions(extensions);
benchmarks.google_message4.BenchmarkMessage41.registerAllExtensions(extensions);
benchmarks.google_message4.BenchmarkMessage4.registerAllExtensions(extensions);
}
return extensions;
}
/** /**
* Return an message instance for one specific dataset, and register the extensions for that * Runs a single test with specific test data. Error messages are displayed to stderr,
* message. * and the return value indicates general success/failure.
*/
public static Message registerBenchmarks(BenchmarkDataset benchmarkDataset) {
if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) {
return com.google.protobuf.benchmarks.BenchmarkMessage1Proto3.GoogleMessage1
.getDefaultInstance();
} else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage1")) {
return com.google.protobuf.benchmarks.BenchmarkMessage1Proto2.GoogleMessage1
.getDefaultInstance();
} else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage2")) {
return com.google.protobuf.benchmarks.BenchmarkMessage2.GoogleMessage2.getDefaultInstance();
} else if (benchmarkDataset.getMessageName().
equals("benchmarks.google_message3.GoogleMessage3")) {
return benchmarks.google_message3.BenchmarkMessage3.GoogleMessage3.getDefaultInstance();
} else if (benchmarkDataset.getMessageName().
equals("benchmarks.google_message4.GoogleMessage4")) {
return benchmarks.google_message4.BenchmarkMessage4.GoogleMessage4.getDefaultInstance();
} else {
return null;
}
}
/**
* Runs a single test. Error messages are displayed to stderr, and the return value indicates
* general success/failure.
*/ */
public static boolean runTest(String file) { public static boolean runTest(String file) {
final Message defaultMessage; byte[] inputData;
BenchmarkDataset benchmarkDataset; BenchmarkDataset benchmarkDataset;
ExtensionRegistry extensions;
final byte[] inputData;
try { try {
inputData = readAllBytes(file); inputData = readAllBytes(file);
benchmarkDataset = BenchmarkDataset.parseFrom(inputData); benchmarkDataset = BenchmarkDataset.parseFrom(inputData);
...@@ -132,134 +86,20 @@ public class ProtoBench { ...@@ -132,134 +86,20 @@ public class ProtoBench {
System.err.println("Unable to get input data"); System.err.println("Unable to get input data");
return false; return false;
} }
defaultMessage = registerBenchmarks(benchmarkDataset);
extensions = getExtensionsRegistry(benchmarkDataset); List<String> argsList = getCaliperOption(benchmarkDataset);
if (defaultMessage == null) { if (argsList == null) {
System.err.println("Unable to get default message " + benchmarkDataset.getMessageName()); System.err.println("Unable to get default message " + benchmarkDataset.getMessageName());
return false; return false;
} }
System.out.println("Benchmarking " + benchmarkDataset.getMessageName() + " with file " + file); argsList.add("-DdataFile=" + file);
argsList.add("com.google.protobuf.ProtoBenchCaliper");
try { try {
List<byte[]> inputDataList = new ArrayList<byte[]>(); String args[] = new String[argsList.size()];
List<ByteArrayInputStream> inputStreamList = new ArrayList<ByteArrayInputStream>(); argsList.toArray(args);
List<ByteString> inputStringList = new ArrayList<ByteString>(); CaliperMain.exitlessMain(args,
List<Message> sampleMessageList = new ArrayList<Message>(); new PrintWriter(System.out, true), new PrintWriter(System.err, true));
for (int i = 0; i < benchmarkDataset.getPayloadCount(); i++) {
byte[] singleInputData = benchmarkDataset.getPayload(i).toByteArray();
inputDataList.add(benchmarkDataset.getPayload(i).toByteArray());
inputStreamList.add(new ByteArrayInputStream(benchmarkDataset.getPayload(i).toByteArray()));
inputStringList.add(benchmarkDataset.getPayload(i));
sampleMessageList.add(
defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build());
}
FileOutputStream devNullTemp = null;
CodedOutputStream reuseDevNullTemp = null;
try {
devNullTemp = new FileOutputStream("/dev/null");
reuseDevNullTemp = CodedOutputStream.newInstance(devNullTemp);
} catch (FileNotFoundException e) {
// ignore: this is probably Windows, where /dev/null does not exist
}
final FileOutputStream devNull = devNullTemp;
final CodedOutputStream reuseDevNull = reuseDevNullTemp;
benchmark(
"Serialize to byte string",
inputData.length,
new Action() {
public void execute() {
for (int i = 0; i < sampleMessageList.size(); i++) {
sampleMessageList.get(i).toByteString();
}
}
});
benchmark(
"Serialize to byte array",
inputData.length,
new Action() {
public void execute() {
for (int i = 0; i < sampleMessageList.size(); i++) {
sampleMessageList.get(i).toByteString();
}
}
});
benchmark(
"Serialize to memory stream",
inputData.length,
new Action() {
public void execute() throws IOException {
ByteArrayOutputStream output = new ByteArrayOutputStream();
for (int i = 0; i < sampleMessageList.size(); i++) {
sampleMessageList.get(i).writeTo(output);
}
}
});
if (devNull != null) {
benchmark(
"Serialize to /dev/null with FileOutputStream",
inputData.length,
new Action() {
public void execute() throws IOException {
for (int i = 0; i < sampleMessageList.size(); i++) {
sampleMessageList.get(i).writeTo(devNull);
}
}
});
benchmark(
"Serialize to /dev/null reusing FileOutputStream",
inputData.length,
new Action() {
public void execute() throws IOException {
for (int i = 0; i < sampleMessageList.size(); i++) {
sampleMessageList.get(i).writeTo(reuseDevNull);
reuseDevNull.flush(); // force the write to the OutputStream
}
}
});
}
benchmark(
"Deserialize from byte string",
inputData.length,
new Action() {
public void execute() throws IOException {
for (int i = 0; i < inputStringList.size(); i++) {
defaultMessage
.newBuilderForType()
.mergeFrom(inputStringList.get(i), extensions)
.build();
}
}
});
benchmark(
"Deserialize from byte array",
inputData.length,
new Action() {
public void execute() throws IOException {
for (int i = 0; i < inputDataList.size(); i++) {
defaultMessage
.newBuilderForType()
.mergeFrom(CodedInputStream.newInstance(inputDataList.get(i)), extensions)
.build();
}
}
});
benchmark(
"Deserialize from memory stream",
inputData.length,
new Action() {
public void execute() throws IOException {
for (int i = 0; i < inputStreamList.size(); i++) {
defaultMessage
.newBuilderForType()
.mergeFrom(CodedInputStream.newInstance(inputStreamList.get(i)), extensions)
.build();
inputStreamList.get(i).reset();
}
}
});
System.out.println();
return true; return true;
} catch (Exception e) { } catch (Exception e) {
System.err.println("Error: " + e.getMessage()); System.err.println("Error: " + e.getMessage());
...@@ -269,51 +109,40 @@ public class ProtoBench { ...@@ -269,51 +109,40 @@ public class ProtoBench {
} }
} }
private static void benchmark(String name, long dataSize, Action action) throws IOException {
// Make sure it's JITted "reasonably" hard before running the first progress test
for (int i=0; i < 100; i++) {
action.execute();
}
// Run it progressively more times until we've got a reasonable sample private static List<String> getCaliperOption(final BenchmarkDataset benchmarkDataset) {
int iterations = 1; List<String> temp = new ArrayList<String>();
long elapsed = timeAction(action, iterations); if (benchmarkDataset.getMessageName().equals("benchmarks.proto3.GoogleMessage1")) {
while (elapsed < MIN_SAMPLE_TIME_MS) { temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE1_PROTO3");
iterations *= 2; } else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage1")) {
elapsed = timeAction(action, iterations); temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE1_PROTO2");
} else if (benchmarkDataset.getMessageName().equals("benchmarks.proto2.GoogleMessage2")) {
temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE2");
} else if (benchmarkDataset.getMessageName().
equals("benchmarks.google_message3.GoogleMessage3")) {
temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE3");
} else if (benchmarkDataset.getMessageName().
equals("benchmarks.google_message4.GoogleMessage4")) {
temp.add("-DbenchmarkMessageType=GOOGLE_MESSAGE4");
} else {
return null;
} }
// Upscale the sample to the target time. Do this in floating point arithmetic temp.add("-i");
// to avoid overflow issues. temp.add("runtime");
iterations = (int) ((TARGET_TIME_MS / (double) elapsed) * iterations); temp.add("-b");
elapsed = timeAction(action, iterations); String benchmarkNames = "serializeToByteString,serializeToByteArray,serializeToMemoryStream"
System.out.println(name + ": " + iterations + " iterations in " + ",deserializeFromByteString,deserializeFromByteArray,deserializeFromMemoryStream";
+ (elapsed/1000f) + "s; " temp.add(benchmarkNames);
+ (iterations * dataSize) / (elapsed * 1024 * 1024 / 1000f) temp.add("-Cinstrument.runtime.options.timingInterval=3000ms");
+ "MB/s");
}
private static long timeAction(Action action, int iterations) throws IOException { return temp;
System.gc();
long start = System.currentTimeMillis();
for (int i = 0; i < iterations; i++) {
action.execute();
}
long end = System.currentTimeMillis();
return end - start;
} }
private static byte[] readAllBytes(String filename) throws IOException { public static byte[] readAllBytes(String filename) throws IOException {
RandomAccessFile file = new RandomAccessFile(new File(filename), "r"); RandomAccessFile file = new RandomAccessFile(new File(filename), "r");
byte[] content = new byte[(int) file.length()]; byte[] content = new byte[(int) file.length()];
file.readFully(content); file.readFully(content);
return content; return content;
} }
/**
* Interface used to capture a single action to benchmark.
*/
interface Action {
void execute() throws IOException;
}
} }
benchmarks/java/src/main/java/com/google/protobuf/ProtoBenchCaliper.java 0 → 100755
View file @ 43caa38d
package com.google.protobuf;
import com.google.caliper.BeforeExperiment;
import com.google.caliper.Benchmark;
import com.google.caliper.Param;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.ExtensionRegistry;
import com.google.protobuf.Message;
import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class ProtoBenchCaliper {
public enum BenchmarkMessageType {
GOOGLE_MESSAGE1_PROTO3 {
@Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
@Override
Message getDefaultInstance() {
return com.google.protobuf.benchmarks.BenchmarkMessage1Proto3.GoogleMessage1
.getDefaultInstance();
}
},
GOOGLE_MESSAGE1_PROTO2 {
@Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
@Override
Message getDefaultInstance() {
return com.google.protobuf.benchmarks.BenchmarkMessage1Proto2.GoogleMessage1
.getDefaultInstance();
}
},
GOOGLE_MESSAGE2 {
@Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
@Override
Message getDefaultInstance() {
return com.google.protobuf.benchmarks.BenchmarkMessage2.GoogleMessage2.getDefaultInstance();
}
},
GOOGLE_MESSAGE3 {
@Override
ExtensionRegistry getExtensionRegistry() {
ExtensionRegistry extensions = ExtensionRegistry.newInstance();
benchmarks.google_message3.BenchmarkMessage38.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage37.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage36.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage35.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage34.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage33.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage32.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage31.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage3.registerAllExtensions(extensions);
return extensions;
}
@Override
Message getDefaultInstance() {
return benchmarks.google_message3.BenchmarkMessage3.GoogleMessage3.getDefaultInstance();
}
},
GOOGLE_MESSAGE4 {
@Override
ExtensionRegistry getExtensionRegistry() {
ExtensionRegistry extensions = ExtensionRegistry.newInstance();
benchmarks.google_message4.BenchmarkMessage43.registerAllExtensions(extensions);
benchmarks.google_message4.BenchmarkMessage42.registerAllExtensions(extensions);
benchmarks.google_message4.BenchmarkMessage41.registerAllExtensions(extensions);
benchmarks.google_message4.BenchmarkMessage4.registerAllExtensions(extensions);
return extensions;
}
@Override
Message getDefaultInstance() {
return benchmarks.google_message4.BenchmarkMessage4.GoogleMessage4.getDefaultInstance();
}
};
abstract ExtensionRegistry getExtensionRegistry();
abstract Message getDefaultInstance();
}
@Param
private BenchmarkMessageType benchmarkMessageType;
@Param
private String dataFile;
private byte[] inputData;
private BenchmarkDataset benchmarkDataset;
private Message defaultMessage;
private ExtensionRegistry extensions;
private List<byte[]> inputDataList;
private List<ByteArrayInputStream> inputStreamList;
private List<ByteString> inputStringList;
private List<Message> sampleMessageList;
private int counter;
@BeforeExperiment
void setUp() throws IOException {
inputData = ProtoBench.readAllBytes(dataFile);
benchmarkDataset = BenchmarkDataset.parseFrom(inputData);
defaultMessage = benchmarkMessageType.getDefaultInstance();
extensions = benchmarkMessageType.getExtensionRegistry();
inputDataList = new ArrayList<byte[]>();
inputStreamList = new ArrayList<ByteArrayInputStream>();
inputStringList = new ArrayList<ByteString>();
sampleMessageList = new ArrayList<Message>();
for (int i = 0; i < benchmarkDataset.getPayloadCount(); i++) {
byte[] singleInputData = benchmarkDataset.getPayload(i).toByteArray();
inputDataList.add(benchmarkDataset.getPayload(i).toByteArray());
inputStreamList.add(new ByteArrayInputStream(
benchmarkDataset.getPayload(i).toByteArray()));
inputStringList.add(benchmarkDataset.getPayload(i));
sampleMessageList.add(
defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build());
}
counter = 0;
}
@Benchmark
void serializeToByteString(int reps) throws IOException {
for (int i = 0; i < reps; i++) {
sampleMessageList.get(counter % sampleMessageList.size()).toByteString();
counter++;
}
}
@Benchmark
void serializeToByteArray(int reps) throws IOException {
for (int i = 0; i < reps; i++) {
sampleMessageList.get(counter % sampleMessageList.size()).toByteArray();
counter++;
}
}
@Benchmark
void serializeToMemoryStream(int reps) throws IOException {
for (int i = 0; i < reps; i++) {
ByteArrayOutputStream output = new ByteArrayOutputStream();
sampleMessageList.get(counter % sampleMessageList.size()).writeTo(output);
counter++;
}
}
@Benchmark
void deserializeFromByteString(int reps) throws IOException {
for (int i = 0; i < reps; i++) {
defaultMessage
.newBuilderForType()
.mergeFrom(inputStringList.get(counter % inputStringList.size()), extensions)
.build();
counter++;
}
}
@Benchmark
void deserializeFromByteArray(int reps) throws IOException {
for (int i = 0; i < reps; i++) {
defaultMessage
.newBuilderForType()
.mergeFrom(inputDataList.get(counter % inputDataList.size()), extensions)
.build();
counter++;
}
}
@Benchmark
void deserializeFromMemoryStream(int reps) throws IOException {
for (int i = 0; i < reps; i++) {
defaultMessage
.newBuilderForType()
.mergeFrom(inputStreamList.get(counter % inputStreamList.size()), extensions)
.build();
inputStreamList.get(counter % inputStreamList.size()).reset();
counter++;
}
}
}
tests.sh
View file @ 43caa38d
...@@ -44,9 +44,6 @@ build_cpp() { ...@@ -44,9 +44,6 @@ build_cpp() {
# appears to be missing it: https://github.com/travis-ci/travis-ci/issues/6996 # appears to be missing it: https://github.com/travis-ci/travis-ci/issues/6996
if [[ $(type cmake 2>/dev/null) ]]; then if [[ $(type cmake 2>/dev/null) ]]; then
# Verify benchmarking code can build successfully. # Verify benchmarking code can build successfully.
git submodule init
git submodule update
cd third_party/benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ../..
cd benchmarks && make cpp-benchmark && cd .. cd benchmarks && make cpp-benchmark && cd ..
else else
echo "" echo ""
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment