Skip to content

P
protobuf
Commit 5e732e35 authored by Yilun Chong's avatar Yilun Chong
Browse files
Options

Add caliper supported to java benchmark

parent 9021f623
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 399 additions and 16 deletions
.gitmodules
View file @ 5e732e35
[submodule "third_party/benchmark"]
path = third_party/benchmark
url = https://github.com/google/benchmark.git
benchmarks/Makefile.am 100644 → 100755
View file @ 5e732e35
......@@ -20,7 +20,7 @@ benchmarks_protoc_inputs_proto2 = \
datasets/google_message4/benchmark_message4_3.proto
make_tmp_dir:
mkdir -p 'tmp'
mkdir -p 'tmp/java/src/main/java'
touch make_tmp_dir
if USE_EXTERNAL_PROTOC
......@@ -39,11 +39,11 @@ else
# relative to srcdir, which may not be the same as the current directory when
# building out-of-tree.
protoc_middleman: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs) $(well_known_type_protoc_inputs)
oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp $(benchmarks_protoc_inputs) )
oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java $(benchmarks_protoc_inputs) )
touch protoc_middleman
protoc_middleman2: make_tmp_dir $(top_srcdir)/src/protoc$(EXEEXT) $(benchmarks_protoc_inputs_proto2) $(well_known_type_protoc_inputs)
oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp $(benchmarks_protoc_inputs_proto2) )
oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. -I$(top_srcdir)/src --cpp_out=$$oldpwd --java_out=$$oldpwd/tmp/java/src/main/java $(benchmarks_protoc_inputs_proto2) )
touch protoc_middleman2
endif
......@@ -124,18 +124,19 @@ cpp: protoc_middleman protoc_middleman2 cpp-benchmark
############# JAVA RULES ##############
javac_middleman: ProtoBench.java protoc_middleman protoc_middleman2
jar=`ls $(top_srcdir)/java/util/target/*.jar` && \
jar1=`ls $(top_srcdir)/java/core/target/*.jar` && \
javac -d tmp -classpath ../java/target/classes:$$jar:$$jar1 ProtoBench.java $(benchmark_java_protoc_outputs_proto2) $(benchmarks_java_protoc_outputs) `find tmp -type f -name "*.java"`
java_benchmark_testing_files =
java/src/main/java/com/google/protocolbuffers/ProtoBench.java \
java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java
javac_middleman: $(java_benchmark_testing_files) protoc_middleman protoc_middleman2
cp -r java tmp && cd tmp/java && mvn clean compile assembly:single
cd ../..
@touch javac_middleman
java-benchmark: javac_middleman
@echo "Writing shortcut script java-benchmark..."
@echo '#! /bin/sh' > java-benchmark
@jar=`ls $(top_srcdir)/java/util/target/*.jar` && \
jar1=`ls $(top_srcdir)/java/core/target/*.jar` && \
echo java -classpath tmp:../java/target/classes:$$jar:$$jar1 com.google.protocolbuffers.ProtoBench '$$@' >> java-benchmark
@echo 'java -cp '"tmp/java/target/*.jar"' com.google.protocolbuffers.ProtoBench $$@' >> java-benchmark
@chmod +x java-benchmark
java: protoc_middleman protoc_middleman2 java-benchmark
......
benchmarks/README.md
View file @ 5e732e35
......@@ -7,6 +7,18 @@ protobuf language runtime.
The schema for the datasets is described in `benchmarks.proto`.
The benchmark is based on some submodules. To initialize the submodues:
For java:
```
$ ./initialize_submodule.sh java
```
For java:
```
$ ./initialize_submodule.sh cpp
```
To run all the benchmark dataset:
For java:
......@@ -26,17 +38,24 @@ To run a specific dataset:
For java:
```
$ make java
$ make java-benchmark
$ ./java-benchmark $(specific generated dataset file name)
```
For cpp:
```
$ make cpp
$ make cpp-benchmark
$ ./cpp-benchmark $(specific generated dataset file name)
```
There's some big testing data not included in the directory initially, you need to
run the following command to download the testing data:
```
$ ./download_data.sh
```
Each data set is in the format of benchmarks.proto:
1. name is the benchmark dataset's name.
2. message_name is the benchmark's message type full name (including package and message name)
......
benchmarks/initialize_submodule.sh 0 → 100755
View file @ 5e732e35
#! /bin/sh
oldpwd=`pwd`
cd "../third_party"
git submodule update --init -r
cd benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ..
cd "$oldpwd"
benchmarks/java/pom.xml 0 → 100755
View file @ 5e732e35
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright 2011 Google Inc.
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<!-- <parent>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-parent</artifactId>
<version>3.5.0</version>
</parent>
<packaging>pom</packaging> -->
<artifactId>protobuf-java-benchmark</artifactId>
<groupId>com.google.protocolbuffers</groupId>
<version>1.0.0</version>
<name>Protocol Buffers [Benchmark]</name>
<description>
The benchmark tools for Protobuf Java.
</description>
<dependencies>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>3.5.0</version>
</dependency>
<dependency>
<groupId>com.google.caliper</groupId>
<artifactId>caliper</artifactId>
<version>1.0-beta-2</version>
</dependency>
</dependencies>
<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.4.1</version>
<configuration>
<!-- get all project dependencies -->
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<!-- MainClass in mainfest make a executable jar -->
<archive>
<manifest>
<mainClass>com.mkyong.core.utils.App</mainClass>
</manifest>
</archive>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<!-- bind to the packaging phase -->
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.5.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.5</version>
<configuration>
<archive>
<manifest>
<addClasspath>true</addClasspath>
<mainClass>com.google.protocolbuffers.ProtoBench</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>2.4</version>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar-no-fork</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.3</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>all</shadedClassifierName>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<manifestEntries>
<Premain-Class>com.google.monitoring.runtime.instrumentation.AllocationInstrumenter</Premain-Class>
<Can-Redefine-Classes>true</Can-Redefine-Classes>
<Can-Retransform-Classes>true</Can-Retransform-Classes>
</manifestEntries>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</pluginManagement>
</build>
</project>
benchmarks/java/src/main/java/com/google/protocolbuffers/ProtoBenchCaliper.java 0 → 100755
View file @ 5e732e35
package com.google.protocolbuffers;
import com.google.caliper.BeforeExperiment;
import com.google.caliper.Benchmark;
import com.google.caliper.Param;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.ExtensionRegistry;
import com.google.protobuf.Message;
import com.google.protobuf.benchmarks.Benchmarks.BenchmarkDataset;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class ProtoBenchCaliper {
public enum BenchmarkMessageType {
GOOGLE_MESSAGE1_PROTO3 {
@Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
@Override
Message getDefaultInstance() {
return com.google.protobuf.benchmarks.BenchmarkMessage1Proto3.GoogleMessage1
.getDefaultInstance();
}
@Override String[] getSpecificCaliperOption() { return null; }
},
GOOGLE_MESSAGE1_PROTO2 {
@Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
@Override
Message getDefaultInstance() {
return com.google.protobuf.benchmarks.BenchmarkMessage1Proto2.GoogleMessage1
.getDefaultInstance();
}
@Override String[] getSpecificCaliperOption() { return null; }
},
GOOGLE_MESSAGE2 {
@Override ExtensionRegistry getExtensionRegistry() { return ExtensionRegistry.newInstance(); }
@Override
Message getDefaultInstance() {
return com.google.protobuf.benchmarks.BenchmarkMessage2.GoogleMessage2.getDefaultInstance();
}
@Override String[] getSpecificCaliperOption() { return null; }
},
GOOGLE_MESSAGE3 {
@Override
ExtensionRegistry getExtensionRegistry() {
ExtensionRegistry extensions = ExtensionRegistry.newInstance();
benchmarks.google_message3.BenchmarkMessage38.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage37.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage36.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage35.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage34.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage33.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage32.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage31.registerAllExtensions(extensions);
benchmarks.google_message3.BenchmarkMessage3.registerAllExtensions(extensions);
return extensions;
}
@Override
Message getDefaultInstance() {
return benchmarks.google_message3.BenchmarkMessage3.GoogleMessage3.getDefaultInstance();
}
@Override
String[] getSpecificCaliperOption() {
String[] opt = new String[1];
opt[0] = "-Cinstrument.runtime.options.timingInterval=3000ms";
return opt;
}
},
GOOGLE_MESSAGE4 {
@Override
ExtensionRegistry getExtensionRegistry() {
ExtensionRegistry extensions = ExtensionRegistry.newInstance();
benchmarks.google_message4.BenchmarkMessage43.registerAllExtensions(extensions);
benchmarks.google_message4.BenchmarkMessage42.registerAllExtensions(extensions);
benchmarks.google_message4.BenchmarkMessage41.registerAllExtensions(extensions);
benchmarks.google_message4.BenchmarkMessage4.registerAllExtensions(extensions);
return extensions;
}
@Override
Message getDefaultInstance() {
return benchmarks.google_message4.BenchmarkMessage4.GoogleMessage4.getDefaultInstance();
}
@Override
String[] getSpecificCaliperOption() {
String[] opt = new String[1];
opt[0] = "-Cinstrument.runtime.options.timingInterval=3000ms";
return opt;
}
};
abstract ExtensionRegistry getExtensionRegistry();
abstract Message getDefaultInstance();
abstract String[] getSpecificCaliperOption();
}
@Param
private BenchmarkMessageType benchmarkMessageType;
@Param
private String dataFile;
private byte[] inputData;
private BenchmarkDataset benchmarkDataset;
private Message defaultMessage;
private ExtensionRegistry extensions;
private List<byte[]> inputDataList;
private List<ByteArrayInputStream> inputStreamList;
private List<ByteString> inputStringList;
private List<Message> sampleMessageList;
private int counter;
private FileOutputStream devNull;
private CodedOutputStream reuseDevNull;
@BeforeExperiment
void setUp() throws IOException {
inputData = ProtoBench.readAllBytes(dataFile);
benchmarkDataset = BenchmarkDataset.parseFrom(inputData);
defaultMessage = benchmarkMessageType.getDefaultInstance();
extensions = benchmarkMessageType.getExtensionRegistry();
inputDataList = new ArrayList<byte[]>();
inputStreamList = new ArrayList<ByteArrayInputStream>();
inputStringList = new ArrayList<ByteString>();
sampleMessageList = new ArrayList<Message>();
for (int i = 0; i < benchmarkDataset.getPayloadCount(); i++) {
byte[] singleInputData = benchmarkDataset.getPayload(i).toByteArray();
inputDataList.add(benchmarkDataset.getPayload(i).toByteArray());
inputStreamList.add(new ByteArrayInputStream(
benchmarkDataset.getPayload(i).toByteArray()));
inputStringList.add(benchmarkDataset.getPayload(i));
sampleMessageList.add(
defaultMessage.newBuilderForType().mergeFrom(singleInputData, extensions).build());
}
devNull = null;
reuseDevNull = null;
try {
devNull = new FileOutputStream("/dev/null");
reuseDevNull = CodedOutputStream.newInstance(devNull);
} catch (FileNotFoundException e) {
// ignore: this is probably Windows, where /dev/null does not exist
}
counter = 0;
}
@Benchmark
void serializeToByteString(int reps) throws IOException {
for (int i = 0; i < reps; i++) {
sampleMessageList.get(counter % sampleMessageList.size()).toByteString();
counter++;
}
}
@Benchmark
void serializeToByteArray(int reps) throws IOException {
for (int i = 0; i < reps; i++) {
sampleMessageList.get(counter % sampleMessageList.size()).toByteArray();
counter++;
}
}
@Benchmark
void serializeToMemoryStream(int reps) throws IOException {
for (int i = 0; i < reps; i++) {
ByteArrayOutputStream output = new ByteArrayOutputStream();
sampleMessageList.get(counter % sampleMessageList.size()).writeTo(output);
counter++;
}
}
@Benchmark
void deserializeFromByteString(int reps) throws IOException {
for (int i = 0; i < reps; i++) {
defaultMessage
.newBuilderForType()
.mergeFrom(inputStringList.get(counter % inputStringList.size()), extensions)
.build();
counter++;
}
}
@Benchmark
void deserializeFromByteArray(int reps) throws IOException {
for (int i = 0; i < reps; i++) {
defaultMessage
.newBuilderForType()
.mergeFrom(inputDataList.get(counter % inputDataList.size()), extensions)
.build();
counter++;
}
}
@Benchmark
void deserializeFromMemoryStream(int reps) throws IOException {
for (int i = 0; i < reps; i++) {
defaultMessage
.newBuilderForType()
.mergeFrom(inputStreamList.get(counter % inputStreamList.size()), extensions)
.build();
inputStreamList.get(counter % inputStreamList.size()).reset();
counter++;
}
}
}
tests.sh
View file @ 5e732e35
......@@ -44,10 +44,7 @@ build_cpp() {
# appears to be missing it: https://github.com/travis-ci/travis-ci/issues/6996
if [[ $(type cmake 2>/dev/null) ]]; then
# Verify benchmarking code can build successfully.
git submodule init
git submodule update
cd third_party/benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ../..
cd benchmarks && make cpp-benchmark && cd ..
cd benchmarks && ./initialize_submodule.sh cpp && make cpp-benchmark && cd ..
else
echo ""
echo "WARNING: Skipping validation of the bench marking code, cmake isn't installed."
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment