Skip to content

P
protobuf
Commit ff9c9e4d authored by jonathan.skeet's avatar jonathan.skeet
Browse files
Options

Initial benchmarking.

parent 1556623f
Hide whitespace changes
Inline Side-by-side
Showing with 467 additions and 0 deletions
benchmarks/ProtoBench.java 0 → 100644
View file @ ff9c9e4d
package com.google.protocolbuffers;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.lang.reflect.Method;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.Message;
public class ProtoBench {
private static long MIN_SAMPLE_TIME_MS = 2 * 1000;
private static long TARGET_TIME_MS = 30 * 1000;
private ProtoBench() {
// Prevent instantiation
}
public static void main(String[] args) {
if (args.length < 2 || (args.length % 2) != 0) {
System.err.println("Usage: ProtoBench <descriptor type name> <input data>");
System.err.println("The descriptor type name is the fully-qualified message name,");
System.err.println("e.g. com.google.protocolbuffers.benchmark.Message1");
System.err.println("(You can specify multiple pairs of descriptor type name and input data.)");
System.exit(1);
}
boolean success = true;
for (int i = 0; i < args.length; i += 2) {
success &= runTest(args[i], args[i + 1]);
}
System.exit(success ? 0 : 1);
}
/**
* Runs a single test. Error messages are displayed to stderr, and the return value
* indicates general success/failure.
*/
public static boolean runTest(String type, String file) {
System.out.println("Benchmarking " + type + " with file " + file);
final Message defaultMessage;
try {
Class<?> clazz = Class.forName(type);
Method method = clazz.getDeclaredMethod("getDefaultInstance");
defaultMessage = (Message) method.invoke(null);
} catch (Exception e) {
// We want to do the same thing with all exceptions. Not generally nice,
// but this is slightly different.
System.err.println("Unable to get default message for " + type);
return false;
}
try {
final byte[] inputData = readAllBytes(file);
final ByteArrayInputStream inputStream = new ByteArrayInputStream(inputData);
final ByteString inputString = ByteString.copyFrom(inputData);
final Message sampleMessage = defaultMessage.newBuilderForType().mergeFrom(inputString).build();
benchmark("Serialize to byte string", inputData.length, new Action() {
public void execute() { sampleMessage.toByteString(); }
});
benchmark("Serialize to byte array", inputData.length, new Action() {
public void execute() { sampleMessage.toByteArray(); }
});
benchmark("Serialize to memory stream", inputData.length, new Action() {
public void execute() throws IOException {
sampleMessage.writeTo(new ByteArrayOutputStream());
}
});
benchmark("Deserialize from byte string", inputData.length, new Action() {
public void execute() throws IOException {
defaultMessage.newBuilderForType().mergeFrom(inputString).build();
}
});
benchmark("Deserialize from byte array", inputData.length, new Action() {
public void execute() throws IOException {
defaultMessage.newBuilderForType()
.mergeFrom(CodedInputStream.newInstance(inputData)).build();
}
});
benchmark("Deserialize from memory stream", inputData.length, new Action() {
public void execute() throws IOException {
defaultMessage.newBuilderForType()
.mergeFrom(CodedInputStream.newInstance(inputStream)).build();
inputStream.reset();
}
});
System.out.println();
return true;
} catch (Exception e) {
System.err.println("Error: " + e.getMessage());
System.err.println("Detailed exception information:");
e.printStackTrace(System.err);
return false;
}
}
private static void benchmark(String name, long dataSize, Action action) throws IOException {
// Make sure it's JITted "reasonably" hard before running the first progress test
for (int i=0; i < 100; i++) {
action.execute();
}
// Run it progressively more times until we've got a reasonable sample
int iterations = 1;
long elapsed = timeAction(action, iterations);
while (elapsed < MIN_SAMPLE_TIME_MS) {
iterations *= 2;
elapsed = timeAction(action, iterations);
}
// Upscale the sample to the target time. Do this in floating point arithmetic
// to avoid overflow issues.
iterations = (int) ((TARGET_TIME_MS / (double) elapsed) * iterations);
elapsed = timeAction(action, iterations);
System.out.println(name + ": " + iterations + " iterations in "
+ (elapsed/1000f) + "s; "
+ (iterations * dataSize) / (elapsed * 1024 * 1024 / 1000f)
+ "MB/s");
}
private static long timeAction(Action action, int iterations) throws IOException {
long start = System.currentTimeMillis();
for (int i = 0; i < iterations; i++) {
action.execute();
}
long end = System.currentTimeMillis();
return end - start;
}
private static byte[] readAllBytes(String filename) throws IOException {
RandomAccessFile file = new RandomAccessFile(new File(filename), "r");
byte[] content = new byte[(int) file.length()];
file.readFully(content);
return content;
}
/**
* Interface used to capture a single action to benchmark.
*/
interface Action {
void execute() throws IOException;
}
}
benchmarks/google_size.proto 0 → 100644
View file @ ff9c9e4d
package benchmarks;
option java_outer_classname = "GoogleSize";
option optimize_for = CODE_SIZE;
message SizeMessage1 {
required string field1 = 1;
optional string field9 = 9;
optional string field18 = 18;
optional bool field80 = 80 [default=false];
optional bool field81 = 81 [default=true];
required int32 field2 = 2;
required int32 field3 = 3;
optional int32 field280 = 280;
optional int32 field6 = 6 [default=0];
optional int64 field22 = 22;
optional string field4 = 4;
repeated fixed64 field5 = 5;
optional bool field59 = 59 [default=false];
optional string field7 = 7;
optional int32 field16 = 16;
optional int32 field130 = 130 [default=0];
optional bool field12 = 12 [default=true];
optional bool field17 = 17 [default=true];
optional bool field13 = 13 [default=true];
optional bool field14 = 14 [default=true];
optional int32 field104 = 104 [default=0];
optional int32 field100 = 100 [default=0];
optional int32 field101 = 101 [default=0];
optional string field102 = 102;
optional string field103 = 103;
optional int32 field29 = 29 [default=0];
optional bool field30 = 30 [default=false];
optional int32 field60 = 60 [default=-1];
optional int32 field271 = 271 [default=-1];
optional int32 field272 = 272 [default=-1];
optional int32 field150 = 150;
optional int32 field23 = 23 [default=0];
optional bool field24 = 24 [default=false];
optional int32 field25 = 25 [default=0];
optional SizeMessage1SubMessage field15 = 15;
optional bool field78 = 78;
optional int32 field67 = 67 [default=0];
optional int32 field68 = 68;
optional int32 field128 = 128 [default=0];
optional string field129 = 129 [default="xxxxxxxxxxxxxxxxxxxxx"];
optional int32 field131 = 131 [default=0];
}
message SizeMessage1SubMessage {
optional int32 field1 = 1 [default=0];
optional int32 field2 = 2 [default=0];
optional int32 field3 = 3 [default=0];
optional string field15 = 15;
optional bool field12 = 12 [default=true];
optional int64 field13 = 13;
optional int64 field14 = 14;
optional int32 field16 = 16;
optional int32 field19 = 19 [default=2];
optional bool field20 = 20 [default=true];
optional bool field28 = 28 [default=true];
optional fixed64 field21 = 21;
optional int32 field22 = 22;
optional bool field23 = 23 [ default=false ];
optional bool field206 = 206 [default=false];
optional fixed32 field203 = 203;
optional int32 field204 = 204;
optional string field205 = 205;
optional uint64 field207 = 207;
optional uint64 field300 = 300;
}
message SizeMessage2 {
optional string field1 = 1;
optional int64 field3 = 3;
optional int64 field4 = 4;
optional int64 field30 = 30;
optional bool field75 = 75 [default=false];
optional string field6 = 6;
optional bytes field2 = 2;
optional int32 field21 = 21 [default=0];
optional int32 field71 = 71;
optional float field25 = 25;
optional int32 field109 = 109 [default=0];
optional int32 field210 = 210 [default=0];
optional int32 field211 = 211 [default=0];
optional int32 field212 = 212 [default=0];
optional int32 field213 = 213 [default=0];
optional int32 field216 = 216 [default=0];
optional int32 field217 = 217 [default=0];
optional int32 field218 = 218 [default=0];
optional int32 field220 = 220 [default=0];
optional int32 field221 = 221 [default=0];
optional float field222 = 222 [default=0.0];
optional int32 field63 = 63;
repeated group Group1 = 10 {
required float field11 = 11;
optional float field26 = 26;
optional string field12 = 12;
optional string field13 = 13;
repeated string field14 = 14;
required uint64 field15 = 15;
optional int32 field5 = 5;
optional string field27 = 27;
optional int32 field28 = 28;
optional string field29 = 29;
optional string field16 = 16;
repeated string field22 = 22;
repeated int32 field73 = 73;
optional int32 field20 = 20 [default=0];
optional string field24 = 24;
optional SizeMessage2GroupedMessage field31 = 31;
}
repeated string field128 = 128;
optional int64 field131 = 131;
repeated string field127 = 127;
optional int32 field129 = 129;
repeated int64 field130 = 130;
optional bool field205 = 205 [default=false];
optional bool field206 = 206 [default=false];
}
message SizeMessage2GroupedMessage {
optional float field1 = 1;
optional float field2 = 2;
optional float field3 = 3 [default=0.0];
optional bool field4 = 4;
optional bool field5 = 5;
optional bool field6 = 6 [default=true];
optional bool field7 = 7 [default=false];
optional float field8 = 8;
optional bool field9 = 9;
optional float field10 = 10;
optional int64 field11 = 11;
}
benchmarks/google_speed.proto 0 → 100644
View file @ ff9c9e4d
package benchmarks;
option java_outer_classname = "GoogleSpeed";
option optimize_for = SPEED;
message SpeedMessage1 {
required string field1 = 1;
optional string field9 = 9;
optional string field18 = 18;
optional bool field80 = 80 [default=false];
optional bool field81 = 81 [default=true];
required int32 field2 = 2;
required int32 field3 = 3;
optional int32 field280 = 280;
optional int32 field6 = 6 [default=0];
optional int64 field22 = 22;
optional string field4 = 4;
repeated fixed64 field5 = 5;
optional bool field59 = 59 [default=false];
optional string field7 = 7;
optional int32 field16 = 16;
optional int32 field130 = 130 [default=0];
optional bool field12 = 12 [default=true];
optional bool field17 = 17 [default=true];
optional bool field13 = 13 [default=true];
optional bool field14 = 14 [default=true];
optional int32 field104 = 104 [default=0];
optional int32 field100 = 100 [default=0];
optional int32 field101 = 101 [default=0];
optional string field102 = 102;
optional string field103 = 103;
optional int32 field29 = 29 [default=0];
optional bool field30 = 30 [default=false];
optional int32 field60 = 60 [default=-1];
optional int32 field271 = 271 [default=-1];
optional int32 field272 = 272 [default=-1];
optional int32 field150 = 150;
optional int32 field23 = 23 [default=0];
optional bool field24 = 24 [default=false];
optional int32 field25 = 25 [default=0];
optional SpeedMessage1SubMessage field15 = 15;
optional bool field78 = 78;
optional int32 field67 = 67 [default=0];
optional int32 field68 = 68;
optional int32 field128 = 128 [default=0];
optional string field129 = 129 [default="xxxxxxxxxxxxxxxxxxxxx"];
optional int32 field131 = 131 [default=0];
}
message SpeedMessage1SubMessage {
optional int32 field1 = 1 [default=0];
optional int32 field2 = 2 [default=0];
optional int32 field3 = 3 [default=0];
optional string field15 = 15;
optional bool field12 = 12 [default=true];
optional int64 field13 = 13;
optional int64 field14 = 14;
optional int32 field16 = 16;
optional int32 field19 = 19 [default=2];
optional bool field20 = 20 [default=true];
optional bool field28 = 28 [default=true];
optional fixed64 field21 = 21;
optional int32 field22 = 22;
optional bool field23 = 23 [ default=false ];
optional bool field206 = 206 [default=false];
optional fixed32 field203 = 203;
optional int32 field204 = 204;
optional string field205 = 205;
optional uint64 field207 = 207;
optional uint64 field300 = 300;
}
message SpeedMessage2 {
optional string field1 = 1;
optional int64 field3 = 3;
optional int64 field4 = 4;
optional int64 field30 = 30;
optional bool field75 = 75 [default=false];
optional string field6 = 6;
optional bytes field2 = 2;
optional int32 field21 = 21 [default=0];
optional int32 field71 = 71;
optional float field25 = 25;
optional int32 field109 = 109 [default=0];
optional int32 field210 = 210 [default=0];
optional int32 field211 = 211 [default=0];
optional int32 field212 = 212 [default=0];
optional int32 field213 = 213 [default=0];
optional int32 field216 = 216 [default=0];
optional int32 field217 = 217 [default=0];
optional int32 field218 = 218 [default=0];
optional int32 field220 = 220 [default=0];
optional int32 field221 = 221 [default=0];
optional float field222 = 222 [default=0.0];
optional int32 field63 = 63;
repeated group Group1 = 10 {
required float field11 = 11;
optional float field26 = 26;
optional string field12 = 12;
optional string field13 = 13;
repeated string field14 = 14;
required uint64 field15 = 15;
optional int32 field5 = 5;
optional string field27 = 27;
optional int32 field28 = 28;
optional string field29 = 29;
optional string field16 = 16;
repeated string field22 = 22;
repeated int32 field73 = 73;
optional int32 field20 = 20 [default=0];
optional string field24 = 24;
optional SpeedMessage2GroupedMessage field31 = 31;
}
repeated string field128 = 128;
optional int64 field131 = 131;
repeated string field127 = 127;
optional int32 field129 = 129;
repeated int64 field130 = 130;
optional bool field205 = 205 [default=false];
optional bool field206 = 206 [default=false];
}
message SpeedMessage2GroupedMessage {
optional float field1 = 1;
optional float field2 = 2;
optional float field3 = 3 [default=0.0];
optional bool field4 = 4;
optional bool field5 = 5;
optional bool field6 = 6 [default=true];
optional bool field7 = 7 [default=false];
optional float field8 = 8;
optional bool field9 = 9;
optional float field10 = 10;
optional int64 field11 = 11;
}
benchmarks/readme.txt 0 → 100644
View file @ ff9c9e4d
Contents
--------
This folder contains three kinds of file:
- Code, such as ProtoBench.java, to build the benchmarking framework.
- Protocol buffer definitions (.proto files)
- Sample data files
If we end up with a lot of different benchmarks it may be worth
separating these out info different directories, but while there are
so few they might as well all be together.
Running a benchmark (Java)
--------------------------
1) Build protoc and the Java protocol buffer library. The examples
below assume a jar file (protobuf.jar) has been built and copied
into this directory.
2) Build ProtoBench:
$ javac -d tmp -cp protobuf.jar ProtoBench.java
3) Generate code for the relevant benchmark protocol buffer, e.g.
$ protoc --java_out=tmp google_size.proto google_speed.proto
4) Build the generated code, e.g.
$ cd tmp
$ javac -d . -cp ../protobuf.jar benchmarks/*.java
5) Run the test. Arguments are given in pairs - the first argument
is the descriptor type; the second is the filename. For example:
$ java -cp .;../protobuf.jar com.google.protocolbuffers.ProtoBench
benchmarks.GoogleSize$SizeMessage1 ../google_message1.dat
benchmarks.GoogleSpeed$SpeedMessage1 ../google_message1.dat
benchmarks.GoogleSize$SizeMessage2 ../google_message2.dat
benchmarks.GoogleSpeed$SpeedMessage2 ../google_message2.dat
6) Wait! Each test runs for around 30 seconds, and there are 6 tests
per class/data combination. The above command would therefore take
about 12 minutes to run.
Benchmarks available
--------------------
From Google:
google_size.proto and google_speed.proto, messages
google_message1.dat and google_message2.dat. The proto files are
equivalent, but optimized differently.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment