Benchmarking

ProtocolBuffers.build

@@ -15,6 +15,7 @@
   
   <property name="src"
             value="${project::get-base-directory()}/src" />
+
   
   <property name="tools-protoc" 
             value="${project::get-base-directory()}/lib/protoc.exe"
@@ -25,6 +26,11 @@
             value="${path::combine(project::get-base-directory(), 'dist')}"
             overwrite="false" />            
 
+  <!-- Directory to find test data -->
+  <property name="testdata-dir"
+            value="${path::combine(project::get-base-directory(), 'testdata')}"
+            overwrite="false" />
+            
   <!-- Base directory to find protos (core, C# options, tests) -->
   <property name="protos-dir"
             value="${path::combine(project::get-base-directory(), 'protos')}"
@@ -37,13 +43,16 @@
   
   <!-- Which version of protogen to use when regenerating source -->
   <property name="tools-protogen-config"
-            value="Debug"
+            value="${build-configuration}"
             overwrite="false" />
   
   <property name="tools-protogen" 
             value="${src}/ProtoGen/bin/${tools-protogen-config}/protogen.exe" 
             overwrite="false"/>
-  
+
+  <property name="tools-protobench" value="${src}/ProtoBench/bin/${build-configuration}/protobench.exe" 
+            overwrite="false"/>
+
   <target name="clean-build"
           description="Rebuilds all source and binaries, including distribution">
 
@@ -96,6 +105,7 @@
       <arg value="--proto_path=${protos-dir}" />
       <arg value="--descriptor_set_out=compiled.pb" />
       <arg file="${protos-dir}/google/protobuf/benchmark.proto" />
+      <arg file="${protos-dir}/google/protobuf/benchmark_speed.proto" />
       <arg file="${protos-dir}/google/protobuf/descriptor.proto" />
       <arg file="${protos-dir}/google/protobuf/csharp_options.proto" />
       <arg file="${protos-dir}/google/protobuf/unittest.proto" />
@@ -142,10 +152,11 @@
     <copy todir="${src}/ProtoBench">
       <fileset basedir="${tmp-dir}">
         <include name="BenchmarkProtoFile.cs" />
+        <include name="BenchmarkSpeedProtoFile.cs" />
       </fileset>
     </copy>
   </target>
-      
+
   <target name="build"
           description="Builds all C# code">
     <msbuild project="${src}/ProtocolBuffers.sln">
@@ -165,7 +176,17 @@
   
   <target name="perf-test"
           description="Runs all performance tests">
-    <fail message="Performance tests not implemented yet" />
+    <exec program="${tools-protobench}"
+          workingdir="${testdata-dir}">
+      <arg value="Google.ProtocolBuffers.BenchmarkProtos.Message1,ProtoBench" />
+      <arg value="benchmark_message1.dat" />
+      <arg value="Google.ProtocolBuffers.BenchmarkProtos.SpeedMessage1,ProtoBench" />
+      <arg value="benchmark_message1.dat" />
+      <arg value="Google.ProtocolBuffers.BenchmarkProtos.Message3,ProtoBench" />
+      <arg value="benchmark_message3.dat" />
+      <arg value="Google.ProtocolBuffers.BenchmarkProtos.SpeedMessage3,ProtoBench" />
+      <arg value="benchmark_message3.dat" />
+    </exec>    
   </target>
   
   <target name="dist" 

protos/google/protobuf/benchmark.proto

@@ -4,8 +4,6 @@ option (google.protobuf.csharp_file_options).umbrella_classname = "BenchmarkProt
 
 package proto2.benchmark.v2_api;
 
-option optimize_for = SPEED;
-
 message Message1 {
   required string field1 = 1;
   optional string field9 = 9;

protos/google/protobuf/benchmark_speed.proto

+import "google/protobuf/csharp_options.proto";
+option (google.protobuf.csharp_file_options).namespace = "Google.ProtocolBuffers.BenchmarkProtos";
+option (google.protobuf.csharp_file_options).umbrella_classname = "BenchmarkSpeedProtoFile";
+
+package proto2.benchmark.v2_api;
+
+option optimize_for = SPEED;
+
+message SpeedMessage1 {
+  required string field1 = 1;
+  optional string field9 = 9;
+  optional string field18 = 18;
+  optional bool field80 = 80 [default=false];
+  optional bool field81 = 81 [default=true];
+  required int32 field2 = 2;
+  required int32 field3 = 3;
+  optional int32 field280 = 280;
+  optional int32 field6 = 6 [default=0];
+  optional int64 field22 = 22;
+  optional string field4 = 4;
+  repeated fixed64 field5 = 5;
+  optional bool field59 = 59 [default=false];
+  optional string field7 = 7;
+  optional int32 field16 = 16;
+  optional int32 field130 = 130 [default=0];
+  optional bool field12 = 12 [default=true];
+  optional bool field17 = 17 [default=true];
+  optional bool field13 = 13 [default=true];
+  optional bool field14 = 14 [default=true];
+  optional int32 field104 = 104 [default=0];
+  optional int32 field100 = 100 [default=0];
+  optional int32 field101 = 101 [default=0];
+  optional string field102 = 102;
+  optional string field103 = 103;
+  optional int32 field29 = 29 [default=0];
+  optional bool field30 = 30 [default=false];
+  optional int32 field60 = 60 [default=-1];
+  optional int32 field271 = 271 [default=-1];
+  optional int32 field272 = 272 [default=-1];
+  optional int32 field150 = 150;
+  optional int32 field23 = 23 [default=0];
+  optional bool field24 = 24 [default=false];
+  optional int32 field25 = 25 [default=0];
+  optional SpeedMessage2 field15 = 15;
+  optional bool field78 = 78;
+  optional int32 field67 = 67 [default=0];
+  optional int32 field68 = 68;
+  optional int32 field128 = 128 [default=0];
+  optional string field129 = 129 [default="xxxxxxxxxxxxxxxxxxxxx"];
+  optional int32 field131 = 131 [default=0];
+}
+
+message SpeedMessage2 {
+  optional int32 field1 = 1 [default=0];
+  optional int32 field2 = 2 [default=0];
+  optional int32 field3 = 3 [default=0];
+  optional string field15 = 15;
+  optional bool field12 = 12 [default=true];
+  optional int64 field13 = 13;
+  optional int64 field14 = 14;
+  optional int32 field16 = 16;
+  optional int32 field19 = 19 [default=2];
+  optional bool field20  = 20 [default=true];
+  optional bool field28 = 28 [default=true];
+  optional fixed64 field21 = 21;
+  optional int32 field22 = 22;
+  optional bool field23 = 23 [ default=false ];
+  optional bool field206 = 206 [default=false];
+  optional fixed32 field203 = 203;
+  optional int32 field204 = 204;
+  optional string field205 = 205;
+  optional uint64 field207 = 207;
+  optional uint64 field300 = 300;
+}
+
+message SpeedMessage3 {
+  optional string field1 = 1;
+  optional int64 field3 = 3;
+  optional int64 field4 = 4;
+  optional int64 field30 = 30;
+  optional bool field75  = 75 [default=false];
+  optional string field6 = 6;
+  optional bytes field2 = 2;
+  optional int32 field21 = 21 [default=0];
+  optional int32 field71 = 71;
+  optional float field25 = 25;
+  optional int32 field109 = 109 [default=0];
+  optional int32 field210 = 210 [default=0];
+  optional int32 field211 = 211 [default=0];
+  optional int32 field212 = 212 [default=0];
+  optional int32 field213 = 213 [default=0];
+  optional int32 field216 = 216 [default=0];
+  optional int32 field217 = 217 [default=0];
+  optional int32 field218 = 218 [default=0];
+  optional int32 field220 = 220 [default=0];
+  optional int32 field221 = 221 [default=0];
+  optional float field222 = 222 [default=0.0];
+  optional int32 field63 = 63;
+
+  repeated group Group1 = 10 {
+    required float field11 = 11;
+    optional float field26 = 26;
+    optional string field12 = 12;
+    optional string field13 = 13;
+    repeated string field14 = 14;
+    required uint64 field15 = 15;
+    optional int32 field5 = 5;
+    optional string field27 = 27;
+    optional int32 field28 = 28;
+    optional string field29 = 29;
+    optional string field16 = 16;
+    repeated string field22 = 22;
+    repeated int32 field73 = 73;
+    optional int32 field20 = 20 [default=0];
+    optional string field24 = 24;
+    optional SpeedMessage4 field31 = 31;
+  }
+  repeated string field128 = 128;
+  optional int64 field131 = 131;
+  repeated string field127 = 127;
+  optional int32 field129 = 129;
+  repeated int64 field130 = 130;
+  optional bool field205 = 205 [default=false];
+  optional bool field206 = 206 [default=false];
+}
+
+message SpeedMessage4 {
+  optional float field1 = 1;
+  optional float field2 = 2;
+  optional float field3 = 3 [default=0.0];
+  optional bool field4 = 4;
+  optional bool field5 = 5;
+  optional bool field6 = 6 [default=true];
+  optional bool field7 = 7 [default=false];
+  optional float field8 = 8;
+  optional bool field9 = 9;
+  optional float field10 = 10;
+  optional int64 field11 = 11;
+}

src/ProtoBench/Program.cs

@@ -16,49 +16,67 @@ namespace ProtoBench {
     delegate void Action();
 
     public static int Main(string[] args) {
-      if (args.Length != 2) {
-        Console.Error.WriteLine("Usage: ProtoBecnh <descriptor type name> <input data>");
+      if (args.Length < 2 || (args.Length % 2) != 0) {
+        Console.Error.WriteLine("Usage: ProtoBench <descriptor type name> <input data>");
         Console.Error.WriteLine("The descriptor type name is the fully-qualified message name,");
         Console.Error.WriteLine("including assembly - e.g. Google.ProtocolBuffers.BenchmarkProtos.Message1,ProtoBench");
+        Console.Error.WriteLine("(You can specify multiple pairs of descriptor type name and input data.)");
         return 1;
       }
+      bool success = true;
+      for (int i = 0; i < args.Length; i += 2) {
+        success &= RunTest(args[i], args[i + 1]);
+      }
+      return success ? 0 : 1;
+    }
+
+    /// <summary>
+    /// Runs a single test. Error messages are displayed to Console.Error, and the return value indicates
+    /// general success/failure.
+    /// </summary>
+    public static bool RunTest(string typeName, string file) {
+      Console.WriteLine("Benchmarking {0} with file {1}", typeName, file);
       IMessage defaultMessage;
       try {
-        defaultMessage = MessageUtil.GetDefaultMessage(args[0]);
+        defaultMessage = MessageUtil.GetDefaultMessage(typeName);
       } catch (ArgumentException e) {
         Console.Error.WriteLine(e.Message);
-        return 1;
+        return false;
       }
       try {
-        IBuilder builder = defaultMessage.WeakCreateBuilderForType();
-        byte[] inputData = File.ReadAllBytes(args[1]);
+        byte[] inputData = File.ReadAllBytes(file);
+        MemoryStream inputStream = new MemoryStream(inputData);
         ByteString inputString = ByteString.CopyFrom(inputData);
         IMessage sampleMessage = defaultMessage.WeakCreateBuilderForType().WeakMergeFrom(inputString).WeakBuild();
         sampleMessage.ToByteString();
-        Console.WriteLine("Benchmarking {0} with file {1}", sampleMessage.GetType().Name, args[1]);
         Benchmark("Serialize to byte string", inputData.Length, () => sampleMessage.ToByteString());
         Benchmark("Serialize to byte array", inputData.Length, () => sampleMessage.ToByteArray());
         Benchmark("Serialize to memory stream", inputData.Length, () => sampleMessage.WriteTo(new MemoryStream()));
+        Benchmark("Deserialize from byte string", inputData.Length, () =>
+          defaultMessage.WeakCreateBuilderForType()
+              .WeakMergeFrom(inputString)
+              .WeakBuild()
+        );
         Benchmark("Deserialize from byte array", inputData.Length, () =>
           defaultMessage.WeakCreateBuilderForType()
               .WeakMergeFrom(CodedInputStream.CreateInstance(inputData))
               .WeakBuild()
         );
-        Benchmark("Deserialize from byte array", inputData.Length, () =>
+        Benchmark("Deserialize from memory stream", inputData.Length, () =>
           defaultMessage.WeakCreateBuilderForType()
-              .WeakMergeFrom(inputString)
+              .WeakMergeFrom(CodedInputStream.CreateInstance(inputStream))
               .WeakBuild()
         );
-        return 0;
+        return true;
       } catch (Exception e) {
         Console.Error.WriteLine("Error: {0}", e.Message);
         Console.Error.WriteLine();
         Console.Error.WriteLine("Detailed exception information: {0}", e);
-        return 1;
+        return false;
       }
     }
 
-    private static void Benchmark(string name, int dataSize, Action action) {
+    private static void Benchmark(string name, long dataSize, Action action) {
       // Make sure it's JITted
       action();
       // Run it progressively more times until we've got a reasonable sample

src/ProtoBench/ProtoBench.csproj

@@ -39,6 +39,7 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="BenchmarkProtoFile.cs" />
+    <Compile Include="BenchmarkSpeedProtoFile.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
   </ItemGroup>