add unit test for nbench functionality (#2253)

src/ngraph/runtime/interpreter/int_backend.cpp

@@ -90,6 +90,10 @@ bool runtime::interpreter::INTBackend::call(shared_ptr<Function> function,
         throw runtime_error("compile() must be called before call().");
     }
     FunctionInstance& instance = fit->second;
+    if (!instance.m_is_compiled)
+    {
+        throw runtime_error("compile() must be called before call().");
+    }
 
     // convert inputs to HostTensor
     vector<void*> func_inputs;

src/tools/nbench/nbench.cpp

@@ -322,6 +322,7 @@ OPTIONS
     }
 
     vector<PerfShape> aggregate_perf_data;
+    int rc = 0;
     for (const string& model : models)
     {
         cout << "\n";
@@ -405,10 +406,12 @@ OPTIONS
         catch (ngraph::unsupported_op& ue)
         {
             cout << "Unsupported op '" << ue.what() << "' in model " << model << endl;
+            rc += 1;
         }
         catch (exception& e)
         {
             cout << "Exception caught on '" << model << "'\n" << e.what() << endl;
+            rc += 1;
         }
     }
 
@@ -421,5 +424,5 @@ OPTIONS
         print_results(aggregate_perf_data, timing_detail);
     }
 
-    return 0;
+    return rc;
 }

test/CMakeLists.txt

@@ -55,6 +55,7 @@ set(SRC
     serialize.cpp
     shape.cpp
     tensor.cpp
+    tools.cpp
     type_prop.cpp
     util.cpp
     uuid.cpp
@@ -200,6 +201,13 @@ if (NGRAPH_CPU_ENABLE)
     target_link_libraries(unit-test PRIVATE libmkldnn)
 endif()
 
+if (NGRAPH_TOOLS_ENABLE)
+    get_property(NBENCH_PATH TARGET nbench PROPERTY BINARY_DIR)
+    set(NBENCH "${NBENCH_PATH}/nbench")
+    target_compile_definitions(unit-test PRIVATE NBENCH_PATH="${NBENCH}")
+    add_dependencies(unit-test nbench)
+endif()
+
 if (NGRAPH_PLAIDML_ENABLE)
     target_link_libraries(unit-test PRIVATE plaidml_backend)
 endif()

test/tools.cpp

+//*****************************************************************************
+// Copyright 2017-2018 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#include <gtest/gtest.h>
+#include <sstream>
+
+#include "ngraph/cpio.hpp"
+#include "ngraph/file_util.hpp"
+#include "ngraph/log.hpp"
+
+using namespace ngraph;
+using namespace std;
+
+TEST(tools, nbench_functional)
+{
+    const string model = "mxnet/mnist_mlp_forward.json";
+    const string model_path = file_util::path_join(SERIALIZED_ZOO, model);
+
+    stringstream ss;
+
+    ss << NBENCH_PATH << " -f " << model_path << " -b INTERPRETER -i 2 -w 2";
+    auto cmd = ss.str();
+    auto f = popen(cmd.c_str(), "r");
+    if (f)
+    {
+        stringstream str;
+        char buffer[256];
+        while (!feof(f))
+        {
+            size_t count = fread(buffer, 1, sizeof(buffer), f);
+            string s = string(buffer, count);
+            str << s;
+        }
+        string output = str.str();
+        auto status = pclose(f);
+        ASSERT_EQ(status, 0) << output;
+    }
+    else
+    {
+        FAIL();
+    }
+}