add interpreter nan check option (#368)

* add interpreter nan check option

* add unit test

src/ngraph/runtime/interpreter/int_call_frame.cpp

@@ -27,6 +27,7 @@ runtime::interpreter::INT_CallFrame::INT_CallFrame(shared_ptr<ExternalFunction>
     : m_external_function(external_function)
     , m_function(func)
     , m_emit_timing(std::getenv("NGRAPH_INTERPRETER_EMIT_TIMING") != nullptr)
+    , m_nan_check(std::getenv("NGRAPH_INTERPRETER_NAN_CHECK") != nullptr)
 {
 }
 
@@ -35,6 +36,10 @@ void runtime::interpreter::INT_CallFrame::call(
     const vector<shared_ptr<runtime::interpreter::INT_TensorView>>& input_tvs,
     const vector<shared_ptr<runtime::interpreter::INT_TensorView>>& output_tvs)
 {
+    if (m_nan_check)
+    {
+        perform_nan_check(input_tvs);
+    }
     unordered_map<descriptor::TensorView*, shared_ptr<runtime::interpreter::INT_TensorView>>
         tensor_map;
 
@@ -149,6 +154,10 @@ void runtime::interpreter::INT_CallFrame::call(
             stopwatch& timer = m_timer_map[op.get()];
             timer.stop();
         }
+        if (m_nan_check)
+        {
+            perform_nan_check(outputs, op.get());
+        }
 
         handle_output_alias(*op, output_alias_map, output_tvs);
 
@@ -306,3 +315,57 @@ vector<runtime::PerformanceCounter>
     }
     return rc;
 }
+
+void runtime::interpreter::INT_CallFrame::perform_nan_check(
+    const vector<shared_ptr<INT_TensorView>>& tvs, const Node* op)
+{
+    size_t arg_number = 1;
+    for (shared_ptr<INT_TensorView> tv : tvs)
+    {
+        const element::Type& type = tv->get_tensor().get_element_type();
+        if (type == element::f32)
+        {
+            const float* data = reinterpret_cast<float*>(tv->get_data_ptr());
+            for (size_t i = 0; i < tv->get_element_count(); i++)
+            {
+                if (isnan(data[i]))
+                {
+                    if (op)
+                    {
+                        throw runtime_error("nan found in op '" + op->get_name() + "' output");
+                    }
+                    else
+                    {
+                        throw runtime_error("nan found in function's input tensor number " +
+                                            to_string(arg_number));
+                    }
+                }
+            }
+        }
+        else if (type == element::f64)
+        {
+            const double* data = reinterpret_cast<double*>(tv->get_data_ptr());
+            for (size_t i = 0; i < tv->get_element_count(); i++)
+            {
+                if (isnan(data[i]))
+                {
+                    if (op)
+                    {
+                        throw runtime_error("nan found in op '" + op->get_name() + "' output");
+                    }
+                    else
+                    {
+                        throw runtime_error("nan found in function's input tensor number " +
+                                            to_string(arg_number));
+                    }
+                }
+            }
+        }
+        arg_number++;
+    }
+}
+
+void runtime::interpreter::INT_CallFrame::set_nan_check(bool value)
+{
+    m_nan_check = value;
+}

src/ngraph/runtime/interpreter/int_call_frame.hpp

@@ -117,6 +117,8 @@ public:
               const std::vector<std::shared_ptr<runtime::TensorView>>& outputs) override;
     std::vector<runtime::PerformanceCounter> get_performance_data() const override;
 
+    void set_nan_check(bool);
+
 private:
     /// @brief Invoke the function with tuples pre-expanded to their underlying
     /// tensor views.
@@ -132,9 +134,13 @@ private:
         const std::unordered_map<descriptor::TensorView*, std::vector<size_t>>& output_alias_map,
         const std::vector<std::shared_ptr<runtime::interpreter::INT_TensorView>>& output_tvs);
 
+    static void perform_nan_check(const std::vector<std::shared_ptr<INT_TensorView>>&,
+                                  const Node* op = nullptr);
+
     std::shared_ptr<ExternalFunction> m_external_function;
     std::shared_ptr<Function> m_function;
     bool m_emit_timing;
+    bool m_nan_check;
     std::unordered_map<const Node*, stopwatch> m_timer_map;
 
     void generate_calls(const element::Type& base_type,

test/CMakeLists.txt

@@ -22,6 +22,7 @@ include_directories(
     )
 
 set (SRC
+    backend_debug_api.cpp
     builder.cpp
     builder_autobroadcast.cpp
     builder_xla.cpp

test/backend_debug_api.cpp

+// ----------------------------------------------------------------------------
+// Copyright 2017 Nervana Systems Inc.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// ----------------------------------------------------------------------------
+
+#include <random>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "ngraph/log.hpp"
+#include "ngraph/ngraph.hpp"
+#include "ngraph/runtime/interpreter/int_call_frame.hpp"
+#include "util/test_tools.hpp"
+
+using namespace std;
+using namespace ngraph;
+
+TEST(INTERPRETER, nan_check_input)
+{
+    auto shape = Shape{4};
+    auto A = make_shared<op::Parameter>(element::f32, shape);
+    auto B = make_shared<op::Parameter>(element::f32, shape);
+    auto f = make_shared<Function>(make_shared<op::Divide>(A, B), op::Parameters{A, B});
+
+    auto manager = runtime::Manager::get("INTERPRETER");
+    auto external = manager->compile(f);
+    auto backend = manager->allocate_backend();
+    auto cf = backend->make_call_frame(external);
+
+    shared_ptr<runtime::interpreter::INT_CallFrame> icf =
+        static_pointer_cast<runtime::interpreter::INT_CallFrame>(cf);
+
+    // Create some tensors for input/output
+    auto a = backend->make_primary_tensor_view(element::f32, shape);
+    copy_data(a, vector<float>{2, 4, NAN, 16});
+    auto b = backend->make_primary_tensor_view(element::f32, shape);
+    copy_data(b, vector<float>{1, 2, 1, 8});
+    auto result = backend->make_primary_tensor_view(element::f32, shape);
+
+    icf->set_nan_check(true);
+    EXPECT_ANY_THROW(icf->call({a, b}, {result}));
+}
+
+TEST(INTERPRETER, nan_check_output)
+{
+    auto shape = Shape{4};
+    auto A = make_shared<op::Parameter>(element::f32, shape);
+    auto B = make_shared<op::Parameter>(element::f32, shape);
+    auto f = make_shared<Function>(make_shared<op::Divide>(A, B), op::Parameters{A, B});
+
+    auto manager = runtime::Manager::get("INTERPRETER");
+    auto external = manager->compile(f);
+    auto backend = manager->allocate_backend();
+    auto cf = backend->make_call_frame(external);
+
+    shared_ptr<runtime::interpreter::INT_CallFrame> icf =
+        static_pointer_cast<runtime::interpreter::INT_CallFrame>(cf);
+
+    // Create some tensors for input/output
+    auto a = backend->make_primary_tensor_view(element::f32, shape);
+    copy_data(a, vector<float>{2, 4, 0, 16});
+    auto b = backend->make_primary_tensor_view(element::f32, shape);
+    copy_data(b, vector<float>{1, 2, 0, 8});
+    auto result = backend->make_primary_tensor_view(element::f32, shape);
+
+    icf->set_nan_check(true);
+    EXPECT_ANY_THROW(icf->call({a, b}, {result}));
+}

test/backend_performance.cpp

@@ -30,17 +30,11 @@
 #include "ngraph/serializer.hpp"
 #include "ngraph/util.hpp"
 #include "util/random.hpp"
+#include "util/test_tools.hpp"
 
 using namespace std;
 using namespace ngraph;
 
-template <typename T>
-static void copy_data(shared_ptr<runtime::TensorView> tv, const vector<T>& data)
-{
-    size_t data_size = data.size() * sizeof(T);
-    tv->write(data.data(), 0, data_size);
-}
-
 static multimap<size_t, string>
     agregate_timing(const vector<runtime::PerformanceCounter>& perf_data)
 {

test/backend_test.in.cpp

@@ -25,17 +25,11 @@
 #include "ngraph/serializer.hpp"
 #include "util/all_close.hpp"
 #include "util/ndarray.hpp"
+#include "util/test_tools.hpp"
 
 using namespace std;
 using namespace ngraph;
 
-template <typename T>
-static void copy_data(shared_ptr<runtime::TensorView> tv, const vector<T>& data)
-{
-    size_t data_size = data.size() * sizeof(T);
-    tv->write(data.data(), 0, data_size);
-}
-
 TEST(${BACKEND_NAME}, aliased_output)
 {
     auto shape = Shape{2, 2};

test/util/test_tools.hpp

@@ -25,3 +25,10 @@ namespace ngraph
 
 bool validate_list(const std::list<std::shared_ptr<ngraph::Node>>& nodes);
 std::shared_ptr<ngraph::Function> make_test_graph();
+
+template <typename T>
+void copy_data(std::shared_ptr<ngraph::runtime::TensorView> tv, const std::vector<T>& data)
+{
+    size_t data_size = data.size() * sizeof(T);
+    tv->write(data.data(), 0, data_size);
+}