CPU nan/inf tensor validation (#553)

* global tracing

* fix compiler errors

* nan/inf validation

* 0644 on mkldnn_utils.cpp

* address Bob's feedback

* 0755 -> 0644

* remove format changes to python dir

src/ngraph/runtime/cpu/cpu_external_function.cpp

@@ -131,6 +131,21 @@ static const string s_output_dir = "cpu_codegen";
 // Temporary Memory Pool alignment
 static const size_t s_memory_pool_alignment = 4096;
 
+static void
+    generate_isnan_isinf_check(codegen::CodeWriter& writer,
+                               std::shared_ptr<Node> node,
+                               const std::vector<ngraph::runtime::cpu::TensorViewWrapper>& out,
+                               const char* funcname)
+{
+    auto ctype = node->get_element_type().c_type_string();
+    writer << "{   // A " << funcname << " for" << node->get_name() << "\n";
+    writer.indent++;
+    writer << " ngraph::check_fp_values<" << ctype << "," << funcname << "> (\"" << node->get_name()
+           << "\", (" << ctype << "*)" << out[0].get_name() << ", " << out[0].get_size() << ");\n";
+    writer.indent--;
+    writer << "}\n";
+}
+
 class StaticInitializers
 {
 public:
@@ -714,6 +729,28 @@ using namespace ngraph::runtime;
                 writer << func_name << "(" << join(names) << ", ctx);\n";
             }
 
+            //skip multi-output nodes since they would be covered by GetOutputElement
+            if (node->get_output_size() == 1 &&
+                //skip non-FP nodes
+                (node->get_element_type() == element::f32 ||
+                 node->get_element_type() == element::f64))
+            {
+                //check inputs and constants?
+                if ((!node->is_parameter() && !node->is_constant()) ||
+                    std::getenv("NGRAPH_CPU_CHECK_PARMS_AND_CONSTS"))
+                {
+                    if (std::getenv("NGRAPH_CPU_NAN_CHECK"))
+                    {
+                        generate_isnan_isinf_check(writer, node, out, "std::isnan");
+                    }
+
+                    if (std::getenv("NGRAPH_CPU_INF_CHECK"))
+                    {
+                        generate_isnan_isinf_check(writer, node, out, "std::isinf");
+                    }
+                }
+            }
+
             // Emit operation epilogue
             if (!node->is_parameter() && !node->is_constant())
             {

src/ngraph/util.hpp

@@ -17,6 +17,8 @@
 #pragma once
 
 #include <chrono>
+#include <cmath>
+#include <functional>
 #include <iostream>
 #include <map>
 #include <memory>
@@ -215,6 +217,30 @@ namespace ngraph
         return y > x ? 0 : x - y;
     }
 
+    template <typename T, bool (*func)(T)>
+    void check_fp_values(const char* name, const T* array, size_t n)
+    {
+        bool (*fPtr)(T) = &std::isinf;
+        const char* cerr_type = fPtr == func ? "Inf" : "NaN";
+        for (size_t i = 0; i < n; i++)
+        {
+            if (func(array[i]))
+            {
+                throw std::runtime_error(std::string("Discovered ") + cerr_type + " in '" + name +
+                                         "'");
+            }
+        }
+    }
+
+    template void
+        check_fp_values<float, std::isinf>(const char* name, const float* array, size_t n);
+    template void
+        check_fp_values<float, std::isnan>(const char* name, const float* array, size_t n);
+    template void
+        check_fp_values<double, std::isinf>(const char* name, const double* array, size_t n);
+    template void
+        check_fp_values<double, std::isnan>(const char* name, const double* array, size_t n);
+
     void* aligned_alloc(size_t alignment, size_t size);
     void aligned_free(void*);
     size_t round_up(size_t size, size_t alignment);