Add Inference Engine (IE) backend (#883)

* ie backend and manager with passing unit tests except for select/function

* fix function_call and select

* simplify implemenation by removing support for convert and select

* remove manager

src/ngraph/CMakeLists.txt

@@ -123,6 +123,7 @@ set (SRC
     runtime/aligned_buffer.cpp
     runtime/backend.cpp
     runtime/host_tensor_view.cpp
+    runtime/ie/ie_backend.cpp
     runtime/interpreter/int_backend.cpp
     runtime/interpreter/int_call_frame.cpp
     runtime/interpreter/int_external_function.cpp

src/ngraph/runtime/ie/ie_backend.cpp

+/*******************************************************************************
+* Copyright 2017-2018 Intel Corporation
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*******************************************************************************/
+
+#include "ngraph/runtime/ie/ie_backend.hpp"
+
+#include "ngraph/descriptor/layout/dense_tensor_view_layout.hpp"
+#include "ngraph/pass/assign_layout.hpp"
+#include "ngraph/pass/liveness.hpp"
+#include "ngraph/pass/manager.hpp"
+
+#include "ngraph/util.hpp"
+
+using namespace std;
+using namespace ngraph;
+
+using descriptor::layout::DenseTensorViewLayout;
+
+static bool static_init()
+{
+    runtime::Backend::register_backend("IE", make_shared<runtime::ie::IE_Backend>());
+    return true;
+};
+
+bool runtime::ie::IE_Backend::init = static_init();
+
+shared_ptr<runtime::TensorView> runtime::ie::IE_Backend::create_tensor(const element::Type& type,
+                                                                       const Shape& shape)
+{
+    return make_shared<runtime::HostTensorView>(type, shape, "external");
+}
+
+shared_ptr<runtime::TensorView> runtime::ie::IE_Backend::create_tensor(const element::Type& type,
+                                                                       const Shape& shape,
+                                                                       void* memory_pointer)
+{
+    return make_shared<runtime::HostTensorView>(type, shape, memory_pointer, "external");
+}
+
+bool runtime::ie::IE_Backend::compile(shared_ptr<Function> function)
+{
+    pass::Manager pass_manager;
+    pass_manager.register_pass<pass::AssignLayout<DenseTensorViewLayout>>();
+    pass_manager.register_pass<pass::Liveness>();
+    pass_manager.run_passes(function);
+
+    return true;
+}
+
+bool runtime::ie::IE_Backend::call(shared_ptr<Function> function,
+                                   const vector<shared_ptr<runtime::TensorView>>& outputs,
+                                   const vector<shared_ptr<runtime::TensorView>>& inputs)
+{
+    validate_call(function, outputs, inputs);
+
+    // TODO: check if function already compiled?
+    compile(function);
+
+    // convert inputs to HostTensorView
+    vector<shared_ptr<runtime::HostTensorView>> func_inputs;
+    for (auto tv : inputs)
+    {
+        func_inputs.push_back(static_pointer_cast<runtime::HostTensorView>(tv));
+    }
+
+    // convert outputs to HostTensorView
+    vector<shared_ptr<runtime::HostTensorView>> func_outputs;
+    for (auto tv : outputs)
+    {
+        func_outputs.push_back(static_pointer_cast<runtime::HostTensorView>(tv));
+    }
+
+    // map function params -> HostTensorView
+    unordered_map<descriptor::TensorView*, shared_ptr<runtime::HostTensorView>> tensor_map;
+    size_t input_count = 0;
+    for (auto param : function->get_parameters())
+    {
+        for (size_t i = 0; i < param->get_output_size(); ++i)
+        {
+            descriptor::TensorView* tv = param->get_output_tensor_view(i).get();
+            tensor_map.insert({tv, func_inputs[input_count++]});
+        }
+    }
+
+    // map function outputs -> HostTensorView
+    for (size_t output_count = 0; output_count < function->get_output_size(); ++output_count)
+    {
+        auto output = function->get_output_op(output_count);
+        if (!dynamic_pointer_cast<op::Result>(output))
+        {
+            throw ngraph_error("One of function's outputs isn't op::Result");
+        }
+        descriptor::TensorView* tv = output->get_output_tensor_view(0).get();
+        tensor_map.insert({tv, func_outputs[output_count]});
+    }
+
+    // for each ordered op in the graph
+    for (shared_ptr<Node> op : function->get_ordered_ops())
+    {
+        if (op->description() == "Parameter")
+        {
+            continue;
+        }
+
+        // get op inputs from map
+        vector<shared_ptr<runtime::HostTensorView>> op_inputs;
+        for (const descriptor::Input& input : op->get_inputs())
+        {
+            descriptor::TensorView* tv = input.get_output().get_tensor_view().get();
+            op_inputs.push_back(tensor_map.at(tv));
+        }
+
+        // get op outputs from map or create
+        vector<shared_ptr<runtime::HostTensorView>> op_outputs;
+        for (size_t i = 0; i < op->get_output_size(); ++i)
+        {
+            descriptor::TensorView* tv = op->get_output_tensor_view(i).get();
+            shared_ptr<runtime::HostTensorView> htv;
+            if (!contains_key(tensor_map, tv))
+            {
+                // the output tensor is not in the tensor map so create a new tensor
+                const Shape& shape = op->get_output_shape(i);
+                const element::Type& type = op->get_output_element_type(i);
+                string name = op->get_output_tensor(i).get_name();
+                htv = make_shared<runtime::HostTensorView>(type, shape, name);
+                tensor_map.insert({tv, htv});
+            }
+            else
+            {
+                htv = tensor_map.at(tv);
+            }
+            op_outputs.push_back(htv);
+        }
+
+        // get op type
+        element::Type type = op->get_element_type();
+        if (!op->get_inputs().empty())
+        {
+            type = op->get_inputs().at(0).get_tensor().get_element_type();
+        }
+
+        generate_calls(type, *op, op_outputs, op_inputs);
+
+        // delete any obsolete tensors
+        for (const descriptor::Tensor* t : op->liveness_free_list)
+        {
+            for (auto it = tensor_map.begin(); it != tensor_map.end(); ++it)
+            {
+                if (it->second->get_tensor().get_name() == t->get_name())
+                {
+                    tensor_map.erase(it);
+                    break;
+                }
+            }
+        }
+    }
+
+    return true;
+}
+
+void runtime::ie::IE_Backend::generate_calls(const element::Type& type,
+                                             Node& op,
+                                             const vector<shared_ptr<HostTensorView>>& outputs,
+                                             const vector<shared_ptr<HostTensorView>>& inputs)
+{
+    if (type == element::boolean)
+    {
+        op_engine<char>(op, outputs, inputs);
+    }
+    else if (type == element::f32)
+    {
+        op_engine<float>(op, outputs, inputs);
+    }
+    else if (type == element::f64)
+    {
+        op_engine<double>(op, outputs, inputs);
+    }
+    else if (type == element::i8)
+    {
+        op_engine<int8_t>(op, outputs, inputs);
+    }
+    else if (type == element::i16)
+    {
+        op_engine<int16_t>(op, outputs, inputs);
+    }
+    else if (type == element::i32)
+    {
+        op_engine<int32_t>(op, outputs, inputs);
+    }
+    else if (type == element::i64)
+    {
+        op_engine<int64_t>(op, outputs, inputs);
+    }
+    else if (type == element::u8)
+    {
+        op_engine<uint8_t>(op, outputs, inputs);
+    }
+    else if (type == element::u16)
+    {
+        op_engine<uint16_t>(op, outputs, inputs);
+    }
+    else if (type == element::u32)
+    {
+        op_engine<uint32_t>(op, outputs, inputs);
+    }
+    else if (type == element::u64)
+    {
+        op_engine<uint64_t>(op, outputs, inputs);
+    }
+    else
+    {
+        stringstream ss;
+        ss << "unsupported element type " << type << " op " << op.get_name();
+        throw ngraph_error(ss.str());
+    }
+}

src/ngraph/runtime/ie/ie_backend.hpp

+/*******************************************************************************
+* Copyright 2017-2018 Intel Corporation
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*******************************************************************************/
+
+#pragma once
+
+#include <memory>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "ngraph/runtime/backend.hpp"
+#include "ngraph/runtime/host_tensor_view.hpp"
+#include "ngraph/runtime/tensor_view.hpp"
+
+#include "ngraph/op/avg_pool.hpp"
+#include "ngraph/op/broadcast.hpp"
+#include "ngraph/op/concat.hpp"
+#include "ngraph/op/constant.hpp"
+#include "ngraph/op/convolution.hpp"
+#include "ngraph/op/dot.hpp"
+#include "ngraph/op/max.hpp"
+#include "ngraph/op/max_pool.hpp"
+#include "ngraph/op/min.hpp"
+#include "ngraph/op/one_hot.hpp"
+#include "ngraph/op/pad.hpp"
+#include "ngraph/op/product.hpp"
+#include "ngraph/op/reduce.hpp"
+#include "ngraph/op/reduce_window.hpp"
+#include "ngraph/op/replace_slice.hpp"
+#include "ngraph/op/reshape.hpp"
+#include "ngraph/op/result.hpp"
+#include "ngraph/op/reverse.hpp"
+#include "ngraph/op/slice.hpp"
+#include "ngraph/op/softmax.hpp"
+#include "ngraph/op/sum.hpp"
+
+#include "ngraph/runtime/reference/abs.hpp"
+#include "ngraph/runtime/reference/acos.hpp"
+#include "ngraph/runtime/reference/add.hpp"
+#include "ngraph/runtime/reference/asin.hpp"
+#include "ngraph/runtime/reference/atan.hpp"
+#include "ngraph/runtime/reference/avg_pool.hpp"
+#include "ngraph/runtime/reference/broadcast.hpp"
+#include "ngraph/runtime/reference/ceiling.hpp"
+#include "ngraph/runtime/reference/concat.hpp"
+#include "ngraph/runtime/reference/constant.hpp"
+#include "ngraph/runtime/reference/convolution.hpp"
+#include "ngraph/runtime/reference/copy.hpp"
+#include "ngraph/runtime/reference/cos.hpp"
+#include "ngraph/runtime/reference/cosh.hpp"
+#include "ngraph/runtime/reference/divide.hpp"
+#include "ngraph/runtime/reference/dot.hpp"
+#include "ngraph/runtime/reference/equal.hpp"
+#include "ngraph/runtime/reference/exp.hpp"
+#include "ngraph/runtime/reference/floor.hpp"
+#include "ngraph/runtime/reference/greater.hpp"
+#include "ngraph/runtime/reference/greater_eq.hpp"
+#include "ngraph/runtime/reference/less.hpp"
+#include "ngraph/runtime/reference/less_eq.hpp"
+#include "ngraph/runtime/reference/log.hpp"
+#include "ngraph/runtime/reference/max.hpp"
+#include "ngraph/runtime/reference/max_pool.hpp"
+#include "ngraph/runtime/reference/maximum.hpp"
+#include "ngraph/runtime/reference/min.hpp"
+#include "ngraph/runtime/reference/minimum.hpp"
+#include "ngraph/runtime/reference/multiply.hpp"
+#include "ngraph/runtime/reference/negate.hpp"
+#include "ngraph/runtime/reference/not.hpp"
+#include "ngraph/runtime/reference/not_equal.hpp"
+#include "ngraph/runtime/reference/one_hot.hpp"
+#include "ngraph/runtime/reference/pad.hpp"
+#include "ngraph/runtime/reference/power.hpp"
+#include "ngraph/runtime/reference/product.hpp"
+#include "ngraph/runtime/reference/reduce.hpp"
+#include "ngraph/runtime/reference/reduce_window.hpp"
+#include "ngraph/runtime/reference/relu.hpp"
+#include "ngraph/runtime/reference/replace_slice.hpp"
+#include "ngraph/runtime/reference/reshape.hpp"
+#include "ngraph/runtime/reference/result.hpp"
+#include "ngraph/runtime/reference/reverse.hpp"
+#include "ngraph/runtime/reference/sign.hpp"
+#include "ngraph/runtime/reference/sin.hpp"
+#include "ngraph/runtime/reference/sinh.hpp"
+#include "ngraph/runtime/reference/slice.hpp"
+#include "ngraph/runtime/reference/softmax.hpp"
+#include "ngraph/runtime/reference/sqrt.hpp"
+#include "ngraph/runtime/reference/subtract.hpp"
+#include "ngraph/runtime/reference/sum.hpp"
+#include "ngraph/runtime/reference/tan.hpp"
+#include "ngraph/runtime/reference/tanh.hpp"
+
+#ifdef NGRAPH_DISTRIBUTED
+#include "ngraph/runtime/reference/allreduce.hpp"
+#endif
+
+namespace ngraph
+{
+    namespace runtime
+    {
+        namespace ie
+        {
+            class IE_Backend : public Backend
+            {
+            public:
+                std::shared_ptr<TensorView> create_tensor(const element::Type& type,
+                                                          const Shape& shape,
+                                                          void* memory_pointer) override;
+
+                std::shared_ptr<TensorView> create_tensor(const element::Type& type,
+                                                          const Shape& shape) override;
+
+                bool compile(std::shared_ptr<Function> function) override;
+
+                bool call(std::shared_ptr<Function> function,
+                          const std::vector<std::shared_ptr<TensorView>>& outputs,
+                          const std::vector<std::shared_ptr<TensorView>>& intputs) override;
+
+            private:
+                static bool init;
+                void generate_calls(const element::Type& type,
+                                    Node& op,
+                                    const std::vector<std::shared_ptr<HostTensorView>>& outputs,
+                                    const std::vector<std::shared_ptr<HostTensorView>>& inputs);
+
+                template <typename T>
+                void op_engine(Node& node,
+                               const std::vector<std::shared_ptr<HostTensorView>>& out,
+                               const std::vector<std::shared_ptr<HostTensorView>>& args)
+                {
+                    std::string node_op = node.description();
+                    if (node_op == "Abs")
+                    {
+                        reference::abs<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                          reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                          out[0]->get_element_count());
+                    }
+                    else if (node_op == "Acos")
+                    {
+                        reference::acos<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                           reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                           out[0]->get_element_count());
+                    }
+                    else if (node_op == "Add")
+                    {
+                        reference::add<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                          reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                                          reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                          out[0]->get_element_count());
+                    }
+#ifdef NGRAPH_DISTRIBUTED
+                    else if (node_op == "AllReduce")
+                    {
+                        reference::allreduce<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                                reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                                args[0]->get_element_type(),
+                                                static_cast<int>(args[0]->get_element_count()));
+                    }
+#endif
+                    else if (node_op == "Asin")
+                    {
+                        reference::asin<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                           reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                           out[0]->get_element_count());
+                    }
+                    else if (node_op == "Atan")
+                    {
+                        reference::atan<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                           reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                           out[0]->get_element_count());
+                    }
+                    else if (node_op == "AvgPool")
+                    {
+                        op::AvgPool* avg_pool = dynamic_cast<op::AvgPool*>(&node);
+
+                        reference::avg_pool<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                               reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                               args[0]->get_shape(),
+                                               out[0]->get_shape(),
+                                               avg_pool->get_window_shape(),
+                                               avg_pool->get_window_movement_strides(),
+                                               avg_pool->get_padding_below(),
+                                               avg_pool->get_padding_above(),
+                                               avg_pool->get_include_padding_in_avg_computation());
+                    }
+                    else if (node_op == "AvgPoolBackprop")
+                    {
+                        op::AvgPoolBackprop* apb = dynamic_cast<op::AvgPoolBackprop*>(&node);
+                        reference::avg_pool_backprop<T>(
+                            reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                            reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                            args[0]->get_shape(),
+                            out[0]->get_shape(),
+                            apb->get_window_shape(),
+                            apb->get_window_movement_strides(),
+                            apb->get_padding_below(),
+                            apb->get_padding_above(),
+                            apb->get_include_padding_in_avg_computation());
+                    }
+                    else if (node_op == "Broadcast")
+                    {
+                        op::Broadcast* broadcast = dynamic_cast<op::Broadcast*>(&node);
+                        Shape in_shape = args[0]->get_shape();
+                        Shape out_shape = out[0]->get_shape();
+                        AxisSet broadcast_axes = broadcast->get_broadcast_axes();
+                        reference::broadcast<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                                reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                                in_shape,
+                                                out_shape,
+                                                broadcast_axes);
+                    }
+                    else if (node_op == "Ceiling")
+                    {
+                        reference::ceiling<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                              reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                              out[0]->get_element_count());
+                    }
+                    else if (node_op == "Concat")
+                    {
+                        const op::Concat* concat = static_cast<const op::Concat*>(&node);
+                        std::vector<const T*> in_args;
+                        std::vector<Shape> in_shapes;
+                        for (std::shared_ptr<HostTensorView> arg : args)
+                        {
+                            in_args.push_back(reinterpret_cast<T*>(arg->get_data_ptr()));
+                            in_shapes.push_back(arg->get_shape());
+                        }
+                        reference::concat<T>(in_args,
+                                             reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                             in_shapes,
+                                             out[0]->get_shape(),
+                                             concat->get_concatenation_axis());
+                    }
+                    else if (node_op == "Constant")
+                    {
+                        const op::Constant* c = static_cast<const op::Constant*>(&node);
+                        reference::constant<T>(reinterpret_cast<const T*>(c->get_data_ptr()),
+                                               reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                               out[0]->get_element_count());
+                    }
+                    else if (node_op == "Convolution")
+                    {
+                        auto c = static_cast<const op::Convolution*>(&node);
+                        reference::convolution<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                                  reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                                                  reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                                  args[0]->get_shape(),
+                                                  args[1]->get_shape(),
+                                                  out[0]->get_shape(),
+                                                  c->get_window_movement_strides(),
+                                                  c->get_window_dilation_strides(),
+                                                  c->get_padding_below(),
+                                                  c->get_padding_above(),
+                                                  c->get_data_dilation_strides(),
+                                                  0,
+                                                  1,
+                                                  1,
+                                                  0,
+                                                  0,
+                                                  1,
+                                                  false);
+                    }
+                    else if (node_op == "ConvolutionBackpropFilters")
+                    {
+                        auto c = static_cast<const op::ConvolutionBackpropFilters*>(&node);
+                        reference::convolution<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                                  reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                                                  reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                                  args[0]->get_shape(),
+                                                  args[1]->get_shape(),
+                                                  out[0]->get_shape(),
+                                                  c->get_window_movement_strides_backward(),
+                                                  c->get_window_dilation_strides_backward(),
+                                                  c->get_padding_below_backward(),
+                                                  c->get_padding_above_backward(),
+                                                  c->get_data_dilation_strides_backward(),
+                                                  1,
+                                                  0,
+                                                  0,
+                                                  1,
+                                                  1,
+                                                  0,
+                                                  false);
+                    }
+                    else if (node_op == "ConvolutionBackpropData")
+                    {
+                        // Note that args[1] and args[0] are switched here from the usual order.
+                        auto c = static_cast<const op::ConvolutionBackpropData*>(&node);
+                        reference::convolution<T>(reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                                                  reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                                  reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                                  args[1]->get_shape(),
+                                                  args[0]->get_shape(),
+                                                  out[0]->get_shape(),
+                                                  c->get_window_movement_strides_backward(),
+                                                  c->get_window_dilation_strides_backward(),
+                                                  c->get_padding_below_backward(),
+                                                  c->get_padding_above_backward(),
+                                                  c->get_data_dilation_strides_backward(),
+                                                  0,
+                                                  1,
+                                                  0,
+                                                  1,
+                                                  0,
+                                                  1,
+                                                  true);
+                    }
+                    else if (node_op == "Cos")
+                    {
+                        reference::cos<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                          reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                          out[0]->get_element_count());
+                    }
+                    else if (node_op == "Cosh")
+                    {
+                        reference::cosh<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                           reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                           out[0]->get_element_count());
+                    }
+                    else if (node_op == "Divide")
+                    {
+                        reference::divide<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                             reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                                             reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                             out[0]->get_element_count());
+                    }
+                    else if (node_op == "Dot")
+                    {
+                        op::Dot* dot = dynamic_cast<op::Dot*>(&node);
+
+                        reference::dot(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                       reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                                       reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                       args[0]->get_shape(),
+                                       args[1]->get_shape(),
+                                       out[0]->get_shape(),
+                                       dot->get_reduction_axes_count());
+                    }
+
+                    else if (node_op == "Equal")
+                    {
+                        reference::equal<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                            reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                                            reinterpret_cast<char*>(out[0]->get_data_ptr()),
+                                            out[0]->get_element_count());
+                    }
+                    else if (node_op == "Exp")
+                    {
+                        reference::exp<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                          reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                          out[0]->get_element_count());
+                    }
+                    else if (node_op == "Floor")
+                    {
+                        reference::floor<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                            reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                            out[0]->get_element_count());
+                    }
+                    else if (node_op == "FunctionCall")
+                    {
+                        std::shared_ptr<Function> function = node.get_functions()[0];
+
+                        std::vector<std::shared_ptr<runtime::TensorView>> outputs;
+                        for (auto tv : out)
+                        {
+                            outputs.push_back(std::static_pointer_cast<runtime::TensorView>(tv));
+                        }
+
+                        std::vector<std::shared_ptr<runtime::TensorView>> inputs;
+                        for (auto tv : args)
+                        {
+                            inputs.push_back(std::static_pointer_cast<runtime::TensorView>(tv));
+                        }
+
+                        call(function, outputs, inputs);
+                    }
+                    else if (node_op == "Greater")
+                    {
+                        reference::greater<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                              reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                                              reinterpret_cast<char*>(out[0]->get_data_ptr()),
+                                              out[0]->get_element_count());
+                    }
+                    else if (node_op == "GreaterEq")
+                    {
+                        reference::greater_eq<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                                 reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                                                 reinterpret_cast<char*>(out[0]->get_data_ptr()),
+                                                 out[0]->get_element_count());
+                    }
+                    else if (node_op == "Less")
+                    {
+                        reference::less<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                           reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                                           reinterpret_cast<char*>(out[0]->get_data_ptr()),
+                                           out[0]->get_element_count());
+                    }
+                    else if (node_op == "LessEq")
+                    {
+                        reference::less_eq<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                              reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                                              reinterpret_cast<char*>(out[0]->get_data_ptr()),
+                                              out[0]->get_element_count());
+                    }
+                    else if (node_op == "Log")
+                    {
+                        reference::log<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                          reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                          out[0]->get_element_count());
+                    }
+                    else if (node_op == "Max")
+                    {
+                        const op::Max* max = static_cast<const op::Max*>(&node);
+                        reference::max<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                          reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                          args[0]->get_shape(),
+                                          out[0]->get_shape(),
+                                          max->get_reduction_axes());
+                    }
+                    else if (node_op == "Maximum")
+                    {
+                        reference::maximum<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                              reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                                              reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                              out[0]->get_element_count());
+                    }
+                    else if (node_op == "MaxPool")
+                    {
+                        op::MaxPool* max_pool = dynamic_cast<op::MaxPool*>(&node);
+
+                        reference::max_pool<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                               reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                               args[0]->get_shape(),
+                                               out[0]->get_shape(),
+                                               max_pool->get_window_shape(),
+                                               max_pool->get_window_movement_strides(),
+                                               max_pool->get_padding_below(),
+                                               max_pool->get_padding_above());
+                    }
+                    else if (node_op == "MaxPoolBackprop")
+                    {
+                        op::MaxPoolBackprop* max_pool_backprop =
+                            dynamic_cast<op::MaxPoolBackprop*>(&node);
+
+                        reference::max_pool_backprop<T>(
+                            reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                            reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                            reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                            args[1]->get_shape(),
+                            out[0]->get_shape(),
+                            max_pool_backprop->get_window_shape(),
+                            max_pool_backprop->get_window_movement_strides(),
+                            max_pool_backprop->get_padding_below(),
+                            max_pool_backprop->get_padding_above());
+                    }
+                    else if (node_op == "Min")
+                    {
+                        const op::Min* min = static_cast<const op::Min*>(&node);
+                        reference::min<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                          reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                          args[0]->get_shape(),
+                                          out[0]->get_shape(),
+                                          min->get_reduction_axes());
+                    }
+                    else if (node_op == "Minimum")
+                    {
+                        reference::minimum<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                              reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                                              reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                              out[0]->get_element_count());
+                    }
+                    else if (node_op == "Multiply")
+                    {
+                        reference::multiply<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                               reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                                               reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                               out[0]->get_element_count());
+                    }
+                    else if (node_op == "Negative")
+                    {
+                        reference::negate<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                             reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                             out[0]->get_element_count());
+                    }
+                    else if (node_op == "Not")
+                    {
+                        reference::logical_not(reinterpret_cast<char*>(args[0]->get_data_ptr()),
+                                               reinterpret_cast<char*>(out[0]->get_data_ptr()),
+                                               out[0]->get_element_count());
+                    }
+                    else if (node_op == "NotEqual")
+                    {
+                        reference::not_equal<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                                reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                                                reinterpret_cast<char*>(out[0]->get_data_ptr()),
+                                                out[0]->get_element_count());
+                    }
+                    else if (node_op == "OneHot")
+                    {
+                        auto oh = static_cast<const op::OneHot*>(&node);
+                        reference::one_hot<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                              reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                              args[0]->get_shape(),
+                                              out[0]->get_shape(),
+                                              oh->get_one_hot_axis());
+                    }
+                    else if (node_op == "Parameter")
+                    {
+                    }
+                    else if (node_op == "Pad")
+                    {
+                        op::Pad* pad = dynamic_cast<op::Pad*>(&node);
+
+                        reference::pad(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                       reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                                       reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                       node.get_inputs().at(0).get_shape(),
+                                       node.get_output_shape(0),
+                                       pad->get_padding_below(),
+                                       pad->get_padding_above(),
+                                       pad->get_padding_interior());
+                    }
+                    else if (node_op == "Power")
+                    {
+                        reference::power<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                            reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                                            reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                            out[0]->get_element_count());
+                    }
+                    else if (node_op == "Product")
+                    {
+                        const op::Product* product = static_cast<const op::Product*>(&node);
+                        reference::product<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                              reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                              args[0]->get_shape(),
+                                              out[0]->get_shape(),
+                                              product->get_reduction_axes());
+                    }
+                    else if (node_op == "Reduce")
+                    {
+                        op::Reduce* reduce = dynamic_cast<op::Reduce*>(&node);
+                        std::shared_ptr<Function> reduction_function = reduce->get_functions()[0];
+
+                        std::function<T(T, T)> f = [this, &node, reduction_function](T x,
+                                                                                     T y) -> T {
+                            auto tx = std::make_shared<HostTensorView>(
+                                node.get_inputs().at(0).get_element_type(),
+                                Shape{},
+                                "reduce_temp_x");
+                            auto ty = std::make_shared<HostTensorView>(
+                                node.get_inputs().at(1).get_element_type(),
+                                Shape{},
+                                "reduce_temp_y");
+                            auto tr = std::make_shared<HostTensorView>(
+                                node.get_output_element_type(0), Shape{}, "reduce_temp_r");
+                            *(reinterpret_cast<T*>(tx->get_data_ptr())) = x;
+                            *(reinterpret_cast<T*>(ty->get_data_ptr())) = y;
+                            call(reduction_function, {tr}, {tx, ty});
+                            return *(reinterpret_cast<T*>(tr->get_data_ptr()));
+                        };
+
+                        reference::reduce(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                          reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                                          reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                          node.get_inputs().at(0).get_shape(),
+                                          node.get_output_shape(0),
+                                          reduce->get_reduction_axes(),
+                                          f);
+                    }
+                    else if (node_op == "ReduceWindow")
+                    {
+                        op::ReduceWindow* reduce_window = dynamic_cast<op::ReduceWindow*>(&node);
+                        std::shared_ptr<Function> reduction_function =
+                            reduce_window->get_functions()[0];
+
+                        std::function<T(T, T)> f = [this, &node, reduction_function](T x,
+                                                                                     T y) -> T {
+                            auto tx = std::make_shared<HostTensorView>(
+                                node.get_inputs().at(0).get_element_type(),
+                                Shape{},
+                                "reduce_window_temp_x");
+                            auto ty = std::make_shared<HostTensorView>(
+                                node.get_inputs().at(1).get_element_type(),
+                                Shape{},
+                                "reduce_window_temp_y");
+                            auto tr = std::make_shared<HostTensorView>(
+                                node.get_output_element_type(0), Shape{}, "reduce_window_temp_r");
+                            *(reinterpret_cast<T*>(tx->get_data_ptr())) = x;
+                            *(reinterpret_cast<T*>(ty->get_data_ptr())) = y;
+                            call(reduction_function, {tr}, {tx, ty});
+                            return *(reinterpret_cast<T*>(tr->get_data_ptr()));
+                        };
+
+                        reference::reduce_window(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                                 reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                                                 reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                                 node.get_inputs().at(0).get_shape(),
+                                                 node.get_output_shape(0),
+                                                 f,
+                                                 reduce_window->get_window_shape(),
+                                                 reduce_window->get_window_movement_strides());
+                    }
+                    else if (node_op == "Relu")
+                    {
+                        reference::relu<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                           reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                           out[0]->get_element_count());
+                    }
+                    else if (node_op == "ReluBackprop")
+                    {
+                        reference::relu_backprop<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                                    reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                                                    reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                                    out[0]->get_element_count());
+                    }
+                    else if (node_op == "ReplaceSlice")
+                    {
+                        const op::ReplaceSlice* slice = static_cast<const op::ReplaceSlice*>(&node);
+                        reference::replace_slice<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                                    reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                                                    reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                                    args[1]->get_shape(),
+                                                    slice->get_lower_bounds(),
+                                                    slice->get_upper_bounds(),
+                                                    slice->get_strides(),
+                                                    out[0]->get_shape());
+                    }
+                    else if (node_op == "Reshape")
+                    {
+                        op::Reshape* reshape = dynamic_cast<op::Reshape*>(&node);
+                        reference::reshape(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                           reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                           args[0]->get_shape(),
+                                           reshape->get_input_order(),
+                                           out[0]->get_shape());
+                    }
+                    else if (node_op == "Result")
+                    {
+                        op::Result* res = dynamic_cast<op::Result*>(&node);
+                        reference::result(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                          reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                          shape_size(res->get_shape()));
+                    }
+                    else if (node_op == "Reverse")
+                    {
+                        op::Reverse* reverse = dynamic_cast<op::Reverse*>(&node);
+                        reference::reverse(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                           reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                           args[0]->get_shape(),
+                                           out[0]->get_shape(),
+                                           reverse->get_reversed_axes());
+                    }
+                    else if (node_op == "Sign")
+                    {
+                        reference::sign<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                           reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                           out[0]->get_element_count());
+                    }
+                    else if (node_op == "Sin")
+                    {
+                        reference::sin<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                          reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                          out[0]->get_element_count());
+                    }
+                    else if (node_op == "Sinh")
+                    {
+                        reference::sinh<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                           reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                           out[0]->get_element_count());
+                    }
+                    else if (node_op == "Slice")
+                    {
+                        const op::Slice* slice = static_cast<const op::Slice*>(&node);
+                        reference::slice<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                            reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                            args[0]->get_shape(),
+                                            slice->get_lower_bounds(),
+                                            slice->get_upper_bounds(),
+                                            slice->get_strides(),
+                                            out[0]->get_shape());
+                    }
+                    else if (node_op == "Softmax")
+                    {
+                        const op::Softmax* softmax = static_cast<const op::Softmax*>(&node);
+                        reference::softmax<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                              reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                              out[0]->get_shape(),
+                                              softmax->get_axes());
+                    }
+                    else if (node_op == "Sqrt")
+                    {
+                        reference::sqrt<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                           reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                           out[0]->get_element_count());
+                    }
+                    else if (node_op == "Subtract")
+                    {
+                        reference::subtract<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                               reinterpret_cast<T*>(args[1]->get_data_ptr()),
+                                               reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                               out[0]->get_element_count());
+                    }
+                    else if (node_op == "Sum")
+                    {
+                        const op::Sum* sum = static_cast<const op::Sum*>(&node);
+                        reference::sum<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                          reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                          args[0]->get_shape(),
+                                          out[0]->get_shape(),
+                                          sum->get_reduction_axes());
+                    }
+                    else if (node_op == "Tan")
+                    {
+                        reference::tan<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                          reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                          out[0]->get_element_count());
+                    }
+                    else if (node_op == "Tanh")
+                    {
+                        reference::tanh<T>(reinterpret_cast<T*>(args[0]->get_data_ptr()),
+                                           reinterpret_cast<T*>(out[0]->get_data_ptr()),
+                                           out[0]->get_element_count());
+                    }
+                    else
+                    {
+                        std::stringstream ss;
+                        ss << "unsupported op " << node_op;
+                        throw ngraph_error(ss.str());
+                    }
+                }
+            };
+        }
+    }
+}

test/CMakeLists.txt

@@ -70,6 +70,7 @@ add_subdirectory(files)
 #================================================================================================
 # TODO add interpreter back to unit tests when it works
 set(BACKEND_NAMES ${BACKEND_NAMES} "INTERPRETER")
+set(BACKEND_NAMES ${BACKEND_NAMES} "IE")
 
 if(MKLDNN_INCLUDE_DIR)
     include_directories(SYSTEM ${MKLDNN_INCLUDE_DIR})

test/autodiff.in.cpp

@@ -829,6 +829,7 @@ TEST(${BACKEND_NAME}, backwards_log)
 
 TEST(${BACKEND_NAME}, backwards_maximum)
 {
+    SKIP_TEST_FOR("IE", "${BACKEND_NAME}"); // no convert support
     auto backend = runtime::Backend::create("${BACKEND_NAME}");
 
     test::Uniform<float> rng(-1.0f, 1.0f);
@@ -847,6 +848,7 @@ TEST(${BACKEND_NAME}, backwards_maximum)
 
 TEST(${BACKEND_NAME}, backwards_minimum)
 {
+    SKIP_TEST_FOR("IE", "${BACKEND_NAME}"); // no convert support
     auto backend = runtime::Backend::create("${BACKEND_NAME}");
 
     test::Uniform<float> rng(-1.0f, 1.0f);
@@ -1017,6 +1019,7 @@ TEST(${BACKEND_NAME}, backwards_reshape)
 
 TEST(${BACKEND_NAME}, backwards_select)
 {
+    SKIP_TEST_FOR("IE", "${BACKEND_NAME}");
     SKIP_TEST_FOR("GPU", "${BACKEND_NAME}");
     SKIP_TEST_FOR("NNP_TESTER", "${BACKEND_NAME}");
 
@@ -1046,6 +1049,7 @@ TEST(${BACKEND_NAME}, backwards_select)
 
 TEST(${BACKEND_NAME}, backwards_select_nested)
 {
+    SKIP_TEST_FOR("IE", "${BACKEND_NAME}");
     SKIP_TEST_FOR("GPU", "${BACKEND_NAME}");
     SKIP_TEST_FOR("NNP_TESTER", "${BACKEND_NAME}");
 

test/backend_test.in.cpp

@@ -1349,6 +1349,7 @@ TEST(${BACKEND_NAME}, notequal)
 
 TEST(${BACKEND_NAME}, select)
 {
+    SKIP_TEST_FOR("IE", "${BACKEND_NAME}");
     SKIP_TEST_FOR("GPU", "${BACKEND_NAME}");
     Shape shape{2, 2, 2};
     auto A = make_shared<op::Parameter>(element::boolean, shape);
@@ -1766,6 +1767,7 @@ TEST(${BACKEND_NAME}, broadcast_matrix_2)
 
 TEST(${BACKEND_NAME}, convert_int32_float32)
 {
+    SKIP_TEST_FOR("IE", "${BACKEND_NAME}");
     Shape shape{2, 2};
     auto A = make_shared<op::Parameter>(element::i32, shape);
     auto f =
@@ -1784,6 +1786,7 @@ TEST(${BACKEND_NAME}, convert_int32_float32)
 
 TEST(${BACKEND_NAME}, convert_int32_bool)
 {
+    SKIP_TEST_FOR("IE", "${BACKEND_NAME}");
     Shape shape{2, 2};
     auto A = make_shared<op::Parameter>(element::i32, shape);
     auto f = make_shared<Function>(make_shared<op::Convert>(A, element::boolean),
@@ -1802,6 +1805,7 @@ TEST(${BACKEND_NAME}, convert_int32_bool)
 
 TEST(${BACKEND_NAME}, convert_float32_bool)
 {
+    SKIP_TEST_FOR("IE", "${BACKEND_NAME}");
     Shape shape{2, 2};
     auto A = make_shared<op::Parameter>(element::f32, shape);
     auto f = make_shared<Function>(make_shared<op::Convert>(A, element::boolean),
@@ -5144,6 +5148,7 @@ TEST(${BACKEND_NAME}, reduce_window_emulating_max_pool_2d_1channel_1image_stride
 //
 TEST(${BACKEND_NAME}, select_and_scatter_with_overlap)
 {
+    SKIP_TEST_FOR("IE", "${BACKEND_NAME}");
     SKIP_TEST_FOR("GPU", "${BACKEND_NAME}");
     Shape shape_sel_a{};
     auto SEL_A = make_shared<op::Parameter>(element::f32, shape_sel_a);
@@ -5198,6 +5203,7 @@ TEST(${BACKEND_NAME}, select_and_scatter_with_overlap)
 //
 TEST(${BACKEND_NAME}, select_and_scatter_without_overlap)
 {
+    SKIP_TEST_FOR("IE", "${BACKEND_NAME}");
     SKIP_TEST_FOR("GPU", "${BACKEND_NAME}");
     Shape shape_sel_a{};
     auto SEL_A = make_shared<op::Parameter>(element::f32, shape_sel_a);
@@ -5252,6 +5258,7 @@ TEST(${BACKEND_NAME}, select_and_scatter_without_overlap)
 //
 TEST(${BACKEND_NAME}, select_and_scatter_3d_without_overlap)
 {
+    SKIP_TEST_FOR("IE", "${BACKEND_NAME}");
     SKIP_TEST_FOR("GPU", "${BACKEND_NAME}");
     Shape shape_sel_a{};
     auto SEL_A = make_shared<op::Parameter>(element::f32, shape_sel_a);