Adding DetectionOutput and Interpolate layers (#2849)

* - Adding DetectionOutput and Interpolate layers

* Adding Unit tests

* Name fix

src/ngraph/CMakeLists.txt

@@ -162,6 +162,10 @@ set (SRC
     op/experimental/quantized_dot_bias.hpp
     op/experimental/transpose.cpp
     op/experimental/transpose.hpp
+    op/experimental/layers/detection_output.cpp
+    op/experimental/layers/detection_output.hpp
+    op/experimental/layers/interpolate.cpp
+    op/experimental/layers/interpolate.hpp
     op/experimental/layers/prior_box.cpp
     op/experimental/layers/prior_box.hpp
     op/experimental/layers/prior_box_clustered.cpp

src/ngraph/op/experimental/layers/detection_output.cpp

+//*****************************************************************************
+// Copyright 2017-2019 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#include "detection_output.hpp"
+
+#include "ngraph/op/constant.hpp"
+
+using namespace std;
+using namespace ngraph;
+
+op::DetectionOutput::DetectionOutput(const std::shared_ptr<Node>& box_logits,
+                                     const std::shared_ptr<Node>& class_preds,
+                                     const std::shared_ptr<Node>& proposals,
+                                     const std::shared_ptr<Node>& aux_class_preds,
+                                     const std::shared_ptr<Node>& aux_box_preds,
+                                     const DetectionOutputAttrs& attrs)
+    : Op("DetectionOutput",
+         check_single_output_args(
+             {box_logits, class_preds, proposals, aux_class_preds, aux_box_preds}))
+    , m_attrs(attrs)
+{
+    constructor_validate_and_infer_types();
+}
+
+void op::DetectionOutput::validate_and_infer_types()
+{
+    if (get_input_partial_shape(0).is_static())
+    {
+        auto box_logits_shape = get_input_partial_shape(0).to_shape();
+        set_output_type(
+            0, element::f32, Shape{1, 1, m_attrs.boxes_top_k[0] * box_logits_shape[0], 7});
+    }
+    else
+    {
+        set_output_type(0, element::f32, PartialShape::dynamic());
+    }
+}
+
+shared_ptr<Node> op::DetectionOutput::copy_with_new_args(const NodeVector& new_args) const
+{
+    check_new_args_count(this, new_args);
+    return make_shared<DetectionOutput>(
+        new_args.at(0), new_args.at(1), new_args.at(2), new_args.at(3), new_args.at(4), m_attrs);
+}

src/ngraph/op/experimental/layers/detection_output.hpp

+//*****************************************************************************
+// Copyright 2017-2019 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#pragma once
+
+#include "ngraph/op/op.hpp"
+
+namespace ngraph
+{
+    namespace op
+    {
+        typedef struct
+        {
+            int num_classes;
+            int background_label_id = 0;
+            int results_top_k = -1;
+            bool add_variance = false;
+            std::vector<int> boxes_top_k = {1};
+            std::string code_type = std::string{"caffe.PriorBoxParameter.CORNER"};
+            bool share_location = true;
+            float nms_threshold;
+            float confidence_threshold = std::numeric_limits<float>::min();
+            bool clip_after_nms = false;
+            bool clip_before_nms = false;
+            bool decrease_label_id = false;
+            bool normalized = false;
+            size_t input_height = 1;
+            size_t input_width = 1;
+            float objectness_score_threshold = 0;
+        } DetectionOutputAttrs;
+
+        /// \brief Layer which performs non-max suppression to
+        /// generate detection output using location and confidence predictions
+        class DetectionOutput : public Op
+        {
+        public:
+            /// \brief Constructs a DetectionOutput operation
+            ///
+            /// \param box_logits			Box logits
+            /// \param class_preds			Class predictions
+            /// \param proposals			Proposals
+            /// \param aux_class_preds		Auxilary class predictions
+            /// \param aux_box_preds		Auxilary box predictions
+            /// \param attrs				Detection Output attributes
+            DetectionOutput(const std::shared_ptr<Node>& box_logits,
+                            const std::shared_ptr<Node>& class_preds,
+                            const std::shared_ptr<Node>& proposals,
+                            const std::shared_ptr<Node>& aux_class_preds,
+                            const std::shared_ptr<Node>& aux_box_preds,
+                            const DetectionOutputAttrs& attrs);
+
+            void validate_and_infer_types() override;
+
+            virtual std::shared_ptr<Node>
+                copy_with_new_args(const NodeVector& new_args) const override;
+
+        private:
+            DetectionOutputAttrs m_attrs;
+        };
+    }
+}

src/ngraph/op/experimental/layers/interpolate.cpp

+//*****************************************************************************
+// Copyright 2017-2019 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#include "interpolate.hpp"
+
+#include "ngraph/op/constant.hpp"
+
+using namespace std;
+using namespace ngraph;
+
+op::Interpolate::Interpolate(const std::shared_ptr<Node>& image, const InterpolateAttrs& attrs)
+    : Op("Interpolate", check_single_output_args({image}))
+    , m_attrs(attrs)
+{
+    constructor_validate_and_infer_types();
+}
+
+void op::Interpolate::validate_and_infer_types()
+{
+    if (get_input_partial_shape(0).is_static())
+    {
+        Shape input_shape = get_input_partial_shape(0).to_shape();
+        Shape output_shape(4);
+        // Assumes {N, C, H, W}
+        output_shape[0] = input_shape[0];
+        output_shape[1] = input_shape[1];
+
+        auto is_zero = [](float value) {
+            return std::fabs(value) < std::numeric_limits<float>::epsilon();
+        };
+
+        bool should_scale = !(is_zero(m_attrs.zoom_factor) && is_zero(m_attrs.shrink_factor) &&
+                              is_zero(m_attrs.scale_factor));
+
+        if (should_scale)
+        {
+            float scale = m_attrs.scale_factor;
+            if (!is_zero(m_attrs.shrink_factor) || !is_zero(m_attrs.zoom_factor))
+            {
+                if (!is_zero(m_attrs.zoom_factor))
+                {
+                    scale = m_attrs.zoom_factor;
+                }
+                if (!is_zero(m_attrs.shrink_factor))
+                {
+                    scale /= m_attrs.shrink_factor;
+                }
+            }
+            output_shape[2] = input_shape[2] * scale;
+            output_shape[3] = input_shape[3] * scale;
+        }
+
+        // Override
+        if (m_attrs.height > 0)
+        {
+            output_shape[2] = m_attrs.height;
+        }
+        if (m_attrs.width > 0)
+        {
+            output_shape[3] = m_attrs.width;
+        }
+
+        set_output_type(0, get_input_element_type(0), output_shape);
+    }
+    else
+    {
+        set_output_type(0, get_input_element_type(0), PartialShape::dynamic());
+    }
+}
+
+shared_ptr<Node> op::Interpolate::copy_with_new_args(const NodeVector& new_args) const
+{
+    check_new_args_count(this, new_args);
+    return make_shared<Interpolate>(new_args.at(0), m_attrs);
+}
+
+op::DynInterpolate::DynInterpolate(const std::shared_ptr<Node>& image,
+                                   const std::shared_ptr<Node>& output_shape,
+                                   const InterpolateAttrs& attrs)
+    : Op("DynInterpolate", check_single_output_args({image, output_shape}))
+    , m_attrs(attrs)
+{
+    constructor_validate_and_infer_types();
+}
+
+void op::DynInterpolate::validate_and_infer_types()
+{
+    set_input_is_relevant_to_shape(1);
+    if (auto const_shape = dynamic_pointer_cast<op::Constant>(get_argument(1)))
+    {
+        NODE_VALIDATION_CHECK(this,
+                              shape_size(const_shape->get_shape()) == 4,
+                              "Layer shape must have rank 4",
+                              const_shape->get_shape());
+
+        auto out_shape = static_cast<const int64_t*>(const_shape->get_data_ptr());
+        Shape output_shape;
+        for (size_t i = 0; i < 4; i++)
+        {
+            output_shape.push_back((out_shape[i] > 0) ? out_shape[i] : 0);
+        }
+        set_output_type(0, get_input_element_type(0), output_shape);
+    }
+    else
+    {
+        set_output_type(0, get_input_element_type(0), PartialShape::dynamic());
+    }
+}
+
+shared_ptr<Node> op::DynInterpolate::copy_with_new_args(const NodeVector& new_args) const
+{
+    check_new_args_count(this, new_args);
+    return make_shared<DynInterpolate>(new_args.at(0), new_args.at(1), m_attrs);
+}

src/ngraph/op/experimental/layers/interpolate.hpp

+//*****************************************************************************
+// Copyright 2017-2019 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#pragma once
+
+#include "ngraph/op/op.hpp"
+
+namespace ngraph
+{
+    namespace op
+    {
+        typedef struct
+        {
+            int height = -1;
+            int width = -1;
+            float zoom_factor = 0;
+            float shrink_factor = 0;
+            float scale_factor = 1.0;
+            bool align_coreners = true;
+            int pad_beg = 0;
+            int pad_end = 0;
+        } InterpolateAttrs;
+
+        /// \brief Layer which performs bilinear interpolation
+        class Interpolate : public Op
+        {
+        public:
+            /// \brief Constructs a Interpolate operation
+            ///
+            /// \param image	Input image
+            /// \param attrs    Interpolation attributes
+            Interpolate(const std::shared_ptr<Node>& image, const InterpolateAttrs& attrs);
+
+            void validate_and_infer_types() override;
+
+            virtual std::shared_ptr<Node>
+                copy_with_new_args(const NodeVector& new_args) const override;
+
+        private:
+            InterpolateAttrs m_attrs;
+        };
+
+        /// \brief Layer which performs bilinear interpolation
+        class DynInterpolate : public Op
+        {
+        public:
+            /// \brief Constructs a DynInterpolate operation
+            ///
+            /// \param image	    Input image
+            /// \param output_shape Output shape
+            /// \param attrs        Interpolation attributes
+            DynInterpolate(const std::shared_ptr<Node>& image,
+                           const std::shared_ptr<Node>& output_shape,
+                           const InterpolateAttrs& attrs);
+
+            void validate_and_infer_types() override;
+
+            virtual std::shared_ptr<Node>
+                copy_with_new_args(const NodeVector& new_args) const override;
+
+        private:
+            InterpolateAttrs m_attrs;
+        };
+    }
+}

test/type_prop_layers.cpp

@@ -17,6 +17,8 @@
 #include "gtest/gtest.h"
 
 #include "ngraph/ngraph.hpp"
+#include "ngraph/op/experimental/layers/detection_output.hpp"
+#include "ngraph/op/experimental/layers/interpolate.hpp"
 #include "ngraph/op/experimental/layers/prior_box.hpp"
 #include "ngraph/op/experimental/layers/prior_box_clustered.hpp"
 #include "ngraph/op/experimental/layers/proposal.hpp"
@@ -27,6 +29,55 @@
 using namespace std;
 using namespace ngraph;
 
+TEST(type_prop_layers, detection_output)
+{
+    auto box_logits = make_shared<op::Parameter>(element::f32, Shape{4, 1, 5, 5});
+    auto class_preds = make_shared<op::Parameter>(element::f32, Shape{2, 1, 4, 5});
+    auto proposals = make_shared<op::Parameter>(element::f32, Shape{2, 1, 4, 5});
+    auto aux_class_preds = make_shared<op::Parameter>(element::f32, Shape{2, 1, 4, 5});
+    auto aux_box_preds = make_shared<op::Parameter>(element::f32, Shape{2, 1, 4, 5});
+    op::DetectionOutputAttrs attrs;
+    attrs.boxes_top_k = {200};
+    auto op = make_shared<op::DetectionOutput>(
+        box_logits, class_preds, proposals, aux_class_preds, aux_box_preds, attrs);
+    ASSERT_EQ(op->get_shape(), (Shape{1, 1, 800, 7}));
+}
+
+TEST(type_prop_layers, interpolate)
+{
+    auto image = make_shared<op::Parameter>(element::f32, Shape{2, 2, 33, 65});
+
+    op::InterpolateAttrs attrs1;
+    attrs1.height = 257;
+    attrs1.width = 513;
+    auto op1 = make_shared<op::Interpolate>(image, attrs1);
+    ASSERT_EQ(op1->get_shape(), (Shape{2, 2, 257, 513}));
+
+    op::InterpolateAttrs attrs2;
+    attrs2.scale_factor = 2;
+    attrs2.width = 513;
+    auto op2 = make_shared<op::Interpolate>(image, attrs2);
+    ASSERT_EQ(op2->get_shape(), (Shape{2, 2, 66, 513}));
+
+    op::InterpolateAttrs attrs3;
+    attrs3.scale_factor = 2;
+    attrs3.height = 257;
+    auto op3 = make_shared<op::Interpolate>(image, attrs3);
+    ASSERT_EQ(op3->get_shape(), (Shape{2, 2, 257, 130}));
+}
+
+TEST(type_prop_layers, dyn_interpolate)
+{
+    auto image = make_shared<op::Parameter>(element::f32, Shape{2, 2, 33, 65});
+    auto output_shape = op::Constant::create<int64_t>(element::i64, Shape{4}, {4, 2, 15, 30});
+
+    op::InterpolateAttrs attrs;
+    attrs.height = 257;
+    attrs.width = 513;
+    auto op = make_shared<op::DynInterpolate>(image, output_shape, attrs);
+    ASSERT_EQ(op->get_shape(), (Shape{4, 2, 15, 30}));
+}
+
 TEST(type_prop_layers, prior_box1)
 {
     auto layer_shape = op::Constant::create<int64_t>(element::i64, Shape{2}, {32, 32});