[SPEC] Add DeformablePSROIPooling v1 (#3954)

* Initial commit

* Moved DeformablePSROIPooling to v1

* Moved DeformablePSROIPooling to v1. Part.2

* Added missing fields

* Added inferance shape

* Added type prop UT

* Added serialization

* Doc + styles applied

* Revert incorrect changes

* Revert incorrect changes. Part.2

* Moved to NGRAPH_API

* integration with master

* Code review remarks introduced

* DeformablePSROIPooling updated to new spec

src/ngraph/CMakeLists.txt

@@ -150,6 +150,8 @@ set (SRC
     op/cum_sum.hpp
     op/crop_and_resize.cpp
     op/crop_and_resize.hpp
+    op/deformable_psroi_pooling.cpp
+    op/deformable_psroi_pooling.hpp
     op/dequantize.cpp
     op/dequantize.hpp
     op/divide.cpp

src/ngraph/op/deformable_psroi_pooling.cpp

+//*****************************************************************************
+// Copyright 2017-2019 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#include "deformable_psroi_pooling.hpp"
+
+using namespace std;
+using namespace ngraph;
+
+constexpr NodeTypeInfo op::v1::DeformablePSROIPooling::type_info;
+
+op::v1::DeformablePSROIPooling::DeformablePSROIPooling(const Output<Node>& input,
+                                                       const Output<Node>& coords,
+                                                       const Output<Node>& offsets,
+                                                       const int64_t output_dim,
+                                                       const float spatial_scale,
+                                                       const int64_t group_size,
+                                                       const std::string mode,
+                                                       int64_t spatial_bins_x,
+                                                       int64_t spatial_bins_y,
+                                                       float trans_std,
+                                                       int64_t part_size)
+    : Op({input, coords, offsets})
+    , m_output_dim(output_dim)
+    , m_spatial_scale(spatial_scale)
+    , m_group_size(group_size)
+    , m_mode(mode)
+    , m_spatial_bins_x(spatial_bins_x)
+    , m_spatial_bins_y(spatial_bins_y)
+    , m_trans_std(trans_std)
+    , m_part_size(part_size)
+{
+    constructor_validate_and_infer_types();
+}
+
+void op::v1::DeformablePSROIPooling::validate_and_infer_types()
+{
+    const auto& input_et = get_input_element_type(0);
+
+    const auto& input_pshape = get_input_partial_shape(0);
+    const auto& box_coords_pshape = get_input_partial_shape(1);
+
+    NODE_VALIDATION_CHECK(this,
+                          input_pshape.rank().is_dynamic() ||
+                              static_cast<size_t>(input_pshape.rank()) == 4,
+                          "Feature map input rank must equal to 4 (input rank: ",
+                          static_cast<size_t>(input_pshape.rank()),
+                          ")");
+    NODE_VALIDATION_CHECK(this,
+                          box_coords_pshape.rank().is_dynamic() ||
+                              static_cast<size_t>(box_coords_pshape.rank()) == 2,
+                          "Box coordinates input rank must equal to 2 (input rank: ",
+                          static_cast<size_t>(box_coords_pshape.rank()),
+                          ")");
+
+    if (get_input_size() == 3) // offsets input is provided
+    {
+        const auto& offsets_pshape = get_input_partial_shape(2);
+        NODE_VALIDATION_CHECK(this,
+                              offsets_pshape.rank().is_dynamic() ||
+                                  static_cast<size_t>(offsets_pshape.rank()) == 4,
+                              "Offsets input rank must equal to 4 (input rank: ",
+                              static_cast<size_t>(offsets_pshape.rank()),
+                              ")");
+    }
+    int64_t output_rank = 4;
+    std::vector<Dimension> output_dim_vec(output_rank, Dimension::dynamic());
+    if (box_coords_pshape[0].is_static())
+    {
+        output_dim_vec[0] = box_coords_pshape.to_shape()[0];
+    }
+    output_dim_vec[1] = m_output_dim;
+    for (int i = 2; i < output_rank; ++i)
+    {
+        output_dim_vec[i] = m_group_size;
+    }
+
+    set_output_type(0, input_et, PartialShape(output_dim_vec));
+}
+
+shared_ptr<Node>
+    op::v1::DeformablePSROIPooling::copy_with_new_args(const NodeVector& new_args) const
+{
+    check_new_args_count(this, new_args);
+    if (new_args.size() == 3)
+    {
+        return make_shared<v1::DeformablePSROIPooling>(new_args.at(0),
+                                                       new_args.at(1),
+                                                       new_args.at(2),
+                                                       m_output_dim,
+                                                       m_spatial_scale,
+                                                       m_group_size,
+                                                       m_mode,
+                                                       m_spatial_bins_x,
+                                                       m_spatial_bins_y,
+                                                       m_trans_std,
+                                                       m_part_size);
+    }
+    else
+    {
+        throw ngraph_error("Not supported number of DeformablePSROIPooling args");
+    }
+}

src/ngraph/op/deformable_psroi_pooling.hpp

+//*****************************************************************************
+// Copyright 2017-2019 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#pragma once
+
+#include "ngraph/op/op.hpp"
+
+namespace ngraph
+{
+    namespace op
+    {
+        namespace v1
+        {
+            class NGRAPH_API DeformablePSROIPooling : public Op
+            {
+            public:
+                static constexpr NodeTypeInfo type_info{"DeformablePSROIPooling", 1};
+                const NodeTypeInfo& get_type_info() const override { return type_info; }
+                DeformablePSROIPooling() = default;
+                /// \brief Constructs a DeformablePSROIPooling operation
+                ///
+                /// \param input           Input tensor with feature maps
+                /// \param coords          Input tensor describing box consisting
+                ///                        of five element tuples
+                /// \param offsets         Input blob with transformation values
+                /// \param output_dim      Pooled output channel number
+                /// \param group_size      Number of groups to encode position-sensitive score maps
+                /// \param spatial_scale   Multiplicative spatial scale factor to translate ROI
+                ///                        coordinates from their input scale to the scale used when
+                ///                        pooling
+                /// \param mode            Specifies mode for pooling.
+                /// \param spatial_bins_x  Specifies numbers of bins to divide the input feature
+                ///                         maps over width
+                /// \param spatial_bins_y  Specifies numbers of bins to divide the input feature
+                ///                        maps over height
+                /// \param no_trans        The flag that specifies whenever third input exists
+                ///                        and contains transformation (offset) values
+                /// \param trans_std       The value that all transformation (offset) values are
+                ///                        multiplied with
+                /// \param part_size       The number of parts the output tensor spatial dimensions
+                ///                        are divided into. Basically it is the height
+                ///                        and width of the third input
+                DeformablePSROIPooling(const Output<Node>& input,
+                                       const Output<Node>& coords,
+                                       const Output<Node>& offsets,
+                                       const int64_t output_dim,
+                                       const float spatial_scale,
+                                       const int64_t group_size = 1,
+                                       const std::string mode = "bilinear_deformable",
+                                       int64_t spatial_bins_x = 1,
+                                       int64_t spatial_bins_y = 1,
+                                       float trans_std = 1,
+                                       int64_t part_size = 1);
+
+                void validate_and_infer_types() override;
+
+                virtual std::shared_ptr<Node>
+                    copy_with_new_args(const NodeVector& new_args) const override;
+
+                int64_t get_output_dim() const { return m_output_dim; }
+                int64_t get_group_size() const { return m_group_size; }
+                float get_spatial_scale() const { return m_spatial_scale; }
+                const std::string& get_mode() const { return m_mode; }
+                int64_t get_spatial_bins_x() const { return m_spatial_bins_x; }
+                int64_t get_spatial_bins_y() const { return m_spatial_bins_y; }
+                float get_trans_std() const { return m_trans_std; }
+                int64_t get_part_size() const { return m_part_size; }
+            private:
+                int64_t m_output_dim;
+                float m_spatial_scale;
+                int64_t m_group_size;
+                std::string m_mode;
+                int64_t m_spatial_bins_x;
+                int64_t m_spatial_bins_y;
+                float m_trans_std;
+                int64_t m_part_size;
+            };
+        }
+    }
+}

src/ngraph/op/op_version_tbl.hpp

@@ -76,6 +76,7 @@ NGRAPH_OP(CropAndResize, ngraph::op, 0)
 NGRAPH_OP(CrossEntropy, ngraph::op, 0)
 NGRAPH_OP(CrossEntropyBackprop, ngraph::op, 0)
 NGRAPH_OP(CumSum, ngraph::op::v0, 0)
+NGRAPH_OP(DeformablePSROIPooling, ngraph::op::v1, 1)
 NGRAPH_OP(DepthToSpace, ngraph::op::v0, 1)
 NGRAPH_OP(Dequantize, ngraph::op, 0)
 NGRAPH_OP(DetectionOutput, ngraph::op::v0, 0)

src/ngraph/ops.hpp

@@ -45,6 +45,7 @@
 #include "ngraph/op/cosh.hpp"
 #include "ngraph/op/crop_and_resize.hpp"
 #include "ngraph/op/cum_sum.hpp"
+#include "ngraph/op/deformable_psroi_pooling.hpp"
 #include "ngraph/op/dequantize.hpp"
 #include "ngraph/op/divide.hpp"
 #include "ngraph/op/dot.hpp"

src/ngraph/opsets/opset1_tbl.hpp

@@ -70,6 +70,7 @@ NGRAPH_OP(Convolution, ngraph::op::v1)
 NGRAPH_OP(ConvolutionBackpropData, ngraph::op::v1)
 NGRAPH_OP(Cos, ngraph::op::v0)
 NGRAPH_OP(Cosh, ngraph::op::v0)
+NGRAPH_OP(DeformablePSROIPooling, ngraph::op::v1)
 NGRAPH_OP(DepthToSpace, ngraph::op::v0)
 NGRAPH_OP(DetectionOutput, ngraph::op::v0)
 NGRAPH_OP(Divide, ngraph::op::v1)

src/ngraph/serializer.cpp

@@ -1325,6 +1325,30 @@ shared_ptr<Node> JSONDeserializer::deserialize_node(json node_js)
         }
         case OP_TYPEID::CTCGreedyDecoder: { break;
         }
+        case OP_TYPEID::DeformablePSROIPooling_v1:
+        {
+            const auto output_dim = node_js.at("output_dim").get<int64_t>();
+            const auto spatial_scale = node_js.at("spatial_scale").get<float>();
+            const auto group_size = node_js.at("group_size").get<int64_t>();
+            const auto mode = node_js.at("mode").get<std::string>();
+            const auto spatial_bins_x = node_js.at("spatial_bins_x").get<int64_t>();
+            const auto spatial_bins_y = node_js.at("spatial_bins_y").get<int64_t>();
+            const auto trans_std = node_js.at("trans_std").get<float>();
+            const auto part_size = node_js.at("part_size").get<int64_t>();
+
+            node = make_shared<op::v1::DeformablePSROIPooling>(args[0],
+                                                               args[1],
+                                                               args[2],
+                                                               output_dim,
+                                                               spatial_scale,
+                                                               group_size,
+                                                               mode,
+                                                               spatial_bins_x,
+                                                               spatial_bins_y,
+                                                               trans_std,
+                                                               part_size);
+            break;
+        }
         case OP_TYPEID::DepthToSpace_v1:
         {
             auto mode = node_js.at("mode").get<op::DepthToSpace::DepthToSpaceMode>();
@@ -3342,6 +3366,19 @@ json JSONSerializer::serialize_node(const Node& n)
     case OP_TYPEID::ReorgYolo: { break;
     }
 
+    case OP_TYPEID::DeformablePSROIPooling_v1:
+    {
+        auto tmp = static_cast<const op::v1::DeformablePSROIPooling*>(&n);
+        node["output_dim"] = tmp->get_output_dim();
+        node["group_size"] = tmp->get_group_size();
+        node["spatial_scale"] = tmp->get_spatial_scale();
+        node["mode"] = tmp->get_mode();
+        node["spatial_bins_x"] = tmp->get_spatial_bins_x();
+        node["spatial_bins_y"] = tmp->get_spatial_bins_y();
+        node["trans_std"] = tmp->get_trans_std();
+        node["part_size"] = tmp->get_part_size();
+        break;
+    }
     case OP_TYPEID::Dequantize:
     {
         auto tmp = static_cast<const op::Dequantize*>(&n);

test/CMakeLists.txt

@@ -120,6 +120,7 @@ set(SRC
     type_prop/convolution.cpp
     type_prop/convolution_bias.cpp
     type_prop/crop_and_resize.cpp
+    type_prop/deformable_psroi_pooling.cpp
     type_prop/depth_to_space.cpp
     type_prop/dequantize.cpp
     type_prop/dot.cpp

test/opset1.cpp

@@ -54,7 +54,7 @@ TEST(opset, check_opset1)
     CHECK_OPSET(op::v0::Cosh, opset1::Cosh)
     CHECK_OPSET(op::v0::CTCGreedyDecoder, opset1::CTCGreedyDecoder)
     // TODO: using op::v0::DeformableConvolution
-    // TODO: using op::v0::DeformablePSROIPooling
+    CHECK_OPSET(op::v1::DeformablePSROIPooling, opset1::DeformablePSROIPooling)
     CHECK_OPSET(op::v0::DepthToSpace, opset1::DepthToSpace)
     CHECK_OPSET(op::v0::DetectionOutput, opset1::DetectionOutput)
     CHECK_OPSET(op::v1::Divide, opset1::Divide)

test/serialize.cpp

@@ -804,3 +804,51 @@ TEST(serialize, space_to_depth)
     EXPECT_EQ(depth_to_space_out->get_block_size(), block_size);
     EXPECT_EQ(depth_to_space_out->get_mode(), mode);
 }
+
+TEST(serialize, deformable_psroi_pooling)
+{
+    auto input = make_shared<op::Parameter>(element::f32, Shape{1, 2, 3, 4});
+    auto coords = make_shared<op::Parameter>(element::f32, Shape{1, 1});
+    auto offsets = make_shared<op::Parameter>(element::f32, Shape{1, 2, 3, 4});
+    const int64_t output_dim = 1;
+    const int64_t group_size = 2;
+    const float spatial_scale = 3;
+    std::string mode = "bilinear_deformable";
+    int64_t spatial_bins_x = 4;
+    int64_t spatial_bins_y = 5;
+    float trans_std = 6.1f;
+    int64_t part_size = 7;
+
+    auto def_psroi_pool_in = make_shared<op::v1::DeformablePSROIPooling>(input,
+                                                                         coords,
+                                                                         offsets,
+                                                                         output_dim,
+                                                                         spatial_scale,
+                                                                         group_size,
+                                                                         mode,
+                                                                         spatial_bins_x,
+                                                                         spatial_bins_y,
+                                                                         trans_std,
+                                                                         part_size);
+
+    auto result = make_shared<op::Result>(def_psroi_pool_in);
+    auto f = make_shared<Function>(ResultVector{result}, ParameterVector{input, coords, offsets});
+    string s = serialize(f);
+
+    shared_ptr<Function> g = deserialize(s);
+    auto g_result = g->get_results().at(0);
+    auto g_def_psroi_pool = g_result->input(0).get_source_output().get_node_shared_ptr();
+    auto def_psroi_pool_out = as_type_ptr<op::v1::DeformablePSROIPooling>(g_def_psroi_pool);
+
+    EXPECT_EQ(def_psroi_pool_out->description(), "DeformablePSROIPooling");
+    EXPECT_EQ(def_psroi_pool_out->get_version(), 1);
+
+    EXPECT_EQ(def_psroi_pool_out->get_output_dim(), output_dim);
+    EXPECT_EQ(def_psroi_pool_out->get_group_size(), group_size);
+    EXPECT_EQ(def_psroi_pool_out->get_spatial_scale(), spatial_scale);
+    EXPECT_EQ(def_psroi_pool_out->get_mode(), mode);
+    EXPECT_EQ(def_psroi_pool_out->get_spatial_bins_x(), spatial_bins_x);
+    EXPECT_EQ(def_psroi_pool_out->get_spatial_bins_y(), spatial_bins_y);
+    EXPECT_EQ(def_psroi_pool_out->get_trans_std(), trans_std);
+    EXPECT_EQ(def_psroi_pool_out->get_part_size(), part_size);
+}

test/type_prop/deformable_psroi_pooling.cpp

+//*****************************************************************************
+// Copyright 2017-2019 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#include "gtest/gtest.h"
+#include "ngraph/ngraph.hpp"
+#include "util/type_prop.hpp"
+
+using namespace std;
+using namespace ngraph;
+
+TEST(type_prop, deformable_psroi_pooling_output_shape)
+{
+    auto input = make_shared<op::Parameter>(element::f32, Shape{1, 1024, 63, 38});
+    auto coords = make_shared<op::Parameter>(element::f32, Shape{300, 5});
+    auto offsets = make_shared<op::Parameter>(element::f32, Shape{1, 2, 3, 4});
+    const int64_t output_dim = 882;
+    const float spatial_scale = 0.0625;
+    const int64_t group_size = 3;
+
+    auto def_psroi_pool = make_shared<op::v1::DeformablePSROIPooling>(
+        input, coords, offsets, output_dim, spatial_scale, group_size);
+
+    ASSERT_EQ(def_psroi_pool->get_output_shape(0), (Shape{300, 882, 3, 3}));
+}
+
+TEST(type_prop, deformable_psroi_pooling_output_shape_2)
+{
+    auto input = make_shared<op::Parameter>(element::f32, Shape{1, 7938, 38, 38});
+    auto coords = make_shared<op::Parameter>(element::f32, Shape{300, 5});
+    auto offsets = make_shared<op::Parameter>(element::f32, Shape{1, 2, 3, 4});
+    const int64_t output_dim = 162;
+    const float spatial_scale = 0.0625;
+    const int64_t group_size = 7;
+
+    auto def_psroi_pool = make_shared<op::v1::DeformablePSROIPooling>(
+        input, coords, offsets, output_dim, spatial_scale, group_size);
+
+    ASSERT_EQ(def_psroi_pool->get_output_shape(0), (Shape{300, 162, 7, 7}));
+}
+
+TEST(type_prop, deformable_psroi_pooling_invalid_input_rank)
+{
+    auto input = make_shared<op::Parameter>(element::f32, Shape{1, 2, 3});
+    auto coords = make_shared<op::Parameter>(element::f32, Shape{1, 2});
+    auto offsets = make_shared<op::Parameter>(element::f32, Shape{1, 2, 3, 4});
+    const int64_t output_dim = 4;
+    const float spatial_scale = 0.9;
+    const int64_t group_size = 7;
+    try
+    {
+        auto def_psroi_pool = make_shared<op::v1::DeformablePSROIPooling>(
+            input, coords, offsets, output_dim, spatial_scale, group_size);
+        // Should have thrown, so fail if it didn't
+        FAIL() << "Ivalid feature map input rank not detected";
+    }
+    catch (const NodeValidationFailure& error)
+    {
+        EXPECT_HAS_SUBSTRING(error.what(),
+                             std::string("Feature map input rank must equal to 4 (input rank: 3)"));
+    }
+    catch (...)
+    {
+        FAIL() << "Deduced type check failed for unexpected reason";
+    }
+}
+
+TEST(type_prop, deformable_psroi_pooling_invalid_box_coordinates_rank)
+{
+    auto input = make_shared<op::Parameter>(element::f32, Shape{1, 2, 3, 4});
+    auto coords = make_shared<op::Parameter>(element::f32, Shape{1, 2, 3});
+    auto offsets = make_shared<op::Parameter>(element::f32, Shape{1, 2, 3, 4});
+    const int64_t output_dim = 4;
+    const float spatial_scale = 0.9;
+    const int64_t group_size = 7;
+    try
+    {
+        auto def_psroi_pool = make_shared<op::v1::DeformablePSROIPooling>(
+            input, coords, offsets, output_dim, spatial_scale, group_size);
+        // Should have thrown, so fail if it didn't
+        FAIL() << "Ivalid box coordinates input rank not detected";
+    }
+    catch (const NodeValidationFailure& error)
+    {
+        EXPECT_HAS_SUBSTRING(
+            error.what(),
+            std::string("Box coordinates input rank must equal to 2 (input rank: 3)"));
+    }
+    catch (...)
+    {
+        FAIL() << "Deduced type check failed for unexpected reason";
+    }
+}
+
+TEST(type_prop, deformable_psroi_pooling_invalid_offstes_rank)
+{
+    auto input = make_shared<op::Parameter>(element::f32, Shape{1, 2, 3, 4});
+    auto coords = make_shared<op::Parameter>(element::f32, Shape{1, 2});
+    auto offsets = make_shared<op::Parameter>(element::f32, Shape{1, 2, 3, 4, 5});
+    const int64_t output_dim = 4;
+    const float spatial_scale = 0.9;
+    const int64_t group_size = 7;
+    try
+    {
+        auto def_psroi_pool = make_shared<op::v1::DeformablePSROIPooling>(
+            input, coords, offsets, output_dim, spatial_scale, group_size);
+        // Should have thrown, so fail if it didn't
+        FAIL() << "Offsets input rank not detected";
+    }
+    catch (const NodeValidationFailure& error)
+    {
+        EXPECT_HAS_SUBSTRING(error.what(),
+                             std::string("Offsets input rank must equal to 4 (input rank: 5)"));
+    }
+    catch (...)
+    {
+        FAIL() << "Deduced type check failed for unexpected reason";
+    }
+}