[SPEC] DeformableConvolution (#3964)

src/ngraph/CMakeLists.txt

@@ -150,6 +150,8 @@ set (SRC
     op/cum_sum.hpp
     op/crop_and_resize.cpp
     op/crop_and_resize.hpp
+    op/deformable_convolution.cpp
+    op/deformable_convolution.hpp
     op/deformable_psroi_pooling.cpp
     op/deformable_psroi_pooling.hpp
     op/dequantize.cpp

src/ngraph/op/deformable_convolution.cpp

+//*****************************************************************************
+// Copyright 2017-2019 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#include "ngraph/op/deformable_convolution.hpp"
+#include "ngraph/axis_vector.hpp"
+#include "ngraph/coordinate_diff.hpp"
+#include "ngraph/op/reshape.hpp"
+#include "ngraph/op/reverse.hpp"
+#include "ngraph/util.hpp"
+#include "ngraph/validation_util.hpp"
+
+using namespace std;
+using namespace ngraph;
+
+constexpr NodeTypeInfo op::v1::DeformableConvolution::type_info;
+
+op::v1::DeformableConvolution::DeformableConvolution(const Output<Node>& arg,
+                                                     const Output<Node>& deformable_values,
+                                                     const Output<Node>& filters,
+                                                     const Strides& strides,
+                                                     const CoordinateDiff& pads_begin,
+                                                     const CoordinateDiff& pads_end,
+                                                     const Strides& dilations,
+                                                     const PadType& auto_pad,
+                                                     const size_t group,
+                                                     const size_t deformable_group)
+    : Op({arg, deformable_values, filters})
+    , m_strides(strides)
+    , m_dilations(dilations)
+    , m_pads_begin(pads_begin)
+    , m_pads_end(pads_end)
+    , m_auto_pad(auto_pad)
+    , m_group(group)
+    , m_deformable_group(deformable_group)
+{
+    constructor_validate_and_infer_types();
+}
+
+void op::v1::DeformableConvolution::validate_and_infer_types()
+{
+    const PartialShape& data_batch_shape = get_input_partial_shape(0);
+    const PartialShape& deformable_values_shape = get_input_partial_shape(1);
+    const PartialShape& filters_shape = get_input_partial_shape(2);
+
+    element::Type data_batch_et = get_input_element_type(0);
+    element::Type deformable_values_et = get_input_element_type(1);
+    element::Type filters_et = get_input_element_type(2);
+
+    if (m_strides.size() == 0)
+    {
+        m_strides = conv_default_strides(this, data_batch_shape, filters_shape);
+    }
+
+    if (m_dilations.size() == 0)
+    {
+        m_dilations = conv_default_strides(this, data_batch_shape, filters_shape);
+    }
+
+    if (m_pads_begin.size() == 0)
+    {
+        m_pads_begin = conv_default_padding(this, data_batch_shape, filters_shape);
+    }
+
+    if (m_pads_end.size() == 0)
+    {
+        m_pads_end = conv_default_padding(this, data_batch_shape, filters_shape);
+    }
+
+    if (m_auto_pad == PadType::SAME_UPPER || m_auto_pad == PadType::SAME_LOWER)
+    {
+        if (data_batch_shape.is_static() && filters_shape.is_static())
+        {
+            m_pads_begin.clear();
+            m_pads_end.clear();
+            auto filter_shape = filters_shape.to_shape();
+            filter_shape.erase(filter_shape.begin(), filter_shape.begin() + 2); // Remove {O,I}
+            infer_auto_padding(data_batch_shape.to_shape(),
+                               filter_shape,
+                               m_strides,
+                               m_dilations,
+                               m_auto_pad,
+                               m_pads_end,
+                               m_pads_begin);
+        }
+    }
+
+    if (deformable_values_shape.rank().is_static())
+    {
+        NODE_VALIDATION_CHECK(
+            this,
+            static_cast<size_t>(deformable_values_shape.rank()) >= 3u,
+            "The deformable values tensor rank is expected to be at least 3, got: ",
+            deformable_values_shape.rank());
+    }
+
+    if (m_group > 1 && data_batch_shape[1].is_static() && filters_shape[0].is_static())
+    {
+        NODE_VALIDATION_CHECK(this,
+                              static_cast<size_t>(data_batch_shape[1]) % m_group == 0,
+                              "The input data shape must be evenly divisible by the 'group' value "
+                              "along the channels axis. Current input shape: ",
+                              data_batch_shape,
+                              ", 'group' attribute value: ",
+                              m_group);
+
+        NODE_VALIDATION_CHECK(
+            this,
+            static_cast<size_t>(filters_shape[0]) % m_group == 0,
+            "The weights shape must be evenly divisible by the 'group' value along "
+            "the channels axis. Current weights shape: ",
+            filters_shape,
+            ", 'group' attribute value: ",
+            m_group);
+    }
+
+    if (m_deformable_group > 1 && deformable_values_shape[1].is_static())
+    {
+        NODE_VALIDATION_CHECK(
+            this,
+            static_cast<size_t>(deformable_values_shape[1]) % m_deformable_group == 0,
+            "The deformable values input must be evenly divisible by the 'deformable group' value "
+            "along the channels axis. Current input shape: ",
+            deformable_values_shape,
+            ", 'deformable group' attribute value: ",
+            m_deformable_group);
+    }
+
+    element::Type result_et;
+    NODE_VALIDATION_CHECK(
+        this,
+        element::Type::merge(result_et, data_batch_et, filters_et),
+        "Element types for data batch and filters do not match (data batch element type: ",
+        data_batch_et,
+        ", filters element type: ",
+        filters_et,
+        ").");
+
+    const PartialShape result_shape =
+        infer_convolution_forward(this,
+                                  data_batch_shape,
+                                  Strides(m_strides.size(), 1), // dummy data dilations
+                                  m_pads_begin,
+                                  m_pads_end,
+                                  filters_shape,
+                                  m_strides,
+                                  m_dilations);
+
+    set_output_type(0, result_et, result_shape);
+}
+
+shared_ptr<Node> op::v1::DeformableConvolution::copy_with_new_args(const NodeVector& new_args) const
+{
+    check_new_args_count(this, new_args);
+    return make_shared<v1::DeformableConvolution>(new_args.at(0),
+                                                  new_args.at(1),
+                                                  new_args.at(2),
+                                                  m_strides,
+                                                  m_pads_begin,
+                                                  m_pads_end,
+                                                  m_dilations,
+                                                  m_auto_pad,
+                                                  m_group,
+                                                  m_deformable_group);
+}

src/ngraph/op/deformable_convolution.hpp

+//*****************************************************************************
+// Copyright 2017-2019 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//*****************************************************************************
+
+#pragma once
+
+#include "ngraph/coordinate_diff.hpp"
+#include "ngraph/op/op.hpp"
+#include "ngraph/op/util/attr_types.hpp"
+
+namespace ngraph
+{
+    namespace op
+    {
+        namespace v1
+        {
+            /// \brief DeformableConvolution operation.
+            class NGRAPH_API DeformableConvolution : public Op
+            {
+            public:
+                static constexpr NodeTypeInfo type_info{"DeformableConvolution", 1};
+                const NodeTypeInfo& get_type_info() const override { return type_info; }
+                /// \brief Constructs a conversion operation.
+                DeformableConvolution() = default;
+                /// \brief Constructs a conversion operation.
+                ///
+                /// \param arg                Node that produces the input tensor.
+                /// \param deformable_values  Node producing the deformable values tensor.
+                /// \param filters            Node producing the filters(kernels) tensor wit OIZYX
+                ///                           layout.
+                /// \param strides            Convolution strides.
+                /// \param pads_begin         Amount of padding to be added to the beginning along
+                ///                           each axis. For example in case of a 2D input the value
+                ///                           of (1, 2) means that 1 element will be added to the
+                ///                           top and 2 elements to the left.
+                /// \param pads_end           Amount of padding to be added to the end along each
+                ///                           axis.
+                /// \param dilations          The distance in width and height between the weights
+                ///                           in the filters tensor.
+                /// \param auto_pad           Specifies how the automatic calculation of padding
+                ///                           should be done.
+                /// \param group              The number of groups which both output and input
+                ///                           should be split into.
+                /// \param deformable_group   The number of groups which deformable values and
+                ///                           output should be split into along the channel axis.
+                DeformableConvolution(const Output<Node>& arg,
+                                      const Output<Node>& deformable_values,
+                                      const Output<Node>& filters,
+                                      const Strides& strides,
+                                      const CoordinateDiff& pads_begin,
+                                      const CoordinateDiff& pads_end,
+                                      const Strides& dilations,
+                                      const PadType& auto_pad = PadType::EXPLICIT,
+                                      const size_t group = 1,
+                                      const size_t deformable_group = 1);
+
+                void validate_and_infer_types() override;
+
+                const Strides& get_strides() const { return m_strides; }
+                void set_strides(const Strides& strides) { m_strides = strides; }
+                const Strides& get_dilations() const { return m_dilations; }
+                void set_dilations(const Strides& dilations) { m_dilations = dilations; }
+                const CoordinateDiff& get_pads_begin() const { return m_pads_begin; }
+                void set_pads_begin(const CoordinateDiff& pads_begin) { m_pads_begin = pads_begin; }
+                const CoordinateDiff& get_pads_end() const { return m_pads_end; }
+                void set_pads_end(const CoordinateDiff& pads_end) { m_pads_end = pads_end; }
+                const PadType& get_auto_pad() const { return m_auto_pad; }
+                void set_auto_pad(const PadType& auto_pad) { m_auto_pad = auto_pad; }
+                size_t get_group() const { return m_group; }
+                void set_group(const size_t group) { m_group = group; }
+                size_t get_deformable_group() const { return m_deformable_group; }
+                void set_deformable_group(const size_t deformable_group)
+                {
+                    m_deformable_group = deformable_group;
+                }
+
+                virtual std::shared_ptr<Node>
+                    copy_with_new_args(const NodeVector& new_args) const override;
+
+            protected:
+                Strides m_strides;
+                Strides m_dilations;
+                CoordinateDiff m_pads_begin;
+                CoordinateDiff m_pads_end;
+                PadType m_auto_pad;
+                size_t m_group;
+                size_t m_deformable_group;
+            };
+        }
+    }
+}

src/ngraph/op/op_version_tbl.hpp

@@ -76,6 +76,7 @@ NGRAPH_OP(CropAndResize, ngraph::op, 0)
 NGRAPH_OP(CrossEntropy, ngraph::op, 0)
 NGRAPH_OP(CrossEntropyBackprop, ngraph::op, 0)
 NGRAPH_OP(CumSum, ngraph::op::v0, 0)
+NGRAPH_OP(DeformableConvolution, ngraph::op::v1, 1)
 NGRAPH_OP(DeformablePSROIPooling, ngraph::op::v1, 1)
 NGRAPH_OP(DepthToSpace, ngraph::op::v0, 1)
 NGRAPH_OP(Dequantize, ngraph::op, 0)

src/ngraph/ops.hpp

@@ -45,6 +45,7 @@
 #include "ngraph/op/cosh.hpp"
 #include "ngraph/op/crop_and_resize.hpp"
 #include "ngraph/op/cum_sum.hpp"
+#include "ngraph/op/deformable_convolution.hpp"
 #include "ngraph/op/deformable_psroi_pooling.hpp"
 #include "ngraph/op/dequantize.hpp"
 #include "ngraph/op/divide.hpp"

src/ngraph/opsets/opset1_tbl.hpp

@@ -70,6 +70,7 @@ NGRAPH_OP(Convolution, ngraph::op::v1)
 NGRAPH_OP(ConvolutionBackpropData, ngraph::op::v1)
 NGRAPH_OP(Cos, ngraph::op::v0)
 NGRAPH_OP(Cosh, ngraph::op::v0)
+NGRAPH_OP(DeformableConvolution, ngraph::op::v1)
 NGRAPH_OP(DeformablePSROIPooling, ngraph::op::v1)
 NGRAPH_OP(DepthToSpace, ngraph::op::v0)
 NGRAPH_OP(DetectionOutput, ngraph::op::v0)

src/ngraph/serializer.cpp

@@ -1325,6 +1325,29 @@ shared_ptr<Node> JSONDeserializer::deserialize_node(json node_js)
         }
         case OP_TYPEID::CTCGreedyDecoder: { break;
         }
+        case OP_TYPEID::DeformableConvolution_v1:
+        {
+            const auto strides = node_js.at("strides").get<vector<size_t>>();
+            const auto dilations = node_js.at("dilations").get<vector<size_t>>();
+            const auto pads_begin = node_js.at("pads_begin").get<vector<std::ptrdiff_t>>();
+            const auto pads_end = node_js.at("pads_end").get<vector<std::ptrdiff_t>>();
+            const auto group = node_js.at("group").get<size_t>();
+            const auto deformable_group = node_js.at("deformable_group").get<size_t>();
+
+            const op::PadType auto_pad = read_pad_type(node_js);
+
+            node = make_shared<op::v1::DeformableConvolution>(args[0],
+                                                              args[1],
+                                                              args[2],
+                                                              strides,
+                                                              pads_begin,
+                                                              pads_end,
+                                                              dilations,
+                                                              auto_pad,
+                                                              group,
+                                                              deformable_group);
+            break;
+        }
         case OP_TYPEID::DeformablePSROIPooling_v1:
         {
             const auto output_dim = node_js.at("output_dim").get<int64_t>();
@@ -3378,7 +3401,18 @@ json JSONSerializer::serialize_node(const Node& n)
     }
     case OP_TYPEID::ReorgYolo: { break;
     }
-
+    case OP_TYPEID::DeformableConvolution_v1:
+    {
+        const auto tmp = static_cast<const op::v1::DeformableConvolution*>(&n);
+        node["strides"] = tmp->get_strides();
+        node["dilations"] = tmp->get_dilations();
+        node["pads_begin"] = tmp->get_pads_begin();
+        node["pads_end"] = tmp->get_pads_end();
+        node["auto_pad"] = tmp->get_auto_pad();
+        node["group"] = tmp->get_group();
+        node["deformable_group"] = tmp->get_deformable_group();
+        break;
+    }
     case OP_TYPEID::DeformablePSROIPooling_v1:
     {
         auto tmp = static_cast<const op::v1::DeformablePSROIPooling*>(&n);

test/type_prop/convolution.cpp

@@ -2862,3 +2862,83 @@ TEST(type_prop, conv_v1_partial_rank)
 
     ASSERT_TRUE(conv->get_output_partial_shape(0).is_dynamic());
 }
+
+TEST(type_prop, deformable_conv_incorrect_group)
+{
+    const PartialShape data_batch_shape{1, 3, 96, 96};
+    const PartialShape deformable_values_shape{1, 50, 5, 5};
+    const PartialShape filters_shape{4, 3, 5, 5};
+
+    auto param0 = make_shared<op::Parameter>(element::f32, data_batch_shape);
+    auto param1 = make_shared<op::Parameter>(element::f32, deformable_values_shape);
+    auto param2 = make_shared<op::Parameter>(element::f32, filters_shape);
+
+    try
+    {
+        make_shared<op::v1::DeformableConvolution>(param0,
+                                                   param1,
+                                                   param2,
+                                                   Strides{},
+                                                   CoordinateDiff{},
+                                                   CoordinateDiff{},
+                                                   Strides{},
+                                                   op::PadType::EXPLICIT,
+                                                   2);
+
+        FAIL() << "DeformableConvolution created with incorrect 'group' value";
+    }
+    catch (const NodeValidationFailure& error)
+    {
+        EXPECT_HAS_SUBSTRING(error.what(), "input data shape must be evenly divisible");
+    }
+
+    try
+    {
+        make_shared<op::v1::DeformableConvolution>(param0,
+                                                   param1,
+                                                   param2,
+                                                   Strides{},
+                                                   CoordinateDiff{},
+                                                   CoordinateDiff{},
+                                                   Strides{},
+                                                   op::PadType::EXPLICIT,
+                                                   3);
+
+        FAIL() << "DeformableConvolution created with incorrect 'group' value";
+    }
+    catch (const NodeValidationFailure& error)
+    {
+        EXPECT_HAS_SUBSTRING(error.what(), "weights shape must be evenly divisible");
+    }
+}
+
+TEST(type_prop, deformable_conv_incorrect_deformable_group)
+{
+    const PartialShape data_batch_shape{1, 3, 96, 96};
+    const PartialShape deformable_values_shape{1, 50, 5, 5};
+    const PartialShape filters_shape{3, 3, 5, 5};
+
+    auto param0 = make_shared<op::Parameter>(element::f32, data_batch_shape);
+    auto param1 = make_shared<op::Parameter>(element::f32, deformable_values_shape);
+    auto param2 = make_shared<op::Parameter>(element::f32, filters_shape);
+
+    try
+    {
+        make_shared<op::v1::DeformableConvolution>(param0,
+                                                   param1,
+                                                   param2,
+                                                   Strides{},
+                                                   CoordinateDiff{},
+                                                   CoordinateDiff{},
+                                                   Strides{},
+                                                   op::PadType::EXPLICIT,
+                                                   1,
+                                                   7);
+
+        FAIL() << "DeformableConvolution created with incorrect 'deformable group' value";
+    }
+    catch (const NodeValidationFailure& error)
+    {
+        EXPECT_HAS_SUBSTRING(error.what(), "deformable values input must be evenly divisible");
+    }
+}