[SPEC] Add new v1::GroupConvolution and v1::GroupConvolutionBackpropData operators. (#3956)

* [SPEC] Add new v1::GroupConvolution and v1::GroupConvolutionBackpropData ops.

* Sort downgrade passes

* Add downgrade pass fro GroupConvolution

* WIP I

* Fix GroupConvolution validate and infer types

* Add upgrade pass for GroupConvolution

* Remove unnecesary get_static_groups() method

* Disable pass for cases when groups are in filters

* Review Fix I

* Move ops to fused/group_conv.hpp

* Use Op instead of FusedOp again

* Move v1::GroupConvolution and v1::GroupConvolutionBackpropData to FusedOp but temporarily disable decomposition

src/ngraph/frontend/onnx_import/op/conv.cpp

@@ -18,13 +18,16 @@
 #include <memory>
 #include <vector>
 
+#include "ngraph/builder/reshape.hpp"
 #include "ngraph/frontend/onnx_import/exceptions.hpp"
 #include "ngraph/frontend/onnx_import/op/conv.hpp"
 #include "ngraph/frontend/onnx_import/utils/convpool.hpp"
 #include "ngraph/op/add.hpp"
 #include "ngraph/op/broadcast.hpp"
 #include "ngraph/op/concat.hpp"
+#include "ngraph/op/constant.hpp"
 #include "ngraph/op/convolution.hpp"
+#include "ngraph/op/fused/group_conv.hpp"
 #include "ngraph/op/slice.hpp"
 #include "ngraph/op/util/attr_types.hpp"
 #include "ngraph/op/util/broadcasting.hpp"
@@ -51,48 +54,21 @@ namespace ngraph
                     {
                         if (groups > 1)
                         {
-                            // Split one convolution op to N ops where N is the number of groups
-                            // and concat results after computation.
-                            // reference:
-                            // https://github.com/NervanaSystems/ngraph-mxnet/blob/fdd692/src/ngraph/ngraph_emitter.cc#L822-L856
-                            std::size_t n_data_channels{data->get_shape().at(1)};
-                            std::size_t n_filters_channels{filters->get_shape().at(0)};
-                            std::size_t data_group_size{n_data_channels / groups};
-                            std::size_t filters_group_size{n_filters_channels / groups};
-                            NodeVector convolution_nodes;
-
-                            // initial bounds for splice
-                            std::vector<std::size_t> data_lower_bounds(data->get_shape().size());
-                            std::vector<std::size_t> data_upper_bounds{data->get_shape()};
-                            std::vector<std::size_t> filters_lower_bounds(
-                                filters->get_shape().size());
-                            std::vector<std::size_t> filters_upper_bounds{filters->get_shape()};
-
-                            for (int64_t group{0}; group < groups; ++group)
-                            {
-                                // slice data
-                                data_lower_bounds[1] = group * data_group_size;
-                                data_upper_bounds[1] = (group + 1) * data_group_size;
-                                auto sliced_data = std::make_shared<ngraph::op::Slice>(
-                                    data, data_lower_bounds, data_upper_bounds);
-                                // slice filters
-                                filters_lower_bounds[0] = group * filters_group_size;
-                                filters_upper_bounds[0] = (group + 1) * filters_group_size;
-                                auto sliced_filters = std::make_shared<ngraph::op::Slice>(
-                                    filters, filters_lower_bounds, filters_upper_bounds);
-
-                                convolution_nodes.push_back(
-                                    std::make_shared<ngraph::op::v1::Convolution>(sliced_data,
-                                                                                  sliced_filters,
-                                                                                  strides,
-                                                                                  padding_below,
-                                                                                  padding_above,
-                                                                                  dilations,
-                                                                                  auto_pad));
-                            }
-                            std::size_t concatenation_axis = 1;
-                            return std::make_shared<ngraph::op::Concat>(convolution_nodes,
-                                                                        concatenation_axis);
+                            auto filters_shape = filters->get_shape();
+                            filters_shape.at(0) = filters_shape.at(0) / groups;
+                            filters_shape.insert(filters_shape.begin(), groups);
+
+                            auto reshaped_filters =
+                                ngraph::builder::reshape(filters, filters_shape);
+
+                            return std::make_shared<ngraph::op::v1::GroupConvolution>(
+                                data,
+                                reshaped_filters,
+                                strides,
+                                padding_below,
+                                padding_above,
+                                dilations,
+                                auto_pad);
                         }
                         else
                         {

src/ngraph/op/op_version_tbl.hpp

@@ -115,7 +115,9 @@ NGRAPH_OP(Greater, ngraph::op::v1, 1)
 NGRAPH_OP(GreaterEq, ngraph::op::v0, 0)
 NGRAPH_OP(GreaterEqual, ngraph::op::v1, 1)
 NGRAPH_OP(GroupConvolution, ngraph::op::v0, 0)
+NGRAPH_OP(GroupConvolution, ngraph::op::v1, 1)
 NGRAPH_OP(GroupConvolutionBackpropData, ngraph::op::v0, 0)
+NGRAPH_OP(GroupConvolutionBackpropData, ngraph::op::v1, 1)
 NGRAPH_OP(GroupConvolutionBackpropFilters, ngraph::op::v0, 0)
 NGRAPH_OP(GroupConvolutionTranspose, ngraph::op::v0, 0)
 NGRAPH_OP(HardSigmoid, ngraph::op::v0, 0)

src/ngraph/opsets/opset1_tbl.hpp

@@ -86,7 +86,8 @@ NGRAPH_OP(Gather, ngraph::op::v1)
 NGRAPH_OP(GatherTree, ngraph::op::v1)
 NGRAPH_OP(Greater, ngraph::op::v1)
 NGRAPH_OP(GreaterEqual, ngraph::op::v1)
-NGRAPH_OP(GroupConvolution, ngraph::op::v0)
+NGRAPH_OP(GroupConvolution, ngraph::op::v1)
+NGRAPH_OP(GroupConvolutionBackpropData, ngraph::op::v1)
 NGRAPH_OP(GRN, ngraph::op::v0)
 NGRAPH_OP(HardSigmoid, ngraph::op::v0)
 NGRAPH_OP(Interpolate, ngraph::op::v0)

src/ngraph/pass/opset0_downgrade.cpp

@@ -19,6 +19,7 @@
 #include <functional>
 #include <numeric>
 
+#include "ngraph/builder/reshape.hpp"
 #include "ngraph/graph_util.hpp"
 #include "ngraph/node.hpp"
 #include "ngraph/op/util/broadcasting.hpp"
@@ -117,12 +118,8 @@ namespace
     {
         const auto data_arg = node->input_value(0);
         const auto filters_arg = node->input_value(1);
-        const PartialShape& data_arg_pshape = node->get_input_partial_shape(0);
-        NGRAPH_CHECK(data_arg_pshape.rank().is_static(),
-                     "Unable to convert Convolution:v1 to Convolution:v0 if data argument "
-                     "rank is dynamic. Node: ",
-                     *node);
-        const size_t num_spatial_dims = static_cast<size_t>(data_arg_pshape.rank()) - 2;
+        const auto strides = node->get_strides();
+        const size_t num_spatial_dims = strides.size();
         auto replacement_node = make_shared<op::v0::Convolution>(data_arg,
                                                                  filters_arg,
                                                                  node->get_strides(),
@@ -140,16 +137,12 @@ namespace
         auto output_shape = as_type_ptr<op::Constant>(node->input_value(2).get_node_shared_ptr());
         const auto data_arg = node->input(0).get_source_output();
         const auto filters_arg = node->input(1).get_source_output();
-        const PartialShape& delta_arg_pshape = node->get_input_partial_shape(1);
-        NGRAPH_CHECK(delta_arg_pshape.rank().is_static(),
-                     "Unable to convert ConvolutionBackpropData:v1 to ConvolutionBackpropData:v0 "
-                     "if delta argument rank is dynamic. Node: ",
-                     *node);
+        const auto strides = node->get_strides();
         NGRAPH_CHECK(output_shape,
                      "Unable to convert ConvolutionBackpropData:v1 to ConvolutionBackpropData:v0 "
                      "if output_shape is not constant. Node: ",
                      *node);
-        const size_t num_spatial_dims = static_cast<size_t>(delta_arg_pshape.rank()) - 2;
+        const size_t num_spatial_dims = strides.size();
 
         auto output_padding = node->get_output_padding();
 
@@ -182,12 +175,8 @@ namespace
                 ->get_shape_val();
         const auto data_arg = node->input_value(0);
         const auto delta_arg = node->input_value(1);
-        const PartialShape& data_arg_pshape = node->get_input_partial_shape(0);
-        NGRAPH_CHECK(data_arg_pshape.rank().is_static(),
-                     "Unable to convert ConvolutionBackpropFilters:v1 to "
-                     "ConvolutionBackpropFilters:v0 if data argument rank is dynamic. Node: ",
-                     *node);
-        const size_t num_spatial_dims = static_cast<size_t>(data_arg_pshape.rank()) - 2;
+        const auto strides = node->get_strides();
+        const size_t num_spatial_dims = strides.size();
         auto replacement_node =
             make_shared<op::v0::ConvolutionBackpropFilters>(data_arg,
                                                             filters_shape,
@@ -256,6 +245,24 @@ namespace
         return true;
     }
 
+    bool op_cast(shared_ptr<op::v1::GroupConvolution> node)
+    {
+        const auto data_arg = node->input_value(0);
+        const auto filters_arg = node->input_value(1);
+        const auto strides = node->get_strides();
+        const size_t num_spatial_dims = strides.size();
+        auto replacement_node = make_shared<op::GroupConvolution>(data_arg,
+                                                                  filters_arg,
+                                                                  node->get_strides(),
+                                                                  node->get_dilations(),
+                                                                  node->get_pads_begin(),
+                                                                  node->get_pads_end(),
+                                                                  Strides(num_spatial_dims, 1),
+                                                                  node->get_auto_pad());
+        replace_node(node, replacement_node);
+        return true;
+    }
+
     bool op_cast(shared_ptr<op::v1::Less> node)
     {
         op_cast_binary_elementwise_node<op::v0::Less, op::v1::Less>(node);

src/ngraph/pass/opset1_upgrade.cpp

@@ -18,6 +18,7 @@
 #include <limits>
 #include <numeric>
 
+#include "ngraph/builder/reshape.hpp"
 #include "ngraph/graph_util.hpp"
 #include "ngraph/ops.hpp"
 #include "ngraph/pass/opset1_upgrade.hpp"
@@ -136,11 +137,9 @@ namespace
         auto data_dilation_strides = node->get_data_dilation_strides();
         auto auto_pad = node->get_pad_type();
 
-        bool is_dds_valid = true;
-        for (auto value : data_dilation_strides)
-        {
-            is_dds_valid = is_dds_valid && (value == 1);
-        }
+        bool is_dds_valid = all_of(data_dilation_strides.begin(),
+                                   data_dilation_strides.end(),
+                                   [](size_t value) { return value == 1; });
 
         NGRAPH_CHECK(is_dds_valid,
                      "Unable to convert Convolution:0 to Convolution:1 with data dilation strides "
@@ -167,11 +166,9 @@ namespace
         auto pads_end = node->get_padding_above_forward();
         auto data_dilation_strides = node->get_data_dilation_strides_forward();
 
-        bool is_dds_valid = true;
-        for (auto value : data_dilation_strides)
-        {
-            is_dds_valid = is_dds_valid && (value == 1);
-        }
+        bool is_dds_valid = all_of(data_dilation_strides.begin(),
+                                   data_dilation_strides.end(),
+                                   [](size_t value) { return value == 1; });
 
         NGRAPH_CHECK(is_dds_valid,
                      "Unable to convert ConvolutionBackpropData:0 to ConvolutionBackpropData:1 "
@@ -271,6 +268,62 @@ namespace
         return true;
     }
 
+    bool op_cast(shared_ptr<op::v0::GroupConvolution> node)
+    {
+        auto strides = node->get_window_movement_strides();
+        auto dilations = node->get_window_dilation_strides();
+        auto pads_begin = node->get_padding_below();
+        auto pads_end = node->get_padding_above();
+        auto data_dilation_strides = node->get_data_dilation_strides();
+        auto auto_pad = node->get_pad_type();
+
+        bool is_dds_valid = all_of(data_dilation_strides.begin(),
+                                   data_dilation_strides.end(),
+                                   [](size_t value) { return value == 1; });
+
+        NGRAPH_CHECK(is_dds_valid,
+                     "Unable to convert GroupConvolution:0 to GroupConvolution:1"
+                     "with data dilation strides other than `1`. Node: ",
+                     *node);
+
+        shared_ptr<Node> replacement_node;
+        if (node->has_groups_in_filters())
+        {
+            replacement_node = make_shared<op::v1::GroupConvolution>(node->input_value(0),
+                                                                     node->input_value(1),
+                                                                     strides,
+                                                                     pads_begin,
+                                                                     pads_end,
+                                                                     dilations,
+                                                                     auto_pad);
+        }
+        else
+        {
+            NGRAPH_CHECK(node->get_input_partial_shape(1).is_static(),
+                         "Unable to convert GroupConvolution:0 to GroupConvolution:1"
+                         "with dynamic filters shape. Node: ",
+                         *node);
+
+            auto filters_shape = node->get_input_shape(1);
+            auto groups = node->get_groups();
+            filters_shape[0] /= groups;
+            filters_shape.insert(filters_shape.begin(), groups);
+
+            auto reshaped_filters = builder::reshape(node->input_value(1), filters_shape);
+
+            replacement_node =
+                make_shared<op::v1::GroupConvolution>(node->input(0).get_source_output(),
+                                                      reshaped_filters,
+                                                      strides,
+                                                      pads_begin,
+                                                      pads_end,
+                                                      dilations,
+                                                      auto_pad);
+        }
+        replace_node(node, replacement_node);
+        return true;
+    }
+
     bool op_cast(shared_ptr<op::Less> node)
     {
         op_cast_binary_elementwise_node<op::v0::Less, op::v1::Less>(node);

src/ngraph/runtime/cpu/pass/cpu_fusion.cpp

@@ -1766,6 +1766,11 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_groupconv_batchnorm_global
             return false;
         }
 
+        if (conv_m->has_groups_in_filters())
+        {
+            return false;
+        }
+
         // new weights = old weights * gamma / sqrt(variance + epsilon)
         // new biases = (-mean) * gamma / sqrt(variance + epsilon) + beta
 

src/ngraph/serializer.cpp

@@ -1178,6 +1178,51 @@ shared_ptr<Node> JSONDeserializer::deserialize_node(json node_js)
             }
             break;
         }
+        case OP_TYPEID::GroupConvolution_v1:
+        {
+            auto strides = node_js.at("strides").get<vector<size_t>>();
+            auto dilations = node_js.at("dilations").get<vector<size_t>>();
+            auto pads_begin = node_js.at("pads_begin").get<vector<std::ptrdiff_t>>();
+            auto pads_end = node_js.at("pads_end").get<vector<std::ptrdiff_t>>();
+
+            op::PadType auto_pad = read_pad_type(node_js);
+
+            node = make_shared<op::v1::GroupConvolution>(
+                args[0], args[1], strides, pads_begin, pads_end, dilations, auto_pad);
+            break;
+        }
+        case OP_TYPEID::GroupConvolutionBackpropData_v1:
+        {
+            auto strides = node_js.at("strides").get<vector<size_t>>();
+            auto dilations = node_js.at("dilations").get<vector<size_t>>();
+            auto pads_begin = node_js.at("pads_begin").get<vector<std::ptrdiff_t>>();
+            auto pads_end = node_js.at("pads_end").get<vector<std::ptrdiff_t>>();
+            auto output_padding = node_js.at("output_padding").get<vector<std::ptrdiff_t>>();
+            if (args.size() == 3)
+            {
+                node = make_shared<op::v1::GroupConvolutionBackpropData>(args[0],
+                                                                         args[1],
+                                                                         args[2],
+                                                                         strides,
+                                                                         pads_begin,
+                                                                         pads_end,
+                                                                         dilations,
+                                                                         read_pad_type(node_js),
+                                                                         output_padding);
+            }
+            else
+            {
+                node = make_shared<op::v1::GroupConvolutionBackpropData>(args[0],
+                                                                         args[1],
+                                                                         strides,
+                                                                         pads_begin,
+                                                                         pads_end,
+                                                                         dilations,
+                                                                         read_pad_type(node_js),
+                                                                         output_padding);
+            }
+            break;
+        }
         case OP_TYPEID::ConvolutionBackpropFilters:
         {
             auto filters_shape = node_js.at("filters_shape").get<vector<size_t>>();
@@ -3296,6 +3341,27 @@ json JSONSerializer::serialize_node(const Node& n)
         node["output_padding"] = tmp->get_output_padding();
         break;
     }
+    case OP_TYPEID::GroupConvolution_v1:
+    {
+        auto tmp = static_cast<const op::v1::GroupConvolution*>(&n);
+        node["strides"] = tmp->get_strides();
+        node["dilations"] = tmp->get_dilations();
+        node["pads_begin"] = tmp->get_pads_begin();
+        node["pads_end"] = tmp->get_pads_end();
+        node["auto_pad"] = tmp->get_auto_pad();
+        break;
+    }
+    case OP_TYPEID::GroupConvolutionBackpropData_v1:
+    {
+        auto tmp = static_cast<const op::v1::GroupConvolutionBackpropData*>(&n);
+        node["strides"] = tmp->get_strides();
+        node["dilations"] = tmp->get_dilations();
+        node["pads_begin"] = tmp->get_pads_begin();
+        node["pads_end"] = tmp->get_pads_end();
+        node["auto_pad"] = tmp->get_auto_pad();
+        node["output_padding"] = tmp->get_output_padding();
+        break;
+    }
     case OP_TYPEID::ConvolutionBackpropFilters:
     {
         auto tmp = static_cast<const op::v0::ConvolutionBackpropFilters*>(&n);

test/opset1.cpp

@@ -70,7 +70,8 @@ TEST(opset, check_opset1)
     CHECK_OPSET(op::v1::Greater, opset1::Greater)
     CHECK_OPSET(op::v1::GreaterEqual, opset1::GreaterEqual)
     CHECK_OPSET(op::v0::GRN, opset1::GRN)
-    CHECK_OPSET(op::v0::GroupConvolution, opset1::GroupConvolution)
+    CHECK_OPSET(op::v1::GroupConvolution, opset1::GroupConvolution)
+    CHECK_OPSET(op::v1::GroupConvolutionBackpropData, opset1::GroupConvolutionBackpropData)
     // CHECK_OPSET(op::v0::GRUCell, opset1::GRUCell)
     // TODO CHECK_OPSET(op::v0::GRUSequence, opset1::GRUSequence)
     CHECK_OPSET(op::v0::HardSigmoid, opset1::HardSigmoid)