Merge pull request #540 from NervanaSystems/louisfeng/NGMX-296-conv_bias

NGMX-296 Convolution + Bias with MKLDNN

src/ngraph/CMakeLists.txt

@@ -186,6 +186,7 @@ if (NGRAPH_CPU_ENABLE AND LLVM_INCLUDE_DIR AND
         runtime/cpu/mkldnn_emitter.cpp
         runtime/cpu/mkldnn_invoke.cpp
         runtime/cpu/mkldnn_utils.cpp
+        runtime/cpu/ops/conv_bias.cpp
         runtime/cpu/ops/convert_layout.cpp
         runtime/cpu/ops/sigmoid.cpp
         runtime/cpu/ops/matmul_bias.cpp

src/ngraph/runtime/cpu/cpu_emitter.cpp

@@ -90,6 +90,7 @@
 #include "ngraph/runtime/cpu/cpu_kernel_emitters.hpp"
 #include "ngraph/runtime/cpu/cpu_op_annotations.hpp"
 #include "ngraph/runtime/cpu/mkldnn_utils.hpp"
+#include "ngraph/runtime/cpu/ops/conv_bias.hpp"
 #include "ngraph/runtime/cpu/ops/convert_layout.hpp"
 #include "ngraph/runtime/cpu/ops/matmul_bias.hpp"
 #include "ngraph/runtime/cpu/ops/sigmoid.hpp"
@@ -2487,6 +2488,134 @@ namespace ngraph
                 }
             }
 
+            template <>
+            void CPU_Emitter::EMITTER_DECL(ngraph::op::ConvolutionBias)
+            {
+                auto convolution = static_cast<const ngraph::op::ConvolutionBias*>(node);
+
+                const TensorViewWrapper& data = args[0];
+                const TensorViewWrapper& weights = args[1];
+                const TensorViewWrapper& bias = args[2];
+                const TensorViewWrapper& result = out[0];
+
+                using namespace runtime::cpu::mkldnn_utils;
+
+                if (mkldnn_utils::use_mkldnn_kernel(node))
+                {
+                    auto data_format = mkldnn_utils::get_input_mkldnn_format(node, 0);
+                    auto weights_format = mkldnn_utils::get_input_mkldnn_format(node, 1);
+                    auto bias_format = mkldnn_utils::get_input_mkldnn_format(node, 2);
+                    auto result_format = mkldnn_utils::get_output_mkldnn_format(node, 0);
+
+                    auto& mkldnn_emitter = external_function->get_mkldnn_emitter();
+                    auto data_desc = mkldnn_emitter->build_memory_descriptor(data, data_format);
+                    auto weights_desc =
+                        mkldnn_emitter->build_memory_descriptor(weights, weights_format);
+                    auto bias_desc = mkldnn_emitter->build_memory_descriptor(bias, bias_format);
+                    auto result_desc =
+                        mkldnn_emitter->build_memory_descriptor(result, result_format);
+
+                    // For dilation, MKLDNN wants to know how many elements to insert between, not how far
+                    // apart to space the elements like nGraph. So we have to subtract 1 from each pos.
+                    Strides window_dilation_strides_adjusted;
+
+                    for (size_t s : convolution->get_window_dilation_strides())
+                    {
+                        window_dilation_strides_adjusted.push_back(s - 1);
+                    }
+
+                    size_t conv_index = mkldnn_emitter->build_convolution_forward(
+                        data_desc,
+                        weights_desc,
+                        bias_desc,
+                        result_desc,
+                        convolution->get_window_movement_strides(),
+                        window_dilation_strides_adjusted,
+                        convolution->get_padding_below(),
+                        convolution->get_padding_above());
+
+                    auto& deps = mkldnn_emitter->get_primitive_deps(conv_index);
+                    writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[0])
+                           << ", " << data.get_name() << ");\n";
+                    writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[1])
+                           << ", " << weights.get_name() << ");\n";
+                    writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[2])
+                           << ", " << bias.get_name() << ");\n";
+                    writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[3])
+                           << ", " << result.get_name() << ");\n";
+
+                    writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, "
+                           << to_string(conv_index) << ");\n";
+                }
+                else
+                {
+                    throw ngraph_error("ConvolutionBias is only supported with MKLDNN kernel.");
+                }
+            }
+
+            template <>
+            void CPU_Emitter::EMITTER_DECL(ngraph::op::ConvolutionBiasBackpropFiltersBias)
+            {
+                auto convolution =
+                    static_cast<const ngraph::op::ConvolutionBiasBackpropFiltersBias*>(node);
+                const TensorViewWrapper& data = args[0];
+                const TensorViewWrapper& delta = args[1];
+                const TensorViewWrapper& weights_delta = out[0];
+                const TensorViewWrapper& bias_delta = out[1];
+
+                using namespace runtime::cpu::mkldnn_utils;
+
+                if (mkldnn_utils::use_mkldnn_kernel(node))
+                {
+                    Strides window_dilation_strides_adjusted;
+                    for (size_t s : convolution->get_window_dilation_strides_forward())
+                    {
+                        window_dilation_strides_adjusted.push_back(s - 1);
+                    }
+
+                    auto data_format = mkldnn_utils::get_input_mkldnn_format(node, 0);
+                    auto delta_format = mkldnn_utils::get_input_mkldnn_format(node, 1);
+                    auto weights_delta_format = mkldnn_utils::get_output_mkldnn_format(node, 0);
+                    auto bias_delta_format = mkldnn_utils::get_output_mkldnn_format(node, 1);
+
+                    auto& mkldnn_emitter = external_function->get_mkldnn_emitter();
+                    auto data_desc = mkldnn_emitter->build_memory_descriptor(data, data_format);
+                    auto delta_desc = mkldnn_emitter->build_memory_descriptor(delta, delta_format);
+                    auto weights_delta_desc = mkldnn_emitter->build_memory_descriptor(
+                        weights_delta, weights_delta_format);
+                    auto bias_delta_desc =
+                        mkldnn_emitter->build_memory_descriptor(bias_delta, bias_delta_format);
+
+                    size_t conv_index = mkldnn_emitter->build_convolution_backward_weights_bias(
+                        data_desc,
+                        delta_desc,
+                        weights_delta_desc,
+                        bias_delta_desc,
+                        convolution->get_window_movement_strides_forward(),
+                        window_dilation_strides_adjusted,
+                        convolution->get_padding_below_forward(),
+                        convolution->get_padding_above_forward());
+
+                    auto& deps = mkldnn_emitter->get_primitive_deps(conv_index);
+                    writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[0])
+                           << ", " << data.get_name() << ");\n";
+                    writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[1])
+                           << ", " << delta.get_name() << ");\n";
+                    writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[2])
+                           << ", " << weights_delta.get_name() << ");\n";
+                    writer << "cpu::mkldnn_utils::set_memory_ptr(ctx, " << to_string(deps[3])
+                           << ", " << bias_delta.get_name() << ");\n";
+
+                    writer << "cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, "
+                           << to_string(conv_index) << ");\n";
+                }
+                else
+                {
+                    throw ngraph_error(
+                        "ConvolutionBiasBackpropFiltersBias is only supported with MKLDNN kernel.");
+                }
+            }
+
             template <>
             void CPU_Emitter::EMITTER_DECL(ngraph::op::Not)
             {

src/ngraph/runtime/cpu/cpu_external_function.cpp

@@ -109,6 +109,7 @@
 #include "ngraph/runtime/cpu/cpu_tensor_view.hpp"
 #include "ngraph/runtime/cpu/cpu_tracing.hpp"
 #include "ngraph/runtime/cpu/mkldnn_utils.hpp"
+#include "ngraph/runtime/cpu/ops/conv_bias.hpp"
 #include "ngraph/runtime/cpu/ops/convert_layout.hpp"
 #include "ngraph/runtime/cpu/ops/matmul_bias.hpp"
 #include "ngraph/runtime/cpu/ops/sigmoid.hpp"
@@ -226,6 +227,10 @@ static const runtime::cpu::OpMap dispatcher{
      &runtime::cpu::CPU_Emitter::emit<op::ConvolutionBackpropFilters>},
     {TI(ngraph::op::ConvolutionBackpropData),
      &runtime::cpu::CPU_Emitter::emit<op::ConvolutionBackpropData>},
+    {TI(ngraph::op::ConvolutionBias), &runtime::cpu::CPU_Emitter::emit<op::ConvolutionBias>},
+    // conv+bias backprop for data share the same implementation as ConvolutionBackpropData
+    {TI(ngraph::op::ConvolutionBiasBackpropFiltersBias),
+     &runtime::cpu::CPU_Emitter::emit<op::ConvolutionBiasBackpropFiltersBias>},
     {TI(ngraph::runtime::cpu::op::ConvertLayout),
      &runtime::cpu::CPU_Emitter::emit<runtime::cpu::op::ConvertLayout>},
     {TI(ngraph::op::Not), &runtime::cpu::CPU_Emitter::emit<op::Not>},

src/ngraph/runtime/cpu/mkldnn_emitter.cpp

@@ -83,6 +83,7 @@ size_t MKLDNNEmitter::build_convolution_forward(const mkldnn::memory::desc& inpu
                                                 const mkldnn::memory::desc& weights_desc,
                                                 const mkldnn::memory::desc& result_desc,
                                                 const ngraph::Strides& strides,
+                                                const ngraph::Strides& dilation_strides,
                                                 const ngraph::CoordinateDiff& padding_below,
                                                 const ngraph::CoordinateDiff& padding_above)
 {
@@ -97,6 +98,7 @@ size_t MKLDNNEmitter::build_convolution_forward(const mkldnn::memory::desc& inpu
           weights_desc,
           result_desc,
           mkldnn::memory::dims(strides.begin(), strides.end()),
+          mkldnn::memory::dims(dilation_strides.begin(), dilation_strides.end()),
           mkldnn::memory::dims(padding_below.begin(), padding_below.end()),
           mkldnn::memory::dims(padding_above.begin(), padding_above.end()),
           mkldnn::padding_kind::zero},
@@ -111,21 +113,24 @@ size_t MKLDNNEmitter::build_convolution_forward(const mkldnn::memory::desc& inpu
 
 size_t MKLDNNEmitter::build_convolution_forward(const mkldnn::memory::desc& input_data_desc,
                                                 const mkldnn::memory::desc& weights_desc,
+                                                const mkldnn::memory::desc& bias_desc,
                                                 const mkldnn::memory::desc& result_desc,
                                                 const ngraph::Strides& strides,
                                                 const ngraph::Strides& dilation_strides,
                                                 const ngraph::CoordinateDiff& padding_below,
                                                 const ngraph::CoordinateDiff& padding_above)
 {
-    size_t input_data_index = build_memory_primitive(input_data_desc);
-    size_t weights_index = build_memory_primitive(weights_desc);
-    size_t result_index = build_memory_primitive(result_desc);
+    const size_t input_data_index = build_memory_primitive(input_data_desc);
+    const size_t weights_index = build_memory_primitive(weights_desc);
+    const size_t bias_index = build_memory_primitive(bias_desc);
+    const size_t result_index = build_memory_primitive(result_desc);
 
-    size_t conv_index = insert_primitive(new mkldnn::convolution_forward(
+    const size_t conv_index = insert_primitive(new mkldnn::convolution_forward(
         {{mkldnn::prop_kind::forward,
           mkldnn::algorithm::convolution_direct,
           input_data_desc,
           weights_desc,
+          bias_desc,
           result_desc,
           mkldnn::memory::dims(strides.begin(), strides.end()),
           mkldnn::memory::dims(dilation_strides.begin(), dilation_strides.end()),
@@ -135,9 +140,68 @@ size_t MKLDNNEmitter::build_convolution_forward(const mkldnn::memory::desc& inpu
          mkldnn_utils::global_cpu_engine},
         *m_mkldnn_primitives[input_data_index],
         *m_mkldnn_primitives[weights_index],
+        *m_mkldnn_primitives[bias_index],
         *m_mkldnn_primitives[result_index]));
 
-    m_primitive_deps[conv_index] = {input_data_index, weights_index, result_index};
+    m_primitive_deps[conv_index] = {input_data_index, weights_index, bias_index, result_index};
+    return conv_index;
+}
+
+size_t MKLDNNEmitter::build_convolution_backward_weights_bias(
+    const mkldnn::memory::desc& in_data_desc,
+    const mkldnn::memory::desc& in_delta_desc,
+    const mkldnn::memory::desc& out_weights_delta_desc,
+    const mkldnn::memory::desc& out_bias_delta_desc,
+    const ngraph::Strides& ng_strides,
+    const ngraph::Strides& ng_dilation_strides,
+    const ngraph::CoordinateDiff& ng_padding_below,
+    const ngraph::CoordinateDiff& ng_padding_above)
+{
+    const size_t in_data_index = build_memory_primitive(in_data_desc);
+    const size_t in_delta_index = build_memory_primitive(in_delta_desc);
+    const size_t out_weights_delta_index = build_memory_primitive(out_weights_delta_desc);
+    const size_t out_bias_delta_index = build_memory_primitive(out_bias_delta_desc);
+
+    mkldnn::memory::dims strides(ng_strides.begin(), ng_strides.end());
+    mkldnn::memory::dims dilation(ng_dilation_strides.begin(), ng_dilation_strides.end());
+    mkldnn::memory::dims padding_l(ng_padding_below.begin(), ng_padding_below.end());
+    mkldnn::memory::dims padding_r(ng_padding_above.begin(), ng_padding_above.end());
+    mkldnn::convolution_forward::primitive_desc fwd_pd{{mkldnn::prop_kind::forward,
+                                                        mkldnn::algorithm::convolution_direct,
+                                                        in_data_desc,
+                                                        out_weights_delta_desc,
+                                                        out_bias_delta_desc,
+                                                        in_delta_desc,
+                                                        strides,
+                                                        dilation,
+                                                        padding_l,
+                                                        padding_r,
+                                                        mkldnn::padding_kind::zero},
+                                                       mkldnn_utils::global_cpu_engine};
+
+    mkldnn::convolution_backward_weights::primitive_desc bwd_pd{
+        {mkldnn::algorithm::convolution_direct,
+         in_data_desc,
+         out_weights_delta_desc,
+         out_bias_delta_desc,
+         in_delta_desc,
+         strides,
+         dilation,
+         padding_l,
+         padding_r,
+         mkldnn::padding_kind::zero},
+        mkldnn_utils::global_cpu_engine,
+        fwd_pd};
+
+    const size_t conv_index = insert_primitive(
+        new mkldnn::convolution_backward_weights(bwd_pd,
+                                                 *m_mkldnn_primitives[in_data_index],
+                                                 *m_mkldnn_primitives[in_delta_index],
+                                                 *m_mkldnn_primitives[out_weights_delta_index],
+                                                 *m_mkldnn_primitives[out_bias_delta_index]));
+
+    m_primitive_deps[conv_index] = {
+        in_data_index, in_delta_index, out_weights_delta_index, out_bias_delta_index};
     return conv_index;
 }
 

src/ngraph/runtime/cpu/mkldnn_emitter.hpp

@@ -57,11 +57,16 @@ namespace ngraph
                                                  const mkldnn::memory::desc& weights_desc,
                                                  const mkldnn::memory::desc& result_desc,
                                                  const ngraph::Strides& strides,
+                                                 const ngraph::Strides& dilation_strides,
                                                  const ngraph::CoordinateDiff& padding_below,
                                                  const ngraph::CoordinateDiff& padding_above);
 
+                /**
+                 * Convolution + bias forward
+                 */
                 size_t build_convolution_forward(const mkldnn::memory::desc& input_data_desc,
                                                  const mkldnn::memory::desc& weights_desc,
+                                                 const mkldnn::memory::desc& bias_desc,
                                                  const mkldnn::memory::desc& result_desc,
                                                  const ngraph::Strides& strides,
                                                  const ngraph::Strides& dilation_strides,
@@ -84,7 +89,18 @@ namespace ngraph
                                                        const ngraph::Strides& dilation_strides,
                                                        const ngraph::CoordinateDiff& padding_below,
                                                        const ngraph::CoordinateDiff& padding_above);
-
+                /**
+                 * Convolution + bias backprop for weights and bias
+                 */
+                size_t build_convolution_backward_weights_bias(
+                    const mkldnn::memory::desc& in_data_desc,
+                    const mkldnn::memory::desc& in_delta_desc,
+                    const mkldnn::memory::desc& out_weights_delta_desc,
+                    const mkldnn::memory::desc& out_bias_delta_desc,
+                    const ngraph::Strides& ng_strides,
+                    const ngraph::Strides& ng_dilation_strides,
+                    const ngraph::CoordinateDiff& ng_padding_below,
+                    const ngraph::CoordinateDiff& ng_padding_above);
                 size_t build_pooling_forward(mkldnn::algorithm pooling_algorithm,
                                              const mkldnn::memory::desc& input_desc,
                                              const mkldnn::memory::desc& result_desc,

src/ngraph/runtime/cpu/mkldnn_utils.cpp

@@ -28,6 +28,7 @@
 #include "ngraph/ops/relu.hpp"
 #include "ngraph/runtime/cpu/cpu_layout_descriptor.hpp"
 #include "ngraph/runtime/cpu/cpu_op_annotations.hpp"
+#include "ngraph/runtime/cpu/ops/conv_bias.hpp"
 #include "ngraph/types/element_type.hpp"
 
 #include "mkldnn_utils.hpp"
@@ -47,6 +48,8 @@ static const std::unordered_set<std::type_index> s_op_registry{
     TI(ngraph::op::Convolution),
     TI(ngraph::op::ConvolutionBackpropData),
     TI(ngraph::op::ConvolutionBackpropFilters),
+    TI(ngraph::op::ConvolutionBias),
+    TI(ngraph::op::ConvolutionBiasBackpropFiltersBias),
     TI(ngraph::op::MaxPool),
     TI(ngraph::op::MaxPoolBackprop),
     TI(ngraph::op::Relu),

src/ngraph/runtime/cpu/ops/conv_bias.cpp

+/*******************************************************************************
+* Copyright 2017-2018 Intel Corporation
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*******************************************************************************/
+
+#include <numeric>
+
+#include "ngraph/ops/convolution.hpp"
+#include "ngraph/ops/get_output_element.hpp"
+#include "ngraph/runtime/cpu/ops/conv_bias.hpp"
+#include "ngraph/util.hpp"
+
+using namespace std;
+using namespace ngraph;
+
+op::ConvolutionBias::ConvolutionBias(const std::shared_ptr<op::Convolution>& conv,
+                                     const std::shared_ptr<Node>& bias)
+    : RequiresTensorViewArgs("ConvolutionBias",
+                             {conv->get_input_op(0), conv->get_input_op(1), bias})
+    , m_window_movement_strides(conv->get_window_movement_strides())
+    , m_window_dilation_strides(conv->get_window_dilation_strides())
+    , m_padding_below(conv->get_padding_below())
+    , m_padding_above(conv->get_padding_above())
+    , m_data_dilation_strides(conv->get_data_dilation_strides())
+{
+    if (conv->get_element_type() != bias->get_element_type())
+    {
+        throw ngraph_error("Convolution's element type isn't equal to bias!");
+    }
+
+    set_value_type_checked(conv->get_element_type(), conv->get_shape());
+}
+
+op::ConvolutionBias::ConvolutionBias(const std::shared_ptr<Node>& data_batch,
+                                     const std::shared_ptr<Node>& filters,
+                                     const std::shared_ptr<Node>& bias,
+                                     const Strides& window_movement_strides,
+                                     const Strides& window_dilation_strides,
+                                     const CoordinateDiff& padding_below,
+                                     const CoordinateDiff& padding_above,
+                                     const Strides& data_dilation_strides)
+    : RequiresTensorViewArgs("ConvolutionBias", {data_batch, filters, bias})
+    , m_window_movement_strides(window_movement_strides)
+    , m_window_dilation_strides(window_dilation_strides)
+    , m_padding_below(padding_below)
+    , m_padding_above(padding_above)
+    , m_data_dilation_strides(data_dilation_strides)
+{
+}
+
+std::shared_ptr<Node> op::ConvolutionBias::copy_with_new_args(const NodeVector& new_args) const
+{
+    if (new_args.size() != 2)
+    {
+        throw ngraph_error("Incorrect number of new arguments");
+    }
+
+    return std::shared_ptr<Node>(new ConvolutionBias(new_args.at(0),
+                                                     new_args.at(1),
+                                                     new_args.at(2),
+                                                     get_window_movement_strides(),
+                                                     get_window_dilation_strides(),
+                                                     get_padding_below(),
+                                                     get_padding_above(),
+                                                     get_data_dilation_strides()));
+}
+
+void op::ConvolutionBias::generate_adjoints(autodiff::Adjoints& adjoints,
+                                            const std::shared_ptr<Node>& delta)
+{
+    auto data = get_input_op(0);
+    const auto data_shape = data->get_shape();
+
+    auto filter = get_input_op(1);
+    const auto filter_shape = filter->get_shape();
+
+    auto bias = get_input_op(2);
+    const auto bias_shape = bias->get_shape();
+
+    // using regular convolution backprop for data
+    adjoints.add_delta(data,
+                       std::make_shared<op::ConvolutionBackpropData>(data_shape,
+                                                                     filter,
+                                                                     delta,
+                                                                     m_window_movement_strides,
+                                                                     m_window_dilation_strides,
+                                                                     m_padding_below,
+                                                                     m_padding_above,
+                                                                     m_data_dilation_strides));
+
+    auto filter_bias_backprop =
+        std::make_shared<op::ConvolutionBiasBackpropFiltersBias>(data,
+                                                                 filter_shape,
+                                                                 bias_shape,
+                                                                 delta,
+                                                                 m_window_movement_strides,
+                                                                 m_window_dilation_strides,
+                                                                 m_padding_below,
+                                                                 m_padding_above,
+                                                                 m_data_dilation_strides);
+    auto filter_delta = std::make_shared<op::GetOutputElement>(filter_bias_backprop, 0);
+    auto bias_delta = std::make_shared<op::GetOutputElement>(filter_bias_backprop, 1);
+
+    adjoints.add_delta(filter, filter_delta);
+    adjoints.add_delta(bias, bias_delta);
+}
+
+op::ConvolutionBiasBackpropFiltersBias::ConvolutionBiasBackpropFiltersBias(
+    const std::shared_ptr<Node>& data_batch,
+    const Shape& filters_shape,
+    const Shape& bias_shape,
+    const std::shared_ptr<Node>& output_delta,
+    const Strides& window_movement_strides_forward,
+    const Strides& window_dilation_strides_forward,
+    const CoordinateDiff& padding_below_forward,
+    const CoordinateDiff& padding_above_forward,
+    const Strides& data_dilation_strides_forward)
+    : RequiresTensorViewArgs("ConvolutionBiasBackpropFiltersBias", {data_batch, output_delta})
+    , m_filters_shape(filters_shape)
+    , m_bias_shape(bias_shape)
+    , m_window_movement_strides_forward(window_movement_strides_forward)
+    , m_window_dilation_strides_forward(window_dilation_strides_forward)
+    , m_padding_below_forward(padding_below_forward)
+    , m_padding_above_forward(padding_above_forward)
+    , m_data_dilation_strides_forward(data_dilation_strides_forward)
+{
+    auto& data_batch_shape = get_input_shape(0);
+    auto& data_batch_et = get_input_element_type(0);
+    auto& output_delta_et = get_input_element_type(1);
+
+    //
+    // Make sure data batch and output delta element types match.
+    //
+    if (data_batch_et != output_delta_et)
+    {
+        throw ngraph_error(
+            "ConvolutionBiasBackpropFilterBias data batch and output delta element types do not "
+            "match");
+    }
+
+    //                              Forward               Backward
+    // Window movement strides      q                     p_f
+    // Window dilation strides      p_f                   q
+    // Padding below                a_x                   a_x
+    // Padding above                b_x                   b_x - (a_x + (S_x - 1)p_x + b_x - (S_f - 1)p_f) % q
+    // Data dilation strides        p_x                   p_x
+
+    for (size_t i = 0; i < filters_shape.size() - 2; i++)
+    {
+        m_window_movement_strides_backward.push_back(window_dilation_strides_forward[i]);
+        m_window_dilation_strides_backward.push_back(window_movement_strides_forward[i]);
+        m_padding_below_backward.push_back(padding_below_forward[i]);
+        m_padding_above_backward.push_back(
+            padding_above_forward[i] -
+            (padding_below_forward[i] +
+             (data_batch_shape[i + 2] - 1) * data_dilation_strides_forward[i] +
+             padding_above_forward[i] -
+             (filters_shape[i + 2] - 1) * window_dilation_strides_forward[i]) %
+                window_movement_strides_forward[i]);
+        m_data_dilation_strides_backward.push_back(data_dilation_strides_forward[i]);
+    }
+
+    add_output(data_batch_et, filters_shape);
+    add_output(data_batch_et, bias_shape);
+}
+
+std::shared_ptr<Node>
+    op::ConvolutionBiasBackpropFiltersBias::copy_with_new_args(const NodeVector& new_args) const
+{
+    if (new_args.size() != 2)
+    {
+        throw ngraph_error("Incorrect number of new arguments");
+    }
+    return std::make_shared<ConvolutionBiasBackpropFiltersBias>(new_args.at(0),
+                                                                m_filters_shape,
+                                                                m_bias_shape,
+                                                                new_args.at(1),
+                                                                m_window_movement_strides_forward,
+                                                                m_window_dilation_strides_forward,
+                                                                m_padding_below_forward,
+                                                                m_padding_above_forward,
+                                                                m_data_dilation_strides_forward);
+}

src/ngraph/runtime/cpu/ops/conv_bias.hpp

+/*******************************************************************************
+* Copyright 2017-2018 Intel Corporation
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*******************************************************************************/
+
+#pragma once
+
+#include "ngraph/ops/convolution.hpp"
+#include "ngraph/ops/util/requires_tensor_view_args.hpp"
+
+namespace ngraph
+{
+    namespace op
+    {
+        /// \brief Convolution + bias forward prop for batched convolution operation.
+        class ConvolutionBias : public util::RequiresTensorViewArgs
+        {
+        public:
+            ConvolutionBias(const std::shared_ptr<op::Convolution>& conv,
+                            const std::shared_ptr<Node>& bias);
+
+            const Strides& get_window_movement_strides() const { return m_window_movement_strides; }
+            const Strides& get_window_dilation_strides() const { return m_window_dilation_strides; }
+            const CoordinateDiff& get_padding_below() const { return m_padding_below; }
+            const CoordinateDiff& get_padding_above() const { return m_padding_above; }
+            const Strides& get_data_dilation_strides() const { return m_data_dilation_strides; }
+            std::shared_ptr<Node> get_bias() { return get_input_op(2); }
+            std::shared_ptr<Node> get_filters() { return get_input_op(1); }
+            std::shared_ptr<Node> get_data_batch() { return get_input_op(0); }
+            virtual std::shared_ptr<Node>
+                copy_with_new_args(const NodeVector& new_args) const override;
+
+            void generate_adjoints(autodiff::Adjoints& adjoints,
+                                   const std::shared_ptr<Node>& delta) override;
+
+        protected:
+            Strides m_window_movement_strides;
+            Strides m_window_dilation_strides;
+            CoordinateDiff m_padding_below;
+            CoordinateDiff m_padding_above;
+            Strides m_data_dilation_strides;
+
+        private:
+            ConvolutionBias(const std::shared_ptr<Node>& data_batch,
+                            const std::shared_ptr<Node>& filters,
+                            const std::shared_ptr<Node>& bias,
+                            const Strides& window_movement_strides,
+                            const Strides& window_dilation_strides,
+                            const CoordinateDiff& padding_below,
+                            const CoordinateDiff& padding_above,
+                            const Strides& data_dilation_strides);
+        };
+
+        /// \brief Filters and bias backprop for batched convolution operation. Data backprop is
+        /// the same as regular convolution backprop for data.
+        class ConvolutionBiasBackpropFiltersBias : public util::RequiresTensorViewArgs
+        {
+        public:
+            ConvolutionBiasBackpropFiltersBias(const std::shared_ptr<Node>& data_batch,
+                                               const Shape& filters_shape,
+                                               const Shape& bias_shape,
+                                               const std::shared_ptr<Node>& output_delta,
+                                               const Strides& window_movement_strides_forward,
+                                               const Strides& window_dilation_strides_forward,
+                                               const CoordinateDiff& padding_below_forward,
+                                               const CoordinateDiff& padding_above_forward,
+                                               const Strides& data_dilation_strides_forward);
+
+            virtual std::shared_ptr<Node>
+                copy_with_new_args(const NodeVector& new_args) const override;
+
+            /// \return The filters tensor shape.
+            const Shape& get_filters_shape() const { return m_filters_shape; }
+            /// \return The bias tensor shape.
+            const Shape& get_bias_shape() const { return m_bias_shape; }
+            /// \return The window movement strides from the forward prop.
+            const Strides& get_window_movement_strides_forward() const
+            {
+                return m_window_movement_strides_forward;
+            }
+            /// \return The window dilation strides from the forward prop.
+            const Strides& get_window_dilation_strides_forward() const
+            {
+                return m_window_dilation_strides_forward;
+            }
+            /// \return The padding-below sizes (possibly negative) from the forward prop.
+            const CoordinateDiff& get_padding_below_forward() const
+            {
+                return m_padding_below_forward;
+            }
+            /// \return The padding-above sizes (possibly negative) from the forward prop.
+            const CoordinateDiff& get_padding_above_forward() const
+            {
+                return m_padding_above_forward;
+            }
+            /// \return The data dilation strides from the forward prop.
+            const Strides& get_data_dilation_strides_forward() const
+            {
+                return m_data_dilation_strides_forward;
+            }
+
+            /// \return The window movement strides for the backward prop.
+            const Strides& get_window_movement_strides_backward() const
+            {
+                return m_window_movement_strides_backward;
+            }
+            /// \return The window dilation strides for the backward prop.
+            const Strides& get_window_dilation_strides_backward() const
+            {
+                return m_window_dilation_strides_backward;
+            }
+            /// \return The padding-below sizes (possibly negative) for the backward prop.
+            const CoordinateDiff& get_padding_below_backward() const
+            {
+                return m_padding_below_backward;
+            }
+            /// \return The padding-above sizes (possibly negative) for the backward prop.
+            const CoordinateDiff& get_padding_above_backward() const
+            {
+                return m_padding_above_backward;
+            }
+            /// \return The data dilation strides for the backward prop.
+            const Strides& get_data_dilation_strides_backward() const
+            {
+                return m_data_dilation_strides_backward;
+            }
+
+        protected:
+            Shape m_filters_shape;
+            Shape m_bias_shape;
+            Strides m_window_movement_strides_forward;
+            Strides m_window_dilation_strides_forward;
+            CoordinateDiff m_padding_below_forward;
+            CoordinateDiff m_padding_above_forward;
+            Strides m_data_dilation_strides_forward;
+
+            Strides m_window_movement_strides_backward;
+            Strides m_window_dilation_strides_backward;
+            CoordinateDiff m_padding_below_backward;
+            CoordinateDiff m_padding_above_backward;
+            Strides m_data_dilation_strides_backward;
+        };
+    }
+}

src/ngraph/runtime/cpu/pass/cpu_assignment.cpp

@@ -32,6 +32,7 @@
 #include "ngraph/ops/relu.hpp"
 #include "ngraph/runtime/cpu/cpu_op_annotations.hpp"
 #include "ngraph/runtime/cpu/mkldnn_utils.hpp"
+#include "ngraph/runtime/cpu/ops/conv_bias.hpp"
 #include "ngraph/runtime/cpu/ops/sigmoid.hpp"
 
 using namespace std;
@@ -153,6 +154,59 @@ namespace ngraph
                     }
                 }
 
+                template <>
+                void CPUAssignment::ASSIGN_DECL(ngraph::op::ConvolutionBias)
+                {
+                    auto convolution = static_cast<op::ConvolutionBias*>(node);
+
+                    auto data_shape = node->get_input_shape(0);
+                    auto weights_shape = node->get_input_shape(1);
+                    auto result_shape = node->get_output_shape(0);
+                    auto data_rank = data_shape.size();
+                    auto weights_rank = weights_shape.size();
+
+                    bool data_dilated = false;
+                    for (size_t s : convolution->get_data_dilation_strides())
+                    {
+                        data_dilated = data_dilated || (s != 1);
+                    }
+
+                    if (!data_dilated && data_rank == 4 && weights_rank == 4 &&
+                        node->get_input_element_type(0) == element::f32)
+                    {
+                        auto op_annotations =
+                            std::make_shared<ngraph::runtime::cpu::CPUOpAnnotations>();
+                        op_annotations->set_mkldnn_op(true);
+                        convolution->set_op_annotations(op_annotations);
+                    }
+                }
+
+                template <>
+                void CPUAssignment::ASSIGN_DECL(ngraph::op::ConvolutionBiasBackpropFiltersBias)
+                {
+                    auto convolution = static_cast<op::ConvolutionBiasBackpropFiltersBias*>(node);
+
+                    auto data_shape = node->get_input_shape(0);
+                    auto delta_shape = node->get_input_shape(1);
+                    auto data_rank = data_shape.size();
+                    auto delta_rank = delta_shape.size();
+
+                    bool data_dilated = false;
+                    for (size_t s : convolution->get_data_dilation_strides_forward())
+                    {
+                        data_dilated = data_dilated || (s != 1);
+                    }
+
+                    if (!data_dilated && data_rank == 4 && delta_rank == 4 &&
+                        node->get_input_element_type(0) == element::f32)
+                    {
+                        auto op_annotations =
+                            std::make_shared<ngraph::runtime::cpu::CPUOpAnnotations>();
+                        op_annotations->set_mkldnn_op(true);
+                        convolution->set_op_annotations(op_annotations);
+                    }
+                }
+
                 template <>
                 void CPUAssignment::ASSIGN_DECL(ngraph::op::AvgPool)
                 {
@@ -266,6 +320,10 @@ static const runtime::cpu::pass::AssignOpMap s_dispatcher{
      &runtime::cpu::pass::CPUAssignment::assign<ngraph::op::ConvolutionBackpropData>},
     {TI(ngraph::op::ConvolutionBackpropFilters),
      &runtime::cpu::pass::CPUAssignment::assign<ngraph::op::ConvolutionBackpropFilters>},
+    {TI(ngraph::op::ConvolutionBias),
+     &runtime::cpu::pass::CPUAssignment::assign<ngraph::op::ConvolutionBias>},
+    {TI(ngraph::op::ConvolutionBiasBackpropFiltersBias),
+     &runtime::cpu::pass::CPUAssignment::assign<ngraph::op::ConvolutionBiasBackpropFiltersBias>},
     {TI(ngraph::op::AvgPool), &runtime::cpu::pass::CPUAssignment::assign<ngraph::op::AvgPool>},
     {TI(ngraph::op::AvgPoolBackprop),
      &runtime::cpu::pass::CPUAssignment::assign<ngraph::op::AvgPoolBackprop>},

src/ngraph/runtime/cpu/pass/cpu_fusion.cpp

@@ -43,6 +43,7 @@
 #include "ngraph/pattern/matcher.hpp"
 #include "ngraph/pattern/op/any.hpp"
 #include "ngraph/pattern/op/label.hpp"
+#include "ngraph/runtime/cpu/ops/conv_bias.hpp"
 #include "ngraph/runtime/cpu/ops/matmul_bias.hpp"
 #include "ngraph/runtime/cpu/ops/sigmoid.hpp"
 
@@ -566,3 +567,37 @@ void ngraph::runtime::cpu::pass::CPUFusion::construct_sigmoid()
     auto m = std::make_shared<ngraph::pattern::Matcher>(divide_1_over_exp, callback);
     this->add_matcher(m);
 }
+
+void ngraph::runtime::cpu::pass::CPUFusion::construct_conv_bias()
+{
+    Shape shape{2, 2, 1, 1};
+    auto data_batch = std::make_shared<pattern::op::Label>(element::f32, shape);
+    auto filters = std::make_shared<pattern::op::Label>(element::f32, shape);
+    auto pbias = std::make_shared<pattern::op::Label>(element::f32, Shape{});
+
+    auto pbroadcast = std::make_shared<op::Broadcast>(pbias, shape, AxisSet{0, 1, 2, 3});
+
+    auto pconv1 = std::make_shared<op::Convolution>(data_batch,
+                                                    filters,
+                                                    Strides{1, 1},
+                                                    Strides{1, 1},
+                                                    CoordinateDiff{0, 0},
+                                                    CoordinateDiff{0, 0},
+                                                    Strides{1, 1});
+    auto p_conv_bias = pbroadcast + pconv1;
+
+    ngraph::pattern::gr_callback_fn callback = [](pattern::Matcher& m) {
+        NGRAPH_DEBUG << "In callback for construct_conv_bias against node = "
+                     << m.match_root()->get_name();
+        auto pattern_map = m.get_pattern_map();
+        std::shared_ptr<Node> nn;
+
+        auto conv = std::dynamic_pointer_cast<op::Convolution>(m.match_root()->get_input_op(0));
+        auto bias = m.match_root()->get_input_op(1)->get_input_op(0);
+        auto conv_bias = std::shared_ptr<Node>(new op::ConvolutionBias(conv, bias));
+        return conv_bias;
+    };
+
+    auto m = std::make_shared<ngraph::pattern::Matcher>(p_conv_bias, callback);
+    this->add_matcher(m);
+}

src/ngraph/runtime/cpu/pass/cpu_fusion.hpp

@@ -44,11 +44,13 @@ public:
         construct_zero_padded_reshaped_conv();
         construct_zero_padded_conv();
         construct_sigmoid();
+        construct_conv_bias();
     }
 
 private:
     void construct_matmul_pattern();
     void construct_matmulbias_pattern();
+    void construct_conv_bias();
     void construct_fprop_bn();
     void construct_sigmoid();
     void construct_zero_padded_reshaped_conv();

src/ngraph/runtime/cpu/pass/cpu_layout.cpp

@@ -37,6 +37,7 @@
 #include "ngraph/runtime/cpu/cpu_layout_descriptor.hpp"
 #include "ngraph/runtime/cpu/cpu_op_annotations.hpp"
 #include "ngraph/runtime/cpu/mkldnn_utils.hpp"
+#include "ngraph/runtime/cpu/ops/conv_bias.hpp"
 #include "ngraph/runtime/cpu/ops/convert_layout.hpp"
 #include "ngraph/runtime/cpu/ops/sigmoid.hpp"
 
@@ -222,12 +223,12 @@ namespace ngraph
         {
             namespace pass
             {
-                template <>
-                void CPULayout::LAYOUT_DECL(ngraph::op::Convolution)
+                template <typename T, bool use_bias>
+                void ConvolutionLayout(std::shared_ptr<ngraph::Node> node,
+                                       vector<memory::format>& prim_input_formats,
+                                       vector<memory::format>& prim_output_formats)
                 {
-                    if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node.get()))
-                    {
-                        auto convolution = static_cast<const ngraph::op::Convolution*>(node.get());
+                    auto convolution = static_cast<const T*>(node.get());
 
                     auto arg0_shape = node->get_input_shape(0);
                     auto arg1_shape = node->get_input_shape(1);
@@ -252,38 +253,88 @@ namespace ngraph
                     memory::dims mkldnn_result_shape(result_shape.begin(), result_shape.end());
                     memory::dims mkldnn_filter_strides(filter_strides.begin(),
                                                        filter_strides.end());
-                        memory::dims mkldnn_dilated_strides(
-                            window_dilation_strides_adjusted.begin(),
+                    memory::dims mkldnn_dilated_strides(window_dilation_strides_adjusted.begin(),
                                                         window_dilation_strides_adjusted.end());
-                        memory::dims mkldnn_padding_below(padding_below.begin(),
-                                                          padding_below.end());
-                        memory::dims mkldnn_padding_above(padding_above.begin(),
-                                                          padding_above.end());
-                        const memory::desc input_data_desc(
-                            mkldnn_arg0_shape, et, memory::format::any);
+                    memory::dims mkldnn_padding_below(padding_below.begin(), padding_below.end());
+                    memory::dims mkldnn_padding_above(padding_above.begin(), padding_above.end());
+                    const memory::desc input_data_desc(mkldnn_arg0_shape, et, memory::format::any);
                     const memory::desc weights_desc(mkldnn_arg1_shape, et, memory::format::any);
-                        const memory::desc result_desc(
-                            mkldnn_result_shape, et, memory::format::any);
-                        convolution_forward::desc fwd_desc(prop_kind::forward,
+                    const memory::desc result_desc(mkldnn_result_shape, et, memory::format::any);
+                    std::unique_ptr<convolution_forward::desc> fwd_desc{nullptr};
+                    if (use_bias)
+                    {
+                        auto arg2_shape = node->get_input_shape(2);
+                        memory::dims mkldnn_arg2_shape(arg2_shape.begin(), arg2_shape.end());
+                        const memory::desc bias_desc(mkldnn_arg2_shape, et, memory::format::any);
+
+                        fwd_desc.reset(new convolution_forward::desc(prop_kind::forward,
                                                                      algorithm::convolution_direct,
                                                                      input_data_desc,
                                                                      weights_desc,
+                                                                     bias_desc, // with bias
                                                                      result_desc,
                                                                      mkldnn_filter_strides,
                                                                      mkldnn_dilated_strides,
                                                                      mkldnn_padding_below,
                                                                      mkldnn_padding_above,
-                                                           padding_kind::zero);
-                        convolution_forward::primitive_desc prim_desc(fwd_desc, cpu_engine);
-                        vector<memory::format> prim_input_formats;
-                        vector<memory::format> prim_output_formats;
+                                                                     padding_kind::zero));
+                    }
+                    else
+                    {
+                        fwd_desc.reset(new convolution_forward::desc(prop_kind::forward,
+                                                                     algorithm::convolution_direct,
+                                                                     input_data_desc,
+                                                                     weights_desc,
+                                                                     result_desc,
+                                                                     mkldnn_filter_strides,
+                                                                     mkldnn_dilated_strides,
+                                                                     mkldnn_padding_below,
+                                                                     mkldnn_padding_above,
+                                                                     padding_kind::zero));
+                    }
+                    convolution_forward::primitive_desc prim_desc(*fwd_desc, cpu_engine);
                     prim_input_formats.push_back(static_cast<memory::format>(
                         prim_desc.src_primitive_desc().desc().data.format));
                     prim_input_formats.push_back(static_cast<memory::format>(
                         prim_desc.weights_primitive_desc().desc().data.format));
+                    if (use_bias)
+                    {
+                        prim_input_formats.push_back(static_cast<memory::format>(
+                            prim_desc.bias_primitive_desc().desc().data.format));
+                    }
                     prim_output_formats.push_back(static_cast<memory::format>(
                         prim_desc.dst_primitive_desc().desc().data.format));
+                }
 
+                template <>
+                void CPULayout::LAYOUT_DECL(ngraph::op::Convolution)
+                {
+                    if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node.get()))
+                    {
+                        vector<memory::format> prim_input_formats;
+                        vector<memory::format> prim_output_formats;
+                        ConvolutionLayout<ngraph::op::Convolution, false>(
+                            node, prim_input_formats, prim_output_formats);
+
+                        node =
+                            insert_input_conversions(external_function, node, prim_input_formats);
+                        set_output_layouts(node, prim_output_formats);
+                    }
+                    else
+                    {
+                        set_default_layouts(external_function, node);
+                    }
+                }
+
+                template <>
+                void CPULayout::LAYOUT_DECL(ngraph::op::ConvolutionBias)
+                {
+                    if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node.get()))
+                    {
+                        vector<memory::format> prim_input_formats;
+                        vector<memory::format> prim_output_formats;
+                        ConvolutionLayout<ngraph::op::ConvolutionBias, true>(
+                            node, prim_input_formats, prim_output_formats);
                         node =
                             insert_input_conversions(external_function, node, prim_input_formats);
                         set_output_layouts(node, prim_output_formats);
@@ -382,17 +433,16 @@ namespace ngraph
                     }
                 }
 
-                template <>
-                void CPULayout::LAYOUT_DECL(ngraph::op::ConvolutionBackpropFilters)
+                template <typename T, bool use_bias>
+                void ConvolutionBackpropFiltersLayout(std::shared_ptr<ngraph::Node> node,
+                                                      vector<memory::format>& prim_input_formats,
+                                                      vector<memory::format>& prim_output_formats)
                 {
-                    if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node.get()))
-                    {
-                        auto convolution =
-                            static_cast<const ngraph::op::ConvolutionBackpropFilters*>(node.get());
+                    auto convolution = static_cast<const T*>(node.get());
 
-                        auto arg0_shape = node->get_input_shape(0);
-                        auto arg1_shape = node->get_input_shape(1);
-                        auto result_shape = node->get_output_shape(0);
+                    auto data_shape = node->get_input_shape(0);
+                    auto delta_shape = node->get_input_shape(1);
+                    auto filters_shape = node->get_output_shape(0);
                     auto filter_strides = convolution->get_window_movement_strides_forward();
                     auto padding_below = convolution->get_padding_below_forward();
                     auto padding_above = convolution->get_padding_above_forward();
@@ -408,57 +458,123 @@ namespace ngraph
                         node->get_input_element_type(0));
 
                     engine cpu_engine(engine::cpu, 0);
-                        memory::dims mkldnn_arg0_shape(arg0_shape.begin(), arg0_shape.end());
-                        memory::dims mkldnn_arg1_shape(arg1_shape.begin(), arg1_shape.end());
-                        memory::dims mkldnn_result_shape(result_shape.begin(), result_shape.end());
+                    memory::dims mkldnn_data_shape(data_shape.begin(), data_shape.end());
+                    memory::dims mkldnn_delta_shape(delta_shape.begin(), delta_shape.end());
+                    memory::dims mkldnn_filters_shape(filters_shape.begin(), filters_shape.end());
                     memory::dims mkldnn_filter_strides(filter_strides.begin(),
                                                        filter_strides.end());
-                        memory::dims mkldnn_dilated_strides(
-                            window_dilation_strides_adjusted.begin(),
+                    memory::dims mkldnn_dilated_strides(window_dilation_strides_adjusted.begin(),
                                                         window_dilation_strides_adjusted.end());
-                        memory::dims mkldnn_padding_below(padding_below.begin(),
-                                                          padding_below.end());
-                        memory::dims mkldnn_padding_above(padding_above.begin(),
-                                                          padding_above.end());
-
-                        const memory::desc data_desc(mkldnn_arg0_shape, et, memory::format::any);
-                        const memory::desc delta_desc(mkldnn_arg1_shape, et, memory::format::any);
-                        const memory::desc result_desc(
-                            mkldnn_result_shape, et, memory::format::any);
-
-                        convolution_backward_weights::desc bwd_desc(algorithm::convolution_direct,
+                    memory::dims mkldnn_padding_below(padding_below.begin(), padding_below.end());
+                    memory::dims mkldnn_padding_above(padding_above.begin(), padding_above.end());
+
+                    const memory::desc data_desc(mkldnn_data_shape, et, memory::format::any);
+                    const memory::desc delta_desc(mkldnn_delta_shape, et, memory::format::any);
+                    const memory::desc filters_desc(mkldnn_filters_shape, et, memory::format::any);
+
+                    std::unique_ptr<convolution_backward_weights::desc> bwd_desc{nullptr};
+                    std::unique_ptr<convolution_forward::desc> fwd_desc{nullptr};
+                    if (use_bias)
+                    {
+                        auto bias_shape = node->get_output_shape(1);
+                        memory::dims mkldnn_bias_shape(bias_shape.begin(), bias_shape.end());
+                        const memory::desc bias_desc(mkldnn_bias_shape, et, memory::format::any);
+                        bwd_desc.reset(
+                            new convolution_backward_weights::desc(algorithm::convolution_direct,
                                                                    data_desc,
-                                                                    result_desc,
+                                                                   filters_desc,
+                                                                   bias_desc,
                                                                    delta_desc,
                                                                    mkldnn_filter_strides,
                                                                    mkldnn_dilated_strides,
                                                                    mkldnn_padding_below,
                                                                    mkldnn_padding_above,
-                                                                    padding_kind::zero);
+                                                                   padding_kind::zero));
 
-                        convolution_forward::desc fwd_desc(prop_kind::forward,
+                        fwd_desc.reset(new convolution_forward::desc(prop_kind::forward,
                                                                      algorithm::convolution_direct,
                                                                      data_desc,
-                                                           result_desc,
+                                                                     filters_desc,
+                                                                     bias_desc,
                                                                      delta_desc,
                                                                      mkldnn_filter_strides,
                                                                      mkldnn_dilated_strides,
                                                                      mkldnn_padding_below,
                                                                      mkldnn_padding_above,
-                                                           padding_kind::zero);
-                        convolution_forward::primitive_desc fwd_prim_desc(fwd_desc, cpu_engine);
+                                                                     padding_kind::zero));
+                    }
+                    else
+                    {
+                        bwd_desc.reset(
+                            new convolution_backward_weights::desc(algorithm::convolution_direct,
+                                                                   data_desc,
+                                                                   filters_desc,
+                                                                   delta_desc,
+                                                                   mkldnn_filter_strides,
+                                                                   mkldnn_dilated_strides,
+                                                                   mkldnn_padding_below,
+                                                                   mkldnn_padding_above,
+                                                                   padding_kind::zero));
 
+                        fwd_desc.reset(new convolution_forward::desc(prop_kind::forward,
+                                                                     algorithm::convolution_direct,
+                                                                     data_desc,
+                                                                     filters_desc,
+                                                                     delta_desc,
+                                                                     mkldnn_filter_strides,
+                                                                     mkldnn_dilated_strides,
+                                                                     mkldnn_padding_below,
+                                                                     mkldnn_padding_above,
+                                                                     padding_kind::zero));
+                    }
+                    convolution_forward::primitive_desc fwd_prim_desc(*fwd_desc, cpu_engine);
                     convolution_backward_weights::primitive_desc prim_desc(
-                            bwd_desc, cpu_engine, fwd_prim_desc);
+                        *bwd_desc, cpu_engine, fwd_prim_desc);
 
-                        vector<memory::format> prim_input_formats;
-                        vector<memory::format> prim_output_formats;
                     prim_input_formats.push_back(static_cast<memory::format>(
                         prim_desc.src_primitive_desc().desc().data.format));
                     prim_input_formats.push_back(static_cast<memory::format>(
                         prim_desc.diff_dst_primitive_desc().desc().data.format));
                     prim_output_formats.push_back(static_cast<memory::format>(
                         prim_desc.diff_weights_primitive_desc().desc().data.format));
+                    if (use_bias)
+                    {
+                        prim_output_formats.push_back(static_cast<memory::format>(
+                            prim_desc.diff_bias_primitive_desc().desc().data.format));
+                    }
+                }
+
+                template <>
+                void CPULayout::LAYOUT_DECL(ngraph::op::ConvolutionBackpropFilters)
+                {
+                    if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node.get()))
+                    {
+                        vector<memory::format> prim_input_formats;
+                        vector<memory::format> prim_output_formats;
+                        ConvolutionBackpropFiltersLayout<ngraph::op::ConvolutionBackpropFilters,
+                                                         false>(
+                            node, prim_input_formats, prim_output_formats);
+
+                        node =
+                            insert_input_conversions(external_function, node, prim_input_formats);
+                        set_output_layouts(node, prim_output_formats);
+                    }
+                    else
+                    {
+                        set_default_layouts(external_function, node);
+                    }
+                }
+
+                template <>
+                void CPULayout::LAYOUT_DECL(ngraph::op::ConvolutionBiasBackpropFiltersBias)
+                {
+                    if (runtime::cpu::mkldnn_utils::use_mkldnn_kernel(node.get()))
+                    {
+                        vector<memory::format> prim_input_formats;
+                        vector<memory::format> prim_output_formats;
+                        ConvolutionBackpropFiltersLayout<
+                            ngraph::op::ConvolutionBiasBackpropFiltersBias,
+                            true>(node, prim_input_formats, prim_output_formats);
 
                         node =
                             insert_input_conversions(external_function, node, prim_input_formats);
@@ -773,6 +889,10 @@ static const runtime::cpu::pass::LayoutOpMap s_dispatcher{
      &runtime::cpu::pass::CPULayout::layout<ngraph::op::ConvolutionBackpropData>},
     {TI(ngraph::op::ConvolutionBackpropFilters),
      &runtime::cpu::pass::CPULayout::layout<ngraph::op::ConvolutionBackpropFilters>},
+    {TI(ngraph::op::ConvolutionBias),
+     &runtime::cpu::pass::CPULayout::layout<ngraph::op::ConvolutionBias>},
+    {TI(ngraph::op::ConvolutionBiasBackpropFiltersBias),
+     &runtime::cpu::pass::CPULayout::layout<ngraph::op::ConvolutionBiasBackpropFiltersBias>},
     {TI(ngraph::op::AvgPool), &runtime::cpu::pass::CPULayout::layout<ngraph::op::AvgPool>},
     {TI(ngraph::op::AvgPoolBackprop),
      &runtime::cpu::pass::CPULayout::layout<ngraph::op::AvgPoolBackprop>},

test/cpu_fusion.cpp

@@ -21,14 +21,18 @@
 #include <memory>
 
 #include "gtest/gtest.h"
+#include "ngraph/file_util.hpp"
 #include "ngraph/graph_util.hpp"
 #include "ngraph/log.hpp"
 #include "ngraph/ngraph.hpp"
 #include "ngraph/ops/batch_norm.hpp"
 #include "ngraph/ops/get_output_element.hpp"
+#include "ngraph/ops/parameter.hpp"
 #include "ngraph/ops/sum.hpp"
 #include "ngraph/pass/graph_rewrite.hpp"
 #include "ngraph/pass/manager.hpp"
+#include "ngraph/pass/reshape_elimination.hpp"
+#include "ngraph/pass/visualize_tree.hpp"
 #include "ngraph/pattern/matcher.hpp"
 #include "ngraph/pattern/op/any.hpp"
 #include "ngraph/pattern/op/label.hpp"
@@ -36,6 +40,7 @@
 #include "ngraph/file_util.hpp"
 #include "ngraph/pass/reshape_elimination.hpp"
 #include "ngraph/pass/visualize_tree.hpp"
+#include "ngraph/runtime/cpu/ops/conv_bias.hpp"
 #include "ngraph/runtime/cpu/ops/matmul_bias.hpp"
 #include "ngraph/runtime/cpu/ops/sigmoid.hpp"
 #include "ngraph/runtime/cpu/pass/cpu_fusion.hpp"
@@ -672,6 +677,190 @@ TEST(cpu_fusion, non_zero_padded_conv)
 
     ASSERT_EQ(count_ops_of_type<op::Pad>(func), 1);
 }
+TEST(cpu_fusion, fuse_conv_bias)
+{
+    pass::Manager pass_manager;
+    pass_manager.register_pass<ngraph::pass::ReshapeElimination>();
+    pass_manager.register_pass<runtime::cpu::pass::CPUFusion>();
+    const string json_path = file_util::path_join(SERIALIZED_ZOO, "conv_bias.json");
+    const string json_string = file_util::read_file_to_string(json_path);
+    stringstream ss(json_string);
+    shared_ptr<Function> func = ngraph::deserialize(ss);
+    pass_manager.run_passes(func);
+    size_t cb = count_ops_of_type<op::ConvolutionBias>(func);
+    ASSERT_GT(cb, 0);
+}
+
+struct ConvolutionBiasTestData
+{
+    size_t n{0};
+    size_t c{0};
+    size_t filter{0};
+    size_t kernel_size{0};
+    size_t w{0};
+    size_t h{0};
+    shared_ptr<runtime::TensorView> data_val;
+    shared_ptr<runtime::TensorView> weights_val;
+    shared_ptr<runtime::TensorView> bias_val;
+    shared_ptr<runtime::TensorView> result_val;
+    shared_ptr<runtime::TensorView> delta_val;
+    shared_ptr<runtime::TensorView> d_data_val;
+    shared_ptr<runtime::TensorView> d_weights_val;
+    shared_ptr<runtime::TensorView> d_bias_val;
+    vector<float> expected_result_val;
+    vector<float> expected_d_data_val;
+    vector<float> expected_d_weights_val;
+    vector<float> expected_d_bias_val;
+
+    Shape data_shape;
+    Shape weights_shape;
+    Shape bias_shape;
+    Shape result_shape;
+    shared_ptr<op::Parameter> data;
+    shared_ptr<op::Parameter> weights;
+    shared_ptr<op::Parameter> bias;
+    shared_ptr<op::Parameter> delta;
+
+    void n1c1h3w3(shared_ptr<runtime::Backend> backend)
+    {
+        n = 1;
+        c = 1;
+        filter = 1;
+        kernel_size = 3;
+        w = 3;
+        h = w;
+
+        data_shape = Shape{n, c, h, w};
+        data = make_shared<op::Parameter>(element::f32, data_shape);
+        weights_shape = Shape{filter, c, kernel_size, kernel_size};
+        weights = make_shared<op::Parameter>(element::f32, weights_shape);
+        bias_shape = Shape{filter};
+        bias = make_shared<op::Parameter>(element::f32, bias_shape);
+        result_shape = Shape{n, filter, 1, 1};
+
+        data_val = backend->make_primary_tensor_view(element::f32, data_shape);
+        copy_data(data_val,
+                  vector<float>{-0.67765152f,
+                                0.10073948f,
+                                0.57595438f,
+                                -0.3469252f,
+                                -0.22134334f,
+                                -1.80471897f,
+                                -0.80642909f,
+                                1.22033095f,
+                                2.23235631f});
+        weights_val = backend->make_primary_tensor_view(element::f32, weights_shape);
+        copy_data(weights_val,
+                  vector<float>{0.20070229f,
+                                -0.54968649f,
+                                -0.19819015f,
+                                -0.38577855f,
+                                1.37109005f,
+                                -0.23789984f,
+                                0.14867957f,
+                                -0.49851316f,
+                                -0.84815776f});
+        bias_val = backend->make_primary_tensor_view(element::f32, bias_shape);
+        copy_data(bias_val, vector<float>{0.07811152f});
+
+        result_val = backend->make_primary_tensor_view(element::f32, result_shape);
+        copy_data(result_val, vector<float>{0});
+
+        delta = make_shared<op::Parameter>(element::f32, result_shape);
+        delta_val = backend->make_primary_tensor_view(element::f32, result_shape);
+        copy_data(delta_val, vector<float>{-2.58936238f});
+
+        d_data_val = backend->make_primary_tensor_view(element::f32, data_shape);
+        copy_data(d_data_val, vector<float>{0, 0, 0, 0, 0, 0, 0, 0, 0});
+
+        d_weights_val = backend->make_primary_tensor_view(element::f32, weights_shape);
+        copy_data(d_weights_val, vector<float>{0, 0, 0, 0, 0, 0, 0, 0, 0});
+
+        d_bias_val = backend->make_primary_tensor_view(element::f32, bias_shape);
+        copy_data(d_bias_val, vector<float>{0});
+
+        expected_result_val = vector<float>{-2.58936238f};
+        expected_d_data_val = vector<float>{-0.51969099f,
+                                            1.42333758f,
+                                            0.5131861f,
+                                            0.99892044f,
+                                            -3.5502491f,
+                                            0.61600888f,
+                                            -0.3849853f,
+                                            1.29083121f,
+                                            2.19618773f};
+        expected_d_weights_val = vector<float>{1.7546854f,
+                                               -0.26085103f,
+                                               -1.49135458f,
+                                               0.89831507f,
+                                               0.57313812f,
+                                               4.67307138f,
+                                               2.08813715f,
+                                               -3.15987897f,
+                                               -5.7803793f};
+        expected_d_bias_val = vector<float>{-2.58936238f};
+    }
+};
+
+TEST(cpu_fusion, conv_bias_fprop_n1c1h3w3)
+{
+    auto manager = runtime::Manager::get("CPU");
+    auto backend = manager->allocate_backend();
+
+    ConvolutionBiasTestData conv_test;
+    conv_test.n1c1h3w3(backend);
+
+    auto convolution = make_shared<op::Convolution>(conv_test.data, conv_test.weights);
+    auto convolution_bias = make_shared<op::ConvolutionBias>(convolution, conv_test.bias);
+
+    auto f = make_shared<Function>(
+        convolution_bias, op::ParameterVector{conv_test.data, conv_test.weights, conv_test.bias});
+    auto external = manager->compile(f);
+    auto cf = backend->make_call_frame(external);
+
+    cf->call({conv_test.data_val, conv_test.weights_val, conv_test.bias_val},
+             {conv_test.result_val});
+    auto result_vec = read_vector<float>(conv_test.result_val);
+
+    EXPECT_TRUE(
+        test::all_close(conv_test.expected_result_val, read_vector<float>(conv_test.result_val)));
+}
+
+TEST(cpu_fusion, conv_bias_bprop_n1c1h3w3)
+{
+    auto manager = runtime::Manager::get("CPU");
+    auto backend = manager->allocate_backend();
+
+    ConvolutionBiasTestData conv_test;
+    conv_test.n1c1h3w3(backend);
+
+    auto convolution = make_shared<op::Convolution>(conv_test.data, conv_test.weights);
+    auto convolution_bias = make_shared<op::ConvolutionBias>(convolution, conv_test.bias);
+
+    auto f = make_shared<Function>(
+        convolution_bias, op::ParameterVector{conv_test.data, conv_test.weights, conv_test.bias});
+
+    auto d_data = convolution_bias->backprop_node(conv_test.data, conv_test.delta);
+    auto d_weights = convolution_bias->backprop_node(conv_test.weights, conv_test.delta);
+    auto d_bias = convolution_bias->backprop_node(conv_test.bias, conv_test.delta);
+
+    auto df = make_shared<Function>(
+        NodeVector{d_data, d_weights, d_bias},
+        op::ParameterVector{conv_test.data, conv_test.weights, conv_test.bias, conv_test.delta});
+
+    auto external = manager->compile(df);
+    auto cf = backend->make_call_frame(external);
+
+    cf->call({conv_test.data_val, conv_test.weights_val, conv_test.bias_val, conv_test.delta_val},
+             {conv_test.d_data_val, conv_test.d_weights_val, conv_test.d_bias_val});
+
+    EXPECT_TRUE(
+        test::all_close(conv_test.expected_d_data_val, read_vector<float>(conv_test.d_data_val)));
+    EXPECT_TRUE(test::all_close(conv_test.expected_d_weights_val,
+                                read_vector<float>(conv_test.d_weights_val)));
+    EXPECT_TRUE(
+        test::all_close(conv_test.expected_d_bias_val, read_vector<float>(conv_test.d_bias_val)));
+}
 
 TEST(cpu_fusion, sigmoid_fprop_fusion)
 {

test/models/conv_bias.json

+[{
+  "name" : "Function_4",
+  "ops" : [
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_747",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_747_0"],
+      "shape" : [1000]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_746",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_746_0"],
+      "shape" : [ 1000, 4096 ]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_738",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_738_0"],
+      "shape" : [4096]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_737",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_737_0"],
+      "shape" : [ 4096, 4096 ]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_729",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_729_0"],
+      "shape" : [4096]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_728",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_728_0"],
+      "shape" : [ 4096, 25088 ]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_717",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_717_0"],
+      "shape" : [512]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_716",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_716_0"],
+      "shape" : [ 512, 512, 3, 3 ]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_707",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_707_0"],
+      "shape" : [512]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_706",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_706_0"],
+      "shape" : [ 512, 512, 3, 3 ]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_696",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_696_0"],
+      "shape" : [512]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_695",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_695_0"],
+      "shape" : [ 512, 512, 3, 3 ]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_686",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_686_0"],
+      "shape" : [512]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_685",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_685_0"],
+      "shape" : [ 512, 256, 3, 3 ]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_675",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_675_0"],
+      "shape" : [256]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_674",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_674_0"],
+      "shape" : [ 256, 256, 3, 3 ]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_665",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_665_0"],
+      "shape" : [256]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_664",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_664_0"],
+      "shape" : [ 256, 128, 3, 3 ]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_654",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_654_0"],
+      "shape" : [128]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_653",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_653_0"],
+      "shape" : [ 128, 64, 3, 3 ]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_643",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_643_0"],
+      "shape" : [64]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_642",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_642_0"],
+      "shape" : [ 64, 3, 3, 3 ]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Parameter_641",
+      "op" : "Parameter",
+      "outputs" : ["Parameter_641_0"],
+      "shape" : [ 32, 3, 224, 224 ]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Constant_743",
+      "op" : "Constant",
+      "outputs" : ["Constant_743_0"],
+      "shape" : [],
+      "value" : ["0"]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Constant_734",
+      "op" : "Constant",
+      "outputs" : ["Constant_734_0"],
+      "shape" : [],
+      "value" : ["0"]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Constant_723",
+      "op" : "Constant",
+      "outputs" : ["Constant_723_0"],
+      "shape" : [],
+      "value" : ["0"]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Constant_713",
+      "op" : "Constant",
+      "outputs" : ["Constant_713_0"],
+      "shape" : [],
+      "value" : ["0"]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Constant_702",
+      "op" : "Constant",
+      "outputs" : ["Constant_702_0"],
+      "shape" : [],
+      "value" : ["0"]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Constant_692",
+      "op" : "Constant",
+      "outputs" : ["Constant_692_0"],
+      "shape" : [],
+      "value" : ["0"]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Constant_681",
+      "op" : "Constant",
+      "outputs" : ["Constant_681_0"],
+      "shape" : [],
+      "value" : ["0"]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Constant_671",
+      "op" : "Constant",
+      "outputs" : ["Constant_671_0"],
+      "shape" : [],
+      "value" : ["0"]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Constant_660",
+      "op" : "Constant",
+      "outputs" : ["Constant_660_0"],
+      "shape" : [],
+      "value" : ["0"]
+    },
+    {
+      "element_type" : "float",
+      "inputs" : [],
+      "name" : "Constant_649",
+      "op" : "Constant",
+      "outputs" : ["Constant_649_0"],
+      "shape" : [],
+      "value" : ["0"]
+    },
+    {
+      "axes" : [0],
+      "inputs" : ["Parameter_747"],
+      "name" : "Broadcast_750",
+      "op" : "Broadcast",
+      "outputs" : ["Broadcast_750_0"],
+      "shape" : [ 32, 1000 ]
+    },
+    {
+      "input_order" : [ 1, 0 ],
+      "inputs" : ["Parameter_746"],
+      "name" : "Reshape_748",
+      "op" : "Reshape",
+      "output_shape" : [ 4096, 1000 ],
+      "outputs" : ["Reshape_748_0"]
+    },
+    {
+      "axes" : [0],
+      "inputs" : ["Parameter_738"],
+      "name" : "Broadcast_741",
+      "op" : "Broadcast",
+      "outputs" : ["Broadcast_741_0"],
+      "shape" : [ 32, 4096 ]
+    },
+    {
+      "input_order" : [ 1, 0 ],
+      "inputs" : ["Parameter_737"],
+      "name" : "Reshape_739",
+      "op" : "Reshape",
+      "output_shape" : [ 4096, 4096 ],
+      "outputs" : ["Reshape_739_0"]
+    },
+    {
+      "axes" : [0],
+      "inputs" : ["Parameter_729"],
+      "name" : "Broadcast_732",
+      "op" : "Broadcast",
+      "outputs" : ["Broadcast_732_0"],
+      "shape" : [ 32, 4096 ]
+    },
+    {
+      "input_order" : [ 1, 0 ],
+      "inputs" : ["Parameter_728"],
+      "name" : "Reshape_730",
+      "op" : "Reshape",
+      "output_shape" : [ 25088, 4096 ],
+      "outputs" : ["Reshape_730_0"]
+    },
+    {
+      "input_order" : [0],
+      "inputs" : ["Parameter_717"],
+      "name" : "Reshape_719",
+      "op" : "Reshape",
+      "output_shape" : [ 1, 512, 1, 1 ],
+      "outputs" : ["Reshape_719_0"]
+    },
+    {
+      "input_order" : [0],
+      "inputs" : ["Parameter_707"],
+      "name" : "Reshape_709",
+      "op" : "Reshape",
+      "output_shape" : [ 1, 512, 1, 1 ],
+      "outputs" : ["Reshape_709_0"]
+    },
+    {
+      "input_order" : [0],
+      "inputs" : ["Parameter_696"],
+      "name" : "Reshape_698",
+      "op" : "Reshape",
+      "output_shape" : [ 1, 512, 1, 1 ],
+      "outputs" : ["Reshape_698_0"]
+    },
+    {
+      "input_order" : [0],
+      "inputs" : ["Parameter_686"],
+      "name" : "Reshape_688",
+      "op" : "Reshape",
+      "output_shape" : [ 1, 512, 1, 1 ],
+      "outputs" : ["Reshape_688_0"]
+    },
+    {
+      "input_order" : [0],
+      "inputs" : ["Parameter_675"],
+      "name" : "Reshape_677",
+      "op" : "Reshape",
+      "output_shape" : [ 1, 256, 1, 1 ],
+      "outputs" : ["Reshape_677_0"]
+    },
+    {
+      "input_order" : [0],
+      "inputs" : ["Parameter_665"],
+      "name" : "Reshape_667",
+      "op" : "Reshape",
+      "output_shape" : [ 1, 256, 1, 1 ],
+      "outputs" : ["Reshape_667_0"]
+    },
+    {
+      "input_order" : [0],
+      "inputs" : ["Parameter_654"],
+      "name" : "Reshape_656",
+      "op" : "Reshape",
+      "output_shape" : [ 1, 128, 1, 1 ],
+      "outputs" : ["Reshape_656_0"]
+    },
+    {
+      "input_order" : [0],
+      "inputs" : ["Parameter_643"],
+      "name" : "Reshape_645",
+      "op" : "Reshape",
+      "output_shape" : [ 1, 64, 1, 1 ],
+      "outputs" : ["Reshape_645_0"]
+    },
+    {
+      "data_dilation_strides" : [ 1, 1 ],
+      "inputs" : [ "Parameter_641", "Parameter_642" ],
+      "name" : "Convolution_644",
+      "op" : "Convolution",
+      "outputs" : ["Convolution_644_0"],
+      "padding_above" : [ 1, 1 ],
+      "padding_below" : [ 1, 1 ],
+      "window_dilation_strides" : [ 1, 1 ],
+      "window_movement_strides" : [ 1, 1 ]
+    },
+    {
+      "axes" : [ 0, 1 ],
+      "inputs" : ["Constant_743"],
+      "name" : "Broadcast_744",
+      "op" : "Broadcast",
+      "outputs" : ["Broadcast_744_0"],
+      "shape" : [ 32, 4096 ]
+    },
+    {
+      "axes" : [ 0, 1 ],
+      "inputs" : ["Constant_734"],
+      "name" : "Broadcast_735",
+      "op" : "Broadcast",
+      "outputs" : ["Broadcast_735_0"],
+      "shape" : [ 32, 4096 ]
+    },
+    {
+      "axes" : [ 0, 1, 2, 3 ],
+      "inputs" : ["Constant_723"],
+      "name" : "Broadcast_724",
+      "op" : "Broadcast",
+      "outputs" : ["Broadcast_724_0"],
+      "shape" : [ 32, 512, 14, 14 ]
+    },
+    {
+      "axes" : [ 0, 1, 2, 3 ],
+      "inputs" : ["Constant_713"],
+      "name" : "Broadcast_714",
+      "op" : "Broadcast",
+      "outputs" : ["Broadcast_714_0"],
+      "shape" : [ 32, 512, 14, 14 ]
+    },
+    {
+      "axes" : [ 0, 1, 2, 3 ],
+      "inputs" : ["Constant_702"],
+      "name" : "Broadcast_703",
+      "op" : "Broadcast",
+      "outputs" : ["Broadcast_703_0"],
+      "shape" : [ 32, 512, 28, 28 ]
+    },
+    {
+      "axes" : [ 0, 1, 2, 3 ],
+      "inputs" : ["Constant_692"],
+      "name" : "Broadcast_693",
+      "op" : "Broadcast",
+      "outputs" : ["Broadcast_693_0"],
+      "shape" : [ 32, 512, 28, 28 ]
+    },
+    {
+      "axes" : [ 0, 1, 2, 3 ],
+      "inputs" : ["Constant_681"],
+      "name" : "Broadcast_682",
+      "op" : "Broadcast",
+      "outputs" : ["Broadcast_682_0"],
+      "shape" : [ 32, 256, 56, 56 ]
+    },
+    {
+      "axes" : [ 0, 1, 2, 3 ],
+      "inputs" : ["Constant_671"],
+      "name" : "Broadcast_672",
+      "op" : "Broadcast",
+      "outputs" : ["Broadcast_672_0"],
+      "shape" : [ 32, 256, 56, 56 ]
+    },
+    {
+      "axes" : [ 0, 1, 2, 3 ],
+      "inputs" : ["Constant_660"],
+      "name" : "Broadcast_661",
+      "op" : "Broadcast",
+      "outputs" : ["Broadcast_661_0"],
+      "shape" : [ 32, 128, 112, 112 ]
+    },
+    {
+      "axes" : [ 0, 1, 2, 3 ],
+      "inputs" : ["Constant_649"],
+      "name" : "Broadcast_650",
+      "op" : "Broadcast",
+      "outputs" : ["Broadcast_650_0"],
+      "shape" : [ 32, 64, 224, 224 ]
+    },
+    {
+      "input_order" : [ 0, 1, 2, 3 ],
+      "inputs" : ["Reshape_719"],
+      "name" : "Reshape_720",
+      "op" : "Reshape",
+      "output_shape" : [512],
+      "outputs" : ["Reshape_720_0"]
+    },
+    {
+      "input_order" : [ 0, 1, 2, 3 ],
+      "inputs" : ["Reshape_709"],
+      "name" : "Reshape_710",
+      "op" : "Reshape",
+      "output_shape" : [512],
+      "outputs" : ["Reshape_710_0"]
+    },
+    {
+      "input_order" : [ 0, 1, 2, 3 ],
+      "inputs" : ["Reshape_698"],
+      "name" : "Reshape_699",
+      "op" : "Reshape",
+      "output_shape" : [512],
+      "outputs" : ["Reshape_699_0"]
+    },
+    {
+      "input_order" : [ 0, 1, 2, 3 ],
+      "inputs" : ["Reshape_688"],
+      "name" : "Reshape_689",
+      "op" : "Reshape",
+      "output_shape" : [512],
+      "outputs" : ["Reshape_689_0"]
+    },
+    {
+      "input_order" : [ 0, 1, 2, 3 ],
+      "inputs" : ["Reshape_677"],
+      "name" : "Reshape_678",
+      "op" : "Reshape",
+      "output_shape" : [256],
+      "outputs" : ["Reshape_678_0"]
+    },
+    {
+      "input_order" : [ 0, 1, 2, 3 ],
+      "inputs" : ["Reshape_667"],
+      "name" : "Reshape_668",
+      "op" : "Reshape",
+      "output_shape" : [256],
+      "outputs" : ["Reshape_668_0"]
+    },
+    {
+      "input_order" : [ 0, 1, 2, 3 ],
+      "inputs" : ["Reshape_656"],
+      "name" : "Reshape_657",
+      "op" : "Reshape",
+      "output_shape" : [128],
+      "outputs" : ["Reshape_657_0"]
+    },
+    {
+      "input_order" : [ 0, 1, 2, 3 ],
+      "inputs" : ["Reshape_645"],
+      "name" : "Reshape_646",
+      "op" : "Reshape",
+      "output_shape" : [64],
+      "outputs" : ["Reshape_646_0"]
+    },
+    {
+      "axes" : [ 0, 2, 3 ],
+      "inputs" : ["Reshape_720"],
+      "name" : "Broadcast_721",
+      "op" : "Broadcast",
+      "outputs" : ["Broadcast_721_0"],
+      "shape" : [ 32, 512, 14, 14 ]
+    },
+    {
+      "axes" : [ 0, 2, 3 ],
+      "inputs" : ["Reshape_710"],
+      "name" : "Broadcast_711",
+      "op" : "Broadcast",
+      "outputs" : ["Broadcast_711_0"],
+      "shape" : [ 32, 512, 14, 14 ]
+    },
+    {
+      "axes" : [ 0, 2, 3 ],
+      "inputs" : ["Reshape_699"],
+      "name" : "Broadcast_700",
+      "op" : "Broadcast",
+      "outputs" : ["Broadcast_700_0"],
+      "shape" : [ 32, 512, 28, 28 ]
+    },
+    {
+      "axes" : [ 0, 2, 3 ],
+      "inputs" : ["Reshape_689"],
+      "name" : "Broadcast_690",
+      "op" : "Broadcast",
+      "outputs" : ["Broadcast_690_0"],
+      "shape" : [ 32, 512, 28, 28 ]
+    },
+    {
+      "axes" : [ 0, 2, 3 ],
+      "inputs" : ["Reshape_678"],
+      "name" : "Broadcast_679",
+      "op" : "Broadcast",
+      "outputs" : ["Broadcast_679_0"],
+      "shape" : [ 32, 256, 56, 56 ]
+    },
+    {
+      "axes" : [ 0, 2, 3 ],
+      "inputs" : ["Reshape_668"],
+      "name" : "Broadcast_669",
+      "op" : "Broadcast",
+      "outputs" : ["Broadcast_669_0"],
+      "shape" : [ 32, 256, 56, 56 ]
+    },
+    {
+      "axes" : [ 0, 2, 3 ],
+      "inputs" : ["Reshape_657"],
+      "name" : "Broadcast_658",
+      "op" : "Broadcast",
+      "outputs" : ["Broadcast_658_0"],
+      "shape" : [ 32, 128, 112, 112 ]
+    },
+    {
+      "axes" : [ 0, 2, 3 ],
+      "inputs" : ["Reshape_646"],
+      "name" : "Broadcast_647",
+      "op" : "Broadcast",
+      "outputs" : ["Broadcast_647_0"],
+      "shape" : [ 32, 64, 224, 224 ]
+    },
+    {
+      "inputs" : [ "Convolution_644", "Broadcast_647" ],
+      "name" : "Add_648",
+      "op" : "Add",
+      "outputs" : ["Add_648_0"]
+    },
+    {
+      "inputs" : [ "Add_648", "Broadcast_650" ],
+      "name" : "Maximum_651",
+      "op" : "Maximum",
+      "outputs" : ["Maximum_651_0"]
+    },
+    {
+      "inputs" : ["Maximum_651"],
+      "name" : "MaxPool_652",
+      "op" : "MaxPool",
+      "outputs" : ["MaxPool_652_0"],
+      "padding_above" : [ 0, 0 ],
+      "padding_below" : [ 0, 0 ],
+      "window_movement_strides" : [ 2, 2 ],
+      "window_shape" : [ 2, 2 ]
+    },
+    {
+      "data_dilation_strides" : [ 1, 1 ],
+      "inputs" : [ "MaxPool_652", "Parameter_653" ],
+      "name" : "Convolution_655",
+      "op" : "Convolution",
+      "outputs" : ["Convolution_655_0"],
+      "padding_above" : [ 1, 1 ],
+      "padding_below" : [ 1, 1 ],
+      "window_dilation_strides" : [ 1, 1 ],
+      "window_movement_strides" : [ 1, 1 ]
+    },
+    {
+      "inputs" : [ "Convolution_655", "Broadcast_658" ],
+      "name" : "Add_659",
+      "op" : "Add",
+      "outputs" : ["Add_659_0"]
+    },
+    {
+      "inputs" : [ "Add_659", "Broadcast_661" ],
+      "name" : "Maximum_662",
+      "op" : "Maximum",
+      "outputs" : ["Maximum_662_0"]
+    },
+    {
+      "inputs" : ["Maximum_662"],
+      "name" : "MaxPool_663",
+      "op" : "MaxPool",
+      "outputs" : ["MaxPool_663_0"],
+      "padding_above" : [ 0, 0 ],
+      "padding_below" : [ 0, 0 ],
+      "window_movement_strides" : [ 2, 2 ],
+      "window_shape" : [ 2, 2 ]
+    },
+    {
+      "data_dilation_strides" : [ 1, 1 ],
+      "inputs" : [ "MaxPool_663", "Parameter_664" ],
+      "name" : "Convolution_666",
+      "op" : "Convolution",
+      "outputs" : ["Convolution_666_0"],
+      "padding_above" : [ 1, 1 ],
+      "padding_below" : [ 1, 1 ],
+      "window_dilation_strides" : [ 1, 1 ],
+      "window_movement_strides" : [ 1, 1 ]
+    },
+    {
+      "inputs" : [ "Convolution_666", "Broadcast_669" ],
+      "name" : "Add_670",
+      "op" : "Add",
+      "outputs" : ["Add_670_0"]
+    },
+    {
+      "inputs" : [ "Add_670", "Broadcast_672" ],
+      "name" : "Maximum_673",
+      "op" : "Maximum",
+      "outputs" : ["Maximum_673_0"]
+    },
+    {
+      "data_dilation_strides" : [ 1, 1 ],
+      "inputs" : [ "Maximum_673", "Parameter_674" ],
+      "name" : "Convolution_676",
+      "op" : "Convolution",
+      "outputs" : ["Convolution_676_0"],
+      "padding_above" : [ 1, 1 ],
+      "padding_below" : [ 1, 1 ],
+      "window_dilation_strides" : [ 1, 1 ],
+      "window_movement_strides" : [ 1, 1 ]
+    },
+    {
+      "inputs" : [ "Convolution_676", "Broadcast_679" ],
+      "name" : "Add_680",
+      "op" : "Add",
+      "outputs" : ["Add_680_0"]
+    },
+    {
+      "inputs" : [ "Add_680", "Broadcast_682" ],
+      "name" : "Maximum_683",
+      "op" : "Maximum",
+      "outputs" : ["Maximum_683_0"]
+    },
+    {
+      "inputs" : ["Maximum_683"],
+      "name" : "MaxPool_684",
+      "op" : "MaxPool",
+      "outputs" : ["MaxPool_684_0"],
+      "padding_above" : [ 0, 0 ],
+      "padding_below" : [ 0, 0 ],
+      "window_movement_strides" : [ 2, 2 ],
+      "window_shape" : [ 2, 2 ]
+    },
+    {
+      "data_dilation_strides" : [ 1, 1 ],
+      "inputs" : [ "MaxPool_684", "Parameter_685" ],
+      "name" : "Convolution_687",
+      "op" : "Convolution",
+      "outputs" : ["Convolution_687_0"],
+      "padding_above" : [ 1, 1 ],
+      "padding_below" : [ 1, 1 ],
+      "window_dilation_strides" : [ 1, 1 ],
+      "window_movement_strides" : [ 1, 1 ]
+    },
+    {
+      "inputs" : [ "Convolution_687", "Broadcast_690" ],
+      "name" : "Add_691",
+      "op" : "Add",
+      "outputs" : ["Add_691_0"]
+    },
+    {
+      "inputs" : [ "Add_691", "Broadcast_693" ],
+      "name" : "Maximum_694",
+      "op" : "Maximum",
+      "outputs" : ["Maximum_694_0"]
+    },
+    {
+      "data_dilation_strides" : [ 1, 1 ],
+      "inputs" : [ "Maximum_694", "Parameter_695" ],
+      "name" : "Convolution_697",
+      "op" : "Convolution",
+      "outputs" : ["Convolution_697_0"],
+      "padding_above" : [ 1, 1 ],
+      "padding_below" : [ 1, 1 ],
+      "window_dilation_strides" : [ 1, 1 ],
+      "window_movement_strides" : [ 1, 1 ]
+    },
+    {
+      "inputs" : [ "Convolution_697", "Broadcast_700" ],
+      "name" : "Add_701",
+      "op" : "Add",
+      "outputs" : ["Add_701_0"]
+    },
+    {
+      "inputs" : [ "Add_701", "Broadcast_703" ],
+      "name" : "Maximum_704",
+      "op" : "Maximum",
+      "outputs" : ["Maximum_704_0"]
+    },
+    {
+      "inputs" : ["Maximum_704"],
+      "name" : "MaxPool_705",
+      "op" : "MaxPool",
+      "outputs" : ["MaxPool_705_0"],
+      "padding_above" : [ 0, 0 ],
+      "padding_below" : [ 0, 0 ],
+      "window_movement_strides" : [ 2, 2 ],
+      "window_shape" : [ 2, 2 ]
+    },
+    {
+      "data_dilation_strides" : [ 1, 1 ],
+      "inputs" : [ "MaxPool_705", "Parameter_706" ],
+      "name" : "Convolution_708",
+      "op" : "Convolution",
+      "outputs" : ["Convolution_708_0"],
+      "padding_above" : [ 1, 1 ],
+      "padding_below" : [ 1, 1 ],
+      "window_dilation_strides" : [ 1, 1 ],
+      "window_movement_strides" : [ 1, 1 ]
+    },
+    {
+      "inputs" : [ "Convolution_708", "Broadcast_711" ],
+      "name" : "Add_712",
+      "op" : "Add",
+      "outputs" : ["Add_712_0"]
+    },
+    {
+      "inputs" : [ "Add_712", "Broadcast_714" ],
+      "name" : "Maximum_715",
+      "op" : "Maximum",
+      "outputs" : ["Maximum_715_0"]
+    },
+    {
+      "data_dilation_strides" : [ 1, 1 ],
+      "inputs" : [ "Maximum_715", "Parameter_716" ],
+      "name" : "Convolution_718",
+      "op" : "Convolution",
+      "outputs" : ["Convolution_718_0"],
+      "padding_above" : [ 1, 1 ],
+      "padding_below" : [ 1, 1 ],
+      "window_dilation_strides" : [ 1, 1 ],
+      "window_movement_strides" : [ 1, 1 ]
+    },
+    {
+      "inputs" : [ "Convolution_718", "Broadcast_721" ],
+      "name" : "Add_722",
+      "op" : "Add",
+      "outputs" : ["Add_722_0"]
+    },
+    {
+      "inputs" : [ "Add_722", "Broadcast_724" ],
+      "name" : "Maximum_725",
+      "op" : "Maximum",
+      "outputs" : ["Maximum_725_0"]
+    },
+    {
+      "inputs" : ["Maximum_725"],
+      "name" : "MaxPool_726",
+      "op" : "MaxPool",
+      "outputs" : ["MaxPool_726_0"],
+      "padding_above" : [ 0, 0 ],
+      "padding_below" : [ 0, 0 ],
+      "window_movement_strides" : [ 2, 2 ],
+      "window_shape" : [ 2, 2 ]
+    },
+    {
+      "input_order" : [ 0, 1, 2, 3 ],
+      "inputs" : ["MaxPool_726"],
+      "name" : "Reshape_727",
+      "op" : "Reshape",
+      "output_shape" : [ 32, 25088 ],
+      "outputs" : ["Reshape_727_0"]
+    },
+    {
+      "inputs" : [ "Reshape_727", "Reshape_730" ],
+      "name" : "Dot_731",
+      "op" : "Dot",
+      "outputs" : ["Dot_731_0"],
+      "reduction_axes_count" : 1
+    },
+    {
+      "inputs" : [ "Dot_731", "Broadcast_732" ],
+      "name" : "Add_733",
+      "op" : "Add",
+      "outputs" : ["Add_733_0"]
+    },
+    {
+      "inputs" : [ "Add_733", "Broadcast_735" ],
+      "name" : "Maximum_736",
+      "op" : "Maximum",
+      "outputs" : ["Maximum_736_0"]
+    },
+    {
+      "inputs" : [ "Maximum_736", "Reshape_739" ],
+      "name" : "Dot_740",
+      "op" : "Dot",
+      "outputs" : ["Dot_740_0"],
+      "reduction_axes_count" : 1
+    },
+    {
+      "inputs" : [ "Dot_740", "Broadcast_741" ],
+      "name" : "Add_742",
+      "op" : "Add",
+      "outputs" : ["Add_742_0"]
+    },
+    {
+      "inputs" : [ "Add_742", "Broadcast_744" ],
+      "name" : "Maximum_745",
+      "op" : "Maximum",
+      "outputs" : ["Maximum_745_0"]
+    },
+    {
+      "inputs" : [ "Maximum_745", "Reshape_748" ],
+      "name" : "Dot_749",
+      "op" : "Dot",
+      "outputs" : ["Dot_749_0"],
+      "reduction_axes_count" : 1
+    },
+    {
+      "inputs" : [ "Dot_749", "Broadcast_750" ],
+      "name" : "Add_751",
+      "op" : "Add",
+      "outputs" : ["Add_751_0"]
+    }
+  ],
+  "parameters" : [
+    "Parameter_641", "Parameter_642", "Parameter_643", "Parameter_653",
+    "Parameter_654", "Parameter_664", "Parameter_665", "Parameter_674",
+    "Parameter_675", "Parameter_685", "Parameter_686", "Parameter_695",
+    "Parameter_696", "Parameter_706", "Parameter_707", "Parameter_716",
+    "Parameter_717", "Parameter_728", "Parameter_729", "Parameter_737",
+    "Parameter_738", "Parameter_746", "Parameter_747"
+  ],
+  "result" : ["Add_751"]
+}]